From 0fab2639a229c7f56d49739e85f738ddd3bf112e Mon Sep 17 00:00:00 2001 From: Josh Spicer Date: Fri, 29 Jul 2022 13:39:41 +0000 Subject: [PATCH 01/30] refactor features-test command and stub out features-package --- .gitignore | 3 +- .vscode/launch.json | 3 +- src/spec-node/devContainersSpecCLI.ts | 93 ++----------------- src/spec-node/featuresCLI/package.ts | 75 +++++++++++++++ .../featuresCLI/packageCommandImpl.ts | 63 +++++++++++++ src/spec-node/featuresCLI/test.ts | 87 +++++++++++++++++ ...ontainerFeatures.ts => testCommandImpl.ts} | 3 +- 7 files changed, 239 insertions(+), 88 deletions(-) create mode 100644 src/spec-node/featuresCLI/package.ts create mode 100644 src/spec-node/featuresCLI/packageCommandImpl.ts create mode 100644 src/spec-node/featuresCLI/test.ts rename src/spec-node/featuresCLI/{testContainerFeatures.ts => testCommandImpl.ts} (99%) diff --git a/.gitignore b/.gitignore index 523c0843a..13261a7a4 100644 --- a/.gitignore +++ b/.gitignore @@ -4,4 +4,5 @@ logs *.log *.tgz tmp -.DS_Store \ No newline at end of file +.DS_Store +.env \ No newline at end of file diff --git a/.vscode/launch.json b/.vscode/launch.json index a2c0a0a8b..03c36fc0a 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -10,10 +10,11 @@ "name": "Launch CLI - up", "program": "${workspaceFolder}/src/spec-node/devContainersSpecCLI.ts", "cwd": "${workspaceFolder}", + "envFile": "${workspaceFolder}/.env", "args": [ "up", "--workspace-folder", - "../devcontainers-features", + "/home/node/example-project", "--log-level", "debug", ], diff --git a/src/spec-node/devContainersSpecCLI.ts b/src/spec-node/devContainersSpecCLI.ts index 90e26e5f0..ff883f963 100644 --- a/src/spec-node/devContainersSpecCLI.ts +++ b/src/spec-node/devContainersSpecCLI.ts @@ -21,11 +21,11 @@ import { getDockerComposeFilePaths } from '../spec-configuration/configuration'; import { workspaceFromPath } from '../spec-utils/workspaces'; import { readDevContainerConfigFile } from './configContainer'; import { getDefaultDevContainerConfigPath, getDevContainerConfigPathIn, uriToFsPath } from '../spec-configuration/configurationCommonUtils'; -import { CLIHost, getCLIHost } from '../spec-common/cliHost'; +import { getCLIHost } from '../spec-common/cliHost'; import { loadNativeModule } from '../spec-common/commonUtils'; import { generateFeaturesConfig, getContainerFeaturesFolder } from '../spec-configuration/containerFeaturesConfiguration'; -import { doFeaturesTestCommand } from './featuresCLI/testContainerFeatures'; -import { PackageConfiguration } from '../spec-utils/product'; +import { featuresTestOptions, featuresTestHandler } from './featuresCLI/test'; +import { featuresPackageHandler, featuresPackageOptions } from './featuresCLI/package'; const defaultDefaultUserEnvProbe: UserEnvProbe = 'loginInteractiveShell'; @@ -49,14 +49,16 @@ const defaultDefaultUserEnvProbe: UserEnvProbe = 'loginInteractiveShell'; y.command('build [path]', 'Build a dev container image', buildOptions, buildHandler); y.command('run-user-commands', 'Run user commands', runUserCommandsOptions, runUserCommandsHandler); y.command('read-configuration', 'Read configuration', readConfigurationOptions, readConfigurationHandler); - y.command('features', 'Features commands', (y: Argv) => - y.command('test', 'Test features', featuresTestOptions, featuresTestHandler)); + y.command('features', 'Features commands', (y: Argv) => { + y.command('test', 'Test features', featuresTestOptions, featuresTestHandler); + y.command('package', 'Package features', featuresPackageOptions, featuresPackageHandler); + }); y.command(restArgs ? ['exec', '*'] : ['exec [args..]'], 'Execute a command on a running dev container', execOptions, execHandler); y.parse(restArgs ? argv.slice(1) : argv); })().catch(console.error); -type UnpackArgv = T extends Argv ? U : T; +export type UnpackArgv = T extends Argv ? U : T; const mountRegex = /^type=(bind|volume),source=([^,]+),target=([^,]+)(?:,external=(true|false))?$/; @@ -832,85 +834,6 @@ export async function doExec({ } } -// -- 'features test' command -function featuresTestOptions(y: Argv) { - return y - .options({ - 'base-image': { type: 'string', alias: 'i', default: 'ubuntu:focal', description: 'Base Image' }, - 'features': { type: 'array', alias: 'f', describe: 'Feature(s) to test as space-separated parameters. Omit to auto-detect all features in collection directory. Cannot be combined with \'-s\'.', }, - 'scenarios': { type: 'string', alias: 's', description: 'Path to scenario test directory (containing scenarios.json). Cannot be combined with \'-f\'.' }, - 'remote-user': { type: 'string', alias: 'u', default: 'root', describe: 'Remote user', }, - 'collection-folder': { type: 'string', alias: 'c', default: '.', description: 'Path to folder containing \'src\' and \'test\' sub-folders.' }, - 'log-level': { choices: ['info' as 'info', 'debug' as 'debug', 'trace' as 'trace'], default: 'info' as 'info', description: 'Log level.' }, - 'quiet': { type: 'boolean', alias: 'q', default: false, description: 'Quiets output' }, - }) - .check(argv => { - if (argv['scenarios'] && argv['features']) { - throw new Error('Cannot combine --scenarios and --features'); - } - return true; - }); -} - -export type FeaturesTestArgs = UnpackArgv>; -export interface FeaturesTestCommandInput { - cliHost: CLIHost; - pkg: PackageConfiguration; - baseImage: string; - collectionFolder: string; - features?: string[]; - scenariosFolder: string | undefined; - remoteUser: string; - quiet: boolean; - logLevel: LogLevel; - disposables: (() => Promise | undefined)[]; -} - -function featuresTestHandler(args: FeaturesTestArgs) { - (async () => await featuresTest(args))().catch(console.error); -} - -async function featuresTest({ - 'base-image': baseImage, - 'collection-folder': collectionFolder, - features, - scenarios: scenariosFolder, - 'remote-user': remoteUser, - quiet, - 'log-level': inputLogLevel, -}: FeaturesTestArgs) { - const disposables: (() => Promise | undefined)[] = []; - const dispose = async () => { - await Promise.all(disposables.map(d => d())); - }; - - const cwd = process.cwd(); - const cliHost = await getCLIHost(cwd, loadNativeModule); - const extensionPath = path.join(__dirname, '..', '..'); - const pkg = await getPackageConfig(extensionPath); - - const logLevel = mapLogLevel(inputLogLevel); - - const args: FeaturesTestCommandInput = { - baseImage, - cliHost, - logLevel, - quiet, - pkg, - collectionFolder: cliHost.path.resolve(collectionFolder), - features: features ? (Array.isArray(features) ? features as string[] : [features]) : undefined, - scenariosFolder: scenariosFolder ? cliHost.path.resolve(scenariosFolder) : undefined, - remoteUser, - disposables - }; - - const exitCode = await doFeaturesTestCommand(args); - - await dispose(); - process.exit(exitCode); -} -// -- End: 'features test' command - function keyValuesToRecord(keyValues: string[]): Record { return keyValues.reduce((envs, env) => { const i = env.indexOf('='); diff --git a/src/spec-node/featuresCLI/package.ts b/src/spec-node/featuresCLI/package.ts new file mode 100644 index 000000000..679ac3447 --- /dev/null +++ b/src/spec-node/featuresCLI/package.ts @@ -0,0 +1,75 @@ +import * as path from 'path'; +import { Argv } from 'yargs'; +import { CLIHost, getCLIHost } from '../../spec-common/cliHost'; +import { loadNativeModule } from '../../spec-common/commonUtils'; +import { LogLevel, mapLogLevel } from '../../spec-utils/log'; +import { getPackageConfig, PackageConfiguration } from '../../spec-utils/product'; +import { UnpackArgv } from '../devContainersSpecCLI'; +import { doFeaturesPackageCommand } from './packageCommandImpl'; + +export function featuresPackageOptions(y: Argv) { + return y + .options({ + // 'features': { type: 'array', alias: 'f', describe: 'Feature(s) to test as space-separated parameters. Omit to auto-detect all features in collection directory. Cannot be combined with \'-s\'.', }, + 'collection-folder': { type: 'string', alias: 'c', default: '.', description: 'Path to folder containing \'src\' and \'test\' sub-folders.' }, + 'output-dir': { type: 'string', alias: 'o', default: 'dist', description: 'Path to write packages to' }, + 'log-level': { choices: ['info' as 'info', 'debug' as 'debug', 'trace' as 'trace'], default: 'info' as 'info', description: 'Log level.' }, + 'quiet': { type: 'boolean', alias: 'q', default: false, description: 'Quiets output' }, + }) + .check(_argv => { + // if (argv['scenarios'] && argv['features']) { + // throw new Error('Cannot combine --scenarios and --features'); + // } + return true; + }); +} + +export type FeaturesPackageArgs = UnpackArgv>; +export interface FeaturesPackageCommandInput { + cliHost: CLIHost; + pkg: PackageConfiguration; + // features?: string[]; + collectionFolder: string; + quiet: boolean; + logLevel: LogLevel; + outputDir: string; + disposables: (() => Promise | undefined)[]; +} + +export function featuresPackageHandler(args: FeaturesPackageArgs) { + (async () => await featuresPackage(args))().catch(console.error); +} + +async function featuresPackage({ + 'collection-folder': collectionFolder, + quiet, + 'log-level': inputLogLevel, + 'output-dir': outputDir, +}: FeaturesPackageArgs) { + const disposables: (() => Promise | undefined)[] = []; + const dispose = async () => { + await Promise.all(disposables.map(d => d())); + }; + + const cwd = process.cwd(); + const cliHost = await getCLIHost(cwd, loadNativeModule); + const extensionPath = path.join(__dirname, '..', '..'); + const pkg = await getPackageConfig(extensionPath); + + const logLevel = mapLogLevel(inputLogLevel); + + const args: FeaturesPackageCommandInput = { + cliHost, + logLevel, + quiet, + pkg, + outputDir: path.resolve(outputDir), + collectionFolder: cliHost.path.resolve(collectionFolder), + disposables + }; + + const exitCode = await doFeaturesPackageCommand(args); + + await dispose(); + process.exit(exitCode); +} \ No newline at end of file diff --git a/src/spec-node/featuresCLI/packageCommandImpl.ts b/src/spec-node/featuresCLI/packageCommandImpl.ts new file mode 100644 index 000000000..6590cb50e --- /dev/null +++ b/src/spec-node/featuresCLI/packageCommandImpl.ts @@ -0,0 +1,63 @@ +import path from 'path'; +import { Feature } from '../../spec-configuration/containerFeaturesConfiguration'; +import { isLocalFile, readLocalDir, readLocalFile, writeLocalFile } from '../../spec-utils/pfs'; +import { FeaturesPackageCommandInput } from './package'; + +export async function doFeaturesPackageCommand(args: FeaturesPackageCommandInput): Promise { + + // For each feature, package each feature and write to 'outputDir/{f}.tgz' + // Returns an array of feature metadata from each processed feature + const metadataOutput = await getFeaturesAndPackage(args.collectionFolder, args.outputDir); + + if (!metadataOutput) { + // ERR + return 1; + } + + // Write the metadata to a file + const metadataOutputPath = path.join(args.outputDir, 'devcontainer-collection.json'); + await writeLocalFile(metadataOutputPath, JSON.stringify(metadataOutput, null, 4)); + + + return 0; +} + +async function tarDirectory(_featureFolder: string, _archiveName: string, _outputDir: string) { + +} + +export async function getFeaturesAndPackage(basePath: string, outputDir: string): Promise { + // const { output } = args; + const featureDirs = await readLocalDir(basePath); + let metadatas: Feature[] = []; + + await Promise.all( + featureDirs.map(async (f: string) => { + // output.write(`Processing feature: ${f}`); + if (!f.startsWith('.')) { + const featureFolder = path.join(basePath, f); + const archiveName = `${f}.tgz`; + + await tarDirectory(featureFolder, archiveName, outputDir); + + const featureJsonPath = path.join(featureFolder, 'devcontainer-feature.json'); + + if (!isLocalFile(featureJsonPath)) { + // core.error(`Feature '${f}' is missing a devcontainer-feature.json`); + // core.setFailed('All features must have a devcontainer-feature.json'); + return; + } + + const featureMetadata: Feature = JSON.parse(await readLocalFile(featureJsonPath, 'utf-8')); + metadatas.push(featureMetadata); + } + }) + ); + + if (metadatas.length === 0) { + // core.setFailed('No features found'); + return; + } + + return metadatas; +} \ No newline at end of file diff --git a/src/spec-node/featuresCLI/test.ts b/src/spec-node/featuresCLI/test.ts new file mode 100644 index 000000000..5c0623869 --- /dev/null +++ b/src/spec-node/featuresCLI/test.ts @@ -0,0 +1,87 @@ +import * as path from 'path'; +import { Argv } from 'yargs'; +import { CLIHost, getCLIHost } from '../../spec-common/cliHost'; +import { loadNativeModule } from '../../spec-common/commonUtils'; +import { LogLevel, mapLogLevel } from '../../spec-utils/log'; +import { getPackageConfig, PackageConfiguration } from '../../spec-utils/product'; +import { UnpackArgv } from '../devContainersSpecCLI'; +import { doFeaturesTestCommand } from './testCommandImpl'; + +// -- 'features test' command +export function featuresTestOptions(y: Argv) { + return y + .options({ + 'base-image': { type: 'string', alias: 'i', default: 'ubuntu:focal', description: 'Base Image' }, + 'features': { type: 'array', alias: 'f', describe: 'Feature(s) to test as space-separated parameters. Omit to auto-detect all features in collection directory. Cannot be combined with \'-s\'.', }, + 'scenarios': { type: 'string', alias: 's', description: 'Path to scenario test directory (containing scenarios.json). Cannot be combined with \'-f\'.' }, + 'remote-user': { type: 'string', alias: 'u', default: 'root', describe: 'Remote user', }, + 'collection-folder': { type: 'string', alias: 'c', default: '.', description: 'Path to folder containing \'src\' and \'test\' sub-folders.' }, + 'log-level': { choices: ['info' as 'info', 'debug' as 'debug', 'trace' as 'trace'], default: 'info' as 'info', description: 'Log level.' }, + 'quiet': { type: 'boolean', alias: 'q', default: false, description: 'Quiets output' }, + }) + .check(argv => { + if (argv['scenarios'] && argv['features']) { + throw new Error('Cannot combine --scenarios and --features'); + } + return true; + }); +} + +export type FeaturesTestArgs = UnpackArgv>; +export interface FeaturesTestCommandInput { + cliHost: CLIHost; + pkg: PackageConfiguration; + baseImage: string; + collectionFolder: string; + features?: string[]; + scenariosFolder: string | undefined; + remoteUser: string; + quiet: boolean; + logLevel: LogLevel; + disposables: (() => Promise | undefined)[]; +} + +export function featuresTestHandler(args: FeaturesTestArgs) { + (async () => await featuresTest(args))().catch(console.error); +} + +async function featuresTest({ + 'base-image': baseImage, + 'collection-folder': collectionFolder, + features, + scenarios: scenariosFolder, + 'remote-user': remoteUser, + quiet, + 'log-level': inputLogLevel, +}: FeaturesTestArgs) { + const disposables: (() => Promise | undefined)[] = []; + const dispose = async () => { + await Promise.all(disposables.map(d => d())); + }; + + const cwd = process.cwd(); + const cliHost = await getCLIHost(cwd, loadNativeModule); + const extensionPath = path.join(__dirname, '..', '..'); + const pkg = await getPackageConfig(extensionPath); + + const logLevel = mapLogLevel(inputLogLevel); + + const args: FeaturesTestCommandInput = { + baseImage, + cliHost, + logLevel, + quiet, + pkg, + collectionFolder: cliHost.path.resolve(collectionFolder), + features: features ? (Array.isArray(features) ? features as string[] : [features]) : undefined, + scenariosFolder: scenariosFolder ? cliHost.path.resolve(scenariosFolder) : undefined, + remoteUser, + disposables + }; + + const exitCode = await doFeaturesTestCommand(args); + + await dispose(); + process.exit(exitCode); +} +// -- End: 'features test' command \ No newline at end of file diff --git a/src/spec-node/featuresCLI/testContainerFeatures.ts b/src/spec-node/featuresCLI/testCommandImpl.ts similarity index 99% rename from src/spec-node/featuresCLI/testContainerFeatures.ts rename to src/spec-node/featuresCLI/testCommandImpl.ts index 4999049e6..68b9e6fa4 100644 --- a/src/spec-node/featuresCLI/testContainerFeatures.ts +++ b/src/spec-node/featuresCLI/testCommandImpl.ts @@ -3,10 +3,11 @@ import chalk from 'chalk'; import { tmpdir } from 'os'; import { CLIHost } from '../../spec-common/cliHost'; import { launch, ProvisionOptions, createDockerParams } from '../devContainers'; -import { doExec, FeaturesTestCommandInput } from '../devContainersSpecCLI'; +import { doExec } from '../devContainersSpecCLI'; import { LaunchResult, staticExecParams, staticProvisionParams, testLibraryScript } from './utils'; import { DockerResolverParameters } from '../utils'; import { DevContainerConfig } from '../../spec-configuration/configuration'; +import { FeaturesTestCommandInput } from './test'; const TEST_LIBRARY_SCRIPT_NAME = 'dev-container-features-test-lib'; From f4783596b0e4a38fe2080a21d31ded87a0b361cc Mon Sep 17 00:00:00 2001 From: Josh Spicer Date: Fri, 29 Jul 2022 16:31:04 +0000 Subject: [PATCH 02/30] packaging complete --- src/spec-node/featuresCLI/package.ts | 53 +++++++++++------- .../featuresCLI/packageCommandImpl.ts | 54 +++++++++---------- 2 files changed, 61 insertions(+), 46 deletions(-) diff --git a/src/spec-node/featuresCLI/package.ts b/src/spec-node/featuresCLI/package.ts index 679ac3447..35c1fdfde 100644 --- a/src/spec-node/featuresCLI/package.ts +++ b/src/spec-node/featuresCLI/package.ts @@ -1,18 +1,20 @@ -import * as path from 'path'; +import path from 'path'; import { Argv } from 'yargs'; import { CLIHost, getCLIHost } from '../../spec-common/cliHost'; import { loadNativeModule } from '../../spec-common/commonUtils'; -import { LogLevel, mapLogLevel } from '../../spec-utils/log'; -import { getPackageConfig, PackageConfiguration } from '../../spec-utils/product'; +import { Log, LogLevel, mapLogLevel } from '../../spec-utils/log'; +import { isLocalFolder, mkdirpLocal } from '../../spec-utils/pfs'; +import { createLog } from '../devContainers'; import { UnpackArgv } from '../devContainersSpecCLI'; +import { getPackageConfig } from '../utils'; import { doFeaturesPackageCommand } from './packageCommandImpl'; export function featuresPackageOptions(y: Argv) { return y .options({ // 'features': { type: 'array', alias: 'f', describe: 'Feature(s) to test as space-separated parameters. Omit to auto-detect all features in collection directory. Cannot be combined with \'-s\'.', }, - 'collection-folder': { type: 'string', alias: 'c', default: '.', description: 'Path to folder containing \'src\' and \'test\' sub-folders.' }, - 'output-dir': { type: 'string', alias: 'o', default: 'dist', description: 'Path to write packages to' }, + 'features-folder': { type: 'string', alias: 'c', default: '.', description: 'Path to folder containing features source code' }, + 'output-dir': { type: 'string', alias: 'o', default: '/tmp/build', description: 'Path to output' }, 'log-level': { choices: ['info' as 'info', 'debug' as 'debug', 'trace' as 'trace'], default: 'info' as 'info', description: 'Log level.' }, 'quiet': { type: 'boolean', alias: 'q', default: false, description: 'Quiets output' }, }) @@ -27,11 +29,9 @@ export function featuresPackageOptions(y: Argv) { export type FeaturesPackageArgs = UnpackArgv>; export interface FeaturesPackageCommandInput { cliHost: CLIHost; - pkg: PackageConfiguration; // features?: string[]; - collectionFolder: string; - quiet: boolean; - logLevel: LogLevel; + featuresFolder: string; + output: Log; outputDir: string; disposables: (() => Promise | undefined)[]; } @@ -41,8 +41,7 @@ export function featuresPackageHandler(args: FeaturesPackageArgs) { } async function featuresPackage({ - 'collection-folder': collectionFolder, - quiet, + 'features-folder': featuresFolder, 'log-level': inputLogLevel, 'output-dir': outputDir, }: FeaturesPackageArgs) { @@ -51,20 +50,36 @@ async function featuresPackage({ await Promise.all(disposables.map(d => d())); }; + const extensionPath = path.join(__dirname, '..', '..', '..'); + const pkg = await getPackageConfig(extensionPath); + const cwd = process.cwd(); const cliHost = await getCLIHost(cwd, loadNativeModule); - const extensionPath = path.join(__dirname, '..', '..'); - const pkg = await getPackageConfig(extensionPath); + const output = createLog({ + logLevel: mapLogLevel(inputLogLevel), + logFormat: 'text', + log: (str) => process.stdout.write(str), + terminalDimensions: undefined, + }, pkg, new Date(), disposables); + + output.write('Packaging features...\n', LogLevel.Info); + + const featuresDirResolved = cliHost.path.resolve(featuresFolder); + if (!(await isLocalFolder(featuresDirResolved))) { + throw new Error(`Features folder '${featuresDirResolved}' does not exist`); + } - const logLevel = mapLogLevel(inputLogLevel); + const outputDirResolved = cliHost.path.resolve(outputDir); + if (!(await isLocalFolder(outputDirResolved))) { + // TODO: Delete folder first? + await mkdirpLocal(outputDirResolved); + } const args: FeaturesPackageCommandInput = { cliHost, - logLevel, - quiet, - pkg, - outputDir: path.resolve(outputDir), - collectionFolder: cliHost.path.resolve(collectionFolder), + featuresFolder: featuresDirResolved, + outputDir: outputDirResolved, + output, disposables }; diff --git a/src/spec-node/featuresCLI/packageCommandImpl.ts b/src/spec-node/featuresCLI/packageCommandImpl.ts index 6590cb50e..3d6c56624 100644 --- a/src/spec-node/featuresCLI/packageCommandImpl.ts +++ b/src/spec-node/featuresCLI/packageCommandImpl.ts @@ -1,16 +1,19 @@ import path from 'path'; +import tar from 'tar'; import { Feature } from '../../spec-configuration/containerFeaturesConfiguration'; +import { LogLevel } from '../../spec-utils/log'; import { isLocalFile, readLocalDir, readLocalFile, writeLocalFile } from '../../spec-utils/pfs'; import { FeaturesPackageCommandInput } from './package'; export async function doFeaturesPackageCommand(args: FeaturesPackageCommandInput): Promise { + const { output } = args; // For each feature, package each feature and write to 'outputDir/{f}.tgz' // Returns an array of feature metadata from each processed feature - const metadataOutput = await getFeaturesAndPackage(args.collectionFolder, args.outputDir); + const metadataOutput = await getFeaturesAndPackage(args); if (!metadataOutput) { - // ERR + output.write('Failed to package features', LogLevel.Error); return 1; } @@ -18,44 +21,41 @@ export async function doFeaturesPackageCommand(args: FeaturesPackageCommandInput const metadataOutputPath = path.join(args.outputDir, 'devcontainer-collection.json'); await writeLocalFile(metadataOutputPath, JSON.stringify(metadataOutput, null, 4)); - return 0; } -async function tarDirectory(_featureFolder: string, _archiveName: string, _outputDir: string) { - +async function tarDirectory(featureFolder: string, archiveName: string, outputDir: string) { + return new Promise((resolve) => resolve(tar.create({ file: path.join(outputDir, archiveName), cwd: featureFolder }, ['.']))); } -export async function getFeaturesAndPackage(basePath: string, outputDir: string): Promise { - // const { output } = args; - const featureDirs = await readLocalDir(basePath); +export async function getFeaturesAndPackage(args: FeaturesPackageCommandInput): Promise { + const { output, featuresFolder, outputDir } = args; + const featureDirs = await readLocalDir(featuresFolder); let metadatas: Feature[] = []; - await Promise.all( - featureDirs.map(async (f: string) => { - // output.write(`Processing feature: ${f}`); - if (!f.startsWith('.')) { - const featureFolder = path.join(basePath, f); - const archiveName = `${f}.tgz`; + for await (const f of featureDirs) { + output.write(`Processing feature: ${f}...`, LogLevel.Info); + if (!f.startsWith('.')) { + const featureFolder = path.join(featuresFolder, f); + const archiveName = `${f}.tgz`; - await tarDirectory(featureFolder, archiveName, outputDir); + await tarDirectory(featureFolder, archiveName, outputDir); - const featureJsonPath = path.join(featureFolder, 'devcontainer-feature.json'); + const featureJsonPath = path.join(featureFolder, 'devcontainer-feature.json'); + if (!isLocalFile(featureJsonPath)) { + output.write(`Feature '${f}' is missing a devcontainer-feature.json`, LogLevel.Error); + return; + } - if (!isLocalFile(featureJsonPath)) { - // core.error(`Feature '${f}' is missing a devcontainer-feature.json`); - // core.setFailed('All features must have a devcontainer-feature.json'); - return; - } + const featureMetadata: Feature = JSON.parse(await readLocalFile(featureJsonPath, 'utf-8')); + metadatas.push(featureMetadata); + } + } - const featureMetadata: Feature = JSON.parse(await readLocalFile(featureJsonPath, 'utf-8')); - metadatas.push(featureMetadata); - } - }) - ); + output.write(metadatas.length.toString(), LogLevel.Info); if (metadatas.length === 0) { - // core.setFailed('No features found'); + output.write('Failed to generate metadata file.', LogLevel.Error); return; } From 180f32c90d948c65ee973ac93604ef3326803af2 Mon Sep 17 00:00:00 2001 From: Josh Spicer Date: Fri, 29 Jul 2022 16:33:12 +0000 Subject: [PATCH 03/30] spacing --- src/spec-node/featuresCLI/packageCommandImpl.ts | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/spec-node/featuresCLI/packageCommandImpl.ts b/src/spec-node/featuresCLI/packageCommandImpl.ts index 3d6c56624..836af9a99 100644 --- a/src/spec-node/featuresCLI/packageCommandImpl.ts +++ b/src/spec-node/featuresCLI/packageCommandImpl.ts @@ -52,12 +52,10 @@ export async function getFeaturesAndPackage(args: FeaturesPackageCommandInput): } } - output.write(metadatas.length.toString(), LogLevel.Info); - if (metadatas.length === 0) { - output.write('Failed to generate metadata file.', LogLevel.Error); return; } + output.write(`Packaged ${metadatas.length.toString()} features!`, LogLevel.Info); return metadatas; } \ No newline at end of file From e855261a5ef167bbf59684ff29da0e37e3269167 Mon Sep 17 00:00:00 2001 From: Josh Spicer Date: Fri, 29 Jul 2022 17:35:46 +0000 Subject: [PATCH 04/30] follow spec --- .../featuresCLI/packageCommandImpl.ts | 27 +++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/src/spec-node/featuresCLI/packageCommandImpl.ts b/src/spec-node/featuresCLI/packageCommandImpl.ts index 836af9a99..7e6612c9b 100644 --- a/src/spec-node/featuresCLI/packageCommandImpl.ts +++ b/src/spec-node/featuresCLI/packageCommandImpl.ts @@ -5,6 +5,22 @@ import { LogLevel } from '../../spec-utils/log'; import { isLocalFile, readLocalDir, readLocalFile, writeLocalFile } from '../../spec-utils/pfs'; import { FeaturesPackageCommandInput } from './package'; + +export interface SourceInformation { + source: string; + owner?: string; + repo?: string; + tag?: string; + ref?: string; + sha?: string; +} + +export interface DevContainerCollectionMetadata { + sourceInformation: SourceInformation; + features: Feature[]; +} + + export async function doFeaturesPackageCommand(args: FeaturesPackageCommandInput): Promise { const { output } = args; @@ -17,9 +33,16 @@ export async function doFeaturesPackageCommand(args: FeaturesPackageCommandInput return 1; } + const collection: DevContainerCollectionMetadata = { + sourceInformation: { + source: 'devcontainer-cli', + }, + features: metadataOutput, + }; + // Write the metadata to a file const metadataOutputPath = path.join(args.outputDir, 'devcontainer-collection.json'); - await writeLocalFile(metadataOutputPath, JSON.stringify(metadataOutput, null, 4)); + await writeLocalFile(metadataOutputPath, JSON.stringify(collection, null, 4)); return 0; } @@ -37,7 +60,7 @@ export async function getFeaturesAndPackage(args: FeaturesPackageCommandInput): output.write(`Processing feature: ${f}...`, LogLevel.Info); if (!f.startsWith('.')) { const featureFolder = path.join(featuresFolder, f); - const archiveName = `${f}.tgz`; + const archiveName = `devcontainer-feature-${f}.tgz`; await tarDirectory(featureFolder, archiveName, outputDir); From e58d6fb177151998486a754e83184f87275e1b29 Mon Sep 17 00:00:00 2001 From: Josh Spicer Date: Fri, 29 Jul 2022 17:43:17 +0000 Subject: [PATCH 05/30] comment out envFile in launch config --- .vscode/launch.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.vscode/launch.json b/.vscode/launch.json index 03c36fc0a..cbf112f7e 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -10,7 +10,7 @@ "name": "Launch CLI - up", "program": "${workspaceFolder}/src/spec-node/devContainersSpecCLI.ts", "cwd": "${workspaceFolder}", - "envFile": "${workspaceFolder}/.env", + // "envFile": "${workspaceFolder}/.env", // Create .env file in the workspace folder to pass environment variables to the CLI. "args": [ "up", "--workspace-folder", From 5ce68f40b125984499d6528414f560ed8abce49f Mon Sep 17 00:00:00 2001 From: Josh Spicer Date: Mon, 1 Aug 2022 15:48:46 +0000 Subject: [PATCH 06/30] Code Review (part 1) - default output to ./output and guard against overwritting - default example project in vscode launch config --- .gitignore | 3 +- .vscode/launch.json | 2 +- src/spec-node/featuresCLI/package.ts | 41 +++++++++++-------- .../featuresCLI/packageCommandImpl.ts | 2 +- src/test/configs/example/.devcontainer.json | 10 +++++ 5 files changed, 37 insertions(+), 21 deletions(-) create mode 100644 src/test/configs/example/.devcontainer.json diff --git a/.gitignore b/.gitignore index 13261a7a4..105572779 100644 --- a/.gitignore +++ b/.gitignore @@ -5,4 +5,5 @@ logs *.tgz tmp .DS_Store -.env \ No newline at end of file +.env +output \ No newline at end of file diff --git a/.vscode/launch.json b/.vscode/launch.json index cbf112f7e..e87675c38 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -14,7 +14,7 @@ "args": [ "up", "--workspace-folder", - "/home/node/example-project", + "${workspaceFolder}/src/test/configs/example", // Edit this example config to test the CLI against a custom configuration. "--log-level", "debug", ], diff --git a/src/spec-node/featuresCLI/package.ts b/src/spec-node/featuresCLI/package.ts index 35c1fdfde..e214aaf37 100644 --- a/src/spec-node/featuresCLI/package.ts +++ b/src/spec-node/featuresCLI/package.ts @@ -3,7 +3,7 @@ import { Argv } from 'yargs'; import { CLIHost, getCLIHost } from '../../spec-common/cliHost'; import { loadNativeModule } from '../../spec-common/commonUtils'; import { Log, LogLevel, mapLogLevel } from '../../spec-utils/log'; -import { isLocalFolder, mkdirpLocal } from '../../spec-utils/pfs'; +import { isLocalFolder, mkdirpLocal, rmLocal } from '../../spec-utils/pfs'; import { createLog } from '../devContainers'; import { UnpackArgv } from '../devContainersSpecCLI'; import { getPackageConfig } from '../utils'; @@ -12,16 +12,12 @@ import { doFeaturesPackageCommand } from './packageCommandImpl'; export function featuresPackageOptions(y: Argv) { return y .options({ - // 'features': { type: 'array', alias: 'f', describe: 'Feature(s) to test as space-separated parameters. Omit to auto-detect all features in collection directory. Cannot be combined with \'-s\'.', }, - 'features-folder': { type: 'string', alias: 'c', default: '.', description: 'Path to folder containing features source code' }, - 'output-dir': { type: 'string', alias: 'o', default: '/tmp/build', description: 'Path to output' }, + 'feature-collection-folder': { type: 'string', alias: 'c', default: '.', description: 'Path to folder containing source code for collection of features' }, + 'output-dir': { type: 'string', alias: 'o', default: './output', description: 'Path to output directory. Will create directories as needed.' }, + 'force-clean-output-dir': { type: 'boolean', alias: 'f', default: false, description: 'Automatically delete previous output directory before packaging' }, 'log-level': { choices: ['info' as 'info', 'debug' as 'debug', 'trace' as 'trace'], default: 'info' as 'info', description: 'Log level.' }, - 'quiet': { type: 'boolean', alias: 'q', default: false, description: 'Quiets output' }, }) .check(_argv => { - // if (argv['scenarios'] && argv['features']) { - // throw new Error('Cannot combine --scenarios and --features'); - // } return true; }); } @@ -29,10 +25,9 @@ export function featuresPackageOptions(y: Argv) { export type FeaturesPackageArgs = UnpackArgv>; export interface FeaturesPackageCommandInput { cliHost: CLIHost; - // features?: string[]; - featuresFolder: string; - output: Log; + srcFolder: string; outputDir: string; + output: Log; disposables: (() => Promise | undefined)[]; } @@ -41,9 +36,10 @@ export function featuresPackageHandler(args: FeaturesPackageArgs) { } async function featuresPackage({ - 'features-folder': featuresFolder, + 'feature-collection-folder': featureCollectionFolder, 'log-level': inputLogLevel, 'output-dir': outputDir, + 'force-clean-output-dir': forceCleanOutputDir, }: FeaturesPackageArgs) { const disposables: (() => Promise | undefined)[] = []; const dispose = async () => { @@ -62,22 +58,31 @@ async function featuresPackage({ terminalDimensions: undefined, }, pkg, new Date(), disposables); - output.write('Packaging features...\n', LogLevel.Info); + output.write('Packaging features...', LogLevel.Info); - const featuresDirResolved = cliHost.path.resolve(featuresFolder); + const featuresDirResolved = cliHost.path.resolve(featureCollectionFolder); if (!(await isLocalFolder(featuresDirResolved))) { throw new Error(`Features folder '${featuresDirResolved}' does not exist`); } const outputDirResolved = cliHost.path.resolve(outputDir); - if (!(await isLocalFolder(outputDirResolved))) { - // TODO: Delete folder first? - await mkdirpLocal(outputDirResolved); + if (await isLocalFolder(outputDirResolved)) { + // Output dir exists. Delete it automatically if '-f' is true + if (forceCleanOutputDir) { + await rmLocal(outputDirResolved, { recursive: true, force: true }); + } + else { + output.write(`Output directory '${outputDirResolved}' already exists. Manually delete, or pass '-f' to continue.`, LogLevel.Warning); + process.exit(1); + } } + // Generate output folder. + await mkdirpLocal(outputDirResolved); + const args: FeaturesPackageCommandInput = { cliHost, - featuresFolder: featuresDirResolved, + srcFolder: featuresDirResolved, outputDir: outputDirResolved, output, disposables diff --git a/src/spec-node/featuresCLI/packageCommandImpl.ts b/src/spec-node/featuresCLI/packageCommandImpl.ts index 7e6612c9b..52b2b8aac 100644 --- a/src/spec-node/featuresCLI/packageCommandImpl.ts +++ b/src/spec-node/featuresCLI/packageCommandImpl.ts @@ -52,7 +52,7 @@ async function tarDirectory(featureFolder: string, archiveName: string, outputDi } export async function getFeaturesAndPackage(args: FeaturesPackageCommandInput): Promise { - const { output, featuresFolder, outputDir } = args; + const { output, srcFolder: featuresFolder, outputDir } = args; const featureDirs = await readLocalDir(featuresFolder); let metadatas: Feature[] = []; diff --git a/src/test/configs/example/.devcontainer.json b/src/test/configs/example/.devcontainer.json new file mode 100644 index 000000000..a7f75ce2c --- /dev/null +++ b/src/test/configs/example/.devcontainer.json @@ -0,0 +1,10 @@ +// Example devcontainer.json configuration, +// wired into the vscode launch task (.vscode/launch.json) +{ + "image": "mcr.microsoft.com/devcontainers/base:latest", + "features": { + "go": { + "verson": "latest" + } + } +} \ No newline at end of file From 2bf85cbdfcf3fee57e12874dc20b3bd3c633fb54 Mon Sep 17 00:00:00 2001 From: Josh Spicer Date: Mon, 1 Aug 2022 16:03:45 +0000 Subject: [PATCH 07/30] code review part 2 --- src/spec-node/featuresCLI/packageCommandImpl.ts | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/spec-node/featuresCLI/packageCommandImpl.ts b/src/spec-node/featuresCLI/packageCommandImpl.ts index 52b2b8aac..a17534905 100644 --- a/src/spec-node/featuresCLI/packageCommandImpl.ts +++ b/src/spec-node/featuresCLI/packageCommandImpl.ts @@ -5,7 +5,6 @@ import { LogLevel } from '../../spec-utils/log'; import { isLocalFile, readLocalDir, readLocalFile, writeLocalFile } from '../../spec-utils/pfs'; import { FeaturesPackageCommandInput } from './package'; - export interface SourceInformation { source: string; owner?: string; @@ -14,13 +13,11 @@ export interface SourceInformation { ref?: string; sha?: string; } - export interface DevContainerCollectionMetadata { sourceInformation: SourceInformation; features: Feature[]; } - export async function doFeaturesPackageCommand(args: FeaturesPackageCommandInput): Promise { const { output } = args; From eb8c767150d81ec050372c6626a35dad360f7ef3 Mon Sep 17 00:00:00 2001 From: Josh Spicer Date: Mon, 1 Aug 2022 16:06:06 +0000 Subject: [PATCH 08/30] validate before tar directory --- src/spec-node/featuresCLI/packageCommandImpl.ts | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/spec-node/featuresCLI/packageCommandImpl.ts b/src/spec-node/featuresCLI/packageCommandImpl.ts index a17534905..7cc941563 100644 --- a/src/spec-node/featuresCLI/packageCommandImpl.ts +++ b/src/spec-node/featuresCLI/packageCommandImpl.ts @@ -59,13 +59,19 @@ export async function getFeaturesAndPackage(args: FeaturesPackageCommandInput): const featureFolder = path.join(featuresFolder, f); const archiveName = `devcontainer-feature-${f}.tgz`; - await tarDirectory(featureFolder, archiveName, outputDir); - + // Validate minimal feature folder structure const featureJsonPath = path.join(featureFolder, 'devcontainer-feature.json'); + const installShPath = path.join(featureFolder, 'install.sh'); if (!isLocalFile(featureJsonPath)) { output.write(`Feature '${f}' is missing a devcontainer-feature.json`, LogLevel.Error); return; } + if (!isLocalFile(installShPath)) { + output.write(`Feature '${f}' is missing an install.sh`, LogLevel.Error); + return; + } + + await tarDirectory(featureFolder, archiveName, outputDir); const featureMetadata: Feature = JSON.parse(await readLocalFile(featureJsonPath, 'utf-8')); metadatas.push(featureMetadata); From dcb0953c2e9af4def473da577e997ae0a9c98581 Mon Sep 17 00:00:00 2001 From: Josh Spicer Date: Mon, 1 Aug 2022 18:57:24 +0000 Subject: [PATCH 09/30] add packaging test --- .gitignore | 3 +- .../src/featureA/devcontainer-feature.json | 15 +++ .../src/featureA/install.sh | 3 + .../src/featureB/devcontainer-feature.json | 15 +++ .../src/featureB/install.sh | 3 + .../src/featureC/devcontainer-feature.json | 15 +++ .../src/featureC/install.sh | 3 + .../src/featureC/other-file.md | 1 + .../featuresSubCommands.test.ts | 99 +++++++++++++++++++ 9 files changed, 156 insertions(+), 1 deletion(-) create mode 100644 src/test/featuresCLICommands/example-source-repo/src/featureA/devcontainer-feature.json create mode 100644 src/test/featuresCLICommands/example-source-repo/src/featureA/install.sh create mode 100644 src/test/featuresCLICommands/example-source-repo/src/featureB/devcontainer-feature.json create mode 100644 src/test/featuresCLICommands/example-source-repo/src/featureB/install.sh create mode 100644 src/test/featuresCLICommands/example-source-repo/src/featureC/devcontainer-feature.json create mode 100644 src/test/featuresCLICommands/example-source-repo/src/featureC/install.sh create mode 100644 src/test/featuresCLICommands/example-source-repo/src/featureC/other-file.md create mode 100644 src/test/featuresCLICommands/featuresSubCommands.test.ts diff --git a/.gitignore b/.gitignore index 105572779..3e7fa80bd 100644 --- a/.gitignore +++ b/.gitignore @@ -6,4 +6,5 @@ logs tmp .DS_Store .env -output \ No newline at end of file +output +src/test/featuresCLICommands/example-source-repo/output/** \ No newline at end of file diff --git a/src/test/featuresCLICommands/example-source-repo/src/featureA/devcontainer-feature.json b/src/test/featuresCLICommands/example-source-repo/src/featureA/devcontainer-feature.json new file mode 100644 index 000000000..bfc1a97ab --- /dev/null +++ b/src/test/featuresCLICommands/example-source-repo/src/featureA/devcontainer-feature.json @@ -0,0 +1,15 @@ +{ + "id": "featureA", + "name": "Feature A", + "description": "Installs feature A", + "options": { + "version": { + "type": "string", + "proposals": [ + "latest" + ], + "default": "latest", + "description": "Set latest version" + } + } +} \ No newline at end of file diff --git a/src/test/featuresCLICommands/example-source-repo/src/featureA/install.sh b/src/test/featuresCLICommands/example-source-repo/src/featureA/install.sh new file mode 100644 index 000000000..a6daecc54 --- /dev/null +++ b/src/test/featuresCLICommands/example-source-repo/src/featureA/install.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +echo "featureA!" \ No newline at end of file diff --git a/src/test/featuresCLICommands/example-source-repo/src/featureB/devcontainer-feature.json b/src/test/featuresCLICommands/example-source-repo/src/featureB/devcontainer-feature.json new file mode 100644 index 000000000..ede04022c --- /dev/null +++ b/src/test/featuresCLICommands/example-source-repo/src/featureB/devcontainer-feature.json @@ -0,0 +1,15 @@ +{ + "id": "featureB", + "name": "Feature B", + "description": "Installs feature B", + "options": { + "version": { + "type": "string", + "proposals": [ + "latest" + ], + "default": "latest", + "description": "Set latest version" + } + } +} \ No newline at end of file diff --git a/src/test/featuresCLICommands/example-source-repo/src/featureB/install.sh b/src/test/featuresCLICommands/example-source-repo/src/featureB/install.sh new file mode 100644 index 000000000..303675b47 --- /dev/null +++ b/src/test/featuresCLICommands/example-source-repo/src/featureB/install.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +echo "featureB!" \ No newline at end of file diff --git a/src/test/featuresCLICommands/example-source-repo/src/featureC/devcontainer-feature.json b/src/test/featuresCLICommands/example-source-repo/src/featureC/devcontainer-feature.json new file mode 100644 index 000000000..792738473 --- /dev/null +++ b/src/test/featuresCLICommands/example-source-repo/src/featureC/devcontainer-feature.json @@ -0,0 +1,15 @@ +{ + "id": "featureC", + "name": "Feature C", + "description": "Installs feature C", + "options": { + "version": { + "type": "string", + "proposals": [ + "latest" + ], + "default": "latest", + "description": "Set latest version" + } + } +} \ No newline at end of file diff --git a/src/test/featuresCLICommands/example-source-repo/src/featureC/install.sh b/src/test/featuresCLICommands/example-source-repo/src/featureC/install.sh new file mode 100644 index 000000000..d388c123f --- /dev/null +++ b/src/test/featuresCLICommands/example-source-repo/src/featureC/install.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +echo "featureC!" \ No newline at end of file diff --git a/src/test/featuresCLICommands/example-source-repo/src/featureC/other-file.md b/src/test/featuresCLICommands/example-source-repo/src/featureC/other-file.md new file mode 100644 index 000000000..d2aeeec3f --- /dev/null +++ b/src/test/featuresCLICommands/example-source-repo/src/featureC/other-file.md @@ -0,0 +1 @@ +hello there \ No newline at end of file diff --git a/src/test/featuresCLICommands/featuresSubCommands.test.ts b/src/test/featuresCLICommands/featuresSubCommands.test.ts new file mode 100644 index 000000000..9173b675e --- /dev/null +++ b/src/test/featuresCLICommands/featuresSubCommands.test.ts @@ -0,0 +1,99 @@ +import { assert } from 'chai'; +import { before } from 'mocha'; +import tar from 'tar'; +import path from 'path'; +import { FeaturesPackageCommandInput } from '../../../src/spec-node/featuresCLI/package'; +import { DevContainerCollectionMetadata, doFeaturesPackageCommand } from '../../../src/spec-node/featuresCLI/packageCommandImpl'; +import { getCLIHost } from '../../spec-common/cliHost'; +import { loadNativeModule } from '../../spec-common/commonUtils'; +import { createLog } from '../../spec-node/devContainers'; +import { getPackageConfig } from '../../spec-node/utils'; +import { mapLogLevel } from '../../spec-utils/log'; +import { isLocalFile, mkdirpLocal, readLocalFile, rmLocal } from '../../spec-utils/pfs'; + +describe('features package command', () => { + + const disposables: (() => Promise | undefined)[] = []; + const dispose = async () => { + await Promise.all(disposables.map(d => d())); + }; + + const srcFolder = `${__dirname}/example-source-repo/src`; + console.log(srcFolder); + const outputDir = `${__dirname}/example-source-repo/output`; + console.log(outputDir); + + // Package + before(async () => { + + await rmLocal(outputDir, { recursive: true, force: true }); + await mkdirpLocal(outputDir); + + const extensionPath = path.join(__dirname, '..', '..', '..'); + const pkg = await getPackageConfig(extensionPath); + + const cwd = process.cwd(); + const cliHost = await getCLIHost(cwd, loadNativeModule); + const output = createLog({ + logLevel: mapLogLevel('trace'), + logFormat: 'text', + log: (str) => process.stdout.write(str), + terminalDimensions: undefined, + }, pkg, new Date(), disposables); + + const args: FeaturesPackageCommandInput = { + cliHost, + srcFolder, + outputDir, + output, + disposables, + }; + + await doFeaturesPackageCommand(args); + + }); + + after(async () => { + await rmLocal(outputDir, { recursive: true, force: true }); + await dispose(); + }); + + it('should generate tgzs', async () => { + const featureAExists = await isLocalFile(`${outputDir}/devcontainer-feature-featureA.tgz`); + const featureBExists = await isLocalFile(`${outputDir}/devcontainer-feature-featureB.tgz`); + const featureCExists = await isLocalFile(`${outputDir}/devcontainer-feature-featureC.tgz`); + assert.isTrue(featureAExists); + assert.isTrue(featureBExists); + assert.isTrue(featureCExists); + }); + + it('should have a valid collection metadata file', async () => { + const collectionMetadataFileExists = await isLocalFile(`${outputDir}/devcontainer-collection.json`); + assert.isTrue(collectionMetadataFileExists); + + const collectionMetadataFile = await readLocalFile(`${outputDir}/devcontainer-collection.json`, 'utf8'); + const collectionMetadata: DevContainerCollectionMetadata = JSON.parse(collectionMetadataFile); + assert.equal(collectionMetadata.features.length, 3); + assert.strictEqual(collectionMetadata.features.find(x => x.id === 'featureA')?.name, 'Feature A'); + }); + + it('should have non-empty tgz files', async () => { + const tgzPath = `${outputDir}/devcontainer-feature-featureC.tgz`; + + await tar.x( + { + file: tgzPath, + cwd: outputDir, + } + ); + + const installShExists = await isLocalFile(`${outputDir}/install.sh`); + assert.isTrue(installShExists); + const jsonExists = await isLocalFile(`${outputDir}/devcontainer-feature.json`); + assert.isTrue(jsonExists); + const otherfileExists = await isLocalFile(`${outputDir}/other-file.md`); + assert.isTrue(otherfileExists); + const otherFileContents = await readLocalFile(`${outputDir}/other-file.md`, 'utf8'); + assert.strictEqual(otherFileContents, 'hello there'); + }); +}); \ No newline at end of file From 5d3a5df3395445963128e4bab2148200cc3f546d Mon Sep 17 00:00:00 2001 From: Josh Spicer Date: Mon, 1 Aug 2022 19:41:56 +0000 Subject: [PATCH 10/30] fix path to test --- src/spec-node/featuresCLI/package.ts | 5 ++++- .../featuresSubCommands.test.ts | 22 +++++++++---------- 2 files changed, 15 insertions(+), 12 deletions(-) rename src/test/{featuresCLICommands => }/featuresSubCommands.test.ts (82%) diff --git a/src/spec-node/featuresCLI/package.ts b/src/spec-node/featuresCLI/package.ts index e214aaf37..0c97deabb 100644 --- a/src/spec-node/featuresCLI/package.ts +++ b/src/spec-node/featuresCLI/package.ts @@ -17,7 +17,10 @@ export function featuresPackageOptions(y: Argv) { 'force-clean-output-dir': { type: 'boolean', alias: 'f', default: false, description: 'Automatically delete previous output directory before packaging' }, 'log-level': { choices: ['info' as 'info', 'debug' as 'debug', 'trace' as 'trace'], default: 'info' as 'info', description: 'Log level.' }, }) - .check(_argv => { + .check(argv => { + if (argv['scenarios'] && argv['features']) { + throw new Error('Cannot combine --scenarios and --features'); + } return true; }); } diff --git a/src/test/featuresCLICommands/featuresSubCommands.test.ts b/src/test/featuresSubCommands.test.ts similarity index 82% rename from src/test/featuresCLICommands/featuresSubCommands.test.ts rename to src/test/featuresSubCommands.test.ts index 9173b675e..ac5ca9432 100644 --- a/src/test/featuresCLICommands/featuresSubCommands.test.ts +++ b/src/test/featuresSubCommands.test.ts @@ -2,14 +2,14 @@ import { assert } from 'chai'; import { before } from 'mocha'; import tar from 'tar'; import path from 'path'; -import { FeaturesPackageCommandInput } from '../../../src/spec-node/featuresCLI/package'; -import { DevContainerCollectionMetadata, doFeaturesPackageCommand } from '../../../src/spec-node/featuresCLI/packageCommandImpl'; -import { getCLIHost } from '../../spec-common/cliHost'; -import { loadNativeModule } from '../../spec-common/commonUtils'; -import { createLog } from '../../spec-node/devContainers'; -import { getPackageConfig } from '../../spec-node/utils'; -import { mapLogLevel } from '../../spec-utils/log'; -import { isLocalFile, mkdirpLocal, readLocalFile, rmLocal } from '../../spec-utils/pfs'; +import { FeaturesPackageCommandInput } from '..//../src/spec-node/featuresCLI/package'; +import { DevContainerCollectionMetadata, doFeaturesPackageCommand } from '../../src/spec-node/featuresCLI/packageCommandImpl'; +import { getCLIHost } from '../spec-common/cliHost'; +import { loadNativeModule } from '..//spec-common/commonUtils'; +import { createLog } from '../spec-node/devContainers'; +import { getPackageConfig } from '../spec-node/utils'; +import { mapLogLevel } from '..//spec-utils/log'; +import { isLocalFile, mkdirpLocal, readLocalFile, rmLocal } from '../spec-utils/pfs'; describe('features package command', () => { @@ -18,9 +18,9 @@ describe('features package command', () => { await Promise.all(disposables.map(d => d())); }; - const srcFolder = `${__dirname}/example-source-repo/src`; + const srcFolder = `${__dirname}/featuresCLICommands/example-source-repo/src`; console.log(srcFolder); - const outputDir = `${__dirname}/example-source-repo/output`; + const outputDir = `${__dirname}/featuresCLICommands/example-source-repo/output`; console.log(outputDir); // Package @@ -29,7 +29,7 @@ describe('features package command', () => { await rmLocal(outputDir, { recursive: true, force: true }); await mkdirpLocal(outputDir); - const extensionPath = path.join(__dirname, '..', '..', '..'); + const extensionPath = path.join(__dirname, '..', '..'); const pkg = await getPackageConfig(extensionPath); const cwd = process.cwd(); From d8d00628950d648dec0408e841589c02daff96ff Mon Sep 17 00:00:00 2001 From: Samruddhi Khandale Date: Tue, 2 Aug 2022 00:20:47 +0000 Subject: [PATCH 11/30] draft --- .gitignore | 3 +- src/spec-node/devContainersSpecCLI.ts | 2 + src/spec-node/featuresCLI/package.ts | 7 +- src/spec-node/featuresCLI/publish.ts | 132 ++++++++++++++++++++++++++ 4 files changed, 140 insertions(+), 4 deletions(-) create mode 100644 src/spec-node/featuresCLI/publish.ts diff --git a/.gitignore b/.gitignore index 105572779..219519f6f 100644 --- a/.gitignore +++ b/.gitignore @@ -6,4 +6,5 @@ logs tmp .DS_Store .env -output \ No newline at end of file +output +package diff --git a/src/spec-node/devContainersSpecCLI.ts b/src/spec-node/devContainersSpecCLI.ts index ff883f963..252028bf1 100644 --- a/src/spec-node/devContainersSpecCLI.ts +++ b/src/spec-node/devContainersSpecCLI.ts @@ -26,6 +26,7 @@ import { loadNativeModule } from '../spec-common/commonUtils'; import { generateFeaturesConfig, getContainerFeaturesFolder } from '../spec-configuration/containerFeaturesConfiguration'; import { featuresTestOptions, featuresTestHandler } from './featuresCLI/test'; import { featuresPackageHandler, featuresPackageOptions } from './featuresCLI/package'; +import { featuresPublishHandler, featuresPublishOptions } from './featuresCLI/publish'; const defaultDefaultUserEnvProbe: UserEnvProbe = 'loginInteractiveShell'; @@ -52,6 +53,7 @@ const defaultDefaultUserEnvProbe: UserEnvProbe = 'loginInteractiveShell'; y.command('features', 'Features commands', (y: Argv) => { y.command('test', 'Test features', featuresTestOptions, featuresTestHandler); y.command('package', 'Package features', featuresPackageOptions, featuresPackageHandler); + y.command('publish', 'Publish features', featuresPublishOptions, featuresPublishHandler); }); y.command(restArgs ? ['exec', '*'] : ['exec [args..]'], 'Execute a command on a running dev container', execOptions, execHandler); y.parse(restArgs ? argv.slice(1) : argv); diff --git a/src/spec-node/featuresCLI/package.ts b/src/spec-node/featuresCLI/package.ts index e214aaf37..8a8aa9c1b 100644 --- a/src/spec-node/featuresCLI/package.ts +++ b/src/spec-node/featuresCLI/package.ts @@ -35,7 +35,7 @@ export function featuresPackageHandler(args: FeaturesPackageArgs) { (async () => await featuresPackage(args))().catch(console.error); } -async function featuresPackage({ +export async function featuresPackage({ 'feature-collection-folder': featureCollectionFolder, 'log-level': inputLogLevel, 'output-dir': outputDir, @@ -88,8 +88,9 @@ async function featuresPackage({ disposables }; - const exitCode = await doFeaturesPackageCommand(args); + await doFeaturesPackageCommand(args); + // const exitCode = await doFeaturesPackageCommand(args); await dispose(); - process.exit(exitCode); + // process.exit(exitCode); } \ No newline at end of file diff --git a/src/spec-node/featuresCLI/publish.ts b/src/spec-node/featuresCLI/publish.ts new file mode 100644 index 000000000..9d0a9313d --- /dev/null +++ b/src/spec-node/featuresCLI/publish.ts @@ -0,0 +1,132 @@ +// import path from 'path'; +import { Argv } from 'yargs'; +// import { getCLIHost } from '../../spec-common/cliHost'; +// import { loadNativeModule } from '../../spec-common/commonUtils'; +import { request } from '../../spec-utils/httpRequest'; +import { createPlainLog, Log, LogLevel, makeLog } from '../../spec-utils/log'; +import { rmLocal } from '../../spec-utils/pfs'; +import { UnpackArgv } from '../devContainersSpecCLI'; +import { FeaturesPackageArgs, featuresPackage } from './package'; + +export const output = makeLog(createPlainLog(text => process.stdout.write(text), () => LogLevel.Trace)); + +export function featuresPublishOptions(y: Argv) { + return y + .options({ + 'feature-collection-folder': { type: 'string', alias: 'c', default: '.', description: 'Path to folder containing source code for collection of features' }, + 'registry': { type: 'string', alias: 'r', default: 'ghcr.io', description: 'Name of the OCI registry.' }, + 'namespace': { type: 'string', alias: 'n', require: true, description: 'Unique indentifier for the collection of features. Example: /' }, + 'log-level': { choices: ['info' as 'info', 'debug' as 'debug', 'trace' as 'trace'], default: 'info' as 'info', description: 'Log level.' }, + 'fea': { type: 'string', alias: 'f', require: true, description: 'Unique indentifier for the collection of features. Example: /' }, + }) + .check(_argv => { + return true; + }); +} + +export type FeaturesPublishArgs = UnpackArgv>; + +export function featuresPublishHandler(args: FeaturesPublishArgs) { + (async () => await featuresPublish(args))().catch(console.error); +} + +async function featuresPublish({ + 'feature-collection-folder': featureCollectionFolder, + 'log-level': inputLogLevel, + 'registry': registry, + 'namespace': namespace, + 'fea': feature +}: FeaturesPublishArgs) { + // Package features + const outputDir = '/tmp/features-output'; + + const packageArgs: FeaturesPackageArgs = { + 'feature-collection-folder': featureCollectionFolder, + 'force-clean-output-dir': true, + 'log-level': inputLogLevel, + 'output-dir': outputDir + }; + + // featuresPackageHandler(packageArgs); + await featuresPackage(packageArgs); + output.write('trying to get tags'); + await getTagsList(feature, registry, namespace, featureCollectionFolder); + + // Cleanup + await rmLocal(outputDir, { recursive: true, force: true }); +} + +async function getTagsList(featureId: string, registry: string, namespace: string, workspace: string) { + console.log(workspace); + const url = `https://${registry}/v2/${namespace}/${featureId}/tags/list`; + const id = `${registry}/${namespace}/${featureId}`; + output.write(`URL: ${url}`); + output.write(`id: ${id}`); + + try { + // const workspaceFolder = path.resolve(process.cwd(), workspace); + // const cwd = workspaceFolder || process.cwd(); + // const cliHost = await getCLIHost(cwd, loadNativeModule); + const env = process.env; + const headers = { + // 'user-agent': 'devcontainer', + 'Authorization': await getAuthenticationToken(env, output, registry, id), + 'Accept': 'application/json', + }; + + const options = { + type: 'GET', + url: url, + headers: headers + }; + + const response = JSON.parse((await request(options, output)).toString()); + output.write('Respone yay'); + output.write(response.toString()); + output.write(response); + // output.write(JSON.parse(response.toString())); + } catch (error) { + output.write('Failed~~~~~'); + } +} + +async function getAuthenticationToken(env: NodeJS.ProcessEnv, output: Log, registry: string, id: string): Promise { + // TODO: Use operating system keychain to get credentials. + // TODO: Fallback to read docker config to get credentials. + + // const githubToken = env['GITHUB_TOKEN']; + + // if (githubToken) { + // output.write('Found GITHUB_TOKEN'); + // return 'Bearer ' + githubToken; + // } else { + console.log(env['GITHUB_TOKEN']); + output.write(`Fetching GHCR token`); + if (registry === 'ghcr.io') { + const token = await getGHCRtoken(output, id); + output.write(token); + output.write('fetched'); + return 'Bearer ' + token; + } + // } + + return ''; +} + +export async function getGHCRtoken(output: Log, id: string) { + const headers = { + 'user-agent': 'devcontainer', + }; + + const url = `https://ghcr.io/token?scope=repo:${id}:pull&service=ghcr.io`; + + const options = { + type: 'GET', + url: url, + headers: headers + }; + + const token = JSON.parse((await request(options, output)).toString()).token; + + return token; +} \ No newline at end of file From bea7064ac59f7aff9270c5056bb7bc59273b1af5 Mon Sep 17 00:00:00 2001 From: Samruddhi Khandale Date: Tue, 2 Aug 2022 19:02:42 +0000 Subject: [PATCH 12/30] working FE --- package.json | 2 + .../containerFeaturesConfiguration.ts | 1 + src/spec-node/featuresCLI/publish.ts | 111 +++++------------- .../featuresCLI/publishCommandImpl.ts | 99 ++++++++++++++++ yarn.lock | 10 ++ 5 files changed, 139 insertions(+), 84 deletions(-) create mode 100644 src/spec-node/featuresCLI/publishCommandImpl.ts diff --git a/package.json b/package.json index 430a091c2..155db9d1b 100644 --- a/package.json +++ b/package.json @@ -45,6 +45,7 @@ "@types/ncp": "^2.0.5", "@types/pull-stream": "^3.6.2", "@types/semver": "^7.3.9", + "@types/semver-compare": "^1.0.1", "@types/shell-quote": "^1.7.1", "@types/tar": "^6.1.1", "@types/yargs": "^17.0.8", @@ -75,6 +76,7 @@ "node-pty": "^0.10.1", "pull-stream": "^3.6.14", "semver": "^7.3.5", + "semver-compare": "1.0.0", "shell-quote": "^1.7.3", "stream-to-pull-stream": "^1.7.3", "tar": "^6.1.11", diff --git a/src/spec-configuration/containerFeaturesConfiguration.ts b/src/spec-configuration/containerFeaturesConfiguration.ts index fbef318f5..605023770 100644 --- a/src/spec-configuration/containerFeaturesConfiguration.ts +++ b/src/spec-configuration/containerFeaturesConfiguration.ts @@ -18,6 +18,7 @@ const V1_ASSET_NAME = 'devcontainer-features.tgz'; export interface Feature { id: string; + version?: string; name: string; description?: string; cachePath?: string; diff --git a/src/spec-node/featuresCLI/publish.ts b/src/spec-node/featuresCLI/publish.ts index 9d0a9313d..5ba64f8c1 100644 --- a/src/spec-node/featuresCLI/publish.ts +++ b/src/spec-node/featuresCLI/publish.ts @@ -1,12 +1,11 @@ -// import path from 'path'; +import path from 'path'; import { Argv } from 'yargs'; -// import { getCLIHost } from '../../spec-common/cliHost'; -// import { loadNativeModule } from '../../spec-common/commonUtils'; -import { request } from '../../spec-utils/httpRequest'; -import { createPlainLog, Log, LogLevel, makeLog } from '../../spec-utils/log'; -import { rmLocal } from '../../spec-utils/pfs'; +import { createPlainLog, LogLevel, makeLog } from '../../spec-utils/log'; +import { readLocalFile, rmLocal } from '../../spec-utils/pfs'; import { UnpackArgv } from '../devContainersSpecCLI'; import { FeaturesPackageArgs, featuresPackage } from './package'; +import { DevContainerCollectionMetadata } from './packageCommandImpl'; +import { getPublishedVersions, getSermanticVersions } from './publishCommandImpl'; export const output = makeLog(createPlainLog(text => process.stdout.write(text), () => LogLevel.Trace)); @@ -16,8 +15,7 @@ export function featuresPublishOptions(y: Argv) { 'feature-collection-folder': { type: 'string', alias: 'c', default: '.', description: 'Path to folder containing source code for collection of features' }, 'registry': { type: 'string', alias: 'r', default: 'ghcr.io', description: 'Name of the OCI registry.' }, 'namespace': { type: 'string', alias: 'n', require: true, description: 'Unique indentifier for the collection of features. Example: /' }, - 'log-level': { choices: ['info' as 'info', 'debug' as 'debug', 'trace' as 'trace'], default: 'info' as 'info', description: 'Log level.' }, - 'fea': { type: 'string', alias: 'f', require: true, description: 'Unique indentifier for the collection of features. Example: /' }, + 'log-level': { choices: ['info' as 'info', 'debug' as 'debug', 'trace' as 'trace'], default: 'info' as 'info', description: 'Log level.' } }) .check(_argv => { return true; @@ -34,8 +32,7 @@ async function featuresPublish({ 'feature-collection-folder': featureCollectionFolder, 'log-level': inputLogLevel, 'registry': registry, - 'namespace': namespace, - 'fea': feature + 'namespace': namespace }: FeaturesPublishArgs) { // Package features const outputDir = '/tmp/features-output'; @@ -47,86 +44,32 @@ async function featuresPublish({ 'output-dir': outputDir }; - // featuresPackageHandler(packageArgs); await featuresPackage(packageArgs); - output.write('trying to get tags'); - await getTagsList(feature, registry, namespace, featureCollectionFolder); - // Cleanup - await rmLocal(outputDir, { recursive: true, force: true }); -} + const metadataOutputPath = path.join(outputDir, 'devcontainer-collection.json'); + const metadata: DevContainerCollectionMetadata = JSON.parse(await readLocalFile(metadataOutputPath, 'utf-8')); -async function getTagsList(featureId: string, registry: string, namespace: string, workspace: string) { - console.log(workspace); - const url = `https://${registry}/v2/${namespace}/${featureId}/tags/list`; - const id = `${registry}/${namespace}/${featureId}`; - output.write(`URL: ${url}`); - output.write(`id: ${id}`); + for (const f of metadata.features) { + output.write(`Processing feature: ${f.id}`, LogLevel.Info); - try { - // const workspaceFolder = path.resolve(process.cwd(), workspace); - // const cwd = workspaceFolder || process.cwd(); - // const cliHost = await getCLIHost(cwd, loadNativeModule); - const env = process.env; - const headers = { - // 'user-agent': 'devcontainer', - 'Authorization': await getAuthenticationToken(env, output, registry, id), - 'Accept': 'application/json', - }; + output.write(`Fetching published versions...`, LogLevel.Info); + const publishedVersions: string[] | undefined = await getPublishedVersions(f.id, registry, namespace, output); + let semanticVersions; - const options = { - type: 'GET', - url: url, - headers: headers - }; + if (publishedVersions !== undefined && f.version !== undefined && publishedVersions.includes(f.version)) { + output.write(`Skipping ${f.id} as ${f.version} is already published...`); + } else { + if (f.version !== undefined && publishedVersions !== undefined) { + semanticVersions = getSermanticVersions(f.version, publishedVersions); - const response = JSON.parse((await request(options, output)).toString()); - output.write('Respone yay'); - output.write(response.toString()); - output.write(response); - // output.write(JSON.parse(response.toString())); - } catch (error) { - output.write('Failed~~~~~'); - } -} - -async function getAuthenticationToken(env: NodeJS.ProcessEnv, output: Log, registry: string, id: string): Promise { - // TODO: Use operating system keychain to get credentials. - // TODO: Fallback to read docker config to get credentials. - - // const githubToken = env['GITHUB_TOKEN']; - - // if (githubToken) { - // output.write('Found GITHUB_TOKEN'); - // return 'Bearer ' + githubToken; - // } else { - console.log(env['GITHUB_TOKEN']); - output.write(`Fetching GHCR token`); - if (registry === 'ghcr.io') { - const token = await getGHCRtoken(output, id); - output.write(token); - output.write('fetched'); - return 'Bearer ' + token; + if (semanticVersions !== null) { + output.write(`Publishing versions ${semanticVersions}`); + // CALL OCI PUSH + } + } } - // } + } - return ''; + // Cleanup + await rmLocal(outputDir, { recursive: true, force: true }); } - -export async function getGHCRtoken(output: Log, id: string) { - const headers = { - 'user-agent': 'devcontainer', - }; - - const url = `https://ghcr.io/token?scope=repo:${id}:pull&service=ghcr.io`; - - const options = { - type: 'GET', - url: url, - headers: headers - }; - - const token = JSON.parse((await request(options, output)).toString()).token; - - return token; -} \ No newline at end of file diff --git a/src/spec-node/featuresCLI/publishCommandImpl.ts b/src/spec-node/featuresCLI/publishCommandImpl.ts new file mode 100644 index 000000000..b21fea8f0 --- /dev/null +++ b/src/spec-node/featuresCLI/publishCommandImpl.ts @@ -0,0 +1,99 @@ +import { request } from '../../spec-utils/httpRequest'; +import { Log, LogLevel } from '../../spec-utils/log'; +import { output } from './publish'; + +const semverCompare = require('semver-compare'); +const semver = require('semver'); + +interface versions { + name: string; + tags: string[]; +} + +export async function getPublishedVersions(featureId: string, registry: string, namespace: string, output: Log) { + const url = `https://${registry}/v2/${namespace}/${featureId}/tags/list`; + const id = `${registry}/${namespace}/${featureId}`; + + try { + const headers = { + 'user-agent': 'devcontainer', + 'Authorization': await getAuthenticationToken(output, registry, id), + 'Accept': 'application/json', + }; + + const options = { + type: 'GET', + url: url, + headers: headers + }; + + const response = await request(options, output); + const publishedVersionsResponse: versions = JSON.parse(response.toString()); + + return publishedVersionsResponse.tags; + } catch (e) { + // Publishing for the first time + if (e && e.code === 403) { + return []; + } + + output.write(`(!) ERR: Failed to publish feature: ${e?.message ?? ''} `, LogLevel.Error); + return undefined; + } +} + +export function getSermanticVersions(version: string, publishedVersions: string[]) { + let semanticVersions: string[] = []; + if (semver.valid(version) === null) { + output.write(`Skipping as version ${version} is not a valid semantic version...`); + return null; + } + + // Add semantic versions ex. 1.2.3 --> [1, 1.2, 1.2.3] + const parsedVersion = semver.parse(version); + + semanticVersions.push(parsedVersion.major); + semanticVersions.push(`${parsedVersion.major}.${parsedVersion.minor}`); + semanticVersions.push(version); + + let publishLatest = true; + if (publishedVersions.length > 0) { + const sortedVersions = publishedVersions.sort(semverCompare); + + // Compare version with the last published version + publishLatest = semverCompare(version, sortedVersions[sortedVersions.length - 1]) === 1 ? true : false; + } + + if (publishLatest) { + semanticVersions.push('latest'); + } + + return semanticVersions; +} + +async function getAuthenticationToken(output: Log, registry: string, id: string): Promise { + if (registry === 'ghcr.io') { + const token = await getGHCRtoken(output, id); + return 'Bearer ' + token; + } + + return ''; +} + +export async function getGHCRtoken(output: Log, id: string) { + const headers = { + 'user-agent': 'devcontainer', + }; + + const url = `https://ghcr.io/token?scope=repo:${id}:pull&service=ghcr.io`; + + const options = { + type: 'GET', + url: url, + headers: headers + }; + + const token = JSON.parse((await request(options, output)).toString()).token; + + return token; +} diff --git a/yarn.lock b/yarn.lock index d0e5ede41..f3f1d333d 100644 --- a/yarn.lock +++ b/yarn.lock @@ -180,6 +180,11 @@ resolved "https://registry.yarnpkg.com/@types/pull-stream/-/pull-stream-3.6.2.tgz#184165017b0764b9a44aff0b555c795ad6cdf9f9" integrity sha512-s5jYmaJH68IQb9JjsemWUZCpaQdotd7B4xfXQtcKvGmQxcBXD/mvSQoi3TzPt2QqpDLjImxccS4en8f8E8O0FA== +"@types/semver-compare@^1.0.1": + version "1.0.1" + resolved "https://registry.yarnpkg.com/@types/semver-compare/-/semver-compare-1.0.1.tgz#17d1dc62c516c133ab01efb7803a537ee6eaf3d5" + integrity sha512-wx2LQVvKlEkhXp/HoKIZ/aSL+TvfJdKco8i0xJS3aR877mg4qBHzNT6+B5a61vewZHo79EdZavskGnRXEC2H6A== + "@types/semver@^7.3.9": version "7.3.9" resolved "https://registry.yarnpkg.com/@types/semver/-/semver-7.3.9.tgz#152c6c20a7688c30b967ec1841d31ace569863fc" @@ -2517,6 +2522,11 @@ safe-buffer@~5.1.0, safe-buffer@~5.1.1: resolved "https://registry.yarnpkg.com/safer-buffer/-/safer-buffer-2.1.2.tgz#44fa161b0187b9549dd84bb91802f9bd8385cd6a" integrity sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg== +semver-compare@1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/semver-compare/-/semver-compare-1.0.0.tgz#0dee216a1c941ab37e9efb1788f6afc5ff5537fc" + integrity sha512-YM3/ITh2MJ5MtzaM429anh+x2jiLVjqILF4m4oyQB18W7Ggea7BfqdH/wGMK7dDiMghv/6WG7znWMwUDzJiXow== + "semver@2 || 3 || 4 || 5", semver@^5.5.0, semver@^5.6.0: version "5.7.1" resolved "https://registry.yarnpkg.com/semver/-/semver-5.7.1.tgz#a954f931aeba508d307bbf069eff0c01c96116f7" From bc7a88fe6c1172adfa27f64b0cea7664b92fa027 Mon Sep 17 00:00:00 2001 From: Samruddhi Khandale Date: Tue, 2 Aug 2022 19:34:07 +0000 Subject: [PATCH 13/30] optimizing code --- src/spec-node/featuresCLI/package.ts | 10 ++-- src/spec-node/featuresCLI/publish.ts | 60 ++++++++++++++----- .../featuresCLI/publishCommandImpl.ts | 11 ++-- 3 files changed, 56 insertions(+), 25 deletions(-) diff --git a/src/spec-node/featuresCLI/package.ts b/src/spec-node/featuresCLI/package.ts index 8a8aa9c1b..a91e9dd25 100644 --- a/src/spec-node/featuresCLI/package.ts +++ b/src/spec-node/featuresCLI/package.ts @@ -40,7 +40,7 @@ export async function featuresPackage({ 'log-level': inputLogLevel, 'output-dir': outputDir, 'force-clean-output-dir': forceCleanOutputDir, -}: FeaturesPackageArgs) { +}: FeaturesPackageArgs, shouldExit: boolean = true) { const disposables: (() => Promise | undefined)[] = []; const dispose = async () => { await Promise.all(disposables.map(d => d())); @@ -88,9 +88,11 @@ export async function featuresPackage({ disposables }; - await doFeaturesPackageCommand(args); - // const exitCode = await doFeaturesPackageCommand(args); + const exitCode = await doFeaturesPackageCommand(args); await dispose(); - // process.exit(exitCode); + + if (shouldExit) { + process.exit(exitCode); + } } \ No newline at end of file diff --git a/src/spec-node/featuresCLI/publish.ts b/src/spec-node/featuresCLI/publish.ts index 5ba64f8c1..ee71fe480 100644 --- a/src/spec-node/featuresCLI/publish.ts +++ b/src/spec-node/featuresCLI/publish.ts @@ -1,14 +1,14 @@ import path from 'path'; import { Argv } from 'yargs'; -import { createPlainLog, LogLevel, makeLog } from '../../spec-utils/log'; -import { readLocalFile, rmLocal } from '../../spec-utils/pfs'; +import { LogLevel, mapLogLevel } from '../../spec-utils/log'; +import { isLocalFile, readLocalFile, rmLocal } from '../../spec-utils/pfs'; +import { getPackageConfig } from '../../spec-utils/product'; +import { createLog } from '../devContainers'; import { UnpackArgv } from '../devContainersSpecCLI'; import { FeaturesPackageArgs, featuresPackage } from './package'; import { DevContainerCollectionMetadata } from './packageCommandImpl'; import { getPublishedVersions, getSermanticVersions } from './publishCommandImpl'; -export const output = makeLog(createPlainLog(text => process.stdout.write(text), () => LogLevel.Trace)); - export function featuresPublishOptions(y: Argv) { return y .options({ @@ -34,6 +34,21 @@ async function featuresPublish({ 'registry': registry, 'namespace': namespace }: FeaturesPublishArgs) { + const disposables: (() => Promise | undefined)[] = []; + + const dispose = async () => { + await Promise.all(disposables.map(d => d())); + }; + + const extensionPath = path.join(__dirname, '..', '..', '..'); + const pkg = await getPackageConfig(extensionPath); + const output = createLog({ + logLevel: mapLogLevel(inputLogLevel), + logFormat: 'text', + log: (str) => process.stdout.write(str), + terminalDimensions: undefined, + }, pkg, new Date(), disposables); + // Package features const outputDir = '/tmp/features-output'; @@ -44,32 +59,45 @@ async function featuresPublish({ 'output-dir': outputDir }; - await featuresPackage(packageArgs); + await featuresPackage(packageArgs, false); const metadataOutputPath = path.join(outputDir, 'devcontainer-collection.json'); + if (!isLocalFile(metadataOutputPath)) { + output.write(`(!) ERR: Failed to fetch ${metadataOutputPath}`, LogLevel.Error); + process.exit(1); + } + const metadata: DevContainerCollectionMetadata = JSON.parse(await readLocalFile(metadataOutputPath, 'utf-8')); + // temp + const exitCode = 0; + for (const f of metadata.features) { - output.write(`Processing feature: ${f.id}`, LogLevel.Info); + output.write('\n'); + output.write(`Processing feature: ${f.id}...`, LogLevel.Info); output.write(`Fetching published versions...`, LogLevel.Info); - const publishedVersions: string[] | undefined = await getPublishedVersions(f.id, registry, namespace, output); - let semanticVersions; + const publishedVersions: string[] = await getPublishedVersions(f.id, registry, namespace, output); - if (publishedVersions !== undefined && f.version !== undefined && publishedVersions.includes(f.version)) { - output.write(`Skipping ${f.id} as ${f.version} is already published...`); + if (f.version !== undefined && publishedVersions.includes(f.version)) { + output.write(`Skipping ${f.id} as ${f.version} is already published...`, LogLevel.Warning); } else { - if (f.version !== undefined && publishedVersions !== undefined) { - semanticVersions = getSermanticVersions(f.version, publishedVersions); + if (f.version !== undefined) { + const semanticVersions: string[] = getSermanticVersions(f.version, publishedVersions, output); + + output.write(`Publishing versions: ${semanticVersions.toString()}...`, LogLevel.Info); - if (semanticVersions !== null) { - output.write(`Publishing versions ${semanticVersions}`); - // CALL OCI PUSH - } + // TODO: CALL OCI PUSH + // exitCode = await doFeaturesPublishCommand(); + + output.write(`Published feature: ${f.id}...`, LogLevel.Info); } } } // Cleanup await rmLocal(outputDir, { recursive: true, force: true }); + + await dispose(); + process.exit(exitCode); } diff --git a/src/spec-node/featuresCLI/publishCommandImpl.ts b/src/spec-node/featuresCLI/publishCommandImpl.ts index b21fea8f0..437453a7d 100644 --- a/src/spec-node/featuresCLI/publishCommandImpl.ts +++ b/src/spec-node/featuresCLI/publishCommandImpl.ts @@ -1,6 +1,5 @@ import { request } from '../../spec-utils/httpRequest'; import { Log, LogLevel } from '../../spec-utils/log'; -import { output } from './publish'; const semverCompare = require('semver-compare'); const semver = require('semver'); @@ -38,15 +37,15 @@ export async function getPublishedVersions(featureId: string, registry: string, } output.write(`(!) ERR: Failed to publish feature: ${e?.message ?? ''} `, LogLevel.Error); - return undefined; + process.exit(1); } } -export function getSermanticVersions(version: string, publishedVersions: string[]) { +export function getSermanticVersions(version: string, publishedVersions: string[], output: Log) { let semanticVersions: string[] = []; if (semver.valid(version) === null) { - output.write(`Skipping as version ${version} is not a valid semantic version...`); - return null; + output.write(`(!) ERR: Version ${version} is not a valid semantic version...`, LogLevel.Error); + process.exit(1); } // Add semantic versions ex. 1.2.3 --> [1, 1.2, 1.2.3] @@ -71,6 +70,7 @@ export function getSermanticVersions(version: string, publishedVersions: string[ return semanticVersions; } +// temp async function getAuthenticationToken(output: Log, registry: string, id: string): Promise { if (registry === 'ghcr.io') { const token = await getGHCRtoken(output, id); @@ -80,6 +80,7 @@ async function getAuthenticationToken(output: Log, registry: string, id: string) return ''; } +// temp export async function getGHCRtoken(output: Log, id: string) { const headers = { 'user-agent': 'devcontainer', From 5503233837c273b57b1579d84bbd40d45376a31d Mon Sep 17 00:00:00 2001 From: Samruddhi Khandale Date: Tue, 2 Aug 2022 21:15:53 +0000 Subject: [PATCH 14/30] nit --- src/spec-node/featuresCLI/publish.ts | 25 +++++++------ .../featuresCLI/publishCommandImpl.ts | 37 ++++++++++++------- 2 files changed, 37 insertions(+), 25 deletions(-) diff --git a/src/spec-node/featuresCLI/publish.ts b/src/spec-node/featuresCLI/publish.ts index ee71fe480..b2f0ed52f 100644 --- a/src/spec-node/featuresCLI/publish.ts +++ b/src/spec-node/featuresCLI/publish.ts @@ -47,7 +47,7 @@ async function featuresPublish({ logFormat: 'text', log: (str) => process.stdout.write(str), terminalDimensions: undefined, - }, pkg, new Date(), disposables); + }, pkg, new Date(), disposables, true); // Package features const outputDir = '/tmp/features-output'; @@ -73,25 +73,26 @@ async function featuresPublish({ const exitCode = 0; for (const f of metadata.features) { - output.write('\n'); output.write(`Processing feature: ${f.id}...`, LogLevel.Info); + if (f.version === undefined) { + output.write(`(!) Version does not exist, skipping ${f.id}...`, LogLevel.Warning); + continue; + } + output.write(`Fetching published versions...`, LogLevel.Info); const publishedVersions: string[] = await getPublishedVersions(f.id, registry, namespace, output); - if (f.version !== undefined && publishedVersions.includes(f.version)) { - output.write(`Skipping ${f.id} as ${f.version} is already published...`, LogLevel.Warning); + if (publishedVersions.includes(f.version)) { + output.write(`(!) Version ${f.version} already exists, skipping ${f.id}...`, LogLevel.Warning); } else { - if (f.version !== undefined) { - const semanticVersions: string[] = getSermanticVersions(f.version, publishedVersions, output); - - output.write(`Publishing versions: ${semanticVersions.toString()}...`, LogLevel.Info); + const semanticVersions: string[] = getSermanticVersions(f.version, publishedVersions, output); + output.write(`Publishing versions: ${semanticVersions.toString()}...`, LogLevel.Info); - // TODO: CALL OCI PUSH - // exitCode = await doFeaturesPublishCommand(); + // TODO: CALL OCI PUSH + // exitCode = await doFeaturesPublishCommand(); - output.write(`Published feature: ${f.id}...`, LogLevel.Info); - } + output.write(`Published feature: ${f.id}...`, LogLevel.Info); } } diff --git a/src/spec-node/featuresCLI/publishCommandImpl.ts b/src/spec-node/featuresCLI/publishCommandImpl.ts index 437453a7d..d30cbbfde 100644 --- a/src/spec-node/featuresCLI/publishCommandImpl.ts +++ b/src/spec-node/featuresCLI/publishCommandImpl.ts @@ -12,11 +12,24 @@ interface versions { export async function getPublishedVersions(featureId: string, registry: string, namespace: string, output: Log) { const url = `https://${registry}/v2/${namespace}/${featureId}/tags/list`; const id = `${registry}/${namespace}/${featureId}`; + let token = ''; + + try { + token = await getAuthenticationToken(registry, id); + } catch (e) { + // Publishing for the first time + if (e?.message.includes('403')) { + return []; + } + + output.write(`(!) ERR: Failed to publish feature: ${e?.message ?? ''} `, LogLevel.Error); + process.exit(1); + } try { const headers = { 'user-agent': 'devcontainer', - 'Authorization': await getAuthenticationToken(output, registry, id), + 'Authorization': token, 'Accept': 'application/json', }; @@ -26,16 +39,11 @@ export async function getPublishedVersions(featureId: string, registry: string, headers: headers }; - const response = await request(options, output); + const response = await request(options); const publishedVersionsResponse: versions = JSON.parse(response.toString()); return publishedVersionsResponse.tags; } catch (e) { - // Publishing for the first time - if (e && e.code === 403) { - return []; - } - output.write(`(!) ERR: Failed to publish feature: ${e?.message ?? ''} `, LogLevel.Error); process.exit(1); } @@ -51,8 +59,11 @@ export function getSermanticVersions(version: string, publishedVersions: string[ // Add semantic versions ex. 1.2.3 --> [1, 1.2, 1.2.3] const parsedVersion = semver.parse(version); - semanticVersions.push(parsedVersion.major); - semanticVersions.push(`${parsedVersion.major}.${parsedVersion.minor}`); + if (parsedVersion.major !== 0) { + semanticVersions.push(parsedVersion.major); + semanticVersions.push(`${parsedVersion.major}.${parsedVersion.minor}`); + } + semanticVersions.push(version); let publishLatest = true; @@ -71,9 +82,9 @@ export function getSermanticVersions(version: string, publishedVersions: string[ } // temp -async function getAuthenticationToken(output: Log, registry: string, id: string): Promise { +async function getAuthenticationToken(registry: string, id: string): Promise { if (registry === 'ghcr.io') { - const token = await getGHCRtoken(output, id); + const token = await getGHCRtoken(id); return 'Bearer ' + token; } @@ -81,7 +92,7 @@ async function getAuthenticationToken(output: Log, registry: string, id: string) } // temp -export async function getGHCRtoken(output: Log, id: string) { +export async function getGHCRtoken(id: string) { const headers = { 'user-agent': 'devcontainer', }; @@ -94,7 +105,7 @@ export async function getGHCRtoken(output: Log, id: string) { headers: headers }; - const token = JSON.parse((await request(options, output)).toString()).token; + const token = JSON.parse((await request(options)).toString()).token; return token; } From fade2097f48a56450d36230d2d3bb612d414c41e Mon Sep 17 00:00:00 2001 From: Samruddhi Khandale Date: Tue, 2 Aug 2022 23:43:02 +0000 Subject: [PATCH 15/30] adding tests --- src/spec-node/featuresCLI/publish.ts | 6 +- .../featuresCLI/publishCommandImpl.ts | 5 ++ src/test/featuresSubCommands.test.ts | 63 ++++++++++++++++++- 3 files changed, 68 insertions(+), 6 deletions(-) diff --git a/src/spec-node/featuresCLI/publish.ts b/src/spec-node/featuresCLI/publish.ts index b2f0ed52f..948cc7412 100644 --- a/src/spec-node/featuresCLI/publish.ts +++ b/src/spec-node/featuresCLI/publish.ts @@ -82,11 +82,9 @@ async function featuresPublish({ output.write(`Fetching published versions...`, LogLevel.Info); const publishedVersions: string[] = await getPublishedVersions(f.id, registry, namespace, output); + const semanticVersions: string[] | undefined = getSermanticVersions(f.version, publishedVersions, output); - if (publishedVersions.includes(f.version)) { - output.write(`(!) Version ${f.version} already exists, skipping ${f.id}...`, LogLevel.Warning); - } else { - const semanticVersions: string[] = getSermanticVersions(f.version, publishedVersions, output); + if (semanticVersions !== undefined) { output.write(`Publishing versions: ${semanticVersions.toString()}...`, LogLevel.Info); // TODO: CALL OCI PUSH diff --git a/src/spec-node/featuresCLI/publishCommandImpl.ts b/src/spec-node/featuresCLI/publishCommandImpl.ts index d30cbbfde..57aab75c4 100644 --- a/src/spec-node/featuresCLI/publishCommandImpl.ts +++ b/src/spec-node/featuresCLI/publishCommandImpl.ts @@ -50,6 +50,11 @@ export async function getPublishedVersions(featureId: string, registry: string, } export function getSermanticVersions(version: string, publishedVersions: string[], output: Log) { + if (publishedVersions.includes(version)) { + output.write(`(!) Version ${version} already exists, skipping ${version}...`, LogLevel.Warning); + return undefined; + } + let semanticVersions: string[] = []; if (semver.valid(version) === null) { output.write(`(!) ERR: Version ${version} is not a valid semantic version...`, LogLevel.Error); diff --git a/src/test/featuresSubCommands.test.ts b/src/test/featuresSubCommands.test.ts index ac5ca9432..7fcc9ff6a 100644 --- a/src/test/featuresSubCommands.test.ts +++ b/src/test/featuresSubCommands.test.ts @@ -8,8 +8,9 @@ import { getCLIHost } from '../spec-common/cliHost'; import { loadNativeModule } from '..//spec-common/commonUtils'; import { createLog } from '../spec-node/devContainers'; import { getPackageConfig } from '../spec-node/utils'; -import { mapLogLevel } from '..//spec-utils/log'; +import { Log, mapLogLevel } from '..//spec-utils/log'; import { isLocalFile, mkdirpLocal, readLocalFile, rmLocal } from '../spec-utils/pfs'; +import { getSermanticVersions } from '../spec-node/featuresCLI/publishCommandImpl'; describe('features package command', () => { @@ -96,4 +97,62 @@ describe('features package command', () => { const otherFileContents = await readLocalFile(`${outputDir}/other-file.md`, 'utf8'); assert.strictEqual(otherFileContents, 'hello there'); }); -}); \ No newline at end of file +}); + +describe('features publish command', () => { + let output: Log; + const disposables: (() => Promise | undefined)[] = []; + const dispose = async () => { + await Promise.all(disposables.map(d => d())); + }; + + before(async () => { + + const extensionPath = path.join(__dirname, '..', '..'); + const pkg = await getPackageConfig(extensionPath); + + output = createLog({ + logLevel: mapLogLevel('trace'), + logFormat: 'text', + log: (str) => process.stdout.write(str), + terminalDimensions: undefined, + }, pkg, new Date(), disposables, true); + }); + + it('should generate correct semantic versions', async () => { + // First publish + let version = '1.0.0'; + let publishedVersions: string[] = []; + let expectedSemVer = ['1', '1.0', '1.0.0', 'latest']; + + let semanticVersions = getSermanticVersions(version, publishedVersions, output); + assert.equal(semanticVersions?.toString(), expectedSemVer.toString()); + + // Publish new major version + version = '2.0.0'; + publishedVersions = ['1', '1.0', '1.0.0', 'latest']; + expectedSemVer = ['2', '2.0', '2.0.0', 'latest']; + + semanticVersions = getSermanticVersions(version, publishedVersions, output); + assert.equal(semanticVersions?.toString(), expectedSemVer.toString()); + + // Publish hotfix version + version = '1.0.1'; + publishedVersions = ['1', '1.0', '1.0.0', '2', '2.0', '2.0.0', 'latest']; + expectedSemVer = ['1', '1.0', '1.0.1']; + + semanticVersions = getSermanticVersions(version, publishedVersions, output); + assert.equal(semanticVersions?.toString(), expectedSemVer.toString()); + + // Re-publish version + version = '1.0.1'; + publishedVersions = ['1', '1.0', '1.0.0', '1.0.1', '2', '2.0', '2.0.0', 'latest']; + + semanticVersions = getSermanticVersions(version, publishedVersions, output); + assert.isUndefined(semanticVersions); + }); + + after(async () => { + await dispose(); + }); +}); From 5e71923b308ed20869196c52fbeb9ded0a0756fe Mon Sep 17 00:00:00 2001 From: Samruddhi Khandale Date: Tue, 9 Aug 2022 18:16:35 +0000 Subject: [PATCH 16/30] resolve merge conflicts --- src/spec-node/devContainersSpecCLI.ts | 2 +- src/spec-node/featuresCLI/testCommandImpl.ts | 632 +++++++++--------- .../src/featureA/devcontainer-feature.json | 15 - .../src/featureA/install.sh | 3 - .../src/featureB/devcontainer-feature.json | 15 - .../src/featureB/install.sh | 3 - .../src/featureC/devcontainer-feature.json | 15 - .../src/featureC/install.sh | 3 - .../src/featureC/other-file.md | 1 - 9 files changed, 317 insertions(+), 372 deletions(-) delete mode 100644 src/test/featuresCLICommands/example-source-repo/src/featureA/devcontainer-feature.json delete mode 100644 src/test/featuresCLICommands/example-source-repo/src/featureA/install.sh delete mode 100644 src/test/featuresCLICommands/example-source-repo/src/featureB/devcontainer-feature.json delete mode 100644 src/test/featuresCLICommands/example-source-repo/src/featureB/install.sh delete mode 100644 src/test/featuresCLICommands/example-source-repo/src/featureC/devcontainer-feature.json delete mode 100644 src/test/featuresCLICommands/example-source-repo/src/featureC/install.sh delete mode 100644 src/test/featuresCLICommands/example-source-repo/src/featureC/other-file.md diff --git a/src/spec-node/devContainersSpecCLI.ts b/src/spec-node/devContainersSpecCLI.ts index f887da470..7ed46b1d0 100644 --- a/src/spec-node/devContainersSpecCLI.ts +++ b/src/spec-node/devContainersSpecCLI.ts @@ -54,7 +54,7 @@ const defaultDefaultUserEnvProbe: UserEnvProbe = 'loginInteractiveShell'; y.command('read-configuration', 'Read configuration', readConfigurationOptions, readConfigurationHandler); y.command('features', 'Features commands', (y: Argv) => { y.command('test', 'Test features', featuresTestOptions, featuresTestHandler); - y.command('package', 'Package features', featuresPackageOptions, featuresPackageHandler); + y.command('package ', 'Package features', featuresPackageOptions, featuresPackageHandler); y.command('publish', 'Publish features', featuresPublishOptions, featuresPublishHandler); }); y.command(restArgs ? ['exec', '*'] : ['exec [args..]'], 'Execute a command on a running dev container', execOptions, execHandler); diff --git a/src/spec-node/featuresCLI/testCommandImpl.ts b/src/spec-node/featuresCLI/testCommandImpl.ts index 68b9e6fa4..d8d2795b8 100644 --- a/src/spec-node/featuresCLI/testCommandImpl.ts +++ b/src/spec-node/featuresCLI/testCommandImpl.ts @@ -12,393 +12,393 @@ import { FeaturesTestCommandInput } from './test'; const TEST_LIBRARY_SCRIPT_NAME = 'dev-container-features-test-lib'; function fail(msg: string) { - log(msg, { prefix: '[-]', stderr: true }); - process.exit(1); + log(msg, { prefix: '[-]', stderr: true }); + process.exit(1); } type Scenarios = { [key: string]: DevContainerConfig }; function log(msg: string, options?: { prefix?: string; info?: boolean; stderr?: boolean }) { - const prefix = options?.prefix || '> '; - const output = `${prefix} ${msg}\n`; + const prefix = options?.prefix || '> '; + const output = `${prefix} ${msg}\n`; - if (options?.stderr) { - process.stderr.write(chalk.red(output)); - } else if (options?.info) { - process.stdout.write(chalk.bold.blue(output)); - } else { - process.stdout.write(chalk.blue(output)); - } + if (options?.stderr) { + process.stderr.write(chalk.red(output)); + } else if (options?.info) { + process.stdout.write(chalk.bold.blue(output)); + } else { + process.stdout.write(chalk.blue(output)); + } } function printFailedTest(feature: string) { - log(`TEST FAILED: ${feature}`, { prefix: '[-]', stderr: true }); + log(`TEST FAILED: ${feature}`, { prefix: '[-]', stderr: true }); } export async function doFeaturesTestCommand(args: FeaturesTestCommandInput): Promise { - const { pkg, scenariosFolder } = args; + const { pkg, scenariosFolder } = args; - process.stdout.write(` + process.stdout.write(` ┌ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ┐ | dev container 'features' | │ v${pkg.version} │ └ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ┘\n\n`); - // There are two modes. - // 1. '--features ...' - A user-provided set of features to test (we expect a parallel 'test' subfolder for each feature) - // 2. '--scenarios ...' - A JSON file codifying a set of features to test (potentially with options & with its own test script) - if (!!scenariosFolder) { - return await runScenarioFeatureTests(args); - } else { - return await runImplicitFeatureTests(args); - } + // There are two modes. + // 1. '--features ...' - A user-provided set of features to test (we expect a parallel 'test' subfolder for each feature) + // 2. '--scenarios ...' - A JSON file codifying a set of features to test (potentially with options & with its own test script) + if (!!scenariosFolder) { + return await runScenarioFeatureTests(args); + } else { + return await runImplicitFeatureTests(args); + } } async function runScenarioFeatureTests(args: FeaturesTestCommandInput): Promise { - const { scenariosFolder, cliHost, collectionFolder } = args; - - const srcDir = `${collectionFolder}/src`; - - if (! await cliHost.isFolder(srcDir)) { - fail(`Folder '${collectionFolder}' does not contain the required 'src' folder.`); - } - - if (!scenariosFolder) { - fail('Must supply a scenarios test folder via --scenarios'); - return 1; // We never reach here, we exit via fail(). - } - - log(`Scenarios: ${scenariosFolder}\n`, { prefix: '\n📊', info: true }); - const scenariosPath = path.join(scenariosFolder, 'scenarios.json'); - - if (!cliHost.isFile(scenariosPath)) { - fail(`scenarios.json not found, expected on at: ${scenariosPath}`); - return 1; // We never reach here, we exit via fail(). - } - - // Read in scenarios.json - const scenariosBuffer = await cliHost.readFile(scenariosPath); - // Parse to json - let scenarios: Scenarios = {}; - try { - scenarios = JSON.parse(scenariosBuffer.toString()); - } catch (e) { - fail(`Failed to parse scenarios.json: ${e.message}`); - return 1; // We never reach here, we exit via fail(). - } - - const testResults: { testName: string; result: boolean }[] = []; - - // For EACH scenario: Spin up a container and exec the scenario test script - for (const [scenarioName, scenarioConfig] of Object.entries(scenarios)) { - log(`Running scenario: ${scenarioName}`); - - // Check if we have a scenario test script, otherwise skip. - const scenarioTestScript = path.join(scenariosFolder, `${scenarioName}.sh`); - if (!cliHost.isFile(scenarioTestScript)) { - log(`No scenario test script found at path ${scenarioTestScript}, skipping scenario...`); - continue; - } - - // Create Container - const workspaceFolder = await generateProjectFromScenario(cliHost, collectionFolder, scenarioName, scenarioConfig); - const params = await generateDockerParams(workspaceFolder, args); - await createContainerFromWorkingDirectory(params, workspaceFolder, args); - - // Execute test script - // Move the test script into the workspaceFolder - const testScript = await cliHost.readFile(scenarioTestScript); - const remoteTestScriptName = `${scenarioName}-test-${Date.now()}.sh`; - await cliHost.writeFile(`${workspaceFolder}/${remoteTestScriptName}`, testScript); - - // Move the test library script into the workspaceFolder - await cliHost.writeFile(`${workspaceFolder}/${TEST_LIBRARY_SCRIPT_NAME}`, Buffer.from(testLibraryScript)); - - // Execute Test - testResults.push({ - testName: scenarioName, - result: await execTest(params, remoteTestScriptName, workspaceFolder) - }); - } - - // Pretty-prints results and returns a status code to indicate success or failure. - return analyzeTestResults(testResults); + const { scenariosFolder, cliHost, collectionFolder } = args; + + const srcDir = `${collectionFolder}/src`; + + if (! await cliHost.isFolder(srcDir)) { + fail(`Folder '${collectionFolder}' does not contain the required 'src' folder.`); + } + + if (!scenariosFolder) { + fail('Must supply a scenarios test folder via --scenarios'); + return 1; // We never reach here, we exit via fail(). + } + + log(`Scenarios: ${scenariosFolder}\n`, { prefix: '\n📊', info: true }); + const scenariosPath = path.join(scenariosFolder, 'scenarios.json'); + + if (!cliHost.isFile(scenariosPath)) { + fail(`scenarios.json not found, expected on at: ${scenariosPath}`); + return 1; // We never reach here, we exit via fail(). + } + + // Read in scenarios.json + const scenariosBuffer = await cliHost.readFile(scenariosPath); + // Parse to json + let scenarios: Scenarios = {}; + try { + scenarios = JSON.parse(scenariosBuffer.toString()); + } catch (e) { + fail(`Failed to parse scenarios.json: ${e.message}`); + return 1; // We never reach here, we exit via fail(). + } + + const testResults: { testName: string; result: boolean }[] = []; + + // For EACH scenario: Spin up a container and exec the scenario test script + for (const [scenarioName, scenarioConfig] of Object.entries(scenarios)) { + log(`Running scenario: ${scenarioName}`); + + // Check if we have a scenario test script, otherwise skip. + const scenarioTestScript = path.join(scenariosFolder, `${scenarioName}.sh`); + if (!cliHost.isFile(scenarioTestScript)) { + log(`No scenario test script found at path ${scenarioTestScript}, skipping scenario...`); + continue; + } + + // Create Container + const workspaceFolder = await generateProjectFromScenario(cliHost, collectionFolder, scenarioName, scenarioConfig); + const params = await generateDockerParams(workspaceFolder, args); + await createContainerFromWorkingDirectory(params, workspaceFolder, args); + + // Execute test script + // Move the test script into the workspaceFolder + const testScript = await cliHost.readFile(scenarioTestScript); + const remoteTestScriptName = `${scenarioName}-test-${Date.now()}.sh`; + await cliHost.writeFile(`${workspaceFolder}/${remoteTestScriptName}`, testScript); + + // Move the test library script into the workspaceFolder + await cliHost.writeFile(`${workspaceFolder}/${TEST_LIBRARY_SCRIPT_NAME}`, Buffer.from(testLibraryScript)); + + // Execute Test + testResults.push({ + testName: scenarioName, + result: await execTest(params, remoteTestScriptName, workspaceFolder) + }); + } + + // Pretty-prints results and returns a status code to indicate success or failure. + return analyzeTestResults(testResults); } async function runImplicitFeatureTests(args: FeaturesTestCommandInput) { - const { baseImage, collectionFolder, remoteUser, cliHost } = args; - let { features } = args; - - const srcDir = `${collectionFolder}/src`; - const testsDir = `${collectionFolder}/test`; - - if (! await cliHost.isFolder(srcDir) || ! await cliHost.isFolder(testsDir)) { - fail(`Folder '${collectionFolder}' does not contain the required 'src' and 'test' folders.`); - } - - log(`baseImage: ${baseImage}`); - log(`Target Folder: ${collectionFolder}`); - - // Parse comma separated list of features - // If a set of '--features' isn't specified, run all features with a 'test' subfolder in random order. - if (!features) { - features = await cliHost.readDir(testsDir); - if (features.length === 0) { - fail(`No features specified and no test folders found in '${testsDir}'`); - } - } - - log(`features: ${features.join(', ')}`); - - // 1. Generate temporary project with 'baseImage' and all the 'features..' - const workspaceFolder = await generateProjectFromFeatures( - cliHost, - baseImage, - collectionFolder, - features, - remoteUser - ); - - const params = await generateDockerParams(workspaceFolder, args); - await createContainerFromWorkingDirectory(params, workspaceFolder, args); - - log('Starting test(s)...\n', { prefix: '\n🏃', info: true }); - - // 3. Exec test script for each feature, in the provided order. - const testResults = []; - for (const feature of features) { - log(`Executing '${feature}' test...`, { prefix: '🧪' }); - const testScriptPath = path.join(collectionFolder, 'test', feature, 'test.sh'); - if (!(await cliHost.isFile(testScriptPath))) { - fail(`Feature ${feature} does not have a test script!`); - } - - // Move the test script into the workspaceFolder - const testScript = await cliHost.readFile(testScriptPath); - const remoteTestScriptName = `${feature}-test-${Date.now()}.sh`; - await cliHost.writeFile(`${workspaceFolder}/${remoteTestScriptName}`, testScript); - - // Move the test library script into the workspaceFolder - await cliHost.writeFile(`${workspaceFolder}/${TEST_LIBRARY_SCRIPT_NAME}`, Buffer.from(testLibraryScript)); - - // Execute Test - const result = await execTest(params, remoteTestScriptName, workspaceFolder); - testResults.push({ - testName: feature, - result, - }); - } - - // Pretty-prints results and returns a status code to indicate success or failure. - return analyzeTestResults(testResults); + const { baseImage, collectionFolder, remoteUser, cliHost } = args; + let { features } = args; + + const srcDir = `${collectionFolder}/src`; + const testsDir = `${collectionFolder}/test`; + + if (! await cliHost.isFolder(srcDir) || ! await cliHost.isFolder(testsDir)) { + fail(`Folder '${collectionFolder}' does not contain the required 'src' and 'test' folders.`); + } + + log(`baseImage: ${baseImage}`); + log(`Target Folder: ${collectionFolder}`); + + // Parse comma separated list of features + // If a set of '--features' isn't specified, run all features with a 'test' subfolder in random order. + if (!features) { + features = await cliHost.readDir(testsDir); + if (features.length === 0) { + fail(`No features specified and no test folders found in '${testsDir}'`); + } + } + + log(`features: ${features.join(', ')}`); + + // 1. Generate temporary project with 'baseImage' and all the 'features..' + const workspaceFolder = await generateProjectFromFeatures( + cliHost, + baseImage, + collectionFolder, + features, + remoteUser + ); + + const params = await generateDockerParams(workspaceFolder, args); + await createContainerFromWorkingDirectory(params, workspaceFolder, args); + + log('Starting test(s)...\n', { prefix: '\n🏃', info: true }); + + // 3. Exec test script for each feature, in the provided order. + const testResults = []; + for (const feature of features) { + log(`Executing '${feature}' test...`, { prefix: '🧪' }); + const testScriptPath = path.join(collectionFolder, 'test', feature, 'test.sh'); + if (!(await cliHost.isFile(testScriptPath))) { + fail(`Feature ${feature} does not have a test script!`); + } + + // Move the test script into the workspaceFolder + const testScript = await cliHost.readFile(testScriptPath); + const remoteTestScriptName = `${feature}-test-${Date.now()}.sh`; + await cliHost.writeFile(`${workspaceFolder}/${remoteTestScriptName}`, testScript); + + // Move the test library script into the workspaceFolder + await cliHost.writeFile(`${workspaceFolder}/${TEST_LIBRARY_SCRIPT_NAME}`, Buffer.from(testLibraryScript)); + + // Execute Test + const result = await execTest(params, remoteTestScriptName, workspaceFolder); + testResults.push({ + testName: feature, + result, + }); + } + + // Pretty-prints results and returns a status code to indicate success or failure. + return analyzeTestResults(testResults); } function analyzeTestResults(testResults: { testName: string; result: boolean }[]): number { - // 4. Print results - const allPassed = testResults.every((x) => x.result); - if (!allPassed) { - testResults.filter((x) => !x.result).forEach((x) => { - printFailedTest(x.testName); - }); - } else { - log('All tests passed!', { prefix: '\n✅', info: true }); - } - - return allPassed ? 0 : 1; + // 4. Print results + const allPassed = testResults.every((x) => x.result); + if (!allPassed) { + testResults.filter((x) => !x.result).forEach((x) => { + printFailedTest(x.testName); + }); + } else { + log('All tests passed!', { prefix: '\n✅', info: true }); + } + + return allPassed ? 0 : 1; } const devcontainerTemplate = ` { - "image": "#{IMAGE}", - "features": { - #{FEATURES} - }, - "remoteUser": "#{REMOTE_USER}" + "image": "#{IMAGE}", + "features": { + #{FEATURES} + }, + "remoteUser": "#{REMOTE_USER}" }`; async function createContainerFromWorkingDirectory(params: DockerResolverParameters, workspaceFolder: string, args: FeaturesTestCommandInput): Promise { - const { quiet, remoteUser, disposables } = args; - log(`workspaceFolder: ${workspaceFolder}`); + const { quiet, remoteUser, disposables } = args; + log(`workspaceFolder: ${workspaceFolder}`); - // 2. Use 'devcontainer-cli up' to build and start a container - log('Building test container...\n', { prefix: '\n⏳', info: true }); - const launchResult: LaunchResult | undefined = await launchProject(params, workspaceFolder, quiet, disposables); - if (!launchResult || !launchResult.containerId) { - fail('Failed to launch container'); - return; - } + // 2. Use 'devcontainer-cli up' to build and start a container + log('Building test container...\n', { prefix: '\n⏳', info: true }); + const launchResult: LaunchResult | undefined = await launchProject(params, workspaceFolder, quiet, disposables); + if (!launchResult || !launchResult.containerId) { + fail('Failed to launch container'); + return; + } - const { containerId } = launchResult; + const { containerId } = launchResult; - log(`Launched container.`, { prefix: '\n🚀', info: true }); - log(`containerId: ${containerId}`); - log(`remoteUser: ${remoteUser}`); + log(`Launched container.`, { prefix: '\n🚀', info: true }); + log(`containerId: ${containerId}`); + log(`remoteUser: ${remoteUser}`); - return launchResult; + return launchResult; } async function createTempDevcontainerFolder(cliHost: CLIHost): Promise { - const systemTmpDir = tmpdir(); - const tmpFolder = path.join(systemTmpDir, 'vsch', 'container-features-test', Date.now().toString()); - await cliHost.mkdirp(`${tmpFolder}/.devcontainer`); - return tmpFolder; + const systemTmpDir = tmpdir(); + const tmpFolder = path.join(systemTmpDir, 'vsch', 'container-features-test', Date.now().toString()); + await cliHost.mkdirp(`${tmpFolder}/.devcontainer`); + return tmpFolder; } async function generateProjectFromFeatures( - cliHost: CLIHost, - baseImage: string, - collectionsDirectory: string, - featuresToTest: string[], - remoteUser: string + cliHost: CLIHost, + baseImage: string, + collectionsDirectory: string, + featuresToTest: string[], + remoteUser: string ): Promise { - const tmpFolder = await createTempDevcontainerFolder(cliHost); + const tmpFolder = await createTempDevcontainerFolder(cliHost); - const features = featuresToTest - .map((x) => `"${collectionsDirectory}/src/${x}": "latest"`) - .join(',\n'); + const features = featuresToTest + .map((x) => `"${collectionsDirectory}/src/${x}": "latest"`) + .join(',\n'); - let template = devcontainerTemplate - .replace('#{IMAGE}', baseImage) - .replace('#{FEATURES}', features) - .replace('#{REMOTE_USER}', remoteUser); + let template = devcontainerTemplate + .replace('#{IMAGE}', baseImage) + .replace('#{FEATURES}', features) + .replace('#{REMOTE_USER}', remoteUser); - await cliHost.writeFile(`${tmpFolder}/.devcontainer/devcontainer.json`, Buffer.from(template)); + await cliHost.writeFile(`${tmpFolder}/.devcontainer/devcontainer.json`, Buffer.from(template)); - return tmpFolder; + return tmpFolder; } async function generateProjectFromScenario( - cliHost: CLIHost, - collectionsDirectory: string, - scenarioId: string, - scenarioObject: DevContainerConfig + cliHost: CLIHost, + collectionsDirectory: string, + scenarioId: string, + scenarioObject: DevContainerConfig ): Promise { - const tmpFolder = await createTempDevcontainerFolder(cliHost); + const tmpFolder = await createTempDevcontainerFolder(cliHost); - let features = scenarioObject.features; - if (!scenarioObject || !features) { - fail(`Scenario '${scenarioId}' is missing features!`); - return ''; // Exits in the 'fail()' before this line is reached. - } + let features = scenarioObject.features; + if (!scenarioObject || !features) { + fail(`Scenario '${scenarioId}' is missing features!`); + return ''; // Exits in the 'fail()' before this line is reached. + } - // Prefix the local path to the collections directory - let updatedFeatures: Record> = {}; - for (const [featureName, featureValue] of Object.entries(features)) { - updatedFeatures[`${collectionsDirectory}/src/${featureName}`] = featureValue; - } - scenarioObject.features = updatedFeatures; + // Prefix the local path to the collections directory + let updatedFeatures: Record> = {}; + for (const [featureName, featureValue] of Object.entries(features)) { + updatedFeatures[`${collectionsDirectory}/src/${featureName}`] = featureValue; + } + scenarioObject.features = updatedFeatures; - await cliHost.writeFile(`${tmpFolder}/.devcontainer/devcontainer.json`, Buffer.from(JSON.stringify(scenarioObject))); + await cliHost.writeFile(`${tmpFolder}/.devcontainer/devcontainer.json`, Buffer.from(JSON.stringify(scenarioObject))); - // tmpFolder will serve as our auto-generated 'workingFolder' - return tmpFolder; + // tmpFolder will serve as our auto-generated 'workingFolder' + return tmpFolder; } async function launchProject(params: DockerResolverParameters, workspaceFolder: string, quiet: boolean, disposables: (() => Promise | undefined)[]): Promise { - const { common } = params; - let response = {} as LaunchResult; - - const options: ProvisionOptions = { - ...staticProvisionParams, - workspaceFolder, - logLevel: common.getLogLevel(), - mountWorkspaceGitRoot: true, - idLabels: [ - `devcontainer.local_folder=${workspaceFolder}` - ], - remoteEnv: common.remoteEnv, - log: text => quiet ? null : process.stderr.write(text), - }; - - try { - if (quiet) { - // Launch container but don't await it to reduce output noise - let isResolved = false; - const p = launch(options, disposables); - p.then(function (res) { - process.stdout.write('\n'); - response = res; - isResolved = true; - }); - while (!isResolved) { - // Just so visual progress with dots - process.stdout.write('.'); - await new Promise((resolve) => setTimeout(resolve, 500)); - } - } else { - // Stream all the container setup logs. - response = await launch(options, disposables); - } - - return { - ...response, - disposables, - }; - } catch (e: any) { - fail(`Failed to launch container:\n\n${e?.message ?? 'Unknown error'}`); - return response; // `fail` exits before we return this. - } + const { common } = params; + let response = {} as LaunchResult; + + const options: ProvisionOptions = { + ...staticProvisionParams, + workspaceFolder, + logLevel: common.getLogLevel(), + mountWorkspaceGitRoot: true, + idLabels: [ + `devcontainer.local_folder=${workspaceFolder}` + ], + remoteEnv: common.remoteEnv, + log: text => quiet ? null : process.stderr.write(text), + }; + + try { + if (quiet) { + // Launch container but don't await it to reduce output noise + let isResolved = false; + const p = launch(options, disposables); + p.then(function (res) { + process.stdout.write('\n'); + response = res; + isResolved = true; + }); + while (!isResolved) { + // Just so visual progress with dots + process.stdout.write('.'); + await new Promise((resolve) => setTimeout(resolve, 500)); + } + } else { + // Stream all the container setup logs. + response = await launch(options, disposables); + } + + return { + ...response, + disposables, + }; + } catch (e: any) { + fail(`Failed to launch container:\n\n${e?.message ?? 'Unknown error'}`); + return response; // `fail` exits before we return this. + } } async function execTest(params: DockerResolverParameters, remoteTestScriptName: string, workspaceFolder: string) { - let cmd = 'chmod'; - let args = ['777', `./${remoteTestScriptName}`, `./${TEST_LIBRARY_SCRIPT_NAME}`]; - await exec(params, cmd, args, workspaceFolder); + let cmd = 'chmod'; + let args = ['777', `./${remoteTestScriptName}`, `./${TEST_LIBRARY_SCRIPT_NAME}`]; + await exec(params, cmd, args, workspaceFolder); - cmd = `./${remoteTestScriptName}`; - args = []; - return await exec(params, cmd, args, workspaceFolder); + cmd = `./${remoteTestScriptName}`; + args = []; + return await exec(params, cmd, args, workspaceFolder); } async function exec(_params: DockerResolverParameters, cmd: string, args: string[], workspaceFolder: string) { - const execArgs = { - ...staticExecParams, - 'workspace-folder': workspaceFolder, - cmd, - args, - _: [ - cmd, - ...args - ] - }; - const result = await doExec(execArgs); - return (result.outcome === 'success'); + const execArgs = { + ...staticExecParams, + 'workspace-folder': workspaceFolder, + cmd, + args, + _: [ + cmd, + ...args + ] + }; + const result = await doExec(execArgs); + return (result.outcome === 'success'); } async function generateDockerParams(workspaceFolder: string, args: FeaturesTestCommandInput): Promise { - const { logLevel, quiet, disposables } = args; - return await createDockerParams({ - workspaceFolder, - dockerPath: undefined, - dockerComposePath: undefined, - containerDataFolder: undefined, - containerSystemDataFolder: undefined, - mountWorkspaceGitRoot: false, - idLabels: [], - configFile: undefined, - overrideConfigFile: undefined, - logLevel, - logFormat: 'text', - log: text => quiet ? null : process.stderr.write(text), - terminalDimensions: undefined, - defaultUserEnvProbe: 'loginInteractiveShell', - removeExistingContainer: false, - buildNoCache: false, - expectExistingContainer: false, - postCreateEnabled: false, - skipNonBlocking: false, - prebuild: false, - persistedFolder: undefined, - additionalMounts: [], - updateRemoteUserUIDDefault: 'never', - remoteEnv: {}, - additionalCacheFroms: [], - omitLoggerHeader: true, - useBuildKit: 'auto', - buildxPlatform: undefined, - buildxPush: false, - }, disposables); + const { logLevel, quiet, disposables } = args; + return await createDockerParams({ + workspaceFolder, + dockerPath: undefined, + dockerComposePath: undefined, + containerDataFolder: undefined, + containerSystemDataFolder: undefined, + mountWorkspaceGitRoot: false, + idLabels: [], + configFile: undefined, + overrideConfigFile: undefined, + logLevel, + logFormat: 'text', + log: text => quiet ? null : process.stderr.write(text), + terminalDimensions: undefined, + defaultUserEnvProbe: 'loginInteractiveShell', + removeExistingContainer: false, + buildNoCache: false, + expectExistingContainer: false, + postCreateEnabled: false, + skipNonBlocking: false, + prebuild: false, + persistedFolder: undefined, + additionalMounts: [], + updateRemoteUserUIDDefault: 'never', + remoteEnv: {}, + additionalCacheFroms: [], + omitLoggerHeader: true, + useBuildKit: 'auto', + buildxPlatform: undefined, + buildxPush: false, + }, disposables); } diff --git a/src/test/featuresCLICommands/example-source-repo/src/featureA/devcontainer-feature.json b/src/test/featuresCLICommands/example-source-repo/src/featureA/devcontainer-feature.json deleted file mode 100644 index bfc1a97ab..000000000 --- a/src/test/featuresCLICommands/example-source-repo/src/featureA/devcontainer-feature.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "id": "featureA", - "name": "Feature A", - "description": "Installs feature A", - "options": { - "version": { - "type": "string", - "proposals": [ - "latest" - ], - "default": "latest", - "description": "Set latest version" - } - } -} \ No newline at end of file diff --git a/src/test/featuresCLICommands/example-source-repo/src/featureA/install.sh b/src/test/featuresCLICommands/example-source-repo/src/featureA/install.sh deleted file mode 100644 index a6daecc54..000000000 --- a/src/test/featuresCLICommands/example-source-repo/src/featureA/install.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/bash - -echo "featureA!" \ No newline at end of file diff --git a/src/test/featuresCLICommands/example-source-repo/src/featureB/devcontainer-feature.json b/src/test/featuresCLICommands/example-source-repo/src/featureB/devcontainer-feature.json deleted file mode 100644 index ede04022c..000000000 --- a/src/test/featuresCLICommands/example-source-repo/src/featureB/devcontainer-feature.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "id": "featureB", - "name": "Feature B", - "description": "Installs feature B", - "options": { - "version": { - "type": "string", - "proposals": [ - "latest" - ], - "default": "latest", - "description": "Set latest version" - } - } -} \ No newline at end of file diff --git a/src/test/featuresCLICommands/example-source-repo/src/featureB/install.sh b/src/test/featuresCLICommands/example-source-repo/src/featureB/install.sh deleted file mode 100644 index 303675b47..000000000 --- a/src/test/featuresCLICommands/example-source-repo/src/featureB/install.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/bash - -echo "featureB!" \ No newline at end of file diff --git a/src/test/featuresCLICommands/example-source-repo/src/featureC/devcontainer-feature.json b/src/test/featuresCLICommands/example-source-repo/src/featureC/devcontainer-feature.json deleted file mode 100644 index 792738473..000000000 --- a/src/test/featuresCLICommands/example-source-repo/src/featureC/devcontainer-feature.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "id": "featureC", - "name": "Feature C", - "description": "Installs feature C", - "options": { - "version": { - "type": "string", - "proposals": [ - "latest" - ], - "default": "latest", - "description": "Set latest version" - } - } -} \ No newline at end of file diff --git a/src/test/featuresCLICommands/example-source-repo/src/featureC/install.sh b/src/test/featuresCLICommands/example-source-repo/src/featureC/install.sh deleted file mode 100644 index d388c123f..000000000 --- a/src/test/featuresCLICommands/example-source-repo/src/featureC/install.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/bash - -echo "featureC!" \ No newline at end of file diff --git a/src/test/featuresCLICommands/example-source-repo/src/featureC/other-file.md b/src/test/featuresCLICommands/example-source-repo/src/featureC/other-file.md deleted file mode 100644 index d2aeeec3f..000000000 --- a/src/test/featuresCLICommands/example-source-repo/src/featureC/other-file.md +++ /dev/null @@ -1 +0,0 @@ -hello there \ No newline at end of file From 5428655b2dd7252ab9a532377b4019af621e4445 Mon Sep 17 00:00:00 2001 From: Samruddhi Khandale Date: Tue, 9 Aug 2022 18:19:52 +0000 Subject: [PATCH 17/30] move tests --- .../featuresCLICommands.test.ts | 63 ++++++- src/test/featuresSubCommands.test.ts | 158 ------------------ 2 files changed, 62 insertions(+), 159 deletions(-) delete mode 100644 src/test/featuresSubCommands.test.ts diff --git a/src/test/container-features/featuresCLICommands.test.ts b/src/test/container-features/featuresCLICommands.test.ts index 9c2f09cc3..60e2f8f98 100644 --- a/src/test/container-features/featuresCLICommands.test.ts +++ b/src/test/container-features/featuresCLICommands.test.ts @@ -1,6 +1,9 @@ import { assert } from 'chai'; import path from 'path'; -import { createPlainLog, LogLevel, makeLog } from '../../spec-utils/log'; +import { createLog } from '../../spec-node/devContainers'; +import { getSermanticVersions } from '../../spec-node/featuresCLI/publishCommandImpl'; +import { getPackageConfig } from '../../spec-node/utils'; +import { createPlainLog, Log, LogLevel, makeLog, mapLogLevel } from '../../spec-utils/log'; import { isLocalFile, readLocalFile } from '../../spec-utils/pfs'; import { shellExec } from '../testUtils'; export const output = makeLog(createPlainLog(text => process.stdout.write(text), () => LogLevel.Trace)); @@ -84,4 +87,62 @@ describe('CLI features subcommands', async function () { assert.strictEqual(json.features.length, 1); assert.isTrue(collectionFileExists); }); +}); + +describe('features publish subcommand', () => { + let output: Log; + const disposables: (() => Promise | undefined)[] = []; + const dispose = async () => { + await Promise.all(disposables.map(d => d())); + }; + + before(async () => { + + const extensionPath = path.join(__dirname, '..', '..'); + const pkg = await getPackageConfig(extensionPath); + + output = createLog({ + logLevel: mapLogLevel('trace'), + logFormat: 'text', + log: (str) => process.stdout.write(str), + terminalDimensions: undefined, + }, pkg, new Date(), disposables, true); + }); + + it('should generate correct semantic versions', async () => { + // First publish + let version = '1.0.0'; + let publishedVersions: string[] = []; + let expectedSemVer = ['1', '1.0', '1.0.0', 'latest']; + + let semanticVersions = getSermanticVersions(version, publishedVersions, output); + assert.equal(semanticVersions?.toString(), expectedSemVer.toString()); + + // Publish new major version + version = '2.0.0'; + publishedVersions = ['1', '1.0', '1.0.0', 'latest']; + expectedSemVer = ['2', '2.0', '2.0.0', 'latest']; + + semanticVersions = getSermanticVersions(version, publishedVersions, output); + assert.equal(semanticVersions?.toString(), expectedSemVer.toString()); + + // Publish hotfix version + version = '1.0.1'; + publishedVersions = ['1', '1.0', '1.0.0', '2', '2.0', '2.0.0', 'latest']; + expectedSemVer = ['1', '1.0', '1.0.1']; + + semanticVersions = getSermanticVersions(version, publishedVersions, output); + assert.equal(semanticVersions?.toString(), expectedSemVer.toString()); + + // Re-publish version + version = '1.0.1'; + publishedVersions = ['1', '1.0', '1.0.0', '1.0.1', '2', '2.0', '2.0.0', 'latest']; + + semanticVersions = getSermanticVersions(version, publishedVersions, output); + assert.isUndefined(semanticVersions); + }); + + after(async () => { + await dispose(); + }); }); \ No newline at end of file diff --git a/src/test/featuresSubCommands.test.ts b/src/test/featuresSubCommands.test.ts deleted file mode 100644 index 7fcc9ff6a..000000000 --- a/src/test/featuresSubCommands.test.ts +++ /dev/null @@ -1,158 +0,0 @@ -import { assert } from 'chai'; -import { before } from 'mocha'; -import tar from 'tar'; -import path from 'path'; -import { FeaturesPackageCommandInput } from '..//../src/spec-node/featuresCLI/package'; -import { DevContainerCollectionMetadata, doFeaturesPackageCommand } from '../../src/spec-node/featuresCLI/packageCommandImpl'; -import { getCLIHost } from '../spec-common/cliHost'; -import { loadNativeModule } from '..//spec-common/commonUtils'; -import { createLog } from '../spec-node/devContainers'; -import { getPackageConfig } from '../spec-node/utils'; -import { Log, mapLogLevel } from '..//spec-utils/log'; -import { isLocalFile, mkdirpLocal, readLocalFile, rmLocal } from '../spec-utils/pfs'; -import { getSermanticVersions } from '../spec-node/featuresCLI/publishCommandImpl'; - -describe('features package command', () => { - - const disposables: (() => Promise | undefined)[] = []; - const dispose = async () => { - await Promise.all(disposables.map(d => d())); - }; - - const srcFolder = `${__dirname}/featuresCLICommands/example-source-repo/src`; - console.log(srcFolder); - const outputDir = `${__dirname}/featuresCLICommands/example-source-repo/output`; - console.log(outputDir); - - // Package - before(async () => { - - await rmLocal(outputDir, { recursive: true, force: true }); - await mkdirpLocal(outputDir); - - const extensionPath = path.join(__dirname, '..', '..'); - const pkg = await getPackageConfig(extensionPath); - - const cwd = process.cwd(); - const cliHost = await getCLIHost(cwd, loadNativeModule); - const output = createLog({ - logLevel: mapLogLevel('trace'), - logFormat: 'text', - log: (str) => process.stdout.write(str), - terminalDimensions: undefined, - }, pkg, new Date(), disposables); - - const args: FeaturesPackageCommandInput = { - cliHost, - srcFolder, - outputDir, - output, - disposables, - }; - - await doFeaturesPackageCommand(args); - - }); - - after(async () => { - await rmLocal(outputDir, { recursive: true, force: true }); - await dispose(); - }); - - it('should generate tgzs', async () => { - const featureAExists = await isLocalFile(`${outputDir}/devcontainer-feature-featureA.tgz`); - const featureBExists = await isLocalFile(`${outputDir}/devcontainer-feature-featureB.tgz`); - const featureCExists = await isLocalFile(`${outputDir}/devcontainer-feature-featureC.tgz`); - assert.isTrue(featureAExists); - assert.isTrue(featureBExists); - assert.isTrue(featureCExists); - }); - - it('should have a valid collection metadata file', async () => { - const collectionMetadataFileExists = await isLocalFile(`${outputDir}/devcontainer-collection.json`); - assert.isTrue(collectionMetadataFileExists); - - const collectionMetadataFile = await readLocalFile(`${outputDir}/devcontainer-collection.json`, 'utf8'); - const collectionMetadata: DevContainerCollectionMetadata = JSON.parse(collectionMetadataFile); - assert.equal(collectionMetadata.features.length, 3); - assert.strictEqual(collectionMetadata.features.find(x => x.id === 'featureA')?.name, 'Feature A'); - }); - - it('should have non-empty tgz files', async () => { - const tgzPath = `${outputDir}/devcontainer-feature-featureC.tgz`; - - await tar.x( - { - file: tgzPath, - cwd: outputDir, - } - ); - - const installShExists = await isLocalFile(`${outputDir}/install.sh`); - assert.isTrue(installShExists); - const jsonExists = await isLocalFile(`${outputDir}/devcontainer-feature.json`); - assert.isTrue(jsonExists); - const otherfileExists = await isLocalFile(`${outputDir}/other-file.md`); - assert.isTrue(otherfileExists); - const otherFileContents = await readLocalFile(`${outputDir}/other-file.md`, 'utf8'); - assert.strictEqual(otherFileContents, 'hello there'); - }); -}); - -describe('features publish command', () => { - let output: Log; - const disposables: (() => Promise | undefined)[] = []; - const dispose = async () => { - await Promise.all(disposables.map(d => d())); - }; - - before(async () => { - - const extensionPath = path.join(__dirname, '..', '..'); - const pkg = await getPackageConfig(extensionPath); - - output = createLog({ - logLevel: mapLogLevel('trace'), - logFormat: 'text', - log: (str) => process.stdout.write(str), - terminalDimensions: undefined, - }, pkg, new Date(), disposables, true); - }); - - it('should generate correct semantic versions', async () => { - // First publish - let version = '1.0.0'; - let publishedVersions: string[] = []; - let expectedSemVer = ['1', '1.0', '1.0.0', 'latest']; - - let semanticVersions = getSermanticVersions(version, publishedVersions, output); - assert.equal(semanticVersions?.toString(), expectedSemVer.toString()); - - // Publish new major version - version = '2.0.0'; - publishedVersions = ['1', '1.0', '1.0.0', 'latest']; - expectedSemVer = ['2', '2.0', '2.0.0', 'latest']; - - semanticVersions = getSermanticVersions(version, publishedVersions, output); - assert.equal(semanticVersions?.toString(), expectedSemVer.toString()); - - // Publish hotfix version - version = '1.0.1'; - publishedVersions = ['1', '1.0', '1.0.0', '2', '2.0', '2.0.0', 'latest']; - expectedSemVer = ['1', '1.0', '1.0.1']; - - semanticVersions = getSermanticVersions(version, publishedVersions, output); - assert.equal(semanticVersions?.toString(), expectedSemVer.toString()); - - // Re-publish version - version = '1.0.1'; - publishedVersions = ['1', '1.0', '1.0.0', '1.0.1', '2', '2.0', '2.0.0', 'latest']; - - semanticVersions = getSermanticVersions(version, publishedVersions, output); - assert.isUndefined(semanticVersions); - }); - - after(async () => { - await dispose(); - }); -}); From 291e37749383eb5205d62bd4c7055012e82a392d Mon Sep 17 00:00:00 2001 From: Samruddhi Khandale Date: Wed, 10 Aug 2022 18:59:58 +0000 Subject: [PATCH 18/30] resolving conflicts --- .../containerFeaturesOCI.ts | 4 +- src/spec-node/devContainersSpecCLI.ts | 2 +- src/spec-node/featuresCLI/package.ts | 9 ++- src/spec-node/featuresCLI/publish.ts | 32 ++++----- .../featuresCLI/publishCommandImpl.ts | 68 ++++++------------- .../featuresCLICommands.test.ts | 29 ++------ 6 files changed, 50 insertions(+), 94 deletions(-) diff --git a/src/spec-configuration/containerFeaturesOCI.ts b/src/spec-configuration/containerFeaturesOCI.ts index a7b21328c..db8953461 100644 --- a/src/spec-configuration/containerFeaturesOCI.ts +++ b/src/spec-configuration/containerFeaturesOCI.ts @@ -255,13 +255,13 @@ export async function fetchRegistryAuthToken(output: Log, registry: string, reso const authReq = await request(options, output); if (!authReq) { - output.write('Failed to get registry auth token', LogLevel.Error); + output.write('(!) ERR: Failed to get registry auth token', LogLevel.Error); return undefined; } const token: string | undefined = JSON.parse(authReq.toString())?.token; if (!token) { - output.write('Failed to parse registry auth token response', LogLevel.Error); + output.write('(!) ERR: Failed to parse registry auth token response', LogLevel.Error); return undefined; } return token; diff --git a/src/spec-node/devContainersSpecCLI.ts b/src/spec-node/devContainersSpecCLI.ts index 7ed46b1d0..b12ed8cba 100644 --- a/src/spec-node/devContainersSpecCLI.ts +++ b/src/spec-node/devContainersSpecCLI.ts @@ -55,7 +55,7 @@ const defaultDefaultUserEnvProbe: UserEnvProbe = 'loginInteractiveShell'; y.command('features', 'Features commands', (y: Argv) => { y.command('test', 'Test features', featuresTestOptions, featuresTestHandler); y.command('package ', 'Package features', featuresPackageOptions, featuresPackageHandler); - y.command('publish', 'Publish features', featuresPublishOptions, featuresPublishHandler); + y.command('publish ', 'Package and publish features', featuresPublishOptions, featuresPublishHandler); }); y.command(restArgs ? ['exec', '*'] : ['exec [args..]'], 'Execute a command on a running dev container', execOptions, execHandler); y.epilog(`devcontainer@${version} ${packageFolder}`); diff --git a/src/spec-node/featuresCLI/package.ts b/src/spec-node/featuresCLI/package.ts index d67e7a3d1..93885b7be 100644 --- a/src/spec-node/featuresCLI/package.ts +++ b/src/spec-node/featuresCLI/package.ts @@ -44,12 +44,12 @@ export function featuresPackageHandler(args: FeaturesPackageArgs) { (async () => await featuresPackage(args))().catch(console.error); } -async function featuresPackage({ +export async function featuresPackage({ 'target': targetFolder, 'log-level': inputLogLevel, 'output-folder': outputDir, 'force-clean-output-folder': forceCleanOutputDir, -}: FeaturesPackageArgs) { +}: FeaturesPackageArgs, shouldExit: boolean = true) { const disposables: (() => Promise | undefined)[] = []; const dispose = async () => { await Promise.all(disposables.map(d => d())); @@ -113,5 +113,8 @@ async function featuresPackage({ const exitCode = await doFeaturesPackageCommand(args); await dispose(); - process.exit(exitCode); + + if (shouldExit) { + process.exit(exitCode); + } } \ No newline at end of file diff --git a/src/spec-node/featuresCLI/publish.ts b/src/spec-node/featuresCLI/publish.ts index 948cc7412..0eb4f07bd 100644 --- a/src/spec-node/featuresCLI/publish.ts +++ b/src/spec-node/featuresCLI/publish.ts @@ -7,16 +7,22 @@ import { createLog } from '../devContainers'; import { UnpackArgv } from '../devContainersSpecCLI'; import { FeaturesPackageArgs, featuresPackage } from './package'; import { DevContainerCollectionMetadata } from './packageCommandImpl'; -import { getPublishedVersions, getSermanticVersions } from './publishCommandImpl'; +import { doFeaturesPublishCommand, getPublishedVersions, getSermanticVersions } from './publishCommandImpl'; + +const targetPositionalDescription = ` +Package and publish features at provided [target] (default is cwd), where [target] is either: + 1. A path to the src folder of the collection with [1..n] features. + 2. A path to a single feature that contains a devcontainer-feature.json. +`; export function featuresPublishOptions(y: Argv) { return y .options({ - 'feature-collection-folder': { type: 'string', alias: 'c', default: '.', description: 'Path to folder containing source code for collection of features' }, 'registry': { type: 'string', alias: 'r', default: 'ghcr.io', description: 'Name of the OCI registry.' }, 'namespace': { type: 'string', alias: 'n', require: true, description: 'Unique indentifier for the collection of features. Example: /' }, 'log-level': { choices: ['info' as 'info', 'debug' as 'debug', 'trace' as 'trace'], default: 'info' as 'info', description: 'Log level.' } }) + .positional('target', { type: 'string', default: '.', description: targetPositionalDescription }) .check(_argv => { return true; }); @@ -29,13 +35,12 @@ export function featuresPublishHandler(args: FeaturesPublishArgs) { } async function featuresPublish({ - 'feature-collection-folder': featureCollectionFolder, + 'target': targetFolder, 'log-level': inputLogLevel, 'registry': registry, 'namespace': namespace }: FeaturesPublishArgs) { const disposables: (() => Promise | undefined)[] = []; - const dispose = async () => { await Promise.all(disposables.map(d => d())); }; @@ -53,10 +58,10 @@ async function featuresPublish({ const outputDir = '/tmp/features-output'; const packageArgs: FeaturesPackageArgs = { - 'feature-collection-folder': featureCollectionFolder, - 'force-clean-output-dir': true, + 'target': targetFolder, 'log-level': inputLogLevel, - 'output-dir': outputDir + 'output-folder': outputDir, + 'force-clean-output-folder': true, }; await featuresPackage(packageArgs, false); @@ -67,16 +72,13 @@ async function featuresPublish({ process.exit(1); } + let exitCode = 0; const metadata: DevContainerCollectionMetadata = JSON.parse(await readLocalFile(metadataOutputPath, 'utf-8')); - - // temp - const exitCode = 0; - for (const f of metadata.features) { output.write(`Processing feature: ${f.id}...`, LogLevel.Info); if (f.version === undefined) { - output.write(`(!) Version does not exist, skipping ${f.id}...`, LogLevel.Warning); + output.write(`(!) WARNING: Version does not exist, skipping ${f.id}...`, LogLevel.Warning); continue; } @@ -86,17 +88,13 @@ async function featuresPublish({ if (semanticVersions !== undefined) { output.write(`Publishing versions: ${semanticVersions.toString()}...`, LogLevel.Info); - - // TODO: CALL OCI PUSH - // exitCode = await doFeaturesPublishCommand(); - + exitCode = doFeaturesPublishCommand(); output.write(`Published feature: ${f.id}...`, LogLevel.Info); } } // Cleanup await rmLocal(outputDir, { recursive: true, force: true }); - await dispose(); process.exit(exitCode); } diff --git a/src/spec-node/featuresCLI/publishCommandImpl.ts b/src/spec-node/featuresCLI/publishCommandImpl.ts index 57aab75c4..d01b751ee 100644 --- a/src/spec-node/featuresCLI/publishCommandImpl.ts +++ b/src/spec-node/featuresCLI/publishCommandImpl.ts @@ -1,3 +1,4 @@ +import { fetchRegistryAuthToken, HEADERS } from '../../spec-configuration/containerFeaturesOCI'; import { request } from '../../spec-utils/httpRequest'; import { Log, LogLevel } from '../../spec-utils/log'; @@ -10,27 +11,21 @@ interface versions { } export async function getPublishedVersions(featureId: string, registry: string, namespace: string, output: Log) { - const url = `https://${registry}/v2/${namespace}/${featureId}/tags/list`; - const id = `${registry}/${namespace}/${featureId}`; - let token = ''; - try { - token = await getAuthenticationToken(registry, id); - } catch (e) { - // Publishing for the first time - if (e?.message.includes('403')) { - return []; - } + const url = `https://${registry}/v2/${namespace}/${featureId}/tags/list`; + const resource = `${registry}/${namespace}/${featureId}`; - output.write(`(!) ERR: Failed to publish feature: ${e?.message ?? ''} `, LogLevel.Error); - process.exit(1); - } + let authToken = await fetchRegistryAuthToken(output, registry, resource, process.env, 'pull'); - try { - const headers = { + if (!authToken) { + output.write(`(!) ERR: Failed to publish feature: ${resource}`, LogLevel.Error); + process.exit(1); + } + + const headers: HEADERS = { 'user-agent': 'devcontainer', - 'Authorization': token, - 'Accept': 'application/json', + 'accept': 'application/json', + 'authorization': `Bearer ${authToken}` }; const options = { @@ -44,6 +39,11 @@ export async function getPublishedVersions(featureId: string, registry: string, return publishedVersionsResponse.tags; } catch (e) { + // Publishing for the first time + if (e?.message.includes('HTTP 404: Not Found')) { + return []; + } + output.write(`(!) ERR: Failed to publish feature: ${e?.message ?? ''} `, LogLevel.Error); process.exit(1); } @@ -51,7 +51,7 @@ export async function getPublishedVersions(featureId: string, registry: string, export function getSermanticVersions(version: string, publishedVersions: string[], output: Log) { if (publishedVersions.includes(version)) { - output.write(`(!) Version ${version} already exists, skipping ${version}...`, LogLevel.Warning); + output.write(`(!) WARNING: Version ${version} already exists, skipping ${version}...`, LogLevel.Warning); return undefined; } @@ -61,7 +61,7 @@ export function getSermanticVersions(version: string, publishedVersions: string[ process.exit(1); } - // Add semantic versions ex. 1.2.3 --> [1, 1.2, 1.2.3] + // Add semantic versions eg. 1.2.3 --> [1, 1.2, 1.2.3] const parsedVersion = semver.parse(version); if (parsedVersion.major !== 0) { @@ -86,31 +86,7 @@ export function getSermanticVersions(version: string, publishedVersions: string[ return semanticVersions; } -// temp -async function getAuthenticationToken(registry: string, id: string): Promise { - if (registry === 'ghcr.io') { - const token = await getGHCRtoken(id); - return 'Bearer ' + token; - } - - return ''; -} - -// temp -export async function getGHCRtoken(id: string) { - const headers = { - 'user-agent': 'devcontainer', - }; - - const url = `https://ghcr.io/token?scope=repo:${id}:pull&service=ghcr.io`; - - const options = { - type: 'GET', - url: url, - headers: headers - }; - - const token = JSON.parse((await request(options)).toString()).token; - - return token; +// TODO: Depends on https://github.com/devcontainers/cli/pull/99 +export function doFeaturesPublishCommand() { + return 0; } diff --git a/src/test/container-features/featuresCLICommands.test.ts b/src/test/container-features/featuresCLICommands.test.ts index 60e2f8f98..76a6ef1aa 100644 --- a/src/test/container-features/featuresCLICommands.test.ts +++ b/src/test/container-features/featuresCLICommands.test.ts @@ -1,9 +1,7 @@ import { assert } from 'chai'; import path from 'path'; -import { createLog } from '../../spec-node/devContainers'; import { getSermanticVersions } from '../../spec-node/featuresCLI/publishCommandImpl'; -import { getPackageConfig } from '../../spec-node/utils'; -import { createPlainLog, Log, LogLevel, makeLog, mapLogLevel } from '../../spec-utils/log'; +import { createPlainLog, LogLevel, makeLog } from '../../spec-utils/log'; import { isLocalFile, readLocalFile } from '../../spec-utils/pfs'; import { shellExec } from '../testUtils'; export const output = makeLog(createPlainLog(text => process.stdout.write(text), () => LogLevel.Trace)); @@ -90,25 +88,6 @@ describe('CLI features subcommands', async function () { }); describe('features publish subcommand', () => { - let output: Log; - const disposables: (() => Promise | undefined)[] = []; - const dispose = async () => { - await Promise.all(disposables.map(d => d())); - }; - - before(async () => { - - const extensionPath = path.join(__dirname, '..', '..'); - const pkg = await getPackageConfig(extensionPath); - - output = createLog({ - logLevel: mapLogLevel('trace'), - logFormat: 'text', - log: (str) => process.stdout.write(str), - terminalDimensions: undefined, - }, pkg, new Date(), disposables, true); - }); - it('should generate correct semantic versions', async () => { // First publish let version = '1.0.0'; @@ -142,7 +121,7 @@ describe('features publish subcommand', () => { assert.isUndefined(semanticVersions); }); - after(async () => { - await dispose(); - }); + it('should test getPublishedVersions()', async () => { + + }); }); \ No newline at end of file From 7b8208189452d1b44546e16ffdab796a66b7d06c Mon Sep 17 00:00:00 2001 From: Samruddhi Khandale Date: Thu, 11 Aug 2022 20:18:13 +0000 Subject: [PATCH 19/30] add test --- .../containerFeaturesOCI.ts | 4 +- .../featuresCLICommands.test.ts | 64 +++++++++++-------- 2 files changed, 38 insertions(+), 30 deletions(-) diff --git a/src/spec-configuration/containerFeaturesOCI.ts b/src/spec-configuration/containerFeaturesOCI.ts index db8953461..a7b21328c 100644 --- a/src/spec-configuration/containerFeaturesOCI.ts +++ b/src/spec-configuration/containerFeaturesOCI.ts @@ -255,13 +255,13 @@ export async function fetchRegistryAuthToken(output: Log, registry: string, reso const authReq = await request(options, output); if (!authReq) { - output.write('(!) ERR: Failed to get registry auth token', LogLevel.Error); + output.write('Failed to get registry auth token', LogLevel.Error); return undefined; } const token: string | undefined = JSON.parse(authReq.toString())?.token; if (!token) { - output.write('(!) ERR: Failed to parse registry auth token response', LogLevel.Error); + output.write('Failed to parse registry auth token response', LogLevel.Error); return undefined; } return token; diff --git a/src/test/container-features/featuresCLICommands.test.ts b/src/test/container-features/featuresCLICommands.test.ts index 76a6ef1aa..3c3263e76 100644 --- a/src/test/container-features/featuresCLICommands.test.ts +++ b/src/test/container-features/featuresCLICommands.test.ts @@ -1,6 +1,6 @@ import { assert } from 'chai'; import path from 'path'; -import { getSermanticVersions } from '../../spec-node/featuresCLI/publishCommandImpl'; +import { getPublishedVersions, getSermanticVersions } from '../../spec-node/featuresCLI/publishCommandImpl'; import { createPlainLog, LogLevel, makeLog } from '../../spec-utils/log'; import { isLocalFile, readLocalFile } from '../../spec-utils/pfs'; import { shellExec } from '../testUtils'; @@ -88,40 +88,48 @@ describe('CLI features subcommands', async function () { }); describe('features publish subcommand', () => { - it('should generate correct semantic versions', async () => { - // First publish - let version = '1.0.0'; - let publishedVersions: string[] = []; - let expectedSemVer = ['1', '1.0', '1.0.0', 'latest']; + it('should generate correct semantic versions', async () => { + // First publish + let version = '1.0.0'; + let publishedVersions: string[] = []; + let expectedSemVer = ['1', '1.0', '1.0.0', 'latest']; - let semanticVersions = getSermanticVersions(version, publishedVersions, output); - assert.equal(semanticVersions?.toString(), expectedSemVer.toString()); + let semanticVersions = getSermanticVersions(version, publishedVersions, output); + assert.equal(semanticVersions?.toString(), expectedSemVer.toString()); - // Publish new major version - version = '2.0.0'; - publishedVersions = ['1', '1.0', '1.0.0', 'latest']; - expectedSemVer = ['2', '2.0', '2.0.0', 'latest']; + // Publish new major version + version = '2.0.0'; + publishedVersions = ['1', '1.0', '1.0.0', 'latest']; + expectedSemVer = ['2', '2.0', '2.0.0', 'latest']; - semanticVersions = getSermanticVersions(version, publishedVersions, output); - assert.equal(semanticVersions?.toString(), expectedSemVer.toString()); + semanticVersions = getSermanticVersions(version, publishedVersions, output); + assert.equal(semanticVersions?.toString(), expectedSemVer.toString()); - // Publish hotfix version - version = '1.0.1'; - publishedVersions = ['1', '1.0', '1.0.0', '2', '2.0', '2.0.0', 'latest']; - expectedSemVer = ['1', '1.0', '1.0.1']; + // Publish hotfix version + version = '1.0.1'; + publishedVersions = ['1', '1.0', '1.0.0', '2', '2.0', '2.0.0', 'latest']; + expectedSemVer = ['1', '1.0', '1.0.1']; - semanticVersions = getSermanticVersions(version, publishedVersions, output); - assert.equal(semanticVersions?.toString(), expectedSemVer.toString()); + semanticVersions = getSermanticVersions(version, publishedVersions, output); + assert.equal(semanticVersions?.toString(), expectedSemVer.toString()); - // Re-publish version - version = '1.0.1'; - publishedVersions = ['1', '1.0', '1.0.0', '1.0.1', '2', '2.0', '2.0.0', 'latest']; + // Re-publish version + version = '1.0.1'; + publishedVersions = ['1', '1.0', '1.0.0', '1.0.1', '2', '2.0', '2.0.0', 'latest']; - semanticVersions = getSermanticVersions(version, publishedVersions, output); - assert.isUndefined(semanticVersions); - }); + semanticVersions = getSermanticVersions(version, publishedVersions, output); + assert.isUndefined(semanticVersions); + }); - it('should test getPublishedVersions()', async () => { + describe('test getPublishedVersions()', async () => { + it('should list published versions', async () => { + const versionsList = await getPublishedVersions('node', 'ghcr.io', 'devcontainers/features', output); + assert.includeMembers(versionsList, ['1', '1.0', '1.0.0', 'latest']); + }); + it('should return empty list for a non-published feature', async () => { + const versionsList = await getPublishedVersions('not-available', 'test.io', 'test/features', output); + assert.isEmpty(versionsList); + }); }); -}); \ No newline at end of file +}); From 39909642e3820f3c166ed3df09a6cd502b021fa0 Mon Sep 17 00:00:00 2001 From: Samruddhi Khandale Date: Thu, 11 Aug 2022 20:24:40 +0000 Subject: [PATCH 20/30] comment user interfacing cmd --- src/spec-node/devContainersSpecCLI.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/spec-node/devContainersSpecCLI.ts b/src/spec-node/devContainersSpecCLI.ts index b12ed8cba..032a64a9a 100644 --- a/src/spec-node/devContainersSpecCLI.ts +++ b/src/spec-node/devContainersSpecCLI.ts @@ -55,7 +55,7 @@ const defaultDefaultUserEnvProbe: UserEnvProbe = 'loginInteractiveShell'; y.command('features', 'Features commands', (y: Argv) => { y.command('test', 'Test features', featuresTestOptions, featuresTestHandler); y.command('package ', 'Package features', featuresPackageOptions, featuresPackageHandler); - y.command('publish ', 'Package and publish features', featuresPublishOptions, featuresPublishHandler); + // y.command('publish ', 'Package and publish features', featuresPublishOptions, featuresPublishHandler); }); y.command(restArgs ? ['exec', '*'] : ['exec [args..]'], 'Execute a command on a running dev container', execOptions, execHandler); y.epilog(`devcontainer@${version} ${packageFolder}`); From c53ff92ec1cfa16fd2bbf2777b9cbaa6463521a2 Mon Sep 17 00:00:00 2001 From: Samruddhi Khandale Date: Thu, 11 Aug 2022 20:34:06 +0000 Subject: [PATCH 21/30] nit --- src/spec-node/devContainersSpecCLI.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/spec-node/devContainersSpecCLI.ts b/src/spec-node/devContainersSpecCLI.ts index 032a64a9a..b7df36203 100644 --- a/src/spec-node/devContainersSpecCLI.ts +++ b/src/spec-node/devContainersSpecCLI.ts @@ -26,7 +26,7 @@ import { loadNativeModule } from '../spec-common/commonUtils'; import { generateFeaturesConfig, getContainerFeaturesFolder } from '../spec-configuration/containerFeaturesConfiguration'; import { featuresTestOptions, featuresTestHandler } from './featuresCLI/test'; import { featuresPackageHandler, featuresPackageOptions } from './featuresCLI/package'; -import { featuresPublishHandler, featuresPublishOptions } from './featuresCLI/publish'; +// import { featuresPublishHandler, featuresPublishOptions } from './featuresCLI/publish'; const defaultDefaultUserEnvProbe: UserEnvProbe = 'loginInteractiveShell'; From bb98a920304de881f2e01c036f44262c553c534b Mon Sep 17 00:00:00 2001 From: Samruddhi Khandale Date: Fri, 12 Aug 2022 21:16:12 +0000 Subject: [PATCH 22/30] wire in OCI push + fix semver bug --- package.json | 2 - src/spec-node/devContainersSpecCLI.ts | 4 +- .../featuresCLI/packageCommandImpl.ts | 10 +-- src/spec-node/featuresCLI/publish.ts | 22 +++++- .../featuresCLI/publishCommandImpl.ts | 43 +++++------ .../featuresCLICommands.test.ts | 72 ++++++++++++++----- yarn.lock | 10 --- 7 files changed, 97 insertions(+), 66 deletions(-) diff --git a/package.json b/package.json index 0272627cb..8c1a54b30 100644 --- a/package.json +++ b/package.json @@ -43,7 +43,6 @@ "@types/ncp": "^2.0.5", "@types/pull-stream": "^3.6.2", "@types/semver": "^7.3.9", - "@types/semver-compare": "^1.0.1", "@types/shell-quote": "^1.7.1", "@types/tar": "^6.1.1", "@types/yargs": "^17.0.8", @@ -74,7 +73,6 @@ "node-pty": "^0.10.1", "pull-stream": "^3.6.14", "semver": "^7.3.5", - "semver-compare": "1.0.0", "shell-quote": "^1.7.3", "stream-to-pull-stream": "^1.7.3", "tar": "^6.1.11", diff --git a/src/spec-node/devContainersSpecCLI.ts b/src/spec-node/devContainersSpecCLI.ts index cb859ce6b..94eb0921e 100644 --- a/src/spec-node/devContainersSpecCLI.ts +++ b/src/spec-node/devContainersSpecCLI.ts @@ -26,7 +26,7 @@ import { loadNativeModule } from '../spec-common/commonUtils'; import { generateFeaturesConfig, getContainerFeaturesFolder } from '../spec-configuration/containerFeaturesConfiguration'; import { featuresTestOptions, featuresTestHandler } from './featuresCLI/test'; import { featuresPackageHandler, featuresPackageOptions } from './featuresCLI/package'; -// import { featuresPublishHandler, featuresPublishOptions } from './featuresCLI/publish'; +import { featuresPublishHandler, featuresPublishOptions } from './featuresCLI/publish'; const defaultDefaultUserEnvProbe: UserEnvProbe = 'loginInteractiveShell'; @@ -55,7 +55,7 @@ const defaultDefaultUserEnvProbe: UserEnvProbe = 'loginInteractiveShell'; y.command('features', 'Features commands', (y: Argv) => { y.command('test', 'Test features', featuresTestOptions, featuresTestHandler); y.command('package ', 'Package features', featuresPackageOptions, featuresPackageHandler); - // y.command('publish ', 'Package and publish features', featuresPublishOptions, featuresPublishHandler); + y.command('publish ', 'Package and publish features', featuresPublishOptions, featuresPublishHandler); }); y.command(restArgs ? ['exec', '*'] : ['exec [args..]'], 'Execute a command on a running dev container', execOptions, execHandler); y.epilog(`devcontainer@${version} ${packageFolder}`); diff --git a/src/spec-node/featuresCLI/packageCommandImpl.ts b/src/spec-node/featuresCLI/packageCommandImpl.ts index 08b9d6977..685373974 100644 --- a/src/spec-node/featuresCLI/packageCommandImpl.ts +++ b/src/spec-node/featuresCLI/packageCommandImpl.ts @@ -17,6 +17,8 @@ export interface DevContainerCollectionMetadata { features: Feature[]; } +export const OCIFeatureCollectionFileName = 'devcontainer-collection.json'; + export async function doFeaturesPackageCommand(args: FeaturesPackageCommandInput): Promise { const { output, isSingleFeature } = args; @@ -44,7 +46,7 @@ export async function doFeaturesPackageCommand(args: FeaturesPackageCommandInput }; // Write the metadata to a file - const metadataOutputPath = path.join(args.outputDir, 'devcontainer-collection.json'); + const metadataOutputPath = path.join(args.outputDir, OCIFeatureCollectionFileName); await writeLocalFile(metadataOutputPath, JSON.stringify(collection, null, 4)); return 0; } @@ -53,7 +55,7 @@ async function tarDirectory(featureFolder: string, archiveName: string, outputDi return new Promise((resolve) => resolve(tar.create({ file: path.join(outputDir, archiveName), cwd: featureFolder }, ['.']))); } -const getArchiveName = (f: string) => `devcontainer-feature-${f}.tgz`; +export const getFeatureArchiveName = (f: string) => `devcontainer-feature-${f}.tgz`; export async function packageSingleFeature(args: FeaturesPackageCommandInput): Promise { const { output, targetFolder, outputDir } = args; @@ -65,7 +67,7 @@ export async function packageSingleFeature(args: FeaturesPackageCommandInput): P output.write(`Feature is missing an id or version in its devcontainer-feature.json`, LogLevel.Error); return; } - const archiveName = getArchiveName(featureMetadata.id); + const archiveName = getFeatureArchiveName(featureMetadata.id); await tarDirectory(targetFolder, archiveName, outputDir); output.write(`Packaged feature '${featureMetadata.id}'`, LogLevel.Info); @@ -85,7 +87,7 @@ export async function packageCollection(args: FeaturesPackageCommandInput): Prom output.write(`Processing feature: ${f}...`, LogLevel.Info); if (!f.startsWith('.')) { const featureFolder = path.join(srcFolder, f); - const archiveName = getArchiveName(f); + const archiveName = getFeatureArchiveName(f); // Validate minimal feature folder structure const featureJsonPath = path.join(featureFolder, 'devcontainer-feature.json'); diff --git a/src/spec-node/featuresCLI/publish.ts b/src/spec-node/featuresCLI/publish.ts index 0eb4f07bd..26be1dcce 100644 --- a/src/spec-node/featuresCLI/publish.ts +++ b/src/spec-node/featuresCLI/publish.ts @@ -6,8 +6,10 @@ import { getPackageConfig } from '../../spec-utils/product'; import { createLog } from '../devContainers'; import { UnpackArgv } from '../devContainersSpecCLI'; import { FeaturesPackageArgs, featuresPackage } from './package'; -import { DevContainerCollectionMetadata } from './packageCommandImpl'; -import { doFeaturesPublishCommand, getPublishedVersions, getSermanticVersions } from './publishCommandImpl'; +import { DevContainerCollectionMetadata, getFeatureArchiveName, OCIFeatureCollectionFileName } from './packageCommandImpl'; +import { getPublishedVersions, getSermanticVersions } from './publishCommandImpl'; +import { pushFeatureCollectionMetadata, pushOCIFeature } from '../../spec-configuration/containerFeaturesOCIPush'; +import { getFeatureRef, OCIFeatureCollectionRef } from '../../spec-configuration/containerFeaturesOCI'; const targetPositionalDescription = ` Package and publish features at provided [target] (default is cwd), where [target] is either: @@ -83,16 +85,30 @@ async function featuresPublish({ } output.write(`Fetching published versions...`, LogLevel.Info); + const resource = `${registry}/${namespace}/${f.id}`; + const ociFeatureRef = getFeatureRef(output, resource); const publishedVersions: string[] = await getPublishedVersions(f.id, registry, namespace, output); const semanticVersions: string[] | undefined = getSermanticVersions(f.version, publishedVersions, output); if (semanticVersions !== undefined) { output.write(`Publishing versions: ${semanticVersions.toString()}...`, LogLevel.Info); - exitCode = doFeaturesPublishCommand(); + const pathToTgz = path.join(outputDir, getFeatureArchiveName(f.id)); + await pushOCIFeature(output, ociFeatureRef, pathToTgz, semanticVersions); output.write(`Published feature: ${f.id}...`, LogLevel.Info); } } + // Publishing Feature Collection Metadata + output.write('Publishing collection metadata...', LogLevel.Info); + const featureCollectionRef: OCIFeatureCollectionRef = { + registry, + path: namespace, + version: 'latest' + }; + const pathToFeatureCollectionFile = path.join(outputDir, OCIFeatureCollectionFileName); + await pushFeatureCollectionMetadata(output, featureCollectionRef, pathToFeatureCollectionFile); + output.write('Published collection metadata...', LogLevel.Info); + // Cleanup await rmLocal(outputDir, { recursive: true, force: true }); await dispose(); diff --git a/src/spec-node/featuresCLI/publishCommandImpl.ts b/src/spec-node/featuresCLI/publishCommandImpl.ts index d01b751ee..b194ad0fa 100644 --- a/src/spec-node/featuresCLI/publishCommandImpl.ts +++ b/src/spec-node/featuresCLI/publishCommandImpl.ts @@ -2,7 +2,6 @@ import { fetchRegistryAuthToken, HEADERS } from '../../spec-configuration/contai import { request } from '../../spec-utils/httpRequest'; import { Log, LogLevel } from '../../spec-utils/log'; -const semverCompare = require('semver-compare'); const semver = require('semver'); interface versions { @@ -49,44 +48,36 @@ export async function getPublishedVersions(featureId: string, registry: string, } } +let semanticVersions: string[] = []; + +function updateSemanticVersionsList(publishedVersions: string[], version: string, range: string, publishVersion: string) { + // Reference: https://github.com/npm/node-semver#ranges-1 + const publishedMaxVersion = semver.maxSatisfying(publishedVersions, range); + if (publishedMaxVersion === null || semver.compare(version, publishedMaxVersion) === 1) { + semanticVersions.push(publishVersion); + } + return; +} + export function getSermanticVersions(version: string, publishedVersions: string[], output: Log) { if (publishedVersions.includes(version)) { output.write(`(!) WARNING: Version ${version} already exists, skipping ${version}...`, LogLevel.Warning); return undefined; } - let semanticVersions: string[] = []; if (semver.valid(version) === null) { output.write(`(!) ERR: Version ${version} is not a valid semantic version...`, LogLevel.Error); process.exit(1); } - // Add semantic versions eg. 1.2.3 --> [1, 1.2, 1.2.3] + // Adds semantic versions depending upon the existings (published) version tags + // eg. 1.2.3 --> [1, 1.2, 1.2.3, latest] const parsedVersion = semver.parse(version); - - if (parsedVersion.major !== 0) { - semanticVersions.push(parsedVersion.major); - semanticVersions.push(`${parsedVersion.major}.${parsedVersion.minor}`); - } - + semanticVersions = []; + updateSemanticVersionsList(publishedVersions, version, `${parsedVersion.major}.x.x`, parsedVersion.major); + updateSemanticVersionsList(publishedVersions, version, `${parsedVersion.major}.${parsedVersion.minor}.x`, `${parsedVersion.major}.${parsedVersion.minor}`); semanticVersions.push(version); - - let publishLatest = true; - if (publishedVersions.length > 0) { - const sortedVersions = publishedVersions.sort(semverCompare); - - // Compare version with the last published version - publishLatest = semverCompare(version, sortedVersions[sortedVersions.length - 1]) === 1 ? true : false; - } - - if (publishLatest) { - semanticVersions.push('latest'); - } + updateSemanticVersionsList(publishedVersions, version, `x.x.x`, 'latest'); return semanticVersions; } - -// TODO: Depends on https://github.com/devcontainers/cli/pull/99 -export function doFeaturesPublishCommand() { - return 0; -} diff --git a/src/test/container-features/featuresCLICommands.test.ts b/src/test/container-features/featuresCLICommands.test.ts index b30f0c444..1c6d97919 100644 --- a/src/test/container-features/featuresCLICommands.test.ts +++ b/src/test/container-features/featuresCLICommands.test.ts @@ -87,37 +87,66 @@ describe('CLI features subcommands', async function () { }); }); -describe('features publish subcommand', () => { - it('should generate correct semantic versions', async () => { - // First publish +describe('test function getSermanticVersions', () => { + it('should generate correct semantic versions for first publishing', async () => { let version = '1.0.0'; let publishedVersions: string[] = []; let expectedSemVer = ['1', '1.0', '1.0.0', 'latest']; let semanticVersions = getSermanticVersions(version, publishedVersions, output); assert.equal(semanticVersions?.toString(), expectedSemVer.toString()); + }); + + it('should generate correct semantic versions for publishing new patch version', async () => { + let version = '1.0.1'; + let publishedVersions = ['1', '1.0', '1.0.0', 'latest']; + let expectedSemVer = ['1', '1.0', '1.0.1', 'latest']; + + let semanticVersions = getSermanticVersions(version, publishedVersions, output); + assert.equal(semanticVersions?.toString(), expectedSemVer.toString()); + }); + + it('should generate correct semantic versions for publishing new minor version', async () => { + let version = '1.1.0'; + let publishedVersions = ['1', '1.0', '1.0.0', '1.0.1', 'latest']; + let expectedSemVer = ['1', '1.1', '1.1.0', 'latest']; + + let semanticVersions = getSermanticVersions(version, publishedVersions, output); + assert.equal(semanticVersions?.toString(), expectedSemVer.toString()); + }); + + it('should generate correct semantic versions for publishing new major version', async () => { + let version = '2.0.0'; + let publishedVersions = ['1', '1.0', '1.0.0', 'latest']; + let expectedSemVer = ['2', '2.0', '2.0.0', 'latest']; - // Publish new major version - version = '2.0.0'; - publishedVersions = ['1', '1.0', '1.0.0', 'latest']; - expectedSemVer = ['2', '2.0', '2.0.0', 'latest']; + let semanticVersions = getSermanticVersions(version, publishedVersions, output); + assert.equal(semanticVersions?.toString(), expectedSemVer.toString()); + }); - semanticVersions = getSermanticVersions(version, publishedVersions, output); + it('should generate correct semantic versions for publishing hotfix patch version', async () => { + let version = '1.0.2'; + let publishedVersions = ['1', '1.0', '1.0.0', '1.0.1', '1.1', '1.1.0', '2', '2.0', '2.0.0', 'latest']; + let expectedSemVer = ['1.0', '1.0.2']; + + let semanticVersions = getSermanticVersions(version, publishedVersions, output); assert.equal(semanticVersions?.toString(), expectedSemVer.toString()); + }); - // Publish hotfix version - version = '1.0.1'; - publishedVersions = ['1', '1.0', '1.0.0', '2', '2.0', '2.0.0', 'latest']; - expectedSemVer = ['1', '1.0', '1.0.1']; + it('should generate correct semantic versions for publishing hotfix minor version', async () => { + let version = '1.0.1'; + let publishedVersions = ['1', '1.0', '1.0.0', '2', '2.0', '2.0.0', 'latest']; + let expectedSemVer = ['1', '1.0', '1.0.1']; - semanticVersions = getSermanticVersions(version, publishedVersions, output); + let semanticVersions = getSermanticVersions(version, publishedVersions, output); assert.equal(semanticVersions?.toString(), expectedSemVer.toString()); + }); - // Re-publish version - version = '1.0.1'; - publishedVersions = ['1', '1.0', '1.0.0', '1.0.1', '2', '2.0', '2.0.0', 'latest']; + it('should return undefined for already published version', async () => { + let version = '1.0.1'; + let publishedVersions = ['1', '1.0', '1.0.0', '1.0.1', '2', '2.0', '2.0.0', 'latest']; - semanticVersions = getSermanticVersions(version, publishedVersions, output); + let semanticVersions = getSermanticVersions(version, publishedVersions, output); assert.isUndefined(semanticVersions); }); @@ -127,8 +156,13 @@ describe('features publish subcommand', () => { assert.includeMembers(versionsList, ['1', '1.0', '1.0.0', 'latest']); }); - it('should return empty list for a non-published feature', async () => { - const versionsList = await getPublishedVersions('not-available', 'test.io', 'test/features', output); + it('should return empty list for a non-published feature with existing resource', async () => { + const versionsList = await getPublishedVersions('not-available', 'ghcr.io', 'devcontainers/features', output); + assert.isEmpty(versionsList); + }); + + it('should return empty list for a non-published feature with non-existing resource', async () => { + const versionsList = await getPublishedVersions('not-available', 'ghcr.io', 'not/available', output); assert.isEmpty(versionsList); }); }); diff --git a/yarn.lock b/yarn.lock index 89ff610e6..de6373714 100644 --- a/yarn.lock +++ b/yarn.lock @@ -180,11 +180,6 @@ resolved "https://registry.yarnpkg.com/@types/pull-stream/-/pull-stream-3.6.2.tgz#184165017b0764b9a44aff0b555c795ad6cdf9f9" integrity sha512-s5jYmaJH68IQb9JjsemWUZCpaQdotd7B4xfXQtcKvGmQxcBXD/mvSQoi3TzPt2QqpDLjImxccS4en8f8E8O0FA== -"@types/semver-compare@^1.0.1": - version "1.0.1" - resolved "https://registry.yarnpkg.com/@types/semver-compare/-/semver-compare-1.0.1.tgz#17d1dc62c516c133ab01efb7803a537ee6eaf3d5" - integrity sha512-wx2LQVvKlEkhXp/HoKIZ/aSL+TvfJdKco8i0xJS3aR877mg4qBHzNT6+B5a61vewZHo79EdZavskGnRXEC2H6A== - "@types/semver@^7.3.9": version "7.3.9" resolved "https://registry.yarnpkg.com/@types/semver/-/semver-7.3.9.tgz#152c6c20a7688c30b967ec1841d31ace569863fc" @@ -2522,11 +2517,6 @@ safe-buffer@~5.1.0, safe-buffer@~5.1.1: resolved "https://registry.yarnpkg.com/safer-buffer/-/safer-buffer-2.1.2.tgz#44fa161b0187b9549dd84bb91802f9bd8385cd6a" integrity sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg== -semver-compare@1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/semver-compare/-/semver-compare-1.0.0.tgz#0dee216a1c941ab37e9efb1788f6afc5ff5537fc" - integrity sha512-YM3/ITh2MJ5MtzaM429anh+x2jiLVjqILF4m4oyQB18W7Ggea7BfqdH/wGMK7dDiMghv/6WG7znWMwUDzJiXow== - "semver@2 || 3 || 4 || 5", semver@^5.5.0, semver@^5.6.0: version "5.7.1" resolved "https://registry.yarnpkg.com/semver/-/semver-5.7.1.tgz#a954f931aeba508d307bbf069eff0c01c96116f7" From 91fa42c435934100d66f7c1507badbbec5dfd4b9 Mon Sep 17 00:00:00 2001 From: Samruddhi Khandale Date: Fri, 12 Aug 2022 22:41:19 +0000 Subject: [PATCH 23/30] addressing comments part 1 --- .../containerFeaturesOCI.ts | 48 +++++++++- src/spec-node/featuresCLI/package.ts | 12 ++- .../featuresCLI/packageCommandImpl.ts | 6 +- src/spec-node/featuresCLI/publish.ts | 54 ++++------- .../featuresCLI/publishCommandImpl.ts | 96 +++++++++---------- .../featuresCLICommands.test.ts | 39 ++++---- 6 files changed, 140 insertions(+), 115 deletions(-) diff --git a/src/spec-configuration/containerFeaturesOCI.ts b/src/spec-configuration/containerFeaturesOCI.ts index 1b6b4149e..dc941fdfc 100644 --- a/src/spec-configuration/containerFeaturesOCI.ts +++ b/src/spec-configuration/containerFeaturesOCI.ts @@ -50,6 +50,11 @@ export interface OCIManifest { annotations?: {}; } +interface versions { + name: string; + tags: string[]; +} + export function getOCIFeatureSet(output: Log, identifier: string, options: boolean | string | Record, manifest: OCIManifest): FeatureSet { const featureRef = getFeatureRef(output, identifier); @@ -155,7 +160,7 @@ export async function fetchOCIFeature(output: Log, env: NodeJS.ProcessEnv, featu throw new Error('FeatureSet is not an OCI featureSet.'); } - const { featureRef } = featureSet.sourceInformation; + const { featureRef } = featureSet.sourceInformation; const blobUrl = `https://${featureSet.sourceInformation.featureRef.registry}/v2/${featureSet.sourceInformation.featureRef.path}/blobs/${featureSet.sourceInformation.manifest?.layers[0].digest}`; output.write(`blob url: ${blobUrl}`, LogLevel.Trace); @@ -292,4 +297,43 @@ export async function fetchRegistryAuthToken(output: Log, registry: string, ociR return undefined; } return token; -} \ No newline at end of file +} + +// Lists published versions of a feature +export async function getPublishedVersions(featureRef: OCIFeatureRef, output: Log) { + try { + const url = `https://${featureRef.registry}/v2/${featureRef.namespace}/${featureRef.id}/tags/list`; + + let authToken = await fetchRegistryAuthToken(output, featureRef.registry, featureRef.resource, process.env, 'pull'); + + if (!authToken) { + output.write(`(!) ERR: Failed to publish feature: ${featureRef.resource}`, LogLevel.Error); + process.exit(1); + } + + const headers: HEADERS = { + 'user-agent': 'devcontainer', + 'accept': 'application/json', + 'authorization': `Bearer ${authToken}` + }; + + const options = { + type: 'GET', + url: url, + headers: headers + }; + + const response = await request(options); + const publishedVersionsResponse: versions = JSON.parse(response.toString()); + + return publishedVersionsResponse.tags; + } catch (e) { + // Publishing for the first time + if (e?.message.includes('HTTP 404: Not Found')) { + return []; + } + + output.write(`(!) ERR: Failed to publish feature: ${e?.message ?? ''} `, LogLevel.Error); + process.exit(1); + } +} diff --git a/src/spec-node/featuresCLI/package.ts b/src/spec-node/featuresCLI/package.ts index 93885b7be..060d03c91 100644 --- a/src/spec-node/featuresCLI/package.ts +++ b/src/spec-node/featuresCLI/package.ts @@ -110,11 +110,17 @@ export async function featuresPackage({ isSingleFeature, }; - const exitCode = await doFeaturesPackageCommand(args); + const collectionMetadata = await doFeaturesPackageCommand(args); await dispose(); if (shouldExit) { - process.exit(exitCode); - } + if (collectionMetadata) { + process.exit(0); + } else { + process.exit(1); + } + } + + return collectionMetadata; } \ No newline at end of file diff --git a/src/spec-node/featuresCLI/packageCommandImpl.ts b/src/spec-node/featuresCLI/packageCommandImpl.ts index 685373974..50986e128 100644 --- a/src/spec-node/featuresCLI/packageCommandImpl.ts +++ b/src/spec-node/featuresCLI/packageCommandImpl.ts @@ -19,7 +19,7 @@ export interface DevContainerCollectionMetadata { export const OCIFeatureCollectionFileName = 'devcontainer-collection.json'; -export async function doFeaturesPackageCommand(args: FeaturesPackageCommandInput): Promise { +export async function doFeaturesPackageCommand(args: FeaturesPackageCommandInput): Promise { const { output, isSingleFeature } = args; // For each feature, package each feature and write to 'outputDir/{f}.tgz' @@ -35,7 +35,7 @@ export async function doFeaturesPackageCommand(args: FeaturesPackageCommandInput if (!metadataOutput) { output.write('Failed to package features', LogLevel.Error); - return 1; + return undefined; } const collection: DevContainerCollectionMetadata = { @@ -48,7 +48,7 @@ export async function doFeaturesPackageCommand(args: FeaturesPackageCommandInput // Write the metadata to a file const metadataOutputPath = path.join(args.outputDir, OCIFeatureCollectionFileName); await writeLocalFile(metadataOutputPath, JSON.stringify(collection, null, 4)); - return 0; + return collection; } async function tarDirectory(featureFolder: string, archiveName: string, outputDir: string) { diff --git a/src/spec-node/featuresCLI/publish.ts b/src/spec-node/featuresCLI/publish.ts index 26be1dcce..63dc650bf 100644 --- a/src/spec-node/featuresCLI/publish.ts +++ b/src/spec-node/featuresCLI/publish.ts @@ -1,15 +1,15 @@ import path from 'path'; +import * as os from 'os'; import { Argv } from 'yargs'; import { LogLevel, mapLogLevel } from '../../spec-utils/log'; -import { isLocalFile, readLocalFile, rmLocal } from '../../spec-utils/pfs'; +import { rmLocal } from '../../spec-utils/pfs'; import { getPackageConfig } from '../../spec-utils/product'; import { createLog } from '../devContainers'; import { UnpackArgv } from '../devContainersSpecCLI'; import { FeaturesPackageArgs, featuresPackage } from './package'; -import { DevContainerCollectionMetadata, getFeatureArchiveName, OCIFeatureCollectionFileName } from './packageCommandImpl'; -import { getPublishedVersions, getSermanticVersions } from './publishCommandImpl'; -import { pushFeatureCollectionMetadata, pushOCIFeature } from '../../spec-configuration/containerFeaturesOCIPush'; -import { getFeatureRef, OCIFeatureCollectionRef } from '../../spec-configuration/containerFeaturesOCI'; +import { OCIFeatureCollectionFileName } from './packageCommandImpl'; +import { doFeaturesPublishCommand } from './publishCommandImpl'; +import { getFeatureRef } from '../../spec-configuration/containerFeaturesOCI'; const targetPositionalDescription = ` Package and publish features at provided [target] (default is cwd), where [target] is either: @@ -57,7 +57,7 @@ async function featuresPublish({ }, pkg, new Date(), disposables, true); // Package features - const outputDir = '/tmp/features-output'; + const outputDir = os.tmpdir(); const packageArgs: FeaturesPackageArgs = { 'target': targetFolder, @@ -66,51 +66,29 @@ async function featuresPublish({ 'force-clean-output-folder': true, }; - await featuresPackage(packageArgs, false); + const metadata = await featuresPackage(packageArgs, false); - const metadataOutputPath = path.join(outputDir, 'devcontainer-collection.json'); - if (!isLocalFile(metadataOutputPath)) { - output.write(`(!) ERR: Failed to fetch ${metadataOutputPath}`, LogLevel.Error); + if (!metadata) { + output.write(`(!) ERR: Failed to fetch ${OCIFeatureCollectionFileName}`, LogLevel.Error); process.exit(1); } let exitCode = 0; - const metadata: DevContainerCollectionMetadata = JSON.parse(await readLocalFile(metadataOutputPath, 'utf-8')); for (const f of metadata.features) { output.write(`Processing feature: ${f.id}...`, LogLevel.Info); - if (f.version === undefined) { + if (!f.version) { output.write(`(!) WARNING: Version does not exist, skipping ${f.id}...`, LogLevel.Warning); continue; } - output.write(`Fetching published versions...`, LogLevel.Info); const resource = `${registry}/${namespace}/${f.id}`; - const ociFeatureRef = getFeatureRef(output, resource); - const publishedVersions: string[] = await getPublishedVersions(f.id, registry, namespace, output); - const semanticVersions: string[] | undefined = getSermanticVersions(f.version, publishedVersions, output); + const featureRef = getFeatureRef(output, resource); + exitCode = await doFeaturesPublishCommand(f.version, featureRef, outputDir, output); - if (semanticVersions !== undefined) { - output.write(`Publishing versions: ${semanticVersions.toString()}...`, LogLevel.Info); - const pathToTgz = path.join(outputDir, getFeatureArchiveName(f.id)); - await pushOCIFeature(output, ociFeatureRef, pathToTgz, semanticVersions); - output.write(`Published feature: ${f.id}...`, LogLevel.Info); - } + // Cleanup + await rmLocal(outputDir, { recursive: true, force: true }); + await dispose(); + process.exit(exitCode); } - - // Publishing Feature Collection Metadata - output.write('Publishing collection metadata...', LogLevel.Info); - const featureCollectionRef: OCIFeatureCollectionRef = { - registry, - path: namespace, - version: 'latest' - }; - const pathToFeatureCollectionFile = path.join(outputDir, OCIFeatureCollectionFileName); - await pushFeatureCollectionMetadata(output, featureCollectionRef, pathToFeatureCollectionFile); - output.write('Published collection metadata...', LogLevel.Info); - - // Cleanup - await rmLocal(outputDir, { recursive: true, force: true }); - await dispose(); - process.exit(exitCode); } diff --git a/src/spec-node/featuresCLI/publishCommandImpl.ts b/src/spec-node/featuresCLI/publishCommandImpl.ts index b194ad0fa..91a950537 100644 --- a/src/spec-node/featuresCLI/publishCommandImpl.ts +++ b/src/spec-node/featuresCLI/publishCommandImpl.ts @@ -1,52 +1,9 @@ -import { fetchRegistryAuthToken, HEADERS } from '../../spec-configuration/containerFeaturesOCI'; -import { request } from '../../spec-utils/httpRequest'; +import path from 'path'; +import * as semver from 'semver'; +import { getPublishedVersions, OCIFeatureCollectionRef, OCIFeatureRef } from '../../spec-configuration/containerFeaturesOCI'; +import { pushFeatureCollectionMetadata, pushOCIFeature } from '../../spec-configuration/containerFeaturesOCIPush'; import { Log, LogLevel } from '../../spec-utils/log'; - -const semver = require('semver'); - -interface versions { - name: string; - tags: string[]; -} - -export async function getPublishedVersions(featureId: string, registry: string, namespace: string, output: Log) { - try { - const url = `https://${registry}/v2/${namespace}/${featureId}/tags/list`; - const resource = `${registry}/${namespace}/${featureId}`; - - let authToken = await fetchRegistryAuthToken(output, registry, resource, process.env, 'pull'); - - if (!authToken) { - output.write(`(!) ERR: Failed to publish feature: ${resource}`, LogLevel.Error); - process.exit(1); - } - - const headers: HEADERS = { - 'user-agent': 'devcontainer', - 'accept': 'application/json', - 'authorization': `Bearer ${authToken}` - }; - - const options = { - type: 'GET', - url: url, - headers: headers - }; - - const response = await request(options); - const publishedVersionsResponse: versions = JSON.parse(response.toString()); - - return publishedVersionsResponse.tags; - } catch (e) { - // Publishing for the first time - if (e?.message.includes('HTTP 404: Not Found')) { - return []; - } - - output.write(`(!) ERR: Failed to publish feature: ${e?.message ?? ''} `, LogLevel.Error); - process.exit(1); - } -} +import { getFeatureArchiveName, OCIFeatureCollectionFileName } from './packageCommandImpl'; let semanticVersions: string[] = []; @@ -65,19 +22,52 @@ export function getSermanticVersions(version: string, publishedVersions: string[ return undefined; } - if (semver.valid(version) === null) { + const parsedVersion = semver.parse(version); + if (!parsedVersion) { output.write(`(!) ERR: Version ${version} is not a valid semantic version...`, LogLevel.Error); process.exit(1); } - // Adds semantic versions depending upon the existings (published) version tags - // eg. 1.2.3 --> [1, 1.2, 1.2.3, latest] - const parsedVersion = semver.parse(version); semanticVersions = []; - updateSemanticVersionsList(publishedVersions, version, `${parsedVersion.major}.x.x`, parsedVersion.major); + + // Adds semantic versions depending upon the existings (published) versions + // eg. 1.2.3 --> [1, 1.2, 1.2.3, latest] + updateSemanticVersionsList(publishedVersions, version, `${parsedVersion.major}.x.x`, `${parsedVersion.major}`); updateSemanticVersionsList(publishedVersions, version, `${parsedVersion.major}.${parsedVersion.minor}.x`, `${parsedVersion.major}.${parsedVersion.minor}`); semanticVersions.push(version); updateSemanticVersionsList(publishedVersions, version, `x.x.x`, 'latest'); return semanticVersions; } + +export async function doFeaturesPublishCommand(version: string, featureRef: OCIFeatureRef, outputDir: string, output: Log): Promise { + output.write(`Fetching published versions...`, LogLevel.Info); + const publishedVersions: string[] = await getPublishedVersions(featureRef, output); + const semanticVersions: string[] | undefined = getSermanticVersions(version, publishedVersions, output); + + if (!!semanticVersions) { + output.write(`Publishing versions: ${semanticVersions.toString()}...`, LogLevel.Info); + const pathToTgz = path.join(outputDir, getFeatureArchiveName(featureRef.id)); + if (! await pushOCIFeature(output, featureRef, pathToTgz, semanticVersions)) { + output.write(`(!) ERR: Failed to publish feature: ${featureRef.resource}`, LogLevel.Error); + return 1; + } + output.write(`Published feature: ${featureRef.id}...`, LogLevel.Info); + } + + // Publishing Feature Collection Metadata + output.write('Publishing collection metadata...', LogLevel.Info); + const featureCollectionRef: OCIFeatureCollectionRef = { + registry: featureRef.registry, + path: featureRef.namespace, + version: 'latest' + }; + const pathToFeatureCollectionFile = path.join(outputDir, OCIFeatureCollectionFileName); + if (! await pushFeatureCollectionMetadata(output, featureCollectionRef, pathToFeatureCollectionFile)) { + output.write(`(!) ERR: Failed to publish collection metadata: ${OCIFeatureCollectionFileName}`, LogLevel.Error); + return 1; + } + output.write('Published collection metadata...', LogLevel.Info); + + return 0; +} diff --git a/src/test/container-features/featuresCLICommands.test.ts b/src/test/container-features/featuresCLICommands.test.ts index 1c6d97919..73e1cabfa 100644 --- a/src/test/container-features/featuresCLICommands.test.ts +++ b/src/test/container-features/featuresCLICommands.test.ts @@ -1,6 +1,7 @@ import { assert } from 'chai'; import path from 'path'; -import { getPublishedVersions, getSermanticVersions } from '../../spec-node/featuresCLI/publishCommandImpl'; +import { getFeatureRef, getPublishedVersions } from '../../spec-configuration/containerFeaturesOCI'; +import { getSermanticVersions } from '../../spec-node/featuresCLI/publishCommandImpl'; import { createPlainLog, LogLevel, makeLog } from '../../spec-utils/log'; import { isLocalFile, readLocalFile } from '../../spec-utils/pfs'; import { shellExec } from '../testUtils'; @@ -149,21 +150,27 @@ describe('test function getSermanticVersions', () => { let semanticVersions = getSermanticVersions(version, publishedVersions, output); assert.isUndefined(semanticVersions); }); +}); + +describe('test function getPublishedVersions', async () => { + it('should list published versions', async () => { + const resource = 'ghcr.io/devcontainers/features/node'; + const featureRef = getFeatureRef(output, resource); + const versionsList = await getPublishedVersions(featureRef, output); + assert.includeMembers(versionsList, ['1', '1.0', '1.0.0', 'latest']); + }); + + it('should return empty list for a non-published feature with existing resource', async () => { + const resource = 'ghcr.io/devcontainers/features/not-available'; + const featureRef = getFeatureRef(output, resource); + const versionsList = await getPublishedVersions(featureRef, output); + assert.isEmpty(versionsList); + }); - describe('test getPublishedVersions()', async () => { - it('should list published versions', async () => { - const versionsList = await getPublishedVersions('node', 'ghcr.io', 'devcontainers/features', output); - assert.includeMembers(versionsList, ['1', '1.0', '1.0.0', 'latest']); - }); - - it('should return empty list for a non-published feature with existing resource', async () => { - const versionsList = await getPublishedVersions('not-available', 'ghcr.io', 'devcontainers/features', output); - assert.isEmpty(versionsList); - }); - - it('should return empty list for a non-published feature with non-existing resource', async () => { - const versionsList = await getPublishedVersions('not-available', 'ghcr.io', 'not/available', output); - assert.isEmpty(versionsList); - }); + it('should return empty list for a non-published feature with non-existing resource', async () => { + const resource = 'ghcr.io/not/available/not-available'; + const featureRef = getFeatureRef(output, resource); + const versionsList = await getPublishedVersions(featureRef, output); + assert.isEmpty(versionsList); }); }); From 58a8e161d3c1bd23de9a109a2dea5df21763a4cb Mon Sep 17 00:00:00 2001 From: Samruddhi Khandale Date: Mon, 15 Aug 2022 20:43:42 +0000 Subject: [PATCH 24/30] addressing comments 2 --- .../containerFeaturesOCI.ts | 6 +- src/spec-node/featuresCLI/package.ts | 64 ++++--------------- .../featuresCLI/packageCommandImpl.ts | 57 +++++++++++++++-- src/spec-node/featuresCLI/publish.ts | 52 +++++++++------ .../featuresCLI/publishCommandImpl.ts | 27 ++++---- .../featuresCLICommands.test.ts | 2 +- 6 files changed, 115 insertions(+), 93 deletions(-) diff --git a/src/spec-configuration/containerFeaturesOCI.ts b/src/spec-configuration/containerFeaturesOCI.ts index dc941fdfc..0a3964e0d 100644 --- a/src/spec-configuration/containerFeaturesOCI.ts +++ b/src/spec-configuration/containerFeaturesOCI.ts @@ -300,7 +300,7 @@ export async function fetchRegistryAuthToken(output: Log, registry: string, ociR } // Lists published versions of a feature -export async function getPublishedVersions(featureRef: OCIFeatureRef, output: Log) { +export async function getPublishedVersions(featureRef: OCIFeatureRef, output: Log): Promise { try { const url = `https://${featureRef.registry}/v2/${featureRef.namespace}/${featureRef.id}/tags/list`; @@ -308,7 +308,7 @@ export async function getPublishedVersions(featureRef: OCIFeatureRef, output: Lo if (!authToken) { output.write(`(!) ERR: Failed to publish feature: ${featureRef.resource}`, LogLevel.Error); - process.exit(1); + return undefined; } const headers: HEADERS = { @@ -334,6 +334,6 @@ export async function getPublishedVersions(featureRef: OCIFeatureRef, output: Lo } output.write(`(!) ERR: Failed to publish feature: ${e?.message ?? ''} `, LogLevel.Error); - process.exit(1); + return undefined; } } diff --git a/src/spec-node/featuresCLI/package.ts b/src/spec-node/featuresCLI/package.ts index 060d03c91..ec249632a 100644 --- a/src/spec-node/featuresCLI/package.ts +++ b/src/spec-node/featuresCLI/package.ts @@ -2,8 +2,7 @@ import path from 'path'; import { Argv } from 'yargs'; import { CLIHost, getCLIHost } from '../../spec-common/cliHost'; import { loadNativeModule } from '../../spec-common/commonUtils'; -import { Log, LogLevel, mapLogLevel } from '../../spec-utils/log'; -import { isLocalFile, isLocalFolder, mkdirpLocal, rmLocal } from '../../spec-utils/pfs'; +import { Log, mapLogLevel } from '../../spec-utils/log'; import { createLog } from '../devContainers'; import { UnpackArgv } from '../devContainersSpecCLI'; import { getPackageConfig } from '../utils'; @@ -37,19 +36,20 @@ export interface FeaturesPackageCommandInput { outputDir: string; output: Log; disposables: (() => Promise | undefined)[]; - isSingleFeature: boolean; // Packaging a collection of many features. Should autodetect. + isSingleFeature?: boolean; // Packaging a collection of many features. Should autodetect. + forceCleanOutputDir?: boolean; } export function featuresPackageHandler(args: FeaturesPackageArgs) { (async () => await featuresPackage(args))().catch(console.error); } -export async function featuresPackage({ +async function featuresPackage({ 'target': targetFolder, 'log-level': inputLogLevel, 'output-folder': outputDir, 'force-clean-output-folder': forceCleanOutputDir, -}: FeaturesPackageArgs, shouldExit: boolean = true) { +}: FeaturesPackageArgs) { const disposables: (() => Promise | undefined)[] = []; const dispose = async () => { await Promise.all(disposables.map(d => d())); @@ -67,60 +67,18 @@ export async function featuresPackage({ terminalDimensions: undefined, }, pkg, new Date(), disposables); - const targetFolderResolved = cliHost.path.resolve(targetFolder); - if (!(await isLocalFolder(targetFolderResolved))) { - throw new Error(`Target folder '${targetFolderResolved}' does not exist`); - } - - const outputDirResolved = cliHost.path.resolve(outputDir); - if (await isLocalFolder(outputDirResolved)) { - // Output dir exists. Delete it automatically if '-f' is true - if (forceCleanOutputDir) { - await rmLocal(outputDirResolved, { recursive: true, force: true }); - } - else { - output.write(`Output directory '${outputDirResolved}' already exists. Manually delete, or pass '-f' to continue.`, LogLevel.Warning); - process.exit(1); - } - } - - // Detect if we're packaging a collection or a single feature - const isValidFolder = await isLocalFolder(cliHost.path.join(targetFolderResolved)); - const isSingleFeature = await isLocalFile(cliHost.path.join(targetFolderResolved, 'devcontainer-feature.json')); - - if (!isValidFolder) { - throw new Error(`Target folder '${targetFolderResolved}' does not exist`); - } - - if (isSingleFeature) { - output.write('Packaging single feature...', LogLevel.Info); - } else { - output.write('Packaging feature collection...', LogLevel.Info); - } - - // Generate output folder. - await mkdirpLocal(outputDirResolved); const args: FeaturesPackageCommandInput = { cliHost, - targetFolder: targetFolderResolved, - outputDir: outputDirResolved, + targetFolder, + outputDir, output, disposables, - isSingleFeature, + forceCleanOutputDir: forceCleanOutputDir }; - const collectionMetadata = await doFeaturesPackageCommand(args); + const exitCode = !!(await doFeaturesPackageCommand(args)) ? 0 : 1; await dispose(); - - if (shouldExit) { - if (collectionMetadata) { - process.exit(0); - } else { - process.exit(1); - } - } - - return collectionMetadata; -} \ No newline at end of file + process.exit(exitCode); +} diff --git a/src/spec-node/featuresCLI/packageCommandImpl.ts b/src/spec-node/featuresCLI/packageCommandImpl.ts index 50986e128..500af4705 100644 --- a/src/spec-node/featuresCLI/packageCommandImpl.ts +++ b/src/spec-node/featuresCLI/packageCommandImpl.ts @@ -2,7 +2,7 @@ import path from 'path'; import tar from 'tar'; import { Feature } from '../../spec-configuration/containerFeaturesConfiguration'; import { LogLevel } from '../../spec-utils/log'; -import { isLocalFile, readLocalDir, readLocalFile, writeLocalFile } from '../../spec-utils/pfs'; +import { isLocalFile, isLocalFolder, mkdirpLocal, readLocalDir, readLocalFile, rmLocal, writeLocalFile } from '../../spec-utils/pfs'; import { FeaturesPackageCommandInput } from './package'; export interface SourceInformation { source: string; @@ -19,8 +19,57 @@ export interface DevContainerCollectionMetadata { export const OCIFeatureCollectionFileName = 'devcontainer-collection.json'; +async function prepPackageCommand(args: FeaturesPackageCommandInput): Promise { + const { cliHost, targetFolder, outputDir, forceCleanOutputDir, output, disposables } = args; + + const targetFolderResolved = cliHost.path.resolve(targetFolder); + if (!(await isLocalFolder(targetFolderResolved))) { + throw new Error(`Target folder '${targetFolderResolved}' does not exist`); + } + + const outputDirResolved = cliHost.path.resolve(outputDir); + if (await isLocalFolder(outputDirResolved)) { + // Output dir exists. Delete it automatically if '-f' is true + if (forceCleanOutputDir) { + await rmLocal(outputDirResolved, { recursive: true, force: true }); + } + else { + output.write(`Output directory '${outputDirResolved}' already exists. Manually delete, or pass '-f' to continue.`, LogLevel.Warning); + process.exit(1); + } + } + + // Detect if we're packaging a collection or a single feature + const isValidFolder = await isLocalFolder(cliHost.path.join(targetFolderResolved)); + const isSingleFeature = await isLocalFile(cliHost.path.join(targetFolderResolved, 'devcontainer-feature.json')); + + if (!isValidFolder) { + throw new Error(`Target folder '${targetFolderResolved}' does not exist`); + } + + if (isSingleFeature) { + output.write('Packaging single feature...', LogLevel.Info); + } else { + output.write('Packaging feature collection...', LogLevel.Info); + } + + // Generate output folder. + await mkdirpLocal(outputDirResolved); + + return { + cliHost, + targetFolder: targetFolderResolved, + outputDir: outputDirResolved, + forceCleanOutputDir, + output, + disposables, + isSingleFeature: isSingleFeature + }; +} + export async function doFeaturesPackageCommand(args: FeaturesPackageCommandInput): Promise { - const { output, isSingleFeature } = args; + args = await prepPackageCommand(args); + const { output, isSingleFeature, outputDir } = args; // For each feature, package each feature and write to 'outputDir/{f}.tgz' // Returns an array of feature metadata from each processed feature @@ -46,7 +95,7 @@ export async function doFeaturesPackageCommand(args: FeaturesPackageCommandInput }; // Write the metadata to a file - const metadataOutputPath = path.join(args.outputDir, OCIFeatureCollectionFileName); + const metadataOutputPath = path.join(outputDir, OCIFeatureCollectionFileName); await writeLocalFile(metadataOutputPath, JSON.stringify(collection, null, 4)); return collection; } @@ -118,4 +167,4 @@ export async function packageCollection(args: FeaturesPackageCommandInput): Prom output.write(`Packaged ${metadatas.length} features!`, LogLevel.Info); return metadatas; -} \ No newline at end of file +} diff --git a/src/spec-node/featuresCLI/publish.ts b/src/spec-node/featuresCLI/publish.ts index 63dc650bf..2690aec65 100644 --- a/src/spec-node/featuresCLI/publish.ts +++ b/src/spec-node/featuresCLI/publish.ts @@ -6,10 +6,12 @@ import { rmLocal } from '../../spec-utils/pfs'; import { getPackageConfig } from '../../spec-utils/product'; import { createLog } from '../devContainers'; import { UnpackArgv } from '../devContainersSpecCLI'; -import { FeaturesPackageArgs, featuresPackage } from './package'; -import { OCIFeatureCollectionFileName } from './packageCommandImpl'; -import { doFeaturesPublishCommand } from './publishCommandImpl'; -import { getFeatureRef } from '../../spec-configuration/containerFeaturesOCI'; +import { FeaturesPackageCommandInput } from './package'; +import { OCIFeatureCollectionFileName, doFeaturesPackageCommand } from './packageCommandImpl'; +import { doFeaturesPublishCommand, doFeaturesPublishMetadata } from './publishCommandImpl'; +import { getFeatureRef, OCIFeatureCollectionRef } from '../../spec-configuration/containerFeaturesOCI'; +import { getCLIHost } from '../../spec-common/cliHost'; +import { loadNativeModule } from '../../spec-common/commonUtils'; const targetPositionalDescription = ` Package and publish features at provided [target] (default is cwd), where [target] is either: @@ -49,31 +51,35 @@ async function featuresPublish({ const extensionPath = path.join(__dirname, '..', '..', '..'); const pkg = await getPackageConfig(extensionPath); + + const cwd = process.cwd(); + const cliHost = await getCLIHost(cwd, loadNativeModule); const output = createLog({ logLevel: mapLogLevel(inputLogLevel), logFormat: 'text', log: (str) => process.stdout.write(str), terminalDimensions: undefined, - }, pkg, new Date(), disposables, true); + }, pkg, new Date(), disposables); // Package features - const outputDir = os.tmpdir(); + const outputDir = path.join(os.tmpdir(), '/features-output'); - const packageArgs: FeaturesPackageArgs = { - 'target': targetFolder, - 'log-level': inputLogLevel, - 'output-folder': outputDir, - 'force-clean-output-folder': true, + const packageArgs: FeaturesPackageCommandInput = { + cliHost, + targetFolder, + outputDir, + output, + disposables, + forceCleanOutputDir: true, }; - const metadata = await featuresPackage(packageArgs, false); + const metadata = await doFeaturesPackageCommand(packageArgs); if (!metadata) { output.write(`(!) ERR: Failed to fetch ${OCIFeatureCollectionFileName}`, LogLevel.Error); process.exit(1); } - let exitCode = 0; for (const f of metadata.features) { output.write(`Processing feature: ${f.id}...`, LogLevel.Info); @@ -84,11 +90,19 @@ async function featuresPublish({ const resource = `${registry}/${namespace}/${f.id}`; const featureRef = getFeatureRef(output, resource); - exitCode = await doFeaturesPublishCommand(f.version, featureRef, outputDir, output); - - // Cleanup - await rmLocal(outputDir, { recursive: true, force: true }); - await dispose(); - process.exit(exitCode); + await doFeaturesPublishCommand(f.version, featureRef, outputDir, output); } + + const featureCollectionRef: OCIFeatureCollectionRef = { + registry: registry, + path: namespace, + version: 'latest' + }; + + await doFeaturesPublishMetadata(featureCollectionRef, outputDir, output); + + // Cleanup + await rmLocal(outputDir, { recursive: true, force: true }); + await dispose(); + process.exit(); } diff --git a/src/spec-node/featuresCLI/publishCommandImpl.ts b/src/spec-node/featuresCLI/publishCommandImpl.ts index 91a950537..73d939bd9 100644 --- a/src/spec-node/featuresCLI/publishCommandImpl.ts +++ b/src/spec-node/featuresCLI/publishCommandImpl.ts @@ -24,8 +24,8 @@ export function getSermanticVersions(version: string, publishedVersions: string[ const parsedVersion = semver.parse(version); if (!parsedVersion) { - output.write(`(!) ERR: Version ${version} is not a valid semantic version...`, LogLevel.Error); - process.exit(1); + output.write(`(!) ERR: Version ${version} is not a valid semantic version, skipping ${version}...`, LogLevel.Error); + return undefined; } semanticVersions = []; @@ -40,9 +40,14 @@ export function getSermanticVersions(version: string, publishedVersions: string[ return semanticVersions; } -export async function doFeaturesPublishCommand(version: string, featureRef: OCIFeatureRef, outputDir: string, output: Log): Promise { +export async function doFeaturesPublishCommand(version: string, featureRef: OCIFeatureRef, outputDir: string, output: Log) { output.write(`Fetching published versions...`, LogLevel.Info); - const publishedVersions: string[] = await getPublishedVersions(featureRef, output); + const publishedVersions = await getPublishedVersions(featureRef, output); + + if (!publishedVersions) { + process.exit(1); + } + const semanticVersions: string[] | undefined = getSermanticVersions(version, publishedVersions, output); if (!!semanticVersions) { @@ -50,24 +55,20 @@ export async function doFeaturesPublishCommand(version: string, featureRef: OCIF const pathToTgz = path.join(outputDir, getFeatureArchiveName(featureRef.id)); if (! await pushOCIFeature(output, featureRef, pathToTgz, semanticVersions)) { output.write(`(!) ERR: Failed to publish feature: ${featureRef.resource}`, LogLevel.Error); - return 1; + process.exit(1); } output.write(`Published feature: ${featureRef.id}...`, LogLevel.Info); } +} +export async function doFeaturesPublishMetadata(featureCollectionRef: OCIFeatureCollectionRef, outputDir: string, output: Log) { // Publishing Feature Collection Metadata output.write('Publishing collection metadata...', LogLevel.Info); - const featureCollectionRef: OCIFeatureCollectionRef = { - registry: featureRef.registry, - path: featureRef.namespace, - version: 'latest' - }; + const pathToFeatureCollectionFile = path.join(outputDir, OCIFeatureCollectionFileName); if (! await pushFeatureCollectionMetadata(output, featureCollectionRef, pathToFeatureCollectionFile)) { output.write(`(!) ERR: Failed to publish collection metadata: ${OCIFeatureCollectionFileName}`, LogLevel.Error); - return 1; + process.exit(1); } output.write('Published collection metadata...', LogLevel.Info); - - return 0; } diff --git a/src/test/container-features/featuresCLICommands.test.ts b/src/test/container-features/featuresCLICommands.test.ts index 73e1cabfa..84654ad91 100644 --- a/src/test/container-features/featuresCLICommands.test.ts +++ b/src/test/container-features/featuresCLICommands.test.ts @@ -156,7 +156,7 @@ describe('test function getPublishedVersions', async () => { it('should list published versions', async () => { const resource = 'ghcr.io/devcontainers/features/node'; const featureRef = getFeatureRef(output, resource); - const versionsList = await getPublishedVersions(featureRef, output); + const versionsList = await getPublishedVersions(featureRef, output) ?? []; assert.includeMembers(versionsList, ['1', '1.0', '1.0.0', 'latest']); }); From ddff7ca0d2b13ab1651b4ca6c22723cfa51c9a4b Mon Sep 17 00:00:00 2001 From: Samruddhi Khandale Date: Mon, 15 Aug 2022 21:08:38 +0000 Subject: [PATCH 25/30] nit --- src/spec-configuration/containerFeaturesOCI.ts | 2 +- src/spec-node/featuresCLI/publishCommandImpl.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/spec-configuration/containerFeaturesOCI.ts b/src/spec-configuration/containerFeaturesOCI.ts index 0a3964e0d..a99687d3e 100644 --- a/src/spec-configuration/containerFeaturesOCI.ts +++ b/src/spec-configuration/containerFeaturesOCI.ts @@ -304,7 +304,7 @@ export async function getPublishedVersions(featureRef: OCIFeatureRef, output: Lo try { const url = `https://${featureRef.registry}/v2/${featureRef.namespace}/${featureRef.id}/tags/list`; - let authToken = await fetchRegistryAuthToken(output, featureRef.registry, featureRef.resource, process.env, 'pull'); + let authToken = await fetchRegistryAuthToken(output, featureRef.registry, featureRef.path, process.env, 'pull'); if (!authToken) { output.write(`(!) ERR: Failed to publish feature: ${featureRef.resource}`, LogLevel.Error); diff --git a/src/spec-node/featuresCLI/publishCommandImpl.ts b/src/spec-node/featuresCLI/publishCommandImpl.ts index 73d939bd9..d5205427a 100644 --- a/src/spec-node/featuresCLI/publishCommandImpl.ts +++ b/src/spec-node/featuresCLI/publishCommandImpl.ts @@ -25,7 +25,7 @@ export function getSermanticVersions(version: string, publishedVersions: string[ const parsedVersion = semver.parse(version); if (!parsedVersion) { output.write(`(!) ERR: Version ${version} is not a valid semantic version, skipping ${version}...`, LogLevel.Error); - return undefined; + process.exit(1); } semanticVersions = []; From 19853c843c7a49fbeef2f9f8af808666f6194a25 Mon Sep 17 00:00:00 2001 From: Samruddhi Khandale Date: Mon, 15 Aug 2022 21:32:21 +0000 Subject: [PATCH 26/30] add retry logic and fix test --- src/spec-configuration/containerFeaturesOCI.ts | 2 +- .../containerFeaturesOCIPush.ts | 16 ++++++++++++++-- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/src/spec-configuration/containerFeaturesOCI.ts b/src/spec-configuration/containerFeaturesOCI.ts index a99687d3e..a57f6253f 100644 --- a/src/spec-configuration/containerFeaturesOCI.ts +++ b/src/spec-configuration/containerFeaturesOCI.ts @@ -329,7 +329,7 @@ export async function getPublishedVersions(featureRef: OCIFeatureRef, output: Lo return publishedVersionsResponse.tags; } catch (e) { // Publishing for the first time - if (e?.message.includes('HTTP 404: Not Found')) { + if (e?.message.includes('HTTP 404: Not Found') || e?.message.includes('HTTP 403: Forbidden')) { return []; } diff --git a/src/spec-configuration/containerFeaturesOCIPush.ts b/src/spec-configuration/containerFeaturesOCIPush.ts index 1d739daa0..477bad196 100644 --- a/src/spec-configuration/containerFeaturesOCIPush.ts +++ b/src/spec-configuration/containerFeaturesOCIPush.ts @@ -157,7 +157,8 @@ export async function putManifestWithTags(output: Log, manifestStr: string, feat for await (const tag of tags) { const url = `https://${featureRef.registry}/v2/${featureRef.path}/manifests/${tag}`; output.write(`PUT -> '${url}'`, LogLevel.Trace); - const { statusCode, resHeaders } = await requestResolveHeaders({ + + const options = { type: 'PUT', url, headers: { @@ -165,7 +166,18 @@ export async function putManifestWithTags(output: Log, manifestStr: string, feat 'Content-Type': 'application/vnd.oci.image.manifest.v1+json', }, data: Buffer.from(manifestStr), - }); + }; + + let { statusCode, resHeaders } = await requestResolveHeaders(options); + + // Retry logic: when request fails with HTTP 429: too many requests + if (statusCode === 429) { + output.write(`Failed to PUT manifest for tag ${tag} due to too many requests. Retrying...`, LogLevel.Warning); + let response = await requestResolveHeaders(options); + statusCode = response.statusCode; + resHeaders = response.resHeaders; + } + if (statusCode !== 201) { output.write(`Failed to PUT manifest for tag ${tag}`, LogLevel.Error); return false; From 6dc2c7f807f174101d3b19583b0ab94a3b1995ec Mon Sep 17 00:00:00 2001 From: Samruddhi Khandale Date: Mon, 15 Aug 2022 21:49:01 +0000 Subject: [PATCH 27/30] nit --- src/spec-configuration/containerFeaturesOCI.ts | 2 +- .../container-features/featuresCLICommands.test.ts | 14 -------------- 2 files changed, 1 insertion(+), 15 deletions(-) diff --git a/src/spec-configuration/containerFeaturesOCI.ts b/src/spec-configuration/containerFeaturesOCI.ts index a57f6253f..a99687d3e 100644 --- a/src/spec-configuration/containerFeaturesOCI.ts +++ b/src/spec-configuration/containerFeaturesOCI.ts @@ -329,7 +329,7 @@ export async function getPublishedVersions(featureRef: OCIFeatureRef, output: Lo return publishedVersionsResponse.tags; } catch (e) { // Publishing for the first time - if (e?.message.includes('HTTP 404: Not Found') || e?.message.includes('HTTP 403: Forbidden')) { + if (e?.message.includes('HTTP 404: Not Found')) { return []; } diff --git a/src/test/container-features/featuresCLICommands.test.ts b/src/test/container-features/featuresCLICommands.test.ts index 84654ad91..45515efd8 100644 --- a/src/test/container-features/featuresCLICommands.test.ts +++ b/src/test/container-features/featuresCLICommands.test.ts @@ -159,18 +159,4 @@ describe('test function getPublishedVersions', async () => { const versionsList = await getPublishedVersions(featureRef, output) ?? []; assert.includeMembers(versionsList, ['1', '1.0', '1.0.0', 'latest']); }); - - it('should return empty list for a non-published feature with existing resource', async () => { - const resource = 'ghcr.io/devcontainers/features/not-available'; - const featureRef = getFeatureRef(output, resource); - const versionsList = await getPublishedVersions(featureRef, output); - assert.isEmpty(versionsList); - }); - - it('should return empty list for a non-published feature with non-existing resource', async () => { - const resource = 'ghcr.io/not/available/not-available'; - const featureRef = getFeatureRef(output, resource); - const versionsList = await getPublishedVersions(featureRef, output); - assert.isEmpty(versionsList); - }); }); From 5ebf1bce1a62076bd4881668e780fbecdbe7b9d0 Mon Sep 17 00:00:00 2001 From: Samruddhi Khandale Date: Tue, 16 Aug 2022 21:07:05 +0000 Subject: [PATCH 28/30] address comments --- src/spec-node/featuresCLI/packageCommandImpl.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/spec-node/featuresCLI/packageCommandImpl.ts b/src/spec-node/featuresCLI/packageCommandImpl.ts index 500af4705..124118adf 100644 --- a/src/spec-node/featuresCLI/packageCommandImpl.ts +++ b/src/spec-node/featuresCLI/packageCommandImpl.ts @@ -34,7 +34,7 @@ async function prepPackageCommand(args: FeaturesPackageCommandInput): Promise Date: Tue, 16 Aug 2022 21:07:14 +0000 Subject: [PATCH 29/30] address comments --- src/spec-configuration/containerFeaturesOCI.ts | 3 ++- src/spec-configuration/containerFeaturesOCIPush.ts | 3 +++ src/spec-configuration/tsconfig.json | 3 +++ 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/src/spec-configuration/containerFeaturesOCI.ts b/src/spec-configuration/containerFeaturesOCI.ts index a99687d3e..08bd6505f 100644 --- a/src/spec-configuration/containerFeaturesOCI.ts +++ b/src/spec-configuration/containerFeaturesOCI.ts @@ -299,7 +299,8 @@ export async function fetchRegistryAuthToken(output: Log, registry: string, ociR return token; } -// Lists published versions of a feature +// Lists published versions of a +// Specification: https://github.com/opencontainers/distribution-spec/blob/v1.0.1/spec.md#content-discovery export async function getPublishedVersions(featureRef: OCIFeatureRef, output: Log): Promise { try { const url = `https://${featureRef.registry}/v2/${featureRef.namespace}/${featureRef.id}/tags/list`; diff --git a/src/spec-configuration/containerFeaturesOCIPush.ts b/src/spec-configuration/containerFeaturesOCIPush.ts index 477bad196..b4d06bed1 100644 --- a/src/spec-configuration/containerFeaturesOCIPush.ts +++ b/src/spec-configuration/containerFeaturesOCIPush.ts @@ -1,6 +1,7 @@ import * as path from 'path'; import * as fs from 'fs'; import * as crypto from 'crypto'; +import { delay } from '../spec-common/async'; import { headRequest, requestResolveHeaders } from '../spec-utils/httpRequest'; import { Log, LogLevel } from '../spec-utils/log'; import { isLocalFile, readLocalFile } from '../spec-utils/pfs'; @@ -173,6 +174,8 @@ export async function putManifestWithTags(output: Log, manifestStr: string, feat // Retry logic: when request fails with HTTP 429: too many requests if (statusCode === 429) { output.write(`Failed to PUT manifest for tag ${tag} due to too many requests. Retrying...`, LogLevel.Warning); + await delay(2000); + let response = await requestResolveHeaders(options); statusCode = response.statusCode; resHeaders = response.resHeaders; diff --git a/src/spec-configuration/tsconfig.json b/src/spec-configuration/tsconfig.json index eff319378..f33845140 100644 --- a/src/spec-configuration/tsconfig.json +++ b/src/spec-configuration/tsconfig.json @@ -1,6 +1,9 @@ { "extends": "../../tsconfig.base.json", "references": [ + { + "path": "../spec-common" + }, { "path": "../spec-utils" } From b0be8ef0088a2fac1b88f49afbda29fc1ae60eaf Mon Sep 17 00:00:00 2001 From: Samruddhi Khandale Date: Wed, 17 Aug 2022 16:10:39 +0000 Subject: [PATCH 30/30] addressing comment: renaming interface --- src/spec-configuration/containerFeaturesOCI.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/spec-configuration/containerFeaturesOCI.ts b/src/spec-configuration/containerFeaturesOCI.ts index 08bd6505f..168942030 100644 --- a/src/spec-configuration/containerFeaturesOCI.ts +++ b/src/spec-configuration/containerFeaturesOCI.ts @@ -50,7 +50,7 @@ export interface OCIManifest { annotations?: {}; } -interface versions { +interface OCITagList { name: string; tags: string[]; } @@ -325,7 +325,7 @@ export async function getPublishedVersions(featureRef: OCIFeatureRef, output: Lo }; const response = await request(options); - const publishedVersionsResponse: versions = JSON.parse(response.toString()); + const publishedVersionsResponse: OCITagList = JSON.parse(response.toString()); return publishedVersionsResponse.tags; } catch (e) {