Introducing the interface for features publish command

src/spec-configuration/containerFeaturesOCI.ts

@@ -50,6 +50,11 @@ export interface OCIManifest {
 	annotations?: {};
 }
 
+interface OCITagList {
+	name: string;
+	tags: string[];
+}
+
 export function getOCIFeatureSet(output: Log, identifier: string, options: boolean | string | Record<string, boolean | string | undefined>, manifest: OCIManifest): FeatureSet {
 
 	const featureRef = getFeatureRef(output, identifier);
@@ -155,7 +160,7 @@ export async function fetchOCIFeature(output: Log, env: NodeJS.ProcessEnv, featu
 		throw new Error('FeatureSet is not an OCI featureSet.');
 	}
 
-	const { featureRef } = featureSet.sourceInformation; 
+	const { featureRef } = featureSet.sourceInformation;
 
 	const blobUrl = `https://${featureSet.sourceInformation.featureRef.registry}/v2/${featureSet.sourceInformation.featureRef.path}/blobs/${featureSet.sourceInformation.manifest?.layers[0].digest}`;
 	output.write(`blob url: ${blobUrl}`, LogLevel.Trace);
@@ -292,4 +297,44 @@ export async function fetchRegistryAuthToken(output: Log, registry: string, ociR
 		return undefined;
 	}
 	return token;
-}
+}
+
+// Lists published versions of a 
+// Specification: https://github.com/opencontainers/distribution-spec/blob/v1.0.1/spec.md#content-discovery
+export async function getPublishedVersions(featureRef: OCIFeatureRef, output: Log): Promise<string[] | undefined> {
+	try {
+		const url = `https://${featureRef.registry}/v2/${featureRef.namespace}/${featureRef.id}/tags/list`;
+
+		let authToken = await fetchRegistryAuthToken(output, featureRef.registry, featureRef.path, process.env, 'pull');
+
+		if (!authToken) {
+			output.write(`(!) ERR: Failed to publish feature: ${featureRef.resource}`, LogLevel.Error);
+			return undefined;
+		}
+
+		const headers: HEADERS = {
+			'user-agent': 'devcontainer',
+			'accept': 'application/json',
+			'authorization': `Bearer ${authToken}`
+		};
+
+		const options = {
+			type: 'GET',
+			url: url,
+			headers: headers
+		};
+
+		const response = await request(options);
+		const publishedVersionsResponse: OCITagList = JSON.parse(response.toString());
+
+		return publishedVersionsResponse.tags;
+	} catch (e) {
+		// Publishing for the first time
+		if (e?.message.includes('HTTP 404: Not Found')) {
+			return [];
+		}
+
+		output.write(`(!) ERR: Failed to publish feature: ${e?.message ?? ''} `, LogLevel.Error);
+		return undefined;
+	}
+}

src/spec-configuration/containerFeaturesOCIPush.ts

@@ -1,6 +1,7 @@
 import * as path from 'path';
 import * as fs from 'fs';
 import * as crypto from 'crypto';
+import { delay } from '../spec-common/async';
 import { headRequest, requestResolveHeaders } from '../spec-utils/httpRequest';
 import { Log, LogLevel } from '../spec-utils/log';
 import { isLocalFile, readLocalFile } from '../spec-utils/pfs';
@@ -157,15 +158,29 @@ export async function putManifestWithTags(output: Log, manifestStr: string, feat
 	for await (const tag of tags) {
 		const url = `https://${featureRef.registry}/v2/${featureRef.path}/manifests/${tag}`;
 		output.write(`PUT -> '${url}'`, LogLevel.Trace);
-		const { statusCode, resHeaders } = await requestResolveHeaders({
+
+		const options = {
 			type: 'PUT',
 			url,
 			headers: {
 				'Authorization': `Bearer ${registryAuthToken}`,
 				'Content-Type': 'application/vnd.oci.image.manifest.v1+json',
 			},
 			data: Buffer.from(manifestStr),
-		});
+		};
+
+		let { statusCode, resHeaders } = await requestResolveHeaders(options);
+
+		// Retry logic: when request fails with HTTP 429: too many requests
+		if (statusCode === 429) {
+			output.write(`Failed to PUT manifest for tag ${tag} due to too many requests. Retrying...`, LogLevel.Warning);
+			await delay(2000);
+
+			let response = await requestResolveHeaders(options);
+			statusCode = response.statusCode;
+			resHeaders = response.resHeaders;
+		}
+
 		if (statusCode !== 201) {
 			output.write(`Failed to PUT manifest for tag ${tag}`, LogLevel.Error);
 			return false;

src/spec-configuration/tsconfig.json

@@ -1,6 +1,9 @@
 {
 	"extends": "../../tsconfig.base.json",
 	"references": [
+		{
+			"path": "../spec-common"
+		},
 		{
 			"path": "../spec-utils"
 		}

src/spec-node/devContainersSpecCLI.ts

@@ -26,6 +26,7 @@ import { loadNativeModule } from '../spec-common/commonUtils';
 import { generateFeaturesConfig, getContainerFeaturesFolder } from '../spec-configuration/containerFeaturesConfiguration';
 import { featuresTestOptions, featuresTestHandler } from './featuresCLI/test';
 import { featuresPackageHandler, featuresPackageOptions } from './featuresCLI/package';
+import { featuresPublishHandler, featuresPublishOptions } from './featuresCLI/publish';
 
 const defaultDefaultUserEnvProbe: UserEnvProbe = 'loginInteractiveShell';
 
@@ -54,6 +55,7 @@ const defaultDefaultUserEnvProbe: UserEnvProbe = 'loginInteractiveShell';
 	y.command('features', 'Features commands', (y: Argv) => {
 		y.command('test', 'Test features', featuresTestOptions, featuresTestHandler);
 		y.command('package <target>', 'Package features', featuresPackageOptions, featuresPackageHandler);
+		y.command('publish <target>', 'Package and publish features', featuresPublishOptions, featuresPublishHandler);
 	});
 	y.command(restArgs ? ['exec', '*'] : ['exec <cmd> [args..]'], 'Execute a command on a running dev container', execOptions, execHandler);
 	y.epilog(`devcontainer@${version} ${packageFolder}`);

src/spec-node/featuresCLI/package.ts

@@ -2,8 +2,7 @@ import path from 'path';
 import { Argv } from 'yargs';
 import { CLIHost, getCLIHost } from '../../spec-common/cliHost';
 import { loadNativeModule } from '../../spec-common/commonUtils';
-import { Log, LogLevel, mapLogLevel } from '../../spec-utils/log';
-import { isLocalFile, isLocalFolder, mkdirpLocal, rmLocal } from '../../spec-utils/pfs';
+import { Log, mapLogLevel } from '../../spec-utils/log';
 import { createLog } from '../devContainers';
 import { UnpackArgv } from '../devContainersSpecCLI';
 import { getPackageConfig } from '../utils';
@@ -37,7 +36,8 @@ export interface FeaturesPackageCommandInput {
 	outputDir: string;
 	output: Log;
 	disposables: (() => Promise<unknown> | undefined)[];
-	isSingleFeature: boolean; // Packaging a collection of many features. Should autodetect.
+	isSingleFeature?: boolean; // Packaging a collection of many features. Should autodetect.
+	forceCleanOutputDir?: boolean;
 }
 
 export function featuresPackageHandler(args: FeaturesPackageArgs) {
@@ -67,51 +67,18 @@ async function featuresPackage({
 		terminalDimensions: undefined,
 	}, pkg, new Date(), disposables);
 
-	const targetFolderResolved = cliHost.path.resolve(targetFolder);
-	if (!(await isLocalFolder(targetFolderResolved))) {
-		throw new Error(`Target folder '${targetFolderResolved}' does not exist`);
-	}
-
-	const outputDirResolved = cliHost.path.resolve(outputDir);
-	if (await isLocalFolder(outputDirResolved)) {
-		// Output dir exists. Delete it automatically if '-f' is true
-		if (forceCleanOutputDir) {
-			await rmLocal(outputDirResolved, { recursive: true, force: true });
-		}
-		else {
-			output.write(`Output directory '${outputDirResolved}' already exists. Manually delete, or pass '-f' to continue.`, LogLevel.Warning);
-			process.exit(1);
-		}
-	}
-
-	// Detect if we're packaging a collection or a single feature
-	const isValidFolder = await isLocalFolder(cliHost.path.join(targetFolderResolved));
-	const isSingleFeature = await isLocalFile(cliHost.path.join(targetFolderResolved, 'devcontainer-feature.json'));
-
-	if (!isValidFolder) {
-		throw new Error(`Target folder '${targetFolderResolved}' does not exist`);
-	}
-
-	if (isSingleFeature) {
-		output.write('Packaging single feature...', LogLevel.Info);
-	} else {
-		output.write('Packaging feature collection...', LogLevel.Info);
-	}
-
-	// Generate output folder.
-	await mkdirpLocal(outputDirResolved);
 
 	const args: FeaturesPackageCommandInput = {
 		cliHost,
-		targetFolder: targetFolderResolved,
-		outputDir: outputDirResolved,
+		targetFolder,
+		outputDir,
 		output,
 		disposables,
-		isSingleFeature,
+		forceCleanOutputDir: forceCleanOutputDir
 	};
 
-	const exitCode = await doFeaturesPackageCommand(args);
+	const exitCode = !!(await doFeaturesPackageCommand(args)) ? 0 : 1;
 
 	await dispose();
 	process.exit(exitCode);
-}
+}

src/spec-node/featuresCLI/packageCommandImpl.ts

@@ -2,7 +2,7 @@ import path from 'path';
 import tar from 'tar';
 import { Feature } from '../../spec-configuration/containerFeaturesConfiguration';
 import { LogLevel } from '../../spec-utils/log';
-import { isLocalFile, readLocalDir, readLocalFile, writeLocalFile } from '../../spec-utils/pfs';
+import { isLocalFile, isLocalFolder, mkdirpLocal, readLocalDir, readLocalFile, rmLocal, writeLocalFile } from '../../spec-utils/pfs';
 import { FeaturesPackageCommandInput } from './package';
 export interface SourceInformation {
 	source: string;
@@ -17,8 +17,59 @@ export interface DevContainerCollectionMetadata {
 	features: Feature[];
 }
 
-export async function doFeaturesPackageCommand(args: FeaturesPackageCommandInput): Promise<number> {
-	const { output, isSingleFeature } = args;
+export const OCIFeatureCollectionFileName = 'devcontainer-collection.json';
+
+async function prepPackageCommand(args: FeaturesPackageCommandInput): Promise<FeaturesPackageCommandInput> {
+	const { cliHost, targetFolder, outputDir, forceCleanOutputDir, output, disposables } = args;
+
+	const targetFolderResolved = cliHost.path.resolve(targetFolder);
+	if (!(await isLocalFolder(targetFolderResolved))) {
+		throw new Error(`Target folder '${targetFolderResolved}' does not exist`);
+	}
+
+	const outputDirResolved = cliHost.path.resolve(outputDir);
+	if (await isLocalFolder(outputDirResolved)) {
+		// Output dir exists. Delete it automatically if '-f' is true
+		if (forceCleanOutputDir) {
+			await rmLocal(outputDirResolved, { recursive: true, force: true });
+		}
+		else {
+			output.write(`(!) ERR: Output directory '${outputDirResolved}' already exists. Manually delete, or pass '-f' to continue.`, LogLevel.Error);
+			process.exit(1);
+		}
+	}
+
+	// Detect if we're packaging a collection or a single feature
+	const isValidFolder = await isLocalFolder(cliHost.path.join(targetFolderResolved));
+	const isSingleFeature = await isLocalFile(cliHost.path.join(targetFolderResolved, 'devcontainer-feature.json'));
+
+	if (!isValidFolder) {
+		throw new Error(`Target folder '${targetFolderResolved}' does not exist`);
+	}
+
+	if (isSingleFeature) {
+		output.write('Packaging single feature...', LogLevel.Info);
+	} else {
+		output.write('Packaging feature collection...', LogLevel.Info);
+	}
+
+	// Generate output folder.
+	await mkdirpLocal(outputDirResolved);
+
+	return {
+		cliHost,
+		targetFolder: targetFolderResolved,
+		outputDir: outputDirResolved,
+		forceCleanOutputDir,
+		output,
+		disposables,
+		isSingleFeature: isSingleFeature
+	};
+}
+
+export async function doFeaturesPackageCommand(args: FeaturesPackageCommandInput): Promise<DevContainerCollectionMetadata | undefined> {
+	args = await prepPackageCommand(args);
+	const { output, isSingleFeature, outputDir } = args;
 
 	// For each feature, package each feature and write to 'outputDir/{f}.tgz'
 	// Returns an array of feature metadata from each processed feature
@@ -33,7 +84,7 @@ export async function doFeaturesPackageCommand(args: FeaturesPackageCommandInput
 
 	if (!metadataOutput) {
 		output.write('Failed to package features', LogLevel.Error);
-		return 1;
+		return undefined;
 	}
 
 	const collection: DevContainerCollectionMetadata = {
@@ -44,16 +95,16 @@ export async function doFeaturesPackageCommand(args: FeaturesPackageCommandInput
 	};
 
 	// Write the metadata to a file
-	const metadataOutputPath = path.join(args.outputDir, 'devcontainer-collection.json');
+	const metadataOutputPath = path.join(outputDir, OCIFeatureCollectionFileName);
 	await writeLocalFile(metadataOutputPath, JSON.stringify(collection, null, 4));
-	return 0;
+	return collection;
 }
 
 async function tarDirectory(featureFolder: string, archiveName: string, outputDir: string) {
 	return new Promise<void>((resolve) => resolve(tar.create({ file: path.join(outputDir, archiveName), cwd: featureFolder }, ['.'])));
 }
 
-const getArchiveName = (f: string) => `devcontainer-feature-${f}.tgz`;
+export const getFeatureArchiveName = (f: string) => `devcontainer-feature-${f}.tgz`;
 
 export async function packageSingleFeature(args: FeaturesPackageCommandInput): Promise<Feature[] | undefined> {
 	const { output, targetFolder, outputDir } = args;
@@ -65,7 +116,7 @@ export async function packageSingleFeature(args: FeaturesPackageCommandInput): P
 		output.write(`Feature is missing an id or version in its devcontainer-feature.json`, LogLevel.Error);
 		return;
 	}
-	const archiveName = getArchiveName(featureMetadata.id);
+	const archiveName = getFeatureArchiveName(featureMetadata.id);
 
 	await tarDirectory(targetFolder, archiveName, outputDir);
 	output.write(`Packaged feature '${featureMetadata.id}'`, LogLevel.Info);
@@ -85,7 +136,7 @@ export async function packageCollection(args: FeaturesPackageCommandInput): Prom
 		output.write(`Processing feature: ${f}...`, LogLevel.Info);
 		if (!f.startsWith('.')) {
 			const featureFolder = path.join(srcFolder, f);
-			const archiveName = getArchiveName(f);
+			const archiveName = getFeatureArchiveName(f);
 
 			// Validate minimal feature folder structure
 			const featureJsonPath = path.join(featureFolder, 'devcontainer-feature.json');
@@ -116,4 +167,4 @@ export async function packageCollection(args: FeaturesPackageCommandInput): Prom
 
 	output.write(`Packaged ${metadatas.length} features!`, LogLevel.Info);
 	return metadatas;
-}
+}