diff --git a/src/spec-configuration/containerFeaturesConfiguration.ts b/src/spec-configuration/containerFeaturesConfiguration.ts index 15914864a..983ef685d 100644 --- a/src/spec-configuration/containerFeaturesConfiguration.ts +++ b/src/spec-configuration/containerFeaturesConfiguration.ts @@ -12,7 +12,7 @@ import { mkdirpLocal, readLocalFile, rmLocal, writeLocalFile, cpDirectoryLocal, import { Log, LogLevel } from '../spec-utils/log'; import { request } from '../spec-utils/httpRequest'; import { computeFeatureInstallationOrder } from './containerFeaturesOrder'; -import { fetchOCIFeature, getOCIFeatureSet, OCIFeatureRef, fetchOCIFeatureManifestIfExists, OCIManifest } from './containerFeaturesOCI'; +import { fetchOCIFeature, getOCIFeatureSet, OCIFeatureRef, fetchOCIFeatureManifestIfExistsFromUserIdentifier, OCIManifest } from './containerFeaturesOCI'; // v1 const V1_ASSET_NAME = 'devcontainer-features.tgz'; @@ -505,7 +505,7 @@ export async function getFeatureIdType(output: Log, env: NodeJS.ProcessEnv, user return { type: 'github-repo', manifest: undefined }; } - const manifest = await fetchOCIFeatureManifestIfExists(output, env, userFeatureId); + const manifest = await fetchOCIFeatureManifestIfExistsFromUserIdentifier(output, env, userFeatureId); if (manifest) { return { type: 'oci', manifest: manifest }; } else { @@ -986,4 +986,3 @@ function getFeatureValueDefaults(feature: Feature) { return defaults; }, {} as Record); } - diff --git a/src/spec-configuration/containerFeaturesOCI.ts b/src/spec-configuration/containerFeaturesOCI.ts index a7b21328c..1b6b4149e 100644 --- a/src/spec-configuration/containerFeaturesOCI.ts +++ b/src/spec-configuration/containerFeaturesOCI.ts @@ -7,262 +7,289 @@ import { FeatureSet } from './containerFeaturesConfiguration'; export type HEADERS = { 'authorization'?: string; 'user-agent': string; 'content-type'?: string; 'accept'?: string }; +export const DEVCONTAINER_MANIFEST_MEDIATYPE = 'application/vnd.devcontainers'; +export const DEVCONTAINER_TAR_LAYER_MEDIATYPE = 'application/vnd.devcontainers.layer.v1+tar'; +export const DEVCONTAINER_COLLECTION_LAYER_MEDIATYPE = 'application/vnd.devcontainers.collection.layer.v1+json'; + +// ghcr.io/devcontainers/features/go:1.0.0 export interface OCIFeatureRef { - id: string; - version: string; - owner: string; - namespace: string; - registry: string; - resource: string; + registry: string; // 'ghcr.io' + owner: string; // 'devcontainers' + namespace: string; // 'devcontainers/features' + path: string; // 'devcontainers/features/go' + resource: string; // 'ghcr.io/devcontainers/features/go' + id: string; // 'go' + version?: string; // '1.0.0' +} + +// ghcr.io/devcontainers/features:latest +export interface OCIFeatureCollectionRef { + registry: string; // 'ghcr.io' + path: string; // 'devcontainers/features' + version: 'latest'; // 'latest' } export interface OCILayer { - mediaType: string; - digest: string; - size: number; - annotations: { - 'org.opencontainers.image.title': string; - }; + mediaType: string; + digest: string; + size: number; + annotations: { + 'org.opencontainers.image.title': string; + }; } export interface OCIManifest { - digest?: string; - schemaVersion: number; - mediaType: string; - config: { - digest: string; - mediaType: string; - size: number; - }; - layers: OCILayer[]; + digest?: string; + schemaVersion: number; + mediaType: string; + config: { + digest: string; + mediaType: string; + size: number; + }; + layers: OCILayer[]; + annotations?: {}; } export function getOCIFeatureSet(output: Log, identifier: string, options: boolean | string | Record, manifest: OCIManifest): FeatureSet { - const featureRef = getFeatureRef(output, identifier); + const featureRef = getFeatureRef(output, identifier); - const feat = { - id: featureRef.id, - name: featureRef.id, - included: true, - value: options - }; + const feat = { + id: featureRef.id, + name: featureRef.id, + included: true, + value: options + }; - let featureSet: FeatureSet = { - sourceInformation: { - type: 'oci', - manifest: manifest, - featureRef: featureRef, + let featureSet: FeatureSet = { + sourceInformation: { + type: 'oci', + manifest: manifest, + featureRef: featureRef, - }, - features: [feat], - }; + }, + features: [feat], + }; - return featureSet; + return featureSet; } export function getFeatureRef(output: Log, resourceAndVersion: string): OCIFeatureRef { - // ex: ghcr.io/codspace/features/ruby:1 - const splitOnColon = resourceAndVersion.split(':'); - const resource = splitOnColon[0]; - const version = splitOnColon[1] ? splitOnColon[1] : 'latest'; - - const splitOnSlash = resource.split('/'); - - const id = splitOnSlash[splitOnSlash.length - 1]; // Aka 'featureName' - Eg: 'ruby' - const owner = splitOnSlash[1]; - const registry = splitOnSlash[0]; - const namespace = splitOnSlash.slice(1, -1).join('/'); - - output.write(`resource: ${resource}`, LogLevel.Trace); - output.write(`id: ${id}`, LogLevel.Trace); - output.write(`version: ${version}`, LogLevel.Trace); - output.write(`owner: ${owner}`, LogLevel.Trace); - output.write(`namespace: ${namespace}`, LogLevel.Trace); - output.write(`registry: ${registry}`, LogLevel.Trace); - - return { - id, - version, - owner, - namespace, - registry, - resource, - }; + // ex: ghcr.io/codspace/features/ruby:1 + const splitOnColon = resourceAndVersion.split(':'); + const resource = splitOnColon[0]; + const version = splitOnColon[1] ? splitOnColon[1] : 'latest'; + + const splitOnSlash = resource.split('/'); + + const id = splitOnSlash[splitOnSlash.length - 1]; // Aka 'featureName' - Eg: 'ruby' + const owner = splitOnSlash[1]; + const registry = splitOnSlash[0]; + const namespace = splitOnSlash.slice(1, -1).join('/'); + + const path = `${namespace}/${id}`; + + output.write(`resource: ${resource}`, LogLevel.Trace); + output.write(`id: ${id}`, LogLevel.Trace); + output.write(`version: ${version}`, LogLevel.Trace); + output.write(`owner: ${owner}`, LogLevel.Trace); + output.write(`namespace: ${namespace}`, LogLevel.Trace); + output.write(`registry: ${registry}`, LogLevel.Trace); + output.write(`path: ${path}`, LogLevel.Trace); + + return { + id, + version, + owner, + namespace, + registry, + resource, + path, + }; +} + + +export async function fetchOCIFeatureManifestIfExistsFromUserIdentifier(output: Log, env: NodeJS.ProcessEnv, identifier: string, manifestDigest?: string, authToken?: string): Promise { + const featureRef = getFeatureRef(output, identifier); + return await fetchOCIFeatureManifestIfExists(output, env, featureRef, manifestDigest, authToken); } // Validate if a manifest exists and is reachable about the declared feature. // Specification: https://github.com/opencontainers/distribution-spec/blob/v1.0.1/spec.md#pulling-manifests -export async function fetchOCIFeatureManifestIfExists(output: Log, env: NodeJS.ProcessEnv, identifier: string, manifestDigest?: string, authToken?: string): Promise { - const featureRef = getFeatureRef(output, identifier); - - // Simple mechanism to avoid making a DNS request for - // something that is not a domain name. - if (featureRef.registry.indexOf('.') < 0) { - return undefined; - } - - // TODO: Always use the manifest digest (the canonical digest) - // instead of the `featureRef.version` by referencing some lock file (if available). - let reference = featureRef.version; - if (manifestDigest) { - reference = manifestDigest; - } - - const manifestUrl = `https://${featureRef.registry}/v2/${featureRef.namespace}/${featureRef.id}/manifests/${reference}`; - output.write(`manifest url: ${manifestUrl}`, LogLevel.Trace); - const manifest = await getFeatureManifest(output, env, manifestUrl, featureRef, authToken); - - if (!manifest) { - output.write('OCI manifest not found', LogLevel.Warning); - return; - } - - if (manifest?.config.mediaType !== 'application/vnd.devcontainers') { - output.write(`(!) Unexpected manifest media type: ${manifest?.config.mediaType}`, LogLevel.Error); - return undefined; - } - - return manifest; +export async function fetchOCIFeatureManifestIfExists(output: Log, env: NodeJS.ProcessEnv, featureRef: OCIFeatureRef | OCIFeatureCollectionRef, manifestDigest?: string, authToken?: string): Promise { + // Simple mechanism to avoid making a DNS request for + // something that is not a domain name. + if (featureRef.registry.indexOf('.') < 0) { + return undefined; + } + + // TODO: Always use the manifest digest (the canonical digest) + // instead of the `featureRef.version` by referencing some lock file (if available). + let reference = featureRef.version; + if (manifestDigest) { + reference = manifestDigest; + } + const manifestUrl = `https://${featureRef.registry}/v2/${featureRef.path}/manifests/${reference}`; + output.write(`manifest url: ${manifestUrl}`, LogLevel.Trace); + const manifest = await getFeatureManifest(output, env, manifestUrl, featureRef, authToken); + + if (!manifest) { + return; + } + + if (manifest?.config.mediaType !== DEVCONTAINER_MANIFEST_MEDIATYPE) { + output.write(`(!) Unexpected manifest media type: ${manifest?.config.mediaType}`, LogLevel.Error); + return undefined; + } + + return manifest; } // Download a feature from which a manifest was previously downloaded. // Specification: https://github.com/opencontainers/distribution-spec/blob/v1.0.1/spec.md#pulling-blobs export async function fetchOCIFeature(output: Log, env: NodeJS.ProcessEnv, featureSet: FeatureSet, ociCacheDir: string, featCachePath: string): Promise { - if (featureSet.sourceInformation.type !== 'oci') { - output.write(`FeatureSet is not an OCI featureSet.`, LogLevel.Error); - throw new Error('FeatureSet is not an OCI featureSet.'); - } + if (featureSet.sourceInformation.type !== 'oci') { + output.write(`FeatureSet is not an OCI featureSet.`, LogLevel.Error); + throw new Error('FeatureSet is not an OCI featureSet.'); + } - const { featureRef } = featureSet.sourceInformation; + const { featureRef } = featureSet.sourceInformation; - const blobUrl = `https://${featureSet.sourceInformation.featureRef.registry}/v2/${featureSet.sourceInformation.featureRef.namespace}/${featureSet.sourceInformation.featureRef.id}/blobs/${featureSet.sourceInformation.manifest?.layers[0].digest}`; - output.write(`blob url: ${blobUrl}`, LogLevel.Trace); + const blobUrl = `https://${featureSet.sourceInformation.featureRef.registry}/v2/${featureSet.sourceInformation.featureRef.path}/blobs/${featureSet.sourceInformation.manifest?.layers[0].digest}`; + output.write(`blob url: ${blobUrl}`, LogLevel.Trace); - const success = await getFeatureBlob(output, env, blobUrl, ociCacheDir, featCachePath, featureRef); + const success = await getFeatureBlob(output, env, blobUrl, ociCacheDir, featCachePath, featureRef); - if (!success) { - throw new Error(`Failed to download package for ${featureSet.sourceInformation.featureRef.resource}`); - } + if (!success) { + throw new Error(`Failed to download package for ${featureSet.sourceInformation.featureRef.resource}`); + } - return true; + return true; } -export async function getFeatureManifest(output: Log, env: NodeJS.ProcessEnv, url: string, featureRef: OCIFeatureRef, authToken?: string): Promise { - try { - const headers: HEADERS = { - 'user-agent': 'devcontainer', - 'accept': 'application/vnd.oci.image.manifest.v1+json', - }; - - const auth = authToken ?? await fetchRegistryAuthToken(output, featureRef.registry, featureRef.resource, env, 'pull'); - if (auth) { - headers['authorization'] = `Bearer ${auth}`; - } - - const options = { - type: 'GET', - url: url, - headers: headers - }; - - const response = await request(options, output); - const manifest: OCIManifest = JSON.parse(response.toString()); - - return manifest; - } catch (e) { - output.write(`error: ${e}`, LogLevel.Error); - return undefined; - } +export async function getFeatureManifest(output: Log, env: NodeJS.ProcessEnv, url: string, featureRef: OCIFeatureRef | OCIFeatureCollectionRef, authToken?: string): Promise { + try { + const headers: HEADERS = { + 'user-agent': 'devcontainer', + 'accept': 'application/vnd.oci.image.manifest.v1+json', + }; + + const auth = authToken ?? await fetchRegistryAuthToken(output, featureRef.registry, featureRef.path, env, 'pull'); + if (auth) { + headers['authorization'] = `Bearer ${auth}`; + } + + const options = { + type: 'GET', + url: url, + headers: headers + }; + + const response = await request(options); + const manifest: OCIManifest = JSON.parse(response.toString()); + + return manifest; + } catch (e) { + return undefined; + } } // Downloads a blob from a registry. export async function getFeatureBlob(output: Log, env: NodeJS.ProcessEnv, url: string, ociCacheDir: string, featCachePath: string, featureRef: OCIFeatureRef, authToken?: string): Promise { - // TODO: Parallelize if multiple layers (not likely). - // TODO: Seeking might be needed if the size is too large. - try { - const tempTarballPath = path.join(ociCacheDir, 'blob.tar'); - - const headers: HEADERS = { - 'user-agent': 'devcontainer', - 'accept': 'application/vnd.oci.image.manifest.v1+json', - }; - - const auth = authToken ?? await fetchRegistryAuthToken(output, featureRef.registry, featureRef.resource, env, 'pull'); - if (auth) { - headers['authorization'] = `Bearer ${auth}`; - } - - const options = { - type: 'GET', - url: url, - headers: headers - }; - - const blob = await request(options, output); - - await mkdirpLocal(featCachePath); - await writeLocalFile(tempTarballPath, blob); - await tar.x( - { - file: tempTarballPath, - cwd: featCachePath, - } - ); - - return true; - } catch (e) { - output.write(`error: ${e}`, LogLevel.Error); - return false; - } + // TODO: Parallelize if multiple layers (not likely). + // TODO: Seeking might be needed if the size is too large. + try { + const tempTarballPath = path.join(ociCacheDir, 'blob.tar'); + + const headers: HEADERS = { + 'user-agent': 'devcontainer', + 'accept': 'application/vnd.oci.image.manifest.v1+json', + }; + + const auth = authToken ?? await fetchRegistryAuthToken(output, featureRef.registry, featureRef.path, env, 'pull'); + if (auth) { + headers['authorization'] = `Bearer ${auth}`; + } + + const options = { + type: 'GET', + url: url, + headers: headers + }; + + const blob = await request(options, output); + + await mkdirpLocal(featCachePath); + await writeLocalFile(tempTarballPath, blob); + await tar.x( + { + file: tempTarballPath, + cwd: featCachePath, + } + ); + + return true; + } catch (e) { + output.write(`error: ${e}`, LogLevel.Error); + return false; + } } // https://github.com/oras-project/oras-go/blob/97a9c43c52f9d89ecf5475bc59bd1f96c8cc61f6/registry/remote/auth/scope.go#L60-L74 -export async function fetchRegistryAuthToken(output: Log, registry: string, resource: string, env: NodeJS.ProcessEnv, operationScopes: string): Promise { - const headers: HEADERS = { - 'user-agent': 'devcontainer' - }; - - // TODO: Read OS keychain/docker config for auth in various registries! - - let userToken = ''; - if (!!env['GITHUB_TOKEN'] && registry === 'ghcr.io') { - userToken = env['GITHUB_TOKEN']; - } else if (!!env['DEVCONTAINERS_OCI_AUTH']) { - // eg: DEVCONTAINERS_OCI_AUTH=domain1:token1,domain2:token2 - const authContexts = env['DEVCONTAINERS_OCI_AUTH'].split(','); - const authContext = authContexts.find(a => a.split(':')[0] === registry); - if (authContext && authContext.length === 2) { - userToken = authContext.split(':')[1]; - } - } else { - output.write('No oauth authentication credentials found.', LogLevel.Trace); - } - - if (userToken) { - const base64Encoded = Buffer.from(`USERNAME:${userToken}`).toString('base64'); - headers['authorization'] = `Basic ${base64Encoded}`; - } - - const url = `https://${registry}/token?scope=repo:${resource}:${operationScopes}&service=${registry}`; - - const options = { - type: 'GET', - url: url, - headers: headers - }; - - const authReq = await request(options, output); - if (!authReq) { - output.write('Failed to get registry auth token', LogLevel.Error); - return undefined; - } - - const token: string | undefined = JSON.parse(authReq.toString())?.token; - if (!token) { - output.write('Failed to parse registry auth token response', LogLevel.Error); - return undefined; - } - return token; +export async function fetchRegistryAuthToken(output: Log, registry: string, ociRepoPath: string, env: NodeJS.ProcessEnv, operationScopes: string): Promise { + const headers: HEADERS = { + 'user-agent': 'devcontainer' + }; + + // TODO: Read OS keychain/docker config for auth in various registries! + + let userToken = ''; + if (!!env['GITHUB_TOKEN'] && registry === 'ghcr.io') { + userToken = env['GITHUB_TOKEN']; + } else if (!!env['DEVCONTAINERS_OCI_AUTH']) { + // eg: DEVCONTAINERS_OCI_AUTH=domain1:token1,domain2:token2 + const authContexts = env['DEVCONTAINERS_OCI_AUTH'].split(','); + const authContext = authContexts.find(a => a.split(':')[0] === registry); + if (authContext && authContext.length === 2) { + userToken = authContext.split(':')[1]; + } + } else { + output.write('No oauth authentication credentials found.', LogLevel.Trace); + } + + if (userToken) { + const base64Encoded = Buffer.from(`USERNAME:${userToken}`).toString('base64'); + headers['authorization'] = `Basic ${base64Encoded}`; + } + + const url = `https://${registry}/token?scope=repo:${ociRepoPath}:${operationScopes}&service=${registry}`; + output.write(`url: ${url}`, LogLevel.Trace); + + const options = { + type: 'GET', + url: url, + headers: headers + }; + + let authReq: Buffer; + try { + authReq = await request(options, output); + } catch (e: any) { + output.write(`Failed to get registry auth token with error: ${e}`, LogLevel.Error); + return undefined; + } + + if (!authReq) { + output.write('Failed to get registry auth token', LogLevel.Error); + return undefined; + } + + const token: string | undefined = JSON.parse(authReq.toString())?.token; + if (!token) { + output.write('Failed to parse registry auth token response', LogLevel.Error); + return undefined; + } + return token; } \ No newline at end of file diff --git a/src/spec-configuration/containerFeaturesOCIPush.ts b/src/spec-configuration/containerFeaturesOCIPush.ts new file mode 100644 index 000000000..1d739daa0 --- /dev/null +++ b/src/spec-configuration/containerFeaturesOCIPush.ts @@ -0,0 +1,350 @@ +import * as path from 'path'; +import * as fs from 'fs'; +import * as crypto from 'crypto'; +import { headRequest, requestResolveHeaders } from '../spec-utils/httpRequest'; +import { Log, LogLevel } from '../spec-utils/log'; +import { isLocalFile, readLocalFile } from '../spec-utils/pfs'; +import { DEVCONTAINER_COLLECTION_LAYER_MEDIATYPE, DEVCONTAINER_TAR_LAYER_MEDIATYPE, fetchOCIFeatureManifestIfExists, fetchRegistryAuthToken, HEADERS, OCIFeatureCollectionRef, OCIFeatureRef, OCILayer, OCIManifest } from './containerFeaturesOCI'; + +// (!) Entrypoint function to push a single feature to a registry. +// Devcontainer Spec : https://containers.dev/implementors/features-distribution/#oci-registry +// OCI Spec : https://github.com/opencontainers/distribution-spec/blob/main/spec.md#push +export async function pushOCIFeature(output: Log, featureRef: OCIFeatureRef, pathToTgz: string, tags: string[]): Promise { + output.write(`Starting push of feature '${featureRef.id}' to '${featureRef.resource}' with tags '${tags.join(', ')}'`); + output.write(`${JSON.stringify(featureRef, null, 2)}`, LogLevel.Trace); + const env = process.env; + + // Generate registry auth token with `pull,push` scopes. + const registryAuthToken = await fetchRegistryAuthToken(output, featureRef.registry, featureRef.path, env, 'pull,push'); + if (!registryAuthToken) { + output.write(`Failed to get registry auth token`, LogLevel.Error); + return false; + } + + // Generate Manifest for given feature artifact. + const manifest = await generateCompleteManifestForIndividualFeature(output, pathToTgz, featureRef); + if (!manifest) { + output.write(`Failed to generate manifest for ${featureRef.id}`, LogLevel.Error); + return false; + } + output.write(`Generated manifest: \n${JSON.stringify(manifest?.manifestObj, undefined, 4)}`, LogLevel.Trace); + + // If the exact manifest digest already exists in the registry, we don't need to push individual blobs (it's already there!) + const existingFeatureManifest = await fetchOCIFeatureManifestIfExists(output, env, featureRef, manifest.digest, registryAuthToken); + if (manifest.digest && existingFeatureManifest) { + output.write(`Not reuploading blobs, digest already exists.`, LogLevel.Trace); + await putManifestWithTags(output, manifest.manifestStr, featureRef, tags, registryAuthToken); + return true; + } + + const blobsToPush = [ + { + name: 'configLayer', + digest: manifest.manifestObj.config.digest, + }, + { + name: 'tgzLayer', + digest: manifest.manifestObj.layers[0].digest, + } + ]; + + // Obtain session ID with `/v2//blobs/uploads/` + const blobPutLocationUriPath = await postUploadSessionId(output, featureRef, registryAuthToken); + if (!blobPutLocationUriPath) { + output.write(`Failed to get upload session ID`, LogLevel.Error); + return false; + } + + for await (const blob of blobsToPush) { + const { name, digest } = blob; + const blobExistsConfigLayer = await checkIfBlobExists(output, featureRef, digest, registryAuthToken); + output.write(`blob: '${name}' with digest '${digest}' ${blobExistsConfigLayer ? 'already exists' : 'does not exist'} in registry.`, LogLevel.Trace); + + // PUT blobs + if (!blobExistsConfigLayer) { + if (!(await putBlob(output, pathToTgz, blobPutLocationUriPath, featureRef, digest, registryAuthToken))) { + output.write(`Failed to PUT blob '${name}' with digest '${digest}'`, LogLevel.Error); + return false; + } + } + } + + // Send a final PUT to combine blobs and tag manifest properly. + await putManifestWithTags(output, manifest.manifestStr, featureRef, tags, registryAuthToken); + + + // Success! + return true; +} + +// (!) Entrypoint function to push a collection metadata/overview file for a set of features to a registry. +// Devcontainer Spec : https://containers.dev/implementors/features-distribution/#oci-registry (see 'devcontainer-collection.json') +// OCI Spec : https://github.com/opencontainers/distribution-spec/blob/main/spec.md#push +export async function pushFeatureCollectionMetadata(output: Log, featureCollectionRef: OCIFeatureCollectionRef, pathToCollectionJson: string): Promise { + output.write(`Starting push of latest feature collection for namespace '${featureCollectionRef.path}' to '${featureCollectionRef.registry}'`); + output.write(`${JSON.stringify(featureCollectionRef, null, 2)}`, LogLevel.Trace); + const env = process.env; + + const registryAuthToken = await fetchRegistryAuthToken(output, featureCollectionRef.registry, featureCollectionRef.path, env, 'pull,push'); + if (!registryAuthToken) { + output.write(`Failed to get registry auth token`, LogLevel.Error); + return false; + } + + // Generate Manifest for collection artifact. + const manifest = await generateCompleteManifestForCollectionFile(output, pathToCollectionJson, featureCollectionRef); + if (!manifest) { + output.write(`Failed to generate manifest for ${featureCollectionRef.path}`, LogLevel.Error); + return false; + } + output.write(`Generated manifest: \n${JSON.stringify(manifest?.manifestObj, undefined, 4)}`, LogLevel.Trace); + + // If the exact manifest digest already exists in the registry, we don't need to push individual blobs (it's already there!) + const existingFeatureManifest = await fetchOCIFeatureManifestIfExists(output, env, featureCollectionRef, manifest.digest, registryAuthToken); + if (manifest.digest && existingFeatureManifest) { + output.write(`Not reuploading blobs, digest already exists.`, LogLevel.Trace); + await putManifestWithTags(output, manifest.manifestStr, featureCollectionRef, ['latest'], registryAuthToken); + return true; + } + + // Obtain session ID with `/v2//blobs/uploads/` + const blobPutLocationUriPath = await postUploadSessionId(output, featureCollectionRef, registryAuthToken); + if (!blobPutLocationUriPath) { + output.write(`Failed to get upload session ID`, LogLevel.Error); + return false; + } + + const blobsToPush = [ + { + name: 'configLayer', + digest: manifest.manifestObj.config.digest, + }, + { + name: 'collectionLayer', + digest: manifest.manifestObj.layers[0].digest, + } + ]; + + for await (const blob of blobsToPush) { + const { name, digest } = blob; + const blobExistsConfigLayer = await checkIfBlobExists(output, featureCollectionRef, digest, registryAuthToken); + output.write(`blob: '${name}' with digest '${digest}' ${blobExistsConfigLayer ? 'already exists' : 'does not exist'} in registry.`, LogLevel.Trace); + + // PUT blobs + if (!blobExistsConfigLayer) { + if (!(await putBlob(output, pathToCollectionJson, blobPutLocationUriPath, featureCollectionRef, digest, registryAuthToken))) { + output.write(`Failed to PUT blob '${name}' with digest '${digest}'`, LogLevel.Error); + return false; + } + } + } + + // Send a final PUT to combine blobs and tag manifest properly. + // Collections are always tagged 'latest' + await putManifestWithTags(output, manifest.manifestStr, featureCollectionRef, ['latest'], registryAuthToken); + + return true; +} + + + +// --- Helper Functions + +// Spec: https://github.com/opencontainers/distribution-spec/blob/main/spec.md#pushing-manifests (PUT /manifests/) +export async function putManifestWithTags(output: Log, manifestStr: string, featureRef: OCIFeatureRef | OCIFeatureCollectionRef, tags: string[], registryAuthToken: string): Promise { + output.write(`Tagging manifest with tags: ${tags.join(', ')}`, LogLevel.Trace); + + for await (const tag of tags) { + const url = `https://${featureRef.registry}/v2/${featureRef.path}/manifests/${tag}`; + output.write(`PUT -> '${url}'`, LogLevel.Trace); + const { statusCode, resHeaders } = await requestResolveHeaders({ + type: 'PUT', + url, + headers: { + 'Authorization': `Bearer ${registryAuthToken}`, + 'Content-Type': 'application/vnd.oci.image.manifest.v1+json', + }, + data: Buffer.from(manifestStr), + }); + if (statusCode !== 201) { + output.write(`Failed to PUT manifest for tag ${tag}`, LogLevel.Error); + return false; + } + + const dockerContentDigestResponseHeader = resHeaders['docker-content-digest'] || resHeaders['Docker-Content-Digest']; + const locationResponseHeader = resHeaders['location'] || resHeaders['Location']; + output.write(`Tagged: ${tag} -> ${locationResponseHeader}`, LogLevel.Info); + output.write(`Returned Content-Digest: ${dockerContentDigestResponseHeader}`, LogLevel.Trace); + } + return true; +} + +// Spec: https://github.com/opencontainers/distribution-spec/blob/main/spec.md#post-then-put (PUT ?digest=) +export async function putBlob(output: Log, pathToBlob: string, blobPutLocationUriPath: string, featureRef: OCIFeatureRef | OCIFeatureCollectionRef, digest: string, registryAuthToken: string): Promise { + output.write(`PUT new blob -> '${digest}'`, LogLevel.Info); + + if (!(await isLocalFile(pathToBlob))) { + output.write(`Blob ${pathToBlob} does not exist`, LogLevel.Error); + return false; + } + + const headers: HEADERS = { + 'user-agent': 'devcontainer', + 'authorization': `Bearer ${registryAuthToken}`, + 'content-type': 'application/octet-stream', + }; + + // OCI distribution spec is ambiguous on whether we get back an absolute or relative path. + let url = ''; + if (blobPutLocationUriPath.startsWith('https://')) { + url = blobPutLocationUriPath; + } else { + url = `https://${featureRef.registry}${blobPutLocationUriPath}`; + } + url += `?digest=${digest}`; + + output.write(`Crafted blob url: ${url}`, LogLevel.Trace); + + const { statusCode } = await requestResolveHeaders({ type: 'PUT', url, headers, data: await readLocalFile(pathToBlob) }); + if (statusCode !== 201) { + output.write(`${statusCode}: Failed to upload blob '${pathToBlob}' to '${url}'`, LogLevel.Error); + return false; + } + + return true; +} + +// Generate a layer that follows the `application/vnd.devcontainers.layer.v1+tar` mediaType +// as defined in: https://containers.dev/implementors/features-distribution/#oci-registry +export async function generateCompleteManifestForIndividualFeature(output: Log, pathToTgz: string, ociFeatureRef: OCIFeatureRef): Promise<{ manifestObj: OCIManifest; manifestStr: string; digest: string } | undefined> { + const tgzLayer = await calculateDataLayer(output, pathToTgz, DEVCONTAINER_TAR_LAYER_MEDIATYPE); + if (!tgzLayer) { + output.write(`Failed to calculate tgz layer.`, LogLevel.Error); + return undefined; + } + + let annotations: { [key: string]: string } | undefined = undefined; + // Specific registries look for certain optional metadata + // in the manifest, in this case for UI presentation. + if (ociFeatureRef.registry === 'ghcr.io') { + annotations = { + 'com.github.package.type': 'devcontainer-feature', + }; + } + return await calculateManifestAndContentDigest(output, tgzLayer, annotations); +} + +// Generate a layer that follows the `application/vnd.devcontainers.collection.layer.v1+json` mediaType +// as defined in: https://containers.dev/implementors/features-distribution/#oci-registry +export async function generateCompleteManifestForCollectionFile(output: Log, pathToCollectionFile: string, collectionRef: OCIFeatureCollectionRef): Promise<{ manifestObj: OCIManifest; manifestStr: string; digest: string } | undefined> { + const collectionMetadataLayer = await calculateDataLayer(output, pathToCollectionFile, DEVCONTAINER_COLLECTION_LAYER_MEDIATYPE); + if (!collectionMetadataLayer) { + output.write(`Failed to calculate collection file layer.`, LogLevel.Error); + return undefined; + } + + let annotations: { [key: string]: string } | undefined = undefined; + // Specific registries look for certain optional metadata + // in the manifest, in this case for UI presentation. + if (collectionRef.registry === 'ghcr.io') { + annotations = { + 'com.github.package.type': 'devcontainer-collection', + }; + } + return await calculateManifestAndContentDigest(output, collectionMetadataLayer, annotations); +} + +// Generic construction of a layer in the manifest and digest for the generated layer. +export async function calculateDataLayer(output: Log, pathToData: string, mediaType: string): Promise { + output.write(`Creating manifest from ${pathToData}`, LogLevel.Trace); + if (!(await isLocalFile(pathToData))) { + output.write(`${pathToData} does not exist.`, LogLevel.Error); + return undefined; + } + + const dataBytes = fs.readFileSync(pathToData); + + const tarSha256 = crypto.createHash('sha256').update(dataBytes).digest('hex'); + output.write(`${pathToData}: sha256:${tarSha256} (size: ${dataBytes.byteLength})`, LogLevel.Info); + + return { + mediaType, + digest: `sha256:${tarSha256}`, + size: dataBytes.byteLength, + annotations: { + 'org.opencontainers.image.title': path.basename(pathToData), + } + }; +} + +// Spec: https://github.com/opencontainers/distribution-spec/blob/main/spec.md#checking-if-content-exists-in-the-registry +// Requires registry auth token. +export async function checkIfBlobExists(output: Log, featureRef: OCIFeatureRef | OCIFeatureCollectionRef, digest: string, authToken: string): Promise { + const headers: HEADERS = { + 'user-agent': 'devcontainer', + 'authorization': `Bearer ${authToken}`, + }; + + const url = `https://${featureRef.registry}/v2/${featureRef.path}/blobs/${digest}`; + const statusCode = await headRequest({ url, headers }, output); + + output.write(`${url}: ${statusCode}`, LogLevel.Trace); + return statusCode === 200; +} + +// Spec: https://github.com/opencontainers/distribution-spec/blob/main/spec.md#post-then-put +// Requires registry auth token. +export async function postUploadSessionId(output: Log, featureRef: OCIFeatureRef | OCIFeatureCollectionRef, authToken: string): Promise { + const headers: HEADERS = { + 'user-agent': 'devcontainer', + 'authorization': `Bearer ${authToken}`, + }; + + const url = `https://${featureRef.registry}/v2/${featureRef.path}/blobs/uploads/`; + output.write(`Generating Upload URL -> ${url}`, LogLevel.Trace); + const { statusCode, resHeaders } = await requestResolveHeaders({ type: 'POST', url, headers }, output); + output.write(`${url}: ${statusCode}`, LogLevel.Trace); + if (statusCode === 202) { + const locationHeader = resHeaders['location'] || resHeaders['Location']; + if (!locationHeader) { + output.write(`${url}: Got 202 status code, but no location header found.`, LogLevel.Error); + return undefined; + } + return locationHeader; + } + return undefined; +} + +export async function calculateManifestAndContentDigest(output: Log, dataLayer: OCILayer, annotations: { [key: string]: string } | undefined) { + // A canonical manifest digest is the sha256 hash of the JSON representation of the manifest, without the signature content. + // See: https://docs.docker.com/registry/spec/api/#content-digests + // Below is an example of a serialized manifest that should resolve to '9726054859c13377c4c3c3c73d15065de59d0c25d61d5652576c0125f2ea8ed3' + // {"schemaVersion":2,"mediaType":"application/vnd.oci.image.manifest.v1+json","config":{"mediaType":"application/vnd.devcontainers","digest":"sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","size":0},"layers":[{"mediaType":"application/vnd.devcontainers.layer.v1+tar","digest":"sha256:b2006e7647191f7b47222ae48df049c6e21a4c5a04acfad0c4ef614d819de4c5","size":15872,"annotations":{"org.opencontainers.image.title":"go.tgz"}}]} + + let manifest: OCIManifest = { + schemaVersion: 2, + mediaType: 'application/vnd.oci.image.manifest.v1+json', + config: { + mediaType: 'application/vnd.devcontainers', + digest: 'sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', // A zero byte digest for the devcontainer mediaType. + size: 0 + }, + layers: [ + dataLayer + ], + }; + + if (annotations) { + manifest.annotations = annotations; + } + + const manifestStringified = JSON.stringify(manifest); + const manifestHash = crypto.createHash('sha256').update(manifestStringified).digest('hex'); + output.write(`Computed Content-Digest -> sha256:${manifestHash} (size: ${manifestHash.length})`, LogLevel.Info); + + return { + manifestStr: manifestStringified, + manifestObj: manifest, + digest: manifestHash, + }; +} \ No newline at end of file diff --git a/src/spec-node/featuresCLI/packageCommandImpl.ts b/src/spec-node/featuresCLI/packageCommandImpl.ts index 43b3aeca5..08b9d6977 100644 --- a/src/spec-node/featuresCLI/packageCommandImpl.ts +++ b/src/spec-node/featuresCLI/packageCommandImpl.ts @@ -4,7 +4,6 @@ import { Feature } from '../../spec-configuration/containerFeaturesConfiguration import { LogLevel } from '../../spec-utils/log'; import { isLocalFile, readLocalDir, readLocalFile, writeLocalFile } from '../../spec-utils/pfs'; import { FeaturesPackageCommandInput } from './package'; - export interface SourceInformation { source: string; owner?: string; @@ -47,7 +46,6 @@ export async function doFeaturesPackageCommand(args: FeaturesPackageCommandInput // Write the metadata to a file const metadataOutputPath = path.join(args.outputDir, 'devcontainer-collection.json'); await writeLocalFile(metadataOutputPath, JSON.stringify(collection, null, 4)); - return 0; } @@ -105,7 +103,7 @@ export async function packageCollection(args: FeaturesPackageCommandInput): Prom const featureMetadata: Feature = JSON.parse(await readLocalFile(featureJsonPath, 'utf-8')); if (!featureMetadata.id || !featureMetadata.version) { - output.write(`Feature '${f}' is missing an id or verion in its devcontainer-feature.json`, LogLevel.Error); + output.write(`Feature '${f}' is missing an id or version in its devcontainer-feature.json`, LogLevel.Error); return; } metadatas.push(featureMetadata); diff --git a/src/spec-utils/httpRequest.ts b/src/spec-utils/httpRequest.ts index 7340d56ca..8eb37d1ef 100644 --- a/src/spec-utils/httpRequest.ts +++ b/src/spec-utils/httpRequest.ts @@ -36,4 +36,54 @@ export function request(options: { type: string; url: string; headers: Record }, output?: Log) { + return new Promise((resolve, reject) => { + const parsed = new url.URL(options.url); + const reqOptions = { + hostname: parsed.hostname, + port: parsed.port, + path: parsed.pathname + parsed.search, + method: 'HEAD', + headers: options.headers, + }; + const req = https.request(reqOptions, res => { + res.on('error', reject); + if (output) { + output.write(`HEAD ${options.url} -> ${res.statusCode}`, LogLevel.Trace); + } + resolve(res.statusCode!); + }); + req.on('error', reject); + req.end(); + }); +} + +// Send HTTP Request. Does not throw on status code, but rather always returns 'statusCode' and 'resHeaders'. +export function requestResolveHeaders(options: { type: string; url: string; headers: Record; data?: Buffer }, _output?: Log) { + return new Promise<{ statusCode: number; resHeaders: Record }>((resolve, reject) => { + const parsed = new url.URL(options.url); + const reqOptions = { + hostname: parsed.hostname, + port: parsed.port, + path: parsed.pathname + parsed.search, + method: options.type, + headers: options.headers, + }; + const req = https.request(reqOptions, res => { + res.on('error', reject); + const result = { + statusCode: res.statusCode!, + resHeaders: res.headers! as Record + }; + resolve(result); + }); + req.on('error', reject); + if (options.data) { + req.write(options.data); + } + req.end(); + }); } \ No newline at end of file diff --git a/src/test/container-features/containerFeaturesOCI.test.ts b/src/test/container-features/containerFeaturesOCI.test.ts index 238d63349..beff3eb82 100644 --- a/src/test/container-features/containerFeaturesOCI.test.ts +++ b/src/test/container-features/containerFeaturesOCI.test.ts @@ -15,6 +15,7 @@ describe('Test OCI Pull', () => { assert.equal(feat.registry, 'ghcr.io'); assert.equal(feat.resource, 'ghcr.io/codspace/features/ruby'); assert.equal(feat.version, '1'); + assert.equal(feat.path, 'codspace/features/ruby'); }); it('Get a manifest by tag', async () => { diff --git a/src/test/container-features/containerFeaturesOCIPush.test.ts b/src/test/container-features/containerFeaturesOCIPush.test.ts new file mode 100644 index 000000000..677764dbd --- /dev/null +++ b/src/test/container-features/containerFeaturesOCIPush.test.ts @@ -0,0 +1,62 @@ +import { assert } from 'chai'; +import { getFeatureRef, fetchOCIFeatureManifestIfExistsFromUserIdentifier, fetchRegistryAuthToken, DEVCONTAINER_TAR_LAYER_MEDIATYPE } from '../../spec-configuration/containerFeaturesOCI'; +import { calculateDataLayer, checkIfBlobExists, calculateManifestAndContentDigest } from '../../spec-configuration/containerFeaturesOCIPush'; +import { createPlainLog, LogLevel, makeLog } from '../../spec-utils/log'; + +export const output = makeLog(createPlainLog(text => process.stdout.write(text), () => LogLevel.Trace)); +const testAssetsDir = `${__dirname}/assets`; + +// NOTE: +// Test depends on https://github.com/codspace/features/pkgs/container/features%2Fgo/29819216?tag=1 +describe('Test OCI Push', () => { + it('Generates the correct tgz manifest layer', async () => { + + // Calculate the tgz layer and digest + const res = await calculateDataLayer(output, `${testAssetsDir}/go.tgz`, DEVCONTAINER_TAR_LAYER_MEDIATYPE); + const expected = { + digest: 'sha256:b2006e7647191f7b47222ae48df049c6e21a4c5a04acfad0c4ef614d819de4c5', + mediaType: 'application/vnd.devcontainers.layer.v1+tar', + size: 15872, + annotations: { + 'org.opencontainers.image.title': 'go.tgz' + } + }; + + if (!res) { + assert.fail(); + } + assert.deepEqual(res, expected); + + // Generate entire manifest to be able to calculate content digest + const { manifestStr, digest } = await calculateManifestAndContentDigest(output, res, undefined); + + // 'Expected' is taken from intermediate value in oras reference implementation, before hash calculation + assert.strictEqual('{"schemaVersion":2,"mediaType":"application/vnd.oci.image.manifest.v1+json","config":{"mediaType":"application/vnd.devcontainers","digest":"sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","size":0},"layers":[{"mediaType":"application/vnd.devcontainers.layer.v1+tar","digest":"sha256:b2006e7647191f7b47222ae48df049c6e21a4c5a04acfad0c4ef614d819de4c5","size":15872,"annotations":{"org.opencontainers.image.title":"go.tgz"}}]}', manifestStr); + + // This is the canonical digest of the manifest + assert.strictEqual('9726054859c13377c4c3c3c73d15065de59d0c25d61d5652576c0125f2ea8ed3', digest); + }); + + it('Can fetch an artifact from a digest reference', async () => { + const manifest = await fetchOCIFeatureManifestIfExistsFromUserIdentifier(output, process.env, 'ghcr.io/codspace/features/go', 'sha256:9726054859c13377c4c3c3c73d15065de59d0c25d61d5652576c0125f2ea8ed3'); + assert.strictEqual(manifest?.layers[0].annotations['org.opencontainers.image.title'], 'go.tgz'); + }); + + it('Can check whether a blob exists', async () => { + const ociFeatureRef = getFeatureRef(output, 'ghcr.io/codspace/features/go:1'); + const { registry, resource } = ociFeatureRef; + const sessionAuth = await fetchRegistryAuthToken(output, registry, resource, process.env, 'pull'); + if (!sessionAuth) { + assert.fail('Could not get registry auth token'); + } + + const tarLayerBlobExists = await checkIfBlobExists(output, ociFeatureRef, 'sha256:b2006e7647191f7b47222ae48df049c6e21a4c5a04acfad0c4ef614d819de4c5', sessionAuth); + assert.isTrue(tarLayerBlobExists); + + const configLayerBlobExists = await checkIfBlobExists(output, ociFeatureRef, 'sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', sessionAuth); + assert.isTrue(configLayerBlobExists); + + const randomStringDoesNotExist = await checkIfBlobExists(output, ociFeatureRef, 'sha256:41af286dc0b172ed2f1ca934fd2278de4a1192302ffa07087cea2682e7d372e3', sessionAuth); + assert.isFalse(randomStringDoesNotExist); + }); +}); \ No newline at end of file diff --git a/src/test/container-features/featureHelpers.test.ts b/src/test/container-features/featureHelpers.test.ts index 458b7ea0c..831fe21ac 100644 --- a/src/test/container-features/featureHelpers.test.ts +++ b/src/test/container-features/featureHelpers.test.ts @@ -203,7 +203,8 @@ describe('validate processFeatureIdentifier', async function () { namespace: 'codspace/features', registry: 'ghcr.io', version: 'latest', - resource: 'ghcr.io/codspace/features/ruby' + resource: 'ghcr.io/codspace/features/ruby', + path: 'codspace/features/ruby', }; if (featureSet.sourceInformation.type === 'oci') { @@ -236,7 +237,8 @@ describe('validate processFeatureIdentifier', async function () { namespace: 'codspace/features', registry: 'ghcr.io', version: '1.0.13', - resource: 'ghcr.io/codspace/features/ruby' + resource: 'ghcr.io/codspace/features/ruby', + path: 'codspace/features/ruby', }; if (featureSet.sourceInformation.type === 'oci') { diff --git a/src/test/container-features/featuresCLICommands.test.ts b/src/test/container-features/featuresCLICommands.test.ts index 9c2f09cc3..020007b25 100644 --- a/src/test/container-features/featuresCLICommands.test.ts +++ b/src/test/container-features/featuresCLICommands.test.ts @@ -32,7 +32,7 @@ describe('CLI features subcommands', async function () { assert.isTrue(success); }); - it('features package subcommand (collection)', async function () { + it('features package subcommand by collection', async function () { const srcFolder = `${__dirname}/example-v2-features-sets/simple/src`; let success = false; try { @@ -61,7 +61,7 @@ describe('CLI features subcommands', async function () { assert.isTrue(collectionFileExists); }); - it('features package subcommand (single feature)', async function () { + it('features package subcommand by single feature', async function () { const singleFeatureFolder = `${__dirname}/example-v2-features-sets/simple/src/color`; let success = false; try {