diff --git a/src/spec-configuration/containerFeaturesOCI.ts b/src/spec-configuration/containerFeaturesOCI.ts index 8cd677bf7..92d2b3c89 100644 --- a/src/spec-configuration/containerFeaturesOCI.ts +++ b/src/spec-configuration/containerFeaturesOCI.ts @@ -1,8 +1,10 @@ import * as path from 'path'; import * as tar from 'tar'; +import * as fs from 'fs'; +import * as crypto from 'crypto'; import { request } from '../spec-utils/httpRequest'; import { Log, LogLevel } from '../spec-utils/log'; -import { mkdirpLocal, writeLocalFile } from '../spec-utils/pfs'; +import { isLocalFile, mkdirpLocal, writeLocalFile } from '../spec-utils/pfs'; import { FeatureSet } from './containerFeaturesConfiguration'; export interface OCIFeatureRef { @@ -14,25 +16,25 @@ export interface OCIFeatureRef { registry: string; } +export interface OCILayer { + mediaType: string; + digest: string; + size: number; + annotations: { + // 'org.opencontainers.image.ref.name': string; + 'org.opencontainers.image.title': string; + }; +} export interface OCIManifest { + digest?: string; schemaVersion: number; mediaType: string; config: { - mediaType: string; digest: string; + mediaType: string; size: number; }; - layers: [ - { - mediaType: string; - digest: string; - size: number; - annotations: { - 'org.opencontainers.image.ref.name': string; - 'org.opencontainers.image.title': string; - }; - } - ]; + layers: OCILayer[]; } export function getOCIFeatureSet(output: Log, identifier: string, options: boolean | string | Record, manifest: OCIManifest): FeatureSet { @@ -91,14 +93,23 @@ export function getFeatureRef(output: Log, identifier: string): OCIFeatureRef { } // Validate if a manifest exists and is reachable about the declared feature. -export async function validateOCIFeature(output: Log, env: NodeJS.ProcessEnv, identifier: string): Promise { +// Specification: https://github.com/opencontainers/distribution-spec/blob/v1.0.1/spec.md#pulling-manifests +export async function validateOCIFeature(output: Log, env: NodeJS.ProcessEnv, identifier: string, manifestDigest?: string): Promise { const featureRef = getFeatureRef(output, identifier); - const manifestUrl = `https://${featureRef.registry}/v2/${featureRef.namespace}/${featureRef.featureName}/manifests/${featureRef.version}`; + let reference = featureRef.version; + if (manifestDigest) { + reference = manifestDigest; + } + + // TODO: Always use the manifest digest (the canonical digest) instead of the `featureRef.version` + // matching some lock file. + const manifestUrl = `https://${featureRef.registry}/v2/${featureRef.namespace}/${featureRef.featureName}/manifests/${reference}`; output.write(`manifest url: ${manifestUrl}`, LogLevel.Trace); const manifest = await getFeatureManifest(output, env, manifestUrl, featureRef); if (manifest?.config.mediaType !== 'application/vnd.devcontainers') { + output.write(`(!) Unexpected manifest media type: ${manifest?.config.mediaType}`, LogLevel.Error); return undefined; } @@ -191,22 +202,26 @@ async function getAuthenticationToken(env: NodeJS.ProcessEnv, output: Log, regis // TODO: Use operating system keychain to get credentials. // TODO: Fallback to read docker config to get credentials. - // TEMP for ghcr.io - const githubToken = env['GITHUB_TOKEN']; - if (registry !== 'ghcr.io' || !githubToken) { - const token = await getAuthToken(output, registry, id); - return 'Bearer ' + token; + // TODO: Un-hardcode ghcr.io + const registryAuthToken = await fetchRegistryAuthToken(output, registry, id, env); + + if (!registryAuthToken) { + return ''; // TODO } - return ''; + return `Bearer ${registryAuthToken}`; } -export async function getAuthToken(output: Log, registry: string, id: string) { - const headers = { - 'user-agent': 'devcontainer', +export async function fetchRegistryAuthToken(output: Log, registry: string, id: string, env: NodeJS.ProcessEnv): Promise { + const headers: { 'authorization'?: string; 'user-agent': string } = { + 'user-agent': 'devcontainer' }; - const url = `https://${registry}/token?scope=repo:${id}:pull&service=ghcr.io`; + if (!!env['GITHUB_TOKEN']) { + headers['authorization'] = `Bearer ${env['GITHUB_TOKEN']}`; + } + + const url = `https://${registry}/token?scope=repo:${id}:pull&service=${registry}`; const options = { type: 'GET', @@ -214,7 +229,85 @@ export async function getAuthToken(output: Log, registry: string, id: string) { headers: headers }; - const token = JSON.parse((await request(options, output)).toString()).token; + const authReq = await request(options, output); + if (!authReq) { + output.write('Failed to get registry auth token', LogLevel.Error); + return undefined; + } + const token = JSON.parse(authReq.toString())?.token; + if (!token) { + output.write('Failed to parse registry auth token response', LogLevel.Error); + return undefined; + } return token; +} + +// -- Push + +// export async function pushOCIFeature(output: Log, env: NodeJS.ProcessEnv, featureSet: FeatureSet, ociCacheDir: string, featCachePath: string, featureRef: OCIFeatureRef): Promise { +// const await validateOCIFeature(output, env, featureRef.id); +// } + +export async function generateManifest(output: Log, pathToTgz: string): Promise { + + const tgzLayer = await calculateTgzLayer(output, pathToTgz); + if (!tgzLayer) { + output.write(`Failed to calculate tgz layer.`, LogLevel.Error); + return undefined; + } + + const { manifestObj, hash } = await calculateContentDigest(output, tgzLayer); + manifestObj.digest = `sha256:${hash}`; + + return manifestObj; +} + +export async function calculateTgzLayer(output: Log, pathToTgz: string): Promise { + output.write(`Creating manifest from ${pathToTgz}`, LogLevel.Trace); + if (!(await isLocalFile(pathToTgz))) { + output.write(`${pathToTgz} does not exist.`, LogLevel.Error); + return undefined; + } + + const tarBytes = fs.readFileSync(pathToTgz); + + const tarSha256 = crypto.createHash('sha256').update(tarBytes).digest('hex'); + output.write(`${pathToTgz}: sha256:${tarSha256} (size: ${tarBytes.byteLength})`, LogLevel.Trace); + + return { + mediaType: 'application/vnd.devcontainers.layer.v1+tar', + digest: `sha256:${tarSha256}`, + size: tarBytes.byteLength, + annotations: { + 'org.opencontainers.image.title': path.basename(pathToTgz), + } + }; +} + +export async function calculateContentDigest(output: Log, tgzLayer: OCILayer) { + // {"schemaVersion":2,"mediaType":"application/vnd.oci.image.manifest.v1+json","config":{"mediaType":"application/vnd.devcontainers","digest":"sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","size":0},"layers":[{"mediaType":"application/vnd.devcontainers.layer.v1+tar","digest":"sha256:b2006e7647191f7b47222ae48df049c6e21a4c5a04acfad0c4ef614d819de4c5","size":15872,"annotations":{"org.opencontainers.image.title":"go.tgz"}}]} + + let manifest: OCIManifest = { + schemaVersion: 2, + mediaType: 'application/vnd.oci.image.manifest.v1+json', + config: { + mediaType: 'application/vnd.devcontainers', + digest: 'sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', + size: 0 + }, + layers: [ + tgzLayer + ] + }; + + const manifestStringified = JSON.stringify(manifest); + const manifestHash = crypto.createHash('sha256').update(manifestStringified).digest('hex'); + output.write(`manifest: sha256:${manifestHash} (size: ${manifestHash.length})`, LogLevel.Trace); + + return { + manifestStr: manifestStringified, + manifestObj: manifest, + hash: manifestHash, + }; } \ No newline at end of file diff --git a/src/test/container-features/assets/go.tgz b/src/test/container-features/assets/go.tgz new file mode 100644 index 000000000..5a831a7ae Binary files /dev/null and b/src/test/container-features/assets/go.tgz differ diff --git a/src/test/container-features/containerFeaturesOCI.offline.test.ts b/src/test/container-features/containerFeaturesOCI.offline.test.ts index 9d07a8a76..7e040d91e 100644 --- a/src/test/container-features/containerFeaturesOCI.offline.test.ts +++ b/src/test/container-features/containerFeaturesOCI.offline.test.ts @@ -1,10 +1,13 @@ +// import { assert } from 'chai'; +// import { getFeatureManifest, getFeatureBlob, getFeatureRef, createManifest } from '../../spec-configuration/containerFeaturesOCI'; import { assert } from 'chai'; -import { getFeatureManifest, getFeatureBlob, getFeatureRef } from '../../spec-configuration/containerFeaturesOCI'; +import { calculateContentDigest, calculateTgzLayer, getFeatureBlob, getFeatureManifest, getFeatureRef, validateOCIFeature } from '../../spec-configuration/containerFeaturesOCI'; import { createPlainLog, LogLevel, makeLog } from '../../spec-utils/log'; export const output = makeLog(createPlainLog(text => process.stdout.write(text), () => LogLevel.Trace)); +const testAssetsDir = `${__dirname}/assets`; -describe('Test OCI', () => { +describe('Test OCI Pull', () => { it('Parse OCI identifier', async () => { const feat = getFeatureRef(output, 'ghcr.io/codspace/features/ruby:1'); @@ -45,4 +48,40 @@ describe('Test OCI', () => { const result = await getFeatureBlob(output, process.env, 'https://ghcr.io/v2/codspace/features/ruby/blobs/sha256:c33008d0dc12d0e631734082401bec692da809eae2ac51e24f58c1cac68fc0c9', '/tmp', '/tmp/featureTest', featureRef); assert.isTrue(result); }); +}); + + +// NOTE: Test depends on https://github.com/codspace/features/pkgs/container/features%2Fgo/29819216?tag=1 +describe('Test Generate Manifests and Digests', () => { + it('Generates the correct tgz manifest layer', async () => { + + // Calculate the tgz layer and digest + const res = await calculateTgzLayer(output, `${testAssetsDir}/go.tgz`); + const expected = { + digest: 'sha256:b2006e7647191f7b47222ae48df049c6e21a4c5a04acfad0c4ef614d819de4c5', + mediaType: 'application/vnd.devcontainers.layer.v1+tar', + size: 15872, + annotations: { + 'org.opencontainers.image.title': 'go.tgz' + } + }; + + if (!res) { + assert.fail(); + } + assert.deepEqual(res, expected); + + // Generate entire manifest to be able to calculate content digest + const { manifestStr, hash } = await calculateContentDigest(output, res); + + // 'Expected' is taken from intermediate value in oras reference implementation, before hash calculation + assert.strictEqual('{"schemaVersion":2,"mediaType":"application/vnd.oci.image.manifest.v1+json","config":{"mediaType":"application/vnd.devcontainers","digest":"sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","size":0},"layers":[{"mediaType":"application/vnd.devcontainers.layer.v1+tar","digest":"sha256:b2006e7647191f7b47222ae48df049c6e21a4c5a04acfad0c4ef614d819de4c5","size":15872,"annotations":{"org.opencontainers.image.title":"go.tgz"}}]}', manifestStr); + + assert.strictEqual('9726054859c13377c4c3c3c73d15065de59d0c25d61d5652576c0125f2ea8ed3', hash); + }); + + it('Can fetch an artifact from a digest reference', async () => { + const manifest = await validateOCIFeature(output, process.env, 'ghcr.io/codspace/features/go', 'sha256:9726054859c13377c4c3c3c73d15065de59d0c25d61d5652576c0125f2ea8ed3'); + assert.strictEqual(manifest?.layers[0].annotations['org.opencontainers.image.title'], 'go.tgz'); + }); }); \ No newline at end of file