Features Distribution Proposal -- Minimal OCI push

src/spec-configuration/containerFeaturesOCI.ts

@@ -1,8 +1,10 @@
 import * as path from 'path';
 import * as tar from 'tar';
+import * as fs from 'fs';
+import * as crypto from 'crypto';
 import { request } from '../spec-utils/httpRequest';
 import { Log, LogLevel } from '../spec-utils/log';
-import { mkdirpLocal, writeLocalFile } from '../spec-utils/pfs';
+import { isLocalFile, mkdirpLocal, writeLocalFile } from '../spec-utils/pfs';
 import { FeatureSet } from './containerFeaturesConfiguration';
 
 export interface OCIFeatureRef {
@@ -14,25 +16,25 @@ export interface OCIFeatureRef {
     registry: string;
 }
 
+export interface OCILayer {
+    mediaType: string;
+    digest: string;
+    size: number;
+    annotations: {
+        // 'org.opencontainers.image.ref.name': string;
+        'org.opencontainers.image.title': string;
+    };
+}
 export interface OCIManifest {
+    digest?: string;
     schemaVersion: number;
     mediaType: string;
     config: {
-        mediaType: string;
         digest: string;
+        mediaType: string;
         size: number;
     };
-    layers: [
-        {
-            mediaType: string;
-            digest: string;
-            size: number;
-            annotations: {
-                'org.opencontainers.image.ref.name': string;
-                'org.opencontainers.image.title': string;
-            };
-        }
-    ];
+    layers: OCILayer[];
 }
 
 export function getOCIFeatureSet(output: Log, identifier: string, options: boolean | string | Record<string, boolean | string | undefined>, manifest: OCIManifest): FeatureSet {
@@ -91,14 +93,23 @@ export function getFeatureRef(output: Log, identifier: string): OCIFeatureRef {
 }
 
 // Validate if a manifest exists and is reachable about the declared feature.
-export async function validateOCIFeature(output: Log, env: NodeJS.ProcessEnv, identifier: string): Promise<OCIManifest | undefined> {
+// Specification: https://github.com/opencontainers/distribution-spec/blob/v1.0.1/spec.md#pulling-manifests
+export async function validateOCIFeature(output: Log, env: NodeJS.ProcessEnv, identifier: string, manifestDigest?: string): Promise<OCIManifest | undefined> {
     const featureRef = getFeatureRef(output, identifier);
 
-    const manifestUrl = `https://${featureRef.registry}/v2/${featureRef.namespace}/${featureRef.featureName}/manifests/${featureRef.version}`;
+    let reference = featureRef.version;
+    if (manifestDigest) {
+        reference = manifestDigest;
+    }
+
+    // TODO: Always use the manifest digest (the canonical digest) instead of the `featureRef.version`
+    //       matching some lock file. 
+    const manifestUrl = `https://${featureRef.registry}/v2/${featureRef.namespace}/${featureRef.featureName}/manifests/${reference}`;
     output.write(`manifest url: ${manifestUrl}`, LogLevel.Trace);
     const manifest = await getFeatureManifest(output, env, manifestUrl, featureRef);
 
     if (manifest?.config.mediaType !== 'application/vnd.devcontainers') {
+        output.write(`(!) Unexpected manifest media type: ${manifest?.config.mediaType}`, LogLevel.Error);
         return undefined;
     }
 
@@ -191,30 +202,112 @@ async function getAuthenticationToken(env: NodeJS.ProcessEnv, output: Log, regis
     // TODO: Use operating system keychain to get credentials.
     // TODO: Fallback to read docker config to get credentials.
 
-    // TEMP for ghcr.io
-    const githubToken = env['GITHUB_TOKEN'];
-    if (registry !== 'ghcr.io' || !githubToken) {
-        const token = await getAuthToken(output, registry, id);
-        return 'Bearer ' + token;
+    // TODO: Un-hardcode ghcr.io
+    const registryAuthToken = await fetchRegistryAuthToken(output, registry, id, env);
+
+    if (!registryAuthToken) {
+        return ''; // TODO
     }
 
-    return '';
+    return `Bearer ${registryAuthToken}`;
 }
 
-export async function getAuthToken(output: Log, registry: string, id: string) {
-    const headers = {
-        'user-agent': 'devcontainer',
+export async function fetchRegistryAuthToken(output: Log, registry: string, id: string, env: NodeJS.ProcessEnv): Promise<string | undefined> {
+    const headers: { 'authorization'?: string; 'user-agent': string } = {
+        'user-agent': 'devcontainer'
     };
 
-    const url = `https://${registry}/token?scope=repo:${id}:pull&service=ghcr.io`;
+    if (!!env['GITHUB_TOKEN']) {
+        headers['authorization'] = `Bearer ${env['GITHUB_TOKEN']}`;
+    }
+
+    const url = `https://${registry}/token?scope=repo:${id}:pull&service=${registry}`;
 
     const options = {
         type: 'GET',
         url: url,
         headers: headers
     };
 
-    const token = JSON.parse((await request(options, output)).toString()).token;
+    const authReq = await request(options, output);
+    if (!authReq) {
+        output.write('Failed to get registry auth token', LogLevel.Error);
+        return undefined;
+    }
 
+    const token = JSON.parse(authReq.toString())?.token;
+    if (!token) {
+        output.write('Failed to parse registry auth token response', LogLevel.Error);
+        return undefined;
+    }
     return token;
+}
+
+// -- Push
+
+// export async function pushOCIFeature(output: Log, env: NodeJS.ProcessEnv, featureSet: FeatureSet, ociCacheDir: string, featCachePath: string, featureRef: OCIFeatureRef): Promise<boolean> {
+//     const await validateOCIFeature(output, env, featureRef.id);
+// }
+
+export async function generateManifest(output: Log, pathToTgz: string): Promise<OCIManifest | undefined> {
+
+    const tgzLayer = await calculateTgzLayer(output, pathToTgz);
+    if (!tgzLayer) {
+        output.write(`Failed to calculate tgz layer.`, LogLevel.Error);
+        return undefined;
+    }
+
+    const { manifestObj, hash } = await calculateContentDigest(output, tgzLayer);
+    manifestObj.digest = `sha256:${hash}`;
+
+    return manifestObj;
+}
+
+export async function calculateTgzLayer(output: Log, pathToTgz: string): Promise<OCILayer | undefined> {
+    output.write(`Creating manifest from ${pathToTgz}`, LogLevel.Trace);
+    if (!(await isLocalFile(pathToTgz))) {
+        output.write(`${pathToTgz} does not exist.`, LogLevel.Error);
+        return undefined;
+    }
+
+    const tarBytes = fs.readFileSync(pathToTgz);
+
+    const tarSha256 = crypto.createHash('sha256').update(tarBytes).digest('hex');
+    output.write(`${pathToTgz}:  sha256:${tarSha256} (size: ${tarBytes.byteLength})`, LogLevel.Trace);
+
+    return {
+        mediaType: 'application/vnd.devcontainers.layer.v1+tar',
+        digest: `sha256:${tarSha256}`,
+        size: tarBytes.byteLength,
+        annotations: {
+            'org.opencontainers.image.title': path.basename(pathToTgz),
+        }
+    };
+}
+
+export async function calculateContentDigest(output: Log, tgzLayer: OCILayer) {
+    // {"schemaVersion":2,"mediaType":"application/vnd.oci.image.manifest.v1+json","config":{"mediaType":"application/vnd.devcontainers","digest":"sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","size":0},"layers":[{"mediaType":"application/vnd.devcontainers.layer.v1+tar","digest":"sha256:b2006e7647191f7b47222ae48df049c6e21a4c5a04acfad0c4ef614d819de4c5","size":15872,"annotations":{"org.opencontainers.image.title":"go.tgz"}}]}
+
+    let manifest: OCIManifest = {
+        schemaVersion: 2,
+        mediaType: 'application/vnd.oci.image.manifest.v1+json',
+        config: {
+            mediaType: 'application/vnd.devcontainers',
+            digest: 'sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855',
+            size: 0
+        },
+        layers: [
+            tgzLayer
+        ]
+    };
+
+    const manifestStringified = JSON.stringify(manifest);
+    const manifestHash = crypto.createHash('sha256').update(manifestStringified).digest('hex');
+    output.write(`manifest:  sha256:${manifestHash} (size: ${manifestHash.length})`, LogLevel.Trace);
+
+    return {
+        manifestStr: manifestStringified,
+        manifestObj: manifest,
+        hash: manifestHash,
+    };
 }

src/test/container-features/containerFeaturesOCI.offline.test.ts

@@ -1,10 +1,13 @@
+// import { assert } from 'chai';
+// import { getFeatureManifest, getFeatureBlob, getFeatureRef, createManifest } from '../../spec-configuration/containerFeaturesOCI';
 import { assert } from 'chai';
-import { getFeatureManifest, getFeatureBlob, getFeatureRef } from '../../spec-configuration/containerFeaturesOCI';
+import { calculateContentDigest, calculateTgzLayer, getFeatureBlob, getFeatureManifest, getFeatureRef, validateOCIFeature } from '../../spec-configuration/containerFeaturesOCI';
 import { createPlainLog, LogLevel, makeLog } from '../../spec-utils/log';
 
 export const output = makeLog(createPlainLog(text => process.stdout.write(text), () => LogLevel.Trace));
+const testAssetsDir = `${__dirname}/assets`;
 
-describe('Test OCI', () => {
+describe('Test OCI Pull', () => {
 
     it('Parse OCI identifier', async () => {
         const feat = getFeatureRef(output, 'ghcr.io/codspace/features/ruby:1');
@@ -45,4 +48,40 @@ describe('Test OCI', () => {
         const result = await getFeatureBlob(output, process.env, 'https://ghcr.io/v2/codspace/features/ruby/blobs/sha256:c33008d0dc12d0e631734082401bec692da809eae2ac51e24f58c1cac68fc0c9', '/tmp', '/tmp/featureTest', featureRef);
         assert.isTrue(result);
     });
+});
+
+
+//  NOTE: Test depends on https://github.com/codspace/features/pkgs/container/features%2Fgo/29819216?tag=1
+describe('Test Generate Manifests and Digests', () => {
+    it('Generates the correct tgz manifest layer', async () => {
+
+        // Calculate the tgz layer and digest
+        const res = await calculateTgzLayer(output, `${testAssetsDir}/go.tgz`);
+        const expected = {
+            digest: 'sha256:b2006e7647191f7b47222ae48df049c6e21a4c5a04acfad0c4ef614d819de4c5',
+            mediaType: 'application/vnd.devcontainers.layer.v1+tar',
+            size: 15872,
+            annotations: {
+                'org.opencontainers.image.title': 'go.tgz'
+            }
+        };
+
+        if (!res) {
+            assert.fail();
+        }
+        assert.deepEqual(res, expected);
+
+        // Generate entire manifest to be able to calculate content digest
+        const { manifestStr, hash } = await calculateContentDigest(output, res);
+
+        // 'Expected' is taken from intermediate value in oras reference implementation, before hash calculation
+        assert.strictEqual('{"schemaVersion":2,"mediaType":"application/vnd.oci.image.manifest.v1+json","config":{"mediaType":"application/vnd.devcontainers","digest":"sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","size":0},"layers":[{"mediaType":"application/vnd.devcontainers.layer.v1+tar","digest":"sha256:b2006e7647191f7b47222ae48df049c6e21a4c5a04acfad0c4ef614d819de4c5","size":15872,"annotations":{"org.opencontainers.image.title":"go.tgz"}}]}', manifestStr);
+
+        assert.strictEqual('9726054859c13377c4c3c3c73d15065de59d0c25d61d5652576c0125f2ea8ed3', hash);
+    });
+
+    it('Can fetch an artifact from a digest reference', async () => {
+        const manifest = await validateOCIFeature(output, process.env, 'ghcr.io/codspace/features/go', 'sha256:9726054859c13377c4c3c3c73d15065de59d0c25d61d5652576c0125f2ea8ed3');
+        assert.strictEqual(manifest?.layers[0].annotations['org.opencontainers.image.title'], 'go.tgz');
+    });
 });