create-legacy-oscontainer: use runvm to build legacy oscontainer

cmd/coreos-assembler.go

@@ -13,7 +13,7 @@ import (
 
 // commands we'd expect to use in the local dev path
 var buildCommands = []string{"init", "fetch", "build", "run", "prune", "clean", "list"}
-var advancedBuildCommands = []string{"buildfetch", "buildupload", "oc-adm-release", "push-container", "upload-oscontainer", "buildextend-extensions"}
+var advancedBuildCommands = []string{"buildfetch", "buildupload", "oc-adm-release", "push-container", "upload-oscontainer", "buildextend-extensions", "create-legacy-oscontainer"}
 var buildextendCommands = []string{"aliyun", "aws", "azure", "digitalocean", "exoscale", "gcp", "ibmcloud", "kubevirt", "live", "metal", "metal4k", "nutanix", "openstack", "qemu", "secex", "virtualbox", "vmware", "vultr"}
 var utilityCommands = []string{"aws-replicate", "build-extensions-container", "compress", "generate-hashlist", "koji-upload", "kola", "push-container-manifest", "remote-build-container", "remote-prune", "remote-session", "sign", "update-variant"}
 var otherCommands = []string{"shell", "meta"}

docs/cosa.md

@@ -72,3 +72,4 @@ Those less commonly used commands are listed here:
 | [tag](https://github.com/coreos/coreos-assembler/blob/main/src/cmd-tag) | Operate on the tags in `builds.json`
 | [test-coreos-installer](https://github.com/coreos/coreos-assembler/blob/main/src/cmd-test-coreos-installer) | Automate an end-to-end run of coreos-installer with the metal image
 | [upload-oscontainer](https://github.com/coreos/coreos-assembler/blob/main/src/cmd-upload-oscontainer) | Upload an oscontainer (historical wrapper for `cosa oscontainer`)
+| [create-legacy-oscontainer](https://github.com/coreos/coreos-assembler/blob/main/src/cmd-create-legacy-oscontainer) | Create an oscontainer in legacy format (i.e. not OSTree-native)

pkg/builds/cosa_v1.go

@@ -1,7 +1,7 @@
 package builds
 
 // generated by 'make schema'
-// source hash: 144450d458f89f637ca487d353af3dfd60096ddbf3179da8e2b42b2bd2d0a6eb
+// source hash: 3508b2f150e72b8e24151d870789809cf4070cec6b4716966a4e8bc585e0c5f1
 
 type AdvisoryDiff []AdvisoryDiffItems
 
@@ -101,6 +101,7 @@ type BuildArtifacts struct {
 	Iso                 *Artifact `json:"iso,omitempty"`
 	Kernel              *Artifact `json:"kernel,omitempty"`
 	KubeVirt            *Artifact `json:"kubevirt,omitempty"`
+	LegacyOscontainer   *Artifact `json:"legacy-oscontainer,omitempty"`
 	LiveInitramfs       *Artifact `json:"live-initramfs,omitempty"`
 	LiveIso             *Artifact `json:"live-iso,omitempty"`
 	LiveKernel          *Artifact `json:"live-kernel,omitempty"`

pkg/builds/schema_doc.go

@@ -1,5 +1,5 @@
 // Generated by ./generate-schema.sh
-// Source hash: 144450d458f89f637ca487d353af3dfd60096ddbf3179da8e2b42b2bd2d0a6eb
+// Source hash: 3508b2f150e72b8e24151d870789809cf4070cec6b4716966a4e8bc585e0c5f1
 // DO NOT EDIT
 
 package builds
@@ -446,6 +446,7 @@ var generatedSchemaJSON = `{
         "digitalocean",
         "exoscale",
         "extensions-container",
+        "legacy-oscontainer",
         "gcp",
         "kubevirt",
         "ibmcloud",
@@ -492,6 +493,12 @@ var generatedSchemaJSON = `{
           "title": "extensions-container",
           "$ref": "#/definitions/artifact"
         },
+        "legacy-oscontainer": {
+          "$id": "#/properties/images/properties/legacy-oscontainer",
+          "type": "object",
+          "title": "legacy-oscontainer",
+          "$ref": "#/definitions/artifact"
+        },
         "qemu": {
           "$id": "#/properties/images/properties/qemu",
           "type": "object",

src/cmd-create-legacy-oscontainer

@@ -0,0 +1,133 @@
+#!/usr/bin/python3 -u
+# Upload an oscontainer.  This is a wrapper for
+# `cosa oscontainer` that just for historical reasons
+# used to live downstream in the redhat-coreos pipeline.
+# In the future we should just have one `cosa oscontainer`
+# command.
+
+import argparse
+import json
+import yaml
+import os
+import shutil
+import subprocess
+import sys
+from cosalib.cmdlib import sha256sum_file
+
+cosa_dir = os.path.dirname(os.path.abspath(__file__))
+sys.path.insert(0, cosa_dir)
+
+from cosalib import cmdlib
+
+
+with open('builds/builds.json') as f:
+    builds = json.load(f)['builds']
+if len(builds) == 0:
+    cmdlib.fatal("No builds found")
+latest_build = builds[0]['id']
+arch = cmdlib.get_basearch()
+latest_build_path = f"builds/{latest_build}/{arch}"
+
+metapath = f"{latest_build_path}/meta.json"
+with open(metapath) as f:
+    meta = json.load(f)
+
+name = meta['name'] + '-' + meta['buildid'] + '-oscontainer.' + arch + '.ociarchive'
+parser = argparse.ArgumentParser()
+parser.add_argument("--arch-tag", help="append arch name to push tag",
+                    action='store_true')
+parser.add_argument("--name", help="oscontainer name",
+                    action='store', default=f'{name}')
+parser.add_argument("--from", help="Base image", default='scratch',
+                    dest='from_image')
+parser.add_argument("--format", help="Format to use for push")
+parser.add_argument("--add-directory", help="Copy in all content from referenced directory DIR",
+                    metavar='DIR', action='append', default=[])
+
+args = parser.parse_args()
+
+# for backcompat, we auto-build extensions if they're missing
+if os.path.exists('src/config/extensions.yaml'):
+    if 'extensions' not in meta:
+        cmdlib.runcmd(['coreos-assembler', 'buildextend-extensions'])
+        with open(metapath) as f:
+            meta = json.load(f)
+    assert 'extensions' in meta
+
+configdir = os.path.abspath('src/config')
+oscconfigpath = f'{configdir}/oscontainer.yaml'
+# XXX: fold oscontainer.yaml handling into oscontainer.py
+configyaml = {}
+if os.path.exists(oscconfigpath):
+    with open(oscconfigpath) as f:
+        configyaml = yaml.safe_load(f)
+
+if 'base' in configyaml:
+    args.from_image = configyaml['base']
+
+print("Preparing to upload oscontainer for build: {}".format(latest_build))
+ostree_commit = meta['ostree-commit']
+
+tmprepo = "{}/tmp/repo".format(os.getcwd())
+# if tmprepo is not a directory, but is unexpectedly a file,
+# just nuke it
+if not os.path.isdir(tmprepo) and os.path.exists(tmprepo):
+    os.remove(tmprepo)
+
+# if tmprepo is not a directory and not a file, recreate from
+# the tarfile
+if not os.path.exists(tmprepo):
+    os.makedirs(tmprepo, exist_ok=True)
+    ostree_commit_tar = meta['images']['ostree']['path']
+    subprocess.check_call(['tar', '-xf',
+                           f'{latest_build_path}/{ostree_commit_tar}',
+                           '-C', tmprepo])
+
+tmp_osreleasedir = 'tmp/usrlib-osrelease'
+subprocess.check_call(['rm', '-rf', tmp_osreleasedir])
+cmdlib.runcmd(['/usr/bin/ostree', 'checkout', '--repo', tmprepo,
+              '--user-mode', '--subpath=/usr/lib/os-release', ostree_commit,
+              tmp_osreleasedir])
+display_name = None
+with open(os.path.join(tmp_osreleasedir, "os-release")) as f:
+    display_name = subprocess.check_output(['/bin/sh', '-c', 'set -euo pipefail; . /proc/self/fd/0 && echo $NAME'], stdin=f, encoding='UTF-8').strip()
+if display_name == "":
+    raise SystemExit(f"Failed to find NAME= in /usr/lib/os-release in commit {ostree_commit}")
+shutil.rmtree(tmp_osreleasedir)
+
+osc_name_and_tag = f"{args.name}:{latest_build}"
+if args.arch_tag:
+    arch = meta.get("coreos-assembler.basearch", cmdlib.get_basearch)
+    osc_name_and_tag = f"{args.name}:{latest_build}-{arch}"
+
+# TODO: Use labels for the build hash and avoid pulling the oscontainer
+# every time we want to poll.
+# TODO: Remove --from
+digestfile = "tmp/oscontainer-digest"
+print("Entering vm to build oscontainer for build: {}".format(latest_build))
+
+cosa_argv = (['/usr/lib/coreos-assembler/create-legacy-oscontainer.sh', '--workdir=./tmp', 'build', f'--from={args.from_image}'])
+for d in args.add_directory:
+    cosa_argv.append(f'--add-directory="{d}"')
+cosa_argv.append(f'--display-name="{display_name}"')
+if 'labeled-packages' in configyaml:
+    pkgs = ' '.join(configyaml['labeled-packages'])
+    cosa_argv.append(f'--labeled-packages="{pkgs}"')
+if args.format is not None:
+    cosa_argv.append(f'--format={args.format}')
+subprocess.check_call(cosa_argv +
+    [f'--digestfile={digestfile}',
+        '--push', tmprepo,
+        meta['ostree-commit'],
+        osc_name_and_tag])
+
+# Inject the oscontainer with SHA256 into the build metadata
+oci_archive = f"{latest_build_path}/{args.name}"
+meta['images']['legacy-oscontainer'] = {'path': args.name,
+                                        'sha256': sha256sum_file(oci_archive),
+                                        'size': os.path.getsize(oci_archive),
+                                        "skip-compression": True}
+metapath_new = f"{metapath}.new"
+with open(metapath_new, 'w') as f:
+    json.dump(meta, f, sort_keys=True)
+shutil.move(metapath_new, metapath)