Add pipeline.yaml

cmd/coreos-assembler.go

@@ -95,6 +95,8 @@ func run(argv []string) error {
 	case "build-extensions-container", // old alias
 		"buildextend-extensions-container":
 		return buildExtensionContainer()
+	case "internals":
+		return runInternals(argv)
 	}
 
 	target := fmt.Sprintf("/usr/lib/coreos-assembler/cmd-%s", cmd)

cmd/internals.go

@@ -0,0 +1,43 @@
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+
+	"github.com/spf13/cobra"
+
+	"github.com/coreos/coreos-assembler/pkg/pipeline"
+)
+
+func runInternals(argv []string) error {
+	cmdInternals := &cobra.Command{
+		Use:   "internals",
+		Short: "Internal APIs of coreos-assembler",
+	}
+
+	cmdGetPipelineJson := &cobra.Command{
+		Use:   "get-pipeline-json",
+		Short: "Output pipeline config JSON",
+		Args:  cobra.ExactArgs(0),
+		RunE: func(cmd *cobra.Command, args []string) error {
+			config, err := pipeline.ReadConfig(".")
+			if err != nil {
+				return err
+			}
+			buf, err := json.Marshal(config)
+			if err != nil {
+				return err
+			}
+			if _, err := fmt.Printf("%s\n", string(buf)); err != nil {
+				return err
+			}
+			return nil
+		},
+	}
+	cmdInternals.AddCommand(cmdGetPipelineJson)
+
+	cmdInternals.SetArgs(argv)
+	cmdInternals.Execute()
+
+	return nil
+}

docs/working.md

@@ -56,6 +56,12 @@ partition or configuring the filesystem types.  If you want to do anything like
 that today it requires forking the assembler and rebuilding it.
 See the fedora-coreos-config for an example.
 
+### `pipeline.yaml`
+
+This YAML file configures default values for various parts of coreos-assembler.
+At the moment, there is only one configuration knob for compression.  Consult
+`/usr/lib/coreos-assembler/pipeline-default.yaml` for canonical documentation.
+
 ## Hacking on "config git"
 
 First you can expand the size of the image; edit `src/config/image.yaml` and

go.mod

@@ -5,6 +5,7 @@ go 1.17
 require (
 	github.com/coreos/stream-metadata-go v0.4.0
 	github.com/spf13/cobra v1.5.0
+	gopkg.in/yaml.v3 v3.0.1
 )
 
 require (

go.sum

@@ -33,6 +33,7 @@ github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQ
 golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f h1:v4INt8xihDGvnrfjMDVXGxw9wrfxYyCjk0KbXjhR55s=
 golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

pkg/pipeline/config.go

@@ -0,0 +1,47 @@
+package pipeline
+
+import (
+	"bytes"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+
+	yaml "gopkg.in/yaml.v3"
+)
+
+// Config is default pipeline configuration, stored in
+// src/config/pipeline.yaml
+type Config struct {
+	// Compressor can currently be one of "xz" or "gzip".  The default
+	// is gzip.
+	Compressor string `json:"compressor"`
+}
+
+func ReadConfig(workdir string) (*Config, error) {
+	buf, err := ioutil.ReadFile("/usr/lib/coreos-assembler/pipeline-default.yaml")
+	if err != nil {
+		return nil, err
+	}
+
+	var config Config
+	dec := yaml.NewDecoder(bytes.NewReader(buf))
+	dec.KnownFields(true)
+	if err := dec.Decode(&config); err != nil {
+		return nil, err
+	}
+
+	buf, err = ioutil.ReadFile(filepath.Join(workdir, "src/config/pipeline.yaml"))
+	if err != nil {
+		if !os.IsNotExist(err) {
+			return nil, err
+		}
+	} else {
+		dec := yaml.NewDecoder(bytes.NewReader(buf))
+		dec.KnownFields(true)
+		if err := dec.Decode(&config); err != nil {
+			return nil, err
+		}
+	}
+
+	return &config, nil
+}

src/cmd-compress

@@ -8,6 +8,7 @@ import sys
 import json
 import shutil
 import argparse
+import subprocess
 
 sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
 from cosalib.builds import Builds
@@ -18,8 +19,6 @@ from cosalib.cmdlib import (
     sha256sum_file,
     write_json)
 
-DEFAULT_COMPRESSOR = 'gzip'
-
 cmd = sys.argv[0].rsplit("/", 1)[1]
 if cmd == "cmd-uncompress" or cmd == "cmd-decompress":
     DEFAULT_MODE = 'uncompress'
@@ -32,8 +31,8 @@ parser.add_argument("--artifact", default=[], action='append',
                     help="Only compress given image ARTIFACT", metavar="ARTIFACT")
 parser.add_argument("--compressor",
                     choices=['xz', 'gzip'],
-                    default=DEFAULT_COMPRESSOR,
-                    help=f"Compressor to use, default is {DEFAULT_COMPRESSOR}")
+                    default=None,
+                    help="Compressor to use")
 parser.add_argument("--mode",
                     choices=['compress', 'uncompress'],
                     default=DEFAULT_MODE,
@@ -42,6 +41,10 @@ parser.add_argument("--fast", action='store_true',
                     help="Override compression to `gzip -1`")
 args = parser.parse_args()
 
+pipeline_config = json.loads(subprocess.check_output(["coreos-assembler", "internals", "get-pipeline-json"]))
+if args.compressor is None:
+    args.compressor = pipeline_config["compressor"]
+
 # that's the tool default
 gzip_level = 6
 if args.fast:

src/pipeline-default.yaml

@@ -0,0 +1,2 @@
+# This file serves as defaults as well as documentation for pipeline.yaml.
+compressor: "gzip"