Add pipeline.yaml

This is motivated by
coreos/fedora-coreos-pipeline#618

Today the coreos-assembler default for `cosa compress` is `gzip`,
and the RHCOS pipeline relies on that.  The FCOS pipeline overrides
it.

My opinion is that this is exactly the kind of thing that should
be configured in the source git - it should not live in "the pipeline".

By having a new `pipeline.yaml` in this repository, we get closer
to the goal of having the default invocations of `cosa $x` do the
same thing as production pipelines.

If accepted I plan to extend this with numerous other things; most
notably for example the set of desired target artifacts i.e.

```
artifacts:
  - aws
  - metal
  - digitalocean
  - ...
```

etc.  Then we can have `cosa buildextend-all`, etc.

This PR includes some ugly bridging of Go logic to Python.

Author: cgwalters

cmd/coreos-assembler.go

@@ -92,6 +92,8 @@ func run(argv []string) error {
 		return runUpdateVariant(argv)
 	case "remote-session":
 		return runRemoteSession(argv)
+	case "internals":
+		return runInternals(argv)
 	case "build-extensions-container":
 		return buildExtensionContainer()
 	}

cmd/internals.go

@@ -0,0 +1,43 @@
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+
+	"github.com/spf13/cobra"
+
+	"github.com/coreos/coreos-assembler/pkg/pipeline"
+)
+
+func runInternals(argv []string) error {
+	cmdInternals := &cobra.Command{
+		Use:   "internals",
+		Short: "Internal APIs of coreos-assembler",
+	}
+
+	cmdGetPipelineJson := &cobra.Command{
+		Use:   "get-pipeline-json",
+		Short: "Output pipeline config JSON",
+		Args:  cobra.ExactArgs(0),
+		RunE: func(cmd *cobra.Command, args []string) error {
+			config, err := pipeline.ReadConfig(".")
+			if err != nil {
+				return err
+			}
+			buf, err := json.Marshal(config)
+			if err != nil {
+				return err
+			}
+			if _, err := fmt.Printf("%s\n", string(buf)); err != nil {
+				return err
+			}
+			return nil
+		},
+	}
+	cmdInternals.AddCommand(cmdGetPipelineJson)
+
+	cmdInternals.SetArgs(argv)
+	cmdInternals.Execute()
+
+	return nil
+}

docs/working.md

@@ -56,6 +56,12 @@ partition or configuring the filesystem types.  If you want to do anything like
 that today it requires forking the assembler and rebuilding it.
 See the fedora-coreos-config for an example.
 
+### `pipeline.yaml`
+
+This YAML file configures default values for various parts of coreos-assembler.
+At the moment, there is only one configuration knob for compression.  Consult
+`/usr/lib/coreos-assembler/pipeline-default.yaml` for canonical documentation.
+
 ## Hacking on "config git"
 
 First you can expand the size of the image; edit `src/config/image.yaml` and

go.mod

@@ -19,4 +19,5 @@ require (
 require (
 	github.com/inconshreveable/mousetrap v1.0.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
+	gopkg.in/yaml.v2 v2.4.0 // indirect
 )

go.sum

@@ -34,6 +34,7 @@ golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f h1:v4INt8xihDGvnrfjMDVXGxw9wrfxYyCjk0KbXjhR55s=
 golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
 gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=

pkg/pipeline/config.go

@@ -0,0 +1,42 @@
+package pipeline
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+
+	yaml "gopkg.in/yaml.v2"
+)
+
+// Config is default pipeline configuration, stored in
+// src/config/pipeline.yaml
+type Config struct {
+	// Compressor can currently be one of "xz" or "gzip".  The default
+	// is gzip.
+	Compressor string `json:"compressor"`
+}
+
+func ReadConfig(workdir string) (*Config, error) {
+	buf, err := ioutil.ReadFile("/usr/lib/coreos-assembler/pipeline-default.yaml")
+	if err != nil {
+		return nil, err
+	}
+
+	var config Config
+	if err := yaml.UnmarshalStrict(buf, &config); err != nil {
+		return nil, err
+	}
+
+	buf, err = ioutil.ReadFile(filepath.Join(workdir, "src/config/pipeline.yaml"))
+	if err != nil {
+		if !os.IsNotExist(err) {
+			return nil, err
+		}
+	} else {
+		if err := yaml.UnmarshalStrict(buf, &config); err != nil {
+			return nil, err
+		}
+	}
+
+	return &config, nil
+}

src/cmd-compress

@@ -8,6 +8,7 @@ import sys
 import json
 import shutil
 import argparse
+import subprocess
 
 sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
 from cosalib.builds import Builds
@@ -18,8 +19,6 @@ from cosalib.cmdlib import (
     sha256sum_file,
     write_json)
 
-DEFAULT_COMPRESSOR = 'gzip'
-
 cmd = sys.argv[0].rsplit("/", 1)[1]
 if cmd == "cmd-uncompress" or cmd == "cmd-decompress":
     DEFAULT_MODE = 'uncompress'
@@ -32,8 +31,8 @@ parser.add_argument("--artifact", default=[], action='append',
                     help="Only compress given image ARTIFACT", metavar="ARTIFACT")
 parser.add_argument("--compressor",
                     choices=['xz', 'gzip'],
-                    default=DEFAULT_COMPRESSOR,
-                    help=f"Compressor to use, default is {DEFAULT_COMPRESSOR}")
+                    default=None,
+                    help=f"Compressor to use")
 parser.add_argument("--mode",
                     choices=['compress', 'uncompress'],
                     default=DEFAULT_MODE,
@@ -42,6 +41,10 @@ parser.add_argument("--fast", action='store_true',
                     help="Override compression to `gzip -1`")
 args = parser.parse_args()
 
+pipeline_config = json.loads(subprocess.check_output(["coreos-assembler", "internals", "get-pipeline-json"]))
+if args.compressor is None:
+    args.compressor = pipeline_config["compressor"]
+
 # that's the tool default
 gzip_level = 6
 if args.fast:

src/pipeline-default.yaml

@@ -0,0 +1,2 @@
+# This file serves as defaults as well as documentation for pipeline.yaml.
+compressor: "gzip"