Skip to content

Commit 933b8b7

Browse files
committed
Add pipeline.yaml
This is motivated by coreos/fedora-coreos-pipeline#618 Today the coreos-assembler default for `cosa compress` is `gzip`, and the RHCOS pipeline relies on that. The FCOS pipeline overrides it. My opinion is that this is exactly the kind of thing that should be configured in the source git - it should not live in "the pipeline". By having a new `pipeline.yaml` in this repository, we get closer to the goal of having the default invocations of `cosa $x` do the same thing as production pipelines. If accepted I plan to extend this with numerous other things; most notably for example the set of desired target artifacts i.e. ``` artifacts: - aws - metal - digitalocean - ... ``` etc. Then we can have `cosa buildextend-all`, etc. This PR includes some ugly bridging of Go logic to Python.
1 parent bc2a455 commit 933b8b7

25 files changed

+11610
-4
lines changed

cmd/coreos-assembler.go

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -95,6 +95,8 @@ func run(argv []string) error {
9595
case "build-extensions-container", // old alias
9696
"buildextend-extensions-container":
9797
return buildExtensionContainer()
98+
case "internals":
99+
return runInternals(argv)
98100
}
99101

100102
target := fmt.Sprintf("/usr/lib/coreos-assembler/cmd-%s", cmd)

cmd/internals.go

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
package main
2+
3+
import (
4+
"encoding/json"
5+
"fmt"
6+
7+
"github.com/spf13/cobra"
8+
9+
"github.com/coreos/coreos-assembler/pkg/pipeline"
10+
)
11+
12+
func runInternals(argv []string) error {
13+
cmdInternals := &cobra.Command{
14+
Use: "internals",
15+
Short: "Internal APIs of coreos-assembler",
16+
}
17+
18+
cmdGetPipelineJson := &cobra.Command{
19+
Use: "get-pipeline-json",
20+
Short: "Output pipeline config JSON",
21+
Args: cobra.ExactArgs(0),
22+
RunE: func(cmd *cobra.Command, args []string) error {
23+
config, err := pipeline.ReadConfig(".")
24+
if err != nil {
25+
return err
26+
}
27+
buf, err := json.Marshal(config)
28+
if err != nil {
29+
return err
30+
}
31+
if _, err := fmt.Printf("%s\n", string(buf)); err != nil {
32+
return err
33+
}
34+
return nil
35+
},
36+
}
37+
cmdInternals.AddCommand(cmdGetPipelineJson)
38+
39+
cmdInternals.SetArgs(argv)
40+
cmdInternals.Execute()
41+
42+
return nil
43+
}

docs/working.md

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,12 @@ partition or configuring the filesystem types. If you want to do anything like
5656
that today it requires forking the assembler and rebuilding it.
5757
See the fedora-coreos-config for an example.
5858

59+
### `pipeline.yaml`
60+
61+
This YAML file configures default values for various parts of coreos-assembler.
62+
At the moment, there is only one configuration knob for compression. Consult
63+
`/usr/lib/coreos-assembler/pipeline-default.yaml` for canonical documentation.
64+
5965
## Hacking on "config git"
6066

6167
First you can expand the size of the image; edit `src/config/image.yaml` and

go.mod

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ go 1.17
55
require (
66
github.com/coreos/stream-metadata-go v0.4.0
77
github.com/spf13/cobra v1.5.0
8+
gopkg.in/yaml.v3 v3.0.1
89
)
910

1011
require (

go.sum

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@ github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQ
3333
golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
3434
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f h1:v4INt8xihDGvnrfjMDVXGxw9wrfxYyCjk0KbXjhR55s=
3535
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
36+
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
3637
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
3738
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
3839
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

pkg/pipeline/config.go

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
package pipeline
2+
3+
import (
4+
"bytes"
5+
"io/ioutil"
6+
"os"
7+
"path/filepath"
8+
9+
yaml "gopkg.in/yaml.v3"
10+
)
11+
12+
// Config is default pipeline configuration, stored in
13+
// src/config/pipeline.yaml
14+
type Config struct {
15+
// Compressor can currently be one of "xz" or "gzip". The default
16+
// is gzip.
17+
Compressor string `json:"compressor"`
18+
}
19+
20+
func ReadConfig(workdir string) (*Config, error) {
21+
buf, err := ioutil.ReadFile("/usr/lib/coreos-assembler/pipeline-default.yaml")
22+
if err != nil {
23+
return nil, err
24+
}
25+
26+
var config Config
27+
dec := yaml.NewDecoder(bytes.NewReader(buf))
28+
dec.KnownFields(true)
29+
if err := dec.Decode(&config); err != nil {
30+
return nil, err
31+
}
32+
33+
buf, err = ioutil.ReadFile(filepath.Join(workdir, "src/config/pipeline.yaml"))
34+
if err != nil {
35+
if !os.IsNotExist(err) {
36+
return nil, err
37+
}
38+
} else {
39+
dec := yaml.NewDecoder(bytes.NewReader(buf))
40+
dec.KnownFields(true)
41+
if err := dec.Decode(&config); err != nil {
42+
return nil, err
43+
}
44+
}
45+
46+
return &config, nil
47+
}

src/cmd-compress

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ import sys
88
import json
99
import shutil
1010
import argparse
11+
import subprocess
1112

1213
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
1314
from cosalib.builds import Builds
@@ -18,8 +19,6 @@ from cosalib.cmdlib import (
1819
sha256sum_file,
1920
write_json)
2021

21-
DEFAULT_COMPRESSOR = 'gzip'
22-
2322
cmd = sys.argv[0].rsplit("/", 1)[1]
2423
if cmd == "cmd-uncompress" or cmd == "cmd-decompress":
2524
DEFAULT_MODE = 'uncompress'
@@ -32,8 +31,8 @@ parser.add_argument("--artifact", default=[], action='append',
3231
help="Only compress given image ARTIFACT", metavar="ARTIFACT")
3332
parser.add_argument("--compressor",
3433
choices=['xz', 'gzip'],
35-
default=DEFAULT_COMPRESSOR,
36-
help=f"Compressor to use, default is {DEFAULT_COMPRESSOR}")
34+
default=None,
35+
help="Compressor to use")
3736
parser.add_argument("--mode",
3837
choices=['compress', 'uncompress'],
3938
default=DEFAULT_MODE,
@@ -42,6 +41,10 @@ parser.add_argument("--fast", action='store_true',
4241
help="Override compression to `gzip -1`")
4342
args = parser.parse_args()
4443

44+
pipeline_config = json.loads(subprocess.check_output(["coreos-assembler", "internals", "get-pipeline-json"]))
45+
if args.compressor is None:
46+
args.compressor = pipeline_config["compressor"]
47+
4548
# that's the tool default
4649
gzip_level = 6
4750
if args.fast:

src/pipeline-default.yaml

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
# This file serves as defaults as well as documentation for pipeline.yaml.
2+
compressor: "gzip"

vendor/gopkg.in/yaml.v3/LICENSE

Lines changed: 50 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

vendor/gopkg.in/yaml.v3/NOTICE

Lines changed: 13 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

vendor/gopkg.in/yaml.v3/README.md

Lines changed: 150 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

0 commit comments

Comments
 (0)