diff --git a/.gitignore b/.gitignore index 1306ab9..5287785 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,7 @@ /_output/ +<<<<<<< HEAD /mixtool +.vscode +======= +/mixtool +>>>>>>> bd0efc3... add mixtool list functionality (#28) diff --git a/cmd/mixtool/generate.go b/cmd/mixtool/generate.go index e431e02..94de0f4 100644 --- a/cmd/mixtool/generate.go +++ b/cmd/mixtool/generate.go @@ -151,7 +151,6 @@ func generateRules(filename string, options mixer.GenerateOptions) error { if err != nil { return err } - return ioutil.WriteFile(options.RulesFilename, out, 0644) } @@ -191,6 +190,14 @@ func generateDashboards(filename string, opts mixer.GenerateOptions) error { return nil } +func generateRulesAlerts(filename string, options mixer.GenerateOptions) ([]byte, error) { + out, err := mixer.GenerateRulesAlerts(filename, options) + if err != nil { + return nil, err + } + return out, nil +} + func generateAll(filename string, opts mixer.GenerateOptions) error { if err := generateAlerts(filename, opts); err != nil { return err diff --git a/cmd/mixtool/install.go b/cmd/mixtool/install.go new file mode 100644 index 0000000..a326284 --- /dev/null +++ b/cmd/mixtool/install.go @@ -0,0 +1,246 @@ +// Copyright 2018 mixtool authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "bytes" + "fmt" + "io/ioutil" + "net/http" + "net/url" + "os" + "path" + "path/filepath" + "strings" + + "github.com/monitoring-mixins/mixtool/pkg/jsonnetbundler" + "github.com/monitoring-mixins/mixtool/pkg/mixer" + + "github.com/urfave/cli" +) + +func installCommand() cli.Command { + return cli.Command{ + Name: "install", + Usage: "Install a mixin", + Description: "Install a mixin from a repository", + Action: installAction, + Flags: []cli.Flag{ + cli.StringFlag{ + Name: "bind-address", + Usage: "Address to bind HTTP server to.", + Value: "http://127.0.0.1:8080", + }, + cli.StringFlag{ + Name: "directory, d", + Usage: "Path where the downloaded mixin is saved. If it doesn't exist already it will be created", + }, + cli.BoolFlag{ + Name: "put, p", + Usage: "Specify this flag when you want to send PUT request to mixtool server once the mixins are generated", + }, + }, + } +} + +// Downloads a mixin from a given repository given by url and places into directory +// by running jb init and jb install +func downloadMixin(url string, jsonnetHome string, directory string) error { + // intialize the jsonnet bundler library + err := jsonnetbundler.InitCommand(directory) + if err != nil { + return fmt.Errorf("jsonnet bundler init failed %v", err) + } + + // by default, set the single flag to false + err = jsonnetbundler.InstallCommand(directory, jsonnetHome, []string{url}, false) + if err != nil { + return fmt.Errorf("jsonnet bundler install failed %v", err) + } + + return nil +} + +// Gets mixins from default website - mostly copied from list.go +func getMixins() ([]mixin, error) { + body, err := queryWebsite(defaultWebsite) + if err != nil { + return nil, err + } + mixins, err := parseMixinJSON(body) + if err != nil { + return nil, err + } + return mixins, nil +} + +func generateMixin(directory string, jsonnetHome string, mixinURL string, options mixer.GenerateOptions) ([]byte, error) { + + mixinBaseDirectory := filepath.Join(directory) + + err := os.Chdir(mixinBaseDirectory) + if err != nil { + return nil, fmt.Errorf("Cannot cd into directory %s", err) + } + + // generate alerts, rules, grafana dashboards + // empty files if not present + + u, err := url.Parse(mixinURL) + if err != nil { + return nil, fmt.Errorf("url parse %v", err) + } + + // absolute directory is the same as the download url stripped of the scheme + absDirectory := path.Join(u.Host, u.Path) + // trim http or www etc, trim repositories with trailing .git + // TODO: what if it's under some different kind of VCS? + absDirectory = strings.TrimLeft(absDirectory, "/:") + absDirectory = strings.TrimSuffix(absDirectory, ".git") + + importFile := filepath.Join(absDirectory, "mixin.libsonnet") + + // generate rules, dashboards, alerts + err = generateAll(importFile, options) + if err != nil { + return nil, fmt.Errorf("generateAll: %w", err) + } + + out, err := generateRulesAlerts(importFile, options) + if err != nil { + return nil, fmt.Errorf("generateRulesAlerts %w", err) + } + + return out, nil + +} + +func putMixin(content []byte, bindAddress string, mixinName string) error { + u, err := url.Parse(bindAddress) + if err != nil { + return err + } + pathName := fmt.Sprintf("/api/v1/rules/%s", mixinName) + u.Path = path.Join(u.Path, pathName) + + r := bytes.NewReader(content) + req, err := http.NewRequest("PUT", u.String(), r) + resp, err := http.DefaultClient.Do(req) + if err != nil { + return fmt.Errorf("response from server %v", err) + } + if resp.StatusCode != 200 { + responseData, err := ioutil.ReadAll(resp.Body) + if err != nil { + return fmt.Errorf("failed to response body in putMixin, %w", err) + } + return fmt.Errorf("non 200 response code: %d, info: %s", resp.StatusCode, string(responseData)) + } + return nil +} + +func installAction(c *cli.Context) error { + directory := c.String("directory") + if directory == "" { + return fmt.Errorf("Must specify a directory to download mixin") + } + + _, err := os.Stat(directory) + if os.IsNotExist(err) { + err = os.MkdirAll(directory, 0755) + if err != nil { + return fmt.Errorf("could not create directory %v", err) + } + } + + mixinPath := c.Args().First() + if mixinPath == "" { + return fmt.Errorf("Expected the url of mixin repository or name of the mixin. Show available mixins using mixtool list") + } + + mixinsList, err := getMixins() + if err != nil { + return fmt.Errorf("getMixins failed %v", err) + } + + var mixinURL string + if _, err := url.ParseRequestURI(mixinPath); err != nil { + // check if the name exists in mixinsList + found := false + for _, m := range mixinsList { + if m.Name == mixinPath { + // join paths together + u, err := url.Parse(m.URL) + if err != nil { + return fmt.Errorf("url parse failed %v", err) + } + u.Path = path.Join(u.Path, m.Subdir) + mixinURL = u.String() + found = true + break + } + } + if !found { + return fmt.Errorf("Could not find mixin with name %s", mixinPath) + } + } else { + mixinURL = mixinPath + } + + if mixinURL == "" { + return fmt.Errorf("Empty mixinURL") + } + + // by default jsonnet packages are downloaded under vendor + jsonnetHome := "vendor" + + err = downloadMixin(mixinURL, jsonnetHome, directory) + if err != nil { + return err + } + + generateCfg := mixer.GenerateOptions{ + AlertsFilename: "alerts.yaml", + RulesFilename: "rules.yaml", + Directory: "dashboards_out", + JPaths: []string{"./vendor"}, + YAML: true, + } + + rulesAlerts, err := generateMixin(directory, jsonnetHome, mixinURL, generateCfg) + if err != nil { + return err + } + + // check if put address flag was set + + if c.Bool("put") { + bindAddress := c.String("bind-address") + // run put requests onto the server + + // TODO: deal with the case where mixinPath is not a URL + _, err := url.ParseRequestURI(mixinPath) + if err == nil { + fmt.Println("TODO: for reloading prometheus, not dealing with mixins by url yet, since name is unknown") + } + + err = putMixin(rulesAlerts, bindAddress, mixinPath) + if err != nil { + return err + } + } + + return nil +} diff --git a/cmd/mixtool/install_test.go b/cmd/mixtool/install_test.go new file mode 100644 index 0000000..4fed349 --- /dev/null +++ b/cmd/mixtool/install_test.go @@ -0,0 +1,86 @@ +package main + +import ( + "fmt" + "io/ioutil" + "os" + "path" + "testing" + + "github.com/monitoring-mixins/mixtool/pkg/mixer" +) + +// Try to install every mixin from the mixin repository +// verify that each package generated has the yaml files +func TestInstallMixin(t *testing.T) { + tmpdir, err := ioutil.TempDir("", "mixtool-install") + if err != nil { + t.Errorf("failed to make directory %v", err) + } + + defer os.RemoveAll(tmpdir) + + body, err := queryWebsite(defaultWebsite) + if err != nil { + t.Errorf("failed to query website %v", err) + } + mixins, err := parseMixinJSON(body) + if err != nil { + t.Errorf("failed to parse mixin body %v", err) + } + + // download each mixin in turn + for _, m := range mixins { + + generateCfg := mixer.GenerateOptions{ + AlertsFilename: "alerts.yaml", + RulesFilename: "rules.yaml", + Directory: "dashboards_out", + JPaths: []string{"vendor"}, + YAML: true, + } + + mixinURL := path.Join(m.URL, m.Subdir) + + fmt.Printf("installing %v\n", mixinURL) + dldir := path.Join(tmpdir, m.Name+"mixin-test") + + err = os.Mkdir(dldir, 0755) + if err != nil { + t.Errorf("failed to create directory %s", dldir) + } + + jsonnetHome := "vendor" + + err = downloadMixin(mixinURL, jsonnetHome, dldir) + if err != nil { + t.Errorf("failed to download mixin at %s: %w", mixinURL, err) + } + + err = generateMixin(dldir, jsonnetHome, mixinURL, generateCfg) + if err != nil { + t.Errorf("failed to generate mixin yaml for %s: %w", mixinURL, err) + } + + // verify that alerts, rules, dashboards exist + err = os.Chdir(dldir) + if err != nil { + t.Errorf("could not cd into %s", dldir) + } + + if _, err := os.Stat("alerts.yaml"); os.IsNotExist(err) { + t.Errorf("expected alerts.yaml in %s", dldir) + } + + if _, err := os.Stat("rules.yaml"); os.IsNotExist(err) { + t.Errorf("expected rules.yaml in %s", dldir) + } + + if _, err := os.Stat("dashboards_out"); os.IsNotExist(err) { + t.Errorf("expected dashboards_out in %s", dldir) + } + + // verify that the output of alerts and rules matches using jsonnet + } + +} diff --git a/cmd/mixtool/lint.go b/cmd/mixtool/lint.go index 6531acc..107466a 100644 --- a/cmd/mixtool/lint.go +++ b/cmd/mixtool/lint.go @@ -79,6 +79,5 @@ func lintAction(c *cli.Context) error { if err := mixer.Lint(os.Stdout, filename, options); err != nil { return fmt.Errorf("failed to lint the file %s: %v", filename, err) } - return nil } diff --git a/cmd/mixtool/list.go b/cmd/mixtool/list.go new file mode 100644 index 0000000..acb897e --- /dev/null +++ b/cmd/mixtool/list.go @@ -0,0 +1,153 @@ +// Copyright 2018 mixtool authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "encoding/json" + "fmt" + "io/ioutil" + "net/http" + "net/url" + "os" + "text/tabwriter" + "time" + + "github.com/fatih/color" + "github.com/urfave/cli" +) + +type mixin struct { + URL string `json:"source"` + Description string `json:"description,omitempty"` + Name string `json:"name"` + Subdir string `json:"subdir"` +} + +const defaultWebsite = "https://monitoring.mixins.dev/mixins.json" + +func listCommand() cli.Command { + return cli.Command{ + Name: "list", + Usage: "List all available mixins", + Description: "List all available mixins as presented on monitoring.mixins.dev", + Action: listAction, + Flags: []cli.Flag{ + cli.StringFlag{ + Name: "path, p", + Usage: "provide the path of a url with a json endpoint or a local json file", + }, + }, + } +} + +func queryWebsite(mixinsWebsite string) ([]byte, error) { + client := http.Client{ + Timeout: time.Second * 10, + } + + req, err := http.NewRequest(http.MethodGet, mixinsWebsite, nil) + if err != nil { + return nil, err + } + + req.Header.Set("User-Agent", "mixtool-list") + res, err := client.Do(req) + if err != nil { + return nil, err + } + defer res.Body.Close() + body, err := ioutil.ReadAll(res.Body) + if err != nil { + return nil, err + } + + return body, nil +} + +func printMixins(mixinsList []mixin) error { + writer := tabwriter.NewWriter(os.Stdout, 4, 8, 0, '\t', tabwriter.TabIndent) + fmt.Fprintln(writer, "name") + fmt.Fprintln(writer, "----") + for _, m := range mixinsList { + // for now, do not print out any of the descriptions + // m.Description = strings.Replace(m.Description, "\n", "", -1) + // // maybe truncate the description if it's too long + // if len(m.Description) <= 0 { + // m.Description = "N/A" + // } + fmt.Fprintf(writer, "%s\n", color.GreenString(m.Name)) + } + + return writer.Flush() +} + +// ParsesMixinJSON expects a top level key mixins which contains a list of mixins +func parseMixinJSON(body []byte) ([]mixin, error) { + var mixins map[string][]mixin + if err := json.Unmarshal(body, &mixins); err != nil { + return nil, fmt.Errorf("failed to unmarshal json: %w", err) + } + mixinsList := mixins["mixins"] + return mixinsList, nil +} + +// if path is not specified, default to the mixins website +// otherwise, try parse as url +// otherwise, try look for a local json file +func listAction(c *cli.Context) error { + path := c.String("path") + var body []byte + var err error + if path == "" { + body, err = queryWebsite(defaultWebsite) + if err != nil { + return err + } + mixins, err := parseMixinJSON(body) + if err != nil { + return err + } + return printMixins(mixins) + } + + _, err = url.ParseRequestURI(path) + if err != nil { + // check if it's a local json file + _, err = os.Stat(path) + if err != nil { + return err + } + body, err = ioutil.ReadFile(path) + if err != nil { + return err + } + mixins, err := parseMixinJSON(body) + if err != nil { + return err + } + return printMixins(mixins) + } + + body, err = queryWebsite(path) + if err != nil { + return err + } + + mixins, err := parseMixinJSON(body) + if err != nil { + return err + } + return printMixins(mixins) +} diff --git a/cmd/mixtool/list_test.go b/cmd/mixtool/list_test.go new file mode 100644 index 0000000..b53fe58 --- /dev/null +++ b/cmd/mixtool/list_test.go @@ -0,0 +1,54 @@ +package main + +import ( + "io/ioutil" + "os" + "testing" +) + +const exampleMixins = ` +{ + "mixins": [ + { + "name": "ceph", + "source": "https://github.com/ceph/ceph-mixins", + "subdir": "", + "description": "A set of Prometheus alerts for Ceph.\n\nThe scope of this project is to provide Ceph specific Prometheus rule files using Prometheus Mixins.\n" + }, + { + "name": "cortex", + "source": "https://github.com/grafana/cortex-jsonnet", + "subdir": "cortex-mixin" + }, + { + "name": "cool-mixin", + "source": "https://github.com", + "subdir": "cool-mixin", + "description": "A fantastic mixin" + } + ] +} +` + +func TestList(t *testing.T) { + err := ioutil.WriteFile("exampleMixinsTest.json", []byte(exampleMixins), 0644) + if err != nil { + t.Errorf("failed to create temp file: %v", err) + } + defer os.Remove("exampleMixinsTest.json") + + body, err := ioutil.ReadFile("exampleMixinsTest.json") + if err != nil { + t.Errorf("failed to read exampleMixinsTest.json %v", err) + } + mixins, err := parseMixinJSON(body) + if err != nil { + t.Errorf("failed to read exampleMixinsTest.json %v", err) + } + exampleMixins := map[string]bool{"ceph": true, "cool-mixin": true, "cortex": true} + for _, m := range mixins { + if _, ok := exampleMixins[m.Name]; !ok { + t.Errorf("failed to find %v in exampleMixinsTest", m.Name) + } + } +} diff --git a/cmd/mixtool/main.go b/cmd/mixtool/main.go index d60bbce..96f26f6 100644 --- a/cmd/mixtool/main.go +++ b/cmd/mixtool/main.go @@ -39,6 +39,8 @@ func main() { lintCommand(), newCommand(), serverCommand(), + listCommand(), + installCommand(), // runbookCommand(), } diff --git a/cmd/mixtool/rules.yaml b/cmd/mixtool/rules.yaml new file mode 100644 index 0000000..a7a437a --- /dev/null +++ b/cmd/mixtool/rules.yaml @@ -0,0 +1,5 @@ +groups: +- name: general + rules: + - alert: Watchdog + expr: vector(1) diff --git a/cmd/mixtool/server.go b/cmd/mixtool/server.go index 19da711..d63d4ae 100644 --- a/cmd/mixtool/server.go +++ b/cmd/mixtool/server.go @@ -16,7 +16,6 @@ package main import ( "bufio" - "bytes" "context" "errors" "fmt" @@ -24,10 +23,15 @@ import ( "io/ioutil" "net/http" "os" + "path/filepath" + + "gopkg.in/yaml.v2" "github.com/urfave/cli" ) +const apiPath = "/api/v1/rules/" + func serverCommand() cli.Command { return cli.Command{ Name: "server", @@ -44,8 +48,8 @@ func serverCommand() cli.Command { Usage: "Prometheus address to reload after provisioning the rule file(s).", }, cli.StringFlag{ - Name: "rule-file", - Usage: "File to provision rules into.", + Name: "config-file", + Usage: "Prometheus configuration file", }, }, Action: serverAction, @@ -54,9 +58,9 @@ func serverCommand() cli.Command { func serverAction(c *cli.Context) error { bindAddress := c.String("bind-address") - http.Handle("/api/v1/rules", &ruleProvisioningHandler{ + http.Handle(apiPath, &ruleProvisioningHandler{ ruleProvisioner: &ruleProvisioner{ - ruleFile: c.String("rule-file"), + configFile: c.String("config-file"), }, prometheusReloader: &prometheusReloader{ prometheusReloadURL: c.String("prometheus-reload-url"), @@ -77,7 +81,10 @@ func (h *ruleProvisioningHandler) ServeHTTP(w http.ResponseWriter, r *http.Reque return } - reloadNecessary, err := h.ruleProvisioner.provision(r.Body) + // TODO: might not be the best place to put this + mixin := r.URL.Path[len(apiPath):] + + reloadNecessary, err := h.ruleProvisioner.provision(r.Body, mixin) if err != nil { http.Error(w, fmt.Sprintf("Internal Server Error: %v", err), http.StatusInternalServerError) return @@ -92,46 +99,116 @@ func (h *ruleProvisioningHandler) ServeHTTP(w http.ResponseWriter, r *http.Reque } type ruleProvisioner struct { - ruleFile string + configFile string } -// provision attempts to provision the rule files read from r, and if identical -// to existing, does not provision them. It returns whether Prometheus should -// be reloaded and if an error has occurred. -func (p *ruleProvisioner) provision(r io.Reader) (bool, error) { - b := bytes.NewBuffer(nil) - tr := io.TeeReader(r, b) - - f, err := os.Open(p.ruleFile) - if err != nil && !os.IsNotExist(err) { - return false, fmt.Errorf("open rule file: %w", err) - } - if os.IsNotExist(err) { - f, err = os.Create(p.ruleFile) - if err != nil { - return false, fmt.Errorf("create rule file: %w", err) +// PUT request +// /api/v1/rules/ +// specify mixin name +// filename determined by server +func (p *ruleProvisioner) provision(r io.Reader, mixinName string) (bool, error) { + newRules, err := ioutil.ReadAll(r) + if err != nil { + return false, fmt.Errorf("unable to read new rules: %w", err) + } + + mixinName = mixinName + ".yaml" + dir := filepath.Dir(p.configFile) + mixinFilename := filepath.Join(dir, mixinName) + + // if the filename under filepath.Join(dir, mixinName) already exists, do nothing + if _, err = os.Stat(mixinFilename); err == nil { + return true, nil + } + + f, err := os.OpenFile(mixinFilename, os.O_RDWR|os.O_CREATE, 0644) + if err != nil { + return false, fmt.Errorf("unable to create new mixin file: %w", err) + } + + // write all the contents into file + n, err := f.Write(newRules) + if err != nil { + return false, fmt.Errorf("error when writing new rules: %w", err) + } + if n != len(newRules) { + return false, fmt.Errorf("writing error, wrote %d bytes, expected %d", n, len(newRules)) + } + + f.Sync() + f.Close() + + // add file's name to config file + configBuf, err := ioutil.ReadFile(p.configFile) + if err != nil { + return false, fmt.Errorf("unable to open prometheus config file: %w", err) + } + + m := make(map[string]interface{}) + err = yaml.Unmarshal(configBuf, &m) + if err != nil { + return false, fmt.Errorf("unable to unmarshal prometheus config file: %w", err) + } + + for k, v := range m { + if k == "rule_files" { + // TODO: not entirely sure if this type assertion is safe + rulemap := v.([]interface{}) + rulemap = append(rulemap, mixinName) + m[k] = rulemap + break } } - equal, err := readersEqual(tr, f) + // create a temporary config file + tempfile, err := ioutil.TempFile(filepath.Dir(p.configFile), "temp-config") + + // marshal back into yaml + newConfig, err := yaml.Marshal(m) + if err != nil { + return false, fmt.Errorf("failed to marhsal yaml: %w", err) + } + + // write contents to temp config file + n, err = tempfile.Write(newConfig) if err != nil { - return false, fmt.Errorf("compare existing rules with provisioned intention: %w", err) + return false, fmt.Errorf("error when writing new rules: %w", err) } - if equal { - return false, nil + if n != len(newConfig) { + return false, fmt.Errorf("writing error, wrote %d bytes, expected %d", n, len(newConfig)) + } + + tempfile.Sync() + + configReader, err := os.OpenFile(p.configFile, os.O_RDONLY, 0644) + if err != nil { + return false, fmt.Errorf("unable to read existing config: %w", err) + } + + newConfigReader, err := os.OpenFile(tempfile.Name(), os.O_RDONLY, 0644) + if err != nil { + return false, fmt.Errorf("unable to open new config file: %w", err) } - if err := f.Truncate(0); err != nil { - return false, fmt.Errorf("truncate file: %w", err) + equal, err := readersEqual(configReader, newConfigReader) + if err != nil { + return false, fmt.Errorf("error from readersEqual: %w", err) } - if _, err := io.Copy(f, b); err != nil { - return false, fmt.Errorf("provision rule to file: %w", err) + if equal { + return false, nil } + if err = os.Rename(tempfile.Name(), p.configFile); err != nil { + return false, fmt.Errorf("cannot rename config file: %w", err) + } return true, nil } +type prometheusReloader struct { + prometheusReloadURL string +} + func readersEqual(r1, r2 io.Reader) (bool, error) { buf1 := bufio.NewReader(r1) buf2 := bufio.NewReader(r2) @@ -153,10 +230,6 @@ func readersEqual(r1, r2 io.Reader) (bool, error) { } } -type prometheusReloader struct { - prometheusReloadURL string -} - func (r *prometheusReloader) triggerReload(ctx context.Context) error { req, err := http.NewRequest("POST", r.prometheusReloadURL, nil) if err != nil { diff --git a/go.mod b/go.mod index c9dc2b0..0808caf 100644 --- a/go.mod +++ b/go.mod @@ -11,19 +11,23 @@ require ( github.com/go-macaron/inject v0.0.0-20160627170012-d8a0b8677191 // indirect github.com/go-macaron/session v0.0.0-20170320172209-b8e286a0dba8 // indirect github.com/gobuffalo/packr v1.30.1 - github.com/gobuffalo/packr/v2 v2.8.0 + github.com/gobuffalo/packr/v2 v2.8.1 github.com/google/go-jsonnet v0.16.1-0.20200908152747-b70cbd441a39 github.com/gosimple/slug v1.2.0 // indirect github.com/grafana/grafana v5.2.4+incompatible github.com/grafana/tanka v0.12.0 github.com/inconshreveable/log15 v0.0.0-20180818164646-67afb5ed74ec // indirect + github.com/jsonnet-bundler/jsonnet-bundler v0.4.0 github.com/pkg/errors v0.9.1 github.com/prometheus/prometheus v1.8.2-0.20200923143134-7e2db3d092f3 github.com/rainycape/unidecode v0.0.0-20150907023854-cb7f23ec59be // indirect github.com/teris-io/shortid v0.0.0-20171029131806-771a37caa5cf // indirect github.com/urfave/cli v1.22.1 + gopkg.in/alecthomas/kingpin.v2 v2.2.6 gopkg.in/bufio.v1 v1.0.0-20140618132640-567b2bfa514e // indirect gopkg.in/ini.v1 v1.38.2 // indirect gopkg.in/macaron.v1 v1.3.1 // indirect gopkg.in/redis.v2 v2.3.2 // indirect + gopkg.in/yaml.v2 v2.3.0 + sigs.k8s.io/yaml v1.2.0 ) diff --git a/go.sum b/go.sum index 7dcd3e0..cba331c 100644 --- a/go.sum +++ b/go.sum @@ -76,9 +76,11 @@ github.com/afex/hystrix-go v0.0.0-20180502004556-fa1af6a1f4f5/go.mod h1:SkGFH1ia github.com/agnivade/levenshtein v1.0.1/go.mod h1:CURSv5d9Uaml+FovSIICkLbAUZ9S4RqaHDIsdSBg7lM= github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af/go.mod h1:K08gAheRH3/J6wwsYMMT4xOr94bZjxIelGM0+d/wbFw= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 h1:JYp7IbQjafoB+tBA3gMyHYHrpOtNuDiK/uB5uXxq5wM= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d h1:UQZhZ2O0vMHr2cI+DC1Mbh0TJxzA3RcLoMsFw+aXw7E= github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= @@ -111,6 +113,7 @@ github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx2 github.com/bradfitz/gomemcache v0.0.0-20180710155616-bc664df96737 h1:rRISKWyXfVxvoa702s91Zl5oREZTrR3yv+tXrrX7G/g= github.com/bradfitz/gomemcache v0.0.0-20180710155616-bc664df96737/go.mod h1:PmM6Mmwb0LSuEubjR8N7PtNe1KxZLtOUHtbeikc5h60= github.com/c-bata/go-prompt v0.2.2/go.mod h1:VzqtzE2ksDBcdln8G7mk2RX9QyGjH+OVqOCSiVIqS34= +github.com/campoy/embedmd v1.0.0/go.mod h1:oxyr9RCiSXg0M3VJ3ks0UGfp98BpSSGr0kpiX3MzVl8= github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ= github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= github.com/cenkalti/backoff/v4 v4.0.2/go.mod h1:eEew/i+1Q6OrCDZh3WiXYv3+nJwBASZ8Bog/87DQnVg= @@ -313,6 +316,8 @@ github.com/gobuffalo/packr/v2 v2.5.1 h1:TFOeY2VoGamPjQLiNDT3mn//ytzk236VMO2j7iHx github.com/gobuffalo/packr/v2 v2.5.1/go.mod h1:8f9c96ITobJlPzI44jj+4tHnEKNt0xXWSVlXRN9X1Iw= github.com/gobuffalo/packr/v2 v2.8.0 h1:IULGd15bQL59ijXLxEvA5wlMxsmx/ZkQv9T282zNVIY= github.com/gobuffalo/packr/v2 v2.8.0/go.mod h1:PDk2k3vGevNE3SwVyVRgQCCXETC9SaONCNSXT1Q8M1g= +github.com/gobuffalo/packr/v2 v2.8.1 h1:tkQpju6i3EtMXJ9uoF5GT6kB+LMTimDWD8Xvbz6zDVA= +github.com/gobuffalo/packr/v2 v2.8.1/go.mod h1:c/PLlOuTU+p3SybaJATW3H6lX/iK7xEz5OeMf+NnJpg= github.com/gobuffalo/syncx v0.0.0-20190224160051-33c29581e754/go.mod h1:HhnNqWY95UYwwW3uSASeV7vtgYkT2t16hJgV3AEPUpw= github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8= github.com/gogo/googleapis v1.1.0/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s= @@ -389,6 +394,7 @@ github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGa github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= +github.com/gorilla/mux v1.7.3 h1:gnP5JzjVOuiZD07fKKToCAOjS0yOpj/qPETTXCCS6hw= github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= @@ -476,6 +482,8 @@ github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/u github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/jsonnet-bundler/jsonnet-bundler v0.4.0 h1:4BKZ6LDqPc2wJDmaKnmYD/vDjUptJtnUpai802MibFc= +github.com/jsonnet-bundler/jsonnet-bundler v0.4.0/go.mod h1:/by7P/OoohkI3q4CgSFqcoFsVY+IaNbzOVDknEsKDeU= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/jsternberg/zap-logfmt v1.0.0/go.mod h1:uvPs/4X51zdkcm5jXl5SYoN+4RK21K8mysFmDaM/h+o= @@ -491,6 +499,8 @@ github.com/karrick/godirwalk v1.10.12/go.mod h1:RoGL9dQei4vP9ilrpETWE8CLOZ1kiN0L github.com/karrick/godirwalk v1.15.3/go.mod h1:j4mkqPuvaLI8mp1DroR3P6ad7cyYd4c1qeJ3RV7ULlk= github.com/karrick/godirwalk v1.15.5 h1:ErdAEFW/cKxQ5+9Gm/hopxB8ki21/di+vyNb9mHnHrA= github.com/karrick/godirwalk v1.15.5/go.mod h1:j4mkqPuvaLI8mp1DroR3P6ad7cyYd4c1qeJ3RV7ULlk= +github.com/karrick/godirwalk v1.15.8 h1:7+rWAZPn9zuRxaIqqT8Ohs2Q2Ac0msBqwRdxNCr2VVs= +github.com/karrick/godirwalk v1.15.8/go.mod h1:j4mkqPuvaLI8mp1DroR3P6ad7cyYd4c1qeJ3RV7ULlk= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= @@ -538,6 +548,7 @@ github.com/mattn/go-colorable v0.1.6 h1:6Su7aK7lXmJ/U79bYtBjLNaha4Fs1Rg9plHpcH+v github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= +github.com/mattn/go-isatty v0.0.6/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84= github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE= @@ -930,6 +941,7 @@ golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20190209173611-3b5209105503/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190310054646-10058d7d4faa/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190321052220-f7bb7a8bee54/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1119,6 +1131,7 @@ google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2 google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.24.0 h1:UhZDfRO8JRQru4/+LlLE0BRKGF8L+PICnvYZmx/fEGA= google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= +gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/bufio.v1 v1.0.0-20140618132640-567b2bfa514e h1:wGA78yza6bu/mWcc4QfBuIEHEtc06xdiU0X8sY36yUU= gopkg.in/bufio.v1 v1.0.0-20140618132640-567b2bfa514e/go.mod h1:xsQCaysVCudhrYTfzYWe577fCe7Ceci+6qjO2Rdc0Z4= diff --git a/pkg/jsonnetbundler/init.go b/pkg/jsonnetbundler/init.go new file mode 100644 index 0000000..d522845 --- /dev/null +++ b/pkg/jsonnetbundler/init.go @@ -0,0 +1,39 @@ +package jsonnetbundler + +import ( + "encoding/json" + "fmt" + "io/ioutil" + "path/filepath" + + "github.com/jsonnet-bundler/jsonnet-bundler/pkg/jsonnetfile" + v1 "github.com/jsonnet-bundler/jsonnet-bundler/spec/v1" +) + +// InitCommand is basically the same as jb init +func InitCommand(dir string) error { + exists, err := jsonnetfile.Exists(jsonnetfile.File) + + if exists { + return fmt.Errorf("jsonnetfile.json already exists") + } + + s := v1.New() + // TODO: disable them by default eventually + // s.LegacyImports = false + + contents, err := json.MarshalIndent(s, "", " ") + if err != nil { + return fmt.Errorf("formatting jsonnetfile contents as json, %s", err.Error()) + } + contents = append(contents, []byte("\n")...) + + filename := filepath.Join(dir, jsonnetfile.File) + + err = ioutil.WriteFile(filename, contents, 0644) + if err != nil { + return fmt.Errorf("Failed to write new jsonnetfile.json, %s", err.Error()) + } + + return nil +} diff --git a/pkg/jsonnetbundler/install.go b/pkg/jsonnetbundler/install.go new file mode 100644 index 0000000..49ef6eb --- /dev/null +++ b/pkg/jsonnetbundler/install.go @@ -0,0 +1,134 @@ +// Copyright 2018 jsonnet-bundler authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package jsonnetbundler + +import ( + "encoding/json" + "fmt" + "io/ioutil" + "os" + "path/filepath" + "reflect" + + "github.com/pkg/errors" + + "github.com/jsonnet-bundler/jsonnet-bundler/pkg" + "github.com/jsonnet-bundler/jsonnet-bundler/pkg/jsonnetfile" + v1 "github.com/jsonnet-bundler/jsonnet-bundler/spec/v1" + "github.com/jsonnet-bundler/jsonnet-bundler/spec/v1/deps" +) + +func InstallCommand(dir, jsonnetHome string, uris []string, single bool) error { + if dir == "" { + dir = "." + } + + jbfilebytes, err := ioutil.ReadFile(filepath.Join(dir, jsonnetfile.File)) + if err != nil { + return fmt.Errorf("failed to load jsonnetfile %s", err.Error()) + } + + jsonnetFile, err := jsonnetfile.Unmarshal(jbfilebytes) + if err != nil { + return err + } + + jblockfilebytes, err := ioutil.ReadFile(filepath.Join(dir, jsonnetfile.LockFile)) + if !os.IsNotExist(err) { + if err != nil { + return fmt.Errorf("failed to load lockfile %s", err.Error()) + } + } + + lockFile, err := jsonnetfile.Unmarshal(jblockfilebytes) + if err != nil { + return err + } + + err = os.MkdirAll(filepath.Join(dir, jsonnetHome, ".tmp"), os.ModePerm) + if err != nil { + return fmt.Errorf("creating vendor folder %s", err.Error()) + } + + for _, u := range uris { + d := deps.Parse(dir, u) + if d == nil { + return fmt.Errorf("Unable to parse package URI %s", u) + } + + if single { + d.Single = true + } + + if !depEqual(jsonnetFile.Dependencies[d.Name()], *d) { + // the dep passed on the cli is different from the jsonnetFile + jsonnetFile.Dependencies[d.Name()] = *d + + // we want to install the passed version (ignore the lock) + delete(lockFile.Dependencies, d.Name()) + } + } + + jsonnetPkgHomeDir := filepath.Join(dir, jsonnetHome) + locked, err := pkg.Ensure(jsonnetFile, jsonnetPkgHomeDir, lockFile.Dependencies) + if err != nil { + return fmt.Errorf("failed to install packages %s", err) + } + + pkg.CleanLegacyName(jsonnetFile.Dependencies) + + err = writeChangedJsonnetFile(jbfilebytes, &jsonnetFile, filepath.Join(dir, jsonnetfile.File)) + if err != nil { + return fmt.Errorf("updating jsonnetfile.json %s", err) + } + + err = writeChangedJsonnetFile(jblockfilebytes, &v1.JsonnetFile{Dependencies: locked}, filepath.Join(dir, jsonnetfile.LockFile)) + if err != nil { + return fmt.Errorf("updating jsonnetfile.lock.json %s", err) + } + + return nil +} + +func depEqual(d1, d2 deps.Dependency) bool { + name := d1.Name() == d2.Name() + version := d1.Version == d2.Version + source := reflect.DeepEqual(d1.Source, d2.Source) + + return name && version && source +} + +func writeJSONFile(name string, d interface{}) error { + b, err := json.MarshalIndent(d, "", " ") + if err != nil { + return errors.Wrap(err, "encoding json") + } + b = append(b, []byte("\n")...) + + return ioutil.WriteFile(name, b, 0644) +} + +func writeChangedJsonnetFile(originalBytes []byte, modified *v1.JsonnetFile, path string) error { + origJsonnetFile, err := jsonnetfile.Unmarshal(originalBytes) + if err != nil { + return err + } + + if reflect.DeepEqual(origJsonnetFile, *modified) { + return nil + } + + return writeJSONFile(path, *modified) +} diff --git a/pkg/mixer/eval.go b/pkg/mixer/eval.go index e1a7922..606eeb2 100644 --- a/pkg/mixer/eval.go +++ b/pkg/mixer/eval.go @@ -24,7 +24,7 @@ func evaluatePrometheusAlerts(vm *jsonnet.VM, filename string) (string, error) { snippet := fmt.Sprintf(` local mixin = (import %q); -if std.objectHas(mixin, "prometheusAlerts") +if std.objectHasAll(mixin, "prometheusAlerts") then mixin.prometheusAlerts else {} `, filename) @@ -36,7 +36,7 @@ func evaluatePrometheusRules(vm *jsonnet.VM, filename string) (string, error) { snippet := fmt.Sprintf(` local mixin = (import %q); -if std.objectHas(mixin, "prometheusRules") +if std.objectHasAll(mixin, "prometheusRules") then mixin.prometheusRules else {} `, filename) @@ -44,11 +44,27 @@ else {} return vm.EvaluateSnippet("", snippet) } +func evaluatePrometheusRulesAlerts(vm *jsonnet.VM, filename string) (string, error) { + snippet := fmt.Sprintf(` +local mixin = (import %q); + +if std.objectHasAll(mixin, "prometheusRules") && std.objectHasAll(mixin, "prometheusAlerts") +then mixin.prometheusRules + mixin.prometheusAlerts +else if std.objectHasAll(mixin, "prometheusRules") +then mixin.prometheusRules +else if std.objectHasAll(mixin, "prometheusAlerts") +then mixin.prometheusAlerts +else {} +`, filename) + + return vm.EvaluateSnippet("", snippet) +} + func evaluateGrafanaDashboards(vm *jsonnet.VM, filename string) (string, error) { snippet := fmt.Sprintf(` local mixin = (import %q); -if std.objectHas(mixin, "grafanaDashboards") +if std.objectHasAll(mixin, "grafanaDashboards") then mixin.grafanaDashboards else {} `, filename) diff --git a/pkg/mixer/generate.go b/pkg/mixer/generate.go index 834ce3a..f731be2 100644 --- a/pkg/mixer/generate.go +++ b/pkg/mixer/generate.go @@ -82,6 +82,26 @@ func GenerateRules(filename string, opts GenerateOptions) ([]byte, error) { return output, nil } +func GenerateRulesAlerts(filename string, opts GenerateOptions) ([]byte, error) { + vm := NewVM(opts.JPaths) + + j, err := evaluatePrometheusRulesAlerts(vm, filename) + if err != nil { + return nil, err + } + + output := []byte(j) + + if opts.YAML { + output, err = yaml.JSONToYAML(output) + if err != nil { + return nil, err + } + } + + return output, nil +} + func GenerateDashboards(filename string, opts GenerateOptions) (map[string]json.RawMessage, error) { vm := NewVM(opts.JPaths)