From 0afb64baa789b543b656802d06db94cbf6107dd2 Mon Sep 17 00:00:00 2001 From: Allen Ma Date: Thu, 29 Oct 2020 17:22:31 -0400 Subject: [PATCH 01/20] add mixtool list functionality --- cmd/mixtool/list.go | 149 ++++++++++++++++++++++++++++++++++++++++++++ cmd/mixtool/main.go | 1 + 2 files changed, 150 insertions(+) create mode 100644 cmd/mixtool/list.go diff --git a/cmd/mixtool/list.go b/cmd/mixtool/list.go new file mode 100644 index 0000000..18f74ba --- /dev/null +++ b/cmd/mixtool/list.go @@ -0,0 +1,149 @@ +// Copyright 2018 mixtool authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "encoding/json" + "fmt" + "io/ioutil" + "net/http" + "net/url" + "os" + "strings" + "text/tabwriter" + "time" + + "github.com/fatih/color" + "github.com/pkg/errors" + + "github.com/urfave/cli" +) + +type mixin struct { + URL string `json:"source"` + Description string `json:"description,omitempty"` + Name string `json:"name"` + Subdir string `json:"subdir"` +} + +type mixins struct { + d map[string][]mixin +} + +func listCommand() cli.Command { + return cli.Command{ + Name: "list", + Usage: "List all available mixins", + Description: "List all available mixins as presented on monitoring.mixins.dev", + Action: listAction, + Flags: []cli.Flag{ + cli.StringFlag{ + Name: "path, p", + Usage: "provide the path of a url with a json endpoint or a local json file", + }, + }, + } +} + +func queryWebsite(mixinsWebsite string) ([]byte, error) { + + client := http.Client{ + Timeout: time.Second * 3, + } + + req, err := http.NewRequest(http.MethodGet, mixinsWebsite, nil) + if err != nil { + return nil, err + } + + req.Header.Set("User-Agent", "mixtool-list") + res, err := client.Do(req) + if err != nil { + return nil, err + } + + if res.Body != nil { + defer res.Body.Close() + } + + body, err := ioutil.ReadAll(res.Body) + if err != nil { + return nil, err + } + + return body, nil +} + +func listAction(c *cli.Context) error { + + // if path is not specified, default to the mixins website + // otherwise, try parse as url + // otherwise, try look for a local json file + + defaultWebsite := "https://monitoring.mixins.dev/mixins.json" + + path := c.String("path") + var body []byte + var err error + if path == "" { + body, err = queryWebsite(defaultWebsite) + if err != nil { + return err + } + } else { + // check if it's a url + _, err := url.ParseRequestURI(path) + if err != nil { + // check if it's a local json file + _, err = os.Stat(path) + if err != nil { + return err + } + body, err = ioutil.ReadFile(path) + if err != nil { + return err + } + } else { + body, err = queryWebsite(path) + if err != nil { + return err + } + } + } + + var mixins map[string][]mixin + jsonErr := json.Unmarshal(body, &mixins) + + if jsonErr != nil { + return errors.New("failed to unmarshal json. Maybe your json schema is incorrect?") + } + + mixinsList := mixins["mixins"] + + writer := tabwriter.NewWriter(os.Stdout, 4, 8, 0, '\t', tabwriter.TabIndent) + fmt.Fprintln(writer, "name\tdescription") + fmt.Fprintln(writer, "----\t----------") + // print all the mixins out + for _, m := range mixinsList { + // chop off the description if it's too long + m.Description = strings.Replace(m.Description, "\n", "", -1) + if len(m.Description) <= 0 { + m.Description = "N/A" + } + fmt.Fprintf(writer, "%s\t%s\n", color.GreenString(m.Name), m.Description) + } + + return writer.Flush() +} diff --git a/cmd/mixtool/main.go b/cmd/mixtool/main.go index d60bbce..eb6404b 100644 --- a/cmd/mixtool/main.go +++ b/cmd/mixtool/main.go @@ -39,6 +39,7 @@ func main() { lintCommand(), newCommand(), serverCommand(), + listCommand(), // runbookCommand(), } From 41ea95c6906f81ef2423ddab29e32f2b639cb54f Mon Sep 17 00:00:00 2001 From: Allen Ma Date: Fri, 30 Oct 2020 16:58:09 -0400 Subject: [PATCH 02/20] add install skeleton --- .gitignore | 1 + cmd/mixtool/install.go | 73 ++++++++++++++++++++++++++++++++++++++++++ cmd/mixtool/main.go | 1 + cmd/mixtool/rules.yaml | 5 +++ cmd/mixtool/server.go | 8 +++++ 5 files changed, 88 insertions(+) create mode 100644 cmd/mixtool/install.go create mode 100644 cmd/mixtool/rules.yaml diff --git a/.gitignore b/.gitignore index 1306ab9..64fb839 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ /_output/ /mixtool +.vscode diff --git a/cmd/mixtool/install.go b/cmd/mixtool/install.go new file mode 100644 index 0000000..3915d59 --- /dev/null +++ b/cmd/mixtool/install.go @@ -0,0 +1,73 @@ +// Copyright 2018 mixtool authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "fmt" + + "github.com/urfave/cli" +) + +// type mixin struct { +// URL string `json:"source"` +// Description string `json:"description,omitempty"` +// Name string `json:"name"` +// Subdir string `json:"subdir"` +// } + +// type mixins struct { +// d map[string][]mixin +// } + +func installCommand() cli.Command { + return cli.Command{ + Name: "install", + Usage: "Install a mixin", + Description: "Install a mixin from a repository", + Action: installAction, + Flags: []cli.Flag{ + cli.StringFlag{ + Name: "bind-address", + Usage: "Address to bind HTTP server to.", + }, + cli.StringFlag{ + Name: "prometheus", + Value: "http://127.0.0.1:9090/", + Usage: "location of the prometheus server", + }, + }, + } +} + +func installAction(c *cli.Context) error { + filename := c.Args().First() + if filename == "" { + return fmt.Errorf("expected one argument, the name of the mixin. Show available mixins using mixtool list") + } + + // process: + + // check if the name of the mixin exists in mixtool list + + // if not, check if the mixin url is valid - if so, interpret as a url to a repo + + // run jb install the mixin + + // run mixtool generate all + + // read files and run mixtool server + + return nil +} diff --git a/cmd/mixtool/main.go b/cmd/mixtool/main.go index eb6404b..96f26f6 100644 --- a/cmd/mixtool/main.go +++ b/cmd/mixtool/main.go @@ -40,6 +40,7 @@ func main() { newCommand(), serverCommand(), listCommand(), + installCommand(), // runbookCommand(), } diff --git a/cmd/mixtool/rules.yaml b/cmd/mixtool/rules.yaml new file mode 100644 index 0000000..a7a437a --- /dev/null +++ b/cmd/mixtool/rules.yaml @@ -0,0 +1,5 @@ +groups: +- name: general + rules: + - alert: Watchdog + expr: vector(1) diff --git a/cmd/mixtool/server.go b/cmd/mixtool/server.go index 19da711..5a84fb3 100644 --- a/cmd/mixtool/server.go +++ b/cmd/mixtool/server.go @@ -84,6 +84,7 @@ func (h *ruleProvisioningHandler) ServeHTTP(w http.ResponseWriter, r *http.Reque } if reloadNecessary { + fmt.Println("reloading prometheus") if err := h.prometheusReloader.triggerReload(ctx); err != nil { http.Error(w, fmt.Sprintf("Internal Server Error: %v", err), http.StatusInternalServerError) return @@ -99,11 +100,13 @@ type ruleProvisioner struct { // to existing, does not provision them. It returns whether Prometheus should // be reloaded and if an error has occurred. func (p *ruleProvisioner) provision(r io.Reader) (bool, error) { + fmt.Println("trying to provision 1") b := bytes.NewBuffer(nil) tr := io.TeeReader(r, b) f, err := os.Open(p.ruleFile) if err != nil && !os.IsNotExist(err) { + fmt.Println(err, p.ruleFile) return false, fmt.Errorf("open rule file: %w", err) } if os.IsNotExist(err) { @@ -121,7 +124,10 @@ func (p *ruleProvisioner) provision(r io.Reader) (bool, error) { return false, nil } + fmt.Println("trying to provision 2") + if err := f.Truncate(0); err != nil { + fmt.Println("err is", err) return false, fmt.Errorf("truncate file: %w", err) } @@ -133,6 +139,7 @@ func (p *ruleProvisioner) provision(r io.Reader) (bool, error) { } func readersEqual(r1, r2 io.Reader) (bool, error) { + fmt.Println("comparing rn") buf1 := bufio.NewReader(r1) buf2 := bufio.NewReader(r2) for { @@ -158,6 +165,7 @@ type prometheusReloader struct { } func (r *prometheusReloader) triggerReload(ctx context.Context) error { + fmt.Println("triggering reload") req, err := http.NewRequest("POST", r.prometheusReloadURL, nil) if err != nil { return fmt.Errorf("create request: %w", err) From 27cd25d578eff4bb148fe57a538e6302986ee989 Mon Sep 17 00:00:00 2001 From: Allen Ma Date: Fri, 30 Oct 2020 17:44:50 -0400 Subject: [PATCH 03/20] syntactical changes to mixtoo list; cleanup code; omitted descriptions --- cmd/mixtool/list.go | 50 +++++++++++++++------------------------------ 1 file changed, 17 insertions(+), 33 deletions(-) diff --git a/cmd/mixtool/list.go b/cmd/mixtool/list.go index 18f74ba..4e30052 100644 --- a/cmd/mixtool/list.go +++ b/cmd/mixtool/list.go @@ -21,13 +21,10 @@ import ( "net/http" "net/url" "os" - "strings" "text/tabwriter" "time" "github.com/fatih/color" - "github.com/pkg/errors" - "github.com/urfave/cli" ) @@ -38,9 +35,7 @@ type mixin struct { Subdir string `json:"subdir"` } -type mixins struct { - d map[string][]mixin -} +const defaultWebsite = "https://monitoring.mixins.dev/mixins.json" func listCommand() cli.Command { return cli.Command{ @@ -58,9 +53,8 @@ func listCommand() cli.Command { } func queryWebsite(mixinsWebsite string) ([]byte, error) { - client := http.Client{ - Timeout: time.Second * 3, + Timeout: time.Second * 10, } req, err := http.NewRequest(http.MethodGet, mixinsWebsite, nil) @@ -73,11 +67,7 @@ func queryWebsite(mixinsWebsite string) ([]byte, error) { if err != nil { return nil, err } - - if res.Body != nil { - defer res.Body.Close() - } - + defer res.Body.Close() body, err := ioutil.ReadAll(res.Body) if err != nil { return nil, err @@ -86,14 +76,10 @@ func queryWebsite(mixinsWebsite string) ([]byte, error) { return body, nil } +// if path is not specified, default to the mixins website +// otherwise, try parse as url +// otherwise, try look for a local json file func listAction(c *cli.Context) error { - - // if path is not specified, default to the mixins website - // otherwise, try parse as url - // otherwise, try look for a local json file - - defaultWebsite := "https://monitoring.mixins.dev/mixins.json" - path := c.String("path") var body []byte var err error @@ -124,25 +110,23 @@ func listAction(c *cli.Context) error { } var mixins map[string][]mixin - jsonErr := json.Unmarshal(body, &mixins) - - if jsonErr != nil { - return errors.New("failed to unmarshal json. Maybe your json schema is incorrect?") + if err := json.Unmarshal(body, &mixins); err != nil { + return fmt.Errorf("failed to unmarshal json: %w", err) } mixinsList := mixins["mixins"] writer := tabwriter.NewWriter(os.Stdout, 4, 8, 0, '\t', tabwriter.TabIndent) - fmt.Fprintln(writer, "name\tdescription") - fmt.Fprintln(writer, "----\t----------") - // print all the mixins out + fmt.Fprintln(writer, "name") + fmt.Fprintln(writer, "----") for _, m := range mixinsList { - // chop off the description if it's too long - m.Description = strings.Replace(m.Description, "\n", "", -1) - if len(m.Description) <= 0 { - m.Description = "N/A" - } - fmt.Fprintf(writer, "%s\t%s\n", color.GreenString(m.Name), m.Description) + // for now, do not print out any of the descriptions + // m.Description = strings.Replace(m.Description, "\n", "", -1) + // // maybe truncate the description if it's too long + // if len(m.Description) <= 0 { + // m.Description = "N/A" + // } + fmt.Fprintf(writer, "%s\n", color.GreenString(m.Name)) } return writer.Flush() From 17e1e82f9f26f61844651b7c15d4afb0f6de3ee9 Mon Sep 17 00:00:00 2001 From: Allen Ma Date: Sat, 7 Nov 2020 14:29:38 -0500 Subject: [PATCH 04/20] add install skeleton to install branch --- cmd/mixtool/generate.go | 2 + cmd/mixtool/install.go | 178 +++++++++++++++++++++++++--------------- cmd/mixtool/list.go | 129 +++++++++++++++++++++-------- go.sum | 4 + 4 files changed, 216 insertions(+), 97 deletions(-) diff --git a/cmd/mixtool/generate.go b/cmd/mixtool/generate.go index e431e02..188e15b 100644 --- a/cmd/mixtool/generate.go +++ b/cmd/mixtool/generate.go @@ -182,6 +182,8 @@ func generateDashboards(filename string, opts mixer.GenerateOptions) error { return nil } + fmt.Println(dashboards) + for name, dashboard := range dashboards { if err := writeDashboard(name, dashboard); err != nil { return err diff --git a/cmd/mixtool/install.go b/cmd/mixtool/install.go index 3915d59..68cdc41 100644 --- a/cmd/mixtool/install.go +++ b/cmd/mixtool/install.go @@ -1,73 +1,123 @@ -// Copyright 2018 mixtool authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +// // Copyright 2018 mixtool authors +// // +// // Licensed under the Apache License, Version 2.0 (the "License"); +// // you may not use this file except in compliance with the License. +// // You may obtain a copy of the License at +// // +// // http://www.apache.org/licenses/LICENSE-2.0 +// // +// // Unless required by applicable law or agreed to in writing, software +// // distributed under the License is distributed on an "AS IS" BASIS, +// // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// // See the License for the specific language governing permissions and +// // limitations under the License. package main -import ( - "fmt" +// package main - "github.com/urfave/cli" -) +// import ( +// "fmt" +// "net/url" +// "os" +// "path" -// type mixin struct { -// URL string `json:"source"` -// Description string `json:"description,omitempty"` -// Name string `json:"name"` -// Subdir string `json:"subdir"` +// // _ "jsonnet-bundler-local/jsonnet-bundler/common" + +// "github.com/urfave/cli" +// ) + +// // type mixin struct { +// // URL string `json:"source"` +// // Description string `json:"description,omitempty"` +// // Name string `json:"name"` +// // Subdir string `json:"subdir"` +// // } + +// func installCommand() cli.Command { +// return cli.Command{ +// Name: "install", +// Usage: "Install a mixin", +// Description: "Install a mixin from a repository", +// Action: installAction, +// Flags: []cli.Flag{ +// cli.StringFlag{ +// Name: "bind-address", +// Usage: "Address to bind HTTP server to.", +// Value: "https://127.0.0.1:8080", +// }, +// cli.StringFlag{ +// Name: "prometheus", +// Value: "http://127.0.0.1:9090/", +// Usage: "location of the prometheus server", +// }, +// cli.StringFlag{ +// Name: "directory, d", +// Usage: "provide the path of where you would like the mixin to be downloaded", +// }, +// }, +// } // } -// type mixins struct { -// d map[string][]mixin +// // Downloads a mixin from a given repository given by url and places into directory +// // by running jb init and jb install +// func downloadMixin(url string, directory string) error { +// os.Chdir(directory) +// fmt.Printf("downloading from %s\n", url) +// // intialize the jsonnet bundler library +// return nil // } -func installCommand() cli.Command { - return cli.Command{ - Name: "install", - Usage: "Install a mixin", - Description: "Install a mixin from a repository", - Action: installAction, - Flags: []cli.Flag{ - cli.StringFlag{ - Name: "bind-address", - Usage: "Address to bind HTTP server to.", - }, - cli.StringFlag{ - Name: "prometheus", - Value: "http://127.0.0.1:9090/", - Usage: "location of the prometheus server", - }, - }, - } -} - -func installAction(c *cli.Context) error { - filename := c.Args().First() - if filename == "" { - return fmt.Errorf("expected one argument, the name of the mixin. Show available mixins using mixtool list") - } - - // process: - - // check if the name of the mixin exists in mixtool list - - // if not, check if the mixin url is valid - if so, interpret as a url to a repo - - // run jb install the mixin - - // run mixtool generate all - - // read files and run mixtool server - - return nil -} +// func installAction(c *cli.Context) error { +// directory := c.String("directory") +// if directory == "" { +// return fmt.Errorf("Expected directory which states where to put downloaded mixin into") +// } + +// mixinPath := c.Args().First() +// if mixinPath == "" { +// return fmt.Errorf("Expected the url of mixin repository or name of the mixin. Show available mixins using mixtool list") +// } + +// mixinsList, err := getMixins() +// if err != nil { +// return err +// } + +// var mixinURL string +// if _, err := url.ParseRequestURI(mixinPath); err != nil { +// // check if the name exists in mixinsList +// found := false +// for _, m := range mixinsList { +// if m.Name == mixinPath { +// // join paths together +// u, err := url.Parse(m.URL) +// if err != nil { +// return err +// } +// u.Path = path.Join(u.Path, m.Subdir) +// mixinURL = u.String() +// found = true +// break +// } +// } +// if !found { +// return fmt.Errorf("could not find mixin with name %s", mixinPath) +// } +// } else { +// mixinURL = mixinPath +// } + +// // make mixinURL consistent +// if mixinURL != "" { +// err = downloadMixin(mixinURL, directory) +// } + +// // run mixtool generate all + +// // read files and run mixtool server + +// // also need to reload grafana + +// return nil +// } diff --git a/cmd/mixtool/list.go b/cmd/mixtool/list.go index 4e30052..adefd65 100644 --- a/cmd/mixtool/list.go +++ b/cmd/mixtool/list.go @@ -76,6 +76,33 @@ func queryWebsite(mixinsWebsite string) ([]byte, error) { return body, nil } +func printMixins(mixins []mixin) error { + writer := tabwriter.NewWriter(os.Stdout, 4, 8, 0, '\t', tabwriter.TabIndent) + fmt.Fprintln(writer, "name") + fmt.Fprintln(writer, "----") + for _, m := range mixinsList { + // for now, do not print out any of the descriptions + // m.Description = strings.Replace(m.Description, "\n", "", -1) + // // maybe truncate the description if it's too long + // if len(m.Description) <= 0 { + // m.Description = "N/A" + // } + fmt.Fprintf(writer, "%s\n", color.GreenString(m.Name)) + } + + return writer.Flush() +} + +// ParsesMixinJSON expects a top level key mixins which contains a list of mixins +func parseMixinJSON(body []byte) ([]mixin, error) { + var mixins map[string][]mixin + if err := json.Unmarshal(body, &mixins); err != nil { + return nil, fmt.Errorf("failed to unmarshal json: %w", err) + } + mixinsList := mixins["mixins"] + return mixinsList, nil +} + // if path is not specified, default to the mixins website // otherwise, try parse as url // otherwise, try look for a local json file @@ -88,46 +115,82 @@ func listAction(c *cli.Context) error { if err != nil { return err } - } else { - // check if it's a url - _, err := url.ParseRequestURI(path) + mixins, err := parseMixinJSON(body) if err != nil { - // check if it's a local json file - _, err = os.Stat(path) - if err != nil { - return err - } - body, err = ioutil.ReadFile(path) - if err != nil { - return err - } - } else { - body, err = queryWebsite(path) - if err != nil { - return err - } + return err } + return printMixins(mixins) } - var mixins map[string][]mixin - if err := json.Unmarshal(body, &mixins); err != nil { - return fmt.Errorf("failed to unmarshal json: %w", err) + _, err := url.ParseRequestURI(path) + if err != nil { + // check if it's a local json file + _, err = os.Stat(path) + if err != nil { + return err + } + body, err = ioutil.ReadFile(path) + if err != nil { + return err + } + mixins, err := parseMixinJSON(body) + if err != nil { + return err + } + return printMixins(mixins) } - mixinsList := mixins["mixins"] + body, err = queryWebsite(path) + if err != nil { + return err + } - writer := tabwriter.NewWriter(os.Stdout, 4, 8, 0, '\t', tabwriter.TabIndent) - fmt.Fprintln(writer, "name") - fmt.Fprintln(writer, "----") - for _, m := range mixinsList { - // for now, do not print out any of the descriptions - // m.Description = strings.Replace(m.Description, "\n", "", -1) - // // maybe truncate the description if it's too long - // if len(m.Description) <= 0 { - // m.Description = "N/A" - // } - fmt.Fprintf(writer, "%s\n", color.GreenString(m.Name)) + mixins, err := parseMixinJSON(body) + if err != nil { + return err } + return printMixins(mixins) - return writer.Flush() } + +// func listAction(c *cli.Context) error { +// path := c.String("path") +// var body []byte +// var err error +// if path == "" { +// body, err = queryWebsite(defaultWebsite) +// if err != nil { +// return err +// } +// } else { +// // check if it's a url +// _, err := url.ParseRequestURI(path) +// if err != nil { +// // check if it's a local json file +// _, err = os.Stat(path) +// if err != nil { +// return err +// } +// body, err = ioutil.ReadFile(path) +// if err != nil { +// return err +// } +// } else { +// body, err = queryWebsite(path) +// if err != nil { +// return err +// } +// } +// } + +// var mixins map[string][]mixin +// if err := json.Unmarshal(body, &mixins); err != nil { +// return fmt.Errorf("failed to unmarshal json: %w", err) +// } + +// mixinsList := mixins["mixins"] + +// err = printMixins(mixinsList) + +// return err +// } diff --git a/go.sum b/go.sum index 7dcd3e0..ee65b57 100644 --- a/go.sum +++ b/go.sum @@ -76,9 +76,11 @@ github.com/afex/hystrix-go v0.0.0-20180502004556-fa1af6a1f4f5/go.mod h1:SkGFH1ia github.com/agnivade/levenshtein v1.0.1/go.mod h1:CURSv5d9Uaml+FovSIICkLbAUZ9S4RqaHDIsdSBg7lM= github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af/go.mod h1:K08gAheRH3/J6wwsYMMT4xOr94bZjxIelGM0+d/wbFw= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 h1:JYp7IbQjafoB+tBA3gMyHYHrpOtNuDiK/uB5uXxq5wM= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d h1:UQZhZ2O0vMHr2cI+DC1Mbh0TJxzA3RcLoMsFw+aXw7E= github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= @@ -389,6 +391,7 @@ github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGa github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= +github.com/gorilla/mux v1.7.3 h1:gnP5JzjVOuiZD07fKKToCAOjS0yOpj/qPETTXCCS6hw= github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= @@ -1119,6 +1122,7 @@ google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2 google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.24.0 h1:UhZDfRO8JRQru4/+LlLE0BRKGF8L+PICnvYZmx/fEGA= google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= +gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/bufio.v1 v1.0.0-20140618132640-567b2bfa514e h1:wGA78yza6bu/mWcc4QfBuIEHEtc06xdiU0X8sY36yUU= gopkg.in/bufio.v1 v1.0.0-20140618132640-567b2bfa514e/go.mod h1:xsQCaysVCudhrYTfzYWe577fCe7Ceci+6qjO2Rdc0Z4= From 98aa30cbc67356c2f4e9dc77f7d455e9f836b34e Mon Sep 17 00:00:00 2001 From: Allen Ma Date: Tue, 10 Nov 2020 10:10:16 -0500 Subject: [PATCH 05/20] add mixtool install wip --- cmd/mixtool/generate.go | 2 - cmd/mixtool/install.go | 336 ++++++++++++++++++++++------------ cmd/mixtool/install_test.go | 83 +++++++++ cmd/mixtool/list.go | 46 +---- cmd/mixtool/list_test.go | 54 ++++++ cmd/mixtool/server.go | 15 +- go.mod | 2 + go.sum | 5 + pkg/jsonnetbundler/init.go | 39 ++++ pkg/jsonnetbundler/install.go | 134 ++++++++++++++ 10 files changed, 551 insertions(+), 165 deletions(-) create mode 100644 cmd/mixtool/install_test.go create mode 100644 cmd/mixtool/list_test.go create mode 100644 pkg/jsonnetbundler/init.go create mode 100644 pkg/jsonnetbundler/install.go diff --git a/cmd/mixtool/generate.go b/cmd/mixtool/generate.go index 188e15b..e431e02 100644 --- a/cmd/mixtool/generate.go +++ b/cmd/mixtool/generate.go @@ -182,8 +182,6 @@ func generateDashboards(filename string, opts mixer.GenerateOptions) error { return nil } - fmt.Println(dashboards) - for name, dashboard := range dashboards { if err := writeDashboard(name, dashboard); err != nil { return err diff --git a/cmd/mixtool/install.go b/cmd/mixtool/install.go index 68cdc41..aaced8e 100644 --- a/cmd/mixtool/install.go +++ b/cmd/mixtool/install.go @@ -1,123 +1,223 @@ -// // Copyright 2018 mixtool authors -// // -// // Licensed under the Apache License, Version 2.0 (the "License"); -// // you may not use this file except in compliance with the License. -// // You may obtain a copy of the License at -// // -// // http://www.apache.org/licenses/LICENSE-2.0 -// // -// // Unless required by applicable law or agreed to in writing, software -// // distributed under the License is distributed on an "AS IS" BASIS, -// // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// // See the License for the specific language governing permissions and -// // limitations under the License. +// Copyright 2018 mixtool authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. package main -// package main - -// import ( -// "fmt" -// "net/url" -// "os" -// "path" - -// // _ "jsonnet-bundler-local/jsonnet-bundler/common" - -// "github.com/urfave/cli" -// ) - -// // type mixin struct { -// // URL string `json:"source"` -// // Description string `json:"description,omitempty"` -// // Name string `json:"name"` -// // Subdir string `json:"subdir"` -// // } - -// func installCommand() cli.Command { -// return cli.Command{ -// Name: "install", -// Usage: "Install a mixin", -// Description: "Install a mixin from a repository", -// Action: installAction, -// Flags: []cli.Flag{ -// cli.StringFlag{ -// Name: "bind-address", -// Usage: "Address to bind HTTP server to.", -// Value: "https://127.0.0.1:8080", -// }, -// cli.StringFlag{ -// Name: "prometheus", -// Value: "http://127.0.0.1:9090/", -// Usage: "location of the prometheus server", -// }, -// cli.StringFlag{ -// Name: "directory, d", -// Usage: "provide the path of where you would like the mixin to be downloaded", -// }, -// }, -// } +import ( + "fmt" + "io/ioutil" + "net/url" + "os" + "path" + "path/filepath" + + "github.com/monitoring-mixins/mixtool/pkg/jsonnetbundler" + "github.com/monitoring-mixins/mixtool/pkg/mixer" + + "github.com/urfave/cli" +) + +// type mixin struct { +// URL string `json:"source"` +// Description string `json:"description,omitempty"` +// Name string `json:"name"` +// Subdir string `json:"subdir"` // } -// // Downloads a mixin from a given repository given by url and places into directory -// // by running jb init and jb install -// func downloadMixin(url string, directory string) error { -// os.Chdir(directory) -// fmt.Printf("downloading from %s\n", url) -// // intialize the jsonnet bundler library -// return nil -// } - -// func installAction(c *cli.Context) error { -// directory := c.String("directory") -// if directory == "" { -// return fmt.Errorf("Expected directory which states where to put downloaded mixin into") -// } - -// mixinPath := c.Args().First() -// if mixinPath == "" { -// return fmt.Errorf("Expected the url of mixin repository or name of the mixin. Show available mixins using mixtool list") -// } - -// mixinsList, err := getMixins() -// if err != nil { -// return err -// } - -// var mixinURL string -// if _, err := url.ParseRequestURI(mixinPath); err != nil { -// // check if the name exists in mixinsList -// found := false -// for _, m := range mixinsList { -// if m.Name == mixinPath { -// // join paths together -// u, err := url.Parse(m.URL) -// if err != nil { -// return err -// } -// u.Path = path.Join(u.Path, m.Subdir) -// mixinURL = u.String() -// found = true -// break -// } -// } -// if !found { -// return fmt.Errorf("could not find mixin with name %s", mixinPath) -// } -// } else { -// mixinURL = mixinPath -// } - -// // make mixinURL consistent -// if mixinURL != "" { -// err = downloadMixin(mixinURL, directory) -// } - -// // run mixtool generate all - -// // read files and run mixtool server - -// // also need to reload grafana - -// return nil -// } +func installCommand() cli.Command { + return cli.Command{ + Name: "install", + Usage: "Install a mixin", + Description: "Install a mixin from a repository", + Action: installAction, + Flags: []cli.Flag{ + cli.StringFlag{ + Name: "bind-address", + Usage: "Address to bind HTTP server to.", + Value: "https://127.0.0.1:8080", + }, + cli.StringFlag{ + Name: "rule-file", + Usage: "File to provision rules into.", + }, + cli.StringFlag{ + Name: "prometheus-reload-url", + Value: "http://127.0.0.1:9090/-/reload", + Usage: "Prometheus address to reload after provisioning the rule file(s).", + }, + cli.StringFlag{ + Name: "directory, d", + Usage: "Path where the downloaded mixin is saved. If it doesn't exist already it will be created", + }, + cli.BoolFlag{ + Name: "run-server, s", + Usage: "Set this flag to run server to reload Prometheus once mixin files are generated. If this flag is set, you also need to specify server's bind-address, and a prometheus reload URL", + }, + }, + } +} + +// Downloads a mixin from a given repository given by url and places into directory +// by running jb init and jb install +func downloadMixin(url string, directory string) error { + // intialize the jsonnet bundler library + err := jsonnetbundler.InitCommand(directory) + if err != nil { + return err + } + + // use vendor directory by default + // by default, set the single flag on + err = jsonnetbundler.InstallCommand(directory, "vendor", []string{url}, false) + if err != nil { + fmt.Println(err) + return err + } + + return nil +} + +// Gets mixins from default website - mostly copied from list.go +func getMixins() ([]mixin, error) { + body, err := queryWebsite(defaultWebsite) + if err != nil { + return nil, err + } + mixins, err := parseMixinJSON(body) + if err != nil { + return nil, err + } + return mixins, nil +} + +func generateMixin(directory string, mixinURL string, options mixer.GenerateOptions) error { + fmt.Println("running generate all") + + err := os.Chdir(directory) + if err != nil { + return fmt.Errorf("Cannot cd into directory %s", err) + } + + files, err := filepath.Glob("*") + fmt.Println("in generatemixin, directory is ", directory, files) + + // create a temporary jsonnet file that + // imports mixin.libsonnet as the "main" file in the mixin configfuration + // then run generate all passing in the vendor folder to find dependencies + // the name of the mixin folder inside vendor seems to be the last fragment of mixin's url + subdir + fragment := filepath.Base(mixinURL) + tempContent := fmt.Sprintf("import \"%s\"", filepath.Join(fragment, "mixin.libsonnet")) + err = ioutil.WriteFile("temp.jsonnet", []byte(tempContent), 0644) + if err != nil { + return err + } + + err = generateAll("temp.jsonnet", options) + if err != nil { + return err + } + + err = os.Remove("temp.jsonnet") + if err != nil { + return err + } + return nil +} + +func installAction(c *cli.Context) error { + directory := c.String("directory") + if directory == "" { + return fmt.Errorf("Must specify a directory to download mixin") + } + + _, err := os.Stat(directory) + if os.IsNotExist(err) { + err = os.MkdirAll(directory, 0755) + if err != nil { + return err + } + } + + mixinPath := c.Args().First() + if mixinPath == "" { + return fmt.Errorf("Expected the url of mixin repository or name of the mixin. Show available mixins using mixtool list") + } + + mixinsList, err := getMixins() + if err != nil { + return err + } + + var mixinURL string + if _, err := url.ParseRequestURI(mixinPath); err != nil { + // check if the name exists in mixinsList + found := false + for _, m := range mixinsList { + if m.Name == mixinPath { + // join paths together + u, err := url.Parse(m.URL) + if err != nil { + return err + } + u.Path = path.Join(u.Path, m.Subdir) + mixinURL = u.String() + found = true + break + } + } + if !found { + return fmt.Errorf("Could not find mixin with name %s", mixinPath) + } + } else { + mixinURL = mixinPath + } + + if mixinURL == "" { + return fmt.Errorf("Empty mixinURL") + } + + err = downloadMixin(mixinURL, directory) + if err != nil { + fmt.Println(err) + return err + } + + generateCfg := mixer.GenerateOptions{ + AlertsFilename: "alerts.yaml", + RulesFilename: "rules.yaml", + Directory: "dashboards_out", + JPaths: []string{"./vendor"}, + YAML: true, + } + + err = generateMixin(directory, mixinURL, generateCfg) + if err != nil { + return err + } + + // read files and run mixtool server + // also need to reload alerts, as well as grafana dashboards + if c.Bool("run-server") { + bindAddress := c.String("bind-address") + promURL := c.String("prometheus-reload-url") + ruleFile := c.String("rule-file") + err = runServer(bindAddress, promURL, ruleFile) + if err != nil { + return err + } + + // call PUT requests to the server + } + return nil +} diff --git a/cmd/mixtool/install_test.go b/cmd/mixtool/install_test.go new file mode 100644 index 0000000..8925d13 --- /dev/null +++ b/cmd/mixtool/install_test.go @@ -0,0 +1,83 @@ +package main + +import ( + "fmt" + "io/ioutil" + "os" + "path" + "testing" + + "github.com/monitoring-mixins/mixtool/pkg/mixer" +) + +// Try to install every mixin from the mixin repository +// verify that each package generated has the yaml files +func TestInstallMixin(t *testing.T) { + tmpdir, err := ioutil.TempDir("", "mixtool-install") + if err != nil { + t.Errorf("failed to make directory %v", err) + } + + // defer os.RemoveAll(tmpdir) + + body, err := queryWebsite(defaultWebsite) + if err != nil { + t.Errorf("failed to query website %v", err) + } + mixins, err := parseMixinJSON(body) + if err != nil { + t.Errorf("failed to parse mixin body %v", err) + } + + // download each mixin in turn + for _, m := range mixins { + + generateCfg := mixer.GenerateOptions{ + AlertsFilename: "alerts.yaml", + RulesFilename: "rules.yaml", + Directory: "dashboards_out", + JPaths: []string{"vendor"}, + YAML: true, + } + + mixinURL := path.Join(m.URL, m.Subdir) + + fmt.Printf("installing %v\n", mixinURL) + dldir := path.Join(tmpdir, m.Name) + + err = os.Mkdir(dldir, 0755) + if err != nil { + t.Errorf("failed to create directory %v", dldir) + } + + err = downloadMixin(mixinURL, dldir) + if err != nil { + t.Errorf("failed to download mixin at %v. %v", mixinURL, err) + } + + err = generateMixin(dldir, mixinURL, generateCfg) + if err != nil { + t.Errorf("failed to generate mixin yaml for %v. %v", mixinURL, err) + } + + // // verify that the contents are correct + // err = os.Chdir(dldir) + // if err != nil { + // t.Errorf("could not cd into %s", dldir) + // } + + // if _, err := os.Stat("alerts.yaml"); os.IsNotExist(err) { + // t.Errorf("expected alerts.yaml in %s", dldir) + // } + + // if _, err := os.Stat("rules.yaml"); os.IsNotExist(err) { + // t.Errorf("expected rules.yaml in %s", dldir) + // } + + // if _, err := os.Stat("dashboards_out"); os.IsNotExist(err) { + // t.Errorf("expected dashboards_out in %s", dldir) + // } + + } + +} diff --git a/cmd/mixtool/list.go b/cmd/mixtool/list.go index adefd65..d2c0e20 100644 --- a/cmd/mixtool/list.go +++ b/cmd/mixtool/list.go @@ -76,7 +76,7 @@ func queryWebsite(mixinsWebsite string) ([]byte, error) { return body, nil } -func printMixins(mixins []mixin) error { +func printMixins(mixinsList []mixin) error { writer := tabwriter.NewWriter(os.Stdout, 4, 8, 0, '\t', tabwriter.TabIndent) fmt.Fprintln(writer, "name") fmt.Fprintln(writer, "----") @@ -122,7 +122,7 @@ func listAction(c *cli.Context) error { return printMixins(mixins) } - _, err := url.ParseRequestURI(path) + _, err = url.ParseRequestURI(path) if err != nil { // check if it's a local json file _, err = os.Stat(path) @@ -152,45 +152,3 @@ func listAction(c *cli.Context) error { return printMixins(mixins) } - -// func listAction(c *cli.Context) error { -// path := c.String("path") -// var body []byte -// var err error -// if path == "" { -// body, err = queryWebsite(defaultWebsite) -// if err != nil { -// return err -// } -// } else { -// // check if it's a url -// _, err := url.ParseRequestURI(path) -// if err != nil { -// // check if it's a local json file -// _, err = os.Stat(path) -// if err != nil { -// return err -// } -// body, err = ioutil.ReadFile(path) -// if err != nil { -// return err -// } -// } else { -// body, err = queryWebsite(path) -// if err != nil { -// return err -// } -// } -// } - -// var mixins map[string][]mixin -// if err := json.Unmarshal(body, &mixins); err != nil { -// return fmt.Errorf("failed to unmarshal json: %w", err) -// } - -// mixinsList := mixins["mixins"] - -// err = printMixins(mixinsList) - -// return err -// } diff --git a/cmd/mixtool/list_test.go b/cmd/mixtool/list_test.go new file mode 100644 index 0000000..b53fe58 --- /dev/null +++ b/cmd/mixtool/list_test.go @@ -0,0 +1,54 @@ +package main + +import ( + "io/ioutil" + "os" + "testing" +) + +const exampleMixins = ` +{ + "mixins": [ + { + "name": "ceph", + "source": "https://github.com/ceph/ceph-mixins", + "subdir": "", + "description": "A set of Prometheus alerts for Ceph.\n\nThe scope of this project is to provide Ceph specific Prometheus rule files using Prometheus Mixins.\n" + }, + { + "name": "cortex", + "source": "https://github.com/grafana/cortex-jsonnet", + "subdir": "cortex-mixin" + }, + { + "name": "cool-mixin", + "source": "https://github.com", + "subdir": "cool-mixin", + "description": "A fantastic mixin" + } + ] +} +` + +func TestList(t *testing.T) { + err := ioutil.WriteFile("exampleMixinsTest.json", []byte(exampleMixins), 0644) + if err != nil { + t.Errorf("failed to create temp file: %v", err) + } + defer os.Remove("exampleMixinsTest.json") + + body, err := ioutil.ReadFile("exampleMixinsTest.json") + if err != nil { + t.Errorf("failed to read exampleMixinsTest.json %v", err) + } + mixins, err := parseMixinJSON(body) + if err != nil { + t.Errorf("failed to read exampleMixinsTest.json %v", err) + } + exampleMixins := map[string]bool{"ceph": true, "cool-mixin": true, "cortex": true} + for _, m := range mixins { + if _, ok := exampleMixins[m.Name]; !ok { + t.Errorf("failed to find %v in exampleMixinsTest", m.Name) + } + } +} diff --git a/cmd/mixtool/server.go b/cmd/mixtool/server.go index 5a84fb3..699513e 100644 --- a/cmd/mixtool/server.go +++ b/cmd/mixtool/server.go @@ -65,6 +65,19 @@ func serverAction(c *cli.Context) error { return http.ListenAndServe(bindAddress, nil) } +func runServer(bindAddress string, promURL string, ruleFile string) error { + fmt.Println("running mixtool server") + http.Handle("/api/v1/rules", &ruleProvisioningHandler{ + ruleProvisioner: &ruleProvisioner{ + ruleFile: ruleFile, + }, + prometheusReloader: &prometheusReloader{ + prometheusReloadURL: promURL, + }, + }) + return http.ListenAndServe(bindAddress, nil) +} + type ruleProvisioningHandler struct { ruleProvisioner *ruleProvisioner prometheusReloader *prometheusReloader @@ -104,7 +117,7 @@ func (p *ruleProvisioner) provision(r io.Reader) (bool, error) { b := bytes.NewBuffer(nil) tr := io.TeeReader(r, b) - f, err := os.Open(p.ruleFile) + f, err := os.OpenFile(p.ruleFile, os.O_RDWR, 0644) if err != nil && !os.IsNotExist(err) { fmt.Println(err, p.ruleFile) return false, fmt.Errorf("open rule file: %w", err) diff --git a/go.mod b/go.mod index c9dc2b0..e854d32 100644 --- a/go.mod +++ b/go.mod @@ -17,11 +17,13 @@ require ( github.com/grafana/grafana v5.2.4+incompatible github.com/grafana/tanka v0.12.0 github.com/inconshreveable/log15 v0.0.0-20180818164646-67afb5ed74ec // indirect + github.com/jsonnet-bundler/jsonnet-bundler v0.4.0 github.com/pkg/errors v0.9.1 github.com/prometheus/prometheus v1.8.2-0.20200923143134-7e2db3d092f3 github.com/rainycape/unidecode v0.0.0-20150907023854-cb7f23ec59be // indirect github.com/teris-io/shortid v0.0.0-20171029131806-771a37caa5cf // indirect github.com/urfave/cli v1.22.1 + gopkg.in/alecthomas/kingpin.v2 v2.2.6 gopkg.in/bufio.v1 v1.0.0-20140618132640-567b2bfa514e // indirect gopkg.in/ini.v1 v1.38.2 // indirect gopkg.in/macaron.v1 v1.3.1 // indirect diff --git a/go.sum b/go.sum index ee65b57..7e6bb68 100644 --- a/go.sum +++ b/go.sum @@ -113,6 +113,7 @@ github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx2 github.com/bradfitz/gomemcache v0.0.0-20180710155616-bc664df96737 h1:rRISKWyXfVxvoa702s91Zl5oREZTrR3yv+tXrrX7G/g= github.com/bradfitz/gomemcache v0.0.0-20180710155616-bc664df96737/go.mod h1:PmM6Mmwb0LSuEubjR8N7PtNe1KxZLtOUHtbeikc5h60= github.com/c-bata/go-prompt v0.2.2/go.mod h1:VzqtzE2ksDBcdln8G7mk2RX9QyGjH+OVqOCSiVIqS34= +github.com/campoy/embedmd v1.0.0/go.mod h1:oxyr9RCiSXg0M3VJ3ks0UGfp98BpSSGr0kpiX3MzVl8= github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ= github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= github.com/cenkalti/backoff/v4 v4.0.2/go.mod h1:eEew/i+1Q6OrCDZh3WiXYv3+nJwBASZ8Bog/87DQnVg= @@ -479,6 +480,8 @@ github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/u github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/jsonnet-bundler/jsonnet-bundler v0.4.0 h1:4BKZ6LDqPc2wJDmaKnmYD/vDjUptJtnUpai802MibFc= +github.com/jsonnet-bundler/jsonnet-bundler v0.4.0/go.mod h1:/by7P/OoohkI3q4CgSFqcoFsVY+IaNbzOVDknEsKDeU= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/jsternberg/zap-logfmt v1.0.0/go.mod h1:uvPs/4X51zdkcm5jXl5SYoN+4RK21K8mysFmDaM/h+o= @@ -541,6 +544,7 @@ github.com/mattn/go-colorable v0.1.6 h1:6Su7aK7lXmJ/U79bYtBjLNaha4Fs1Rg9plHpcH+v github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= +github.com/mattn/go-isatty v0.0.6/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84= github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE= @@ -933,6 +937,7 @@ golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20190209173611-3b5209105503/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190310054646-10058d7d4faa/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190321052220-f7bb7a8bee54/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= diff --git a/pkg/jsonnetbundler/init.go b/pkg/jsonnetbundler/init.go new file mode 100644 index 0000000..d522845 --- /dev/null +++ b/pkg/jsonnetbundler/init.go @@ -0,0 +1,39 @@ +package jsonnetbundler + +import ( + "encoding/json" + "fmt" + "io/ioutil" + "path/filepath" + + "github.com/jsonnet-bundler/jsonnet-bundler/pkg/jsonnetfile" + v1 "github.com/jsonnet-bundler/jsonnet-bundler/spec/v1" +) + +// InitCommand is basically the same as jb init +func InitCommand(dir string) error { + exists, err := jsonnetfile.Exists(jsonnetfile.File) + + if exists { + return fmt.Errorf("jsonnetfile.json already exists") + } + + s := v1.New() + // TODO: disable them by default eventually + // s.LegacyImports = false + + contents, err := json.MarshalIndent(s, "", " ") + if err != nil { + return fmt.Errorf("formatting jsonnetfile contents as json, %s", err.Error()) + } + contents = append(contents, []byte("\n")...) + + filename := filepath.Join(dir, jsonnetfile.File) + + err = ioutil.WriteFile(filename, contents, 0644) + if err != nil { + return fmt.Errorf("Failed to write new jsonnetfile.json, %s", err.Error()) + } + + return nil +} diff --git a/pkg/jsonnetbundler/install.go b/pkg/jsonnetbundler/install.go new file mode 100644 index 0000000..49ef6eb --- /dev/null +++ b/pkg/jsonnetbundler/install.go @@ -0,0 +1,134 @@ +// Copyright 2018 jsonnet-bundler authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package jsonnetbundler + +import ( + "encoding/json" + "fmt" + "io/ioutil" + "os" + "path/filepath" + "reflect" + + "github.com/pkg/errors" + + "github.com/jsonnet-bundler/jsonnet-bundler/pkg" + "github.com/jsonnet-bundler/jsonnet-bundler/pkg/jsonnetfile" + v1 "github.com/jsonnet-bundler/jsonnet-bundler/spec/v1" + "github.com/jsonnet-bundler/jsonnet-bundler/spec/v1/deps" +) + +func InstallCommand(dir, jsonnetHome string, uris []string, single bool) error { + if dir == "" { + dir = "." + } + + jbfilebytes, err := ioutil.ReadFile(filepath.Join(dir, jsonnetfile.File)) + if err != nil { + return fmt.Errorf("failed to load jsonnetfile %s", err.Error()) + } + + jsonnetFile, err := jsonnetfile.Unmarshal(jbfilebytes) + if err != nil { + return err + } + + jblockfilebytes, err := ioutil.ReadFile(filepath.Join(dir, jsonnetfile.LockFile)) + if !os.IsNotExist(err) { + if err != nil { + return fmt.Errorf("failed to load lockfile %s", err.Error()) + } + } + + lockFile, err := jsonnetfile.Unmarshal(jblockfilebytes) + if err != nil { + return err + } + + err = os.MkdirAll(filepath.Join(dir, jsonnetHome, ".tmp"), os.ModePerm) + if err != nil { + return fmt.Errorf("creating vendor folder %s", err.Error()) + } + + for _, u := range uris { + d := deps.Parse(dir, u) + if d == nil { + return fmt.Errorf("Unable to parse package URI %s", u) + } + + if single { + d.Single = true + } + + if !depEqual(jsonnetFile.Dependencies[d.Name()], *d) { + // the dep passed on the cli is different from the jsonnetFile + jsonnetFile.Dependencies[d.Name()] = *d + + // we want to install the passed version (ignore the lock) + delete(lockFile.Dependencies, d.Name()) + } + } + + jsonnetPkgHomeDir := filepath.Join(dir, jsonnetHome) + locked, err := pkg.Ensure(jsonnetFile, jsonnetPkgHomeDir, lockFile.Dependencies) + if err != nil { + return fmt.Errorf("failed to install packages %s", err) + } + + pkg.CleanLegacyName(jsonnetFile.Dependencies) + + err = writeChangedJsonnetFile(jbfilebytes, &jsonnetFile, filepath.Join(dir, jsonnetfile.File)) + if err != nil { + return fmt.Errorf("updating jsonnetfile.json %s", err) + } + + err = writeChangedJsonnetFile(jblockfilebytes, &v1.JsonnetFile{Dependencies: locked}, filepath.Join(dir, jsonnetfile.LockFile)) + if err != nil { + return fmt.Errorf("updating jsonnetfile.lock.json %s", err) + } + + return nil +} + +func depEqual(d1, d2 deps.Dependency) bool { + name := d1.Name() == d2.Name() + version := d1.Version == d2.Version + source := reflect.DeepEqual(d1.Source, d2.Source) + + return name && version && source +} + +func writeJSONFile(name string, d interface{}) error { + b, err := json.MarshalIndent(d, "", " ") + if err != nil { + return errors.Wrap(err, "encoding json") + } + b = append(b, []byte("\n")...) + + return ioutil.WriteFile(name, b, 0644) +} + +func writeChangedJsonnetFile(originalBytes []byte, modified *v1.JsonnetFile, path string) error { + origJsonnetFile, err := jsonnetfile.Unmarshal(originalBytes) + if err != nil { + return err + } + + if reflect.DeepEqual(origJsonnetFile, *modified) { + return nil + } + + return writeJSONFile(path, *modified) +} From d06f32d9ec3770a92e0914c9ee63b16e2358df7a Mon Sep 17 00:00:00 2001 From: Allen Ma Date: Tue, 10 Nov 2020 11:19:34 -0500 Subject: [PATCH 06/20] add seek to server --- cmd/mixtool/install.go | 1 + cmd/mixtool/install_test.go | 34 +++++++++++++++++----------------- cmd/mixtool/server.go | 5 +++++ 3 files changed, 23 insertions(+), 17 deletions(-) diff --git a/cmd/mixtool/install.go b/cmd/mixtool/install.go index aaced8e..60a61ac 100644 --- a/cmd/mixtool/install.go +++ b/cmd/mixtool/install.go @@ -209,6 +209,7 @@ func installAction(c *cli.Context) error { // read files and run mixtool server // also need to reload alerts, as well as grafana dashboards if c.Bool("run-server") { + // should be running outside! bindAddress := c.String("bind-address") promURL := c.String("prometheus-reload-url") ruleFile := c.String("rule-file") diff --git a/cmd/mixtool/install_test.go b/cmd/mixtool/install_test.go index 8925d13..55e18e4 100644 --- a/cmd/mixtool/install_test.go +++ b/cmd/mixtool/install_test.go @@ -60,23 +60,23 @@ func TestInstallMixin(t *testing.T) { t.Errorf("failed to generate mixin yaml for %v. %v", mixinURL, err) } - // // verify that the contents are correct - // err = os.Chdir(dldir) - // if err != nil { - // t.Errorf("could not cd into %s", dldir) - // } - - // if _, err := os.Stat("alerts.yaml"); os.IsNotExist(err) { - // t.Errorf("expected alerts.yaml in %s", dldir) - // } - - // if _, err := os.Stat("rules.yaml"); os.IsNotExist(err) { - // t.Errorf("expected rules.yaml in %s", dldir) - // } - - // if _, err := os.Stat("dashboards_out"); os.IsNotExist(err) { - // t.Errorf("expected dashboards_out in %s", dldir) - // } + // verify that the contents are correct + err = os.Chdir(dldir) + if err != nil { + t.Errorf("could not cd into %s", dldir) + } + + if _, err := os.Stat("alerts.yaml"); os.IsNotExist(err) { + t.Errorf("expected alerts.yaml in %s", dldir) + } + + if _, err := os.Stat("rules.yaml"); os.IsNotExist(err) { + t.Errorf("expected rules.yaml in %s", dldir) + } + + if _, err := os.Stat("dashboards_out"); os.IsNotExist(err) { + t.Errorf("expected dashboards_out in %s", dldir) + } } diff --git a/cmd/mixtool/server.go b/cmd/mixtool/server.go index 699513e..36e1ccd 100644 --- a/cmd/mixtool/server.go +++ b/cmd/mixtool/server.go @@ -144,6 +144,11 @@ func (p *ruleProvisioner) provision(r io.Reader) (bool, error) { return false, fmt.Errorf("truncate file: %w", err) } + _, err = f.Seek(0, 0) + if err != nil { + return false, fmt.Errorf("seek file %w", err) + } + if _, err := io.Copy(f, b); err != nil { return false, fmt.Errorf("provision rule to file: %w", err) } From 4ae0038e89c6e4694783f66e0d0e8c3ebb9211fb Mon Sep 17 00:00:00 2001 From: Allen Ma Date: Fri, 13 Nov 2020 11:45:07 -0500 Subject: [PATCH 07/20] rebasing onto origin install --- .gitignore | 4 ++++ cmd/mixtool/lint.go | 1 - cmd/mixtool/list.go | 1 - 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index 64fb839..5287785 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,7 @@ /_output/ +<<<<<<< HEAD /mixtool .vscode +======= +/mixtool +>>>>>>> bd0efc3... add mixtool list functionality (#28) diff --git a/cmd/mixtool/lint.go b/cmd/mixtool/lint.go index 6531acc..107466a 100644 --- a/cmd/mixtool/lint.go +++ b/cmd/mixtool/lint.go @@ -79,6 +79,5 @@ func lintAction(c *cli.Context) error { if err := mixer.Lint(os.Stdout, filename, options); err != nil { return fmt.Errorf("failed to lint the file %s: %v", filename, err) } - return nil } diff --git a/cmd/mixtool/list.go b/cmd/mixtool/list.go index d2c0e20..acb897e 100644 --- a/cmd/mixtool/list.go +++ b/cmd/mixtool/list.go @@ -150,5 +150,4 @@ func listAction(c *cli.Context) error { return err } return printMixins(mixins) - } From 2dad53e92aa877bf730d09a7942ef96a4d29bd59 Mon Sep 17 00:00:00 2001 From: Allen Ma Date: Fri, 13 Nov 2020 11:27:09 -0500 Subject: [PATCH 08/20] saving --- go.mod | 2 +- go.sum | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index e854d32..a078141 100644 --- a/go.mod +++ b/go.mod @@ -11,7 +11,7 @@ require ( github.com/go-macaron/inject v0.0.0-20160627170012-d8a0b8677191 // indirect github.com/go-macaron/session v0.0.0-20170320172209-b8e286a0dba8 // indirect github.com/gobuffalo/packr v1.30.1 - github.com/gobuffalo/packr/v2 v2.8.0 + github.com/gobuffalo/packr/v2 v2.8.1 github.com/google/go-jsonnet v0.16.1-0.20200908152747-b70cbd441a39 github.com/gosimple/slug v1.2.0 // indirect github.com/grafana/grafana v5.2.4+incompatible diff --git a/go.sum b/go.sum index 7e6bb68..cba331c 100644 --- a/go.sum +++ b/go.sum @@ -316,6 +316,8 @@ github.com/gobuffalo/packr/v2 v2.5.1 h1:TFOeY2VoGamPjQLiNDT3mn//ytzk236VMO2j7iHx github.com/gobuffalo/packr/v2 v2.5.1/go.mod h1:8f9c96ITobJlPzI44jj+4tHnEKNt0xXWSVlXRN9X1Iw= github.com/gobuffalo/packr/v2 v2.8.0 h1:IULGd15bQL59ijXLxEvA5wlMxsmx/ZkQv9T282zNVIY= github.com/gobuffalo/packr/v2 v2.8.0/go.mod h1:PDk2k3vGevNE3SwVyVRgQCCXETC9SaONCNSXT1Q8M1g= +github.com/gobuffalo/packr/v2 v2.8.1 h1:tkQpju6i3EtMXJ9uoF5GT6kB+LMTimDWD8Xvbz6zDVA= +github.com/gobuffalo/packr/v2 v2.8.1/go.mod h1:c/PLlOuTU+p3SybaJATW3H6lX/iK7xEz5OeMf+NnJpg= github.com/gobuffalo/syncx v0.0.0-20190224160051-33c29581e754/go.mod h1:HhnNqWY95UYwwW3uSASeV7vtgYkT2t16hJgV3AEPUpw= github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8= github.com/gogo/googleapis v1.1.0/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s= @@ -497,6 +499,8 @@ github.com/karrick/godirwalk v1.10.12/go.mod h1:RoGL9dQei4vP9ilrpETWE8CLOZ1kiN0L github.com/karrick/godirwalk v1.15.3/go.mod h1:j4mkqPuvaLI8mp1DroR3P6ad7cyYd4c1qeJ3RV7ULlk= github.com/karrick/godirwalk v1.15.5 h1:ErdAEFW/cKxQ5+9Gm/hopxB8ki21/di+vyNb9mHnHrA= github.com/karrick/godirwalk v1.15.5/go.mod h1:j4mkqPuvaLI8mp1DroR3P6ad7cyYd4c1qeJ3RV7ULlk= +github.com/karrick/godirwalk v1.15.8 h1:7+rWAZPn9zuRxaIqqT8Ohs2Q2Ac0msBqwRdxNCr2VVs= +github.com/karrick/godirwalk v1.15.8/go.mod h1:j4mkqPuvaLI8mp1DroR3P6ad7cyYd4c1qeJ3RV7ULlk= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= From 060f6b7ccc4d1b723b6d71e042489b300078517b Mon Sep 17 00:00:00 2001 From: Allen Ma Date: Fri, 13 Nov 2020 12:33:15 -0500 Subject: [PATCH 09/20] add put server - not working yet --- cmd/mixtool/install.go | 83 +++++++++++++++++++++++++++++++----------- cmd/mixtool/server.go | 13 ------- 2 files changed, 61 insertions(+), 35 deletions(-) diff --git a/cmd/mixtool/install.go b/cmd/mixtool/install.go index 60a61ac..615a929 100644 --- a/cmd/mixtool/install.go +++ b/cmd/mixtool/install.go @@ -17,6 +17,7 @@ package main import ( "fmt" "io/ioutil" + "net/http" "net/url" "os" "path" @@ -51,19 +52,10 @@ func installCommand() cli.Command { Name: "rule-file", Usage: "File to provision rules into.", }, - cli.StringFlag{ - Name: "prometheus-reload-url", - Value: "http://127.0.0.1:9090/-/reload", - Usage: "Prometheus address to reload after provisioning the rule file(s).", - }, cli.StringFlag{ Name: "directory, d", Usage: "Path where the downloaded mixin is saved. If it doesn't exist already it will be created", }, - cli.BoolFlag{ - Name: "run-server, s", - Usage: "Set this flag to run server to reload Prometheus once mixin files are generated. If this flag is set, you also need to specify server's bind-address, and a prometheus reload URL", - }, }, } } @@ -116,6 +108,7 @@ func generateMixin(directory string, mixinURL string, options mixer.GenerateOpti // imports mixin.libsonnet as the "main" file in the mixin configfuration // then run generate all passing in the vendor folder to find dependencies // the name of the mixin folder inside vendor seems to be the last fragment of mixin's url + subdir + // TODO: need to get absolute path of the mixin fragment := filepath.Base(mixinURL) tempContent := fmt.Sprintf("import \"%s\"", filepath.Join(fragment, "mixin.libsonnet")) err = ioutil.WriteFile("temp.jsonnet", []byte(tempContent), 0644) @@ -135,6 +128,59 @@ func generateMixin(directory string, mixinURL string, options mixer.GenerateOpti return nil } +func putMixin(directory string, mixinURL string, bindAddress string, options mixer.GenerateOptions) error { + fmt.Println("running put Mixin") + + wd, err := os.Getwd() + fmt.Println("path is", wd) + if err != nil { + return err + } + + // err := os.Chdir(filepath.Join() + if err != nil { + return fmt.Errorf("Cannot cd into directory %s", err) + } + + // // alerts.yaml + // alertsFilename := options.AlertsFilename + // alertsReader, err := os.Open(alertsFilename) + // if err != nil { + // return err + // } + + // rules.yaml + rulesFilename := options.RulesFilename + rulesReader, err := os.Open(rulesFilename) + if err != nil { + return err + } + + u, err := url.Parse(bindAddress) + if err != nil { + return err + } + u.Path = path.Join(u.Path, "/api/v1/rules") + + req, err := http.NewRequest("PUT", u.String(), rulesReader) + resp, err := http.DefaultClient.Do(req) + if err != nil { + return err + } + + if resp.StatusCode == 200 { + fmt.Println("OK") + } else { + fmt.Printf("resp is %v\n", resp.Body) + } + + if err != nil { + return err + } + + return nil +} + func installAction(c *cli.Context) error { directory := c.String("directory") if directory == "" { @@ -206,19 +252,12 @@ func installAction(c *cli.Context) error { return err } - // read files and run mixtool server - // also need to reload alerts, as well as grafana dashboards - if c.Bool("run-server") { - // should be running outside! - bindAddress := c.String("bind-address") - promURL := c.String("prometheus-reload-url") - ruleFile := c.String("rule-file") - err = runServer(bindAddress, promURL, ruleFile) - if err != nil { - return err - } - - // call PUT requests to the server + bindAddress := c.String("bind-address") + // run put requests onto the server + err = putMixin(directory, mixinURL, bindAddress, generateCfg) + if err != nil { + return err } + return nil } diff --git a/cmd/mixtool/server.go b/cmd/mixtool/server.go index 36e1ccd..9131adb 100644 --- a/cmd/mixtool/server.go +++ b/cmd/mixtool/server.go @@ -65,19 +65,6 @@ func serverAction(c *cli.Context) error { return http.ListenAndServe(bindAddress, nil) } -func runServer(bindAddress string, promURL string, ruleFile string) error { - fmt.Println("running mixtool server") - http.Handle("/api/v1/rules", &ruleProvisioningHandler{ - ruleProvisioner: &ruleProvisioner{ - ruleFile: ruleFile, - }, - prometheusReloader: &prometheusReloader{ - prometheusReloadURL: promURL, - }, - }) - return http.ListenAndServe(bindAddress, nil) -} - type ruleProvisioningHandler struct { ruleProvisioner *ruleProvisioner prometheusReloader *prometheusReloader From eba6f919ecf9bb223e0d12ba2048a0ad8f3b0867 Mon Sep 17 00:00:00 2001 From: Allen Ma Date: Sat, 14 Nov 2020 16:32:57 -0500 Subject: [PATCH 10/20] server request for rules work but are getting overwritten instead of saving --- cmd/mixtool/install.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/mixtool/install.go b/cmd/mixtool/install.go index 615a929..bbd56d4 100644 --- a/cmd/mixtool/install.go +++ b/cmd/mixtool/install.go @@ -46,7 +46,7 @@ func installCommand() cli.Command { cli.StringFlag{ Name: "bind-address", Usage: "Address to bind HTTP server to.", - Value: "https://127.0.0.1:8080", + Value: "http://127.0.0.1:8080", }, cli.StringFlag{ Name: "rule-file", @@ -165,7 +165,7 @@ func putMixin(directory string, mixinURL string, bindAddress string, options mix req, err := http.NewRequest("PUT", u.String(), rulesReader) resp, err := http.DefaultClient.Do(req) if err != nil { - return err + return fmt.Errorf("response from server %v", err) } if resp.StatusCode == 200 { From cd86e238728a0c6d2ec75860b9e351d4bb93393e Mon Sep 17 00:00:00 2001 From: Allen Ma Date: Sat, 14 Nov 2020 17:27:07 -0500 Subject: [PATCH 11/20] change symlink import to use absolute imports, still using temp.jsonnet - not sure how to do it otherwise without adding lots of duplicate code --- cmd/mixtool/generate.go | 2 +- cmd/mixtool/install.go | 52 ++++++++++++++++++++++----------- pkg/mixer/eval_snippet.go | 60 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 97 insertions(+), 17 deletions(-) create mode 100644 pkg/mixer/eval_snippet.go diff --git a/cmd/mixtool/generate.go b/cmd/mixtool/generate.go index e431e02..02d1d43 100644 --- a/cmd/mixtool/generate.go +++ b/cmd/mixtool/generate.go @@ -151,7 +151,7 @@ func generateRules(filename string, options mixer.GenerateOptions) error { if err != nil { return err } - + fmt.Println("inside generate rules", options.RulesFilename) return ioutil.WriteFile(options.RulesFilename, out, 0644) } diff --git a/cmd/mixtool/install.go b/cmd/mixtool/install.go index bbd56d4..9e37b15 100644 --- a/cmd/mixtool/install.go +++ b/cmd/mixtool/install.go @@ -62,16 +62,15 @@ func installCommand() cli.Command { // Downloads a mixin from a given repository given by url and places into directory // by running jb init and jb install -func downloadMixin(url string, directory string) error { +func downloadMixin(url string, jsonnetHome string, directory string) error { // intialize the jsonnet bundler library err := jsonnetbundler.InitCommand(directory) if err != nil { return err } - // use vendor directory by default - // by default, set the single flag on - err = jsonnetbundler.InstallCommand(directory, "vendor", []string{url}, false) + // by default, set the single flag to false + err = jsonnetbundler.InstallCommand(directory, jsonnetHome, []string{url}, false) if err != nil { fmt.Println(err) return err @@ -93,10 +92,12 @@ func getMixins() ([]mixin, error) { return mixins, nil } -func generateMixin(directory string, mixinURL string, options mixer.GenerateOptions) error { +func generateMixin(directory string, jsonnetHome string, mixinURL string, options mixer.GenerateOptions) error { fmt.Println("running generate all") - err := os.Chdir(directory) + mixinBaseDirectory := filepath.Join(directory) + + err := os.Chdir(mixinBaseDirectory) if err != nil { return fmt.Errorf("Cannot cd into directory %s", err) } @@ -104,13 +105,28 @@ func generateMixin(directory string, mixinURL string, options mixer.GenerateOpti files, err := filepath.Glob("*") fmt.Println("in generatemixin, directory is ", directory, files) + // generate alerts, rules, grafana dashboards + // empty files if not present + + u, err := url.Parse(mixinURL) + if err != nil { + return err + } + + // absolute directory is the same as the download url stripped of the scheme + absDirectory := path.Join(u.Host, u.Path) + + fmt.Println("absDirectory is", absDirectory) + // create a temporary jsonnet file that // imports mixin.libsonnet as the "main" file in the mixin configfuration // then run generate all passing in the vendor folder to find dependencies // the name of the mixin folder inside vendor seems to be the last fragment of mixin's url + subdir // TODO: need to get absolute path of the mixin - fragment := filepath.Base(mixinURL) - tempContent := fmt.Sprintf("import \"%s\"", filepath.Join(fragment, "mixin.libsonnet")) + tempContent := fmt.Sprintf("import \"%s\"", filepath.Join(absDirectory, "mixin.libsonnet")) + + // evaluate prometheus rules and alerts + // since generateall expects a filename but here we do not need to provide a filename err = ioutil.WriteFile("temp.jsonnet", []byte(tempContent), 0644) if err != nil { return err @@ -126,6 +142,7 @@ func generateMixin(directory string, mixinURL string, options mixer.GenerateOpti return err } return nil + } func putMixin(directory string, mixinURL string, bindAddress string, options mixer.GenerateOptions) error { @@ -233,7 +250,10 @@ func installAction(c *cli.Context) error { return fmt.Errorf("Empty mixinURL") } - err = downloadMixin(mixinURL, directory) + // by default jsonnet packages are downloaded under vendor + jsonnetHome := "vendor" + + err = downloadMixin(mixinURL, jsonnetHome, directory) if err != nil { fmt.Println(err) return err @@ -247,17 +267,17 @@ func installAction(c *cli.Context) error { YAML: true, } - err = generateMixin(directory, mixinURL, generateCfg) + err = generateMixin(directory, jsonnetHome, mixinURL, generateCfg) if err != nil { return err } - bindAddress := c.String("bind-address") - // run put requests onto the server - err = putMixin(directory, mixinURL, bindAddress, generateCfg) - if err != nil { - return err - } + // bindAddress := c.String("bind-address") + // // run put requests onto the server + // err = putMixin(directory, mixinURL, bindAddress, generateCfg) + // if err != nil { + // return err + // } return nil } diff --git a/pkg/mixer/eval_snippet.go b/pkg/mixer/eval_snippet.go new file mode 100644 index 0000000..2f1caf3 --- /dev/null +++ b/pkg/mixer/eval_snippet.go @@ -0,0 +1,60 @@ +// Copyright 2018 mixtool authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package mixer + +import ( + "fmt" + + "github.com/google/go-jsonnet" +) + +// functions almost identical to eval.go except instead of reading contents from a file +// it formats in a importString and evaluates it instead + +func evaluatePrometheusAlertsSnippet(vm *jsonnet.VM, importStr string) (string, error) { + snippet := fmt.Sprintf(` +local mixin = (%s); + +if std.objectHas(mixin, "prometheusAlerts") +then mixin.prometheusAlerts +else {} +`, importStr) + + return vm.EvaluateSnippet("", snippet) +} + +func evaluatePrometheusRulesSnippet(vm *jsonnet.VM, importStr string) (string, error) { + snippet := fmt.Sprintf(` +local mixin = (%s); + +if std.objectHas(mixin, "prometheusRules") +then mixin.prometheusRules +else {} +`, importStr) + + return vm.EvaluateSnippet("", snippet) +} + +func evaluateGrafanaDashboardsSnippet(vm *jsonnet.VM, importStr string) (string, error) { + snippet := fmt.Sprintf(` +local mixin = (%s); + +if std.objectHas(mixin, "grafanaDashboards") +then mixin.grafanaDashboards +else {} +`, importStr) + + return vm.EvaluateSnippet("", snippet) +} From 0e65d852db46dc6af0bf72a9c87f00a1c470ffa1 Mon Sep 17 00:00:00 2001 From: Allen Ma Date: Tue, 17 Nov 2020 00:55:41 -0500 Subject: [PATCH 12/20] remove temp.jsonnet hack, add vm evaluation --- cmd/mixtool/eval_snippet.go | 112 +++++++++++++++++++++++++++++++++ cmd/mixtool/install.go | 119 ++++++++++++++++++++++++++++++++---- go.mod | 1 + pkg/mixer/eval_snippet.go | 60 ------------------ 4 files changed, 219 insertions(+), 73 deletions(-) create mode 100644 cmd/mixtool/eval_snippet.go delete mode 100644 pkg/mixer/eval_snippet.go diff --git a/cmd/mixtool/eval_snippet.go b/cmd/mixtool/eval_snippet.go new file mode 100644 index 0000000..2740b58 --- /dev/null +++ b/cmd/mixtool/eval_snippet.go @@ -0,0 +1,112 @@ +// Copyright 2018 mixtool authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "encoding/json" + "fmt" + + "github.com/google/go-jsonnet" + "github.com/monitoring-mixins/mixtool/pkg/mixer" + "github.com/pkg/errors" + "sigs.k8s.io/yaml" +) + +func evaluateInstallAlerts(importStr string, opts mixer.GenerateOptions) ([]byte, error) { + vm := mixer.NewVM(opts.JPaths) + + j, err := evaluateAlerts(vm, importStr) + if err != nil { + return nil, err + } + + output := []byte(j) + + if opts.YAML { + output, err = yaml.JSONToYAML(output) + if err != nil { + return nil, err + } + } + + return output, nil +} + +func evaluateInstallRules(importStr string, opts mixer.GenerateOptions) ([]byte, error) { + vm := mixer.NewVM(opts.JPaths) + + j, err := evaluateRules(vm, importStr) + if err != nil { + return nil, err + } + + output := []byte(j) + + if opts.YAML { + output, err = yaml.JSONToYAML(output) + if err != nil { + return nil, err + } + } + return output, nil +} + +func evaluateInstallDashboards(importStr string, opts mixer.GenerateOptions) (map[string]json.RawMessage, error) { + vm := mixer.NewVM(opts.JPaths) + + j, err := evaluateDashboards(vm, importStr) + if err != nil { + return nil, err + } + + var dashboards map[string]json.RawMessage + if err := json.Unmarshal([]byte(j), &dashboards); err != nil { + return nil, errors.Wrap(err, "failed to unmarshal dashboards") + } + + return dashboards, nil + +} + +func evaluateRules(vm *jsonnet.VM, importStr string) (string, error) { + snippet := fmt.Sprintf(` + local mixin = (%s); + if std.objectHasAll(mixin, "prometheusRules") + then mixin.prometheusRules + else {} + `, importStr) + return vm.EvaluateSnippet("", snippet) +} + +func evaluateAlerts(vm *jsonnet.VM, importStr string) (string, error) { + snippet := fmt.Sprintf(` + local mixin = (%s); + if std.objectHasAll(mixin, "prometheusAlerts") + then mixin.prometheusAlerts + else {} + `, importStr) + return vm.EvaluateSnippet("", snippet) +} + +func evaluateDashboards(vm *jsonnet.VM, importStr string) (string, error) { + snippet := fmt.Sprintf(` + local mixin = (%s); + if std.objectHasAll(mixin, "grafanaDashboards") + then mixin.grafanaDashboards + else {} + `, importStr) + + return vm.EvaluateSnippet("", snippet) +} diff --git a/cmd/mixtool/install.go b/cmd/mixtool/install.go index 9e37b15..3aae6e2 100644 --- a/cmd/mixtool/install.go +++ b/cmd/mixtool/install.go @@ -15,6 +15,7 @@ package main import ( + "encoding/json" "fmt" "io/ioutil" "net/http" @@ -25,6 +26,7 @@ import ( "github.com/monitoring-mixins/mixtool/pkg/jsonnetbundler" "github.com/monitoring-mixins/mixtool/pkg/mixer" + "github.com/pkg/errors" "github.com/urfave/cli" ) @@ -123,24 +125,105 @@ func generateMixin(directory string, jsonnetHome string, mixinURL string, option // then run generate all passing in the vendor folder to find dependencies // the name of the mixin folder inside vendor seems to be the last fragment of mixin's url + subdir // TODO: need to get absolute path of the mixin - tempContent := fmt.Sprintf("import \"%s\"", filepath.Join(absDirectory, "mixin.libsonnet")) + + // note - need to somehow explicitly pick up +:: hidden fields in thanos + tempContent := fmt.Sprintf( + `import "%s"`, filepath.Join(absDirectory, "mixin.libsonnet")) + + // generate rules, dashboards, alerts + err = evaluateMixin(tempContent, options) + if err != nil { + return err + } + + // tempContent := fmt.Sprintf( + // `local mixin = (import "%s"); + // mixin.grafanaDashboards`, filepath.Join(absDirectory, "mixin.libsonnet")) // evaluate prometheus rules and alerts // since generateall expects a filename but here we do not need to provide a filename - err = ioutil.WriteFile("temp.jsonnet", []byte(tempContent), 0644) + // err = ioutil.WriteFile("temp.jsonnet", []byte(tempContent), 0644) + // if err != nil { + // return err + // } + + // err = generateAll("temp.jsonnet", options) + // if err != nil { + // return err + // } + + // err = os.Remove("temp.jsonnet") + // if err != nil { + // return err + // } + // return nil + return nil + +} + +// generateMixin generates the mixin given an jsonnet importString and writes +// to files specified in options +func evaluateMixin(importStr string, options mixer.GenerateOptions) error { + fmt.Println("inside evaluateMixin") + + // rules + + out, err := evaluateInstallRules(importStr, options) + if err != nil { + return err + } + + err = ioutil.WriteFile(options.RulesFilename, out, 0644) + if err != nil { + return err + } + + // alerts + + out, err = evaluateInstallAlerts(importStr, options) if err != nil { return err } - err = generateAll("temp.jsonnet", options) + err = ioutil.WriteFile(options.AlertsFilename, out, 0644) if err != nil { return err } - err = os.Remove("temp.jsonnet") + // dashboards + + dashboards, err := evaluateInstallDashboards(importStr, options) if err != nil { return err } + + if options.Directory == "" { + return errors.New("missing directory flag to tell where to write to") + } + + if err := os.MkdirAll(options.Directory, 0755); err != nil { + return err + } + + // Creating this func so that we can make proper use of defer + writeDashboard := func(name string, dashboard json.RawMessage) error { + file, err := os.Create(filepath.Join(options.Directory, name)) + if err != nil { + return errors.Wrap(err, "failed to create dashboard file") + } + defer file.Close() + + file.Write(dashboard) + + return nil + } + + for name, dashboard := range dashboards { + if err := writeDashboard(name, dashboard); err != nil { + return err + } + } + return nil } @@ -155,17 +238,17 @@ func putMixin(directory string, mixinURL string, bindAddress string, options mix } // err := os.Chdir(filepath.Join() - if err != nil { - return fmt.Errorf("Cannot cd into directory %s", err) - } - - // // alerts.yaml - // alertsFilename := options.AlertsFilename - // alertsReader, err := os.Open(alertsFilename) // if err != nil { - // return err + // return fmt.Errorf("Cannot cd into directory %s", err) // } + // alerts.yaml + alertsFilename := options.AlertsFilename + alertsReader, err := os.Open(alertsFilename) + if err != nil { + return err + } + // rules.yaml rulesFilename := options.RulesFilename rulesReader, err := os.Open(rulesFilename) @@ -179,6 +262,7 @@ func putMixin(directory string, mixinURL string, bindAddress string, options mix } u.Path = path.Join(u.Path, "/api/v1/rules") + // request for rules req, err := http.NewRequest("PUT", u.String(), rulesReader) resp, err := http.DefaultClient.Do(req) if err != nil { @@ -191,8 +275,17 @@ func putMixin(directory string, mixinURL string, bindAddress string, options mix fmt.Printf("resp is %v\n", resp.Body) } + // same request but for alerts + req, err = http.NewRequest("PUT", u.String(), alertsReader) + resp, err = http.DefaultClient.Do(req) if err != nil { - return err + return fmt.Errorf("response from server %v", err) + } + + if resp.StatusCode == 200 { + fmt.Println("OK") + } else { + fmt.Printf("resp is %v\n", resp.Body) } return nil diff --git a/go.mod b/go.mod index a078141..a627017 100644 --- a/go.mod +++ b/go.mod @@ -28,4 +28,5 @@ require ( gopkg.in/ini.v1 v1.38.2 // indirect gopkg.in/macaron.v1 v1.3.1 // indirect gopkg.in/redis.v2 v2.3.2 // indirect + sigs.k8s.io/yaml v1.2.0 ) diff --git a/pkg/mixer/eval_snippet.go b/pkg/mixer/eval_snippet.go deleted file mode 100644 index 2f1caf3..0000000 --- a/pkg/mixer/eval_snippet.go +++ /dev/null @@ -1,60 +0,0 @@ -// Copyright 2018 mixtool authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package mixer - -import ( - "fmt" - - "github.com/google/go-jsonnet" -) - -// functions almost identical to eval.go except instead of reading contents from a file -// it formats in a importString and evaluates it instead - -func evaluatePrometheusAlertsSnippet(vm *jsonnet.VM, importStr string) (string, error) { - snippet := fmt.Sprintf(` -local mixin = (%s); - -if std.objectHas(mixin, "prometheusAlerts") -then mixin.prometheusAlerts -else {} -`, importStr) - - return vm.EvaluateSnippet("", snippet) -} - -func evaluatePrometheusRulesSnippet(vm *jsonnet.VM, importStr string) (string, error) { - snippet := fmt.Sprintf(` -local mixin = (%s); - -if std.objectHas(mixin, "prometheusRules") -then mixin.prometheusRules -else {} -`, importStr) - - return vm.EvaluateSnippet("", snippet) -} - -func evaluateGrafanaDashboardsSnippet(vm *jsonnet.VM, importStr string) (string, error) { - snippet := fmt.Sprintf(` -local mixin = (%s); - -if std.objectHas(mixin, "grafanaDashboards") -then mixin.grafanaDashboards -else {} -`, importStr) - - return vm.EvaluateSnippet("", snippet) -} From 9f243842e60cc08b0d3e53d371f9dff3b932615a Mon Sep 17 00:00:00 2001 From: Allen Ma Date: Tue, 17 Nov 2020 02:18:38 -0500 Subject: [PATCH 13/20] add flag for PUT, restructured code for generate --- cmd/mixtool/eval_snippet.go | 112 ----------------------- cmd/mixtool/generate_install.go | 93 +++++++++++++++++++ cmd/mixtool/install.go | 156 +++++--------------------------- pkg/mixer/eval_install.go | 38 ++++++++ pkg/mixer/generate_install.go | 64 +++++++++++++ 5 files changed, 219 insertions(+), 244 deletions(-) delete mode 100644 cmd/mixtool/eval_snippet.go create mode 100644 cmd/mixtool/generate_install.go create mode 100644 pkg/mixer/eval_install.go create mode 100644 pkg/mixer/generate_install.go diff --git a/cmd/mixtool/eval_snippet.go b/cmd/mixtool/eval_snippet.go deleted file mode 100644 index 2740b58..0000000 --- a/cmd/mixtool/eval_snippet.go +++ /dev/null @@ -1,112 +0,0 @@ -// Copyright 2018 mixtool authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package main - -import ( - "encoding/json" - "fmt" - - "github.com/google/go-jsonnet" - "github.com/monitoring-mixins/mixtool/pkg/mixer" - "github.com/pkg/errors" - "sigs.k8s.io/yaml" -) - -func evaluateInstallAlerts(importStr string, opts mixer.GenerateOptions) ([]byte, error) { - vm := mixer.NewVM(opts.JPaths) - - j, err := evaluateAlerts(vm, importStr) - if err != nil { - return nil, err - } - - output := []byte(j) - - if opts.YAML { - output, err = yaml.JSONToYAML(output) - if err != nil { - return nil, err - } - } - - return output, nil -} - -func evaluateInstallRules(importStr string, opts mixer.GenerateOptions) ([]byte, error) { - vm := mixer.NewVM(opts.JPaths) - - j, err := evaluateRules(vm, importStr) - if err != nil { - return nil, err - } - - output := []byte(j) - - if opts.YAML { - output, err = yaml.JSONToYAML(output) - if err != nil { - return nil, err - } - } - return output, nil -} - -func evaluateInstallDashboards(importStr string, opts mixer.GenerateOptions) (map[string]json.RawMessage, error) { - vm := mixer.NewVM(opts.JPaths) - - j, err := evaluateDashboards(vm, importStr) - if err != nil { - return nil, err - } - - var dashboards map[string]json.RawMessage - if err := json.Unmarshal([]byte(j), &dashboards); err != nil { - return nil, errors.Wrap(err, "failed to unmarshal dashboards") - } - - return dashboards, nil - -} - -func evaluateRules(vm *jsonnet.VM, importStr string) (string, error) { - snippet := fmt.Sprintf(` - local mixin = (%s); - if std.objectHasAll(mixin, "prometheusRules") - then mixin.prometheusRules - else {} - `, importStr) - return vm.EvaluateSnippet("", snippet) -} - -func evaluateAlerts(vm *jsonnet.VM, importStr string) (string, error) { - snippet := fmt.Sprintf(` - local mixin = (%s); - if std.objectHasAll(mixin, "prometheusAlerts") - then mixin.prometheusAlerts - else {} - `, importStr) - return vm.EvaluateSnippet("", snippet) -} - -func evaluateDashboards(vm *jsonnet.VM, importStr string) (string, error) { - snippet := fmt.Sprintf(` - local mixin = (%s); - if std.objectHasAll(mixin, "grafanaDashboards") - then mixin.grafanaDashboards - else {} - `, importStr) - - return vm.EvaluateSnippet("", snippet) -} diff --git a/cmd/mixtool/generate_install.go b/cmd/mixtool/generate_install.go new file mode 100644 index 0000000..0ef89ed --- /dev/null +++ b/cmd/mixtool/generate_install.go @@ -0,0 +1,93 @@ +// Copyright 2018 mixtool authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "encoding/json" + "fmt" + "io/ioutil" + "os" + "path/filepath" + + "github.com/monitoring-mixins/mixtool/pkg/mixer" + "github.com/pkg/errors" +) + +// generateMixin generates the mixin given an jsonnet importString and writes +// to files specified in options +func generateAllMixin(importStr string, options mixer.GenerateOptions) error { + fmt.Println("inside evaluateMixin") + + // rules + + out, err := mixer.GenerateInstallRules(importStr, options) + if err != nil { + return err + } + + err = ioutil.WriteFile(options.RulesFilename, out, 0644) + if err != nil { + return err + } + + // alerts + + out, err = mixer.GenerateInstallAlerts(importStr, options) + if err != nil { + return err + } + + err = ioutil.WriteFile(options.AlertsFilename, out, 0644) + if err != nil { + return err + } + + // dashboards + + dashboards, err := mixer.GenerateInstallDashboards(importStr, options) + if err != nil { + return err + } + + if options.Directory == "" { + return errors.New("missing directory flag to tell where to write to") + } + + if err := os.MkdirAll(options.Directory, 0755); err != nil { + return err + } + + // Creating this func so that we can make proper use of defer + writeDashboard := func(name string, dashboard json.RawMessage) error { + file, err := os.Create(filepath.Join(options.Directory, name)) + if err != nil { + return errors.Wrap(err, "failed to create dashboard file") + } + defer file.Close() + + file.Write(dashboard) + + return nil + } + + for name, dashboard := range dashboards { + if err := writeDashboard(name, dashboard); err != nil { + return err + } + } + + return nil + +} diff --git a/cmd/mixtool/install.go b/cmd/mixtool/install.go index 3aae6e2..bda3aa9 100644 --- a/cmd/mixtool/install.go +++ b/cmd/mixtool/install.go @@ -15,9 +15,7 @@ package main import ( - "encoding/json" "fmt" - "io/ioutil" "net/http" "net/url" "os" @@ -26,7 +24,6 @@ import ( "github.com/monitoring-mixins/mixtool/pkg/jsonnetbundler" "github.com/monitoring-mixins/mixtool/pkg/mixer" - "github.com/pkg/errors" "github.com/urfave/cli" ) @@ -50,14 +47,14 @@ func installCommand() cli.Command { Usage: "Address to bind HTTP server to.", Value: "http://127.0.0.1:8080", }, - cli.StringFlag{ - Name: "rule-file", - Usage: "File to provision rules into.", - }, cli.StringFlag{ Name: "directory, d", Usage: "Path where the downloaded mixin is saved. If it doesn't exist already it will be created", }, + cli.BoolFlag{ + Name: "put, p", + Usage: "Specify this flag when you want to send PUT request to mixtool server once the mixins are generated", + }, }, } } @@ -68,14 +65,13 @@ func downloadMixin(url string, jsonnetHome string, directory string) error { // intialize the jsonnet bundler library err := jsonnetbundler.InitCommand(directory) if err != nil { - return err + return fmt.Errorf("jsonnet bundler init failed %v", err) } // by default, set the single flag to false err = jsonnetbundler.InstallCommand(directory, jsonnetHome, []string{url}, false) if err != nil { - fmt.Println(err) - return err + return fmt.Errorf("jsonnet bundler install failed %v", err) } return nil @@ -112,7 +108,7 @@ func generateMixin(directory string, jsonnetHome string, mixinURL string, option u, err := url.Parse(mixinURL) if err != nil { - return err + return fmt.Errorf("url parse %v", err) } // absolute directory is the same as the download url stripped of the scheme @@ -120,108 +116,13 @@ func generateMixin(directory string, jsonnetHome string, mixinURL string, option fmt.Println("absDirectory is", absDirectory) - // create a temporary jsonnet file that - // imports mixin.libsonnet as the "main" file in the mixin configfuration - // then run generate all passing in the vendor folder to find dependencies - // the name of the mixin folder inside vendor seems to be the last fragment of mixin's url + subdir - // TODO: need to get absolute path of the mixin - - // note - need to somehow explicitly pick up +:: hidden fields in thanos tempContent := fmt.Sprintf( `import "%s"`, filepath.Join(absDirectory, "mixin.libsonnet")) // generate rules, dashboards, alerts - err = evaluateMixin(tempContent, options) + err = generateAllMixin(tempContent, options) if err != nil { - return err - } - - // tempContent := fmt.Sprintf( - // `local mixin = (import "%s"); - // mixin.grafanaDashboards`, filepath.Join(absDirectory, "mixin.libsonnet")) - - // evaluate prometheus rules and alerts - // since generateall expects a filename but here we do not need to provide a filename - // err = ioutil.WriteFile("temp.jsonnet", []byte(tempContent), 0644) - // if err != nil { - // return err - // } - - // err = generateAll("temp.jsonnet", options) - // if err != nil { - // return err - // } - - // err = os.Remove("temp.jsonnet") - // if err != nil { - // return err - // } - // return nil - return nil - -} - -// generateMixin generates the mixin given an jsonnet importString and writes -// to files specified in options -func evaluateMixin(importStr string, options mixer.GenerateOptions) error { - fmt.Println("inside evaluateMixin") - - // rules - - out, err := evaluateInstallRules(importStr, options) - if err != nil { - return err - } - - err = ioutil.WriteFile(options.RulesFilename, out, 0644) - if err != nil { - return err - } - - // alerts - - out, err = evaluateInstallAlerts(importStr, options) - if err != nil { - return err - } - - err = ioutil.WriteFile(options.AlertsFilename, out, 0644) - if err != nil { - return err - } - - // dashboards - - dashboards, err := evaluateInstallDashboards(importStr, options) - if err != nil { - return err - } - - if options.Directory == "" { - return errors.New("missing directory flag to tell where to write to") - } - - if err := os.MkdirAll(options.Directory, 0755); err != nil { - return err - } - - // Creating this func so that we can make proper use of defer - writeDashboard := func(name string, dashboard json.RawMessage) error { - file, err := os.Create(filepath.Join(options.Directory, name)) - if err != nil { - return errors.Wrap(err, "failed to create dashboard file") - } - defer file.Close() - - file.Write(dashboard) - - return nil - } - - for name, dashboard := range dashboards { - if err := writeDashboard(name, dashboard); err != nil { - return err - } + return fmt.Errorf("generateAllMixins %v", err) } return nil @@ -229,18 +130,6 @@ func evaluateMixin(importStr string, options mixer.GenerateOptions) error { } func putMixin(directory string, mixinURL string, bindAddress string, options mixer.GenerateOptions) error { - fmt.Println("running put Mixin") - - wd, err := os.Getwd() - fmt.Println("path is", wd) - if err != nil { - return err - } - - // err := os.Chdir(filepath.Join() - // if err != nil { - // return fmt.Errorf("Cannot cd into directory %s", err) - // } // alerts.yaml alertsFilename := options.AlertsFilename @@ -270,7 +159,7 @@ func putMixin(directory string, mixinURL string, bindAddress string, options mix } if resp.StatusCode == 200 { - fmt.Println("OK") + fmt.Println("PUT rules OK") } else { fmt.Printf("resp is %v\n", resp.Body) } @@ -283,7 +172,7 @@ func putMixin(directory string, mixinURL string, bindAddress string, options mix } if resp.StatusCode == 200 { - fmt.Println("OK") + fmt.Println("PUT alerts OK") } else { fmt.Printf("resp is %v\n", resp.Body) } @@ -301,7 +190,7 @@ func installAction(c *cli.Context) error { if os.IsNotExist(err) { err = os.MkdirAll(directory, 0755) if err != nil { - return err + return fmt.Errorf("could not create directory %v", err) } } @@ -312,7 +201,7 @@ func installAction(c *cli.Context) error { mixinsList, err := getMixins() if err != nil { - return err + return fmt.Errorf("getMixins failed %v", err) } var mixinURL string @@ -324,7 +213,7 @@ func installAction(c *cli.Context) error { // join paths together u, err := url.Parse(m.URL) if err != nil { - return err + return fmt.Errorf("url parse failed %v", err) } u.Path = path.Join(u.Path, m.Subdir) mixinURL = u.String() @@ -348,7 +237,6 @@ func installAction(c *cli.Context) error { err = downloadMixin(mixinURL, jsonnetHome, directory) if err != nil { - fmt.Println(err) return err } @@ -365,12 +253,16 @@ func installAction(c *cli.Context) error { return err } - // bindAddress := c.String("bind-address") - // // run put requests onto the server - // err = putMixin(directory, mixinURL, bindAddress, generateCfg) - // if err != nil { - // return err - // } + // check if put address flag was set + + if c.Bool("put") { + bindAddress := c.String("bind-address") + // run put requests onto the server + err = putMixin(directory, mixinURL, bindAddress, generateCfg) + if err != nil { + return err + } + } return nil } diff --git a/pkg/mixer/eval_install.go b/pkg/mixer/eval_install.go new file mode 100644 index 0000000..eeb701a --- /dev/null +++ b/pkg/mixer/eval_install.go @@ -0,0 +1,38 @@ +package mixer + +import ( + "fmt" + + "github.com/google/go-jsonnet" +) + +func evaluateRules(vm *jsonnet.VM, importStr string) (string, error) { + snippet := fmt.Sprintf(` + local mixin = (%s); + if std.objectHasAll(mixin, "prometheusRules") + then mixin.prometheusRules + else {} + `, importStr) + return vm.EvaluateSnippet("", snippet) +} + +func evaluateAlerts(vm *jsonnet.VM, importStr string) (string, error) { + snippet := fmt.Sprintf(` + local mixin = (%s); + if std.objectHasAll(mixin, "prometheusAlerts") + then mixin.prometheusAlerts + else {} + `, importStr) + return vm.EvaluateSnippet("", snippet) +} + +func evaluateDashboards(vm *jsonnet.VM, importStr string) (string, error) { + snippet := fmt.Sprintf(` + local mixin = (%s); + if std.objectHasAll(mixin, "grafanaDashboards") + then mixin.grafanaDashboards + else {} + `, importStr) + + return vm.EvaluateSnippet("", snippet) +} diff --git a/pkg/mixer/generate_install.go b/pkg/mixer/generate_install.go new file mode 100644 index 0000000..32055ef --- /dev/null +++ b/pkg/mixer/generate_install.go @@ -0,0 +1,64 @@ +package mixer + +import ( + "encoding/json" + + "github.com/pkg/errors" + "sigs.k8s.io/yaml" +) + +func GenerateInstallAlerts(importStr string, opts GenerateOptions) ([]byte, error) { + vm := NewVM(opts.JPaths) + + j, err := evaluateAlerts(vm, importStr) + if err != nil { + return nil, err + } + + output := []byte(j) + + if opts.YAML { + output, err = yaml.JSONToYAML(output) + if err != nil { + return nil, err + } + } + + return output, nil +} + +func GenerateInstallRules(importStr string, opts GenerateOptions) ([]byte, error) { + vm := NewVM(opts.JPaths) + + j, err := evaluateRules(vm, importStr) + if err != nil { + return nil, err + } + + output := []byte(j) + + if opts.YAML { + output, err = yaml.JSONToYAML(output) + if err != nil { + return nil, err + } + } + return output, nil +} + +func GenerateInstallDashboards(importStr string, opts GenerateOptions) (map[string]json.RawMessage, error) { + vm := NewVM(opts.JPaths) + + j, err := evaluateDashboards(vm, importStr) + if err != nil { + return nil, err + } + + var dashboards map[string]json.RawMessage + if err := json.Unmarshal([]byte(j), &dashboards); err != nil { + return nil, errors.Wrap(err, "failed to unmarshal dashboards") + } + + return dashboards, nil + +} From 3c493dea5397fffb304ac8ca52bf75fc79f0de42 Mon Sep 17 00:00:00 2001 From: Allen Ma Date: Wed, 18 Nov 2020 14:52:27 -0500 Subject: [PATCH 14/20] remove superfluous code for generate - pass in a filename string now --- cmd/mixtool/generate_install.go | 93 --------------------------------- cmd/mixtool/install.go | 9 ++-- cmd/mixtool/install_test.go | 19 ++++--- pkg/mixer/eval.go | 6 +-- pkg/mixer/eval_install.go | 38 -------------- pkg/mixer/generate_install.go | 64 ----------------------- 6 files changed, 20 insertions(+), 209 deletions(-) delete mode 100644 cmd/mixtool/generate_install.go delete mode 100644 pkg/mixer/eval_install.go delete mode 100644 pkg/mixer/generate_install.go diff --git a/cmd/mixtool/generate_install.go b/cmd/mixtool/generate_install.go deleted file mode 100644 index 0ef89ed..0000000 --- a/cmd/mixtool/generate_install.go +++ /dev/null @@ -1,93 +0,0 @@ -// Copyright 2018 mixtool authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package main - -import ( - "encoding/json" - "fmt" - "io/ioutil" - "os" - "path/filepath" - - "github.com/monitoring-mixins/mixtool/pkg/mixer" - "github.com/pkg/errors" -) - -// generateMixin generates the mixin given an jsonnet importString and writes -// to files specified in options -func generateAllMixin(importStr string, options mixer.GenerateOptions) error { - fmt.Println("inside evaluateMixin") - - // rules - - out, err := mixer.GenerateInstallRules(importStr, options) - if err != nil { - return err - } - - err = ioutil.WriteFile(options.RulesFilename, out, 0644) - if err != nil { - return err - } - - // alerts - - out, err = mixer.GenerateInstallAlerts(importStr, options) - if err != nil { - return err - } - - err = ioutil.WriteFile(options.AlertsFilename, out, 0644) - if err != nil { - return err - } - - // dashboards - - dashboards, err := mixer.GenerateInstallDashboards(importStr, options) - if err != nil { - return err - } - - if options.Directory == "" { - return errors.New("missing directory flag to tell where to write to") - } - - if err := os.MkdirAll(options.Directory, 0755); err != nil { - return err - } - - // Creating this func so that we can make proper use of defer - writeDashboard := func(name string, dashboard json.RawMessage) error { - file, err := os.Create(filepath.Join(options.Directory, name)) - if err != nil { - return errors.Wrap(err, "failed to create dashboard file") - } - defer file.Close() - - file.Write(dashboard) - - return nil - } - - for name, dashboard := range dashboards { - if err := writeDashboard(name, dashboard); err != nil { - return err - } - } - - return nil - -} diff --git a/cmd/mixtool/install.go b/cmd/mixtool/install.go index bda3aa9..3e99734 100644 --- a/cmd/mixtool/install.go +++ b/cmd/mixtool/install.go @@ -21,6 +21,7 @@ import ( "os" "path" "path/filepath" + "strings" "github.com/monitoring-mixins/mixtool/pkg/jsonnetbundler" "github.com/monitoring-mixins/mixtool/pkg/mixer" @@ -114,13 +115,15 @@ func generateMixin(directory string, jsonnetHome string, mixinURL string, option // absolute directory is the same as the download url stripped of the scheme absDirectory := path.Join(u.Host, u.Path) + // strip leading slashes and colons + absDirectory = strings.TrimLeft(absDirectory, "/:") + fmt.Println("absDirectory is", absDirectory) - tempContent := fmt.Sprintf( - `import "%s"`, filepath.Join(absDirectory, "mixin.libsonnet")) + importFile := filepath.Join(absDirectory, "mixin.libsonnet") // generate rules, dashboards, alerts - err = generateAllMixin(tempContent, options) + err = generateAll(importFile, options) if err != nil { return fmt.Errorf("generateAllMixins %v", err) } diff --git a/cmd/mixtool/install_test.go b/cmd/mixtool/install_test.go index 55e18e4..4fed349 100644 --- a/cmd/mixtool/install_test.go +++ b/cmd/mixtool/install_test.go @@ -18,7 +18,7 @@ func TestInstallMixin(t *testing.T) { t.Errorf("failed to make directory %v", err) } - // defer os.RemoveAll(tmpdir) + defer os.RemoveAll(tmpdir) body, err := queryWebsite(defaultWebsite) if err != nil { @@ -43,24 +43,26 @@ func TestInstallMixin(t *testing.T) { mixinURL := path.Join(m.URL, m.Subdir) fmt.Printf("installing %v\n", mixinURL) - dldir := path.Join(tmpdir, m.Name) + dldir := path.Join(tmpdir, m.Name+"mixin-test") err = os.Mkdir(dldir, 0755) if err != nil { - t.Errorf("failed to create directory %v", dldir) + t.Errorf("failed to create directory %s", dldir) } - err = downloadMixin(mixinURL, dldir) + jsonnetHome := "vendor" + + err = downloadMixin(mixinURL, jsonnetHome, dldir) if err != nil { - t.Errorf("failed to download mixin at %v. %v", mixinURL, err) + t.Errorf("failed to download mixin at %s: %w", mixinURL, err) } - err = generateMixin(dldir, mixinURL, generateCfg) + err = generateMixin(dldir, jsonnetHome, mixinURL, generateCfg) if err != nil { - t.Errorf("failed to generate mixin yaml for %v. %v", mixinURL, err) + t.Errorf("failed to generate mixin yaml for %s: %w", mixinURL, err) } - // verify that the contents are correct + // verify that alerts, rules, dashboards exist err = os.Chdir(dldir) if err != nil { t.Errorf("could not cd into %s", dldir) @@ -78,6 +80,7 @@ func TestInstallMixin(t *testing.T) { t.Errorf("expected dashboards_out in %s", dldir) } + // verify that the output of alerts and rules matches using jsonnet } } diff --git a/pkg/mixer/eval.go b/pkg/mixer/eval.go index e1a7922..4c29771 100644 --- a/pkg/mixer/eval.go +++ b/pkg/mixer/eval.go @@ -24,7 +24,7 @@ func evaluatePrometheusAlerts(vm *jsonnet.VM, filename string) (string, error) { snippet := fmt.Sprintf(` local mixin = (import %q); -if std.objectHas(mixin, "prometheusAlerts") +if std.objectHasAll(mixin, "prometheusAlerts") then mixin.prometheusAlerts else {} `, filename) @@ -36,7 +36,7 @@ func evaluatePrometheusRules(vm *jsonnet.VM, filename string) (string, error) { snippet := fmt.Sprintf(` local mixin = (import %q); -if std.objectHas(mixin, "prometheusRules") +if std.objectHasAll(mixin, "prometheusRules") then mixin.prometheusRules else {} `, filename) @@ -48,7 +48,7 @@ func evaluateGrafanaDashboards(vm *jsonnet.VM, filename string) (string, error) snippet := fmt.Sprintf(` local mixin = (import %q); -if std.objectHas(mixin, "grafanaDashboards") +if std.objectHasAll(mixin, "grafanaDashboards") then mixin.grafanaDashboards else {} `, filename) diff --git a/pkg/mixer/eval_install.go b/pkg/mixer/eval_install.go deleted file mode 100644 index eeb701a..0000000 --- a/pkg/mixer/eval_install.go +++ /dev/null @@ -1,38 +0,0 @@ -package mixer - -import ( - "fmt" - - "github.com/google/go-jsonnet" -) - -func evaluateRules(vm *jsonnet.VM, importStr string) (string, error) { - snippet := fmt.Sprintf(` - local mixin = (%s); - if std.objectHasAll(mixin, "prometheusRules") - then mixin.prometheusRules - else {} - `, importStr) - return vm.EvaluateSnippet("", snippet) -} - -func evaluateAlerts(vm *jsonnet.VM, importStr string) (string, error) { - snippet := fmt.Sprintf(` - local mixin = (%s); - if std.objectHasAll(mixin, "prometheusAlerts") - then mixin.prometheusAlerts - else {} - `, importStr) - return vm.EvaluateSnippet("", snippet) -} - -func evaluateDashboards(vm *jsonnet.VM, importStr string) (string, error) { - snippet := fmt.Sprintf(` - local mixin = (%s); - if std.objectHasAll(mixin, "grafanaDashboards") - then mixin.grafanaDashboards - else {} - `, importStr) - - return vm.EvaluateSnippet("", snippet) -} diff --git a/pkg/mixer/generate_install.go b/pkg/mixer/generate_install.go deleted file mode 100644 index 32055ef..0000000 --- a/pkg/mixer/generate_install.go +++ /dev/null @@ -1,64 +0,0 @@ -package mixer - -import ( - "encoding/json" - - "github.com/pkg/errors" - "sigs.k8s.io/yaml" -) - -func GenerateInstallAlerts(importStr string, opts GenerateOptions) ([]byte, error) { - vm := NewVM(opts.JPaths) - - j, err := evaluateAlerts(vm, importStr) - if err != nil { - return nil, err - } - - output := []byte(j) - - if opts.YAML { - output, err = yaml.JSONToYAML(output) - if err != nil { - return nil, err - } - } - - return output, nil -} - -func GenerateInstallRules(importStr string, opts GenerateOptions) ([]byte, error) { - vm := NewVM(opts.JPaths) - - j, err := evaluateRules(vm, importStr) - if err != nil { - return nil, err - } - - output := []byte(j) - - if opts.YAML { - output, err = yaml.JSONToYAML(output) - if err != nil { - return nil, err - } - } - return output, nil -} - -func GenerateInstallDashboards(importStr string, opts GenerateOptions) (map[string]json.RawMessage, error) { - vm := NewVM(opts.JPaths) - - j, err := evaluateDashboards(vm, importStr) - if err != nil { - return nil, err - } - - var dashboards map[string]json.RawMessage - if err := json.Unmarshal([]byte(j), &dashboards); err != nil { - return nil, errors.Wrap(err, "failed to unmarshal dashboards") - } - - return dashboards, nil - -} From 607aed2e32604728de1aae860f0917c99eff1a96 Mon Sep 17 00:00:00 2001 From: Allen Ma Date: Thu, 19 Nov 2020 21:59:48 -0500 Subject: [PATCH 15/20] change server and install --- cmd/mixtool/install.go | 42 +++++++++++++------------ cmd/mixtool/server.go | 71 ++++++++++++++++++++++-------------------- 2 files changed, 59 insertions(+), 54 deletions(-) diff --git a/cmd/mixtool/install.go b/cmd/mixtool/install.go index 3e99734..4d23497 100644 --- a/cmd/mixtool/install.go +++ b/cmd/mixtool/install.go @@ -141,12 +141,14 @@ func putMixin(directory string, mixinURL string, bindAddress string, options mix return err } - // rules.yaml - rulesFilename := options.RulesFilename - rulesReader, err := os.Open(rulesFilename) - if err != nil { - return err - } + // // rules.yaml + // rulesFilename := options.RulesFilename + // rulesReader, err := os.Open(rulesFilename) + // if err != nil { + // return err + // } + + // merge rules and alerts together u, err := url.Parse(bindAddress) if err != nil { @@ -154,22 +156,22 @@ func putMixin(directory string, mixinURL string, bindAddress string, options mix } u.Path = path.Join(u.Path, "/api/v1/rules") - // request for rules - req, err := http.NewRequest("PUT", u.String(), rulesReader) - resp, err := http.DefaultClient.Do(req) - if err != nil { - return fmt.Errorf("response from server %v", err) - } + // // request for rules + // req, err := http.NewRequest("PUT", u.String(), rulesReader) + // resp, err := http.DefaultClient.Do(req) + // if err != nil { + // return fmt.Errorf("response from server %v", err) + // } - if resp.StatusCode == 200 { - fmt.Println("PUT rules OK") - } else { - fmt.Printf("resp is %v\n", resp.Body) - } + // if resp.StatusCode == 200 { + // fmt.Println("PUT rules OK") + // } else { + // fmt.Printf("resp is %v\n", resp.Body) + // } // same request but for alerts - req, err = http.NewRequest("PUT", u.String(), alertsReader) - resp, err = http.DefaultClient.Do(req) + req, err := http.NewRequest("PUT", u.String(), alertsReader) + resp, err := http.DefaultClient.Do(req) if err != nil { return fmt.Errorf("response from server %v", err) } @@ -177,7 +179,7 @@ func putMixin(directory string, mixinURL string, bindAddress string, options mix if resp.StatusCode == 200 { fmt.Println("PUT alerts OK") } else { - fmt.Printf("resp is %v\n", resp.Body) + return fmt.Errorf("response code: %d resp is %v", resp.StatusCode, resp.Body) } return nil diff --git a/cmd/mixtool/server.go b/cmd/mixtool/server.go index 9131adb..f683629 100644 --- a/cmd/mixtool/server.go +++ b/cmd/mixtool/server.go @@ -16,7 +16,6 @@ package main import ( "bufio" - "bytes" "context" "errors" "fmt" @@ -24,6 +23,7 @@ import ( "io/ioutil" "net/http" "os" + "path/filepath" "github.com/urfave/cli" ) @@ -84,7 +84,6 @@ func (h *ruleProvisioningHandler) ServeHTTP(w http.ResponseWriter, r *http.Reque } if reloadNecessary { - fmt.Println("reloading prometheus") if err := h.prometheusReloader.triggerReload(ctx); err != nil { http.Error(w, fmt.Sprintf("Internal Server Error: %v", err), http.StatusInternalServerError) return @@ -100,51 +99,60 @@ type ruleProvisioner struct { // to existing, does not provision them. It returns whether Prometheus should // be reloaded and if an error has occurred. func (p *ruleProvisioner) provision(r io.Reader) (bool, error) { - fmt.Println("trying to provision 1") - b := bytes.NewBuffer(nil) - tr := io.TeeReader(r, b) - - f, err := os.OpenFile(p.ruleFile, os.O_RDWR, 0644) - if err != nil && !os.IsNotExist(err) { - fmt.Println(err, p.ruleFile) - return false, fmt.Errorf("open rule file: %w", err) - } - if os.IsNotExist(err) { - f, err = os.Create(p.ruleFile) - if err != nil { - return false, fmt.Errorf("create rule file: %w", err) - } + newData, err := ioutil.ReadAll(r) + if err != nil { + return false, fmt.Errorf("unable to read new rules: %w", err) } - equal, err := readersEqual(tr, f) + tempfile, err := ioutil.TempFile(filepath.Dir(p.ruleFile), "temp-mixtool") if err != nil { - return false, fmt.Errorf("compare existing rules with provisioned intention: %w", err) + return false, fmt.Errorf("unable to create temp file: %w", err) } - if equal { - return false, nil + + n, err := tempfile.Write(newData) + if err != nil { + return false, fmt.Errorf("error when writing new rules: %w", err) } - fmt.Println("trying to provision 2") + if n != len(newData) { + return false, fmt.Errorf("writing error, wrote %d bytes, expected %d", n, len(newData)) + } + + tempfile.Sync() + + ruleFileReader, err := os.OpenFile(p.ruleFile, os.O_RDWR, 0644) + if err != nil { + return false, fmt.Errorf("unable to read existing rules: %w", err) + } - if err := f.Truncate(0); err != nil { - fmt.Println("err is", err) - return false, fmt.Errorf("truncate file: %w", err) + newFileReader, err := os.OpenFile(tempfile.Name(), os.O_RDWR, 0644) + if err != nil { + return false, fmt.Errorf("unable to open new rules file: %w", err) } - _, err = f.Seek(0, 0) + equal, err := readersEqual(newFileReader, ruleFileReader) if err != nil { - return false, fmt.Errorf("seek file %w", err) + return false, fmt.Errorf("error from readersEqual: %w", err) } - if _, err := io.Copy(f, b); err != nil { - return false, fmt.Errorf("provision rule to file: %w", err) + if equal { + fmt.Println("don't need to reload") + return false, nil } + fmt.Printf("need to remove oldrulefile @ %s, replace with newfile name %s\n", p.ruleFile, tempfile.Name()) + + if err = os.Rename(tempfile.Name(), p.ruleFile); err != nil { + return false, fmt.Errorf("cannot rename rules file: %w", err) + } return true, nil } +type prometheusReloader struct { + prometheusReloadURL string +} + func readersEqual(r1, r2 io.Reader) (bool, error) { - fmt.Println("comparing rn") buf1 := bufio.NewReader(r1) buf2 := bufio.NewReader(r2) for { @@ -165,12 +173,7 @@ func readersEqual(r1, r2 io.Reader) (bool, error) { } } -type prometheusReloader struct { - prometheusReloadURL string -} - func (r *prometheusReloader) triggerReload(ctx context.Context) error { - fmt.Println("triggering reload") req, err := http.NewRequest("POST", r.prometheusReloadURL, nil) if err != nil { return fmt.Errorf("create request: %w", err) From 448b64658340fd57677b77984d2a9d1d21d0dd5f Mon Sep 17 00:00:00 2001 From: Allen Ma Date: Mon, 23 Nov 2020 20:31:22 -0500 Subject: [PATCH 16/20] merge rules and alerts into rules-alerts.yaml for now --- cmd/mixtool/generate.go | 10 +++++++++- cmd/mixtool/install.go | 43 +++++++++++++++++++++++++++-------------- pkg/mixer/eval.go | 16 +++++++++++++++ pkg/mixer/generate.go | 20 +++++++++++++++++++ 4 files changed, 74 insertions(+), 15 deletions(-) diff --git a/cmd/mixtool/generate.go b/cmd/mixtool/generate.go index 02d1d43..8066640 100644 --- a/cmd/mixtool/generate.go +++ b/cmd/mixtool/generate.go @@ -151,7 +151,6 @@ func generateRules(filename string, options mixer.GenerateOptions) error { if err != nil { return err } - fmt.Println("inside generate rules", options.RulesFilename) return ioutil.WriteFile(options.RulesFilename, out, 0644) } @@ -191,6 +190,15 @@ func generateDashboards(filename string, opts mixer.GenerateOptions) error { return nil } +func generateRulesAlerts(filename string, options mixer.GenerateOptions) error { + out, err := mixer.GenerateRulesAlerts(filename, options) + if err != nil { + return err + } + fmt.Println("inside generate rules, alerts") + return ioutil.WriteFile("rules-alerts.yaml", out, 0644) +} + func generateAll(filename string, opts mixer.GenerateOptions) error { if err := generateAlerts(filename, opts); err != nil { return err diff --git a/cmd/mixtool/install.go b/cmd/mixtool/install.go index 4d23497..8226637 100644 --- a/cmd/mixtool/install.go +++ b/cmd/mixtool/install.go @@ -29,13 +29,6 @@ import ( "github.com/urfave/cli" ) -// type mixin struct { -// URL string `json:"source"` -// Description string `json:"description,omitempty"` -// Name string `json:"name"` -// Subdir string `json:"subdir"` -// } - func installCommand() cli.Command { return cli.Command{ Name: "install", @@ -128,18 +121,23 @@ func generateMixin(directory string, jsonnetHome string, mixinURL string, option return fmt.Errorf("generateAllMixins %v", err) } + err = generateRulesAlerts(importFile, options) + if err != nil { + return fmt.Errorf("generateRulesAlerts %v", err) + } + return nil } func putMixin(directory string, mixinURL string, bindAddress string, options mixer.GenerateOptions) error { - // alerts.yaml - alertsFilename := options.AlertsFilename - alertsReader, err := os.Open(alertsFilename) - if err != nil { - return err - } + // // alerts.yaml + // alertsFilename := options.AlertsFilename + // alertsReader, err := os.Open(alertsFilename) + // if err != nil { + // return err + // } // // rules.yaml // rulesFilename := options.RulesFilename @@ -170,6 +168,24 @@ func putMixin(directory string, mixinURL string, bindAddress string, options mix // } // same request but for alerts + // req, err := http.NewRequest("PUT", u.String(), alertsReader) + // resp, err := http.DefaultClient.Do(req) + // if err != nil { + // return fmt.Errorf("response from server %v", err) + // } + + // if resp.StatusCode == 200 { + // fmt.Println("PUT alerts OK") + // } else { + // return fmt.Errorf("response code: %d resp is %v", resp.StatusCode, resp.Body) + // } + + // temporary: generate rules and alerts in a single file + alertsReader, err := os.Open("rules-alerts.yaml") + if err != nil { + return err + } + req, err := http.NewRequest("PUT", u.String(), alertsReader) resp, err := http.DefaultClient.Do(req) if err != nil { @@ -181,7 +197,6 @@ func putMixin(directory string, mixinURL string, bindAddress string, options mix } else { return fmt.Errorf("response code: %d resp is %v", resp.StatusCode, resp.Body) } - return nil } diff --git a/pkg/mixer/eval.go b/pkg/mixer/eval.go index 4c29771..606eeb2 100644 --- a/pkg/mixer/eval.go +++ b/pkg/mixer/eval.go @@ -44,6 +44,22 @@ else {} return vm.EvaluateSnippet("", snippet) } +func evaluatePrometheusRulesAlerts(vm *jsonnet.VM, filename string) (string, error) { + snippet := fmt.Sprintf(` +local mixin = (import %q); + +if std.objectHasAll(mixin, "prometheusRules") && std.objectHasAll(mixin, "prometheusAlerts") +then mixin.prometheusRules + mixin.prometheusAlerts +else if std.objectHasAll(mixin, "prometheusRules") +then mixin.prometheusRules +else if std.objectHasAll(mixin, "prometheusAlerts") +then mixin.prometheusAlerts +else {} +`, filename) + + return vm.EvaluateSnippet("", snippet) +} + func evaluateGrafanaDashboards(vm *jsonnet.VM, filename string) (string, error) { snippet := fmt.Sprintf(` local mixin = (import %q); diff --git a/pkg/mixer/generate.go b/pkg/mixer/generate.go index 834ce3a..f731be2 100644 --- a/pkg/mixer/generate.go +++ b/pkg/mixer/generate.go @@ -82,6 +82,26 @@ func GenerateRules(filename string, opts GenerateOptions) ([]byte, error) { return output, nil } +func GenerateRulesAlerts(filename string, opts GenerateOptions) ([]byte, error) { + vm := NewVM(opts.JPaths) + + j, err := evaluatePrometheusRulesAlerts(vm, filename) + if err != nil { + return nil, err + } + + output := []byte(j) + + if opts.YAML { + output, err = yaml.JSONToYAML(output) + if err != nil { + return nil, err + } + } + + return output, nil +} + func GenerateDashboards(filename string, opts GenerateOptions) (map[string]json.RawMessage, error) { vm := NewVM(opts.JPaths) From 5777aeacb50ef650dbeeae4d64ceaaf0871e4c88 Mon Sep 17 00:00:00 2001 From: Allen Ma Date: Wed, 25 Nov 2020 23:23:58 -0500 Subject: [PATCH 17/20] add changes to server --- cmd/mixtool/server.go | 90 +++++++++++-------------------------------- go.mod | 1 + 2 files changed, 23 insertions(+), 68 deletions(-) diff --git a/cmd/mixtool/server.go b/cmd/mixtool/server.go index f683629..16fa4c2 100644 --- a/cmd/mixtool/server.go +++ b/cmd/mixtool/server.go @@ -15,15 +15,14 @@ package main import ( - "bufio" "context" - "errors" "fmt" "io" "io/ioutil" "net/http" "os" - "path/filepath" + + "gopkg.in/yaml.v2" "github.com/urfave/cli" ) @@ -44,8 +43,8 @@ func serverCommand() cli.Command { Usage: "Prometheus address to reload after provisioning the rule file(s).", }, cli.StringFlag{ - Name: "rule-file", - Usage: "File to provision rules into.", + Name: "config-file", + Usage: "Prometheus configuration file", }, }, Action: serverAction, @@ -56,7 +55,7 @@ func serverAction(c *cli.Context) error { bindAddress := c.String("bind-address") http.Handle("/api/v1/rules", &ruleProvisioningHandler{ ruleProvisioner: &ruleProvisioner{ - ruleFile: c.String("rule-file"), + configFile: c.String("config-file"), }, prometheusReloader: &prometheusReloader{ prometheusReloadURL: c.String("prometheus-reload-url"), @@ -92,59 +91,35 @@ func (h *ruleProvisioningHandler) ServeHTTP(w http.ResponseWriter, r *http.Reque } type ruleProvisioner struct { - ruleFile string + configFile string } -// provision attempts to provision the rule files read from r, and if identical -// to existing, does not provision them. It returns whether Prometheus should -// be reloaded and if an error has occurred. +// PUT request +// /api/v1/rules/ +// name specify +// filename determined by server + +// provision attempts to provision the rule files read from r +// expects {rule-filename: "filename", data: "groups: ...."} +// makes new file and dumps data into rule-filename +// edits prometheus configuration to include that entry func (p *ruleProvisioner) provision(r io.Reader) (bool, error) { - newData, err := ioutil.ReadAll(r) - if err != nil { - return false, fmt.Errorf("unable to read new rules: %w", err) - } - tempfile, err := ioutil.TempFile(filepath.Dir(p.ruleFile), "temp-mixtool") + configReader, err := os.OpenFile(p.configFile, os.O_RDWR, 0644) if err != nil { - return false, fmt.Errorf("unable to create temp file: %w", err) + return false, fmt.Errorf("unable to open prometheus config file: %w", err) } - n, err := tempfile.Write(newData) + configBuf, err := ioutil.ReadAll(configReader) if err != nil { - return false, fmt.Errorf("error when writing new rules: %w", err) + return false, fmt.Errorf("unable to read prometheus config file: %w", err) } - - if n != len(newData) { - return false, fmt.Errorf("writing error, wrote %d bytes, expected %d", n, len(newData)) - } - - tempfile.Sync() - - ruleFileReader, err := os.OpenFile(p.ruleFile, os.O_RDWR, 0644) - if err != nil { - return false, fmt.Errorf("unable to read existing rules: %w", err) - } - - newFileReader, err := os.OpenFile(tempfile.Name(), os.O_RDWR, 0644) + m := make(map[string]interface{}) + err = yaml.Unmarshal(configBuf, &m) if err != nil { - return false, fmt.Errorf("unable to open new rules file: %w", err) + return false, fmt.Errorf("unable to unmarshal prometheus config file: %w", err) } - equal, err := readersEqual(newFileReader, ruleFileReader) - if err != nil { - return false, fmt.Errorf("error from readersEqual: %w", err) - } - - if equal { - fmt.Println("don't need to reload") - return false, nil - } - - fmt.Printf("need to remove oldrulefile @ %s, replace with newfile name %s\n", p.ruleFile, tempfile.Name()) - - if err = os.Rename(tempfile.Name(), p.ruleFile); err != nil { - return false, fmt.Errorf("cannot rename rules file: %w", err) - } return true, nil } @@ -152,27 +127,6 @@ type prometheusReloader struct { prometheusReloadURL string } -func readersEqual(r1, r2 io.Reader) (bool, error) { - buf1 := bufio.NewReader(r1) - buf2 := bufio.NewReader(r2) - for { - b1, err1 := buf1.ReadByte() - b2, err2 := buf2.ReadByte() - if err1 != nil && !errors.Is(err1, io.EOF) { - return false, err1 - } - if err2 != nil && !errors.Is(err2, io.EOF) { - return false, err2 - } - if errors.Is(err1, io.EOF) || errors.Is(err2, io.EOF) { - return err1 == err2, nil - } - if b1 != b2 { - return false, nil - } - } -} - func (r *prometheusReloader) triggerReload(ctx context.Context) error { req, err := http.NewRequest("POST", r.prometheusReloadURL, nil) if err != nil { diff --git a/go.mod b/go.mod index a627017..0808caf 100644 --- a/go.mod +++ b/go.mod @@ -28,5 +28,6 @@ require ( gopkg.in/ini.v1 v1.38.2 // indirect gopkg.in/macaron.v1 v1.3.1 // indirect gopkg.in/redis.v2 v2.3.2 // indirect + gopkg.in/yaml.v2 v2.3.0 sigs.k8s.io/yaml v1.2.0 ) From c5de2a7a08c2cb4645039af476968b4f05cd6cfe Mon Sep 17 00:00:00 2001 From: Allen Ma Date: Sun, 13 Dec 2020 17:30:47 -0500 Subject: [PATCH 18/20] add multiple mixin functionality by creating multiple temp files on server side --- cmd/mixtool/generate.go | 7 ++- cmd/mixtool/install.go | 88 ++++++++---------------------- cmd/mixtool/server.go | 116 ++++++++++++++++++++++++++++++++++++++-- 3 files changed, 137 insertions(+), 74 deletions(-) diff --git a/cmd/mixtool/generate.go b/cmd/mixtool/generate.go index 8066640..94de0f4 100644 --- a/cmd/mixtool/generate.go +++ b/cmd/mixtool/generate.go @@ -190,13 +190,12 @@ func generateDashboards(filename string, opts mixer.GenerateOptions) error { return nil } -func generateRulesAlerts(filename string, options mixer.GenerateOptions) error { +func generateRulesAlerts(filename string, options mixer.GenerateOptions) ([]byte, error) { out, err := mixer.GenerateRulesAlerts(filename, options) if err != nil { - return err + return nil, err } - fmt.Println("inside generate rules, alerts") - return ioutil.WriteFile("rules-alerts.yaml", out, 0644) + return out, nil } func generateAll(filename string, opts mixer.GenerateOptions) error { diff --git a/cmd/mixtool/install.go b/cmd/mixtool/install.go index 8226637..8a9deeb 100644 --- a/cmd/mixtool/install.go +++ b/cmd/mixtool/install.go @@ -15,7 +15,9 @@ package main import ( + "bytes" "fmt" + "io/ioutil" "net/http" "net/url" "os" @@ -84,14 +86,13 @@ func getMixins() ([]mixin, error) { return mixins, nil } -func generateMixin(directory string, jsonnetHome string, mixinURL string, options mixer.GenerateOptions) error { - fmt.Println("running generate all") +func generateMixin(directory string, jsonnetHome string, mixinURL string, options mixer.GenerateOptions) ([]byte, error) { mixinBaseDirectory := filepath.Join(directory) err := os.Chdir(mixinBaseDirectory) if err != nil { - return fmt.Errorf("Cannot cd into directory %s", err) + return nil, fmt.Errorf("Cannot cd into directory %s", err) } files, err := filepath.Glob("*") @@ -102,15 +103,15 @@ func generateMixin(directory string, jsonnetHome string, mixinURL string, option u, err := url.Parse(mixinURL) if err != nil { - return fmt.Errorf("url parse %v", err) + return nil, fmt.Errorf("url parse %v", err) } // absolute directory is the same as the download url stripped of the scheme absDirectory := path.Join(u.Host, u.Path) - - // strip leading slashes and colons + // trim http or www etc, trim repositories with trailing .git + // TODO: what if it's under some different kind of VCS? absDirectory = strings.TrimLeft(absDirectory, "/:") - + absDirectory = strings.TrimSuffix(absDirectory, ".git") fmt.Println("absDirectory is", absDirectory) importFile := filepath.Join(absDirectory, "mixin.libsonnet") @@ -118,84 +119,39 @@ func generateMixin(directory string, jsonnetHome string, mixinURL string, option // generate rules, dashboards, alerts err = generateAll(importFile, options) if err != nil { - return fmt.Errorf("generateAllMixins %v", err) + return nil, fmt.Errorf("generateAll: %w", err) } - err = generateRulesAlerts(importFile, options) + out, err := generateRulesAlerts(importFile, options) if err != nil { - return fmt.Errorf("generateRulesAlerts %v", err) + return nil, fmt.Errorf("generateRulesAlerts %w", err) } - return nil + return out, nil } -func putMixin(directory string, mixinURL string, bindAddress string, options mixer.GenerateOptions) error { - - // // alerts.yaml - // alertsFilename := options.AlertsFilename - // alertsReader, err := os.Open(alertsFilename) - // if err != nil { - // return err - // } - - // // rules.yaml - // rulesFilename := options.RulesFilename - // rulesReader, err := os.Open(rulesFilename) - // if err != nil { - // return err - // } - - // merge rules and alerts together - +func putMixin(content []byte, bindAddress string) error { u, err := url.Parse(bindAddress) if err != nil { return err } u.Path = path.Join(u.Path, "/api/v1/rules") - // // request for rules - // req, err := http.NewRequest("PUT", u.String(), rulesReader) - // resp, err := http.DefaultClient.Do(req) - // if err != nil { - // return fmt.Errorf("response from server %v", err) - // } - - // if resp.StatusCode == 200 { - // fmt.Println("PUT rules OK") - // } else { - // fmt.Printf("resp is %v\n", resp.Body) - // } - - // same request but for alerts - // req, err := http.NewRequest("PUT", u.String(), alertsReader) - // resp, err := http.DefaultClient.Do(req) - // if err != nil { - // return fmt.Errorf("response from server %v", err) - // } - - // if resp.StatusCode == 200 { - // fmt.Println("PUT alerts OK") - // } else { - // return fmt.Errorf("response code: %d resp is %v", resp.StatusCode, resp.Body) - // } - - // temporary: generate rules and alerts in a single file - alertsReader, err := os.Open("rules-alerts.yaml") - if err != nil { - return err - } - - req, err := http.NewRequest("PUT", u.String(), alertsReader) + r := bytes.NewReader(content) + req, err := http.NewRequest("PUT", u.String(), r) resp, err := http.DefaultClient.Do(req) if err != nil { return fmt.Errorf("response from server %v", err) } - if resp.StatusCode == 200 { fmt.Println("PUT alerts OK") } else { - return fmt.Errorf("response code: %d resp is %v", resp.StatusCode, resp.Body) + responseData, err := ioutil.ReadAll(resp.Body) + if err != nil { + return fmt.Errorf("failed to response body in putMixin, %w", err) + } + return fmt.Errorf("non 200 response code: %d, info: %s", resp.StatusCode, string(responseData)) } return nil } @@ -268,7 +224,7 @@ func installAction(c *cli.Context) error { YAML: true, } - err = generateMixin(directory, jsonnetHome, mixinURL, generateCfg) + rulesAlerts, err := generateMixin(directory, jsonnetHome, mixinURL, generateCfg) if err != nil { return err } @@ -278,7 +234,7 @@ func installAction(c *cli.Context) error { if c.Bool("put") { bindAddress := c.String("bind-address") // run put requests onto the server - err = putMixin(directory, mixinURL, bindAddress, generateCfg) + err = putMixin(rulesAlerts, bindAddress) if err != nil { return err } diff --git a/cmd/mixtool/server.go b/cmd/mixtool/server.go index 16fa4c2..00adcbf 100644 --- a/cmd/mixtool/server.go +++ b/cmd/mixtool/server.go @@ -15,12 +15,15 @@ package main import ( + "bufio" "context" + "errors" "fmt" "io" "io/ioutil" "net/http" "os" + "path/filepath" "gopkg.in/yaml.v2" @@ -76,6 +79,8 @@ func (h *ruleProvisioningHandler) ServeHTTP(w http.ResponseWriter, r *http.Reque return } + fmt.Println("serve http") + reloadNecessary, err := h.ruleProvisioner.provision(r.Body) if err != nil { http.Error(w, fmt.Sprintf("Internal Server Error: %v", err), http.StatusInternalServerError) @@ -104,22 +109,104 @@ type ruleProvisioner struct { // makes new file and dumps data into rule-filename // edits prometheus configuration to include that entry func (p *ruleProvisioner) provision(r io.Reader) (bool, error) { + fmt.Println("provision") + newRules, err := ioutil.ReadAll(r) + if err != nil { + return false, fmt.Errorf("unable to read new rules: %w", err) + } - configReader, err := os.OpenFile(p.configFile, os.O_RDWR, 0644) + // dir := filepath.Dir(p.configFile) + + f, err := ioutil.TempFile(filepath.Dir(p.configFile), "temp-mixinname") + // f, err := os.OpenFile(filepath.Join(dir, "test-mixin"), os.O_RDWR|os.O_CREATE, 0644) if err != nil { - return false, fmt.Errorf("unable to open prometheus config file: %w", err) + return false, fmt.Errorf("unable to create new mixin file: %w", err) } - configBuf, err := ioutil.ReadAll(configReader) + // write all the contents into file + n, err := f.Write(newRules) + if err != nil { + return false, fmt.Errorf("error when writing new rules: %w", err) + } + if n != len(newRules) { + return false, fmt.Errorf("writing error, wrote %d bytes, expected %d", n, len(newRules)) + } + + f.Sync() + f.Close() + + // add file's name to config file + + configBuf, err := ioutil.ReadFile(p.configFile) if err != nil { - return false, fmt.Errorf("unable to read prometheus config file: %w", err) + return false, fmt.Errorf("unable to open prometheus config file: %w", err) } + m := make(map[string]interface{}) err = yaml.Unmarshal(configBuf, &m) if err != nil { return false, fmt.Errorf("unable to unmarshal prometheus config file: %w", err) } + for k, v := range m { + fmt.Printf("k is %s\n", k) + // check explicitly for rule_files + if k == "rule_files" { + // add the new file's name under rule-files + // TODO: not entirely sure if this type assertion is safe + rulemap := v.([]interface{}) + rulemap = append(rulemap, f.Name()) + m[k] = rulemap + fmt.Printf("value of v %v\n", rulemap) + break + } + } + + // create a temporary config file + tempfile, err := ioutil.TempFile(filepath.Dir(p.configFile), "temp-config") + + // marshal back into yaml + newConfig, err := yaml.Marshal(m) + if err != nil { + return false, fmt.Errorf("failed to marhsal yaml: %w", err) + } + + // write contents to temp config file + n, err = tempfile.Write(newConfig) + if err != nil { + return false, fmt.Errorf("error when writing new rules: %w", err) + } + if n != len(newConfig) { + return false, fmt.Errorf("writing error, wrote %d bytes, expected %d", n, len(newConfig)) + } + + tempfile.Sync() + + // TODO: can we just use existing configreader? + configReader, err := os.OpenFile(p.configFile, os.O_RDONLY, 0644) + if err != nil { + return false, fmt.Errorf("unable to read existing config: %w", err) + } + + newConfigReader, err := os.OpenFile(tempfile.Name(), os.O_RDONLY, 0644) + if err != nil { + return false, fmt.Errorf("unable to open new config file: %w", err) + } + + // check the difference between config file and temp file + equal, err := readersEqual(configReader, newConfigReader) + if err != nil { + return false, fmt.Errorf("error from readersEqual: %w", err) + } + + if equal { + fmt.Println("don't need to do anything!") + return false, nil + } + + if err = os.Rename(tempfile.Name(), p.configFile); err != nil { + return false, fmt.Errorf("cannot rename config file: %w", err) + } return true, nil } @@ -127,6 +214,27 @@ type prometheusReloader struct { prometheusReloadURL string } +func readersEqual(r1, r2 io.Reader) (bool, error) { + buf1 := bufio.NewReader(r1) + buf2 := bufio.NewReader(r2) + for { + b1, err1 := buf1.ReadByte() + b2, err2 := buf2.ReadByte() + if err1 != nil && !errors.Is(err1, io.EOF) { + return false, err1 + } + if err2 != nil && !errors.Is(err2, io.EOF) { + return false, err2 + } + if errors.Is(err1, io.EOF) || errors.Is(err2, io.EOF) { + return err1 == err2, nil + } + if b1 != b2 { + return false, nil + } + } +} + func (r *prometheusReloader) triggerReload(ctx context.Context) error { req, err := http.NewRequest("POST", r.prometheusReloadURL, nil) if err != nil { From c89078c6aa9d5638124cfd9c0ca054efc4d03658 Mon Sep 17 00:00:00 2001 From: Allen Ma Date: Sun, 13 Dec 2020 18:28:21 -0500 Subject: [PATCH 19/20] add support for more than one mixin via api/rules/v1/ --- cmd/mixtool/install.go | 14 +++++++++++--- cmd/mixtool/server.go | 18 ++++++++++-------- 2 files changed, 21 insertions(+), 11 deletions(-) diff --git a/cmd/mixtool/install.go b/cmd/mixtool/install.go index 8a9deeb..d79ad6e 100644 --- a/cmd/mixtool/install.go +++ b/cmd/mixtool/install.go @@ -131,12 +131,13 @@ func generateMixin(directory string, jsonnetHome string, mixinURL string, option } -func putMixin(content []byte, bindAddress string) error { +func putMixin(content []byte, bindAddress string, mixinName string) error { u, err := url.Parse(bindAddress) if err != nil { return err } - u.Path = path.Join(u.Path, "/api/v1/rules") + pathName := fmt.Sprintf("/api/v1/rules/%s", mixinName) + u.Path = path.Join(u.Path, pathName) r := bytes.NewReader(content) req, err := http.NewRequest("PUT", u.String(), r) @@ -234,7 +235,14 @@ func installAction(c *cli.Context) error { if c.Bool("put") { bindAddress := c.String("bind-address") // run put requests onto the server - err = putMixin(rulesAlerts, bindAddress) + + // TODO: deal with the case where mixinPath is not a URL + _, err := url.ParseRequestURI(mixinPath) + if err == nil { + fmt.Println("TODO: for reloading prometheus, not dealing with mixins by url yet, since name is unknown") + } + + err = putMixin(rulesAlerts, bindAddress, mixinPath) if err != nil { return err } diff --git a/cmd/mixtool/server.go b/cmd/mixtool/server.go index 00adcbf..bf43633 100644 --- a/cmd/mixtool/server.go +++ b/cmd/mixtool/server.go @@ -30,6 +30,8 @@ import ( "github.com/urfave/cli" ) +const apiPath = "/api/v1/rules/" + func serverCommand() cli.Command { return cli.Command{ Name: "server", @@ -56,7 +58,7 @@ func serverCommand() cli.Command { func serverAction(c *cli.Context) error { bindAddress := c.String("bind-address") - http.Handle("/api/v1/rules", &ruleProvisioningHandler{ + http.Handle(apiPath, &ruleProvisioningHandler{ ruleProvisioner: &ruleProvisioner{ configFile: c.String("config-file"), }, @@ -79,9 +81,11 @@ func (h *ruleProvisioningHandler) ServeHTTP(w http.ResponseWriter, r *http.Reque return } - fmt.Println("serve http") + // TODO: might not be the best place to put this + mixin := r.URL.Path[len(apiPath):] + fmt.Println("the mixin is ", mixin) - reloadNecessary, err := h.ruleProvisioner.provision(r.Body) + reloadNecessary, err := h.ruleProvisioner.provision(r.Body, mixin) if err != nil { http.Error(w, fmt.Sprintf("Internal Server Error: %v", err), http.StatusInternalServerError) return @@ -108,17 +112,15 @@ type ruleProvisioner struct { // expects {rule-filename: "filename", data: "groups: ...."} // makes new file and dumps data into rule-filename // edits prometheus configuration to include that entry -func (p *ruleProvisioner) provision(r io.Reader) (bool, error) { +func (p *ruleProvisioner) provision(r io.Reader, mixinName string) (bool, error) { fmt.Println("provision") newRules, err := ioutil.ReadAll(r) if err != nil { return false, fmt.Errorf("unable to read new rules: %w", err) } - // dir := filepath.Dir(p.configFile) - - f, err := ioutil.TempFile(filepath.Dir(p.configFile), "temp-mixinname") - // f, err := os.OpenFile(filepath.Join(dir, "test-mixin"), os.O_RDWR|os.O_CREATE, 0644) + dir := filepath.Dir(p.configFile) + f, err := os.OpenFile(filepath.Join(dir, mixinName), os.O_RDWR|os.O_CREATE, 0644) if err != nil { return false, fmt.Errorf("unable to create new mixin file: %w", err) } From 454c4eda7acb4c2174e7ba2ed688f6ade9673060 Mon Sep 17 00:00:00 2001 From: Allen Ma Date: Mon, 14 Dec 2020 16:20:21 -0500 Subject: [PATCH 20/20] add elementary support for multiple mixins --- cmd/mixtool/install.go | 8 +------- cmd/mixtool/server.go | 29 +++++++++++------------------ 2 files changed, 12 insertions(+), 25 deletions(-) diff --git a/cmd/mixtool/install.go b/cmd/mixtool/install.go index d79ad6e..a326284 100644 --- a/cmd/mixtool/install.go +++ b/cmd/mixtool/install.go @@ -95,9 +95,6 @@ func generateMixin(directory string, jsonnetHome string, mixinURL string, option return nil, fmt.Errorf("Cannot cd into directory %s", err) } - files, err := filepath.Glob("*") - fmt.Println("in generatemixin, directory is ", directory, files) - // generate alerts, rules, grafana dashboards // empty files if not present @@ -112,7 +109,6 @@ func generateMixin(directory string, jsonnetHome string, mixinURL string, option // TODO: what if it's under some different kind of VCS? absDirectory = strings.TrimLeft(absDirectory, "/:") absDirectory = strings.TrimSuffix(absDirectory, ".git") - fmt.Println("absDirectory is", absDirectory) importFile := filepath.Join(absDirectory, "mixin.libsonnet") @@ -145,9 +141,7 @@ func putMixin(content []byte, bindAddress string, mixinName string) error { if err != nil { return fmt.Errorf("response from server %v", err) } - if resp.StatusCode == 200 { - fmt.Println("PUT alerts OK") - } else { + if resp.StatusCode != 200 { responseData, err := ioutil.ReadAll(resp.Body) if err != nil { return fmt.Errorf("failed to response body in putMixin, %w", err) diff --git a/cmd/mixtool/server.go b/cmd/mixtool/server.go index bf43633..d63d4ae 100644 --- a/cmd/mixtool/server.go +++ b/cmd/mixtool/server.go @@ -83,7 +83,6 @@ func (h *ruleProvisioningHandler) ServeHTTP(w http.ResponseWriter, r *http.Reque // TODO: might not be the best place to put this mixin := r.URL.Path[len(apiPath):] - fmt.Println("the mixin is ", mixin) reloadNecessary, err := h.ruleProvisioner.provision(r.Body, mixin) if err != nil { @@ -105,22 +104,24 @@ type ruleProvisioner struct { // PUT request // /api/v1/rules/ -// name specify +// specify mixin name // filename determined by server - -// provision attempts to provision the rule files read from r -// expects {rule-filename: "filename", data: "groups: ...."} -// makes new file and dumps data into rule-filename -// edits prometheus configuration to include that entry func (p *ruleProvisioner) provision(r io.Reader, mixinName string) (bool, error) { - fmt.Println("provision") newRules, err := ioutil.ReadAll(r) if err != nil { return false, fmt.Errorf("unable to read new rules: %w", err) } + mixinName = mixinName + ".yaml" dir := filepath.Dir(p.configFile) - f, err := os.OpenFile(filepath.Join(dir, mixinName), os.O_RDWR|os.O_CREATE, 0644) + mixinFilename := filepath.Join(dir, mixinName) + + // if the filename under filepath.Join(dir, mixinName) already exists, do nothing + if _, err = os.Stat(mixinFilename); err == nil { + return true, nil + } + + f, err := os.OpenFile(mixinFilename, os.O_RDWR|os.O_CREATE, 0644) if err != nil { return false, fmt.Errorf("unable to create new mixin file: %w", err) } @@ -138,7 +139,6 @@ func (p *ruleProvisioner) provision(r io.Reader, mixinName string) (bool, error) f.Close() // add file's name to config file - configBuf, err := ioutil.ReadFile(p.configFile) if err != nil { return false, fmt.Errorf("unable to open prometheus config file: %w", err) @@ -151,15 +151,11 @@ func (p *ruleProvisioner) provision(r io.Reader, mixinName string) (bool, error) } for k, v := range m { - fmt.Printf("k is %s\n", k) - // check explicitly for rule_files if k == "rule_files" { - // add the new file's name under rule-files // TODO: not entirely sure if this type assertion is safe rulemap := v.([]interface{}) - rulemap = append(rulemap, f.Name()) + rulemap = append(rulemap, mixinName) m[k] = rulemap - fmt.Printf("value of v %v\n", rulemap) break } } @@ -184,7 +180,6 @@ func (p *ruleProvisioner) provision(r io.Reader, mixinName string) (bool, error) tempfile.Sync() - // TODO: can we just use existing configreader? configReader, err := os.OpenFile(p.configFile, os.O_RDONLY, 0644) if err != nil { return false, fmt.Errorf("unable to read existing config: %w", err) @@ -195,14 +190,12 @@ func (p *ruleProvisioner) provision(r io.Reader, mixinName string) (bool, error) return false, fmt.Errorf("unable to open new config file: %w", err) } - // check the difference between config file and temp file equal, err := readersEqual(configReader, newConfigReader) if err != nil { return false, fmt.Errorf("error from readersEqual: %w", err) } if equal { - fmt.Println("don't need to do anything!") return false, nil }