Skip to content

Commit

Permalink
Support filters based on consul meta
Browse files Browse the repository at this point in the history
  • Loading branch information
Emilien Kenler committed Oct 7, 2017
1 parent 3c9a8e0 commit 0a424f8
Show file tree
Hide file tree
Showing 6 changed files with 162 additions and 31 deletions.
41 changes: 36 additions & 5 deletions collectd/checker.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,14 +2,17 @@ package collectd

import (
"bytes"
"fmt"
"regexp"
"strings"
"sync"
"sync/atomic"

"github.com/hashicorp/hil"
"github.com/Sirupsen/logrus"
"github.com/hashicorp/hil"

"github.com/MiLk/nmp/config"
"github.com/MiLk/nmp/consul"
"github.com/MiLk/nmp/shared"
)

Expand Down Expand Up @@ -61,16 +64,44 @@ func (checker *Checker) checkRecord(record CollectdRecord) ([]shared.CheckResult
}
value := buf.String()

// Load host specific thresholds
critical := rule.Check.Critical
warning := rule.Check.Warning
for pattern, threshold := range rule.Check.Thresholds {

matchName := "default"

// Load meta specific thresholds
for pattern, threshold := range rule.Check.MetaThresholds {
node, err := consul.GetNode(record.HostShort)
if err != nil {
checker.logger.Error(err)
continue
}
if node == nil {
continue
}

splitted := strings.SplitN(pattern, ":", 2)
v, ok := node.Meta[splitted[0]]
if !ok || v != splitted[1] {
continue
}

matchName = fmt.Sprintf("meta:%s", pattern)

critical = threshold.Critical
warning = threshold.Warning
break
}

// Load host specific thresholds
for pattern, threshold := range rule.Check.HostThresholds {
matched, err := regexp.MatchString(pattern, record.Host)
if err != nil {
checker.logger.Error(err)
continue
}
if matched {
matchName = fmt.Sprintf("host:%s", pattern)
critical = threshold.Critical
warning = threshold.Warning
break
Expand All @@ -84,7 +115,7 @@ func (checker *Checker) checkRecord(record CollectdRecord) ([]shared.CheckResult
continue
}
if result != nil {
checker.logger.Infof("CRITICAL: %s - %s | %+v | %s %s %s\n", record.Host, rule.Name, result, value, rule.Check.Comparator, critical)
checker.logger.Infof("CRITICAL: %s - %s - %s | %+v | %s %s %s\n", record.Host, rule.Name, matchName, result, value, rule.Check.Comparator, critical)
results = append(results, *result)
continue
}
Expand All @@ -96,7 +127,7 @@ func (checker *Checker) checkRecord(record CollectdRecord) ([]shared.CheckResult
continue
}
if result != nil {
checker.logger.Infof("WARNING: %s - %s | %+v | %s %s %s\n", record.Host, rule.Name, result, value, rule.Check.Comparator, warning)
checker.logger.Infof("WARNING: %s - %s - %s | %+v | %s %s %s\n", record.Host, rule.Name, matchName, result, value, rule.Check.Comparator, warning)
results = append(results, *result)
continue
}
Expand Down
6 changes: 6 additions & 0 deletions collectd/transformer.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package collectd

import (
"reflect"
"strings"
"sync"
"sync/atomic"

Expand Down Expand Up @@ -31,6 +32,11 @@ func (transformer *Transformer) TransformRecord(tag string, record shared.TinyRe
switch k {
case "host":
transformed.Host = v.(string)
if idx := strings.IndexByte(transformed.Host, '.'); idx > -1 {
transformed.HostShort = transformed.Host[:idx]
} else {
transformed.HostShort = transformed.Host
}
case "plugin":
transformed.Plugin = v.(string)
case "plugin_instance":
Expand Down
1 change: 1 addition & 0 deletions collectd/types.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ type CollectdRecord struct {
Timestamp uint64
Raw map[string]interface{}
Host string
HostShort string
Plugin string
PluginInstance string
Type string
Expand Down
59 changes: 34 additions & 25 deletions config/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -25,19 +25,39 @@ type CheckThreshold struct {
Critical hil.EvaluationResult `hcl:"-"`
}

type CheckThresholdMap map[string]CheckThreshold

func (m CheckThresholdMap) Parse(hilConfig *hil.EvalConfig) (err error) {
for k, threshold := range m {
threshold.Critical, err = ParseHIL(threshold.CriticalTpl, hilConfig)
if err != nil {
return
}

threshold.Warning, err = ParseHIL(threshold.WarningTpl, hilConfig)
if err != nil {
return
}

m[k] = threshold
}
return
}

type Check struct {
Plugin string `hcl:"plugin"`
PluginInstance string `hcl:"plugin_instance"`
Type string `hcl:"type"`
TypeInstance string `hcl:"type_instance"`
Comparator Comparator `hcl:"comparator"`
WarningTpl string `hcl:"warning"`
CriticalTpl string `hcl:"critical"`
Warning hil.EvaluationResult `hcl:"-"`
Critical hil.EvaluationResult `hcl:"-"`
ValueTpl string `hcl:"value"`
Value *template.Template `hcl:"-"`
Thresholds map[string]CheckThreshold `hcl:"host"`
Plugin string `hcl:"plugin"`
PluginInstance string `hcl:"plugin_instance"`
Type string `hcl:"type"`
TypeInstance string `hcl:"type_instance"`
Comparator Comparator `hcl:"comparator"`
WarningTpl string `hcl:"warning"`
CriticalTpl string `hcl:"critical"`
Warning hil.EvaluationResult `hcl:"-"`
Critical hil.EvaluationResult `hcl:"-"`
ValueTpl string `hcl:"value"`
Value *template.Template `hcl:"-"`
HostThresholds CheckThresholdMap `hcl:"host"`
MetaThresholds CheckThresholdMap `hcl:"meta"`
}

type Config struct {
Expand Down Expand Up @@ -100,19 +120,8 @@ func loadFileHcl(root string) (*Config, error) {
return nil, err
}

for hostname, threshold := range check.Thresholds {
threshold.Critical, err = ParseHIL(threshold.CriticalTpl, hilConfig)
if err != nil {
return nil, err
}

threshold.Warning, err = ParseHIL(threshold.WarningTpl, hilConfig)
if err != nil {
return nil, err
}

check.Thresholds[hostname] = threshold
}
check.HostThresholds.Parse(hilConfig)
check.MetaThresholds.Parse(hilConfig)

out.Checks[name] = check
}
Expand Down
84 changes: 84 additions & 0 deletions consul/consul.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,84 @@
package consul

import (
"fmt"
"time"

"github.com/hashicorp/consul/api"
"github.com/patrickmn/go-cache"
)

var nodeCache *cache.Cache

func init() {
nodeCache = cache.New(5*time.Minute, 10*time.Minute)
err := loadEverything()
if err != nil {
fmt.Println(err)
}
}

func loadEverything() error {
client, err := api.NewClient(&api.Config{})
if err != nil {
return err
}

dcs, err := client.Catalog().Datacenters()
if err != nil {
return err
}

for _, dc := range dcs {
nodes, _, err := client.Catalog().Nodes(&api.QueryOptions{
Datacenter: dc,
})
if err != nil {
return err
}
for _, n := range nodes {
if n != nil {
nodeCache.Set(n.Node, n, cache.DefaultExpiration)
}
}
}
return nil
}

func GetNode(host string) (*api.Node, error) {
n, found := nodeCache.Get(host)
if found {
return n.(*api.Node), nil
}

client, err := api.NewClient(&api.Config{})
if err != nil {
return nil, err
}

dcs, err := client.Catalog().Datacenters()
if err != nil {
return nil, err
}

var node *api.CatalogNode
for _, dc := range dcs {
var err error
if node, _, err = client.Catalog().Node(host, &api.QueryOptions{
Datacenter: dc,
}); err != nil {
return nil, err
}
if node != nil {
break
}
}

if node == nil {
return nil, nil
}

nodeCache.Set(host, node.Node, cache.DefaultExpiration)

return node.Node, nil
}
2 changes: 1 addition & 1 deletion nagios/writer.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,9 @@ package nagios
import (
"fmt"
"io/ioutil"
"os"
"sync"
"sync/atomic"
"os"

"github.com/Sirupsen/logrus"
)
Expand Down

0 comments on commit 0a424f8

Please sign in to comment.