diff --git a/Makefile b/Makefile index a43395a..2bd5487 100644 --- a/Makefile +++ b/Makefile @@ -17,6 +17,7 @@ SUM_FILES = build/sha256sum build/md5sum GO_FLAGS = GO_BUILD = go build $(GO_FLAGS) -ldflags "-X 'main.version=$(VERSION)'" -o $@ $< +export GO111MODULE=on .PHONY: all clean docker test version @@ -36,7 +37,7 @@ test: go vet ./... go test -v ./... -build: build/$(NAME) +build: build/$(NAME) build/$(NAME)-client docker: docker build -t $(ORG_NAME)/$(NAME):builder -f docker/builder/Dockerfile . @@ -45,6 +46,9 @@ docker: build/$(NAME): $(NAME)/main.go $(GO_BUILD) +build/$(NAME)-client: $(NAME)-client/main.go + $(GO_BUILD) + ######################################################### # Prepare artifact directory and set outputs for upload # ######################################################### @@ -70,16 +74,20 @@ artifact/%: build/% | artifact # Prepare everything for packaging .ONESHELL: -build/pkg: build/$(NAME)_linux_x64 $(NAME).toml +build/pkg: build/$(NAME)-client_linux_x64 build/$(NAME)_linux_x64 $(NAME).toml cd build mkdir -p pkg/etc/$(NAME)/example/ mkdir -p pkg/usr/bin cp -l $(NAME)_linux_x64 pkg/usr/bin/$(NAME) + cp -l $(NAME)-client_linux_x64 pkg/usr/bin/$(NAME)-client cp -l ../$(NAME).toml pkg/etc/$(NAME)/example/ build/$(NAME)_linux_x64: $(NAME)/main.go GOOS=linux GOARCH=amd64 $(GO_BUILD) +build/$(NAME)-client_linux_x64: $(NAME)-client/main.go + GOOS=linux GOARCH=amd64 $(GO_BUILD) + packages: $(PKG_FILES) $(SUM_FILES) # md5 and sha256 sum-files for packages diff --git a/README.md b/README.md index 977b196..eddac33 100644 --- a/README.md +++ b/README.md @@ -37,8 +37,9 @@ Most of the time you need to use default (recommended) configuration of grafsy, - `supervisor` - supervisor manager which is used to run Grafsy. e.g. systemd or supervisord. Default is none - `clientSendInterval` - the interval, after which client will send data to graphite. In seconds - `metricsPerSecond` - maximum amount of metrics which can be processed per second - In case of problems with connection/amount of metrics, this configuration will save up to `MetricsPerSecond*ClientSendInterval*10` metrics in retryDir + In case of problems with connection/amount of metrics, this configuration will save up to `MetricsPerSecond*RetryKeepSecs` metrics in retryDir Also these 2 params are exactly allocating memory +- `retryKeepSecs` - how many seconds should be kept in retry files, at least - `allowedMetrics` - regexp of allowed metric. Every metric which is not passing check against regexp will be removed - `log` - main log file, `-` is treated as STDOUT - `hostname` - alias to use instead of os.Hostname() result @@ -83,17 +84,25 @@ replaceWith = "servers.HOSTNAME.software.pdns" ``` This will ask Grafsy to replace all kinds of metric starting with **pdns** or aggregation prefixes **^(SUM|AVG|MIN|MAX).pdns** to **servers.HOSTNAME.software.pdns** where *HOSTNAME* will be replaced with os.Hostname() output +# Client + +The `grafsy-client` binary is implemented for easy metrics sending from generators to a grafsy daemon. You only need to specify the config file, if a not-default one is used. +It sends either metrics from specified files or from STDIN. + +``` +Usage: ./build/grafsy-client [args] [file1 [fileN...]] + Or: metrics-generator | ./build/grafsy-client [args] +``` + # Installation - Install go https://golang.org/doc/install -- Make a proper structure of directories: `mkdir -p /opt/go/src /opt/go/bin /opt/go/pkg` -- Setup g GOPATH variable: `export GOPATH=/opt/go` -- Clone this project to src: `go get github.com/leoleovich/grafsy` -- Fetch dependencies: `cd /opt/go/github.com/leoleovich/grafsy && go get ./...` -- Compile project: `go install github.com/leoleovich/grafsy/grafsy` -- Copy config file: `mkdir /etc/grafsy && cp /opt/go/src/github.com/leoleovich/grafsy/grafsy.toml /etc/grafsy/` +- Clone this project: `git clohe https://github.com/leoleovich/grafsy.git && cd grafsy` +- Compile project: `make` +- Copy a config file: `mkdir /etc/grafsy && cp grafsy.toml /etc/grafsy/` - Change your settings, e.g. `carbonAddrs` -- Run it `/opt/go/bin/grafsy` +- Run it `./build/grafsy` +- Send metrics via client `metrics-generator | ./build/grafsy-client` ## Docker diff --git a/config.go b/config.go index 9e527d2..dad164a 100644 --- a/config.go +++ b/config.go @@ -51,6 +51,9 @@ type Config struct { // Data, which was not sent will be buffered in this directory. RetryDir string + // Time in seconds to keep metrics in retry file, at least + RetryKeepSecs int + // Prefix for metric to sum. // Do not forget to include it in allowedMetrics if you change it. SumPrefix string @@ -136,7 +139,7 @@ type LocalConfig struct { func (conf *Config) LoadConfig(configFile string) error { if _, err := toml.DecodeFile(configFile, conf); err != nil { - return errors.New("Failed to parse config file" + err.Error()) + return errors.New("Failed to parse config file: " + err.Error()) } if conf.ClientSendInterval < 1 || conf.AggrInterval < 1 || conf.AggrPerSecond < 1 || @@ -145,6 +148,11 @@ func (conf *Config) LoadConfig(configFile string) error { "MetricsPerSecond, ConnectTimeout must be greater than 0") } + if conf.RetryKeepSecs <= 0 { + // Backward compatibility with old behavior + conf.RetryKeepSecs = conf.ClientSendInterval * 10 + } + if conf.MonitoringPath == "" { // This will be replaced later by monitoring routine conf.MonitoringPath = "HOSTNAME" @@ -262,7 +270,7 @@ func (conf *Config) GenerateLocalConfig() (*LocalConfig, error) { /* Retry file will take only 10 full buffers */ - fileMetricSize: conf.MetricsPerSecond * conf.ClientSendInterval * 10, + fileMetricSize: conf.MetricsPerSecond * conf.RetryKeepSecs, lg: lg, allowedMetrics: regexp.MustCompile(conf.AllowedMetrics), aggrRegexp: regexp.MustCompile(fmt.Sprintf("^(%s|%s|%s|%s)..*", conf.AvgPrefix, conf.SumPrefix, conf.MinPrefix, conf.MaxPrefix)), diff --git a/grafsy-client/main.go b/grafsy-client/main.go new file mode 100644 index 0000000..4ef12e1 --- /dev/null +++ b/grafsy-client/main.go @@ -0,0 +1,82 @@ +package main + +import ( + "bufio" + "flag" + "fmt" + "log" + "net" + "os" + "time" + + "github.com/leoleovich/grafsy" +) + +var version = "dev" + +func main() { + var configFile string + var connectionTimeout int + printVersion := false + flag.Usage = func() { + fmt.Fprintf(os.Stderr, "Usage: %s [args] [file1 [fileN...]]\n", os.Args[0]) + fmt.Fprintf(os.Stderr, " Or: metrics-generator | %s [args]\n\n", os.Args[0]) + fmt.Fprintln(os.Stderr, "Reads metrics from files or STDIN and writes to grafsy LocalBind address.") + fmt.Fprintln(os.Stderr, "If STDIN contains something, then files will be ignored") + fmt.Fprintf(os.Stderr, "\nArgs:\n") + flag.PrintDefaults() + } + flag.StringVar(&configFile, "c", "/etc/grafsy/grafsy.toml", "Path to config file.") + flag.BoolVar(&printVersion, "v", printVersion, "Print version and exit") + flag.IntVar(&connectionTimeout, "w", 50, "Timeout ") + flag.Parse() + fileNames := flag.Args() + + if printVersion { + fmt.Printf("Version: %v\n", version) + os.Exit(0) + } + + var conf grafsy.Config + err := conf.LoadConfig(configFile) + if err != nil { + log.Fatalln(err) + } + + conn, err := net.DialTimeout("tcp", conf.LocalBind, time.Duration(connectionTimeout)*time.Second) + if err != nil { + log.Fatalf("Fail to establish connection: %v\n", err) + } + defer conn.Close() + + stat, err := os.Stdin.Stat() + if err != nil { + log.Fatalf("Error in STDIN: %v\n", err) + } + + // If STDIN is CharDevice, then it contains something. + // Send it to grafsy daemon + if (stat.Mode() & os.ModeCharDevice) == 0 { + stdin := bufio.NewReader(os.Stdin) + connWriter := bufio.NewWriter(conn) + stdin.WriteTo(connWriter) + return + } + + if len(fileNames) == 0 { + flag.Usage() + } + + for _, fileName := range fileNames { + file, err := os.Open(fileName) + if err != nil { + log.Printf("Failed to open file %s: %v\n", fileName, err) + continue + } + defer file.Close() + + r := bufio.NewReader(file) + connWriter := bufio.NewWriter(conn) + r.WriteTo(connWriter) + } +} diff --git a/grafsy.toml b/grafsy.toml index d3855ae..ffe9165 100644 --- a/grafsy.toml +++ b/grafsy.toml @@ -19,6 +19,9 @@ useACL = false retryDir = "/tmp/grafsy/retry" +# By default this one is equal to clientSendInterval*10 +retryKeepSecs = 200 + sumPrefix = "SUM." avgPrefix = "AVG." minPrefix = "MIN."