Skip to content

Commit

Permalink
Merge pull request #20 from leoleovich/client_and_retry
Browse files Browse the repository at this point in the history
Implement grafsy-client
Add retryKeepSecs configuration parameter
  • Loading branch information
Felixoid authored Aug 2, 2020
2 parents a5d3f96 + 9fb8b98 commit 9ed51c7
Show file tree
Hide file tree
Showing 5 changed files with 122 additions and 12 deletions.
12 changes: 10 additions & 2 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ SUM_FILES = build/sha256sum build/md5sum
GO_FLAGS =
GO_BUILD = go build $(GO_FLAGS) -ldflags "-X 'main.version=$(VERSION)'" -o $@ $<

export GO111MODULE=on

.PHONY: all clean docker test version

Expand All @@ -36,7 +37,7 @@ test:
go vet ./...
go test -v ./...

build: build/$(NAME)
build: build/$(NAME) build/$(NAME)-client

docker:
docker build -t $(ORG_NAME)/$(NAME):builder -f docker/builder/Dockerfile .
Expand All @@ -45,6 +46,9 @@ docker:
build/$(NAME): $(NAME)/main.go
$(GO_BUILD)

build/$(NAME)-client: $(NAME)-client/main.go
$(GO_BUILD)

#########################################################
# Prepare artifact directory and set outputs for upload #
#########################################################
Expand All @@ -70,16 +74,20 @@ artifact/%: build/% | artifact

# Prepare everything for packaging
.ONESHELL:
build/pkg: build/$(NAME)_linux_x64 $(NAME).toml
build/pkg: build/$(NAME)-client_linux_x64 build/$(NAME)_linux_x64 $(NAME).toml
cd build
mkdir -p pkg/etc/$(NAME)/example/
mkdir -p pkg/usr/bin
cp -l $(NAME)_linux_x64 pkg/usr/bin/$(NAME)
cp -l $(NAME)-client_linux_x64 pkg/usr/bin/$(NAME)-client
cp -l ../$(NAME).toml pkg/etc/$(NAME)/example/

build/$(NAME)_linux_x64: $(NAME)/main.go
GOOS=linux GOARCH=amd64 $(GO_BUILD)

build/$(NAME)-client_linux_x64: $(NAME)-client/main.go
GOOS=linux GOARCH=amd64 $(GO_BUILD)

packages: $(PKG_FILES) $(SUM_FILES)

# md5 and sha256 sum-files for packages
Expand Down
25 changes: 17 additions & 8 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -37,8 +37,9 @@ Most of the time you need to use default (recommended) configuration of grafsy,
- `supervisor` - supervisor manager which is used to run Grafsy. e.g. systemd or supervisord. Default is none
- `clientSendInterval` - the interval, after which client will send data to graphite. In seconds
- `metricsPerSecond` - maximum amount of metrics which can be processed per second
In case of problems with connection/amount of metrics, this configuration will save up to `MetricsPerSecond*ClientSendInterval*10` metrics in retryDir
In case of problems with connection/amount of metrics, this configuration will save up to `MetricsPerSecond*RetryKeepSecs` metrics in retryDir
Also these 2 params are exactly allocating memory
- `retryKeepSecs` - how many seconds should be kept in retry files, at least
- `allowedMetrics` - regexp of allowed metric. Every metric which is not passing check against regexp will be removed
- `log` - main log file, `-` is treated as STDOUT
- `hostname` - alias to use instead of os.Hostname() result
Expand Down Expand Up @@ -83,17 +84,25 @@ replaceWith = "servers.HOSTNAME.software.pdns"
```
This will ask Grafsy to replace all kinds of metric starting with **pdns** or aggregation prefixes **^(SUM|AVG|MIN|MAX).pdns** to **servers.HOSTNAME.software.pdns** where *HOSTNAME* will be replaced with os.Hostname() output

# Client

The `grafsy-client` binary is implemented for easy metrics sending from generators to a grafsy daemon. You only need to specify the config file, if a not-default one is used.
It sends either metrics from specified files or from STDIN.

```
Usage: ./build/grafsy-client [args] [file1 [fileN...]]
Or: metrics-generator | ./build/grafsy-client [args]
```

# Installation

- Install go https://golang.org/doc/install
- Make a proper structure of directories: `mkdir -p /opt/go/src /opt/go/bin /opt/go/pkg`
- Setup g GOPATH variable: `export GOPATH=/opt/go`
- Clone this project to src: `go get github.com/leoleovich/grafsy`
- Fetch dependencies: `cd /opt/go/github.com/leoleovich/grafsy && go get ./...`
- Compile project: `go install github.com/leoleovich/grafsy/grafsy`
- Copy config file: `mkdir /etc/grafsy && cp /opt/go/src/github.com/leoleovich/grafsy/grafsy.toml /etc/grafsy/`
- Clone this project: `git clohe https://github.com/leoleovich/grafsy.git && cd grafsy`
- Compile project: `make`
- Copy a config file: `mkdir /etc/grafsy && cp grafsy.toml /etc/grafsy/`
- Change your settings, e.g. `carbonAddrs`
- Run it `/opt/go/bin/grafsy`
- Run it `./build/grafsy`
- Send metrics via client `metrics-generator | ./build/grafsy-client`

## Docker

Expand Down
12 changes: 10 additions & 2 deletions config.go
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,9 @@ type Config struct {
// Data, which was not sent will be buffered in this directory.
RetryDir string

// Time in seconds to keep metrics in retry file, at least
RetryKeepSecs int

// Prefix for metric to sum.
// Do not forget to include it in allowedMetrics if you change it.
SumPrefix string
Expand Down Expand Up @@ -136,7 +139,7 @@ type LocalConfig struct {
func (conf *Config) LoadConfig(configFile string) error {

if _, err := toml.DecodeFile(configFile, conf); err != nil {
return errors.New("Failed to parse config file" + err.Error())
return errors.New("Failed to parse config file: " + err.Error())
}

if conf.ClientSendInterval < 1 || conf.AggrInterval < 1 || conf.AggrPerSecond < 1 ||
Expand All @@ -145,6 +148,11 @@ func (conf *Config) LoadConfig(configFile string) error {
"MetricsPerSecond, ConnectTimeout must be greater than 0")
}

if conf.RetryKeepSecs <= 0 {
// Backward compatibility with old behavior
conf.RetryKeepSecs = conf.ClientSendInterval * 10
}

if conf.MonitoringPath == "" {
// This will be replaced later by monitoring routine
conf.MonitoringPath = "HOSTNAME"
Expand Down Expand Up @@ -262,7 +270,7 @@ func (conf *Config) GenerateLocalConfig() (*LocalConfig, error) {
/*
Retry file will take only 10 full buffers
*/
fileMetricSize: conf.MetricsPerSecond * conf.ClientSendInterval * 10,
fileMetricSize: conf.MetricsPerSecond * conf.RetryKeepSecs,
lg: lg,
allowedMetrics: regexp.MustCompile(conf.AllowedMetrics),
aggrRegexp: regexp.MustCompile(fmt.Sprintf("^(%s|%s|%s|%s)..*", conf.AvgPrefix, conf.SumPrefix, conf.MinPrefix, conf.MaxPrefix)),
Expand Down
82 changes: 82 additions & 0 deletions grafsy-client/main.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,82 @@
package main

import (
"bufio"
"flag"
"fmt"
"log"
"net"
"os"
"time"

"github.com/leoleovich/grafsy"
)

var version = "dev"

func main() {
var configFile string
var connectionTimeout int
printVersion := false
flag.Usage = func() {
fmt.Fprintf(os.Stderr, "Usage: %s [args] [file1 [fileN...]]\n", os.Args[0])
fmt.Fprintf(os.Stderr, " Or: metrics-generator | %s [args]\n\n", os.Args[0])
fmt.Fprintln(os.Stderr, "Reads metrics from files or STDIN and writes to grafsy LocalBind address.")
fmt.Fprintln(os.Stderr, "If STDIN contains something, then files will be ignored")
fmt.Fprintf(os.Stderr, "\nArgs:\n")
flag.PrintDefaults()
}
flag.StringVar(&configFile, "c", "/etc/grafsy/grafsy.toml", "Path to config file.")
flag.BoolVar(&printVersion, "v", printVersion, "Print version and exit")
flag.IntVar(&connectionTimeout, "w", 50, "Timeout ")
flag.Parse()
fileNames := flag.Args()

if printVersion {
fmt.Printf("Version: %v\n", version)
os.Exit(0)
}

var conf grafsy.Config
err := conf.LoadConfig(configFile)
if err != nil {
log.Fatalln(err)
}

conn, err := net.DialTimeout("tcp", conf.LocalBind, time.Duration(connectionTimeout)*time.Second)
if err != nil {
log.Fatalf("Fail to establish connection: %v\n", err)
}
defer conn.Close()

stat, err := os.Stdin.Stat()
if err != nil {
log.Fatalf("Error in STDIN: %v\n", err)
}

// If STDIN is CharDevice, then it contains something.
// Send it to grafsy daemon
if (stat.Mode() & os.ModeCharDevice) == 0 {
stdin := bufio.NewReader(os.Stdin)
connWriter := bufio.NewWriter(conn)
stdin.WriteTo(connWriter)
return
}

if len(fileNames) == 0 {
flag.Usage()
}

for _, fileName := range fileNames {
file, err := os.Open(fileName)
if err != nil {
log.Printf("Failed to open file %s: %v\n", fileName, err)
continue
}
defer file.Close()

r := bufio.NewReader(file)
connWriter := bufio.NewWriter(conn)
r.WriteTo(connWriter)
}
}
3 changes: 3 additions & 0 deletions grafsy.toml
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,9 @@ useACL = false

retryDir = "/tmp/grafsy/retry"

# By default this one is equal to clientSendInterval*10
retryKeepSecs = 200

sumPrefix = "SUM."
avgPrefix = "AVG."
minPrefix = "MIN."
Expand Down

0 comments on commit 9ed51c7

Please sign in to comment.