Skip to content

Commit 5d1d832

Browse files
author
Vladimir Smirnov
authored
Merge pull request #743 from go-graphite/multifetch
Multifetch
2 parents 2484afd + 8f5ee35 commit 5d1d832

File tree

425 files changed

+55882
-11419
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

425 files changed

+55882
-11419
lines changed

‎cmd/carbonapi/config/config.go‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -107,6 +107,8 @@ type ConfigType struct {
107107
TruncateTimeMap map[time.Duration]time.Duration `mapstructure:"truncateTime"`
108108
TruncateTime []DurationTruncate `mapstructure:"-" json:"-"` // produce from TruncateTimeMap and sort in reverse order
109109

110+
CombineMultipleTargetsInOne bool `mapstructure:"combineMultipleTargetsInOne"`
111+
110112
ResponseCache cache.BytesCache `mapstructure:"-" json:"-"`
111113
BackendCache cache.BytesCache `mapstructure:"-" json:"-"`
112114

‎cmd/carbonapi/config/init.go‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,10 @@ import (
1616
zipperConfig "github.com/go-graphite/carbonapi/zipper/config"
1717

1818
"github.com/ansel1/merry"
19+
"github.com/lomik/zapwriter"
20+
"github.com/spf13/viper"
21+
"go.uber.org/zap"
22+
1923
"github.com/go-graphite/carbonapi/cache"
2024
"github.com/go-graphite/carbonapi/expr/functions"
2125
"github.com/go-graphite/carbonapi/expr/functions/cairo/png"
@@ -25,9 +29,6 @@ import (
2529
"github.com/go-graphite/carbonapi/limiter"
2630
"github.com/go-graphite/carbonapi/pkg/parser"
2731
zipperTypes "github.com/go-graphite/carbonapi/zipper/types"
28-
"github.com/lomik/zapwriter"
29-
"github.com/spf13/viper"
30-
"go.uber.org/zap"
3132
)
3233

3334
var graphTemplates map[string]png.PictureParams
@@ -394,13 +395,12 @@ func SetUpViper(logger *zap.Logger, configPath *string, viperPrefix string) {
394395
viper.SetDefault("upstreams.concurrencyLimit", 0)
395396
viper.SetDefault("upstreams.keepAliveInterval", "30s")
396397
viper.SetDefault("upstreams.maxIdleConnsPerHost", 100)
397-
viper.SetDefault("upstreams.carbonsearch.backend", "")
398-
viper.SetDefault("upstreams.carbonsearch.prefix", "virt.v1.*")
399398
viper.SetDefault("upstreams.scaleToCommonStep", true)
400399
viper.SetDefault("upstreams.graphite09compat", false)
401400
viper.SetDefault("expireDelaySec", 600)
402401
viper.SetDefault("useCachingDNSResolver", false)
403402
viper.SetDefault("logger", map[string]string{})
403+
viper.SetDefault("combineMultipleTargetsInOne", false)
404404
viper.AutomaticEnv()
405405

406406
err := viper.Unmarshal(&Config)

‎cmd/carbonapi/http/capability.go‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2,15 +2,16 @@ package http
22

33
import (
44
"encoding/json"
5-
"io/ioutil"
5+
"io"
66
"net/http"
77
"os"
88
"time"
99

10-
"github.com/go-graphite/carbonapi/zipper/httpHeaders"
1110
pb "github.com/go-graphite/protocol/carbonapi_v3_pb"
1211
"github.com/lomik/zapwriter"
1312
"go.uber.org/zap"
13+
14+
"github.com/go-graphite/carbonapi/zipper/httpHeaders"
1415
)
1516

1617
func capabilityHandler(wr http.ResponseWriter, req *http.Request) {
@@ -33,7 +34,7 @@ func capabilityHandler(wr http.ResponseWriter, req *http.Request) {
3334
}
3435

3536
if formatCode, ok := knownFormats[format]; ok {
36-
body, err := ioutil.ReadAll(req.Body)
37+
body, err := io.ReadAll(req.Body)
3738
if err != nil {
3839
accessLogger.Error("find failed",
3940
zap.Duration("runtime_seconds", time.Since(t0)),

‎cmd/carbonapi/http/find_handlers.go‎

Lines changed: 8 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -4,24 +4,25 @@ import (
44
"bytes"
55
"encoding/json"
66
"fmt"
7-
"io/ioutil"
7+
"io"
88
"net/http"
99
"sort"
1010
"strings"
1111
"time"
1212

1313
"github.com/ansel1/merry"
14-
"github.com/go-graphite/carbonapi/carbonapipb"
15-
"github.com/go-graphite/carbonapi/cmd/carbonapi/config"
16-
"github.com/go-graphite/carbonapi/date"
17-
"github.com/go-graphite/carbonapi/intervalset"
18-
utilctx "github.com/go-graphite/carbonapi/util/ctx"
1914
pbv2 "github.com/go-graphite/protocol/carbonapi_v2_pb"
2015
pbv3 "github.com/go-graphite/protocol/carbonapi_v3_pb"
2116
pickle "github.com/lomik/og-rek"
2217
"github.com/lomik/zapwriter"
2318
"github.com/maruel/natural"
2419
uuid "github.com/satori/go.uuid"
20+
21+
"github.com/go-graphite/carbonapi/carbonapipb"
22+
"github.com/go-graphite/carbonapi/cmd/carbonapi/config"
23+
"github.com/go-graphite/carbonapi/date"
24+
"github.com/go-graphite/carbonapi/intervalset"
25+
utilctx "github.com/go-graphite/carbonapi/util/ctx"
2526
)
2627

2728
// Find handler and it's helper functions
@@ -235,7 +236,7 @@ func findHandler(w http.ResponseWriter, r *http.Request) {
235236
var pv3Request pbv3.MultiGlobRequest
236237

237238
if format == protoV3Format {
238-
body, err := ioutil.ReadAll(r.Body)
239+
body, err := io.ReadAll(r.Body)
239240
if err != nil {
240241
accessLogDetails.HTTPCode = http.StatusBadRequest
241242
accessLogDetails.Reason = "failed to parse message body: " + err.Error()

‎cmd/carbonapi/http/render_handler.go‎

Lines changed: 41 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -5,13 +5,19 @@ import (
55
"encoding/gob"
66
"errors"
77
"fmt"
8-
"io/ioutil"
8+
"io"
99
"net/http"
1010
"strconv"
1111
"strings"
1212
"time"
1313

1414
"github.com/ansel1/merry"
15+
pb "github.com/go-graphite/protocol/carbonapi_v3_pb"
16+
"github.com/lomik/zapwriter"
17+
"github.com/msaf1980/go-stringutils"
18+
uuid "github.com/satori/go.uuid"
19+
"go.uber.org/zap"
20+
1521
"github.com/go-graphite/carbonapi/carbonapipb"
1622
"github.com/go-graphite/carbonapi/cmd/carbonapi/config"
1723
"github.com/go-graphite/carbonapi/date"
@@ -21,11 +27,6 @@ import (
2127
"github.com/go-graphite/carbonapi/pkg/parser"
2228
utilctx "github.com/go-graphite/carbonapi/util/ctx"
2329
"github.com/go-graphite/carbonapi/zipper/helper"
24-
pb "github.com/go-graphite/protocol/carbonapi_v3_pb"
25-
"github.com/lomik/zapwriter"
26-
stringutils "github.com/msaf1980/go-stringutils"
27-
uuid "github.com/satori/go.uuid"
28-
"go.uber.org/zap"
2930
)
3031

3132
func cleanupParams(r *http.Request) {
@@ -205,7 +206,7 @@ func renderHandler(w http.ResponseWriter, r *http.Request) {
205206
}
206207

207208
if format == protoV3Format {
208-
body, err := ioutil.ReadAll(r.Body)
209+
body, err := io.ReadAll(r.Body)
209210
if err != nil {
210211
accessLogDetails.HTTPCode = http.StatusBadRequest
211212
accessLogDetails.Reason = "failed to parse message body: " + err.Error()
@@ -291,23 +292,46 @@ func renderHandler(w http.ResponseWriter, r *http.Request) {
291292
results = make([]*types.MetricData, 0)
292293
values := make(map[parser.MetricRequest][]*types.MetricData)
293294

294-
for _, target := range targets {
295-
exp, e, err := parser.ParseExpr(target)
296-
if err != nil || e != "" {
297-
msg := buildParseErrorString(target, e, err)
298-
setError(w, accessLogDetails, msg, http.StatusBadRequest, uid.String())
299-
logAsError = true
300-
return
295+
if config.Config.CombineMultipleTargetsInOne && len(targets) > 0 {
296+
exprs := make([]parser.Expr, 0, len(targets))
297+
for _, target := range targets {
298+
exp, e, err := parser.ParseExpr(target)
299+
if err != nil || e != "" {
300+
msg := buildParseErrorString(target, e, err)
301+
setError(w, accessLogDetails, msg, http.StatusBadRequest, uid.String())
302+
logAsError = true
303+
return
304+
}
305+
exprs = append(exprs, exp)
301306
}
302307

303308
ApiMetrics.RenderRequests.Add(1)
304309

305-
result, err := expr.FetchAndEvalExp(ctx, exp, from32, until32, values)
306-
if err != nil {
307-
errors[target] = merry.Wrap(err)
310+
result, errs := expr.FetchAndEvalExprs(ctx, exprs, from32, until32, values)
311+
if errs != nil {
312+
errors = errs
308313
}
309314

310315
results = append(results, result...)
316+
} else {
317+
for _, target := range targets {
318+
exp, e, err := parser.ParseExpr(target)
319+
if err != nil || e != "" {
320+
msg := buildParseErrorString(target, e, err)
321+
setError(w, accessLogDetails, msg, http.StatusBadRequest, uid.String())
322+
logAsError = true
323+
return
324+
}
325+
326+
ApiMetrics.RenderRequests.Add(1)
327+
328+
result, err := expr.FetchAndEvalExp(ctx, exp, from32, until32, values)
329+
if err != nil {
330+
errors[target] = merry.Wrap(err)
331+
}
332+
333+
results = append(results, result...)
334+
}
311335
}
312336

313337
for mFetch := range values {

‎cmd/carbonapi/main.go‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,11 +11,12 @@ import (
1111
"os"
1212
"sync"
1313

14-
"github.com/go-graphite/carbonapi/pkg/tlsconfig"
1514
"github.com/gorilla/handlers"
1615
"github.com/lomik/zapwriter"
1716
"go.uber.org/zap"
1817

18+
"github.com/go-graphite/carbonapi/pkg/tlsconfig"
19+
1920
"github.com/go-graphite/carbonapi/cmd/carbonapi/config"
2021
"github.com/go-graphite/carbonapi/cmd/carbonapi/helper"
2122
carbonapiHttp "github.com/go-graphite/carbonapi/cmd/carbonapi/http"

0 commit comments

Comments
 (0)