diff --git a/cmd/tsbs_generate_queries/databases/clickhouse/common.go b/cmd/tsbs_generate_queries/databases/clickhouse/common.go index b4f29b9f0..0705a95a0 100644 --- a/cmd/tsbs_generate_queries/databases/clickhouse/common.go +++ b/cmd/tsbs_generate_queries/databases/clickhouse/common.go @@ -3,6 +3,7 @@ package clickhouse import ( "time" + "github.com/timescale/tsbs/cmd/tsbs_generate_queries/uses/dea" "github.com/timescale/tsbs/cmd/tsbs_generate_queries/uses/devops" "github.com/timescale/tsbs/cmd/tsbs_generate_queries/utils" "github.com/timescale/tsbs/pkg/query" @@ -10,7 +11,8 @@ import ( // BaseGenerator contains settings specific for ClickHouse. type BaseGenerator struct { - UseTags bool + UseTags bool + PropertyAccessMode string } // GenerateEmptyQuery returns an empty query.ClickHouse. @@ -42,3 +44,22 @@ func (g *BaseGenerator) NewDevops(start, end time.Time, scale int) (utils.QueryG return devops, nil } + +// NewDEA creates a new devops use case query generator. +func (g *BaseGenerator) NewDEA(start, end time.Time, scale int) (utils.QueryGenerator, error) { + core, err := dea.NewCore(start, end, scale) + + if err != nil { + return nil, err + } + + devops := &DEA{ + BaseGenerator: g, + Core: core, + } + + return devops, nil +} + +// ClickHouse understands and can compare time presented as strings of this format +const clickhouseTimeStringFormat = "2006-01-02 15:04:05" diff --git a/cmd/tsbs_generate_queries/databases/clickhouse/dea.go b/cmd/tsbs_generate_queries/databases/clickhouse/dea.go new file mode 100644 index 000000000..3ce68a046 --- /dev/null +++ b/cmd/tsbs_generate_queries/databases/clickhouse/dea.go @@ -0,0 +1,108 @@ +package clickhouse + +import ( + "fmt" + "strings" + + "github.com/timescale/tsbs/cmd/tsbs_generate_queries/uses/dea" + "github.com/timescale/tsbs/pkg/query" +) + +// Devops produces ClickHouse-specific queries for all the devops query types. +type DEA struct { + *dea.Core + *BaseGenerator +} + +const ( + Json string = "json" + Map string = "map" +) + +func Must[T any](v T, err error) T { + if err != nil { + panic(err) + } + return v +} + +func (d *DEA) getJSONProperty(key string) string { + return fmt.Sprintf("simpleJSONExtractRaw(propertiesJson, '%s')", key) +} + +func (d *DEA) getMapProperty(key string) string { + return fmt.Sprintf("propertiesMap['%s']", key) +} + +func (d *DEA) getProperty(key string) string { + switch d.PropertyAccessMode { + case Json: + return d.getJSONProperty(key) + case Map: + return d.getMapProperty(key) + default: + panic(fmt.Sprintf("unknown access mode %s", d.PropertyAccessMode)) + } +} + +func (d *DEA) getPropertyAlias(property string) string { + return fmt.Sprintf("a_%s", property) +} + +func (d *DEA) getAliasedProperties(keys []string) []string { + aliasedProps := make([]string, len(keys)) + + for i := range keys { + aliasedProps[i] = fmt.Sprintf("%s as %s", d.getProperty(keys[i]), d.getPropertyAlias(keys[i])) + } + + return aliasedProps +} + +func (d *DEA) NestedWhere(qi query.Query, nProperties int) { + randomProperties, err := d.GetRandomProperties(4, dea.AvailableStrProperties) + panicIfErr(err) + + selectClauses := strings.Join(d.getAliasedProperties(randomProperties), ", ") + + whereClauses := fmt.Sprintf("%s = '%s' AND %s != '' AND (%s LIKE '%%%s%%' OR %s LIKE '%%%s')", + d.getPropertyAlias(randomProperties[0]), + Must(d.GetRandomPropertyValue(randomProperties[0])), + d.getPropertyAlias(randomProperties[1]), + d.getPropertyAlias(randomProperties[2]), + Must(d.GetRandomPropertyValue(randomProperties[2]))[2:6], + d.getPropertyAlias(randomProperties[3]), + Must(d.GetRandomPropertyValue(randomProperties[3]))[2:]) + + // GetRandomSubsetPerm - use this + + sql := fmt.Sprintf(` + SELECT toStartOfHour(created_at) AS hour, %s FROM events WHERE %s + `, selectClauses, whereClauses) + + humanLabel := "ClickHouse nested and dynamic" + humanDesc := fmt.Sprintf("%s: nested where query with dynamic data access", humanLabel) + d.fillInQuery(qi, humanLabel, humanDesc, dea.TableName, sql) +} + +func (d *DEA) EventHistogram(qi query.Query) { + sql := fmt.Sprintf(` + SELECT + toDate(timestamp, 'Europe/Prague') as day, + uniq(user_id) as visitors, + count() as cnt + FROM events + WHERE + name LIKE 'package_changed_to%' + AND tenant_id = '57e8dcb8673eec55468b4567' + AND timestamp >= now() - INTERVAL 30 DAY + GROUP BY + day + ORDER BY + day WITH FILL + FROM toDate(now() - INTERVAL 30 DAY, 'Europe/Prague') + TO toDate(now(), 'Europe/Prague') + `) + + d.fillInQuery(qi, "ClickHouse Event histogram", "Clickhouse Event histogram", dea.TableName, sql) +} diff --git a/cmd/tsbs_generate_queries/databases/clickhouse/devops.go b/cmd/tsbs_generate_queries/databases/clickhouse/devops.go index a10144bde..43678749f 100644 --- a/cmd/tsbs_generate_queries/databases/clickhouse/devops.go +++ b/cmd/tsbs_generate_queries/databases/clickhouse/devops.go @@ -67,18 +67,17 @@ func (d *Devops) getSelectClausesAggMetrics(aggregateFunction string, metrics [] return selectAggregateClauses } -// ClickHouse understands and can compare time presented as strings of this format -const clickhouseTimeStringFormat = "2006-01-02 15:04:05" - // MaxAllCPU selects the MAX of all metrics under 'cpu' per hour for nhosts hosts, // e.g. in pseudo-SQL: // // SELECT MAX(metric1), ..., MAX(metricN) // FROM cpu // WHERE -// (hostname = '$HOSTNAME_1' OR ... OR hostname = '$HOSTNAME_N') -// AND time >= '$HOUR_START' -// AND time < '$HOUR_END' +// +// (hostname = '$HOSTNAME_1' OR ... OR hostname = '$HOSTNAME_N') +// AND time >= '$HOUR_START' +// AND time < '$HOUR_END' +// // GROUP BY hour // ORDER BY hour // @@ -290,9 +289,11 @@ func (d *Devops) LastPointPerHost(qi query.Query) { // SELECT minute, max(metric1), ..., max(metricN) // FROM cpu // WHERE -// (hostname = '$HOSTNAME_1' OR ... OR hostname = '$HOSTNAME_N') -// AND time >= '$HOUR_START' -// AND time < '$HOUR_END' +// +// (hostname = '$HOSTNAME_1' OR ... OR hostname = '$HOSTNAME_N') +// AND time >= '$HOUR_START' +// AND time < '$HOUR_END' +// // GROUP BY minute // ORDER BY minute ASC // diff --git a/cmd/tsbs_generate_queries/main.go b/cmd/tsbs_generate_queries/main.go index b5970f42c..ebb9c8593 100644 --- a/cmd/tsbs_generate_queries/main.go +++ b/cmd/tsbs_generate_queries/main.go @@ -4,12 +4,14 @@ package main import ( "fmt" - "github.com/timescale/tsbs/pkg/query/config" "os" "time" + "github.com/timescale/tsbs/pkg/query/config" + "github.com/blagojts/viper" "github.com/spf13/pflag" + "github.com/timescale/tsbs/cmd/tsbs_generate_queries/uses/dea" "github.com/timescale/tsbs/cmd/tsbs_generate_queries/uses/devops" "github.com/timescale/tsbs/cmd/tsbs_generate_queries/uses/iot" "github.com/timescale/tsbs/cmd/tsbs_generate_queries/utils" @@ -18,6 +20,9 @@ import ( ) var useCaseMatrix = map[string]map[string]utils.QueryFillerMaker{ + "dea": { + dea.LabelNestedWhere: dea.NewNestedWhere(1), + }, "devops": { devops.LabelSingleGroupby + "-1-1-1": devops.NewSingleGroupby(1, 1, 1), devops.LabelSingleGroupby + "-1-1-12": devops.NewSingleGroupby(1, 1, 12), diff --git a/cmd/tsbs_generate_queries/uses/dea/common.go b/cmd/tsbs_generate_queries/uses/dea/common.go new file mode 100644 index 000000000..709e71cf6 --- /dev/null +++ b/cmd/tsbs_generate_queries/uses/dea/common.go @@ -0,0 +1,82 @@ +package dea + +import ( + "fmt" + "math/rand" + "time" + + "github.com/timescale/tsbs/cmd/tsbs_generate_queries/uses/common" + "github.com/timescale/tsbs/pkg/query" +) + +const ( + TableName = "events" + + LabelNestedWhere = "nested-where" +) + +const ( + UrlProperty = "url" + EmailProperty = "email" + NameProperty = "name" + PhoneProperty = "phone" + FavoriteNumberProperty = "age" +) + +const ( + NavigationEvent = "navigation" +) + +var ( + AvailableStrProperties = []string{UrlProperty, EmailProperty, NameProperty, PhoneProperty} + AvailableStrPropertyValues = map[string][]string{ + UrlProperty: {"https://www.seznam.cz"}, + EmailProperty: {"hello@world.com"}, + NameProperty: {"john doe"}, + PhoneProperty: {"+420602303222"}, + } +) + +// Core is the common component of all generators for all systems. +type Core struct { + *common.Core +} + +// NewCore returns a new Core for the given time range and cardinality +func NewCore(start, end time.Time, scale int) (*Core, error) { + c, err := common.NewCore(start, end, scale) + return &Core{Core: c}, err +} + +type NestedWhereFiller interface { + NestedWhere(query.Query, int) +} + +func (d *Core) GetRandomProperties(n int, properties []string) ([]string, error) { + if n < 1 { + return nil, fmt.Errorf("number of properties cannot be < 1; got %d", n) + } + if n > len(properties) { + return nil, fmt.Errorf("number of properties (%d) larger than total properties (%d)", n, len(properties)) + } + + randomNumbers, err := common.GetRandomSubsetPerm(n, len(properties)) + if err != nil { + return nil, err + } + + selectedProperties := make([]string, n) + for i, n := range randomNumbers { + selectedProperties[i] = properties[n] + } + + return selectedProperties, nil +} + +func (d *Core) GetRandomPropertyValue(property string) (string, error) { + if values, ok := AvailableStrPropertyValues[property]; ok { + return values[rand.Intn(len(values))], nil + } else { + return "", fmt.Errorf("no values for %s", property) + } +} diff --git a/cmd/tsbs_generate_queries/uses/dea/nested_where.go b/cmd/tsbs_generate_queries/uses/dea/nested_where.go new file mode 100644 index 000000000..7861dd027 --- /dev/null +++ b/cmd/tsbs_generate_queries/uses/dea/nested_where.go @@ -0,0 +1,31 @@ +package dea + +import ( + "github.com/timescale/tsbs/cmd/tsbs_generate_queries/uses/common" + "github.com/timescale/tsbs/cmd/tsbs_generate_queries/utils" + "github.com/timescale/tsbs/pkg/query" +) + +// NestedWhere contains info for filling in avg load queries. +type NestedWhere struct { + core utils.QueryGenerator +} + +// NewNestedWhere creates a new avg load query filler. +func NewNestedWhere(scale int) utils.QueryFillerMaker { + return func(core utils.QueryGenerator) utils.QueryFiller { + return &NestedWhere{ + core: core, + } + } +} + +// Fill fills in the query.Query with query details. +func (i *NestedWhere) Fill(q query.Query) query.Query { + fc, ok := i.core.(NestedWhereFiller) + if !ok { + common.PanicUnimplementedQuery(i.core) + } + fc.NestedWhere(q, 1) + return q +} diff --git a/go.mod b/go.mod index 1106116ed..836ed472f 100644 --- a/go.mod +++ b/go.mod @@ -1,16 +1,14 @@ module github.com/timescale/tsbs -go 1.14 +go 1.20 require ( github.com/HdrHistogram/hdrhistogram-go v1.0.0 github.com/SiriDB/go-siridb-connector v0.0.0-20190110105621-86b34c44c921 - github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d // indirect github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883 github.com/aws/aws-sdk-go v1.35.13 github.com/blagojts/viper v1.6.3-0.20200313094124-068f44cf5e69 github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8 - github.com/go-ole/go-ole v1.2.4 // indirect github.com/gocql/gocql v0.0.0-20190810123941-df4b9cc33030 github.com/golang/protobuf v1.4.2 github.com/golang/snappy v0.0.1 @@ -26,7 +24,6 @@ require ( github.com/spf13/cobra v1.0.0 github.com/spf13/pflag v1.0.5 github.com/timescale/promscale v0.0.0-20201006153045-6a66a36f5c84 - github.com/tklauser/go-sysconf v0.3.5 // indirect github.com/transceptor-technology/go-qpack v0.0.0-20190116123619-49a14b216a45 github.com/valyala/fasthttp v1.15.1 go.uber.org/atomic v1.6.0 @@ -34,3 +31,50 @@ require ( golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e gopkg.in/yaml.v2 v2.3.0 ) + +require ( + github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d // indirect + github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 // indirect + github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d // indirect + github.com/andybalholm/brotli v1.0.0 // indirect + github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58 // indirect + github.com/fsnotify/fsnotify v1.4.9 // indirect + github.com/go-ole/go-ole v1.2.4 // indirect + github.com/gogo/protobuf v1.3.1 // indirect + github.com/grpc-ecosystem/grpc-gateway v1.14.8 // indirect + github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed // indirect + github.com/hashicorp/hcl v1.0.0 // indirect + github.com/inconshreveable/mousetrap v1.0.0 // indirect + github.com/jackc/chunkreader/v2 v2.0.1 // indirect + github.com/jackc/pgconn v1.6.3 // indirect + github.com/jackc/pgio v1.0.0 // indirect + github.com/jackc/pgpassfile v1.0.0 // indirect + github.com/jackc/pgproto3/v2 v2.0.2 // indirect + github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b // indirect + github.com/jackc/pgtype v1.4.2 // indirect + github.com/jmespath/go-jmespath v0.4.0 // indirect + github.com/klauspost/compress v1.10.10 // indirect + github.com/magiconair/properties v1.8.1 // indirect + github.com/mitchellh/mapstructure v1.2.2 // indirect + github.com/pelletier/go-toml v1.4.0 // indirect + github.com/sergi/go-diff v1.0.0 // indirect + github.com/sirupsen/logrus v1.9.0 // indirect + github.com/spf13/afero v1.2.2 // indirect + github.com/spf13/cast v1.3.0 // indirect + github.com/spf13/jwalterweatherman v1.0.0 // indirect + github.com/stretchr/testify v1.8.0 // indirect + github.com/subosito/gotenv v1.2.0 // indirect + github.com/tklauser/go-sysconf v0.3.5 // indirect + github.com/tklauser/numcpus v0.2.2 // indirect + github.com/valyala/bytebufferpool v1.0.0 // indirect + golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a // indirect + golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8 // indirect + golang.org/x/text v0.3.3 // indirect + golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect + google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d // indirect + google.golang.org/grpc v1.32.0 // indirect + google.golang.org/protobuf v1.25.0 // indirect + gopkg.in/alecthomas/kingpin.v2 v2.2.6 // indirect + gopkg.in/inf.v0 v0.9.1 // indirect + gopkg.in/ini.v1 v1.51.0 // indirect +) diff --git a/go.sum b/go.sum index 6b3ac40ef..3990d068a 100644 --- a/go.sum +++ b/go.sum @@ -499,7 +499,6 @@ github.com/influxdata/promql/v2 v2.12.0/go.mod h1:fxOPu+DY0bqCTCECchSRtWfc+0X19y github.com/influxdata/roaring v0.4.13-0.20180809181101-fc520f41fab6/go.mod h1:bSgUQ7q5ZLSO+bKBGqJiCBGAl+9DxyW63zLTujjUlOE= github.com/influxdata/tdigest v0.0.0-20181121200506-bf2b5ad3c0a9/go.mod h1:Js0mqiSBE6Ffsg94weZZ2c+v/ciT8QRHFOap7EKDrR0= github.com/influxdata/usage-client v0.0.0-20160829180054-6d3895376368/go.mod h1:Wbbw6tYNvwa5dlB6304Sd+82Z3f7PmVZHVKU637d4po= -github.com/jackc/chunkreader v1.0.0 h1:4s39bBR8ByfqH+DKm8rQA3E1LHZWB9XWcrz8fqaZbe0= github.com/jackc/chunkreader v1.0.0/go.mod h1:RT6O25fNZIuasFJRyZ4R/Y2BbhasbmZXF9QQ7T3kePo= github.com/jackc/chunkreader/v2 v2.0.0/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk= github.com/jackc/chunkreader/v2 v2.0.1 h1:i+RDz65UE+mmpjTfyz0MoVTnzeYxroil2G82ki7MGG8= @@ -519,7 +518,6 @@ github.com/jackc/pgmock v0.0.0-20190831213851-13a1b77aafa2 h1:JVX6jT/XfzNqIjye47 github.com/jackc/pgmock v0.0.0-20190831213851-13a1b77aafa2/go.mod h1:fGZlG77KXmcq05nJLRkk0+p82V8B8Dw8KN2/V9c/OAE= github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= -github.com/jackc/pgproto3 v1.1.0 h1:FYYE4yRw+AgI8wXIinMlNjBbp/UitDJwfj5LqqewP1A= github.com/jackc/pgproto3 v1.1.0/go.mod h1:eR5FA3leWg7p9aeAqi37XOTgTIbkABlvcPB3E5rlc78= github.com/jackc/pgproto3/v2 v2.0.0-alpha1.0.20190420180111-c116219b62db/go.mod h1:bhq50y+xrl9n5mRYyCBFKkpRVTLYJVWeCc+mEAI3yXA= github.com/jackc/pgproto3/v2 v2.0.0-alpha1.0.20190609003834-432c2951c711/go.mod h1:uH0AWtUmuShn0bcesswc4aBTWGvw0cAxIJp+6OB//Wg= @@ -596,7 +594,6 @@ github.com/klauspost/crc32 v0.0.0-20161016154125-cb6bfca970f6/go.mod h1:+ZoRqAPR github.com/klauspost/pgzip v1.0.2-0.20170402124221-0bf5dcad4ada/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/konsorten/go-windows-terminal-sequences v1.0.3 h1:CE8S1cTafDpPvMhIxNJKvHsGVBgn1xWYf1NbHQhywc8= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= @@ -828,11 +825,8 @@ github.com/sergi/go-diff v1.0.0 h1:Kpca3qRNrduNnOQeazBd0ysaKrUJiIuISHxogkT9RPQ= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/shazow/go-diff v0.0.0-20160112020656-b6b7b6733b8c/go.mod h1:/PevMnwAxekIXwN8qQyfc5gl2NlkB3CQlkizAbOkeBs= github.com/shirou/gopsutil v0.0.0-20190901111213-e4ec7b275ada/go.mod h1:WWnYX4lzhCH5h/3YBfyVA3VbLYjlMZZAQcW9ojMexNc= -github.com/shirou/gopsutil v2.18.12+incompatible h1:1eaJvGomDnH74/5cF4CTmTbLHAriGFsTZppLXDX93OM= -github.com/shirou/gopsutil v2.18.12+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= github.com/shirou/gopsutil v3.21.3+incompatible h1:uenXGGa8ESCQq+dbgtl916dmg6PSAz2cXov0uORQ9v8= github.com/shirou/gopsutil v3.21.3+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= -github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4 h1:udFKJ0aHUL60LboW/A+DfgoHVedieIzIXE8uylPue0U= github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4/go.mod h1:qsXQc7+bwAM3Q1u/4XEfrquwF8Lw7D7y5cD8CuHnfIc= github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4= github.com/shopspring/decimal v0.0.0-20200227202807-02e2044944cc h1:jUIKcSPO9MoMJBbEoyE/RJoE8vz7Mb8AjvifMMwSyvY= @@ -847,8 +841,9 @@ github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPx github.com/sirupsen/logrus v1.4.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= -github.com/sirupsen/logrus v1.6.0 h1:UBcNElsrwanuuMsnGSlYmtmgbb23qDR5dG+6X6Oo89I= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= +github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= +github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s= @@ -883,13 +878,17 @@ github.com/streadway/handy v0.0.0-20190108123426-d5acb3125c2a/go.mod h1:qNTQ5P5J github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/testify v1.2.0/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= -github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/tdakkota/asciicheck v0.0.0-20200416190851-d7f85be797a2/go.mod h1:yHp0ai0Z9gUljN3o0xMhYJnH/IcvkdTBOX2fmJ93JEM= @@ -902,6 +901,7 @@ github.com/timescale/promscale v0.0.0-20201006153045-6a66a36f5c84/go.mod h1:rkhy github.com/tinylib/msgp v1.0.2/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE= github.com/tklauser/go-sysconf v0.3.5 h1:uu3Xl4nkLzQfXNsWn15rPc/HQCJKObbt1dKJeWp3vU4= github.com/tklauser/go-sysconf v0.3.5/go.mod h1:MkWzOF4RMCshBAMXuhXJs64Rte09mITnppBXY/rYEFI= +github.com/tklauser/numcpus v0.2.2 h1:oyhllyrScuYI6g+h/zUvNXNp1wy7x8qQy3t/piefldA= github.com/tklauser/numcpus v0.2.2/go.mod h1:x3qojaO3uyYt0i56EW/VUYs7uBvdl2fkfZFu0T9wgjM= github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= @@ -1022,7 +1022,6 @@ golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.3.0 h1:RM4zey1++hCTbCVQfnWeKs9/IEsaBLA8vTkd0WVtmH4= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20170114055629-f2499483f923/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -1136,10 +1135,10 @@ golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200821140526-fda516888d29/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200908134130-d2e65c121b96 h1:gJciq3lOg0eS9fSZJcoHfv7q1BfC6cJfnmSSKL1yu3Q= golang.org/x/sys v0.0.0-20200908134130-d2e65c121b96/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210316164454-77fc1eacc6aa h1:ZYxPR6aca/uhfRJyaOAtflSHjJYiktO7QnJC5ut7iY4= golang.org/x/sys v0.0.0-20210316164454-77fc1eacc6aa/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8 h1:0A+M6Uqn+Eje4kHMK80dtF3JCXC4ykBgQG4Fe06QRhQ= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1348,8 +1347,9 @@ gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776 h1:tQIYjPdBoyREyB9XMu+nnTclpTYkz2zFM+lzLJFO4gQ= gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools v0.0.0-20181223230014-1083505acf35/go.mod h1:R//lfYlUuTOTfblYI3lGoAAAebUdzjvbmQsuB7Ykd90= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/internal/inputs/generator_queries.go b/internal/inputs/generator_queries.go index 44a9ec58c..3221d3dcf 100644 --- a/internal/inputs/generator_queries.go +++ b/internal/inputs/generator_queries.go @@ -40,6 +40,10 @@ type IoTGeneratorMaker interface { NewIoT(start, end time.Time, scale int) (queryUtils.QueryGenerator, error) } +type DEAGeneratorMaker interface { + NewDEA(start, end time.Time, scale int) (queryUtils.QueryGenerator, error) +} + // QueryGenerator is a type of Generator for creating queries to test against a // database. The output is specific to the type of database (due to each using // different querying techniques, e.g. SQL or REST), but is consumed by TSBS @@ -196,6 +200,14 @@ func (g *QueryGenerator) getUseCaseGenerator(c *config.QueryGeneratorConfig) (qu } return devopsFactory.NewDevops(g.tsStart, g.tsEnd, scale) + case common.UseCaseDEA: + deaFactory, ok := factory.(DEAGeneratorMaker) + + if !ok { + return nil, fmt.Errorf(errUseCaseNotImplementedFmt, c.Use, c.Format) + } + + return deaFactory.NewDEA(g.tsStart, g.tsEnd, scale) default: return nil, fmt.Errorf(errUnknownUseCaseFmt, c.Use) } diff --git a/pkg/data/usecases/common/common.go b/pkg/data/usecases/common/common.go index d23188888..cb8806406 100644 --- a/pkg/data/usecases/common/common.go +++ b/pkg/data/usecases/common/common.go @@ -24,6 +24,7 @@ const ( UseCaseDevops = "devops" UseCaseIoT = "iot" UseCaseDevopsGeneric = "devops-generic" + UseCaseDEA = "dea" ) var UseCaseChoices = []string{ @@ -32,4 +33,5 @@ var UseCaseChoices = []string{ UseCaseDevops, UseCaseIoT, UseCaseDevopsGeneric, + UseCaseDEA, } diff --git a/pkg/query/config/config.go b/pkg/query/config/config.go index 86459e93b..374f05d78 100644 --- a/pkg/query/config/config.go +++ b/pkg/query/config/config.go @@ -2,6 +2,7 @@ package config import ( "fmt" + "github.com/spf13/pflag" "github.com/timescale/tsbs/internal/utils" "github.com/timescale/tsbs/pkg/data/usecases/common" @@ -25,7 +26,8 @@ type QueryGeneratorConfig struct { TimescaleUseTags bool `mapstructure:"timescale-use-tags"` TimescaleUseTimeBucket bool `mapstructure:"timescale-use-time-bucket"` - ClickhouseUseTags bool `mapstructure:"clickhouse-use-tags"` + ClickhouseUseTags bool `mapstructure:"clickhouse-use-tags"` + ClickhousePropertyAccessMode string `mapstructure:"clickhouse-property-access-mode"` MongoUseNaive bool `mapstructure:"mongo-use-native"` DbName string `mapstructure:"db-name"` @@ -57,6 +59,7 @@ func (c *QueryGeneratorConfig) AddToFlagSet(fs *pflag.FlagSet) { "The number of round-robin serialization groups. Use this to scale up data generation to multiple processes.") fs.Bool("clickhouse-use-tags", true, "ClickHouse only: Use separate tags table when querying") + fs.String("clickhouse-property-access-mode", "json", "ClickHouse only (DEA): Use json or map") fs.Bool("mongo-use-naive", true, "MongoDB only: Generate queries for the 'naive' data storage format for Mongo") fs.Bool("timescale-use-json", false, "TimescaleDB only: Use separate JSON tags table when querying") fs.Bool("timescale-use-tags", true, "TimescaleDB only: Use separate tags table when querying") diff --git a/pkg/query/factories/init_factories.go b/pkg/query/factories/init_factories.go index ff3faf47d..f1b6f4730 100644 --- a/pkg/query/factories/init_factories.go +++ b/pkg/query/factories/init_factories.go @@ -20,7 +20,8 @@ func InitQueryFactories(config *config.QueryGeneratorConfig) map[string]interfac factories := make(map[string]interface{}) factories[constants.FormatCassandra] = &cassandra.BaseGenerator{} factories[constants.FormatClickhouse] = &clickhouse.BaseGenerator{ - UseTags: config.ClickhouseUseTags, + UseTags: config.ClickhouseUseTags, + PropertyAccessMode: config.ClickhousePropertyAccessMode, } factories[constants.FormatCrateDB] = &cratedb.BaseGenerator{} factories[constants.FormatInflux] = &influx.BaseGenerator{} diff --git a/pkg/targets/clickhouse/creator.go b/pkg/targets/clickhouse/creator.go index 9e991d36b..becb9b20a 100644 --- a/pkg/targets/clickhouse/creator.go +++ b/pkg/targets/clickhouse/creator.go @@ -143,7 +143,9 @@ func createMetricsTable(conf *ClickhouseConfig, db *sqlx.DB, tableName string, f tags_id UInt32, %s, additional_tags String DEFAULT '' - ) ENGINE = MergeTree(created_date, (tags_id, created_at), 8192) + ) ENGINE = MergeTree() PARTITION BY toYYYYMM(created_date) + ORDER BY (tags_id, created_at) + SETTINGS index_granularity = 8192; `, tableName, strings.Join(columnsWithType, ",")) @@ -180,7 +182,9 @@ func generateTagsTableQuery(tagNames, tagTypes []string) string { "created_at DateTime DEFAULT now(),\n"+ "id UInt32,\n"+ "%s"+ - ") ENGINE = MergeTree(created_date, (%s), 8192)", + ") ENGINE = MergeTree() PARTITION BY toYYYYMM(created_date)"+ + "ORDER BY (%s)"+ + "SETTINGS index_granularity = 8192;", cols, index) }