Skip to content

Commit 80bdaf8

Browse files
yashmehrotrayashmehrotra
authored
feat: models,views and schema for config cost (#2068)
* feat: models,views and schema for config cost * chore: fix tests * chore: add currency-safe config cost storage and rollups * chore: add cost compaction with configurable properties * chore: address coderabbit comments
1 parent 3095cf6 commit 80bdaf8

32 files changed

Lines changed: 1475 additions & 209 deletions

‎functions/cost_overlap.sql‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
-- Bucket and window helpers for config cost.
2+
3+
-- Fraction of a cost row's charge period that falls inside a reporting window.
4+
-- Rows coarser than the window straddle it, so rollups prorate with this rather than
5+
-- including or excluding the whole row.
6+
--
7+
-- Both ranges are half-open [start, end). A row fully inside the window returns 1 and one
8+
-- fully outside returns 0, so the common case stays exact.
9+
CREATE OR REPLACE FUNCTION cost_window_overlap(
10+
p_start timestamptz, p_end timestamptz, w_start timestamptz, w_end timestamptz
11+
) RETURNS numeric LANGUAGE sql IMMUTABLE PARALLEL SAFE AS $$
12+
SELECT GREATEST(0, EXTRACT(epoch FROM (LEAST(p_end, w_end) - GREATEST(p_start, w_start))))
13+
/ NULLIF(EXTRACT(epoch FROM (p_end - p_start)), 0);
14+
$$;
15+
16+
-- Start of the bucket of the given width containing ts, anchored on the Unix epoch.
17+
--
18+
-- The width is a parameter rather than a property lookup so this stays IMMUTABLE, which is
19+
-- what lets it be used from a materialized view and an index. The compaction job reads the
20+
-- configured level widths and passes them in.
21+
--
22+
-- Anchoring on the epoch makes the common widths fall on natural boundaries — 3600 gives
23+
-- clock hours, 86400 gives UTC midnights — while any width still tiles the timeline
24+
-- without gaps, so a finer bucket always nests inside exactly one coarser bucket provided
25+
-- the widths divide each other.
26+
CREATE OR REPLACE FUNCTION cost_bucket(ts timestamptz, width_seconds bigint)
27+
RETURNS timestamptz LANGUAGE sql IMMUTABLE PARALLEL SAFE AS $$
28+
SELECT to_timestamp(floor(EXTRACT(epoch FROM ts) / width_seconds) * width_seconds);
29+
$$;

‎functions/drop.sql‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,20 @@
11
DROP VIEW IF EXISTS configs CASCADE;
2+
DROP FUNCTION IF EXISTS refresh_config_cost_summary CASCADE;
3+
4+
-- The materialized view depends on config_costs columns. Drop it before Atlas changes
5+
-- the table, then recreate it from the current definition in 006_config_views.sql.
6+
DROP MATERIALIZED VIEW IF EXISTS config_cost_summary CASCADE;
7+
8+
-- Former name of config_cost_summary; dropped so upgrades do not leave it behind
9+
-- holding a stale dependency on columns the schema apply is about to change.
10+
DROP FUNCTION IF EXISTS refresh_config_costs_rollup CASCADE;
11+
DROP MATERIALIZED VIEW IF EXISTS config_costs_rollup CASCADE;
12+
13+
-- config_summary & config_class_summary aggregate cost straight off config_items,
14+
-- so they hold a dependency on those columns and must go before the schema apply.
15+
DROP VIEW IF EXISTS config_summary CASCADE;
16+
17+
DROP VIEW IF EXISTS config_class_summary CASCADE;
218

319
DROP VIEW IF EXISTS config_detail CASCADE;
420

‎go.mod‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -75,6 +75,7 @@ require (
7575
github.com/samber/lo v1.53.0
7676
github.com/samber/oops v1.22.0
7777
github.com/sethvargo/go-retry v0.3.0
78+
github.com/shopspring/decimal v1.4.0
7879
github.com/spf13/cobra v1.10.2
7980
github.com/spf13/pflag v1.0.10
8081
github.com/stretchr/testify v1.11.1
@@ -362,7 +363,6 @@ require (
362363
github.com/sergi/go-diff v1.4.0 // indirect
363364
github.com/shirou/gopsutil/v3 v3.24.5 // indirect
364365
github.com/shoenig/go-m1cpu v0.1.7 // indirect
365-
github.com/shopspring/decimal v1.4.0 // indirect
366366
github.com/sirupsen/logrus v1.9.4 // indirect
367367
github.com/skeema/knownhosts v1.3.2 // indirect
368368
github.com/spiffe/go-spiffe/v2 v2.6.0 // indirect

‎migrate/dependency_test.go‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,8 @@ func TestDependencyMap(t *testing.T) {
4646

4747
expected := map[string][]string{
4848
"functions/drop.sql": {"views/006_config_views.sql", "views/021_notification.sql", "views/038_config_access.sql"},
49-
"views/006_config_views.sql": {"views/021_notification.sql"},
49+
"functions/cost_overlap.sql": {"views/006_config_views.sql"},
50+
"views/006_config_views.sql": {"views/014_config_item_by_type.sql", "views/021_notification.sql"},
5051
}
5152

5253
g.Expect(graph).To(gomega.HaveLen(len(expected)))

‎models/config.go‎

Lines changed: 69 additions & 79 deletions
Original file line numberDiff line numberDiff line change
@@ -133,33 +133,29 @@ type ConfigLocation struct {
133133

134134
// ConfigItem represents the config item database table
135135
type ConfigItem struct {
136-
ID uuid.UUID `json:"id" faker:"uuid_hyphenated" gorm:"default:generate_ulid()"`
137-
ScraperID *string `json:"scraper_id,omitempty"`
138-
AgentID uuid.UUID `json:"agent_id,omitempty"`
139-
ConfigClass string `json:"config_class" faker:"oneof:File,EC2Instance,KubernetesPod" `
140-
ExternalID pq.StringArray `gorm:"type:[]text" json:"external_id,omitempty"`
141-
Type *string `json:"type"`
142-
Status *string `json:"status" gorm:"default:null"`
143-
Ready bool `json:"ready"`
144-
Health *Health `json:"health"`
145-
Name *string `json:"name,omitempty" faker:"name"`
146-
Description *string `json:"description"`
147-
Config *string `json:"config"`
148-
Source *string `json:"source,omitempty"`
149-
ParentID *uuid.UUID `json:"parent_id,omitempty" faker:"-"`
150-
Path string `json:"path,omitempty" faker:"-"`
151-
CostPerMinute float64 `gorm:"column:cost_per_minute;default:null" json:"cost_per_minute,omitempty"`
152-
CostTotal1d float64 `gorm:"column:cost_total_1d;default:null" json:"cost_total_1d,omitempty"`
153-
CostTotal7d float64 `gorm:"column:cost_total_7d;default:null" json:"cost_total_7d,omitempty"`
154-
CostTotal30d float64 `gorm:"column:cost_total_30d;default:null" json:"cost_total_30d,omitempty"`
155-
Labels *types.JSONStringMap `json:"labels,omitempty" faker:"labels"`
156-
Tags types.JSONStringMap `json:"tags,omitempty" faker:"tags"`
157-
Properties *types.Properties `json:"properties,omitempty"`
158-
CreatedAt time.Time `json:"created_at" gorm:"<-:create"`
159-
InsertedAt time.Time `json:"inserted_at" gorm:"->;default:now()"`
160-
UpdatedAt *time.Time `json:"updated_at" gorm:"autoUpdateTime:false"`
161-
DeletedAt *time.Time `json:"deleted_at,omitempty"`
162-
DeleteReason string `json:"delete_reason,omitempty"`
136+
ID uuid.UUID `json:"id" faker:"uuid_hyphenated" gorm:"default:generate_ulid()"`
137+
ScraperID *string `json:"scraper_id,omitempty"`
138+
AgentID uuid.UUID `json:"agent_id,omitempty"`
139+
ConfigClass string `json:"config_class" faker:"oneof:File,EC2Instance,KubernetesPod" `
140+
ExternalID pq.StringArray `gorm:"type:[]text" json:"external_id,omitempty"`
141+
Type *string `json:"type"`
142+
Status *string `json:"status" gorm:"default:null"`
143+
Ready bool `json:"ready"`
144+
Health *Health `json:"health"`
145+
Name *string `json:"name,omitempty" faker:"name"`
146+
Description *string `json:"description"`
147+
Config *string `json:"config"`
148+
Source *string `json:"source,omitempty"`
149+
ParentID *uuid.UUID `json:"parent_id,omitempty" faker:"-"`
150+
Path string `json:"path,omitempty" faker:"-"`
151+
Labels *types.JSONStringMap `json:"labels,omitempty" faker:"labels"`
152+
Tags types.JSONStringMap `json:"tags,omitempty" faker:"tags"`
153+
Properties *types.Properties `json:"properties,omitempty"`
154+
CreatedAt time.Time `json:"created_at" gorm:"<-:create"`
155+
InsertedAt time.Time `json:"inserted_at" gorm:"->;default:now()"`
156+
UpdatedAt *time.Time `json:"updated_at" gorm:"autoUpdateTime:false"`
157+
DeletedAt *time.Time `json:"deleted_at,omitempty"`
158+
DeleteReason string `json:"delete_reason,omitempty"`
163159

164160
configJson map[string]any `json:"-" yaml:"-" gorm:"-"`
165161
}
@@ -266,10 +262,6 @@ func (c ConfigItem) PrettyRow(opts interface{}) map[string]api.Text {
266262
row["status"] = clicky.Text(lo.FromPtr(c.Status), "text-gray-700")
267263
}
268264

269-
if c.CostTotal30d > 0 {
270-
row["cost"] = clicky.Text(fmt.Sprintf("$%.2f", c.CostTotal30d), "text-green-700")
271-
}
272-
273265
if c.CreatedAt != (time.Time{}) {
274266
row["age"] = api.Human(time.Since(c.CreatedAt), "text-gray-600")
275267
}
@@ -1038,32 +1030,34 @@ func (e ExternalID) WhereClause(db *gorm.DB) *gorm.DB {
10381030

10391031
// ConfigItemSummary represents the configs view
10401032
type ConfigItemSummary struct {
1041-
ID uuid.UUID `json:"id" gorm:"primaryKey"`
1042-
ScraperID *string `json:"scraper_id,omitempty"`
1043-
ConfigClass string `json:"config_class"`
1044-
ExternalID pq.StringArray `gorm:"type:[]text" json:"external_id,omitempty"`
1045-
Type *string `json:"type"`
1046-
Name *string `json:"name,omitempty"`
1047-
Namespace *string `json:"namespace,omitempty"`
1048-
Description *string `json:"description"`
1049-
Source *string `json:"source,omitempty"`
1050-
Labels *types.JSONStringMap `json:"labels,omitempty"`
1051-
Tags types.JSONStringMap `json:"tags,omitempty"`
1052-
CreatedBy *uuid.UUID `json:"created_by,omitempty"`
1053-
CreatedAt time.Time `json:"created_at"`
1054-
UpdatedAt *time.Time `json:"updated_at"`
1055-
DeletedAt *time.Time `json:"deleted_at,omitempty"`
1056-
CostPerMinute float64 `gorm:"column:cost_per_minute" json:"cost_per_minute,omitempty"`
1057-
CostTotal1d float64 `gorm:"column:cost_total_1d" json:"cost_total_1d,omitempty"`
1058-
CostTotal7d float64 `gorm:"column:cost_total_7d" json:"cost_total_7d,omitempty"`
1059-
CostTotal30d float64 `gorm:"column:cost_total_30d" json:"cost_total_30d,omitempty"`
1060-
AgentID uuid.UUID `json:"agent_id,omitempty"`
1061-
Status *string `json:"status"`
1062-
Health *Health `json:"health"`
1063-
Ready bool `json:"ready"`
1064-
Path string `json:"path,omitempty"`
1065-
Changes int `json:"changes,omitempty"`
1066-
Analysis *types.JSONMap `json:"analysis,omitempty"`
1033+
ID uuid.UUID `json:"id" gorm:"primaryKey"`
1034+
ScraperID *string `json:"scraper_id,omitempty"`
1035+
ConfigClass string `json:"config_class"`
1036+
ExternalID pq.StringArray `gorm:"type:[]text" json:"external_id,omitempty"`
1037+
Type *string `json:"type"`
1038+
Name *string `json:"name,omitempty"`
1039+
Namespace *string `json:"namespace,omitempty"`
1040+
Description *string `json:"description"`
1041+
Source *string `json:"source,omitempty"`
1042+
Labels *types.JSONStringMap `json:"labels,omitempty"`
1043+
Tags types.JSONStringMap `json:"tags,omitempty"`
1044+
CreatedBy *uuid.UUID `json:"created_by,omitempty"`
1045+
CreatedAt time.Time `json:"created_at"`
1046+
UpdatedAt *time.Time `json:"updated_at"`
1047+
DeletedAt *time.Time `json:"deleted_at,omitempty"`
1048+
CostPerMinute *float64 `gorm:"column:cost_per_minute" json:"cost_per_minute,omitempty"`
1049+
CostTotal1h *float64 `gorm:"column:cost_total_1h" json:"cost_total_1h,omitempty"`
1050+
CostTotal1d *float64 `gorm:"column:cost_total_1d" json:"cost_total_1d,omitempty"`
1051+
CostTotal30d *float64 `gorm:"column:cost_total_30d" json:"cost_total_30d,omitempty"`
1052+
BillingCurrency *string `gorm:"column:billing_currency" json:"billing_currency,omitempty"`
1053+
MixedCurrency bool `gorm:"column:mixed_currency" json:"mixed_currency"`
1054+
AgentID uuid.UUID `json:"agent_id,omitempty"`
1055+
Status *string `json:"status"`
1056+
Health *Health `json:"health"`
1057+
Ready bool `json:"ready"`
1058+
Path string `json:"path,omitempty"`
1059+
Changes int `json:"changes,omitempty"`
1060+
Analysis *types.JSONMap `json:"analysis,omitempty"`
10671061
}
10681062

10691063
func (ConfigItemSummary) TableName() string {
@@ -1079,27 +1073,23 @@ func (c ConfigItemSummary) GetAgentID() string {
10791073

10801074
func (c ConfigItemSummary) ToConfigItem() ConfigItem {
10811075
return ConfigItem{
1082-
ID: c.ID,
1083-
ScraperID: c.ScraperID,
1084-
AgentID: c.AgentID,
1085-
ConfigClass: c.ConfigClass,
1086-
ExternalID: c.ExternalID,
1087-
Type: c.Type,
1088-
Status: c.Status,
1089-
Ready: c.Ready,
1090-
Health: c.Health,
1091-
Name: c.Name,
1092-
Description: c.Description,
1093-
Source: c.Source,
1094-
Path: c.Path,
1095-
CostPerMinute: c.CostPerMinute,
1096-
CostTotal1d: c.CostTotal1d,
1097-
CostTotal7d: c.CostTotal7d,
1098-
CostTotal30d: c.CostTotal30d,
1099-
Labels: c.Labels,
1100-
Tags: c.Tags,
1101-
CreatedAt: c.CreatedAt,
1102-
UpdatedAt: c.UpdatedAt,
1103-
DeletedAt: c.DeletedAt,
1076+
ID: c.ID,
1077+
ScraperID: c.ScraperID,
1078+
AgentID: c.AgentID,
1079+
ConfigClass: c.ConfigClass,
1080+
ExternalID: c.ExternalID,
1081+
Type: c.Type,
1082+
Status: c.Status,
1083+
Ready: c.Ready,
1084+
Health: c.Health,
1085+
Name: c.Name,
1086+
Description: c.Description,
1087+
Source: c.Source,
1088+
Path: c.Path,
1089+
Labels: c.Labels,
1090+
Tags: c.Tags,
1091+
CreatedAt: c.CreatedAt,
1092+
UpdatedAt: c.UpdatedAt,
1093+
DeletedAt: c.DeletedAt,
11041094
}
11051095
}

0 commit comments

Comments
 (0)