From f0dfec87610e2458fc211d32760056c280adf28d Mon Sep 17 00:00:00 2001 From: Sparsh Date: Tue, 5 Aug 2025 12:54:59 +0530 Subject: [PATCH 1/8] Initial commit for the fastly plugin Signed-off-by: Sparsh --- pkg/plugins/fastly/cmd/main/main.go | 340 ++++++++++++++++++ pkg/plugins/fastly/cmd/main/main_test.go | 1 + pkg/plugins/fastly/cmd/validator/main/main.go | 201 +++++++++++ pkg/plugins/fastly/fastly-config-sample.json | 9 + pkg/plugins/fastly/go.mod | 66 ++++ pkg/plugins/fastly/go.sum | 213 +++++++++++ pkg/plugins/fastly/plugin/fastlyconfig.go | 44 +++ pkg/plugins/fastly/plugin/services.go | 314 ++++++++++++++++ pkg/plugins/fastly/plugin/types.go | 173 +++++++++ pkg/plugins/fastly/tests/fastly_test.go | 236 ++++++++++++ 10 files changed, 1597 insertions(+) create mode 100644 pkg/plugins/fastly/cmd/main/main.go create mode 100644 pkg/plugins/fastly/cmd/main/main_test.go create mode 100644 pkg/plugins/fastly/cmd/validator/main/main.go create mode 100644 pkg/plugins/fastly/fastly-config-sample.json create mode 100644 pkg/plugins/fastly/go.mod create mode 100644 pkg/plugins/fastly/go.sum create mode 100644 pkg/plugins/fastly/plugin/fastlyconfig.go create mode 100644 pkg/plugins/fastly/plugin/services.go create mode 100644 pkg/plugins/fastly/plugin/types.go create mode 100644 pkg/plugins/fastly/tests/fastly_test.go diff --git a/pkg/plugins/fastly/cmd/main/main.go b/pkg/plugins/fastly/cmd/main/main.go new file mode 100644 index 0000000..c674c90 --- /dev/null +++ b/pkg/plugins/fastly/cmd/main/main.go @@ -0,0 +1,340 @@ +package main + +import ( + "context" + "encoding/json" + "fmt" + "os" + "time" + + "github.com/hashicorp/go-plugin" + commonconfig "github.com/opencost/opencost-plugins/pkg/common/config" + fastlyplugin "github.com/opencost/opencost-plugins/pkg/plugins/fastly/plugin" + "github.com/opencost/opencost/core/pkg/log" + "github.com/opencost/opencost/core/pkg/model/pb" + "github.com/opencost/opencost/core/pkg/opencost" + ocplugin "github.com/opencost/opencost/core/pkg/plugin" + "golang.org/x/time/rate" + "google.golang.org/protobuf/types/known/timestamppb" +) + +// handshakeConfigs are used to just do a basic handshake between +// a plugin and host. If the handshake fails, a user-friendly error is shown. +var handshakeConfig = plugin.HandshakeConfig{ + ProtocolVersion: 1, + MagicCookieKey: "PLUGIN_NAME", + MagicCookieValue: "fastly", +} + +// FastlyCostSource implements the CustomCostSource interface +type FastlyCostSource struct { + config *fastlyplugin.FastlyConfig + client *fastlyplugin.FastlyClient + rateLimiter *rate.Limiter + ctx context.Context + costCalculator *fastlyplugin.CostCalculator + serviceMapper *fastlyplugin.ServiceMapper +} + +// GetCustomCosts retrieves custom costs from Fastly for the given time window +func (f *FastlyCostSource) GetCustomCosts(req *pb.CustomCostRequest) []*pb.CustomCostResponse { + results := []*pb.CustomCostResponse{} + + // Get windows based on resolution + targets, err := opencost.GetWindows(req.Start.AsTime(), req.End.AsTime(), req.Resolution.AsDuration()) + if err != nil { + log.Errorf("error getting windows: %v", err) + errResp := pb.CustomCostResponse{ + Errors: []string{fmt.Sprintf("error getting windows: %v", err)}, + } + results = append(results, &errResp) + return results + } + + for _, target := range targets { + // Skip future windows + if target.Start().After(time.Now().UTC()) { + log.Debugf("skipping future window %v", target) + continue + } + + log.Debugf("fetching Fastly costs for window %v", target) + result := f.getFastlyCostsForWindow(target) + results = append(results, result) + } + + return results +} + +func (f *FastlyCostSource) getFastlyCostsForWindow(window opencost.Window) *pb.CustomCostResponse { + ccResp := &pb.CustomCostResponse{ + Metadata: map[string]string{"api_version": "v1"}, + CostSource: "infrastructure", + Domain: "fastly", + Version: "v1", + Currency: "USD", + Start: timestamppb.New(*window.Start()), + End: timestamppb.New(*window.End()), + Errors: []string{}, + Costs: []*pb.CustomCost{}, + } + + // Rate limit check + if err := f.rateLimiter.Wait(f.ctx); err != nil { + ccResp.Errors = append(ccResp.Errors, fmt.Sprintf("rate limiter error: %v", err)) + return ccResp + } + + // Fetch invoices from Fastly API + invoices, err := f.client.FetchInvoices(f.ctx, *window.Start(), *window.End()) + if err != nil { + ccResp.Errors = append(ccResp.Errors, fmt.Sprintf("error fetching invoices: %v", err)) + return ccResp + } + + // Process invoices into costs + costs := f.processInvoices(invoices, window) + + // Optionally fetch usage data for more detailed cost allocation + if f.config.EnableUsageDetail { + usageCosts, err := f.fetchAndProcessUsage(window) + if err != nil { + log.Warnf("error fetching usage details: %v", err) + // Continue with invoice-based costs only + } else { + costs = f.reconcileCosts(costs, usageCosts) + } + } + + ccResp.Costs = costs + return ccResp +} + +func (f *FastlyCostSource) processInvoices(invoices []fastlyplugin.Invoice, window opencost.Window) []*pb.CustomCost { + costs := []*pb.CustomCost{} + + for _, invoice := range invoices { + // Process each line item in the invoice + for _, lineItem := range invoice.LineItems { + cost := f.createCostFromLineItem(invoice, lineItem, window) + if cost != nil { + costs = append(costs, cost) + } + } + } + + return costs +} + +func (f *FastlyCostSource) fetchAndProcessUsage(window opencost.Window) ([]*pb.CustomCost, error) { + costs := []*pb.CustomCost{} + + // Fetch all services if not already cached + if f.serviceMapper == nil { + services, err := f.client.FetchServices(f.ctx) + if err != nil { + return nil, fmt.Errorf("error fetching services: %v", err) + } + f.serviceMapper = fastlyplugin.NewServiceMapper(services) + } + + // Get service IDs to process + serviceIDs := []string{} + for _, service := range f.serviceMapper.GetServices() { + // Apply filters if configured + if len(f.config.ServiceFilters) > 0 { + found := false + for _, filter := range f.config.ServiceFilters { + if service.ID == filter { + found = true + break + } + } + if !found { + continue + } + } + + // Apply exclusions if configured + excluded := false + for _, exclude := range f.config.ExcludeServices { + if service.ID == exclude { + excluded = true + break + } + } + if excluded { + continue + } + + serviceIDs = append(serviceIDs, service.ID) + } + + // Fetch usage for each service + for _, serviceID := range serviceIDs { + usage, err := f.client.FetchUsage(f.ctx, *window.Start(), *window.End(), serviceID) + if err != nil { + log.Warnf("error fetching usage for service %s: %v", serviceID, err) + continue + } + + // Calculate costs from usage + for _, u := range usage { + allocations := f.costCalculator.CalculateUsageCost(u) + for _, alloc := range allocations { + cost := f.createCostFromAllocation(alloc) + costs = append(costs, cost) + } + } + } + + return costs, nil +} + +func (f *FastlyCostSource) createCostFromAllocation(alloc fastlyplugin.CostAllocation) *pb.CustomCost { + serviceName := f.serviceMapper.GetServiceName(alloc.ServiceID) + attrs := f.serviceMapper.GetServiceAttributes(alloc.ServiceID) + + return &pb.CustomCost{ + AccountName: f.config.FastlyAccountID, + ChargeCategory: "usage", + Description: fmt.Sprintf("%s - %s", serviceName, alloc.CostType), + ResourceName: alloc.CostType, + ResourceType: "cdn", + Id: fmt.Sprintf("%s-%s-%s-%s", alloc.ServiceID, alloc.CostType, alloc.Region, alloc.StartTime.Format("2006-01-02-15")), + ProviderId: alloc.ServiceID, + BilledCost: alloc.Amount, + ListCost: alloc.Amount, + UsageQuantity: alloc.Units, + UsageUnit: alloc.UnitType, + Zone: alloc.Region, + Labels: map[string]string{ + "service_id": alloc.ServiceID, + "service_name": serviceName, + "cost_type": alloc.CostType, + "region": alloc.Region, + "service_attributes": fmt.Sprintf("%v", attrs), + }, + ExtendedAttributes: &pb.CustomCostExtendedAttributes{ + ServiceName: stringPtr("fastly-cdn"), + ServiceCategory: stringPtr("infrastructure"), + Provider: stringPtr("fastly"), + AccountId: stringPtr(f.config.FastlyAccountID), + }, + } +} + +func (f *FastlyCostSource) reconcileCosts(invoiceCosts, usageCosts []*pb.CustomCost) []*pb.CustomCost { + // If we have detailed usage costs, prefer those over invoice line items + // This provides more granular cost allocation + if len(usageCosts) > 0 { + return usageCosts + } + return invoiceCosts +} + +func (f *FastlyCostSource) createCostFromLineItem(invoice fastlyplugin.Invoice, item fastlyplugin.LineItem, window opencost.Window) *pb.CustomCost { + // Calculate the daily rate for the line item + dailyRate := item.Total / float32(invoice.GetDaysInBillingPeriod()) + + // Calculate the cost for this specific window + windowDuration := window.End().Sub(*window.Start()).Hours() / 24.0 + windowCost := dailyRate * float32(windowDuration) + + return &pb.CustomCost{ + AccountName: invoice.CustomerID, + ChargeCategory: "usage", + Description: item.Description, + ResourceName: item.ServiceType, + ResourceType: "cdn", + Id: fmt.Sprintf("%s-%s-%s", invoice.ID, item.ServiceType, window.Start().Format("2006-01-02")), + ProviderId: invoice.ID, + BilledCost: windowCost, + ListCost: windowCost, + ListUnitPrice: item.Rate, + UsageQuantity: item.Units, + UsageUnit: item.UnitType, + Labels: map[string]string{ + "service_type": item.ServiceType, + "region": item.Region, + }, + Zone: item.Region, + ExtendedAttributes: &pb.CustomCostExtendedAttributes{ + BillingPeriodStart: timestamppb.New(invoice.BillingStartDate), + BillingPeriodEnd: timestamppb.New(invoice.BillingEndDate), + ServiceName: stringPtr("fastly-cdn"), + ServiceCategory: stringPtr("infrastructure"), + Provider: stringPtr("fastly"), + AccountId: stringPtr(invoice.CustomerID), + }, + } +} + +func main() { + configFile, err := commonconfig.GetConfigFilePath() + if err != nil { + log.Fatalf("error opening config file: %v", err) + } + + fastlyConfig, err := getFastlyConfig(configFile) + if err != nil { + log.Fatalf("error building Fastly config: %v", err) + } + + log.SetLogLevel(fastlyConfig.LogLevel) + + // Create rate limiter + rateLimiter := rate.NewLimiter(rate.Limit(fastlyConfig.RateLimitPerSecond), 1) + + // Create Fastly API client + client := fastlyplugin.NewFastlyClient(fastlyConfig.FastlyAPIToken) + + // Create cost calculator + costCalculator := fastlyplugin.NewCostCalculator() + + // Create Fastly cost source + fastlyCostSrc := &FastlyCostSource{ + config: fastlyConfig, + client: client, + rateLimiter: rateLimiter, + ctx: context.Background(), + costCalculator: costCalculator, + } + + // Plugin map for the CustomCostSource + var pluginMap = map[string]plugin.Plugin{ + "CustomCostSource": &ocplugin.CustomCostPlugin{Impl: fastlyCostSrc}, + } + + plugin.Serve(&plugin.ServeConfig{ + HandshakeConfig: handshakeConfig, + Plugins: pluginMap, + GRPCServer: plugin.DefaultGRPCServer, + }) +} + +func getFastlyConfig(configFilePath string) (*fastlyplugin.FastlyConfig, error) { + var result fastlyplugin.FastlyConfig + + bytes, err := os.ReadFile(configFilePath) + if err != nil { + return nil, fmt.Errorf("error reading config file for Fastly config @ %s: %v", configFilePath, err) + } + + err = json.Unmarshal(bytes, &result) + if err != nil { + return nil, fmt.Errorf("error marshaling json into Fastly config: %v", err) + } + + // Validate config + if err := result.Validate(); err != nil { + return nil, err + } + + return &result, nil +} + +// Helper function to convert a string to a string pointer +func stringPtr(s string) *string { + return &s +} diff --git a/pkg/plugins/fastly/cmd/main/main_test.go b/pkg/plugins/fastly/cmd/main/main_test.go new file mode 100644 index 0000000..06ab7d0 --- /dev/null +++ b/pkg/plugins/fastly/cmd/main/main_test.go @@ -0,0 +1 @@ +package main diff --git a/pkg/plugins/fastly/cmd/validator/main/main.go b/pkg/plugins/fastly/cmd/validator/main/main.go new file mode 100644 index 0000000..2f1f561 --- /dev/null +++ b/pkg/plugins/fastly/cmd/validator/main/main.go @@ -0,0 +1,201 @@ +package main + +import ( + "encoding/json" + "fmt" + "os" + "time" + + "github.com/hashicorp/go-multierror" + "github.com/opencost/opencost/core/pkg/log" + "github.com/opencost/opencost/core/pkg/model/pb" + "google.golang.org/protobuf/encoding/protojson" +) + +// Validator for Fastly plugin integration tests +func main() { + if len(os.Args) < 3 { + fmt.Println("Usage: validator ") + os.Exit(1) + } + + dailyProtobufFilePath := os.Args[1] + hourlyProtobufFilePath := os.Args[2] + + // Read and validate daily data + dailyData, err := os.ReadFile(dailyProtobufFilePath) + if err != nil { + fmt.Printf("Error reading daily protobuf file: %v\n", err) + os.Exit(1) + } + + dailyCustomCostResponses, err := Unmarshal(dailyData) + if err != nil { + fmt.Printf("Error unmarshalling daily protobuf data: %v\n", err) + os.Exit(1) + } + + fmt.Printf("Successfully unmarshalled %d daily custom cost responses\n", len(dailyCustomCostResponses)) + + // Read and validate hourly data + hourlyData, err := os.ReadFile(hourlyProtobufFilePath) + if err != nil { + fmt.Printf("Error reading hourly protobuf file: %v\n", err) + os.Exit(1) + } + + hourlyCustomCostResponses, err := Unmarshal(hourlyData) + if err != nil { + fmt.Printf("Error unmarshalling hourly protobuf data: %v\n", err) + os.Exit(1) + } + + fmt.Printf("Successfully unmarshalled %d hourly custom cost responses\n", len(hourlyCustomCostResponses)) + + // Validate the responses + isValid := validate(dailyCustomCostResponses, hourlyCustomCostResponses) + if !isValid { + os.Exit(1) + } else { + fmt.Println("Validation successful") + } +} + +func validate(respDaily, respHourly []*pb.CustomCostResponse) bool { + if len(respDaily) == 0 { + log.Errorf("no daily response received from fastly plugin") + return false + } + + if len(respHourly) == 0 { + log.Errorf("no hourly response received from fastly plugin") + return false + } + + var multiErr error + + // Check for errors in responses + for _, resp := range respDaily { + if len(resp.Errors) > 0 { + multiErr = multierror.Append(multiErr, fmt.Errorf("errors occurred in daily response: %v", resp.Errors)) + } + } + + for _, resp := range respHourly { + if len(resp.Errors) > 0 { + multiErr = multierror.Append(multiErr, fmt.Errorf("errors occurred in hourly response: %v", resp.Errors)) + } + } + + if multiErr != nil { + log.Errorf("Errors occurred during plugin testing for fastly: %v", multiErr) + return false + } + + // Validate daily costs + seenCosts := map[string]bool{} + totalDailyCost := float32(0.0) + + for _, resp := range respDaily { + // Skip empty responses for recent dates + if len(resp.Costs) == 0 && resp.Start.AsTime().After(time.Now().Truncate(24*time.Hour).Add(-1*time.Minute)) { + log.Debugf("today's daily costs returned by plugin fastly are empty, skipping: %v", resp) + continue + } + + for _, cost := range resp.Costs { + totalDailyCost += cost.GetBilledCost() + seenCosts[cost.GetResourceName()] = true + + // Validate cost sanity + if cost.GetBilledCost() < 0 { + log.Errorf("negative cost found for %v", cost) + return false + } + + // Check for reasonable cost values (adjust based on your expected ranges) + if cost.GetBilledCost() > 10000 { + log.Errorf("unexpectedly high daily cost for %v: %f", cost.GetResourceName(), cost.GetBilledCost()) + return false + } + } + } + + // Validate we have some expected cost types + expectedCosts := []string{ + "cdn_bandwidth", + "cdn_requests", + // Add more expected cost types as needed + } + + foundExpectedCost := false + for _, expectedCost := range expectedCosts { + if seenCosts[expectedCost] { + foundExpectedCost = true + break + } + } + + if !foundExpectedCost && totalDailyCost > 0 { + log.Warnf("None of the expected cost types found, but costs exist. Seen costs: %v", seenCosts) + } + + // Validate domain + for _, resp := range respDaily { + if resp.Domain != "fastly" { + log.Errorf("daily domain returned by plugin does not match expected 'fastly': %s", resp.Domain) + return false + } + } + + // Validate hourly costs + seenHourlyCosts := map[string]bool{} + totalHourlyCost := float32(0.0) + + for _, resp := range respHourly { + for _, cost := range resp.Costs { + seenHourlyCosts[cost.GetResourceName()] = true + totalHourlyCost += cost.GetBilledCost() + + if cost.GetBilledCost() < 0 { + log.Errorf("negative hourly cost found for %v", cost) + return false + } + + // Hourly costs should be smaller than daily + if cost.GetBilledCost() > 1000 { + log.Errorf("unexpectedly high hourly cost for %v: %f", cost.GetResourceName(), cost.GetBilledCost()) + return false + } + } + } + + // Basic sanity check - if we have daily costs, we should have hourly costs + if totalDailyCost > 0 && totalHourlyCost == 0 { + log.Errorf("daily costs exist but no hourly costs found") + return false + } + + log.Infof("Validation passed. Daily costs: %f, Hourly costs: %f", totalDailyCost, totalHourlyCost) + log.Infof("Cost types seen - Daily: %v, Hourly: %v", seenCosts, seenHourlyCosts) + + return true +} + +func Unmarshal(data []byte) ([]*pb.CustomCostResponse, error) { + var raw []json.RawMessage + if err := json.Unmarshal(data, &raw); err != nil { + return nil, err + } + + protoResps := make([]*pb.CustomCostResponse, len(raw)) + for i, r := range raw { + p := &pb.CustomCostResponse{} + if err := protojson.Unmarshal(r, p); err != nil { + return nil, err + } + protoResps[i] = p + } + + return protoResps, nil +} diff --git a/pkg/plugins/fastly/fastly-config-sample.json b/pkg/plugins/fastly/fastly-config-sample.json new file mode 100644 index 0000000..44683c1 --- /dev/null +++ b/pkg/plugins/fastly/fastly-config-sample.json @@ -0,0 +1,9 @@ +{ + "fastly_api_token": "YOUR_FASTLY_API_TOKEN_HERE", + "fastly_account_id": "", + "log_level": "info", + "rate_limit_per_second": 1.0, + "enable_usage_detail": false, + "service_filters": [], + "exclude_services": [] +} diff --git a/pkg/plugins/fastly/go.mod b/pkg/plugins/fastly/go.mod new file mode 100644 index 0000000..301ed7a --- /dev/null +++ b/pkg/plugins/fastly/go.mod @@ -0,0 +1,66 @@ +module github.com/opencost/opencost-plugins/pkg/plugins/fastly + +go 1.24.2 + +replace github.com/opencost/opencost-plugins/pkg/common => ../../common + +require ( + github.com/hashicorp/go-multierror v1.1.1 + github.com/hashicorp/go-plugin v1.6.3 + github.com/opencost/opencost-plugins/pkg/common v0.0.0-20250403142249-d8eaf3258224 + github.com/opencost/opencost/core v0.0.0-20250805023318-1e1643d24b0e + golang.org/x/time v0.12.0 + google.golang.org/protobuf v1.36.6 +) + +require ( + github.com/fatih/color v1.18.0 // indirect + github.com/fsnotify/fsnotify v1.7.0 // indirect + github.com/fxamacker/cbor/v2 v2.7.0 // indirect + github.com/go-logr/logr v1.4.3 // indirect + github.com/goccy/go-json v0.10.5 // indirect + github.com/gogo/protobuf v1.3.2 // indirect + github.com/golang/protobuf v1.5.4 // indirect + github.com/hashicorp/errwrap v1.1.0 // indirect + github.com/hashicorp/go-hclog v1.6.3 // indirect + github.com/hashicorp/hcl v1.0.0 // indirect + github.com/hashicorp/yamux v0.1.2 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/magiconair/properties v1.8.7 // indirect + github.com/mattn/go-colorable v0.1.14 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect + github.com/mitchellh/mapstructure v1.5.0 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/oklog/run v1.2.0 // indirect + github.com/patrickmn/go-cache v2.1.0+incompatible // indirect + github.com/pelletier/go-toml/v2 v2.1.1 // indirect + github.com/rs/zerolog v1.32.0 // indirect + github.com/sagikazarmark/locafero v0.4.0 // indirect + github.com/sagikazarmark/slog-shim v0.1.0 // indirect + github.com/sourcegraph/conc v0.3.0 // indirect + github.com/spf13/afero v1.11.0 // indirect + github.com/spf13/cast v1.6.0 // indirect + github.com/spf13/pflag v1.0.5 // indirect + github.com/spf13/viper v1.18.2 // indirect + github.com/subosito/gotenv v1.6.0 // indirect + github.com/x448/float16 v0.8.4 // indirect + go.uber.org/multierr v1.11.0 // indirect + golang.org/x/exp v0.0.0-20240222234643-814bf88cf225 // indirect + golang.org/x/net v0.42.0 // indirect + golang.org/x/sys v0.34.0 // indirect + golang.org/x/text v0.27.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20250804133106-a7a43d27e69b // indirect + google.golang.org/grpc v1.74.2 // indirect + gopkg.in/inf.v0 v0.9.1 // indirect + gopkg.in/ini.v1 v1.67.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect + k8s.io/api v0.33.1 // indirect + k8s.io/apimachinery v0.33.1 // indirect + k8s.io/klog/v2 v2.130.1 // indirect + k8s.io/utils v0.0.0-20250321185631-1f6e0b77f77e // indirect + sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3 // indirect + sigs.k8s.io/randfill v1.0.0 // indirect + sigs.k8s.io/structured-merge-diff/v4 v4.6.0 // indirect + sigs.k8s.io/yaml v1.4.0 // indirect +) diff --git a/pkg/plugins/fastly/go.sum b/pkg/plugins/fastly/go.sum new file mode 100644 index 0000000..6e2fa93 --- /dev/null +++ b/pkg/plugins/fastly/go.sum @@ -0,0 +1,213 @@ +github.com/bufbuild/protocompile v0.4.0 h1:LbFKd2XowZvQ/kajzguUp2DC9UEIQhIq77fZZlaQsNA= +github.com/bufbuild/protocompile v0.4.0/go.mod h1:3v93+mbWn/v3xzN+31nwkJfrEpAUwp+BagBSZWx+TP8= +github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= +github.com/fatih/color v1.18.0 h1:S8gINlzdQ840/4pfAwic/ZE0djQEH3wM94VfqLTZcOM= +github.com/fatih/color v1.18.0/go.mod h1:4FelSpRwEGDpQ12mAdzqdOukCy4u8WUtOY6lkT/6HfU= +github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= +github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= +github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= +github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= +github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv5E= +github.com/fxamacker/cbor/v2 v2.7.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ= +github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= +github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4= +github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= +github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= +github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-hclog v1.6.3 h1:Qr2kF+eVWjTiYmU7Y31tYlP1h0q/X3Nl3tPGdaB11/k= +github.com/hashicorp/go-hclog v1.6.3/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M= +github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= +github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= +github.com/hashicorp/go-plugin v1.6.3 h1:xgHB+ZUSYeuJi96WtxEjzi23uh7YQpznjGh0U0UUrwg= +github.com/hashicorp/go-plugin v1.6.3/go.mod h1:MRobyh+Wc/nYy1V4KAXUiYfzxoYhs7V1mlH1Z7iY2h0= +github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= +github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/hashicorp/yamux v0.1.2 h1:XtB8kyFOyHXYVFnwT5C3+Bdo8gArse7j2AQ0DA0Uey8= +github.com/hashicorp/yamux v0.1.2/go.mod h1:C+zze2n6e/7wshOZep2A70/aQU6QBRWJO/G6FT1wIns= +github.com/jhump/protoreflect v1.15.1 h1:HUMERORf3I3ZdX05WaQ6MIpd/NJ434hTp5YiKgfCL6c= +github.com/jhump/protoreflect v1.15.1/go.mod h1:jD/2GMKKE6OqX8qTjhADU1e6DShO+gavG9e0Q693nKo= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= +github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= +github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= +github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= +github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE= +github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8= +github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= +github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= +github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= +github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/oklog/run v1.2.0 h1:O8x3yXwah4A73hJdlrwo/2X6J62gE5qTMusH0dvz60E= +github.com/oklog/run v1.2.0/go.mod h1:mgDbKRSwPhJfesJ4PntqFUbKQRZ50NgmZTSPlFA0YFk= +github.com/opencost/opencost/core v0.0.0-20250805023318-1e1643d24b0e h1:RSXlPL7sB0NSpS55hAExcsKsq0xlEgwZIIZZw346NI8= +github.com/opencost/opencost/core v0.0.0-20250805023318-1e1643d24b0e/go.mod h1:qKHpAsBp1gS3EkWy/HWfy19doPxlzhv9t5CfauZNN/k= +github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc= +github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ= +github.com/pelletier/go-toml/v2 v2.1.1 h1:LWAJwfNvjQZCFIDKWYQaM62NcYeYViCmWIwmOStowAI= +github.com/pelletier/go-toml/v2 v2.1.1/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= +github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= +github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= +github.com/rs/zerolog v1.32.0 h1:keLypqrlIjaFsbmJOBdB/qvyF8KEtCWHwobLp5l/mQ0= +github.com/rs/zerolog v1.32.0/go.mod h1:/7mN4D5sKwJLZQ2b/znpjC3/GQWY/xaDXUM0kKWRHss= +github.com/sagikazarmark/locafero v0.4.0 h1:HApY1R9zGo4DBgr7dqsTH/JJxLTTsOt7u6keLGt6kNQ= +github.com/sagikazarmark/locafero v0.4.0/go.mod h1:Pe1W6UlPYUk/+wc/6KFhbORCfqzgYEpgQ3O5fPuL3H4= +github.com/sagikazarmark/slog-shim v0.1.0 h1:diDBnUNK9N/354PgrxMywXnAwEr1QZcOr6gto+ugjYE= +github.com/sagikazarmark/slog-shim v0.1.0/go.mod h1:SrcSrq8aKtyuqEI1uvTDTK1arOWRIczQRv+GVI1AkeQ= +github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo= +github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0= +github.com/spf13/afero v1.11.0 h1:WJQKhtpdm3v2IzqG8VMqrr6Rf3UYpEF239Jy9wNepM8= +github.com/spf13/afero v1.11.0/go.mod h1:GH9Y3pIexgf1MTIWtNGyogA5MwRIDXGUr+hbWNoBjkY= +github.com/spf13/cast v1.6.0 h1:GEiTHELF+vaR5dhz3VqZfFSzZjYbgeKDpBxQVS4GYJ0= +github.com/spf13/cast v1.6.0/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/viper v1.18.2 h1:LUXCnvUvSM6FXAsj6nnfc8Q2tp1dIgUfY9Kc8GsSOiQ= +github.com/spf13/viper v1.18.2/go.mod h1:EKmWIqdnk5lOcmR72yw6hS+8OPYcwD0jteitLMVB+yk= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= +github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= +github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= +github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= +go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= +go.opentelemetry.io/otel v1.36.0 h1:UumtzIklRBY6cI/lllNZlALOF5nNIzJVb16APdvgTXg= +go.opentelemetry.io/otel v1.36.0/go.mod h1:/TcFMXYjyRNh8khOAO9ybYkqaDBb/70aVwkNML4pP8E= +go.opentelemetry.io/otel/metric v1.36.0 h1:MoWPKVhQvJ+eeXWHFBOPoBOi20jh6Iq2CcCREuTYufE= +go.opentelemetry.io/otel/metric v1.36.0/go.mod h1:zC7Ks+yeyJt4xig9DEw9kuUFe5C3zLbVjV2PzT6qzbs= +go.opentelemetry.io/otel/sdk v1.36.0 h1:b6SYIuLRs88ztox4EyrvRti80uXIFy+Sqzoh9kFULbs= +go.opentelemetry.io/otel/sdk v1.36.0/go.mod h1:+lC+mTgD+MUWfjJubi2vvXWcVxyr9rmlshZni72pXeY= +go.opentelemetry.io/otel/sdk/metric v1.36.0 h1:r0ntwwGosWGaa0CrSt8cuNuTcccMXERFwHX4dThiPis= +go.opentelemetry.io/otel/sdk/metric v1.36.0/go.mod h1:qTNOhFDfKRwX0yXOqJYegL5WRaW376QbB7P4Pb0qva4= +go.opentelemetry.io/otel/trace v1.36.0 h1:ahxWNuqZjpdiFAyrIoQ4GIiAIhxAunQR6MUoKrsNd4w= +go.opentelemetry.io/otel/trace v1.36.0/go.mod h1:gQ+OnDZzrybY4k4seLzPAWNwVBBVlF2szhehOBB/tGA= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/exp v0.0.0-20240222234643-814bf88cf225 h1:LfspQV/FYTatPTr/3HzIcmiUFH7PGP+OQ6mgDYo3yuQ= +golang.org/x/exp v0.0.0-20240222234643-814bf88cf225/go.mod h1:CxmFvTBINI24O/j8iY7H1xHzx2i4OsyguNBmN/uPtqc= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.42.0 h1:jzkYrhi3YQWD6MLBJcsklgQsoAcw89EcZbJw8Z614hs= +golang.org/x/net v0.42.0/go.mod h1:FF1RA5d3u7nAYA4z2TkclSCKh68eSXtiFwcWQpPXdt8= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.34.0 h1:H5Y5sJ2L2JRdyv7ROF1he/lPdvFsd0mJHFw2ThKHxLA= +golang.org/x/sys v0.34.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.27.0 h1:4fGWRpyh641NLlecmyl4LOe6yDdfaYNrGb2zdfo4JV4= +golang.org/x/text v0.27.0/go.mod h1:1D28KMCvyooCX9hBiosv5Tz/+YLxj0j7XhWjpSUF7CU= +golang.org/x/time v0.12.0 h1:ScB/8o8olJvc+CQPWrK3fPZNfh7qgwCrY0zJmoEQLSE= +golang.org/x/time v0.12.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250804133106-a7a43d27e69b h1:zPKJod4w6F1+nRGDI9ubnXYhU9NSWoFAijkHkUXeTK8= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250804133106-a7a43d27e69b/go.mod h1:qQ0YXyHHx3XkvlzUtpXDkS29lDSafHMZBAZDc03LQ3A= +google.golang.org/grpc v1.74.2 h1:WoosgB65DlWVC9FqI82dGsZhWFNBSLjQ84bjROOpMu4= +google.golang.org/grpc v1.74.2/go.mod h1:CtQ+BGjaAIXHs/5YS3i473GqwBBa1zGQNevxdeBEXrM= +google.golang.org/protobuf v1.36.6 h1:z1NpPI8ku2WgiWnf+t9wTPsn6eP1L7ksHUlkfLvd9xY= +google.golang.org/protobuf v1.36.6/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= +gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= +gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +k8s.io/api v0.33.1 h1:tA6Cf3bHnLIrUK4IqEgb2v++/GYUtqiu9sRVk3iBXyw= +k8s.io/api v0.33.1/go.mod h1:87esjTn9DRSRTD4fWMXamiXxJhpOIREjWOSjsW1kEHw= +k8s.io/apimachinery v0.33.1 h1:mzqXWV8tW9Rw4VeW9rEkqvnxj59k1ezDUl20tFK/oM4= +k8s.io/apimachinery v0.33.1/go.mod h1:BHW0YOu7n22fFv/JkYOEfkUYNRN0fj0BlvMFWA7b+SM= +k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= +k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= +k8s.io/utils v0.0.0-20250321185631-1f6e0b77f77e h1:KqK5c/ghOm8xkHYhlodbp6i6+r+ChV2vuAuVRdFbLro= +k8s.io/utils v0.0.0-20250321185631-1f6e0b77f77e/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3 h1:/Rv+M11QRah1itp8VhT6HoVx1Ray9eB4DBr+K+/sCJ8= +sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3/go.mod h1:18nIHnGi6636UCz6m8i4DhaJ65T6EruyzmoQqI2BVDo= +sigs.k8s.io/randfill v0.0.0-20250304075658-069ef1bbf016/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY= +sigs.k8s.io/randfill v1.0.0 h1:JfjMILfT8A6RbawdsK2JXGBR5AQVfd+9TbzrlneTyrU= +sigs.k8s.io/randfill v1.0.0/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY= +sigs.k8s.io/structured-merge-diff/v4 v4.6.0 h1:IUA9nvMmnKWcj5jl84xn+T5MnlZKThmUW1TdblaLVAc= +sigs.k8s.io/structured-merge-diff/v4 v4.6.0/go.mod h1:dDy58f92j70zLsuZVuUX5Wp9vtxXpaZnkPGWeqDfCps= +sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E= +sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY= diff --git a/pkg/plugins/fastly/plugin/fastlyconfig.go b/pkg/plugins/fastly/plugin/fastlyconfig.go new file mode 100644 index 0000000..27b52f0 --- /dev/null +++ b/pkg/plugins/fastly/plugin/fastlyconfig.go @@ -0,0 +1,44 @@ +package plugin + +import "fmt" + +// FastlyConfig represents the configuration needed to authenticate with Fastly API +type FastlyConfig struct { + // FastlyAPIToken is the API token for authenticating with Fastly + FastlyAPIToken string `json:"fastly_api_token"` + + // FastlyAccountID is the account ID for the Fastly account (optional) + FastlyAccountID string `json:"fastly_account_id,omitempty"` + + // LogLevel controls the verbosity of logging + LogLevel string `json:"log_level,omitempty"` + + // RateLimitPerSecond controls API request rate limiting + RateLimitPerSecond float64 `json:"rate_limit_per_second,omitempty"` + + // EnableUsageDetail enables fetching detailed usage data for cost allocation + EnableUsageDetail bool `json:"enable_usage_detail,omitempty"` + + // ServiceFilters limits cost retrieval to specific services + ServiceFilters []string `json:"service_filters,omitempty"` + + // ExcludeServices excludes specific services from cost retrieval + ExcludeServices []string `json:"exclude_services,omitempty"` +} + +// Validate checks if the configuration is valid +func (c *FastlyConfig) Validate() error { + if c.FastlyAPIToken == "" { + return fmt.Errorf("fastly_api_token is required") + } + + if c.LogLevel == "" { + c.LogLevel = "info" + } + + if c.RateLimitPerSecond <= 0 { + c.RateLimitPerSecond = 1.0 // Default to 1 request per second + } + + return nil +} diff --git a/pkg/plugins/fastly/plugin/services.go b/pkg/plugins/fastly/plugin/services.go new file mode 100644 index 0000000..8ed398c --- /dev/null +++ b/pkg/plugins/fastly/plugin/services.go @@ -0,0 +1,314 @@ +package plugin + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "time" + + "github.com/opencost/opencost/core/pkg/log" +) + +// FastlyClient provides methods to interact with Fastly API +type FastlyClient struct { + APIToken string + HTTPClient *http.Client + BaseURL string +} + +// NewFastlyClient creates a new Fastly API client +func NewFastlyClient(apiToken string) *FastlyClient { + return &FastlyClient{ + APIToken: apiToken, + HTTPClient: &http.Client{ + Timeout: 30 * time.Second, + }, + BaseURL: "https://api.fastly.com", + } +} + +// FetchInvoices retrieves invoices for the specified time range +func (c *FastlyClient) FetchInvoices(ctx context.Context, start, end time.Time) ([]Invoice, error) { + allInvoices := []Invoice{} + cursor := "" + + for { + url := fmt.Sprintf("%s/billing/v3/invoices?from=%s&to=%s&limit=100", + c.BaseURL, + start.Format("2006-01-02"), + end.Format("2006-01-02")) + + if cursor != "" { + url += "&cursor=" + cursor + } + + invoices, meta, err := c.fetchInvoicePage(ctx, url) + if err != nil { + return nil, err + } + + allInvoices = append(allInvoices, invoices...) + + if meta.NextCursor == "" { + break + } + cursor = meta.NextCursor + } + + return allInvoices, nil +} + +func (c *FastlyClient) fetchInvoicePage(ctx context.Context, url string) ([]Invoice, Meta, error) { + req, err := http.NewRequestWithContext(ctx, "GET", url, nil) + if err != nil { + return nil, Meta{}, err + } + + req.Header.Add("Fastly-Key", c.APIToken) + req.Header.Add("Accept", "application/json") + + resp, err := c.HTTPClient.Do(req) + if err != nil { + return nil, Meta{}, err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return nil, Meta{}, fmt.Errorf("API request failed with status %d", resp.StatusCode) + } + + var response InvoiceResponse + if err := json.NewDecoder(resp.Body).Decode(&response); err != nil { + return nil, Meta{}, err + } + + return response.Data, response.Meta, nil +} + +// FetchUsage retrieves usage data for the specified time range +func (c *FastlyClient) FetchUsage(ctx context.Context, start, end time.Time, serviceID string) ([]Usage, error) { + allUsage := []Usage{} + + // Fastly usage API typically provides hourly granularity + current := start + for current.Before(end) { + nextHour := current.Add(time.Hour) + if nextHour.After(end) { + nextHour = end + } + + usage, err := c.fetchUsageForPeriod(ctx, current, nextHour, serviceID) + if err != nil { + log.Warnf("error fetching usage for period %v-%v: %v", current, nextHour, err) + // Continue with next period instead of failing completely + current = nextHour + continue + } + + allUsage = append(allUsage, usage...) + current = nextHour + } + + return allUsage, nil +} + +func (c *FastlyClient) fetchUsageForPeriod(ctx context.Context, start, end time.Time, serviceID string) ([]Usage, error) { + url := fmt.Sprintf("%s/stats/usage_by_service?from=%s&to=%s&by=hour", + c.BaseURL, + start.Format("2006-01-02T15:04:05Z"), + end.Format("2006-01-02T15:04:05Z")) + + if serviceID != "" { + url += "&service_id=" + serviceID + } + + req, err := http.NewRequestWithContext(ctx, "GET", url, nil) + if err != nil { + return nil, err + } + + req.Header.Add("Fastly-Key", c.APIToken) + req.Header.Add("Accept", "application/json") + + resp, err := c.HTTPClient.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("usage API request failed with status %d", resp.StatusCode) + } + + var response UsageResponse + if err := json.NewDecoder(resp.Body).Decode(&response); err != nil { + return nil, err + } + + return response.Data, nil +} + +// FetchServices retrieves all services for the account +func (c *FastlyClient) FetchServices(ctx context.Context) ([]ServiceDetail, error) { + url := fmt.Sprintf("%s/service", c.BaseURL) + + req, err := http.NewRequestWithContext(ctx, "GET", url, nil) + if err != nil { + return nil, err + } + + req.Header.Add("Fastly-Key", c.APIToken) + req.Header.Add("Accept", "application/json") + + resp, err := c.HTTPClient.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("services API request failed with status %d", resp.StatusCode) + } + + var services []ServiceDetail + if err := json.NewDecoder(resp.Body).Decode(&services); err != nil { + return nil, err + } + + return services, nil +} + +// CostCalculator provides methods for calculating costs from usage data +type CostCalculator struct { + // Pricing rates can be configured here + BandwidthRate float32 // Cost per GB + RequestRate float32 // Cost per 10k requests + ComputeRate float32 // Cost per compute second + ShieldingRate float32 // Cost per GB for origin shielding +} + +// NewCostCalculator creates a new cost calculator with default rates +func NewCostCalculator() *CostCalculator { + return &CostCalculator{ + BandwidthRate: 0.05, // $0.05 per GB + RequestRate: 0.001, // $0.001 per 10k requests + ComputeRate: 0.0001, // $0.0001 per compute second + ShieldingRate: 0.02, // $0.02 per GB for shielding + } +} + +// CalculateUsageCost calculates cost from usage metrics +func (calc *CostCalculator) CalculateUsageCost(usage Usage) []CostAllocation { + allocations := []CostAllocation{} + + for region, metrics := range usage.Usage { + // Bandwidth cost + if metrics.Bandwidth > 0 { + bandwidthGB := float32(metrics.Bandwidth) / (1024 * 1024 * 1024) + allocations = append(allocations, CostAllocation{ + ServiceID: usage.ServiceID, + ServiceName: usage.ServiceName, + Region: region, + CostType: "bandwidth", + Amount: bandwidthGB * calc.BandwidthRate, + Units: bandwidthGB, + UnitType: "GB", + StartTime: usage.StartTime, + EndTime: usage.EndTime, + }) + } + + // Request cost + if metrics.Requests > 0 { + requestsIn10k := float32(metrics.Requests) / 10000 + allocations = append(allocations, CostAllocation{ + ServiceID: usage.ServiceID, + ServiceName: usage.ServiceName, + Region: region, + CostType: "requests", + Amount: requestsIn10k * calc.RequestRate, + Units: requestsIn10k, + UnitType: "10k_requests", + StartTime: usage.StartTime, + EndTime: usage.EndTime, + }) + } + + // Compute cost + if metrics.ComputeDuration > 0 { + computeSeconds := float32(metrics.ComputeDuration) / 1000 + allocations = append(allocations, CostAllocation{ + ServiceID: usage.ServiceID, + ServiceName: usage.ServiceName, + Region: region, + CostType: "compute", + Amount: computeSeconds * calc.ComputeRate, + Units: computeSeconds, + UnitType: "compute_seconds", + StartTime: usage.StartTime, + EndTime: usage.EndTime, + }) + } + + // Origin shielding cost (cached bandwidth) + if metrics.BandwidthCached > 0 { + cachedGB := float32(metrics.BandwidthCached) / (1024 * 1024 * 1024) + allocations = append(allocations, CostAllocation{ + ServiceID: usage.ServiceID, + ServiceName: usage.ServiceName, + Region: region, + CostType: "origin_shielding", + Amount: cachedGB * calc.ShieldingRate, + Units: cachedGB, + UnitType: "GB", + StartTime: usage.StartTime, + EndTime: usage.EndTime, + }) + } + } + + return allocations +} + +// ServiceMapper provides methods for enriching cost data with service information +type ServiceMapper struct { + services map[string]ServiceDetail +} + +// NewServiceMapper creates a new service mapper +func NewServiceMapper(services []ServiceDetail) *ServiceMapper { + serviceMap := make(map[string]ServiceDetail) + for _, service := range services { + serviceMap[service.ID] = service + } + + return &ServiceMapper{ + services: serviceMap, + } +} + +// GetServiceName returns the service name for a given service ID +func (sm *ServiceMapper) GetServiceName(serviceID string) string { + if service, ok := sm.services[serviceID]; ok { + return service.Name + } + return serviceID +} + +// GetServiceAttributes returns the attributes for a given service ID +func (sm *ServiceMapper) GetServiceAttributes(serviceID string) map[string]string { + if service, ok := sm.services[serviceID]; ok { + return service.Attributes + } + return map[string]string{} +} + +// GetServices returns all services +func (sm *ServiceMapper) GetServices() []ServiceDetail { + services := make([]ServiceDetail, 0, len(sm.services)) + for _, service := range sm.services { + services = append(services, service) + } + return services +} diff --git a/pkg/plugins/fastly/plugin/types.go b/pkg/plugins/fastly/plugin/types.go new file mode 100644 index 0000000..7505d4f --- /dev/null +++ b/pkg/plugins/fastly/plugin/types.go @@ -0,0 +1,173 @@ +package plugin + +import ( + "fmt" + "time" +) + +// InvoiceResponse represents the response from Fastly's invoice API +type InvoiceResponse struct { + Data []Invoice `json:"data"` + Meta Meta `json:"meta"` +} + +// Invoice represents a Fastly invoice +type Invoice struct { + ID string `json:"id"` + CustomerID string `json:"customer_id"` + InvoiceNumber string `json:"invoice_number"` + State string `json:"state"` + Total float32 `json:"total"` + Region string `json:"region"` + BillingStartDate time.Time `json:"billing_start_date"` + BillingEndDate time.Time `json:"billing_end_date"` + IssuedOn time.Time `json:"issued_on"` + DueOn time.Time `json:"due_on"` + Currency string `json:"currency_code"` + LineItems []LineItem `json:"line_items"` +} + +// LineItem represents a line item in a Fastly invoice +type LineItem struct { + ID string `json:"id"` + Description string `json:"description"` + Amount float32 `json:"amount"` + Rate float32 `json:"rate"` + Units float32 `json:"units"` + UnitType string `json:"unit_type"` + ServiceType string `json:"service_type"` + ServiceID string `json:"service_id"` + Total float32 `json:"total"` + Region string `json:"region"` + ProductLine string `json:"product_line"` +} + +// Meta represents pagination metadata +type Meta struct { + NextCursor string `json:"next_cursor"` + Limit int `json:"limit"` +} + +// GetDaysInBillingPeriod calculates the number of days in the billing period +func (i *Invoice) GetDaysInBillingPeriod() int { + return int(i.BillingEndDate.Sub(i.BillingStartDate).Hours() / 24) +} + +// UsageResponse represents the response from Fastly's usage API +type UsageResponse struct { + Data []Usage `json:"data"` + Meta Meta `json:"meta"` +} + +// Usage represents usage data from Fastly +type Usage struct { + ServiceID string `json:"service_id"` + ServiceName string `json:"service_name"` + StartTime time.Time `json:"start_time"` + EndTime time.Time `json:"end_time"` + Region string `json:"region"` + Usage map[string]UsageMetric `json:"usage"` +} + +// UsageMetric represents a specific usage metric +type UsageMetric struct { + Requests int64 `json:"requests"` + Bandwidth int64 `json:"bandwidth"` + BandwidthCached int64 `json:"bandwidth_cached"` + ComputeRequests int64 `json:"compute_requests"` + ComputeDuration float64 `json:"compute_duration_ms"` +} + +// ServiceDetail represents detailed information about a Fastly service +type ServiceDetail struct { + ID string `json:"id"` + Name string `json:"name"` + CustomerID string `json:"customer_id"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` + ActiveVersion int `json:"active_version"` + Type string `json:"type"` + Attributes map[string]string `json:"attributes"` +} + +// CostAllocation helps with allocating costs across different services and regions +type CostAllocation struct { + ServiceID string + ServiceName string + Region string + CostType string + Amount float32 + Units float32 + UnitType string + StartTime time.Time + EndTime time.Time +} + +// GetCostCategory returns the cost category based on the service type +func GetCostCategory(serviceType string) string { + switch serviceType { + case "cdn_bandwidth", "cdn_requests", "origin_shielding": + return "networking" + case "compute", "compute_requests": + return "compute" + case "logs", "real_time_analytics": + return "observability" + case "waf", "ddos_protection": + return "security" + default: + return "other" + } +} + +// IsWithinTimeRange checks if an invoice covers the specified time range +func (i *Invoice) IsWithinTimeRange(start, end time.Time) bool { + // Check if invoice period overlaps with requested time range + return !(i.BillingEndDate.Before(start) || i.BillingStartDate.After(end)) +} + +// GetProportionalCost calculates the proportional cost for a specific time window +func (i *Invoice) GetProportionalCost(windowStart, windowEnd time.Time) float32 { + // Calculate overlap between invoice period and window + overlapStart := i.BillingStartDate + if windowStart.After(overlapStart) { + overlapStart = windowStart + } + + overlapEnd := i.BillingEndDate + if windowEnd.Before(overlapEnd) { + overlapEnd = windowEnd + } + + // If no overlap, return 0 + if overlapEnd.Before(overlapStart) || overlapEnd.Equal(overlapStart) { + return 0 + } + + // Calculate proportion + overlapDays := overlapEnd.Sub(overlapStart).Hours() / 24.0 + totalDays := float64(i.GetDaysInBillingPeriod()) + + if totalDays == 0 { + return 0 + } + + proportion := float32(overlapDays / totalDays) + return i.Total * proportion +} + +// Validate checks if the invoice data is valid +func (i *Invoice) Validate() error { + if i.ID == "" { + return fmt.Errorf("invoice ID is empty") + } + + if i.BillingEndDate.Before(i.BillingStartDate) { + return fmt.Errorf("billing end date is before start date") + } + + if i.Total < 0 { + return fmt.Errorf("invoice total is negative") + } + + return nil +} diff --git a/pkg/plugins/fastly/tests/fastly_test.go b/pkg/plugins/fastly/tests/fastly_test.go new file mode 100644 index 0000000..2bda438 --- /dev/null +++ b/pkg/plugins/fastly/tests/fastly_test.go @@ -0,0 +1,236 @@ +package tests + +import ( + "os" + "testing" + "time" + + fastlyplugin "github.com/opencost/opencost-plugins/pkg/plugins/fastly/plugin" + "github.com/opencost/opencost/core/pkg/log" + "github.com/opencost/opencost/core/pkg/model/pb" + "github.com/opencost/opencost/core/pkg/opencost" + "github.com/opencost/opencost/core/pkg/util/timeutil" + "golang.org/x/time/rate" + "google.golang.org/protobuf/types/known/timestamppb" +) + +func TestGetCustomCosts(t *testing.T) { + // Read necessary env vars + fastlyAPIToken := os.Getenv("FASTLY_API_TOKEN") + + if fastlyAPIToken == "" { + log.Warnf("FASTLY_API_TOKEN undefined, skipping test") + t.Skip() + return + } + + config := fastlyplugin.FastlyConfig{ + FastlyAPIToken: fastlyAPIToken, + LogLevel: "debug", + RateLimitPerSecond: 1.0, + } + + // Validate config + if err := config.Validate(); err != nil { + t.Fatalf("config validation failed: %v", err) + } + + // Test rate limiter creation + rateLimiter := rate.NewLimiter(rate.Limit(config.RateLimitPerSecond), 1) + + // Verify rate limiter is configured correctly + if rateLimiter.Limit() != rate.Limit(config.RateLimitPerSecond) { + t.Errorf("Rate limiter limit mismatch: got %v, want %v", rateLimiter.Limit(), config.RateLimitPerSecond) + } + + t.Log("Config validation and rate limiter setup passed") +} + +func TestFastlyConfig(t *testing.T) { + tests := []struct { + name string + config fastlyplugin.FastlyConfig + wantErr bool + }{ + { + name: "valid config", + config: fastlyplugin.FastlyConfig{ + FastlyAPIToken: "test-token", + LogLevel: "info", + }, + wantErr: false, + }, + { + name: "missing API token", + config: fastlyplugin.FastlyConfig{ + LogLevel: "info", + }, + wantErr: true, + }, + { + name: "default log level", + config: fastlyplugin.FastlyConfig{ + FastlyAPIToken: "test-token", + }, + wantErr: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := tt.config.Validate() + if (err != nil) != tt.wantErr { + t.Errorf("Validate() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} + +func TestInvoiceProcessing(t *testing.T) { + // Test invoice processing logic + invoice := fastlyplugin.Invoice{ + ID: "inv-123", + CustomerID: "cust-456", + BillingStartDate: time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC), + BillingEndDate: time.Date(2024, 1, 31, 0, 0, 0, 0, time.UTC), + Total: 1000.0, + Currency: "USD", + LineItems: []fastlyplugin.LineItem{ + { + ID: "li-1", + Description: "CDN Bandwidth", + ServiceType: "cdn_bandwidth", + Amount: 500.0, + Rate: 0.05, + Units: 10000, + UnitType: "GB", + Total: 500.0, + Region: "us-east-1", + }, + { + ID: "li-2", + Description: "CDN Requests", + ServiceType: "cdn_requests", + Amount: 300.0, + Rate: 0.001, + Units: 300000, + UnitType: "requests", + Total: 300.0, + Region: "us-east-1", + }, + }, + } + + days := invoice.GetDaysInBillingPeriod() + if days != 30 { + t.Errorf("Expected 30 days, got %d", days) + } + + // Test daily rate calculation + for _, item := range invoice.LineItems { + dailyRate := item.Total / float32(days) + if dailyRate <= 0 { + t.Errorf("Daily rate should be positive, got %f", dailyRate) + } + } +} + +func TestCostResponseStructure(t *testing.T) { + // Test the structure of a cost response + windowStart := time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC) + windowEnd := time.Date(2024, 1, 2, 0, 0, 0, 0, time.UTC) + + resp := &pb.CustomCostResponse{ + Metadata: map[string]string{"api_version": "v1"}, + CostSource: "infrastructure", + Domain: "fastly", + Version: "v1", + Currency: "USD", + Start: timestamppb.New(windowStart), + End: timestamppb.New(windowEnd), + Errors: []string{}, + Costs: []*pb.CustomCost{ + { + AccountName: "cust-456", + ChargeCategory: "usage", + Description: "CDN Bandwidth", + ResourceName: "cdn_bandwidth", + ResourceType: "cdn", + Id: "inv-123-cdn_bandwidth-2024-01-01", + BilledCost: 16.67, + ListCost: 16.67, + ListUnitPrice: 0.05, + UsageQuantity: 333.33, + UsageUnit: "GB", + Labels: map[string]string{ + "service_type": "cdn_bandwidth", + "region": "us-east-1", + }, + Zone: "us-east-1", + }, + }, + } + + // Validate response structure + if resp.Domain != "fastly" { + t.Errorf("Expected domain 'fastly', got %s", resp.Domain) + } + + if len(resp.Costs) != 1 { + t.Errorf("Expected 1 cost, got %d", len(resp.Costs)) + } + + if resp.Costs[0].ResourceType != "cdn" { + t.Errorf("Expected resource type 'cdn', got %s", resp.Costs[0].ResourceType) + } +} + +func TestWindowProcessing(t *testing.T) { + // Test window processing for different resolutions + tests := []struct { + name string + start time.Time + end time.Time + resolution time.Duration + expected int + }{ + { + name: "daily resolution", + start: time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC), + end: time.Date(2024, 1, 8, 0, 0, 0, 0, time.UTC), + resolution: timeutil.Day, + expected: 7, + }, + { + name: "hourly resolution", + start: time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC), + end: time.Date(2024, 1, 1, 6, 0, 0, 0, time.UTC), + resolution: time.Hour, + expected: 6, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // Test window calculation + windows, err := opencost.GetWindows(tt.start, tt.end, tt.resolution) + if err != nil { + t.Fatalf("Failed to get windows: %v", err) + } + + if len(windows) != tt.expected { + t.Errorf("Expected %d windows, got %d", tt.expected, len(windows)) + } + + // Verify each window has the correct duration + for i, window := range windows { + duration := window.End().Sub(*window.Start()) + if duration != tt.resolution { + t.Errorf("Window %d has incorrect duration: got %v, want %v", i, duration, tt.resolution) + } + } + + t.Logf("Test case %s: got %d windows as expected", tt.name, len(windows)) + }) + } +} From 9f9d237c5edf3645c1e1a7433a149074e7972886 Mon Sep 17 00:00:00 2001 From: Sparsh Date: Tue, 5 Aug 2025 16:37:55 +0530 Subject: [PATCH 2/8] Add Fastly cost integration plugin for OpenCost This commit implements a Fastly plugin for OpenCost that integrates with the Fastly billing API to retrieve and display cloud costs. The plugin: 1. Fetches invoice data from Fastly's API (both historical and month-to-date) 2. Converts Fastly billing items to OpenCost's CustomCost model 3. Implements proper rate limiting to respect Fastly API constraints 4. Caches invoice data to minimize redundant API calls 5. Prorates costs across time windows when invoice periods don't align with requested time ranges The plugin allows OpenCost users to monitor and analyze their Fastly CDN costs alongside other cloud expenses. Signed-off-by: Sparsh --- pkg/plugins/fastly/cmd/main/main.go | 464 ++++++++++-------- pkg/plugins/fastly/cmd/main/main_test.go | 167 +++++++ pkg/plugins/fastly/cmd/validator/main/main.go | 131 ++--- pkg/plugins/fastly/fastly-config-sample.json | 9 - .../fastly/fastlyplugin/fastlyconfig.go | 6 + pkg/plugins/fastly/fastlyplugin/invoices.go | 46 ++ pkg/plugins/fastly/go.mod | 30 +- pkg/plugins/fastly/go.sum | 70 ++- pkg/plugins/fastly/plugin/fastlyconfig.go | 44 -- pkg/plugins/fastly/plugin/services.go | 314 ------------ pkg/plugins/fastly/plugin/types.go | 173 ------- pkg/plugins/fastly/tests/fastly_test.go | 347 ++++++------- 12 files changed, 739 insertions(+), 1062 deletions(-) delete mode 100644 pkg/plugins/fastly/fastly-config-sample.json create mode 100644 pkg/plugins/fastly/fastlyplugin/fastlyconfig.go create mode 100644 pkg/plugins/fastly/fastlyplugin/invoices.go delete mode 100644 pkg/plugins/fastly/plugin/fastlyconfig.go delete mode 100644 pkg/plugins/fastly/plugin/services.go delete mode 100644 pkg/plugins/fastly/plugin/types.go diff --git a/pkg/plugins/fastly/cmd/main/main.go b/pkg/plugins/fastly/cmd/main/main.go index c674c90..195f882 100644 --- a/pkg/plugins/fastly/cmd/main/main.go +++ b/pkg/plugins/fastly/cmd/main/main.go @@ -4,12 +4,16 @@ import ( "context" "encoding/json" "fmt" + "io" + "net/http" + "net/url" "os" + "strings" + "sync" "time" "github.com/hashicorp/go-plugin" - commonconfig "github.com/opencost/opencost-plugins/pkg/common/config" - fastlyplugin "github.com/opencost/opencost-plugins/pkg/plugins/fastly/plugin" + "github.com/opencost/opencost-plugins/pkg/plugins/fastly/fastlyplugin" "github.com/opencost/opencost/core/pkg/log" "github.com/opencost/opencost/core/pkg/model/pb" "github.com/opencost/opencost/core/pkg/opencost" @@ -18,39 +22,55 @@ import ( "google.golang.org/protobuf/types/known/timestamppb" ) +const ( + fastlyAPIBaseURL = "https://api.fastly.com" +) + // handshakeConfigs are used to just do a basic handshake between -// a plugin and host. If the handshake fails, a user-friendly error is shown. +// a plugin and host. If the handshake fails, a user friendly error is shown. +// This prevents users from executing bad plugins or executing a plugin +// directory. It is a UX feature, not a security feature. var handshakeConfig = plugin.HandshakeConfig{ ProtocolVersion: 1, MagicCookieKey: "PLUGIN_NAME", MagicCookieValue: "fastly", } -// FastlyCostSource implements the CustomCostSource interface +// Implementation of CustomCostSource type FastlyCostSource struct { - config *fastlyplugin.FastlyConfig - client *fastlyplugin.FastlyClient - rateLimiter *rate.Limiter - ctx context.Context - costCalculator *fastlyplugin.CostCalculator - serviceMapper *fastlyplugin.ServiceMapper + apiKey string + httpClient *http.Client + rateLimiter *rate.Limiter + invoiceCache map[string][]fastlyplugin.Invoice + invoiceCacheMux sync.Mutex } -// GetCustomCosts retrieves custom costs from Fastly for the given time window func (f *FastlyCostSource) GetCustomCosts(req *pb.CustomCostRequest) []*pb.CustomCostResponse { results := []*pb.CustomCostResponse{} - // Get windows based on resolution targets, err := opencost.GetWindows(req.Start.AsTime(), req.End.AsTime(), req.Resolution.AsDuration()) + + // Fetch all invoices at once for the entire period + startTime := req.Start.AsTime() + endTime := req.End.AsTime() + allInvoices, err := f.getInvoicesForPeriod(&startTime, &endTime) + if err != nil { log.Errorf("error getting windows: %v", err) - errResp := pb.CustomCostResponse{ - Errors: []string{fmt.Sprintf("error getting windows: %v", err)}, - } + startTime := req.Start.AsTime() + endTime := req.End.AsTime() + errResp := boilerplateFastlyCustomCost(opencost.NewWindow(&startTime, &endTime)) + errResp.Errors = []string{fmt.Sprintf("error getting windows: %v", err)} results = append(results, &errResp) return results } + // Store in cache + cacheKey := fmt.Sprintf("%s-%s", req.Start.AsTime().Format("2006-01"), req.End.AsTime().Format("2006-01")) + f.invoiceCacheMux.Lock() + f.invoiceCache[cacheKey] = allInvoices + f.invoiceCacheMux.Unlock() + for _, target := range targets { // Skip future windows if target.Start().After(time.Now().UTC()) { @@ -67,243 +87,277 @@ func (f *FastlyCostSource) GetCustomCosts(req *pb.CustomCostRequest) []*pb.Custo } func (f *FastlyCostSource) getFastlyCostsForWindow(window opencost.Window) *pb.CustomCostResponse { - ccResp := &pb.CustomCostResponse{ - Metadata: map[string]string{"api_version": "v1"}, - CostSource: "infrastructure", - Domain: "fastly", - Version: "v1", - Currency: "USD", - Start: timestamppb.New(*window.Start()), - End: timestamppb.New(*window.End()), - Errors: []string{}, - Costs: []*pb.CustomCost{}, - } + ccResp := boilerplateFastlyCustomCost(window) + costs := []*pb.CustomCost{} - // Rate limit check - if err := f.rateLimiter.Wait(f.ctx); err != nil { - ccResp.Errors = append(ccResp.Errors, fmt.Sprintf("rate limiter error: %v", err)) - return ccResp + // Get invoices for the window period + invoices, err := f.getInvoicesForPeriod(window.Start(), window.End()) + if err != nil { + log.Errorf("error fetching invoices: %v", err) + ccResp.Errors = append(ccResp.Errors, err.Error()) + return &ccResp } - // Fetch invoices from Fastly API - invoices, err := f.client.FetchInvoices(f.ctx, *window.Start(), *window.End()) - if err != nil { - ccResp.Errors = append(ccResp.Errors, fmt.Sprintf("error fetching invoices: %v", err)) - return ccResp + // Convert invoices to custom costs + for _, invoice := range invoices { + invoiceCosts := f.convertInvoiceToCosts(invoice, window) + costs = append(costs, invoiceCosts...) } - // Process invoices into costs - costs := f.processInvoices(invoices, window) + ccResp.Costs = costs + return &ccResp +} + +func (f *FastlyCostSource) getInvoicesForPeriod(start, end *time.Time) ([]fastlyplugin.Invoice, error) { + allInvoices := []fastlyplugin.Invoice{} + cursor := "" + hasMore := true + + for hasMore { + // Rate limiting + if f.rateLimiter.Tokens() < 1.0 { + log.Infof("fastly rate limit reached. holding request until rate capacity is back") + } + err := f.rateLimiter.WaitN(context.TODO(), 1) + if err != nil { + return nil, fmt.Errorf("error waiting on rate limiter: %v", err) + } + + // Build request URL + reqURL := fmt.Sprintf("%s/billing/v3/invoices", fastlyAPIBaseURL) + params := url.Values{} + params.Add("billing_start_date", start.Format("2006-01-02")) + params.Add("billing_end_date", end.Format("2006-01-02")) + params.Add("limit", "200") + if cursor != "" { + params.Add("cursor", cursor) + } + reqURL = fmt.Sprintf("%s?%s", reqURL, params.Encode()) + + // Make request + req, err := http.NewRequest("GET", reqURL, nil) + if err != nil { + return nil, fmt.Errorf("error creating request: %v", err) + } + req.Header.Set("Accept", "application/json") + req.Header.Set("Fastly-Key", f.apiKey) - // Optionally fetch usage data for more detailed cost allocation - if f.config.EnableUsageDetail { - usageCosts, err := f.fetchAndProcessUsage(window) + resp, err := f.httpClient.Do(req) if err != nil { - log.Warnf("error fetching usage details: %v", err) - // Continue with invoice-based costs only + return nil, fmt.Errorf("error making request: %v", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + body, _ := io.ReadAll(resp.Body) + return nil, fmt.Errorf("API request failed with status %d: %s", resp.StatusCode, string(body)) + } + + // Parse response + var invoiceResp fastlyplugin.InvoiceListResponse + if err := json.NewDecoder(resp.Body).Decode(&invoiceResp); err != nil { + return nil, fmt.Errorf("error decoding response: %v", err) + } + + allInvoices = append(allInvoices, invoiceResp.Data...) + + // Check for more pages + if invoiceResp.Meta.NextCursor == "" { + hasMore = false } else { - costs = f.reconcileCosts(costs, usageCosts) + cursor = invoiceResp.Meta.NextCursor } } - ccResp.Costs = costs - return ccResp + // Also get month-to-date if the window includes current month + now := time.Now() + if start.Before(now) && end.After(time.Date(now.Year(), now.Month(), 1, 0, 0, 0, 0, time.UTC)) { + mtdInvoice, err := f.getMonthToDateInvoice() + if err != nil { + log.Warnf("error fetching month-to-date invoice: %v", err) + } else if mtdInvoice != nil { + allInvoices = append(allInvoices, *mtdInvoice) + } + } + + return allInvoices, nil } -func (f *FastlyCostSource) processInvoices(invoices []fastlyplugin.Invoice, window opencost.Window) []*pb.CustomCost { - costs := []*pb.CustomCost{} +func (f *FastlyCostSource) getMonthToDateInvoice() (*fastlyplugin.Invoice, error) { + // Rate limiting + if f.rateLimiter.Tokens() < 1.0 { + log.Infof("fastly rate limit reached. holding request until rate capacity is back") + } + err := f.rateLimiter.WaitN(context.TODO(), 1) + if err != nil { + return nil, fmt.Errorf("error waiting on rate limiter: %v", err) + } - for _, invoice := range invoices { - // Process each line item in the invoice - for _, lineItem := range invoice.LineItems { - cost := f.createCostFromLineItem(invoice, lineItem, window) - if cost != nil { - costs = append(costs, cost) - } - } + reqURL := fmt.Sprintf("%s/billing/v3/invoices/month-to-date", fastlyAPIBaseURL) + req, err := http.NewRequest("GET", reqURL, nil) + if err != nil { + return nil, fmt.Errorf("error creating request: %v", err) } + req.Header.Set("Accept", "application/json") + req.Header.Set("Fastly-Key", f.apiKey) - return costs + resp, err := f.httpClient.Do(req) + if err != nil { + return nil, fmt.Errorf("error making request: %v", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + body, _ := io.ReadAll(resp.Body) + return nil, fmt.Errorf("API request failed with status %d: %s", resp.StatusCode, string(body)) + } + + var invoice fastlyplugin.Invoice + if err := json.NewDecoder(resp.Body).Decode(&invoice); err != nil { + return nil, fmt.Errorf("error decoding response: %v", err) + } + + return &invoice, nil } -func (f *FastlyCostSource) fetchAndProcessUsage(window opencost.Window) ([]*pb.CustomCost, error) { +func (f *FastlyCostSource) convertInvoiceToCosts(invoice fastlyplugin.Invoice, window opencost.Window) []*pb.CustomCost { costs := []*pb.CustomCost{} - // Fetch all services if not already cached - if f.serviceMapper == nil { - services, err := f.client.FetchServices(f.ctx) - if err != nil { - return nil, fmt.Errorf("error fetching services: %v", err) - } - f.serviceMapper = fastlyplugin.NewServiceMapper(services) + // Parse invoice dates + startDate, err := fastlyplugin.ParseFastlyDate(invoice.BillingStartDate) + if err != nil { + log.Errorf("error parsing billing start date: %v", err) + return costs + } + endDate, err := fastlyplugin.ParseFastlyDate(invoice.BillingEndDate) + if err != nil { + log.Errorf("error parsing billing end date: %v", err) + return costs } - // Get service IDs to process - serviceIDs := []string{} - for _, service := range f.serviceMapper.GetServices() { - // Apply filters if configured - if len(f.config.ServiceFilters) > 0 { - found := false - for _, filter := range f.config.ServiceFilters { - if service.ID == filter { - found = true - break - } - } - if !found { - continue - } - } + // Check if invoice overlaps with window + invoiceStart := startDate + invoiceEnd := endDate + windowStart := *window.Start() + windowEnd := *window.End() - // Apply exclusions if configured - excluded := false - for _, exclude := range f.config.ExcludeServices { - if service.ID == exclude { - excluded = true - break - } - } - if excluded { - continue - } + if invoiceEnd.Before(windowStart) || invoiceStart.After(windowEnd) { + return costs + } - serviceIDs = append(serviceIDs, service.ID) + // Calculate the overlap duration for prorating + overlapStart := invoiceStart + if windowStart.After(invoiceStart) { + overlapStart = windowStart + } + overlapEnd := invoiceEnd + if windowEnd.Before(invoiceEnd) { + overlapEnd = windowEnd } - // Fetch usage for each service - for _, serviceID := range serviceIDs { - usage, err := f.client.FetchUsage(f.ctx, *window.Start(), *window.End(), serviceID) - if err != nil { - log.Warnf("error fetching usage for service %s: %v", serviceID, err) + overlapHours := overlapEnd.Sub(overlapStart).Hours() + totalInvoiceHours := invoiceEnd.Sub(invoiceStart).Hours() + prorateRatio := float32(1.0) + if totalInvoiceHours > 0 { + prorateRatio = float32(overlapHours / totalInvoiceHours) + } + + // Convert each line item to a cost + for _, item := range invoice.TransactionLineItems { + // Skip zero-amount items + if item.Amount == 0 { continue } - // Calculate costs from usage - for _, u := range usage { - allocations := f.costCalculator.CalculateUsageCost(u) - for _, alloc := range allocations { - cost := f.createCostFromAllocation(alloc) - costs = append(costs, cost) - } + // Create provider ID + providerID := fmt.Sprintf("%s/%s/%s", invoice.InvoiceID, item.ProductName, item.UsageType) + + // Prorate the cost based on window overlap + billedCost := float32(item.Amount) * prorateRatio + usageQuantity := float32(item.Units) * prorateRatio + + cost := &pb.CustomCost{ + Zone: item.Region, + AccountName: invoice.CustomerID, + ChargeCategory: "usage", + Description: item.Description, + ResourceName: item.UsageType, + ResourceType: item.ProductGroup, + Id: invoice.InvoiceID, + ProviderId: providerID, + Labels: map[string]string{ + "product_line": item.ProductLine, + "product_name": item.ProductName, + "credit_coupon_code": item.CreditCouponCode, + }, + ListCost: billedCost, + ListUnitPrice: float32(item.Rate), + BilledCost: billedCost, + UsageQuantity: usageQuantity, + UsageUnit: getUsageUnit(item.UsageType), } + + costs = append(costs, cost) } - return costs, nil + return costs } -func (f *FastlyCostSource) createCostFromAllocation(alloc fastlyplugin.CostAllocation) *pb.CustomCost { - serviceName := f.serviceMapper.GetServiceName(alloc.ServiceID) - attrs := f.serviceMapper.GetServiceAttributes(alloc.ServiceID) - - return &pb.CustomCost{ - AccountName: f.config.FastlyAccountID, - ChargeCategory: "usage", - Description: fmt.Sprintf("%s - %s", serviceName, alloc.CostType), - ResourceName: alloc.CostType, - ResourceType: "cdn", - Id: fmt.Sprintf("%s-%s-%s-%s", alloc.ServiceID, alloc.CostType, alloc.Region, alloc.StartTime.Format("2006-01-02-15")), - ProviderId: alloc.ServiceID, - BilledCost: alloc.Amount, - ListCost: alloc.Amount, - UsageQuantity: alloc.Units, - UsageUnit: alloc.UnitType, - Zone: alloc.Region, - Labels: map[string]string{ - "service_id": alloc.ServiceID, - "service_name": serviceName, - "cost_type": alloc.CostType, - "region": alloc.Region, - "service_attributes": fmt.Sprintf("%v", attrs), - }, - ExtendedAttributes: &pb.CustomCostExtendedAttributes{ - ServiceName: stringPtr("fastly-cdn"), - ServiceCategory: stringPtr("infrastructure"), - Provider: stringPtr("fastly"), - AccountId: stringPtr(f.config.FastlyAccountID), - }, +func getUsageUnit(usageType string) string { + // Map common usage types to units + usageType = strings.ToLower(usageType) + if strings.Contains(usageType, "bandwidth") { + return "GB" } -} - -func (f *FastlyCostSource) reconcileCosts(invoiceCosts, usageCosts []*pb.CustomCost) []*pb.CustomCost { - // If we have detailed usage costs, prefer those over invoice line items - // This provides more granular cost allocation - if len(usageCosts) > 0 { - return usageCosts + if strings.Contains(usageType, "request") { + return "requests" + } + if strings.Contains(usageType, "compute") { + return "hours" } - return invoiceCosts + return "units" } -func (f *FastlyCostSource) createCostFromLineItem(invoice fastlyplugin.Invoice, item fastlyplugin.LineItem, window opencost.Window) *pb.CustomCost { - // Calculate the daily rate for the line item - dailyRate := item.Total / float32(invoice.GetDaysInBillingPeriod()) - - // Calculate the cost for this specific window - windowDuration := window.End().Sub(*window.Start()).Hours() / 24.0 - windowCost := dailyRate * float32(windowDuration) - - return &pb.CustomCost{ - AccountName: invoice.CustomerID, - ChargeCategory: "usage", - Description: item.Description, - ResourceName: item.ServiceType, - ResourceType: "cdn", - Id: fmt.Sprintf("%s-%s-%s", invoice.ID, item.ServiceType, window.Start().Format("2006-01-02")), - ProviderId: invoice.ID, - BilledCost: windowCost, - ListCost: windowCost, - ListUnitPrice: item.Rate, - UsageQuantity: item.Units, - UsageUnit: item.UnitType, - Labels: map[string]string{ - "service_type": item.ServiceType, - "region": item.Region, - }, - Zone: item.Region, - ExtendedAttributes: &pb.CustomCostExtendedAttributes{ - BillingPeriodStart: timestamppb.New(invoice.BillingStartDate), - BillingPeriodEnd: timestamppb.New(invoice.BillingEndDate), - ServiceName: stringPtr("fastly-cdn"), - ServiceCategory: stringPtr("infrastructure"), - Provider: stringPtr("fastly"), - AccountId: stringPtr(invoice.CustomerID), - }, +func boilerplateFastlyCustomCost(win opencost.Window) pb.CustomCostResponse { + return pb.CustomCostResponse{ + Metadata: map[string]string{"api_client_version": "v3"}, + CostSource: "billing", + Domain: "fastly", + Version: "v1", + Currency: "USD", + Start: timestamppb.New(*win.Start()), + End: timestamppb.New(*win.End()), + Errors: []string{}, + Costs: []*pb.CustomCost{}, } } func main() { - configFile, err := commonconfig.GetConfigFilePath() - if err != nil { - log.Fatalf("error opening config file: %v", err) + // Get config file path from environment variable or use default + configFile := os.Getenv("PLUGIN_CONFIG_FILE") + if configFile == "" { + configFile = "/opt/opencost/plugin/config.json" } fastlyConfig, err := getFastlyConfig(configFile) if err != nil { log.Fatalf("error building Fastly config: %v", err) } - log.SetLogLevel(fastlyConfig.LogLevel) - // Create rate limiter - rateLimiter := rate.NewLimiter(rate.Limit(fastlyConfig.RateLimitPerSecond), 1) - - // Create Fastly API client - client := fastlyplugin.NewFastlyClient(fastlyConfig.FastlyAPIToken) + // Fastly rate limiting - be conservative + rateLimiter := rate.NewLimiter(rate.Every(time.Second), 10) - // Create cost calculator - costCalculator := fastlyplugin.NewCostCalculator() - - // Create Fastly cost source - fastlyCostSrc := &FastlyCostSource{ - config: fastlyConfig, - client: client, - rateLimiter: rateLimiter, - ctx: context.Background(), - costCalculator: costCalculator, + fastlyCostSrc := FastlyCostSource{ + apiKey: fastlyConfig.FastlyAPIKey, + httpClient: &http.Client{Timeout: 30 * time.Second}, + rateLimiter: rateLimiter, + invoiceCache: make(map[string][]fastlyplugin.Invoice), } - // Plugin map for the CustomCostSource + // pluginMap is the map of plugins we can dispense. var pluginMap = map[string]plugin.Plugin{ - "CustomCostSource": &ocplugin.CustomCostPlugin{Impl: fastlyCostSrc}, + "CustomCostSource": &ocplugin.CustomCostPlugin{Impl: &fastlyCostSrc}, } plugin.Serve(&plugin.ServeConfig{ @@ -315,26 +369,18 @@ func main() { func getFastlyConfig(configFilePath string) (*fastlyplugin.FastlyConfig, error) { var result fastlyplugin.FastlyConfig - bytes, err := os.ReadFile(configFilePath) if err != nil { return nil, fmt.Errorf("error reading config file for Fastly config @ %s: %v", configFilePath, err) } - err = json.Unmarshal(bytes, &result) if err != nil { - return nil, fmt.Errorf("error marshaling json into Fastly config: %v", err) + return nil, fmt.Errorf("error marshaling json into Fastly config %v", err) } - // Validate config - if err := result.Validate(); err != nil { - return nil, err + if result.LogLevel == "" { + result.LogLevel = "info" } return &result, nil } - -// Helper function to convert a string to a string pointer -func stringPtr(s string) *string { - return &s -} diff --git a/pkg/plugins/fastly/cmd/main/main_test.go b/pkg/plugins/fastly/cmd/main/main_test.go index 06ab7d0..bf24601 100644 --- a/pkg/plugins/fastly/cmd/main/main_test.go +++ b/pkg/plugins/fastly/cmd/main/main_test.go @@ -1 +1,168 @@ package main + +import ( + "net/http" + "os" + "testing" + "time" + + "github.com/opencost/opencost-plugins/pkg/plugins/fastly/fastlyplugin" + "github.com/opencost/opencost/core/pkg/log" + "github.com/opencost/opencost/core/pkg/model/pb" + "github.com/opencost/opencost/core/pkg/opencost" + "github.com/opencost/opencost/core/pkg/util/timeutil" + "golang.org/x/time/rate" + "google.golang.org/protobuf/types/known/durationpb" + "google.golang.org/protobuf/types/known/timestamppb" +) + +func TestGetCustomCosts(t *testing.T) { + // read necessary env vars. If any are missing, log warning and skip test + fastlyAPIKey := os.Getenv("FASTLY_API_KEY") + + if fastlyAPIKey == "" { + log.Warnf("FASTLY_API_KEY undefined, skipping test") + t.Skip() + return + } + + // write out config + config := fastlyplugin.FastlyConfig{ + FastlyAPIKey: fastlyAPIKey, + LogLevel: "debug", + } + + rateLimiter := rate.NewLimiter(rate.Every(time.Second), 10) + fastlyCostSrc := FastlyCostSource{ + apiKey: config.FastlyAPIKey, + httpClient: &http.Client{Timeout: 30 * time.Second}, + rateLimiter: rateLimiter, + invoiceCache: make(map[string][]fastlyplugin.Invoice), + } + + // Query for last month's data + now := time.Now() + windowStart := time.Date(now.Year(), now.Month()-1, 1, 0, 0, 0, 0, time.UTC) + windowEnd := time.Date(now.Year(), now.Month(), 1, 0, 0, 0, 0, time.UTC) + + req := &pb.CustomCostRequest{ + Start: timestamppb.New(windowStart), + End: timestamppb.New(windowEnd), + Resolution: durationpb.New(timeutil.Day), + } + + log.SetLogLevel("trace") + resp := fastlyCostSrc.GetCustomCosts(req) + + if len(resp) == 0 { + t.Fatalf("empty response") + } + + // Check for errors + for _, r := range resp { + if len(r.Errors) > 0 { + t.Errorf("errors in response: %v", r.Errors) + } + } + + // Verify domain + for _, r := range resp { + if r.Domain != "fastly" { + t.Errorf("expected domain 'fastly', got %s", r.Domain) + } + } + + // Log some results for debugging + totalCosts := 0 + totalBilled := float32(0) + for _, r := range resp { + totalCosts += len(r.Costs) + for _, cost := range r.Costs { + totalBilled += cost.BilledCost + } + } + t.Logf("Total responses: %d, Total costs: %d, Total billed: %.2f", len(resp), totalCosts, totalBilled) +} + +func TestConvertInvoiceToCosts(t *testing.T) { + fastlyCostSrc := FastlyCostSource{} + + // Create a test invoice + invoice := fastlyplugin.Invoice{ + CustomerID: "test-customer-123", + InvoiceID: "inv-123", + BillingStartDate: "2024-01-01T00:00:00Z", + BillingEndDate: "2024-01-31T23:59:59Z", + CurrencyCode: "USD", + TransactionLineItems: []fastlyplugin.TransactionLineItem{ + { + Description: "CDN Bandwidth", + Amount: 100.50, + Rate: 0.05, + Units: 2010, + ProductName: "CDN", + ProductGroup: "Full Site Delivery", + ProductLine: "Network Services", + Region: "Global", + UsageType: "bandwidth", + }, + { + Description: "Compute Requests", + Amount: 50.25, + Rate: 0.001, + Units: 50250, + ProductName: "Compute@Edge", + ProductGroup: "Compute", + ProductLine: "Edge Computing", + Region: "US-East", + UsageType: "requests", + }, + }, + } + + // Create a window that overlaps with the invoice + windowStart := time.Date(2024, 1, 15, 0, 0, 0, 0, time.UTC) + windowEnd := time.Date(2024, 1, 16, 0, 0, 0, 0, time.UTC) + window := opencost.NewWindow(&windowStart, &windowEnd) + + costs := fastlyCostSrc.convertInvoiceToCosts(invoice, window) + + if len(costs) != 2 { + t.Errorf("expected 2 costs, got %d", len(costs)) + } + + // Verify costs are prorated correctly + for _, cost := range costs { + if cost.BilledCost <= 0 { + t.Errorf("expected positive billed cost, got %f", cost.BilledCost) + } + if cost.UsageQuantity <= 0 { + t.Errorf("expected positive usage quantity, got %f", cost.UsageQuantity) + } + // Since we're getting 1 day out of 31, costs should be roughly 1/31 of the total + if cost.ResourceName == "bandwidth" && cost.BilledCost > 5 { + t.Errorf("bandwidth cost seems too high for 1-day proration: %f", cost.BilledCost) + } + } +} + +func TestGetUsageUnit(t *testing.T) { + tests := []struct { + usageType string + expected string + }{ + {"bandwidth", "GB"}, + {"requests", "requests"}, + {"compute_hours", "hours"}, + {"Bandwidth_GB", "GB"}, + {"API_Requests", "requests"}, + {"custom_metric", "units"}, + } + + for _, test := range tests { + result := getUsageUnit(test.usageType) + if result != test.expected { + t.Errorf("getUsageUnit(%s) = %s, expected %s", test.usageType, result, test.expected) + } + } +} diff --git a/pkg/plugins/fastly/cmd/validator/main/main.go b/pkg/plugins/fastly/cmd/validator/main/main.go index 2f1f561..016411f 100644 --- a/pkg/plugins/fastly/cmd/validator/main/main.go +++ b/pkg/plugins/fastly/cmd/validator/main/main.go @@ -12,24 +12,30 @@ import ( "google.golang.org/protobuf/encoding/protojson" ) -// Validator for Fastly plugin integration tests +// the validator is designed to allow plugin implementors to validate their plugin information +// as called by the central test harness. +// this avoids having to ask folks to re-implement the test harness over again for each plugin + +// the integration test harness provides a path to a protobuf file for each window +// the validator can then read that in and further validate the response data +// using the domain knowledge of each plugin author func main() { + // first arg is the path to the daily protobuf file if len(os.Args) < 3 { fmt.Println("Usage: validator ") os.Exit(1) } dailyProtobufFilePath := os.Args[1] - hourlyProtobufFilePath := os.Args[2] - // Read and validate daily data - dailyData, err := os.ReadFile(dailyProtobufFilePath) + // read in the protobuf file + data, err := os.ReadFile(dailyProtobufFilePath) if err != nil { fmt.Printf("Error reading daily protobuf file: %v\n", err) os.Exit(1) } - dailyCustomCostResponses, err := Unmarshal(dailyData) + dailyCustomCostResponses, err := Unmarshal(data) if err != nil { fmt.Printf("Error unmarshalling daily protobuf data: %v\n", err) os.Exit(1) @@ -37,14 +43,17 @@ func main() { fmt.Printf("Successfully unmarshalled %d daily custom cost responses\n", len(dailyCustomCostResponses)) - // Read and validate hourly data - hourlyData, err := os.ReadFile(hourlyProtobufFilePath) + // second arg is the path to the hourly protobuf file + hourlyProtobufFilePath := os.Args[2] + + data, err = os.ReadFile(hourlyProtobufFilePath) if err != nil { fmt.Printf("Error reading hourly protobuf file: %v\n", err) os.Exit(1) } - hourlyCustomCostResponses, err := Unmarshal(hourlyData) + // read in the protobuf file + hourlyCustomCostResponses, err := Unmarshal(data) if err != nil { fmt.Printf("Error unmarshalling hourly protobuf data: %v\n", err) os.Exit(1) @@ -52,9 +61,9 @@ func main() { fmt.Printf("Successfully unmarshalled %d hourly custom cost responses\n", len(hourlyCustomCostResponses)) - // Validate the responses - isValid := validate(dailyCustomCostResponses, hourlyCustomCostResponses) - if !isValid { + // validate the custom cost response data + isvalid := validate(dailyCustomCostResponses, hourlyCustomCostResponses) + if !isvalid { os.Exit(1) } else { fmt.Println("Validation successful") @@ -74,7 +83,7 @@ func validate(respDaily, respHourly []*pb.CustomCostResponse) bool { var multiErr error - // Check for errors in responses + // parse the response and look for errors for _, resp := range respDaily { if len(resp.Errors) > 0 { multiErr = multierror.Append(multiErr, fmt.Errorf("errors occurred in daily response: %v", resp.Errors)) @@ -82,102 +91,109 @@ func validate(respDaily, respHourly []*pb.CustomCostResponse) bool { } for _, resp := range respHourly { - if len(resp.Errors) > 0 { + if resp.Errors != nil { multiErr = multierror.Append(multiErr, fmt.Errorf("errors occurred in hourly response: %v", resp.Errors)) } } + // check if any errors occurred if multiErr != nil { log.Errorf("Errors occurred during plugin testing for fastly: %v", multiErr) return false } - // Validate daily costs - seenCosts := map[string]bool{} + seenResourceTypes := map[string]bool{} + seenProductGroups := map[string]bool{} totalDailyCost := float32(0.0) + // verify that the returned costs are non zero for _, resp := range respDaily { - // Skip empty responses for recent dates if len(resp.Costs) == 0 && resp.Start.AsTime().After(time.Now().Truncate(24*time.Hour).Add(-1*time.Minute)) { - log.Debugf("today's daily costs returned by plugin fastly are empty, skipping: %v", resp) + log.Debugf("today's daily costs returned by fastly plugin are empty, skipping: %v", resp) continue } for _, cost := range resp.Costs { totalDailyCost += cost.GetBilledCost() - seenCosts[cost.GetResourceName()] = true + seenResourceTypes[cost.GetResourceType()] = true + seenProductGroups[cost.GetResourceName()] = true - // Validate cost sanity - if cost.GetBilledCost() < 0 { - log.Errorf("negative cost found for %v", cost) - return false + if cost.GetBilledCost() == 0 { + log.Debugf("got zero cost for %v", cost) } - // Check for reasonable cost values (adjust based on your expected ranges) - if cost.GetBilledCost() > 10000 { - log.Errorf("unexpectedly high daily cost for %v: %f", cost.GetResourceName(), cost.GetBilledCost()) + // Sanity check - individual line items shouldn't be extremely high + if cost.GetBilledCost() > 100000 { + log.Errorf("daily cost returned by fastly plugin for %v is greater than 100,000", cost) return false } } } - // Validate we have some expected cost types - expectedCosts := []string{ - "cdn_bandwidth", - "cdn_requests", - // Add more expected cost types as needed + // Fastly should have some costs unless it's a free/developer account + if totalDailyCost == 0 { + log.Warnf("daily costs returned by fastly plugin are zero - this might be expected for developer accounts") } - foundExpectedCost := false - for _, expectedCost := range expectedCosts { - if seenCosts[expectedCost] { - foundExpectedCost = true + // Check that we see some expected product groups + expectedProductGroups := []string{ + "Full Site Delivery", + "Compute", + "Security", + "Network Services", + } + + foundAnyExpected := false + for _, expected := range expectedProductGroups { + if seenResourceTypes[expected] { + foundAnyExpected = true break } } - if !foundExpectedCost && totalDailyCost > 0 { - log.Warnf("None of the expected cost types found, but costs exist. Seen costs: %v", seenCosts) + if len(seenResourceTypes) > 0 && !foundAnyExpected { + log.Warnf("none of the expected product groups found in fastly response. Seen: %v", seenResourceTypes) } - // Validate domain + // verify the domain matches the plugin name for _, resp := range respDaily { if resp.Domain != "fastly" { - log.Errorf("daily domain returned by plugin does not match expected 'fastly': %s", resp.Domain) + log.Errorf("daily domain returned by fastly plugin does not match plugin name") return false } } - // Validate hourly costs - seenHourlyCosts := map[string]bool{} + // Check hourly responses totalHourlyCost := float32(0.0) - for _, resp := range respHourly { for _, cost := range resp.Costs { - seenHourlyCosts[cost.GetResourceName()] = true totalHourlyCost += cost.GetBilledCost() - - if cost.GetBilledCost() < 0 { - log.Errorf("negative hourly cost found for %v", cost) - return false - } - - // Hourly costs should be smaller than daily - if cost.GetBilledCost() > 1000 { - log.Errorf("unexpectedly high hourly cost for %v: %f", cost.GetResourceName(), cost.GetBilledCost()) + if cost.GetBilledCost() > 10000 { + log.Errorf("hourly cost returned by fastly plugin for %v is greater than 10,000", cost) return false } } } - // Basic sanity check - if we have daily costs, we should have hourly costs - if totalDailyCost > 0 && totalHourlyCost == 0 { - log.Errorf("daily costs exist but no hourly costs found") - return false + if totalHourlyCost == 0 { + log.Warnf("hourly costs returned by fastly plugin are zero - this might be expected for developer accounts") } - log.Infof("Validation passed. Daily costs: %f, Hourly costs: %f", totalDailyCost, totalHourlyCost) - log.Infof("Cost types seen - Daily: %v, Hourly: %v", seenCosts, seenHourlyCosts) + // Verify currency is USD + for _, resp := range respDaily { + if resp.Currency != "USD" { + log.Errorf("expected currency USD, got %s", resp.Currency) + return false + } + } + + // Verify cost source + for _, resp := range respDaily { + if resp.CostSource != "billing" { + log.Errorf("expected cost source 'billing', got %s", resp.CostSource) + return false + } + } return true } @@ -187,7 +203,6 @@ func Unmarshal(data []byte) ([]*pb.CustomCostResponse, error) { if err := json.Unmarshal(data, &raw); err != nil { return nil, err } - protoResps := make([]*pb.CustomCostResponse, len(raw)) for i, r := range raw { p := &pb.CustomCostResponse{} diff --git a/pkg/plugins/fastly/fastly-config-sample.json b/pkg/plugins/fastly/fastly-config-sample.json deleted file mode 100644 index 44683c1..0000000 --- a/pkg/plugins/fastly/fastly-config-sample.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "fastly_api_token": "YOUR_FASTLY_API_TOKEN_HERE", - "fastly_account_id": "", - "log_level": "info", - "rate_limit_per_second": 1.0, - "enable_usage_detail": false, - "service_filters": [], - "exclude_services": [] -} diff --git a/pkg/plugins/fastly/fastlyplugin/fastlyconfig.go b/pkg/plugins/fastly/fastlyplugin/fastlyconfig.go new file mode 100644 index 0000000..8bc7c65 --- /dev/null +++ b/pkg/plugins/fastly/fastlyplugin/fastlyconfig.go @@ -0,0 +1,6 @@ +package fastlyplugin + +type FastlyConfig struct { + FastlyAPIKey string `json:"fastly_api_key"` + LogLevel string `json:"log_level"` +} diff --git a/pkg/plugins/fastly/fastlyplugin/invoices.go b/pkg/plugins/fastly/fastlyplugin/invoices.go new file mode 100644 index 0000000..8c1d63f --- /dev/null +++ b/pkg/plugins/fastly/fastlyplugin/invoices.go @@ -0,0 +1,46 @@ +package fastlyplugin + +import "time" + +type InvoiceListResponse struct { + Data []Invoice `json:"data"` + Meta PaginationMetadata `json:"meta"` +} + +type Invoice struct { + CustomerID string `json:"customer_id"` + InvoiceID string `json:"invoice_id"` + InvoicePostedOn string `json:"invoice_posted_on"` + BillingStartDate string `json:"billing_start_date"` + BillingEndDate string `json:"billing_end_date"` + StatementNumber string `json:"statement_number"` + CurrencyCode string `json:"currency_code"` + MonthlyTransactionAmount float64 `json:"monthly_transaction_amount"` + PaymentStatus string `json:"payment_status,omitempty"` + TransactionLineItems []TransactionLineItem `json:"transaction_line_items"` +} + +type TransactionLineItem struct { + Description string `json:"description"` + Amount float64 `json:"amount"` + CreditCouponCode string `json:"credit_coupon_code"` + Rate float64 `json:"rate"` + Units float64 `json:"units"` + ProductName string `json:"product_name"` + ProductGroup string `json:"product_group"` + ProductLine string `json:"product_line"` + Region string `json:"region"` + UsageType string `json:"usage_type"` +} + +type PaginationMetadata struct { + NextCursor string `json:"next_cursor"` + Limit int `json:"limit"` + Total int `json:"total"` + Sort string `json:"sort"` +} + +// Helper function to parse date strings +func ParseFastlyDate(dateStr string) (time.Time, error) { + return time.Parse(time.RFC3339, dateStr) +} diff --git a/pkg/plugins/fastly/go.mod b/pkg/plugins/fastly/go.mod index 301ed7a..4d07174 100644 --- a/pkg/plugins/fastly/go.mod +++ b/pkg/plugins/fastly/go.mod @@ -2,37 +2,35 @@ module github.com/opencost/opencost-plugins/pkg/plugins/fastly go 1.24.2 -replace github.com/opencost/opencost-plugins/pkg/common => ../../common - require ( github.com/hashicorp/go-multierror v1.1.1 github.com/hashicorp/go-plugin v1.6.3 - github.com/opencost/opencost-plugins/pkg/common v0.0.0-20250403142249-d8eaf3258224 + github.com/opencost/opencost-plugins/test v0.0.0-20240429172518-a50cd1290864 github.com/opencost/opencost/core v0.0.0-20250805023318-1e1643d24b0e golang.org/x/time v0.12.0 - google.golang.org/protobuf v1.36.6 + google.golang.org/protobuf v1.36.5 ) require ( - github.com/fatih/color v1.18.0 // indirect + github.com/fatih/color v1.16.0 // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect github.com/fxamacker/cbor/v2 v2.7.0 // indirect - github.com/go-logr/logr v1.4.3 // indirect + github.com/go-logr/logr v1.4.2 // indirect github.com/goccy/go-json v0.10.5 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/protobuf v1.5.4 // indirect - github.com/hashicorp/errwrap v1.1.0 // indirect - github.com/hashicorp/go-hclog v1.6.3 // indirect + github.com/hashicorp/errwrap v1.0.0 // indirect + github.com/hashicorp/go-hclog v1.6.2 // indirect github.com/hashicorp/hcl v1.0.0 // indirect - github.com/hashicorp/yamux v0.1.2 // indirect + github.com/hashicorp/yamux v0.1.1 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/magiconair/properties v1.8.7 // indirect - github.com/mattn/go-colorable v0.1.14 // indirect + github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect - github.com/oklog/run v1.2.0 // indirect + github.com/oklog/run v1.1.0 // indirect github.com/patrickmn/go-cache v2.1.0+incompatible // indirect github.com/pelletier/go-toml/v2 v2.1.1 // indirect github.com/rs/zerolog v1.32.0 // indirect @@ -47,11 +45,11 @@ require ( github.com/x448/float16 v0.8.4 // indirect go.uber.org/multierr v1.11.0 // indirect golang.org/x/exp v0.0.0-20240222234643-814bf88cf225 // indirect - golang.org/x/net v0.42.0 // indirect - golang.org/x/sys v0.34.0 // indirect - golang.org/x/text v0.27.0 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20250804133106-a7a43d27e69b // indirect - google.golang.org/grpc v1.74.2 // indirect + golang.org/x/net v0.38.0 // indirect + golang.org/x/sys v0.31.0 // indirect + golang.org/x/text v0.23.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20241209162323-e6fa225c2576 // indirect + google.golang.org/grpc v1.68.1 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/pkg/plugins/fastly/go.sum b/pkg/plugins/fastly/go.sum index 6e2fa93..91f4470 100644 --- a/pkg/plugins/fastly/go.sum +++ b/pkg/plugins/fastly/go.sum @@ -6,18 +6,16 @@ github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= -github.com/fatih/color v1.18.0 h1:S8gINlzdQ840/4pfAwic/ZE0djQEH3wM94VfqLTZcOM= -github.com/fatih/color v1.18.0/go.mod h1:4FelSpRwEGDpQ12mAdzqdOukCy4u8WUtOY6lkT/6HfU= +github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM= +github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE= github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv5E= github.com/fxamacker/cbor/v2 v2.7.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ= -github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= -github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= -github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= -github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= +github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4= github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= @@ -29,21 +27,18 @@ github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeN github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= -github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= -github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= -github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= -github.com/hashicorp/go-hclog v1.6.3 h1:Qr2kF+eVWjTiYmU7Y31tYlP1h0q/X3Nl3tPGdaB11/k= -github.com/hashicorp/go-hclog v1.6.3/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M= +github.com/hashicorp/go-hclog v1.6.2 h1:NOtoftovWkDheyUM/8JW3QMiXyxJK3uHRK7wV04nD2I= +github.com/hashicorp/go-hclog v1.6.2/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M= github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= github.com/hashicorp/go-plugin v1.6.3 h1:xgHB+ZUSYeuJi96WtxEjzi23uh7YQpznjGh0U0UUrwg= github.com/hashicorp/go-plugin v1.6.3/go.mod h1:MRobyh+Wc/nYy1V4KAXUiYfzxoYhs7V1mlH1Z7iY2h0= github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= -github.com/hashicorp/yamux v0.1.2 h1:XtB8kyFOyHXYVFnwT5C3+Bdo8gArse7j2AQ0DA0Uey8= -github.com/hashicorp/yamux v0.1.2/go.mod h1:C+zze2n6e/7wshOZep2A70/aQU6QBRWJO/G6FT1wIns= +github.com/hashicorp/yamux v0.1.1 h1:yrQxtgseBDrq9Y652vSRDvsKCJKOUD+GzTS4Y0Y8pvE= +github.com/hashicorp/yamux v0.1.1/go.mod h1:CtWFDAQgb7dxtzFs4tWbplKIe2jSi3+5vKbgIO0SLnQ= github.com/jhump/protoreflect v1.15.1 h1:HUMERORf3I3ZdX05WaQ6MIpd/NJ434hTp5YiKgfCL6c= github.com/jhump/protoreflect v1.15.1/go.mod h1:jD/2GMKKE6OqX8qTjhADU1e6DShO+gavG9e0Q693nKo= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= @@ -58,9 +53,8 @@ github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0V github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= +github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= -github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE= -github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= @@ -74,8 +68,10 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= -github.com/oklog/run v1.2.0 h1:O8x3yXwah4A73hJdlrwo/2X6J62gE5qTMusH0dvz60E= -github.com/oklog/run v1.2.0/go.mod h1:mgDbKRSwPhJfesJ4PntqFUbKQRZ50NgmZTSPlFA0YFk= +github.com/oklog/run v1.1.0 h1:GEenZ1cK0+q0+wsJew9qUg/DyD8k3JzYsZAi5gYi2mA= +github.com/oklog/run v1.1.0/go.mod h1:sVPdnTZT1zYwAJeCMu2Th4T21pA3FPOQRfWjQlk7DVU= +github.com/opencost/opencost-plugins/test v0.0.0-20240429172518-a50cd1290864 h1:r0Nlj0Az+XhFdOdik/dzZygeMvzM/cXo0JszHETiii0= +github.com/opencost/opencost-plugins/test v0.0.0-20240429172518-a50cd1290864/go.mod h1:6B39vhLt/C97HxMoTZehNF9cf9z7qRd9EzKUYKgvPg4= github.com/opencost/opencost/core v0.0.0-20250805023318-1e1643d24b0e h1:RSXlPL7sB0NSpS55hAExcsKsq0xlEgwZIIZZw346NI8= github.com/opencost/opencost/core v0.0.0-20250805023318-1e1643d24b0e/go.mod h1:qKHpAsBp1gS3EkWy/HWfy19doPxlzhv9t5CfauZNN/k= github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc= @@ -121,18 +117,6 @@ github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= -go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= -go.opentelemetry.io/otel v1.36.0 h1:UumtzIklRBY6cI/lllNZlALOF5nNIzJVb16APdvgTXg= -go.opentelemetry.io/otel v1.36.0/go.mod h1:/TcFMXYjyRNh8khOAO9ybYkqaDBb/70aVwkNML4pP8E= -go.opentelemetry.io/otel/metric v1.36.0 h1:MoWPKVhQvJ+eeXWHFBOPoBOi20jh6Iq2CcCREuTYufE= -go.opentelemetry.io/otel/metric v1.36.0/go.mod h1:zC7Ks+yeyJt4xig9DEw9kuUFe5C3zLbVjV2PzT6qzbs= -go.opentelemetry.io/otel/sdk v1.36.0 h1:b6SYIuLRs88ztox4EyrvRti80uXIFy+Sqzoh9kFULbs= -go.opentelemetry.io/otel/sdk v1.36.0/go.mod h1:+lC+mTgD+MUWfjJubi2vvXWcVxyr9rmlshZni72pXeY= -go.opentelemetry.io/otel/sdk/metric v1.36.0 h1:r0ntwwGosWGaa0CrSt8cuNuTcccMXERFwHX4dThiPis= -go.opentelemetry.io/otel/sdk/metric v1.36.0/go.mod h1:qTNOhFDfKRwX0yXOqJYegL5WRaW376QbB7P4Pb0qva4= -go.opentelemetry.io/otel/trace v1.36.0 h1:ahxWNuqZjpdiFAyrIoQ4GIiAIhxAunQR6MUoKrsNd4w= -go.opentelemetry.io/otel/trace v1.36.0/go.mod h1:gQ+OnDZzrybY4k4seLzPAWNwVBBVlF2szhehOBB/tGA= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= @@ -146,8 +130,8 @@ golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.42.0 h1:jzkYrhi3YQWD6MLBJcsklgQsoAcw89EcZbJw8Z614hs= -golang.org/x/net v0.42.0/go.mod h1:FF1RA5d3u7nAYA4z2TkclSCKh68eSXtiFwcWQpPXdt8= +golang.org/x/net v0.38.0 h1:vRMAPTMaeGqVhG5QyLJHqNDwecKTomGeqbnfZyKlBI8= +golang.org/x/net v0.38.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -162,12 +146,14 @@ golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.34.0 h1:H5Y5sJ2L2JRdyv7ROF1he/lPdvFsd0mJHFw2ThKHxLA= -golang.org/x/sys v0.34.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/sys v0.31.0 h1:ioabZlmFYtWhL+TRYpcnNlLwhyxaM9kWTDEmfnprqik= +golang.org/x/sys v0.31.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.27.0 h1:4fGWRpyh641NLlecmyl4LOe6yDdfaYNrGb2zdfo4JV4= -golang.org/x/text v0.27.0/go.mod h1:1D28KMCvyooCX9hBiosv5Tz/+YLxj0j7XhWjpSUF7CU= +golang.org/x/text v0.23.0 h1:D71I7dUrlY+VX0gQShAThNGHFxZ13dGLBHQLVl1mJlY= +golang.org/x/text v0.23.0/go.mod h1:/BLNzu4aZCJ1+kcD0DNRotWKage4q2rGVAg4o22unh4= +golang.org/x/time v0.9.0 h1:EsRrnYcQiGH+5FfbgvV4AP7qEZstoyrHB0DzarOQ4ZY= +golang.org/x/time v0.9.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/time v0.12.0 h1:ScB/8o8olJvc+CQPWrK3fPZNfh7qgwCrY0zJmoEQLSE= golang.org/x/time v0.12.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -178,12 +164,12 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/genproto/googleapis/rpc v0.0.0-20250804133106-a7a43d27e69b h1:zPKJod4w6F1+nRGDI9ubnXYhU9NSWoFAijkHkUXeTK8= -google.golang.org/genproto/googleapis/rpc v0.0.0-20250804133106-a7a43d27e69b/go.mod h1:qQ0YXyHHx3XkvlzUtpXDkS29lDSafHMZBAZDc03LQ3A= -google.golang.org/grpc v1.74.2 h1:WoosgB65DlWVC9FqI82dGsZhWFNBSLjQ84bjROOpMu4= -google.golang.org/grpc v1.74.2/go.mod h1:CtQ+BGjaAIXHs/5YS3i473GqwBBa1zGQNevxdeBEXrM= -google.golang.org/protobuf v1.36.6 h1:z1NpPI8ku2WgiWnf+t9wTPsn6eP1L7ksHUlkfLvd9xY= -google.golang.org/protobuf v1.36.6/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY= +google.golang.org/genproto/googleapis/rpc v0.0.0-20241209162323-e6fa225c2576 h1:8ZmaLZE4XWrtU3MyClkYqqtl6Oegr3235h7jxsDyqCY= +google.golang.org/genproto/googleapis/rpc v0.0.0-20241209162323-e6fa225c2576/go.mod h1:5uTbfoYQed2U9p3KIj2/Zzm02PYhndfdmML0qC3q3FU= +google.golang.org/grpc v1.68.1 h1:oI5oTa11+ng8r8XMMN7jAOmWfPZWbYpCFaMUTACxkM0= +google.golang.org/grpc v1.68.1/go.mod h1:+q1XYFJjShcqn0QZHvCyeR4CXPA+llXIeUIfIe00waw= +google.golang.org/protobuf v1.36.5 h1:tPhr+woSbjfYvY6/GPufUoYizxw1cF/yFoxJ2fmpwlM= +google.golang.org/protobuf v1.36.5/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= diff --git a/pkg/plugins/fastly/plugin/fastlyconfig.go b/pkg/plugins/fastly/plugin/fastlyconfig.go deleted file mode 100644 index 27b52f0..0000000 --- a/pkg/plugins/fastly/plugin/fastlyconfig.go +++ /dev/null @@ -1,44 +0,0 @@ -package plugin - -import "fmt" - -// FastlyConfig represents the configuration needed to authenticate with Fastly API -type FastlyConfig struct { - // FastlyAPIToken is the API token for authenticating with Fastly - FastlyAPIToken string `json:"fastly_api_token"` - - // FastlyAccountID is the account ID for the Fastly account (optional) - FastlyAccountID string `json:"fastly_account_id,omitempty"` - - // LogLevel controls the verbosity of logging - LogLevel string `json:"log_level,omitempty"` - - // RateLimitPerSecond controls API request rate limiting - RateLimitPerSecond float64 `json:"rate_limit_per_second,omitempty"` - - // EnableUsageDetail enables fetching detailed usage data for cost allocation - EnableUsageDetail bool `json:"enable_usage_detail,omitempty"` - - // ServiceFilters limits cost retrieval to specific services - ServiceFilters []string `json:"service_filters,omitempty"` - - // ExcludeServices excludes specific services from cost retrieval - ExcludeServices []string `json:"exclude_services,omitempty"` -} - -// Validate checks if the configuration is valid -func (c *FastlyConfig) Validate() error { - if c.FastlyAPIToken == "" { - return fmt.Errorf("fastly_api_token is required") - } - - if c.LogLevel == "" { - c.LogLevel = "info" - } - - if c.RateLimitPerSecond <= 0 { - c.RateLimitPerSecond = 1.0 // Default to 1 request per second - } - - return nil -} diff --git a/pkg/plugins/fastly/plugin/services.go b/pkg/plugins/fastly/plugin/services.go deleted file mode 100644 index 8ed398c..0000000 --- a/pkg/plugins/fastly/plugin/services.go +++ /dev/null @@ -1,314 +0,0 @@ -package plugin - -import ( - "context" - "encoding/json" - "fmt" - "net/http" - "time" - - "github.com/opencost/opencost/core/pkg/log" -) - -// FastlyClient provides methods to interact with Fastly API -type FastlyClient struct { - APIToken string - HTTPClient *http.Client - BaseURL string -} - -// NewFastlyClient creates a new Fastly API client -func NewFastlyClient(apiToken string) *FastlyClient { - return &FastlyClient{ - APIToken: apiToken, - HTTPClient: &http.Client{ - Timeout: 30 * time.Second, - }, - BaseURL: "https://api.fastly.com", - } -} - -// FetchInvoices retrieves invoices for the specified time range -func (c *FastlyClient) FetchInvoices(ctx context.Context, start, end time.Time) ([]Invoice, error) { - allInvoices := []Invoice{} - cursor := "" - - for { - url := fmt.Sprintf("%s/billing/v3/invoices?from=%s&to=%s&limit=100", - c.BaseURL, - start.Format("2006-01-02"), - end.Format("2006-01-02")) - - if cursor != "" { - url += "&cursor=" + cursor - } - - invoices, meta, err := c.fetchInvoicePage(ctx, url) - if err != nil { - return nil, err - } - - allInvoices = append(allInvoices, invoices...) - - if meta.NextCursor == "" { - break - } - cursor = meta.NextCursor - } - - return allInvoices, nil -} - -func (c *FastlyClient) fetchInvoicePage(ctx context.Context, url string) ([]Invoice, Meta, error) { - req, err := http.NewRequestWithContext(ctx, "GET", url, nil) - if err != nil { - return nil, Meta{}, err - } - - req.Header.Add("Fastly-Key", c.APIToken) - req.Header.Add("Accept", "application/json") - - resp, err := c.HTTPClient.Do(req) - if err != nil { - return nil, Meta{}, err - } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK { - return nil, Meta{}, fmt.Errorf("API request failed with status %d", resp.StatusCode) - } - - var response InvoiceResponse - if err := json.NewDecoder(resp.Body).Decode(&response); err != nil { - return nil, Meta{}, err - } - - return response.Data, response.Meta, nil -} - -// FetchUsage retrieves usage data for the specified time range -func (c *FastlyClient) FetchUsage(ctx context.Context, start, end time.Time, serviceID string) ([]Usage, error) { - allUsage := []Usage{} - - // Fastly usage API typically provides hourly granularity - current := start - for current.Before(end) { - nextHour := current.Add(time.Hour) - if nextHour.After(end) { - nextHour = end - } - - usage, err := c.fetchUsageForPeriod(ctx, current, nextHour, serviceID) - if err != nil { - log.Warnf("error fetching usage for period %v-%v: %v", current, nextHour, err) - // Continue with next period instead of failing completely - current = nextHour - continue - } - - allUsage = append(allUsage, usage...) - current = nextHour - } - - return allUsage, nil -} - -func (c *FastlyClient) fetchUsageForPeriod(ctx context.Context, start, end time.Time, serviceID string) ([]Usage, error) { - url := fmt.Sprintf("%s/stats/usage_by_service?from=%s&to=%s&by=hour", - c.BaseURL, - start.Format("2006-01-02T15:04:05Z"), - end.Format("2006-01-02T15:04:05Z")) - - if serviceID != "" { - url += "&service_id=" + serviceID - } - - req, err := http.NewRequestWithContext(ctx, "GET", url, nil) - if err != nil { - return nil, err - } - - req.Header.Add("Fastly-Key", c.APIToken) - req.Header.Add("Accept", "application/json") - - resp, err := c.HTTPClient.Do(req) - if err != nil { - return nil, err - } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf("usage API request failed with status %d", resp.StatusCode) - } - - var response UsageResponse - if err := json.NewDecoder(resp.Body).Decode(&response); err != nil { - return nil, err - } - - return response.Data, nil -} - -// FetchServices retrieves all services for the account -func (c *FastlyClient) FetchServices(ctx context.Context) ([]ServiceDetail, error) { - url := fmt.Sprintf("%s/service", c.BaseURL) - - req, err := http.NewRequestWithContext(ctx, "GET", url, nil) - if err != nil { - return nil, err - } - - req.Header.Add("Fastly-Key", c.APIToken) - req.Header.Add("Accept", "application/json") - - resp, err := c.HTTPClient.Do(req) - if err != nil { - return nil, err - } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf("services API request failed with status %d", resp.StatusCode) - } - - var services []ServiceDetail - if err := json.NewDecoder(resp.Body).Decode(&services); err != nil { - return nil, err - } - - return services, nil -} - -// CostCalculator provides methods for calculating costs from usage data -type CostCalculator struct { - // Pricing rates can be configured here - BandwidthRate float32 // Cost per GB - RequestRate float32 // Cost per 10k requests - ComputeRate float32 // Cost per compute second - ShieldingRate float32 // Cost per GB for origin shielding -} - -// NewCostCalculator creates a new cost calculator with default rates -func NewCostCalculator() *CostCalculator { - return &CostCalculator{ - BandwidthRate: 0.05, // $0.05 per GB - RequestRate: 0.001, // $0.001 per 10k requests - ComputeRate: 0.0001, // $0.0001 per compute second - ShieldingRate: 0.02, // $0.02 per GB for shielding - } -} - -// CalculateUsageCost calculates cost from usage metrics -func (calc *CostCalculator) CalculateUsageCost(usage Usage) []CostAllocation { - allocations := []CostAllocation{} - - for region, metrics := range usage.Usage { - // Bandwidth cost - if metrics.Bandwidth > 0 { - bandwidthGB := float32(metrics.Bandwidth) / (1024 * 1024 * 1024) - allocations = append(allocations, CostAllocation{ - ServiceID: usage.ServiceID, - ServiceName: usage.ServiceName, - Region: region, - CostType: "bandwidth", - Amount: bandwidthGB * calc.BandwidthRate, - Units: bandwidthGB, - UnitType: "GB", - StartTime: usage.StartTime, - EndTime: usage.EndTime, - }) - } - - // Request cost - if metrics.Requests > 0 { - requestsIn10k := float32(metrics.Requests) / 10000 - allocations = append(allocations, CostAllocation{ - ServiceID: usage.ServiceID, - ServiceName: usage.ServiceName, - Region: region, - CostType: "requests", - Amount: requestsIn10k * calc.RequestRate, - Units: requestsIn10k, - UnitType: "10k_requests", - StartTime: usage.StartTime, - EndTime: usage.EndTime, - }) - } - - // Compute cost - if metrics.ComputeDuration > 0 { - computeSeconds := float32(metrics.ComputeDuration) / 1000 - allocations = append(allocations, CostAllocation{ - ServiceID: usage.ServiceID, - ServiceName: usage.ServiceName, - Region: region, - CostType: "compute", - Amount: computeSeconds * calc.ComputeRate, - Units: computeSeconds, - UnitType: "compute_seconds", - StartTime: usage.StartTime, - EndTime: usage.EndTime, - }) - } - - // Origin shielding cost (cached bandwidth) - if metrics.BandwidthCached > 0 { - cachedGB := float32(metrics.BandwidthCached) / (1024 * 1024 * 1024) - allocations = append(allocations, CostAllocation{ - ServiceID: usage.ServiceID, - ServiceName: usage.ServiceName, - Region: region, - CostType: "origin_shielding", - Amount: cachedGB * calc.ShieldingRate, - Units: cachedGB, - UnitType: "GB", - StartTime: usage.StartTime, - EndTime: usage.EndTime, - }) - } - } - - return allocations -} - -// ServiceMapper provides methods for enriching cost data with service information -type ServiceMapper struct { - services map[string]ServiceDetail -} - -// NewServiceMapper creates a new service mapper -func NewServiceMapper(services []ServiceDetail) *ServiceMapper { - serviceMap := make(map[string]ServiceDetail) - for _, service := range services { - serviceMap[service.ID] = service - } - - return &ServiceMapper{ - services: serviceMap, - } -} - -// GetServiceName returns the service name for a given service ID -func (sm *ServiceMapper) GetServiceName(serviceID string) string { - if service, ok := sm.services[serviceID]; ok { - return service.Name - } - return serviceID -} - -// GetServiceAttributes returns the attributes for a given service ID -func (sm *ServiceMapper) GetServiceAttributes(serviceID string) map[string]string { - if service, ok := sm.services[serviceID]; ok { - return service.Attributes - } - return map[string]string{} -} - -// GetServices returns all services -func (sm *ServiceMapper) GetServices() []ServiceDetail { - services := make([]ServiceDetail, 0, len(sm.services)) - for _, service := range sm.services { - services = append(services, service) - } - return services -} diff --git a/pkg/plugins/fastly/plugin/types.go b/pkg/plugins/fastly/plugin/types.go deleted file mode 100644 index 7505d4f..0000000 --- a/pkg/plugins/fastly/plugin/types.go +++ /dev/null @@ -1,173 +0,0 @@ -package plugin - -import ( - "fmt" - "time" -) - -// InvoiceResponse represents the response from Fastly's invoice API -type InvoiceResponse struct { - Data []Invoice `json:"data"` - Meta Meta `json:"meta"` -} - -// Invoice represents a Fastly invoice -type Invoice struct { - ID string `json:"id"` - CustomerID string `json:"customer_id"` - InvoiceNumber string `json:"invoice_number"` - State string `json:"state"` - Total float32 `json:"total"` - Region string `json:"region"` - BillingStartDate time.Time `json:"billing_start_date"` - BillingEndDate time.Time `json:"billing_end_date"` - IssuedOn time.Time `json:"issued_on"` - DueOn time.Time `json:"due_on"` - Currency string `json:"currency_code"` - LineItems []LineItem `json:"line_items"` -} - -// LineItem represents a line item in a Fastly invoice -type LineItem struct { - ID string `json:"id"` - Description string `json:"description"` - Amount float32 `json:"amount"` - Rate float32 `json:"rate"` - Units float32 `json:"units"` - UnitType string `json:"unit_type"` - ServiceType string `json:"service_type"` - ServiceID string `json:"service_id"` - Total float32 `json:"total"` - Region string `json:"region"` - ProductLine string `json:"product_line"` -} - -// Meta represents pagination metadata -type Meta struct { - NextCursor string `json:"next_cursor"` - Limit int `json:"limit"` -} - -// GetDaysInBillingPeriod calculates the number of days in the billing period -func (i *Invoice) GetDaysInBillingPeriod() int { - return int(i.BillingEndDate.Sub(i.BillingStartDate).Hours() / 24) -} - -// UsageResponse represents the response from Fastly's usage API -type UsageResponse struct { - Data []Usage `json:"data"` - Meta Meta `json:"meta"` -} - -// Usage represents usage data from Fastly -type Usage struct { - ServiceID string `json:"service_id"` - ServiceName string `json:"service_name"` - StartTime time.Time `json:"start_time"` - EndTime time.Time `json:"end_time"` - Region string `json:"region"` - Usage map[string]UsageMetric `json:"usage"` -} - -// UsageMetric represents a specific usage metric -type UsageMetric struct { - Requests int64 `json:"requests"` - Bandwidth int64 `json:"bandwidth"` - BandwidthCached int64 `json:"bandwidth_cached"` - ComputeRequests int64 `json:"compute_requests"` - ComputeDuration float64 `json:"compute_duration_ms"` -} - -// ServiceDetail represents detailed information about a Fastly service -type ServiceDetail struct { - ID string `json:"id"` - Name string `json:"name"` - CustomerID string `json:"customer_id"` - CreatedAt time.Time `json:"created_at"` - UpdatedAt time.Time `json:"updated_at"` - ActiveVersion int `json:"active_version"` - Type string `json:"type"` - Attributes map[string]string `json:"attributes"` -} - -// CostAllocation helps with allocating costs across different services and regions -type CostAllocation struct { - ServiceID string - ServiceName string - Region string - CostType string - Amount float32 - Units float32 - UnitType string - StartTime time.Time - EndTime time.Time -} - -// GetCostCategory returns the cost category based on the service type -func GetCostCategory(serviceType string) string { - switch serviceType { - case "cdn_bandwidth", "cdn_requests", "origin_shielding": - return "networking" - case "compute", "compute_requests": - return "compute" - case "logs", "real_time_analytics": - return "observability" - case "waf", "ddos_protection": - return "security" - default: - return "other" - } -} - -// IsWithinTimeRange checks if an invoice covers the specified time range -func (i *Invoice) IsWithinTimeRange(start, end time.Time) bool { - // Check if invoice period overlaps with requested time range - return !(i.BillingEndDate.Before(start) || i.BillingStartDate.After(end)) -} - -// GetProportionalCost calculates the proportional cost for a specific time window -func (i *Invoice) GetProportionalCost(windowStart, windowEnd time.Time) float32 { - // Calculate overlap between invoice period and window - overlapStart := i.BillingStartDate - if windowStart.After(overlapStart) { - overlapStart = windowStart - } - - overlapEnd := i.BillingEndDate - if windowEnd.Before(overlapEnd) { - overlapEnd = windowEnd - } - - // If no overlap, return 0 - if overlapEnd.Before(overlapStart) || overlapEnd.Equal(overlapStart) { - return 0 - } - - // Calculate proportion - overlapDays := overlapEnd.Sub(overlapStart).Hours() / 24.0 - totalDays := float64(i.GetDaysInBillingPeriod()) - - if totalDays == 0 { - return 0 - } - - proportion := float32(overlapDays / totalDays) - return i.Total * proportion -} - -// Validate checks if the invoice data is valid -func (i *Invoice) Validate() error { - if i.ID == "" { - return fmt.Errorf("invoice ID is empty") - } - - if i.BillingEndDate.Before(i.BillingStartDate) { - return fmt.Errorf("billing end date is before start date") - } - - if i.Total < 0 { - return fmt.Errorf("invoice total is negative") - } - - return nil -} diff --git a/pkg/plugins/fastly/tests/fastly_test.go b/pkg/plugins/fastly/tests/fastly_test.go index 2bda438..ad4a395 100644 --- a/pkg/plugins/fastly/tests/fastly_test.go +++ b/pkg/plugins/fastly/tests/fastly_test.go @@ -1,236 +1,189 @@ package tests import ( + "encoding/json" + "io/fs" "os" + "path/filepath" + "runtime" "testing" "time" - fastlyplugin "github.com/opencost/opencost-plugins/pkg/plugins/fastly/plugin" + "github.com/opencost/opencost-plugins/pkg/plugins/fastly/fastlyplugin" + "github.com/opencost/opencost-plugins/test/pkg/harness" "github.com/opencost/opencost/core/pkg/log" "github.com/opencost/opencost/core/pkg/model/pb" - "github.com/opencost/opencost/core/pkg/opencost" "github.com/opencost/opencost/core/pkg/util/timeutil" - "golang.org/x/time/rate" + "google.golang.org/protobuf/types/known/durationpb" "google.golang.org/protobuf/types/known/timestamppb" ) -func TestGetCustomCosts(t *testing.T) { - // Read necessary env vars - fastlyAPIToken := os.Getenv("FASTLY_API_TOKEN") +func TestFastlyCostRetrieval(t *testing.T) { + // Query for last month's data + now := time.Now() + windowStart := time.Date(now.Year(), now.Month()-1, 1, 0, 0, 0, 0, time.UTC) + windowEnd := time.Date(now.Year(), now.Month(), 1, 0, 0, 0, 0, time.UTC) - if fastlyAPIToken == "" { - log.Warnf("FASTLY_API_TOKEN undefined, skipping test") - t.Skip() - return - } + response := getFastlyResponse(t, windowStart, windowEnd, timeutil.Day) - config := fastlyplugin.FastlyConfig{ - FastlyAPIToken: fastlyAPIToken, - LogLevel: "debug", - RateLimitPerSecond: 1.0, + // confirm no errors in result + if len(response) == 0 { + t.Fatalf("empty response") + } + for _, resp := range response { + if len(resp.Errors) > 0 { + t.Fatalf("got errors in response: %v", resp.Errors) + } } - // Validate config - if err := config.Validate(); err != nil { - t.Fatalf("config validation failed: %v", err) + // confirm results have correct provider + for _, resp := range response { + if resp.Domain != "fastly" { + t.Fatalf("unexpected domain. expected fastly, got %s", resp.Domain) + } } - // Test rate limiter creation - rateLimiter := rate.NewLimiter(rate.Limit(config.RateLimitPerSecond), 1) + // check some attributes of the cost response + totalCosts := 0 + totalBilled := float32(0) + for _, resp := range response { + // May have zero costs for some days + totalCosts += len(resp.Costs) - // Verify rate limiter is configured correctly - if rateLimiter.Limit() != rate.Limit(config.RateLimitPerSecond) { - t.Errorf("Rate limiter limit mismatch: got %v, want %v", rateLimiter.Limit(), config.RateLimitPerSecond) + for _, cost := range resp.Costs { + totalBilled += cost.BilledCost + + // Verify required fields are populated + if cost.ProviderId == "" { + t.Errorf("empty ProviderId") + } + if cost.ResourceType == "" { + t.Errorf("empty ResourceType") + } + if cost.ResourceName == "" { + t.Errorf("empty ResourceName") + } + } } - t.Log("Config validation and rate limiter setup passed") + t.Logf("Total responses: %d, Total costs: %d, Total billed: %.2f", len(response), totalCosts, totalBilled) } -func TestFastlyConfig(t *testing.T) { - tests := []struct { - name string - config fastlyplugin.FastlyConfig - wantErr bool - }{ - { - name: "valid config", - config: fastlyplugin.FastlyConfig{ - FastlyAPIToken: "test-token", - LogLevel: "info", - }, - wantErr: false, - }, - { - name: "missing API token", - config: fastlyplugin.FastlyConfig{ - LogLevel: "info", - }, - wantErr: true, - }, - { - name: "default log level", - config: fastlyplugin.FastlyConfig{ - FastlyAPIToken: "test-token", - }, - wantErr: false, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - err := tt.config.Validate() - if (err != nil) != tt.wantErr { - t.Errorf("Validate() error = %v, wantErr %v", err, tt.wantErr) - } - }) +func TestFastlyFutureWindow(t *testing.T) { + // query for the future + windowStart := time.Now().UTC().Truncate(time.Hour).Add(time.Hour) + windowEnd := windowStart.Add(time.Hour) + + response := getFastlyResponse(t, windowStart, windowEnd, time.Hour) + + // when we query for data in the future, we expect to get back no data AND no errors + if len(response) > 0 { + t.Fatalf("got non-empty response for future window") } } -func TestInvoiceProcessing(t *testing.T) { - // Test invoice processing logic - invoice := fastlyplugin.Invoice{ - ID: "inv-123", - CustomerID: "cust-456", - BillingStartDate: time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC), - BillingEndDate: time.Date(2024, 1, 31, 0, 0, 0, 0, time.UTC), - Total: 1000.0, - Currency: "USD", - LineItems: []fastlyplugin.LineItem{ - { - ID: "li-1", - Description: "CDN Bandwidth", - ServiceType: "cdn_bandwidth", - Amount: 500.0, - Rate: 0.05, - Units: 10000, - UnitType: "GB", - Total: 500.0, - Region: "us-east-1", - }, - { - ID: "li-2", - Description: "CDN Requests", - ServiceType: "cdn_requests", - Amount: 300.0, - Rate: 0.001, - Units: 300000, - UnitType: "requests", - Total: 300.0, - Region: "us-east-1", - }, - }, - } - - days := invoice.GetDaysInBillingPeriod() - if days != 30 { - t.Errorf("Expected 30 days, got %d", days) - } - - // Test daily rate calculation - for _, item := range invoice.LineItems { - dailyRate := item.Total / float32(days) - if dailyRate <= 0 { - t.Errorf("Daily rate should be positive, got %f", dailyRate) +func TestFastlyHourlyData(t *testing.T) { + // query for hourly data from yesterday + now := time.Now() + windowStart := time.Date(now.Year(), now.Month(), now.Day()-1, 0, 0, 0, 0, time.UTC) + windowEnd := windowStart.Add(24 * time.Hour) + + response := getFastlyResponse(t, windowStart, windowEnd, time.Hour) + + // Should get 24 responses for hourly data + if len(response) != 24 { + t.Errorf("expected 24 hourly responses, got %d", len(response)) + } + + // Verify each response + for i, resp := range response { + if len(resp.Errors) > 0 { + t.Errorf("errors in hourly response %d: %v", i, resp.Errors) + } + + // Verify timestamps + expectedStart := windowStart.Add(time.Duration(i) * time.Hour) + expectedEnd := expectedStart.Add(time.Hour) + + if !resp.Start.AsTime().Equal(expectedStart) { + t.Errorf("response %d: expected start %v, got %v", i, expectedStart, resp.Start.AsTime()) + } + if !resp.End.AsTime().Equal(expectedEnd) { + t.Errorf("response %d: expected end %v, got %v", i, expectedEnd, resp.End.AsTime()) } } } -func TestCostResponseStructure(t *testing.T) { - // Test the structure of a cost response - windowStart := time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC) - windowEnd := time.Date(2024, 1, 2, 0, 0, 0, 0, time.UTC) - - resp := &pb.CustomCostResponse{ - Metadata: map[string]string{"api_version": "v1"}, - CostSource: "infrastructure", - Domain: "fastly", - Version: "v1", - Currency: "USD", - Start: timestamppb.New(windowStart), - End: timestamppb.New(windowEnd), - Errors: []string{}, - Costs: []*pb.CustomCost{ - { - AccountName: "cust-456", - ChargeCategory: "usage", - Description: "CDN Bandwidth", - ResourceName: "cdn_bandwidth", - ResourceType: "cdn", - Id: "inv-123-cdn_bandwidth-2024-01-01", - BilledCost: 16.67, - ListCost: 16.67, - ListUnitPrice: 0.05, - UsageQuantity: 333.33, - UsageUnit: "GB", - Labels: map[string]string{ - "service_type": "cdn_bandwidth", - "region": "us-east-1", - }, - Zone: "us-east-1", - }, - }, - } - - // Validate response structure - if resp.Domain != "fastly" { - t.Errorf("Expected domain 'fastly', got %s", resp.Domain) - } - - if len(resp.Costs) != 1 { - t.Errorf("Expected 1 cost, got %d", len(resp.Costs)) - } - - if resp.Costs[0].ResourceType != "cdn" { - t.Errorf("Expected resource type 'cdn', got %s", resp.Costs[0].ResourceType) +func TestFastlyMonthToDate(t *testing.T) { + // Query for current month to date + now := time.Now() + windowStart := time.Date(now.Year(), now.Month(), 1, 0, 0, 0, 0, time.UTC) + windowEnd := now.Truncate(24 * time.Hour) + + response := getFastlyResponse(t, windowStart, windowEnd, timeutil.Day) + + if len(response) == 0 { + t.Fatalf("empty response for month-to-date") + } + + // Should include current month data + foundCurrentMonth := false + for _, resp := range response { + if resp.Start.AsTime().Month() == now.Month() { + foundCurrentMonth = true + break + } + } + + if !foundCurrentMonth { + t.Errorf("no data found for current month") } } -func TestWindowProcessing(t *testing.T) { - // Test window processing for different resolutions - tests := []struct { - name string - start time.Time - end time.Time - resolution time.Duration - expected int - }{ - { - name: "daily resolution", - start: time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC), - end: time.Date(2024, 1, 8, 0, 0, 0, 0, time.UTC), - resolution: timeutil.Day, - expected: 7, - }, - { - name: "hourly resolution", - start: time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC), - end: time.Date(2024, 1, 1, 6, 0, 0, 0, time.UTC), - resolution: time.Hour, - expected: 6, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - // Test window calculation - windows, err := opencost.GetWindows(tt.start, tt.end, tt.resolution) - if err != nil { - t.Fatalf("Failed to get windows: %v", err) - } +func getFastlyResponse(t *testing.T, windowStart, windowEnd time.Time, step time.Duration) []*pb.CustomCostResponse { + // read necessary env vars. If any are missing, log warning and skip test + fastlyAPIKey := os.Getenv("FASTLY_API_KEY") - if len(windows) != tt.expected { - t.Errorf("Expected %d windows, got %d", tt.expected, len(windows)) - } + if fastlyAPIKey == "" { + log.Warnf("FASTLY_API_KEY undefined, skipping test") + t.Skip() + return nil + } - // Verify each window has the correct duration - for i, window := range windows { - duration := window.End().Sub(*window.Start()) - if duration != tt.resolution { - t.Errorf("Window %d has incorrect duration: got %v, want %v", i, duration, tt.resolution) - } - } + // write out config to temp file using contents of env vars + config := fastlyplugin.FastlyConfig{ + FastlyAPIKey: fastlyAPIKey, + LogLevel: "debug", + } + + // set up custom cost request + file, err := os.CreateTemp("", "fastly_config.json") + if err != nil { + t.Fatalf("could not create temp config dir: %v", err) + } + defer os.Remove(file.Name()) + + data, err := json.MarshalIndent(config, "", " ") + if err != nil { + t.Fatalf("could not marshal json: %v", err) + } - t.Logf("Test case %s: got %d windows as expected", tt.name, len(windows)) - }) + err = os.WriteFile(file.Name(), data, fs.FileMode(os.O_RDWR)) + if err != nil { + t.Fatalf("could not write file: %v", err) } + + // invoke plugin via harness + _, filename, _, _ := runtime.Caller(0) + parent := filepath.Dir(filename) + pluginRoot := filepath.Dir(parent) + pluginFile := pluginRoot + "/cmd/main/main.go" + + req := pb.CustomCostRequest{ + Start: timestamppb.New(windowStart), + End: timestamppb.New(windowEnd), + Resolution: durationpb.New(step), + } + + return harness.InvokePlugin(file.Name(), pluginFile, &req) } From bd2dd7ae71c3757d5a2cda550d7f775f4d92974b Mon Sep 17 00:00:00 2001 From: Sparsh Date: Tue, 5 Aug 2025 18:44:56 +0530 Subject: [PATCH 3/8] fixed the api request header and updated test cases Signed-off-by: Sparsh --- pkg/plugins/fastly/cmd/main/main.go | 44 +- pkg/plugins/fastly/cmd/main/main_test.go | 604 ++++++++++++++++++----- 2 files changed, 530 insertions(+), 118 deletions(-) diff --git a/pkg/plugins/fastly/cmd/main/main.go b/pkg/plugins/fastly/cmd/main/main.go index 195f882..39b709b 100644 --- a/pkg/plugins/fastly/cmd/main/main.go +++ b/pkg/plugins/fastly/cmd/main/main.go @@ -141,6 +141,7 @@ func (f *FastlyCostSource) getInvoicesForPeriod(start, end *time.Time) ([]fastly } req.Header.Set("Accept", "application/json") req.Header.Set("Fastly-Key", f.apiKey) + req.Header.Set("Host", "api.fastly.com") resp, err := f.httpClient.Do(req) if err != nil { @@ -200,6 +201,45 @@ func (f *FastlyCostSource) getMonthToDateInvoice() (*fastlyplugin.Invoice, error } req.Header.Set("Accept", "application/json") req.Header.Set("Fastly-Key", f.apiKey) + req.Header.Set("Host", "api.fastly.com") + + resp, err := f.httpClient.Do(req) + if err != nil { + return nil, fmt.Errorf("error making request: %v", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + body, _ := io.ReadAll(resp.Body) + return nil, fmt.Errorf("API request failed with status %d: %s", resp.StatusCode, string(body)) + } + + var invoice fastlyplugin.Invoice + if err := json.NewDecoder(resp.Body).Decode(&invoice); err != nil { + return nil, fmt.Errorf("error decoding response: %v", err) + } + + return &invoice, nil +} + +func (f *FastlyCostSource) getInvoiceByID(invoiceID string) (*fastlyplugin.Invoice, error) { + // Rate limiting + if f.rateLimiter.Tokens() < 1.0 { + log.Infof("fastly rate limit reached. holding request until rate capacity is back") + } + err := f.rateLimiter.WaitN(context.TODO(), 1) + if err != nil { + return nil, fmt.Errorf("error waiting on rate limiter: %v", err) + } + + reqURL := fmt.Sprintf("%s/billing/v3/invoices/%s", fastlyAPIBaseURL, invoiceID) + req, err := http.NewRequest("GET", reqURL, nil) + if err != nil { + return nil, fmt.Errorf("error creating request: %v", err) + } + req.Header.Set("Accept", "application/json") + req.Header.Set("Fastly-Key", f.apiKey) + req.Header.Set("Host", "api.fastly.com") resp, err := f.httpClient.Do(req) if err != nil { @@ -334,9 +374,9 @@ func boilerplateFastlyCustomCost(win opencost.Window) pb.CustomCostResponse { func main() { // Get config file path from environment variable or use default - configFile := os.Getenv("PLUGIN_CONFIG_FILE") + configFile := os.Getenv("FASTLY_PLUGIN_CONFIG_FILE") if configFile == "" { - configFile = "/opt/opencost/plugin/config.json" + configFile = "/opt/opencost/plugin/fastlyconfig.json" } fastlyConfig, err := getFastlyConfig(configFile) diff --git a/pkg/plugins/fastly/cmd/main/main_test.go b/pkg/plugins/fastly/cmd/main/main_test.go index bf24601..67f1491 100644 --- a/pkg/plugins/fastly/cmd/main/main_test.go +++ b/pkg/plugins/fastly/cmd/main/main_test.go @@ -1,168 +1,540 @@ package main import ( + "fmt" + "io" "net/http" + "net/url" "os" + "path/filepath" + "strings" "testing" "time" "github.com/opencost/opencost-plugins/pkg/plugins/fastly/fastlyplugin" - "github.com/opencost/opencost/core/pkg/log" - "github.com/opencost/opencost/core/pkg/model/pb" "github.com/opencost/opencost/core/pkg/opencost" - "github.com/opencost/opencost/core/pkg/util/timeutil" "golang.org/x/time/rate" - "google.golang.org/protobuf/types/known/durationpb" - "google.golang.org/protobuf/types/known/timestamppb" ) -func TestGetCustomCosts(t *testing.T) { - // read necessary env vars. If any are missing, log warning and skip test - fastlyAPIKey := os.Getenv("FASTLY_API_KEY") +// mockHTTPTransport implements http.RoundTripper for testing +type mockHTTPTransport struct { + responses map[string]*http.Response +} + +// URL query parameter order-insensitive matching +func (m *mockHTTPTransport) RoundTrip(req *http.Request) (*http.Response, error) { + // Try exact match first + if response, ok := m.responses[req.URL.String()]; ok { + return response, nil + } + + // If exact match fails, try matching by normalized URL + // (comparing host, path and query parameters regardless of order) + requestURL := req.URL.String() + for mockURL, response := range m.responses { + if haveSameParams(mockURL, requestURL) { + return response, nil + } + } + + return nil, fmt.Errorf("no mock response found for URL: %s", req.URL.String()) +} + +// Helper function to check if two URLs have the same parameters regardless of order +func haveSameParams(url1, url2 string) bool { + u1, err := url.Parse(url1) + if err != nil { + return false + } + u2, err := url.Parse(url2) + if err != nil { + return false + } + + // Check if host and path match + if u1.Host != u2.Host || u1.Path != u2.Path { + return false + } + + // Check if query parameters are the same (regardless of order) + q1 := u1.Query() + q2 := u2.Query() + + if len(q1) != len(q2) { + return false + } + + for k, v1 := range q1 { + v2, ok := q2[k] + if !ok { + return false + } + if len(v1) != len(v2) { + return false + } + for i := range v1 { + if v1[i] != v2[i] { + return false + } + } + } + + return true +} + +// createMockResponse creates a mock HTTP response with given status code and body +func createMockResponse(statusCode int, body string) *http.Response { + return &http.Response{ + StatusCode: statusCode, + Body: io.NopCloser(strings.NewReader(body)), + Header: make(http.Header), + } +} + +func TestGetFastlyConfig(t *testing.T) { + // Arrange + tempDir := t.TempDir() + configPath := filepath.Join(tempDir, "config.json") + configData := `{ + "fastly_api_key": "test-api-key", + "log_level": "debug" + }` + err := os.WriteFile(configPath, []byte(configData), 0644) + if err != nil { + t.Fatalf("Failed to create test config file: %v", err) + } + + // Act + config, err := getFastlyConfig(configPath) + + // Assert + if err != nil { + t.Fatalf("getFastlyConfig returned error: %v", err) + } + if config == nil { + t.Fatal("Expected config to be non-nil") + } + if config.FastlyAPIKey != "test-api-key" { + t.Errorf("Expected API key 'test-api-key', got '%s'", config.FastlyAPIKey) + } + if config.LogLevel != "debug" { + t.Errorf("Expected log level 'debug', got '%s'", config.LogLevel) + } +} + +func TestGetFastlyConfigDefaultLogLevel(t *testing.T) { + // Arrange + tempDir := t.TempDir() + configPath := filepath.Join(tempDir, "config.json") + configData := `{ + "fastly_api_key": "test-api-key" + }` + err := os.WriteFile(configPath, []byte(configData), 0644) + if err != nil { + t.Fatalf("Failed to create test config file: %v", err) + } + + // Act + config, err := getFastlyConfig(configPath) + + // Assert + if err != nil { + t.Fatalf("getFastlyConfig returned error: %v", err) + } + if config.LogLevel != "info" { + t.Errorf("Expected default log level 'info', got '%s'", config.LogLevel) + } +} + +func TestGetFastlyConfigInvalidPath(t *testing.T) { + // Arrange + configPath := "/nonexistent/path/to/config.json" + + // Act + config, err := getFastlyConfig(configPath) + + // Assert + if err == nil { + t.Error("Expected error for nonexistent config file, got nil") + } + if config != nil { + t.Error("Expected nil config for invalid path") + } +} + +func TestGetFastlyConfigInvalidJSON(t *testing.T) { + // Arrange + tempDir := t.TempDir() + configPath := filepath.Join(tempDir, "config.json") + invalidJSON := `{ invalid json }` + err := os.WriteFile(configPath, []byte(invalidJSON), 0644) + if err != nil { + t.Fatalf("Failed to create test config file: %v", err) + } + + // Act + config, err := getFastlyConfig(configPath) + + // Assert + if err == nil { + t.Error("Expected error for invalid JSON, got nil") + } + if config != nil { + t.Error("Expected nil config for invalid JSON") + } +} + +func TestGetMonthToDateInvoice(t *testing.T) { + // Arrange + mockTransport := &mockHTTPTransport{ + responses: map[string]*http.Response{ + "https://api.fastly.com/billing/v3/invoices/month-to-date": createMockResponse(200, `{ + "customer_id": "test-customer", + "invoice_id": "mtd-12345", + "billing_start_date": "2024-01-01T00:00:00Z", + "billing_end_date": "2024-01-31T00:00:00Z", + "currency_code": "USD", + "monthly_transaction_amount": 50.75, + "transaction_line_items": [ + { + "description": "CDN Bandwidth", + "amount": 50.75, + "rate": 0.05, + "units": 1015, + "product_name": "CDN", + "product_group": "Full Site Delivery", + "product_line": "Network Services", + "region": "Global", + "usage_type": "bandwidth" + } + ] + }`), + }, + } + + rateLimiter := rate.NewLimiter(rate.Every(time.Second), 10) + fastlyCostSrc := FastlyCostSource{ + apiKey: "test-api-key", + httpClient: &http.Client{Transport: mockTransport}, + rateLimiter: rateLimiter, + invoiceCache: make(map[string][]fastlyplugin.Invoice), + } + + // Act + invoice, err := fastlyCostSrc.getMonthToDateInvoice() - if fastlyAPIKey == "" { - log.Warnf("FASTLY_API_KEY undefined, skipping test") - t.Skip() - return + // Assert + if err != nil { + t.Fatalf("Failed to get month-to-date invoice: %v", err) + } + if invoice == nil { + t.Fatal("Received nil invoice") + } + if invoice.InvoiceID != "mtd-12345" { + t.Errorf("Expected invoice ID mtd-12345, got %s", invoice.InvoiceID) } + if len(invoice.TransactionLineItems) != 1 { + t.Errorf("Expected 1 transaction line item, got %d", len(invoice.TransactionLineItems)) + } +} - // write out config - config := fastlyplugin.FastlyConfig{ - FastlyAPIKey: fastlyAPIKey, - LogLevel: "debug", +func TestGetMonthToDateInvoiceError(t *testing.T) { + // Arrange + mockTransport := &mockHTTPTransport{ + responses: map[string]*http.Response{ + "https://api.fastly.com/billing/v3/invoices/month-to-date": createMockResponse(403, `{ + "msg": "Unauthorized", + "detail": "Invalid API key" + }`), + }, } rateLimiter := rate.NewLimiter(rate.Every(time.Second), 10) fastlyCostSrc := FastlyCostSource{ - apiKey: config.FastlyAPIKey, - httpClient: &http.Client{Timeout: 30 * time.Second}, + apiKey: "invalid-api-key", + httpClient: &http.Client{Transport: mockTransport}, rateLimiter: rateLimiter, invoiceCache: make(map[string][]fastlyplugin.Invoice), } - // Query for last month's data - now := time.Now() - windowStart := time.Date(now.Year(), now.Month()-1, 1, 0, 0, 0, 0, time.UTC) - windowEnd := time.Date(now.Year(), now.Month(), 1, 0, 0, 0, 0, time.UTC) + // Act + invoice, err := fastlyCostSrc.getMonthToDateInvoice() - req := &pb.CustomCostRequest{ - Start: timestamppb.New(windowStart), - End: timestamppb.New(windowEnd), - Resolution: durationpb.New(timeutil.Day), + // Assert + if err == nil { + t.Error("Expected error for unauthorized request, got nil") + } + if invoice != nil { + t.Error("Expected nil invoice for error response") } +} - log.SetLogLevel("trace") - resp := fastlyCostSrc.GetCustomCosts(req) +func TestGetInvoicesForPeriod(t *testing.T) { + // Arrange + mockTransport := &mockHTTPTransport{ + responses: map[string]*http.Response{ + "https://api.fastly.com/billing/v3/invoices?billing_start_date=2024-01-01&billing_end_date=2024-01-31&limit=200": createMockResponse(200, `{ + "data": [ + { + "customer_id": "test-customer", + "invoice_id": "inv-12345", + "billing_start_date": "2024-01-01T00:00:00Z", + "billing_end_date": "2024-01-31T00:00:00Z", + "currency_code": "USD", + "monthly_transaction_amount": 100.50, + "transaction_line_items": [ + { + "description": "CDN Bandwidth", + "amount": 100.50, + "rate": 0.05, + "units": 2010, + "product_name": "CDN", + "product_group": "Full Site Delivery", + "product_line": "Network Services", + "region": "Global", + "usage_type": "bandwidth" + } + ] + } + ], + "meta": { + "next_cursor": "", + "limit": 200, + "total": 1, + "sort": "billing_start_date" + } + }`), + }, + } - if len(resp) == 0 { - t.Fatalf("empty response") + rateLimiter := rate.NewLimiter(rate.Every(time.Second), 10) + fastlyCostSrc := FastlyCostSource{ + apiKey: "test-api-key", + httpClient: &http.Client{Transport: mockTransport}, + rateLimiter: rateLimiter, + invoiceCache: make(map[string][]fastlyplugin.Invoice), } - // Check for errors - for _, r := range resp { - if len(r.Errors) > 0 { - t.Errorf("errors in response: %v", r.Errors) - } + startDate := time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC) + endDate := time.Date(2024, 1, 31, 0, 0, 0, 0, time.UTC) + + // Act + invoices, err := fastlyCostSrc.getInvoicesForPeriod(&startDate, &endDate) + + // Assert + if err != nil { + t.Fatalf("Failed to get invoices for period: %v", err) + } + if len(invoices) != 1 { + t.Errorf("Expected 1 invoice, got %d", len(invoices)) } + if invoices[0].InvoiceID != "inv-12345" { + t.Errorf("Expected invoice ID inv-12345, got %s", invoices[0].InvoiceID) + } +} - // Verify domain - for _, r := range resp { - if r.Domain != "fastly" { - t.Errorf("expected domain 'fastly', got %s", r.Domain) - } +func TestGetInvoicesForPeriodWithPagination(t *testing.T) { + // Arrange + mockTransport := &mockHTTPTransport{ + responses: map[string]*http.Response{ + // First page + "https://api.fastly.com/billing/v3/invoices?billing_start_date=2024-01-01&billing_end_date=2024-01-31&limit=200": createMockResponse(200, `{ + "data": [ + { + "customer_id": "test-customer", + "invoice_id": "inv-12345", + "billing_start_date": "2024-01-01T00:00:00Z", + "billing_end_date": "2024-01-31T00:00:00Z", + "transaction_line_items": [ + { + "description": "CDN Bandwidth", + "amount": 100.50, + "rate": 0.05, + "units": 2010, + "product_name": "CDN", + "product_group": "Full Site Delivery", + "usage_type": "bandwidth" + } + ] + } + ], + "meta": { + "next_cursor": "page2", + "limit": 200, + "total": 2 + } + }`), + // Second page + "https://api.fastly.com/billing/v3/invoices?billing_start_date=2024-01-01&billing_end_date=2024-01-31&limit=200&cursor=page2": createMockResponse(200, `{ + "data": [ + { + "customer_id": "test-customer", + "invoice_id": "inv-67890", + "billing_start_date": "2024-01-01T00:00:00Z", + "billing_end_date": "2024-01-31T00:00:00Z", + "transaction_line_items": [ + { + "description": "Compute Requests", + "amount": 75.25, + "rate": 0.001, + "units": 75250, + "product_name": "Compute", + "product_group": "Compute", + "usage_type": "requests" + } + ] + } + ], + "meta": { + "next_cursor": "", + "limit": 200, + "total": 2 + } + }`), + }, } - // Log some results for debugging - totalCosts := 0 - totalBilled := float32(0) - for _, r := range resp { - totalCosts += len(r.Costs) - for _, cost := range r.Costs { - totalBilled += cost.BilledCost + rateLimiter := rate.NewLimiter(rate.Every(time.Second), 10) + fastlyCostSrc := FastlyCostSource{ + apiKey: "test-api-key", + httpClient: &http.Client{Transport: mockTransport}, + rateLimiter: rateLimiter, + invoiceCache: make(map[string][]fastlyplugin.Invoice), + } + + startDate := time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC) + endDate := time.Date(2024, 1, 31, 0, 0, 0, 0, time.UTC) + + // Act + invoices, err := fastlyCostSrc.getInvoicesForPeriod(&startDate, &endDate) + + // Assert + if err != nil { + t.Fatalf("Failed to get invoices for period: %v", err) + } + if len(invoices) != 2 { + t.Errorf("Expected 2 invoices from pagination, got %d", len(invoices)) + } + + foundFirst := false + foundSecond := false + for _, inv := range invoices { + if inv.InvoiceID == "inv-12345" { + foundFirst = true + } else if inv.InvoiceID == "inv-67890" { + foundSecond = true } } - t.Logf("Total responses: %d, Total costs: %d, Total billed: %.2f", len(resp), totalCosts, totalBilled) + + if !foundFirst { + t.Error("First invoice not found in results") + } + if !foundSecond { + t.Error("Second invoice not found in results") + } } -func TestConvertInvoiceToCosts(t *testing.T) { - fastlyCostSrc := FastlyCostSource{} - - // Create a test invoice - invoice := fastlyplugin.Invoice{ - CustomerID: "test-customer-123", - InvoiceID: "inv-123", - BillingStartDate: "2024-01-01T00:00:00Z", - BillingEndDate: "2024-01-31T23:59:59Z", - CurrencyCode: "USD", - TransactionLineItems: []fastlyplugin.TransactionLineItem{ - { - Description: "CDN Bandwidth", - Amount: 100.50, - Rate: 0.05, - Units: 2010, - ProductName: "CDN", - ProductGroup: "Full Site Delivery", - ProductLine: "Network Services", - Region: "Global", - UsageType: "bandwidth", - }, - { - Description: "Compute Requests", - Amount: 50.25, - Rate: 0.001, - Units: 50250, - ProductName: "Compute@Edge", - ProductGroup: "Compute", - ProductLine: "Edge Computing", - Region: "US-East", - UsageType: "requests", - }, +func TestGetFastlyCostsForWindow(t *testing.T) { + // Arrange + mockTransport := &mockHTTPTransport{ + responses: map[string]*http.Response{ + "https://api.fastly.com/billing/v3/invoices?billing_start_date=2024-01-01&billing_end_date=2024-01-31&limit=200": createMockResponse(200, `{ + "data": [ + { + "customer_id": "test-customer", + "invoice_id": "inv-12345", + "billing_start_date": "2024-01-01T00:00:00Z", + "billing_end_date": "2024-01-31T00:00:00Z", + "transaction_line_items": [ + { + "description": "CDN Bandwidth", + "amount": 100.50, + "rate": 0.05, + "units": 2010, + "product_name": "CDN", + "product_group": "Full Site Delivery", + "region": "Global", + "usage_type": "bandwidth" + } + ] + } + ], + "meta": { + "next_cursor": "", + "limit": 200, + "total": 1 + } + }`), }, } - // Create a window that overlaps with the invoice - windowStart := time.Date(2024, 1, 15, 0, 0, 0, 0, time.UTC) - windowEnd := time.Date(2024, 1, 16, 0, 0, 0, 0, time.UTC) - window := opencost.NewWindow(&windowStart, &windowEnd) + rateLimiter := rate.NewLimiter(rate.Every(time.Second), 10) + fastlyCostSrc := FastlyCostSource{ + apiKey: "test-api-key", + httpClient: &http.Client{Transport: mockTransport}, + rateLimiter: rateLimiter, + invoiceCache: make(map[string][]fastlyplugin.Invoice), + } + + startDate := time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC) + endDate := time.Date(2024, 1, 31, 23, 59, 59, 0, time.UTC) + window := opencost.NewWindow(&startDate, &endDate) - costs := fastlyCostSrc.convertInvoiceToCosts(invoice, window) + // Act + result := fastlyCostSrc.getFastlyCostsForWindow(window) - if len(costs) != 2 { - t.Errorf("expected 2 costs, got %d", len(costs)) + // Assert + if result == nil { + t.Fatal("Expected non-nil result") + } + if len(result.Costs) != 1 { + t.Errorf("Expected 1 cost item, got %d", len(result.Costs)) + } + if len(result.Errors) > 0 { + t.Errorf("Unexpected errors in response: %v", result.Errors) } - // Verify costs are prorated correctly - for _, cost := range costs { - if cost.BilledCost <= 0 { - t.Errorf("expected positive billed cost, got %f", cost.BilledCost) + // Check cost properties + if len(result.Costs) > 0 { + cost := result.Costs[0] + if cost.ResourceType != "Full Site Delivery" { + t.Errorf("Expected resource type 'Full Site Delivery', got '%s'", cost.ResourceType) } - if cost.UsageQuantity <= 0 { - t.Errorf("expected positive usage quantity, got %f", cost.UsageQuantity) - } - // Since we're getting 1 day out of 31, costs should be roughly 1/31 of the total - if cost.ResourceName == "bandwidth" && cost.BilledCost > 5 { - t.Errorf("bandwidth cost seems too high for 1-day proration: %f", cost.BilledCost) + if cost.UsageUnit != "GB" { + t.Errorf("Expected usage unit 'GB' for bandwidth, got '%s'", cost.UsageUnit) } } } -func TestGetUsageUnit(t *testing.T) { - tests := []struct { - usageType string - expected string - }{ - {"bandwidth", "GB"}, - {"requests", "requests"}, - {"compute_hours", "hours"}, - {"Bandwidth_GB", "GB"}, - {"API_Requests", "requests"}, - {"custom_metric", "units"}, - } - - for _, test := range tests { - result := getUsageUnit(test.usageType) - if result != test.expected { - t.Errorf("getUsageUnit(%s) = %s, expected %s", test.usageType, result, test.expected) - } +func TestBoilerplateFastlyCustomCost(t *testing.T) { + // Arrange + startDate := time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC) + endDate := time.Date(2024, 1, 31, 0, 0, 0, 0, time.UTC) + window := opencost.NewWindow(&startDate, &endDate) + + // Act + result := boilerplateFastlyCustomCost(window) + + // Assert + if result.Domain != "fastly" { + t.Errorf("Expected domain 'fastly', got '%s'", result.Domain) + } + if result.CostSource != "billing" { + t.Errorf("Expected cost source 'billing', got '%s'", result.CostSource) + } + if result.Currency != "USD" { + t.Errorf("Expected currency 'USD', got '%s'", result.Currency) + } + if result.Start.AsTime().Format(time.RFC3339) != startDate.Format(time.RFC3339) { + t.Errorf("Expected start time %s, got %s", startDate.Format(time.RFC3339), result.Start.AsTime().Format(time.RFC3339)) + } + if result.End.AsTime().Format(time.RFC3339) != endDate.Format(time.RFC3339) { + t.Errorf("Expected end time %s, got %s", endDate.Format(time.RFC3339), result.End.AsTime().Format(time.RFC3339)) + } + if len(result.Errors) != 0 { + t.Errorf("Expected empty errors, got %v", result.Errors) + } + if len(result.Costs) != 0 { + t.Errorf("Expected empty costs, got %d costs", len(result.Costs)) } } From 7f3c43caacaf46fd9e9268ac3c35932f33f7f2fa Mon Sep 17 00:00:00 2001 From: Sparsh Date: Tue, 5 Aug 2025 19:24:34 +0530 Subject: [PATCH 4/8] made harness tests work Signed-off-by: Sparsh --- pkg/plugins/fastly/cmd/main/main.go | 14 +++++++++++--- pkg/plugins/fastly/fastlyplugin/invoices.go | 2 +- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/pkg/plugins/fastly/cmd/main/main.go b/pkg/plugins/fastly/cmd/main/main.go index 39b709b..72dafa7 100644 --- a/pkg/plugins/fastly/cmd/main/main.go +++ b/pkg/plugins/fastly/cmd/main/main.go @@ -373,10 +373,18 @@ func boilerplateFastlyCustomCost(win opencost.Window) pb.CustomCostResponse { } func main() { - // Get config file path from environment variable or use default - configFile := os.Getenv("FASTLY_PLUGIN_CONFIG_FILE") + // Check command line args first + configFile := "" + if len(os.Args) > 1 { + configFile = os.Args[1] + } + + // If no command line args, try environment variable or default if configFile == "" { - configFile = "/opt/opencost/plugin/fastlyconfig.json" + configFile = os.Getenv("FASTLY_PLUGIN_CONFIG_FILE") + if configFile == "" { + configFile = "/opt/opencost/plugin/fastlyconfig.json" + } } fastlyConfig, err := getFastlyConfig(configFile) diff --git a/pkg/plugins/fastly/fastlyplugin/invoices.go b/pkg/plugins/fastly/fastlyplugin/invoices.go index 8c1d63f..8452686 100644 --- a/pkg/plugins/fastly/fastlyplugin/invoices.go +++ b/pkg/plugins/fastly/fastlyplugin/invoices.go @@ -15,7 +15,7 @@ type Invoice struct { BillingEndDate string `json:"billing_end_date"` StatementNumber string `json:"statement_number"` CurrencyCode string `json:"currency_code"` - MonthlyTransactionAmount float64 `json:"monthly_transaction_amount"` + MonthlyTransactionAmount string `json:"monthly_transaction_amount"` PaymentStatus string `json:"payment_status,omitempty"` TransactionLineItems []TransactionLineItem `json:"transaction_line_items"` } From 8d9b2f2137335338bcbc8c28c85c481e0bfb26e2 Mon Sep 17 00:00:00 2001 From: Sparsh Date: Thu, 7 Aug 2025 13:57:57 +0530 Subject: [PATCH 5/8] Enhance Fastly plugin with UUID generation and improved rate limiting This commit implements several key improvements to the Fastly OpenCost plugin: Core Enhancements: - Replace shared invoice IDs with unique UUIDs for each cost item using github.com/google/uuid - Update rate limiting from 600 to 5400 requests per minute to better utilize Fastly's 6000 req/min limit - Add comprehensive request validation with proper error handling for edge cases Performance & Reliability: - Implement invoice caching to reduce redundant API calls - Batch invoice fetching for entire periods instead of per-window requests - Add proper future date handling (return empty response instead of errors) - Enhanced error logging with more descriptive messages Testing Improvements: - Add comprehensive test coverage with TestUniqueUUIDGeneration - Replace transport mocking with HTTPClient interface for better testability - Add request validation tests covering all edge cases - Include test utilities and mock data generation tools API Integration: - Remove redundant Host headers from API requests - Improve charge categorization (commitment, credit, support, tax, usage) - Enhanced usage unit mapping for better cost attribution - Add support for credits/discounts with proper labeling Signed-off-by: Sparsh --- pkg/plugins/fastly/cmd/main/main.go | 260 +++++-- pkg/plugins/fastly/cmd/main/main_test.go | 728 ++++++++++++++---- pkg/plugins/fastly/cmd/validator/main/main.go | 8 +- pkg/plugins/fastly/fastlyplugin/invoices.go | 59 +- pkg/plugins/fastly/go.mod | 1 + pkg/plugins/fastly/go.sum | 4 +- .../fastly/tests/fastly_daily_test.json | 53 ++ .../fastly/tests/fastly_hourly_test.json | 34 + pkg/plugins/fastly/tests/mock_test.go | 347 +++++++++ pkg/plugins/fastly/utils/create_test_files.go | 146 ++++ .../fastly/utils/generate_test_data.go | 129 ++++ 11 files changed, 1580 insertions(+), 189 deletions(-) create mode 100644 pkg/plugins/fastly/tests/fastly_daily_test.json create mode 100644 pkg/plugins/fastly/tests/fastly_hourly_test.json create mode 100644 pkg/plugins/fastly/tests/mock_test.go create mode 100644 pkg/plugins/fastly/utils/create_test_files.go create mode 100644 pkg/plugins/fastly/utils/generate_test_data.go diff --git a/pkg/plugins/fastly/cmd/main/main.go b/pkg/plugins/fastly/cmd/main/main.go index 72dafa7..14676b6 100644 --- a/pkg/plugins/fastly/cmd/main/main.go +++ b/pkg/plugins/fastly/cmd/main/main.go @@ -12,6 +12,7 @@ import ( "sync" "time" + "github.com/google/uuid" "github.com/hashicorp/go-plugin" "github.com/opencost/opencost-plugins/pkg/plugins/fastly/fastlyplugin" "github.com/opencost/opencost/core/pkg/log" @@ -36,19 +37,94 @@ var handshakeConfig = plugin.HandshakeConfig{ MagicCookieValue: "fastly", } +// HTTPClient interface for better testability +type HTTPClient interface { + Do(req *http.Request) (*http.Response, error) +} + // Implementation of CustomCostSource type FastlyCostSource struct { apiKey string - httpClient *http.Client + httpClient HTTPClient // Changed from *http.Client to HTTPClient interface rateLimiter *rate.Limiter invoiceCache map[string][]fastlyplugin.Invoice invoiceCacheMux sync.Mutex } +// validateRequest validates the incoming request and returns any errors +func validateRequest(req *pb.CustomCostRequest) []string { + var errors []string + now := time.Now() + + // 1. Check if resolution is less than an hour (Fastly supports hourly data) + if req.Resolution.AsDuration() < time.Hour { + resolutionMessage := "Resolution should be at least one hour for Fastly billing data" + log.Warn(resolutionMessage) + errors = append(errors, resolutionMessage) + } + + // 2. Check if the date range is reasonable (not too far in the past) + // Fastly typically keeps detailed billing data for the last 12 months + twelveMonthsAgo := now.AddDate(0, -12, 0) + if req.Start.AsTime().Before(twelveMonthsAgo) { + startDateMessage := fmt.Sprintf("Start date is more than 12 months in the past. Fastly billing data may not be available before %s", + twelveMonthsAgo.Format("2006-01-02")) + log.Warn(startDateMessage) + errors = append(errors, startDateMessage) + } + + // 3. Check if end time is after start time + if req.End.AsTime().Before(req.Start.AsTime()) { + dateRangeMessage := "End date cannot be before start date" + log.Error(dateRangeMessage) + errors = append(errors, dateRangeMessage) + } + + // Note: Future date validation is handled earlier in GetCustomCosts to return empty response + + // 5. Warn if the date range is very large (performance consideration) + daysDiff := req.End.AsTime().Sub(req.Start.AsTime()).Hours() / 24 + if daysDiff > 90 { + performanceMessage := fmt.Sprintf("Large date range requested (%.0f days). This may take longer to process", daysDiff) + log.Info(performanceMessage) + // This is just a warning, not an error + } + + return errors +} + func (f *FastlyCostSource) GetCustomCosts(req *pb.CustomCostRequest) []*pb.CustomCostResponse { results := []*pb.CustomCostResponse{} + // Check if requesting future data - return empty response if so + now := time.Now().UTC() + if req.Start.AsTime().After(now) || req.End.AsTime().After(now.Add(time.Hour)) { + log.Debugf("skipping future window request: start=%v, end=%v", req.Start.AsTime(), req.End.AsTime()) + return results // Return empty array for future windows + } + + // Validate the request for other issues + requestErrors := validateRequest(req) + if len(requestErrors) > 0 { + // Return error response if validation fails + startTime := req.Start.AsTime() + endTime := req.End.AsTime() + errResp := boilerplateFastlyCustomCost(opencost.NewWindow(&startTime, &endTime)) + errResp.Errors = requestErrors + results = append(results, &errResp) + return results + } + targets, err := opencost.GetWindows(req.Start.AsTime(), req.End.AsTime(), req.Resolution.AsDuration()) + if err != nil { + log.Errorf("error getting windows: %v", err) + startTime := req.Start.AsTime() + endTime := req.End.AsTime() + errResp := boilerplateFastlyCustomCost(opencost.NewWindow(&startTime, &endTime)) + errResp.Errors = []string{fmt.Sprintf("error getting windows: %v", err)} + results = append(results, &errResp) + return results + } // Fetch all invoices at once for the entire period startTime := req.Start.AsTime() @@ -56,11 +132,9 @@ func (f *FastlyCostSource) GetCustomCosts(req *pb.CustomCostRequest) []*pb.Custo allInvoices, err := f.getInvoicesForPeriod(&startTime, &endTime) if err != nil { - log.Errorf("error getting windows: %v", err) - startTime := req.Start.AsTime() - endTime := req.End.AsTime() + log.Errorf("error fetching invoices for period: %v", err) errResp := boilerplateFastlyCustomCost(opencost.NewWindow(&startTime, &endTime)) - errResp.Errors = []string{fmt.Sprintf("error getting windows: %v", err)} + errResp.Errors = []string{fmt.Sprintf("error fetching invoices: %v", err)} results = append(results, &errResp) return results } @@ -79,27 +153,19 @@ func (f *FastlyCostSource) GetCustomCosts(req *pb.CustomCostRequest) []*pb.Custo } log.Debugf("fetching Fastly costs for window %v", target) - result := f.getFastlyCostsForWindow(target) + result := f.getFastlyCostsForWindow(target, allInvoices) results = append(results, result) } return results } -func (f *FastlyCostSource) getFastlyCostsForWindow(window opencost.Window) *pb.CustomCostResponse { +func (f *FastlyCostSource) getFastlyCostsForWindow(window opencost.Window, allInvoices []fastlyplugin.Invoice) *pb.CustomCostResponse { ccResp := boilerplateFastlyCustomCost(window) costs := []*pb.CustomCost{} - // Get invoices for the window period - invoices, err := f.getInvoicesForPeriod(window.Start(), window.End()) - if err != nil { - log.Errorf("error fetching invoices: %v", err) - ccResp.Errors = append(ccResp.Errors, err.Error()) - return &ccResp - } - - // Convert invoices to custom costs - for _, invoice := range invoices { + // Filter invoices that overlap with this window + for _, invoice := range allInvoices { invoiceCosts := f.convertInvoiceToCosts(invoice, window) costs = append(costs, invoiceCosts...) } @@ -109,10 +175,24 @@ func (f *FastlyCostSource) getFastlyCostsForWindow(window opencost.Window) *pb.C } func (f *FastlyCostSource) getInvoicesForPeriod(start, end *time.Time) ([]fastlyplugin.Invoice, error) { + // Check cache first + cacheKey := fmt.Sprintf("%s_%s", start.Format("2006-01-02"), end.Format("2006-01-02")) + f.invoiceCacheMux.Lock() + if cached, ok := f.invoiceCache[cacheKey]; ok { + f.invoiceCacheMux.Unlock() + log.Debugf("returning cached invoices for period %s to %s", start.Format("2006-01-02"), end.Format("2006-01-02")) + return cached, nil + } + f.invoiceCacheMux.Unlock() + allInvoices := []fastlyplugin.Invoice{} cursor := "" hasMore := true + // Format dates for API - ensure we use proper date format + startStr := start.Format("2006-01-02") + endStr := end.Format("2006-01-02") + for hasMore { // Rate limiting if f.rateLimiter.Tokens() < 1.0 { @@ -126,14 +206,16 @@ func (f *FastlyCostSource) getInvoicesForPeriod(start, end *time.Time) ([]fastly // Build request URL reqURL := fmt.Sprintf("%s/billing/v3/invoices", fastlyAPIBaseURL) params := url.Values{} - params.Add("billing_start_date", start.Format("2006-01-02")) - params.Add("billing_end_date", end.Format("2006-01-02")) + params.Add("billing_start_date", startStr) + params.Add("billing_end_date", endStr) params.Add("limit", "200") if cursor != "" { params.Add("cursor", cursor) } reqURL = fmt.Sprintf("%s?%s", reqURL, params.Encode()) + log.Debugf("Fetching invoices from: %s", reqURL) + // Make request req, err := http.NewRequest("GET", reqURL, nil) if err != nil { @@ -141,7 +223,6 @@ func (f *FastlyCostSource) getInvoicesForPeriod(start, end *time.Time) ([]fastly } req.Header.Set("Accept", "application/json") req.Header.Set("Fastly-Key", f.apiKey) - req.Header.Set("Host", "api.fastly.com") resp, err := f.httpClient.Do(req) if err != nil { @@ -160,6 +241,7 @@ func (f *FastlyCostSource) getInvoicesForPeriod(start, end *time.Time) ([]fastly return nil, fmt.Errorf("error decoding response: %v", err) } + log.Debugf("Retrieved %d invoices, total: %d", len(invoiceResp.Data), invoiceResp.Meta.Total) allInvoices = append(allInvoices, invoiceResp.Data...) // Check for more pages @@ -177,10 +259,17 @@ func (f *FastlyCostSource) getInvoicesForPeriod(start, end *time.Time) ([]fastly if err != nil { log.Warnf("error fetching month-to-date invoice: %v", err) } else if mtdInvoice != nil { + log.Debugf("Including month-to-date invoice: %s", mtdInvoice.InvoiceID) allInvoices = append(allInvoices, *mtdInvoice) } } + // Store in cache + f.invoiceCacheMux.Lock() + f.invoiceCache[cacheKey] = allInvoices + f.invoiceCacheMux.Unlock() + + log.Infof("Total invoices retrieved for period %s to %s: %d", startStr, endStr, len(allInvoices)) return allInvoices, nil } @@ -201,7 +290,6 @@ func (f *FastlyCostSource) getMonthToDateInvoice() (*fastlyplugin.Invoice, error } req.Header.Set("Accept", "application/json") req.Header.Set("Fastly-Key", f.apiKey) - req.Header.Set("Host", "api.fastly.com") resp, err := f.httpClient.Do(req) if err != nil { @@ -239,7 +327,6 @@ func (f *FastlyCostSource) getInvoiceByID(invoiceID string) (*fastlyplugin.Invoi } req.Header.Set("Accept", "application/json") req.Header.Set("Fastly-Key", f.apiKey) - req.Header.Set("Host", "api.fastly.com") resp, err := f.httpClient.Do(req) if err != nil { @@ -266,12 +353,12 @@ func (f *FastlyCostSource) convertInvoiceToCosts(invoice fastlyplugin.Invoice, w // Parse invoice dates startDate, err := fastlyplugin.ParseFastlyDate(invoice.BillingStartDate) if err != nil { - log.Errorf("error parsing billing start date: %v", err) + log.Errorf("error parsing billing start date for invoice %s: %v", invoice.InvoiceID, err) return costs } endDate, err := fastlyplugin.ParseFastlyDate(invoice.BillingEndDate) if err != nil { - log.Errorf("error parsing billing end date: %v", err) + log.Errorf("error parsing billing end date for invoice %s: %v", invoice.InvoiceID, err) return costs } @@ -302,10 +389,13 @@ func (f *FastlyCostSource) convertInvoiceToCosts(invoice fastlyplugin.Invoice, w prorateRatio = float32(overlapHours / totalInvoiceHours) } + log.Debugf("Invoice %s: overlap %.2f hours of %.2f total hours (%.2f%%)", + invoice.InvoiceID, overlapHours, totalInvoiceHours, prorateRatio*100) + // Convert each line item to a cost for _, item := range invoice.TransactionLineItems { - // Skip zero-amount items - if item.Amount == 0 { + // Skip zero-amount items unless they have units (could be usage tracking) + if item.Amount == 0 && item.Units == 0 { continue } @@ -316,25 +406,39 @@ func (f *FastlyCostSource) convertInvoiceToCosts(invoice fastlyplugin.Invoice, w billedCost := float32(item.Amount) * prorateRatio usageQuantity := float32(item.Units) * prorateRatio + // Handle region - default to "Global" if empty + region := item.Region + if region == "" { + region = "Global" + } + cost := &pb.CustomCost{ - Zone: item.Region, + Zone: region, AccountName: invoice.CustomerID, - ChargeCategory: "usage", + ChargeCategory: getChargeCategory(item), Description: item.Description, - ResourceName: item.UsageType, - ResourceType: item.ProductGroup, - Id: invoice.InvoiceID, + ResourceName: item.UsageType, // UsageType goes to ResourceName + ResourceType: item.ProductGroup, // ProductGroup goes to ResourceType + Id: uuid.New().String(), ProviderId: providerID, Labels: map[string]string{ "product_line": item.ProductLine, "product_name": item.ProductName, "credit_coupon_code": item.CreditCouponCode, + "currency": invoice.CurrencyCode, }, ListCost: billedCost, ListUnitPrice: float32(item.Rate), BilledCost: billedCost, UsageQuantity: usageQuantity, - UsageUnit: getUsageUnit(item.UsageType), + UsageUnit: getUsageUnit(item.UsageType, item.ProductName), + } + + // Add additional label for credits/discounts + if item.Amount < 0 { + cost.Labels["cost_type"] = "credit" + } else { + cost.Labels["cost_type"] = "charge" } costs = append(costs, cost) @@ -343,24 +447,76 @@ func (f *FastlyCostSource) convertInvoiceToCosts(invoice fastlyplugin.Invoice, w return costs } -func getUsageUnit(usageType string) string { - // Map common usage types to units - usageType = strings.ToLower(usageType) - if strings.Contains(usageType, "bandwidth") { +// Enhanced getUsageUnit function with more comprehensive mapping +func getUsageUnit(usageType string, productName string) string { + // Normalize to lowercase for comparison + usageTypeLower := strings.ToLower(usageType) + productNameLower := strings.ToLower(productName) + + // Check usage type first + switch { + case strings.Contains(usageTypeLower, "bandwidth"): return "GB" - } - if strings.Contains(usageType, "request") { + case strings.Contains(usageTypeLower, "request"): return "requests" - } - if strings.Contains(usageType, "compute") { + case strings.Contains(usageTypeLower, "compute"): + return "compute-hours" + case strings.Contains(usageTypeLower, "storage"): + return "GB" + case strings.Contains(usageTypeLower, "committed amount"): + return "USD" + case strings.Contains(usageTypeLower, "minute"): + return "minutes" + case strings.Contains(usageTypeLower, "hour"): return "hours" + case strings.Contains(usageTypeLower, "invocation"): + return "invocations" + case strings.Contains(usageTypeLower, "log"): + return "log-lines" + } + + // Check product name as fallback + switch { + case strings.Contains(productNameLower, "cdn"): + return "GB" + case strings.Contains(productNameLower, "compute"): + return "compute-hours" + case strings.Contains(productNameLower, "waf") || strings.Contains(productNameLower, "security"): + return "requests" + case strings.Contains(productNameLower, "image"): + return "transformations" + case strings.Contains(productNameLower, "video"): + return "minutes" } + + // Default return "units" } +// getChargeCategory determines the charge category based on the line item +func getChargeCategory(item fastlyplugin.TransactionLineItem) string { + descLower := strings.ToLower(item.Description) + + switch { + case strings.Contains(descLower, "minimum"): + return "commitment" + case strings.Contains(descLower, "credit") || item.Amount < 0: + return "credit" + case strings.Contains(descLower, "support"): + return "support" + case strings.Contains(descLower, "tax"): + return "tax" + default: + return "usage" + } +} + func boilerplateFastlyCustomCost(win opencost.Window) pb.CustomCostResponse { return pb.CustomCostResponse{ - Metadata: map[string]string{"api_client_version": "v3"}, + Metadata: map[string]string{ + "api_client_version": "v3", + "plugin_version": "v1.1.0", // Bumped version for Phase 1 enhancements + }, CostSource: "billing", Domain: "fastly", Version: "v1", @@ -372,6 +528,13 @@ func boilerplateFastlyCustomCost(win opencost.Window) pb.CustomCostResponse { } } +// getFastlyHTTPClient returns an HTTP client for Fastly API +func getFastlyHTTPClient() HTTPClient { + return &http.Client{ + Timeout: 30 * time.Second, + } +} + func main() { // Check command line args first configFile := "" @@ -393,12 +556,18 @@ func main() { } log.SetLogLevel(fastlyConfig.LogLevel) - // Fastly rate limiting - be conservative - rateLimiter := rate.NewLimiter(rate.Every(time.Second), 10) + // Validate API key + if fastlyConfig.FastlyAPIKey == "" { + log.Fatalf("Fastly API key is required but not provided in config") + } + + // Fastly rate limiting - 6000 requests per minute = 100 requests per second + // Being slightly conservative at 90 requests per second to avoid hitting limits + rateLimiter := rate.NewLimiter(rate.Limit(90), 100) fastlyCostSrc := FastlyCostSource{ apiKey: fastlyConfig.FastlyAPIKey, - httpClient: &http.Client{Timeout: 30 * time.Second}, + httpClient: getFastlyHTTPClient(), // Use the new function rateLimiter: rateLimiter, invoiceCache: make(map[string][]fastlyplugin.Invoice), } @@ -408,6 +577,7 @@ func main() { "CustomCostSource": &ocplugin.CustomCostPlugin{Impl: &fastlyCostSrc}, } + log.Infof("Starting Fastly plugin server v1.1.0...") plugin.Serve(&plugin.ServeConfig{ HandshakeConfig: handshakeConfig, Plugins: pluginMap, @@ -423,7 +593,7 @@ func getFastlyConfig(configFilePath string) (*fastlyplugin.FastlyConfig, error) } err = json.Unmarshal(bytes, &result) if err != nil { - return nil, fmt.Errorf("error marshaling json into Fastly config %v", err) + return nil, fmt.Errorf("error marshaling json into Fastly config: %v", err) } if result.LogLevel == "" { diff --git a/pkg/plugins/fastly/cmd/main/main_test.go b/pkg/plugins/fastly/cmd/main/main_test.go index 67f1491..cbd60b9 100644 --- a/pkg/plugins/fastly/cmd/main/main_test.go +++ b/pkg/plugins/fastly/cmd/main/main_test.go @@ -4,7 +4,7 @@ import ( "fmt" "io" "net/http" - "net/url" + _ "net/url" "os" "path/filepath" "strings" @@ -12,82 +12,133 @@ import ( "time" "github.com/opencost/opencost-plugins/pkg/plugins/fastly/fastlyplugin" + "github.com/opencost/opencost/core/pkg/model/pb" "github.com/opencost/opencost/core/pkg/opencost" "golang.org/x/time/rate" + "google.golang.org/protobuf/types/known/durationpb" + "google.golang.org/protobuf/types/known/timestamppb" ) -// mockHTTPTransport implements http.RoundTripper for testing -type mockHTTPTransport struct { - responses map[string]*http.Response +// MockHTTPClient implements HTTPClient interface for testing +type MockHTTPClient struct { + DoFunc func(req *http.Request) (*http.Response, error) } -// URL query parameter order-insensitive matching -func (m *mockHTTPTransport) RoundTrip(req *http.Request) (*http.Response, error) { - // Try exact match first - if response, ok := m.responses[req.URL.String()]; ok { - return response, nil +func (m *MockHTTPClient) Do(req *http.Request) (*http.Response, error) { + if m.DoFunc != nil { + return m.DoFunc(req) } - - // If exact match fails, try matching by normalized URL - // (comparing host, path and query parameters regardless of order) - requestURL := req.URL.String() - for mockURL, response := range m.responses { - if haveSameParams(mockURL, requestURL) { - return response, nil - } - } - - return nil, fmt.Errorf("no mock response found for URL: %s", req.URL.String()) + return nil, fmt.Errorf("DoFunc not implemented") } -// Helper function to check if two URLs have the same parameters regardless of order -func haveSameParams(url1, url2 string) bool { - u1, err := url.Parse(url1) - if err != nil { - return false - } - u2, err := url.Parse(url2) - if err != nil { - return false +// createMockResponse creates a mock HTTP response with given status code and body +func createMockResponse(statusCode int, body string) *http.Response { + return &http.Response{ + StatusCode: statusCode, + Body: io.NopCloser(strings.NewReader(body)), + Header: make(http.Header), } +} - // Check if host and path match - if u1.Host != u2.Host || u1.Path != u2.Path { - return false +// Test request validation +func TestValidateRequest(t *testing.T) { + now := time.Now() + + tests := []struct { + name string + req *pb.CustomCostRequest + expectedErrors int + errorContains []string + }{ + { + name: "Valid request", + req: &pb.CustomCostRequest{ + Start: timestamppb.New(now.AddDate(0, -1, 0)), + End: timestamppb.New(now), + Resolution: durationpb.New(24 * time.Hour), + }, + expectedErrors: 0, + errorContains: []string{}, + }, + { + name: "Resolution less than an hour", + req: &pb.CustomCostRequest{ + Start: timestamppb.New(now.AddDate(0, -1, 0)), + End: timestamppb.New(now), + Resolution: durationpb.New(30 * time.Minute), + }, + expectedErrors: 1, + errorContains: []string{"Resolution should be at least one hour"}, + }, + { + name: "Start date more than 12 months ago", + req: &pb.CustomCostRequest{ + Start: timestamppb.New(now.AddDate(0, -13, 0)), + End: timestamppb.New(now), + Resolution: durationpb.New(24 * time.Hour), + }, + expectedErrors: 1, + errorContains: []string{"more than 12 months in the past"}, + }, + { + name: "End date before start date", + req: &pb.CustomCostRequest{ + Start: timestamppb.New(now), + End: timestamppb.New(now.AddDate(0, -1, 0)), + Resolution: durationpb.New(24 * time.Hour), + }, + expectedErrors: 1, + errorContains: []string{"End date cannot be before start date"}, + }, + { + name: "Future dates requested", + req: &pb.CustomCostRequest{ + Start: timestamppb.New(now.AddDate(0, 1, 0)), + End: timestamppb.New(now.AddDate(0, 2, 0)), + Resolution: durationpb.New(24 * time.Hour), + }, + expectedErrors: 0, // Future dates now return empty response instead of error + errorContains: []string{}, + }, + { + name: "Multiple validation errors", + req: &pb.CustomCostRequest{ + Start: timestamppb.New(now.AddDate(0, -13, 0)), // Too far in past + End: timestamppb.New(now.AddDate(0, -14, 0)), // Before start + Resolution: durationpb.New(30 * time.Minute), // Too small + }, + expectedErrors: 3, + errorContains: []string{ + "Resolution should be at least one hour", + "more than 12 months in the past", + "End date cannot be before start date", + }, + }, } - // Check if query parameters are the same (regardless of order) - q1 := u1.Query() - q2 := u2.Query() + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + errors := validateRequest(tt.req) - if len(q1) != len(q2) { - return false - } - - for k, v1 := range q1 { - v2, ok := q2[k] - if !ok { - return false - } - if len(v1) != len(v2) { - return false - } - for i := range v1 { - if v1[i] != v2[i] { - return false + if len(errors) != tt.expectedErrors { + t.Errorf("Expected %d errors, got %d. Errors: %v", + tt.expectedErrors, len(errors), errors) } - } - } - return true -} - -// createMockResponse creates a mock HTTP response with given status code and body -func createMockResponse(statusCode int, body string) *http.Response { - return &http.Response{ - StatusCode: statusCode, - Body: io.NopCloser(strings.NewReader(body)), - Header: make(http.Header), + for _, expectedContains := range tt.errorContains { + found := false + for _, err := range errors { + if strings.Contains(err, expectedContains) { + found = true + break + } + } + if !found { + t.Errorf("Expected error containing '%s' not found in %v", + expectedContains, errors) + } + } + }) } } @@ -186,9 +237,20 @@ func TestGetFastlyConfigInvalidJSON(t *testing.T) { func TestGetMonthToDateInvoice(t *testing.T) { // Arrange - mockTransport := &mockHTTPTransport{ - responses: map[string]*http.Response{ - "https://api.fastly.com/billing/v3/invoices/month-to-date": createMockResponse(200, `{ + mockClient := &MockHTTPClient{ + DoFunc: func(req *http.Request) (*http.Response, error) { + // Verify the request + if req.Method != "GET" { + t.Errorf("Expected GET request, got %s", req.Method) + } + if !strings.Contains(req.URL.Path, "/billing/v3/invoices/month-to-date") { + t.Errorf("Expected month-to-date endpoint, got %s", req.URL.Path) + } + if req.Header.Get("Fastly-Key") != "test-api-key" { + t.Errorf("Expected API key in header") + } + + return createMockResponse(200, `{ "customer_id": "test-customer", "invoice_id": "mtd-12345", "billing_start_date": "2024-01-01T00:00:00Z", @@ -208,14 +270,14 @@ func TestGetMonthToDateInvoice(t *testing.T) { "usage_type": "bandwidth" } ] - }`), + }`), nil }, } rateLimiter := rate.NewLimiter(rate.Every(time.Second), 10) fastlyCostSrc := FastlyCostSource{ apiKey: "test-api-key", - httpClient: &http.Client{Transport: mockTransport}, + httpClient: mockClient, rateLimiter: rateLimiter, invoiceCache: make(map[string][]fastlyplugin.Invoice), } @@ -240,19 +302,19 @@ func TestGetMonthToDateInvoice(t *testing.T) { func TestGetMonthToDateInvoiceError(t *testing.T) { // Arrange - mockTransport := &mockHTTPTransport{ - responses: map[string]*http.Response{ - "https://api.fastly.com/billing/v3/invoices/month-to-date": createMockResponse(403, `{ + mockClient := &MockHTTPClient{ + DoFunc: func(req *http.Request) (*http.Response, error) { + return createMockResponse(403, `{ "msg": "Unauthorized", "detail": "Invalid API key" - }`), + }`), nil }, } rateLimiter := rate.NewLimiter(rate.Every(time.Second), 10) fastlyCostSrc := FastlyCostSource{ apiKey: "invalid-api-key", - httpClient: &http.Client{Transport: mockTransport}, + httpClient: mockClient, rateLimiter: rateLimiter, invoiceCache: make(map[string][]fastlyplugin.Invoice), } @@ -271,9 +333,18 @@ func TestGetMonthToDateInvoiceError(t *testing.T) { func TestGetInvoicesForPeriod(t *testing.T) { // Arrange - mockTransport := &mockHTTPTransport{ - responses: map[string]*http.Response{ - "https://api.fastly.com/billing/v3/invoices?billing_start_date=2024-01-01&billing_end_date=2024-01-31&limit=200": createMockResponse(200, `{ + mockClient := &MockHTTPClient{ + DoFunc: func(req *http.Request) (*http.Response, error) { + // Verify request parameters + query := req.URL.Query() + if query.Get("billing_start_date") != "2024-01-01" { + t.Errorf("Expected billing_start_date=2024-01-01, got %s", query.Get("billing_start_date")) + } + if query.Get("billing_end_date") != "2024-01-31" { + t.Errorf("Expected billing_end_date=2024-01-31, got %s", query.Get("billing_end_date")) + } + + return createMockResponse(200, `{ "data": [ { "customer_id": "test-customer", @@ -303,14 +374,14 @@ func TestGetInvoicesForPeriod(t *testing.T) { "total": 1, "sort": "billing_start_date" } - }`), + }`), nil }, } rateLimiter := rate.NewLimiter(rate.Every(time.Second), 10) fastlyCostSrc := FastlyCostSource{ apiKey: "test-api-key", - httpClient: &http.Client{Transport: mockTransport}, + httpClient: mockClient, rateLimiter: rateLimiter, invoiceCache: make(map[string][]fastlyplugin.Invoice), } @@ -335,69 +406,83 @@ func TestGetInvoicesForPeriod(t *testing.T) { func TestGetInvoicesForPeriodWithPagination(t *testing.T) { // Arrange - mockTransport := &mockHTTPTransport{ - responses: map[string]*http.Response{ - // First page - "https://api.fastly.com/billing/v3/invoices?billing_start_date=2024-01-01&billing_end_date=2024-01-31&limit=200": createMockResponse(200, `{ - "data": [ - { - "customer_id": "test-customer", - "invoice_id": "inv-12345", - "billing_start_date": "2024-01-01T00:00:00Z", - "billing_end_date": "2024-01-31T00:00:00Z", - "transaction_line_items": [ - { - "description": "CDN Bandwidth", - "amount": 100.50, - "rate": 0.05, - "units": 2010, - "product_name": "CDN", - "product_group": "Full Site Delivery", - "usage_type": "bandwidth" - } - ] + callCount := 0 + mockClient := &MockHTTPClient{ + DoFunc: func(req *http.Request) (*http.Response, error) { + callCount++ + query := req.URL.Query() + + if callCount == 1 { + // First page + if query.Get("cursor") != "" { + t.Errorf("Expected no cursor on first call") + } + return createMockResponse(200, `{ + "data": [ + { + "customer_id": "test-customer", + "invoice_id": "inv-12345", + "billing_start_date": "2024-01-01T00:00:00Z", + "billing_end_date": "2024-01-31T00:00:00Z", + "transaction_line_items": [ + { + "description": "CDN Bandwidth", + "amount": 100.50, + "rate": 0.05, + "units": 2010, + "product_name": "CDN", + "product_group": "Full Site Delivery", + "usage_type": "bandwidth" + } + ] + } + ], + "meta": { + "next_cursor": "page2", + "limit": 200, + "total": 2 } - ], - "meta": { - "next_cursor": "page2", - "limit": 200, - "total": 2 - } - }`), - // Second page - "https://api.fastly.com/billing/v3/invoices?billing_start_date=2024-01-01&billing_end_date=2024-01-31&limit=200&cursor=page2": createMockResponse(200, `{ - "data": [ - { - "customer_id": "test-customer", - "invoice_id": "inv-67890", - "billing_start_date": "2024-01-01T00:00:00Z", - "billing_end_date": "2024-01-31T00:00:00Z", - "transaction_line_items": [ - { - "description": "Compute Requests", - "amount": 75.25, - "rate": 0.001, - "units": 75250, - "product_name": "Compute", - "product_group": "Compute", - "usage_type": "requests" - } - ] + }`), nil + } else if callCount == 2 { + // Second page + if query.Get("cursor") != "page2" { + t.Errorf("Expected cursor=page2, got %s", query.Get("cursor")) + } + return createMockResponse(200, `{ + "data": [ + { + "customer_id": "test-customer", + "invoice_id": "inv-67890", + "billing_start_date": "2024-01-01T00:00:00Z", + "billing_end_date": "2024-01-31T00:00:00Z", + "transaction_line_items": [ + { + "description": "Compute Requests", + "amount": 75.25, + "rate": 0.001, + "units": 75250, + "product_name": "Compute", + "product_group": "Compute", + "usage_type": "requests" + } + ] + } + ], + "meta": { + "next_cursor": "", + "limit": 200, + "total": 2 } - ], - "meta": { - "next_cursor": "", - "limit": 200, - "total": 2 - } - }`), + }`), nil + } + return nil, fmt.Errorf("unexpected call count: %d", callCount) }, } rateLimiter := rate.NewLimiter(rate.Every(time.Second), 10) fastlyCostSrc := FastlyCostSource{ apiKey: "test-api-key", - httpClient: &http.Client{Transport: mockTransport}, + httpClient: mockClient, rateLimiter: rateLimiter, invoiceCache: make(map[string][]fastlyplugin.Invoice), } @@ -415,6 +500,9 @@ func TestGetInvoicesForPeriodWithPagination(t *testing.T) { if len(invoices) != 2 { t.Errorf("Expected 2 invoices from pagination, got %d", len(invoices)) } + if callCount != 2 { + t.Errorf("Expected 2 API calls for pagination, got %d", callCount) + } foundFirst := false foundSecond := false @@ -436,9 +524,9 @@ func TestGetInvoicesForPeriodWithPagination(t *testing.T) { func TestGetFastlyCostsForWindow(t *testing.T) { // Arrange - mockTransport := &mockHTTPTransport{ - responses: map[string]*http.Response{ - "https://api.fastly.com/billing/v3/invoices?billing_start_date=2024-01-01&billing_end_date=2024-01-31&limit=200": createMockResponse(200, `{ + mockClient := &MockHTTPClient{ + DoFunc: func(req *http.Request) (*http.Response, error) { + return createMockResponse(200, `{ "data": [ { "customer_id": "test-customer", @@ -464,14 +552,14 @@ func TestGetFastlyCostsForWindow(t *testing.T) { "limit": 200, "total": 1 } - }`), + }`), nil }, } rateLimiter := rate.NewLimiter(rate.Every(time.Second), 10) fastlyCostSrc := FastlyCostSource{ apiKey: "test-api-key", - httpClient: &http.Client{Transport: mockTransport}, + httpClient: mockClient, rateLimiter: rateLimiter, invoiceCache: make(map[string][]fastlyplugin.Invoice), } @@ -481,7 +569,12 @@ func TestGetFastlyCostsForWindow(t *testing.T) { window := opencost.NewWindow(&startDate, &endDate) // Act - result := fastlyCostSrc.getFastlyCostsForWindow(window) + // First get invoices for the period to pass to getFastlyCostsForWindow + invoices, err := fastlyCostSrc.getInvoicesForPeriod(&startDate, &endDate) + if err != nil { + t.Fatalf("Failed to get invoices: %v", err) + } + result := fastlyCostSrc.getFastlyCostsForWindow(window, invoices) // Assert if result == nil { @@ -506,6 +599,206 @@ func TestGetFastlyCostsForWindow(t *testing.T) { } } +// Test GetCustomCosts with validation errors +func TestGetCustomCostsWithValidationErrors(t *testing.T) { + mockClient := &MockHTTPClient{ + DoFunc: func(req *http.Request) (*http.Response, error) { + // Should not be called due to validation errors + t.Error("HTTP client should not be called when validation fails") + return nil, fmt.Errorf("should not be called") + }, + } + + rateLimiter := rate.NewLimiter(rate.Every(time.Second), 10) + fastlyCostSrc := FastlyCostSource{ + apiKey: "test-api-key", + httpClient: mockClient, + rateLimiter: rateLimiter, + invoiceCache: make(map[string][]fastlyplugin.Invoice), + } + + // Create request with multiple validation errors (non-future) + now := time.Now() + req := &pb.CustomCostRequest{ + Start: timestamppb.New(now.AddDate(0, -13, 0)), // Too far in past + End: timestamppb.New(now.AddDate(0, -14, 0)), // Before start + Resolution: durationpb.New(30 * time.Minute), // Too small + } + + // Act + responses := fastlyCostSrc.GetCustomCosts(req) + + // Assert + if len(responses) != 1 { + t.Fatalf("Expected 1 response with errors, got %d", len(responses)) + } + + if len(responses[0].Errors) != 3 { + t.Errorf("Expected 3 validation errors, got %d: %v", + len(responses[0].Errors), responses[0].Errors) + } + + // Verify error messages + expectedErrors := []string{ + "Resolution should be at least one hour", + "more than 12 months in the past", + "End date cannot be before start date", + } + + for _, expected := range expectedErrors { + found := false + for _, err := range responses[0].Errors { + if strings.Contains(err, expected) { + found = true + break + } + } + if !found { + t.Errorf("Expected error containing '%s' not found in %v", + expected, responses[0].Errors) + } + } +} + +// Test GetCustomCosts with successful response +func TestGetCustomCostsSuccess(t *testing.T) { + // Create valid request with recent dates (within last 12 months) + now := time.Now() + startDate := now.AddDate(0, -1, 0).Truncate(24 * time.Hour) // 1 month ago + endDate := startDate.Add(24 * time.Hour) // 1 day later + + mockClient := &MockHTTPClient{ + DoFunc: func(req *http.Request) (*http.Response, error) { + // Use the dynamic start and end dates for the response + responseBody := fmt.Sprintf(`{ + "data": [ + { + "customer_id": "test-customer", + "invoice_id": "inv-12345", + "billing_start_date": "%s", + "billing_end_date": "%s", + "currency_code": "USD", + "monthly_transaction_amount": 100.50, + "transaction_line_items": [ + { + "description": "CDN Bandwidth", + "amount": 100.50, + "rate": 0.05, + "units": 2010, + "product_name": "CDN", + "product_group": "Full Site Delivery", + "region": "Global", + "usage_type": "bandwidth" + } + ] + } + ], + "meta": { + "next_cursor": "", + "limit": 200, + "total": 1 + } + }`, startDate.Format(time.RFC3339), endDate.Format(time.RFC3339)) + return createMockResponse(200, responseBody), nil + }, + } + + rateLimiter := rate.NewLimiter(rate.Every(time.Second), 10) + fastlyCostSrc := FastlyCostSource{ + apiKey: "test-api-key", + httpClient: mockClient, + rateLimiter: rateLimiter, + invoiceCache: make(map[string][]fastlyplugin.Invoice), + } + req := &pb.CustomCostRequest{ + Start: timestamppb.New(startDate), + End: timestamppb.New(endDate), + Resolution: durationpb.New(24 * time.Hour), + } + + // Act + responses := fastlyCostSrc.GetCustomCosts(req) + + // Assert + if len(responses) != 1 { + t.Fatalf("Expected 1 response, got %d", len(responses)) + } + + if len(responses[0].Errors) > 0 { + t.Errorf("Expected no errors, got: %v", responses[0].Errors) + } + + if responses[0].Domain != "fastly" { + t.Errorf("Expected domain 'fastly', got %s", responses[0].Domain) + } + + if len(responses[0].Costs) != 1 { + t.Errorf("Expected 1 cost, got %d", len(responses[0].Costs)) + } +} + +// Test caching behavior +func TestInvoiceCaching(t *testing.T) { + callCount := 0 + mockClient := &MockHTTPClient{ + DoFunc: func(req *http.Request) (*http.Response, error) { + callCount++ + return createMockResponse(200, `{ + "data": [ + { + "customer_id": "test-customer", + "invoice_id": "inv-cached", + "billing_start_date": "2024-01-01T00:00:00Z", + "billing_end_date": "2024-01-31T00:00:00Z", + "transaction_line_items": [] + } + ], + "meta": { + "next_cursor": "", + "limit": 200, + "total": 1 + } + }`), nil + }, + } + + rateLimiter := rate.NewLimiter(rate.Every(time.Second), 10) + fastlyCostSrc := FastlyCostSource{ + apiKey: "test-api-key", + httpClient: mockClient, + rateLimiter: rateLimiter, + invoiceCache: make(map[string][]fastlyplugin.Invoice), + } + + startDate := time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC) + endDate := time.Date(2024, 1, 31, 0, 0, 0, 0, time.UTC) + + // First call - should hit the API + invoices1, err1 := fastlyCostSrc.getInvoicesForPeriod(&startDate, &endDate) + if err1 != nil { + t.Fatalf("First call failed: %v", err1) + } + + // Second call - should use cache + invoices2, err2 := fastlyCostSrc.getInvoicesForPeriod(&startDate, &endDate) + if err2 != nil { + t.Fatalf("Second call failed: %v", err2) + } + + // Assert + if callCount != 1 { + t.Errorf("Expected 1 API call (second should use cache), got %d", callCount) + } + + if len(invoices1) != len(invoices2) { + t.Errorf("Cached response differs from original") + } + + if invoices1[0].InvoiceID != invoices2[0].InvoiceID { + t.Errorf("Cached invoice ID differs from original") + } +} + func TestBoilerplateFastlyCustomCost(t *testing.T) { // Arrange startDate := time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC) @@ -525,6 +818,9 @@ func TestBoilerplateFastlyCustomCost(t *testing.T) { if result.Currency != "USD" { t.Errorf("Expected currency 'USD', got '%s'", result.Currency) } + if result.Metadata["plugin_version"] != "v1.1.0" { + t.Errorf("Expected plugin version 'v1.1.0', got '%s'", result.Metadata["plugin_version"]) + } if result.Start.AsTime().Format(time.RFC3339) != startDate.Format(time.RFC3339) { t.Errorf("Expected start time %s, got %s", startDate.Format(time.RFC3339), result.Start.AsTime().Format(time.RFC3339)) } @@ -538,3 +834,161 @@ func TestBoilerplateFastlyCustomCost(t *testing.T) { t.Errorf("Expected empty costs, got %d costs", len(result.Costs)) } } + +// Test GetCustomCosts with future dates returns empty response +func TestGetCustomCostsFutureRequest(t *testing.T) { + mockClient := &MockHTTPClient{ + DoFunc: func(req *http.Request) (*http.Response, error) { + // Should not be called for future requests + t.Error("HTTP client should not be called for future requests") + return nil, fmt.Errorf("should not be called") + }, + } + + rateLimiter := rate.NewLimiter(rate.Every(time.Second), 10) + fastlyCostSrc := FastlyCostSource{ + apiKey: "test-api-key", + httpClient: mockClient, + rateLimiter: rateLimiter, + invoiceCache: make(map[string][]fastlyplugin.Invoice), + } + + // Create request for future dates + now := time.Now().UTC().Truncate(time.Hour) + req := &pb.CustomCostRequest{ + Start: timestamppb.New(now.Add(time.Hour)), // 1 hour in the future + End: timestamppb.New(now.Add(2 * time.Hour)), // 2 hours in the future + Resolution: durationpb.New(time.Hour), + } + + // Act + responses := fastlyCostSrc.GetCustomCosts(req) + + // Assert + if len(responses) != 0 { + t.Fatalf("Expected empty response for future request, got %d responses", len(responses)) + } +} + +// Test that each cost item gets a unique UUID +func TestUniqueUUIDGeneration(t *testing.T) { + // Create valid request + now := time.Now() + startDate := now.AddDate(0, -1, 0).Truncate(24 * time.Hour) + endDate := startDate.Add(24 * time.Hour) + + mockClient := &MockHTTPClient{ + DoFunc: func(req *http.Request) (*http.Response, error) { + // Return mock response with multiple line items from the same invoice + responseBody := fmt.Sprintf(`{ + "data": [ + { + "customer_id": "test-customer", + "invoice_id": "inv-12345", + "billing_start_date": "%s", + "billing_end_date": "%s", + "currency_code": "USD", + "transaction_line_items": [`, startDate.Format(time.RFC3339), endDate.Format(time.RFC3339)) + responseBody += ` + { + "description": "CDN Bandwidth", + "amount": 100.50, + "rate": 0.05, + "units": 2010, + "product_name": "CDN", + "product_group": "Full Site Delivery", + "usage_type": "bandwidth" + }, + { + "description": "Compute Requests", + "amount": 25.75, + "rate": 0.001, + "units": 25750, + "product_name": "Compute", + "product_group": "Edge Computing", + "usage_type": "requests" + }, + { + "description": "WAF Requests", + "amount": 15.25, + "rate": 0.0005, + "units": 30500, + "product_name": "WAF", + "product_group": "Security", + "usage_type": "requests" + } + ] + } + ], + "meta": { + "next_cursor": "", + "limit": 200, + "total": 1 + } + }` + return createMockResponse(200, responseBody), nil + }, + } + + rateLimiter := rate.NewLimiter(rate.Every(time.Second), 10) + fastlyCostSrc := FastlyCostSource{ + apiKey: "test-api-key", + httpClient: mockClient, + rateLimiter: rateLimiter, + invoiceCache: make(map[string][]fastlyplugin.Invoice), + } + + req := &pb.CustomCostRequest{ + Start: timestamppb.New(startDate), + End: timestamppb.New(endDate), + Resolution: durationpb.New(24 * time.Hour), + } + + // Act + responses := fastlyCostSrc.GetCustomCosts(req) + + // Assert + if len(responses) != 1 { + t.Fatalf("Expected 1 response, got %d", len(responses)) + } + + if len(responses[0].Costs) != 3 { + t.Fatalf("Expected 3 costs (3 line items), got %d", len(responses[0].Costs)) + } + + // Check that all IDs are unique and are valid UUIDs + seenIDs := make(map[string]bool) + for i, cost := range responses[0].Costs { + // Check ID is not empty + if cost.Id == "" { + t.Errorf("Cost %d has empty ID", i) + } + + // Check ID is unique + if seenIDs[cost.Id] { + t.Errorf("Duplicate ID found: %s", cost.Id) + } + seenIDs[cost.Id] = true + + // Check ID looks like a UUID (contains hyphens and is 36 characters) + if len(cost.Id) != 36 { + t.Errorf("Cost %d ID '%s' is not 36 characters (expected UUID format)", i, cost.Id) + } + if !strings.Contains(cost.Id, "-") { + t.Errorf("Cost %d ID '%s' doesn't contain hyphens (expected UUID format)", i, cost.Id) + } + + // Verify ProviderId is still constructed properly (should be different from ID) + expectedProviderID := fmt.Sprintf("inv-12345/%s/%s", cost.Labels["product_name"], cost.ResourceName) + if cost.ProviderId != expectedProviderID { + t.Errorf("Cost %d has unexpected ProviderId. Expected %s, got %s", i, expectedProviderID, cost.ProviderId) + } + + // Verify ID is different from ProviderId + if cost.Id == cost.ProviderId { + t.Errorf("Cost %d ID should be different from ProviderId, both are: %s", i, cost.Id) + } + } + + t.Logf("Successfully generated %d unique UUIDs for cost items", len(responses[0].Costs)) +} diff --git a/pkg/plugins/fastly/cmd/validator/main/main.go b/pkg/plugins/fastly/cmd/validator/main/main.go index 016411f..39b1186 100644 --- a/pkg/plugins/fastly/cmd/validator/main/main.go +++ b/pkg/plugins/fastly/cmd/validator/main/main.go @@ -116,7 +116,7 @@ func validate(respDaily, respHourly []*pb.CustomCostResponse) bool { for _, cost := range resp.Costs { totalDailyCost += cost.GetBilledCost() seenResourceTypes[cost.GetResourceType()] = true - seenProductGroups[cost.GetResourceName()] = true + seenProductGroups[cost.GetResourceType()] = true if cost.GetBilledCost() == 0 { log.Debugf("got zero cost for %v", cost) @@ -145,14 +145,14 @@ func validate(respDaily, respHourly []*pb.CustomCostResponse) bool { foundAnyExpected := false for _, expected := range expectedProductGroups { - if seenResourceTypes[expected] { + if seenProductGroups[expected] { foundAnyExpected = true break } } - if len(seenResourceTypes) > 0 && !foundAnyExpected { - log.Warnf("none of the expected product groups found in fastly response. Seen: %v", seenResourceTypes) + if len(seenProductGroups) > 0 && !foundAnyExpected { + log.Warnf("none of the expected product groups found in fastly response. Seen product groups: %v", seenProductGroups) } // verify the domain matches the plugin name diff --git a/pkg/plugins/fastly/fastlyplugin/invoices.go b/pkg/plugins/fastly/fastlyplugin/invoices.go index 8452686..186047e 100644 --- a/pkg/plugins/fastly/fastlyplugin/invoices.go +++ b/pkg/plugins/fastly/fastlyplugin/invoices.go @@ -1,6 +1,11 @@ package fastlyplugin -import "time" +import ( + "encoding/json" + "fmt" + "strconv" + "time" +) type InvoiceListResponse struct { Data []Invoice `json:"data"` @@ -40,6 +45,58 @@ type PaginationMetadata struct { Sort string `json:"sort"` } +// Custom UnmarshalJSON for Invoice to handle monthly_transaction_amount +// which can come as either a number or string from the API +func (i *Invoice) UnmarshalJSON(data []byte) error { + // Define a temporary struct with the same fields but using interface{} for monthly_transaction_amount + type TempInvoice struct { + CustomerID string `json:"customer_id"` + InvoiceID string `json:"invoice_id"` + InvoicePostedOn string `json:"invoice_posted_on"` + BillingStartDate string `json:"billing_start_date"` + BillingEndDate string `json:"billing_end_date"` + StatementNumber string `json:"statement_number"` + CurrencyCode string `json:"currency_code"` + MonthlyTransactionAmount interface{} `json:"monthly_transaction_amount"` + PaymentStatus string `json:"payment_status,omitempty"` + TransactionLineItems []TransactionLineItem `json:"transaction_line_items"` + } + + var temp TempInvoice + if err := json.Unmarshal(data, &temp); err != nil { + return err + } + + // Copy all fields except MonthlyTransactionAmount + i.CustomerID = temp.CustomerID + i.InvoiceID = temp.InvoiceID + i.InvoicePostedOn = temp.InvoicePostedOn + i.BillingStartDate = temp.BillingStartDate + i.BillingEndDate = temp.BillingEndDate + i.StatementNumber = temp.StatementNumber + i.CurrencyCode = temp.CurrencyCode + i.PaymentStatus = temp.PaymentStatus + i.TransactionLineItems = temp.TransactionLineItems + + // Handle MonthlyTransactionAmount conversion + switch v := temp.MonthlyTransactionAmount.(type) { + case string: + i.MonthlyTransactionAmount = v + case float64: + i.MonthlyTransactionAmount = strconv.FormatFloat(v, 'f', -1, 64) + case int: + i.MonthlyTransactionAmount = strconv.Itoa(v) + default: + if v == nil { + i.MonthlyTransactionAmount = "" + } else { + return fmt.Errorf("unsupported type for monthly_transaction_amount: %T", v) + } + } + + return nil +} + // Helper function to parse date strings func ParseFastlyDate(dateStr string) (time.Time, error) { return time.Parse(time.RFC3339, dateStr) diff --git a/pkg/plugins/fastly/go.mod b/pkg/plugins/fastly/go.mod index 4d07174..243f3d0 100644 --- a/pkg/plugins/fastly/go.mod +++ b/pkg/plugins/fastly/go.mod @@ -19,6 +19,7 @@ require ( github.com/goccy/go-json v0.10.5 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/protobuf v1.5.4 // indirect + github.com/google/uuid v1.6.0 // indirect github.com/hashicorp/errwrap v1.0.0 // indirect github.com/hashicorp/go-hclog v1.6.2 // indirect github.com/hashicorp/hcl v1.0.0 // indirect diff --git a/pkg/plugins/fastly/go.sum b/pkg/plugins/fastly/go.sum index 91f4470..abc02bb 100644 --- a/pkg/plugins/fastly/go.sum +++ b/pkg/plugins/fastly/go.sum @@ -27,6 +27,8 @@ github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeN github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-hclog v1.6.2 h1:NOtoftovWkDheyUM/8JW3QMiXyxJK3uHRK7wV04nD2I= @@ -152,8 +154,6 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.23.0 h1:D71I7dUrlY+VX0gQShAThNGHFxZ13dGLBHQLVl1mJlY= golang.org/x/text v0.23.0/go.mod h1:/BLNzu4aZCJ1+kcD0DNRotWKage4q2rGVAg4o22unh4= -golang.org/x/time v0.9.0 h1:EsRrnYcQiGH+5FfbgvV4AP7qEZstoyrHB0DzarOQ4ZY= -golang.org/x/time v0.9.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/time v0.12.0 h1:ScB/8o8olJvc+CQPWrK3fPZNfh7qgwCrY0zJmoEQLSE= golang.org/x/time v0.12.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= diff --git a/pkg/plugins/fastly/tests/fastly_daily_test.json b/pkg/plugins/fastly/tests/fastly_daily_test.json new file mode 100644 index 0000000..a1cb65f --- /dev/null +++ b/pkg/plugins/fastly/tests/fastly_daily_test.json @@ -0,0 +1,53 @@ +[ + { + "metadata": { + "api_client_version": "v3" + }, + "costSource": "billing", + "domain": "fastly", + "version": "v1", + "currency": "USD", + "start": "2025-07-01T00:00:00Z", + "end": "2025-07-02T00:00:00Z", + "costs": [ + { + "zone": "North America", + "accountName": "test-customer-123", + "chargeCategory": "usage", + "description": "CDN Bandwidth Usage", + "resourceName": "bandwidth", + "resourceType": "Full Site Delivery", + "id": "inv-test-001", + "providerId": "inv-test-001/Full Site Delivery/bandwidth", + "billedCost": 75.5, + "listCost": 75.5, + "listUnitPrice": 0.12, + "usageQuantity": 629.17, + "usageUnit": "GB", + "labels": { + "product_line": "CDN", + "product_name": "Full Site Delivery" + } + }, + { + "zone": "Global", + "accountName": "test-customer-123", + "chargeCategory": "usage", + "description": "Edge Compute Usage", + "resourceName": "compute_time", + "resourceType": "Compute", + "id": "inv-test-001", + "providerId": "inv-test-001/Compute@Edge/compute_time", + "billedCost": 50, + "listCost": 50, + "listUnitPrice": 1, + "usageQuantity": 50, + "usageUnit": "hours", + "labels": { + "product_line": "Edge Compute", + "product_name": "Compute@Edge" + } + } + ] + } +] \ No newline at end of file diff --git a/pkg/plugins/fastly/tests/fastly_hourly_test.json b/pkg/plugins/fastly/tests/fastly_hourly_test.json new file mode 100644 index 0000000..5b2c3d3 --- /dev/null +++ b/pkg/plugins/fastly/tests/fastly_hourly_test.json @@ -0,0 +1,34 @@ +[ + { + "metadata": { + "api_client_version": "v3" + }, + "costSource": "billing", + "domain": "fastly", + "version": "v1", + "currency": "USD", + "start": "2025-07-01T00:00:00Z", + "end": "2025-07-01T01:00:00Z", + "costs": [ + { + "zone": "North America", + "accountName": "test-customer-123", + "chargeCategory": "usage", + "description": "CDN Bandwidth Usage - Hourly", + "resourceName": "bandwidth", + "resourceType": "Full Site Delivery", + "id": "inv-test-hourly-001", + "providerId": "inv-test-hourly-001/Full Site Delivery/bandwidth", + "billedCost": 3.15, + "listCost": 3.15, + "listUnitPrice": 0.12, + "usageQuantity": 26.25, + "usageUnit": "GB", + "labels": { + "product_line": "CDN", + "product_name": "Full Site Delivery" + } + } + ] + } +] \ No newline at end of file diff --git a/pkg/plugins/fastly/tests/mock_test.go b/pkg/plugins/fastly/tests/mock_test.go new file mode 100644 index 0000000..f467451 --- /dev/null +++ b/pkg/plugins/fastly/tests/mock_test.go @@ -0,0 +1,347 @@ +package tests + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "os" + "testing" + "time" + + "github.com/opencost/opencost-plugins/pkg/plugins/fastly/fastlyplugin" + "github.com/opencost/opencost/core/pkg/model/pb" + "google.golang.org/protobuf/types/known/timestamppb" +) + +// Mock Fastly API server for testing +func createMockFastlyServer() *httptest.Server { + return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/billing/v3/invoices": + // Mock invoice list response + response := fastlyplugin.InvoiceListResponse{ + Data: []fastlyplugin.Invoice{ + { + CustomerID: "mock-customer-123", + InvoiceID: "inv-mock-001", + InvoicePostedOn: "2024-01-01T00:00:00Z", + BillingStartDate: "2024-01-01T00:00:00Z", + BillingEndDate: "2024-01-31T23:59:59Z", + StatementNumber: "STMT-001", + CurrencyCode: "USD", + MonthlyTransactionAmount: "150.75", + PaymentStatus: "paid", + TransactionLineItems: []fastlyplugin.TransactionLineItem{ + { + Description: "CDN Bandwidth - North America", + Amount: 75.50, + CreditCouponCode: "", + Rate: 0.12, + Units: 629.17, + ProductName: "Full Site Delivery", + ProductGroup: "Full Site Delivery", + ProductLine: "CDN", + Region: "North America", + UsageType: "bandwidth", + }, + { + Description: "Image Optimization", + Amount: 25.25, + CreditCouponCode: "", + Rate: 0.05, + Units: 505.0, + ProductName: "Image Optimization", + ProductGroup: "Full Site Delivery", + ProductLine: "CDN", + Region: "Global", + UsageType: "image_processing", + }, + { + Description: "Compute@Edge", + Amount: 50.00, + CreditCouponCode: "", + Rate: 1.00, + Units: 50.0, + ProductName: "Compute@Edge", + ProductGroup: "Compute", + ProductLine: "Edge Compute", + Region: "Global", + UsageType: "compute_time", + }, + }, + }, + }, + Meta: fastlyplugin.PaginationMetadata{ + NextCursor: "", + Limit: 200, + Total: 1, + Sort: "billing_start_date", + }, + } + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(response) + + case "/billing/v3/invoices/month-to-date": + // Mock month-to-date response + invoice := fastlyplugin.Invoice{ + CustomerID: "mock-customer-123", + InvoiceID: "mtd-mock-001", + InvoicePostedOn: time.Now().Format(time.RFC3339), + BillingStartDate: time.Date(time.Now().Year(), time.Now().Month(), 1, 0, 0, 0, 0, time.UTC).Format(time.RFC3339), + BillingEndDate: time.Now().Format(time.RFC3339), + StatementNumber: "MTD-001", + CurrencyCode: "USD", + MonthlyTransactionAmount: "45.25", + PaymentStatus: "pending", + TransactionLineItems: []fastlyplugin.TransactionLineItem{ + { + Description: "CDN Bandwidth - Current Month", + Amount: 45.25, + CreditCouponCode: "", + Rate: 0.12, + Units: 377.08, + ProductName: "Full Site Delivery", + ProductGroup: "Full Site Delivery", + ProductLine: "CDN", + Region: "North America", + UsageType: "bandwidth", + }, + }, + } + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(invoice) + + default: + w.WriteHeader(http.StatusNotFound) + w.Write([]byte(`{"msg": "Not Found"}`)) + } + })) +} + +// Test with mock data instead of real API +func TestFastlyMockCostRetrieval(t *testing.T) { + // Create mock server + mockServer := createMockFastlyServer() + defer mockServer.Close() + + // Create temp config file + config := fastlyplugin.FastlyConfig{ + FastlyAPIKey: "mock-api-key", + LogLevel: "debug", + } + + configFile, err := os.CreateTemp("", "fastly_mock_config.json") + if err != nil { + t.Fatalf("could not create temp config file: %v", err) + } + defer os.Remove(configFile.Name()) + + configData, err := json.MarshalIndent(config, "", " ") + if err != nil { + t.Fatalf("could not marshal config: %v", err) + } + + if _, err := configFile.Write(configData); err != nil { + t.Fatalf("could not write config file: %v", err) + } + configFile.Close() + + // Set up test parameters + now := time.Now() + windowStart := time.Date(now.Year(), now.Month()-1, 1, 0, 0, 0, 0, time.UTC) + windowEnd := time.Date(now.Year(), now.Month(), 1, 0, 0, 0, 0, time.UTC) + + // Note: You would need to modify the main plugin code to accept a custom HTTP client + // or base URL for testing. This is a limitation of the current implementation. + // For now, we can test the data structures and basic functionality without the full request. + + t.Log("Mock server created successfully. To fully test, you would need to:") + t.Log("1. Modify the FastlyCostSource to accept a custom base URL") + t.Log("2. Or inject a custom HTTP client") + t.Log("3. Then point it to the mock server URL:", mockServer.URL) + + // For now, we can test the data structures and basic functionality + testInvoice := fastlyplugin.Invoice{ + CustomerID: "test-customer", + InvoiceID: "test-invoice-001", + MonthlyTransactionAmount: "100.50", + BillingStartDate: windowStart.Format(time.RFC3339), + BillingEndDate: windowEnd.Format(time.RFC3339), + CurrencyCode: "USD", + TransactionLineItems: []fastlyplugin.TransactionLineItem{ + { + Description: "Test CDN Usage", + Amount: 100.50, + Rate: 0.12, + Units: 837.5, + ProductName: "Full Site Delivery", + ProductGroup: "Full Site Delivery", + ProductLine: "CDN", + Region: "North America", + UsageType: "bandwidth", + }, + }, + } + + // Test JSON marshaling/unmarshaling + data, err := json.Marshal(testInvoice) + if err != nil { + t.Fatalf("failed to marshal test invoice: %v", err) + } + + var unmarshaled fastlyplugin.Invoice + if err := json.Unmarshal(data, &unmarshaled); err != nil { + t.Fatalf("failed to unmarshal test invoice: %v", err) + } + + // Verify the monthly transaction amount is preserved as string + if unmarshaled.MonthlyTransactionAmount != "100.50" { + t.Errorf("expected MonthlyTransactionAmount to be '100.50', got %s", unmarshaled.MonthlyTransactionAmount) + } + + // Test with numeric input (simulating API response) + jsonWithNumber := `{ + "customer_id": "test-customer", + "invoice_id": "test-invoice-001", + "monthly_transaction_amount": 100.50, + "billing_start_date": "2024-01-01T00:00:00Z", + "billing_end_date": "2024-01-31T23:59:59Z", + "currency_code": "USD", + "transaction_line_items": [] + }` + + var invoiceFromNumber fastlyplugin.Invoice + if err := json.Unmarshal([]byte(jsonWithNumber), &invoiceFromNumber); err != nil { + t.Fatalf("failed to unmarshal invoice with numeric monthly_transaction_amount: %v", err) + } + + // Verify numeric input is converted to string + if invoiceFromNumber.MonthlyTransactionAmount != "100.5" { + t.Errorf("expected MonthlyTransactionAmount to be '100.5', got %s", invoiceFromNumber.MonthlyTransactionAmount) + } + + t.Log("Mock data tests passed successfully!") +} + +// Generate test protobuf files for validator testing +func TestGenerateValidatorTestData(t *testing.T) { + // Create mock daily response + now := time.Now() + dailyStart := time.Date(now.Year(), now.Month()-1, 1, 0, 0, 0, 0, time.UTC) + dailyEnd := dailyStart.AddDate(0, 0, 1) + + dailyResponse := &pb.CustomCostResponse{ + Metadata: map[string]string{"api_client_version": "v3"}, + CostSource: "billing", + Domain: "fastly", + Version: "v1", + Currency: "USD", + Start: timestamppb.New(dailyStart), + End: timestamppb.New(dailyEnd), + Errors: []string{}, + Costs: []*pb.CustomCost{ + { + Zone: "North America", + AccountName: "test-customer-123", + ChargeCategory: "usage", + Description: "CDN Bandwidth Usage", + ResourceName: "bandwidth", + ResourceType: "Full Site Delivery", + Id: "inv-test-001", + ProviderId: "inv-test-001/Full Site Delivery/bandwidth", + Labels: map[string]string{ + "product_line": "CDN", + "product_name": "Full Site Delivery", + }, + ListCost: 75.50, + ListUnitPrice: 0.12, + BilledCost: 75.50, + UsageQuantity: 629.17, + UsageUnit: "GB", + }, + { + Zone: "Global", + AccountName: "test-customer-123", + ChargeCategory: "usage", + Description: "Edge Compute Usage", + ResourceName: "compute_time", + ResourceType: "Compute", + Id: "inv-test-001", + ProviderId: "inv-test-001/Compute@Edge/compute_time", + Labels: map[string]string{ + "product_line": "Edge Compute", + "product_name": "Compute@Edge", + }, + ListCost: 50.00, + ListUnitPrice: 1.00, + BilledCost: 50.00, + UsageQuantity: 50.0, + UsageUnit: "hours", + }, + }, + } + + // Create mock hourly response (smaller amounts) + hourlyStart := dailyStart + hourlyEnd := hourlyStart.Add(time.Hour) + + hourlyResponse := &pb.CustomCostResponse{ + Metadata: map[string]string{"api_client_version": "v3"}, + CostSource: "billing", + Domain: "fastly", + Version: "v1", + Currency: "USD", + Start: timestamppb.New(hourlyStart), + End: timestamppb.New(hourlyEnd), + Errors: []string{}, + Costs: []*pb.CustomCost{ + { + Zone: "North America", + AccountName: "test-customer-123", + ChargeCategory: "usage", + Description: "CDN Bandwidth Usage - Hourly", + ResourceName: "bandwidth", + ResourceType: "Full Site Delivery", + Id: "inv-test-hourly-001", + ProviderId: "inv-test-hourly-001/Full Site Delivery/bandwidth", + Labels: map[string]string{ + "product_line": "CDN", + "product_name": "Full Site Delivery", + }, + ListCost: 3.15, + ListUnitPrice: 0.12, + BilledCost: 3.15, + UsageQuantity: 26.25, + UsageUnit: "GB", + }, + }, + } + + // Create temporary files for testing + dailyFile, err := os.CreateTemp("", "fastly_daily_test.json") + if err != nil { + t.Fatalf("could not create daily test file: %v", err) + } + defer os.Remove(dailyFile.Name()) + + hourlyFile, err := os.CreateTemp("", "fastly_hourly_test.json") + if err != nil { + t.Fatalf("could not create hourly test file: %v", err) + } + defer os.Remove(hourlyFile.Name()) + + // Write responses as JSON arrays (as expected by validator) + dailyData, _ := json.MarshalIndent([]*pb.CustomCostResponse{dailyResponse}, "", " ") + hourlyData, _ := json.MarshalIndent([]*pb.CustomCostResponse{hourlyResponse}, "", " ") + + dailyFile.Write(dailyData) + hourlyFile.Write(hourlyData) + dailyFile.Close() + hourlyFile.Close() + + t.Logf("Test data files created:") + t.Logf("Daily: %s", dailyFile.Name()) + t.Logf("Hourly: %s", hourlyFile.Name()) + t.Logf("\nTo test the validator, run:") + t.Logf("go run ./cmd/validator/main/main.go %s %s", dailyFile.Name(), hourlyFile.Name()) +} \ No newline at end of file diff --git a/pkg/plugins/fastly/utils/create_test_files.go b/pkg/plugins/fastly/utils/create_test_files.go new file mode 100644 index 0000000..174e5b4 --- /dev/null +++ b/pkg/plugins/fastly/utils/create_test_files.go @@ -0,0 +1,146 @@ +package main + +import ( + "encoding/json" + "fmt" + "os" + "time" + + "github.com/opencost/opencost/core/pkg/model/pb" + "google.golang.org/protobuf/encoding/protojson" + "google.golang.org/protobuf/types/known/timestamppb" +) + +func main() { + // Create mock daily response + now := time.Now() + dailyStart := time.Date(now.Year(), now.Month()-1, 1, 0, 0, 0, 0, time.UTC) + dailyEnd := dailyStart.AddDate(0, 0, 1) + + dailyResponse := &pb.CustomCostResponse{ + Metadata: map[string]string{"api_client_version": "v3"}, + CostSource: "billing", + Domain: "fastly", + Version: "v1", + Currency: "USD", + Start: timestamppb.New(dailyStart), + End: timestamppb.New(dailyEnd), + Errors: []string{}, + Costs: []*pb.CustomCost{ + { + Zone: "North America", + AccountName: "test-customer-123", + ChargeCategory: "usage", + Description: "CDN Bandwidth Usage", + ResourceName: "bandwidth", + ResourceType: "Full Site Delivery", + Id: "inv-test-001", + ProviderId: "inv-test-001/Full Site Delivery/bandwidth", + Labels: map[string]string{ + "product_line": "CDN", + "product_name": "Full Site Delivery", + }, + ListCost: 75.50, + ListUnitPrice: 0.12, + BilledCost: 75.50, + UsageQuantity: 629.17, + UsageUnit: "GB", + }, + { + Zone: "Global", + AccountName: "test-customer-123", + ChargeCategory: "usage", + Description: "Edge Compute Usage", + ResourceName: "compute_time", + ResourceType: "Compute", + Id: "inv-test-001", + ProviderId: "inv-test-001/Compute@Edge/compute_time", + Labels: map[string]string{ + "product_line": "Edge Compute", + "product_name": "Compute@Edge", + }, + ListCost: 50.00, + ListUnitPrice: 1.00, + BilledCost: 50.00, + UsageQuantity: 50.0, + UsageUnit: "hours", + }, + }, + } + + // Create mock hourly response (smaller amounts) + hourlyStart := dailyStart + hourlyEnd := hourlyStart.Add(time.Hour) + + hourlyResponse := &pb.CustomCostResponse{ + Metadata: map[string]string{"api_client_version": "v3"}, + CostSource: "billing", + Domain: "fastly", + Version: "v1", + Currency: "USD", + Start: timestamppb.New(hourlyStart), + End: timestamppb.New(hourlyEnd), + Errors: []string{}, + Costs: []*pb.CustomCost{ + { + Zone: "North America", + AccountName: "test-customer-123", + ChargeCategory: "usage", + Description: "CDN Bandwidth Usage - Hourly", + ResourceName: "bandwidth", + ResourceType: "Full Site Delivery", + Id: "inv-test-hourly-001", + ProviderId: "inv-test-hourly-001/Full Site Delivery/bandwidth", + Labels: map[string]string{ + "product_line": "CDN", + "product_name": "Full Site Delivery", + }, + ListCost: 3.15, + ListUnitPrice: 0.12, + BilledCost: 3.15, + UsageQuantity: 26.25, + UsageUnit: "GB", + }, + }, + } + + // Write daily test data + dailyFile := "fastly_daily_test.json" + // Create array as expected by validator + responses := []*pb.CustomCostResponse{dailyResponse} + // Marshal each response individually and create JSON array + var responseArray []json.RawMessage + for _, resp := range responses { + protoData, _ := protojson.Marshal(resp) + responseArray = append(responseArray, json.RawMessage(protoData)) + } + dailyData, _ := json.MarshalIndent(responseArray, "", " ") + err := os.WriteFile(dailyFile, dailyData, 0644) + if err != nil { + fmt.Printf("Error writing daily file: %v\n", err) + return + } + + // Write hourly test data + hourlyFile := "fastly_hourly_test.json" + // Create array as expected by validator + hourlyResponses := []*pb.CustomCostResponse{hourlyResponse} + // Marshal each response individually and create JSON array + var hourlyResponseArray []json.RawMessage + for _, resp := range hourlyResponses { + protoData, _ := protojson.Marshal(resp) + hourlyResponseArray = append(hourlyResponseArray, json.RawMessage(protoData)) + } + hourlyData, _ := json.MarshalIndent(hourlyResponseArray, "", " ") + err = os.WriteFile(hourlyFile, hourlyData, 0644) + if err != nil { + fmt.Printf("Error writing hourly file: %v\n", err) + return + } + + fmt.Printf("Test data files created successfully:\n") + fmt.Printf("Daily: %s\n", dailyFile) + fmt.Printf("Hourly: %s\n", hourlyFile) + fmt.Printf("\nTo test the validator, run:\n") + fmt.Printf("go run ../cmd/validator/main/main.go %s %s\n", dailyFile, hourlyFile) +} diff --git a/pkg/plugins/fastly/utils/generate_test_data.go b/pkg/plugins/fastly/utils/generate_test_data.go new file mode 100644 index 0000000..6144578 --- /dev/null +++ b/pkg/plugins/fastly/utils/generate_test_data.go @@ -0,0 +1,129 @@ +package main + +import ( + "encoding/json" + "fmt" + "os" + "time" + + "github.com/opencost/opencost/core/pkg/model/pb" + "google.golang.org/protobuf/types/known/timestamppb" +) + +func main() { + // Create mock daily response + now := time.Now() + dailyStart := time.Date(now.Year(), now.Month()-1, 1, 0, 0, 0, 0, time.UTC) + dailyEnd := dailyStart.AddDate(0, 0, 1) + + dailyResponse := &pb.CustomCostResponse{ + Metadata: map[string]string{"api_client_version": "v3"}, + CostSource: "billing", + Domain: "fastly", + Version: "v1", + Currency: "USD", + Start: timestamppb.New(dailyStart), + End: timestamppb.New(dailyEnd), + Errors: []string{}, + Costs: []*pb.CustomCost{ + { + Zone: "North America", + AccountName: "test-customer-123", + ChargeCategory: "usage", + Description: "CDN Bandwidth Usage", + ResourceName: "bandwidth", + ResourceType: "Full Site Delivery", + Id: "inv-test-001", + ProviderId: "inv-test-001/Full Site Delivery/bandwidth", + Labels: map[string]string{ + "product_line": "CDN", + "product_name": "Full Site Delivery", + }, + ListCost: 75.50, + ListUnitPrice: 0.12, + BilledCost: 75.50, + UsageQuantity: 629.17, + UsageUnit: "GB", + }, + { + Zone: "Global", + AccountName: "test-customer-123", + ChargeCategory: "usage", + Description: "Edge Compute Usage", + ResourceName: "compute_time", + ResourceType: "Compute", + Id: "inv-test-001", + ProviderId: "inv-test-001/Compute@Edge/compute_time", + Labels: map[string]string{ + "product_line": "Edge Compute", + "product_name": "Compute@Edge", + }, + ListCost: 50.00, + ListUnitPrice: 1.00, + BilledCost: 50.00, + UsageQuantity: 50.0, + UsageUnit: "hours", + }, + }, + } + + // Create mock hourly response (smaller amounts) + hourlyStart := dailyStart + hourlyEnd := hourlyStart.Add(time.Hour) + + hourlyResponse := &pb.CustomCostResponse{ + Metadata: map[string]string{"api_client_version": "v3"}, + CostSource: "billing", + Domain: "fastly", + Version: "v1", + Currency: "USD", + Start: timestamppb.New(hourlyStart), + End: timestamppb.New(hourlyEnd), + Errors: []string{}, + Costs: []*pb.CustomCost{ + { + Zone: "North America", + AccountName: "test-customer-123", + ChargeCategory: "usage", + Description: "CDN Bandwidth Usage - Hourly", + ResourceName: "bandwidth", + ResourceType: "Full Site Delivery", + Id: "inv-test-hourly-001", + ProviderId: "inv-test-hourly-001/Full Site Delivery/bandwidth", + Labels: map[string]string{ + "product_line": "CDN", + "product_name": "Full Site Delivery", + }, + ListCost: 3.15, + ListUnitPrice: 0.12, + BilledCost: 3.15, + UsageQuantity: 26.25, + UsageUnit: "GB", + }, + }, + } + + // Write daily test data + dailyFile := "fastly_daily_test.json" + dailyData, _ := json.MarshalIndent([]*pb.CustomCostResponse{dailyResponse}, "", " ") + err := os.WriteFile(dailyFile, dailyData, 0644) + if err != nil { + fmt.Printf("Error writing daily file: %v\n", err) + return + } + + // Write hourly test data + hourlyFile := "fastly_hourly_test.json" + hourlyData, _ := json.MarshalIndent([]*pb.CustomCostResponse{hourlyResponse}, "", " ") + err = os.WriteFile(hourlyFile, hourlyData, 0644) + if err != nil { + fmt.Printf("Error writing hourly file: %v\n", err) + return + } + + fmt.Printf("Test data files created successfully:\n") + fmt.Printf("Daily: %s\n", dailyFile) + fmt.Printf("Hourly: %s\n", hourlyFile) + fmt.Printf("\nTo test the validator, run:\n") + fmt.Printf("go run ./cmd/validator/main/main.go %s %s\n", dailyFile, hourlyFile) +} From bcc8eb1adfa7436be8c917c2fbeb8da1ea2fa3fa Mon Sep 17 00:00:00 2001 From: Sparsh Date: Thu, 7 Aug 2025 14:14:45 +0530 Subject: [PATCH 6/8] Added missing tests for get invoice by ID Signed-off-by: Sparsh --- pkg/plugins/fastly/cmd/main/main_test.go | 154 +++++++++++++++++++++++ 1 file changed, 154 insertions(+) diff --git a/pkg/plugins/fastly/cmd/main/main_test.go b/pkg/plugins/fastly/cmd/main/main_test.go index cbd60b9..7918244 100644 --- a/pkg/plugins/fastly/cmd/main/main_test.go +++ b/pkg/plugins/fastly/cmd/main/main_test.go @@ -992,3 +992,157 @@ func TestUniqueUUIDGeneration(t *testing.T) { t.Logf("Successfully generated %d unique UUIDs for cost items", len(responses[0].Costs)) } + +func TestGetInvoiceByID(t *testing.T) { + // Arrange + mockClient := &MockHTTPClient{ + DoFunc: func(req *http.Request) (*http.Response, error) { + // Verify the request + if req.Method != "GET" { + t.Errorf("Expected GET request, got %s", req.Method) + } + if !strings.Contains(req.URL.Path, "/billing/v3/invoices/inv-12345") { + t.Errorf("Expected invoice by ID endpoint, got %s", req.URL.Path) + } + if req.Header.Get("Fastly-Key") != "test-api-key" { + t.Errorf("Expected API key in header") + } + + return createMockResponse(200, `{ + "customer_id": "test-customer", + "invoice_id": "inv-12345", + "billing_start_date": "2024-01-01T00:00:00Z", + "billing_end_date": "2024-01-31T00:00:00Z", + "currency_code": "USD", + "transaction_line_items": [ + { + "description": "CDN Bandwidth", + "amount": 100.50, + "rate": 0.05, + "units": 2010, + "product_name": "CDN", + "product_group": "Full Site Delivery", + "product_line": "Delivery", + "usage_type": "bandwidth", + "region": "North America" + } + ] + }`), nil + }, + } + + rateLimiter := rate.NewLimiter(rate.Every(time.Second), 10) + fastlyCostSrc := FastlyCostSource{ + apiKey: "test-api-key", + httpClient: mockClient, + rateLimiter: rateLimiter, + invoiceCache: make(map[string][]fastlyplugin.Invoice), + } + + // Act + invoice, err := fastlyCostSrc.getInvoiceByID("inv-12345") + + // Assert + if err != nil { + t.Fatalf("Unexpected error: %v", err) + } + + if invoice == nil { + t.Fatal("Expected invoice, got nil") + } + + if invoice.InvoiceID != "inv-12345" { + t.Errorf("Expected invoice ID 'inv-12345', got '%s'", invoice.InvoiceID) + } + + if invoice.CustomerID != "test-customer" { + t.Errorf("Expected customer ID 'test-customer', got '%s'", invoice.CustomerID) + } + + if len(invoice.TransactionLineItems) != 1 { + t.Errorf("Expected 1 line item, got %d", len(invoice.TransactionLineItems)) + } + + lineItem := invoice.TransactionLineItems[0] + if lineItem.Description != "CDN Bandwidth" { + t.Errorf("Expected description 'CDN Bandwidth', got '%s'", lineItem.Description) + } + + if lineItem.Amount != 100.50 { + t.Errorf("Expected amount 100.50, got %f", lineItem.Amount) + } +} + +func TestGetInvoiceByIDError(t *testing.T) { + // Arrange + mockClient := &MockHTTPClient{ + DoFunc: func(req *http.Request) (*http.Response, error) { + // Verify the request URL contains the invoice ID + if !strings.Contains(req.URL.Path, "/billing/v3/invoices/invalid-invoice") { + t.Errorf("Expected invoice by ID endpoint with invalid-invoice, got %s", req.URL.Path) + } + + return createMockResponse(404, `{ + "msg": "Not Found", + "detail": "Invoice not found" + }`), nil + }, + } + + rateLimiter := rate.NewLimiter(rate.Every(time.Second), 10) + fastlyCostSrc := FastlyCostSource{ + apiKey: "test-api-key", + httpClient: mockClient, + rateLimiter: rateLimiter, + invoiceCache: make(map[string][]fastlyplugin.Invoice), + } + + // Act + invoice, err := fastlyCostSrc.getInvoiceByID("invalid-invoice") + + // Assert + if err == nil { + t.Fatal("Expected error, got nil") + } + + if invoice != nil { + t.Error("Expected nil invoice on error") + } + + if !strings.Contains(err.Error(), "404") { + t.Errorf("Expected error message to contain '404', got: %v", err) + } +} + +func TestGetInvoiceByIDNetworkError(t *testing.T) { + // Arrange + mockClient := &MockHTTPClient{ + DoFunc: func(req *http.Request) (*http.Response, error) { + return nil, fmt.Errorf("network connection failed") + }, + } + + rateLimiter := rate.NewLimiter(rate.Every(time.Second), 10) + fastlyCostSrc := FastlyCostSource{ + apiKey: "test-api-key", + httpClient: mockClient, + rateLimiter: rateLimiter, + invoiceCache: make(map[string][]fastlyplugin.Invoice), + } + + // Act + invoice, err := fastlyCostSrc.getInvoiceByID("inv-12345") + + // Assert + if err == nil { + t.Fatal("Expected error, got nil") + } + + if invoice != nil { + t.Error("Expected nil invoice on network error") + } + + if !strings.Contains(err.Error(), "network connection failed") { + t.Errorf("Expected error message to contain 'network connection failed', got: %v", err) + } +} From 6ecb18576e7dbb9928873648baf2a4e21b2dfea8 Mon Sep 17 00:00:00 2001 From: Sparsh Date: Thu, 7 Aug 2025 16:13:48 +0530 Subject: [PATCH 7/8] Added fastly invoices api data model mapping with FOCUS and added extended attributes for the same Signed-off-by: Sparsh --- pkg/plugins/fastly/cmd/main/main.go | 54 ++++++++++++++++++++-- pkg/plugins/fastly/fastly_focus_mapping.md | 15 ++++++ 2 files changed, 64 insertions(+), 5 deletions(-) create mode 100644 pkg/plugins/fastly/fastly_focus_mapping.md diff --git a/pkg/plugins/fastly/cmd/main/main.go b/pkg/plugins/fastly/cmd/main/main.go index 14676b6..5f15e9c 100644 --- a/pkg/plugins/fastly/cmd/main/main.go +++ b/pkg/plugins/fastly/cmd/main/main.go @@ -412,6 +412,49 @@ func (f *FastlyCostSource) convertInvoiceToCosts(invoice fastlyplugin.Invoice, w region = "Global" } + // Create extended attributes following FOCUS mapping + // According to fastly_focus_mapping.md: + // AccountId maps to customer_id (Fastly customer/account ID) + extendedAttrs := pb.CustomCostExtendedAttributes{ + AccountId: &invoice.CustomerID, // FOCUS: AccountId → customer_id + } + + // Use product_line as SubAccountId if available for better granularity + if item.ProductLine != "" { + extendedAttrs.SubAccountId = &item.ProductLine // FOCUS: Subcategory → product_line + } + + // Add billing period information + // Parse ISO 8601 dates and convert to timestamppb.Timestamp + if billingStartTime, err := fastlyplugin.ParseFastlyDate(invoice.BillingStartDate); err == nil { + extendedAttrs.BillingPeriodStart = timestamppb.New(billingStartTime) // FOCUS: BillingPeriodStart → billing_start_date + } + if billingEndTime, err := fastlyplugin.ParseFastlyDate(invoice.BillingEndDate); err == nil { + extendedAttrs.BillingPeriodEnd = timestamppb.New(billingEndTime) // FOCUS: BillingPeriodEnd → billing_end_date + } + + // Add service information if available + if item.ProductGroup != "" { + extendedAttrs.ServiceCategory = &item.ProductGroup // FOCUS: ServiceCategory → product_group + } + if item.ProductName != "" { + extendedAttrs.ServiceName = &item.ProductName // FOCUS: ServiceName → product_name + } + + // Add pricing information + if item.Units > 0 { + units := float32(item.Units) + extendedAttrs.PricingQuantity = &units // FOCUS: PricingQuantity → units + } + if item.UsageType != "" { + extendedAttrs.PricingUnit = &item.UsageType // FOCUS: PricingUnit → usage_type + } + + // Add discount code if available + if item.CreditCouponCode != "" { + extendedAttrs.CommitmentDiscountId = &item.CreditCouponCode // FOCUS: CommitmentDiscountId → credit_coupon_code + } + cost := &pb.CustomCost{ Zone: region, AccountName: invoice.CustomerID, @@ -427,11 +470,12 @@ func (f *FastlyCostSource) convertInvoiceToCosts(invoice fastlyplugin.Invoice, w "credit_coupon_code": item.CreditCouponCode, "currency": invoice.CurrencyCode, }, - ListCost: billedCost, - ListUnitPrice: float32(item.Rate), - BilledCost: billedCost, - UsageQuantity: usageQuantity, - UsageUnit: getUsageUnit(item.UsageType, item.ProductName), + ListCost: billedCost, + ListUnitPrice: float32(item.Rate), + BilledCost: billedCost, + UsageQuantity: usageQuantity, + UsageUnit: getUsageUnit(item.UsageType, item.ProductName), + ExtendedAttributes: &extendedAttrs, } // Add additional label for credits/discounts diff --git a/pkg/plugins/fastly/fastly_focus_mapping.md b/pkg/plugins/fastly/fastly_focus_mapping.md new file mode 100644 index 0000000..1851b3e --- /dev/null +++ b/pkg/plugins/fastly/fastly_focus_mapping.md @@ -0,0 +1,15 @@ +# Fastly Invoice – FOCUS Attribute Mapping + +| FOCUS custom‑cost attribute | Fastly field | Notes | +|---|---|---| +| BillingPeriodStart | `billing_start_date` | Billing cycle start (ISO‑8601). | +| BillingPeriodEnd | `billing_end_date` | Billing cycle end (ISO‑8601). | +| AccountId | `customer_id` | Fastly customer (account) ID. | +| Subcategory | `product_line` | Functional grouping, e.g. “Network Services”. | +| CommitmentDiscountId | `credit_coupon_code` | Discount / credit coupon code. | +| EffectiveCost | `amount` (line‑item) or `monthly_transaction_amount` (invoice total) | Map depending on desired granularity. | +| ServiceCategory | `product_group` | Broad service family, e.g. “Compute”. | +| ServiceName | `product_name` | Specific product name. | +| PricingQuantity | `units` | Usage quantity. | +| PricingUnit | `usage_type` | Unit of measure (requests, bandwidth, etc.). | +| PricingCategory | `product_group` | Geographic vs. functional category. | From 8623ceab20a811c3664bcd787caeaa684f1a3d90 Mon Sep 17 00:00:00 2001 From: Sparsh Date: Thu, 18 Sep 2025 14:16:40 +0530 Subject: [PATCH 8/8] Dummy commit to run the backend workflow on github Signed-off-by: Sparsh --- pkg/plugins/fastly/tests/fastly_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/plugins/fastly/tests/fastly_test.go b/pkg/plugins/fastly/tests/fastly_test.go index ad4a395..b3d6555 100644 --- a/pkg/plugins/fastly/tests/fastly_test.go +++ b/pkg/plugins/fastly/tests/fastly_test.go @@ -156,7 +156,7 @@ func getFastlyResponse(t *testing.T, windowStart, windowEnd time.Time, step time LogLevel: "debug", } - // set up custom cost request + // set up custom cost request and write to temp file file, err := os.CreateTemp("", "fastly_config.json") if err != nil { t.Fatalf("could not create temp config dir: %v", err)