feat: new Fastly plugin

pkg/plugins/fastly/cmd/validator/main/main.go

@@ -0,0 +1,216 @@
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"os"
+	"time"
+
+	"github.com/hashicorp/go-multierror"
+	"github.com/opencost/opencost/core/pkg/log"
+	"github.com/opencost/opencost/core/pkg/model/pb"
+	"google.golang.org/protobuf/encoding/protojson"
+)
+
+// the validator is designed to allow plugin implementors to validate their plugin information
+// as called by the central test harness.
+// this avoids having to ask folks to re-implement the test harness over again for each plugin
+
+// the integration test harness provides a path to a protobuf file for each window
+// the validator can then read that in and further validate the response data
+// using the domain knowledge of each plugin author
+func main() {
+	// first arg is the path to the daily protobuf file
+	if len(os.Args) < 3 {
+		fmt.Println("Usage: validator <path-to-daily-protobuf-file> <path-to-hourly-protobuf-file>")
+		os.Exit(1)
+	}
+
+	dailyProtobufFilePath := os.Args[1]
+
+	// read in the protobuf file
+	data, err := os.ReadFile(dailyProtobufFilePath)
+	if err != nil {
+		fmt.Printf("Error reading daily protobuf file: %v\n", err)
+		os.Exit(1)
+	}
+
+	dailyCustomCostResponses, err := Unmarshal(data)
+	if err != nil {
+		fmt.Printf("Error unmarshalling daily protobuf data: %v\n", err)
+		os.Exit(1)
+	}
+
+	fmt.Printf("Successfully unmarshalled %d daily custom cost responses\n", len(dailyCustomCostResponses))
+
+	// second arg is the path to the hourly protobuf file
+	hourlyProtobufFilePath := os.Args[2]
+
+	data, err = os.ReadFile(hourlyProtobufFilePath)
+	if err != nil {
+		fmt.Printf("Error reading hourly protobuf file: %v\n", err)
+		os.Exit(1)
+	}
+
+	// read in the protobuf file
+	hourlyCustomCostResponses, err := Unmarshal(data)
+	if err != nil {
+		fmt.Printf("Error unmarshalling hourly protobuf data: %v\n", err)
+		os.Exit(1)
+	}
+
+	fmt.Printf("Successfully unmarshalled %d hourly custom cost responses\n", len(hourlyCustomCostResponses))
+
+	// validate the custom cost response data
+	isvalid := validate(dailyCustomCostResponses, hourlyCustomCostResponses)
+	if !isvalid {
+		os.Exit(1)
+	} else {
+		fmt.Println("Validation successful")
+	}
+}
+
+func validate(respDaily, respHourly []*pb.CustomCostResponse) bool {
+	if len(respDaily) == 0 {
+		log.Errorf("no daily response received from fastly plugin")
+		return false
+	}
+
+	if len(respHourly) == 0 {
+		log.Errorf("no hourly response received from fastly plugin")
+		return false
+	}
+
+	var multiErr error
+
+	// parse the response and look for errors
+	for _, resp := range respDaily {
+		if len(resp.Errors) > 0 {
+			multiErr = multierror.Append(multiErr, fmt.Errorf("errors occurred in daily response: %v", resp.Errors))
+		}
+	}
+
+	for _, resp := range respHourly {
+		if resp.Errors != nil {
+			multiErr = multierror.Append(multiErr, fmt.Errorf("errors occurred in hourly response: %v", resp.Errors))
+		}
+	}
+
+	// check if any errors occurred
+	if multiErr != nil {
+		log.Errorf("Errors occurred during plugin testing for fastly: %v", multiErr)
+		return false
+	}
+
+	seenResourceTypes := map[string]bool{}
+	seenProductGroups := map[string]bool{}
+	totalDailyCost := float32(0.0)
+
+	// verify that the returned costs are non zero
+	for _, resp := range respDaily {
+		if len(resp.Costs) == 0 && resp.Start.AsTime().After(time.Now().Truncate(24*time.Hour).Add(-1*time.Minute)) {
+			log.Debugf("today's daily costs returned by fastly plugin are empty, skipping: %v", resp)
+			continue
+		}
+
+		for _, cost := range resp.Costs {
+			totalDailyCost += cost.GetBilledCost()
+			seenResourceTypes[cost.GetResourceType()] = true
+			seenProductGroups[cost.GetResourceType()] = true
+
+			if cost.GetBilledCost() == 0 {
+				log.Debugf("got zero cost for %v", cost)
+			}
+
+			// Sanity check - individual line items shouldn't be extremely high
+			if cost.GetBilledCost() > 100000 {
+				log.Errorf("daily cost returned by fastly plugin for %v is greater than 100,000", cost)
+				return false
+			}
+		}
+	}
+
+	// Fastly should have some costs unless it's a free/developer account
+	if totalDailyCost == 0 {
+		log.Warnf("daily costs returned by fastly plugin are zero - this might be expected for developer accounts")
+	}
+
+	// Check that we see some expected product groups
+	expectedProductGroups := []string{
+		"Full Site Delivery",
+		"Compute",
+		"Security",
+		"Network Services",
+	}
+
+	foundAnyExpected := false
+	for _, expected := range expectedProductGroups {
+		if seenProductGroups[expected] {
+			foundAnyExpected = true
+			break
+		}
+	}
+
+	if len(seenProductGroups) > 0 && !foundAnyExpected {
+		log.Warnf("none of the expected product groups found in fastly response. Seen product groups: %v", seenProductGroups)
+	}
+
+	// verify the domain matches the plugin name
+	for _, resp := range respDaily {
+		if resp.Domain != "fastly" {
+			log.Errorf("daily domain returned by fastly plugin does not match plugin name")
+			return false
+		}
+	}
+
+	// Check hourly responses
+	totalHourlyCost := float32(0.0)
+	for _, resp := range respHourly {
+		for _, cost := range resp.Costs {
+			totalHourlyCost += cost.GetBilledCost()
+			if cost.GetBilledCost() > 10000 {
+				log.Errorf("hourly cost returned by fastly plugin for %v is greater than 10,000", cost)
+				return false
+			}
+		}
+	}
+
+	if totalHourlyCost == 0 {
+		log.Warnf("hourly costs returned by fastly plugin are zero - this might be expected for developer accounts")
+	}
+
+	// Verify currency is USD
+	for _, resp := range respDaily {
+		if resp.Currency != "USD" {
+			log.Errorf("expected currency USD, got %s", resp.Currency)
+			return false
+		}
+	}
+
+	// Verify cost source
+	for _, resp := range respDaily {
+		if resp.CostSource != "billing" {
+			log.Errorf("expected cost source 'billing', got %s", resp.CostSource)
+			return false
+		}
+	}
+
+	return true
+}
+
+func Unmarshal(data []byte) ([]*pb.CustomCostResponse, error) {
+	var raw []json.RawMessage
+	if err := json.Unmarshal(data, &raw); err != nil {
+		return nil, err
+	}
+	protoResps := make([]*pb.CustomCostResponse, len(raw))
+	for i, r := range raw {
+		p := &pb.CustomCostResponse{}
+		if err := protojson.Unmarshal(r, p); err != nil {
+			return nil, err
+		}
+		protoResps[i] = p
+	}
+
+	return protoResps, nil
+}

pkg/plugins/fastly/fastly_focus_mapping.md

@@ -0,0 +1,15 @@
+# Fastly Invoice – FOCUS Attribute Mapping
+
+| FOCUS custom‑cost attribute | Fastly field | Notes |
+|---|---|---|
+| BillingPeriodStart | `billing_start_date` | Billing cycle start (ISO‑8601). |
+| BillingPeriodEnd | `billing_end_date` | Billing cycle end (ISO‑8601). |
+| AccountId | `customer_id` | Fastly customer (account) ID. |
+| Subcategory | `product_line` | Functional grouping, e.g. “Network Services”. |
+| CommitmentDiscountId | `credit_coupon_code` | Discount / credit coupon code. |
+| EffectiveCost | `amount` (line‑item) or `monthly_transaction_amount` (invoice total) | Map depending on desired granularity. |
+| ServiceCategory | `product_group` | Broad service family, e.g. “Compute”. |
+| ServiceName | `product_name` | Specific product name. |
+| PricingQuantity | `units` | Usage quantity. |
+| PricingUnit | `usage_type` | Unit of measure (requests, bandwidth, etc.). |
+| PricingCategory | `product_group` | Geographic vs. functional category. |

pkg/plugins/fastly/fastlyplugin/fastlyconfig.go

@@ -0,0 +1,6 @@
+package fastlyplugin
+
+type FastlyConfig struct {
+	FastlyAPIKey string `json:"fastly_api_key"`
+	LogLevel     string `json:"log_level"`
+}

pkg/plugins/fastly/fastlyplugin/invoices.go

@@ -0,0 +1,103 @@
+package fastlyplugin
+
+import (
+	"encoding/json"
+	"fmt"
+	"strconv"
+	"time"
+)
+
+type InvoiceListResponse struct {
+	Data []Invoice          `json:"data"`
+	Meta PaginationMetadata `json:"meta"`
+}
+
+type Invoice struct {
+	CustomerID               string                `json:"customer_id"`
+	InvoiceID                string                `json:"invoice_id"`
+	InvoicePostedOn          string                `json:"invoice_posted_on"`
+	BillingStartDate         string                `json:"billing_start_date"`
+	BillingEndDate           string                `json:"billing_end_date"`
+	StatementNumber          string                `json:"statement_number"`
+	CurrencyCode             string                `json:"currency_code"`
+	MonthlyTransactionAmount string                `json:"monthly_transaction_amount"`
+	PaymentStatus            string                `json:"payment_status,omitempty"`
+	TransactionLineItems     []TransactionLineItem `json:"transaction_line_items"`
+}
+
+type TransactionLineItem struct {
+	Description      string  `json:"description"`
+	Amount           float64 `json:"amount"`
+	CreditCouponCode string  `json:"credit_coupon_code"`
+	Rate             float64 `json:"rate"`
+	Units            float64 `json:"units"`
+	ProductName      string  `json:"product_name"`
+	ProductGroup     string  `json:"product_group"`
+	ProductLine      string  `json:"product_line"`
+	Region           string  `json:"region"`
+	UsageType        string  `json:"usage_type"`
+}
+
+type PaginationMetadata struct {
+	NextCursor string `json:"next_cursor"`
+	Limit      int    `json:"limit"`
+	Total      int    `json:"total"`
+	Sort       string `json:"sort"`
+}
+
+// Custom UnmarshalJSON for Invoice to handle monthly_transaction_amount 
+// which can come as either a number or string from the API
+func (i *Invoice) UnmarshalJSON(data []byte) error {
+	// Define a temporary struct with the same fields but using interface{} for monthly_transaction_amount
+	type TempInvoice struct {
+		CustomerID               string                `json:"customer_id"`
+		InvoiceID                string                `json:"invoice_id"`
+		InvoicePostedOn          string                `json:"invoice_posted_on"`
+		BillingStartDate         string                `json:"billing_start_date"`
+		BillingEndDate           string                `json:"billing_end_date"`
+		StatementNumber          string                `json:"statement_number"`
+		CurrencyCode             string                `json:"currency_code"`
+		MonthlyTransactionAmount interface{}           `json:"monthly_transaction_amount"`
+		PaymentStatus            string                `json:"payment_status,omitempty"`
+		TransactionLineItems     []TransactionLineItem `json:"transaction_line_items"`
+	}
+
+	var temp TempInvoice
+	if err := json.Unmarshal(data, &temp); err != nil {
+		return err
+	}
+
+	// Copy all fields except MonthlyTransactionAmount
+	i.CustomerID = temp.CustomerID
+	i.InvoiceID = temp.InvoiceID
+	i.InvoicePostedOn = temp.InvoicePostedOn
+	i.BillingStartDate = temp.BillingStartDate
+	i.BillingEndDate = temp.BillingEndDate
+	i.StatementNumber = temp.StatementNumber
+	i.CurrencyCode = temp.CurrencyCode
+	i.PaymentStatus = temp.PaymentStatus
+	i.TransactionLineItems = temp.TransactionLineItems
+
+	// Handle MonthlyTransactionAmount conversion
+	switch v := temp.MonthlyTransactionAmount.(type) {
+	case string:
+		i.MonthlyTransactionAmount = v
+	case float64:
+		i.MonthlyTransactionAmount = strconv.FormatFloat(v, 'f', -1, 64)
+	case int:
+		i.MonthlyTransactionAmount = strconv.Itoa(v)
+	default:
+		if v == nil {
+			i.MonthlyTransactionAmount = ""
+		} else {
+			return fmt.Errorf("unsupported type for monthly_transaction_amount: %T", v)
+		}
+	}
+
+	return nil
+}
+
+// Helper function to parse date strings
+func ParseFastlyDate(dateStr string) (time.Time, error) {
+	return time.Parse(time.RFC3339, dateStr)
+}