Merge pull request #1338 from circleci/put-back-honeycomb

Revert "Remove honeycomb otel provider"

Author: pete-woods

README.md

@@ -20,8 +20,9 @@ See example / template repo [here](https://github.com/circleci/ex-service-templa
 - `httpserver/ginrouter` A common base for configuring a Gin router instance.
 - `httpserver/healthcheck` A healthcheck HTTP server that can accept all the checks from a `system`.
 - `mongoex` **Experimental** Common patterns using when talking to MongoDB.
-- `o11y` Observability that is currently backed by Otel. It also supports outputting
+- `o11y` Observability that is currently backed by Honeycomb. It also supports outputting
   trace data as JSON and plain or colored text output.
+- `o11y/honeycomb` The honeycomb-backed implementation of `o11y`.
 - `o11y/wrappers/o11ygin` `o11y` middleware for the Gin router.
 - `o11y/wrappers/o11ynethttp` `o11y` middleware for the standard Go HTTP server.
 - `rabbit` **Experimental** RabbitMQ publishing client.

config/o11y/doc.go

@@ -0,0 +1,5 @@
+/*
+Package o11y is the primary entrypoint for wiring up a standard configuration of the o11y
+observability system.
+*/
+package o11y

config/o11y/o11y.go

@@ -0,0 +1,177 @@
+package o11y
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"math"
+	"os"
+	"time"
+
+	"github.com/DataDog/datadog-go/statsd"
+	"github.com/cenkalti/backoff/v5"
+	"github.com/honeycombio/libhoney-go/transmission"
+	"github.com/rollbar/rollbar-go"
+
+	"github.com/circleci/ex/config/secret"
+	"github.com/circleci/ex/o11y"
+	"github.com/circleci/ex/o11y/honeycomb"
+)
+
+type Config struct {
+	Statsd            string
+	RollbarToken      secret.String
+	RollbarEnv        string
+	RollbarServerRoot string
+	HoneycombEnabled  bool
+	HoneycombDataset  string
+	HoneycombHost     string
+	HoneycombKey      secret.String
+	SampleTraces      bool
+	SampleKeyFunc     func(map[string]any) string
+	SampleRates       map[string]int
+	Format            string
+	Version           string
+	Service           string
+	StatsNamespace    string
+
+	// Optional
+	Mode                    string
+	Debug                   bool
+	RollbarDisabled         bool
+	StatsdTelemetryDisabled bool
+	Writer                  io.Writer
+
+	// Sender allows setting a custom honeycomb sender, Typically the build-in one is preferred.
+	Sender transmission.Sender
+
+	// Metrics allows setting a custom metrics client. Typically setting Statsd/StatsNamespace is preferred
+	Metrics o11y.ClosableMetricsProvider
+}
+
+// Setup is the primary entrypoint to initialize the o11y system.
+//
+//nolint:funlen
+func Setup(ctx context.Context, o Config) (context.Context, func(context.Context), error) {
+	honeyConfig, err := honeyComb(o)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	hostname, _ := os.Hostname()
+
+	if o.Metrics != nil {
+		honeyConfig.Metrics = o.Metrics
+	} else if o.Statsd == "" {
+		honeyConfig.Metrics = &statsd.NoOpClient{}
+	} else {
+		tags := []string{
+			"service:" + o.Service,
+			"version:" + o.Version,
+			"hostname:" + hostname,
+		}
+		if o.Mode != "" {
+			tags = append(tags, "mode:"+o.Mode)
+		}
+
+		statsdOpts := []statsd.Option{
+			statsd.WithNamespace(o.StatsNamespace),
+			statsd.WithTags(tags),
+		}
+		if o.StatsdTelemetryDisabled {
+			statsdOpts = append(statsdOpts, statsd.WithoutTelemetry())
+		}
+
+		stats, err := backoff.Retry(ctx, func() (*statsd.Client, error) {
+			return statsd.New(o.Statsd, statsdOpts...)
+		}, backoff.WithBackOff(backoff.NewConstantBackOff(time.Second)), backoff.WithMaxTries(31))
+		if err != nil {
+			return ctx, nil, err
+		}
+
+		honeyConfig.Metrics = stats
+	}
+
+	o11yProvider := honeycomb.New(honeyConfig)
+	o11yProvider.AddGlobalField("service", o.Service)
+	o11yProvider.AddGlobalField("version", o.Version)
+	if o.Mode != "" {
+		o11yProvider.AddGlobalField("mode", o.Mode)
+	}
+
+	if o.RollbarToken != "" {
+		client := rollbar.NewAsync(o.RollbarToken.Raw(), o.RollbarEnv, o.Version, hostname, o.RollbarServerRoot)
+		client.SetEnabled(!o.RollbarDisabled)
+		client.Message(rollbar.INFO, "Deployment")
+		o11yProvider = rollBarHoneycombProvider{
+			Provider:      o11yProvider,
+			rollBarClient: client,
+		}
+	}
+
+	ctx = o11y.WithProvider(ctx, o11yProvider)
+
+	return ctx, o11yProvider.Close, nil
+}
+
+type rollBarHoneycombProvider struct {
+	o11y.Provider
+	rollBarClient *rollbar.Client
+}
+
+func (p rollBarHoneycombProvider) Close(ctx context.Context) {
+	p.Provider.Close(ctx)
+	_ = p.rollBarClient.Close()
+}
+
+func (p rollBarHoneycombProvider) RollBarClient() *rollbar.Client {
+	return p.rollBarClient
+}
+
+func honeyComb(o Config) (honeycomb.Config, error) {
+	if o.SampleKeyFunc == nil {
+		o.SampleKeyFunc = func(fields map[string]interface{}) string {
+			// defaults for gin server
+			return fmt.Sprintf("%s %s %v",
+				fields["http.server_name"],
+				fields["http.route"],
+				fields["http.status_code"],
+			)
+		}
+	}
+
+	conf := honeycomb.Config{
+		Host:          o.HoneycombHost,
+		Dataset:       o.HoneycombDataset,
+		Key:           string(o.HoneycombKey),
+		Format:        o.Format,
+		SendTraces:    o.HoneycombEnabled,
+		SampleTraces:  o.SampleTraces,
+		SampleKeyFunc: o.SampleKeyFunc,
+		SampleRates:   o.SampleRates,
+		ServiceName:   o.Service,
+		Debug:         o.Debug,
+		Writer:        o.Writer,
+		Sender:        o.Sender,
+	}
+	return conf, conf.Validate()
+}
+
+func clampToUintMax(v int) uint {
+	if int64(v) >= math.MaxUint32 {
+		return math.MaxUint32 - 1
+	}
+
+	//nolint gosec:G115 // This overflow is handled above
+	return uint(v)
+}
+
+// OtelSampleRates adapts the root o11y package configured map[string]int
+// sample rates to the Otel-required map[string]uint
+func (c *Config) OtelSampleRates() map[string]uint {
+	adapted := make(map[string]uint, len(c.SampleRates))
+	for k, v := range c.SampleRates {
+		adapted[k] = clampToUintMax(v)
+	}
+	return adapted
+}

config/o11y/o11y_test.go

@@ -0,0 +1,135 @@
+package o11y_test
+
+import (
+	"bytes"
+	"context"
+	"strings"
+	"testing"
+
+	"gotest.tools/v3/assert"
+	"gotest.tools/v3/assert/cmp"
+	"gotest.tools/v3/poll"
+
+	o11yconfig "github.com/circleci/ex/config/o11y"
+	"github.com/circleci/ex/config/secret"
+	"github.com/circleci/ex/o11y"
+	"github.com/circleci/ex/o11y/honeycomb"
+	"github.com/circleci/ex/testing/fakestatsd"
+)
+
+func TestO11Y_SecretRedacted(t *testing.T) {
+	// confirm that honeycomb uses the json marshaller and that we dont see secrets
+	buf := bytes.Buffer{}
+	provider := honeycomb.New(honeycomb.Config{
+		Writer: &buf,
+	})
+	ctx := context.Background()
+	ctx, span := provider.StartSpan(ctx, "secret test")
+	s := secret.String("super-secret")
+	span.AddField("secret", s)
+	span.End()
+	provider.Close(ctx)
+	assert.Check(t, !strings.Contains(buf.String(), "super-secret"), buf.String())
+	assert.Check(t, cmp.Contains(buf.String(), "REDACTED"))
+}
+
+func TestO11Y_SecretRedactedColor(t *testing.T) {
+	// This test can be run to *VISUALLY* confirm that color honeycomb
+	// formatter uses the json marshaller under the hood and that we dont see secrets.
+	// Once you provide a writer the Format is ignored so this test does
+	// not assert on anything since we cant catch the output
+	//
+	// The test is left in so that it can be eyeballed with -v
+	// The other buffer based test does assert and it is very likely if secrets
+	// leak in this test then the above test will fail in any case
+	provider := honeycomb.New(honeycomb.Config{
+		Format: "color",
+	})
+	ctx := context.Background()
+	ctx, span := provider.StartSpan(ctx, "secret test")
+	s := secret.String("super-secret")
+	span.AddField("secret", s)
+	span.End()
+	provider.Close(ctx)
+}
+
+func TestSetup_Wiring(t *testing.T) {
+	s := fakestatsd.New(t)
+
+	ctx := context.Background()
+	ctx, cleanup, err := o11yconfig.Setup(ctx, o11yconfig.Config{
+		Statsd:            s.Addr(),
+		RollbarToken:      "qwertyuiop",
+		RollbarDisabled:   true,
+		RollbarEnv:        "production",
+		RollbarServerRoot: "github.com/circleci/ex",
+		HoneycombEnabled:  false,
+		HoneycombDataset:  "does-not-exist",
+		HoneycombKey:      "1234567890",
+		SampleTraces:      false,
+		Format:            "color",
+		Version:           "1.2.3",
+		Service:           "test-service",
+		StatsNamespace:    "test.service",
+		Mode:              "banana",
+		Debug:             true,
+	})
+	assert.Assert(t, err)
+
+	t.Run("Send metric", func(t *testing.T) {
+		p := o11y.FromContext(ctx)
+		err = p.MetricsProvider().Count("my_count", 1, []string{"mytag:myvalue"}, 1)
+		assert.Check(t, err)
+	})
+
+	t.Run("Cleanup provider", func(t *testing.T) {
+		cleanup(ctx)
+	})
+
+	t.Run("Check metrics received", func(t *testing.T) {
+		poll.WaitOn(t, func(t poll.LogT) poll.Result {
+			metrics := s.Metrics()
+			if len(metrics) == 0 {
+				return poll.Continue("no metrics found yet")
+			}
+			return poll.Success()
+		})
+
+		metrics := s.Metrics()
+		assert.Assert(t, cmp.Len(metrics, 1))
+		metric := metrics[0]
+		assert.Check(t, cmp.Equal("test.service.my_count", metric.Name))
+		assert.Check(t, cmp.Equal("1|c|", metric.Value))
+		assert.Check(t, cmp.Contains(metric.Tags, "service:test-service"))
+		assert.Check(t, cmp.Contains(metric.Tags, "version:1.2.3"))
+		assert.Check(t, cmp.Contains(metric.Tags, "mode:banana"))
+		assert.Check(t, cmp.Contains(metric.Tags, "mytag:myvalue"))
+	})
+}
+
+func TestSetup_WithWriter(t *testing.T) {
+	buf := bytes.Buffer{}
+	ctx := context.Background()
+	ctx, cleanup, err := o11yconfig.Setup(ctx, o11yconfig.Config{
+		Writer: &buf,
+	})
+	assert.Assert(t, err)
+	defer cleanup(ctx)
+
+	o11y.Log(ctx, "some log output")
+
+	assert.Check(t, cmp.Contains(buf.String(), "some log output"))
+}
+
+func TestConfig_OTelSampleRates(t *testing.T) {
+	conf := o11yconfig.Config{
+		SampleRates: map[string]int{
+			"foo": 128,
+		},
+	}
+	otelSampleRates := conf.OtelSampleRates()
+
+	assert.Check(t, cmp.DeepEqual(otelSampleRates, map[string]uint{
+		"foo": uint(128),
+	}))
+}

config/o11y/otel.go

@@ -1,14 +1,9 @@
-/*
-Package o11y is the primary entrypoint for wiring up a standard configuration of the o11y
-observability system.
-*/
 package o11y
 
 import (
 	"context"
 	"fmt"
 	"io"
-	"math"
 	"os"
 	"time"
 
@@ -23,9 +18,6 @@ import (
 	"github.com/circleci/ex/o11y/otel"
 )
 
-// SampleOut is the maximum value the sample rate can be
-const SampleOut = math.MaxUint32
-
 // OtelConfig contains all the things we need to configure for otel based instrumentation.
 type OtelConfig struct {
 	GrpcHostAndPort string