Skip to content
This repository was archived by the owner on Aug 17, 2020. It is now read-only.

scope.yml config file support #187

Open
wants to merge 12 commits into
base: master
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
49 changes: 30 additions & 19 deletions agent/agent.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ import (
"github.com/mitchellh/go-homedir"
"github.com/opentracing/opentracing-go"

"go.undefinedlabs.com/scopeagent/env"
"go.undefinedlabs.com/scopeagent/config"
scopeError "go.undefinedlabs.com/scopeagent/errors"
"go.undefinedlabs.com/scopeagent/instrumentation"
scopetesting "go.undefinedlabs.com/scopeagent/instrumentation/testing"
Expand Down Expand Up @@ -66,6 +66,8 @@ var (

testingModeFrequency = time.Second
nonTestingModeFrequency = time.Minute

cfg = config.Get()
)

func WithApiKey(apiKey string) Option {
Expand Down Expand Up @@ -210,13 +212,13 @@ func NewAgent(options ...Option) (*Agent, error) {
agent.logger = log.New(ioutil.Discard, "", 0)
}

agent.debugMode = agent.debugMode || env.ScopeDebug.Value
agent.debugMode = agent.debugMode || *cfg.Debug

configProfile := GetConfigCurrentProfile()

if agent.apiKey == "" || agent.apiEndpoint == "" {
if dsn, set := env.ScopeDsn.Tuple(); set && dsn != "" {
dsnApiKey, dsnApiEndpoint, dsnErr := parseDSN(dsn)
if cfg.Dsn != nil && *cfg.Dsn != "" {
dsnApiKey, dsnApiEndpoint, dsnErr := parseDSN(*cfg.Dsn)
if dsnErr != nil {
agent.logger.Printf("Error parsing dsn value: %v\n", dsnErr)
} else {
Expand All @@ -229,8 +231,8 @@ func NewAgent(options ...Option) (*Agent, error) {
}

if agent.apiKey == "" {
if apiKey, set := env.ScopeApiKey.Tuple(); set && apiKey != "" {
agent.apiKey = apiKey
if cfg.ApiKey != nil && *cfg.ApiKey != "" {
agent.apiKey = *cfg.ApiKey
} else if configProfile != nil {
agent.logger.Println("API key found in the native app configuration")
agent.apiKey = configProfile.ApiKey
Expand All @@ -242,12 +244,13 @@ func NewAgent(options ...Option) (*Agent, error) {
}

if agent.apiEndpoint == "" {
if endpoint, set := env.ScopeApiEndpoint.Tuple(); set && endpoint != "" {
agent.apiEndpoint = endpoint
if cfg.ApiEndpoint != nil && *cfg.ApiEndpoint != "" {
agent.apiEndpoint = *cfg.ApiEndpoint
} else if configProfile != nil {
agent.logger.Println("API endpoint found in the native app configuration")
agent.apiEndpoint = configProfile.ApiEndpoint
} else {
endpoint := "https://app.scope.dev"
agent.logger.Printf("using default endpoint: %v\n", endpoint)
agent.apiEndpoint = endpoint
}
Expand Down Expand Up @@ -287,13 +290,19 @@ func NewAgent(options ...Option) (*Agent, error) {
agent.metadata[tags.GoVersion] = runtime.Version()

// Service name
addElementToMapIfEmpty(agent.metadata, tags.Service, env.ScopeService.Value)
if cfg.Service != nil {
addElementToMapIfEmpty(agent.metadata, tags.Service, *cfg.Service)
}

// Configurations
addElementToMapIfEmpty(agent.metadata, tags.ConfigurationKeys, env.ScopeConfiguration.Value)
if cfg.Configuration != nil {
addElementToMapIfEmpty(agent.metadata, tags.ConfigurationKeys, cfg.Configuration)
}

// Metadata
addToMapIfEmpty(agent.metadata, env.ScopeMetadata.Value)
if cfg.Metadata != nil {
addToMapIfEmpty(agent.metadata, cfg.Metadata)
}

// Git data
addToMapIfEmpty(agent.metadata, getGitInfoFromEnv())
Expand Down Expand Up @@ -323,17 +332,19 @@ func NewAgent(options ...Option) (*Agent, error) {
agent.metadata[tags.SourceRoot] = sourceRoot

if !agent.testingMode {
if env.ScopeTestingMode.IsSet {
agent.testingMode = env.ScopeTestingMode.Value
if cfg.TestingMode != nil {
agent.testingMode = *cfg.TestingMode
} else {
agent.testingMode = agent.metadata[tags.CI].(bool)
}
}

if agent.failRetriesCount == 0 {
agent.failRetriesCount = env.ScopeTestingFailRetries.Value
if agent.failRetriesCount == 0 && cfg.Instrumentation.TestsFrameworks.FailRetries != nil {
agent.failRetriesCount = *cfg.Instrumentation.TestsFrameworks.FailRetries
}
if cfg.Instrumentation.TestsFrameworks.PanicAsFail != nil {
agent.panicAsFail = agent.panicAsFail || *cfg.Instrumentation.TestsFrameworks.PanicAsFail
}
agent.panicAsFail = agent.panicAsFail || env.ScopeTestingPanicAsFail.Value

if agent.debugMode {
agent.logMetadata()
Expand Down Expand Up @@ -362,7 +373,7 @@ func NewAgent(options ...Option) (*Agent, error) {
instrumentation.SetTracer(agent.tracer)
instrumentation.SetLogger(agent.logger)
instrumentation.SetSourceRoot(sourceRoot)
if agent.setGlobalTracer || env.ScopeTracerGlobal.Value {
if agent.setGlobalTracer || (cfg.Tracer.Global != nil && *cfg.Tracer.Global) {
opentracing.SetGlobalTracer(agent.Tracer())
}

Expand Down Expand Up @@ -457,8 +468,8 @@ func generateAgentID() string {
}

func getLogPath() (string, error) {
if env.ScopeLoggerRoot.IsSet {
return env.ScopeLoggerRoot.Value, nil
if cfg.Logger.Root != nil {
return *cfg.Logger.Root, nil
}

logFolder := ""
Expand Down
6 changes: 4 additions & 2 deletions agent/agent_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ import (
"testing"
"time"

"go.undefinedlabs.com/scopeagent/env"
"go.undefinedlabs.com/scopeagent/config"
"go.undefinedlabs.com/scopeagent/tags"
)

Expand Down Expand Up @@ -134,7 +134,9 @@ func sameElements(a, b []string) bool {
}

func TestTildeExpandRaceMetadata(t *testing.T) {
env.ScopeSourceRoot.Value = "~/scope"
cfg := config.Get()
sroot := "~/scope"
cfg.SourceRoot = &sroot
agent, err := NewAgent(WithApiKey("123"), WithTestingModeEnabled())
if err != nil {
t.Fatal(err)
Expand Down
23 changes: 13 additions & 10 deletions agent/git.go
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,6 @@ import (
"strings"

"github.com/google/uuid"
"go.undefinedlabs.com/scopeagent/env"
"go.undefinedlabs.com/scopeagent/tags"
)

Expand Down Expand Up @@ -163,6 +162,10 @@ func getGitFolder() (string, error) {
}

func getGitDiff() *GitDiff {
if cfg.Instrumentation.DiffSummary != nil && !*cfg.Instrumentation.DiffSummary {
return nil
}

var diff string
if diffBytes, err := exec.Command("git", "diff", "--numstat").Output(); err == nil {
diff = string(diffBytes)
Expand Down Expand Up @@ -229,20 +232,20 @@ func getGitInfoFromGitFolder() map[string]interface{} {
func getGitInfoFromEnv() map[string]interface{} {
gitInfo := map[string]interface{}{}

if repository, set := env.ScopeRepository.Tuple(); set && repository != "" {
gitInfo[tags.Repository] = repository
if cfg.Repository != nil && *cfg.Repository != "" {
gitInfo[tags.Repository] = *cfg.Repository
}
if commit, set := env.ScopeCommitSha.Tuple(); set && commit != "" {
gitInfo[tags.Commit] = commit
if cfg.CommitSha != nil && *cfg.CommitSha != "" {
gitInfo[tags.Commit] = *cfg.CommitSha
}
if sourceRoot, set := env.ScopeSourceRoot.Tuple(); set && sourceRoot != "" {
if cfg.SourceRoot != nil && *cfg.SourceRoot != "" {
// We check if is a valid and existing folder
if fInfo, err := os.Stat(sourceRoot); err == nil && fInfo.IsDir() {
gitInfo[tags.SourceRoot] = sourceRoot
if fInfo, err := os.Stat(*cfg.SourceRoot); err == nil && fInfo.IsDir() {
gitInfo[tags.SourceRoot] = *cfg.SourceRoot
}
}
if branch, set := env.ScopeBranch.Tuple(); set && branch != "" {
gitInfo[tags.Branch] = branch
if cfg.Branch != nil && *cfg.Branch != "" {
gitInfo[tags.Branch] = *cfg.Branch
}

return gitInfo
Expand Down
21 changes: 16 additions & 5 deletions agent/recorder.go
Original file line number Diff line number Diff line change
Expand Up @@ -38,8 +38,10 @@ type (
debugMode bool
metadata map[string]interface{}

payloadSpans []PayloadSpan
payloadEvents []PayloadEvent
payloadSpans []PayloadSpan
payloadEvents []PayloadEvent
maxSpansPerPayload int
maxEventsPerPayload int

flushFrequency time.Duration
url string
Expand Down Expand Up @@ -82,6 +84,16 @@ func NewSpanRecorder(agent *Agent) *SpanRecorder {
r.url = agent.getUrl("api/agent/ingest")
r.client = &http.Client{}
r.stats = &RecorderStats{}
if cfg.Tracer.Dispatcher.Events.MaxPayloadSize != nil {
r.maxEventsPerPayload = *cfg.Tracer.Dispatcher.Events.MaxPayloadSize
} else {
r.maxEventsPerPayload = 1000
}
if cfg.Tracer.Dispatcher.Spans.MaxPayloadSize != nil {
r.maxSpansPerPayload = *cfg.Tracer.Dispatcher.Spans.MaxPayloadSize
} else {
r.maxSpansPerPayload = 1000
}
r.t.Go(r.loop)
return r
}
Expand Down Expand Up @@ -142,11 +154,10 @@ func (r *SpanRecorder) loop() error {
// Sends the spans in the buffer to Scope
func (r *SpanRecorder) sendSpans() (error, bool) {
atomic.AddInt64(&r.stats.sendSpansCalls, 1)
const batchSize = 1000
var lastError error
for {
spans, spMore, spTotal := r.popPayloadSpan(batchSize)
events, evMore, evTotal := r.popPayloadEvents(batchSize)
spans, spMore, spTotal := r.popPayloadSpan(r.maxSpansPerPayload)
events, evMore, evTotal := r.popPayloadEvents(r.maxEventsPerPayload)

payload := map[string]interface{}{
"metadata": r.metadata,
Expand Down
70 changes: 70 additions & 0 deletions config/types.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,70 @@
package config

type (
ScopeConfig struct {
Dsn *string `env:"SCOPE_DSN"`
ApiKey *string `env:"SCOPE_APIKEY"`
ApiEndpoint *string `env:"SCOPE_API_ENDPOINT"`
Service *string `yaml:"service" env:"SCOPE_SERVICE" default:"default"`
Repository *string `yaml:"repository" env:"SCOPE_REPOSITORY"`
CommitSha *string `yaml:"commit_sha" env:"SCOPE_COMMIT_SHA"`
Branch *string `yaml:"branch" env:"SCOPE_BRANCH"`
SourceRoot *string `yaml:"source_root" env:"SCOPE_SOURCE_ROOT"`
Logger LoggerConfig `yaml:"logger"`
Metadata map[string]interface{} `yaml:"metadata" env:"SCOPE_METADATA"`
Configuration []string `yaml:"configuration" env:"SCOPE_CONFIGURATION" default:"platform.name, platform.architecture, go.version"`
TestingMode *bool `yaml:"testing_mode" env:"SCOPE_TESTING_MODE" default:"false"`
Instrumentation InstrumentationConfig `yaml:"instrumentation"`
Tracer TracerConfig `yaml:"tracer"`
Debug *bool `env:"SCOPE_DEBUG" default:"false"`
ConfigPath *string
LoadError error
}
LoggerConfig struct {
Root *string `yaml:"root" env:"SCOPE_LOGGER_ROOT, SCOPE_LOG_ROOT_PATH"`
}
InstrumentationConfig struct {
DiffSummary *bool `yaml:"diff_summary" env:"SCOPE_INSTRUMENTATION_DIFF_SUMMARY" default:"true"`
TestsFrameworks InstrumentationTestsFrameworksConfig `yaml:"tests_frameworks"`
DB InstrumentationDatabaseConfig `yaml:"db"`
Http InstrumentationHttpConfig `yaml:"http"`
Logger InstrumentationLoggerConfig `yaml:"logger"`
}
InstrumentationTestsFrameworksConfig struct {
FailRetries *int `yaml:"fail_retries" env:"SCOPE_INSTRUMENTATION_TESTS_FRAMEWORKS_FAIL_RETRIES" default:"0"`
PanicAsFail *bool `yaml:"panic_as_fail" env:"SCOPE_INSTRUMENTATION_TESTS_FRAMEWORKS_PANIC_AS_FAIL" default:"false"`
}
InstrumentationDatabaseConfig struct {
StatementValues *bool `yaml:"statement_values" env:"SCOPE_INSTRUMENTATION_DB_STATEMENT_VALUES" default:"false"`
Stacktrace *bool `yaml:"stacktrace" env:"SCOPE_INSTRUMENTATION_DB_STACKTRACE" default:"false"`
}
InstrumentationHttpConfig struct {
Client *bool `yaml:"client" env:"SCOPE_INSTRUMENTATION_HTTP_CLIENT" default:"true"`
Server *bool `yaml:"server" env:"SCOPE_INSTRUMENTATION_HTTP_SERVER" default:"true"`
Payloads *bool `yaml:"payloads" env:"SCOPE_INSTRUMENTATION_HTTP_PAYLOADS" default:"false"`
Stacktrace *bool `yaml:"stacktrace" env:"SCOPE_INSTRUMENTATION_HTTP_STACKTRACE" default:"false"`
Headers []string `yaml:"headers" env:"SCOPE_INSTRUMENTATION_HTTP_HEADERS"`
}
InstrumentationLoggerConfig struct {
StandardLogger *bool `yaml:"standard_logger" env:"SCOPE_INSTRUMENTATION_LOGGER_STANDARD_LOGGER" default:"true"`
StandardOutput *bool `yaml:"standard_output" env:"SCOPE_INSTRUMENTATION_LOGGER_STANDARD_OUTPUT" default:"false"`
StandardError *bool `yaml:"standard_error" env:"SCOPE_INSTRUMENTATION_LOGGER_STANDARD_ERROR" default:"false"`
}
TracerConfig struct {
Global *bool `yaml:"global" env:"SCOPE_TRACER_GLOBAL, SCOPE_SET_GLOBAL_TRACER" default:"false"`
Dispatcher TracerDispatcherConfig `yaml:"dispatcher"`
}
TracerDispatcherConfig struct {
HealthCheckFrequency *int `yaml:"healthcheck_frecuency" env:"SCOPE_TRACER_DISPATCHER_HEALTHCHECK_FRECUENCY" default:"1000"`
HealthCheckFrequencyInTestMode *int `yaml:"healthcheck_frecuency_in_testmode" env:"SCOPE_TRACER_DISPATCHER_HEALTHCHECK_FRECUENCY_IN_TESTMODE" default:"60000"`
ConcurrencyLevel *int `yaml:"concurrency_level" env:"SCOPE_TRACER_DISPATCHER_CONCURRENCY_LEVEL" default:"1"`
Spans TracerDispatcherSpansConfig `yaml:"spans"`
Events TracerDispatcherEventsConfig `yaml:"events"`
}
TracerDispatcherSpansConfig struct {
MaxPayloadSize *int `yaml:"max_payload_size" env:"SCOPE_TRACER_DISPATCHER_SPANS_MAX_PAYLOAD_SIZE" default:"1000"`
}
TracerDispatcherEventsConfig struct {
MaxPayloadSize *int `yaml:"max_payload_size" env:"SCOPE_TRACER_DISPATCHER_EVENTS_MAX_PAYLOAD_SIZE" default:"1000"`
}
)
81 changes: 81 additions & 0 deletions config/vars.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,81 @@
package config

import (
"errors"
"fmt"
"io/ioutil"
"os"
"path/filepath"
"sync"

"gopkg.in/yaml.v2"

"github.com/undefinedlabs/go-env"
)

var (
current *ScopeConfig
m sync.RWMutex
)

func Get() *ScopeConfig {
// We check is already loaded with a reader lock
m.RLock()
if current != nil {
defer m.RUnlock()
return current
}
m.RUnlock()

// Is not loaded we block to load it
m.Lock()
defer m.Unlock()
if current != nil {
return current
}
var config ScopeConfig
content, path, err := readConfigurationFile()
if err == nil {
config.ConfigPath = path
_ = yaml.Unmarshal(content, &config)
if config.Metadata != nil {
for k, v := range config.Metadata {
if str, ok := v.(string); ok {
config.Metadata[k] = os.ExpandEnv(str)
}
}
}
} else {
config.LoadError = err
}
_, err = env.UnmarshalFromEnviron(&config)
if err != nil {
config.LoadError = err
}
current = &config
return current
}

func readConfigurationFile() ([]byte, *string, error) {
dir, err := os.Getwd()
if err != nil {
return nil, nil, err
}
for {
rel, _ := filepath.Rel("/", dir)
// Exit the loop once we reach the basePath.
if rel == "." {
break
}

path := fmt.Sprintf("%v/scope.yml", dir)
dat, err := ioutil.ReadFile(path)
if err == nil {
return dat, &path, nil
}

// Going up!
dir += "/.."
}
return nil, nil, errors.New("configuration not found")
}
Loading