FFM-8662 Add WaitForInitialzed / Improve Auth Retries

Author: erdirowlands

README.md

@@ -29,12 +29,48 @@ To follow along with our test code sample, make sure you’ve:
 - [Created a server SDK key and made a copy of it](https://ngdocs.harness.io/article/1j7pdkqh7j-create-a-feature-flag#step_3_create_an_sdk_key)
 
 
-### Install the SDK
+## Install the SDK
 Install the golang SDK using go
 ```golang
 go get github.com/harness/ff-golang-server-sdk
 ```
 
+## Initialize the SDK
+### Non-blocking Initialization (Default)
+By default, when initializing the Harness Feature Flags client, the initialization process is non-blocking. This means that the client creation call will return immediately, allowing your application to continue its startup process without waiting for the client to be fully initialized. Here’s an example of creating a non-blocking client:
+
+```go
+client, err := harness.NewCfClient(apiKey)
+```
+In this scenario, the client will initialize in the background, making it possible to use the client even if it hasn’t finished initializing. 
+Be mindful that if you attempt to evaluate a feature flag before the client has fully initialized, it will return the default value provided in the evaluation call.
+
+### Blocking Initialization
+In some cases, you may want your application to wait for the client to finish initializing before continuing. To achieve this, you can use the `WithWaitForInitialized` option, which will block until the client is fully initialized. Example usage:
+
+```go
+client, err := harness.NewCfClient(sdkKey, harness.WithWaitForInitialized(true))
+
+if err != nil {
+log.ErrorF("could not connect to FF servers %s", err)
+}
+```
+
+
+In this example, WaitForInitialized will block for up to 5 authentication attempts. If the client is not initialized within 5 authentication attempts, it will return an error.
+
+This can be useful if you need to unblock after a certain time. **NOTE**: if you evaluate a feature flag in this state
+the default variation will be returned.
+
+```go
+// Try to authenticate only 5 times before returning a result
+client, err := harness.NewCfClient(sdkKey, harness.WithWaitForInitialized(true), harness.WithMaxAuthRetries(5))
+
+if err != nil {
+log.Fatalf("client did not initialize in time: %s", err)
+}
+```
+
 ### Code Sample
 The following is a complete code example that you can use to test the `harnessappdemodarkmode` Flag you created on the Harness Platform. When you run the code it will:
 - Connect to the FF service.
@@ -114,6 +150,8 @@ export FF_API_KEY=<your key here>
 docker run -e FF_API_KEY=$FF_API_KEY -v $(pwd):/app -w /app golang:1.17 go run examples/getting_started.go
 ```
 
+
+
 ### Additional Reading
 
 For further examples and config options, see the [Golang SDK Reference](https://ngdocs.harness.io/article/4c8wljx60w-feature-flag-sdks-go-application).

analyticsservice/analytics.go

@@ -24,7 +24,7 @@ const (
 	variationValueAttribute      string = "featureValue"
 	targetAttribute              string = "target"
 	sdkVersionAttribute          string = "SDK_VERSION"
-	sdkVersion                   string = "1.0.0"
+	sdkVersion                   string = "1.12.0"
 	sdkTypeAttribute             string = "SDK_TYPE"
 	sdkType                      string = "server"
 	sdkLanguageAttribute         string = "SDK_LANGUAGE"

client/client.go

@@ -6,6 +6,7 @@ import (
 	"errors"
 	"fmt"
 	"log"
+	"math/rand"
 	"net/http"
 	"strings"
 	"sync"
@@ -53,8 +54,10 @@ type CfClient struct {
 	streamConnectedLock sync.RWMutex
 	authenticated       chan struct{}
 	postEvalChan        chan evaluation.PostEvalData
-	initialized         bool
-	initializedLock     sync.RWMutex
+	initializedBool     bool
+	initializedBoolLock sync.RWMutex
+	initialized         chan struct{}
+	initializedErr      chan error
 	analyticsService    *analyticsservice.AnalyticsService
 	clusterIdentifier   string
 	stop                chan struct{}
@@ -82,10 +85,13 @@ func NewCfClient(sdkKey string, options ...ConfigOption) (*CfClient, error) {
 		postEvalChan:      make(chan evaluation.PostEvalData),
 		stop:              make(chan struct{}),
 		stopped:           newAtomicBool(false),
+		initialized:       make(chan struct{}),
+		initializedErr:    make(chan error),
 	}
 
 	if sdkKey == "" {
-		return client, types.ErrSdkCantBeEmpty
+		config.Logger.Errorf("Initialization failed: SDK Key cannot be empty. Please provide a valid SDK Key to initialize the client.")
+		return client, EmptySDKKeyError
 	}
 
 	var err error
@@ -106,6 +112,25 @@ func NewCfClient(sdkKey string, options ...ConfigOption) (*CfClient, error) {
 	}
 
 	client.start()
+	if config.waitForInitialized {
+		var initErr error
+
+		select {
+		case <-client.initialized:
+			return client, nil
+		case err := <-client.initializedErr:
+			initErr = err
+		}
+
+		if initErr != nil {
+			config.Logger.Errorf("Initialization failed: '%v'", initErr)
+			// We return the client but leave it in un-initialized state by not setting the relevant initialized flag.
+			// This ensures any subsequent calls to the client don't potentially result in a panic. For example, if a user
+			// calls BoolVariation we can log that the client is not initialized and return the user the default variation.
+			return client, initErr
+		}
+	}
+
 	return client, nil
 }
 
@@ -117,7 +142,11 @@ func (c *CfClient) start() {
 		cancel()
 	}()
 
-	go c.initAuthentication(ctx)
+	go func() {
+		if err := c.initAuthentication(context.Background()); err != nil {
+			c.initializedErr <- err
+		}
+	}()
 	go c.setAnalyticsServiceClient(ctx)
 	go c.pullCronJob(ctx)
 }
@@ -141,15 +170,15 @@ func (c *CfClient) GetClusterIdentifier() string {
 // and successfully retrieved flags.  If it takes longer than 1 minute the call will timeout and return an error.
 func (c *CfClient) IsInitialized() (bool, error) {
 	for i := 0; i < 30; i++ {
-		c.initializedLock.RLock()
-		if c.initialized {
-			c.initializedLock.RUnlock()
+		c.initializedBoolLock.RLock()
+		if c.initializedBool {
+			c.initializedBoolLock.RUnlock()
 			return true, nil
 		}
-		c.initializedLock.RUnlock()
-		time.Sleep(time.Second * 2)
+		c.initializedBoolLock.RUnlock()
+		c.config.sleeper.Sleep(time.Second * 2)
 	}
-	return false, fmt.Errorf("timeout waiting to initialize")
+	return false, InitializeTimeoutError{}
 }
 
 func (c *CfClient) retrieve(ctx context.Context) bool {
@@ -185,9 +214,12 @@ func (c *CfClient) retrieve(ctx context.Context) bool {
 	}
 
 	if ok {
-		c.initializedLock.Lock()
-		c.initialized = true
-		c.initializedLock.Unlock()
+		// This flag is used by `IsInitialized` so set to true.
+		c.initializedBoolLock.Lock()
+		c.initializedBool = true
+		c.initializedBoolLock.Unlock()
+
+		close(c.initialized)
 	}
 	return ok
 }
@@ -233,15 +265,45 @@ func (c *CfClient) streamConnect(ctx context.Context) {
 	c.streamConnected = true
 }
 
-func (c *CfClient) initAuthentication(ctx context.Context) {
-	// attempt to authenticate every minute until we succeed
+func (c *CfClient) initAuthentication(ctx context.Context) error {
+	baseDelay := 1 * time.Second
+	maxDelay := 1 * time.Minute
+	factor := 2.0
+	currentDelay := baseDelay
+
+	attempts := 0
+
 	for {
 		err := c.authenticate(ctx)
 		if err == nil {
-			return
+			return nil
+		}
+
+		var nonRetryableAuthError NonRetryableAuthError
+		if errors.As(err, &nonRetryableAuthError) {
+			c.config.Logger.Error("Authentication failed with a non-retryable error: '%s %s' Default variations will now be served", nonRetryableAuthError.StatusCode, nonRetryableAuthError.Message)
+			return err
+		}
+
+		// -1 is the default maxAuthRetries option and indicates there should be no max attempts
+		if c.config.maxAuthRetries != -1 && attempts >= c.config.maxAuthRetries {
+			c.config.Logger.Errorf("Authentication failed with error: '%s'. Exceeded max attempts: '%v'.", err, c.config.maxAuthRetries)
+			return err
+		}
+
+		jitter := time.Duration(rand.Float64() * float64(currentDelay))
+		delayWithJitter := currentDelay + jitter
+
+		c.config.Logger.Errorf("Authentication failed with error: '%s'. Retrying in %v.", err, delayWithJitter)
+		c.config.sleeper.Sleep(delayWithJitter)
+
+		currentDelay *= time.Duration(factor)
+		if currentDelay > maxDelay {
+			currentDelay = maxDelay
 		}
-		c.config.Logger.Errorf("Authentication failed. Trying again in 1 minute: %s", err)
-		time.Sleep(1 * time.Minute)
+
+		attempts++
+
 	}
 }
 
@@ -262,9 +324,31 @@ func (c *CfClient) authenticate(ctx context.Context) error {
 	if err != nil {
 		return err
 	}
-	// should be login to harness and get account data (JWT token)
+
+	responseError := findErrorInResponse(response)
+
+	// Indicate that we should retry
+	if responseError != nil && responseError.Code == "500" {
+		return RetryableAuthError{
+			StatusCode: responseError.Code,
+			Message:    responseError.Message,
+		}
+	}
+
+	// Indicate that we shouldn't retry on non-500 errors
+	if responseError != nil {
+		return NonRetryableAuthError{
+			StatusCode: responseError.Code,
+			Message:    responseError.Message,
+		}
+	}
+
+	// Defensive check to handle the case that all responses are nil
 	if response.JSON200 == nil {
-		return fmt.Errorf("error while authenticating %v", ErrUnauthorized)
+		return RetryableAuthError{
+			StatusCode: "No errpr status code returned from server",
+			Message:    "No error message returned from server ",
+		}
 	}
 
 	c.token = response.JSON200.AuthToken
@@ -427,6 +511,10 @@ func (c *CfClient) setAnalyticsServiceClient(ctx context.Context) {
 //
 // Returns defaultValue if there is an error or if the flag doesn't exist
 func (c *CfClient) BoolVariation(key string, target *evaluation.Target, defaultValue bool) (bool, error) {
+	if !c.initializedBool {
+		c.config.Logger.Info("Error when calling BoolVariation and returning default variation: 'Client is not initialized'")
+		return defaultValue, fmt.Errorf("%w: Client is not initialized", DefaultVariationReturnedError)
+	}
 	value := c.evaluator.BoolVariation(key, target, defaultValue)
 	return value, nil
 }
@@ -435,6 +523,10 @@ func (c *CfClient) BoolVariation(key string, target *evaluation.Target, defaultV
 //
 // Returns defaultValue if there is an error or if the flag doesn't exist
 func (c *CfClient) StringVariation(key string, target *evaluation.Target, defaultValue string) (string, error) {
+	if !c.initializedBool {
+		c.config.Logger.Info("Error when calling StringVariation and returning default variation: 'Client is not initialized'")
+		return defaultValue, fmt.Errorf("%w: Client is not initialized", DefaultVariationReturnedError)
+	}
 	value := c.evaluator.StringVariation(key, target, defaultValue)
 	return value, nil
 }
@@ -443,6 +535,10 @@ func (c *CfClient) StringVariation(key string, target *evaluation.Target, defaul
 //
 // Returns defaultValue if there is an error or if the flag doesn't exist
 func (c *CfClient) IntVariation(key string, target *evaluation.Target, defaultValue int64) (int64, error) {
+	if !c.initializedBool {
+		c.config.Logger.Info("Error when calling IntVariation and returning default variation: 'Client is not initialized'")
+		return defaultValue, fmt.Errorf("%w: Client is not initialized", DefaultVariationReturnedError)
+	}
 	value := c.evaluator.IntVariation(key, target, int(defaultValue))
 	return int64(value), nil
 }
@@ -451,6 +547,10 @@ func (c *CfClient) IntVariation(key string, target *evaluation.Target, defaultVa
 //
 // Returns defaultValue if there is an error or if the flag doesn't exist
 func (c *CfClient) NumberVariation(key string, target *evaluation.Target, defaultValue float64) (float64, error) {
+	if !c.initializedBool {
+		c.config.Logger.Info("Error when calling NumberVariation and returning default variation: 'Client is not initialized'")
+		return defaultValue, fmt.Errorf("%w: Client is not initialized", DefaultVariationReturnedError)
+	}
 	value := c.evaluator.NumberVariation(key, target, defaultValue)
 	return value, nil
 }
@@ -460,13 +560,20 @@ func (c *CfClient) NumberVariation(key string, target *evaluation.Target, defaul
 //
 // Returns defaultValue if there is an error or if the flag doesn't exist
 func (c *CfClient) JSONVariation(key string, target *evaluation.Target, defaultValue types.JSON) (types.JSON, error) {
+	if !c.initializedBool {
+		c.config.Logger.Info("Error when calling JSONVariation and returning default variation: 'Client is not initialized'")
+		return defaultValue, fmt.Errorf("%w: Client is not initialized", DefaultVariationReturnedError)
+	}
 	value := c.evaluator.JSONVariation(key, target, defaultValue)
 	return value, nil
 }
 
 // Close shuts down the Feature Flag client. After calling this, the client
 // should no longer be used
 func (c *CfClient) Close() error {
+	if !c.initializedBool {
+		return errors.New("attempted to close client that is not initialized")
+	}
 	if c.stopped.get() {
 		return errors.New("client already closed")
 	}
@@ -526,3 +633,14 @@ func getLogger(options ...ConfigOption) logger.Logger {
 	}
 	return dummyConfig.Logger
 }
+
+// findErrorInResponse parses an auth response and returns the response error if it exists
+func findErrorInResponse(resp *rest.AuthenticateResponse) *rest.Error {
+	responseErrors := []*rest.Error{resp.JSON401, resp.JSON403, resp.JSON404, resp.JSON500}
+	for _, responseError := range responseErrors {
+		if responseError != nil {
+			return responseError
+		}
+	}
+	return nil
+}