Skip to content

Commit 5996323

Browse files
arewmarewm
committed
Add configuration for custom bundle resolver backoff
A bundle resolver is susceptible to flakes in network communication. In order to increase reliability, we can add backoff retries to mitigate transient issues. While it may be possible to add a generic retry functionality around resolvers, implementation will likely be simpler if the retry functionality is isolated to the individual resolvers. Future work is needed to add retries to git resolver requests. Partially addresses #8571 Signed-off-by: arewm <[email protected]>
1 parent 00ed658 commit 5996323

File tree

4 files changed

+151
-6
lines changed

4 files changed

+151
-6
lines changed

‎docs/bundle-resolver.md

+8-3
Original file line numberDiff line numberDiff line change
@@ -36,9 +36,14 @@ for the name, namespace and defaults that the resolver ships with.
3636

3737
### Options
3838

39-
| Option Name | Description | Example Values |
40-
|---------------------------|--------------------------------------------------------------|-----------------------|
41-
| `default-kind` | The default layer kind in the bundle image. | `task`, `pipeline` |
39+
| Option Name | Description | Example Values |
40+
|----------------------|-------------------------------------------------------------------|-----------------------|
41+
| `backoff-duration` | The initial duration for a backoff. | `500ms`, `2s` |
42+
| `backoff-factor` | The factor by which the sleep duration increases every step. | `2.5`, `4.0` |
43+
| `backoff-jitter` | A random amount of additioan sleep between 0 andduration * jitter.| `0.1`, `0.5` |
44+
| `backoff-steps` | The number of backoffs to attempt. | `3`, `7` |
45+
| `backoff-cap` | The maxumum backoff duration. If reached, remaining steps are zeroed.| `10s`, `20s` |
46+
| `default-kind` | The default layer kind in the bundle image. | `task`, `pipeline` |
4247

4348
## Usage
4449

‎pkg/resolution/resolver/bundle/bundle.go

+10-2
Original file line numberDiff line numberDiff line change
@@ -142,9 +142,17 @@ func retrieveImage(ctx context.Context, keychain authn.Keychain, ref string) (st
142142
if err != nil {
143143
return "", nil, fmt.Errorf("%s is an unparseable image reference: %w", ref, err)
144144
}
145+
customRetryBackoff, err := GetBundleResolverBackoff(ctx)
146+
if err == nil {
147+
img, err := remote.Image(imgRef, remote.WithAuthFromKeychain(keychain), remote.WithContext(ctx),
148+
remote.WithRetryBackoff(customRetryBackoff))
145149

146-
img, err := remote.Image(imgRef, remote.WithAuthFromKeychain(keychain), remote.WithContext(ctx))
147-
return imgRef.Context().Name(), img, err
150+
return imgRef.Context().Name(), img, err
151+
} else {
152+
img, err := remote.Image(imgRef, remote.WithAuthFromKeychain(keychain), remote.WithContext(ctx))
153+
154+
return imgRef.Context().Name(), img, err
155+
}
148156
}
149157

150158
// checkImageCompliance will perform common checks to ensure the Tekton Bundle is compliant to our spec.

‎pkg/resolution/resolver/bundle/config.go

+94-1
Original file line numberDiff line numberDiff line change
@@ -13,14 +13,107 @@ limitations under the License.
1313

1414
package bundle
1515

16+
import (
17+
"context"
18+
"fmt"
19+
"strconv"
20+
"time"
21+
22+
"github.com/google/go-containerregistry/pkg/v1/remote"
23+
"github.com/tektoncd/pipeline/pkg/resolution/resolver/framework"
24+
)
25+
1626
const (
1727
// ConfigServiceAccount is the configuration field name for controlling
1828
// the Service Account name to use for bundle requests.
1929
ConfigServiceAccount = "default-service-account"
2030
// ConfigKind is the configuration field name for controlling
2131
// what the layer name in the bundle image is.
2232
ConfigKind = "default-kind"
23-
// DefaultTimeoutKey is the configuration field name for controlling
33+
// ConfigTimeoutKey is the configuration field name for controlling
2434
// the maximum duration of a resolution request for a file from registry.
2535
ConfigTimeoutKey = "fetch-timeout"
36+
// ConfigBackoffDuration is the configuration field name for controlling
37+
// the initial duration of a backoff when a bundle resolution fails
38+
ConfigBackoffDuration = "backoff-duration"
39+
DefaultBackoffDuration = 2.0 * time.Second
40+
// ConfigBackoffFactor is the configuration field name for controlling
41+
// the factor by which successive backoffs will increase when a bundle
42+
// resolution fails
43+
ConfigBackoffFactor = "backoff-factor"
44+
DefaultBackoffFactor = 2.0
45+
// ConfigBackoffJitter is the configuration field name for controlling
46+
// the randomness applied to backoff durations when a bundle resolution fails
47+
ConfigBackoffJitter = "backoff-jitter"
48+
DefaultBackoffJitter = 0.1
49+
// ConfigBackoffSteps is the configuration field name for controlling
50+
// the number of attempted backoffs to retry when a bundle resolution fails
51+
ConfigBackoffSteps = "backoff-steps"
52+
DefaultBackoffSteps = 2
53+
// ConfigBackoffCap is the configuration field name for controlling
54+
// the maximum duration to try when backing off
55+
ConfigBackoffCap = "backoff-cap"
56+
DefaultBackoffCap = 10 * time.Second
2657
)
58+
59+
// func {
60+
// conf := framework.GetResolverConfigFromContext(ctx)
61+
// }
62+
63+
// GetBundleResolverBackoff returns a BundleResolverBackoffConfig to
64+
// be passed when resolving remote images. This can be configured with the
65+
// backoff-duration, backoff-factor, backoff-jitter, and backoff-steps
66+
// fields in the bundle-resolver-config ConfigMap.
67+
func GetBundleResolverBackoff(ctx context.Context) (remote.Backoff, error) {
68+
conf := framework.GetResolverConfigFromContext(ctx)
69+
70+
customRetryBackoff := remote.Backoff{
71+
Duration: DefaultBackoffDuration,
72+
Factor: DefaultBackoffFactor,
73+
Jitter: DefaultBackoffJitter,
74+
Steps: DefaultBackoffSteps,
75+
Cap: DefaultBackoffCap,
76+
}
77+
if v, ok := conf[ConfigBackoffDuration]; ok {
78+
var err error
79+
duration, err := time.ParseDuration(v)
80+
if err != nil {
81+
return customRetryBackoff, fmt.Errorf("error parsing backoff duration value %s: %w", v, err)
82+
}
83+
customRetryBackoff.Duration = duration
84+
}
85+
if v, ok := conf[ConfigBackoffFactor]; ok {
86+
var err error
87+
factor, err := strconv.ParseFloat(string(v), 64)
88+
if err != nil {
89+
return customRetryBackoff, fmt.Errorf("error parsing backoff factor value %s: %w", v, err)
90+
}
91+
customRetryBackoff.Factor = factor
92+
}
93+
if v, ok := conf[ConfigBackoffJitter]; ok {
94+
var err error
95+
jitter, err := strconv.ParseFloat(string(v), 64)
96+
if err != nil {
97+
return customRetryBackoff, fmt.Errorf("error parsing backoff jitter value %s: %w", v, err)
98+
}
99+
customRetryBackoff.Jitter = jitter
100+
}
101+
if v, ok := conf[ConfigBackoffSteps]; ok {
102+
var err error
103+
steps, err := strconv.Atoi(string(v))
104+
if err != nil {
105+
return customRetryBackoff, fmt.Errorf("error parsing backoff steps value %s: %w", v, err)
106+
}
107+
customRetryBackoff.Steps = steps
108+
}
109+
if v, ok := conf[ConfigBackoffCap]; ok {
110+
var err error
111+
cap, err := time.ParseDuration(v)
112+
if err != nil {
113+
return customRetryBackoff, fmt.Errorf("error parsing backoff steps value %s: %w", v, err)
114+
}
115+
customRetryBackoff.Cap = cap
116+
}
117+
118+
return customRetryBackoff, nil
119+
}

‎pkg/resolution/resolver/bundle/resolver_test.go

+39
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ import (
2222
"fmt"
2323
"net/http/httptest"
2424
"net/url"
25+
"strconv"
2526
"strings"
2627
"testing"
2728
"time"
@@ -740,3 +741,41 @@ func TestGetResolutionTimeoutCustom(t *testing.T) {
740741
t.Fatalf("expected timeout from config to be returned")
741742
}
742743
}
744+
745+
func TestGetResolutionBackoffCustom(t *testing.T) {
746+
// resolver := bundle.Resolver{}
747+
// defaultTimeout := 30 * time.Minute
748+
configBackoffDuration := 7.0 * time.Second
749+
configBackoffFactor := 7.0
750+
configBackoffJitter := 0.5
751+
configBackoffSteps := 3
752+
configBackoffCap := 20 * time.Second
753+
config := map[string]string{
754+
bundle.ConfigBackoffDuration: configBackoffDuration.String(),
755+
bundle.ConfigBackoffFactor: strconv.FormatFloat(configBackoffFactor, 'f', -1, 64),
756+
bundle.ConfigBackoffJitter: strconv.FormatFloat(configBackoffJitter, 'f', -1, 64),
757+
bundle.ConfigBackoffSteps: strconv.Itoa(configBackoffSteps),
758+
bundle.ConfigBackoffCap: configBackoffCap.String(),
759+
}
760+
ctx := framework.InjectResolverConfigToContext(context.Background(), config)
761+
backoffConfig, err := bundle.GetBundleResolverBackoff(ctx)
762+
// timeout, err := resolver.GetResolutionTimeout(ctx, defaultTimeout, map[string]string{})
763+
if err != nil {
764+
t.Fatalf("couldn't get backoff config: %v", err)
765+
}
766+
if backoffConfig.Duration != configBackoffDuration {
767+
t.Fatalf("expected duration from config to be returned")
768+
}
769+
if backoffConfig.Factor != configBackoffFactor {
770+
t.Fatalf("expected backoff from config to be returned")
771+
}
772+
if backoffConfig.Jitter != configBackoffJitter {
773+
t.Fatalf("expected jitter from config to be returned")
774+
}
775+
if backoffConfig.Steps != configBackoffSteps {
776+
t.Fatalf("expected steps from config to be returned")
777+
}
778+
if backoffConfig.Cap != configBackoffCap {
779+
t.Fatalf("expected steps from config to be returned")
780+
}
781+
}

0 commit comments

Comments
 (0)