Skip to content

Commit 0f94f03

Browse files
arewmarewm
committed
Add configuration for custom bundle resolver backoff
A bundle resolver is susceptible to flakes in network communication. In order to increase reliability, we can add backoff retries to mitigate transient issues. While it may be possible to add a generic retry functionality around resolvers, implementation will likely be simpler if the retry functionality is isolated to the individual resolvers. Future work is needed to add retries to git resolver requests. Partially addresses #8571 Signed-off-by: arewm <[email protected]>
1 parent 00ed658 commit 0f94f03

File tree

4 files changed

+147
-6
lines changed

4 files changed

+147
-6
lines changed

‎docs/bundle-resolver.md

+8-3
Original file line numberDiff line numberDiff line change
@@ -36,9 +36,14 @@ for the name, namespace and defaults that the resolver ships with.
3636

3737
### Options
3838

39-
| Option Name | Description | Example Values |
40-
|---------------------------|--------------------------------------------------------------|-----------------------|
41-
| `default-kind` | The default layer kind in the bundle image. | `task`, `pipeline` |
39+
| Option Name | Description | Example Values |
40+
|----------------------|-------------------------------------------------------------------|-----------------------|
41+
| `backoff-duration` | The initial duration for a backoff. | `500ms`, `2s` |
42+
| `backoff-factor` | The factor by which the sleep duration increases every step. | `2.5`, `4.0` |
43+
| `backoff-jitter` | A random amount of additioan sleep between 0 andduration * jitter.| `0.1`, `0.5` |
44+
| `backoff-steps` | The number of backoffs to attempt. | `3`, `7` |
45+
| `backoff-cap` | The maxumum backoff duration. If reached, remaining steps are zeroed.| `10s`, `20s` |
46+
| `default-kind` | The default layer kind in the bundle image. | `task`, `pipeline` |
4247

4348
## Usage
4449

‎pkg/resolution/resolver/bundle/bundle.go

+10-2
Original file line numberDiff line numberDiff line change
@@ -142,9 +142,17 @@ func retrieveImage(ctx context.Context, keychain authn.Keychain, ref string) (st
142142
if err != nil {
143143
return "", nil, fmt.Errorf("%s is an unparseable image reference: %w", ref, err)
144144
}
145+
customRetryBackoff, err := GetBundleResolverBackoff(ctx)
146+
if err == nil {
147+
img, err := remote.Image(imgRef, remote.WithAuthFromKeychain(keychain), remote.WithContext(ctx),
148+
remote.WithRetryBackoff(customRetryBackoff))
145149

146-
img, err := remote.Image(imgRef, remote.WithAuthFromKeychain(keychain), remote.WithContext(ctx))
147-
return imgRef.Context().Name(), img, err
150+
return imgRef.Context().Name(), img, err
151+
} else {
152+
img, err := remote.Image(imgRef, remote.WithAuthFromKeychain(keychain), remote.WithContext(ctx))
153+
154+
return imgRef.Context().Name(), img, err
155+
}
148156
}
149157

150158
// checkImageCompliance will perform common checks to ensure the Tekton Bundle is compliant to our spec.

‎pkg/resolution/resolver/bundle/config.go

+90-1
Original file line numberDiff line numberDiff line change
@@ -13,14 +13,103 @@ limitations under the License.
1313

1414
package bundle
1515

16+
import (
17+
"context"
18+
"fmt"
19+
"strconv"
20+
"time"
21+
22+
"github.com/google/go-containerregistry/pkg/v1/remote"
23+
"github.com/tektoncd/pipeline/pkg/resolution/resolver/framework"
24+
)
25+
1626
const (
1727
// ConfigServiceAccount is the configuration field name for controlling
1828
// the Service Account name to use for bundle requests.
1929
ConfigServiceAccount = "default-service-account"
2030
// ConfigKind is the configuration field name for controlling
2131
// what the layer name in the bundle image is.
2232
ConfigKind = "default-kind"
23-
// DefaultTimeoutKey is the configuration field name for controlling
33+
// ConfigTimeoutKey is the configuration field name for controlling
2434
// the maximum duration of a resolution request for a file from registry.
2535
ConfigTimeoutKey = "fetch-timeout"
36+
// ConfigBackoffDuration is the configuration field name for controlling
37+
// the initial duration of a backoff when a bundle resolution fails
38+
ConfigBackoffDuration = "backoff-duration"
39+
DefaultBackoffDuration = 2.0 * time.Second
40+
// ConfigBackoffFactor is the configuration field name for controlling
41+
// the factor by which successive backoffs will increase when a bundle
42+
// resolution fails
43+
ConfigBackoffFactor = "backoff-factor"
44+
DefaultBackoffFactor = 2.0
45+
// ConfigBackoffJitter is the configuration field name for controlling
46+
// the randomness applied to backoff durations when a bundle resolution fails
47+
ConfigBackoffJitter = "backoff-jitter"
48+
DefaultBackoffJitter = 0.1
49+
// ConfigBackoffSteps is the configuration field name for controlling
50+
// the number of attempted backoffs to retry when a bundle resolution fails
51+
ConfigBackoffSteps = "backoff-steps"
52+
DefaultBackoffSteps = 2
53+
// ConfigBackoffCap is the configuration field name for controlling
54+
// the maximum duration to try when backing off
55+
ConfigBackoffCap = "backoff-cap"
56+
DefaultBackoffCap = 10 * time.Second
2657
)
58+
59+
// GetBundleResolverBackoff returns a remote.Backoff to
60+
// be passed when resolving remote images. This can be configured with the
61+
// backoff-duration, backoff-factor, backoff-jitter, backoff-steps, and backoff-cap
62+
// fields in the bundle-resolver-config ConfigMap.
63+
func GetBundleResolverBackoff(ctx context.Context) (remote.Backoff, error) {
64+
conf := framework.GetResolverConfigFromContext(ctx)
65+
66+
customRetryBackoff := remote.Backoff{
67+
Duration: DefaultBackoffDuration,
68+
Factor: DefaultBackoffFactor,
69+
Jitter: DefaultBackoffJitter,
70+
Steps: DefaultBackoffSteps,
71+
Cap: DefaultBackoffCap,
72+
}
73+
if v, ok := conf[ConfigBackoffDuration]; ok {
74+
var err error
75+
duration, err := time.ParseDuration(v)
76+
if err != nil {
77+
return customRetryBackoff, fmt.Errorf("error parsing backoff duration value %s: %w", v, err)
78+
}
79+
customRetryBackoff.Duration = duration
80+
}
81+
if v, ok := conf[ConfigBackoffFactor]; ok {
82+
var err error
83+
factor, err := strconv.ParseFloat(v, 64)
84+
if err != nil {
85+
return customRetryBackoff, fmt.Errorf("error parsing backoff factor value %s: %w", v, err)
86+
}
87+
customRetryBackoff.Factor = factor
88+
}
89+
if v, ok := conf[ConfigBackoffJitter]; ok {
90+
var err error
91+
jitter, err := strconv.ParseFloat(v, 64)
92+
if err != nil {
93+
return customRetryBackoff, fmt.Errorf("error parsing backoff jitter value %s: %w", v, err)
94+
}
95+
customRetryBackoff.Jitter = jitter
96+
}
97+
if v, ok := conf[ConfigBackoffSteps]; ok {
98+
var err error
99+
steps, err := strconv.Atoi(v)
100+
if err != nil {
101+
return customRetryBackoff, fmt.Errorf("error parsing backoff steps value %s: %w", v, err)
102+
}
103+
customRetryBackoff.Steps = steps
104+
}
105+
if v, ok := conf[ConfigBackoffCap]; ok {
106+
var err error
107+
cap, err := time.ParseDuration(v)
108+
if err != nil {
109+
return customRetryBackoff, fmt.Errorf("error parsing backoff steps value %s: %w", v, err)
110+
}
111+
customRetryBackoff.Cap = cap
112+
}
113+
114+
return customRetryBackoff, nil
115+
}

‎pkg/resolution/resolver/bundle/resolver_test.go

+39
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ import (
2222
"fmt"
2323
"net/http/httptest"
2424
"net/url"
25+
"strconv"
2526
"strings"
2627
"testing"
2728
"time"
@@ -740,3 +741,41 @@ func TestGetResolutionTimeoutCustom(t *testing.T) {
740741
t.Fatalf("expected timeout from config to be returned")
741742
}
742743
}
744+
745+
func TestGetResolutionBackoffCustom(t *testing.T) {
746+
// resolver := bundle.Resolver{}
747+
// defaultTimeout := 30 * time.Minute
748+
configBackoffDuration := 7.0 * time.Second
749+
configBackoffFactor := 7.0
750+
configBackoffJitter := 0.5
751+
configBackoffSteps := 3
752+
configBackoffCap := 20 * time.Second
753+
config := map[string]string{
754+
bundle.ConfigBackoffDuration: configBackoffDuration.String(),
755+
bundle.ConfigBackoffFactor: strconv.FormatFloat(configBackoffFactor, 'f', -1, 64),
756+
bundle.ConfigBackoffJitter: strconv.FormatFloat(configBackoffJitter, 'f', -1, 64),
757+
bundle.ConfigBackoffSteps: strconv.Itoa(configBackoffSteps),
758+
bundle.ConfigBackoffCap: configBackoffCap.String(),
759+
}
760+
ctx := framework.InjectResolverConfigToContext(context.Background(), config)
761+
backoffConfig, err := bundle.GetBundleResolverBackoff(ctx)
762+
// timeout, err := resolver.GetResolutionTimeout(ctx, defaultTimeout, map[string]string{})
763+
if err != nil {
764+
t.Fatalf("couldn't get backoff config: %v", err)
765+
}
766+
if backoffConfig.Duration != configBackoffDuration {
767+
t.Fatalf("expected duration from config to be returned")
768+
}
769+
if backoffConfig.Factor != configBackoffFactor {
770+
t.Fatalf("expected backoff from config to be returned")
771+
}
772+
if backoffConfig.Jitter != configBackoffJitter {
773+
t.Fatalf("expected jitter from config to be returned")
774+
}
775+
if backoffConfig.Steps != configBackoffSteps {
776+
t.Fatalf("expected steps from config to be returned")
777+
}
778+
if backoffConfig.Cap != configBackoffCap {
779+
t.Fatalf("expected steps from config to be returned")
780+
}
781+
}

0 commit comments

Comments
 (0)