Skip to content

Commit ad3525f

Browse files
mitchbnemitchbne
authored
Merge pull request #56 from buildkite/add-template-key-to-cache
Introduce 'template' key to cache
2 parents f780a88 + 69f1906 commit ad3525f

9 files changed

Lines changed: 555 additions & 93 deletions

File tree

internal/commands/cache.go renamed to internal/cache/cache.go

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
package commands
1+
package cache
22

33
import (
44
"fmt"
@@ -7,6 +7,8 @@ import (
77
)
88

99
type Cache struct {
10+
// Template of the cache entry.
11+
Template string
1012
// The registry to use which defaults to "~".
1113
Registry string
1214
// ID of the cache entry to save.

internal/commands/cache_test.go renamed to internal/cache/cache_test.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
package commands
1+
package cache
22

33
import (
44
"testing"

‎internal/commands/commands.go‎

Lines changed: 2 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,12 @@
11
package commands
22

33
import (
4-
"fmt"
54
"math"
6-
"strings"
75

86
"github.com/buildkite/zstash/internal/api"
97
"github.com/buildkite/zstash/internal/archive"
8+
"github.com/buildkite/zstash/internal/cache"
109
"github.com/buildkite/zstash/internal/console"
11-
"github.com/buildkite/zstash/internal/key"
12-
"github.com/rs/zerolog/log"
1310
)
1411

1512
type CommonFlags struct {
@@ -25,45 +22,10 @@ type Globals struct {
2522
Version string
2623
Client api.Client
2724
Printer *console.Printer
28-
Caches []Cache
25+
Caches []cache.Cache
2926
Common CommonFlags
3027
}
3128

32-
// checkPath validates the provided path and returns a list of paths.
33-
func checkPath(paths []string) ([]string, error) {
34-
if len(paths) == 0 {
35-
return nil, fmt.Errorf("no paths provided")
36-
}
37-
38-
return paths, nil
39-
}
40-
41-
// restoreKeys generates a list of restore keys from the provided ID and restore key list.
42-
func restoreKeys(id string, restoreKeyTemplates []string) ([]string, error) {
43-
restoreKeys := make([]string, len(restoreKeyTemplates))
44-
45-
log.Debug().Str("id", id).Strs("restore_keys", restoreKeyTemplates).Msg("templating restore keys")
46-
47-
for n, restoreKeyTemplate := range restoreKeyTemplates {
48-
49-
// trim quotes and whitespace
50-
restoreKeyTemplate = strings.Trim(restoreKeyTemplate, "\"' \t")
51-
52-
log.Debug().Str("restore_key_template", restoreKeyTemplate).Msg("templating restore key")
53-
54-
restoreKey, err := key.Template(id, restoreKeyTemplate)
55-
if err != nil {
56-
return nil, fmt.Errorf("failed to template restore key: %w", err)
57-
}
58-
59-
log.Debug().Str("restore_key", restoreKey).Msg("templated restore key")
60-
61-
restoreKeys[n] = restoreKey
62-
}
63-
64-
return restoreKeys, nil
65-
}
66-
6729
// calculate the compression ratio
6830
func compressionRatio(archiveInfo *archive.ArchiveInfo) float64 {
6931
if archiveInfo.Size == 0 {

‎internal/commands/restore.go‎

Lines changed: 14 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,8 @@ import (
1111

1212
"github.com/buildkite/zstash/internal/api"
1313
"github.com/buildkite/zstash/internal/archive"
14-
"github.com/buildkite/zstash/internal/key"
14+
"github.com/buildkite/zstash/internal/cache"
15+
"github.com/buildkite/zstash/internal/configuration"
1516
"github.com/buildkite/zstash/internal/store"
1617
"github.com/buildkite/zstash/internal/trace"
1718
"github.com/charmbracelet/lipgloss"
@@ -37,7 +38,13 @@ func (cmd *RestoreCmd) Run(ctx context.Context, globals *Globals) error {
3738

3839
log.Info().Str("version", globals.Version).Msg("Running RestoreCmd")
3940

40-
for _, cache := range globals.Caches {
41+
// Augment `cli.Caches` with template values.
42+
caches, err := configuration.ExpandCacheConfiguration(globals.Caches)
43+
if err != nil {
44+
return fmt.Errorf("failed to load cache configuration: %w", err)
45+
}
46+
47+
for _, cache := range caches {
4148
if len(cmd.Ids) > 0 && !slices.Contains(cmd.Ids, cache.ID) {
4249
log.Debug().Str("id", cache.ID).Msg("Skipping cache restore for ID")
4350
continue
@@ -55,7 +62,7 @@ func (cmd *RestoreCmd) Run(ctx context.Context, globals *Globals) error {
5562
return nil
5663
}
5764

58-
func (cmd *RestoreCmd) restoreCache(ctx context.Context, cache Cache, globals *Globals) error {
65+
func (cmd *RestoreCmd) restoreCache(ctx context.Context, cache cache.Cache, globals *Globals) error {
5966
ctx, span := trace.Start(ctx, "restoreCache")
6067
defer span.End()
6168

@@ -169,26 +176,11 @@ type extractionResult struct {
169176
paths []string
170177
}
171178

172-
func (cmd *RestoreCmd) validateAndPrepare(ctx context.Context, span oteltrace.Span, cache Cache) (*restoreData, error) {
173-
paths, err := checkPath(cache.Paths)
174-
if err != nil {
175-
return nil, trace.NewError(span, "failed to check paths: %w", err)
176-
}
177-
178-
cacheKey, err := key.Template(cache.ID, cache.Key)
179-
if err != nil {
180-
return nil, trace.NewError(span, "failed to template key: %w", err)
181-
}
182-
183-
fallbackCacheKeys, err := restoreKeys(cache.ID, cache.FallbackKeys)
184-
if err != nil {
185-
return nil, trace.NewError(span, "failed to restore keys: %w", err)
186-
}
187-
179+
func (cmd *RestoreCmd) validateAndPrepare(ctx context.Context, span oteltrace.Span, cache cache.Cache) (*restoreData, error) {
188180
return &restoreData{
189-
paths: paths,
190-
cacheKey: cacheKey,
191-
fallbackCacheKeys: fallbackCacheKeys,
181+
paths: cache.Paths,
182+
cacheKey: cache.Key,
183+
fallbackCacheKeys: cache.FallbackKeys,
192184
}, nil
193185
}
194186

‎internal/commands/save.go‎

Lines changed: 20 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -10,8 +10,9 @@ import (
1010

1111
"github.com/buildkite/zstash/internal/api"
1212
"github.com/buildkite/zstash/internal/archive"
13+
"github.com/buildkite/zstash/internal/cache"
14+
"github.com/buildkite/zstash/internal/configuration"
1315
"github.com/buildkite/zstash/internal/console"
14-
"github.com/buildkite/zstash/internal/key"
1516
"github.com/buildkite/zstash/internal/store"
1617
"github.com/buildkite/zstash/internal/trace"
1718
"github.com/charmbracelet/lipgloss"
@@ -31,7 +32,13 @@ func (cmd *SaveCmd) Run(ctx context.Context, globals *Globals) error {
3132

3233
log.Info().Str("version", globals.Version).Msg("Running SaveCmd")
3334

34-
for _, cache := range globals.Caches {
35+
// Augment `cli.Caches` with template values.
36+
caches, err := configuration.ExpandCacheConfiguration(globals.Caches)
37+
if err != nil {
38+
return fmt.Errorf("failed to load cache configuration: %w", err)
39+
}
40+
41+
for _, cache := range caches {
3542
if len(cmd.Ids) > 0 && !slices.Contains(cmd.Ids, cache.ID) {
3643
log.Debug().Str("id", cache.ID).Msg("Skipping cache save for ID")
3744
continue
@@ -49,7 +56,7 @@ func (cmd *SaveCmd) Run(ctx context.Context, globals *Globals) error {
4956
return nil
5057
}
5158

52-
func (cmd *SaveCmd) saveCache(ctx context.Context, cache Cache, globals *Globals) error {
59+
func (cmd *SaveCmd) saveCache(ctx context.Context, cache cache.Cache, globals *Globals) error {
5360
ctx, span := trace.Start(ctx, "saveCache")
5461
defer span.End()
5562

@@ -110,34 +117,34 @@ func (cmd *SaveCmd) saveCache(ctx context.Context, cache Cache, globals *Globals
110117

111118
globals.Printer.Info("📦", "Creating new cache entry for key: %s store: %s", data.cacheKey, cacheRegistryResp.Store)
112119

113-
// Phase 3: Build archive
120+
// Phase 2: Build archive
114121
archiveResult, err := cmd.buildArchive(ctx, data, globals.Printer)
115122
if err != nil {
116123
return err
117124
}
118125

119-
// Phase 4: Register cache entry
126+
// Phase 3: Register cache entry
120127
registrationResult, err := cmd.registerCacheEntry(ctx, data, archiveResult, globals.Client, globals.Common, cache.Registry, cacheRegistryResp.Store)
121128
if err != nil {
122129
return err
123130
}
124131

125132
globals.Printer.Info("🚀", "Registering cache entry with upload ID: %s", registrationResult.uploadID)
126133

127-
// Phase 5: Upload archive
134+
// Phase 4: Upload archive
128135
uploadResult, err := cmd.uploadArchive(ctx, data.cacheKey, cacheRegistryResp.Store, archiveResult, globals.Printer, globals.Common)
129136
if err != nil {
130137
return err
131138
}
132139

133-
// Phase 6: Commit cache
140+
// Phase 5: Commit cache
134141
if err := cmd.commitCache(ctx, globals.Client, cache.Registry, registrationResult.uploadID); err != nil {
135142
return err
136143
}
137144

138145
globals.Printer.Success("🎉", "Cache committed successfully")
139146

140-
// Phase 7: Print summary
147+
// Phase 6: Print summary
141148
t := table.New().
142149
Border(lipgloss.NormalBorder()).
143150
Row("Key", data.cacheKey).
@@ -175,31 +182,16 @@ type uploadResult struct {
175182
transferInfo *store.TransferInfo
176183
}
177184

178-
func (cmd *SaveCmd) validateAndPrepare(cache Cache) (*saveData, error) {
179-
paths, err := checkPath(cache.Paths)
180-
if err != nil {
181-
return nil, fmt.Errorf("failed to check paths: %w", err)
182-
}
183-
184-
cacheKey, err := key.Template(cache.ID, cache.Key)
185-
if err != nil {
186-
return nil, fmt.Errorf("failed to template key: %w", err)
187-
}
188-
189-
fallbackKeys, err := restoreKeys(cache.ID, cache.FallbackKeys)
190-
if err != nil {
191-
return nil, fmt.Errorf("failed to restore keys: %w", err)
192-
}
193-
194-
_, err = checkPathsExist(paths)
185+
func (cmd *SaveCmd) validateAndPrepare(cache cache.Cache) (*saveData, error) {
186+
_, err := checkPathsExist(cache.Paths)
195187
if err != nil {
196188
return nil, fmt.Errorf("failed to check paths exist: %w", err)
197189
}
198190

199191
return &saveData{
200-
paths: paths,
201-
cacheKey: cacheKey,
202-
fallbackKeys: fallbackKeys,
192+
paths: cache.Paths,
193+
cacheKey: cache.Key,
194+
fallbackKeys: cache.FallbackKeys,
203195
}, nil
204196
}
205197

0 commit comments

Comments
 (0)