Skip to content
Merged
Show file tree
Hide file tree
Changes from 16 commits
Commits
Show all changes
24 commits
Select commit Hold shift + click to select a range
0f8f14d
Merge branch 'main' of github.com:entireio/cli into fix/checkpoints-v…
pfleidi May 7, 2026
b575cfc
Merge branch 'main' of github.com:entireio/cli into fix/checkpoints-v…
pfleidi May 7, 2026
a5aac69
Fix v2 full generation rotation push
pfleidi May 8, 2026
600cbd4
Detect stale v2 full generation archives
pfleidi May 8, 2026
a599cde
Handle multiple remote v2 rotations
pfleidi May 8, 2026
43fa9dc
Cover post-rotation v2 current pushes
pfleidi May 8, 2026
bc82aaf
Cover repeated local v2 rotations
pfleidi May 8, 2026
aa9ac82
Push all v2 archived generation refs
pfleidi May 8, 2026
65d9a46
Use SplitSeq for v2 remote ref scans
pfleidi May 8, 2026
d654296
Publish v2 archives from pending records
pfleidi May 8, 2026
090dc30
Rollback unqueued migrated v2 archives
pfleidi May 8, 2026
6c48c47
Push pending v2 archives without active refs
pfleidi May 8, 2026
ed4eddd
Guard empty v2 ref pushes
pfleidi May 8, 2026
3dccb8d
Lock pending v2 full generation markers
pfleidi May 8, 2026
3a95e50
Avoid redundant v2 generation lookups
pfleidi May 8, 2026
004ab54
Simplify v2 pending publication push path
pfleidi May 8, 2026
9c73ee3
Merge branch 'main' of github.com:entireio/cli into fix/checkpoints-v…
pfleidi May 8, 2026
aed8210
Harden v2 rotation recovery fetches
pfleidi May 8, 2026
cdcdec6
Merge remote-tracking branch 'origin/main' into fix/checkpoints-v2-ro…
pfleidi May 8, 2026
3ea8f1c
Refine v2 pending publication push handling
pfleidi May 9, 2026
25e04b8
Validate pending v2 archive publication refs
pfleidi May 9, 2026
d014d6d
Clean up fetched v2 archive temp refs
pfleidi May 9, 2026
48c2096
Reuse v2 push repository state
pfleidi May 9, 2026
c2446ed
Require pending v2 rotation marker before reset
pfleidi May 9, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 16 additions & 0 deletions cmd/entire/cli/checkpoint/v2_generation.go
Original file line number Diff line number Diff line change
Expand Up @@ -570,6 +570,22 @@ func (s *V2GitStore) RotateCurrentGenerationIfNeeded(ctx context.Context, maxChe
return archiveRefName, false, nil
}

if err := s.AppendPendingFullGenerationPublication(ctx, PendingV2FullGenerationPublication{
ArchiveRefName: archiveRefName.String(),
ArchiveCommitHash: archiveCommitHash.String(),
PreviousFullCurrentHash: currentRef.Hash().String(),
ResetFullCurrentRootHash: orphanCommitHash.String(),
QueuedAt: time.Now().UTC(),
}); err != nil {
logging.Warn(ctx, "rotation: failed to record pending full rotation; continuing with rotated refs",
"error", err,
slog.String("archive_ref", string(archiveRefName)),
slog.String("previous_full_current_hash", currentRef.Hash().String()),
slog.String("archive_commit_hash", archiveCommitHash.String()),
slog.String("reset_full_current_root_hash", orphanCommitHash.String()),
)
}
Comment thread
pfleidi marked this conversation as resolved.
Outdated

logging.Info(ctx, "generation rotation complete",
slog.Int("archived_generation", archiveNumber),
slog.String("archive_ref", string(archiveRefName)),
Expand Down
63 changes: 63 additions & 0 deletions cmd/entire/cli/checkpoint/v2_generation_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,8 @@ import (
"context"
"encoding/json"
"fmt"
"os"
"path/filepath"
"testing"
"time"

Expand Down Expand Up @@ -447,6 +449,67 @@ func TestRotateGeneration_ArchivesCurrentAndCreatesNewOrphan(t *testing.T) {
assert.Empty(t, freshTree.Entries, "fresh tree should be empty (no generation.json)")
}

func TestRotateGeneration_SucceedsWhenPendingMarkerCannotBeRecorded(t *testing.T) {
t.Parallel()
repo := initTestRepo(t)
store := NewV2GitStore(repo, "origin")
ctx := context.Background()

populateFullCurrent(t, store, 3, 0)

worktree, err := repo.Worktree()
require.NoError(t, err)
blockingPath := filepath.Join(worktree.Filesystem.Root(), ".git", pendingV2FullGenerationPublicationDirName)
require.NoError(t, os.WriteFile(blockingPath, []byte("not a directory"), 0o600))

refName, rotated, err := store.RotateCurrentGenerationIfNeeded(ctx, 3)
require.NoError(t, err)
require.True(t, rotated)
require.Equal(t, ArchivedGenerationRefName(1), refName)

_, currentTreeHash, err := store.GetRefState(plumbing.ReferenceName(paths.V2FullCurrentRefName))
require.NoError(t, err)
currentCount, err := store.CountCheckpointsInTree(currentTreeHash)
require.NoError(t, err)
assert.Equal(t, 0, currentCount)

_, archiveTreeHash, err := store.GetRefState(refName)
require.NoError(t, err)
archiveCount, err := store.CountCheckpointsInTree(archiveTreeHash)
require.NoError(t, err)
assert.Equal(t, 3, archiveCount)
}

func TestRemovePendingFullGenerationPublications_PreservesLaterQueuedEntries(t *testing.T) {
t.Parallel()
repo := initTestRepo(t)
store := NewV2GitStore(repo, "origin")
ctx := context.Background()

first := PendingV2FullGenerationPublication{
ArchiveRefName: paths.V2FullRefPrefix + "0000000000001",
ArchiveCommitHash: "1111111111111111111111111111111111111111",
QueuedAt: time.Date(2026, 3, 19, 1, 2, 3, 0, time.UTC),
}
later := PendingV2FullGenerationPublication{
ArchiveRefName: paths.V2FullRefPrefix + "0000000000002",
ArchiveCommitHash: "2222222222222222222222222222222222222222",
QueuedAt: time.Date(2026, 3, 19, 4, 5, 6, 0, time.UTC),
}

require.NoError(t, store.AppendPendingFullGenerationPublication(ctx, first))
snapshot, err := store.ReadPendingFullGenerationPublications(ctx)
require.NoError(t, err)
require.Equal(t, []PendingV2FullGenerationPublication{first}, snapshot)

require.NoError(t, store.AppendPendingFullGenerationPublication(ctx, later))
require.NoError(t, store.RemovePendingFullGenerationPublications(ctx, snapshot))

remaining, err := store.ReadPendingFullGenerationPublications(ctx)
require.NoError(t, err)
assert.Equal(t, []PendingV2FullGenerationPublication{later}, remaining)
}

func TestResetFullCurrentRefIfUnchangedRejectsConcurrentChange(t *testing.T) {
t.Parallel()
repo := initTestRepo(t)
Expand Down
248 changes: 248 additions & 0 deletions cmd/entire/cli/checkpoint/v2_pending_rotation.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,248 @@
package checkpoint

import (
"context"
"encoding/json"
"errors"
"fmt"
"os"
"os/exec"
"path/filepath"
"strings"
"time"

"github.com/entireio/cli/cmd/entire/cli/jsonutil"
"github.com/entireio/cli/cmd/entire/cli/lockfile"
"github.com/go-git/go-git/v6"
)

const (
pendingV2FullGenerationPublicationVersion = 1
pendingV2FullGenerationPublicationDirName = "entire-v2-rotations"
pendingV2FullGenerationPublicationFile = "pending.json"
pendingV2FullGenerationPublicationLock = "pending.lock"
pendingV2FullGenerationPublicationLockTTL = 5 * time.Second
)

type PendingV2FullGenerationPublication struct {
ArchiveRefName string `json:"archive_ref_name"`
ArchiveCommitHash string `json:"archive_commit_hash"`
// PreviousFullCurrentHash and ResetFullCurrentRootHash are set when the
// archive publication came from a local /full/current rotation.
PreviousFullCurrentHash string `json:"previous_full_current_hash,omitempty"`
ResetFullCurrentRootHash string `json:"reset_full_current_root_hash,omitempty"`
QueuedAt time.Time `json:"queued_at"`
}

type pendingV2FullGenerationPublicationState struct {
Version int `json:"version"`
Publications []PendingV2FullGenerationPublication `json:"publications"`
}

func (s *V2GitStore) AppendPendingFullGenerationPublication(ctx context.Context, publication PendingV2FullGenerationPublication) error {
return s.AppendPendingFullGenerationPublications(ctx, []PendingV2FullGenerationPublication{publication})
}

func (s *V2GitStore) AppendPendingFullGenerationPublications(ctx context.Context, publications []PendingV2FullGenerationPublication) error {
if len(publications) == 0 {
return nil
}
return s.withPendingFullGenerationPublicationLock(ctx, func() error {
state, err := s.readPendingFullGenerationPublicationState(ctx)
if err != nil {
return err
}
state.Version = pendingV2FullGenerationPublicationVersion
state.Publications = append(state.Publications, publications...)
return s.writePendingFullGenerationPublicationState(ctx, state)
})
}

func (s *V2GitStore) ReadPendingFullGenerationPublications(ctx context.Context) ([]PendingV2FullGenerationPublication, error) {
state, err := s.readPendingFullGenerationPublicationState(ctx)
if err != nil {
return nil, err
}
return state.Publications, nil
}

func (s *V2GitStore) RemovePendingFullGenerationPublications(ctx context.Context, publications []PendingV2FullGenerationPublication) error {
if len(publications) == 0 {
return nil
}
return s.withPendingFullGenerationPublicationLock(ctx, func() error {
state, err := s.readPendingFullGenerationPublicationState(ctx)
if err != nil {
return err
}
previousCount := len(state.Publications)
state.Publications = removePendingFullGenerationPublications(state.Publications, publications)
if len(state.Publications) == previousCount {
return nil
}
if len(state.Publications) == 0 {
return s.removePendingFullGenerationPublicationFile(ctx)
}
state.Version = pendingV2FullGenerationPublicationVersion
return s.writePendingFullGenerationPublicationState(ctx, state)
})
}

func removePendingFullGenerationPublications(current, remove []PendingV2FullGenerationPublication) []PendingV2FullGenerationPublication {
removeCounts := make(map[PendingV2FullGenerationPublication]int, len(remove))
for _, publication := range remove {
removeCounts[comparablePendingFullGenerationPublication(publication)]++
}

remaining := make([]PendingV2FullGenerationPublication, 0, len(current))
for _, publication := range current {
key := comparablePendingFullGenerationPublication(publication)
if removeCounts[key] > 0 {
removeCounts[key]--
continue
}
remaining = append(remaining, publication)
}
return remaining
}

func comparablePendingFullGenerationPublication(publication PendingV2FullGenerationPublication) PendingV2FullGenerationPublication {
publication.QueuedAt = publication.QueuedAt.Round(0).UTC()
return publication
}

func (s *V2GitStore) removePendingFullGenerationPublicationFile(ctx context.Context) error {
path, err := s.pendingFullGenerationPublicationFilePath(ctx)
if err != nil {
return err
}
if err := os.Remove(path); err != nil && !os.IsNotExist(err) {
return fmt.Errorf("remove pending v2 full generation publications: %w", err)
}
return nil
}

func (s *V2GitStore) withPendingFullGenerationPublicationLock(ctx context.Context, fn func() error) (err error) {
lockPath, err := s.pendingFullGenerationPublicationLockPath(ctx)
if err != nil {
return err
}
if err := os.MkdirAll(filepath.Dir(lockPath), 0o750); err != nil {
return fmt.Errorf("create pending v2 full generation publication lock dir: %w", err)
}
if err := lockfile.WithTimeout(ctx, lockPath, pendingV2FullGenerationPublicationLockTTL, fn); err != nil {
return fmt.Errorf("pending v2 full generation publication lock: %w", err)
}
return nil
}

func (s *V2GitStore) readPendingFullGenerationPublicationState(ctx context.Context) (pendingV2FullGenerationPublicationState, error) {
path, err := s.pendingFullGenerationPublicationFilePath(ctx)
if err != nil {
return pendingV2FullGenerationPublicationState{}, err
}
data, err := os.ReadFile(path) //nolint:gosec // path is under git common dir
if os.IsNotExist(err) {
return pendingV2FullGenerationPublicationState{Version: pendingV2FullGenerationPublicationVersion}, nil
}
if err != nil {
return pendingV2FullGenerationPublicationState{}, fmt.Errorf("read pending v2 full generation publications: %w", err)
}

var state pendingV2FullGenerationPublicationState
if err := json.Unmarshal(data, &state); err != nil {
return pendingV2FullGenerationPublicationState{}, fmt.Errorf("parse pending v2 full generation publications: %w", err)
}
if state.Version != pendingV2FullGenerationPublicationVersion {
return pendingV2FullGenerationPublicationState{}, fmt.Errorf("unsupported pending v2 full generation publication version %d", state.Version)
}
return state, nil
}

func (s *V2GitStore) writePendingFullGenerationPublicationState(ctx context.Context, state pendingV2FullGenerationPublicationState) error {
path, err := s.pendingFullGenerationPublicationFilePath(ctx)
if err != nil {
return err
}
if err := os.MkdirAll(filepath.Dir(path), 0o750); err != nil {
return fmt.Errorf("create pending v2 full generation publication dir: %w", err)
}

data, err := jsonutil.MarshalIndentWithNewline(state, "", " ")
if err != nil {
return fmt.Errorf("marshal pending v2 full generation publications: %w", err)
}

tmpFile, err := os.CreateTemp(filepath.Dir(path), pendingV2FullGenerationPublicationFile+".*.tmp")
if err != nil {
return fmt.Errorf("create pending v2 full generation publication temp file: %w", err)
}
tmpName := tmpFile.Name()
removeTmp := true
defer func() {
if removeTmp {
_ = os.Remove(tmpName)
}
}()

if _, err := tmpFile.Write(data); err != nil {
_ = tmpFile.Close()
return fmt.Errorf("write pending v2 full generation publications: %w", err)
}
if err := tmpFile.Close(); err != nil {
return fmt.Errorf("close pending v2 full generation publications: %w", err)
}
if err := os.Rename(tmpName, path); err != nil {
return fmt.Errorf("replace pending v2 full generation publications: %w", err)
}
removeTmp = false
return nil
}

func (s *V2GitStore) pendingFullGenerationPublicationFilePath(ctx context.Context) (string, error) {
commonDir, err := s.gitCommonDir(ctx)
if err != nil {
return "", err
}
return filepath.Join(commonDir, pendingV2FullGenerationPublicationDirName, pendingV2FullGenerationPublicationFile), nil
}

func (s *V2GitStore) pendingFullGenerationPublicationLockPath(ctx context.Context) (string, error) {
commonDir, err := s.gitCommonDir(ctx)
if err != nil {
return "", err
}
return filepath.Join(commonDir, pendingV2FullGenerationPublicationDirName, pendingV2FullGenerationPublicationLock), nil
}

func (s *V2GitStore) gitCommonDir(ctx context.Context) (string, error) {
s.commonDirOnce.Do(func() {
s.commonDir, s.commonDirErr = resolveGitCommonDir(ctx, s.repo)
})
return s.commonDir, s.commonDirErr
}

func resolveGitCommonDir(ctx context.Context, repo *git.Repository) (string, error) {
worktree, err := repo.Worktree()
if err != nil {
return "", fmt.Errorf("open worktree for pending v2 full generation publications: %w", err)
}
root := worktree.Filesystem.Root()
if root == "" {
return "", errors.New("resolve worktree root for pending v2 full generation publications")
}

cmd := exec.CommandContext(ctx, "git", "-C", root, "rev-parse", "--git-common-dir")
output, err := cmd.Output()
if err != nil {
return "", fmt.Errorf("resolve git common dir for pending v2 full generation publications: %w", err)
}
commonDir := strings.TrimSpace(string(output))
if commonDir == "" {
return "", errors.New("resolve git common dir for pending v2 full generation publications: empty output")
}
if !filepath.IsAbs(commonDir) {
commonDir = filepath.Join(root, commonDir)
}
return filepath.Clean(commonDir), nil
}
2 changes: 1 addition & 1 deletion cmd/entire/cli/checkpoint/v2_read.go
Original file line number Diff line number Diff line change
Expand Up @@ -421,7 +421,7 @@ func (s *V2GitStore) fetchRemoteFullRefs(ctx context.Context) error {
}

var refSpecs []string
for _, line := range strings.Split(strings.TrimSpace(string(output)), "\n") {
for line := range strings.SplitSeq(strings.TrimSpace(string(output)), "\n") {
if line == "" {
continue
}
Expand Down
5 changes: 5 additions & 0 deletions cmd/entire/cli/checkpoint/v2_store.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package checkpoint
import (
"context"
"fmt"
"sync"

"github.com/go-git/go-git/v6"
"github.com/go-git/go-git/v6/plumbing"
Expand Down Expand Up @@ -36,6 +37,10 @@ type V2GitStore struct {
// cat-file fallback covers partial-clone-filtered blobs that go-git's
// storer can't see).
blobFetcher BlobFetchFunc

commonDirOnce sync.Once
commonDir string
commonDirErr error
}

// maxCheckpoints returns the effective rotation threshold.
Expand Down
Loading
Loading