Merge branch 'worktree-agent-a3db4d35a1277b036' into integrate/delivery-catchup
This commit is contained in:
@@ -45,3 +45,19 @@ change.
|
||||
Recovery checklist: retain `connection-reconnect` and unfinished feature
|
||||
branches; merge only complete committed trees that pass mapped checks; do not
|
||||
infer delivery from a dirty worktree or an old deployed-backend capture.
|
||||
|
||||
## Catch-up merge when main moved (2026-10-31)
|
||||
|
||||
Two delivery runs finishing close together used to fail the second with
|
||||
"delivery item commit cannot fast-forward canonical main" (run #211), although a
|
||||
retry of its Land stage succeeded. Now, while holding the delivery lock, the gate
|
||||
merges canonical main's HEAD into the item workspace (`git merge --no-edit`, an
|
||||
ordinary merge commit), runs the declared checks on the merged commit in the
|
||||
disposable candidate worktree, and only then fast-forwards main. A conflicting
|
||||
merge is aborted, the workspace is verified back at its prior commit and clean,
|
||||
and the step fails with "main moved and the item no longer merges cleanly:
|
||||
<files>". If checks fail on the merged tree, main is untouched; the merge commit
|
||||
stays on the item branch and the diagnostic names the catch-up. A successful
|
||||
catch-up is written to the server log (a success-step diagnostic renders as a
|
||||
stop in the TUI). Nothing is forced onto main and history is never rewritten;
|
||||
the clean-checkout, access and ancestry checks are unchanged.
|
||||
|
||||
@@ -10,7 +10,8 @@ Inspect actual source and diffs, run relevant tests and record exact results.
|
||||
A model summary, committed branch or successful merge is not proof of runtime
|
||||
behavior. Missing or partial diff capture must be reported as incomplete.
|
||||
With delivery=true, the stage returns a clean candidate and the server checks and
|
||||
lands it; direct recovery can complete a reviewed merge without another pipeline.
|
||||
lands it, first merging a moved main and re-checking the merged tree; only a real
|
||||
conflict fails. Direct recovery can complete a reviewed merge without another pipeline.
|
||||
|
||||
Keep artifacts under the supplied run/stage artifact directory. Mark unverified
|
||||
builds as WIP. Verify any download URL serves the real bytes before reporting it.
|
||||
|
||||
@@ -238,6 +238,9 @@ published on its own; keep those copies in sync with this list.
|
||||
The committer prepares the complete clean candidate and resolves integration conflicts.
|
||||
When `delivery=true`, the server executes the declared checks and lands after the
|
||||
stage returns success; do not run the post-landing ancestry check beforehand.
|
||||
If canonical `main` moved meanwhile, the server merges it into the item (a normal
|
||||
merge commit), re-runs the checks on the merged tree and lands that; a conflicting
|
||||
catch-up is aborted with the workspace unchanged and the conflicting files named.
|
||||
Without a delivery gate, the original workflow or manager completes the merge
|
||||
and verifies ancestry. No separate merger pipeline is required (`references/queue.md`).
|
||||
|
||||
|
||||
@@ -26,68 +26,127 @@ const deliveryCleanupTimeout = 10 * time.Second
|
||||
// before fast-forwarding canonical main. Settlement deliberately performs only
|
||||
// non-mutating repository checks after that mutation: a declared check must
|
||||
// never be the first signal that the landed tree is known-red.
|
||||
func verifyDelivery(ctx context.Context, workspace string, access stage.LandingAccess, checks []string) error {
|
||||
//
|
||||
// When canonical main moved after the item's last integration (for example a
|
||||
// concurrent delivery landed first), the gate catches up while still holding
|
||||
// the delivery lock: it merges main into the item workspace with an ordinary
|
||||
// merge commit, runs the declared checks on that merged commit, and only then
|
||||
// fast-forwards main. A conflicting merge is aborted, leaving the workspace
|
||||
// exactly as it was. The returned note is non-empty when a catch-up merge
|
||||
// happened, so the caller can record it.
|
||||
func verifyDelivery(ctx context.Context, workspace string, access stage.LandingAccess, checks []string) (string, error) {
|
||||
lock, err := acquireDeliveryLock(ctx, access.LockPath)
|
||||
if err != nil {
|
||||
return err
|
||||
return "", err
|
||||
}
|
||||
defer lock.Close()
|
||||
current, err := landingAccess(workspace)
|
||||
if err != nil {
|
||||
return fmt.Errorf("delivery access changed: %w", err)
|
||||
return "", fmt.Errorf("delivery access changed: %w", err)
|
||||
}
|
||||
if current != access {
|
||||
return fmt.Errorf("delivery access changed during stage")
|
||||
return "", fmt.Errorf("delivery access changed during stage")
|
||||
}
|
||||
for _, directory := range []string{workspace, access.MainWorktree} {
|
||||
status, err := gitOutput(directory, "status", "--porcelain=v1", "--untracked-files=all")
|
||||
if err != nil {
|
||||
return fmt.Errorf("delivery checkout status: %w", err)
|
||||
return "", fmt.Errorf("delivery checkout status: %w", err)
|
||||
}
|
||||
if status != "" {
|
||||
return fmt.Errorf("delivery checkout is not clean: %s", strings.Split(status, "\n")[0])
|
||||
return "", fmt.Errorf("delivery checkout is not clean: %s", strings.Split(status, "\n")[0])
|
||||
}
|
||||
}
|
||||
itemHead, err := gitOutput(workspace, "rev-parse", "HEAD")
|
||||
if err != nil {
|
||||
return fmt.Errorf("delivery item commit: %w", err)
|
||||
}
|
||||
if err := runDeliveryCandidate(ctx, workspace, itemHead, checks); err != nil {
|
||||
return err
|
||||
return "", fmt.Errorf("delivery item commit: %w", err)
|
||||
}
|
||||
mainHead, err := gitOutput(access.MainWorktree, "rev-parse", "HEAD")
|
||||
if err != nil {
|
||||
return fmt.Errorf("delivery main commit: %w", err)
|
||||
return "", fmt.Errorf("delivery main commit: %w", err)
|
||||
}
|
||||
note := ""
|
||||
if exec.CommandContext(ctx, "git", "-C", workspace, "merge-base", "--is-ancestor", mainHead, itemHead).Run() != nil {
|
||||
merged, err := catchUpDeliveryItem(ctx, workspace, itemHead, mainHead)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
note = fmt.Sprintf("delivery gate: canonical main moved to %s; merged it into item %s as %s and re-ran checks on the merged tree", shortCommit(mainHead), shortCommit(itemHead), shortCommit(merged))
|
||||
itemHead = merged
|
||||
}
|
||||
if err := runDeliveryCandidate(ctx, workspace, itemHead, checks); err != nil {
|
||||
if note != "" {
|
||||
return note, fmt.Errorf("%s; %w", note, err)
|
||||
}
|
||||
return "", err
|
||||
}
|
||||
if err := exec.CommandContext(ctx, "git", "-C", workspace, "merge-base", "--is-ancestor", mainHead, itemHead).Run(); err != nil {
|
||||
return fmt.Errorf("delivery item commit cannot fast-forward canonical main")
|
||||
return note, fmt.Errorf("delivery item commit cannot fast-forward canonical main")
|
||||
}
|
||||
if output, err := exec.CommandContext(ctx, "git", "-C", access.MainWorktree, "merge", "--ff-only", itemHead).CombinedOutput(); err != nil {
|
||||
return fmt.Errorf("delivery fast-forward canonical main: %w: %s", err, strings.TrimSpace(string(output)))
|
||||
return note, fmt.Errorf("delivery fast-forward canonical main: %w: %s", err, strings.TrimSpace(string(output)))
|
||||
}
|
||||
for _, directory := range []string{workspace, access.MainWorktree} {
|
||||
status, err := gitOutput(directory, "status", "--porcelain=v1", "--untracked-files=all")
|
||||
if err != nil {
|
||||
return fmt.Errorf("delivery settlement status: %w", err)
|
||||
return note, fmt.Errorf("delivery settlement status: %w", err)
|
||||
}
|
||||
if status != "" {
|
||||
return fmt.Errorf("delivery checkout is not clean after checks: %s", strings.Split(status, "\n")[0])
|
||||
return note, fmt.Errorf("delivery checkout is not clean after checks: %s", strings.Split(status, "\n")[0])
|
||||
}
|
||||
}
|
||||
// Re-derive at the settlement boundary. A success means exactly the checked
|
||||
// item commit is now reachable from the same canonical main worktree.
|
||||
current, err = landingAccess(workspace)
|
||||
if err != nil || current != access {
|
||||
return fmt.Errorf("delivery access changed during settlement")
|
||||
return note, fmt.Errorf("delivery access changed during settlement")
|
||||
}
|
||||
mainHead, err = gitOutput(access.MainWorktree, "rev-parse", "HEAD")
|
||||
if err != nil {
|
||||
return fmt.Errorf("delivery main commit: %w", err)
|
||||
return note, fmt.Errorf("delivery main commit: %w", err)
|
||||
}
|
||||
if err := exec.CommandContext(ctx, "git", "-C", workspace, "merge-base", "--is-ancestor", itemHead, mainHead).Run(); err != nil {
|
||||
return fmt.Errorf("delivery item commit is not an ancestor of canonical main")
|
||||
return note, fmt.Errorf("delivery item commit is not an ancestor of canonical main")
|
||||
}
|
||||
return nil
|
||||
return note, nil
|
||||
}
|
||||
|
||||
// catchUpDeliveryItem merges canonical main into the clean item workspace with a
|
||||
// normal merge commit and returns the new item HEAD. It never rewrites history:
|
||||
// a failed merge is aborted and the workspace is verified to be back at
|
||||
// itemHead with a clean checkout.
|
||||
func catchUpDeliveryItem(ctx context.Context, workspace, itemHead, mainHead string) (string, error) {
|
||||
output, mergeErr := boundedCommandOutput(ctx, workspace, "merge", "--no-edit", mainHead)
|
||||
if mergeErr == nil {
|
||||
merged, err := gitOutput(workspace, "rev-parse", "HEAD")
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("delivery catch-up merge commit: %w", err)
|
||||
}
|
||||
return merged, nil
|
||||
}
|
||||
conflicts, _ := gitOutput(workspace, "diff", "--name-only", "--diff-filter=U")
|
||||
abortCtx, cancel := context.WithTimeout(context.Background(), deliveryCleanupTimeout)
|
||||
defer cancel()
|
||||
if _, err := gitOutput(workspace, "rev-parse", "-q", "--verify", "MERGE_HEAD"); err == nil {
|
||||
if abortOutput, err := exec.CommandContext(abortCtx, "git", "-C", workspace, "merge", "--abort").CombinedOutput(); err != nil {
|
||||
return "", fmt.Errorf("main moved and the catch-up merge could not be aborted: %w: %s", err, strings.TrimSpace(string(abortOutput)))
|
||||
}
|
||||
}
|
||||
head, headErr := gitOutput(workspace, "rev-parse", "HEAD")
|
||||
status, statusErr := gitOutput(workspace, "status", "--porcelain=v1", "--untracked-files=all")
|
||||
if headErr != nil || statusErr != nil || head != itemHead || status != "" {
|
||||
return "", fmt.Errorf("main moved and the catch-up merge did not restore the item workspace to %s", shortCommit(itemHead))
|
||||
}
|
||||
if conflicts != "" {
|
||||
return "", fmt.Errorf("main moved and the item no longer merges cleanly: %s", strings.Join(strings.Fields(conflicts), ", "))
|
||||
}
|
||||
return "", fmt.Errorf("main moved and the catch-up merge of %s failed: %w: %s", shortCommit(mainHead), mergeErr, strings.TrimSpace(string(output)))
|
||||
}
|
||||
|
||||
func shortCommit(commit string) string {
|
||||
if len(commit) > 12 {
|
||||
return commit[:12]
|
||||
}
|
||||
return commit
|
||||
}
|
||||
|
||||
func acquireDeliveryLock(ctx context.Context, path string) (*os.File, error) {
|
||||
|
||||
@@ -83,19 +83,19 @@ func TestVerifyDeliveryRequiresCleanLandedMainAndPassingChecks(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := verifyDelivery(context.Background(), item, access, []string{"test -f delivered"}); err != nil {
|
||||
if _, err := verifyDelivery(context.Background(), item, access, []string{"test -f delivered"}); err != nil {
|
||||
t.Fatalf("complete landed delivery rejected: %v", err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(item, "missing-helper"), []byte("dirty\n"), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := verifyDelivery(context.Background(), item, access, []string{"true"}); err == nil {
|
||||
if _, err := verifyDelivery(context.Background(), item, access, []string{"true"}); err == nil {
|
||||
t.Fatal("dirty task-owned helper was accepted")
|
||||
}
|
||||
if err := os.Remove(filepath.Join(item, "missing-helper")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := verifyDelivery(context.Background(), item, access, []string{"false"}); err == nil {
|
||||
if _, err := verifyDelivery(context.Background(), item, access, []string{"false"}); err == nil {
|
||||
t.Fatal("failed required check was accepted")
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(item, "unlanded"), []byte("no\n"), 0600); err != nil {
|
||||
@@ -103,13 +103,123 @@ func TestVerifyDeliveryRequiresCleanLandedMainAndPassingChecks(t *testing.T) {
|
||||
}
|
||||
runGit(t, item, "add", "unlanded")
|
||||
runGit(t, item, "commit", "-m", "unlanded")
|
||||
if err := verifyDelivery(context.Background(), item, access, []string{"test -f unlanded"}); err != nil {
|
||||
committed := runGitOutput(t, item, "rev-parse", "HEAD")
|
||||
note, err := verifyDelivery(context.Background(), item, access, []string{"test -f unlanded"})
|
||||
if err != nil {
|
||||
t.Fatalf("complete unlanded item was not landed after candidate verification: %v", err)
|
||||
}
|
||||
if note != "" {
|
||||
t.Fatalf("fast-forwardable item reported a catch-up merge: %s", note)
|
||||
}
|
||||
mainHead := runGitOutput(t, main, "rev-parse", "HEAD")
|
||||
itemHead := runGitOutput(t, item, "rev-parse", "HEAD")
|
||||
if mainHead != itemHead {
|
||||
t.Fatal("verified item was not fast-forwarded into canonical main")
|
||||
if mainHead != itemHead || itemHead != committed {
|
||||
t.Fatal("verified item was not fast-forwarded into canonical main unchanged")
|
||||
}
|
||||
}
|
||||
|
||||
// newDeliveryRepo creates a canonical main worktree with one base commit and a
|
||||
// linked item worktree on branch item.
|
||||
func newDeliveryRepo(t *testing.T) (main, item string) {
|
||||
t.Helper()
|
||||
root := t.TempDir()
|
||||
main = filepath.Join(root, "main")
|
||||
runGit(t, root, "init", "-b", "main", main)
|
||||
runGit(t, main, "config", "user.email", "test@example.invalid")
|
||||
runGit(t, main, "config", "user.name", "test")
|
||||
commitFile(t, main, "README", "base\n")
|
||||
item = filepath.Join(root, "item")
|
||||
runGit(t, main, "worktree", "add", "-b", "item", item)
|
||||
return main, item
|
||||
}
|
||||
|
||||
func commitFile(t *testing.T, directory, name, content string) {
|
||||
t.Helper()
|
||||
if err := os.WriteFile(filepath.Join(directory, name), []byte(content), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
runGit(t, directory, "add", name)
|
||||
runGit(t, directory, "commit", "-m", "change "+name)
|
||||
}
|
||||
|
||||
func TestVerifyDeliveryCatchesUpWithUnrelatedMainChange(t *testing.T) {
|
||||
main, item := newDeliveryRepo(t)
|
||||
commitFile(t, item, "work", "item\n")
|
||||
itemBefore := runGitOutput(t, item, "rev-parse", "HEAD")
|
||||
commitFile(t, main, "other", "concurrent\n")
|
||||
mainBefore := runGitOutput(t, main, "rev-parse", "HEAD")
|
||||
access, err := landingAccess(item)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// The check only passes on the merged tree, proving it ran after the merge.
|
||||
note, err := verifyDelivery(context.Background(), item, access, []string{"test -f work && test -f other"})
|
||||
if err != nil {
|
||||
t.Fatalf("cleanly mergeable item was rejected: %v", err)
|
||||
}
|
||||
if !strings.Contains(note, "merged") {
|
||||
t.Fatalf("catch-up merge was not reported: %q", note)
|
||||
}
|
||||
itemHead := runGitOutput(t, item, "rev-parse", "HEAD")
|
||||
if runGitOutput(t, main, "rev-parse", "HEAD") != itemHead {
|
||||
t.Fatal("canonical main was not fast-forwarded to the merge")
|
||||
}
|
||||
parents := strings.Fields(runGitOutput(t, item, "rev-list", "--parents", "-n", "1", "HEAD"))
|
||||
if len(parents) != 3 || parents[1] != itemBefore || parents[2] != mainBefore {
|
||||
t.Fatalf("expected a merge of item %s and main %s, got %v", itemBefore, mainBefore, parents)
|
||||
}
|
||||
if status := runGitOutput(t, item, "status", "--porcelain"); status != "" {
|
||||
t.Fatalf("item workspace is not clean: %s", status)
|
||||
}
|
||||
}
|
||||
|
||||
func TestVerifyDeliveryAbortsConflictingCatchUp(t *testing.T) {
|
||||
main, item := newDeliveryRepo(t)
|
||||
commitFile(t, item, "README", "item\n")
|
||||
itemBefore := runGitOutput(t, item, "rev-parse", "HEAD")
|
||||
commitFile(t, main, "README", "main\n")
|
||||
mainBefore := runGitOutput(t, main, "rev-parse", "HEAD")
|
||||
access, err := landingAccess(item)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, err = verifyDelivery(context.Background(), item, access, []string{"true"})
|
||||
if err == nil || !strings.Contains(err.Error(), "main moved and the item no longer merges cleanly: README") {
|
||||
t.Fatalf("conflicting catch-up did not fail clearly: %v", err)
|
||||
}
|
||||
if runGitOutput(t, item, "rev-parse", "HEAD") != itemBefore {
|
||||
t.Fatal("conflicting catch-up changed the item commit")
|
||||
}
|
||||
if status := runGitOutput(t, item, "status", "--porcelain", "--untracked-files=all"); status != "" {
|
||||
t.Fatalf("conflicting catch-up left the workspace dirty: %s", status)
|
||||
}
|
||||
if exec.Command("git", "-C", item, "rev-parse", "-q", "--verify", "MERGE_HEAD").Run() == nil {
|
||||
t.Fatal("conflicting catch-up left a merge in progress")
|
||||
}
|
||||
if runGitOutput(t, main, "rev-parse", "HEAD") != mainBefore {
|
||||
t.Fatal("conflicting catch-up changed canonical main")
|
||||
}
|
||||
}
|
||||
|
||||
func TestVerifyDeliveryFailingChecksAfterCatchUpLeaveMainUnchanged(t *testing.T) {
|
||||
main, item := newDeliveryRepo(t)
|
||||
commitFile(t, item, "work", "item\n")
|
||||
commitFile(t, main, "other", "concurrent\n")
|
||||
mainBefore := runGitOutput(t, main, "rev-parse", "HEAD")
|
||||
access, err := landingAccess(item)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Passes on the item tree alone, fails on the merged tree.
|
||||
_, err = verifyDelivery(context.Background(), item, access, []string{"test -f work", "test ! -f other"})
|
||||
if err == nil || !strings.Contains(err.Error(), "merged") || !strings.Contains(err.Error(), "test ! -f other") {
|
||||
t.Fatalf("failing merged-tree check did not fail with catch-up context: %v", err)
|
||||
}
|
||||
if runGitOutput(t, main, "rev-parse", "HEAD") != mainBefore {
|
||||
t.Fatal("failing merged-tree check changed canonical main")
|
||||
}
|
||||
if status := runGitOutput(t, main, "status", "--porcelain", "--untracked-files=all"); status != "" {
|
||||
t.Fatalf("canonical main checkout changed: %s", status)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -136,7 +246,7 @@ func TestVerifyDeliveryFailingCandidateLeavesMainUnchanged(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
before := runGitOutput(t, main, "rev-parse", "HEAD")
|
||||
if err := verifyDelivery(context.Background(), item, access, []string{"test -f candidate", "false"}); err == nil {
|
||||
if _, err := verifyDelivery(context.Background(), item, access, []string{"test -f candidate", "false"}); err == nil {
|
||||
t.Fatal("failing candidate check landed main")
|
||||
}
|
||||
if after := runGitOutput(t, main, "rev-parse", "HEAD"); after != before {
|
||||
|
||||
@@ -1917,8 +1917,14 @@ func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, na
|
||||
err = checksErr
|
||||
} else {
|
||||
gateCtx, gateCancel := context.WithTimeout(stageCtx, modelConfig.Timeout)
|
||||
err = verifyDelivery(gateCtx, workspace, landing, checks)
|
||||
var catchUp string
|
||||
catchUp, err = verifyDelivery(gateCtx, workspace, landing, checks)
|
||||
gateCancel()
|
||||
// A failed gate already carries the note in its diagnostic. A step
|
||||
// diagnostic on success renders as a stop, so success is logged.
|
||||
if catchUp != "" {
|
||||
log.Printf("run %d stage %s: %s", id, key, catchUp)
|
||||
}
|
||||
}
|
||||
if err != nil {
|
||||
step.Status = "failed"
|
||||
|
||||
Reference in New Issue
Block a user