diff --git a/docs/notes/delivery-stabilization.md b/docs/notes/delivery-stabilization.md index bba21fd9..345125ef 100644 --- a/docs/notes/delivery-stabilization.md +++ b/docs/notes/delivery-stabilization.md @@ -45,3 +45,19 @@ change. Recovery checklist: retain `connection-reconnect` and unfinished feature branches; merge only complete committed trees that pass mapped checks; do not infer delivery from a dirty worktree or an old deployed-backend capture. + +## Catch-up merge when main moved (2026-10-31) + +Two delivery runs finishing close together used to fail the second with +"delivery item commit cannot fast-forward canonical main" (run #211), although a +retry of its Land stage succeeded. Now, while holding the delivery lock, the gate +merges canonical main's HEAD into the item workspace (`git merge --no-edit`, an +ordinary merge commit), runs the declared checks on the merged commit in the +disposable candidate worktree, and only then fast-forwards main. A conflicting +merge is aborted, the workspace is verified back at its prior commit and clean, +and the step fails with "main moved and the item no longer merges cleanly: +". If checks fail on the merged tree, main is untouched; the merge commit +stays on the item branch and the diagnostic names the catch-up. A successful +catch-up is written to the server log (a success-step diagnostic renders as a +stop in the TUI). Nothing is forced onto main and history is never rewritten; +the clean-checkout, access and ancestry checks are unchanged. diff --git a/internal/author/skill/references/delivery.md b/internal/author/skill/references/delivery.md index 56d77568..bc27855a 100644 --- a/internal/author/skill/references/delivery.md +++ b/internal/author/skill/references/delivery.md @@ -10,7 +10,8 @@ Inspect actual source and diffs, run relevant tests and record exact results. A model summary, committed branch or successful merge is not proof of runtime behavior. Missing or partial diff capture must be reported as incomplete. With delivery=true, the stage returns a clean candidate and the server checks and -lands it; direct recovery can complete a reviewed merge without another pipeline. +lands it, first merging a moved main and re-checking the merged tree; only a real +conflict fails. Direct recovery can complete a reviewed merge without another pipeline. Keep artifacts under the supplied run/stage artifact directory. Mark unverified builds as WIP. Verify any download URL serves the real bytes before reporting it. diff --git a/internal/author/skill/references/format.md b/internal/author/skill/references/format.md index 4977b317..537e8b21 100644 --- a/internal/author/skill/references/format.md +++ b/internal/author/skill/references/format.md @@ -238,6 +238,9 @@ published on its own; keep those copies in sync with this list. The committer prepares the complete clean candidate and resolves integration conflicts. When `delivery=true`, the server executes the declared checks and lands after the stage returns success; do not run the post-landing ancestry check beforehand. +If canonical `main` moved meanwhile, the server merges it into the item (a normal +merge commit), re-runs the checks on the merged tree and lands that; a conflicting +catch-up is aborted with the workspace unchanged and the conflicting files named. Without a delivery gate, the original workflow or manager completes the merge and verifies ancestry. No separate merger pipeline is required (`references/queue.md`). diff --git a/internal/server/delivery_gate.go b/internal/server/delivery_gate.go index df1dc7c5..44c5b6b8 100644 --- a/internal/server/delivery_gate.go +++ b/internal/server/delivery_gate.go @@ -26,68 +26,127 @@ const deliveryCleanupTimeout = 10 * time.Second // before fast-forwarding canonical main. Settlement deliberately performs only // non-mutating repository checks after that mutation: a declared check must // never be the first signal that the landed tree is known-red. -func verifyDelivery(ctx context.Context, workspace string, access stage.LandingAccess, checks []string) error { +// +// When canonical main moved after the item's last integration (for example a +// concurrent delivery landed first), the gate catches up while still holding +// the delivery lock: it merges main into the item workspace with an ordinary +// merge commit, runs the declared checks on that merged commit, and only then +// fast-forwards main. A conflicting merge is aborted, leaving the workspace +// exactly as it was. The returned note is non-empty when a catch-up merge +// happened, so the caller can record it. +func verifyDelivery(ctx context.Context, workspace string, access stage.LandingAccess, checks []string) (string, error) { lock, err := acquireDeliveryLock(ctx, access.LockPath) if err != nil { - return err + return "", err } defer lock.Close() current, err := landingAccess(workspace) if err != nil { - return fmt.Errorf("delivery access changed: %w", err) + return "", fmt.Errorf("delivery access changed: %w", err) } if current != access { - return fmt.Errorf("delivery access changed during stage") + return "", fmt.Errorf("delivery access changed during stage") } for _, directory := range []string{workspace, access.MainWorktree} { status, err := gitOutput(directory, "status", "--porcelain=v1", "--untracked-files=all") if err != nil { - return fmt.Errorf("delivery checkout status: %w", err) + return "", fmt.Errorf("delivery checkout status: %w", err) } if status != "" { - return fmt.Errorf("delivery checkout is not clean: %s", strings.Split(status, "\n")[0]) + return "", fmt.Errorf("delivery checkout is not clean: %s", strings.Split(status, "\n")[0]) } } itemHead, err := gitOutput(workspace, "rev-parse", "HEAD") if err != nil { - return fmt.Errorf("delivery item commit: %w", err) - } - if err := runDeliveryCandidate(ctx, workspace, itemHead, checks); err != nil { - return err + return "", fmt.Errorf("delivery item commit: %w", err) } mainHead, err := gitOutput(access.MainWorktree, "rev-parse", "HEAD") if err != nil { - return fmt.Errorf("delivery main commit: %w", err) + return "", fmt.Errorf("delivery main commit: %w", err) + } + note := "" + if exec.CommandContext(ctx, "git", "-C", workspace, "merge-base", "--is-ancestor", mainHead, itemHead).Run() != nil { + merged, err := catchUpDeliveryItem(ctx, workspace, itemHead, mainHead) + if err != nil { + return "", err + } + note = fmt.Sprintf("delivery gate: canonical main moved to %s; merged it into item %s as %s and re-ran checks on the merged tree", shortCommit(mainHead), shortCommit(itemHead), shortCommit(merged)) + itemHead = merged + } + if err := runDeliveryCandidate(ctx, workspace, itemHead, checks); err != nil { + if note != "" { + return note, fmt.Errorf("%s; %w", note, err) + } + return "", err } if err := exec.CommandContext(ctx, "git", "-C", workspace, "merge-base", "--is-ancestor", mainHead, itemHead).Run(); err != nil { - return fmt.Errorf("delivery item commit cannot fast-forward canonical main") + return note, fmt.Errorf("delivery item commit cannot fast-forward canonical main") } if output, err := exec.CommandContext(ctx, "git", "-C", access.MainWorktree, "merge", "--ff-only", itemHead).CombinedOutput(); err != nil { - return fmt.Errorf("delivery fast-forward canonical main: %w: %s", err, strings.TrimSpace(string(output))) + return note, fmt.Errorf("delivery fast-forward canonical main: %w: %s", err, strings.TrimSpace(string(output))) } for _, directory := range []string{workspace, access.MainWorktree} { status, err := gitOutput(directory, "status", "--porcelain=v1", "--untracked-files=all") if err != nil { - return fmt.Errorf("delivery settlement status: %w", err) + return note, fmt.Errorf("delivery settlement status: %w", err) } if status != "" { - return fmt.Errorf("delivery checkout is not clean after checks: %s", strings.Split(status, "\n")[0]) + return note, fmt.Errorf("delivery checkout is not clean after checks: %s", strings.Split(status, "\n")[0]) } } // Re-derive at the settlement boundary. A success means exactly the checked // item commit is now reachable from the same canonical main worktree. current, err = landingAccess(workspace) if err != nil || current != access { - return fmt.Errorf("delivery access changed during settlement") + return note, fmt.Errorf("delivery access changed during settlement") } mainHead, err = gitOutput(access.MainWorktree, "rev-parse", "HEAD") if err != nil { - return fmt.Errorf("delivery main commit: %w", err) + return note, fmt.Errorf("delivery main commit: %w", err) } if err := exec.CommandContext(ctx, "git", "-C", workspace, "merge-base", "--is-ancestor", itemHead, mainHead).Run(); err != nil { - return fmt.Errorf("delivery item commit is not an ancestor of canonical main") + return note, fmt.Errorf("delivery item commit is not an ancestor of canonical main") } - return nil + return note, nil +} + +// catchUpDeliveryItem merges canonical main into the clean item workspace with a +// normal merge commit and returns the new item HEAD. It never rewrites history: +// a failed merge is aborted and the workspace is verified to be back at +// itemHead with a clean checkout. +func catchUpDeliveryItem(ctx context.Context, workspace, itemHead, mainHead string) (string, error) { + output, mergeErr := boundedCommandOutput(ctx, workspace, "merge", "--no-edit", mainHead) + if mergeErr == nil { + merged, err := gitOutput(workspace, "rev-parse", "HEAD") + if err != nil { + return "", fmt.Errorf("delivery catch-up merge commit: %w", err) + } + return merged, nil + } + conflicts, _ := gitOutput(workspace, "diff", "--name-only", "--diff-filter=U") + abortCtx, cancel := context.WithTimeout(context.Background(), deliveryCleanupTimeout) + defer cancel() + if _, err := gitOutput(workspace, "rev-parse", "-q", "--verify", "MERGE_HEAD"); err == nil { + if abortOutput, err := exec.CommandContext(abortCtx, "git", "-C", workspace, "merge", "--abort").CombinedOutput(); err != nil { + return "", fmt.Errorf("main moved and the catch-up merge could not be aborted: %w: %s", err, strings.TrimSpace(string(abortOutput))) + } + } + head, headErr := gitOutput(workspace, "rev-parse", "HEAD") + status, statusErr := gitOutput(workspace, "status", "--porcelain=v1", "--untracked-files=all") + if headErr != nil || statusErr != nil || head != itemHead || status != "" { + return "", fmt.Errorf("main moved and the catch-up merge did not restore the item workspace to %s", shortCommit(itemHead)) + } + if conflicts != "" { + return "", fmt.Errorf("main moved and the item no longer merges cleanly: %s", strings.Join(strings.Fields(conflicts), ", ")) + } + return "", fmt.Errorf("main moved and the catch-up merge of %s failed: %w: %s", shortCommit(mainHead), mergeErr, strings.TrimSpace(string(output))) +} + +func shortCommit(commit string) string { + if len(commit) > 12 { + return commit[:12] + } + return commit } func acquireDeliveryLock(ctx context.Context, path string) (*os.File, error) { diff --git a/internal/server/landing_access_test.go b/internal/server/landing_access_test.go index 9d2d7ae8..af56b8db 100644 --- a/internal/server/landing_access_test.go +++ b/internal/server/landing_access_test.go @@ -83,19 +83,19 @@ func TestVerifyDeliveryRequiresCleanLandedMainAndPassingChecks(t *testing.T) { if err != nil { t.Fatal(err) } - if err := verifyDelivery(context.Background(), item, access, []string{"test -f delivered"}); err != nil { + if _, err := verifyDelivery(context.Background(), item, access, []string{"test -f delivered"}); err != nil { t.Fatalf("complete landed delivery rejected: %v", err) } if err := os.WriteFile(filepath.Join(item, "missing-helper"), []byte("dirty\n"), 0600); err != nil { t.Fatal(err) } - if err := verifyDelivery(context.Background(), item, access, []string{"true"}); err == nil { + if _, err := verifyDelivery(context.Background(), item, access, []string{"true"}); err == nil { t.Fatal("dirty task-owned helper was accepted") } if err := os.Remove(filepath.Join(item, "missing-helper")); err != nil { t.Fatal(err) } - if err := verifyDelivery(context.Background(), item, access, []string{"false"}); err == nil { + if _, err := verifyDelivery(context.Background(), item, access, []string{"false"}); err == nil { t.Fatal("failed required check was accepted") } if err := os.WriteFile(filepath.Join(item, "unlanded"), []byte("no\n"), 0600); err != nil { @@ -103,13 +103,123 @@ func TestVerifyDeliveryRequiresCleanLandedMainAndPassingChecks(t *testing.T) { } runGit(t, item, "add", "unlanded") runGit(t, item, "commit", "-m", "unlanded") - if err := verifyDelivery(context.Background(), item, access, []string{"test -f unlanded"}); err != nil { + committed := runGitOutput(t, item, "rev-parse", "HEAD") + note, err := verifyDelivery(context.Background(), item, access, []string{"test -f unlanded"}) + if err != nil { t.Fatalf("complete unlanded item was not landed after candidate verification: %v", err) } + if note != "" { + t.Fatalf("fast-forwardable item reported a catch-up merge: %s", note) + } mainHead := runGitOutput(t, main, "rev-parse", "HEAD") itemHead := runGitOutput(t, item, "rev-parse", "HEAD") - if mainHead != itemHead { - t.Fatal("verified item was not fast-forwarded into canonical main") + if mainHead != itemHead || itemHead != committed { + t.Fatal("verified item was not fast-forwarded into canonical main unchanged") + } +} + +// newDeliveryRepo creates a canonical main worktree with one base commit and a +// linked item worktree on branch item. +func newDeliveryRepo(t *testing.T) (main, item string) { + t.Helper() + root := t.TempDir() + main = filepath.Join(root, "main") + runGit(t, root, "init", "-b", "main", main) + runGit(t, main, "config", "user.email", "test@example.invalid") + runGit(t, main, "config", "user.name", "test") + commitFile(t, main, "README", "base\n") + item = filepath.Join(root, "item") + runGit(t, main, "worktree", "add", "-b", "item", item) + return main, item +} + +func commitFile(t *testing.T, directory, name, content string) { + t.Helper() + if err := os.WriteFile(filepath.Join(directory, name), []byte(content), 0600); err != nil { + t.Fatal(err) + } + runGit(t, directory, "add", name) + runGit(t, directory, "commit", "-m", "change "+name) +} + +func TestVerifyDeliveryCatchesUpWithUnrelatedMainChange(t *testing.T) { + main, item := newDeliveryRepo(t) + commitFile(t, item, "work", "item\n") + itemBefore := runGitOutput(t, item, "rev-parse", "HEAD") + commitFile(t, main, "other", "concurrent\n") + mainBefore := runGitOutput(t, main, "rev-parse", "HEAD") + access, err := landingAccess(item) + if err != nil { + t.Fatal(err) + } + // The check only passes on the merged tree, proving it ran after the merge. + note, err := verifyDelivery(context.Background(), item, access, []string{"test -f work && test -f other"}) + if err != nil { + t.Fatalf("cleanly mergeable item was rejected: %v", err) + } + if !strings.Contains(note, "merged") { + t.Fatalf("catch-up merge was not reported: %q", note) + } + itemHead := runGitOutput(t, item, "rev-parse", "HEAD") + if runGitOutput(t, main, "rev-parse", "HEAD") != itemHead { + t.Fatal("canonical main was not fast-forwarded to the merge") + } + parents := strings.Fields(runGitOutput(t, item, "rev-list", "--parents", "-n", "1", "HEAD")) + if len(parents) != 3 || parents[1] != itemBefore || parents[2] != mainBefore { + t.Fatalf("expected a merge of item %s and main %s, got %v", itemBefore, mainBefore, parents) + } + if status := runGitOutput(t, item, "status", "--porcelain"); status != "" { + t.Fatalf("item workspace is not clean: %s", status) + } +} + +func TestVerifyDeliveryAbortsConflictingCatchUp(t *testing.T) { + main, item := newDeliveryRepo(t) + commitFile(t, item, "README", "item\n") + itemBefore := runGitOutput(t, item, "rev-parse", "HEAD") + commitFile(t, main, "README", "main\n") + mainBefore := runGitOutput(t, main, "rev-parse", "HEAD") + access, err := landingAccess(item) + if err != nil { + t.Fatal(err) + } + _, err = verifyDelivery(context.Background(), item, access, []string{"true"}) + if err == nil || !strings.Contains(err.Error(), "main moved and the item no longer merges cleanly: README") { + t.Fatalf("conflicting catch-up did not fail clearly: %v", err) + } + if runGitOutput(t, item, "rev-parse", "HEAD") != itemBefore { + t.Fatal("conflicting catch-up changed the item commit") + } + if status := runGitOutput(t, item, "status", "--porcelain", "--untracked-files=all"); status != "" { + t.Fatalf("conflicting catch-up left the workspace dirty: %s", status) + } + if exec.Command("git", "-C", item, "rev-parse", "-q", "--verify", "MERGE_HEAD").Run() == nil { + t.Fatal("conflicting catch-up left a merge in progress") + } + if runGitOutput(t, main, "rev-parse", "HEAD") != mainBefore { + t.Fatal("conflicting catch-up changed canonical main") + } +} + +func TestVerifyDeliveryFailingChecksAfterCatchUpLeaveMainUnchanged(t *testing.T) { + main, item := newDeliveryRepo(t) + commitFile(t, item, "work", "item\n") + commitFile(t, main, "other", "concurrent\n") + mainBefore := runGitOutput(t, main, "rev-parse", "HEAD") + access, err := landingAccess(item) + if err != nil { + t.Fatal(err) + } + // Passes on the item tree alone, fails on the merged tree. + _, err = verifyDelivery(context.Background(), item, access, []string{"test -f work", "test ! -f other"}) + if err == nil || !strings.Contains(err.Error(), "merged") || !strings.Contains(err.Error(), "test ! -f other") { + t.Fatalf("failing merged-tree check did not fail with catch-up context: %v", err) + } + if runGitOutput(t, main, "rev-parse", "HEAD") != mainBefore { + t.Fatal("failing merged-tree check changed canonical main") + } + if status := runGitOutput(t, main, "status", "--porcelain", "--untracked-files=all"); status != "" { + t.Fatalf("canonical main checkout changed: %s", status) } } @@ -136,7 +246,7 @@ func TestVerifyDeliveryFailingCandidateLeavesMainUnchanged(t *testing.T) { t.Fatal(err) } before := runGitOutput(t, main, "rev-parse", "HEAD") - if err := verifyDelivery(context.Background(), item, access, []string{"test -f candidate", "false"}); err == nil { + if _, err := verifyDelivery(context.Background(), item, access, []string{"test -f candidate", "false"}); err == nil { t.Fatal("failing candidate check landed main") } if after := runGitOutput(t, main, "rev-parse", "HEAD"); after != before { diff --git a/internal/server/runner.go b/internal/server/runner.go index 0aa69732..eaade908 100644 --- a/internal/server/runner.go +++ b/internal/server/runner.go @@ -1917,8 +1917,14 @@ func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, na err = checksErr } else { gateCtx, gateCancel := context.WithTimeout(stageCtx, modelConfig.Timeout) - err = verifyDelivery(gateCtx, workspace, landing, checks) + var catchUp string + catchUp, err = verifyDelivery(gateCtx, workspace, landing, checks) gateCancel() + // A failed gate already carries the note in its diagnostic. A step + // diagnostic on success renders as a stop, so success is logged. + if catchUp != "" { + log.Printf("run %d stage %s: %s", id, key, catchUp) + } } if err != nil { step.Status = "failed"