package server import ( "bytes" "context" "errors" "fmt" "io" "os" "os/exec" "path/filepath" "strings" "syscall" "time" "git.a-wan.net/alan/pipelines/internal/stage" "git.a-wan.net/alan/pipelines/internal/workenv" ) // verifyDelivery proves that the stage landed a complete, committed tree before // its success can settle the run. It re-derives the capability after the stage: // a stage result and its input cannot select either the landing path or checks. const maxDeliveryCheckOutput = 64 << 10 const deliveryCleanupTimeout = 10 * time.Second // verifyDelivery verifies the committed item tree in a disposable worktree // before fast-forwarding canonical main. Settlement deliberately performs only // non-mutating repository checks after that mutation: a declared check must // never be the first signal that the landed tree is known-red. // // When canonical main moved after the item's last integration (for example a // concurrent delivery landed first), the gate catches up while still holding // the delivery lock: it merges main into the item workspace with an ordinary // merge commit, runs the declared checks on that merged commit, and only then // fast-forwards main. A conflicting merge is aborted, leaving the workspace // exactly as it was. The returned note is non-empty when a catch-up merge // happened, so the caller can record it. // // Work the land stage left uncommitted in the item workspace is committed // first (commitMessage): the workspace is the run's own tree, so everything in // it is the run's work, and a partial file list must never drop part of it. // The operator's main checkout need not be clean: the fast-forward is Git's // own `merge --ff-only`, which refuses without touching anything when a local // change or untracked file would be overwritten. func verifyDelivery(ctx context.Context, workspace string, access stage.LandingAccess, checks []string, commitMessage string) (string, error) { lock, err := acquireDeliveryLock(ctx, access.LockPath) if err != nil { return "", err } defer lock.Close() current, err := landingAccess(workspace) if err != nil { return "", fmt.Errorf("delivery access changed: %w", err) } if current != access { return "", fmt.Errorf("delivery access changed during stage") } if err := commitDeliveryLeftovers(ctx, workspace, commitMessage); err != nil { return "", err } status, err := gitOutput(workspace, "status", "--porcelain=v1", "--untracked-files=all") if err != nil { return "", fmt.Errorf("delivery checkout status: %w", err) } if status != "" { return "", fmt.Errorf("delivery checkout is not clean: %s", strings.Split(status, "\n")[0]) } itemHead, err := gitOutput(workspace, "rev-parse", "HEAD") if err != nil { return "", fmt.Errorf("delivery item commit: %w", err) } mainHead, err := gitOutput(access.MainWorktree, "rev-parse", "HEAD") if err != nil { return "", fmt.Errorf("delivery main commit: %w", err) } note := "" if exec.CommandContext(ctx, "git", "-C", workspace, "merge-base", "--is-ancestor", mainHead, itemHead).Run() != nil { merged, err := catchUpDeliveryItem(ctx, workspace, itemHead, mainHead) if err != nil { return "", err } note = fmt.Sprintf("delivery gate: canonical main moved to %s; merged it into item %s as %s and re-ran checks on the merged tree", shortCommit(mainHead), shortCommit(itemHead), shortCommit(merged)) itemHead = merged } if err := runDeliveryCandidate(ctx, workspace, itemHead, checks); err != nil { if note != "" { return note, fmt.Errorf("%s; %w", note, err) } return "", err } if err := exec.CommandContext(ctx, "git", "-C", workspace, "merge-base", "--is-ancestor", mainHead, itemHead).Run(); err != nil { return note, fmt.Errorf("delivery item commit cannot fast-forward canonical main") } if output, err := exec.CommandContext(ctx, "git", "-C", access.MainWorktree, "merge", "--ff-only", itemHead).CombinedOutput(); err != nil { return note, fmt.Errorf("delivery fast-forward canonical main (the main checkout has local changes that conflict with the item; the item is checked and ready on its branch): %w: %s", err, strings.TrimSpace(string(output))) } status, err = gitOutput(workspace, "status", "--porcelain=v1", "--untracked-files=all") if err != nil { return note, fmt.Errorf("delivery settlement status: %w", err) } if status != "" { return note, fmt.Errorf("delivery checkout is not clean after checks: %s", strings.Split(status, "\n")[0]) } // Re-derive at the settlement boundary. A success means exactly the checked // item commit is now reachable from the same canonical main worktree. current, err = landingAccess(workspace) if err != nil || current != access { return note, fmt.Errorf("delivery access changed during settlement") } mainHead, err = gitOutput(access.MainWorktree, "rev-parse", "HEAD") if err != nil { return note, fmt.Errorf("delivery main commit: %w", err) } if err := exec.CommandContext(ctx, "git", "-C", workspace, "merge-base", "--is-ancestor", itemHead, mainHead).Run(); err != nil { return note, fmt.Errorf("delivery item commit is not an ancestor of canonical main") } return note, nil } // commitDeliveryLeftovers commits everything still uncommitted in the item // workspace (tracked and untracked, respecting .gitignore). An empty message // or a clean tree commits nothing. func commitDeliveryLeftovers(ctx context.Context, workspace, message string) error { status, err := gitOutput(workspace, "status", "--porcelain=v1", "--untracked-files=all") if err != nil { return fmt.Errorf("delivery checkout status: %w", err) } if status == "" || strings.TrimSpace(message) == "" { return nil } if output, err := boundedCommandOutput(ctx, workspace, "add", "--all"); err != nil { return fmt.Errorf("delivery commit of remaining changes: %w: %s", err, strings.TrimSpace(string(output))) } if output, err := boundedCommandOutput(ctx, workspace, "commit", "--no-verify", "-m", message); err != nil { return fmt.Errorf("delivery commit of remaining changes: %w: %s", err, strings.TrimSpace(string(output))) } return nil } // catchUpDeliveryItem merges canonical main into the clean item workspace with a // normal merge commit and returns the new item HEAD. It never rewrites history: // a failed merge is aborted and the workspace is verified to be back at // itemHead with a clean checkout. func catchUpDeliveryItem(ctx context.Context, workspace, itemHead, mainHead string) (string, error) { output, mergeErr := boundedCommandOutput(ctx, workspace, "merge", "--no-edit", mainHead) if mergeErr == nil { merged, err := gitOutput(workspace, "rev-parse", "HEAD") if err != nil { return "", fmt.Errorf("delivery catch-up merge commit: %w", err) } return merged, nil } conflicts, _ := gitOutput(workspace, "diff", "--name-only", "--diff-filter=U") abortCtx, cancel := context.WithTimeout(context.Background(), deliveryCleanupTimeout) defer cancel() if _, err := gitOutput(workspace, "rev-parse", "-q", "--verify", "MERGE_HEAD"); err == nil { if abortOutput, err := exec.CommandContext(abortCtx, "git", "-C", workspace, "merge", "--abort").CombinedOutput(); err != nil { return "", fmt.Errorf("main moved and the catch-up merge could not be aborted: %w: %s", err, strings.TrimSpace(string(abortOutput))) } } head, headErr := gitOutput(workspace, "rev-parse", "HEAD") status, statusErr := gitOutput(workspace, "status", "--porcelain=v1", "--untracked-files=all") if headErr != nil || statusErr != nil || head != itemHead || status != "" { return "", fmt.Errorf("main moved and the catch-up merge did not restore the item workspace to %s", shortCommit(itemHead)) } if conflicts != "" { return "", fmt.Errorf("main moved and the item no longer merges cleanly: %s", strings.Join(strings.Fields(conflicts), ", ")) } return "", fmt.Errorf("main moved and the catch-up merge of %s failed: %w: %s", shortCommit(mainHead), mergeErr, strings.TrimSpace(string(output))) } func shortCommit(commit string) string { if len(commit) > 12 { return commit[:12] } return commit } func acquireDeliveryLock(ctx context.Context, path string) (*os.File, error) { file, err := os.OpenFile(path, os.O_CREATE|os.O_RDWR, 0600) if err != nil { return nil, fmt.Errorf("delivery lock: %w", err) } for { err := syscall.Flock(int(file.Fd()), syscall.LOCK_EX|syscall.LOCK_NB) if err == nil { return file, nil } if !errors.Is(err, syscall.EWOULDBLOCK) && !errors.Is(err, syscall.EAGAIN) { _ = file.Close() return nil, fmt.Errorf("delivery lock: %w", err) } select { case <-ctx.Done(): _ = file.Close() return nil, fmt.Errorf("delivery lock: %w", ctx.Err()) case <-time.After(20 * time.Millisecond): } } } func runDeliveryCandidate(ctx context.Context, workspace, itemHead string, checks []string) error { candidate, err := os.MkdirTemp(filepath.Dir(workspace), ".pipelines-delivery-") if err != nil { return fmt.Errorf("create delivery candidate: %w", err) } if err := os.Remove(candidate); err != nil { return fmt.Errorf("prepare delivery candidate: %w", err) } if output, err := boundedCommandOutput(ctx, workspace, "worktree", "add", "--detach", candidate, itemHead); err != nil { cleanupDeliveryCandidate(workspace, candidate) return fmt.Errorf("create delivery candidate: %w: %s", err, strings.TrimSpace(string(output))) } defer cleanupDeliveryCandidate(workspace, candidate) for _, check := range checks { if err := runDeliveryCheck(ctx, candidate, check); err != nil { return err } } return nil } // cleanupDeliveryCandidate always uses its own bounded context: the stage context // may already be canceled when a check fails or times out. func cleanupDeliveryCandidate(workspace, candidate string) { ctx, cancel := context.WithTimeout(context.Background(), deliveryCleanupTimeout) defer cancel() _ = exec.CommandContext(ctx, "git", "-C", workspace, "worktree", "remove", "--force", candidate).Run() _ = os.RemoveAll(candidate) } func boundedCommandOutput(ctx context.Context, directory string, args ...string) ([]byte, error) { cmd := exec.CommandContext(ctx, "git", append([]string{"-C", directory}, args...)...) cmd.Env = workenv.CommandEnv(ctx, directory) var output boundedDeliveryOutput cmd.Stdout = &output cmd.Stderr = &output err := cmd.Run() return []byte(output.String()), err } func runDeliveryCheck(ctx context.Context, directory, check string) error { cmd := exec.CommandContext(ctx, "sh", "-c", check) cmd.Dir = directory // The run workspace's own environment (direnv/mise), as a terminal there // would have it; nil inherits the server's. cmd.Env = workenv.CommandEnv(ctx, directory) var output boundedDeliveryOutput cmd.Stdout = &output cmd.Stderr = &output if err := cmd.Run(); err != nil { return fmt.Errorf("delivery check %q failed: %w: %s", check, err, output.String()) } return nil } type boundedDeliveryOutput struct{ bytes.Buffer } func (b *boundedDeliveryOutput) Write(p []byte) (int, error) { remaining := maxDeliveryCheckOutput - b.Len() if remaining > 0 { _, _ = b.Buffer.Write(p[:min(remaining, len(p))]) } return len(p), nil } func (b *boundedDeliveryOutput) String() string { text := strings.TrimSpace(b.Buffer.String()) if b.Len() == maxDeliveryCheckOutput { return text + " (output truncated)" } return text } var _ io.Writer = (*boundedDeliveryOutput)(nil)