Files
pipelines/internal/server/delivery_gate.go
T
alan 532ba48927 Run every pipeline run in its own worktree
A fresh run started from a repository's main checkout gets its own
worktree and branch next to the repository, so runs never share a tree
or see the operator's uncommitted work, and several can run at once.
Retries and runs started in a linked worktree keep their directory. A
clean, landed run tree is removed after success; anything else is kept.

The delivery gate commits what the land stage left uncommitted in a run
worktree, and no longer requires the operator's main checkout to be
clean: Git's fast-forward refuses safely when a local change conflicts.
2026-11-10 16:06:40 +00:00

276 lines
11 KiB
Go

package server
import (
"bytes"
"context"
"errors"
"fmt"
"io"
"os"
"os/exec"
"path/filepath"
"strings"
"syscall"
"time"
"git.a-wan.net/alan/pipelines/internal/stage"
"git.a-wan.net/alan/pipelines/internal/workenv"
)
// verifyDelivery proves that the stage landed a complete, committed tree before
// its success can settle the run. It re-derives the capability after the stage:
// a stage result and its input cannot select either the landing path or checks.
const maxDeliveryCheckOutput = 64 << 10
const deliveryCleanupTimeout = 10 * time.Second
// verifyDelivery verifies the committed item tree in a disposable worktree
// before fast-forwarding canonical main. Settlement deliberately performs only
// non-mutating repository checks after that mutation: a declared check must
// never be the first signal that the landed tree is known-red.
//
// When canonical main moved after the item's last integration (for example a
// concurrent delivery landed first), the gate catches up while still holding
// the delivery lock: it merges main into the item workspace with an ordinary
// merge commit, runs the declared checks on that merged commit, and only then
// fast-forwards main. A conflicting merge is aborted, leaving the workspace
// exactly as it was. The returned note is non-empty when a catch-up merge
// happened, so the caller can record it.
//
// Work the land stage left uncommitted in the item workspace is committed
// first (commitMessage): the workspace is the run's own tree, so everything in
// it is the run's work, and a partial file list must never drop part of it.
// The operator's main checkout need not be clean: the fast-forward is Git's
// own `merge --ff-only`, which refuses without touching anything when a local
// change or untracked file would be overwritten.
func verifyDelivery(ctx context.Context, workspace string, access stage.LandingAccess, checks []string, commitMessage string) (string, error) {
lock, err := acquireDeliveryLock(ctx, access.LockPath)
if err != nil {
return "", err
}
defer lock.Close()
current, err := landingAccess(workspace)
if err != nil {
return "", fmt.Errorf("delivery access changed: %w", err)
}
if current != access {
return "", fmt.Errorf("delivery access changed during stage")
}
if err := commitDeliveryLeftovers(ctx, workspace, commitMessage); err != nil {
return "", err
}
status, err := gitOutput(workspace, "status", "--porcelain=v1", "--untracked-files=all")
if err != nil {
return "", fmt.Errorf("delivery checkout status: %w", err)
}
if status != "" {
return "", fmt.Errorf("delivery checkout is not clean: %s", strings.Split(status, "\n")[0])
}
itemHead, err := gitOutput(workspace, "rev-parse", "HEAD")
if err != nil {
return "", fmt.Errorf("delivery item commit: %w", err)
}
mainHead, err := gitOutput(access.MainWorktree, "rev-parse", "HEAD")
if err != nil {
return "", fmt.Errorf("delivery main commit: %w", err)
}
note := ""
if exec.CommandContext(ctx, "git", "-C", workspace, "merge-base", "--is-ancestor", mainHead, itemHead).Run() != nil {
merged, err := catchUpDeliveryItem(ctx, workspace, itemHead, mainHead)
if err != nil {
return "", err
}
note = fmt.Sprintf("delivery gate: canonical main moved to %s; merged it into item %s as %s and re-ran checks on the merged tree", shortCommit(mainHead), shortCommit(itemHead), shortCommit(merged))
itemHead = merged
}
if err := runDeliveryCandidate(ctx, workspace, itemHead, checks); err != nil {
if note != "" {
return note, fmt.Errorf("%s; %w", note, err)
}
return "", err
}
if err := exec.CommandContext(ctx, "git", "-C", workspace, "merge-base", "--is-ancestor", mainHead, itemHead).Run(); err != nil {
return note, fmt.Errorf("delivery item commit cannot fast-forward canonical main")
}
if output, err := exec.CommandContext(ctx, "git", "-C", access.MainWorktree, "merge", "--ff-only", itemHead).CombinedOutput(); err != nil {
return note, fmt.Errorf("delivery fast-forward canonical main (the main checkout has local changes that conflict with the item; the item is checked and ready on its branch): %w: %s", err, strings.TrimSpace(string(output)))
}
status, err = gitOutput(workspace, "status", "--porcelain=v1", "--untracked-files=all")
if err != nil {
return note, fmt.Errorf("delivery settlement status: %w", err)
}
if status != "" {
return note, fmt.Errorf("delivery checkout is not clean after checks: %s", strings.Split(status, "\n")[0])
}
// Re-derive at the settlement boundary. A success means exactly the checked
// item commit is now reachable from the same canonical main worktree.
current, err = landingAccess(workspace)
if err != nil || current != access {
return note, fmt.Errorf("delivery access changed during settlement")
}
mainHead, err = gitOutput(access.MainWorktree, "rev-parse", "HEAD")
if err != nil {
return note, fmt.Errorf("delivery main commit: %w", err)
}
if err := exec.CommandContext(ctx, "git", "-C", workspace, "merge-base", "--is-ancestor", itemHead, mainHead).Run(); err != nil {
return note, fmt.Errorf("delivery item commit is not an ancestor of canonical main")
}
return note, nil
}
// commitDeliveryLeftovers commits everything still uncommitted in the item
// workspace (tracked and untracked, respecting .gitignore). An empty message
// or a clean tree commits nothing.
func commitDeliveryLeftovers(ctx context.Context, workspace, message string) error {
status, err := gitOutput(workspace, "status", "--porcelain=v1", "--untracked-files=all")
if err != nil {
return fmt.Errorf("delivery checkout status: %w", err)
}
if status == "" || strings.TrimSpace(message) == "" {
return nil
}
if output, err := boundedCommandOutput(ctx, workspace, "add", "--all"); err != nil {
return fmt.Errorf("delivery commit of remaining changes: %w: %s", err, strings.TrimSpace(string(output)))
}
if output, err := boundedCommandOutput(ctx, workspace, "commit", "--no-verify", "-m", message); err != nil {
return fmt.Errorf("delivery commit of remaining changes: %w: %s", err, strings.TrimSpace(string(output)))
}
return nil
}
// catchUpDeliveryItem merges canonical main into the clean item workspace with a
// normal merge commit and returns the new item HEAD. It never rewrites history:
// a failed merge is aborted and the workspace is verified to be back at
// itemHead with a clean checkout.
func catchUpDeliveryItem(ctx context.Context, workspace, itemHead, mainHead string) (string, error) {
output, mergeErr := boundedCommandOutput(ctx, workspace, "merge", "--no-edit", mainHead)
if mergeErr == nil {
merged, err := gitOutput(workspace, "rev-parse", "HEAD")
if err != nil {
return "", fmt.Errorf("delivery catch-up merge commit: %w", err)
}
return merged, nil
}
conflicts, _ := gitOutput(workspace, "diff", "--name-only", "--diff-filter=U")
abortCtx, cancel := context.WithTimeout(context.Background(), deliveryCleanupTimeout)
defer cancel()
if _, err := gitOutput(workspace, "rev-parse", "-q", "--verify", "MERGE_HEAD"); err == nil {
if abortOutput, err := exec.CommandContext(abortCtx, "git", "-C", workspace, "merge", "--abort").CombinedOutput(); err != nil {
return "", fmt.Errorf("main moved and the catch-up merge could not be aborted: %w: %s", err, strings.TrimSpace(string(abortOutput)))
}
}
head, headErr := gitOutput(workspace, "rev-parse", "HEAD")
status, statusErr := gitOutput(workspace, "status", "--porcelain=v1", "--untracked-files=all")
if headErr != nil || statusErr != nil || head != itemHead || status != "" {
return "", fmt.Errorf("main moved and the catch-up merge did not restore the item workspace to %s", shortCommit(itemHead))
}
if conflicts != "" {
return "", fmt.Errorf("main moved and the item no longer merges cleanly: %s", strings.Join(strings.Fields(conflicts), ", "))
}
return "", fmt.Errorf("main moved and the catch-up merge of %s failed: %w: %s", shortCommit(mainHead), mergeErr, strings.TrimSpace(string(output)))
}
func shortCommit(commit string) string {
if len(commit) > 12 {
return commit[:12]
}
return commit
}
func acquireDeliveryLock(ctx context.Context, path string) (*os.File, error) {
file, err := os.OpenFile(path, os.O_CREATE|os.O_RDWR, 0600)
if err != nil {
return nil, fmt.Errorf("delivery lock: %w", err)
}
for {
err := syscall.Flock(int(file.Fd()), syscall.LOCK_EX|syscall.LOCK_NB)
if err == nil {
return file, nil
}
if !errors.Is(err, syscall.EWOULDBLOCK) && !errors.Is(err, syscall.EAGAIN) {
_ = file.Close()
return nil, fmt.Errorf("delivery lock: %w", err)
}
select {
case <-ctx.Done():
_ = file.Close()
return nil, fmt.Errorf("delivery lock: %w", ctx.Err())
case <-time.After(20 * time.Millisecond):
}
}
}
func runDeliveryCandidate(ctx context.Context, workspace, itemHead string, checks []string) error {
candidate, err := os.MkdirTemp(filepath.Dir(workspace), ".pipelines-delivery-")
if err != nil {
return fmt.Errorf("create delivery candidate: %w", err)
}
if err := os.Remove(candidate); err != nil {
return fmt.Errorf("prepare delivery candidate: %w", err)
}
if output, err := boundedCommandOutput(ctx, workspace, "worktree", "add", "--detach", candidate, itemHead); err != nil {
cleanupDeliveryCandidate(workspace, candidate)
return fmt.Errorf("create delivery candidate: %w: %s", err, strings.TrimSpace(string(output)))
}
defer cleanupDeliveryCandidate(workspace, candidate)
for _, check := range checks {
if err := runDeliveryCheck(ctx, candidate, check); err != nil {
return err
}
}
return nil
}
// cleanupDeliveryCandidate always uses its own bounded context: the stage context
// may already be canceled when a check fails or times out.
func cleanupDeliveryCandidate(workspace, candidate string) {
ctx, cancel := context.WithTimeout(context.Background(), deliveryCleanupTimeout)
defer cancel()
_ = exec.CommandContext(ctx, "git", "-C", workspace, "worktree", "remove", "--force", candidate).Run()
_ = os.RemoveAll(candidate)
}
func boundedCommandOutput(ctx context.Context, directory string, args ...string) ([]byte, error) {
cmd := exec.CommandContext(ctx, "git", append([]string{"-C", directory}, args...)...)
cmd.Env = workenv.CommandEnv(ctx, directory)
var output boundedDeliveryOutput
cmd.Stdout = &output
cmd.Stderr = &output
err := cmd.Run()
return []byte(output.String()), err
}
func runDeliveryCheck(ctx context.Context, directory, check string) error {
cmd := exec.CommandContext(ctx, "sh", "-c", check)
cmd.Dir = directory
// The run workspace's own environment (direnv/mise), as a terminal there
// would have it; nil inherits the server's.
cmd.Env = workenv.CommandEnv(ctx, directory)
var output boundedDeliveryOutput
cmd.Stdout = &output
cmd.Stderr = &output
if err := cmd.Run(); err != nil {
return fmt.Errorf("delivery check %q failed: %w: %s", check, err, output.String())
}
return nil
}
type boundedDeliveryOutput struct{ bytes.Buffer }
func (b *boundedDeliveryOutput) Write(p []byte) (int, error) {
remaining := maxDeliveryCheckOutput - b.Len()
if remaining > 0 {
_, _ = b.Buffer.Write(p[:min(remaining, len(p))])
}
return len(p), nil
}
func (b *boundedDeliveryOutput) String() string {
text := strings.TrimSpace(b.Buffer.String())
if b.Len() == maxDeliveryCheckOutput {
return text + " (output truncated)"
}
return text
}
var _ io.Writer = (*boundedDeliveryOutput)(nil)