Files
pipelines/internal/server/delivery_gate.go
T
alan be932c846f Catch up a moved main in the delivery gate before landing
When canonical main is not an ancestor of the item HEAD, the gate now
merges main into the item workspace under the delivery lock, re-runs the
declared checks on the merged commit, and fast-forwards main to it. A
conflicting merge is aborted with the workspace verified unchanged and a
diagnostic naming the files. Successful catch-ups are logged.
2026-10-31 15:24:12 +00:00

245 lines
9.5 KiB
Go

package server
import (
"bytes"
"context"
"errors"
"fmt"
"io"
"os"
"os/exec"
"path/filepath"
"strings"
"syscall"
"time"
"git.a-wan.net/alan/pipelines/internal/stage"
)
// verifyDelivery proves that the stage landed a complete, committed tree before
// its success can settle the run. It re-derives the capability after the stage:
// a stage result and its input cannot select either the landing path or checks.
const maxDeliveryCheckOutput = 64 << 10
const deliveryCleanupTimeout = 10 * time.Second
// verifyDelivery verifies the committed item tree in a disposable worktree
// before fast-forwarding canonical main. Settlement deliberately performs only
// non-mutating repository checks after that mutation: a declared check must
// never be the first signal that the landed tree is known-red.
//
// When canonical main moved after the item's last integration (for example a
// concurrent delivery landed first), the gate catches up while still holding
// the delivery lock: it merges main into the item workspace with an ordinary
// merge commit, runs the declared checks on that merged commit, and only then
// fast-forwards main. A conflicting merge is aborted, leaving the workspace
// exactly as it was. The returned note is non-empty when a catch-up merge
// happened, so the caller can record it.
func verifyDelivery(ctx context.Context, workspace string, access stage.LandingAccess, checks []string) (string, error) {
lock, err := acquireDeliveryLock(ctx, access.LockPath)
if err != nil {
return "", err
}
defer lock.Close()
current, err := landingAccess(workspace)
if err != nil {
return "", fmt.Errorf("delivery access changed: %w", err)
}
if current != access {
return "", fmt.Errorf("delivery access changed during stage")
}
for _, directory := range []string{workspace, access.MainWorktree} {
status, err := gitOutput(directory, "status", "--porcelain=v1", "--untracked-files=all")
if err != nil {
return "", fmt.Errorf("delivery checkout status: %w", err)
}
if status != "" {
return "", fmt.Errorf("delivery checkout is not clean: %s", strings.Split(status, "\n")[0])
}
}
itemHead, err := gitOutput(workspace, "rev-parse", "HEAD")
if err != nil {
return "", fmt.Errorf("delivery item commit: %w", err)
}
mainHead, err := gitOutput(access.MainWorktree, "rev-parse", "HEAD")
if err != nil {
return "", fmt.Errorf("delivery main commit: %w", err)
}
note := ""
if exec.CommandContext(ctx, "git", "-C", workspace, "merge-base", "--is-ancestor", mainHead, itemHead).Run() != nil {
merged, err := catchUpDeliveryItem(ctx, workspace, itemHead, mainHead)
if err != nil {
return "", err
}
note = fmt.Sprintf("delivery gate: canonical main moved to %s; merged it into item %s as %s and re-ran checks on the merged tree", shortCommit(mainHead), shortCommit(itemHead), shortCommit(merged))
itemHead = merged
}
if err := runDeliveryCandidate(ctx, workspace, itemHead, checks); err != nil {
if note != "" {
return note, fmt.Errorf("%s; %w", note, err)
}
return "", err
}
if err := exec.CommandContext(ctx, "git", "-C", workspace, "merge-base", "--is-ancestor", mainHead, itemHead).Run(); err != nil {
return note, fmt.Errorf("delivery item commit cannot fast-forward canonical main")
}
if output, err := exec.CommandContext(ctx, "git", "-C", access.MainWorktree, "merge", "--ff-only", itemHead).CombinedOutput(); err != nil {
return note, fmt.Errorf("delivery fast-forward canonical main: %w: %s", err, strings.TrimSpace(string(output)))
}
for _, directory := range []string{workspace, access.MainWorktree} {
status, err := gitOutput(directory, "status", "--porcelain=v1", "--untracked-files=all")
if err != nil {
return note, fmt.Errorf("delivery settlement status: %w", err)
}
if status != "" {
return note, fmt.Errorf("delivery checkout is not clean after checks: %s", strings.Split(status, "\n")[0])
}
}
// Re-derive at the settlement boundary. A success means exactly the checked
// item commit is now reachable from the same canonical main worktree.
current, err = landingAccess(workspace)
if err != nil || current != access {
return note, fmt.Errorf("delivery access changed during settlement")
}
mainHead, err = gitOutput(access.MainWorktree, "rev-parse", "HEAD")
if err != nil {
return note, fmt.Errorf("delivery main commit: %w", err)
}
if err := exec.CommandContext(ctx, "git", "-C", workspace, "merge-base", "--is-ancestor", itemHead, mainHead).Run(); err != nil {
return note, fmt.Errorf("delivery item commit is not an ancestor of canonical main")
}
return note, nil
}
// catchUpDeliveryItem merges canonical main into the clean item workspace with a
// normal merge commit and returns the new item HEAD. It never rewrites history:
// a failed merge is aborted and the workspace is verified to be back at
// itemHead with a clean checkout.
func catchUpDeliveryItem(ctx context.Context, workspace, itemHead, mainHead string) (string, error) {
output, mergeErr := boundedCommandOutput(ctx, workspace, "merge", "--no-edit", mainHead)
if mergeErr == nil {
merged, err := gitOutput(workspace, "rev-parse", "HEAD")
if err != nil {
return "", fmt.Errorf("delivery catch-up merge commit: %w", err)
}
return merged, nil
}
conflicts, _ := gitOutput(workspace, "diff", "--name-only", "--diff-filter=U")
abortCtx, cancel := context.WithTimeout(context.Background(), deliveryCleanupTimeout)
defer cancel()
if _, err := gitOutput(workspace, "rev-parse", "-q", "--verify", "MERGE_HEAD"); err == nil {
if abortOutput, err := exec.CommandContext(abortCtx, "git", "-C", workspace, "merge", "--abort").CombinedOutput(); err != nil {
return "", fmt.Errorf("main moved and the catch-up merge could not be aborted: %w: %s", err, strings.TrimSpace(string(abortOutput)))
}
}
head, headErr := gitOutput(workspace, "rev-parse", "HEAD")
status, statusErr := gitOutput(workspace, "status", "--porcelain=v1", "--untracked-files=all")
if headErr != nil || statusErr != nil || head != itemHead || status != "" {
return "", fmt.Errorf("main moved and the catch-up merge did not restore the item workspace to %s", shortCommit(itemHead))
}
if conflicts != "" {
return "", fmt.Errorf("main moved and the item no longer merges cleanly: %s", strings.Join(strings.Fields(conflicts), ", "))
}
return "", fmt.Errorf("main moved and the catch-up merge of %s failed: %w: %s", shortCommit(mainHead), mergeErr, strings.TrimSpace(string(output)))
}
func shortCommit(commit string) string {
if len(commit) > 12 {
return commit[:12]
}
return commit
}
func acquireDeliveryLock(ctx context.Context, path string) (*os.File, error) {
file, err := os.OpenFile(path, os.O_CREATE|os.O_RDWR, 0600)
if err != nil {
return nil, fmt.Errorf("delivery lock: %w", err)
}
for {
err := syscall.Flock(int(file.Fd()), syscall.LOCK_EX|syscall.LOCK_NB)
if err == nil {
return file, nil
}
if !errors.Is(err, syscall.EWOULDBLOCK) && !errors.Is(err, syscall.EAGAIN) {
_ = file.Close()
return nil, fmt.Errorf("delivery lock: %w", err)
}
select {
case <-ctx.Done():
_ = file.Close()
return nil, fmt.Errorf("delivery lock: %w", ctx.Err())
case <-time.After(20 * time.Millisecond):
}
}
}
func runDeliveryCandidate(ctx context.Context, workspace, itemHead string, checks []string) error {
candidate, err := os.MkdirTemp(filepath.Dir(workspace), ".pipelines-delivery-")
if err != nil {
return fmt.Errorf("create delivery candidate: %w", err)
}
if err := os.Remove(candidate); err != nil {
return fmt.Errorf("prepare delivery candidate: %w", err)
}
if output, err := boundedCommandOutput(ctx, workspace, "worktree", "add", "--detach", candidate, itemHead); err != nil {
cleanupDeliveryCandidate(workspace, candidate)
return fmt.Errorf("create delivery candidate: %w: %s", err, strings.TrimSpace(string(output)))
}
defer cleanupDeliveryCandidate(workspace, candidate)
for _, check := range checks {
if err := runDeliveryCheck(ctx, candidate, check); err != nil {
return err
}
}
return nil
}
// cleanupDeliveryCandidate always uses its own bounded context: the stage context
// may already be canceled when a check fails or times out.
func cleanupDeliveryCandidate(workspace, candidate string) {
ctx, cancel := context.WithTimeout(context.Background(), deliveryCleanupTimeout)
defer cancel()
_ = exec.CommandContext(ctx, "git", "-C", workspace, "worktree", "remove", "--force", candidate).Run()
_ = os.RemoveAll(candidate)
}
func boundedCommandOutput(ctx context.Context, directory string, args ...string) ([]byte, error) {
cmd := exec.CommandContext(ctx, "git", append([]string{"-C", directory}, args...)...)
var output boundedDeliveryOutput
cmd.Stdout = &output
cmd.Stderr = &output
err := cmd.Run()
return []byte(output.String()), err
}
func runDeliveryCheck(ctx context.Context, directory, check string) error {
cmd := exec.CommandContext(ctx, "sh", "-c", check)
cmd.Dir = directory
var output boundedDeliveryOutput
cmd.Stdout = &output
cmd.Stderr = &output
if err := cmd.Run(); err != nil {
return fmt.Errorf("delivery check %q failed: %w: %s", check, err, output.String())
}
return nil
}
type boundedDeliveryOutput struct{ bytes.Buffer }
func (b *boundedDeliveryOutput) Write(p []byte) (int, error) {
remaining := maxDeliveryCheckOutput - b.Len()
if remaining > 0 {
_, _ = b.Buffer.Write(p[:min(remaining, len(p))])
}
return len(p), nil
}
func (b *boundedDeliveryOutput) String() string {
text := strings.TrimSpace(b.Buffer.String())
if b.Len() == maxDeliveryCheckOutput {
return text + " (output truncated)"
}
return text
}
var _ io.Writer = (*boundedDeliveryOutput)(nil)