Files
pipelines/internal/server/run_worktree.go
T
alan 532ba48927 Run every pipeline run in its own worktree
A fresh run started from a repository's main checkout gets its own
worktree and branch next to the repository, so runs never share a tree
or see the operator's uncommitted work, and several can run at once.
Retries and runs started in a linked worktree keep their directory. A
clean, landed run tree is removed after success; anything else is kept.

The delivery gate commits what the land stage left uncommitted in a run
worktree, and no longer requires the operator's main checkout to be
clean: Git's fast-forward refuses safely when a local change conflicts.
2026-11-10 16:06:40 +00:00

198 lines
7.1 KiB
Go

package server
import (
"context"
"crypto/rand"
"encoding/hex"
"fmt"
"os"
"os/exec"
"path/filepath"
"regexp"
"strings"
"time"
)
// runWorktreeBranchPrefix names the branch of every worktree the server creates
// for a run, so cleanup can tell its own trees from an operator's.
const runWorktreeBranchPrefix = "pipelines/run-"
// runWorktreesDirSuffix is appended to a repository's directory name for the
// sibling directory that holds its run worktrees: /repos/app -> /repos/app-runs.
const runWorktreesDirSuffix = "-runs"
// isolateRunWorkspace gives a new run its own Git worktree when it was started
// from a repository's main checkout, so concurrent runs never share a tree and a
// run never sees (or commits) the operator's uncommitted work. A workspace that
// is not in a Git repository, or is already a linked worktree (an item's tree,
// or a retry reusing its run's tree), is returned unchanged. A subdirectory of
// the main checkout maps to the same subdirectory of the new worktree.
//
// The worktree starts at the checkout's current HEAD on a new branch
// pipelines/run-<slug> and lives in <repo>-runs/<slug> next to the repository.
func isolateRunWorkspace(ctx context.Context, workspace, title string) (string, error) {
top, err := gitOutput(workspace, "rev-parse", "--show-toplevel")
if err != nil {
return workspace, nil // Not a Git checkout: nothing to isolate.
}
gitDir, err := gitOutput(workspace, "rev-parse", "--path-format=absolute", "--git-dir")
if err != nil {
return "", fmt.Errorf("run worktree: %w", err)
}
commonDir, err := gitOutput(workspace, "rev-parse", "--path-format=absolute", "--git-common-dir")
if err != nil {
return "", fmt.Errorf("run worktree: %w", err)
}
if filepath.Clean(gitDir) != filepath.Clean(commonDir) {
return workspace, nil // Already a linked worktree.
}
if _, err := gitOutput(workspace, "rev-parse", "--verify", "HEAD"); err != nil {
return workspace, nil // No commit to branch from yet.
}
top, err = filepath.EvalSymlinks(top)
if err != nil {
return "", fmt.Errorf("run worktree: %w", err)
}
rel, err := filepath.Rel(top, workspace)
if err != nil || rel == ".." || strings.HasPrefix(rel, ".."+string(os.PathSeparator)) {
return "", fmt.Errorf("run worktree: workspace %s is outside its repository %s", workspace, top)
}
slug := runWorktreeSlug(time.Now(), title)
root := filepath.Join(filepath.Dir(top), filepath.Base(top)+runWorktreesDirSuffix)
if err := os.MkdirAll(root, 0o755); err != nil {
return "", fmt.Errorf("run worktree: %w", err)
}
tree := filepath.Join(root, slug)
branch := runWorktreeBranchPrefix + slug
cmd := exec.CommandContext(ctx, "git", "-C", top, "worktree", "add", "-b", branch, tree, "HEAD")
if output, err := cmd.CombinedOutput(); err != nil {
return "", fmt.Errorf("run worktree: git worktree add %s: %w: %s", tree, err, strings.TrimSpace(string(output)))
}
resolved, err := filepath.EvalSymlinks(tree)
if err != nil {
return "", fmt.Errorf("run worktree: %w", err)
}
return filepath.Join(resolved, rel), nil
}
var runSlugUnsafe = regexp.MustCompile(`[^a-z0-9]+`)
// runWorktreeSlug is a sortable, readable and unique directory/branch name:
// 20261110-1530-short-title-ab12.
func runWorktreeSlug(now time.Time, title string) string {
words := strings.Trim(runSlugUnsafe.ReplaceAllString(strings.ToLower(title), "-"), "-")
if len(words) > 40 {
words = strings.TrimRight(words[:40], "-")
}
random := make([]byte, 2)
_, _ = rand.Read(random)
parts := []string{now.UTC().Format("20060102-1504")}
if words != "" {
parts = append(parts, words)
}
parts = append(parts, hex.EncodeToString(random))
return strings.Join(parts, "-")
}
// removeLandedRunWorktree deletes a run worktree the server created once its
// work is safe: the tree is clean and its branch is already in the main
// main branch (landed, or no commits at all). Anything else — an
// operator's tree, uncommitted files, unlanded commits — is kept, so a failed
// or unfinished run never loses work. It reports whether the tree was removed.
func removeLandedRunWorktree(workspace string) bool {
if !isRunWorktree(workspace) {
return false
}
top, err := gitOutput(workspace, "rev-parse", "--show-toplevel")
if err != nil {
return false
}
branch, err := gitOutput(top, "symbolic-ref", "--quiet", "--short", "HEAD")
if err != nil {
return false
}
if status, err := gitOutput(top, "status", "--porcelain=v1", "--untracked-files=all"); err != nil || status != "" {
return false
}
main, err := firstWorktree(top)
if err != nil {
return false
}
// Landed means reachable from the repository's main branch.
if exec.Command("git", "-C", top, "merge-base", "--is-ancestor", "HEAD", "refs/heads/main").Run() != nil {
return false
}
if exec.Command("git", "-C", main, "worktree", "remove", top).Run() != nil {
return false
}
_ = exec.Command("git", "-C", main, "branch", "-d", branch).Run()
return true
}
// firstWorktree is the repository's main (first-listed) worktree, the one
// linked worktrees are managed from.
func firstWorktree(directory string) (string, error) {
list, err := gitOutput(directory, "worktree", "list", "--porcelain")
if err != nil {
return "", err
}
first, _, _ := strings.Cut(list, "\n")
if !strings.HasPrefix(first, "worktree ") {
return "", fmt.Errorf("no main worktree")
}
return strings.TrimPrefix(first, "worktree "), nil
}
// discardRunWorktree removes a run worktree that was created for a run that
// never started (refused or a duplicate request), with its branch. It only
// touches trees the server created.
func discardRunWorktree(workspace string) {
if !isRunWorktree(workspace) {
return
}
top, err := gitOutput(workspace, "rev-parse", "--show-toplevel")
if err != nil {
return
}
branch, err := gitOutput(top, "symbolic-ref", "--quiet", "--short", "HEAD")
if err != nil {
return
}
main, err := firstWorktree(top)
if err != nil {
return
}
_ = exec.Command("git", "-C", main, "worktree", "remove", "--force", top).Run()
_ = exec.Command("git", "-C", main, "branch", "-D", branch).Run()
}
// deliveryCommitMessage is the message for the work a land stage left
// uncommitted: the run's title and id. Only a run in a server-created run
// worktree gets one; elsewhere the tree may hold someone else's work, so the
// gate keeps refusing an unclean checkout there.
func (r *runner) deliveryCommitMessage(id int64) string {
c, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
detail, err := r.store.GetRun(c, id)
if err != nil || detail == nil || !isRunWorktree(detail.Workspace) {
return ""
}
title := strings.TrimSpace(detail.Title)
if title == "" {
title = detail.Name
}
return fmt.Sprintf("%s\n\nPipelines run %d.", title, id)
}
// isRunWorktree reports whether directory is inside a worktree the server
// created for a run.
func isRunWorktree(directory string) bool {
top, err := gitOutput(directory, "rev-parse", "--show-toplevel")
if err != nil {
return false
}
branch, err := gitOutput(top, "symbolic-ref", "--quiet", "--short", "HEAD")
return err == nil && strings.HasPrefix(branch, runWorktreeBranchPrefix) &&
strings.HasSuffix(filepath.Base(filepath.Dir(top)), runWorktreesDirSuffix)
}