Files
pipelines/internal/server/turns.go
T
alan c2b992542b Tell the next stage what failed when an error edge leads onward
A failed visit was recorded on its step only, so a landing gate that sent
the run back to implement gave the implementer nothing about which checks
failed. The diagnostic (bounded) is now the next stage's previous output
and a memory entry.
2026-11-11 00:39:36 +00:00

418 lines
14 KiB
Go

package server
import (
"encoding/json"
"errors"
"fmt"
"io"
"io/fs"
"os"
"path"
"path/filepath"
"sort"
"strconv"
"strings"
"syscall"
"git.a-wan.net/alan/pipelines/internal/protocol"
"git.a-wan.net/alan/pipelines/internal/stage"
"git.a-wan.net/alan/pipelines/internal/turn"
)
// turnRoot is the run-level artifact directory shared by every stage turn. The
// per-visit directories live under <run>/stages/ and public/ is run-level.
func (r *runner) turnRoot(runID int64) string {
return filepath.Join(r.artifactsRoot, strconv.FormatInt(runID, 10))
}
// turnPaths returns the deterministic layout for one visit. The zero-padded
// sequence is part of the directory name, and a retry is admitted as a new run
// with its own artifact root, so a later visit never reuses an earlier turn's
// files.
func (r *runner) turnPaths(runID int64, sequence int, stageKey string) turn.Paths {
return turn.Layout(r.turnRoot(runID), sequence, stageKey)
}
// stageTurnPaths projects the turn layout onto the runtime-only paths a model
// stage receives.
func stageTurnPaths(paths turn.Paths) stage.TurnPaths {
return stage.TurnPaths{
Dir: paths.TurnDir,
Input: paths.InputPath,
Memory: paths.MemoryPath,
History: paths.HistoryPath,
Output: paths.OutputPath,
Private: paths.PrivateDir,
Public: paths.PublicDir,
}
}
// prepareTurn creates the visit layout and writes the model's input.md,
// memory.md (essential memory, capped at turn.EssentialCap) and history.md (every
// completed turn in full) before the visit starts. A failure here is a run failure: the file
// protocol is the stage contract, not best-effort evidence.
//
// retryInput is the complete merged input for a selected-stage retry and nil for
// an ordinary visit. A retry override is persisted on the step, but the model
// reads input.md rather than the step row, so the same values must be written
// into the file or the operator's correction is invisible to the model.
func (r *runner) prepareTurn(paths turn.Paths, state *runState, retryInput stage.Values) error {
if err := paths.Ensure(); err != nil {
return err
}
input := turnInputWithRetry(state.turnInput(), retryInput)
history := turn.RenderHistory(state.turns)
memory := turn.RenderEssential(turn.Essential{
Task: state.prompt,
StageKey: paths.StageKey,
Entries: state.turns,
ChangedFiles: state.capturedList(),
InputPath: paths.InputPath,
HistoryPath: paths.HistoryPath,
})
for _, file := range []struct{ path, content string }{
{paths.InputPath, input},
{paths.MemoryPath, memory},
{paths.HistoryPath, history},
} {
if err := os.WriteFile(file.path, []byte(file.content), 0o600); err != nil {
return fmt.Errorf("write %s: %w", file.path, err)
}
}
return nil
}
// turnInputWithRetry appends the merged retry input to the ordinary turn input.
// It is rendered as a fenced JSON object because the values are operator data,
// not prose; encoding/json sorts map keys so the file is deterministic.
func turnInputWithRetry(base string, retryInput stage.Values) string {
if len(retryInput) == 0 {
return base
}
encoded, err := json.MarshalIndent(retryInput, "", " ")
if err != nil {
// A retry input that cannot be rendered is still shown as a visible
// section; the stage can choose its error outcome rather than silently
// receiving a correction it cannot see.
return base + "\n\n# Retry input\n\nThe retry input could not be rendered.\n"
}
if !strings.HasSuffix(base, "\n") {
base += "\n"
}
return base + "\n# Retry input\n\nThis retry was given this complete input:\n\n```json\n" + string(encoded) + "\n```\n"
}
// turnInput is the run prompt on the first visit; later visits carry the run
// prompt and the previous visit's output.
func (s *runState) turnInput() string {
if len(s.turns) == 0 && strings.TrimSpace(s.lastOutput) == "" {
return s.prompt
}
if strings.TrimSpace(s.lastOutput) == "" {
return turn.RenderInput(s.prompt, "_(no output)_\n")
}
return turn.RenderInput(s.prompt, s.lastOutput)
}
// turnOutput renders a completed step for later memory. Model stages store their
// body under the single markdown key; human and other stages keep a small
// fenced rendering of their recorded evidence.
func turnOutput(output stage.Values) string {
if markdown, ok := output["markdown"].(string); ok {
return markdown
}
if len(output) == 0 {
return ""
}
data, err := json.MarshalIndent(output, "", " ")
if err != nil {
return ""
}
return "```json\n" + string(data) + "\n```\n"
}
// recordTurn appends one completed visit to the run's accumulated memory. The
// public file list is a system-written addition computed from the filesystem,
// never a model claim.
func (s *runState) recordTurn(step protocol.RunStep, publicFiles []string) {
output := turnOutput(step.Output)
s.turns = append(s.turns, turn.MemoryEntry{
Stage: step.Stage,
Visit: s.visits[step.Stage],
Outcome: step.Outcome,
Output: output,
PublicFiles: publicFiles,
})
s.lastOutput = output
}
// maxFailureDetail bounds the failure text carried to the next stage. A
// delivery check's quiet output lists its failures first, so the head is kept.
const maxFailureDetail = 6000
// recordFailure makes a failed visit that routes onward (an error edge to
// another stage) visible to that stage. Without it the diagnostic exists only
// on the stored step: a landing gate that fails its checks would send the run
// back to the implementer with nothing saying which checks failed.
func (s *runState) recordFailure(step protocol.RunStep) {
detail := strings.TrimSpace(step.Diagnostic)
if len(detail) > maxFailureDetail {
detail = strings.ToValidUTF8(detail[:maxFailureDetail], "") + "\n[truncated]"
}
output := "# outcome\n\nerror\n\n# handoff\n\nThe " + step.Stage +
" stage failed and sent the run back. Fix the cause below; keep the work already done.\n\n~~~\n" +
detail + "\n~~~\n"
s.turns = append(s.turns, turn.MemoryEntry{
Stage: step.Stage,
Visit: s.visits[step.Stage],
Outcome: step.Outcome,
Output: output,
})
s.lastOutput = output
}
// publicFileSet lists regular files below the run's public directory as paths
// relative to the run directory, retaining the "public/" prefix.
func publicFileSet(runRoot string) map[string]bool {
files := map[string]bool{}
public := filepath.Join(runRoot, "public")
_ = filepath.WalkDir(public, func(name string, entry fs.DirEntry, err error) error {
if err != nil {
return nil
}
if entry.IsDir() || entry.Type()&fs.ModeSymlink != 0 || !entry.Type().IsRegular() {
return nil
}
relative, err := filepath.Rel(runRoot, name)
if err != nil || strings.HasPrefix(relative, ".."+string(os.PathSeparator)) {
return nil
}
files[filepath.ToSlash(relative)] = true
return nil
})
return files
}
// addedPublicFiles returns the deterministic sorted difference between two
// public file snapshots.
func addedPublicFiles(before, after map[string]bool) []string {
added := make([]string, 0, len(after))
for name := range after {
if !before[name] {
added = append(added, name)
}
}
sort.Strings(added)
return added
}
// publicArtifacts returns the public/ entries in a persisted step inventory that
// have not been seen in an earlier turn. scanArtifacts records the run-level
// public directory for every model visit, so the persisted artifacts are enough
// to rebuild each turn's system-written file list after a restart or retry
// without reading the filesystem.
func publicArtifacts(artifacts []protocol.StageArtifact, seen map[string]bool) []string {
added := []string{}
for _, artifact := range artifacts {
if !strings.HasPrefix(artifact.Name, "public/") || seen[artifact.Name] {
continue
}
seen[artifact.Name] = true
added = append(added, artifact.Name)
}
sort.Strings(added)
return added
}
// markPublicFiles filters names already attributed to an earlier turn. A prefix
// record carries the exact additions of its turn, so retries do not have to
// guess from a source inventory that may already include carried files.
func markPublicFiles(names []string, seen map[string]bool) []string {
added := make([]string, 0, len(names))
for _, name := range names {
if !strings.HasPrefix(name, "public/") || seen[name] {
continue
}
seen[name] = true
added = append(added, name)
}
sort.Strings(added)
return added
}
// retryPublicBySequence attributes public files to completed steps of the
// source attempt by first appearance. A failed step still advances the seen set
// so files it wrote are not later attributed to an unrelated completed turn,
// matching the live before/after snapshot.
func retryPublicBySequence(detail *protocol.RunDetail, failedSequence int) map[int][]string {
result := map[int][]string{}
if detail == nil {
return result
}
seen := map[string]bool{}
for _, step := range detail.Steps {
if step.Sequence >= failedSequence {
break
}
added := publicArtifacts(step.Artifacts, seen)
if step.Status == "completed" {
result[step.Sequence] = added
}
}
return result
}
// recordPublicFiles returns the public files a retried prefix turn added. New
// checkpoints carry the list directly; legacy checkpoints fall back to the
// source step's persisted inventory.
func recordPublicFiles(record retryPrefixRecord, detail *protocol.RunDetail, seen map[string]bool) []string {
if record.publicKnown {
return markPublicFiles(record.public, seen)
}
return publicArtifacts(stepArtifacts(detail, record.sequence), seen)
}
// carryRetryPublic copies the completed prefix's recorded public files into the
// retried run's artifact root. A retry is a new run with a new root, so without
// this the memory lists files the new visit cannot read. The copy is limited to
// files attributed to completed prefix turns; files written by a failed visit
// are not carried. The source is opened through os.Root and each name is
// re-checked so a symlink cannot escape either artifact root.
func (r *runner) carryRetryPublic(targetID, sourceID int64, failedSequence int, handoff map[string]any, source *protocol.RunDetail) error {
files := retryPublicCarryFiles(source, failedSequence, handoff)
if len(files) == 0 {
return nil
}
sourceRootPath := filepath.Join(r.artifactsRoot, strconv.FormatInt(sourceID, 10))
targetRootPath := filepath.Join(r.artifactsRoot, strconv.FormatInt(targetID, 10))
if err := os.MkdirAll(targetRootPath, 0o700); err != nil {
return fmt.Errorf("carry retry public: create %s: %w", targetRootPath, err)
}
sourceRoot, err := os.OpenRoot(sourceRootPath)
if err != nil {
return fmt.Errorf("carry retry public: open %s: %w", sourceRootPath, err)
}
defer sourceRoot.Close()
targetRoot, err := os.OpenRoot(targetRootPath)
if err != nil {
return fmt.Errorf("carry retry public: open %s: %w", targetRootPath, err)
}
defer targetRoot.Close()
for _, name := range files {
if path.Clean(name) != name || path.IsAbs(name) || !strings.HasPrefix(name, "public/") {
return fmt.Errorf("carry retry public: invalid recorded name %q", name)
}
info, statErr := sourceRoot.Lstat(name)
if statErr != nil {
if errors.Is(statErr, fs.ErrNotExist) {
continue
}
return fmt.Errorf("carry retry public: read %s: %w", name, statErr)
}
if info.Mode()&os.ModeSymlink != 0 || !info.Mode().IsRegular() {
continue
}
src, openErr := sourceRoot.OpenFile(name, os.O_RDONLY|syscall.O_NOFOLLOW, 0)
if openErr != nil {
return fmt.Errorf("carry retry public: open %s: %w", name, openErr)
}
if err := targetRoot.MkdirAll(path.Dir(name), 0o700); err != nil {
src.Close()
return fmt.Errorf("carry retry public: create %s: %w", path.Dir(name), err)
}
dst, createErr := targetRoot.OpenFile(name, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o600)
if createErr != nil {
src.Close()
return fmt.Errorf("carry retry public: create %s: %w", name, createErr)
}
_, copyErr := io.Copy(dst, src)
closeErr := dst.Close()
src.Close()
if copyErr != nil {
return fmt.Errorf("carry retry public: copy %s: %w", name, copyErr)
}
if closeErr != nil {
return fmt.Errorf("carry retry public: write %s: %w", name, closeErr)
}
}
return nil
}
// retryPublicCarryFiles is the union of the public files recorded for the
// retried prefix. Inherited records already carry their exact additions; an
// immediate record without one falls back to first-appearance attribution from
// the source attempt.
func retryPublicCarryFiles(detail *protocol.RunDetail, failedSequence int, handoff map[string]any) []string {
records := retryPrefixRecords(handoff["prefix_provenance"])
bySequence := retryPublicBySequence(detail, failedSequence)
files := map[string]bool{}
inherited := map[string]bool{}
for _, record := range records {
if !record.publicKnown {
continue
}
for _, name := range record.public {
if strings.HasPrefix(name, "public/") {
files[name] = true
inherited[name] = true
}
}
}
if len(records) == 0 {
for _, names := range bySequence {
for _, name := range names {
files[name] = true
}
}
} else {
for _, record := range records {
if record.publicKnown {
continue
}
for _, name := range bySequence[record.sequence] {
if !inherited[name] {
files[name] = true
}
}
}
}
result := make([]string, 0, len(files))
for name := range files {
result = append(result, name)
}
sort.Strings(result)
return result
}
// stepArtifacts finds one persisted visit's artifact inventory by sequence.
func stepArtifacts(detail *protocol.RunDetail, sequence int) []protocol.StageArtifact {
if detail == nil {
return nil
}
for _, step := range detail.Steps {
if step.Sequence == sequence {
return step.Artifacts
}
}
return nil
}
// ensureTurnOutput writes a best-effort output.md for a completed model visit
// that did not go through the model prompt path (for example the runner's
// execute fixture). The real model path writes output.md from the final
// message before the step is recorded.
func ensureTurnOutput(paths turn.Paths, result stage.Result) {
if paths.OutputPath == "" {
return
}
if _, err := os.Stat(paths.OutputPath); err == nil {
return
}
var b strings.Builder
b.WriteString("# outcome\n")
b.WriteString(result.Outcome)
b.WriteString("\n\n")
b.WriteString(turnOutput(result.Output))
_ = os.WriteFile(paths.OutputPath, []byte(b.String()), 0o600)
}