c2b992542b
A failed visit was recorded on its step only, so a landing gate that sent the run back to implement gave the implementer nothing about which checks failed. The diagnostic (bounded) is now the next stage's previous output and a memory entry.
418 lines
14 KiB
Go
418 lines
14 KiB
Go
package server
|
|
|
|
import (
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"io/fs"
|
|
"os"
|
|
"path"
|
|
"path/filepath"
|
|
"sort"
|
|
"strconv"
|
|
"strings"
|
|
"syscall"
|
|
|
|
"git.a-wan.net/alan/pipelines/internal/protocol"
|
|
"git.a-wan.net/alan/pipelines/internal/stage"
|
|
"git.a-wan.net/alan/pipelines/internal/turn"
|
|
)
|
|
|
|
// turnRoot is the run-level artifact directory shared by every stage turn. The
|
|
// per-visit directories live under <run>/stages/ and public/ is run-level.
|
|
func (r *runner) turnRoot(runID int64) string {
|
|
return filepath.Join(r.artifactsRoot, strconv.FormatInt(runID, 10))
|
|
}
|
|
|
|
// turnPaths returns the deterministic layout for one visit. The zero-padded
|
|
// sequence is part of the directory name, and a retry is admitted as a new run
|
|
// with its own artifact root, so a later visit never reuses an earlier turn's
|
|
// files.
|
|
func (r *runner) turnPaths(runID int64, sequence int, stageKey string) turn.Paths {
|
|
return turn.Layout(r.turnRoot(runID), sequence, stageKey)
|
|
}
|
|
|
|
// stageTurnPaths projects the turn layout onto the runtime-only paths a model
|
|
// stage receives.
|
|
func stageTurnPaths(paths turn.Paths) stage.TurnPaths {
|
|
return stage.TurnPaths{
|
|
Dir: paths.TurnDir,
|
|
Input: paths.InputPath,
|
|
Memory: paths.MemoryPath,
|
|
History: paths.HistoryPath,
|
|
Output: paths.OutputPath,
|
|
Private: paths.PrivateDir,
|
|
Public: paths.PublicDir,
|
|
}
|
|
}
|
|
|
|
// prepareTurn creates the visit layout and writes the model's input.md,
|
|
// memory.md (essential memory, capped at turn.EssentialCap) and history.md (every
|
|
// completed turn in full) before the visit starts. A failure here is a run failure: the file
|
|
// protocol is the stage contract, not best-effort evidence.
|
|
//
|
|
// retryInput is the complete merged input for a selected-stage retry and nil for
|
|
// an ordinary visit. A retry override is persisted on the step, but the model
|
|
// reads input.md rather than the step row, so the same values must be written
|
|
// into the file or the operator's correction is invisible to the model.
|
|
func (r *runner) prepareTurn(paths turn.Paths, state *runState, retryInput stage.Values) error {
|
|
if err := paths.Ensure(); err != nil {
|
|
return err
|
|
}
|
|
input := turnInputWithRetry(state.turnInput(), retryInput)
|
|
history := turn.RenderHistory(state.turns)
|
|
memory := turn.RenderEssential(turn.Essential{
|
|
Task: state.prompt,
|
|
StageKey: paths.StageKey,
|
|
Entries: state.turns,
|
|
ChangedFiles: state.capturedList(),
|
|
InputPath: paths.InputPath,
|
|
HistoryPath: paths.HistoryPath,
|
|
})
|
|
for _, file := range []struct{ path, content string }{
|
|
{paths.InputPath, input},
|
|
{paths.MemoryPath, memory},
|
|
{paths.HistoryPath, history},
|
|
} {
|
|
if err := os.WriteFile(file.path, []byte(file.content), 0o600); err != nil {
|
|
return fmt.Errorf("write %s: %w", file.path, err)
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// turnInputWithRetry appends the merged retry input to the ordinary turn input.
|
|
// It is rendered as a fenced JSON object because the values are operator data,
|
|
// not prose; encoding/json sorts map keys so the file is deterministic.
|
|
func turnInputWithRetry(base string, retryInput stage.Values) string {
|
|
if len(retryInput) == 0 {
|
|
return base
|
|
}
|
|
encoded, err := json.MarshalIndent(retryInput, "", " ")
|
|
if err != nil {
|
|
// A retry input that cannot be rendered is still shown as a visible
|
|
// section; the stage can choose its error outcome rather than silently
|
|
// receiving a correction it cannot see.
|
|
return base + "\n\n# Retry input\n\nThe retry input could not be rendered.\n"
|
|
}
|
|
if !strings.HasSuffix(base, "\n") {
|
|
base += "\n"
|
|
}
|
|
return base + "\n# Retry input\n\nThis retry was given this complete input:\n\n```json\n" + string(encoded) + "\n```\n"
|
|
}
|
|
|
|
// turnInput is the run prompt on the first visit; later visits carry the run
|
|
// prompt and the previous visit's output.
|
|
func (s *runState) turnInput() string {
|
|
if len(s.turns) == 0 && strings.TrimSpace(s.lastOutput) == "" {
|
|
return s.prompt
|
|
}
|
|
if strings.TrimSpace(s.lastOutput) == "" {
|
|
return turn.RenderInput(s.prompt, "_(no output)_\n")
|
|
}
|
|
return turn.RenderInput(s.prompt, s.lastOutput)
|
|
}
|
|
|
|
// turnOutput renders a completed step for later memory. Model stages store their
|
|
// body under the single markdown key; human and other stages keep a small
|
|
// fenced rendering of their recorded evidence.
|
|
func turnOutput(output stage.Values) string {
|
|
if markdown, ok := output["markdown"].(string); ok {
|
|
return markdown
|
|
}
|
|
if len(output) == 0 {
|
|
return ""
|
|
}
|
|
data, err := json.MarshalIndent(output, "", " ")
|
|
if err != nil {
|
|
return ""
|
|
}
|
|
return "```json\n" + string(data) + "\n```\n"
|
|
}
|
|
|
|
// recordTurn appends one completed visit to the run's accumulated memory. The
|
|
// public file list is a system-written addition computed from the filesystem,
|
|
// never a model claim.
|
|
func (s *runState) recordTurn(step protocol.RunStep, publicFiles []string) {
|
|
output := turnOutput(step.Output)
|
|
s.turns = append(s.turns, turn.MemoryEntry{
|
|
Stage: step.Stage,
|
|
Visit: s.visits[step.Stage],
|
|
Outcome: step.Outcome,
|
|
Output: output,
|
|
PublicFiles: publicFiles,
|
|
})
|
|
s.lastOutput = output
|
|
}
|
|
|
|
// maxFailureDetail bounds the failure text carried to the next stage. A
|
|
// delivery check's quiet output lists its failures first, so the head is kept.
|
|
const maxFailureDetail = 6000
|
|
|
|
// recordFailure makes a failed visit that routes onward (an error edge to
|
|
// another stage) visible to that stage. Without it the diagnostic exists only
|
|
// on the stored step: a landing gate that fails its checks would send the run
|
|
// back to the implementer with nothing saying which checks failed.
|
|
func (s *runState) recordFailure(step protocol.RunStep) {
|
|
detail := strings.TrimSpace(step.Diagnostic)
|
|
if len(detail) > maxFailureDetail {
|
|
detail = strings.ToValidUTF8(detail[:maxFailureDetail], "") + "\n[truncated]"
|
|
}
|
|
output := "# outcome\n\nerror\n\n# handoff\n\nThe " + step.Stage +
|
|
" stage failed and sent the run back. Fix the cause below; keep the work already done.\n\n~~~\n" +
|
|
detail + "\n~~~\n"
|
|
s.turns = append(s.turns, turn.MemoryEntry{
|
|
Stage: step.Stage,
|
|
Visit: s.visits[step.Stage],
|
|
Outcome: step.Outcome,
|
|
Output: output,
|
|
})
|
|
s.lastOutput = output
|
|
}
|
|
|
|
// publicFileSet lists regular files below the run's public directory as paths
|
|
// relative to the run directory, retaining the "public/" prefix.
|
|
func publicFileSet(runRoot string) map[string]bool {
|
|
files := map[string]bool{}
|
|
public := filepath.Join(runRoot, "public")
|
|
_ = filepath.WalkDir(public, func(name string, entry fs.DirEntry, err error) error {
|
|
if err != nil {
|
|
return nil
|
|
}
|
|
if entry.IsDir() || entry.Type()&fs.ModeSymlink != 0 || !entry.Type().IsRegular() {
|
|
return nil
|
|
}
|
|
relative, err := filepath.Rel(runRoot, name)
|
|
if err != nil || strings.HasPrefix(relative, ".."+string(os.PathSeparator)) {
|
|
return nil
|
|
}
|
|
files[filepath.ToSlash(relative)] = true
|
|
return nil
|
|
})
|
|
return files
|
|
}
|
|
|
|
// addedPublicFiles returns the deterministic sorted difference between two
|
|
// public file snapshots.
|
|
func addedPublicFiles(before, after map[string]bool) []string {
|
|
added := make([]string, 0, len(after))
|
|
for name := range after {
|
|
if !before[name] {
|
|
added = append(added, name)
|
|
}
|
|
}
|
|
sort.Strings(added)
|
|
return added
|
|
}
|
|
|
|
// publicArtifacts returns the public/ entries in a persisted step inventory that
|
|
// have not been seen in an earlier turn. scanArtifacts records the run-level
|
|
// public directory for every model visit, so the persisted artifacts are enough
|
|
// to rebuild each turn's system-written file list after a restart or retry
|
|
// without reading the filesystem.
|
|
func publicArtifacts(artifacts []protocol.StageArtifact, seen map[string]bool) []string {
|
|
added := []string{}
|
|
for _, artifact := range artifacts {
|
|
if !strings.HasPrefix(artifact.Name, "public/") || seen[artifact.Name] {
|
|
continue
|
|
}
|
|
seen[artifact.Name] = true
|
|
added = append(added, artifact.Name)
|
|
}
|
|
sort.Strings(added)
|
|
return added
|
|
}
|
|
|
|
// markPublicFiles filters names already attributed to an earlier turn. A prefix
|
|
// record carries the exact additions of its turn, so retries do not have to
|
|
// guess from a source inventory that may already include carried files.
|
|
func markPublicFiles(names []string, seen map[string]bool) []string {
|
|
added := make([]string, 0, len(names))
|
|
for _, name := range names {
|
|
if !strings.HasPrefix(name, "public/") || seen[name] {
|
|
continue
|
|
}
|
|
seen[name] = true
|
|
added = append(added, name)
|
|
}
|
|
sort.Strings(added)
|
|
return added
|
|
}
|
|
|
|
// retryPublicBySequence attributes public files to completed steps of the
|
|
// source attempt by first appearance. A failed step still advances the seen set
|
|
// so files it wrote are not later attributed to an unrelated completed turn,
|
|
// matching the live before/after snapshot.
|
|
func retryPublicBySequence(detail *protocol.RunDetail, failedSequence int) map[int][]string {
|
|
result := map[int][]string{}
|
|
if detail == nil {
|
|
return result
|
|
}
|
|
seen := map[string]bool{}
|
|
for _, step := range detail.Steps {
|
|
if step.Sequence >= failedSequence {
|
|
break
|
|
}
|
|
added := publicArtifacts(step.Artifacts, seen)
|
|
if step.Status == "completed" {
|
|
result[step.Sequence] = added
|
|
}
|
|
}
|
|
return result
|
|
}
|
|
|
|
// recordPublicFiles returns the public files a retried prefix turn added. New
|
|
// checkpoints carry the list directly; legacy checkpoints fall back to the
|
|
// source step's persisted inventory.
|
|
func recordPublicFiles(record retryPrefixRecord, detail *protocol.RunDetail, seen map[string]bool) []string {
|
|
if record.publicKnown {
|
|
return markPublicFiles(record.public, seen)
|
|
}
|
|
return publicArtifacts(stepArtifacts(detail, record.sequence), seen)
|
|
}
|
|
|
|
// carryRetryPublic copies the completed prefix's recorded public files into the
|
|
// retried run's artifact root. A retry is a new run with a new root, so without
|
|
// this the memory lists files the new visit cannot read. The copy is limited to
|
|
// files attributed to completed prefix turns; files written by a failed visit
|
|
// are not carried. The source is opened through os.Root and each name is
|
|
// re-checked so a symlink cannot escape either artifact root.
|
|
func (r *runner) carryRetryPublic(targetID, sourceID int64, failedSequence int, handoff map[string]any, source *protocol.RunDetail) error {
|
|
files := retryPublicCarryFiles(source, failedSequence, handoff)
|
|
if len(files) == 0 {
|
|
return nil
|
|
}
|
|
sourceRootPath := filepath.Join(r.artifactsRoot, strconv.FormatInt(sourceID, 10))
|
|
targetRootPath := filepath.Join(r.artifactsRoot, strconv.FormatInt(targetID, 10))
|
|
if err := os.MkdirAll(targetRootPath, 0o700); err != nil {
|
|
return fmt.Errorf("carry retry public: create %s: %w", targetRootPath, err)
|
|
}
|
|
sourceRoot, err := os.OpenRoot(sourceRootPath)
|
|
if err != nil {
|
|
return fmt.Errorf("carry retry public: open %s: %w", sourceRootPath, err)
|
|
}
|
|
defer sourceRoot.Close()
|
|
targetRoot, err := os.OpenRoot(targetRootPath)
|
|
if err != nil {
|
|
return fmt.Errorf("carry retry public: open %s: %w", targetRootPath, err)
|
|
}
|
|
defer targetRoot.Close()
|
|
for _, name := range files {
|
|
if path.Clean(name) != name || path.IsAbs(name) || !strings.HasPrefix(name, "public/") {
|
|
return fmt.Errorf("carry retry public: invalid recorded name %q", name)
|
|
}
|
|
info, statErr := sourceRoot.Lstat(name)
|
|
if statErr != nil {
|
|
if errors.Is(statErr, fs.ErrNotExist) {
|
|
continue
|
|
}
|
|
return fmt.Errorf("carry retry public: read %s: %w", name, statErr)
|
|
}
|
|
if info.Mode()&os.ModeSymlink != 0 || !info.Mode().IsRegular() {
|
|
continue
|
|
}
|
|
src, openErr := sourceRoot.OpenFile(name, os.O_RDONLY|syscall.O_NOFOLLOW, 0)
|
|
if openErr != nil {
|
|
return fmt.Errorf("carry retry public: open %s: %w", name, openErr)
|
|
}
|
|
if err := targetRoot.MkdirAll(path.Dir(name), 0o700); err != nil {
|
|
src.Close()
|
|
return fmt.Errorf("carry retry public: create %s: %w", path.Dir(name), err)
|
|
}
|
|
dst, createErr := targetRoot.OpenFile(name, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o600)
|
|
if createErr != nil {
|
|
src.Close()
|
|
return fmt.Errorf("carry retry public: create %s: %w", name, createErr)
|
|
}
|
|
_, copyErr := io.Copy(dst, src)
|
|
closeErr := dst.Close()
|
|
src.Close()
|
|
if copyErr != nil {
|
|
return fmt.Errorf("carry retry public: copy %s: %w", name, copyErr)
|
|
}
|
|
if closeErr != nil {
|
|
return fmt.Errorf("carry retry public: write %s: %w", name, closeErr)
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// retryPublicCarryFiles is the union of the public files recorded for the
|
|
// retried prefix. Inherited records already carry their exact additions; an
|
|
// immediate record without one falls back to first-appearance attribution from
|
|
// the source attempt.
|
|
func retryPublicCarryFiles(detail *protocol.RunDetail, failedSequence int, handoff map[string]any) []string {
|
|
records := retryPrefixRecords(handoff["prefix_provenance"])
|
|
bySequence := retryPublicBySequence(detail, failedSequence)
|
|
files := map[string]bool{}
|
|
inherited := map[string]bool{}
|
|
for _, record := range records {
|
|
if !record.publicKnown {
|
|
continue
|
|
}
|
|
for _, name := range record.public {
|
|
if strings.HasPrefix(name, "public/") {
|
|
files[name] = true
|
|
inherited[name] = true
|
|
}
|
|
}
|
|
}
|
|
if len(records) == 0 {
|
|
for _, names := range bySequence {
|
|
for _, name := range names {
|
|
files[name] = true
|
|
}
|
|
}
|
|
} else {
|
|
for _, record := range records {
|
|
if record.publicKnown {
|
|
continue
|
|
}
|
|
for _, name := range bySequence[record.sequence] {
|
|
if !inherited[name] {
|
|
files[name] = true
|
|
}
|
|
}
|
|
}
|
|
}
|
|
result := make([]string, 0, len(files))
|
|
for name := range files {
|
|
result = append(result, name)
|
|
}
|
|
sort.Strings(result)
|
|
return result
|
|
}
|
|
|
|
// stepArtifacts finds one persisted visit's artifact inventory by sequence.
|
|
func stepArtifacts(detail *protocol.RunDetail, sequence int) []protocol.StageArtifact {
|
|
if detail == nil {
|
|
return nil
|
|
}
|
|
for _, step := range detail.Steps {
|
|
if step.Sequence == sequence {
|
|
return step.Artifacts
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// ensureTurnOutput writes a best-effort output.md for a completed model visit
|
|
// that did not go through the model prompt path (for example the runner's
|
|
// execute fixture). The real model path writes output.md from the final
|
|
// message before the step is recorded.
|
|
func ensureTurnOutput(paths turn.Paths, result stage.Result) {
|
|
if paths.OutputPath == "" {
|
|
return
|
|
}
|
|
if _, err := os.Stat(paths.OutputPath); err == nil {
|
|
return
|
|
}
|
|
var b strings.Builder
|
|
b.WriteString("# outcome\n")
|
|
b.WriteString(result.Outcome)
|
|
b.WriteString("\n\n")
|
|
b.WriteString(turnOutput(result.Output))
|
|
_ = os.WriteFile(paths.OutputPath, []byte(b.String()), 0o600)
|
|
}
|