538143554c
Every stage used to be told to read memory.md, the full output of every earlier turn (19 KB average, 92 KB max), and paid for it on each model call. memory.md is now essential memory, hard-capped at 4 KB and inlined in the prompt: the task, the previous stage's # handoff section (or its truncated output head), a one-line-per-stage outcome ledger and the server-captured changed files. history.md keeps the old full memory on disk; read-capable stages are pointed at it and input.md for on-demand detail, and it is never inlined. The prompt asks each stage for a short # handoff section.
393 lines
13 KiB
Go
393 lines
13 KiB
Go
package server
|
|
|
|
import (
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"io/fs"
|
|
"os"
|
|
"path"
|
|
"path/filepath"
|
|
"sort"
|
|
"strconv"
|
|
"strings"
|
|
"syscall"
|
|
|
|
"git.a-wan.net/alan/pipelines/internal/protocol"
|
|
"git.a-wan.net/alan/pipelines/internal/stage"
|
|
"git.a-wan.net/alan/pipelines/internal/turn"
|
|
)
|
|
|
|
// turnRoot is the run-level artifact directory shared by every stage turn. The
|
|
// per-visit directories live under <run>/stages/ and public/ is run-level.
|
|
func (r *runner) turnRoot(runID int64) string {
|
|
return filepath.Join(r.artifactsRoot, strconv.FormatInt(runID, 10))
|
|
}
|
|
|
|
// turnPaths returns the deterministic layout for one visit. The zero-padded
|
|
// sequence is part of the directory name, and a retry is admitted as a new run
|
|
// with its own artifact root, so a later visit never reuses an earlier turn's
|
|
// files.
|
|
func (r *runner) turnPaths(runID int64, sequence int, stageKey string) turn.Paths {
|
|
return turn.Layout(r.turnRoot(runID), sequence, stageKey)
|
|
}
|
|
|
|
// stageTurnPaths projects the turn layout onto the runtime-only paths a model
|
|
// stage receives.
|
|
func stageTurnPaths(paths turn.Paths) stage.TurnPaths {
|
|
return stage.TurnPaths{
|
|
Dir: paths.TurnDir,
|
|
Input: paths.InputPath,
|
|
Memory: paths.MemoryPath,
|
|
History: paths.HistoryPath,
|
|
Output: paths.OutputPath,
|
|
Private: paths.PrivateDir,
|
|
Public: paths.PublicDir,
|
|
}
|
|
}
|
|
|
|
// prepareTurn creates the visit layout and writes the model's input.md,
|
|
// memory.md (essential memory, capped at turn.EssentialCap) and history.md (every
|
|
// completed turn in full) before the visit starts. A failure here is a run failure: the file
|
|
// protocol is the stage contract, not best-effort evidence.
|
|
//
|
|
// retryInput is the complete merged input for a selected-stage retry and nil for
|
|
// an ordinary visit. A retry override is persisted on the step, but the model
|
|
// reads input.md rather than the step row, so the same values must be written
|
|
// into the file or the operator's correction is invisible to the model.
|
|
func (r *runner) prepareTurn(paths turn.Paths, state *runState, retryInput stage.Values) error {
|
|
if err := paths.Ensure(); err != nil {
|
|
return err
|
|
}
|
|
input := turnInputWithRetry(state.turnInput(), retryInput)
|
|
history := turn.RenderHistory(state.turns)
|
|
memory := turn.RenderEssential(turn.Essential{
|
|
Task: state.prompt,
|
|
StageKey: paths.StageKey,
|
|
Entries: state.turns,
|
|
ChangedFiles: state.capturedList(),
|
|
InputPath: paths.InputPath,
|
|
HistoryPath: paths.HistoryPath,
|
|
})
|
|
for _, file := range []struct{ path, content string }{
|
|
{paths.InputPath, input},
|
|
{paths.MemoryPath, memory},
|
|
{paths.HistoryPath, history},
|
|
} {
|
|
if err := os.WriteFile(file.path, []byte(file.content), 0o600); err != nil {
|
|
return fmt.Errorf("write %s: %w", file.path, err)
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// turnInputWithRetry appends the merged retry input to the ordinary turn input.
|
|
// It is rendered as a fenced JSON object because the values are operator data,
|
|
// not prose; encoding/json sorts map keys so the file is deterministic.
|
|
func turnInputWithRetry(base string, retryInput stage.Values) string {
|
|
if len(retryInput) == 0 {
|
|
return base
|
|
}
|
|
encoded, err := json.MarshalIndent(retryInput, "", " ")
|
|
if err != nil {
|
|
// A retry input that cannot be rendered is still shown as a visible
|
|
// section; the stage can choose its error outcome rather than silently
|
|
// receiving a correction it cannot see.
|
|
return base + "\n\n# Retry input\n\nThe retry input could not be rendered.\n"
|
|
}
|
|
if !strings.HasSuffix(base, "\n") {
|
|
base += "\n"
|
|
}
|
|
return base + "\n# Retry input\n\nThis retry was given this complete input:\n\n```json\n" + string(encoded) + "\n```\n"
|
|
}
|
|
|
|
// turnInput is the run prompt on the first visit; later visits carry the run
|
|
// prompt and the previous visit's output.
|
|
func (s *runState) turnInput() string {
|
|
if len(s.turns) == 0 && strings.TrimSpace(s.lastOutput) == "" {
|
|
return s.prompt
|
|
}
|
|
if strings.TrimSpace(s.lastOutput) == "" {
|
|
return turn.RenderInput(s.prompt, "_(no output)_\n")
|
|
}
|
|
return turn.RenderInput(s.prompt, s.lastOutput)
|
|
}
|
|
|
|
// turnOutput renders a completed step for later memory. Model stages store their
|
|
// body under the single markdown key; human and other stages keep a small
|
|
// fenced rendering of their recorded evidence.
|
|
func turnOutput(output stage.Values) string {
|
|
if markdown, ok := output["markdown"].(string); ok {
|
|
return markdown
|
|
}
|
|
if len(output) == 0 {
|
|
return ""
|
|
}
|
|
data, err := json.MarshalIndent(output, "", " ")
|
|
if err != nil {
|
|
return ""
|
|
}
|
|
return "```json\n" + string(data) + "\n```\n"
|
|
}
|
|
|
|
// recordTurn appends one completed visit to the run's accumulated memory. The
|
|
// public file list is a system-written addition computed from the filesystem,
|
|
// never a model claim.
|
|
func (s *runState) recordTurn(step protocol.RunStep, publicFiles []string) {
|
|
output := turnOutput(step.Output)
|
|
s.turns = append(s.turns, turn.MemoryEntry{
|
|
Stage: step.Stage,
|
|
Visit: s.visits[step.Stage],
|
|
Outcome: step.Outcome,
|
|
Output: output,
|
|
PublicFiles: publicFiles,
|
|
})
|
|
s.lastOutput = output
|
|
}
|
|
|
|
// publicFileSet lists regular files below the run's public directory as paths
|
|
// relative to the run directory, retaining the "public/" prefix.
|
|
func publicFileSet(runRoot string) map[string]bool {
|
|
files := map[string]bool{}
|
|
public := filepath.Join(runRoot, "public")
|
|
_ = filepath.WalkDir(public, func(name string, entry fs.DirEntry, err error) error {
|
|
if err != nil {
|
|
return nil
|
|
}
|
|
if entry.IsDir() || entry.Type()&fs.ModeSymlink != 0 || !entry.Type().IsRegular() {
|
|
return nil
|
|
}
|
|
relative, err := filepath.Rel(runRoot, name)
|
|
if err != nil || strings.HasPrefix(relative, ".."+string(os.PathSeparator)) {
|
|
return nil
|
|
}
|
|
files[filepath.ToSlash(relative)] = true
|
|
return nil
|
|
})
|
|
return files
|
|
}
|
|
|
|
// addedPublicFiles returns the deterministic sorted difference between two
|
|
// public file snapshots.
|
|
func addedPublicFiles(before, after map[string]bool) []string {
|
|
added := make([]string, 0, len(after))
|
|
for name := range after {
|
|
if !before[name] {
|
|
added = append(added, name)
|
|
}
|
|
}
|
|
sort.Strings(added)
|
|
return added
|
|
}
|
|
|
|
// publicArtifacts returns the public/ entries in a persisted step inventory that
|
|
// have not been seen in an earlier turn. scanArtifacts records the run-level
|
|
// public directory for every model visit, so the persisted artifacts are enough
|
|
// to rebuild each turn's system-written file list after a restart or retry
|
|
// without reading the filesystem.
|
|
func publicArtifacts(artifacts []protocol.StageArtifact, seen map[string]bool) []string {
|
|
added := []string{}
|
|
for _, artifact := range artifacts {
|
|
if !strings.HasPrefix(artifact.Name, "public/") || seen[artifact.Name] {
|
|
continue
|
|
}
|
|
seen[artifact.Name] = true
|
|
added = append(added, artifact.Name)
|
|
}
|
|
sort.Strings(added)
|
|
return added
|
|
}
|
|
|
|
// markPublicFiles filters names already attributed to an earlier turn. A prefix
|
|
// record carries the exact additions of its turn, so retries do not have to
|
|
// guess from a source inventory that may already include carried files.
|
|
func markPublicFiles(names []string, seen map[string]bool) []string {
|
|
added := make([]string, 0, len(names))
|
|
for _, name := range names {
|
|
if !strings.HasPrefix(name, "public/") || seen[name] {
|
|
continue
|
|
}
|
|
seen[name] = true
|
|
added = append(added, name)
|
|
}
|
|
sort.Strings(added)
|
|
return added
|
|
}
|
|
|
|
// retryPublicBySequence attributes public files to completed steps of the
|
|
// source attempt by first appearance. A failed step still advances the seen set
|
|
// so files it wrote are not later attributed to an unrelated completed turn,
|
|
// matching the live before/after snapshot.
|
|
func retryPublicBySequence(detail *protocol.RunDetail, failedSequence int) map[int][]string {
|
|
result := map[int][]string{}
|
|
if detail == nil {
|
|
return result
|
|
}
|
|
seen := map[string]bool{}
|
|
for _, step := range detail.Steps {
|
|
if step.Sequence >= failedSequence {
|
|
break
|
|
}
|
|
added := publicArtifacts(step.Artifacts, seen)
|
|
if step.Status == "completed" {
|
|
result[step.Sequence] = added
|
|
}
|
|
}
|
|
return result
|
|
}
|
|
|
|
// recordPublicFiles returns the public files a retried prefix turn added. New
|
|
// checkpoints carry the list directly; legacy checkpoints fall back to the
|
|
// source step's persisted inventory.
|
|
func recordPublicFiles(record retryPrefixRecord, detail *protocol.RunDetail, seen map[string]bool) []string {
|
|
if record.publicKnown {
|
|
return markPublicFiles(record.public, seen)
|
|
}
|
|
return publicArtifacts(stepArtifacts(detail, record.sequence), seen)
|
|
}
|
|
|
|
// carryRetryPublic copies the completed prefix's recorded public files into the
|
|
// retried run's artifact root. A retry is a new run with a new root, so without
|
|
// this the memory lists files the new visit cannot read. The copy is limited to
|
|
// files attributed to completed prefix turns; files written by a failed visit
|
|
// are not carried. The source is opened through os.Root and each name is
|
|
// re-checked so a symlink cannot escape either artifact root.
|
|
func (r *runner) carryRetryPublic(targetID, sourceID int64, failedSequence int, handoff map[string]any, source *protocol.RunDetail) error {
|
|
files := retryPublicCarryFiles(source, failedSequence, handoff)
|
|
if len(files) == 0 {
|
|
return nil
|
|
}
|
|
sourceRootPath := filepath.Join(r.artifactsRoot, strconv.FormatInt(sourceID, 10))
|
|
targetRootPath := filepath.Join(r.artifactsRoot, strconv.FormatInt(targetID, 10))
|
|
if err := os.MkdirAll(targetRootPath, 0o700); err != nil {
|
|
return fmt.Errorf("carry retry public: create %s: %w", targetRootPath, err)
|
|
}
|
|
sourceRoot, err := os.OpenRoot(sourceRootPath)
|
|
if err != nil {
|
|
return fmt.Errorf("carry retry public: open %s: %w", sourceRootPath, err)
|
|
}
|
|
defer sourceRoot.Close()
|
|
targetRoot, err := os.OpenRoot(targetRootPath)
|
|
if err != nil {
|
|
return fmt.Errorf("carry retry public: open %s: %w", targetRootPath, err)
|
|
}
|
|
defer targetRoot.Close()
|
|
for _, name := range files {
|
|
if path.Clean(name) != name || path.IsAbs(name) || !strings.HasPrefix(name, "public/") {
|
|
return fmt.Errorf("carry retry public: invalid recorded name %q", name)
|
|
}
|
|
info, statErr := sourceRoot.Lstat(name)
|
|
if statErr != nil {
|
|
if errors.Is(statErr, fs.ErrNotExist) {
|
|
continue
|
|
}
|
|
return fmt.Errorf("carry retry public: read %s: %w", name, statErr)
|
|
}
|
|
if info.Mode()&os.ModeSymlink != 0 || !info.Mode().IsRegular() {
|
|
continue
|
|
}
|
|
src, openErr := sourceRoot.OpenFile(name, os.O_RDONLY|syscall.O_NOFOLLOW, 0)
|
|
if openErr != nil {
|
|
return fmt.Errorf("carry retry public: open %s: %w", name, openErr)
|
|
}
|
|
if err := targetRoot.MkdirAll(path.Dir(name), 0o700); err != nil {
|
|
src.Close()
|
|
return fmt.Errorf("carry retry public: create %s: %w", path.Dir(name), err)
|
|
}
|
|
dst, createErr := targetRoot.OpenFile(name, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o600)
|
|
if createErr != nil {
|
|
src.Close()
|
|
return fmt.Errorf("carry retry public: create %s: %w", name, createErr)
|
|
}
|
|
_, copyErr := io.Copy(dst, src)
|
|
closeErr := dst.Close()
|
|
src.Close()
|
|
if copyErr != nil {
|
|
return fmt.Errorf("carry retry public: copy %s: %w", name, copyErr)
|
|
}
|
|
if closeErr != nil {
|
|
return fmt.Errorf("carry retry public: write %s: %w", name, closeErr)
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// retryPublicCarryFiles is the union of the public files recorded for the
|
|
// retried prefix. Inherited records already carry their exact additions; an
|
|
// immediate record without one falls back to first-appearance attribution from
|
|
// the source attempt.
|
|
func retryPublicCarryFiles(detail *protocol.RunDetail, failedSequence int, handoff map[string]any) []string {
|
|
records := retryPrefixRecords(handoff["prefix_provenance"])
|
|
bySequence := retryPublicBySequence(detail, failedSequence)
|
|
files := map[string]bool{}
|
|
inherited := map[string]bool{}
|
|
for _, record := range records {
|
|
if !record.publicKnown {
|
|
continue
|
|
}
|
|
for _, name := range record.public {
|
|
if strings.HasPrefix(name, "public/") {
|
|
files[name] = true
|
|
inherited[name] = true
|
|
}
|
|
}
|
|
}
|
|
if len(records) == 0 {
|
|
for _, names := range bySequence {
|
|
for _, name := range names {
|
|
files[name] = true
|
|
}
|
|
}
|
|
} else {
|
|
for _, record := range records {
|
|
if record.publicKnown {
|
|
continue
|
|
}
|
|
for _, name := range bySequence[record.sequence] {
|
|
if !inherited[name] {
|
|
files[name] = true
|
|
}
|
|
}
|
|
}
|
|
}
|
|
result := make([]string, 0, len(files))
|
|
for name := range files {
|
|
result = append(result, name)
|
|
}
|
|
sort.Strings(result)
|
|
return result
|
|
}
|
|
|
|
// stepArtifacts finds one persisted visit's artifact inventory by sequence.
|
|
func stepArtifacts(detail *protocol.RunDetail, sequence int) []protocol.StageArtifact {
|
|
if detail == nil {
|
|
return nil
|
|
}
|
|
for _, step := range detail.Steps {
|
|
if step.Sequence == sequence {
|
|
return step.Artifacts
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// ensureTurnOutput writes a best-effort output.md for a completed model visit
|
|
// that did not go through the model prompt path (for example the runner's
|
|
// execute fixture). The real model path writes output.md from the final
|
|
// message before the step is recorded.
|
|
func ensureTurnOutput(paths turn.Paths, result stage.Result) {
|
|
if paths.OutputPath == "" {
|
|
return
|
|
}
|
|
if _, err := os.Stat(paths.OutputPath); err == nil {
|
|
return
|
|
}
|
|
var b strings.Builder
|
|
b.WriteString("# outcome\n")
|
|
b.WriteString(result.Outcome)
|
|
b.WriteString("\n\n")
|
|
b.WriteString(turnOutput(result.Output))
|
|
_ = os.WriteFile(paths.OutputPath, []byte(b.String()), 0o600)
|
|
}
|