538143554c
Every stage used to be told to read memory.md, the full output of every earlier turn (19 KB average, 92 KB max), and paid for it on each model call. memory.md is now essential memory, hard-capped at 4 KB and inlined in the prompt: the task, the previous stage's # handoff section (or its truncated output head), a one-line-per-stage outcome ledger and the server-captured changed files. history.md keeps the old full memory on disk; read-capable stages are pointed at it and input.md for on-demand detail, and it is never inlined. The prompt asks each stage for a short # handoff section.
362 lines
14 KiB
Go
362 lines
14 KiB
Go
package server
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"os"
|
|
"path/filepath"
|
|
"strconv"
|
|
"strings"
|
|
"testing"
|
|
|
|
"git.a-wan.net/alan/pipelines/internal/pipeline"
|
|
"git.a-wan.net/alan/pipelines/internal/protocol"
|
|
"git.a-wan.net/alan/pipelines/internal/stage"
|
|
"git.a-wan.net/alan/pipelines/internal/turn"
|
|
)
|
|
|
|
// TestTurnsAccumulateMemoryAndListFilesAsArtifacts runs two real visits and
|
|
// proves the file contract end to end: input.md carries the run prompt and then
|
|
// the previous turn's output, memory.md accumulates completed turns plus the
|
|
// system-written public file list, and all three turn files are inventoried.
|
|
func TestTurnsAccumulateMemoryAndListFilesAsArtifacts(t *testing.T) {
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
defer cancel()
|
|
store, err := pipeline.Open(filepath.Join(t.TempDir(), "turns.db"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer store.Close()
|
|
if _, err := store.Submit(ctx, "turns", pipeline.Template([]string{"first", "second"}, true)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
items, err := store.List(ctx)
|
|
if err != nil || len(items) != 1 {
|
|
t.Fatalf("submit/list: %+v %v", items, err)
|
|
}
|
|
runner, err := newRunner(ctx, store)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var secondInput, secondMemory, secondHistory string
|
|
runner.execute = func(_ context.Context, def stage.Definition, _ stage.Values) (stage.Result, error) {
|
|
if def.Turn.Input == "" || def.Turn.Memory == "" || def.Turn.Output == "" || def.Turn.Private == "" || def.Turn.Public == "" {
|
|
t.Fatalf("runner did not provide the turn paths: %+v", def.Turn)
|
|
}
|
|
switch def.Key {
|
|
case "first":
|
|
if err := os.WriteFile(filepath.Join(def.Turn.Public, "shared.txt"), []byte("shared"), 0o600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := os.WriteFile(filepath.Join(def.Turn.Private, "secret.txt"), []byte("turn one only"), 0o600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return stage.Result{Outcome: "success", Output: stage.Values{"markdown": "first body\n"}}, nil
|
|
case "second":
|
|
data, err := os.ReadFile(def.Turn.Input)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
secondInput = string(data)
|
|
data, err = os.ReadFile(def.Turn.Memory)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
secondMemory = string(data)
|
|
if def.Turn.History == "" {
|
|
t.Fatalf("runner did not provide the history path: %+v", def.Turn)
|
|
}
|
|
data, err = os.ReadFile(def.Turn.History)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
secondHistory = string(data)
|
|
if _, err := os.Stat(filepath.Join(def.Turn.Private, "secret.txt")); !os.IsNotExist(err) {
|
|
t.Fatalf("a later turn received an earlier private file: %v", err)
|
|
}
|
|
if _, err := os.Stat(filepath.Join(def.Turn.Public, "shared.txt")); err != nil {
|
|
t.Fatalf("shared public artifact was not visible to the later turn: %v", err)
|
|
}
|
|
if err := os.WriteFile(filepath.Join(def.Turn.Public, "second.txt"), []byte("second"), 0o600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return stage.Result{Outcome: "success", Output: stage.Values{"markdown": "second body\n"}}, nil
|
|
}
|
|
t.Fatalf("unexpected stage %q", def.Key)
|
|
return stage.Result{}, nil
|
|
}
|
|
run, err := runner.start(protocol.Message{PipelineID: items[0].ID, Version: 1, RequestKey: "turns", Workspace: t.TempDir(), Input: map[string]any{"prompt": "original task"}})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
runner.wg.Wait()
|
|
detail, err := store.GetRun(ctx, run.ID)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if detail.Status != "succeeded" || len(detail.Steps) != 2 {
|
|
t.Fatalf("run = %q steps=%d: %+v", detail.Status, len(detail.Steps), detail.RunSummary)
|
|
}
|
|
if secondInput != "# Run task\n\noriginal task\n\n# Previous turn output\n\nfirst body\n" {
|
|
t.Fatalf("second input.md = %q, want the run task and the previous turn's output", secondInput)
|
|
}
|
|
for _, want := range []string{"## first · visit 1", "first body", "Files added to public/:\n- public/shared.txt"} {
|
|
if !strings.Contains(secondHistory, want) {
|
|
t.Fatalf("second history.md missing %q:\n%s", want, secondHistory)
|
|
}
|
|
}
|
|
for _, want := range []string{"# Essential memory", "original task", "## Previous stage: first · visit 1 · outcome success", "first body", "Added to public/: public/shared.txt", "- first · visit 1 · outcome success"} {
|
|
if !strings.Contains(secondMemory, want) {
|
|
t.Fatalf("second memory.md missing %q:\n%s", want, secondMemory)
|
|
}
|
|
}
|
|
if len(secondMemory) > turn.EssentialCap {
|
|
t.Fatalf("memory.md is %d bytes, over the %d cap", len(secondMemory), turn.EssentialCap)
|
|
}
|
|
for _, name := range []string{"input.md", "memory.md", "history.md", "output.md"} {
|
|
if _, ok := artifactNamed(detail.Steps[1].Artifacts, name); !ok {
|
|
t.Fatalf("second visit did not list %s: %+v", name, detail.Steps[1].Artifacts)
|
|
}
|
|
}
|
|
if _, ok := artifactNamed(detail.Steps[0].Artifacts, "private/secret.txt"); !ok {
|
|
t.Fatalf("first visit did not list its private artifact: %+v", detail.Steps[0].Artifacts)
|
|
}
|
|
if _, ok := artifactNamed(detail.Steps[0].Artifacts, "public/shared.txt"); !ok {
|
|
t.Fatalf("first visit did not list the shared public artifact: %+v", detail.Steps[0].Artifacts)
|
|
}
|
|
if _, ok := artifactNamed(detail.Steps[1].Artifacts, "private/secret.txt"); ok {
|
|
t.Fatalf("second visit inherited the first turn's private artifact: %+v", detail.Steps[1].Artifacts)
|
|
}
|
|
}
|
|
|
|
// TestRebuildRunStateRestoresPublicFileLists proves a durable resume rebuilds
|
|
// each turn's system-written public list from persisted step artifacts instead
|
|
// of rendering the "none" placeholder for every prior turn.
|
|
func TestRebuildRunStateRestoresPublicFileLists(t *testing.T) {
|
|
detail := &protocol.RunDetail{
|
|
Input: map[string]any{"prompt": "task"},
|
|
Steps: []protocol.RunStep{
|
|
{Sequence: 1, Stage: "first", Status: "completed", Outcome: "success", Output: map[string]any{"markdown": "first body\n"}, Artifacts: []protocol.StageArtifact{{Name: "input.md"}, {Name: "public/a.png"}}},
|
|
{Sequence: 2, Stage: "second", Status: "completed", Outcome: "success", Output: map[string]any{"markdown": "second body\n"}, Artifacts: []protocol.StageArtifact{{Name: "public/a.png"}, {Name: "public/b.txt"}}},
|
|
},
|
|
}
|
|
state := rebuildRunState(detail, protocol.RunStep{Sequence: 2, Stage: "second", Output: map[string]any{"markdown": "second body\n"}})
|
|
memory := turn.RenderHistory(state.turns)
|
|
for _, want := range []string{
|
|
"## first · visit 1",
|
|
"Files added to public/:\n- public/a.png",
|
|
"## second · visit 1",
|
|
"Files added to public/:\n- public/b.txt",
|
|
} {
|
|
if !strings.Contains(memory, want) {
|
|
t.Fatalf("resumed memory missing %q:\n%s", want, memory)
|
|
}
|
|
}
|
|
if strings.Count(memory, "public/a.png") != 1 {
|
|
t.Fatalf("public file was repeated on a later turn:\n%s", memory)
|
|
}
|
|
}
|
|
|
|
// TestRebuildRetryStateRestoresPublicFileLists proves an inherited retry
|
|
// prefix still lists the files that turn added, using the source step's
|
|
// persisted artifact inventory.
|
|
func TestRebuildRetryStateRestoresPublicFileLists(t *testing.T) {
|
|
detail := &protocol.RunDetail{
|
|
Input: map[string]any{"prompt": "task"},
|
|
Steps: []protocol.RunStep{
|
|
{Sequence: 1, Stage: "first", Status: "completed", Outcome: "success", Output: map[string]any{"markdown": "first body\n"}, Artifacts: []protocol.StageArtifact{{Name: "public/report.pdf"}}},
|
|
{Sequence: 2, Stage: "second", Status: "failed", Output: map[string]any{"error": "boom"}},
|
|
},
|
|
}
|
|
handoff := map[string]any{
|
|
"prefix": map[string]any{"first": map[string]any{"markdown": "first body\n"}},
|
|
"prefix_provenance": []any{map[string]any{"stage": "first", "sequence": float64(1)}},
|
|
}
|
|
state := rebuildRetryState(detail, detail.Steps[1], handoff)
|
|
memory := turn.RenderHistory(state.turns)
|
|
if !strings.Contains(memory, "Files added to public/:\n- public/report.pdf") {
|
|
t.Fatalf("retry memory lost the inherited public file list:\n%s", memory)
|
|
}
|
|
}
|
|
|
|
// TestRetryCarriesPublicFilesAcrossAttempts proves a selected retry starts a
|
|
// new artifact root but still reads the completed prefix's public files and
|
|
// lists them in memory, including a retry of a retry.
|
|
func TestRetryCarriesPublicFilesAcrossAttempts(t *testing.T) {
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
defer cancel()
|
|
store, err := pipeline.Open(filepath.Join(t.TempDir(), "retry-turns.db"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer store.Close()
|
|
if _, err := store.Submit(ctx, "retry-turns", pipeline.Template([]string{"a", "b"}, true)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
items, err := store.List(ctx)
|
|
if err != nil || len(items) != 1 {
|
|
t.Fatalf("submit/list: %+v %v", items, err)
|
|
}
|
|
runner, err := newRunner(ctx, store)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
attempts := 0
|
|
runner.execute = func(_ context.Context, def stage.Definition, _ stage.Values) (stage.Result, error) {
|
|
switch def.Key {
|
|
case "a":
|
|
if err := os.WriteFile(filepath.Join(def.Turn.Public, "shared.txt"), []byte("shared"), 0o600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return stage.Result{Outcome: "success", Output: stage.Values{"markdown": "a body\n"}}, nil
|
|
case "b":
|
|
attempts++
|
|
if _, err := os.Stat(filepath.Join(def.Turn.Public, "shared.txt")); err != nil {
|
|
t.Fatalf("attempt %d cannot read the carried public file: %v", attempts, err)
|
|
}
|
|
memory, err := os.ReadFile(def.Turn.Memory)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if !strings.Contains(string(memory), "public/shared.txt") {
|
|
t.Fatalf("attempt %d memory did not list the carried public file:\n%s", attempts, memory)
|
|
}
|
|
if attempts < 3 {
|
|
return stage.Result{}, fmt.Errorf("deliberate failure %d", attempts)
|
|
}
|
|
return stage.Result{Outcome: "success", Output: stage.Values{"markdown": "b body\n"}}, nil
|
|
}
|
|
t.Fatalf("unexpected stage %q", def.Key)
|
|
return stage.Result{}, nil
|
|
}
|
|
original, err := runner.start(protocol.Message{PipelineID: items[0].ID, Version: 1, RequestKey: "retry-turns", Workspace: t.TempDir(), Input: map[string]any{"prompt": "original task"}})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
runner.wg.Wait()
|
|
original, err = store.GetRun(ctx, original.ID)
|
|
if err != nil || original.Status != "failed" || len(original.Steps) != 2 {
|
|
t.Fatalf("initial run did not stop at b: %+v %v", original, err)
|
|
}
|
|
first, err := runner.retry(protocol.Message{RunID: original.ID, RequestKey: "retry-one", RetryStage: "b"})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
runner.wg.Wait()
|
|
first, err = store.GetRun(ctx, first.ID)
|
|
if err != nil || first.Status != "failed" {
|
|
t.Fatalf("first retry did not fail at b: %+v %v", first, err)
|
|
}
|
|
second, err := runner.retry(protocol.Message{RunID: first.ID, RequestKey: "retry-two", RetryStage: "b"})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
runner.wg.Wait()
|
|
second, err = store.GetRun(ctx, second.ID)
|
|
if err != nil || second.Status != "succeeded" {
|
|
t.Fatalf("transitive retry did not succeed: %+v %v", second, err)
|
|
}
|
|
if attempts != 3 {
|
|
t.Fatalf("b ran %d times, want original plus two retries", attempts)
|
|
}
|
|
carried := filepath.Join(runner.artifactsRoot, strconv.FormatInt(second.ID, 10), "public", "shared.txt")
|
|
if data, err := os.ReadFile(carried); err != nil || string(data) != "shared" {
|
|
t.Fatalf("transitive retry did not carry the public file: %q %v", data, err)
|
|
}
|
|
paths := runner.turnPaths(second.ID, 2, "b")
|
|
memory, err := os.ReadFile(paths.HistoryPath)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
for _, want := range []string{"## a · visit 1", "public/shared.txt"} {
|
|
if !strings.Contains(string(memory), want) {
|
|
t.Fatalf("transitive retry memory missing %q:\n%s", want, memory)
|
|
}
|
|
}
|
|
if _, ok := artifactNamed(second.Steps[0].Artifacts, "public/shared.txt"); !ok {
|
|
t.Fatalf("transitive retry did not list the carried public file: %+v", second.Steps[0].Artifacts)
|
|
}
|
|
}
|
|
|
|
// TestTurnLayoutMatchesRunnerDirectories pins the runner's artifact layout to
|
|
// internal/turn.Layout so the server and the protocol cannot drift.
|
|
func TestTurnLayoutMatchesRunnerDirectories(t *testing.T) {
|
|
root := t.TempDir()
|
|
r := &runner{artifactsRoot: root}
|
|
paths := r.turnPaths(7, 2, "implementer")
|
|
want := turn.Layout(filepath.Join(root, "7"), 2, "implementer")
|
|
if paths != want {
|
|
t.Fatalf("runner layout = %#v, want %#v", paths, want)
|
|
}
|
|
if paths.InputPath != filepath.Join(root, "7", "stages", "02-implementer", "input.md") || paths.PublicDir != filepath.Join(root, "7", "public") {
|
|
t.Fatalf("unexpected turn paths: %#v", paths)
|
|
}
|
|
}
|
|
|
|
// TestTurnPrivateAndPublicSeparation proves a visit may read and write its own
|
|
// private directory and the shared public directory, while an earlier visit's
|
|
// private directory is never opened to it.
|
|
func TestTurnPrivateAndPublicSeparation(t *testing.T) {
|
|
root := t.TempDir()
|
|
r := &runner{artifactsRoot: root}
|
|
first := r.turnPaths(3, 1, "one")
|
|
second := r.turnPaths(3, 2, "two")
|
|
for _, paths := range []turn.Paths{first, second} {
|
|
if err := paths.Ensure(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
if err := os.WriteFile(filepath.Join(first.PrivateDir, "secret.txt"), []byte("first only"), 0o600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := os.WriteFile(filepath.Join(first.PublicDir, "shared.txt"), []byte("shared"), 0o600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
access := r.artifactAccess(3, 2, []string{"1"}, second.TurnDir)
|
|
private, _ := filepath.EvalSymlinks(second.PrivateDir)
|
|
public, _ := filepath.EvalSymlinks(second.PublicDir)
|
|
firstPrivate, _ := filepath.EvalSymlinks(first.PrivateDir)
|
|
firstTurn, _ := filepath.EvalSymlinks(first.TurnDir)
|
|
if access.Write != private {
|
|
t.Fatalf("write scope = %q, want the second turn's private directory", access.Write)
|
|
}
|
|
if access.WriteShared != public {
|
|
t.Fatalf("shared write scope = %q, want public", access.WriteShared)
|
|
}
|
|
for _, directory := range access.Read {
|
|
if directory == firstPrivate || directory == firstTurn {
|
|
t.Fatalf("second visit can read the first visit's private turn: %#v", access.Read)
|
|
}
|
|
}
|
|
foundPublic := false
|
|
for _, directory := range access.Read {
|
|
if directory == public {
|
|
foundPublic = true
|
|
}
|
|
}
|
|
if !foundPublic {
|
|
t.Fatalf("second visit cannot read public: %#v", access.Read)
|
|
}
|
|
}
|
|
|
|
// TestTurnMissingPrivateGrantsNoWrite proves a broken visit layout fails closed
|
|
// instead of widening write access to the server-owned turn files.
|
|
func TestTurnMissingPrivateGrantsNoWrite(t *testing.T) {
|
|
root := t.TempDir()
|
|
r := &runner{artifactsRoot: root}
|
|
paths := r.turnPaths(4, 1, "one")
|
|
if err := paths.Ensure(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := os.RemoveAll(paths.PrivateDir); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
access := r.artifactAccess(4, 1, nil, paths.TurnDir)
|
|
if access.Write != "" {
|
|
t.Fatalf("missing private/ granted write scope %q", access.Write)
|
|
}
|
|
if access.WriteShared == "" {
|
|
t.Fatal("shared public scope was lost with private/")
|
|
}
|
|
}
|