e37b864c40
Each model visit now gets <run>/stages/<seq>-<stage>/ with input.md, memory.md, output.md, private/ and a shared run-level public/. The model reads the two input files, follows the stage's instructions verbatim and returns complete markdown; only the # outcome section is parsed. - add internal/turn layout, render and output parsing - write input.md/memory.md before a visit and output.md from the final message; inventory all turn files as stage artifacts - accumulate completed turns and system-written public file additions in memory, including across retries and resume - remove the JSON envelope, strict field validation, line format and the repair ladder for model stages - republish pipeline authoring prompts and docs in the new style Checks: go build ./..., go vet ./..., go test -count=1 ./..., gofmt -l .
346 lines
14 KiB
Go
346 lines
14 KiB
Go
package server
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"os"
|
|
"path/filepath"
|
|
"strconv"
|
|
"strings"
|
|
"testing"
|
|
|
|
"git.a-wan.net/alan/pipelines/internal/pipeline"
|
|
"git.a-wan.net/alan/pipelines/internal/protocol"
|
|
"git.a-wan.net/alan/pipelines/internal/stage"
|
|
"git.a-wan.net/alan/pipelines/internal/turn"
|
|
)
|
|
|
|
// TestTurnsAccumulateMemoryAndListFilesAsArtifacts runs two real visits and
|
|
// proves the file contract end to end: input.md carries the run prompt and then
|
|
// the previous turn's output, memory.md accumulates completed turns plus the
|
|
// system-written public file list, and all three turn files are inventoried.
|
|
func TestTurnsAccumulateMemoryAndListFilesAsArtifacts(t *testing.T) {
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
defer cancel()
|
|
store, err := pipeline.Open(filepath.Join(t.TempDir(), "turns.db"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer store.Close()
|
|
if _, err := store.Submit(ctx, "turns", pipeline.Template([]string{"first", "second"}, true)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
items, err := store.List(ctx)
|
|
if err != nil || len(items) != 1 {
|
|
t.Fatalf("submit/list: %+v %v", items, err)
|
|
}
|
|
runner, err := newRunner(ctx, store)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var secondInput, secondMemory string
|
|
runner.execute = func(_ context.Context, def stage.Definition, _ stage.Values) (stage.Result, error) {
|
|
if def.Turn.Input == "" || def.Turn.Memory == "" || def.Turn.Output == "" || def.Turn.Private == "" || def.Turn.Public == "" {
|
|
t.Fatalf("runner did not provide the turn paths: %+v", def.Turn)
|
|
}
|
|
switch def.Key {
|
|
case "first":
|
|
if err := os.WriteFile(filepath.Join(def.Turn.Public, "shared.txt"), []byte("shared"), 0o600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := os.WriteFile(filepath.Join(def.Turn.Private, "secret.txt"), []byte("turn one only"), 0o600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return stage.Result{Outcome: "success", Output: stage.Values{"markdown": "first body\n"}}, nil
|
|
case "second":
|
|
data, err := os.ReadFile(def.Turn.Input)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
secondInput = string(data)
|
|
data, err = os.ReadFile(def.Turn.Memory)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
secondMemory = string(data)
|
|
if _, err := os.Stat(filepath.Join(def.Turn.Private, "secret.txt")); !os.IsNotExist(err) {
|
|
t.Fatalf("a later turn received an earlier private file: %v", err)
|
|
}
|
|
if _, err := os.Stat(filepath.Join(def.Turn.Public, "shared.txt")); err != nil {
|
|
t.Fatalf("shared public artifact was not visible to the later turn: %v", err)
|
|
}
|
|
if err := os.WriteFile(filepath.Join(def.Turn.Public, "second.txt"), []byte("second"), 0o600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return stage.Result{Outcome: "success", Output: stage.Values{"markdown": "second body\n"}}, nil
|
|
}
|
|
t.Fatalf("unexpected stage %q", def.Key)
|
|
return stage.Result{}, nil
|
|
}
|
|
run, err := runner.start(protocol.Message{PipelineID: items[0].ID, Version: 1, RequestKey: "turns", Workspace: t.TempDir(), Input: map[string]any{"prompt": "original task"}})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
runner.wg.Wait()
|
|
detail, err := store.GetRun(ctx, run.ID)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if detail.Status != "succeeded" || len(detail.Steps) != 2 {
|
|
t.Fatalf("run = %q steps=%d: %+v", detail.Status, len(detail.Steps), detail.RunSummary)
|
|
}
|
|
if secondInput != "# Previous turn output\n\nfirst body\n" {
|
|
t.Fatalf("second input.md = %q, want the previous turn's output", secondInput)
|
|
}
|
|
for _, want := range []string{"## first · visit 1", "first body", "Files added to public/:\n- public/shared.txt"} {
|
|
if !strings.Contains(secondMemory, want) {
|
|
t.Fatalf("second memory.md missing %q:\n%s", want, secondMemory)
|
|
}
|
|
}
|
|
for _, name := range []string{"input.md", "memory.md", "output.md"} {
|
|
if _, ok := artifactNamed(detail.Steps[1].Artifacts, name); !ok {
|
|
t.Fatalf("second visit did not list %s: %+v", name, detail.Steps[1].Artifacts)
|
|
}
|
|
}
|
|
if _, ok := artifactNamed(detail.Steps[0].Artifacts, "private/secret.txt"); !ok {
|
|
t.Fatalf("first visit did not list its private artifact: %+v", detail.Steps[0].Artifacts)
|
|
}
|
|
if _, ok := artifactNamed(detail.Steps[0].Artifacts, "public/shared.txt"); !ok {
|
|
t.Fatalf("first visit did not list the shared public artifact: %+v", detail.Steps[0].Artifacts)
|
|
}
|
|
if _, ok := artifactNamed(detail.Steps[1].Artifacts, "private/secret.txt"); ok {
|
|
t.Fatalf("second visit inherited the first turn's private artifact: %+v", detail.Steps[1].Artifacts)
|
|
}
|
|
}
|
|
|
|
// TestRebuildRunStateRestoresPublicFileLists proves a durable resume rebuilds
|
|
// each turn's system-written public list from persisted step artifacts instead
|
|
// of rendering the "none" placeholder for every prior turn.
|
|
func TestRebuildRunStateRestoresPublicFileLists(t *testing.T) {
|
|
detail := &protocol.RunDetail{
|
|
Input: map[string]any{"prompt": "task"},
|
|
Steps: []protocol.RunStep{
|
|
{Sequence: 1, Stage: "first", Status: "completed", Outcome: "success", Output: map[string]any{"markdown": "first body\n"}, Artifacts: []protocol.StageArtifact{{Name: "input.md"}, {Name: "public/a.png"}}},
|
|
{Sequence: 2, Stage: "second", Status: "completed", Outcome: "success", Output: map[string]any{"markdown": "second body\n"}, Artifacts: []protocol.StageArtifact{{Name: "public/a.png"}, {Name: "public/b.txt"}}},
|
|
},
|
|
}
|
|
state := rebuildRunState(detail, protocol.RunStep{Sequence: 2, Stage: "second", Output: map[string]any{"markdown": "second body\n"}})
|
|
memory := turn.RenderMemory(state.turns)
|
|
for _, want := range []string{
|
|
"## first · visit 1",
|
|
"Files added to public/:\n- public/a.png",
|
|
"## second · visit 1",
|
|
"Files added to public/:\n- public/b.txt",
|
|
} {
|
|
if !strings.Contains(memory, want) {
|
|
t.Fatalf("resumed memory missing %q:\n%s", want, memory)
|
|
}
|
|
}
|
|
if strings.Count(memory, "public/a.png") != 1 {
|
|
t.Fatalf("public file was repeated on a later turn:\n%s", memory)
|
|
}
|
|
}
|
|
|
|
// TestRebuildRetryStateRestoresPublicFileLists proves an inherited retry
|
|
// prefix still lists the files that turn added, using the source step's
|
|
// persisted artifact inventory.
|
|
func TestRebuildRetryStateRestoresPublicFileLists(t *testing.T) {
|
|
detail := &protocol.RunDetail{
|
|
Input: map[string]any{"prompt": "task"},
|
|
Steps: []protocol.RunStep{
|
|
{Sequence: 1, Stage: "first", Status: "completed", Outcome: "success", Output: map[string]any{"markdown": "first body\n"}, Artifacts: []protocol.StageArtifact{{Name: "public/report.pdf"}}},
|
|
{Sequence: 2, Stage: "second", Status: "failed", Output: map[string]any{"error": "boom"}},
|
|
},
|
|
}
|
|
handoff := map[string]any{
|
|
"prefix": map[string]any{"first": map[string]any{"markdown": "first body\n"}},
|
|
"prefix_provenance": []any{map[string]any{"stage": "first", "sequence": float64(1)}},
|
|
}
|
|
state := rebuildRetryState(detail, detail.Steps[1], handoff)
|
|
memory := turn.RenderMemory(state.turns)
|
|
if !strings.Contains(memory, "Files added to public/:\n- public/report.pdf") {
|
|
t.Fatalf("retry memory lost the inherited public file list:\n%s", memory)
|
|
}
|
|
}
|
|
|
|
// TestRetryCarriesPublicFilesAcrossAttempts proves a selected retry starts a
|
|
// new artifact root but still reads the completed prefix's public files and
|
|
// lists them in memory, including a retry of a retry.
|
|
func TestRetryCarriesPublicFilesAcrossAttempts(t *testing.T) {
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
defer cancel()
|
|
store, err := pipeline.Open(filepath.Join(t.TempDir(), "retry-turns.db"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer store.Close()
|
|
if _, err := store.Submit(ctx, "retry-turns", pipeline.Template([]string{"a", "b"}, true)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
items, err := store.List(ctx)
|
|
if err != nil || len(items) != 1 {
|
|
t.Fatalf("submit/list: %+v %v", items, err)
|
|
}
|
|
runner, err := newRunner(ctx, store)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
attempts := 0
|
|
runner.execute = func(_ context.Context, def stage.Definition, _ stage.Values) (stage.Result, error) {
|
|
switch def.Key {
|
|
case "a":
|
|
if err := os.WriteFile(filepath.Join(def.Turn.Public, "shared.txt"), []byte("shared"), 0o600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return stage.Result{Outcome: "success", Output: stage.Values{"markdown": "a body\n"}}, nil
|
|
case "b":
|
|
attempts++
|
|
if _, err := os.Stat(filepath.Join(def.Turn.Public, "shared.txt")); err != nil {
|
|
t.Fatalf("attempt %d cannot read the carried public file: %v", attempts, err)
|
|
}
|
|
memory, err := os.ReadFile(def.Turn.Memory)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if !strings.Contains(string(memory), "public/shared.txt") {
|
|
t.Fatalf("attempt %d memory did not list the carried public file:\n%s", attempts, memory)
|
|
}
|
|
if attempts < 3 {
|
|
return stage.Result{}, fmt.Errorf("deliberate failure %d", attempts)
|
|
}
|
|
return stage.Result{Outcome: "success", Output: stage.Values{"markdown": "b body\n"}}, nil
|
|
}
|
|
t.Fatalf("unexpected stage %q", def.Key)
|
|
return stage.Result{}, nil
|
|
}
|
|
original, err := runner.start(protocol.Message{PipelineID: items[0].ID, Version: 1, RequestKey: "retry-turns", Workspace: t.TempDir(), Input: map[string]any{"prompt": "original task"}})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
runner.wg.Wait()
|
|
original, err = store.GetRun(ctx, original.ID)
|
|
if err != nil || original.Status != "failed" || len(original.Steps) != 2 {
|
|
t.Fatalf("initial run did not stop at b: %+v %v", original, err)
|
|
}
|
|
first, err := runner.retry(protocol.Message{RunID: original.ID, RequestKey: "retry-one", RetryStage: "b"})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
runner.wg.Wait()
|
|
first, err = store.GetRun(ctx, first.ID)
|
|
if err != nil || first.Status != "failed" {
|
|
t.Fatalf("first retry did not fail at b: %+v %v", first, err)
|
|
}
|
|
second, err := runner.retry(protocol.Message{RunID: first.ID, RequestKey: "retry-two", RetryStage: "b"})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
runner.wg.Wait()
|
|
second, err = store.GetRun(ctx, second.ID)
|
|
if err != nil || second.Status != "succeeded" {
|
|
t.Fatalf("transitive retry did not succeed: %+v %v", second, err)
|
|
}
|
|
if attempts != 3 {
|
|
t.Fatalf("b ran %d times, want original plus two retries", attempts)
|
|
}
|
|
carried := filepath.Join(runner.artifactsRoot, strconv.FormatInt(second.ID, 10), "public", "shared.txt")
|
|
if data, err := os.ReadFile(carried); err != nil || string(data) != "shared" {
|
|
t.Fatalf("transitive retry did not carry the public file: %q %v", data, err)
|
|
}
|
|
paths := runner.turnPaths(second.ID, 2, "b")
|
|
memory, err := os.ReadFile(paths.MemoryPath)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
for _, want := range []string{"## a · visit 1", "public/shared.txt"} {
|
|
if !strings.Contains(string(memory), want) {
|
|
t.Fatalf("transitive retry memory missing %q:\n%s", want, memory)
|
|
}
|
|
}
|
|
if _, ok := artifactNamed(second.Steps[0].Artifacts, "public/shared.txt"); !ok {
|
|
t.Fatalf("transitive retry did not list the carried public file: %+v", second.Steps[0].Artifacts)
|
|
}
|
|
}
|
|
|
|
// TestTurnLayoutMatchesRunnerDirectories pins the runner's artifact layout to
|
|
// internal/turn.Layout so the server and the protocol cannot drift.
|
|
func TestTurnLayoutMatchesRunnerDirectories(t *testing.T) {
|
|
root := t.TempDir()
|
|
r := &runner{artifactsRoot: root}
|
|
paths := r.turnPaths(7, 2, "implementer")
|
|
want := turn.Layout(filepath.Join(root, "7"), 2, "implementer")
|
|
if paths != want {
|
|
t.Fatalf("runner layout = %#v, want %#v", paths, want)
|
|
}
|
|
if paths.InputPath != filepath.Join(root, "7", "stages", "02-implementer", "input.md") || paths.PublicDir != filepath.Join(root, "7", "public") {
|
|
t.Fatalf("unexpected turn paths: %#v", paths)
|
|
}
|
|
}
|
|
|
|
// TestTurnPrivateAndPublicSeparation proves a visit may read and write its own
|
|
// private directory and the shared public directory, while an earlier visit's
|
|
// private directory is never opened to it.
|
|
func TestTurnPrivateAndPublicSeparation(t *testing.T) {
|
|
root := t.TempDir()
|
|
r := &runner{artifactsRoot: root}
|
|
first := r.turnPaths(3, 1, "one")
|
|
second := r.turnPaths(3, 2, "two")
|
|
for _, paths := range []turn.Paths{first, second} {
|
|
if err := paths.Ensure(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
if err := os.WriteFile(filepath.Join(first.PrivateDir, "secret.txt"), []byte("first only"), 0o600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := os.WriteFile(filepath.Join(first.PublicDir, "shared.txt"), []byte("shared"), 0o600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
access := r.artifactAccess(3, 2, []string{"1"}, second.TurnDir)
|
|
private, _ := filepath.EvalSymlinks(second.PrivateDir)
|
|
public, _ := filepath.EvalSymlinks(second.PublicDir)
|
|
firstPrivate, _ := filepath.EvalSymlinks(first.PrivateDir)
|
|
firstTurn, _ := filepath.EvalSymlinks(first.TurnDir)
|
|
if access.Write != private {
|
|
t.Fatalf("write scope = %q, want the second turn's private directory", access.Write)
|
|
}
|
|
if access.WriteShared != public {
|
|
t.Fatalf("shared write scope = %q, want public", access.WriteShared)
|
|
}
|
|
for _, directory := range access.Read {
|
|
if directory == firstPrivate || directory == firstTurn {
|
|
t.Fatalf("second visit can read the first visit's private turn: %#v", access.Read)
|
|
}
|
|
}
|
|
foundPublic := false
|
|
for _, directory := range access.Read {
|
|
if directory == public {
|
|
foundPublic = true
|
|
}
|
|
}
|
|
if !foundPublic {
|
|
t.Fatalf("second visit cannot read public: %#v", access.Read)
|
|
}
|
|
}
|
|
|
|
// TestTurnMissingPrivateGrantsNoWrite proves a broken visit layout fails closed
|
|
// instead of widening write access to the server-owned turn files.
|
|
func TestTurnMissingPrivateGrantsNoWrite(t *testing.T) {
|
|
root := t.TempDir()
|
|
r := &runner{artifactsRoot: root}
|
|
paths := r.turnPaths(4, 1, "one")
|
|
if err := paths.Ensure(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := os.RemoveAll(paths.PrivateDir); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
access := r.artifactAccess(4, 1, nil, paths.TurnDir)
|
|
if access.Write != "" {
|
|
t.Fatalf("missing private/ granted write scope %q", access.Write)
|
|
}
|
|
if access.WriteShared == "" {
|
|
t.Fatal("shared public scope was lost with private/")
|
|
}
|
|
}
|