Files
pipelines/internal/server/turn_test.go
T
alan e37b864c40 stage turns: replace JSON results with file-based markdown turns
Each model visit now gets <run>/stages/<seq>-<stage>/ with input.md,
memory.md, output.md, private/ and a shared run-level public/. The model
reads the two input files, follows the stage's instructions verbatim and
returns complete markdown; only the # outcome section is parsed.

- add internal/turn layout, render and output parsing
- write input.md/memory.md before a visit and output.md from the final
  message; inventory all turn files as stage artifacts
- accumulate completed turns and system-written public file additions in
  memory, including across retries and resume
- remove the JSON envelope, strict field validation, line format and the
  repair ladder for model stages
- republish pipeline authoring prompts and docs in the new style

Checks: go build ./..., go vet ./..., go test -count=1 ./..., gofmt -l .
2026-10-19 22:27:44 +01:00

346 lines
14 KiB
Go

package server
import (
"context"
"fmt"
"os"
"path/filepath"
"strconv"
"strings"
"testing"
"git.a-wan.net/alan/pipelines/internal/pipeline"
"git.a-wan.net/alan/pipelines/internal/protocol"
"git.a-wan.net/alan/pipelines/internal/stage"
"git.a-wan.net/alan/pipelines/internal/turn"
)
// TestTurnsAccumulateMemoryAndListFilesAsArtifacts runs two real visits and
// proves the file contract end to end: input.md carries the run prompt and then
// the previous turn's output, memory.md accumulates completed turns plus the
// system-written public file list, and all three turn files are inventoried.
func TestTurnsAccumulateMemoryAndListFilesAsArtifacts(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
store, err := pipeline.Open(filepath.Join(t.TempDir(), "turns.db"))
if err != nil {
t.Fatal(err)
}
defer store.Close()
if _, err := store.Submit(ctx, "turns", pipeline.Template([]string{"first", "second"}, true)); err != nil {
t.Fatal(err)
}
items, err := store.List(ctx)
if err != nil || len(items) != 1 {
t.Fatalf("submit/list: %+v %v", items, err)
}
runner, err := newRunner(ctx, store)
if err != nil {
t.Fatal(err)
}
var secondInput, secondMemory string
runner.execute = func(_ context.Context, def stage.Definition, _ stage.Values) (stage.Result, error) {
if def.Turn.Input == "" || def.Turn.Memory == "" || def.Turn.Output == "" || def.Turn.Private == "" || def.Turn.Public == "" {
t.Fatalf("runner did not provide the turn paths: %+v", def.Turn)
}
switch def.Key {
case "first":
if err := os.WriteFile(filepath.Join(def.Turn.Public, "shared.txt"), []byte("shared"), 0o600); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(def.Turn.Private, "secret.txt"), []byte("turn one only"), 0o600); err != nil {
t.Fatal(err)
}
return stage.Result{Outcome: "success", Output: stage.Values{"markdown": "first body\n"}}, nil
case "second":
data, err := os.ReadFile(def.Turn.Input)
if err != nil {
t.Fatal(err)
}
secondInput = string(data)
data, err = os.ReadFile(def.Turn.Memory)
if err != nil {
t.Fatal(err)
}
secondMemory = string(data)
if _, err := os.Stat(filepath.Join(def.Turn.Private, "secret.txt")); !os.IsNotExist(err) {
t.Fatalf("a later turn received an earlier private file: %v", err)
}
if _, err := os.Stat(filepath.Join(def.Turn.Public, "shared.txt")); err != nil {
t.Fatalf("shared public artifact was not visible to the later turn: %v", err)
}
if err := os.WriteFile(filepath.Join(def.Turn.Public, "second.txt"), []byte("second"), 0o600); err != nil {
t.Fatal(err)
}
return stage.Result{Outcome: "success", Output: stage.Values{"markdown": "second body\n"}}, nil
}
t.Fatalf("unexpected stage %q", def.Key)
return stage.Result{}, nil
}
run, err := runner.start(protocol.Message{PipelineID: items[0].ID, Version: 1, RequestKey: "turns", Workspace: t.TempDir(), Input: map[string]any{"prompt": "original task"}})
if err != nil {
t.Fatal(err)
}
runner.wg.Wait()
detail, err := store.GetRun(ctx, run.ID)
if err != nil {
t.Fatal(err)
}
if detail.Status != "succeeded" || len(detail.Steps) != 2 {
t.Fatalf("run = %q steps=%d: %+v", detail.Status, len(detail.Steps), detail.RunSummary)
}
if secondInput != "# Previous turn output\n\nfirst body\n" {
t.Fatalf("second input.md = %q, want the previous turn's output", secondInput)
}
for _, want := range []string{"## first · visit 1", "first body", "Files added to public/:\n- public/shared.txt"} {
if !strings.Contains(secondMemory, want) {
t.Fatalf("second memory.md missing %q:\n%s", want, secondMemory)
}
}
for _, name := range []string{"input.md", "memory.md", "output.md"} {
if _, ok := artifactNamed(detail.Steps[1].Artifacts, name); !ok {
t.Fatalf("second visit did not list %s: %+v", name, detail.Steps[1].Artifacts)
}
}
if _, ok := artifactNamed(detail.Steps[0].Artifacts, "private/secret.txt"); !ok {
t.Fatalf("first visit did not list its private artifact: %+v", detail.Steps[0].Artifacts)
}
if _, ok := artifactNamed(detail.Steps[0].Artifacts, "public/shared.txt"); !ok {
t.Fatalf("first visit did not list the shared public artifact: %+v", detail.Steps[0].Artifacts)
}
if _, ok := artifactNamed(detail.Steps[1].Artifacts, "private/secret.txt"); ok {
t.Fatalf("second visit inherited the first turn's private artifact: %+v", detail.Steps[1].Artifacts)
}
}
// TestRebuildRunStateRestoresPublicFileLists proves a durable resume rebuilds
// each turn's system-written public list from persisted step artifacts instead
// of rendering the "none" placeholder for every prior turn.
func TestRebuildRunStateRestoresPublicFileLists(t *testing.T) {
detail := &protocol.RunDetail{
Input: map[string]any{"prompt": "task"},
Steps: []protocol.RunStep{
{Sequence: 1, Stage: "first", Status: "completed", Outcome: "success", Output: map[string]any{"markdown": "first body\n"}, Artifacts: []protocol.StageArtifact{{Name: "input.md"}, {Name: "public/a.png"}}},
{Sequence: 2, Stage: "second", Status: "completed", Outcome: "success", Output: map[string]any{"markdown": "second body\n"}, Artifacts: []protocol.StageArtifact{{Name: "public/a.png"}, {Name: "public/b.txt"}}},
},
}
state := rebuildRunState(detail, protocol.RunStep{Sequence: 2, Stage: "second", Output: map[string]any{"markdown": "second body\n"}})
memory := turn.RenderMemory(state.turns)
for _, want := range []string{
"## first · visit 1",
"Files added to public/:\n- public/a.png",
"## second · visit 1",
"Files added to public/:\n- public/b.txt",
} {
if !strings.Contains(memory, want) {
t.Fatalf("resumed memory missing %q:\n%s", want, memory)
}
}
if strings.Count(memory, "public/a.png") != 1 {
t.Fatalf("public file was repeated on a later turn:\n%s", memory)
}
}
// TestRebuildRetryStateRestoresPublicFileLists proves an inherited retry
// prefix still lists the files that turn added, using the source step's
// persisted artifact inventory.
func TestRebuildRetryStateRestoresPublicFileLists(t *testing.T) {
detail := &protocol.RunDetail{
Input: map[string]any{"prompt": "task"},
Steps: []protocol.RunStep{
{Sequence: 1, Stage: "first", Status: "completed", Outcome: "success", Output: map[string]any{"markdown": "first body\n"}, Artifacts: []protocol.StageArtifact{{Name: "public/report.pdf"}}},
{Sequence: 2, Stage: "second", Status: "failed", Output: map[string]any{"error": "boom"}},
},
}
handoff := map[string]any{
"prefix": map[string]any{"first": map[string]any{"markdown": "first body\n"}},
"prefix_provenance": []any{map[string]any{"stage": "first", "sequence": float64(1)}},
}
state := rebuildRetryState(detail, detail.Steps[1], handoff)
memory := turn.RenderMemory(state.turns)
if !strings.Contains(memory, "Files added to public/:\n- public/report.pdf") {
t.Fatalf("retry memory lost the inherited public file list:\n%s", memory)
}
}
// TestRetryCarriesPublicFilesAcrossAttempts proves a selected retry starts a
// new artifact root but still reads the completed prefix's public files and
// lists them in memory, including a retry of a retry.
func TestRetryCarriesPublicFilesAcrossAttempts(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
store, err := pipeline.Open(filepath.Join(t.TempDir(), "retry-turns.db"))
if err != nil {
t.Fatal(err)
}
defer store.Close()
if _, err := store.Submit(ctx, "retry-turns", pipeline.Template([]string{"a", "b"}, true)); err != nil {
t.Fatal(err)
}
items, err := store.List(ctx)
if err != nil || len(items) != 1 {
t.Fatalf("submit/list: %+v %v", items, err)
}
runner, err := newRunner(ctx, store)
if err != nil {
t.Fatal(err)
}
attempts := 0
runner.execute = func(_ context.Context, def stage.Definition, _ stage.Values) (stage.Result, error) {
switch def.Key {
case "a":
if err := os.WriteFile(filepath.Join(def.Turn.Public, "shared.txt"), []byte("shared"), 0o600); err != nil {
t.Fatal(err)
}
return stage.Result{Outcome: "success", Output: stage.Values{"markdown": "a body\n"}}, nil
case "b":
attempts++
if _, err := os.Stat(filepath.Join(def.Turn.Public, "shared.txt")); err != nil {
t.Fatalf("attempt %d cannot read the carried public file: %v", attempts, err)
}
memory, err := os.ReadFile(def.Turn.Memory)
if err != nil {
t.Fatal(err)
}
if !strings.Contains(string(memory), "public/shared.txt") {
t.Fatalf("attempt %d memory did not list the carried public file:\n%s", attempts, memory)
}
if attempts < 3 {
return stage.Result{}, fmt.Errorf("deliberate failure %d", attempts)
}
return stage.Result{Outcome: "success", Output: stage.Values{"markdown": "b body\n"}}, nil
}
t.Fatalf("unexpected stage %q", def.Key)
return stage.Result{}, nil
}
original, err := runner.start(protocol.Message{PipelineID: items[0].ID, Version: 1, RequestKey: "retry-turns", Workspace: t.TempDir(), Input: map[string]any{"prompt": "original task"}})
if err != nil {
t.Fatal(err)
}
runner.wg.Wait()
original, err = store.GetRun(ctx, original.ID)
if err != nil || original.Status != "failed" || len(original.Steps) != 2 {
t.Fatalf("initial run did not stop at b: %+v %v", original, err)
}
first, err := runner.retry(protocol.Message{RunID: original.ID, RequestKey: "retry-one", RetryStage: "b"})
if err != nil {
t.Fatal(err)
}
runner.wg.Wait()
first, err = store.GetRun(ctx, first.ID)
if err != nil || first.Status != "failed" {
t.Fatalf("first retry did not fail at b: %+v %v", first, err)
}
second, err := runner.retry(protocol.Message{RunID: first.ID, RequestKey: "retry-two", RetryStage: "b"})
if err != nil {
t.Fatal(err)
}
runner.wg.Wait()
second, err = store.GetRun(ctx, second.ID)
if err != nil || second.Status != "succeeded" {
t.Fatalf("transitive retry did not succeed: %+v %v", second, err)
}
if attempts != 3 {
t.Fatalf("b ran %d times, want original plus two retries", attempts)
}
carried := filepath.Join(runner.artifactsRoot, strconv.FormatInt(second.ID, 10), "public", "shared.txt")
if data, err := os.ReadFile(carried); err != nil || string(data) != "shared" {
t.Fatalf("transitive retry did not carry the public file: %q %v", data, err)
}
paths := runner.turnPaths(second.ID, 2, "b")
memory, err := os.ReadFile(paths.MemoryPath)
if err != nil {
t.Fatal(err)
}
for _, want := range []string{"## a · visit 1", "public/shared.txt"} {
if !strings.Contains(string(memory), want) {
t.Fatalf("transitive retry memory missing %q:\n%s", want, memory)
}
}
if _, ok := artifactNamed(second.Steps[0].Artifacts, "public/shared.txt"); !ok {
t.Fatalf("transitive retry did not list the carried public file: %+v", second.Steps[0].Artifacts)
}
}
// TestTurnLayoutMatchesRunnerDirectories pins the runner's artifact layout to
// internal/turn.Layout so the server and the protocol cannot drift.
func TestTurnLayoutMatchesRunnerDirectories(t *testing.T) {
root := t.TempDir()
r := &runner{artifactsRoot: root}
paths := r.turnPaths(7, 2, "implementer")
want := turn.Layout(filepath.Join(root, "7"), 2, "implementer")
if paths != want {
t.Fatalf("runner layout = %#v, want %#v", paths, want)
}
if paths.InputPath != filepath.Join(root, "7", "stages", "02-implementer", "input.md") || paths.PublicDir != filepath.Join(root, "7", "public") {
t.Fatalf("unexpected turn paths: %#v", paths)
}
}
// TestTurnPrivateAndPublicSeparation proves a visit may read and write its own
// private directory and the shared public directory, while an earlier visit's
// private directory is never opened to it.
func TestTurnPrivateAndPublicSeparation(t *testing.T) {
root := t.TempDir()
r := &runner{artifactsRoot: root}
first := r.turnPaths(3, 1, "one")
second := r.turnPaths(3, 2, "two")
for _, paths := range []turn.Paths{first, second} {
if err := paths.Ensure(); err != nil {
t.Fatal(err)
}
}
if err := os.WriteFile(filepath.Join(first.PrivateDir, "secret.txt"), []byte("first only"), 0o600); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(first.PublicDir, "shared.txt"), []byte("shared"), 0o600); err != nil {
t.Fatal(err)
}
access := r.artifactAccess(3, 2, []string{"1"}, second.TurnDir)
private, _ := filepath.EvalSymlinks(second.PrivateDir)
public, _ := filepath.EvalSymlinks(second.PublicDir)
firstPrivate, _ := filepath.EvalSymlinks(first.PrivateDir)
firstTurn, _ := filepath.EvalSymlinks(first.TurnDir)
if access.Write != private {
t.Fatalf("write scope = %q, want the second turn's private directory", access.Write)
}
if access.WriteShared != public {
t.Fatalf("shared write scope = %q, want public", access.WriteShared)
}
for _, directory := range access.Read {
if directory == firstPrivate || directory == firstTurn {
t.Fatalf("second visit can read the first visit's private turn: %#v", access.Read)
}
}
foundPublic := false
for _, directory := range access.Read {
if directory == public {
foundPublic = true
}
}
if !foundPublic {
t.Fatalf("second visit cannot read public: %#v", access.Read)
}
}
// TestTurnMissingPrivateGrantsNoWrite proves a broken visit layout fails closed
// instead of widening write access to the server-owned turn files.
func TestTurnMissingPrivateGrantsNoWrite(t *testing.T) {
root := t.TempDir()
r := &runner{artifactsRoot: root}
paths := r.turnPaths(4, 1, "one")
if err := paths.Ensure(); err != nil {
t.Fatal(err)
}
if err := os.RemoveAll(paths.PrivateDir); err != nil {
t.Fatal(err)
}
access := r.artifactAccess(4, 1, nil, paths.TurnDir)
if access.Write != "" {
t.Fatalf("missing private/ granted write scope %q", access.Write)
}
if access.WriteShared == "" {
t.Fatal("shared public scope was lost with private/")
}
}