Model stages: outcome-first output shape and one in-session outcome repair

The prompt now says output.md must start with # outcome and the outcome
word alone on the next line, then # handoff, then detail. Runs 241/243
followed the old "start with # handoff" wording and dropped the outcome.

When a completed turn lacks a valid outcome, the OpenCode/agent and Pi
clients send one short follow-up in the same session asking only for the
outcome line. An outcome-only reply goes at the top of the original report;
a still-invalid reply fails with the same validation error as before. The
follow-up's tokens accrue to the session usage. No outcome is inferred.

Missing tools/evidence now name blocked when declared, else error. The
parser tolerates a trailing period on the outcome word.
This commit is contained in:
2026-11-04 17:20:58 +00:00
parent b5927505cc
commit 88dfc051e5
13 changed files with 457 additions and 16 deletions
+12 -2
View File
@@ -59,12 +59,21 @@ success
The rest of the output is free-form markdown.
```
Matching ignores case and surrounding whitespace and returns the canonical edge
Matching ignores case, surrounding whitespace and one trailing period, and returns the canonical edge
spelling. Headings and words inside fenced code blocks are ignored, so a code
sample cannot add a false outcome. Missing, duplicate, empty or undeclared
outcomes fail the stage. The parser never guesses, repairs the text, or looks for
any other section.
When a turn completes but its final message has no valid outcome, the harness
client (OpenCode or Pi) sends one short follow-up in the same session asking
only for the `# outcome` line and listing the allowed outcomes, without redoing
work. An outcome-only reply is placed at the top of the original output (any
invalid outcome section is replaced); a full reply replaces it. The outcome is
always the model's explicit answer, never inferred from prose. Its tokens count
toward the stage usage. If the reply is still invalid, the stage fails with the
same validation error as before.
Everything after the outcome value line is retained as the turn body under the
single `markdown` key of the internal output map. No JSON envelope, output
schema, field selection, `output_contract`, or format-repair pass exists for
@@ -81,7 +90,8 @@ facts:
- the paths to `memory.md`, `input.md` and `output.md` (or their contents for a
text-only stage);
- the stage prompt verbatim;
- the worked outcome example and the complete allowed outcome list;
- the required shape, outcome first (`# outcome` and one of the allowed
outcomes, then `# handoff`, then any detail), with a worked example;
- the execution time and timezone;
- a granted resource target, when one exists;
- the writable `private/` and shared `public/` artifact directories, when they
+6
View File
@@ -241,6 +241,12 @@ func (c *AgentClient) Generate(ctx context.Context, config ModelConfig, prompt s
if len(data) > 0 {
c.Transcript = string(data)
}
if err == nil {
text, data, err = c.repairOutput(ctx, config, c.SessionID, text)
if len(data) > 0 {
c.Transcript = string(data)
}
}
// Budgeted stages already settle through the single watch.finish read. For an
// unbudgeted V2 stage the watch was never started, so record best-effort final
// session usage from exactly one bounded aggregate read; missing/invalid usage
+10 -5
View File
@@ -46,13 +46,14 @@ func TestModelFailureCategoriesAreSanitizedAndDistinct(t *testing.T) {
// TestAgentPostEditValidationFailureRetainsWorkWithoutReplay reproduces a
// tool-using stage that edits once and then returns a nested-duplicate final
// result. The validation failure must not re-prompt the agent, must retain the
// admitted session, transcript and evidence, and must carry the precise category.
// result. The only extra prompt is the single outcome repair follow-up, which
// does not redo work; when the reply is still invalid the stage fails as
// before, retaining the admitted session, transcript, evidence and category.
func TestAgentPostEditValidationFailureRetainsWorkWithoutReplay(t *testing.T) {
dir := t.TempDir()
path := filepath.Join(dir, "README.md")
review := "# outcome\nsuccess\n\n# evidence\n\nprivate-evidence\n\n# outcome\nerror\n"
creates, edits, owned := 0, 0, 0
creates, edits, repairs, owned := 0, 0, 0, 0
agent := &AgentClient{OpenCodeClient: OpenCodeClient{Directory: dir}, OnSession: func(string, bool) error { owned++; return nil }}
agent.command = func(_ context.Context, _ string, args ...string) ([]byte, error) {
if args[1] == "delete" {
@@ -63,6 +64,10 @@ func TestAgentPostEditValidationFailureRetainsWorkWithoutReplay(t *testing.T) {
creates++
return []byte(fmt.Sprintf(`{"data":{"id":"ses_%d"}}`, creates)), nil
case "/api/session/ses_1/prompt":
if strings.Contains(args[len(args)-1], "no valid `# outcome`") {
repairs++
return []byte(`{}`), nil
}
edits++
if err := os.WriteFile(path, []byte("edited once"), 0600); err != nil {
t.Fatal(err)
@@ -85,8 +90,8 @@ func TestAgentPostEditValidationFailureRetainsWorkWithoutReplay(t *testing.T) {
if strings.Contains(err.Error(), "HARNESS TRANSPORT") || strings.Contains(err.Error(), "private-evidence") {
t.Fatalf("validation diagnostic leaked content or a false category: %v", err)
}
if creates != 1 || edits != 1 || owned != 2 || agent.SessionID != "ses_1" || result.Output != nil {
t.Fatal("validation failure replayed work or dropped retained ownership", creates, edits, owned, agent.SessionID, result)
if creates != 1 || edits != 1 || repairs != 1 || owned != 2 || agent.SessionID != "ses_1" || result.Output != nil {
t.Fatal("validation failure replayed work or dropped retained ownership", creates, edits, repairs, owned, agent.SessionID, result)
}
if !json.Valid([]byte(agent.Transcript)) || !strings.Contains(agent.Transcript, "private-evidence") {
t.Fatal("retained readback evidence missing", agent.Transcript)
+24 -3
View File
@@ -105,19 +105,40 @@ func modelPrompt(def Definition, spec ModelSpec, input Values) (string, error) {
}
b.WriteString("\n\nProduce ")
b.WriteString(def.Turn.Output)
b.WriteString(" as markdown; return its complete content as your final message (the server writes that file). Start it with a short # handoff section (at most about 1 KB): what you did, files changed, and open issues or findings the next stage must act on. The next stage receives only that section as essential memory; the rest of your output stays in history. The only required section is the outcome, for example:\n\n# handoff\n...\n\n# outcome\n")
b.WriteString(" as markdown and return its complete content as your final message (the server writes that file; no JSON envelope). ")
b.WriteString(outputShape(outcomes))
b.WriteString(" The next stage receives only the handoff as essential memory. For example:\n\n# outcome\n")
b.WriteString(preferredOutcome(outcomes))
fmt.Fprintf(&b, "\n\nAllowed outcomes: %s.\n\nEverything else is free-form markdown for the next stage and the operator. The server does not parse or validate any other section. Return the complete output.md content as your final message; do not wrap it in a JSON envelope.\n", strings.Join(outcomes, ", "))
b.WriteString("\n\n# handoff\nWhat you did, files changed, open issues for the next stage.\n\n# details\n...\n")
fmt.Fprintf(&b, "\nExecution time: %s. Timezone: %s.\n", time.Now().Format(time.RFC3339), time.Local.String())
if def.Turn.Public != "" {
fmt.Fprintf(&b, "\nShared artifacts go to %s; files there are visible to every later stage. That absolute path is the shared folder: a relative public/ inside the workspace is a different, unshared directory. Turn-local work goes to %s and is not handed to later stages.\n", def.Turn.Public, def.Turn.Private)
}
b.WriteString("\n")
b.WriteString(toolsLine)
b.WriteString(" Do not invent file edits, command results, screenshots, or observations. If the task requires unavailable tools or evidence, choose the stage's error outcome and explain the limitation. Never claim unperformed work succeeded.")
b.WriteString(" Do not invent file edits, command results, screenshots, or observations. If the task requires unavailable tools or evidence, ")
b.WriteString(limitationOutcome(outcomes))
b.WriteString(" and explain the limitation. Never claim unperformed work succeeded.")
return b.String(), nil
}
// outputShape is the one sentence stating the required output.md layout,
// outcome first. The repair follow-up reuses it so both say the same thing.
func outputShape(outcomes []string) string {
return "output.md must start with `# outcome` and the outcome word alone on the next line, one of: " + strings.Join(outcomes, ", ") + "; then `# handoff` (a few lines); then any detail."
}
// limitationOutcome names the outcome for missing tools or evidence: blocked
// when the stage declares it, otherwise error.
func limitationOutcome(outcomes []string) string {
for _, want := range []string{"blocked", "error"} {
if slices.Contains(outcomes, want) {
return "choose `" + want + "`"
}
}
return "choose the stage's error outcome"
}
// serverChangedFiles reads the runner's reserved changed-path list from stage
// input, accepting the []string the runner supplies and the []any JSON decoding
// produces for a replayed handoff. Other values are ignored.
+88
View File
@@ -0,0 +1,88 @@
package stage
import (
"context"
"errors"
"strings"
"git.a-wan.net/alan/pipelines/internal/turn"
)
// outputRepair is the single outcome repair follow-up. When a turn completes
// but its final message lacks a valid outcome, the client sends one short
// message in the same session asking only for the outcome, instead of failing
// a stage whose admitted work a full retry would repeat. The outcome is always
// stated by the model; the server never infers one from prose.
type outputRepair struct{ outcomes []string }
type outputRepairKey struct{}
// withOutputRepair lets a model client repair a missing or invalid outcome
// for the given declared outcomes while its session is still open.
func withOutputRepair(ctx context.Context, outcomes []string) context.Context {
return context.WithValue(ctx, outputRepairKey{}, &outputRepair{outcomes: outcomes})
}
// outputRepairFrom returns the attempt's repair, or nil when none applies.
func outputRepairFrom(ctx context.Context) *outputRepair {
repair, _ := ctx.Value(outputRepairKey{}).(*outputRepair)
return repair
}
// followUp reports whether text needs the outcome follow-up and returns its
// message. Valid and empty outputs need none: an empty output has no report
// to keep, so it fails as before.
func (r *outputRepair) followUp(text string) (string, bool) {
if r == nil || len(r.outcomes) == 0 || strings.TrimSpace(text) == "" {
return "", false
}
_, _, err := turn.ParseOutput(text, r.outcomes)
if !errors.Is(err, turn.ErrMissingOutcome) && !errors.Is(err, turn.ErrOutcomeNotAllowed) && !errors.Is(err, turn.ErrDuplicateOutcome) {
return "", false
}
allowed := strings.Join(r.outcomes, ", ")
return "Your final message has no valid `# outcome` section. Do not redo any work or repeat your report; the server keeps it. " +
outputShape(r.outcomes) + " Reply with only these two lines:\n\n# outcome\n<one of: " + allowed + ">\n", true
}
// merge combines the original output with the follow-up reply. An
// outcome-only reply is placed at the top of the original report, replacing
// any invalid outcome section. A reply carrying a full document replaces the
// original. An invalid reply keeps the original, so validation fails as before.
func (r *outputRepair) merge(original, reply string) string {
outcome, body, err := turn.ParseOutput(reply, r.outcomes)
if err != nil {
return original
}
if strings.TrimSpace(body) == "" {
return turn.WithOutcome(original, outcome)
}
return reply
}
// repairOutput sends the outcome follow-up in an open OpenCode session when
// text needs it, waits for that turn and returns the merged output with the
// raw final readback. Tokens it uses accrue to the same session, so the
// existing budget watch and session usage read count them.
func (c OpenCodeClient) repairOutput(ctx context.Context, config ModelConfig, sessionID, text string) (string, []byte, error) {
repair := outputRepairFrom(ctx)
message, ok := repair.followUp(text)
if !ok {
return text, nil, nil
}
if _, err := c.call(ctx, "post", "/api/session/"+sessionID+"/prompt", generationBody("text", message, config, c.Backend)); err != nil {
return "", nil, err
}
if c.Backend == "v1" {
if _, err := c.call(ctx, "post", "/api/experimental/session/"+sessionID+"/wait", nil); err != nil {
return "", nil, err
}
} else if err := c.waitForSessionIdle(ctx, sessionID); err != nil {
return "", nil, err
}
data, reply, err := c.readFinalAgentResult(ctx, sessionID)
if err != nil {
return "", data, err
}
return repair.merge(text, reply), data, nil
}
+171
View File
@@ -0,0 +1,171 @@
package stage
import (
"context"
"encoding/json"
"errors"
"fmt"
"os"
"strings"
"testing"
"time"
"git.a-wan.net/alan/pipelines/internal/turn"
)
// repairAgent serves one V2 agent session whose first final message is
// handoffOnly and whose follow-up reply is reply. It records the call order.
func repairAgent(handoffOnly, reply string, calls *[]string) *AgentClient {
prompts := 0
return &AgentClient{OpenCodeClient: OpenCodeClient{Backend: "v2", sessionPollInterval: time.Millisecond, command: func(_ context.Context, _ string, args ...string) ([]byte, error) {
path := args[2]
switch {
case path == "/api/session" && args[1] == "post":
return []byte(`{"data":{"id":"ses_1"}}`), nil
case path == "/api/session/ses_1/prompt":
prompts++
body := args[len(args)-1]
if prompts == 2 && !strings.Contains(body, "no valid `# outcome`") {
return nil, fmt.Errorf("second prompt is not the repair follow-up: %s", body)
}
*calls = append(*calls, fmt.Sprintf("prompt%d", prompts))
return []byte(`{}`), nil
case path == "/api/session/active":
return []byte(`{"data":{}}`), nil
case strings.HasSuffix(path, finalAssistantReadback):
text := handoffOnly
if prompts == 2 {
text = reply
}
return json.Marshal(map[string]any{"data": []any{map[string]any{"finish": "stop", "content": []any{map[string]string{"type": "text", "text": text}}}}})
case path == "/api/session/ses_1":
*calls = append(*calls, "usage")
return usageResponse("ses_1", .2, 42), nil
case strings.HasSuffix(path, "/interrupt"):
return []byte(`{}`), nil
}
return nil, fmt.Errorf("unexpected path %s", path)
}}}
}
// TestHandoffOnlyOutputIsRepairedInSession reproduces runs 241/243: the model
// starts with # handoff and omits the outcome. One follow-up in the same
// session asks only for the outcome; its reply is used and the report kept.
func TestHandoffOnlyOutputIsRepairedInSession(t *testing.T) {
report := "# handoff\n\nImplemented cache.go; tests pass.\n\n# details\n\nLong report.\n"
var calls []string
agent := repairAgent(report, "# outcome\nsuccess\n", &calls)
def := turnDefinition(t)
def.Blocks["tools"] = "edit"
budget := NewBudget(0, 0)
result, err := (ModelStage{Client: agent}).Run(WithBudget(context.Background(), budget), def, nil)
if err != nil || result.Outcome != "success" {
t.Fatalf("repair did not recover the outcome: %v %v", result, err)
}
written, _ := os.ReadFile(def.Turn.Output)
if string(written) != "# outcome\nsuccess\n\n"+report {
t.Fatalf("output.md = %q", written)
}
if body := result.Output["markdown"].(string); !strings.Contains(body, "Implemented cache.go") || !strings.Contains(body, "Long report.") {
t.Fatalf("report dropped: %q", body)
}
// The usage read follows the follow-up, so its tokens are counted.
if strings.Join(calls, ",") != "prompt1,prompt2,usage" || budget.Snapshot().UsedTokens != 42 {
t.Fatalf("calls=%v usage=%+v", calls, budget.Snapshot())
}
}
// TestFailedRepairFailsAsBefore proves a follow-up that still lacks a valid
// outcome fails with today's missing-outcome error and keeps the report.
func TestFailedRepairFailsAsBefore(t *testing.T) {
report := "# handoff\n\nDid the work.\n"
for _, reply := range []string{"I think it went well.\n", "# outcome\nmaybe\n"} {
var calls []string
def := turnDefinition(t)
def.Blocks["tools"] = "edit"
_, err := (ModelStage{Client: repairAgent(report, reply, &calls)}).Run(context.Background(), def, nil)
if !errors.Is(err, ErrInvalidModelResult) || !strings.Contains(err.Error(), turn.ErrMissingOutcome.Error()) || !strings.Contains(err.Error(), "FINAL RESULT VALIDATION") {
t.Fatalf("reply %q: err = %v", reply, err)
}
if strings.Count(strings.Join(calls, ","), "prompt") != 2 {
t.Fatalf("expected exactly one follow-up: %v", calls)
}
if written, _ := os.ReadFile(def.Turn.Output); string(written) != report {
t.Fatalf("output.md = %q", written)
}
}
}
// TestValidOutputSendsNoFollowUp keeps a valid first answer to one prompt.
func TestValidOutputSendsNoFollowUp(t *testing.T) {
var calls []string
def := turnDefinition(t)
def.Blocks["tools"] = "edit"
result, err := (ModelStage{Client: repairAgent("# outcome\nerror\n\n# handoff\n\nblocked\n", "", &calls)}).Run(context.Background(), def, nil)
if err != nil || result.Outcome != "error" || strings.Count(strings.Join(calls, ","), "prompt") != 1 {
t.Fatalf("%v %v %v", result, err, calls)
}
}
// TestModelPromptStatesOutcomeFirst checks the required shape: outcome first,
// then handoff, then detail, with the allowed outcomes listed.
func TestModelPromptStatesOutcomeFirst(t *testing.T) {
def := modelDefinition()
spec, err := (ModelStage{}).Spec(def)
if err != nil {
t.Fatal(err)
}
prompt, err := modelPrompt(def, spec, nil)
if err != nil {
t.Fatal(err)
}
if !strings.Contains(prompt, "output.md must start with `# outcome` and the outcome word alone on the next line, one of: error, success; then `# handoff` (a few lines); then any detail.") {
t.Fatalf("shape sentence missing: %s", prompt)
}
outcome, handoff := strings.Index(prompt, "\n# outcome\n"), strings.Index(prompt, "\n# handoff\n")
if outcome < 0 || handoff < 0 || outcome > handoff {
t.Fatalf("example does not put the outcome before the handoff: %s", prompt)
}
if strings.Contains(prompt, "Start it with a short # handoff") {
t.Fatal("prompt still asks to start with the handoff")
}
if !strings.Contains(prompt, "unavailable tools or evidence, choose `error` and explain") {
t.Fatalf("limitation outcome not error: %s", prompt)
}
}
func TestLimitationOutcomePrefersBlocked(t *testing.T) {
for outcomes, want := range map[string]string{
"blocked,error,success": "choose `blocked`",
"error,success": "choose `error`",
"done,retry": "choose the stage's error outcome",
} {
if got := limitationOutcome(strings.Split(outcomes, ",")); got != want {
t.Fatalf("%s: %q, want %q", outcomes, got, want)
}
}
}
func TestRepairMerge(t *testing.T) {
r := &outputRepair{outcomes: []string{"error", "success"}}
if _, ok := r.followUp("# outcome\nsuccess\n"); ok {
t.Fatal("valid output needs no follow-up")
}
if _, ok := r.followUp(" \n"); ok {
t.Fatal("empty output is not repaired")
}
message, ok := r.followUp("# handoff\n\nx\n")
if !ok || !strings.Contains(message, "error, success") || !strings.Contains(message, "Do not redo any work") {
t.Fatalf("follow-up = %q", message)
}
if got := r.merge("# handoff\n\nx\n", "# outcome\nSuccess\n"); got != "# outcome\nsuccess\n\n# handoff\n\nx\n" {
t.Fatalf("outcome-only merge = %q", got)
}
full := "# outcome\nerror\n\n# handoff\n\ny\n"
if got := r.merge("# handoff\n\nx\n", full); got != full {
t.Fatalf("full reply merge = %q", got)
}
if got := r.merge("# handoff\n\nx\n", "no outcome here"); got != "# handoff\n\nx\n" {
t.Fatalf("invalid reply merge = %q", got)
}
}
+3 -1
View File
@@ -61,7 +61,9 @@ func (m ModelStage) Run(ctx context.Context, def Definition, input Values) (Resu
}
spec.Config.Budget.Attempt()
attemptCtx, cancel := context.WithTimeoutCause(ctx, spec.Config.Timeout, &stageTimeoutCause{timeout: spec.Config.Timeout})
text, generateErr := client.Generate(attemptCtx, spec.Config, prompt)
// A completed turn without a valid outcome gets one in-session follow-up
// asking only for the outcome, rather than a full retry of its work.
text, generateErr := client.Generate(withOutputRepair(attemptCtx, outcomesFor(def.Edges)), spec.Config, prompt)
if generateErr == nil && attemptCtx.Err() != nil {
generateErr = attemptCtx.Err()
}
+1 -1
View File
@@ -78,7 +78,7 @@ func TestModelPromptUsesTurnFilesAndNoJSONEnvelope(t *testing.T) {
"Do the task.",
"# outcome",
"success",
"free-form markdown",
"must start with `# outcome`",
} {
if !strings.Contains(prompt, want) {
t.Fatalf("prompt missing %q:\n%s", want, prompt)
+7
View File
@@ -431,6 +431,13 @@ func (c *OpenCodeClient) Generate(ctx context.Context, config ModelConfig, promp
if err != nil {
return "", err
}
text, data, err = c.repairOutput(ctx, config, session.Data.ID, text)
if len(data) > 0 {
c.FormatAudit = string(data)
}
if err != nil {
return "", err
}
if !config.Budget.Enabled() {
// As with AgentClient, missing usage is not a generation failure when
// no limit is enabled (and a nil ledger needs no aggregate read).
+24 -1
View File
@@ -329,8 +329,31 @@ func (c *PiClient) Generate(ctx context.Context, config ModelConfig, prompt stri
if _, err = c.rpc(ctx, "prompt", map[string]any{"message": prompt}); err != nil {
return "", err
}
if text, err = c.settleTurn(ctx); err != nil {
return "", err
}
// One outcome repair follow-up in the same Pi session; get_session_stats
// reports session totals, so its tokens are counted by the re-read.
repair := outputRepairFrom(ctx)
if message, ok := repair.followUp(text); ok {
c.settled = false
if _, err = c.rpc(ctx, "prompt", map[string]any{"message": message}); err != nil {
return "", err
}
reply, err := c.settleTurn(ctx)
if err != nil {
return "", err
}
text = repair.merge(text, reply)
}
return text, nil
}
// settleTurn waits for the prompted Pi turn to settle, records its session
// usage and returns the final assistant text.
func (c *PiClient) settleTurn(ctx context.Context) (string, error) {
for !c.settled {
if _, err = c.receive(ctx); err != nil {
if _, err := c.receive(ctx); err != nil {
return "", err
}
}
+21 -2
View File
@@ -30,6 +30,12 @@ done
if os.Getenv("PI_FIXTURE_CASE") == "missing" {
delete(message, "usage")
}
// repair: the first turn drops its outcome; the follow-up states only it.
repair := os.Getenv("PI_FIXTURE_CASE") == "repair"
prompts, total := 0, 0
if repair {
message["content"] = []map[string]string{{"type": "text", "text": "# handoff\n\nReport kept.\n"}}
}
emit := func(v any) { data, _ := json.Marshal(v); fmt.Println(string(data)) }
scanner := bufio.NewScanner(os.Stdin)
for scanner.Scan() {
@@ -45,12 +51,17 @@ done
emit(map[string]any{"type": "response", "id": request["id"], "success": true})
continue
}
prompts++
total += 10
if repair && prompts == 2 {
message = map[string]any{"role": "assistant", "stopReason": "stop", "timestamp": 2, "content": []map[string]string{{"type": "text", "text": "# outcome\nsuccess\n"}}, "usage": map[string]any{"totalTokens": 10, "cost": map[string]any{"total": 0.1}}}
}
emit(map[string]any{"type": "message_end", "message": message})
emit(map[string]string{"type": "agent_settled"})
case "abort":
emit(map[string]string{"type": "agent_settled"})
case "get_session_stats":
data = map[string]any{"sessionId": sessionID, "assistantMessages": 1, "tokens": map[string]int{"total": 10}, "cost": 0.1}
data = map[string]any{"sessionId": sessionID, "assistantMessages": max(prompts, 1), "tokens": map[string]int{"total": max(total, 10)}, "cost": 0.1 * float64(max(prompts, 1))}
case "get_messages":
data = map[string]any{"messages": []any{message}}
}
@@ -60,7 +71,7 @@ done
}
func TestPiAuditedRPCAndBudgets(t *testing.T) {
for _, which := range []string{"success", "budget", "missing", "hang"} {
for _, which := range []string{"success", "budget", "missing", "hang", "repair"} {
t.Run(which, func(t *testing.T) {
root := t.TempDir()
t.Setenv("XDG_CACHE_HOME", root)
@@ -87,12 +98,20 @@ func TestPiAuditedRPCAndBudgets(t *testing.T) {
ctx, cancel = context.WithTimeout(context.Background(), 200*time.Millisecond)
}
defer cancel()
if which == "repair" {
ctx = withOutputRepair(ctx, []string{"error", "success"})
}
text, err := client.Generate(ctx, ModelConfig{Provider: "fixture", Model: "model", Budget: budget}, "task")
switch which {
case "success":
if err != nil || text == "" || budget.Snapshot().UsedTokens != 10 {
t.Fatal(text, err, budget.Snapshot())
}
case "repair":
// The follow-up's tokens count: two turns of 10 each.
if err != nil || text != "# outcome\nsuccess\n\n# handoff\n\nReport kept.\n" || budget.Snapshot().UsedTokens != 20 {
t.Fatal(text, err, budget.Snapshot())
}
case "budget":
if !errors.Is(err, ErrBudgetExceeded) {
t.Fatal(err)
+58 -1
View File
@@ -64,7 +64,8 @@ func ParseOutput(markdown string, allowedOutcomes []string) (Outcome string, Bod
if value == len(lines) {
return "", "", fmt.Errorf("%w: outcome heading has no value", ErrMissingOutcome)
}
word := strings.TrimSpace(lines[value].text)
// The value line is the word alone; a trailing period is tolerated.
word := strings.TrimSpace(strings.TrimSuffix(strings.TrimSpace(lines[value].text), "."))
canonical, ok := allowedOutcome(word, allowedOutcomes)
if !ok {
return "", "", fmt.Errorf("%w: %q", ErrOutcomeNotAllowed, word)
@@ -89,6 +90,62 @@ func allowedOutcome(word string, outcomes []string) (string, bool) {
return "", false
}
// WithOutcome returns markdown with a leading "# outcome" section holding the
// given word, replacing any outcome headings (and their value lines) already in
// it. The word is never derived from the text: callers pass an outcome the
// model stated explicitly, such as its reply to the outcome repair follow-up.
// Every other byte of the original document is kept in order.
func WithOutcome(markdown, outcome string) string {
lines := markdownLines(markdown)
drop := make(map[int]bool)
var fence fenceState
for i := 0; i < len(lines); i++ {
text := lines[i].text
if fence.open() {
if fence.closes(text) {
fence = fenceState{}
}
continue
}
if marker, ok := openingFence(text); ok {
fence = marker
continue
}
if !isOutcomeHeading(text) {
continue
}
drop[i] = true
value := i + 1
for value < len(lines) && strings.TrimSpace(lines[value].text) == "" {
value++
}
if value == len(lines) {
continue
}
_, _, heading := atxHeading(lines[value].text)
_, fenced := openingFence(lines[value].text)
if !heading && !fenced {
drop[value] = true
i = value
}
}
var b strings.Builder
b.WriteString("# outcome\n")
b.WriteString(outcome)
b.WriteString("\n\n")
for i, line := range lines {
if drop[i] {
continue
}
start := 0
if i > 0 {
start = lines[i-1].next
}
b.WriteString(markdown[start:line.next])
}
return b.String()
}
type markdownLine struct {
text string
next int
+32
View File
@@ -143,6 +143,38 @@ func TestParseOutputReturnsCanonicalOutcome(t *testing.T) {
}
}
func TestParseOutputToleratesTrailingPeriod(t *testing.T) {
for _, input := range []string{"# outcome\nsuccess.\n", "# outcome\n Success . \n"} {
if outcome, _, err := ParseOutput(input, []string{"success"}); err != nil || outcome != "success" {
t.Fatalf("%q: %q %v", input, outcome, err)
}
}
if _, _, err := ParseOutput("# outcome\nsuccess, mostly\n", []string{"success"}); !errors.Is(err, ErrOutcomeNotAllowed) {
t.Fatalf("prose accepted as outcome: %v", err)
}
}
func TestWithOutcome(t *testing.T) {
tests := []struct{ name, input, want string }{
{"no outcome", "# handoff\n\nx\n", "# outcome\nsuccess\n\n# handoff\n\nx\n"},
{"invalid value replaced", "# handoff\n\nx\n\n# outcome\nmaybe\n\n# log\n", "# outcome\nsuccess\n\n# handoff\n\nx\n\n\n# log\n"},
{"duplicates removed", "## outcome\nerror\n# body\n## outcome\nsuccess\n", "# outcome\nsuccess\n\n# body\n"},
{"empty heading keeps next heading", "# outcome\n\n# handoff\nx\n", "# outcome\nsuccess\n\n\n# handoff\nx\n"},
{"fenced kept", "```\n# outcome\nerror\n```\n", "# outcome\nsuccess\n\n```\n# outcome\nerror\n```\n"},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := WithOutcome(tt.input, "success")
if got != tt.want {
t.Fatalf("WithOutcome = %q, want %q", got, tt.want)
}
if outcome, _, err := ParseOutput(got, []string{"success", "error"}); err != nil || outcome != "success" {
t.Fatalf("result does not parse: %q %v", outcome, err)
}
})
}
}
func TestParseOutputRequiresWhitespaceAfterHeadingMarker(t *testing.T) {
if _, _, err := ParseOutput("#outcome\nsuccess\n", []string{"success"}); !errors.Is(err, ErrMissingOutcome) {
t.Fatalf("error = %v, want missing outcome", err)