diff --git a/docs/model-results.md b/docs/model-results.md index 65300f4a..a0a635f6 100644 --- a/docs/model-results.md +++ b/docs/model-results.md @@ -59,12 +59,21 @@ success The rest of the output is free-form markdown. ``` -Matching ignores case and surrounding whitespace and returns the canonical edge +Matching ignores case, surrounding whitespace and one trailing period, and returns the canonical edge spelling. Headings and words inside fenced code blocks are ignored, so a code sample cannot add a false outcome. Missing, duplicate, empty or undeclared outcomes fail the stage. The parser never guesses, repairs the text, or looks for any other section. +When a turn completes but its final message has no valid outcome, the harness +client (OpenCode or Pi) sends one short follow-up in the same session asking +only for the `# outcome` line and listing the allowed outcomes, without redoing +work. An outcome-only reply is placed at the top of the original output (any +invalid outcome section is replaced); a full reply replaces it. The outcome is +always the model's explicit answer, never inferred from prose. Its tokens count +toward the stage usage. If the reply is still invalid, the stage fails with the +same validation error as before. + Everything after the outcome value line is retained as the turn body under the single `markdown` key of the internal output map. No JSON envelope, output schema, field selection, `output_contract`, or format-repair pass exists for @@ -81,7 +90,8 @@ facts: - the paths to `memory.md`, `input.md` and `output.md` (or their contents for a text-only stage); - the stage prompt verbatim; -- the worked outcome example and the complete allowed outcome list; +- the required shape, outcome first (`# outcome` and one of the allowed + outcomes, then `# handoff`, then any detail), with a worked example; - the execution time and timezone; - a granted resource target, when one exists; - the writable `private/` and shared `public/` artifact directories, when they diff --git a/internal/stage/agent.go b/internal/stage/agent.go index e71b7e6f..007df8cb 100644 --- a/internal/stage/agent.go +++ b/internal/stage/agent.go @@ -241,6 +241,12 @@ func (c *AgentClient) Generate(ctx context.Context, config ModelConfig, prompt s if len(data) > 0 { c.Transcript = string(data) } + if err == nil { + text, data, err = c.repairOutput(ctx, config, c.SessionID, text) + if len(data) > 0 { + c.Transcript = string(data) + } + } // Budgeted stages already settle through the single watch.finish read. For an // unbudgeted V2 stage the watch was never started, so record best-effort final // session usage from exactly one bounded aggregate read; missing/invalid usage diff --git a/internal/stage/model_diagnostic_test.go b/internal/stage/model_diagnostic_test.go index af4128ed..88cf42a6 100644 --- a/internal/stage/model_diagnostic_test.go +++ b/internal/stage/model_diagnostic_test.go @@ -46,13 +46,14 @@ func TestModelFailureCategoriesAreSanitizedAndDistinct(t *testing.T) { // TestAgentPostEditValidationFailureRetainsWorkWithoutReplay reproduces a // tool-using stage that edits once and then returns a nested-duplicate final -// result. The validation failure must not re-prompt the agent, must retain the -// admitted session, transcript and evidence, and must carry the precise category. +// result. The only extra prompt is the single outcome repair follow-up, which +// does not redo work; when the reply is still invalid the stage fails as +// before, retaining the admitted session, transcript, evidence and category. func TestAgentPostEditValidationFailureRetainsWorkWithoutReplay(t *testing.T) { dir := t.TempDir() path := filepath.Join(dir, "README.md") review := "# outcome\nsuccess\n\n# evidence\n\nprivate-evidence\n\n# outcome\nerror\n" - creates, edits, owned := 0, 0, 0 + creates, edits, repairs, owned := 0, 0, 0, 0 agent := &AgentClient{OpenCodeClient: OpenCodeClient{Directory: dir}, OnSession: func(string, bool) error { owned++; return nil }} agent.command = func(_ context.Context, _ string, args ...string) ([]byte, error) { if args[1] == "delete" { @@ -63,6 +64,10 @@ func TestAgentPostEditValidationFailureRetainsWorkWithoutReplay(t *testing.T) { creates++ return []byte(fmt.Sprintf(`{"data":{"id":"ses_%d"}}`, creates)), nil case "/api/session/ses_1/prompt": + if strings.Contains(args[len(args)-1], "no valid `# outcome`") { + repairs++ + return []byte(`{}`), nil + } edits++ if err := os.WriteFile(path, []byte("edited once"), 0600); err != nil { t.Fatal(err) @@ -85,8 +90,8 @@ func TestAgentPostEditValidationFailureRetainsWorkWithoutReplay(t *testing.T) { if strings.Contains(err.Error(), "HARNESS TRANSPORT") || strings.Contains(err.Error(), "private-evidence") { t.Fatalf("validation diagnostic leaked content or a false category: %v", err) } - if creates != 1 || edits != 1 || owned != 2 || agent.SessionID != "ses_1" || result.Output != nil { - t.Fatal("validation failure replayed work or dropped retained ownership", creates, edits, owned, agent.SessionID, result) + if creates != 1 || edits != 1 || repairs != 1 || owned != 2 || agent.SessionID != "ses_1" || result.Output != nil { + t.Fatal("validation failure replayed work or dropped retained ownership", creates, edits, repairs, owned, agent.SessionID, result) } if !json.Valid([]byte(agent.Transcript)) || !strings.Contains(agent.Transcript, "private-evidence") { t.Fatal("retained readback evidence missing", agent.Transcript) diff --git a/internal/stage/model_prompt.go b/internal/stage/model_prompt.go index 456bb5fd..1b7ee346 100644 --- a/internal/stage/model_prompt.go +++ b/internal/stage/model_prompt.go @@ -105,19 +105,40 @@ func modelPrompt(def Definition, spec ModelSpec, input Values) (string, error) { } b.WriteString("\n\nProduce ") b.WriteString(def.Turn.Output) - b.WriteString(" as markdown; return its complete content as your final message (the server writes that file). Start it with a short # handoff section (at most about 1 KB): what you did, files changed, and open issues or findings the next stage must act on. The next stage receives only that section as essential memory; the rest of your output stays in history. The only required section is the outcome, for example:\n\n# handoff\n...\n\n# outcome\n") + b.WriteString(" as markdown and return its complete content as your final message (the server writes that file; no JSON envelope). ") + b.WriteString(outputShape(outcomes)) + b.WriteString(" The next stage receives only the handoff as essential memory. For example:\n\n# outcome\n") b.WriteString(preferredOutcome(outcomes)) - fmt.Fprintf(&b, "\n\nAllowed outcomes: %s.\n\nEverything else is free-form markdown for the next stage and the operator. The server does not parse or validate any other section. Return the complete output.md content as your final message; do not wrap it in a JSON envelope.\n", strings.Join(outcomes, ", ")) + b.WriteString("\n\n# handoff\nWhat you did, files changed, open issues for the next stage.\n\n# details\n...\n") fmt.Fprintf(&b, "\nExecution time: %s. Timezone: %s.\n", time.Now().Format(time.RFC3339), time.Local.String()) if def.Turn.Public != "" { fmt.Fprintf(&b, "\nShared artifacts go to %s; files there are visible to every later stage. That absolute path is the shared folder: a relative public/ inside the workspace is a different, unshared directory. Turn-local work goes to %s and is not handed to later stages.\n", def.Turn.Public, def.Turn.Private) } b.WriteString("\n") b.WriteString(toolsLine) - b.WriteString(" Do not invent file edits, command results, screenshots, or observations. If the task requires unavailable tools or evidence, choose the stage's error outcome and explain the limitation. Never claim unperformed work succeeded.") + b.WriteString(" Do not invent file edits, command results, screenshots, or observations. If the task requires unavailable tools or evidence, ") + b.WriteString(limitationOutcome(outcomes)) + b.WriteString(" and explain the limitation. Never claim unperformed work succeeded.") return b.String(), nil } +// outputShape is the one sentence stating the required output.md layout, +// outcome first. The repair follow-up reuses it so both say the same thing. +func outputShape(outcomes []string) string { + return "output.md must start with `# outcome` and the outcome word alone on the next line, one of: " + strings.Join(outcomes, ", ") + "; then `# handoff` (a few lines); then any detail." +} + +// limitationOutcome names the outcome for missing tools or evidence: blocked +// when the stage declares it, otherwise error. +func limitationOutcome(outcomes []string) string { + for _, want := range []string{"blocked", "error"} { + if slices.Contains(outcomes, want) { + return "choose `" + want + "`" + } + } + return "choose the stage's error outcome" +} + // serverChangedFiles reads the runner's reserved changed-path list from stage // input, accepting the []string the runner supplies and the []any JSON decoding // produces for a replayed handoff. Other values are ignored. diff --git a/internal/stage/model_repair.go b/internal/stage/model_repair.go new file mode 100644 index 00000000..db75e888 --- /dev/null +++ b/internal/stage/model_repair.go @@ -0,0 +1,88 @@ +package stage + +import ( + "context" + "errors" + "strings" + + "git.a-wan.net/alan/pipelines/internal/turn" +) + +// outputRepair is the single outcome repair follow-up. When a turn completes +// but its final message lacks a valid outcome, the client sends one short +// message in the same session asking only for the outcome, instead of failing +// a stage whose admitted work a full retry would repeat. The outcome is always +// stated by the model; the server never infers one from prose. +type outputRepair struct{ outcomes []string } + +type outputRepairKey struct{} + +// withOutputRepair lets a model client repair a missing or invalid outcome +// for the given declared outcomes while its session is still open. +func withOutputRepair(ctx context.Context, outcomes []string) context.Context { + return context.WithValue(ctx, outputRepairKey{}, &outputRepair{outcomes: outcomes}) +} + +// outputRepairFrom returns the attempt's repair, or nil when none applies. +func outputRepairFrom(ctx context.Context) *outputRepair { + repair, _ := ctx.Value(outputRepairKey{}).(*outputRepair) + return repair +} + +// followUp reports whether text needs the outcome follow-up and returns its +// message. Valid and empty outputs need none: an empty output has no report +// to keep, so it fails as before. +func (r *outputRepair) followUp(text string) (string, bool) { + if r == nil || len(r.outcomes) == 0 || strings.TrimSpace(text) == "" { + return "", false + } + _, _, err := turn.ParseOutput(text, r.outcomes) + if !errors.Is(err, turn.ErrMissingOutcome) && !errors.Is(err, turn.ErrOutcomeNotAllowed) && !errors.Is(err, turn.ErrDuplicateOutcome) { + return "", false + } + allowed := strings.Join(r.outcomes, ", ") + return "Your final message has no valid `# outcome` section. Do not redo any work or repeat your report; the server keeps it. " + + outputShape(r.outcomes) + " Reply with only these two lines:\n\n# outcome\n\n", true +} + +// merge combines the original output with the follow-up reply. An +// outcome-only reply is placed at the top of the original report, replacing +// any invalid outcome section. A reply carrying a full document replaces the +// original. An invalid reply keeps the original, so validation fails as before. +func (r *outputRepair) merge(original, reply string) string { + outcome, body, err := turn.ParseOutput(reply, r.outcomes) + if err != nil { + return original + } + if strings.TrimSpace(body) == "" { + return turn.WithOutcome(original, outcome) + } + return reply +} + +// repairOutput sends the outcome follow-up in an open OpenCode session when +// text needs it, waits for that turn and returns the merged output with the +// raw final readback. Tokens it uses accrue to the same session, so the +// existing budget watch and session usage read count them. +func (c OpenCodeClient) repairOutput(ctx context.Context, config ModelConfig, sessionID, text string) (string, []byte, error) { + repair := outputRepairFrom(ctx) + message, ok := repair.followUp(text) + if !ok { + return text, nil, nil + } + if _, err := c.call(ctx, "post", "/api/session/"+sessionID+"/prompt", generationBody("text", message, config, c.Backend)); err != nil { + return "", nil, err + } + if c.Backend == "v1" { + if _, err := c.call(ctx, "post", "/api/experimental/session/"+sessionID+"/wait", nil); err != nil { + return "", nil, err + } + } else if err := c.waitForSessionIdle(ctx, sessionID); err != nil { + return "", nil, err + } + data, reply, err := c.readFinalAgentResult(ctx, sessionID) + if err != nil { + return "", data, err + } + return repair.merge(text, reply), data, nil +} diff --git a/internal/stage/model_repair_test.go b/internal/stage/model_repair_test.go new file mode 100644 index 00000000..4ba6d178 --- /dev/null +++ b/internal/stage/model_repair_test.go @@ -0,0 +1,171 @@ +package stage + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "os" + "strings" + "testing" + "time" + + "git.a-wan.net/alan/pipelines/internal/turn" +) + +// repairAgent serves one V2 agent session whose first final message is +// handoffOnly and whose follow-up reply is reply. It records the call order. +func repairAgent(handoffOnly, reply string, calls *[]string) *AgentClient { + prompts := 0 + return &AgentClient{OpenCodeClient: OpenCodeClient{Backend: "v2", sessionPollInterval: time.Millisecond, command: func(_ context.Context, _ string, args ...string) ([]byte, error) { + path := args[2] + switch { + case path == "/api/session" && args[1] == "post": + return []byte(`{"data":{"id":"ses_1"}}`), nil + case path == "/api/session/ses_1/prompt": + prompts++ + body := args[len(args)-1] + if prompts == 2 && !strings.Contains(body, "no valid `# outcome`") { + return nil, fmt.Errorf("second prompt is not the repair follow-up: %s", body) + } + *calls = append(*calls, fmt.Sprintf("prompt%d", prompts)) + return []byte(`{}`), nil + case path == "/api/session/active": + return []byte(`{"data":{}}`), nil + case strings.HasSuffix(path, finalAssistantReadback): + text := handoffOnly + if prompts == 2 { + text = reply + } + return json.Marshal(map[string]any{"data": []any{map[string]any{"finish": "stop", "content": []any{map[string]string{"type": "text", "text": text}}}}}) + case path == "/api/session/ses_1": + *calls = append(*calls, "usage") + return usageResponse("ses_1", .2, 42), nil + case strings.HasSuffix(path, "/interrupt"): + return []byte(`{}`), nil + } + return nil, fmt.Errorf("unexpected path %s", path) + }}} +} + +// TestHandoffOnlyOutputIsRepairedInSession reproduces runs 241/243: the model +// starts with # handoff and omits the outcome. One follow-up in the same +// session asks only for the outcome; its reply is used and the report kept. +func TestHandoffOnlyOutputIsRepairedInSession(t *testing.T) { + report := "# handoff\n\nImplemented cache.go; tests pass.\n\n# details\n\nLong report.\n" + var calls []string + agent := repairAgent(report, "# outcome\nsuccess\n", &calls) + def := turnDefinition(t) + def.Blocks["tools"] = "edit" + budget := NewBudget(0, 0) + result, err := (ModelStage{Client: agent}).Run(WithBudget(context.Background(), budget), def, nil) + if err != nil || result.Outcome != "success" { + t.Fatalf("repair did not recover the outcome: %v %v", result, err) + } + written, _ := os.ReadFile(def.Turn.Output) + if string(written) != "# outcome\nsuccess\n\n"+report { + t.Fatalf("output.md = %q", written) + } + if body := result.Output["markdown"].(string); !strings.Contains(body, "Implemented cache.go") || !strings.Contains(body, "Long report.") { + t.Fatalf("report dropped: %q", body) + } + // The usage read follows the follow-up, so its tokens are counted. + if strings.Join(calls, ",") != "prompt1,prompt2,usage" || budget.Snapshot().UsedTokens != 42 { + t.Fatalf("calls=%v usage=%+v", calls, budget.Snapshot()) + } +} + +// TestFailedRepairFailsAsBefore proves a follow-up that still lacks a valid +// outcome fails with today's missing-outcome error and keeps the report. +func TestFailedRepairFailsAsBefore(t *testing.T) { + report := "# handoff\n\nDid the work.\n" + for _, reply := range []string{"I think it went well.\n", "# outcome\nmaybe\n"} { + var calls []string + def := turnDefinition(t) + def.Blocks["tools"] = "edit" + _, err := (ModelStage{Client: repairAgent(report, reply, &calls)}).Run(context.Background(), def, nil) + if !errors.Is(err, ErrInvalidModelResult) || !strings.Contains(err.Error(), turn.ErrMissingOutcome.Error()) || !strings.Contains(err.Error(), "FINAL RESULT VALIDATION") { + t.Fatalf("reply %q: err = %v", reply, err) + } + if strings.Count(strings.Join(calls, ","), "prompt") != 2 { + t.Fatalf("expected exactly one follow-up: %v", calls) + } + if written, _ := os.ReadFile(def.Turn.Output); string(written) != report { + t.Fatalf("output.md = %q", written) + } + } +} + +// TestValidOutputSendsNoFollowUp keeps a valid first answer to one prompt. +func TestValidOutputSendsNoFollowUp(t *testing.T) { + var calls []string + def := turnDefinition(t) + def.Blocks["tools"] = "edit" + result, err := (ModelStage{Client: repairAgent("# outcome\nerror\n\n# handoff\n\nblocked\n", "", &calls)}).Run(context.Background(), def, nil) + if err != nil || result.Outcome != "error" || strings.Count(strings.Join(calls, ","), "prompt") != 1 { + t.Fatalf("%v %v %v", result, err, calls) + } +} + +// TestModelPromptStatesOutcomeFirst checks the required shape: outcome first, +// then handoff, then detail, with the allowed outcomes listed. +func TestModelPromptStatesOutcomeFirst(t *testing.T) { + def := modelDefinition() + spec, err := (ModelStage{}).Spec(def) + if err != nil { + t.Fatal(err) + } + prompt, err := modelPrompt(def, spec, nil) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(prompt, "output.md must start with `# outcome` and the outcome word alone on the next line, one of: error, success; then `# handoff` (a few lines); then any detail.") { + t.Fatalf("shape sentence missing: %s", prompt) + } + outcome, handoff := strings.Index(prompt, "\n# outcome\n"), strings.Index(prompt, "\n# handoff\n") + if outcome < 0 || handoff < 0 || outcome > handoff { + t.Fatalf("example does not put the outcome before the handoff: %s", prompt) + } + if strings.Contains(prompt, "Start it with a short # handoff") { + t.Fatal("prompt still asks to start with the handoff") + } + if !strings.Contains(prompt, "unavailable tools or evidence, choose `error` and explain") { + t.Fatalf("limitation outcome not error: %s", prompt) + } +} + +func TestLimitationOutcomePrefersBlocked(t *testing.T) { + for outcomes, want := range map[string]string{ + "blocked,error,success": "choose `blocked`", + "error,success": "choose `error`", + "done,retry": "choose the stage's error outcome", + } { + if got := limitationOutcome(strings.Split(outcomes, ",")); got != want { + t.Fatalf("%s: %q, want %q", outcomes, got, want) + } + } +} + +func TestRepairMerge(t *testing.T) { + r := &outputRepair{outcomes: []string{"error", "success"}} + if _, ok := r.followUp("# outcome\nsuccess\n"); ok { + t.Fatal("valid output needs no follow-up") + } + if _, ok := r.followUp(" \n"); ok { + t.Fatal("empty output is not repaired") + } + message, ok := r.followUp("# handoff\n\nx\n") + if !ok || !strings.Contains(message, "error, success") || !strings.Contains(message, "Do not redo any work") { + t.Fatalf("follow-up = %q", message) + } + if got := r.merge("# handoff\n\nx\n", "# outcome\nSuccess\n"); got != "# outcome\nsuccess\n\n# handoff\n\nx\n" { + t.Fatalf("outcome-only merge = %q", got) + } + full := "# outcome\nerror\n\n# handoff\n\ny\n" + if got := r.merge("# handoff\n\nx\n", full); got != full { + t.Fatalf("full reply merge = %q", got) + } + if got := r.merge("# handoff\n\nx\n", "no outcome here"); got != "# handoff\n\nx\n" { + t.Fatalf("invalid reply merge = %q", got) + } +} diff --git a/internal/stage/model_run.go b/internal/stage/model_run.go index 774e39e9..8839b4c7 100644 --- a/internal/stage/model_run.go +++ b/internal/stage/model_run.go @@ -61,7 +61,9 @@ func (m ModelStage) Run(ctx context.Context, def Definition, input Values) (Resu } spec.Config.Budget.Attempt() attemptCtx, cancel := context.WithTimeoutCause(ctx, spec.Config.Timeout, &stageTimeoutCause{timeout: spec.Config.Timeout}) - text, generateErr := client.Generate(attemptCtx, spec.Config, prompt) + // A completed turn without a valid outcome gets one in-session follow-up + // asking only for the outcome, rather than a full retry of its work. + text, generateErr := client.Generate(withOutputRepair(attemptCtx, outcomesFor(def.Edges)), spec.Config, prompt) if generateErr == nil && attemptCtx.Err() != nil { generateErr = attemptCtx.Err() } diff --git a/internal/stage/model_test.go b/internal/stage/model_test.go index 4d246b90..8d85b22c 100644 --- a/internal/stage/model_test.go +++ b/internal/stage/model_test.go @@ -78,7 +78,7 @@ func TestModelPromptUsesTurnFilesAndNoJSONEnvelope(t *testing.T) { "Do the task.", "# outcome", "success", - "free-form markdown", + "must start with `# outcome`", } { if !strings.Contains(prompt, want) { t.Fatalf("prompt missing %q:\n%s", want, prompt) diff --git a/internal/stage/opencode.go b/internal/stage/opencode.go index f47643e2..3aa964d0 100644 --- a/internal/stage/opencode.go +++ b/internal/stage/opencode.go @@ -431,6 +431,13 @@ func (c *OpenCodeClient) Generate(ctx context.Context, config ModelConfig, promp if err != nil { return "", err } + text, data, err = c.repairOutput(ctx, config, session.Data.ID, text) + if len(data) > 0 { + c.FormatAudit = string(data) + } + if err != nil { + return "", err + } if !config.Budget.Enabled() { // As with AgentClient, missing usage is not a generation failure when // no limit is enabled (and a nil ledger needs no aggregate read). diff --git a/internal/stage/pi.go b/internal/stage/pi.go index 6228509f..c13b4353 100644 --- a/internal/stage/pi.go +++ b/internal/stage/pi.go @@ -329,8 +329,31 @@ func (c *PiClient) Generate(ctx context.Context, config ModelConfig, prompt stri if _, err = c.rpc(ctx, "prompt", map[string]any{"message": prompt}); err != nil { return "", err } + if text, err = c.settleTurn(ctx); err != nil { + return "", err + } + // One outcome repair follow-up in the same Pi session; get_session_stats + // reports session totals, so its tokens are counted by the re-read. + repair := outputRepairFrom(ctx) + if message, ok := repair.followUp(text); ok { + c.settled = false + if _, err = c.rpc(ctx, "prompt", map[string]any{"message": message}); err != nil { + return "", err + } + reply, err := c.settleTurn(ctx) + if err != nil { + return "", err + } + text = repair.merge(text, reply) + } + return text, nil +} + +// settleTurn waits for the prompted Pi turn to settle, records its session +// usage and returns the final assistant text. +func (c *PiClient) settleTurn(ctx context.Context) (string, error) { for !c.settled { - if _, err = c.receive(ctx); err != nil { + if _, err := c.receive(ctx); err != nil { return "", err } } diff --git a/internal/stage/pi_test.go b/internal/stage/pi_test.go index 4f83c75e..c4a9ae3d 100644 --- a/internal/stage/pi_test.go +++ b/internal/stage/pi_test.go @@ -30,6 +30,12 @@ done if os.Getenv("PI_FIXTURE_CASE") == "missing" { delete(message, "usage") } + // repair: the first turn drops its outcome; the follow-up states only it. + repair := os.Getenv("PI_FIXTURE_CASE") == "repair" + prompts, total := 0, 0 + if repair { + message["content"] = []map[string]string{{"type": "text", "text": "# handoff\n\nReport kept.\n"}} + } emit := func(v any) { data, _ := json.Marshal(v); fmt.Println(string(data)) } scanner := bufio.NewScanner(os.Stdin) for scanner.Scan() { @@ -45,12 +51,17 @@ done emit(map[string]any{"type": "response", "id": request["id"], "success": true}) continue } + prompts++ + total += 10 + if repair && prompts == 2 { + message = map[string]any{"role": "assistant", "stopReason": "stop", "timestamp": 2, "content": []map[string]string{{"type": "text", "text": "# outcome\nsuccess\n"}}, "usage": map[string]any{"totalTokens": 10, "cost": map[string]any{"total": 0.1}}} + } emit(map[string]any{"type": "message_end", "message": message}) emit(map[string]string{"type": "agent_settled"}) case "abort": emit(map[string]string{"type": "agent_settled"}) case "get_session_stats": - data = map[string]any{"sessionId": sessionID, "assistantMessages": 1, "tokens": map[string]int{"total": 10}, "cost": 0.1} + data = map[string]any{"sessionId": sessionID, "assistantMessages": max(prompts, 1), "tokens": map[string]int{"total": max(total, 10)}, "cost": 0.1 * float64(max(prompts, 1))} case "get_messages": data = map[string]any{"messages": []any{message}} } @@ -60,7 +71,7 @@ done } func TestPiAuditedRPCAndBudgets(t *testing.T) { - for _, which := range []string{"success", "budget", "missing", "hang"} { + for _, which := range []string{"success", "budget", "missing", "hang", "repair"} { t.Run(which, func(t *testing.T) { root := t.TempDir() t.Setenv("XDG_CACHE_HOME", root) @@ -87,12 +98,20 @@ func TestPiAuditedRPCAndBudgets(t *testing.T) { ctx, cancel = context.WithTimeout(context.Background(), 200*time.Millisecond) } defer cancel() + if which == "repair" { + ctx = withOutputRepair(ctx, []string{"error", "success"}) + } text, err := client.Generate(ctx, ModelConfig{Provider: "fixture", Model: "model", Budget: budget}, "task") switch which { case "success": if err != nil || text == "" || budget.Snapshot().UsedTokens != 10 { t.Fatal(text, err, budget.Snapshot()) } + case "repair": + // The follow-up's tokens count: two turns of 10 each. + if err != nil || text != "# outcome\nsuccess\n\n# handoff\n\nReport kept.\n" || budget.Snapshot().UsedTokens != 20 { + t.Fatal(text, err, budget.Snapshot()) + } case "budget": if !errors.Is(err, ErrBudgetExceeded) { t.Fatal(err) diff --git a/internal/turn/output.go b/internal/turn/output.go index 3dfc0ebb..4e176706 100644 --- a/internal/turn/output.go +++ b/internal/turn/output.go @@ -64,7 +64,8 @@ func ParseOutput(markdown string, allowedOutcomes []string) (Outcome string, Bod if value == len(lines) { return "", "", fmt.Errorf("%w: outcome heading has no value", ErrMissingOutcome) } - word := strings.TrimSpace(lines[value].text) + // The value line is the word alone; a trailing period is tolerated. + word := strings.TrimSpace(strings.TrimSuffix(strings.TrimSpace(lines[value].text), ".")) canonical, ok := allowedOutcome(word, allowedOutcomes) if !ok { return "", "", fmt.Errorf("%w: %q", ErrOutcomeNotAllowed, word) @@ -89,6 +90,62 @@ func allowedOutcome(word string, outcomes []string) (string, bool) { return "", false } +// WithOutcome returns markdown with a leading "# outcome" section holding the +// given word, replacing any outcome headings (and their value lines) already in +// it. The word is never derived from the text: callers pass an outcome the +// model stated explicitly, such as its reply to the outcome repair follow-up. +// Every other byte of the original document is kept in order. +func WithOutcome(markdown, outcome string) string { + lines := markdownLines(markdown) + drop := make(map[int]bool) + var fence fenceState + for i := 0; i < len(lines); i++ { + text := lines[i].text + if fence.open() { + if fence.closes(text) { + fence = fenceState{} + } + continue + } + if marker, ok := openingFence(text); ok { + fence = marker + continue + } + if !isOutcomeHeading(text) { + continue + } + drop[i] = true + value := i + 1 + for value < len(lines) && strings.TrimSpace(lines[value].text) == "" { + value++ + } + if value == len(lines) { + continue + } + _, _, heading := atxHeading(lines[value].text) + _, fenced := openingFence(lines[value].text) + if !heading && !fenced { + drop[value] = true + i = value + } + } + var b strings.Builder + b.WriteString("# outcome\n") + b.WriteString(outcome) + b.WriteString("\n\n") + for i, line := range lines { + if drop[i] { + continue + } + start := 0 + if i > 0 { + start = lines[i-1].next + } + b.WriteString(markdown[start:line.next]) + } + return b.String() +} + type markdownLine struct { text string next int diff --git a/internal/turn/output_test.go b/internal/turn/output_test.go index a42dc92b..a6c7cfc9 100644 --- a/internal/turn/output_test.go +++ b/internal/turn/output_test.go @@ -143,6 +143,38 @@ func TestParseOutputReturnsCanonicalOutcome(t *testing.T) { } } +func TestParseOutputToleratesTrailingPeriod(t *testing.T) { + for _, input := range []string{"# outcome\nsuccess.\n", "# outcome\n Success . \n"} { + if outcome, _, err := ParseOutput(input, []string{"success"}); err != nil || outcome != "success" { + t.Fatalf("%q: %q %v", input, outcome, err) + } + } + if _, _, err := ParseOutput("# outcome\nsuccess, mostly\n", []string{"success"}); !errors.Is(err, ErrOutcomeNotAllowed) { + t.Fatalf("prose accepted as outcome: %v", err) + } +} + +func TestWithOutcome(t *testing.T) { + tests := []struct{ name, input, want string }{ + {"no outcome", "# handoff\n\nx\n", "# outcome\nsuccess\n\n# handoff\n\nx\n"}, + {"invalid value replaced", "# handoff\n\nx\n\n# outcome\nmaybe\n\n# log\n", "# outcome\nsuccess\n\n# handoff\n\nx\n\n\n# log\n"}, + {"duplicates removed", "## outcome\nerror\n# body\n## outcome\nsuccess\n", "# outcome\nsuccess\n\n# body\n"}, + {"empty heading keeps next heading", "# outcome\n\n# handoff\nx\n", "# outcome\nsuccess\n\n\n# handoff\nx\n"}, + {"fenced kept", "```\n# outcome\nerror\n```\n", "# outcome\nsuccess\n\n```\n# outcome\nerror\n```\n"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := WithOutcome(tt.input, "success") + if got != tt.want { + t.Fatalf("WithOutcome = %q, want %q", got, tt.want) + } + if outcome, _, err := ParseOutput(got, []string{"success", "error"}); err != nil || outcome != "success" { + t.Fatalf("result does not parse: %q %v", outcome, err) + } + }) + } +} + func TestParseOutputRequiresWhitespaceAfterHeadingMarker(t *testing.T) { if _, _, err := ParseOutput("#outcome\nsuccess\n", []string{"success"}); !errors.Is(err, ErrMissingOutcome) { t.Fatalf("error = %v, want missing outcome", err)