From b05277a57d64074fdfb3707cdfce45deaabf0711 Mon Sep 17 00:00:00 2001 From: Alan Silva Date: Fri, 16 Oct 2026 21:34:33 +0100 Subject: [PATCH] Add line-oriented result format for model and agent stages Introduce a short, markdown-ish result format whose first non-empty line is the declared outcome and whose "# output" section carries the routing data, so stage output stops carrying multi-kilobyte prose inside a JSON envelope. - New internal/stage/line_result.go: resultFormat() gates legacy JSON ('{' first) vs the new line format; decodeLineResult() parses outcome-from-first line, requires a JSON-object "# output" section, stores extra sections verbatim under normalised keys, and records a reserved result_parse status (parsed/partial/failed). Failures return a salvage Result plus ErrInvalidModelResult, and never route an undeclared outcome. - internal/stage/model.go: write the raw response to raw-output.md (0600, best-effort) before parsing, and route the line format through decodeLineResult while leaving the JSON path (stageResultText -> decodeModelResult -> repairModelResult) unchanged. - internal/stage/agent.go: agentResultText returns line-format text verbatim so the "# output" body is not stripped by the JSON-object extractor. - internal/server/runner.go: merge a non-empty salvage Result into the error envelope under "salvage" on failure (a no-op for the legacy JSON path). - internal/stage/line_result_test.go: raw-text fixtures covering happy path, missing/undeclared outcome, missing "# output", best-effort extras, adversarial prose, legacy JSON unchanged, >=64KB responses, and raw-output.md written before parsing. --- internal/server/runner.go | 3 + internal/stage/agent.go | 5 + internal/stage/line_result.go | 229 ++++++++++++++++++++++++++++ internal/stage/line_result_test.go | 232 +++++++++++++++++++++++++++++ internal/stage/model.go | 13 +- 5 files changed, 479 insertions(+), 3 deletions(-) create mode 100644 internal/stage/line_result.go create mode 100644 internal/stage/line_result_test.go diff --git a/internal/server/runner.go b/internal/server/runner.go index bbb6b324..f1f42ff3 100644 --- a/internal/server/runner.go +++ b/internal/server/runner.go @@ -680,6 +680,9 @@ func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, na step.Diagnostic = err.Error() step.Outcome = "error" step.Output = stage.Values{"error": err.Error(), "stage": key, "input": input} + if len(result.Output) > 0 { + step.Output["salvage"] = result.Output + } } // A failed or timed-out step carries the session pointer even when the // bounded transcript would otherwise be only its stage_limits envelope. diff --git a/internal/stage/agent.go b/internal/stage/agent.go index 18f35dad..2d3aa1ea 100644 --- a/internal/stage/agent.go +++ b/internal/stage/agent.go @@ -276,6 +276,11 @@ func completedAgentResult(data []byte) (string, error) { // Agents may emit a short explanation before their final result. Preserve the // full response in the audit, but pass only a complete final JSON object to the // same strict stage validator. Never infer outcomes from prose or tool output. +// The line-oriented format is returned verbatim so its '# output' body is not +// stripped by the JSON-object extractor. func agentResultText(text string) string { + if resultFormat(text) == "line" { + return text + } return stageResultText(text) } diff --git a/internal/stage/line_result.go b/internal/stage/line_result.go new file mode 100644 index 00000000..7b99d3aa --- /dev/null +++ b/internal/stage/line_result.go @@ -0,0 +1,229 @@ +package stage + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "strings" + "unicode" +) + +// resultFormat decides which decoder to use for a raw stage response. +// +// The legacy JSON path is kept for responses that look like JSON envelopes, +// including presentation wrappers (a single surrounding markdown fence or a +// short prose prefix before the JSON object). The line-oriented format is +// chosen only when a section heading ("# ") appears before any JSON object, +// because the line format's required "# output" section always precedes the +// JSON body. This keeps existing JSON responses, including fenced or +// prose-prefixed results, on the unchanged decodeModelResult path. +func resultFormat(raw string) string { + trimmed := strings.TrimSpace(raw) + // Strip a single surrounding markdown fence so fenced JSON still routes + // through the unchanged legacy path. + for _, prefix := range []string{"```json\n", "```\n"} { + if strings.HasPrefix(trimmed, prefix) && strings.HasSuffix(trimmed, "\n```") { + trimmed = strings.TrimSuffix(strings.TrimPrefix(trimmed, prefix), "\n```") + trimmed = strings.TrimSpace(trimmed) + break + } + } + if len(trimmed) > 0 && trimmed[0] == '{' { + return "json" + } + heading := headingLineIndex(trimmed) + brace := strings.IndexByte(trimmed, '{') + if heading >= 0 && (brace < 0 || heading < brace) { + return "line" + } + return "json" +} + +// headingLineIndex returns the index of the first "# " heading line, or -1. +func headingLineIndex(s string) int { + if strings.HasPrefix(s, "# ") { + return 0 + } + if i := strings.Index(s, "\n# "); i >= 0 { + return i + 1 + } + return -1 +} + +// lineResultStatus records how completely the line-oriented response was parsed. +// It is stored in Result.Output under the reserved key resultParseKey. +const ( + lineParseParsed = "parsed" + lineParsePartial = "partial" + lineParseFailed = "failed" +) + +const resultParseKey = "result_parse" + +// writeRawOutputFile persists the full raw response to the stage artifacts +// directory before any parsing, so the closest record is always retained. +// It is best-effort: empty ArtifactsDir or a write error is ignored. +func writeRawOutputFile(artifactsDir, text string) { + if artifactsDir == "" { + return + } + _ = os.WriteFile(filepath.Join(artifactsDir, "raw-output.md"), []byte(text), 0600) +} + +type lineSection struct { + heading string + lines []string +} + +func (s lineSection) body() string { + return trimOneTrailingNewline(strings.Join(s.lines, "\n")) +} + +// decodeLineResult parses the line-oriented result format. +// +// The first non-empty line is the outcome. It must be one of the declared edges. +// Subsequent lines beginning with "# " start sections. The required "# output" +// section must contain a JSON object. Every other section is stored verbatim +// under a normalised key derived from its heading. +// +// The function never returns a nil Result: on failure it returns whatever was +// salvaged plus a non-nil error. The returned status string is one of +// lineParseParsed, lineParsePartial or lineParseFailed and is also inserted into +// Result.Output under resultParseKey. +func decodeLineResult(raw string, edges map[string]string) (Result, string, error) { + status := lineParseParsed + result := Result{Output: Values{}} + + lines := strings.Split(raw, "\n") + i := 0 + for i < len(lines) && strings.TrimSpace(lines[i]) == "" { + i++ + } + if i >= len(lines) { + result.Output[resultParseKey] = lineParseFailed + return result, lineParseFailed, fmt.Errorf("%w: missing outcome", ErrInvalidModelResult) + } + + outcome := strings.TrimSpace(lines[i]) + i++ + + sections := parseLineSections(lines, i) + + if strings.HasPrefix(outcome, "# ") { + // The first non-empty line is a section heading, not an outcome. + salvage := Result{Output: Values{}} + addLineSectionsToOutput(salvage.Output, sections, &status) + salvage.Output[resultParseKey] = lineParseFailed + return salvage, lineParseFailed, fmt.Errorf("%w: missing outcome", ErrInvalidModelResult) + } + + if _, ok := edges[outcome]; !ok { + // Salvage any extra sections but do not route: the outcome is not declared. + salvage := Result{Output: Values{}} + addLineSectionsToOutput(salvage.Output, sections, &status) + salvage.Output[resultParseKey] = lineParseFailed + if outcome == "" { + return salvage, lineParseFailed, fmt.Errorf("%w: missing outcome", ErrInvalidModelResult) + } + return salvage, lineParseFailed, fmt.Errorf("%w: undeclared outcome", ErrInvalidModelResult) + } + result.Outcome = outcome + + addLineSectionsToOutput(result.Output, sections, &status) + + outputSection := findLineSection(sections, "output") + if outputSection == nil { + result.Output[resultParseKey] = lineParseFailed + return result, lineParseFailed, fmt.Errorf("%w: missing output section", ErrInvalidModelResult) + } + + var output Values + if err := json.Unmarshal([]byte(outputSection.body()), &output); err != nil { + result.Output[resultParseKey] = lineParseFailed + return result, lineParseFailed, fmt.Errorf("%w: output must be an object", ErrInvalidModelResult) + } + if output == nil { + result.Output[resultParseKey] = lineParseFailed + return result, lineParseFailed, fmt.Errorf("%w: output must be an object", ErrInvalidModelResult) + } + for k, v := range output { + result.Output[k] = v + } + + result.Output[resultParseKey] = status + return result, status, nil +} + +func parseLineSections(lines []string, start int) []lineSection { + var sections []lineSection + var current *lineSection + for i := start; i < len(lines); i++ { + line := lines[i] + if strings.HasPrefix(line, "# ") { + if current != nil { + sections = append(sections, *current) + } + current = &lineSection{heading: strings.TrimSpace(line[2:])} + continue + } + if current != nil { + current.lines = append(current.lines, line) + } + } + if current != nil { + sections = append(sections, *current) + } + return sections +} + +func findLineSection(sections []lineSection, name string) *lineSection { + for i := range sections { + if normaliseHeading(sections[i].heading) == name { + return §ions[i] + } + } + return nil +} + +func addLineSectionsToOutput(out Values, sections []lineSection, status *string) { + seen := map[string]bool{} + for _, s := range sections { + key := normaliseHeading(s.heading) + if key == "output" { + continue + } + if key == resultParseKey || key == "" || seen[key] { + *status = lineParsePartial + continue + } + seen[key] = true + out[key] = s.body() + } +} + +func trimOneTrailingNewline(s string) string { + if strings.HasSuffix(s, "\n") { + return s[:len(s)-1] + } + return s +} + +func normaliseHeading(s string) string { + s = strings.ToLower(strings.TrimSpace(s)) + var b strings.Builder + b.Grow(len(s)) + inSpace := false + for _, r := range s { + if unicode.IsSpace(r) { + inSpace = true + continue + } + if inSpace && b.Len() > 0 { + b.WriteByte('_') + } + inSpace = false + b.WriteRune(r) + } + return b.String() +} diff --git a/internal/stage/line_result_test.go b/internal/stage/line_result_test.go new file mode 100644 index 00000000..1424f2ec --- /dev/null +++ b/internal/stage/line_result_test.go @@ -0,0 +1,232 @@ +package stage + +import ( + "context" + "errors" + "os" + "path/filepath" + "strings" + "testing" +) + +func lineEdges() map[string]string { + return map[string]string{"success": "#ok", "error": "#fail", "rejected": "#rejected"} +} + +func TestResultFormatPrecedence(t *testing.T) { + for _, tc := range []struct { + raw string + want string + }{ + {`{"outcome":"success","output":{}}`, "json"}, + {" \n\n{\"outcome\":\"success\"}", "json"}, + {"success\n# output\n{}", "line"}, + {" success\n# output\n{}", "line"}, + {"# output\n{}", "line"}, + } { + if got := resultFormat(tc.raw); got != tc.want { + t.Errorf("resultFormat(%q) = %q, want %q", tc.raw, got, tc.want) + } + } +} + +func TestDecodeLineResultHappyPath(t *testing.T) { + raw := "rejected\n\n# output\n{\"task\":\"x\"}\n\n# findings\n- BLOCKER: prose\n\n# verification evidence\n{raw json}\n" + result, status, err := decodeLineResult(raw, lineEdges()) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if status != lineParseParsed { + t.Fatalf("status = %q, want parsed", status) + } + if result.Outcome != "rejected" { + t.Fatalf("outcome = %q, want rejected", result.Outcome) + } + if result.Output["task"] != "x" { + t.Fatalf("output task = %v", result.Output["task"]) + } + if got, want := result.Output["findings"], "- BLOCKER: prose"; got != want { + t.Fatalf("findings = %q, want %q", got, want) + } + if got, want := result.Output["verification_evidence"], "{raw json}"; got != want { + t.Fatalf("verification_evidence = %q, want %q", got, want) + } + if result.Output[resultParseKey] != lineParseParsed { + t.Fatalf("result_parse = %v", result.Output[resultParseKey]) + } +} + +func TestDecodeLineResultMissingOutcome(t *testing.T) { + raw := "\n\n# output\n{\"task\":\"x\"}\n" + result, status, err := decodeLineResult(raw, lineEdges()) + if err == nil || !errors.Is(err, ErrInvalidModelResult) || !strings.Contains(err.Error(), "missing outcome") { + t.Fatalf("expected missing-outcome error, got %v", err) + } + if status != lineParseFailed { + t.Fatalf("status = %q, want failed", status) + } + if result.Output[resultParseKey] != lineParseFailed { + t.Fatalf("result_parse = %v", result.Output[resultParseKey]) + } +} + +func TestDecodeLineResultUndeclaredOutcome(t *testing.T) { + raw := "bogus\n\n# output\n{\"task\":\"x\"}\n# findings\nkept\n" + result, status, err := decodeLineResult(raw, lineEdges()) + if err == nil || !errors.Is(err, ErrInvalidModelResult) || !strings.Contains(err.Error(), "undeclared outcome") { + t.Fatalf("expected undeclared-outcome error, got %v", err) + } + if status != lineParseFailed { + t.Fatalf("status = %q, want failed", status) + } + if result.Outcome != "" { + t.Fatalf("outcome = %q, want empty", result.Outcome) + } + if got, want := result.Output["findings"], "kept"; got != want { + t.Fatalf("findings salvage = %q, want %q", got, want) + } +} + +func TestDecodeLineResultMissingOutput(t *testing.T) { + raw := "success\n\n# findings\nkept\n" + result, status, err := decodeLineResult(raw, lineEdges()) + if err == nil || !errors.Is(err, ErrInvalidModelResult) { + t.Fatalf("expected error, got %v", err) + } + if status != lineParseFailed { + t.Fatalf("status = %q, want failed", status) + } + if got, want := result.Output["findings"], "kept"; got != want { + t.Fatalf("findings salvage = %q, want %q", got, want) + } +} + +func TestDecodeLineResultBestEffortExtras(t *testing.T) { + raw := "success\n# \nempty heading body\n# !@#\nodd heading body\n# big\n" + strings.Repeat("x", 70<<10) + "\n# output\n{\"ok\":true}\n" + result, status, err := decodeLineResult(raw, lineEdges()) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if status != lineParsePartial { + t.Fatalf("status = %q, want partial", status) + } + if result.Output["ok"] != true { + t.Fatalf("output ok = %v", result.Output["ok"]) + } + if result.Output["big"] == nil { + t.Fatal("missing big section") + } + if len(result.Output["big"].(string)) != 70<<10 { + t.Fatalf("big section length = %d", len(result.Output["big"].(string))) + } + if result.Output[resultParseKey] != lineParsePartial { + t.Fatalf("result_parse = %v", result.Output[resultParseKey]) + } +} + +func TestDecodeLineResultAdversarialProse(t *testing.T) { + raw := "success\n# findings\n- \"double\" and 'single' quotes\n```go\nfmt.Println(\"hi\")\n```\n{\"looks\":\"json\"}\nrejected\n# output\n{\"ok\":true}\n" + result, status, err := decodeLineResult(raw, lineEdges()) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if status != lineParseParsed { + t.Fatalf("status = %q, want parsed", status) + } + if result.Outcome != "success" { + t.Fatalf("outcome = %q, want success", result.Outcome) + } + body := result.Output["findings"].(string) + if !strings.Contains(body, `"double"`) || !strings.Contains(body, "'single'") || !strings.Contains(body, "```go") || !strings.Contains(body, "rejected") { + t.Fatalf("prose not verbatim: %q", body) + } +} + +func TestDecodeLineResultLargeResponse(t *testing.T) { + payload := strings.Repeat("x", 65<<10) + raw := "success\n# output\n{\"task\":\"" + payload + "\"}\n# findings\nok\n" + result, status, err := decodeLineResult(raw, lineEdges()) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if status != lineParseParsed { + t.Fatalf("status = %q, want parsed", status) + } + if result.Output["task"] != payload { + t.Fatal("payload mismatch") + } +} + +func TestModelRunLineFormatWritesRawOutputAndParses(t *testing.T) { + def := modelDefinition() + def.Edges = lineEdges() + def.ArtifactsDir = t.TempDir() + raw := "rejected\n\n# output\n{\"task\":\"x\"}\n\n# findings\n- BLOCKER\n" + client := generatorFunc(func(context.Context, ModelConfig, string) (string, error) { return raw, nil }) + result, err := (ModelStage{Client: client}).Run(context.Background(), def, nil) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if result.Outcome != "rejected" { + t.Fatalf("outcome = %q", result.Outcome) + } + if result.Output["task"] != "x" || result.Output["findings"] != "- BLOCKER" { + t.Fatalf("output mismatch: %+v", result.Output) + } + if result.Output[resultParseKey] != lineParseParsed { + t.Fatalf("result_parse = %v", result.Output[resultParseKey]) + } + got, err := os.ReadFile(filepath.Join(def.ArtifactsDir, "raw-output.md")) + if err != nil { + t.Fatalf("raw-output.md not written: %v", err) + } + if string(got) != raw { + t.Fatalf("raw-output.md mismatch") + } +} + +func TestModelRunLineFormatFailedParseStillWritesRawOutput(t *testing.T) { + def := modelDefinition() + def.Edges = lineEdges() + def.ArtifactsDir = t.TempDir() + raw := "bogus\n\n# findings\nkept\n" + client := generatorFunc(func(context.Context, ModelConfig, string) (string, error) { return raw, nil }) + result, err := (ModelStage{Client: client}).Run(context.Background(), def, nil) + if err == nil || !errors.Is(err, ErrInvalidModelResult) { + t.Fatalf("expected invalid result error, got %v", err) + } + if result.Outcome != "" { + t.Fatalf("outcome = %q, want empty", result.Outcome) + } + got, err := os.ReadFile(filepath.Join(def.ArtifactsDir, "raw-output.md")) + if err != nil { + t.Fatalf("raw-output.md not written: %v", err) + } + if string(got) != raw { + t.Fatalf("raw-output.md mismatch") + } +} + +func TestModelRunLegacyJSONPathUnchanged(t *testing.T) { + def := modelDefinition() + raw := `{"outcome":"success","output":{"task":"x"}}` + client := generatorFunc(func(context.Context, ModelConfig, string) (string, error) { return raw, nil }) + result, err := (ModelStage{Client: client}).Run(context.Background(), def, nil) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if result.Outcome != "success" || result.Output["task"] != "x" { + t.Fatalf("legacy path broken: %+v", result) + } +} + +func TestAgentResultTextPreservesLineFormat(t *testing.T) { + line := "success\n# output\n{\"ok\":true}\n# prose\n```json\n{\"x\":1}\n```\n" + if got := agentResultText(line); got != line { + t.Fatalf("agentResultText altered line format: %q", got) + } + json := "explaining\n{\"outcome\":\"success\",\"output\":{}}" + if got := agentResultText(json); got != "{\"outcome\":\"success\",\"output\":{}}" { + t.Fatalf("agentResultText did not extract JSON: %q", got) + } +} diff --git a/internal/stage/model.go b/internal/stage/model.go index dc52a161..4d1c4bfb 100644 --- a/internal/stage/model.go +++ b/internal/stage/model.go @@ -255,9 +255,16 @@ func (m ModelStage) Run(ctx context.Context, def Definition, input Values) (Resu } replayPostAdmission := false if generateErr == nil { - result, resultErr := decodeModelResult(stageResultText(text), def.Edges) - if resultErr != nil { - result, resultErr = repairModelResult(attemptCtx, client, spec.Config, text, def.Edges, resultErr) + var result Result + var resultErr error + if resultFormat(text) == "line" { + writeRawOutputFile(def.ArtifactsDir, text) + result, _, resultErr = decodeLineResult(text, def.Edges) + } else { + result, resultErr = decodeModelResult(stageResultText(text), def.Edges) + if resultErr != nil { + result, resultErr = repairModelResult(attemptCtx, client, spec.Config, text, def.Edges, resultErr) + } } if attemptCtx.Err() != nil { resultErr = errors.Join(resultErr, attemptCtx.Err())