Add line-oriented result format for model and agent stages

Introduce a short, markdown-ish result format whose first non-empty line is
the declared outcome and whose "# output" section carries the routing data,
so stage output stops carrying multi-kilobyte prose inside a JSON envelope.

- New internal/stage/line_result.go: resultFormat() gates legacy JSON ('{'
  first) vs the new line format; decodeLineResult() parses outcome-from-first
  line, requires a JSON-object "# output" section, stores extra sections
  verbatim under normalised keys, and records a reserved result_parse status
  (parsed/partial/failed). Failures return a salvage Result plus
  ErrInvalidModelResult, and never route an undeclared outcome.
- internal/stage/model.go: write the raw response to raw-output.md (0600,
  best-effort) before parsing, and route the line format through
  decodeLineResult while leaving the JSON path (stageResultText ->
  decodeModelResult -> repairModelResult) unchanged.
- internal/stage/agent.go: agentResultText returns line-format text verbatim
  so the "# output" body is not stripped by the JSON-object extractor.
- internal/server/runner.go: merge a non-empty salvage Result into the error
  envelope under "salvage" on failure (a no-op for the legacy JSON path).
- internal/stage/line_result_test.go: raw-text fixtures covering happy path,
  missing/undeclared outcome, missing "# output", best-effort extras,
  adversarial prose, legacy JSON unchanged, >=64KB responses, and
  raw-output.md written before parsing.
This commit is contained in:
2026-10-16 21:34:33 +01:00
parent 8c1eea53a9
commit b05277a57d
5 changed files with 479 additions and 3 deletions
+3
View File
@@ -680,6 +680,9 @@ func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, na
step.Diagnostic = err.Error()
step.Outcome = "error"
step.Output = stage.Values{"error": err.Error(), "stage": key, "input": input}
if len(result.Output) > 0 {
step.Output["salvage"] = result.Output
}
}
// A failed or timed-out step carries the session pointer even when the
// bounded transcript would otherwise be only its stage_limits envelope.
+5
View File
@@ -276,6 +276,11 @@ func completedAgentResult(data []byte) (string, error) {
// Agents may emit a short explanation before their final result. Preserve the
// full response in the audit, but pass only a complete final JSON object to the
// same strict stage validator. Never infer outcomes from prose or tool output.
// The line-oriented format is returned verbatim so its '# output' body is not
// stripped by the JSON-object extractor.
func agentResultText(text string) string {
if resultFormat(text) == "line" {
return text
}
return stageResultText(text)
}
+229
View File
@@ -0,0 +1,229 @@
package stage
import (
"encoding/json"
"fmt"
"os"
"path/filepath"
"strings"
"unicode"
)
// resultFormat decides which decoder to use for a raw stage response.
//
// The legacy JSON path is kept for responses that look like JSON envelopes,
// including presentation wrappers (a single surrounding markdown fence or a
// short prose prefix before the JSON object). The line-oriented format is
// chosen only when a section heading ("# ") appears before any JSON object,
// because the line format's required "# output" section always precedes the
// JSON body. This keeps existing JSON responses, including fenced or
// prose-prefixed results, on the unchanged decodeModelResult path.
func resultFormat(raw string) string {
trimmed := strings.TrimSpace(raw)
// Strip a single surrounding markdown fence so fenced JSON still routes
// through the unchanged legacy path.
for _, prefix := range []string{"```json\n", "```\n"} {
if strings.HasPrefix(trimmed, prefix) && strings.HasSuffix(trimmed, "\n```") {
trimmed = strings.TrimSuffix(strings.TrimPrefix(trimmed, prefix), "\n```")
trimmed = strings.TrimSpace(trimmed)
break
}
}
if len(trimmed) > 0 && trimmed[0] == '{' {
return "json"
}
heading := headingLineIndex(trimmed)
brace := strings.IndexByte(trimmed, '{')
if heading >= 0 && (brace < 0 || heading < brace) {
return "line"
}
return "json"
}
// headingLineIndex returns the index of the first "# " heading line, or -1.
func headingLineIndex(s string) int {
if strings.HasPrefix(s, "# ") {
return 0
}
if i := strings.Index(s, "\n# "); i >= 0 {
return i + 1
}
return -1
}
// lineResultStatus records how completely the line-oriented response was parsed.
// It is stored in Result.Output under the reserved key resultParseKey.
const (
lineParseParsed = "parsed"
lineParsePartial = "partial"
lineParseFailed = "failed"
)
const resultParseKey = "result_parse"
// writeRawOutputFile persists the full raw response to the stage artifacts
// directory before any parsing, so the closest record is always retained.
// It is best-effort: empty ArtifactsDir or a write error is ignored.
func writeRawOutputFile(artifactsDir, text string) {
if artifactsDir == "" {
return
}
_ = os.WriteFile(filepath.Join(artifactsDir, "raw-output.md"), []byte(text), 0600)
}
type lineSection struct {
heading string
lines []string
}
func (s lineSection) body() string {
return trimOneTrailingNewline(strings.Join(s.lines, "\n"))
}
// decodeLineResult parses the line-oriented result format.
//
// The first non-empty line is the outcome. It must be one of the declared edges.
// Subsequent lines beginning with "# " start sections. The required "# output"
// section must contain a JSON object. Every other section is stored verbatim
// under a normalised key derived from its heading.
//
// The function never returns a nil Result: on failure it returns whatever was
// salvaged plus a non-nil error. The returned status string is one of
// lineParseParsed, lineParsePartial or lineParseFailed and is also inserted into
// Result.Output under resultParseKey.
func decodeLineResult(raw string, edges map[string]string) (Result, string, error) {
status := lineParseParsed
result := Result{Output: Values{}}
lines := strings.Split(raw, "\n")
i := 0
for i < len(lines) && strings.TrimSpace(lines[i]) == "" {
i++
}
if i >= len(lines) {
result.Output[resultParseKey] = lineParseFailed
return result, lineParseFailed, fmt.Errorf("%w: missing outcome", ErrInvalidModelResult)
}
outcome := strings.TrimSpace(lines[i])
i++
sections := parseLineSections(lines, i)
if strings.HasPrefix(outcome, "# ") {
// The first non-empty line is a section heading, not an outcome.
salvage := Result{Output: Values{}}
addLineSectionsToOutput(salvage.Output, sections, &status)
salvage.Output[resultParseKey] = lineParseFailed
return salvage, lineParseFailed, fmt.Errorf("%w: missing outcome", ErrInvalidModelResult)
}
if _, ok := edges[outcome]; !ok {
// Salvage any extra sections but do not route: the outcome is not declared.
salvage := Result{Output: Values{}}
addLineSectionsToOutput(salvage.Output, sections, &status)
salvage.Output[resultParseKey] = lineParseFailed
if outcome == "" {
return salvage, lineParseFailed, fmt.Errorf("%w: missing outcome", ErrInvalidModelResult)
}
return salvage, lineParseFailed, fmt.Errorf("%w: undeclared outcome", ErrInvalidModelResult)
}
result.Outcome = outcome
addLineSectionsToOutput(result.Output, sections, &status)
outputSection := findLineSection(sections, "output")
if outputSection == nil {
result.Output[resultParseKey] = lineParseFailed
return result, lineParseFailed, fmt.Errorf("%w: missing output section", ErrInvalidModelResult)
}
var output Values
if err := json.Unmarshal([]byte(outputSection.body()), &output); err != nil {
result.Output[resultParseKey] = lineParseFailed
return result, lineParseFailed, fmt.Errorf("%w: output must be an object", ErrInvalidModelResult)
}
if output == nil {
result.Output[resultParseKey] = lineParseFailed
return result, lineParseFailed, fmt.Errorf("%w: output must be an object", ErrInvalidModelResult)
}
for k, v := range output {
result.Output[k] = v
}
result.Output[resultParseKey] = status
return result, status, nil
}
func parseLineSections(lines []string, start int) []lineSection {
var sections []lineSection
var current *lineSection
for i := start; i < len(lines); i++ {
line := lines[i]
if strings.HasPrefix(line, "# ") {
if current != nil {
sections = append(sections, *current)
}
current = &lineSection{heading: strings.TrimSpace(line[2:])}
continue
}
if current != nil {
current.lines = append(current.lines, line)
}
}
if current != nil {
sections = append(sections, *current)
}
return sections
}
func findLineSection(sections []lineSection, name string) *lineSection {
for i := range sections {
if normaliseHeading(sections[i].heading) == name {
return &sections[i]
}
}
return nil
}
func addLineSectionsToOutput(out Values, sections []lineSection, status *string) {
seen := map[string]bool{}
for _, s := range sections {
key := normaliseHeading(s.heading)
if key == "output" {
continue
}
if key == resultParseKey || key == "" || seen[key] {
*status = lineParsePartial
continue
}
seen[key] = true
out[key] = s.body()
}
}
func trimOneTrailingNewline(s string) string {
if strings.HasSuffix(s, "\n") {
return s[:len(s)-1]
}
return s
}
func normaliseHeading(s string) string {
s = strings.ToLower(strings.TrimSpace(s))
var b strings.Builder
b.Grow(len(s))
inSpace := false
for _, r := range s {
if unicode.IsSpace(r) {
inSpace = true
continue
}
if inSpace && b.Len() > 0 {
b.WriteByte('_')
}
inSpace = false
b.WriteRune(r)
}
return b.String()
}
+232
View File
@@ -0,0 +1,232 @@
package stage
import (
"context"
"errors"
"os"
"path/filepath"
"strings"
"testing"
)
func lineEdges() map[string]string {
return map[string]string{"success": "#ok", "error": "#fail", "rejected": "#rejected"}
}
func TestResultFormatPrecedence(t *testing.T) {
for _, tc := range []struct {
raw string
want string
}{
{`{"outcome":"success","output":{}}`, "json"},
{" \n\n{\"outcome\":\"success\"}", "json"},
{"success\n# output\n{}", "line"},
{" success\n# output\n{}", "line"},
{"# output\n{}", "line"},
} {
if got := resultFormat(tc.raw); got != tc.want {
t.Errorf("resultFormat(%q) = %q, want %q", tc.raw, got, tc.want)
}
}
}
func TestDecodeLineResultHappyPath(t *testing.T) {
raw := "rejected\n\n# output\n{\"task\":\"x\"}\n\n# findings\n- BLOCKER: prose\n\n# verification evidence\n{raw json}\n"
result, status, err := decodeLineResult(raw, lineEdges())
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if status != lineParseParsed {
t.Fatalf("status = %q, want parsed", status)
}
if result.Outcome != "rejected" {
t.Fatalf("outcome = %q, want rejected", result.Outcome)
}
if result.Output["task"] != "x" {
t.Fatalf("output task = %v", result.Output["task"])
}
if got, want := result.Output["findings"], "- BLOCKER: prose"; got != want {
t.Fatalf("findings = %q, want %q", got, want)
}
if got, want := result.Output["verification_evidence"], "{raw json}"; got != want {
t.Fatalf("verification_evidence = %q, want %q", got, want)
}
if result.Output[resultParseKey] != lineParseParsed {
t.Fatalf("result_parse = %v", result.Output[resultParseKey])
}
}
func TestDecodeLineResultMissingOutcome(t *testing.T) {
raw := "\n\n# output\n{\"task\":\"x\"}\n"
result, status, err := decodeLineResult(raw, lineEdges())
if err == nil || !errors.Is(err, ErrInvalidModelResult) || !strings.Contains(err.Error(), "missing outcome") {
t.Fatalf("expected missing-outcome error, got %v", err)
}
if status != lineParseFailed {
t.Fatalf("status = %q, want failed", status)
}
if result.Output[resultParseKey] != lineParseFailed {
t.Fatalf("result_parse = %v", result.Output[resultParseKey])
}
}
func TestDecodeLineResultUndeclaredOutcome(t *testing.T) {
raw := "bogus\n\n# output\n{\"task\":\"x\"}\n# findings\nkept\n"
result, status, err := decodeLineResult(raw, lineEdges())
if err == nil || !errors.Is(err, ErrInvalidModelResult) || !strings.Contains(err.Error(), "undeclared outcome") {
t.Fatalf("expected undeclared-outcome error, got %v", err)
}
if status != lineParseFailed {
t.Fatalf("status = %q, want failed", status)
}
if result.Outcome != "" {
t.Fatalf("outcome = %q, want empty", result.Outcome)
}
if got, want := result.Output["findings"], "kept"; got != want {
t.Fatalf("findings salvage = %q, want %q", got, want)
}
}
func TestDecodeLineResultMissingOutput(t *testing.T) {
raw := "success\n\n# findings\nkept\n"
result, status, err := decodeLineResult(raw, lineEdges())
if err == nil || !errors.Is(err, ErrInvalidModelResult) {
t.Fatalf("expected error, got %v", err)
}
if status != lineParseFailed {
t.Fatalf("status = %q, want failed", status)
}
if got, want := result.Output["findings"], "kept"; got != want {
t.Fatalf("findings salvage = %q, want %q", got, want)
}
}
func TestDecodeLineResultBestEffortExtras(t *testing.T) {
raw := "success\n# \nempty heading body\n# !@#\nodd heading body\n# big\n" + strings.Repeat("x", 70<<10) + "\n# output\n{\"ok\":true}\n"
result, status, err := decodeLineResult(raw, lineEdges())
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if status != lineParsePartial {
t.Fatalf("status = %q, want partial", status)
}
if result.Output["ok"] != true {
t.Fatalf("output ok = %v", result.Output["ok"])
}
if result.Output["big"] == nil {
t.Fatal("missing big section")
}
if len(result.Output["big"].(string)) != 70<<10 {
t.Fatalf("big section length = %d", len(result.Output["big"].(string)))
}
if result.Output[resultParseKey] != lineParsePartial {
t.Fatalf("result_parse = %v", result.Output[resultParseKey])
}
}
func TestDecodeLineResultAdversarialProse(t *testing.T) {
raw := "success\n# findings\n- \"double\" and 'single' quotes\n```go\nfmt.Println(\"hi\")\n```\n{\"looks\":\"json\"}\nrejected\n# output\n{\"ok\":true}\n"
result, status, err := decodeLineResult(raw, lineEdges())
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if status != lineParseParsed {
t.Fatalf("status = %q, want parsed", status)
}
if result.Outcome != "success" {
t.Fatalf("outcome = %q, want success", result.Outcome)
}
body := result.Output["findings"].(string)
if !strings.Contains(body, `"double"`) || !strings.Contains(body, "'single'") || !strings.Contains(body, "```go") || !strings.Contains(body, "rejected") {
t.Fatalf("prose not verbatim: %q", body)
}
}
func TestDecodeLineResultLargeResponse(t *testing.T) {
payload := strings.Repeat("x", 65<<10)
raw := "success\n# output\n{\"task\":\"" + payload + "\"}\n# findings\nok\n"
result, status, err := decodeLineResult(raw, lineEdges())
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if status != lineParseParsed {
t.Fatalf("status = %q, want parsed", status)
}
if result.Output["task"] != payload {
t.Fatal("payload mismatch")
}
}
func TestModelRunLineFormatWritesRawOutputAndParses(t *testing.T) {
def := modelDefinition()
def.Edges = lineEdges()
def.ArtifactsDir = t.TempDir()
raw := "rejected\n\n# output\n{\"task\":\"x\"}\n\n# findings\n- BLOCKER\n"
client := generatorFunc(func(context.Context, ModelConfig, string) (string, error) { return raw, nil })
result, err := (ModelStage{Client: client}).Run(context.Background(), def, nil)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if result.Outcome != "rejected" {
t.Fatalf("outcome = %q", result.Outcome)
}
if result.Output["task"] != "x" || result.Output["findings"] != "- BLOCKER" {
t.Fatalf("output mismatch: %+v", result.Output)
}
if result.Output[resultParseKey] != lineParseParsed {
t.Fatalf("result_parse = %v", result.Output[resultParseKey])
}
got, err := os.ReadFile(filepath.Join(def.ArtifactsDir, "raw-output.md"))
if err != nil {
t.Fatalf("raw-output.md not written: %v", err)
}
if string(got) != raw {
t.Fatalf("raw-output.md mismatch")
}
}
func TestModelRunLineFormatFailedParseStillWritesRawOutput(t *testing.T) {
def := modelDefinition()
def.Edges = lineEdges()
def.ArtifactsDir = t.TempDir()
raw := "bogus\n\n# findings\nkept\n"
client := generatorFunc(func(context.Context, ModelConfig, string) (string, error) { return raw, nil })
result, err := (ModelStage{Client: client}).Run(context.Background(), def, nil)
if err == nil || !errors.Is(err, ErrInvalidModelResult) {
t.Fatalf("expected invalid result error, got %v", err)
}
if result.Outcome != "" {
t.Fatalf("outcome = %q, want empty", result.Outcome)
}
got, err := os.ReadFile(filepath.Join(def.ArtifactsDir, "raw-output.md"))
if err != nil {
t.Fatalf("raw-output.md not written: %v", err)
}
if string(got) != raw {
t.Fatalf("raw-output.md mismatch")
}
}
func TestModelRunLegacyJSONPathUnchanged(t *testing.T) {
def := modelDefinition()
raw := `{"outcome":"success","output":{"task":"x"}}`
client := generatorFunc(func(context.Context, ModelConfig, string) (string, error) { return raw, nil })
result, err := (ModelStage{Client: client}).Run(context.Background(), def, nil)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if result.Outcome != "success" || result.Output["task"] != "x" {
t.Fatalf("legacy path broken: %+v", result)
}
}
func TestAgentResultTextPreservesLineFormat(t *testing.T) {
line := "success\n# output\n{\"ok\":true}\n# prose\n```json\n{\"x\":1}\n```\n"
if got := agentResultText(line); got != line {
t.Fatalf("agentResultText altered line format: %q", got)
}
json := "explaining\n{\"outcome\":\"success\",\"output\":{}}"
if got := agentResultText(json); got != "{\"outcome\":\"success\",\"output\":{}}" {
t.Fatalf("agentResultText did not extract JSON: %q", got)
}
}
+10 -3
View File
@@ -255,9 +255,16 @@ func (m ModelStage) Run(ctx context.Context, def Definition, input Values) (Resu
}
replayPostAdmission := false
if generateErr == nil {
result, resultErr := decodeModelResult(stageResultText(text), def.Edges)
if resultErr != nil {
result, resultErr = repairModelResult(attemptCtx, client, spec.Config, text, def.Edges, resultErr)
var result Result
var resultErr error
if resultFormat(text) == "line" {
writeRawOutputFile(def.ArtifactsDir, text)
result, _, resultErr = decodeLineResult(text, def.Edges)
} else {
result, resultErr = decodeModelResult(stageResultText(text), def.Edges)
if resultErr != nil {
result, resultErr = repairModelResult(attemptCtx, client, spec.Config, text, def.Edges, resultErr)
}
}
if attemptCtx.Err() != nil {
resultErr = errors.Join(resultErr, attemptCtx.Err())