Accept a single model result wrapped in prose or Markdown

This commit is contained in:
2026-10-09 14:59:52 +01:00
parent 6bd0e28ce6
commit b2dce357ca
5 changed files with 74 additions and 16 deletions
+1 -13
View File
@@ -168,17 +168,5 @@ func (c *AgentClient) Generate(ctx context.Context, config ModelConfig, prompt s
// full response in the audit, but pass only a complete final JSON object to the
// same strict stage validator. Never infer outcomes from prose or tool output.
func agentResultText(text string) string {
trimmed := strings.TrimSpace(text)
for start := strings.IndexByte(trimmed, '{'); start >= 0; {
candidate := trimmed[start:]
if json.Valid([]byte(candidate)) {
return candidate
}
next := strings.IndexByte(trimmed[start+1:], '{')
if next < 0 {
break
}
start += next + 1
}
return text
return stageResultText(text)
}
+2 -2
View File
@@ -56,12 +56,12 @@ func TestAgentOwnershipPermissionsAndInterrupt(t *testing.T) {
func TestAgentResultExtractionRemainsStrict(t *testing.T) {
edges := modelDefinition().Edges
for _, text := range []string{`{"outcome":"success","output":{"nested":{"x":1}}}`, "Observed failure; final result:\n" + `{"outcome":"error","output":{"reason":"blocked"}}`} {
for _, text := range []string{`{"outcome":"success","output":{"nested":{"x":1}}}`, "Observed failure; final result:\n" + `{"outcome":"error","output":{"reason":"blocked"}}`, "```json\n{\"outcome\":\"success\",\"output\":{}}\n```", `{"outcome":"success","output":{}} trailing`} {
if _, err := decodeModelResult(agentResultText(text), edges); err != nil {
t.Fatal(err)
}
}
for _, text := range []string{`success`, "```json\n{\"outcome\":\"success\",\"output\":{}}\n```", `{"outcome":"success","output":{}} trailing`, `{"outcome":"not-declared","output":{}}`, `{"outcome":"success","outcome":"error","output":{}}`} {
for _, text := range []string{`success`, `{"outcome":"success","output":{}} {"outcome":"error","output":{}}`, `{"outcome":"not-declared","output":{}}`, `{"outcome":"success","outcome":"error","output":{}}`, `{"broken": {"outcome":"success","output":{}}`} {
if _, err := decodeModelResult(agentResultText(text), edges); err == nil {
t.Fatal("accepted invalid agent result", text)
}
+1 -1
View File
@@ -127,7 +127,7 @@ func (m ModelStage) Run(ctx context.Context, def Definition, input Values) (Resu
text, err = client.Generate(ctx, spec.Config, prompt)
if err == nil {
var result Result
result, err = decodeModelResult(text, def.Edges)
result, err = decodeModelResult(stageResultText(text), def.Edges)
if err == nil {
return result, nil
}
+36
View File
@@ -59,6 +59,42 @@ Stage definition and runtime input:
` + string(data), nil
}
// stageResultText tolerates presentation around one complete result object.
// Never pick a winner among multiple objects or salvage nested JSON from a
// malformed outer object. The extracted object still gets strict validation.
func stageResultText(text string) string {
remaining := strings.TrimSpace(text)
var result string
for {
start := strings.IndexByte(remaining, '{')
if start < 0 {
break
}
// An array containing an object is not a stage result envelope.
if strings.HasPrefix(strings.TrimSpace(remaining[:start]), "[") {
return text
}
decoder := json.NewDecoder(strings.NewReader(remaining[start:]))
var object json.RawMessage
if err := decoder.Decode(&object); err != nil {
return text
}
if result != "" {
return text
}
result = string(object)
remaining = remaining[start+int(decoder.InputOffset()):]
var extra any
if json.NewDecoder(strings.NewReader(remaining)).Decode(&extra) == nil {
return text
}
}
if result != "" {
return result
}
return text
}
func decodeModelResult(text string, edges map[string]string) (Result, error) {
var wire struct {
Outcome string `json:"outcome"`
+34
View File
@@ -110,6 +110,40 @@ func TestModelSpecAndValidation(t *testing.T) {
type generatorFunc func(context.Context, ModelConfig, string) (string, error)
func TestModelRunAcceptsSingleResultWithPresentationWithoutRegeneration(t *testing.T) {
resultJSON := `{"outcome":"success","output":{"task":"Keep {nested} text","plan":["Inspect README","Edit QML"]}}`
for _, response := range []string{
resultJSON + "\nPlan ready for implementation.",
"```json\n" + resultJSON + "\n```",
"The plan follows:\n```json\n" + resultJSON + "\n```\nReady.",
} {
t.Run(response, func(t *testing.T) {
calls := 0
client := generatorFunc(func(context.Context, ModelConfig, string) (string, error) { calls++; return response, nil })
result, err := (ModelStage{Client: client}).Run(context.Background(), modelDefinition(), Values{"prompt": "task"})
if err != nil || result.Outcome != "success" || calls != 1 || result.Output["task"] != "Keep {nested} text" {
t.Fatal("formatting failed or regenerated model", result, err, calls)
}
})
}
if _, err := decodeModelResult(resultJSON+"\nPlan ready.", modelDefinition().Edges); err == nil || !strings.Contains(err.Error(), "trailing response content") {
t.Fatal("did not reproduce reported strict-parser error", err)
}
for _, response := range []string{
resultJSON + "\n" + resultJSON,
"```json\n" + resultJSON + "\n```\nAlternative:\n" + resultJSON,
"[" + resultJSON + "]", resultJSON + " null", resultJSON + " []",
`{"broken":` + resultJSON,
`{"outcome":"success","outcome":"error","output":{}}`,
`{"outcome":"success","output":null}`, `{"outcome":"missing","output":{}}`,
} {
client := generatorFunc(func(context.Context, ModelConfig, string) (string, error) { return response, nil })
if _, err := (ModelStage{Client: client}).Run(context.Background(), modelDefinition(), nil); !errors.Is(err, ErrInvalidModelResult) {
t.Fatal("accepted malformed or ambiguous response", response, err)
}
}
}
func (f generatorFunc) Generate(ctx context.Context, config ModelConfig, prompt string) (string, error) {
return f(ctx, config, prompt)
}