Accept a single model result wrapped in prose or Markdown
This commit is contained in:
+1
-13
@@ -168,17 +168,5 @@ func (c *AgentClient) Generate(ctx context.Context, config ModelConfig, prompt s
|
||||
// full response in the audit, but pass only a complete final JSON object to the
|
||||
// same strict stage validator. Never infer outcomes from prose or tool output.
|
||||
func agentResultText(text string) string {
|
||||
trimmed := strings.TrimSpace(text)
|
||||
for start := strings.IndexByte(trimmed, '{'); start >= 0; {
|
||||
candidate := trimmed[start:]
|
||||
if json.Valid([]byte(candidate)) {
|
||||
return candidate
|
||||
}
|
||||
next := strings.IndexByte(trimmed[start+1:], '{')
|
||||
if next < 0 {
|
||||
break
|
||||
}
|
||||
start += next + 1
|
||||
}
|
||||
return text
|
||||
return stageResultText(text)
|
||||
}
|
||||
|
||||
@@ -56,12 +56,12 @@ func TestAgentOwnershipPermissionsAndInterrupt(t *testing.T) {
|
||||
|
||||
func TestAgentResultExtractionRemainsStrict(t *testing.T) {
|
||||
edges := modelDefinition().Edges
|
||||
for _, text := range []string{`{"outcome":"success","output":{"nested":{"x":1}}}`, "Observed failure; final result:\n" + `{"outcome":"error","output":{"reason":"blocked"}}`} {
|
||||
for _, text := range []string{`{"outcome":"success","output":{"nested":{"x":1}}}`, "Observed failure; final result:\n" + `{"outcome":"error","output":{"reason":"blocked"}}`, "```json\n{\"outcome\":\"success\",\"output\":{}}\n```", `{"outcome":"success","output":{}} trailing`} {
|
||||
if _, err := decodeModelResult(agentResultText(text), edges); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
for _, text := range []string{`success`, "```json\n{\"outcome\":\"success\",\"output\":{}}\n```", `{"outcome":"success","output":{}} trailing`, `{"outcome":"not-declared","output":{}}`, `{"outcome":"success","outcome":"error","output":{}}`} {
|
||||
for _, text := range []string{`success`, `{"outcome":"success","output":{}} {"outcome":"error","output":{}}`, `{"outcome":"not-declared","output":{}}`, `{"outcome":"success","outcome":"error","output":{}}`, `{"broken": {"outcome":"success","output":{}}`} {
|
||||
if _, err := decodeModelResult(agentResultText(text), edges); err == nil {
|
||||
t.Fatal("accepted invalid agent result", text)
|
||||
}
|
||||
|
||||
@@ -127,7 +127,7 @@ func (m ModelStage) Run(ctx context.Context, def Definition, input Values) (Resu
|
||||
text, err = client.Generate(ctx, spec.Config, prompt)
|
||||
if err == nil {
|
||||
var result Result
|
||||
result, err = decodeModelResult(text, def.Edges)
|
||||
result, err = decodeModelResult(stageResultText(text), def.Edges)
|
||||
if err == nil {
|
||||
return result, nil
|
||||
}
|
||||
|
||||
@@ -59,6 +59,42 @@ Stage definition and runtime input:
|
||||
` + string(data), nil
|
||||
}
|
||||
|
||||
// stageResultText tolerates presentation around one complete result object.
|
||||
// Never pick a winner among multiple objects or salvage nested JSON from a
|
||||
// malformed outer object. The extracted object still gets strict validation.
|
||||
func stageResultText(text string) string {
|
||||
remaining := strings.TrimSpace(text)
|
||||
var result string
|
||||
for {
|
||||
start := strings.IndexByte(remaining, '{')
|
||||
if start < 0 {
|
||||
break
|
||||
}
|
||||
// An array containing an object is not a stage result envelope.
|
||||
if strings.HasPrefix(strings.TrimSpace(remaining[:start]), "[") {
|
||||
return text
|
||||
}
|
||||
decoder := json.NewDecoder(strings.NewReader(remaining[start:]))
|
||||
var object json.RawMessage
|
||||
if err := decoder.Decode(&object); err != nil {
|
||||
return text
|
||||
}
|
||||
if result != "" {
|
||||
return text
|
||||
}
|
||||
result = string(object)
|
||||
remaining = remaining[start+int(decoder.InputOffset()):]
|
||||
var extra any
|
||||
if json.NewDecoder(strings.NewReader(remaining)).Decode(&extra) == nil {
|
||||
return text
|
||||
}
|
||||
}
|
||||
if result != "" {
|
||||
return result
|
||||
}
|
||||
return text
|
||||
}
|
||||
|
||||
func decodeModelResult(text string, edges map[string]string) (Result, error) {
|
||||
var wire struct {
|
||||
Outcome string `json:"outcome"`
|
||||
|
||||
@@ -110,6 +110,40 @@ func TestModelSpecAndValidation(t *testing.T) {
|
||||
|
||||
type generatorFunc func(context.Context, ModelConfig, string) (string, error)
|
||||
|
||||
func TestModelRunAcceptsSingleResultWithPresentationWithoutRegeneration(t *testing.T) {
|
||||
resultJSON := `{"outcome":"success","output":{"task":"Keep {nested} text","plan":["Inspect README","Edit QML"]}}`
|
||||
for _, response := range []string{
|
||||
resultJSON + "\nPlan ready for implementation.",
|
||||
"```json\n" + resultJSON + "\n```",
|
||||
"The plan follows:\n```json\n" + resultJSON + "\n```\nReady.",
|
||||
} {
|
||||
t.Run(response, func(t *testing.T) {
|
||||
calls := 0
|
||||
client := generatorFunc(func(context.Context, ModelConfig, string) (string, error) { calls++; return response, nil })
|
||||
result, err := (ModelStage{Client: client}).Run(context.Background(), modelDefinition(), Values{"prompt": "task"})
|
||||
if err != nil || result.Outcome != "success" || calls != 1 || result.Output["task"] != "Keep {nested} text" {
|
||||
t.Fatal("formatting failed or regenerated model", result, err, calls)
|
||||
}
|
||||
})
|
||||
}
|
||||
if _, err := decodeModelResult(resultJSON+"\nPlan ready.", modelDefinition().Edges); err == nil || !strings.Contains(err.Error(), "trailing response content") {
|
||||
t.Fatal("did not reproduce reported strict-parser error", err)
|
||||
}
|
||||
for _, response := range []string{
|
||||
resultJSON + "\n" + resultJSON,
|
||||
"```json\n" + resultJSON + "\n```\nAlternative:\n" + resultJSON,
|
||||
"[" + resultJSON + "]", resultJSON + " null", resultJSON + " []",
|
||||
`{"broken":` + resultJSON,
|
||||
`{"outcome":"success","outcome":"error","output":{}}`,
|
||||
`{"outcome":"success","output":null}`, `{"outcome":"missing","output":{}}`,
|
||||
} {
|
||||
client := generatorFunc(func(context.Context, ModelConfig, string) (string, error) { return response, nil })
|
||||
if _, err := (ModelStage{Client: client}).Run(context.Background(), modelDefinition(), nil); !errors.Is(err, ErrInvalidModelResult) {
|
||||
t.Fatal("accepted malformed or ambiguous response", response, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (f generatorFunc) Generate(ctx context.Context, config ModelConfig, prompt string) (string, error) {
|
||||
return f(ctx, config, prompt)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user