diff --git a/docs/model-results.md b/docs/model-results.md index 8d122a2e..93e62ba7 100644 --- a/docs/model-results.md +++ b/docs/model-results.md @@ -31,29 +31,51 @@ schema-invalid result is not reformatted to invent a missing field or verdict. Format recovery has a separate bounded budget, even with `retries=0`: -1. Append up to eight missing closing delimiters when the existing response - consists of complete JSON tokens. Validate the result again. No model call. -2. If necessary, make at most one fresh, deny-all, retained text-generation - request with only the malformed response and precise parsing error. The - original implementing/reviewing agent is never re-prompted. The formatter - has a 30-second deadline within the original stage deadline and a 32 KiB - input/output bound. +1. A deterministic local pass fixes only the structural error that is derivable + from the payload itself, in order: append up to eight missing closing + delimiters when the existing response consists of complete JSON tokens; + remove a single unambiguous trailing comma before a closing `}`/`]`; or + insert exactly the one comma/colon the parser expected at its reported byte. + Each candidate must pass lossless syntax validation and the full envelope + validation. No model call. An ambiguous payload - an unescaped quote or colon + inside a string value, or a stray delimiter with no single completion - is + never guessed at and falls through. +2. If the local pass cannot repair it, make at most one fresh, deny-all, retained + text-generation request with only the malformed response and precise parsing + error. The original implementing/reviewing agent is never re-prompted. The + formatter's deadline is derived from the payload size and the attempt's own + remaining budget rather than a fixed short constant, so a large result is not + guaranteed to time out; it stays within the stage deadline, is floored for + ordinary payloads, and keeps the 32 KiB input/output bound. 3. Accept only insertion of missing commas/colons or trailing closing delimiters, with at most 16 insertions. Every existing string, number, literal and structural delimiter must remain verbatim and in order. Validate the full result again. Changed approval, outcome, evidence, reordered data, deleted fields, incomplete strings and fabricated values are rejected. +Every terminal path records an explicit recovery status: `repaired` (a fully +lossless local or formatter recovery), `partial` (the candidate is valid JSON but +still fails envelope validation, so it is never returned as a completed result), +or `unrecoverable` (nothing recovered, or the formatter errored or timed out). +The unrecoverable path retains the bounded original payload together with the +precise parse error and its byte offset, and the status is included in the +`FINAL RESULT VALIDATION` diagnostic so an operator can see exactly what the model +emitted. + If recovery fails, report the original precise error and repair failure; never rerun project edits, commands, or the entire stage to repair final formatting. Configured execution retries remain for retryable text-generation transport errors; an explicit read-only stage may additionally re-emit a malformed final result and retry a post-admission deadline or transport failure up to `retries`. A stage whose tools include `edit`/`shell`, or that has no tools block (the legacy -read/glob/grep/edit/shell bundle), never has admitted tool work replayed. +read/glob/grep/edit/shell bundle), never has admitted tool work replayed. The +deterministic local pass and the fresh deny-all, tool-free formatter session never +re-prompt or replay the admitted stage session, so a shell-enabled review stage +gains malformed-result recovery without any replay of admitted work. Repair records include the original/candidate response, parsing diagnostic, -method and acceptance flag in the bounded transcript preview. `pipelines -inspect --json` and the TUI evidence section expose these records. +method, recovery status, byte offset and acceptance flag in the bounded transcript +preview. `pipelines inspect --json` and the TUI evidence section expose +these records. OpenCode CLI failures include the HTTP status when supplied, process exit code, and a recognized safe error category (such as session not found or service @@ -98,9 +120,18 @@ exactly 262144 bytes of incomplete JSON (`invalid agent messages`); a reviewer result was rejected for a duplicate nested `performance_review` field; and an onboarding visual-review result failed with `invalid character 'z' after object key:value pair at byte 3968`, where unsafe lossless repair was correctly rejected. -These are source-level findings covered by mocked `internal/stage` fixtures. An -updated checkout or installed skill is **not** evidence that a user's running -server contains the fix; deploying/restarting is a separate authorized action. + +A shell-enabled `ui_reviewer` stage (run 55, pipeline 9 v6, 2026-10-15) failed with +`invalid character ':' after object key:value pair at byte 7837`, where the +isolated formatter hit its own 30-second deadline. The malformed payload was +retrieved read-only with `pipelines inspect 55 --json` and reproduced the exact +byte offset 7837. It is an unescaped quote/colon inside a string value - an +ambiguous case - so the deterministic pass correctly reports it unrecoverable +rather than guessing, while the new derived deadline removes the fixed-deadline +failure mode for larger payloads. These are source-level findings covered by mocked +`internal/stage` fixtures. An updated checkout or installed skill is **not** +evidence that a user's running server contains the fix; deploying/restarting is a +separate authorized action. OpenCode V2's published prompt/generate API does not accept a structured-output field, so a prose `output_contract` does **not** enforce structured output: V2 diff --git a/internal/author/skill/references/debugging.md b/internal/author/skill/references/debugging.md index 28d20f45..3f699106 100644 --- a/internal/author/skill/references/debugging.md +++ b/internal/author/skill/references/debugging.md @@ -31,12 +31,19 @@ Each playbook is *symptom → how to confirm → what to do*. - **Confirm** Read the diagnostic and the retained final result: the stage's result does not match its `output` contract. A retained transcript can be a valid bounded preview envelope, not the complete history. -- **Do** A malformed result on an explicit read-only stage may be re-emitted up to - `retries` with its `format_repairs` evidence retained; a stage with `edit`/`shell` - or no tools block is never replayed. Never accept prose, flattened output or a - tool-call envelope as success. If one stage keeps producing invalid JSON, tighten - its `output`/`prompt` in the **pipeline definition** (`references/format.md`) — a - pipeline fix, not a project edit. +- **Do** Recovery is bounded and work-preserving. A deterministic local pass first + appends missing closers, drops one trailing comma, or inserts the single + comma/colon the parser expected; only if that is ambiguous does one deny-all + formatter request run, with a deadline derived from the payload size and the + attempt's remaining budget. Each path records a `repaired`, `partial` or + `unrecoverable` status in `format_repairs`; a partial or unrecoverable payload + is never accepted as complete. On an explicit read-only stage a malformed result + may be re-emitted up to `retries`; a stage with `edit`/`shell` or no tools block + is never replayed, though its malformed result can still be repaired by the + deterministic pass or the fresh deny-all formatter session. Never accept prose, + flattened output or a tool-call envelope as success. If one stage keeps + producing invalid JSON, tighten its `output`/`prompt` in the **pipeline + definition** (`references/format.md`) — a pipeline fix, not a project edit. ## A reviewer or visual reviewer that blocks, rejects repeatedly, or hangs diff --git a/internal/author/skill/references/format.md b/internal/author/skill/references/format.md index fa1ccb08..dba0f2ea 100644 --- a/internal/author/skill/references/format.md +++ b/internal/author/skill/references/format.md @@ -105,15 +105,23 @@ Use this block for intentional file/command restrictions; prose is not a runtime permission policy. Keep ordinary workflows language-neutral and unrestricted within their run directory unless the user asks for a narrower scope. -Malformed final JSON has bounded, syntax-only recovery even with `retries=0`; -it never changes verdict/evidence. On an explicit read-only stage a rejected -malformed result may be re-emitted up to `retries` with its `format_repairs` -evidence retained; a stage with `edit`/`shell` or no tools block never replays -stage tools or re-emits the result. Compatible native stable -backends request schema output with schema retries disabled; a retained deny-all, -no-inference compatibility probe identifies known format incompatibility. Incompatible -native backends and V2 require strict local envelope validation. Do not promise -recovery of truncated strings, missing evidence or invalid outcomes. +Malformed final JSON has bounded recovery even with `retries=0`; it never changes +verdict/evidence. Recovery tries a deterministic local pass first (append missing +closing delimiters, drop one trailing comma, or insert the single comma/colon the +parser expected), then at most one deny-all formatter request whose deadline is +derived from the payload size and the attempt's remaining budget rather than a +fixed short constant. Every path records `repaired`, `partial` or `unrecoverable` +in `format_repairs`; a partial or unrecoverable result is never returned as +complete, and an ambiguous payload (an unescaped quote/colon inside a string +value) is never guessed at. On an explicit read-only stage a rejected malformed +result may be re-emitted up to `retries`; a stage with `edit`/`shell` or no tools +block never replays stage tools or re-emits the result, though the deterministic +pass and the fresh deny-all, tool-free formatter session still apply. Compatible +native stable backends request schema output with schema retries disabled; a +retained deny-all, no-inference compatibility probe identifies known format +incompatibility. Incompatible native backends and V2 require strict local envelope +validation. Do not promise recovery of truncated strings, missing evidence or +invalid outcomes. Return the strict `{ "outcome": "declared-edge-key", "output": { ... } }` envelope. Do not treat tool-call/DSML markup, plain prose or flattened output as success, or rerun tool work to fix a response's formatting. diff --git a/internal/author/skill/references/verification.md b/internal/author/skill/references/verification.md index 90049b73..f5a1dc9b 100644 --- a/internal/author/skill/references/verification.md +++ b/internal/author/skill/references/verification.md @@ -94,6 +94,24 @@ field is sent. A review-only fresh execution is a future recovery idea, **not implemented**. Onboarding UI visual approval remains unresolved and must not be reported complete. +## Bounded malformed-result recovery: 2026-10-16 + +Source-level, mocked `internal/stage` regression coverage (not live-provider +evidence and not a deployment): malformed final-result recovery now tries a +deterministic local pass first (append missing closers, drop one trailing comma, +or insert the single comma/colon the parser expected), then at most one deny-all +formatter request whose deadline is derived from the payload size and the +attempt's remaining budget rather than a fixed 30-second constant. Every path +records `repaired`, `partial` or `unrecoverable` in `format_repairs`; a `partial` +or `unrecoverable` payload is never returned as a completed result, and an +ambiguous payload (an unescaped quote/colon inside a string value) is never +guessed at. A shell-enabled stage's malformed but deterministically recoverable +result now completes without replaying admitted tool work; `safeToReplay` is +unchanged. The run-55 `ui_reviewer` payload was retrieved read-only with +`pipelines inspect 55 --json` and reproduced its exact byte offset 7837; it is the +ambiguous error class, so it is reported unrecoverable rather than repaired. This +does not update or restart any running server. + ## Failure-handling source changes: 2026-10-15 Source-level, mocked `internal/stage` and `internal/server` regression coverage diff --git a/internal/stage/budget_test.go b/internal/stage/budget_test.go index c545ac17..21627ac9 100644 --- a/internal/stage/budget_test.go +++ b/internal/stage/budget_test.go @@ -370,7 +370,7 @@ func TestFormatRepairUsesTheSameBudget(t *testing.T) { } if strings.HasSuffix(path, "/generate") { generated++ - text := `{"outcome":"success" "output":{}}` + text := `{"outcome":"success" "output"{}}` if generated > 1 { text = `{"outcome":"success","output":{}}` } diff --git a/internal/stage/model_prompt.go b/internal/stage/model_prompt.go index c5903b50..04f9b880 100644 --- a/internal/stage/model_prompt.go +++ b/internal/stage/model_prompt.go @@ -100,7 +100,15 @@ func stageResultText(text string) string { decoder := json.NewDecoder(strings.NewReader(remaining[start:])) var object json.RawMessage if err := decoder.Decode(&object); err != nil { - return text + // The object is malformed. Strip surrounding prose from a single + // balanced span so a following deterministic syntax repair sees just + // the object; strict envelope validation still follows. + if result == "" { + if span, ok := objectSpan(remaining[start:]); ok && !strings.Contains(remaining[start+len(span):], "{") { + result = span + } + } + break } if result != "" { return text @@ -118,6 +126,41 @@ func stageResultText(text string) string { return text } +// objectSpan returns the balanced object that begins at text[0], respecting JSON +// string escaping, so surrounding prose can be stripped from an otherwise +// malformed object. Strict envelope validation still follows. +func objectSpan(text string) (string, bool) { + if len(text) == 0 || text[0] != '{' { + return "", false + } + depth, inString, escape := 0, false, false + for i := 0; i < len(text); i++ { + b := text[i] + if inString { + if escape { + escape = false + } else if b == '\\' { + escape = true + } else if b == '"' { + inString = false + } + continue + } + switch b { + case '"': + inString = true + case '{', '[': + depth++ + case '}', ']': + depth-- + if depth == 0 { + return text[:i+1], true + } + } + } + return "", false +} + func toolMarkup(text string) bool { depth, inString, escape := 0, false, false for i := 0; i < len(text); i++ { diff --git a/internal/stage/pi_local_test.go b/internal/stage/pi_local_test.go index e85c6d19..2be43ccd 100644 --- a/internal/stage/pi_local_test.go +++ b/internal/stage/pi_local_test.go @@ -53,7 +53,7 @@ func TestPiInstalledLocalIntegration(t *testing.T) { delta := map[string]any{"role": "assistant", "content": `{"outcome":"success","output":{"summary":"done"}}`} finish := "stop" if which == "repair" && call == 1 { - delta["content"] = `{"outcome":"success" "output":{"summary":"done"}}` + delta["content"] = `{"outcome":"success" "output"{"summary":"done"}}` } if call == 1 && which != "budget" && which != "repair" { name := "write" diff --git a/internal/stage/pi_repair_test.go b/internal/stage/pi_repair_test.go index 037413f2..2a382370 100644 --- a/internal/stage/pi_repair_test.go +++ b/internal/stage/pi_repair_test.go @@ -35,12 +35,12 @@ func TestPiIsolatedRepairOwnershipAndCumulativeUsage(t *testing.T) { return nil }} config := ModelConfig{Provider: "fixture", Model: "model", Budget: NewBudget(0, 100)} - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) defer cancel() if _, err := c.Generate(ctx, config, "task"); err != nil { t.Fatal(err) } - original := `{"outcome":"success" "output":{"summary":"done"}}` + original := `{"outcome":"success" "output"{"summary":"done"}}` result, err := repairModelResult(ctx, c, config, original, map[string]string{"success": "#success", "error": "#failure"}, fmt.Errorf("missing comma")) if err != nil || result.Outcome != "success" || result.Output["summary"] != "done" { t.Fatal(result, err) diff --git a/internal/stage/result_repair.go b/internal/stage/result_repair.go index 05e90b67..948faae7 100644 --- a/internal/stage/result_repair.go +++ b/internal/stage/result_repair.go @@ -35,12 +35,23 @@ type formatRepairer interface { type repairRecorder interface{ RecordFormatRepair(FormatRepair) } type FormatRepair struct { Method string `json:"method"` + Status string `json:"status"` Accepted bool `json:"accepted"` Diagnostic string `json:"diagnostic"` Original string `json:"original"` Candidate string `json:"candidate"` + ByteOffset int `json:"byte_offset,omitempty"` } +// Repair outcome words, retained in the format_repairs evidence and surfaced in +// the failure diagnostic: a fully lossless repair, a valid-JSON candidate that +// still fails envelope validation, or nothing recoverable. +const ( + repairStatusRepaired = "repaired" + repairStatusPartial = "partial" + repairStatusUnrecoverable = "unrecoverable" +) + func (c OpenCodeClient) RepairFormat(ctx context.Context, config ModelConfig, text string, parseError error) (string, error) { c.connection = nil // Keep the primary stage's ownership record; the formatter is independent. @@ -80,65 +91,235 @@ func repairModelResult(ctx context.Context, client ModelGenerator, config ModelC return Result{}, err } original := stageResultText(text) - record := func(candidate, method string, accepted bool) { + offset := resultByteOffset(original) + record := func(candidate, method, status string, accepted bool, byteOffset int) { if recorder, ok := client.(repairRecorder); ok { bounded := original if len(bounded) > 32<<10 { bounded = bounded[:32<<10] + "\n[truncated]" } - recorder.RecordFormatRepair(FormatRepair{Original: bounded, Candidate: candidate, Diagnostic: parseError.Error(), Accepted: accepted, Method: method}) + recorder.RecordFormatRepair(FormatRepair{Original: bounded, Candidate: candidate, Diagnostic: parseError.Error(), Accepted: accepted, Method: method, Status: status, ByteOffset: byteOffset}) } } if len(original) > 32<<10 { - record("", "validation", false) - return Result{}, fmt.Errorf("%w; format repair skipped: response exceeds 32 KiB repair bound", parseError) + record("", "validation", repairStatusUnrecoverable, false, offset) + return Result{}, fmt.Errorf("%w; format repair skipped: response exceeds 32 KiB repair bound; recovery status %s", parseError, repairStatusUnrecoverable) } if json.Valid([]byte(original)) { - record("", "validation", false) - return Result{}, parseError + // Syntactically valid but the envelope is wrong (unknown, duplicate or + // missing fields). No syntax repair can help, and a partial result is + // never accepted as complete. + record("", "validation", repairStatusPartial, false, 0) + return Result{}, fmt.Errorf("%w; recovery status %s", parseError, repairStatusPartial) } if tokens, ok := jsonLexemes(original); !ok || len(tokens) == 0 || tokens[0] != "{" { - record("", "validation", false) - return Result{}, fmt.Errorf("%w; unsafe to repair incomplete values or a non-object response", parseError) + record("", "validation", repairStatusUnrecoverable, false, offset) + return Result{}, fmt.Errorf("%w; unsafe to repair incomplete values or a non-object response; recovery status %s", parseError, repairStatusUnrecoverable) } - // The most common truncation has one unique repair: close the existing - // containers. This cannot change a verdict or any completed evidence value. - if candidate, ok := closeJSONContainers(original); ok { - if result, err := decodeModelResult(candidate, edges); err == nil { - record(candidate, "closing delimiters", true) - return result, nil - } + // Cheapest option first: a deterministic, local repair that only inserts + // delimiters already implied by the payload or removes one trailing comma. + // It never invents, reorders or rewrites a value. + if result, candidate, method, ok := deterministicRepair(original, edges); ok { + record(candidate, method, repairStatusRepaired, true, offset) + return result, nil } formatter, ok := client.(formatRepairer) if !ok { - return Result{}, fmt.Errorf("%w; no isolated format repair available", parseError) + record("", "validation", repairStatusUnrecoverable, false, offset) + return Result{}, fmt.Errorf("%w; no isolated format repair available; recovery status %s", parseError, repairStatusUnrecoverable) } - // One formatter request, independent of execution/transport retries. - repairCtx, cancel := context.WithTimeout(ctx, 30*time.Second) + // One formatter request, independent of execution/transport retries. The + // deadline is derived from the payload size and the attempt's own remaining + // budget rather than a fixed short constant, so a larger result is not + // guaranteed to time out while the attempt context remains the outer bound. + remaining := time.Hour + if deadline, ok := ctx.Deadline(); ok { + remaining = time.Until(deadline) + } + repairCtx, cancel := context.WithTimeout(ctx, repairDeadline(len(original), remaining)) defer cancel() candidate, err := formatter.RepairFormat(repairCtx, config, original, parseError) if err := ctx.Err(); err != nil { return Result{}, err } if err != nil { - return Result{}, fmt.Errorf("%w; isolated format repair failed: %w", parseError, err) + record("", "tool-free formatter", repairStatusUnrecoverable, false, offset) + return Result{}, fmt.Errorf("%w; isolated format repair failed: %w; recovery status %s", parseError, err, repairStatusUnrecoverable) } candidate = stageResultText(candidate) if len(candidate) > 32<<10 { - return Result{}, fmt.Errorf("%w; format repair rejected: response exceeds 32 KiB repair bound", parseError) + record(candidate, "tool-free formatter", repairStatusUnrecoverable, false, offset) + return Result{}, fmt.Errorf("%w; format repair rejected: response exceeds 32 KiB repair bound; recovery status %s", parseError, repairStatusUnrecoverable) } if !losslessSyntaxRepair(original, candidate) { - record(candidate, "tool-free formatter", false) - return Result{}, fmt.Errorf("%w; format repair rejected: changed values or existing structure", parseError) + record(candidate, "tool-free formatter", repairStatusUnrecoverable, false, offset) + return Result{}, fmt.Errorf("%w; format repair rejected: changed values or existing structure; recovery status %s", parseError, repairStatusUnrecoverable) } result, err := decodeModelResult(candidate, edges) - record(candidate, "tool-free formatter", err == nil) if err != nil { - return Result{}, fmt.Errorf("%w; format repair still invalid: %v", parseError, err) + // A candidate that is valid JSON but still fails the envelope is a + // partial salvage, never a completed result. + status := repairStatusUnrecoverable + if json.Valid([]byte(candidate)) { + status = repairStatusPartial + } + record(candidate, "tool-free formatter", status, false, offset) + return Result{}, fmt.Errorf("%w; format repair still invalid: %v; recovery status %s", parseError, err, status) } + record(candidate, "tool-free formatter", repairStatusRepaired, true, offset) return result, nil } +// deterministicRepair repairs only the unambiguous structural errors that can be +// derived from the payload text itself: appended closing delimiters, one removed +// trailing comma before a closer, or one inserted comma/colon at the parser's +// reported byte. It returns the decoded result, the repaired candidate, the +// recorded method and whether the repair succeeded. Anything ambiguous (for +// example an unescaped quote or colon inside a string value, where more than one +// completion is possible) returns not-ok so the bounded model re-emit is tried +// instead of guessing. Every accepted candidate is value-preserving: it passes +// losslessSyntaxRepair before decodeModelResult. +func deterministicRepair(original string, edges map[string]string) (Result, string, string, bool) { + if candidate, ok := closeJSONContainers(original); ok { + if result, err := decodeModelResult(candidate, edges); err == nil { + return result, candidate, "deterministic closing delimiters", true + } + } + // A single trailing comma before a closing delimiter is a structural + // deletion. Reversed lossless validation proves the original is the stripped + // text plus exactly that one delimiter, so no value changed. + if candidate, ok := stripTrailingComma(original); ok { + if losslessSyntaxRepair(candidate, original) { + if result, err := decodeModelResult(candidate, edges); err == nil { + return result, candidate, "deterministic trailing comma", true + } + } + } + offset, ok := syntaxErrorOffset(original) + if !ok { + return Result{}, "", "", false + } + at := offset - 1 + if at < 0 || at > len(original) { + return Result{}, "", "", false + } + var ( + repaired Result + candidate string + matches int + ) + for _, delimiter := range []string{",", ":"} { + inserted := original[:at] + delimiter + original[at:] + if !json.Valid([]byte(inserted)) || !losslessSyntaxRepair(original, inserted) { + continue + } + result, err := decodeModelResult(inserted, edges) + if err != nil { + continue + } + repaired, candidate, matches = result, inserted, matches+1 + } + if matches != 1 { + // No completion, or more than one: never guess. + return Result{}, "", "", false + } + return repaired, candidate, fmt.Sprintf("deterministic delimiter at byte %d", offset), true +} + +// syntaxErrorOffset reports the byte the JSON parser rejected, matching the +// offset retained by modelJSONError. An unexpected end of input is not a +// structural delimiter error and reports not-ok. +func syntaxErrorOffset(text string) (int, bool) { + syntax, ok := json.Unmarshal([]byte(text), new(json.RawMessage)).(*json.SyntaxError) + if !ok { + return 0, false + } + return int(syntax.Offset), true +} + +// resultByteOffset re-derives the retained byte offset for a malformed payload, +// falling back to the payload length for truncation, exactly as modelJSONError. +func resultByteOffset(text string) int { + if offset, ok := syntaxErrorOffset(text); ok { + return offset + } + return len(text) +} + +// stripTrailingComma removes one comma that immediately precedes a closing +// delimiter. More than one candidate is ambiguous and reports not-ok. +func stripTrailingComma(text string) (string, bool) { + tokens, ok := jsonLexemes(text) + if !ok { + return "", false + } + offsets, ok := lexemeOffsets(text, tokens) + if !ok { + return "", false + } + found := -1 + for i, token := range tokens { + if token != "," || i+1 >= len(tokens) || (tokens[i+1] != "}" && tokens[i+1] != "]") { + continue + } + if found >= 0 { + return "", false + } + found = offsets[i] + } + if found < 0 { + return "", false + } + return text[:found] + text[found+1:], true +} + +// lexemeOffsets locates each token returned by jsonLexemes in the source text. +// Tokens contain no whitespace and are consumed in order, so a forward scan from +// the end of the previous token finds each position without re-lexing. +func lexemeOffsets(text string, tokens []string) ([]int, bool) { + offsets := make([]int, len(tokens)) + pos := 0 + for i, token := range tokens { + index := strings.Index(text[pos:], token) + if index < 0 { + return nil, false + } + offsets[i] = pos + index + pos += index + len(token) + } + return offsets, true +} + +// Isolated formatter deadline bounds. The allowance grows by one second per KiB +// of payload up to a ceiling, is floored for ordinary payloads, and is clamped +// to the attempt's remaining budget so the stage/attempt context stays the outer +// bound and a large result is not guaranteed to time out. +const ( + repairDeadlineFloor = 30 * time.Second + repairDeadlineMax = 10 * time.Minute + repairDeadlineMargin = 2 * time.Second +) + +// repairDeadline derives the formatter deadline. It is O(1) and monotonic +// non-decreasing in payloadBytes; it never exceeds remaining and, when the +// remaining budget allows, never drops below repairDeadlineFloor. +func repairDeadline(payloadBytes int, remaining time.Duration) time.Duration { + if remaining <= 0 { + return 0 + } + allowance := repairDeadlineFloor + time.Duration(payloadBytes/1024)*time.Second + if allowance > repairDeadlineMax { + allowance = repairDeadlineMax + } + if limit := remaining - repairDeadlineMargin; allowance > limit { + allowance = limit + } + if allowance < 0 { + return 0 + } + return allowance +} + // Lexemes preserve strings (including escape sequences), numbers and structural // delimiters verbatim. A formatter cannot change approval, omit evidence, move // keys, flatten objects, or replace an error outcome with success. diff --git a/internal/stage/result_repair_test.go b/internal/stage/result_repair_test.go index 566210df..88d4b5f6 100644 --- a/internal/stage/result_repair_test.go +++ b/internal/stage/result_repair_test.go @@ -9,6 +9,7 @@ import ( "path/filepath" "strings" "testing" + "time" ) type repairFixture struct { @@ -46,7 +47,10 @@ func TestMissingBraceRepairPreservesApprovalWithZeroExecutionRetries(t *testing. } func TestToolFreeFormatterGetsExactErrorAndCannotChangeVerdictOrEvidence(t *testing.T) { - original := `{"outcome":"error" "output":{"approval":false,"evidence":["check failed"]}}` + // Two structural errors (missing comma and missing colon) so the + // deterministic single-delimiter pass cannot repair it and the isolated + // formatter path is exercised. + original := `{"outcome":"error" "output"{"approval":false,"evidence":["check failed"]}}` correct := `{"outcome":"error", "output":{"approval":false,"evidence":["check failed"]}}` for _, candidate := range []string{correct, strings.Replace(correct, `"error"`, `"success"`, 1), strings.Replace(correct, "false", "true", 1), strings.Replace(correct, "check failed", "check passed", 1), `{"outcome":"error","output":{"approval":false}}`} { f := &repairFixture{response: original, candidate: candidate} @@ -73,8 +77,8 @@ func TestToolFreeFormatterGetsExactErrorAndCannotChangeVerdictOrEvidence(t *test func TestAgentFormattingRepairUsesFreshDenyAllSessionWithoutReplayingEdit(t *testing.T) { dir := t.TempDir() path := filepath.Join(dir, "README.md") - original := `{"outcome":"success" "output":{"approval":true,"evidence":"edited once"}}` - candidate := strings.Replace(original, `"success" "output"`, `"success", "output"`, 1) + original := `{"outcome":"success" "output"{"approval":true,"evidence":"edited once"}}` + candidate := strings.Replace(original, `"success" "output"{`, `"success", "output":{`, 1) creates, edits, generates, owned := 0, 0, 0, 0 agent := &AgentClient{OpenCodeClient: OpenCodeClient{Directory: dir}, OnSession: func(string, bool) error { owned++; return nil }} agent.command = func(_ context.Context, _ string, args ...string) ([]byte, error) { @@ -206,3 +210,164 @@ func TestLosslessSyntaxRepairRejectsFlippedValuesAndNonStructuralTokens(t *testi } } } + +// TestDeterministicRepairOnlyUnambiguousStructuralErrors proves the local pass +// repairs exactly the derivable cases (missing comma, missing colon, one trailing +// comma, missing closers) and never guesses at an ambiguous unescaped quote or +// colon inside a string value. +func TestDeterministicRepairOnlyUnambiguousStructuralErrors(t *testing.T) { + edges := modelDefinition().Edges + for _, tc := range []struct { + name, original, method string + repaired bool + }{ + {"missing comma", `{"outcome":"error" "output":{"approval":false}}`, "deterministic delimiter at byte 20", true}, + {"missing colon", `{"outcome" "success","output":{"a":true}}`, "deterministic delimiter at byte 12", true}, + {"trailing comma", `{"outcome":"success","output":{"a":true,}}`, "deterministic trailing comma", true}, + {"missing closing brace", `{"outcome":"success","output":{"a":true}`, "deterministic closing delimiters", true}, + {"unescaped quote before colon", `{"outcome":"rejected","output":{"summary":"same graph.":"","platform":"x"}}`, "", false}, + {"stray colon between values", `{"outcome":"success","output":{"a":1:2}}`, "", false}, + {"unescaped quote inside value", `{"outcome":"success","output":{"summary":"he said "hi""}}`, "", false}, + } { + t.Run(tc.name, func(t *testing.T) { + result, candidate, method, ok := deterministicRepair(tc.original, edges) + if ok != tc.repaired || method != tc.method { + t.Fatalf("repaired=%v method=%q candidate=%q result=%v", ok, method, candidate, result) + } + if ok { + if candidate == tc.original || !json.Valid([]byte(candidate)) { + t.Fatalf("candidate was not a real repair: %q", candidate) + } + if result.Outcome == "" { + t.Fatalf("repair did not decode: %+v", result) + } + } + }) + } +} + +// TestRepairStatusesAndEvidence proves the repaired/partial/unrecoverable outcome +// is recorded and surfaced in the failure diagnostic, and that a partial parse is +// never returned as a completed result. +func TestRepairStatusesAndEvidence(t *testing.T) { + t.Run("deterministic repaired", func(t *testing.T) { + original := `{"outcome":"success" "output":{"approval":true}}` + f := &repairFixture{response: original} + result, err := (ModelStage{Client: f}).Run(context.Background(), modelDefinition(), nil) + if err != nil || result.Output["approval"] != true || f.repairCalls != 0 || f.generateCalls != 1 { + t.Fatalf("deterministic repair failed: %v %v %+v", result, err, f) + } + if len(f.audits) != 1 || f.audits[0].Status != repairStatusRepaired || !f.audits[0].Accepted || !strings.HasPrefix(f.audits[0].Method, "deterministic delimiter at byte ") { + t.Fatalf("repaired evidence wrong: %+v", f.audits) + } + }) + t.Run("formatter partial", func(t *testing.T) { + original := `{"outcome":"success","output":{"a":1} "orphan":true}` + candidate := `{"outcome":"success","output":{"a":1}, "orphan":true}` + f := &repairFixture{response: original, candidate: candidate} + result, err := (ModelStage{Client: f}).Run(context.Background(), modelDefinition(), nil) + if err == nil || result.Output != nil || !errors.Is(err, ErrInvalidModelResult) || !strings.Contains(err.Error(), "recovery status "+repairStatusPartial) { + t.Fatalf("partial candidate was accepted as complete: %v %v", result, err) + } + if len(f.audits) != 1 || f.audits[0].Status != repairStatusPartial || f.audits[0].Accepted { + t.Fatalf("partial evidence wrong: %+v", f.audits) + } + }) + t.Run("unrecoverable retains offset", func(t *testing.T) { + original := `{"outcome":"rejected","output":{"summary":"same graph.":"","platform":"x"}}` + f := &repairFixture{response: original, candidate: original} + result, err := (ModelStage{Client: f}).Run(context.Background(), modelDefinition(), nil) + if err == nil || result.Output != nil || !strings.Contains(err.Error(), "recovery status "+repairStatusUnrecoverable) { + t.Fatalf("unrecoverable payload not reported: %v %v", result, err) + } + if len(f.audits) != 1 || f.audits[0].Status != repairStatusUnrecoverable || f.audits[0].Accepted || f.audits[0].ByteOffset == 0 { + t.Fatalf("unrecoverable evidence wrong: %+v", f.audits) + } + }) +} + +// TestPresentationStrippingBeforeDeterministicRepair proves a single fence and +// surrounding prose are stripped before the local repair pass. +func TestPresentationStrippingBeforeDeterministicRepair(t *testing.T) { + malformed := `{"outcome":"success" "output":{"approval":true}}` + for _, wrapped := range []string{ + "```json\n" + malformed + "\n```", + "Here is the result:\n" + malformed + "\nHope that helps.", + "```json\nHere:\n" + malformed + "\n```", + } { + f := &repairFixture{response: wrapped} + result, err := (ModelStage{Client: f}).Run(context.Background(), modelDefinition(), nil) + if err != nil || result.Output["approval"] != true || f.repairCalls != 0 { + t.Fatalf("presentation not stripped before repair: %q %v %v", wrapped, result, err) + } + } +} + +// TestRepairDeadlineIsMonotonicFlooredAndClamped proves the derived deadline, not +// a fixed constant, bounds the single formatter request. +func TestRepairDeadlineIsMonotonicFlooredAndClamped(t *testing.T) { + if got := repairDeadline(0, time.Hour); got != repairDeadlineFloor { + t.Fatalf("floor not applied: %s", got) + } + if got := repairDeadline(1<<30, 24*time.Hour); got != repairDeadlineMax { + t.Fatalf("ceiling not applied: %s", got) + } + if got := repairDeadline(10, 0); got != 0 { + t.Fatalf("zero remaining should yield zero: %s", got) + } + if clamped := repairDeadline(1<<20, 5*time.Second); clamped != 5*time.Second-repairDeadlineMargin || clamped > 5*time.Second { + t.Fatalf("remaining budget was not the bound: %s", clamped) + } + previous := time.Duration(0) + for size := 0; size <= 64<<10; size += 1024 { + got := repairDeadline(size, time.Hour) + if got < previous { + t.Fatalf("deadline not monotonic at %d: %s < %s", size, got, previous) + } + previous = got + } +} + +// TestShellStageRecoverableMalformedResultDoesNotFailRun proves a shell-enabled +// stage whose final result is malformed but deterministically recoverable +// completes without replaying admitted tool work, and safeToReplay is unchanged. +func TestShellStageRecoverableMalformedResultDoesNotFailRun(t *testing.T) { + dir := t.TempDir() + malformed := `{"outcome":"success" "output":{"approval":true,"evidence":["shell verification passed"]}}` + creates, prompts, owned := 0, 0, 0 + agent := &AgentClient{OpenCodeClient: OpenCodeClient{Directory: dir}, OnSession: func(string, bool) error { owned++; return nil }} + agent.command = func(_ context.Context, _ string, args ...string) ([]byte, error) { + if args[1] == "delete" { + t.Fatal("deterministic repair must retain the admitted session") + } + switch args[2] { + case "/api/session": + creates++ + return []byte(`{"data":{"id":"ses_shell_1"}}`), nil + case "/api/session/ses_shell_1/prompt": + prompts++ + return []byte(`{}`), nil + case "/api/session/active": + return []byte(`{"data":{}}`), nil + case "/api/session/ses_shell_1" + finalAssistantReadback: + return json.Marshal(map[string]any{"data": []any{map[string]any{"finish": "stop", "content": []any{map[string]string{"type": "text", "text": malformed}}}}}) + default: + return nil, fmt.Errorf("unexpected API call %s", args[2]) + } + } + def := modelDefinition() + def.Blocks["tools"] = "shell" + if safeToReplay(def) { + t.Fatal("shell-enabled stage must not become replayable") + } + result, err := (ModelStage{Client: agent}).Run(context.Background(), def, nil) + if err != nil || result.Outcome != "success" || result.Output["approval"] != true { + t.Fatalf("deterministically recoverable shell-stage result failed the run: %v %v", result, err) + } + if creates != 1 || prompts != 1 || owned != 2 { + t.Fatalf("shell stage work was replayed: creates=%d prompts=%d owned=%d", creates, prompts, owned) + } + if !json.Valid([]byte(agent.Transcript)) || !strings.Contains(agent.Transcript, `"status":"repaired"`) || !strings.Contains(agent.Transcript, "deterministic delimiter") { + t.Fatalf("deterministic repair evidence missing: %s", agent.Transcript) + } +} diff --git a/internal/stage/retry_safety_test.go b/internal/stage/retry_safety_test.go index 25d9691d..f5f812e9 100644 --- a/internal/stage/retry_safety_test.go +++ b/internal/stage/retry_safety_test.go @@ -154,7 +154,7 @@ func TestMalformedResultToleratedOnlyForReadOnlyStages(t *testing.T) { if err != nil || result.Output["ok"] != true || calls != 2 { t.Fatalf("read-only malformed result was not tolerated: %v %v calls=%d", result, err, calls) } - if len(client.repairs) != 1 || client.repairs[0].Accepted || client.repairs[0].Method != "validation" || client.repairs[0].Diagnostic == "" { + if len(client.repairs) != 1 || client.repairs[0].Accepted || client.repairs[0].Method != "validation" || client.repairs[0].Status != repairStatusPartial || client.repairs[0].Diagnostic == "" { t.Fatalf("format_repairs evidence missing or wrong: %+v", client.repairs) } })