7b72b8f20f
Treat a failure to reach the harness before any prompt or tool call is admitted (connecting or creating the session) as a distinct retryable reach failure, so a transport blip at the first stage no longer ends a mutating run; keep post-admission readback/poll/agent failures retried only on an explicit read-only stage and never replay admitted tool work. Preserve the true cause when the context is already done (join ctx.Err with the typed transport error) and report a sanitized unknown provider error name/status instead of a blanket summary. Make the isolated format-repair deadline proportional to the stage timeout (floored, capped and clamped to remaining minus a margin) so a slow-but-repairable result is recovered without ever extending the stage deadline; the bounded ladder, value guard, deny-all formatter session, cumulative usage accounting and no-tool-replay rule are unchanged. Update docs/model-results.md, docs/pipeline-performance-review.md and the packaged skill references; add stage, opencode and server fixtures.
351 lines
12 KiB
Go
351 lines
12 KiB
Go
package stage
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"regexp"
|
|
"strconv"
|
|
"strings"
|
|
"time"
|
|
)
|
|
|
|
const ModelType = "model"
|
|
|
|
const ModelID = "deepseek-v4.1-flash"
|
|
|
|
// Retained model-stage failure categories. A strict final-result validation
|
|
// failure is safe and never replays admitted tool work; a harness transport,
|
|
// readback or wait failure is a different operator action. Labels are fixed and
|
|
// sanitized: no provider or model response body is copied into them.
|
|
const (
|
|
finalResultValidation = "FINAL RESULT VALIDATION"
|
|
harnessTransport = "HARNESS TRANSPORT"
|
|
)
|
|
|
|
// finalResultGuidance is a fixed sanitized sentence appended only to validation
|
|
// diagnostics. It states the retained side effects and the cost of a full retry.
|
|
const finalResultGuidance = "completed edits and tool side effects are retained and not undone; a full retry repeats work"
|
|
|
|
// harnessTransportFailure is the single classifier shared by both transport
|
|
// sentinels and the typed OpenCode error: a request, poll or readback failure
|
|
// that reports as the fixed HARNESS TRANSPORT category. A bare context deadline
|
|
// is classified separately so a genuine stage overrun is never relabeled as a
|
|
// transport failure.
|
|
func harnessTransportFailure(err error) bool {
|
|
var openCode *openCodeError
|
|
return errors.Is(err, ErrHarnessUnreachable) || errors.Is(err, ErrHarnessTransport) || errors.As(err, &openCode)
|
|
}
|
|
|
|
// harnessReachFailure reports whether err is a pre-admission harness reach
|
|
// failure. It is distinct from a post-admission ErrHarnessTransport failure: no
|
|
// session or prompt was admitted, so the retry replays no work.
|
|
func harnessReachFailure(err error) bool {
|
|
return errors.Is(err, ErrHarnessUnreachable)
|
|
}
|
|
|
|
// admittedWorker is implemented by a client that can admit tool work before its
|
|
// final result is read, so the retry loop can refuse to replay it without a
|
|
// per-client type switch.
|
|
type admittedWorker interface{ workAdmitted() bool }
|
|
|
|
// workAdmitted reports whether the client admitted any prompt or tool call, so
|
|
// the retry loop may retry a transport failure on any stage only when it is
|
|
// false, and only a read-only stage otherwise.
|
|
func (c *AgentClient) workAdmitted() bool { return c.WorkStarted }
|
|
func (c *PiClient) workAdmitted() bool { return c.WorkStarted }
|
|
|
|
// modelFailureCategory classifies a stage failure using error identity only, so
|
|
// the label stays sanitized and bounded. Validation is checked first: a
|
|
// structured-output failure is never reported as a transport failure and vice versa.
|
|
func modelFailureCategory(err error) string {
|
|
switch {
|
|
case errors.Is(err, ErrInvalidModelResult):
|
|
return finalResultValidation
|
|
case harnessTransportFailure(err), errors.Is(err, context.DeadlineExceeded):
|
|
return harnessTransport
|
|
default:
|
|
return ""
|
|
}
|
|
}
|
|
|
|
// wrapModelStageFailure preserves the underlying detail via %w while adding the
|
|
// stage key and, where known, the failure category. Budget/cleanup and other
|
|
// unclassified failures keep their previous single-prefix form.
|
|
func wrapModelStageFailure(key string, err error) error {
|
|
if err == nil {
|
|
return nil
|
|
}
|
|
switch modelFailureCategory(err) {
|
|
case finalResultValidation:
|
|
return fmt.Errorf("model stage %s: %s: %w; %s", key, finalResultValidation, err, finalResultGuidance)
|
|
case harnessTransport:
|
|
return fmt.Errorf("model stage %s: %s: %w", key, harnessTransport, err)
|
|
default:
|
|
return fmt.Errorf("model stage %s: %w", key, err)
|
|
}
|
|
}
|
|
|
|
// replayablePostAdmission reports whether a post-admission error can be retried
|
|
// from the start without risking a repeated tool mutation: a stage context
|
|
// deadline or a harness transport failure. A structured-output validation
|
|
// failure is classified separately as ErrInvalidModelResult.
|
|
func replayablePostAdmission(err error) bool {
|
|
return errors.Is(err, context.DeadlineExceeded) || errors.Is(err, ErrHarnessTransport)
|
|
}
|
|
|
|
// toleratedModelResultRetry reports whether a malformed final result may be
|
|
// re-emitted. It requires an explicit read-only stage, remaining attempts,
|
|
// live work, available budget and a plain validation error; fail-closed
|
|
// accounting and cleanup errors still stop immediately.
|
|
func toleratedModelResultRetry(ctx context.Context, def Definition, config ModelConfig, attempt int, resultErr error) bool {
|
|
return errors.Is(resultErr, ErrInvalidModelResult) &&
|
|
!errors.Is(resultErr, ErrBudgetExceeded) &&
|
|
!errors.Is(resultErr, ErrBudgetAccounting) &&
|
|
!errors.Is(resultErr, ErrStageCleanup) &&
|
|
safeToReplay(def) && attempt < config.Retries && ctx.Err() == nil && config.Budget != nil && config.Budget.Check() == nil
|
|
}
|
|
|
|
var providerID = regexp.MustCompile(`^[A-Za-z0-9_.-]+$`)
|
|
var modelID = regexp.MustCompile(`^[A-Za-z0-9_./:-]+$`)
|
|
|
|
func ValidModel(provider, model string) bool {
|
|
return len(provider) <= 200 && len(model) <= 300 && providerID.MatchString(provider) && modelID.MatchString(model)
|
|
}
|
|
|
|
// ModelConfig and ModelSpec group the settings/contracts that belong specifically
|
|
// to an LLM stage. Other stage types need not have a prompt, provider, or model.
|
|
type ModelConfig struct {
|
|
Harness string
|
|
Provider string
|
|
Model string
|
|
Retries int
|
|
Timeout time.Duration
|
|
BudgetUSD float64
|
|
BudgetTokens int64
|
|
Budget *Budget
|
|
ResultSchema map[string]any
|
|
}
|
|
|
|
type ModelSpec struct {
|
|
Output string
|
|
Prompt string
|
|
Config ModelConfig
|
|
}
|
|
|
|
// ModelGenerator isolates the provider integration from stage instructions/results.
|
|
type ModelGenerator interface {
|
|
Generate(context.Context, ModelConfig, string) (string, error)
|
|
}
|
|
|
|
type ModelStage struct {
|
|
Client ModelGenerator // nil uses the locally authenticated OpenCode service
|
|
}
|
|
|
|
var _ Stage = ModelStage{}
|
|
|
|
func (m ModelStage) Validate(def Definition) error {
|
|
_, err := m.Spec(def)
|
|
return err
|
|
}
|
|
|
|
func (ModelStage) Spec(def Definition) (ModelSpec, error) {
|
|
if EffectiveType(def.Type) != ModelType {
|
|
return ModelSpec{}, fmt.Errorf("expected model stage, got %q", def.Type)
|
|
}
|
|
for _, block := range []string{"output", "prompt", "configs"} {
|
|
if strings.TrimSpace(def.Blocks[block]) == "" {
|
|
return ModelSpec{}, fmt.Errorf("required block %s is missing or empty", block)
|
|
}
|
|
}
|
|
for block := range def.Blocks {
|
|
switch block {
|
|
case "output", "prompt", "edges", "configs", "tools", "permissions":
|
|
default:
|
|
return ModelSpec{}, fmt.Errorf("unexpected block %q for model stage", block)
|
|
}
|
|
}
|
|
if _, _, err := ToolNames(def); err != nil {
|
|
return ModelSpec{}, err
|
|
}
|
|
if _, err := StagePermissions(def); err != nil {
|
|
return ModelSpec{}, err
|
|
}
|
|
for _, key := range []string{"provider", "model"} {
|
|
if def.Configs[key] == "" {
|
|
return ModelSpec{}, fmt.Errorf("missing config %s", key)
|
|
}
|
|
}
|
|
for key := range def.Configs {
|
|
switch key {
|
|
case "provider", "model", "retries", "timeout", "budget_usd", "budget_tokens", "harness":
|
|
default:
|
|
return ModelSpec{}, fmt.Errorf("unknown config %q", key)
|
|
}
|
|
}
|
|
if !ValidModel(def.Configs["provider"], def.Configs["model"]) {
|
|
return ModelSpec{}, fmt.Errorf("invalid provider/model identifier; use the exact IDs validated during setup")
|
|
}
|
|
backend := def.Configs["harness"]
|
|
if backend == "" {
|
|
backend = "opencode"
|
|
}
|
|
if backend != "opencode" && backend != "pi" {
|
|
return ModelSpec{}, fmt.Errorf("harness must be opencode or pi")
|
|
}
|
|
if backend == "pi" {
|
|
tools, _, _ := ToolNames(def)
|
|
if _, err := piTools(tools); err != nil {
|
|
return ModelSpec{}, err
|
|
}
|
|
}
|
|
retries := DefaultRetries
|
|
var err error
|
|
if value := def.Configs["retries"]; value != "" {
|
|
retries, err = strconv.Atoi(value)
|
|
}
|
|
if err != nil || retries < 0 || retries > 100 {
|
|
return ModelSpec{}, fmt.Errorf("retries must be an integer from 0 to 100")
|
|
}
|
|
timeout, err := StageTimeout(def)
|
|
if err != nil {
|
|
return ModelSpec{}, err
|
|
}
|
|
usd, tokens, err := parseBudget(def)
|
|
if err != nil {
|
|
return ModelSpec{}, err
|
|
}
|
|
return ModelSpec{
|
|
Output: def.Blocks["output"], Prompt: def.Blocks["prompt"],
|
|
Config: ModelConfig{Harness: backend, Provider: def.Configs["provider"], Model: def.Configs["model"], Retries: retries, Timeout: timeout, BudgetUSD: usd, BudgetTokens: tokens},
|
|
}, nil
|
|
}
|
|
|
|
func (m ModelStage) Run(ctx context.Context, def Definition, input Values) (Result, error) {
|
|
if err := ctx.Err(); err != nil {
|
|
return Result{}, err
|
|
}
|
|
spec, err := m.Spec(def)
|
|
if err != nil {
|
|
return Result{}, err
|
|
}
|
|
spec.Config.ResultSchema = resultSchema(def.Edges)
|
|
prompt, err := modelPrompt(def, spec, input)
|
|
if err != nil {
|
|
return Result{}, err
|
|
}
|
|
spec.Config.Budget = budgetFromContext(ctx)
|
|
if spec.Config.Budget == nil {
|
|
spec.Config.Budget = NewBudget(spec.Config.BudgetUSD, spec.Config.BudgetTokens)
|
|
}
|
|
client := m.Client
|
|
if client == nil {
|
|
if spec.Config.Harness == "pi" {
|
|
return Result{}, fmt.Errorf("Pi stages require an audited Pi client")
|
|
}
|
|
tools, _, _ := ToolNames(def)
|
|
if len(tools) > 0 {
|
|
return Result{}, fmt.Errorf("tool-enabled stages require an audited agent client")
|
|
}
|
|
client = &OpenCodeClient{}
|
|
}
|
|
if spec.Config.Budget.Enabled() {
|
|
capable, ok := client.(interface{ BudgetCapable() bool })
|
|
if !ok || !capable.BudgetCapable() {
|
|
return Result{}, fmt.Errorf("%w: generator cannot enforce usage limits", ErrBudgetAccounting)
|
|
}
|
|
}
|
|
// Replaying admitted tool work can repeat mutations. Only a stage whose
|
|
// explicit tools cannot mutate the run directory (safeToReplay) may
|
|
// automatically retry work that started after admission.
|
|
if agent, ok := client.(*AgentClient); ok {
|
|
if tools, explicit, _ := ToolNames(def); explicit {
|
|
agent.Tools = tools
|
|
}
|
|
agent.Policy, _ = StagePermissions(def)
|
|
}
|
|
for attempt := 0; attempt <= spec.Config.Retries; attempt++ {
|
|
if pi, ok := client.(*PiClient); ok {
|
|
pi.Policy, _ = StagePermissions(def)
|
|
}
|
|
if err := ctx.Err(); err != nil {
|
|
return Result{}, err
|
|
}
|
|
if err := spec.Config.Budget.Check(); err != nil {
|
|
return Result{}, err
|
|
}
|
|
spec.Config.Budget.Attempt()
|
|
attemptCtx, cancel := context.WithTimeout(ctx, spec.Config.Timeout)
|
|
text, generateErr := client.Generate(attemptCtx, spec.Config, prompt)
|
|
if generateErr == nil && attemptCtx.Err() != nil {
|
|
generateErr = attemptCtx.Err()
|
|
}
|
|
replayPostAdmission := false
|
|
if generateErr == nil {
|
|
var result Result
|
|
var resultErr error
|
|
if resultFormat(text) == "line" {
|
|
writeRawOutputFile(def.ArtifactsDir, text)
|
|
result, _, resultErr = decodeLineResult(text, def.Edges)
|
|
} else {
|
|
result, resultErr = decodeModelResult(stageResultText(text), def.Edges)
|
|
if resultErr != nil {
|
|
result, resultErr = repairModelResult(attemptCtx, client, spec.Config, text, def.Edges, resultErr)
|
|
}
|
|
}
|
|
if attemptCtx.Err() != nil {
|
|
resultErr = errors.Join(resultErr, attemptCtx.Err())
|
|
}
|
|
cancel()
|
|
if resultErr == nil {
|
|
return result, nil
|
|
}
|
|
// A malformed final result mutates nothing on an explicit read-only
|
|
// stage, so it may be re-emitted within the retry budget. A mutating
|
|
// or legacy-bundle stage retains its admitted work and returns now.
|
|
if toleratedModelResultRetry(ctx, def, spec.Config, attempt, resultErr) {
|
|
err = resultErr
|
|
} else {
|
|
return result, wrapModelStageFailure(def.Key, resultErr)
|
|
}
|
|
} else {
|
|
err = generateErr
|
|
if attemptCtx.Err() != nil && !errors.Is(err, ErrBudgetExceeded) && !errors.Is(err, ErrBudgetAccounting) {
|
|
err = errors.Join(err, attemptCtx.Err())
|
|
}
|
|
cancel()
|
|
if ctx.Err() != nil {
|
|
return Result{}, ctx.Err()
|
|
}
|
|
if errors.Is(err, ErrBudgetExceeded) || errors.Is(err, ErrBudgetAccounting) || errors.Is(err, ErrStageCleanup) {
|
|
break
|
|
}
|
|
replayPostAdmission = safeToReplay(def) && replayablePostAdmission(err)
|
|
// A reach failure observed before any work was admitted replays
|
|
// nothing, so it is retryable within the existing budget for any
|
|
// stage. After admission, only an explicit read-only stage may
|
|
// retry; admitted tool work is never replayed.
|
|
workStarted := false
|
|
if worker, ok := client.(admittedWorker); ok {
|
|
workStarted = worker.workAdmitted()
|
|
}
|
|
if workStarted && !replayPostAdmission {
|
|
break
|
|
}
|
|
preAdmissionReach := !workStarted && harnessReachFailure(err)
|
|
if attempt == spec.Config.Retries || (!retryableModelError(err) && !errors.Is(err, context.DeadlineExceeded) && !replayPostAdmission && !preAdmissionReach) {
|
|
break
|
|
}
|
|
}
|
|
delay := time.Duration(1<<min(attempt, 4)) * 100 * time.Millisecond
|
|
timer := time.NewTimer(delay)
|
|
select {
|
|
case <-ctx.Done():
|
|
timer.Stop()
|
|
return Result{}, ctx.Err()
|
|
case <-timer.C:
|
|
}
|
|
}
|
|
return Result{}, wrapModelStageFailure(def.Key, err)
|
|
}
|