Files
pipelines/internal/stage/model.go
T
alan b05277a57d Add line-oriented result format for model and agent stages
Introduce a short, markdown-ish result format whose first non-empty line is
the declared outcome and whose "# output" section carries the routing data,
so stage output stops carrying multi-kilobyte prose inside a JSON envelope.

- New internal/stage/line_result.go: resultFormat() gates legacy JSON ('{'
  first) vs the new line format; decodeLineResult() parses outcome-from-first
  line, requires a JSON-object "# output" section, stores extra sections
  verbatim under normalised keys, and records a reserved result_parse status
  (parsed/partial/failed). Failures return a salvage Result plus
  ErrInvalidModelResult, and never route an undeclared outcome.
- internal/stage/model.go: write the raw response to raw-output.md (0600,
  best-effort) before parsing, and route the line format through
  decodeLineResult while leaving the JSON path (stageResultText ->
  decodeModelResult -> repairModelResult) unchanged.
- internal/stage/agent.go: agentResultText returns line-format text verbatim
  so the "# output" body is not stripped by the JSON-object extractor.
- internal/server/runner.go: merge a non-empty salvage Result into the error
  envelope under "salvage" on failure (a no-op for the legacy JSON path).
- internal/stage/line_result_test.go: raw-text fixtures covering happy path,
  missing/undeclared outcome, missing "# output", best-effort extras,
  adversarial prose, legacy JSON unchanged, >=64KB responses, and
  raw-output.md written before parsing.
2026-10-16 21:34:33 +01:00

320 lines
11 KiB
Go

package stage
import (
"context"
"errors"
"fmt"
"regexp"
"strconv"
"strings"
"time"
)
const ModelType = "model"
const ModelID = "deepseek-v4.1-flash"
// Retained model-stage failure categories. A strict final-result validation
// failure is safe and never replays admitted tool work; a harness transport,
// readback or wait failure is a different operator action. Labels are fixed and
// sanitized: no provider or model response body is copied into them.
const (
finalResultValidation = "FINAL RESULT VALIDATION"
harnessTransport = "HARNESS TRANSPORT"
)
// finalResultGuidance is a fixed sanitized sentence appended only to validation
// diagnostics. It states the retained side effects and the cost of a full retry.
const finalResultGuidance = "completed edits and tool side effects are retained and not undone; a full retry repeats work"
// modelFailureCategory classifies a stage failure using error identity only, so
// the label stays sanitized and bounded. Validation is checked first: a
// structured-output failure is never reported as a transport failure and vice versa.
func modelFailureCategory(err error) string {
var openCode *openCodeError
switch {
case errors.Is(err, ErrInvalidModelResult):
return finalResultValidation
case errors.Is(err, ErrHarnessTransport), errors.As(err, &openCode), errors.Is(err, context.DeadlineExceeded):
return harnessTransport
default:
return ""
}
}
// wrapModelStageFailure preserves the underlying detail via %w while adding the
// stage key and, where known, the failure category. Budget/cleanup and other
// unclassified failures keep their previous single-prefix form.
func wrapModelStageFailure(key string, err error) error {
if err == nil {
return nil
}
switch modelFailureCategory(err) {
case finalResultValidation:
return fmt.Errorf("model stage %s: %s: %w; %s", key, finalResultValidation, err, finalResultGuidance)
case harnessTransport:
return fmt.Errorf("model stage %s: %s: %w", key, harnessTransport, err)
default:
return fmt.Errorf("model stage %s: %w", key, err)
}
}
// replayablePostAdmission reports whether a post-admission error can be retried
// from the start without risking a repeated tool mutation: a stage context
// deadline or a harness transport failure. A structured-output validation
// failure is classified separately as ErrInvalidModelResult.
func replayablePostAdmission(err error) bool {
return errors.Is(err, context.DeadlineExceeded) || errors.Is(err, ErrHarnessTransport)
}
// toleratedModelResultRetry reports whether a malformed final result may be
// re-emitted. It requires an explicit read-only stage, remaining attempts,
// live work, available budget and a plain validation error; fail-closed
// accounting and cleanup errors still stop immediately.
func toleratedModelResultRetry(ctx context.Context, def Definition, config ModelConfig, attempt int, resultErr error) bool {
return errors.Is(resultErr, ErrInvalidModelResult) &&
!errors.Is(resultErr, ErrBudgetExceeded) &&
!errors.Is(resultErr, ErrBudgetAccounting) &&
!errors.Is(resultErr, ErrStageCleanup) &&
safeToReplay(def) && attempt < config.Retries && ctx.Err() == nil && config.Budget != nil && config.Budget.Check() == nil
}
var providerID = regexp.MustCompile(`^[A-Za-z0-9_.-]+$`)
var modelID = regexp.MustCompile(`^[A-Za-z0-9_./:-]+$`)
func ValidModel(provider, model string) bool {
return len(provider) <= 200 && len(model) <= 300 && providerID.MatchString(provider) && modelID.MatchString(model)
}
// ModelConfig and ModelSpec group the settings/contracts that belong specifically
// to an LLM stage. Other stage types need not have a prompt, provider, or model.
type ModelConfig struct {
Harness string
Provider string
Model string
Retries int
Timeout time.Duration
BudgetUSD float64
BudgetTokens int64
Budget *Budget
ResultSchema map[string]any
}
type ModelSpec struct {
Output string
Prompt string
Config ModelConfig
}
// ModelGenerator isolates the provider integration from stage instructions/results.
type ModelGenerator interface {
Generate(context.Context, ModelConfig, string) (string, error)
}
type ModelStage struct {
Client ModelGenerator // nil uses the locally authenticated OpenCode service
}
var _ Stage = ModelStage{}
func (m ModelStage) Validate(def Definition) error {
_, err := m.Spec(def)
return err
}
func (ModelStage) Spec(def Definition) (ModelSpec, error) {
if EffectiveType(def.Type) != ModelType {
return ModelSpec{}, fmt.Errorf("expected model stage, got %q", def.Type)
}
for _, block := range []string{"output", "prompt", "configs"} {
if strings.TrimSpace(def.Blocks[block]) == "" {
return ModelSpec{}, fmt.Errorf("required block %s is missing or empty", block)
}
}
for block := range def.Blocks {
switch block {
case "output", "prompt", "edges", "configs", "tools", "permissions":
default:
return ModelSpec{}, fmt.Errorf("unexpected block %q for model stage", block)
}
}
if _, _, err := ToolNames(def); err != nil {
return ModelSpec{}, err
}
if _, err := StagePermissions(def); err != nil {
return ModelSpec{}, err
}
for _, key := range []string{"provider", "model"} {
if def.Configs[key] == "" {
return ModelSpec{}, fmt.Errorf("missing config %s", key)
}
}
for key := range def.Configs {
switch key {
case "provider", "model", "retries", "timeout", "budget_usd", "budget_tokens", "harness":
default:
return ModelSpec{}, fmt.Errorf("unknown config %q", key)
}
}
if !ValidModel(def.Configs["provider"], def.Configs["model"]) {
return ModelSpec{}, fmt.Errorf("invalid provider/model identifier; use the exact IDs validated during setup")
}
backend := def.Configs["harness"]
if backend == "" {
backend = "opencode"
}
if backend != "opencode" && backend != "pi" {
return ModelSpec{}, fmt.Errorf("harness must be opencode or pi")
}
if backend == "pi" {
tools, _, _ := ToolNames(def)
if _, err := piTools(tools); err != nil {
return ModelSpec{}, err
}
}
retries := DefaultRetries
var err error
if value := def.Configs["retries"]; value != "" {
retries, err = strconv.Atoi(value)
}
if err != nil || retries < 0 || retries > 100 {
return ModelSpec{}, fmt.Errorf("retries must be an integer from 0 to 100")
}
timeout, err := StageTimeout(def)
if err != nil {
return ModelSpec{}, err
}
usd, tokens, err := parseBudget(def)
if err != nil {
return ModelSpec{}, err
}
return ModelSpec{
Output: def.Blocks["output"], Prompt: def.Blocks["prompt"],
Config: ModelConfig{Harness: backend, Provider: def.Configs["provider"], Model: def.Configs["model"], Retries: retries, Timeout: timeout, BudgetUSD: usd, BudgetTokens: tokens},
}, nil
}
func (m ModelStage) Run(ctx context.Context, def Definition, input Values) (Result, error) {
if err := ctx.Err(); err != nil {
return Result{}, err
}
spec, err := m.Spec(def)
if err != nil {
return Result{}, err
}
spec.Config.ResultSchema = resultSchema(def.Edges)
prompt, err := modelPrompt(def, spec, input)
if err != nil {
return Result{}, err
}
spec.Config.Budget = budgetFromContext(ctx)
if spec.Config.Budget == nil {
spec.Config.Budget = NewBudget(spec.Config.BudgetUSD, spec.Config.BudgetTokens)
}
client := m.Client
if client == nil {
if spec.Config.Harness == "pi" {
return Result{}, fmt.Errorf("Pi stages require an audited Pi client")
}
tools, _, _ := ToolNames(def)
if len(tools) > 0 {
return Result{}, fmt.Errorf("tool-enabled stages require an audited agent client")
}
client = &OpenCodeClient{}
}
if spec.Config.Budget.Enabled() {
capable, ok := client.(interface{ BudgetCapable() bool })
if !ok || !capable.BudgetCapable() {
return Result{}, fmt.Errorf("%w: generator cannot enforce usage limits", ErrBudgetAccounting)
}
}
// Replaying admitted tool work can repeat mutations. Only a stage whose
// explicit tools cannot mutate the run directory (safeToReplay) may
// automatically retry work that started after admission.
if agent, ok := client.(*AgentClient); ok {
if tools, explicit, _ := ToolNames(def); explicit {
agent.Tools = tools
}
agent.Policy, _ = StagePermissions(def)
}
for attempt := 0; attempt <= spec.Config.Retries; attempt++ {
if pi, ok := client.(*PiClient); ok {
pi.Policy, _ = StagePermissions(def)
}
if err := ctx.Err(); err != nil {
return Result{}, err
}
if err := spec.Config.Budget.Check(); err != nil {
return Result{}, err
}
spec.Config.Budget.Attempt()
attemptCtx, cancel := context.WithTimeout(ctx, spec.Config.Timeout)
text, generateErr := client.Generate(attemptCtx, spec.Config, prompt)
if generateErr == nil && attemptCtx.Err() != nil {
generateErr = attemptCtx.Err()
}
replayPostAdmission := false
if generateErr == nil {
var result Result
var resultErr error
if resultFormat(text) == "line" {
writeRawOutputFile(def.ArtifactsDir, text)
result, _, resultErr = decodeLineResult(text, def.Edges)
} else {
result, resultErr = decodeModelResult(stageResultText(text), def.Edges)
if resultErr != nil {
result, resultErr = repairModelResult(attemptCtx, client, spec.Config, text, def.Edges, resultErr)
}
}
if attemptCtx.Err() != nil {
resultErr = errors.Join(resultErr, attemptCtx.Err())
}
cancel()
if resultErr == nil {
return result, nil
}
// A malformed final result mutates nothing on an explicit read-only
// stage, so it may be re-emitted within the retry budget. A mutating
// or legacy-bundle stage retains its admitted work and returns now.
if toleratedModelResultRetry(ctx, def, spec.Config, attempt, resultErr) {
err = resultErr
} else {
return result, wrapModelStageFailure(def.Key, resultErr)
}
} else {
err = generateErr
if attemptCtx.Err() != nil && !errors.Is(err, ErrBudgetExceeded) && !errors.Is(err, ErrBudgetAccounting) {
err = errors.Join(err, attemptCtx.Err())
}
cancel()
if ctx.Err() != nil {
return Result{}, ctx.Err()
}
if errors.Is(err, ErrBudgetExceeded) || errors.Is(err, ErrBudgetAccounting) || errors.Is(err, ErrStageCleanup) {
break
}
replayPostAdmission = safeToReplay(def) && replayablePostAdmission(err)
// Never blindly replay admitted tool work: only a read-only stage may
// retry a post-admission deadline or transport failure.
if agent, ok := client.(*AgentClient); ok && agent.WorkStarted && !replayPostAdmission {
break
}
if pi, ok := client.(*PiClient); ok && pi.WorkStarted && !replayPostAdmission {
break
}
if attempt == spec.Config.Retries || (!retryableModelError(err) && !errors.Is(err, context.DeadlineExceeded) && !replayPostAdmission) {
break
}
}
delay := time.Duration(1<<min(attempt, 4)) * 100 * time.Millisecond
timer := time.NewTimer(delay)
select {
case <-ctx.Done():
timer.Stop()
return Result{}, ctx.Err()
case <-timer.C:
}
}
return Result{}, wrapModelStageFailure(def.Key, err)
}