Files
pipelines/internal/stage/opencode.go
T
alan f57457aec9 Make agent-stage retries safe and retain timeout failure evidence
Fix four stage failure-handling and observability defects (A2, A3, A6, A9).

- A2: add safeToReplay(def) as the single definition of retryable post-admission
  work - only an explicit tools block that selects neither edit nor shell. The
  agent/pi WorkStarted guards now break only when the stage is not safe to
  replay, so read-only stages genuinely retry a post-admission timeout while
  mutating and legacy-bundle stages still never replay admitted work.
- A6: re-emit a malformed final result on an explicit read-only stage up to the
  retry budget, preserving format_repairs evidence; mutating stages keep the
  immediate return and their session.
- A3: timed-out stages retain usable evidence. waitForSessionIdle records one
  bounded sanitized OperationEvidence on deadline; the agent deferred cleanup
  does a bounded best-effort latest-assistant readback and settles knowable V2
  usage; RetainSession adds a retained_session pointer to failed step
  transcripts so the session is visible beneath the BoundTranscript threshold.
- A9: a stage outcome routed to #failure now yields a non-empty run diagnostic
  ("stage <name> returned outcome <outcome>") while preferring an existing step
  diagnostic.

Adds focused stage and server tests for retry safety, malformed-result
re-emission, timeout evidence retention and run-level diagnostics; updates the
packaged skill references and docs/cli.md, docs/model-results.md.
2026-10-15 18:27:22 +01:00

413 lines
15 KiB
Go

package stage
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"git.a-wan.net/alan/pipelines/internal/opencode"
"io"
"os"
"os/exec"
"regexp"
"strconv"
"time"
)
const maxModelRequest = 64 << 10 // CLI request body is passed as one OS argument.
const maxModelResponse = 1 << 20
// OpenCodeClient reuses the selected OpenCode CLI and its authenticated accounts.
// No API keys are copied into pipeline definitions, subprocess arguments, or logs.
// A fresh location-scoped session avoids unrelated user conversation history and
// is retained for debugging. Generation is text-only: no agent tools are run.
type OpenCodeClient struct {
Directory string // empty uses the pipeline server's working directory
Backend string
Reference string
FormatAudit string
// ArtifactsDir is exported to the spawned OpenCode CLI as
// PIPELINES_ARTIFACTS_DIR where the runtime controls the process. V2 agent
// tools execute inside the shared `opencode serve` process, so its
// environment cannot be set per stage; the stage prompt is the guaranteed
// carrier of the artifacts directory there.
ArtifactsDir string
Operations *OperationLog
OnGenerationSession func(id, backend, reference string, active bool) error
connection *opencode.Connection
command func(context.Context, string, ...string) ([]byte, error)
sessionPollInterval time.Duration // test seam; defaults to defaultSessionPollInterval
}
// ArtifactsEnv names the environment variable carrying a stage's artifacts
// directory to a locally spawned harness.
const ArtifactsEnv = "PIPELINES_ARTIFACTS_DIR"
// defaultSessionPollInterval bounds the delay between V2 active-session checks.
// The V2 /api/experimental/session/{id}/wait request blocks server-side until the
// CLI's own request timeout (observed ~6 minutes, HTTP 499) expires, which a long
// agent stage exceeds; short active-session polls avoid that timeout.
const defaultSessionPollInterval = time.Second
type openCodeError struct {
operation string
status int
cause error
diagnostic string
}
func (e *openCodeError) Unwrap() error { return e.cause }
func (e *openCodeError) Error() string {
detail := ""
if e.diagnostic != "" {
detail = ": " + e.diagnostic
}
if e.status > 0 {
return fmt.Sprintf("OpenCode %s failed (HTTP %d)%s", e.operation, e.status, detail)
}
return fmt.Sprintf("OpenCode %s command failed%s", e.operation, detail)
}
func (e *openCodeError) Retryable() bool {
return e.status == 429 || e.status == 502 || e.status == 503 || e.status == 504
}
var sessionIDPattern = regexp.MustCompile(`^ses[A-Za-z0-9_-]+$`)
var httpStatusPattern = regexp.MustCompile(`HTTP (\d{3})`)
func (c OpenCodeClient) call(ctx context.Context, method, path string, body any) ([]byte, error) {
if c.connection != nil && c.connection.Backend == "v1" {
started := time.Now()
data, err := c.connection.Call(ctx, method, path, body)
if probe := c.connection.FormatProbe; probe != nil {
c.Operations.recordFormatProbe(*probe)
if !probe.Supported && probe.Diagnostic != "" {
c.Operations.record(OperationEvidence{Operation: "get /session/" + probe.SessionID + "/message (format compatibility probe)", Backend: c.Backend, Executable: opencode.Executable(), Directory: c.Directory, ExitCode: -1, HTTPStatus: 400, Diagnostic: probe.Diagnostic, Stdout: probe.Response}, started)
}
c.connection.FormatProbe = nil
}
if err != nil {
status := 0
if match := httpStatusPattern.FindStringSubmatch(err.Error()); len(match) > 0 {
status, _ = strconv.Atoi(match[1])
}
evidenceStatus := status
if evidenceStatus == 0 {
evidenceStatus = c.connection.LastHTTPStatus
}
c.Operations.record(OperationEvidence{Operation: method + " " + path, Backend: c.Backend, Executable: opencode.Executable(), Directory: c.Directory, ExitCode: -1, HTTPStatus: evidenceStatus, NativeOperation: c.connection.LastHTTPOperation, Diagnostic: err.Error(), Stdout: string(data)}, started)
if ctx.Err() != nil {
return nil, ctx.Err()
}
return nil, &openCodeError{operation: method + " " + path, status: status, cause: err, diagnostic: err.Error()}
}
return data, nil
}
args := []string{"api", method, path}
if body != nil {
data, err := json.Marshal(body)
if err != nil {
return nil, err
}
if len(data) > maxModelRequest {
return nil, fmt.Errorf("OpenCode request exceeds 64 KiB limit")
}
args = append(args, "--data", string(data))
}
directory := c.Directory
if directory == "" {
var err error
directory, err = os.Getwd()
if err != nil {
return nil, err
}
}
if c.command != nil {
return c.command(ctx, directory, args...)
}
cmd := opencode.CommandContext(ctx, opencode.Executable(), args...)
cmd.Dir = directory
if c.ArtifactsDir != "" {
cmd.Env = append(os.Environ(), ArtifactsEnv+"="+c.ArtifactsDir)
}
stdout := &boundedBuffer{limit: maxModelResponse}
stderr := &boundedBuffer{limit: 8 << 10}
cmd.Stdout = stdout
cmd.Stderr = stderr
started := time.Now()
if err := cmd.Run(); err != nil {
status, code := 0, -1
if match := httpStatusPattern.FindStringSubmatch(stderr.String()); len(match) > 0 {
status, _ = strconv.Atoi(match[1])
}
var exit *exec.ExitError
if errors.As(err, &exit) {
code = exit.ExitCode()
}
c.Operations.record(OperationEvidence{Operation: method + " " + path, Backend: c.Backend, Executable: cmd.Path, Directory: directory, ExitCode: code, HTTPStatus: status, Diagnostic: safeCommandDiagnostic(err, stdout.Bytes()), Stdout: stdout.String(), Stderr: stderr.String()}, started)
if ctx.Err() != nil {
return nil, ctx.Err()
}
// Do not embed provider responses, prompts, credentials, or CLI logs in errors.
// Keep the raw cause so errors.Is(err, exec.ErrWaitDelay) still holds while
// Error() stays sanitized. This branch is only reached with a non-nil Run
// error, so valid-looking partial stdout is never returned as success.
return nil, &openCodeError{operation: method + " " + path, status: status, cause: err, diagnostic: safeCommandDiagnostic(err, stdout.Bytes())}
}
if stdout.overflow {
return nil, fmt.Errorf("OpenCode response exceeds 1 MiB limit")
}
return stdout.Bytes(), nil
}
func safeCommandDiagnostic(err error, data []byte) string {
var exit *exec.ExitError
detail := "could not execute CLI"
if errors.As(err, &exit) {
detail = fmt.Sprintf("CLI exit code %d", exit.ExitCode())
}
var response struct {
Tag string `json:"_tag"`
}
if json.Unmarshal(data, &response) == nil {
// Never copy provider messages, arbitrary stderr or response bodies.
reasons := map[string]string{"SessionNotFoundError": "session not found on the selected OpenCode service", "LocationNotFoundError": "session location not found", "ServiceUnavailableError": "OpenCode service unavailable", "UnauthorizedError": "OpenCode service authentication failed", "InvalidRequestError": "OpenCode rejected the request", "SessionBusyError": "OpenCode session is busy"}
if reason := reasons[response.Tag]; reason != "" {
detail += "; " + reason
}
}
return detail
}
// activeSessionState is the documented GET /api/session/active entry shape:
// {"data":{"ses_ID":{"type":"running"}}}. An absent session is inactive.
type activeSessionState struct {
Type string `json:"type"`
}
// waitForSessionIdle polls GET /api/session/active through the managed CLI until
// the admitted session is absent (inactive). It is the single V2 wait helper used
// by every V2 generation path; it never issues the long blocking .../wait request,
// uses a context-aware delay with no busy loop, and fails closed on malformed,
// non-object or unsupported responses instead of treating bad JSON as inactive.
func (c OpenCodeClient) waitForSessionIdle(ctx context.Context, sessionID string) error {
interval := c.sessionPollInterval
if interval <= 0 {
interval = defaultSessionPollInterval
}
started := time.Now()
for {
active, err := c.sessionActive(ctx, sessionID)
if err != nil {
return err
}
if !active {
return nil
}
timer := time.NewTimer(interval)
select {
case <-ctx.Done():
timer.Stop()
// A settled session is the only success signal; retain one bounded
// sanitized record so a timed-out wait is not an empty transcript.
c.Operations.record(OperationEvidence{
Operation: "wait for session idle",
Backend: c.Backend,
Executable: opencode.Executable(),
Directory: c.Directory,
ExitCode: -1,
Diagnostic: "stage deadline reached while waiting for the agent session to become idle",
}, started)
return ctx.Err()
case <-timer.C:
}
}
}
// sessionActive reports whether sessionID is currently active. Only the target
// session is examined; entries for other sessions are ignored entirely, so a
// foreign session's state never affects this wait. The target's absence is
// inactive. Missing, null or non-object data, and the target present with any
// state other than the documented "running" type, are errors so a bad response
// can never be mistaken for a settled session.
func (c OpenCodeClient) sessionActive(ctx context.Context, sessionID string) (bool, error) {
data, err := c.call(ctx, "get", "/api/session/active", nil)
if err != nil {
return false, err
}
var response struct {
Data map[string]activeSessionState `json:"data"`
}
if err := json.Unmarshal(data, &response); err != nil || response.Data == nil {
return false, fmt.Errorf("%w: OpenCode returned an invalid active-session response", ErrHarnessTransport)
}
state, ok := response.Data[sessionID]
if !ok {
return false, nil
}
if state.Type != "running" {
return false, fmt.Errorf("%w: OpenCode reported an unsupported active-session state", ErrHarnessTransport)
}
return true, nil
}
func (c *OpenCodeClient) Generate(ctx context.Context, config ModelConfig, prompt string) (text string, err error) {
if c.Operations != nil {
c.Operations.Provider, c.Operations.Model = config.Provider, config.Model
}
if c.command == nil {
connection, err := opencode.Open(ctx, opencode.Executable(), c.Directory, c.Backend)
if err != nil {
return "", err
}
defer connection.Close()
c.connection = connection
c.Backend = connection.Backend
}
directory := c.Directory
if directory == "" {
directory, err = os.Getwd()
if err != nil {
return "", err
}
}
// Fail before creating a session if the generation payload is too large.
body := generationBody("prompt", prompt, config, c.Backend)
encoded, err := json.Marshal(body)
if err != nil {
return "", err
}
if len(encoded) > maxModelRequest {
return "", fmt.Errorf("OpenCode request exceeds 64 KiB limit")
}
created, err := c.call(ctx, "post", "/api/session", map[string]any{
"title": "Pipeline stage generation",
"location": map[string]string{"directory": directory},
"model": map[string]string{"providerID": config.Provider, "id": config.Model},
"permissions": []map[string]string{{"action": "*", "resource": "*", "effect": "deny"}},
})
if err != nil {
return "", err
}
var session struct {
Data struct {
ID string `json:"id"`
} `json:"data"`
}
if err := json.Unmarshal(created, &session); err != nil || !sessionIDPattern.MatchString(session.Data.ID) {
return "", fmt.Errorf("OpenCode returned an invalid session ID")
}
var watch *budgetWatch
defer func() {
watch.stopWatch(ctx, &err)
defer watch.finish(ctx, &err)
cleanup, cancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second)
defer cancel()
if err != nil {
if stopErr := c.Interrupt(cleanup, session.Data.ID); stopErr != nil {
err = fmt.Errorf("retained OpenCode session %s; interrupt failed: %w", session.Data.ID, errors.Join(err, stopErr, ErrStageCleanup))
return
}
}
if c.OnGenerationSession != nil {
if persistErr := c.OnGenerationSession(session.Data.ID, c.Backend, c.Reference, false); persistErr != nil {
text = ""
err = errors.Join(err, persistErr, ErrStageCleanup)
}
}
if err != nil {
err = fmt.Errorf("retained OpenCode session %s: %w", session.Data.ID, err)
}
}()
if c.connection != nil && c.Backend == "v1" {
c.Reference, err = c.connection.Reference()
if err != nil {
return "", err
}
}
if c.OnGenerationSession != nil {
if err = c.OnGenerationSession(session.Data.ID, c.Backend, c.Reference, true); err != nil {
return "", err
}
}
ctx, watch, err = c.watchBudget(ctx, config.Budget, session.Data.ID)
if err != nil {
return "", err
}
// V2 /generate returns text only and does not persist messages or update
// session usage. Budgeted calls must use tracked, deny-all prompting instead.
if c.Backend == "v2" && config.Budget.Enabled() {
if _, err = c.call(ctx, "post", "/api/session/"+session.Data.ID+"/prompt", generationBody("text", prompt, config, c.Backend)); err != nil {
return "", err
}
if err = c.waitForSessionIdle(ctx, session.Data.ID); err != nil {
return "", err
}
var data []byte
data, text, err = c.readFinalAgentResult(ctx, session.Data.ID)
if len(data) > 0 {
c.FormatAudit = string(data)
}
if err != nil {
return "", err
}
// A completed tracked response must have accounted tokens; session-zero
// placeholders are not proof of a free, zero-token provider request.
usageCtx, usageCancel := context.WithTimeout(context.WithoutCancel(ctx), 2*time.Second)
usage, usageErr := c.call(usageCtx, "get", "/api/session/"+session.Data.ID, nil)
usageCancel()
if usageErr != nil {
return "", fmt.Errorf("%w: %v", ErrBudgetAccounting, usageErr)
}
if usageErr = config.Budget.readUsage(session.Data.ID, usage, false, true); usageErr != nil {
return "", usageErr
}
if config.Budget.Snapshot().LimitTokens > 0 {
var response struct {
Data struct{ reportedUsage } `json:"data"`
}
if json.Unmarshal(usage, &response) != nil {
return "", ErrBudgetAccounting
}
u := response.Data.Tokens
if u == nil || u.Input == nil || u.Output == nil || u.Reasoning == nil || u.Cache == nil || u.Cache.Read == nil || u.Cache.Write == nil || *u.Input+*u.Output+*u.Reasoning+*u.Cache.Read+*u.Cache.Write == 0 {
return "", fmt.Errorf("%w: completed V2 response reported zero tokens", ErrBudgetAccounting)
}
}
return text, nil
}
response, err := c.call(ctx, "post", "/api/session/"+session.Data.ID+"/generate", body)
if err != nil {
return "", err
}
var generated struct {
Data struct {
Text *string `json:"text"`
} `json:"data"`
}
if err := json.Unmarshal(response, &generated); err != nil || generated.Data.Text == nil {
return "", fmt.Errorf("OpenCode returned an invalid generation response")
}
return *generated.Data.Text, nil
}
type boundedBuffer struct {
bytes.Buffer
limit int
overflow bool
}
func (b *boundedBuffer) Write(p []byte) (int, error) {
n := len(p)
remaining := b.limit - b.Len()
if n > remaining {
b.overflow = true
p = p[:remaining]
}
_, _ = b.Buffer.Write(p)
return n, nil // Drain subprocess output without retaining unbounded data.
}
var _ io.Writer = (*boundedBuffer)(nil)