aa8eddd686
A stopping server no longer interrupts harness sessions. Runs are canceled with a shutdown cause, so stage clients leave admitted sessions running and owned; start-up recovery re-adopts live OpenCode V2 stage sessions (wait, read back the result, continue the graph), restarts unadmitted visits, continues runs between stages, and falls back to interrupted with a reason when the harness is gone. Chats with live work are recorded on shutdown and reattached on start; turns that ended during the downtime are folded in from history. The server-spawned V2 chat serve is now a detached owned service. PIPELINES_SHUTDOWN_SESSIONS=stop restores the old behaviour.
498 lines
18 KiB
Go
498 lines
18 KiB
Go
package stage
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"git.a-wan.net/alan/pipelines/internal/opencode"
|
|
"io"
|
|
"os"
|
|
"os/exec"
|
|
"regexp"
|
|
"strconv"
|
|
"time"
|
|
)
|
|
|
|
const maxModelRequest = 64 << 10 // CLI request body is passed as one OS argument.
|
|
const maxModelResponse = 1 << 20
|
|
|
|
// OpenCodeClient reuses the selected OpenCode CLI and its authenticated accounts.
|
|
// No API keys are copied into pipeline definitions, subprocess arguments, or logs.
|
|
// A fresh location-scoped session avoids unrelated user conversation history and
|
|
// is retained for debugging. Generation is text-only: no agent tools are run.
|
|
type OpenCodeClient struct {
|
|
Directory string // empty uses the pipeline server's working directory
|
|
Backend string
|
|
Reference string
|
|
FormatAudit string
|
|
// ArtifactsDir is exported to the spawned OpenCode CLI as
|
|
// PIPELINES_ARTIFACTS_DIR where the runtime controls the process. V2 agent
|
|
// tools execute inside the shared `opencode serve` process, so its
|
|
// environment cannot be set per stage; the stage prompt is the guaranteed
|
|
// carrier of the artifacts directory there.
|
|
ArtifactsDir string
|
|
Operations *OperationLog
|
|
OnGenerationSession func(id, backend, reference string, active bool) error
|
|
connection *opencode.Connection
|
|
command func(context.Context, string, ...string) ([]byte, error)
|
|
sessionPollInterval time.Duration // test seam; defaults to defaultSessionPollInterval
|
|
}
|
|
|
|
// ArtifactsEnv names the environment variable carrying a stage's artifacts
|
|
// directory to a locally spawned harness.
|
|
const ArtifactsEnv = "PIPELINES_ARTIFACTS_DIR"
|
|
|
|
// defaultSessionPollInterval bounds the delay between V2 active-session checks.
|
|
// The V2 /api/experimental/session/{id}/wait request blocks server-side until the
|
|
// CLI's own request timeout (observed ~6 minutes, HTTP 499) expires, which a long
|
|
// agent stage exceeds; short active-session polls avoid that timeout.
|
|
const defaultSessionPollInterval = time.Second
|
|
|
|
type openCodeError struct {
|
|
operation string
|
|
status int
|
|
cause error
|
|
diagnostic string
|
|
}
|
|
|
|
func (e *openCodeError) Unwrap() error { return e.cause }
|
|
|
|
func (e *openCodeError) Error() string {
|
|
detail := ""
|
|
if e.diagnostic != "" {
|
|
detail = ": " + e.diagnostic
|
|
}
|
|
if e.status > 0 {
|
|
return fmt.Sprintf("OpenCode %s failed (HTTP %d)%s", e.operation, e.status, detail)
|
|
}
|
|
return fmt.Sprintf("OpenCode %s command failed%s", e.operation, detail)
|
|
}
|
|
func (e *openCodeError) Retryable() bool {
|
|
return e.status == 429 || e.status == 502 || e.status == 503 || e.status == 504
|
|
}
|
|
|
|
var sessionIDPattern = regexp.MustCompile(`^ses[A-Za-z0-9_-]+$`)
|
|
var httpStatusPattern = regexp.MustCompile(`HTTP (\d{3})`)
|
|
|
|
func (c OpenCodeClient) call(ctx context.Context, method, path string, body any) ([]byte, error) {
|
|
if c.connection != nil && c.connection.Backend == "v1" {
|
|
started := time.Now()
|
|
data, err := c.connection.Call(ctx, method, path, body)
|
|
if probe := c.connection.FormatProbe; probe != nil {
|
|
c.Operations.recordFormatProbe(*probe)
|
|
if !probe.Supported && probe.Diagnostic != "" {
|
|
c.Operations.record(OperationEvidence{Operation: "get /session/" + probe.SessionID + "/message (format compatibility probe)", Backend: c.Backend, Executable: opencode.Executable(), Directory: c.Directory, ExitCode: -1, HTTPStatus: 400, Diagnostic: probe.Diagnostic, Stdout: probe.Response}, started)
|
|
}
|
|
c.connection.FormatProbe = nil
|
|
}
|
|
if err != nil {
|
|
status := 0
|
|
if match := httpStatusPattern.FindStringSubmatch(err.Error()); len(match) > 0 {
|
|
status, _ = strconv.Atoi(match[1])
|
|
}
|
|
evidenceStatus := status
|
|
if evidenceStatus == 0 {
|
|
evidenceStatus = c.connection.LastHTTPStatus
|
|
}
|
|
c.Operations.record(OperationEvidence{Operation: method + " " + path, Backend: c.Backend, Executable: opencode.Executable(), Directory: c.Directory, ExitCode: -1, HTTPStatus: evidenceStatus, NativeOperation: c.connection.LastHTTPOperation, Diagnostic: err.Error(), Stdout: string(data)}, started)
|
|
transport := &openCodeError{operation: method + " " + path, status: status, cause: err, diagnostic: err.Error()}
|
|
if ctxErr := ctx.Err(); ctxErr != nil {
|
|
// Keep the transport shape (operation/status) with the context
|
|
// cause: a deadline that fired while the request was in flight is
|
|
// both, and neither may be discarded.
|
|
return nil, errors.Join(ctxErr, transport)
|
|
}
|
|
return nil, transport
|
|
}
|
|
return data, nil
|
|
}
|
|
args := []string{"api", method, path}
|
|
if body != nil {
|
|
data, err := json.Marshal(body)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if len(data) > maxModelRequest {
|
|
return nil, fmt.Errorf("OpenCode request exceeds 64 KiB limit")
|
|
}
|
|
args = append(args, "--data", string(data))
|
|
}
|
|
directory := c.Directory
|
|
if directory == "" {
|
|
var err error
|
|
directory, err = os.Getwd()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
}
|
|
if c.command != nil {
|
|
return c.command(ctx, directory, args...)
|
|
}
|
|
cmd := opencode.CommandContext(ctx, opencode.Executable(), args...)
|
|
cmd.Dir = directory
|
|
if c.ArtifactsDir != "" {
|
|
cmd.Env = append(os.Environ(), ArtifactsEnv+"="+c.ArtifactsDir)
|
|
}
|
|
if c.connection != nil {
|
|
c.connection.PinCLI(cmd)
|
|
}
|
|
stdout := &boundedBuffer{limit: maxModelResponse}
|
|
stderr := &boundedBuffer{limit: 8 << 10}
|
|
cmd.Stdout = stdout
|
|
cmd.Stderr = stderr
|
|
started := time.Now()
|
|
if err := cmd.Run(); err != nil {
|
|
status, code := 0, -1
|
|
if match := httpStatusPattern.FindStringSubmatch(stderr.String()); len(match) > 0 {
|
|
status, _ = strconv.Atoi(match[1])
|
|
}
|
|
var exit *exec.ExitError
|
|
if errors.As(err, &exit) {
|
|
code = exit.ExitCode()
|
|
}
|
|
c.Operations.record(OperationEvidence{Operation: method + " " + path, Backend: c.Backend, Executable: cmd.Path, Directory: directory, ExitCode: code, HTTPStatus: status, Diagnostic: safeCommandDiagnostic(err, stdout.Bytes()), Stdout: stdout.String(), Stderr: stderr.String()}, started)
|
|
// Do not embed provider responses, prompts, credentials, or CLI logs in errors.
|
|
// Keep the raw cause so errors.Is(err, exec.ErrWaitDelay) still holds while
|
|
// Error() stays sanitized. This branch is only reached with a non-nil Run
|
|
// error, so valid-looking partial stdout is never returned as success.
|
|
transport := &openCodeError{operation: method + " " + path, status: status, cause: err, diagnostic: safeCommandDiagnostic(err, stdout.Bytes())}
|
|
if ctxErr := ctx.Err(); ctxErr != nil {
|
|
// Keep the sanitized transport detail (operation/status/exit) together
|
|
// with errors.Is(err, context.DeadlineExceeded) so the true cause is
|
|
// not replaced by a bare context error.
|
|
return nil, errors.Join(ctxErr, transport)
|
|
}
|
|
return nil, transport
|
|
}
|
|
if stdout.overflow {
|
|
return nil, fmt.Errorf("OpenCode response exceeds 1 MiB limit")
|
|
}
|
|
return stdout.Bytes(), nil
|
|
}
|
|
|
|
func safeCommandDiagnostic(err error, data []byte) string {
|
|
var exit *exec.ExitError
|
|
detail := "could not execute CLI"
|
|
if errors.As(err, &exit) {
|
|
detail = fmt.Sprintf("CLI exit code %d", exit.ExitCode())
|
|
}
|
|
var response struct {
|
|
Tag string `json:"_tag"`
|
|
}
|
|
if json.Unmarshal(data, &response) == nil {
|
|
// Never copy provider messages, arbitrary stderr or response bodies.
|
|
reasons := map[string]string{"SessionNotFoundError": "session not found on the selected OpenCode service", "LocationNotFoundError": "session location not found", "ServiceUnavailableError": "OpenCode service unavailable", "UnauthorizedError": "OpenCode service authentication failed", "InvalidRequestError": "OpenCode rejected the request", "SessionBusyError": "OpenCode session is busy"}
|
|
if reason := reasons[response.Tag]; reason != "" {
|
|
detail += "; " + reason
|
|
}
|
|
}
|
|
return detail
|
|
}
|
|
|
|
// activeSessionState is the documented GET /api/session/active entry shape:
|
|
// {"data":{"ses_ID":{"type":"running"}}}. An absent session is inactive.
|
|
type activeSessionState struct {
|
|
Type string `json:"type"`
|
|
}
|
|
|
|
// waitForSessionIdle polls GET /api/session/active through the managed CLI until
|
|
// the admitted session is absent (inactive). It is the single V2 wait helper used
|
|
// by every V2 generation path; it never issues the long blocking .../wait request,
|
|
// uses a context-aware delay with no busy loop, and fails closed on malformed,
|
|
// non-object or unsupported responses instead of treating bad JSON as inactive.
|
|
func (c OpenCodeClient) waitForSessionIdle(ctx context.Context, sessionID string) error {
|
|
interval := c.sessionPollInterval
|
|
if interval <= 0 {
|
|
interval = defaultSessionPollInterval
|
|
}
|
|
started := time.Now()
|
|
for {
|
|
active, err := c.sessionActive(ctx, sessionID)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if !active {
|
|
return nil
|
|
}
|
|
timer := time.NewTimer(interval)
|
|
select {
|
|
case <-ctx.Done():
|
|
timer.Stop()
|
|
waitErr := fmt.Errorf("waiting for session idle; retained OpenCode session %.128s; stage had not finished: %w", RedactEvidence(sessionID), ctx.Err())
|
|
var timeout *stageTimeoutCause
|
|
if errors.Is(ctx.Err(), context.DeadlineExceeded) && errors.As(context.Cause(ctx), &timeout) {
|
|
waitErr = &stageDeadlineError{timeout: timeout.timeout, session: sessionID}
|
|
}
|
|
// A settled session is the only success signal; retain one bounded
|
|
// sanitized record so a timed-out wait is not an empty transcript.
|
|
c.Operations.record(OperationEvidence{
|
|
Operation: "wait for session idle",
|
|
Backend: c.Backend,
|
|
Executable: opencode.Executable(),
|
|
Directory: c.Directory,
|
|
ExitCode: -1,
|
|
Diagnostic: waitErr.Error(),
|
|
}, started)
|
|
return waitErr
|
|
case <-timer.C:
|
|
}
|
|
}
|
|
}
|
|
|
|
// sessionActive reports whether sessionID is currently active. Only the target
|
|
// session is examined; entries for other sessions are ignored entirely, so a
|
|
// foreign session's state never affects this wait. The target's absence is
|
|
// inactive. Missing, null or non-object data, and the target present with any
|
|
// state other than the documented "running" type, are errors so a bad response
|
|
// can never be mistaken for a settled session.
|
|
func (c OpenCodeClient) sessionActive(ctx context.Context, sessionID string) (bool, error) {
|
|
data, err := c.call(ctx, "get", "/api/session/active", nil)
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
var response struct {
|
|
Data map[string]activeSessionState `json:"data"`
|
|
}
|
|
if err := json.Unmarshal(data, &response); err != nil || response.Data == nil {
|
|
return false, fmt.Errorf("%w: OpenCode returned an invalid active-session response", ErrHarnessTransport)
|
|
}
|
|
state, ok := response.Data[sessionID]
|
|
if !ok {
|
|
return false, nil
|
|
}
|
|
if state.Type != "running" {
|
|
return false, fmt.Errorf("%w: OpenCode reported an unsupported active-session state", ErrHarnessTransport)
|
|
}
|
|
return true, nil
|
|
}
|
|
|
|
func (c *OpenCodeClient) Generate(ctx context.Context, config ModelConfig, prompt string) (text string, err error) {
|
|
c.FormatAudit = ""
|
|
if c.Operations != nil {
|
|
c.Operations.Provider, c.Operations.Model = config.Provider, config.Model
|
|
}
|
|
// A re-adopted session was admitted by a previous server process (see
|
|
// WithResumeSession): wait for it and read its result, never prompt again.
|
|
resume := TakeResumeSession(ctx)
|
|
if resume != "" && c.command == nil {
|
|
connection, err := resumeConnection(c.Reference, c.Backend, c.Directory)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
defer connection.Close()
|
|
c.connection = connection
|
|
c.Backend = connection.Backend
|
|
} else if c.command == nil {
|
|
connection, err := opencode.OpenStage(ctx, opencode.Executable(), c.Directory, c.Backend)
|
|
if err != nil {
|
|
// Connecting to the service failed before any session existed.
|
|
return "", harnessUnreachable(err)
|
|
}
|
|
defer connection.Close()
|
|
c.connection = connection
|
|
c.Backend = connection.Backend
|
|
if c.Backend == "v2" {
|
|
c.Reference, err = connection.Reference()
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
}
|
|
}
|
|
directory := c.Directory
|
|
if directory == "" {
|
|
directory, err = os.Getwd()
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
}
|
|
// Fail before creating a session if the generation payload is too large.
|
|
promptKey := "prompt"
|
|
if c.Backend == "v2" {
|
|
promptKey = "text"
|
|
}
|
|
body := generationBody(promptKey, prompt, config, c.Backend)
|
|
encoded, err := json.Marshal(body)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
if len(encoded) > maxModelRequest {
|
|
return "", fmt.Errorf("OpenCode request exceeds 64 KiB limit")
|
|
}
|
|
model, err := openCodeModelSelection(config, c.Backend)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
var session struct {
|
|
Data struct {
|
|
ID string `json:"id"`
|
|
} `json:"data"`
|
|
}
|
|
if resume != "" {
|
|
if c.Backend != "v2" || !sessionIDPattern.MatchString(resume) {
|
|
return "", fmt.Errorf("retained OpenCode session %.128s cannot be re-adopted on backend %q", RedactEvidence(resume), c.Backend)
|
|
}
|
|
session.Data.ID = resume
|
|
} else {
|
|
created, err := c.call(ctx, "post", "/api/session", map[string]any{
|
|
"title": "Pipeline stage generation",
|
|
"location": map[string]string{"directory": directory},
|
|
"model": model,
|
|
"permissions": []map[string]string{{"action": "*", "resource": "*", "effect": "deny"}},
|
|
})
|
|
if err != nil {
|
|
if variant := model["variant"]; variant != "" {
|
|
err = fmt.Errorf("OpenCode model selection for reasoning_effort=%s (variant %q) failed: %w", variant, variant, err)
|
|
}
|
|
// No prompt or tool call was admitted: a request failure here is a
|
|
// reachable retry rather than a reason to end the run.
|
|
return "", harnessUnreachable(err)
|
|
}
|
|
if err := json.Unmarshal(created, &session); err != nil || !sessionIDPattern.MatchString(session.Data.ID) {
|
|
return "", fmt.Errorf("OpenCode returned an invalid session ID")
|
|
}
|
|
}
|
|
var watch *budgetWatch
|
|
promptStarted := false
|
|
defer func() {
|
|
watch.stopWatch(ctx, &err)
|
|
defer watch.finish(ctx, &err)
|
|
if err != nil && Detached(ctx) {
|
|
// The server is restarting: leave the session running and owned.
|
|
err = fmt.Errorf("%w: retained OpenCode session %s left running", ErrDetached, session.Data.ID)
|
|
return
|
|
}
|
|
cleanup, cancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second)
|
|
defer cancel()
|
|
if err != nil {
|
|
stopErr := c.InterruptAndConfirm(cleanup, session.Data.ID)
|
|
// Read after settlement so a late assistant turn remains available as
|
|
// failure evidence. Never replay a prompt to reconstruct missing turns.
|
|
if promptStarted && c.Backend == "v2" {
|
|
readCtx, readCancel := context.WithTimeout(context.WithoutCancel(ctx), 3*time.Second)
|
|
data, readErr := c.call(readCtx, "get", "/api/session/"+session.Data.ID+finalAssistantReadback, nil)
|
|
readCancel()
|
|
if readErr == nil && len(data) > 0 {
|
|
c.FormatAudit = RedactEvidence(string(data))
|
|
}
|
|
if watch == nil {
|
|
c.settleV2Usage(ctx, config.Budget, session.Data.ID)
|
|
}
|
|
}
|
|
if stopErr != nil {
|
|
err = fmt.Errorf("retained OpenCode session %s; interrupt failed: %w", session.Data.ID, errors.Join(err, stopErr, ErrStageCleanup))
|
|
return
|
|
}
|
|
}
|
|
if c.OnGenerationSession != nil {
|
|
if persistErr := c.OnGenerationSession(session.Data.ID, c.Backend, c.Reference, false); persistErr != nil {
|
|
text = ""
|
|
err = errors.Join(err, persistErr, ErrStageCleanup)
|
|
}
|
|
}
|
|
if err != nil {
|
|
err = fmt.Errorf("retained OpenCode session %s: %w", session.Data.ID, err)
|
|
}
|
|
}()
|
|
if c.connection != nil && c.Backend == "v1" {
|
|
c.Reference, err = c.connection.Reference()
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
}
|
|
if c.OnGenerationSession != nil {
|
|
if err = c.OnGenerationSession(session.Data.ID, c.Backend, c.Reference, true); err != nil {
|
|
return "", err
|
|
}
|
|
}
|
|
ctx, watch, err = c.watchBudget(ctx, config.Budget, session.Data.ID)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
// V2 /generate does not persist messages or update session usage. Every V2
|
|
// model call uses tracked prompting under the session's deny-all tool policy,
|
|
// including unbudgeted text-only stages, so its exact native turns survive.
|
|
if c.Backend == "v2" {
|
|
promptStarted = true
|
|
if resume == "" {
|
|
if _, err = c.call(ctx, "post", "/api/session/"+session.Data.ID+"/prompt", body); err != nil {
|
|
return "", err
|
|
}
|
|
}
|
|
if err = c.waitForSessionIdle(ctx, session.Data.ID); err != nil {
|
|
return "", err
|
|
}
|
|
var data []byte
|
|
data, text, err = c.readFinalAgentResult(ctx, session.Data.ID)
|
|
if len(data) > 0 {
|
|
c.FormatAudit = string(data)
|
|
}
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
if !config.Budget.Enabled() {
|
|
// As with AgentClient, missing usage is not a generation failure when
|
|
// no limit is enabled (and a nil ledger needs no aggregate read).
|
|
c.settleV2Usage(ctx, config.Budget, session.Data.ID)
|
|
return text, nil
|
|
}
|
|
// A completed tracked response must have accounted tokens; session-zero
|
|
// placeholders are not proof of a free, zero-token provider request.
|
|
usageCtx, usageCancel := context.WithTimeout(context.WithoutCancel(ctx), 2*time.Second)
|
|
usage, usageErr := c.call(usageCtx, "get", "/api/session/"+session.Data.ID, nil)
|
|
usageCancel()
|
|
if usageErr != nil {
|
|
return "", fmt.Errorf("%w: %v", ErrBudgetAccounting, usageErr)
|
|
}
|
|
if usageErr = config.Budget.readUsage(session.Data.ID, usage, false, true); usageErr != nil {
|
|
return "", usageErr
|
|
}
|
|
if config.Budget.Snapshot().LimitTokens > 0 {
|
|
var response struct {
|
|
Data struct{ reportedUsage } `json:"data"`
|
|
}
|
|
if json.Unmarshal(usage, &response) != nil {
|
|
return "", ErrBudgetAccounting
|
|
}
|
|
u := response.Data.Tokens
|
|
if u == nil || u.Input == nil || u.Output == nil || u.Reasoning == nil || u.Cache == nil || u.Cache.Read == nil || u.Cache.Write == nil || *u.Input+*u.Output+*u.Reasoning+*u.Cache.Read+*u.Cache.Write == 0 {
|
|
return "", fmt.Errorf("%w: completed V2 response reported zero tokens", ErrBudgetAccounting)
|
|
}
|
|
}
|
|
return text, nil
|
|
}
|
|
response, err := c.call(ctx, "post", "/api/session/"+session.Data.ID+"/generate", body)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
var generated struct {
|
|
Data struct {
|
|
Text *string `json:"text"`
|
|
} `json:"data"`
|
|
}
|
|
if err := json.Unmarshal(response, &generated); err != nil || generated.Data.Text == nil {
|
|
return "", fmt.Errorf("OpenCode returned an invalid generation response")
|
|
}
|
|
return *generated.Data.Text, nil
|
|
}
|
|
|
|
type boundedBuffer struct {
|
|
bytes.Buffer
|
|
limit int
|
|
overflow bool
|
|
}
|
|
|
|
func (b *boundedBuffer) Write(p []byte) (int, error) {
|
|
n := len(p)
|
|
remaining := b.limit - b.Len()
|
|
if n > remaining {
|
|
b.overflow = true
|
|
p = p[:remaining]
|
|
}
|
|
_, _ = b.Buffer.Write(p)
|
|
return n, nil // Drain subprocess output without retaining unbounded data.
|
|
}
|
|
|
|
var _ io.Writer = (*boundedBuffer)(nil)
|