Keep OpenCode/Codex chat and stage sessions running across server restarts

A stopping server no longer interrupts harness sessions. Runs are canceled
with a shutdown cause, so stage clients leave admitted sessions running and
owned; start-up recovery re-adopts live OpenCode V2 stage sessions (wait,
read back the result, continue the graph), restarts unadmitted visits,
continues runs between stages, and falls back to interrupted with a reason
when the harness is gone. Chats with live work are recorded on shutdown and
reattached on start; turns that ended during the downtime are folded in
from history. The server-spawned V2 chat serve is now a detached owned
service. PIPELINES_SHUTDOWN_SESSIONS=stop restores the old behaviour.
This commit is contained in:
2026-11-04 16:17:08 +00:00
parent 9351cd7d74
commit aa8eddd686
26 changed files with 2318 additions and 99 deletions
+44 -19
View File
@@ -273,7 +273,18 @@ func (c *OpenCodeClient) Generate(ctx context.Context, config ModelConfig, promp
if c.Operations != nil {
c.Operations.Provider, c.Operations.Model = config.Provider, config.Model
}
if c.command == nil {
// A re-adopted session was admitted by a previous server process (see
// WithResumeSession): wait for it and read its result, never prompt again.
resume := TakeResumeSession(ctx)
if resume != "" && c.command == nil {
connection, err := resumeConnection(c.Reference, c.Backend, c.Directory)
if err != nil {
return "", err
}
defer connection.Close()
c.connection = connection
c.Backend = connection.Backend
} else if c.command == nil {
connection, err := opencode.OpenStage(ctx, opencode.Executable(), c.Directory, c.Backend)
if err != nil {
// Connecting to the service failed before any session existed.
@@ -313,33 +324,45 @@ func (c *OpenCodeClient) Generate(ctx context.Context, config ModelConfig, promp
if err != nil {
return "", err
}
created, err := c.call(ctx, "post", "/api/session", map[string]any{
"title": "Pipeline stage generation",
"location": map[string]string{"directory": directory},
"model": model,
"permissions": []map[string]string{{"action": "*", "resource": "*", "effect": "deny"}},
})
if err != nil {
if variant := model["variant"]; variant != "" {
err = fmt.Errorf("OpenCode model selection for reasoning_effort=%s (variant %q) failed: %w", variant, variant, err)
}
// No prompt or tool call was admitted: a request failure here is a
// reachable retry rather than a reason to end the run.
return "", harnessUnreachable(err)
}
var session struct {
Data struct {
ID string `json:"id"`
} `json:"data"`
}
if err := json.Unmarshal(created, &session); err != nil || !sessionIDPattern.MatchString(session.Data.ID) {
return "", fmt.Errorf("OpenCode returned an invalid session ID")
if resume != "" {
if c.Backend != "v2" || !sessionIDPattern.MatchString(resume) {
return "", fmt.Errorf("retained OpenCode session %.128s cannot be re-adopted on backend %q", RedactEvidence(resume), c.Backend)
}
session.Data.ID = resume
} else {
created, err := c.call(ctx, "post", "/api/session", map[string]any{
"title": "Pipeline stage generation",
"location": map[string]string{"directory": directory},
"model": model,
"permissions": []map[string]string{{"action": "*", "resource": "*", "effect": "deny"}},
})
if err != nil {
if variant := model["variant"]; variant != "" {
err = fmt.Errorf("OpenCode model selection for reasoning_effort=%s (variant %q) failed: %w", variant, variant, err)
}
// No prompt or tool call was admitted: a request failure here is a
// reachable retry rather than a reason to end the run.
return "", harnessUnreachable(err)
}
if err := json.Unmarshal(created, &session); err != nil || !sessionIDPattern.MatchString(session.Data.ID) {
return "", fmt.Errorf("OpenCode returned an invalid session ID")
}
}
var watch *budgetWatch
promptStarted := false
defer func() {
watch.stopWatch(ctx, &err)
defer watch.finish(ctx, &err)
if err != nil && Detached(ctx) {
// The server is restarting: leave the session running and owned.
err = fmt.Errorf("%w: retained OpenCode session %s left running", ErrDetached, session.Data.ID)
return
}
cleanup, cancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second)
defer cancel()
if err != nil {
@@ -392,8 +415,10 @@ func (c *OpenCodeClient) Generate(ctx context.Context, config ModelConfig, promp
// including unbudgeted text-only stages, so its exact native turns survive.
if c.Backend == "v2" {
promptStarted = true
if _, err = c.call(ctx, "post", "/api/session/"+session.Data.ID+"/prompt", body); err != nil {
return "", err
if resume == "" {
if _, err = c.call(ctx, "post", "/api/session/"+session.Data.ID+"/prompt", body); err != nil {
return "", err
}
}
if err = c.waitForSessionIdle(ctx, session.Data.ID); err != nil {
return "", err