Keep OpenCode/Codex chat and stage sessions running across server restarts
A stopping server no longer interrupts harness sessions. Runs are canceled with a shutdown cause, so stage clients leave admitted sessions running and owned; start-up recovery re-adopts live OpenCode V2 stage sessions (wait, read back the result, continue the graph), restarts unadmitted visits, continues runs between stages, and falls back to interrupted with a reason when the harness is gone. Chats with live work are recorded on shutdown and reattached on start; turns that ended during the downtime are folded in from history. The server-spawned V2 chat serve is now a detached owned service. PIPELINES_SHUTDOWN_SESSIONS=stop restores the old behaviour.
This commit is contained in:
@@ -49,8 +49,18 @@ func main() {
|
||||
if err := store.AcquireServerLock(); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
|
||||
signals, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
|
||||
defer stop()
|
||||
// The server's own context ends after a signal, once the chats left live on
|
||||
// their harness have been recorded (see the shutdown goroutine below).
|
||||
ctx, cancelServer := context.WithCancel(context.Background())
|
||||
defer cancelServer()
|
||||
// A restart or stop leaves harness sessions running and the next start
|
||||
// reattaches them; PIPELINES_SHUTDOWN_SESSIONS=stop restores stopping them.
|
||||
detach, err := shutdownDetaches(os.Getenv(shutdownSessionsEnvKey))
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
// A brand-new database starts with the bundled examples; an existing or
|
||||
// already-seeded one is never changed, even if its pipelines were deleted.
|
||||
if seeded, err := server.SeedExamples(ctx, store); err != nil {
|
||||
@@ -62,7 +72,7 @@ func main() {
|
||||
// Each new run gets a bounded one-line title from the configured default
|
||||
// harness; the summariser falls back to the prompt's first line, so title
|
||||
// generation never blocks or fails run creation.
|
||||
options := server.Options{AllowAuthoring: *allowAuthoring, ArtifactsRoot: artifactsRoot, RunTitles: summarise.GenerateTitle, StepVerdicts: summarise.GenerateVerdictReason, VerificationFixtures: server.VerificationFixturesFromEnvironment(os.Getenv)}
|
||||
options := server.Options{AllowAuthoring: *allowAuthoring, ArtifactsRoot: artifactsRoot, RunTitles: summarise.GenerateTitle, StepVerdicts: summarise.GenerateVerdictReason, VerificationFixtures: server.VerificationFixturesFromEnvironment(os.Getenv), DetachSessions: detach}
|
||||
recorded := ""
|
||||
if value, err := store.OpenCodeEndpoint(ctx); err != nil {
|
||||
log.Printf("OpenCode endpoint setting unreadable: %v", err)
|
||||
@@ -117,7 +127,7 @@ func main() {
|
||||
}
|
||||
}
|
||||
if options.Chat == nil {
|
||||
if conn, err := opencode.OpenChat(ctx, opencode.Executable(), ""); err != nil {
|
||||
if conn, err := openChat(ctx, detach); err != nil {
|
||||
log.Printf("OpenCode chat unavailable: %v", err)
|
||||
} else if conn.URL == "" {
|
||||
conn.Close()
|
||||
@@ -141,7 +151,11 @@ func main() {
|
||||
chats := server.NewChatManager(ctx, options.Chat, store)
|
||||
if len(chats.Backends()) > 0 {
|
||||
options.Chat = chats
|
||||
defer chats.Close()
|
||||
if detach {
|
||||
defer chats.DetachSessions()
|
||||
} else {
|
||||
defer chats.Close()
|
||||
}
|
||||
}
|
||||
srv := &http.Server{Addr: *addr, Handler: server.HandlerWithOptions(ctx, options, store), ReadHeaderTimeout: 5 * time.Second}
|
||||
// Deferred before WaitRuns so it runs after the runs settle: the owned stage
|
||||
@@ -149,7 +163,16 @@ func main() {
|
||||
defer opencode.StopIdleStageService()
|
||||
defer server.WaitRuns(srv.Handler)
|
||||
go func() {
|
||||
<-ctx.Done()
|
||||
select {
|
||||
case <-signals.Done():
|
||||
case <-ctx.Done():
|
||||
}
|
||||
if detach {
|
||||
// Record the live chats while their rooms still exist.
|
||||
server.DetachSessions(srv.Handler)
|
||||
log.Printf("shutdown: harness sessions are left running; the next start reattaches them")
|
||||
}
|
||||
cancelServer()
|
||||
shutdownCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
if err := srv.Shutdown(shutdownCtx); err != nil {
|
||||
@@ -172,3 +195,36 @@ func main() {
|
||||
log.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// shutdownSessionsEnvKey selects what a stopping server does with harness
|
||||
// sessions: "keep" (the default) leaves them running for the next start to
|
||||
// reattach; "stop" interrupts runs and closes chat clients as before.
|
||||
const shutdownSessionsEnvKey = "PIPELINES_SHUTDOWN_SESSIONS"
|
||||
|
||||
func shutdownDetaches(value string) (bool, error) {
|
||||
switch value {
|
||||
case "", "keep":
|
||||
return true, nil
|
||||
case "stop":
|
||||
return false, nil
|
||||
}
|
||||
return false, errors.New(shutdownSessionsEnvKey + " must be keep or stop")
|
||||
}
|
||||
|
||||
// openChat opens the OpenCode service chat brokers from when no endpoint is
|
||||
// attached. When sessions are kept across restarts, a V2 installation uses the
|
||||
// detached Pipelines-owned chat service, which outlives this process; otherwise
|
||||
// (and for V1) it is a serve child that ends with the server.
|
||||
func openChat(ctx context.Context, detach bool) (*opencode.Connection, error) {
|
||||
executable := opencode.Executable()
|
||||
if detach {
|
||||
if backend, err := opencode.Detect(ctx, executable); err == nil && backend == "v2" {
|
||||
directory, err := os.Getwd()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return opencode.EnsureOwnedChatService(ctx, executable, directory)
|
||||
}
|
||||
}
|
||||
return opencode.OpenChat(ctx, executable, "")
|
||||
}
|
||||
|
||||
@@ -6,6 +6,45 @@ when the `claude` CLI is installed. The selected backend is stored on the
|
||||
operator-session row and returned as `backend` on every `chat_sessions` row, so a
|
||||
restart does not have to guess from the session id.
|
||||
|
||||
## Restarts keep sessions running
|
||||
|
||||
A server restart or stop does not stop chat sessions. Only an operator action
|
||||
stops one: closing the chat, or Stop on its turn. On shutdown the server records
|
||||
each chat that still has work on its harness (a turn in flight, a pending
|
||||
question or approval, or running background tasks) in `chat_live_sessions`.
|
||||
It then detaches each backend instead of closing it, and never sends an
|
||||
interrupt.
|
||||
|
||||
On the next start each recorded chat is reattached. The server asks the harness
|
||||
whether the turn is still running, re-subscribes to its event stream with no
|
||||
client attached, re-reads pending approvals/questions and background shells,
|
||||
and holds queued prompts behind the turn. If a turn ended while the server was
|
||||
down, or its end event was lost with the old stream, the end is folded in from
|
||||
the harness's history. Clients get a `resync`, the reply notifies once (with
|
||||
the text the harness recorded), and queued prompts are delivered in order. The
|
||||
older start-up probe of recently used chats still runs for anything not
|
||||
recorded, for example after a crash.
|
||||
|
||||
Per backend (the contract is `chatSessionDetacher` in
|
||||
`internal/server/chat_detach.go`):
|
||||
|
||||
- **OpenCode**: sessions live in the OpenCode service. An attached endpoint
|
||||
(the operator's CLI-managed service) already outlives the server. Without
|
||||
one, a V2 server now uses a detached Pipelines-owned chat service
|
||||
(`opencode serve` in its own session, registered in
|
||||
`<cache>/pipelines/private/opencode-chat-service.json`). The next server
|
||||
reuses it at the same address and password, and it is not stopped on
|
||||
shutdown. A V1 serve child still ends with the server.
|
||||
- **Codex**: threads and turns live in the App Server. Detaching closes the
|
||||
control connection, and the next server reconciles through `thread/read`.
|
||||
An approval or question the App Server sent to the old connection is bound to
|
||||
it, so one still pending at shutdown cannot be answered after the restart.
|
||||
- **Claude Code**: closed as before until its host implements
|
||||
`DetachSessions`.
|
||||
|
||||
`PIPELINES_SHUTDOWN_SESSIONS=stop` restores the old shutdown, which closes the
|
||||
chat clients and records nothing. See [Installation](install.md#restarts-keep-sessions-running).
|
||||
|
||||
## Codex
|
||||
|
||||
The adapter in `internal/codex` connects to the Codex App Server's Unix
|
||||
|
||||
@@ -92,7 +92,10 @@ How the stage service behaves:
|
||||
session is active on it. A busy service keeps its old limits until it is idle.
|
||||
- **Lifetime.** It runs detached in its own process session, so a stopping
|
||||
Pipelines service does not take active stage work with it. On server shutdown
|
||||
it is stopped only when no session is active on it. Otherwise restart
|
||||
it is stopped only when no session is active on it. A busy service keeps
|
||||
running, and the next start re-adopts its sessions and continues their runs
|
||||
(see [Installation](install.md#restarts-keep-sessions-running)). With
|
||||
`PIPELINES_SHUTDOWN_SESSIONS=stop`, or when the service is gone, restart
|
||||
recovery interrupts the retained sessions through it, as before.
|
||||
- `pipelines debug` is unchanged: it opens a fresh interactive harness
|
||||
conversation, not the stage session.
|
||||
|
||||
+46
-2
@@ -173,8 +173,8 @@ another installation.
|
||||
| Command | Effect |
|
||||
| --- | --- |
|
||||
| `pipelines start` | start server, web proxy and ntfy; print the startup report |
|
||||
| `pipelines stop` | stop all three (active runs are interrupted; edits are not rolled back) |
|
||||
| `pipelines restart` | stop then start, keeping data |
|
||||
| `pipelines stop` | stop all three; harness sessions keep running and the next start reattaches them (see below) |
|
||||
| `pipelines restart` | stop then start, keeping data; chat turns and stage sessions continue |
|
||||
| `pipelines doctor` | startup report, health of server and ntfy, OpenCode API check (no inference) |
|
||||
| `pipelines status` | server, host and ntfy publishing health, from the running server |
|
||||
| `pipelines logs` | follow `logs/server.log` |
|
||||
@@ -184,6 +184,50 @@ another installation.
|
||||
| `pipelines service disable` | disable autostart and stop the supervisor |
|
||||
| `pipelines service status` | autostart and supervisor state |
|
||||
|
||||
### Restarts keep sessions running
|
||||
|
||||
A server restart, `pipelines stop` and `pipelines update` stop the Pipelines
|
||||
processes, not the work running on harnesses. On SIGTERM the server leaves
|
||||
every OpenCode and Codex session running, records what it needs to reattach,
|
||||
and exits. It does not interrupt or abort anything. On the next start:
|
||||
|
||||
- **Runs**: a stage whose OpenCode V2 session is still alive is re-adopted. The
|
||||
runner waits for that same session (no new prompt), or reads the result it
|
||||
reached while the server was down, and then continues the graph. A stage
|
||||
that had not yet handed work to a harness starts again. A run between stages
|
||||
continues with its next stage. The stage timeout restarts when the stage is
|
||||
reattached. Its file-change list is marked partial, because the snapshot
|
||||
taken before the restart is gone.
|
||||
- **Chats**: turns, pending questions/approvals, background tasks and queued
|
||||
prompts are picked up again, and a reply that arrived during the downtime
|
||||
still notifies. See [Chat backends](chat-backends.md#restarts-keep-sessions-running).
|
||||
|
||||
If the harness is gone, the old behaviour applies and the run is interrupted
|
||||
with the reason in its diagnostic, for example "Server restarted; the OpenCode
|
||||
service running stage … was gone". This happens when the OpenCode service
|
||||
process died, or the session belongs to a Pi or OpenCode V1 process that ended
|
||||
with the server.
|
||||
|
||||
Only explicit operator actions stop sessions:
|
||||
|
||||
- `pipelines cancel <run>` (or Cancel in the app) interrupts a run's stage.
|
||||
- Closing a chat, or Stop on its turn, interrupts that chat.
|
||||
|
||||
To stop everything: cancel the active runs and close or stop the busy chats,
|
||||
then run `pipelines stop`. The owned OpenCode stage service stops itself at
|
||||
shutdown when idle. The owned chat service, used only without an attached
|
||||
endpoint, keeps running and is reused. Stop it by killing the `pid` in
|
||||
`~/Library/Caches/pipelines/private/opencode-chat-service.json`
|
||||
(`~/.cache/...` on Linux). To make every shutdown interrupt runs and close chat
|
||||
clients as before, add `"PIPELINES_SHUTDOWN_SESSIONS": "stop"` to
|
||||
`opencode_env` in `<prefix>/pipelines-install.json` and restart.
|
||||
|
||||
Limit on Linux: systemd stops every process in the service's cgroup. The
|
||||
OpenCode services the server starts (stage service, owned chat service) end
|
||||
with `pipelines stop`/`restart`, and their runs fall back as described above.
|
||||
An attached OpenCode endpoint and the Codex App Server are separate and
|
||||
survive. macOS launchd does not kill them.
|
||||
|
||||
The login service is `~/Library/LaunchAgents/net.a-wan.pipelines.plist` on
|
||||
macOS and `~/.config/systemd/user/net.a-wan.pipelines.service` on Linux. It
|
||||
runs `manage.py supervise`, which owns all three processes. This is login
|
||||
|
||||
@@ -176,6 +176,14 @@ func (c *Chat) Close() {
|
||||
_ = c.conn.Close(websocket.StatusNormalClosure, "")
|
||||
}
|
||||
|
||||
// DetachSessions is the server-restart shutdown: threads and their turns live
|
||||
// in the Codex App Server, not in this client, so closing the control
|
||||
// connection leaves them running; no turn is interrupted. The next server
|
||||
// process reconnects and reconciles each thread through thread/read. An
|
||||
// approval or question the App Server sent to this connection is bound to it,
|
||||
// so one still pending at shutdown is not answerable after the restart.
|
||||
func (c *Chat) DetachSessions() { c.Close() }
|
||||
|
||||
func (c *Chat) readLoop() {
|
||||
for {
|
||||
_, data, err := c.conn.Read(c.ctx)
|
||||
|
||||
@@ -0,0 +1,48 @@
|
||||
package codex
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// A server restart detaches from the App Server without interrupting the
|
||||
// thread's turn; the next server's fresh connection still sees the turn in
|
||||
// progress, then completed, through the read-only turn list.
|
||||
func TestDetachSessionsLeavesTurnRunningForNextConnection(t *testing.T) {
|
||||
var completed atomic.Bool
|
||||
var mu sync.Mutex
|
||||
var methods []string
|
||||
appServer := func(method string, _ json.RawMessage) any {
|
||||
mu.Lock()
|
||||
methods = append(methods, method)
|
||||
mu.Unlock()
|
||||
status := "inProgress"
|
||||
if completed.Load() {
|
||||
status = "completed"
|
||||
}
|
||||
return json.RawMessage(`{"data":[{"id":"turn","status":"` + status + `","items":[]}],"nextCursor":null}`)
|
||||
}
|
||||
before := modelTestChat(t, appServer)
|
||||
if running, err := before.ProbeTurn(context.Background(), fixtureThread); err != nil || !running {
|
||||
t.Fatalf("turn not running before the restart: %v %v", running, err)
|
||||
}
|
||||
before.DetachSessions()
|
||||
after := modelTestChat(t, appServer)
|
||||
if running, err := after.ProbeTurn(context.Background(), fixtureThread); err != nil || !running {
|
||||
t.Fatalf("turn did not survive the restart: %v %v", running, err)
|
||||
}
|
||||
completed.Store(true) // finished while nobody was watching
|
||||
if running, err := after.ProbeTurn(context.Background(), fixtureThread); err != nil || running {
|
||||
t.Fatalf("completed turn still reported running: %v %v", running, err)
|
||||
}
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
for _, method := range methods {
|
||||
if method == "turn/interrupt" {
|
||||
t.Fatalf("detaching interrupted the turn: %v", methods)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -304,6 +304,13 @@ func (c *Chat) Interrupt(ctx context.Context, id string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// DetachSessions is the server-restart shutdown. Sessions, turns, questions,
|
||||
// permissions and shells live in the OpenCode service, not in this adapter, so
|
||||
// there is nothing to stop: the connection's owner decides whether the service
|
||||
// itself outlives the server (an attached endpoint and the detached owned chat
|
||||
// service do; a V1 serve child does not).
|
||||
func (c *Chat) DetachSessions() {}
|
||||
|
||||
// Messages reads one session's stored turns. On the owned serve backend it
|
||||
// reuses Connection.Messages, including its read-only OutputFormat readback
|
||||
// fallback, instead of duplicating that path. It maps the verified assistant
|
||||
|
||||
@@ -0,0 +1,64 @@
|
||||
package opencode
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"strings"
|
||||
"syscall"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// The server-spawned chat service is detached and registered: a second server
|
||||
// process reuses the same running service (sessions survive the restart), and
|
||||
// it is separate from the stage service.
|
||||
func TestOwnedChatServiceIsReusedAcrossServerProcesses(t *testing.T) {
|
||||
executable := fakeOwnedStageEnv(t)
|
||||
t.Setenv(ChatConfigEnvKey, "")
|
||||
t.Cleanup(func() {
|
||||
path, _ := ChatServiceRegistrationPath()
|
||||
if state, err := readOwnedStageState(path); err == nil {
|
||||
_ = syscall.Kill(state.PID, syscall.SIGKILL)
|
||||
}
|
||||
})
|
||||
first, err := EnsureOwnedChatService(context.Background(), executable, t.TempDir())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
path, _ := ChatServiceRegistrationPath()
|
||||
stagePath, _ := StageServiceRegistrationPath()
|
||||
if first.credentialFile != path || path == stagePath || !strings.HasPrefix(first.URL, "http://127.0.0.1:") {
|
||||
t.Fatalf("chat connection = %+v", first)
|
||||
}
|
||||
pid := ownedPID(t, first)
|
||||
first.Close() // the previous server process exits
|
||||
second, err := EnsureOwnedChatService(context.Background(), executable, t.TempDir())
|
||||
if err != nil || second.URL != first.URL || ownedPID(t, second) != pid {
|
||||
t.Fatalf("chat service was not reused by the next server: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// ReferenceAlive tells a live retained service from one whose process is gone,
|
||||
// without reviving it.
|
||||
func TestReferenceAliveDoesNotRevive(t *testing.T) {
|
||||
executable := fakeOwnedStageEnv(t)
|
||||
c, err := OpenStage(context.Background(), executable, t.TempDir(), "v2")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
reference, err := c.Reference()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := ReferenceAlive(context.Background(), reference); err != nil {
|
||||
t.Fatalf("live service reported gone: %v", err)
|
||||
}
|
||||
pid := ownedPID(t, c)
|
||||
_ = syscall.Kill(pid, syscall.SIGKILL)
|
||||
waitGone(t, pid)
|
||||
if err := ReferenceAlive(context.Background(), reference); !errors.Is(err, ErrServiceNotRunning) {
|
||||
t.Fatalf("gone service not reported: %v", err)
|
||||
}
|
||||
if _, healthy := c.ownedInfo(context.Background()); healthy {
|
||||
t.Fatal("ReferenceAlive revived the service")
|
||||
}
|
||||
}
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
"crypto/rand"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
@@ -90,23 +91,65 @@ func (c *Connection) Reference() (string, error) {
|
||||
return f.Name(), nil
|
||||
}
|
||||
|
||||
func FromReference(path string) (*Connection, error) {
|
||||
type connectionReference struct{ Backend, URL, Password, Directory, CredentialFile string }
|
||||
|
||||
func readReference(path string) (connectionReference, error) {
|
||||
var reference connectionReference
|
||||
info, err := os.Lstat(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return reference, err
|
||||
}
|
||||
if !info.Mode().IsRegular() || info.Mode().Perm() != 0600 || info.Size() > MaxRequest {
|
||||
return nil, fmt.Errorf("invalid private OpenCode connection reference")
|
||||
return reference, fmt.Errorf("invalid private OpenCode connection reference")
|
||||
}
|
||||
if stat, ok := info.Sys().(*syscall.Stat_t); !ok || stat.Uid != uint32(os.Getuid()) {
|
||||
return nil, fmt.Errorf("OpenCode reference belongs to another user")
|
||||
return reference, fmt.Errorf("OpenCode reference belongs to another user")
|
||||
}
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return reference, err
|
||||
}
|
||||
var reference struct{ Backend, URL, Password, Directory, CredentialFile string }
|
||||
if err = json.Unmarshal(data, &reference); err != nil {
|
||||
err = json.Unmarshal(data, &reference)
|
||||
return reference, err
|
||||
}
|
||||
|
||||
// ErrServiceNotRunning reports that a retained OpenCode service no longer
|
||||
// answers: its process is gone, and with it any session turn it was running.
|
||||
var ErrServiceNotRunning = errors.New("the OpenCode service is not running")
|
||||
|
||||
// ReferenceAlive reports whether the execution service a retained reference
|
||||
// names is running right now, without starting or reviving it. Restart recovery
|
||||
// uses it to tell a session that kept running across a Pipelines restart from
|
||||
// one whose harness process is gone.
|
||||
func ReferenceAlive(ctx context.Context, path string) error {
|
||||
reference, err := readReference(path)
|
||||
if err != nil {
|
||||
return fmt.Errorf("retained OpenCode reference unreadable: %w", err)
|
||||
}
|
||||
if reference.Backend != "v2" {
|
||||
return fmt.Errorf("the V1 OpenCode stage service belonged to the stopped server process")
|
||||
}
|
||||
if !filepath.IsAbs(reference.CredentialFile) {
|
||||
return fmt.Errorf("invalid V2 service credential reference")
|
||||
}
|
||||
c, err := serviceConnection(reference.CredentialFile, reference.Directory)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if c.URL != reference.URL {
|
||||
return fmt.Errorf("retained OpenCode service endpoint changed")
|
||||
}
|
||||
probe, cancel := context.WithTimeout(ctx, 3*time.Second)
|
||||
defer cancel()
|
||||
if err := c.probeHealth(probe, "v2"); err != nil {
|
||||
return fmt.Errorf("%w at %s", ErrServiceNotRunning, c.URL)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func FromReference(path string) (*Connection, error) {
|
||||
reference, err := readReference(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if reference.Backend == "v2" {
|
||||
|
||||
@@ -54,14 +54,56 @@ type ownedStageState struct {
|
||||
Identity string `json:"identity"`
|
||||
}
|
||||
|
||||
// StageServiceRegistrationPath is the private registration of the owned stage
|
||||
// service, beside the private connection references.
|
||||
func StageServiceRegistrationPath() (string, error) {
|
||||
// ownedService names one Pipelines-owned detached `opencode serve`: the stage
|
||||
// service, or the chat service a server without an attached endpoint uses.
|
||||
type ownedService struct {
|
||||
name string // for logs: "stage" or "chat"
|
||||
file string // registration file name, without .json
|
||||
}
|
||||
|
||||
var (
|
||||
stageOwnedService = ownedService{name: "stage", file: "opencode-stage-service"}
|
||||
chatOwnedService = ownedService{name: "chat", file: "opencode-chat-service"}
|
||||
)
|
||||
|
||||
func (s ownedService) registrationPath() (string, error) {
|
||||
cache, err := os.UserCacheDir()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return filepath.Join(cache, "pipelines", "private", "opencode-stage-service.json"), nil
|
||||
return filepath.Join(cache, "pipelines", "private", s.file+".json"), nil
|
||||
}
|
||||
|
||||
// StageServiceRegistrationPath is the private registration of the owned stage
|
||||
// service, beside the private connection references.
|
||||
func StageServiceRegistrationPath() (string, error) {
|
||||
return stageOwnedService.registrationPath()
|
||||
}
|
||||
|
||||
// ChatServiceRegistrationPath is the private registration of the owned chat
|
||||
// service (see EnsureOwnedChatService).
|
||||
func ChatServiceRegistrationPath() (string, error) {
|
||||
return chatOwnedService.registrationPath()
|
||||
}
|
||||
|
||||
// EnsureOwnedChatService returns a connection to the detached Pipelines-owned
|
||||
// V2 chat service, starting or reviving it when needed. Unlike a serve child of
|
||||
// the server, it keeps running when the server stops, so chat turns, questions
|
||||
// and background shells survive a restart; the next server reuses the same
|
||||
// endpoint and password from the private registration. Chat-only limits
|
||||
// (PIPELINES_OPENCODE_CHAT_CONFIG) apply to it alone.
|
||||
func EnsureOwnedChatService(ctx context.Context, executable, directory string) (*Connection, error) {
|
||||
overrides, err := chatConfigOverrides()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
content := ""
|
||||
if overrides != nil {
|
||||
if content, err = configContent(os.Getenv("OPENCODE_CONFIG_CONTENT"), overrides); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return ensureOwnedService(ctx, chatOwnedService, content, executable, directory)
|
||||
}
|
||||
|
||||
func isOwnedStageRegistration(path string) bool {
|
||||
@@ -81,7 +123,11 @@ func ensureOwnedStageService(ctx context.Context, executable, directory string)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
path, err := StageServiceRegistrationPath()
|
||||
return ensureOwnedService(ctx, stageOwnedService, content, executable, directory)
|
||||
}
|
||||
|
||||
func ensureOwnedService(ctx context.Context, service ownedService, content, executable, directory string) (*Connection, error) {
|
||||
path, err := service.registrationPath()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -92,7 +138,7 @@ func ensureOwnedStageService(ctx context.Context, executable, directory string)
|
||||
defer ownedStageMu.Unlock()
|
||||
unlock, err := lockFile(path + ".lock")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("OpenCode stage service lock: %w", err)
|
||||
return nil, fmt.Errorf("OpenCode %s service lock: %w", service.name, err)
|
||||
}
|
||||
defer unlock()
|
||||
|
||||
@@ -106,11 +152,11 @@ func ensureOwnedStageService(ctx context.Context, executable, directory string)
|
||||
case healthy && state.Identity == identity:
|
||||
return ownedStageConnection(path, executable, directory)
|
||||
case healthy && current.ownedIdle(ctx):
|
||||
log.Printf("OpenCode stage service configuration changed; restarting the idle service")
|
||||
log.Printf("OpenCode %s service configuration changed; restarting the idle service", service.name)
|
||||
stopOwnedPID(pid)
|
||||
case healthy:
|
||||
// Never interrupt active stage work to apply a new limit.
|
||||
log.Printf("OpenCode stage service configuration changed; keeping the busy service until it is idle")
|
||||
log.Printf("OpenCode %s service configuration changed; keeping the busy service until it is idle", service.name)
|
||||
return ownedStageConnection(path, executable, directory)
|
||||
}
|
||||
if u, err := url.Parse(state.URL); err == nil {
|
||||
@@ -135,7 +181,7 @@ func ensureOwnedStageService(ctx context.Context, executable, directory string)
|
||||
case <-time.After(time.Duration(attempt) * time.Second):
|
||||
}
|
||||
}
|
||||
started, err := startOwnedStage(ctx, executable, port, state.Password, content)
|
||||
started, err := startOwnedStage(ctx, service, executable, port, state.Password, content)
|
||||
if err == nil {
|
||||
started.Identity = identity
|
||||
if err := writeOwnedStageState(path, started); err != nil {
|
||||
@@ -197,7 +243,7 @@ func ownedStageIdentity(executable, content string) string {
|
||||
|
||||
// startOwnedStage starts one detached serve process and waits until it answers
|
||||
// the authenticated V2 info contract.
|
||||
func startOwnedStage(ctx context.Context, executable string, port int, password, content string) (ownedStageState, error) {
|
||||
func startOwnedStage(ctx context.Context, service ownedService, executable string, port int, password, content string) (ownedStageState, error) {
|
||||
if port == 0 {
|
||||
free, err := freePort()
|
||||
if err != nil {
|
||||
@@ -214,7 +260,7 @@ func startOwnedStage(ctx context.Context, executable string, port int, password,
|
||||
// Its own session: a stopping Pipelines service (and its process group)
|
||||
// does not take active stage work with it.
|
||||
cmd.SysProcAttr = &syscall.SysProcAttr{Setsid: true}
|
||||
logPath, _ := StageServiceRegistrationPath()
|
||||
logPath, _ := service.registrationPath()
|
||||
logPath = strings.TrimSuffix(logPath, ".json") + ".log"
|
||||
logFile, err := os.OpenFile(logPath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0600)
|
||||
if err != nil {
|
||||
@@ -224,7 +270,7 @@ func startOwnedStage(ctx context.Context, executable string, port int, password,
|
||||
cmd.Stdin = nil
|
||||
if err := cmd.Start(); err != nil {
|
||||
logFile.Close()
|
||||
return ownedStageState{}, fmt.Errorf("OpenCode stage service could not start: %w", err)
|
||||
return ownedStageState{}, fmt.Errorf("OpenCode %s service could not start: %w", service.name, err)
|
||||
}
|
||||
logFile.Close()
|
||||
exited := make(chan struct{})
|
||||
@@ -241,13 +287,13 @@ func startOwnedStage(ctx context.Context, executable string, port int, password,
|
||||
}
|
||||
select {
|
||||
case <-exited:
|
||||
return ownedStageState{}, fmt.Errorf("OpenCode stage service exited before becoming ready (see %s)", logPath)
|
||||
return ownedStageState{}, fmt.Errorf("OpenCode %s service exited before becoming ready (see %s)", service.name, logPath)
|
||||
case <-startup.Done():
|
||||
_ = cmd.Process.Kill()
|
||||
if ctx.Err() != nil {
|
||||
return ownedStageState{}, ctx.Err()
|
||||
}
|
||||
return ownedStageState{}, fmt.Errorf("OpenCode stage service did not become ready within %s (see %s)", ownedStageStartTimeout, logPath)
|
||||
return ownedStageState{}, fmt.Errorf("OpenCode %s service did not become ready within %s (see %s)", service.name, ownedStageStartTimeout, logPath)
|
||||
case <-time.After(150 * time.Millisecond):
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,90 @@
|
||||
package pipeline
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
)
|
||||
|
||||
// ChatLiveSession is one chat a stopping server left live on its harness: a
|
||||
// turn in flight, an unanswered question or approval, or running background
|
||||
// tasks. The next server process reattaches each one (and reads back the final
|
||||
// state of a turn that ended while it was down).
|
||||
type ChatLiveSession struct {
|
||||
SessionID string
|
||||
Backend string
|
||||
// Turn is true when a turn was in flight; NativeTurnID names it when the
|
||||
// harness reported one, so a reply is announced once.
|
||||
Turn bool
|
||||
NativeTurnID string
|
||||
// Attention is true when a question/approval was pending or background
|
||||
// tasks were running.
|
||||
Attention bool
|
||||
RecordedAt time.Time
|
||||
}
|
||||
|
||||
func (s *Store) initChatLiveSessions() error {
|
||||
_, err := s.db.Exec(`CREATE TABLE IF NOT EXISTS chat_live_sessions (
|
||||
session_id TEXT PRIMARY KEY,
|
||||
backend TEXT NOT NULL DEFAULT '',
|
||||
turn INTEGER NOT NULL DEFAULT 0,
|
||||
native_turn_id TEXT NOT NULL DEFAULT '',
|
||||
attention INTEGER NOT NULL DEFAULT 0,
|
||||
recorded_at INTEGER NOT NULL)`)
|
||||
return err
|
||||
}
|
||||
|
||||
// RecordChatLiveSessions replaces the recorded set with sessions.
|
||||
func (s *Store) RecordChatLiveSessions(ctx context.Context, sessions []ChatLiveSession) error {
|
||||
tx, err := s.db.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer tx.Rollback()
|
||||
if _, err := tx.ExecContext(ctx, `DELETE FROM chat_live_sessions`); err != nil {
|
||||
return err
|
||||
}
|
||||
for _, session := range sessions {
|
||||
at := session.RecordedAt
|
||||
if at.IsZero() {
|
||||
at = time.Now()
|
||||
}
|
||||
if _, err := tx.ExecContext(ctx, `INSERT OR REPLACE INTO chat_live_sessions(session_id,backend,turn,native_turn_id,attention,recorded_at) VALUES(?,?,?,?,?,?)`,
|
||||
session.SessionID, session.Backend, session.Turn, session.NativeTurnID, session.Attention, at.UnixMilli()); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return tx.Commit()
|
||||
}
|
||||
|
||||
// TakeChatLiveSessions returns the recorded set and clears it, so each start
|
||||
// reattaches what the previous process recorded exactly once.
|
||||
func (s *Store) TakeChatLiveSessions(ctx context.Context) ([]ChatLiveSession, error) {
|
||||
tx, err := s.db.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer tx.Rollback()
|
||||
rows, err := tx.QueryContext(ctx, `SELECT session_id,backend,turn,native_turn_id,attention,recorded_at FROM chat_live_sessions ORDER BY recorded_at, session_id`)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var sessions []ChatLiveSession
|
||||
for rows.Next() {
|
||||
var session ChatLiveSession
|
||||
var at int64
|
||||
if err := rows.Scan(&session.SessionID, &session.Backend, &session.Turn, &session.NativeTurnID, &session.Attention, &at); err != nil {
|
||||
rows.Close()
|
||||
return nil, err
|
||||
}
|
||||
session.RecordedAt = time.UnixMilli(at)
|
||||
sessions = append(sessions, session)
|
||||
}
|
||||
rows.Close()
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if _, err := tx.ExecContext(ctx, `DELETE FROM chat_live_sessions`); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return sessions, tx.Commit()
|
||||
}
|
||||
@@ -190,18 +190,57 @@ func (s *Store) addColumnIfMissing(table, column, definition string) error {
|
||||
return err
|
||||
}
|
||||
|
||||
// RestartStep marks a visit that a stopped server had begun but had not yet
|
||||
// handed to any harness session as interrupted, so its run can begin the same
|
||||
// stage again under a new sequence. Nothing was admitted, so nothing replays.
|
||||
func (s *Store) RestartStep(ctx context.Context, id int64, sequence int, diagnostic string) error {
|
||||
return affected(s.db.ExecContext(ctx, "UPDATE run_steps SET status='interrupted',agent_active=0,diagnostic=?,finished_at=CURRENT_TIMESTAMP WHERE run_id=? AND sequence=? AND status='running' AND session_id=''", diagnostic, id, sequence))
|
||||
}
|
||||
|
||||
func (s *Store) RecoverRuns(ctx context.Context) error {
|
||||
return s.RecoverRunsKeeping(ctx, nil, nil)
|
||||
}
|
||||
|
||||
// RecoverRunsKeeping is RecoverRuns for a server that reattaches work across a
|
||||
// restart: runs in keep are left exactly as the previous process left them (the
|
||||
// runner continues them), and a run in reasons is interrupted with that reason
|
||||
// instead of the generic one, so the operator sees why it was not reattached.
|
||||
func (s *Store) RecoverRunsKeeping(ctx context.Context, keep map[int64]bool, reasons map[int64]string) error {
|
||||
tx, err := s.db.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer tx.Rollback()
|
||||
kept, keptArgs := "", []any{}
|
||||
for id := range keep {
|
||||
if kept != "" {
|
||||
kept += ","
|
||||
}
|
||||
kept += "?"
|
||||
keptArgs = append(keptArgs, id)
|
||||
}
|
||||
stepFilter, runFilter := "", ""
|
||||
if kept != "" {
|
||||
stepFilter = " AND run_id NOT IN (" + kept + ")"
|
||||
runFilter = " AND id NOT IN (" + kept + ")"
|
||||
}
|
||||
// No replay: pending visits never began. A cleanup agent that was still
|
||||
// active is marked interrupted, and its run is interrupted below.
|
||||
if _, err := tx.ExecContext(ctx, "UPDATE run_steps SET status=CASE WHEN (SELECT cancel_requested FROM pipeline_runs WHERE id=run_steps.run_id)=1 THEN 'canceled' ELSE 'interrupted' END,agent_active=0,diagnostic='Server stopped; stage was not resumed',finished_at=CURRENT_TIMESTAMP WHERE (status='running' OR agent_active=1) AND status<>? AND run_id IN (SELECT id FROM pipeline_runs WHERE cleanup_unconfirmed=0)", protocol.StepStatusWaiting); err != nil {
|
||||
if _, err := tx.ExecContext(ctx, "UPDATE run_steps SET status=CASE WHEN (SELECT cancel_requested FROM pipeline_runs WHERE id=run_steps.run_id)=1 THEN 'canceled' ELSE 'interrupted' END,agent_active=0,diagnostic='Server stopped; stage was not resumed',finished_at=CURRENT_TIMESTAMP WHERE (status='running' OR agent_active=1) AND status<>? AND run_id IN (SELECT id FROM pipeline_runs WHERE cleanup_unconfirmed=0)"+stepFilter, append([]any{protocol.StepStatusWaiting}, keptArgs...)...); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := tx.ExecContext(ctx, "UPDATE pipeline_runs SET status=CASE WHEN cancel_requested=1 THEN 'canceled' ELSE 'interrupted' END,diagnostic='Server stopped; owned sessions stopped; run was not resumed',finished_at=CURRENT_TIMESTAMP WHERE status IN ('queued','running') AND status<>? AND cleanup_unconfirmed=0", protocol.RunStatusWaiting); err != nil {
|
||||
for id, reason := range reasons {
|
||||
if keep[id] {
|
||||
continue
|
||||
}
|
||||
if _, err := tx.ExecContext(ctx, "UPDATE run_steps SET diagnostic=? WHERE run_id=? AND status='interrupted' AND sequence=(SELECT MAX(sequence) FROM run_steps WHERE run_id=?)", "Server restarted; "+reason, id, id); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := tx.ExecContext(ctx, "UPDATE pipeline_runs SET status='interrupted',diagnostic=?,finished_at=CURRENT_TIMESTAMP WHERE id=? AND status IN ('queued','running') AND cancel_requested=0 AND cleanup_unconfirmed=0", "Server restarted; "+reason+"; run was not resumed", id); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if _, err := tx.ExecContext(ctx, "UPDATE pipeline_runs SET status=CASE WHEN cancel_requested=1 THEN 'canceled' ELSE 'interrupted' END,diagnostic='Server stopped; owned sessions stopped; run was not resumed',finished_at=CURRENT_TIMESTAMP WHERE status IN ('queued','running') AND status<>? AND cleanup_unconfirmed=0"+runFilter, append([]any{protocol.RunStatusWaiting}, keptArgs...)...); err != nil {
|
||||
return err
|
||||
}
|
||||
// An export checkpoint is a paused handoff, not an interrupted model call.
|
||||
|
||||
@@ -93,6 +93,10 @@ CREATE TABLE IF NOT EXISTS submissions (id INTEGER PRIMARY KEY, name TEXT NOT NU
|
||||
db.Close()
|
||||
return nil, err
|
||||
}
|
||||
if err := s.initChatLiveSessions(); err != nil {
|
||||
db.Close()
|
||||
return nil, err
|
||||
}
|
||||
if err := s.initAppNotifications(); err != nil {
|
||||
db.Close()
|
||||
return nil, err
|
||||
|
||||
@@ -0,0 +1,243 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.a-wan.net/alan/pipelines/internal/pipeline"
|
||||
"git.a-wan.net/alan/pipelines/internal/protocol"
|
||||
)
|
||||
|
||||
// Chat sessions survive a Pipelines restart. Only an operator action (closing
|
||||
// the chat, or Stop on its turn) stops one.
|
||||
//
|
||||
// Per-backend contract. A chat backend whose sessions can outlive the server
|
||||
// process implements chatSessionDetacher. On shutdown the server calls
|
||||
// DetachSessions instead of Close: the backend releases its own connection or
|
||||
// process handles but must not interrupt, abort or kill any session, turn,
|
||||
// question or background task. On the next start the server reattaches through
|
||||
// the ordinary broker surface: chatTurnProber.ProbeTurn (is the turn still
|
||||
// running?), Stream (re-subscribe), Permissions/Forms (pending questions and
|
||||
// approvals), Shells (background tasks) and Messages (read back a turn that
|
||||
// ended while the server was down). A backend without DetachSessions is closed
|
||||
// as before, and its sessions end with the server.
|
||||
//
|
||||
// OpenCode (an attached or detached owned service) and Codex (its App Server)
|
||||
// implement it; a Claude Code host that keeps its CLI processes alive can
|
||||
// implement the same method.
|
||||
type chatSessionDetacher interface {
|
||||
DetachSessions()
|
||||
}
|
||||
|
||||
// chatLiveStore records, on shutdown, the chats left live on their harness so
|
||||
// the next start reattaches exactly those; *pipeline.Store implements it.
|
||||
type chatLiveStore interface {
|
||||
RecordChatLiveSessions(ctx context.Context, sessions []pipeline.ChatLiveSession) error
|
||||
TakeChatLiveSessions(ctx context.Context) ([]pipeline.ChatLiveSession, error)
|
||||
}
|
||||
|
||||
// chatReattachPoll is how often a reattached turn re-asks its harness whether
|
||||
// it is still running, to fold in an end the re-subscribed stream did not see.
|
||||
var chatReattachPoll = 2 * time.Second
|
||||
|
||||
// detachSessions records every chat with work still live on its harness: a
|
||||
// turn in flight, a pending question or approval, or running background tasks.
|
||||
// It must run before rooms are torn down (before the server context is
|
||||
// canceled), and it never touches the sessions themselves.
|
||||
func (h *chatHub) detachSessions(ctx context.Context) error {
|
||||
if h.liveStore == nil {
|
||||
return nil
|
||||
}
|
||||
h.mu.Lock()
|
||||
var live []pipeline.ChatLiveSession
|
||||
for id, room := range h.rooms {
|
||||
turn := room.running || room.observedTurn
|
||||
if step := h.turns[id]; step != nil && step.step != nil && step.step.Status == protocol.ChatStepStarted && !step.step.Intermediate {
|
||||
turn = true
|
||||
}
|
||||
attention := len(room.pending) > 0 || len(room.forms) > 0 || h.backgroundTasksLocked(id) > 0
|
||||
if turn || attention {
|
||||
live = append(live, pipeline.ChatLiveSession{SessionID: id, Turn: turn, NativeTurnID: room.nativeTurnID, Attention: attention, RecordedAt: h.clock()})
|
||||
}
|
||||
}
|
||||
h.mu.Unlock()
|
||||
if len(live) > 0 {
|
||||
log.Printf("chat: leaving %d chat session(s) running on their harness for the next start", len(live))
|
||||
}
|
||||
return h.liveStore.RecordChatLiveSessions(ctx, live)
|
||||
}
|
||||
|
||||
// reattachSessions is the start-up half of detachSessions: each recorded chat
|
||||
// is observed again (stream, pending questions, background tasks) while its
|
||||
// harness still has work, and a turn that ended while the server was down is
|
||||
// folded in from the harness's history, so its reply still notifies and its
|
||||
// queued prompts advance, as if it had been observed live.
|
||||
func (h *chatHub) reattachSessions(ctx context.Context, broker chatBroker) {
|
||||
if h.liveStore == nil || broker == nil {
|
||||
return
|
||||
}
|
||||
live, err := h.liveStore.TakeChatLiveSessions(ctx)
|
||||
if err != nil {
|
||||
log.Printf("chat reattach: %v", err)
|
||||
return
|
||||
}
|
||||
for _, session := range live {
|
||||
if ctx.Err() != nil {
|
||||
return
|
||||
}
|
||||
if !validChatSessionID(session.SessionID) {
|
||||
continue
|
||||
}
|
||||
h.reattachSession(ctx, broker, session)
|
||||
}
|
||||
}
|
||||
|
||||
func (h *chatHub) reattachSession(ctx context.Context, broker chatBroker, session pipeline.ChatLiveSession) {
|
||||
id := session.SessionID
|
||||
running := false
|
||||
if prober, ok := broker.(chatTurnProber); ok {
|
||||
probeCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
|
||||
busy, err := prober.ProbeTurn(probeCtx, id)
|
||||
cancel()
|
||||
if err != nil {
|
||||
// Unknown is not idle: keep observing rather than guess it ended.
|
||||
log.Printf("chat reattach probe session=%s failed (%T)", id, err)
|
||||
busy = session.Turn
|
||||
}
|
||||
running = busy
|
||||
}
|
||||
if running || session.Attention {
|
||||
h.observeUnattended(id, broker)
|
||||
}
|
||||
if !session.Turn {
|
||||
return
|
||||
}
|
||||
if !running {
|
||||
h.foldEndedTurn(ctx, broker, session, nil)
|
||||
return
|
||||
}
|
||||
h.mu.Lock()
|
||||
room := h.rooms[id]
|
||||
if room == nil {
|
||||
h.mu.Unlock()
|
||||
return
|
||||
}
|
||||
// The turn the previous process watched is still running: hold the queue
|
||||
// behind it and remember it as started, so its end notifies as usual.
|
||||
turn := h.turnLocked(id)
|
||||
if turn.step == nil || turn.step.Status == protocol.ChatStepEnded {
|
||||
turn.step = &protocol.ChatStep{Status: protocol.ChatStepStarted, TurnID: session.NativeTurnID}
|
||||
}
|
||||
if room.nativeTurnID == "" {
|
||||
room.nativeTurnID = session.NativeTurnID
|
||||
}
|
||||
room.observedTurn = true
|
||||
h.mu.Unlock()
|
||||
go h.awaitReattachedTurn(room, broker, session)
|
||||
}
|
||||
|
||||
// awaitReattachedTurn waits for a reattached turn to end. The stream normally
|
||||
// reports it; when the harness reports the turn idle without the stream having
|
||||
// said so, the end is folded in from history.
|
||||
func (h *chatHub) awaitReattachedTurn(room *chatRoom, broker chatBroker, session pipeline.ChatLiveSession) {
|
||||
prober, _ := broker.(chatTurnProber)
|
||||
started := h.clock()
|
||||
ticker := time.NewTicker(chatReattachPoll)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-room.ctx.Done():
|
||||
return
|
||||
case <-ticker.C:
|
||||
}
|
||||
h.mu.Lock()
|
||||
ended := !room.observedTurn || h.rooms[room.sessionID] != room
|
||||
h.mu.Unlock()
|
||||
if ended || prober == nil || h.clock().Sub(started) >= h.unattended.turnLimit {
|
||||
return
|
||||
}
|
||||
probeCtx, cancel := context.WithTimeout(room.ctx, 5*time.Second)
|
||||
busy, err := prober.ProbeTurn(probeCtx, room.sessionID)
|
||||
cancel()
|
||||
if err != nil || busy {
|
||||
continue
|
||||
}
|
||||
h.foldEndedTurn(room.ctx, broker, session, room)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// foldEndedTurn settles a turn that ended without the hub observing its end:
|
||||
// attached clients are told to re-read the authoritative history, the room's
|
||||
// turn slot is released (so queued prompts are delivered) and the reply is
|
||||
// announced once, with its text read back from the harness.
|
||||
func (h *chatHub) foldEndedTurn(ctx context.Context, broker chatBroker, session pipeline.ChatLiveSession, room *chatRoom) {
|
||||
id := session.SessionID
|
||||
readCtx, cancel := context.WithTimeout(ctx, 10*time.Second)
|
||||
page, err := broker.Messages(readCtx, id, 20, "")
|
||||
cancel()
|
||||
if err != nil {
|
||||
log.Printf("chat reattach history session=%s failed (%T)", id, err)
|
||||
}
|
||||
replyID, body := lastReply(page.Messages)
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
if room != nil && h.rooms[id] == room && room.observedTurn {
|
||||
end := protocol.ChatEventFrame{SessionID: id, Kind: protocol.ChatEventStep, Step: &protocol.ChatStep{Status: protocol.ChatStepEnded, TurnID: room.nativeTurnID}}
|
||||
h.applyLocked(end)
|
||||
h.finishTurnLocked(room)
|
||||
resync := protocol.ChatEventFrame{SessionID: id, Kind: protocol.ChatEventResync}
|
||||
for _, sub := range room.subs {
|
||||
enqueueChatLocked(sub, protocol.Message{Type: protocol.ChatEvent, ChatEvent: &resync})
|
||||
enqueueChatLocked(sub, protocol.Message{Type: protocol.ChatEvent, ChatEvent: &end})
|
||||
}
|
||||
if room.watching && len(room.subs) == 0 && len(room.queue) == 0 && !h.keepUnattendedLocked(room, false, false) {
|
||||
room.watching = false
|
||||
h.releaseRoomLocked(room)
|
||||
}
|
||||
}
|
||||
if replyID == "" || !h.attentionWanted() {
|
||||
return
|
||||
}
|
||||
key := session.NativeTurnID
|
||||
if key == "" {
|
||||
key = replyID
|
||||
}
|
||||
if !h.claimChatAttentionLocked("reply\x00" + id + "\x00" + key) {
|
||||
return
|
||||
}
|
||||
if body == "" {
|
||||
body = chatReplyFallbackBody
|
||||
}
|
||||
log.Printf("chat session %s: turn finished while the server was restarting; reply read back from history", id)
|
||||
h.startChatAttention(chatAttention{sessionID: id, reply: true, body: body, id: chatNotificationID(id, "reply", key)})
|
||||
}
|
||||
|
||||
// lastReply returns the id and bounded text of the assistant reply that ends
|
||||
// a history page (messages after the last user message), or "" when the page
|
||||
// ends with the user's own message.
|
||||
func lastReply(messages []protocol.ChatMessage) (string, string) {
|
||||
for i := len(messages) - 1; i >= 0; i-- {
|
||||
message := messages[i]
|
||||
if message.Role == protocol.ChatRoleUser {
|
||||
return "", ""
|
||||
}
|
||||
if message.Kind != "" {
|
||||
continue
|
||||
}
|
||||
var text strings.Builder
|
||||
for _, part := range message.Parts {
|
||||
if part.Type != protocol.ChatPartText || part.Text == "" {
|
||||
continue
|
||||
}
|
||||
if text.Len() > 0 {
|
||||
text.WriteString("\n\n")
|
||||
}
|
||||
text.WriteString(part.Text)
|
||||
}
|
||||
return message.ID, truncateRunes(paragraphs(text.String()), chatReplyBodyRunes)
|
||||
}
|
||||
return "", ""
|
||||
}
|
||||
@@ -0,0 +1,203 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http/httptest"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.a-wan.net/alan/pipelines/internal/protocol"
|
||||
)
|
||||
|
||||
// restartChat is a harness fake whose sessions live outside the server, the
|
||||
// way an OpenCode service or the Codex App Server keeps them: a turn keeps
|
||||
// running (probe) and its history (Messages) is readable after the server
|
||||
// that started it is gone.
|
||||
type restartChat struct {
|
||||
*observedChat
|
||||
hmu sync.Mutex
|
||||
history []protocol.ChatMessage
|
||||
}
|
||||
|
||||
func newRestartChat() *restartChat { return &restartChat{observedChat: newObservedChat()} }
|
||||
|
||||
func (c *restartChat) Messages(context.Context, string, int, string) (protocol.ChatHistoryPage, error) {
|
||||
c.hmu.Lock()
|
||||
defer c.hmu.Unlock()
|
||||
return protocol.ChatHistoryPage{Messages: append([]protocol.ChatMessage(nil), c.history...)}, nil
|
||||
}
|
||||
|
||||
func (c *restartChat) setProbe(running bool) {
|
||||
c.smu.Lock()
|
||||
c.probe["ses_1"] = running
|
||||
c.smu.Unlock()
|
||||
}
|
||||
|
||||
// chatRestartServer is one server process over a shared database.
|
||||
type chatRestartServer struct {
|
||||
ctx context.Context
|
||||
cancel context.CancelFunc
|
||||
srv *httptest.Server
|
||||
handler *runHandler
|
||||
}
|
||||
|
||||
func startChatRestartServer(t *testing.T, broker chatBroker, path string) *chatRestartServer {
|
||||
t.Helper()
|
||||
store := openQueueStore(t, path)
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
|
||||
handler := HandlerWithOptions(ctx, Options{Chat: broker, DetachSessions: true}, store).(*runHandler)
|
||||
s := &chatRestartServer{ctx: ctx, cancel: cancel, srv: httptest.NewServer(handler), handler: handler}
|
||||
t.Cleanup(s.stop)
|
||||
return s
|
||||
}
|
||||
|
||||
// shutdown is the server's SIGTERM path: record live chats, then stop.
|
||||
func (s *chatRestartServer) shutdown() {
|
||||
DetachSessions(s.handler)
|
||||
s.stop()
|
||||
WaitRuns(s.handler)
|
||||
}
|
||||
|
||||
func (s *chatRestartServer) stop() {
|
||||
s.srv.CloseClientConnections()
|
||||
s.srv.Close()
|
||||
s.cancel()
|
||||
}
|
||||
|
||||
func restartChatDB(t *testing.T) string {
|
||||
t.Helper()
|
||||
path := filepath.Join(t.TempDir(), "chat.db")
|
||||
seed := openQueueStore(t, path)
|
||||
if err := seed.RecordOperatorSession(context.Background(), "ses_1", t.TempDir(), "Release prep", protocol.ChatSessionTagOperator); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := seed.UpdateNotificationSettings(context.Background(), map[string]any{protocol.NotifyChatReplyFinished: true}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
seed.Close()
|
||||
return path
|
||||
}
|
||||
|
||||
func fastReattach(t *testing.T) {
|
||||
t.Helper()
|
||||
fastUnattended(t)
|
||||
t.Setenv("PIPELINES_NTFY_URL", "")
|
||||
poll := chatReattachPoll
|
||||
chatReattachPoll = 20 * time.Millisecond
|
||||
t.Cleanup(func() { chatReattachPoll = poll })
|
||||
}
|
||||
|
||||
// startTurnThenRestart prompts on the first server, holds the turn in flight
|
||||
// (and optionally queues a second prompt), and shuts that server down.
|
||||
func startTurnThenRestart(t *testing.T, path string, queueSecond bool) {
|
||||
t.Helper()
|
||||
first := newRestartChat()
|
||||
server := startChatRestartServer(t, first, path)
|
||||
conn := dialChatConn(t, server.ctx, server.srv)
|
||||
sendPrompt(t, server.ctx, conn, "p1", "build it", "op-1")
|
||||
first.push(stepFrameFor(protocol.ChatStepStarted, "turn_1"))
|
||||
waitFor(t, "first prompt", func() bool { return first.promptCount() == 1 })
|
||||
if queueSecond {
|
||||
if ack := sendPrompt(t, server.ctx, conn, "p2", "then ship it", "op-2"); !ack.ChatQueued {
|
||||
t.Fatalf("second prompt was not queued behind the running turn: %+v", ack)
|
||||
}
|
||||
}
|
||||
server.shutdown()
|
||||
}
|
||||
|
||||
func replyNotes(t *testing.T, path string) []protocol.AppNotification {
|
||||
t.Helper()
|
||||
return chatNotes(t, openQueueStore(t, path), protocol.NotificationChatReply)
|
||||
}
|
||||
|
||||
// A turn in flight at shutdown that finished while the server was down is
|
||||
// folded in from the harness's history: its reply notifies once, with the
|
||||
// text the harness recorded during the downtime.
|
||||
func TestChatTurnFinishedDuringRestartIsFoldedIn(t *testing.T) {
|
||||
fastReattach(t)
|
||||
path := restartChatDB(t)
|
||||
startTurnThenRestart(t, path, false)
|
||||
second := newRestartChat()
|
||||
second.setProbe(false) // ended while the server was down
|
||||
second.history = []protocol.ChatMessage{
|
||||
{ID: "msg_u", SessionID: "ses_1", Role: protocol.ChatRoleUser, Parts: []protocol.ChatPart{{Type: protocol.ChatPartText, Text: "build it"}}},
|
||||
{ID: "msg_a", SessionID: "ses_1", Role: "assistant", Parts: []protocol.ChatPart{{Type: protocol.ChatPartText, Text: "Built while you were away."}}},
|
||||
}
|
||||
startChatRestartServer(t, second, path)
|
||||
waitFor(t, "reply folded in from history", func() bool { return len(replyNotes(t, path)) == 1 })
|
||||
if notes := replyNotes(t, path); !strings.Contains(notes[0].Message, "Built while you were away.") {
|
||||
t.Fatalf("reply body not read back from history: %+v", notes[0])
|
||||
}
|
||||
}
|
||||
|
||||
// A turn still running after the restart keeps its queued prompt waiting;
|
||||
// when the harness reports the turn idle (its end event was lost with the old
|
||||
// stream), the end is folded in and the queued prompt is delivered.
|
||||
func TestChatQueuedPromptDeliveredAfterRestart(t *testing.T) {
|
||||
fastReattach(t)
|
||||
path := restartChatDB(t)
|
||||
startTurnThenRestart(t, path, true)
|
||||
second := newRestartChat()
|
||||
second.setProbe(true)
|
||||
second.history = []protocol.ChatMessage{{ID: "msg_a", SessionID: "ses_1", Role: "assistant", Parts: []protocol.ChatPart{{Type: protocol.ChatPartText, Text: "First part done."}}}}
|
||||
startChatRestartServer(t, second, path)
|
||||
waitFor(t, "reattached stream", func() bool { active, _ := second.streams(); return active >= 1 })
|
||||
time.Sleep(2 * chatQueueRecoverySettle)
|
||||
if second.promptCount() != 0 {
|
||||
t.Fatalf("queued prompt delivered into a turn still running: %v", second.promptTexts())
|
||||
}
|
||||
second.setProbe(false)
|
||||
waitFor(t, "queued prompt delivered", func() bool { return second.promptCount() == 1 })
|
||||
if texts := second.promptTexts(); texts[0] != "then ship it" {
|
||||
t.Fatalf("wrong prompt delivered after restart: %v", texts)
|
||||
}
|
||||
waitFor(t, "reply notified once", func() bool { return len(replyNotes(t, path)) == 1 })
|
||||
}
|
||||
|
||||
// A question pending when the server stopped is observed again after the
|
||||
// restart with no client attached, and a client that opens the chat sees it.
|
||||
func TestChatPendingQuestionSurvivesRestart(t *testing.T) {
|
||||
fastReattach(t)
|
||||
path := restartChatDB(t)
|
||||
question := protocol.ChatPermissionFrame{SessionID: "ses_1", RequestID: "req_1", Action: "bash", Resources: []string{"make release"}}
|
||||
first := newRestartChat()
|
||||
first.perm = []protocol.ChatPermissionFrame{question}
|
||||
server := startChatRestartServer(t, first, path)
|
||||
conn := dialChatConn(t, server.ctx, server.srv)
|
||||
sendPrompt(t, server.ctx, conn, "p1", "release", "op-1")
|
||||
first.push(stepFrameFor(protocol.ChatStepStarted, "turn_1"))
|
||||
waitFor(t, "question surfaced", func() bool { return len(server.handler.hub.pendingFrames("ses_1")) == 1 })
|
||||
server.shutdown()
|
||||
|
||||
second := newRestartChat()
|
||||
second.setProbe(true)
|
||||
second.perm = []protocol.ChatPermissionFrame{question}
|
||||
restarted := startChatRestartServer(t, second, path)
|
||||
waitFor(t, "question observed after restart", func() bool { return len(restarted.handler.hub.pendingFrames("ses_1")) == 1 })
|
||||
// A client opening the chat after the restart is shown the question and
|
||||
// its answer reaches the harness.
|
||||
phone := dialChatConn(t, restarted.ctx, restarted.srv)
|
||||
writeChat(t, restarted.ctx, phone, protocol.Message{Type: protocol.ChatSubscribe, ID: "s1", ChatSubscribe: &protocol.ChatQuery{SessionID: "ses_1"}})
|
||||
readUntil(t, restarted.ctx, phone, protocol.ChatSubscribe)
|
||||
if pending := nextPermission(t, restarted.ctx, phone, "req_1"); pending.Resolved {
|
||||
t.Fatalf("question resolved by the restart: %+v", pending)
|
||||
}
|
||||
writeChat(t, restarted.ctx, phone, protocol.Message{Type: protocol.ChatReply, ID: "a1", ChatReply: &protocol.ChatReplyRequest{SessionID: "ses_1", RequestID: "req_1", Decision: protocol.ChatDecisionOnce, Client: "Phone"}})
|
||||
readUntil(t, restarted.ctx, phone, protocol.ChatReply)
|
||||
waitFor(t, "answer delivered to the harness", func() bool { return second.replyCount() == 1 })
|
||||
}
|
||||
|
||||
// Nothing was in flight: nothing is recorded and nothing is observed.
|
||||
func TestChatIdleShutdownRecordsNothing(t *testing.T) {
|
||||
fastReattach(t)
|
||||
path := restartChatDB(t)
|
||||
server := startChatRestartServer(t, newRestartChat(), path)
|
||||
server.shutdown()
|
||||
live, err := openQueueStore(t, path).TakeChatLiveSessions(context.Background())
|
||||
if err != nil || len(live) != 0 {
|
||||
t.Fatalf("idle shutdown recorded live chats: %+v %v", live, err)
|
||||
}
|
||||
}
|
||||
@@ -54,6 +54,9 @@ type chatHub struct {
|
||||
// queueStore persists prompts queued behind a running turn; nil keeps the
|
||||
// queue in memory only (tests and store-less servers).
|
||||
queueStore chatPromptQueueStore
|
||||
// liveStore records the chats a stopping server leaves live on their
|
||||
// harness (chat_detach.go); nil reattaches nothing beyond the start probe.
|
||||
liveStore chatLiveStore
|
||||
// recentOps/recentOpOrder are the bounded record of accepted prompt
|
||||
// operation ids, so a client retry is acknowledged instead of re-sent.
|
||||
recentOps map[string]struct{}
|
||||
|
||||
@@ -183,6 +183,28 @@ func (m *chatManager) Close() {
|
||||
closeHarness(claudeClient)
|
||||
}
|
||||
|
||||
// DetachSessions is the shutdown counterpart of Close for a server that keeps
|
||||
// chat sessions running across a restart (see chat_detach.go). Each harness
|
||||
// that implements chatSessionDetacher releases its handles without stopping a
|
||||
// session; one that does not is closed as before. The OpenCode connection's
|
||||
// lifetime belongs to the caller (cmd/pipelines-server), so it is only detached.
|
||||
func (m *chatManager) DetachSessions() {
|
||||
m.mu.Lock()
|
||||
m.closed = true
|
||||
codexClient, claudeClient := m.codex, m.claude
|
||||
m.mu.Unlock()
|
||||
if detacher, ok := m.opencode.(chatSessionDetacher); ok {
|
||||
detacher.DetachSessions()
|
||||
}
|
||||
for _, client := range []chatBroker{codexClient, claudeClient} {
|
||||
if detacher, ok := client.(chatSessionDetacher); ok {
|
||||
detacher.DetachSessions()
|
||||
continue
|
||||
}
|
||||
closeHarness(client)
|
||||
}
|
||||
}
|
||||
|
||||
// Backends lists the harnesses available for a new session.
|
||||
func (m *chatManager) Backends() []string {
|
||||
backends := []string{}
|
||||
|
||||
@@ -0,0 +1,385 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"syscall"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/coder/websocket"
|
||||
"github.com/coder/websocket/wsjson"
|
||||
|
||||
"git.a-wan.net/alan/pipelines/internal/pipeline"
|
||||
"git.a-wan.net/alan/pipelines/internal/protocol"
|
||||
)
|
||||
|
||||
// TestRestartKeepsRealOpenCodeSessions is the opt-in end-to-end restart check
|
||||
// against the installed OpenCode binary and a real provider:
|
||||
//
|
||||
// PIPELINES_RESTART_LIVE=1 go test ./internal/server -run TestRestartKeepsRealOpenCodeSessions -v -timeout 15m
|
||||
//
|
||||
// It builds pipelines-server, runs it on a scratch HOME/XDG tree, a scratch
|
||||
// database and a free loopback port (with only the operator's OpenCode
|
||||
// provider login copied in), starts a chat turn and a one-stage run that each
|
||||
// run `sleep 30`, restarts the server mid-turn, and checks both finish
|
||||
// instead of being aborted. Every scratch process is killed afterwards.
|
||||
func TestRestartKeepsRealOpenCodeSessions(t *testing.T) {
|
||||
if os.Getenv("PIPELINES_RESTART_LIVE") != "1" {
|
||||
t.Skip("set PIPELINES_RESTART_LIVE=1 to run the real-OpenCode restart check")
|
||||
}
|
||||
realHome, err := os.UserHomeDir()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
root, err := os.MkdirTemp("", "pipelines-restart-live-")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Logf("scratch root %s", root)
|
||||
home := filepath.Join(root, "home")
|
||||
env := scratchEnvironment(t, realHome, home)
|
||||
binary := filepath.Join(root, "pipelines-server")
|
||||
build := exec.Command("go", "build", "-o", binary, "./cmd/pipelines-server")
|
||||
build.Dir = filepath.Join("..", "..")
|
||||
if out, err := build.CombinedOutput(); err != nil {
|
||||
t.Fatalf("build: %v\n%s", err, out)
|
||||
}
|
||||
// Chat creation activates the manager skill, so install the current
|
||||
// source's skills into the scratch harness roots.
|
||||
skills := exec.Command("go", "run", "./cmd/pipelines", "skill", "install")
|
||||
skills.Dir, skills.Env = filepath.Join("..", ".."), env
|
||||
if out, err := skills.CombinedOutput(); err != nil {
|
||||
t.Fatalf("skill install: %v\n%s", err, out)
|
||||
}
|
||||
db := filepath.Join(root, "data", "pipelines.db")
|
||||
if err := os.MkdirAll(filepath.Dir(db), 0700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
store, err := pipeline.Open(db)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := store.Submit(context.Background(), "restart-live", restartLiveStage); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
items, _ := store.List(context.Background())
|
||||
store.Close()
|
||||
addr := freeLoopback(t)
|
||||
logPath := filepath.Join(root, "server.log")
|
||||
server := startScratchServer(t, binary, db, addr, env, logPath)
|
||||
t.Cleanup(func() {
|
||||
server.stop()
|
||||
killScratchServices(t, home)
|
||||
})
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 12*time.Minute)
|
||||
defer cancel()
|
||||
|
||||
chatDir := filepath.Join(root, "chat-work")
|
||||
runDir := filepath.Join(root, "run-work")
|
||||
for _, dir := range []string{chatDir, runDir} {
|
||||
if err := os.MkdirAll(dir, 0700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
client := dialLive(t, ctx, addr)
|
||||
created := client.request(t, ctx, protocol.Message{Type: protocol.ChatCreate, ID: "c1", ChatCreate: &protocol.ChatCreateRequest{Title: "restart live", Directory: chatDir, OperationID: "create-1"}}, protocol.ChatCreate)
|
||||
if created.ChatSession == nil {
|
||||
t.Fatalf("chat_create = %+v", created)
|
||||
}
|
||||
sessionID := created.ChatSession.ID
|
||||
prompted := time.Now()
|
||||
client.request(t, ctx, protocol.Message{Type: protocol.ChatPrompt, ID: "p1", ChatPrompt: &protocol.ChatPromptRequest{SessionID: sessionID, OperationID: "op-1", Client: "restart-live",
|
||||
Text: "Use your shell tool to run exactly this command once: sleep 30 . Wait for it to finish. Then reply with exactly the text RESTART-CHAT-OK and nothing else."}}, protocol.ChatPrompt)
|
||||
started := client.request(t, ctx, protocol.Message{Type: protocol.RunPipeline, ID: "r1", PipelineID: items[0].ID, Version: 1, RequestKey: "restart-live", Workspace: runDir, Input: map[string]any{"prompt": "restart live check"}}, protocol.RunStarted)
|
||||
if started.RunDetail == nil {
|
||||
t.Fatalf("run_pipeline = %+v", started)
|
||||
}
|
||||
runID := started.RunDetail.ID
|
||||
// Wait until both harness sessions are executing the sleep.
|
||||
waitLive(t, ctx, "chat shell running", func() bool { return chatShellStatus(t, ctx, addr, sessionID) == "running" })
|
||||
var stageSession string
|
||||
waitLive(t, ctx, "stage session admitted", func() bool {
|
||||
d := getLiveRun(t, ctx, addr, runID)
|
||||
if len(d.Steps) == 1 && d.Steps[0].SessionID != "" {
|
||||
stageSession = d.Steps[0].SessionID
|
||||
return true
|
||||
}
|
||||
return false
|
||||
})
|
||||
time.Sleep(8 * time.Second) // the stage's shell is inside its sleep
|
||||
client.close()
|
||||
|
||||
t.Log("restarting the scratch server mid-turn")
|
||||
server.stop()
|
||||
server = startScratchServer(t, binary, db, addr, env, logPath)
|
||||
|
||||
var reply string
|
||||
waitLiveFor(t, ctx, 5*time.Minute, "chat reply after restart", func() bool {
|
||||
reply = chatReply(t, ctx, addr, sessionID)
|
||||
return strings.Contains(reply, "RESTART-CHAT-OK")
|
||||
})
|
||||
if status := chatShellStatus(t, ctx, addr, sessionID); status != "completed" {
|
||||
t.Fatalf("chat shell was not completed after the restart: %q", status)
|
||||
}
|
||||
if elapsed := time.Since(prompted); elapsed < 30*time.Second {
|
||||
t.Fatalf("chat reply arrived before the sleep could finish (%s): the turn was cut short", elapsed)
|
||||
}
|
||||
var final *protocol.RunDetail
|
||||
waitLiveFor(t, ctx, 6*time.Minute, "run finished after restart", func() bool {
|
||||
final = getLiveRun(t, ctx, addr, runID)
|
||||
return final.FinishedAt != ""
|
||||
})
|
||||
if final.Status != "succeeded" || len(final.Steps) != 1 || final.Steps[0].Status != "completed" || final.Steps[0].SessionID != stageSession {
|
||||
t.Fatalf("run did not continue on its own session: status=%s diagnostic=%q steps=%+v", final.Status, final.Diagnostic, final.Steps)
|
||||
}
|
||||
logData, _ := os.ReadFile(logPath)
|
||||
for _, want := range []string{"leaving 1 chat session(s) running", "left running for the next start to reattach", "reattached stage one"} {
|
||||
if !bytes.Contains(logData, []byte(want)) {
|
||||
t.Errorf("server log lacks %q", want)
|
||||
}
|
||||
}
|
||||
t.Logf("chat reply %q; run %d %s on session %s", reply, runID, final.Status, stageSession)
|
||||
}
|
||||
|
||||
const restartLiveStage = "# stage one\n\n```prompt\nUse your shell tool to run exactly this command once: sleep 30 . Wait for it to finish. Then write the turn output: a short body saying RESTART-STAGE-OK, with the outcome success.\n```\n\n```tools\nshell\n```\n\n```edges\nsuccess -> #success\nerror -> #failure\n```\n\n```configs\ntype=model\nprovider=opencode-go\nmodel=deepseek-v4.1-flash\nretries=0\ntimeout=15m\n```\n"
|
||||
|
||||
// scratchEnvironment is the server's environment: a private HOME/XDG tree with
|
||||
// only the operator's OpenCode provider login and a minimal configuration.
|
||||
func scratchEnvironment(t *testing.T, realHome, home string) []string {
|
||||
t.Helper()
|
||||
config, data, cache := filepath.Join(home, ".config"), filepath.Join(home, ".local", "share"), filepath.Join(home, ".cache")
|
||||
for _, dir := range []string{filepath.Join(config, "opencode"), filepath.Join(data, "opencode"), cache} {
|
||||
if err := os.MkdirAll(dir, 0700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
auth, err := os.ReadFile(filepath.Join(realHome, ".local", "share", "opencode", "auth.json"))
|
||||
if err != nil {
|
||||
t.Skipf("no OpenCode login to copy: %v", err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(data, "opencode", "auth.json"), auth, 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// A cold model catalog can leave the first service unable to resolve the
|
||||
// model; seed the public catalog cache (not a credential).
|
||||
if catalog, err := os.ReadFile(filepath.Join(realHome, ".cache", "opencode", "models.json")); err == nil {
|
||||
if err := os.MkdirAll(filepath.Join(cache, "opencode"), 0700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(cache, "opencode", "models.json"), catalog, 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
settings := `{"$schema":"https://opencode.ai/config.json","model":"opencode-go/deepseek-v4.1-flash","permissions":[{"action":"shell","resource":"*","effect":"allow"}]}`
|
||||
if err := os.WriteFile(filepath.Join(config, "opencode", "opencode.json"), []byte(settings), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var env []string
|
||||
for _, entry := range os.Environ() {
|
||||
key, _, _ := strings.Cut(entry, "=")
|
||||
switch {
|
||||
case key == "HOME", strings.HasPrefix(key, "XDG_"), strings.HasPrefix(key, "PIPELINES_"), strings.HasPrefix(key, "OPENCODE_"):
|
||||
continue
|
||||
}
|
||||
env = append(env, entry)
|
||||
}
|
||||
return append(env, "HOME="+home, "XDG_CONFIG_HOME="+config, "XDG_DATA_HOME="+data, "XDG_CACHE_HOME="+cache, "XDG_STATE_HOME="+filepath.Join(home, ".local", "state"),
|
||||
"PIPELINES_OPENCODE="+opencodePath(), "PIPELINES_NTFY_URL=", "PIPELINES_MDNS=off")
|
||||
}
|
||||
|
||||
func opencodePath() string {
|
||||
if path, err := exec.LookPath("opencode"); err == nil {
|
||||
return path
|
||||
}
|
||||
return "/opt/homebrew/bin/opencode"
|
||||
}
|
||||
|
||||
type scratchServer struct {
|
||||
t *testing.T
|
||||
cmd *exec.Cmd
|
||||
}
|
||||
|
||||
func startScratchServer(t *testing.T, binary, db, addr string, env []string, logPath string) *scratchServer {
|
||||
t.Helper()
|
||||
logFile, err := os.OpenFile(logPath, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0600)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cmd := exec.Command(binary, "-db", db, "-addr", addr, "-mdns=false")
|
||||
cmd.Env, cmd.Stdout, cmd.Stderr = env, logFile, logFile
|
||||
if err := cmd.Start(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
logFile.Close()
|
||||
s := &scratchServer{t: t, cmd: cmd}
|
||||
deadline := time.Now().Add(90 * time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
if resp, err := http.Get("http://" + addr + "/healthz"); err == nil {
|
||||
resp.Body.Close()
|
||||
if resp.StatusCode == http.StatusOK {
|
||||
return s
|
||||
}
|
||||
}
|
||||
time.Sleep(200 * time.Millisecond)
|
||||
}
|
||||
s.stop()
|
||||
data, _ := os.ReadFile(logPath)
|
||||
t.Fatalf("scratch server did not become healthy:\n%s", data)
|
||||
return nil
|
||||
}
|
||||
|
||||
// stop is a service stop/restart: SIGTERM, then wait for a clean exit.
|
||||
func (s *scratchServer) stop() {
|
||||
if s == nil || s.cmd == nil || s.cmd.Process == nil || s.cmd.ProcessState != nil {
|
||||
return
|
||||
}
|
||||
_ = s.cmd.Process.Signal(syscall.SIGTERM)
|
||||
done := make(chan struct{})
|
||||
go func() { _ = s.cmd.Wait(); close(done) }()
|
||||
select {
|
||||
case <-done:
|
||||
case <-time.After(60 * time.Second):
|
||||
_ = s.cmd.Process.Kill()
|
||||
<-done
|
||||
s.t.Error("scratch server did not stop within 60s of SIGTERM")
|
||||
}
|
||||
}
|
||||
|
||||
// killScratchServices stops the detached OpenCode services the scratch server
|
||||
// owned (their pids are in its private registrations).
|
||||
func killScratchServices(t *testing.T, home string) {
|
||||
for _, name := range []string{"opencode-stage-service.json", "opencode-chat-service.json"} {
|
||||
data, err := os.ReadFile(filepath.Join(home, "Library", "Caches", "pipelines", "private", name))
|
||||
if err != nil {
|
||||
data, err = os.ReadFile(filepath.Join(home, ".cache", "pipelines", "private", name))
|
||||
}
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
var state struct{ PID int }
|
||||
if json.Unmarshal(data, &state) == nil && state.PID > 0 {
|
||||
_ = syscall.Kill(state.PID, syscall.SIGTERM)
|
||||
time.Sleep(time.Second)
|
||||
_ = syscall.Kill(state.PID, syscall.SIGKILL)
|
||||
t.Logf("stopped scratch %s (pid %d)", strings.TrimSuffix(name, ".json"), state.PID)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func freeLoopback(t *testing.T) string {
|
||||
t.Helper()
|
||||
listener, err := net.Listen("tcp", "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer listener.Close()
|
||||
return "127.0.0.1:" + strconv.Itoa(listener.Addr().(*net.TCPAddr).Port)
|
||||
}
|
||||
|
||||
type liveClient struct{ conn *websocket.Conn }
|
||||
|
||||
func dialLive(t *testing.T, ctx context.Context, addr string) *liveClient {
|
||||
t.Helper()
|
||||
conn, _, err := websocket.Dial(ctx, "ws://"+addr+"/connect", nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
conn.SetReadLimit(16 << 20)
|
||||
return &liveClient{conn: conn}
|
||||
}
|
||||
|
||||
func (c *liveClient) close() { c.conn.CloseNow() }
|
||||
|
||||
func (c *liveClient) request(t *testing.T, ctx context.Context, msg protocol.Message, want string) protocol.Message {
|
||||
t.Helper()
|
||||
if err := wsjson.Write(ctx, c.conn, msg); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
read, cancel := context.WithTimeout(ctx, 2*time.Minute)
|
||||
defer cancel()
|
||||
for {
|
||||
var reply protocol.Message
|
||||
if err := wsjson.Read(read, c.conn, &reply); err != nil {
|
||||
t.Fatalf("waiting for %s: %v", want, err)
|
||||
}
|
||||
if reply.ID != msg.ID {
|
||||
continue
|
||||
}
|
||||
if reply.Type == protocol.Error {
|
||||
t.Fatalf("%s refused: %s", msg.Type, reply.Error)
|
||||
}
|
||||
if reply.Type == want {
|
||||
return reply
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func liveQuery(t *testing.T, ctx context.Context, addr string, msg protocol.Message, want string) protocol.Message {
|
||||
t.Helper()
|
||||
c := dialLive(t, ctx, addr)
|
||||
defer c.close()
|
||||
return c.request(t, ctx, msg, want)
|
||||
}
|
||||
|
||||
func getLiveRun(t *testing.T, ctx context.Context, addr string, id int64) *protocol.RunDetail {
|
||||
t.Helper()
|
||||
reply := liveQuery(t, ctx, addr, protocol.Message{Type: protocol.GetRun, ID: "g", RunID: id}, protocol.Run)
|
||||
if reply.RunDetail == nil {
|
||||
t.Fatalf("get_run = %+v", reply)
|
||||
}
|
||||
return reply.RunDetail
|
||||
}
|
||||
|
||||
func liveMessages(t *testing.T, ctx context.Context, addr, session string) []protocol.ChatMessage {
|
||||
t.Helper()
|
||||
reply := liveQuery(t, ctx, addr, protocol.Message{Type: protocol.ChatMessages, ID: "m", ChatQuery: &protocol.ChatQuery{SessionID: session, Limit: 50}}, protocol.ChatMessages)
|
||||
if reply.ChatHistory != nil {
|
||||
return reply.ChatHistory.Messages
|
||||
}
|
||||
return reply.ChatMessages
|
||||
}
|
||||
|
||||
// chatShellStatus reports the status of the newest shell tool call.
|
||||
func chatShellStatus(t *testing.T, ctx context.Context, addr, session string) string {
|
||||
status := ""
|
||||
for _, message := range liveMessages(t, ctx, addr, session) {
|
||||
for _, part := range message.Parts {
|
||||
if part.Type == protocol.ChatPartTool && part.State != nil && strings.Contains(strings.ToLower(part.Name+string(part.State.Input)), "sleep") {
|
||||
status = part.State.Status
|
||||
}
|
||||
}
|
||||
}
|
||||
return status
|
||||
}
|
||||
|
||||
func chatReply(t *testing.T, ctx context.Context, addr, session string) string {
|
||||
_, text := lastReply(liveMessages(t, ctx, addr, session))
|
||||
return text
|
||||
}
|
||||
|
||||
func waitLive(t *testing.T, ctx context.Context, what string, check func() bool) {
|
||||
t.Helper()
|
||||
waitLiveFor(t, ctx, 3*time.Minute, what, check)
|
||||
}
|
||||
|
||||
func waitLiveFor(t *testing.T, ctx context.Context, limit time.Duration, what string, check func() bool) {
|
||||
t.Helper()
|
||||
deadline := time.Now().Add(limit)
|
||||
for time.Now().Before(deadline) && ctx.Err() == nil {
|
||||
if check() {
|
||||
return
|
||||
}
|
||||
time.Sleep(2 * time.Second)
|
||||
}
|
||||
t.Fatal(fmt.Sprintf("timed out waiting for %s", what))
|
||||
}
|
||||
@@ -0,0 +1,220 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log"
|
||||
"time"
|
||||
|
||||
"git.a-wan.net/alan/pipelines/internal/opencode"
|
||||
"git.a-wan.net/alan/pipelines/internal/pipeline"
|
||||
"git.a-wan.net/alan/pipelines/internal/protocol"
|
||||
"git.a-wan.net/alan/pipelines/internal/stage"
|
||||
)
|
||||
|
||||
// Runs survive a server restart. With detach enabled (the installed server),
|
||||
// a stopping server cancels each run's context with stage.ErrServerShutdown:
|
||||
// stage clients leave their admitted harness session running and owned, and the
|
||||
// loop returns without settling anything. The next process plans, before it
|
||||
// serves requests, how to continue every run it finds in flight:
|
||||
//
|
||||
// - a running visit with a live OpenCode V2 session is re-adopted: the loop
|
||||
// re-enters that visit, attaches to the session, waits for it to settle (or
|
||||
// reads the result it reached while the server was down) and continues the
|
||||
// graph as if it had watched it;
|
||||
// - a visit begun but never handed to a harness is marked interrupted and the
|
||||
// same stage starts again (nothing was admitted, so nothing replays);
|
||||
// - a run between visits continues with its next stage;
|
||||
// - a session whose harness process is gone, or that belongs to a harness the
|
||||
// server owned as a child (Pi, OpenCode V1), falls back to the previous
|
||||
// behaviour: the run is interrupted, with the reason in its diagnostic.
|
||||
//
|
||||
// Only an operator cancel (cancel_run) stops a stage session.
|
||||
|
||||
type reattachKind int
|
||||
|
||||
const (
|
||||
reattachAdopt reattachKind = iota
|
||||
reattachRestart
|
||||
reattachBetween
|
||||
)
|
||||
|
||||
// reattachPlan is one run start-up recovery kept for reattachRuns.
|
||||
type reattachPlan struct {
|
||||
runID int64
|
||||
kind reattachKind
|
||||
// step is the running visit (adopt/restart) or the last completed one.
|
||||
step protocol.RunStep
|
||||
owner pipeline.SessionOwner
|
||||
}
|
||||
|
||||
// adoptedVisit is the visit a runLoop re-adopts instead of starting.
|
||||
type adoptedVisit struct {
|
||||
sequence int
|
||||
owner pipeline.SessionOwner
|
||||
}
|
||||
|
||||
// stageServiceAlive reports whether a retained stage session's harness service
|
||||
// is still running. It is a seam for tests.
|
||||
var stageServiceAlive = func(ctx context.Context, owner pipeline.SessionOwner) error {
|
||||
return opencode.ReferenceAlive(ctx, owner.Reference)
|
||||
}
|
||||
|
||||
// interruptRecoveredSession stops a previously active session that start-up
|
||||
// recovery does not re-adopt. It is a seam for tests.
|
||||
var interruptRecoveredSession = func(ctx context.Context, owner pipeline.SessionOwner) error {
|
||||
if owner.Backend == "pi" {
|
||||
return stage.InterruptPi(ctx, owner.Reference)
|
||||
}
|
||||
return (stage.OpenCodeClient{Backend: owner.Backend, Directory: owner.Directory, Reference: owner.Reference}).InterruptAndConfirm(ctx, owner.ID)
|
||||
}
|
||||
|
||||
// runContext derives one run's execution context. With detach, a server
|
||||
// shutdown cancels it with stage.ErrServerShutdown as the cause, while the
|
||||
// returned cancel (an operator cancel or the run's own end) uses the ordinary
|
||||
// context.Canceled, so stage clients can tell the two apart.
|
||||
func (r *runner) runContext() (context.Context, context.CancelFunc) {
|
||||
if !r.detach {
|
||||
return context.WithCancel(r.ctx)
|
||||
}
|
||||
ctx, cancel := context.WithCancelCause(context.WithoutCancel(r.ctx))
|
||||
stop := context.AfterFunc(r.ctx, func() { cancel(stage.ErrServerShutdown) })
|
||||
return ctx, func() { stop(); cancel(context.Canceled) }
|
||||
}
|
||||
|
||||
// planReattach decides, per in-flight run, whether start-up continues it.
|
||||
// Runs it does not plan keep today's recovery (interrupted); reasons carries the
|
||||
// explanation for the ones that could not be reattached.
|
||||
// gone lists runs whose session's service is not running: there is nothing
|
||||
// left to interrupt, so recovery only clears their ownership.
|
||||
func planReattach(ctx context.Context, s *pipeline.Store) ([]reattachPlan, map[int64]string, map[int64]bool, error) {
|
||||
reasons, gone := map[int64]string{}, map[int64]bool{}
|
||||
rows, err := s.ListRuns(ctx)
|
||||
if err != nil {
|
||||
return nil, nil, nil, err
|
||||
}
|
||||
var plans []reattachPlan
|
||||
for _, row := range rows {
|
||||
if row.Status != protocol.RunStatusRunning || row.CancelRequested || row.CleanupUnconfirmed {
|
||||
continue
|
||||
}
|
||||
detail, err := s.GetRun(ctx, row.ID)
|
||||
if err != nil {
|
||||
return nil, nil, nil, err
|
||||
}
|
||||
if len(detail.Steps) == 0 {
|
||||
continue
|
||||
}
|
||||
last := detail.Steps[0]
|
||||
for _, step := range detail.Steps {
|
||||
if step.Sequence > last.Sequence {
|
||||
last = step
|
||||
}
|
||||
}
|
||||
switch last.Status {
|
||||
case protocol.StepStatusRunning:
|
||||
if last.SessionID == "" {
|
||||
plans = append(plans, reattachPlan{runID: row.ID, kind: reattachRestart, step: last})
|
||||
continue
|
||||
}
|
||||
owner, err := s.RunSessionOwner(ctx, row.ID, last.Sequence)
|
||||
if err != nil {
|
||||
return nil, nil, nil, err
|
||||
}
|
||||
owner.RunID = row.ID
|
||||
if owner.Backend != "v2" {
|
||||
name := "OpenCode V1"
|
||||
if owner.Backend == "pi" {
|
||||
name = "Pi"
|
||||
}
|
||||
reasons[row.ID] = fmt.Sprintf("%s stage sessions belong to the stopped server process, so stage %s could not be reattached", name, last.Stage)
|
||||
continue
|
||||
}
|
||||
probe, cancel := context.WithTimeout(ctx, 5*time.Second)
|
||||
aliveErr := stageServiceAlive(probe, owner)
|
||||
cancel()
|
||||
if aliveErr != nil {
|
||||
gone[row.ID] = errors.Is(aliveErr, opencode.ErrServiceNotRunning)
|
||||
reasons[row.ID] = fmt.Sprintf("the OpenCode service running stage %s was gone (%v), so its session could not be reattached", last.Stage, aliveErr)
|
||||
continue
|
||||
}
|
||||
plans = append(plans, reattachPlan{runID: row.ID, kind: reattachAdopt, step: last, owner: owner})
|
||||
case "completed":
|
||||
plans = append(plans, reattachPlan{runID: row.ID, kind: reattachBetween, step: last})
|
||||
}
|
||||
}
|
||||
return plans, reasons, gone, nil
|
||||
}
|
||||
|
||||
// reattachRuns continues every run start-up recovery kept. It is called once,
|
||||
// after the runner's collaborators are wired and before requests are served.
|
||||
func (r *runner) reattachRuns() {
|
||||
plans := r.reattach
|
||||
r.reattach = nil
|
||||
for _, plan := range plans {
|
||||
r.reattachRun(plan)
|
||||
}
|
||||
}
|
||||
|
||||
func (r *runner) reattachRun(plan reattachPlan) {
|
||||
id := plan.runID
|
||||
ctx, cancel := context.WithTimeout(r.ctx, 10*time.Second)
|
||||
defer cancel()
|
||||
if plan.kind == reattachRestart {
|
||||
if err := r.store.RestartStep(ctx, id, plan.step.Sequence, "Server restarted before this stage handed work to a harness; the stage was started again"); err != nil {
|
||||
log.Printf("run %d: restart stage %s after server restart: %v", id, plan.step.Stage, err)
|
||||
}
|
||||
}
|
||||
detail, err := r.store.GetRun(ctx, id)
|
||||
if err != nil {
|
||||
log.Printf("run %d: read for reattach: %v", id, err)
|
||||
return
|
||||
}
|
||||
r.rememberRunIdentity(id, identityOfRunSummary(detail.RunSummary))
|
||||
workspace := detail.Workspace
|
||||
request := protocol.Message{WritePaths: detail.WritePaths, AllowTools: detail.AllowTools, Model: detail.Model, StageModels: detail.StageModels}
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
switch plan.kind {
|
||||
case reattachBetween:
|
||||
log.Printf("run %d: continuing after stage %s following a server restart", id, plan.step.Stage)
|
||||
r.launch(id, workspace, func(ctx context.Context) { r.resume(ctx, id, detail, workspace, plan.step) })
|
||||
default:
|
||||
state := rebuildAdoptState(detail, plan)
|
||||
if plan.kind == reattachAdopt {
|
||||
log.Printf("run %d: reattached stage %s #%d session %s after a server restart", id, plan.step.Stage, plan.step.Sequence, plan.owner.ID)
|
||||
} else {
|
||||
log.Printf("run %d: starting stage %s again after a server restart", id, plan.step.Stage)
|
||||
}
|
||||
r.launch(id, workspace, func(ctx context.Context) {
|
||||
r.runLoop(ctx, id, detail.Graph, detail.Name, workspace, request, state)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// rebuildAdoptState restores the loop state at the planned visit: memory and
|
||||
// visit counts from every other persisted step, the handoff input from the
|
||||
// visit before it, and (for an adopted visit) the session to re-adopt.
|
||||
func rebuildAdoptState(detail *protocol.RunDetail, plan reattachPlan) *runState {
|
||||
decided := protocol.RunStep{Target: plan.step.Stage, Output: detail.Input}
|
||||
previous := 0
|
||||
for _, step := range detail.Steps {
|
||||
if step.Sequence < plan.step.Sequence && step.Sequence > previous {
|
||||
previous = step.Sequence
|
||||
decided = step
|
||||
decided.Target = plan.step.Stage
|
||||
}
|
||||
}
|
||||
state := rebuildRunState(detail, decided)
|
||||
state.key = plan.step.Stage
|
||||
// The interrupted or adopted visit was counted; the loop counts it again.
|
||||
if state.visits[plan.step.Stage] > 0 {
|
||||
state.visits[plan.step.Stage]--
|
||||
}
|
||||
if plan.kind == reattachAdopt {
|
||||
state.nextSequence = plan.step.Sequence
|
||||
state.adopt = &adoptedVisit{sequence: plan.step.Sequence, owner: plan.owner}
|
||||
}
|
||||
return state
|
||||
}
|
||||
@@ -0,0 +1,262 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.a-wan.net/alan/pipelines/internal/opencode"
|
||||
"git.a-wan.net/alan/pipelines/internal/pipeline"
|
||||
"git.a-wan.net/alan/pipelines/internal/protocol"
|
||||
"git.a-wan.net/alan/pipelines/internal/stage"
|
||||
)
|
||||
|
||||
// restartFixture runs a two-stage pipeline on a detaching runner, stops that
|
||||
// "server" while stage one is in flight, and reopens the store for a second
|
||||
// runner, the way a restart does.
|
||||
type restartFixture struct {
|
||||
path, root, workspace string
|
||||
runID int64
|
||||
}
|
||||
|
||||
// stopMidStage starts the run and shuts the first runner down while stage one
|
||||
// is executing. firstStage is the first process's executor for stage one; it
|
||||
// must block until its context ends.
|
||||
func stopMidStage(t *testing.T, firstStage func(ctx context.Context, s *pipeline.Store, id int64) (stage.Result, error)) restartFixture {
|
||||
t.Helper()
|
||||
f := restartFixture{path: filepath.Join(t.TempDir(), "restart.db"), root: t.TempDir(), workspace: t.TempDir()}
|
||||
s, err := pipeline.Open(f.path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ctx, shutdown := context.WithCancel(context.Background())
|
||||
if _, err := s.Submit(ctx, "restart", pipeline.Template([]string{"one", "two"}, true)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
items, _ := s.List(ctx)
|
||||
r, err := newRunnerReattaching(ctx, s, f.root, true)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
r.exportSession = validGateExport
|
||||
entered := make(chan struct{})
|
||||
var id atomic.Int64
|
||||
r.execute = func(ctx context.Context, def stage.Definition, _ stage.Values) (stage.Result, error) {
|
||||
if def.Key != "one" {
|
||||
t.Errorf("the first process reached stage %s", def.Key)
|
||||
return stage.Result{}, fmt.Errorf("unexpected stage")
|
||||
}
|
||||
for id.Load() == 0 {
|
||||
time.Sleep(time.Millisecond)
|
||||
}
|
||||
close(entered)
|
||||
return firstStage(ctx, s, id.Load())
|
||||
}
|
||||
d, err := r.start(protocol.Message{PipelineID: items[0].ID, Version: 1, RequestKey: "restart", Workspace: f.workspace, Input: map[string]any{"prompt": "task"}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
id.Store(d.ID)
|
||||
f.runID = d.ID
|
||||
<-entered
|
||||
shutdown() // SIGTERM
|
||||
r.wg.Wait()
|
||||
r.schedulerWG.Wait()
|
||||
persisted, err := s.GetRun(context.Background(), f.runID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if persisted.Status != protocol.RunStatusRunning || len(persisted.Steps) != 1 || persisted.Steps[0].Status != protocol.StepStatusRunning {
|
||||
t.Fatalf("shutdown settled the run instead of leaving it in flight: %+v", persisted)
|
||||
}
|
||||
if err := s.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return f
|
||||
}
|
||||
|
||||
// restart opens the second process's runner on the same database.
|
||||
func (f restartFixture) restart(t *testing.T, execute func(context.Context, *pipeline.Store, stage.Definition) (stage.Result, error)) (*runner, *pipeline.Store) {
|
||||
t.Helper()
|
||||
s, err := pipeline.Open(f.path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
r, err := newRunnerReattaching(ctx, s, f.root, true)
|
||||
if err != nil {
|
||||
cancel()
|
||||
s.Close()
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { cancel(); r.wg.Wait(); r.schedulerWG.Wait(); s.Close() })
|
||||
r.exportSession = validGateExport
|
||||
r.execute = func(ctx context.Context, def stage.Definition, _ stage.Values) (stage.Result, error) {
|
||||
return execute(ctx, s, def)
|
||||
}
|
||||
return r, s
|
||||
}
|
||||
|
||||
func seamRestartRecovery(t *testing.T, alive error) *atomic.Int32 {
|
||||
t.Helper()
|
||||
interrupts := &atomic.Int32{}
|
||||
previousAlive, previousInterrupt := stageServiceAlive, interruptRecoveredSession
|
||||
stageServiceAlive = func(context.Context, pipeline.SessionOwner) error { return alive }
|
||||
interruptRecoveredSession = func(context.Context, pipeline.SessionOwner) error { interrupts.Add(1); return nil }
|
||||
t.Cleanup(func() { stageServiceAlive, interruptRecoveredSession = previousAlive, previousInterrupt })
|
||||
return interrupts
|
||||
}
|
||||
|
||||
// ownedLiveStage is a first-process stage one that admitted session
|
||||
// ses_live (recorded like a real client does) and is still running at shutdown.
|
||||
func ownedLiveStage(ctx context.Context, s *pipeline.Store, id int64) (stage.Result, error) {
|
||||
if err := s.RunSession(context.Background(), id, 1, "ses_live", true, "v2", "ref-live"); err != nil {
|
||||
return stage.Result{}, err
|
||||
}
|
||||
<-ctx.Done()
|
||||
if stage.Detached(ctx) {
|
||||
return stage.Result{}, stage.ErrDetached
|
||||
}
|
||||
return stage.Result{}, ctx.Err()
|
||||
}
|
||||
|
||||
// TestStageRunContinuesToNextStageAfterRestart: the stage in flight at a server
|
||||
// restart is re-adopted (never re-prompted, never interrupted), its result is
|
||||
// folded in, and the run continues with the next stage to success.
|
||||
func TestStageRunContinuesToNextStageAfterRestart(t *testing.T) {
|
||||
interrupts := seamRestartRecovery(t, nil)
|
||||
f := stopMidStage(t, ownedLiveStage)
|
||||
var resumed atomic.Value
|
||||
var secondRan atomic.Int32
|
||||
r, s := f.restart(t, func(ctx context.Context, s *pipeline.Store, def stage.Definition) (stage.Result, error) {
|
||||
if def.Key == "one" {
|
||||
resumed.Store(stage.TakeResumeSession(ctx))
|
||||
// The real client releases ownership once the session settled.
|
||||
if err := s.RunSession(context.Background(), f.runID, 1, "ses_live", false, "v2", "ref-live"); err != nil {
|
||||
return stage.Result{}, err
|
||||
}
|
||||
return stage.Result{Outcome: "success", Output: stage.Values{"markdown": "finished while the server was down"}}, nil
|
||||
}
|
||||
secondRan.Add(1)
|
||||
return stage.Result{Outcome: "success", Output: stage.Values{"markdown": "next stage"}}, nil
|
||||
})
|
||||
if interrupts.Load() != 0 {
|
||||
t.Fatal("start-up recovery interrupted a session that survived the restart")
|
||||
}
|
||||
r.reattachRuns()
|
||||
r.wg.Wait()
|
||||
d, err := s.GetRun(context.Background(), f.runID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got, _ := resumed.Load().(string); got != "ses_live" {
|
||||
t.Fatalf("stage one was not re-adopted from its session: %q", got)
|
||||
}
|
||||
if d.Status != "succeeded" || len(d.Steps) != 2 || secondRan.Load() != 1 {
|
||||
t.Fatalf("run did not continue to the next stage: %+v", d)
|
||||
}
|
||||
one := d.Steps[0]
|
||||
if one.Sequence != 1 || one.Status != "completed" || one.Output["markdown"] != "finished while the server was down" {
|
||||
t.Fatalf("adopted visit not folded in: %+v", one)
|
||||
}
|
||||
if one.FileChanges == nil || one.FileChanges.Status != "partial" || !strings.Contains(one.FileChanges.Diagnostic, "restart") {
|
||||
t.Fatalf("adopted visit claims a complete change capture: %+v", one.FileChanges)
|
||||
}
|
||||
}
|
||||
|
||||
// TestStageRunWithHarnessGoneFallsBackWithReason: when the harness process
|
||||
// is gone after the restart, the run is interrupted as before, naming why.
|
||||
func TestStageRunWithHarnessGoneFallsBackWithReason(t *testing.T) {
|
||||
interrupts := seamRestartRecovery(t, fmt.Errorf("%w at http://127.0.0.1:1", opencode.ErrServiceNotRunning))
|
||||
f := stopMidStage(t, ownedLiveStage)
|
||||
r, s := f.restart(t, func(context.Context, *pipeline.Store, stage.Definition) (stage.Result, error) {
|
||||
t.Error("a run whose harness was gone executed again")
|
||||
return stage.Result{}, nil
|
||||
})
|
||||
r.reattachRuns()
|
||||
r.wg.Wait()
|
||||
if interrupts.Load() != 0 {
|
||||
t.Fatal("recovery tried to interrupt a session on a service that is gone")
|
||||
}
|
||||
d, err := s.GetRun(context.Background(), f.runID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if d.Status != "interrupted" || !strings.Contains(d.Diagnostic, "was gone") || !strings.Contains(d.Steps[0].Diagnostic, "was gone") {
|
||||
t.Fatalf("harness-gone fallback lacks a clear reason: status=%s run=%q step=%q", d.Status, d.Diagnostic, d.Steps[0].Diagnostic)
|
||||
}
|
||||
if owners, _ := s.ActiveSessionOwners(context.Background()); len(owners) != 0 {
|
||||
t.Fatalf("a gone session is still recorded as active: %+v", owners)
|
||||
}
|
||||
}
|
||||
|
||||
// TestStageNotYetAdmittedStartsAgainAfterRestart: a visit the stopped server
|
||||
// had begun but not handed to any session is started again, nothing replayed.
|
||||
func TestStageNotYetAdmittedStartsAgainAfterRestart(t *testing.T) {
|
||||
seamRestartRecovery(t, nil)
|
||||
f := stopMidStage(t, func(ctx context.Context, _ *pipeline.Store, _ int64) (stage.Result, error) {
|
||||
<-ctx.Done()
|
||||
return stage.Result{}, ctx.Err()
|
||||
})
|
||||
var resumed atomic.Value
|
||||
resumed.Store("")
|
||||
r, s := f.restart(t, func(ctx context.Context, _ *pipeline.Store, def stage.Definition) (stage.Result, error) {
|
||||
if def.Key == "one" {
|
||||
resumed.Store(stage.TakeResumeSession(ctx))
|
||||
}
|
||||
return stage.Result{Outcome: "success", Output: stage.Values{"markdown": def.Key}}, nil
|
||||
})
|
||||
r.reattachRuns()
|
||||
r.wg.Wait()
|
||||
d, _ := s.GetRun(context.Background(), f.runID)
|
||||
if d.Status != "succeeded" || len(d.Steps) != 3 || d.Steps[0].Status != "interrupted" || d.Steps[1].Stage != "one" || d.Steps[1].Status != "completed" {
|
||||
t.Fatalf("unadmitted stage was not started again: %+v", d)
|
||||
}
|
||||
if got := resumed.Load().(string); got != "" {
|
||||
t.Fatalf("a fresh visit was treated as a re-adoption of %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestOperatorCancelStillStopsDetachingRun: only shutdown detaches; an
|
||||
// operator cancel on a detaching server still settles the run as canceled.
|
||||
func TestOperatorCancelStillStopsDetachingRun(t *testing.T) {
|
||||
s, err := pipeline.Open(filepath.Join(t.TempDir(), "cancel.db"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
if _, err := s.Submit(ctx, "cancel", pipeline.Template([]string{"one"}, true)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
items, _ := s.List(ctx)
|
||||
r, err := newRunnerReattaching(ctx, s, t.TempDir(), true)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() { cancel(); r.wg.Wait(); r.schedulerWG.Wait(); s.Close() }()
|
||||
entered := make(chan struct{})
|
||||
detached := &atomic.Bool{}
|
||||
r.execute = func(ctx context.Context, _ stage.Definition, _ stage.Values) (stage.Result, error) {
|
||||
close(entered)
|
||||
<-ctx.Done()
|
||||
detached.Store(stage.Detached(ctx))
|
||||
return stage.Result{}, ctx.Err()
|
||||
}
|
||||
d, err := r.start(protocol.Message{PipelineID: items[0].ID, Version: 1, RequestKey: "cancel", Workspace: t.TempDir(), Input: map[string]any{"prompt": "task"}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
<-entered
|
||||
if err := r.cancel(d.ID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
r.wg.Wait()
|
||||
final, _ := s.GetRun(context.Background(), d.ID)
|
||||
if detached.Load() || final.Status != "canceled" {
|
||||
t.Fatalf("operator cancel did not stop the run: detached=%v %+v", detached.Load(), final)
|
||||
}
|
||||
}
|
||||
+90
-23
@@ -84,6 +84,10 @@ type runner struct {
|
||||
// sizeKick wakes the background run-size backfill (sizeLoop); nil when the
|
||||
// loop is not running (a runner built directly in a test).
|
||||
sizeKick chan struct{}
|
||||
// detach leaves harness sessions running when the server stops; reattach
|
||||
// holds the runs start-up recovery kept for reattachRuns to continue.
|
||||
detach bool
|
||||
reattach []reattachPlan
|
||||
}
|
||||
|
||||
func newRunner(ctx context.Context, s *pipeline.Store) (*runner, error) {
|
||||
@@ -96,6 +100,14 @@ func newRunner(ctx context.Context, s *pipeline.Store) (*runner, error) {
|
||||
// starts and is never written afterwards, so those goroutines read it without
|
||||
// locking.
|
||||
func newRunnerAt(ctx context.Context, s *pipeline.Store, artifactsRoot string) (*runner, error) {
|
||||
return newRunnerReattaching(ctx, s, artifactsRoot, false)
|
||||
}
|
||||
|
||||
// newRunnerReattaching builds the runner. With detach set, a server shutdown
|
||||
// leaves stage sessions running (see stage.ErrServerShutdown) and start-up
|
||||
// recovery re-adopts the runs a previous process left in flight instead of
|
||||
// interrupting them; reattachRuns then continues them.
|
||||
func newRunnerReattaching(ctx context.Context, s *pipeline.Store, artifactsRoot string, detach bool) (*runner, error) {
|
||||
if err := s.AcquireRunnerLock(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -109,19 +121,30 @@ func newRunnerAt(ctx context.Context, s *pipeline.Store, artifactsRoot string) (
|
||||
if err = s.BindLegacyScheduleWorkspace(ctx, workspace); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var plans []reattachPlan
|
||||
keep, reasons, gone := map[int64]bool{}, map[int64]string{}, map[int64]bool{}
|
||||
if detach {
|
||||
if plans, reasons, gone, err = planReattach(ctx, s); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, plan := range plans {
|
||||
keep[plan.runID] = true
|
||||
}
|
||||
}
|
||||
sessions, err := s.ActiveSessionOwners(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, owner := range sessions {
|
||||
stop, cancel := context.WithTimeout(ctx, 20*time.Second)
|
||||
var err error
|
||||
if owner.Backend == "pi" {
|
||||
err = stage.InterruptPi(stop, owner.Reference)
|
||||
} else {
|
||||
err = (stage.OpenCodeClient{Backend: owner.Backend, Directory: owner.Directory, Reference: owner.Reference}).InterruptAndConfirm(stop, owner.ID)
|
||||
if keep[owner.RunID] {
|
||||
continue // Re-adopted below: never interrupt work that survived.
|
||||
}
|
||||
var err error
|
||||
if !gone[owner.RunID] {
|
||||
stop, cancel := context.WithTimeout(ctx, 20*time.Second)
|
||||
err = interruptRecoveredSession(stop, owner)
|
||||
cancel()
|
||||
}
|
||||
cancel()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("cannot interrupt previously active agent %s: %w", owner.ID, err)
|
||||
}
|
||||
@@ -129,7 +152,7 @@ func newRunnerAt(ctx context.Context, s *pipeline.Store, artifactsRoot string) (
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
if err := s.RecoverRuns(ctx); err != nil {
|
||||
if err := s.RecoverRunsKeeping(ctx, keep, reasons); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
root := artifactsRoot
|
||||
@@ -146,7 +169,7 @@ func newRunnerAt(ctx context.Context, s *pipeline.Store, artifactsRoot string) (
|
||||
active[row.ID] = activeRun{cancel: func() {}, workspace: row.Workspace}
|
||||
}
|
||||
}
|
||||
r := &runner{ctx: ctx, store: s, artifactsRoot: root, active: active, idents: map[int64]runIdentity{}, attentionSent: map[int64]map[string]bool{}, notifyModes: map[int64]string{}, exportSession: stage.ExportSession}
|
||||
r := &runner{ctx: ctx, store: s, artifactsRoot: root, active: active, idents: map[int64]runIdentity{}, attentionSent: map[int64]map[string]bool{}, notifyModes: map[int64]string{}, exportSession: stage.ExportSession, detach: detach, reattach: plans}
|
||||
if err := r.startSessionExportJobs(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -341,7 +364,7 @@ func (r *runner) startWithSchedule(request protocol.Message, origin *pipeline.Sc
|
||||
// the registration when it returns. A fresh run and a resumed decision both go
|
||||
// through it, so registration and teardown live once. Callers must hold r.mu.
|
||||
func (r *runner) launch(id int64, workspace string, fn func(context.Context)) {
|
||||
ctx, cancel := context.WithCancel(r.ctx)
|
||||
ctx, cancel := r.runContext()
|
||||
generation := new(byte)
|
||||
r.active[id] = activeRun{cancel: cancel, workspace: workspace, generation: generation}
|
||||
r.wg.Add(1)
|
||||
@@ -980,6 +1003,9 @@ type runState struct {
|
||||
// have been resolved. It is intentionally not part of run input or memory.
|
||||
retryOverride stage.Values
|
||||
priorBudgets map[string]stage.BudgetSnapshot
|
||||
// adopt names the visit to re-adopt from a previous server process instead
|
||||
// of starting it; it is consumed by the loop's first iteration.
|
||||
adopt *adoptedVisit
|
||||
}
|
||||
|
||||
func newRunState(key string, input stage.Values) *runState {
|
||||
@@ -1528,6 +1554,15 @@ func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, na
|
||||
finish := func(status, outcome, diagnostic string) {
|
||||
r.finishRunState(id, name, workspace, status, outcome, diagnostic, state.completed, state.nextSequence-1, state.key, state.input)
|
||||
}
|
||||
// canceled settles an operator cancel. A server shutdown is not a cancel:
|
||||
// the run is left exactly as persisted for the next process to continue.
|
||||
canceled := func() {
|
||||
if stage.Detached(ctx) {
|
||||
log.Printf("run %d: server stopping; run left in flight for the next start to continue", id)
|
||||
return
|
||||
}
|
||||
finish("canceled", "", ctx.Err().Error())
|
||||
}
|
||||
defer func() {
|
||||
if recovered := recover(); recovered != nil {
|
||||
finish("failed", "error", "stage execution panicked")
|
||||
@@ -1554,7 +1589,7 @@ func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, na
|
||||
}
|
||||
for sequence := state.nextSequence; sequence <= 30; sequence++ {
|
||||
if ctx.Err() != nil {
|
||||
finish("canceled", "", ctx.Err().Error())
|
||||
canceled()
|
||||
return
|
||||
}
|
||||
key := state.key
|
||||
@@ -1600,8 +1635,15 @@ func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, na
|
||||
return
|
||||
}
|
||||
}
|
||||
// adopted is the visit a previous server process began and left running
|
||||
// across its restart: its step row, turn files and session already exist.
|
||||
adopted := state.adopt
|
||||
if adopted != nil && adopted.sequence != sequence {
|
||||
adopted = nil
|
||||
}
|
||||
state.adopt = nil
|
||||
_, declaresSkip := def.Edges["skipped"]
|
||||
skip := !isHuman && declaresSkip && autoSkipEligible(def.Configs, state)
|
||||
skip := adopted == nil && !isHuman && declaresSkip && autoSkipEligible(def.Configs, state)
|
||||
state.nextSequence = sequence + 1
|
||||
state.visits[key]++
|
||||
// Publish the run's server-captured changed paths into the input memory
|
||||
@@ -1643,13 +1685,15 @@ func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, na
|
||||
retryInput = input
|
||||
state.retryOverride = nil
|
||||
}
|
||||
if err := r.store.BeginStep(ctx, id, sequence, key, input); err != nil {
|
||||
if ctx.Err() != nil {
|
||||
finish("canceled", "", ctx.Err().Error())
|
||||
if adopted == nil {
|
||||
if err := r.store.BeginStep(ctx, id, sequence, key, input); err != nil {
|
||||
if ctx.Err() != nil {
|
||||
canceled()
|
||||
return
|
||||
}
|
||||
finish("failed", "error", err.Error())
|
||||
return
|
||||
}
|
||||
finish("failed", "error", err.Error())
|
||||
return
|
||||
}
|
||||
// queued -> running: report the new current stage and the step start.
|
||||
r.emitRun(protocol.RunEventFrame{
|
||||
@@ -1668,7 +1712,7 @@ func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, na
|
||||
if err := r.store.MarkWaiting(ctx, id, sequence, key, evidence); err != nil {
|
||||
r.mu.Unlock()
|
||||
if ctx.Err() != nil {
|
||||
finish("canceled", "", ctx.Err().Error())
|
||||
canceled()
|
||||
return
|
||||
}
|
||||
finish("failed", "error", err.Error())
|
||||
@@ -1744,9 +1788,11 @@ func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, na
|
||||
var publicBefore map[string]bool
|
||||
if stage.EffectiveType(def.Type) == stage.ModelType {
|
||||
visitTurn = r.turnPaths(id, sequence, key)
|
||||
if turnErr := r.prepareTurn(visitTurn, state, retryInput); turnErr != nil {
|
||||
finish("failed", "error", turnErr.Error())
|
||||
return
|
||||
if adopted == nil {
|
||||
if turnErr := r.prepareTurn(visitTurn, state, retryInput); turnErr != nil {
|
||||
finish("failed", "error", turnErr.Error())
|
||||
return
|
||||
}
|
||||
}
|
||||
artifactsDir = visitTurn.TurnDir
|
||||
def.ArtifactsDir = artifactsDir
|
||||
@@ -1779,6 +1825,11 @@ func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, na
|
||||
var piClient *stage.PiClient
|
||||
var err error
|
||||
stageCtx := ctx
|
||||
if adopted != nil {
|
||||
// Attach to the surviving session instead of prompting again.
|
||||
client.Backend, client.Reference = adopted.owner.Backend, adopted.owner.Reference
|
||||
stageCtx = stage.WithResumeSession(stageCtx, adopted.owner.ID)
|
||||
}
|
||||
var modelConfig stage.ModelConfig
|
||||
var budgetBefore stage.BudgetSnapshot
|
||||
if stage.EffectiveType(def.Type) == stage.ModelType {
|
||||
@@ -1846,6 +1897,9 @@ func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, na
|
||||
stageSession = client.SessionID
|
||||
} else if stage.EffectiveType(def.Type) == stage.ModelType {
|
||||
plain := &stage.OpenCodeClient{Directory: workspace, Operations: operations, ArtifactsDir: artifactsDir}
|
||||
if adopted != nil {
|
||||
plain.Backend, plain.Reference = adopted.owner.Backend, adopted.owner.Reference
|
||||
}
|
||||
plain.OnGenerationSession = func(session, backend, reference string, active bool) error {
|
||||
c, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
@@ -1860,6 +1914,13 @@ func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, na
|
||||
result, err = stage.DefaultRegistry().Run(stageCtx, def, input)
|
||||
}
|
||||
}()
|
||||
if r.detach && stage.Detached(ctx) {
|
||||
// The server is stopping. The stage left its session running and
|
||||
// owned (or never admitted one); nothing about this visit is settled
|
||||
// here. The next process re-adopts the session and continues.
|
||||
log.Printf("run %d stage %s #%d: server stopping; stage left running for the next start to reattach", id, key, sequence)
|
||||
return
|
||||
}
|
||||
ownershipCtx, ownershipCancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
activeSessions, ownershipErr := r.store.ActiveAgentSessionsForRun(ownershipCtx, id)
|
||||
ownershipCancel()
|
||||
@@ -1877,6 +1938,12 @@ func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, na
|
||||
after := filediff.Capture(snapshotCtx, workspace, before)
|
||||
snapshotCancel()
|
||||
changes := filediff.Compare(before, after)
|
||||
if adopted != nil && changes.Status == "complete" {
|
||||
// The pre-visit snapshot belonged to the stopped process; only the
|
||||
// changes made after reattaching were compared.
|
||||
changes.Status = "partial"
|
||||
changes.Diagnostic = "Server restarted during this stage; changes made before the restart are not listed."
|
||||
}
|
||||
if errors.Is(err, stage.ErrStageCleanup) {
|
||||
changes.Status = "partial"
|
||||
changes.Diagnostic = "Agent cleanup was not confirmed; files may still be changing."
|
||||
@@ -2062,7 +2129,7 @@ func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, na
|
||||
return
|
||||
}
|
||||
if ctx.Err() != nil {
|
||||
finish("canceled", "", ctx.Err().Error())
|
||||
canceled()
|
||||
return
|
||||
}
|
||||
// Advance the in-memory UI-surface evidence only after the step is
|
||||
@@ -2099,7 +2166,7 @@ func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, na
|
||||
},
|
||||
}, false)
|
||||
if ctx.Err() != nil {
|
||||
finish("canceled", "", ctx.Err().Error())
|
||||
canceled()
|
||||
return
|
||||
}
|
||||
if terminal {
|
||||
|
||||
@@ -58,6 +58,12 @@ type Options struct {
|
||||
// target-aware verification. Definitions can name only a registered ID; they
|
||||
// cannot select an endpoint or supply provenance.
|
||||
VerificationFixtures []VerificationFixture
|
||||
// DetachSessions keeps harness sessions running across a server restart:
|
||||
// a stopping server leaves stage and chat sessions running (DetachSessions
|
||||
// on the handler records the chats) and the next start reattaches them.
|
||||
// False keeps the previous behaviour (tests, embedded servers): a stopping
|
||||
// server interrupts its runs and the next start marks them interrupted.
|
||||
DetachSessions bool
|
||||
}
|
||||
|
||||
// Handler serves the persistent connection with default options. Canceling ctx
|
||||
@@ -77,7 +83,7 @@ func HandlerWithOptions(ctx context.Context, options Options, stores ...*pipelin
|
||||
var runs *runner
|
||||
var runErr error
|
||||
if len(stores) > 0 {
|
||||
runs, runErr = newRunnerAt(ctx, stores[0], options.ArtifactsRoot)
|
||||
runs, runErr = newRunnerReattaching(ctx, stores[0], options.ArtifactsRoot, options.DetachSessions)
|
||||
if runErr == nil {
|
||||
runs.events = events
|
||||
}
|
||||
@@ -115,15 +121,11 @@ func HandlerWithOptions(ctx context.Context, options Options, stores ...*pipelin
|
||||
settingsStore = stores[0]
|
||||
}
|
||||
hub.settings = settingsStore
|
||||
if options.Chat != nil && registry != nil {
|
||||
// Keep observing chats whose turn a previous process left running, so
|
||||
// their reply or question still notifies with no client attached.
|
||||
go hub.observeRunningTurns(ctx, options.Chat)
|
||||
}
|
||||
if len(stores) > 0 {
|
||||
// Prompts queued behind a running turn are durable and owned by their
|
||||
// session; resume delivering any a previous process left queued.
|
||||
hub.queueStore = stores[0]
|
||||
hub.liveStore = stores[0]
|
||||
if options.Chat != nil {
|
||||
go hub.resumeQueued(ctx, options.Chat)
|
||||
}
|
||||
@@ -449,12 +451,44 @@ func HandlerWithOptions(ctx context.Context, options Options, stores ...*pipelin
|
||||
}
|
||||
}
|
||||
mux.HandleFunc("GET /connect", connect)
|
||||
return &runHandler{Handler: mux, runner: runs}
|
||||
if options.Chat != nil && registry != nil {
|
||||
// Reattach the chats a previous process left live (and fold in turns
|
||||
// that ended while it was down), then keep observing any other chat
|
||||
// whose turn is still running, so a reply or question still notifies
|
||||
// with no client attached.
|
||||
go func() {
|
||||
hub.reattachSessions(ctx, options.Chat)
|
||||
hub.observeRunningTurns(ctx, options.Chat)
|
||||
}()
|
||||
}
|
||||
if runs != nil {
|
||||
// Continue the runs a previous process left in flight, now that the
|
||||
// runner's collaborators are wired and before any request is served.
|
||||
runs.reattachRuns()
|
||||
}
|
||||
return &runHandler{Handler: mux, runner: runs, hub: hub}
|
||||
}
|
||||
|
||||
type runHandler struct {
|
||||
http.Handler
|
||||
runner *runner
|
||||
hub *chatHub
|
||||
}
|
||||
|
||||
// DetachSessions records the chats left live on their harness so the next
|
||||
// start reattaches them. Call it when the server is stopping, before the
|
||||
// handler's context is canceled (rooms are torn down after that) and before the
|
||||
// store closes. It never interrupts a session.
|
||||
func DetachSessions(handler http.Handler) {
|
||||
h, ok := handler.(*runHandler)
|
||||
if !ok || h.hub == nil {
|
||||
return
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
if err := h.hub.detachSessions(ctx); err != nil {
|
||||
log.Printf("record live chat sessions: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// WaitRuns drains active execution before the database is closed on shutdown.
|
||||
|
||||
+63
-19
@@ -27,6 +27,23 @@ type AgentClient struct {
|
||||
WorkStarted bool
|
||||
}
|
||||
|
||||
// resumeConnection reopens the retained execution service a previous server
|
||||
// process recorded for a session, refusing a different backend or directory.
|
||||
func resumeConnection(reference, backend, directory string) (*opencode.Connection, error) {
|
||||
if reference == "" {
|
||||
return nil, fmt.Errorf("retained OpenCode session has no execution-service reference")
|
||||
}
|
||||
connection, err := opencode.FromReference(reference)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if (backend != "" && connection.Backend != backend) || connection.Directory != directory {
|
||||
connection.Close()
|
||||
return nil, fmt.Errorf("retained OpenCode execution context mismatch")
|
||||
}
|
||||
return connection, nil
|
||||
}
|
||||
|
||||
func (c OpenCodeClient) Interrupt(ctx context.Context, id string) error {
|
||||
return c.interrupt(ctx, id, false)
|
||||
}
|
||||
@@ -80,7 +97,18 @@ func (c *AgentClient) Generate(ctx context.Context, config ModelConfig, prompt s
|
||||
if c.Operations != nil {
|
||||
c.Operations.Provider, c.Operations.Model = config.Provider, config.Model
|
||||
}
|
||||
if c.command == nil {
|
||||
// A re-adopted session was admitted by a previous server process: attach
|
||||
// to its retained service and wait for it; never create or prompt again.
|
||||
resume := TakeResumeSession(ctx)
|
||||
if resume != "" && c.command == nil {
|
||||
connection, err := resumeConnection(c.Reference, c.Backend, c.Directory)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer connection.Close()
|
||||
c.connection = connection
|
||||
c.Backend = connection.Backend
|
||||
} else if c.command == nil {
|
||||
connection, err := opencode.OpenStage(ctx, opencode.Executable(), c.Directory, c.Backend)
|
||||
if err != nil {
|
||||
// Connecting to the service failed before any session existed.
|
||||
@@ -101,24 +129,31 @@ func (c *AgentClient) Generate(ctx context.Context, config ModelConfig, prompt s
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
data, err := c.call(ctx, "post", "/api/session", map[string]any{"title": "Pipeline audited agent stage", "agent": "build", "location": map[string]string{"directory": c.Directory}, "model": model, "permissions": rules})
|
||||
if err != nil {
|
||||
if variant := model["variant"]; variant != "" {
|
||||
err = fmt.Errorf("OpenCode model selection for reasoning_effort=%s (variant %q) failed: %w", variant, variant, err)
|
||||
if resume != "" {
|
||||
if c.Backend != "v2" || !sessionIDPattern.MatchString(resume) {
|
||||
return "", fmt.Errorf("retained OpenCode session %.128s cannot be re-adopted on backend %q", RedactEvidence(resume), c.Backend)
|
||||
}
|
||||
// No prompt or tool call was admitted: a request failure here is a
|
||||
// reachable retry rather than a reason to end the run.
|
||||
return "", harnessUnreachable(err)
|
||||
c.SessionID = resume
|
||||
} else {
|
||||
data, err := c.call(ctx, "post", "/api/session", map[string]any{"title": "Pipeline audited agent stage", "agent": "build", "location": map[string]string{"directory": c.Directory}, "model": model, "permissions": rules})
|
||||
if err != nil {
|
||||
if variant := model["variant"]; variant != "" {
|
||||
err = fmt.Errorf("OpenCode model selection for reasoning_effort=%s (variant %q) failed: %w", variant, variant, err)
|
||||
}
|
||||
// No prompt or tool call was admitted: a request failure here is a
|
||||
// reachable retry rather than a reason to end the run.
|
||||
return "", harnessUnreachable(err)
|
||||
}
|
||||
var created struct {
|
||||
Data struct {
|
||||
ID string `json:"id"`
|
||||
} `json:"data"`
|
||||
}
|
||||
if json.Unmarshal(data, &created) != nil || !sessionIDPattern.MatchString(created.Data.ID) {
|
||||
return "", fmt.Errorf("invalid agent session")
|
||||
}
|
||||
c.SessionID = created.Data.ID
|
||||
}
|
||||
var created struct {
|
||||
Data struct {
|
||||
ID string `json:"id"`
|
||||
} `json:"data"`
|
||||
}
|
||||
if json.Unmarshal(data, &created) != nil || !sessionIDPattern.MatchString(created.Data.ID) {
|
||||
return "", fmt.Errorf("invalid agent session")
|
||||
}
|
||||
c.SessionID = created.Data.ID
|
||||
if c.connection != nil && c.Backend == "v1" {
|
||||
c.Reference, err = c.connection.Reference()
|
||||
if err != nil {
|
||||
@@ -134,6 +169,12 @@ func (c *AgentClient) Generate(ctx context.Context, config ModelConfig, prompt s
|
||||
defer func() {
|
||||
watch.stopWatch(ctx, &err)
|
||||
defer watch.finish(ctx, &err)
|
||||
if err != nil && Detached(ctx) {
|
||||
// The server is restarting: leave the admitted session running and
|
||||
// its ownership recorded, so the next process re-adopts it.
|
||||
err = fmt.Errorf("%w: retained OpenCode session %s left running", ErrDetached, c.SessionID)
|
||||
return
|
||||
}
|
||||
cleanup, cancel := context.WithTimeout(context.WithoutCancel(ctx), 15*time.Second)
|
||||
defer cancel()
|
||||
if err != nil {
|
||||
@@ -181,8 +222,10 @@ func (c *AgentClient) Generate(ctx context.Context, config ModelConfig, prompt s
|
||||
return "", err
|
||||
}
|
||||
c.WorkStarted = true
|
||||
if _, err = c.call(ctx, "post", "/api/session/"+c.SessionID+"/prompt", generationBody("text", prompt, config, c.Backend)); err != nil {
|
||||
return "", err
|
||||
if resume == "" {
|
||||
if _, err = c.call(ctx, "post", "/api/session/"+c.SessionID+"/prompt", generationBody("text", prompt, config, c.Backend)); err != nil {
|
||||
return "", err
|
||||
}
|
||||
}
|
||||
if c.Backend == "v1" {
|
||||
// Native V1 prompts block until completion; keep the existing settlement
|
||||
@@ -193,6 +236,7 @@ func (c *AgentClient) Generate(ctx context.Context, config ModelConfig, prompt s
|
||||
} else if err = c.waitForSessionIdle(ctx, c.SessionID); err != nil {
|
||||
return "", err
|
||||
}
|
||||
var data []byte
|
||||
data, text, err = c.readFinalAgentResult(ctx, c.SessionID)
|
||||
if len(data) > 0 {
|
||||
c.Transcript = string(data)
|
||||
|
||||
@@ -0,0 +1,61 @@
|
||||
package stage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"sync"
|
||||
)
|
||||
|
||||
// A Pipelines server restart must not stop harness work. The runner cancels a
|
||||
// run's context with ErrServerShutdown as its cause when the server process is
|
||||
// stopping (as opposed to an operator cancel, whose cause is context.Canceled).
|
||||
// A stage client that sees this cause leaves its admitted session running: it
|
||||
// neither interrupts it nor clears its recorded ownership, and returns an error
|
||||
// wrapping ErrDetached. The next server process re-adopts the session through
|
||||
// WithResumeSession and waits for it instead of prompting again.
|
||||
|
||||
// ErrServerShutdown is the cancellation cause of a run context whose server is
|
||||
// stopping or restarting.
|
||||
var ErrServerShutdown = errors.New("pipelines server is shutting down; harness sessions are left running")
|
||||
|
||||
// ErrDetached reports that a stage left its admitted harness session running
|
||||
// for the next server process to re-adopt.
|
||||
var ErrDetached = errors.New("stage session detached for server restart")
|
||||
|
||||
// Detached reports whether ctx was canceled because the server is stopping.
|
||||
func Detached(ctx context.Context) bool {
|
||||
return ctx.Err() != nil && errors.Is(context.Cause(ctx), ErrServerShutdown)
|
||||
}
|
||||
|
||||
type resumeKey struct{}
|
||||
|
||||
type resumeHolder struct {
|
||||
mu sync.Mutex
|
||||
session string
|
||||
}
|
||||
|
||||
// WithResumeSession marks the next model generation under ctx as a re-adoption
|
||||
// of an already admitted session: the client attaches to sessionID, waits for it
|
||||
// to settle and reads its final result, without creating a session or sending a
|
||||
// prompt. The mark is consumed by the first generation, so a later retry or
|
||||
// repair attempt starts fresh as usual.
|
||||
func WithResumeSession(ctx context.Context, sessionID string) context.Context {
|
||||
if sessionID == "" {
|
||||
return ctx
|
||||
}
|
||||
return context.WithValue(ctx, resumeKey{}, &resumeHolder{session: sessionID})
|
||||
}
|
||||
|
||||
// TakeResumeSession returns and consumes the session marked by
|
||||
// WithResumeSession, or "" when the generation is a fresh one.
|
||||
func TakeResumeSession(ctx context.Context) string {
|
||||
holder, _ := ctx.Value(resumeKey{}).(*resumeHolder)
|
||||
if holder == nil {
|
||||
return ""
|
||||
}
|
||||
holder.mu.Lock()
|
||||
defer holder.mu.Unlock()
|
||||
id := holder.session
|
||||
holder.session = ""
|
||||
return id
|
||||
}
|
||||
@@ -0,0 +1,159 @@
|
||||
package stage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// fakeV2Session answers the V2 calls a stage makes for one session, records
|
||||
// every request, and reports the session running until finish is closed.
|
||||
type fakeV2Session struct {
|
||||
t *testing.T
|
||||
id string
|
||||
finish chan struct{}
|
||||
mu sync.Mutex
|
||||
calls []string
|
||||
}
|
||||
|
||||
func (f *fakeV2Session) command(_ context.Context, _ string, args ...string) ([]byte, error) {
|
||||
f.mu.Lock()
|
||||
f.calls = append(f.calls, args[1]+" "+args[2])
|
||||
f.mu.Unlock()
|
||||
switch path := args[2]; {
|
||||
case path == "/api/session":
|
||||
return []byte(`{"data":{"id":"` + f.id + `"}}`), nil
|
||||
case strings.HasSuffix(path, "/prompt"), strings.HasSuffix(path, "/interrupt"):
|
||||
return []byte(`{}`), nil
|
||||
case path == "/api/session/active":
|
||||
select {
|
||||
case <-f.finish:
|
||||
return []byte(`{"data":{}}`), nil
|
||||
default:
|
||||
return []byte(`{"data":{"` + f.id + `":{"type":"running"}}}`), nil
|
||||
}
|
||||
case path == "/api/session/"+f.id+finalAssistantReadback:
|
||||
return []byte(`{"data":[{"finish":"stop","content":[{"type":"text","text":"# outcome\nsuccess\n\nreattached reply\n"}]}]}`), nil
|
||||
}
|
||||
return nil, fmt.Errorf("unexpected request %v", args)
|
||||
}
|
||||
|
||||
func (f *fakeV2Session) called(prefix string) bool {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
for _, call := range f.calls {
|
||||
if strings.HasPrefix(call, prefix) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// TestAgentDetachesOnServerShutdown: a server shutdown (not an operator cancel)
|
||||
// leaves the admitted session running and owned: no interrupt, no release.
|
||||
func TestAgentDetachesOnServerShutdown(t *testing.T) {
|
||||
fake := &fakeV2Session{t: t, id: "ses_detach", finish: make(chan struct{})}
|
||||
var released []string
|
||||
client := &AgentClient{OnSession: func(id string, active bool) error {
|
||||
if !active {
|
||||
released = append(released, id)
|
||||
}
|
||||
return nil
|
||||
}}
|
||||
client.Backend, client.Directory, client.command, client.sessionPollInterval = "v2", "/work", fake.command, time.Millisecond
|
||||
ctx, cancel := context.WithCancelCause(context.Background())
|
||||
go func() {
|
||||
for !fake.called("post /api/session/ses_detach/prompt") {
|
||||
time.Sleep(time.Millisecond)
|
||||
}
|
||||
cancel(ErrServerShutdown)
|
||||
}()
|
||||
_, err := client.Generate(ctx, ModelConfig{}, "work")
|
||||
if !errors.Is(err, ErrDetached) {
|
||||
t.Fatalf("shutdown did not detach: %v", err)
|
||||
}
|
||||
if fake.called("post /api/session/ses_detach/interrupt") {
|
||||
t.Fatal("a server shutdown interrupted the stage session")
|
||||
}
|
||||
if len(released) != 0 {
|
||||
t.Fatalf("a detached session's ownership was released: %v", released)
|
||||
}
|
||||
}
|
||||
|
||||
// TestAgentOperatorCancelStillInterrupts keeps the explicit stop path.
|
||||
func TestAgentOperatorCancelStillInterrupts(t *testing.T) {
|
||||
fake := &fakeV2Session{t: t, id: "ses_cancel", finish: make(chan struct{})}
|
||||
client := &AgentClient{}
|
||||
client.Backend, client.Directory, client.command, client.sessionPollInterval = "v2", "/work", fake.command, time.Millisecond
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
go func() {
|
||||
for !fake.called("post /api/session/ses_cancel/prompt") {
|
||||
time.Sleep(time.Millisecond)
|
||||
}
|
||||
cancel()
|
||||
// Interrupt confirmation waits for the session to settle.
|
||||
for !fake.called("post /api/session/ses_cancel/interrupt") {
|
||||
time.Sleep(time.Millisecond)
|
||||
}
|
||||
close(fake.finish)
|
||||
}()
|
||||
if _, err := client.Generate(ctx, ModelConfig{}, "work"); err == nil || errors.Is(err, ErrDetached) {
|
||||
t.Fatalf("operator cancel did not fail the stage: %v", err)
|
||||
}
|
||||
if !fake.called("post /api/session/ses_cancel/interrupt") {
|
||||
t.Fatal("operator cancel did not interrupt the session")
|
||||
}
|
||||
}
|
||||
|
||||
// TestAgentResumeAttachesWithoutPrompting: a re-adopted session is waited on
|
||||
// and read back; it is never created or prompted again, and the resume mark is
|
||||
// consumed so a retry starts fresh.
|
||||
func TestAgentResumeAttachesWithoutPrompting(t *testing.T) {
|
||||
fake := &fakeV2Session{t: t, id: "ses_adopted", finish: make(chan struct{})}
|
||||
var owned []bool
|
||||
client := &AgentClient{OnSession: func(id string, active bool) error {
|
||||
if id != "ses_adopted" {
|
||||
t.Errorf("ownership for %s", id)
|
||||
}
|
||||
owned = append(owned, active)
|
||||
return nil
|
||||
}}
|
||||
client.Backend, client.Directory, client.command, client.sessionPollInterval = "v2", "/work", fake.command, time.Millisecond
|
||||
ctx := WithResumeSession(context.Background(), "ses_adopted")
|
||||
go func() {
|
||||
time.Sleep(20 * time.Millisecond)
|
||||
close(fake.finish)
|
||||
}()
|
||||
text, err := client.Generate(ctx, ModelConfig{}, "work")
|
||||
if err != nil || !strings.Contains(text, "reattached reply") {
|
||||
t.Fatalf("resume = %q, %v", text, err)
|
||||
}
|
||||
if fake.called("post /api/session ") || fake.called("post /api/session/ses_adopted/prompt") {
|
||||
t.Fatalf("a re-adopted session was created or prompted again: %v", fake.calls)
|
||||
}
|
||||
if len(owned) != 2 || !owned[0] || owned[1] {
|
||||
t.Fatalf("ownership was not re-asserted then released: %v", owned)
|
||||
}
|
||||
if TakeResumeSession(ctx) != "" {
|
||||
t.Fatal("the resume mark was not consumed")
|
||||
}
|
||||
}
|
||||
|
||||
// TestPlainResumeReadsResultReachedWhileDown: a text-only session that ended
|
||||
// during the downtime is folded in from its final assistant message.
|
||||
func TestPlainResumeReadsResultReachedWhileDown(t *testing.T) {
|
||||
fake := &fakeV2Session{t: t, id: "ses_plain", finish: make(chan struct{})}
|
||||
close(fake.finish)
|
||||
client := &OpenCodeClient{Backend: "v2", Directory: "/work", command: fake.command, sessionPollInterval: time.Millisecond}
|
||||
text, err := client.Generate(WithResumeSession(context.Background(), "ses_plain"), ModelConfig{}, "prompt")
|
||||
if err != nil || !strings.Contains(text, "reattached reply") {
|
||||
t.Fatalf("resume = %q, %v", text, err)
|
||||
}
|
||||
if fake.called("post /api/session/ses_plain/prompt") || fake.called("post /api/session ") {
|
||||
t.Fatalf("a re-adopted session was prompted again: %v", fake.calls)
|
||||
}
|
||||
}
|
||||
+44
-19
@@ -273,7 +273,18 @@ func (c *OpenCodeClient) Generate(ctx context.Context, config ModelConfig, promp
|
||||
if c.Operations != nil {
|
||||
c.Operations.Provider, c.Operations.Model = config.Provider, config.Model
|
||||
}
|
||||
if c.command == nil {
|
||||
// A re-adopted session was admitted by a previous server process (see
|
||||
// WithResumeSession): wait for it and read its result, never prompt again.
|
||||
resume := TakeResumeSession(ctx)
|
||||
if resume != "" && c.command == nil {
|
||||
connection, err := resumeConnection(c.Reference, c.Backend, c.Directory)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer connection.Close()
|
||||
c.connection = connection
|
||||
c.Backend = connection.Backend
|
||||
} else if c.command == nil {
|
||||
connection, err := opencode.OpenStage(ctx, opencode.Executable(), c.Directory, c.Backend)
|
||||
if err != nil {
|
||||
// Connecting to the service failed before any session existed.
|
||||
@@ -313,33 +324,45 @@ func (c *OpenCodeClient) Generate(ctx context.Context, config ModelConfig, promp
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
created, err := c.call(ctx, "post", "/api/session", map[string]any{
|
||||
"title": "Pipeline stage generation",
|
||||
"location": map[string]string{"directory": directory},
|
||||
"model": model,
|
||||
"permissions": []map[string]string{{"action": "*", "resource": "*", "effect": "deny"}},
|
||||
})
|
||||
if err != nil {
|
||||
if variant := model["variant"]; variant != "" {
|
||||
err = fmt.Errorf("OpenCode model selection for reasoning_effort=%s (variant %q) failed: %w", variant, variant, err)
|
||||
}
|
||||
// No prompt or tool call was admitted: a request failure here is a
|
||||
// reachable retry rather than a reason to end the run.
|
||||
return "", harnessUnreachable(err)
|
||||
}
|
||||
var session struct {
|
||||
Data struct {
|
||||
ID string `json:"id"`
|
||||
} `json:"data"`
|
||||
}
|
||||
if err := json.Unmarshal(created, &session); err != nil || !sessionIDPattern.MatchString(session.Data.ID) {
|
||||
return "", fmt.Errorf("OpenCode returned an invalid session ID")
|
||||
if resume != "" {
|
||||
if c.Backend != "v2" || !sessionIDPattern.MatchString(resume) {
|
||||
return "", fmt.Errorf("retained OpenCode session %.128s cannot be re-adopted on backend %q", RedactEvidence(resume), c.Backend)
|
||||
}
|
||||
session.Data.ID = resume
|
||||
} else {
|
||||
created, err := c.call(ctx, "post", "/api/session", map[string]any{
|
||||
"title": "Pipeline stage generation",
|
||||
"location": map[string]string{"directory": directory},
|
||||
"model": model,
|
||||
"permissions": []map[string]string{{"action": "*", "resource": "*", "effect": "deny"}},
|
||||
})
|
||||
if err != nil {
|
||||
if variant := model["variant"]; variant != "" {
|
||||
err = fmt.Errorf("OpenCode model selection for reasoning_effort=%s (variant %q) failed: %w", variant, variant, err)
|
||||
}
|
||||
// No prompt or tool call was admitted: a request failure here is a
|
||||
// reachable retry rather than a reason to end the run.
|
||||
return "", harnessUnreachable(err)
|
||||
}
|
||||
if err := json.Unmarshal(created, &session); err != nil || !sessionIDPattern.MatchString(session.Data.ID) {
|
||||
return "", fmt.Errorf("OpenCode returned an invalid session ID")
|
||||
}
|
||||
}
|
||||
var watch *budgetWatch
|
||||
promptStarted := false
|
||||
defer func() {
|
||||
watch.stopWatch(ctx, &err)
|
||||
defer watch.finish(ctx, &err)
|
||||
if err != nil && Detached(ctx) {
|
||||
// The server is restarting: leave the session running and owned.
|
||||
err = fmt.Errorf("%w: retained OpenCode session %s left running", ErrDetached, session.Data.ID)
|
||||
return
|
||||
}
|
||||
cleanup, cancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second)
|
||||
defer cancel()
|
||||
if err != nil {
|
||||
@@ -392,8 +415,10 @@ func (c *OpenCodeClient) Generate(ctx context.Context, config ModelConfig, promp
|
||||
// including unbudgeted text-only stages, so its exact native turns survive.
|
||||
if c.Backend == "v2" {
|
||||
promptStarted = true
|
||||
if _, err = c.call(ctx, "post", "/api/session/"+session.Data.ID+"/prompt", body); err != nil {
|
||||
return "", err
|
||||
if resume == "" {
|
||||
if _, err = c.call(ctx, "post", "/api/session/"+session.Data.ID+"/prompt", body); err != nil {
|
||||
return "", err
|
||||
}
|
||||
}
|
||||
if err = c.waitForSessionIdle(ctx, session.Data.ID); err != nil {
|
||||
return "", err
|
||||
|
||||
Reference in New Issue
Block a user