Preserve independent background sessions through owner shutdown and idle cleanup

This commit is contained in:
2026-11-07 16:23:20 +00:00
parent 4ead2abdbf
commit b15597662f
9 changed files with 360 additions and 51 deletions
@@ -0,0 +1,43 @@
package opencode
import (
"context"
"net/http"
"net/http/httptest"
"testing"
)
func TestOwnedIdleRequiresAuthoritativeBackgroundCompletion(t *testing.T) {
for _, tc := range []struct {
name, body string
status int
idle bool
}{
{"empty envelope", `{"data":[]}`, 200, true},
{"empty list", `[]`, 200, true},
{"exited", `{"data":[{"status":"exited"}]}`, 200, true},
{"running", `{"data":[{"status":"running"}]}`, 200, false},
{"missing status", `{"data":[{}]}`, 200, false},
{"missing membership", `{}`, 200, false},
{"null membership", `{"data":null}`, 200, false},
{"invalid response", `bad`, 200, false},
{"readback unavailable", ``, 503, false},
} {
t.Run(tc.name, func(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
if r.URL.Path == "/api/session/active" {
_, _ = w.Write([]byte(`{"data":{}}`))
return
}
w.WriteHeader(tc.status)
_, _ = w.Write([]byte(tc.body))
}))
defer server.Close()
c := &Connection{Backend: "v2", URL: server.URL, password: "test"}
if got := c.ownedIdle(context.Background()); got != tc.idle {
t.Fatalf("idle=%v want%v", got, tc.idle)
}
})
}
}
+30 -4
View File
@@ -426,8 +426,10 @@ func (c *Connection) ownedInfo(ctx context.Context) (int, bool) {
return info.PID, true
}
// ownedIdle reports whether no session is active on this process; any error
// counts as busy so active work is never stopped on a guess.
// ownedIdle requires both the foreground drain and background shells to be
// empty/finished. A completed turn can leave real shells running on this
// service; stopping it would kill those tasks across a server restart.
// Any missing, malformed or unavailable readback counts as busy.
func (c *Connection) ownedIdle(ctx context.Context) bool {
probe, cancel := context.WithTimeout(ctx, ownedStageProbeTimeout)
defer cancel()
@@ -438,11 +440,35 @@ func (c *Connection) ownedIdle(ctx context.Context) bool {
var active struct {
Data map[string]json.RawMessage `json:"data"`
}
return json.Unmarshal(data, &active) == nil && active.Data != nil && len(active.Data) == 0
if json.Unmarshal(data, &active) != nil || active.Data == nil || len(active.Data) != 0 {
return false
}
data, err = c.nativeRequest(probe, "get", "/api/shell", nil, "")
if err != nil {
return false
}
var shells []struct {
Status string `json:"status"`
}
var response struct {
Data json.RawMessage `json:"data"`
}
if json.Unmarshal(data, &response) == nil && response.Data != nil {
data = response.Data
}
if json.Unmarshal(data, &shells) != nil || shells == nil {
return false
}
for _, shell := range shells {
if shell.Status != "exited" {
return false
}
}
return true
}
// StopIdleStageService stops the owned stage service on server shutdown when no
// session is active on it. The registration (URL and password) is kept, so a
// foreground turn or background shell is active on it. The registration (URL and password) is kept, so a
// later start reuses the same endpoint and retained references stay valid. A
// busy service keeps running so restart recovery can interrupt it properly.
func StopIdleStageService() {
+128
View File
@@ -3,9 +3,12 @@ package opencode
import (
"context"
"encoding/json"
"fmt"
"git.a-wan.net/alan/pipelines/internal/protocol"
"net/http"
"os"
"path/filepath"
"strconv"
"strings"
"syscall"
"testing"
@@ -63,6 +66,57 @@ func runFakeOpenCodeServe() {
}
_, _ = w.Write([]byte(`{"data":{}}`))
}))
// A real task in this independent service keeps producing output while no
// Pipelines adapter is connected. The marker is its explicit completion gate.
outputPath := os.Getenv("FAKE_SHELL_OUTPUT")
if outputPath != "" {
go func() {
for i := 0; ; i++ {
if _, err := os.Stat(os.Getenv("FAKE_SHELL_FILE")); err != nil {
return
}
f, err := os.OpenFile(outputPath, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0600)
if err != nil {
return
}
_, _ = fmt.Fprintln(f, i)
_ = f.Close()
time.Sleep(20 * time.Millisecond)
}
}()
}
shellInfo := func() map[string]any {
status := "exited"
if _, err := os.Stat(os.Getenv("FAKE_SHELL_FILE")); err == nil {
status = "running"
}
return map[string]any{"id": "sh_background", "status": status, "metadata": map[string]string{"sessionID": "ses_background"}, "command": "background heartbeat"}
}
mux.HandleFunc("/api/shell", auth(func(w http.ResponseWriter) {
_, running := os.Stat(os.Getenv("FAKE_SHELL_FILE"))
_, ran := os.Stat(outputPath)
if running == nil || (outputPath != "" && ran == nil) {
_ = json.NewEncoder(w).Encode(map[string]any{"data": []any{shellInfo()}})
return
}
_, _ = w.Write([]byte(`{"data":[]}`))
}))
mux.HandleFunc("/api/shell/sh_background", auth(func(w http.ResponseWriter) { _ = json.NewEncoder(w).Encode(map[string]any{"data": shellInfo()}) }))
mux.HandleFunc("/api/shell/sh_background/output", func(w http.ResponseWriter, r *http.Request) {
auth(func(w http.ResponseWriter) {
data, _ := os.ReadFile(outputPath)
cursor, _ := strconv.Atoi(r.URL.Query().Get("cursor"))
limit, _ := strconv.Atoi(r.URL.Query().Get("limit"))
if cursor > len(data) {
cursor = len(data)
}
end := cursor + limit
if end > len(data) {
end = len(data)
}
_ = json.NewEncoder(w).Encode(map[string]any{"data": map[string]any{"output": string(data[cursor:end]), "cursor": end, "size": len(data), "truncated": false}})
})(w, r)
})
mux.HandleFunc("/api/env", auth(func(w http.ResponseWriter) {
values := map[string]*string{}
for _, key := range strings.Split(os.Getenv("FAKE_ENV_KEYS"), ",") {
@@ -87,6 +141,7 @@ func fakeOwnedStageEnv(t *testing.T) string {
t.Setenv(OpenCodeServiceStateEnvKey, t.TempDir())
t.Setenv(fakeOpenCodeEnv, "1")
t.Setenv("FAKE_BUSY_FILE", filepath.Join(t.TempDir(), "busy"))
t.Setenv("FAKE_SHELL_FILE", filepath.Join(t.TempDir(), "background"))
for _, key := range []string{StageServiceEnvKey, StageToolOutputMaxBytesEnvKey, StageToolOutputMaxLinesEnvKey, StageCompactionEnvKey, StageConfigEnvKey, "OPENCODE_CONFIG_CONTENT", "FAKE_FAIL"} {
t.Setenv(key, "")
}
@@ -287,3 +342,76 @@ func waitGone(t *testing.T, pid int) {
}
t.Fatalf("process %d still running", pid)
}
// Foreground idle does not imply service idle. This is a process-level restart:
// the adapter closes, shutdown cleanup runs, the task keeps writing while the
// server is away, and a new adapter reattaches the same native shell/output.
func TestBackgroundOnlyStageSurvivesOwnerRestart(t *testing.T) {
executable := fakeOwnedStageEnv(t)
marker := os.Getenv("FAKE_SHELL_FILE")
if err := os.WriteFile(marker, nil, 0600); err != nil {
t.Fatal(err)
}
output := filepath.Join(t.TempDir(), "output")
t.Setenv("FAKE_SHELL_OUTPUT", output)
first, err := OpenStage(context.Background(), executable, t.TempDir(), "v2")
if err != nil {
t.Fatal(err)
}
pid := ownedPID(t, first)
ref, err := first.Reference()
if err != nil {
t.Fatal(err)
}
before := NewChat(first)
shells, err := before.Shells(context.Background(), "ses_background")
if err != nil || len(shells) != 1 {
t.Fatalf("native task: %+v %v", shells, err)
}
query := protocol.ChatShellOutputQuery{SessionID: "ses_background", ShellID: shells[0].ID, Limit: 4096}
page, err := before.ShellOutput(context.Background(), query)
if err != nil {
t.Fatal(err)
}
first.Close()
StopIdleStageService()
time.Sleep(100 * time.Millisecond)
second, err := FromReference(ref)
if err != nil {
t.Fatal(err)
}
defer second.Close()
if ownedPID(t, second) != pid {
t.Fatal("restart replaced the live background service")
}
after := NewChat(second)
got, err := after.Shells(context.Background(), "ses_background")
if err != nil || len(got) != 1 || got[0].ID != shells[0].ID || got[0].Status != protocol.ChatShellRunning {
t.Fatalf("reattached task %+v %v", got, err)
}
continued, err := after.ShellOutput(context.Background(), query)
if err != nil || continued.Size <= page.Size {
t.Fatalf("task did not progress during restart: %+v -> %+v %v", page, continued, err)
}
t.Logf("backend PID=%d, native shell=%s, output bytes before=%d after=%d", pid, got[0].ID, page.Size, continued.Size)
// Configuration replacement and idle timeout must protect the same task too.
t.Setenv(StageToolOutputMaxBytesEnvKey, "1234")
changed, err := OpenStage(context.Background(), executable, t.TempDir(), "v2")
if err != nil || ownedPID(t, changed) != pid {
t.Fatalf("background service replaced on config update: %v", err)
}
path, _ := StageServiceRegistrationPath()
ownedStageLocksMu.Lock()
ownedStageUsed[path] = time.Now().Add(-time.Hour)
ownedStageLocksMu.Unlock()
if stopIdleOwnedStage(path, time.Minute) {
t.Fatal("idle reaper stopped background task")
}
_ = os.Remove(marker)
terminal, err := after.ShellOutput(context.Background(), query)
if err != nil || terminal.Shell.ID != query.ShellID || terminal.Shell.Status != protocol.ChatShellExited {
t.Fatalf("completion not observable: %+v %v", terminal, err)
}
StopIdleStageService()
waitGone(t, pid)
}