Preserve independent background sessions through owner shutdown and idle cleanup
This commit is contained in:
@@ -0,0 +1,43 @@
|
||||
package opencode
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestOwnedIdleRequiresAuthoritativeBackgroundCompletion(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name, body string
|
||||
status int
|
||||
idle bool
|
||||
}{
|
||||
{"empty envelope", `{"data":[]}`, 200, true},
|
||||
{"empty list", `[]`, 200, true},
|
||||
{"exited", `{"data":[{"status":"exited"}]}`, 200, true},
|
||||
{"running", `{"data":[{"status":"running"}]}`, 200, false},
|
||||
{"missing status", `{"data":[{}]}`, 200, false},
|
||||
{"missing membership", `{}`, 200, false},
|
||||
{"null membership", `{"data":null}`, 200, false},
|
||||
{"invalid response", `bad`, 200, false},
|
||||
{"readback unavailable", ``, 503, false},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
if r.URL.Path == "/api/session/active" {
|
||||
_, _ = w.Write([]byte(`{"data":{}}`))
|
||||
return
|
||||
}
|
||||
w.WriteHeader(tc.status)
|
||||
_, _ = w.Write([]byte(tc.body))
|
||||
}))
|
||||
defer server.Close()
|
||||
c := &Connection{Backend: "v2", URL: server.URL, password: "test"}
|
||||
if got := c.ownedIdle(context.Background()); got != tc.idle {
|
||||
t.Fatalf("idle=%v want%v", got, tc.idle)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -426,8 +426,10 @@ func (c *Connection) ownedInfo(ctx context.Context) (int, bool) {
|
||||
return info.PID, true
|
||||
}
|
||||
|
||||
// ownedIdle reports whether no session is active on this process; any error
|
||||
// counts as busy so active work is never stopped on a guess.
|
||||
// ownedIdle requires both the foreground drain and background shells to be
|
||||
// empty/finished. A completed turn can leave real shells running on this
|
||||
// service; stopping it would kill those tasks across a server restart.
|
||||
// Any missing, malformed or unavailable readback counts as busy.
|
||||
func (c *Connection) ownedIdle(ctx context.Context) bool {
|
||||
probe, cancel := context.WithTimeout(ctx, ownedStageProbeTimeout)
|
||||
defer cancel()
|
||||
@@ -438,11 +440,35 @@ func (c *Connection) ownedIdle(ctx context.Context) bool {
|
||||
var active struct {
|
||||
Data map[string]json.RawMessage `json:"data"`
|
||||
}
|
||||
return json.Unmarshal(data, &active) == nil && active.Data != nil && len(active.Data) == 0
|
||||
if json.Unmarshal(data, &active) != nil || active.Data == nil || len(active.Data) != 0 {
|
||||
return false
|
||||
}
|
||||
data, err = c.nativeRequest(probe, "get", "/api/shell", nil, "")
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
var shells []struct {
|
||||
Status string `json:"status"`
|
||||
}
|
||||
var response struct {
|
||||
Data json.RawMessage `json:"data"`
|
||||
}
|
||||
if json.Unmarshal(data, &response) == nil && response.Data != nil {
|
||||
data = response.Data
|
||||
}
|
||||
if json.Unmarshal(data, &shells) != nil || shells == nil {
|
||||
return false
|
||||
}
|
||||
for _, shell := range shells {
|
||||
if shell.Status != "exited" {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// StopIdleStageService stops the owned stage service on server shutdown when no
|
||||
// session is active on it. The registration (URL and password) is kept, so a
|
||||
// foreground turn or background shell is active on it. The registration (URL and password) is kept, so a
|
||||
// later start reuses the same endpoint and retained references stay valid. A
|
||||
// busy service keeps running so restart recovery can interrupt it properly.
|
||||
func StopIdleStageService() {
|
||||
|
||||
@@ -3,9 +3,12 @@ package opencode
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"git.a-wan.net/alan/pipelines/internal/protocol"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"syscall"
|
||||
"testing"
|
||||
@@ -63,6 +66,57 @@ func runFakeOpenCodeServe() {
|
||||
}
|
||||
_, _ = w.Write([]byte(`{"data":{}}`))
|
||||
}))
|
||||
// A real task in this independent service keeps producing output while no
|
||||
// Pipelines adapter is connected. The marker is its explicit completion gate.
|
||||
outputPath := os.Getenv("FAKE_SHELL_OUTPUT")
|
||||
if outputPath != "" {
|
||||
go func() {
|
||||
for i := 0; ; i++ {
|
||||
if _, err := os.Stat(os.Getenv("FAKE_SHELL_FILE")); err != nil {
|
||||
return
|
||||
}
|
||||
f, err := os.OpenFile(outputPath, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0600)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
_, _ = fmt.Fprintln(f, i)
|
||||
_ = f.Close()
|
||||
time.Sleep(20 * time.Millisecond)
|
||||
}
|
||||
}()
|
||||
}
|
||||
shellInfo := func() map[string]any {
|
||||
status := "exited"
|
||||
if _, err := os.Stat(os.Getenv("FAKE_SHELL_FILE")); err == nil {
|
||||
status = "running"
|
||||
}
|
||||
return map[string]any{"id": "sh_background", "status": status, "metadata": map[string]string{"sessionID": "ses_background"}, "command": "background heartbeat"}
|
||||
}
|
||||
mux.HandleFunc("/api/shell", auth(func(w http.ResponseWriter) {
|
||||
_, running := os.Stat(os.Getenv("FAKE_SHELL_FILE"))
|
||||
_, ran := os.Stat(outputPath)
|
||||
if running == nil || (outputPath != "" && ran == nil) {
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{"data": []any{shellInfo()}})
|
||||
return
|
||||
}
|
||||
_, _ = w.Write([]byte(`{"data":[]}`))
|
||||
}))
|
||||
mux.HandleFunc("/api/shell/sh_background", auth(func(w http.ResponseWriter) { _ = json.NewEncoder(w).Encode(map[string]any{"data": shellInfo()}) }))
|
||||
mux.HandleFunc("/api/shell/sh_background/output", func(w http.ResponseWriter, r *http.Request) {
|
||||
auth(func(w http.ResponseWriter) {
|
||||
data, _ := os.ReadFile(outputPath)
|
||||
cursor, _ := strconv.Atoi(r.URL.Query().Get("cursor"))
|
||||
limit, _ := strconv.Atoi(r.URL.Query().Get("limit"))
|
||||
if cursor > len(data) {
|
||||
cursor = len(data)
|
||||
}
|
||||
end := cursor + limit
|
||||
if end > len(data) {
|
||||
end = len(data)
|
||||
}
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{"data": map[string]any{"output": string(data[cursor:end]), "cursor": end, "size": len(data), "truncated": false}})
|
||||
})(w, r)
|
||||
})
|
||||
mux.HandleFunc("/api/env", auth(func(w http.ResponseWriter) {
|
||||
values := map[string]*string{}
|
||||
for _, key := range strings.Split(os.Getenv("FAKE_ENV_KEYS"), ",") {
|
||||
@@ -87,6 +141,7 @@ func fakeOwnedStageEnv(t *testing.T) string {
|
||||
t.Setenv(OpenCodeServiceStateEnvKey, t.TempDir())
|
||||
t.Setenv(fakeOpenCodeEnv, "1")
|
||||
t.Setenv("FAKE_BUSY_FILE", filepath.Join(t.TempDir(), "busy"))
|
||||
t.Setenv("FAKE_SHELL_FILE", filepath.Join(t.TempDir(), "background"))
|
||||
for _, key := range []string{StageServiceEnvKey, StageToolOutputMaxBytesEnvKey, StageToolOutputMaxLinesEnvKey, StageCompactionEnvKey, StageConfigEnvKey, "OPENCODE_CONFIG_CONTENT", "FAKE_FAIL"} {
|
||||
t.Setenv(key, "")
|
||||
}
|
||||
@@ -287,3 +342,76 @@ func waitGone(t *testing.T, pid int) {
|
||||
}
|
||||
t.Fatalf("process %d still running", pid)
|
||||
}
|
||||
|
||||
// Foreground idle does not imply service idle. This is a process-level restart:
|
||||
// the adapter closes, shutdown cleanup runs, the task keeps writing while the
|
||||
// server is away, and a new adapter reattaches the same native shell/output.
|
||||
func TestBackgroundOnlyStageSurvivesOwnerRestart(t *testing.T) {
|
||||
executable := fakeOwnedStageEnv(t)
|
||||
marker := os.Getenv("FAKE_SHELL_FILE")
|
||||
if err := os.WriteFile(marker, nil, 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
output := filepath.Join(t.TempDir(), "output")
|
||||
t.Setenv("FAKE_SHELL_OUTPUT", output)
|
||||
first, err := OpenStage(context.Background(), executable, t.TempDir(), "v2")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
pid := ownedPID(t, first)
|
||||
ref, err := first.Reference()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
before := NewChat(first)
|
||||
shells, err := before.Shells(context.Background(), "ses_background")
|
||||
if err != nil || len(shells) != 1 {
|
||||
t.Fatalf("native task: %+v %v", shells, err)
|
||||
}
|
||||
query := protocol.ChatShellOutputQuery{SessionID: "ses_background", ShellID: shells[0].ID, Limit: 4096}
|
||||
page, err := before.ShellOutput(context.Background(), query)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
first.Close()
|
||||
StopIdleStageService()
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
second, err := FromReference(ref)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer second.Close()
|
||||
if ownedPID(t, second) != pid {
|
||||
t.Fatal("restart replaced the live background service")
|
||||
}
|
||||
after := NewChat(second)
|
||||
got, err := after.Shells(context.Background(), "ses_background")
|
||||
if err != nil || len(got) != 1 || got[0].ID != shells[0].ID || got[0].Status != protocol.ChatShellRunning {
|
||||
t.Fatalf("reattached task %+v %v", got, err)
|
||||
}
|
||||
continued, err := after.ShellOutput(context.Background(), query)
|
||||
if err != nil || continued.Size <= page.Size {
|
||||
t.Fatalf("task did not progress during restart: %+v -> %+v %v", page, continued, err)
|
||||
}
|
||||
t.Logf("backend PID=%d, native shell=%s, output bytes before=%d after=%d", pid, got[0].ID, page.Size, continued.Size)
|
||||
// Configuration replacement and idle timeout must protect the same task too.
|
||||
t.Setenv(StageToolOutputMaxBytesEnvKey, "1234")
|
||||
changed, err := OpenStage(context.Background(), executable, t.TempDir(), "v2")
|
||||
if err != nil || ownedPID(t, changed) != pid {
|
||||
t.Fatalf("background service replaced on config update: %v", err)
|
||||
}
|
||||
path, _ := StageServiceRegistrationPath()
|
||||
ownedStageLocksMu.Lock()
|
||||
ownedStageUsed[path] = time.Now().Add(-time.Hour)
|
||||
ownedStageLocksMu.Unlock()
|
||||
if stopIdleOwnedStage(path, time.Minute) {
|
||||
t.Fatal("idle reaper stopped background task")
|
||||
}
|
||||
_ = os.Remove(marker)
|
||||
terminal, err := after.ShellOutput(context.Background(), query)
|
||||
if err != nil || terminal.Shell.ID != query.ShellID || terminal.Shell.Status != protocol.ChatShellExited {
|
||||
t.Fatalf("completion not observable: %+v %v", terminal, err)
|
||||
}
|
||||
StopIdleStageService()
|
||||
waitGone(t, pid)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user