From 31d8f4a6d5cf74b3ef6416313278ccb51b324f9f Mon Sep 17 00:00:00 2001 From: Alan Silva Date: Sun, 18 Oct 2026 11:17:01 +0100 Subject: [PATCH 1/7] Distinguish active stage deadlines from harness transport failures Identify configured attempt expiry during V2 active-session idle waits and retain bounded phase, timeout and session diagnostics without claiming a lost response. Preserve genuine operation failures, error identities, redaction, cleanup and admitted-work replay protections. Add stage regressions and scratch-server CLI/websocket retention coverage. Checked packaged templates in examples and the pipeline template generator: none needs updating because timeout configuration and contracts are unchanged. Checked packaged authoring skill references: debugging.md needs a STAGE DEADLINE playbook and clarified transport guidance. Recorded that stale shared documentation in docs/notes/stage-timeout-diagnostics.md for docs-fold; shared documentation was deliberately not edited. Verification: go build ./...; go vet ./...; go test ./...; go test -count=1 ./internal/server ./internal/stage ./internal/pipeline ./internal/protocol; go test -count=1 -run TestStageDeadlineDiagnosticThroughCLIAndWebsocket -v ./internal/server; gofmt -l .; git diff --check. All passed. Live-provider behavior not tested; no deploy or deployed-service restart. --- docs/notes/stage-timeout-diagnostics.md | 42 +++++++ internal/server/stage_deadline_test.go | 138 +++++++++++++++++++++++ internal/stage/model_diagnostic_test.go | 4 +- internal/stage/model_failure.go | 27 ++++- internal/stage/model_run.go | 9 +- internal/stage/opencode.go | 9 +- internal/stage/retry_safety_test.go | 2 +- internal/stage/stage_deadline_test.go | 140 ++++++++++++++++++++++++ 8 files changed, 363 insertions(+), 8 deletions(-) create mode 100644 docs/notes/stage-timeout-diagnostics.md create mode 100644 internal/server/stage_deadline_test.go create mode 100644 internal/stage/stage_deadline_test.go diff --git a/docs/notes/stage-timeout-diagnostics.md b/docs/notes/stage-timeout-diagnostics.md new file mode 100644 index 00000000..a9989418 --- /dev/null +++ b/docs/notes/stage-timeout-diagnostics.md @@ -0,0 +1,42 @@ +# Stage-timeout diagnostics + +## Source behavior + +The V2 active-session wait now reports `STAGE DEADLINE` when the configured +attempt timer expires between successful polls of a running admitted session. +Its bounded diagnostic names the idle-wait phase, configured timeout, retained +session, and that the stage had not finished (active at the last poll). It does +not claim that a completed response was lost. Cancellation is no longer recorded +as a stage deadline. A bare deadline alone is not `HARNESS TRANSPORT`. + +Actual request/HTTP/operation failures keep `HARNESS TRANSPORT`, their sanitized +operation/status evidence and underlying error identities, including when joined +with a deadline. A parent deadline no longer discards an accompanying transport +failure. Final-result validation, budget accounting, cleanup, interrupts, +side effects and replay policy are unchanged. No timeout was increased. + +## Documentation to fold manually + +`internal/author/skill/references/debugging.md` needs a separate stage-deadline +playbook alongside its HARNESS TRANSPORT section: inspect the retained session, +last activity, configured limits and unfinished final-result evidence before +deciding what to do. Do not blindly retry an admitted mutating stage; edits/tool +side effects remain. Explain the distinction between a configured idle-wait +overrun, cancellation, an actual failed request with a deadline, budget failure, +and final-result validation. Do not infer that all deadline errors are transport +failures or that a successful last tool means the stage returned a result. + +## Evidence and deployment + +2026-10-18: installed server build metadata (`go version -m +/Users/alan/.local/share/pipelines/bin/pipelines-server`) identifies revision +`0d1ddc73a444fa24110ba5223646856c8f053d29`. Comparing that revision with this +worktree's initial HEAD showed identical model-failure classification, model +attempt timing, V2 polling and agent wait/cleanup paths. The runner difference +was unrelated session-export HTML-only metadata. + +Regression tests use isolated loopback servers and scratch SQLite databases, +with a controlled V2 CLI fixture for the admitted-active-session deadline and +real loopback HTTP/refused-connection checks for native transport evidence. +This is not a live-provider timeout test. No deployment or deployed-service +restart is part of this item; retained historical runs are not rewritten. diff --git a/internal/server/stage_deadline_test.go b/internal/server/stage_deadline_test.go new file mode 100644 index 00000000..611e0afb --- /dev/null +++ b/internal/server/stage_deadline_test.go @@ -0,0 +1,138 @@ +package server + +import ( + "bytes" + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" + + "git.a-wan.net/alan/pipelines/internal/cli" + "git.a-wan.net/alan/pipelines/internal/pipeline" + "git.a-wan.net/alan/pipelines/internal/protocol" +) + +// Exercise the real V2 CLI/poll path, runner, SQLite retention and websocket CLI +// against an isolated loopback server/database, never the configured deployment. +func TestStageDeadlineDiagnosticThroughCLIAndWebsocket(t *testing.T) { + ntfy := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write([]byte(`{"id":"scratch-receipt","event":"message"}`)) + })) + defer ntfy.Close() + t.Setenv("PIPELINES_NTFY_URL", ntfy.URL+"/scratch") + t.Setenv("PIPELINES_NTFY_TOKEN", "") + t.Setenv("PIPELINES_NTFY_TOKEN_FILE", "") + for _, mode := range []string{"active", "http", "connection"} { + t.Run(mode, func(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + dir := t.TempDir() + executable := filepath.Join(dir, "opencode") + script := `#!/bin/sh +if [ "$1" = "--help" ]; then echo ' api'; exit 0; fi +printf '%s %s\n' "$2" "$3" >> "@STATE@/calls" +case "$3" in + /api/session) echo '{"data":{"id":"ses_deadline"}}' ;; + */prompt) echo 'edited once' >> "@STATE@/edited"; echo '{}' ;; + /api/session/active) + if [ -f "@STATE@/interrupted" ]; then echo '{"data":{}}'; exit 0; fi + case "@MODE@" in + active) echo '{"data":{"ses_deadline":{"type":"running"}}}' ;; + http) echo 'HTTP 503 token=private-token' >&2; exit 1 ;; + connection) echo 'connection refused token=private-token' >&2; exit 1 ;; + esac ;; + */interrupt) touch "@STATE@/interrupted"; echo '{}' ;; + */message*) echo '{"data":[{"content":[{"type":"text","text":"still working"}]}]}' ;; + /api/session/ses_deadline) echo '{"data":{"id":"ses_deadline"}}' ;; + *) echo '{}' ;; +esac +` + script = strings.ReplaceAll(strings.ReplaceAll(script, "@STATE@", dir), "@MODE@", mode) + if err := os.WriteFile(executable, []byte(script), 0700); err != nil { + t.Fatal(err) + } + t.Setenv("PIPELINES_OPENCODE", executable) + store, err := pipeline.Open(filepath.Join(dir, "runs.db")) + if err != nil { + t.Fatal(err) + } + defer store.Close() + source := pipeline.Template([]string{"worker"}, true) + source = strings.Replace(source, "```tools\n```", "```tools\nedit\n```", 1) + source = strings.Replace(source, "timeout=1h", "timeout=500ms", 1) + if _, err := store.Submit(ctx, "deadline", source); err != nil { + t.Fatal(err) + } + handler := Handler(ctx, store) + defer func() { cancel(); WaitRuns(handler) }() + srv := httptest.NewServer(handler) + defer srv.Close() + url := "ws" + strings.TrimPrefix(srv.URL, "http") + "/connect" + var out, stderr bytes.Buffer + app := cli.App{Out: &out, Err: &stderr, In: strings.NewReader("")} + if err := app.Run(ctx, []string{"--url", url, "run", "deadline", "task", "--yes", "--json", "--wait", "--timeout", "10s", "--workspace", dir}); err == nil || !strings.Contains(err.Error(), "run ended failed") { + t.Fatalf("run not failed: %v, %s, %s", err, out.String(), stderr.String()) + } + var response protocol.Message + if err := json.Unmarshal(out.Bytes(), &response); err != nil { + t.Fatal(err, out.String()) + } + detail := response.RunDetail + if detail == nil || detail.Status != "failed" || len(detail.Steps) != 1 { + t.Fatal("missing failed run/step", out.String()) + } + step := detail.Steps[0] + if step.SessionID != "ses_deadline" || step.AgentActive || step.Output["error"] != step.Diagnostic || step.Status != "failed" || detail.Diagnostic != step.Diagnostic { + t.Fatalf("incorrect retained failure: %+v", detail) + } + wants := []string{"HARNESS TRANSPORT", "get /api/session/active", "CLI exit code 1"} + if mode == "active" { + wants = []string{"STAGE DEADLINE", "timeout 500ms", "waiting for session idle", "ses_deadline", "stage had not finished"} + if strings.Contains(detail.Diagnostic, "HARNESS TRANSPORT") || strings.Contains(detail.Diagnostic, "response lost") { + t.Fatal("active stage overrun reported as transport", detail.Diagnostic) + } + if strings.Contains(detail.Diagnostic, "\n") { + t.Fatal("duplicate context error appended to bounded stage diagnostic", detail.Diagnostic) + } + for _, want := range []string{"opencode_failures", "wait for session idle", "timeout 500ms", "retained_session", "still working"} { + if !strings.Contains(step.Transcript, want) { + t.Fatalf("missing retained evidence %q: %s", want, step.Transcript) + } + } + } else { + if !strings.Contains(step.Transcript, "[REDACTED]") { + t.Fatal("request failure evidence missing", step.Transcript) + } + if mode == "http" { + wants = append(wants, "HTTP 503") + } else if !strings.Contains(step.Transcript, "connection refused") { + t.Fatal("underlying connection evidence missing", step.Transcript) + } + } + for _, want := range wants { + if !strings.Contains(detail.Diagnostic, want) { + t.Fatalf("missing diagnostic %q: %s", want, detail.Diagnostic) + } + } + if strings.Contains(detail.Diagnostic+step.Transcript, "private-token") { + t.Fatal("credential leaked in retained diagnostic/evidence") + } + edited, err := os.ReadFile(filepath.Join(dir, "edited")) + if err != nil || string(edited) != "edited once\n" { + t.Fatalf("admitted tool work replayed or lost: %q, %v", edited, err) + } + if _, err := os.Stat(filepath.Join(dir, "interrupted")); err != nil { + t.Fatal("cleanup did not interrupt", err) + } + stored, err := store.GetRun(ctx, detail.ID) + if err != nil || stored.Diagnostic != detail.Diagnostic || stored.Steps[0].Transcript != step.Transcript { + t.Fatal("diagnostic/evidence not persisted in scratch database", err) + } + t.Logf("scratch run %s: %s", mode, stored.Diagnostic) + }) + } +} diff --git a/internal/stage/model_diagnostic_test.go b/internal/stage/model_diagnostic_test.go index e61a5347..00be8b01 100644 --- a/internal/stage/model_diagnostic_test.go +++ b/internal/stage/model_diagnostic_test.go @@ -36,8 +36,8 @@ func TestModelFailureCategoriesAreSanitizedAndDistinct(t *testing.T) { if typed := wrapModelStageFailure("worker", &openCodeError{operation: "get /api/session", status: 503}); !strings.Contains(typed.Error(), "HARNESS TRANSPORT") || !errors.As(typed, new(*openCodeError)) { t.Fatalf("typed harness error was not classified: %v", typed) } - if deadline := wrapModelStageFailure("worker", context.DeadlineExceeded); !strings.Contains(deadline.Error(), "HARNESS TRANSPORT") || !errors.Is(deadline, context.DeadlineExceeded) { - t.Fatalf("context deadline was not classified as harness: %v", deadline) + if deadline := wrapModelStageFailure("worker", context.DeadlineExceeded); strings.Contains(deadline.Error(), "HARNESS TRANSPORT") || !errors.Is(deadline, context.DeadlineExceeded) { + t.Fatalf("bare context deadline was misclassified or lost: %v", deadline) } if modelFailureCategory(nil) != "" || wrapModelStageFailure("worker", nil) != nil { t.Fatal("nil failure was classified") diff --git a/internal/stage/model_failure.go b/internal/stage/model_failure.go index 6cef2b1b..2c6af418 100644 --- a/internal/stage/model_failure.go +++ b/internal/stage/model_failure.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "time" ) // Retained model-stage failure categories. A strict final-result validation @@ -13,8 +14,27 @@ import ( const ( finalResultValidation = "FINAL RESULT VALIDATION" harnessTransport = "HARNESS TRANSPORT" + stageDeadline = "STAGE DEADLINE" ) +// stageTimeoutCause identifies this attempt's configured timer, not a parent +// cancellation or a request's own deadline. It is carried through budget contexts. +type stageTimeoutCause struct{ timeout time.Duration } + +func (e *stageTimeoutCause) Error() string { return "configured stage timeout reached" } +func (e *stageTimeoutCause) Unwrap() error { return context.DeadlineExceeded } + +type stageDeadlineError struct { + timeout time.Duration + session string +} + +func (e *stageDeadlineError) Error() string { + return fmt.Sprintf("configured stage deadline reached (timeout %s) while waiting for session idle; retained OpenCode session %.128s was still active at the last poll; stage had not finished", e.timeout, RedactEvidence(e.session)) +} + +func (e *stageDeadlineError) Unwrap() error { return context.DeadlineExceeded } + // finalResultGuidance is a fixed sanitized sentence appended only to validation // diagnostics. It states the retained side effects and the cost of a full retry. const finalResultGuidance = "completed edits and tool side effects are retained and not undone; a full retry repeats work" @@ -51,11 +71,14 @@ func (c *PiClient) workAdmitted() bool { return c.WorkStarted } // the label stays sanitized and bounded. Validation is checked first: a // structured-output failure is never reported as a transport failure and vice versa. func modelFailureCategory(err error) string { + var deadline *stageDeadlineError switch { case errors.Is(err, ErrInvalidModelResult): return finalResultValidation - case harnessTransportFailure(err), errors.Is(err, context.DeadlineExceeded): + case harnessTransportFailure(err): return harnessTransport + case errors.As(err, &deadline): + return stageDeadline default: return "" } @@ -73,6 +96,8 @@ func wrapModelStageFailure(key string, err error) error { return fmt.Errorf("model stage %s: %s: %w; %s", key, finalResultValidation, err, finalResultGuidance) case harnessTransport: return fmt.Errorf("model stage %s: %s: %w", key, harnessTransport, err) + case stageDeadline: + return fmt.Errorf("model stage %s: %s: %w", key, stageDeadline, err) default: return fmt.Errorf("model stage %s: %w", key, err) } diff --git a/internal/stage/model_run.go b/internal/stage/model_run.go index e1979706..41300243 100644 --- a/internal/stage/model_run.go +++ b/internal/stage/model_run.go @@ -61,7 +61,7 @@ func (m ModelStage) Run(ctx context.Context, def Definition, input Values) (Resu return Result{}, err } spec.Config.Budget.Attempt() - attemptCtx, cancel := context.WithTimeout(ctx, spec.Config.Timeout) + attemptCtx, cancel := context.WithTimeoutCause(ctx, spec.Config.Timeout, &stageTimeoutCause{timeout: spec.Config.Timeout}) text, generateErr := client.Generate(attemptCtx, spec.Config, prompt) if generateErr == nil && attemptCtx.Err() != nil { generateErr = attemptCtx.Err() @@ -96,11 +96,16 @@ func (m ModelStage) Run(ctx context.Context, def Definition, input Values) (Resu } } else { err = generateErr - if attemptCtx.Err() != nil && !errors.Is(err, ErrBudgetExceeded) && !errors.Is(err, ErrBudgetAccounting) { + // Do not append the same context error twice: the wait/transport + // error may already preserve it, with a more precise diagnostic. + if attemptCtx.Err() != nil && !errors.Is(err, attemptCtx.Err()) && !errors.Is(err, ErrBudgetExceeded) && !errors.Is(err, ErrBudgetAccounting) { err = errors.Join(err, attemptCtx.Err()) } cancel() if ctx.Err() != nil { + if harnessTransportFailure(err) { + return Result{}, wrapModelStageFailure(def.Key, err) + } return Result{}, ctx.Err() } if errors.Is(err, ErrBudgetExceeded) || errors.Is(err, ErrBudgetAccounting) || errors.Is(err, ErrStageCleanup) { diff --git a/internal/stage/opencode.go b/internal/stage/opencode.go index 38d7722e..bdf28bfd 100644 --- a/internal/stage/opencode.go +++ b/internal/stage/opencode.go @@ -217,6 +217,11 @@ func (c OpenCodeClient) waitForSessionIdle(ctx context.Context, sessionID string select { case <-ctx.Done(): timer.Stop() + waitErr := fmt.Errorf("waiting for session idle; retained OpenCode session %.128s; stage had not finished: %w", RedactEvidence(sessionID), ctx.Err()) + var timeout *stageTimeoutCause + if errors.Is(ctx.Err(), context.DeadlineExceeded) && errors.As(context.Cause(ctx), &timeout) { + waitErr = &stageDeadlineError{timeout: timeout.timeout, session: sessionID} + } // A settled session is the only success signal; retain one bounded // sanitized record so a timed-out wait is not an empty transcript. c.Operations.record(OperationEvidence{ @@ -225,9 +230,9 @@ func (c OpenCodeClient) waitForSessionIdle(ctx context.Context, sessionID string Executable: opencode.Executable(), Directory: c.Directory, ExitCode: -1, - Diagnostic: "stage deadline reached while waiting for the agent session to become idle", + Diagnostic: waitErr.Error(), }, started) - return ctx.Err() + return waitErr case <-timer.C: } } diff --git a/internal/stage/retry_safety_test.go b/internal/stage/retry_safety_test.go index f5f812e9..f0a40541 100644 --- a/internal/stage/retry_safety_test.go +++ b/internal/stage/retry_safety_test.go @@ -251,7 +251,7 @@ func TestWaitForSessionIdleRecordsOneTimeoutEvidence(t *testing.T) { t.Fatalf("expected exactly one timeout evidence, got %d", len(log.Failures)) } entry := log.Failures[0] - if entry.Operation != "wait for session idle" || !strings.Contains(entry.Diagnostic, "deadline reached") { + if entry.Operation != "wait for session idle" || !strings.Contains(entry.Diagnostic, "context deadline exceeded") { t.Fatalf("timeout evidence not sanitized as expected: %+v", entry) } clean := &OperationLog{} diff --git a/internal/stage/stage_deadline_test.go b/internal/stage/stage_deadline_test.go new file mode 100644 index 00000000..2f2f3aea --- /dev/null +++ b/internal/stage/stage_deadline_test.go @@ -0,0 +1,140 @@ +package stage + +import ( + "context" + "errors" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "git.a-wan.net/alan/pipelines/internal/opencode" +) + +func TestConfiguredStageDeadlineWhileSessionActive(t *testing.T) { + def := modelDefinition() + def.Blocks["tools"] = "edit" + def.Configs["timeout"] = "20ms" + def.Configs["retries"] = "2" + log := &OperationLog{} + creates, prompts, interrupts := 0, 0, 0 + agent := &AgentClient{OpenCodeClient: OpenCodeClient{Backend: "v2", Operations: log, sessionPollInterval: time.Hour}} + agent.command = func(_ context.Context, _ string, args ...string) ([]byte, error) { + switch args[2] { + case "/api/session": + creates++ + return []byte(`{"data":{"id":"ses_deadline"}}`), nil + case "/api/session/ses_deadline/prompt": + prompts++ + return []byte(`{}`), nil + case "/api/session/active": + return []byte(`{"data":{"ses_deadline":{"type":"running"}}}`), nil + case "/api/session/ses_deadline/interrupt": + interrupts++ + return []byte(`{}`), nil + default: + // Best-effort timeout readback/usage cannot convert the failure to success. + return []byte(`{}`), nil + } + } + result, err := (ModelStage{Client: agent}).Run(context.Background(), def, nil) + if !errors.Is(err, context.DeadlineExceeded) || modelFailureCategory(err) != stageDeadline || errors.Is(err, ErrHarnessTransport) { + t.Fatalf("deadline identity/category: %v", err) + } + for _, want := range []string{"STAGE DEADLINE", "worker", "timeout 20ms", "waiting for session idle", "ses_deadline", "stage had not finished"} { + if !strings.Contains(err.Error(), want) { + t.Fatalf("missing %q: %v", want, err) + } + } + if result.Output != nil || creates != 1 || prompts != 1 || interrupts != 1 || !agent.WorkStarted { + t.Fatalf("work replayed, cleanup skipped or incomplete turn accepted: %+v creates=%d prompts=%d interrupts=%d", result, creates, prompts, interrupts) + } + if len(log.Failures) != 1 || !strings.Contains(log.Failures[0].Diagnostic, "timeout 20ms") { + t.Fatalf("deadline evidence missing: %+v", log.Failures) + } +} + +func TestWaitCancellationIsNotConfiguredStageDeadline(t *testing.T) { + for _, cause := range []error{context.Canceled, ErrBudgetExceeded, ErrBudgetAccounting} { + t.Run(cause.Error(), func(t *testing.T) { + attempt, stop := context.WithTimeoutCause(context.Background(), time.Hour, &stageTimeoutCause{timeout: time.Hour}) + defer stop() + ctx, cancel := context.WithCancelCause(attempt) + log := &OperationLog{} + client := OpenCodeClient{Operations: log, command: func(context.Context, string, ...string) ([]byte, error) { + cancel(cause) + return []byte(`{"data":{"ses_wait":{"type":"running"}}}`), nil + }} + err := client.waitForSessionIdle(ctx, "ses_wait") + if !errors.Is(err, context.Canceled) || modelFailureCategory(err) != "" || len(log.Failures) != 1 { + t.Fatalf("cancellation misclassified: %v, %+v", err, log.Failures) + } + if strings.Contains(log.Failures[0].Diagnostic, "deadline") { + t.Fatalf("cancellation recorded as deadline: %+v", log.Failures) + } + }) + } +} + +func TestDeadlineWithTransportRetainsUnderlyingIdentity(t *testing.T) { + transport := &openCodeError{operation: "get /api/session/active", status: 503, cause: errors.New("request failed")} + err := wrapModelStageFailure("worker", errors.Join(&stageDeadlineError{timeout: time.Second, session: "ses_wait"}, transport)) + var got *openCodeError + if modelFailureCategory(err) != harnessTransport || !errors.Is(err, context.DeadlineExceeded) || !errors.As(err, &got) || got != transport || !errors.Is(err, transport.cause) { + t.Fatalf("combined failure lost transport/deadline identity: %v", err) + } + if !strings.Contains(err.Error(), "HTTP 503") || !strings.Contains(err.Error(), "get /api/session/active") { + t.Fatalf("combined failure lost operation/status: %v", err) + } + bounded := (&stageDeadlineError{timeout: time.Hour, session: strings.Repeat("s", 10000) + " Bearer private-token"}).Error() + if len(bounded) > 512 || strings.Contains(bounded, "private-token") { + t.Fatal("deadline diagnostic was not bounded/redacted") + } +} + +func TestParentDeadlinePreservesModelTransportFailure(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Millisecond) + defer cancel() + transport := &openCodeError{operation: "get /api/session/active", status: 504, cause: errors.New("request failed")} + client := generatorFunc(func(ctx context.Context, _ ModelConfig, _ string) (string, error) { + <-ctx.Done() + return "", errors.Join(ctx.Err(), transport) + }) + _, err := (ModelStage{Client: client}).Run(ctx, modelDefinition(), nil) + var got *openCodeError + if !errors.Is(err, context.DeadlineExceeded) || !errors.As(err, &got) || got != transport || modelFailureCategory(err) != harnessTransport { + t.Fatalf("parent deadline erased underlying request failure: %v", err) + } +} + +func TestNativeHTTPAndConnectionFailuresRemainTransport(t *testing.T) { + for _, mode := range []string{"http", "connection"} { + t.Run(mode, func(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusServiceUnavailable) + _, _ = w.Write([]byte(`{"message":"service unavailable","token":"private-token"}`)) + })) + defer srv.Close() + if mode == "connection" { + srv.Close() + } + log := &OperationLog{} + client := OpenCodeClient{Backend: "v1", Operations: log, connection: &opencode.Connection{Backend: "v1", URL: srv.URL}} + _, err := client.call(context.Background(), "get", "/api/session", nil) + var typed *openCodeError + if !errors.As(err, &typed) || modelFailureCategory(err) != harnessTransport || len(log.Failures) != 1 { + t.Fatalf("real request failure misclassified: %v %+v", err, log.Failures) + } + if mode == "http" && (typed.status != 503 || log.Failures[0].HTTPStatus != 503 || !strings.Contains(log.Failures[0].Stdout, "service unavailable")) { + t.Fatalf("HTTP status/evidence lost: %v %+v", err, log.Failures) + } + if mode == "connection" && (log.Failures[0].Diagnostic != "OpenCode native API request failed" || log.Failures[0].NativeOperation != "GET /session") { + t.Fatalf("connection evidence lost: %+v", log.Failures) + } + if strings.Contains(err.Error()+log.Transcript(""), "private-token") { + t.Fatal("credential leaked in transport diagnostic/evidence") + } + }) + } +} From 31aec05f99e71ac59e60e3727863c25538620282 Mon Sep 17 00:00:00 2001 From: Alan Silva Date: Sun, 18 Oct 2026 12:02:37 +0100 Subject: [PATCH 2/7] Scope stage artifact access by run visit Derive validated same-run visit artifact capabilities in the runner and pass them to OpenCode and Pi clients. Permit supported file tools to read completed prior visits and write only the current visit, preserving workspace, secret, database, and shell restrictions.\n\nChecked packaged templates and authoring skill references: neither needs updating in this item; docs/notes/scoped-artifact-access.md records the shared documentation staleness for the docs-fold writer. --- docs/notes/scoped-artifact-access.md | 20 ++++++++ internal/server/artifacts_test.go | 55 +++++++++++++++++++++ internal/server/runner.go | 67 ++++++++++++++++++++++++-- internal/stage/agent.go | 17 ++++--- internal/stage/pi.go | 3 +- internal/stage/pi_policy.mjs | 35 +++++++++++++- internal/stage/pi_policy_test.go | 72 ++++++++++++++++++++++++++++ internal/stage/tools.go | 34 +++++++++++++ internal/stage/tools_test.go | 35 ++++++++++++++ 9 files changed, 325 insertions(+), 13 deletions(-) create mode 100644 docs/notes/scoped-artifact-access.md create mode 100644 internal/stage/pi_policy_test.go diff --git a/docs/notes/scoped-artifact-access.md b/docs/notes/scoped-artifact-access.md new file mode 100644 index 00000000..75943280 --- /dev/null +++ b/docs/notes/scoped-artifact-access.md @@ -0,0 +1,20 @@ +# Scoped stage artifact access + +## What this item changed + +The server now derives one exact artifact capability for an OpenCode or Pi +model-stage visit: its own directory is writable when the edit tool is present, +and its own plus completed earlier visit directories in the same run are +readable when a file-reading tool is present. No artifact root, sibling, future +visit, other run, traversal path, or symlink escape is granted. The scope is +empty if the runner cannot validate its artifact root. + +Pi retains its lease and explicit extension policy; unrestricted shell remains +outside this file-tool boundary and is not an OS sandbox. + +## Shared documents this change makes stale + +- `docs/stage-artifacts.md` and `internal/author/skill/references/operations.md` + should explain that model-stage artifact directories are usable by the + supported file tools only within the exact same-run visit scope, not as a + general external-directory permission. diff --git a/internal/server/artifacts_test.go b/internal/server/artifacts_test.go index 573b851b..6c4c7804 100644 --- a/internal/server/artifacts_test.go +++ b/internal/server/artifacts_test.go @@ -105,6 +105,61 @@ func TestArtifactsScanMissingDirectoryIsEmpty(t *testing.T) { } } +func TestArtifactAccessUsesOnlyCurrentAndEarlierVisits(t *testing.T) { + root := t.TempDir() + r := &runner{artifactsRoot: root} + for _, directory := range []string{"7/1", "7/2", "7/3", "8/1"} { + if err := os.MkdirAll(filepath.Join(root, directory), 0700); err != nil { + t.Fatal(err) + } + } + access := r.artifactAccess(7, 2, []string{"1", "3", "../8/1"}, filepath.Join(root, "7", "2")) + current, _ := filepath.EvalSymlinks(filepath.Join(root, "7", "2")) + prior, _ := filepath.EvalSymlinks(filepath.Join(root, "7", "1")) + if access.Write != current || len(access.Read) != 2 || access.Read[0] != access.Write || access.Read[1] != prior { + t.Fatalf("artifact scope = %#v", access) + } + if got := r.artifactAccess(7, 2, nil, filepath.Join(root, "8", "1")); got.Write != "" || len(got.Read) != 0 { + t.Fatalf("other-run current scope accepted: %#v", got) + } + if got := r.artifactAccess(7, 2, nil, filepath.Join(root, "7", "3")); got.Write != "" || len(got.Read) != 0 { + t.Fatalf("sibling current scope accepted: %#v", got) + } + if err := os.WriteFile(filepath.Join(root, "7", "4"), []byte("not a directory"), 0600); err != nil { + t.Fatal(err) + } + if got := r.artifactAccess(7, 4, nil, filepath.Join(root, "7", "4")); got.Write != "" || len(got.Read) != 0 { + t.Fatalf("file current scope accepted: %#v", got) + } + if err := os.Symlink(filepath.Join(root, "7", "2"), filepath.Join(root, "7", "5")); err != nil { + t.Fatal(err) + } + if got := r.artifactAccess(7, 5, nil, filepath.Join(root, "7", "5")); got.Write != "" || len(got.Read) != 0 { + t.Fatalf("symlinked current scope accepted: %#v", got) + } +} + +func TestRunnerPassesDerivedArtifactAccessToAgentClient(t *testing.T) { + root := t.TempDir() + for _, directory := range []string{"7/1", "7/2"} { + if err := os.MkdirAll(filepath.Join(root, directory), 0700); err != nil { + t.Fatal(err) + } + } + r := &runner{artifactsRoot: root} + client := r.agentClient(7, 2, []string{"1", "3"}, t.TempDir(), filepath.Join(root, "7", "2"), &stage.OperationLog{}, nil, []string{"read", "edit"}) + if client.ArtifactAccess.Write == "" || len(client.ArtifactAccess.Read) != 2 { + t.Fatalf("runner did not pass scoped artifact access: %#v", client.ArtifactAccess) + } + if client.ArtifactAccess.Read[1] == filepath.Join(root, "7", "3") { + t.Fatal("runner passed a future visit to the client") + } + piClient := r.piClient(7, 2, []string{"1", "3"}, "pi", t.TempDir(), filepath.Join(root, "7", "2"), nil, []string{"read", "edit"}) + if piClient.ArtifactAccess.Write != client.ArtifactAccess.Write || len(piClient.ArtifactAccess.Read) != len(client.ArtifactAccess.Read) || piClient.ArtifactAccess.Read[1] != client.ArtifactAccess.Read[1] { + t.Fatalf("runner passed different artifact scope to Pi client: %#v", piClient.ArtifactAccess) + } +} + // runWithArtifact executes one real run whose stage writes a file into the // runner-provided artifacts directory. It returns the live handler and the // recorded step so tests can serve the artifact over the real HTTP endpoint. diff --git a/internal/server/runner.go b/internal/server/runner.go index 0c532247..11471617 100644 --- a/internal/server/runner.go +++ b/internal/server/runner.go @@ -580,7 +580,8 @@ type runState struct { capturedSeen map[string]bool // completed counts the run's finished steps, so a delta can report the // running total without re-reading the store. - completed int + completed int + artifactDirs []string } func newRunState(key string, input stage.Values) *runState { @@ -606,10 +607,67 @@ func rebuildRunState(detail *protocol.RunDetail, decided protocol.RunStep) *runS // Seed the UI-surface accumulator from the persisted evidence so a // resumed run reaches the same skip decision with no extra query. state.recordChanges(step.FileChanges, step.Artifacts) + // Only completed, recorded visits are eligible for a later stage's + // review capability. The path is reconstructed by the runner below. + if step.Status == "completed" { + state.artifactDirs = append(state.artifactDirs, filepath.Join("", strconv.Itoa(step.Sequence))) + } } return state } +// artifactAccess derives exact visit directories; it never consumes a path from +// a definition or model result. An unreadable or symlinked root fails closed. +func (r *runner) artifactAccess(runID int64, sequence int, prior []string, current string) stage.ArtifactAccess { + root, err := filepath.EvalSymlinks(r.artifactsRoot) + if err != nil { + return stage.ArtifactAccess{} + } + runRoot := filepath.Join(root, strconv.FormatInt(runID, 10)) + visitDirectory := func(visit int) (string, bool) { + if visit <= 0 { + return "", false + } + directory := filepath.Join(runRoot, strconv.Itoa(visit)) + resolved, err := filepath.EvalSymlinks(directory) + if err != nil || resolved != directory { + return "", false + } + info, err := os.Stat(resolved) + if err != nil || !info.IsDir() { + return "", false + } + return resolved, true + } + access := stage.ArtifactAccess{} + if directory, ok := visitDirectory(sequence); ok { + resolved, err := filepath.EvalSymlinks(current) + if err != nil || resolved != directory { + return access + } + access.Write = directory + access.Read = append(access.Read, directory) + } + for _, priorSequence := range prior { + visit, err := strconv.Atoi(priorSequence) + if err != nil || visit <= 0 || visit >= sequence { + continue + } + if directory, ok := visitDirectory(visit); ok { + access.Read = append(access.Read, directory) + } + } + return access +} + +func (r *runner) agentClient(runID int64, sequence int, prior []string, workspace, artifactsDir string, operations *stage.OperationLog, writePaths, tools []string) *stage.AgentClient { + return &stage.AgentClient{OpenCodeClient: stage.OpenCodeClient{Directory: workspace, Operations: operations, ArtifactsDir: artifactsDir}, WritePaths: writePaths, Tools: tools, ArtifactAccess: r.artifactAccess(runID, sequence, prior, artifactsDir)} +} + +func (r *runner) piClient(runID int64, sequence int, prior []string, executable, workspace, artifactsDir string, writePaths, tools []string) *stage.PiClient { + return &stage.PiClient{Executable: executable, Directory: workspace, Tools: tools, WritePaths: writePaths, ArtifactsDir: artifactsDir, ArtifactAccess: r.artifactAccess(runID, sequence, prior, artifactsDir)} +} + // serverChangedFilesField is the reserved run-input memory field under which the // runner publishes the server-captured changed paths. A stage selects it with // `"server_changed_files":"input.server_changed_files"`, alongside another @@ -977,7 +1035,7 @@ func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, na } } operations := &stage.OperationLog{} - client := &stage.AgentClient{OpenCodeClient: stage.OpenCodeClient{Directory: workspace, Operations: operations, ArtifactsDir: artifactsDir}, WritePaths: request.WritePaths, Tools: tools} + client := r.agentClient(id, sequence, state.artifactDirs, workspace, artifactsDir, operations, request.WritePaths, tools) client.OnSession = func(session string, active bool) error { c, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() @@ -1033,7 +1091,7 @@ func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, na err = fmt.Errorf("Pi is not configured on execution server") return } - piClient = &stage.PiClient{Executable: profile.Executable, Directory: workspace, Tools: tools, WritePaths: request.WritePaths, ArtifactsDir: artifactsDir} + piClient = r.piClient(id, sequence, state.artifactDirs, profile.Executable, workspace, artifactsDir, request.WritePaths, tools) piClient.OnOwnership = func(session, reference string, active bool) error { c, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() @@ -1148,6 +1206,9 @@ func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, na state.recordChanges(step.FileChanges, step.Artifacts) if step.Status == "completed" { state.completed++ + if artifactsDir != "" { + state.artifactDirs = append(state.artifactDirs, strconv.Itoa(sequence)) + } } // Step finished: report its outcome, routing target, diagnostic and the // artifacts recorded for the visit, plus the run's running step count. diff --git a/internal/stage/agent.go b/internal/stage/agent.go index e0c3974d..d3109d20 100644 --- a/internal/stage/agent.go +++ b/internal/stage/agent.go @@ -14,13 +14,16 @@ import ( // before any prompt is admitted, so restart recovery can interrupt the agent. type AgentClient struct { OpenCodeClient - WritePaths []string - Tools []string // nil is the legacy manual approval bundle; empty denies all - OnSession func(string, bool) error - SessionID string - Transcript string - Policy ToolPolicy - WorkStarted bool + WritePaths []string + Tools []string // nil is the legacy manual approval bundle; empty denies all + // ArtifactAccess is supplied only by the runner. Write is this visit's + // directory; Read contains this and completed earlier visits in the same run. + ArtifactAccess ArtifactAccess + OnSession func(string, bool) error + SessionID string + Transcript string + Policy ToolPolicy + WorkStarted bool } func (c OpenCodeClient) Interrupt(ctx context.Context, id string) error { diff --git a/internal/stage/pi.go b/internal/stage/pi.go index 6d93e4d7..3e0b1bf1 100644 --- a/internal/stage/pi.go +++ b/internal/stage/pi.go @@ -28,6 +28,7 @@ type PiClient struct { // ArtifactsDir is exported to the spawned Pi process as // PIPELINES_ARTIFACTS_DIR so its tools can write stage artifacts. ArtifactsDir string + ArtifactAccess ArtifactAccess Policy ToolPolicy OnSession func(string, bool) error OnOwnership func(string, string, bool) error @@ -128,7 +129,7 @@ func (c *PiClient) Generate(ctx context.Context, config ModelConfig, prompt stri if err != nil { return "", err } - policy := map[string]any{"directory": c.Directory, "tools": tools, "paths": c.WritePaths, "edit": c.Policy.Edit, "shell": c.Policy.Shell} + policy := map[string]any{"directory": c.Directory, "tools": tools, "paths": c.WritePaths, "edit": c.Policy.Edit, "shell": c.Policy.Shell, "artifact_write": c.ArtifactAccess.Write, "artifact_read": c.ArtifactAccess.Read} if c.WritePaths == nil { policy["paths"] = []string{} } diff --git a/internal/stage/pi_policy.mjs b/internal/stage/pi_policy.mjs index aec60e93..03be3440 100644 --- a/internal/stage/pi_policy.mjs +++ b/internal/stage/pi_policy.mjs @@ -8,6 +8,24 @@ export function permit(policy, tool, args) { if (policy.shell !== null && !policy.shell.includes(args.command)) return "Shell command is not permitted"; return null; } + const artifact = (absolute, write) => { + const permitted = write ? [policy.artifact_write] : policy.artifact_read; + for (const directory of permitted) { + if (typeof directory !== "string" || !path.isAbsolute(directory)) continue; + let parent = absolute; + while (!fs.existsSync(parent)) { + const next = path.dirname(parent); + if (next === parent) break; + parent = next; + } + if (!fs.existsSync(parent)) continue; + const resolved = fs.realpathSync(parent); + const target = path.join(resolved, path.relative(parent, absolute)); + const root = fs.realpathSync(directory); + if (target === root || target.startsWith(root + path.sep)) return true; + } + return false; + }; if (tool === "edit" || tool === "write") { if (typeof args.path !== "string") return "Missing file path"; if (args.path.startsWith("~") || args.path.startsWith("@")) return "Use a literal path, not a harness path shortcut"; @@ -20,7 +38,10 @@ export function permit(policy, tool, args) { } const resolved = fs.realpathSync(parent); const root = fs.realpathSync(policy.directory); - if (resolved !== root && !resolved.startsWith(root + path.sep)) return "Edit escapes run directory"; + if (resolved !== root && !resolved.startsWith(root + path.sep)) { + if (!artifact(absolute, true)) return "Edit escapes run directory"; + return null; + } const relative = path.relative(path.resolve(policy.directory), absolute).split(path.sep).join("/"); const canonical = path.relative(root, path.join(resolved, path.relative(parent, absolute))).split(path.sep).join("/"); if (relative.startsWith("../") || path.isAbsolute(relative)) return "Edit escapes run directory"; @@ -31,9 +52,19 @@ export function permit(policy, tool, args) { }); if (policy.edit !== null && (!matches(relative) || !matches(canonical))) return "Edit is outside stage patterns"; } - if (tool === "read" && typeof args.path === "string") { + if (["read", "grep", "find"].includes(tool) && typeof args.path === "string") { const name = path.basename(args.path); if (/^(.*\.env(?:\..*)?|pipelines\.db.*)$/.test(name)) return "Protected file cannot be read"; + const absolute = path.resolve(policy.directory, args.path); + let parent = absolute; + while (!fs.existsSync(parent)) { + const next = path.dirname(parent); + if (next === parent) return "Cannot resolve read path"; + parent = next; + } + const resolved = fs.realpathSync(parent); + const root = fs.realpathSync(policy.directory); + if (resolved !== root && !resolved.startsWith(root + path.sep) && !artifact(absolute, false)) return "Read escapes run directory"; } return null; } diff --git a/internal/stage/pi_policy_test.go b/internal/stage/pi_policy_test.go new file mode 100644 index 00000000..6b1018a6 --- /dev/null +++ b/internal/stage/pi_policy_test.go @@ -0,0 +1,72 @@ +package stage + +import ( + "encoding/json" + "os" + "os/exec" + "path/filepath" + "testing" +) + +func TestPiPolicyArtifactScope(t *testing.T) { + if _, err := exec.LookPath("node"); err != nil { + t.Skip("node is required to exercise the Pi policy extension") + } + root := t.TempDir() + workspace := filepath.Join(root, "workspace") + current := filepath.Join(root, "artifacts [7]", "2") + prior := filepath.Join(root, "artifacts [7]", "1") + sibling := filepath.Join(root, "artifacts [7]", "3") + for _, directory := range []string{workspace, current, prior, sibling} { + if err := os.MkdirAll(directory, 0700); err != nil { + t.Fatal(err) + } + } + policy := map[string]any{ + "directory": workspace, + "tools": []string{"read", "grep", "find", "edit"}, + "paths": []string{}, + "edit": nil, + "shell": nil, + "artifact_write": current, + "artifact_read": []string{current, prior}, + } + policyJSON, err := json.Marshal(policy) + if err != nil { + t.Fatal(err) + } + policyFile := filepath.Join(root, "policy.mjs") + if err := os.WriteFile(policyFile, piPolicy, 0600); err != nil { + t.Fatal(err) + } + script := `const { permit } = await import(process.argv[1]); +const policy = JSON.parse(process.argv[2]); +const checks = JSON.parse(process.argv[3]); +console.log(JSON.stringify(checks.map(([tool, path]) => permit(policy, tool, {path}))));` + checks := [][2]string{ + {"edit", filepath.Join(current, "report.md")}, + {"edit", filepath.Join(prior, "report.md")}, + {"read", filepath.Join(prior, "report.md")}, + {"grep", filepath.Join(prior, "report.md")}, + {"find", filepath.Join(current, "reports")}, + {"read", filepath.Join(sibling, "report.md")}, + {"grep", filepath.Join(root, "other-run", "1", "report.md")}, + {"find", filepath.Join(current, "..", "3", "report.md")}, + {"read", filepath.Join(prior, ".env")}, + } + checksJSON, err := json.Marshal(checks) + if err != nil { + t.Fatal(err) + } + output, err := exec.Command("node", "--input-type=module", "--eval", script, "file://"+policyFile, string(policyJSON), string(checksJSON)).Output() + if err != nil { + t.Fatal(err) + } + var got []any + if err := json.Unmarshal(output, &got); err != nil { + t.Fatal(err) + } + if len(got) != len(checks) || got[0] != nil || got[1] == nil || got[2] != nil || got[3] != nil || got[4] != nil || got[5] == nil || got[6] == nil || got[7] == nil || got[8] == nil { + t.Fatalf("Pi artifact policy results = %#v", got) + } +} diff --git a/internal/stage/tools.go b/internal/stage/tools.go index 566519cb..77477987 100644 --- a/internal/stage/tools.go +++ b/internal/stage/tools.go @@ -7,6 +7,13 @@ import ( "strings" ) +// ArtifactAccess is runtime-only, server-derived access to artifact visits. +// Empty fields deliberately grant nothing outside the workspace. +type ArtifactAccess struct { + Write string + Read []string +} + // ToolNames validates model-stage permissions, not general graph semantics. // A present empty block explicitly denies all tools. func ToolNames(def Definition) ([]string, bool, error) { @@ -128,12 +135,39 @@ func (c *AgentClient) permissionRules() []map[string]string { } } } + // OpenCode V2 resolves an external file to its containing directory and + // checks external_directory against /*. Do not grant a parent: + // each visit is deliberately an independent capability. + readArtifacts := false + writeArtifacts := false + for _, tool := range tools { + readArtifacts = readArtifacts || tool == "read" || tool == "glob" || tool == "grep" + writeArtifacts = writeArtifacts || tool == "edit" + } + if readArtifacts { + for _, directory := range c.ArtifactAccess.Read { + if artifactDirectory(directory) { + allow("external_directory", filepath.ToSlash(filepath.Join(directory, "*"))) + } + } + } + if writeArtifacts && artifactDirectory(c.ArtifactAccess.Write) { + allow("external_directory", filepath.ToSlash(filepath.Join(c.ArtifactAccess.Write, "*"))) + allow("edit", filepath.ToSlash(filepath.Join(c.ArtifactAccess.Write, "*"))) + } + for _, pattern := range []string{"../*", "*/../*"} { + rules = append(rules, map[string]string{"action": "external_directory", "resource": pattern, "effect": "deny"}) + } for _, pattern := range []string{"*.env", "*.env.*", "pipelines.db*"} { rules = append(rules, map[string]string{"action": "read", "resource": pattern, "effect": "deny"}) } return rules } +func artifactDirectory(directory string) bool { + return directory != "" && filepath.IsAbs(directory) && filepath.Clean(directory) == directory && !strings.ContainsAny(directory, "*?\\") +} + func matchesEditPolicy(patterns []string, path string) bool { for _, pattern := range patterns { // OpenCode resource wildcards match across directory separators. diff --git a/internal/stage/tools_test.go b/internal/stage/tools_test.go index fa561166..ed823bc1 100644 --- a/internal/stage/tools_test.go +++ b/internal/stage/tools_test.go @@ -139,3 +139,38 @@ func TestDefaultEditScopeIsLanguageIndependentRunDirectory(t *testing.T) { t.Fatal("external directories were enabled") } } + +func TestArtifactPermissionsAreExactAndToolScoped(t *testing.T) { + current := "/artifacts/run [7]/2" + prior := "/artifacts/run [7]/1" + sibling := "/artifacts/run 7/3" + otherRun := "/artifacts/run 8/1" + client := &AgentClient{OpenCodeClient: OpenCodeClient{Directory: "/repo"}, Tools: []string{"read", "edit"}, ArtifactAccess: ArtifactAccess{Write: current, Read: []string{current, prior}}} + rules := client.permissionRules() + for _, path := range []string{current + "/report.md", prior + "/report.md"} { + if permissionEffect(rules, "external_directory", path) != "allow" { + t.Fatalf("artifact directory denied: %q", path) + } + } + if permissionEffect(rules, "edit", current+"/report.md") != "allow" || permissionEffect(rules, "edit", prior+"/report.md") != "deny" { + t.Fatal("artifact writes were not limited to the current visit") + } + for _, path := range []string{sibling + "/report.md", otherRun + "/report.md", current + "/../3/report.md", "/artifacts/run 7/20/report.md"} { + if permissionEffect(rules, "external_directory", path) != "deny" { + t.Fatalf("unapproved artifact scope allowed: %q", path) + } + } + client.Tools = []string{} + if permissionEffect(client.permissionRules(), "external_directory", current+"/report.md") != "deny" { + t.Fatal("no-tool stage gained artifact access") + } + client.Tools = []string{"read"} + rules = client.permissionRules() + if permissionEffect(rules, "external_directory", prior+"/report.md") != "allow" || permissionEffect(rules, "edit", current+"/report.md") != "deny" { + t.Fatal("read-only artifact stage gained write access") + } + client.ArtifactAccess = ArtifactAccess{} + if permissionEffect(client.permissionRules(), "external_directory", current+"/report.md") != "deny" { + t.Fatal("missing artifact scope enabled an external directory") + } +} From 269a8d65bbf1866207717efbec7a13522c8f7274 Mon Sep 17 00:00:00 2001 From: Alan Silva Date: Sun, 18 Oct 2026 12:27:24 +0100 Subject: [PATCH 3/7] Guard socket recovery from stale callbacks Serialize socket replacement and callbacks so a superseded socket cannot clear or regress the active connection, and make reconnect backoff safe after a successful open. Add focused recovery regression coverage.\n\nChecked packaged templates and packaged authoring skill references; neither needs updating for this Android transport-only change. --- .../data/transport/PipelinesClient.kt | 172 +++++++++++++----- .../data/transport/ConnectionRecoveryTest.kt | 60 ++++++ 2 files changed, 184 insertions(+), 48 deletions(-) create mode 100644 android/app/src/test/java/net/awan/pipelines/data/transport/ConnectionRecoveryTest.kt diff --git a/android/app/src/main/java/net/awan/pipelines/data/transport/PipelinesClient.kt b/android/app/src/main/java/net/awan/pipelines/data/transport/PipelinesClient.kt index 886d99e3..b649f361 100644 --- a/android/app/src/main/java/net/awan/pipelines/data/transport/PipelinesClient.kt +++ b/android/app/src/main/java/net/awan/pipelines/data/transport/PipelinesClient.kt @@ -78,11 +78,33 @@ class PipelinesClient( @Volatile private var manualClose = false + /** Serializes socket replacement with callbacks arriving on OkHttp threads. */ + private val connectionLock = Any() + private var reconnectJob: Job? = null private var attempt = 0 /** Opens (or re-opens) the connection to a validated server URL. */ fun connect(validated: ServerUrl.Validated) { + synchronized(connectionLock) { + connectLocked(validated) + } + } + + /** + * Re-opens a connection the transport lost without reversing an explicit + * [disconnect]. The check and connection replacement share one lock, so a + * Disconnect racing a foreground liveness probe wins even after that probe + * has decided recovery is needed. + */ + fun recover(validated: ServerUrl.Validated) { + synchronized(connectionLock) { + if (manualClose) return + connectLocked(validated) + } + } + + private fun connectLocked(validated: ServerUrl.Validated) { val httpUrl = validated.httpUrl ?: throw IllegalArgumentException("server URL is not valid") endpoint = httpUrl manualClose = false @@ -94,14 +116,16 @@ class PipelinesClient( /** Explicit user disconnect: stops reconnection and closes the socket. */ fun disconnect() { - manualClose = true - reconnectJob?.cancel() - reconnectJob = null - val socket = webSocket - webSocket = null - socket?.close(NORMAL_CLOSURE, "client closing") - failPending("Disconnected") - _state.value = ConnectionState.Idle + synchronized(connectionLock) { + manualClose = true + reconnectJob?.cancel() + reconnectJob = null + val socket = webSocket + webSocket = null + socket?.close(NORMAL_CLOSURE, "client closing") + failPending("Disconnected") + _state.value = ConnectionState.Idle + } } fun close() { @@ -115,14 +139,20 @@ class PipelinesClient( */ suspend fun call(request: Message): Message = withContext(ioDispatcher) { sendLock.withLock { - val socket = webSocket ?: throw NotConnectedException() - if (_state.value !is ConnectionState.Connected) throw NotConnectedException() - val id = "android-" + sequence.incrementAndGet() - val deferred = CompletableDeferred() - pending[id] = deferred - try { + val (id, deferred) = synchronized(connectionLock) { + val socket = webSocket ?: throw NotConnectedException() + if (_state.value !is ConnectionState.Connected) throw NotConnectedException() + val id = "android-" + sequence.incrementAndGet() + val deferred = CompletableDeferred() + pending[id] = deferred val payload = json.encodeToString(Message.serializer(), request.copy(id = id)) - if (!socket.send(payload)) throw IOException("websocket send failed") + if (!socket.send(payload)) { + pending.remove(id) + throw IOException("websocket send failed") + } + id to deferred + } + try { val response = withTimeout(requestTimeoutMillis) { deferred.await() } if (response.type == Protocol.ERROR) { throw ProtocolException(response.error?.ifBlank { "server error" } ?: "server error") @@ -144,12 +174,14 @@ class PipelinesClient( */ suspend fun sendOnly(request: Message): String = withContext(ioDispatcher) { sendLock.withLock { - val socket = webSocket ?: throw NotConnectedException() - if (_state.value !is ConnectionState.Connected) throw NotConnectedException() - val id = "android-" + sequence.incrementAndGet() - val payload = json.encodeToString(Message.serializer(), request.copy(id = id)) - if (!socket.send(payload)) throw IOException("websocket send failed") - id + synchronized(connectionLock) { + val socket = webSocket ?: throw NotConnectedException() + if (_state.value !is ConnectionState.Connected) throw NotConnectedException() + val id = "android-" + sequence.incrementAndGet() + val payload = json.encodeToString(Message.serializer(), request.copy(id = id)) + if (!socket.send(payload)) throw IOException("websocket send failed") + id + } } } @@ -166,14 +198,34 @@ class PipelinesClient( } } - private fun open() { + private fun open() = synchronized(connectionLock) { openLocked() } + + private fun openLocked() { val current = endpoint ?: return attempt += 1 _state.value = ConnectionState.Connecting(attempt) val request = Request.Builder().url(current).build() + // Replace the current socket, then close the one it superseded. Its + // callbacks are ignored from then on (see [isCurrent]), so a failure the + // old socket reports late — the common case when the process was frozen + // while it died — can neither clear the new socket nor schedule a + // reconnect that would undo it. + val superseded = webSocket webSocket = httpClient.newWebSocket(request, listener) + // A request sent over the superseded socket cannot receive a reply from + // the replacement. Fail it now rather than leaving a screen waiting for + // the ordinary request timeout after recovery. + if (superseded != null) failPending("Connection replaced") + superseded?.close(NORMAL_CLOSURE, "superseded") } + /** + * Whether [socket] is still the connection this client acts on. A socket this + * client has already replaced (a deliberate reconnect, or the foreground + * recovery re-establishing a frozen one) must never drive the state again. + */ + private fun isCurrent(socket: WebSocket): Boolean = webSocket === socket + private fun failPending(reason: String) { val snapshot = pending.keys.toList() for (key in snapshot) { @@ -181,7 +233,8 @@ class PipelinesClient( } } - private fun handleClosed(reason: String) { + private fun handleClosed(socket: WebSocket, reason: String) { + if (!isCurrent(socket)) return webSocket = null failPending(reason) if (manualClose) { @@ -194,52 +247,65 @@ class PipelinesClient( private fun scheduleReconnect(reason: String) { if (reconnectJob?.isActive == true) return _state.value = ConnectionState.Disconnected(reason, retrying = true) - val nextAttempt = attempt - val backoff = (BASE_BACKOFF_MILLIS shl (nextAttempt - 1).coerceAtMost(6)) - .coerceAtMost(MAX_BACKOFF_MILLIS) + val backoff = reconnectBackoffMillis(attempt) val jitter = Random.nextLong(0, 400) reconnectJob = scope.launch { delay(backoff + jitter) - if (!manualClose && endpoint != null) open() + synchronized(connectionLock) { + if (!manualClose && endpoint != null) open() + } } } private val listener = object : WebSocketListener() { override fun onOpen(webSocket: WebSocket, response: Response) { - attempt = 0 - _state.value = ConnectionState.Connected(build = null, serverTime = null) + synchronized(connectionLock) { + if (!isCurrent(webSocket)) return + attempt = 0 + _state.value = ConnectionState.Connected(build = null, serverTime = null) + } } override fun onMessage(webSocket: WebSocket, text: String) { - val message = try { - json.decodeFromString(Message.serializer(), text) - } catch (_: Exception) { - // Malformed frame: never misattribute it to a request. - return - } - val id = message.id - val deferred = if (id != null) pending.remove(id) else null - if (deferred != null) { - deferred.complete(message) - } else { - // Unsolicited envelopes (the connection's initial pipeline_list, - // server-pushed chat frames, stream errors) carry no matching id: - // hand them to the event stream instead of dropping them. - _events.tryEmit(message) + synchronized(connectionLock) { + if (!isCurrent(webSocket)) return + val message = try { + json.decodeFromString(Message.serializer(), text) + } catch (_: Exception) { + // Malformed frame: never misattribute it to a request. + return + } + val id = message.id + val deferred = if (id != null) pending.remove(id) else null + if (deferred != null) { + deferred.complete(message) + } else { + // Unsolicited envelopes (the connection's initial pipeline_list, + // server-pushed chat frames, stream errors) carry no matching id: + // hand them to the event stream instead of dropping them. + _events.tryEmit(message) + } } } override fun onClosed(webSocket: WebSocket, code: Int, reason: String) { - handleClosed("Connection closed (${code})") + synchronized(connectionLock) { + handleClosed(webSocket, "Connection closed (${code})") + } } override fun onClosing(webSocket: WebSocket, code: Int, reason: String) { - webSocket.close(NORMAL_CLOSURE, null) + synchronized(connectionLock) { + if (!isCurrent(webSocket)) return + webSocket.close(NORMAL_CLOSURE, null) + } } override fun onFailure(webSocket: WebSocket, t: Throwable, response: Response?) { - val reason = t.message?.ifBlank { t.javaClass.simpleName } ?: "connection failed" - handleClosed(reason) + synchronized(connectionLock) { + val reason = t.message?.ifBlank { t.javaClass.simpleName } ?: "connection failed" + handleClosed(webSocket, reason) + } } } @@ -247,6 +313,16 @@ class PipelinesClient( private const val NORMAL_CLOSURE = 1000 private const val BASE_BACKOFF_MILLIS = 1_000L private const val MAX_BACKOFF_MILLIS = 30_000L + + /** + * The successful open callback resets [attempt] to zero. A later socket + * failure must therefore begin at the base delay, never shift by -1 + * (which Kotlin masks into a very large shift count). + */ + internal fun reconnectBackoffMillis(attempt: Int): Long { + val exponent = (attempt - 1).coerceIn(0, 6) + return (BASE_BACKOFF_MILLIS shl exponent).coerceAtMost(MAX_BACKOFF_MILLIS) + } } } diff --git a/android/app/src/test/java/net/awan/pipelines/data/transport/ConnectionRecoveryTest.kt b/android/app/src/test/java/net/awan/pipelines/data/transport/ConnectionRecoveryTest.kt new file mode 100644 index 00000000..13e2bd1a --- /dev/null +++ b/android/app/src/test/java/net/awan/pipelines/data/transport/ConnectionRecoveryTest.kt @@ -0,0 +1,60 @@ +package net.awan.pipelines.data.transport + +import org.junit.Assert.assertEquals +import org.junit.Test + +/** + * The return-to-foreground policy, in the one pure place it lives: an explicit + * Disconnect is never overridden, a dropped or in-flight connection is + * re-established at once, and a still-Connected socket is trusted only after it + * has been confirmed to answer. + */ +class ConnectionRecoveryTest { + + @Test + fun aDeliberateDisconnectIsNeverOverridden() { + // Idle is only ever reached by a deliberate Disconnect (the client starts + // Idle and never returns to it by itself), so returning must leave it. + assertEquals( + ForegroundAction.LeaveAlone, + foregroundAction(ConnectionState.Idle, confirmedAlive = false), + ) + } + + @Test + fun aDroppedConnectionReconnectsAtOnce() { + assertEquals( + ForegroundAction.Reconnect, + foregroundAction(ConnectionState.Disconnected("socket closed", retrying = true), confirmedAlive = false), + ) + } + + @Test + fun anOpenAlreadyInFlightIsRestartedRatherThanWaitedOn() { + assertEquals( + ForegroundAction.Reconnect, + foregroundAction(ConnectionState.Connecting(3), confirmedAlive = false), + ) + } + + @Test + fun aConnectedSocketIsTrustedOnlyAfterItAnswers() { + val connected = ConnectionState.Connected(build = null, serverTime = null) + assertEquals(ForegroundAction.LeaveAlone, foregroundAction(connected, confirmedAlive = true)) + // The defect: a socket the process still calls Connected but that answers + // nothing must be reconnected, not trusted. + assertEquals(ForegroundAction.Reconnect, foregroundAction(connected, confirmedAlive = false)) + } + + @Test + fun aFailureAfterConnectedStartsAtTheBaseBackoff() { + // onOpen resets the client's attempt counter to zero. The next failure + // must not pass -1 to the shift used for exponential backoff. + assertEquals(1_000L, PipelinesClient.reconnectBackoffMillis(attempt = 0)) + } + + @Test + fun reconnectBackoffRemainsBounded() { + assertEquals(30_000L, PipelinesClient.reconnectBackoffMillis(attempt = Int.MAX_VALUE)) + } +} From 9130ae8c7a62db011fc399233ce37c8d87035cfd Mon Sep 17 00:00:00 2001 From: Alan Silva Date: Sun, 18 Oct 2026 12:30:30 +0100 Subject: [PATCH 4/7] Add stage-exclusive resource waiting Add durable FIFO capacity-one resource leases for executable stages, including cleanup-confirmed release, conservative restart quarantine, queue state events, and CLI/TUI/Android waiting presentation. Add scheduler, cleanup, protocol, client, and Android coverage plus a proposed Android v3 resource draft. Checked packaged templates and authoring skill references: neither needs updating in this item. The per-item docs note records the shared documentation that docs-fold must update; no shared README or packaged reference was edited. --- android/AGENTS.md | 30 +- .../awan/pipelines/data/protocol/Message.kt | 14 + .../java/net/awan/pipelines/ui/RunEvents.kt | 19 +- .../pipelines/ui/screens/RunHistoryScreen.kt | 1 + .../pipelines/ui/screens/RunSummaryTab.kt | 1 + .../net/awan/pipelines/ui/ResourceWaitTest.kt | 32 ++ docs/notes/phone-resource-wait.md | 158 +++++++++ internal/cli/app.go | 4 +- internal/pipeline/control.go | 15 +- internal/pipeline/resources.go | 277 +++++++++++++++ internal/pipeline/resources_test.go | 288 +++++++++++++++ internal/pipeline/retention.go | 29 +- internal/pipeline/runs.go | 11 +- internal/pipeline/runs_mutate.go | 4 +- internal/pipeline/runs_schema.go | 32 +- internal/pipeline/stage.go | 1 + internal/pipeline/store.go | 10 +- internal/protocol/message.go | 6 + internal/protocol/resources_test.go | 29 ++ internal/protocol/runevents.go | 7 +- internal/protocol/runs.go | 28 +- internal/server/resource_cli_test.go | 154 +++++++++ internal/server/resources.go | 60 ++++ internal/server/resources_test.go | 327 ++++++++++++++++++ internal/server/retry_test.go | 1 + internal/server/runevents.go | 29 +- internal/server/runner.go | 109 ++++-- .../server/testdata/android-resource-draft.md | 312 +++++++++++++++++ .../server/testdata/android-v2-published.md | 296 ++++++++++++++++ .../server/testdata/android-v3-published.md | 296 ++++++++++++++++ internal/stage/agent.go | 25 +- internal/stage/agent_test.go | 4 +- internal/stage/budget_test.go | 6 + internal/stage/opencode.go | 2 +- internal/stage/pi.go | 15 +- internal/stage/resource_cleanup_test.go | 44 +++ internal/tui/resources_test.go | 43 +++ internal/tui/results.go | 5 +- internal/tui/run_detail.go | 12 +- 39 files changed, 2657 insertions(+), 79 deletions(-) create mode 100644 android/app/src/test/java/net/awan/pipelines/ui/ResourceWaitTest.kt create mode 100644 docs/notes/phone-resource-wait.md create mode 100644 internal/pipeline/resources.go create mode 100644 internal/pipeline/resources_test.go create mode 100644 internal/protocol/resources_test.go create mode 100644 internal/server/resource_cli_test.go create mode 100644 internal/server/resources.go create mode 100644 internal/server/resources_test.go create mode 100644 internal/server/testdata/android-resource-draft.md create mode 100644 internal/server/testdata/android-v2-published.md create mode 100644 internal/server/testdata/android-v3-published.md create mode 100644 internal/stage/resource_cleanup_test.go create mode 100644 internal/tui/resources_test.go diff --git a/android/AGENTS.md b/android/AGENTS.md index f6dddbd2..2e035722 100644 --- a/android/AGENTS.md +++ b/android/AGENTS.md @@ -20,16 +20,37 @@ time while iterating — the full suite is for the end. ## The dev device +Only the **UI reviewer**, while its stage holds a declared runner-exclusive resource +for the selected phone, may use the device. Implementers and code reviewers do code, +builds, unit tests and source review only: **no adb, installs, captures, device-setting +changes or main-build restoration**. Never acquire a phone using sleep/check loops. +Do not use the phone until compatible resource scheduling is deployed and every device +user has opted in or has been reconciled by the operator. Existing pinned runs are not +retroactively protected. The resource for the device below is `phone-47241FDAS00DLK`. + The only device is the operator's **Pixel 9 Pro XL, serial `47241FDAS00DLK`**. The operator's Pixel 10 Pro Fold (`58281FDCG000JF`) is theirs: use it only where an item's brief explicitly allows it, and never make it a default. **Never start or use an emulator.** -Before any capture: `svc power stayon usb`, `settings put global stay_on_while_plugged_in 7`, +The UI reviewer records original app connection/workspace, display size and power/settings +before changing them. Before any capture: `svc power stayon usb`, `settings put global stay_on_while_plugged_in 7`, `input keyevent KEYCODE_WAKEUP`, `wm dismiss-keyguard`. Capture with `adb -s shell screencap -p /sdcard/shot.png` then `adb pull` — never `exec-out > file`. Wide layout: `wm size 2000x2244`; the fold's unfolded shape is `wm size 2076x2152`; `wm size reset` afterwards either way. +Create controlled fixtures for changed screens/states (including human-decision sheets) +on a scratch server/database and throwaway session; never depend on incidental production +state, change a real operator session, or restart a deployed service. Use the configured +authoritative endpoint, never a scratch URL as a project default. Keep all shell/ADB work +foreground and awaited: no detached/background processes may outlive the lease. +UI review includes restoring the captured original settings and main's build before its +successful return. Confirm every tool has exited. If capture, restoration or cleanup fails, +report the failure explicitly; do not claim verification or readiness. Uncertain termination +must fail closed and keep ownership quarantined, not delete a lock or expire it by age. +Return `output.resource_cleanup_confirmed=false` if restoration/readiness is uncertain; +that generic release veto retains the runner lease even if the harness reports idle. + ## Geometry is tokenised, and a test enforces it Every dimension comes from `ui/theme/Dimens.kt` (plus the layout itself) — no new hardcoded `dp`, no @@ -53,13 +74,14 @@ genuinely needs a new token goes there, named for its purpose. ## Publishing a build Copy the APK into the running run's artifacts directory as `pipelines-debug-wip-.apk` and -report the LAN URL. After any install, leave the operator's phone on **main's** build: +report the LAN URL. The UI reviewer alone, under the same lease, restores **main's** build after an install: `/private/var/folders/jp/q0qvjyg54fl_c5vhhk1ylz2m0000gn/T/opencode/deploy-app.sh main`. ## Traps - A build made in a worktree that is behind main is missing features main has — sync before building, and say which revision you built. -- Changing the app's server URL for a test (a stub) must be undone: `ws://10.0.0.75:8080/connect`, - workspace `/Users/alan/repos/pipelines`, then force-stop and relaunch. +- Restore the app's captured original connection and workspace after a fixture test, + using the configured authoritative endpoint; then force-stop and relaunch under the lease. + Never substitute a hardcoded or disposable scratch endpoint for the original configuration. - `mStayOn`/screen timeout are already set; do not "fix" the device by locking or sleeping it. diff --git a/android/app/src/main/java/net/awan/pipelines/data/protocol/Message.kt b/android/app/src/main/java/net/awan/pipelines/data/protocol/Message.kt index 0a1e2d40..b0a5c168 100644 --- a/android/app/src/main/java/net/awan/pipelines/data/protocol/Message.kt +++ b/android/app/src/main/java/net/awan/pipelines/data/protocol/Message.kt @@ -315,6 +315,7 @@ data class SessionExport( @Serializable data class RunSummary( + @SerialName("resource_wait") val resourceWait: ResourceWait? = null, val id: Long = 0, @SerialName("pipeline_id") val pipelineId: Long = 0, val name: String = "", @@ -370,6 +371,7 @@ data class RunStep( // declared directly with the exact internal/protocol/runs.go wire names. @Serializable data class RunDetail( + @SerialName("resource_wait") val resourceWait: ResourceWait? = null, val id: Long = 0, @SerialName("pipeline_id") val pipelineId: Long = 0, val name: String = "", @@ -405,6 +407,8 @@ data class RunDetail( */ @Serializable data class RunEventFrame( + @SerialName("resource_wait") val resourceWait: ResourceWait? = null, + @SerialName("resource_wait_set") val resourceWaitSet: Boolean = false, @SerialName("run_id") val runId: Long = 0, val workspace: String = "", // The run's pipeline identity and stored title, repeated on every frame so a @@ -428,6 +432,16 @@ data class RunEventFrame( val step: RunStepDelta? = null, ) +@Serializable +data class ResourceWait( + val resource: String = "", + val stage: String = "", + val sequence: Int = 0, + @SerialName("queued_since") val queuedSince: String = "", +) { + fun statusLabel(): String = "Waiting for resource $resource" +} + /** * The compact change of one step within a [RunEventFrame] * (internal/protocol/runevents.go RunStepDelta): a delta, not a RunStep, so it diff --git a/android/app/src/main/java/net/awan/pipelines/ui/RunEvents.kt b/android/app/src/main/java/net/awan/pipelines/ui/RunEvents.kt index ecf6a66c..a222ba55 100644 --- a/android/app/src/main/java/net/awan/pipelines/ui/RunEvents.kt +++ b/android/app/src/main/java/net/awan/pipelines/ui/RunEvents.kt @@ -101,6 +101,7 @@ internal fun foldRunEvent(runs: List, frame: RunEventFrame): RunList internal fun RunEventFrame.newRunSummary(): RunSummary? { if (pipelineId <= 0 || name.isBlank()) return null return RunSummary( + resourceWait = resourceWait, id = runId, pipelineId = pipelineId, name = name, @@ -124,10 +125,11 @@ internal fun RunEventFrame.newRunSummary(): RunSummary? { */ internal fun foldRunEvent(detail: RunDetail, frame: RunEventFrame): RunDetail { if (frame.runId != detail.id) return detail - if (isStaleRunFrame(detail.status, frame.status)) return detail + if (isStaleRunFrame(detail.status, frame.status, frame.resourceWaitSet)) return detail val merged = detail.fields().applying(frame) val steps = frame.step?.let { upsertStep(detail.steps, it) } ?: detail.steps val updated = detail.copy( + resourceWait = merged.resourceWait, status = merged.status, outcome = merged.outcome, currentStage = merged.currentStage, @@ -169,8 +171,9 @@ private fun advanceRunStatus(current: String, incoming: String): String = * run already settled. Dropping every field (not just the status) is what keeps * a late `running` delta from also rewriting the stage or step of a settled run. */ -private fun isStaleRunFrame(current: String, incoming: String): Boolean = - isStaleTransition(current, incoming, ::runLifecycleRank) +private fun isStaleRunFrame(current: String, incoming: String, resourceWaitSet: Boolean = false): Boolean = + if (resourceWaitSet && incoming == "queued" && !isTerminalRunStatus(current)) false + else isStaleTransition(current, incoming, ::runLifecycleRank) /** The step counterpart of [isStaleRunFrame]: a settled visit drops a stale delta whole. */ private fun isStaleStepDelta(current: String, incoming: String): Boolean = @@ -213,6 +216,7 @@ private fun advanceStatus(current: String, incoming: String, rank: (String) -> I /** The run-state fields a frame advances, shared by the summary row and the detail. */ private data class RunStateFields( + val resourceWait: net.awan.pipelines.data.protocol.ResourceWait?, val status: String, val outcome: String, val currentStage: String, @@ -226,8 +230,10 @@ private data class RunStateFields( * detail can never apply a frame differently. */ fun applying(frame: RunEventFrame): RunStateFields { - val status = advanceRunStatus(status, frame.status) + val status = if (frame.resourceWaitSet && frame.status == "queued" && !isTerminalRunStatus(status)) + "queued" else advanceRunStatus(status, frame.status) return RunStateFields( + resourceWait = if (isTerminalRunStatus(status)) null else if (frame.resourceWaitSet) frame.resourceWait else resourceWait, status = status, outcome = frame.outcome.ifBlank { outcome }, currentStage = frame.currentStage.ifBlank { currentStage }, @@ -240,6 +246,7 @@ private data class RunStateFields( } private fun RunSummary.fields() = RunStateFields( + resourceWait = resourceWait, status = status, outcome = outcome, currentStage = currentStage, @@ -250,6 +257,7 @@ private fun RunSummary.fields() = RunStateFields( ) private fun RunDetail.fields() = RunStateFields( + resourceWait = resourceWait, status = status, outcome = outcome, currentStage = currentStage, @@ -267,11 +275,12 @@ private fun executingFor(status: String, current: Boolean): Boolean = when (stat } private fun RunSummary.applying(frame: RunEventFrame): RunSummary { - if (isStaleRunFrame(status, frame.status)) return this + if (isStaleRunFrame(status, frame.status, frame.resourceWaitSet)) return this val current = fields() val merged = current.applying(frame) if (merged == current) return this return copy( + resourceWait = merged.resourceWait, status = merged.status, outcome = merged.outcome, currentStage = merged.currentStage, diff --git a/android/app/src/main/java/net/awan/pipelines/ui/screens/RunHistoryScreen.kt b/android/app/src/main/java/net/awan/pipelines/ui/screens/RunHistoryScreen.kt index 94093a1e..eacf2ace 100644 --- a/android/app/src/main/java/net/awan/pipelines/ui/screens/RunHistoryScreen.kt +++ b/android/app/src/main/java/net/awan/pipelines/ui/screens/RunHistoryScreen.kt @@ -615,6 +615,7 @@ private fun VersionChip(version: String) { /** Progress from the fields a summary actually has: completed visits + stage. */ private fun progressText(run: RunSummary): String = buildString { + run.resourceWait?.let { append(it.statusLabel()); append(" · ") } append(visitCount(run.completedSteps, "completed")) if (run.currentStage.isNotBlank()) append(" · stage ${run.currentStage}") } diff --git a/android/app/src/main/java/net/awan/pipelines/ui/screens/RunSummaryTab.kt b/android/app/src/main/java/net/awan/pipelines/ui/screens/RunSummaryTab.kt index 559a7029..418cad8d 100644 --- a/android/app/src/main/java/net/awan/pipelines/ui/screens/RunSummaryTab.kt +++ b/android/app/src/main/java/net/awan/pipelines/ui/screens/RunSummaryTab.kt @@ -133,6 +133,7 @@ internal fun RunSummaryTab( ) KeyValueRow("Pipeline", "id ${run.pipelineId} · v${run.version}") KeyValueRow("Stage", run.currentStage) + run.resourceWait?.let { Text(it.statusLabel(), style = text.meta) } KeyValueRow("Workspace", run.workspace) KeyValueRow("Content hash", run.contentHash, monospace = true) KeyValueRow("Started", formatLocalTimestamp(run.createdAt).orEmpty()) diff --git a/android/app/src/test/java/net/awan/pipelines/ui/ResourceWaitTest.kt b/android/app/src/test/java/net/awan/pipelines/ui/ResourceWaitTest.kt new file mode 100644 index 00000000..5bcdd82c --- /dev/null +++ b/android/app/src/test/java/net/awan/pipelines/ui/ResourceWaitTest.kt @@ -0,0 +1,32 @@ +package net.awan.pipelines.ui + +import kotlinx.serialization.json.Json +import net.awan.pipelines.data.protocol.ResourceWait +import net.awan.pipelines.data.protocol.RunDetail +import net.awan.pipelines.data.protocol.RunEventFrame +import net.awan.pipelines.data.protocol.RunSummary +import org.junit.Assert.* +import org.junit.Test + +class ResourceWaitTest { + @Test fun optionalMetadataAndPauseResumeClearing() { + val json = Json { ignoreUnknownKeys = true } + assertNull(json.decodeFromString("""{"id":1,"status":"queued"}""").resourceWait) + val wait = ResourceWait("phone-X", "review", 2, "2026-10-18T10:00:00Z") + val queued = RunEventFrame(runId = 1, status = "queued", resourceWaitSet = true, resourceWait = wait) + val detail = foldRunEvent(RunDetail(id = 1, status = "running"), queued) + assertEquals("queued", detail.status) + assertEquals(wait, detail.resourceWait) + assertEquals("Waiting for resource phone-X", wait.statusLabel()) + val rows = foldRunEvent(listOf(RunSummary(id = 1, status = "running")), queued).runs + assertEquals(wait, rows.single().resourceWait) + val clear = RunEventFrame(runId = 1, status = "running", resourceWaitSet = true) + assertNull(foldRunEvent(detail, clear).resourceWait) + assertNull(foldRunEvent(rows, clear).runs.single().resourceWait) + assertNull(foldRunEvent(detail, RunEventFrame(runId = 1, status = "canceled")).resourceWait) + assertEquals("succeeded", foldRunEvent(RunDetail(id = 1, status = "succeeded"), queued).status) + val decoded = json.decodeFromString("""{"id":1,"status":"queued","resource_wait":{"resource":"phone-X","stage":"review","sequence":2,"queued_since":"now"}}""") + assertEquals("phone-X", decoded.resourceWait?.resource) + assertFalse(isWaitingRunStatus(decoded.status)) + } +} diff --git a/docs/notes/phone-resource-wait.md b/docs/notes/phone-resource-wait.md new file mode 100644 index 00000000..33d6b3d2 --- /dev/null +++ b/docs/notes/phone-resource-wait.md @@ -0,0 +1,158 @@ +# Stage-exclusive resources — 2026-10-18, not deployed + +This is source work for run 355, **not a deployment, installation or publication**. +No physical phone, real operator session or deployed service was used or changed. +The Android role split does not retrospectively gate pinned versions or existing runs. + +## Contract and scope + +An executable non-human stage can declare one runner-owned control block: + +```resources +{"exclusive":"phone-47241FDAS00DLK"} +``` + +The key is 1–128 ASCII identifier bytes (letter first, then letters, digits, `_`, `-`). +Duplicate/unknown fields, multiple resources and human-stage resources are rejected. +One capacity-one resource per stage visit, one server/store authority; no expressions, +nested acquisition, distributed scheduling, capacity settings or multi-resource requests. +Resource-free definitions retain their execution path. Old servers reject this new block; +do not publish to an incompatible server. + +Acquisition follows selection and automatic UI-skip evaluation, before BeginStep, +visits/sequences, artifacts, filediff, stage timeout, budget setup or executor calls. +Queued runs retain workspace reservation, use legacy `queued` (never human `waiting`), +and carry bounded `resource_wait` identity/stage/sequence/queued-since metadata. +Events use `resource_wait_set=true` with absent `resource_wait` to clear caches. +CLI lists and TUI/Android details show “Waiting for resource …”; human decisions and +attention notifications remain separate. Older clients still see an unfinished run. + +SQLite owners and FIFO waiters are indexed by resource/order and run identity. +Private random capabilities attribute ownership to resource/run/stage/sequence and +never enter wire data, model input, transcripts or artifacts. Only the full capability +and positive cleanup evidence authorize release; mismatches and repeated release cannot +release a subsequent owner. One buffered channel per live queued run, no resource +polling, waiter tickers, model-based acquisition, queue driver or new daemon. + +The gate holds through tool cleanup. OpenCode interruption now confirms session idle/ +V1 settlement before clearing session ownership. Pi recovery requires both its receipt +and process/process-group exit, not merely abort acknowledgement. Executor panic, +ErrStageCleanup, retained active sessions and failed cleanup queries quarantine the owner. +An explicit `output.resource_cleanup_confirmed=false` (or malformed value) also vetoes +release for failed restoration; true never substitutes for harness cleanup evidence. +The generic runner does not inspect Android settings or magically verify restoration. +The Android UI reviewer must keep all device work foreground/awaited, restore original +settings/connection and main's build, and report uncertainty with the release veto. + +## Restart and retention + +Automatic FIFO progress is promised only within a live server process. Startup interrupts +persisted queued resource visits, removes their waiter rows, and never replays an executor. +Durably cleanup-confirmed owners can be reclaimed through the token-checked delete helper. +Unconfirmed owners remain quarantined, including crashes before BeginStep, before session +registration, or after cleanup but before its confirmation is persisted. Session interruption +alone cannot prove resource-specific restoration. Age never authorizes reclamation. +Quarantined ownership/session evidence survives recovery and protects even terminal runs +from retention. Reconciliation requires inspected cleanup and restoration, not lock deletion. +There is intentionally no force-unlock CLI or automatic uncertain-owner reclamation. + +## Android draft, provenance and activation + +Read-only retrieval: `pipelines show Android --version 2 --json` returned pipeline 20, +v2, hash `8c93aeb9b76f1e6e7696415d6308d786ec08d805a1a346fbabe4d252d5d65032`. +`python3` recomputed SHA-256 of the returned source and asserted equality. +The exact source is retained as `internal/server/testdata/android-v2-published.md`; +the proposed source is `internal/server/testdata/android-resource-draft.md`. +The provenance test compares graph fields/outcomes unchanged, apart from the approved +three stage prompts and the UI review resource; every stage keeps its published v3 model: Terra for implementer/committer and Luna for reviewer/UI reviewer. No unrelated published definition was edited. + +Artifact handoff recovery: the validated v3 draft was copied to this run's retained +artifact directory as `android-resource-draft.md`. The similarly named artifact from +run 355 remains the old v2 baseline and must not be published. The proposed source in +this workspace and this run's artifact have matching SHA-256 digests +`a10d8b9fa3dc6a1875b10469061ed25fb4a7a404d463d1a2d304a309d55aa5d9`. + +Activation, requiring separate explicit operator authorization: +1. Review/land the source change and independently rerun the required checks. +2. Arrange a compatible deployment separately; nothing here deploys or restarts it. +3. Reconcile every existing phone user, pinned/live run and any quarantined owner. +4. Transfer the **workspace proposed draft**, provenance and this note into the retained + stage artifacts after resolving the artifact edit permission. +5. Validate that retained draft against the configured authoritative compatible endpoint; + only then explicitly publish it. Do not use a disposable endpoint as a project default. +6. Opt in other device users explicitly before relying on mutual exclusion system-wide. + +Read-only published inventory (`pipelines show --json`, v2 for each): +- UI sweep: `survey` and `ui_reviewer` request visual/screenshots work; phone work there + needs explicit resource opt-in and restoration/fixture policy. +- Generic: `reviewer` and `ui_reviewer` have visual/device verification wording; when + targeting a phone, separate code review from leased device review first. +- Plan and land: `implementer`, `reviewer`, `ui_reviewer` contain rendered-surface work; + they need an explicit technology-specific role split/lease before phone use. +- Probe: one harness smoke stage; no default phone declaration. Any device-specific + probe needs explicit opt-in, never an inferred resource or Android stage-name special case. +Server/TUI/other definitions were not changed. Inventory is not activation evidence. + +## Verification evidence and remaining limits + +Executed focused checks: +- Run 358 recovery revalidation: `go build ./... && go vet ./... && go test ./...` + passed; `go test -race ./internal/pipeline ./internal/server ./internal/stage -run + 'Resource|Scheduler|Concurrent|Human|Decision|RetryWaits' -count=1` passed; and the + Android map invocation with `JAVA_HOME`/`ANDROID_HOME`, installed Gradle, + `--offline --no-daemon --rerun-tasks :app:assembleDebug :app:testDebugUnitTest` + executed all 41 tasks and reported `BUILD SUCCESSFUL`. These remain build/unit and + fake-executor scheduler evidence, + not physical-phone verification or screenshots. +- Recovery after integrating main `88595eb`: `go build ./...`, `go vet ./...`, + `go test ./internal/server -run Resource -count=1`, + `go test ./internal/pipeline ./internal/protocol ./internal/tui -run Resource -count=1`, + `go test ./internal/stage -count=1`, + `go test -race ./internal/pipeline ./internal/server ./internal/stage -run 'Resource|Scheduler|Concurrent|Human|Decision|RetryWaits' -count=1`, and + `go test ./...` all passed. `gofmt -l .` and `git diff --check` produced no output. +- Recovery Android build/unit check with the map's offline, no-daemon Gradle invocation: + `:app:assembleDebug :app:testDebugUnitTest` passed. This remains build/unit evidence, + not phone verification or screenshots. +- `go test ./internal/server ./internal/pipeline ./internal/protocol ./internal/tui -run Resource -count=1`: + passed. Includes fake-executor queue >30ms timeout, independent-resource progress, + FIFO/mismatch/duplicate/grant-cancel cycles, blocked cleanup, panic/quarantine, + persistence/BeginStep/query failure, no-acquire skip, restart/retention, wire defaults, + narrow/wide/plain TUI frames and real CLI/WebSocket scratch exercise. +- `go test ./internal/stage -count=1`: passed; the later full suite also included + `TestResourcePiReceiptRequiresProcessExit` (a test-owned sleep process, not a device). +- Android map invocation with `JAVA_HOME`/`ANDROID_HOME` and installed Gradle, + `--offline --no-daemon :app:assembleDebug :app:testDebugUnitTest`: BUILD SUCCESSFUL. + These are build/unit checks, not device verification or screenshots. +- After the Go tree settled: `go build ./...`, `go vet ./...`, `go test ./...` + all exited zero (including author-package tests). +- `go test -race ./internal/pipeline ./internal/server ./internal/stage -run 'Resource|Scheduler|Concurrent|Human|Decision|RetryWaits' -count=1`: + all three packages passed, including concurrent grant/cancel cycles. +- `gofmt -l .` and `git diff --check`: exited zero with no output. + +The scratch CLI test uses the real CLI and WebSocket handler with temporary SQLite and +an explicitly **FAKE executor**. It validates the proposed Android draft on compatible +scratch code, observes B as queued with zero steps while A holds fake-phone, inspects +subscription metadata, releases A, runs `wait`, and observes B succeeded automatically. +It does not call a model or use a physical phone. Test log output is evidence, not a +live-deployment claim; no retained CLI transcript was written to the denied artifact path. + +Live OpenCode/Pi shell/ADB termination and Android visual/device restoration are **not +tested** here. The checks prove scheduler behavior with controlled fakes, plus test-owned +process-exit checks, not device readiness on the current deployment. +Commit/merge/landing belong to the later committer stage; no commit or push here. + +## Stale shared documentation for docs-fold (do not edit in this item) + +- `docs/cli.md`: resource queue presentation, inspection metadata, unchanged wait/decide semantics. +- `internal/author/skill/references/format.md`: optional resources block and validation/key bounds. +- `internal/author/skill/references/control.md`: runner-owned declaration, timeout/accounting boundary, + release veto and automatic skip placement. +- `internal/author/skill/references/queue.md`: FIFO resource waits vs workspace exclusion and human waits. +- `internal/author/skill/references/operations.md`: authority, restart interruption/quarantine, + inspected recovery, retention and explicit compatible-deployment/publication prerequisites. +- `internal/author/skill/references/verification.md`: fake-vs-device evidence, fixtures and leased restoration. +- `internal/author/skill/references/orchestration.md`: no queue drivers/polling/extra agents; explicit + opt-in of every device user, no retroactive gating or distributed guarantees. + +No packaged skill/reference, shared README/topic documentation, pipeline registry, +installation, deployment, provider setting or `internal/stage/model_failure.go` was edited. diff --git a/internal/cli/app.go b/internal/cli/app.go index 8311ca35..5e26e393 100644 --- a/internal/cli/app.go +++ b/internal/cli/app.go @@ -774,7 +774,7 @@ func (a App) Run(ctx context.Context, args []string) error { } case "runs": for _, run := range r.Runs { - fmt.Fprintf(a.Out, "%d\t%s\t%s\tv%d\t%s\n", run.ID, run.Label(), run.Name, run.Version, run.Status) + fmt.Fprintf(a.Out, "%d\t%s\t%s\tv%d\t%s\n", run.ID, run.Label(), run.Name, run.Version, run.StatusLabel()) } case "ping": fmt.Fprintln(a.Out, "Connected ·", r.Time) @@ -1484,7 +1484,7 @@ func (e *waitLimitError) Runs() string { // waitLine is the one-line terminal report `pipelines wait` prints: the run id, // its status and outcome, its stage and how long it took. func waitLine(d *protocol.RunDetail) string { - line := fmt.Sprintf("run %d · %s", d.ID, d.Status) + line := fmt.Sprintf("run %d · %s", d.ID, d.StatusLabel()) if outcome := strings.TrimSpace(d.Outcome); outcome != "" { line += " (" + outcome + ")" } diff --git a/internal/pipeline/control.go b/internal/pipeline/control.go index c603497e..9d32e974 100644 --- a/internal/pipeline/control.go +++ b/internal/pipeline/control.go @@ -10,7 +10,7 @@ import ( ) func decodeControl(text string, value any) error { - // Both control blocks are flat objects. Reject duplicate keys before decoding. + // Control blocks are flat objects. Reject duplicate keys before decoding. keys := json.NewDecoder(strings.NewReader(text)) if token, err := keys.Token(); err != nil || token != json.Delim('{') { return fmt.Errorf("expected object") @@ -44,6 +44,12 @@ func decodeControl(text string, value any) error { } func parseControl(s *Stage, fields map[string]string) error { + if text, ok := fields["resources"]; ok { + if err := decodeControl(text, &s.Resources); err != nil || s.Resources == nil || !ValidResourceKey(s.Resources.Exclusive) { + return fmt.Errorf("resources must contain exactly one exclusive key (1–128 ASCII letters, digits, _ or -; start with a letter)") + } + delete(fields, "resources") + } if text, ok := fields["limits"]; ok { if err := decodeControl(text, &s.Limits); err != nil || s.Limits == nil { return fmt.Errorf("limits must be an object with max_visits and optional on outcomes") @@ -76,6 +82,9 @@ func parseControl(s *Stage, fields map[string]string) error { } func validateControl(s Stage, stages map[string]bool) error { + if s.Resources != nil && s.Type == "human" { + return fmt.Errorf("stage %s: human stages cannot declare resources", s.Key) + } if s.Limits != nil { for _, outcome := range s.Limits.On { target, exists := s.Edges[outcome] @@ -95,3 +104,7 @@ func validateControl(s Stage, stages map[string]bool) error { } return nil } + +func ValidResourceKey(key string) bool { + return len(key) <= 128 && identifier.MatchString(key) +} diff --git a/internal/pipeline/resources.go b/internal/pipeline/resources.go new file mode 100644 index 00000000..1a1720b1 --- /dev/null +++ b/internal/pipeline/resources.go @@ -0,0 +1,277 @@ +package pipeline + +import ( + "context" + "crypto/rand" + "database/sql" + "encoding/hex" + "encoding/json" + "fmt" + "time" + + "git.a-wan.net/alan/pipelines/internal/protocol" +) + +// ResourceVisit identifies a pending visit, before it consumes execution accounting. +type ResourceVisit struct { + Resource string + RunID int64 + Stage string + Sequence int +} + +// ResourceLease is an opaque capability. Its credential is deliberately not exported +// or serializable, and must never be put in model input or public wire metadata. +type ResourceLease struct { + ResourceVisit + token string +} + +// RequestResource acquires immediately or registers one FIFO waiter/channel. The +// mutex covers only short DB transactions and channel bookkeeping, never execution. +func (s *Store) RequestResource(ctx context.Context, visit ResourceVisit) (ResourceLease, <-chan ResourceLease, *protocol.ResourceWait, error) { + s.resourceMu.Lock() + defer s.resourceMu.Unlock() + if !ValidResourceKey(visit.Resource) || visit.RunID <= 0 || visit.Sequence <= 0 || !identifier.MatchString(visit.Stage) { + return ResourceLease{}, nil, nil, fmt.Errorf("invalid resource visit") + } + if ch := s.resourceWaiters[visit]; ch != nil { + wait, err := s.resourceWait(ctx, visit.RunID) + return ResourceLease{}, ch, wait, err + } + credential := make([]byte, 32) + if _, err := rand.Read(credential); err != nil { + return ResourceLease{}, nil, nil, err + } + lease := ResourceLease{ResourceVisit: visit, token: hex.EncodeToString(credential)} + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + return ResourceLease{}, nil, nil, err + } + defer tx.Rollback() + var status string + if err := tx.QueryRowContext(ctx, "SELECT status FROM pipeline_runs WHERE id=?", visit.RunID).Scan(&status); err != nil { + return ResourceLease{}, nil, nil, err + } + if status != protocol.RunStatusQueued && status != protocol.RunStatusRunning { + return ResourceLease{}, nil, nil, fmt.Errorf("run is not active") + } + var existing ResourceLease + err = tx.QueryRowContext(ctx, "SELECT resource_key,run_id,stage_key,sequence,token FROM resource_owners WHERE run_id=?", visit.RunID).Scan(&existing.Resource, &existing.RunID, &existing.Stage, &existing.Sequence, &existing.token) + if err == nil { + if existing.ResourceVisit != visit { + return ResourceLease{}, nil, nil, fmt.Errorf("run already owns another resource visit") + } + return existing, nil, nil, nil + } + if err != sql.ErrNoRows { + return ResourceLease{}, nil, nil, err + } + result, err := tx.ExecContext(ctx, `INSERT INTO resource_owners(resource_key,run_id,stage_key,sequence,token) +VALUES(?,?,?,?,?) ON CONFLICT(resource_key) DO NOTHING`, visit.Resource, visit.RunID, visit.Stage, visit.Sequence, lease.token) + if err != nil { + return ResourceLease{}, nil, nil, err + } + n, err := result.RowsAffected() + if err != nil { + return ResourceLease{}, nil, nil, err + } + if n == 1 { + return lease, nil, nil, tx.Commit() + } + wait := &protocol.ResourceWait{Resource: visit.Resource, Stage: visit.Stage, Sequence: visit.Sequence, QueuedSince: time.Now().UTC().Format(time.RFC3339Nano)} + encoded, err := json.Marshal(wait) + if err != nil { + return ResourceLease{}, nil, nil, err + } + if _, err := tx.ExecContext(ctx, `INSERT INTO resource_waiters(resource_key,run_id,stage_key,sequence,token,queued_at) VALUES(?,?,?,?,?,?)`, visit.Resource, visit.RunID, visit.Stage, visit.Sequence, lease.token, wait.QueuedSince); err != nil { + return ResourceLease{}, nil, nil, err + } + if err := affected(tx.ExecContext(ctx, "UPDATE pipeline_runs SET status='queued',current_stage=?,resource_wait=? WHERE id=? AND status IN ('queued','running')", visit.Stage, string(encoded), visit.RunID)); err != nil { + return ResourceLease{}, nil, nil, err + } + if err := tx.Commit(); err != nil { + return ResourceLease{}, nil, nil, err + } + if s.resourceWaiters == nil { + s.resourceWaiters = make(map[ResourceVisit]chan ResourceLease) + } + ch := make(chan ResourceLease, 1) + s.resourceWaiters[visit] = ch + return ResourceLease{}, ch, wait, nil +} + +func (s *Store) resourceWait(ctx context.Context, id int64) (*protocol.ResourceWait, error) { + var wait protocol.ResourceWait + err := s.db.QueryRowContext(ctx, "SELECT resource_key,stage_key,sequence,queued_at FROM resource_waiters WHERE run_id=?", id).Scan(&wait.Resource, &wait.Stage, &wait.Sequence, &wait.QueuedSince) + if err == sql.ErrNoRows { + return nil, nil + } + return &wait, err +} + +// CancelResource removes a queued visit, or returns a concurrently reserved grant +// to the queue before any execution is allowed. It cannot cancel another visit. +func (s *Store) CancelResource(ctx context.Context, visit ResourceVisit) error { + s.resourceMu.Lock() + defer s.resourceMu.Unlock() + defer delete(s.resourceWaiters, visit) + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + return err + } + defer tx.Rollback() + deleted, err := tx.ExecContext(ctx, "DELETE FROM resource_waiters WHERE resource_key=? AND run_id=? AND stage_key=? AND sequence=?", visit.Resource, visit.RunID, visit.Stage, visit.Sequence) + if err != nil { + return err + } + n, err := deleted.RowsAffected() + if err != nil { + return err + } + if n > 0 { + if _, err := tx.ExecContext(ctx, "UPDATE pipeline_runs SET resource_wait='' WHERE id=?", visit.RunID); err != nil { + return err + } + } + ch := s.resourceWaiters[visit] + // A grant is buffered while this registry entry exists. Cancellation can + // reclaim only that not-yet-accepted capability, never an executing owner. + var grant ResourceLease + if ch != nil { + select { + case grant = <-ch: + default: + } + } + if err := tx.Commit(); err != nil { + return err + } + if grant.token != "" { + if err := s.SetResourceCleanup(ctx, grant, true); err != nil { + return err + } + return s.releaseResourceLocked(ctx, grant) + } + return nil +} + +// AcceptResource removes the channel exactly once. Cancellation is rechecked by +// the runner before BeginStep; a canceled accepted grant is released as unused. +func (s *Store) AcceptResource(lease ResourceLease) { + s.resourceMu.Lock() + delete(s.resourceWaiters, lease.ResourceVisit) + s.resourceMu.Unlock() +} + +// HasResourceOwner is used only for orphan/restart reconciliation, never for a +// resource-free stage visit. +func (s *Store) HasResourceOwner(ctx context.Context, runID int64) (bool, error) { + var exists bool + err := s.db.QueryRowContext(ctx, "SELECT EXISTS(SELECT 1 FROM resource_owners WHERE run_id=?)", runID).Scan(&exists) + return exists, err +} + +func (s *Store) ConfirmRecoveredSession(ctx context.Context, owner SessionOwner) error { + _, err := s.db.ExecContext(ctx, "UPDATE run_steps SET agent_active=0 WHERE run_id=? AND session_id=?", owner.RunID, owner.ID) + return err +} + +// SetResourceCleanup records positive quiescence durably before release. False +// quarantines the owner; neither age nor another run can remove the lease. +func (s *Store) SetResourceCleanup(ctx context.Context, lease ResourceLease, confirmed bool) error { + diagnostic := "" + if !confirmed { + diagnostic = "Cleanup/restoration unconfirmed; resource quarantined pending inspected reconciliation" + } + return affected(s.db.ExecContext(ctx, `UPDATE resource_owners SET cleanup_confirmed=?,diagnostic=? WHERE resource_key=? AND run_id=? AND stage_key=? AND sequence=? AND token=?`, confirmed, diagnostic, lease.Resource, lease.RunID, lease.Stage, lease.Sequence, lease.token)) +} + +// ReleaseResource requires the complete capability AND positive cleanup evidence. +// Mismatches and repeated releases are harmless, including after transfer. +func (s *Store) ReleaseResource(ctx context.Context, lease ResourceLease) error { + s.resourceMu.Lock() + defer s.resourceMu.Unlock() + return s.releaseResourceLocked(ctx, lease) +} + +func (s *Store) releaseResourceLocked(ctx context.Context, lease ResourceLease) error { + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + return err + } + defer tx.Rollback() + n, err := deleteResourceOwner(ctx, tx, lease) + if err != nil { + return err + } + if n == 0 { + return nil + } + var next ResourceLease + next.Resource = lease.Resource + err = tx.QueryRowContext(ctx, `SELECT run_id,stage_key,sequence,token FROM resource_waiters WHERE resource_key=? ORDER BY queue_order LIMIT 1`, lease.Resource).Scan(&next.RunID, &next.Stage, &next.Sequence, &next.token) + if err != nil && err != sql.ErrNoRows { + return err + } + if err == nil { + if s.resourceWaiters[next.ResourceVisit] == nil { + return fmt.Errorf("resource waiter has no live runner; refusing handoff") + } + if _, err := tx.ExecContext(ctx, `INSERT INTO resource_owners(resource_key,run_id,stage_key,sequence,token) VALUES(?,?,?,?,?)`, next.Resource, next.RunID, next.Stage, next.Sequence, next.token); err != nil { + return err + } + if _, err := tx.ExecContext(ctx, "DELETE FROM resource_waiters WHERE run_id=?", next.RunID); err != nil { + return err + } + if _, err := tx.ExecContext(ctx, "UPDATE pipeline_runs SET resource_wait='' WHERE id=?", next.RunID); err != nil { + return err + } + } + if err := tx.Commit(); err != nil { + return err + } + if next.token != "" { + s.resourceWaiters[next.ResourceVisit] <- next + } + return nil +} + +func deleteResourceOwner(ctx context.Context, tx *sql.Tx, lease ResourceLease) (int64, error) { + result, err := tx.ExecContext(ctx, `DELETE FROM resource_owners WHERE resource_key=? AND run_id=? AND stage_key=? AND sequence=? AND token=? AND cleanup_confirmed=1`, lease.Resource, lease.RunID, lease.Stage, lease.Sequence, lease.token) + if err != nil { + return 0, err + } + return result.RowsAffected() +} + +// recoverResourceOwners runs only at startup, after pending waiters have been +// interrupted and removed. Every reclamation uses the persisted capability; +// absent cleanup evidence is never replaced with a timestamp-based expiry. +func recoverResourceOwners(ctx context.Context, tx *sql.Tx) error { + rows, err := tx.QueryContext(ctx, "SELECT resource_key,run_id,stage_key,sequence,token FROM resource_owners WHERE cleanup_confirmed=1") + if err != nil { + return err + } + var leases []ResourceLease + for rows.Next() { + var lease ResourceLease + if err := rows.Scan(&lease.Resource, &lease.RunID, &lease.Stage, &lease.Sequence, &lease.token); err != nil { + rows.Close() + return err + } + leases = append(leases, lease) + } + err = rows.Err() + rows.Close() + if err != nil { + return err + } + for _, lease := range leases { + if _, err := deleteResourceOwner(ctx, tx, lease); err != nil { + return err + } + } + _, err = tx.ExecContext(ctx, "UPDATE resource_owners SET diagnostic='Server stopped; resource quarantined pending inspected cleanup/restoration'") + return err +} diff --git a/internal/pipeline/resources_test.go b/internal/pipeline/resources_test.go new file mode 100644 index 00000000..58ae2e6a --- /dev/null +++ b/internal/pipeline/resources_test.go @@ -0,0 +1,288 @@ +package pipeline + +import ( + "context" + "encoding/json" + "fmt" + "path/filepath" + "strings" + "sync" + "testing" + "time" +) + +func TestResourceDeclaration(t *testing.T) { + base := Template([]string{"work"}, true) + for _, value := range []string{`{"exclusive":"phone-X"}`, `{ "exclusive" : "A_1" }`} { + graph, err := Parse(base + "```resources\n" + value + "\n```\n") + if err != nil || graph.Stages[0].Resources == nil { + t.Fatalf("%s: %v", value, err) + } + if _, ok := StageDefinition(graph.Stages[0]).Blocks["resources"]; ok { + t.Fatal("runner control reached executor") + } + encoded, _ := json.Marshal(graph) + if !strings.Contains(string(encoded), `"resources"`) { + t.Fatal("resource lost in persisted graph") + } + } + for _, value := range []string{`{}`, `null`, `[]`, `{"exclusive":""}`, `{"exclusive":"a b"}`, `{"exclusive":"a","exclusive":"b"}`, `{"exclusive":"a","extra":1}`, `{"exclusive":["a","b"]}`, fmt.Sprintf(`{"exclusive":%q}`, strings.Repeat("a", 129))} { + if _, err := Parse(base + "```resources\n" + value + "\n```\n"); err == nil { + t.Fatalf("accepted %s", value) + } + } + human := "# stage gate\n```prompt\nChoose\n```\n```configs\ntype=human\n```\n```edges\ngo -> #success\n```\n```resources\n{\"exclusive\":\"phone\"}\n```\n" + if _, err := Parse(human); err == nil { + t.Fatal("human acquired a resource") + } +} + +func resourceStore(t *testing.T) (*Store, func(string) ResourceVisit) { + t.Helper() + ctx := context.Background() + s, err := Open(filepath.Join(t.TempDir(), "resources.db")) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { s.Close() }) + if _, err := s.Submit(ctx, "resource", Template([]string{"work"}, true)); err != nil { + t.Fatal(err) + } + items, _ := s.List(ctx) + d, _ := s.Get(ctx, items[0].ID, 1) + count := 0 + return s, func(resource string) ResourceVisit { + count++ + id, _, err := s.CreateRun(ctx, d, fmt.Sprint(count), "workspace", false, nil, nil) + if err != nil { + t.Fatal(err) + } + return ResourceVisit{Resource: resource, RunID: id, Stage: "work", Sequence: 1} + } +} + +func acquireTestResource(t *testing.T, s *Store, visit ResourceVisit) ResourceLease { + t.Helper() + l, ch, _, err := s.RequestResource(context.Background(), visit) + if err != nil || ch != nil { + t.Fatalf("acquire: %v %v", ch, err) + } + return l +} + +func releaseTestResource(t *testing.T, s *Store, lease ResourceLease) { + t.Helper() + if err := s.SetResourceCleanup(context.Background(), lease, true); err != nil { + t.Fatal(err) + } + if err := s.ReleaseResource(context.Background(), lease); err != nil { + t.Fatal(err) + } +} + +func TestResourceFIFOOwnershipCancellationAndRegistry(t *testing.T) { + s, create := resourceStore(t) + ctx := context.Background() + a := acquireTestResource(t, s, create("X")) + duplicate := acquireTestResource(t, s, a.ResourceVisit) + if duplicate != a { + t.Fatal("duplicate acquisition changed capability") + } + var visits []ResourceVisit + var channels []<-chan ResourceLease + for i := 0; i < 4; i++ { + v := create("X") + _, ch, wait, err := s.RequestResource(ctx, v) + if err != nil || ch == nil || wait == nil { + t.Fatal("missing waiter", err) + } + _, again, _, err := s.RequestResource(ctx, v) + if err != nil || again != ch { + t.Fatal("duplicate waiter registered", err) + } + visits = append(visits, v) + channels = append(channels, ch) + } + if err := s.ReleaseResource(ctx, a); err != nil { + t.Fatal(err) + } // no cleanup receipt + for _, ch := range channels { + select { + case <-ch: + t.Fatal("unconfirmed release granted") + default: + } + } + if err := s.SetResourceCleanup(ctx, a, true); err != nil { + t.Fatal(err) + } + for _, corrupt := range []func(*ResourceLease){func(l *ResourceLease) { l.RunID++ }, func(l *ResourceLease) { l.Stage = "other" }, func(l *ResourceLease) { l.Sequence++ }, func(l *ResourceLease) { l.token = "wrong" }} { + bad := a + corrupt(&bad) + if err := s.ReleaseResource(ctx, bad); err != nil { + t.Fatal(err) + } + } + for _, ch := range channels { + select { + case <-ch: + t.Fatal("unconfirmed/mismatched release granted") + default: + } + } + if err := s.CancelResource(ctx, visits[1]); err != nil { + t.Fatal(err) + } + releaseTestResource(t, s, a) + for _, i := range []int{0, 2, 3} { + var grant ResourceLease + select { + case grant = <-channels[i]: + case <-time.After(time.Second): + t.Fatal("FIFO grant not delivered") + } + s.AcceptResource(grant) + if grant.ResourceVisit != visits[i] { + t.Fatal("wrong FIFO owner") + } + // An old owner's repeated release cannot release the next owner. + if err := s.ReleaseResource(ctx, a); err != nil { + t.Fatal(err) + } + for _, j := range []int{2, 3} { + if j > i { + select { + case <-channels[j]: + t.Fatal("multiple grants") + default: + } + } + } + releaseTestResource(t, s, grant) + } + if len(s.resourceWaiters) != 0 { + t.Fatal("channel registry leaked") + } + for i := 0; i < 30; i++ { + owner := acquireTestResource(t, s, create("X")) + v := create("X") + _, _, _, err := s.RequestResource(ctx, v) + if err != nil { + t.Fatal(err) + } + releaseTestResource(t, s, owner) // race window: granted but not accepted + if err := s.CancelResource(ctx, v); err != nil { + t.Fatal(err) + } + if len(s.resourceWaiters) != 0 { + t.Fatal("grant/cancel leaked channel") + } + } + for i := 0; i < 30; i++ { + owner := acquireTestResource(t, s, create("X")) + v := create("X") + if _, _, _, err := s.RequestResource(ctx, v); err != nil { + t.Fatal(err) + } + if err := s.SetResourceCleanup(ctx, owner, true); err != nil { + t.Fatal(err) + } + var wg sync.WaitGroup + wg.Add(2) + go func() { + defer wg.Done() + if err := s.ReleaseResource(ctx, owner); err != nil { + t.Error(err) + } + }() + go func() { + defer wg.Done() + if err := s.CancelResource(ctx, v); err != nil { + t.Error(err) + } + }() + wg.Wait() + if len(s.resourceWaiters) != 0 { + t.Fatal("concurrent grant/cancel leaked channel") + } + } + var owners, waiters int + s.db.QueryRow("SELECT COUNT(*) FROM resource_owners").Scan(&owners) + s.db.QueryRow("SELECT COUNT(*) FROM resource_waiters").Scan(&waiters) + if owners != 0 || waiters != 0 { + t.Fatalf("stranded owners/waiters: %d/%d", owners, waiters) + } +} + +func TestResourceCancellationPersistenceFailureRemovesChannel(t *testing.T) { + s, create := resourceStore(t) + acquireTestResource(t, s, create("X")) + v := create("X") + if _, _, _, err := s.RequestResource(context.Background(), v); err != nil { + t.Fatal(err) + } + if err := s.Close(); err != nil { + t.Fatal(err) + } + if err := s.CancelResource(context.Background(), v); err == nil { + t.Fatal("closed DB cancellation claimed success") + } + if len(s.resourceWaiters) != 0 { + t.Fatal("persistence failure leaked channel") + } +} + +func TestResourceRestartQuarantineAndRetention(t *testing.T) { + for _, confirmed := range []bool{false, true} { + t.Run(fmt.Sprint(confirmed), func(t *testing.T) { + s, create := resourceStore(t) + ctx := context.Background() + a := acquireTestResource(t, s, create("X")) + b := create("X") + if _, _, _, err := s.RequestResource(ctx, b); err != nil { + t.Fatal(err) + } + if confirmed { + if err := s.SetResourceCleanup(ctx, a, true); err != nil { + t.Fatal(err) + } + } + path := s.path + if err := s.Close(); err != nil { + t.Fatal(err) + } + reopened, err := Open(path) + if err != nil { + t.Fatal(err) + } + defer reopened.Close() + if err := reopened.RecoverRuns(ctx); err != nil { + t.Fatal(err) + } + if err := reopened.RecoverRuns(ctx); err != nil { + t.Fatal(err) + } + detail, err := reopened.GetRun(ctx, b.RunID) + if err != nil || detail.Status != "interrupted" || detail.ResourceWait != nil || len(detail.Steps) != 0 || !strings.Contains(detail.Diagnostic, "resource") { + t.Fatal("queued visit replayed/lost diagnostic", detail, err) + } + exists, err := reopened.HasResourceOwner(ctx, a.RunID) + if err != nil || exists == confirmed { + t.Fatal("incorrect recovery disposition", exists, err) + } + report, err := reopened.RetentionReport(ctx, RetentionPolicy{Keep: 0}, "", time.Now()) + if err != nil { + t.Fatal(err) + } + protected := false + for _, p := range report.Protected { + if p.ID == a.RunID { + protected = true + } + } + if protected == confirmed { + t.Fatal("quarantined evidence not protected") + } + }) + } +} diff --git a/internal/pipeline/retention.go b/internal/pipeline/retention.go index c4873632..0e7a38f5 100644 --- a/internal/pipeline/retention.go +++ b/internal/pipeline/retention.go @@ -130,11 +130,12 @@ type retentionQuerier interface { // retentionRunInfo is one pipeline_runs row plus the step aggregates protection // needs, read in the same statement so the report never issues a per-run query. type retentionRunInfo struct { - id int64 - status string - createdAt time.Time - waitingSteps int - activeSteps int + id int64 + status string + createdAt time.Time + waitingSteps int + activeSteps int + resourceOwners int } // protectionReasons lists why a run must not be removed for a reason intrinsic @@ -155,6 +156,9 @@ func (r retentionRunInfo) protectionReasons() []string { if r.activeSteps > 0 { reasons = append(reasons, "cleanup agent still active (agent_active=1)") } + if r.resourceOwners > 0 { + reasons = append(reasons, "resource ownership retained (possibly quarantined)") + } return reasons } @@ -171,9 +175,20 @@ type retentionSelection struct { // report is a bounded number of round trips rather than one query per run. func selectRetentionRuns(ctx context.Context, q retentionQuerier, policy RetentionPolicy, now time.Time) (retentionSelection, error) { var sel retentionSelection + // Maintenance is deliberately non-migrating and may inspect a database + // whose server predates resource support. + var resources bool + if err := q.QueryRowContext(ctx, "SELECT EXISTS(SELECT 1 FROM sqlite_master WHERE type='table' AND name='resource_owners')").Scan(&resources); err != nil { + return sel, err + } + owners := "0" + if resources { + owners = "(SELECT COUNT(*) FROM resource_owners o WHERE o.run_id=r.id)" + } rows, err := q.QueryContext(ctx, `SELECT r.id,r.status,strftime('%Y-%m-%dT%H:%M:%SZ',r.created_at), COALESCE((SELECT COUNT(*) FROM run_steps s WHERE s.run_id=r.id AND s.status=?),0), -COALESCE((SELECT COUNT(*) FROM run_steps s WHERE s.run_id=r.id AND s.agent_active=1),0) `+ +COALESCE((SELECT COUNT(*) FROM run_steps s WHERE s.run_id=r.id AND s.agent_active=1),0), +`+owners+` `+ runFrom+` ORDER BY r.id DESC`, protocol.StepStatusWaiting) if err != nil { return sel, err @@ -182,7 +197,7 @@ COALESCE((SELECT COUNT(*) FROM run_steps s WHERE s.run_id=r.id AND s.agent_activ for rows.Next() { var run retentionRunInfo var created string - if err := rows.Scan(&run.id, &run.status, &created, &run.waitingSteps, &run.activeSteps); err != nil { + if err := rows.Scan(&run.id, &run.status, &created, &run.waitingSteps, &run.activeSteps, &run.resourceOwners); err != nil { return sel, err } run.createdAt = parseRetentionTime(created) diff --git a/internal/pipeline/runs.go b/internal/pipeline/runs.go index 5b58328c..76e1de11 100644 --- a/internal/pipeline/runs.go +++ b/internal/pipeline/runs.go @@ -21,14 +21,19 @@ const runStamps = `strftime('%Y-%m-%dT%H:%M:%SZ',r.created_at),COALESCE(strftime const runSelect = `SELECT r.id,r.pipeline_id,p.name,r.version,v.content_hash,r.status,r.outcome,r.current_stage, ` + runStamps + `,r.diagnostic, CASE WHEN r.status='succeeded' THEN r.output_json ELSE COALESCE((SELECT output_json FROM run_steps WHERE run_id=r.id AND status='completed' ORDER BY sequence DESC LIMIT 1),'{}') END, -(SELECT COUNT(*) FROM run_steps WHERE run_id=r.id AND status='completed'),r.workspace,r.title,r.model,r.stage_models +(SELECT COUNT(*) FROM run_steps WHERE run_id=r.id AND status='completed'),r.workspace,r.title,r.model,r.stage_models,r.resource_wait ` + runFrom + ` JOIN pipeline_versions v ON v.pipeline_id=r.pipeline_id AND v.version=r.version` func scanRun(row interface{ Scan(...any) error }, r *protocol.RunSummary) error { - var output, stageModels string - if err := row.Scan(&r.ID, &r.PipelineID, &r.Name, &r.Version, &r.ContentHash, &r.Status, &r.Outcome, &r.CurrentStage, &r.CreatedAt, &r.FinishedAt, &r.Diagnostic, &output, &r.CompletedSteps, &r.Workspace, &r.Title, &r.Model, &stageModels); err != nil { + var output, stageModels, resourceWait string + if err := row.Scan(&r.ID, &r.PipelineID, &r.Name, &r.Version, &r.ContentHash, &r.Status, &r.Outcome, &r.CurrentStage, &r.CreatedAt, &r.FinishedAt, &r.Diagnostic, &output, &r.CompletedSteps, &r.Workspace, &r.Title, &r.Model, &stageModels, &resourceWait); err != nil { return err } + if resourceWait != "" { + if err := json.Unmarshal([]byte(resourceWait), &r.ResourceWait); err != nil { + return err + } + } if stageModels != "" && stageModels != "{}" { if err := json.Unmarshal([]byte(stageModels), &r.StageModels); err != nil { return err diff --git a/internal/pipeline/runs_mutate.go b/internal/pipeline/runs_mutate.go index d27e7268..d370a29f 100644 --- a/internal/pipeline/runs_mutate.go +++ b/internal/pipeline/runs_mutate.go @@ -44,7 +44,7 @@ func (s *Store) BeginStep(ctx context.Context, id int64, sequence int, key strin return err } defer tx.Rollback() - if err := affected(tx.ExecContext(ctx, "UPDATE pipeline_runs SET status='running',current_stage=? WHERE id=? AND status IN ('queued','running')", key, id)); err != nil { + if err := affected(tx.ExecContext(ctx, "UPDATE pipeline_runs SET status='running',current_stage=?,resource_wait='' WHERE id=? AND status IN ('queued','running')", key, id)); err != nil { return err } if _, err := tx.ExecContext(ctx, "INSERT INTO run_steps(run_id,sequence,stage_key,input_json) VALUES (?,?,?,?)", id, sequence, key, string(encoded)); err != nil { @@ -310,5 +310,5 @@ func (s *Store) FinishRun(ctx context.Context, id int64, status, outcome, diagno if err != nil { return err } - return affected(s.db.ExecContext(ctx, "UPDATE pipeline_runs SET status=?,outcome=?,diagnostic=?,output_json=?,finished_at=CURRENT_TIMESTAMP WHERE id=? AND status IN ('queued','running')", status, outcome, diagnostic, string(data), id)) + return affected(s.db.ExecContext(ctx, "UPDATE pipeline_runs SET status=?,outcome=?,diagnostic=?,output_json=?,finished_at=CURRENT_TIMESTAMP,resource_wait='' WHERE id=? AND status IN ('queued','running')", status, outcome, diagnostic, string(data), id)) } diff --git a/internal/pipeline/runs_schema.go b/internal/pipeline/runs_schema.go index a8c1c78c..8f861061 100644 --- a/internal/pipeline/runs_schema.go +++ b/internal/pipeline/runs_schema.go @@ -51,13 +51,28 @@ CREATE TABLE IF NOT EXISTS run_session_owners ( run_id INTEGER NOT NULL, sequence INTEGER NOT NULL, backend TEXT NOT NULL, reference TEXT NOT NULL DEFAULT '', PRIMARY KEY(run_id,sequence), FOREIGN KEY(run_id,sequence) REFERENCES run_steps(run_id,sequence)); CREATE TABLE IF NOT EXISTS run_retries ( -run_id INTEGER PRIMARY KEY REFERENCES pipeline_runs(id), original_run_id INTEGER NOT NULL REFERENCES pipeline_runs(id));`) +run_id INTEGER PRIMARY KEY REFERENCES pipeline_runs(id), original_run_id INTEGER NOT NULL REFERENCES pipeline_runs(id)); +CREATE TABLE IF NOT EXISTS resource_owners ( +resource_key TEXT PRIMARY KEY, run_id INTEGER NOT NULL REFERENCES pipeline_runs(id), +stage_key TEXT NOT NULL, sequence INTEGER NOT NULL, token TEXT NOT NULL, +cleanup_confirmed INTEGER NOT NULL DEFAULT 0, diagnostic TEXT NOT NULL DEFAULT '', +acquired_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP); +CREATE INDEX IF NOT EXISTS resource_owner_run ON resource_owners(run_id); +CREATE UNIQUE INDEX IF NOT EXISTS resource_owner_visit ON resource_owners(run_id); +CREATE TABLE IF NOT EXISTS resource_waiters ( +queue_order INTEGER PRIMARY KEY AUTOINCREMENT, resource_key TEXT NOT NULL, +run_id INTEGER NOT NULL UNIQUE REFERENCES pipeline_runs(id), stage_key TEXT NOT NULL, +sequence INTEGER NOT NULL, token TEXT NOT NULL, queued_at TEXT NOT NULL); +CREATE INDEX IF NOT EXISTS resource_wait_fifo ON resource_waiters(resource_key,queue_order);`) if err != nil { return err } if err := s.migrateRunTitles(); err != nil { return err } + if err := s.addColumnIfMissing("pipeline_runs", "resource_wait", "TEXT NOT NULL DEFAULT ''"); err != nil { + return err + } return s.migrateRunModels() } @@ -122,16 +137,29 @@ func (s *Store) addColumnIfMissing(table, column, definition string) error { } func (s *Store) RecoverRuns(ctx context.Context) error { + s.resourceMu.Lock() + defer s.resourceMu.Unlock() tx, err := s.db.BeginTx(ctx, nil) if err != nil { return err } defer tx.Rollback() - if _, err := tx.ExecContext(ctx, "UPDATE run_steps SET status='interrupted',agent_active=0,diagnostic='Server stopped; stage was not resumed',finished_at=CURRENT_TIMESTAMP WHERE (status='running' OR agent_active=1) AND status<>?", protocol.StepStatusWaiting); err != nil { + // No replay: pending visits never began. Uncertain owners survive recovery, + // including the crash window before BeginStep/session registration. + if _, err := tx.ExecContext(ctx, "UPDATE pipeline_runs SET diagnostic='Server stopped while queued for a resource; pending visit was not resumed',status='interrupted',finished_at=CURRENT_TIMESTAMP,resource_wait='' WHERE id IN (SELECT run_id FROM resource_waiters); DELETE FROM resource_waiters"); err != nil { + return err + } + if err := recoverResourceOwners(ctx, tx); err != nil { + return err + } + if _, err := tx.ExecContext(ctx, "UPDATE run_steps SET status='interrupted',agent_active=CASE WHEN EXISTS(SELECT 1 FROM resource_owners o WHERE o.run_id=run_steps.run_id AND o.sequence=run_steps.sequence) THEN agent_active ELSE 0 END,diagnostic='Server stopped; stage was not resumed',finished_at=CURRENT_TIMESTAMP WHERE (status='running' OR agent_active=1) AND status<>?", protocol.StepStatusWaiting); err != nil { return err } if _, err := tx.ExecContext(ctx, "UPDATE pipeline_runs SET status='interrupted',diagnostic='Server stopped; run was not resumed',finished_at=CURRENT_TIMESTAMP WHERE status IN ('queued','running') AND status<>?", protocol.RunStatusWaiting); err != nil { return err } + if _, err := tx.ExecContext(ctx, "UPDATE pipeline_runs SET diagnostic='Server stopped; resource ownership quarantined: inspect retained session, tool termination and restoration before reconciliation' WHERE id IN (SELECT run_id FROM resource_owners)"); err != nil { + return err + } return tx.Commit() } diff --git a/internal/pipeline/stage.go b/internal/pipeline/stage.go index d2a04b44..3afb8135 100644 --- a/internal/pipeline/stage.go +++ b/internal/pipeline/stage.go @@ -18,6 +18,7 @@ func StageDefinition(s Stage) stage.Definition { delete(blocks, "input") // Retired contracts in historical graph JSON are ignored. delete(blocks, "limits") delete(blocks, "memory") + delete(blocks, "resources") for name, value := range map[string]string{"output": s.Output, "prompt": s.Prompt} { if value != "" { blocks[name] = value diff --git a/internal/pipeline/store.go b/internal/pipeline/store.go index 46466d51..abcfe29a 100644 --- a/internal/pipeline/store.go +++ b/internal/pipeline/store.go @@ -18,10 +18,12 @@ import ( ) type Store struct { - db *sql.DB - dir string - path string - mu sync.Mutex + db *sql.DB + dir string + path string + mu sync.Mutex + resourceMu sync.Mutex + resourceWaiters map[ResourceVisit]chan ResourceLease } // openDB opens one SQLite connection pool for a DSN. Every open path shares it diff --git a/internal/protocol/message.go b/internal/protocol/message.go index 74065770..f3d1f8d5 100644 --- a/internal/protocol/message.go +++ b/internal/protocol/message.go @@ -19,6 +19,7 @@ const ( ) type Stage struct { + Resources *StageResources `json:"resources,omitempty"` Limits *StageLimits `json:"limits,omitempty"` Memory map[string]string `json:"memory,omitempty"` Notify *NotificationPolicy `json:"notify,omitempty"` @@ -33,6 +34,11 @@ type Stage struct { Blocks map[string]string `json:"blocks,omitempty"` } +// StageResources declares one capacity-one resource, local to this store authority. +type StageResources struct { + Exclusive string `json:"exclusive"` +} + // Limits are enforced by the runner and never included in the model prompt. type StageLimits struct { MaxVisits int `json:"max_visits"` diff --git a/internal/protocol/resources_test.go b/internal/protocol/resources_test.go new file mode 100644 index 00000000..f6c8cd41 --- /dev/null +++ b/internal/protocol/resources_test.go @@ -0,0 +1,29 @@ +package protocol + +import ( + "encoding/json" + "strings" + "testing" +) + +func TestResourceWaitOlderClientCompatibility(t *testing.T) { + data, err := json.Marshal(RunSummary{ID: 1, Status: RunStatusQueued, ResourceWait: &ResourceWait{Resource: "X", Stage: "review", Sequence: 2, QueuedSince: "now"}}) + if err != nil { + t.Fatal(err) + } + var old struct { + ID int64 `json:"id"` + Status string `json:"status"` + } + if err := json.Unmarshal(data, &old); err != nil || old.Status != "queued" { + t.Fatal("old client no longer waits", old, err) + } + clear, err := json.Marshal(RunEventFrame{RunID: 1, ResourceWaitSet: true}) + if err != nil || !strings.Contains(string(clear), `"resource_wait_set":true`) { + t.Fatal("no explicit clear", string(clear), err) + } + var current RunSummary + if err := json.Unmarshal([]byte(`{"id":2,"status":"running"}`), ¤t); err != nil || current.ResourceWait != nil { + t.Fatal("missing field changed old payload", err) + } +} diff --git a/internal/protocol/runevents.go b/internal/protocol/runevents.go index f3b424fd..240ad2b1 100644 --- a/internal/protocol/runevents.go +++ b/internal/protocol/runevents.go @@ -24,8 +24,11 @@ const ( // from the frame instead of re-reading the whole list. They repeat on every // frame (pipeline identity never changes mid-run). type RunEventFrame struct { - RunID int64 `json:"run_id"` - Workspace string `json:"workspace,omitempty"` + // ResourceWaitSet makes a nil ResourceWait an explicit cache clear. + ResourceWaitSet bool `json:"resource_wait_set,omitempty"` + ResourceWait *ResourceWait `json:"resource_wait,omitempty"` + RunID int64 `json:"run_id"` + Workspace string `json:"workspace,omitempty"` // PipelineID, Name, Title and Version identify the run's pipeline and its // operator-facing title, so a new run's row can be built without a fetch. PipelineID int64 `json:"pipeline_id,omitempty"` diff --git a/internal/protocol/runs.go b/internal/protocol/runs.go index 87d94a1e..fb3f2830 100644 --- a/internal/protocol/runs.go +++ b/internal/protocol/runs.go @@ -118,9 +118,10 @@ type SessionExport struct { } type RunSummary struct { - ID int64 `json:"id"` - PipelineID int64 `json:"pipeline_id"` - Name string `json:"name"` + ResourceWait *ResourceWait `json:"resource_wait,omitempty"` + ID int64 `json:"id"` + PipelineID int64 `json:"pipeline_id"` + Name string `json:"name"` // Title is the operator-facing, bounded single-line description of what the // run is for, generated from its prompt. It is never empty for a new run; // historical rows predating the title column may omit it. @@ -145,6 +146,19 @@ type RunSummary struct { StageModels map[string]string `json:"stage_models,omitempty"` } +// ResourceWait is bounded public queue metadata. Ownership credentials stay private. +type ResourceWait struct { + Resource string `json:"resource"` + Stage string `json:"stage"` + Sequence int `json:"sequence"` + QueuedSince string `json:"queued_since"` +} + +// ResourceCleanupConfirmedKey lets a stage report failed restoration without +// pretending that tool idleness also proves the resource is ready. An explicit +// false (or malformed value) vetoes release; it never authorizes release alone. +const ResourceCleanupConfirmedKey = "resource_cleanup_confirmed" + // Label is the operator-facing name of a run: the generated Title when the // server stored one, otherwise the pipeline Name (historical rows predate // titles). It is the single definition shared by the CLI and TUI. @@ -155,6 +169,14 @@ func (r RunSummary) Label() string { return r.Name } +// StatusLabel separates resource contention from a human decision everywhere. +func (r RunSummary) StatusLabel() string { + if r.ResourceWait != nil { + return "Waiting for resource " + r.ResourceWait.Resource + } + return r.Status +} + type RunStep struct { Control *ControlEvidence `json:"control,omitempty"` FileChanges *StageFileChanges `json:"file_changes,omitempty"` diff --git a/internal/server/resource_cli_test.go b/internal/server/resource_cli_test.go new file mode 100644 index 00000000..a377dc70 --- /dev/null +++ b/internal/server/resource_cli_test.go @@ -0,0 +1,154 @@ +package server + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/json" + "fmt" + "net/http/httptest" + "os" + "path/filepath" + "reflect" + "strings" + "testing" + "time" + + "git.a-wan.net/alan/pipelines/internal/cli" + "git.a-wan.net/alan/pipelines/internal/pipeline" + "git.a-wan.net/alan/pipelines/internal/protocol" + "git.a-wan.net/alan/pipelines/internal/stage" +) + +// This is a real CLI/WebSocket/scratch-SQLite exercise with an explicitly FAKE +// executor. It never calls inference, a phone, or a deployed server. +func TestResourceScratchCLIWaitingThenProgress(t *testing.T) { + t.Setenv("PIPELINES_HARNESS_CONFIG", filepath.Join(t.TempDir(), "harnesses.json")) + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + store, err := pipeline.Open(filepath.Join(t.TempDir(), "scratch.db")) + if err != nil { + t.Fatal(err) + } + defer store.Close() + source := pipeline.Template([]string{"work"}, true) + "```resources\n{\"exclusive\":\"fake-phone\"}\n```\n" + if _, err := store.Submit(ctx, "scratch-resource", source); err != nil { + t.Fatal(err) + } + handler := Handler(ctx, store).(*runHandler) + started, release := make(chan struct{}), make(chan struct{}) + handler.runner.execute = func(ctx context.Context, _ stage.Definition, input stage.Values) (stage.Result, error) { + if input["prompt"] == "A" { + close(started) + select { + case <-release: + case <-ctx.Done(): + return stage.Result{}, ctx.Err() + } + } + return stage.Result{Outcome: "success", Output: stage.Values{"execution": "FAKE executor; no device or inference"}}, nil + } + handler.runner.events.interval = time.Millisecond + events := make(chan protocol.RunEventFrame, 64) + sub := handler.runner.events.subscribe("cli-evidence", "", nil, func(m protocol.Message) error { + if m.RunEvent != nil { + events <- *m.RunEvent + } + return nil + }) + sub.ready() + defer sub.close() + srv := httptest.NewServer(handler) + defer srv.Close() + defer func() { cancel(); WaitRuns(handler) }() + endpoint := "ws" + strings.TrimPrefix(srv.URL, "http") + "/connect" + run := func(args ...string) string { + var out bytes.Buffer + app := cli.App{Out: &out, Err: &out, In: strings.NewReader("")} + if err := app.Run(ctx, append([]string{"--url", endpoint}, args...)); err != nil { + t.Fatalf("CLI %v: %v\n%s", args, err, out.String()) + } + t.Logf("pipelines %s\n%s", strings.Join(args, " "), out.String()) + return out.String() + } + parseRun := func(output string) int64 { + var m protocol.Message + if err := json.Unmarshal([]byte(output), &m); err != nil || m.RunDetail == nil { + t.Fatal("CLI run response", output, err) + } + return m.RunDetail.ID + } + run("validate", "testdata/android-resource-draft.md", "--json") + a := parseRun(run("run", "scratch-resource", "A", "--workspace", t.TempDir(), "--yes", "--json", "--request-key", "a")) + <-started + b := parseRun(run("run", "scratch-resource", "B", "--workspace", t.TempDir(), "--yes", "--json", "--request-key", "b")) + awaitResourceEvent(t, events, b, func(e protocol.RunEventFrame) bool { return e.ResourceWait != nil }) + var detail protocol.RunDetail + if err := json.Unmarshal([]byte(run("inspect", fmt.Sprint(b), "--json")), &detail); err != nil || detail.ResourceWait == nil || detail.Status != "queued" || len(detail.Steps) != 0 { + t.Fatal("CLI did not observe queue", err) + } + if output := run("runs"); !strings.Contains(output, "Waiting for resource fake-phone") { + t.Fatal("list omitted resource status", output) + } + var snapshot *protocol.RunEventFrame + handleRunEvents(protocol.Message{Type: protocol.Subscribe, RunIDs: []int64{b}}, handler.runner.events, handler.runner, "snapshot-evidence", func(m protocol.Message) error { + if m.RunEvent != nil { + snapshot = m.RunEvent + } + return nil + }) + handler.runner.events.unsubscribeClient("snapshot-evidence") + if snapshot == nil || snapshot.ResourceWait == nil || !snapshot.ResourceWaitSet { + t.Fatal("subscription snapshot omitted resource wait") + } + close(release) + run("wait", fmt.Sprint(a), fmt.Sprint(b), "--json") + detail = protocol.RunDetail{} + if err := json.Unmarshal([]byte(run("inspect", fmt.Sprint(b), "--json")), &detail); err != nil || detail.Status != "succeeded" || detail.ResourceWait != nil || len(detail.Steps) != 1 { + t.Fatal("automatic progress not observed", err) + } + t.Log("Observed queued resource wait followed by automatic progress; FAKE executor only, no phone/inference/deployment.") +} + +func TestResourceAndroidDraftPublishedV3Provenance(t *testing.T) { + baseline, err := os.ReadFile("testdata/android-v3-published.md") + if err != nil { + t.Fatal(err) + } + if hash := fmt.Sprintf("%x", sha256.Sum256(baseline)); hash != "4a3a4b8b947eb20c115c5470e6e378f117d5fa291a7b6c8ae966a3745bc826de" { + t.Fatal("baseline is not retrieved Android v3", hash) + } + draft, err := os.ReadFile("testdata/android-resource-draft.md") + if err != nil { + t.Fatal(err) + } + old, err := pipeline.Parse(string(baseline)) + if err != nil { + t.Fatal(err) + } + updated, err := pipeline.Parse(string(draft)) + if err != nil { + t.Fatal(err) + } + if len(old.Stages) != len(updated.Stages) { + t.Fatal("draft changed graph shape") + } + for i := range old.Stages { + a, b := old.Stages[i], updated.Stages[i] + if b.Configs["provider"] != "openai" || b.Configs["model"] != a.Configs["model"] { + t.Fatal("draft changed approved model") + } + if b.Key == "ui_reviewer" { + if b.Resources == nil || b.Resources.Exclusive != "phone-47241FDAS00DLK" { + t.Fatal("reviewer resource missing") + } + } else if b.Resources != nil { + t.Fatal("non-UI stage acquired phone") + } + a.Prompt, b.Prompt = "", "" + a.Resources, b.Resources = nil, nil + if !reflect.DeepEqual(a, b) { + t.Fatal("draft changed fields outside approved role split/resource", a.Key) + } + } +} diff --git a/internal/server/resources.go b/internal/server/resources.go new file mode 100644 index 00000000..ecb5b7a5 --- /dev/null +++ b/internal/server/resources.go @@ -0,0 +1,60 @@ +package server + +import ( + "context" + "errors" + "time" + + "git.a-wan.net/alan/pipelines/internal/pipeline" + "git.a-wan.net/alan/pipelines/internal/protocol" + "git.a-wan.net/alan/pipelines/internal/stage" +) + +func (r *runner) acquireResource(ctx context.Context, visit pipeline.ResourceVisit, workspace string, completed int) (pipeline.ResourceLease, error) { + lease, ready, wait, err := r.store.RequestResource(ctx, visit) + if err != nil { + return lease, err + } + if ready != nil { + r.emitRun(protocol.RunEventFrame{RunID: visit.RunID, Workspace: workspace, Status: protocol.RunStatusQueued, CurrentStage: visit.Stage, CompletedSteps: completed, ResourceWaitSet: true, ResourceWait: wait}, false) + select { + case lease = <-ready: + r.store.AcceptResource(lease) + case <-ctx.Done(): + cleanup, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + if err := r.store.CancelResource(cleanup, visit); err != nil { + return lease, errors.Join(ctx.Err(), err) + } + return lease, ctx.Err() + } + } + if ctx.Err() != nil { + err := r.releaseResource(lease, true) // Never launched, including a concurrent grant. + return pipeline.ResourceLease{}, errors.Join(ctx.Err(), err) + } + r.emitRun(protocol.RunEventFrame{RunID: visit.RunID, Workspace: workspace, ResourceWaitSet: true}, false) + return lease, nil +} + +func (r *runner) releaseResource(lease pipeline.ResourceLease, confirmed bool) error { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + if err := r.store.SetResourceCleanup(ctx, lease, confirmed); err != nil { + return err + } + if confirmed { + return r.store.ReleaseResource(ctx, lease) + } + return nil +} + +func (r *runner) resourceQuiescent(id int64, executionErr error, panicked bool) bool { + if panicked || errors.Is(executionErr, stage.ErrStageCleanup) { + return false + } + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + sessions, err := r.store.ActiveAgentSessionsForRun(ctx, id) + return err == nil && len(sessions) == 0 +} diff --git a/internal/server/resources_test.go b/internal/server/resources_test.go new file mode 100644 index 00000000..837560eb --- /dev/null +++ b/internal/server/resources_test.go @@ -0,0 +1,327 @@ +package server + +import ( + "context" + "database/sql" + "errors" + "fmt" + "path/filepath" + "strings" + "sync/atomic" + "testing" + "time" + + "git.a-wan.net/alan/pipelines/internal/pipeline" + "git.a-wan.net/alan/pipelines/internal/protocol" + "git.a-wan.net/alan/pipelines/internal/stage" +) + +func resourceRunner(t *testing.T) (*runner, <-chan protocol.RunEventFrame, func(string, string, string) *protocol.RunDetail) { + t.Helper() + ctx, cancel := context.WithCancel(context.Background()) + s, err := pipeline.Open(filepath.Join(t.TempDir(), "resources.db")) + if err != nil { + t.Fatal(err) + } + r, err := newRunner(ctx, s) + if err != nil { + t.Fatal(err) + } + r.events = newRunEventHub() + r.events.interval = time.Millisecond + events := make(chan protocol.RunEventFrame, 128) + sub := r.events.subscribe("resource-test", "", nil, func(m protocol.Message) error { + if m.RunEvent != nil { + events <- *m.RunEvent + } + return nil + }) + sub.ready() + t.Cleanup(func() { cancel(); r.wg.Wait(); sub.close(); s.Close() }) + count := 0 + return r, events, func(resource, timeout, prompt string) *protocol.RunDetail { + count++ + source := strings.Replace(pipeline.Template([]string{"work"}, true), "timeout=1h", "timeout="+timeout, 1) + if resource != "" { + source += "```resources\n{\"exclusive\":\"" + resource + "\"}\n```\n" + } + name := fmt.Sprint("resource-", count) + if _, err := s.Submit(ctx, name, source); err != nil { + t.Fatal(err) + } + items, _ := s.List(ctx) + var pid int64 + for _, p := range items { + if p.Name == name { + pid = p.ID + } + } + d, err := r.start(protocol.Message{PipelineID: pid, Version: 1, RequestKey: name, Workspace: t.TempDir(), Input: map[string]any{"prompt": prompt}}) + if err != nil { + t.Fatal(err) + } + return d + } +} + +func awaitResourceEvent(t *testing.T, events <-chan protocol.RunEventFrame, id int64, predicate func(protocol.RunEventFrame) bool) protocol.RunEventFrame { + t.Helper() + timer := time.NewTimer(3 * time.Second) + defer timer.Stop() + for { + select { + case event := <-events: + if event.RunID == id && predicate(event) { + return event + } + case <-timer.C: + t.Fatalf("resource event missing for run %d", id) + return protocol.RunEventFrame{} + } + } +} + +func TestResourcePauseBeyondTimeoutAndAutomaticResume(t *testing.T) { + r, events, start := resourceRunner(t) + aStarted, releaseA := make(chan struct{}), make(chan struct{}) + bStarted := make(chan struct{}, 1) + var bCalls atomic.Int32 + r.execute = func(ctx context.Context, _ stage.Definition, input stage.Values) (stage.Result, error) { + switch input["prompt"] { + case "A": + close(aStarted) + select { + case <-releaseA: + case <-ctx.Done(): + return stage.Result{}, ctx.Err() + } + case "B": + bCalls.Add(1) + deadline, ok := ctx.Deadline() + if !ok || time.Until(deadline) < 15*time.Millisecond { + return stage.Result{}, errors.New("execution timeout was consumed in queue") + } + bStarted <- struct{}{} + } + return stage.Result{Outcome: "success", Output: stage.Values{}}, nil + } + a := start("X", "2s", "A") + <-aStarted + b := start("X", "30ms", "B") + awaitResourceEvent(t, events, b.ID, func(e protocol.RunEventFrame) bool { return e.ResourceWait != nil }) + <-time.After(70 * time.Millisecond) // Deliberately longer than B's whole execution timeout. + d, err := r.store.GetRun(context.Background(), b.ID) + if err != nil || d.Status != protocol.RunStatusQueued || d.ResourceWait == nil || len(d.Steps) != 0 || bCalls.Load() != 0 { + t.Fatal("queued execution consumed accounting", d, err, bCalls.Load()) + } + if _, err := r.decide(protocol.Message{RunID: b.ID, Option: "success"}); err == nil { + t.Fatal("resource wait exposed a human decision") + } + c := start("Y", "1s", "C") + awaitResourceEvent(t, events, c.ID, func(e protocol.RunEventFrame) bool { return e.Status == "succeeded" }) + select { + case <-bStarted: + t.Fatal("B ran while A held X") + default: + } + owner, err := r.store.GetRun(context.Background(), a.ID) + if err != nil || owner.Status != "running" { + t.Fatal("A did not remain owner", err) + } + close(releaseA) + awaitResourceEvent(t, events, b.ID, func(e protocol.RunEventFrame) bool { return e.Status == "succeeded" }) + r.wg.Wait() + d, err = r.store.GetRun(context.Background(), b.ID) + if err != nil || bCalls.Load() != 1 || d.ResourceWait != nil || len(d.Steps) != 1 || d.Steps[0].Control.Visit != 1 || d.Steps[0].Sequence != 1 { + t.Fatal("resume did not execute exactly one fresh visit", d, err) + } +} + +func TestResourceCancellationWaitsForOwnerCleanup(t *testing.T) { + r, events, start := resourceRunner(t) + started, cleaning, cleanupDone := make(chan struct{}), make(chan struct{}), make(chan struct{}) + var waitingCalls atomic.Int32 + r.execute = func(ctx context.Context, _ stage.Definition, input stage.Values) (stage.Result, error) { + if input["prompt"] == "owner" { + close(started) + <-ctx.Done() + close(cleaning) + <-cleanupDone + return stage.Result{}, ctx.Err() + } + waitingCalls.Add(1) + return stage.Result{Outcome: "success", Output: stage.Values{}}, nil + } + a := start("X", "2s", "owner") + <-started + b := start("X", "30ms", "waiter") + awaitResourceEvent(t, events, b.ID, func(e protocol.RunEventFrame) bool { return e.ResourceWait != nil }) + if err := r.cancel(a.ID); err != nil { + t.Fatal(err) + } + <-cleaning + select { + case <-time.After(50 * time.Millisecond): + if waitingCalls.Load() != 0 { + t.Fatal("handoff before cleanup") + } + } + close(cleanupDone) + awaitResourceEvent(t, events, b.ID, func(e protocol.RunEventFrame) bool { return e.Status == "succeeded" }) + if waitingCalls.Load() != 1 { + t.Fatal("waiter did not resume exactly once") + } +} + +func TestResourceQuarantineAndQueuedCancellation(t *testing.T) { + for _, failure := range []string{"cleanup", "panic", "restoration"} { + t.Run(failure, func(t *testing.T) { + r, events, start := resourceRunner(t) + started, release := make(chan struct{}), make(chan struct{}) + var calls atomic.Int32 + r.execute = func(ctx context.Context, _ stage.Definition, input stage.Values) (stage.Result, error) { + if input["prompt"] == "owner" { + close(started) + select { + case <-release: + case <-ctx.Done(): + } + if failure == "panic" { + panic("unknown cleanup") + } + if failure == "restoration" { + return stage.Result{Outcome: "success", Output: stage.Values{protocol.ResourceCleanupConfirmedKey: false}}, nil + } + return stage.Result{}, stage.ErrStageCleanup + } + calls.Add(1) + return stage.Result{Outcome: "success", Output: stage.Values{}}, nil + } + a := start("X", "1s", "owner") + <-started + b := start("X", "30ms", "waiter") + awaitResourceEvent(t, events, b.ID, func(e protocol.RunEventFrame) bool { return e.ResourceWait != nil }) + close(release) + awaitResourceEvent(t, events, a.ID, func(e protocol.RunEventFrame) bool { return e.Status == "failed" }) + if err := r.cancel(b.ID); err != nil { + t.Fatal(err) + } + awaitResourceEvent(t, events, b.ID, func(e protocol.RunEventFrame) bool { + return e.Status == "canceled" && e.ResourceWaitSet && e.ResourceWait == nil + }) + r.wg.Wait() + if calls.Load() != 0 { + t.Fatal("quarantined resource handed off") + } + exists, err := r.store.HasResourceOwner(context.Background(), a.ID) + if err != nil || !exists { + t.Fatal("quarantine evidence lost", err) + } + d, err := r.store.GetRun(context.Background(), b.ID) + if err != nil || d.ResourceWait != nil || len(d.Steps) != 0 { + t.Fatal("canceled queue consumed step", err) + } + }) + } +} + +func TestResourceEventClearCoalescing(t *testing.T) { + frame := protocol.RunEventFrame{ResourceWaitSet: true, ResourceWait: &protocol.ResourceWait{Resource: "X"}} + mergeRunEventFrame(&frame, &protocol.RunEventFrame{ResourceWaitSet: true}) + if !frame.ResourceWaitSet || frame.ResourceWait != nil { + t.Fatal("coalescer retained stale resource wait") + } +} + +func TestResourceReleaseOnBeginAndPersistenceFailure(t *testing.T) { + for _, failure := range []string{"begin", "persist"} { + t.Run(failure, func(t *testing.T) { + r, events, start := resourceRunner(t) + db, err := sql.Open("sqlite", filepath.Join(r.store.Directory(), "resources.db")) + if err != nil { + t.Fatal(err) + } + defer db.Close() + trigger := `CREATE TRIGGER resource_fail BEFORE INSERT ON run_steps WHEN json_extract(NEW.input_json,'$.prompt')='fail' BEGIN SELECT RAISE(FAIL,'test BeginStep failure'); END` + if failure == "persist" { + trigger = `CREATE TRIGGER resource_fail BEFORE UPDATE OF output_json ON run_steps WHEN json_extract(OLD.input_json,'$.prompt')='fail' BEGIN SELECT RAISE(FAIL,'test persistence failure'); END` + } + if _, err := db.Exec(trigger); err != nil { + t.Fatal(err) + } + var calls atomic.Int32 + r.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) { + calls.Add(1) + return stage.Result{Outcome: "success", Output: stage.Values{}}, nil + } + a := start("X", "1s", "fail") + awaitResourceEvent(t, events, a.ID, func(e protocol.RunEventFrame) bool { return e.Status == "failed" }) + r.wg.Wait() + if exists, err := r.store.HasResourceOwner(context.Background(), a.ID); err != nil || exists { + t.Fatal("safe failure stranded resource", err) + } + if failure == "begin" && calls.Load() != 0 { + t.Fatal("BeginStep failure executed work") + } + b := start("X", "1s", "pass") + awaitResourceEvent(t, events, b.ID, func(e protocol.RunEventFrame) bool { return e.Status == "succeeded" }) + }) + } +} + +func TestResourceCleanupQueryFailureQuarantines(t *testing.T) { + r, events, start := resourceRunner(t) + r.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) { + if err := r.store.Close(); err != nil { + return stage.Result{}, err + } + return stage.Result{Outcome: "success", Output: stage.Values{}}, nil + } + a := start("X", "1s", "query-failure") + awaitResourceEvent(t, events, a.ID, func(e protocol.RunEventFrame) bool { return e.Status == "failed" }) + r.wg.Wait() + reopened, err := pipeline.Open(filepath.Join(r.store.Directory(), "resources.db")) + if err != nil { + t.Fatal(err) + } + defer reopened.Close() + if err := reopened.RecoverRuns(context.Background()); err != nil { + t.Fatal(err) + } + if exists, err := reopened.HasResourceOwner(context.Background(), a.ID); err != nil || !exists { + t.Fatal("query failure released uncertain owner", err) + } +} + +func TestResourceSkippedVisitDoesNotAcquire(t *testing.T) { + r, events, _ := resourceRunner(t) + ctx := context.Background() + source := strings.Replace(pipeline.Template([]string{"review"}, true), "success -> #success", "success -> #success\nskipped -> #success", 1) + "```resources\n{\"exclusive\":\"X\"}\n```\n" + if _, err := r.store.Submit(ctx, "skip", source); err != nil { + t.Fatal(err) + } + items, _ := r.store.List(ctx) + d, _ := r.store.Get(ctx, items[0].ID, 1) + owner, _, err := r.store.CreateRun(ctx, d, "held", t.TempDir(), false, nil, nil) + if err != nil { + t.Fatal(err) + } + lease, ch, _, err := r.store.RequestResource(ctx, pipeline.ResourceVisit{Resource: "X", RunID: owner, Stage: "review", Sequence: 1}) + if err != nil || ch != nil { + t.Fatal("owner fixture failed", err) + } + defer r.releaseResource(lease, true) + r.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) { + t.Error("skipped stage executed") + return stage.Result{}, nil + } + run, err := r.start(protocol.Message{PipelineID: d.ID, Version: 1, RequestKey: "skip", Workspace: t.TempDir(), Input: map[string]any{"prompt": "no UI"}}) + if err != nil { + t.Fatal(err) + } + awaitResourceEvent(t, events, run.ID, func(e protocol.RunEventFrame) bool { return e.Status == "succeeded" }) + detail, err := r.store.GetRun(ctx, run.ID) + if err != nil || len(detail.Steps) != 1 || detail.Steps[0].Outcome != "skipped" { + t.Fatal("skip not preserved", detail, err) + } +} diff --git a/internal/server/retry_test.go b/internal/server/retry_test.go index acde318d..152201ce 100644 --- a/internal/server/retry_test.go +++ b/internal/server/retry_test.go @@ -29,6 +29,7 @@ func TestRetryCreatesFreshOpenCodeSessionAndRetainsFailedSession(t *testing.T) { if [ "$1" = "--help" ]; then echo ' api'; exit 0; fi printf '%%s %%s %%s\n' "$PWD" "$2" "$3" >> %q case "$3" in + /api/session/active) echo '{"data":{}}' ;; /api/session) n=0; if [ -f %q ]; then n=$(cat %q); fi n=$((n+1)); echo "$n" > %q diff --git a/internal/server/runevents.go b/internal/server/runevents.go index e35d683d..87cac293 100644 --- a/internal/server/runevents.go +++ b/internal/server/runevents.go @@ -424,6 +424,9 @@ func (s *runSubscription) close() { } func mergeRunEventFrame(into, in *protocol.RunEventFrame) { + if in.ResourceWaitSet { + into.ResourceWaitSet, into.ResourceWait = true, in.ResourceWait + } if in.Workspace != "" { into.Workspace = in.Workspace } @@ -500,18 +503,20 @@ func handleRunEvents(request protocol.Message, events *runEventHub, runs *runner continue } frame := protocol.RunEventFrame{ - RunID: summary.ID, - Workspace: summary.Workspace, - PipelineID: summary.PipelineID, - Name: summary.Name, - Title: summary.Title, - Version: summary.Version, - Status: summary.Status, - Outcome: summary.Outcome, - CurrentStage: summary.CurrentStage, - CompletedSteps: summary.CompletedSteps, - CreatedAt: summary.CreatedAt, - FinishedAt: summary.FinishedAt, + ResourceWaitSet: true, + ResourceWait: summary.ResourceWait, + RunID: summary.ID, + Workspace: summary.Workspace, + PipelineID: summary.PipelineID, + Name: summary.Name, + Title: summary.Title, + Version: summary.Version, + Status: summary.Status, + Outcome: summary.Outcome, + CurrentStage: summary.CurrentStage, + CompletedSteps: summary.CompletedSteps, + CreatedAt: summary.CreatedAt, + FinishedAt: summary.FinishedAt, } if !sub.writeSnapshot(protocol.Message{Type: protocol.RunEvent, RunEvent: &frame}) { // The connection write failed; the subscription is closed and the diff --git a/internal/server/runner.go b/internal/server/runner.go index 11471617..efaa8787 100644 --- a/internal/server/runner.go +++ b/internal/server/runner.go @@ -78,12 +78,19 @@ func newRunner(ctx context.Context, s *pipeline.Store) (*runner, error) { if owner.Backend == "pi" { err = stage.InterruptPi(stop, owner.Reference) } else { - err = (stage.OpenCodeClient{Backend: owner.Backend, Directory: owner.Directory, Reference: owner.Reference}).Interrupt(stop, owner.ID) + err = (stage.OpenCodeClient{Backend: owner.Backend, Directory: owner.Directory, Reference: owner.Reference}).InterruptAndConfirm(stop, owner.ID) } cancel() if err != nil { + if resource, checkErr := s.HasResourceOwner(ctx, owner.RunID); checkErr == nil && resource { + log.Printf("resource owner run=%d remains quarantined: cleanup not confirmed", owner.RunID) + continue + } return nil, fmt.Errorf("cannot interrupt previously active agent %s: %w", owner.ID, err) } + if err := s.ConfirmRecoveredSession(ctx, owner); err != nil { + return nil, err + } } if err := s.RecoverRuns(ctx); err != nil { return nil, err @@ -213,6 +220,9 @@ func (r *runner) start(request protocol.Message, retryOf ...int64) (*protocol.Ru // workspace) must not block new work. for _, owner := range owners { if _, ok := r.active[owner.RunID]; !ok { + if resource, checkErr := r.store.HasResourceOwner(r.ctx, owner.RunID); checkErr == nil && resource && owner.Directory != workspace { + continue + } r.mu.Unlock() return nil, fmt.Errorf("an agent may still be active; restart recovery must interrupt it before new work") } @@ -775,13 +785,14 @@ func (r *runner) finishRunState(id int64, name, workspace, status, outcome, diag // Push the terminal delta before the attention notification, so a slow ntfy // never delays the live update the app folds. r.emitRun(protocol.RunEventFrame{ - RunID: id, - Workspace: workspace, - Status: status, - Outcome: outcome, - CurrentStage: currentStage, - CompletedSteps: completed, - FinishedAt: time.Now().UTC().Format(time.RFC3339), + RunID: id, + Workspace: workspace, + Status: status, + ResourceWaitSet: true, + Outcome: outcome, + CurrentStage: currentStage, + CompletedSteps: completed, + FinishedAt: time.Now().UTC().Format(time.RFC3339), }, true) r.notifyRun(id, name, status, sequence, currentStage, diagnostic) // The run is terminal: nothing more can notify for it. @@ -849,6 +860,21 @@ func (r *runner) finishStage(id int64, name, workspace string, selected protocol } func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, name, workspace string, request protocol.Message, state *runState) { + var held *pipeline.ResourceLease + resourceQuiet := true + release := func() error { + if held != nil { + err := r.releaseResource(*held, resourceQuiet) + held = nil + return err + } + return nil + } + defer func() { + if err := release(); err != nil { + log.Printf("resource cleanup run=%d: %v", id, err) + } + }() finish := func(status, outcome, diagnostic string) { r.finishRunState(id, name, workspace, status, outcome, diagnostic, state.completed, state.nextSequence-1, state.key, state.input) } @@ -877,7 +903,6 @@ func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, na effectiveModels = resolved } for sequence := state.nextSequence; sequence <= 30; sequence++ { - state.nextSequence = sequence + 1 if ctx.Err() != nil { finish("canceled", "", ctx.Err().Error()) return @@ -909,24 +934,42 @@ func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, na } } isHuman := stage.EffectiveType(def.Type) == stage.HumanType - state.visits[key]++ maxVisits := 5 if selected.Limits != nil { maxVisits = selected.Limits.MaxVisits } - // A human stage pauses for a person; neither the global visit budget nor - // the per-stage max_visits may kill it. The shared 30-step run budget - // still bounds every run. + // Validate the next visit without consuming it. An already-exhausted + // stage must not queue for a resource it can never execute. if !isHuman { - if state.visits[key] > 5 { + if state.visits[key]+1 > 5 { finish("failed", "error", "stage visit budget exceeded") return } - if state.visits[key] > maxVisits { + if state.visits[key]+1 > maxVisits { finish("failed", "error", fmt.Sprintf("stage %s visit limit reached (%d)", key, maxVisits)) return } } + _, declaresSkip := def.Edges["skipped"] + skip := !isHuman && declaresSkip && !state.uiSurface + if selected.Resources != nil && !skip { + if isHuman || !pipeline.ValidResourceKey(selected.Resources.Exclusive) { + finish("failed", "error", "invalid persisted resource declaration") + return + } + lease, err := r.acquireResource(ctx, pipeline.ResourceVisit{Resource: selected.Resources.Exclusive, RunID: id, Stage: key, Sequence: sequence}, workspace, state.completed) + if err != nil { + if ctx.Err() != nil { + finish("canceled", "", err.Error()) + } else { + finish("failed", "error", err.Error()) + } + return + } + held, resourceQuiet = &lease, true + } + state.nextSequence = sequence + 1 + state.visits[key]++ // Publish the run's server-captured changed paths into the input memory // value before resolving bindings, so a stage selecting // input.server_changed_files always resolves (an empty list when nothing @@ -948,12 +991,13 @@ func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, na } // queued -> running: report the new current stage and the step start. r.emitRun(protocol.RunEventFrame{ - RunID: id, - Workspace: workspace, - Status: protocol.RunStatusRunning, - CurrentStage: key, - CompletedSteps: state.completed, - Step: &protocol.RunStepDelta{Sequence: sequence, Stage: key, Status: protocol.StepStatusRunning}, + RunID: id, + Workspace: workspace, + Status: protocol.RunStatusRunning, + CurrentStage: key, + ResourceWaitSet: true, + CompletedSteps: state.completed, + Step: &protocol.RunStepDelta{Sequence: sequence, Stage: key, Status: protocol.StepStatusRunning}, }, false) if isHuman { // A human step is O(1): one BeginStep plus one MarkWaiting, no @@ -987,7 +1031,7 @@ func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, na // the model call and its capture work, record the step as completed and // route along the declared edge. Only a declared route is honoured, and no // other stage or outcome routing changes. - if _, declaresSkip := def.Edges["skipped"]; declaresSkip && !state.uiSurface { + if skip { reason := skipReasonLine(state) step := protocol.RunStep{ Sequence: sequence, @@ -1068,6 +1112,9 @@ func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, na snapshotCancel() var stageSession string panicked := false + if held != nil { + resourceQuiet = false + } func() { defer func() { if recover() != nil { @@ -1075,6 +1122,10 @@ func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, na err = fmt.Errorf("stage execution panicked") } }() + if ctx.Err() != nil { + err = ctx.Err() + return + } if r.execute != nil { timeout, _ := stage.StageTimeout(def) executeCtx, cancel := context.WithTimeout(stageCtx, timeout) @@ -1119,6 +1170,12 @@ func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, na result, err = stage.DefaultRegistry().Run(ctx, def, input) } }() + if held != nil { + resourceQuiet = r.resourceQuiescent(id, err, panicked) + if confirmed, reported := result.Output[protocol.ResourceCleanupConfirmedKey]; reported && confirmed != true { + resourceQuiet = false + } + } // Capture after cleanup, even when the stage's context was canceled. snapshotCtx, snapshotCancel = context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second) after := filediff.Capture(snapshotCtx, workspace, before) @@ -1236,6 +1293,14 @@ func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, na return } } + if held != nil && !resourceQuiet { + finish("failed", "error", "resource cleanup/restoration not confirmed; ownership quarantined pending inspection") + return + } + if err := release(); err != nil { + finish("failed", "error", "resource release failed; retained ownership needs reconciliation: "+err.Error()) + return + } if ctx.Err() != nil { finish("canceled", "", ctx.Err().Error()) return diff --git a/internal/server/testdata/android-resource-draft.md b/internal/server/testdata/android-resource-draft.md new file mode 100644 index 00000000..e822ce78 --- /dev/null +++ b/internal/server/testdata/android-resource-draft.md @@ -0,0 +1,312 @@ +# stage implementer + +```limits +{"max_visits":5,"on":["blocked_retryable"]} +``` + + +```memory +{"task": "input.prompt", "code_findings": "reviewer.findings?", "ui_findings": "ui_reviewer.findings?"} +``` + +```output +{ task, acceptance_checks, changed_files, implementation_summary, new_helpers } +``` + +```prompt +Implement input.prompt in the run directory using the repository's actual languages and +file types; create files as needed. + +Read android/AGENTS.md first. It is the map of this project - how to build, test, run and capture here, where +the pieces live, and the conventions that apply. Do not re-derive what it tells you and do not go +looking for what it already names; if it is wrong or out of date, say so in your output. + +Read the surrounding code before editing so you extend the existing helpers and patterns rather than +duplicating them. Keep each change as small as the task allows and follow the repository's own +conventions and error handling. + +This is the Android app. Build and test it with the project's own Gradle invocation as the map +gives it, and treat the map's geometry tokens as the only source of dimensions - a UI change that +adds a raw value fails the repository's own check. Perform code/build/unit tests only; +no adb, install, capture, device-setting changes or restoring main's build. The UI reviewer +alone verifies rendered surfaces under its declared runner resource lease. + +Record the acceptance checks in acceptance_checks, the actual changes in changed_files, and any new +shared helper in new_helpers. Verify with the checks this repository uses - as the map gives them - +and report the exact commands and observed output. Address any concrete findings the runner supplies +and note the actual fixes. Return blocked if the required scope or tools are unavailable. Do not +commit or push; the committer stage lands the work. +``` + +```tools +read +glob +grep +edit +shell +``` + +```configs +provider=openai +model=gpt-5.6-terra +retries=1 +timeout=30m +``` + + +```edges +success -> reviewer +blocked_retryable -> implementer +blocked -> #failure +error -> #failure +``` + + +# stage reviewer + +```limits +{"max_visits": 3, "on": ["rejected"]} +``` + +```memory +{"task": "input.prompt", "acceptance_checks": "implementer.acceptance_checks", "changed_files": "implementer.changed_files", "implementation": "implementer.implementation_summary?"} +``` + + +```output +{ summary, changed_files, verification_evidence, duplication_review, performance_review, findings } +``` + +```prompt +Verify with builds, unit tests and source review as android/AGENTS.md gives them. +Do not use adb, install, capture, change device settings or restore main's build. +Device verification belongs exclusively to the UI reviewer under its runner resource lease. +Enforce the map's geometry-token rule: a raw dimension outside the token file is a must-fix finding. + +Review the actual changes against the task in input.prompt and the acceptance +checks the implementer recorded. Run the repository's relevant verification checks +with the stage tools (for example build, lint, tests or type checks) and report the +exact commands and observed results in verification_evidence. + +Independently assess code quality: +- Duplication: search the run directory for repeated or near-identical functions, + blocks and copy-pasted logic introduced by the change; identify code that should be + shared and record concrete locations in duplication_review. +- Performance and resources: flag redundant computation, unnecessary dependencies, + and repeated I/O or database/filesystem work that wastes time, CPU or memory; + record it in performance_review. +- Workarounds: flag hacks, suppressed errors and shortcuts that trade reliability for + speed in findings. + +Keep verification proportionate to the change: exercise the files and screens the diff +actually touches, capture at most one screenshot per changed surface plus one regression +screen, and do not repeat sweeps or measurements that an earlier visit already recorded. + +Return rejected ONLY when at least one must-fix finding exists: a defect that blocks +acceptance and that you can locate concretely (file and line), with evidence and a +remedy. Do not return rejected for advisory-only, cosmetic or nit items. When there +is no must-fix finding but you have non-blocking notes (duplication, performance or +workaround observations), record them in findings and return advisory, which +continues the run without another implementer visit. Return success only when the +observed checks pass and you have nothing further to record. Do not edit files or +claim unexecuted checks. +``` + +```tools +read +glob +grep +shell +``` + +```configs +provider=openai +model=gpt-5.6-luna +retries=1 +timeout=45m +``` + + +```edges +success -> ui_reviewer +advisory -> ui_reviewer +rejected -> implementer +error -> #failure +``` + + +# stage ui_reviewer + +```resources +{"exclusive":"phone-47241FDAS00DLK"} +``` + +```limits +{"max_visits": 3, "on": ["rejected"]} +``` + +```memory +{"task": "input.prompt", "acceptance_checks": "implementer.acceptance_checks", "changed_files": "implementer.changed_files", "implementation": "implementer.implementation_summary?"} +``` + + +```output +{ summary, changed_files, ui_applicable, screenshots, ui_review, platform_conventions, findings } +``` + +```prompt +Decide whether this change touches a user-facing UI. If it does not (a backend, +library, CLI, config or data-only change with no rendered surface), set +ui_applicable=false, explain briefly in summary, and return skipped. Do not invent +screenshots or visual work. + +When the change does affect a user-facing UI, verify it visually: +Read android/AGENTS.md first. You alone perform phone work while the runner holds +this stage's declared resource lease. Use controlled fixtures for changed states, +including human-decision sheets, on a scratch server/database and throwaway session. +Never mutate an operator session or restart deployed services. Record original +settings and connection/workspace; use the configured authoritative endpoint, not a +hardcoded scratch default. Keep every shell/ADB operation foreground and awaited. +Restore original settings and main's build before successful return, under the same +lease. Report capture/restoration failures explicitly; do not approve incomplete +verification or advertise readiness when cleanup/restoration is uncertain. +Include resource_cleanup_confirmed in output: true only after confirmed restoration; +false on any uncertainty (a release veto that quarantines ownership even if tools are idle). +- Build and launch the application the way the repository actually does it, then + capture screenshots of every relevant screen, state and viewport (default, empty, + loading, error, long content, small and large sizes) with the target platform's + screenshot tooling through shell. Prefer a headless or scripted capture when the + project supports one. Save the images under the run directory and list the exact + capture commands and image paths in screenshots. +- Read the captured images and review them as a UI/UX designer: layout, spacing, + alignment, typographic hierarchy, color and contrast, iconography, interaction + and focus states, loading/empty/error states, accessibility, and consistency. +- Judge the UI against established industry patterns and, above all, the native + conventions of the platform the app targets (operating system, toolkit or web + standards). Record the platform you reviewed against in platform_conventions, + and any deviations and concrete fixes in findings. Infer the target platform from + the task and the repository's actual UI toolkit when it is unstated. +- Approve only when the screenshots show the task and the acceptance checks are met + and the UI is correct, consistent and native to the target platform; otherwise + return rejected with specific, actionable findings the implementer can fix. + +Keep the capture pass proportionate: capture each changed state once (for example one +collapsed and one expanded section, one run state, one stuck header) instead of every +screen at multiple viewports or pan directions, and reuse what this run already captured +rather than re-doing it. + +If the change is UI-facing but screenshots genuinely cannot be captured +(for example no runnable UI or no display), return blocked rather than approving. +Do not edit files, and do not approve a UI you did not actually see. + +Return rejected only for a must-fix visual defect with a concrete location, evidence +and remedy; record non-blocking polish, duplication or convention notes in findings +and return advisory, which continues the run without another implementer visit. +``` + +```tools +read +glob +grep +shell +``` + +```configs +provider=openai +model=gpt-5.6-luna +retries=1 +timeout=45m +``` + + +```edges +skipped -> committer +approved -> committer +advisory -> committer +rejected -> implementer +blocked -> #failure +error -> #failure +``` + +# stage committer + +```memory +{"task": "input.prompt", "changed_files": "implementer.changed_files", "review": "reviewer.summary?", "ui_review": "ui_reviewer.summary?"} +``` + +```output +{ commit, files, conflicts_resolved, summary } +``` + +```prompt +Commit the work this run produced, and resolve any conflict that is present. This stage +runs after the reviews have approved and is the only stage allowed to commit. + +Before committing, check that the change is reflected where it belongs: the packaged +templates, the packaged +authoring skill references. Update anything the change makes stale, and state in the +commit body what you checked, saying explicitly when none of them needs updating. + +Stage exactly the files this run changed - the paths in changed_files that still exist - +using the repository's own version control commands, and commit them as one commit whose +subject says what the change does and whose body summarises it. Do not stage unrelated +files, generated or ignored files, or anything another change owns. Do not invent work +that was not done; describe what is actually in the diff. + +If the workspace is in a conflicted state - unmerged paths, or a merge, rebase or +cherry-pick left unfinished - resolve each conflict: keep both intents where they are +compatible, and choose the result that satisfies the task where they are not, then +complete the operation so the tree is clean and the commit is finished. Never commit +conflict markers, and never discard the other side's work silently; record what you +resolved. + +Do not push, tag, revert, or rewrite unrelated history. If there is nothing to commit, +report that instead of creating an empty commit. Report the commit identifier, the files +committed, whatever you resolved, and anything you deliberately left uncommitted. + + +Finally, bring this branch up to date with main, in this same stage. After committing the work +above, merge main into this branch here, resolve any conflict here - keeping both intents where they +are compatible, and choosing the result that satisfies this task where they are not - run the +repository's checks again after resolving, and commit the completed merge with a message naming what +was combined. The point is that this branch can later be merged back without conflict: leave the +tree clean and the merge finished. Never leave conflict markers, never discard the other side's work +silently. If the merge genuinely cannot be completed, say so with the path and the reason instead of +leaving the merge half-finished. + +Then land it: this branch is the work and main is where it belongs, so merge this branch into main +as the last act of this stage. Find the repository's main working tree with `git worktree list` (the +entry checked out on main, not this one) and merge this branch into it there with `--no-ff`, with a +message saying what the change does. Two lanes can land at the same moment, so take a lock first - +`mkdir` a lock directory beside main's `git` directory in a bounded retry loop, and remove it when +done - and retry briefly if another lane holds it. Then verify with +`git merge-base --is-ancestor main` and report the commit main now has. If main cannot +be updated, say exactly why and leave this branch committed and clean rather than leaving main +half-merged. There is no separate merge pipeline and nothing to hand this to: if it cannot be landed +here, it is reported here. + + +Do not edit README or other shared documentation files: an item records what it changed and which documentation is now stale in its own commit body, and the shared documents are folded by a single writer. + +``` + +```tools +read +glob +grep +edit +shell +``` + +```configs +provider=openai +model=gpt-5.6-terra +retries=1 +timeout=30m +``` + +```edges +success -> #success +blocked -> #failure +error -> #failure +``` diff --git a/internal/server/testdata/android-v2-published.md b/internal/server/testdata/android-v2-published.md new file mode 100644 index 00000000..43bf14a9 --- /dev/null +++ b/internal/server/testdata/android-v2-published.md @@ -0,0 +1,296 @@ +# stage implementer + +```limits +{"max_visits":5,"on":["blocked_retryable"]} +``` + + +```memory +{"task": "input.prompt", "code_findings": "reviewer.findings?", "ui_findings": "ui_reviewer.findings?"} +``` + +```output +{ task, acceptance_checks, changed_files, implementation_summary, new_helpers } +``` + +```prompt +Implement input.prompt in the run directory using the repository's actual languages and +file types; create files as needed. + +Read android/AGENTS.md first. It is the map of this project - how to build, test, run and capture here, where +the pieces live, and the conventions that apply. Do not re-derive what it tells you and do not go +looking for what it already names; if it is wrong or out of date, say so in your output. + +Read the surrounding code before editing so you extend the existing helpers and patterns rather than +duplicating them. Keep each change as small as the task allows and follow the repository's own +conventions and error handling. + +This is the Android app. Build and test it with the project's own Gradle invocation as the map +gives it, and treat the map's geometry tokens as the only source of dimensions - a UI change that +adds a raw value fails the repository's own check. A change with a rendered surface is verified on +the device the map names (never an emulator), capturing what the map says to capture. + +Record the acceptance checks in acceptance_checks, the actual changes in changed_files, and any new +shared helper in new_helpers. Verify with the checks this repository uses - as the map gives them - +and report the exact commands and observed output. Address any concrete findings the runner supplies +and note the actual fixes. Return blocked if the required scope or tools are unavailable. Do not +commit or push; the committer stage lands the work. +``` + +```tools +read +glob +grep +edit +shell +``` + +```configs +provider=openai +model=gpt-6.1-sol-fast +retries=1 +timeout=30m +``` + + +```edges +success -> reviewer +blocked_retryable -> implementer +blocked -> #failure +error -> #failure +``` + + +# stage reviewer + +```limits +{"max_visits": 3, "on": ["rejected"]} +``` + +```memory +{"task": "input.prompt", "acceptance_checks": "implementer.acceptance_checks", "changed_files": "implementer.changed_files", "implementation": "implementer.implementation_summary?"} +``` + + +```output +{ summary, changed_files, verification_evidence, duplication_review, performance_review, findings } +``` + +```prompt +Verify with the build and the unit tests as android/AGENTS.md gives them, and for a +change with a rendered surface check the captures against the running app on the device the map +names. Enforce the map's geometry-token rule: a raw dimension outside the token file is a must-fix +finding. + +Review the actual changes against the task in input.prompt and the acceptance +checks the implementer recorded. Run the repository's relevant verification checks +with the stage tools (for example build, lint, tests or type checks) and report the +exact commands and observed results in verification_evidence. + +Independently assess code quality: +- Duplication: search the run directory for repeated or near-identical functions, + blocks and copy-pasted logic introduced by the change; identify code that should be + shared and record concrete locations in duplication_review. +- Performance and resources: flag redundant computation, unnecessary dependencies, + and repeated I/O or database/filesystem work that wastes time, CPU or memory; + record it in performance_review. +- Workarounds: flag hacks, suppressed errors and shortcuts that trade reliability for + speed in findings. + +Keep verification proportionate to the change: exercise the files and screens the diff +actually touches, capture at most one screenshot per changed surface plus one regression +screen, and do not repeat sweeps or measurements that an earlier visit already recorded. + +Return rejected ONLY when at least one must-fix finding exists: a defect that blocks +acceptance and that you can locate concretely (file and line), with evidence and a +remedy. Do not return rejected for advisory-only, cosmetic or nit items. When there +is no must-fix finding but you have non-blocking notes (duplication, performance or +workaround observations), record them in findings and return advisory, which +continues the run without another implementer visit. Return success only when the +observed checks pass and you have nothing further to record. Do not edit files or +claim unexecuted checks. +``` + +```tools +read +glob +grep +shell +``` + +```configs +provider=openai +model=gpt-6.1-sol-fast +retries=1 +timeout=45m +``` + + +```edges +success -> ui_reviewer +advisory -> ui_reviewer +rejected -> implementer +error -> #failure +``` + + +# stage ui_reviewer + +```limits +{"max_visits": 3, "on": ["rejected"]} +``` + +```memory +{"task": "input.prompt", "acceptance_checks": "implementer.acceptance_checks", "changed_files": "implementer.changed_files", "implementation": "implementer.implementation_summary?"} +``` + + +```output +{ summary, changed_files, ui_applicable, screenshots, ui_review, platform_conventions, findings } +``` + +```prompt +Decide whether this change touches a user-facing UI. If it does not (a backend, +library, CLI, config or data-only change with no rendered surface), set +ui_applicable=false, explain briefly in summary, and return skipped. Do not invent +screenshots or visual work. + +When the change does affect a user-facing UI, verify it visually: +- Build and launch the application the way the repository actually does it, then + capture screenshots of every relevant screen, state and viewport (default, empty, + loading, error, long content, small and large sizes) with the target platform's + screenshot tooling through shell. Prefer a headless or scripted capture when the + project supports one. Save the images under the run directory and list the exact + capture commands and image paths in screenshots. +- Read the captured images and review them as a UI/UX designer: layout, spacing, + alignment, typographic hierarchy, color and contrast, iconography, interaction + and focus states, loading/empty/error states, accessibility, and consistency. +- Judge the UI against established industry patterns and, above all, the native + conventions of the platform the app targets (operating system, toolkit or web + standards). Record the platform you reviewed against in platform_conventions, + and any deviations and concrete fixes in findings. Infer the target platform from + the task and the repository's actual UI toolkit when it is unstated. +- Approve only when the screenshots show the task and the acceptance checks are met + and the UI is correct, consistent and native to the target platform; otherwise + return rejected with specific, actionable findings the implementer can fix. + +Keep the capture pass proportionate: capture each changed state once (for example one +collapsed and one expanded section, one run state, one stuck header) instead of every +screen at multiple viewports or pan directions, and reuse what this run already captured +rather than re-doing it. + +If the change is UI-facing but screenshots genuinely cannot be captured +(for example no runnable UI or no display), return blocked rather than approving. +Do not edit files, and do not approve a UI you did not actually see. + +Return rejected only for a must-fix visual defect with a concrete location, evidence +and remedy; record non-blocking polish, duplication or convention notes in findings +and return advisory, which continues the run without another implementer visit. +``` + +```tools +read +glob +grep +shell +``` + +```configs +provider=openai +model=gpt-6.1-sol-fast +retries=1 +timeout=45m +``` + + +```edges +skipped -> committer +approved -> committer +advisory -> committer +rejected -> implementer +blocked -> #failure +error -> #failure +``` + +# stage committer + +```memory +{"task": "input.prompt", "changed_files": "implementer.changed_files", "review": "reviewer.summary?", "ui_review": "ui_reviewer.summary?"} +``` + +```output +{ commit, files, conflicts_resolved, summary } +``` + +```prompt +Commit the work this run produced, and resolve any conflict that is present. This stage +runs after the reviews have approved and is the only stage allowed to commit. + +Before committing, check that the change is reflected where it belongs: the packaged +templates, the packaged +authoring skill references. Update anything the change makes stale, and state in the +commit body what you checked, saying explicitly when none of them needs updating. + +Stage exactly the files this run changed - the paths in changed_files that still exist - +using the repository's own version control commands, and commit them as one commit whose +subject says what the change does and whose body summarises it. Do not stage unrelated +files, generated or ignored files, or anything another change owns. Do not invent work +that was not done; describe what is actually in the diff. + +If the workspace is in a conflicted state - unmerged paths, or a merge, rebase or +cherry-pick left unfinished - resolve each conflict: keep both intents where they are +compatible, and choose the result that satisfies the task where they are not, then +complete the operation so the tree is clean and the commit is finished. Never commit +conflict markers, and never discard the other side's work silently; record what you +resolved. + +Do not push, tag, revert, or rewrite unrelated history. If there is nothing to commit, +report that instead of creating an empty commit. Report the commit identifier, the files +committed, whatever you resolved, and anything you deliberately left uncommitted. + + +Finally, bring this branch up to date with main, in this same stage. After committing the work +above, merge main into this branch here, resolve any conflict here - keeping both intents where they +are compatible, and choosing the result that satisfies this task where they are not - run the +repository's checks again after resolving, and commit the completed merge with a message naming what +was combined. The point is that this branch can later be merged back without conflict: leave the +tree clean and the merge finished. Never leave conflict markers, never discard the other side's work +silently. If the merge genuinely cannot be completed, say so with the path and the reason instead of +leaving the merge half-finished. + +Then land it: this branch is the work and main is where it belongs, so merge this branch into main +as the last act of this stage. Find the repository's main working tree with `git worktree list` (the +entry checked out on main, not this one) and merge this branch into it there with `--no-ff`, with a +message saying what the change does. Two lanes can land at the same moment, so take a lock first - +`mkdir` a lock directory beside main's `git` directory in a bounded retry loop, and remove it when +done - and retry briefly if another lane holds it. Then verify with +`git merge-base --is-ancestor main` and report the commit main now has. If main cannot +be updated, say exactly why and leave this branch committed and clean rather than leaving main +half-merged. There is no separate merge pipeline and nothing to hand this to: if it cannot be landed +here, it is reported here. + + +Do not edit README or other shared documentation files: an item records what it changed and which documentation is now stale in its own commit body, and the shared documents are folded by a single writer. + +``` + +```tools +read +glob +grep +edit +shell +``` + +```configs +provider=openai +model=gpt-6.1-sol-fast +retries=1 +timeout=30m +``` + +```edges +success -> #success +blocked -> #failure +error -> #failure +``` diff --git a/internal/server/testdata/android-v3-published.md b/internal/server/testdata/android-v3-published.md new file mode 100644 index 00000000..344e537b --- /dev/null +++ b/internal/server/testdata/android-v3-published.md @@ -0,0 +1,296 @@ +# stage implementer + +```limits +{"max_visits":5,"on":["blocked_retryable"]} +``` + + +```memory +{"task": "input.prompt", "code_findings": "reviewer.findings?", "ui_findings": "ui_reviewer.findings?"} +``` + +```output +{ task, acceptance_checks, changed_files, implementation_summary, new_helpers } +``` + +```prompt +Implement input.prompt in the run directory using the repository's actual languages and +file types; create files as needed. + +Read android/AGENTS.md first. It is the map of this project - how to build, test, run and capture here, where +the pieces live, and the conventions that apply. Do not re-derive what it tells you and do not go +looking for what it already names; if it is wrong or out of date, say so in your output. + +Read the surrounding code before editing so you extend the existing helpers and patterns rather than +duplicating them. Keep each change as small as the task allows and follow the repository's own +conventions and error handling. + +This is the Android app. Build and test it with the project's own Gradle invocation as the map +gives it, and treat the map's geometry tokens as the only source of dimensions - a UI change that +adds a raw value fails the repository's own check. A change with a rendered surface is verified on +the device the map names (never an emulator), capturing what the map says to capture. + +Record the acceptance checks in acceptance_checks, the actual changes in changed_files, and any new +shared helper in new_helpers. Verify with the checks this repository uses - as the map gives them - +and report the exact commands and observed output. Address any concrete findings the runner supplies +and note the actual fixes. Return blocked if the required scope or tools are unavailable. Do not +commit or push; the committer stage lands the work. +``` + +```tools +read +glob +grep +edit +shell +``` + +```configs +provider=openai +model=gpt-5.6-terra +retries=1 +timeout=30m +``` + + +```edges +success -> reviewer +blocked_retryable -> implementer +blocked -> #failure +error -> #failure +``` + + +# stage reviewer + +```limits +{"max_visits": 3, "on": ["rejected"]} +``` + +```memory +{"task": "input.prompt", "acceptance_checks": "implementer.acceptance_checks", "changed_files": "implementer.changed_files", "implementation": "implementer.implementation_summary?"} +``` + + +```output +{ summary, changed_files, verification_evidence, duplication_review, performance_review, findings } +``` + +```prompt +Verify with the build and the unit tests as android/AGENTS.md gives them, and for a +change with a rendered surface check the captures against the running app on the device the map +names. Enforce the map's geometry-token rule: a raw dimension outside the token file is a must-fix +finding. + +Review the actual changes against the task in input.prompt and the acceptance +checks the implementer recorded. Run the repository's relevant verification checks +with the stage tools (for example build, lint, tests or type checks) and report the +exact commands and observed results in verification_evidence. + +Independently assess code quality: +- Duplication: search the run directory for repeated or near-identical functions, + blocks and copy-pasted logic introduced by the change; identify code that should be + shared and record concrete locations in duplication_review. +- Performance and resources: flag redundant computation, unnecessary dependencies, + and repeated I/O or database/filesystem work that wastes time, CPU or memory; + record it in performance_review. +- Workarounds: flag hacks, suppressed errors and shortcuts that trade reliability for + speed in findings. + +Keep verification proportionate to the change: exercise the files and screens the diff +actually touches, capture at most one screenshot per changed surface plus one regression +screen, and do not repeat sweeps or measurements that an earlier visit already recorded. + +Return rejected ONLY when at least one must-fix finding exists: a defect that blocks +acceptance and that you can locate concretely (file and line), with evidence and a +remedy. Do not return rejected for advisory-only, cosmetic or nit items. When there +is no must-fix finding but you have non-blocking notes (duplication, performance or +workaround observations), record them in findings and return advisory, which +continues the run without another implementer visit. Return success only when the +observed checks pass and you have nothing further to record. Do not edit files or +claim unexecuted checks. +``` + +```tools +read +glob +grep +shell +``` + +```configs +provider=openai +model=gpt-5.6-luna +retries=1 +timeout=45m +``` + + +```edges +success -> ui_reviewer +advisory -> ui_reviewer +rejected -> implementer +error -> #failure +``` + + +# stage ui_reviewer + +```limits +{"max_visits": 3, "on": ["rejected"]} +``` + +```memory +{"task": "input.prompt", "acceptance_checks": "implementer.acceptance_checks", "changed_files": "implementer.changed_files", "implementation": "implementer.implementation_summary?"} +``` + + +```output +{ summary, changed_files, ui_applicable, screenshots, ui_review, platform_conventions, findings } +``` + +```prompt +Decide whether this change touches a user-facing UI. If it does not (a backend, +library, CLI, config or data-only change with no rendered surface), set +ui_applicable=false, explain briefly in summary, and return skipped. Do not invent +screenshots or visual work. + +When the change does affect a user-facing UI, verify it visually: +- Build and launch the application the way the repository actually does it, then + capture screenshots of every relevant screen, state and viewport (default, empty, + loading, error, long content, small and large sizes) with the target platform's + screenshot tooling through shell. Prefer a headless or scripted capture when the + project supports one. Save the images under the run directory and list the exact + capture commands and image paths in screenshots. +- Read the captured images and review them as a UI/UX designer: layout, spacing, + alignment, typographic hierarchy, color and contrast, iconography, interaction + and focus states, loading/empty/error states, accessibility, and consistency. +- Judge the UI against established industry patterns and, above all, the native + conventions of the platform the app targets (operating system, toolkit or web + standards). Record the platform you reviewed against in platform_conventions, + and any deviations and concrete fixes in findings. Infer the target platform from + the task and the repository's actual UI toolkit when it is unstated. +- Approve only when the screenshots show the task and the acceptance checks are met + and the UI is correct, consistent and native to the target platform; otherwise + return rejected with specific, actionable findings the implementer can fix. + +Keep the capture pass proportionate: capture each changed state once (for example one +collapsed and one expanded section, one run state, one stuck header) instead of every +screen at multiple viewports or pan directions, and reuse what this run already captured +rather than re-doing it. + +If the change is UI-facing but screenshots genuinely cannot be captured +(for example no runnable UI or no display), return blocked rather than approving. +Do not edit files, and do not approve a UI you did not actually see. + +Return rejected only for a must-fix visual defect with a concrete location, evidence +and remedy; record non-blocking polish, duplication or convention notes in findings +and return advisory, which continues the run without another implementer visit. +``` + +```tools +read +glob +grep +shell +``` + +```configs +provider=openai +model=gpt-5.6-luna +retries=1 +timeout=45m +``` + + +```edges +skipped -> committer +approved -> committer +advisory -> committer +rejected -> implementer +blocked -> #failure +error -> #failure +``` + +# stage committer + +```memory +{"task": "input.prompt", "changed_files": "implementer.changed_files", "review": "reviewer.summary?", "ui_review": "ui_reviewer.summary?"} +``` + +```output +{ commit, files, conflicts_resolved, summary } +``` + +```prompt +Commit the work this run produced, and resolve any conflict that is present. This stage +runs after the reviews have approved and is the only stage allowed to commit. + +Before committing, check that the change is reflected where it belongs: the packaged +templates, the packaged +authoring skill references. Update anything the change makes stale, and state in the +commit body what you checked, saying explicitly when none of them needs updating. + +Stage exactly the files this run changed - the paths in changed_files that still exist - +using the repository's own version control commands, and commit them as one commit whose +subject says what the change does and whose body summarises it. Do not stage unrelated +files, generated or ignored files, or anything another change owns. Do not invent work +that was not done; describe what is actually in the diff. + +If the workspace is in a conflicted state - unmerged paths, or a merge, rebase or +cherry-pick left unfinished - resolve each conflict: keep both intents where they are +compatible, and choose the result that satisfies the task where they are not, then +complete the operation so the tree is clean and the commit is finished. Never commit +conflict markers, and never discard the other side's work silently; record what you +resolved. + +Do not push, tag, revert, or rewrite unrelated history. If there is nothing to commit, +report that instead of creating an empty commit. Report the commit identifier, the files +committed, whatever you resolved, and anything you deliberately left uncommitted. + + +Finally, bring this branch up to date with main, in this same stage. After committing the work +above, merge main into this branch here, resolve any conflict here - keeping both intents where they +are compatible, and choosing the result that satisfies this task where they are not - run the +repository's checks again after resolving, and commit the completed merge with a message naming what +was combined. The point is that this branch can later be merged back without conflict: leave the +tree clean and the merge finished. Never leave conflict markers, never discard the other side's work +silently. If the merge genuinely cannot be completed, say so with the path and the reason instead of +leaving the merge half-finished. + +Then land it: this branch is the work and main is where it belongs, so merge this branch into main +as the last act of this stage. Find the repository's main working tree with `git worktree list` (the +entry checked out on main, not this one) and merge this branch into it there with `--no-ff`, with a +message saying what the change does. Two lanes can land at the same moment, so take a lock first - +`mkdir` a lock directory beside main's `git` directory in a bounded retry loop, and remove it when +done - and retry briefly if another lane holds it. Then verify with +`git merge-base --is-ancestor main` and report the commit main now has. If main cannot +be updated, say exactly why and leave this branch committed and clean rather than leaving main +half-merged. There is no separate merge pipeline and nothing to hand this to: if it cannot be landed +here, it is reported here. + + +Do not edit README or other shared documentation files: an item records what it changed and which documentation is now stale in its own commit body, and the shared documents are folded by a single writer. + +``` + +```tools +read +glob +grep +edit +shell +``` + +```configs +provider=openai +model=gpt-5.6-terra +retries=1 +timeout=30m +``` + +```edges +success -> #success +blocked -> #failure +error -> #failure +``` diff --git a/internal/stage/agent.go b/internal/stage/agent.go index d3109d20..57b23b12 100644 --- a/internal/stage/agent.go +++ b/internal/stage/agent.go @@ -27,6 +27,14 @@ type AgentClient struct { } func (c OpenCodeClient) Interrupt(ctx context.Context, id string) error { + return c.interrupt(ctx, id, false) +} + +func (c OpenCodeClient) InterruptAndConfirm(ctx context.Context, id string) error { + return c.interrupt(ctx, id, true) +} + +func (c OpenCodeClient) interrupt(ctx context.Context, id string, confirm bool) error { if !sessionIDPattern.MatchString(id) { return fmt.Errorf("invalid session ID") } @@ -46,7 +54,20 @@ func (c OpenCodeClient) Interrupt(ctx context.Context, id string) error { c.Backend = connection.Backend } _, err := c.call(ctx, "post", "/api/session/"+id+"/interrupt", map[string]any{}) - return err + if err != nil || !confirm { + return err + } + // An interrupt acknowledgement is not a cleanup receipt. Keep ownership + // until the harness positively reports settlement, bounded by the caller. + if c.Backend == "v1" { + _, err := c.call(ctx, "post", "/api/experimental/session/"+id+"/wait", nil) + return err + } + // The failed stage wait already records deadline evidence. Cleanup settlement + // must still be confirmed, but must not add a second duplicate operation entry. + confirmed := c + confirmed.Operations = nil + return confirmed.waitForSessionIdle(ctx, id) } func (c *AgentClient) Generate(ctx context.Context, config ModelConfig, prompt string) (text string, err error) { @@ -100,7 +121,7 @@ func (c *AgentClient) Generate(ctx context.Context, config ModelConfig, prompt s defer cancel() if err != nil { c.captureTimedOutEvidence(ctx, err, config, watch) - if stopErr := c.Interrupt(cleanup, c.SessionID); stopErr != nil { + if stopErr := c.InterruptAndConfirm(cleanup, c.SessionID); stopErr != nil { err = fmt.Errorf("agent interrupt failed; retained session %s needs inspection: %w", c.SessionID, errors.Join(err, stopErr, ErrStageCleanup)) return } diff --git a/internal/stage/agent_test.go b/internal/stage/agent_test.go index 2a8bcb58..c18d0721 100644 --- a/internal/stage/agent_test.go +++ b/internal/stage/agent_test.go @@ -52,8 +52,8 @@ func TestAgentOwnershipPermissionsAndInterrupt(t *testing.T) { t.Fatal("unexpected call", args) return nil, nil } - if _, err := client.Generate(context.Background(), ModelConfig{}, "This call has no tools."); err == nil || !interrupted || owned { - t.Fatal("failed generation did not stop and release ownership") + if _, err := client.Generate(context.Background(), ModelConfig{}, "This call has no tools."); !errors.Is(err, ErrStageCleanup) || !interrupted || !owned { + t.Fatal("interrupt acknowledgement without idle confirmation must retain ownership") } } diff --git a/internal/stage/budget_test.go b/internal/stage/budget_test.go index 47b0ee08..21426679 100644 --- a/internal/stage/budget_test.go +++ b/internal/stage/budget_test.go @@ -82,6 +82,9 @@ func TestBudgetsAccumulateAcrossRetriesAndBlockFurtherGeneration(t *testing.T) { var sessions, generations atomic.Int64 client := &OpenCodeClient{command: func(ctx context.Context, _ string, args ...string) ([]byte, error) { path := args[2] + if path == "/api/session/active" { + return []byte(`{"data":{}}`), nil + } if path == "/api/session" { return []byte(fmt.Sprintf(`{"data":{"id":"ses_attempt_%d"}}`, sessions.Add(1))), nil } @@ -147,6 +150,9 @@ func TestLiveAgentBudgetInterruptsAndDoesNotReplay(t *testing.T) { return []byte(`{"data":{"id":"ses_agent_budget"}}`), nil } if path == "/api/session/active" { + if interrupts.Load() > 0 { + return []byte(`{"data":{}}`), nil + } <-ctx.Done() return nil, ctx.Err() } diff --git a/internal/stage/opencode.go b/internal/stage/opencode.go index bdf28bfd..d58b0bf1 100644 --- a/internal/stage/opencode.go +++ b/internal/stage/opencode.go @@ -321,7 +321,7 @@ func (c *OpenCodeClient) Generate(ctx context.Context, config ModelConfig, promp cleanup, cancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second) defer cancel() if err != nil { - if stopErr := c.Interrupt(cleanup, session.Data.ID); stopErr != nil { + if stopErr := c.InterruptAndConfirm(cleanup, session.Data.ID); stopErr != nil { err = fmt.Errorf("retained OpenCode session %s; interrupt failed: %w", session.Data.ID, errors.Join(err, stopErr, ErrStageCleanup)) return } diff --git a/internal/stage/pi.go b/internal/stage/pi.go index 3e0b1bf1..e4849f79 100644 --- a/internal/stage/pi.go +++ b/internal/stage/pi.go @@ -238,6 +238,10 @@ func (c *PiClient) Generate(ctx context.Context, config ModelConfig, prompt stri } close(done) close(heartbeatDone) + // agent_end/abort acknowledgement and the parent's exit alone do not + // prove its shell descendants exited. A surviving process group retains + // ownership rather than handing the resource to another stage. + quiet = quiet && errors.Is(syscall.Kill(-c.cmd.Process.Pid, 0), syscall.ESRCH) if quiet { _ = os.WriteFile(filepath.Join(c.Reference, "quiescent"), []byte("quiescent"), 0600) } @@ -532,9 +536,16 @@ func InterruptPi(ctx context.Context, reference string) error { if !filepath.IsAbs(reference) { return fmt.Errorf("invalid retained Pi reference") } - if _, err := os.Stat(filepath.Join(reference, "owner.json")); err != nil { + data, err := os.ReadFile(filepath.Join(reference, "owner.json")) + if err != nil { return errors.Join(err, ErrStageCleanup) } + var owner struct { + PID int `json:"pid"` + } + if json.Unmarshal(data, &owner) != nil || owner.PID <= 1 { + return errors.Join(fmt.Errorf("invalid retained Pi owner"), ErrStageCleanup) + } old := time.Unix(0, 0) if err := os.Chtimes(filepath.Join(reference, "lease"), old, old); err != nil { return err @@ -542,7 +553,7 @@ func InterruptPi(ctx context.Context, reference string) error { ticker := time.NewTicker(100 * time.Millisecond) defer ticker.Stop() for { - if data, err := os.ReadFile(filepath.Join(reference, "quiescent")); err == nil && string(data) == "quiescent" { + if data, err := os.ReadFile(filepath.Join(reference, "quiescent")); err == nil && string(data) == "quiescent" && errors.Is(syscall.Kill(owner.PID, 0), syscall.ESRCH) && errors.Is(syscall.Kill(-owner.PID, 0), syscall.ESRCH) { return nil } select { diff --git a/internal/stage/resource_cleanup_test.go b/internal/stage/resource_cleanup_test.go new file mode 100644 index 00000000..c57d4080 --- /dev/null +++ b/internal/stage/resource_cleanup_test.go @@ -0,0 +1,44 @@ +package stage + +import ( + "context" + "encoding/json" + "errors" + "os" + "os/exec" + "path/filepath" + "syscall" + "testing" + "time" +) + +func TestResourcePiReceiptRequiresProcessExit(t *testing.T) { + // A test-owned process only: no Pi provider, shell/ADB command or phone. + cmd := exec.Command("sleep", "5") + cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true} + if err := cmd.Start(); err != nil { + t.Fatal(err) + } + defer func() { cmd.Process.Kill(); cmd.Wait() }() + dir := t.TempDir() + owner, _ := json.Marshal(map[string]any{"pid": cmd.Process.Pid}) + for name, data := range map[string][]byte{"owner.json": owner, "lease": []byte("lease"), "quiescent": []byte("quiescent")} { + if err := os.WriteFile(filepath.Join(dir, name), data, 0600); err != nil { + t.Fatal(err) + } + } + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Millisecond) + defer cancel() + if err := InterruptPi(ctx, dir); !errors.Is(err, ErrStageCleanup) { + t.Fatal("receipt authorized handoff while process lived", err) + } + if err := cmd.Process.Kill(); err != nil { + t.Fatal(err) + } + cmd.Wait() + ctx2, cancel2 := context.WithTimeout(context.Background(), time.Second) + defer cancel2() + if err := InterruptPi(ctx2, dir); err != nil { + t.Fatal("confirmed process exit not accepted", err) + } +} diff --git a/internal/tui/resources_test.go b/internal/tui/resources_test.go new file mode 100644 index 00000000..a2e34475 --- /dev/null +++ b/internal/tui/resources_test.go @@ -0,0 +1,43 @@ +package tui + +import ( + "fmt" + "strings" + "testing" + + "git.a-wan.net/alan/pipelines/internal/protocol" + "github.com/charmbracelet/lipgloss" + "github.com/charmbracelet/x/ansi" + "github.com/muesli/termenv" +) + +func TestResourceWaitFrames(t *testing.T) { + previous := lipgloss.ColorProfile() + defer lipgloss.SetColorProfile(previous) + for _, width := range []int{60, 120} { + for _, plain := range []bool{false, true} { + t.Run(fmt.Sprintf("width-%d-plain-%t", width, plain), func(t *testing.T) { + if plain { + lipgloss.SetColorProfile(termenv.Ascii) + } else { + lipgloss.SetColorProfile(termenv.TrueColor) + } + m := decisionModel() + m.width = width + m.runDetail.Status = protocol.RunStatusQueued + m.runDetail.Steps = nil + m.runDetail.ResourceWait = &protocol.ResourceWait{Resource: "phone-X", Stage: "review", Sequence: 1, QueuedSince: "2026-10-18T10:00:00Z"} + frame := ansi.Strip(m.runsDetailsView(24)) + if !strings.Contains(frame, "Waiting for resource phone-X") || strings.Contains(frame, "c decide") || strings.Contains(frame, "Waiting for a person") { + t.Fatal("wrong resource wait frame:\n", frame) + } + for _, line := range strings.Split(frame, "\n") { + if ansi.StringWidth(line) > width { + t.Fatalf("overflow: %q", line) + } + } + t.Log("Rendered resource wait (no phone):\n" + frame) + }) + } + } +} diff --git a/internal/tui/results.go b/internal/tui/results.go index a4c1b610..3ef6840a 100644 --- a/internal/tui/results.go +++ b/internal/tui/results.go @@ -11,6 +11,9 @@ import ( ) func resultLabel(r protocol.RunSummary) string { + if r.ResourceWait != nil { + return r.StatusLabel() + " · stage " + r.ResourceWait.Stage + } preview := ansi.Strip(r.ResultPreview) if r.Status == "succeeded" { if preview != "" { @@ -55,7 +58,7 @@ func (m Model) pipelineResultLines() []string { if r == nil { return []string{"Results · This version has not run yet. No accomplishments recorded."} } - lines := []string{fmt.Sprintf("Results · run #%d · %s · %s · %d completed visits · %s", r.ID, r.Label(), ansi.Strip(m.progress(r.Status)), r.CompletedSteps, runDuration(r.CreatedAt, r.FinishedAt)), resultLabel(*r)} + lines := []string{fmt.Sprintf("Results · run #%d · %s · %s · %d completed visits · %s", r.ID, r.Label(), ansi.Strip(m.runSummaryStatus(*r)), r.CompletedSteps, runDuration(r.CreatedAt, r.FinishedAt)), resultLabel(*r)} if r.Status != "succeeded" && r.ResultPreview != "" && r.Diagnostic != "" { lines = append(lines, "Partial output: "+ansi.Strip(r.ResultPreview)) } diff --git a/internal/tui/run_detail.go b/internal/tui/run_detail.go index 79154144..87c4c04a 100644 --- a/internal/tui/run_detail.go +++ b/internal/tui/run_detail.go @@ -275,6 +275,13 @@ func runStatus(status string) string { return lipgloss.NewStyle().Foreground(color).Render(status) } +func (m Model) runSummaryStatus(run protocol.RunSummary) string { + if run.ResourceWait != nil { + return run.StatusLabel() + } + return m.progress(run.Status) +} + func runDuration(start, end string) string { first, err := time.Parse(time.RFC3339, start) if err != nil { @@ -687,7 +694,7 @@ func (m Model) runsDetailsView(height int) string { start, end := selectionWindow(len(runs), m.runIndex, max(1, (height-4-len(rows))/2)) for i := start; i < end; i++ { r := runs[i] - text := fmt.Sprintf("#%d %s · v%d · %s · %s", r.ID, r.Label(), r.Version, m.progress(r.Status), dateLabel(r.FinishedAt)) + text := fmt.Sprintf("#%d %s · v%d · %s · %s", r.ID, r.Label(), r.Version, m.runSummaryStatus(r), dateLabel(r.FinishedAt)) rows = append(rows, menuOption(text, i == m.runIndex)) rows = append(rows, " "+ansi.Truncate(resultLabel(r), max(1, m.width-6), "…")) } @@ -710,7 +717,8 @@ func (m Model) runsDetailsView(height int) string { return panel(m.width, height, []string{bar(title, fmt.Sprint(len(runs)), max(1, m.width-4), true)}, rows, footer, true) } r := m.runDetail - header := bar(fmt.Sprintf("#%d %s · v%d", r.ID, r.Label(), r.Version), m.progress(r.Status)+" · "+runDuration(r.CreatedAt, r.FinishedAt), m.width, false) + status := m.runSummaryStatus(r.RunSummary) + header := bar(fmt.Sprintf("#%d %s · v%d", r.ID, r.Label(), r.Version), status+" · "+runDuration(r.CreatedAt, r.FinishedAt), m.width, false) if height <= 6 { return fitBlock([]string{header, runHighlight(m.runSelectionLabel(), m.width), m.runActions() + " · Tab pane · Esc back"}, m.width, height) } From 3c47b08d985572c9a3a28d1683c5c8c3b8fa1105 Mon Sep 17 00:00:00 2001 From: Alan Silva Date: Sun, 18 Oct 2026 12:37:20 +0100 Subject: [PATCH 5/7] Restore connections on app foregrounding Add a process-lifecycle foreground reconnector and pure recovery policy, wiring repository recovery through the validated saved endpoint. Include lifecycle-process and focused unit coverage while preserving explicit Disconnect and stale socket callback guards. Checked packaged templates and packaged authoring skill references: neither documents Android connection lifecycle behavior, so neither needs updating. Shared android README changes are recorded in docs/notes/foreground-reconnect.md for the docs-fold writer. --- android/app/build.gradle.kts | 1 + .../awan/pipelines/PipelinesApplication.kt | 34 +++++ .../pipelines/data/ForegroundReconnector.kt | 81 ++++++++++ .../pipelines/data/PipelinesRepository.kt | 3 + .../data/transport/ConnectionRecovery.kt | 40 +++++ .../data/ForegroundReconnectorTest.kt | 138 ++++++++++++++++++ android/gradle/libs.versions.toml | 1 + docs/notes/foreground-reconnect.md | 95 ++++++++++++ 8 files changed, 393 insertions(+) create mode 100644 android/app/src/main/java/net/awan/pipelines/data/ForegroundReconnector.kt create mode 100644 android/app/src/main/java/net/awan/pipelines/data/transport/ConnectionRecovery.kt create mode 100644 android/app/src/test/java/net/awan/pipelines/data/ForegroundReconnectorTest.kt create mode 100644 docs/notes/foreground-reconnect.md diff --git a/android/app/build.gradle.kts b/android/app/build.gradle.kts index 4b89f8dc..02d1b391 100644 --- a/android/app/build.gradle.kts +++ b/android/app/build.gradle.kts @@ -58,6 +58,7 @@ dependencies { implementation(libs.androidx.compose.material.icons.extended) implementation(libs.androidx.navigation.compose) implementation(libs.androidx.lifecycle.runtime.compose) + implementation(libs.androidx.lifecycle.process) implementation(libs.androidx.lifecycle.viewmodel.compose) implementation(libs.androidx.lifecycle.viewmodel.ktx) implementation(libs.kotlinx.coroutines.android) diff --git a/android/app/src/main/java/net/awan/pipelines/PipelinesApplication.kt b/android/app/src/main/java/net/awan/pipelines/PipelinesApplication.kt index 8d36876e..e0a93b0e 100644 --- a/android/app/src/main/java/net/awan/pipelines/PipelinesApplication.kt +++ b/android/app/src/main/java/net/awan/pipelines/PipelinesApplication.kt @@ -1,12 +1,15 @@ package net.awan.pipelines import android.app.Application +import androidx.lifecycle.ProcessLifecycleOwner import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.SupervisorJob +import net.awan.pipelines.data.ForegroundReconnector import net.awan.pipelines.data.PipelinesRepository import net.awan.pipelines.data.SettingsRepository import net.awan.pipelines.data.transport.PipelinesClient +import net.awan.pipelines.data.transport.ServerUrl import net.awan.pipelines.notify.RunNotifier /** @@ -30,8 +33,39 @@ class PipelinesApplication : Application() { */ val runNotifier: RunNotifier by lazy { RunNotifier(this) } + /** + * The one owner of return-to-foreground recovery: when the process comes back + * to the front it restores a connection the background took away, with no + * interaction. It is created once with the Application and observes the + * process lifecycle, so every Activity and ViewModel recreation leaves it + * untouched. The liveness check is the existing `ping`; the reconnect is the + * same shared-client connect the app's own Connect/Reconnect uses, so the + * policy lives only in `foregroundAction` and the client's backoff stays the + * only reconnect mechanism. + */ + private val foregroundReconnector: ForegroundReconnector by lazy { + ForegroundReconnector( + state = { repository.connectionState.value }, + confirmAlive = { + // The existing ping is the positive check: on a dead socket it + // throws (or never answers) instead of returning, so reaching the + // next line is the proof of life. + repository.ping() + true + }, + reconnect = { + // A saved URL that no longer validates is left for Settings to + // report; there is nothing to reconnect to. + val validated = ServerUrl.validate(settings.serverUrl.value) + if (validated.valid) repository.recover(validated) + }, + scope = applicationScope, + ) + } + override fun onCreate() { super.onCreate() runNotifier.ensureChannel() + ProcessLifecycleOwner.get().lifecycle.addObserver(foregroundReconnector) } } diff --git a/android/app/src/main/java/net/awan/pipelines/data/ForegroundReconnector.kt b/android/app/src/main/java/net/awan/pipelines/data/ForegroundReconnector.kt new file mode 100644 index 00000000..1819acbc --- /dev/null +++ b/android/app/src/main/java/net/awan/pipelines/data/ForegroundReconnector.kt @@ -0,0 +1,81 @@ +package net.awan.pipelines.data + +import androidx.lifecycle.DefaultLifecycleObserver +import androidx.lifecycle.LifecycleOwner +import kotlinx.coroutines.CancellationException +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.launch +import kotlinx.coroutines.withTimeoutOrNull +import net.awan.pipelines.data.transport.ConnectionState +import net.awan.pipelines.data.transport.ForegroundAction +import net.awan.pipelines.data.transport.foregroundAction + +/** + * The one lifecycle-aware owner of the app's return-to-foreground recovery. + * + * While another app is in front Android freezes — and often kills — this process, + * so the socket it holds dies or goes quiet and the client's own bounded backoff + * either never starts or starts too late for the operator, who comes back and + * taps Connect. The client's backoff (see `PipelinesClient.scheduleReconnect`) is + * still the only reconnect mechanism; this owner only decides *when* to drive it, + * and does so the moment the app becomes visible again. + * + * It observes the whole process lifecycle (a `ProcessLifecycleOwner`), never the + * Activity or the ViewModel, both of which Android recreates: the owner is + * created once with the Application and survives every recreation. The decision + * itself is the one pure [foregroundAction] policy; this class supplies only the + * current state, a positive liveness check and the reconnect effect, so the + * policy is unit-testable with no device, socket or framework. + */ +class ForegroundReconnector( + private val state: () -> ConnectionState, + private val confirmAlive: suspend () -> Boolean, + private val reconnect: () -> Unit, + private val scope: CoroutineScope, + private val confirmTimeoutMillis: Long = CONFIRM_TIMEOUT_MILLIS, +) : DefaultLifecycleObserver { + + /** + * The app became visible again. A state the policy calls reconnecting is + * reconnected at once — the pending backoff delay is cancelled by the + * reconnect effect, so the operator never waits it out. A socket the state + * still calls Connected is first confirmed to answer, so a silent + * "Connected" is never trusted; the state is re-read afterwards, because the + * socket's own failure callback may have run while the check was in flight. + */ + override fun onStart(owner: LifecycleOwner) { + val before = state() + if (before !is ConnectionState.Connected) { + if (foregroundAction(before, confirmedAlive = false) == ForegroundAction.Reconnect) reconnect() + return + } + scope.launch { + val alive = confirm() + if (foregroundAction(state(), alive) == ForegroundAction.Reconnect) reconnect() + } + } + + /** + * Confirms a supposedly-connected socket still answers, bounded so a dead one + * cannot hold recovery: the check runs off the main thread on the app scope + * and a probe that fails, throws or outlives its budget counts as not alive. + */ + private suspend fun confirm(): Boolean = + withTimeoutOrNull(confirmTimeoutMillis) { + try { + confirmAlive() + } catch (t: Throwable) { + if (t is CancellationException) throw t + false + } + } ?: false + + companion object { + /** + * Bounds the liveness check. A healthy round trip on the trusted LAN/VPN + * is far shorter, so a socket that cannot answer is noticed promptly + * instead of holding the "Connected" the operator is looking at. + */ + const val CONFIRM_TIMEOUT_MILLIS = 3_000L + } +} diff --git a/android/app/src/main/java/net/awan/pipelines/data/PipelinesRepository.kt b/android/app/src/main/java/net/awan/pipelines/data/PipelinesRepository.kt index 1479576f..9ad104dd 100644 --- a/android/app/src/main/java/net/awan/pipelines/data/PipelinesRepository.kt +++ b/android/app/src/main/java/net/awan/pipelines/data/PipelinesRepository.kt @@ -66,6 +66,9 @@ class PipelinesRepository( /** Opens the single shared connection to a validated server URL. */ fun connect(validated: ServerUrl.Validated) = client.connect(validated) + /** Recovers a lost connection without reversing an explicit Disconnect. */ + fun recover(validated: ServerUrl.Validated) = client.recover(validated) + /** Explicit disconnect; the client then stays Idle. */ fun disconnect() = client.disconnect() diff --git a/android/app/src/main/java/net/awan/pipelines/data/transport/ConnectionRecovery.kt b/android/app/src/main/java/net/awan/pipelines/data/transport/ConnectionRecovery.kt new file mode 100644 index 00000000..87b44b1b --- /dev/null +++ b/android/app/src/main/java/net/awan/pipelines/data/transport/ConnectionRecovery.kt @@ -0,0 +1,40 @@ +package net.awan.pipelines.data.transport + +/** What the app does with the connection the moment it becomes visible again. */ +enum class ForegroundAction { + /** Leave the connection exactly as it is. */ + LeaveAlone, + + /** Re-establish the connection now, cancelling any pending reconnect delay. */ + Reconnect, +} + +/** + * The app's return-to-foreground recovery policy, in one pure place. + * + * Android freezes and often kills a backgrounded app's process, so the socket + * the app holds dies or goes quiet while another app is in front and nothing in + * the process notices in time. Coming back must restore that connection without + * the operator tapping Connect — but an explicit Disconnect is the operator's + * own decision and stays authoritative. A deliberate Disconnect is the only way + * the client reaches [ConnectionState.Idle], so Idle is left alone; every other + * state reconnects. + * + * A still [ConnectionState.Connected] socket is not trusted on return: a frozen + * process can leave it claiming Connected while it answers nothing, which is + * exactly the stale state this replaces. The caller therefore checks it with a + * positive probe (the existing ping) and passes the verdict as [confirmedAlive]; + * only a connection just confirmed to answer is left alone. + * + * @param state the connection state at the moment the app became visible again. + * @param confirmedAlive whether a [ConnectionState.Connected] socket has just + * been confirmed to answer. It is consulted only for Connected; every other + * state carries its own answer. + */ +fun foregroundAction(state: ConnectionState, confirmedAlive: Boolean): ForegroundAction = when (state) { + ConnectionState.Idle -> ForegroundAction.LeaveAlone + is ConnectionState.Connected -> + if (confirmedAlive) ForegroundAction.LeaveAlone else ForegroundAction.Reconnect + is ConnectionState.Connecting -> ForegroundAction.Reconnect + is ConnectionState.Disconnected -> ForegroundAction.Reconnect +} diff --git a/android/app/src/test/java/net/awan/pipelines/data/ForegroundReconnectorTest.kt b/android/app/src/test/java/net/awan/pipelines/data/ForegroundReconnectorTest.kt new file mode 100644 index 00000000..7491ae26 --- /dev/null +++ b/android/app/src/test/java/net/awan/pipelines/data/ForegroundReconnectorTest.kt @@ -0,0 +1,138 @@ +package net.awan.pipelines.data + +import androidx.lifecycle.Lifecycle +import androidx.lifecycle.LifecycleOwner +import java.util.concurrent.CountDownLatch +import java.util.concurrent.TimeUnit +import java.util.concurrent.atomic.AtomicInteger +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.delay +import net.awan.pipelines.data.transport.ConnectionState +import org.junit.Assert.assertEquals +import org.junit.Test + +/** + * The one lifecycle-aware owner of return-to-foreground recovery: it applies the + * pure `foregroundAction` policy to the state it reads, confirms a supposedly + * connected socket before trusting it, and reconnects when — and only when — the + * policy says so. The state, the liveness check and the reconnect are injected, + * so the owner is exercised with no device, socket or Android framework. The + * owner never touches the LifecycleOwner it is handed. + */ +class ForegroundReconnectorTest { + + private val reconnects = AtomicInteger(0) + private val probes = AtomicInteger(0) + + private val owner = object : LifecycleOwner { + override val lifecycle: Lifecycle get() = throw UnsupportedOperationException("not used") + } + + private val connected = ConnectionState.Connected(build = null, serverTime = null) + + /** + * A reconnector whose probe answers [alive] without suspending, so + * `Dispatchers.Unconfined` runs the whole `onStart` decision inline and the + * assertions below hold the moment it returns. + */ + private fun reconnector( + state: () -> ConnectionState, + alive: suspend () -> Boolean = { true }, + ) = ForegroundReconnector( + state = state, + confirmAlive = { probes.incrementAndGet(); alive() }, + reconnect = { reconnects.incrementAndGet() }, + scope = CoroutineScope(Dispatchers.Unconfined), + ) + + @Test + fun aDeliberateDisconnectIsLeftAloneAndNotEvenProbed() { + reconnector(state = { ConnectionState.Idle }).onStart(owner) + assertEquals(0, probes.get()) + assertEquals(0, reconnects.get()) + } + + @Test + fun aDroppedConnectionReconnectsImmediatelyWithoutProbing() { + reconnector(state = { ConnectionState.Disconnected("socket closed", retrying = true) }).onStart(owner) + assertEquals("a dropped socket has nothing to confirm", 0, probes.get()) + assertEquals(1, reconnects.get()) + } + + @Test + fun aConfirmedConnectedSocketIsLeftAlone() { + reconnector(state = { connected }).onStart(owner) + assertEquals(1, probes.get()) + assertEquals(0, reconnects.get()) + } + + @Test + fun aConnectedSocketThatDoesNotAnswerReconnects() { + reconnector(state = { connected }, alive = { false }).onStart(owner) + assertEquals(1, probes.get()) + assertEquals(1, reconnects.get()) + } + + @Test + fun aProbeThatThrowsIsNotAlive() { + reconnector( + state = { connected }, + alive = { throw IllegalStateException("socket closed") }, + ).onStart(owner) + assertEquals(1, reconnects.get()) + } + + @Test + fun aSocketThatDiesWhileTheProbeRunsStillReconnects() { + // The socket's own failure callback lands mid-check; the owner re-reads + // the state and the policy then decides on what is actually there. + var state: ConnectionState = connected + reconnector( + state = { state }, + alive = { + state = ConnectionState.Disconnected("socket closed", retrying = true) + false + }, + ).onStart(owner) + assertEquals(1, reconnects.get()) + } + + @Test + fun aDisconnectWhileTheProbeRunsStaysDisconnected() { + // The user can tap Disconnect while the bounded foreground probe is in + // flight. Re-reading state after that probe keeps the explicit choice + // authoritative rather than reconnecting over it. + var state: ConnectionState = connected + reconnector( + state = { state }, + alive = { + state = ConnectionState.Idle + true + }, + ).onStart(owner) + assertEquals(0, reconnects.get()) + } + + @Test + fun aProbeThatNeverAnswersIsNotAliveWhenItsBudgetExpires() { + val done = CountDownLatch(1) + val reconnector = ForegroundReconnector( + state = { connected }, + confirmAlive = { delay(5_000); true }, + reconnect = { + reconnects.incrementAndGet() + done.countDown() + }, + scope = CoroutineScope(Dispatchers.Unconfined), + confirmTimeoutMillis = 20L, + ) + reconnector.onStart(owner) + assertEquals( + "the bounded check must give up on its own rather than hold the UI's Connected", + true, + done.await(2, TimeUnit.SECONDS), + ) + assertEquals(1, reconnects.get()) + } +} diff --git a/android/gradle/libs.versions.toml b/android/gradle/libs.versions.toml index c1d8f4df..9932d8a4 100644 --- a/android/gradle/libs.versions.toml +++ b/android/gradle/libs.versions.toml @@ -23,6 +23,7 @@ androidx-compose-material3 = { group = "androidx.compose.material3", name = "mat androidx-compose-material-icons-extended = { group = "androidx.compose.material", name = "material-icons-extended" } androidx-navigation-compose = { group = "androidx.navigation", name = "navigation-compose", version.ref = "navigationCompose" } androidx-lifecycle-runtime-compose = { group = "androidx.lifecycle", name = "lifecycle-runtime-compose", version.ref = "lifecycle" } +androidx-lifecycle-process = { group = "androidx.lifecycle", name = "lifecycle-process", version.ref = "lifecycle" } androidx-lifecycle-viewmodel-compose = { group = "androidx.lifecycle", name = "lifecycle-viewmodel-compose", version.ref = "lifecycle" } androidx-lifecycle-viewmodel-ktx = { group = "androidx.lifecycle", name = "lifecycle-viewmodel-ktx", version.ref = "lifecycle" } kotlinx-coroutines-android = { group = "org.jetbrains.kotlinx", name = "kotlinx-coroutines-android", version.ref = "coroutines" } diff --git a/docs/notes/foreground-reconnect.md b/docs/notes/foreground-reconnect.md new file mode 100644 index 00000000..1125f0e5 --- /dev/null +++ b/docs/notes/foreground-reconnect.md @@ -0,0 +1,95 @@ +# Android: coming back to the app restores the connection + +## What this item changed + +Android app only. + +- `data/transport/ConnectionRecovery.kt` (new): `ForegroundAction` and the one + pure `foregroundAction(state, confirmedAlive)` policy — `Idle` → leave alone (a + deliberate Disconnect is the only way the client reaches Idle), `Connected` with + a confirmed-alive socket → leave alone, `Connected` unconfirmed → reconnect, + `Connecting`/`Disconnected` → reconnect. +- `data/ForegroundReconnector.kt` (new): the one lifecycle-aware owner. It is + created once with the Application and observes `ProcessLifecycleOwner`, never + the Activity or the ViewModel (both recreated). On `ON_START` it applies the + policy: a non-Connected state reconnects at once, and the reconnect cancels the + client's pending backoff delay, so the operator never waits it out; a state + that still claims `Connected` is first confirmed with the existing `ping`, + bounded to 3 s, and the state is re-read afterwards (the socket's own failure + callback can land while the check is in flight). +- `PipelinesApplication.kt`: wires the owner to the process lifecycle; + `confirmAlive` is `repository.ping()`, `reconnect` is + `ServerUrl.validate(...) + repository.recover(...)`. Recovery shares the + client's state lock with Disconnect, so a Disconnect after the probe has made + its decision but before its reconnect effect runs stays authoritative. Manual + Connect/Reconnect continue to use `repository.connect(...)`; the client's + bounded backoff stays the only reconnect mechanism and the policy lives in the + one pure place. +- `data/transport/PipelinesClient.kt`: `open()` now closes the socket it + supersedes, and every `WebSocketListener` callback ignores a socket that is no + longer the current one (`isCurrent`). A failure the replaced socket reports late + — the common case when the process was frozen while it died — can then neither + clear the new connection nor schedule a reconnect that would undo the recovery. +- `app/build.gradle.kts` and `gradle/libs.versions.toml`: + `androidx.lifecycle:lifecycle-process` 2.8.7 (already in the offline cache, + with its `androidx.startup`/`androidx.tracing` transitives). +- Tests: `data/transport/ConnectionRecoveryTest.kt` (5 cases) and + `data/ForegroundReconnectorTest.kt` (8 cases). + +## Shared documents this change makes stale + +`android/README.md`. + +- The Configuration section (`android/README.md:99-101`) — "Tap **Connect** to + open the single shared WebSocket (probed with `ping`, showing the server + build). Reconnect uses bounded exponential backoff; **Disconnect** stops it." — + should record that coming back to the app restores the connection on its own: + the process coming to the front reconnects immediately, cancelling the pending + backoff delay, and when the last state still claims Connected it first confirms + the socket with `ping` before trusting it. An explicit Disconnect stays + authoritative and is never reconnected over; the manual Reconnect control + stays, but normal recovery never needs it. +- The Architecture block (`android/README.md:310-323`) lists `data/transport` and + `data`. It should name the two new files — `data/transport/ConnectionRecovery.kt` + (the pure return-to-foreground policy) and `data/ForegroundReconnector.kt` (the + process-lifecycle owner) — and the new `androidx.lifecycle:lifecycle-process` + dependency behind the owner. + +## Not stale + +- `android/AGENTS.md` names neither the presence nor the absence of a lifecycle + observer; its "Where things are" summary stays correct (it could optionally add + the new owner beside `data/PipelinesRepository.kt`). +- `docs/app-integration-verification.md` — its connection mentions are about + restoring the stub URL, which this item does not change. +- No packaged OpenCode skill reference (`internal/author/skill/references/`) + documents the app's connection lifecycle, so nothing there is stale. + +## Verified on the device, and what was not + +Built and captured on the operator's Pixel 9 Pro XL (`47241FDAS00DLK`) from +`51a2ba4` + this change; after merging main the same tree (now `b93eee2`) was +rebuilt and the full unit suite re-run (691 tests, 0 failures). The operator's +`ws://10.0.0.75:8080/connect` and `/Users/alan/repos/pipelines` were used +throughout and were not changed. The phone was left on main's build. + +Captured, all with `android/scripts/capture_png.sh` (every image validated): + +- a **live** socket on return: the server log shows a `ping`/`pong` on the same + connection at the second of the return and no reconnect — the positive check + runs and a healthy connection is left alone; +- a **dead** socket on return: the reconnect and re-subscribe land in the same + second as the return, with no interaction (the process was frozen, so the + client's own backoff had not run); +- **no network** on return: the banner reads "Disconnected: Failed to connect to + …:8080 · retrying with backoff" — never a false Connected — and recovery + follows on its own once the network returns; +- an explicit **Disconnect**: still "Not connected. Enter the server URL in + Settings, then connect." after returning, with no reconnect in the server log. + +**Not verified live:** the stale-Connected probe branch. In every capture the +frozen socket's own failure was delivered to the client before the lifecycle +owner ran, so the client already read `Disconnected` and no probe was attempted +(confirmed by the client's request-id sequence never skipping a value in the +server log). That branch — `Connected` still claimed, probe fails, reconnect — is +covered by unit tests only. From e9d902213ed310ec8f94b84bb74597e18dc0c594 Mon Sep 17 00:00:00 2001 From: Alan Silva Date: Sun, 18 Oct 2026 12:54:36 +0100 Subject: [PATCH 6/7] Preserve background shell visibility across readback races Reconcile authoritative shell snapshots with bounded lifecycle journals so concurrent exits and creates cannot be overwritten, coalesce observation setup, and retain verified ownership across reconnects. Add deterministic reconciliation and cancellation regressions plus stable limited-visibility handling.\n\nChecked packaged templates and authoring skill references: none need updating. The task-owned docs note records the stale shared chat documentation for docs-fold; shared documentation was not edited. --- docs/notes/background-shell-visibility.md | 37 +++ internal/opencode/chat.go | 209 +++++++++++++- .../opencode/chat_shell_reconcile_test.go | 116 ++++++++ internal/opencode/chat_shell_snapshot.go | 72 +++++ internal/opencode/chat_v2_test.go | 42 +++ internal/server/chat.go | 12 + internal/server/chat_hub.go | 12 +- internal/server/chat_multiclient_test.go | 2 + internal/server/chat_room.go | 186 ++++++++++++- internal/server/chat_shell_reconcile_test.go | 261 ++++++++++++++++++ internal/server/chat_stream.go | 13 +- internal/server/chat_test.go | 99 ++++++- 12 files changed, 1036 insertions(+), 25 deletions(-) create mode 100644 docs/notes/background-shell-visibility.md create mode 100644 internal/opencode/chat_shell_reconcile_test.go create mode 100644 internal/opencode/chat_shell_snapshot.go create mode 100644 internal/server/chat_shell_reconcile_test.go diff --git a/docs/notes/background-shell-visibility.md b/docs/notes/background-shell-visibility.md new file mode 100644 index 00000000..583e5f90 --- /dev/null +++ b/docs/notes/background-shell-visibility.md @@ -0,0 +1,37 @@ +# Background shell visibility + +The chat server now opens the broker stream for read-only subscribers and reconciles verified running shell operations from OpenCode's authoritative `/api/shell` readback. The packaged OpenCode skill references and chat documentation should describe that shell visibility survives reconnects only when OpenCode reports explicit `metadata.sessionID` ownership; shared working directories are not ownership evidence. + +## Current source behavior + +Observers share one initial readback per room/stream attachment. The adapter and hub +fold bounded lifecycle journals over each snapshot, so a stale read cannot resurrect +an exited shell or erase a concurrent create. Shell replay and live delivery enqueue +in fold order. Cancellation rejects old-room callbacks and releases local identity +state before reconnect. Journals retain at most 256 identities and 1 MiB of id/command +text; capacity/read failures surface stable limited-visibility text, never broker +response bodies. No shell polling was added. OpenCode's terminal `timeout` and +`killed` statuses are excluded from retained running operations as well as `exited`. + +## Verification evidence — 2026-10-18 + +`opencode api get /openapi.json` authenticated through supported service discovery. +The live schema advertises shell list/detail/output and generic `metadata`; it does +not type `metadata.sessionID`. A throwaway session +`ses_eb12ab7b3ffe0ZAle7PHpftHMG` and its own `sleep 1` shell +`sh_14ed57134001e6fuYqvn62pKit` verified actual create/detail readback preserving +explicit `metadata.sessionID`, status `running`, subsequent `exited`/exit 0, and +absence from the running list. This verifies explicit API metadata, not automatic +agent-tool ownership injection. The CLI event attempt yielded no captured events; +live-provider event ordering is **not tested**. Forced ordering evidence comes from +channel/barrier regressions, including a real WebSocket handler on a loopback scratch +server with fresh SQLite, not from the deployed service. + +## Documentation follow-up / deployment + +`docs/chat-verification.md` and the chat subscribe/replay section of `docs/cli.md` +are stale: docs-fold should describe authoritative ownership/readback, coalesced +observation, bounded reconciliation and limited-visibility errors, distinguishing +fixture ordering evidence from the live readback above. Shared docs and packaged +skill references were not edited here. No deployment, installation, service restart +or physical-phone/UI verification was performed; this changes no rendered layout. diff --git a/internal/opencode/chat.go b/internal/opencode/chat.go index 40adc005..8fae8cd9 100644 --- a/internal/opencode/chat.go +++ b/internal/opencode/chat.go @@ -28,7 +28,9 @@ var ErrPermissionUnsupported = errors.New("OpenCode permission approval is not i // semantics, the session-id shape and the 64 KiB / 1 MiB bounds of Connection so // no second transport or credential path is introduced. type Chat struct { - conn *Connection + conn *Connection + shellMu sync.Mutex + shells map[string]*ChatStreamState // mu serializes the adapter's HTTP calls. Connection records its last // operation and status on the shared connection, so two calls must not // overlap; the permission poller runs alongside the read loop's @@ -294,6 +296,114 @@ func decodeChatPermissions(sessionID string, data []byte) ([]protocol.ChatPermis return frames, nil } +// Shells reads the endpoint's authoritative shell list and returns only running +// operations whose metadata explicitly names sessionID. The shell API is global; +// filtering locally avoids guessing an unsupported query filter and never assigns +// ownership from a shared working directory. A successful read also seeds the +// stream state used to attribute later shell.exited events, which carry no session. +func (c *Chat) Shells(ctx context.Context, sessionID string) ([]protocol.ChatShell, error) { + if !ValidSessionID(sessionID) { + return nil, fmt.Errorf("invalid OpenCode session id") + } + state, err := c.streamStateContext(ctx, sessionID) + if err != nil { + return nil, err + } + // One read per state; waiting and I/O never hold the event-state mutex. + state.mu.Lock() + if state.readGate == nil { + state.readGate = make(chan struct{}, 1) + } + gate := state.readGate + state.mu.Unlock() + select { + case gate <- struct{}{}: + case <-ctx.Done(): + return nil, ctx.Err() + } + defer func() { <-gate }() + state.mu.Lock() + state.readback = &ShellSnapshot{} + state.mu.Unlock() + defer func() { state.mu.Lock(); state.readback = nil; state.mu.Unlock() }() + data, err := c.call(ctx, "get", "/api/shell", nil) + if err != nil { + return nil, err + } + var response struct { + Data []chatShellInfo `json:"data"` + } + if err := json.Unmarshal(data, &response); err != nil || response.Data == nil { + if err := json.Unmarshal(data, &response.Data); err != nil { + return nil, fmt.Errorf("invalid OpenCode shell response") + } + } + out := make([]protocol.ChatShell, 0, len(response.Data)) + for _, info := range response.Data { + if info.ID == "" || info.shellSessionID() != sessionID || (info.Status != "" && info.Status != protocol.ChatShellRunning) { + continue + } + status := info.Status + if status == "" { + status = protocol.ChatShellRunning + } + out = append(out, protocol.ChatShell{ID: info.ID, Status: status, Command: info.Command}) + } + state.mu.Lock() + defer state.mu.Unlock() + out, err = state.readback.Merge(out) + if err != nil { + return nil, err + } + if err := ctx.Err(); err != nil { + return nil, err + } + ids := make([]string, 0, len(out)) + for _, shell := range out { + ids = append(ids, shell.ID) + } + state.replaceShellsLocked(ids) + return out, nil +} + +// streamState returns the session's retained, verified shell identities. It is +// deliberately kept across stream replacement so an exit after reconnect remains +// attributable; Shells reconciles its contents from authoritative readback on +// each attach rather than replacing the pointer a live scanner is using. +func (c *Chat) streamState(sessionID string) *ChatStreamState { + state, _ := c.streamStateContext(context.Background(), sessionID) + return state +} + +func (c *Chat) streamStateContext(ctx context.Context, sessionID string) (*ChatStreamState, error) { + c.shellMu.Lock() + defer c.shellMu.Unlock() + if err := ctx.Err(); err != nil { + return nil, err + } + if c.shells == nil { + c.shells = map[string]*ChatStreamState{} + } + state := c.shells[sessionID] + if state == nil { + state = &ChatStreamState{} + c.shells[sessionID] = state + } + return state, nil +} + +// ReleaseShells discards verified shell identities after the last room observer +// disconnects. A future attach calls Shells for a fresh authoritative snapshot, +// so this bounds state without attributing a foreign shell during the gap. +func (c *Chat) ReleaseShells(sessionID string) { + if sessionID == "" { + return + } + c.shellMu.Lock() + delete(c.shells, sessionID) + c.shellMu.Unlock() +} + // Stream subscribes to the OpenCode event firehose and invokes emit for every // frame belonging to sessionID. It reads one bounded SSE frame at a time and // never polls the per-session message log; the global firehose is the only @@ -306,6 +416,12 @@ func (c *Chat) Stream(ctx context.Context, id string, ready func(), emit func(pr if !sessionID.MatchString(id) { return fmt.Errorf("invalid OpenCode session id") } + // Pin state before opening/announcing the stream so a canceled scanner + // cannot acquire a replacement room's identity map. + state, err := c.streamStateContext(ctx, id) + if err != nil { + return err + } body, err := c.conn.EventStream(ctx) if err != nil { return err @@ -314,7 +430,7 @@ func (c *Chat) Stream(ctx context.Context, id string, ready func(), emit func(pr if ready != nil { ready() } - return scanChatEvents(ctx, body, id, emit) + return c.scanChatEventsWithState(ctx, body, id, state, emit) } // scanChatEvents turns an SSE body into protocol frames, one bounded frame at a @@ -322,10 +438,13 @@ func (c *Chat) Stream(ctx context.Context, id string, ready func(), emit func(pr // malformed JSON fails the stream instead of producing a malformed frame. The // last assistant message id is retained so a later tool/step event that omits it // still attributes every frame to a stable message. -func scanChatEvents(ctx context.Context, body io.Reader, sessionID string, emit func(protocol.ChatEventFrame) error) error { +func (c *Chat) scanChatEvents(ctx context.Context, body io.Reader, sessionID string, emit func(protocol.ChatEventFrame) error) error { + return c.scanChatEventsWithState(ctx, body, sessionID, c.streamState(sessionID), emit) +} + +func (c *Chat) scanChatEventsWithState(ctx context.Context, body io.Reader, sessionID string, state *ChatStreamState, emit func(protocol.ChatEventFrame) error) error { scanner := bufio.NewScanner(body) scanner.Buffer(make([]byte, 0, 64<<10), MaxResponse) - state := &ChatStreamState{} for scanner.Scan() { if err := ctx.Err(); err != nil { return err @@ -346,7 +465,7 @@ func scanChatEvents(ctx context.Context, body io.Reader, sessionID string, emit continue } if frame.MessageID != "" { - state.LastMessageID = frame.MessageID + state.setLastMessageID(frame.MessageID) } if err := emit(frame); err != nil { return err @@ -449,8 +568,29 @@ func (i *chatShellInfo) shellSessionID() string { // what attributes the exit to its session: another session's shell is an // unknown id and is ignored. type ChatStreamState struct { + mu sync.Mutex LastMessageID string Shells map[string]struct{} + readGate chan struct{} + readback *ShellSnapshot +} + +func (s *ChatStreamState) lastMessageID() string { + if s == nil { + return "" + } + s.mu.Lock() + defer s.mu.Unlock() + return s.LastMessageID +} + +func (s *ChatStreamState) setLastMessageID(id string) { + if s == nil || id == "" { + return + } + s.mu.Lock() + s.LastMessageID = id + s.mu.Unlock() } // knowsShell reports whether id is a shell this stream has already surfaced for @@ -459,6 +599,8 @@ func (s *ChatStreamState) knowsShell(id string) bool { if s == nil || id == "" { return false } + s.mu.Lock() + defer s.mu.Unlock() _, ok := s.Shells[id] return ok } @@ -468,6 +610,8 @@ func (s *ChatStreamState) markShell(id string) { if s == nil || id == "" { return } + s.mu.Lock() + defer s.mu.Unlock() if s.Shells == nil { s.Shells = map[string]struct{}{} } @@ -476,12 +620,37 @@ func (s *ChatStreamState) markShell(id string) { // forgetShell drops id once its exit has been surfaced. func (s *ChatStreamState) forgetShell(id string) { - if s == nil || s.Shells == nil { + if s == nil { + return + } + s.mu.Lock() + defer s.mu.Unlock() + if s.Shells == nil { return } delete(s.Shells, id) } +// replaceShells applies one authoritative running-shell snapshot without +// replacing the state object an already attached event scanner owns. +func (s *ChatStreamState) replaceShells(ids []string) { + if s == nil { + return + } + s.mu.Lock() + defer s.mu.Unlock() + s.replaceShellsLocked(ids) +} + +func (s *ChatStreamState) replaceShellsLocked(ids []string) { + s.Shells = make(map[string]struct{}, len(ids)) + for _, id := range ids { + if id != "" { + s.Shells[id] = struct{}{} + } + } +} + // NormalizeChatEvent maps one OpenCode firehose `data:` payload to a // protocol.ChatEventFrame, folding the small per-session context ([stream]) it // needs across frames. ok is false for an event type this adapter does not @@ -528,7 +697,7 @@ func NormalizeChatEvent(payload []byte, sessionID string, stream *ChatStreamStat messageID = data.MessageIDAlt } if messageID == "" && stream != nil { - messageID = stream.LastMessageID + messageID = stream.lastMessageID() } // Every pushed frame must carry a stable message id so the client can // attribute it; without one this is not a chat frame. @@ -590,20 +759,38 @@ func normalizeShellEvent(eventType string, data chatEvent, sessionID string, str if status == "" { status = protocol.ChatShellRunning } - stream.markShell(data.Info.ID) + shell := &protocol.ChatShell{ID: data.Info.ID, Status: status, Command: data.Info.Command} + if stream != nil { + stream.mu.Lock() + stream.readback.Record(shell) + if stream.Shells == nil { + stream.Shells = map[string]struct{}{} + } + if status == protocol.ChatShellRunning && len(stream.Shells) < chatShellIdentityLimit { + stream.Shells[shell.ID] = struct{}{} + } + stream.mu.Unlock() + } return protocol.ChatEventFrame{ SessionID: sessionID, Kind: protocol.ChatEventShell, - Shell: &protocol.ChatShell{ID: data.Info.ID, Status: status, Command: data.Info.Command}, + Shell: shell, }, true, nil } if data.ID == "" { return protocol.ChatEventFrame{}, false, fmt.Errorf("malformed OpenCode shell.exited event") } - if !stream.knowsShell(data.ID) { + if stream == nil { + return protocol.ChatEventFrame{}, false, nil + } + stream.mu.Lock() + stream.readback.Record(&protocol.ChatShell{ID: data.ID, Status: protocol.ChatShellExited}) + _, known := stream.Shells[data.ID] + delete(stream.Shells, data.ID) + stream.mu.Unlock() + if !known { return protocol.ChatEventFrame{}, false, nil } - stream.forgetShell(data.ID) status := data.Status if status == "" { status = protocol.ChatShellExited diff --git a/internal/opencode/chat_shell_reconcile_test.go b/internal/opencode/chat_shell_reconcile_test.go new file mode 100644 index 00000000..087080d0 --- /dev/null +++ b/internal/opencode/chat_shell_reconcile_test.go @@ -0,0 +1,116 @@ +package opencode + +import ( + "context" + "fmt" + "net/http" + "net/http/httptest" + "testing" + "time" + + "git.a-wan.net/alan/pipelines/internal/protocol" +) + +func TestChatShellReadbackBarrier(t *testing.T) { + for _, known := range []bool{false, true} { + t.Run(fmt.Sprint(known), func(t *testing.T) { + read, apply := make(chan struct{}), make(chan struct{}) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + close(read) + select { + case <-apply: + fmt.Fprint(w, `{"data":[{"id":"sh_old","status":"running","metadata":{"sessionID":"ses_1"}},{"id":"sh_foreign","status":"running","metadata":{"sessionID":"ses_other"}}]}`) + case <-r.Context().Done(): + } + })) + defer srv.Close() + c := NewChat(&Connection{Backend: "v2", URL: srv.URL}) + state := c.streamState("ses_1") + if known { + state.markShell("sh_old") + } + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + defer cancel() + result := make(chan []protocol.ChatShell, 1) + errs := make(chan error, 1) + go func() { shells, err := c.Shells(ctx, "ses_1"); result <- shells; errs <- err }() + select { + case <-read: + case <-ctx.Done(): + t.Fatal("read did not reach barrier") + } + // Both a previously verified exit and an exit first learned while + // reading must suppress the stale running snapshot. + frame, ok, err := NormalizeChatEvent([]byte(`{"type":"shell.exited","data":{"id":"sh_old","status":"exited","exit":0}}`), "ses_1", state) + if err != nil || ok != known || (ok && frame.Shell.Status != protocol.ChatShellExited) { + t.Fatalf("exit = %+v %v %v", frame, ok, err) + } + frame, ok, err = NormalizeChatEvent([]byte(`{"type":"shell.created","data":{"info":{"id":"sh_new","status":"running","command":"sleep 10","metadata":{"sessionID":"ses_1"}}}}`), "ses_1", state) + if err != nil || !ok || frame.Shell.ID != "sh_new" { + t.Fatalf("create = %+v %v %v", frame, ok, err) + } + close(apply) + got := <-result + if err := <-errs; err != nil { + t.Fatal(err) + } + if len(got) != 1 || got[0].ID != "sh_new" || got[0].Command != "sleep 10" { + t.Fatalf("readback lost events/leaked ownership: %+v", got) + } + if state.knowsShell("sh_old") || state.knowsShell("sh_foreign") || !state.knowsShell("sh_new") { + t.Fatal("adapter identity state regressed") + } + // User-visible subsequent exit remains attributable after reconciliation. + frame, ok, err = NormalizeChatEvent([]byte(`{"type":"shell.exited","data":{"id":"sh_new","status":"exited","exit":0}}`), "ses_1", state) + if err != nil || !ok || frame.Shell.ID != "sh_new" { + t.Fatalf("new shell exit lost: %+v %v %v", frame, ok, err) + } + }) + } +} + +func TestChatShellReadbackCancelWaiter(t *testing.T) { + read := make(chan struct{}) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { close(read); <-r.Context().Done() })) + defer srv.Close() + c := NewChat(&Connection{Backend: "v2", URL: srv.URL}) + ctx, cancel := context.WithCancel(context.Background()) + result := make(chan error, 2) + go func() { _, err := c.Shells(ctx, "ses_1"); result <- err }() + select { + case <-read: + case <-time.After(time.Second): + t.Fatal("read stuck") + } + go func() { _, err := c.Shells(ctx, "ses_1"); result <- err }() + cancel() + for range 2 { + select { + case err := <-result: + if err == nil { + t.Fatal("canceled read succeeded") + } + case <-time.After(time.Second): + t.Fatal("read or waiter stuck") + } + } + old := c.streamState("ses_1") + c.ReleaseShells("ses_1") + if c.streamState("ses_1") == old { + t.Fatal("reconnect reused released state") + } +} + +func TestShellSnapshotBounded(t *testing.T) { + var snapshot ShellSnapshot + for i := 0; i <= chatShellIdentityLimit; i++ { + snapshot.Record(&protocol.ChatShell{ID: fmt.Sprintf("sh_%d", i), Status: protocol.ChatShellExited}) + } + if len(snapshot.changes) != chatShellIdentityLimit { + t.Fatalf("unbounded journal: %d", len(snapshot.changes)) + } + if _, err := snapshot.Merge([]protocol.ChatShell{{ID: "sh_0", Status: protocol.ChatShellRunning}}); err == nil { + t.Fatal("overflow accepted stale snapshot") + } +} diff --git a/internal/opencode/chat_shell_snapshot.go b/internal/opencode/chat_shell_snapshot.go new file mode 100644 index 00000000..8559a741 --- /dev/null +++ b/internal/opencode/chat_shell_snapshot.go @@ -0,0 +1,72 @@ +package opencode + +import ( + "fmt" + "sort" + + "git.a-wan.net/alan/pipelines/internal/protocol" +) + +// ShellSnapshot records changes while an authoritative read is in flight. Its +// owner serializes access. Unknown exits may be recorded, but can only remove an +// id: they never establish ownership. Overflow rejects the snapshot, not events. +type ShellSnapshot struct { + changes map[string]protocol.ChatShell + bytes int + overflow bool +} + +const chatShellIdentityLimit = 256 + +func (s *ShellSnapshot) Record(shell *protocol.ChatShell) { + if s == nil || shell == nil || shell.ID == "" || s.overflow { + return + } + if s.changes == nil { + s.changes = make(map[string]protocol.ChatShell) + } + old := s.changes[shell.ID] + n := s.bytes - len(old.ID) - len(old.Command) + len(shell.ID) + len(shell.Command) + if n > MaxResponse || (old.ID == "" && len(s.changes) >= chatShellIdentityLimit) { + s.overflow = true + return + } + // A terminal event wins even if a duplicate create is delivered later. + if old.ID != "" && old.Status != protocol.ChatShellRunning { + return + } + s.changes[shell.ID] = *shell + s.bytes = n +} + +func (s *ShellSnapshot) Merge(shells []protocol.ChatShell) ([]protocol.ChatShell, error) { + if s.overflow { + return nil, fmt.Errorf("shell reconciliation capacity exceeded") + } + byID := make(map[string]protocol.ChatShell) + for _, shell := range shells { + if shell.ID != "" && shell.Status == protocol.ChatShellRunning { + byID[shell.ID] = shell + } + } + for id, shell := range s.changes { + if shell.Status == protocol.ChatShellRunning { + byID[id] = shell + } else { + delete(byID, id) + } + } + if len(byID) > chatShellIdentityLimit { + return nil, fmt.Errorf("shell reconciliation capacity exceeded") + } + ids := make([]string, 0, len(byID)) + for id := range byID { + ids = append(ids, id) + } + sort.Strings(ids) + out := make([]protocol.ChatShell, 0, len(ids)) + for _, id := range ids { + out = append(out, byID[id]) + } + return out, nil +} diff --git a/internal/opencode/chat_v2_test.go b/internal/opencode/chat_v2_test.go index 00cb2242..25fc7024 100644 --- a/internal/opencode/chat_v2_test.go +++ b/internal/opencode/chat_v2_test.go @@ -395,3 +395,45 @@ func TestOpenChatRejectsUnrunnableCLI(t *testing.T) { t.Fatal("a missing CLI must fail detection, not open an endpoint") } } + +// TestChatShellReadbackSeedsReplacementStream proves an authoritative shell +// readback survives a later stream replacement: shell.exited has no session id, +// so it must still clear the verified shell after reconnect. +func TestChatShellReadbackSeedsReplacementStream(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/api/shell": + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"data":[{"id":"sh_1","status":"running","command":"sleep 30","metadata":{"sessionID":"ses_1"}},{"id":"sh_other","status":"running","command":"sleep 30","metadata":{"sessionID":"ses_other"}}]}`)) + case "/api/event": + w.Header().Set("Content-Type", "text/event-stream") + _, _ = w.Write([]byte("data: {\"type\":\"shell.exited\",\"data\":{\"id\":\"sh_1\",\"status\":\"exited\",\"exit\":0}}\n\n")) + } + })) + defer srv.Close() + chat := NewChat(&Connection{Backend: "v2", URL: srv.URL}) + shells, err := chat.Shells(context.Background(), "ses_1") + if err != nil || len(shells) != 1 || shells[0].ID != "sh_1" { + t.Fatalf("Shells = %+v, %v", shells, err) + } + var got []protocol.ChatEventFrame + if err := chat.Stream(context.Background(), "ses_1", nil, func(frame protocol.ChatEventFrame) error { got = append(got, frame); return nil }); err != nil { + t.Fatal(err) + } + if len(got) != 1 || got[0].Shell == nil || got[0].Shell.ID != "sh_1" || got[0].Shell.Status != protocol.ChatShellExited { + t.Fatalf("exit after readback = %+v", got) + } +} + +func TestChatReleaseShellsBoundsDisconnectedState(t *testing.T) { + chat := NewChat(&Connection{}) + state := chat.streamState("ses_1") + state.markShell("sh_1") + chat.ReleaseShells("ses_1") + chat.shellMu.Lock() + _, retained := chat.shells["ses_1"] + chat.shellMu.Unlock() + if retained { + t.Fatal("released session shell state was retained") + } +} diff --git a/internal/server/chat.go b/internal/server/chat.go index 1601fbfc..1a26dfa6 100644 --- a/internal/server/chat.go +++ b/internal/server/chat.go @@ -32,9 +32,21 @@ type chatBroker interface { // Permissions lists the session's pending OpenCode approval requests so the // handler can push chat_permission frames. Permissions(ctx context.Context, sessionID string) ([]protocol.ChatPermissionFrame, error) + // Shells returns only shell operations whose ownership is authoritatively + // verified for sessionID. It is the reconnect/readback source. + Shells(ctx context.Context, sessionID string) ([]protocol.ChatShell, error) Reply(ctx context.Context, request protocol.ChatReplyRequest) error } +// chatShellReleaser is implemented by brokers that retain verified shell ids for +// attributing exit events. The hub calls it only after the last observer leaves +// a room; a later attach obtains a fresh authoritative snapshot. +// This is a bounded local-cache operation: it must not perform I/O or call the +// hub, which serializes it with room replacement under its mutex. +type chatShellReleaser interface { + ReleaseShells(sessionID string) +} + // operatorRegistry is the server-side operator-session seam. *pipeline.Store // satisfies it. It is a table of its own, separate from the pipeline stage // ownership (SessionOwner/run_session_owners) that settles a run: chat_sessions diff --git a/internal/server/chat_hub.go b/internal/server/chat_hub.go index 134e3fd6..9bd7e32e 100644 --- a/internal/server/chat_hub.go +++ b/internal/server/chat_hub.go @@ -133,6 +133,13 @@ func (h *chatHub) apply(frame protocol.ChatEventFrame) { } h.mu.Lock() defer h.mu.Unlock() + h.applyLocked(frame) +} + +func (h *chatHub) applyLocked(frame protocol.ChatEventFrame) { + if frame.SessionID == "" { + return + } turn := h.turnLocked(frame.SessionID) switch frame.Kind { case protocol.ChatEventStep: @@ -144,6 +151,9 @@ func (h *chatHub) apply(frame protocol.ChatEventFrame) { case protocol.ChatEventShell: // A shell frame carries no message id (it outlives the tool call that // started it), so it is folded before the message-id gate below. + if room := h.rooms[frame.SessionID]; room != nil { + room.shellReadback.Record(frame.Shell) + } turn.applyShell(frame.Shell) return } @@ -240,7 +250,7 @@ func (t *chatTurn) applyShell(shell *protocol.ChatShell) { if shell == nil || shell.ID == "" { return } - if shell.Status == protocol.ChatShellExited { + if shell.Status != protocol.ChatShellRunning { old, ok := t.shells[shell.ID] if !ok { return diff --git a/internal/server/chat_multiclient_test.go b/internal/server/chat_multiclient_test.go index 43d373a5..d6b3dde6 100644 --- a/internal/server/chat_multiclient_test.go +++ b/internal/server/chat_multiclient_test.go @@ -82,6 +82,8 @@ func (c *channelChat) Session(context.Context, string) (protocol.ChatSession, bo return protocol.ChatSession{}, false, nil } +func (c *channelChat) Shells(context.Context, string) ([]protocol.ChatShell, error) { return nil, nil } + func (c *channelChat) Permissions(context.Context, string) ([]protocol.ChatPermissionFrame, error) { c.mu.Lock() defer c.mu.Unlock() diff --git a/internal/server/chat_room.go b/internal/server/chat_room.go index 9ae2450c..a8d0ecc5 100644 --- a/internal/server/chat_room.go +++ b/internal/server/chat_room.go @@ -106,6 +106,11 @@ type chatQueuedPrompt struct { acked bool } +type chatObservation struct { + done chan struct{} + err error // published by closing done; belongs to this attempt, not the next +} + // chatRoom is one session's shared live channel. Every field is guarded by the // owning chatHub's mutex. type chatRoom struct { @@ -117,6 +122,11 @@ type chatRoom struct { queue []*chatQueuedPrompt running bool streaming bool + // observing is non-nil while one caller is attaching the shared broker + // stream. It prevents concurrent subscribers from opening duplicate global + // event streams before streaming can be published. + observing *chatObservation + shellReadback *opencode.ShellSnapshot // buffering holds the turn's frames until the prompting client's ack has been // written, so the ack always precedes the frames it acknowledges. It is set // only while the very first turn subscribes to the firehose. @@ -160,6 +170,81 @@ func (h *chatHub) roomLocked(sessionID string) *chatRoom { return room } +// ensureObserving starts the firehose even for a read-only subscriber, then +// reconciles retained shells from the endpoint's authoritative list. A bounded +// journal folds concurrent lifecycle events over that snapshot atomically. All +// observers share this reconciliation, with no broker I/O under the hub mutex. +func (h *chatHub) ensureObserving(room *chatRoom, broker chatBroker) error { + if broker == nil { + return errors.New("chat is not configured on this server") + } + h.mu.Lock() + if err := room.ctx.Err(); err != nil { + h.mu.Unlock() + return err + } + if waiting := room.observing; waiting != nil { + h.mu.Unlock() + select { + case <-waiting.done: + return waiting.err + case <-room.ctx.Done(): + return room.ctx.Err() + } + } + if room.streaming { + h.mu.Unlock() + return nil + } + room.observing = &chatObservation{done: make(chan struct{})} + waiting := room.observing + room.shellReadback = &opencode.ShellSnapshot{} + h.mu.Unlock() + var observeErr error + defer func() { + h.mu.Lock() + room.shellReadback = nil + waiting.err = observeErr + room.observing = nil + close(waiting.done) + h.mu.Unlock() + }() + if err := h.startStream(room, broker); err != nil { + observeErr = err + return err + } + shells, err := broker.Shells(room.ctx, room.sessionID) + h.mu.Lock() + if room.ctx.Err() != nil || h.rooms[room.sessionID] != room { + h.mu.Unlock() + observeErr = context.Canceled + return observeErr + } + if err == nil { + shells, err = room.shellReadback.Merge(shells) + } + if err != nil { + // The firehose remains useful when its authoritative shell readback is + // temporarily unavailable. Do not turn that read failure into a chat + // failure (or claim the last retained snapshot is current): clear the + // replay cache and tell attached clients that shell visibility is limited. + // Preserve events already folded during this read, but clear stale replay. + live, mergeErr := room.shellReadback.Merge(nil) + if mergeErr == nil { + h.replaceShellsLocked(room.sessionID, live) + } + h.mu.Unlock() + // Never log broker bodies/URLs: they may contain credentials or private output. + log.Printf("chat shell readback session=%s failed (%T)", room.sessionID, err) + h.fanout(room.sessionID, protocol.Message{Type: protocol.Error, Error: "background shell visibility is limited"}, "") + return nil + } + h.replaceShellsLocked(room.sessionID, shells) + room.shellReadback = nil + h.mu.Unlock() + return nil +} + // attach registers write as one client's subscription to sessionID's stream and // returns it, or the existing subscription when the same connection has already // attached. It is idempotent so a client may prompt and subscribe without @@ -209,6 +294,11 @@ func (h *chatHub) detach(sessionID string, sub *chatSub) { if len(room.subs) == 0 { delete(h.rooms, sessionID) room.cancel() + // ReleaseShells is a local identity-cache operation (no broker I/O). + // Complete it before a reconnect can acquire this session's new room. + if releaser, ok := room.broker.(chatShellReleaser); ok { + releaser.ReleaseShells(sessionID) + } } h.mu.Unlock() } @@ -265,9 +355,17 @@ func (h *chatHub) broadcastUser(frame protocol.ChatEventFrame, exceptClientID st // once it is written. A step-ended frame finishes the turn, letting the queue // advance. func (h *chatHub) emit(frame protocol.ChatEventFrame) { - h.apply(frame) + h.emitRoom(nil, frame) +} + +func (h *chatHub) emitRoom(source *chatRoom, frame protocol.ChatEventFrame) { h.mu.Lock() room := h.rooms[frame.SessionID] + if source != nil && (source != room || source.ctx.Err() != nil) { + h.mu.Unlock() + return + } + h.applyLocked(frame) buffered := false var targets []*chatSub if room != nil { @@ -283,6 +381,22 @@ func (h *chatHub) emit(frame protocol.ChatEventFrame) { if room != nil && frame.Kind == protocol.ChatEventStep && frame.Step != nil && frame.Step.Status == protocol.ChatStepEnded { h.finishTurnLocked(room) } + // Shell replay and live lifecycle delivery must enqueue in the same order + // as folding. Otherwise an exit can overtake an older running replay. + if frame.Kind == protocol.ChatEventShell && !buffered { + msg := protocol.Message{Type: protocol.ChatEvent, ChatEvent: &frame} + var slow []*chatSub + for _, sub := range targets { + if !enqueueChatLocked(sub, msg) { + slow = append(slow, sub) + } + } + h.mu.Unlock() + for _, sub := range slow { + sub.close() + } + return + } h.mu.Unlock() if buffered { return @@ -293,6 +407,62 @@ func (h *chatHub) emit(frame protocol.ChatEventFrame) { } } +// enqueueChatLocked never blocks or calls detach under the hub mutex. +func enqueueChatLocked(sub *chatSub, msg protocol.Message) bool { + select { + case <-sub.done: + return true + default: + } + select { + case sub.ch <- msg: + return true + default: + return false + } +} + +func (h *chatHub) replayShells(sub *chatSub) { + h.mu.Lock() + slow := false + if h.rooms[sub.room.sessionID] == sub.room && sub.room.subs[sub.clientID] == sub { + turn := h.turnLocked(sub.room.sessionID) + for _, id := range turn.shellOrder { + shell := turn.shells[id] + frame := protocol.ChatEventFrame{SessionID: sub.room.sessionID, Kind: protocol.ChatEventShell, Shell: &shell} + if !enqueueChatLocked(sub, protocol.Message{Type: protocol.ChatEvent, ChatEvent: &frame}) { + slow = true + break + } + } + } + h.mu.Unlock() + if slow { + sub.close() + } +} + +// replaceShells reconciles a successful authoritative readback. It drops stale +// retained entries (including exits missed while disconnected) and upserts the +// verified running set without fabricating a lifecycle event for foreign shells. +func (h *chatHub) replaceShells(sessionID string, shells []protocol.ChatShell) { + h.mu.Lock() + defer h.mu.Unlock() + h.replaceShellsLocked(sessionID, shells) +} + +func (h *chatHub) replaceShellsLocked(sessionID string, shells []protocol.ChatShell) { + turn := h.turnLocked(sessionID) + for _, shell := range turn.shells { + turn.shrink(len(shell.Command)) + } + turn.shells = nil + turn.shellOrder = nil + for i := range shells { + turn.applyShell(&shells[i]) + } +} + // finishTurnLocked marks the room's turn finished and releases the turn runner's // wait. Callers must hold h.mu. func (h *chatHub) finishTurnLocked(room *chatRoom) { @@ -331,14 +501,20 @@ func (h *chatHub) flushBuffered(sessionID string) { targets = append(targets, sub) } } - h.mu.Unlock() + var slow []*chatSub for i := range pending { frame := pending[i] msg := protocol.Message{Type: protocol.ChatEvent, ChatEvent: &frame} for _, sub := range targets { - sub.send(msg) + if !enqueueChatLocked(sub, msg) { + slow = append(slow, sub) + } } } + h.mu.Unlock() + for _, sub := range slow { + sub.close() + } } // submitPrompt accepts one prompt: it starts the turn runner immediately when no @@ -476,7 +652,7 @@ func (h *chatHub) runTurn(room *chatRoom, p *chatQueuedPrompt, done chan struct{ return } if first { - if err := h.startStream(room, broker); err != nil { + if err := h.ensureObserving(room, broker); err != nil { h.failPrompt(room, p, err) h.endRoomTurn(room) return @@ -550,7 +726,7 @@ func (h *chatHub) startStream(room *chatRoom, broker chatBroker) error { result <- broker.Stream(ctx, room.sessionID, func() { once.Do(func() { close(attached) }) }, func(frame protocol.ChatEventFrame) error { - h.emit(frame) + h.emitRoom(room, frame) return nil }) }() diff --git a/internal/server/chat_shell_reconcile_test.go b/internal/server/chat_shell_reconcile_test.go new file mode 100644 index 00000000..9ada615b --- /dev/null +++ b/internal/server/chat_shell_reconcile_test.go @@ -0,0 +1,261 @@ +package server + +import ( + "context" + "net/http/httptest" + "strings" + "sync/atomic" + "testing" + "time" + + "git.a-wan.net/alan/pipelines/internal/protocol" + "github.com/coder/websocket" + "github.com/coder/websocket/wsjson" +) + +type shellBarrierBroker struct { + *fakeChat + read chan struct{} + apply chan struct{} + events chan shellBarrierEvent + stopped chan struct{} + reads atomic.Int32 + snapshot []protocol.ChatShell +} + +// Exercise the real websocket handler on a loopback scratch server and a fresh +// SQLite database. A marker on the shared stream bounds the replay assertion +// without sleeps, polling or a negative-read timeout. +func TestChatShellReconciliationScratchServer(t *testing.T) { + b := newShellBarrierBroker() + b.snapshot = []protocol.ChatShell{{ID: "sh_old", Status: protocol.ChatShellRunning}} + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + srv := httptest.NewServer(HandlerWithOptions(ctx, Options{Chat: b}, newOperatorStore(t))) + defer srv.Close() + conn, _, err := websocket.Dial(ctx, "ws"+strings.TrimPrefix(srv.URL, "http")+"/connect", nil) + if err != nil { + t.Fatal(err) + } + defer conn.CloseNow() + subscribe := func(id string) { + t.Helper() + if err := wsjson.Write(ctx, conn, protocol.Message{Type: protocol.ChatSubscribe, ID: id, ChatSubscribe: &protocol.ChatQuery{SessionID: "ses_1"}}); err != nil { + t.Fatal(err) + } + } + read := func() protocol.Message { + t.Helper() + var m protocol.Message + if err := wsjson.Read(ctx, conn, &m); err != nil { + t.Fatal(err) + } + return m + } + if m := read(); m.Type != protocol.PipelineList { + t.Fatalf("server greeting = %s", m.Type) + } + subscribe("first") + if m := read(); m.ID != "first" || m.Type != protocol.ChatSubscribe { + t.Fatalf("ack = %+v", m) + } + shellBarrierWait(t, b.read) + done := make(chan struct{}) + b.events <- shellBarrierEvent{protocol.ChatShell{ID: "sh_old", Status: protocol.ChatShellExited}, done} + shellBarrierWait(t, done) + if m := read(); m.ChatEvent == nil || m.ChatEvent.Shell.Status != protocol.ChatShellExited { + t.Fatalf("exit not visible: %+v", m) + } + close(b.apply) + subscribe("second") + for { + m := read() + if m.ChatEvent != nil && m.ChatEvent.Shell.Status == protocol.ChatShellRunning { + t.Fatalf("stale replay: %+v", m) + } + if m.ID == "second" { + break + } + } + done = make(chan struct{}) + b.events <- shellBarrierEvent{protocol.ChatShell{ID: "sh_marker", Status: protocol.ChatShellExited}, done} + shellBarrierWait(t, done) + for { + m := read() + if m.ChatEvent == nil { + continue + } + if m.ChatEvent.Shell.Status == protocol.ChatShellRunning { + t.Fatalf("ghost visible after apply: %+v", m) + } + if m.ChatEvent.Shell.ID == "sh_marker" { + break + } + } + if b.reads.Load() != 1 { + t.Fatalf("shared observer read count = %d", b.reads.Load()) + } + conn.CloseNow() + shellBarrierWait(t, b.stopped) +} + +type shellBarrierEvent struct { + shell protocol.ChatShell + done chan struct{} +} + +func newShellBarrierBroker() *shellBarrierBroker { + return &shellBarrierBroker{fakeChat: scriptedChat(), read: make(chan struct{}, 8), apply: make(chan struct{}), events: make(chan shellBarrierEvent), stopped: make(chan struct{})} +} + +func (b *shellBarrierBroker) Shells(ctx context.Context, _ string) ([]protocol.ChatShell, error) { + b.reads.Add(1) + snapshot := append([]protocol.ChatShell(nil), b.snapshot...) + b.read <- struct{}{} + select { + case <-b.apply: + return snapshot, nil + case <-ctx.Done(): + return nil, ctx.Err() + } +} + +func (b *shellBarrierBroker) Stream(ctx context.Context, id string, ready func(), emit func(protocol.ChatEventFrame) error) error { + defer close(b.stopped) + ready() + for { + select { + case <-ctx.Done(): + return ctx.Err() + case event := <-b.events: + if err := emit(protocol.ChatEventFrame{SessionID: id, Kind: protocol.ChatEventShell, Shell: &event.shell}); err != nil { + return err + } + close(event.done) + } + } +} + +func shellBarrierWait(t *testing.T, ch <-chan struct{}) { + t.Helper() + select { + case <-ch: + case <-time.After(3 * time.Second): + t.Fatal("barrier timed out") + } +} + +func TestChatShellReconciliationBarrier(t *testing.T) { + for _, status := range []string{protocol.ChatShellExited, protocol.ChatShellRunning, "timeout", "killed"} { + t.Run(status, func(t *testing.T) { + b := newShellBarrierBroker() + b.snapshot = []protocol.ChatShell{{ID: "sh_old", Status: protocol.ChatShellRunning}} + h := newChatHub() + messages := make(chan protocol.Message, 32) + sub := h.attach("ses_1", "one", "", b, func(m protocol.Message) error { messages <- m; return nil }) + t.Cleanup(sub.close) + result := make(chan error, 2) + go func() { result <- h.ensureObserving(sub.room, b) }() + shellBarrierWait(t, b.read) + // A second observer must share the in-flight read, not apply an + // independently completing older/newer snapshot out of order. + go func() { result <- h.ensureObserving(sub.room, b) }() + shell := protocol.ChatShell{ID: "sh_old", Status: status} + if status == protocol.ChatShellRunning { + shell.ID = "sh_new" + shell.Command = "sleep 10" + } + done := make(chan struct{}) + select { + case b.events <- shellBarrierEvent{shell, done}: + case <-time.After(time.Second): + t.Fatal("event blocked behind readback") + } + shellBarrierWait(t, done) + select { + case m := <-messages: + if m.ChatEvent == nil || m.ChatEvent.Shell == nil || m.ChatEvent.Shell.ID != shell.ID || m.ChatEvent.Shell.Status != status { + t.Fatalf("live shell = %+v", m) + } + case <-time.After(time.Second): + t.Fatal("live event not visible during read") + } + close(b.apply) + for range 2 { + select { + case err := <-result: + if err != nil { + t.Fatal(err) + } + case <-time.After(time.Second): + t.Fatal("observer stuck") + } + } + if b.reads.Load() != 1 { + t.Fatalf("redundant/out-of-order reads: %d", b.reads.Load()) + } + snapshot := h.shellSnapshot("ses_1") + if status == protocol.ChatShellRunning { + if len(snapshot) != 2 { + t.Fatalf("new create lost: %+v", snapshot) + } + } else if len(snapshot) != 0 { + t.Fatalf("terminal shell resurrected: %+v", snapshot) + } + // Assert the same retained state is replayed to the client. + h.replayShells(sub) + for range len(snapshot) { + select { + case m := <-messages: + if m.ChatEvent == nil || m.ChatEvent.Shell.Status != protocol.ChatShellRunning { + t.Fatalf("replay = %+v", m) + } + case <-time.After(time.Second): + t.Fatal("replay missing") + } + } + sub.close() + shellBarrierWait(t, b.stopped) + }) + } +} + +func TestChatShellReconciliationCancelReconnectBarrier(t *testing.T) { + b := newShellBarrierBroker() + h := newChatHub() + sub := h.attach("ses_1", "one", "", b, func(protocol.Message) error { return nil }) + results := make(chan error, 2) + go func() { results <- h.ensureObserving(sub.room, b) }() + shellBarrierWait(t, b.read) + go func() { results <- h.ensureObserving(sub.room, b) }() + sub.close() + for range 2 { + select { + case err := <-results: + if err == nil { + t.Fatal("canceled observer succeeded") + } + case <-time.After(time.Second): + t.Fatal("canceled waiter stuck") + } + } + shellBarrierWait(t, b.stopped) + next := newShellBarrierBroker() + next.snapshot = []protocol.ChatShell{{ID: "sh_new", Status: protocol.ChatShellRunning}} + close(next.apply) + newSub := h.attach("ses_1", "two", "", next, func(protocol.Message) error { return nil }) + t.Cleanup(newSub.close) + if err := h.ensureObserving(newSub.room, next); err != nil { + t.Fatal(err) + } + // A late callback from the canceled stream cannot poison the new room, + // nor can a foreign-session callback from the current stream. + h.emitRoom(sub.room, protocol.ChatEventFrame{SessionID: "ses_1", Kind: protocol.ChatEventShell, Shell: &protocol.ChatShell{ID: "sh_ghost", Status: protocol.ChatShellRunning}}) + h.emitRoom(newSub.room, protocol.ChatEventFrame{SessionID: "ses_foreign", Kind: protocol.ChatEventShell, Shell: &protocol.ChatShell{ID: "sh_foreign", Status: protocol.ChatShellRunning}}) + got := h.shellSnapshot("ses_1") + if len(got) != 1 || got[0].ID != "sh_new" || len(h.shellSnapshot("ses_foreign")) != 0 { + t.Fatalf("reconnect state = %+v", got) + } + newSub.close() + shellBarrierWait(t, next.stopped) +} diff --git a/internal/server/chat_stream.go b/internal/server/chat_stream.go index 76c175e6..89ba6404 100644 --- a/internal/server/chat_stream.go +++ b/internal/server/chat_stream.go @@ -111,7 +111,14 @@ func handleChatSubscribe(request protocol.Message, broker chatBroker, hub *chatH _ = write(protocol.Message{Type: protocol.Error, ID: request.ID, Error: "could not attach to session " + query.SessionID}) return } + // Acknowledge before the stream is allowed to fan out: otherwise an immediately + // available retained shell can overtake the subscribe reply on this connection. _ = write(protocol.Message{Type: protocol.ChatSubscribe, ID: request.ID}) + if err := hub.ensureObserving(sub.room, broker); err != nil { + sub.close() + _ = write(protocol.Message{Type: protocol.Error, ID: request.ID, Error: "could not observe session " + query.SessionID}) + return + } // Replay every still-unresolved approval request the room holds, not only the // most recent one, so a client that attaches after two requests were raised // still sees both cards. @@ -124,9 +131,5 @@ func handleChatSubscribe(request protocol.Message, broker chatBroker, hub *chatH // shows them rather than nothing until they exit. The message snapshot cannot // carry a shell — a shell outlives the turn and is not a turn — so this is the // shell half of the resume. - for _, shell := range hub.shellSnapshot(query.SessionID) { - running := shell - frame := protocol.ChatEventFrame{SessionID: query.SessionID, Kind: protocol.ChatEventShell, Shell: &running} - sub.send(protocol.Message{Type: protocol.ChatEvent, ChatEvent: &frame}) - } + hub.replayShells(sub) } diff --git a/internal/server/chat_test.go b/internal/server/chat_test.go index 29dbc4bb..d4a15cd2 100644 --- a/internal/server/chat_test.go +++ b/internal/server/chat_test.go @@ -3,6 +3,7 @@ package server import ( "context" "encoding/json" + "errors" "fmt" "net/http" "net/http/httptest" @@ -52,6 +53,8 @@ type fakeChat struct { permissionSessionID string permissions []protocol.ChatPermissionFrame permissionErr error + shells []protocol.ChatShell + shellErr error sessionCalls int sessions []protocol.ChatSession sessionErr error @@ -62,6 +65,8 @@ type fakeChat struct { started chan struct{} done chan struct{} startOnce sync.Once + streamCalls int + readyGate <-chan struct{} block bool subscribed bool promptBeforeSubscribe bool @@ -91,13 +96,18 @@ func (f *fakeChat) Prompt(context.Context, string, string) error { func (f *fakeChat) Stream(ctx context.Context, _ string, ready func(), emit func(protocol.ChatEventFrame) error) error { f.mu.Lock() f.subscribed = true + f.streamCalls++ + readyGate := f.readyGate f.mu.Unlock() - if ready != nil { - ready() - } if f.started != nil { f.startOnce.Do(func() { close(f.started) }) } + if readyGate != nil { + <-readyGate + } + if ready != nil { + ready() + } if f.block { <-ctx.Done() if f.done != nil { @@ -116,6 +126,42 @@ func (f *fakeChat) Stream(ctx context.Context, _ string, ready func(), emit func return nil } +// TestChatHubConcurrentObserversShareStream pins the start-up handoff: two +// clients that subscribe before the broker says ready still share one firehose. +func TestChatHubConcurrentObserversShareStream(t *testing.T) { + broker := scriptedChat() + broker.block = true + broker.started = make(chan struct{}) + gate := make(chan struct{}) + broker.readyGate = gate + hub := newChatHub() + hub.mu.Lock() + room := hub.roomLocked("ses_1") + hub.mu.Unlock() + + errs := make(chan error, 2) + go func() { errs <- hub.ensureObserving(room, broker) }() + select { + case <-broker.started: + case <-time.After(time.Second): + t.Fatal("first observer did not start the broker stream") + } + go func() { errs <- hub.ensureObserving(room, broker) }() + close(gate) + for range 2 { + if err := <-errs; err != nil { + t.Fatalf("ensureObserving: %v", err) + } + } + broker.mu.Lock() + calls := broker.streamCalls + broker.mu.Unlock() + if calls != 1 { + t.Fatalf("broker Stream calls = %d, want 1", calls) + } + room.cancel() +} + func (f *fakeChat) Interrupt(context.Context, string) error { f.mu.Lock() defer f.mu.Unlock() @@ -151,6 +197,12 @@ func (f *fakeChat) Reply(_ context.Context, request protocol.ChatReplyRequest) e // Permissions returns the seeded pending requests, or blocks until the caller's // context is canceled when permBlock is set (used to prove the poller stops on // disconnect). Every call is counted so a test can assert polling happened. +func (f *fakeChat) Shells(context.Context, string) ([]protocol.ChatShell, error) { + f.mu.Lock() + defer f.mu.Unlock() + return append([]protocol.ChatShell(nil), f.shells...), f.shellErr +} + func (f *fakeChat) Permissions(ctx context.Context, sessionID string) ([]protocol.ChatPermissionFrame, error) { f.mu.Lock() f.permissionCalls++ @@ -682,6 +734,9 @@ func TestChatInterruptEndsRunningPrompt(t *testing.T) { case r.Method == http.MethodGet && r.URL.Path == "/api/permission/request": w.Header().Set("Content-Type", "application/json") _, _ = w.Write([]byte(`{"location":{"directory":"/work"},"data":[]}`)) + case r.Method == http.MethodGet && r.URL.Path == "/api/shell": + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"data":[]}`)) case r.Method == http.MethodGet && r.URL.Path == "/api/event": flusher, ok := w.(http.Flusher) if !ok { @@ -995,6 +1050,7 @@ func TestChatPromptStreamsShellFrame(t *testing.T) { // shows the background process instead of nothing until it exits. func TestChatSubscribeReplaysRunningShell(t *testing.T) { broker := scriptedChat() + broker.shells = []protocol.ChatShell{{ID: "sh_1", Status: protocol.ChatShellRunning, Command: "sleep 120"}} broker.script = append([]protocol.ChatEventFrame{ {SessionID: "ses_1", Kind: protocol.ChatEventShell, Shell: &protocol.ChatShell{ID: "sh_1", Status: protocol.ChatShellRunning, Command: "sleep 120"}}, }, broker.script...) @@ -1039,6 +1095,40 @@ func TestChatSubscribeReplaysRunningShell(t *testing.T) { } } +// TestChatSubscribeKeepsStreamWhenShellReadbackFails ensures a temporary +// authoritative readback failure does not make chat unavailable or replay a +// stale process as running. The subscriber receives an explicit limitation +// while its broker stream remains attached for future lifecycle events. +func TestChatSubscribeKeepsStreamWhenShellReadbackFails(t *testing.T) { + broker := scriptedChat() + broker.block = true + broker.shellErr = errors.New("shell endpoint unavailable: https://user:secret@example.invalid private response body") + ctx, _, conn := dialChat(t, broker) + if err := wsjson.Write(ctx, conn, protocol.Message{Type: protocol.ChatSubscribe, ID: "sub1", ChatSubscribe: &protocol.ChatQuery{SessionID: "ses_1"}}); err != nil { + t.Fatal(err) + } + var ack protocol.Message + if err := wsjson.Read(ctx, conn, &ack); err != nil { + t.Fatal(err) + } + if ack.Type != protocol.ChatSubscribe || ack.ID != "sub1" { + t.Fatalf("subscribe ack = %+v", ack) + } + var limited protocol.Message + if err := wsjson.Read(ctx, conn, &limited); err != nil { + t.Fatal(err) + } + if limited.Type != protocol.Error || limited.Error != "background shell visibility is limited" { + t.Fatalf("limited visibility notice = %+v", limited) + } + broker.mu.Lock() + calls := broker.streamCalls + broker.mu.Unlock() + if calls != 1 { + t.Fatalf("broker Stream calls = %d, want 1", calls) + } +} + // TestChatPromptSubscribesBeforePrompt pins the subscribe-before-prompt order: // the firehose has no replay cursor, so the subscription must be attached before // the prompt is posted or deltas emitted in between are lost. @@ -1079,6 +1169,9 @@ func realAdapterChat(t *testing.T, frames []string) []protocol.ChatEventFrame { case r.Method == http.MethodGet && r.URL.Path == "/api/permission/request": w.Header().Set("Content-Type", "application/json") _, _ = w.Write([]byte(`{"location":{"directory":"/work"},"data":[]}`)) + case r.Method == http.MethodGet && r.URL.Path == "/api/shell": + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"data":[]}`)) case r.Method == http.MethodGet && r.URL.Path == "/api/event": streams.Add(1) w.Header().Set("Content-Type", "text/event-stream") From 2cdb9a9e9300a12d1ef65e8bc6b5f9cb5431536e Mon Sep 17 00:00:00 2001 From: Alan Silva Date: Sun, 18 Oct 2026 14:50:18 +0100 Subject: [PATCH 7/7] Stabilize guarded model delivery Add validated OpenCode-only delivery capabilities, canonical linked-worktree access checks, isolated candidate verification, and guarded main settlement. Wire the bounded reconnect-foreground controlled HTTP fixture through ordinary server construction so target verification obtains server-controlled provenance and rejects unknown or incompatible fixture evidence. Checked packaged templates and packaged authoring skill references; no updates are needed because this draft-only infrastructure activates no definition. Recorded the stale reference and activation guidance in docs/notes/delivery-stabilization.md for the docs fold. --- cmd/pipelines-server/main.go | 2 +- docs/notes/delivery-stabilization.md | 47 +++ internal/server/delivery_gate.go | 185 +++++++++++ internal/server/landing_access.go | 130 ++++++++ internal/server/landing_access_test.go | 304 +++++++++++++++++++ internal/server/runner.go | 61 +++- internal/server/verification_fixture.go | 135 ++++++++ internal/server/verification_fixture_test.go | 148 +++++++++ internal/server/websocket.go | 8 + internal/stage/agent.go | 1 + internal/stage/model.go | 16 +- internal/stage/tools.go | 75 +++++ internal/stage/tools_test.go | 104 +++++++ internal/stage/verification.go | 97 ++++++ internal/stage/verification_test.go | 27 ++ 15 files changed, 1335 insertions(+), 5 deletions(-) create mode 100644 docs/notes/delivery-stabilization.md create mode 100644 internal/server/delivery_gate.go create mode 100644 internal/server/landing_access.go create mode 100644 internal/server/landing_access_test.go create mode 100644 internal/server/verification_fixture.go create mode 100644 internal/server/verification_fixture_test.go create mode 100644 internal/stage/verification.go create mode 100644 internal/stage/verification_test.go diff --git a/cmd/pipelines-server/main.go b/cmd/pipelines-server/main.go index ec789886..54415ddf 100644 --- a/cmd/pipelines-server/main.go +++ b/cmd/pipelines-server/main.go @@ -47,7 +47,7 @@ func main() { // Each new run gets a bounded one-line title from the configured default // harness; the summariser falls back to the prompt's first line, so title // generation never blocks or fails run creation. - options := server.Options{ArtifactsRoot: artifactsRoot, RunTitles: summarise.GenerateTitle, StepVerdicts: summarise.GenerateVerdictReason} + options := server.Options{ArtifactsRoot: artifactsRoot, RunTitles: summarise.GenerateTitle, StepVerdicts: summarise.GenerateVerdictReason, VerificationFixtures: server.VerificationFixturesFromEnvironment(os.Getenv)} recorded := "" if value, err := store.OpenCodeEndpoint(ctx); err != nil { log.Printf("OpenCode endpoint setting unreadable: %v", err) diff --git a/docs/notes/delivery-stabilization.md b/docs/notes/delivery-stabilization.md new file mode 100644 index 00000000..bba21fd9 --- /dev/null +++ b/docs/notes/delivery-stabilization.md @@ -0,0 +1,47 @@ +# Delivery stabilization + +Date: 2026-10-18. + +The server now has draft-only `delivery=true` support for OpenCode model stages. +At run time it derives the canonical `main` worktree and linked-worktree Git +metadata from Git rather than accepting paths from a stage prompt. It verifies +the committed item tree in a disposable candidate worktree before fast-forwarding +canonical `main`; declared checks are bounded configuration, not proof that any +deployed definition has used them. It is denied for Pi (until equivalent +capability enforcement exists), foreign repositories, missing main worktrees, +and malformed capabilities. This item does not publish or deploy a definition. + +The source also supports an optional controlled-fixture verification declaration +(`backend`, `revision`, `capability`, and `fixture`). Ordinary server setup wires +an explicit, bounded `reconnect-foreground` HTTP fixture registry when +`PIPELINES_VERIFICATION_RECONNECT_FOREGROUND_URL` and its expected `BACKEND`, +`REVISION`, `CAPABILITY`, and `STATE` values are configured. There is no default +remote URL. The server, not the model, probes that registered endpoint and +collects its returned backend identity, build revision, capability, fixture ID, +and state before comparing provenance. Unknown fixtures, incomplete setup, +wrong backend/revision/capability/state, non-200 responses, cancellation, and +oversized or malformed responses fail closed. A model result's labels are not +evidence. This is controlled local/backend fixture infrastructure, not evidence +of a rendered layout or a live Android device. +Definitions must be scratch-validated against the deployed server only after +the server support is deployed; no future delivery or verification definition +is activated by this source change. + +`internal/author/skill/references/format.md` and +`internal/author/skill/references/queue.md` are stale: a later docs fold should +describe `delivery=true`, OpenCode-only availability, candidate verification, +and the required activation order (deploy server support before scratch-validating +and then publishing a definition). +OpenCode V2 resolves external resources at a containing directory, so delivery +uses the verified repository metadata directory rather than an impossible +single-lock-file scope. It does not grant a parent, home, foreign repository, +or global policy. Unrestricted shell permissions are not an OS sandbox. + +Activation remains draft-only: deploy compatible server support, configure the +controlled fixture endpoint and matching provenance, scratch-validate the +fixture, then publish a definition. No definition is activated by this source +change. + +Recovery checklist: retain `connection-reconnect` and unfinished feature +branches; merge only complete committed trees that pass mapped checks; do not +infer delivery from a dirty worktree or an old deployed-backend capture. diff --git a/internal/server/delivery_gate.go b/internal/server/delivery_gate.go new file mode 100644 index 00000000..df1dc7c5 --- /dev/null +++ b/internal/server/delivery_gate.go @@ -0,0 +1,185 @@ +package server + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "strings" + "syscall" + "time" + + "git.a-wan.net/alan/pipelines/internal/stage" +) + +// verifyDelivery proves that the stage landed a complete, committed tree before +// its success can settle the run. It re-derives the capability after the stage: +// a stage result and its input cannot select either the landing path or checks. +const maxDeliveryCheckOutput = 64 << 10 +const deliveryCleanupTimeout = 10 * time.Second + +// verifyDelivery verifies the committed item tree in a disposable worktree +// before fast-forwarding canonical main. Settlement deliberately performs only +// non-mutating repository checks after that mutation: a declared check must +// never be the first signal that the landed tree is known-red. +func verifyDelivery(ctx context.Context, workspace string, access stage.LandingAccess, checks []string) error { + lock, err := acquireDeliveryLock(ctx, access.LockPath) + if err != nil { + return err + } + defer lock.Close() + current, err := landingAccess(workspace) + if err != nil { + return fmt.Errorf("delivery access changed: %w", err) + } + if current != access { + return fmt.Errorf("delivery access changed during stage") + } + for _, directory := range []string{workspace, access.MainWorktree} { + status, err := gitOutput(directory, "status", "--porcelain=v1", "--untracked-files=all") + if err != nil { + return fmt.Errorf("delivery checkout status: %w", err) + } + if status != "" { + return fmt.Errorf("delivery checkout is not clean: %s", strings.Split(status, "\n")[0]) + } + } + itemHead, err := gitOutput(workspace, "rev-parse", "HEAD") + if err != nil { + return fmt.Errorf("delivery item commit: %w", err) + } + if err := runDeliveryCandidate(ctx, workspace, itemHead, checks); err != nil { + return err + } + mainHead, err := gitOutput(access.MainWorktree, "rev-parse", "HEAD") + if err != nil { + return fmt.Errorf("delivery main commit: %w", err) + } + if err := exec.CommandContext(ctx, "git", "-C", workspace, "merge-base", "--is-ancestor", mainHead, itemHead).Run(); err != nil { + return fmt.Errorf("delivery item commit cannot fast-forward canonical main") + } + if output, err := exec.CommandContext(ctx, "git", "-C", access.MainWorktree, "merge", "--ff-only", itemHead).CombinedOutput(); err != nil { + return fmt.Errorf("delivery fast-forward canonical main: %w: %s", err, strings.TrimSpace(string(output))) + } + for _, directory := range []string{workspace, access.MainWorktree} { + status, err := gitOutput(directory, "status", "--porcelain=v1", "--untracked-files=all") + if err != nil { + return fmt.Errorf("delivery settlement status: %w", err) + } + if status != "" { + return fmt.Errorf("delivery checkout is not clean after checks: %s", strings.Split(status, "\n")[0]) + } + } + // Re-derive at the settlement boundary. A success means exactly the checked + // item commit is now reachable from the same canonical main worktree. + current, err = landingAccess(workspace) + if err != nil || current != access { + return fmt.Errorf("delivery access changed during settlement") + } + mainHead, err = gitOutput(access.MainWorktree, "rev-parse", "HEAD") + if err != nil { + return fmt.Errorf("delivery main commit: %w", err) + } + if err := exec.CommandContext(ctx, "git", "-C", workspace, "merge-base", "--is-ancestor", itemHead, mainHead).Run(); err != nil { + return fmt.Errorf("delivery item commit is not an ancestor of canonical main") + } + return nil +} + +func acquireDeliveryLock(ctx context.Context, path string) (*os.File, error) { + file, err := os.OpenFile(path, os.O_CREATE|os.O_RDWR, 0600) + if err != nil { + return nil, fmt.Errorf("delivery lock: %w", err) + } + for { + err := syscall.Flock(int(file.Fd()), syscall.LOCK_EX|syscall.LOCK_NB) + if err == nil { + return file, nil + } + if !errors.Is(err, syscall.EWOULDBLOCK) && !errors.Is(err, syscall.EAGAIN) { + _ = file.Close() + return nil, fmt.Errorf("delivery lock: %w", err) + } + select { + case <-ctx.Done(): + _ = file.Close() + return nil, fmt.Errorf("delivery lock: %w", ctx.Err()) + case <-time.After(20 * time.Millisecond): + } + } +} + +func runDeliveryCandidate(ctx context.Context, workspace, itemHead string, checks []string) error { + candidate, err := os.MkdirTemp(filepath.Dir(workspace), ".pipelines-delivery-") + if err != nil { + return fmt.Errorf("create delivery candidate: %w", err) + } + if err := os.Remove(candidate); err != nil { + return fmt.Errorf("prepare delivery candidate: %w", err) + } + if output, err := boundedCommandOutput(ctx, workspace, "worktree", "add", "--detach", candidate, itemHead); err != nil { + cleanupDeliveryCandidate(workspace, candidate) + return fmt.Errorf("create delivery candidate: %w: %s", err, strings.TrimSpace(string(output))) + } + defer cleanupDeliveryCandidate(workspace, candidate) + for _, check := range checks { + if err := runDeliveryCheck(ctx, candidate, check); err != nil { + return err + } + } + return nil +} + +// cleanupDeliveryCandidate always uses its own bounded context: the stage context +// may already be canceled when a check fails or times out. +func cleanupDeliveryCandidate(workspace, candidate string) { + ctx, cancel := context.WithTimeout(context.Background(), deliveryCleanupTimeout) + defer cancel() + _ = exec.CommandContext(ctx, "git", "-C", workspace, "worktree", "remove", "--force", candidate).Run() + _ = os.RemoveAll(candidate) +} + +func boundedCommandOutput(ctx context.Context, directory string, args ...string) ([]byte, error) { + cmd := exec.CommandContext(ctx, "git", append([]string{"-C", directory}, args...)...) + var output boundedDeliveryOutput + cmd.Stdout = &output + cmd.Stderr = &output + err := cmd.Run() + return []byte(output.String()), err +} + +func runDeliveryCheck(ctx context.Context, directory, check string) error { + cmd := exec.CommandContext(ctx, "sh", "-c", check) + cmd.Dir = directory + var output boundedDeliveryOutput + cmd.Stdout = &output + cmd.Stderr = &output + if err := cmd.Run(); err != nil { + return fmt.Errorf("delivery check %q failed: %w: %s", check, err, output.String()) + } + return nil +} + +type boundedDeliveryOutput struct{ bytes.Buffer } + +func (b *boundedDeliveryOutput) Write(p []byte) (int, error) { + remaining := maxDeliveryCheckOutput - b.Len() + if remaining > 0 { + _, _ = b.Buffer.Write(p[:min(remaining, len(p))]) + } + return len(p), nil +} + +func (b *boundedDeliveryOutput) String() string { + text := strings.TrimSpace(b.Buffer.String()) + if b.Len() == maxDeliveryCheckOutput { + return text + " (output truncated)" + } + return text +} + +var _ io.Writer = (*boundedDeliveryOutput)(nil) diff --git a/internal/server/landing_access.go b/internal/server/landing_access.go new file mode 100644 index 00000000..a373d625 --- /dev/null +++ b/internal/server/landing_access.go @@ -0,0 +1,130 @@ +package server + +import ( + "bytes" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + + "git.a-wan.net/alan/pipelines/internal/stage" +) + +// landingAccess derives delivery paths from Git's verified worktree metadata, +// never from stage input. A missing canonical main worktree fails closed. +func landingAccess(workspace string) (stage.LandingAccess, error) { + workspace, err := canonicalDirectory(workspace) + if err != nil { + return stage.LandingAccess{}, fmt.Errorf("delivery workspace: %w", err) + } + gitDir, err := gitOutput(workspace, "rev-parse", "--git-dir") + if err != nil { + return stage.LandingAccess{}, fmt.Errorf("delivery workspace is not a Git worktree: %w", err) + } + commonDir, err := gitOutput(workspace, "rev-parse", "--git-common-dir") + if err != nil { + return stage.LandingAccess{}, fmt.Errorf("delivery repository metadata: %w", err) + } + gitDir, err = canonicalDirectory(resolveGitPath(workspace, gitDir)) + if err != nil { + return stage.LandingAccess{}, fmt.Errorf("delivery Git metadata: %w", err) + } + commonDir, err = canonicalDirectory(resolveGitPath(workspace, commonDir)) + if err != nil { + return stage.LandingAccess{}, fmt.Errorf("delivery common Git metadata: %w", err) + } + main, err := mainWorktree(workspace, commonDir) + if err != nil { + return stage.LandingAccess{}, err + } + // Delivery originates in a linked item worktree. Its metadata must be under + // the repository's canonical common directory, never a standalone or foreign + // checkout. Git's common directory is the bounded capability needed for refs, + // worktree state, and the landing lock. + if !pathWithin(filepath.Join(commonDir, "worktrees"), gitDir) { + return stage.LandingAccess{}, fmt.Errorf("delivery workspace is not a linked Git worktree") + } + return stage.LandingAccess{MainWorktree: main, GitMetadata: commonDir, LockPath: filepath.Join(commonDir, "pipelines-landing.lock")}, nil +} + +func mainWorktree(workspace, commonDir string) (string, error) { + list, err := gitOutput(workspace, "worktree", "list", "--porcelain") + if err != nil { + return "", fmt.Errorf("list repository worktrees: %w", err) + } + for _, record := range strings.Split(strings.TrimSpace(list), "\n\n") { + lines := strings.Split(record, "\n") + if len(lines) == 0 || !strings.HasPrefix(lines[0], "worktree ") || !worktreeHasBranch(lines, "refs/heads/main") { + continue + } + candidate, err := canonicalDirectory(strings.TrimPrefix(lines[0], "worktree ")) + if err != nil || candidate == workspace { + continue + } + candidateGitDir, err := gitOutput(candidate, "rev-parse", "--git-dir") + if err != nil { + continue + } + candidateGitDir, err = canonicalDirectory(resolveGitPath(candidate, candidateGitDir)) + if err != nil || candidateGitDir != commonDir { + continue + } + candidateCommon, err := gitOutput(candidate, "rev-parse", "--git-common-dir") + if err != nil { + continue + } + candidateCommon, err = canonicalDirectory(resolveGitPath(candidate, candidateCommon)) + if err == nil && candidateCommon == commonDir { + return candidate, nil + } + } + return "", fmt.Errorf("no canonical main worktree in the delivery repository") +} + +func worktreeHasBranch(lines []string, branch string) bool { + for _, line := range lines[1:] { + if line == "branch "+branch { + return true + } + } + return false +} + +func canonicalDirectory(path string) (string, error) { + resolved, err := filepath.EvalSymlinks(path) + if err != nil { + return "", err + } + info, err := os.Stat(resolved) + if err != nil { + return "", err + } + if !info.IsDir() { + return "", fmt.Errorf("not a directory") + } + return filepath.Clean(resolved), nil +} + +func resolveGitPath(workspace, value string) string { + if filepath.IsAbs(value) { + return value + } + return filepath.Join(workspace, value) +} + +func pathWithin(parent, child string) bool { + rel, err := filepath.Rel(parent, child) + return err == nil && rel != "." && rel != ".." && !strings.HasPrefix(rel, ".."+string(filepath.Separator)) +} + +func gitOutput(directory string, args ...string) (string, error) { + cmd := exec.Command("git", append([]string{"-C", directory}, args...)...) + var stderr bytes.Buffer + cmd.Stderr = &stderr + data, err := cmd.Output() + if err != nil { + return "", fmt.Errorf("git %s: %w: %s", strings.Join(args, " "), err, strings.TrimSpace(stderr.String())) + } + return strings.TrimSpace(string(data)), nil +} diff --git a/internal/server/landing_access_test.go b/internal/server/landing_access_test.go new file mode 100644 index 00000000..9d2d7ae8 --- /dev/null +++ b/internal/server/landing_access_test.go @@ -0,0 +1,304 @@ +package server + +import ( + "context" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + + "git.a-wan.net/alan/pipelines/internal/pipeline" + "git.a-wan.net/alan/pipelines/internal/protocol" + "git.a-wan.net/alan/pipelines/internal/stage" +) + +func TestLandingAccessAcceptsOnlyVerifiedSameRepositoryMainWorktree(t *testing.T) { + root := t.TempDir() + main := filepath.Join(root, "main") + runGit(t, root, "init", "-b", "main", main) + runGit(t, main, "config", "user.email", "test@example.invalid") + runGit(t, main, "config", "user.name", "test") + if err := os.WriteFile(filepath.Join(main, "README"), []byte("base\n"), 0600); err != nil { + t.Fatal(err) + } + runGit(t, main, "add", "README") + runGit(t, main, "commit", "-m", "base") + item := filepath.Join(root, "item") + runGit(t, main, "worktree", "add", "-b", "item", item) + access, err := landingAccess(item) + if err != nil { + t.Fatal(err) + } + canonicalMain, err := canonicalDirectory(main) + if err != nil { + t.Fatal(err) + } + if access.MainWorktree != canonicalMain || access.GitMetadata != filepath.Join(canonicalMain, ".git") || access.LockPath != filepath.Join(canonicalMain, ".git", "pipelines-landing.lock") { + t.Fatalf("unexpected access: %+v", access) + } + foreign := filepath.Join(root, "foreign") + runGit(t, root, "init", "-b", "main", foreign) + if _, err := landingAccess(foreign); err == nil { + t.Fatal("foreign repository without an item worktree gained landing access") + } + if _, err := landingAccess(main); err == nil { + t.Fatal("main worktree gained linked-worktree delivery access") + } + if err := os.RemoveAll(item); err != nil { + t.Fatal(err) + } + runGit(t, main, "worktree", "prune") + if _, err := landingAccess(main); err == nil { + t.Fatal("repository without a linked delivery worktree gained landing access") + } + secondItem := filepath.Join(root, "second-item") + runGit(t, main, "worktree", "add", "-b", "second-item", secondItem) + runGit(t, main, "branch", "-m", "main", "trunk") + if _, err := landingAccess(secondItem); err == nil { + t.Fatal("repository without a canonical main worktree gained landing access") + } +} + +func TestVerifyDeliveryRequiresCleanLandedMainAndPassingChecks(t *testing.T) { + root := t.TempDir() + main := filepath.Join(root, "main") + runGit(t, root, "init", "-b", "main", main) + runGit(t, main, "config", "user.email", "test@example.invalid") + runGit(t, main, "config", "user.name", "test") + if err := os.WriteFile(filepath.Join(main, "README"), []byte("base\n"), 0600); err != nil { + t.Fatal(err) + } + runGit(t, main, "add", "README") + runGit(t, main, "commit", "-m", "base") + item := filepath.Join(root, "item") + runGit(t, main, "worktree", "add", "-b", "item", item) + if err := os.WriteFile(filepath.Join(item, "delivered"), []byte("complete\n"), 0600); err != nil { + t.Fatal(err) + } + runGit(t, item, "add", "delivered") + runGit(t, item, "commit", "-m", "delivery") + runGit(t, main, "merge", "--ff-only", "item") + access, err := landingAccess(item) + if err != nil { + t.Fatal(err) + } + if err := verifyDelivery(context.Background(), item, access, []string{"test -f delivered"}); err != nil { + t.Fatalf("complete landed delivery rejected: %v", err) + } + if err := os.WriteFile(filepath.Join(item, "missing-helper"), []byte("dirty\n"), 0600); err != nil { + t.Fatal(err) + } + if err := verifyDelivery(context.Background(), item, access, []string{"true"}); err == nil { + t.Fatal("dirty task-owned helper was accepted") + } + if err := os.Remove(filepath.Join(item, "missing-helper")); err != nil { + t.Fatal(err) + } + if err := verifyDelivery(context.Background(), item, access, []string{"false"}); err == nil { + t.Fatal("failed required check was accepted") + } + if err := os.WriteFile(filepath.Join(item, "unlanded"), []byte("no\n"), 0600); err != nil { + t.Fatal(err) + } + runGit(t, item, "add", "unlanded") + runGit(t, item, "commit", "-m", "unlanded") + if err := verifyDelivery(context.Background(), item, access, []string{"test -f unlanded"}); err != nil { + t.Fatalf("complete unlanded item was not landed after candidate verification: %v", err) + } + mainHead := runGitOutput(t, main, "rev-parse", "HEAD") + itemHead := runGitOutput(t, item, "rev-parse", "HEAD") + if mainHead != itemHead { + t.Fatal("verified item was not fast-forwarded into canonical main") + } +} + +func TestVerifyDeliveryFailingCandidateLeavesMainUnchanged(t *testing.T) { + root := t.TempDir() + main := filepath.Join(root, "main") + runGit(t, root, "init", "-b", "main", main) + runGit(t, main, "config", "user.email", "test@example.invalid") + runGit(t, main, "config", "user.name", "test") + if err := os.WriteFile(filepath.Join(main, "README"), []byte("base\n"), 0600); err != nil { + t.Fatal(err) + } + runGit(t, main, "add", "README") + runGit(t, main, "commit", "-m", "base") + item := filepath.Join(root, "item") + runGit(t, main, "worktree", "add", "-b", "item", item) + if err := os.WriteFile(filepath.Join(item, "candidate"), []byte("candidate\n"), 0600); err != nil { + t.Fatal(err) + } + runGit(t, item, "add", "candidate") + runGit(t, item, "commit", "-m", "candidate") + access, err := landingAccess(item) + if err != nil { + t.Fatal(err) + } + before := runGitOutput(t, main, "rev-parse", "HEAD") + if err := verifyDelivery(context.Background(), item, access, []string{"test -f candidate", "false"}); err == nil { + t.Fatal("failing candidate check landed main") + } + if after := runGitOutput(t, main, "rev-parse", "HEAD"); after != before { + t.Fatal("failing candidate check changed canonical main") + } +} + +func TestRunnerSettlesSuccessfulDeliveryAfterCandidateLanding(t *testing.T) { + ctx := context.Background() + root := t.TempDir() + main := filepath.Join(root, "main") + runGit(t, root, "init", "-b", "main", main) + runGit(t, main, "config", "user.email", "test@example.invalid") + runGit(t, main, "config", "user.name", "test") + if err := os.WriteFile(filepath.Join(main, "README"), []byte("base\n"), 0600); err != nil { + t.Fatal(err) + } + runGit(t, main, "add", "README") + runGit(t, main, "commit", "-m", "base") + item := filepath.Join(root, "item") + runGit(t, main, "worktree", "add", "-b", "item", item) + if err := os.WriteFile(filepath.Join(item, "unlanded"), []byte("no\n"), 0600); err != nil { + t.Fatal(err) + } + runGit(t, item, "add", "unlanded") + runGit(t, item, "commit", "-m", "unlanded") + + store, err := pipeline.Open(filepath.Join(root, "runner.db")) + if err != nil { + t.Fatal(err) + } + defer store.Close() + source := pipeline.Template([]string{"committer"}, true) + source = strings.Replace(source, "```configs", "```checks\ntrue\n```\n\n```configs", 1) + source = strings.Replace(source, "timeout=1h", "timeout=1h\ndelivery=true", 1) + if _, err := store.Submit(ctx, "delivery", source); err != nil { + t.Fatal(err) + } + items, err := store.List(ctx) + if err != nil { + t.Fatal(err) + } + r, err := newRunner(ctx, store) + if err != nil { + t.Fatal(err) + } + r.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) { + return stage.Result{Outcome: "success", Output: stage.Values{}}, nil + } + run, err := r.start(protocol.Message{PipelineID: items[0].ID, Version: 1, RequestKey: "unlanded", Workspace: item, Input: map[string]any{"prompt": "task"}}) + if err != nil { + t.Fatal(err) + } + r.wg.Wait() + detail, err := store.GetRun(ctx, run.ID) + if err != nil { + t.Fatal(err) + } + if detail.Status != "succeeded" || len(detail.Steps) != 1 { + t.Fatalf("complete delivery did not settle successfully: %+v", detail) + } + if runGitOutput(t, main, "rev-parse", "HEAD") != runGitOutput(t, item, "rev-parse", "HEAD") { + t.Fatal("runner did not land the verified item commit") + } +} + +func TestRunnerRejectsWrongBackendVerificationEvidence(t *testing.T) { + ctx := context.Background() + source := pipeline.Template([]string{"worker"}, true) + source = strings.Replace(source, "```configs", "```verification\nbackend=branch-api\nrevision=abc123\ncapability=controlled-state\nfixture=reconnect-foreground\n```\n\n```configs", 1) + r, store, id := submitRunner(t, ctx, source) + defer store.Close() + r.verifyTarget = func(context.Context, stage.VerificationTarget) (stage.VerificationEvidence, error) { + return stage.VerificationEvidence{Backend: "old-api", Revision: "abc123", Capability: "controlled-state", Fixture: "reconnect-foreground"}, nil + } + r.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) { + return stage.Result{Outcome: "success", Output: stage.Values{}}, nil + } + run := startRun(t, r, id, "wrong-backend-evidence") + detail, err := store.GetRun(ctx, run.ID) + if err != nil { + t.Fatal(err) + } + if detail.Status != "failed" || len(detail.Steps) != 1 || !strings.Contains(detail.Steps[0].Diagnostic, "backend") { + t.Fatalf("wrong-backend evidence settled as success: %+v", detail) + } +} + +func TestRunnerIgnoresModelSuppliedVerificationEvidence(t *testing.T) { + ctx := context.Background() + source := pipeline.Template([]string{"worker"}, true) + source = strings.Replace(source, "```configs", "```verification\nbackend=branch-api\nrevision=abc123\ncapability=controlled-state\nfixture=reconnect-foreground\n```\n\n```configs", 1) + r, store, id := submitRunner(t, ctx, source) + defer store.Close() + r.verifyTarget = func(context.Context, stage.VerificationTarget) (stage.VerificationEvidence, error) { + return stage.VerificationEvidence{Backend: "branch-api", Revision: "abc123", Capability: "controlled-state", Fixture: "reconnect-foreground"}, nil + } + r.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) { + return stage.Result{Outcome: "success", Output: stage.Values{"verification": map[string]any{"backend": "old-api", "revision": "stale", "capability": "invented", "fixture": "other"}}}, nil + } + run := startRun(t, r, id, "model-supplied-evidence") + detail, err := store.GetRun(ctx, run.ID) + if err != nil { + t.Fatal(err) + } + if detail.Status != "succeeded" || len(detail.Steps) != 1 { + t.Fatalf("controlled verification did not settle: %+v", detail) + } +} + +func TestMainWorktreeRejectsNearMatchAndLinkedMain(t *testing.T) { + for _, test := range []struct { + name string + branch string + }{ + {name: "mainline is not main", branch: "mainline"}, + {name: "linked main is not canonical", branch: "main"}, + } { + t.Run(test.name, func(t *testing.T) { + root := t.TempDir() + primary := filepath.Join(root, "primary") + runGit(t, root, "init", "-b", "trunk", primary) + runGit(t, primary, "config", "user.email", "test@example.invalid") + runGit(t, primary, "config", "user.name", "test") + if err := os.WriteFile(filepath.Join(primary, "README"), []byte("base\n"), 0600); err != nil { + t.Fatal(err) + } + runGit(t, primary, "add", "README") + runGit(t, primary, "commit", "-m", "base") + item := filepath.Join(root, "item") + runGit(t, primary, "worktree", "add", "-b", "item", item) + candidate := filepath.Join(root, test.branch) + runGit(t, primary, "worktree", "add", "-b", test.branch, candidate) + + commonDir, err := gitOutput(item, "rev-parse", "--git-common-dir") + if err != nil { + t.Fatal(err) + } + commonDir, err = canonicalDirectory(resolveGitPath(item, commonDir)) + if err != nil { + t.Fatal(err) + } + if _, err := mainWorktree(item, commonDir); err == nil { + t.Fatalf("%s gained canonical main access", test.name) + } + }) + } +} + +func runGit(t *testing.T, directory string, args ...string) { + t.Helper() + cmd := exec.Command("git", append([]string{"-C", directory}, args...)...) + if output, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("git %v: %v: %s", args, err, output) + } +} + +func runGitOutput(t *testing.T, directory string, args ...string) string { + t.Helper() + output, err := exec.Command("git", append([]string{"-C", directory}, args...)...).Output() + if err != nil { + t.Fatalf("git %v: %v", args, err) + } + return strings.TrimSpace(string(output)) +} diff --git a/internal/server/runner.go b/internal/server/runner.go index efaa8787..260397fe 100644 --- a/internal/server/runner.go +++ b/internal/server/runner.go @@ -58,8 +58,11 @@ type runner struct { // directory. It defaults to stage.ExportSession; a failure is best-effort and // never changes the stage's status, outcome, target or result validation. exportSession func(context.Context, stage.SessionExportRequest) (stage.SessionExportResult, error) - wg sync.WaitGroup - schedulerWG sync.WaitGroup + // verifyTarget is installed only by server wiring for a controlled fixture + // workflow. Leaving it nil fails closed for target-aware definitions. + verifyTarget stage.VerificationRunner + wg sync.WaitGroup + schedulerWG sync.WaitGroup // attentionMu guards attentionSent, the per-run set of run-attention events // already delivered — by a stage's own notify policy or by the run-level // default — so neither republishes the same genuine event. @@ -1079,7 +1082,22 @@ func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, na } } operations := &stage.OperationLog{} + var landing stage.LandingAccess + delivery, deliveryErr := stage.DeliveryEnabled(def) + if deliveryErr != nil { + finish("failed", "error", deliveryErr.Error()) + return + } + if delivery { + access, accessErr := landingAccess(workspace) + if accessErr != nil { + finish("failed", "error", "delivery access denied: "+accessErr.Error()) + return + } + landing = access + } client := r.agentClient(id, sequence, state.artifactDirs, workspace, artifactsDir, operations, request.WritePaths, tools) + client.LandingAccess = landing client.OnSession = func(session string, active bool) error { c, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() @@ -1222,6 +1240,45 @@ func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, na step.Output["salvage"] = result.Output } } + if err == nil && result.Outcome == "success" { + target, targetErr := stage.VerificationTargetFor(def) + if targetErr != nil { + err = targetErr + } else if target != nil { + if r.verifyTarget == nil { + err = fmt.Errorf("verification target %q requires a server-controlled fixture runner", target.Fixture) + } else { + evidence, verificationErr := r.verifyTarget(stageCtx, *target) + if verificationErr != nil { + err = verificationErr + } else { + err = stage.VerifyTargetEvidence(*target, evidence) + } + } + } + if err != nil { + step.Status = "failed" + step.Diagnostic = err.Error() + step.Outcome = "error" + step.Output = stage.Values{"error": err.Error(), "stage": key, "input": input} + } + } + if err == nil && delivery && result.Outcome == "success" { + checks, checksErr := stage.DeliveryChecks(def) + if checksErr != nil { + err = checksErr + } else { + gateCtx, gateCancel := context.WithTimeout(stageCtx, modelConfig.Timeout) + err = verifyDelivery(gateCtx, workspace, landing, checks) + gateCancel() + } + if err != nil { + step.Status = "failed" + step.Diagnostic = err.Error() + step.Outcome = "error" + step.Output = stage.Values{"error": err.Error(), "stage": key, "input": input} + } + } // A failed or timed-out step carries the session pointer even when the // bounded transcript would otherwise be only its stage_limits envelope. if err != nil && stageSession != "" { diff --git a/internal/server/verification_fixture.go b/internal/server/verification_fixture.go new file mode 100644 index 00000000..767723ab --- /dev/null +++ b/internal/server/verification_fixture.go @@ -0,0 +1,135 @@ +package server + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "os" + "strings" + "time" + + "git.a-wan.net/alan/pipelines/internal/stage" +) + +const ( + verificationFixtureID = "reconnect-foreground" + verificationFixtureResponseLimit int64 = 64 << 10 + verificationFixtureTimeout = 10 * time.Second +) + +// VerificationFixture is a server-owned, explicitly registered probe. Its +// endpoint and expected provenance come from server configuration, never a +// stage definition or result. +type VerificationFixture struct { + ID, Endpoint, Backend, Revision, Capability, State string +} + +// VerificationFixturesFromEnvironment returns the single supported fixture +// registration when its endpoint is explicitly configured. There is no default +// endpoint: an unconfigured server fails target-aware verification closed. +func VerificationFixturesFromEnvironment(getenv func(string) string) []VerificationFixture { + if getenv == nil { + getenv = os.Getenv + } + endpoint := strings.TrimSpace(getenv("PIPELINES_VERIFICATION_RECONNECT_FOREGROUND_URL")) + if endpoint == "" { + return nil + } + return []VerificationFixture{{ + ID: verificationFixtureID, + Endpoint: endpoint, + Backend: strings.TrimSpace(getenv("PIPELINES_VERIFICATION_RECONNECT_FOREGROUND_BACKEND")), + Revision: strings.TrimSpace(getenv("PIPELINES_VERIFICATION_RECONNECT_FOREGROUND_REVISION")), + Capability: strings.TrimSpace(getenv("PIPELINES_VERIFICATION_RECONNECT_FOREGROUND_CAPABILITY")), + State: strings.TrimSpace(getenv("PIPELINES_VERIFICATION_RECONNECT_FOREGROUND_STATE")), + }} +} + +type verificationFixtureRegistry struct { + fixtures map[string]VerificationFixture + err error +} + +func newVerificationFixtureRegistry(fixtures []VerificationFixture) *verificationFixtureRegistry { + r := &verificationFixtureRegistry{fixtures: make(map[string]VerificationFixture, len(fixtures))} + for _, fixture := range fixtures { + if err := validVerificationFixture(fixture); err != nil { + r.err = err + return r + } + if _, exists := r.fixtures[fixture.ID]; exists { + r.err = fmt.Errorf("duplicate verification fixture %q", fixture.ID) + return r + } + r.fixtures[fixture.ID] = fixture + } + return r +} + +func validVerificationFixture(fixture VerificationFixture) error { + if fixture.ID != verificationFixtureID { + return fmt.Errorf("unsupported verification fixture %q", fixture.ID) + } + for _, value := range []string{fixture.Endpoint, fixture.Backend, fixture.Revision, fixture.Capability, fixture.State} { + if value == "" || len(value) > 256 { + return fmt.Errorf("verification fixture %q has incomplete configuration", fixture.ID) + } + } + u, err := url.Parse(fixture.Endpoint) + if err != nil || u.Host == "" || (u.Scheme != "http" && u.Scheme != "https") { + return fmt.Errorf("verification fixture %q has invalid endpoint", fixture.ID) + } + return nil +} + +func (r *verificationFixtureRegistry) verify(ctx context.Context, target stage.VerificationTarget) (stage.VerificationEvidence, error) { + if r.err != nil { + return stage.VerificationEvidence{}, r.err + } + fixture, exists := r.fixtures[target.Fixture] + if !exists { + return stage.VerificationEvidence{}, fmt.Errorf("unknown verification fixture %q", target.Fixture) + } + if fixture.Backend != target.Backend || fixture.Revision != target.Revision || fixture.Capability != target.Capability { + return stage.VerificationEvidence{}, fmt.Errorf("verification target does not match registered fixture %q", target.Fixture) + } + probeCtx, cancel := context.WithTimeout(ctx, verificationFixtureTimeout) + defer cancel() + req, err := http.NewRequestWithContext(probeCtx, http.MethodGet, fixture.Endpoint, nil) + if err != nil { + return stage.VerificationEvidence{}, fmt.Errorf("verification fixture %q request: %w", fixture.ID, err) + } + client := &http.Client{Timeout: verificationFixtureTimeout, CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }} + response, err := client.Do(req) + if err != nil { + return stage.VerificationEvidence{}, fmt.Errorf("verification fixture %q probe: %w", fixture.ID, err) + } + defer response.Body.Close() + if response.StatusCode != http.StatusOK { + return stage.VerificationEvidence{}, fmt.Errorf("verification fixture %q probe returned %s", fixture.ID, response.Status) + } + var observed struct { + Backend string `json:"backend"` + Revision string `json:"revision"` + Capability string `json:"capability"` + Fixture string `json:"fixture"` + State string `json:"state"` + } + body, err := io.ReadAll(io.LimitReader(response.Body, verificationFixtureResponseLimit+1)) + if err != nil { + return stage.VerificationEvidence{}, fmt.Errorf("verification fixture %q response: %w", fixture.ID, err) + } + if int64(len(body)) > verificationFixtureResponseLimit { + return stage.VerificationEvidence{}, fmt.Errorf("verification fixture %q response exceeds %d bytes", fixture.ID, verificationFixtureResponseLimit) + } + if err := json.Unmarshal(body, &observed); err != nil { + return stage.VerificationEvidence{}, fmt.Errorf("verification fixture %q response: %w", fixture.ID, err) + } + if observed.Backend != fixture.Backend || observed.Revision != fixture.Revision || observed.Capability != fixture.Capability || observed.Fixture != fixture.ID || observed.State != fixture.State { + return stage.VerificationEvidence{}, fmt.Errorf("verification fixture %q returned incompatible backend, revision, capability, fixture, or state", fixture.ID) + } + return stage.VerificationEvidence{Backend: observed.Backend, Revision: observed.Revision, Capability: observed.Capability, Fixture: observed.Fixture}, nil +} diff --git a/internal/server/verification_fixture_test.go b/internal/server/verification_fixture_test.go new file mode 100644 index 00000000..3f71ca96 --- /dev/null +++ b/internal/server/verification_fixture_test.go @@ -0,0 +1,148 @@ +package server + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + "git.a-wan.net/alan/pipelines/internal/pipeline" + "git.a-wan.net/alan/pipelines/internal/protocol" + "git.a-wan.net/alan/pipelines/internal/stage" +) + +func TestHandlerRunsRegisteredVerificationFixture(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + var mu sync.Mutex + response := map[string]string{"backend": "branch-api", "revision": "abc123", "capability": "controlled-state", "fixture": verificationFixtureID, "state": "ready"} + calls := 0 + fixture := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + mu.Lock() + defer mu.Unlock() + calls++ + _ = json.NewEncoder(w).Encode(response) + })) + defer fixture.Close() + + store, err := pipeline.Open(filepath.Join(t.TempDir(), "fixture.db")) + if err != nil { + t.Fatal(err) + } + defer store.Close() + source := pipeline.Template([]string{"worker"}, true) + source = strings.Replace(source, "```configs", "```verification\nbackend=branch-api\nrevision=abc123\ncapability=controlled-state\nfixture="+verificationFixtureID+"\n```\n\n```configs", 1) + if _, err := store.Submit(ctx, "controlled-fixture", source); err != nil { + t.Fatal(err) + } + items, err := store.List(ctx) + if err != nil { + t.Fatal(err) + } + handler := HandlerWithOptions(ctx, Options{VerificationFixtures: []VerificationFixture{{ID: verificationFixtureID, Endpoint: fixture.URL, Backend: "branch-api", Revision: "abc123", Capability: "controlled-state", State: "ready"}}}, store).(*runHandler) + defer func() { + cancel() + WaitRuns(handler) + }() + r := handler.runner + if r.verifyTarget == nil { + t.Fatal("normal server construction did not wire controlled fixture verification") + } + r.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) { + // These forged model labels must not influence fixture provenance. + return stage.Result{Outcome: "success", Output: stage.Values{"verification": map[string]any{"backend": "old-api", "revision": "stale", "capability": "forged", "fixture": "other"}}}, nil + } + start := func(key string) *protocol.RunDetail { + run, err := r.start(protocol.Message{PipelineID: items[0].ID, Version: 1, RequestKey: key, Workspace: t.TempDir(), Input: map[string]any{"prompt": "verify"}}) + if err != nil { + t.Fatal(err) + } + r.wg.Wait() + detail, err := store.GetRun(ctx, run.ID) + if err != nil { + t.Fatal(err) + } + return detail + } + if detail := start("compatible"); detail.Status != "succeeded" { + t.Fatalf("compatible controlled fixture did not settle: %+v", detail) + } + mu.Lock() + response["revision"] = "old-build" + mu.Unlock() + if detail := start("stale"); detail.Status != "failed" || !strings.Contains(detail.Steps[0].Diagnostic, "incompatible") { + t.Fatalf("stale fixture provenance settled as success: %+v", detail) + } + mu.Lock() + if calls != 2 { + t.Fatalf("fixture probes = %d, want one per target-aware stage", calls) + } + mu.Unlock() +} + +func TestVerificationFixtureCancellationClosesProbe(t *testing.T) { + started := make(chan struct{}) + canceled := make(chan struct{}) + fixture := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + close(started) + <-r.Context().Done() + close(canceled) + })) + defer fixture.Close() + registry := newVerificationFixtureRegistry([]VerificationFixture{{ID: verificationFixtureID, Endpoint: fixture.URL, Backend: "branch-api", Revision: "abc123", Capability: "controlled-state", State: "ready"}}) + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan error, 1) + go func() { + _, err := registry.verify(ctx, stage.VerificationTarget{Backend: "branch-api", Revision: "abc123", Capability: "controlled-state", Fixture: verificationFixtureID}) + done <- err + }() + select { + case <-started: + case <-time.After(time.Second): + t.Fatal("fixture probe did not start") + } + cancel() + select { + case err := <-done: + if err == nil { + t.Fatal("canceled fixture probe succeeded") + } + case <-time.After(time.Second): + t.Fatal("canceled fixture probe did not return") + } + select { + case <-canceled: + case <-time.After(time.Second): + t.Fatal("canceled fixture request was not cleaned up") + } +} + +func TestVerificationFixtureRejectsUnknownAndMismatchedProvenance(t *testing.T) { + base := map[string]string{"backend": "branch-api", "revision": "abc123", "capability": "controlled-state", "fixture": verificationFixtureID, "state": "ready"} + for field, wrong := range map[string]string{"backend": "old-api", "revision": "old-build", "capability": "other-state", "state": "stale"} { + t.Run(field, func(t *testing.T) { + response := make(map[string]string, len(base)) + for key, value := range base { + response[key] = value + } + response[field] = wrong + fixture := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _ = json.NewEncoder(w).Encode(response) + })) + defer fixture.Close() + registry := newVerificationFixtureRegistry([]VerificationFixture{{ID: verificationFixtureID, Endpoint: fixture.URL, Backend: base["backend"], Revision: base["revision"], Capability: base["capability"], State: base["state"]}}) + _, err := registry.verify(context.Background(), stage.VerificationTarget{Backend: base["backend"], Revision: base["revision"], Capability: base["capability"], Fixture: verificationFixtureID}) + if err == nil || !strings.Contains(err.Error(), "incompatible") { + t.Fatalf("mismatched %s provenance was accepted: %v", field, err) + } + }) + } + registry := newVerificationFixtureRegistry(nil) + if _, err := registry.verify(context.Background(), stage.VerificationTarget{Backend: "branch-api", Revision: "abc123", Capability: "controlled-state", Fixture: "unknown"}); err == nil || !strings.Contains(err.Error(), "unknown") { + t.Fatalf("unknown fixture was accepted: %v", err) + } +} diff --git a/internal/server/websocket.go b/internal/server/websocket.go index adb3184b..5e598223 100644 --- a/internal/server/websocket.go +++ b/internal/server/websocket.go @@ -49,6 +49,10 @@ type Options struct { // read time with no model call, which keeps tests and unconfigured servers // deterministic. StepVerdicts func(context.Context, map[string]any) string + // VerificationFixtures are explicit server-owned HTTP fixtures for optional + // target-aware verification. Definitions can name only a registered ID; they + // cannot select an endpoint or supply provenance. + VerificationFixtures []VerificationFixture } // Handler serves the persistent connection with default options. Canceling ctx @@ -81,6 +85,10 @@ func HandlerWithOptions(ctx context.Context, options Options, stores ...*pipelin if runErr == nil && options.StepVerdicts != nil { runs.verdict = options.StepVerdicts } + if runErr == nil { + fixtures := newVerificationFixtureRegistry(options.VerificationFixtures) + runs.verifyTarget = fixtures.verify + } if runErr == nil { runs.schedulerWG.Add(1) go runs.scheduleLoop() diff --git a/internal/stage/agent.go b/internal/stage/agent.go index 57b23b12..dd91d1f3 100644 --- a/internal/stage/agent.go +++ b/internal/stage/agent.go @@ -19,6 +19,7 @@ type AgentClient struct { // ArtifactAccess is supplied only by the runner. Write is this visit's // directory; Read contains this and completed earlier visits in the same run. ArtifactAccess ArtifactAccess + LandingAccess LandingAccess OnSession func(string, bool) error SessionID string Transcript string diff --git a/internal/stage/model.go b/internal/stage/model.go index 13e3d1ff..18465c67 100644 --- a/internal/stage/model.go +++ b/internal/stage/model.go @@ -59,7 +59,7 @@ func (ModelStage) Spec(def Definition) (ModelSpec, error) { } for block := range def.Blocks { switch block { - case "output", "prompt", "edges", "configs", "tools", "permissions": + case "output", "prompt", "edges", "configs", "tools", "permissions", "checks", "verification": default: return ModelSpec{}, fmt.Errorf("unexpected block %q for model stage", block) } @@ -70,6 +70,9 @@ func (ModelStage) Spec(def Definition) (ModelSpec, error) { if _, err := StagePermissions(def); err != nil { return ModelSpec{}, err } + if _, err := VerificationTargetFor(def); err != nil { + return ModelSpec{}, err + } for _, key := range []string{"provider", "model"} { if def.Configs[key] == "" { return ModelSpec{}, fmt.Errorf("missing config %s", key) @@ -77,7 +80,7 @@ func (ModelStage) Spec(def Definition) (ModelSpec, error) { } for key := range def.Configs { switch key { - case "provider", "model", "retries", "timeout", "budget_usd", "budget_tokens", "harness": + case "provider", "model", "retries", "timeout", "budget_usd", "budget_tokens", "harness", "delivery": default: return ModelSpec{}, fmt.Errorf("unknown config %q", key) } @@ -92,6 +95,15 @@ func (ModelStage) Spec(def Definition) (ModelSpec, error) { if backend != "opencode" && backend != "pi" { return ModelSpec{}, fmt.Errorf("harness must be opencode or pi") } + if delivery := def.Configs["delivery"]; delivery != "" && delivery != "true" { + return ModelSpec{}, fmt.Errorf("delivery must be true when set") + } + if def.Configs["delivery"] == "true" && backend != "opencode" { + return ModelSpec{}, fmt.Errorf("delivery is supported only by the opencode harness") + } + if _, present := def.Blocks["checks"]; present && def.Configs["delivery"] != "true" { + return ModelSpec{}, fmt.Errorf("checks block is supported only by delivery stages") + } if backend == "pi" { tools, _, _ := ToolNames(def) if _, err := piTools(tools); err != nil { diff --git a/internal/stage/tools.go b/internal/stage/tools.go index 77477987..2f567e3b 100644 --- a/internal/stage/tools.go +++ b/internal/stage/tools.go @@ -14,6 +14,77 @@ type ArtifactAccess struct { Read []string } +// LandingAccess is the server-derived external capability for a verified +// delivery operation. Empty or malformed values grant nothing. +type LandingAccess struct { + MainWorktree string + GitMetadata string + LockPath string +} + +func (a LandingAccess) valid() bool { + for _, path := range []string{a.MainWorktree, a.GitMetadata, a.LockPath} { + if path == "" || !filepath.IsAbs(path) || filepath.Clean(path) != path || strings.ContainsAny(path, "*?\\") { + return false + } + } + // The server grants the canonical repository metadata directory as OpenCode + // V2's smallest external-directory unit. Keep the lock inside that directory + // so a caller cannot turn a valid metadata capability into access elsewhere. + return a.GitMetadata == filepath.Join(a.MainWorktree, ".git") && + a.LockPath == filepath.Join(a.GitMetadata, "pipelines-landing.lock") +} + +// DeliveryEnabled is the narrow, declarative capability check used before the +// runner supplies any landing paths. Only a validated model definition can ask +// for it; runtime paths are still derived by the server. +func DeliveryEnabled(def Definition) (bool, error) { + delivery := def.Configs["delivery"] + if delivery == "" { + return false, nil + } + if delivery != "true" { + return false, fmt.Errorf("delivery must be true when set") + } + if EffectiveType(def.Type) != ModelType { + return false, fmt.Errorf("delivery is supported only by model stages") + } + if _, err := (ModelStage{}).Spec(def); err != nil { + return false, fmt.Errorf("invalid delivery model stage: %w", err) + } + if _, err := DeliveryChecks(def); err != nil { + return false, err + } + return true, nil +} + +// DeliveryChecks returns the bounded, definition-declared checks that the +// runner executes against a committed delivery candidate before canonical main +// is changed. They +// are intentionally a block rather than a generic runner setting: each +// workflow owns its repository-specific verification commands. +func DeliveryChecks(def Definition) ([]string, error) { + text := strings.TrimSpace(def.Blocks["checks"]) + if text == "" { + return nil, fmt.Errorf("delivery stages require a nonempty checks block") + } + var checks []string + for _, line := range strings.Split(text, "\n") { + check := strings.TrimSpace(line) + if check == "" { + continue + } + if len(check) > 4096 { + return nil, fmt.Errorf("delivery check exceeds 4096 bytes") + } + checks = append(checks, check) + } + if len(checks) == 0 || len(checks) > 20 { + return nil, fmt.Errorf("delivery stages require 1 to 20 checks") + } + return checks, nil +} + // ToolNames validates model-stage permissions, not general graph semantics. // A present empty block explicitly denies all tools. func ToolNames(def Definition) ([]string, bool, error) { @@ -155,6 +226,10 @@ func (c *AgentClient) permissionRules() []map[string]string { allow("external_directory", filepath.ToSlash(filepath.Join(c.ArtifactAccess.Write, "*"))) allow("edit", filepath.ToSlash(filepath.Join(c.ArtifactAccess.Write, "*"))) } + if c.LandingAccess.valid() { + allow("external_directory", filepath.ToSlash(filepath.Join(c.LandingAccess.MainWorktree, "*"))) + allow("external_directory", filepath.ToSlash(filepath.Join(c.LandingAccess.GitMetadata, "*"))) + } for _, pattern := range []string{"../*", "*/../*"} { rules = append(rules, map[string]string{"action": "external_directory", "resource": pattern, "effect": "deny"}) } diff --git a/internal/stage/tools_test.go b/internal/stage/tools_test.go index ed823bc1..e7a20c0f 100644 --- a/internal/stage/tools_test.go +++ b/internal/stage/tools_test.go @@ -174,3 +174,107 @@ func TestArtifactPermissionsAreExactAndToolScoped(t *testing.T) { t.Fatal("missing artifact scope enabled an external directory") } } + +func TestLandingPermissionsAreExactAndServerDerived(t *testing.T) { + access := LandingAccess{MainWorktree: "/repos/pipelines", GitMetadata: "/repos/pipelines/.git", LockPath: "/repos/pipelines/.git/pipelines-landing.lock"} + client := &AgentClient{Tools: []string{"read", "shell"}, LandingAccess: access} + rules := client.permissionRules() + for _, path := range []string{"/repos/pipelines/go.mod", "/repos/pipelines/.git/worktrees/item/HEAD", "/repos/pipelines/.git/pipelines-landing.lock"} { + if permissionEffect(rules, "external_directory", path) != "allow" { + t.Fatalf("verified landing path denied: %q", path) + } + } + for _, path := range []string{"/repos/other/go.mod", "/repos/pipelines/../other/go.mod", "/Users/alan/.git/config", "/.git/config"} { + if permissionEffect(rules, "external_directory", path) != "deny" { + t.Fatalf("unverified landing path allowed: %q", path) + } + } + client.LandingAccess.MainWorktree = "/repos/pipelines/../other" + if permissionEffect(client.permissionRules(), "external_directory", "/repos/other/go.mod") != "deny" { + t.Fatal("non-canonical landing capability allowed") + } + client.LandingAccess = LandingAccess{MainWorktree: "/repos/pipelines", GitMetadata: "/repos/pipelines/.git", LockPath: "/tmp/pipelines-landing.lock"} + if permissionEffect(client.permissionRules(), "external_directory", "/tmp/pipelines-landing.lock") != "deny" { + t.Fatal("landing lock outside verified metadata allowed") + } + client.LandingAccess = LandingAccess{MainWorktree: "/repos/pipelines", GitMetadata: "/repos/other/.git", LockPath: "/repos/other/.git/pipelines-landing.lock"} + if permissionEffect(client.permissionRules(), "external_directory", "/repos/other/.git/config") != "deny" { + t.Fatal("unrelated Git metadata allowed") + } +} + +func TestDeliveryEnabledRequiresValidatedModelStage(t *testing.T) { + def := modelDefinition() + def.Configs["delivery"] = "true" + def.Blocks["checks"] = "go test ./..." + if enabled, err := DeliveryEnabled(def); err != nil || !enabled { + t.Fatalf("valid delivery stage = %v, %v", enabled, err) + } + for name, change := range map[string]func(*Definition){ + "malformed value": func(d *Definition) { d.Configs["delivery"] = "yes" }, + "non-model": func(d *Definition) { d.Type = HumanType }, + "invalid model": func(d *Definition) { d.Configs["unexpected"] = "value" }, + } { + t.Run(name, func(t *testing.T) { + copy := def + copy.Configs = map[string]string{} + for key, value := range def.Configs { + copy.Configs[key] = value + } + change(©) + if enabled, err := DeliveryEnabled(copy); err == nil || enabled { + t.Fatalf("invalid delivery stage = %v, %v", enabled, err) + } + }) + } +} + +func TestDeliveryRejectsPiBeforeAgentExecution(t *testing.T) { + def := modelDefinition() + def.Configs["delivery"] = "true" + def.Configs["harness"] = "pi" + def.Blocks["checks"] = "true" + if err := (ModelStage{}).Validate(def); err == nil || !strings.Contains(err.Error(), "opencode harness") { + t.Fatal("Pi delivery was accepted", err) + } + delete(def.Configs, "delivery") + delete(def.Blocks, "checks") + if err := (ModelStage{}).Validate(def); err != nil { + t.Fatal("Pi non-delivery was changed", err) + } +} + +func TestDeliveryChecksRequireBoundedDeliveryOnlyDeclaration(t *testing.T) { + def := modelDefinition() + def.Configs["delivery"] = "true" + for name, mutate := range map[string]func(*Definition){ + "missing": func(d *Definition) {}, + "empty": func(d *Definition) { d.Blocks["checks"] = "\n" }, + "non delivery": func(d *Definition) { d.Blocks["checks"] = "go test ./..." }, + "too many checks": func(d *Definition) { d.Blocks["checks"] = strings.Repeat("true\n", 21) }, + "oversized check": func(d *Definition) { d.Blocks["checks"] = strings.Repeat("x", 4097) }, + } { + t.Run(name, func(t *testing.T) { + copy := def + copy.Blocks = map[string]string{} + copy.Configs = map[string]string{} + for key, value := range def.Blocks { + copy.Blocks[key] = value + } + for key, value := range def.Configs { + copy.Configs[key] = value + } + mutate(©) + if name == "non delivery" { + delete(copy.Configs, "delivery") + if _, err := (ModelStage{}).Spec(copy); err == nil { + t.Fatal("non-delivery checks block was accepted") + } + return + } + if _, err := DeliveryEnabled(copy); err == nil { + t.Fatal("invalid delivery checks were accepted") + } + }) + } +} diff --git a/internal/stage/verification.go b/internal/stage/verification.go new file mode 100644 index 00000000..a186aa15 --- /dev/null +++ b/internal/stage/verification.go @@ -0,0 +1,97 @@ +package stage + +import ( + "context" + "fmt" + "strings" +) + +// VerificationTarget declares the controlled fixture that can substantiate a +// target-specific behavior claim. It is intentionally data-only: the runner +// does not execute fixture input supplied by a stage result. +type VerificationTarget struct { + Backend string + Revision string + Capability string + Fixture string +} + +// VerificationEvidence is returned by a server-controlled fixture workflow, +// never parsed from a model's result. +type VerificationEvidence struct { + Backend string + Revision string + Capability string + Fixture string +} + +// VerificationRunner is the narrow server seam for a controlled fixture or +// workflow. The runner selects it during server setup; model output cannot +// select a backend, revision, fixture, or executable. +type VerificationRunner func(context.Context, VerificationTarget) (VerificationEvidence, error) + +// VerificationTargetFor parses an optional, bounded verification block. A +// declared target requires all provenance fields so an old or other-backend +// fixture cannot be presented as evidence for the current target. +func VerificationTargetFor(def Definition) (*VerificationTarget, error) { + text, declared := def.Blocks["verification"] + if !declared { + return nil, nil + } + if len(text) > 4096 { + return nil, fmt.Errorf("verification block exceeds 4096 bytes") + } + values := map[string]string{} + for _, line := range strings.Split(text, "\n") { + line = strings.TrimSpace(line) + if line == "" { + continue + } + key, value, ok := strings.Cut(line, "=") + key, value = strings.TrimSpace(key), strings.TrimSpace(value) + if !ok || value == "" || len(value) > 256 { + return nil, fmt.Errorf("invalid verification entry %q", line) + } + switch key { + case "backend", "revision", "capability", "fixture": + default: + return nil, fmt.Errorf("unknown verification entry %q", key) + } + if _, exists := values[key]; exists { + return nil, fmt.Errorf("duplicate verification entry %q", key) + } + values[key] = value + } + target := &VerificationTarget{Backend: values["backend"], Revision: values["revision"], Capability: values["capability"], Fixture: values["fixture"]} + if target.Backend == "" || target.Revision == "" || target.Capability == "" || target.Fixture == "" { + return nil, fmt.Errorf("verification requires backend, revision, capability, and fixture") + } + return target, nil +} + +// VerifyTargetEvidence compares server-collected controlled-fixture provenance +// with the workflow declaration. It deliberately accepts no model result: a +// stage cannot substantiate its own target-specific behavior claim. +func VerifyTargetEvidence(target VerificationTarget, evidence VerificationEvidence) error { + for key, want := range map[string]string{ + "backend": target.Backend, "revision": target.Revision, "capability": target.Capability, "fixture": target.Fixture, + } { + got := map[string]string{ + "backend": evidence.Backend, "revision": evidence.Revision, "capability": evidence.Capability, "fixture": evidence.Fixture, + }[key] + if got != want { + return fmt.Errorf("verification evidence %s does not match declared target", key) + } + } + return nil +} + +// VerifyTargetResult is retained for callers that only need declaration +// validation. Result-provided provenance is intentionally not evidence. +func VerifyTargetResult(def Definition, _ Values) error { + target, err := VerificationTargetFor(def) + if err != nil || target == nil { + return err + } + return fmt.Errorf("verification target %q requires a server-controlled fixture runner", target.Fixture) +} diff --git a/internal/stage/verification_test.go b/internal/stage/verification_test.go new file mode 100644 index 00000000..d52d5b5a --- /dev/null +++ b/internal/stage/verification_test.go @@ -0,0 +1,27 @@ +package stage + +import "testing" + +func TestTargetVerificationRejectsWrongBackendEvidence(t *testing.T) { + def := modelDefinition() + def.Blocks["verification"] = "backend=branch-api\nrevision=abc123\ncapability=controlled-state\nfixture=reconnect-foreground" + if err := (ModelStage{}).Validate(def); err != nil { + t.Fatal(err) + } + target, err := VerificationTargetFor(def) + if err != nil { + t.Fatal(err) + } + good := VerificationEvidence{Backend: "branch-api", Revision: "abc123", Capability: "controlled-state", Fixture: "reconnect-foreground"} + if err := VerifyTargetEvidence(*target, good); err != nil { + t.Fatalf("matching fixture evidence rejected: %v", err) + } + wrongBackend := VerificationEvidence{Backend: "old-api", Revision: "abc123", Capability: "controlled-state", Fixture: "reconnect-foreground"} + if err := VerifyTargetEvidence(*target, wrongBackend); err == nil { + t.Fatal("wrong-backend fixture evidence accepted") + } + def.Blocks["verification"] = "backend=branch-api\nrevision=abc123" + if err := (ModelStage{}).Validate(def); err == nil { + t.Fatal("incomplete target verification declaration accepted") + } +}