From 31d8f4a6d5cf74b3ef6416313278ccb51b324f9f Mon Sep 17 00:00:00 2001 From: Alan Silva Date: Sun, 18 Oct 2026 11:17:01 +0100 Subject: [PATCH 1/2] Distinguish active stage deadlines from harness transport failures Identify configured attempt expiry during V2 active-session idle waits and retain bounded phase, timeout and session diagnostics without claiming a lost response. Preserve genuine operation failures, error identities, redaction, cleanup and admitted-work replay protections. Add stage regressions and scratch-server CLI/websocket retention coverage. Checked packaged templates in examples and the pipeline template generator: none needs updating because timeout configuration and contracts are unchanged. Checked packaged authoring skill references: debugging.md needs a STAGE DEADLINE playbook and clarified transport guidance. Recorded that stale shared documentation in docs/notes/stage-timeout-diagnostics.md for docs-fold; shared documentation was deliberately not edited. Verification: go build ./...; go vet ./...; go test ./...; go test -count=1 ./internal/server ./internal/stage ./internal/pipeline ./internal/protocol; go test -count=1 -run TestStageDeadlineDiagnosticThroughCLIAndWebsocket -v ./internal/server; gofmt -l .; git diff --check. All passed. Live-provider behavior not tested; no deploy or deployed-service restart. --- docs/notes/stage-timeout-diagnostics.md | 42 +++++++ internal/server/stage_deadline_test.go | 138 +++++++++++++++++++++++ internal/stage/model_diagnostic_test.go | 4 +- internal/stage/model_failure.go | 27 ++++- internal/stage/model_run.go | 9 +- internal/stage/opencode.go | 9 +- internal/stage/retry_safety_test.go | 2 +- internal/stage/stage_deadline_test.go | 140 ++++++++++++++++++++++++ 8 files changed, 363 insertions(+), 8 deletions(-) create mode 100644 docs/notes/stage-timeout-diagnostics.md create mode 100644 internal/server/stage_deadline_test.go create mode 100644 internal/stage/stage_deadline_test.go diff --git a/docs/notes/stage-timeout-diagnostics.md b/docs/notes/stage-timeout-diagnostics.md new file mode 100644 index 00000000..a9989418 --- /dev/null +++ b/docs/notes/stage-timeout-diagnostics.md @@ -0,0 +1,42 @@ +# Stage-timeout diagnostics + +## Source behavior + +The V2 active-session wait now reports `STAGE DEADLINE` when the configured +attempt timer expires between successful polls of a running admitted session. +Its bounded diagnostic names the idle-wait phase, configured timeout, retained +session, and that the stage had not finished (active at the last poll). It does +not claim that a completed response was lost. Cancellation is no longer recorded +as a stage deadline. A bare deadline alone is not `HARNESS TRANSPORT`. + +Actual request/HTTP/operation failures keep `HARNESS TRANSPORT`, their sanitized +operation/status evidence and underlying error identities, including when joined +with a deadline. A parent deadline no longer discards an accompanying transport +failure. Final-result validation, budget accounting, cleanup, interrupts, +side effects and replay policy are unchanged. No timeout was increased. + +## Documentation to fold manually + +`internal/author/skill/references/debugging.md` needs a separate stage-deadline +playbook alongside its HARNESS TRANSPORT section: inspect the retained session, +last activity, configured limits and unfinished final-result evidence before +deciding what to do. Do not blindly retry an admitted mutating stage; edits/tool +side effects remain. Explain the distinction between a configured idle-wait +overrun, cancellation, an actual failed request with a deadline, budget failure, +and final-result validation. Do not infer that all deadline errors are transport +failures or that a successful last tool means the stage returned a result. + +## Evidence and deployment + +2026-10-18: installed server build metadata (`go version -m +/Users/alan/.local/share/pipelines/bin/pipelines-server`) identifies revision +`0d1ddc73a444fa24110ba5223646856c8f053d29`. Comparing that revision with this +worktree's initial HEAD showed identical model-failure classification, model +attempt timing, V2 polling and agent wait/cleanup paths. The runner difference +was unrelated session-export HTML-only metadata. + +Regression tests use isolated loopback servers and scratch SQLite databases, +with a controlled V2 CLI fixture for the admitted-active-session deadline and +real loopback HTTP/refused-connection checks for native transport evidence. +This is not a live-provider timeout test. No deployment or deployed-service +restart is part of this item; retained historical runs are not rewritten. diff --git a/internal/server/stage_deadline_test.go b/internal/server/stage_deadline_test.go new file mode 100644 index 00000000..611e0afb --- /dev/null +++ b/internal/server/stage_deadline_test.go @@ -0,0 +1,138 @@ +package server + +import ( + "bytes" + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" + + "git.a-wan.net/alan/pipelines/internal/cli" + "git.a-wan.net/alan/pipelines/internal/pipeline" + "git.a-wan.net/alan/pipelines/internal/protocol" +) + +// Exercise the real V2 CLI/poll path, runner, SQLite retention and websocket CLI +// against an isolated loopback server/database, never the configured deployment. +func TestStageDeadlineDiagnosticThroughCLIAndWebsocket(t *testing.T) { + ntfy := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write([]byte(`{"id":"scratch-receipt","event":"message"}`)) + })) + defer ntfy.Close() + t.Setenv("PIPELINES_NTFY_URL", ntfy.URL+"/scratch") + t.Setenv("PIPELINES_NTFY_TOKEN", "") + t.Setenv("PIPELINES_NTFY_TOKEN_FILE", "") + for _, mode := range []string{"active", "http", "connection"} { + t.Run(mode, func(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + dir := t.TempDir() + executable := filepath.Join(dir, "opencode") + script := `#!/bin/sh +if [ "$1" = "--help" ]; then echo ' api'; exit 0; fi +printf '%s %s\n' "$2" "$3" >> "@STATE@/calls" +case "$3" in + /api/session) echo '{"data":{"id":"ses_deadline"}}' ;; + */prompt) echo 'edited once' >> "@STATE@/edited"; echo '{}' ;; + /api/session/active) + if [ -f "@STATE@/interrupted" ]; then echo '{"data":{}}'; exit 0; fi + case "@MODE@" in + active) echo '{"data":{"ses_deadline":{"type":"running"}}}' ;; + http) echo 'HTTP 503 token=private-token' >&2; exit 1 ;; + connection) echo 'connection refused token=private-token' >&2; exit 1 ;; + esac ;; + */interrupt) touch "@STATE@/interrupted"; echo '{}' ;; + */message*) echo '{"data":[{"content":[{"type":"text","text":"still working"}]}]}' ;; + /api/session/ses_deadline) echo '{"data":{"id":"ses_deadline"}}' ;; + *) echo '{}' ;; +esac +` + script = strings.ReplaceAll(strings.ReplaceAll(script, "@STATE@", dir), "@MODE@", mode) + if err := os.WriteFile(executable, []byte(script), 0700); err != nil { + t.Fatal(err) + } + t.Setenv("PIPELINES_OPENCODE", executable) + store, err := pipeline.Open(filepath.Join(dir, "runs.db")) + if err != nil { + t.Fatal(err) + } + defer store.Close() + source := pipeline.Template([]string{"worker"}, true) + source = strings.Replace(source, "```tools\n```", "```tools\nedit\n```", 1) + source = strings.Replace(source, "timeout=1h", "timeout=500ms", 1) + if _, err := store.Submit(ctx, "deadline", source); err != nil { + t.Fatal(err) + } + handler := Handler(ctx, store) + defer func() { cancel(); WaitRuns(handler) }() + srv := httptest.NewServer(handler) + defer srv.Close() + url := "ws" + strings.TrimPrefix(srv.URL, "http") + "/connect" + var out, stderr bytes.Buffer + app := cli.App{Out: &out, Err: &stderr, In: strings.NewReader("")} + if err := app.Run(ctx, []string{"--url", url, "run", "deadline", "task", "--yes", "--json", "--wait", "--timeout", "10s", "--workspace", dir}); err == nil || !strings.Contains(err.Error(), "run ended failed") { + t.Fatalf("run not failed: %v, %s, %s", err, out.String(), stderr.String()) + } + var response protocol.Message + if err := json.Unmarshal(out.Bytes(), &response); err != nil { + t.Fatal(err, out.String()) + } + detail := response.RunDetail + if detail == nil || detail.Status != "failed" || len(detail.Steps) != 1 { + t.Fatal("missing failed run/step", out.String()) + } + step := detail.Steps[0] + if step.SessionID != "ses_deadline" || step.AgentActive || step.Output["error"] != step.Diagnostic || step.Status != "failed" || detail.Diagnostic != step.Diagnostic { + t.Fatalf("incorrect retained failure: %+v", detail) + } + wants := []string{"HARNESS TRANSPORT", "get /api/session/active", "CLI exit code 1"} + if mode == "active" { + wants = []string{"STAGE DEADLINE", "timeout 500ms", "waiting for session idle", "ses_deadline", "stage had not finished"} + if strings.Contains(detail.Diagnostic, "HARNESS TRANSPORT") || strings.Contains(detail.Diagnostic, "response lost") { + t.Fatal("active stage overrun reported as transport", detail.Diagnostic) + } + if strings.Contains(detail.Diagnostic, "\n") { + t.Fatal("duplicate context error appended to bounded stage diagnostic", detail.Diagnostic) + } + for _, want := range []string{"opencode_failures", "wait for session idle", "timeout 500ms", "retained_session", "still working"} { + if !strings.Contains(step.Transcript, want) { + t.Fatalf("missing retained evidence %q: %s", want, step.Transcript) + } + } + } else { + if !strings.Contains(step.Transcript, "[REDACTED]") { + t.Fatal("request failure evidence missing", step.Transcript) + } + if mode == "http" { + wants = append(wants, "HTTP 503") + } else if !strings.Contains(step.Transcript, "connection refused") { + t.Fatal("underlying connection evidence missing", step.Transcript) + } + } + for _, want := range wants { + if !strings.Contains(detail.Diagnostic, want) { + t.Fatalf("missing diagnostic %q: %s", want, detail.Diagnostic) + } + } + if strings.Contains(detail.Diagnostic+step.Transcript, "private-token") { + t.Fatal("credential leaked in retained diagnostic/evidence") + } + edited, err := os.ReadFile(filepath.Join(dir, "edited")) + if err != nil || string(edited) != "edited once\n" { + t.Fatalf("admitted tool work replayed or lost: %q, %v", edited, err) + } + if _, err := os.Stat(filepath.Join(dir, "interrupted")); err != nil { + t.Fatal("cleanup did not interrupt", err) + } + stored, err := store.GetRun(ctx, detail.ID) + if err != nil || stored.Diagnostic != detail.Diagnostic || stored.Steps[0].Transcript != step.Transcript { + t.Fatal("diagnostic/evidence not persisted in scratch database", err) + } + t.Logf("scratch run %s: %s", mode, stored.Diagnostic) + }) + } +} diff --git a/internal/stage/model_diagnostic_test.go b/internal/stage/model_diagnostic_test.go index e61a5347..00be8b01 100644 --- a/internal/stage/model_diagnostic_test.go +++ b/internal/stage/model_diagnostic_test.go @@ -36,8 +36,8 @@ func TestModelFailureCategoriesAreSanitizedAndDistinct(t *testing.T) { if typed := wrapModelStageFailure("worker", &openCodeError{operation: "get /api/session", status: 503}); !strings.Contains(typed.Error(), "HARNESS TRANSPORT") || !errors.As(typed, new(*openCodeError)) { t.Fatalf("typed harness error was not classified: %v", typed) } - if deadline := wrapModelStageFailure("worker", context.DeadlineExceeded); !strings.Contains(deadline.Error(), "HARNESS TRANSPORT") || !errors.Is(deadline, context.DeadlineExceeded) { - t.Fatalf("context deadline was not classified as harness: %v", deadline) + if deadline := wrapModelStageFailure("worker", context.DeadlineExceeded); strings.Contains(deadline.Error(), "HARNESS TRANSPORT") || !errors.Is(deadline, context.DeadlineExceeded) { + t.Fatalf("bare context deadline was misclassified or lost: %v", deadline) } if modelFailureCategory(nil) != "" || wrapModelStageFailure("worker", nil) != nil { t.Fatal("nil failure was classified") diff --git a/internal/stage/model_failure.go b/internal/stage/model_failure.go index 6cef2b1b..2c6af418 100644 --- a/internal/stage/model_failure.go +++ b/internal/stage/model_failure.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "time" ) // Retained model-stage failure categories. A strict final-result validation @@ -13,8 +14,27 @@ import ( const ( finalResultValidation = "FINAL RESULT VALIDATION" harnessTransport = "HARNESS TRANSPORT" + stageDeadline = "STAGE DEADLINE" ) +// stageTimeoutCause identifies this attempt's configured timer, not a parent +// cancellation or a request's own deadline. It is carried through budget contexts. +type stageTimeoutCause struct{ timeout time.Duration } + +func (e *stageTimeoutCause) Error() string { return "configured stage timeout reached" } +func (e *stageTimeoutCause) Unwrap() error { return context.DeadlineExceeded } + +type stageDeadlineError struct { + timeout time.Duration + session string +} + +func (e *stageDeadlineError) Error() string { + return fmt.Sprintf("configured stage deadline reached (timeout %s) while waiting for session idle; retained OpenCode session %.128s was still active at the last poll; stage had not finished", e.timeout, RedactEvidence(e.session)) +} + +func (e *stageDeadlineError) Unwrap() error { return context.DeadlineExceeded } + // finalResultGuidance is a fixed sanitized sentence appended only to validation // diagnostics. It states the retained side effects and the cost of a full retry. const finalResultGuidance = "completed edits and tool side effects are retained and not undone; a full retry repeats work" @@ -51,11 +71,14 @@ func (c *PiClient) workAdmitted() bool { return c.WorkStarted } // the label stays sanitized and bounded. Validation is checked first: a // structured-output failure is never reported as a transport failure and vice versa. func modelFailureCategory(err error) string { + var deadline *stageDeadlineError switch { case errors.Is(err, ErrInvalidModelResult): return finalResultValidation - case harnessTransportFailure(err), errors.Is(err, context.DeadlineExceeded): + case harnessTransportFailure(err): return harnessTransport + case errors.As(err, &deadline): + return stageDeadline default: return "" } @@ -73,6 +96,8 @@ func wrapModelStageFailure(key string, err error) error { return fmt.Errorf("model stage %s: %s: %w; %s", key, finalResultValidation, err, finalResultGuidance) case harnessTransport: return fmt.Errorf("model stage %s: %s: %w", key, harnessTransport, err) + case stageDeadline: + return fmt.Errorf("model stage %s: %s: %w", key, stageDeadline, err) default: return fmt.Errorf("model stage %s: %w", key, err) } diff --git a/internal/stage/model_run.go b/internal/stage/model_run.go index e1979706..41300243 100644 --- a/internal/stage/model_run.go +++ b/internal/stage/model_run.go @@ -61,7 +61,7 @@ func (m ModelStage) Run(ctx context.Context, def Definition, input Values) (Resu return Result{}, err } spec.Config.Budget.Attempt() - attemptCtx, cancel := context.WithTimeout(ctx, spec.Config.Timeout) + attemptCtx, cancel := context.WithTimeoutCause(ctx, spec.Config.Timeout, &stageTimeoutCause{timeout: spec.Config.Timeout}) text, generateErr := client.Generate(attemptCtx, spec.Config, prompt) if generateErr == nil && attemptCtx.Err() != nil { generateErr = attemptCtx.Err() @@ -96,11 +96,16 @@ func (m ModelStage) Run(ctx context.Context, def Definition, input Values) (Resu } } else { err = generateErr - if attemptCtx.Err() != nil && !errors.Is(err, ErrBudgetExceeded) && !errors.Is(err, ErrBudgetAccounting) { + // Do not append the same context error twice: the wait/transport + // error may already preserve it, with a more precise diagnostic. + if attemptCtx.Err() != nil && !errors.Is(err, attemptCtx.Err()) && !errors.Is(err, ErrBudgetExceeded) && !errors.Is(err, ErrBudgetAccounting) { err = errors.Join(err, attemptCtx.Err()) } cancel() if ctx.Err() != nil { + if harnessTransportFailure(err) { + return Result{}, wrapModelStageFailure(def.Key, err) + } return Result{}, ctx.Err() } if errors.Is(err, ErrBudgetExceeded) || errors.Is(err, ErrBudgetAccounting) || errors.Is(err, ErrStageCleanup) { diff --git a/internal/stage/opencode.go b/internal/stage/opencode.go index 38d7722e..bdf28bfd 100644 --- a/internal/stage/opencode.go +++ b/internal/stage/opencode.go @@ -217,6 +217,11 @@ func (c OpenCodeClient) waitForSessionIdle(ctx context.Context, sessionID string select { case <-ctx.Done(): timer.Stop() + waitErr := fmt.Errorf("waiting for session idle; retained OpenCode session %.128s; stage had not finished: %w", RedactEvidence(sessionID), ctx.Err()) + var timeout *stageTimeoutCause + if errors.Is(ctx.Err(), context.DeadlineExceeded) && errors.As(context.Cause(ctx), &timeout) { + waitErr = &stageDeadlineError{timeout: timeout.timeout, session: sessionID} + } // A settled session is the only success signal; retain one bounded // sanitized record so a timed-out wait is not an empty transcript. c.Operations.record(OperationEvidence{ @@ -225,9 +230,9 @@ func (c OpenCodeClient) waitForSessionIdle(ctx context.Context, sessionID string Executable: opencode.Executable(), Directory: c.Directory, ExitCode: -1, - Diagnostic: "stage deadline reached while waiting for the agent session to become idle", + Diagnostic: waitErr.Error(), }, started) - return ctx.Err() + return waitErr case <-timer.C: } } diff --git a/internal/stage/retry_safety_test.go b/internal/stage/retry_safety_test.go index f5f812e9..f0a40541 100644 --- a/internal/stage/retry_safety_test.go +++ b/internal/stage/retry_safety_test.go @@ -251,7 +251,7 @@ func TestWaitForSessionIdleRecordsOneTimeoutEvidence(t *testing.T) { t.Fatalf("expected exactly one timeout evidence, got %d", len(log.Failures)) } entry := log.Failures[0] - if entry.Operation != "wait for session idle" || !strings.Contains(entry.Diagnostic, "deadline reached") { + if entry.Operation != "wait for session idle" || !strings.Contains(entry.Diagnostic, "context deadline exceeded") { t.Fatalf("timeout evidence not sanitized as expected: %+v", entry) } clean := &OperationLog{} diff --git a/internal/stage/stage_deadline_test.go b/internal/stage/stage_deadline_test.go new file mode 100644 index 00000000..2f2f3aea --- /dev/null +++ b/internal/stage/stage_deadline_test.go @@ -0,0 +1,140 @@ +package stage + +import ( + "context" + "errors" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "git.a-wan.net/alan/pipelines/internal/opencode" +) + +func TestConfiguredStageDeadlineWhileSessionActive(t *testing.T) { + def := modelDefinition() + def.Blocks["tools"] = "edit" + def.Configs["timeout"] = "20ms" + def.Configs["retries"] = "2" + log := &OperationLog{} + creates, prompts, interrupts := 0, 0, 0 + agent := &AgentClient{OpenCodeClient: OpenCodeClient{Backend: "v2", Operations: log, sessionPollInterval: time.Hour}} + agent.command = func(_ context.Context, _ string, args ...string) ([]byte, error) { + switch args[2] { + case "/api/session": + creates++ + return []byte(`{"data":{"id":"ses_deadline"}}`), nil + case "/api/session/ses_deadline/prompt": + prompts++ + return []byte(`{}`), nil + case "/api/session/active": + return []byte(`{"data":{"ses_deadline":{"type":"running"}}}`), nil + case "/api/session/ses_deadline/interrupt": + interrupts++ + return []byte(`{}`), nil + default: + // Best-effort timeout readback/usage cannot convert the failure to success. + return []byte(`{}`), nil + } + } + result, err := (ModelStage{Client: agent}).Run(context.Background(), def, nil) + if !errors.Is(err, context.DeadlineExceeded) || modelFailureCategory(err) != stageDeadline || errors.Is(err, ErrHarnessTransport) { + t.Fatalf("deadline identity/category: %v", err) + } + for _, want := range []string{"STAGE DEADLINE", "worker", "timeout 20ms", "waiting for session idle", "ses_deadline", "stage had not finished"} { + if !strings.Contains(err.Error(), want) { + t.Fatalf("missing %q: %v", want, err) + } + } + if result.Output != nil || creates != 1 || prompts != 1 || interrupts != 1 || !agent.WorkStarted { + t.Fatalf("work replayed, cleanup skipped or incomplete turn accepted: %+v creates=%d prompts=%d interrupts=%d", result, creates, prompts, interrupts) + } + if len(log.Failures) != 1 || !strings.Contains(log.Failures[0].Diagnostic, "timeout 20ms") { + t.Fatalf("deadline evidence missing: %+v", log.Failures) + } +} + +func TestWaitCancellationIsNotConfiguredStageDeadline(t *testing.T) { + for _, cause := range []error{context.Canceled, ErrBudgetExceeded, ErrBudgetAccounting} { + t.Run(cause.Error(), func(t *testing.T) { + attempt, stop := context.WithTimeoutCause(context.Background(), time.Hour, &stageTimeoutCause{timeout: time.Hour}) + defer stop() + ctx, cancel := context.WithCancelCause(attempt) + log := &OperationLog{} + client := OpenCodeClient{Operations: log, command: func(context.Context, string, ...string) ([]byte, error) { + cancel(cause) + return []byte(`{"data":{"ses_wait":{"type":"running"}}}`), nil + }} + err := client.waitForSessionIdle(ctx, "ses_wait") + if !errors.Is(err, context.Canceled) || modelFailureCategory(err) != "" || len(log.Failures) != 1 { + t.Fatalf("cancellation misclassified: %v, %+v", err, log.Failures) + } + if strings.Contains(log.Failures[0].Diagnostic, "deadline") { + t.Fatalf("cancellation recorded as deadline: %+v", log.Failures) + } + }) + } +} + +func TestDeadlineWithTransportRetainsUnderlyingIdentity(t *testing.T) { + transport := &openCodeError{operation: "get /api/session/active", status: 503, cause: errors.New("request failed")} + err := wrapModelStageFailure("worker", errors.Join(&stageDeadlineError{timeout: time.Second, session: "ses_wait"}, transport)) + var got *openCodeError + if modelFailureCategory(err) != harnessTransport || !errors.Is(err, context.DeadlineExceeded) || !errors.As(err, &got) || got != transport || !errors.Is(err, transport.cause) { + t.Fatalf("combined failure lost transport/deadline identity: %v", err) + } + if !strings.Contains(err.Error(), "HTTP 503") || !strings.Contains(err.Error(), "get /api/session/active") { + t.Fatalf("combined failure lost operation/status: %v", err) + } + bounded := (&stageDeadlineError{timeout: time.Hour, session: strings.Repeat("s", 10000) + " Bearer private-token"}).Error() + if len(bounded) > 512 || strings.Contains(bounded, "private-token") { + t.Fatal("deadline diagnostic was not bounded/redacted") + } +} + +func TestParentDeadlinePreservesModelTransportFailure(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Millisecond) + defer cancel() + transport := &openCodeError{operation: "get /api/session/active", status: 504, cause: errors.New("request failed")} + client := generatorFunc(func(ctx context.Context, _ ModelConfig, _ string) (string, error) { + <-ctx.Done() + return "", errors.Join(ctx.Err(), transport) + }) + _, err := (ModelStage{Client: client}).Run(ctx, modelDefinition(), nil) + var got *openCodeError + if !errors.Is(err, context.DeadlineExceeded) || !errors.As(err, &got) || got != transport || modelFailureCategory(err) != harnessTransport { + t.Fatalf("parent deadline erased underlying request failure: %v", err) + } +} + +func TestNativeHTTPAndConnectionFailuresRemainTransport(t *testing.T) { + for _, mode := range []string{"http", "connection"} { + t.Run(mode, func(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusServiceUnavailable) + _, _ = w.Write([]byte(`{"message":"service unavailable","token":"private-token"}`)) + })) + defer srv.Close() + if mode == "connection" { + srv.Close() + } + log := &OperationLog{} + client := OpenCodeClient{Backend: "v1", Operations: log, connection: &opencode.Connection{Backend: "v1", URL: srv.URL}} + _, err := client.call(context.Background(), "get", "/api/session", nil) + var typed *openCodeError + if !errors.As(err, &typed) || modelFailureCategory(err) != harnessTransport || len(log.Failures) != 1 { + t.Fatalf("real request failure misclassified: %v %+v", err, log.Failures) + } + if mode == "http" && (typed.status != 503 || log.Failures[0].HTTPStatus != 503 || !strings.Contains(log.Failures[0].Stdout, "service unavailable")) { + t.Fatalf("HTTP status/evidence lost: %v %+v", err, log.Failures) + } + if mode == "connection" && (log.Failures[0].Diagnostic != "OpenCode native API request failed" || log.Failures[0].NativeOperation != "GET /session") { + t.Fatalf("connection evidence lost: %+v", log.Failures) + } + if strings.Contains(err.Error()+log.Transcript(""), "private-token") { + t.Fatal("credential leaked in transport diagnostic/evidence") + } + }) + } +} From 31aec05f99e71ac59e60e3727863c25538620282 Mon Sep 17 00:00:00 2001 From: Alan Silva Date: Sun, 18 Oct 2026 12:02:37 +0100 Subject: [PATCH 2/2] Scope stage artifact access by run visit Derive validated same-run visit artifact capabilities in the runner and pass them to OpenCode and Pi clients. Permit supported file tools to read completed prior visits and write only the current visit, preserving workspace, secret, database, and shell restrictions.\n\nChecked packaged templates and authoring skill references: neither needs updating in this item; docs/notes/scoped-artifact-access.md records the shared documentation staleness for the docs-fold writer. --- docs/notes/scoped-artifact-access.md | 20 ++++++++ internal/server/artifacts_test.go | 55 +++++++++++++++++++++ internal/server/runner.go | 67 ++++++++++++++++++++++++-- internal/stage/agent.go | 17 ++++--- internal/stage/pi.go | 3 +- internal/stage/pi_policy.mjs | 35 +++++++++++++- internal/stage/pi_policy_test.go | 72 ++++++++++++++++++++++++++++ internal/stage/tools.go | 34 +++++++++++++ internal/stage/tools_test.go | 35 ++++++++++++++ 9 files changed, 325 insertions(+), 13 deletions(-) create mode 100644 docs/notes/scoped-artifact-access.md create mode 100644 internal/stage/pi_policy_test.go diff --git a/docs/notes/scoped-artifact-access.md b/docs/notes/scoped-artifact-access.md new file mode 100644 index 00000000..75943280 --- /dev/null +++ b/docs/notes/scoped-artifact-access.md @@ -0,0 +1,20 @@ +# Scoped stage artifact access + +## What this item changed + +The server now derives one exact artifact capability for an OpenCode or Pi +model-stage visit: its own directory is writable when the edit tool is present, +and its own plus completed earlier visit directories in the same run are +readable when a file-reading tool is present. No artifact root, sibling, future +visit, other run, traversal path, or symlink escape is granted. The scope is +empty if the runner cannot validate its artifact root. + +Pi retains its lease and explicit extension policy; unrestricted shell remains +outside this file-tool boundary and is not an OS sandbox. + +## Shared documents this change makes stale + +- `docs/stage-artifacts.md` and `internal/author/skill/references/operations.md` + should explain that model-stage artifact directories are usable by the + supported file tools only within the exact same-run visit scope, not as a + general external-directory permission. diff --git a/internal/server/artifacts_test.go b/internal/server/artifacts_test.go index 573b851b..6c4c7804 100644 --- a/internal/server/artifacts_test.go +++ b/internal/server/artifacts_test.go @@ -105,6 +105,61 @@ func TestArtifactsScanMissingDirectoryIsEmpty(t *testing.T) { } } +func TestArtifactAccessUsesOnlyCurrentAndEarlierVisits(t *testing.T) { + root := t.TempDir() + r := &runner{artifactsRoot: root} + for _, directory := range []string{"7/1", "7/2", "7/3", "8/1"} { + if err := os.MkdirAll(filepath.Join(root, directory), 0700); err != nil { + t.Fatal(err) + } + } + access := r.artifactAccess(7, 2, []string{"1", "3", "../8/1"}, filepath.Join(root, "7", "2")) + current, _ := filepath.EvalSymlinks(filepath.Join(root, "7", "2")) + prior, _ := filepath.EvalSymlinks(filepath.Join(root, "7", "1")) + if access.Write != current || len(access.Read) != 2 || access.Read[0] != access.Write || access.Read[1] != prior { + t.Fatalf("artifact scope = %#v", access) + } + if got := r.artifactAccess(7, 2, nil, filepath.Join(root, "8", "1")); got.Write != "" || len(got.Read) != 0 { + t.Fatalf("other-run current scope accepted: %#v", got) + } + if got := r.artifactAccess(7, 2, nil, filepath.Join(root, "7", "3")); got.Write != "" || len(got.Read) != 0 { + t.Fatalf("sibling current scope accepted: %#v", got) + } + if err := os.WriteFile(filepath.Join(root, "7", "4"), []byte("not a directory"), 0600); err != nil { + t.Fatal(err) + } + if got := r.artifactAccess(7, 4, nil, filepath.Join(root, "7", "4")); got.Write != "" || len(got.Read) != 0 { + t.Fatalf("file current scope accepted: %#v", got) + } + if err := os.Symlink(filepath.Join(root, "7", "2"), filepath.Join(root, "7", "5")); err != nil { + t.Fatal(err) + } + if got := r.artifactAccess(7, 5, nil, filepath.Join(root, "7", "5")); got.Write != "" || len(got.Read) != 0 { + t.Fatalf("symlinked current scope accepted: %#v", got) + } +} + +func TestRunnerPassesDerivedArtifactAccessToAgentClient(t *testing.T) { + root := t.TempDir() + for _, directory := range []string{"7/1", "7/2"} { + if err := os.MkdirAll(filepath.Join(root, directory), 0700); err != nil { + t.Fatal(err) + } + } + r := &runner{artifactsRoot: root} + client := r.agentClient(7, 2, []string{"1", "3"}, t.TempDir(), filepath.Join(root, "7", "2"), &stage.OperationLog{}, nil, []string{"read", "edit"}) + if client.ArtifactAccess.Write == "" || len(client.ArtifactAccess.Read) != 2 { + t.Fatalf("runner did not pass scoped artifact access: %#v", client.ArtifactAccess) + } + if client.ArtifactAccess.Read[1] == filepath.Join(root, "7", "3") { + t.Fatal("runner passed a future visit to the client") + } + piClient := r.piClient(7, 2, []string{"1", "3"}, "pi", t.TempDir(), filepath.Join(root, "7", "2"), nil, []string{"read", "edit"}) + if piClient.ArtifactAccess.Write != client.ArtifactAccess.Write || len(piClient.ArtifactAccess.Read) != len(client.ArtifactAccess.Read) || piClient.ArtifactAccess.Read[1] != client.ArtifactAccess.Read[1] { + t.Fatalf("runner passed different artifact scope to Pi client: %#v", piClient.ArtifactAccess) + } +} + // runWithArtifact executes one real run whose stage writes a file into the // runner-provided artifacts directory. It returns the live handler and the // recorded step so tests can serve the artifact over the real HTTP endpoint. diff --git a/internal/server/runner.go b/internal/server/runner.go index 0c532247..11471617 100644 --- a/internal/server/runner.go +++ b/internal/server/runner.go @@ -580,7 +580,8 @@ type runState struct { capturedSeen map[string]bool // completed counts the run's finished steps, so a delta can report the // running total without re-reading the store. - completed int + completed int + artifactDirs []string } func newRunState(key string, input stage.Values) *runState { @@ -606,10 +607,67 @@ func rebuildRunState(detail *protocol.RunDetail, decided protocol.RunStep) *runS // Seed the UI-surface accumulator from the persisted evidence so a // resumed run reaches the same skip decision with no extra query. state.recordChanges(step.FileChanges, step.Artifacts) + // Only completed, recorded visits are eligible for a later stage's + // review capability. The path is reconstructed by the runner below. + if step.Status == "completed" { + state.artifactDirs = append(state.artifactDirs, filepath.Join("", strconv.Itoa(step.Sequence))) + } } return state } +// artifactAccess derives exact visit directories; it never consumes a path from +// a definition or model result. An unreadable or symlinked root fails closed. +func (r *runner) artifactAccess(runID int64, sequence int, prior []string, current string) stage.ArtifactAccess { + root, err := filepath.EvalSymlinks(r.artifactsRoot) + if err != nil { + return stage.ArtifactAccess{} + } + runRoot := filepath.Join(root, strconv.FormatInt(runID, 10)) + visitDirectory := func(visit int) (string, bool) { + if visit <= 0 { + return "", false + } + directory := filepath.Join(runRoot, strconv.Itoa(visit)) + resolved, err := filepath.EvalSymlinks(directory) + if err != nil || resolved != directory { + return "", false + } + info, err := os.Stat(resolved) + if err != nil || !info.IsDir() { + return "", false + } + return resolved, true + } + access := stage.ArtifactAccess{} + if directory, ok := visitDirectory(sequence); ok { + resolved, err := filepath.EvalSymlinks(current) + if err != nil || resolved != directory { + return access + } + access.Write = directory + access.Read = append(access.Read, directory) + } + for _, priorSequence := range prior { + visit, err := strconv.Atoi(priorSequence) + if err != nil || visit <= 0 || visit >= sequence { + continue + } + if directory, ok := visitDirectory(visit); ok { + access.Read = append(access.Read, directory) + } + } + return access +} + +func (r *runner) agentClient(runID int64, sequence int, prior []string, workspace, artifactsDir string, operations *stage.OperationLog, writePaths, tools []string) *stage.AgentClient { + return &stage.AgentClient{OpenCodeClient: stage.OpenCodeClient{Directory: workspace, Operations: operations, ArtifactsDir: artifactsDir}, WritePaths: writePaths, Tools: tools, ArtifactAccess: r.artifactAccess(runID, sequence, prior, artifactsDir)} +} + +func (r *runner) piClient(runID int64, sequence int, prior []string, executable, workspace, artifactsDir string, writePaths, tools []string) *stage.PiClient { + return &stage.PiClient{Executable: executable, Directory: workspace, Tools: tools, WritePaths: writePaths, ArtifactsDir: artifactsDir, ArtifactAccess: r.artifactAccess(runID, sequence, prior, artifactsDir)} +} + // serverChangedFilesField is the reserved run-input memory field under which the // runner publishes the server-captured changed paths. A stage selects it with // `"server_changed_files":"input.server_changed_files"`, alongside another @@ -977,7 +1035,7 @@ func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, na } } operations := &stage.OperationLog{} - client := &stage.AgentClient{OpenCodeClient: stage.OpenCodeClient{Directory: workspace, Operations: operations, ArtifactsDir: artifactsDir}, WritePaths: request.WritePaths, Tools: tools} + client := r.agentClient(id, sequence, state.artifactDirs, workspace, artifactsDir, operations, request.WritePaths, tools) client.OnSession = func(session string, active bool) error { c, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() @@ -1033,7 +1091,7 @@ func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, na err = fmt.Errorf("Pi is not configured on execution server") return } - piClient = &stage.PiClient{Executable: profile.Executable, Directory: workspace, Tools: tools, WritePaths: request.WritePaths, ArtifactsDir: artifactsDir} + piClient = r.piClient(id, sequence, state.artifactDirs, profile.Executable, workspace, artifactsDir, request.WritePaths, tools) piClient.OnOwnership = func(session, reference string, active bool) error { c, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() @@ -1148,6 +1206,9 @@ func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, na state.recordChanges(step.FileChanges, step.Artifacts) if step.Status == "completed" { state.completed++ + if artifactsDir != "" { + state.artifactDirs = append(state.artifactDirs, strconv.Itoa(sequence)) + } } // Step finished: report its outcome, routing target, diagnostic and the // artifacts recorded for the visit, plus the run's running step count. diff --git a/internal/stage/agent.go b/internal/stage/agent.go index e0c3974d..d3109d20 100644 --- a/internal/stage/agent.go +++ b/internal/stage/agent.go @@ -14,13 +14,16 @@ import ( // before any prompt is admitted, so restart recovery can interrupt the agent. type AgentClient struct { OpenCodeClient - WritePaths []string - Tools []string // nil is the legacy manual approval bundle; empty denies all - OnSession func(string, bool) error - SessionID string - Transcript string - Policy ToolPolicy - WorkStarted bool + WritePaths []string + Tools []string // nil is the legacy manual approval bundle; empty denies all + // ArtifactAccess is supplied only by the runner. Write is this visit's + // directory; Read contains this and completed earlier visits in the same run. + ArtifactAccess ArtifactAccess + OnSession func(string, bool) error + SessionID string + Transcript string + Policy ToolPolicy + WorkStarted bool } func (c OpenCodeClient) Interrupt(ctx context.Context, id string) error { diff --git a/internal/stage/pi.go b/internal/stage/pi.go index 6d93e4d7..3e0b1bf1 100644 --- a/internal/stage/pi.go +++ b/internal/stage/pi.go @@ -28,6 +28,7 @@ type PiClient struct { // ArtifactsDir is exported to the spawned Pi process as // PIPELINES_ARTIFACTS_DIR so its tools can write stage artifacts. ArtifactsDir string + ArtifactAccess ArtifactAccess Policy ToolPolicy OnSession func(string, bool) error OnOwnership func(string, string, bool) error @@ -128,7 +129,7 @@ func (c *PiClient) Generate(ctx context.Context, config ModelConfig, prompt stri if err != nil { return "", err } - policy := map[string]any{"directory": c.Directory, "tools": tools, "paths": c.WritePaths, "edit": c.Policy.Edit, "shell": c.Policy.Shell} + policy := map[string]any{"directory": c.Directory, "tools": tools, "paths": c.WritePaths, "edit": c.Policy.Edit, "shell": c.Policy.Shell, "artifact_write": c.ArtifactAccess.Write, "artifact_read": c.ArtifactAccess.Read} if c.WritePaths == nil { policy["paths"] = []string{} } diff --git a/internal/stage/pi_policy.mjs b/internal/stage/pi_policy.mjs index aec60e93..03be3440 100644 --- a/internal/stage/pi_policy.mjs +++ b/internal/stage/pi_policy.mjs @@ -8,6 +8,24 @@ export function permit(policy, tool, args) { if (policy.shell !== null && !policy.shell.includes(args.command)) return "Shell command is not permitted"; return null; } + const artifact = (absolute, write) => { + const permitted = write ? [policy.artifact_write] : policy.artifact_read; + for (const directory of permitted) { + if (typeof directory !== "string" || !path.isAbsolute(directory)) continue; + let parent = absolute; + while (!fs.existsSync(parent)) { + const next = path.dirname(parent); + if (next === parent) break; + parent = next; + } + if (!fs.existsSync(parent)) continue; + const resolved = fs.realpathSync(parent); + const target = path.join(resolved, path.relative(parent, absolute)); + const root = fs.realpathSync(directory); + if (target === root || target.startsWith(root + path.sep)) return true; + } + return false; + }; if (tool === "edit" || tool === "write") { if (typeof args.path !== "string") return "Missing file path"; if (args.path.startsWith("~") || args.path.startsWith("@")) return "Use a literal path, not a harness path shortcut"; @@ -20,7 +38,10 @@ export function permit(policy, tool, args) { } const resolved = fs.realpathSync(parent); const root = fs.realpathSync(policy.directory); - if (resolved !== root && !resolved.startsWith(root + path.sep)) return "Edit escapes run directory"; + if (resolved !== root && !resolved.startsWith(root + path.sep)) { + if (!artifact(absolute, true)) return "Edit escapes run directory"; + return null; + } const relative = path.relative(path.resolve(policy.directory), absolute).split(path.sep).join("/"); const canonical = path.relative(root, path.join(resolved, path.relative(parent, absolute))).split(path.sep).join("/"); if (relative.startsWith("../") || path.isAbsolute(relative)) return "Edit escapes run directory"; @@ -31,9 +52,19 @@ export function permit(policy, tool, args) { }); if (policy.edit !== null && (!matches(relative) || !matches(canonical))) return "Edit is outside stage patterns"; } - if (tool === "read" && typeof args.path === "string") { + if (["read", "grep", "find"].includes(tool) && typeof args.path === "string") { const name = path.basename(args.path); if (/^(.*\.env(?:\..*)?|pipelines\.db.*)$/.test(name)) return "Protected file cannot be read"; + const absolute = path.resolve(policy.directory, args.path); + let parent = absolute; + while (!fs.existsSync(parent)) { + const next = path.dirname(parent); + if (next === parent) return "Cannot resolve read path"; + parent = next; + } + const resolved = fs.realpathSync(parent); + const root = fs.realpathSync(policy.directory); + if (resolved !== root && !resolved.startsWith(root + path.sep) && !artifact(absolute, false)) return "Read escapes run directory"; } return null; } diff --git a/internal/stage/pi_policy_test.go b/internal/stage/pi_policy_test.go new file mode 100644 index 00000000..6b1018a6 --- /dev/null +++ b/internal/stage/pi_policy_test.go @@ -0,0 +1,72 @@ +package stage + +import ( + "encoding/json" + "os" + "os/exec" + "path/filepath" + "testing" +) + +func TestPiPolicyArtifactScope(t *testing.T) { + if _, err := exec.LookPath("node"); err != nil { + t.Skip("node is required to exercise the Pi policy extension") + } + root := t.TempDir() + workspace := filepath.Join(root, "workspace") + current := filepath.Join(root, "artifacts [7]", "2") + prior := filepath.Join(root, "artifacts [7]", "1") + sibling := filepath.Join(root, "artifacts [7]", "3") + for _, directory := range []string{workspace, current, prior, sibling} { + if err := os.MkdirAll(directory, 0700); err != nil { + t.Fatal(err) + } + } + policy := map[string]any{ + "directory": workspace, + "tools": []string{"read", "grep", "find", "edit"}, + "paths": []string{}, + "edit": nil, + "shell": nil, + "artifact_write": current, + "artifact_read": []string{current, prior}, + } + policyJSON, err := json.Marshal(policy) + if err != nil { + t.Fatal(err) + } + policyFile := filepath.Join(root, "policy.mjs") + if err := os.WriteFile(policyFile, piPolicy, 0600); err != nil { + t.Fatal(err) + } + script := `const { permit } = await import(process.argv[1]); +const policy = JSON.parse(process.argv[2]); +const checks = JSON.parse(process.argv[3]); +console.log(JSON.stringify(checks.map(([tool, path]) => permit(policy, tool, {path}))));` + checks := [][2]string{ + {"edit", filepath.Join(current, "report.md")}, + {"edit", filepath.Join(prior, "report.md")}, + {"read", filepath.Join(prior, "report.md")}, + {"grep", filepath.Join(prior, "report.md")}, + {"find", filepath.Join(current, "reports")}, + {"read", filepath.Join(sibling, "report.md")}, + {"grep", filepath.Join(root, "other-run", "1", "report.md")}, + {"find", filepath.Join(current, "..", "3", "report.md")}, + {"read", filepath.Join(prior, ".env")}, + } + checksJSON, err := json.Marshal(checks) + if err != nil { + t.Fatal(err) + } + output, err := exec.Command("node", "--input-type=module", "--eval", script, "file://"+policyFile, string(policyJSON), string(checksJSON)).Output() + if err != nil { + t.Fatal(err) + } + var got []any + if err := json.Unmarshal(output, &got); err != nil { + t.Fatal(err) + } + if len(got) != len(checks) || got[0] != nil || got[1] == nil || got[2] != nil || got[3] != nil || got[4] != nil || got[5] == nil || got[6] == nil || got[7] == nil || got[8] == nil { + t.Fatalf("Pi artifact policy results = %#v", got) + } +} diff --git a/internal/stage/tools.go b/internal/stage/tools.go index 566519cb..77477987 100644 --- a/internal/stage/tools.go +++ b/internal/stage/tools.go @@ -7,6 +7,13 @@ import ( "strings" ) +// ArtifactAccess is runtime-only, server-derived access to artifact visits. +// Empty fields deliberately grant nothing outside the workspace. +type ArtifactAccess struct { + Write string + Read []string +} + // ToolNames validates model-stage permissions, not general graph semantics. // A present empty block explicitly denies all tools. func ToolNames(def Definition) ([]string, bool, error) { @@ -128,12 +135,39 @@ func (c *AgentClient) permissionRules() []map[string]string { } } } + // OpenCode V2 resolves an external file to its containing directory and + // checks external_directory against /*. Do not grant a parent: + // each visit is deliberately an independent capability. + readArtifacts := false + writeArtifacts := false + for _, tool := range tools { + readArtifacts = readArtifacts || tool == "read" || tool == "glob" || tool == "grep" + writeArtifacts = writeArtifacts || tool == "edit" + } + if readArtifacts { + for _, directory := range c.ArtifactAccess.Read { + if artifactDirectory(directory) { + allow("external_directory", filepath.ToSlash(filepath.Join(directory, "*"))) + } + } + } + if writeArtifacts && artifactDirectory(c.ArtifactAccess.Write) { + allow("external_directory", filepath.ToSlash(filepath.Join(c.ArtifactAccess.Write, "*"))) + allow("edit", filepath.ToSlash(filepath.Join(c.ArtifactAccess.Write, "*"))) + } + for _, pattern := range []string{"../*", "*/../*"} { + rules = append(rules, map[string]string{"action": "external_directory", "resource": pattern, "effect": "deny"}) + } for _, pattern := range []string{"*.env", "*.env.*", "pipelines.db*"} { rules = append(rules, map[string]string{"action": "read", "resource": pattern, "effect": "deny"}) } return rules } +func artifactDirectory(directory string) bool { + return directory != "" && filepath.IsAbs(directory) && filepath.Clean(directory) == directory && !strings.ContainsAny(directory, "*?\\") +} + func matchesEditPolicy(patterns []string, path string) bool { for _, pattern := range patterns { // OpenCode resource wildcards match across directory separators. diff --git a/internal/stage/tools_test.go b/internal/stage/tools_test.go index fa561166..ed823bc1 100644 --- a/internal/stage/tools_test.go +++ b/internal/stage/tools_test.go @@ -139,3 +139,38 @@ func TestDefaultEditScopeIsLanguageIndependentRunDirectory(t *testing.T) { t.Fatal("external directories were enabled") } } + +func TestArtifactPermissionsAreExactAndToolScoped(t *testing.T) { + current := "/artifacts/run [7]/2" + prior := "/artifacts/run [7]/1" + sibling := "/artifacts/run 7/3" + otherRun := "/artifacts/run 8/1" + client := &AgentClient{OpenCodeClient: OpenCodeClient{Directory: "/repo"}, Tools: []string{"read", "edit"}, ArtifactAccess: ArtifactAccess{Write: current, Read: []string{current, prior}}} + rules := client.permissionRules() + for _, path := range []string{current + "/report.md", prior + "/report.md"} { + if permissionEffect(rules, "external_directory", path) != "allow" { + t.Fatalf("artifact directory denied: %q", path) + } + } + if permissionEffect(rules, "edit", current+"/report.md") != "allow" || permissionEffect(rules, "edit", prior+"/report.md") != "deny" { + t.Fatal("artifact writes were not limited to the current visit") + } + for _, path := range []string{sibling + "/report.md", otherRun + "/report.md", current + "/../3/report.md", "/artifacts/run 7/20/report.md"} { + if permissionEffect(rules, "external_directory", path) != "deny" { + t.Fatalf("unapproved artifact scope allowed: %q", path) + } + } + client.Tools = []string{} + if permissionEffect(client.permissionRules(), "external_directory", current+"/report.md") != "deny" { + t.Fatal("no-tool stage gained artifact access") + } + client.Tools = []string{"read"} + rules = client.permissionRules() + if permissionEffect(rules, "external_directory", prior+"/report.md") != "allow" || permissionEffect(rules, "edit", current+"/report.md") != "deny" { + t.Fatal("read-only artifact stage gained write access") + } + client.ArtifactAccess = ArtifactAccess{} + if permissionEffect(client.permissionRules(), "external_directory", current+"/report.md") != "deny" { + t.Fatal("missing artifact scope enabled an external directory") + } +}