diff --git a/android/app/src/main/java/net/awan/pipelines/data/PipelinesRepository.kt b/android/app/src/main/java/net/awan/pipelines/data/PipelinesRepository.kt index ab08e775..a8e163d2 100644 --- a/android/app/src/main/java/net/awan/pipelines/data/PipelinesRepository.kt +++ b/android/app/src/main/java/net/awan/pipelines/data/PipelinesRepository.kt @@ -13,6 +13,8 @@ import net.awan.pipelines.data.protocol.ChatPromptRequest import net.awan.pipelines.data.protocol.ChatProtocol import net.awan.pipelines.data.protocol.ChatQuery import net.awan.pipelines.data.protocol.ChatSession +import net.awan.pipelines.data.protocol.ChatShellOutputPage +import net.awan.pipelines.data.protocol.ChatShell import net.awan.pipelines.data.protocol.Message import net.awan.pipelines.data.protocol.Graph import net.awan.pipelines.data.protocol.PipelineDetail @@ -312,6 +314,22 @@ class PipelinesRepository( return response.chatHistory ?: ChatHistoryPage(messages = response.chatMessages ?: emptyList()) } + /** Reads one bounded page from a verified, currently observable shell. */ + suspend fun chatShellOutput(sessionId: String, shellId: String, cursor: Long = 0, limit: Int = ChatProtocol.SHELL_OUTPUT_LIMIT): ChatShellOutputPage { + if (sessionId.isBlank() || shellId.isBlank()) throw ProtocolException("Select a shell first.") + if (cursor < 0 || limit <= 0 || limit > ChatProtocol.SHELL_OUTPUT_LIMIT) { + throw ProtocolException("Shell output bounds are invalid.") + } + return client.call(ChatProtocol.shellOutputRequest(sessionId, shellId, cursor, limit)).chatShellOutput + ?: throw ProtocolException("The server returned no shell output page.") + } + + /** Lists currently observed shells owned by this operator chat session. */ + suspend fun chatShells(sessionId: String): List { + if (sessionId.isBlank()) throw ProtocolException("Select a chat session first.") + return client.call(ChatProtocol.shellsRequest(sessionId)).chatShells ?: emptyList() + } + /** * Attaches this connection to [sessionId]'s shared live stream, so it sees * every frame any client's turn produces — the other devices' prompts and the diff --git a/android/app/src/main/java/net/awan/pipelines/data/protocol/Chat.kt b/android/app/src/main/java/net/awan/pipelines/data/protocol/Chat.kt index 0390de90..5483dd5c 100644 --- a/android/app/src/main/java/net/awan/pipelines/data/protocol/Chat.kt +++ b/android/app/src/main/java/net/awan/pipelines/data/protocol/Chat.kt @@ -204,6 +204,24 @@ data class ChatShell( val exit: Int? = null, ) +@Serializable +data class ChatShellOutputQuery( + @SerialName("session_id") val sessionId: String = "", + @SerialName("shell_id") val shellId: String = "", + val cursor: Long = 0, + val limit: Int = ChatProtocol.SHELL_OUTPUT_LIMIT, +) + +@Serializable +data class ChatShellOutputPage( + val shell: ChatShell = ChatShell(), + val output: String = "", + val cursor: Long = 0, + val size: Long = 0, + val truncated: Boolean = false, + val unavailable: Boolean = false, +) + @Serializable data class ChatStep( val status: String = "", @@ -285,6 +303,7 @@ object ChatProtocol { const val SHELL_RUNNING = "running" const val SHELL_EXITED = "exited" + const val SHELL_OUTPUT_LIMIT = 64 * 1024 /** * A prompt another client sent, echoed live to this client so it appears in @@ -362,6 +381,15 @@ object ChatProtocol { fun interruptRequest(sessionId: String): Message = Message(type = Protocol.CHAT_INTERRUPT, chatQuery = ChatQuery(sessionId = sessionId)) + fun shellOutputRequest(sessionId: String, shellId: String, cursor: Long = 0, limit: Int = SHELL_OUTPUT_LIMIT): Message = + Message( + type = Protocol.CHAT_SHELL_OUTPUT, + chatShellOutputQuery = ChatShellOutputQuery(sessionId, shellId, cursor, limit), + ) + + fun shellsRequest(sessionId: String): Message = + Message(type = Protocol.CHAT_SHELLS, chatQuery = ChatQuery(sessionId = sessionId)) + /** * The `chat_close` frame that finishes [sessionId]: the server stops any * running turn (through the same interrupt) and records the session closed on diff --git a/android/app/src/main/java/net/awan/pipelines/data/protocol/Message.kt b/android/app/src/main/java/net/awan/pipelines/data/protocol/Message.kt index b9845ea7..7014e6d3 100644 --- a/android/app/src/main/java/net/awan/pipelines/data/protocol/Message.kt +++ b/android/app/src/main/java/net/awan/pipelines/data/protocol/Message.kt @@ -79,6 +79,8 @@ object Protocol { // The operator's directory choices for a new chat (home first, then the // directories of previous operator sessions, most recently used first). const val CHAT_DIRECTORIES = "chat_directories" + const val CHAT_SHELLS = "chat_shells" + const val CHAT_SHELL_OUTPUT = "chat_shell_output" // Server-pushed frames with no matching request. const val CHAT_EVENT = "chat_event" const val CHAT_PERMISSION = "chat_permission" @@ -712,4 +714,7 @@ data class Message( // close/reopen requests themselves ride on chat_query like chat_interrupt. @SerialName("chat_close") val chatClose: ChatCloseResult? = null, @SerialName("chat_directories") val chatDirectories: List? = null, + @SerialName("chat_shell_output_query") val chatShellOutputQuery: ChatShellOutputQuery? = null, + @SerialName("chat_shells") val chatShells: List? = null, + @SerialName("chat_shell_output") val chatShellOutput: ChatShellOutputPage? = null, ) diff --git a/android/app/src/test/java/net/awan/pipelines/data/protocol/ChatProtocolTest.kt b/android/app/src/test/java/net/awan/pipelines/data/protocol/ChatProtocolTest.kt index ca5412c8..fa170210 100644 --- a/android/app/src/test/java/net/awan/pipelines/data/protocol/ChatProtocolTest.kt +++ b/android/app/src/test/java/net/awan/pipelines/data/protocol/ChatProtocolTest.kt @@ -405,6 +405,36 @@ class ChatProtocolTest { assertEquals(1792112367999L, decodedReply.chatClose!!.closedAt) } + @Test + fun shellOutputRequestAndPageUseBoundedWireNames() { + val request = ChatProtocol.shellOutputRequest("ses_1", "sh_1", 7, 128) + val decoded = PipelinesJson.decodeFromString(Message.serializer(), PipelinesJson.encodeToString(Message.serializer(), request)) + assertEquals(Protocol.CHAT_SHELL_OUTPUT, decoded.type) + assertEquals("ses_1", decoded.chatShellOutputQuery!!.sessionId) + assertEquals("sh_1", decoded.chatShellOutputQuery!!.shellId) + assertEquals(7L, decoded.chatShellOutputQuery!!.cursor) + assertEquals(128, decoded.chatShellOutputQuery!!.limit) + + val page = PipelinesJson.decodeFromString(Message.serializer(), """ + {"type":"chat_shell_output","chat_shell_output":{"shell":{"id":"sh_1","status":"running"},"output":"marker","cursor":13,"size":20,"truncated":true}} + """.trimIndent()) + assertEquals("marker", page.chatShellOutput!!.output) + assertTrue(page.chatShellOutput!!.truncated) + } + + @Test + fun shellsRequestUsesSessionQueryAndPreservesListWireName() { + val request = ChatProtocol.shellsRequest("ses_1") + val decoded = PipelinesJson.decodeFromString(Message.serializer(), PipelinesJson.encodeToString(Message.serializer(), request)) + assertEquals(Protocol.CHAT_SHELLS, decoded.type) + assertEquals("ses_1", decoded.chatQuery!!.sessionId) + val reply = PipelinesJson.decodeFromString(Message.serializer(), """ + {"type":"chat_shells","chat_shells":[{"id":"sh_1","status":"running","command":"printf marker"}]} + """.trimIndent()) + assertEquals("sh_1", reply.chatShells!!.single().id) + assertEquals(ChatProtocol.SHELL_RUNNING, reply.chatShells.single().status) + } + private fun toolDuration(part: ChatPart): Long? { val time = part.time ?: return null return time.completed - time.ran diff --git a/cmd/pipelines-large-chat-fixture/main.go b/cmd/pipelines-large-chat-fixture/main.go index d0e4ee73..17b10e8e 100644 --- a/cmd/pipelines-large-chat-fixture/main.go +++ b/cmd/pipelines-large-chat-fixture/main.go @@ -29,6 +29,7 @@ import ( func main() { addr := flag.String("addr", "127.0.0.1:0", "Pipelines scratch listen address") scratch := flag.String("scratch-dir", "", "required scratch directory for the database and workspace") + exited := flag.Bool("shell-exited", false, "serve the selected fixture shell as observed exited") flag.Parse() if *scratch == "" { fatal(fmt.Errorf("-scratch-dir is required")) @@ -46,7 +47,7 @@ func main() { } defer upstream.Close() upstreamURL := "http://" + upstream.Addr().String() - upstreamServer := &http.Server{Handler: server.LargeChatFixtureHandler(workspace), ReadHeaderTimeout: 5 * time.Second} + upstreamServer := &http.Server{Handler: server.LargeChatFixtureHandlerWithShell(workspace, *exited), ReadHeaderTimeout: 5 * time.Second} go func() { _ = upstreamServer.Serve(upstream) }() ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) @@ -76,17 +77,85 @@ func main() { if err != nil { fatal(err) } + shellReadbacks, err := verifyShellReadback(ctx, endpoint, *exited) + if err != nil { + fatal(err) + } + <-ctx.Done() + shutdown, stop := context.WithTimeout(context.Background(), 5*time.Second) + pipelinesErr := pipelinesServer.Shutdown(shutdown) + upstreamErr := upstreamServer.Shutdown(shutdown) + stop() + if pipelinesErr != nil { + fatal(pipelinesErr) + } + if upstreamErr != nil { + fatal(upstreamErr) + } info := buildinfo.Current() _ = json.NewEncoder(os.Stdout).Encode(map[string]any{ "endpoint": endpoint, "upstream_endpoint": upstreamURL, "backend": attached.Connection.Backend, "revision": info.Commit, "session_id": server.LargeChatFixtureSessionID, "workspace": workspace, "newest_readback": readbacks[0], "older_readback": readbacks[1], + "shell_readbacks": shellReadbacks, "clean_shutdown": true, "shutdown_exit_status": 0, }) - <-ctx.Done() - shutdown, stop := context.WithTimeout(context.Background(), 5*time.Second) - defer stop() - _ = pipelinesServer.Shutdown(shutdown) - _ = upstreamServer.Shutdown(shutdown) +} + +func verifyShellReadback(ctx context.Context, endpoint string, exited bool) (map[string]any, error) { + evidence := map[string]any{"test_id": "fixture-shell-owned-and-foreign-refusal", "negative": []string{}} + ws, _, err := websocket.Dial(ctx, "ws"+strings.TrimPrefix(endpoint, "http")+"/connect", nil) + if err != nil { + return evidence, err + } + defer ws.Close(websocket.StatusNormalClosure, "fixture complete") + if err := wsjson.Write(ctx, ws, protocol.Message{Type: protocol.ChatShells, ID: "fixture-shells", ChatQuery: &protocol.ChatQuery{SessionID: server.LargeChatFixtureSessionID}}); err != nil { + return evidence, err + } + var listed protocol.Message + if listed, err = readFixtureResponse(ctx, ws, protocol.ChatShells); err != nil { + return evidence, err + } + if !exited && (len(listed.ChatShells) != 1 || listed.ChatShells[0].ID != server.LargeChatFixtureShellID) { + return evidence, fmt.Errorf("fixture shell is not listed") + } + if err := wsjson.Write(ctx, ws, protocol.Message{Type: protocol.ChatShellOutput, ID: "fixture-output", ChatShellOutputQuery: &protocol.ChatShellOutputQuery{SessionID: server.LargeChatFixtureSessionID, ShellID: server.LargeChatFixtureShellID, Limit: 64 << 10}}); err != nil { + return evidence, err + } + page, err := readFixtureResponse(ctx, ws, protocol.ChatShellOutput) + if err != nil { + return evidence, err + } + if page.ChatShellOutput == nil || page.ChatShellOutput.Output != server.LargeChatFixtureShellOutput || page.ChatShellOutput.Shell.Status != map[bool]string{true: "exited", false: "running"}[exited] { + return evidence, fmt.Errorf("unexpected shell readback: %+v", page.ChatShellOutput) + } + owned := map[string]any{"shell_id": page.ChatShellOutput.Shell.ID, "output": page.ChatShellOutput.Output, "cursor": page.ChatShellOutput.Cursor, "status": page.ChatShellOutput.Shell.Status} + if page.ChatShellOutput.Shell.Exit != nil { + owned["exit"] = *page.ChatShellOutput.Shell.Exit + } + evidence["owned"] = owned + negative := evidence["negative"].([]string) + negativeRequests := []protocol.Message{ + {Type: protocol.ChatShells, ID: "foreign-list", ChatQuery: &protocol.ChatQuery{SessionID: server.LargeChatFixtureForeignSessionID}}, + {Type: protocol.ChatShells, ID: "unknown-list", ChatQuery: &protocol.ChatQuery{SessionID: "ses_unregistered_fixture"}}, + {Type: protocol.ChatShellOutput, ID: "foreign-output", ChatShellOutputQuery: &protocol.ChatShellOutputQuery{SessionID: server.LargeChatFixtureSessionID, ShellID: server.LargeChatFixtureForeignShellID, Limit: 64 << 10}}, + {Type: protocol.ChatShellOutput, ID: "unknown-output", ChatShellOutputQuery: &protocol.ChatShellOutputQuery{SessionID: "ses_unregistered_fixture", ShellID: server.LargeChatFixtureShellID, Limit: 64 << 10}}, + } + for _, request := range negativeRequests { + if err := wsjson.Write(ctx, ws, request); err != nil { + return evidence, err + } + var response protocol.Message + if err := wsjson.Read(ctx, ws, &response); err != nil { + return evidence, err + } + if response.Type != protocol.Error || strings.Contains(response.Error, server.LargeChatFixtureForeignShellOutput) { + return evidence, fmt.Errorf("negative request %s leaked or succeeded: %+v", request.ID, response) + } + negative = append(negative, request.ID) + } + evidence["negative"] = negative + evidence["negative_count"] = len(negative) + return evidence, nil } func verifyBroker(ctx context.Context, endpoint string) ([2]map[string]any, error) { diff --git a/docs/notes/background-task-output-api.md b/docs/notes/background-task-output-api.md new file mode 100644 index 00000000..dd705034 --- /dev/null +++ b/docs/notes/background-task-output-api.md @@ -0,0 +1,40 @@ +# Background task output API + +Changed: 2026-10-18. The chat websocket now has `chat_shells` for current, +operator-session-owned running shells and `chat_shell_output` for a selected, +bounded (64 KiB) absolute-byte output page. Ownership requires both the +operator-session registry and provider shell metadata `sessionID`; a shared cwd +is not evidence. Output reads first verify provider detail and explicitly mark +removed/provider-unavailable shells unavailable. This is not completed-task +history or restart retention. + +Stale shared documentation: the packaged OpenCode operational references under +`internal/author/skill/references/` need a later docs-fold update describing the +two additive websocket messages, 64 KiB maximum page, and current-only history. + +Fixture evidence (2026-10-18): `go build -o /pipelines-large-chat-fixture +./cmd/pipelines-large-chat-fixture`; then `/pipelines-large-chat-fixture +-scratch-dir /run -shell-exited`, stopped with SIGTERM and awaited. The +foreground run exercised authenticated `chat_sessions`, three reconnecting history +pages, `chat_shells`, bounded `chat_shell_output`, and four executable negative +readbacks (`foreign-list`, `unknown-list`, `foreign-output`, `unknown-output`) against +scratch DB/workspace only. Its JSON contains `clean_shutdown: true` and +`shutdown_exit_status: 0`, plus the selected exited shell's `exit: 0` readback. +The evidence revision was `ded79277783b6ccdc25eb4ea25fe55d8632b89c7`; session +`ses_large_chat_fixture`, shell `sh_large_chat_fixture`, and fixed marker +`fixture shell output marker\n`. The current list remains running-shell discovery. + +Integration coverage added: `TestChatShellOutputRejectsBoundsBeforeRegistryOrProvider`, +`TestChatShellOutputRealHandlerWebsocketOwnedAndForeign`, and +`TestChatShellOutputRealHandlerExitedReadback`, +`TestChatShellOutputRealHandlerUnavailableAndReconnect`, and +`TestChatShellOutputRejectsMalformedAndOversizedProviderPages`, and +`TestChatShellOutputRealHandlerCancellationStopsProvider` in +`internal/server/chat_shell_output_test.go`. Run with +`go test ./internal/server -run 'TestChatShellOutput' -count=1 -v` and +`go test -race ./internal/server -run 'TestChatShellOutput(RealHandler|RejectsBounds)' -count=1 -v`. +These tests use an authenticated in-process HTTP provider, a scratch SQLite operator +registry, and real `/connect` websocket traffic. The fixture executes foreign +session/shell refusal requests and records positive/negative JSON readbacks under +`shell_readbacks`; it remains scratch-only evidence, not evidence of a real provider, +model, production workspace, or device. diff --git a/docs/notes/inspected-resource-recovery.md b/docs/notes/inspected-resource-recovery.md new file mode 100644 index 00000000..b3986890 --- /dev/null +++ b/docs/notes/inspected-resource-recovery.md @@ -0,0 +1,19 @@ +# Inspected resource recovery + +The operator recovery contract now keeps quarantined resource ownership and its +audit transactionally coupled to FIFO handoff. Inspection is redacted and +reconciliation is accepted only over a loopback operator connection with the +exact owner visit plus positive cleanup/restoration evidence. The persisted +capability remains private to the authoritative live store; missing waiter +channels fail closed without changing owner or audit state. + +Use the configured `PIPELINES_URL` endpoint, not a disposable URL. First run +the redacted resource inspection, then reconcile only after the parked visit +has been inspected and quiesced, supplying its exact run/stage/sequence and +explicit evidence (including `no mutating action occurred` where applicable). +The old deployed server does not support this action; deploy a compatible +server before attempting recovery. This is an operator trust boundary, not an +OS sandbox or unrestricted authorization grant. + +The shared README and packaged OpenCode skill references should be updated by +the manual `docs-fold` process; they are intentionally not edited here. diff --git a/docs/notes/retry-from-failed-stage.md b/docs/notes/retry-from-failed-stage.md new file mode 100644 index 00000000..57dd006d --- /dev/null +++ b/docs/notes/retry-from-failed-stage.md @@ -0,0 +1,11 @@ +# Retry from failed stage + +As of 2026-10-18, retry requests default to `resume`: the server records a +durable checkpoint for the terminal failed visit and starts the new attempt at +that visit with its completed-prefix handoff context. `--mode=restart` retains +the older full-restart behavior. Older clients that omit the mode therefore +resume rather than restart. + +The CLI and protocol now expose the mode and retry provenance. The shared CLI +documentation and packaged skill references should be folded from this note by +the manual `docs-fold` pipeline. diff --git a/internal/cli/app.go b/internal/cli/app.go index 5e26e393..cadada39 100644 --- a/internal/cli/app.go +++ b/internal/cli/app.go @@ -7,6 +7,7 @@ import ( "flag" "fmt" "io" + "net/url" "os" "os/exec" "path/filepath" @@ -36,7 +37,8 @@ type App struct { type options struct { harness string url, name, prompt, title, template, workspace, paths, key, opencode, model string - option, feedback string + retryMode string + option, feedback, evidence, actor string output string conflict string kind string @@ -45,7 +47,9 @@ type options struct { keepSet bool all bool json, source, noOpen, yes, allow, createOnly, force, wait bool - version, expected, limit int + version, expected, limit, recoverySequence int + recoveryRun int64 + recoveryStage, resource string timeout, waitTimeout time.Duration stageModels stageModelFlags } @@ -159,6 +163,7 @@ func (a App) flags(command string) (*flag.FlagSet, *options) { f.StringVar(&o.key, "request-key", "", "Idempotency key (default: generated unique key)") f.BoolVar(&o.wait, "wait", false, "Wait for the new run to finish") f.DurationVar(&o.waitTimeout, "wait-timeout", 0, "Client limit for --wait (0 waits until the run ends); independent of --timeout") + f.StringVar(&o.retryMode, "mode", "resume", "Retry mode: resume failed stage or restart from entry") case "cancel", "draft-delete": f.BoolVar(&o.yes, "yes", false, "Confirm this action without an interactive prompt") case "retention": @@ -177,6 +182,13 @@ func (a App) flags(command string) (*flag.FlagSet, *options) { case "decide": f.StringVar(&o.option, "option", "", "Option key to choose among the waiting stage's edges (required)") f.StringVar(&o.feedback, "feedback", "", "Optional free-text feedback recorded with the decision") + case "resource-inspect": + f.StringVar(&o.resource, "resource", "", "Exclusive resource key") + case "resource-reconcile": + f.StringVar(&o.evidence, "evidence", "", "Bounded positive cleanup/restoration evidence (required)") + f.StringVar(&o.actor, "actor", "operator", "Bounded operator identity") + f.BoolVar(&o.allow, "no-mutating-action", false, "Evidence explicitly records that no mutating action occurred") + f.BoolVar(&o.yes, "yes", false, "Confirm reconciliation without prompting") case "debug": f.StringVar(&o.harness, "harness", "", "Debugging harness: configured opencode/pi, or nvim") f.BoolVar(&o.yes, "yes", false, "Approve opening a debugging session without prompting") @@ -462,8 +474,16 @@ func (a App) Run(ctx context.Context, args []string) error { if requestTimeout <= 0 { requestTimeout = 30 * time.Second } + connectURL := o.url + if command == "resource-inspect" || command == "resource-reconcile" { + var transportErr error + connectURL, transportErr = operatorTransportURL(o.url) + if transportErr != nil { + return transportErr + } + } dialCtx, cancel := context.WithTimeout(ctx, requestTimeout) - c, err := client.Open(dialCtx, o.url) + c, err := client.Open(dialCtx, connectURL) cancel() if err != nil { if command == "status" && len(pos) == 0 { @@ -897,6 +917,51 @@ func (a App) Run(ctx context.Context, args []string) error { } } return fmt.Errorf("template %q not found", key) + case "resource-inspect": + if len(pos) > 1 { + return fmt.Errorf("resource-inspect takes one resource key") + } + if o.resource == "" && len(pos) == 1 { + o.resource = pos[0] + } + if strings.TrimSpace(o.resource) == "" { + return fmt.Errorf("resource-inspect requires a resource key") + } + r, err := call(protocol.Message{Type: protocol.InspectResource, Resource: o.resource}) + if err != nil { + return err + } + if r.ResourceInspection == nil { + return fmt.Errorf("server did not return resource inspection") + } + return a.output(r.ResourceInspection) + case "resource-reconcile": + if len(pos) != 4 { + return fmt.Errorf("resource-reconcile requires resource, run-id, stage and sequence") + } + runID, err := strconv.ParseInt(pos[1], 10, 64) + if err != nil || runID <= 0 { + return fmt.Errorf("run ID must be positive") + } + sequence, err := strconv.Atoi(pos[3]) + if err != nil || sequence <= 0 { + return fmt.Errorf("sequence must be positive") + } + if strings.TrimSpace(o.evidence) == "" { + return fmt.Errorf("--evidence is required") + } + if err := a.confirmAction(fmt.Sprintf("Reconcile quarantined resource %s for run #%d?", pos[0], runID), o.yes); err != nil { + return err + } + r, err := call(protocol.Message{Type: protocol.ReconcileResource, Resource: pos[0], ResourceRecovery: &protocol.ResourceRecoveryRequest{Resource: pos[0], RunID: runID, Stage: pos[2], Sequence: sequence, Actor: o.actor, Evidence: &protocol.ResourceRecoveryEvidence{CleanupConfirmed: true, NoMutatingAction: o.allow, Summary: o.evidence}}}) + if err != nil { + return err + } + if o.json { + return a.output(r) + } + _, err = fmt.Fprintf(a.Out, "Reconciled resource %s for run #%d\n", pos[0], runID) + return err case "inspect", "cancel", "retry": value, err := one() if err != nil { @@ -920,11 +985,14 @@ func (a App) Run(ctx context.Context, args []string) error { } typeName = protocol.RetryRun key = o.key + if o.retryMode != "resume" && o.retryMode != "restart" { + return fmt.Errorf("--mode must be resume or restart") + } if key == "" { key = fmt.Sprintf("retry-%d-%d", os.Getpid(), time.Now().UnixNano()) } } - r, err := call(protocol.Message{Type: typeName, RunID: id, RequestKey: key}) + r, err := call(protocol.Message{Type: typeName, RunID: id, RequestKey: key, RetryMode: o.retryMode}) if err != nil { return err } @@ -1039,6 +1107,16 @@ func (a App) Run(ctx context.Context, args []string) error { return nil } +func operatorTransportURL(raw string) (string, error) { + u, err := url.Parse(raw) + if err != nil || u.Scheme == "" || u.Host == "" { + return "", fmt.Errorf("invalid configured pipeline URL") + } + u.Path = "/operator-connect" + u.RawPath = "" + return u.String(), nil +} + func (a App) output(value any) error { encoder := json.NewEncoder(a.Out) encoder.SetIndent("", " ") diff --git a/internal/cli/commands.go b/internal/cli/commands.go index a490e56c..f1e7f9fb 100644 --- a/internal/cli/commands.go +++ b/internal/cli/commands.go @@ -2,39 +2,41 @@ package cli // Help is also the public command inventory for agents and parity checks. var usage = map[string]string{ - "provider-check": "provider-check --model PROVIDER/MODEL [--yes] [--json]", - "new": "new [name] [prompt] [--template KEY] [--no-open | --json]", - "edit": "edit [name|id] [prompt] [--version N] [--no-open | --json]", - "drafts": "drafts [--json]", - "draft-delete": "draft-delete [--json]", - "validate": "validate [--json]", - "submit": "submit [--name NAME] [--create-only] [--yes] [--json]", - "list": "list [--json]", - "search": "search QUERY [--type KIND] [--limit N] [--json]", - "show": "show [--version N] [--json]", - "versions": "versions [--json]", - "export": "export [name|id ...] [--output FILE.zip] [--all] [--version N]", - "import": "import [FILE.zip] [--all] [--on-conflict ask|keep|both|import] [--yes] [--json]", - "templates": "templates [--json]", - "template": "template [--source | --json]", - "run": "run [name|id] [prompt] [--title TITLE] [--model PROVIDER/MODEL] [--stage-model STAGE=PROVIDER/MODEL ...] [--json [--wait [--wait-timeout DURATION]]]", - "runs": "runs [--pipeline NAME] [--json]", - "inspect": "inspect [--json]", - "wait": "wait [run-id ...] [--timeout DURATION] [--json]", - "cancel": "cancel [--json]", - "decide": "decide [stage] --option KEY [--feedback TEXT] [--json]", - "retry": "retry [--json [--wait [--wait-timeout DURATION]]]", - "title": "title \"\" [--json]", - "debug": "debug <run-id> [--json | --no-open]", - "chat-adopt": "chat-adopt <session-id> [--json]", - "connection": "connection [--json]", - "ping": "ping [--json]", - "version": "version [--json]", - "status": "status [--json]", - "stats": "stats [--json]", - "rpc": "rpc <request.json|->", - "retention": "retention report|apply [--older-than DURATION] [--keep N] [--db FILE] [--artifacts DIR] [--json] [--yes]", - "tui": "tui", + "provider-check": "provider-check --model PROVIDER/MODEL [--yes] [--json]", + "new": "new [name] [prompt] [--template KEY] [--no-open | --json]", + "edit": "edit [name|id] [prompt] [--version N] [--no-open | --json]", + "drafts": "drafts [--json]", + "draft-delete": "draft-delete <draft.md> [--json]", + "validate": "validate <draft.md> [--json]", + "submit": "submit <draft.md> [--name NAME] [--create-only] [--yes] [--json]", + "list": "list [--json]", + "search": "search QUERY [--type KIND] [--limit N] [--json]", + "show": "show <name|id> [--version N] [--json]", + "versions": "versions <name|id> [--json]", + "export": "export [name|id ...] [--output FILE.zip] [--all] [--version N]", + "import": "import [FILE.zip] [--all] [--on-conflict ask|keep|both|import] [--yes] [--json]", + "templates": "templates [--json]", + "template": "template <key> [--source | --json]", + "run": "run [name|id] [prompt] [--title TITLE] [--model PROVIDER/MODEL] [--stage-model STAGE=PROVIDER/MODEL ...] [--json [--wait [--wait-timeout DURATION]]]", + "runs": "runs [--pipeline NAME] [--json]", + "inspect": "inspect <run-id> [--json]", + "wait": "wait <run-id> [run-id ...] [--timeout DURATION] [--json]", + "cancel": "cancel <run-id> [--json]", + "decide": "decide <run-id> [stage] --option KEY [--feedback TEXT] [--json]", + "retry": "retry <run-id> [--mode resume|restart] [--json [--wait [--wait-timeout DURATION]]]", + "title": "title <run-id> \"<title>\" [--json]", + "debug": "debug <run-id> [--json | --no-open]", + "chat-adopt": "chat-adopt <session-id> [--json]", + "connection": "connection [--json]", + "ping": "ping [--json]", + "version": "version [--json]", + "status": "status [--json]", + "stats": "stats [--json]", + "resource-inspect": "resource-inspect <resource> [--json]", + "resource-reconcile": "resource-reconcile <resource> <run-id> <stage> <sequence> --evidence TEXT [--no-mutating-action] [--yes] [--json]", + "rpc": "rpc <request.json|->", + "retention": "retention report|apply [--older-than DURATION] [--keep N] [--db FILE] [--artifacts DIR] [--json] [--yes]", + "tui": "tui", } var notes = map[string]string{ @@ -49,7 +51,7 @@ var notes = map[string]string{ "validate": "Uses the same server parser as submission. Does not persist or activate schedules. Structural validity does not prove runtime/tool availability.", "submit": "Confirm publication at the prompt, or use --yes for explicit automation; schedules activate on acceptance. Managed drafts provide the name and base-version guard. --force never bypasses validation.", "run": "Runs in your current directory after action confirmation (--yes skips it). Select a pipeline and compose a multiline task with Ctrl+S. Stage tools are enabled automatically; no file list or typed RUN confirmation. Cancellation does not undo edits. --title sets an explicit run title; without it the title is derived from the prompt. --model PROVIDER/MODEL overrides the model for every stage of that run; --stage-model STAGE=PROVIDER/MODEL overrides one stage and is repeatable, with a repeated key last-wins. Precedence is per-stage > whole-run > the model pinned in the pipeline version; overrides apply to that run only and never change the pinned definition. An unknown stage key or a malformed provider/model is refused before the run starts, naming the valid stage keys. In JSON mode --wait polls until terminal; --wait-timeout bounds only the client wait after admission and is independent of per-request --timeout. A reached wait limit leaves the run active: inspect read-only with `inspect <run-id> --json`, or reattach with --wait reusing the original --request-key (never an automatic replay or cancel).", - "retry": "Start a new attempt of a failed run with the same initial input, directory, pipeline version, and edit scope. Original history is retained. Already-completed side effects are not undone. --wait/--wait-timeout behave as for run: a reached wait limit leaves the new run active; inspect read-only with `inspect <run-id> --json`, or reattach with --wait and the original --request-key instead of starting a duplicate retry.", + "retry": "Resume a failed stage by default, restoring the completed prefix handoff; use --mode=restart to start at entry. Original history is retained and already-completed side effects are not undone. --wait/--wait-timeout behave as for run: a reached wait limit leaves the new run active; inspect read-only or reattach with --wait and the original --request-key instead of starting a duplicate retry.", "title": "Rename a run's operator-facing title after the fact without touching its input, steps or execution. The title is collapsed to a single bounded line and recorded as explicitly set. `pipelines runs` and `pipelines inspect` then show it.", "wait": "Block until each named run reaches a terminal status - succeeded, failed, canceled or interrupted, and also through a run waiting on a person - then print one line per run with its id, status and outcome, its stage and how long it took, and exit 0 even for a run that finished unsuccessfully. Several ids wait together in one command. The exit is non-zero only for a client-side failure: an unknown run id, a connection that could not be established, or --timeout expiring while a run is still active; a timeout says so plainly, never cancels, retries or otherwise touches a run, and is never reported as its outcome. Read-only: it needs no workspace, is not refused by the workspace guard and writes nothing. --json prints the final run envelope instead of the line (one envelope for one id, an array for several).", "decide": "Choose an option for a run waiting on a human stage. The run must be `waiting`; the option must be one of that stage's non-error edges. The stage is inferred from the run's current stage unless named. The choice and optional --feedback are recorded as the step's evidence (visible in inspect and the TUI) and the run resumes along the chosen edge. An unknown option is refused with the valid list; a repeated decision for an already-decided visit does not double-apply. Works across a server restart because the wait is persisted.", diff --git a/internal/cli/help.go b/internal/cli/help.go index dd3585ef..51ddf8d7 100644 --- a/internal/cli/help.go +++ b/internal/cli/help.go @@ -29,7 +29,7 @@ EXECUTE wait <run-id> ... Block until runs finish, then report each (read-only) cancel <run-id> Cancel a run; changes are NOT rolled back decide <run-id> [stage] Choose an option for a run waiting on a person - retry <run-id> Restart a failed run with its original input and directory + retry <run-id> Resume a failed stage (use --mode=restart for a full restart) title <run-id> "<title>" Rename a run after the fact; input/steps unchanged debug <run-id> Investigate a run with a configured harness or nvim chat-adopt <session-id> Adopt an existing OpenCode session into chat_sessions diff --git a/internal/opencode/chat.go b/internal/opencode/chat.go index 8be73857..6342f772 100644 --- a/internal/opencode/chat.go +++ b/internal/opencode/chat.go @@ -33,6 +33,10 @@ type Chat struct { conn *Connection shellMu sync.Mutex shells map[string]*ChatStreamState + // shellOwners retains ownership from the last authoritative global shell + // listing so a shell known to belong to another session is rejected before + // its detail or output endpoint is queried. + shellOwners map[string]string // mu serializes the adapter's HTTP calls. Connection records its last // operation and status on the shared connection, so two calls must not // overlap; the permission poller runs alongside the read loop's @@ -50,6 +54,21 @@ func NewChat(conn *Connection) *Chat { return &Chat{conn: conn} } // chat adapter and callers that must reject a request before any OpenCode call. func ValidSessionID(id string) bool { return sessionID.MatchString(id) } +// ValidShellID accepts only the provider's opaque shell-id alphabet. Keeping +// this check shared by the websocket boundary and provider adapter ensures an +// invalid id is rejected before either network call or path construction. +func ValidShellID(id string) bool { + if len(id) <= 3 || len(id) > 128 || !strings.HasPrefix(id, "sh_") { + return false + } + for _, c := range id[3:] { + if (c < 'a' || c > 'z') && (c < 'A' || c > 'Z') && (c < '0' || c > '9') && c != '_' && c != '-' { + return false + } + } + return true +} + // call serializes the adapter's HTTP calls and forwards them through the shared // Connection, so no second transport is introduced and Connection's recorded // last operation/status is never written by two goroutines at once. @@ -390,9 +409,19 @@ func (c *Chat) Shells(ctx context.Context, sessionID string) ([]protocol.ChatShe return nil, fmt.Errorf("invalid OpenCode shell response") } } + c.shellMu.Lock() + if c.shellOwners == nil { + c.shellOwners = make(map[string]string) + } + for _, info := range response.Data { + if ValidShellID(info.ID) && ValidSessionID(info.shellSessionID()) { + c.shellOwners[info.ID] = info.shellSessionID() + } + } + c.shellMu.Unlock() out := make([]protocol.ChatShell, 0, len(response.Data)) for _, info := range response.Data { - if info.ID == "" || info.shellSessionID() != sessionID || (info.Status != "" && info.Status != protocol.ChatShellRunning) { + if info.ID == "" || info.shellSessionID() != sessionID || (info.Status != "" && info.Status != protocol.ChatShellRunning && info.Status != protocol.ChatShellExited) { continue } status := info.Status @@ -418,6 +447,58 @@ func (c *Chat) Shells(ctx context.Context, sessionID string) ([]protocol.ChatShe return out, nil } +// ShellOutput validates explicit session metadata from the authoritative detail +// record before reading the selected bounded output page. A shared directory is +// never ownership evidence. +func (c *Chat) ShellOutput(ctx context.Context, query protocol.ChatShellOutputQuery) (protocol.ChatShellOutputPage, error) { + if !ValidSessionID(query.SessionID) || !ValidShellID(query.ShellID) || query.Cursor < 0 || query.Cursor > protocol.ChatShellOutputMaxCursor || query.Limit <= 0 || query.Limit > protocol.ChatShellOutputLimit { + return protocol.ChatShellOutputPage{}, fmt.Errorf("invalid shell output query") + } + c.shellMu.Lock() + owner := c.shellOwners[query.ShellID] + c.shellMu.Unlock() + if owner != "" && owner != query.SessionID { + return protocol.ChatShellOutputPage{}, fmt.Errorf("shell is not available for this session") + } + data, err := c.call(ctx, "get", "/api/shell/"+query.ShellID, nil) + if err != nil { + if c.conn.LastHTTPStatus == 404 { + return protocol.ChatShellOutputPage{Unavailable: true}, nil + } + return protocol.ChatShellOutputPage{}, err + } + var detail struct { + Data chatShellInfo `json:"data"` + } + if err := json.Unmarshal(data, &detail); err != nil || detail.Data.ID != query.ShellID || detail.Data.shellSessionID() != query.SessionID || (detail.Data.Status != "" && detail.Data.Status != protocol.ChatShellRunning && detail.Data.Status != protocol.ChatShellExited) { + return protocol.ChatShellOutputPage{}, fmt.Errorf("shell is not available for this session") + } + shell := protocol.ChatShell{ID: detail.Data.ID, Status: detail.Data.Status, Command: detail.Data.Command, Exit: detail.Data.Exit} + if shell.Status == "" { + shell.Status = protocol.ChatShellRunning + } + path := fmt.Sprintf("/api/shell/%s/output?cursor=%d&limit=%d", query.ShellID, query.Cursor, query.Limit) + data, err = c.call(ctx, "get", path, nil) + if err != nil { + if c.conn.LastHTTPStatus == 404 { + return protocol.ChatShellOutputPage{Shell: shell, Unavailable: true}, nil + } + return protocol.ChatShellOutputPage{}, err + } + var response struct { + Data *struct { + Output *string `json:"output"` + Cursor *int64 `json:"cursor"` + Size *int64 `json:"size"` + Truncated *bool `json:"truncated"` + } `json:"data"` + } + if err := json.Unmarshal(data, &response); err != nil || response.Data == nil || response.Data.Output == nil || response.Data.Cursor == nil || response.Data.Size == nil || response.Data.Truncated == nil || len(*response.Data.Output) > query.Limit || *response.Data.Cursor < query.Cursor || *response.Data.Cursor > protocol.ChatShellOutputMaxCursor || *response.Data.Size < *response.Data.Cursor || *response.Data.Size > protocol.ChatShellOutputMaxCursor || *response.Data.Cursor-query.Cursor != int64(len(*response.Data.Output)) { + return protocol.ChatShellOutputPage{}, fmt.Errorf("invalid OpenCode shell output response") + } + return protocol.ChatShellOutputPage{Shell: shell, Output: *response.Data.Output, Cursor: *response.Data.Cursor, Size: *response.Data.Size, Truncated: *response.Data.Truncated}, nil +} + // streamState returns the session's retained, verified shell identities. It is // deliberately kept across stream replacement so an exit after reconnect remains // attributable; Shells reconciles its contents from authoritative readback on @@ -592,6 +673,7 @@ type chatShellInfo struct { ID string `json:"id"` Status string `json:"status"` Command string `json:"command"` + Exit *int `json:"exit"` Metadata map[string]any `json:"metadata"` } diff --git a/internal/opencode/chat_v2_test.go b/internal/opencode/chat_v2_test.go index fd00dbae..dfcb3973 100644 --- a/internal/opencode/chat_v2_test.go +++ b/internal/opencode/chat_v2_test.go @@ -7,6 +7,7 @@ import ( "net/http" "net/http/httptest" "path/filepath" + "strings" "testing" "time" @@ -425,6 +426,57 @@ func TestChatShellReadbackSeedsReplacementStream(t *testing.T) { } } +func TestChatShellOutputVerifiesOwnerAndBoundsPage(t *testing.T) { + var outputPath string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + switch r.URL.Path { + case "/api/shell/sh_1": + _, _ = w.Write([]byte(`{"data":{"id":"sh_1","status":"exited","command":"job","metadata":{"sessionID":"ses_1"}}}`)) + case "/api/shell/sh_1/output": + outputPath = r.URL.RawQuery + _, _ = w.Write([]byte(`{"data":{"output":"marker","cursor":6,"size":6,"truncated":false}}`)) + default: + http.NotFound(w, r) + } + })) + defer srv.Close() + page, err := NewChat(&Connection{Backend: "v2", URL: srv.URL}).ShellOutput(context.Background(), protocol.ChatShellOutputQuery{SessionID: "ses_1", ShellID: "sh_1", Limit: 64 << 10}) + if err != nil || page.Output != "marker" || page.Shell.Status != protocol.ChatShellExited { + t.Fatalf("ShellOutput = %+v, %v", page, err) + } + if !strings.Contains(outputPath, "cursor=0") || !strings.Contains(outputPath, "limit=65536") { + t.Fatalf("output query = %q", outputPath) + } +} + +func TestChatShellOutputRejectsMissingRequiredPageFields(t *testing.T) { + for _, body := range []string{ + `{"data":{}}`, + `{"data":{"output":"","cursor":0,"size":0}}`, + `{"data":{"output":"","size":0,"truncated":false}}`, + `{"data":{"output":"","cursor":0,"truncated":false}}`, + } { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + switch r.URL.Path { + case "/api/shell/sh_1": + _, _ = w.Write([]byte(`{"data":{"id":"sh_1","status":"running","metadata":{"sessionID":"ses_1"}}}`)) + case "/api/shell/sh_1/output": + _, _ = w.Write([]byte(body)) + default: + http.NotFound(w, r) + } + })) + chat := NewChat(&Connection{Backend: "v2", URL: srv.URL}) + _, err := chat.ShellOutput(context.Background(), protocol.ChatShellOutputQuery{SessionID: "ses_1", ShellID: "sh_1", Limit: 1}) + srv.Close() + if err == nil { + t.Fatalf("response %s was accepted", body) + } + } +} + func TestChatReleaseShellsBoundsDisconnectedState(t *testing.T) { chat := NewChat(&Connection{}) state := chat.streamState("ses_1") diff --git a/internal/pipeline/resources.go b/internal/pipeline/resources.go index 1a1720b1..5d4907a8 100644 --- a/internal/pipeline/resources.go +++ b/internal/pipeline/resources.go @@ -3,15 +3,184 @@ package pipeline import ( "context" "crypto/rand" + "crypto/sha256" "database/sql" "encoding/hex" "encoding/json" "fmt" + "strings" "time" + "unicode/utf8" "git.a-wan.net/alan/pipelines/internal/protocol" ) +const maxRecoveryText = 512 + +func boundedRecoveryText(value string) string { + value = strings.TrimSpace(value) + return value +} + +// InspectResource exposes only bounded state; capability tokens never leave this +// package or enter a protocol value. +func (s *Store) InspectResource(ctx context.Context, resource string) (protocol.ResourceInspection, error) { + if !ValidResourceKey(resource) { + return protocol.ResourceInspection{}, fmt.Errorf("invalid resource") + } + s.resourceMu.Lock() + defer s.resourceMu.Unlock() + out := protocol.ResourceInspection{Resource: resource, Waiters: []protocol.ResourceWaiter{}, Audits: []protocol.ResourceRecoveryAudit{}} + var runID int64 + var stage string + var seq, cleanup int + err := s.db.QueryRowContext(ctx, `SELECT run_id,stage_key,sequence,cleanup_confirmed FROM resource_owners WHERE resource_key=?`, resource).Scan(&runID, &stage, &seq, &cleanup) + if err == nil { + var active int + if err := s.db.QueryRowContext(ctx, `SELECT COALESCE(MAX(agent_active),0) FROM run_steps WHERE run_id=? AND sequence=?`, runID, seq).Scan(&active); err != nil { + return out, err + } + out.Owner = &protocol.ResourceOwner{RunID: runID, Stage: stage, Sequence: seq, CleanupConfirmed: cleanup != 0, Quarantined: cleanup == 0, AgentActive: active != 0} + } else if err != sql.ErrNoRows { + return out, err + } + rows, err := s.db.QueryContext(ctx, `SELECT run_id,stage_key,sequence,queued_at FROM resource_waiters WHERE resource_key=? ORDER BY queue_order LIMIT 32`, resource) + if err != nil { + return out, err + } + for rows.Next() { + var w protocol.ResourceWaiter + if err := rows.Scan(&w.RunID, &w.Stage, &w.Sequence, &w.QueuedAt); err != nil { + rows.Close() + return out, err + } + out.Waiters = append(out.Waiters, w) + } + if err := rows.Err(); err != nil { + rows.Close() + return out, err + } + rows.Close() + audits, err := s.db.QueryContext(ctx, `SELECT run_id,stage_key,sequence,actor,source,evidence,result,created_at FROM resource_recovery_audit WHERE resource_key=? ORDER BY rowid DESC LIMIT 16`, resource) + if err != nil { + return out, err + } + defer audits.Close() + for audits.Next() { + var a protocol.ResourceRecoveryAudit + a.Resource = resource + if err := audits.Scan(&a.RunID, &a.Stage, &a.Sequence, &a.Actor, &a.Source, &a.Evidence, &a.Result, &a.CreatedAt); err != nil { + audits.Close() + return out, err + } + out.Audits = append(out.Audits, a) + } + return out, audits.Err() +} + +// ReconcileResource is the sole operator recovery mutation. The persisted +// capability is compared only inside this authoritative store operation. +func (s *Store) ReconcileResource(ctx context.Context, request protocol.ResourceRecoveryRequest) error { + if !ValidResourceKey(request.Resource) || request.RunID <= 0 || request.Sequence <= 0 || !identifier.MatchString(request.Stage) { + return fmt.Errorf("invalid resource recovery identity") + } + if request.Evidence == nil || !request.Evidence.CleanupConfirmed || !validRecoveryEvidence(request.Evidence.Summary) { + return fmt.Errorf("positive cleanup/restoration evidence is required") + } + if strings.TrimSpace(request.Actor) == "" || len(request.Actor) > maxRecoveryText { + return fmt.Errorf("bounded actor is required") + } + s.resourceMu.Lock() + defer s.resourceMu.Unlock() + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + return err + } + defer tx.Rollback() + var token, stage string + var runID int64 + var seq, cleanup int + err = tx.QueryRowContext(ctx, `SELECT run_id,stage_key,sequence,token,cleanup_confirmed FROM resource_owners WHERE resource_key=?`, request.Resource).Scan(&runID, &stage, &seq, &token, &cleanup) + if err == sql.ErrNoRows { + var count int + if err := tx.QueryRowContext(ctx, `SELECT COUNT(*) FROM resource_recovery_audit WHERE resource_key=? AND run_id=? AND stage_key=? AND sequence=? AND result='reconciled'`, request.Resource, request.RunID, request.Stage, request.Sequence).Scan(&count); err != nil { + return err + } + if count > 0 { + return nil + } + return fmt.Errorf("resource owner not found") + } + if err != nil { + return err + } + if runID != request.RunID || stage != request.Stage || seq != request.Sequence { + return fmt.Errorf("resource owner visit is stale") + } + if cleanup != 0 { + return fmt.Errorf("resource owner is not quarantined") + } + var status string + if err := tx.QueryRowContext(ctx, `SELECT status FROM pipeline_runs WHERE id=?`, runID).Scan(&status); err != nil { + return err + } + if status == protocol.RunStatusQueued || status == protocol.RunStatusWaiting || status == protocol.RunStatusRunning { + return fmt.Errorf("owning run is still active") + } + var active int + if err := tx.QueryRowContext(ctx, `SELECT COALESCE(MAX(agent_active),0) FROM run_steps WHERE run_id=? AND sequence=?`, runID, seq).Scan(&active); err != nil { + return err + } + if active != 0 { + return fmt.Errorf("owning agent is still active") + } + evidence := boundedRecoveryText(request.Evidence.Summary) + if request.Evidence.NoMutatingAction { + evidence = boundedRecoveryText("no mutating action occurred; " + evidence) + } + evidenceHash := sha256.Sum256([]byte(evidence)) + if _, err := tx.ExecContext(ctx, `INSERT INTO resource_recovery_audit(resource_key,run_id,stage_key,sequence,actor,source,evidence,result) VALUES(?,?,?,?,?,?,?,?)`, request.Resource, runID, stage, seq, boundedRecoveryText(request.Actor), "loopback-operator", "sha256:"+hex.EncodeToString(evidenceHash[:]), "reconciled"); err != nil { + return err + } + if _, err := tx.ExecContext(ctx, `UPDATE resource_owners SET cleanup_confirmed=1,diagnostic='' WHERE resource_key=? AND run_id=? AND stage_key=? AND sequence=? AND token=? AND cleanup_confirmed=0`, request.Resource, runID, stage, seq, token); err != nil { + return err + } + owner := ResourceLease{ResourceVisit: ResourceVisit{Resource: request.Resource, RunID: runID, Stage: stage, Sequence: seq}, token: token} + if _, err := deleteResourceOwner(ctx, tx, owner); err != nil { + return err + } + next, err := s.handoffResourceTx(ctx, tx, request.Resource) + if err != nil { + return err + } + if err := tx.Commit(); err != nil { + return err + } + if next.token != "" { + s.resourceWaiters[next.ResourceVisit] <- next + } + return nil +} + +func validRecoveryEvidence(value string) bool { + value = strings.TrimSpace(value) + if value == "" || len(value) > maxRecoveryText || !utf8.ValidString(value) { + return false + } + if strings.EqualFold(value, "true") || strings.EqualFold(value, "false") || strings.EqualFold(value, "null") { + return false + } + if strings.HasPrefix(value, "{") || strings.HasPrefix(value, "[") { + return false + } + for _, r := range value { + if r == '\x00' || (r < 0x20 && r != '\t' && r != '\n' && r != '\r') { + return false + } + } + return true +} + // ResourceVisit identifies a pending visit, before it consumes execution accounting. type ResourceVisit struct { Resource string @@ -208,26 +377,10 @@ func (s *Store) releaseResourceLocked(ctx context.Context, lease ResourceLease) if n == 0 { return nil } - var next ResourceLease - next.Resource = lease.Resource - err = tx.QueryRowContext(ctx, `SELECT run_id,stage_key,sequence,token FROM resource_waiters WHERE resource_key=? ORDER BY queue_order LIMIT 1`, lease.Resource).Scan(&next.RunID, &next.Stage, &next.Sequence, &next.token) - if err != nil && err != sql.ErrNoRows { + next, err := s.handoffResourceTx(ctx, tx, lease.Resource) + if err != nil { return err } - if err == nil { - if s.resourceWaiters[next.ResourceVisit] == nil { - return fmt.Errorf("resource waiter has no live runner; refusing handoff") - } - if _, err := tx.ExecContext(ctx, `INSERT INTO resource_owners(resource_key,run_id,stage_key,sequence,token) VALUES(?,?,?,?,?)`, next.Resource, next.RunID, next.Stage, next.Sequence, next.token); err != nil { - return err - } - if _, err := tx.ExecContext(ctx, "DELETE FROM resource_waiters WHERE run_id=?", next.RunID); err != nil { - return err - } - if _, err := tx.ExecContext(ctx, "UPDATE pipeline_runs SET resource_wait='' WHERE id=?", next.RunID); err != nil { - return err - } - } if err := tx.Commit(); err != nil { return err } @@ -237,6 +390,34 @@ func (s *Store) releaseResourceLocked(ctx context.Context, lease ResourceLease) return nil } +// handoffResourceTx is shared by ordinary release and inspected recovery. It +// validates the in-memory waiter before changing durable ownership so a stale +// server cannot grant a capability to a runner that no longer exists. +func (s *Store) handoffResourceTx(ctx context.Context, tx *sql.Tx, resource string) (ResourceLease, error) { + var next ResourceLease + next.Resource = resource + err := tx.QueryRowContext(ctx, `SELECT run_id,stage_key,sequence,token FROM resource_waiters WHERE resource_key=? ORDER BY queue_order LIMIT 1`, resource).Scan(&next.RunID, &next.Stage, &next.Sequence, &next.token) + if err == sql.ErrNoRows { + return ResourceLease{}, nil + } + if err != nil { + return ResourceLease{}, err + } + if s.resourceWaiters[next.ResourceVisit] == nil { + return ResourceLease{}, fmt.Errorf("resource waiter has no live runner; refusing handoff") + } + if _, err := tx.ExecContext(ctx, `INSERT INTO resource_owners(resource_key,run_id,stage_key,sequence,token) VALUES(?,?,?,?,?)`, next.Resource, next.RunID, next.Stage, next.Sequence, next.token); err != nil { + return ResourceLease{}, err + } + if _, err := tx.ExecContext(ctx, `DELETE FROM resource_waiters WHERE run_id=?`, next.RunID); err != nil { + return ResourceLease{}, err + } + if _, err := tx.ExecContext(ctx, "UPDATE pipeline_runs SET resource_wait='' WHERE id=?", next.RunID); err != nil { + return ResourceLease{}, err + } + return next, nil +} + func deleteResourceOwner(ctx context.Context, tx *sql.Tx, lease ResourceLease) (int64, error) { result, err := tx.ExecContext(ctx, `DELETE FROM resource_owners WHERE resource_key=? AND run_id=? AND stage_key=? AND sequence=? AND token=? AND cleanup_confirmed=1`, lease.Resource, lease.RunID, lease.Stage, lease.Sequence, lease.token) if err != nil { diff --git a/internal/pipeline/resources_test.go b/internal/pipeline/resources_test.go index 58ae2e6a..c9a1c4b3 100644 --- a/internal/pipeline/resources_test.go +++ b/internal/pipeline/resources_test.go @@ -9,6 +9,8 @@ import ( "sync" "testing" "time" + + "git.a-wan.net/alan/pipelines/internal/protocol" ) func TestResourceDeclaration(t *testing.T) { @@ -286,3 +288,243 @@ func TestResourceRestartQuarantineAndRetention(t *testing.T) { }) } } + +func TestResourceReconcileMissingWaiterPreservesQuarantine(t *testing.T) { + s, create := resourceStore(t) + ctx := context.Background() + owner := acquireTestResource(t, s, create("X")) + if _, err := s.db.ExecContext(ctx, "UPDATE pipeline_runs SET status='interrupted' WHERE id=?", owner.RunID); err != nil { + t.Fatal(err) + } + waiter := create("X") + if _, ch, _, err := s.RequestResource(ctx, waiter); err != nil || ch == nil { + t.Fatalf("register waiter: %v", err) + } + // Simulate a runner disappearing while its durable waiter remains. The + // authoritative operation must reject the handoff before audit or cleanup. + delete(s.resourceWaiters, waiter) + err := s.ReconcileResource(ctx, protocol.ResourceRecoveryRequest{ + Resource: owner.Resource, + RunID: owner.RunID, + Stage: owner.Stage, + Sequence: owner.Sequence, + Actor: "operator", + Evidence: &protocol.ResourceRecoveryEvidence{CleanupConfirmed: true, NoMutatingAction: true, Summary: "inspection confirmed no mutating action occurred"}, + }) + if err == nil { + t.Fatal("reconciled without a live waiter channel") + } + var cleanup, audits, owners int + if err := s.db.QueryRow("SELECT cleanup_confirmed FROM resource_owners WHERE resource_key=?", owner.Resource).Scan(&cleanup); err != nil { + t.Fatal(err) + } + if err := s.db.QueryRow("SELECT COUNT(*) FROM resource_recovery_audit WHERE resource_key=?", owner.Resource).Scan(&audits); err != nil { + t.Fatal(err) + } + if err := s.db.QueryRow("SELECT COUNT(*) FROM resource_owners WHERE resource_key=?", owner.Resource).Scan(&owners); err != nil { + t.Fatal(err) + } + if cleanup != 0 || audits != 0 || owners != 1 { + t.Fatalf("failed recovery changed state: cleanup=%d audits=%d owners=%d", cleanup, audits, owners) + } +} + +func TestResourceReconcileTransfersAndAudits(t *testing.T) { + s, create := resourceStore(t) + ctx := context.Background() + owner := acquireTestResource(t, s, create("X")) + if _, err := s.db.ExecContext(ctx, "UPDATE pipeline_runs SET status='interrupted' WHERE id=?", owner.RunID); err != nil { + t.Fatal(err) + } + waiter := create("X") + _, ch, _, err := s.RequestResource(ctx, waiter) + if err != nil || ch == nil { + t.Fatalf("register waiter: %v", err) + } + if err := s.ReconcileResource(ctx, protocol.ResourceRecoveryRequest{ + Resource: owner.Resource, + RunID: owner.RunID, + Stage: owner.Stage, + Sequence: owner.Sequence, + Actor: "operator", + Evidence: &protocol.ResourceRecoveryEvidence{CleanupConfirmed: true, NoMutatingAction: true, Summary: "inspection confirmed no mutating action occurred"}, + }); err != nil { + t.Fatal(err) + } + select { + case next := <-ch: + s.AcceptResource(next) + if next.ResourceVisit != waiter { + t.Fatalf("wrong handoff: %#v", next.ResourceVisit) + } + case <-time.After(time.Second): + t.Fatal("reconciliation did not wake waiter") + } + var audits int + if err := s.db.QueryRow("SELECT COUNT(*) FROM resource_recovery_audit WHERE resource_key=? AND result='reconciled'", owner.Resource).Scan(&audits); err != nil { + t.Fatal(err) + } + if audits != 1 { + t.Fatalf("recovery audit count=%d", audits) + } +} + +func TestResourceReconcileReplayAfterSuccessorHandoff(t *testing.T) { + s, create := resourceStore(t) + ctx := context.Background() + owner := acquireTestResource(t, s, create("X")) + if _, err := s.db.ExecContext(ctx, "UPDATE pipeline_runs SET status='interrupted' WHERE id=?", owner.RunID); err != nil { + t.Fatal(err) + } + successor := create("X") + _, ch, _, err := s.RequestResource(ctx, successor) + if err != nil || ch == nil { + t.Fatalf("register successor: %v", err) + } + request := protocol.ResourceRecoveryRequest{ + Resource: owner.Resource, + RunID: owner.RunID, + Stage: owner.Stage, + Sequence: owner.Sequence, + Actor: "operator", + Evidence: &protocol.ResourceRecoveryEvidence{ + CleanupConfirmed: true, + NoMutatingAction: true, + Summary: "inspection confirmed no mutating action occurred", + }, + } + if err := s.ReconcileResource(ctx, request); err != nil { + t.Fatal(err) + } + var successorLease ResourceLease + select { + case successorLease = <-ch: + case <-time.After(time.Second): + t.Fatal("reconciliation did not hand off to successor") + } + if successorLease.ResourceVisit != successor { + t.Fatalf("wrong successor visit: %#v", successorLease.ResourceVisit) + } + if successorLease.token == "" { + t.Fatal("successor capability was empty") + } + s.AcceptResource(successorLease) + + // The old request is an idempotent replay after the handoff. It must not + // match or release the successor's private capability. + if err := s.ReconcileResource(ctx, request); err == nil { + t.Fatal("old recovery replay was accepted against successor owner") + } + inspection, err := s.InspectResource(ctx, owner.Resource) + if err != nil { + t.Fatal(err) + } + if inspection.Owner == nil || inspection.Owner.RunID != successor.RunID || inspection.Owner.Stage != successor.Stage || inspection.Owner.Sequence != successor.Sequence { + t.Fatalf("successor was not preserved: %#v", inspection.Owner) + } + if len(inspection.Waiters) != 0 || len(inspection.Audits) != 1 || inspection.Audits[0].RunID != owner.RunID { + t.Fatalf("replay changed handoff state: waiters=%d audits=%#v", len(inspection.Waiters), inspection.Audits) + } + if len(s.resourceWaiters) != 0 { + t.Fatal("successor channel registry leaked") + } + var owners, waiters, audits int + if err := s.db.QueryRowContext(ctx, "SELECT COUNT(*) FROM resource_owners WHERE resource_key=?", owner.Resource).Scan(&owners); err != nil { + t.Fatal(err) + } + if err := s.db.QueryRowContext(ctx, "SELECT COUNT(*) FROM resource_waiters WHERE resource_key=?", owner.Resource).Scan(&waiters); err != nil { + t.Fatal(err) + } + if err := s.db.QueryRowContext(ctx, "SELECT COUNT(*) FROM resource_recovery_audit WHERE resource_key=?", owner.Resource).Scan(&audits); err != nil { + t.Fatal(err) + } + if owners != 1 || waiters != 0 || audits != 1 { + t.Fatalf("unexpected post-replay counts: owners=%d waiters=%d audits=%d", owners, waiters, audits) + } +} + +func TestResourceReconcileRequiresQuarantinedInactiveOwner(t *testing.T) { + ctx := context.Background() + test := func(t *testing.T, setup func(*Store, ResourceLease) int) { + s, create := resourceStore(t) + owner := acquireTestResource(t, s, create("X")) + expectedCleanup := setup(s, owner) + err := s.ReconcileResource(ctx, protocol.ResourceRecoveryRequest{ + Resource: owner.Resource, RunID: owner.RunID, Stage: owner.Stage, Sequence: owner.Sequence, + Actor: "operator", Evidence: &protocol.ResourceRecoveryEvidence{CleanupConfirmed: true, Summary: "verified"}, + }) + if err == nil { + t.Fatal("reconciled unsafe owner") + } + var owners, audits, cleanup int + if err := s.db.QueryRow("SELECT COUNT(*) FROM resource_owners").Scan(&owners); err != nil { + t.Fatal(err) + } + if err := s.db.QueryRow("SELECT COUNT(*) FROM resource_recovery_audit").Scan(&audits); err != nil { + t.Fatal(err) + } + if err := s.db.QueryRow("SELECT cleanup_confirmed FROM resource_owners WHERE resource_key=?", owner.Resource).Scan(&cleanup); err != nil { + t.Fatal(err) + } + if owners != 1 || audits != 0 || cleanup != expectedCleanup { + t.Fatalf("unsafe recovery changed state: owners=%d audits=%d cleanup=%d", owners, audits, cleanup) + } + } + t.Run("active run", func(t *testing.T) { + test(t, func(s *Store, owner ResourceLease) int { + if _, err := s.db.Exec("UPDATE pipeline_runs SET status=? WHERE id=?", protocol.RunStatusWaiting, owner.RunID); err != nil { + t.Fatal(err) + } + return 0 + }) + }) + t.Run("active agent", func(t *testing.T) { + test(t, func(s *Store, owner ResourceLease) int { + if _, err := s.db.Exec("UPDATE pipeline_runs SET status='interrupted' WHERE id=?", owner.RunID); err != nil { + t.Fatal(err) + } + if _, err := s.db.Exec("INSERT INTO run_steps(run_id,sequence,stage_key,status,agent_active,input_json) VALUES(?,?,?,?,?, '{}')", owner.RunID, owner.Sequence, owner.Stage, "running", 1); err != nil { + t.Fatal(err) + } + return 0 + }) + }) + t.Run("already confirmed", func(t *testing.T) { + test(t, func(s *Store, owner ResourceLease) int { + if err := s.SetResourceCleanup(ctx, owner, true); err != nil { + t.Fatal(err) + } + return 1 + }) + }) +} + +func TestResourceReconcileRejectsMalformedEvidenceBeforeMutation(t *testing.T) { + ctx := context.Background() + for _, summary := range []string{"", "true", "FALSE", "null", "verified\x00cleanup", strings.Repeat("x", maxRecoveryText+1)} { + t.Run(fmt.Sprintf("%q", summary), func(t *testing.T) { + s, create := resourceStore(t) + owner := acquireTestResource(t, s, create("X")) + err := s.ReconcileResource(ctx, protocol.ResourceRecoveryRequest{ + Resource: owner.Resource, RunID: owner.RunID, Stage: owner.Stage, Sequence: owner.Sequence, + Actor: "operator", Evidence: &protocol.ResourceRecoveryEvidence{CleanupConfirmed: true, Summary: summary}, + }) + if err == nil { + t.Fatal("accepted malformed evidence") + } + var owners, audits, cleanup int + if err := s.db.QueryRow("SELECT COUNT(*) FROM resource_owners").Scan(&owners); err != nil { + t.Fatal(err) + } + if err := s.db.QueryRow("SELECT COUNT(*) FROM resource_recovery_audit").Scan(&audits); err != nil { + t.Fatal(err) + } + if err := s.db.QueryRow("SELECT cleanup_confirmed FROM resource_owners WHERE resource_key=?", owner.Resource).Scan(&cleanup); err != nil { + t.Fatal(err) + } + if owners != 1 || audits != 0 || cleanup != 0 { + t.Fatalf("invalid evidence changed state: owners=%d audits=%d cleanup=%d", owners, audits, cleanup) + } + }) + } +} diff --git a/internal/pipeline/retry_checkpoints.go b/internal/pipeline/retry_checkpoints.go new file mode 100644 index 00000000..af8de1f7 --- /dev/null +++ b/internal/pipeline/retry_checkpoints.go @@ -0,0 +1,52 @@ +package pipeline + +import ( + "context" + "database/sql" + "encoding/json" + "fmt" + + "git.a-wan.net/alan/pipelines/internal/protocol" +) + +// SaveRetryCheckpoint records the immutable source and the completed-prefix +// handoff before a retry is launched. It intentionally stores no step or usage +// rows: those remain evidence belonging only to the source attempt. +func (s *Store) SaveRetryCheckpoint(ctx context.Context, runID, sourceID int64, mode string, sequence int, source *protocol.RunDetail, handoff map[string]any) error { + if mode == "" { + mode = "resume" + } + data, err := json.Marshal(handoff) + if err != nil { + return err + } + s.mu.Lock() + defer s.mu.Unlock() + var oldSource int64 + var oldMode, oldHandoff, oldHash string + var oldSeq, oldVersion int + err = s.db.QueryRowContext(ctx, `SELECT source_run_id,mode,failed_sequence,handoff_json,source_version,source_hash FROM retry_checkpoints WHERE run_id=?`, runID).Scan(&oldSource, &oldMode, &oldSeq, &oldHandoff, &oldVersion, &oldHash) + if err == nil { + if oldSource != sourceID || oldMode != mode || oldSeq != sequence || oldHandoff != string(data) || oldVersion != source.Version || oldHash != source.ContentHash { + return fmt.Errorf("retry checkpoint conflicts with the existing request") + } + return nil + } + if err != sql.ErrNoRows { + return err + } + _, err = s.db.ExecContext(ctx, `INSERT INTO retry_checkpoints(run_id,source_run_id,mode,failed_sequence,handoff_json,source_version,source_hash) VALUES(?,?,?,?,?,?,?)`, runID, sourceID, mode, sequence, string(data), source.Version, source.ContentHash) + return err +} + +func (s *Store) RetryCheckpoint(ctx context.Context, runID int64) (sourceID int64, mode string, sequence int, handoff map[string]any, err error) { + s.mu.Lock() + defer s.mu.Unlock() + var data string + err = s.db.QueryRowContext(ctx, `SELECT source_run_id,mode,failed_sequence,handoff_json FROM retry_checkpoints WHERE run_id=?`, runID).Scan(&sourceID, &mode, &sequence, &data) + if err != nil { + return 0, "", 0, nil, err + } + err = json.Unmarshal([]byte(data), &handoff) + return +} diff --git a/internal/pipeline/runs.go b/internal/pipeline/runs.go index 76e1de11..0ff3577e 100644 --- a/internal/pipeline/runs.go +++ b/internal/pipeline/runs.go @@ -100,7 +100,7 @@ func (s *Store) GetRun(ctx context.Context, id int64) (*protocol.RunDetail, erro if err := json.Unmarshal([]byte(graphJSON), &r.Graph); err != nil { return nil, err } - if err := tx.QueryRowContext(ctx, "SELECT workspace,allow_tools,write_paths,input_json,output_json,COALESCE((SELECT original_run_id FROM run_retries WHERE run_id=pipeline_runs.id),0),title_source,title_model FROM pipeline_runs WHERE id=?", id).Scan(&r.Workspace, &r.AllowTools, &paths, &input, &output, &r.RetryOf, &r.TitleSource, &r.TitleModel); err != nil { + if err := tx.QueryRowContext(ctx, "SELECT workspace,allow_tools,write_paths,input_json,output_json,COALESCE((SELECT original_run_id FROM run_retries WHERE run_id=pipeline_runs.id),0),COALESCE((SELECT mode FROM retry_checkpoints WHERE run_id=pipeline_runs.id),CASE WHEN EXISTS(SELECT 1 FROM run_retries WHERE run_id=pipeline_runs.id) THEN 'restart' ELSE '' END),COALESCE((SELECT source_run_id FROM retry_checkpoints WHERE run_id=pipeline_runs.id),0),title_source,title_model FROM pipeline_runs WHERE id=?", id).Scan(&r.Workspace, &r.AllowTools, &paths, &input, &output, &r.RetryOf, &r.RetryMode, &r.RetrySource, &r.TitleSource, &r.TitleModel); err != nil { return nil, err } if err := decodeValues(input, &r.Input); err != nil { diff --git a/internal/pipeline/runs_schema.go b/internal/pipeline/runs_schema.go index 8f861061..ff766442 100644 --- a/internal/pipeline/runs_schema.go +++ b/internal/pipeline/runs_schema.go @@ -52,6 +52,10 @@ run_id INTEGER NOT NULL, sequence INTEGER NOT NULL, backend TEXT NOT NULL, refer PRIMARY KEY(run_id,sequence), FOREIGN KEY(run_id,sequence) REFERENCES run_steps(run_id,sequence)); CREATE TABLE IF NOT EXISTS run_retries ( run_id INTEGER PRIMARY KEY REFERENCES pipeline_runs(id), original_run_id INTEGER NOT NULL REFERENCES pipeline_runs(id)); +CREATE TABLE IF NOT EXISTS retry_checkpoints ( + run_id INTEGER PRIMARY KEY REFERENCES pipeline_runs(id), source_run_id INTEGER NOT NULL REFERENCES pipeline_runs(id), + mode TEXT NOT NULL, failed_sequence INTEGER NOT NULL, handoff_json TEXT NOT NULL, + source_version INTEGER NOT NULL, source_hash TEXT NOT NULL, created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP); CREATE TABLE IF NOT EXISTS resource_owners ( resource_key TEXT PRIMARY KEY, run_id INTEGER NOT NULL REFERENCES pipeline_runs(id), stage_key TEXT NOT NULL, sequence INTEGER NOT NULL, token TEXT NOT NULL, @@ -73,6 +77,12 @@ CREATE INDEX IF NOT EXISTS resource_wait_fifo ON resource_waiters(resource_key,q if err := s.addColumnIfMissing("pipeline_runs", "resource_wait", "TEXT NOT NULL DEFAULT ''"); err != nil { return err } + if _, err := s.db.Exec(`CREATE TABLE IF NOT EXISTS resource_recovery_audit ( +resource_key TEXT NOT NULL, run_id INTEGER NOT NULL, stage_key TEXT NOT NULL, +sequence INTEGER NOT NULL, actor TEXT NOT NULL, source TEXT NOT NULL, +evidence TEXT NOT NULL, result TEXT NOT NULL, created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP)`); err != nil { + return err + } return s.migrateRunModels() } diff --git a/internal/protocol/chat.go b/internal/protocol/chat.go index 8d0cc609..52d4a0e8 100644 --- a/internal/protocol/chat.go +++ b/internal/protocol/chat.go @@ -54,6 +54,10 @@ const ( // load the choices without first listing sessions and chat_sessions stays a // pure, fail-closed operator-session list. ChatDirectories = "chat_directories" + // ChatShells lists current, verified shells belonging to one operator chat + // session. ChatShellOutput reads a bounded page from one such shell. + ChatShells = "chat_shells" + ChatShellOutput = "chat_shell_output" // ChatEvent and ChatPermission are the server-pushed frames; they are the // only types the server sends without a matching request. ChatEvent = "chat_event" @@ -89,8 +93,10 @@ const ( // ChatShell.Status values, mirroring the OpenCode shell events. A running shell // is one whose command has started and not yet exited; exited is final. const ( - ChatShellRunning = "running" - ChatShellExited = "exited" + ChatShellRunning = "running" + ChatShellExited = "exited" + ChatShellOutputLimit = 64 << 10 + ChatShellOutputMaxCursor = int64(1 << 62) ) // ChatStep.Status values, mirroring the OpenCode session.step events. @@ -363,6 +369,26 @@ type ChatShell struct { Exit *int `json:"exit,omitempty"` } +// ChatShellOutputQuery names one selected shell and a bounded absolute-byte +// output page. Cursor is decimal so it can be passed directly to OpenCode. +type ChatShellOutputQuery struct { + SessionID string `json:"session_id"` + ShellID string `json:"shell_id"` + Cursor int64 `json:"cursor,omitempty"` + Limit int `json:"limit,omitempty"` +} + +// ChatShellOutputPage is a bounded, lazy output read. Unavailable is explicit when +// an exited shell is no longer directly verifiable upstream. +type ChatShellOutputPage struct { + Shell ChatShell `json:"shell"` + Output string `json:"output,omitempty"` + Cursor int64 `json:"cursor,omitempty"` + Size int64 `json:"size,omitempty"` + Truncated bool `json:"truncated,omitempty"` + Unavailable bool `json:"unavailable,omitempty"` +} + // ChatPermissionFrame is the server-pushed chat_permission payload: one pending // OpenCode permission request the phone must approve or reject. It is a distinct // Message field (like ChatEvent) rather than a ChatEvent.Kind, so it does not diff --git a/internal/protocol/message.go b/internal/protocol/message.go index 1fbaa296..9b1a42ac 100644 --- a/internal/protocol/message.go +++ b/internal/protocol/message.go @@ -3,19 +3,23 @@ package protocol import "git.a-wan.net/alan/pipelines/internal/buildinfo" const ( - Ping = "ping" - Pong = "pong" - Error = "error" - Submit = "submit" - Submitted = "submitted" - List = "list_pipelines" - PipelineList = "pipelines" - GetPipeline = "get_pipeline" - Pipeline = "pipeline" - ListTemplates = "list_templates" - Templates = "templates" - Validate = "validate" - Validated = "validated" + Ping = "ping" + Pong = "pong" + Error = "error" + Submit = "submit" + Submitted = "submitted" + List = "list_pipelines" + PipelineList = "pipelines" + GetPipeline = "get_pipeline" + Pipeline = "pipeline" + ListTemplates = "list_templates" + Templates = "templates" + Validate = "validate" + Validated = "validated" + InspectResource = "inspect_resource" + ResourceInspectionResult = "resource_inspection" + ReconcileResource = "reconcile_resource" + ResourceReconciled = "resource_reconciled" ) type Stage struct { @@ -155,6 +159,10 @@ type Message struct { // them; the same fields are echoed on the run record. Model string `json:"model,omitempty"` StageModels map[string]string `json:"stage_models,omitempty"` + // Empty (or "resume") retries the failed visit; "restart" starts at the + // entry stage. Older clients omit this field and retain resume behavior. + RetryMode string `json:"retry_mode,omitempty"` + RetrySource int64 `json:"-"` // Search is the broad, read-only search traffic over pipelines, versions, // runs, steps, field text and artifacts. See search.go for the per-type // payloads and the ranking/kind/bound vocabulary. @@ -187,10 +195,16 @@ type Message struct { // chat_directories response (home first, then recent operator-session // directories, deduplicated and bounded). Additive/omitempty so older peers // ignore it. - ChatDirectories []string `json:"chat_directories,omitempty"` + ChatDirectories []string `json:"chat_directories,omitempty"` + ChatShellOutputQuery *ChatShellOutputQuery `json:"chat_shell_output_query,omitempty"` + ChatShells []ChatShell `json:"chat_shells,omitempty"` + ChatShellOutput *ChatShellOutputPage `json:"chat_shell_output,omitempty"` // RunEvent is the server-pushed run state delta (see runevents.go) and // RunIDs the optional subscribe scope, so a client asks for the runs it // displays. Both are optional so older peers ignore them. - RunEvent *RunEventFrame `json:"run_event,omitempty"` - RunIDs []int64 `json:"run_ids,omitempty"` + RunEvent *RunEventFrame `json:"run_event,omitempty"` + RunIDs []int64 `json:"run_ids,omitempty"` + Resource string `json:"resource,omitempty"` + ResourceInspection *ResourceInspection `json:"resource_inspection,omitempty"` + ResourceRecovery *ResourceRecoveryRequest `json:"resource_recovery,omitempty"` } diff --git a/internal/protocol/resources.go b/internal/protocol/resources.go new file mode 100644 index 00000000..4f9c3c53 --- /dev/null +++ b/internal/protocol/resources.go @@ -0,0 +1,46 @@ +package protocol + +type ResourceInspection struct { + Resource string `json:"resource"` + Owner *ResourceOwner `json:"owner,omitempty"` + Waiters []ResourceWaiter `json:"waiters,omitempty"` + Audits []ResourceRecoveryAudit `json:"audits,omitempty"` +} +type ResourceOwner struct { + RunID int64 `json:"run_id"` + Stage string `json:"stage"` + Sequence int `json:"sequence"` + CleanupConfirmed bool `json:"cleanup_confirmed"` + Quarantined bool `json:"quarantined"` + AgentActive bool `json:"agent_active"` +} +type ResourceWaiter struct { + RunID int64 `json:"run_id"` + Stage string `json:"stage"` + Sequence int `json:"sequence"` + QueuedAt string `json:"queued_at"` +} +type ResourceRecoveryAudit struct { + Resource string `json:"resource"` + RunID int64 `json:"run_id"` + Stage string `json:"stage"` + Sequence int `json:"sequence"` + Actor string `json:"actor"` + Source string `json:"source"` + Evidence string `json:"evidence"` + Result string `json:"result"` + CreatedAt string `json:"created_at"` +} +type ResourceRecoveryEvidence struct { + CleanupConfirmed bool `json:"cleanup_confirmed"` + NoMutatingAction bool `json:"no_mutating_action"` + Summary string `json:"summary"` +} +type ResourceRecoveryRequest struct { + Resource string `json:"resource"` + RunID int64 `json:"run_id"` + Stage string `json:"stage"` + Sequence int `json:"sequence"` + Evidence *ResourceRecoveryEvidence `json:"evidence,omitempty"` + Actor string `json:"actor"` +} diff --git a/internal/protocol/runs.go b/internal/protocol/runs.go index fb3f2830..ae9b98ea 100644 --- a/internal/protocol/runs.go +++ b/internal/protocol/runs.go @@ -370,14 +370,16 @@ type NotificationReceipt struct { type RunDetail struct { RunSummary - Graph Graph `json:"graph"` - RetryOf int64 `json:"retry_of,omitempty"` - Workspace string `json:"workspace"` - AllowTools bool `json:"allow_tools"` - WritePaths []string `json:"write_paths"` - Input map[string]any `json:"input"` - Output map[string]any `json:"output"` - Steps []RunStep `json:"steps"` + Graph Graph `json:"graph"` + RetryOf int64 `json:"retry_of,omitempty"` + RetryMode string `json:"retry_mode,omitempty"` + RetrySource int64 `json:"retry_source,omitempty"` + Workspace string `json:"workspace"` + AllowTools bool `json:"allow_tools"` + WritePaths []string `json:"write_paths"` + Input map[string]any `json:"input"` + Output map[string]any `json:"output"` + Steps []RunStep `json:"steps"` // Usage is the totalled reported usage of the run's model stages, summed in // one pass over Steps. It is absent for a run with no model usage recorded // (including historical runs). diff --git a/internal/server/chat.go b/internal/server/chat.go index ab03ef05..3d4ddf5c 100644 --- a/internal/server/chat.go +++ b/internal/server/chat.go @@ -42,6 +42,10 @@ type chatBroker interface { Reply(ctx context.Context, request protocol.ChatReplyRequest) error } +type chatShellOutputReader interface { + ShellOutput(ctx context.Context, query protocol.ChatShellOutputQuery) (protocol.ChatShellOutputPage, error) +} + // chatShellReleaser is implemented by brokers that retain verified shell ids for // attributing exit events. The hub calls it only after the last observer leaves // a room; a later attach obtains a fresh authoritative snapshot. @@ -128,6 +132,12 @@ func handleChat(ctx context.Context, request protocol.Message, broker chatBroker case protocol.ChatReopen: handleChatReopen(ctx, request, registry, write) return true + case protocol.ChatShells: + handleChatShells(ctx, request, broker, registry, write) + return true + case protocol.ChatShellOutput: + handleChatShellOutput(ctx, request, broker, registry, write) + return true } return false } diff --git a/internal/server/chat_shell_output.go b/internal/server/chat_shell_output.go new file mode 100644 index 00000000..bc4d5c09 --- /dev/null +++ b/internal/server/chat_shell_output.go @@ -0,0 +1,76 @@ +package server + +import ( + "context" + "fmt" + + "git.a-wan.net/alan/pipelines/internal/opencode" + "git.a-wan.net/alan/pipelines/internal/protocol" +) + +func operatorChatSession(ctx context.Context, registry operatorRegistry, id string) error { + if registry == nil { + return fmt.Errorf("operator session registry is not configured on this server") + } + _, found, err := registry.OperatorSession(ctx, id) + if err != nil { + return err + } + if !found { + return fmt.Errorf("operator session is not available") + } + return nil +} + +func handleChatShells(ctx context.Context, request protocol.Message, broker chatBroker, registry operatorRegistry, write func(protocol.Message) error) { + query := request.ChatQuery + if broker == nil { + _ = write(protocol.Message{Type: protocol.Error, ID: request.ID, Error: "chat is not configured on this server"}) + return + } + if query == nil || !opencode.ValidSessionID(query.SessionID) { + _ = write(protocol.Message{Type: protocol.Error, ID: request.ID, Error: "a valid OpenCode session id is required"}) + return + } + if err := operatorChatSession(ctx, registry, query.SessionID); err != nil { + _ = write(protocol.Message{Type: protocol.Error, ID: request.ID, Error: err.Error()}) + return + } + shells, err := broker.Shells(ctx, query.SessionID) + if err != nil { + _ = write(protocol.Message{Type: protocol.Error, ID: request.ID, Error: err.Error()}) + return + } + _ = write(protocol.Message{Type: protocol.ChatShells, ID: request.ID, ChatShells: shells}) +} + +func handleChatShellOutput(ctx context.Context, request protocol.Message, broker chatBroker, registry operatorRegistry, write func(protocol.Message) error) { + query := request.ChatShellOutputQuery + if broker == nil { + _ = write(protocol.Message{Type: protocol.Error, ID: request.ID, Error: "chat is not configured on this server"}) + return + } + if query == nil || !opencode.ValidSessionID(query.SessionID) || !validShellID(query.ShellID) || query.Cursor < 0 || query.Cursor > protocol.ChatShellOutputMaxCursor || query.Limit <= 0 || query.Limit > protocol.ChatShellOutputLimit { + _ = write(protocol.Message{Type: protocol.Error, ID: request.ID, Error: "a valid bounded shell output query is required"}) + return + } + if err := operatorChatSession(ctx, registry, query.SessionID); err != nil { + _ = write(protocol.Message{Type: protocol.Error, ID: request.ID, Error: err.Error()}) + return + } + reader, ok := broker.(chatShellOutputReader) + if !ok { + _ = write(protocol.Message{Type: protocol.Error, ID: request.ID, Error: "shell output is not configured on this server"}) + return + } + output, err := reader.ShellOutput(ctx, *query) + if err != nil { + _ = write(protocol.Message{Type: protocol.Error, ID: request.ID, Error: err.Error()}) + return + } + _ = write(protocol.Message{Type: protocol.ChatShellOutput, ID: request.ID, ChatShellOutput: &output}) +} + +func validShellID(id string) bool { + return opencode.ValidShellID(id) +} diff --git a/internal/server/chat_shell_output_test.go b/internal/server/chat_shell_output_test.go new file mode 100644 index 00000000..7c66d48a --- /dev/null +++ b/internal/server/chat_shell_output_test.go @@ -0,0 +1,280 @@ +package server + +import ( + "context" + "fmt" + "net/http" + "net/http/httptest" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/coder/websocket" + "github.com/coder/websocket/wsjson" + + "git.a-wan.net/alan/pipelines/internal/opencode" + "git.a-wan.net/alan/pipelines/internal/protocol" +) + +type shellOutputTestBroker struct { + *fakeChat + outputCalls int +} + +func (b *shellOutputTestBroker) ShellOutput(context.Context, protocol.ChatShellOutputQuery) (protocol.ChatShellOutputPage, error) { + b.outputCalls++ + return protocol.ChatShellOutputPage{Output: "unexpected"}, nil +} + +func TestChatShellOutputRejectsBoundsBeforeRegistryOrProvider(t *testing.T) { + store := newOperatorStore(t) + broker := &shellOutputTestBroker{fakeChat: scriptedChat()} + queries := []protocol.ChatShellOutputQuery{ + {SessionID: "ses_1", ShellID: "sh_1", Limit: 0}, + {SessionID: "ses_1", ShellID: "sh_1", Limit: -1}, + {SessionID: "ses_1", ShellID: "sh_1", Cursor: protocol.ChatShellOutputMaxCursor + 1, Limit: 1}, + {SessionID: "ses_1", ShellID: "sh_1/../x", Limit: 1}, + {SessionID: "ses_1", ShellID: "sh_1%2f", Limit: 1}, + {SessionID: "ses_1", ShellID: "sh_1?cursor=0", Limit: 1}, + {SessionID: "ses_1", ShellID: "sh_1#fragment", Limit: 1}, + {SessionID: "ses_1", ShellID: "sh_%ZZ", Limit: 1}, + {SessionID: "ses_1", ShellID: "sh_" + string(make([]byte, 126)), Limit: 1}, + } + for _, query := range queries { + var response protocol.Message + handleChatShellOutput(context.Background(), protocol.Message{ChatShellOutputQuery: &query}, broker, store, func(m protocol.Message) error { + response = m + return nil + }) + if response.Type != protocol.Error { + t.Fatalf("query %+v response = %+v", query, response) + } + } + if broker.outputCalls != 0 { + t.Fatalf("invalid queries reached provider %d times", broker.outputCalls) + } +} + +// TestChatShellOutputRealHandlerWebsocketOwnedAndForeign drives the real +// authenticated adapter, HTTP fixture, HandlerWithOptions, and /connect path. +// The foreign detail deliberately has a marker that would be returned by its +// output route; the session metadata check must reject it first. +func TestChatShellOutputRealHandlerWebsocketOwnedAndForeign(t *testing.T) { + testChatShellOutputOwnedAndForeign(t, false) +} + +func TestChatShellOutputRealHandlerExitedReadback(t *testing.T) { + testChatShellOutputOwnedAndForeign(t, true) +} + +func testChatShellOutputOwnedAndForeign(t *testing.T, exited bool) { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + var providerRequests atomic.Int32 + fixtureHandler := LargeChatFixtureHandlerWithShell(t.TempDir(), exited) + fixture := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + providerRequests.Add(1) + fixtureHandler.ServeHTTP(w, r) + })) + defer fixture.Close() + attached, err := opencode.OpenEndpointAttached(ctx, fixture.URL, LargeChatFixturePassword, "", "") + if err != nil { + t.Fatal(err) + } + store := newOperatorStore(t) + if err := store.RecordOperatorSession(ctx, LargeChatFixtureSessionID, t.TempDir(), "owned", protocol.ChatSessionTagOperator); err != nil { + t.Fatal(err) + } + srv := httptest.NewServer(HandlerWithOptions(ctx, Options{Chat: opencode.NewChat(attached.Connection), OperatorRegistry: store})) + defer srv.Close() + ws := dialShellOutputWebsocket(t, ctx, srv) + defer ws.CloseNow() + + writeChat(t, ctx, ws, protocol.Message{Type: protocol.ChatShells, ID: "owned-shells", ChatQuery: &protocol.ChatQuery{SessionID: LargeChatFixtureSessionID}}) + owned := readShellOutputMessage(t, ctx, ws) + if owned.Type != protocol.ChatShells || (exited && len(owned.ChatShells) != 0) || (!exited && (len(owned.ChatShells) != 1 || owned.ChatShells[0].ID != LargeChatFixtureShellID || owned.ChatShells[0].Status != protocol.ChatShellRunning)) { + t.Fatalf("owned shell listing = %+v", owned) + } + wantStatus := protocol.ChatShellRunning + if exited { + wantStatus = protocol.ChatShellExited + } + writeChat(t, ctx, ws, protocol.Message{Type: protocol.ChatShellOutput, ID: "owned-output", ChatShellOutputQuery: &protocol.ChatShellOutputQuery{SessionID: LargeChatFixtureSessionID, ShellID: LargeChatFixtureShellID, Limit: protocol.ChatShellOutputLimit}}) + page := readShellOutputMessage(t, ctx, ws) + if page.Type != protocol.ChatShellOutput || page.ChatShellOutput == nil || page.ChatShellOutput.Output != LargeChatFixtureShellOutput || page.ChatShellOutput.Cursor != int64(len(LargeChatFixtureShellOutput)) || page.ChatShellOutput.Shell.Status != wantStatus { + t.Fatalf("owned output page = %+v", page) + } + if exited { + if page.ChatShellOutput.Shell.Exit == nil || *page.ChatShellOutput.Shell.Exit != 0 { + t.Fatalf("selected exited shell readback = %+v", page.ChatShellOutput.Shell) + } + } else if page.ChatShellOutput.Shell.Exit != nil { + t.Fatalf("running shell unexpectedly has exit = %+v", page.ChatShellOutput.Shell.Exit) + } + if page.ChatShellOutput.Cursor != int64(len(page.ChatShellOutput.Output)) || page.ChatShellOutput.Size != page.ChatShellOutput.Cursor { + t.Fatalf("non-deterministic output cursor/size = %+v", page.ChatShellOutput) + } + refusals := []struct { + name string + request protocol.Message + }{ + {name: "foreign-output", request: protocol.Message{Type: protocol.ChatShellOutput, ID: "foreign-output", ChatShellOutputQuery: &protocol.ChatShellOutputQuery{SessionID: LargeChatFixtureSessionID, ShellID: LargeChatFixtureForeignShellID, Limit: protocol.ChatShellOutputLimit}}}, + {name: "foreign-list", request: protocol.Message{Type: protocol.ChatShells, ID: "foreign-list", ChatQuery: &protocol.ChatQuery{SessionID: LargeChatFixtureForeignSessionID}}}, + {name: "syntax-invalid-list", request: protocol.Message{Type: protocol.ChatShells, ID: "invalid-list", ChatQuery: &protocol.ChatQuery{SessionID: "ses_bad/invalid"}}}, + {name: "unknown-output", request: protocol.Message{Type: protocol.ChatShellOutput, ID: "unknown-output", ChatShellOutputQuery: &protocol.ChatShellOutputQuery{SessionID: "ses_unregistered_fixture", ShellID: LargeChatFixtureShellID, Limit: protocol.ChatShellOutputLimit}}}, + } + for _, refusal := range refusals { + beforeRefusal := providerRequests.Load() + writeChat(t, ctx, ws, refusal.request) + response := readShellOutputMessage(t, ctx, ws) + if response.Type != protocol.Error || strings.Contains(response.Error, LargeChatFixtureForeignShellOutput) { + t.Fatalf("%s response leaked or succeeded: %+v", refusal.name, response) + } + if got := providerRequests.Load(); got != beforeRefusal { + t.Fatalf("%s reached provider: requests %d -> %d", refusal.name, beforeRefusal, got) + } + } +} + +func TestChatShellOutputRealHandlerUnavailableAndReconnect(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + fixture := httptest.NewServer(LargeChatFixtureHandlerWithShell(t.TempDir(), false)) + defer fixture.Close() + attached, err := opencode.OpenEndpointAttached(ctx, fixture.URL, LargeChatFixturePassword, "", "") + if err != nil { + t.Fatal(err) + } + store := newOperatorStore(t) + if err := store.RecordOperatorSession(ctx, LargeChatFixtureSessionID, t.TempDir(), "owned", protocol.ChatSessionTagOperator); err != nil { + t.Fatal(err) + } + srv := httptest.NewServer(HandlerWithOptions(ctx, Options{Chat: opencode.NewChat(attached.Connection), OperatorRegistry: store})) + defer srv.Close() + ws := dialShellOutputWebsocket(t, ctx, srv) + writeChat(t, ctx, ws, protocol.Message{Type: protocol.ChatShellOutput, ID: "missing-output", ChatShellOutputQuery: &protocol.ChatShellOutputQuery{SessionID: LargeChatFixtureSessionID, ShellID: "sh_missing_fixture", Limit: 64}}) + missing := readShellOutputMessage(t, ctx, ws) + if missing.Type != protocol.ChatShellOutput || missing.ChatShellOutput == nil || !missing.ChatShellOutput.Unavailable { + t.Fatalf("404 shell output = %+v", missing) + } + ws = dialShellOutputWebsocket(t, ctx, srv) + defer ws.CloseNow() + writeChat(t, ctx, ws, protocol.Message{Type: protocol.ChatShells, ID: "reconnect", ChatQuery: &protocol.ChatQuery{SessionID: LargeChatFixtureSessionID}}) + if got := readShellOutputMessage(t, ctx, ws); got.Type != protocol.ChatShells { + t.Fatalf("reconnect shell listing = %+v", got) + } + if _, err := opencode.OpenEndpointAttached(ctx, fixture.URL, "wrong-password", "", ""); err == nil { + t.Fatal("wrong provider authentication unexpectedly succeeded") + } +} + +func TestChatShellOutputRejectsMalformedAndOversizedProviderPages(t *testing.T) { + for _, tc := range []struct { + name string + output string + }{ + {name: "malformed", output: `{"data":{"output":"x"}}`}, + {name: "oversized", output: fmt.Sprintf(`{"data":{"output":%q,"cursor":%d,"size":%d,"truncated":false}}`, strings.Repeat("x", protocol.ChatShellOutputLimit+1), protocol.ChatShellOutputLimit+1, protocol.ChatShellOutputLimit+1)}, + } { + t.Run(tc.name, func(t *testing.T) { + fixture := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if !largeChatFixtureAuthenticated(w, r) { + return + } + w.Header().Set("Content-Type", "application/json") + switch r.URL.Path { + case "/api/info": + _, _ = w.Write([]byte(`{"version":"fixture","urls":[]}`)) + case "/api/event": + w.Header().Set("Content-Type", "text/event-stream") + case "/api/shell/sh_malformed_fixture": + _, _ = w.Write([]byte(`{"data":{"id":"sh_malformed_fixture","status":"running","metadata":{"sessionID":"ses_malformed_fixture"}}}`)) + case "/api/shell/sh_malformed_fixture/output": + _, _ = w.Write([]byte(tc.output)) + default: + if r.URL.Path == "/api/shell" { + _, _ = w.Write([]byte(`{"data":[]}`)) + return + } + http.NotFound(w, r) + } + })) + defer fixture.Close() + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + attached, err := opencode.OpenEndpointAttached(ctx, fixture.URL, LargeChatFixturePassword, "", "") + if err != nil { + t.Fatal(err) + } + _, err = opencode.NewChat(attached.Connection).ShellOutput(ctx, protocol.ChatShellOutputQuery{SessionID: "ses_malformed_fixture", ShellID: "sh_malformed_fixture", Limit: protocol.ChatShellOutputLimit}) + if err == nil { + t.Fatalf("%s provider page was accepted", tc.name) + } + }) + } +} + +func TestChatShellOutputRealHandlerCancellationStopsProvider(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + started := make(chan struct{}) + canceled := make(chan struct{}) + fixtureHandler := LargeChatFixtureHandlerWithShell(t.TempDir(), false) + fixture := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/api/shell/sh_cancel_fixture" { + if !largeChatFixtureAuthenticated(w, r) { + return + } + close(started) + <-r.Context().Done() + close(canceled) + return + } + fixtureHandler.ServeHTTP(w, r) + })) + defer fixture.Close() + attached, err := opencode.OpenEndpointAttached(ctx, fixture.URL, LargeChatFixturePassword, "", "") + if err != nil { + t.Fatal(err) + } + store := newOperatorStore(t) + if err := store.RecordOperatorSession(ctx, "ses_cancel_fixture", t.TempDir(), "cancel", protocol.ChatSessionTagOperator); err != nil { + t.Fatal(err) + } + serverCtx, serverCancel := context.WithCancel(ctx) + defer serverCancel() + srv := httptest.NewServer(HandlerWithOptions(serverCtx, Options{Chat: opencode.NewChat(attached.Connection), OperatorRegistry: store})) + defer srv.Close() + ws := dialShellOutputWebsocket(t, ctx, srv) + writeChat(t, ctx, ws, protocol.Message{Type: protocol.ChatShellOutput, ID: "cancel-output", ChatShellOutputQuery: &protocol.ChatShellOutputQuery{SessionID: "ses_cancel_fixture", ShellID: "sh_cancel_fixture", Limit: 64}}) + select { + case <-started: + case <-ctx.Done(): + t.Fatal("provider detail request did not start") + } + serverCancel() + select { + case <-canceled: + case <-ctx.Done(): + t.Fatal("canceling handler did not cancel provider request") + } +} + +func dialShellOutputWebsocket(t *testing.T, ctx context.Context, srv *httptest.Server) *websocket.Conn { + t.Helper() + ws, _, err := websocket.Dial(ctx, "ws"+strings.TrimPrefix(srv.URL, "http")+"/connect", nil) + if err != nil { + t.Fatal(err) + } + return ws +} + +func readShellOutputMessage(t *testing.T, ctx context.Context, ws *websocket.Conn) protocol.Message { + t.Helper() + var message protocol.Message + if err := wsjson.Read(ctx, ws, &message); err != nil { + t.Fatal(err) + } + return message +} diff --git a/internal/server/concurrency_test.go b/internal/server/concurrency_test.go index 33d51d05..81aa2236 100644 --- a/internal/server/concurrency_test.go +++ b/internal/server/concurrency_test.go @@ -341,20 +341,23 @@ func TestRetryConflictsOnlyWithSameWorkspaceActiveRun(t *testing.T) { if err != nil || failed.Status != "failed" { t.Fatalf("original run did not fail as intended: %+v %v", failed, err) } - resolvedB, err := filepath.EvalSymlinks(b) + entered := make(chan struct{}, 4) + resolvedA, err := filepath.EvalSymlinks(a) if err != nil { t.Fatal(err) } - entered := make(chan struct{}, 4) - release := make(chan struct{}) + releaseA := make(chan struct{}) + releaseB := make(chan struct{}) r.execute = func(ctx context.Context, _ stage.Definition, input stage.Values) (stage.Result, error) { entered <- struct{}{} - if input["pwd"] == resolvedB { - select { - case <-release: - case <-ctx.Done(): - return stage.Result{}, ctx.Err() - } + release := releaseB + if input["pwd"] == resolvedA { + release = releaseA + } + select { + case <-release: + case <-ctx.Done(): + return stage.Result{}, ctx.Err() } return stage.Result{Outcome: "success", Output: stage.Values{}}, nil } @@ -366,6 +369,7 @@ func TestRetryConflictsOnlyWithSameWorkspaceActiveRun(t *testing.T) { if _, err := r.retry(protocol.Message{RunID: original.ID, RequestKey: "retry-a"}); err == nil || !strings.Contains(err.Error(), fmt.Sprintf("run %d is already active", activeA.ID)) || !strings.Contains(err.Error(), "different workspace") { t.Fatalf("retry did not conflict with same-workspace active run: %v", err) } + close(releaseA) // A different active workspace must not block the retry of the original. if _, err := r.start(protocol.Message{PipelineID: pipelineID, Version: 1, RequestKey: "active-b", Workspace: b, Input: map[string]any{"prompt": "task"}}); err != nil { t.Fatal(err) @@ -378,7 +382,7 @@ func TestRetryConflictsOnlyWithSameWorkspaceActiveRun(t *testing.T) { if retried.Workspace != failed.Workspace { t.Fatalf("retry did not reuse the original workspace: %q != %q", retried.Workspace, failed.Workspace) } - close(release) + close(releaseB) r.wg.Wait() } diff --git a/internal/server/human_test.go b/internal/server/human_test.go index c5ac1dfd..30d9307c 100644 --- a/internal/server/human_test.go +++ b/internal/server/human_test.go @@ -270,7 +270,7 @@ func TestHumanStageCancelRetryAndRebuild(t *testing.T) { t.Fatalf("post-decision failure did not fail the run: %+v", failed.RunSummary) } r3.execute = successExecute - retried, err := r3.retry(protocol.Message{RunID: failing.ID, RequestKey: "retry-human"}) + retried, err := r3.retry(protocol.Message{RunID: failing.ID, RequestKey: "retry-human", RetryMode: "restart"}) if err != nil { t.Fatal(err) } diff --git a/internal/server/large_chat_fixture.go b/internal/server/large_chat_fixture.go index 12062dd5..be620321 100644 --- a/internal/server/large_chat_fixture.go +++ b/internal/server/large_chat_fixture.go @@ -13,6 +13,11 @@ const LargeChatFixtureSessionID = "ses_large_chat_fixture" // LargeChatFixturePassword authenticates the disposable fixture endpoint. It is // fixed fixture data, not an operator credential. const LargeChatFixturePassword = "large-chat-fixture" +const LargeChatFixtureShellID = "sh_large_chat_fixture" +const LargeChatFixtureShellOutput = "fixture shell output marker\n" +const LargeChatFixtureForeignSessionID = "ses_foreign_fixture" +const LargeChatFixtureForeignShellID = "sh_foreign_fixture" +const LargeChatFixtureForeignShellOutput = "FOREIGN SHELL MARKER MUST NOT LEAK\n" const ( LargeChatFixtureNewestAnswer = "Controlled newest answer: large history fixture" @@ -24,6 +29,13 @@ const ( // It deliberately serves fixed, paged data: no model is invoked and no production // history is read or written. func LargeChatFixtureHandler(workspace string) http.Handler { + return LargeChatFixtureHandlerWithShell(workspace, false) +} + +// LargeChatFixtureHandlerWithShell is the foreground fixture surface used by +// the readback check. exited selects the provider's observed terminal state; +// the output remains fixed and bounded in either state. +func LargeChatFixtureHandlerWithShell(workspace string, exited bool) http.Handler { mux := http.NewServeMux() mux.HandleFunc("GET /api/info", func(w http.ResponseWriter, r *http.Request) { if !largeChatFixtureAuthenticated(w, r) { @@ -63,6 +75,46 @@ func LargeChatFixtureHandler(workspace string) http.Handler { } writeLargeChatFixturePage(w, page) }) + status := "running" + if exited { + status = "exited" + } + mux.HandleFunc("GET /api/shell", func(w http.ResponseWriter, r *http.Request) { + if !largeChatFixtureAuthenticated(w, r) { + return + } + writeLargeChatFixtureJSON(w, fmt.Sprintf(`{"data":[{"id":%q,"status":%q,"command":"printf fixture","metadata":{"sessionID":%q}},{"id":%q,"status":"running","command":"printf foreign","metadata":{"sessionID":%q}}]}`, LargeChatFixtureShellID, status, LargeChatFixtureSessionID, LargeChatFixtureForeignShellID, LargeChatFixtureForeignSessionID)) + }) + mux.HandleFunc("GET /api/shell/"+LargeChatFixtureShellID, func(w http.ResponseWriter, r *http.Request) { + if !largeChatFixtureAuthenticated(w, r) { + return + } + exit := "null" + if exited { + exit = "0" + } + writeLargeChatFixtureJSON(w, fmt.Sprintf(`{"data":{"id":%q,"status":%q,"command":"printf fixture","exit":%s,"metadata":{"sessionID":%q}}}`, LargeChatFixtureShellID, status, exit, LargeChatFixtureSessionID)) + }) + mux.HandleFunc("GET /api/shell/"+LargeChatFixtureForeignShellID, func(w http.ResponseWriter, r *http.Request) { + if !largeChatFixtureAuthenticated(w, r) { + return + } + writeLargeChatFixtureJSON(w, fmt.Sprintf(`{"data":{"id":%q,"status":"running","command":"printf foreign","metadata":{"sessionID":%q}}}`, LargeChatFixtureForeignShellID, LargeChatFixtureForeignSessionID)) + }) + mux.HandleFunc("GET /api/shell/"+LargeChatFixtureShellID+"/output", func(w http.ResponseWriter, r *http.Request) { + if !largeChatFixtureAuthenticated(w, r) { + return + } + cursor := int64(len(LargeChatFixtureShellOutput)) + writeLargeChatFixtureJSON(w, fmt.Sprintf(`{"data":{"output":%q,"cursor":%d,"size":%d,"truncated":false}}`, LargeChatFixtureShellOutput, cursor, cursor)) + }) + mux.HandleFunc("GET /api/shell/"+LargeChatFixtureForeignShellID+"/output", func(w http.ResponseWriter, r *http.Request) { + if !largeChatFixtureAuthenticated(w, r) { + return + } + cursor := int64(len(LargeChatFixtureForeignShellOutput)) + writeLargeChatFixtureJSON(w, fmt.Sprintf(`{"data":{"output":%q,"cursor":%d,"size":%d,"truncated":false}}`, LargeChatFixtureForeignShellOutput, cursor, cursor)) + }) return mux } diff --git a/internal/server/resource_cli_test.go b/internal/server/resource_cli_test.go index a377dc70..7147a7f4 100644 --- a/internal/server/resource_cli_test.go +++ b/internal/server/resource_cli_test.go @@ -6,6 +6,7 @@ import ( "crypto/sha256" "encoding/json" "fmt" + "net" "net/http/httptest" "os" "path/filepath" @@ -15,11 +16,30 @@ import ( "time" "git.a-wan.net/alan/pipelines/internal/cli" + "git.a-wan.net/alan/pipelines/internal/client" "git.a-wan.net/alan/pipelines/internal/pipeline" "git.a-wan.net/alan/pipelines/internal/protocol" "git.a-wan.net/alan/pipelines/internal/stage" + "github.com/coder/websocket" + "github.com/coder/websocket/wsjson" ) +type fixedRemoteListener struct{ net.Listener } + +func (l fixedRemoteListener) Accept() (net.Conn, error) { + conn, err := l.Listener.Accept() + if err != nil { + return nil, err + } + return fixedRemoteConn{Conn: conn}, nil +} + +type fixedRemoteConn struct{ net.Conn } + +func (fixedRemoteConn) RemoteAddr() net.Addr { + return &net.TCPAddr{IP: net.ParseIP("203.0.113.9"), Port: 443} +} + // This is a real CLI/WebSocket/scratch-SQLite exercise with an explicitly FAKE // executor. It never calls inference, a phone, or a deployed server. func TestResourceScratchCLIWaitingThenProgress(t *testing.T) { @@ -110,6 +130,241 @@ func TestResourceScratchCLIWaitingThenProgress(t *testing.T) { t.Log("Observed queued resource wait followed by automatic progress; FAKE executor only, no phone/inference/deployment.") } +func TestResourceOperatorTransportCLIRecoveryBoundary(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + store, err := pipeline.Open(filepath.Join(t.TempDir(), "operator.db")) + if err != nil { + t.Fatal(err) + } + defer store.Close() + source := pipeline.Template([]string{"work"}, true) + "```resources\n{\"exclusive\":\"scratch-resource\"}\n```\n" + if _, err := store.Submit(ctx, "operator-resource", source); err != nil { + t.Fatal(err) + } + items, err := store.List(ctx) + if err != nil { + t.Fatal(err) + } + detail, err := store.Get(ctx, items[0].ID, 1) + if err != nil { + t.Fatal(err) + } + runID, _, err := store.CreateRun(ctx, detail, "operator-recovery", t.TempDir(), false, nil, nil) + if err != nil { + t.Fatal(err) + } + visit := pipeline.ResourceVisit{Resource: "scratch-resource", RunID: runID, Stage: "work", Sequence: 1} + if _, ch, _, err := store.RequestResource(ctx, visit); err != nil || ch != nil { + t.Fatalf("acquire resource: channel=%v err=%v", ch, err) + } + if err := store.RecoverRuns(ctx); err != nil { + t.Fatal(err) + } + handler := Handler(ctx, store) + srv := httptest.NewServer(handler) + defer srv.Close() + normal, err := client.Open(ctx, "ws"+strings.TrimPrefix(srv.URL, "http")+"/connect") + if err != nil { + t.Fatal(err) + } + if _, err := normal.Call(ctx, protocol.Message{Type: protocol.InspectResource, Resource: visit.Resource}); err == nil || !strings.Contains(err.Error(), "operator transport") { + t.Fatalf("ordinary connect accepted recovery: %v", err) + } + normal.Close() + endpoint := "ws" + strings.TrimPrefix(srv.URL, "http") + "/connect" + runCLI := func(args ...string) string { + var out bytes.Buffer + app := cli.App{Out: &out, Err: &out, In: strings.NewReader("")} + if err := app.Run(ctx, append([]string{"--url", endpoint}, args...)); err != nil { + t.Fatalf("CLI %v: %v\n%s", args, err, out.String()) + } + return out.String() + } + inspection := runCLI("resource-inspect", visit.Resource, "--json") + if strings.Contains(inspection, "token") || strings.Contains(inspection, "credential") { + t.Fatalf("inspection leaked capability: %s", inspection) + } + runCLI("resource-reconcile", visit.Resource, fmt.Sprint(runID), visit.Stage, fmt.Sprint(visit.Sequence), "--evidence", "verified no mutating action", "--yes", "--json") + inspection = runCLI("resource-inspect", visit.Resource, "--json") + var recovered protocol.ResourceInspection + if err := json.Unmarshal([]byte(inspection), &recovered); err != nil { + t.Fatal(err) + } + if recovered.Owner != nil || len(recovered.Audits) != 1 || recovered.Audits[0].RunID != runID || recovered.Audits[0].Result != "reconciled" { + t.Fatalf("CLI recovery state was not persisted: %s", inspection) + } +} + +func TestResourceOperatorCLIRejectsStaleAndNonQuarantinedRecovery(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + store, err := pipeline.Open(filepath.Join(t.TempDir(), "operator-rejections.db")) + if err != nil { + t.Fatal(err) + } + defer store.Close() + if _, err := store.Submit(ctx, "operator-rejections", pipeline.Template([]string{"work"}, true)+"```resources\n{\"exclusive\":\"scratch-resource\"}\n```\n"); err != nil { + t.Fatal(err) + } + items, err := store.List(ctx) + if err != nil { + t.Fatal(err) + } + detail, err := store.Get(ctx, items[0].ID, 1) + if err != nil { + t.Fatal(err) + } + runID, _, err := store.CreateRun(ctx, detail, "operator-rejections", t.TempDir(), false, nil, nil) + if err != nil { + t.Fatal(err) + } + visit := pipeline.ResourceVisit{Resource: "scratch-resource", RunID: runID, Stage: "work", Sequence: 1} + lease, ch, _, err := store.RequestResource(ctx, visit) + if err != nil || ch != nil { + t.Fatalf("acquire resource: channel=%v err=%v", ch, err) + } + if err := store.RecoverRuns(ctx); err != nil { + t.Fatal(err) + } + handler := Handler(ctx, store) + srv := httptest.NewServer(handler) + defer srv.Close() + endpoint := "ws" + strings.TrimPrefix(srv.URL, "http") + "/connect" + runCLI := func(args ...string) error { + var out bytes.Buffer + app := cli.App{Out: &out, Err: &out, In: strings.NewReader("")} + err := app.Run(ctx, append([]string{"--url", endpoint}, args...)) + if strings.Contains(out.String(), "token") || strings.Contains(out.String(), "credential") { + t.Fatalf("private capability appeared in CLI output: %s", out.String()) + } + return err + } + if err := runCLI("resource-reconcile", visit.Resource, fmt.Sprint(runID+1), visit.Stage, fmt.Sprint(visit.Sequence), "--evidence", "verified", "--yes", "--json"); err == nil { + t.Fatal("accepted wrong owner") + } + if err := runCLI("resource-reconcile", visit.Resource, fmt.Sprint(runID), "stale-stage", fmt.Sprint(visit.Sequence), "--evidence", "verified", "--yes", "--json"); err == nil { + t.Fatal("accepted stale stage") + } + if err := runCLI("resource-reconcile", visit.Resource, fmt.Sprint(runID), visit.Stage, "2", "--evidence", "verified", "--yes", "--json"); err == nil { + t.Fatal("accepted stale sequence") + } + if err := store.SetResourceCleanup(ctx, lease, true); err != nil { + t.Fatal(err) + } + if err := runCLI("resource-reconcile", visit.Resource, fmt.Sprint(runID), visit.Stage, fmt.Sprint(visit.Sequence), "--evidence", "verified", "--yes", "--json"); err == nil { + t.Fatal("accepted non-quarantined owner") + } + inspection, err := store.InspectResource(ctx, visit.Resource) + if err != nil { + t.Fatal(err) + } + if inspection.Owner == nil || !inspection.Owner.CleanupConfirmed || len(inspection.Audits) != 0 { + t.Fatalf("rejected recovery changed state: owner=%#v audits=%d", inspection.Owner, len(inspection.Audits)) + } +} + +func TestResourceOperatorTransportDoesNotTrustRemoteOrForwardedHeaders(t *testing.T) { + for _, remote := range []string{"203.0.113.9:443", "198.51.100.7:8080", "not-an-address"} { + if isLoopbackPeer(remote) { + t.Fatalf("accepted non-loopback peer %q", remote) + } + } + for _, remote := range []string{"127.0.0.1:443", "[::1]:443"} { + if !isLoopbackPeer(remote) { + t.Fatalf("rejected loopback peer %q", remote) + } + } + // X-Forwarded-For is deliberately not consulted: a proxy-supplied header + // cannot turn a remote WebSocket peer into the local operator transport. +} + +func TestResourceOperatorTransportRejectsForgedForwardedPeer(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + store, err := pipeline.Open(filepath.Join(t.TempDir(), "operator-transport.db")) + if err != nil { + t.Fatal(err) + } + defer store.Close() + if _, err := store.Submit(ctx, "operator-transport", pipeline.Template([]string{"work"}, true)+"```resources\n{\"exclusive\":\"scratch-resource\"}\n```\n"); err != nil { + t.Fatal(err) + } + items, err := store.List(ctx) + if err != nil { + t.Fatal(err) + } + detail, err := store.Get(ctx, items[0].ID, 1) + if err != nil { + t.Fatal(err) + } + runID, _, err := store.CreateRun(ctx, detail, "operator-transport", t.TempDir(), false, nil, nil) + if err != nil { + t.Fatal(err) + } + visit := pipeline.ResourceVisit{Resource: "scratch-resource", RunID: runID, Stage: "work", Sequence: 1} + if _, ch, _, err := store.RequestResource(ctx, visit); err != nil || ch != nil { + t.Fatalf("acquire resource: channel=%v err=%v", ch, err) + } + if err := store.RecoverRuns(ctx); err != nil { + t.Fatal(err) + } + before, err := store.InspectResource(ctx, visit.Resource) + if err != nil { + t.Fatal(err) + } + + server := httptest.NewUnstartedServer(Handler(ctx, store)) + server.Listener = fixedRemoteListener{Listener: server.Listener} + server.Start() + defer server.Close() + conn, _, err := websocket.Dial(ctx, "ws"+strings.TrimPrefix(server.URL, "http")+"/operator-connect", &websocket.DialOptions{HTTPHeader: map[string][]string{"X-Forwarded-For": {"127.0.0.1"}}}) + if err != nil { + t.Fatal(err) + } + defer conn.CloseNow() + var welcome protocol.Message + if err := wsjson.Read(ctx, conn, &welcome); err != nil { + t.Fatal(err) + } + if err := wsjson.Write(ctx, conn, protocol.Message{Type: protocol.InspectResource, Resource: visit.Resource, ID: "forged"}); err != nil { + t.Fatal(err) + } + var refused protocol.Message + if err := wsjson.Read(ctx, conn, &refused); err != nil { + t.Fatal(err) + } + if refused.Type != protocol.Error || !strings.Contains(refused.Error, "operator transport") { + t.Fatalf("forged peer was accepted: %#v", refused) + } + after, err := store.InspectResource(ctx, visit.Resource) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(before, after) { + t.Fatalf("forged request mutated recovery state: before=%#v after=%#v", before, after) + } + + trusted := httptest.NewServer(Handler(ctx, store)) + defer trusted.Close() + normal, err := client.Open(ctx, "ws"+strings.TrimPrefix(trusted.URL, "http")+"/connect") + if err != nil { + t.Fatal(err) + } + if _, err := normal.Call(ctx, protocol.Message{Type: protocol.InspectResource, Resource: visit.Resource}); err == nil || !strings.Contains(err.Error(), "operator transport") { + t.Fatalf("ordinary connect accepted recovery: %v", err) + } + normal.Close() + operator, err := client.Open(ctx, "ws"+strings.TrimPrefix(trusted.URL, "http")+"/operator-connect") + if err != nil { + t.Fatal(err) + } + defer operator.Close() + if _, err := operator.Call(ctx, protocol.Message{Type: protocol.InspectResource, Resource: visit.Resource}); err != nil { + t.Fatalf("trusted operator transport refused inspection: %v", err) + } +} + func TestResourceAndroidDraftPublishedV3Provenance(t *testing.T) { baseline, err := os.ReadFile("testdata/android-v3-published.md") if err != nil { diff --git a/internal/server/retry_cleanup_test.go b/internal/server/retry_cleanup_test.go index ffe00230..a82a4f45 100644 --- a/internal/server/retry_cleanup_test.go +++ b/internal/server/retry_cleanup_test.go @@ -43,14 +43,14 @@ func TestRetryWaitsForExecutionCleanupEvenAfterTerminalStatus(t *testing.T) { if err != nil || !runs[0].Executing { t.Fatal("list advertised stopped run during cleanup", runs, err) } - if _, err := r.retry(protocol.Message{RunID: id, RequestKey: "retry"}); err == nil || !strings.Contains(err.Error(), "still executing") { + if _, err := r.retry(protocol.Message{RunID: id, RequestKey: "retry", RetryMode: "restart"}); err == nil || !strings.Contains(err.Error(), "still executing") { t.Fatal("retry accepted during execution cleanup", err) } delete(r.active, id) r.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) { return stage.Result{Outcome: "success", Output: stage.Values{"summary": "done"}}, nil } - if _, err := r.retry(protocol.Message{RunID: id, RequestKey: "retry"}); err != nil { + if _, err := r.retry(protocol.Message{RunID: id, RequestKey: "retry", RetryMode: "restart"}); err != nil { t.Fatal("retry rejected after cleanup", err) } r.wg.Wait() diff --git a/internal/server/retry_test.go b/internal/server/retry_test.go index 152201ce..93f5f279 100644 --- a/internal/server/retry_test.go +++ b/internal/server/retry_test.go @@ -183,7 +183,7 @@ func TestRetryReplaysInitialInputAndPinnedVersionThroughCLI(t *testing.T) { if err != nil || len(beforeRetry) != 1 { t.Fatal("unconfirmed retry created a run", err) } - result, err := runCLI("retry", fmt.Sprint(original.ID), "--yes", "--json", "--wait", "--timeout", "5s", "--request-key", "retry-key") + result, err := runCLI("retry", fmt.Sprint(original.ID), "--yes", "--json", "--wait", "--timeout", "5s", "--request-key", "retry-key", "--mode", "restart") if err != nil { t.Fatal(err) } @@ -236,7 +236,7 @@ func TestRetryRejectsUnavailableOriginalDirectoryAndNonfailedRuns(t *testing.T) t.Fatal(err) } } - _, err = r.retry(protocol.Message{RunID: id, RequestKey: "retry-" + status}) + _, err = r.retry(protocol.Message{RunID: id, RequestKey: "retry-" + status, RetryMode: "restart"}) if err == nil { t.Fatal("invalid retry accepted", status) } @@ -249,6 +249,95 @@ func TestRetryRejectsUnavailableOriginalDirectoryAndNonfailedRuns(t *testing.T) } } +func TestRetryResumeReusesPrefixHandoffAndReplaysRequestAfterReopen(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + dir := t.TempDir() + store, err := pipeline.Open(filepath.Join(dir, "resume.db")) + if err != nil { + t.Fatal(err) + } + source := "# stage a\n\n```output\nA\n```\n\n```prompt\nA\n```\n\n```edges\nsuccess -> b\nerror -> #failure\n```\n\n```configs\ntype=model\nprovider=test\nmodel=test\n```\n\n# stage b\n\n```output\nB\n```\n\n```prompt\nB\n```\n\n```edges\nsuccess -> c\nerror -> #failure\n```\n\n```configs\ntype=model\nprovider=test\nmodel=test\n```\n\n# stage c\n\n```output\nC\n```\n\n```prompt\nC\n```\n\n```edges\nsuccess -> #success\nerror -> #failure\n```\n\n```configs\ntype=model\nprovider=test\nmodel=test\n```\n" + if _, err := store.Submit(ctx, "resume", source); err != nil { + t.Fatal(err) + } + items, _ := store.List(ctx) + workspace := t.TempDir() + var aCalls, bCalls, cCalls int + r, err := newRunner(ctx, store) + if err != nil { + t.Fatal(err) + } + r.execute = func(_ context.Context, def stage.Definition, input stage.Values) (stage.Result, error) { + switch def.Key { + case "a": + aCalls++ + if input["prompt"] != "task" { + t.Errorf("entry received altered input: %#v", input) + } + return stage.Result{Outcome: "success", Output: stage.Values{"handoff": "from-a"}}, nil + case "b": + bCalls++ + if input["handoff"] != "from-a" { + t.Errorf("resume lost exact prefix handoff: %#v", input) + } + if bCalls == 1 { + return stage.Result{}, fmt.Errorf("deliberate failure") + } + return stage.Result{Outcome: "success", Output: stage.Values{"handoff": "from-b"}}, nil + case "c": + cCalls++ + if input["handoff"] != "from-b" { + t.Errorf("C received wrong handoff: %#v", input) + } + return stage.Result{Outcome: "success", Output: stage.Values{"summary": "done"}}, nil + default: + t.Fatalf("unexpected stage %q", def.Key) + return stage.Result{}, nil + } + } + original, err := r.start(protocol.Message{PipelineID: items[0].ID, Version: 1, RequestKey: "original", Workspace: workspace, Input: map[string]any{"prompt": "task"}}) + if err != nil { + t.Fatal(err) + } + r.wg.Wait() + original, err = store.GetRun(ctx, original.ID) + if err != nil || original.Status != "failed" || len(original.Steps) != 2 { + t.Fatalf("initial run did not stop at B: %+v %v", original, err) + } + retried, err := r.retry(protocol.Message{RunID: original.ID, RequestKey: "resume-key"}) + if err != nil { + t.Fatal(err) + } + r.wg.Wait() + retried, err = store.GetRun(ctx, retried.ID) + if err != nil || retried.Status != "succeeded" || retried.RetryMode != "resume" || retried.RetrySource != original.ID { + t.Fatalf("resume did not complete with provenance: %+v %v", retried, err) + } + if aCalls != 1 || bCalls != 2 || cCalls != 1 { + t.Fatalf("retry duplicated prefix or suffix: A=%d B=%d C=%d", aCalls, bCalls, cCalls) + } + if len(retried.Steps) != 2 || retried.Steps[0].Stage != "b" || retried.Steps[1].Stage != "c" { + t.Fatalf("retry persisted the wrong steps: %+v", retried.Steps) + } + if err := store.Close(); err != nil { + t.Fatal(err) + } + store, err = pipeline.Open(filepath.Join(dir, "resume.db")) + if err != nil { + t.Fatal(err) + } + defer store.Close() + r2, err := newRunner(ctx, store) + if err != nil { + t.Fatal(err) + } + replayed, err := r2.retry(protocol.Message{RunID: original.ID, RequestKey: "resume-key"}) + if err != nil || replayed.ID != retried.ID { + t.Fatalf("persisted request-key replay lost its run: %+v %v", replayed, err) + } +} + // TestReadOnlyStageRetriesPostAdmissionTransportButMutatingStageDoesNot proves, // through the real runner and CLI fixture, that a read-only stage retries a // post-admission transport (final-readback) failure with a fresh session while diff --git a/internal/server/runner.go b/internal/server/runner.go index 260397fe..4c722216 100644 --- a/internal/server/runner.go +++ b/internal/server/runner.go @@ -285,6 +285,15 @@ func (r *runner) start(request protocol.Message, retryOf ...int64) (*protocol.Ru r.rememberRunIdentity(id, identityOfRunSummary(detail.RunSummary)) } if created { + if request.RetrySource > 0 && request.RetryMode != "restart" { + checkpoint, checkpointErr := r.retrySourceCheckpoint(request.RetrySource, d) + if checkpointErr != nil { + return nil, checkpointErr + } + if err := r.store.SaveRetryCheckpoint(r.ctx, id, request.RetrySource, "resume", checkpoint.failed.Sequence, checkpoint.detail, checkpoint.handoff); err != nil { + return nil, err + } + } // Emit the created delta before launching: the queued frame reaches // subscribers before the running/step-started transition, so a client // sees the run appear and then advance. @@ -404,6 +413,65 @@ func (r *runner) listRuns() ([]protocol.RunSummary, error) { return items, err } +type retryCheckpoint struct { + detail *protocol.RunDetail + failed protocol.RunStep + handoff map[string]any +} + +func (r *runner) retrySourceCheckpoint(sourceID int64, pinned *protocol.PipelineDetail) (retryCheckpoint, error) { + source, err := r.store.GetRun(r.ctx, sourceID) + if err != nil { + return retryCheckpoint{}, err + } + if source.Status != "failed" || source.FinishedAt == "" { + return retryCheckpoint{}, fmt.Errorf("retry source %d is not a terminal failed run", sourceID) + } + if source.PipelineID != pinned.ID || source.Version != pinned.Version || source.ContentHash != pinned.ContentHash { + return retryCheckpoint{}, fmt.Errorf("retry source %d is pinned to a different pipeline version", sourceID) + } + if len(source.Steps) == 0 { + return retryCheckpoint{}, fmt.Errorf("retry source %d has no failed stage checkpoint", sourceID) + } + failed := source.Steps[len(source.Steps)-1] + if (failed.Status != "completed" && failed.Status != "failed") || failed.Target != protocol.FailureEnd { + return retryCheckpoint{}, fmt.Errorf("retry source %d has no safe failed-stage checkpoint", sourceID) + } + var handoff map[string]any + if failed.Input == nil { + return retryCheckpoint{}, fmt.Errorf("retry source %d is missing failed-stage handoff context", sourceID) + } + handoff = map[string]any{"input": failed.Input, "sequence": failed.Sequence, "stage": failed.Stage} + prefix := map[string]any{} + for _, step := range source.Steps { + if step.Sequence >= failed.Sequence || step.Status != "completed" { + continue + } + prefix[step.Stage] = step.Output + } + handoff["prefix"] = prefix + for _, declared := range pinned.Graph.Stages { + if declared.Key != failed.Stage { + continue + } + typ := stage.EffectiveType(declared.Type) + if typ == stage.HumanType || typ == stage.NtfyType { + return retryCheckpoint{}, fmt.Errorf("retry source %d cannot replay completed external stage %s", sourceID, failed.Stage) + } + if typ == stage.ModelType { + tools, explicit, toolErr := stage.ToolNames(pipeline.StageDefinition(declared)) + if toolErr != nil { + return retryCheckpoint{}, toolErr + } + if explicit && len(tools) > 0 { + return retryCheckpoint{}, fmt.Errorf("retry source %d cannot safely replay admitted native stage %s", sourceID, failed.Stage) + } + } + return retryCheckpoint{detail: source, failed: failed, handoff: handoff}, nil + } + return retryCheckpoint{}, fmt.Errorf("retry source %d failed stage is not in the pinned graph", sourceID) +} + func (r *runner) retry(request protocol.Message) (*protocol.RunDetail, error) { if request.RunID <= 0 { return nil, fmt.Errorf("retry requires a positive run ID") @@ -418,7 +486,26 @@ func (r *runner) retry(request protocol.Message) (*protocol.RunDetail, error) { if original.Status != "failed" || original.FinishedAt == "" { return nil, fmt.Errorf("only failed runs can be retried (run %d is %s)", original.ID, original.Status) } - return r.start(protocol.Message{PipelineID: original.PipelineID, Version: original.Version, RequestKey: request.RequestKey, Workspace: original.Workspace, Input: original.Input, AllowTools: original.AllowTools, WritePaths: original.WritePaths, Model: original.Model, StageModels: original.StageModels}, original.ID) + if request.RequestKey == "" { + return nil, fmt.Errorf("retry requires an idempotency request key") + } + mode := request.RetryMode + if mode == "" { + mode = "resume" + } + if mode != "resume" && mode != "restart" { + return nil, fmt.Errorf("retry mode must be resume or restart") + } + d, err := r.store.Get(r.ctx, original.PipelineID, original.Version) + if err != nil { + return nil, err + } + if mode == "resume" { + if _, err := r.retrySourceCheckpoint(original.ID, d); err != nil { + return nil, err + } + } + return r.start(protocol.Message{PipelineID: original.PipelineID, Version: original.Version, RequestKey: request.RequestKey, RetryMode: mode, RetrySource: original.ID, Workspace: original.Workspace, Input: original.Input, AllowTools: original.AllowTools, WritePaths: original.WritePaths, Model: original.Model, StageModels: original.StageModels}, original.ID) } func (r *runner) cancel(id int64) error { @@ -520,6 +607,35 @@ func (r *runner) decide(request protocol.Message) (*protocol.RunDetail, error) { } func (r *runner) run(ctx context.Context, id int64, d *protocol.PipelineDetail, workspace string, request protocol.Message) { + if request.RetrySource > 0 && request.RetryMode != "restart" { + source, err := r.store.GetRun(ctx, request.RetrySource) + if err != nil { + r.finishRunState(id, d.Name, workspace, "failed", "error", err.Error(), 0, 0, "", nil) + return + } + checkpoint, err := r.retrySourceCheckpoint(request.RetrySource, d) + if err != nil { + r.finishRunState(id, d.Name, workspace, "failed", "error", err.Error(), 0, 0, "", nil) + return + } + storedSource, storedMode, storedSequence, storedHandoff, checkpointErr := r.store.RetryCheckpoint(ctx, id) + if checkpointErr != nil || storedSource != request.RetrySource || storedMode != "resume" || storedSequence != checkpoint.failed.Sequence || storedHandoff == nil { + if checkpointErr == nil { + checkpointErr = fmt.Errorf("retry checkpoint is incomplete") + } + r.finishRunState(id, d.Name, workspace, "failed", "error", checkpointErr.Error(), 0, 0, "", nil) + return + } + if input, ok := storedHandoff["input"].(map[string]any); ok { + checkpoint.failed.Input = input + } else { + r.finishRunState(id, d.Name, workspace, "failed", "error", "retry checkpoint is missing handoff context", 0, 0, "", nil) + return + } + state := rebuildRetryState(source, checkpoint.failed) + r.runLoop(ctx, id, d.Graph, d.Name, workspace, request, state) + return + } original, copyErr := copyValues(stage.Values(request.Input)) if copyErr != nil { r.finishRunState(id, d.Name, workspace, "failed", "error", copyErr.Error(), 0, 0, "", stage.Values(request.Input)) @@ -629,6 +745,34 @@ func rebuildRunState(detail *protocol.RunDetail, decided protocol.RunStep) *runS return state } +// rebuildRetryState restores only the completed prefix. The failed visit is +// deliberately not counted and its step/evidence is not copied into the new +// attempt, so retries cannot duplicate execution or usage aggregates. +func rebuildRetryState(detail *protocol.RunDetail, failed protocol.RunStep) *runState { + state := newRunState(failed.Stage, stage.Values(failed.Input)) + state.nextSequence = failed.Sequence + if original, err := copyValues(stage.Values(detail.Input)); err == nil { + state.memory["input"] = memoryValue{output: original} + } + for _, step := range detail.Steps { + if step.Sequence >= failed.Sequence { + break + } + state.visits[step.Stage]++ + if step.Status == "completed" { + if values, err := copyValues(stage.Values(step.Output)); err == nil { + state.memory[step.Stage] = memoryValue{output: values, sequence: step.Sequence} + } + state.completed++ + } + state.recordChanges(step.FileChanges, step.Artifacts) + if step.Status == "completed" { + state.artifactDirs = append(state.artifactDirs, filepath.Join("", strconv.Itoa(step.Sequence))) + } + } + return state +} + // artifactAccess derives exact visit directories; it never consumes a path from // a definition or model result. An unreadable or symlinked root fails closed. func (r *runner) artifactAccess(runID int64, sequence int, prior []string, current string) stage.ArtifactAccess { diff --git a/internal/server/websocket.go b/internal/server/websocket.go index 5e598223..9003a433 100644 --- a/internal/server/websocket.go +++ b/internal/server/websocket.go @@ -2,7 +2,9 @@ package server import ( "context" + "fmt" "log" + "net" "net/http" "strings" "sync" @@ -119,7 +121,7 @@ func HandlerWithOptions(ctx context.Context, options Options, stores ...*pipelin } _, _ = w.Write([]byte(`{"healthy":true}`)) }) - mux.HandleFunc("GET /connect", func(w http.ResponseWriter, r *http.Request) { + connect := func(w http.ResponseWriter, r *http.Request) { conn, err := websocket.Accept(w, r, nil) if err != nil { return @@ -171,7 +173,7 @@ func HandlerWithOptions(ctx context.Context, options Options, stores ...*pipelin } for { var request protocol.Message - if err := wsjson.Read(ctx, conn, &request); err != nil { + if err := wsjson.Read(connCtx, conn, &request); err != nil { return } log.Printf("command received: client=%s type=%q id=%q pipeline=%d version=%d", r.RemoteAddr, request.Type, request.ID, request.PipelineID, request.Version) @@ -268,6 +270,32 @@ func HandlerWithOptions(ctx context.Context, options Options, stores ...*pipelin response.Error = err.Error() } } + } else if (request.Type == protocol.InspectResource || request.Type == protocol.ReconcileResource) && len(stores) > 0 { + var err error + response.Type = protocol.ResourceInspectionResult + // Recovery has a separate transport path. A stage/model client may + // also be local, so loopback alone is not an authority boundary. + // The CLI uses /operator-connect; the ordinary /connect path is + // deliberately refused even from loopback. + if r.URL.Path != "/operator-connect" || !isLoopbackPeer(r.RemoteAddr) { + response.Type = protocol.Error + response.Error = "resource recovery requires the local operator transport" + } else if request.Type == protocol.InspectResource { + var inspection protocol.ResourceInspection + inspection, err = stores[0].InspectResource(ctx, request.Resource) + response.ResourceInspection = &inspection + } else { + response.Type = protocol.ResourceReconciled + if request.ResourceRecovery == nil { + err = fmt.Errorf("resource recovery evidence is required") + } else { + err = stores[0].ReconcileResource(ctx, *request.ResourceRecovery) + } + } + if err != nil { + response.Type = protocol.Error + response.Error = err.Error() + } } else if runs != nil && (request.Type == protocol.RunPipeline || request.Type == protocol.RetryRun || request.Type == protocol.ListRuns || request.Type == protocol.GetRun || request.Type == protocol.CancelRun || request.Type == protocol.DecideRun || request.Type == protocol.SetRunTitle || request.Type == protocol.GetRunEvidence) { var err error switch request.Type { @@ -323,10 +351,21 @@ func HandlerWithOptions(ctx context.Context, options Options, stores ...*pipelin } log.Printf("response sent: client=%s type=%q id=%q time=%q error=%q version=%d", r.RemoteAddr, response.Type, response.ID, response.Time, response.Error, selectedVersion) } - }) + } + mux.HandleFunc("GET /connect", connect) + mux.HandleFunc("GET /operator-connect", connect) return &runHandler{Handler: mux, runner: runs} } +func isLoopbackPeer(remoteAddr string) bool { + host, _, err := net.SplitHostPort(remoteAddr) + if err != nil { + return false + } + ip := net.ParseIP(host) + return ip != nil && ip.IsLoopback() +} + type runHandler struct { http.Handler runner *runner diff --git a/internal/tui/retry_test.go b/internal/tui/retry_test.go index 904e9a13..85d3a3d0 100644 --- a/internal/tui/retry_test.go +++ b/internal/tui/retry_test.go @@ -32,6 +32,12 @@ func TestRetryShortcutAndPaletteSendServerRetryWithoutReplacingDirectory(t *test if err != nil { t.Fatal(err) } + if err := store.BeginStep(ctx, id, 1, "one", map[string]any{"prompt": "original task"}); err != nil { + t.Fatal(err) + } + if err := store.FinishStep(ctx, id, protocol.RunStep{Sequence: 1, Stage: "one", Status: "failed", Outcome: "error", Target: protocol.FailureEnd, Input: map[string]any{"prompt": "original task"}}); err != nil { + t.Fatal(err) + } if err := store.FinishRun(ctx, id, "failed", "error", "original failure", nil); err != nil { t.Fatal(err) }