Merge branch 'main' into chat-cross-client-echo-order

This commit is contained in:
2026-10-18 21:46:00 +01:00
34 changed files with 2072 additions and 110 deletions
@@ -13,6 +13,8 @@ import net.awan.pipelines.data.protocol.ChatPromptRequest
import net.awan.pipelines.data.protocol.ChatProtocol
import net.awan.pipelines.data.protocol.ChatQuery
import net.awan.pipelines.data.protocol.ChatSession
import net.awan.pipelines.data.protocol.ChatShellOutputPage
import net.awan.pipelines.data.protocol.ChatShell
import net.awan.pipelines.data.protocol.Message
import net.awan.pipelines.data.protocol.Graph
import net.awan.pipelines.data.protocol.PipelineDetail
@@ -312,6 +314,22 @@ class PipelinesRepository(
return response.chatHistory ?: ChatHistoryPage(messages = response.chatMessages ?: emptyList())
}
/** Reads one bounded page from a verified, currently observable shell. */
suspend fun chatShellOutput(sessionId: String, shellId: String, cursor: Long = 0, limit: Int = ChatProtocol.SHELL_OUTPUT_LIMIT): ChatShellOutputPage {
if (sessionId.isBlank() || shellId.isBlank()) throw ProtocolException("Select a shell first.")
if (cursor < 0 || limit <= 0 || limit > ChatProtocol.SHELL_OUTPUT_LIMIT) {
throw ProtocolException("Shell output bounds are invalid.")
}
return client.call(ChatProtocol.shellOutputRequest(sessionId, shellId, cursor, limit)).chatShellOutput
?: throw ProtocolException("The server returned no shell output page.")
}
/** Lists currently observed shells owned by this operator chat session. */
suspend fun chatShells(sessionId: String): List<ChatShell> {
if (sessionId.isBlank()) throw ProtocolException("Select a chat session first.")
return client.call(ChatProtocol.shellsRequest(sessionId)).chatShells ?: emptyList()
}
/**
* Attaches this connection to [sessionId]'s shared live stream, so it sees
* every frame any client's turn produces — the other devices' prompts and the
@@ -204,6 +204,24 @@ data class ChatShell(
val exit: Int? = null,
)
@Serializable
data class ChatShellOutputQuery(
@SerialName("session_id") val sessionId: String = "",
@SerialName("shell_id") val shellId: String = "",
val cursor: Long = 0,
val limit: Int = ChatProtocol.SHELL_OUTPUT_LIMIT,
)
@Serializable
data class ChatShellOutputPage(
val shell: ChatShell = ChatShell(),
val output: String = "",
val cursor: Long = 0,
val size: Long = 0,
val truncated: Boolean = false,
val unavailable: Boolean = false,
)
@Serializable
data class ChatStep(
val status: String = "",
@@ -285,6 +303,7 @@ object ChatProtocol {
const val SHELL_RUNNING = "running"
const val SHELL_EXITED = "exited"
const val SHELL_OUTPUT_LIMIT = 64 * 1024
/**
* A prompt another client sent, echoed live to this client so it appears in
@@ -362,6 +381,15 @@ object ChatProtocol {
fun interruptRequest(sessionId: String): Message =
Message(type = Protocol.CHAT_INTERRUPT, chatQuery = ChatQuery(sessionId = sessionId))
fun shellOutputRequest(sessionId: String, shellId: String, cursor: Long = 0, limit: Int = SHELL_OUTPUT_LIMIT): Message =
Message(
type = Protocol.CHAT_SHELL_OUTPUT,
chatShellOutputQuery = ChatShellOutputQuery(sessionId, shellId, cursor, limit),
)
fun shellsRequest(sessionId: String): Message =
Message(type = Protocol.CHAT_SHELLS, chatQuery = ChatQuery(sessionId = sessionId))
/**
* The `chat_close` frame that finishes [sessionId]: the server stops any
* running turn (through the same interrupt) and records the session closed on
@@ -79,6 +79,8 @@ object Protocol {
// The operator's directory choices for a new chat (home first, then the
// directories of previous operator sessions, most recently used first).
const val CHAT_DIRECTORIES = "chat_directories"
const val CHAT_SHELLS = "chat_shells"
const val CHAT_SHELL_OUTPUT = "chat_shell_output"
// Server-pushed frames with no matching request.
const val CHAT_EVENT = "chat_event"
const val CHAT_PERMISSION = "chat_permission"
@@ -712,4 +714,7 @@ data class Message(
// close/reopen requests themselves ride on chat_query like chat_interrupt.
@SerialName("chat_close") val chatClose: ChatCloseResult? = null,
@SerialName("chat_directories") val chatDirectories: List<String>? = null,
@SerialName("chat_shell_output_query") val chatShellOutputQuery: ChatShellOutputQuery? = null,
@SerialName("chat_shells") val chatShells: List<ChatShell>? = null,
@SerialName("chat_shell_output") val chatShellOutput: ChatShellOutputPage? = null,
)
@@ -405,6 +405,36 @@ class ChatProtocolTest {
assertEquals(1792112367999L, decodedReply.chatClose!!.closedAt)
}
@Test
fun shellOutputRequestAndPageUseBoundedWireNames() {
val request = ChatProtocol.shellOutputRequest("ses_1", "sh_1", 7, 128)
val decoded = PipelinesJson.decodeFromString(Message.serializer(), PipelinesJson.encodeToString(Message.serializer(), request))
assertEquals(Protocol.CHAT_SHELL_OUTPUT, decoded.type)
assertEquals("ses_1", decoded.chatShellOutputQuery!!.sessionId)
assertEquals("sh_1", decoded.chatShellOutputQuery!!.shellId)
assertEquals(7L, decoded.chatShellOutputQuery!!.cursor)
assertEquals(128, decoded.chatShellOutputQuery!!.limit)
val page = PipelinesJson.decodeFromString(Message.serializer(), """
{"type":"chat_shell_output","chat_shell_output":{"shell":{"id":"sh_1","status":"running"},"output":"marker","cursor":13,"size":20,"truncated":true}}
""".trimIndent())
assertEquals("marker", page.chatShellOutput!!.output)
assertTrue(page.chatShellOutput!!.truncated)
}
@Test
fun shellsRequestUsesSessionQueryAndPreservesListWireName() {
val request = ChatProtocol.shellsRequest("ses_1")
val decoded = PipelinesJson.decodeFromString(Message.serializer(), PipelinesJson.encodeToString(Message.serializer(), request))
assertEquals(Protocol.CHAT_SHELLS, decoded.type)
assertEquals("ses_1", decoded.chatQuery!!.sessionId)
val reply = PipelinesJson.decodeFromString(Message.serializer(), """
{"type":"chat_shells","chat_shells":[{"id":"sh_1","status":"running","command":"printf marker"}]}
""".trimIndent())
assertEquals("sh_1", reply.chatShells!!.single().id)
assertEquals(ChatProtocol.SHELL_RUNNING, reply.chatShells.single().status)
}
private fun toolDuration(part: ChatPart): Long? {
val time = part.time ?: return null
return time.completed - time.ran
+75 -6
View File
@@ -29,6 +29,7 @@ import (
func main() {
addr := flag.String("addr", "127.0.0.1:0", "Pipelines scratch listen address")
scratch := flag.String("scratch-dir", "", "required scratch directory for the database and workspace")
exited := flag.Bool("shell-exited", false, "serve the selected fixture shell as observed exited")
flag.Parse()
if *scratch == "" {
fatal(fmt.Errorf("-scratch-dir is required"))
@@ -46,7 +47,7 @@ func main() {
}
defer upstream.Close()
upstreamURL := "http://" + upstream.Addr().String()
upstreamServer := &http.Server{Handler: server.LargeChatFixtureHandler(workspace), ReadHeaderTimeout: 5 * time.Second}
upstreamServer := &http.Server{Handler: server.LargeChatFixtureHandlerWithShell(workspace, *exited), ReadHeaderTimeout: 5 * time.Second}
go func() { _ = upstreamServer.Serve(upstream) }()
ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
@@ -76,17 +77,85 @@ func main() {
if err != nil {
fatal(err)
}
shellReadbacks, err := verifyShellReadback(ctx, endpoint, *exited)
if err != nil {
fatal(err)
}
<-ctx.Done()
shutdown, stop := context.WithTimeout(context.Background(), 5*time.Second)
pipelinesErr := pipelinesServer.Shutdown(shutdown)
upstreamErr := upstreamServer.Shutdown(shutdown)
stop()
if pipelinesErr != nil {
fatal(pipelinesErr)
}
if upstreamErr != nil {
fatal(upstreamErr)
}
info := buildinfo.Current()
_ = json.NewEncoder(os.Stdout).Encode(map[string]any{
"endpoint": endpoint, "upstream_endpoint": upstreamURL, "backend": attached.Connection.Backend,
"revision": info.Commit, "session_id": server.LargeChatFixtureSessionID, "workspace": workspace,
"newest_readback": readbacks[0], "older_readback": readbacks[1],
"shell_readbacks": shellReadbacks, "clean_shutdown": true, "shutdown_exit_status": 0,
})
<-ctx.Done()
shutdown, stop := context.WithTimeout(context.Background(), 5*time.Second)
defer stop()
_ = pipelinesServer.Shutdown(shutdown)
_ = upstreamServer.Shutdown(shutdown)
}
func verifyShellReadback(ctx context.Context, endpoint string, exited bool) (map[string]any, error) {
evidence := map[string]any{"test_id": "fixture-shell-owned-and-foreign-refusal", "negative": []string{}}
ws, _, err := websocket.Dial(ctx, "ws"+strings.TrimPrefix(endpoint, "http")+"/connect", nil)
if err != nil {
return evidence, err
}
defer ws.Close(websocket.StatusNormalClosure, "fixture complete")
if err := wsjson.Write(ctx, ws, protocol.Message{Type: protocol.ChatShells, ID: "fixture-shells", ChatQuery: &protocol.ChatQuery{SessionID: server.LargeChatFixtureSessionID}}); err != nil {
return evidence, err
}
var listed protocol.Message
if listed, err = readFixtureResponse(ctx, ws, protocol.ChatShells); err != nil {
return evidence, err
}
if !exited && (len(listed.ChatShells) != 1 || listed.ChatShells[0].ID != server.LargeChatFixtureShellID) {
return evidence, fmt.Errorf("fixture shell is not listed")
}
if err := wsjson.Write(ctx, ws, protocol.Message{Type: protocol.ChatShellOutput, ID: "fixture-output", ChatShellOutputQuery: &protocol.ChatShellOutputQuery{SessionID: server.LargeChatFixtureSessionID, ShellID: server.LargeChatFixtureShellID, Limit: 64 << 10}}); err != nil {
return evidence, err
}
page, err := readFixtureResponse(ctx, ws, protocol.ChatShellOutput)
if err != nil {
return evidence, err
}
if page.ChatShellOutput == nil || page.ChatShellOutput.Output != server.LargeChatFixtureShellOutput || page.ChatShellOutput.Shell.Status != map[bool]string{true: "exited", false: "running"}[exited] {
return evidence, fmt.Errorf("unexpected shell readback: %+v", page.ChatShellOutput)
}
owned := map[string]any{"shell_id": page.ChatShellOutput.Shell.ID, "output": page.ChatShellOutput.Output, "cursor": page.ChatShellOutput.Cursor, "status": page.ChatShellOutput.Shell.Status}
if page.ChatShellOutput.Shell.Exit != nil {
owned["exit"] = *page.ChatShellOutput.Shell.Exit
}
evidence["owned"] = owned
negative := evidence["negative"].([]string)
negativeRequests := []protocol.Message{
{Type: protocol.ChatShells, ID: "foreign-list", ChatQuery: &protocol.ChatQuery{SessionID: server.LargeChatFixtureForeignSessionID}},
{Type: protocol.ChatShells, ID: "unknown-list", ChatQuery: &protocol.ChatQuery{SessionID: "ses_unregistered_fixture"}},
{Type: protocol.ChatShellOutput, ID: "foreign-output", ChatShellOutputQuery: &protocol.ChatShellOutputQuery{SessionID: server.LargeChatFixtureSessionID, ShellID: server.LargeChatFixtureForeignShellID, Limit: 64 << 10}},
{Type: protocol.ChatShellOutput, ID: "unknown-output", ChatShellOutputQuery: &protocol.ChatShellOutputQuery{SessionID: "ses_unregistered_fixture", ShellID: server.LargeChatFixtureShellID, Limit: 64 << 10}},
}
for _, request := range negativeRequests {
if err := wsjson.Write(ctx, ws, request); err != nil {
return evidence, err
}
var response protocol.Message
if err := wsjson.Read(ctx, ws, &response); err != nil {
return evidence, err
}
if response.Type != protocol.Error || strings.Contains(response.Error, server.LargeChatFixtureForeignShellOutput) {
return evidence, fmt.Errorf("negative request %s leaked or succeeded: %+v", request.ID, response)
}
negative = append(negative, request.ID)
}
evidence["negative"] = negative
evidence["negative_count"] = len(negative)
return evidence, nil
}
func verifyBroker(ctx context.Context, endpoint string) ([2]map[string]any, error) {
+40
View File
@@ -0,0 +1,40 @@
# Background task output API
Changed: 2026-10-18. The chat websocket now has `chat_shells` for current,
operator-session-owned running shells and `chat_shell_output` for a selected,
bounded (64 KiB) absolute-byte output page. Ownership requires both the
operator-session registry and provider shell metadata `sessionID`; a shared cwd
is not evidence. Output reads first verify provider detail and explicitly mark
removed/provider-unavailable shells unavailable. This is not completed-task
history or restart retention.
Stale shared documentation: the packaged OpenCode operational references under
`internal/author/skill/references/` need a later docs-fold update describing the
two additive websocket messages, 64 KiB maximum page, and current-only history.
Fixture evidence (2026-10-18): `go build -o <scratch>/pipelines-large-chat-fixture
./cmd/pipelines-large-chat-fixture`; then `<scratch>/pipelines-large-chat-fixture
-scratch-dir <scratch>/run -shell-exited`, stopped with SIGTERM and awaited. The
foreground run exercised authenticated `chat_sessions`, three reconnecting history
pages, `chat_shells`, bounded `chat_shell_output`, and four executable negative
readbacks (`foreign-list`, `unknown-list`, `foreign-output`, `unknown-output`) against
scratch DB/workspace only. Its JSON contains `clean_shutdown: true` and
`shutdown_exit_status: 0`, plus the selected exited shell's `exit: 0` readback.
The evidence revision was `ded79277783b6ccdc25eb4ea25fe55d8632b89c7`; session
`ses_large_chat_fixture`, shell `sh_large_chat_fixture`, and fixed marker
`fixture shell output marker\n`. The current list remains running-shell discovery.
Integration coverage added: `TestChatShellOutputRejectsBoundsBeforeRegistryOrProvider`,
`TestChatShellOutputRealHandlerWebsocketOwnedAndForeign`, and
`TestChatShellOutputRealHandlerExitedReadback`,
`TestChatShellOutputRealHandlerUnavailableAndReconnect`, and
`TestChatShellOutputRejectsMalformedAndOversizedProviderPages`, and
`TestChatShellOutputRealHandlerCancellationStopsProvider` in
`internal/server/chat_shell_output_test.go`. Run with
`go test ./internal/server -run 'TestChatShellOutput' -count=1 -v` and
`go test -race ./internal/server -run 'TestChatShellOutput(RealHandler|RejectsBounds)' -count=1 -v`.
These tests use an authenticated in-process HTTP provider, a scratch SQLite operator
registry, and real `/connect` websocket traffic. The fixture executes foreign
session/shell refusal requests and records positive/negative JSON readbacks under
`shell_readbacks`; it remains scratch-only evidence, not evidence of a real provider,
model, production workspace, or device.
+19
View File
@@ -0,0 +1,19 @@
# Inspected resource recovery
The operator recovery contract now keeps quarantined resource ownership and its
audit transactionally coupled to FIFO handoff. Inspection is redacted and
reconciliation is accepted only over a loopback operator connection with the
exact owner visit plus positive cleanup/restoration evidence. The persisted
capability remains private to the authoritative live store; missing waiter
channels fail closed without changing owner or audit state.
Use the configured `PIPELINES_URL` endpoint, not a disposable URL. First run
the redacted resource inspection, then reconcile only after the parked visit
has been inspected and quiesced, supplying its exact run/stage/sequence and
explicit evidence (including `no mutating action occurred` where applicable).
The old deployed server does not support this action; deploy a compatible
server before attempting recovery. This is an operator trust boundary, not an
OS sandbox or unrestricted authorization grant.
The shared README and packaged OpenCode skill references should be updated by
the manual `docs-fold` process; they are intentionally not edited here.
+11
View File
@@ -0,0 +1,11 @@
# Retry from failed stage
As of 2026-10-18, retry requests default to `resume`: the server records a
durable checkpoint for the terminal failed visit and starts the new attempt at
that visit with its completed-prefix handoff context. `--mode=restart` retains
the older full-restart behavior. Older clients that omit the mode therefore
resume rather than restart.
The CLI and protocol now expose the mode and retry provenance. The shared CLI
documentation and packaged skill references should be folded from this note by
the manual `docs-fold` pipeline.
+82 -4
View File
@@ -7,6 +7,7 @@ import (
"flag"
"fmt"
"io"
"net/url"
"os"
"os/exec"
"path/filepath"
@@ -36,7 +37,8 @@ type App struct {
type options struct {
harness string
url, name, prompt, title, template, workspace, paths, key, opencode, model string
option, feedback string
retryMode string
option, feedback, evidence, actor string
output string
conflict string
kind string
@@ -45,7 +47,9 @@ type options struct {
keepSet bool
all bool
json, source, noOpen, yes, allow, createOnly, force, wait bool
version, expected, limit int
version, expected, limit, recoverySequence int
recoveryRun int64
recoveryStage, resource string
timeout, waitTimeout time.Duration
stageModels stageModelFlags
}
@@ -159,6 +163,7 @@ func (a App) flags(command string) (*flag.FlagSet, *options) {
f.StringVar(&o.key, "request-key", "", "Idempotency key (default: generated unique key)")
f.BoolVar(&o.wait, "wait", false, "Wait for the new run to finish")
f.DurationVar(&o.waitTimeout, "wait-timeout", 0, "Client limit for --wait (0 waits until the run ends); independent of --timeout")
f.StringVar(&o.retryMode, "mode", "resume", "Retry mode: resume failed stage or restart from entry")
case "cancel", "draft-delete":
f.BoolVar(&o.yes, "yes", false, "Confirm this action without an interactive prompt")
case "retention":
@@ -177,6 +182,13 @@ func (a App) flags(command string) (*flag.FlagSet, *options) {
case "decide":
f.StringVar(&o.option, "option", "", "Option key to choose among the waiting stage's edges (required)")
f.StringVar(&o.feedback, "feedback", "", "Optional free-text feedback recorded with the decision")
case "resource-inspect":
f.StringVar(&o.resource, "resource", "", "Exclusive resource key")
case "resource-reconcile":
f.StringVar(&o.evidence, "evidence", "", "Bounded positive cleanup/restoration evidence (required)")
f.StringVar(&o.actor, "actor", "operator", "Bounded operator identity")
f.BoolVar(&o.allow, "no-mutating-action", false, "Evidence explicitly records that no mutating action occurred")
f.BoolVar(&o.yes, "yes", false, "Confirm reconciliation without prompting")
case "debug":
f.StringVar(&o.harness, "harness", "", "Debugging harness: configured opencode/pi, or nvim")
f.BoolVar(&o.yes, "yes", false, "Approve opening a debugging session without prompting")
@@ -462,8 +474,16 @@ func (a App) Run(ctx context.Context, args []string) error {
if requestTimeout <= 0 {
requestTimeout = 30 * time.Second
}
connectURL := o.url
if command == "resource-inspect" || command == "resource-reconcile" {
var transportErr error
connectURL, transportErr = operatorTransportURL(o.url)
if transportErr != nil {
return transportErr
}
}
dialCtx, cancel := context.WithTimeout(ctx, requestTimeout)
c, err := client.Open(dialCtx, o.url)
c, err := client.Open(dialCtx, connectURL)
cancel()
if err != nil {
if command == "status" && len(pos) == 0 {
@@ -897,6 +917,51 @@ func (a App) Run(ctx context.Context, args []string) error {
}
}
return fmt.Errorf("template %q not found", key)
case "resource-inspect":
if len(pos) > 1 {
return fmt.Errorf("resource-inspect takes one resource key")
}
if o.resource == "" && len(pos) == 1 {
o.resource = pos[0]
}
if strings.TrimSpace(o.resource) == "" {
return fmt.Errorf("resource-inspect requires a resource key")
}
r, err := call(protocol.Message{Type: protocol.InspectResource, Resource: o.resource})
if err != nil {
return err
}
if r.ResourceInspection == nil {
return fmt.Errorf("server did not return resource inspection")
}
return a.output(r.ResourceInspection)
case "resource-reconcile":
if len(pos) != 4 {
return fmt.Errorf("resource-reconcile requires resource, run-id, stage and sequence")
}
runID, err := strconv.ParseInt(pos[1], 10, 64)
if err != nil || runID <= 0 {
return fmt.Errorf("run ID must be positive")
}
sequence, err := strconv.Atoi(pos[3])
if err != nil || sequence <= 0 {
return fmt.Errorf("sequence must be positive")
}
if strings.TrimSpace(o.evidence) == "" {
return fmt.Errorf("--evidence is required")
}
if err := a.confirmAction(fmt.Sprintf("Reconcile quarantined resource %s for run #%d?", pos[0], runID), o.yes); err != nil {
return err
}
r, err := call(protocol.Message{Type: protocol.ReconcileResource, Resource: pos[0], ResourceRecovery: &protocol.ResourceRecoveryRequest{Resource: pos[0], RunID: runID, Stage: pos[2], Sequence: sequence, Actor: o.actor, Evidence: &protocol.ResourceRecoveryEvidence{CleanupConfirmed: true, NoMutatingAction: o.allow, Summary: o.evidence}}})
if err != nil {
return err
}
if o.json {
return a.output(r)
}
_, err = fmt.Fprintf(a.Out, "Reconciled resource %s for run #%d\n", pos[0], runID)
return err
case "inspect", "cancel", "retry":
value, err := one()
if err != nil {
@@ -920,11 +985,14 @@ func (a App) Run(ctx context.Context, args []string) error {
}
typeName = protocol.RetryRun
key = o.key
if o.retryMode != "resume" && o.retryMode != "restart" {
return fmt.Errorf("--mode must be resume or restart")
}
if key == "" {
key = fmt.Sprintf("retry-%d-%d", os.Getpid(), time.Now().UnixNano())
}
}
r, err := call(protocol.Message{Type: typeName, RunID: id, RequestKey: key})
r, err := call(protocol.Message{Type: typeName, RunID: id, RequestKey: key, RetryMode: o.retryMode})
if err != nil {
return err
}
@@ -1039,6 +1107,16 @@ func (a App) Run(ctx context.Context, args []string) error {
return nil
}
func operatorTransportURL(raw string) (string, error) {
u, err := url.Parse(raw)
if err != nil || u.Scheme == "" || u.Host == "" {
return "", fmt.Errorf("invalid configured pipeline URL")
}
u.Path = "/operator-connect"
u.RawPath = ""
return u.String(), nil
}
func (a App) output(value any) error {
encoder := json.NewEncoder(a.Out)
encoder.SetIndent("", " ")
+36 -34
View File
@@ -2,39 +2,41 @@ package cli
// Help is also the public command inventory for agents and parity checks.
var usage = map[string]string{
"provider-check": "provider-check --model PROVIDER/MODEL [--yes] [--json]",
"new": "new [name] [prompt] [--template KEY] [--no-open | --json]",
"edit": "edit [name|id] [prompt] [--version N] [--no-open | --json]",
"drafts": "drafts [--json]",
"draft-delete": "draft-delete <draft.md> [--json]",
"validate": "validate <draft.md> [--json]",
"submit": "submit <draft.md> [--name NAME] [--create-only] [--yes] [--json]",
"list": "list [--json]",
"search": "search QUERY [--type KIND] [--limit N] [--json]",
"show": "show <name|id> [--version N] [--json]",
"versions": "versions <name|id> [--json]",
"export": "export [name|id ...] [--output FILE.zip] [--all] [--version N]",
"import": "import [FILE.zip] [--all] [--on-conflict ask|keep|both|import] [--yes] [--json]",
"templates": "templates [--json]",
"template": "template <key> [--source | --json]",
"run": "run [name|id] [prompt] [--title TITLE] [--model PROVIDER/MODEL] [--stage-model STAGE=PROVIDER/MODEL ...] [--json [--wait [--wait-timeout DURATION]]]",
"runs": "runs [--pipeline NAME] [--json]",
"inspect": "inspect <run-id> [--json]",
"wait": "wait <run-id> [run-id ...] [--timeout DURATION] [--json]",
"cancel": "cancel <run-id> [--json]",
"decide": "decide <run-id> [stage] --option KEY [--feedback TEXT] [--json]",
"retry": "retry <run-id> [--json [--wait [--wait-timeout DURATION]]]",
"title": "title <run-id> \"<title>\" [--json]",
"debug": "debug <run-id> [--json | --no-open]",
"chat-adopt": "chat-adopt <session-id> [--json]",
"connection": "connection [--json]",
"ping": "ping [--json]",
"version": "version [--json]",
"status": "status [--json]",
"stats": "stats [--json]",
"rpc": "rpc <request.json|->",
"retention": "retention report|apply [--older-than DURATION] [--keep N] [--db FILE] [--artifacts DIR] [--json] [--yes]",
"tui": "tui",
"provider-check": "provider-check --model PROVIDER/MODEL [--yes] [--json]",
"new": "new [name] [prompt] [--template KEY] [--no-open | --json]",
"edit": "edit [name|id] [prompt] [--version N] [--no-open | --json]",
"drafts": "drafts [--json]",
"draft-delete": "draft-delete <draft.md> [--json]",
"validate": "validate <draft.md> [--json]",
"submit": "submit <draft.md> [--name NAME] [--create-only] [--yes] [--json]",
"list": "list [--json]",
"search": "search QUERY [--type KIND] [--limit N] [--json]",
"show": "show <name|id> [--version N] [--json]",
"versions": "versions <name|id> [--json]",
"export": "export [name|id ...] [--output FILE.zip] [--all] [--version N]",
"import": "import [FILE.zip] [--all] [--on-conflict ask|keep|both|import] [--yes] [--json]",
"templates": "templates [--json]",
"template": "template <key> [--source | --json]",
"run": "run [name|id] [prompt] [--title TITLE] [--model PROVIDER/MODEL] [--stage-model STAGE=PROVIDER/MODEL ...] [--json [--wait [--wait-timeout DURATION]]]",
"runs": "runs [--pipeline NAME] [--json]",
"inspect": "inspect <run-id> [--json]",
"wait": "wait <run-id> [run-id ...] [--timeout DURATION] [--json]",
"cancel": "cancel <run-id> [--json]",
"decide": "decide <run-id> [stage] --option KEY [--feedback TEXT] [--json]",
"retry": "retry <run-id> [--mode resume|restart] [--json [--wait [--wait-timeout DURATION]]]",
"title": "title <run-id> \"<title>\" [--json]",
"debug": "debug <run-id> [--json | --no-open]",
"chat-adopt": "chat-adopt <session-id> [--json]",
"connection": "connection [--json]",
"ping": "ping [--json]",
"version": "version [--json]",
"status": "status [--json]",
"stats": "stats [--json]",
"resource-inspect": "resource-inspect <resource> [--json]",
"resource-reconcile": "resource-reconcile <resource> <run-id> <stage> <sequence> --evidence TEXT [--no-mutating-action] [--yes] [--json]",
"rpc": "rpc <request.json|->",
"retention": "retention report|apply [--older-than DURATION] [--keep N] [--db FILE] [--artifacts DIR] [--json] [--yes]",
"tui": "tui",
}
var notes = map[string]string{
@@ -49,7 +51,7 @@ var notes = map[string]string{
"validate": "Uses the same server parser as submission. Does not persist or activate schedules. Structural validity does not prove runtime/tool availability.",
"submit": "Confirm publication at the prompt, or use --yes for explicit automation; schedules activate on acceptance. Managed drafts provide the name and base-version guard. --force never bypasses validation.",
"run": "Runs in your current directory after action confirmation (--yes skips it). Select a pipeline and compose a multiline task with Ctrl+S. Stage tools are enabled automatically; no file list or typed RUN confirmation. Cancellation does not undo edits. --title sets an explicit run title; without it the title is derived from the prompt. --model PROVIDER/MODEL overrides the model for every stage of that run; --stage-model STAGE=PROVIDER/MODEL overrides one stage and is repeatable, with a repeated key last-wins. Precedence is per-stage > whole-run > the model pinned in the pipeline version; overrides apply to that run only and never change the pinned definition. An unknown stage key or a malformed provider/model is refused before the run starts, naming the valid stage keys. In JSON mode --wait polls until terminal; --wait-timeout bounds only the client wait after admission and is independent of per-request --timeout. A reached wait limit leaves the run active: inspect read-only with `inspect <run-id> --json`, or reattach with --wait reusing the original --request-key (never an automatic replay or cancel).",
"retry": "Start a new attempt of a failed run with the same initial input, directory, pipeline version, and edit scope. Original history is retained. Already-completed side effects are not undone. --wait/--wait-timeout behave as for run: a reached wait limit leaves the new run active; inspect read-only with `inspect <run-id> --json`, or reattach with --wait and the original --request-key instead of starting a duplicate retry.",
"retry": "Resume a failed stage by default, restoring the completed prefix handoff; use --mode=restart to start at entry. Original history is retained and already-completed side effects are not undone. --wait/--wait-timeout behave as for run: a reached wait limit leaves the new run active; inspect read-only or reattach with --wait and the original --request-key instead of starting a duplicate retry.",
"title": "Rename a run's operator-facing title after the fact without touching its input, steps or execution. The title is collapsed to a single bounded line and recorded as explicitly set. `pipelines runs` and `pipelines inspect` then show it.",
"wait": "Block until each named run reaches a terminal status - succeeded, failed, canceled or interrupted, and also through a run waiting on a person - then print one line per run with its id, status and outcome, its stage and how long it took, and exit 0 even for a run that finished unsuccessfully. Several ids wait together in one command. The exit is non-zero only for a client-side failure: an unknown run id, a connection that could not be established, or --timeout expiring while a run is still active; a timeout says so plainly, never cancels, retries or otherwise touches a run, and is never reported as its outcome. Read-only: it needs no workspace, is not refused by the workspace guard and writes nothing. --json prints the final run envelope instead of the line (one envelope for one id, an array for several).",
"decide": "Choose an option for a run waiting on a human stage. The run must be `waiting`; the option must be one of that stage's non-error edges. The stage is inferred from the run's current stage unless named. The choice and optional --feedback are recorded as the step's evidence (visible in inspect and the TUI) and the run resumes along the chosen edge. An unknown option is refused with the valid list; a repeated decision for an already-decided visit does not double-apply. Works across a server restart because the wait is persisted.",
+1 -1
View File
@@ -29,7 +29,7 @@ EXECUTE
wait <run-id> ... Block until runs finish, then report each (read-only)
cancel <run-id> Cancel a run; changes are NOT rolled back
decide <run-id> [stage] Choose an option for a run waiting on a person
retry <run-id> Restart a failed run with its original input and directory
retry <run-id> Resume a failed stage (use --mode=restart for a full restart)
title <run-id> "<title>" Rename a run after the fact; input/steps unchanged
debug <run-id> Investigate a run with a configured harness or nvim
chat-adopt <session-id> Adopt an existing OpenCode session into chat_sessions
+83 -1
View File
@@ -33,6 +33,10 @@ type Chat struct {
conn *Connection
shellMu sync.Mutex
shells map[string]*ChatStreamState
// shellOwners retains ownership from the last authoritative global shell
// listing so a shell known to belong to another session is rejected before
// its detail or output endpoint is queried.
shellOwners map[string]string
// mu serializes the adapter's HTTP calls. Connection records its last
// operation and status on the shared connection, so two calls must not
// overlap; the permission poller runs alongside the read loop's
@@ -50,6 +54,21 @@ func NewChat(conn *Connection) *Chat { return &Chat{conn: conn} }
// chat adapter and callers that must reject a request before any OpenCode call.
func ValidSessionID(id string) bool { return sessionID.MatchString(id) }
// ValidShellID accepts only the provider's opaque shell-id alphabet. Keeping
// this check shared by the websocket boundary and provider adapter ensures an
// invalid id is rejected before either network call or path construction.
func ValidShellID(id string) bool {
if len(id) <= 3 || len(id) > 128 || !strings.HasPrefix(id, "sh_") {
return false
}
for _, c := range id[3:] {
if (c < 'a' || c > 'z') && (c < 'A' || c > 'Z') && (c < '0' || c > '9') && c != '_' && c != '-' {
return false
}
}
return true
}
// call serializes the adapter's HTTP calls and forwards them through the shared
// Connection, so no second transport is introduced and Connection's recorded
// last operation/status is never written by two goroutines at once.
@@ -390,9 +409,19 @@ func (c *Chat) Shells(ctx context.Context, sessionID string) ([]protocol.ChatShe
return nil, fmt.Errorf("invalid OpenCode shell response")
}
}
c.shellMu.Lock()
if c.shellOwners == nil {
c.shellOwners = make(map[string]string)
}
for _, info := range response.Data {
if ValidShellID(info.ID) && ValidSessionID(info.shellSessionID()) {
c.shellOwners[info.ID] = info.shellSessionID()
}
}
c.shellMu.Unlock()
out := make([]protocol.ChatShell, 0, len(response.Data))
for _, info := range response.Data {
if info.ID == "" || info.shellSessionID() != sessionID || (info.Status != "" && info.Status != protocol.ChatShellRunning) {
if info.ID == "" || info.shellSessionID() != sessionID || (info.Status != "" && info.Status != protocol.ChatShellRunning && info.Status != protocol.ChatShellExited) {
continue
}
status := info.Status
@@ -418,6 +447,58 @@ func (c *Chat) Shells(ctx context.Context, sessionID string) ([]protocol.ChatShe
return out, nil
}
// ShellOutput validates explicit session metadata from the authoritative detail
// record before reading the selected bounded output page. A shared directory is
// never ownership evidence.
func (c *Chat) ShellOutput(ctx context.Context, query protocol.ChatShellOutputQuery) (protocol.ChatShellOutputPage, error) {
if !ValidSessionID(query.SessionID) || !ValidShellID(query.ShellID) || query.Cursor < 0 || query.Cursor > protocol.ChatShellOutputMaxCursor || query.Limit <= 0 || query.Limit > protocol.ChatShellOutputLimit {
return protocol.ChatShellOutputPage{}, fmt.Errorf("invalid shell output query")
}
c.shellMu.Lock()
owner := c.shellOwners[query.ShellID]
c.shellMu.Unlock()
if owner != "" && owner != query.SessionID {
return protocol.ChatShellOutputPage{}, fmt.Errorf("shell is not available for this session")
}
data, err := c.call(ctx, "get", "/api/shell/"+query.ShellID, nil)
if err != nil {
if c.conn.LastHTTPStatus == 404 {
return protocol.ChatShellOutputPage{Unavailable: true}, nil
}
return protocol.ChatShellOutputPage{}, err
}
var detail struct {
Data chatShellInfo `json:"data"`
}
if err := json.Unmarshal(data, &detail); err != nil || detail.Data.ID != query.ShellID || detail.Data.shellSessionID() != query.SessionID || (detail.Data.Status != "" && detail.Data.Status != protocol.ChatShellRunning && detail.Data.Status != protocol.ChatShellExited) {
return protocol.ChatShellOutputPage{}, fmt.Errorf("shell is not available for this session")
}
shell := protocol.ChatShell{ID: detail.Data.ID, Status: detail.Data.Status, Command: detail.Data.Command, Exit: detail.Data.Exit}
if shell.Status == "" {
shell.Status = protocol.ChatShellRunning
}
path := fmt.Sprintf("/api/shell/%s/output?cursor=%d&limit=%d", query.ShellID, query.Cursor, query.Limit)
data, err = c.call(ctx, "get", path, nil)
if err != nil {
if c.conn.LastHTTPStatus == 404 {
return protocol.ChatShellOutputPage{Shell: shell, Unavailable: true}, nil
}
return protocol.ChatShellOutputPage{}, err
}
var response struct {
Data *struct {
Output *string `json:"output"`
Cursor *int64 `json:"cursor"`
Size *int64 `json:"size"`
Truncated *bool `json:"truncated"`
} `json:"data"`
}
if err := json.Unmarshal(data, &response); err != nil || response.Data == nil || response.Data.Output == nil || response.Data.Cursor == nil || response.Data.Size == nil || response.Data.Truncated == nil || len(*response.Data.Output) > query.Limit || *response.Data.Cursor < query.Cursor || *response.Data.Cursor > protocol.ChatShellOutputMaxCursor || *response.Data.Size < *response.Data.Cursor || *response.Data.Size > protocol.ChatShellOutputMaxCursor || *response.Data.Cursor-query.Cursor != int64(len(*response.Data.Output)) {
return protocol.ChatShellOutputPage{}, fmt.Errorf("invalid OpenCode shell output response")
}
return protocol.ChatShellOutputPage{Shell: shell, Output: *response.Data.Output, Cursor: *response.Data.Cursor, Size: *response.Data.Size, Truncated: *response.Data.Truncated}, nil
}
// streamState returns the session's retained, verified shell identities. It is
// deliberately kept across stream replacement so an exit after reconnect remains
// attributable; Shells reconciles its contents from authoritative readback on
@@ -592,6 +673,7 @@ type chatShellInfo struct {
ID string `json:"id"`
Status string `json:"status"`
Command string `json:"command"`
Exit *int `json:"exit"`
Metadata map[string]any `json:"metadata"`
}
+52
View File
@@ -7,6 +7,7 @@ import (
"net/http"
"net/http/httptest"
"path/filepath"
"strings"
"testing"
"time"
@@ -425,6 +426,57 @@ func TestChatShellReadbackSeedsReplacementStream(t *testing.T) {
}
}
func TestChatShellOutputVerifiesOwnerAndBoundsPage(t *testing.T) {
var outputPath string
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
switch r.URL.Path {
case "/api/shell/sh_1":
_, _ = w.Write([]byte(`{"data":{"id":"sh_1","status":"exited","command":"job","metadata":{"sessionID":"ses_1"}}}`))
case "/api/shell/sh_1/output":
outputPath = r.URL.RawQuery
_, _ = w.Write([]byte(`{"data":{"output":"marker","cursor":6,"size":6,"truncated":false}}`))
default:
http.NotFound(w, r)
}
}))
defer srv.Close()
page, err := NewChat(&Connection{Backend: "v2", URL: srv.URL}).ShellOutput(context.Background(), protocol.ChatShellOutputQuery{SessionID: "ses_1", ShellID: "sh_1", Limit: 64 << 10})
if err != nil || page.Output != "marker" || page.Shell.Status != protocol.ChatShellExited {
t.Fatalf("ShellOutput = %+v, %v", page, err)
}
if !strings.Contains(outputPath, "cursor=0") || !strings.Contains(outputPath, "limit=65536") {
t.Fatalf("output query = %q", outputPath)
}
}
func TestChatShellOutputRejectsMissingRequiredPageFields(t *testing.T) {
for _, body := range []string{
`{"data":{}}`,
`{"data":{"output":"","cursor":0,"size":0}}`,
`{"data":{"output":"","size":0,"truncated":false}}`,
`{"data":{"output":"","cursor":0,"truncated":false}}`,
} {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
switch r.URL.Path {
case "/api/shell/sh_1":
_, _ = w.Write([]byte(`{"data":{"id":"sh_1","status":"running","metadata":{"sessionID":"ses_1"}}}`))
case "/api/shell/sh_1/output":
_, _ = w.Write([]byte(body))
default:
http.NotFound(w, r)
}
}))
chat := NewChat(&Connection{Backend: "v2", URL: srv.URL})
_, err := chat.ShellOutput(context.Background(), protocol.ChatShellOutputQuery{SessionID: "ses_1", ShellID: "sh_1", Limit: 1})
srv.Close()
if err == nil {
t.Fatalf("response %s was accepted", body)
}
}
}
func TestChatReleaseShellsBoundsDisconnectedState(t *testing.T) {
chat := NewChat(&Connection{})
state := chat.streamState("ses_1")
+199 -18
View File
@@ -3,15 +3,184 @@ package pipeline
import (
"context"
"crypto/rand"
"crypto/sha256"
"database/sql"
"encoding/hex"
"encoding/json"
"fmt"
"strings"
"time"
"unicode/utf8"
"git.a-wan.net/alan/pipelines/internal/protocol"
)
const maxRecoveryText = 512
func boundedRecoveryText(value string) string {
value = strings.TrimSpace(value)
return value
}
// InspectResource exposes only bounded state; capability tokens never leave this
// package or enter a protocol value.
func (s *Store) InspectResource(ctx context.Context, resource string) (protocol.ResourceInspection, error) {
if !ValidResourceKey(resource) {
return protocol.ResourceInspection{}, fmt.Errorf("invalid resource")
}
s.resourceMu.Lock()
defer s.resourceMu.Unlock()
out := protocol.ResourceInspection{Resource: resource, Waiters: []protocol.ResourceWaiter{}, Audits: []protocol.ResourceRecoveryAudit{}}
var runID int64
var stage string
var seq, cleanup int
err := s.db.QueryRowContext(ctx, `SELECT run_id,stage_key,sequence,cleanup_confirmed FROM resource_owners WHERE resource_key=?`, resource).Scan(&runID, &stage, &seq, &cleanup)
if err == nil {
var active int
if err := s.db.QueryRowContext(ctx, `SELECT COALESCE(MAX(agent_active),0) FROM run_steps WHERE run_id=? AND sequence=?`, runID, seq).Scan(&active); err != nil {
return out, err
}
out.Owner = &protocol.ResourceOwner{RunID: runID, Stage: stage, Sequence: seq, CleanupConfirmed: cleanup != 0, Quarantined: cleanup == 0, AgentActive: active != 0}
} else if err != sql.ErrNoRows {
return out, err
}
rows, err := s.db.QueryContext(ctx, `SELECT run_id,stage_key,sequence,queued_at FROM resource_waiters WHERE resource_key=? ORDER BY queue_order LIMIT 32`, resource)
if err != nil {
return out, err
}
for rows.Next() {
var w protocol.ResourceWaiter
if err := rows.Scan(&w.RunID, &w.Stage, &w.Sequence, &w.QueuedAt); err != nil {
rows.Close()
return out, err
}
out.Waiters = append(out.Waiters, w)
}
if err := rows.Err(); err != nil {
rows.Close()
return out, err
}
rows.Close()
audits, err := s.db.QueryContext(ctx, `SELECT run_id,stage_key,sequence,actor,source,evidence,result,created_at FROM resource_recovery_audit WHERE resource_key=? ORDER BY rowid DESC LIMIT 16`, resource)
if err != nil {
return out, err
}
defer audits.Close()
for audits.Next() {
var a protocol.ResourceRecoveryAudit
a.Resource = resource
if err := audits.Scan(&a.RunID, &a.Stage, &a.Sequence, &a.Actor, &a.Source, &a.Evidence, &a.Result, &a.CreatedAt); err != nil {
audits.Close()
return out, err
}
out.Audits = append(out.Audits, a)
}
return out, audits.Err()
}
// ReconcileResource is the sole operator recovery mutation. The persisted
// capability is compared only inside this authoritative store operation.
func (s *Store) ReconcileResource(ctx context.Context, request protocol.ResourceRecoveryRequest) error {
if !ValidResourceKey(request.Resource) || request.RunID <= 0 || request.Sequence <= 0 || !identifier.MatchString(request.Stage) {
return fmt.Errorf("invalid resource recovery identity")
}
if request.Evidence == nil || !request.Evidence.CleanupConfirmed || !validRecoveryEvidence(request.Evidence.Summary) {
return fmt.Errorf("positive cleanup/restoration evidence is required")
}
if strings.TrimSpace(request.Actor) == "" || len(request.Actor) > maxRecoveryText {
return fmt.Errorf("bounded actor is required")
}
s.resourceMu.Lock()
defer s.resourceMu.Unlock()
tx, err := s.db.BeginTx(ctx, nil)
if err != nil {
return err
}
defer tx.Rollback()
var token, stage string
var runID int64
var seq, cleanup int
err = tx.QueryRowContext(ctx, `SELECT run_id,stage_key,sequence,token,cleanup_confirmed FROM resource_owners WHERE resource_key=?`, request.Resource).Scan(&runID, &stage, &seq, &token, &cleanup)
if err == sql.ErrNoRows {
var count int
if err := tx.QueryRowContext(ctx, `SELECT COUNT(*) FROM resource_recovery_audit WHERE resource_key=? AND run_id=? AND stage_key=? AND sequence=? AND result='reconciled'`, request.Resource, request.RunID, request.Stage, request.Sequence).Scan(&count); err != nil {
return err
}
if count > 0 {
return nil
}
return fmt.Errorf("resource owner not found")
}
if err != nil {
return err
}
if runID != request.RunID || stage != request.Stage || seq != request.Sequence {
return fmt.Errorf("resource owner visit is stale")
}
if cleanup != 0 {
return fmt.Errorf("resource owner is not quarantined")
}
var status string
if err := tx.QueryRowContext(ctx, `SELECT status FROM pipeline_runs WHERE id=?`, runID).Scan(&status); err != nil {
return err
}
if status == protocol.RunStatusQueued || status == protocol.RunStatusWaiting || status == protocol.RunStatusRunning {
return fmt.Errorf("owning run is still active")
}
var active int
if err := tx.QueryRowContext(ctx, `SELECT COALESCE(MAX(agent_active),0) FROM run_steps WHERE run_id=? AND sequence=?`, runID, seq).Scan(&active); err != nil {
return err
}
if active != 0 {
return fmt.Errorf("owning agent is still active")
}
evidence := boundedRecoveryText(request.Evidence.Summary)
if request.Evidence.NoMutatingAction {
evidence = boundedRecoveryText("no mutating action occurred; " + evidence)
}
evidenceHash := sha256.Sum256([]byte(evidence))
if _, err := tx.ExecContext(ctx, `INSERT INTO resource_recovery_audit(resource_key,run_id,stage_key,sequence,actor,source,evidence,result) VALUES(?,?,?,?,?,?,?,?)`, request.Resource, runID, stage, seq, boundedRecoveryText(request.Actor), "loopback-operator", "sha256:"+hex.EncodeToString(evidenceHash[:]), "reconciled"); err != nil {
return err
}
if _, err := tx.ExecContext(ctx, `UPDATE resource_owners SET cleanup_confirmed=1,diagnostic='' WHERE resource_key=? AND run_id=? AND stage_key=? AND sequence=? AND token=? AND cleanup_confirmed=0`, request.Resource, runID, stage, seq, token); err != nil {
return err
}
owner := ResourceLease{ResourceVisit: ResourceVisit{Resource: request.Resource, RunID: runID, Stage: stage, Sequence: seq}, token: token}
if _, err := deleteResourceOwner(ctx, tx, owner); err != nil {
return err
}
next, err := s.handoffResourceTx(ctx, tx, request.Resource)
if err != nil {
return err
}
if err := tx.Commit(); err != nil {
return err
}
if next.token != "" {
s.resourceWaiters[next.ResourceVisit] <- next
}
return nil
}
func validRecoveryEvidence(value string) bool {
value = strings.TrimSpace(value)
if value == "" || len(value) > maxRecoveryText || !utf8.ValidString(value) {
return false
}
if strings.EqualFold(value, "true") || strings.EqualFold(value, "false") || strings.EqualFold(value, "null") {
return false
}
if strings.HasPrefix(value, "{") || strings.HasPrefix(value, "[") {
return false
}
for _, r := range value {
if r == '\x00' || (r < 0x20 && r != '\t' && r != '\n' && r != '\r') {
return false
}
}
return true
}
// ResourceVisit identifies a pending visit, before it consumes execution accounting.
type ResourceVisit struct {
Resource string
@@ -208,26 +377,10 @@ func (s *Store) releaseResourceLocked(ctx context.Context, lease ResourceLease)
if n == 0 {
return nil
}
var next ResourceLease
next.Resource = lease.Resource
err = tx.QueryRowContext(ctx, `SELECT run_id,stage_key,sequence,token FROM resource_waiters WHERE resource_key=? ORDER BY queue_order LIMIT 1`, lease.Resource).Scan(&next.RunID, &next.Stage, &next.Sequence, &next.token)
if err != nil && err != sql.ErrNoRows {
next, err := s.handoffResourceTx(ctx, tx, lease.Resource)
if err != nil {
return err
}
if err == nil {
if s.resourceWaiters[next.ResourceVisit] == nil {
return fmt.Errorf("resource waiter has no live runner; refusing handoff")
}
if _, err := tx.ExecContext(ctx, `INSERT INTO resource_owners(resource_key,run_id,stage_key,sequence,token) VALUES(?,?,?,?,?)`, next.Resource, next.RunID, next.Stage, next.Sequence, next.token); err != nil {
return err
}
if _, err := tx.ExecContext(ctx, "DELETE FROM resource_waiters WHERE run_id=?", next.RunID); err != nil {
return err
}
if _, err := tx.ExecContext(ctx, "UPDATE pipeline_runs SET resource_wait='' WHERE id=?", next.RunID); err != nil {
return err
}
}
if err := tx.Commit(); err != nil {
return err
}
@@ -237,6 +390,34 @@ func (s *Store) releaseResourceLocked(ctx context.Context, lease ResourceLease)
return nil
}
// handoffResourceTx is shared by ordinary release and inspected recovery. It
// validates the in-memory waiter before changing durable ownership so a stale
// server cannot grant a capability to a runner that no longer exists.
func (s *Store) handoffResourceTx(ctx context.Context, tx *sql.Tx, resource string) (ResourceLease, error) {
var next ResourceLease
next.Resource = resource
err := tx.QueryRowContext(ctx, `SELECT run_id,stage_key,sequence,token FROM resource_waiters WHERE resource_key=? ORDER BY queue_order LIMIT 1`, resource).Scan(&next.RunID, &next.Stage, &next.Sequence, &next.token)
if err == sql.ErrNoRows {
return ResourceLease{}, nil
}
if err != nil {
return ResourceLease{}, err
}
if s.resourceWaiters[next.ResourceVisit] == nil {
return ResourceLease{}, fmt.Errorf("resource waiter has no live runner; refusing handoff")
}
if _, err := tx.ExecContext(ctx, `INSERT INTO resource_owners(resource_key,run_id,stage_key,sequence,token) VALUES(?,?,?,?,?)`, next.Resource, next.RunID, next.Stage, next.Sequence, next.token); err != nil {
return ResourceLease{}, err
}
if _, err := tx.ExecContext(ctx, `DELETE FROM resource_waiters WHERE run_id=?`, next.RunID); err != nil {
return ResourceLease{}, err
}
if _, err := tx.ExecContext(ctx, "UPDATE pipeline_runs SET resource_wait='' WHERE id=?", next.RunID); err != nil {
return ResourceLease{}, err
}
return next, nil
}
func deleteResourceOwner(ctx context.Context, tx *sql.Tx, lease ResourceLease) (int64, error) {
result, err := tx.ExecContext(ctx, `DELETE FROM resource_owners WHERE resource_key=? AND run_id=? AND stage_key=? AND sequence=? AND token=? AND cleanup_confirmed=1`, lease.Resource, lease.RunID, lease.Stage, lease.Sequence, lease.token)
if err != nil {
+242
View File
@@ -9,6 +9,8 @@ import (
"sync"
"testing"
"time"
"git.a-wan.net/alan/pipelines/internal/protocol"
)
func TestResourceDeclaration(t *testing.T) {
@@ -286,3 +288,243 @@ func TestResourceRestartQuarantineAndRetention(t *testing.T) {
})
}
}
func TestResourceReconcileMissingWaiterPreservesQuarantine(t *testing.T) {
s, create := resourceStore(t)
ctx := context.Background()
owner := acquireTestResource(t, s, create("X"))
if _, err := s.db.ExecContext(ctx, "UPDATE pipeline_runs SET status='interrupted' WHERE id=?", owner.RunID); err != nil {
t.Fatal(err)
}
waiter := create("X")
if _, ch, _, err := s.RequestResource(ctx, waiter); err != nil || ch == nil {
t.Fatalf("register waiter: %v", err)
}
// Simulate a runner disappearing while its durable waiter remains. The
// authoritative operation must reject the handoff before audit or cleanup.
delete(s.resourceWaiters, waiter)
err := s.ReconcileResource(ctx, protocol.ResourceRecoveryRequest{
Resource: owner.Resource,
RunID: owner.RunID,
Stage: owner.Stage,
Sequence: owner.Sequence,
Actor: "operator",
Evidence: &protocol.ResourceRecoveryEvidence{CleanupConfirmed: true, NoMutatingAction: true, Summary: "inspection confirmed no mutating action occurred"},
})
if err == nil {
t.Fatal("reconciled without a live waiter channel")
}
var cleanup, audits, owners int
if err := s.db.QueryRow("SELECT cleanup_confirmed FROM resource_owners WHERE resource_key=?", owner.Resource).Scan(&cleanup); err != nil {
t.Fatal(err)
}
if err := s.db.QueryRow("SELECT COUNT(*) FROM resource_recovery_audit WHERE resource_key=?", owner.Resource).Scan(&audits); err != nil {
t.Fatal(err)
}
if err := s.db.QueryRow("SELECT COUNT(*) FROM resource_owners WHERE resource_key=?", owner.Resource).Scan(&owners); err != nil {
t.Fatal(err)
}
if cleanup != 0 || audits != 0 || owners != 1 {
t.Fatalf("failed recovery changed state: cleanup=%d audits=%d owners=%d", cleanup, audits, owners)
}
}
func TestResourceReconcileTransfersAndAudits(t *testing.T) {
s, create := resourceStore(t)
ctx := context.Background()
owner := acquireTestResource(t, s, create("X"))
if _, err := s.db.ExecContext(ctx, "UPDATE pipeline_runs SET status='interrupted' WHERE id=?", owner.RunID); err != nil {
t.Fatal(err)
}
waiter := create("X")
_, ch, _, err := s.RequestResource(ctx, waiter)
if err != nil || ch == nil {
t.Fatalf("register waiter: %v", err)
}
if err := s.ReconcileResource(ctx, protocol.ResourceRecoveryRequest{
Resource: owner.Resource,
RunID: owner.RunID,
Stage: owner.Stage,
Sequence: owner.Sequence,
Actor: "operator",
Evidence: &protocol.ResourceRecoveryEvidence{CleanupConfirmed: true, NoMutatingAction: true, Summary: "inspection confirmed no mutating action occurred"},
}); err != nil {
t.Fatal(err)
}
select {
case next := <-ch:
s.AcceptResource(next)
if next.ResourceVisit != waiter {
t.Fatalf("wrong handoff: %#v", next.ResourceVisit)
}
case <-time.After(time.Second):
t.Fatal("reconciliation did not wake waiter")
}
var audits int
if err := s.db.QueryRow("SELECT COUNT(*) FROM resource_recovery_audit WHERE resource_key=? AND result='reconciled'", owner.Resource).Scan(&audits); err != nil {
t.Fatal(err)
}
if audits != 1 {
t.Fatalf("recovery audit count=%d", audits)
}
}
func TestResourceReconcileReplayAfterSuccessorHandoff(t *testing.T) {
s, create := resourceStore(t)
ctx := context.Background()
owner := acquireTestResource(t, s, create("X"))
if _, err := s.db.ExecContext(ctx, "UPDATE pipeline_runs SET status='interrupted' WHERE id=?", owner.RunID); err != nil {
t.Fatal(err)
}
successor := create("X")
_, ch, _, err := s.RequestResource(ctx, successor)
if err != nil || ch == nil {
t.Fatalf("register successor: %v", err)
}
request := protocol.ResourceRecoveryRequest{
Resource: owner.Resource,
RunID: owner.RunID,
Stage: owner.Stage,
Sequence: owner.Sequence,
Actor: "operator",
Evidence: &protocol.ResourceRecoveryEvidence{
CleanupConfirmed: true,
NoMutatingAction: true,
Summary: "inspection confirmed no mutating action occurred",
},
}
if err := s.ReconcileResource(ctx, request); err != nil {
t.Fatal(err)
}
var successorLease ResourceLease
select {
case successorLease = <-ch:
case <-time.After(time.Second):
t.Fatal("reconciliation did not hand off to successor")
}
if successorLease.ResourceVisit != successor {
t.Fatalf("wrong successor visit: %#v", successorLease.ResourceVisit)
}
if successorLease.token == "" {
t.Fatal("successor capability was empty")
}
s.AcceptResource(successorLease)
// The old request is an idempotent replay after the handoff. It must not
// match or release the successor's private capability.
if err := s.ReconcileResource(ctx, request); err == nil {
t.Fatal("old recovery replay was accepted against successor owner")
}
inspection, err := s.InspectResource(ctx, owner.Resource)
if err != nil {
t.Fatal(err)
}
if inspection.Owner == nil || inspection.Owner.RunID != successor.RunID || inspection.Owner.Stage != successor.Stage || inspection.Owner.Sequence != successor.Sequence {
t.Fatalf("successor was not preserved: %#v", inspection.Owner)
}
if len(inspection.Waiters) != 0 || len(inspection.Audits) != 1 || inspection.Audits[0].RunID != owner.RunID {
t.Fatalf("replay changed handoff state: waiters=%d audits=%#v", len(inspection.Waiters), inspection.Audits)
}
if len(s.resourceWaiters) != 0 {
t.Fatal("successor channel registry leaked")
}
var owners, waiters, audits int
if err := s.db.QueryRowContext(ctx, "SELECT COUNT(*) FROM resource_owners WHERE resource_key=?", owner.Resource).Scan(&owners); err != nil {
t.Fatal(err)
}
if err := s.db.QueryRowContext(ctx, "SELECT COUNT(*) FROM resource_waiters WHERE resource_key=?", owner.Resource).Scan(&waiters); err != nil {
t.Fatal(err)
}
if err := s.db.QueryRowContext(ctx, "SELECT COUNT(*) FROM resource_recovery_audit WHERE resource_key=?", owner.Resource).Scan(&audits); err != nil {
t.Fatal(err)
}
if owners != 1 || waiters != 0 || audits != 1 {
t.Fatalf("unexpected post-replay counts: owners=%d waiters=%d audits=%d", owners, waiters, audits)
}
}
func TestResourceReconcileRequiresQuarantinedInactiveOwner(t *testing.T) {
ctx := context.Background()
test := func(t *testing.T, setup func(*Store, ResourceLease) int) {
s, create := resourceStore(t)
owner := acquireTestResource(t, s, create("X"))
expectedCleanup := setup(s, owner)
err := s.ReconcileResource(ctx, protocol.ResourceRecoveryRequest{
Resource: owner.Resource, RunID: owner.RunID, Stage: owner.Stage, Sequence: owner.Sequence,
Actor: "operator", Evidence: &protocol.ResourceRecoveryEvidence{CleanupConfirmed: true, Summary: "verified"},
})
if err == nil {
t.Fatal("reconciled unsafe owner")
}
var owners, audits, cleanup int
if err := s.db.QueryRow("SELECT COUNT(*) FROM resource_owners").Scan(&owners); err != nil {
t.Fatal(err)
}
if err := s.db.QueryRow("SELECT COUNT(*) FROM resource_recovery_audit").Scan(&audits); err != nil {
t.Fatal(err)
}
if err := s.db.QueryRow("SELECT cleanup_confirmed FROM resource_owners WHERE resource_key=?", owner.Resource).Scan(&cleanup); err != nil {
t.Fatal(err)
}
if owners != 1 || audits != 0 || cleanup != expectedCleanup {
t.Fatalf("unsafe recovery changed state: owners=%d audits=%d cleanup=%d", owners, audits, cleanup)
}
}
t.Run("active run", func(t *testing.T) {
test(t, func(s *Store, owner ResourceLease) int {
if _, err := s.db.Exec("UPDATE pipeline_runs SET status=? WHERE id=?", protocol.RunStatusWaiting, owner.RunID); err != nil {
t.Fatal(err)
}
return 0
})
})
t.Run("active agent", func(t *testing.T) {
test(t, func(s *Store, owner ResourceLease) int {
if _, err := s.db.Exec("UPDATE pipeline_runs SET status='interrupted' WHERE id=?", owner.RunID); err != nil {
t.Fatal(err)
}
if _, err := s.db.Exec("INSERT INTO run_steps(run_id,sequence,stage_key,status,agent_active,input_json) VALUES(?,?,?,?,?, '{}')", owner.RunID, owner.Sequence, owner.Stage, "running", 1); err != nil {
t.Fatal(err)
}
return 0
})
})
t.Run("already confirmed", func(t *testing.T) {
test(t, func(s *Store, owner ResourceLease) int {
if err := s.SetResourceCleanup(ctx, owner, true); err != nil {
t.Fatal(err)
}
return 1
})
})
}
func TestResourceReconcileRejectsMalformedEvidenceBeforeMutation(t *testing.T) {
ctx := context.Background()
for _, summary := range []string{"", "true", "FALSE", "null", "verified\x00cleanup", strings.Repeat("x", maxRecoveryText+1)} {
t.Run(fmt.Sprintf("%q", summary), func(t *testing.T) {
s, create := resourceStore(t)
owner := acquireTestResource(t, s, create("X"))
err := s.ReconcileResource(ctx, protocol.ResourceRecoveryRequest{
Resource: owner.Resource, RunID: owner.RunID, Stage: owner.Stage, Sequence: owner.Sequence,
Actor: "operator", Evidence: &protocol.ResourceRecoveryEvidence{CleanupConfirmed: true, Summary: summary},
})
if err == nil {
t.Fatal("accepted malformed evidence")
}
var owners, audits, cleanup int
if err := s.db.QueryRow("SELECT COUNT(*) FROM resource_owners").Scan(&owners); err != nil {
t.Fatal(err)
}
if err := s.db.QueryRow("SELECT COUNT(*) FROM resource_recovery_audit").Scan(&audits); err != nil {
t.Fatal(err)
}
if err := s.db.QueryRow("SELECT cleanup_confirmed FROM resource_owners WHERE resource_key=?", owner.Resource).Scan(&cleanup); err != nil {
t.Fatal(err)
}
if owners != 1 || audits != 0 || cleanup != 0 {
t.Fatalf("invalid evidence changed state: owners=%d audits=%d cleanup=%d", owners, audits, cleanup)
}
})
}
}
+52
View File
@@ -0,0 +1,52 @@
package pipeline
import (
"context"
"database/sql"
"encoding/json"
"fmt"
"git.a-wan.net/alan/pipelines/internal/protocol"
)
// SaveRetryCheckpoint records the immutable source and the completed-prefix
// handoff before a retry is launched. It intentionally stores no step or usage
// rows: those remain evidence belonging only to the source attempt.
func (s *Store) SaveRetryCheckpoint(ctx context.Context, runID, sourceID int64, mode string, sequence int, source *protocol.RunDetail, handoff map[string]any) error {
if mode == "" {
mode = "resume"
}
data, err := json.Marshal(handoff)
if err != nil {
return err
}
s.mu.Lock()
defer s.mu.Unlock()
var oldSource int64
var oldMode, oldHandoff, oldHash string
var oldSeq, oldVersion int
err = s.db.QueryRowContext(ctx, `SELECT source_run_id,mode,failed_sequence,handoff_json,source_version,source_hash FROM retry_checkpoints WHERE run_id=?`, runID).Scan(&oldSource, &oldMode, &oldSeq, &oldHandoff, &oldVersion, &oldHash)
if err == nil {
if oldSource != sourceID || oldMode != mode || oldSeq != sequence || oldHandoff != string(data) || oldVersion != source.Version || oldHash != source.ContentHash {
return fmt.Errorf("retry checkpoint conflicts with the existing request")
}
return nil
}
if err != sql.ErrNoRows {
return err
}
_, err = s.db.ExecContext(ctx, `INSERT INTO retry_checkpoints(run_id,source_run_id,mode,failed_sequence,handoff_json,source_version,source_hash) VALUES(?,?,?,?,?,?,?)`, runID, sourceID, mode, sequence, string(data), source.Version, source.ContentHash)
return err
}
func (s *Store) RetryCheckpoint(ctx context.Context, runID int64) (sourceID int64, mode string, sequence int, handoff map[string]any, err error) {
s.mu.Lock()
defer s.mu.Unlock()
var data string
err = s.db.QueryRowContext(ctx, `SELECT source_run_id,mode,failed_sequence,handoff_json FROM retry_checkpoints WHERE run_id=?`, runID).Scan(&sourceID, &mode, &sequence, &data)
if err != nil {
return 0, "", 0, nil, err
}
err = json.Unmarshal([]byte(data), &handoff)
return
}
+1 -1
View File
@@ -100,7 +100,7 @@ func (s *Store) GetRun(ctx context.Context, id int64) (*protocol.RunDetail, erro
if err := json.Unmarshal([]byte(graphJSON), &r.Graph); err != nil {
return nil, err
}
if err := tx.QueryRowContext(ctx, "SELECT workspace,allow_tools,write_paths,input_json,output_json,COALESCE((SELECT original_run_id FROM run_retries WHERE run_id=pipeline_runs.id),0),title_source,title_model FROM pipeline_runs WHERE id=?", id).Scan(&r.Workspace, &r.AllowTools, &paths, &input, &output, &r.RetryOf, &r.TitleSource, &r.TitleModel); err != nil {
if err := tx.QueryRowContext(ctx, "SELECT workspace,allow_tools,write_paths,input_json,output_json,COALESCE((SELECT original_run_id FROM run_retries WHERE run_id=pipeline_runs.id),0),COALESCE((SELECT mode FROM retry_checkpoints WHERE run_id=pipeline_runs.id),CASE WHEN EXISTS(SELECT 1 FROM run_retries WHERE run_id=pipeline_runs.id) THEN 'restart' ELSE '' END),COALESCE((SELECT source_run_id FROM retry_checkpoints WHERE run_id=pipeline_runs.id),0),title_source,title_model FROM pipeline_runs WHERE id=?", id).Scan(&r.Workspace, &r.AllowTools, &paths, &input, &output, &r.RetryOf, &r.RetryMode, &r.RetrySource, &r.TitleSource, &r.TitleModel); err != nil {
return nil, err
}
if err := decodeValues(input, &r.Input); err != nil {
+10
View File
@@ -52,6 +52,10 @@ run_id INTEGER NOT NULL, sequence INTEGER NOT NULL, backend TEXT NOT NULL, refer
PRIMARY KEY(run_id,sequence), FOREIGN KEY(run_id,sequence) REFERENCES run_steps(run_id,sequence));
CREATE TABLE IF NOT EXISTS run_retries (
run_id INTEGER PRIMARY KEY REFERENCES pipeline_runs(id), original_run_id INTEGER NOT NULL REFERENCES pipeline_runs(id));
CREATE TABLE IF NOT EXISTS retry_checkpoints (
run_id INTEGER PRIMARY KEY REFERENCES pipeline_runs(id), source_run_id INTEGER NOT NULL REFERENCES pipeline_runs(id),
mode TEXT NOT NULL, failed_sequence INTEGER NOT NULL, handoff_json TEXT NOT NULL,
source_version INTEGER NOT NULL, source_hash TEXT NOT NULL, created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP);
CREATE TABLE IF NOT EXISTS resource_owners (
resource_key TEXT PRIMARY KEY, run_id INTEGER NOT NULL REFERENCES pipeline_runs(id),
stage_key TEXT NOT NULL, sequence INTEGER NOT NULL, token TEXT NOT NULL,
@@ -73,6 +77,12 @@ CREATE INDEX IF NOT EXISTS resource_wait_fifo ON resource_waiters(resource_key,q
if err := s.addColumnIfMissing("pipeline_runs", "resource_wait", "TEXT NOT NULL DEFAULT ''"); err != nil {
return err
}
if _, err := s.db.Exec(`CREATE TABLE IF NOT EXISTS resource_recovery_audit (
resource_key TEXT NOT NULL, run_id INTEGER NOT NULL, stage_key TEXT NOT NULL,
sequence INTEGER NOT NULL, actor TEXT NOT NULL, source TEXT NOT NULL,
evidence TEXT NOT NULL, result TEXT NOT NULL, created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP)`); err != nil {
return err
}
return s.migrateRunModels()
}
+28 -2
View File
@@ -54,6 +54,10 @@ const (
// load the choices without first listing sessions and chat_sessions stays a
// pure, fail-closed operator-session list.
ChatDirectories = "chat_directories"
// ChatShells lists current, verified shells belonging to one operator chat
// session. ChatShellOutput reads a bounded page from one such shell.
ChatShells = "chat_shells"
ChatShellOutput = "chat_shell_output"
// ChatEvent and ChatPermission are the server-pushed frames; they are the
// only types the server sends without a matching request.
ChatEvent = "chat_event"
@@ -89,8 +93,10 @@ const (
// ChatShell.Status values, mirroring the OpenCode shell events. A running shell
// is one whose command has started and not yet exited; exited is final.
const (
ChatShellRunning = "running"
ChatShellExited = "exited"
ChatShellRunning = "running"
ChatShellExited = "exited"
ChatShellOutputLimit = 64 << 10
ChatShellOutputMaxCursor = int64(1 << 62)
)
// ChatStep.Status values, mirroring the OpenCode session.step events.
@@ -363,6 +369,26 @@ type ChatShell struct {
Exit *int `json:"exit,omitempty"`
}
// ChatShellOutputQuery names one selected shell and a bounded absolute-byte
// output page. Cursor is decimal so it can be passed directly to OpenCode.
type ChatShellOutputQuery struct {
SessionID string `json:"session_id"`
ShellID string `json:"shell_id"`
Cursor int64 `json:"cursor,omitempty"`
Limit int `json:"limit,omitempty"`
}
// ChatShellOutputPage is a bounded, lazy output read. Unavailable is explicit when
// an exited shell is no longer directly verifiable upstream.
type ChatShellOutputPage struct {
Shell ChatShell `json:"shell"`
Output string `json:"output,omitempty"`
Cursor int64 `json:"cursor,omitempty"`
Size int64 `json:"size,omitempty"`
Truncated bool `json:"truncated,omitempty"`
Unavailable bool `json:"unavailable,omitempty"`
}
// ChatPermissionFrame is the server-pushed chat_permission payload: one pending
// OpenCode permission request the phone must approve or reject. It is a distinct
// Message field (like ChatEvent) rather than a ChatEvent.Kind, so it does not
+30 -16
View File
@@ -3,19 +3,23 @@ package protocol
import "git.a-wan.net/alan/pipelines/internal/buildinfo"
const (
Ping = "ping"
Pong = "pong"
Error = "error"
Submit = "submit"
Submitted = "submitted"
List = "list_pipelines"
PipelineList = "pipelines"
GetPipeline = "get_pipeline"
Pipeline = "pipeline"
ListTemplates = "list_templates"
Templates = "templates"
Validate = "validate"
Validated = "validated"
Ping = "ping"
Pong = "pong"
Error = "error"
Submit = "submit"
Submitted = "submitted"
List = "list_pipelines"
PipelineList = "pipelines"
GetPipeline = "get_pipeline"
Pipeline = "pipeline"
ListTemplates = "list_templates"
Templates = "templates"
Validate = "validate"
Validated = "validated"
InspectResource = "inspect_resource"
ResourceInspectionResult = "resource_inspection"
ReconcileResource = "reconcile_resource"
ResourceReconciled = "resource_reconciled"
)
type Stage struct {
@@ -155,6 +159,10 @@ type Message struct {
// them; the same fields are echoed on the run record.
Model string `json:"model,omitempty"`
StageModels map[string]string `json:"stage_models,omitempty"`
// Empty (or "resume") retries the failed visit; "restart" starts at the
// entry stage. Older clients omit this field and retain resume behavior.
RetryMode string `json:"retry_mode,omitempty"`
RetrySource int64 `json:"-"`
// Search is the broad, read-only search traffic over pipelines, versions,
// runs, steps, field text and artifacts. See search.go for the per-type
// payloads and the ranking/kind/bound vocabulary.
@@ -187,10 +195,16 @@ type Message struct {
// chat_directories response (home first, then recent operator-session
// directories, deduplicated and bounded). Additive/omitempty so older peers
// ignore it.
ChatDirectories []string `json:"chat_directories,omitempty"`
ChatDirectories []string `json:"chat_directories,omitempty"`
ChatShellOutputQuery *ChatShellOutputQuery `json:"chat_shell_output_query,omitempty"`
ChatShells []ChatShell `json:"chat_shells,omitempty"`
ChatShellOutput *ChatShellOutputPage `json:"chat_shell_output,omitempty"`
// RunEvent is the server-pushed run state delta (see runevents.go) and
// RunIDs the optional subscribe scope, so a client asks for the runs it
// displays. Both are optional so older peers ignore them.
RunEvent *RunEventFrame `json:"run_event,omitempty"`
RunIDs []int64 `json:"run_ids,omitempty"`
RunEvent *RunEventFrame `json:"run_event,omitempty"`
RunIDs []int64 `json:"run_ids,omitempty"`
Resource string `json:"resource,omitempty"`
ResourceInspection *ResourceInspection `json:"resource_inspection,omitempty"`
ResourceRecovery *ResourceRecoveryRequest `json:"resource_recovery,omitempty"`
}
+46
View File
@@ -0,0 +1,46 @@
package protocol
type ResourceInspection struct {
Resource string `json:"resource"`
Owner *ResourceOwner `json:"owner,omitempty"`
Waiters []ResourceWaiter `json:"waiters,omitempty"`
Audits []ResourceRecoveryAudit `json:"audits,omitempty"`
}
type ResourceOwner struct {
RunID int64 `json:"run_id"`
Stage string `json:"stage"`
Sequence int `json:"sequence"`
CleanupConfirmed bool `json:"cleanup_confirmed"`
Quarantined bool `json:"quarantined"`
AgentActive bool `json:"agent_active"`
}
type ResourceWaiter struct {
RunID int64 `json:"run_id"`
Stage string `json:"stage"`
Sequence int `json:"sequence"`
QueuedAt string `json:"queued_at"`
}
type ResourceRecoveryAudit struct {
Resource string `json:"resource"`
RunID int64 `json:"run_id"`
Stage string `json:"stage"`
Sequence int `json:"sequence"`
Actor string `json:"actor"`
Source string `json:"source"`
Evidence string `json:"evidence"`
Result string `json:"result"`
CreatedAt string `json:"created_at"`
}
type ResourceRecoveryEvidence struct {
CleanupConfirmed bool `json:"cleanup_confirmed"`
NoMutatingAction bool `json:"no_mutating_action"`
Summary string `json:"summary"`
}
type ResourceRecoveryRequest struct {
Resource string `json:"resource"`
RunID int64 `json:"run_id"`
Stage string `json:"stage"`
Sequence int `json:"sequence"`
Evidence *ResourceRecoveryEvidence `json:"evidence,omitempty"`
Actor string `json:"actor"`
}
+10 -8
View File
@@ -370,14 +370,16 @@ type NotificationReceipt struct {
type RunDetail struct {
RunSummary
Graph Graph `json:"graph"`
RetryOf int64 `json:"retry_of,omitempty"`
Workspace string `json:"workspace"`
AllowTools bool `json:"allow_tools"`
WritePaths []string `json:"write_paths"`
Input map[string]any `json:"input"`
Output map[string]any `json:"output"`
Steps []RunStep `json:"steps"`
Graph Graph `json:"graph"`
RetryOf int64 `json:"retry_of,omitempty"`
RetryMode string `json:"retry_mode,omitempty"`
RetrySource int64 `json:"retry_source,omitempty"`
Workspace string `json:"workspace"`
AllowTools bool `json:"allow_tools"`
WritePaths []string `json:"write_paths"`
Input map[string]any `json:"input"`
Output map[string]any `json:"output"`
Steps []RunStep `json:"steps"`
// Usage is the totalled reported usage of the run's model stages, summed in
// one pass over Steps. It is absent for a run with no model usage recorded
// (including historical runs).
+10
View File
@@ -42,6 +42,10 @@ type chatBroker interface {
Reply(ctx context.Context, request protocol.ChatReplyRequest) error
}
type chatShellOutputReader interface {
ShellOutput(ctx context.Context, query protocol.ChatShellOutputQuery) (protocol.ChatShellOutputPage, error)
}
// chatShellReleaser is implemented by brokers that retain verified shell ids for
// attributing exit events. The hub calls it only after the last observer leaves
// a room; a later attach obtains a fresh authoritative snapshot.
@@ -128,6 +132,12 @@ func handleChat(ctx context.Context, request protocol.Message, broker chatBroker
case protocol.ChatReopen:
handleChatReopen(ctx, request, registry, write)
return true
case protocol.ChatShells:
handleChatShells(ctx, request, broker, registry, write)
return true
case protocol.ChatShellOutput:
handleChatShellOutput(ctx, request, broker, registry, write)
return true
}
return false
}
+76
View File
@@ -0,0 +1,76 @@
package server
import (
"context"
"fmt"
"git.a-wan.net/alan/pipelines/internal/opencode"
"git.a-wan.net/alan/pipelines/internal/protocol"
)
func operatorChatSession(ctx context.Context, registry operatorRegistry, id string) error {
if registry == nil {
return fmt.Errorf("operator session registry is not configured on this server")
}
_, found, err := registry.OperatorSession(ctx, id)
if err != nil {
return err
}
if !found {
return fmt.Errorf("operator session is not available")
}
return nil
}
func handleChatShells(ctx context.Context, request protocol.Message, broker chatBroker, registry operatorRegistry, write func(protocol.Message) error) {
query := request.ChatQuery
if broker == nil {
_ = write(protocol.Message{Type: protocol.Error, ID: request.ID, Error: "chat is not configured on this server"})
return
}
if query == nil || !opencode.ValidSessionID(query.SessionID) {
_ = write(protocol.Message{Type: protocol.Error, ID: request.ID, Error: "a valid OpenCode session id is required"})
return
}
if err := operatorChatSession(ctx, registry, query.SessionID); err != nil {
_ = write(protocol.Message{Type: protocol.Error, ID: request.ID, Error: err.Error()})
return
}
shells, err := broker.Shells(ctx, query.SessionID)
if err != nil {
_ = write(protocol.Message{Type: protocol.Error, ID: request.ID, Error: err.Error()})
return
}
_ = write(protocol.Message{Type: protocol.ChatShells, ID: request.ID, ChatShells: shells})
}
func handleChatShellOutput(ctx context.Context, request protocol.Message, broker chatBroker, registry operatorRegistry, write func(protocol.Message) error) {
query := request.ChatShellOutputQuery
if broker == nil {
_ = write(protocol.Message{Type: protocol.Error, ID: request.ID, Error: "chat is not configured on this server"})
return
}
if query == nil || !opencode.ValidSessionID(query.SessionID) || !validShellID(query.ShellID) || query.Cursor < 0 || query.Cursor > protocol.ChatShellOutputMaxCursor || query.Limit <= 0 || query.Limit > protocol.ChatShellOutputLimit {
_ = write(protocol.Message{Type: protocol.Error, ID: request.ID, Error: "a valid bounded shell output query is required"})
return
}
if err := operatorChatSession(ctx, registry, query.SessionID); err != nil {
_ = write(protocol.Message{Type: protocol.Error, ID: request.ID, Error: err.Error()})
return
}
reader, ok := broker.(chatShellOutputReader)
if !ok {
_ = write(protocol.Message{Type: protocol.Error, ID: request.ID, Error: "shell output is not configured on this server"})
return
}
output, err := reader.ShellOutput(ctx, *query)
if err != nil {
_ = write(protocol.Message{Type: protocol.Error, ID: request.ID, Error: err.Error()})
return
}
_ = write(protocol.Message{Type: protocol.ChatShellOutput, ID: request.ID, ChatShellOutput: &output})
}
func validShellID(id string) bool {
return opencode.ValidShellID(id)
}
+280
View File
@@ -0,0 +1,280 @@
package server
import (
"context"
"fmt"
"net/http"
"net/http/httptest"
"strings"
"sync/atomic"
"testing"
"time"
"github.com/coder/websocket"
"github.com/coder/websocket/wsjson"
"git.a-wan.net/alan/pipelines/internal/opencode"
"git.a-wan.net/alan/pipelines/internal/protocol"
)
type shellOutputTestBroker struct {
*fakeChat
outputCalls int
}
func (b *shellOutputTestBroker) ShellOutput(context.Context, protocol.ChatShellOutputQuery) (protocol.ChatShellOutputPage, error) {
b.outputCalls++
return protocol.ChatShellOutputPage{Output: "unexpected"}, nil
}
func TestChatShellOutputRejectsBoundsBeforeRegistryOrProvider(t *testing.T) {
store := newOperatorStore(t)
broker := &shellOutputTestBroker{fakeChat: scriptedChat()}
queries := []protocol.ChatShellOutputQuery{
{SessionID: "ses_1", ShellID: "sh_1", Limit: 0},
{SessionID: "ses_1", ShellID: "sh_1", Limit: -1},
{SessionID: "ses_1", ShellID: "sh_1", Cursor: protocol.ChatShellOutputMaxCursor + 1, Limit: 1},
{SessionID: "ses_1", ShellID: "sh_1/../x", Limit: 1},
{SessionID: "ses_1", ShellID: "sh_1%2f", Limit: 1},
{SessionID: "ses_1", ShellID: "sh_1?cursor=0", Limit: 1},
{SessionID: "ses_1", ShellID: "sh_1#fragment", Limit: 1},
{SessionID: "ses_1", ShellID: "sh_%ZZ", Limit: 1},
{SessionID: "ses_1", ShellID: "sh_" + string(make([]byte, 126)), Limit: 1},
}
for _, query := range queries {
var response protocol.Message
handleChatShellOutput(context.Background(), protocol.Message{ChatShellOutputQuery: &query}, broker, store, func(m protocol.Message) error {
response = m
return nil
})
if response.Type != protocol.Error {
t.Fatalf("query %+v response = %+v", query, response)
}
}
if broker.outputCalls != 0 {
t.Fatalf("invalid queries reached provider %d times", broker.outputCalls)
}
}
// TestChatShellOutputRealHandlerWebsocketOwnedAndForeign drives the real
// authenticated adapter, HTTP fixture, HandlerWithOptions, and /connect path.
// The foreign detail deliberately has a marker that would be returned by its
// output route; the session metadata check must reject it first.
func TestChatShellOutputRealHandlerWebsocketOwnedAndForeign(t *testing.T) {
testChatShellOutputOwnedAndForeign(t, false)
}
func TestChatShellOutputRealHandlerExitedReadback(t *testing.T) {
testChatShellOutputOwnedAndForeign(t, true)
}
func testChatShellOutputOwnedAndForeign(t *testing.T, exited bool) {
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
var providerRequests atomic.Int32
fixtureHandler := LargeChatFixtureHandlerWithShell(t.TempDir(), exited)
fixture := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
providerRequests.Add(1)
fixtureHandler.ServeHTTP(w, r)
}))
defer fixture.Close()
attached, err := opencode.OpenEndpointAttached(ctx, fixture.URL, LargeChatFixturePassword, "", "")
if err != nil {
t.Fatal(err)
}
store := newOperatorStore(t)
if err := store.RecordOperatorSession(ctx, LargeChatFixtureSessionID, t.TempDir(), "owned", protocol.ChatSessionTagOperator); err != nil {
t.Fatal(err)
}
srv := httptest.NewServer(HandlerWithOptions(ctx, Options{Chat: opencode.NewChat(attached.Connection), OperatorRegistry: store}))
defer srv.Close()
ws := dialShellOutputWebsocket(t, ctx, srv)
defer ws.CloseNow()
writeChat(t, ctx, ws, protocol.Message{Type: protocol.ChatShells, ID: "owned-shells", ChatQuery: &protocol.ChatQuery{SessionID: LargeChatFixtureSessionID}})
owned := readShellOutputMessage(t, ctx, ws)
if owned.Type != protocol.ChatShells || (exited && len(owned.ChatShells) != 0) || (!exited && (len(owned.ChatShells) != 1 || owned.ChatShells[0].ID != LargeChatFixtureShellID || owned.ChatShells[0].Status != protocol.ChatShellRunning)) {
t.Fatalf("owned shell listing = %+v", owned)
}
wantStatus := protocol.ChatShellRunning
if exited {
wantStatus = protocol.ChatShellExited
}
writeChat(t, ctx, ws, protocol.Message{Type: protocol.ChatShellOutput, ID: "owned-output", ChatShellOutputQuery: &protocol.ChatShellOutputQuery{SessionID: LargeChatFixtureSessionID, ShellID: LargeChatFixtureShellID, Limit: protocol.ChatShellOutputLimit}})
page := readShellOutputMessage(t, ctx, ws)
if page.Type != protocol.ChatShellOutput || page.ChatShellOutput == nil || page.ChatShellOutput.Output != LargeChatFixtureShellOutput || page.ChatShellOutput.Cursor != int64(len(LargeChatFixtureShellOutput)) || page.ChatShellOutput.Shell.Status != wantStatus {
t.Fatalf("owned output page = %+v", page)
}
if exited {
if page.ChatShellOutput.Shell.Exit == nil || *page.ChatShellOutput.Shell.Exit != 0 {
t.Fatalf("selected exited shell readback = %+v", page.ChatShellOutput.Shell)
}
} else if page.ChatShellOutput.Shell.Exit != nil {
t.Fatalf("running shell unexpectedly has exit = %+v", page.ChatShellOutput.Shell.Exit)
}
if page.ChatShellOutput.Cursor != int64(len(page.ChatShellOutput.Output)) || page.ChatShellOutput.Size != page.ChatShellOutput.Cursor {
t.Fatalf("non-deterministic output cursor/size = %+v", page.ChatShellOutput)
}
refusals := []struct {
name string
request protocol.Message
}{
{name: "foreign-output", request: protocol.Message{Type: protocol.ChatShellOutput, ID: "foreign-output", ChatShellOutputQuery: &protocol.ChatShellOutputQuery{SessionID: LargeChatFixtureSessionID, ShellID: LargeChatFixtureForeignShellID, Limit: protocol.ChatShellOutputLimit}}},
{name: "foreign-list", request: protocol.Message{Type: protocol.ChatShells, ID: "foreign-list", ChatQuery: &protocol.ChatQuery{SessionID: LargeChatFixtureForeignSessionID}}},
{name: "syntax-invalid-list", request: protocol.Message{Type: protocol.ChatShells, ID: "invalid-list", ChatQuery: &protocol.ChatQuery{SessionID: "ses_bad/invalid"}}},
{name: "unknown-output", request: protocol.Message{Type: protocol.ChatShellOutput, ID: "unknown-output", ChatShellOutputQuery: &protocol.ChatShellOutputQuery{SessionID: "ses_unregistered_fixture", ShellID: LargeChatFixtureShellID, Limit: protocol.ChatShellOutputLimit}}},
}
for _, refusal := range refusals {
beforeRefusal := providerRequests.Load()
writeChat(t, ctx, ws, refusal.request)
response := readShellOutputMessage(t, ctx, ws)
if response.Type != protocol.Error || strings.Contains(response.Error, LargeChatFixtureForeignShellOutput) {
t.Fatalf("%s response leaked or succeeded: %+v", refusal.name, response)
}
if got := providerRequests.Load(); got != beforeRefusal {
t.Fatalf("%s reached provider: requests %d -> %d", refusal.name, beforeRefusal, got)
}
}
}
func TestChatShellOutputRealHandlerUnavailableAndReconnect(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
fixture := httptest.NewServer(LargeChatFixtureHandlerWithShell(t.TempDir(), false))
defer fixture.Close()
attached, err := opencode.OpenEndpointAttached(ctx, fixture.URL, LargeChatFixturePassword, "", "")
if err != nil {
t.Fatal(err)
}
store := newOperatorStore(t)
if err := store.RecordOperatorSession(ctx, LargeChatFixtureSessionID, t.TempDir(), "owned", protocol.ChatSessionTagOperator); err != nil {
t.Fatal(err)
}
srv := httptest.NewServer(HandlerWithOptions(ctx, Options{Chat: opencode.NewChat(attached.Connection), OperatorRegistry: store}))
defer srv.Close()
ws := dialShellOutputWebsocket(t, ctx, srv)
writeChat(t, ctx, ws, protocol.Message{Type: protocol.ChatShellOutput, ID: "missing-output", ChatShellOutputQuery: &protocol.ChatShellOutputQuery{SessionID: LargeChatFixtureSessionID, ShellID: "sh_missing_fixture", Limit: 64}})
missing := readShellOutputMessage(t, ctx, ws)
if missing.Type != protocol.ChatShellOutput || missing.ChatShellOutput == nil || !missing.ChatShellOutput.Unavailable {
t.Fatalf("404 shell output = %+v", missing)
}
ws = dialShellOutputWebsocket(t, ctx, srv)
defer ws.CloseNow()
writeChat(t, ctx, ws, protocol.Message{Type: protocol.ChatShells, ID: "reconnect", ChatQuery: &protocol.ChatQuery{SessionID: LargeChatFixtureSessionID}})
if got := readShellOutputMessage(t, ctx, ws); got.Type != protocol.ChatShells {
t.Fatalf("reconnect shell listing = %+v", got)
}
if _, err := opencode.OpenEndpointAttached(ctx, fixture.URL, "wrong-password", "", ""); err == nil {
t.Fatal("wrong provider authentication unexpectedly succeeded")
}
}
func TestChatShellOutputRejectsMalformedAndOversizedProviderPages(t *testing.T) {
for _, tc := range []struct {
name string
output string
}{
{name: "malformed", output: `{"data":{"output":"x"}}`},
{name: "oversized", output: fmt.Sprintf(`{"data":{"output":%q,"cursor":%d,"size":%d,"truncated":false}}`, strings.Repeat("x", protocol.ChatShellOutputLimit+1), protocol.ChatShellOutputLimit+1, protocol.ChatShellOutputLimit+1)},
} {
t.Run(tc.name, func(t *testing.T) {
fixture := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if !largeChatFixtureAuthenticated(w, r) {
return
}
w.Header().Set("Content-Type", "application/json")
switch r.URL.Path {
case "/api/info":
_, _ = w.Write([]byte(`{"version":"fixture","urls":[]}`))
case "/api/event":
w.Header().Set("Content-Type", "text/event-stream")
case "/api/shell/sh_malformed_fixture":
_, _ = w.Write([]byte(`{"data":{"id":"sh_malformed_fixture","status":"running","metadata":{"sessionID":"ses_malformed_fixture"}}}`))
case "/api/shell/sh_malformed_fixture/output":
_, _ = w.Write([]byte(tc.output))
default:
if r.URL.Path == "/api/shell" {
_, _ = w.Write([]byte(`{"data":[]}`))
return
}
http.NotFound(w, r)
}
}))
defer fixture.Close()
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
attached, err := opencode.OpenEndpointAttached(ctx, fixture.URL, LargeChatFixturePassword, "", "")
if err != nil {
t.Fatal(err)
}
_, err = opencode.NewChat(attached.Connection).ShellOutput(ctx, protocol.ChatShellOutputQuery{SessionID: "ses_malformed_fixture", ShellID: "sh_malformed_fixture", Limit: protocol.ChatShellOutputLimit})
if err == nil {
t.Fatalf("%s provider page was accepted", tc.name)
}
})
}
}
func TestChatShellOutputRealHandlerCancellationStopsProvider(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
started := make(chan struct{})
canceled := make(chan struct{})
fixtureHandler := LargeChatFixtureHandlerWithShell(t.TempDir(), false)
fixture := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path == "/api/shell/sh_cancel_fixture" {
if !largeChatFixtureAuthenticated(w, r) {
return
}
close(started)
<-r.Context().Done()
close(canceled)
return
}
fixtureHandler.ServeHTTP(w, r)
}))
defer fixture.Close()
attached, err := opencode.OpenEndpointAttached(ctx, fixture.URL, LargeChatFixturePassword, "", "")
if err != nil {
t.Fatal(err)
}
store := newOperatorStore(t)
if err := store.RecordOperatorSession(ctx, "ses_cancel_fixture", t.TempDir(), "cancel", protocol.ChatSessionTagOperator); err != nil {
t.Fatal(err)
}
serverCtx, serverCancel := context.WithCancel(ctx)
defer serverCancel()
srv := httptest.NewServer(HandlerWithOptions(serverCtx, Options{Chat: opencode.NewChat(attached.Connection), OperatorRegistry: store}))
defer srv.Close()
ws := dialShellOutputWebsocket(t, ctx, srv)
writeChat(t, ctx, ws, protocol.Message{Type: protocol.ChatShellOutput, ID: "cancel-output", ChatShellOutputQuery: &protocol.ChatShellOutputQuery{SessionID: "ses_cancel_fixture", ShellID: "sh_cancel_fixture", Limit: 64}})
select {
case <-started:
case <-ctx.Done():
t.Fatal("provider detail request did not start")
}
serverCancel()
select {
case <-canceled:
case <-ctx.Done():
t.Fatal("canceling handler did not cancel provider request")
}
}
func dialShellOutputWebsocket(t *testing.T, ctx context.Context, srv *httptest.Server) *websocket.Conn {
t.Helper()
ws, _, err := websocket.Dial(ctx, "ws"+strings.TrimPrefix(srv.URL, "http")+"/connect", nil)
if err != nil {
t.Fatal(err)
}
return ws
}
func readShellOutputMessage(t *testing.T, ctx context.Context, ws *websocket.Conn) protocol.Message {
t.Helper()
var message protocol.Message
if err := wsjson.Read(ctx, ws, &message); err != nil {
t.Fatal(err)
}
return message
}
+14 -10
View File
@@ -341,20 +341,23 @@ func TestRetryConflictsOnlyWithSameWorkspaceActiveRun(t *testing.T) {
if err != nil || failed.Status != "failed" {
t.Fatalf("original run did not fail as intended: %+v %v", failed, err)
}
resolvedB, err := filepath.EvalSymlinks(b)
entered := make(chan struct{}, 4)
resolvedA, err := filepath.EvalSymlinks(a)
if err != nil {
t.Fatal(err)
}
entered := make(chan struct{}, 4)
release := make(chan struct{})
releaseA := make(chan struct{})
releaseB := make(chan struct{})
r.execute = func(ctx context.Context, _ stage.Definition, input stage.Values) (stage.Result, error) {
entered <- struct{}{}
if input["pwd"] == resolvedB {
select {
case <-release:
case <-ctx.Done():
return stage.Result{}, ctx.Err()
}
release := releaseB
if input["pwd"] == resolvedA {
release = releaseA
}
select {
case <-release:
case <-ctx.Done():
return stage.Result{}, ctx.Err()
}
return stage.Result{Outcome: "success", Output: stage.Values{}}, nil
}
@@ -366,6 +369,7 @@ func TestRetryConflictsOnlyWithSameWorkspaceActiveRun(t *testing.T) {
if _, err := r.retry(protocol.Message{RunID: original.ID, RequestKey: "retry-a"}); err == nil || !strings.Contains(err.Error(), fmt.Sprintf("run %d is already active", activeA.ID)) || !strings.Contains(err.Error(), "different workspace") {
t.Fatalf("retry did not conflict with same-workspace active run: %v", err)
}
close(releaseA)
// A different active workspace must not block the retry of the original.
if _, err := r.start(protocol.Message{PipelineID: pipelineID, Version: 1, RequestKey: "active-b", Workspace: b, Input: map[string]any{"prompt": "task"}}); err != nil {
t.Fatal(err)
@@ -378,7 +382,7 @@ func TestRetryConflictsOnlyWithSameWorkspaceActiveRun(t *testing.T) {
if retried.Workspace != failed.Workspace {
t.Fatalf("retry did not reuse the original workspace: %q != %q", retried.Workspace, failed.Workspace)
}
close(release)
close(releaseB)
r.wg.Wait()
}
+1 -1
View File
@@ -270,7 +270,7 @@ func TestHumanStageCancelRetryAndRebuild(t *testing.T) {
t.Fatalf("post-decision failure did not fail the run: %+v", failed.RunSummary)
}
r3.execute = successExecute
retried, err := r3.retry(protocol.Message{RunID: failing.ID, RequestKey: "retry-human"})
retried, err := r3.retry(protocol.Message{RunID: failing.ID, RequestKey: "retry-human", RetryMode: "restart"})
if err != nil {
t.Fatal(err)
}
+52
View File
@@ -13,6 +13,11 @@ const LargeChatFixtureSessionID = "ses_large_chat_fixture"
// LargeChatFixturePassword authenticates the disposable fixture endpoint. It is
// fixed fixture data, not an operator credential.
const LargeChatFixturePassword = "large-chat-fixture"
const LargeChatFixtureShellID = "sh_large_chat_fixture"
const LargeChatFixtureShellOutput = "fixture shell output marker\n"
const LargeChatFixtureForeignSessionID = "ses_foreign_fixture"
const LargeChatFixtureForeignShellID = "sh_foreign_fixture"
const LargeChatFixtureForeignShellOutput = "FOREIGN SHELL MARKER MUST NOT LEAK\n"
const (
LargeChatFixtureNewestAnswer = "Controlled newest answer: large history fixture"
@@ -24,6 +29,13 @@ const (
// It deliberately serves fixed, paged data: no model is invoked and no production
// history is read or written.
func LargeChatFixtureHandler(workspace string) http.Handler {
return LargeChatFixtureHandlerWithShell(workspace, false)
}
// LargeChatFixtureHandlerWithShell is the foreground fixture surface used by
// the readback check. exited selects the provider's observed terminal state;
// the output remains fixed and bounded in either state.
func LargeChatFixtureHandlerWithShell(workspace string, exited bool) http.Handler {
mux := http.NewServeMux()
mux.HandleFunc("GET /api/info", func(w http.ResponseWriter, r *http.Request) {
if !largeChatFixtureAuthenticated(w, r) {
@@ -63,6 +75,46 @@ func LargeChatFixtureHandler(workspace string) http.Handler {
}
writeLargeChatFixturePage(w, page)
})
status := "running"
if exited {
status = "exited"
}
mux.HandleFunc("GET /api/shell", func(w http.ResponseWriter, r *http.Request) {
if !largeChatFixtureAuthenticated(w, r) {
return
}
writeLargeChatFixtureJSON(w, fmt.Sprintf(`{"data":[{"id":%q,"status":%q,"command":"printf fixture","metadata":{"sessionID":%q}},{"id":%q,"status":"running","command":"printf foreign","metadata":{"sessionID":%q}}]}`, LargeChatFixtureShellID, status, LargeChatFixtureSessionID, LargeChatFixtureForeignShellID, LargeChatFixtureForeignSessionID))
})
mux.HandleFunc("GET /api/shell/"+LargeChatFixtureShellID, func(w http.ResponseWriter, r *http.Request) {
if !largeChatFixtureAuthenticated(w, r) {
return
}
exit := "null"
if exited {
exit = "0"
}
writeLargeChatFixtureJSON(w, fmt.Sprintf(`{"data":{"id":%q,"status":%q,"command":"printf fixture","exit":%s,"metadata":{"sessionID":%q}}}`, LargeChatFixtureShellID, status, exit, LargeChatFixtureSessionID))
})
mux.HandleFunc("GET /api/shell/"+LargeChatFixtureForeignShellID, func(w http.ResponseWriter, r *http.Request) {
if !largeChatFixtureAuthenticated(w, r) {
return
}
writeLargeChatFixtureJSON(w, fmt.Sprintf(`{"data":{"id":%q,"status":"running","command":"printf foreign","metadata":{"sessionID":%q}}}`, LargeChatFixtureForeignShellID, LargeChatFixtureForeignSessionID))
})
mux.HandleFunc("GET /api/shell/"+LargeChatFixtureShellID+"/output", func(w http.ResponseWriter, r *http.Request) {
if !largeChatFixtureAuthenticated(w, r) {
return
}
cursor := int64(len(LargeChatFixtureShellOutput))
writeLargeChatFixtureJSON(w, fmt.Sprintf(`{"data":{"output":%q,"cursor":%d,"size":%d,"truncated":false}}`, LargeChatFixtureShellOutput, cursor, cursor))
})
mux.HandleFunc("GET /api/shell/"+LargeChatFixtureForeignShellID+"/output", func(w http.ResponseWriter, r *http.Request) {
if !largeChatFixtureAuthenticated(w, r) {
return
}
cursor := int64(len(LargeChatFixtureForeignShellOutput))
writeLargeChatFixtureJSON(w, fmt.Sprintf(`{"data":{"output":%q,"cursor":%d,"size":%d,"truncated":false}}`, LargeChatFixtureForeignShellOutput, cursor, cursor))
})
return mux
}
+255
View File
@@ -6,6 +6,7 @@ import (
"crypto/sha256"
"encoding/json"
"fmt"
"net"
"net/http/httptest"
"os"
"path/filepath"
@@ -15,11 +16,30 @@ import (
"time"
"git.a-wan.net/alan/pipelines/internal/cli"
"git.a-wan.net/alan/pipelines/internal/client"
"git.a-wan.net/alan/pipelines/internal/pipeline"
"git.a-wan.net/alan/pipelines/internal/protocol"
"git.a-wan.net/alan/pipelines/internal/stage"
"github.com/coder/websocket"
"github.com/coder/websocket/wsjson"
)
type fixedRemoteListener struct{ net.Listener }
func (l fixedRemoteListener) Accept() (net.Conn, error) {
conn, err := l.Listener.Accept()
if err != nil {
return nil, err
}
return fixedRemoteConn{Conn: conn}, nil
}
type fixedRemoteConn struct{ net.Conn }
func (fixedRemoteConn) RemoteAddr() net.Addr {
return &net.TCPAddr{IP: net.ParseIP("203.0.113.9"), Port: 443}
}
// This is a real CLI/WebSocket/scratch-SQLite exercise with an explicitly FAKE
// executor. It never calls inference, a phone, or a deployed server.
func TestResourceScratchCLIWaitingThenProgress(t *testing.T) {
@@ -110,6 +130,241 @@ func TestResourceScratchCLIWaitingThenProgress(t *testing.T) {
t.Log("Observed queued resource wait followed by automatic progress; FAKE executor only, no phone/inference/deployment.")
}
func TestResourceOperatorTransportCLIRecoveryBoundary(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
store, err := pipeline.Open(filepath.Join(t.TempDir(), "operator.db"))
if err != nil {
t.Fatal(err)
}
defer store.Close()
source := pipeline.Template([]string{"work"}, true) + "```resources\n{\"exclusive\":\"scratch-resource\"}\n```\n"
if _, err := store.Submit(ctx, "operator-resource", source); err != nil {
t.Fatal(err)
}
items, err := store.List(ctx)
if err != nil {
t.Fatal(err)
}
detail, err := store.Get(ctx, items[0].ID, 1)
if err != nil {
t.Fatal(err)
}
runID, _, err := store.CreateRun(ctx, detail, "operator-recovery", t.TempDir(), false, nil, nil)
if err != nil {
t.Fatal(err)
}
visit := pipeline.ResourceVisit{Resource: "scratch-resource", RunID: runID, Stage: "work", Sequence: 1}
if _, ch, _, err := store.RequestResource(ctx, visit); err != nil || ch != nil {
t.Fatalf("acquire resource: channel=%v err=%v", ch, err)
}
if err := store.RecoverRuns(ctx); err != nil {
t.Fatal(err)
}
handler := Handler(ctx, store)
srv := httptest.NewServer(handler)
defer srv.Close()
normal, err := client.Open(ctx, "ws"+strings.TrimPrefix(srv.URL, "http")+"/connect")
if err != nil {
t.Fatal(err)
}
if _, err := normal.Call(ctx, protocol.Message{Type: protocol.InspectResource, Resource: visit.Resource}); err == nil || !strings.Contains(err.Error(), "operator transport") {
t.Fatalf("ordinary connect accepted recovery: %v", err)
}
normal.Close()
endpoint := "ws" + strings.TrimPrefix(srv.URL, "http") + "/connect"
runCLI := func(args ...string) string {
var out bytes.Buffer
app := cli.App{Out: &out, Err: &out, In: strings.NewReader("")}
if err := app.Run(ctx, append([]string{"--url", endpoint}, args...)); err != nil {
t.Fatalf("CLI %v: %v\n%s", args, err, out.String())
}
return out.String()
}
inspection := runCLI("resource-inspect", visit.Resource, "--json")
if strings.Contains(inspection, "token") || strings.Contains(inspection, "credential") {
t.Fatalf("inspection leaked capability: %s", inspection)
}
runCLI("resource-reconcile", visit.Resource, fmt.Sprint(runID), visit.Stage, fmt.Sprint(visit.Sequence), "--evidence", "verified no mutating action", "--yes", "--json")
inspection = runCLI("resource-inspect", visit.Resource, "--json")
var recovered protocol.ResourceInspection
if err := json.Unmarshal([]byte(inspection), &recovered); err != nil {
t.Fatal(err)
}
if recovered.Owner != nil || len(recovered.Audits) != 1 || recovered.Audits[0].RunID != runID || recovered.Audits[0].Result != "reconciled" {
t.Fatalf("CLI recovery state was not persisted: %s", inspection)
}
}
func TestResourceOperatorCLIRejectsStaleAndNonQuarantinedRecovery(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
store, err := pipeline.Open(filepath.Join(t.TempDir(), "operator-rejections.db"))
if err != nil {
t.Fatal(err)
}
defer store.Close()
if _, err := store.Submit(ctx, "operator-rejections", pipeline.Template([]string{"work"}, true)+"```resources\n{\"exclusive\":\"scratch-resource\"}\n```\n"); err != nil {
t.Fatal(err)
}
items, err := store.List(ctx)
if err != nil {
t.Fatal(err)
}
detail, err := store.Get(ctx, items[0].ID, 1)
if err != nil {
t.Fatal(err)
}
runID, _, err := store.CreateRun(ctx, detail, "operator-rejections", t.TempDir(), false, nil, nil)
if err != nil {
t.Fatal(err)
}
visit := pipeline.ResourceVisit{Resource: "scratch-resource", RunID: runID, Stage: "work", Sequence: 1}
lease, ch, _, err := store.RequestResource(ctx, visit)
if err != nil || ch != nil {
t.Fatalf("acquire resource: channel=%v err=%v", ch, err)
}
if err := store.RecoverRuns(ctx); err != nil {
t.Fatal(err)
}
handler := Handler(ctx, store)
srv := httptest.NewServer(handler)
defer srv.Close()
endpoint := "ws" + strings.TrimPrefix(srv.URL, "http") + "/connect"
runCLI := func(args ...string) error {
var out bytes.Buffer
app := cli.App{Out: &out, Err: &out, In: strings.NewReader("")}
err := app.Run(ctx, append([]string{"--url", endpoint}, args...))
if strings.Contains(out.String(), "token") || strings.Contains(out.String(), "credential") {
t.Fatalf("private capability appeared in CLI output: %s", out.String())
}
return err
}
if err := runCLI("resource-reconcile", visit.Resource, fmt.Sprint(runID+1), visit.Stage, fmt.Sprint(visit.Sequence), "--evidence", "verified", "--yes", "--json"); err == nil {
t.Fatal("accepted wrong owner")
}
if err := runCLI("resource-reconcile", visit.Resource, fmt.Sprint(runID), "stale-stage", fmt.Sprint(visit.Sequence), "--evidence", "verified", "--yes", "--json"); err == nil {
t.Fatal("accepted stale stage")
}
if err := runCLI("resource-reconcile", visit.Resource, fmt.Sprint(runID), visit.Stage, "2", "--evidence", "verified", "--yes", "--json"); err == nil {
t.Fatal("accepted stale sequence")
}
if err := store.SetResourceCleanup(ctx, lease, true); err != nil {
t.Fatal(err)
}
if err := runCLI("resource-reconcile", visit.Resource, fmt.Sprint(runID), visit.Stage, fmt.Sprint(visit.Sequence), "--evidence", "verified", "--yes", "--json"); err == nil {
t.Fatal("accepted non-quarantined owner")
}
inspection, err := store.InspectResource(ctx, visit.Resource)
if err != nil {
t.Fatal(err)
}
if inspection.Owner == nil || !inspection.Owner.CleanupConfirmed || len(inspection.Audits) != 0 {
t.Fatalf("rejected recovery changed state: owner=%#v audits=%d", inspection.Owner, len(inspection.Audits))
}
}
func TestResourceOperatorTransportDoesNotTrustRemoteOrForwardedHeaders(t *testing.T) {
for _, remote := range []string{"203.0.113.9:443", "198.51.100.7:8080", "not-an-address"} {
if isLoopbackPeer(remote) {
t.Fatalf("accepted non-loopback peer %q", remote)
}
}
for _, remote := range []string{"127.0.0.1:443", "[::1]:443"} {
if !isLoopbackPeer(remote) {
t.Fatalf("rejected loopback peer %q", remote)
}
}
// X-Forwarded-For is deliberately not consulted: a proxy-supplied header
// cannot turn a remote WebSocket peer into the local operator transport.
}
func TestResourceOperatorTransportRejectsForgedForwardedPeer(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
store, err := pipeline.Open(filepath.Join(t.TempDir(), "operator-transport.db"))
if err != nil {
t.Fatal(err)
}
defer store.Close()
if _, err := store.Submit(ctx, "operator-transport", pipeline.Template([]string{"work"}, true)+"```resources\n{\"exclusive\":\"scratch-resource\"}\n```\n"); err != nil {
t.Fatal(err)
}
items, err := store.List(ctx)
if err != nil {
t.Fatal(err)
}
detail, err := store.Get(ctx, items[0].ID, 1)
if err != nil {
t.Fatal(err)
}
runID, _, err := store.CreateRun(ctx, detail, "operator-transport", t.TempDir(), false, nil, nil)
if err != nil {
t.Fatal(err)
}
visit := pipeline.ResourceVisit{Resource: "scratch-resource", RunID: runID, Stage: "work", Sequence: 1}
if _, ch, _, err := store.RequestResource(ctx, visit); err != nil || ch != nil {
t.Fatalf("acquire resource: channel=%v err=%v", ch, err)
}
if err := store.RecoverRuns(ctx); err != nil {
t.Fatal(err)
}
before, err := store.InspectResource(ctx, visit.Resource)
if err != nil {
t.Fatal(err)
}
server := httptest.NewUnstartedServer(Handler(ctx, store))
server.Listener = fixedRemoteListener{Listener: server.Listener}
server.Start()
defer server.Close()
conn, _, err := websocket.Dial(ctx, "ws"+strings.TrimPrefix(server.URL, "http")+"/operator-connect", &websocket.DialOptions{HTTPHeader: map[string][]string{"X-Forwarded-For": {"127.0.0.1"}}})
if err != nil {
t.Fatal(err)
}
defer conn.CloseNow()
var welcome protocol.Message
if err := wsjson.Read(ctx, conn, &welcome); err != nil {
t.Fatal(err)
}
if err := wsjson.Write(ctx, conn, protocol.Message{Type: protocol.InspectResource, Resource: visit.Resource, ID: "forged"}); err != nil {
t.Fatal(err)
}
var refused protocol.Message
if err := wsjson.Read(ctx, conn, &refused); err != nil {
t.Fatal(err)
}
if refused.Type != protocol.Error || !strings.Contains(refused.Error, "operator transport") {
t.Fatalf("forged peer was accepted: %#v", refused)
}
after, err := store.InspectResource(ctx, visit.Resource)
if err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(before, after) {
t.Fatalf("forged request mutated recovery state: before=%#v after=%#v", before, after)
}
trusted := httptest.NewServer(Handler(ctx, store))
defer trusted.Close()
normal, err := client.Open(ctx, "ws"+strings.TrimPrefix(trusted.URL, "http")+"/connect")
if err != nil {
t.Fatal(err)
}
if _, err := normal.Call(ctx, protocol.Message{Type: protocol.InspectResource, Resource: visit.Resource}); err == nil || !strings.Contains(err.Error(), "operator transport") {
t.Fatalf("ordinary connect accepted recovery: %v", err)
}
normal.Close()
operator, err := client.Open(ctx, "ws"+strings.TrimPrefix(trusted.URL, "http")+"/operator-connect")
if err != nil {
t.Fatal(err)
}
defer operator.Close()
if _, err := operator.Call(ctx, protocol.Message{Type: protocol.InspectResource, Resource: visit.Resource}); err != nil {
t.Fatalf("trusted operator transport refused inspection: %v", err)
}
}
func TestResourceAndroidDraftPublishedV3Provenance(t *testing.T) {
baseline, err := os.ReadFile("testdata/android-v3-published.md")
if err != nil {
+2 -2
View File
@@ -43,14 +43,14 @@ func TestRetryWaitsForExecutionCleanupEvenAfterTerminalStatus(t *testing.T) {
if err != nil || !runs[0].Executing {
t.Fatal("list advertised stopped run during cleanup", runs, err)
}
if _, err := r.retry(protocol.Message{RunID: id, RequestKey: "retry"}); err == nil || !strings.Contains(err.Error(), "still executing") {
if _, err := r.retry(protocol.Message{RunID: id, RequestKey: "retry", RetryMode: "restart"}); err == nil || !strings.Contains(err.Error(), "still executing") {
t.Fatal("retry accepted during execution cleanup", err)
}
delete(r.active, id)
r.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) {
return stage.Result{Outcome: "success", Output: stage.Values{"summary": "done"}}, nil
}
if _, err := r.retry(protocol.Message{RunID: id, RequestKey: "retry"}); err != nil {
if _, err := r.retry(protocol.Message{RunID: id, RequestKey: "retry", RetryMode: "restart"}); err != nil {
t.Fatal("retry rejected after cleanup", err)
}
r.wg.Wait()
+91 -2
View File
@@ -183,7 +183,7 @@ func TestRetryReplaysInitialInputAndPinnedVersionThroughCLI(t *testing.T) {
if err != nil || len(beforeRetry) != 1 {
t.Fatal("unconfirmed retry created a run", err)
}
result, err := runCLI("retry", fmt.Sprint(original.ID), "--yes", "--json", "--wait", "--timeout", "5s", "--request-key", "retry-key")
result, err := runCLI("retry", fmt.Sprint(original.ID), "--yes", "--json", "--wait", "--timeout", "5s", "--request-key", "retry-key", "--mode", "restart")
if err != nil {
t.Fatal(err)
}
@@ -236,7 +236,7 @@ func TestRetryRejectsUnavailableOriginalDirectoryAndNonfailedRuns(t *testing.T)
t.Fatal(err)
}
}
_, err = r.retry(protocol.Message{RunID: id, RequestKey: "retry-" + status})
_, err = r.retry(protocol.Message{RunID: id, RequestKey: "retry-" + status, RetryMode: "restart"})
if err == nil {
t.Fatal("invalid retry accepted", status)
}
@@ -249,6 +249,95 @@ func TestRetryRejectsUnavailableOriginalDirectoryAndNonfailedRuns(t *testing.T)
}
}
func TestRetryResumeReusesPrefixHandoffAndReplaysRequestAfterReopen(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
dir := t.TempDir()
store, err := pipeline.Open(filepath.Join(dir, "resume.db"))
if err != nil {
t.Fatal(err)
}
source := "# stage a\n\n```output\nA\n```\n\n```prompt\nA\n```\n\n```edges\nsuccess -> b\nerror -> #failure\n```\n\n```configs\ntype=model\nprovider=test\nmodel=test\n```\n\n# stage b\n\n```output\nB\n```\n\n```prompt\nB\n```\n\n```edges\nsuccess -> c\nerror -> #failure\n```\n\n```configs\ntype=model\nprovider=test\nmodel=test\n```\n\n# stage c\n\n```output\nC\n```\n\n```prompt\nC\n```\n\n```edges\nsuccess -> #success\nerror -> #failure\n```\n\n```configs\ntype=model\nprovider=test\nmodel=test\n```\n"
if _, err := store.Submit(ctx, "resume", source); err != nil {
t.Fatal(err)
}
items, _ := store.List(ctx)
workspace := t.TempDir()
var aCalls, bCalls, cCalls int
r, err := newRunner(ctx, store)
if err != nil {
t.Fatal(err)
}
r.execute = func(_ context.Context, def stage.Definition, input stage.Values) (stage.Result, error) {
switch def.Key {
case "a":
aCalls++
if input["prompt"] != "task" {
t.Errorf("entry received altered input: %#v", input)
}
return stage.Result{Outcome: "success", Output: stage.Values{"handoff": "from-a"}}, nil
case "b":
bCalls++
if input["handoff"] != "from-a" {
t.Errorf("resume lost exact prefix handoff: %#v", input)
}
if bCalls == 1 {
return stage.Result{}, fmt.Errorf("deliberate failure")
}
return stage.Result{Outcome: "success", Output: stage.Values{"handoff": "from-b"}}, nil
case "c":
cCalls++
if input["handoff"] != "from-b" {
t.Errorf("C received wrong handoff: %#v", input)
}
return stage.Result{Outcome: "success", Output: stage.Values{"summary": "done"}}, nil
default:
t.Fatalf("unexpected stage %q", def.Key)
return stage.Result{}, nil
}
}
original, err := r.start(protocol.Message{PipelineID: items[0].ID, Version: 1, RequestKey: "original", Workspace: workspace, Input: map[string]any{"prompt": "task"}})
if err != nil {
t.Fatal(err)
}
r.wg.Wait()
original, err = store.GetRun(ctx, original.ID)
if err != nil || original.Status != "failed" || len(original.Steps) != 2 {
t.Fatalf("initial run did not stop at B: %+v %v", original, err)
}
retried, err := r.retry(protocol.Message{RunID: original.ID, RequestKey: "resume-key"})
if err != nil {
t.Fatal(err)
}
r.wg.Wait()
retried, err = store.GetRun(ctx, retried.ID)
if err != nil || retried.Status != "succeeded" || retried.RetryMode != "resume" || retried.RetrySource != original.ID {
t.Fatalf("resume did not complete with provenance: %+v %v", retried, err)
}
if aCalls != 1 || bCalls != 2 || cCalls != 1 {
t.Fatalf("retry duplicated prefix or suffix: A=%d B=%d C=%d", aCalls, bCalls, cCalls)
}
if len(retried.Steps) != 2 || retried.Steps[0].Stage != "b" || retried.Steps[1].Stage != "c" {
t.Fatalf("retry persisted the wrong steps: %+v", retried.Steps)
}
if err := store.Close(); err != nil {
t.Fatal(err)
}
store, err = pipeline.Open(filepath.Join(dir, "resume.db"))
if err != nil {
t.Fatal(err)
}
defer store.Close()
r2, err := newRunner(ctx, store)
if err != nil {
t.Fatal(err)
}
replayed, err := r2.retry(protocol.Message{RunID: original.ID, RequestKey: "resume-key"})
if err != nil || replayed.ID != retried.ID {
t.Fatalf("persisted request-key replay lost its run: %+v %v", replayed, err)
}
}
// TestReadOnlyStageRetriesPostAdmissionTransportButMutatingStageDoesNot proves,
// through the real runner and CLI fixture, that a read-only stage retries a
// post-admission transport (final-readback) failure with a fresh session while
+145 -1
View File
@@ -285,6 +285,15 @@ func (r *runner) start(request protocol.Message, retryOf ...int64) (*protocol.Ru
r.rememberRunIdentity(id, identityOfRunSummary(detail.RunSummary))
}
if created {
if request.RetrySource > 0 && request.RetryMode != "restart" {
checkpoint, checkpointErr := r.retrySourceCheckpoint(request.RetrySource, d)
if checkpointErr != nil {
return nil, checkpointErr
}
if err := r.store.SaveRetryCheckpoint(r.ctx, id, request.RetrySource, "resume", checkpoint.failed.Sequence, checkpoint.detail, checkpoint.handoff); err != nil {
return nil, err
}
}
// Emit the created delta before launching: the queued frame reaches
// subscribers before the running/step-started transition, so a client
// sees the run appear and then advance.
@@ -404,6 +413,65 @@ func (r *runner) listRuns() ([]protocol.RunSummary, error) {
return items, err
}
type retryCheckpoint struct {
detail *protocol.RunDetail
failed protocol.RunStep
handoff map[string]any
}
func (r *runner) retrySourceCheckpoint(sourceID int64, pinned *protocol.PipelineDetail) (retryCheckpoint, error) {
source, err := r.store.GetRun(r.ctx, sourceID)
if err != nil {
return retryCheckpoint{}, err
}
if source.Status != "failed" || source.FinishedAt == "" {
return retryCheckpoint{}, fmt.Errorf("retry source %d is not a terminal failed run", sourceID)
}
if source.PipelineID != pinned.ID || source.Version != pinned.Version || source.ContentHash != pinned.ContentHash {
return retryCheckpoint{}, fmt.Errorf("retry source %d is pinned to a different pipeline version", sourceID)
}
if len(source.Steps) == 0 {
return retryCheckpoint{}, fmt.Errorf("retry source %d has no failed stage checkpoint", sourceID)
}
failed := source.Steps[len(source.Steps)-1]
if (failed.Status != "completed" && failed.Status != "failed") || failed.Target != protocol.FailureEnd {
return retryCheckpoint{}, fmt.Errorf("retry source %d has no safe failed-stage checkpoint", sourceID)
}
var handoff map[string]any
if failed.Input == nil {
return retryCheckpoint{}, fmt.Errorf("retry source %d is missing failed-stage handoff context", sourceID)
}
handoff = map[string]any{"input": failed.Input, "sequence": failed.Sequence, "stage": failed.Stage}
prefix := map[string]any{}
for _, step := range source.Steps {
if step.Sequence >= failed.Sequence || step.Status != "completed" {
continue
}
prefix[step.Stage] = step.Output
}
handoff["prefix"] = prefix
for _, declared := range pinned.Graph.Stages {
if declared.Key != failed.Stage {
continue
}
typ := stage.EffectiveType(declared.Type)
if typ == stage.HumanType || typ == stage.NtfyType {
return retryCheckpoint{}, fmt.Errorf("retry source %d cannot replay completed external stage %s", sourceID, failed.Stage)
}
if typ == stage.ModelType {
tools, explicit, toolErr := stage.ToolNames(pipeline.StageDefinition(declared))
if toolErr != nil {
return retryCheckpoint{}, toolErr
}
if explicit && len(tools) > 0 {
return retryCheckpoint{}, fmt.Errorf("retry source %d cannot safely replay admitted native stage %s", sourceID, failed.Stage)
}
}
return retryCheckpoint{detail: source, failed: failed, handoff: handoff}, nil
}
return retryCheckpoint{}, fmt.Errorf("retry source %d failed stage is not in the pinned graph", sourceID)
}
func (r *runner) retry(request protocol.Message) (*protocol.RunDetail, error) {
if request.RunID <= 0 {
return nil, fmt.Errorf("retry requires a positive run ID")
@@ -418,7 +486,26 @@ func (r *runner) retry(request protocol.Message) (*protocol.RunDetail, error) {
if original.Status != "failed" || original.FinishedAt == "" {
return nil, fmt.Errorf("only failed runs can be retried (run %d is %s)", original.ID, original.Status)
}
return r.start(protocol.Message{PipelineID: original.PipelineID, Version: original.Version, RequestKey: request.RequestKey, Workspace: original.Workspace, Input: original.Input, AllowTools: original.AllowTools, WritePaths: original.WritePaths, Model: original.Model, StageModels: original.StageModels}, original.ID)
if request.RequestKey == "" {
return nil, fmt.Errorf("retry requires an idempotency request key")
}
mode := request.RetryMode
if mode == "" {
mode = "resume"
}
if mode != "resume" && mode != "restart" {
return nil, fmt.Errorf("retry mode must be resume or restart")
}
d, err := r.store.Get(r.ctx, original.PipelineID, original.Version)
if err != nil {
return nil, err
}
if mode == "resume" {
if _, err := r.retrySourceCheckpoint(original.ID, d); err != nil {
return nil, err
}
}
return r.start(protocol.Message{PipelineID: original.PipelineID, Version: original.Version, RequestKey: request.RequestKey, RetryMode: mode, RetrySource: original.ID, Workspace: original.Workspace, Input: original.Input, AllowTools: original.AllowTools, WritePaths: original.WritePaths, Model: original.Model, StageModels: original.StageModels}, original.ID)
}
func (r *runner) cancel(id int64) error {
@@ -520,6 +607,35 @@ func (r *runner) decide(request protocol.Message) (*protocol.RunDetail, error) {
}
func (r *runner) run(ctx context.Context, id int64, d *protocol.PipelineDetail, workspace string, request protocol.Message) {
if request.RetrySource > 0 && request.RetryMode != "restart" {
source, err := r.store.GetRun(ctx, request.RetrySource)
if err != nil {
r.finishRunState(id, d.Name, workspace, "failed", "error", err.Error(), 0, 0, "", nil)
return
}
checkpoint, err := r.retrySourceCheckpoint(request.RetrySource, d)
if err != nil {
r.finishRunState(id, d.Name, workspace, "failed", "error", err.Error(), 0, 0, "", nil)
return
}
storedSource, storedMode, storedSequence, storedHandoff, checkpointErr := r.store.RetryCheckpoint(ctx, id)
if checkpointErr != nil || storedSource != request.RetrySource || storedMode != "resume" || storedSequence != checkpoint.failed.Sequence || storedHandoff == nil {
if checkpointErr == nil {
checkpointErr = fmt.Errorf("retry checkpoint is incomplete")
}
r.finishRunState(id, d.Name, workspace, "failed", "error", checkpointErr.Error(), 0, 0, "", nil)
return
}
if input, ok := storedHandoff["input"].(map[string]any); ok {
checkpoint.failed.Input = input
} else {
r.finishRunState(id, d.Name, workspace, "failed", "error", "retry checkpoint is missing handoff context", 0, 0, "", nil)
return
}
state := rebuildRetryState(source, checkpoint.failed)
r.runLoop(ctx, id, d.Graph, d.Name, workspace, request, state)
return
}
original, copyErr := copyValues(stage.Values(request.Input))
if copyErr != nil {
r.finishRunState(id, d.Name, workspace, "failed", "error", copyErr.Error(), 0, 0, "", stage.Values(request.Input))
@@ -629,6 +745,34 @@ func rebuildRunState(detail *protocol.RunDetail, decided protocol.RunStep) *runS
return state
}
// rebuildRetryState restores only the completed prefix. The failed visit is
// deliberately not counted and its step/evidence is not copied into the new
// attempt, so retries cannot duplicate execution or usage aggregates.
func rebuildRetryState(detail *protocol.RunDetail, failed protocol.RunStep) *runState {
state := newRunState(failed.Stage, stage.Values(failed.Input))
state.nextSequence = failed.Sequence
if original, err := copyValues(stage.Values(detail.Input)); err == nil {
state.memory["input"] = memoryValue{output: original}
}
for _, step := range detail.Steps {
if step.Sequence >= failed.Sequence {
break
}
state.visits[step.Stage]++
if step.Status == "completed" {
if values, err := copyValues(stage.Values(step.Output)); err == nil {
state.memory[step.Stage] = memoryValue{output: values, sequence: step.Sequence}
}
state.completed++
}
state.recordChanges(step.FileChanges, step.Artifacts)
if step.Status == "completed" {
state.artifactDirs = append(state.artifactDirs, filepath.Join("", strconv.Itoa(step.Sequence)))
}
}
return state
}
// artifactAccess derives exact visit directories; it never consumes a path from
// a definition or model result. An unreadable or symlinked root fails closed.
func (r *runner) artifactAccess(runID int64, sequence int, prior []string, current string) stage.ArtifactAccess {
+42 -3
View File
@@ -2,7 +2,9 @@ package server
import (
"context"
"fmt"
"log"
"net"
"net/http"
"strings"
"sync"
@@ -119,7 +121,7 @@ func HandlerWithOptions(ctx context.Context, options Options, stores ...*pipelin
}
_, _ = w.Write([]byte(`{"healthy":true}`))
})
mux.HandleFunc("GET /connect", func(w http.ResponseWriter, r *http.Request) {
connect := func(w http.ResponseWriter, r *http.Request) {
conn, err := websocket.Accept(w, r, nil)
if err != nil {
return
@@ -171,7 +173,7 @@ func HandlerWithOptions(ctx context.Context, options Options, stores ...*pipelin
}
for {
var request protocol.Message
if err := wsjson.Read(ctx, conn, &request); err != nil {
if err := wsjson.Read(connCtx, conn, &request); err != nil {
return
}
log.Printf("command received: client=%s type=%q id=%q pipeline=%d version=%d", r.RemoteAddr, request.Type, request.ID, request.PipelineID, request.Version)
@@ -268,6 +270,32 @@ func HandlerWithOptions(ctx context.Context, options Options, stores ...*pipelin
response.Error = err.Error()
}
}
} else if (request.Type == protocol.InspectResource || request.Type == protocol.ReconcileResource) && len(stores) > 0 {
var err error
response.Type = protocol.ResourceInspectionResult
// Recovery has a separate transport path. A stage/model client may
// also be local, so loopback alone is not an authority boundary.
// The CLI uses /operator-connect; the ordinary /connect path is
// deliberately refused even from loopback.
if r.URL.Path != "/operator-connect" || !isLoopbackPeer(r.RemoteAddr) {
response.Type = protocol.Error
response.Error = "resource recovery requires the local operator transport"
} else if request.Type == protocol.InspectResource {
var inspection protocol.ResourceInspection
inspection, err = stores[0].InspectResource(ctx, request.Resource)
response.ResourceInspection = &inspection
} else {
response.Type = protocol.ResourceReconciled
if request.ResourceRecovery == nil {
err = fmt.Errorf("resource recovery evidence is required")
} else {
err = stores[0].ReconcileResource(ctx, *request.ResourceRecovery)
}
}
if err != nil {
response.Type = protocol.Error
response.Error = err.Error()
}
} else if runs != nil && (request.Type == protocol.RunPipeline || request.Type == protocol.RetryRun || request.Type == protocol.ListRuns || request.Type == protocol.GetRun || request.Type == protocol.CancelRun || request.Type == protocol.DecideRun || request.Type == protocol.SetRunTitle || request.Type == protocol.GetRunEvidence) {
var err error
switch request.Type {
@@ -323,10 +351,21 @@ func HandlerWithOptions(ctx context.Context, options Options, stores ...*pipelin
}
log.Printf("response sent: client=%s type=%q id=%q time=%q error=%q version=%d", r.RemoteAddr, response.Type, response.ID, response.Time, response.Error, selectedVersion)
}
})
}
mux.HandleFunc("GET /connect", connect)
mux.HandleFunc("GET /operator-connect", connect)
return &runHandler{Handler: mux, runner: runs}
}
func isLoopbackPeer(remoteAddr string) bool {
host, _, err := net.SplitHostPort(remoteAddr)
if err != nil {
return false
}
ip := net.ParseIP(host)
return ip != nil && ip.IsLoopback()
}
type runHandler struct {
http.Handler
runner *runner
+6
View File
@@ -32,6 +32,12 @@ func TestRetryShortcutAndPaletteSendServerRetryWithoutReplacingDirectory(t *test
if err != nil {
t.Fatal(err)
}
if err := store.BeginStep(ctx, id, 1, "one", map[string]any{"prompt": "original task"}); err != nil {
t.Fatal(err)
}
if err := store.FinishStep(ctx, id, protocol.RunStep{Sequence: 1, Stage: "one", Status: "failed", Outcome: "error", Target: protocol.FailureEnd, Input: map[string]any{"prompt": "original task"}}); err != nil {
t.Fatal(err)
}
if err := store.FinishRun(ctx, id, "failed", "error", "original failure", nil); err != nil {
t.Fatal(err)
}