From 9130ae8c7a62db011fc399233ce37c8d87035cfd Mon Sep 17 00:00:00 2001 From: Alan Silva Date: Sun, 18 Oct 2026 12:30:30 +0100 Subject: [PATCH] Add stage-exclusive resource waiting Add durable FIFO capacity-one resource leases for executable stages, including cleanup-confirmed release, conservative restart quarantine, queue state events, and CLI/TUI/Android waiting presentation. Add scheduler, cleanup, protocol, client, and Android coverage plus a proposed Android v3 resource draft. Checked packaged templates and authoring skill references: neither needs updating in this item. The per-item docs note records the shared documentation that docs-fold must update; no shared README or packaged reference was edited. --- android/AGENTS.md | 30 +- .../awan/pipelines/data/protocol/Message.kt | 14 + .../java/net/awan/pipelines/ui/RunEvents.kt | 19 +- .../pipelines/ui/screens/RunHistoryScreen.kt | 1 + .../pipelines/ui/screens/RunSummaryTab.kt | 1 + .../net/awan/pipelines/ui/ResourceWaitTest.kt | 32 ++ docs/notes/phone-resource-wait.md | 158 +++++++++ internal/cli/app.go | 4 +- internal/pipeline/control.go | 15 +- internal/pipeline/resources.go | 277 +++++++++++++++ internal/pipeline/resources_test.go | 288 +++++++++++++++ internal/pipeline/retention.go | 29 +- internal/pipeline/runs.go | 11 +- internal/pipeline/runs_mutate.go | 4 +- internal/pipeline/runs_schema.go | 32 +- internal/pipeline/stage.go | 1 + internal/pipeline/store.go | 10 +- internal/protocol/message.go | 6 + internal/protocol/resources_test.go | 29 ++ internal/protocol/runevents.go | 7 +- internal/protocol/runs.go | 28 +- internal/server/resource_cli_test.go | 154 +++++++++ internal/server/resources.go | 60 ++++ internal/server/resources_test.go | 327 ++++++++++++++++++ internal/server/retry_test.go | 1 + internal/server/runevents.go | 29 +- internal/server/runner.go | 109 ++++-- .../server/testdata/android-resource-draft.md | 312 +++++++++++++++++ .../server/testdata/android-v2-published.md | 296 ++++++++++++++++ .../server/testdata/android-v3-published.md | 296 ++++++++++++++++ internal/stage/agent.go | 25 +- internal/stage/agent_test.go | 4 +- internal/stage/budget_test.go | 6 + internal/stage/opencode.go | 2 +- internal/stage/pi.go | 15 +- internal/stage/resource_cleanup_test.go | 44 +++ internal/tui/resources_test.go | 43 +++ internal/tui/results.go | 5 +- internal/tui/run_detail.go | 12 +- 39 files changed, 2657 insertions(+), 79 deletions(-) create mode 100644 android/app/src/test/java/net/awan/pipelines/ui/ResourceWaitTest.kt create mode 100644 docs/notes/phone-resource-wait.md create mode 100644 internal/pipeline/resources.go create mode 100644 internal/pipeline/resources_test.go create mode 100644 internal/protocol/resources_test.go create mode 100644 internal/server/resource_cli_test.go create mode 100644 internal/server/resources.go create mode 100644 internal/server/resources_test.go create mode 100644 internal/server/testdata/android-resource-draft.md create mode 100644 internal/server/testdata/android-v2-published.md create mode 100644 internal/server/testdata/android-v3-published.md create mode 100644 internal/stage/resource_cleanup_test.go create mode 100644 internal/tui/resources_test.go diff --git a/android/AGENTS.md b/android/AGENTS.md index f6dddbd2..2e035722 100644 --- a/android/AGENTS.md +++ b/android/AGENTS.md @@ -20,16 +20,37 @@ time while iterating — the full suite is for the end. ## The dev device +Only the **UI reviewer**, while its stage holds a declared runner-exclusive resource +for the selected phone, may use the device. Implementers and code reviewers do code, +builds, unit tests and source review only: **no adb, installs, captures, device-setting +changes or main-build restoration**. Never acquire a phone using sleep/check loops. +Do not use the phone until compatible resource scheduling is deployed and every device +user has opted in or has been reconciled by the operator. Existing pinned runs are not +retroactively protected. The resource for the device below is `phone-47241FDAS00DLK`. + The only device is the operator's **Pixel 9 Pro XL, serial `47241FDAS00DLK`**. The operator's Pixel 10 Pro Fold (`58281FDCG000JF`) is theirs: use it only where an item's brief explicitly allows it, and never make it a default. **Never start or use an emulator.** -Before any capture: `svc power stayon usb`, `settings put global stay_on_while_plugged_in 7`, +The UI reviewer records original app connection/workspace, display size and power/settings +before changing them. Before any capture: `svc power stayon usb`, `settings put global stay_on_while_plugged_in 7`, `input keyevent KEYCODE_WAKEUP`, `wm dismiss-keyguard`. Capture with `adb -s shell screencap -p /sdcard/shot.png` then `adb pull` — never `exec-out > file`. Wide layout: `wm size 2000x2244`; the fold's unfolded shape is `wm size 2076x2152`; `wm size reset` afterwards either way. +Create controlled fixtures for changed screens/states (including human-decision sheets) +on a scratch server/database and throwaway session; never depend on incidental production +state, change a real operator session, or restart a deployed service. Use the configured +authoritative endpoint, never a scratch URL as a project default. Keep all shell/ADB work +foreground and awaited: no detached/background processes may outlive the lease. +UI review includes restoring the captured original settings and main's build before its +successful return. Confirm every tool has exited. If capture, restoration or cleanup fails, +report the failure explicitly; do not claim verification or readiness. Uncertain termination +must fail closed and keep ownership quarantined, not delete a lock or expire it by age. +Return `output.resource_cleanup_confirmed=false` if restoration/readiness is uncertain; +that generic release veto retains the runner lease even if the harness reports idle. + ## Geometry is tokenised, and a test enforces it Every dimension comes from `ui/theme/Dimens.kt` (plus the layout itself) — no new hardcoded `dp`, no @@ -53,13 +74,14 @@ genuinely needs a new token goes there, named for its purpose. ## Publishing a build Copy the APK into the running run's artifacts directory as `pipelines-debug-wip-.apk` and -report the LAN URL. After any install, leave the operator's phone on **main's** build: +report the LAN URL. The UI reviewer alone, under the same lease, restores **main's** build after an install: `/private/var/folders/jp/q0qvjyg54fl_c5vhhk1ylz2m0000gn/T/opencode/deploy-app.sh main`. ## Traps - A build made in a worktree that is behind main is missing features main has — sync before building, and say which revision you built. -- Changing the app's server URL for a test (a stub) must be undone: `ws://10.0.0.75:8080/connect`, - workspace `/Users/alan/repos/pipelines`, then force-stop and relaunch. +- Restore the app's captured original connection and workspace after a fixture test, + using the configured authoritative endpoint; then force-stop and relaunch under the lease. + Never substitute a hardcoded or disposable scratch endpoint for the original configuration. - `mStayOn`/screen timeout are already set; do not "fix" the device by locking or sleeping it. diff --git a/android/app/src/main/java/net/awan/pipelines/data/protocol/Message.kt b/android/app/src/main/java/net/awan/pipelines/data/protocol/Message.kt index 0a1e2d40..b0a5c168 100644 --- a/android/app/src/main/java/net/awan/pipelines/data/protocol/Message.kt +++ b/android/app/src/main/java/net/awan/pipelines/data/protocol/Message.kt @@ -315,6 +315,7 @@ data class SessionExport( @Serializable data class RunSummary( + @SerialName("resource_wait") val resourceWait: ResourceWait? = null, val id: Long = 0, @SerialName("pipeline_id") val pipelineId: Long = 0, val name: String = "", @@ -370,6 +371,7 @@ data class RunStep( // declared directly with the exact internal/protocol/runs.go wire names. @Serializable data class RunDetail( + @SerialName("resource_wait") val resourceWait: ResourceWait? = null, val id: Long = 0, @SerialName("pipeline_id") val pipelineId: Long = 0, val name: String = "", @@ -405,6 +407,8 @@ data class RunDetail( */ @Serializable data class RunEventFrame( + @SerialName("resource_wait") val resourceWait: ResourceWait? = null, + @SerialName("resource_wait_set") val resourceWaitSet: Boolean = false, @SerialName("run_id") val runId: Long = 0, val workspace: String = "", // The run's pipeline identity and stored title, repeated on every frame so a @@ -428,6 +432,16 @@ data class RunEventFrame( val step: RunStepDelta? = null, ) +@Serializable +data class ResourceWait( + val resource: String = "", + val stage: String = "", + val sequence: Int = 0, + @SerialName("queued_since") val queuedSince: String = "", +) { + fun statusLabel(): String = "Waiting for resource $resource" +} + /** * The compact change of one step within a [RunEventFrame] * (internal/protocol/runevents.go RunStepDelta): a delta, not a RunStep, so it diff --git a/android/app/src/main/java/net/awan/pipelines/ui/RunEvents.kt b/android/app/src/main/java/net/awan/pipelines/ui/RunEvents.kt index ecf6a66c..a222ba55 100644 --- a/android/app/src/main/java/net/awan/pipelines/ui/RunEvents.kt +++ b/android/app/src/main/java/net/awan/pipelines/ui/RunEvents.kt @@ -101,6 +101,7 @@ internal fun foldRunEvent(runs: List, frame: RunEventFrame): RunList internal fun RunEventFrame.newRunSummary(): RunSummary? { if (pipelineId <= 0 || name.isBlank()) return null return RunSummary( + resourceWait = resourceWait, id = runId, pipelineId = pipelineId, name = name, @@ -124,10 +125,11 @@ internal fun RunEventFrame.newRunSummary(): RunSummary? { */ internal fun foldRunEvent(detail: RunDetail, frame: RunEventFrame): RunDetail { if (frame.runId != detail.id) return detail - if (isStaleRunFrame(detail.status, frame.status)) return detail + if (isStaleRunFrame(detail.status, frame.status, frame.resourceWaitSet)) return detail val merged = detail.fields().applying(frame) val steps = frame.step?.let { upsertStep(detail.steps, it) } ?: detail.steps val updated = detail.copy( + resourceWait = merged.resourceWait, status = merged.status, outcome = merged.outcome, currentStage = merged.currentStage, @@ -169,8 +171,9 @@ private fun advanceRunStatus(current: String, incoming: String): String = * run already settled. Dropping every field (not just the status) is what keeps * a late `running` delta from also rewriting the stage or step of a settled run. */ -private fun isStaleRunFrame(current: String, incoming: String): Boolean = - isStaleTransition(current, incoming, ::runLifecycleRank) +private fun isStaleRunFrame(current: String, incoming: String, resourceWaitSet: Boolean = false): Boolean = + if (resourceWaitSet && incoming == "queued" && !isTerminalRunStatus(current)) false + else isStaleTransition(current, incoming, ::runLifecycleRank) /** The step counterpart of [isStaleRunFrame]: a settled visit drops a stale delta whole. */ private fun isStaleStepDelta(current: String, incoming: String): Boolean = @@ -213,6 +216,7 @@ private fun advanceStatus(current: String, incoming: String, rank: (String) -> I /** The run-state fields a frame advances, shared by the summary row and the detail. */ private data class RunStateFields( + val resourceWait: net.awan.pipelines.data.protocol.ResourceWait?, val status: String, val outcome: String, val currentStage: String, @@ -226,8 +230,10 @@ private data class RunStateFields( * detail can never apply a frame differently. */ fun applying(frame: RunEventFrame): RunStateFields { - val status = advanceRunStatus(status, frame.status) + val status = if (frame.resourceWaitSet && frame.status == "queued" && !isTerminalRunStatus(status)) + "queued" else advanceRunStatus(status, frame.status) return RunStateFields( + resourceWait = if (isTerminalRunStatus(status)) null else if (frame.resourceWaitSet) frame.resourceWait else resourceWait, status = status, outcome = frame.outcome.ifBlank { outcome }, currentStage = frame.currentStage.ifBlank { currentStage }, @@ -240,6 +246,7 @@ private data class RunStateFields( } private fun RunSummary.fields() = RunStateFields( + resourceWait = resourceWait, status = status, outcome = outcome, currentStage = currentStage, @@ -250,6 +257,7 @@ private fun RunSummary.fields() = RunStateFields( ) private fun RunDetail.fields() = RunStateFields( + resourceWait = resourceWait, status = status, outcome = outcome, currentStage = currentStage, @@ -267,11 +275,12 @@ private fun executingFor(status: String, current: Boolean): Boolean = when (stat } private fun RunSummary.applying(frame: RunEventFrame): RunSummary { - if (isStaleRunFrame(status, frame.status)) return this + if (isStaleRunFrame(status, frame.status, frame.resourceWaitSet)) return this val current = fields() val merged = current.applying(frame) if (merged == current) return this return copy( + resourceWait = merged.resourceWait, status = merged.status, outcome = merged.outcome, currentStage = merged.currentStage, diff --git a/android/app/src/main/java/net/awan/pipelines/ui/screens/RunHistoryScreen.kt b/android/app/src/main/java/net/awan/pipelines/ui/screens/RunHistoryScreen.kt index 94093a1e..eacf2ace 100644 --- a/android/app/src/main/java/net/awan/pipelines/ui/screens/RunHistoryScreen.kt +++ b/android/app/src/main/java/net/awan/pipelines/ui/screens/RunHistoryScreen.kt @@ -615,6 +615,7 @@ private fun VersionChip(version: String) { /** Progress from the fields a summary actually has: completed visits + stage. */ private fun progressText(run: RunSummary): String = buildString { + run.resourceWait?.let { append(it.statusLabel()); append(" · ") } append(visitCount(run.completedSteps, "completed")) if (run.currentStage.isNotBlank()) append(" · stage ${run.currentStage}") } diff --git a/android/app/src/main/java/net/awan/pipelines/ui/screens/RunSummaryTab.kt b/android/app/src/main/java/net/awan/pipelines/ui/screens/RunSummaryTab.kt index 559a7029..418cad8d 100644 --- a/android/app/src/main/java/net/awan/pipelines/ui/screens/RunSummaryTab.kt +++ b/android/app/src/main/java/net/awan/pipelines/ui/screens/RunSummaryTab.kt @@ -133,6 +133,7 @@ internal fun RunSummaryTab( ) KeyValueRow("Pipeline", "id ${run.pipelineId} · v${run.version}") KeyValueRow("Stage", run.currentStage) + run.resourceWait?.let { Text(it.statusLabel(), style = text.meta) } KeyValueRow("Workspace", run.workspace) KeyValueRow("Content hash", run.contentHash, monospace = true) KeyValueRow("Started", formatLocalTimestamp(run.createdAt).orEmpty()) diff --git a/android/app/src/test/java/net/awan/pipelines/ui/ResourceWaitTest.kt b/android/app/src/test/java/net/awan/pipelines/ui/ResourceWaitTest.kt new file mode 100644 index 00000000..5bcdd82c --- /dev/null +++ b/android/app/src/test/java/net/awan/pipelines/ui/ResourceWaitTest.kt @@ -0,0 +1,32 @@ +package net.awan.pipelines.ui + +import kotlinx.serialization.json.Json +import net.awan.pipelines.data.protocol.ResourceWait +import net.awan.pipelines.data.protocol.RunDetail +import net.awan.pipelines.data.protocol.RunEventFrame +import net.awan.pipelines.data.protocol.RunSummary +import org.junit.Assert.* +import org.junit.Test + +class ResourceWaitTest { + @Test fun optionalMetadataAndPauseResumeClearing() { + val json = Json { ignoreUnknownKeys = true } + assertNull(json.decodeFromString("""{"id":1,"status":"queued"}""").resourceWait) + val wait = ResourceWait("phone-X", "review", 2, "2026-10-18T10:00:00Z") + val queued = RunEventFrame(runId = 1, status = "queued", resourceWaitSet = true, resourceWait = wait) + val detail = foldRunEvent(RunDetail(id = 1, status = "running"), queued) + assertEquals("queued", detail.status) + assertEquals(wait, detail.resourceWait) + assertEquals("Waiting for resource phone-X", wait.statusLabel()) + val rows = foldRunEvent(listOf(RunSummary(id = 1, status = "running")), queued).runs + assertEquals(wait, rows.single().resourceWait) + val clear = RunEventFrame(runId = 1, status = "running", resourceWaitSet = true) + assertNull(foldRunEvent(detail, clear).resourceWait) + assertNull(foldRunEvent(rows, clear).runs.single().resourceWait) + assertNull(foldRunEvent(detail, RunEventFrame(runId = 1, status = "canceled")).resourceWait) + assertEquals("succeeded", foldRunEvent(RunDetail(id = 1, status = "succeeded"), queued).status) + val decoded = json.decodeFromString("""{"id":1,"status":"queued","resource_wait":{"resource":"phone-X","stage":"review","sequence":2,"queued_since":"now"}}""") + assertEquals("phone-X", decoded.resourceWait?.resource) + assertFalse(isWaitingRunStatus(decoded.status)) + } +} diff --git a/docs/notes/phone-resource-wait.md b/docs/notes/phone-resource-wait.md new file mode 100644 index 00000000..33d6b3d2 --- /dev/null +++ b/docs/notes/phone-resource-wait.md @@ -0,0 +1,158 @@ +# Stage-exclusive resources — 2026-10-18, not deployed + +This is source work for run 355, **not a deployment, installation or publication**. +No physical phone, real operator session or deployed service was used or changed. +The Android role split does not retrospectively gate pinned versions or existing runs. + +## Contract and scope + +An executable non-human stage can declare one runner-owned control block: + +```resources +{"exclusive":"phone-47241FDAS00DLK"} +``` + +The key is 1–128 ASCII identifier bytes (letter first, then letters, digits, `_`, `-`). +Duplicate/unknown fields, multiple resources and human-stage resources are rejected. +One capacity-one resource per stage visit, one server/store authority; no expressions, +nested acquisition, distributed scheduling, capacity settings or multi-resource requests. +Resource-free definitions retain their execution path. Old servers reject this new block; +do not publish to an incompatible server. + +Acquisition follows selection and automatic UI-skip evaluation, before BeginStep, +visits/sequences, artifacts, filediff, stage timeout, budget setup or executor calls. +Queued runs retain workspace reservation, use legacy `queued` (never human `waiting`), +and carry bounded `resource_wait` identity/stage/sequence/queued-since metadata. +Events use `resource_wait_set=true` with absent `resource_wait` to clear caches. +CLI lists and TUI/Android details show “Waiting for resource …”; human decisions and +attention notifications remain separate. Older clients still see an unfinished run. + +SQLite owners and FIFO waiters are indexed by resource/order and run identity. +Private random capabilities attribute ownership to resource/run/stage/sequence and +never enter wire data, model input, transcripts or artifacts. Only the full capability +and positive cleanup evidence authorize release; mismatches and repeated release cannot +release a subsequent owner. One buffered channel per live queued run, no resource +polling, waiter tickers, model-based acquisition, queue driver or new daemon. + +The gate holds through tool cleanup. OpenCode interruption now confirms session idle/ +V1 settlement before clearing session ownership. Pi recovery requires both its receipt +and process/process-group exit, not merely abort acknowledgement. Executor panic, +ErrStageCleanup, retained active sessions and failed cleanup queries quarantine the owner. +An explicit `output.resource_cleanup_confirmed=false` (or malformed value) also vetoes +release for failed restoration; true never substitutes for harness cleanup evidence. +The generic runner does not inspect Android settings or magically verify restoration. +The Android UI reviewer must keep all device work foreground/awaited, restore original +settings/connection and main's build, and report uncertainty with the release veto. + +## Restart and retention + +Automatic FIFO progress is promised only within a live server process. Startup interrupts +persisted queued resource visits, removes their waiter rows, and never replays an executor. +Durably cleanup-confirmed owners can be reclaimed through the token-checked delete helper. +Unconfirmed owners remain quarantined, including crashes before BeginStep, before session +registration, or after cleanup but before its confirmation is persisted. Session interruption +alone cannot prove resource-specific restoration. Age never authorizes reclamation. +Quarantined ownership/session evidence survives recovery and protects even terminal runs +from retention. Reconciliation requires inspected cleanup and restoration, not lock deletion. +There is intentionally no force-unlock CLI or automatic uncertain-owner reclamation. + +## Android draft, provenance and activation + +Read-only retrieval: `pipelines show Android --version 2 --json` returned pipeline 20, +v2, hash `8c93aeb9b76f1e6e7696415d6308d786ec08d805a1a346fbabe4d252d5d65032`. +`python3` recomputed SHA-256 of the returned source and asserted equality. +The exact source is retained as `internal/server/testdata/android-v2-published.md`; +the proposed source is `internal/server/testdata/android-resource-draft.md`. +The provenance test compares graph fields/outcomes unchanged, apart from the approved +three stage prompts and the UI review resource; every stage keeps its published v3 model: Terra for implementer/committer and Luna for reviewer/UI reviewer. No unrelated published definition was edited. + +Artifact handoff recovery: the validated v3 draft was copied to this run's retained +artifact directory as `android-resource-draft.md`. The similarly named artifact from +run 355 remains the old v2 baseline and must not be published. The proposed source in +this workspace and this run's artifact have matching SHA-256 digests +`a10d8b9fa3dc6a1875b10469061ed25fb4a7a404d463d1a2d304a309d55aa5d9`. + +Activation, requiring separate explicit operator authorization: +1. Review/land the source change and independently rerun the required checks. +2. Arrange a compatible deployment separately; nothing here deploys or restarts it. +3. Reconcile every existing phone user, pinned/live run and any quarantined owner. +4. Transfer the **workspace proposed draft**, provenance and this note into the retained + stage artifacts after resolving the artifact edit permission. +5. Validate that retained draft against the configured authoritative compatible endpoint; + only then explicitly publish it. Do not use a disposable endpoint as a project default. +6. Opt in other device users explicitly before relying on mutual exclusion system-wide. + +Read-only published inventory (`pipelines show --json`, v2 for each): +- UI sweep: `survey` and `ui_reviewer` request visual/screenshots work; phone work there + needs explicit resource opt-in and restoration/fixture policy. +- Generic: `reviewer` and `ui_reviewer` have visual/device verification wording; when + targeting a phone, separate code review from leased device review first. +- Plan and land: `implementer`, `reviewer`, `ui_reviewer` contain rendered-surface work; + they need an explicit technology-specific role split/lease before phone use. +- Probe: one harness smoke stage; no default phone declaration. Any device-specific + probe needs explicit opt-in, never an inferred resource or Android stage-name special case. +Server/TUI/other definitions were not changed. Inventory is not activation evidence. + +## Verification evidence and remaining limits + +Executed focused checks: +- Run 358 recovery revalidation: `go build ./... && go vet ./... && go test ./...` + passed; `go test -race ./internal/pipeline ./internal/server ./internal/stage -run + 'Resource|Scheduler|Concurrent|Human|Decision|RetryWaits' -count=1` passed; and the + Android map invocation with `JAVA_HOME`/`ANDROID_HOME`, installed Gradle, + `--offline --no-daemon --rerun-tasks :app:assembleDebug :app:testDebugUnitTest` + executed all 41 tasks and reported `BUILD SUCCESSFUL`. These remain build/unit and + fake-executor scheduler evidence, + not physical-phone verification or screenshots. +- Recovery after integrating main `88595eb`: `go build ./...`, `go vet ./...`, + `go test ./internal/server -run Resource -count=1`, + `go test ./internal/pipeline ./internal/protocol ./internal/tui -run Resource -count=1`, + `go test ./internal/stage -count=1`, + `go test -race ./internal/pipeline ./internal/server ./internal/stage -run 'Resource|Scheduler|Concurrent|Human|Decision|RetryWaits' -count=1`, and + `go test ./...` all passed. `gofmt -l .` and `git diff --check` produced no output. +- Recovery Android build/unit check with the map's offline, no-daemon Gradle invocation: + `:app:assembleDebug :app:testDebugUnitTest` passed. This remains build/unit evidence, + not phone verification or screenshots. +- `go test ./internal/server ./internal/pipeline ./internal/protocol ./internal/tui -run Resource -count=1`: + passed. Includes fake-executor queue >30ms timeout, independent-resource progress, + FIFO/mismatch/duplicate/grant-cancel cycles, blocked cleanup, panic/quarantine, + persistence/BeginStep/query failure, no-acquire skip, restart/retention, wire defaults, + narrow/wide/plain TUI frames and real CLI/WebSocket scratch exercise. +- `go test ./internal/stage -count=1`: passed; the later full suite also included + `TestResourcePiReceiptRequiresProcessExit` (a test-owned sleep process, not a device). +- Android map invocation with `JAVA_HOME`/`ANDROID_HOME` and installed Gradle, + `--offline --no-daemon :app:assembleDebug :app:testDebugUnitTest`: BUILD SUCCESSFUL. + These are build/unit checks, not device verification or screenshots. +- After the Go tree settled: `go build ./...`, `go vet ./...`, `go test ./...` + all exited zero (including author-package tests). +- `go test -race ./internal/pipeline ./internal/server ./internal/stage -run 'Resource|Scheduler|Concurrent|Human|Decision|RetryWaits' -count=1`: + all three packages passed, including concurrent grant/cancel cycles. +- `gofmt -l .` and `git diff --check`: exited zero with no output. + +The scratch CLI test uses the real CLI and WebSocket handler with temporary SQLite and +an explicitly **FAKE executor**. It validates the proposed Android draft on compatible +scratch code, observes B as queued with zero steps while A holds fake-phone, inspects +subscription metadata, releases A, runs `wait`, and observes B succeeded automatically. +It does not call a model or use a physical phone. Test log output is evidence, not a +live-deployment claim; no retained CLI transcript was written to the denied artifact path. + +Live OpenCode/Pi shell/ADB termination and Android visual/device restoration are **not +tested** here. The checks prove scheduler behavior with controlled fakes, plus test-owned +process-exit checks, not device readiness on the current deployment. +Commit/merge/landing belong to the later committer stage; no commit or push here. + +## Stale shared documentation for docs-fold (do not edit in this item) + +- `docs/cli.md`: resource queue presentation, inspection metadata, unchanged wait/decide semantics. +- `internal/author/skill/references/format.md`: optional resources block and validation/key bounds. +- `internal/author/skill/references/control.md`: runner-owned declaration, timeout/accounting boundary, + release veto and automatic skip placement. +- `internal/author/skill/references/queue.md`: FIFO resource waits vs workspace exclusion and human waits. +- `internal/author/skill/references/operations.md`: authority, restart interruption/quarantine, + inspected recovery, retention and explicit compatible-deployment/publication prerequisites. +- `internal/author/skill/references/verification.md`: fake-vs-device evidence, fixtures and leased restoration. +- `internal/author/skill/references/orchestration.md`: no queue drivers/polling/extra agents; explicit + opt-in of every device user, no retroactive gating or distributed guarantees. + +No packaged skill/reference, shared README/topic documentation, pipeline registry, +installation, deployment, provider setting or `internal/stage/model_failure.go` was edited. diff --git a/internal/cli/app.go b/internal/cli/app.go index 8311ca35..5e26e393 100644 --- a/internal/cli/app.go +++ b/internal/cli/app.go @@ -774,7 +774,7 @@ func (a App) Run(ctx context.Context, args []string) error { } case "runs": for _, run := range r.Runs { - fmt.Fprintf(a.Out, "%d\t%s\t%s\tv%d\t%s\n", run.ID, run.Label(), run.Name, run.Version, run.Status) + fmt.Fprintf(a.Out, "%d\t%s\t%s\tv%d\t%s\n", run.ID, run.Label(), run.Name, run.Version, run.StatusLabel()) } case "ping": fmt.Fprintln(a.Out, "Connected ·", r.Time) @@ -1484,7 +1484,7 @@ func (e *waitLimitError) Runs() string { // waitLine is the one-line terminal report `pipelines wait` prints: the run id, // its status and outcome, its stage and how long it took. func waitLine(d *protocol.RunDetail) string { - line := fmt.Sprintf("run %d · %s", d.ID, d.Status) + line := fmt.Sprintf("run %d · %s", d.ID, d.StatusLabel()) if outcome := strings.TrimSpace(d.Outcome); outcome != "" { line += " (" + outcome + ")" } diff --git a/internal/pipeline/control.go b/internal/pipeline/control.go index c603497e..9d32e974 100644 --- a/internal/pipeline/control.go +++ b/internal/pipeline/control.go @@ -10,7 +10,7 @@ import ( ) func decodeControl(text string, value any) error { - // Both control blocks are flat objects. Reject duplicate keys before decoding. + // Control blocks are flat objects. Reject duplicate keys before decoding. keys := json.NewDecoder(strings.NewReader(text)) if token, err := keys.Token(); err != nil || token != json.Delim('{') { return fmt.Errorf("expected object") @@ -44,6 +44,12 @@ func decodeControl(text string, value any) error { } func parseControl(s *Stage, fields map[string]string) error { + if text, ok := fields["resources"]; ok { + if err := decodeControl(text, &s.Resources); err != nil || s.Resources == nil || !ValidResourceKey(s.Resources.Exclusive) { + return fmt.Errorf("resources must contain exactly one exclusive key (1–128 ASCII letters, digits, _ or -; start with a letter)") + } + delete(fields, "resources") + } if text, ok := fields["limits"]; ok { if err := decodeControl(text, &s.Limits); err != nil || s.Limits == nil { return fmt.Errorf("limits must be an object with max_visits and optional on outcomes") @@ -76,6 +82,9 @@ func parseControl(s *Stage, fields map[string]string) error { } func validateControl(s Stage, stages map[string]bool) error { + if s.Resources != nil && s.Type == "human" { + return fmt.Errorf("stage %s: human stages cannot declare resources", s.Key) + } if s.Limits != nil { for _, outcome := range s.Limits.On { target, exists := s.Edges[outcome] @@ -95,3 +104,7 @@ func validateControl(s Stage, stages map[string]bool) error { } return nil } + +func ValidResourceKey(key string) bool { + return len(key) <= 128 && identifier.MatchString(key) +} diff --git a/internal/pipeline/resources.go b/internal/pipeline/resources.go new file mode 100644 index 00000000..1a1720b1 --- /dev/null +++ b/internal/pipeline/resources.go @@ -0,0 +1,277 @@ +package pipeline + +import ( + "context" + "crypto/rand" + "database/sql" + "encoding/hex" + "encoding/json" + "fmt" + "time" + + "git.a-wan.net/alan/pipelines/internal/protocol" +) + +// ResourceVisit identifies a pending visit, before it consumes execution accounting. +type ResourceVisit struct { + Resource string + RunID int64 + Stage string + Sequence int +} + +// ResourceLease is an opaque capability. Its credential is deliberately not exported +// or serializable, and must never be put in model input or public wire metadata. +type ResourceLease struct { + ResourceVisit + token string +} + +// RequestResource acquires immediately or registers one FIFO waiter/channel. The +// mutex covers only short DB transactions and channel bookkeeping, never execution. +func (s *Store) RequestResource(ctx context.Context, visit ResourceVisit) (ResourceLease, <-chan ResourceLease, *protocol.ResourceWait, error) { + s.resourceMu.Lock() + defer s.resourceMu.Unlock() + if !ValidResourceKey(visit.Resource) || visit.RunID <= 0 || visit.Sequence <= 0 || !identifier.MatchString(visit.Stage) { + return ResourceLease{}, nil, nil, fmt.Errorf("invalid resource visit") + } + if ch := s.resourceWaiters[visit]; ch != nil { + wait, err := s.resourceWait(ctx, visit.RunID) + return ResourceLease{}, ch, wait, err + } + credential := make([]byte, 32) + if _, err := rand.Read(credential); err != nil { + return ResourceLease{}, nil, nil, err + } + lease := ResourceLease{ResourceVisit: visit, token: hex.EncodeToString(credential)} + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + return ResourceLease{}, nil, nil, err + } + defer tx.Rollback() + var status string + if err := tx.QueryRowContext(ctx, "SELECT status FROM pipeline_runs WHERE id=?", visit.RunID).Scan(&status); err != nil { + return ResourceLease{}, nil, nil, err + } + if status != protocol.RunStatusQueued && status != protocol.RunStatusRunning { + return ResourceLease{}, nil, nil, fmt.Errorf("run is not active") + } + var existing ResourceLease + err = tx.QueryRowContext(ctx, "SELECT resource_key,run_id,stage_key,sequence,token FROM resource_owners WHERE run_id=?", visit.RunID).Scan(&existing.Resource, &existing.RunID, &existing.Stage, &existing.Sequence, &existing.token) + if err == nil { + if existing.ResourceVisit != visit { + return ResourceLease{}, nil, nil, fmt.Errorf("run already owns another resource visit") + } + return existing, nil, nil, nil + } + if err != sql.ErrNoRows { + return ResourceLease{}, nil, nil, err + } + result, err := tx.ExecContext(ctx, `INSERT INTO resource_owners(resource_key,run_id,stage_key,sequence,token) +VALUES(?,?,?,?,?) ON CONFLICT(resource_key) DO NOTHING`, visit.Resource, visit.RunID, visit.Stage, visit.Sequence, lease.token) + if err != nil { + return ResourceLease{}, nil, nil, err + } + n, err := result.RowsAffected() + if err != nil { + return ResourceLease{}, nil, nil, err + } + if n == 1 { + return lease, nil, nil, tx.Commit() + } + wait := &protocol.ResourceWait{Resource: visit.Resource, Stage: visit.Stage, Sequence: visit.Sequence, QueuedSince: time.Now().UTC().Format(time.RFC3339Nano)} + encoded, err := json.Marshal(wait) + if err != nil { + return ResourceLease{}, nil, nil, err + } + if _, err := tx.ExecContext(ctx, `INSERT INTO resource_waiters(resource_key,run_id,stage_key,sequence,token,queued_at) VALUES(?,?,?,?,?,?)`, visit.Resource, visit.RunID, visit.Stage, visit.Sequence, lease.token, wait.QueuedSince); err != nil { + return ResourceLease{}, nil, nil, err + } + if err := affected(tx.ExecContext(ctx, "UPDATE pipeline_runs SET status='queued',current_stage=?,resource_wait=? WHERE id=? AND status IN ('queued','running')", visit.Stage, string(encoded), visit.RunID)); err != nil { + return ResourceLease{}, nil, nil, err + } + if err := tx.Commit(); err != nil { + return ResourceLease{}, nil, nil, err + } + if s.resourceWaiters == nil { + s.resourceWaiters = make(map[ResourceVisit]chan ResourceLease) + } + ch := make(chan ResourceLease, 1) + s.resourceWaiters[visit] = ch + return ResourceLease{}, ch, wait, nil +} + +func (s *Store) resourceWait(ctx context.Context, id int64) (*protocol.ResourceWait, error) { + var wait protocol.ResourceWait + err := s.db.QueryRowContext(ctx, "SELECT resource_key,stage_key,sequence,queued_at FROM resource_waiters WHERE run_id=?", id).Scan(&wait.Resource, &wait.Stage, &wait.Sequence, &wait.QueuedSince) + if err == sql.ErrNoRows { + return nil, nil + } + return &wait, err +} + +// CancelResource removes a queued visit, or returns a concurrently reserved grant +// to the queue before any execution is allowed. It cannot cancel another visit. +func (s *Store) CancelResource(ctx context.Context, visit ResourceVisit) error { + s.resourceMu.Lock() + defer s.resourceMu.Unlock() + defer delete(s.resourceWaiters, visit) + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + return err + } + defer tx.Rollback() + deleted, err := tx.ExecContext(ctx, "DELETE FROM resource_waiters WHERE resource_key=? AND run_id=? AND stage_key=? AND sequence=?", visit.Resource, visit.RunID, visit.Stage, visit.Sequence) + if err != nil { + return err + } + n, err := deleted.RowsAffected() + if err != nil { + return err + } + if n > 0 { + if _, err := tx.ExecContext(ctx, "UPDATE pipeline_runs SET resource_wait='' WHERE id=?", visit.RunID); err != nil { + return err + } + } + ch := s.resourceWaiters[visit] + // A grant is buffered while this registry entry exists. Cancellation can + // reclaim only that not-yet-accepted capability, never an executing owner. + var grant ResourceLease + if ch != nil { + select { + case grant = <-ch: + default: + } + } + if err := tx.Commit(); err != nil { + return err + } + if grant.token != "" { + if err := s.SetResourceCleanup(ctx, grant, true); err != nil { + return err + } + return s.releaseResourceLocked(ctx, grant) + } + return nil +} + +// AcceptResource removes the channel exactly once. Cancellation is rechecked by +// the runner before BeginStep; a canceled accepted grant is released as unused. +func (s *Store) AcceptResource(lease ResourceLease) { + s.resourceMu.Lock() + delete(s.resourceWaiters, lease.ResourceVisit) + s.resourceMu.Unlock() +} + +// HasResourceOwner is used only for orphan/restart reconciliation, never for a +// resource-free stage visit. +func (s *Store) HasResourceOwner(ctx context.Context, runID int64) (bool, error) { + var exists bool + err := s.db.QueryRowContext(ctx, "SELECT EXISTS(SELECT 1 FROM resource_owners WHERE run_id=?)", runID).Scan(&exists) + return exists, err +} + +func (s *Store) ConfirmRecoveredSession(ctx context.Context, owner SessionOwner) error { + _, err := s.db.ExecContext(ctx, "UPDATE run_steps SET agent_active=0 WHERE run_id=? AND session_id=?", owner.RunID, owner.ID) + return err +} + +// SetResourceCleanup records positive quiescence durably before release. False +// quarantines the owner; neither age nor another run can remove the lease. +func (s *Store) SetResourceCleanup(ctx context.Context, lease ResourceLease, confirmed bool) error { + diagnostic := "" + if !confirmed { + diagnostic = "Cleanup/restoration unconfirmed; resource quarantined pending inspected reconciliation" + } + return affected(s.db.ExecContext(ctx, `UPDATE resource_owners SET cleanup_confirmed=?,diagnostic=? WHERE resource_key=? AND run_id=? AND stage_key=? AND sequence=? AND token=?`, confirmed, diagnostic, lease.Resource, lease.RunID, lease.Stage, lease.Sequence, lease.token)) +} + +// ReleaseResource requires the complete capability AND positive cleanup evidence. +// Mismatches and repeated releases are harmless, including after transfer. +func (s *Store) ReleaseResource(ctx context.Context, lease ResourceLease) error { + s.resourceMu.Lock() + defer s.resourceMu.Unlock() + return s.releaseResourceLocked(ctx, lease) +} + +func (s *Store) releaseResourceLocked(ctx context.Context, lease ResourceLease) error { + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + return err + } + defer tx.Rollback() + n, err := deleteResourceOwner(ctx, tx, lease) + if err != nil { + return err + } + if n == 0 { + return nil + } + var next ResourceLease + next.Resource = lease.Resource + err = tx.QueryRowContext(ctx, `SELECT run_id,stage_key,sequence,token FROM resource_waiters WHERE resource_key=? ORDER BY queue_order LIMIT 1`, lease.Resource).Scan(&next.RunID, &next.Stage, &next.Sequence, &next.token) + if err != nil && err != sql.ErrNoRows { + return err + } + if err == nil { + if s.resourceWaiters[next.ResourceVisit] == nil { + return fmt.Errorf("resource waiter has no live runner; refusing handoff") + } + if _, err := tx.ExecContext(ctx, `INSERT INTO resource_owners(resource_key,run_id,stage_key,sequence,token) VALUES(?,?,?,?,?)`, next.Resource, next.RunID, next.Stage, next.Sequence, next.token); err != nil { + return err + } + if _, err := tx.ExecContext(ctx, "DELETE FROM resource_waiters WHERE run_id=?", next.RunID); err != nil { + return err + } + if _, err := tx.ExecContext(ctx, "UPDATE pipeline_runs SET resource_wait='' WHERE id=?", next.RunID); err != nil { + return err + } + } + if err := tx.Commit(); err != nil { + return err + } + if next.token != "" { + s.resourceWaiters[next.ResourceVisit] <- next + } + return nil +} + +func deleteResourceOwner(ctx context.Context, tx *sql.Tx, lease ResourceLease) (int64, error) { + result, err := tx.ExecContext(ctx, `DELETE FROM resource_owners WHERE resource_key=? AND run_id=? AND stage_key=? AND sequence=? AND token=? AND cleanup_confirmed=1`, lease.Resource, lease.RunID, lease.Stage, lease.Sequence, lease.token) + if err != nil { + return 0, err + } + return result.RowsAffected() +} + +// recoverResourceOwners runs only at startup, after pending waiters have been +// interrupted and removed. Every reclamation uses the persisted capability; +// absent cleanup evidence is never replaced with a timestamp-based expiry. +func recoverResourceOwners(ctx context.Context, tx *sql.Tx) error { + rows, err := tx.QueryContext(ctx, "SELECT resource_key,run_id,stage_key,sequence,token FROM resource_owners WHERE cleanup_confirmed=1") + if err != nil { + return err + } + var leases []ResourceLease + for rows.Next() { + var lease ResourceLease + if err := rows.Scan(&lease.Resource, &lease.RunID, &lease.Stage, &lease.Sequence, &lease.token); err != nil { + rows.Close() + return err + } + leases = append(leases, lease) + } + err = rows.Err() + rows.Close() + if err != nil { + return err + } + for _, lease := range leases { + if _, err := deleteResourceOwner(ctx, tx, lease); err != nil { + return err + } + } + _, err = tx.ExecContext(ctx, "UPDATE resource_owners SET diagnostic='Server stopped; resource quarantined pending inspected cleanup/restoration'") + return err +} diff --git a/internal/pipeline/resources_test.go b/internal/pipeline/resources_test.go new file mode 100644 index 00000000..58ae2e6a --- /dev/null +++ b/internal/pipeline/resources_test.go @@ -0,0 +1,288 @@ +package pipeline + +import ( + "context" + "encoding/json" + "fmt" + "path/filepath" + "strings" + "sync" + "testing" + "time" +) + +func TestResourceDeclaration(t *testing.T) { + base := Template([]string{"work"}, true) + for _, value := range []string{`{"exclusive":"phone-X"}`, `{ "exclusive" : "A_1" }`} { + graph, err := Parse(base + "```resources\n" + value + "\n```\n") + if err != nil || graph.Stages[0].Resources == nil { + t.Fatalf("%s: %v", value, err) + } + if _, ok := StageDefinition(graph.Stages[0]).Blocks["resources"]; ok { + t.Fatal("runner control reached executor") + } + encoded, _ := json.Marshal(graph) + if !strings.Contains(string(encoded), `"resources"`) { + t.Fatal("resource lost in persisted graph") + } + } + for _, value := range []string{`{}`, `null`, `[]`, `{"exclusive":""}`, `{"exclusive":"a b"}`, `{"exclusive":"a","exclusive":"b"}`, `{"exclusive":"a","extra":1}`, `{"exclusive":["a","b"]}`, fmt.Sprintf(`{"exclusive":%q}`, strings.Repeat("a", 129))} { + if _, err := Parse(base + "```resources\n" + value + "\n```\n"); err == nil { + t.Fatalf("accepted %s", value) + } + } + human := "# stage gate\n```prompt\nChoose\n```\n```configs\ntype=human\n```\n```edges\ngo -> #success\n```\n```resources\n{\"exclusive\":\"phone\"}\n```\n" + if _, err := Parse(human); err == nil { + t.Fatal("human acquired a resource") + } +} + +func resourceStore(t *testing.T) (*Store, func(string) ResourceVisit) { + t.Helper() + ctx := context.Background() + s, err := Open(filepath.Join(t.TempDir(), "resources.db")) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { s.Close() }) + if _, err := s.Submit(ctx, "resource", Template([]string{"work"}, true)); err != nil { + t.Fatal(err) + } + items, _ := s.List(ctx) + d, _ := s.Get(ctx, items[0].ID, 1) + count := 0 + return s, func(resource string) ResourceVisit { + count++ + id, _, err := s.CreateRun(ctx, d, fmt.Sprint(count), "workspace", false, nil, nil) + if err != nil { + t.Fatal(err) + } + return ResourceVisit{Resource: resource, RunID: id, Stage: "work", Sequence: 1} + } +} + +func acquireTestResource(t *testing.T, s *Store, visit ResourceVisit) ResourceLease { + t.Helper() + l, ch, _, err := s.RequestResource(context.Background(), visit) + if err != nil || ch != nil { + t.Fatalf("acquire: %v %v", ch, err) + } + return l +} + +func releaseTestResource(t *testing.T, s *Store, lease ResourceLease) { + t.Helper() + if err := s.SetResourceCleanup(context.Background(), lease, true); err != nil { + t.Fatal(err) + } + if err := s.ReleaseResource(context.Background(), lease); err != nil { + t.Fatal(err) + } +} + +func TestResourceFIFOOwnershipCancellationAndRegistry(t *testing.T) { + s, create := resourceStore(t) + ctx := context.Background() + a := acquireTestResource(t, s, create("X")) + duplicate := acquireTestResource(t, s, a.ResourceVisit) + if duplicate != a { + t.Fatal("duplicate acquisition changed capability") + } + var visits []ResourceVisit + var channels []<-chan ResourceLease + for i := 0; i < 4; i++ { + v := create("X") + _, ch, wait, err := s.RequestResource(ctx, v) + if err != nil || ch == nil || wait == nil { + t.Fatal("missing waiter", err) + } + _, again, _, err := s.RequestResource(ctx, v) + if err != nil || again != ch { + t.Fatal("duplicate waiter registered", err) + } + visits = append(visits, v) + channels = append(channels, ch) + } + if err := s.ReleaseResource(ctx, a); err != nil { + t.Fatal(err) + } // no cleanup receipt + for _, ch := range channels { + select { + case <-ch: + t.Fatal("unconfirmed release granted") + default: + } + } + if err := s.SetResourceCleanup(ctx, a, true); err != nil { + t.Fatal(err) + } + for _, corrupt := range []func(*ResourceLease){func(l *ResourceLease) { l.RunID++ }, func(l *ResourceLease) { l.Stage = "other" }, func(l *ResourceLease) { l.Sequence++ }, func(l *ResourceLease) { l.token = "wrong" }} { + bad := a + corrupt(&bad) + if err := s.ReleaseResource(ctx, bad); err != nil { + t.Fatal(err) + } + } + for _, ch := range channels { + select { + case <-ch: + t.Fatal("unconfirmed/mismatched release granted") + default: + } + } + if err := s.CancelResource(ctx, visits[1]); err != nil { + t.Fatal(err) + } + releaseTestResource(t, s, a) + for _, i := range []int{0, 2, 3} { + var grant ResourceLease + select { + case grant = <-channels[i]: + case <-time.After(time.Second): + t.Fatal("FIFO grant not delivered") + } + s.AcceptResource(grant) + if grant.ResourceVisit != visits[i] { + t.Fatal("wrong FIFO owner") + } + // An old owner's repeated release cannot release the next owner. + if err := s.ReleaseResource(ctx, a); err != nil { + t.Fatal(err) + } + for _, j := range []int{2, 3} { + if j > i { + select { + case <-channels[j]: + t.Fatal("multiple grants") + default: + } + } + } + releaseTestResource(t, s, grant) + } + if len(s.resourceWaiters) != 0 { + t.Fatal("channel registry leaked") + } + for i := 0; i < 30; i++ { + owner := acquireTestResource(t, s, create("X")) + v := create("X") + _, _, _, err := s.RequestResource(ctx, v) + if err != nil { + t.Fatal(err) + } + releaseTestResource(t, s, owner) // race window: granted but not accepted + if err := s.CancelResource(ctx, v); err != nil { + t.Fatal(err) + } + if len(s.resourceWaiters) != 0 { + t.Fatal("grant/cancel leaked channel") + } + } + for i := 0; i < 30; i++ { + owner := acquireTestResource(t, s, create("X")) + v := create("X") + if _, _, _, err := s.RequestResource(ctx, v); err != nil { + t.Fatal(err) + } + if err := s.SetResourceCleanup(ctx, owner, true); err != nil { + t.Fatal(err) + } + var wg sync.WaitGroup + wg.Add(2) + go func() { + defer wg.Done() + if err := s.ReleaseResource(ctx, owner); err != nil { + t.Error(err) + } + }() + go func() { + defer wg.Done() + if err := s.CancelResource(ctx, v); err != nil { + t.Error(err) + } + }() + wg.Wait() + if len(s.resourceWaiters) != 0 { + t.Fatal("concurrent grant/cancel leaked channel") + } + } + var owners, waiters int + s.db.QueryRow("SELECT COUNT(*) FROM resource_owners").Scan(&owners) + s.db.QueryRow("SELECT COUNT(*) FROM resource_waiters").Scan(&waiters) + if owners != 0 || waiters != 0 { + t.Fatalf("stranded owners/waiters: %d/%d", owners, waiters) + } +} + +func TestResourceCancellationPersistenceFailureRemovesChannel(t *testing.T) { + s, create := resourceStore(t) + acquireTestResource(t, s, create("X")) + v := create("X") + if _, _, _, err := s.RequestResource(context.Background(), v); err != nil { + t.Fatal(err) + } + if err := s.Close(); err != nil { + t.Fatal(err) + } + if err := s.CancelResource(context.Background(), v); err == nil { + t.Fatal("closed DB cancellation claimed success") + } + if len(s.resourceWaiters) != 0 { + t.Fatal("persistence failure leaked channel") + } +} + +func TestResourceRestartQuarantineAndRetention(t *testing.T) { + for _, confirmed := range []bool{false, true} { + t.Run(fmt.Sprint(confirmed), func(t *testing.T) { + s, create := resourceStore(t) + ctx := context.Background() + a := acquireTestResource(t, s, create("X")) + b := create("X") + if _, _, _, err := s.RequestResource(ctx, b); err != nil { + t.Fatal(err) + } + if confirmed { + if err := s.SetResourceCleanup(ctx, a, true); err != nil { + t.Fatal(err) + } + } + path := s.path + if err := s.Close(); err != nil { + t.Fatal(err) + } + reopened, err := Open(path) + if err != nil { + t.Fatal(err) + } + defer reopened.Close() + if err := reopened.RecoverRuns(ctx); err != nil { + t.Fatal(err) + } + if err := reopened.RecoverRuns(ctx); err != nil { + t.Fatal(err) + } + detail, err := reopened.GetRun(ctx, b.RunID) + if err != nil || detail.Status != "interrupted" || detail.ResourceWait != nil || len(detail.Steps) != 0 || !strings.Contains(detail.Diagnostic, "resource") { + t.Fatal("queued visit replayed/lost diagnostic", detail, err) + } + exists, err := reopened.HasResourceOwner(ctx, a.RunID) + if err != nil || exists == confirmed { + t.Fatal("incorrect recovery disposition", exists, err) + } + report, err := reopened.RetentionReport(ctx, RetentionPolicy{Keep: 0}, "", time.Now()) + if err != nil { + t.Fatal(err) + } + protected := false + for _, p := range report.Protected { + if p.ID == a.RunID { + protected = true + } + } + if protected == confirmed { + t.Fatal("quarantined evidence not protected") + } + }) + } +} diff --git a/internal/pipeline/retention.go b/internal/pipeline/retention.go index c4873632..0e7a38f5 100644 --- a/internal/pipeline/retention.go +++ b/internal/pipeline/retention.go @@ -130,11 +130,12 @@ type retentionQuerier interface { // retentionRunInfo is one pipeline_runs row plus the step aggregates protection // needs, read in the same statement so the report never issues a per-run query. type retentionRunInfo struct { - id int64 - status string - createdAt time.Time - waitingSteps int - activeSteps int + id int64 + status string + createdAt time.Time + waitingSteps int + activeSteps int + resourceOwners int } // protectionReasons lists why a run must not be removed for a reason intrinsic @@ -155,6 +156,9 @@ func (r retentionRunInfo) protectionReasons() []string { if r.activeSteps > 0 { reasons = append(reasons, "cleanup agent still active (agent_active=1)") } + if r.resourceOwners > 0 { + reasons = append(reasons, "resource ownership retained (possibly quarantined)") + } return reasons } @@ -171,9 +175,20 @@ type retentionSelection struct { // report is a bounded number of round trips rather than one query per run. func selectRetentionRuns(ctx context.Context, q retentionQuerier, policy RetentionPolicy, now time.Time) (retentionSelection, error) { var sel retentionSelection + // Maintenance is deliberately non-migrating and may inspect a database + // whose server predates resource support. + var resources bool + if err := q.QueryRowContext(ctx, "SELECT EXISTS(SELECT 1 FROM sqlite_master WHERE type='table' AND name='resource_owners')").Scan(&resources); err != nil { + return sel, err + } + owners := "0" + if resources { + owners = "(SELECT COUNT(*) FROM resource_owners o WHERE o.run_id=r.id)" + } rows, err := q.QueryContext(ctx, `SELECT r.id,r.status,strftime('%Y-%m-%dT%H:%M:%SZ',r.created_at), COALESCE((SELECT COUNT(*) FROM run_steps s WHERE s.run_id=r.id AND s.status=?),0), -COALESCE((SELECT COUNT(*) FROM run_steps s WHERE s.run_id=r.id AND s.agent_active=1),0) `+ +COALESCE((SELECT COUNT(*) FROM run_steps s WHERE s.run_id=r.id AND s.agent_active=1),0), +`+owners+` `+ runFrom+` ORDER BY r.id DESC`, protocol.StepStatusWaiting) if err != nil { return sel, err @@ -182,7 +197,7 @@ COALESCE((SELECT COUNT(*) FROM run_steps s WHERE s.run_id=r.id AND s.agent_activ for rows.Next() { var run retentionRunInfo var created string - if err := rows.Scan(&run.id, &run.status, &created, &run.waitingSteps, &run.activeSteps); err != nil { + if err := rows.Scan(&run.id, &run.status, &created, &run.waitingSteps, &run.activeSteps, &run.resourceOwners); err != nil { return sel, err } run.createdAt = parseRetentionTime(created) diff --git a/internal/pipeline/runs.go b/internal/pipeline/runs.go index 5b58328c..76e1de11 100644 --- a/internal/pipeline/runs.go +++ b/internal/pipeline/runs.go @@ -21,14 +21,19 @@ const runStamps = `strftime('%Y-%m-%dT%H:%M:%SZ',r.created_at),COALESCE(strftime const runSelect = `SELECT r.id,r.pipeline_id,p.name,r.version,v.content_hash,r.status,r.outcome,r.current_stage, ` + runStamps + `,r.diagnostic, CASE WHEN r.status='succeeded' THEN r.output_json ELSE COALESCE((SELECT output_json FROM run_steps WHERE run_id=r.id AND status='completed' ORDER BY sequence DESC LIMIT 1),'{}') END, -(SELECT COUNT(*) FROM run_steps WHERE run_id=r.id AND status='completed'),r.workspace,r.title,r.model,r.stage_models +(SELECT COUNT(*) FROM run_steps WHERE run_id=r.id AND status='completed'),r.workspace,r.title,r.model,r.stage_models,r.resource_wait ` + runFrom + ` JOIN pipeline_versions v ON v.pipeline_id=r.pipeline_id AND v.version=r.version` func scanRun(row interface{ Scan(...any) error }, r *protocol.RunSummary) error { - var output, stageModels string - if err := row.Scan(&r.ID, &r.PipelineID, &r.Name, &r.Version, &r.ContentHash, &r.Status, &r.Outcome, &r.CurrentStage, &r.CreatedAt, &r.FinishedAt, &r.Diagnostic, &output, &r.CompletedSteps, &r.Workspace, &r.Title, &r.Model, &stageModels); err != nil { + var output, stageModels, resourceWait string + if err := row.Scan(&r.ID, &r.PipelineID, &r.Name, &r.Version, &r.ContentHash, &r.Status, &r.Outcome, &r.CurrentStage, &r.CreatedAt, &r.FinishedAt, &r.Diagnostic, &output, &r.CompletedSteps, &r.Workspace, &r.Title, &r.Model, &stageModels, &resourceWait); err != nil { return err } + if resourceWait != "" { + if err := json.Unmarshal([]byte(resourceWait), &r.ResourceWait); err != nil { + return err + } + } if stageModels != "" && stageModels != "{}" { if err := json.Unmarshal([]byte(stageModels), &r.StageModels); err != nil { return err diff --git a/internal/pipeline/runs_mutate.go b/internal/pipeline/runs_mutate.go index d27e7268..d370a29f 100644 --- a/internal/pipeline/runs_mutate.go +++ b/internal/pipeline/runs_mutate.go @@ -44,7 +44,7 @@ func (s *Store) BeginStep(ctx context.Context, id int64, sequence int, key strin return err } defer tx.Rollback() - if err := affected(tx.ExecContext(ctx, "UPDATE pipeline_runs SET status='running',current_stage=? WHERE id=? AND status IN ('queued','running')", key, id)); err != nil { + if err := affected(tx.ExecContext(ctx, "UPDATE pipeline_runs SET status='running',current_stage=?,resource_wait='' WHERE id=? AND status IN ('queued','running')", key, id)); err != nil { return err } if _, err := tx.ExecContext(ctx, "INSERT INTO run_steps(run_id,sequence,stage_key,input_json) VALUES (?,?,?,?)", id, sequence, key, string(encoded)); err != nil { @@ -310,5 +310,5 @@ func (s *Store) FinishRun(ctx context.Context, id int64, status, outcome, diagno if err != nil { return err } - return affected(s.db.ExecContext(ctx, "UPDATE pipeline_runs SET status=?,outcome=?,diagnostic=?,output_json=?,finished_at=CURRENT_TIMESTAMP WHERE id=? AND status IN ('queued','running')", status, outcome, diagnostic, string(data), id)) + return affected(s.db.ExecContext(ctx, "UPDATE pipeline_runs SET status=?,outcome=?,diagnostic=?,output_json=?,finished_at=CURRENT_TIMESTAMP,resource_wait='' WHERE id=? AND status IN ('queued','running')", status, outcome, diagnostic, string(data), id)) } diff --git a/internal/pipeline/runs_schema.go b/internal/pipeline/runs_schema.go index a8c1c78c..8f861061 100644 --- a/internal/pipeline/runs_schema.go +++ b/internal/pipeline/runs_schema.go @@ -51,13 +51,28 @@ CREATE TABLE IF NOT EXISTS run_session_owners ( run_id INTEGER NOT NULL, sequence INTEGER NOT NULL, backend TEXT NOT NULL, reference TEXT NOT NULL DEFAULT '', PRIMARY KEY(run_id,sequence), FOREIGN KEY(run_id,sequence) REFERENCES run_steps(run_id,sequence)); CREATE TABLE IF NOT EXISTS run_retries ( -run_id INTEGER PRIMARY KEY REFERENCES pipeline_runs(id), original_run_id INTEGER NOT NULL REFERENCES pipeline_runs(id));`) +run_id INTEGER PRIMARY KEY REFERENCES pipeline_runs(id), original_run_id INTEGER NOT NULL REFERENCES pipeline_runs(id)); +CREATE TABLE IF NOT EXISTS resource_owners ( +resource_key TEXT PRIMARY KEY, run_id INTEGER NOT NULL REFERENCES pipeline_runs(id), +stage_key TEXT NOT NULL, sequence INTEGER NOT NULL, token TEXT NOT NULL, +cleanup_confirmed INTEGER NOT NULL DEFAULT 0, diagnostic TEXT NOT NULL DEFAULT '', +acquired_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP); +CREATE INDEX IF NOT EXISTS resource_owner_run ON resource_owners(run_id); +CREATE UNIQUE INDEX IF NOT EXISTS resource_owner_visit ON resource_owners(run_id); +CREATE TABLE IF NOT EXISTS resource_waiters ( +queue_order INTEGER PRIMARY KEY AUTOINCREMENT, resource_key TEXT NOT NULL, +run_id INTEGER NOT NULL UNIQUE REFERENCES pipeline_runs(id), stage_key TEXT NOT NULL, +sequence INTEGER NOT NULL, token TEXT NOT NULL, queued_at TEXT NOT NULL); +CREATE INDEX IF NOT EXISTS resource_wait_fifo ON resource_waiters(resource_key,queue_order);`) if err != nil { return err } if err := s.migrateRunTitles(); err != nil { return err } + if err := s.addColumnIfMissing("pipeline_runs", "resource_wait", "TEXT NOT NULL DEFAULT ''"); err != nil { + return err + } return s.migrateRunModels() } @@ -122,16 +137,29 @@ func (s *Store) addColumnIfMissing(table, column, definition string) error { } func (s *Store) RecoverRuns(ctx context.Context) error { + s.resourceMu.Lock() + defer s.resourceMu.Unlock() tx, err := s.db.BeginTx(ctx, nil) if err != nil { return err } defer tx.Rollback() - if _, err := tx.ExecContext(ctx, "UPDATE run_steps SET status='interrupted',agent_active=0,diagnostic='Server stopped; stage was not resumed',finished_at=CURRENT_TIMESTAMP WHERE (status='running' OR agent_active=1) AND status<>?", protocol.StepStatusWaiting); err != nil { + // No replay: pending visits never began. Uncertain owners survive recovery, + // including the crash window before BeginStep/session registration. + if _, err := tx.ExecContext(ctx, "UPDATE pipeline_runs SET diagnostic='Server stopped while queued for a resource; pending visit was not resumed',status='interrupted',finished_at=CURRENT_TIMESTAMP,resource_wait='' WHERE id IN (SELECT run_id FROM resource_waiters); DELETE FROM resource_waiters"); err != nil { + return err + } + if err := recoverResourceOwners(ctx, tx); err != nil { + return err + } + if _, err := tx.ExecContext(ctx, "UPDATE run_steps SET status='interrupted',agent_active=CASE WHEN EXISTS(SELECT 1 FROM resource_owners o WHERE o.run_id=run_steps.run_id AND o.sequence=run_steps.sequence) THEN agent_active ELSE 0 END,diagnostic='Server stopped; stage was not resumed',finished_at=CURRENT_TIMESTAMP WHERE (status='running' OR agent_active=1) AND status<>?", protocol.StepStatusWaiting); err != nil { return err } if _, err := tx.ExecContext(ctx, "UPDATE pipeline_runs SET status='interrupted',diagnostic='Server stopped; run was not resumed',finished_at=CURRENT_TIMESTAMP WHERE status IN ('queued','running') AND status<>?", protocol.RunStatusWaiting); err != nil { return err } + if _, err := tx.ExecContext(ctx, "UPDATE pipeline_runs SET diagnostic='Server stopped; resource ownership quarantined: inspect retained session, tool termination and restoration before reconciliation' WHERE id IN (SELECT run_id FROM resource_owners)"); err != nil { + return err + } return tx.Commit() } diff --git a/internal/pipeline/stage.go b/internal/pipeline/stage.go index d2a04b44..3afb8135 100644 --- a/internal/pipeline/stage.go +++ b/internal/pipeline/stage.go @@ -18,6 +18,7 @@ func StageDefinition(s Stage) stage.Definition { delete(blocks, "input") // Retired contracts in historical graph JSON are ignored. delete(blocks, "limits") delete(blocks, "memory") + delete(blocks, "resources") for name, value := range map[string]string{"output": s.Output, "prompt": s.Prompt} { if value != "" { blocks[name] = value diff --git a/internal/pipeline/store.go b/internal/pipeline/store.go index 46466d51..abcfe29a 100644 --- a/internal/pipeline/store.go +++ b/internal/pipeline/store.go @@ -18,10 +18,12 @@ import ( ) type Store struct { - db *sql.DB - dir string - path string - mu sync.Mutex + db *sql.DB + dir string + path string + mu sync.Mutex + resourceMu sync.Mutex + resourceWaiters map[ResourceVisit]chan ResourceLease } // openDB opens one SQLite connection pool for a DSN. Every open path shares it diff --git a/internal/protocol/message.go b/internal/protocol/message.go index 74065770..f3d1f8d5 100644 --- a/internal/protocol/message.go +++ b/internal/protocol/message.go @@ -19,6 +19,7 @@ const ( ) type Stage struct { + Resources *StageResources `json:"resources,omitempty"` Limits *StageLimits `json:"limits,omitempty"` Memory map[string]string `json:"memory,omitempty"` Notify *NotificationPolicy `json:"notify,omitempty"` @@ -33,6 +34,11 @@ type Stage struct { Blocks map[string]string `json:"blocks,omitempty"` } +// StageResources declares one capacity-one resource, local to this store authority. +type StageResources struct { + Exclusive string `json:"exclusive"` +} + // Limits are enforced by the runner and never included in the model prompt. type StageLimits struct { MaxVisits int `json:"max_visits"` diff --git a/internal/protocol/resources_test.go b/internal/protocol/resources_test.go new file mode 100644 index 00000000..f6c8cd41 --- /dev/null +++ b/internal/protocol/resources_test.go @@ -0,0 +1,29 @@ +package protocol + +import ( + "encoding/json" + "strings" + "testing" +) + +func TestResourceWaitOlderClientCompatibility(t *testing.T) { + data, err := json.Marshal(RunSummary{ID: 1, Status: RunStatusQueued, ResourceWait: &ResourceWait{Resource: "X", Stage: "review", Sequence: 2, QueuedSince: "now"}}) + if err != nil { + t.Fatal(err) + } + var old struct { + ID int64 `json:"id"` + Status string `json:"status"` + } + if err := json.Unmarshal(data, &old); err != nil || old.Status != "queued" { + t.Fatal("old client no longer waits", old, err) + } + clear, err := json.Marshal(RunEventFrame{RunID: 1, ResourceWaitSet: true}) + if err != nil || !strings.Contains(string(clear), `"resource_wait_set":true`) { + t.Fatal("no explicit clear", string(clear), err) + } + var current RunSummary + if err := json.Unmarshal([]byte(`{"id":2,"status":"running"}`), ¤t); err != nil || current.ResourceWait != nil { + t.Fatal("missing field changed old payload", err) + } +} diff --git a/internal/protocol/runevents.go b/internal/protocol/runevents.go index f3b424fd..240ad2b1 100644 --- a/internal/protocol/runevents.go +++ b/internal/protocol/runevents.go @@ -24,8 +24,11 @@ const ( // from the frame instead of re-reading the whole list. They repeat on every // frame (pipeline identity never changes mid-run). type RunEventFrame struct { - RunID int64 `json:"run_id"` - Workspace string `json:"workspace,omitempty"` + // ResourceWaitSet makes a nil ResourceWait an explicit cache clear. + ResourceWaitSet bool `json:"resource_wait_set,omitempty"` + ResourceWait *ResourceWait `json:"resource_wait,omitempty"` + RunID int64 `json:"run_id"` + Workspace string `json:"workspace,omitempty"` // PipelineID, Name, Title and Version identify the run's pipeline and its // operator-facing title, so a new run's row can be built without a fetch. PipelineID int64 `json:"pipeline_id,omitempty"` diff --git a/internal/protocol/runs.go b/internal/protocol/runs.go index 87d94a1e..fb3f2830 100644 --- a/internal/protocol/runs.go +++ b/internal/protocol/runs.go @@ -118,9 +118,10 @@ type SessionExport struct { } type RunSummary struct { - ID int64 `json:"id"` - PipelineID int64 `json:"pipeline_id"` - Name string `json:"name"` + ResourceWait *ResourceWait `json:"resource_wait,omitempty"` + ID int64 `json:"id"` + PipelineID int64 `json:"pipeline_id"` + Name string `json:"name"` // Title is the operator-facing, bounded single-line description of what the // run is for, generated from its prompt. It is never empty for a new run; // historical rows predating the title column may omit it. @@ -145,6 +146,19 @@ type RunSummary struct { StageModels map[string]string `json:"stage_models,omitempty"` } +// ResourceWait is bounded public queue metadata. Ownership credentials stay private. +type ResourceWait struct { + Resource string `json:"resource"` + Stage string `json:"stage"` + Sequence int `json:"sequence"` + QueuedSince string `json:"queued_since"` +} + +// ResourceCleanupConfirmedKey lets a stage report failed restoration without +// pretending that tool idleness also proves the resource is ready. An explicit +// false (or malformed value) vetoes release; it never authorizes release alone. +const ResourceCleanupConfirmedKey = "resource_cleanup_confirmed" + // Label is the operator-facing name of a run: the generated Title when the // server stored one, otherwise the pipeline Name (historical rows predate // titles). It is the single definition shared by the CLI and TUI. @@ -155,6 +169,14 @@ func (r RunSummary) Label() string { return r.Name } +// StatusLabel separates resource contention from a human decision everywhere. +func (r RunSummary) StatusLabel() string { + if r.ResourceWait != nil { + return "Waiting for resource " + r.ResourceWait.Resource + } + return r.Status +} + type RunStep struct { Control *ControlEvidence `json:"control,omitempty"` FileChanges *StageFileChanges `json:"file_changes,omitempty"` diff --git a/internal/server/resource_cli_test.go b/internal/server/resource_cli_test.go new file mode 100644 index 00000000..a377dc70 --- /dev/null +++ b/internal/server/resource_cli_test.go @@ -0,0 +1,154 @@ +package server + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/json" + "fmt" + "net/http/httptest" + "os" + "path/filepath" + "reflect" + "strings" + "testing" + "time" + + "git.a-wan.net/alan/pipelines/internal/cli" + "git.a-wan.net/alan/pipelines/internal/pipeline" + "git.a-wan.net/alan/pipelines/internal/protocol" + "git.a-wan.net/alan/pipelines/internal/stage" +) + +// This is a real CLI/WebSocket/scratch-SQLite exercise with an explicitly FAKE +// executor. It never calls inference, a phone, or a deployed server. +func TestResourceScratchCLIWaitingThenProgress(t *testing.T) { + t.Setenv("PIPELINES_HARNESS_CONFIG", filepath.Join(t.TempDir(), "harnesses.json")) + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + store, err := pipeline.Open(filepath.Join(t.TempDir(), "scratch.db")) + if err != nil { + t.Fatal(err) + } + defer store.Close() + source := pipeline.Template([]string{"work"}, true) + "```resources\n{\"exclusive\":\"fake-phone\"}\n```\n" + if _, err := store.Submit(ctx, "scratch-resource", source); err != nil { + t.Fatal(err) + } + handler := Handler(ctx, store).(*runHandler) + started, release := make(chan struct{}), make(chan struct{}) + handler.runner.execute = func(ctx context.Context, _ stage.Definition, input stage.Values) (stage.Result, error) { + if input["prompt"] == "A" { + close(started) + select { + case <-release: + case <-ctx.Done(): + return stage.Result{}, ctx.Err() + } + } + return stage.Result{Outcome: "success", Output: stage.Values{"execution": "FAKE executor; no device or inference"}}, nil + } + handler.runner.events.interval = time.Millisecond + events := make(chan protocol.RunEventFrame, 64) + sub := handler.runner.events.subscribe("cli-evidence", "", nil, func(m protocol.Message) error { + if m.RunEvent != nil { + events <- *m.RunEvent + } + return nil + }) + sub.ready() + defer sub.close() + srv := httptest.NewServer(handler) + defer srv.Close() + defer func() { cancel(); WaitRuns(handler) }() + endpoint := "ws" + strings.TrimPrefix(srv.URL, "http") + "/connect" + run := func(args ...string) string { + var out bytes.Buffer + app := cli.App{Out: &out, Err: &out, In: strings.NewReader("")} + if err := app.Run(ctx, append([]string{"--url", endpoint}, args...)); err != nil { + t.Fatalf("CLI %v: %v\n%s", args, err, out.String()) + } + t.Logf("pipelines %s\n%s", strings.Join(args, " "), out.String()) + return out.String() + } + parseRun := func(output string) int64 { + var m protocol.Message + if err := json.Unmarshal([]byte(output), &m); err != nil || m.RunDetail == nil { + t.Fatal("CLI run response", output, err) + } + return m.RunDetail.ID + } + run("validate", "testdata/android-resource-draft.md", "--json") + a := parseRun(run("run", "scratch-resource", "A", "--workspace", t.TempDir(), "--yes", "--json", "--request-key", "a")) + <-started + b := parseRun(run("run", "scratch-resource", "B", "--workspace", t.TempDir(), "--yes", "--json", "--request-key", "b")) + awaitResourceEvent(t, events, b, func(e protocol.RunEventFrame) bool { return e.ResourceWait != nil }) + var detail protocol.RunDetail + if err := json.Unmarshal([]byte(run("inspect", fmt.Sprint(b), "--json")), &detail); err != nil || detail.ResourceWait == nil || detail.Status != "queued" || len(detail.Steps) != 0 { + t.Fatal("CLI did not observe queue", err) + } + if output := run("runs"); !strings.Contains(output, "Waiting for resource fake-phone") { + t.Fatal("list omitted resource status", output) + } + var snapshot *protocol.RunEventFrame + handleRunEvents(protocol.Message{Type: protocol.Subscribe, RunIDs: []int64{b}}, handler.runner.events, handler.runner, "snapshot-evidence", func(m protocol.Message) error { + if m.RunEvent != nil { + snapshot = m.RunEvent + } + return nil + }) + handler.runner.events.unsubscribeClient("snapshot-evidence") + if snapshot == nil || snapshot.ResourceWait == nil || !snapshot.ResourceWaitSet { + t.Fatal("subscription snapshot omitted resource wait") + } + close(release) + run("wait", fmt.Sprint(a), fmt.Sprint(b), "--json") + detail = protocol.RunDetail{} + if err := json.Unmarshal([]byte(run("inspect", fmt.Sprint(b), "--json")), &detail); err != nil || detail.Status != "succeeded" || detail.ResourceWait != nil || len(detail.Steps) != 1 { + t.Fatal("automatic progress not observed", err) + } + t.Log("Observed queued resource wait followed by automatic progress; FAKE executor only, no phone/inference/deployment.") +} + +func TestResourceAndroidDraftPublishedV3Provenance(t *testing.T) { + baseline, err := os.ReadFile("testdata/android-v3-published.md") + if err != nil { + t.Fatal(err) + } + if hash := fmt.Sprintf("%x", sha256.Sum256(baseline)); hash != "4a3a4b8b947eb20c115c5470e6e378f117d5fa291a7b6c8ae966a3745bc826de" { + t.Fatal("baseline is not retrieved Android v3", hash) + } + draft, err := os.ReadFile("testdata/android-resource-draft.md") + if err != nil { + t.Fatal(err) + } + old, err := pipeline.Parse(string(baseline)) + if err != nil { + t.Fatal(err) + } + updated, err := pipeline.Parse(string(draft)) + if err != nil { + t.Fatal(err) + } + if len(old.Stages) != len(updated.Stages) { + t.Fatal("draft changed graph shape") + } + for i := range old.Stages { + a, b := old.Stages[i], updated.Stages[i] + if b.Configs["provider"] != "openai" || b.Configs["model"] != a.Configs["model"] { + t.Fatal("draft changed approved model") + } + if b.Key == "ui_reviewer" { + if b.Resources == nil || b.Resources.Exclusive != "phone-47241FDAS00DLK" { + t.Fatal("reviewer resource missing") + } + } else if b.Resources != nil { + t.Fatal("non-UI stage acquired phone") + } + a.Prompt, b.Prompt = "", "" + a.Resources, b.Resources = nil, nil + if !reflect.DeepEqual(a, b) { + t.Fatal("draft changed fields outside approved role split/resource", a.Key) + } + } +} diff --git a/internal/server/resources.go b/internal/server/resources.go new file mode 100644 index 00000000..ecb5b7a5 --- /dev/null +++ b/internal/server/resources.go @@ -0,0 +1,60 @@ +package server + +import ( + "context" + "errors" + "time" + + "git.a-wan.net/alan/pipelines/internal/pipeline" + "git.a-wan.net/alan/pipelines/internal/protocol" + "git.a-wan.net/alan/pipelines/internal/stage" +) + +func (r *runner) acquireResource(ctx context.Context, visit pipeline.ResourceVisit, workspace string, completed int) (pipeline.ResourceLease, error) { + lease, ready, wait, err := r.store.RequestResource(ctx, visit) + if err != nil { + return lease, err + } + if ready != nil { + r.emitRun(protocol.RunEventFrame{RunID: visit.RunID, Workspace: workspace, Status: protocol.RunStatusQueued, CurrentStage: visit.Stage, CompletedSteps: completed, ResourceWaitSet: true, ResourceWait: wait}, false) + select { + case lease = <-ready: + r.store.AcceptResource(lease) + case <-ctx.Done(): + cleanup, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + if err := r.store.CancelResource(cleanup, visit); err != nil { + return lease, errors.Join(ctx.Err(), err) + } + return lease, ctx.Err() + } + } + if ctx.Err() != nil { + err := r.releaseResource(lease, true) // Never launched, including a concurrent grant. + return pipeline.ResourceLease{}, errors.Join(ctx.Err(), err) + } + r.emitRun(protocol.RunEventFrame{RunID: visit.RunID, Workspace: workspace, ResourceWaitSet: true}, false) + return lease, nil +} + +func (r *runner) releaseResource(lease pipeline.ResourceLease, confirmed bool) error { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + if err := r.store.SetResourceCleanup(ctx, lease, confirmed); err != nil { + return err + } + if confirmed { + return r.store.ReleaseResource(ctx, lease) + } + return nil +} + +func (r *runner) resourceQuiescent(id int64, executionErr error, panicked bool) bool { + if panicked || errors.Is(executionErr, stage.ErrStageCleanup) { + return false + } + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + sessions, err := r.store.ActiveAgentSessionsForRun(ctx, id) + return err == nil && len(sessions) == 0 +} diff --git a/internal/server/resources_test.go b/internal/server/resources_test.go new file mode 100644 index 00000000..837560eb --- /dev/null +++ b/internal/server/resources_test.go @@ -0,0 +1,327 @@ +package server + +import ( + "context" + "database/sql" + "errors" + "fmt" + "path/filepath" + "strings" + "sync/atomic" + "testing" + "time" + + "git.a-wan.net/alan/pipelines/internal/pipeline" + "git.a-wan.net/alan/pipelines/internal/protocol" + "git.a-wan.net/alan/pipelines/internal/stage" +) + +func resourceRunner(t *testing.T) (*runner, <-chan protocol.RunEventFrame, func(string, string, string) *protocol.RunDetail) { + t.Helper() + ctx, cancel := context.WithCancel(context.Background()) + s, err := pipeline.Open(filepath.Join(t.TempDir(), "resources.db")) + if err != nil { + t.Fatal(err) + } + r, err := newRunner(ctx, s) + if err != nil { + t.Fatal(err) + } + r.events = newRunEventHub() + r.events.interval = time.Millisecond + events := make(chan protocol.RunEventFrame, 128) + sub := r.events.subscribe("resource-test", "", nil, func(m protocol.Message) error { + if m.RunEvent != nil { + events <- *m.RunEvent + } + return nil + }) + sub.ready() + t.Cleanup(func() { cancel(); r.wg.Wait(); sub.close(); s.Close() }) + count := 0 + return r, events, func(resource, timeout, prompt string) *protocol.RunDetail { + count++ + source := strings.Replace(pipeline.Template([]string{"work"}, true), "timeout=1h", "timeout="+timeout, 1) + if resource != "" { + source += "```resources\n{\"exclusive\":\"" + resource + "\"}\n```\n" + } + name := fmt.Sprint("resource-", count) + if _, err := s.Submit(ctx, name, source); err != nil { + t.Fatal(err) + } + items, _ := s.List(ctx) + var pid int64 + for _, p := range items { + if p.Name == name { + pid = p.ID + } + } + d, err := r.start(protocol.Message{PipelineID: pid, Version: 1, RequestKey: name, Workspace: t.TempDir(), Input: map[string]any{"prompt": prompt}}) + if err != nil { + t.Fatal(err) + } + return d + } +} + +func awaitResourceEvent(t *testing.T, events <-chan protocol.RunEventFrame, id int64, predicate func(protocol.RunEventFrame) bool) protocol.RunEventFrame { + t.Helper() + timer := time.NewTimer(3 * time.Second) + defer timer.Stop() + for { + select { + case event := <-events: + if event.RunID == id && predicate(event) { + return event + } + case <-timer.C: + t.Fatalf("resource event missing for run %d", id) + return protocol.RunEventFrame{} + } + } +} + +func TestResourcePauseBeyondTimeoutAndAutomaticResume(t *testing.T) { + r, events, start := resourceRunner(t) + aStarted, releaseA := make(chan struct{}), make(chan struct{}) + bStarted := make(chan struct{}, 1) + var bCalls atomic.Int32 + r.execute = func(ctx context.Context, _ stage.Definition, input stage.Values) (stage.Result, error) { + switch input["prompt"] { + case "A": + close(aStarted) + select { + case <-releaseA: + case <-ctx.Done(): + return stage.Result{}, ctx.Err() + } + case "B": + bCalls.Add(1) + deadline, ok := ctx.Deadline() + if !ok || time.Until(deadline) < 15*time.Millisecond { + return stage.Result{}, errors.New("execution timeout was consumed in queue") + } + bStarted <- struct{}{} + } + return stage.Result{Outcome: "success", Output: stage.Values{}}, nil + } + a := start("X", "2s", "A") + <-aStarted + b := start("X", "30ms", "B") + awaitResourceEvent(t, events, b.ID, func(e protocol.RunEventFrame) bool { return e.ResourceWait != nil }) + <-time.After(70 * time.Millisecond) // Deliberately longer than B's whole execution timeout. + d, err := r.store.GetRun(context.Background(), b.ID) + if err != nil || d.Status != protocol.RunStatusQueued || d.ResourceWait == nil || len(d.Steps) != 0 || bCalls.Load() != 0 { + t.Fatal("queued execution consumed accounting", d, err, bCalls.Load()) + } + if _, err := r.decide(protocol.Message{RunID: b.ID, Option: "success"}); err == nil { + t.Fatal("resource wait exposed a human decision") + } + c := start("Y", "1s", "C") + awaitResourceEvent(t, events, c.ID, func(e protocol.RunEventFrame) bool { return e.Status == "succeeded" }) + select { + case <-bStarted: + t.Fatal("B ran while A held X") + default: + } + owner, err := r.store.GetRun(context.Background(), a.ID) + if err != nil || owner.Status != "running" { + t.Fatal("A did not remain owner", err) + } + close(releaseA) + awaitResourceEvent(t, events, b.ID, func(e protocol.RunEventFrame) bool { return e.Status == "succeeded" }) + r.wg.Wait() + d, err = r.store.GetRun(context.Background(), b.ID) + if err != nil || bCalls.Load() != 1 || d.ResourceWait != nil || len(d.Steps) != 1 || d.Steps[0].Control.Visit != 1 || d.Steps[0].Sequence != 1 { + t.Fatal("resume did not execute exactly one fresh visit", d, err) + } +} + +func TestResourceCancellationWaitsForOwnerCleanup(t *testing.T) { + r, events, start := resourceRunner(t) + started, cleaning, cleanupDone := make(chan struct{}), make(chan struct{}), make(chan struct{}) + var waitingCalls atomic.Int32 + r.execute = func(ctx context.Context, _ stage.Definition, input stage.Values) (stage.Result, error) { + if input["prompt"] == "owner" { + close(started) + <-ctx.Done() + close(cleaning) + <-cleanupDone + return stage.Result{}, ctx.Err() + } + waitingCalls.Add(1) + return stage.Result{Outcome: "success", Output: stage.Values{}}, nil + } + a := start("X", "2s", "owner") + <-started + b := start("X", "30ms", "waiter") + awaitResourceEvent(t, events, b.ID, func(e protocol.RunEventFrame) bool { return e.ResourceWait != nil }) + if err := r.cancel(a.ID); err != nil { + t.Fatal(err) + } + <-cleaning + select { + case <-time.After(50 * time.Millisecond): + if waitingCalls.Load() != 0 { + t.Fatal("handoff before cleanup") + } + } + close(cleanupDone) + awaitResourceEvent(t, events, b.ID, func(e protocol.RunEventFrame) bool { return e.Status == "succeeded" }) + if waitingCalls.Load() != 1 { + t.Fatal("waiter did not resume exactly once") + } +} + +func TestResourceQuarantineAndQueuedCancellation(t *testing.T) { + for _, failure := range []string{"cleanup", "panic", "restoration"} { + t.Run(failure, func(t *testing.T) { + r, events, start := resourceRunner(t) + started, release := make(chan struct{}), make(chan struct{}) + var calls atomic.Int32 + r.execute = func(ctx context.Context, _ stage.Definition, input stage.Values) (stage.Result, error) { + if input["prompt"] == "owner" { + close(started) + select { + case <-release: + case <-ctx.Done(): + } + if failure == "panic" { + panic("unknown cleanup") + } + if failure == "restoration" { + return stage.Result{Outcome: "success", Output: stage.Values{protocol.ResourceCleanupConfirmedKey: false}}, nil + } + return stage.Result{}, stage.ErrStageCleanup + } + calls.Add(1) + return stage.Result{Outcome: "success", Output: stage.Values{}}, nil + } + a := start("X", "1s", "owner") + <-started + b := start("X", "30ms", "waiter") + awaitResourceEvent(t, events, b.ID, func(e protocol.RunEventFrame) bool { return e.ResourceWait != nil }) + close(release) + awaitResourceEvent(t, events, a.ID, func(e protocol.RunEventFrame) bool { return e.Status == "failed" }) + if err := r.cancel(b.ID); err != nil { + t.Fatal(err) + } + awaitResourceEvent(t, events, b.ID, func(e protocol.RunEventFrame) bool { + return e.Status == "canceled" && e.ResourceWaitSet && e.ResourceWait == nil + }) + r.wg.Wait() + if calls.Load() != 0 { + t.Fatal("quarantined resource handed off") + } + exists, err := r.store.HasResourceOwner(context.Background(), a.ID) + if err != nil || !exists { + t.Fatal("quarantine evidence lost", err) + } + d, err := r.store.GetRun(context.Background(), b.ID) + if err != nil || d.ResourceWait != nil || len(d.Steps) != 0 { + t.Fatal("canceled queue consumed step", err) + } + }) + } +} + +func TestResourceEventClearCoalescing(t *testing.T) { + frame := protocol.RunEventFrame{ResourceWaitSet: true, ResourceWait: &protocol.ResourceWait{Resource: "X"}} + mergeRunEventFrame(&frame, &protocol.RunEventFrame{ResourceWaitSet: true}) + if !frame.ResourceWaitSet || frame.ResourceWait != nil { + t.Fatal("coalescer retained stale resource wait") + } +} + +func TestResourceReleaseOnBeginAndPersistenceFailure(t *testing.T) { + for _, failure := range []string{"begin", "persist"} { + t.Run(failure, func(t *testing.T) { + r, events, start := resourceRunner(t) + db, err := sql.Open("sqlite", filepath.Join(r.store.Directory(), "resources.db")) + if err != nil { + t.Fatal(err) + } + defer db.Close() + trigger := `CREATE TRIGGER resource_fail BEFORE INSERT ON run_steps WHEN json_extract(NEW.input_json,'$.prompt')='fail' BEGIN SELECT RAISE(FAIL,'test BeginStep failure'); END` + if failure == "persist" { + trigger = `CREATE TRIGGER resource_fail BEFORE UPDATE OF output_json ON run_steps WHEN json_extract(OLD.input_json,'$.prompt')='fail' BEGIN SELECT RAISE(FAIL,'test persistence failure'); END` + } + if _, err := db.Exec(trigger); err != nil { + t.Fatal(err) + } + var calls atomic.Int32 + r.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) { + calls.Add(1) + return stage.Result{Outcome: "success", Output: stage.Values{}}, nil + } + a := start("X", "1s", "fail") + awaitResourceEvent(t, events, a.ID, func(e protocol.RunEventFrame) bool { return e.Status == "failed" }) + r.wg.Wait() + if exists, err := r.store.HasResourceOwner(context.Background(), a.ID); err != nil || exists { + t.Fatal("safe failure stranded resource", err) + } + if failure == "begin" && calls.Load() != 0 { + t.Fatal("BeginStep failure executed work") + } + b := start("X", "1s", "pass") + awaitResourceEvent(t, events, b.ID, func(e protocol.RunEventFrame) bool { return e.Status == "succeeded" }) + }) + } +} + +func TestResourceCleanupQueryFailureQuarantines(t *testing.T) { + r, events, start := resourceRunner(t) + r.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) { + if err := r.store.Close(); err != nil { + return stage.Result{}, err + } + return stage.Result{Outcome: "success", Output: stage.Values{}}, nil + } + a := start("X", "1s", "query-failure") + awaitResourceEvent(t, events, a.ID, func(e protocol.RunEventFrame) bool { return e.Status == "failed" }) + r.wg.Wait() + reopened, err := pipeline.Open(filepath.Join(r.store.Directory(), "resources.db")) + if err != nil { + t.Fatal(err) + } + defer reopened.Close() + if err := reopened.RecoverRuns(context.Background()); err != nil { + t.Fatal(err) + } + if exists, err := reopened.HasResourceOwner(context.Background(), a.ID); err != nil || !exists { + t.Fatal("query failure released uncertain owner", err) + } +} + +func TestResourceSkippedVisitDoesNotAcquire(t *testing.T) { + r, events, _ := resourceRunner(t) + ctx := context.Background() + source := strings.Replace(pipeline.Template([]string{"review"}, true), "success -> #success", "success -> #success\nskipped -> #success", 1) + "```resources\n{\"exclusive\":\"X\"}\n```\n" + if _, err := r.store.Submit(ctx, "skip", source); err != nil { + t.Fatal(err) + } + items, _ := r.store.List(ctx) + d, _ := r.store.Get(ctx, items[0].ID, 1) + owner, _, err := r.store.CreateRun(ctx, d, "held", t.TempDir(), false, nil, nil) + if err != nil { + t.Fatal(err) + } + lease, ch, _, err := r.store.RequestResource(ctx, pipeline.ResourceVisit{Resource: "X", RunID: owner, Stage: "review", Sequence: 1}) + if err != nil || ch != nil { + t.Fatal("owner fixture failed", err) + } + defer r.releaseResource(lease, true) + r.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) { + t.Error("skipped stage executed") + return stage.Result{}, nil + } + run, err := r.start(protocol.Message{PipelineID: d.ID, Version: 1, RequestKey: "skip", Workspace: t.TempDir(), Input: map[string]any{"prompt": "no UI"}}) + if err != nil { + t.Fatal(err) + } + awaitResourceEvent(t, events, run.ID, func(e protocol.RunEventFrame) bool { return e.Status == "succeeded" }) + detail, err := r.store.GetRun(ctx, run.ID) + if err != nil || len(detail.Steps) != 1 || detail.Steps[0].Outcome != "skipped" { + t.Fatal("skip not preserved", detail, err) + } +} diff --git a/internal/server/retry_test.go b/internal/server/retry_test.go index acde318d..152201ce 100644 --- a/internal/server/retry_test.go +++ b/internal/server/retry_test.go @@ -29,6 +29,7 @@ func TestRetryCreatesFreshOpenCodeSessionAndRetainsFailedSession(t *testing.T) { if [ "$1" = "--help" ]; then echo ' api'; exit 0; fi printf '%%s %%s %%s\n' "$PWD" "$2" "$3" >> %q case "$3" in + /api/session/active) echo '{"data":{}}' ;; /api/session) n=0; if [ -f %q ]; then n=$(cat %q); fi n=$((n+1)); echo "$n" > %q diff --git a/internal/server/runevents.go b/internal/server/runevents.go index e35d683d..87cac293 100644 --- a/internal/server/runevents.go +++ b/internal/server/runevents.go @@ -424,6 +424,9 @@ func (s *runSubscription) close() { } func mergeRunEventFrame(into, in *protocol.RunEventFrame) { + if in.ResourceWaitSet { + into.ResourceWaitSet, into.ResourceWait = true, in.ResourceWait + } if in.Workspace != "" { into.Workspace = in.Workspace } @@ -500,18 +503,20 @@ func handleRunEvents(request protocol.Message, events *runEventHub, runs *runner continue } frame := protocol.RunEventFrame{ - RunID: summary.ID, - Workspace: summary.Workspace, - PipelineID: summary.PipelineID, - Name: summary.Name, - Title: summary.Title, - Version: summary.Version, - Status: summary.Status, - Outcome: summary.Outcome, - CurrentStage: summary.CurrentStage, - CompletedSteps: summary.CompletedSteps, - CreatedAt: summary.CreatedAt, - FinishedAt: summary.FinishedAt, + ResourceWaitSet: true, + ResourceWait: summary.ResourceWait, + RunID: summary.ID, + Workspace: summary.Workspace, + PipelineID: summary.PipelineID, + Name: summary.Name, + Title: summary.Title, + Version: summary.Version, + Status: summary.Status, + Outcome: summary.Outcome, + CurrentStage: summary.CurrentStage, + CompletedSteps: summary.CompletedSteps, + CreatedAt: summary.CreatedAt, + FinishedAt: summary.FinishedAt, } if !sub.writeSnapshot(protocol.Message{Type: protocol.RunEvent, RunEvent: &frame}) { // The connection write failed; the subscription is closed and the diff --git a/internal/server/runner.go b/internal/server/runner.go index 11471617..efaa8787 100644 --- a/internal/server/runner.go +++ b/internal/server/runner.go @@ -78,12 +78,19 @@ func newRunner(ctx context.Context, s *pipeline.Store) (*runner, error) { if owner.Backend == "pi" { err = stage.InterruptPi(stop, owner.Reference) } else { - err = (stage.OpenCodeClient{Backend: owner.Backend, Directory: owner.Directory, Reference: owner.Reference}).Interrupt(stop, owner.ID) + err = (stage.OpenCodeClient{Backend: owner.Backend, Directory: owner.Directory, Reference: owner.Reference}).InterruptAndConfirm(stop, owner.ID) } cancel() if err != nil { + if resource, checkErr := s.HasResourceOwner(ctx, owner.RunID); checkErr == nil && resource { + log.Printf("resource owner run=%d remains quarantined: cleanup not confirmed", owner.RunID) + continue + } return nil, fmt.Errorf("cannot interrupt previously active agent %s: %w", owner.ID, err) } + if err := s.ConfirmRecoveredSession(ctx, owner); err != nil { + return nil, err + } } if err := s.RecoverRuns(ctx); err != nil { return nil, err @@ -213,6 +220,9 @@ func (r *runner) start(request protocol.Message, retryOf ...int64) (*protocol.Ru // workspace) must not block new work. for _, owner := range owners { if _, ok := r.active[owner.RunID]; !ok { + if resource, checkErr := r.store.HasResourceOwner(r.ctx, owner.RunID); checkErr == nil && resource && owner.Directory != workspace { + continue + } r.mu.Unlock() return nil, fmt.Errorf("an agent may still be active; restart recovery must interrupt it before new work") } @@ -775,13 +785,14 @@ func (r *runner) finishRunState(id int64, name, workspace, status, outcome, diag // Push the terminal delta before the attention notification, so a slow ntfy // never delays the live update the app folds. r.emitRun(protocol.RunEventFrame{ - RunID: id, - Workspace: workspace, - Status: status, - Outcome: outcome, - CurrentStage: currentStage, - CompletedSteps: completed, - FinishedAt: time.Now().UTC().Format(time.RFC3339), + RunID: id, + Workspace: workspace, + Status: status, + ResourceWaitSet: true, + Outcome: outcome, + CurrentStage: currentStage, + CompletedSteps: completed, + FinishedAt: time.Now().UTC().Format(time.RFC3339), }, true) r.notifyRun(id, name, status, sequence, currentStage, diagnostic) // The run is terminal: nothing more can notify for it. @@ -849,6 +860,21 @@ func (r *runner) finishStage(id int64, name, workspace string, selected protocol } func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, name, workspace string, request protocol.Message, state *runState) { + var held *pipeline.ResourceLease + resourceQuiet := true + release := func() error { + if held != nil { + err := r.releaseResource(*held, resourceQuiet) + held = nil + return err + } + return nil + } + defer func() { + if err := release(); err != nil { + log.Printf("resource cleanup run=%d: %v", id, err) + } + }() finish := func(status, outcome, diagnostic string) { r.finishRunState(id, name, workspace, status, outcome, diagnostic, state.completed, state.nextSequence-1, state.key, state.input) } @@ -877,7 +903,6 @@ func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, na effectiveModels = resolved } for sequence := state.nextSequence; sequence <= 30; sequence++ { - state.nextSequence = sequence + 1 if ctx.Err() != nil { finish("canceled", "", ctx.Err().Error()) return @@ -909,24 +934,42 @@ func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, na } } isHuman := stage.EffectiveType(def.Type) == stage.HumanType - state.visits[key]++ maxVisits := 5 if selected.Limits != nil { maxVisits = selected.Limits.MaxVisits } - // A human stage pauses for a person; neither the global visit budget nor - // the per-stage max_visits may kill it. The shared 30-step run budget - // still bounds every run. + // Validate the next visit without consuming it. An already-exhausted + // stage must not queue for a resource it can never execute. if !isHuman { - if state.visits[key] > 5 { + if state.visits[key]+1 > 5 { finish("failed", "error", "stage visit budget exceeded") return } - if state.visits[key] > maxVisits { + if state.visits[key]+1 > maxVisits { finish("failed", "error", fmt.Sprintf("stage %s visit limit reached (%d)", key, maxVisits)) return } } + _, declaresSkip := def.Edges["skipped"] + skip := !isHuman && declaresSkip && !state.uiSurface + if selected.Resources != nil && !skip { + if isHuman || !pipeline.ValidResourceKey(selected.Resources.Exclusive) { + finish("failed", "error", "invalid persisted resource declaration") + return + } + lease, err := r.acquireResource(ctx, pipeline.ResourceVisit{Resource: selected.Resources.Exclusive, RunID: id, Stage: key, Sequence: sequence}, workspace, state.completed) + if err != nil { + if ctx.Err() != nil { + finish("canceled", "", err.Error()) + } else { + finish("failed", "error", err.Error()) + } + return + } + held, resourceQuiet = &lease, true + } + state.nextSequence = sequence + 1 + state.visits[key]++ // Publish the run's server-captured changed paths into the input memory // value before resolving bindings, so a stage selecting // input.server_changed_files always resolves (an empty list when nothing @@ -948,12 +991,13 @@ func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, na } // queued -> running: report the new current stage and the step start. r.emitRun(protocol.RunEventFrame{ - RunID: id, - Workspace: workspace, - Status: protocol.RunStatusRunning, - CurrentStage: key, - CompletedSteps: state.completed, - Step: &protocol.RunStepDelta{Sequence: sequence, Stage: key, Status: protocol.StepStatusRunning}, + RunID: id, + Workspace: workspace, + Status: protocol.RunStatusRunning, + CurrentStage: key, + ResourceWaitSet: true, + CompletedSteps: state.completed, + Step: &protocol.RunStepDelta{Sequence: sequence, Stage: key, Status: protocol.StepStatusRunning}, }, false) if isHuman { // A human step is O(1): one BeginStep plus one MarkWaiting, no @@ -987,7 +1031,7 @@ func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, na // the model call and its capture work, record the step as completed and // route along the declared edge. Only a declared route is honoured, and no // other stage or outcome routing changes. - if _, declaresSkip := def.Edges["skipped"]; declaresSkip && !state.uiSurface { + if skip { reason := skipReasonLine(state) step := protocol.RunStep{ Sequence: sequence, @@ -1068,6 +1112,9 @@ func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, na snapshotCancel() var stageSession string panicked := false + if held != nil { + resourceQuiet = false + } func() { defer func() { if recover() != nil { @@ -1075,6 +1122,10 @@ func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, na err = fmt.Errorf("stage execution panicked") } }() + if ctx.Err() != nil { + err = ctx.Err() + return + } if r.execute != nil { timeout, _ := stage.StageTimeout(def) executeCtx, cancel := context.WithTimeout(stageCtx, timeout) @@ -1119,6 +1170,12 @@ func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, na result, err = stage.DefaultRegistry().Run(ctx, def, input) } }() + if held != nil { + resourceQuiet = r.resourceQuiescent(id, err, panicked) + if confirmed, reported := result.Output[protocol.ResourceCleanupConfirmedKey]; reported && confirmed != true { + resourceQuiet = false + } + } // Capture after cleanup, even when the stage's context was canceled. snapshotCtx, snapshotCancel = context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second) after := filediff.Capture(snapshotCtx, workspace, before) @@ -1236,6 +1293,14 @@ func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, na return } } + if held != nil && !resourceQuiet { + finish("failed", "error", "resource cleanup/restoration not confirmed; ownership quarantined pending inspection") + return + } + if err := release(); err != nil { + finish("failed", "error", "resource release failed; retained ownership needs reconciliation: "+err.Error()) + return + } if ctx.Err() != nil { finish("canceled", "", ctx.Err().Error()) return diff --git a/internal/server/testdata/android-resource-draft.md b/internal/server/testdata/android-resource-draft.md new file mode 100644 index 00000000..e822ce78 --- /dev/null +++ b/internal/server/testdata/android-resource-draft.md @@ -0,0 +1,312 @@ +# stage implementer + +```limits +{"max_visits":5,"on":["blocked_retryable"]} +``` + + +```memory +{"task": "input.prompt", "code_findings": "reviewer.findings?", "ui_findings": "ui_reviewer.findings?"} +``` + +```output +{ task, acceptance_checks, changed_files, implementation_summary, new_helpers } +``` + +```prompt +Implement input.prompt in the run directory using the repository's actual languages and +file types; create files as needed. + +Read android/AGENTS.md first. It is the map of this project - how to build, test, run and capture here, where +the pieces live, and the conventions that apply. Do not re-derive what it tells you and do not go +looking for what it already names; if it is wrong or out of date, say so in your output. + +Read the surrounding code before editing so you extend the existing helpers and patterns rather than +duplicating them. Keep each change as small as the task allows and follow the repository's own +conventions and error handling. + +This is the Android app. Build and test it with the project's own Gradle invocation as the map +gives it, and treat the map's geometry tokens as the only source of dimensions - a UI change that +adds a raw value fails the repository's own check. Perform code/build/unit tests only; +no adb, install, capture, device-setting changes or restoring main's build. The UI reviewer +alone verifies rendered surfaces under its declared runner resource lease. + +Record the acceptance checks in acceptance_checks, the actual changes in changed_files, and any new +shared helper in new_helpers. Verify with the checks this repository uses - as the map gives them - +and report the exact commands and observed output. Address any concrete findings the runner supplies +and note the actual fixes. Return blocked if the required scope or tools are unavailable. Do not +commit or push; the committer stage lands the work. +``` + +```tools +read +glob +grep +edit +shell +``` + +```configs +provider=openai +model=gpt-5.6-terra +retries=1 +timeout=30m +``` + + +```edges +success -> reviewer +blocked_retryable -> implementer +blocked -> #failure +error -> #failure +``` + + +# stage reviewer + +```limits +{"max_visits": 3, "on": ["rejected"]} +``` + +```memory +{"task": "input.prompt", "acceptance_checks": "implementer.acceptance_checks", "changed_files": "implementer.changed_files", "implementation": "implementer.implementation_summary?"} +``` + + +```output +{ summary, changed_files, verification_evidence, duplication_review, performance_review, findings } +``` + +```prompt +Verify with builds, unit tests and source review as android/AGENTS.md gives them. +Do not use adb, install, capture, change device settings or restore main's build. +Device verification belongs exclusively to the UI reviewer under its runner resource lease. +Enforce the map's geometry-token rule: a raw dimension outside the token file is a must-fix finding. + +Review the actual changes against the task in input.prompt and the acceptance +checks the implementer recorded. Run the repository's relevant verification checks +with the stage tools (for example build, lint, tests or type checks) and report the +exact commands and observed results in verification_evidence. + +Independently assess code quality: +- Duplication: search the run directory for repeated or near-identical functions, + blocks and copy-pasted logic introduced by the change; identify code that should be + shared and record concrete locations in duplication_review. +- Performance and resources: flag redundant computation, unnecessary dependencies, + and repeated I/O or database/filesystem work that wastes time, CPU or memory; + record it in performance_review. +- Workarounds: flag hacks, suppressed errors and shortcuts that trade reliability for + speed in findings. + +Keep verification proportionate to the change: exercise the files and screens the diff +actually touches, capture at most one screenshot per changed surface plus one regression +screen, and do not repeat sweeps or measurements that an earlier visit already recorded. + +Return rejected ONLY when at least one must-fix finding exists: a defect that blocks +acceptance and that you can locate concretely (file and line), with evidence and a +remedy. Do not return rejected for advisory-only, cosmetic or nit items. When there +is no must-fix finding but you have non-blocking notes (duplication, performance or +workaround observations), record them in findings and return advisory, which +continues the run without another implementer visit. Return success only when the +observed checks pass and you have nothing further to record. Do not edit files or +claim unexecuted checks. +``` + +```tools +read +glob +grep +shell +``` + +```configs +provider=openai +model=gpt-5.6-luna +retries=1 +timeout=45m +``` + + +```edges +success -> ui_reviewer +advisory -> ui_reviewer +rejected -> implementer +error -> #failure +``` + + +# stage ui_reviewer + +```resources +{"exclusive":"phone-47241FDAS00DLK"} +``` + +```limits +{"max_visits": 3, "on": ["rejected"]} +``` + +```memory +{"task": "input.prompt", "acceptance_checks": "implementer.acceptance_checks", "changed_files": "implementer.changed_files", "implementation": "implementer.implementation_summary?"} +``` + + +```output +{ summary, changed_files, ui_applicable, screenshots, ui_review, platform_conventions, findings } +``` + +```prompt +Decide whether this change touches a user-facing UI. If it does not (a backend, +library, CLI, config or data-only change with no rendered surface), set +ui_applicable=false, explain briefly in summary, and return skipped. Do not invent +screenshots or visual work. + +When the change does affect a user-facing UI, verify it visually: +Read android/AGENTS.md first. You alone perform phone work while the runner holds +this stage's declared resource lease. Use controlled fixtures for changed states, +including human-decision sheets, on a scratch server/database and throwaway session. +Never mutate an operator session or restart deployed services. Record original +settings and connection/workspace; use the configured authoritative endpoint, not a +hardcoded scratch default. Keep every shell/ADB operation foreground and awaited. +Restore original settings and main's build before successful return, under the same +lease. Report capture/restoration failures explicitly; do not approve incomplete +verification or advertise readiness when cleanup/restoration is uncertain. +Include resource_cleanup_confirmed in output: true only after confirmed restoration; +false on any uncertainty (a release veto that quarantines ownership even if tools are idle). +- Build and launch the application the way the repository actually does it, then + capture screenshots of every relevant screen, state and viewport (default, empty, + loading, error, long content, small and large sizes) with the target platform's + screenshot tooling through shell. Prefer a headless or scripted capture when the + project supports one. Save the images under the run directory and list the exact + capture commands and image paths in screenshots. +- Read the captured images and review them as a UI/UX designer: layout, spacing, + alignment, typographic hierarchy, color and contrast, iconography, interaction + and focus states, loading/empty/error states, accessibility, and consistency. +- Judge the UI against established industry patterns and, above all, the native + conventions of the platform the app targets (operating system, toolkit or web + standards). Record the platform you reviewed against in platform_conventions, + and any deviations and concrete fixes in findings. Infer the target platform from + the task and the repository's actual UI toolkit when it is unstated. +- Approve only when the screenshots show the task and the acceptance checks are met + and the UI is correct, consistent and native to the target platform; otherwise + return rejected with specific, actionable findings the implementer can fix. + +Keep the capture pass proportionate: capture each changed state once (for example one +collapsed and one expanded section, one run state, one stuck header) instead of every +screen at multiple viewports or pan directions, and reuse what this run already captured +rather than re-doing it. + +If the change is UI-facing but screenshots genuinely cannot be captured +(for example no runnable UI or no display), return blocked rather than approving. +Do not edit files, and do not approve a UI you did not actually see. + +Return rejected only for a must-fix visual defect with a concrete location, evidence +and remedy; record non-blocking polish, duplication or convention notes in findings +and return advisory, which continues the run without another implementer visit. +``` + +```tools +read +glob +grep +shell +``` + +```configs +provider=openai +model=gpt-5.6-luna +retries=1 +timeout=45m +``` + + +```edges +skipped -> committer +approved -> committer +advisory -> committer +rejected -> implementer +blocked -> #failure +error -> #failure +``` + +# stage committer + +```memory +{"task": "input.prompt", "changed_files": "implementer.changed_files", "review": "reviewer.summary?", "ui_review": "ui_reviewer.summary?"} +``` + +```output +{ commit, files, conflicts_resolved, summary } +``` + +```prompt +Commit the work this run produced, and resolve any conflict that is present. This stage +runs after the reviews have approved and is the only stage allowed to commit. + +Before committing, check that the change is reflected where it belongs: the packaged +templates, the packaged +authoring skill references. Update anything the change makes stale, and state in the +commit body what you checked, saying explicitly when none of them needs updating. + +Stage exactly the files this run changed - the paths in changed_files that still exist - +using the repository's own version control commands, and commit them as one commit whose +subject says what the change does and whose body summarises it. Do not stage unrelated +files, generated or ignored files, or anything another change owns. Do not invent work +that was not done; describe what is actually in the diff. + +If the workspace is in a conflicted state - unmerged paths, or a merge, rebase or +cherry-pick left unfinished - resolve each conflict: keep both intents where they are +compatible, and choose the result that satisfies the task where they are not, then +complete the operation so the tree is clean and the commit is finished. Never commit +conflict markers, and never discard the other side's work silently; record what you +resolved. + +Do not push, tag, revert, or rewrite unrelated history. If there is nothing to commit, +report that instead of creating an empty commit. Report the commit identifier, the files +committed, whatever you resolved, and anything you deliberately left uncommitted. + + +Finally, bring this branch up to date with main, in this same stage. After committing the work +above, merge main into this branch here, resolve any conflict here - keeping both intents where they +are compatible, and choosing the result that satisfies this task where they are not - run the +repository's checks again after resolving, and commit the completed merge with a message naming what +was combined. The point is that this branch can later be merged back without conflict: leave the +tree clean and the merge finished. Never leave conflict markers, never discard the other side's work +silently. If the merge genuinely cannot be completed, say so with the path and the reason instead of +leaving the merge half-finished. + +Then land it: this branch is the work and main is where it belongs, so merge this branch into main +as the last act of this stage. Find the repository's main working tree with `git worktree list` (the +entry checked out on main, not this one) and merge this branch into it there with `--no-ff`, with a +message saying what the change does. Two lanes can land at the same moment, so take a lock first - +`mkdir` a lock directory beside main's `git` directory in a bounded retry loop, and remove it when +done - and retry briefly if another lane holds it. Then verify with +`git merge-base --is-ancestor main` and report the commit main now has. If main cannot +be updated, say exactly why and leave this branch committed and clean rather than leaving main +half-merged. There is no separate merge pipeline and nothing to hand this to: if it cannot be landed +here, it is reported here. + + +Do not edit README or other shared documentation files: an item records what it changed and which documentation is now stale in its own commit body, and the shared documents are folded by a single writer. + +``` + +```tools +read +glob +grep +edit +shell +``` + +```configs +provider=openai +model=gpt-5.6-terra +retries=1 +timeout=30m +``` + +```edges +success -> #success +blocked -> #failure +error -> #failure +``` diff --git a/internal/server/testdata/android-v2-published.md b/internal/server/testdata/android-v2-published.md new file mode 100644 index 00000000..43bf14a9 --- /dev/null +++ b/internal/server/testdata/android-v2-published.md @@ -0,0 +1,296 @@ +# stage implementer + +```limits +{"max_visits":5,"on":["blocked_retryable"]} +``` + + +```memory +{"task": "input.prompt", "code_findings": "reviewer.findings?", "ui_findings": "ui_reviewer.findings?"} +``` + +```output +{ task, acceptance_checks, changed_files, implementation_summary, new_helpers } +``` + +```prompt +Implement input.prompt in the run directory using the repository's actual languages and +file types; create files as needed. + +Read android/AGENTS.md first. It is the map of this project - how to build, test, run and capture here, where +the pieces live, and the conventions that apply. Do not re-derive what it tells you and do not go +looking for what it already names; if it is wrong or out of date, say so in your output. + +Read the surrounding code before editing so you extend the existing helpers and patterns rather than +duplicating them. Keep each change as small as the task allows and follow the repository's own +conventions and error handling. + +This is the Android app. Build and test it with the project's own Gradle invocation as the map +gives it, and treat the map's geometry tokens as the only source of dimensions - a UI change that +adds a raw value fails the repository's own check. A change with a rendered surface is verified on +the device the map names (never an emulator), capturing what the map says to capture. + +Record the acceptance checks in acceptance_checks, the actual changes in changed_files, and any new +shared helper in new_helpers. Verify with the checks this repository uses - as the map gives them - +and report the exact commands and observed output. Address any concrete findings the runner supplies +and note the actual fixes. Return blocked if the required scope or tools are unavailable. Do not +commit or push; the committer stage lands the work. +``` + +```tools +read +glob +grep +edit +shell +``` + +```configs +provider=openai +model=gpt-6.1-sol-fast +retries=1 +timeout=30m +``` + + +```edges +success -> reviewer +blocked_retryable -> implementer +blocked -> #failure +error -> #failure +``` + + +# stage reviewer + +```limits +{"max_visits": 3, "on": ["rejected"]} +``` + +```memory +{"task": "input.prompt", "acceptance_checks": "implementer.acceptance_checks", "changed_files": "implementer.changed_files", "implementation": "implementer.implementation_summary?"} +``` + + +```output +{ summary, changed_files, verification_evidence, duplication_review, performance_review, findings } +``` + +```prompt +Verify with the build and the unit tests as android/AGENTS.md gives them, and for a +change with a rendered surface check the captures against the running app on the device the map +names. Enforce the map's geometry-token rule: a raw dimension outside the token file is a must-fix +finding. + +Review the actual changes against the task in input.prompt and the acceptance +checks the implementer recorded. Run the repository's relevant verification checks +with the stage tools (for example build, lint, tests or type checks) and report the +exact commands and observed results in verification_evidence. + +Independently assess code quality: +- Duplication: search the run directory for repeated or near-identical functions, + blocks and copy-pasted logic introduced by the change; identify code that should be + shared and record concrete locations in duplication_review. +- Performance and resources: flag redundant computation, unnecessary dependencies, + and repeated I/O or database/filesystem work that wastes time, CPU or memory; + record it in performance_review. +- Workarounds: flag hacks, suppressed errors and shortcuts that trade reliability for + speed in findings. + +Keep verification proportionate to the change: exercise the files and screens the diff +actually touches, capture at most one screenshot per changed surface plus one regression +screen, and do not repeat sweeps or measurements that an earlier visit already recorded. + +Return rejected ONLY when at least one must-fix finding exists: a defect that blocks +acceptance and that you can locate concretely (file and line), with evidence and a +remedy. Do not return rejected for advisory-only, cosmetic or nit items. When there +is no must-fix finding but you have non-blocking notes (duplication, performance or +workaround observations), record them in findings and return advisory, which +continues the run without another implementer visit. Return success only when the +observed checks pass and you have nothing further to record. Do not edit files or +claim unexecuted checks. +``` + +```tools +read +glob +grep +shell +``` + +```configs +provider=openai +model=gpt-6.1-sol-fast +retries=1 +timeout=45m +``` + + +```edges +success -> ui_reviewer +advisory -> ui_reviewer +rejected -> implementer +error -> #failure +``` + + +# stage ui_reviewer + +```limits +{"max_visits": 3, "on": ["rejected"]} +``` + +```memory +{"task": "input.prompt", "acceptance_checks": "implementer.acceptance_checks", "changed_files": "implementer.changed_files", "implementation": "implementer.implementation_summary?"} +``` + + +```output +{ summary, changed_files, ui_applicable, screenshots, ui_review, platform_conventions, findings } +``` + +```prompt +Decide whether this change touches a user-facing UI. If it does not (a backend, +library, CLI, config or data-only change with no rendered surface), set +ui_applicable=false, explain briefly in summary, and return skipped. Do not invent +screenshots or visual work. + +When the change does affect a user-facing UI, verify it visually: +- Build and launch the application the way the repository actually does it, then + capture screenshots of every relevant screen, state and viewport (default, empty, + loading, error, long content, small and large sizes) with the target platform's + screenshot tooling through shell. Prefer a headless or scripted capture when the + project supports one. Save the images under the run directory and list the exact + capture commands and image paths in screenshots. +- Read the captured images and review them as a UI/UX designer: layout, spacing, + alignment, typographic hierarchy, color and contrast, iconography, interaction + and focus states, loading/empty/error states, accessibility, and consistency. +- Judge the UI against established industry patterns and, above all, the native + conventions of the platform the app targets (operating system, toolkit or web + standards). Record the platform you reviewed against in platform_conventions, + and any deviations and concrete fixes in findings. Infer the target platform from + the task and the repository's actual UI toolkit when it is unstated. +- Approve only when the screenshots show the task and the acceptance checks are met + and the UI is correct, consistent and native to the target platform; otherwise + return rejected with specific, actionable findings the implementer can fix. + +Keep the capture pass proportionate: capture each changed state once (for example one +collapsed and one expanded section, one run state, one stuck header) instead of every +screen at multiple viewports or pan directions, and reuse what this run already captured +rather than re-doing it. + +If the change is UI-facing but screenshots genuinely cannot be captured +(for example no runnable UI or no display), return blocked rather than approving. +Do not edit files, and do not approve a UI you did not actually see. + +Return rejected only for a must-fix visual defect with a concrete location, evidence +and remedy; record non-blocking polish, duplication or convention notes in findings +and return advisory, which continues the run without another implementer visit. +``` + +```tools +read +glob +grep +shell +``` + +```configs +provider=openai +model=gpt-6.1-sol-fast +retries=1 +timeout=45m +``` + + +```edges +skipped -> committer +approved -> committer +advisory -> committer +rejected -> implementer +blocked -> #failure +error -> #failure +``` + +# stage committer + +```memory +{"task": "input.prompt", "changed_files": "implementer.changed_files", "review": "reviewer.summary?", "ui_review": "ui_reviewer.summary?"} +``` + +```output +{ commit, files, conflicts_resolved, summary } +``` + +```prompt +Commit the work this run produced, and resolve any conflict that is present. This stage +runs after the reviews have approved and is the only stage allowed to commit. + +Before committing, check that the change is reflected where it belongs: the packaged +templates, the packaged +authoring skill references. Update anything the change makes stale, and state in the +commit body what you checked, saying explicitly when none of them needs updating. + +Stage exactly the files this run changed - the paths in changed_files that still exist - +using the repository's own version control commands, and commit them as one commit whose +subject says what the change does and whose body summarises it. Do not stage unrelated +files, generated or ignored files, or anything another change owns. Do not invent work +that was not done; describe what is actually in the diff. + +If the workspace is in a conflicted state - unmerged paths, or a merge, rebase or +cherry-pick left unfinished - resolve each conflict: keep both intents where they are +compatible, and choose the result that satisfies the task where they are not, then +complete the operation so the tree is clean and the commit is finished. Never commit +conflict markers, and never discard the other side's work silently; record what you +resolved. + +Do not push, tag, revert, or rewrite unrelated history. If there is nothing to commit, +report that instead of creating an empty commit. Report the commit identifier, the files +committed, whatever you resolved, and anything you deliberately left uncommitted. + + +Finally, bring this branch up to date with main, in this same stage. After committing the work +above, merge main into this branch here, resolve any conflict here - keeping both intents where they +are compatible, and choosing the result that satisfies this task where they are not - run the +repository's checks again after resolving, and commit the completed merge with a message naming what +was combined. The point is that this branch can later be merged back without conflict: leave the +tree clean and the merge finished. Never leave conflict markers, never discard the other side's work +silently. If the merge genuinely cannot be completed, say so with the path and the reason instead of +leaving the merge half-finished. + +Then land it: this branch is the work and main is where it belongs, so merge this branch into main +as the last act of this stage. Find the repository's main working tree with `git worktree list` (the +entry checked out on main, not this one) and merge this branch into it there with `--no-ff`, with a +message saying what the change does. Two lanes can land at the same moment, so take a lock first - +`mkdir` a lock directory beside main's `git` directory in a bounded retry loop, and remove it when +done - and retry briefly if another lane holds it. Then verify with +`git merge-base --is-ancestor main` and report the commit main now has. If main cannot +be updated, say exactly why and leave this branch committed and clean rather than leaving main +half-merged. There is no separate merge pipeline and nothing to hand this to: if it cannot be landed +here, it is reported here. + + +Do not edit README or other shared documentation files: an item records what it changed and which documentation is now stale in its own commit body, and the shared documents are folded by a single writer. + +``` + +```tools +read +glob +grep +edit +shell +``` + +```configs +provider=openai +model=gpt-6.1-sol-fast +retries=1 +timeout=30m +``` + +```edges +success -> #success +blocked -> #failure +error -> #failure +``` diff --git a/internal/server/testdata/android-v3-published.md b/internal/server/testdata/android-v3-published.md new file mode 100644 index 00000000..344e537b --- /dev/null +++ b/internal/server/testdata/android-v3-published.md @@ -0,0 +1,296 @@ +# stage implementer + +```limits +{"max_visits":5,"on":["blocked_retryable"]} +``` + + +```memory +{"task": "input.prompt", "code_findings": "reviewer.findings?", "ui_findings": "ui_reviewer.findings?"} +``` + +```output +{ task, acceptance_checks, changed_files, implementation_summary, new_helpers } +``` + +```prompt +Implement input.prompt in the run directory using the repository's actual languages and +file types; create files as needed. + +Read android/AGENTS.md first. It is the map of this project - how to build, test, run and capture here, where +the pieces live, and the conventions that apply. Do not re-derive what it tells you and do not go +looking for what it already names; if it is wrong or out of date, say so in your output. + +Read the surrounding code before editing so you extend the existing helpers and patterns rather than +duplicating them. Keep each change as small as the task allows and follow the repository's own +conventions and error handling. + +This is the Android app. Build and test it with the project's own Gradle invocation as the map +gives it, and treat the map's geometry tokens as the only source of dimensions - a UI change that +adds a raw value fails the repository's own check. A change with a rendered surface is verified on +the device the map names (never an emulator), capturing what the map says to capture. + +Record the acceptance checks in acceptance_checks, the actual changes in changed_files, and any new +shared helper in new_helpers. Verify with the checks this repository uses - as the map gives them - +and report the exact commands and observed output. Address any concrete findings the runner supplies +and note the actual fixes. Return blocked if the required scope or tools are unavailable. Do not +commit or push; the committer stage lands the work. +``` + +```tools +read +glob +grep +edit +shell +``` + +```configs +provider=openai +model=gpt-5.6-terra +retries=1 +timeout=30m +``` + + +```edges +success -> reviewer +blocked_retryable -> implementer +blocked -> #failure +error -> #failure +``` + + +# stage reviewer + +```limits +{"max_visits": 3, "on": ["rejected"]} +``` + +```memory +{"task": "input.prompt", "acceptance_checks": "implementer.acceptance_checks", "changed_files": "implementer.changed_files", "implementation": "implementer.implementation_summary?"} +``` + + +```output +{ summary, changed_files, verification_evidence, duplication_review, performance_review, findings } +``` + +```prompt +Verify with the build and the unit tests as android/AGENTS.md gives them, and for a +change with a rendered surface check the captures against the running app on the device the map +names. Enforce the map's geometry-token rule: a raw dimension outside the token file is a must-fix +finding. + +Review the actual changes against the task in input.prompt and the acceptance +checks the implementer recorded. Run the repository's relevant verification checks +with the stage tools (for example build, lint, tests or type checks) and report the +exact commands and observed results in verification_evidence. + +Independently assess code quality: +- Duplication: search the run directory for repeated or near-identical functions, + blocks and copy-pasted logic introduced by the change; identify code that should be + shared and record concrete locations in duplication_review. +- Performance and resources: flag redundant computation, unnecessary dependencies, + and repeated I/O or database/filesystem work that wastes time, CPU or memory; + record it in performance_review. +- Workarounds: flag hacks, suppressed errors and shortcuts that trade reliability for + speed in findings. + +Keep verification proportionate to the change: exercise the files and screens the diff +actually touches, capture at most one screenshot per changed surface plus one regression +screen, and do not repeat sweeps or measurements that an earlier visit already recorded. + +Return rejected ONLY when at least one must-fix finding exists: a defect that blocks +acceptance and that you can locate concretely (file and line), with evidence and a +remedy. Do not return rejected for advisory-only, cosmetic or nit items. When there +is no must-fix finding but you have non-blocking notes (duplication, performance or +workaround observations), record them in findings and return advisory, which +continues the run without another implementer visit. Return success only when the +observed checks pass and you have nothing further to record. Do not edit files or +claim unexecuted checks. +``` + +```tools +read +glob +grep +shell +``` + +```configs +provider=openai +model=gpt-5.6-luna +retries=1 +timeout=45m +``` + + +```edges +success -> ui_reviewer +advisory -> ui_reviewer +rejected -> implementer +error -> #failure +``` + + +# stage ui_reviewer + +```limits +{"max_visits": 3, "on": ["rejected"]} +``` + +```memory +{"task": "input.prompt", "acceptance_checks": "implementer.acceptance_checks", "changed_files": "implementer.changed_files", "implementation": "implementer.implementation_summary?"} +``` + + +```output +{ summary, changed_files, ui_applicable, screenshots, ui_review, platform_conventions, findings } +``` + +```prompt +Decide whether this change touches a user-facing UI. If it does not (a backend, +library, CLI, config or data-only change with no rendered surface), set +ui_applicable=false, explain briefly in summary, and return skipped. Do not invent +screenshots or visual work. + +When the change does affect a user-facing UI, verify it visually: +- Build and launch the application the way the repository actually does it, then + capture screenshots of every relevant screen, state and viewport (default, empty, + loading, error, long content, small and large sizes) with the target platform's + screenshot tooling through shell. Prefer a headless or scripted capture when the + project supports one. Save the images under the run directory and list the exact + capture commands and image paths in screenshots. +- Read the captured images and review them as a UI/UX designer: layout, spacing, + alignment, typographic hierarchy, color and contrast, iconography, interaction + and focus states, loading/empty/error states, accessibility, and consistency. +- Judge the UI against established industry patterns and, above all, the native + conventions of the platform the app targets (operating system, toolkit or web + standards). Record the platform you reviewed against in platform_conventions, + and any deviations and concrete fixes in findings. Infer the target platform from + the task and the repository's actual UI toolkit when it is unstated. +- Approve only when the screenshots show the task and the acceptance checks are met + and the UI is correct, consistent and native to the target platform; otherwise + return rejected with specific, actionable findings the implementer can fix. + +Keep the capture pass proportionate: capture each changed state once (for example one +collapsed and one expanded section, one run state, one stuck header) instead of every +screen at multiple viewports or pan directions, and reuse what this run already captured +rather than re-doing it. + +If the change is UI-facing but screenshots genuinely cannot be captured +(for example no runnable UI or no display), return blocked rather than approving. +Do not edit files, and do not approve a UI you did not actually see. + +Return rejected only for a must-fix visual defect with a concrete location, evidence +and remedy; record non-blocking polish, duplication or convention notes in findings +and return advisory, which continues the run without another implementer visit. +``` + +```tools +read +glob +grep +shell +``` + +```configs +provider=openai +model=gpt-5.6-luna +retries=1 +timeout=45m +``` + + +```edges +skipped -> committer +approved -> committer +advisory -> committer +rejected -> implementer +blocked -> #failure +error -> #failure +``` + +# stage committer + +```memory +{"task": "input.prompt", "changed_files": "implementer.changed_files", "review": "reviewer.summary?", "ui_review": "ui_reviewer.summary?"} +``` + +```output +{ commit, files, conflicts_resolved, summary } +``` + +```prompt +Commit the work this run produced, and resolve any conflict that is present. This stage +runs after the reviews have approved and is the only stage allowed to commit. + +Before committing, check that the change is reflected where it belongs: the packaged +templates, the packaged +authoring skill references. Update anything the change makes stale, and state in the +commit body what you checked, saying explicitly when none of them needs updating. + +Stage exactly the files this run changed - the paths in changed_files that still exist - +using the repository's own version control commands, and commit them as one commit whose +subject says what the change does and whose body summarises it. Do not stage unrelated +files, generated or ignored files, or anything another change owns. Do not invent work +that was not done; describe what is actually in the diff. + +If the workspace is in a conflicted state - unmerged paths, or a merge, rebase or +cherry-pick left unfinished - resolve each conflict: keep both intents where they are +compatible, and choose the result that satisfies the task where they are not, then +complete the operation so the tree is clean and the commit is finished. Never commit +conflict markers, and never discard the other side's work silently; record what you +resolved. + +Do not push, tag, revert, or rewrite unrelated history. If there is nothing to commit, +report that instead of creating an empty commit. Report the commit identifier, the files +committed, whatever you resolved, and anything you deliberately left uncommitted. + + +Finally, bring this branch up to date with main, in this same stage. After committing the work +above, merge main into this branch here, resolve any conflict here - keeping both intents where they +are compatible, and choosing the result that satisfies this task where they are not - run the +repository's checks again after resolving, and commit the completed merge with a message naming what +was combined. The point is that this branch can later be merged back without conflict: leave the +tree clean and the merge finished. Never leave conflict markers, never discard the other side's work +silently. If the merge genuinely cannot be completed, say so with the path and the reason instead of +leaving the merge half-finished. + +Then land it: this branch is the work and main is where it belongs, so merge this branch into main +as the last act of this stage. Find the repository's main working tree with `git worktree list` (the +entry checked out on main, not this one) and merge this branch into it there with `--no-ff`, with a +message saying what the change does. Two lanes can land at the same moment, so take a lock first - +`mkdir` a lock directory beside main's `git` directory in a bounded retry loop, and remove it when +done - and retry briefly if another lane holds it. Then verify with +`git merge-base --is-ancestor main` and report the commit main now has. If main cannot +be updated, say exactly why and leave this branch committed and clean rather than leaving main +half-merged. There is no separate merge pipeline and nothing to hand this to: if it cannot be landed +here, it is reported here. + + +Do not edit README or other shared documentation files: an item records what it changed and which documentation is now stale in its own commit body, and the shared documents are folded by a single writer. + +``` + +```tools +read +glob +grep +edit +shell +``` + +```configs +provider=openai +model=gpt-5.6-terra +retries=1 +timeout=30m +``` + +```edges +success -> #success +blocked -> #failure +error -> #failure +``` diff --git a/internal/stage/agent.go b/internal/stage/agent.go index d3109d20..57b23b12 100644 --- a/internal/stage/agent.go +++ b/internal/stage/agent.go @@ -27,6 +27,14 @@ type AgentClient struct { } func (c OpenCodeClient) Interrupt(ctx context.Context, id string) error { + return c.interrupt(ctx, id, false) +} + +func (c OpenCodeClient) InterruptAndConfirm(ctx context.Context, id string) error { + return c.interrupt(ctx, id, true) +} + +func (c OpenCodeClient) interrupt(ctx context.Context, id string, confirm bool) error { if !sessionIDPattern.MatchString(id) { return fmt.Errorf("invalid session ID") } @@ -46,7 +54,20 @@ func (c OpenCodeClient) Interrupt(ctx context.Context, id string) error { c.Backend = connection.Backend } _, err := c.call(ctx, "post", "/api/session/"+id+"/interrupt", map[string]any{}) - return err + if err != nil || !confirm { + return err + } + // An interrupt acknowledgement is not a cleanup receipt. Keep ownership + // until the harness positively reports settlement, bounded by the caller. + if c.Backend == "v1" { + _, err := c.call(ctx, "post", "/api/experimental/session/"+id+"/wait", nil) + return err + } + // The failed stage wait already records deadline evidence. Cleanup settlement + // must still be confirmed, but must not add a second duplicate operation entry. + confirmed := c + confirmed.Operations = nil + return confirmed.waitForSessionIdle(ctx, id) } func (c *AgentClient) Generate(ctx context.Context, config ModelConfig, prompt string) (text string, err error) { @@ -100,7 +121,7 @@ func (c *AgentClient) Generate(ctx context.Context, config ModelConfig, prompt s defer cancel() if err != nil { c.captureTimedOutEvidence(ctx, err, config, watch) - if stopErr := c.Interrupt(cleanup, c.SessionID); stopErr != nil { + if stopErr := c.InterruptAndConfirm(cleanup, c.SessionID); stopErr != nil { err = fmt.Errorf("agent interrupt failed; retained session %s needs inspection: %w", c.SessionID, errors.Join(err, stopErr, ErrStageCleanup)) return } diff --git a/internal/stage/agent_test.go b/internal/stage/agent_test.go index 2a8bcb58..c18d0721 100644 --- a/internal/stage/agent_test.go +++ b/internal/stage/agent_test.go @@ -52,8 +52,8 @@ func TestAgentOwnershipPermissionsAndInterrupt(t *testing.T) { t.Fatal("unexpected call", args) return nil, nil } - if _, err := client.Generate(context.Background(), ModelConfig{}, "This call has no tools."); err == nil || !interrupted || owned { - t.Fatal("failed generation did not stop and release ownership") + if _, err := client.Generate(context.Background(), ModelConfig{}, "This call has no tools."); !errors.Is(err, ErrStageCleanup) || !interrupted || !owned { + t.Fatal("interrupt acknowledgement without idle confirmation must retain ownership") } } diff --git a/internal/stage/budget_test.go b/internal/stage/budget_test.go index 47b0ee08..21426679 100644 --- a/internal/stage/budget_test.go +++ b/internal/stage/budget_test.go @@ -82,6 +82,9 @@ func TestBudgetsAccumulateAcrossRetriesAndBlockFurtherGeneration(t *testing.T) { var sessions, generations atomic.Int64 client := &OpenCodeClient{command: func(ctx context.Context, _ string, args ...string) ([]byte, error) { path := args[2] + if path == "/api/session/active" { + return []byte(`{"data":{}}`), nil + } if path == "/api/session" { return []byte(fmt.Sprintf(`{"data":{"id":"ses_attempt_%d"}}`, sessions.Add(1))), nil } @@ -147,6 +150,9 @@ func TestLiveAgentBudgetInterruptsAndDoesNotReplay(t *testing.T) { return []byte(`{"data":{"id":"ses_agent_budget"}}`), nil } if path == "/api/session/active" { + if interrupts.Load() > 0 { + return []byte(`{"data":{}}`), nil + } <-ctx.Done() return nil, ctx.Err() } diff --git a/internal/stage/opencode.go b/internal/stage/opencode.go index bdf28bfd..d58b0bf1 100644 --- a/internal/stage/opencode.go +++ b/internal/stage/opencode.go @@ -321,7 +321,7 @@ func (c *OpenCodeClient) Generate(ctx context.Context, config ModelConfig, promp cleanup, cancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second) defer cancel() if err != nil { - if stopErr := c.Interrupt(cleanup, session.Data.ID); stopErr != nil { + if stopErr := c.InterruptAndConfirm(cleanup, session.Data.ID); stopErr != nil { err = fmt.Errorf("retained OpenCode session %s; interrupt failed: %w", session.Data.ID, errors.Join(err, stopErr, ErrStageCleanup)) return } diff --git a/internal/stage/pi.go b/internal/stage/pi.go index 3e0b1bf1..e4849f79 100644 --- a/internal/stage/pi.go +++ b/internal/stage/pi.go @@ -238,6 +238,10 @@ func (c *PiClient) Generate(ctx context.Context, config ModelConfig, prompt stri } close(done) close(heartbeatDone) + // agent_end/abort acknowledgement and the parent's exit alone do not + // prove its shell descendants exited. A surviving process group retains + // ownership rather than handing the resource to another stage. + quiet = quiet && errors.Is(syscall.Kill(-c.cmd.Process.Pid, 0), syscall.ESRCH) if quiet { _ = os.WriteFile(filepath.Join(c.Reference, "quiescent"), []byte("quiescent"), 0600) } @@ -532,9 +536,16 @@ func InterruptPi(ctx context.Context, reference string) error { if !filepath.IsAbs(reference) { return fmt.Errorf("invalid retained Pi reference") } - if _, err := os.Stat(filepath.Join(reference, "owner.json")); err != nil { + data, err := os.ReadFile(filepath.Join(reference, "owner.json")) + if err != nil { return errors.Join(err, ErrStageCleanup) } + var owner struct { + PID int `json:"pid"` + } + if json.Unmarshal(data, &owner) != nil || owner.PID <= 1 { + return errors.Join(fmt.Errorf("invalid retained Pi owner"), ErrStageCleanup) + } old := time.Unix(0, 0) if err := os.Chtimes(filepath.Join(reference, "lease"), old, old); err != nil { return err @@ -542,7 +553,7 @@ func InterruptPi(ctx context.Context, reference string) error { ticker := time.NewTicker(100 * time.Millisecond) defer ticker.Stop() for { - if data, err := os.ReadFile(filepath.Join(reference, "quiescent")); err == nil && string(data) == "quiescent" { + if data, err := os.ReadFile(filepath.Join(reference, "quiescent")); err == nil && string(data) == "quiescent" && errors.Is(syscall.Kill(owner.PID, 0), syscall.ESRCH) && errors.Is(syscall.Kill(-owner.PID, 0), syscall.ESRCH) { return nil } select { diff --git a/internal/stage/resource_cleanup_test.go b/internal/stage/resource_cleanup_test.go new file mode 100644 index 00000000..c57d4080 --- /dev/null +++ b/internal/stage/resource_cleanup_test.go @@ -0,0 +1,44 @@ +package stage + +import ( + "context" + "encoding/json" + "errors" + "os" + "os/exec" + "path/filepath" + "syscall" + "testing" + "time" +) + +func TestResourcePiReceiptRequiresProcessExit(t *testing.T) { + // A test-owned process only: no Pi provider, shell/ADB command or phone. + cmd := exec.Command("sleep", "5") + cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true} + if err := cmd.Start(); err != nil { + t.Fatal(err) + } + defer func() { cmd.Process.Kill(); cmd.Wait() }() + dir := t.TempDir() + owner, _ := json.Marshal(map[string]any{"pid": cmd.Process.Pid}) + for name, data := range map[string][]byte{"owner.json": owner, "lease": []byte("lease"), "quiescent": []byte("quiescent")} { + if err := os.WriteFile(filepath.Join(dir, name), data, 0600); err != nil { + t.Fatal(err) + } + } + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Millisecond) + defer cancel() + if err := InterruptPi(ctx, dir); !errors.Is(err, ErrStageCleanup) { + t.Fatal("receipt authorized handoff while process lived", err) + } + if err := cmd.Process.Kill(); err != nil { + t.Fatal(err) + } + cmd.Wait() + ctx2, cancel2 := context.WithTimeout(context.Background(), time.Second) + defer cancel2() + if err := InterruptPi(ctx2, dir); err != nil { + t.Fatal("confirmed process exit not accepted", err) + } +} diff --git a/internal/tui/resources_test.go b/internal/tui/resources_test.go new file mode 100644 index 00000000..a2e34475 --- /dev/null +++ b/internal/tui/resources_test.go @@ -0,0 +1,43 @@ +package tui + +import ( + "fmt" + "strings" + "testing" + + "git.a-wan.net/alan/pipelines/internal/protocol" + "github.com/charmbracelet/lipgloss" + "github.com/charmbracelet/x/ansi" + "github.com/muesli/termenv" +) + +func TestResourceWaitFrames(t *testing.T) { + previous := lipgloss.ColorProfile() + defer lipgloss.SetColorProfile(previous) + for _, width := range []int{60, 120} { + for _, plain := range []bool{false, true} { + t.Run(fmt.Sprintf("width-%d-plain-%t", width, plain), func(t *testing.T) { + if plain { + lipgloss.SetColorProfile(termenv.Ascii) + } else { + lipgloss.SetColorProfile(termenv.TrueColor) + } + m := decisionModel() + m.width = width + m.runDetail.Status = protocol.RunStatusQueued + m.runDetail.Steps = nil + m.runDetail.ResourceWait = &protocol.ResourceWait{Resource: "phone-X", Stage: "review", Sequence: 1, QueuedSince: "2026-10-18T10:00:00Z"} + frame := ansi.Strip(m.runsDetailsView(24)) + if !strings.Contains(frame, "Waiting for resource phone-X") || strings.Contains(frame, "c decide") || strings.Contains(frame, "Waiting for a person") { + t.Fatal("wrong resource wait frame:\n", frame) + } + for _, line := range strings.Split(frame, "\n") { + if ansi.StringWidth(line) > width { + t.Fatalf("overflow: %q", line) + } + } + t.Log("Rendered resource wait (no phone):\n" + frame) + }) + } + } +} diff --git a/internal/tui/results.go b/internal/tui/results.go index a4c1b610..3ef6840a 100644 --- a/internal/tui/results.go +++ b/internal/tui/results.go @@ -11,6 +11,9 @@ import ( ) func resultLabel(r protocol.RunSummary) string { + if r.ResourceWait != nil { + return r.StatusLabel() + " · stage " + r.ResourceWait.Stage + } preview := ansi.Strip(r.ResultPreview) if r.Status == "succeeded" { if preview != "" { @@ -55,7 +58,7 @@ func (m Model) pipelineResultLines() []string { if r == nil { return []string{"Results · This version has not run yet. No accomplishments recorded."} } - lines := []string{fmt.Sprintf("Results · run #%d · %s · %s · %d completed visits · %s", r.ID, r.Label(), ansi.Strip(m.progress(r.Status)), r.CompletedSteps, runDuration(r.CreatedAt, r.FinishedAt)), resultLabel(*r)} + lines := []string{fmt.Sprintf("Results · run #%d · %s · %s · %d completed visits · %s", r.ID, r.Label(), ansi.Strip(m.runSummaryStatus(*r)), r.CompletedSteps, runDuration(r.CreatedAt, r.FinishedAt)), resultLabel(*r)} if r.Status != "succeeded" && r.ResultPreview != "" && r.Diagnostic != "" { lines = append(lines, "Partial output: "+ansi.Strip(r.ResultPreview)) } diff --git a/internal/tui/run_detail.go b/internal/tui/run_detail.go index 79154144..87c4c04a 100644 --- a/internal/tui/run_detail.go +++ b/internal/tui/run_detail.go @@ -275,6 +275,13 @@ func runStatus(status string) string { return lipgloss.NewStyle().Foreground(color).Render(status) } +func (m Model) runSummaryStatus(run protocol.RunSummary) string { + if run.ResourceWait != nil { + return run.StatusLabel() + } + return m.progress(run.Status) +} + func runDuration(start, end string) string { first, err := time.Parse(time.RFC3339, start) if err != nil { @@ -687,7 +694,7 @@ func (m Model) runsDetailsView(height int) string { start, end := selectionWindow(len(runs), m.runIndex, max(1, (height-4-len(rows))/2)) for i := start; i < end; i++ { r := runs[i] - text := fmt.Sprintf("#%d %s · v%d · %s · %s", r.ID, r.Label(), r.Version, m.progress(r.Status), dateLabel(r.FinishedAt)) + text := fmt.Sprintf("#%d %s · v%d · %s · %s", r.ID, r.Label(), r.Version, m.runSummaryStatus(r), dateLabel(r.FinishedAt)) rows = append(rows, menuOption(text, i == m.runIndex)) rows = append(rows, " "+ansi.Truncate(resultLabel(r), max(1, m.width-6), "…")) } @@ -710,7 +717,8 @@ func (m Model) runsDetailsView(height int) string { return panel(m.width, height, []string{bar(title, fmt.Sprint(len(runs)), max(1, m.width-4), true)}, rows, footer, true) } r := m.runDetail - header := bar(fmt.Sprintf("#%d %s · v%d", r.ID, r.Label(), r.Version), m.progress(r.Status)+" · "+runDuration(r.CreatedAt, r.FinishedAt), m.width, false) + status := m.runSummaryStatus(r.RunSummary) + header := bar(fmt.Sprintf("#%d %s · v%d", r.ID, r.Label(), r.Version), status+" · "+runDuration(r.CreatedAt, r.FinishedAt), m.width, false) if height <= 6 { return fitBlock([]string{header, runHighlight(m.runSelectionLabel(), m.width), m.runActions() + " · Tab pane · Esc back"}, m.width, height) }