diff --git a/android/AGENTS.md b/android/AGENTS.md index f6dddbd2..2e035722 100644 --- a/android/AGENTS.md +++ b/android/AGENTS.md @@ -20,16 +20,37 @@ time while iterating — the full suite is for the end. ## The dev device +Only the **UI reviewer**, while its stage holds a declared runner-exclusive resource +for the selected phone, may use the device. Implementers and code reviewers do code, +builds, unit tests and source review only: **no adb, installs, captures, device-setting +changes or main-build restoration**. Never acquire a phone using sleep/check loops. +Do not use the phone until compatible resource scheduling is deployed and every device +user has opted in or has been reconciled by the operator. Existing pinned runs are not +retroactively protected. The resource for the device below is `phone-47241FDAS00DLK`. + The only device is the operator's **Pixel 9 Pro XL, serial `47241FDAS00DLK`**. The operator's Pixel 10 Pro Fold (`58281FDCG000JF`) is theirs: use it only where an item's brief explicitly allows it, and never make it a default. **Never start or use an emulator.** -Before any capture: `svc power stayon usb`, `settings put global stay_on_while_plugged_in 7`, +The UI reviewer records original app connection/workspace, display size and power/settings +before changing them. Before any capture: `svc power stayon usb`, `settings put global stay_on_while_plugged_in 7`, `input keyevent KEYCODE_WAKEUP`, `wm dismiss-keyguard`. Capture with `adb -s shell screencap -p /sdcard/shot.png` then `adb pull` — never `exec-out > file`. Wide layout: `wm size 2000x2244`; the fold's unfolded shape is `wm size 2076x2152`; `wm size reset` afterwards either way. +Create controlled fixtures for changed screens/states (including human-decision sheets) +on a scratch server/database and throwaway session; never depend on incidental production +state, change a real operator session, or restart a deployed service. Use the configured +authoritative endpoint, never a scratch URL as a project default. Keep all shell/ADB work +foreground and awaited: no detached/background processes may outlive the lease. +UI review includes restoring the captured original settings and main's build before its +successful return. Confirm every tool has exited. If capture, restoration or cleanup fails, +report the failure explicitly; do not claim verification or readiness. Uncertain termination +must fail closed and keep ownership quarantined, not delete a lock or expire it by age. +Return `output.resource_cleanup_confirmed=false` if restoration/readiness is uncertain; +that generic release veto retains the runner lease even if the harness reports idle. + ## Geometry is tokenised, and a test enforces it Every dimension comes from `ui/theme/Dimens.kt` (plus the layout itself) — no new hardcoded `dp`, no @@ -53,13 +74,14 @@ genuinely needs a new token goes there, named for its purpose. ## Publishing a build Copy the APK into the running run's artifacts directory as `pipelines-debug-wip-.apk` and -report the LAN URL. After any install, leave the operator's phone on **main's** build: +report the LAN URL. The UI reviewer alone, under the same lease, restores **main's** build after an install: `/private/var/folders/jp/q0qvjyg54fl_c5vhhk1ylz2m0000gn/T/opencode/deploy-app.sh main`. ## Traps - A build made in a worktree that is behind main is missing features main has — sync before building, and say which revision you built. -- Changing the app's server URL for a test (a stub) must be undone: `ws://10.0.0.75:8080/connect`, - workspace `/Users/alan/repos/pipelines`, then force-stop and relaunch. +- Restore the app's captured original connection and workspace after a fixture test, + using the configured authoritative endpoint; then force-stop and relaunch under the lease. + Never substitute a hardcoded or disposable scratch endpoint for the original configuration. - `mStayOn`/screen timeout are already set; do not "fix" the device by locking or sleeping it. diff --git a/android/app/build.gradle.kts b/android/app/build.gradle.kts index 4b89f8dc..02d1b391 100644 --- a/android/app/build.gradle.kts +++ b/android/app/build.gradle.kts @@ -58,6 +58,7 @@ dependencies { implementation(libs.androidx.compose.material.icons.extended) implementation(libs.androidx.navigation.compose) implementation(libs.androidx.lifecycle.runtime.compose) + implementation(libs.androidx.lifecycle.process) implementation(libs.androidx.lifecycle.viewmodel.compose) implementation(libs.androidx.lifecycle.viewmodel.ktx) implementation(libs.kotlinx.coroutines.android) diff --git a/android/app/src/main/java/net/awan/pipelines/PipelinesApplication.kt b/android/app/src/main/java/net/awan/pipelines/PipelinesApplication.kt index 8d36876e..e0a93b0e 100644 --- a/android/app/src/main/java/net/awan/pipelines/PipelinesApplication.kt +++ b/android/app/src/main/java/net/awan/pipelines/PipelinesApplication.kt @@ -1,12 +1,15 @@ package net.awan.pipelines import android.app.Application +import androidx.lifecycle.ProcessLifecycleOwner import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.SupervisorJob +import net.awan.pipelines.data.ForegroundReconnector import net.awan.pipelines.data.PipelinesRepository import net.awan.pipelines.data.SettingsRepository import net.awan.pipelines.data.transport.PipelinesClient +import net.awan.pipelines.data.transport.ServerUrl import net.awan.pipelines.notify.RunNotifier /** @@ -30,8 +33,39 @@ class PipelinesApplication : Application() { */ val runNotifier: RunNotifier by lazy { RunNotifier(this) } + /** + * The one owner of return-to-foreground recovery: when the process comes back + * to the front it restores a connection the background took away, with no + * interaction. It is created once with the Application and observes the + * process lifecycle, so every Activity and ViewModel recreation leaves it + * untouched. The liveness check is the existing `ping`; the reconnect is the + * same shared-client connect the app's own Connect/Reconnect uses, so the + * policy lives only in `foregroundAction` and the client's backoff stays the + * only reconnect mechanism. + */ + private val foregroundReconnector: ForegroundReconnector by lazy { + ForegroundReconnector( + state = { repository.connectionState.value }, + confirmAlive = { + // The existing ping is the positive check: on a dead socket it + // throws (or never answers) instead of returning, so reaching the + // next line is the proof of life. + repository.ping() + true + }, + reconnect = { + // A saved URL that no longer validates is left for Settings to + // report; there is nothing to reconnect to. + val validated = ServerUrl.validate(settings.serverUrl.value) + if (validated.valid) repository.recover(validated) + }, + scope = applicationScope, + ) + } + override fun onCreate() { super.onCreate() runNotifier.ensureChannel() + ProcessLifecycleOwner.get().lifecycle.addObserver(foregroundReconnector) } } diff --git a/android/app/src/main/java/net/awan/pipelines/data/ForegroundReconnector.kt b/android/app/src/main/java/net/awan/pipelines/data/ForegroundReconnector.kt new file mode 100644 index 00000000..1819acbc --- /dev/null +++ b/android/app/src/main/java/net/awan/pipelines/data/ForegroundReconnector.kt @@ -0,0 +1,81 @@ +package net.awan.pipelines.data + +import androidx.lifecycle.DefaultLifecycleObserver +import androidx.lifecycle.LifecycleOwner +import kotlinx.coroutines.CancellationException +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.launch +import kotlinx.coroutines.withTimeoutOrNull +import net.awan.pipelines.data.transport.ConnectionState +import net.awan.pipelines.data.transport.ForegroundAction +import net.awan.pipelines.data.transport.foregroundAction + +/** + * The one lifecycle-aware owner of the app's return-to-foreground recovery. + * + * While another app is in front Android freezes — and often kills — this process, + * so the socket it holds dies or goes quiet and the client's own bounded backoff + * either never starts or starts too late for the operator, who comes back and + * taps Connect. The client's backoff (see `PipelinesClient.scheduleReconnect`) is + * still the only reconnect mechanism; this owner only decides *when* to drive it, + * and does so the moment the app becomes visible again. + * + * It observes the whole process lifecycle (a `ProcessLifecycleOwner`), never the + * Activity or the ViewModel, both of which Android recreates: the owner is + * created once with the Application and survives every recreation. The decision + * itself is the one pure [foregroundAction] policy; this class supplies only the + * current state, a positive liveness check and the reconnect effect, so the + * policy is unit-testable with no device, socket or framework. + */ +class ForegroundReconnector( + private val state: () -> ConnectionState, + private val confirmAlive: suspend () -> Boolean, + private val reconnect: () -> Unit, + private val scope: CoroutineScope, + private val confirmTimeoutMillis: Long = CONFIRM_TIMEOUT_MILLIS, +) : DefaultLifecycleObserver { + + /** + * The app became visible again. A state the policy calls reconnecting is + * reconnected at once — the pending backoff delay is cancelled by the + * reconnect effect, so the operator never waits it out. A socket the state + * still calls Connected is first confirmed to answer, so a silent + * "Connected" is never trusted; the state is re-read afterwards, because the + * socket's own failure callback may have run while the check was in flight. + */ + override fun onStart(owner: LifecycleOwner) { + val before = state() + if (before !is ConnectionState.Connected) { + if (foregroundAction(before, confirmedAlive = false) == ForegroundAction.Reconnect) reconnect() + return + } + scope.launch { + val alive = confirm() + if (foregroundAction(state(), alive) == ForegroundAction.Reconnect) reconnect() + } + } + + /** + * Confirms a supposedly-connected socket still answers, bounded so a dead one + * cannot hold recovery: the check runs off the main thread on the app scope + * and a probe that fails, throws or outlives its budget counts as not alive. + */ + private suspend fun confirm(): Boolean = + withTimeoutOrNull(confirmTimeoutMillis) { + try { + confirmAlive() + } catch (t: Throwable) { + if (t is CancellationException) throw t + false + } + } ?: false + + companion object { + /** + * Bounds the liveness check. A healthy round trip on the trusted LAN/VPN + * is far shorter, so a socket that cannot answer is noticed promptly + * instead of holding the "Connected" the operator is looking at. + */ + const val CONFIRM_TIMEOUT_MILLIS = 3_000L + } +} diff --git a/android/app/src/main/java/net/awan/pipelines/data/PipelinesRepository.kt b/android/app/src/main/java/net/awan/pipelines/data/PipelinesRepository.kt index 1479576f..9ad104dd 100644 --- a/android/app/src/main/java/net/awan/pipelines/data/PipelinesRepository.kt +++ b/android/app/src/main/java/net/awan/pipelines/data/PipelinesRepository.kt @@ -66,6 +66,9 @@ class PipelinesRepository( /** Opens the single shared connection to a validated server URL. */ fun connect(validated: ServerUrl.Validated) = client.connect(validated) + /** Recovers a lost connection without reversing an explicit Disconnect. */ + fun recover(validated: ServerUrl.Validated) = client.recover(validated) + /** Explicit disconnect; the client then stays Idle. */ fun disconnect() = client.disconnect() diff --git a/android/app/src/main/java/net/awan/pipelines/data/protocol/Message.kt b/android/app/src/main/java/net/awan/pipelines/data/protocol/Message.kt index 0a1e2d40..b0a5c168 100644 --- a/android/app/src/main/java/net/awan/pipelines/data/protocol/Message.kt +++ b/android/app/src/main/java/net/awan/pipelines/data/protocol/Message.kt @@ -315,6 +315,7 @@ data class SessionExport( @Serializable data class RunSummary( + @SerialName("resource_wait") val resourceWait: ResourceWait? = null, val id: Long = 0, @SerialName("pipeline_id") val pipelineId: Long = 0, val name: String = "", @@ -370,6 +371,7 @@ data class RunStep( // declared directly with the exact internal/protocol/runs.go wire names. @Serializable data class RunDetail( + @SerialName("resource_wait") val resourceWait: ResourceWait? = null, val id: Long = 0, @SerialName("pipeline_id") val pipelineId: Long = 0, val name: String = "", @@ -405,6 +407,8 @@ data class RunDetail( */ @Serializable data class RunEventFrame( + @SerialName("resource_wait") val resourceWait: ResourceWait? = null, + @SerialName("resource_wait_set") val resourceWaitSet: Boolean = false, @SerialName("run_id") val runId: Long = 0, val workspace: String = "", // The run's pipeline identity and stored title, repeated on every frame so a @@ -428,6 +432,16 @@ data class RunEventFrame( val step: RunStepDelta? = null, ) +@Serializable +data class ResourceWait( + val resource: String = "", + val stage: String = "", + val sequence: Int = 0, + @SerialName("queued_since") val queuedSince: String = "", +) { + fun statusLabel(): String = "Waiting for resource $resource" +} + /** * The compact change of one step within a [RunEventFrame] * (internal/protocol/runevents.go RunStepDelta): a delta, not a RunStep, so it diff --git a/android/app/src/main/java/net/awan/pipelines/data/transport/ConnectionRecovery.kt b/android/app/src/main/java/net/awan/pipelines/data/transport/ConnectionRecovery.kt new file mode 100644 index 00000000..87b44b1b --- /dev/null +++ b/android/app/src/main/java/net/awan/pipelines/data/transport/ConnectionRecovery.kt @@ -0,0 +1,40 @@ +package net.awan.pipelines.data.transport + +/** What the app does with the connection the moment it becomes visible again. */ +enum class ForegroundAction { + /** Leave the connection exactly as it is. */ + LeaveAlone, + + /** Re-establish the connection now, cancelling any pending reconnect delay. */ + Reconnect, +} + +/** + * The app's return-to-foreground recovery policy, in one pure place. + * + * Android freezes and often kills a backgrounded app's process, so the socket + * the app holds dies or goes quiet while another app is in front and nothing in + * the process notices in time. Coming back must restore that connection without + * the operator tapping Connect — but an explicit Disconnect is the operator's + * own decision and stays authoritative. A deliberate Disconnect is the only way + * the client reaches [ConnectionState.Idle], so Idle is left alone; every other + * state reconnects. + * + * A still [ConnectionState.Connected] socket is not trusted on return: a frozen + * process can leave it claiming Connected while it answers nothing, which is + * exactly the stale state this replaces. The caller therefore checks it with a + * positive probe (the existing ping) and passes the verdict as [confirmedAlive]; + * only a connection just confirmed to answer is left alone. + * + * @param state the connection state at the moment the app became visible again. + * @param confirmedAlive whether a [ConnectionState.Connected] socket has just + * been confirmed to answer. It is consulted only for Connected; every other + * state carries its own answer. + */ +fun foregroundAction(state: ConnectionState, confirmedAlive: Boolean): ForegroundAction = when (state) { + ConnectionState.Idle -> ForegroundAction.LeaveAlone + is ConnectionState.Connected -> + if (confirmedAlive) ForegroundAction.LeaveAlone else ForegroundAction.Reconnect + is ConnectionState.Connecting -> ForegroundAction.Reconnect + is ConnectionState.Disconnected -> ForegroundAction.Reconnect +} diff --git a/android/app/src/main/java/net/awan/pipelines/data/transport/PipelinesClient.kt b/android/app/src/main/java/net/awan/pipelines/data/transport/PipelinesClient.kt index 886d99e3..b649f361 100644 --- a/android/app/src/main/java/net/awan/pipelines/data/transport/PipelinesClient.kt +++ b/android/app/src/main/java/net/awan/pipelines/data/transport/PipelinesClient.kt @@ -78,11 +78,33 @@ class PipelinesClient( @Volatile private var manualClose = false + /** Serializes socket replacement with callbacks arriving on OkHttp threads. */ + private val connectionLock = Any() + private var reconnectJob: Job? = null private var attempt = 0 /** Opens (or re-opens) the connection to a validated server URL. */ fun connect(validated: ServerUrl.Validated) { + synchronized(connectionLock) { + connectLocked(validated) + } + } + + /** + * Re-opens a connection the transport lost without reversing an explicit + * [disconnect]. The check and connection replacement share one lock, so a + * Disconnect racing a foreground liveness probe wins even after that probe + * has decided recovery is needed. + */ + fun recover(validated: ServerUrl.Validated) { + synchronized(connectionLock) { + if (manualClose) return + connectLocked(validated) + } + } + + private fun connectLocked(validated: ServerUrl.Validated) { val httpUrl = validated.httpUrl ?: throw IllegalArgumentException("server URL is not valid") endpoint = httpUrl manualClose = false @@ -94,14 +116,16 @@ class PipelinesClient( /** Explicit user disconnect: stops reconnection and closes the socket. */ fun disconnect() { - manualClose = true - reconnectJob?.cancel() - reconnectJob = null - val socket = webSocket - webSocket = null - socket?.close(NORMAL_CLOSURE, "client closing") - failPending("Disconnected") - _state.value = ConnectionState.Idle + synchronized(connectionLock) { + manualClose = true + reconnectJob?.cancel() + reconnectJob = null + val socket = webSocket + webSocket = null + socket?.close(NORMAL_CLOSURE, "client closing") + failPending("Disconnected") + _state.value = ConnectionState.Idle + } } fun close() { @@ -115,14 +139,20 @@ class PipelinesClient( */ suspend fun call(request: Message): Message = withContext(ioDispatcher) { sendLock.withLock { - val socket = webSocket ?: throw NotConnectedException() - if (_state.value !is ConnectionState.Connected) throw NotConnectedException() - val id = "android-" + sequence.incrementAndGet() - val deferred = CompletableDeferred() - pending[id] = deferred - try { + val (id, deferred) = synchronized(connectionLock) { + val socket = webSocket ?: throw NotConnectedException() + if (_state.value !is ConnectionState.Connected) throw NotConnectedException() + val id = "android-" + sequence.incrementAndGet() + val deferred = CompletableDeferred() + pending[id] = deferred val payload = json.encodeToString(Message.serializer(), request.copy(id = id)) - if (!socket.send(payload)) throw IOException("websocket send failed") + if (!socket.send(payload)) { + pending.remove(id) + throw IOException("websocket send failed") + } + id to deferred + } + try { val response = withTimeout(requestTimeoutMillis) { deferred.await() } if (response.type == Protocol.ERROR) { throw ProtocolException(response.error?.ifBlank { "server error" } ?: "server error") @@ -144,12 +174,14 @@ class PipelinesClient( */ suspend fun sendOnly(request: Message): String = withContext(ioDispatcher) { sendLock.withLock { - val socket = webSocket ?: throw NotConnectedException() - if (_state.value !is ConnectionState.Connected) throw NotConnectedException() - val id = "android-" + sequence.incrementAndGet() - val payload = json.encodeToString(Message.serializer(), request.copy(id = id)) - if (!socket.send(payload)) throw IOException("websocket send failed") - id + synchronized(connectionLock) { + val socket = webSocket ?: throw NotConnectedException() + if (_state.value !is ConnectionState.Connected) throw NotConnectedException() + val id = "android-" + sequence.incrementAndGet() + val payload = json.encodeToString(Message.serializer(), request.copy(id = id)) + if (!socket.send(payload)) throw IOException("websocket send failed") + id + } } } @@ -166,14 +198,34 @@ class PipelinesClient( } } - private fun open() { + private fun open() = synchronized(connectionLock) { openLocked() } + + private fun openLocked() { val current = endpoint ?: return attempt += 1 _state.value = ConnectionState.Connecting(attempt) val request = Request.Builder().url(current).build() + // Replace the current socket, then close the one it superseded. Its + // callbacks are ignored from then on (see [isCurrent]), so a failure the + // old socket reports late — the common case when the process was frozen + // while it died — can neither clear the new socket nor schedule a + // reconnect that would undo it. + val superseded = webSocket webSocket = httpClient.newWebSocket(request, listener) + // A request sent over the superseded socket cannot receive a reply from + // the replacement. Fail it now rather than leaving a screen waiting for + // the ordinary request timeout after recovery. + if (superseded != null) failPending("Connection replaced") + superseded?.close(NORMAL_CLOSURE, "superseded") } + /** + * Whether [socket] is still the connection this client acts on. A socket this + * client has already replaced (a deliberate reconnect, or the foreground + * recovery re-establishing a frozen one) must never drive the state again. + */ + private fun isCurrent(socket: WebSocket): Boolean = webSocket === socket + private fun failPending(reason: String) { val snapshot = pending.keys.toList() for (key in snapshot) { @@ -181,7 +233,8 @@ class PipelinesClient( } } - private fun handleClosed(reason: String) { + private fun handleClosed(socket: WebSocket, reason: String) { + if (!isCurrent(socket)) return webSocket = null failPending(reason) if (manualClose) { @@ -194,52 +247,65 @@ class PipelinesClient( private fun scheduleReconnect(reason: String) { if (reconnectJob?.isActive == true) return _state.value = ConnectionState.Disconnected(reason, retrying = true) - val nextAttempt = attempt - val backoff = (BASE_BACKOFF_MILLIS shl (nextAttempt - 1).coerceAtMost(6)) - .coerceAtMost(MAX_BACKOFF_MILLIS) + val backoff = reconnectBackoffMillis(attempt) val jitter = Random.nextLong(0, 400) reconnectJob = scope.launch { delay(backoff + jitter) - if (!manualClose && endpoint != null) open() + synchronized(connectionLock) { + if (!manualClose && endpoint != null) open() + } } } private val listener = object : WebSocketListener() { override fun onOpen(webSocket: WebSocket, response: Response) { - attempt = 0 - _state.value = ConnectionState.Connected(build = null, serverTime = null) + synchronized(connectionLock) { + if (!isCurrent(webSocket)) return + attempt = 0 + _state.value = ConnectionState.Connected(build = null, serverTime = null) + } } override fun onMessage(webSocket: WebSocket, text: String) { - val message = try { - json.decodeFromString(Message.serializer(), text) - } catch (_: Exception) { - // Malformed frame: never misattribute it to a request. - return - } - val id = message.id - val deferred = if (id != null) pending.remove(id) else null - if (deferred != null) { - deferred.complete(message) - } else { - // Unsolicited envelopes (the connection's initial pipeline_list, - // server-pushed chat frames, stream errors) carry no matching id: - // hand them to the event stream instead of dropping them. - _events.tryEmit(message) + synchronized(connectionLock) { + if (!isCurrent(webSocket)) return + val message = try { + json.decodeFromString(Message.serializer(), text) + } catch (_: Exception) { + // Malformed frame: never misattribute it to a request. + return + } + val id = message.id + val deferred = if (id != null) pending.remove(id) else null + if (deferred != null) { + deferred.complete(message) + } else { + // Unsolicited envelopes (the connection's initial pipeline_list, + // server-pushed chat frames, stream errors) carry no matching id: + // hand them to the event stream instead of dropping them. + _events.tryEmit(message) + } } } override fun onClosed(webSocket: WebSocket, code: Int, reason: String) { - handleClosed("Connection closed (${code})") + synchronized(connectionLock) { + handleClosed(webSocket, "Connection closed (${code})") + } } override fun onClosing(webSocket: WebSocket, code: Int, reason: String) { - webSocket.close(NORMAL_CLOSURE, null) + synchronized(connectionLock) { + if (!isCurrent(webSocket)) return + webSocket.close(NORMAL_CLOSURE, null) + } } override fun onFailure(webSocket: WebSocket, t: Throwable, response: Response?) { - val reason = t.message?.ifBlank { t.javaClass.simpleName } ?: "connection failed" - handleClosed(reason) + synchronized(connectionLock) { + val reason = t.message?.ifBlank { t.javaClass.simpleName } ?: "connection failed" + handleClosed(webSocket, reason) + } } } @@ -247,6 +313,16 @@ class PipelinesClient( private const val NORMAL_CLOSURE = 1000 private const val BASE_BACKOFF_MILLIS = 1_000L private const val MAX_BACKOFF_MILLIS = 30_000L + + /** + * The successful open callback resets [attempt] to zero. A later socket + * failure must therefore begin at the base delay, never shift by -1 + * (which Kotlin masks into a very large shift count). + */ + internal fun reconnectBackoffMillis(attempt: Int): Long { + val exponent = (attempt - 1).coerceIn(0, 6) + return (BASE_BACKOFF_MILLIS shl exponent).coerceAtMost(MAX_BACKOFF_MILLIS) + } } } diff --git a/android/app/src/main/java/net/awan/pipelines/ui/RunEvents.kt b/android/app/src/main/java/net/awan/pipelines/ui/RunEvents.kt index ecf6a66c..a222ba55 100644 --- a/android/app/src/main/java/net/awan/pipelines/ui/RunEvents.kt +++ b/android/app/src/main/java/net/awan/pipelines/ui/RunEvents.kt @@ -101,6 +101,7 @@ internal fun foldRunEvent(runs: List, frame: RunEventFrame): RunList internal fun RunEventFrame.newRunSummary(): RunSummary? { if (pipelineId <= 0 || name.isBlank()) return null return RunSummary( + resourceWait = resourceWait, id = runId, pipelineId = pipelineId, name = name, @@ -124,10 +125,11 @@ internal fun RunEventFrame.newRunSummary(): RunSummary? { */ internal fun foldRunEvent(detail: RunDetail, frame: RunEventFrame): RunDetail { if (frame.runId != detail.id) return detail - if (isStaleRunFrame(detail.status, frame.status)) return detail + if (isStaleRunFrame(detail.status, frame.status, frame.resourceWaitSet)) return detail val merged = detail.fields().applying(frame) val steps = frame.step?.let { upsertStep(detail.steps, it) } ?: detail.steps val updated = detail.copy( + resourceWait = merged.resourceWait, status = merged.status, outcome = merged.outcome, currentStage = merged.currentStage, @@ -169,8 +171,9 @@ private fun advanceRunStatus(current: String, incoming: String): String = * run already settled. Dropping every field (not just the status) is what keeps * a late `running` delta from also rewriting the stage or step of a settled run. */ -private fun isStaleRunFrame(current: String, incoming: String): Boolean = - isStaleTransition(current, incoming, ::runLifecycleRank) +private fun isStaleRunFrame(current: String, incoming: String, resourceWaitSet: Boolean = false): Boolean = + if (resourceWaitSet && incoming == "queued" && !isTerminalRunStatus(current)) false + else isStaleTransition(current, incoming, ::runLifecycleRank) /** The step counterpart of [isStaleRunFrame]: a settled visit drops a stale delta whole. */ private fun isStaleStepDelta(current: String, incoming: String): Boolean = @@ -213,6 +216,7 @@ private fun advanceStatus(current: String, incoming: String, rank: (String) -> I /** The run-state fields a frame advances, shared by the summary row and the detail. */ private data class RunStateFields( + val resourceWait: net.awan.pipelines.data.protocol.ResourceWait?, val status: String, val outcome: String, val currentStage: String, @@ -226,8 +230,10 @@ private data class RunStateFields( * detail can never apply a frame differently. */ fun applying(frame: RunEventFrame): RunStateFields { - val status = advanceRunStatus(status, frame.status) + val status = if (frame.resourceWaitSet && frame.status == "queued" && !isTerminalRunStatus(status)) + "queued" else advanceRunStatus(status, frame.status) return RunStateFields( + resourceWait = if (isTerminalRunStatus(status)) null else if (frame.resourceWaitSet) frame.resourceWait else resourceWait, status = status, outcome = frame.outcome.ifBlank { outcome }, currentStage = frame.currentStage.ifBlank { currentStage }, @@ -240,6 +246,7 @@ private data class RunStateFields( } private fun RunSummary.fields() = RunStateFields( + resourceWait = resourceWait, status = status, outcome = outcome, currentStage = currentStage, @@ -250,6 +257,7 @@ private fun RunSummary.fields() = RunStateFields( ) private fun RunDetail.fields() = RunStateFields( + resourceWait = resourceWait, status = status, outcome = outcome, currentStage = currentStage, @@ -267,11 +275,12 @@ private fun executingFor(status: String, current: Boolean): Boolean = when (stat } private fun RunSummary.applying(frame: RunEventFrame): RunSummary { - if (isStaleRunFrame(status, frame.status)) return this + if (isStaleRunFrame(status, frame.status, frame.resourceWaitSet)) return this val current = fields() val merged = current.applying(frame) if (merged == current) return this return copy( + resourceWait = merged.resourceWait, status = merged.status, outcome = merged.outcome, currentStage = merged.currentStage, diff --git a/android/app/src/main/java/net/awan/pipelines/ui/screens/RunHistoryScreen.kt b/android/app/src/main/java/net/awan/pipelines/ui/screens/RunHistoryScreen.kt index 94093a1e..eacf2ace 100644 --- a/android/app/src/main/java/net/awan/pipelines/ui/screens/RunHistoryScreen.kt +++ b/android/app/src/main/java/net/awan/pipelines/ui/screens/RunHistoryScreen.kt @@ -615,6 +615,7 @@ private fun VersionChip(version: String) { /** Progress from the fields a summary actually has: completed visits + stage. */ private fun progressText(run: RunSummary): String = buildString { + run.resourceWait?.let { append(it.statusLabel()); append(" · ") } append(visitCount(run.completedSteps, "completed")) if (run.currentStage.isNotBlank()) append(" · stage ${run.currentStage}") } diff --git a/android/app/src/main/java/net/awan/pipelines/ui/screens/RunSummaryTab.kt b/android/app/src/main/java/net/awan/pipelines/ui/screens/RunSummaryTab.kt index 47e1b3db..893da31d 100644 --- a/android/app/src/main/java/net/awan/pipelines/ui/screens/RunSummaryTab.kt +++ b/android/app/src/main/java/net/awan/pipelines/ui/screens/RunSummaryTab.kt @@ -135,6 +135,7 @@ internal fun RunSummaryTab( ) KeyValueRow("Pipeline", "id ${run.pipelineId} · v${run.version}") KeyValueRow("Stage", run.currentStage) + run.resourceWait?.let { Text(it.statusLabel(), style = text.meta) } KeyValueRow("Workspace", run.workspace) KeyValueRow("Content hash", run.contentHash, monospace = true) KeyValueRow("Started", formatLocalTimestamp(run.createdAt).orEmpty()) diff --git a/android/app/src/test/java/net/awan/pipelines/data/ForegroundReconnectorTest.kt b/android/app/src/test/java/net/awan/pipelines/data/ForegroundReconnectorTest.kt new file mode 100644 index 00000000..7491ae26 --- /dev/null +++ b/android/app/src/test/java/net/awan/pipelines/data/ForegroundReconnectorTest.kt @@ -0,0 +1,138 @@ +package net.awan.pipelines.data + +import androidx.lifecycle.Lifecycle +import androidx.lifecycle.LifecycleOwner +import java.util.concurrent.CountDownLatch +import java.util.concurrent.TimeUnit +import java.util.concurrent.atomic.AtomicInteger +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.delay +import net.awan.pipelines.data.transport.ConnectionState +import org.junit.Assert.assertEquals +import org.junit.Test + +/** + * The one lifecycle-aware owner of return-to-foreground recovery: it applies the + * pure `foregroundAction` policy to the state it reads, confirms a supposedly + * connected socket before trusting it, and reconnects when — and only when — the + * policy says so. The state, the liveness check and the reconnect are injected, + * so the owner is exercised with no device, socket or Android framework. The + * owner never touches the LifecycleOwner it is handed. + */ +class ForegroundReconnectorTest { + + private val reconnects = AtomicInteger(0) + private val probes = AtomicInteger(0) + + private val owner = object : LifecycleOwner { + override val lifecycle: Lifecycle get() = throw UnsupportedOperationException("not used") + } + + private val connected = ConnectionState.Connected(build = null, serverTime = null) + + /** + * A reconnector whose probe answers [alive] without suspending, so + * `Dispatchers.Unconfined` runs the whole `onStart` decision inline and the + * assertions below hold the moment it returns. + */ + private fun reconnector( + state: () -> ConnectionState, + alive: suspend () -> Boolean = { true }, + ) = ForegroundReconnector( + state = state, + confirmAlive = { probes.incrementAndGet(); alive() }, + reconnect = { reconnects.incrementAndGet() }, + scope = CoroutineScope(Dispatchers.Unconfined), + ) + + @Test + fun aDeliberateDisconnectIsLeftAloneAndNotEvenProbed() { + reconnector(state = { ConnectionState.Idle }).onStart(owner) + assertEquals(0, probes.get()) + assertEquals(0, reconnects.get()) + } + + @Test + fun aDroppedConnectionReconnectsImmediatelyWithoutProbing() { + reconnector(state = { ConnectionState.Disconnected("socket closed", retrying = true) }).onStart(owner) + assertEquals("a dropped socket has nothing to confirm", 0, probes.get()) + assertEquals(1, reconnects.get()) + } + + @Test + fun aConfirmedConnectedSocketIsLeftAlone() { + reconnector(state = { connected }).onStart(owner) + assertEquals(1, probes.get()) + assertEquals(0, reconnects.get()) + } + + @Test + fun aConnectedSocketThatDoesNotAnswerReconnects() { + reconnector(state = { connected }, alive = { false }).onStart(owner) + assertEquals(1, probes.get()) + assertEquals(1, reconnects.get()) + } + + @Test + fun aProbeThatThrowsIsNotAlive() { + reconnector( + state = { connected }, + alive = { throw IllegalStateException("socket closed") }, + ).onStart(owner) + assertEquals(1, reconnects.get()) + } + + @Test + fun aSocketThatDiesWhileTheProbeRunsStillReconnects() { + // The socket's own failure callback lands mid-check; the owner re-reads + // the state and the policy then decides on what is actually there. + var state: ConnectionState = connected + reconnector( + state = { state }, + alive = { + state = ConnectionState.Disconnected("socket closed", retrying = true) + false + }, + ).onStart(owner) + assertEquals(1, reconnects.get()) + } + + @Test + fun aDisconnectWhileTheProbeRunsStaysDisconnected() { + // The user can tap Disconnect while the bounded foreground probe is in + // flight. Re-reading state after that probe keeps the explicit choice + // authoritative rather than reconnecting over it. + var state: ConnectionState = connected + reconnector( + state = { state }, + alive = { + state = ConnectionState.Idle + true + }, + ).onStart(owner) + assertEquals(0, reconnects.get()) + } + + @Test + fun aProbeThatNeverAnswersIsNotAliveWhenItsBudgetExpires() { + val done = CountDownLatch(1) + val reconnector = ForegroundReconnector( + state = { connected }, + confirmAlive = { delay(5_000); true }, + reconnect = { + reconnects.incrementAndGet() + done.countDown() + }, + scope = CoroutineScope(Dispatchers.Unconfined), + confirmTimeoutMillis = 20L, + ) + reconnector.onStart(owner) + assertEquals( + "the bounded check must give up on its own rather than hold the UI's Connected", + true, + done.await(2, TimeUnit.SECONDS), + ) + assertEquals(1, reconnects.get()) + } +} diff --git a/android/app/src/test/java/net/awan/pipelines/data/transport/ConnectionRecoveryTest.kt b/android/app/src/test/java/net/awan/pipelines/data/transport/ConnectionRecoveryTest.kt new file mode 100644 index 00000000..13e2bd1a --- /dev/null +++ b/android/app/src/test/java/net/awan/pipelines/data/transport/ConnectionRecoveryTest.kt @@ -0,0 +1,60 @@ +package net.awan.pipelines.data.transport + +import org.junit.Assert.assertEquals +import org.junit.Test + +/** + * The return-to-foreground policy, in the one pure place it lives: an explicit + * Disconnect is never overridden, a dropped or in-flight connection is + * re-established at once, and a still-Connected socket is trusted only after it + * has been confirmed to answer. + */ +class ConnectionRecoveryTest { + + @Test + fun aDeliberateDisconnectIsNeverOverridden() { + // Idle is only ever reached by a deliberate Disconnect (the client starts + // Idle and never returns to it by itself), so returning must leave it. + assertEquals( + ForegroundAction.LeaveAlone, + foregroundAction(ConnectionState.Idle, confirmedAlive = false), + ) + } + + @Test + fun aDroppedConnectionReconnectsAtOnce() { + assertEquals( + ForegroundAction.Reconnect, + foregroundAction(ConnectionState.Disconnected("socket closed", retrying = true), confirmedAlive = false), + ) + } + + @Test + fun anOpenAlreadyInFlightIsRestartedRatherThanWaitedOn() { + assertEquals( + ForegroundAction.Reconnect, + foregroundAction(ConnectionState.Connecting(3), confirmedAlive = false), + ) + } + + @Test + fun aConnectedSocketIsTrustedOnlyAfterItAnswers() { + val connected = ConnectionState.Connected(build = null, serverTime = null) + assertEquals(ForegroundAction.LeaveAlone, foregroundAction(connected, confirmedAlive = true)) + // The defect: a socket the process still calls Connected but that answers + // nothing must be reconnected, not trusted. + assertEquals(ForegroundAction.Reconnect, foregroundAction(connected, confirmedAlive = false)) + } + + @Test + fun aFailureAfterConnectedStartsAtTheBaseBackoff() { + // onOpen resets the client's attempt counter to zero. The next failure + // must not pass -1 to the shift used for exponential backoff. + assertEquals(1_000L, PipelinesClient.reconnectBackoffMillis(attempt = 0)) + } + + @Test + fun reconnectBackoffRemainsBounded() { + assertEquals(30_000L, PipelinesClient.reconnectBackoffMillis(attempt = Int.MAX_VALUE)) + } +} diff --git a/android/app/src/test/java/net/awan/pipelines/ui/ResourceWaitTest.kt b/android/app/src/test/java/net/awan/pipelines/ui/ResourceWaitTest.kt new file mode 100644 index 00000000..5bcdd82c --- /dev/null +++ b/android/app/src/test/java/net/awan/pipelines/ui/ResourceWaitTest.kt @@ -0,0 +1,32 @@ +package net.awan.pipelines.ui + +import kotlinx.serialization.json.Json +import net.awan.pipelines.data.protocol.ResourceWait +import net.awan.pipelines.data.protocol.RunDetail +import net.awan.pipelines.data.protocol.RunEventFrame +import net.awan.pipelines.data.protocol.RunSummary +import org.junit.Assert.* +import org.junit.Test + +class ResourceWaitTest { + @Test fun optionalMetadataAndPauseResumeClearing() { + val json = Json { ignoreUnknownKeys = true } + assertNull(json.decodeFromString("""{"id":1,"status":"queued"}""").resourceWait) + val wait = ResourceWait("phone-X", "review", 2, "2026-10-18T10:00:00Z") + val queued = RunEventFrame(runId = 1, status = "queued", resourceWaitSet = true, resourceWait = wait) + val detail = foldRunEvent(RunDetail(id = 1, status = "running"), queued) + assertEquals("queued", detail.status) + assertEquals(wait, detail.resourceWait) + assertEquals("Waiting for resource phone-X", wait.statusLabel()) + val rows = foldRunEvent(listOf(RunSummary(id = 1, status = "running")), queued).runs + assertEquals(wait, rows.single().resourceWait) + val clear = RunEventFrame(runId = 1, status = "running", resourceWaitSet = true) + assertNull(foldRunEvent(detail, clear).resourceWait) + assertNull(foldRunEvent(rows, clear).runs.single().resourceWait) + assertNull(foldRunEvent(detail, RunEventFrame(runId = 1, status = "canceled")).resourceWait) + assertEquals("succeeded", foldRunEvent(RunDetail(id = 1, status = "succeeded"), queued).status) + val decoded = json.decodeFromString("""{"id":1,"status":"queued","resource_wait":{"resource":"phone-X","stage":"review","sequence":2,"queued_since":"now"}}""") + assertEquals("phone-X", decoded.resourceWait?.resource) + assertFalse(isWaitingRunStatus(decoded.status)) + } +} diff --git a/android/gradle/libs.versions.toml b/android/gradle/libs.versions.toml index c1d8f4df..9932d8a4 100644 --- a/android/gradle/libs.versions.toml +++ b/android/gradle/libs.versions.toml @@ -23,6 +23,7 @@ androidx-compose-material3 = { group = "androidx.compose.material3", name = "mat androidx-compose-material-icons-extended = { group = "androidx.compose.material", name = "material-icons-extended" } androidx-navigation-compose = { group = "androidx.navigation", name = "navigation-compose", version.ref = "navigationCompose" } androidx-lifecycle-runtime-compose = { group = "androidx.lifecycle", name = "lifecycle-runtime-compose", version.ref = "lifecycle" } +androidx-lifecycle-process = { group = "androidx.lifecycle", name = "lifecycle-process", version.ref = "lifecycle" } androidx-lifecycle-viewmodel-compose = { group = "androidx.lifecycle", name = "lifecycle-viewmodel-compose", version.ref = "lifecycle" } androidx-lifecycle-viewmodel-ktx = { group = "androidx.lifecycle", name = "lifecycle-viewmodel-ktx", version.ref = "lifecycle" } kotlinx-coroutines-android = { group = "org.jetbrains.kotlinx", name = "kotlinx-coroutines-android", version.ref = "coroutines" } diff --git a/cmd/pipelines-server/main.go b/cmd/pipelines-server/main.go index ec789886..54415ddf 100644 --- a/cmd/pipelines-server/main.go +++ b/cmd/pipelines-server/main.go @@ -47,7 +47,7 @@ func main() { // Each new run gets a bounded one-line title from the configured default // harness; the summariser falls back to the prompt's first line, so title // generation never blocks or fails run creation. - options := server.Options{ArtifactsRoot: artifactsRoot, RunTitles: summarise.GenerateTitle, StepVerdicts: summarise.GenerateVerdictReason} + options := server.Options{ArtifactsRoot: artifactsRoot, RunTitles: summarise.GenerateTitle, StepVerdicts: summarise.GenerateVerdictReason, VerificationFixtures: server.VerificationFixturesFromEnvironment(os.Getenv)} recorded := "" if value, err := store.OpenCodeEndpoint(ctx); err != nil { log.Printf("OpenCode endpoint setting unreadable: %v", err) diff --git a/docs/notes/background-shell-visibility.md b/docs/notes/background-shell-visibility.md new file mode 100644 index 00000000..583e5f90 --- /dev/null +++ b/docs/notes/background-shell-visibility.md @@ -0,0 +1,37 @@ +# Background shell visibility + +The chat server now opens the broker stream for read-only subscribers and reconciles verified running shell operations from OpenCode's authoritative `/api/shell` readback. The packaged OpenCode skill references and chat documentation should describe that shell visibility survives reconnects only when OpenCode reports explicit `metadata.sessionID` ownership; shared working directories are not ownership evidence. + +## Current source behavior + +Observers share one initial readback per room/stream attachment. The adapter and hub +fold bounded lifecycle journals over each snapshot, so a stale read cannot resurrect +an exited shell or erase a concurrent create. Shell replay and live delivery enqueue +in fold order. Cancellation rejects old-room callbacks and releases local identity +state before reconnect. Journals retain at most 256 identities and 1 MiB of id/command +text; capacity/read failures surface stable limited-visibility text, never broker +response bodies. No shell polling was added. OpenCode's terminal `timeout` and +`killed` statuses are excluded from retained running operations as well as `exited`. + +## Verification evidence — 2026-10-18 + +`opencode api get /openapi.json` authenticated through supported service discovery. +The live schema advertises shell list/detail/output and generic `metadata`; it does +not type `metadata.sessionID`. A throwaway session +`ses_eb12ab7b3ffe0ZAle7PHpftHMG` and its own `sleep 1` shell +`sh_14ed57134001e6fuYqvn62pKit` verified actual create/detail readback preserving +explicit `metadata.sessionID`, status `running`, subsequent `exited`/exit 0, and +absence from the running list. This verifies explicit API metadata, not automatic +agent-tool ownership injection. The CLI event attempt yielded no captured events; +live-provider event ordering is **not tested**. Forced ordering evidence comes from +channel/barrier regressions, including a real WebSocket handler on a loopback scratch +server with fresh SQLite, not from the deployed service. + +## Documentation follow-up / deployment + +`docs/chat-verification.md` and the chat subscribe/replay section of `docs/cli.md` +are stale: docs-fold should describe authoritative ownership/readback, coalesced +observation, bounded reconciliation and limited-visibility errors, distinguishing +fixture ordering evidence from the live readback above. Shared docs and packaged +skill references were not edited here. No deployment, installation, service restart +or physical-phone/UI verification was performed; this changes no rendered layout. diff --git a/docs/notes/delivery-stabilization.md b/docs/notes/delivery-stabilization.md new file mode 100644 index 00000000..bba21fd9 --- /dev/null +++ b/docs/notes/delivery-stabilization.md @@ -0,0 +1,47 @@ +# Delivery stabilization + +Date: 2026-10-18. + +The server now has draft-only `delivery=true` support for OpenCode model stages. +At run time it derives the canonical `main` worktree and linked-worktree Git +metadata from Git rather than accepting paths from a stage prompt. It verifies +the committed item tree in a disposable candidate worktree before fast-forwarding +canonical `main`; declared checks are bounded configuration, not proof that any +deployed definition has used them. It is denied for Pi (until equivalent +capability enforcement exists), foreign repositories, missing main worktrees, +and malformed capabilities. This item does not publish or deploy a definition. + +The source also supports an optional controlled-fixture verification declaration +(`backend`, `revision`, `capability`, and `fixture`). Ordinary server setup wires +an explicit, bounded `reconnect-foreground` HTTP fixture registry when +`PIPELINES_VERIFICATION_RECONNECT_FOREGROUND_URL` and its expected `BACKEND`, +`REVISION`, `CAPABILITY`, and `STATE` values are configured. There is no default +remote URL. The server, not the model, probes that registered endpoint and +collects its returned backend identity, build revision, capability, fixture ID, +and state before comparing provenance. Unknown fixtures, incomplete setup, +wrong backend/revision/capability/state, non-200 responses, cancellation, and +oversized or malformed responses fail closed. A model result's labels are not +evidence. This is controlled local/backend fixture infrastructure, not evidence +of a rendered layout or a live Android device. +Definitions must be scratch-validated against the deployed server only after +the server support is deployed; no future delivery or verification definition +is activated by this source change. + +`internal/author/skill/references/format.md` and +`internal/author/skill/references/queue.md` are stale: a later docs fold should +describe `delivery=true`, OpenCode-only availability, candidate verification, +and the required activation order (deploy server support before scratch-validating +and then publishing a definition). +OpenCode V2 resolves external resources at a containing directory, so delivery +uses the verified repository metadata directory rather than an impossible +single-lock-file scope. It does not grant a parent, home, foreign repository, +or global policy. Unrestricted shell permissions are not an OS sandbox. + +Activation remains draft-only: deploy compatible server support, configure the +controlled fixture endpoint and matching provenance, scratch-validate the +fixture, then publish a definition. No definition is activated by this source +change. + +Recovery checklist: retain `connection-reconnect` and unfinished feature +branches; merge only complete committed trees that pass mapped checks; do not +infer delivery from a dirty worktree or an old deployed-backend capture. diff --git a/docs/notes/foreground-reconnect.md b/docs/notes/foreground-reconnect.md new file mode 100644 index 00000000..1125f0e5 --- /dev/null +++ b/docs/notes/foreground-reconnect.md @@ -0,0 +1,95 @@ +# Android: coming back to the app restores the connection + +## What this item changed + +Android app only. + +- `data/transport/ConnectionRecovery.kt` (new): `ForegroundAction` and the one + pure `foregroundAction(state, confirmedAlive)` policy — `Idle` → leave alone (a + deliberate Disconnect is the only way the client reaches Idle), `Connected` with + a confirmed-alive socket → leave alone, `Connected` unconfirmed → reconnect, + `Connecting`/`Disconnected` → reconnect. +- `data/ForegroundReconnector.kt` (new): the one lifecycle-aware owner. It is + created once with the Application and observes `ProcessLifecycleOwner`, never + the Activity or the ViewModel (both recreated). On `ON_START` it applies the + policy: a non-Connected state reconnects at once, and the reconnect cancels the + client's pending backoff delay, so the operator never waits it out; a state + that still claims `Connected` is first confirmed with the existing `ping`, + bounded to 3 s, and the state is re-read afterwards (the socket's own failure + callback can land while the check is in flight). +- `PipelinesApplication.kt`: wires the owner to the process lifecycle; + `confirmAlive` is `repository.ping()`, `reconnect` is + `ServerUrl.validate(...) + repository.recover(...)`. Recovery shares the + client's state lock with Disconnect, so a Disconnect after the probe has made + its decision but before its reconnect effect runs stays authoritative. Manual + Connect/Reconnect continue to use `repository.connect(...)`; the client's + bounded backoff stays the only reconnect mechanism and the policy lives in the + one pure place. +- `data/transport/PipelinesClient.kt`: `open()` now closes the socket it + supersedes, and every `WebSocketListener` callback ignores a socket that is no + longer the current one (`isCurrent`). A failure the replaced socket reports late + — the common case when the process was frozen while it died — can then neither + clear the new connection nor schedule a reconnect that would undo the recovery. +- `app/build.gradle.kts` and `gradle/libs.versions.toml`: + `androidx.lifecycle:lifecycle-process` 2.8.7 (already in the offline cache, + with its `androidx.startup`/`androidx.tracing` transitives). +- Tests: `data/transport/ConnectionRecoveryTest.kt` (5 cases) and + `data/ForegroundReconnectorTest.kt` (8 cases). + +## Shared documents this change makes stale + +`android/README.md`. + +- The Configuration section (`android/README.md:99-101`) — "Tap **Connect** to + open the single shared WebSocket (probed with `ping`, showing the server + build). Reconnect uses bounded exponential backoff; **Disconnect** stops it." — + should record that coming back to the app restores the connection on its own: + the process coming to the front reconnects immediately, cancelling the pending + backoff delay, and when the last state still claims Connected it first confirms + the socket with `ping` before trusting it. An explicit Disconnect stays + authoritative and is never reconnected over; the manual Reconnect control + stays, but normal recovery never needs it. +- The Architecture block (`android/README.md:310-323`) lists `data/transport` and + `data`. It should name the two new files — `data/transport/ConnectionRecovery.kt` + (the pure return-to-foreground policy) and `data/ForegroundReconnector.kt` (the + process-lifecycle owner) — and the new `androidx.lifecycle:lifecycle-process` + dependency behind the owner. + +## Not stale + +- `android/AGENTS.md` names neither the presence nor the absence of a lifecycle + observer; its "Where things are" summary stays correct (it could optionally add + the new owner beside `data/PipelinesRepository.kt`). +- `docs/app-integration-verification.md` — its connection mentions are about + restoring the stub URL, which this item does not change. +- No packaged OpenCode skill reference (`internal/author/skill/references/`) + documents the app's connection lifecycle, so nothing there is stale. + +## Verified on the device, and what was not + +Built and captured on the operator's Pixel 9 Pro XL (`47241FDAS00DLK`) from +`51a2ba4` + this change; after merging main the same tree (now `b93eee2`) was +rebuilt and the full unit suite re-run (691 tests, 0 failures). The operator's +`ws://10.0.0.75:8080/connect` and `/Users/alan/repos/pipelines` were used +throughout and were not changed. The phone was left on main's build. + +Captured, all with `android/scripts/capture_png.sh` (every image validated): + +- a **live** socket on return: the server log shows a `ping`/`pong` on the same + connection at the second of the return and no reconnect — the positive check + runs and a healthy connection is left alone; +- a **dead** socket on return: the reconnect and re-subscribe land in the same + second as the return, with no interaction (the process was frozen, so the + client's own backoff had not run); +- **no network** on return: the banner reads "Disconnected: Failed to connect to + …:8080 · retrying with backoff" — never a false Connected — and recovery + follows on its own once the network returns; +- an explicit **Disconnect**: still "Not connected. Enter the server URL in + Settings, then connect." after returning, with no reconnect in the server log. + +**Not verified live:** the stale-Connected probe branch. In every capture the +frozen socket's own failure was delivered to the client before the lifecycle +owner ran, so the client already read `Disconnected` and no probe was attempted +(confirmed by the client's request-id sequence never skipping a value in the +server log). That branch — `Connected` still claimed, probe fails, reconnect — is +covered by unit tests only. diff --git a/docs/notes/phone-resource-wait.md b/docs/notes/phone-resource-wait.md new file mode 100644 index 00000000..33d6b3d2 --- /dev/null +++ b/docs/notes/phone-resource-wait.md @@ -0,0 +1,158 @@ +# Stage-exclusive resources — 2026-10-18, not deployed + +This is source work for run 355, **not a deployment, installation or publication**. +No physical phone, real operator session or deployed service was used or changed. +The Android role split does not retrospectively gate pinned versions or existing runs. + +## Contract and scope + +An executable non-human stage can declare one runner-owned control block: + +```resources +{"exclusive":"phone-47241FDAS00DLK"} +``` + +The key is 1–128 ASCII identifier bytes (letter first, then letters, digits, `_`, `-`). +Duplicate/unknown fields, multiple resources and human-stage resources are rejected. +One capacity-one resource per stage visit, one server/store authority; no expressions, +nested acquisition, distributed scheduling, capacity settings or multi-resource requests. +Resource-free definitions retain their execution path. Old servers reject this new block; +do not publish to an incompatible server. + +Acquisition follows selection and automatic UI-skip evaluation, before BeginStep, +visits/sequences, artifacts, filediff, stage timeout, budget setup or executor calls. +Queued runs retain workspace reservation, use legacy `queued` (never human `waiting`), +and carry bounded `resource_wait` identity/stage/sequence/queued-since metadata. +Events use `resource_wait_set=true` with absent `resource_wait` to clear caches. +CLI lists and TUI/Android details show “Waiting for resource …”; human decisions and +attention notifications remain separate. Older clients still see an unfinished run. + +SQLite owners and FIFO waiters are indexed by resource/order and run identity. +Private random capabilities attribute ownership to resource/run/stage/sequence and +never enter wire data, model input, transcripts or artifacts. Only the full capability +and positive cleanup evidence authorize release; mismatches and repeated release cannot +release a subsequent owner. One buffered channel per live queued run, no resource +polling, waiter tickers, model-based acquisition, queue driver or new daemon. + +The gate holds through tool cleanup. OpenCode interruption now confirms session idle/ +V1 settlement before clearing session ownership. Pi recovery requires both its receipt +and process/process-group exit, not merely abort acknowledgement. Executor panic, +ErrStageCleanup, retained active sessions and failed cleanup queries quarantine the owner. +An explicit `output.resource_cleanup_confirmed=false` (or malformed value) also vetoes +release for failed restoration; true never substitutes for harness cleanup evidence. +The generic runner does not inspect Android settings or magically verify restoration. +The Android UI reviewer must keep all device work foreground/awaited, restore original +settings/connection and main's build, and report uncertainty with the release veto. + +## Restart and retention + +Automatic FIFO progress is promised only within a live server process. Startup interrupts +persisted queued resource visits, removes their waiter rows, and never replays an executor. +Durably cleanup-confirmed owners can be reclaimed through the token-checked delete helper. +Unconfirmed owners remain quarantined, including crashes before BeginStep, before session +registration, or after cleanup but before its confirmation is persisted. Session interruption +alone cannot prove resource-specific restoration. Age never authorizes reclamation. +Quarantined ownership/session evidence survives recovery and protects even terminal runs +from retention. Reconciliation requires inspected cleanup and restoration, not lock deletion. +There is intentionally no force-unlock CLI or automatic uncertain-owner reclamation. + +## Android draft, provenance and activation + +Read-only retrieval: `pipelines show Android --version 2 --json` returned pipeline 20, +v2, hash `8c93aeb9b76f1e6e7696415d6308d786ec08d805a1a346fbabe4d252d5d65032`. +`python3` recomputed SHA-256 of the returned source and asserted equality. +The exact source is retained as `internal/server/testdata/android-v2-published.md`; +the proposed source is `internal/server/testdata/android-resource-draft.md`. +The provenance test compares graph fields/outcomes unchanged, apart from the approved +three stage prompts and the UI review resource; every stage keeps its published v3 model: Terra for implementer/committer and Luna for reviewer/UI reviewer. No unrelated published definition was edited. + +Artifact handoff recovery: the validated v3 draft was copied to this run's retained +artifact directory as `android-resource-draft.md`. The similarly named artifact from +run 355 remains the old v2 baseline and must not be published. The proposed source in +this workspace and this run's artifact have matching SHA-256 digests +`a10d8b9fa3dc6a1875b10469061ed25fb4a7a404d463d1a2d304a309d55aa5d9`. + +Activation, requiring separate explicit operator authorization: +1. Review/land the source change and independently rerun the required checks. +2. Arrange a compatible deployment separately; nothing here deploys or restarts it. +3. Reconcile every existing phone user, pinned/live run and any quarantined owner. +4. Transfer the **workspace proposed draft**, provenance and this note into the retained + stage artifacts after resolving the artifact edit permission. +5. Validate that retained draft against the configured authoritative compatible endpoint; + only then explicitly publish it. Do not use a disposable endpoint as a project default. +6. Opt in other device users explicitly before relying on mutual exclusion system-wide. + +Read-only published inventory (`pipelines show --json`, v2 for each): +- UI sweep: `survey` and `ui_reviewer` request visual/screenshots work; phone work there + needs explicit resource opt-in and restoration/fixture policy. +- Generic: `reviewer` and `ui_reviewer` have visual/device verification wording; when + targeting a phone, separate code review from leased device review first. +- Plan and land: `implementer`, `reviewer`, `ui_reviewer` contain rendered-surface work; + they need an explicit technology-specific role split/lease before phone use. +- Probe: one harness smoke stage; no default phone declaration. Any device-specific + probe needs explicit opt-in, never an inferred resource or Android stage-name special case. +Server/TUI/other definitions were not changed. Inventory is not activation evidence. + +## Verification evidence and remaining limits + +Executed focused checks: +- Run 358 recovery revalidation: `go build ./... && go vet ./... && go test ./...` + passed; `go test -race ./internal/pipeline ./internal/server ./internal/stage -run + 'Resource|Scheduler|Concurrent|Human|Decision|RetryWaits' -count=1` passed; and the + Android map invocation with `JAVA_HOME`/`ANDROID_HOME`, installed Gradle, + `--offline --no-daemon --rerun-tasks :app:assembleDebug :app:testDebugUnitTest` + executed all 41 tasks and reported `BUILD SUCCESSFUL`. These remain build/unit and + fake-executor scheduler evidence, + not physical-phone verification or screenshots. +- Recovery after integrating main `88595eb`: `go build ./...`, `go vet ./...`, + `go test ./internal/server -run Resource -count=1`, + `go test ./internal/pipeline ./internal/protocol ./internal/tui -run Resource -count=1`, + `go test ./internal/stage -count=1`, + `go test -race ./internal/pipeline ./internal/server ./internal/stage -run 'Resource|Scheduler|Concurrent|Human|Decision|RetryWaits' -count=1`, and + `go test ./...` all passed. `gofmt -l .` and `git diff --check` produced no output. +- Recovery Android build/unit check with the map's offline, no-daemon Gradle invocation: + `:app:assembleDebug :app:testDebugUnitTest` passed. This remains build/unit evidence, + not phone verification or screenshots. +- `go test ./internal/server ./internal/pipeline ./internal/protocol ./internal/tui -run Resource -count=1`: + passed. Includes fake-executor queue >30ms timeout, independent-resource progress, + FIFO/mismatch/duplicate/grant-cancel cycles, blocked cleanup, panic/quarantine, + persistence/BeginStep/query failure, no-acquire skip, restart/retention, wire defaults, + narrow/wide/plain TUI frames and real CLI/WebSocket scratch exercise. +- `go test ./internal/stage -count=1`: passed; the later full suite also included + `TestResourcePiReceiptRequiresProcessExit` (a test-owned sleep process, not a device). +- Android map invocation with `JAVA_HOME`/`ANDROID_HOME` and installed Gradle, + `--offline --no-daemon :app:assembleDebug :app:testDebugUnitTest`: BUILD SUCCESSFUL. + These are build/unit checks, not device verification or screenshots. +- After the Go tree settled: `go build ./...`, `go vet ./...`, `go test ./...` + all exited zero (including author-package tests). +- `go test -race ./internal/pipeline ./internal/server ./internal/stage -run 'Resource|Scheduler|Concurrent|Human|Decision|RetryWaits' -count=1`: + all three packages passed, including concurrent grant/cancel cycles. +- `gofmt -l .` and `git diff --check`: exited zero with no output. + +The scratch CLI test uses the real CLI and WebSocket handler with temporary SQLite and +an explicitly **FAKE executor**. It validates the proposed Android draft on compatible +scratch code, observes B as queued with zero steps while A holds fake-phone, inspects +subscription metadata, releases A, runs `wait`, and observes B succeeded automatically. +It does not call a model or use a physical phone. Test log output is evidence, not a +live-deployment claim; no retained CLI transcript was written to the denied artifact path. + +Live OpenCode/Pi shell/ADB termination and Android visual/device restoration are **not +tested** here. The checks prove scheduler behavior with controlled fakes, plus test-owned +process-exit checks, not device readiness on the current deployment. +Commit/merge/landing belong to the later committer stage; no commit or push here. + +## Stale shared documentation for docs-fold (do not edit in this item) + +- `docs/cli.md`: resource queue presentation, inspection metadata, unchanged wait/decide semantics. +- `internal/author/skill/references/format.md`: optional resources block and validation/key bounds. +- `internal/author/skill/references/control.md`: runner-owned declaration, timeout/accounting boundary, + release veto and automatic skip placement. +- `internal/author/skill/references/queue.md`: FIFO resource waits vs workspace exclusion and human waits. +- `internal/author/skill/references/operations.md`: authority, restart interruption/quarantine, + inspected recovery, retention and explicit compatible-deployment/publication prerequisites. +- `internal/author/skill/references/verification.md`: fake-vs-device evidence, fixtures and leased restoration. +- `internal/author/skill/references/orchestration.md`: no queue drivers/polling/extra agents; explicit + opt-in of every device user, no retroactive gating or distributed guarantees. + +No packaged skill/reference, shared README/topic documentation, pipeline registry, +installation, deployment, provider setting or `internal/stage/model_failure.go` was edited. diff --git a/docs/notes/scoped-artifact-access.md b/docs/notes/scoped-artifact-access.md new file mode 100644 index 00000000..75943280 --- /dev/null +++ b/docs/notes/scoped-artifact-access.md @@ -0,0 +1,20 @@ +# Scoped stage artifact access + +## What this item changed + +The server now derives one exact artifact capability for an OpenCode or Pi +model-stage visit: its own directory is writable when the edit tool is present, +and its own plus completed earlier visit directories in the same run are +readable when a file-reading tool is present. No artifact root, sibling, future +visit, other run, traversal path, or symlink escape is granted. The scope is +empty if the runner cannot validate its artifact root. + +Pi retains its lease and explicit extension policy; unrestricted shell remains +outside this file-tool boundary and is not an OS sandbox. + +## Shared documents this change makes stale + +- `docs/stage-artifacts.md` and `internal/author/skill/references/operations.md` + should explain that model-stage artifact directories are usable by the + supported file tools only within the exact same-run visit scope, not as a + general external-directory permission. diff --git a/docs/notes/stage-timeout-diagnostics.md b/docs/notes/stage-timeout-diagnostics.md new file mode 100644 index 00000000..a9989418 --- /dev/null +++ b/docs/notes/stage-timeout-diagnostics.md @@ -0,0 +1,42 @@ +# Stage-timeout diagnostics + +## Source behavior + +The V2 active-session wait now reports `STAGE DEADLINE` when the configured +attempt timer expires between successful polls of a running admitted session. +Its bounded diagnostic names the idle-wait phase, configured timeout, retained +session, and that the stage had not finished (active at the last poll). It does +not claim that a completed response was lost. Cancellation is no longer recorded +as a stage deadline. A bare deadline alone is not `HARNESS TRANSPORT`. + +Actual request/HTTP/operation failures keep `HARNESS TRANSPORT`, their sanitized +operation/status evidence and underlying error identities, including when joined +with a deadline. A parent deadline no longer discards an accompanying transport +failure. Final-result validation, budget accounting, cleanup, interrupts, +side effects and replay policy are unchanged. No timeout was increased. + +## Documentation to fold manually + +`internal/author/skill/references/debugging.md` needs a separate stage-deadline +playbook alongside its HARNESS TRANSPORT section: inspect the retained session, +last activity, configured limits and unfinished final-result evidence before +deciding what to do. Do not blindly retry an admitted mutating stage; edits/tool +side effects remain. Explain the distinction between a configured idle-wait +overrun, cancellation, an actual failed request with a deadline, budget failure, +and final-result validation. Do not infer that all deadline errors are transport +failures or that a successful last tool means the stage returned a result. + +## Evidence and deployment + +2026-10-18: installed server build metadata (`go version -m +/Users/alan/.local/share/pipelines/bin/pipelines-server`) identifies revision +`0d1ddc73a444fa24110ba5223646856c8f053d29`. Comparing that revision with this +worktree's initial HEAD showed identical model-failure classification, model +attempt timing, V2 polling and agent wait/cleanup paths. The runner difference +was unrelated session-export HTML-only metadata. + +Regression tests use isolated loopback servers and scratch SQLite databases, +with a controlled V2 CLI fixture for the admitted-active-session deadline and +real loopback HTTP/refused-connection checks for native transport evidence. +This is not a live-provider timeout test. No deployment or deployed-service +restart is part of this item; retained historical runs are not rewritten. diff --git a/internal/cli/app.go b/internal/cli/app.go index 8311ca35..5e26e393 100644 --- a/internal/cli/app.go +++ b/internal/cli/app.go @@ -774,7 +774,7 @@ func (a App) Run(ctx context.Context, args []string) error { } case "runs": for _, run := range r.Runs { - fmt.Fprintf(a.Out, "%d\t%s\t%s\tv%d\t%s\n", run.ID, run.Label(), run.Name, run.Version, run.Status) + fmt.Fprintf(a.Out, "%d\t%s\t%s\tv%d\t%s\n", run.ID, run.Label(), run.Name, run.Version, run.StatusLabel()) } case "ping": fmt.Fprintln(a.Out, "Connected ·", r.Time) @@ -1484,7 +1484,7 @@ func (e *waitLimitError) Runs() string { // waitLine is the one-line terminal report `pipelines wait` prints: the run id, // its status and outcome, its stage and how long it took. func waitLine(d *protocol.RunDetail) string { - line := fmt.Sprintf("run %d · %s", d.ID, d.Status) + line := fmt.Sprintf("run %d · %s", d.ID, d.StatusLabel()) if outcome := strings.TrimSpace(d.Outcome); outcome != "" { line += " (" + outcome + ")" } diff --git a/internal/opencode/chat.go b/internal/opencode/chat.go index 40adc005..8fae8cd9 100644 --- a/internal/opencode/chat.go +++ b/internal/opencode/chat.go @@ -28,7 +28,9 @@ var ErrPermissionUnsupported = errors.New("OpenCode permission approval is not i // semantics, the session-id shape and the 64 KiB / 1 MiB bounds of Connection so // no second transport or credential path is introduced. type Chat struct { - conn *Connection + conn *Connection + shellMu sync.Mutex + shells map[string]*ChatStreamState // mu serializes the adapter's HTTP calls. Connection records its last // operation and status on the shared connection, so two calls must not // overlap; the permission poller runs alongside the read loop's @@ -294,6 +296,114 @@ func decodeChatPermissions(sessionID string, data []byte) ([]protocol.ChatPermis return frames, nil } +// Shells reads the endpoint's authoritative shell list and returns only running +// operations whose metadata explicitly names sessionID. The shell API is global; +// filtering locally avoids guessing an unsupported query filter and never assigns +// ownership from a shared working directory. A successful read also seeds the +// stream state used to attribute later shell.exited events, which carry no session. +func (c *Chat) Shells(ctx context.Context, sessionID string) ([]protocol.ChatShell, error) { + if !ValidSessionID(sessionID) { + return nil, fmt.Errorf("invalid OpenCode session id") + } + state, err := c.streamStateContext(ctx, sessionID) + if err != nil { + return nil, err + } + // One read per state; waiting and I/O never hold the event-state mutex. + state.mu.Lock() + if state.readGate == nil { + state.readGate = make(chan struct{}, 1) + } + gate := state.readGate + state.mu.Unlock() + select { + case gate <- struct{}{}: + case <-ctx.Done(): + return nil, ctx.Err() + } + defer func() { <-gate }() + state.mu.Lock() + state.readback = &ShellSnapshot{} + state.mu.Unlock() + defer func() { state.mu.Lock(); state.readback = nil; state.mu.Unlock() }() + data, err := c.call(ctx, "get", "/api/shell", nil) + if err != nil { + return nil, err + } + var response struct { + Data []chatShellInfo `json:"data"` + } + if err := json.Unmarshal(data, &response); err != nil || response.Data == nil { + if err := json.Unmarshal(data, &response.Data); err != nil { + return nil, fmt.Errorf("invalid OpenCode shell response") + } + } + out := make([]protocol.ChatShell, 0, len(response.Data)) + for _, info := range response.Data { + if info.ID == "" || info.shellSessionID() != sessionID || (info.Status != "" && info.Status != protocol.ChatShellRunning) { + continue + } + status := info.Status + if status == "" { + status = protocol.ChatShellRunning + } + out = append(out, protocol.ChatShell{ID: info.ID, Status: status, Command: info.Command}) + } + state.mu.Lock() + defer state.mu.Unlock() + out, err = state.readback.Merge(out) + if err != nil { + return nil, err + } + if err := ctx.Err(); err != nil { + return nil, err + } + ids := make([]string, 0, len(out)) + for _, shell := range out { + ids = append(ids, shell.ID) + } + state.replaceShellsLocked(ids) + return out, nil +} + +// streamState returns the session's retained, verified shell identities. It is +// deliberately kept across stream replacement so an exit after reconnect remains +// attributable; Shells reconciles its contents from authoritative readback on +// each attach rather than replacing the pointer a live scanner is using. +func (c *Chat) streamState(sessionID string) *ChatStreamState { + state, _ := c.streamStateContext(context.Background(), sessionID) + return state +} + +func (c *Chat) streamStateContext(ctx context.Context, sessionID string) (*ChatStreamState, error) { + c.shellMu.Lock() + defer c.shellMu.Unlock() + if err := ctx.Err(); err != nil { + return nil, err + } + if c.shells == nil { + c.shells = map[string]*ChatStreamState{} + } + state := c.shells[sessionID] + if state == nil { + state = &ChatStreamState{} + c.shells[sessionID] = state + } + return state, nil +} + +// ReleaseShells discards verified shell identities after the last room observer +// disconnects. A future attach calls Shells for a fresh authoritative snapshot, +// so this bounds state without attributing a foreign shell during the gap. +func (c *Chat) ReleaseShells(sessionID string) { + if sessionID == "" { + return + } + c.shellMu.Lock() + delete(c.shells, sessionID) + c.shellMu.Unlock() +} + // Stream subscribes to the OpenCode event firehose and invokes emit for every // frame belonging to sessionID. It reads one bounded SSE frame at a time and // never polls the per-session message log; the global firehose is the only @@ -306,6 +416,12 @@ func (c *Chat) Stream(ctx context.Context, id string, ready func(), emit func(pr if !sessionID.MatchString(id) { return fmt.Errorf("invalid OpenCode session id") } + // Pin state before opening/announcing the stream so a canceled scanner + // cannot acquire a replacement room's identity map. + state, err := c.streamStateContext(ctx, id) + if err != nil { + return err + } body, err := c.conn.EventStream(ctx) if err != nil { return err @@ -314,7 +430,7 @@ func (c *Chat) Stream(ctx context.Context, id string, ready func(), emit func(pr if ready != nil { ready() } - return scanChatEvents(ctx, body, id, emit) + return c.scanChatEventsWithState(ctx, body, id, state, emit) } // scanChatEvents turns an SSE body into protocol frames, one bounded frame at a @@ -322,10 +438,13 @@ func (c *Chat) Stream(ctx context.Context, id string, ready func(), emit func(pr // malformed JSON fails the stream instead of producing a malformed frame. The // last assistant message id is retained so a later tool/step event that omits it // still attributes every frame to a stable message. -func scanChatEvents(ctx context.Context, body io.Reader, sessionID string, emit func(protocol.ChatEventFrame) error) error { +func (c *Chat) scanChatEvents(ctx context.Context, body io.Reader, sessionID string, emit func(protocol.ChatEventFrame) error) error { + return c.scanChatEventsWithState(ctx, body, sessionID, c.streamState(sessionID), emit) +} + +func (c *Chat) scanChatEventsWithState(ctx context.Context, body io.Reader, sessionID string, state *ChatStreamState, emit func(protocol.ChatEventFrame) error) error { scanner := bufio.NewScanner(body) scanner.Buffer(make([]byte, 0, 64<<10), MaxResponse) - state := &ChatStreamState{} for scanner.Scan() { if err := ctx.Err(); err != nil { return err @@ -346,7 +465,7 @@ func scanChatEvents(ctx context.Context, body io.Reader, sessionID string, emit continue } if frame.MessageID != "" { - state.LastMessageID = frame.MessageID + state.setLastMessageID(frame.MessageID) } if err := emit(frame); err != nil { return err @@ -449,8 +568,29 @@ func (i *chatShellInfo) shellSessionID() string { // what attributes the exit to its session: another session's shell is an // unknown id and is ignored. type ChatStreamState struct { + mu sync.Mutex LastMessageID string Shells map[string]struct{} + readGate chan struct{} + readback *ShellSnapshot +} + +func (s *ChatStreamState) lastMessageID() string { + if s == nil { + return "" + } + s.mu.Lock() + defer s.mu.Unlock() + return s.LastMessageID +} + +func (s *ChatStreamState) setLastMessageID(id string) { + if s == nil || id == "" { + return + } + s.mu.Lock() + s.LastMessageID = id + s.mu.Unlock() } // knowsShell reports whether id is a shell this stream has already surfaced for @@ -459,6 +599,8 @@ func (s *ChatStreamState) knowsShell(id string) bool { if s == nil || id == "" { return false } + s.mu.Lock() + defer s.mu.Unlock() _, ok := s.Shells[id] return ok } @@ -468,6 +610,8 @@ func (s *ChatStreamState) markShell(id string) { if s == nil || id == "" { return } + s.mu.Lock() + defer s.mu.Unlock() if s.Shells == nil { s.Shells = map[string]struct{}{} } @@ -476,12 +620,37 @@ func (s *ChatStreamState) markShell(id string) { // forgetShell drops id once its exit has been surfaced. func (s *ChatStreamState) forgetShell(id string) { - if s == nil || s.Shells == nil { + if s == nil { + return + } + s.mu.Lock() + defer s.mu.Unlock() + if s.Shells == nil { return } delete(s.Shells, id) } +// replaceShells applies one authoritative running-shell snapshot without +// replacing the state object an already attached event scanner owns. +func (s *ChatStreamState) replaceShells(ids []string) { + if s == nil { + return + } + s.mu.Lock() + defer s.mu.Unlock() + s.replaceShellsLocked(ids) +} + +func (s *ChatStreamState) replaceShellsLocked(ids []string) { + s.Shells = make(map[string]struct{}, len(ids)) + for _, id := range ids { + if id != "" { + s.Shells[id] = struct{}{} + } + } +} + // NormalizeChatEvent maps one OpenCode firehose `data:` payload to a // protocol.ChatEventFrame, folding the small per-session context ([stream]) it // needs across frames. ok is false for an event type this adapter does not @@ -528,7 +697,7 @@ func NormalizeChatEvent(payload []byte, sessionID string, stream *ChatStreamStat messageID = data.MessageIDAlt } if messageID == "" && stream != nil { - messageID = stream.LastMessageID + messageID = stream.lastMessageID() } // Every pushed frame must carry a stable message id so the client can // attribute it; without one this is not a chat frame. @@ -590,20 +759,38 @@ func normalizeShellEvent(eventType string, data chatEvent, sessionID string, str if status == "" { status = protocol.ChatShellRunning } - stream.markShell(data.Info.ID) + shell := &protocol.ChatShell{ID: data.Info.ID, Status: status, Command: data.Info.Command} + if stream != nil { + stream.mu.Lock() + stream.readback.Record(shell) + if stream.Shells == nil { + stream.Shells = map[string]struct{}{} + } + if status == protocol.ChatShellRunning && len(stream.Shells) < chatShellIdentityLimit { + stream.Shells[shell.ID] = struct{}{} + } + stream.mu.Unlock() + } return protocol.ChatEventFrame{ SessionID: sessionID, Kind: protocol.ChatEventShell, - Shell: &protocol.ChatShell{ID: data.Info.ID, Status: status, Command: data.Info.Command}, + Shell: shell, }, true, nil } if data.ID == "" { return protocol.ChatEventFrame{}, false, fmt.Errorf("malformed OpenCode shell.exited event") } - if !stream.knowsShell(data.ID) { + if stream == nil { + return protocol.ChatEventFrame{}, false, nil + } + stream.mu.Lock() + stream.readback.Record(&protocol.ChatShell{ID: data.ID, Status: protocol.ChatShellExited}) + _, known := stream.Shells[data.ID] + delete(stream.Shells, data.ID) + stream.mu.Unlock() + if !known { return protocol.ChatEventFrame{}, false, nil } - stream.forgetShell(data.ID) status := data.Status if status == "" { status = protocol.ChatShellExited diff --git a/internal/opencode/chat_shell_reconcile_test.go b/internal/opencode/chat_shell_reconcile_test.go new file mode 100644 index 00000000..087080d0 --- /dev/null +++ b/internal/opencode/chat_shell_reconcile_test.go @@ -0,0 +1,116 @@ +package opencode + +import ( + "context" + "fmt" + "net/http" + "net/http/httptest" + "testing" + "time" + + "git.a-wan.net/alan/pipelines/internal/protocol" +) + +func TestChatShellReadbackBarrier(t *testing.T) { + for _, known := range []bool{false, true} { + t.Run(fmt.Sprint(known), func(t *testing.T) { + read, apply := make(chan struct{}), make(chan struct{}) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + close(read) + select { + case <-apply: + fmt.Fprint(w, `{"data":[{"id":"sh_old","status":"running","metadata":{"sessionID":"ses_1"}},{"id":"sh_foreign","status":"running","metadata":{"sessionID":"ses_other"}}]}`) + case <-r.Context().Done(): + } + })) + defer srv.Close() + c := NewChat(&Connection{Backend: "v2", URL: srv.URL}) + state := c.streamState("ses_1") + if known { + state.markShell("sh_old") + } + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + defer cancel() + result := make(chan []protocol.ChatShell, 1) + errs := make(chan error, 1) + go func() { shells, err := c.Shells(ctx, "ses_1"); result <- shells; errs <- err }() + select { + case <-read: + case <-ctx.Done(): + t.Fatal("read did not reach barrier") + } + // Both a previously verified exit and an exit first learned while + // reading must suppress the stale running snapshot. + frame, ok, err := NormalizeChatEvent([]byte(`{"type":"shell.exited","data":{"id":"sh_old","status":"exited","exit":0}}`), "ses_1", state) + if err != nil || ok != known || (ok && frame.Shell.Status != protocol.ChatShellExited) { + t.Fatalf("exit = %+v %v %v", frame, ok, err) + } + frame, ok, err = NormalizeChatEvent([]byte(`{"type":"shell.created","data":{"info":{"id":"sh_new","status":"running","command":"sleep 10","metadata":{"sessionID":"ses_1"}}}}`), "ses_1", state) + if err != nil || !ok || frame.Shell.ID != "sh_new" { + t.Fatalf("create = %+v %v %v", frame, ok, err) + } + close(apply) + got := <-result + if err := <-errs; err != nil { + t.Fatal(err) + } + if len(got) != 1 || got[0].ID != "sh_new" || got[0].Command != "sleep 10" { + t.Fatalf("readback lost events/leaked ownership: %+v", got) + } + if state.knowsShell("sh_old") || state.knowsShell("sh_foreign") || !state.knowsShell("sh_new") { + t.Fatal("adapter identity state regressed") + } + // User-visible subsequent exit remains attributable after reconciliation. + frame, ok, err = NormalizeChatEvent([]byte(`{"type":"shell.exited","data":{"id":"sh_new","status":"exited","exit":0}}`), "ses_1", state) + if err != nil || !ok || frame.Shell.ID != "sh_new" { + t.Fatalf("new shell exit lost: %+v %v %v", frame, ok, err) + } + }) + } +} + +func TestChatShellReadbackCancelWaiter(t *testing.T) { + read := make(chan struct{}) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { close(read); <-r.Context().Done() })) + defer srv.Close() + c := NewChat(&Connection{Backend: "v2", URL: srv.URL}) + ctx, cancel := context.WithCancel(context.Background()) + result := make(chan error, 2) + go func() { _, err := c.Shells(ctx, "ses_1"); result <- err }() + select { + case <-read: + case <-time.After(time.Second): + t.Fatal("read stuck") + } + go func() { _, err := c.Shells(ctx, "ses_1"); result <- err }() + cancel() + for range 2 { + select { + case err := <-result: + if err == nil { + t.Fatal("canceled read succeeded") + } + case <-time.After(time.Second): + t.Fatal("read or waiter stuck") + } + } + old := c.streamState("ses_1") + c.ReleaseShells("ses_1") + if c.streamState("ses_1") == old { + t.Fatal("reconnect reused released state") + } +} + +func TestShellSnapshotBounded(t *testing.T) { + var snapshot ShellSnapshot + for i := 0; i <= chatShellIdentityLimit; i++ { + snapshot.Record(&protocol.ChatShell{ID: fmt.Sprintf("sh_%d", i), Status: protocol.ChatShellExited}) + } + if len(snapshot.changes) != chatShellIdentityLimit { + t.Fatalf("unbounded journal: %d", len(snapshot.changes)) + } + if _, err := snapshot.Merge([]protocol.ChatShell{{ID: "sh_0", Status: protocol.ChatShellRunning}}); err == nil { + t.Fatal("overflow accepted stale snapshot") + } +} diff --git a/internal/opencode/chat_shell_snapshot.go b/internal/opencode/chat_shell_snapshot.go new file mode 100644 index 00000000..8559a741 --- /dev/null +++ b/internal/opencode/chat_shell_snapshot.go @@ -0,0 +1,72 @@ +package opencode + +import ( + "fmt" + "sort" + + "git.a-wan.net/alan/pipelines/internal/protocol" +) + +// ShellSnapshot records changes while an authoritative read is in flight. Its +// owner serializes access. Unknown exits may be recorded, but can only remove an +// id: they never establish ownership. Overflow rejects the snapshot, not events. +type ShellSnapshot struct { + changes map[string]protocol.ChatShell + bytes int + overflow bool +} + +const chatShellIdentityLimit = 256 + +func (s *ShellSnapshot) Record(shell *protocol.ChatShell) { + if s == nil || shell == nil || shell.ID == "" || s.overflow { + return + } + if s.changes == nil { + s.changes = make(map[string]protocol.ChatShell) + } + old := s.changes[shell.ID] + n := s.bytes - len(old.ID) - len(old.Command) + len(shell.ID) + len(shell.Command) + if n > MaxResponse || (old.ID == "" && len(s.changes) >= chatShellIdentityLimit) { + s.overflow = true + return + } + // A terminal event wins even if a duplicate create is delivered later. + if old.ID != "" && old.Status != protocol.ChatShellRunning { + return + } + s.changes[shell.ID] = *shell + s.bytes = n +} + +func (s *ShellSnapshot) Merge(shells []protocol.ChatShell) ([]protocol.ChatShell, error) { + if s.overflow { + return nil, fmt.Errorf("shell reconciliation capacity exceeded") + } + byID := make(map[string]protocol.ChatShell) + for _, shell := range shells { + if shell.ID != "" && shell.Status == protocol.ChatShellRunning { + byID[shell.ID] = shell + } + } + for id, shell := range s.changes { + if shell.Status == protocol.ChatShellRunning { + byID[id] = shell + } else { + delete(byID, id) + } + } + if len(byID) > chatShellIdentityLimit { + return nil, fmt.Errorf("shell reconciliation capacity exceeded") + } + ids := make([]string, 0, len(byID)) + for id := range byID { + ids = append(ids, id) + } + sort.Strings(ids) + out := make([]protocol.ChatShell, 0, len(ids)) + for _, id := range ids { + out = append(out, byID[id]) + } + return out, nil +} diff --git a/internal/opencode/chat_v2_test.go b/internal/opencode/chat_v2_test.go index 00cb2242..25fc7024 100644 --- a/internal/opencode/chat_v2_test.go +++ b/internal/opencode/chat_v2_test.go @@ -395,3 +395,45 @@ func TestOpenChatRejectsUnrunnableCLI(t *testing.T) { t.Fatal("a missing CLI must fail detection, not open an endpoint") } } + +// TestChatShellReadbackSeedsReplacementStream proves an authoritative shell +// readback survives a later stream replacement: shell.exited has no session id, +// so it must still clear the verified shell after reconnect. +func TestChatShellReadbackSeedsReplacementStream(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/api/shell": + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"data":[{"id":"sh_1","status":"running","command":"sleep 30","metadata":{"sessionID":"ses_1"}},{"id":"sh_other","status":"running","command":"sleep 30","metadata":{"sessionID":"ses_other"}}]}`)) + case "/api/event": + w.Header().Set("Content-Type", "text/event-stream") + _, _ = w.Write([]byte("data: {\"type\":\"shell.exited\",\"data\":{\"id\":\"sh_1\",\"status\":\"exited\",\"exit\":0}}\n\n")) + } + })) + defer srv.Close() + chat := NewChat(&Connection{Backend: "v2", URL: srv.URL}) + shells, err := chat.Shells(context.Background(), "ses_1") + if err != nil || len(shells) != 1 || shells[0].ID != "sh_1" { + t.Fatalf("Shells = %+v, %v", shells, err) + } + var got []protocol.ChatEventFrame + if err := chat.Stream(context.Background(), "ses_1", nil, func(frame protocol.ChatEventFrame) error { got = append(got, frame); return nil }); err != nil { + t.Fatal(err) + } + if len(got) != 1 || got[0].Shell == nil || got[0].Shell.ID != "sh_1" || got[0].Shell.Status != protocol.ChatShellExited { + t.Fatalf("exit after readback = %+v", got) + } +} + +func TestChatReleaseShellsBoundsDisconnectedState(t *testing.T) { + chat := NewChat(&Connection{}) + state := chat.streamState("ses_1") + state.markShell("sh_1") + chat.ReleaseShells("ses_1") + chat.shellMu.Lock() + _, retained := chat.shells["ses_1"] + chat.shellMu.Unlock() + if retained { + t.Fatal("released session shell state was retained") + } +} diff --git a/internal/pipeline/control.go b/internal/pipeline/control.go index c603497e..9d32e974 100644 --- a/internal/pipeline/control.go +++ b/internal/pipeline/control.go @@ -10,7 +10,7 @@ import ( ) func decodeControl(text string, value any) error { - // Both control blocks are flat objects. Reject duplicate keys before decoding. + // Control blocks are flat objects. Reject duplicate keys before decoding. keys := json.NewDecoder(strings.NewReader(text)) if token, err := keys.Token(); err != nil || token != json.Delim('{') { return fmt.Errorf("expected object") @@ -44,6 +44,12 @@ func decodeControl(text string, value any) error { } func parseControl(s *Stage, fields map[string]string) error { + if text, ok := fields["resources"]; ok { + if err := decodeControl(text, &s.Resources); err != nil || s.Resources == nil || !ValidResourceKey(s.Resources.Exclusive) { + return fmt.Errorf("resources must contain exactly one exclusive key (1–128 ASCII letters, digits, _ or -; start with a letter)") + } + delete(fields, "resources") + } if text, ok := fields["limits"]; ok { if err := decodeControl(text, &s.Limits); err != nil || s.Limits == nil { return fmt.Errorf("limits must be an object with max_visits and optional on outcomes") @@ -76,6 +82,9 @@ func parseControl(s *Stage, fields map[string]string) error { } func validateControl(s Stage, stages map[string]bool) error { + if s.Resources != nil && s.Type == "human" { + return fmt.Errorf("stage %s: human stages cannot declare resources", s.Key) + } if s.Limits != nil { for _, outcome := range s.Limits.On { target, exists := s.Edges[outcome] @@ -95,3 +104,7 @@ func validateControl(s Stage, stages map[string]bool) error { } return nil } + +func ValidResourceKey(key string) bool { + return len(key) <= 128 && identifier.MatchString(key) +} diff --git a/internal/pipeline/resources.go b/internal/pipeline/resources.go new file mode 100644 index 00000000..1a1720b1 --- /dev/null +++ b/internal/pipeline/resources.go @@ -0,0 +1,277 @@ +package pipeline + +import ( + "context" + "crypto/rand" + "database/sql" + "encoding/hex" + "encoding/json" + "fmt" + "time" + + "git.a-wan.net/alan/pipelines/internal/protocol" +) + +// ResourceVisit identifies a pending visit, before it consumes execution accounting. +type ResourceVisit struct { + Resource string + RunID int64 + Stage string + Sequence int +} + +// ResourceLease is an opaque capability. Its credential is deliberately not exported +// or serializable, and must never be put in model input or public wire metadata. +type ResourceLease struct { + ResourceVisit + token string +} + +// RequestResource acquires immediately or registers one FIFO waiter/channel. The +// mutex covers only short DB transactions and channel bookkeeping, never execution. +func (s *Store) RequestResource(ctx context.Context, visit ResourceVisit) (ResourceLease, <-chan ResourceLease, *protocol.ResourceWait, error) { + s.resourceMu.Lock() + defer s.resourceMu.Unlock() + if !ValidResourceKey(visit.Resource) || visit.RunID <= 0 || visit.Sequence <= 0 || !identifier.MatchString(visit.Stage) { + return ResourceLease{}, nil, nil, fmt.Errorf("invalid resource visit") + } + if ch := s.resourceWaiters[visit]; ch != nil { + wait, err := s.resourceWait(ctx, visit.RunID) + return ResourceLease{}, ch, wait, err + } + credential := make([]byte, 32) + if _, err := rand.Read(credential); err != nil { + return ResourceLease{}, nil, nil, err + } + lease := ResourceLease{ResourceVisit: visit, token: hex.EncodeToString(credential)} + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + return ResourceLease{}, nil, nil, err + } + defer tx.Rollback() + var status string + if err := tx.QueryRowContext(ctx, "SELECT status FROM pipeline_runs WHERE id=?", visit.RunID).Scan(&status); err != nil { + return ResourceLease{}, nil, nil, err + } + if status != protocol.RunStatusQueued && status != protocol.RunStatusRunning { + return ResourceLease{}, nil, nil, fmt.Errorf("run is not active") + } + var existing ResourceLease + err = tx.QueryRowContext(ctx, "SELECT resource_key,run_id,stage_key,sequence,token FROM resource_owners WHERE run_id=?", visit.RunID).Scan(&existing.Resource, &existing.RunID, &existing.Stage, &existing.Sequence, &existing.token) + if err == nil { + if existing.ResourceVisit != visit { + return ResourceLease{}, nil, nil, fmt.Errorf("run already owns another resource visit") + } + return existing, nil, nil, nil + } + if err != sql.ErrNoRows { + return ResourceLease{}, nil, nil, err + } + result, err := tx.ExecContext(ctx, `INSERT INTO resource_owners(resource_key,run_id,stage_key,sequence,token) +VALUES(?,?,?,?,?) ON CONFLICT(resource_key) DO NOTHING`, visit.Resource, visit.RunID, visit.Stage, visit.Sequence, lease.token) + if err != nil { + return ResourceLease{}, nil, nil, err + } + n, err := result.RowsAffected() + if err != nil { + return ResourceLease{}, nil, nil, err + } + if n == 1 { + return lease, nil, nil, tx.Commit() + } + wait := &protocol.ResourceWait{Resource: visit.Resource, Stage: visit.Stage, Sequence: visit.Sequence, QueuedSince: time.Now().UTC().Format(time.RFC3339Nano)} + encoded, err := json.Marshal(wait) + if err != nil { + return ResourceLease{}, nil, nil, err + } + if _, err := tx.ExecContext(ctx, `INSERT INTO resource_waiters(resource_key,run_id,stage_key,sequence,token,queued_at) VALUES(?,?,?,?,?,?)`, visit.Resource, visit.RunID, visit.Stage, visit.Sequence, lease.token, wait.QueuedSince); err != nil { + return ResourceLease{}, nil, nil, err + } + if err := affected(tx.ExecContext(ctx, "UPDATE pipeline_runs SET status='queued',current_stage=?,resource_wait=? WHERE id=? AND status IN ('queued','running')", visit.Stage, string(encoded), visit.RunID)); err != nil { + return ResourceLease{}, nil, nil, err + } + if err := tx.Commit(); err != nil { + return ResourceLease{}, nil, nil, err + } + if s.resourceWaiters == nil { + s.resourceWaiters = make(map[ResourceVisit]chan ResourceLease) + } + ch := make(chan ResourceLease, 1) + s.resourceWaiters[visit] = ch + return ResourceLease{}, ch, wait, nil +} + +func (s *Store) resourceWait(ctx context.Context, id int64) (*protocol.ResourceWait, error) { + var wait protocol.ResourceWait + err := s.db.QueryRowContext(ctx, "SELECT resource_key,stage_key,sequence,queued_at FROM resource_waiters WHERE run_id=?", id).Scan(&wait.Resource, &wait.Stage, &wait.Sequence, &wait.QueuedSince) + if err == sql.ErrNoRows { + return nil, nil + } + return &wait, err +} + +// CancelResource removes a queued visit, or returns a concurrently reserved grant +// to the queue before any execution is allowed. It cannot cancel another visit. +func (s *Store) CancelResource(ctx context.Context, visit ResourceVisit) error { + s.resourceMu.Lock() + defer s.resourceMu.Unlock() + defer delete(s.resourceWaiters, visit) + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + return err + } + defer tx.Rollback() + deleted, err := tx.ExecContext(ctx, "DELETE FROM resource_waiters WHERE resource_key=? AND run_id=? AND stage_key=? AND sequence=?", visit.Resource, visit.RunID, visit.Stage, visit.Sequence) + if err != nil { + return err + } + n, err := deleted.RowsAffected() + if err != nil { + return err + } + if n > 0 { + if _, err := tx.ExecContext(ctx, "UPDATE pipeline_runs SET resource_wait='' WHERE id=?", visit.RunID); err != nil { + return err + } + } + ch := s.resourceWaiters[visit] + // A grant is buffered while this registry entry exists. Cancellation can + // reclaim only that not-yet-accepted capability, never an executing owner. + var grant ResourceLease + if ch != nil { + select { + case grant = <-ch: + default: + } + } + if err := tx.Commit(); err != nil { + return err + } + if grant.token != "" { + if err := s.SetResourceCleanup(ctx, grant, true); err != nil { + return err + } + return s.releaseResourceLocked(ctx, grant) + } + return nil +} + +// AcceptResource removes the channel exactly once. Cancellation is rechecked by +// the runner before BeginStep; a canceled accepted grant is released as unused. +func (s *Store) AcceptResource(lease ResourceLease) { + s.resourceMu.Lock() + delete(s.resourceWaiters, lease.ResourceVisit) + s.resourceMu.Unlock() +} + +// HasResourceOwner is used only for orphan/restart reconciliation, never for a +// resource-free stage visit. +func (s *Store) HasResourceOwner(ctx context.Context, runID int64) (bool, error) { + var exists bool + err := s.db.QueryRowContext(ctx, "SELECT EXISTS(SELECT 1 FROM resource_owners WHERE run_id=?)", runID).Scan(&exists) + return exists, err +} + +func (s *Store) ConfirmRecoveredSession(ctx context.Context, owner SessionOwner) error { + _, err := s.db.ExecContext(ctx, "UPDATE run_steps SET agent_active=0 WHERE run_id=? AND session_id=?", owner.RunID, owner.ID) + return err +} + +// SetResourceCleanup records positive quiescence durably before release. False +// quarantines the owner; neither age nor another run can remove the lease. +func (s *Store) SetResourceCleanup(ctx context.Context, lease ResourceLease, confirmed bool) error { + diagnostic := "" + if !confirmed { + diagnostic = "Cleanup/restoration unconfirmed; resource quarantined pending inspected reconciliation" + } + return affected(s.db.ExecContext(ctx, `UPDATE resource_owners SET cleanup_confirmed=?,diagnostic=? WHERE resource_key=? AND run_id=? AND stage_key=? AND sequence=? AND token=?`, confirmed, diagnostic, lease.Resource, lease.RunID, lease.Stage, lease.Sequence, lease.token)) +} + +// ReleaseResource requires the complete capability AND positive cleanup evidence. +// Mismatches and repeated releases are harmless, including after transfer. +func (s *Store) ReleaseResource(ctx context.Context, lease ResourceLease) error { + s.resourceMu.Lock() + defer s.resourceMu.Unlock() + return s.releaseResourceLocked(ctx, lease) +} + +func (s *Store) releaseResourceLocked(ctx context.Context, lease ResourceLease) error { + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + return err + } + defer tx.Rollback() + n, err := deleteResourceOwner(ctx, tx, lease) + if err != nil { + return err + } + if n == 0 { + return nil + } + var next ResourceLease + next.Resource = lease.Resource + err = tx.QueryRowContext(ctx, `SELECT run_id,stage_key,sequence,token FROM resource_waiters WHERE resource_key=? ORDER BY queue_order LIMIT 1`, lease.Resource).Scan(&next.RunID, &next.Stage, &next.Sequence, &next.token) + if err != nil && err != sql.ErrNoRows { + return err + } + if err == nil { + if s.resourceWaiters[next.ResourceVisit] == nil { + return fmt.Errorf("resource waiter has no live runner; refusing handoff") + } + if _, err := tx.ExecContext(ctx, `INSERT INTO resource_owners(resource_key,run_id,stage_key,sequence,token) VALUES(?,?,?,?,?)`, next.Resource, next.RunID, next.Stage, next.Sequence, next.token); err != nil { + return err + } + if _, err := tx.ExecContext(ctx, "DELETE FROM resource_waiters WHERE run_id=?", next.RunID); err != nil { + return err + } + if _, err := tx.ExecContext(ctx, "UPDATE pipeline_runs SET resource_wait='' WHERE id=?", next.RunID); err != nil { + return err + } + } + if err := tx.Commit(); err != nil { + return err + } + if next.token != "" { + s.resourceWaiters[next.ResourceVisit] <- next + } + return nil +} + +func deleteResourceOwner(ctx context.Context, tx *sql.Tx, lease ResourceLease) (int64, error) { + result, err := tx.ExecContext(ctx, `DELETE FROM resource_owners WHERE resource_key=? AND run_id=? AND stage_key=? AND sequence=? AND token=? AND cleanup_confirmed=1`, lease.Resource, lease.RunID, lease.Stage, lease.Sequence, lease.token) + if err != nil { + return 0, err + } + return result.RowsAffected() +} + +// recoverResourceOwners runs only at startup, after pending waiters have been +// interrupted and removed. Every reclamation uses the persisted capability; +// absent cleanup evidence is never replaced with a timestamp-based expiry. +func recoverResourceOwners(ctx context.Context, tx *sql.Tx) error { + rows, err := tx.QueryContext(ctx, "SELECT resource_key,run_id,stage_key,sequence,token FROM resource_owners WHERE cleanup_confirmed=1") + if err != nil { + return err + } + var leases []ResourceLease + for rows.Next() { + var lease ResourceLease + if err := rows.Scan(&lease.Resource, &lease.RunID, &lease.Stage, &lease.Sequence, &lease.token); err != nil { + rows.Close() + return err + } + leases = append(leases, lease) + } + err = rows.Err() + rows.Close() + if err != nil { + return err + } + for _, lease := range leases { + if _, err := deleteResourceOwner(ctx, tx, lease); err != nil { + return err + } + } + _, err = tx.ExecContext(ctx, "UPDATE resource_owners SET diagnostic='Server stopped; resource quarantined pending inspected cleanup/restoration'") + return err +} diff --git a/internal/pipeline/resources_test.go b/internal/pipeline/resources_test.go new file mode 100644 index 00000000..58ae2e6a --- /dev/null +++ b/internal/pipeline/resources_test.go @@ -0,0 +1,288 @@ +package pipeline + +import ( + "context" + "encoding/json" + "fmt" + "path/filepath" + "strings" + "sync" + "testing" + "time" +) + +func TestResourceDeclaration(t *testing.T) { + base := Template([]string{"work"}, true) + for _, value := range []string{`{"exclusive":"phone-X"}`, `{ "exclusive" : "A_1" }`} { + graph, err := Parse(base + "```resources\n" + value + "\n```\n") + if err != nil || graph.Stages[0].Resources == nil { + t.Fatalf("%s: %v", value, err) + } + if _, ok := StageDefinition(graph.Stages[0]).Blocks["resources"]; ok { + t.Fatal("runner control reached executor") + } + encoded, _ := json.Marshal(graph) + if !strings.Contains(string(encoded), `"resources"`) { + t.Fatal("resource lost in persisted graph") + } + } + for _, value := range []string{`{}`, `null`, `[]`, `{"exclusive":""}`, `{"exclusive":"a b"}`, `{"exclusive":"a","exclusive":"b"}`, `{"exclusive":"a","extra":1}`, `{"exclusive":["a","b"]}`, fmt.Sprintf(`{"exclusive":%q}`, strings.Repeat("a", 129))} { + if _, err := Parse(base + "```resources\n" + value + "\n```\n"); err == nil { + t.Fatalf("accepted %s", value) + } + } + human := "# stage gate\n```prompt\nChoose\n```\n```configs\ntype=human\n```\n```edges\ngo -> #success\n```\n```resources\n{\"exclusive\":\"phone\"}\n```\n" + if _, err := Parse(human); err == nil { + t.Fatal("human acquired a resource") + } +} + +func resourceStore(t *testing.T) (*Store, func(string) ResourceVisit) { + t.Helper() + ctx := context.Background() + s, err := Open(filepath.Join(t.TempDir(), "resources.db")) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { s.Close() }) + if _, err := s.Submit(ctx, "resource", Template([]string{"work"}, true)); err != nil { + t.Fatal(err) + } + items, _ := s.List(ctx) + d, _ := s.Get(ctx, items[0].ID, 1) + count := 0 + return s, func(resource string) ResourceVisit { + count++ + id, _, err := s.CreateRun(ctx, d, fmt.Sprint(count), "workspace", false, nil, nil) + if err != nil { + t.Fatal(err) + } + return ResourceVisit{Resource: resource, RunID: id, Stage: "work", Sequence: 1} + } +} + +func acquireTestResource(t *testing.T, s *Store, visit ResourceVisit) ResourceLease { + t.Helper() + l, ch, _, err := s.RequestResource(context.Background(), visit) + if err != nil || ch != nil { + t.Fatalf("acquire: %v %v", ch, err) + } + return l +} + +func releaseTestResource(t *testing.T, s *Store, lease ResourceLease) { + t.Helper() + if err := s.SetResourceCleanup(context.Background(), lease, true); err != nil { + t.Fatal(err) + } + if err := s.ReleaseResource(context.Background(), lease); err != nil { + t.Fatal(err) + } +} + +func TestResourceFIFOOwnershipCancellationAndRegistry(t *testing.T) { + s, create := resourceStore(t) + ctx := context.Background() + a := acquireTestResource(t, s, create("X")) + duplicate := acquireTestResource(t, s, a.ResourceVisit) + if duplicate != a { + t.Fatal("duplicate acquisition changed capability") + } + var visits []ResourceVisit + var channels []<-chan ResourceLease + for i := 0; i < 4; i++ { + v := create("X") + _, ch, wait, err := s.RequestResource(ctx, v) + if err != nil || ch == nil || wait == nil { + t.Fatal("missing waiter", err) + } + _, again, _, err := s.RequestResource(ctx, v) + if err != nil || again != ch { + t.Fatal("duplicate waiter registered", err) + } + visits = append(visits, v) + channels = append(channels, ch) + } + if err := s.ReleaseResource(ctx, a); err != nil { + t.Fatal(err) + } // no cleanup receipt + for _, ch := range channels { + select { + case <-ch: + t.Fatal("unconfirmed release granted") + default: + } + } + if err := s.SetResourceCleanup(ctx, a, true); err != nil { + t.Fatal(err) + } + for _, corrupt := range []func(*ResourceLease){func(l *ResourceLease) { l.RunID++ }, func(l *ResourceLease) { l.Stage = "other" }, func(l *ResourceLease) { l.Sequence++ }, func(l *ResourceLease) { l.token = "wrong" }} { + bad := a + corrupt(&bad) + if err := s.ReleaseResource(ctx, bad); err != nil { + t.Fatal(err) + } + } + for _, ch := range channels { + select { + case <-ch: + t.Fatal("unconfirmed/mismatched release granted") + default: + } + } + if err := s.CancelResource(ctx, visits[1]); err != nil { + t.Fatal(err) + } + releaseTestResource(t, s, a) + for _, i := range []int{0, 2, 3} { + var grant ResourceLease + select { + case grant = <-channels[i]: + case <-time.After(time.Second): + t.Fatal("FIFO grant not delivered") + } + s.AcceptResource(grant) + if grant.ResourceVisit != visits[i] { + t.Fatal("wrong FIFO owner") + } + // An old owner's repeated release cannot release the next owner. + if err := s.ReleaseResource(ctx, a); err != nil { + t.Fatal(err) + } + for _, j := range []int{2, 3} { + if j > i { + select { + case <-channels[j]: + t.Fatal("multiple grants") + default: + } + } + } + releaseTestResource(t, s, grant) + } + if len(s.resourceWaiters) != 0 { + t.Fatal("channel registry leaked") + } + for i := 0; i < 30; i++ { + owner := acquireTestResource(t, s, create("X")) + v := create("X") + _, _, _, err := s.RequestResource(ctx, v) + if err != nil { + t.Fatal(err) + } + releaseTestResource(t, s, owner) // race window: granted but not accepted + if err := s.CancelResource(ctx, v); err != nil { + t.Fatal(err) + } + if len(s.resourceWaiters) != 0 { + t.Fatal("grant/cancel leaked channel") + } + } + for i := 0; i < 30; i++ { + owner := acquireTestResource(t, s, create("X")) + v := create("X") + if _, _, _, err := s.RequestResource(ctx, v); err != nil { + t.Fatal(err) + } + if err := s.SetResourceCleanup(ctx, owner, true); err != nil { + t.Fatal(err) + } + var wg sync.WaitGroup + wg.Add(2) + go func() { + defer wg.Done() + if err := s.ReleaseResource(ctx, owner); err != nil { + t.Error(err) + } + }() + go func() { + defer wg.Done() + if err := s.CancelResource(ctx, v); err != nil { + t.Error(err) + } + }() + wg.Wait() + if len(s.resourceWaiters) != 0 { + t.Fatal("concurrent grant/cancel leaked channel") + } + } + var owners, waiters int + s.db.QueryRow("SELECT COUNT(*) FROM resource_owners").Scan(&owners) + s.db.QueryRow("SELECT COUNT(*) FROM resource_waiters").Scan(&waiters) + if owners != 0 || waiters != 0 { + t.Fatalf("stranded owners/waiters: %d/%d", owners, waiters) + } +} + +func TestResourceCancellationPersistenceFailureRemovesChannel(t *testing.T) { + s, create := resourceStore(t) + acquireTestResource(t, s, create("X")) + v := create("X") + if _, _, _, err := s.RequestResource(context.Background(), v); err != nil { + t.Fatal(err) + } + if err := s.Close(); err != nil { + t.Fatal(err) + } + if err := s.CancelResource(context.Background(), v); err == nil { + t.Fatal("closed DB cancellation claimed success") + } + if len(s.resourceWaiters) != 0 { + t.Fatal("persistence failure leaked channel") + } +} + +func TestResourceRestartQuarantineAndRetention(t *testing.T) { + for _, confirmed := range []bool{false, true} { + t.Run(fmt.Sprint(confirmed), func(t *testing.T) { + s, create := resourceStore(t) + ctx := context.Background() + a := acquireTestResource(t, s, create("X")) + b := create("X") + if _, _, _, err := s.RequestResource(ctx, b); err != nil { + t.Fatal(err) + } + if confirmed { + if err := s.SetResourceCleanup(ctx, a, true); err != nil { + t.Fatal(err) + } + } + path := s.path + if err := s.Close(); err != nil { + t.Fatal(err) + } + reopened, err := Open(path) + if err != nil { + t.Fatal(err) + } + defer reopened.Close() + if err := reopened.RecoverRuns(ctx); err != nil { + t.Fatal(err) + } + if err := reopened.RecoverRuns(ctx); err != nil { + t.Fatal(err) + } + detail, err := reopened.GetRun(ctx, b.RunID) + if err != nil || detail.Status != "interrupted" || detail.ResourceWait != nil || len(detail.Steps) != 0 || !strings.Contains(detail.Diagnostic, "resource") { + t.Fatal("queued visit replayed/lost diagnostic", detail, err) + } + exists, err := reopened.HasResourceOwner(ctx, a.RunID) + if err != nil || exists == confirmed { + t.Fatal("incorrect recovery disposition", exists, err) + } + report, err := reopened.RetentionReport(ctx, RetentionPolicy{Keep: 0}, "", time.Now()) + if err != nil { + t.Fatal(err) + } + protected := false + for _, p := range report.Protected { + if p.ID == a.RunID { + protected = true + } + } + if protected == confirmed { + t.Fatal("quarantined evidence not protected") + } + }) + } +} diff --git a/internal/pipeline/retention.go b/internal/pipeline/retention.go index c4873632..0e7a38f5 100644 --- a/internal/pipeline/retention.go +++ b/internal/pipeline/retention.go @@ -130,11 +130,12 @@ type retentionQuerier interface { // retentionRunInfo is one pipeline_runs row plus the step aggregates protection // needs, read in the same statement so the report never issues a per-run query. type retentionRunInfo struct { - id int64 - status string - createdAt time.Time - waitingSteps int - activeSteps int + id int64 + status string + createdAt time.Time + waitingSteps int + activeSteps int + resourceOwners int } // protectionReasons lists why a run must not be removed for a reason intrinsic @@ -155,6 +156,9 @@ func (r retentionRunInfo) protectionReasons() []string { if r.activeSteps > 0 { reasons = append(reasons, "cleanup agent still active (agent_active=1)") } + if r.resourceOwners > 0 { + reasons = append(reasons, "resource ownership retained (possibly quarantined)") + } return reasons } @@ -171,9 +175,20 @@ type retentionSelection struct { // report is a bounded number of round trips rather than one query per run. func selectRetentionRuns(ctx context.Context, q retentionQuerier, policy RetentionPolicy, now time.Time) (retentionSelection, error) { var sel retentionSelection + // Maintenance is deliberately non-migrating and may inspect a database + // whose server predates resource support. + var resources bool + if err := q.QueryRowContext(ctx, "SELECT EXISTS(SELECT 1 FROM sqlite_master WHERE type='table' AND name='resource_owners')").Scan(&resources); err != nil { + return sel, err + } + owners := "0" + if resources { + owners = "(SELECT COUNT(*) FROM resource_owners o WHERE o.run_id=r.id)" + } rows, err := q.QueryContext(ctx, `SELECT r.id,r.status,strftime('%Y-%m-%dT%H:%M:%SZ',r.created_at), COALESCE((SELECT COUNT(*) FROM run_steps s WHERE s.run_id=r.id AND s.status=?),0), -COALESCE((SELECT COUNT(*) FROM run_steps s WHERE s.run_id=r.id AND s.agent_active=1),0) `+ +COALESCE((SELECT COUNT(*) FROM run_steps s WHERE s.run_id=r.id AND s.agent_active=1),0), +`+owners+` `+ runFrom+` ORDER BY r.id DESC`, protocol.StepStatusWaiting) if err != nil { return sel, err @@ -182,7 +197,7 @@ COALESCE((SELECT COUNT(*) FROM run_steps s WHERE s.run_id=r.id AND s.agent_activ for rows.Next() { var run retentionRunInfo var created string - if err := rows.Scan(&run.id, &run.status, &created, &run.waitingSteps, &run.activeSteps); err != nil { + if err := rows.Scan(&run.id, &run.status, &created, &run.waitingSteps, &run.activeSteps, &run.resourceOwners); err != nil { return sel, err } run.createdAt = parseRetentionTime(created) diff --git a/internal/pipeline/runs.go b/internal/pipeline/runs.go index 5b58328c..76e1de11 100644 --- a/internal/pipeline/runs.go +++ b/internal/pipeline/runs.go @@ -21,14 +21,19 @@ const runStamps = `strftime('%Y-%m-%dT%H:%M:%SZ',r.created_at),COALESCE(strftime const runSelect = `SELECT r.id,r.pipeline_id,p.name,r.version,v.content_hash,r.status,r.outcome,r.current_stage, ` + runStamps + `,r.diagnostic, CASE WHEN r.status='succeeded' THEN r.output_json ELSE COALESCE((SELECT output_json FROM run_steps WHERE run_id=r.id AND status='completed' ORDER BY sequence DESC LIMIT 1),'{}') END, -(SELECT COUNT(*) FROM run_steps WHERE run_id=r.id AND status='completed'),r.workspace,r.title,r.model,r.stage_models +(SELECT COUNT(*) FROM run_steps WHERE run_id=r.id AND status='completed'),r.workspace,r.title,r.model,r.stage_models,r.resource_wait ` + runFrom + ` JOIN pipeline_versions v ON v.pipeline_id=r.pipeline_id AND v.version=r.version` func scanRun(row interface{ Scan(...any) error }, r *protocol.RunSummary) error { - var output, stageModels string - if err := row.Scan(&r.ID, &r.PipelineID, &r.Name, &r.Version, &r.ContentHash, &r.Status, &r.Outcome, &r.CurrentStage, &r.CreatedAt, &r.FinishedAt, &r.Diagnostic, &output, &r.CompletedSteps, &r.Workspace, &r.Title, &r.Model, &stageModels); err != nil { + var output, stageModels, resourceWait string + if err := row.Scan(&r.ID, &r.PipelineID, &r.Name, &r.Version, &r.ContentHash, &r.Status, &r.Outcome, &r.CurrentStage, &r.CreatedAt, &r.FinishedAt, &r.Diagnostic, &output, &r.CompletedSteps, &r.Workspace, &r.Title, &r.Model, &stageModels, &resourceWait); err != nil { return err } + if resourceWait != "" { + if err := json.Unmarshal([]byte(resourceWait), &r.ResourceWait); err != nil { + return err + } + } if stageModels != "" && stageModels != "{}" { if err := json.Unmarshal([]byte(stageModels), &r.StageModels); err != nil { return err diff --git a/internal/pipeline/runs_mutate.go b/internal/pipeline/runs_mutate.go index d27e7268..d370a29f 100644 --- a/internal/pipeline/runs_mutate.go +++ b/internal/pipeline/runs_mutate.go @@ -44,7 +44,7 @@ func (s *Store) BeginStep(ctx context.Context, id int64, sequence int, key strin return err } defer tx.Rollback() - if err := affected(tx.ExecContext(ctx, "UPDATE pipeline_runs SET status='running',current_stage=? WHERE id=? AND status IN ('queued','running')", key, id)); err != nil { + if err := affected(tx.ExecContext(ctx, "UPDATE pipeline_runs SET status='running',current_stage=?,resource_wait='' WHERE id=? AND status IN ('queued','running')", key, id)); err != nil { return err } if _, err := tx.ExecContext(ctx, "INSERT INTO run_steps(run_id,sequence,stage_key,input_json) VALUES (?,?,?,?)", id, sequence, key, string(encoded)); err != nil { @@ -310,5 +310,5 @@ func (s *Store) FinishRun(ctx context.Context, id int64, status, outcome, diagno if err != nil { return err } - return affected(s.db.ExecContext(ctx, "UPDATE pipeline_runs SET status=?,outcome=?,diagnostic=?,output_json=?,finished_at=CURRENT_TIMESTAMP WHERE id=? AND status IN ('queued','running')", status, outcome, diagnostic, string(data), id)) + return affected(s.db.ExecContext(ctx, "UPDATE pipeline_runs SET status=?,outcome=?,diagnostic=?,output_json=?,finished_at=CURRENT_TIMESTAMP,resource_wait='' WHERE id=? AND status IN ('queued','running')", status, outcome, diagnostic, string(data), id)) } diff --git a/internal/pipeline/runs_schema.go b/internal/pipeline/runs_schema.go index a8c1c78c..8f861061 100644 --- a/internal/pipeline/runs_schema.go +++ b/internal/pipeline/runs_schema.go @@ -51,13 +51,28 @@ CREATE TABLE IF NOT EXISTS run_session_owners ( run_id INTEGER NOT NULL, sequence INTEGER NOT NULL, backend TEXT NOT NULL, reference TEXT NOT NULL DEFAULT '', PRIMARY KEY(run_id,sequence), FOREIGN KEY(run_id,sequence) REFERENCES run_steps(run_id,sequence)); CREATE TABLE IF NOT EXISTS run_retries ( -run_id INTEGER PRIMARY KEY REFERENCES pipeline_runs(id), original_run_id INTEGER NOT NULL REFERENCES pipeline_runs(id));`) +run_id INTEGER PRIMARY KEY REFERENCES pipeline_runs(id), original_run_id INTEGER NOT NULL REFERENCES pipeline_runs(id)); +CREATE TABLE IF NOT EXISTS resource_owners ( +resource_key TEXT PRIMARY KEY, run_id INTEGER NOT NULL REFERENCES pipeline_runs(id), +stage_key TEXT NOT NULL, sequence INTEGER NOT NULL, token TEXT NOT NULL, +cleanup_confirmed INTEGER NOT NULL DEFAULT 0, diagnostic TEXT NOT NULL DEFAULT '', +acquired_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP); +CREATE INDEX IF NOT EXISTS resource_owner_run ON resource_owners(run_id); +CREATE UNIQUE INDEX IF NOT EXISTS resource_owner_visit ON resource_owners(run_id); +CREATE TABLE IF NOT EXISTS resource_waiters ( +queue_order INTEGER PRIMARY KEY AUTOINCREMENT, resource_key TEXT NOT NULL, +run_id INTEGER NOT NULL UNIQUE REFERENCES pipeline_runs(id), stage_key TEXT NOT NULL, +sequence INTEGER NOT NULL, token TEXT NOT NULL, queued_at TEXT NOT NULL); +CREATE INDEX IF NOT EXISTS resource_wait_fifo ON resource_waiters(resource_key,queue_order);`) if err != nil { return err } if err := s.migrateRunTitles(); err != nil { return err } + if err := s.addColumnIfMissing("pipeline_runs", "resource_wait", "TEXT NOT NULL DEFAULT ''"); err != nil { + return err + } return s.migrateRunModels() } @@ -122,16 +137,29 @@ func (s *Store) addColumnIfMissing(table, column, definition string) error { } func (s *Store) RecoverRuns(ctx context.Context) error { + s.resourceMu.Lock() + defer s.resourceMu.Unlock() tx, err := s.db.BeginTx(ctx, nil) if err != nil { return err } defer tx.Rollback() - if _, err := tx.ExecContext(ctx, "UPDATE run_steps SET status='interrupted',agent_active=0,diagnostic='Server stopped; stage was not resumed',finished_at=CURRENT_TIMESTAMP WHERE (status='running' OR agent_active=1) AND status<>?", protocol.StepStatusWaiting); err != nil { + // No replay: pending visits never began. Uncertain owners survive recovery, + // including the crash window before BeginStep/session registration. + if _, err := tx.ExecContext(ctx, "UPDATE pipeline_runs SET diagnostic='Server stopped while queued for a resource; pending visit was not resumed',status='interrupted',finished_at=CURRENT_TIMESTAMP,resource_wait='' WHERE id IN (SELECT run_id FROM resource_waiters); DELETE FROM resource_waiters"); err != nil { + return err + } + if err := recoverResourceOwners(ctx, tx); err != nil { + return err + } + if _, err := tx.ExecContext(ctx, "UPDATE run_steps SET status='interrupted',agent_active=CASE WHEN EXISTS(SELECT 1 FROM resource_owners o WHERE o.run_id=run_steps.run_id AND o.sequence=run_steps.sequence) THEN agent_active ELSE 0 END,diagnostic='Server stopped; stage was not resumed',finished_at=CURRENT_TIMESTAMP WHERE (status='running' OR agent_active=1) AND status<>?", protocol.StepStatusWaiting); err != nil { return err } if _, err := tx.ExecContext(ctx, "UPDATE pipeline_runs SET status='interrupted',diagnostic='Server stopped; run was not resumed',finished_at=CURRENT_TIMESTAMP WHERE status IN ('queued','running') AND status<>?", protocol.RunStatusWaiting); err != nil { return err } + if _, err := tx.ExecContext(ctx, "UPDATE pipeline_runs SET diagnostic='Server stopped; resource ownership quarantined: inspect retained session, tool termination and restoration before reconciliation' WHERE id IN (SELECT run_id FROM resource_owners)"); err != nil { + return err + } return tx.Commit() } diff --git a/internal/pipeline/stage.go b/internal/pipeline/stage.go index d2a04b44..3afb8135 100644 --- a/internal/pipeline/stage.go +++ b/internal/pipeline/stage.go @@ -18,6 +18,7 @@ func StageDefinition(s Stage) stage.Definition { delete(blocks, "input") // Retired contracts in historical graph JSON are ignored. delete(blocks, "limits") delete(blocks, "memory") + delete(blocks, "resources") for name, value := range map[string]string{"output": s.Output, "prompt": s.Prompt} { if value != "" { blocks[name] = value diff --git a/internal/pipeline/store.go b/internal/pipeline/store.go index 46466d51..abcfe29a 100644 --- a/internal/pipeline/store.go +++ b/internal/pipeline/store.go @@ -18,10 +18,12 @@ import ( ) type Store struct { - db *sql.DB - dir string - path string - mu sync.Mutex + db *sql.DB + dir string + path string + mu sync.Mutex + resourceMu sync.Mutex + resourceWaiters map[ResourceVisit]chan ResourceLease } // openDB opens one SQLite connection pool for a DSN. Every open path shares it diff --git a/internal/protocol/message.go b/internal/protocol/message.go index 74065770..f3d1f8d5 100644 --- a/internal/protocol/message.go +++ b/internal/protocol/message.go @@ -19,6 +19,7 @@ const ( ) type Stage struct { + Resources *StageResources `json:"resources,omitempty"` Limits *StageLimits `json:"limits,omitempty"` Memory map[string]string `json:"memory,omitempty"` Notify *NotificationPolicy `json:"notify,omitempty"` @@ -33,6 +34,11 @@ type Stage struct { Blocks map[string]string `json:"blocks,omitempty"` } +// StageResources declares one capacity-one resource, local to this store authority. +type StageResources struct { + Exclusive string `json:"exclusive"` +} + // Limits are enforced by the runner and never included in the model prompt. type StageLimits struct { MaxVisits int `json:"max_visits"` diff --git a/internal/protocol/resources_test.go b/internal/protocol/resources_test.go new file mode 100644 index 00000000..f6c8cd41 --- /dev/null +++ b/internal/protocol/resources_test.go @@ -0,0 +1,29 @@ +package protocol + +import ( + "encoding/json" + "strings" + "testing" +) + +func TestResourceWaitOlderClientCompatibility(t *testing.T) { + data, err := json.Marshal(RunSummary{ID: 1, Status: RunStatusQueued, ResourceWait: &ResourceWait{Resource: "X", Stage: "review", Sequence: 2, QueuedSince: "now"}}) + if err != nil { + t.Fatal(err) + } + var old struct { + ID int64 `json:"id"` + Status string `json:"status"` + } + if err := json.Unmarshal(data, &old); err != nil || old.Status != "queued" { + t.Fatal("old client no longer waits", old, err) + } + clear, err := json.Marshal(RunEventFrame{RunID: 1, ResourceWaitSet: true}) + if err != nil || !strings.Contains(string(clear), `"resource_wait_set":true`) { + t.Fatal("no explicit clear", string(clear), err) + } + var current RunSummary + if err := json.Unmarshal([]byte(`{"id":2,"status":"running"}`), ¤t); err != nil || current.ResourceWait != nil { + t.Fatal("missing field changed old payload", err) + } +} diff --git a/internal/protocol/runevents.go b/internal/protocol/runevents.go index f3b424fd..240ad2b1 100644 --- a/internal/protocol/runevents.go +++ b/internal/protocol/runevents.go @@ -24,8 +24,11 @@ const ( // from the frame instead of re-reading the whole list. They repeat on every // frame (pipeline identity never changes mid-run). type RunEventFrame struct { - RunID int64 `json:"run_id"` - Workspace string `json:"workspace,omitempty"` + // ResourceWaitSet makes a nil ResourceWait an explicit cache clear. + ResourceWaitSet bool `json:"resource_wait_set,omitempty"` + ResourceWait *ResourceWait `json:"resource_wait,omitempty"` + RunID int64 `json:"run_id"` + Workspace string `json:"workspace,omitempty"` // PipelineID, Name, Title and Version identify the run's pipeline and its // operator-facing title, so a new run's row can be built without a fetch. PipelineID int64 `json:"pipeline_id,omitempty"` diff --git a/internal/protocol/runs.go b/internal/protocol/runs.go index 87d94a1e..fb3f2830 100644 --- a/internal/protocol/runs.go +++ b/internal/protocol/runs.go @@ -118,9 +118,10 @@ type SessionExport struct { } type RunSummary struct { - ID int64 `json:"id"` - PipelineID int64 `json:"pipeline_id"` - Name string `json:"name"` + ResourceWait *ResourceWait `json:"resource_wait,omitempty"` + ID int64 `json:"id"` + PipelineID int64 `json:"pipeline_id"` + Name string `json:"name"` // Title is the operator-facing, bounded single-line description of what the // run is for, generated from its prompt. It is never empty for a new run; // historical rows predating the title column may omit it. @@ -145,6 +146,19 @@ type RunSummary struct { StageModels map[string]string `json:"stage_models,omitempty"` } +// ResourceWait is bounded public queue metadata. Ownership credentials stay private. +type ResourceWait struct { + Resource string `json:"resource"` + Stage string `json:"stage"` + Sequence int `json:"sequence"` + QueuedSince string `json:"queued_since"` +} + +// ResourceCleanupConfirmedKey lets a stage report failed restoration without +// pretending that tool idleness also proves the resource is ready. An explicit +// false (or malformed value) vetoes release; it never authorizes release alone. +const ResourceCleanupConfirmedKey = "resource_cleanup_confirmed" + // Label is the operator-facing name of a run: the generated Title when the // server stored one, otherwise the pipeline Name (historical rows predate // titles). It is the single definition shared by the CLI and TUI. @@ -155,6 +169,14 @@ func (r RunSummary) Label() string { return r.Name } +// StatusLabel separates resource contention from a human decision everywhere. +func (r RunSummary) StatusLabel() string { + if r.ResourceWait != nil { + return "Waiting for resource " + r.ResourceWait.Resource + } + return r.Status +} + type RunStep struct { Control *ControlEvidence `json:"control,omitempty"` FileChanges *StageFileChanges `json:"file_changes,omitempty"` diff --git a/internal/server/artifacts_test.go b/internal/server/artifacts_test.go index 573b851b..6c4c7804 100644 --- a/internal/server/artifacts_test.go +++ b/internal/server/artifacts_test.go @@ -105,6 +105,61 @@ func TestArtifactsScanMissingDirectoryIsEmpty(t *testing.T) { } } +func TestArtifactAccessUsesOnlyCurrentAndEarlierVisits(t *testing.T) { + root := t.TempDir() + r := &runner{artifactsRoot: root} + for _, directory := range []string{"7/1", "7/2", "7/3", "8/1"} { + if err := os.MkdirAll(filepath.Join(root, directory), 0700); err != nil { + t.Fatal(err) + } + } + access := r.artifactAccess(7, 2, []string{"1", "3", "../8/1"}, filepath.Join(root, "7", "2")) + current, _ := filepath.EvalSymlinks(filepath.Join(root, "7", "2")) + prior, _ := filepath.EvalSymlinks(filepath.Join(root, "7", "1")) + if access.Write != current || len(access.Read) != 2 || access.Read[0] != access.Write || access.Read[1] != prior { + t.Fatalf("artifact scope = %#v", access) + } + if got := r.artifactAccess(7, 2, nil, filepath.Join(root, "8", "1")); got.Write != "" || len(got.Read) != 0 { + t.Fatalf("other-run current scope accepted: %#v", got) + } + if got := r.artifactAccess(7, 2, nil, filepath.Join(root, "7", "3")); got.Write != "" || len(got.Read) != 0 { + t.Fatalf("sibling current scope accepted: %#v", got) + } + if err := os.WriteFile(filepath.Join(root, "7", "4"), []byte("not a directory"), 0600); err != nil { + t.Fatal(err) + } + if got := r.artifactAccess(7, 4, nil, filepath.Join(root, "7", "4")); got.Write != "" || len(got.Read) != 0 { + t.Fatalf("file current scope accepted: %#v", got) + } + if err := os.Symlink(filepath.Join(root, "7", "2"), filepath.Join(root, "7", "5")); err != nil { + t.Fatal(err) + } + if got := r.artifactAccess(7, 5, nil, filepath.Join(root, "7", "5")); got.Write != "" || len(got.Read) != 0 { + t.Fatalf("symlinked current scope accepted: %#v", got) + } +} + +func TestRunnerPassesDerivedArtifactAccessToAgentClient(t *testing.T) { + root := t.TempDir() + for _, directory := range []string{"7/1", "7/2"} { + if err := os.MkdirAll(filepath.Join(root, directory), 0700); err != nil { + t.Fatal(err) + } + } + r := &runner{artifactsRoot: root} + client := r.agentClient(7, 2, []string{"1", "3"}, t.TempDir(), filepath.Join(root, "7", "2"), &stage.OperationLog{}, nil, []string{"read", "edit"}) + if client.ArtifactAccess.Write == "" || len(client.ArtifactAccess.Read) != 2 { + t.Fatalf("runner did not pass scoped artifact access: %#v", client.ArtifactAccess) + } + if client.ArtifactAccess.Read[1] == filepath.Join(root, "7", "3") { + t.Fatal("runner passed a future visit to the client") + } + piClient := r.piClient(7, 2, []string{"1", "3"}, "pi", t.TempDir(), filepath.Join(root, "7", "2"), nil, []string{"read", "edit"}) + if piClient.ArtifactAccess.Write != client.ArtifactAccess.Write || len(piClient.ArtifactAccess.Read) != len(client.ArtifactAccess.Read) || piClient.ArtifactAccess.Read[1] != client.ArtifactAccess.Read[1] { + t.Fatalf("runner passed different artifact scope to Pi client: %#v", piClient.ArtifactAccess) + } +} + // runWithArtifact executes one real run whose stage writes a file into the // runner-provided artifacts directory. It returns the live handler and the // recorded step so tests can serve the artifact over the real HTTP endpoint. diff --git a/internal/server/chat.go b/internal/server/chat.go index 1601fbfc..1a26dfa6 100644 --- a/internal/server/chat.go +++ b/internal/server/chat.go @@ -32,9 +32,21 @@ type chatBroker interface { // Permissions lists the session's pending OpenCode approval requests so the // handler can push chat_permission frames. Permissions(ctx context.Context, sessionID string) ([]protocol.ChatPermissionFrame, error) + // Shells returns only shell operations whose ownership is authoritatively + // verified for sessionID. It is the reconnect/readback source. + Shells(ctx context.Context, sessionID string) ([]protocol.ChatShell, error) Reply(ctx context.Context, request protocol.ChatReplyRequest) error } +// chatShellReleaser is implemented by brokers that retain verified shell ids for +// attributing exit events. The hub calls it only after the last observer leaves +// a room; a later attach obtains a fresh authoritative snapshot. +// This is a bounded local-cache operation: it must not perform I/O or call the +// hub, which serializes it with room replacement under its mutex. +type chatShellReleaser interface { + ReleaseShells(sessionID string) +} + // operatorRegistry is the server-side operator-session seam. *pipeline.Store // satisfies it. It is a table of its own, separate from the pipeline stage // ownership (SessionOwner/run_session_owners) that settles a run: chat_sessions diff --git a/internal/server/chat_hub.go b/internal/server/chat_hub.go index 134e3fd6..9bd7e32e 100644 --- a/internal/server/chat_hub.go +++ b/internal/server/chat_hub.go @@ -133,6 +133,13 @@ func (h *chatHub) apply(frame protocol.ChatEventFrame) { } h.mu.Lock() defer h.mu.Unlock() + h.applyLocked(frame) +} + +func (h *chatHub) applyLocked(frame protocol.ChatEventFrame) { + if frame.SessionID == "" { + return + } turn := h.turnLocked(frame.SessionID) switch frame.Kind { case protocol.ChatEventStep: @@ -144,6 +151,9 @@ func (h *chatHub) apply(frame protocol.ChatEventFrame) { case protocol.ChatEventShell: // A shell frame carries no message id (it outlives the tool call that // started it), so it is folded before the message-id gate below. + if room := h.rooms[frame.SessionID]; room != nil { + room.shellReadback.Record(frame.Shell) + } turn.applyShell(frame.Shell) return } @@ -240,7 +250,7 @@ func (t *chatTurn) applyShell(shell *protocol.ChatShell) { if shell == nil || shell.ID == "" { return } - if shell.Status == protocol.ChatShellExited { + if shell.Status != protocol.ChatShellRunning { old, ok := t.shells[shell.ID] if !ok { return diff --git a/internal/server/chat_multiclient_test.go b/internal/server/chat_multiclient_test.go index 43d373a5..d6b3dde6 100644 --- a/internal/server/chat_multiclient_test.go +++ b/internal/server/chat_multiclient_test.go @@ -82,6 +82,8 @@ func (c *channelChat) Session(context.Context, string) (protocol.ChatSession, bo return protocol.ChatSession{}, false, nil } +func (c *channelChat) Shells(context.Context, string) ([]protocol.ChatShell, error) { return nil, nil } + func (c *channelChat) Permissions(context.Context, string) ([]protocol.ChatPermissionFrame, error) { c.mu.Lock() defer c.mu.Unlock() diff --git a/internal/server/chat_room.go b/internal/server/chat_room.go index 9ae2450c..a8d0ecc5 100644 --- a/internal/server/chat_room.go +++ b/internal/server/chat_room.go @@ -106,6 +106,11 @@ type chatQueuedPrompt struct { acked bool } +type chatObservation struct { + done chan struct{} + err error // published by closing done; belongs to this attempt, not the next +} + // chatRoom is one session's shared live channel. Every field is guarded by the // owning chatHub's mutex. type chatRoom struct { @@ -117,6 +122,11 @@ type chatRoom struct { queue []*chatQueuedPrompt running bool streaming bool + // observing is non-nil while one caller is attaching the shared broker + // stream. It prevents concurrent subscribers from opening duplicate global + // event streams before streaming can be published. + observing *chatObservation + shellReadback *opencode.ShellSnapshot // buffering holds the turn's frames until the prompting client's ack has been // written, so the ack always precedes the frames it acknowledges. It is set // only while the very first turn subscribes to the firehose. @@ -160,6 +170,81 @@ func (h *chatHub) roomLocked(sessionID string) *chatRoom { return room } +// ensureObserving starts the firehose even for a read-only subscriber, then +// reconciles retained shells from the endpoint's authoritative list. A bounded +// journal folds concurrent lifecycle events over that snapshot atomically. All +// observers share this reconciliation, with no broker I/O under the hub mutex. +func (h *chatHub) ensureObserving(room *chatRoom, broker chatBroker) error { + if broker == nil { + return errors.New("chat is not configured on this server") + } + h.mu.Lock() + if err := room.ctx.Err(); err != nil { + h.mu.Unlock() + return err + } + if waiting := room.observing; waiting != nil { + h.mu.Unlock() + select { + case <-waiting.done: + return waiting.err + case <-room.ctx.Done(): + return room.ctx.Err() + } + } + if room.streaming { + h.mu.Unlock() + return nil + } + room.observing = &chatObservation{done: make(chan struct{})} + waiting := room.observing + room.shellReadback = &opencode.ShellSnapshot{} + h.mu.Unlock() + var observeErr error + defer func() { + h.mu.Lock() + room.shellReadback = nil + waiting.err = observeErr + room.observing = nil + close(waiting.done) + h.mu.Unlock() + }() + if err := h.startStream(room, broker); err != nil { + observeErr = err + return err + } + shells, err := broker.Shells(room.ctx, room.sessionID) + h.mu.Lock() + if room.ctx.Err() != nil || h.rooms[room.sessionID] != room { + h.mu.Unlock() + observeErr = context.Canceled + return observeErr + } + if err == nil { + shells, err = room.shellReadback.Merge(shells) + } + if err != nil { + // The firehose remains useful when its authoritative shell readback is + // temporarily unavailable. Do not turn that read failure into a chat + // failure (or claim the last retained snapshot is current): clear the + // replay cache and tell attached clients that shell visibility is limited. + // Preserve events already folded during this read, but clear stale replay. + live, mergeErr := room.shellReadback.Merge(nil) + if mergeErr == nil { + h.replaceShellsLocked(room.sessionID, live) + } + h.mu.Unlock() + // Never log broker bodies/URLs: they may contain credentials or private output. + log.Printf("chat shell readback session=%s failed (%T)", room.sessionID, err) + h.fanout(room.sessionID, protocol.Message{Type: protocol.Error, Error: "background shell visibility is limited"}, "") + return nil + } + h.replaceShellsLocked(room.sessionID, shells) + room.shellReadback = nil + h.mu.Unlock() + return nil +} + // attach registers write as one client's subscription to sessionID's stream and // returns it, or the existing subscription when the same connection has already // attached. It is idempotent so a client may prompt and subscribe without @@ -209,6 +294,11 @@ func (h *chatHub) detach(sessionID string, sub *chatSub) { if len(room.subs) == 0 { delete(h.rooms, sessionID) room.cancel() + // ReleaseShells is a local identity-cache operation (no broker I/O). + // Complete it before a reconnect can acquire this session's new room. + if releaser, ok := room.broker.(chatShellReleaser); ok { + releaser.ReleaseShells(sessionID) + } } h.mu.Unlock() } @@ -265,9 +355,17 @@ func (h *chatHub) broadcastUser(frame protocol.ChatEventFrame, exceptClientID st // once it is written. A step-ended frame finishes the turn, letting the queue // advance. func (h *chatHub) emit(frame protocol.ChatEventFrame) { - h.apply(frame) + h.emitRoom(nil, frame) +} + +func (h *chatHub) emitRoom(source *chatRoom, frame protocol.ChatEventFrame) { h.mu.Lock() room := h.rooms[frame.SessionID] + if source != nil && (source != room || source.ctx.Err() != nil) { + h.mu.Unlock() + return + } + h.applyLocked(frame) buffered := false var targets []*chatSub if room != nil { @@ -283,6 +381,22 @@ func (h *chatHub) emit(frame protocol.ChatEventFrame) { if room != nil && frame.Kind == protocol.ChatEventStep && frame.Step != nil && frame.Step.Status == protocol.ChatStepEnded { h.finishTurnLocked(room) } + // Shell replay and live lifecycle delivery must enqueue in the same order + // as folding. Otherwise an exit can overtake an older running replay. + if frame.Kind == protocol.ChatEventShell && !buffered { + msg := protocol.Message{Type: protocol.ChatEvent, ChatEvent: &frame} + var slow []*chatSub + for _, sub := range targets { + if !enqueueChatLocked(sub, msg) { + slow = append(slow, sub) + } + } + h.mu.Unlock() + for _, sub := range slow { + sub.close() + } + return + } h.mu.Unlock() if buffered { return @@ -293,6 +407,62 @@ func (h *chatHub) emit(frame protocol.ChatEventFrame) { } } +// enqueueChatLocked never blocks or calls detach under the hub mutex. +func enqueueChatLocked(sub *chatSub, msg protocol.Message) bool { + select { + case <-sub.done: + return true + default: + } + select { + case sub.ch <- msg: + return true + default: + return false + } +} + +func (h *chatHub) replayShells(sub *chatSub) { + h.mu.Lock() + slow := false + if h.rooms[sub.room.sessionID] == sub.room && sub.room.subs[sub.clientID] == sub { + turn := h.turnLocked(sub.room.sessionID) + for _, id := range turn.shellOrder { + shell := turn.shells[id] + frame := protocol.ChatEventFrame{SessionID: sub.room.sessionID, Kind: protocol.ChatEventShell, Shell: &shell} + if !enqueueChatLocked(sub, protocol.Message{Type: protocol.ChatEvent, ChatEvent: &frame}) { + slow = true + break + } + } + } + h.mu.Unlock() + if slow { + sub.close() + } +} + +// replaceShells reconciles a successful authoritative readback. It drops stale +// retained entries (including exits missed while disconnected) and upserts the +// verified running set without fabricating a lifecycle event for foreign shells. +func (h *chatHub) replaceShells(sessionID string, shells []protocol.ChatShell) { + h.mu.Lock() + defer h.mu.Unlock() + h.replaceShellsLocked(sessionID, shells) +} + +func (h *chatHub) replaceShellsLocked(sessionID string, shells []protocol.ChatShell) { + turn := h.turnLocked(sessionID) + for _, shell := range turn.shells { + turn.shrink(len(shell.Command)) + } + turn.shells = nil + turn.shellOrder = nil + for i := range shells { + turn.applyShell(&shells[i]) + } +} + // finishTurnLocked marks the room's turn finished and releases the turn runner's // wait. Callers must hold h.mu. func (h *chatHub) finishTurnLocked(room *chatRoom) { @@ -331,14 +501,20 @@ func (h *chatHub) flushBuffered(sessionID string) { targets = append(targets, sub) } } - h.mu.Unlock() + var slow []*chatSub for i := range pending { frame := pending[i] msg := protocol.Message{Type: protocol.ChatEvent, ChatEvent: &frame} for _, sub := range targets { - sub.send(msg) + if !enqueueChatLocked(sub, msg) { + slow = append(slow, sub) + } } } + h.mu.Unlock() + for _, sub := range slow { + sub.close() + } } // submitPrompt accepts one prompt: it starts the turn runner immediately when no @@ -476,7 +652,7 @@ func (h *chatHub) runTurn(room *chatRoom, p *chatQueuedPrompt, done chan struct{ return } if first { - if err := h.startStream(room, broker); err != nil { + if err := h.ensureObserving(room, broker); err != nil { h.failPrompt(room, p, err) h.endRoomTurn(room) return @@ -550,7 +726,7 @@ func (h *chatHub) startStream(room *chatRoom, broker chatBroker) error { result <- broker.Stream(ctx, room.sessionID, func() { once.Do(func() { close(attached) }) }, func(frame protocol.ChatEventFrame) error { - h.emit(frame) + h.emitRoom(room, frame) return nil }) }() diff --git a/internal/server/chat_shell_reconcile_test.go b/internal/server/chat_shell_reconcile_test.go new file mode 100644 index 00000000..9ada615b --- /dev/null +++ b/internal/server/chat_shell_reconcile_test.go @@ -0,0 +1,261 @@ +package server + +import ( + "context" + "net/http/httptest" + "strings" + "sync/atomic" + "testing" + "time" + + "git.a-wan.net/alan/pipelines/internal/protocol" + "github.com/coder/websocket" + "github.com/coder/websocket/wsjson" +) + +type shellBarrierBroker struct { + *fakeChat + read chan struct{} + apply chan struct{} + events chan shellBarrierEvent + stopped chan struct{} + reads atomic.Int32 + snapshot []protocol.ChatShell +} + +// Exercise the real websocket handler on a loopback scratch server and a fresh +// SQLite database. A marker on the shared stream bounds the replay assertion +// without sleeps, polling or a negative-read timeout. +func TestChatShellReconciliationScratchServer(t *testing.T) { + b := newShellBarrierBroker() + b.snapshot = []protocol.ChatShell{{ID: "sh_old", Status: protocol.ChatShellRunning}} + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + srv := httptest.NewServer(HandlerWithOptions(ctx, Options{Chat: b}, newOperatorStore(t))) + defer srv.Close() + conn, _, err := websocket.Dial(ctx, "ws"+strings.TrimPrefix(srv.URL, "http")+"/connect", nil) + if err != nil { + t.Fatal(err) + } + defer conn.CloseNow() + subscribe := func(id string) { + t.Helper() + if err := wsjson.Write(ctx, conn, protocol.Message{Type: protocol.ChatSubscribe, ID: id, ChatSubscribe: &protocol.ChatQuery{SessionID: "ses_1"}}); err != nil { + t.Fatal(err) + } + } + read := func() protocol.Message { + t.Helper() + var m protocol.Message + if err := wsjson.Read(ctx, conn, &m); err != nil { + t.Fatal(err) + } + return m + } + if m := read(); m.Type != protocol.PipelineList { + t.Fatalf("server greeting = %s", m.Type) + } + subscribe("first") + if m := read(); m.ID != "first" || m.Type != protocol.ChatSubscribe { + t.Fatalf("ack = %+v", m) + } + shellBarrierWait(t, b.read) + done := make(chan struct{}) + b.events <- shellBarrierEvent{protocol.ChatShell{ID: "sh_old", Status: protocol.ChatShellExited}, done} + shellBarrierWait(t, done) + if m := read(); m.ChatEvent == nil || m.ChatEvent.Shell.Status != protocol.ChatShellExited { + t.Fatalf("exit not visible: %+v", m) + } + close(b.apply) + subscribe("second") + for { + m := read() + if m.ChatEvent != nil && m.ChatEvent.Shell.Status == protocol.ChatShellRunning { + t.Fatalf("stale replay: %+v", m) + } + if m.ID == "second" { + break + } + } + done = make(chan struct{}) + b.events <- shellBarrierEvent{protocol.ChatShell{ID: "sh_marker", Status: protocol.ChatShellExited}, done} + shellBarrierWait(t, done) + for { + m := read() + if m.ChatEvent == nil { + continue + } + if m.ChatEvent.Shell.Status == protocol.ChatShellRunning { + t.Fatalf("ghost visible after apply: %+v", m) + } + if m.ChatEvent.Shell.ID == "sh_marker" { + break + } + } + if b.reads.Load() != 1 { + t.Fatalf("shared observer read count = %d", b.reads.Load()) + } + conn.CloseNow() + shellBarrierWait(t, b.stopped) +} + +type shellBarrierEvent struct { + shell protocol.ChatShell + done chan struct{} +} + +func newShellBarrierBroker() *shellBarrierBroker { + return &shellBarrierBroker{fakeChat: scriptedChat(), read: make(chan struct{}, 8), apply: make(chan struct{}), events: make(chan shellBarrierEvent), stopped: make(chan struct{})} +} + +func (b *shellBarrierBroker) Shells(ctx context.Context, _ string) ([]protocol.ChatShell, error) { + b.reads.Add(1) + snapshot := append([]protocol.ChatShell(nil), b.snapshot...) + b.read <- struct{}{} + select { + case <-b.apply: + return snapshot, nil + case <-ctx.Done(): + return nil, ctx.Err() + } +} + +func (b *shellBarrierBroker) Stream(ctx context.Context, id string, ready func(), emit func(protocol.ChatEventFrame) error) error { + defer close(b.stopped) + ready() + for { + select { + case <-ctx.Done(): + return ctx.Err() + case event := <-b.events: + if err := emit(protocol.ChatEventFrame{SessionID: id, Kind: protocol.ChatEventShell, Shell: &event.shell}); err != nil { + return err + } + close(event.done) + } + } +} + +func shellBarrierWait(t *testing.T, ch <-chan struct{}) { + t.Helper() + select { + case <-ch: + case <-time.After(3 * time.Second): + t.Fatal("barrier timed out") + } +} + +func TestChatShellReconciliationBarrier(t *testing.T) { + for _, status := range []string{protocol.ChatShellExited, protocol.ChatShellRunning, "timeout", "killed"} { + t.Run(status, func(t *testing.T) { + b := newShellBarrierBroker() + b.snapshot = []protocol.ChatShell{{ID: "sh_old", Status: protocol.ChatShellRunning}} + h := newChatHub() + messages := make(chan protocol.Message, 32) + sub := h.attach("ses_1", "one", "", b, func(m protocol.Message) error { messages <- m; return nil }) + t.Cleanup(sub.close) + result := make(chan error, 2) + go func() { result <- h.ensureObserving(sub.room, b) }() + shellBarrierWait(t, b.read) + // A second observer must share the in-flight read, not apply an + // independently completing older/newer snapshot out of order. + go func() { result <- h.ensureObserving(sub.room, b) }() + shell := protocol.ChatShell{ID: "sh_old", Status: status} + if status == protocol.ChatShellRunning { + shell.ID = "sh_new" + shell.Command = "sleep 10" + } + done := make(chan struct{}) + select { + case b.events <- shellBarrierEvent{shell, done}: + case <-time.After(time.Second): + t.Fatal("event blocked behind readback") + } + shellBarrierWait(t, done) + select { + case m := <-messages: + if m.ChatEvent == nil || m.ChatEvent.Shell == nil || m.ChatEvent.Shell.ID != shell.ID || m.ChatEvent.Shell.Status != status { + t.Fatalf("live shell = %+v", m) + } + case <-time.After(time.Second): + t.Fatal("live event not visible during read") + } + close(b.apply) + for range 2 { + select { + case err := <-result: + if err != nil { + t.Fatal(err) + } + case <-time.After(time.Second): + t.Fatal("observer stuck") + } + } + if b.reads.Load() != 1 { + t.Fatalf("redundant/out-of-order reads: %d", b.reads.Load()) + } + snapshot := h.shellSnapshot("ses_1") + if status == protocol.ChatShellRunning { + if len(snapshot) != 2 { + t.Fatalf("new create lost: %+v", snapshot) + } + } else if len(snapshot) != 0 { + t.Fatalf("terminal shell resurrected: %+v", snapshot) + } + // Assert the same retained state is replayed to the client. + h.replayShells(sub) + for range len(snapshot) { + select { + case m := <-messages: + if m.ChatEvent == nil || m.ChatEvent.Shell.Status != protocol.ChatShellRunning { + t.Fatalf("replay = %+v", m) + } + case <-time.After(time.Second): + t.Fatal("replay missing") + } + } + sub.close() + shellBarrierWait(t, b.stopped) + }) + } +} + +func TestChatShellReconciliationCancelReconnectBarrier(t *testing.T) { + b := newShellBarrierBroker() + h := newChatHub() + sub := h.attach("ses_1", "one", "", b, func(protocol.Message) error { return nil }) + results := make(chan error, 2) + go func() { results <- h.ensureObserving(sub.room, b) }() + shellBarrierWait(t, b.read) + go func() { results <- h.ensureObserving(sub.room, b) }() + sub.close() + for range 2 { + select { + case err := <-results: + if err == nil { + t.Fatal("canceled observer succeeded") + } + case <-time.After(time.Second): + t.Fatal("canceled waiter stuck") + } + } + shellBarrierWait(t, b.stopped) + next := newShellBarrierBroker() + next.snapshot = []protocol.ChatShell{{ID: "sh_new", Status: protocol.ChatShellRunning}} + close(next.apply) + newSub := h.attach("ses_1", "two", "", next, func(protocol.Message) error { return nil }) + t.Cleanup(newSub.close) + if err := h.ensureObserving(newSub.room, next); err != nil { + t.Fatal(err) + } + // A late callback from the canceled stream cannot poison the new room, + // nor can a foreign-session callback from the current stream. + h.emitRoom(sub.room, protocol.ChatEventFrame{SessionID: "ses_1", Kind: protocol.ChatEventShell, Shell: &protocol.ChatShell{ID: "sh_ghost", Status: protocol.ChatShellRunning}}) + h.emitRoom(newSub.room, protocol.ChatEventFrame{SessionID: "ses_foreign", Kind: protocol.ChatEventShell, Shell: &protocol.ChatShell{ID: "sh_foreign", Status: protocol.ChatShellRunning}}) + got := h.shellSnapshot("ses_1") + if len(got) != 1 || got[0].ID != "sh_new" || len(h.shellSnapshot("ses_foreign")) != 0 { + t.Fatalf("reconnect state = %+v", got) + } + newSub.close() + shellBarrierWait(t, next.stopped) +} diff --git a/internal/server/chat_stream.go b/internal/server/chat_stream.go index 76c175e6..89ba6404 100644 --- a/internal/server/chat_stream.go +++ b/internal/server/chat_stream.go @@ -111,7 +111,14 @@ func handleChatSubscribe(request protocol.Message, broker chatBroker, hub *chatH _ = write(protocol.Message{Type: protocol.Error, ID: request.ID, Error: "could not attach to session " + query.SessionID}) return } + // Acknowledge before the stream is allowed to fan out: otherwise an immediately + // available retained shell can overtake the subscribe reply on this connection. _ = write(protocol.Message{Type: protocol.ChatSubscribe, ID: request.ID}) + if err := hub.ensureObserving(sub.room, broker); err != nil { + sub.close() + _ = write(protocol.Message{Type: protocol.Error, ID: request.ID, Error: "could not observe session " + query.SessionID}) + return + } // Replay every still-unresolved approval request the room holds, not only the // most recent one, so a client that attaches after two requests were raised // still sees both cards. @@ -124,9 +131,5 @@ func handleChatSubscribe(request protocol.Message, broker chatBroker, hub *chatH // shows them rather than nothing until they exit. The message snapshot cannot // carry a shell — a shell outlives the turn and is not a turn — so this is the // shell half of the resume. - for _, shell := range hub.shellSnapshot(query.SessionID) { - running := shell - frame := protocol.ChatEventFrame{SessionID: query.SessionID, Kind: protocol.ChatEventShell, Shell: &running} - sub.send(protocol.Message{Type: protocol.ChatEvent, ChatEvent: &frame}) - } + hub.replayShells(sub) } diff --git a/internal/server/chat_test.go b/internal/server/chat_test.go index 29dbc4bb..d4a15cd2 100644 --- a/internal/server/chat_test.go +++ b/internal/server/chat_test.go @@ -3,6 +3,7 @@ package server import ( "context" "encoding/json" + "errors" "fmt" "net/http" "net/http/httptest" @@ -52,6 +53,8 @@ type fakeChat struct { permissionSessionID string permissions []protocol.ChatPermissionFrame permissionErr error + shells []protocol.ChatShell + shellErr error sessionCalls int sessions []protocol.ChatSession sessionErr error @@ -62,6 +65,8 @@ type fakeChat struct { started chan struct{} done chan struct{} startOnce sync.Once + streamCalls int + readyGate <-chan struct{} block bool subscribed bool promptBeforeSubscribe bool @@ -91,13 +96,18 @@ func (f *fakeChat) Prompt(context.Context, string, string) error { func (f *fakeChat) Stream(ctx context.Context, _ string, ready func(), emit func(protocol.ChatEventFrame) error) error { f.mu.Lock() f.subscribed = true + f.streamCalls++ + readyGate := f.readyGate f.mu.Unlock() - if ready != nil { - ready() - } if f.started != nil { f.startOnce.Do(func() { close(f.started) }) } + if readyGate != nil { + <-readyGate + } + if ready != nil { + ready() + } if f.block { <-ctx.Done() if f.done != nil { @@ -116,6 +126,42 @@ func (f *fakeChat) Stream(ctx context.Context, _ string, ready func(), emit func return nil } +// TestChatHubConcurrentObserversShareStream pins the start-up handoff: two +// clients that subscribe before the broker says ready still share one firehose. +func TestChatHubConcurrentObserversShareStream(t *testing.T) { + broker := scriptedChat() + broker.block = true + broker.started = make(chan struct{}) + gate := make(chan struct{}) + broker.readyGate = gate + hub := newChatHub() + hub.mu.Lock() + room := hub.roomLocked("ses_1") + hub.mu.Unlock() + + errs := make(chan error, 2) + go func() { errs <- hub.ensureObserving(room, broker) }() + select { + case <-broker.started: + case <-time.After(time.Second): + t.Fatal("first observer did not start the broker stream") + } + go func() { errs <- hub.ensureObserving(room, broker) }() + close(gate) + for range 2 { + if err := <-errs; err != nil { + t.Fatalf("ensureObserving: %v", err) + } + } + broker.mu.Lock() + calls := broker.streamCalls + broker.mu.Unlock() + if calls != 1 { + t.Fatalf("broker Stream calls = %d, want 1", calls) + } + room.cancel() +} + func (f *fakeChat) Interrupt(context.Context, string) error { f.mu.Lock() defer f.mu.Unlock() @@ -151,6 +197,12 @@ func (f *fakeChat) Reply(_ context.Context, request protocol.ChatReplyRequest) e // Permissions returns the seeded pending requests, or blocks until the caller's // context is canceled when permBlock is set (used to prove the poller stops on // disconnect). Every call is counted so a test can assert polling happened. +func (f *fakeChat) Shells(context.Context, string) ([]protocol.ChatShell, error) { + f.mu.Lock() + defer f.mu.Unlock() + return append([]protocol.ChatShell(nil), f.shells...), f.shellErr +} + func (f *fakeChat) Permissions(ctx context.Context, sessionID string) ([]protocol.ChatPermissionFrame, error) { f.mu.Lock() f.permissionCalls++ @@ -682,6 +734,9 @@ func TestChatInterruptEndsRunningPrompt(t *testing.T) { case r.Method == http.MethodGet && r.URL.Path == "/api/permission/request": w.Header().Set("Content-Type", "application/json") _, _ = w.Write([]byte(`{"location":{"directory":"/work"},"data":[]}`)) + case r.Method == http.MethodGet && r.URL.Path == "/api/shell": + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"data":[]}`)) case r.Method == http.MethodGet && r.URL.Path == "/api/event": flusher, ok := w.(http.Flusher) if !ok { @@ -995,6 +1050,7 @@ func TestChatPromptStreamsShellFrame(t *testing.T) { // shows the background process instead of nothing until it exits. func TestChatSubscribeReplaysRunningShell(t *testing.T) { broker := scriptedChat() + broker.shells = []protocol.ChatShell{{ID: "sh_1", Status: protocol.ChatShellRunning, Command: "sleep 120"}} broker.script = append([]protocol.ChatEventFrame{ {SessionID: "ses_1", Kind: protocol.ChatEventShell, Shell: &protocol.ChatShell{ID: "sh_1", Status: protocol.ChatShellRunning, Command: "sleep 120"}}, }, broker.script...) @@ -1039,6 +1095,40 @@ func TestChatSubscribeReplaysRunningShell(t *testing.T) { } } +// TestChatSubscribeKeepsStreamWhenShellReadbackFails ensures a temporary +// authoritative readback failure does not make chat unavailable or replay a +// stale process as running. The subscriber receives an explicit limitation +// while its broker stream remains attached for future lifecycle events. +func TestChatSubscribeKeepsStreamWhenShellReadbackFails(t *testing.T) { + broker := scriptedChat() + broker.block = true + broker.shellErr = errors.New("shell endpoint unavailable: https://user:secret@example.invalid private response body") + ctx, _, conn := dialChat(t, broker) + if err := wsjson.Write(ctx, conn, protocol.Message{Type: protocol.ChatSubscribe, ID: "sub1", ChatSubscribe: &protocol.ChatQuery{SessionID: "ses_1"}}); err != nil { + t.Fatal(err) + } + var ack protocol.Message + if err := wsjson.Read(ctx, conn, &ack); err != nil { + t.Fatal(err) + } + if ack.Type != protocol.ChatSubscribe || ack.ID != "sub1" { + t.Fatalf("subscribe ack = %+v", ack) + } + var limited protocol.Message + if err := wsjson.Read(ctx, conn, &limited); err != nil { + t.Fatal(err) + } + if limited.Type != protocol.Error || limited.Error != "background shell visibility is limited" { + t.Fatalf("limited visibility notice = %+v", limited) + } + broker.mu.Lock() + calls := broker.streamCalls + broker.mu.Unlock() + if calls != 1 { + t.Fatalf("broker Stream calls = %d, want 1", calls) + } +} + // TestChatPromptSubscribesBeforePrompt pins the subscribe-before-prompt order: // the firehose has no replay cursor, so the subscription must be attached before // the prompt is posted or deltas emitted in between are lost. @@ -1079,6 +1169,9 @@ func realAdapterChat(t *testing.T, frames []string) []protocol.ChatEventFrame { case r.Method == http.MethodGet && r.URL.Path == "/api/permission/request": w.Header().Set("Content-Type", "application/json") _, _ = w.Write([]byte(`{"location":{"directory":"/work"},"data":[]}`)) + case r.Method == http.MethodGet && r.URL.Path == "/api/shell": + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"data":[]}`)) case r.Method == http.MethodGet && r.URL.Path == "/api/event": streams.Add(1) w.Header().Set("Content-Type", "text/event-stream") diff --git a/internal/server/delivery_gate.go b/internal/server/delivery_gate.go new file mode 100644 index 00000000..df1dc7c5 --- /dev/null +++ b/internal/server/delivery_gate.go @@ -0,0 +1,185 @@ +package server + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "strings" + "syscall" + "time" + + "git.a-wan.net/alan/pipelines/internal/stage" +) + +// verifyDelivery proves that the stage landed a complete, committed tree before +// its success can settle the run. It re-derives the capability after the stage: +// a stage result and its input cannot select either the landing path or checks. +const maxDeliveryCheckOutput = 64 << 10 +const deliveryCleanupTimeout = 10 * time.Second + +// verifyDelivery verifies the committed item tree in a disposable worktree +// before fast-forwarding canonical main. Settlement deliberately performs only +// non-mutating repository checks after that mutation: a declared check must +// never be the first signal that the landed tree is known-red. +func verifyDelivery(ctx context.Context, workspace string, access stage.LandingAccess, checks []string) error { + lock, err := acquireDeliveryLock(ctx, access.LockPath) + if err != nil { + return err + } + defer lock.Close() + current, err := landingAccess(workspace) + if err != nil { + return fmt.Errorf("delivery access changed: %w", err) + } + if current != access { + return fmt.Errorf("delivery access changed during stage") + } + for _, directory := range []string{workspace, access.MainWorktree} { + status, err := gitOutput(directory, "status", "--porcelain=v1", "--untracked-files=all") + if err != nil { + return fmt.Errorf("delivery checkout status: %w", err) + } + if status != "" { + return fmt.Errorf("delivery checkout is not clean: %s", strings.Split(status, "\n")[0]) + } + } + itemHead, err := gitOutput(workspace, "rev-parse", "HEAD") + if err != nil { + return fmt.Errorf("delivery item commit: %w", err) + } + if err := runDeliveryCandidate(ctx, workspace, itemHead, checks); err != nil { + return err + } + mainHead, err := gitOutput(access.MainWorktree, "rev-parse", "HEAD") + if err != nil { + return fmt.Errorf("delivery main commit: %w", err) + } + if err := exec.CommandContext(ctx, "git", "-C", workspace, "merge-base", "--is-ancestor", mainHead, itemHead).Run(); err != nil { + return fmt.Errorf("delivery item commit cannot fast-forward canonical main") + } + if output, err := exec.CommandContext(ctx, "git", "-C", access.MainWorktree, "merge", "--ff-only", itemHead).CombinedOutput(); err != nil { + return fmt.Errorf("delivery fast-forward canonical main: %w: %s", err, strings.TrimSpace(string(output))) + } + for _, directory := range []string{workspace, access.MainWorktree} { + status, err := gitOutput(directory, "status", "--porcelain=v1", "--untracked-files=all") + if err != nil { + return fmt.Errorf("delivery settlement status: %w", err) + } + if status != "" { + return fmt.Errorf("delivery checkout is not clean after checks: %s", strings.Split(status, "\n")[0]) + } + } + // Re-derive at the settlement boundary. A success means exactly the checked + // item commit is now reachable from the same canonical main worktree. + current, err = landingAccess(workspace) + if err != nil || current != access { + return fmt.Errorf("delivery access changed during settlement") + } + mainHead, err = gitOutput(access.MainWorktree, "rev-parse", "HEAD") + if err != nil { + return fmt.Errorf("delivery main commit: %w", err) + } + if err := exec.CommandContext(ctx, "git", "-C", workspace, "merge-base", "--is-ancestor", itemHead, mainHead).Run(); err != nil { + return fmt.Errorf("delivery item commit is not an ancestor of canonical main") + } + return nil +} + +func acquireDeliveryLock(ctx context.Context, path string) (*os.File, error) { + file, err := os.OpenFile(path, os.O_CREATE|os.O_RDWR, 0600) + if err != nil { + return nil, fmt.Errorf("delivery lock: %w", err) + } + for { + err := syscall.Flock(int(file.Fd()), syscall.LOCK_EX|syscall.LOCK_NB) + if err == nil { + return file, nil + } + if !errors.Is(err, syscall.EWOULDBLOCK) && !errors.Is(err, syscall.EAGAIN) { + _ = file.Close() + return nil, fmt.Errorf("delivery lock: %w", err) + } + select { + case <-ctx.Done(): + _ = file.Close() + return nil, fmt.Errorf("delivery lock: %w", ctx.Err()) + case <-time.After(20 * time.Millisecond): + } + } +} + +func runDeliveryCandidate(ctx context.Context, workspace, itemHead string, checks []string) error { + candidate, err := os.MkdirTemp(filepath.Dir(workspace), ".pipelines-delivery-") + if err != nil { + return fmt.Errorf("create delivery candidate: %w", err) + } + if err := os.Remove(candidate); err != nil { + return fmt.Errorf("prepare delivery candidate: %w", err) + } + if output, err := boundedCommandOutput(ctx, workspace, "worktree", "add", "--detach", candidate, itemHead); err != nil { + cleanupDeliveryCandidate(workspace, candidate) + return fmt.Errorf("create delivery candidate: %w: %s", err, strings.TrimSpace(string(output))) + } + defer cleanupDeliveryCandidate(workspace, candidate) + for _, check := range checks { + if err := runDeliveryCheck(ctx, candidate, check); err != nil { + return err + } + } + return nil +} + +// cleanupDeliveryCandidate always uses its own bounded context: the stage context +// may already be canceled when a check fails or times out. +func cleanupDeliveryCandidate(workspace, candidate string) { + ctx, cancel := context.WithTimeout(context.Background(), deliveryCleanupTimeout) + defer cancel() + _ = exec.CommandContext(ctx, "git", "-C", workspace, "worktree", "remove", "--force", candidate).Run() + _ = os.RemoveAll(candidate) +} + +func boundedCommandOutput(ctx context.Context, directory string, args ...string) ([]byte, error) { + cmd := exec.CommandContext(ctx, "git", append([]string{"-C", directory}, args...)...) + var output boundedDeliveryOutput + cmd.Stdout = &output + cmd.Stderr = &output + err := cmd.Run() + return []byte(output.String()), err +} + +func runDeliveryCheck(ctx context.Context, directory, check string) error { + cmd := exec.CommandContext(ctx, "sh", "-c", check) + cmd.Dir = directory + var output boundedDeliveryOutput + cmd.Stdout = &output + cmd.Stderr = &output + if err := cmd.Run(); err != nil { + return fmt.Errorf("delivery check %q failed: %w: %s", check, err, output.String()) + } + return nil +} + +type boundedDeliveryOutput struct{ bytes.Buffer } + +func (b *boundedDeliveryOutput) Write(p []byte) (int, error) { + remaining := maxDeliveryCheckOutput - b.Len() + if remaining > 0 { + _, _ = b.Buffer.Write(p[:min(remaining, len(p))]) + } + return len(p), nil +} + +func (b *boundedDeliveryOutput) String() string { + text := strings.TrimSpace(b.Buffer.String()) + if b.Len() == maxDeliveryCheckOutput { + return text + " (output truncated)" + } + return text +} + +var _ io.Writer = (*boundedDeliveryOutput)(nil) diff --git a/internal/server/landing_access.go b/internal/server/landing_access.go new file mode 100644 index 00000000..a373d625 --- /dev/null +++ b/internal/server/landing_access.go @@ -0,0 +1,130 @@ +package server + +import ( + "bytes" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + + "git.a-wan.net/alan/pipelines/internal/stage" +) + +// landingAccess derives delivery paths from Git's verified worktree metadata, +// never from stage input. A missing canonical main worktree fails closed. +func landingAccess(workspace string) (stage.LandingAccess, error) { + workspace, err := canonicalDirectory(workspace) + if err != nil { + return stage.LandingAccess{}, fmt.Errorf("delivery workspace: %w", err) + } + gitDir, err := gitOutput(workspace, "rev-parse", "--git-dir") + if err != nil { + return stage.LandingAccess{}, fmt.Errorf("delivery workspace is not a Git worktree: %w", err) + } + commonDir, err := gitOutput(workspace, "rev-parse", "--git-common-dir") + if err != nil { + return stage.LandingAccess{}, fmt.Errorf("delivery repository metadata: %w", err) + } + gitDir, err = canonicalDirectory(resolveGitPath(workspace, gitDir)) + if err != nil { + return stage.LandingAccess{}, fmt.Errorf("delivery Git metadata: %w", err) + } + commonDir, err = canonicalDirectory(resolveGitPath(workspace, commonDir)) + if err != nil { + return stage.LandingAccess{}, fmt.Errorf("delivery common Git metadata: %w", err) + } + main, err := mainWorktree(workspace, commonDir) + if err != nil { + return stage.LandingAccess{}, err + } + // Delivery originates in a linked item worktree. Its metadata must be under + // the repository's canonical common directory, never a standalone or foreign + // checkout. Git's common directory is the bounded capability needed for refs, + // worktree state, and the landing lock. + if !pathWithin(filepath.Join(commonDir, "worktrees"), gitDir) { + return stage.LandingAccess{}, fmt.Errorf("delivery workspace is not a linked Git worktree") + } + return stage.LandingAccess{MainWorktree: main, GitMetadata: commonDir, LockPath: filepath.Join(commonDir, "pipelines-landing.lock")}, nil +} + +func mainWorktree(workspace, commonDir string) (string, error) { + list, err := gitOutput(workspace, "worktree", "list", "--porcelain") + if err != nil { + return "", fmt.Errorf("list repository worktrees: %w", err) + } + for _, record := range strings.Split(strings.TrimSpace(list), "\n\n") { + lines := strings.Split(record, "\n") + if len(lines) == 0 || !strings.HasPrefix(lines[0], "worktree ") || !worktreeHasBranch(lines, "refs/heads/main") { + continue + } + candidate, err := canonicalDirectory(strings.TrimPrefix(lines[0], "worktree ")) + if err != nil || candidate == workspace { + continue + } + candidateGitDir, err := gitOutput(candidate, "rev-parse", "--git-dir") + if err != nil { + continue + } + candidateGitDir, err = canonicalDirectory(resolveGitPath(candidate, candidateGitDir)) + if err != nil || candidateGitDir != commonDir { + continue + } + candidateCommon, err := gitOutput(candidate, "rev-parse", "--git-common-dir") + if err != nil { + continue + } + candidateCommon, err = canonicalDirectory(resolveGitPath(candidate, candidateCommon)) + if err == nil && candidateCommon == commonDir { + return candidate, nil + } + } + return "", fmt.Errorf("no canonical main worktree in the delivery repository") +} + +func worktreeHasBranch(lines []string, branch string) bool { + for _, line := range lines[1:] { + if line == "branch "+branch { + return true + } + } + return false +} + +func canonicalDirectory(path string) (string, error) { + resolved, err := filepath.EvalSymlinks(path) + if err != nil { + return "", err + } + info, err := os.Stat(resolved) + if err != nil { + return "", err + } + if !info.IsDir() { + return "", fmt.Errorf("not a directory") + } + return filepath.Clean(resolved), nil +} + +func resolveGitPath(workspace, value string) string { + if filepath.IsAbs(value) { + return value + } + return filepath.Join(workspace, value) +} + +func pathWithin(parent, child string) bool { + rel, err := filepath.Rel(parent, child) + return err == nil && rel != "." && rel != ".." && !strings.HasPrefix(rel, ".."+string(filepath.Separator)) +} + +func gitOutput(directory string, args ...string) (string, error) { + cmd := exec.Command("git", append([]string{"-C", directory}, args...)...) + var stderr bytes.Buffer + cmd.Stderr = &stderr + data, err := cmd.Output() + if err != nil { + return "", fmt.Errorf("git %s: %w: %s", strings.Join(args, " "), err, strings.TrimSpace(stderr.String())) + } + return strings.TrimSpace(string(data)), nil +} diff --git a/internal/server/landing_access_test.go b/internal/server/landing_access_test.go new file mode 100644 index 00000000..9d2d7ae8 --- /dev/null +++ b/internal/server/landing_access_test.go @@ -0,0 +1,304 @@ +package server + +import ( + "context" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + + "git.a-wan.net/alan/pipelines/internal/pipeline" + "git.a-wan.net/alan/pipelines/internal/protocol" + "git.a-wan.net/alan/pipelines/internal/stage" +) + +func TestLandingAccessAcceptsOnlyVerifiedSameRepositoryMainWorktree(t *testing.T) { + root := t.TempDir() + main := filepath.Join(root, "main") + runGit(t, root, "init", "-b", "main", main) + runGit(t, main, "config", "user.email", "test@example.invalid") + runGit(t, main, "config", "user.name", "test") + if err := os.WriteFile(filepath.Join(main, "README"), []byte("base\n"), 0600); err != nil { + t.Fatal(err) + } + runGit(t, main, "add", "README") + runGit(t, main, "commit", "-m", "base") + item := filepath.Join(root, "item") + runGit(t, main, "worktree", "add", "-b", "item", item) + access, err := landingAccess(item) + if err != nil { + t.Fatal(err) + } + canonicalMain, err := canonicalDirectory(main) + if err != nil { + t.Fatal(err) + } + if access.MainWorktree != canonicalMain || access.GitMetadata != filepath.Join(canonicalMain, ".git") || access.LockPath != filepath.Join(canonicalMain, ".git", "pipelines-landing.lock") { + t.Fatalf("unexpected access: %+v", access) + } + foreign := filepath.Join(root, "foreign") + runGit(t, root, "init", "-b", "main", foreign) + if _, err := landingAccess(foreign); err == nil { + t.Fatal("foreign repository without an item worktree gained landing access") + } + if _, err := landingAccess(main); err == nil { + t.Fatal("main worktree gained linked-worktree delivery access") + } + if err := os.RemoveAll(item); err != nil { + t.Fatal(err) + } + runGit(t, main, "worktree", "prune") + if _, err := landingAccess(main); err == nil { + t.Fatal("repository without a linked delivery worktree gained landing access") + } + secondItem := filepath.Join(root, "second-item") + runGit(t, main, "worktree", "add", "-b", "second-item", secondItem) + runGit(t, main, "branch", "-m", "main", "trunk") + if _, err := landingAccess(secondItem); err == nil { + t.Fatal("repository without a canonical main worktree gained landing access") + } +} + +func TestVerifyDeliveryRequiresCleanLandedMainAndPassingChecks(t *testing.T) { + root := t.TempDir() + main := filepath.Join(root, "main") + runGit(t, root, "init", "-b", "main", main) + runGit(t, main, "config", "user.email", "test@example.invalid") + runGit(t, main, "config", "user.name", "test") + if err := os.WriteFile(filepath.Join(main, "README"), []byte("base\n"), 0600); err != nil { + t.Fatal(err) + } + runGit(t, main, "add", "README") + runGit(t, main, "commit", "-m", "base") + item := filepath.Join(root, "item") + runGit(t, main, "worktree", "add", "-b", "item", item) + if err := os.WriteFile(filepath.Join(item, "delivered"), []byte("complete\n"), 0600); err != nil { + t.Fatal(err) + } + runGit(t, item, "add", "delivered") + runGit(t, item, "commit", "-m", "delivery") + runGit(t, main, "merge", "--ff-only", "item") + access, err := landingAccess(item) + if err != nil { + t.Fatal(err) + } + if err := verifyDelivery(context.Background(), item, access, []string{"test -f delivered"}); err != nil { + t.Fatalf("complete landed delivery rejected: %v", err) + } + if err := os.WriteFile(filepath.Join(item, "missing-helper"), []byte("dirty\n"), 0600); err != nil { + t.Fatal(err) + } + if err := verifyDelivery(context.Background(), item, access, []string{"true"}); err == nil { + t.Fatal("dirty task-owned helper was accepted") + } + if err := os.Remove(filepath.Join(item, "missing-helper")); err != nil { + t.Fatal(err) + } + if err := verifyDelivery(context.Background(), item, access, []string{"false"}); err == nil { + t.Fatal("failed required check was accepted") + } + if err := os.WriteFile(filepath.Join(item, "unlanded"), []byte("no\n"), 0600); err != nil { + t.Fatal(err) + } + runGit(t, item, "add", "unlanded") + runGit(t, item, "commit", "-m", "unlanded") + if err := verifyDelivery(context.Background(), item, access, []string{"test -f unlanded"}); err != nil { + t.Fatalf("complete unlanded item was not landed after candidate verification: %v", err) + } + mainHead := runGitOutput(t, main, "rev-parse", "HEAD") + itemHead := runGitOutput(t, item, "rev-parse", "HEAD") + if mainHead != itemHead { + t.Fatal("verified item was not fast-forwarded into canonical main") + } +} + +func TestVerifyDeliveryFailingCandidateLeavesMainUnchanged(t *testing.T) { + root := t.TempDir() + main := filepath.Join(root, "main") + runGit(t, root, "init", "-b", "main", main) + runGit(t, main, "config", "user.email", "test@example.invalid") + runGit(t, main, "config", "user.name", "test") + if err := os.WriteFile(filepath.Join(main, "README"), []byte("base\n"), 0600); err != nil { + t.Fatal(err) + } + runGit(t, main, "add", "README") + runGit(t, main, "commit", "-m", "base") + item := filepath.Join(root, "item") + runGit(t, main, "worktree", "add", "-b", "item", item) + if err := os.WriteFile(filepath.Join(item, "candidate"), []byte("candidate\n"), 0600); err != nil { + t.Fatal(err) + } + runGit(t, item, "add", "candidate") + runGit(t, item, "commit", "-m", "candidate") + access, err := landingAccess(item) + if err != nil { + t.Fatal(err) + } + before := runGitOutput(t, main, "rev-parse", "HEAD") + if err := verifyDelivery(context.Background(), item, access, []string{"test -f candidate", "false"}); err == nil { + t.Fatal("failing candidate check landed main") + } + if after := runGitOutput(t, main, "rev-parse", "HEAD"); after != before { + t.Fatal("failing candidate check changed canonical main") + } +} + +func TestRunnerSettlesSuccessfulDeliveryAfterCandidateLanding(t *testing.T) { + ctx := context.Background() + root := t.TempDir() + main := filepath.Join(root, "main") + runGit(t, root, "init", "-b", "main", main) + runGit(t, main, "config", "user.email", "test@example.invalid") + runGit(t, main, "config", "user.name", "test") + if err := os.WriteFile(filepath.Join(main, "README"), []byte("base\n"), 0600); err != nil { + t.Fatal(err) + } + runGit(t, main, "add", "README") + runGit(t, main, "commit", "-m", "base") + item := filepath.Join(root, "item") + runGit(t, main, "worktree", "add", "-b", "item", item) + if err := os.WriteFile(filepath.Join(item, "unlanded"), []byte("no\n"), 0600); err != nil { + t.Fatal(err) + } + runGit(t, item, "add", "unlanded") + runGit(t, item, "commit", "-m", "unlanded") + + store, err := pipeline.Open(filepath.Join(root, "runner.db")) + if err != nil { + t.Fatal(err) + } + defer store.Close() + source := pipeline.Template([]string{"committer"}, true) + source = strings.Replace(source, "```configs", "```checks\ntrue\n```\n\n```configs", 1) + source = strings.Replace(source, "timeout=1h", "timeout=1h\ndelivery=true", 1) + if _, err := store.Submit(ctx, "delivery", source); err != nil { + t.Fatal(err) + } + items, err := store.List(ctx) + if err != nil { + t.Fatal(err) + } + r, err := newRunner(ctx, store) + if err != nil { + t.Fatal(err) + } + r.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) { + return stage.Result{Outcome: "success", Output: stage.Values{}}, nil + } + run, err := r.start(protocol.Message{PipelineID: items[0].ID, Version: 1, RequestKey: "unlanded", Workspace: item, Input: map[string]any{"prompt": "task"}}) + if err != nil { + t.Fatal(err) + } + r.wg.Wait() + detail, err := store.GetRun(ctx, run.ID) + if err != nil { + t.Fatal(err) + } + if detail.Status != "succeeded" || len(detail.Steps) != 1 { + t.Fatalf("complete delivery did not settle successfully: %+v", detail) + } + if runGitOutput(t, main, "rev-parse", "HEAD") != runGitOutput(t, item, "rev-parse", "HEAD") { + t.Fatal("runner did not land the verified item commit") + } +} + +func TestRunnerRejectsWrongBackendVerificationEvidence(t *testing.T) { + ctx := context.Background() + source := pipeline.Template([]string{"worker"}, true) + source = strings.Replace(source, "```configs", "```verification\nbackend=branch-api\nrevision=abc123\ncapability=controlled-state\nfixture=reconnect-foreground\n```\n\n```configs", 1) + r, store, id := submitRunner(t, ctx, source) + defer store.Close() + r.verifyTarget = func(context.Context, stage.VerificationTarget) (stage.VerificationEvidence, error) { + return stage.VerificationEvidence{Backend: "old-api", Revision: "abc123", Capability: "controlled-state", Fixture: "reconnect-foreground"}, nil + } + r.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) { + return stage.Result{Outcome: "success", Output: stage.Values{}}, nil + } + run := startRun(t, r, id, "wrong-backend-evidence") + detail, err := store.GetRun(ctx, run.ID) + if err != nil { + t.Fatal(err) + } + if detail.Status != "failed" || len(detail.Steps) != 1 || !strings.Contains(detail.Steps[0].Diagnostic, "backend") { + t.Fatalf("wrong-backend evidence settled as success: %+v", detail) + } +} + +func TestRunnerIgnoresModelSuppliedVerificationEvidence(t *testing.T) { + ctx := context.Background() + source := pipeline.Template([]string{"worker"}, true) + source = strings.Replace(source, "```configs", "```verification\nbackend=branch-api\nrevision=abc123\ncapability=controlled-state\nfixture=reconnect-foreground\n```\n\n```configs", 1) + r, store, id := submitRunner(t, ctx, source) + defer store.Close() + r.verifyTarget = func(context.Context, stage.VerificationTarget) (stage.VerificationEvidence, error) { + return stage.VerificationEvidence{Backend: "branch-api", Revision: "abc123", Capability: "controlled-state", Fixture: "reconnect-foreground"}, nil + } + r.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) { + return stage.Result{Outcome: "success", Output: stage.Values{"verification": map[string]any{"backend": "old-api", "revision": "stale", "capability": "invented", "fixture": "other"}}}, nil + } + run := startRun(t, r, id, "model-supplied-evidence") + detail, err := store.GetRun(ctx, run.ID) + if err != nil { + t.Fatal(err) + } + if detail.Status != "succeeded" || len(detail.Steps) != 1 { + t.Fatalf("controlled verification did not settle: %+v", detail) + } +} + +func TestMainWorktreeRejectsNearMatchAndLinkedMain(t *testing.T) { + for _, test := range []struct { + name string + branch string + }{ + {name: "mainline is not main", branch: "mainline"}, + {name: "linked main is not canonical", branch: "main"}, + } { + t.Run(test.name, func(t *testing.T) { + root := t.TempDir() + primary := filepath.Join(root, "primary") + runGit(t, root, "init", "-b", "trunk", primary) + runGit(t, primary, "config", "user.email", "test@example.invalid") + runGit(t, primary, "config", "user.name", "test") + if err := os.WriteFile(filepath.Join(primary, "README"), []byte("base\n"), 0600); err != nil { + t.Fatal(err) + } + runGit(t, primary, "add", "README") + runGit(t, primary, "commit", "-m", "base") + item := filepath.Join(root, "item") + runGit(t, primary, "worktree", "add", "-b", "item", item) + candidate := filepath.Join(root, test.branch) + runGit(t, primary, "worktree", "add", "-b", test.branch, candidate) + + commonDir, err := gitOutput(item, "rev-parse", "--git-common-dir") + if err != nil { + t.Fatal(err) + } + commonDir, err = canonicalDirectory(resolveGitPath(item, commonDir)) + if err != nil { + t.Fatal(err) + } + if _, err := mainWorktree(item, commonDir); err == nil { + t.Fatalf("%s gained canonical main access", test.name) + } + }) + } +} + +func runGit(t *testing.T, directory string, args ...string) { + t.Helper() + cmd := exec.Command("git", append([]string{"-C", directory}, args...)...) + if output, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("git %v: %v: %s", args, err, output) + } +} + +func runGitOutput(t *testing.T, directory string, args ...string) string { + t.Helper() + output, err := exec.Command("git", append([]string{"-C", directory}, args...)...).Output() + if err != nil { + t.Fatalf("git %v: %v", args, err) + } + return strings.TrimSpace(string(output)) +} diff --git a/internal/server/resource_cli_test.go b/internal/server/resource_cli_test.go new file mode 100644 index 00000000..a377dc70 --- /dev/null +++ b/internal/server/resource_cli_test.go @@ -0,0 +1,154 @@ +package server + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/json" + "fmt" + "net/http/httptest" + "os" + "path/filepath" + "reflect" + "strings" + "testing" + "time" + + "git.a-wan.net/alan/pipelines/internal/cli" + "git.a-wan.net/alan/pipelines/internal/pipeline" + "git.a-wan.net/alan/pipelines/internal/protocol" + "git.a-wan.net/alan/pipelines/internal/stage" +) + +// This is a real CLI/WebSocket/scratch-SQLite exercise with an explicitly FAKE +// executor. It never calls inference, a phone, or a deployed server. +func TestResourceScratchCLIWaitingThenProgress(t *testing.T) { + t.Setenv("PIPELINES_HARNESS_CONFIG", filepath.Join(t.TempDir(), "harnesses.json")) + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + store, err := pipeline.Open(filepath.Join(t.TempDir(), "scratch.db")) + if err != nil { + t.Fatal(err) + } + defer store.Close() + source := pipeline.Template([]string{"work"}, true) + "```resources\n{\"exclusive\":\"fake-phone\"}\n```\n" + if _, err := store.Submit(ctx, "scratch-resource", source); err != nil { + t.Fatal(err) + } + handler := Handler(ctx, store).(*runHandler) + started, release := make(chan struct{}), make(chan struct{}) + handler.runner.execute = func(ctx context.Context, _ stage.Definition, input stage.Values) (stage.Result, error) { + if input["prompt"] == "A" { + close(started) + select { + case <-release: + case <-ctx.Done(): + return stage.Result{}, ctx.Err() + } + } + return stage.Result{Outcome: "success", Output: stage.Values{"execution": "FAKE executor; no device or inference"}}, nil + } + handler.runner.events.interval = time.Millisecond + events := make(chan protocol.RunEventFrame, 64) + sub := handler.runner.events.subscribe("cli-evidence", "", nil, func(m protocol.Message) error { + if m.RunEvent != nil { + events <- *m.RunEvent + } + return nil + }) + sub.ready() + defer sub.close() + srv := httptest.NewServer(handler) + defer srv.Close() + defer func() { cancel(); WaitRuns(handler) }() + endpoint := "ws" + strings.TrimPrefix(srv.URL, "http") + "/connect" + run := func(args ...string) string { + var out bytes.Buffer + app := cli.App{Out: &out, Err: &out, In: strings.NewReader("")} + if err := app.Run(ctx, append([]string{"--url", endpoint}, args...)); err != nil { + t.Fatalf("CLI %v: %v\n%s", args, err, out.String()) + } + t.Logf("pipelines %s\n%s", strings.Join(args, " "), out.String()) + return out.String() + } + parseRun := func(output string) int64 { + var m protocol.Message + if err := json.Unmarshal([]byte(output), &m); err != nil || m.RunDetail == nil { + t.Fatal("CLI run response", output, err) + } + return m.RunDetail.ID + } + run("validate", "testdata/android-resource-draft.md", "--json") + a := parseRun(run("run", "scratch-resource", "A", "--workspace", t.TempDir(), "--yes", "--json", "--request-key", "a")) + <-started + b := parseRun(run("run", "scratch-resource", "B", "--workspace", t.TempDir(), "--yes", "--json", "--request-key", "b")) + awaitResourceEvent(t, events, b, func(e protocol.RunEventFrame) bool { return e.ResourceWait != nil }) + var detail protocol.RunDetail + if err := json.Unmarshal([]byte(run("inspect", fmt.Sprint(b), "--json")), &detail); err != nil || detail.ResourceWait == nil || detail.Status != "queued" || len(detail.Steps) != 0 { + t.Fatal("CLI did not observe queue", err) + } + if output := run("runs"); !strings.Contains(output, "Waiting for resource fake-phone") { + t.Fatal("list omitted resource status", output) + } + var snapshot *protocol.RunEventFrame + handleRunEvents(protocol.Message{Type: protocol.Subscribe, RunIDs: []int64{b}}, handler.runner.events, handler.runner, "snapshot-evidence", func(m protocol.Message) error { + if m.RunEvent != nil { + snapshot = m.RunEvent + } + return nil + }) + handler.runner.events.unsubscribeClient("snapshot-evidence") + if snapshot == nil || snapshot.ResourceWait == nil || !snapshot.ResourceWaitSet { + t.Fatal("subscription snapshot omitted resource wait") + } + close(release) + run("wait", fmt.Sprint(a), fmt.Sprint(b), "--json") + detail = protocol.RunDetail{} + if err := json.Unmarshal([]byte(run("inspect", fmt.Sprint(b), "--json")), &detail); err != nil || detail.Status != "succeeded" || detail.ResourceWait != nil || len(detail.Steps) != 1 { + t.Fatal("automatic progress not observed", err) + } + t.Log("Observed queued resource wait followed by automatic progress; FAKE executor only, no phone/inference/deployment.") +} + +func TestResourceAndroidDraftPublishedV3Provenance(t *testing.T) { + baseline, err := os.ReadFile("testdata/android-v3-published.md") + if err != nil { + t.Fatal(err) + } + if hash := fmt.Sprintf("%x", sha256.Sum256(baseline)); hash != "4a3a4b8b947eb20c115c5470e6e378f117d5fa291a7b6c8ae966a3745bc826de" { + t.Fatal("baseline is not retrieved Android v3", hash) + } + draft, err := os.ReadFile("testdata/android-resource-draft.md") + if err != nil { + t.Fatal(err) + } + old, err := pipeline.Parse(string(baseline)) + if err != nil { + t.Fatal(err) + } + updated, err := pipeline.Parse(string(draft)) + if err != nil { + t.Fatal(err) + } + if len(old.Stages) != len(updated.Stages) { + t.Fatal("draft changed graph shape") + } + for i := range old.Stages { + a, b := old.Stages[i], updated.Stages[i] + if b.Configs["provider"] != "openai" || b.Configs["model"] != a.Configs["model"] { + t.Fatal("draft changed approved model") + } + if b.Key == "ui_reviewer" { + if b.Resources == nil || b.Resources.Exclusive != "phone-47241FDAS00DLK" { + t.Fatal("reviewer resource missing") + } + } else if b.Resources != nil { + t.Fatal("non-UI stage acquired phone") + } + a.Prompt, b.Prompt = "", "" + a.Resources, b.Resources = nil, nil + if !reflect.DeepEqual(a, b) { + t.Fatal("draft changed fields outside approved role split/resource", a.Key) + } + } +} diff --git a/internal/server/resources.go b/internal/server/resources.go new file mode 100644 index 00000000..ecb5b7a5 --- /dev/null +++ b/internal/server/resources.go @@ -0,0 +1,60 @@ +package server + +import ( + "context" + "errors" + "time" + + "git.a-wan.net/alan/pipelines/internal/pipeline" + "git.a-wan.net/alan/pipelines/internal/protocol" + "git.a-wan.net/alan/pipelines/internal/stage" +) + +func (r *runner) acquireResource(ctx context.Context, visit pipeline.ResourceVisit, workspace string, completed int) (pipeline.ResourceLease, error) { + lease, ready, wait, err := r.store.RequestResource(ctx, visit) + if err != nil { + return lease, err + } + if ready != nil { + r.emitRun(protocol.RunEventFrame{RunID: visit.RunID, Workspace: workspace, Status: protocol.RunStatusQueued, CurrentStage: visit.Stage, CompletedSteps: completed, ResourceWaitSet: true, ResourceWait: wait}, false) + select { + case lease = <-ready: + r.store.AcceptResource(lease) + case <-ctx.Done(): + cleanup, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + if err := r.store.CancelResource(cleanup, visit); err != nil { + return lease, errors.Join(ctx.Err(), err) + } + return lease, ctx.Err() + } + } + if ctx.Err() != nil { + err := r.releaseResource(lease, true) // Never launched, including a concurrent grant. + return pipeline.ResourceLease{}, errors.Join(ctx.Err(), err) + } + r.emitRun(protocol.RunEventFrame{RunID: visit.RunID, Workspace: workspace, ResourceWaitSet: true}, false) + return lease, nil +} + +func (r *runner) releaseResource(lease pipeline.ResourceLease, confirmed bool) error { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + if err := r.store.SetResourceCleanup(ctx, lease, confirmed); err != nil { + return err + } + if confirmed { + return r.store.ReleaseResource(ctx, lease) + } + return nil +} + +func (r *runner) resourceQuiescent(id int64, executionErr error, panicked bool) bool { + if panicked || errors.Is(executionErr, stage.ErrStageCleanup) { + return false + } + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + sessions, err := r.store.ActiveAgentSessionsForRun(ctx, id) + return err == nil && len(sessions) == 0 +} diff --git a/internal/server/resources_test.go b/internal/server/resources_test.go new file mode 100644 index 00000000..837560eb --- /dev/null +++ b/internal/server/resources_test.go @@ -0,0 +1,327 @@ +package server + +import ( + "context" + "database/sql" + "errors" + "fmt" + "path/filepath" + "strings" + "sync/atomic" + "testing" + "time" + + "git.a-wan.net/alan/pipelines/internal/pipeline" + "git.a-wan.net/alan/pipelines/internal/protocol" + "git.a-wan.net/alan/pipelines/internal/stage" +) + +func resourceRunner(t *testing.T) (*runner, <-chan protocol.RunEventFrame, func(string, string, string) *protocol.RunDetail) { + t.Helper() + ctx, cancel := context.WithCancel(context.Background()) + s, err := pipeline.Open(filepath.Join(t.TempDir(), "resources.db")) + if err != nil { + t.Fatal(err) + } + r, err := newRunner(ctx, s) + if err != nil { + t.Fatal(err) + } + r.events = newRunEventHub() + r.events.interval = time.Millisecond + events := make(chan protocol.RunEventFrame, 128) + sub := r.events.subscribe("resource-test", "", nil, func(m protocol.Message) error { + if m.RunEvent != nil { + events <- *m.RunEvent + } + return nil + }) + sub.ready() + t.Cleanup(func() { cancel(); r.wg.Wait(); sub.close(); s.Close() }) + count := 0 + return r, events, func(resource, timeout, prompt string) *protocol.RunDetail { + count++ + source := strings.Replace(pipeline.Template([]string{"work"}, true), "timeout=1h", "timeout="+timeout, 1) + if resource != "" { + source += "```resources\n{\"exclusive\":\"" + resource + "\"}\n```\n" + } + name := fmt.Sprint("resource-", count) + if _, err := s.Submit(ctx, name, source); err != nil { + t.Fatal(err) + } + items, _ := s.List(ctx) + var pid int64 + for _, p := range items { + if p.Name == name { + pid = p.ID + } + } + d, err := r.start(protocol.Message{PipelineID: pid, Version: 1, RequestKey: name, Workspace: t.TempDir(), Input: map[string]any{"prompt": prompt}}) + if err != nil { + t.Fatal(err) + } + return d + } +} + +func awaitResourceEvent(t *testing.T, events <-chan protocol.RunEventFrame, id int64, predicate func(protocol.RunEventFrame) bool) protocol.RunEventFrame { + t.Helper() + timer := time.NewTimer(3 * time.Second) + defer timer.Stop() + for { + select { + case event := <-events: + if event.RunID == id && predicate(event) { + return event + } + case <-timer.C: + t.Fatalf("resource event missing for run %d", id) + return protocol.RunEventFrame{} + } + } +} + +func TestResourcePauseBeyondTimeoutAndAutomaticResume(t *testing.T) { + r, events, start := resourceRunner(t) + aStarted, releaseA := make(chan struct{}), make(chan struct{}) + bStarted := make(chan struct{}, 1) + var bCalls atomic.Int32 + r.execute = func(ctx context.Context, _ stage.Definition, input stage.Values) (stage.Result, error) { + switch input["prompt"] { + case "A": + close(aStarted) + select { + case <-releaseA: + case <-ctx.Done(): + return stage.Result{}, ctx.Err() + } + case "B": + bCalls.Add(1) + deadline, ok := ctx.Deadline() + if !ok || time.Until(deadline) < 15*time.Millisecond { + return stage.Result{}, errors.New("execution timeout was consumed in queue") + } + bStarted <- struct{}{} + } + return stage.Result{Outcome: "success", Output: stage.Values{}}, nil + } + a := start("X", "2s", "A") + <-aStarted + b := start("X", "30ms", "B") + awaitResourceEvent(t, events, b.ID, func(e protocol.RunEventFrame) bool { return e.ResourceWait != nil }) + <-time.After(70 * time.Millisecond) // Deliberately longer than B's whole execution timeout. + d, err := r.store.GetRun(context.Background(), b.ID) + if err != nil || d.Status != protocol.RunStatusQueued || d.ResourceWait == nil || len(d.Steps) != 0 || bCalls.Load() != 0 { + t.Fatal("queued execution consumed accounting", d, err, bCalls.Load()) + } + if _, err := r.decide(protocol.Message{RunID: b.ID, Option: "success"}); err == nil { + t.Fatal("resource wait exposed a human decision") + } + c := start("Y", "1s", "C") + awaitResourceEvent(t, events, c.ID, func(e protocol.RunEventFrame) bool { return e.Status == "succeeded" }) + select { + case <-bStarted: + t.Fatal("B ran while A held X") + default: + } + owner, err := r.store.GetRun(context.Background(), a.ID) + if err != nil || owner.Status != "running" { + t.Fatal("A did not remain owner", err) + } + close(releaseA) + awaitResourceEvent(t, events, b.ID, func(e protocol.RunEventFrame) bool { return e.Status == "succeeded" }) + r.wg.Wait() + d, err = r.store.GetRun(context.Background(), b.ID) + if err != nil || bCalls.Load() != 1 || d.ResourceWait != nil || len(d.Steps) != 1 || d.Steps[0].Control.Visit != 1 || d.Steps[0].Sequence != 1 { + t.Fatal("resume did not execute exactly one fresh visit", d, err) + } +} + +func TestResourceCancellationWaitsForOwnerCleanup(t *testing.T) { + r, events, start := resourceRunner(t) + started, cleaning, cleanupDone := make(chan struct{}), make(chan struct{}), make(chan struct{}) + var waitingCalls atomic.Int32 + r.execute = func(ctx context.Context, _ stage.Definition, input stage.Values) (stage.Result, error) { + if input["prompt"] == "owner" { + close(started) + <-ctx.Done() + close(cleaning) + <-cleanupDone + return stage.Result{}, ctx.Err() + } + waitingCalls.Add(1) + return stage.Result{Outcome: "success", Output: stage.Values{}}, nil + } + a := start("X", "2s", "owner") + <-started + b := start("X", "30ms", "waiter") + awaitResourceEvent(t, events, b.ID, func(e protocol.RunEventFrame) bool { return e.ResourceWait != nil }) + if err := r.cancel(a.ID); err != nil { + t.Fatal(err) + } + <-cleaning + select { + case <-time.After(50 * time.Millisecond): + if waitingCalls.Load() != 0 { + t.Fatal("handoff before cleanup") + } + } + close(cleanupDone) + awaitResourceEvent(t, events, b.ID, func(e protocol.RunEventFrame) bool { return e.Status == "succeeded" }) + if waitingCalls.Load() != 1 { + t.Fatal("waiter did not resume exactly once") + } +} + +func TestResourceQuarantineAndQueuedCancellation(t *testing.T) { + for _, failure := range []string{"cleanup", "panic", "restoration"} { + t.Run(failure, func(t *testing.T) { + r, events, start := resourceRunner(t) + started, release := make(chan struct{}), make(chan struct{}) + var calls atomic.Int32 + r.execute = func(ctx context.Context, _ stage.Definition, input stage.Values) (stage.Result, error) { + if input["prompt"] == "owner" { + close(started) + select { + case <-release: + case <-ctx.Done(): + } + if failure == "panic" { + panic("unknown cleanup") + } + if failure == "restoration" { + return stage.Result{Outcome: "success", Output: stage.Values{protocol.ResourceCleanupConfirmedKey: false}}, nil + } + return stage.Result{}, stage.ErrStageCleanup + } + calls.Add(1) + return stage.Result{Outcome: "success", Output: stage.Values{}}, nil + } + a := start("X", "1s", "owner") + <-started + b := start("X", "30ms", "waiter") + awaitResourceEvent(t, events, b.ID, func(e protocol.RunEventFrame) bool { return e.ResourceWait != nil }) + close(release) + awaitResourceEvent(t, events, a.ID, func(e protocol.RunEventFrame) bool { return e.Status == "failed" }) + if err := r.cancel(b.ID); err != nil { + t.Fatal(err) + } + awaitResourceEvent(t, events, b.ID, func(e protocol.RunEventFrame) bool { + return e.Status == "canceled" && e.ResourceWaitSet && e.ResourceWait == nil + }) + r.wg.Wait() + if calls.Load() != 0 { + t.Fatal("quarantined resource handed off") + } + exists, err := r.store.HasResourceOwner(context.Background(), a.ID) + if err != nil || !exists { + t.Fatal("quarantine evidence lost", err) + } + d, err := r.store.GetRun(context.Background(), b.ID) + if err != nil || d.ResourceWait != nil || len(d.Steps) != 0 { + t.Fatal("canceled queue consumed step", err) + } + }) + } +} + +func TestResourceEventClearCoalescing(t *testing.T) { + frame := protocol.RunEventFrame{ResourceWaitSet: true, ResourceWait: &protocol.ResourceWait{Resource: "X"}} + mergeRunEventFrame(&frame, &protocol.RunEventFrame{ResourceWaitSet: true}) + if !frame.ResourceWaitSet || frame.ResourceWait != nil { + t.Fatal("coalescer retained stale resource wait") + } +} + +func TestResourceReleaseOnBeginAndPersistenceFailure(t *testing.T) { + for _, failure := range []string{"begin", "persist"} { + t.Run(failure, func(t *testing.T) { + r, events, start := resourceRunner(t) + db, err := sql.Open("sqlite", filepath.Join(r.store.Directory(), "resources.db")) + if err != nil { + t.Fatal(err) + } + defer db.Close() + trigger := `CREATE TRIGGER resource_fail BEFORE INSERT ON run_steps WHEN json_extract(NEW.input_json,'$.prompt')='fail' BEGIN SELECT RAISE(FAIL,'test BeginStep failure'); END` + if failure == "persist" { + trigger = `CREATE TRIGGER resource_fail BEFORE UPDATE OF output_json ON run_steps WHEN json_extract(OLD.input_json,'$.prompt')='fail' BEGIN SELECT RAISE(FAIL,'test persistence failure'); END` + } + if _, err := db.Exec(trigger); err != nil { + t.Fatal(err) + } + var calls atomic.Int32 + r.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) { + calls.Add(1) + return stage.Result{Outcome: "success", Output: stage.Values{}}, nil + } + a := start("X", "1s", "fail") + awaitResourceEvent(t, events, a.ID, func(e protocol.RunEventFrame) bool { return e.Status == "failed" }) + r.wg.Wait() + if exists, err := r.store.HasResourceOwner(context.Background(), a.ID); err != nil || exists { + t.Fatal("safe failure stranded resource", err) + } + if failure == "begin" && calls.Load() != 0 { + t.Fatal("BeginStep failure executed work") + } + b := start("X", "1s", "pass") + awaitResourceEvent(t, events, b.ID, func(e protocol.RunEventFrame) bool { return e.Status == "succeeded" }) + }) + } +} + +func TestResourceCleanupQueryFailureQuarantines(t *testing.T) { + r, events, start := resourceRunner(t) + r.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) { + if err := r.store.Close(); err != nil { + return stage.Result{}, err + } + return stage.Result{Outcome: "success", Output: stage.Values{}}, nil + } + a := start("X", "1s", "query-failure") + awaitResourceEvent(t, events, a.ID, func(e protocol.RunEventFrame) bool { return e.Status == "failed" }) + r.wg.Wait() + reopened, err := pipeline.Open(filepath.Join(r.store.Directory(), "resources.db")) + if err != nil { + t.Fatal(err) + } + defer reopened.Close() + if err := reopened.RecoverRuns(context.Background()); err != nil { + t.Fatal(err) + } + if exists, err := reopened.HasResourceOwner(context.Background(), a.ID); err != nil || !exists { + t.Fatal("query failure released uncertain owner", err) + } +} + +func TestResourceSkippedVisitDoesNotAcquire(t *testing.T) { + r, events, _ := resourceRunner(t) + ctx := context.Background() + source := strings.Replace(pipeline.Template([]string{"review"}, true), "success -> #success", "success -> #success\nskipped -> #success", 1) + "```resources\n{\"exclusive\":\"X\"}\n```\n" + if _, err := r.store.Submit(ctx, "skip", source); err != nil { + t.Fatal(err) + } + items, _ := r.store.List(ctx) + d, _ := r.store.Get(ctx, items[0].ID, 1) + owner, _, err := r.store.CreateRun(ctx, d, "held", t.TempDir(), false, nil, nil) + if err != nil { + t.Fatal(err) + } + lease, ch, _, err := r.store.RequestResource(ctx, pipeline.ResourceVisit{Resource: "X", RunID: owner, Stage: "review", Sequence: 1}) + if err != nil || ch != nil { + t.Fatal("owner fixture failed", err) + } + defer r.releaseResource(lease, true) + r.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) { + t.Error("skipped stage executed") + return stage.Result{}, nil + } + run, err := r.start(protocol.Message{PipelineID: d.ID, Version: 1, RequestKey: "skip", Workspace: t.TempDir(), Input: map[string]any{"prompt": "no UI"}}) + if err != nil { + t.Fatal(err) + } + awaitResourceEvent(t, events, run.ID, func(e protocol.RunEventFrame) bool { return e.Status == "succeeded" }) + detail, err := r.store.GetRun(ctx, run.ID) + if err != nil || len(detail.Steps) != 1 || detail.Steps[0].Outcome != "skipped" { + t.Fatal("skip not preserved", detail, err) + } +} diff --git a/internal/server/retry_test.go b/internal/server/retry_test.go index acde318d..152201ce 100644 --- a/internal/server/retry_test.go +++ b/internal/server/retry_test.go @@ -29,6 +29,7 @@ func TestRetryCreatesFreshOpenCodeSessionAndRetainsFailedSession(t *testing.T) { if [ "$1" = "--help" ]; then echo ' api'; exit 0; fi printf '%%s %%s %%s\n' "$PWD" "$2" "$3" >> %q case "$3" in + /api/session/active) echo '{"data":{}}' ;; /api/session) n=0; if [ -f %q ]; then n=$(cat %q); fi n=$((n+1)); echo "$n" > %q diff --git a/internal/server/runevents.go b/internal/server/runevents.go index e35d683d..87cac293 100644 --- a/internal/server/runevents.go +++ b/internal/server/runevents.go @@ -424,6 +424,9 @@ func (s *runSubscription) close() { } func mergeRunEventFrame(into, in *protocol.RunEventFrame) { + if in.ResourceWaitSet { + into.ResourceWaitSet, into.ResourceWait = true, in.ResourceWait + } if in.Workspace != "" { into.Workspace = in.Workspace } @@ -500,18 +503,20 @@ func handleRunEvents(request protocol.Message, events *runEventHub, runs *runner continue } frame := protocol.RunEventFrame{ - RunID: summary.ID, - Workspace: summary.Workspace, - PipelineID: summary.PipelineID, - Name: summary.Name, - Title: summary.Title, - Version: summary.Version, - Status: summary.Status, - Outcome: summary.Outcome, - CurrentStage: summary.CurrentStage, - CompletedSteps: summary.CompletedSteps, - CreatedAt: summary.CreatedAt, - FinishedAt: summary.FinishedAt, + ResourceWaitSet: true, + ResourceWait: summary.ResourceWait, + RunID: summary.ID, + Workspace: summary.Workspace, + PipelineID: summary.PipelineID, + Name: summary.Name, + Title: summary.Title, + Version: summary.Version, + Status: summary.Status, + Outcome: summary.Outcome, + CurrentStage: summary.CurrentStage, + CompletedSteps: summary.CompletedSteps, + CreatedAt: summary.CreatedAt, + FinishedAt: summary.FinishedAt, } if !sub.writeSnapshot(protocol.Message{Type: protocol.RunEvent, RunEvent: &frame}) { // The connection write failed; the subscription is closed and the diff --git a/internal/server/runner.go b/internal/server/runner.go index 0c532247..260397fe 100644 --- a/internal/server/runner.go +++ b/internal/server/runner.go @@ -58,8 +58,11 @@ type runner struct { // directory. It defaults to stage.ExportSession; a failure is best-effort and // never changes the stage's status, outcome, target or result validation. exportSession func(context.Context, stage.SessionExportRequest) (stage.SessionExportResult, error) - wg sync.WaitGroup - schedulerWG sync.WaitGroup + // verifyTarget is installed only by server wiring for a controlled fixture + // workflow. Leaving it nil fails closed for target-aware definitions. + verifyTarget stage.VerificationRunner + wg sync.WaitGroup + schedulerWG sync.WaitGroup // attentionMu guards attentionSent, the per-run set of run-attention events // already delivered — by a stage's own notify policy or by the run-level // default — so neither republishes the same genuine event. @@ -78,12 +81,19 @@ func newRunner(ctx context.Context, s *pipeline.Store) (*runner, error) { if owner.Backend == "pi" { err = stage.InterruptPi(stop, owner.Reference) } else { - err = (stage.OpenCodeClient{Backend: owner.Backend, Directory: owner.Directory, Reference: owner.Reference}).Interrupt(stop, owner.ID) + err = (stage.OpenCodeClient{Backend: owner.Backend, Directory: owner.Directory, Reference: owner.Reference}).InterruptAndConfirm(stop, owner.ID) } cancel() if err != nil { + if resource, checkErr := s.HasResourceOwner(ctx, owner.RunID); checkErr == nil && resource { + log.Printf("resource owner run=%d remains quarantined: cleanup not confirmed", owner.RunID) + continue + } return nil, fmt.Errorf("cannot interrupt previously active agent %s: %w", owner.ID, err) } + if err := s.ConfirmRecoveredSession(ctx, owner); err != nil { + return nil, err + } } if err := s.RecoverRuns(ctx); err != nil { return nil, err @@ -213,6 +223,9 @@ func (r *runner) start(request protocol.Message, retryOf ...int64) (*protocol.Ru // workspace) must not block new work. for _, owner := range owners { if _, ok := r.active[owner.RunID]; !ok { + if resource, checkErr := r.store.HasResourceOwner(r.ctx, owner.RunID); checkErr == nil && resource && owner.Directory != workspace { + continue + } r.mu.Unlock() return nil, fmt.Errorf("an agent may still be active; restart recovery must interrupt it before new work") } @@ -580,7 +593,8 @@ type runState struct { capturedSeen map[string]bool // completed counts the run's finished steps, so a delta can report the // running total without re-reading the store. - completed int + completed int + artifactDirs []string } func newRunState(key string, input stage.Values) *runState { @@ -606,10 +620,67 @@ func rebuildRunState(detail *protocol.RunDetail, decided protocol.RunStep) *runS // Seed the UI-surface accumulator from the persisted evidence so a // resumed run reaches the same skip decision with no extra query. state.recordChanges(step.FileChanges, step.Artifacts) + // Only completed, recorded visits are eligible for a later stage's + // review capability. The path is reconstructed by the runner below. + if step.Status == "completed" { + state.artifactDirs = append(state.artifactDirs, filepath.Join("", strconv.Itoa(step.Sequence))) + } } return state } +// artifactAccess derives exact visit directories; it never consumes a path from +// a definition or model result. An unreadable or symlinked root fails closed. +func (r *runner) artifactAccess(runID int64, sequence int, prior []string, current string) stage.ArtifactAccess { + root, err := filepath.EvalSymlinks(r.artifactsRoot) + if err != nil { + return stage.ArtifactAccess{} + } + runRoot := filepath.Join(root, strconv.FormatInt(runID, 10)) + visitDirectory := func(visit int) (string, bool) { + if visit <= 0 { + return "", false + } + directory := filepath.Join(runRoot, strconv.Itoa(visit)) + resolved, err := filepath.EvalSymlinks(directory) + if err != nil || resolved != directory { + return "", false + } + info, err := os.Stat(resolved) + if err != nil || !info.IsDir() { + return "", false + } + return resolved, true + } + access := stage.ArtifactAccess{} + if directory, ok := visitDirectory(sequence); ok { + resolved, err := filepath.EvalSymlinks(current) + if err != nil || resolved != directory { + return access + } + access.Write = directory + access.Read = append(access.Read, directory) + } + for _, priorSequence := range prior { + visit, err := strconv.Atoi(priorSequence) + if err != nil || visit <= 0 || visit >= sequence { + continue + } + if directory, ok := visitDirectory(visit); ok { + access.Read = append(access.Read, directory) + } + } + return access +} + +func (r *runner) agentClient(runID int64, sequence int, prior []string, workspace, artifactsDir string, operations *stage.OperationLog, writePaths, tools []string) *stage.AgentClient { + return &stage.AgentClient{OpenCodeClient: stage.OpenCodeClient{Directory: workspace, Operations: operations, ArtifactsDir: artifactsDir}, WritePaths: writePaths, Tools: tools, ArtifactAccess: r.artifactAccess(runID, sequence, prior, artifactsDir)} +} + +func (r *runner) piClient(runID int64, sequence int, prior []string, executable, workspace, artifactsDir string, writePaths, tools []string) *stage.PiClient { + return &stage.PiClient{Executable: executable, Directory: workspace, Tools: tools, WritePaths: writePaths, ArtifactsDir: artifactsDir, ArtifactAccess: r.artifactAccess(runID, sequence, prior, artifactsDir)} +} + // serverChangedFilesField is the reserved run-input memory field under which the // runner publishes the server-captured changed paths. A stage selects it with // `"server_changed_files":"input.server_changed_files"`, alongside another @@ -717,13 +788,14 @@ func (r *runner) finishRunState(id int64, name, workspace, status, outcome, diag // Push the terminal delta before the attention notification, so a slow ntfy // never delays the live update the app folds. r.emitRun(protocol.RunEventFrame{ - RunID: id, - Workspace: workspace, - Status: status, - Outcome: outcome, - CurrentStage: currentStage, - CompletedSteps: completed, - FinishedAt: time.Now().UTC().Format(time.RFC3339), + RunID: id, + Workspace: workspace, + Status: status, + ResourceWaitSet: true, + Outcome: outcome, + CurrentStage: currentStage, + CompletedSteps: completed, + FinishedAt: time.Now().UTC().Format(time.RFC3339), }, true) r.notifyRun(id, name, status, sequence, currentStage, diagnostic) // The run is terminal: nothing more can notify for it. @@ -791,6 +863,21 @@ func (r *runner) finishStage(id int64, name, workspace string, selected protocol } func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, name, workspace string, request protocol.Message, state *runState) { + var held *pipeline.ResourceLease + resourceQuiet := true + release := func() error { + if held != nil { + err := r.releaseResource(*held, resourceQuiet) + held = nil + return err + } + return nil + } + defer func() { + if err := release(); err != nil { + log.Printf("resource cleanup run=%d: %v", id, err) + } + }() finish := func(status, outcome, diagnostic string) { r.finishRunState(id, name, workspace, status, outcome, diagnostic, state.completed, state.nextSequence-1, state.key, state.input) } @@ -819,7 +906,6 @@ func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, na effectiveModels = resolved } for sequence := state.nextSequence; sequence <= 30; sequence++ { - state.nextSequence = sequence + 1 if ctx.Err() != nil { finish("canceled", "", ctx.Err().Error()) return @@ -851,24 +937,42 @@ func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, na } } isHuman := stage.EffectiveType(def.Type) == stage.HumanType - state.visits[key]++ maxVisits := 5 if selected.Limits != nil { maxVisits = selected.Limits.MaxVisits } - // A human stage pauses for a person; neither the global visit budget nor - // the per-stage max_visits may kill it. The shared 30-step run budget - // still bounds every run. + // Validate the next visit without consuming it. An already-exhausted + // stage must not queue for a resource it can never execute. if !isHuman { - if state.visits[key] > 5 { + if state.visits[key]+1 > 5 { finish("failed", "error", "stage visit budget exceeded") return } - if state.visits[key] > maxVisits { + if state.visits[key]+1 > maxVisits { finish("failed", "error", fmt.Sprintf("stage %s visit limit reached (%d)", key, maxVisits)) return } } + _, declaresSkip := def.Edges["skipped"] + skip := !isHuman && declaresSkip && !state.uiSurface + if selected.Resources != nil && !skip { + if isHuman || !pipeline.ValidResourceKey(selected.Resources.Exclusive) { + finish("failed", "error", "invalid persisted resource declaration") + return + } + lease, err := r.acquireResource(ctx, pipeline.ResourceVisit{Resource: selected.Resources.Exclusive, RunID: id, Stage: key, Sequence: sequence}, workspace, state.completed) + if err != nil { + if ctx.Err() != nil { + finish("canceled", "", err.Error()) + } else { + finish("failed", "error", err.Error()) + } + return + } + held, resourceQuiet = &lease, true + } + state.nextSequence = sequence + 1 + state.visits[key]++ // Publish the run's server-captured changed paths into the input memory // value before resolving bindings, so a stage selecting // input.server_changed_files always resolves (an empty list when nothing @@ -890,12 +994,13 @@ func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, na } // queued -> running: report the new current stage and the step start. r.emitRun(protocol.RunEventFrame{ - RunID: id, - Workspace: workspace, - Status: protocol.RunStatusRunning, - CurrentStage: key, - CompletedSteps: state.completed, - Step: &protocol.RunStepDelta{Sequence: sequence, Stage: key, Status: protocol.StepStatusRunning}, + RunID: id, + Workspace: workspace, + Status: protocol.RunStatusRunning, + CurrentStage: key, + ResourceWaitSet: true, + CompletedSteps: state.completed, + Step: &protocol.RunStepDelta{Sequence: sequence, Stage: key, Status: protocol.StepStatusRunning}, }, false) if isHuman { // A human step is O(1): one BeginStep plus one MarkWaiting, no @@ -929,7 +1034,7 @@ func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, na // the model call and its capture work, record the step as completed and // route along the declared edge. Only a declared route is honoured, and no // other stage or outcome routing changes. - if _, declaresSkip := def.Edges["skipped"]; declaresSkip && !state.uiSurface { + if skip { reason := skipReasonLine(state) step := protocol.RunStep{ Sequence: sequence, @@ -977,7 +1082,22 @@ func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, na } } operations := &stage.OperationLog{} - client := &stage.AgentClient{OpenCodeClient: stage.OpenCodeClient{Directory: workspace, Operations: operations, ArtifactsDir: artifactsDir}, WritePaths: request.WritePaths, Tools: tools} + var landing stage.LandingAccess + delivery, deliveryErr := stage.DeliveryEnabled(def) + if deliveryErr != nil { + finish("failed", "error", deliveryErr.Error()) + return + } + if delivery { + access, accessErr := landingAccess(workspace) + if accessErr != nil { + finish("failed", "error", "delivery access denied: "+accessErr.Error()) + return + } + landing = access + } + client := r.agentClient(id, sequence, state.artifactDirs, workspace, artifactsDir, operations, request.WritePaths, tools) + client.LandingAccess = landing client.OnSession = func(session string, active bool) error { c, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() @@ -1010,6 +1130,9 @@ func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, na snapshotCancel() var stageSession string panicked := false + if held != nil { + resourceQuiet = false + } func() { defer func() { if recover() != nil { @@ -1017,6 +1140,10 @@ func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, na err = fmt.Errorf("stage execution panicked") } }() + if ctx.Err() != nil { + err = ctx.Err() + return + } if r.execute != nil { timeout, _ := stage.StageTimeout(def) executeCtx, cancel := context.WithTimeout(stageCtx, timeout) @@ -1033,7 +1160,7 @@ func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, na err = fmt.Errorf("Pi is not configured on execution server") return } - piClient = &stage.PiClient{Executable: profile.Executable, Directory: workspace, Tools: tools, WritePaths: request.WritePaths, ArtifactsDir: artifactsDir} + piClient = r.piClient(id, sequence, state.artifactDirs, profile.Executable, workspace, artifactsDir, request.WritePaths, tools) piClient.OnOwnership = func(session, reference string, active bool) error { c, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() @@ -1061,6 +1188,12 @@ func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, na result, err = stage.DefaultRegistry().Run(ctx, def, input) } }() + if held != nil { + resourceQuiet = r.resourceQuiescent(id, err, panicked) + if confirmed, reported := result.Output[protocol.ResourceCleanupConfirmedKey]; reported && confirmed != true { + resourceQuiet = false + } + } // Capture after cleanup, even when the stage's context was canceled. snapshotCtx, snapshotCancel = context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second) after := filediff.Capture(snapshotCtx, workspace, before) @@ -1107,6 +1240,45 @@ func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, na step.Output["salvage"] = result.Output } } + if err == nil && result.Outcome == "success" { + target, targetErr := stage.VerificationTargetFor(def) + if targetErr != nil { + err = targetErr + } else if target != nil { + if r.verifyTarget == nil { + err = fmt.Errorf("verification target %q requires a server-controlled fixture runner", target.Fixture) + } else { + evidence, verificationErr := r.verifyTarget(stageCtx, *target) + if verificationErr != nil { + err = verificationErr + } else { + err = stage.VerifyTargetEvidence(*target, evidence) + } + } + } + if err != nil { + step.Status = "failed" + step.Diagnostic = err.Error() + step.Outcome = "error" + step.Output = stage.Values{"error": err.Error(), "stage": key, "input": input} + } + } + if err == nil && delivery && result.Outcome == "success" { + checks, checksErr := stage.DeliveryChecks(def) + if checksErr != nil { + err = checksErr + } else { + gateCtx, gateCancel := context.WithTimeout(stageCtx, modelConfig.Timeout) + err = verifyDelivery(gateCtx, workspace, landing, checks) + gateCancel() + } + if err != nil { + step.Status = "failed" + step.Diagnostic = err.Error() + step.Outcome = "error" + step.Output = stage.Values{"error": err.Error(), "stage": key, "input": input} + } + } // A failed or timed-out step carries the session pointer even when the // bounded transcript would otherwise be only its stage_limits envelope. if err != nil && stageSession != "" { @@ -1148,6 +1320,9 @@ func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, na state.recordChanges(step.FileChanges, step.Artifacts) if step.Status == "completed" { state.completed++ + if artifactsDir != "" { + state.artifactDirs = append(state.artifactDirs, strconv.Itoa(sequence)) + } } // Step finished: report its outcome, routing target, diagnostic and the // artifacts recorded for the visit, plus the run's running step count. @@ -1175,6 +1350,14 @@ func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, na return } } + if held != nil && !resourceQuiet { + finish("failed", "error", "resource cleanup/restoration not confirmed; ownership quarantined pending inspection") + return + } + if err := release(); err != nil { + finish("failed", "error", "resource release failed; retained ownership needs reconciliation: "+err.Error()) + return + } if ctx.Err() != nil { finish("canceled", "", ctx.Err().Error()) return diff --git a/internal/server/stage_deadline_test.go b/internal/server/stage_deadline_test.go new file mode 100644 index 00000000..611e0afb --- /dev/null +++ b/internal/server/stage_deadline_test.go @@ -0,0 +1,138 @@ +package server + +import ( + "bytes" + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" + + "git.a-wan.net/alan/pipelines/internal/cli" + "git.a-wan.net/alan/pipelines/internal/pipeline" + "git.a-wan.net/alan/pipelines/internal/protocol" +) + +// Exercise the real V2 CLI/poll path, runner, SQLite retention and websocket CLI +// against an isolated loopback server/database, never the configured deployment. +func TestStageDeadlineDiagnosticThroughCLIAndWebsocket(t *testing.T) { + ntfy := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write([]byte(`{"id":"scratch-receipt","event":"message"}`)) + })) + defer ntfy.Close() + t.Setenv("PIPELINES_NTFY_URL", ntfy.URL+"/scratch") + t.Setenv("PIPELINES_NTFY_TOKEN", "") + t.Setenv("PIPELINES_NTFY_TOKEN_FILE", "") + for _, mode := range []string{"active", "http", "connection"} { + t.Run(mode, func(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + dir := t.TempDir() + executable := filepath.Join(dir, "opencode") + script := `#!/bin/sh +if [ "$1" = "--help" ]; then echo ' api'; exit 0; fi +printf '%s %s\n' "$2" "$3" >> "@STATE@/calls" +case "$3" in + /api/session) echo '{"data":{"id":"ses_deadline"}}' ;; + */prompt) echo 'edited once' >> "@STATE@/edited"; echo '{}' ;; + /api/session/active) + if [ -f "@STATE@/interrupted" ]; then echo '{"data":{}}'; exit 0; fi + case "@MODE@" in + active) echo '{"data":{"ses_deadline":{"type":"running"}}}' ;; + http) echo 'HTTP 503 token=private-token' >&2; exit 1 ;; + connection) echo 'connection refused token=private-token' >&2; exit 1 ;; + esac ;; + */interrupt) touch "@STATE@/interrupted"; echo '{}' ;; + */message*) echo '{"data":[{"content":[{"type":"text","text":"still working"}]}]}' ;; + /api/session/ses_deadline) echo '{"data":{"id":"ses_deadline"}}' ;; + *) echo '{}' ;; +esac +` + script = strings.ReplaceAll(strings.ReplaceAll(script, "@STATE@", dir), "@MODE@", mode) + if err := os.WriteFile(executable, []byte(script), 0700); err != nil { + t.Fatal(err) + } + t.Setenv("PIPELINES_OPENCODE", executable) + store, err := pipeline.Open(filepath.Join(dir, "runs.db")) + if err != nil { + t.Fatal(err) + } + defer store.Close() + source := pipeline.Template([]string{"worker"}, true) + source = strings.Replace(source, "```tools\n```", "```tools\nedit\n```", 1) + source = strings.Replace(source, "timeout=1h", "timeout=500ms", 1) + if _, err := store.Submit(ctx, "deadline", source); err != nil { + t.Fatal(err) + } + handler := Handler(ctx, store) + defer func() { cancel(); WaitRuns(handler) }() + srv := httptest.NewServer(handler) + defer srv.Close() + url := "ws" + strings.TrimPrefix(srv.URL, "http") + "/connect" + var out, stderr bytes.Buffer + app := cli.App{Out: &out, Err: &stderr, In: strings.NewReader("")} + if err := app.Run(ctx, []string{"--url", url, "run", "deadline", "task", "--yes", "--json", "--wait", "--timeout", "10s", "--workspace", dir}); err == nil || !strings.Contains(err.Error(), "run ended failed") { + t.Fatalf("run not failed: %v, %s, %s", err, out.String(), stderr.String()) + } + var response protocol.Message + if err := json.Unmarshal(out.Bytes(), &response); err != nil { + t.Fatal(err, out.String()) + } + detail := response.RunDetail + if detail == nil || detail.Status != "failed" || len(detail.Steps) != 1 { + t.Fatal("missing failed run/step", out.String()) + } + step := detail.Steps[0] + if step.SessionID != "ses_deadline" || step.AgentActive || step.Output["error"] != step.Diagnostic || step.Status != "failed" || detail.Diagnostic != step.Diagnostic { + t.Fatalf("incorrect retained failure: %+v", detail) + } + wants := []string{"HARNESS TRANSPORT", "get /api/session/active", "CLI exit code 1"} + if mode == "active" { + wants = []string{"STAGE DEADLINE", "timeout 500ms", "waiting for session idle", "ses_deadline", "stage had not finished"} + if strings.Contains(detail.Diagnostic, "HARNESS TRANSPORT") || strings.Contains(detail.Diagnostic, "response lost") { + t.Fatal("active stage overrun reported as transport", detail.Diagnostic) + } + if strings.Contains(detail.Diagnostic, "\n") { + t.Fatal("duplicate context error appended to bounded stage diagnostic", detail.Diagnostic) + } + for _, want := range []string{"opencode_failures", "wait for session idle", "timeout 500ms", "retained_session", "still working"} { + if !strings.Contains(step.Transcript, want) { + t.Fatalf("missing retained evidence %q: %s", want, step.Transcript) + } + } + } else { + if !strings.Contains(step.Transcript, "[REDACTED]") { + t.Fatal("request failure evidence missing", step.Transcript) + } + if mode == "http" { + wants = append(wants, "HTTP 503") + } else if !strings.Contains(step.Transcript, "connection refused") { + t.Fatal("underlying connection evidence missing", step.Transcript) + } + } + for _, want := range wants { + if !strings.Contains(detail.Diagnostic, want) { + t.Fatalf("missing diagnostic %q: %s", want, detail.Diagnostic) + } + } + if strings.Contains(detail.Diagnostic+step.Transcript, "private-token") { + t.Fatal("credential leaked in retained diagnostic/evidence") + } + edited, err := os.ReadFile(filepath.Join(dir, "edited")) + if err != nil || string(edited) != "edited once\n" { + t.Fatalf("admitted tool work replayed or lost: %q, %v", edited, err) + } + if _, err := os.Stat(filepath.Join(dir, "interrupted")); err != nil { + t.Fatal("cleanup did not interrupt", err) + } + stored, err := store.GetRun(ctx, detail.ID) + if err != nil || stored.Diagnostic != detail.Diagnostic || stored.Steps[0].Transcript != step.Transcript { + t.Fatal("diagnostic/evidence not persisted in scratch database", err) + } + t.Logf("scratch run %s: %s", mode, stored.Diagnostic) + }) + } +} diff --git a/internal/server/testdata/android-resource-draft.md b/internal/server/testdata/android-resource-draft.md new file mode 100644 index 00000000..e822ce78 --- /dev/null +++ b/internal/server/testdata/android-resource-draft.md @@ -0,0 +1,312 @@ +# stage implementer + +```limits +{"max_visits":5,"on":["blocked_retryable"]} +``` + + +```memory +{"task": "input.prompt", "code_findings": "reviewer.findings?", "ui_findings": "ui_reviewer.findings?"} +``` + +```output +{ task, acceptance_checks, changed_files, implementation_summary, new_helpers } +``` + +```prompt +Implement input.prompt in the run directory using the repository's actual languages and +file types; create files as needed. + +Read android/AGENTS.md first. It is the map of this project - how to build, test, run and capture here, where +the pieces live, and the conventions that apply. Do not re-derive what it tells you and do not go +looking for what it already names; if it is wrong or out of date, say so in your output. + +Read the surrounding code before editing so you extend the existing helpers and patterns rather than +duplicating them. Keep each change as small as the task allows and follow the repository's own +conventions and error handling. + +This is the Android app. Build and test it with the project's own Gradle invocation as the map +gives it, and treat the map's geometry tokens as the only source of dimensions - a UI change that +adds a raw value fails the repository's own check. Perform code/build/unit tests only; +no adb, install, capture, device-setting changes or restoring main's build. The UI reviewer +alone verifies rendered surfaces under its declared runner resource lease. + +Record the acceptance checks in acceptance_checks, the actual changes in changed_files, and any new +shared helper in new_helpers. Verify with the checks this repository uses - as the map gives them - +and report the exact commands and observed output. Address any concrete findings the runner supplies +and note the actual fixes. Return blocked if the required scope or tools are unavailable. Do not +commit or push; the committer stage lands the work. +``` + +```tools +read +glob +grep +edit +shell +``` + +```configs +provider=openai +model=gpt-5.6-terra +retries=1 +timeout=30m +``` + + +```edges +success -> reviewer +blocked_retryable -> implementer +blocked -> #failure +error -> #failure +``` + + +# stage reviewer + +```limits +{"max_visits": 3, "on": ["rejected"]} +``` + +```memory +{"task": "input.prompt", "acceptance_checks": "implementer.acceptance_checks", "changed_files": "implementer.changed_files", "implementation": "implementer.implementation_summary?"} +``` + + +```output +{ summary, changed_files, verification_evidence, duplication_review, performance_review, findings } +``` + +```prompt +Verify with builds, unit tests and source review as android/AGENTS.md gives them. +Do not use adb, install, capture, change device settings or restore main's build. +Device verification belongs exclusively to the UI reviewer under its runner resource lease. +Enforce the map's geometry-token rule: a raw dimension outside the token file is a must-fix finding. + +Review the actual changes against the task in input.prompt and the acceptance +checks the implementer recorded. Run the repository's relevant verification checks +with the stage tools (for example build, lint, tests or type checks) and report the +exact commands and observed results in verification_evidence. + +Independently assess code quality: +- Duplication: search the run directory for repeated or near-identical functions, + blocks and copy-pasted logic introduced by the change; identify code that should be + shared and record concrete locations in duplication_review. +- Performance and resources: flag redundant computation, unnecessary dependencies, + and repeated I/O or database/filesystem work that wastes time, CPU or memory; + record it in performance_review. +- Workarounds: flag hacks, suppressed errors and shortcuts that trade reliability for + speed in findings. + +Keep verification proportionate to the change: exercise the files and screens the diff +actually touches, capture at most one screenshot per changed surface plus one regression +screen, and do not repeat sweeps or measurements that an earlier visit already recorded. + +Return rejected ONLY when at least one must-fix finding exists: a defect that blocks +acceptance and that you can locate concretely (file and line), with evidence and a +remedy. Do not return rejected for advisory-only, cosmetic or nit items. When there +is no must-fix finding but you have non-blocking notes (duplication, performance or +workaround observations), record them in findings and return advisory, which +continues the run without another implementer visit. Return success only when the +observed checks pass and you have nothing further to record. Do not edit files or +claim unexecuted checks. +``` + +```tools +read +glob +grep +shell +``` + +```configs +provider=openai +model=gpt-5.6-luna +retries=1 +timeout=45m +``` + + +```edges +success -> ui_reviewer +advisory -> ui_reviewer +rejected -> implementer +error -> #failure +``` + + +# stage ui_reviewer + +```resources +{"exclusive":"phone-47241FDAS00DLK"} +``` + +```limits +{"max_visits": 3, "on": ["rejected"]} +``` + +```memory +{"task": "input.prompt", "acceptance_checks": "implementer.acceptance_checks", "changed_files": "implementer.changed_files", "implementation": "implementer.implementation_summary?"} +``` + + +```output +{ summary, changed_files, ui_applicable, screenshots, ui_review, platform_conventions, findings } +``` + +```prompt +Decide whether this change touches a user-facing UI. If it does not (a backend, +library, CLI, config or data-only change with no rendered surface), set +ui_applicable=false, explain briefly in summary, and return skipped. Do not invent +screenshots or visual work. + +When the change does affect a user-facing UI, verify it visually: +Read android/AGENTS.md first. You alone perform phone work while the runner holds +this stage's declared resource lease. Use controlled fixtures for changed states, +including human-decision sheets, on a scratch server/database and throwaway session. +Never mutate an operator session or restart deployed services. Record original +settings and connection/workspace; use the configured authoritative endpoint, not a +hardcoded scratch default. Keep every shell/ADB operation foreground and awaited. +Restore original settings and main's build before successful return, under the same +lease. Report capture/restoration failures explicitly; do not approve incomplete +verification or advertise readiness when cleanup/restoration is uncertain. +Include resource_cleanup_confirmed in output: true only after confirmed restoration; +false on any uncertainty (a release veto that quarantines ownership even if tools are idle). +- Build and launch the application the way the repository actually does it, then + capture screenshots of every relevant screen, state and viewport (default, empty, + loading, error, long content, small and large sizes) with the target platform's + screenshot tooling through shell. Prefer a headless or scripted capture when the + project supports one. Save the images under the run directory and list the exact + capture commands and image paths in screenshots. +- Read the captured images and review them as a UI/UX designer: layout, spacing, + alignment, typographic hierarchy, color and contrast, iconography, interaction + and focus states, loading/empty/error states, accessibility, and consistency. +- Judge the UI against established industry patterns and, above all, the native + conventions of the platform the app targets (operating system, toolkit or web + standards). Record the platform you reviewed against in platform_conventions, + and any deviations and concrete fixes in findings. Infer the target platform from + the task and the repository's actual UI toolkit when it is unstated. +- Approve only when the screenshots show the task and the acceptance checks are met + and the UI is correct, consistent and native to the target platform; otherwise + return rejected with specific, actionable findings the implementer can fix. + +Keep the capture pass proportionate: capture each changed state once (for example one +collapsed and one expanded section, one run state, one stuck header) instead of every +screen at multiple viewports or pan directions, and reuse what this run already captured +rather than re-doing it. + +If the change is UI-facing but screenshots genuinely cannot be captured +(for example no runnable UI or no display), return blocked rather than approving. +Do not edit files, and do not approve a UI you did not actually see. + +Return rejected only for a must-fix visual defect with a concrete location, evidence +and remedy; record non-blocking polish, duplication or convention notes in findings +and return advisory, which continues the run without another implementer visit. +``` + +```tools +read +glob +grep +shell +``` + +```configs +provider=openai +model=gpt-5.6-luna +retries=1 +timeout=45m +``` + + +```edges +skipped -> committer +approved -> committer +advisory -> committer +rejected -> implementer +blocked -> #failure +error -> #failure +``` + +# stage committer + +```memory +{"task": "input.prompt", "changed_files": "implementer.changed_files", "review": "reviewer.summary?", "ui_review": "ui_reviewer.summary?"} +``` + +```output +{ commit, files, conflicts_resolved, summary } +``` + +```prompt +Commit the work this run produced, and resolve any conflict that is present. This stage +runs after the reviews have approved and is the only stage allowed to commit. + +Before committing, check that the change is reflected where it belongs: the packaged +templates, the packaged +authoring skill references. Update anything the change makes stale, and state in the +commit body what you checked, saying explicitly when none of them needs updating. + +Stage exactly the files this run changed - the paths in changed_files that still exist - +using the repository's own version control commands, and commit them as one commit whose +subject says what the change does and whose body summarises it. Do not stage unrelated +files, generated or ignored files, or anything another change owns. Do not invent work +that was not done; describe what is actually in the diff. + +If the workspace is in a conflicted state - unmerged paths, or a merge, rebase or +cherry-pick left unfinished - resolve each conflict: keep both intents where they are +compatible, and choose the result that satisfies the task where they are not, then +complete the operation so the tree is clean and the commit is finished. Never commit +conflict markers, and never discard the other side's work silently; record what you +resolved. + +Do not push, tag, revert, or rewrite unrelated history. If there is nothing to commit, +report that instead of creating an empty commit. Report the commit identifier, the files +committed, whatever you resolved, and anything you deliberately left uncommitted. + + +Finally, bring this branch up to date with main, in this same stage. After committing the work +above, merge main into this branch here, resolve any conflict here - keeping both intents where they +are compatible, and choosing the result that satisfies this task where they are not - run the +repository's checks again after resolving, and commit the completed merge with a message naming what +was combined. The point is that this branch can later be merged back without conflict: leave the +tree clean and the merge finished. Never leave conflict markers, never discard the other side's work +silently. If the merge genuinely cannot be completed, say so with the path and the reason instead of +leaving the merge half-finished. + +Then land it: this branch is the work and main is where it belongs, so merge this branch into main +as the last act of this stage. Find the repository's main working tree with `git worktree list` (the +entry checked out on main, not this one) and merge this branch into it there with `--no-ff`, with a +message saying what the change does. Two lanes can land at the same moment, so take a lock first - +`mkdir` a lock directory beside main's `git` directory in a bounded retry loop, and remove it when +done - and retry briefly if another lane holds it. Then verify with +`git merge-base --is-ancestor main` and report the commit main now has. If main cannot +be updated, say exactly why and leave this branch committed and clean rather than leaving main +half-merged. There is no separate merge pipeline and nothing to hand this to: if it cannot be landed +here, it is reported here. + + +Do not edit README or other shared documentation files: an item records what it changed and which documentation is now stale in its own commit body, and the shared documents are folded by a single writer. + +``` + +```tools +read +glob +grep +edit +shell +``` + +```configs +provider=openai +model=gpt-5.6-terra +retries=1 +timeout=30m +``` + +```edges +success -> #success +blocked -> #failure +error -> #failure +``` diff --git a/internal/server/testdata/android-v2-published.md b/internal/server/testdata/android-v2-published.md new file mode 100644 index 00000000..43bf14a9 --- /dev/null +++ b/internal/server/testdata/android-v2-published.md @@ -0,0 +1,296 @@ +# stage implementer + +```limits +{"max_visits":5,"on":["blocked_retryable"]} +``` + + +```memory +{"task": "input.prompt", "code_findings": "reviewer.findings?", "ui_findings": "ui_reviewer.findings?"} +``` + +```output +{ task, acceptance_checks, changed_files, implementation_summary, new_helpers } +``` + +```prompt +Implement input.prompt in the run directory using the repository's actual languages and +file types; create files as needed. + +Read android/AGENTS.md first. It is the map of this project - how to build, test, run and capture here, where +the pieces live, and the conventions that apply. Do not re-derive what it tells you and do not go +looking for what it already names; if it is wrong or out of date, say so in your output. + +Read the surrounding code before editing so you extend the existing helpers and patterns rather than +duplicating them. Keep each change as small as the task allows and follow the repository's own +conventions and error handling. + +This is the Android app. Build and test it with the project's own Gradle invocation as the map +gives it, and treat the map's geometry tokens as the only source of dimensions - a UI change that +adds a raw value fails the repository's own check. A change with a rendered surface is verified on +the device the map names (never an emulator), capturing what the map says to capture. + +Record the acceptance checks in acceptance_checks, the actual changes in changed_files, and any new +shared helper in new_helpers. Verify with the checks this repository uses - as the map gives them - +and report the exact commands and observed output. Address any concrete findings the runner supplies +and note the actual fixes. Return blocked if the required scope or tools are unavailable. Do not +commit or push; the committer stage lands the work. +``` + +```tools +read +glob +grep +edit +shell +``` + +```configs +provider=openai +model=gpt-6.1-sol-fast +retries=1 +timeout=30m +``` + + +```edges +success -> reviewer +blocked_retryable -> implementer +blocked -> #failure +error -> #failure +``` + + +# stage reviewer + +```limits +{"max_visits": 3, "on": ["rejected"]} +``` + +```memory +{"task": "input.prompt", "acceptance_checks": "implementer.acceptance_checks", "changed_files": "implementer.changed_files", "implementation": "implementer.implementation_summary?"} +``` + + +```output +{ summary, changed_files, verification_evidence, duplication_review, performance_review, findings } +``` + +```prompt +Verify with the build and the unit tests as android/AGENTS.md gives them, and for a +change with a rendered surface check the captures against the running app on the device the map +names. Enforce the map's geometry-token rule: a raw dimension outside the token file is a must-fix +finding. + +Review the actual changes against the task in input.prompt and the acceptance +checks the implementer recorded. Run the repository's relevant verification checks +with the stage tools (for example build, lint, tests or type checks) and report the +exact commands and observed results in verification_evidence. + +Independently assess code quality: +- Duplication: search the run directory for repeated or near-identical functions, + blocks and copy-pasted logic introduced by the change; identify code that should be + shared and record concrete locations in duplication_review. +- Performance and resources: flag redundant computation, unnecessary dependencies, + and repeated I/O or database/filesystem work that wastes time, CPU or memory; + record it in performance_review. +- Workarounds: flag hacks, suppressed errors and shortcuts that trade reliability for + speed in findings. + +Keep verification proportionate to the change: exercise the files and screens the diff +actually touches, capture at most one screenshot per changed surface plus one regression +screen, and do not repeat sweeps or measurements that an earlier visit already recorded. + +Return rejected ONLY when at least one must-fix finding exists: a defect that blocks +acceptance and that you can locate concretely (file and line), with evidence and a +remedy. Do not return rejected for advisory-only, cosmetic or nit items. When there +is no must-fix finding but you have non-blocking notes (duplication, performance or +workaround observations), record them in findings and return advisory, which +continues the run without another implementer visit. Return success only when the +observed checks pass and you have nothing further to record. Do not edit files or +claim unexecuted checks. +``` + +```tools +read +glob +grep +shell +``` + +```configs +provider=openai +model=gpt-6.1-sol-fast +retries=1 +timeout=45m +``` + + +```edges +success -> ui_reviewer +advisory -> ui_reviewer +rejected -> implementer +error -> #failure +``` + + +# stage ui_reviewer + +```limits +{"max_visits": 3, "on": ["rejected"]} +``` + +```memory +{"task": "input.prompt", "acceptance_checks": "implementer.acceptance_checks", "changed_files": "implementer.changed_files", "implementation": "implementer.implementation_summary?"} +``` + + +```output +{ summary, changed_files, ui_applicable, screenshots, ui_review, platform_conventions, findings } +``` + +```prompt +Decide whether this change touches a user-facing UI. If it does not (a backend, +library, CLI, config or data-only change with no rendered surface), set +ui_applicable=false, explain briefly in summary, and return skipped. Do not invent +screenshots or visual work. + +When the change does affect a user-facing UI, verify it visually: +- Build and launch the application the way the repository actually does it, then + capture screenshots of every relevant screen, state and viewport (default, empty, + loading, error, long content, small and large sizes) with the target platform's + screenshot tooling through shell. Prefer a headless or scripted capture when the + project supports one. Save the images under the run directory and list the exact + capture commands and image paths in screenshots. +- Read the captured images and review them as a UI/UX designer: layout, spacing, + alignment, typographic hierarchy, color and contrast, iconography, interaction + and focus states, loading/empty/error states, accessibility, and consistency. +- Judge the UI against established industry patterns and, above all, the native + conventions of the platform the app targets (operating system, toolkit or web + standards). Record the platform you reviewed against in platform_conventions, + and any deviations and concrete fixes in findings. Infer the target platform from + the task and the repository's actual UI toolkit when it is unstated. +- Approve only when the screenshots show the task and the acceptance checks are met + and the UI is correct, consistent and native to the target platform; otherwise + return rejected with specific, actionable findings the implementer can fix. + +Keep the capture pass proportionate: capture each changed state once (for example one +collapsed and one expanded section, one run state, one stuck header) instead of every +screen at multiple viewports or pan directions, and reuse what this run already captured +rather than re-doing it. + +If the change is UI-facing but screenshots genuinely cannot be captured +(for example no runnable UI or no display), return blocked rather than approving. +Do not edit files, and do not approve a UI you did not actually see. + +Return rejected only for a must-fix visual defect with a concrete location, evidence +and remedy; record non-blocking polish, duplication or convention notes in findings +and return advisory, which continues the run without another implementer visit. +``` + +```tools +read +glob +grep +shell +``` + +```configs +provider=openai +model=gpt-6.1-sol-fast +retries=1 +timeout=45m +``` + + +```edges +skipped -> committer +approved -> committer +advisory -> committer +rejected -> implementer +blocked -> #failure +error -> #failure +``` + +# stage committer + +```memory +{"task": "input.prompt", "changed_files": "implementer.changed_files", "review": "reviewer.summary?", "ui_review": "ui_reviewer.summary?"} +``` + +```output +{ commit, files, conflicts_resolved, summary } +``` + +```prompt +Commit the work this run produced, and resolve any conflict that is present. This stage +runs after the reviews have approved and is the only stage allowed to commit. + +Before committing, check that the change is reflected where it belongs: the packaged +templates, the packaged +authoring skill references. Update anything the change makes stale, and state in the +commit body what you checked, saying explicitly when none of them needs updating. + +Stage exactly the files this run changed - the paths in changed_files that still exist - +using the repository's own version control commands, and commit them as one commit whose +subject says what the change does and whose body summarises it. Do not stage unrelated +files, generated or ignored files, or anything another change owns. Do not invent work +that was not done; describe what is actually in the diff. + +If the workspace is in a conflicted state - unmerged paths, or a merge, rebase or +cherry-pick left unfinished - resolve each conflict: keep both intents where they are +compatible, and choose the result that satisfies the task where they are not, then +complete the operation so the tree is clean and the commit is finished. Never commit +conflict markers, and never discard the other side's work silently; record what you +resolved. + +Do not push, tag, revert, or rewrite unrelated history. If there is nothing to commit, +report that instead of creating an empty commit. Report the commit identifier, the files +committed, whatever you resolved, and anything you deliberately left uncommitted. + + +Finally, bring this branch up to date with main, in this same stage. After committing the work +above, merge main into this branch here, resolve any conflict here - keeping both intents where they +are compatible, and choosing the result that satisfies this task where they are not - run the +repository's checks again after resolving, and commit the completed merge with a message naming what +was combined. The point is that this branch can later be merged back without conflict: leave the +tree clean and the merge finished. Never leave conflict markers, never discard the other side's work +silently. If the merge genuinely cannot be completed, say so with the path and the reason instead of +leaving the merge half-finished. + +Then land it: this branch is the work and main is where it belongs, so merge this branch into main +as the last act of this stage. Find the repository's main working tree with `git worktree list` (the +entry checked out on main, not this one) and merge this branch into it there with `--no-ff`, with a +message saying what the change does. Two lanes can land at the same moment, so take a lock first - +`mkdir` a lock directory beside main's `git` directory in a bounded retry loop, and remove it when +done - and retry briefly if another lane holds it. Then verify with +`git merge-base --is-ancestor main` and report the commit main now has. If main cannot +be updated, say exactly why and leave this branch committed and clean rather than leaving main +half-merged. There is no separate merge pipeline and nothing to hand this to: if it cannot be landed +here, it is reported here. + + +Do not edit README or other shared documentation files: an item records what it changed and which documentation is now stale in its own commit body, and the shared documents are folded by a single writer. + +``` + +```tools +read +glob +grep +edit +shell +``` + +```configs +provider=openai +model=gpt-6.1-sol-fast +retries=1 +timeout=30m +``` + +```edges +success -> #success +blocked -> #failure +error -> #failure +``` diff --git a/internal/server/testdata/android-v3-published.md b/internal/server/testdata/android-v3-published.md new file mode 100644 index 00000000..344e537b --- /dev/null +++ b/internal/server/testdata/android-v3-published.md @@ -0,0 +1,296 @@ +# stage implementer + +```limits +{"max_visits":5,"on":["blocked_retryable"]} +``` + + +```memory +{"task": "input.prompt", "code_findings": "reviewer.findings?", "ui_findings": "ui_reviewer.findings?"} +``` + +```output +{ task, acceptance_checks, changed_files, implementation_summary, new_helpers } +``` + +```prompt +Implement input.prompt in the run directory using the repository's actual languages and +file types; create files as needed. + +Read android/AGENTS.md first. It is the map of this project - how to build, test, run and capture here, where +the pieces live, and the conventions that apply. Do not re-derive what it tells you and do not go +looking for what it already names; if it is wrong or out of date, say so in your output. + +Read the surrounding code before editing so you extend the existing helpers and patterns rather than +duplicating them. Keep each change as small as the task allows and follow the repository's own +conventions and error handling. + +This is the Android app. Build and test it with the project's own Gradle invocation as the map +gives it, and treat the map's geometry tokens as the only source of dimensions - a UI change that +adds a raw value fails the repository's own check. A change with a rendered surface is verified on +the device the map names (never an emulator), capturing what the map says to capture. + +Record the acceptance checks in acceptance_checks, the actual changes in changed_files, and any new +shared helper in new_helpers. Verify with the checks this repository uses - as the map gives them - +and report the exact commands and observed output. Address any concrete findings the runner supplies +and note the actual fixes. Return blocked if the required scope or tools are unavailable. Do not +commit or push; the committer stage lands the work. +``` + +```tools +read +glob +grep +edit +shell +``` + +```configs +provider=openai +model=gpt-5.6-terra +retries=1 +timeout=30m +``` + + +```edges +success -> reviewer +blocked_retryable -> implementer +blocked -> #failure +error -> #failure +``` + + +# stage reviewer + +```limits +{"max_visits": 3, "on": ["rejected"]} +``` + +```memory +{"task": "input.prompt", "acceptance_checks": "implementer.acceptance_checks", "changed_files": "implementer.changed_files", "implementation": "implementer.implementation_summary?"} +``` + + +```output +{ summary, changed_files, verification_evidence, duplication_review, performance_review, findings } +``` + +```prompt +Verify with the build and the unit tests as android/AGENTS.md gives them, and for a +change with a rendered surface check the captures against the running app on the device the map +names. Enforce the map's geometry-token rule: a raw dimension outside the token file is a must-fix +finding. + +Review the actual changes against the task in input.prompt and the acceptance +checks the implementer recorded. Run the repository's relevant verification checks +with the stage tools (for example build, lint, tests or type checks) and report the +exact commands and observed results in verification_evidence. + +Independently assess code quality: +- Duplication: search the run directory for repeated or near-identical functions, + blocks and copy-pasted logic introduced by the change; identify code that should be + shared and record concrete locations in duplication_review. +- Performance and resources: flag redundant computation, unnecessary dependencies, + and repeated I/O or database/filesystem work that wastes time, CPU or memory; + record it in performance_review. +- Workarounds: flag hacks, suppressed errors and shortcuts that trade reliability for + speed in findings. + +Keep verification proportionate to the change: exercise the files and screens the diff +actually touches, capture at most one screenshot per changed surface plus one regression +screen, and do not repeat sweeps or measurements that an earlier visit already recorded. + +Return rejected ONLY when at least one must-fix finding exists: a defect that blocks +acceptance and that you can locate concretely (file and line), with evidence and a +remedy. Do not return rejected for advisory-only, cosmetic or nit items. When there +is no must-fix finding but you have non-blocking notes (duplication, performance or +workaround observations), record them in findings and return advisory, which +continues the run without another implementer visit. Return success only when the +observed checks pass and you have nothing further to record. Do not edit files or +claim unexecuted checks. +``` + +```tools +read +glob +grep +shell +``` + +```configs +provider=openai +model=gpt-5.6-luna +retries=1 +timeout=45m +``` + + +```edges +success -> ui_reviewer +advisory -> ui_reviewer +rejected -> implementer +error -> #failure +``` + + +# stage ui_reviewer + +```limits +{"max_visits": 3, "on": ["rejected"]} +``` + +```memory +{"task": "input.prompt", "acceptance_checks": "implementer.acceptance_checks", "changed_files": "implementer.changed_files", "implementation": "implementer.implementation_summary?"} +``` + + +```output +{ summary, changed_files, ui_applicable, screenshots, ui_review, platform_conventions, findings } +``` + +```prompt +Decide whether this change touches a user-facing UI. If it does not (a backend, +library, CLI, config or data-only change with no rendered surface), set +ui_applicable=false, explain briefly in summary, and return skipped. Do not invent +screenshots or visual work. + +When the change does affect a user-facing UI, verify it visually: +- Build and launch the application the way the repository actually does it, then + capture screenshots of every relevant screen, state and viewport (default, empty, + loading, error, long content, small and large sizes) with the target platform's + screenshot tooling through shell. Prefer a headless or scripted capture when the + project supports one. Save the images under the run directory and list the exact + capture commands and image paths in screenshots. +- Read the captured images and review them as a UI/UX designer: layout, spacing, + alignment, typographic hierarchy, color and contrast, iconography, interaction + and focus states, loading/empty/error states, accessibility, and consistency. +- Judge the UI against established industry patterns and, above all, the native + conventions of the platform the app targets (operating system, toolkit or web + standards). Record the platform you reviewed against in platform_conventions, + and any deviations and concrete fixes in findings. Infer the target platform from + the task and the repository's actual UI toolkit when it is unstated. +- Approve only when the screenshots show the task and the acceptance checks are met + and the UI is correct, consistent and native to the target platform; otherwise + return rejected with specific, actionable findings the implementer can fix. + +Keep the capture pass proportionate: capture each changed state once (for example one +collapsed and one expanded section, one run state, one stuck header) instead of every +screen at multiple viewports or pan directions, and reuse what this run already captured +rather than re-doing it. + +If the change is UI-facing but screenshots genuinely cannot be captured +(for example no runnable UI or no display), return blocked rather than approving. +Do not edit files, and do not approve a UI you did not actually see. + +Return rejected only for a must-fix visual defect with a concrete location, evidence +and remedy; record non-blocking polish, duplication or convention notes in findings +and return advisory, which continues the run without another implementer visit. +``` + +```tools +read +glob +grep +shell +``` + +```configs +provider=openai +model=gpt-5.6-luna +retries=1 +timeout=45m +``` + + +```edges +skipped -> committer +approved -> committer +advisory -> committer +rejected -> implementer +blocked -> #failure +error -> #failure +``` + +# stage committer + +```memory +{"task": "input.prompt", "changed_files": "implementer.changed_files", "review": "reviewer.summary?", "ui_review": "ui_reviewer.summary?"} +``` + +```output +{ commit, files, conflicts_resolved, summary } +``` + +```prompt +Commit the work this run produced, and resolve any conflict that is present. This stage +runs after the reviews have approved and is the only stage allowed to commit. + +Before committing, check that the change is reflected where it belongs: the packaged +templates, the packaged +authoring skill references. Update anything the change makes stale, and state in the +commit body what you checked, saying explicitly when none of them needs updating. + +Stage exactly the files this run changed - the paths in changed_files that still exist - +using the repository's own version control commands, and commit them as one commit whose +subject says what the change does and whose body summarises it. Do not stage unrelated +files, generated or ignored files, or anything another change owns. Do not invent work +that was not done; describe what is actually in the diff. + +If the workspace is in a conflicted state - unmerged paths, or a merge, rebase or +cherry-pick left unfinished - resolve each conflict: keep both intents where they are +compatible, and choose the result that satisfies the task where they are not, then +complete the operation so the tree is clean and the commit is finished. Never commit +conflict markers, and never discard the other side's work silently; record what you +resolved. + +Do not push, tag, revert, or rewrite unrelated history. If there is nothing to commit, +report that instead of creating an empty commit. Report the commit identifier, the files +committed, whatever you resolved, and anything you deliberately left uncommitted. + + +Finally, bring this branch up to date with main, in this same stage. After committing the work +above, merge main into this branch here, resolve any conflict here - keeping both intents where they +are compatible, and choosing the result that satisfies this task where they are not - run the +repository's checks again after resolving, and commit the completed merge with a message naming what +was combined. The point is that this branch can later be merged back without conflict: leave the +tree clean and the merge finished. Never leave conflict markers, never discard the other side's work +silently. If the merge genuinely cannot be completed, say so with the path and the reason instead of +leaving the merge half-finished. + +Then land it: this branch is the work and main is where it belongs, so merge this branch into main +as the last act of this stage. Find the repository's main working tree with `git worktree list` (the +entry checked out on main, not this one) and merge this branch into it there with `--no-ff`, with a +message saying what the change does. Two lanes can land at the same moment, so take a lock first - +`mkdir` a lock directory beside main's `git` directory in a bounded retry loop, and remove it when +done - and retry briefly if another lane holds it. Then verify with +`git merge-base --is-ancestor main` and report the commit main now has. If main cannot +be updated, say exactly why and leave this branch committed and clean rather than leaving main +half-merged. There is no separate merge pipeline and nothing to hand this to: if it cannot be landed +here, it is reported here. + + +Do not edit README or other shared documentation files: an item records what it changed and which documentation is now stale in its own commit body, and the shared documents are folded by a single writer. + +``` + +```tools +read +glob +grep +edit +shell +``` + +```configs +provider=openai +model=gpt-5.6-terra +retries=1 +timeout=30m +``` + +```edges +success -> #success +blocked -> #failure +error -> #failure +``` diff --git a/internal/server/verification_fixture.go b/internal/server/verification_fixture.go new file mode 100644 index 00000000..767723ab --- /dev/null +++ b/internal/server/verification_fixture.go @@ -0,0 +1,135 @@ +package server + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "os" + "strings" + "time" + + "git.a-wan.net/alan/pipelines/internal/stage" +) + +const ( + verificationFixtureID = "reconnect-foreground" + verificationFixtureResponseLimit int64 = 64 << 10 + verificationFixtureTimeout = 10 * time.Second +) + +// VerificationFixture is a server-owned, explicitly registered probe. Its +// endpoint and expected provenance come from server configuration, never a +// stage definition or result. +type VerificationFixture struct { + ID, Endpoint, Backend, Revision, Capability, State string +} + +// VerificationFixturesFromEnvironment returns the single supported fixture +// registration when its endpoint is explicitly configured. There is no default +// endpoint: an unconfigured server fails target-aware verification closed. +func VerificationFixturesFromEnvironment(getenv func(string) string) []VerificationFixture { + if getenv == nil { + getenv = os.Getenv + } + endpoint := strings.TrimSpace(getenv("PIPELINES_VERIFICATION_RECONNECT_FOREGROUND_URL")) + if endpoint == "" { + return nil + } + return []VerificationFixture{{ + ID: verificationFixtureID, + Endpoint: endpoint, + Backend: strings.TrimSpace(getenv("PIPELINES_VERIFICATION_RECONNECT_FOREGROUND_BACKEND")), + Revision: strings.TrimSpace(getenv("PIPELINES_VERIFICATION_RECONNECT_FOREGROUND_REVISION")), + Capability: strings.TrimSpace(getenv("PIPELINES_VERIFICATION_RECONNECT_FOREGROUND_CAPABILITY")), + State: strings.TrimSpace(getenv("PIPELINES_VERIFICATION_RECONNECT_FOREGROUND_STATE")), + }} +} + +type verificationFixtureRegistry struct { + fixtures map[string]VerificationFixture + err error +} + +func newVerificationFixtureRegistry(fixtures []VerificationFixture) *verificationFixtureRegistry { + r := &verificationFixtureRegistry{fixtures: make(map[string]VerificationFixture, len(fixtures))} + for _, fixture := range fixtures { + if err := validVerificationFixture(fixture); err != nil { + r.err = err + return r + } + if _, exists := r.fixtures[fixture.ID]; exists { + r.err = fmt.Errorf("duplicate verification fixture %q", fixture.ID) + return r + } + r.fixtures[fixture.ID] = fixture + } + return r +} + +func validVerificationFixture(fixture VerificationFixture) error { + if fixture.ID != verificationFixtureID { + return fmt.Errorf("unsupported verification fixture %q", fixture.ID) + } + for _, value := range []string{fixture.Endpoint, fixture.Backend, fixture.Revision, fixture.Capability, fixture.State} { + if value == "" || len(value) > 256 { + return fmt.Errorf("verification fixture %q has incomplete configuration", fixture.ID) + } + } + u, err := url.Parse(fixture.Endpoint) + if err != nil || u.Host == "" || (u.Scheme != "http" && u.Scheme != "https") { + return fmt.Errorf("verification fixture %q has invalid endpoint", fixture.ID) + } + return nil +} + +func (r *verificationFixtureRegistry) verify(ctx context.Context, target stage.VerificationTarget) (stage.VerificationEvidence, error) { + if r.err != nil { + return stage.VerificationEvidence{}, r.err + } + fixture, exists := r.fixtures[target.Fixture] + if !exists { + return stage.VerificationEvidence{}, fmt.Errorf("unknown verification fixture %q", target.Fixture) + } + if fixture.Backend != target.Backend || fixture.Revision != target.Revision || fixture.Capability != target.Capability { + return stage.VerificationEvidence{}, fmt.Errorf("verification target does not match registered fixture %q", target.Fixture) + } + probeCtx, cancel := context.WithTimeout(ctx, verificationFixtureTimeout) + defer cancel() + req, err := http.NewRequestWithContext(probeCtx, http.MethodGet, fixture.Endpoint, nil) + if err != nil { + return stage.VerificationEvidence{}, fmt.Errorf("verification fixture %q request: %w", fixture.ID, err) + } + client := &http.Client{Timeout: verificationFixtureTimeout, CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }} + response, err := client.Do(req) + if err != nil { + return stage.VerificationEvidence{}, fmt.Errorf("verification fixture %q probe: %w", fixture.ID, err) + } + defer response.Body.Close() + if response.StatusCode != http.StatusOK { + return stage.VerificationEvidence{}, fmt.Errorf("verification fixture %q probe returned %s", fixture.ID, response.Status) + } + var observed struct { + Backend string `json:"backend"` + Revision string `json:"revision"` + Capability string `json:"capability"` + Fixture string `json:"fixture"` + State string `json:"state"` + } + body, err := io.ReadAll(io.LimitReader(response.Body, verificationFixtureResponseLimit+1)) + if err != nil { + return stage.VerificationEvidence{}, fmt.Errorf("verification fixture %q response: %w", fixture.ID, err) + } + if int64(len(body)) > verificationFixtureResponseLimit { + return stage.VerificationEvidence{}, fmt.Errorf("verification fixture %q response exceeds %d bytes", fixture.ID, verificationFixtureResponseLimit) + } + if err := json.Unmarshal(body, &observed); err != nil { + return stage.VerificationEvidence{}, fmt.Errorf("verification fixture %q response: %w", fixture.ID, err) + } + if observed.Backend != fixture.Backend || observed.Revision != fixture.Revision || observed.Capability != fixture.Capability || observed.Fixture != fixture.ID || observed.State != fixture.State { + return stage.VerificationEvidence{}, fmt.Errorf("verification fixture %q returned incompatible backend, revision, capability, fixture, or state", fixture.ID) + } + return stage.VerificationEvidence{Backend: observed.Backend, Revision: observed.Revision, Capability: observed.Capability, Fixture: observed.Fixture}, nil +} diff --git a/internal/server/verification_fixture_test.go b/internal/server/verification_fixture_test.go new file mode 100644 index 00000000..3f71ca96 --- /dev/null +++ b/internal/server/verification_fixture_test.go @@ -0,0 +1,148 @@ +package server + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + "git.a-wan.net/alan/pipelines/internal/pipeline" + "git.a-wan.net/alan/pipelines/internal/protocol" + "git.a-wan.net/alan/pipelines/internal/stage" +) + +func TestHandlerRunsRegisteredVerificationFixture(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + var mu sync.Mutex + response := map[string]string{"backend": "branch-api", "revision": "abc123", "capability": "controlled-state", "fixture": verificationFixtureID, "state": "ready"} + calls := 0 + fixture := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + mu.Lock() + defer mu.Unlock() + calls++ + _ = json.NewEncoder(w).Encode(response) + })) + defer fixture.Close() + + store, err := pipeline.Open(filepath.Join(t.TempDir(), "fixture.db")) + if err != nil { + t.Fatal(err) + } + defer store.Close() + source := pipeline.Template([]string{"worker"}, true) + source = strings.Replace(source, "```configs", "```verification\nbackend=branch-api\nrevision=abc123\ncapability=controlled-state\nfixture="+verificationFixtureID+"\n```\n\n```configs", 1) + if _, err := store.Submit(ctx, "controlled-fixture", source); err != nil { + t.Fatal(err) + } + items, err := store.List(ctx) + if err != nil { + t.Fatal(err) + } + handler := HandlerWithOptions(ctx, Options{VerificationFixtures: []VerificationFixture{{ID: verificationFixtureID, Endpoint: fixture.URL, Backend: "branch-api", Revision: "abc123", Capability: "controlled-state", State: "ready"}}}, store).(*runHandler) + defer func() { + cancel() + WaitRuns(handler) + }() + r := handler.runner + if r.verifyTarget == nil { + t.Fatal("normal server construction did not wire controlled fixture verification") + } + r.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) { + // These forged model labels must not influence fixture provenance. + return stage.Result{Outcome: "success", Output: stage.Values{"verification": map[string]any{"backend": "old-api", "revision": "stale", "capability": "forged", "fixture": "other"}}}, nil + } + start := func(key string) *protocol.RunDetail { + run, err := r.start(protocol.Message{PipelineID: items[0].ID, Version: 1, RequestKey: key, Workspace: t.TempDir(), Input: map[string]any{"prompt": "verify"}}) + if err != nil { + t.Fatal(err) + } + r.wg.Wait() + detail, err := store.GetRun(ctx, run.ID) + if err != nil { + t.Fatal(err) + } + return detail + } + if detail := start("compatible"); detail.Status != "succeeded" { + t.Fatalf("compatible controlled fixture did not settle: %+v", detail) + } + mu.Lock() + response["revision"] = "old-build" + mu.Unlock() + if detail := start("stale"); detail.Status != "failed" || !strings.Contains(detail.Steps[0].Diagnostic, "incompatible") { + t.Fatalf("stale fixture provenance settled as success: %+v", detail) + } + mu.Lock() + if calls != 2 { + t.Fatalf("fixture probes = %d, want one per target-aware stage", calls) + } + mu.Unlock() +} + +func TestVerificationFixtureCancellationClosesProbe(t *testing.T) { + started := make(chan struct{}) + canceled := make(chan struct{}) + fixture := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + close(started) + <-r.Context().Done() + close(canceled) + })) + defer fixture.Close() + registry := newVerificationFixtureRegistry([]VerificationFixture{{ID: verificationFixtureID, Endpoint: fixture.URL, Backend: "branch-api", Revision: "abc123", Capability: "controlled-state", State: "ready"}}) + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan error, 1) + go func() { + _, err := registry.verify(ctx, stage.VerificationTarget{Backend: "branch-api", Revision: "abc123", Capability: "controlled-state", Fixture: verificationFixtureID}) + done <- err + }() + select { + case <-started: + case <-time.After(time.Second): + t.Fatal("fixture probe did not start") + } + cancel() + select { + case err := <-done: + if err == nil { + t.Fatal("canceled fixture probe succeeded") + } + case <-time.After(time.Second): + t.Fatal("canceled fixture probe did not return") + } + select { + case <-canceled: + case <-time.After(time.Second): + t.Fatal("canceled fixture request was not cleaned up") + } +} + +func TestVerificationFixtureRejectsUnknownAndMismatchedProvenance(t *testing.T) { + base := map[string]string{"backend": "branch-api", "revision": "abc123", "capability": "controlled-state", "fixture": verificationFixtureID, "state": "ready"} + for field, wrong := range map[string]string{"backend": "old-api", "revision": "old-build", "capability": "other-state", "state": "stale"} { + t.Run(field, func(t *testing.T) { + response := make(map[string]string, len(base)) + for key, value := range base { + response[key] = value + } + response[field] = wrong + fixture := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _ = json.NewEncoder(w).Encode(response) + })) + defer fixture.Close() + registry := newVerificationFixtureRegistry([]VerificationFixture{{ID: verificationFixtureID, Endpoint: fixture.URL, Backend: base["backend"], Revision: base["revision"], Capability: base["capability"], State: base["state"]}}) + _, err := registry.verify(context.Background(), stage.VerificationTarget{Backend: base["backend"], Revision: base["revision"], Capability: base["capability"], Fixture: verificationFixtureID}) + if err == nil || !strings.Contains(err.Error(), "incompatible") { + t.Fatalf("mismatched %s provenance was accepted: %v", field, err) + } + }) + } + registry := newVerificationFixtureRegistry(nil) + if _, err := registry.verify(context.Background(), stage.VerificationTarget{Backend: "branch-api", Revision: "abc123", Capability: "controlled-state", Fixture: "unknown"}); err == nil || !strings.Contains(err.Error(), "unknown") { + t.Fatalf("unknown fixture was accepted: %v", err) + } +} diff --git a/internal/server/websocket.go b/internal/server/websocket.go index adb3184b..5e598223 100644 --- a/internal/server/websocket.go +++ b/internal/server/websocket.go @@ -49,6 +49,10 @@ type Options struct { // read time with no model call, which keeps tests and unconfigured servers // deterministic. StepVerdicts func(context.Context, map[string]any) string + // VerificationFixtures are explicit server-owned HTTP fixtures for optional + // target-aware verification. Definitions can name only a registered ID; they + // cannot select an endpoint or supply provenance. + VerificationFixtures []VerificationFixture } // Handler serves the persistent connection with default options. Canceling ctx @@ -81,6 +85,10 @@ func HandlerWithOptions(ctx context.Context, options Options, stores ...*pipelin if runErr == nil && options.StepVerdicts != nil { runs.verdict = options.StepVerdicts } + if runErr == nil { + fixtures := newVerificationFixtureRegistry(options.VerificationFixtures) + runs.verifyTarget = fixtures.verify + } if runErr == nil { runs.schedulerWG.Add(1) go runs.scheduleLoop() diff --git a/internal/stage/agent.go b/internal/stage/agent.go index e0c3974d..dd91d1f3 100644 --- a/internal/stage/agent.go +++ b/internal/stage/agent.go @@ -14,16 +14,28 @@ import ( // before any prompt is admitted, so restart recovery can interrupt the agent. type AgentClient struct { OpenCodeClient - WritePaths []string - Tools []string // nil is the legacy manual approval bundle; empty denies all - OnSession func(string, bool) error - SessionID string - Transcript string - Policy ToolPolicy - WorkStarted bool + WritePaths []string + Tools []string // nil is the legacy manual approval bundle; empty denies all + // ArtifactAccess is supplied only by the runner. Write is this visit's + // directory; Read contains this and completed earlier visits in the same run. + ArtifactAccess ArtifactAccess + LandingAccess LandingAccess + OnSession func(string, bool) error + SessionID string + Transcript string + Policy ToolPolicy + WorkStarted bool } func (c OpenCodeClient) Interrupt(ctx context.Context, id string) error { + return c.interrupt(ctx, id, false) +} + +func (c OpenCodeClient) InterruptAndConfirm(ctx context.Context, id string) error { + return c.interrupt(ctx, id, true) +} + +func (c OpenCodeClient) interrupt(ctx context.Context, id string, confirm bool) error { if !sessionIDPattern.MatchString(id) { return fmt.Errorf("invalid session ID") } @@ -43,7 +55,20 @@ func (c OpenCodeClient) Interrupt(ctx context.Context, id string) error { c.Backend = connection.Backend } _, err := c.call(ctx, "post", "/api/session/"+id+"/interrupt", map[string]any{}) - return err + if err != nil || !confirm { + return err + } + // An interrupt acknowledgement is not a cleanup receipt. Keep ownership + // until the harness positively reports settlement, bounded by the caller. + if c.Backend == "v1" { + _, err := c.call(ctx, "post", "/api/experimental/session/"+id+"/wait", nil) + return err + } + // The failed stage wait already records deadline evidence. Cleanup settlement + // must still be confirmed, but must not add a second duplicate operation entry. + confirmed := c + confirmed.Operations = nil + return confirmed.waitForSessionIdle(ctx, id) } func (c *AgentClient) Generate(ctx context.Context, config ModelConfig, prompt string) (text string, err error) { @@ -97,7 +122,7 @@ func (c *AgentClient) Generate(ctx context.Context, config ModelConfig, prompt s defer cancel() if err != nil { c.captureTimedOutEvidence(ctx, err, config, watch) - if stopErr := c.Interrupt(cleanup, c.SessionID); stopErr != nil { + if stopErr := c.InterruptAndConfirm(cleanup, c.SessionID); stopErr != nil { err = fmt.Errorf("agent interrupt failed; retained session %s needs inspection: %w", c.SessionID, errors.Join(err, stopErr, ErrStageCleanup)) return } diff --git a/internal/stage/agent_test.go b/internal/stage/agent_test.go index 2a8bcb58..c18d0721 100644 --- a/internal/stage/agent_test.go +++ b/internal/stage/agent_test.go @@ -52,8 +52,8 @@ func TestAgentOwnershipPermissionsAndInterrupt(t *testing.T) { t.Fatal("unexpected call", args) return nil, nil } - if _, err := client.Generate(context.Background(), ModelConfig{}, "This call has no tools."); err == nil || !interrupted || owned { - t.Fatal("failed generation did not stop and release ownership") + if _, err := client.Generate(context.Background(), ModelConfig{}, "This call has no tools."); !errors.Is(err, ErrStageCleanup) || !interrupted || !owned { + t.Fatal("interrupt acknowledgement without idle confirmation must retain ownership") } } diff --git a/internal/stage/budget_test.go b/internal/stage/budget_test.go index 47b0ee08..21426679 100644 --- a/internal/stage/budget_test.go +++ b/internal/stage/budget_test.go @@ -82,6 +82,9 @@ func TestBudgetsAccumulateAcrossRetriesAndBlockFurtherGeneration(t *testing.T) { var sessions, generations atomic.Int64 client := &OpenCodeClient{command: func(ctx context.Context, _ string, args ...string) ([]byte, error) { path := args[2] + if path == "/api/session/active" { + return []byte(`{"data":{}}`), nil + } if path == "/api/session" { return []byte(fmt.Sprintf(`{"data":{"id":"ses_attempt_%d"}}`, sessions.Add(1))), nil } @@ -147,6 +150,9 @@ func TestLiveAgentBudgetInterruptsAndDoesNotReplay(t *testing.T) { return []byte(`{"data":{"id":"ses_agent_budget"}}`), nil } if path == "/api/session/active" { + if interrupts.Load() > 0 { + return []byte(`{"data":{}}`), nil + } <-ctx.Done() return nil, ctx.Err() } diff --git a/internal/stage/model.go b/internal/stage/model.go index 13e3d1ff..18465c67 100644 --- a/internal/stage/model.go +++ b/internal/stage/model.go @@ -59,7 +59,7 @@ func (ModelStage) Spec(def Definition) (ModelSpec, error) { } for block := range def.Blocks { switch block { - case "output", "prompt", "edges", "configs", "tools", "permissions": + case "output", "prompt", "edges", "configs", "tools", "permissions", "checks", "verification": default: return ModelSpec{}, fmt.Errorf("unexpected block %q for model stage", block) } @@ -70,6 +70,9 @@ func (ModelStage) Spec(def Definition) (ModelSpec, error) { if _, err := StagePermissions(def); err != nil { return ModelSpec{}, err } + if _, err := VerificationTargetFor(def); err != nil { + return ModelSpec{}, err + } for _, key := range []string{"provider", "model"} { if def.Configs[key] == "" { return ModelSpec{}, fmt.Errorf("missing config %s", key) @@ -77,7 +80,7 @@ func (ModelStage) Spec(def Definition) (ModelSpec, error) { } for key := range def.Configs { switch key { - case "provider", "model", "retries", "timeout", "budget_usd", "budget_tokens", "harness": + case "provider", "model", "retries", "timeout", "budget_usd", "budget_tokens", "harness", "delivery": default: return ModelSpec{}, fmt.Errorf("unknown config %q", key) } @@ -92,6 +95,15 @@ func (ModelStage) Spec(def Definition) (ModelSpec, error) { if backend != "opencode" && backend != "pi" { return ModelSpec{}, fmt.Errorf("harness must be opencode or pi") } + if delivery := def.Configs["delivery"]; delivery != "" && delivery != "true" { + return ModelSpec{}, fmt.Errorf("delivery must be true when set") + } + if def.Configs["delivery"] == "true" && backend != "opencode" { + return ModelSpec{}, fmt.Errorf("delivery is supported only by the opencode harness") + } + if _, present := def.Blocks["checks"]; present && def.Configs["delivery"] != "true" { + return ModelSpec{}, fmt.Errorf("checks block is supported only by delivery stages") + } if backend == "pi" { tools, _, _ := ToolNames(def) if _, err := piTools(tools); err != nil { diff --git a/internal/stage/model_diagnostic_test.go b/internal/stage/model_diagnostic_test.go index e61a5347..00be8b01 100644 --- a/internal/stage/model_diagnostic_test.go +++ b/internal/stage/model_diagnostic_test.go @@ -36,8 +36,8 @@ func TestModelFailureCategoriesAreSanitizedAndDistinct(t *testing.T) { if typed := wrapModelStageFailure("worker", &openCodeError{operation: "get /api/session", status: 503}); !strings.Contains(typed.Error(), "HARNESS TRANSPORT") || !errors.As(typed, new(*openCodeError)) { t.Fatalf("typed harness error was not classified: %v", typed) } - if deadline := wrapModelStageFailure("worker", context.DeadlineExceeded); !strings.Contains(deadline.Error(), "HARNESS TRANSPORT") || !errors.Is(deadline, context.DeadlineExceeded) { - t.Fatalf("context deadline was not classified as harness: %v", deadline) + if deadline := wrapModelStageFailure("worker", context.DeadlineExceeded); strings.Contains(deadline.Error(), "HARNESS TRANSPORT") || !errors.Is(deadline, context.DeadlineExceeded) { + t.Fatalf("bare context deadline was misclassified or lost: %v", deadline) } if modelFailureCategory(nil) != "" || wrapModelStageFailure("worker", nil) != nil { t.Fatal("nil failure was classified") diff --git a/internal/stage/model_failure.go b/internal/stage/model_failure.go index 6cef2b1b..2c6af418 100644 --- a/internal/stage/model_failure.go +++ b/internal/stage/model_failure.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "time" ) // Retained model-stage failure categories. A strict final-result validation @@ -13,8 +14,27 @@ import ( const ( finalResultValidation = "FINAL RESULT VALIDATION" harnessTransport = "HARNESS TRANSPORT" + stageDeadline = "STAGE DEADLINE" ) +// stageTimeoutCause identifies this attempt's configured timer, not a parent +// cancellation or a request's own deadline. It is carried through budget contexts. +type stageTimeoutCause struct{ timeout time.Duration } + +func (e *stageTimeoutCause) Error() string { return "configured stage timeout reached" } +func (e *stageTimeoutCause) Unwrap() error { return context.DeadlineExceeded } + +type stageDeadlineError struct { + timeout time.Duration + session string +} + +func (e *stageDeadlineError) Error() string { + return fmt.Sprintf("configured stage deadline reached (timeout %s) while waiting for session idle; retained OpenCode session %.128s was still active at the last poll; stage had not finished", e.timeout, RedactEvidence(e.session)) +} + +func (e *stageDeadlineError) Unwrap() error { return context.DeadlineExceeded } + // finalResultGuidance is a fixed sanitized sentence appended only to validation // diagnostics. It states the retained side effects and the cost of a full retry. const finalResultGuidance = "completed edits and tool side effects are retained and not undone; a full retry repeats work" @@ -51,11 +71,14 @@ func (c *PiClient) workAdmitted() bool { return c.WorkStarted } // the label stays sanitized and bounded. Validation is checked first: a // structured-output failure is never reported as a transport failure and vice versa. func modelFailureCategory(err error) string { + var deadline *stageDeadlineError switch { case errors.Is(err, ErrInvalidModelResult): return finalResultValidation - case harnessTransportFailure(err), errors.Is(err, context.DeadlineExceeded): + case harnessTransportFailure(err): return harnessTransport + case errors.As(err, &deadline): + return stageDeadline default: return "" } @@ -73,6 +96,8 @@ func wrapModelStageFailure(key string, err error) error { return fmt.Errorf("model stage %s: %s: %w; %s", key, finalResultValidation, err, finalResultGuidance) case harnessTransport: return fmt.Errorf("model stage %s: %s: %w", key, harnessTransport, err) + case stageDeadline: + return fmt.Errorf("model stage %s: %s: %w", key, stageDeadline, err) default: return fmt.Errorf("model stage %s: %w", key, err) } diff --git a/internal/stage/model_run.go b/internal/stage/model_run.go index e1979706..41300243 100644 --- a/internal/stage/model_run.go +++ b/internal/stage/model_run.go @@ -61,7 +61,7 @@ func (m ModelStage) Run(ctx context.Context, def Definition, input Values) (Resu return Result{}, err } spec.Config.Budget.Attempt() - attemptCtx, cancel := context.WithTimeout(ctx, spec.Config.Timeout) + attemptCtx, cancel := context.WithTimeoutCause(ctx, spec.Config.Timeout, &stageTimeoutCause{timeout: spec.Config.Timeout}) text, generateErr := client.Generate(attemptCtx, spec.Config, prompt) if generateErr == nil && attemptCtx.Err() != nil { generateErr = attemptCtx.Err() @@ -96,11 +96,16 @@ func (m ModelStage) Run(ctx context.Context, def Definition, input Values) (Resu } } else { err = generateErr - if attemptCtx.Err() != nil && !errors.Is(err, ErrBudgetExceeded) && !errors.Is(err, ErrBudgetAccounting) { + // Do not append the same context error twice: the wait/transport + // error may already preserve it, with a more precise diagnostic. + if attemptCtx.Err() != nil && !errors.Is(err, attemptCtx.Err()) && !errors.Is(err, ErrBudgetExceeded) && !errors.Is(err, ErrBudgetAccounting) { err = errors.Join(err, attemptCtx.Err()) } cancel() if ctx.Err() != nil { + if harnessTransportFailure(err) { + return Result{}, wrapModelStageFailure(def.Key, err) + } return Result{}, ctx.Err() } if errors.Is(err, ErrBudgetExceeded) || errors.Is(err, ErrBudgetAccounting) || errors.Is(err, ErrStageCleanup) { diff --git a/internal/stage/opencode.go b/internal/stage/opencode.go index 38d7722e..d58b0bf1 100644 --- a/internal/stage/opencode.go +++ b/internal/stage/opencode.go @@ -217,6 +217,11 @@ func (c OpenCodeClient) waitForSessionIdle(ctx context.Context, sessionID string select { case <-ctx.Done(): timer.Stop() + waitErr := fmt.Errorf("waiting for session idle; retained OpenCode session %.128s; stage had not finished: %w", RedactEvidence(sessionID), ctx.Err()) + var timeout *stageTimeoutCause + if errors.Is(ctx.Err(), context.DeadlineExceeded) && errors.As(context.Cause(ctx), &timeout) { + waitErr = &stageDeadlineError{timeout: timeout.timeout, session: sessionID} + } // A settled session is the only success signal; retain one bounded // sanitized record so a timed-out wait is not an empty transcript. c.Operations.record(OperationEvidence{ @@ -225,9 +230,9 @@ func (c OpenCodeClient) waitForSessionIdle(ctx context.Context, sessionID string Executable: opencode.Executable(), Directory: c.Directory, ExitCode: -1, - Diagnostic: "stage deadline reached while waiting for the agent session to become idle", + Diagnostic: waitErr.Error(), }, started) - return ctx.Err() + return waitErr case <-timer.C: } } @@ -316,7 +321,7 @@ func (c *OpenCodeClient) Generate(ctx context.Context, config ModelConfig, promp cleanup, cancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second) defer cancel() if err != nil { - if stopErr := c.Interrupt(cleanup, session.Data.ID); stopErr != nil { + if stopErr := c.InterruptAndConfirm(cleanup, session.Data.ID); stopErr != nil { err = fmt.Errorf("retained OpenCode session %s; interrupt failed: %w", session.Data.ID, errors.Join(err, stopErr, ErrStageCleanup)) return } diff --git a/internal/stage/pi.go b/internal/stage/pi.go index 6d93e4d7..e4849f79 100644 --- a/internal/stage/pi.go +++ b/internal/stage/pi.go @@ -28,6 +28,7 @@ type PiClient struct { // ArtifactsDir is exported to the spawned Pi process as // PIPELINES_ARTIFACTS_DIR so its tools can write stage artifacts. ArtifactsDir string + ArtifactAccess ArtifactAccess Policy ToolPolicy OnSession func(string, bool) error OnOwnership func(string, string, bool) error @@ -128,7 +129,7 @@ func (c *PiClient) Generate(ctx context.Context, config ModelConfig, prompt stri if err != nil { return "", err } - policy := map[string]any{"directory": c.Directory, "tools": tools, "paths": c.WritePaths, "edit": c.Policy.Edit, "shell": c.Policy.Shell} + policy := map[string]any{"directory": c.Directory, "tools": tools, "paths": c.WritePaths, "edit": c.Policy.Edit, "shell": c.Policy.Shell, "artifact_write": c.ArtifactAccess.Write, "artifact_read": c.ArtifactAccess.Read} if c.WritePaths == nil { policy["paths"] = []string{} } @@ -237,6 +238,10 @@ func (c *PiClient) Generate(ctx context.Context, config ModelConfig, prompt stri } close(done) close(heartbeatDone) + // agent_end/abort acknowledgement and the parent's exit alone do not + // prove its shell descendants exited. A surviving process group retains + // ownership rather than handing the resource to another stage. + quiet = quiet && errors.Is(syscall.Kill(-c.cmd.Process.Pid, 0), syscall.ESRCH) if quiet { _ = os.WriteFile(filepath.Join(c.Reference, "quiescent"), []byte("quiescent"), 0600) } @@ -531,9 +536,16 @@ func InterruptPi(ctx context.Context, reference string) error { if !filepath.IsAbs(reference) { return fmt.Errorf("invalid retained Pi reference") } - if _, err := os.Stat(filepath.Join(reference, "owner.json")); err != nil { + data, err := os.ReadFile(filepath.Join(reference, "owner.json")) + if err != nil { return errors.Join(err, ErrStageCleanup) } + var owner struct { + PID int `json:"pid"` + } + if json.Unmarshal(data, &owner) != nil || owner.PID <= 1 { + return errors.Join(fmt.Errorf("invalid retained Pi owner"), ErrStageCleanup) + } old := time.Unix(0, 0) if err := os.Chtimes(filepath.Join(reference, "lease"), old, old); err != nil { return err @@ -541,7 +553,7 @@ func InterruptPi(ctx context.Context, reference string) error { ticker := time.NewTicker(100 * time.Millisecond) defer ticker.Stop() for { - if data, err := os.ReadFile(filepath.Join(reference, "quiescent")); err == nil && string(data) == "quiescent" { + if data, err := os.ReadFile(filepath.Join(reference, "quiescent")); err == nil && string(data) == "quiescent" && errors.Is(syscall.Kill(owner.PID, 0), syscall.ESRCH) && errors.Is(syscall.Kill(-owner.PID, 0), syscall.ESRCH) { return nil } select { diff --git a/internal/stage/pi_policy.mjs b/internal/stage/pi_policy.mjs index aec60e93..03be3440 100644 --- a/internal/stage/pi_policy.mjs +++ b/internal/stage/pi_policy.mjs @@ -8,6 +8,24 @@ export function permit(policy, tool, args) { if (policy.shell !== null && !policy.shell.includes(args.command)) return "Shell command is not permitted"; return null; } + const artifact = (absolute, write) => { + const permitted = write ? [policy.artifact_write] : policy.artifact_read; + for (const directory of permitted) { + if (typeof directory !== "string" || !path.isAbsolute(directory)) continue; + let parent = absolute; + while (!fs.existsSync(parent)) { + const next = path.dirname(parent); + if (next === parent) break; + parent = next; + } + if (!fs.existsSync(parent)) continue; + const resolved = fs.realpathSync(parent); + const target = path.join(resolved, path.relative(parent, absolute)); + const root = fs.realpathSync(directory); + if (target === root || target.startsWith(root + path.sep)) return true; + } + return false; + }; if (tool === "edit" || tool === "write") { if (typeof args.path !== "string") return "Missing file path"; if (args.path.startsWith("~") || args.path.startsWith("@")) return "Use a literal path, not a harness path shortcut"; @@ -20,7 +38,10 @@ export function permit(policy, tool, args) { } const resolved = fs.realpathSync(parent); const root = fs.realpathSync(policy.directory); - if (resolved !== root && !resolved.startsWith(root + path.sep)) return "Edit escapes run directory"; + if (resolved !== root && !resolved.startsWith(root + path.sep)) { + if (!artifact(absolute, true)) return "Edit escapes run directory"; + return null; + } const relative = path.relative(path.resolve(policy.directory), absolute).split(path.sep).join("/"); const canonical = path.relative(root, path.join(resolved, path.relative(parent, absolute))).split(path.sep).join("/"); if (relative.startsWith("../") || path.isAbsolute(relative)) return "Edit escapes run directory"; @@ -31,9 +52,19 @@ export function permit(policy, tool, args) { }); if (policy.edit !== null && (!matches(relative) || !matches(canonical))) return "Edit is outside stage patterns"; } - if (tool === "read" && typeof args.path === "string") { + if (["read", "grep", "find"].includes(tool) && typeof args.path === "string") { const name = path.basename(args.path); if (/^(.*\.env(?:\..*)?|pipelines\.db.*)$/.test(name)) return "Protected file cannot be read"; + const absolute = path.resolve(policy.directory, args.path); + let parent = absolute; + while (!fs.existsSync(parent)) { + const next = path.dirname(parent); + if (next === parent) return "Cannot resolve read path"; + parent = next; + } + const resolved = fs.realpathSync(parent); + const root = fs.realpathSync(policy.directory); + if (resolved !== root && !resolved.startsWith(root + path.sep) && !artifact(absolute, false)) return "Read escapes run directory"; } return null; } diff --git a/internal/stage/pi_policy_test.go b/internal/stage/pi_policy_test.go new file mode 100644 index 00000000..6b1018a6 --- /dev/null +++ b/internal/stage/pi_policy_test.go @@ -0,0 +1,72 @@ +package stage + +import ( + "encoding/json" + "os" + "os/exec" + "path/filepath" + "testing" +) + +func TestPiPolicyArtifactScope(t *testing.T) { + if _, err := exec.LookPath("node"); err != nil { + t.Skip("node is required to exercise the Pi policy extension") + } + root := t.TempDir() + workspace := filepath.Join(root, "workspace") + current := filepath.Join(root, "artifacts [7]", "2") + prior := filepath.Join(root, "artifacts [7]", "1") + sibling := filepath.Join(root, "artifacts [7]", "3") + for _, directory := range []string{workspace, current, prior, sibling} { + if err := os.MkdirAll(directory, 0700); err != nil { + t.Fatal(err) + } + } + policy := map[string]any{ + "directory": workspace, + "tools": []string{"read", "grep", "find", "edit"}, + "paths": []string{}, + "edit": nil, + "shell": nil, + "artifact_write": current, + "artifact_read": []string{current, prior}, + } + policyJSON, err := json.Marshal(policy) + if err != nil { + t.Fatal(err) + } + policyFile := filepath.Join(root, "policy.mjs") + if err := os.WriteFile(policyFile, piPolicy, 0600); err != nil { + t.Fatal(err) + } + script := `const { permit } = await import(process.argv[1]); +const policy = JSON.parse(process.argv[2]); +const checks = JSON.parse(process.argv[3]); +console.log(JSON.stringify(checks.map(([tool, path]) => permit(policy, tool, {path}))));` + checks := [][2]string{ + {"edit", filepath.Join(current, "report.md")}, + {"edit", filepath.Join(prior, "report.md")}, + {"read", filepath.Join(prior, "report.md")}, + {"grep", filepath.Join(prior, "report.md")}, + {"find", filepath.Join(current, "reports")}, + {"read", filepath.Join(sibling, "report.md")}, + {"grep", filepath.Join(root, "other-run", "1", "report.md")}, + {"find", filepath.Join(current, "..", "3", "report.md")}, + {"read", filepath.Join(prior, ".env")}, + } + checksJSON, err := json.Marshal(checks) + if err != nil { + t.Fatal(err) + } + output, err := exec.Command("node", "--input-type=module", "--eval", script, "file://"+policyFile, string(policyJSON), string(checksJSON)).Output() + if err != nil { + t.Fatal(err) + } + var got []any + if err := json.Unmarshal(output, &got); err != nil { + t.Fatal(err) + } + if len(got) != len(checks) || got[0] != nil || got[1] == nil || got[2] != nil || got[3] != nil || got[4] != nil || got[5] == nil || got[6] == nil || got[7] == nil || got[8] == nil { + t.Fatalf("Pi artifact policy results = %#v", got) + } +} diff --git a/internal/stage/resource_cleanup_test.go b/internal/stage/resource_cleanup_test.go new file mode 100644 index 00000000..c57d4080 --- /dev/null +++ b/internal/stage/resource_cleanup_test.go @@ -0,0 +1,44 @@ +package stage + +import ( + "context" + "encoding/json" + "errors" + "os" + "os/exec" + "path/filepath" + "syscall" + "testing" + "time" +) + +func TestResourcePiReceiptRequiresProcessExit(t *testing.T) { + // A test-owned process only: no Pi provider, shell/ADB command or phone. + cmd := exec.Command("sleep", "5") + cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true} + if err := cmd.Start(); err != nil { + t.Fatal(err) + } + defer func() { cmd.Process.Kill(); cmd.Wait() }() + dir := t.TempDir() + owner, _ := json.Marshal(map[string]any{"pid": cmd.Process.Pid}) + for name, data := range map[string][]byte{"owner.json": owner, "lease": []byte("lease"), "quiescent": []byte("quiescent")} { + if err := os.WriteFile(filepath.Join(dir, name), data, 0600); err != nil { + t.Fatal(err) + } + } + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Millisecond) + defer cancel() + if err := InterruptPi(ctx, dir); !errors.Is(err, ErrStageCleanup) { + t.Fatal("receipt authorized handoff while process lived", err) + } + if err := cmd.Process.Kill(); err != nil { + t.Fatal(err) + } + cmd.Wait() + ctx2, cancel2 := context.WithTimeout(context.Background(), time.Second) + defer cancel2() + if err := InterruptPi(ctx2, dir); err != nil { + t.Fatal("confirmed process exit not accepted", err) + } +} diff --git a/internal/stage/retry_safety_test.go b/internal/stage/retry_safety_test.go index f5f812e9..f0a40541 100644 --- a/internal/stage/retry_safety_test.go +++ b/internal/stage/retry_safety_test.go @@ -251,7 +251,7 @@ func TestWaitForSessionIdleRecordsOneTimeoutEvidence(t *testing.T) { t.Fatalf("expected exactly one timeout evidence, got %d", len(log.Failures)) } entry := log.Failures[0] - if entry.Operation != "wait for session idle" || !strings.Contains(entry.Diagnostic, "deadline reached") { + if entry.Operation != "wait for session idle" || !strings.Contains(entry.Diagnostic, "context deadline exceeded") { t.Fatalf("timeout evidence not sanitized as expected: %+v", entry) } clean := &OperationLog{} diff --git a/internal/stage/stage_deadline_test.go b/internal/stage/stage_deadline_test.go new file mode 100644 index 00000000..2f2f3aea --- /dev/null +++ b/internal/stage/stage_deadline_test.go @@ -0,0 +1,140 @@ +package stage + +import ( + "context" + "errors" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "git.a-wan.net/alan/pipelines/internal/opencode" +) + +func TestConfiguredStageDeadlineWhileSessionActive(t *testing.T) { + def := modelDefinition() + def.Blocks["tools"] = "edit" + def.Configs["timeout"] = "20ms" + def.Configs["retries"] = "2" + log := &OperationLog{} + creates, prompts, interrupts := 0, 0, 0 + agent := &AgentClient{OpenCodeClient: OpenCodeClient{Backend: "v2", Operations: log, sessionPollInterval: time.Hour}} + agent.command = func(_ context.Context, _ string, args ...string) ([]byte, error) { + switch args[2] { + case "/api/session": + creates++ + return []byte(`{"data":{"id":"ses_deadline"}}`), nil + case "/api/session/ses_deadline/prompt": + prompts++ + return []byte(`{}`), nil + case "/api/session/active": + return []byte(`{"data":{"ses_deadline":{"type":"running"}}}`), nil + case "/api/session/ses_deadline/interrupt": + interrupts++ + return []byte(`{}`), nil + default: + // Best-effort timeout readback/usage cannot convert the failure to success. + return []byte(`{}`), nil + } + } + result, err := (ModelStage{Client: agent}).Run(context.Background(), def, nil) + if !errors.Is(err, context.DeadlineExceeded) || modelFailureCategory(err) != stageDeadline || errors.Is(err, ErrHarnessTransport) { + t.Fatalf("deadline identity/category: %v", err) + } + for _, want := range []string{"STAGE DEADLINE", "worker", "timeout 20ms", "waiting for session idle", "ses_deadline", "stage had not finished"} { + if !strings.Contains(err.Error(), want) { + t.Fatalf("missing %q: %v", want, err) + } + } + if result.Output != nil || creates != 1 || prompts != 1 || interrupts != 1 || !agent.WorkStarted { + t.Fatalf("work replayed, cleanup skipped or incomplete turn accepted: %+v creates=%d prompts=%d interrupts=%d", result, creates, prompts, interrupts) + } + if len(log.Failures) != 1 || !strings.Contains(log.Failures[0].Diagnostic, "timeout 20ms") { + t.Fatalf("deadline evidence missing: %+v", log.Failures) + } +} + +func TestWaitCancellationIsNotConfiguredStageDeadline(t *testing.T) { + for _, cause := range []error{context.Canceled, ErrBudgetExceeded, ErrBudgetAccounting} { + t.Run(cause.Error(), func(t *testing.T) { + attempt, stop := context.WithTimeoutCause(context.Background(), time.Hour, &stageTimeoutCause{timeout: time.Hour}) + defer stop() + ctx, cancel := context.WithCancelCause(attempt) + log := &OperationLog{} + client := OpenCodeClient{Operations: log, command: func(context.Context, string, ...string) ([]byte, error) { + cancel(cause) + return []byte(`{"data":{"ses_wait":{"type":"running"}}}`), nil + }} + err := client.waitForSessionIdle(ctx, "ses_wait") + if !errors.Is(err, context.Canceled) || modelFailureCategory(err) != "" || len(log.Failures) != 1 { + t.Fatalf("cancellation misclassified: %v, %+v", err, log.Failures) + } + if strings.Contains(log.Failures[0].Diagnostic, "deadline") { + t.Fatalf("cancellation recorded as deadline: %+v", log.Failures) + } + }) + } +} + +func TestDeadlineWithTransportRetainsUnderlyingIdentity(t *testing.T) { + transport := &openCodeError{operation: "get /api/session/active", status: 503, cause: errors.New("request failed")} + err := wrapModelStageFailure("worker", errors.Join(&stageDeadlineError{timeout: time.Second, session: "ses_wait"}, transport)) + var got *openCodeError + if modelFailureCategory(err) != harnessTransport || !errors.Is(err, context.DeadlineExceeded) || !errors.As(err, &got) || got != transport || !errors.Is(err, transport.cause) { + t.Fatalf("combined failure lost transport/deadline identity: %v", err) + } + if !strings.Contains(err.Error(), "HTTP 503") || !strings.Contains(err.Error(), "get /api/session/active") { + t.Fatalf("combined failure lost operation/status: %v", err) + } + bounded := (&stageDeadlineError{timeout: time.Hour, session: strings.Repeat("s", 10000) + " Bearer private-token"}).Error() + if len(bounded) > 512 || strings.Contains(bounded, "private-token") { + t.Fatal("deadline diagnostic was not bounded/redacted") + } +} + +func TestParentDeadlinePreservesModelTransportFailure(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Millisecond) + defer cancel() + transport := &openCodeError{operation: "get /api/session/active", status: 504, cause: errors.New("request failed")} + client := generatorFunc(func(ctx context.Context, _ ModelConfig, _ string) (string, error) { + <-ctx.Done() + return "", errors.Join(ctx.Err(), transport) + }) + _, err := (ModelStage{Client: client}).Run(ctx, modelDefinition(), nil) + var got *openCodeError + if !errors.Is(err, context.DeadlineExceeded) || !errors.As(err, &got) || got != transport || modelFailureCategory(err) != harnessTransport { + t.Fatalf("parent deadline erased underlying request failure: %v", err) + } +} + +func TestNativeHTTPAndConnectionFailuresRemainTransport(t *testing.T) { + for _, mode := range []string{"http", "connection"} { + t.Run(mode, func(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusServiceUnavailable) + _, _ = w.Write([]byte(`{"message":"service unavailable","token":"private-token"}`)) + })) + defer srv.Close() + if mode == "connection" { + srv.Close() + } + log := &OperationLog{} + client := OpenCodeClient{Backend: "v1", Operations: log, connection: &opencode.Connection{Backend: "v1", URL: srv.URL}} + _, err := client.call(context.Background(), "get", "/api/session", nil) + var typed *openCodeError + if !errors.As(err, &typed) || modelFailureCategory(err) != harnessTransport || len(log.Failures) != 1 { + t.Fatalf("real request failure misclassified: %v %+v", err, log.Failures) + } + if mode == "http" && (typed.status != 503 || log.Failures[0].HTTPStatus != 503 || !strings.Contains(log.Failures[0].Stdout, "service unavailable")) { + t.Fatalf("HTTP status/evidence lost: %v %+v", err, log.Failures) + } + if mode == "connection" && (log.Failures[0].Diagnostic != "OpenCode native API request failed" || log.Failures[0].NativeOperation != "GET /session") { + t.Fatalf("connection evidence lost: %+v", log.Failures) + } + if strings.Contains(err.Error()+log.Transcript(""), "private-token") { + t.Fatal("credential leaked in transport diagnostic/evidence") + } + }) + } +} diff --git a/internal/stage/tools.go b/internal/stage/tools.go index 566519cb..2f567e3b 100644 --- a/internal/stage/tools.go +++ b/internal/stage/tools.go @@ -7,6 +7,84 @@ import ( "strings" ) +// ArtifactAccess is runtime-only, server-derived access to artifact visits. +// Empty fields deliberately grant nothing outside the workspace. +type ArtifactAccess struct { + Write string + Read []string +} + +// LandingAccess is the server-derived external capability for a verified +// delivery operation. Empty or malformed values grant nothing. +type LandingAccess struct { + MainWorktree string + GitMetadata string + LockPath string +} + +func (a LandingAccess) valid() bool { + for _, path := range []string{a.MainWorktree, a.GitMetadata, a.LockPath} { + if path == "" || !filepath.IsAbs(path) || filepath.Clean(path) != path || strings.ContainsAny(path, "*?\\") { + return false + } + } + // The server grants the canonical repository metadata directory as OpenCode + // V2's smallest external-directory unit. Keep the lock inside that directory + // so a caller cannot turn a valid metadata capability into access elsewhere. + return a.GitMetadata == filepath.Join(a.MainWorktree, ".git") && + a.LockPath == filepath.Join(a.GitMetadata, "pipelines-landing.lock") +} + +// DeliveryEnabled is the narrow, declarative capability check used before the +// runner supplies any landing paths. Only a validated model definition can ask +// for it; runtime paths are still derived by the server. +func DeliveryEnabled(def Definition) (bool, error) { + delivery := def.Configs["delivery"] + if delivery == "" { + return false, nil + } + if delivery != "true" { + return false, fmt.Errorf("delivery must be true when set") + } + if EffectiveType(def.Type) != ModelType { + return false, fmt.Errorf("delivery is supported only by model stages") + } + if _, err := (ModelStage{}).Spec(def); err != nil { + return false, fmt.Errorf("invalid delivery model stage: %w", err) + } + if _, err := DeliveryChecks(def); err != nil { + return false, err + } + return true, nil +} + +// DeliveryChecks returns the bounded, definition-declared checks that the +// runner executes against a committed delivery candidate before canonical main +// is changed. They +// are intentionally a block rather than a generic runner setting: each +// workflow owns its repository-specific verification commands. +func DeliveryChecks(def Definition) ([]string, error) { + text := strings.TrimSpace(def.Blocks["checks"]) + if text == "" { + return nil, fmt.Errorf("delivery stages require a nonempty checks block") + } + var checks []string + for _, line := range strings.Split(text, "\n") { + check := strings.TrimSpace(line) + if check == "" { + continue + } + if len(check) > 4096 { + return nil, fmt.Errorf("delivery check exceeds 4096 bytes") + } + checks = append(checks, check) + } + if len(checks) == 0 || len(checks) > 20 { + return nil, fmt.Errorf("delivery stages require 1 to 20 checks") + } + return checks, nil +} + // ToolNames validates model-stage permissions, not general graph semantics. // A present empty block explicitly denies all tools. func ToolNames(def Definition) ([]string, bool, error) { @@ -128,12 +206,43 @@ func (c *AgentClient) permissionRules() []map[string]string { } } } + // OpenCode V2 resolves an external file to its containing directory and + // checks external_directory against /*. Do not grant a parent: + // each visit is deliberately an independent capability. + readArtifacts := false + writeArtifacts := false + for _, tool := range tools { + readArtifacts = readArtifacts || tool == "read" || tool == "glob" || tool == "grep" + writeArtifacts = writeArtifacts || tool == "edit" + } + if readArtifacts { + for _, directory := range c.ArtifactAccess.Read { + if artifactDirectory(directory) { + allow("external_directory", filepath.ToSlash(filepath.Join(directory, "*"))) + } + } + } + if writeArtifacts && artifactDirectory(c.ArtifactAccess.Write) { + allow("external_directory", filepath.ToSlash(filepath.Join(c.ArtifactAccess.Write, "*"))) + allow("edit", filepath.ToSlash(filepath.Join(c.ArtifactAccess.Write, "*"))) + } + if c.LandingAccess.valid() { + allow("external_directory", filepath.ToSlash(filepath.Join(c.LandingAccess.MainWorktree, "*"))) + allow("external_directory", filepath.ToSlash(filepath.Join(c.LandingAccess.GitMetadata, "*"))) + } + for _, pattern := range []string{"../*", "*/../*"} { + rules = append(rules, map[string]string{"action": "external_directory", "resource": pattern, "effect": "deny"}) + } for _, pattern := range []string{"*.env", "*.env.*", "pipelines.db*"} { rules = append(rules, map[string]string{"action": "read", "resource": pattern, "effect": "deny"}) } return rules } +func artifactDirectory(directory string) bool { + return directory != "" && filepath.IsAbs(directory) && filepath.Clean(directory) == directory && !strings.ContainsAny(directory, "*?\\") +} + func matchesEditPolicy(patterns []string, path string) bool { for _, pattern := range patterns { // OpenCode resource wildcards match across directory separators. diff --git a/internal/stage/tools_test.go b/internal/stage/tools_test.go index fa561166..e7a20c0f 100644 --- a/internal/stage/tools_test.go +++ b/internal/stage/tools_test.go @@ -139,3 +139,142 @@ func TestDefaultEditScopeIsLanguageIndependentRunDirectory(t *testing.T) { t.Fatal("external directories were enabled") } } + +func TestArtifactPermissionsAreExactAndToolScoped(t *testing.T) { + current := "/artifacts/run [7]/2" + prior := "/artifacts/run [7]/1" + sibling := "/artifacts/run 7/3" + otherRun := "/artifacts/run 8/1" + client := &AgentClient{OpenCodeClient: OpenCodeClient{Directory: "/repo"}, Tools: []string{"read", "edit"}, ArtifactAccess: ArtifactAccess{Write: current, Read: []string{current, prior}}} + rules := client.permissionRules() + for _, path := range []string{current + "/report.md", prior + "/report.md"} { + if permissionEffect(rules, "external_directory", path) != "allow" { + t.Fatalf("artifact directory denied: %q", path) + } + } + if permissionEffect(rules, "edit", current+"/report.md") != "allow" || permissionEffect(rules, "edit", prior+"/report.md") != "deny" { + t.Fatal("artifact writes were not limited to the current visit") + } + for _, path := range []string{sibling + "/report.md", otherRun + "/report.md", current + "/../3/report.md", "/artifacts/run 7/20/report.md"} { + if permissionEffect(rules, "external_directory", path) != "deny" { + t.Fatalf("unapproved artifact scope allowed: %q", path) + } + } + client.Tools = []string{} + if permissionEffect(client.permissionRules(), "external_directory", current+"/report.md") != "deny" { + t.Fatal("no-tool stage gained artifact access") + } + client.Tools = []string{"read"} + rules = client.permissionRules() + if permissionEffect(rules, "external_directory", prior+"/report.md") != "allow" || permissionEffect(rules, "edit", current+"/report.md") != "deny" { + t.Fatal("read-only artifact stage gained write access") + } + client.ArtifactAccess = ArtifactAccess{} + if permissionEffect(client.permissionRules(), "external_directory", current+"/report.md") != "deny" { + t.Fatal("missing artifact scope enabled an external directory") + } +} + +func TestLandingPermissionsAreExactAndServerDerived(t *testing.T) { + access := LandingAccess{MainWorktree: "/repos/pipelines", GitMetadata: "/repos/pipelines/.git", LockPath: "/repos/pipelines/.git/pipelines-landing.lock"} + client := &AgentClient{Tools: []string{"read", "shell"}, LandingAccess: access} + rules := client.permissionRules() + for _, path := range []string{"/repos/pipelines/go.mod", "/repos/pipelines/.git/worktrees/item/HEAD", "/repos/pipelines/.git/pipelines-landing.lock"} { + if permissionEffect(rules, "external_directory", path) != "allow" { + t.Fatalf("verified landing path denied: %q", path) + } + } + for _, path := range []string{"/repos/other/go.mod", "/repos/pipelines/../other/go.mod", "/Users/alan/.git/config", "/.git/config"} { + if permissionEffect(rules, "external_directory", path) != "deny" { + t.Fatalf("unverified landing path allowed: %q", path) + } + } + client.LandingAccess.MainWorktree = "/repos/pipelines/../other" + if permissionEffect(client.permissionRules(), "external_directory", "/repos/other/go.mod") != "deny" { + t.Fatal("non-canonical landing capability allowed") + } + client.LandingAccess = LandingAccess{MainWorktree: "/repos/pipelines", GitMetadata: "/repos/pipelines/.git", LockPath: "/tmp/pipelines-landing.lock"} + if permissionEffect(client.permissionRules(), "external_directory", "/tmp/pipelines-landing.lock") != "deny" { + t.Fatal("landing lock outside verified metadata allowed") + } + client.LandingAccess = LandingAccess{MainWorktree: "/repos/pipelines", GitMetadata: "/repos/other/.git", LockPath: "/repos/other/.git/pipelines-landing.lock"} + if permissionEffect(client.permissionRules(), "external_directory", "/repos/other/.git/config") != "deny" { + t.Fatal("unrelated Git metadata allowed") + } +} + +func TestDeliveryEnabledRequiresValidatedModelStage(t *testing.T) { + def := modelDefinition() + def.Configs["delivery"] = "true" + def.Blocks["checks"] = "go test ./..." + if enabled, err := DeliveryEnabled(def); err != nil || !enabled { + t.Fatalf("valid delivery stage = %v, %v", enabled, err) + } + for name, change := range map[string]func(*Definition){ + "malformed value": func(d *Definition) { d.Configs["delivery"] = "yes" }, + "non-model": func(d *Definition) { d.Type = HumanType }, + "invalid model": func(d *Definition) { d.Configs["unexpected"] = "value" }, + } { + t.Run(name, func(t *testing.T) { + copy := def + copy.Configs = map[string]string{} + for key, value := range def.Configs { + copy.Configs[key] = value + } + change(©) + if enabled, err := DeliveryEnabled(copy); err == nil || enabled { + t.Fatalf("invalid delivery stage = %v, %v", enabled, err) + } + }) + } +} + +func TestDeliveryRejectsPiBeforeAgentExecution(t *testing.T) { + def := modelDefinition() + def.Configs["delivery"] = "true" + def.Configs["harness"] = "pi" + def.Blocks["checks"] = "true" + if err := (ModelStage{}).Validate(def); err == nil || !strings.Contains(err.Error(), "opencode harness") { + t.Fatal("Pi delivery was accepted", err) + } + delete(def.Configs, "delivery") + delete(def.Blocks, "checks") + if err := (ModelStage{}).Validate(def); err != nil { + t.Fatal("Pi non-delivery was changed", err) + } +} + +func TestDeliveryChecksRequireBoundedDeliveryOnlyDeclaration(t *testing.T) { + def := modelDefinition() + def.Configs["delivery"] = "true" + for name, mutate := range map[string]func(*Definition){ + "missing": func(d *Definition) {}, + "empty": func(d *Definition) { d.Blocks["checks"] = "\n" }, + "non delivery": func(d *Definition) { d.Blocks["checks"] = "go test ./..." }, + "too many checks": func(d *Definition) { d.Blocks["checks"] = strings.Repeat("true\n", 21) }, + "oversized check": func(d *Definition) { d.Blocks["checks"] = strings.Repeat("x", 4097) }, + } { + t.Run(name, func(t *testing.T) { + copy := def + copy.Blocks = map[string]string{} + copy.Configs = map[string]string{} + for key, value := range def.Blocks { + copy.Blocks[key] = value + } + for key, value := range def.Configs { + copy.Configs[key] = value + } + mutate(©) + if name == "non delivery" { + delete(copy.Configs, "delivery") + if _, err := (ModelStage{}).Spec(copy); err == nil { + t.Fatal("non-delivery checks block was accepted") + } + return + } + if _, err := DeliveryEnabled(copy); err == nil { + t.Fatal("invalid delivery checks were accepted") + } + }) + } +} diff --git a/internal/stage/verification.go b/internal/stage/verification.go new file mode 100644 index 00000000..a186aa15 --- /dev/null +++ b/internal/stage/verification.go @@ -0,0 +1,97 @@ +package stage + +import ( + "context" + "fmt" + "strings" +) + +// VerificationTarget declares the controlled fixture that can substantiate a +// target-specific behavior claim. It is intentionally data-only: the runner +// does not execute fixture input supplied by a stage result. +type VerificationTarget struct { + Backend string + Revision string + Capability string + Fixture string +} + +// VerificationEvidence is returned by a server-controlled fixture workflow, +// never parsed from a model's result. +type VerificationEvidence struct { + Backend string + Revision string + Capability string + Fixture string +} + +// VerificationRunner is the narrow server seam for a controlled fixture or +// workflow. The runner selects it during server setup; model output cannot +// select a backend, revision, fixture, or executable. +type VerificationRunner func(context.Context, VerificationTarget) (VerificationEvidence, error) + +// VerificationTargetFor parses an optional, bounded verification block. A +// declared target requires all provenance fields so an old or other-backend +// fixture cannot be presented as evidence for the current target. +func VerificationTargetFor(def Definition) (*VerificationTarget, error) { + text, declared := def.Blocks["verification"] + if !declared { + return nil, nil + } + if len(text) > 4096 { + return nil, fmt.Errorf("verification block exceeds 4096 bytes") + } + values := map[string]string{} + for _, line := range strings.Split(text, "\n") { + line = strings.TrimSpace(line) + if line == "" { + continue + } + key, value, ok := strings.Cut(line, "=") + key, value = strings.TrimSpace(key), strings.TrimSpace(value) + if !ok || value == "" || len(value) > 256 { + return nil, fmt.Errorf("invalid verification entry %q", line) + } + switch key { + case "backend", "revision", "capability", "fixture": + default: + return nil, fmt.Errorf("unknown verification entry %q", key) + } + if _, exists := values[key]; exists { + return nil, fmt.Errorf("duplicate verification entry %q", key) + } + values[key] = value + } + target := &VerificationTarget{Backend: values["backend"], Revision: values["revision"], Capability: values["capability"], Fixture: values["fixture"]} + if target.Backend == "" || target.Revision == "" || target.Capability == "" || target.Fixture == "" { + return nil, fmt.Errorf("verification requires backend, revision, capability, and fixture") + } + return target, nil +} + +// VerifyTargetEvidence compares server-collected controlled-fixture provenance +// with the workflow declaration. It deliberately accepts no model result: a +// stage cannot substantiate its own target-specific behavior claim. +func VerifyTargetEvidence(target VerificationTarget, evidence VerificationEvidence) error { + for key, want := range map[string]string{ + "backend": target.Backend, "revision": target.Revision, "capability": target.Capability, "fixture": target.Fixture, + } { + got := map[string]string{ + "backend": evidence.Backend, "revision": evidence.Revision, "capability": evidence.Capability, "fixture": evidence.Fixture, + }[key] + if got != want { + return fmt.Errorf("verification evidence %s does not match declared target", key) + } + } + return nil +} + +// VerifyTargetResult is retained for callers that only need declaration +// validation. Result-provided provenance is intentionally not evidence. +func VerifyTargetResult(def Definition, _ Values) error { + target, err := VerificationTargetFor(def) + if err != nil || target == nil { + return err + } + return fmt.Errorf("verification target %q requires a server-controlled fixture runner", target.Fixture) +} diff --git a/internal/stage/verification_test.go b/internal/stage/verification_test.go new file mode 100644 index 00000000..d52d5b5a --- /dev/null +++ b/internal/stage/verification_test.go @@ -0,0 +1,27 @@ +package stage + +import "testing" + +func TestTargetVerificationRejectsWrongBackendEvidence(t *testing.T) { + def := modelDefinition() + def.Blocks["verification"] = "backend=branch-api\nrevision=abc123\ncapability=controlled-state\nfixture=reconnect-foreground" + if err := (ModelStage{}).Validate(def); err != nil { + t.Fatal(err) + } + target, err := VerificationTargetFor(def) + if err != nil { + t.Fatal(err) + } + good := VerificationEvidence{Backend: "branch-api", Revision: "abc123", Capability: "controlled-state", Fixture: "reconnect-foreground"} + if err := VerifyTargetEvidence(*target, good); err != nil { + t.Fatalf("matching fixture evidence rejected: %v", err) + } + wrongBackend := VerificationEvidence{Backend: "old-api", Revision: "abc123", Capability: "controlled-state", Fixture: "reconnect-foreground"} + if err := VerifyTargetEvidence(*target, wrongBackend); err == nil { + t.Fatal("wrong-backend fixture evidence accepted") + } + def.Blocks["verification"] = "backend=branch-api\nrevision=abc123" + if err := (ModelStage{}).Validate(def); err == nil { + t.Fatal("incomplete target verification declaration accepted") + } +} diff --git a/internal/tui/resources_test.go b/internal/tui/resources_test.go new file mode 100644 index 00000000..a2e34475 --- /dev/null +++ b/internal/tui/resources_test.go @@ -0,0 +1,43 @@ +package tui + +import ( + "fmt" + "strings" + "testing" + + "git.a-wan.net/alan/pipelines/internal/protocol" + "github.com/charmbracelet/lipgloss" + "github.com/charmbracelet/x/ansi" + "github.com/muesli/termenv" +) + +func TestResourceWaitFrames(t *testing.T) { + previous := lipgloss.ColorProfile() + defer lipgloss.SetColorProfile(previous) + for _, width := range []int{60, 120} { + for _, plain := range []bool{false, true} { + t.Run(fmt.Sprintf("width-%d-plain-%t", width, plain), func(t *testing.T) { + if plain { + lipgloss.SetColorProfile(termenv.Ascii) + } else { + lipgloss.SetColorProfile(termenv.TrueColor) + } + m := decisionModel() + m.width = width + m.runDetail.Status = protocol.RunStatusQueued + m.runDetail.Steps = nil + m.runDetail.ResourceWait = &protocol.ResourceWait{Resource: "phone-X", Stage: "review", Sequence: 1, QueuedSince: "2026-10-18T10:00:00Z"} + frame := ansi.Strip(m.runsDetailsView(24)) + if !strings.Contains(frame, "Waiting for resource phone-X") || strings.Contains(frame, "c decide") || strings.Contains(frame, "Waiting for a person") { + t.Fatal("wrong resource wait frame:\n", frame) + } + for _, line := range strings.Split(frame, "\n") { + if ansi.StringWidth(line) > width { + t.Fatalf("overflow: %q", line) + } + } + t.Log("Rendered resource wait (no phone):\n" + frame) + }) + } + } +} diff --git a/internal/tui/results.go b/internal/tui/results.go index a4c1b610..3ef6840a 100644 --- a/internal/tui/results.go +++ b/internal/tui/results.go @@ -11,6 +11,9 @@ import ( ) func resultLabel(r protocol.RunSummary) string { + if r.ResourceWait != nil { + return r.StatusLabel() + " · stage " + r.ResourceWait.Stage + } preview := ansi.Strip(r.ResultPreview) if r.Status == "succeeded" { if preview != "" { @@ -55,7 +58,7 @@ func (m Model) pipelineResultLines() []string { if r == nil { return []string{"Results · This version has not run yet. No accomplishments recorded."} } - lines := []string{fmt.Sprintf("Results · run #%d · %s · %s · %d completed visits · %s", r.ID, r.Label(), ansi.Strip(m.progress(r.Status)), r.CompletedSteps, runDuration(r.CreatedAt, r.FinishedAt)), resultLabel(*r)} + lines := []string{fmt.Sprintf("Results · run #%d · %s · %s · %d completed visits · %s", r.ID, r.Label(), ansi.Strip(m.runSummaryStatus(*r)), r.CompletedSteps, runDuration(r.CreatedAt, r.FinishedAt)), resultLabel(*r)} if r.Status != "succeeded" && r.ResultPreview != "" && r.Diagnostic != "" { lines = append(lines, "Partial output: "+ansi.Strip(r.ResultPreview)) } diff --git a/internal/tui/run_detail.go b/internal/tui/run_detail.go index 79154144..87c4c04a 100644 --- a/internal/tui/run_detail.go +++ b/internal/tui/run_detail.go @@ -275,6 +275,13 @@ func runStatus(status string) string { return lipgloss.NewStyle().Foreground(color).Render(status) } +func (m Model) runSummaryStatus(run protocol.RunSummary) string { + if run.ResourceWait != nil { + return run.StatusLabel() + } + return m.progress(run.Status) +} + func runDuration(start, end string) string { first, err := time.Parse(time.RFC3339, start) if err != nil { @@ -687,7 +694,7 @@ func (m Model) runsDetailsView(height int) string { start, end := selectionWindow(len(runs), m.runIndex, max(1, (height-4-len(rows))/2)) for i := start; i < end; i++ { r := runs[i] - text := fmt.Sprintf("#%d %s · v%d · %s · %s", r.ID, r.Label(), r.Version, m.progress(r.Status), dateLabel(r.FinishedAt)) + text := fmt.Sprintf("#%d %s · v%d · %s · %s", r.ID, r.Label(), r.Version, m.runSummaryStatus(r), dateLabel(r.FinishedAt)) rows = append(rows, menuOption(text, i == m.runIndex)) rows = append(rows, " "+ansi.Truncate(resultLabel(r), max(1, m.width-6), "…")) } @@ -710,7 +717,8 @@ func (m Model) runsDetailsView(height int) string { return panel(m.width, height, []string{bar(title, fmt.Sprint(len(runs)), max(1, m.width-4), true)}, rows, footer, true) } r := m.runDetail - header := bar(fmt.Sprintf("#%d %s · v%d", r.ID, r.Label(), r.Version), m.progress(r.Status)+" · "+runDuration(r.CreatedAt, r.FinishedAt), m.width, false) + status := m.runSummaryStatus(r.RunSummary) + header := bar(fmt.Sprintf("#%d %s · v%d", r.ID, r.Label(), r.Version), status+" · "+runDuration(r.CreatedAt, r.FinishedAt), m.width, false) if height <= 6 { return fitBlock([]string{header, runHighlight(m.runSelectionLabel(), m.width), m.runActions() + " · Tab pane · Esc back"}, m.width, height) }