Merge main into pill-run-decision for Android recovery

This commit is contained in:
2026-10-18 15:06:52 +01:00
85 changed files with 6293 additions and 178 deletions
+26 -4
View File
@@ -20,16 +20,37 @@ time while iterating — the full suite is for the end.
## The dev device
Only the **UI reviewer**, while its stage holds a declared runner-exclusive resource
for the selected phone, may use the device. Implementers and code reviewers do code,
builds, unit tests and source review only: **no adb, installs, captures, device-setting
changes or main-build restoration**. Never acquire a phone using sleep/check loops.
Do not use the phone until compatible resource scheduling is deployed and every device
user has opted in or has been reconciled by the operator. Existing pinned runs are not
retroactively protected. The resource for the device below is `phone-47241FDAS00DLK`.
The only device is the operator's **Pixel 9 Pro XL, serial `47241FDAS00DLK`**. The operator's
Pixel 10 Pro Fold (`58281FDCG000JF`) is theirs: use it only where an item's brief explicitly allows
it, and never make it a default. **Never start or use an emulator.**
Before any capture: `svc power stayon usb`, `settings put global stay_on_while_plugged_in 7`,
The UI reviewer records original app connection/workspace, display size and power/settings
before changing them. Before any capture: `svc power stayon usb`, `settings put global stay_on_while_plugged_in 7`,
`input keyevent KEYCODE_WAKEUP`, `wm dismiss-keyguard`. Capture with
`adb -s <serial> shell screencap -p /sdcard/shot.png` then `adb pull` — never `exec-out > file`.
Wide layout: `wm size 2000x2244`; the fold's unfolded shape is `wm size 2076x2152`; `wm size reset`
afterwards either way.
Create controlled fixtures for changed screens/states (including human-decision sheets)
on a scratch server/database and throwaway session; never depend on incidental production
state, change a real operator session, or restart a deployed service. Use the configured
authoritative endpoint, never a scratch URL as a project default. Keep all shell/ADB work
foreground and awaited: no detached/background processes may outlive the lease.
UI review includes restoring the captured original settings and main's build before its
successful return. Confirm every tool has exited. If capture, restoration or cleanup fails,
report the failure explicitly; do not claim verification or readiness. Uncertain termination
must fail closed and keep ownership quarantined, not delete a lock or expire it by age.
Return `output.resource_cleanup_confirmed=false` if restoration/readiness is uncertain;
that generic release veto retains the runner lease even if the harness reports idle.
## Geometry is tokenised, and a test enforces it
Every dimension comes from `ui/theme/Dimens.kt` (plus the layout itself) — no new hardcoded `dp`, no
@@ -53,13 +74,14 @@ genuinely needs a new token goes there, named for its purpose.
## Publishing a build
Copy the APK into the running run's artifacts directory as `pipelines-debug-wip-<short-HEAD>.apk` and
report the LAN URL. After any install, leave the operator's phone on **main's** build:
report the LAN URL. The UI reviewer alone, under the same lease, restores **main's** build after an install:
`/private/var/folders/jp/q0qvjyg54fl_c5vhhk1ylz2m0000gn/T/opencode/deploy-app.sh main`.
## Traps
- A build made in a worktree that is behind main is missing features main has — sync before building,
and say which revision you built.
- Changing the app's server URL for a test (a stub) must be undone: `ws://10.0.0.75:8080/connect`,
workspace `/Users/alan/repos/pipelines`, then force-stop and relaunch.
- Restore the app's captured original connection and workspace after a fixture test,
using the configured authoritative endpoint; then force-stop and relaunch under the lease.
Never substitute a hardcoded or disposable scratch endpoint for the original configuration.
- `mStayOn`/screen timeout are already set; do not "fix" the device by locking or sleeping it.
+1
View File
@@ -58,6 +58,7 @@ dependencies {
implementation(libs.androidx.compose.material.icons.extended)
implementation(libs.androidx.navigation.compose)
implementation(libs.androidx.lifecycle.runtime.compose)
implementation(libs.androidx.lifecycle.process)
implementation(libs.androidx.lifecycle.viewmodel.compose)
implementation(libs.androidx.lifecycle.viewmodel.ktx)
implementation(libs.kotlinx.coroutines.android)
@@ -1,12 +1,15 @@
package net.awan.pipelines
import android.app.Application
import androidx.lifecycle.ProcessLifecycleOwner
import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.SupervisorJob
import net.awan.pipelines.data.ForegroundReconnector
import net.awan.pipelines.data.PipelinesRepository
import net.awan.pipelines.data.SettingsRepository
import net.awan.pipelines.data.transport.PipelinesClient
import net.awan.pipelines.data.transport.ServerUrl
import net.awan.pipelines.notify.RunNotifier
/**
@@ -30,8 +33,39 @@ class PipelinesApplication : Application() {
*/
val runNotifier: RunNotifier by lazy { RunNotifier(this) }
/**
* The one owner of return-to-foreground recovery: when the process comes back
* to the front it restores a connection the background took away, with no
* interaction. It is created once with the Application and observes the
* process lifecycle, so every Activity and ViewModel recreation leaves it
* untouched. The liveness check is the existing `ping`; the reconnect is the
* same shared-client connect the app's own Connect/Reconnect uses, so the
* policy lives only in `foregroundAction` and the client's backoff stays the
* only reconnect mechanism.
*/
private val foregroundReconnector: ForegroundReconnector by lazy {
ForegroundReconnector(
state = { repository.connectionState.value },
confirmAlive = {
// The existing ping is the positive check: on a dead socket it
// throws (or never answers) instead of returning, so reaching the
// next line is the proof of life.
repository.ping()
true
},
reconnect = {
// A saved URL that no longer validates is left for Settings to
// report; there is nothing to reconnect to.
val validated = ServerUrl.validate(settings.serverUrl.value)
if (validated.valid) repository.recover(validated)
},
scope = applicationScope,
)
}
override fun onCreate() {
super.onCreate()
runNotifier.ensureChannel()
ProcessLifecycleOwner.get().lifecycle.addObserver(foregroundReconnector)
}
}
@@ -0,0 +1,81 @@
package net.awan.pipelines.data
import androidx.lifecycle.DefaultLifecycleObserver
import androidx.lifecycle.LifecycleOwner
import kotlinx.coroutines.CancellationException
import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.launch
import kotlinx.coroutines.withTimeoutOrNull
import net.awan.pipelines.data.transport.ConnectionState
import net.awan.pipelines.data.transport.ForegroundAction
import net.awan.pipelines.data.transport.foregroundAction
/**
* The one lifecycle-aware owner of the app's return-to-foreground recovery.
*
* While another app is in front Android freezes — and often kills — this process,
* so the socket it holds dies or goes quiet and the client's own bounded backoff
* either never starts or starts too late for the operator, who comes back and
* taps Connect. The client's backoff (see `PipelinesClient.scheduleReconnect`) is
* still the only reconnect mechanism; this owner only decides *when* to drive it,
* and does so the moment the app becomes visible again.
*
* It observes the whole process lifecycle (a `ProcessLifecycleOwner`), never the
* Activity or the ViewModel, both of which Android recreates: the owner is
* created once with the Application and survives every recreation. The decision
* itself is the one pure [foregroundAction] policy; this class supplies only the
* current state, a positive liveness check and the reconnect effect, so the
* policy is unit-testable with no device, socket or framework.
*/
class ForegroundReconnector(
private val state: () -> ConnectionState,
private val confirmAlive: suspend () -> Boolean,
private val reconnect: () -> Unit,
private val scope: CoroutineScope,
private val confirmTimeoutMillis: Long = CONFIRM_TIMEOUT_MILLIS,
) : DefaultLifecycleObserver {
/**
* The app became visible again. A state the policy calls reconnecting is
* reconnected at once — the pending backoff delay is cancelled by the
* reconnect effect, so the operator never waits it out. A socket the state
* still calls Connected is first confirmed to answer, so a silent
* "Connected" is never trusted; the state is re-read afterwards, because the
* socket's own failure callback may have run while the check was in flight.
*/
override fun onStart(owner: LifecycleOwner) {
val before = state()
if (before !is ConnectionState.Connected) {
if (foregroundAction(before, confirmedAlive = false) == ForegroundAction.Reconnect) reconnect()
return
}
scope.launch {
val alive = confirm()
if (foregroundAction(state(), alive) == ForegroundAction.Reconnect) reconnect()
}
}
/**
* Confirms a supposedly-connected socket still answers, bounded so a dead one
* cannot hold recovery: the check runs off the main thread on the app scope
* and a probe that fails, throws or outlives its budget counts as not alive.
*/
private suspend fun confirm(): Boolean =
withTimeoutOrNull(confirmTimeoutMillis) {
try {
confirmAlive()
} catch (t: Throwable) {
if (t is CancellationException) throw t
false
}
} ?: false
companion object {
/**
* Bounds the liveness check. A healthy round trip on the trusted LAN/VPN
* is far shorter, so a socket that cannot answer is noticed promptly
* instead of holding the "Connected" the operator is looking at.
*/
const val CONFIRM_TIMEOUT_MILLIS = 3_000L
}
}
@@ -66,6 +66,9 @@ class PipelinesRepository(
/** Opens the single shared connection to a validated server URL. */
fun connect(validated: ServerUrl.Validated) = client.connect(validated)
/** Recovers a lost connection without reversing an explicit Disconnect. */
fun recover(validated: ServerUrl.Validated) = client.recover(validated)
/** Explicit disconnect; the client then stays Idle. */
fun disconnect() = client.disconnect()
@@ -315,6 +315,7 @@ data class SessionExport(
@Serializable
data class RunSummary(
@SerialName("resource_wait") val resourceWait: ResourceWait? = null,
val id: Long = 0,
@SerialName("pipeline_id") val pipelineId: Long = 0,
val name: String = "",
@@ -370,6 +371,7 @@ data class RunStep(
// declared directly with the exact internal/protocol/runs.go wire names.
@Serializable
data class RunDetail(
@SerialName("resource_wait") val resourceWait: ResourceWait? = null,
val id: Long = 0,
@SerialName("pipeline_id") val pipelineId: Long = 0,
val name: String = "",
@@ -405,6 +407,8 @@ data class RunDetail(
*/
@Serializable
data class RunEventFrame(
@SerialName("resource_wait") val resourceWait: ResourceWait? = null,
@SerialName("resource_wait_set") val resourceWaitSet: Boolean = false,
@SerialName("run_id") val runId: Long = 0,
val workspace: String = "",
// The run's pipeline identity and stored title, repeated on every frame so a
@@ -428,6 +432,16 @@ data class RunEventFrame(
val step: RunStepDelta? = null,
)
@Serializable
data class ResourceWait(
val resource: String = "",
val stage: String = "",
val sequence: Int = 0,
@SerialName("queued_since") val queuedSince: String = "",
) {
fun statusLabel(): String = "Waiting for resource $resource"
}
/**
* The compact change of one step within a [RunEventFrame]
* (internal/protocol/runevents.go RunStepDelta): a delta, not a RunStep, so it
@@ -0,0 +1,40 @@
package net.awan.pipelines.data.transport
/** What the app does with the connection the moment it becomes visible again. */
enum class ForegroundAction {
/** Leave the connection exactly as it is. */
LeaveAlone,
/** Re-establish the connection now, cancelling any pending reconnect delay. */
Reconnect,
}
/**
* The app's return-to-foreground recovery policy, in one pure place.
*
* Android freezes and often kills a backgrounded app's process, so the socket
* the app holds dies or goes quiet while another app is in front and nothing in
* the process notices in time. Coming back must restore that connection without
* the operator tapping Connect — but an explicit Disconnect is the operator's
* own decision and stays authoritative. A deliberate Disconnect is the only way
* the client reaches [ConnectionState.Idle], so Idle is left alone; every other
* state reconnects.
*
* A still [ConnectionState.Connected] socket is not trusted on return: a frozen
* process can leave it claiming Connected while it answers nothing, which is
* exactly the stale state this replaces. The caller therefore checks it with a
* positive probe (the existing ping) and passes the verdict as [confirmedAlive];
* only a connection just confirmed to answer is left alone.
*
* @param state the connection state at the moment the app became visible again.
* @param confirmedAlive whether a [ConnectionState.Connected] socket has just
* been confirmed to answer. It is consulted only for Connected; every other
* state carries its own answer.
*/
fun foregroundAction(state: ConnectionState, confirmedAlive: Boolean): ForegroundAction = when (state) {
ConnectionState.Idle -> ForegroundAction.LeaveAlone
is ConnectionState.Connected ->
if (confirmedAlive) ForegroundAction.LeaveAlone else ForegroundAction.Reconnect
is ConnectionState.Connecting -> ForegroundAction.Reconnect
is ConnectionState.Disconnected -> ForegroundAction.Reconnect
}
@@ -78,11 +78,33 @@ class PipelinesClient(
@Volatile
private var manualClose = false
/** Serializes socket replacement with callbacks arriving on OkHttp threads. */
private val connectionLock = Any()
private var reconnectJob: Job? = null
private var attempt = 0
/** Opens (or re-opens) the connection to a validated server URL. */
fun connect(validated: ServerUrl.Validated) {
synchronized(connectionLock) {
connectLocked(validated)
}
}
/**
* Re-opens a connection the transport lost without reversing an explicit
* [disconnect]. The check and connection replacement share one lock, so a
* Disconnect racing a foreground liveness probe wins even after that probe
* has decided recovery is needed.
*/
fun recover(validated: ServerUrl.Validated) {
synchronized(connectionLock) {
if (manualClose) return
connectLocked(validated)
}
}
private fun connectLocked(validated: ServerUrl.Validated) {
val httpUrl = validated.httpUrl ?: throw IllegalArgumentException("server URL is not valid")
endpoint = httpUrl
manualClose = false
@@ -94,14 +116,16 @@ class PipelinesClient(
/** Explicit user disconnect: stops reconnection and closes the socket. */
fun disconnect() {
manualClose = true
reconnectJob?.cancel()
reconnectJob = null
val socket = webSocket
webSocket = null
socket?.close(NORMAL_CLOSURE, "client closing")
failPending("Disconnected")
_state.value = ConnectionState.Idle
synchronized(connectionLock) {
manualClose = true
reconnectJob?.cancel()
reconnectJob = null
val socket = webSocket
webSocket = null
socket?.close(NORMAL_CLOSURE, "client closing")
failPending("Disconnected")
_state.value = ConnectionState.Idle
}
}
fun close() {
@@ -115,14 +139,20 @@ class PipelinesClient(
*/
suspend fun call(request: Message): Message = withContext(ioDispatcher) {
sendLock.withLock {
val socket = webSocket ?: throw NotConnectedException()
if (_state.value !is ConnectionState.Connected) throw NotConnectedException()
val id = "android-" + sequence.incrementAndGet()
val deferred = CompletableDeferred<Message>()
pending[id] = deferred
try {
val (id, deferred) = synchronized(connectionLock) {
val socket = webSocket ?: throw NotConnectedException()
if (_state.value !is ConnectionState.Connected) throw NotConnectedException()
val id = "android-" + sequence.incrementAndGet()
val deferred = CompletableDeferred<Message>()
pending[id] = deferred
val payload = json.encodeToString(Message.serializer(), request.copy(id = id))
if (!socket.send(payload)) throw IOException("websocket send failed")
if (!socket.send(payload)) {
pending.remove(id)
throw IOException("websocket send failed")
}
id to deferred
}
try {
val response = withTimeout(requestTimeoutMillis) { deferred.await() }
if (response.type == Protocol.ERROR) {
throw ProtocolException(response.error?.ifBlank { "server error" } ?: "server error")
@@ -144,12 +174,14 @@ class PipelinesClient(
*/
suspend fun sendOnly(request: Message): String = withContext(ioDispatcher) {
sendLock.withLock {
val socket = webSocket ?: throw NotConnectedException()
if (_state.value !is ConnectionState.Connected) throw NotConnectedException()
val id = "android-" + sequence.incrementAndGet()
val payload = json.encodeToString(Message.serializer(), request.copy(id = id))
if (!socket.send(payload)) throw IOException("websocket send failed")
id
synchronized(connectionLock) {
val socket = webSocket ?: throw NotConnectedException()
if (_state.value !is ConnectionState.Connected) throw NotConnectedException()
val id = "android-" + sequence.incrementAndGet()
val payload = json.encodeToString(Message.serializer(), request.copy(id = id))
if (!socket.send(payload)) throw IOException("websocket send failed")
id
}
}
}
@@ -166,14 +198,34 @@ class PipelinesClient(
}
}
private fun open() {
private fun open() = synchronized(connectionLock) { openLocked() }
private fun openLocked() {
val current = endpoint ?: return
attempt += 1
_state.value = ConnectionState.Connecting(attempt)
val request = Request.Builder().url(current).build()
// Replace the current socket, then close the one it superseded. Its
// callbacks are ignored from then on (see [isCurrent]), so a failure the
// old socket reports late — the common case when the process was frozen
// while it died — can neither clear the new socket nor schedule a
// reconnect that would undo it.
val superseded = webSocket
webSocket = httpClient.newWebSocket(request, listener)
// A request sent over the superseded socket cannot receive a reply from
// the replacement. Fail it now rather than leaving a screen waiting for
// the ordinary request timeout after recovery.
if (superseded != null) failPending("Connection replaced")
superseded?.close(NORMAL_CLOSURE, "superseded")
}
/**
* Whether [socket] is still the connection this client acts on. A socket this
* client has already replaced (a deliberate reconnect, or the foreground
* recovery re-establishing a frozen one) must never drive the state again.
*/
private fun isCurrent(socket: WebSocket): Boolean = webSocket === socket
private fun failPending(reason: String) {
val snapshot = pending.keys.toList()
for (key in snapshot) {
@@ -181,7 +233,8 @@ class PipelinesClient(
}
}
private fun handleClosed(reason: String) {
private fun handleClosed(socket: WebSocket, reason: String) {
if (!isCurrent(socket)) return
webSocket = null
failPending(reason)
if (manualClose) {
@@ -194,52 +247,65 @@ class PipelinesClient(
private fun scheduleReconnect(reason: String) {
if (reconnectJob?.isActive == true) return
_state.value = ConnectionState.Disconnected(reason, retrying = true)
val nextAttempt = attempt
val backoff = (BASE_BACKOFF_MILLIS shl (nextAttempt - 1).coerceAtMost(6))
.coerceAtMost(MAX_BACKOFF_MILLIS)
val backoff = reconnectBackoffMillis(attempt)
val jitter = Random.nextLong(0, 400)
reconnectJob = scope.launch {
delay(backoff + jitter)
if (!manualClose && endpoint != null) open()
synchronized(connectionLock) {
if (!manualClose && endpoint != null) open()
}
}
}
private val listener = object : WebSocketListener() {
override fun onOpen(webSocket: WebSocket, response: Response) {
attempt = 0
_state.value = ConnectionState.Connected(build = null, serverTime = null)
synchronized(connectionLock) {
if (!isCurrent(webSocket)) return
attempt = 0
_state.value = ConnectionState.Connected(build = null, serverTime = null)
}
}
override fun onMessage(webSocket: WebSocket, text: String) {
val message = try {
json.decodeFromString(Message.serializer(), text)
} catch (_: Exception) {
// Malformed frame: never misattribute it to a request.
return
}
val id = message.id
val deferred = if (id != null) pending.remove(id) else null
if (deferred != null) {
deferred.complete(message)
} else {
// Unsolicited envelopes (the connection's initial pipeline_list,
// server-pushed chat frames, stream errors) carry no matching id:
// hand them to the event stream instead of dropping them.
_events.tryEmit(message)
synchronized(connectionLock) {
if (!isCurrent(webSocket)) return
val message = try {
json.decodeFromString(Message.serializer(), text)
} catch (_: Exception) {
// Malformed frame: never misattribute it to a request.
return
}
val id = message.id
val deferred = if (id != null) pending.remove(id) else null
if (deferred != null) {
deferred.complete(message)
} else {
// Unsolicited envelopes (the connection's initial pipeline_list,
// server-pushed chat frames, stream errors) carry no matching id:
// hand them to the event stream instead of dropping them.
_events.tryEmit(message)
}
}
}
override fun onClosed(webSocket: WebSocket, code: Int, reason: String) {
handleClosed("Connection closed (${code})")
synchronized(connectionLock) {
handleClosed(webSocket, "Connection closed (${code})")
}
}
override fun onClosing(webSocket: WebSocket, code: Int, reason: String) {
webSocket.close(NORMAL_CLOSURE, null)
synchronized(connectionLock) {
if (!isCurrent(webSocket)) return
webSocket.close(NORMAL_CLOSURE, null)
}
}
override fun onFailure(webSocket: WebSocket, t: Throwable, response: Response?) {
val reason = t.message?.ifBlank { t.javaClass.simpleName } ?: "connection failed"
handleClosed(reason)
synchronized(connectionLock) {
val reason = t.message?.ifBlank { t.javaClass.simpleName } ?: "connection failed"
handleClosed(webSocket, reason)
}
}
}
@@ -247,6 +313,16 @@ class PipelinesClient(
private const val NORMAL_CLOSURE = 1000
private const val BASE_BACKOFF_MILLIS = 1_000L
private const val MAX_BACKOFF_MILLIS = 30_000L
/**
* The successful open callback resets [attempt] to zero. A later socket
* failure must therefore begin at the base delay, never shift by -1
* (which Kotlin masks into a very large shift count).
*/
internal fun reconnectBackoffMillis(attempt: Int): Long {
val exponent = (attempt - 1).coerceIn(0, 6)
return (BASE_BACKOFF_MILLIS shl exponent).coerceAtMost(MAX_BACKOFF_MILLIS)
}
}
}
@@ -101,6 +101,7 @@ internal fun foldRunEvent(runs: List<RunSummary>, frame: RunEventFrame): RunList
internal fun RunEventFrame.newRunSummary(): RunSummary? {
if (pipelineId <= 0 || name.isBlank()) return null
return RunSummary(
resourceWait = resourceWait,
id = runId,
pipelineId = pipelineId,
name = name,
@@ -124,10 +125,11 @@ internal fun RunEventFrame.newRunSummary(): RunSummary? {
*/
internal fun foldRunEvent(detail: RunDetail, frame: RunEventFrame): RunDetail {
if (frame.runId != detail.id) return detail
if (isStaleRunFrame(detail.status, frame.status)) return detail
if (isStaleRunFrame(detail.status, frame.status, frame.resourceWaitSet)) return detail
val merged = detail.fields().applying(frame)
val steps = frame.step?.let { upsertStep(detail.steps, it) } ?: detail.steps
val updated = detail.copy(
resourceWait = merged.resourceWait,
status = merged.status,
outcome = merged.outcome,
currentStage = merged.currentStage,
@@ -169,8 +171,9 @@ private fun advanceRunStatus(current: String, incoming: String): String =
* run already settled. Dropping every field (not just the status) is what keeps
* a late `running` delta from also rewriting the stage or step of a settled run.
*/
private fun isStaleRunFrame(current: String, incoming: String): Boolean =
isStaleTransition(current, incoming, ::runLifecycleRank)
private fun isStaleRunFrame(current: String, incoming: String, resourceWaitSet: Boolean = false): Boolean =
if (resourceWaitSet && incoming == "queued" && !isTerminalRunStatus(current)) false
else isStaleTransition(current, incoming, ::runLifecycleRank)
/** The step counterpart of [isStaleRunFrame]: a settled visit drops a stale delta whole. */
private fun isStaleStepDelta(current: String, incoming: String): Boolean =
@@ -213,6 +216,7 @@ private fun advanceStatus(current: String, incoming: String, rank: (String) -> I
/** The run-state fields a frame advances, shared by the summary row and the detail. */
private data class RunStateFields(
val resourceWait: net.awan.pipelines.data.protocol.ResourceWait?,
val status: String,
val outcome: String,
val currentStage: String,
@@ -226,8 +230,10 @@ private data class RunStateFields(
* detail can never apply a frame differently.
*/
fun applying(frame: RunEventFrame): RunStateFields {
val status = advanceRunStatus(status, frame.status)
val status = if (frame.resourceWaitSet && frame.status == "queued" && !isTerminalRunStatus(status))
"queued" else advanceRunStatus(status, frame.status)
return RunStateFields(
resourceWait = if (isTerminalRunStatus(status)) null else if (frame.resourceWaitSet) frame.resourceWait else resourceWait,
status = status,
outcome = frame.outcome.ifBlank { outcome },
currentStage = frame.currentStage.ifBlank { currentStage },
@@ -240,6 +246,7 @@ private data class RunStateFields(
}
private fun RunSummary.fields() = RunStateFields(
resourceWait = resourceWait,
status = status,
outcome = outcome,
currentStage = currentStage,
@@ -250,6 +257,7 @@ private fun RunSummary.fields() = RunStateFields(
)
private fun RunDetail.fields() = RunStateFields(
resourceWait = resourceWait,
status = status,
outcome = outcome,
currentStage = currentStage,
@@ -267,11 +275,12 @@ private fun executingFor(status: String, current: Boolean): Boolean = when (stat
}
private fun RunSummary.applying(frame: RunEventFrame): RunSummary {
if (isStaleRunFrame(status, frame.status)) return this
if (isStaleRunFrame(status, frame.status, frame.resourceWaitSet)) return this
val current = fields()
val merged = current.applying(frame)
if (merged == current) return this
return copy(
resourceWait = merged.resourceWait,
status = merged.status,
outcome = merged.outcome,
currentStage = merged.currentStage,
@@ -615,6 +615,7 @@ private fun VersionChip(version: String) {
/** Progress from the fields a summary actually has: completed visits + stage. */
private fun progressText(run: RunSummary): String = buildString {
run.resourceWait?.let { append(it.statusLabel()); append(" · ") }
append(visitCount(run.completedSteps, "completed"))
if (run.currentStage.isNotBlank()) append(" · stage ${run.currentStage}")
}
@@ -135,6 +135,7 @@ internal fun RunSummaryTab(
)
KeyValueRow("Pipeline", "id ${run.pipelineId} · v${run.version}")
KeyValueRow("Stage", run.currentStage)
run.resourceWait?.let { Text(it.statusLabel(), style = text.meta) }
KeyValueRow("Workspace", run.workspace)
KeyValueRow("Content hash", run.contentHash, monospace = true)
KeyValueRow("Started", formatLocalTimestamp(run.createdAt).orEmpty())
@@ -0,0 +1,138 @@
package net.awan.pipelines.data
import androidx.lifecycle.Lifecycle
import androidx.lifecycle.LifecycleOwner
import java.util.concurrent.CountDownLatch
import java.util.concurrent.TimeUnit
import java.util.concurrent.atomic.AtomicInteger
import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.delay
import net.awan.pipelines.data.transport.ConnectionState
import org.junit.Assert.assertEquals
import org.junit.Test
/**
* The one lifecycle-aware owner of return-to-foreground recovery: it applies the
* pure `foregroundAction` policy to the state it reads, confirms a supposedly
* connected socket before trusting it, and reconnects when — and only when — the
* policy says so. The state, the liveness check and the reconnect are injected,
* so the owner is exercised with no device, socket or Android framework. The
* owner never touches the LifecycleOwner it is handed.
*/
class ForegroundReconnectorTest {
private val reconnects = AtomicInteger(0)
private val probes = AtomicInteger(0)
private val owner = object : LifecycleOwner {
override val lifecycle: Lifecycle get() = throw UnsupportedOperationException("not used")
}
private val connected = ConnectionState.Connected(build = null, serverTime = null)
/**
* A reconnector whose probe answers [alive] without suspending, so
* `Dispatchers.Unconfined` runs the whole `onStart` decision inline and the
* assertions below hold the moment it returns.
*/
private fun reconnector(
state: () -> ConnectionState,
alive: suspend () -> Boolean = { true },
) = ForegroundReconnector(
state = state,
confirmAlive = { probes.incrementAndGet(); alive() },
reconnect = { reconnects.incrementAndGet() },
scope = CoroutineScope(Dispatchers.Unconfined),
)
@Test
fun aDeliberateDisconnectIsLeftAloneAndNotEvenProbed() {
reconnector(state = { ConnectionState.Idle }).onStart(owner)
assertEquals(0, probes.get())
assertEquals(0, reconnects.get())
}
@Test
fun aDroppedConnectionReconnectsImmediatelyWithoutProbing() {
reconnector(state = { ConnectionState.Disconnected("socket closed", retrying = true) }).onStart(owner)
assertEquals("a dropped socket has nothing to confirm", 0, probes.get())
assertEquals(1, reconnects.get())
}
@Test
fun aConfirmedConnectedSocketIsLeftAlone() {
reconnector(state = { connected }).onStart(owner)
assertEquals(1, probes.get())
assertEquals(0, reconnects.get())
}
@Test
fun aConnectedSocketThatDoesNotAnswerReconnects() {
reconnector(state = { connected }, alive = { false }).onStart(owner)
assertEquals(1, probes.get())
assertEquals(1, reconnects.get())
}
@Test
fun aProbeThatThrowsIsNotAlive() {
reconnector(
state = { connected },
alive = { throw IllegalStateException("socket closed") },
).onStart(owner)
assertEquals(1, reconnects.get())
}
@Test
fun aSocketThatDiesWhileTheProbeRunsStillReconnects() {
// The socket's own failure callback lands mid-check; the owner re-reads
// the state and the policy then decides on what is actually there.
var state: ConnectionState = connected
reconnector(
state = { state },
alive = {
state = ConnectionState.Disconnected("socket closed", retrying = true)
false
},
).onStart(owner)
assertEquals(1, reconnects.get())
}
@Test
fun aDisconnectWhileTheProbeRunsStaysDisconnected() {
// The user can tap Disconnect while the bounded foreground probe is in
// flight. Re-reading state after that probe keeps the explicit choice
// authoritative rather than reconnecting over it.
var state: ConnectionState = connected
reconnector(
state = { state },
alive = {
state = ConnectionState.Idle
true
},
).onStart(owner)
assertEquals(0, reconnects.get())
}
@Test
fun aProbeThatNeverAnswersIsNotAliveWhenItsBudgetExpires() {
val done = CountDownLatch(1)
val reconnector = ForegroundReconnector(
state = { connected },
confirmAlive = { delay(5_000); true },
reconnect = {
reconnects.incrementAndGet()
done.countDown()
},
scope = CoroutineScope(Dispatchers.Unconfined),
confirmTimeoutMillis = 20L,
)
reconnector.onStart(owner)
assertEquals(
"the bounded check must give up on its own rather than hold the UI's Connected",
true,
done.await(2, TimeUnit.SECONDS),
)
assertEquals(1, reconnects.get())
}
}
@@ -0,0 +1,60 @@
package net.awan.pipelines.data.transport
import org.junit.Assert.assertEquals
import org.junit.Test
/**
* The return-to-foreground policy, in the one pure place it lives: an explicit
* Disconnect is never overridden, a dropped or in-flight connection is
* re-established at once, and a still-Connected socket is trusted only after it
* has been confirmed to answer.
*/
class ConnectionRecoveryTest {
@Test
fun aDeliberateDisconnectIsNeverOverridden() {
// Idle is only ever reached by a deliberate Disconnect (the client starts
// Idle and never returns to it by itself), so returning must leave it.
assertEquals(
ForegroundAction.LeaveAlone,
foregroundAction(ConnectionState.Idle, confirmedAlive = false),
)
}
@Test
fun aDroppedConnectionReconnectsAtOnce() {
assertEquals(
ForegroundAction.Reconnect,
foregroundAction(ConnectionState.Disconnected("socket closed", retrying = true), confirmedAlive = false),
)
}
@Test
fun anOpenAlreadyInFlightIsRestartedRatherThanWaitedOn() {
assertEquals(
ForegroundAction.Reconnect,
foregroundAction(ConnectionState.Connecting(3), confirmedAlive = false),
)
}
@Test
fun aConnectedSocketIsTrustedOnlyAfterItAnswers() {
val connected = ConnectionState.Connected(build = null, serverTime = null)
assertEquals(ForegroundAction.LeaveAlone, foregroundAction(connected, confirmedAlive = true))
// The defect: a socket the process still calls Connected but that answers
// nothing must be reconnected, not trusted.
assertEquals(ForegroundAction.Reconnect, foregroundAction(connected, confirmedAlive = false))
}
@Test
fun aFailureAfterConnectedStartsAtTheBaseBackoff() {
// onOpen resets the client's attempt counter to zero. The next failure
// must not pass -1 to the shift used for exponential backoff.
assertEquals(1_000L, PipelinesClient.reconnectBackoffMillis(attempt = 0))
}
@Test
fun reconnectBackoffRemainsBounded() {
assertEquals(30_000L, PipelinesClient.reconnectBackoffMillis(attempt = Int.MAX_VALUE))
}
}
@@ -0,0 +1,32 @@
package net.awan.pipelines.ui
import kotlinx.serialization.json.Json
import net.awan.pipelines.data.protocol.ResourceWait
import net.awan.pipelines.data.protocol.RunDetail
import net.awan.pipelines.data.protocol.RunEventFrame
import net.awan.pipelines.data.protocol.RunSummary
import org.junit.Assert.*
import org.junit.Test
class ResourceWaitTest {
@Test fun optionalMetadataAndPauseResumeClearing() {
val json = Json { ignoreUnknownKeys = true }
assertNull(json.decodeFromString<RunDetail>("""{"id":1,"status":"queued"}""").resourceWait)
val wait = ResourceWait("phone-X", "review", 2, "2026-10-18T10:00:00Z")
val queued = RunEventFrame(runId = 1, status = "queued", resourceWaitSet = true, resourceWait = wait)
val detail = foldRunEvent(RunDetail(id = 1, status = "running"), queued)
assertEquals("queued", detail.status)
assertEquals(wait, detail.resourceWait)
assertEquals("Waiting for resource phone-X", wait.statusLabel())
val rows = foldRunEvent(listOf(RunSummary(id = 1, status = "running")), queued).runs
assertEquals(wait, rows.single().resourceWait)
val clear = RunEventFrame(runId = 1, status = "running", resourceWaitSet = true)
assertNull(foldRunEvent(detail, clear).resourceWait)
assertNull(foldRunEvent(rows, clear).runs.single().resourceWait)
assertNull(foldRunEvent(detail, RunEventFrame(runId = 1, status = "canceled")).resourceWait)
assertEquals("succeeded", foldRunEvent(RunDetail(id = 1, status = "succeeded"), queued).status)
val decoded = json.decodeFromString<RunDetail>("""{"id":1,"status":"queued","resource_wait":{"resource":"phone-X","stage":"review","sequence":2,"queued_since":"now"}}""")
assertEquals("phone-X", decoded.resourceWait?.resource)
assertFalse(isWaitingRunStatus(decoded.status))
}
}
+1
View File
@@ -23,6 +23,7 @@ androidx-compose-material3 = { group = "androidx.compose.material3", name = "mat
androidx-compose-material-icons-extended = { group = "androidx.compose.material", name = "material-icons-extended" }
androidx-navigation-compose = { group = "androidx.navigation", name = "navigation-compose", version.ref = "navigationCompose" }
androidx-lifecycle-runtime-compose = { group = "androidx.lifecycle", name = "lifecycle-runtime-compose", version.ref = "lifecycle" }
androidx-lifecycle-process = { group = "androidx.lifecycle", name = "lifecycle-process", version.ref = "lifecycle" }
androidx-lifecycle-viewmodel-compose = { group = "androidx.lifecycle", name = "lifecycle-viewmodel-compose", version.ref = "lifecycle" }
androidx-lifecycle-viewmodel-ktx = { group = "androidx.lifecycle", name = "lifecycle-viewmodel-ktx", version.ref = "lifecycle" }
kotlinx-coroutines-android = { group = "org.jetbrains.kotlinx", name = "kotlinx-coroutines-android", version.ref = "coroutines" }
+1 -1
View File
@@ -47,7 +47,7 @@ func main() {
// Each new run gets a bounded one-line title from the configured default
// harness; the summariser falls back to the prompt's first line, so title
// generation never blocks or fails run creation.
options := server.Options{ArtifactsRoot: artifactsRoot, RunTitles: summarise.GenerateTitle, StepVerdicts: summarise.GenerateVerdictReason}
options := server.Options{ArtifactsRoot: artifactsRoot, RunTitles: summarise.GenerateTitle, StepVerdicts: summarise.GenerateVerdictReason, VerificationFixtures: server.VerificationFixturesFromEnvironment(os.Getenv)}
recorded := ""
if value, err := store.OpenCodeEndpoint(ctx); err != nil {
log.Printf("OpenCode endpoint setting unreadable: %v", err)
+37
View File
@@ -0,0 +1,37 @@
# Background shell visibility
The chat server now opens the broker stream for read-only subscribers and reconciles verified running shell operations from OpenCode's authoritative `/api/shell` readback. The packaged OpenCode skill references and chat documentation should describe that shell visibility survives reconnects only when OpenCode reports explicit `metadata.sessionID` ownership; shared working directories are not ownership evidence.
## Current source behavior
Observers share one initial readback per room/stream attachment. The adapter and hub
fold bounded lifecycle journals over each snapshot, so a stale read cannot resurrect
an exited shell or erase a concurrent create. Shell replay and live delivery enqueue
in fold order. Cancellation rejects old-room callbacks and releases local identity
state before reconnect. Journals retain at most 256 identities and 1 MiB of id/command
text; capacity/read failures surface stable limited-visibility text, never broker
response bodies. No shell polling was added. OpenCode's terminal `timeout` and
`killed` statuses are excluded from retained running operations as well as `exited`.
## Verification evidence — 2026-10-18
`opencode api get /openapi.json` authenticated through supported service discovery.
The live schema advertises shell list/detail/output and generic `metadata`; it does
not type `metadata.sessionID`. A throwaway session
`ses_eb12ab7b3ffe0ZAle7PHpftHMG` and its own `sleep 1` shell
`sh_14ed57134001e6fuYqvn62pKit` verified actual create/detail readback preserving
explicit `metadata.sessionID`, status `running`, subsequent `exited`/exit 0, and
absence from the running list. This verifies explicit API metadata, not automatic
agent-tool ownership injection. The CLI event attempt yielded no captured events;
live-provider event ordering is **not tested**. Forced ordering evidence comes from
channel/barrier regressions, including a real WebSocket handler on a loopback scratch
server with fresh SQLite, not from the deployed service.
## Documentation follow-up / deployment
`docs/chat-verification.md` and the chat subscribe/replay section of `docs/cli.md`
are stale: docs-fold should describe authoritative ownership/readback, coalesced
observation, bounded reconciliation and limited-visibility errors, distinguishing
fixture ordering evidence from the live readback above. Shared docs and packaged
skill references were not edited here. No deployment, installation, service restart
or physical-phone/UI verification was performed; this changes no rendered layout.
+47
View File
@@ -0,0 +1,47 @@
# Delivery stabilization
Date: 2026-10-18.
The server now has draft-only `delivery=true` support for OpenCode model stages.
At run time it derives the canonical `main` worktree and linked-worktree Git
metadata from Git rather than accepting paths from a stage prompt. It verifies
the committed item tree in a disposable candidate worktree before fast-forwarding
canonical `main`; declared checks are bounded configuration, not proof that any
deployed definition has used them. It is denied for Pi (until equivalent
capability enforcement exists), foreign repositories, missing main worktrees,
and malformed capabilities. This item does not publish or deploy a definition.
The source also supports an optional controlled-fixture verification declaration
(`backend`, `revision`, `capability`, and `fixture`). Ordinary server setup wires
an explicit, bounded `reconnect-foreground` HTTP fixture registry when
`PIPELINES_VERIFICATION_RECONNECT_FOREGROUND_URL` and its expected `BACKEND`,
`REVISION`, `CAPABILITY`, and `STATE` values are configured. There is no default
remote URL. The server, not the model, probes that registered endpoint and
collects its returned backend identity, build revision, capability, fixture ID,
and state before comparing provenance. Unknown fixtures, incomplete setup,
wrong backend/revision/capability/state, non-200 responses, cancellation, and
oversized or malformed responses fail closed. A model result's labels are not
evidence. This is controlled local/backend fixture infrastructure, not evidence
of a rendered layout or a live Android device.
Definitions must be scratch-validated against the deployed server only after
the server support is deployed; no future delivery or verification definition
is activated by this source change.
`internal/author/skill/references/format.md` and
`internal/author/skill/references/queue.md` are stale: a later docs fold should
describe `delivery=true`, OpenCode-only availability, candidate verification,
and the required activation order (deploy server support before scratch-validating
and then publishing a definition).
OpenCode V2 resolves external resources at a containing directory, so delivery
uses the verified repository metadata directory rather than an impossible
single-lock-file scope. It does not grant a parent, home, foreign repository,
or global policy. Unrestricted shell permissions are not an OS sandbox.
Activation remains draft-only: deploy compatible server support, configure the
controlled fixture endpoint and matching provenance, scratch-validate the
fixture, then publish a definition. No definition is activated by this source
change.
Recovery checklist: retain `connection-reconnect` and unfinished feature
branches; merge only complete committed trees that pass mapped checks; do not
infer delivery from a dirty worktree or an old deployed-backend capture.
+95
View File
@@ -0,0 +1,95 @@
# Android: coming back to the app restores the connection
## What this item changed
Android app only.
- `data/transport/ConnectionRecovery.kt` (new): `ForegroundAction` and the one
pure `foregroundAction(state, confirmedAlive)` policy — `Idle` → leave alone (a
deliberate Disconnect is the only way the client reaches Idle), `Connected` with
a confirmed-alive socket → leave alone, `Connected` unconfirmed → reconnect,
`Connecting`/`Disconnected` → reconnect.
- `data/ForegroundReconnector.kt` (new): the one lifecycle-aware owner. It is
created once with the Application and observes `ProcessLifecycleOwner`, never
the Activity or the ViewModel (both recreated). On `ON_START` it applies the
policy: a non-Connected state reconnects at once, and the reconnect cancels the
client's pending backoff delay, so the operator never waits it out; a state
that still claims `Connected` is first confirmed with the existing `ping`,
bounded to 3 s, and the state is re-read afterwards (the socket's own failure
callback can land while the check is in flight).
- `PipelinesApplication.kt`: wires the owner to the process lifecycle;
`confirmAlive` is `repository.ping()`, `reconnect` is
`ServerUrl.validate(...) + repository.recover(...)`. Recovery shares the
client's state lock with Disconnect, so a Disconnect after the probe has made
its decision but before its reconnect effect runs stays authoritative. Manual
Connect/Reconnect continue to use `repository.connect(...)`; the client's
bounded backoff stays the only reconnect mechanism and the policy lives in the
one pure place.
- `data/transport/PipelinesClient.kt`: `open()` now closes the socket it
supersedes, and every `WebSocketListener` callback ignores a socket that is no
longer the current one (`isCurrent`). A failure the replaced socket reports late
— the common case when the process was frozen while it died — can then neither
clear the new connection nor schedule a reconnect that would undo the recovery.
- `app/build.gradle.kts` and `gradle/libs.versions.toml`:
`androidx.lifecycle:lifecycle-process` 2.8.7 (already in the offline cache,
with its `androidx.startup`/`androidx.tracing` transitives).
- Tests: `data/transport/ConnectionRecoveryTest.kt` (5 cases) and
`data/ForegroundReconnectorTest.kt` (8 cases).
## Shared documents this change makes stale
`android/README.md`.
- The Configuration section (`android/README.md:99-101`) — "Tap **Connect** to
open the single shared WebSocket (probed with `ping`, showing the server
build). Reconnect uses bounded exponential backoff; **Disconnect** stops it." —
should record that coming back to the app restores the connection on its own:
the process coming to the front reconnects immediately, cancelling the pending
backoff delay, and when the last state still claims Connected it first confirms
the socket with `ping` before trusting it. An explicit Disconnect stays
authoritative and is never reconnected over; the manual Reconnect control
stays, but normal recovery never needs it.
- The Architecture block (`android/README.md:310-323`) lists `data/transport` and
`data`. It should name the two new files — `data/transport/ConnectionRecovery.kt`
(the pure return-to-foreground policy) and `data/ForegroundReconnector.kt` (the
process-lifecycle owner) — and the new `androidx.lifecycle:lifecycle-process`
dependency behind the owner.
## Not stale
- `android/AGENTS.md` names neither the presence nor the absence of a lifecycle
observer; its "Where things are" summary stays correct (it could optionally add
the new owner beside `data/PipelinesRepository.kt`).
- `docs/app-integration-verification.md` — its connection mentions are about
restoring the stub URL, which this item does not change.
- No packaged OpenCode skill reference (`internal/author/skill/references/`)
documents the app's connection lifecycle, so nothing there is stale.
## Verified on the device, and what was not
Built and captured on the operator's Pixel 9 Pro XL (`47241FDAS00DLK`) from
`51a2ba4` + this change; after merging main the same tree (now `b93eee2`) was
rebuilt and the full unit suite re-run (691 tests, 0 failures). The operator's
`ws://10.0.0.75:8080/connect` and `/Users/alan/repos/pipelines` were used
throughout and were not changed. The phone was left on main's build.
Captured, all with `android/scripts/capture_png.sh` (every image validated):
- a **live** socket on return: the server log shows a `ping`/`pong` on the same
connection at the second of the return and no reconnect — the positive check
runs and a healthy connection is left alone;
- a **dead** socket on return: the reconnect and re-subscribe land in the same
second as the return, with no interaction (the process was frozen, so the
client's own backoff had not run);
- **no network** on return: the banner reads "Disconnected: Failed to connect to
…:8080 · retrying with backoff" — never a false Connected — and recovery
follows on its own once the network returns;
- an explicit **Disconnect**: still "Not connected. Enter the server URL in
Settings, then connect." after returning, with no reconnect in the server log.
**Not verified live:** the stale-Connected probe branch. In every capture the
frozen socket's own failure was delivered to the client before the lifecycle
owner ran, so the client already read `Disconnected` and no probe was attempted
(confirmed by the client's request-id sequence never skipping a value in the
server log). That branch — `Connected` still claimed, probe fails, reconnect — is
covered by unit tests only.
+158
View File
@@ -0,0 +1,158 @@
# Stage-exclusive resources — 2026-10-18, not deployed
This is source work for run 355, **not a deployment, installation or publication**.
No physical phone, real operator session or deployed service was used or changed.
The Android role split does not retrospectively gate pinned versions or existing runs.
## Contract and scope
An executable non-human stage can declare one runner-owned control block:
```resources
{"exclusive":"phone-47241FDAS00DLK"}
```
The key is 1–128 ASCII identifier bytes (letter first, then letters, digits, `_`, `-`).
Duplicate/unknown fields, multiple resources and human-stage resources are rejected.
One capacity-one resource per stage visit, one server/store authority; no expressions,
nested acquisition, distributed scheduling, capacity settings or multi-resource requests.
Resource-free definitions retain their execution path. Old servers reject this new block;
do not publish to an incompatible server.
Acquisition follows selection and automatic UI-skip evaluation, before BeginStep,
visits/sequences, artifacts, filediff, stage timeout, budget setup or executor calls.
Queued runs retain workspace reservation, use legacy `queued` (never human `waiting`),
and carry bounded `resource_wait` identity/stage/sequence/queued-since metadata.
Events use `resource_wait_set=true` with absent `resource_wait` to clear caches.
CLI lists and TUI/Android details show “Waiting for resource …”; human decisions and
attention notifications remain separate. Older clients still see an unfinished run.
SQLite owners and FIFO waiters are indexed by resource/order and run identity.
Private random capabilities attribute ownership to resource/run/stage/sequence and
never enter wire data, model input, transcripts or artifacts. Only the full capability
and positive cleanup evidence authorize release; mismatches and repeated release cannot
release a subsequent owner. One buffered channel per live queued run, no resource
polling, waiter tickers, model-based acquisition, queue driver or new daemon.
The gate holds through tool cleanup. OpenCode interruption now confirms session idle/
V1 settlement before clearing session ownership. Pi recovery requires both its receipt
and process/process-group exit, not merely abort acknowledgement. Executor panic,
ErrStageCleanup, retained active sessions and failed cleanup queries quarantine the owner.
An explicit `output.resource_cleanup_confirmed=false` (or malformed value) also vetoes
release for failed restoration; true never substitutes for harness cleanup evidence.
The generic runner does not inspect Android settings or magically verify restoration.
The Android UI reviewer must keep all device work foreground/awaited, restore original
settings/connection and main's build, and report uncertainty with the release veto.
## Restart and retention
Automatic FIFO progress is promised only within a live server process. Startup interrupts
persisted queued resource visits, removes their waiter rows, and never replays an executor.
Durably cleanup-confirmed owners can be reclaimed through the token-checked delete helper.
Unconfirmed owners remain quarantined, including crashes before BeginStep, before session
registration, or after cleanup but before its confirmation is persisted. Session interruption
alone cannot prove resource-specific restoration. Age never authorizes reclamation.
Quarantined ownership/session evidence survives recovery and protects even terminal runs
from retention. Reconciliation requires inspected cleanup and restoration, not lock deletion.
There is intentionally no force-unlock CLI or automatic uncertain-owner reclamation.
## Android draft, provenance and activation
Read-only retrieval: `pipelines show Android --version 2 --json` returned pipeline 20,
v2, hash `8c93aeb9b76f1e6e7696415d6308d786ec08d805a1a346fbabe4d252d5d65032`.
`python3` recomputed SHA-256 of the returned source and asserted equality.
The exact source is retained as `internal/server/testdata/android-v2-published.md`;
the proposed source is `internal/server/testdata/android-resource-draft.md`.
The provenance test compares graph fields/outcomes unchanged, apart from the approved
three stage prompts and the UI review resource; every stage keeps its published v3 model: Terra for implementer/committer and Luna for reviewer/UI reviewer. No unrelated published definition was edited.
Artifact handoff recovery: the validated v3 draft was copied to this run's retained
artifact directory as `android-resource-draft.md`. The similarly named artifact from
run 355 remains the old v2 baseline and must not be published. The proposed source in
this workspace and this run's artifact have matching SHA-256 digests
`a10d8b9fa3dc6a1875b10469061ed25fb4a7a404d463d1a2d304a309d55aa5d9`.
Activation, requiring separate explicit operator authorization:
1. Review/land the source change and independently rerun the required checks.
2. Arrange a compatible deployment separately; nothing here deploys or restarts it.
3. Reconcile every existing phone user, pinned/live run and any quarantined owner.
4. Transfer the **workspace proposed draft**, provenance and this note into the retained
stage artifacts after resolving the artifact edit permission.
5. Validate that retained draft against the configured authoritative compatible endpoint;
only then explicitly publish it. Do not use a disposable endpoint as a project default.
6. Opt in other device users explicitly before relying on mutual exclusion system-wide.
Read-only published inventory (`pipelines show <name> --json`, v2 for each):
- UI sweep: `survey` and `ui_reviewer` request visual/screenshots work; phone work there
needs explicit resource opt-in and restoration/fixture policy.
- Generic: `reviewer` and `ui_reviewer` have visual/device verification wording; when
targeting a phone, separate code review from leased device review first.
- Plan and land: `implementer`, `reviewer`, `ui_reviewer` contain rendered-surface work;
they need an explicit technology-specific role split/lease before phone use.
- Probe: one harness smoke stage; no default phone declaration. Any device-specific
probe needs explicit opt-in, never an inferred resource or Android stage-name special case.
Server/TUI/other definitions were not changed. Inventory is not activation evidence.
## Verification evidence and remaining limits
Executed focused checks:
- Run 358 recovery revalidation: `go build ./... && go vet ./... && go test ./...`
passed; `go test -race ./internal/pipeline ./internal/server ./internal/stage -run
'Resource|Scheduler|Concurrent|Human|Decision|RetryWaits' -count=1` passed; and the
Android map invocation with `JAVA_HOME`/`ANDROID_HOME`, installed Gradle,
`--offline --no-daemon --rerun-tasks :app:assembleDebug :app:testDebugUnitTest`
executed all 41 tasks and reported `BUILD SUCCESSFUL`. These remain build/unit and
fake-executor scheduler evidence,
not physical-phone verification or screenshots.
- Recovery after integrating main `88595eb`: `go build ./...`, `go vet ./...`,
`go test ./internal/server -run Resource -count=1`,
`go test ./internal/pipeline ./internal/protocol ./internal/tui -run Resource -count=1`,
`go test ./internal/stage -count=1`,
`go test -race ./internal/pipeline ./internal/server ./internal/stage -run 'Resource|Scheduler|Concurrent|Human|Decision|RetryWaits' -count=1`, and
`go test ./...` all passed. `gofmt -l .` and `git diff --check` produced no output.
- Recovery Android build/unit check with the map's offline, no-daemon Gradle invocation:
`:app:assembleDebug :app:testDebugUnitTest` passed. This remains build/unit evidence,
not phone verification or screenshots.
- `go test ./internal/server ./internal/pipeline ./internal/protocol ./internal/tui -run Resource -count=1`:
passed. Includes fake-executor queue >30ms timeout, independent-resource progress,
FIFO/mismatch/duplicate/grant-cancel cycles, blocked cleanup, panic/quarantine,
persistence/BeginStep/query failure, no-acquire skip, restart/retention, wire defaults,
narrow/wide/plain TUI frames and real CLI/WebSocket scratch exercise.
- `go test ./internal/stage -count=1`: passed; the later full suite also included
`TestResourcePiReceiptRequiresProcessExit` (a test-owned sleep process, not a device).
- Android map invocation with `JAVA_HOME`/`ANDROID_HOME` and installed Gradle,
`--offline --no-daemon :app:assembleDebug :app:testDebugUnitTest`: BUILD SUCCESSFUL.
These are build/unit checks, not device verification or screenshots.
- After the Go tree settled: `go build ./...`, `go vet ./...`, `go test ./...`
all exited zero (including author-package tests).
- `go test -race ./internal/pipeline ./internal/server ./internal/stage -run 'Resource|Scheduler|Concurrent|Human|Decision|RetryWaits' -count=1`:
all three packages passed, including concurrent grant/cancel cycles.
- `gofmt -l .` and `git diff --check`: exited zero with no output.
The scratch CLI test uses the real CLI and WebSocket handler with temporary SQLite and
an explicitly **FAKE executor**. It validates the proposed Android draft on compatible
scratch code, observes B as queued with zero steps while A holds fake-phone, inspects
subscription metadata, releases A, runs `wait`, and observes B succeeded automatically.
It does not call a model or use a physical phone. Test log output is evidence, not a
live-deployment claim; no retained CLI transcript was written to the denied artifact path.
Live OpenCode/Pi shell/ADB termination and Android visual/device restoration are **not
tested** here. The checks prove scheduler behavior with controlled fakes, plus test-owned
process-exit checks, not device readiness on the current deployment.
Commit/merge/landing belong to the later committer stage; no commit or push here.
## Stale shared documentation for docs-fold (do not edit in this item)
- `docs/cli.md`: resource queue presentation, inspection metadata, unchanged wait/decide semantics.
- `internal/author/skill/references/format.md`: optional resources block and validation/key bounds.
- `internal/author/skill/references/control.md`: runner-owned declaration, timeout/accounting boundary,
release veto and automatic skip placement.
- `internal/author/skill/references/queue.md`: FIFO resource waits vs workspace exclusion and human waits.
- `internal/author/skill/references/operations.md`: authority, restart interruption/quarantine,
inspected recovery, retention and explicit compatible-deployment/publication prerequisites.
- `internal/author/skill/references/verification.md`: fake-vs-device evidence, fixtures and leased restoration.
- `internal/author/skill/references/orchestration.md`: no queue drivers/polling/extra agents; explicit
opt-in of every device user, no retroactive gating or distributed guarantees.
No packaged skill/reference, shared README/topic documentation, pipeline registry,
installation, deployment, provider setting or `internal/stage/model_failure.go` was edited.
+20
View File
@@ -0,0 +1,20 @@
# Scoped stage artifact access
## What this item changed
The server now derives one exact artifact capability for an OpenCode or Pi
model-stage visit: its own directory is writable when the edit tool is present,
and its own plus completed earlier visit directories in the same run are
readable when a file-reading tool is present. No artifact root, sibling, future
visit, other run, traversal path, or symlink escape is granted. The scope is
empty if the runner cannot validate its artifact root.
Pi retains its lease and explicit extension policy; unrestricted shell remains
outside this file-tool boundary and is not an OS sandbox.
## Shared documents this change makes stale
- `docs/stage-artifacts.md` and `internal/author/skill/references/operations.md`
should explain that model-stage artifact directories are usable by the
supported file tools only within the exact same-run visit scope, not as a
general external-directory permission.
+42
View File
@@ -0,0 +1,42 @@
# Stage-timeout diagnostics
## Source behavior
The V2 active-session wait now reports `STAGE DEADLINE` when the configured
attempt timer expires between successful polls of a running admitted session.
Its bounded diagnostic names the idle-wait phase, configured timeout, retained
session, and that the stage had not finished (active at the last poll). It does
not claim that a completed response was lost. Cancellation is no longer recorded
as a stage deadline. A bare deadline alone is not `HARNESS TRANSPORT`.
Actual request/HTTP/operation failures keep `HARNESS TRANSPORT`, their sanitized
operation/status evidence and underlying error identities, including when joined
with a deadline. A parent deadline no longer discards an accompanying transport
failure. Final-result validation, budget accounting, cleanup, interrupts,
side effects and replay policy are unchanged. No timeout was increased.
## Documentation to fold manually
`internal/author/skill/references/debugging.md` needs a separate stage-deadline
playbook alongside its HARNESS TRANSPORT section: inspect the retained session,
last activity, configured limits and unfinished final-result evidence before
deciding what to do. Do not blindly retry an admitted mutating stage; edits/tool
side effects remain. Explain the distinction between a configured idle-wait
overrun, cancellation, an actual failed request with a deadline, budget failure,
and final-result validation. Do not infer that all deadline errors are transport
failures or that a successful last tool means the stage returned a result.
## Evidence and deployment
2026-10-18: installed server build metadata (`go version -m
/Users/alan/.local/share/pipelines/bin/pipelines-server`) identifies revision
`0d1ddc73a444fa24110ba5223646856c8f053d29`. Comparing that revision with this
worktree's initial HEAD showed identical model-failure classification, model
attempt timing, V2 polling and agent wait/cleanup paths. The runner difference
was unrelated session-export HTML-only metadata.
Regression tests use isolated loopback servers and scratch SQLite databases,
with a controlled V2 CLI fixture for the admitted-active-session deadline and
real loopback HTTP/refused-connection checks for native transport evidence.
This is not a live-provider timeout test. No deployment or deployed-service
restart is part of this item; retained historical runs are not rewritten.
+2 -2
View File
@@ -774,7 +774,7 @@ func (a App) Run(ctx context.Context, args []string) error {
}
case "runs":
for _, run := range r.Runs {
fmt.Fprintf(a.Out, "%d\t%s\t%s\tv%d\t%s\n", run.ID, run.Label(), run.Name, run.Version, run.Status)
fmt.Fprintf(a.Out, "%d\t%s\t%s\tv%d\t%s\n", run.ID, run.Label(), run.Name, run.Version, run.StatusLabel())
}
case "ping":
fmt.Fprintln(a.Out, "Connected ·", r.Time)
@@ -1484,7 +1484,7 @@ func (e *waitLimitError) Runs() string {
// waitLine is the one-line terminal report `pipelines wait` prints: the run id,
// its status and outcome, its stage and how long it took.
func waitLine(d *protocol.RunDetail) string {
line := fmt.Sprintf("run %d · %s", d.ID, d.Status)
line := fmt.Sprintf("run %d · %s", d.ID, d.StatusLabel())
if outcome := strings.TrimSpace(d.Outcome); outcome != "" {
line += " (" + outcome + ")"
}
+198 -11
View File
@@ -28,7 +28,9 @@ var ErrPermissionUnsupported = errors.New("OpenCode permission approval is not i
// semantics, the session-id shape and the 64 KiB / 1 MiB bounds of Connection so
// no second transport or credential path is introduced.
type Chat struct {
conn *Connection
conn *Connection
shellMu sync.Mutex
shells map[string]*ChatStreamState
// mu serializes the adapter's HTTP calls. Connection records its last
// operation and status on the shared connection, so two calls must not
// overlap; the permission poller runs alongside the read loop's
@@ -294,6 +296,114 @@ func decodeChatPermissions(sessionID string, data []byte) ([]protocol.ChatPermis
return frames, nil
}
// Shells reads the endpoint's authoritative shell list and returns only running
// operations whose metadata explicitly names sessionID. The shell API is global;
// filtering locally avoids guessing an unsupported query filter and never assigns
// ownership from a shared working directory. A successful read also seeds the
// stream state used to attribute later shell.exited events, which carry no session.
func (c *Chat) Shells(ctx context.Context, sessionID string) ([]protocol.ChatShell, error) {
if !ValidSessionID(sessionID) {
return nil, fmt.Errorf("invalid OpenCode session id")
}
state, err := c.streamStateContext(ctx, sessionID)
if err != nil {
return nil, err
}
// One read per state; waiting and I/O never hold the event-state mutex.
state.mu.Lock()
if state.readGate == nil {
state.readGate = make(chan struct{}, 1)
}
gate := state.readGate
state.mu.Unlock()
select {
case gate <- struct{}{}:
case <-ctx.Done():
return nil, ctx.Err()
}
defer func() { <-gate }()
state.mu.Lock()
state.readback = &ShellSnapshot{}
state.mu.Unlock()
defer func() { state.mu.Lock(); state.readback = nil; state.mu.Unlock() }()
data, err := c.call(ctx, "get", "/api/shell", nil)
if err != nil {
return nil, err
}
var response struct {
Data []chatShellInfo `json:"data"`
}
if err := json.Unmarshal(data, &response); err != nil || response.Data == nil {
if err := json.Unmarshal(data, &response.Data); err != nil {
return nil, fmt.Errorf("invalid OpenCode shell response")
}
}
out := make([]protocol.ChatShell, 0, len(response.Data))
for _, info := range response.Data {
if info.ID == "" || info.shellSessionID() != sessionID || (info.Status != "" && info.Status != protocol.ChatShellRunning) {
continue
}
status := info.Status
if status == "" {
status = protocol.ChatShellRunning
}
out = append(out, protocol.ChatShell{ID: info.ID, Status: status, Command: info.Command})
}
state.mu.Lock()
defer state.mu.Unlock()
out, err = state.readback.Merge(out)
if err != nil {
return nil, err
}
if err := ctx.Err(); err != nil {
return nil, err
}
ids := make([]string, 0, len(out))
for _, shell := range out {
ids = append(ids, shell.ID)
}
state.replaceShellsLocked(ids)
return out, nil
}
// streamState returns the session's retained, verified shell identities. It is
// deliberately kept across stream replacement so an exit after reconnect remains
// attributable; Shells reconciles its contents from authoritative readback on
// each attach rather than replacing the pointer a live scanner is using.
func (c *Chat) streamState(sessionID string) *ChatStreamState {
state, _ := c.streamStateContext(context.Background(), sessionID)
return state
}
func (c *Chat) streamStateContext(ctx context.Context, sessionID string) (*ChatStreamState, error) {
c.shellMu.Lock()
defer c.shellMu.Unlock()
if err := ctx.Err(); err != nil {
return nil, err
}
if c.shells == nil {
c.shells = map[string]*ChatStreamState{}
}
state := c.shells[sessionID]
if state == nil {
state = &ChatStreamState{}
c.shells[sessionID] = state
}
return state, nil
}
// ReleaseShells discards verified shell identities after the last room observer
// disconnects. A future attach calls Shells for a fresh authoritative snapshot,
// so this bounds state without attributing a foreign shell during the gap.
func (c *Chat) ReleaseShells(sessionID string) {
if sessionID == "" {
return
}
c.shellMu.Lock()
delete(c.shells, sessionID)
c.shellMu.Unlock()
}
// Stream subscribes to the OpenCode event firehose and invokes emit for every
// frame belonging to sessionID. It reads one bounded SSE frame at a time and
// never polls the per-session message log; the global firehose is the only
@@ -306,6 +416,12 @@ func (c *Chat) Stream(ctx context.Context, id string, ready func(), emit func(pr
if !sessionID.MatchString(id) {
return fmt.Errorf("invalid OpenCode session id")
}
// Pin state before opening/announcing the stream so a canceled scanner
// cannot acquire a replacement room's identity map.
state, err := c.streamStateContext(ctx, id)
if err != nil {
return err
}
body, err := c.conn.EventStream(ctx)
if err != nil {
return err
@@ -314,7 +430,7 @@ func (c *Chat) Stream(ctx context.Context, id string, ready func(), emit func(pr
if ready != nil {
ready()
}
return scanChatEvents(ctx, body, id, emit)
return c.scanChatEventsWithState(ctx, body, id, state, emit)
}
// scanChatEvents turns an SSE body into protocol frames, one bounded frame at a
@@ -322,10 +438,13 @@ func (c *Chat) Stream(ctx context.Context, id string, ready func(), emit func(pr
// malformed JSON fails the stream instead of producing a malformed frame. The
// last assistant message id is retained so a later tool/step event that omits it
// still attributes every frame to a stable message.
func scanChatEvents(ctx context.Context, body io.Reader, sessionID string, emit func(protocol.ChatEventFrame) error) error {
func (c *Chat) scanChatEvents(ctx context.Context, body io.Reader, sessionID string, emit func(protocol.ChatEventFrame) error) error {
return c.scanChatEventsWithState(ctx, body, sessionID, c.streamState(sessionID), emit)
}
func (c *Chat) scanChatEventsWithState(ctx context.Context, body io.Reader, sessionID string, state *ChatStreamState, emit func(protocol.ChatEventFrame) error) error {
scanner := bufio.NewScanner(body)
scanner.Buffer(make([]byte, 0, 64<<10), MaxResponse)
state := &ChatStreamState{}
for scanner.Scan() {
if err := ctx.Err(); err != nil {
return err
@@ -346,7 +465,7 @@ func scanChatEvents(ctx context.Context, body io.Reader, sessionID string, emit
continue
}
if frame.MessageID != "" {
state.LastMessageID = frame.MessageID
state.setLastMessageID(frame.MessageID)
}
if err := emit(frame); err != nil {
return err
@@ -449,8 +568,29 @@ func (i *chatShellInfo) shellSessionID() string {
// what attributes the exit to its session: another session's shell is an
// unknown id and is ignored.
type ChatStreamState struct {
mu sync.Mutex
LastMessageID string
Shells map[string]struct{}
readGate chan struct{}
readback *ShellSnapshot
}
func (s *ChatStreamState) lastMessageID() string {
if s == nil {
return ""
}
s.mu.Lock()
defer s.mu.Unlock()
return s.LastMessageID
}
func (s *ChatStreamState) setLastMessageID(id string) {
if s == nil || id == "" {
return
}
s.mu.Lock()
s.LastMessageID = id
s.mu.Unlock()
}
// knowsShell reports whether id is a shell this stream has already surfaced for
@@ -459,6 +599,8 @@ func (s *ChatStreamState) knowsShell(id string) bool {
if s == nil || id == "" {
return false
}
s.mu.Lock()
defer s.mu.Unlock()
_, ok := s.Shells[id]
return ok
}
@@ -468,6 +610,8 @@ func (s *ChatStreamState) markShell(id string) {
if s == nil || id == "" {
return
}
s.mu.Lock()
defer s.mu.Unlock()
if s.Shells == nil {
s.Shells = map[string]struct{}{}
}
@@ -476,12 +620,37 @@ func (s *ChatStreamState) markShell(id string) {
// forgetShell drops id once its exit has been surfaced.
func (s *ChatStreamState) forgetShell(id string) {
if s == nil || s.Shells == nil {
if s == nil {
return
}
s.mu.Lock()
defer s.mu.Unlock()
if s.Shells == nil {
return
}
delete(s.Shells, id)
}
// replaceShells applies one authoritative running-shell snapshot without
// replacing the state object an already attached event scanner owns.
func (s *ChatStreamState) replaceShells(ids []string) {
if s == nil {
return
}
s.mu.Lock()
defer s.mu.Unlock()
s.replaceShellsLocked(ids)
}
func (s *ChatStreamState) replaceShellsLocked(ids []string) {
s.Shells = make(map[string]struct{}, len(ids))
for _, id := range ids {
if id != "" {
s.Shells[id] = struct{}{}
}
}
}
// NormalizeChatEvent maps one OpenCode firehose `data:` payload to a
// protocol.ChatEventFrame, folding the small per-session context ([stream]) it
// needs across frames. ok is false for an event type this adapter does not
@@ -528,7 +697,7 @@ func NormalizeChatEvent(payload []byte, sessionID string, stream *ChatStreamStat
messageID = data.MessageIDAlt
}
if messageID == "" && stream != nil {
messageID = stream.LastMessageID
messageID = stream.lastMessageID()
}
// Every pushed frame must carry a stable message id so the client can
// attribute it; without one this is not a chat frame.
@@ -590,20 +759,38 @@ func normalizeShellEvent(eventType string, data chatEvent, sessionID string, str
if status == "" {
status = protocol.ChatShellRunning
}
stream.markShell(data.Info.ID)
shell := &protocol.ChatShell{ID: data.Info.ID, Status: status, Command: data.Info.Command}
if stream != nil {
stream.mu.Lock()
stream.readback.Record(shell)
if stream.Shells == nil {
stream.Shells = map[string]struct{}{}
}
if status == protocol.ChatShellRunning && len(stream.Shells) < chatShellIdentityLimit {
stream.Shells[shell.ID] = struct{}{}
}
stream.mu.Unlock()
}
return protocol.ChatEventFrame{
SessionID: sessionID,
Kind: protocol.ChatEventShell,
Shell: &protocol.ChatShell{ID: data.Info.ID, Status: status, Command: data.Info.Command},
Shell: shell,
}, true, nil
}
if data.ID == "" {
return protocol.ChatEventFrame{}, false, fmt.Errorf("malformed OpenCode shell.exited event")
}
if !stream.knowsShell(data.ID) {
if stream == nil {
return protocol.ChatEventFrame{}, false, nil
}
stream.mu.Lock()
stream.readback.Record(&protocol.ChatShell{ID: data.ID, Status: protocol.ChatShellExited})
_, known := stream.Shells[data.ID]
delete(stream.Shells, data.ID)
stream.mu.Unlock()
if !known {
return protocol.ChatEventFrame{}, false, nil
}
stream.forgetShell(data.ID)
status := data.Status
if status == "" {
status = protocol.ChatShellExited
@@ -0,0 +1,116 @@
package opencode
import (
"context"
"fmt"
"net/http"
"net/http/httptest"
"testing"
"time"
"git.a-wan.net/alan/pipelines/internal/protocol"
)
func TestChatShellReadbackBarrier(t *testing.T) {
for _, known := range []bool{false, true} {
t.Run(fmt.Sprint(known), func(t *testing.T) {
read, apply := make(chan struct{}), make(chan struct{})
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
close(read)
select {
case <-apply:
fmt.Fprint(w, `{"data":[{"id":"sh_old","status":"running","metadata":{"sessionID":"ses_1"}},{"id":"sh_foreign","status":"running","metadata":{"sessionID":"ses_other"}}]}`)
case <-r.Context().Done():
}
}))
defer srv.Close()
c := NewChat(&Connection{Backend: "v2", URL: srv.URL})
state := c.streamState("ses_1")
if known {
state.markShell("sh_old")
}
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
defer cancel()
result := make(chan []protocol.ChatShell, 1)
errs := make(chan error, 1)
go func() { shells, err := c.Shells(ctx, "ses_1"); result <- shells; errs <- err }()
select {
case <-read:
case <-ctx.Done():
t.Fatal("read did not reach barrier")
}
// Both a previously verified exit and an exit first learned while
// reading must suppress the stale running snapshot.
frame, ok, err := NormalizeChatEvent([]byte(`{"type":"shell.exited","data":{"id":"sh_old","status":"exited","exit":0}}`), "ses_1", state)
if err != nil || ok != known || (ok && frame.Shell.Status != protocol.ChatShellExited) {
t.Fatalf("exit = %+v %v %v", frame, ok, err)
}
frame, ok, err = NormalizeChatEvent([]byte(`{"type":"shell.created","data":{"info":{"id":"sh_new","status":"running","command":"sleep 10","metadata":{"sessionID":"ses_1"}}}}`), "ses_1", state)
if err != nil || !ok || frame.Shell.ID != "sh_new" {
t.Fatalf("create = %+v %v %v", frame, ok, err)
}
close(apply)
got := <-result
if err := <-errs; err != nil {
t.Fatal(err)
}
if len(got) != 1 || got[0].ID != "sh_new" || got[0].Command != "sleep 10" {
t.Fatalf("readback lost events/leaked ownership: %+v", got)
}
if state.knowsShell("sh_old") || state.knowsShell("sh_foreign") || !state.knowsShell("sh_new") {
t.Fatal("adapter identity state regressed")
}
// User-visible subsequent exit remains attributable after reconciliation.
frame, ok, err = NormalizeChatEvent([]byte(`{"type":"shell.exited","data":{"id":"sh_new","status":"exited","exit":0}}`), "ses_1", state)
if err != nil || !ok || frame.Shell.ID != "sh_new" {
t.Fatalf("new shell exit lost: %+v %v %v", frame, ok, err)
}
})
}
}
func TestChatShellReadbackCancelWaiter(t *testing.T) {
read := make(chan struct{})
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { close(read); <-r.Context().Done() }))
defer srv.Close()
c := NewChat(&Connection{Backend: "v2", URL: srv.URL})
ctx, cancel := context.WithCancel(context.Background())
result := make(chan error, 2)
go func() { _, err := c.Shells(ctx, "ses_1"); result <- err }()
select {
case <-read:
case <-time.After(time.Second):
t.Fatal("read stuck")
}
go func() { _, err := c.Shells(ctx, "ses_1"); result <- err }()
cancel()
for range 2 {
select {
case err := <-result:
if err == nil {
t.Fatal("canceled read succeeded")
}
case <-time.After(time.Second):
t.Fatal("read or waiter stuck")
}
}
old := c.streamState("ses_1")
c.ReleaseShells("ses_1")
if c.streamState("ses_1") == old {
t.Fatal("reconnect reused released state")
}
}
func TestShellSnapshotBounded(t *testing.T) {
var snapshot ShellSnapshot
for i := 0; i <= chatShellIdentityLimit; i++ {
snapshot.Record(&protocol.ChatShell{ID: fmt.Sprintf("sh_%d", i), Status: protocol.ChatShellExited})
}
if len(snapshot.changes) != chatShellIdentityLimit {
t.Fatalf("unbounded journal: %d", len(snapshot.changes))
}
if _, err := snapshot.Merge([]protocol.ChatShell{{ID: "sh_0", Status: protocol.ChatShellRunning}}); err == nil {
t.Fatal("overflow accepted stale snapshot")
}
}
+72
View File
@@ -0,0 +1,72 @@
package opencode
import (
"fmt"
"sort"
"git.a-wan.net/alan/pipelines/internal/protocol"
)
// ShellSnapshot records changes while an authoritative read is in flight. Its
// owner serializes access. Unknown exits may be recorded, but can only remove an
// id: they never establish ownership. Overflow rejects the snapshot, not events.
type ShellSnapshot struct {
changes map[string]protocol.ChatShell
bytes int
overflow bool
}
const chatShellIdentityLimit = 256
func (s *ShellSnapshot) Record(shell *protocol.ChatShell) {
if s == nil || shell == nil || shell.ID == "" || s.overflow {
return
}
if s.changes == nil {
s.changes = make(map[string]protocol.ChatShell)
}
old := s.changes[shell.ID]
n := s.bytes - len(old.ID) - len(old.Command) + len(shell.ID) + len(shell.Command)
if n > MaxResponse || (old.ID == "" && len(s.changes) >= chatShellIdentityLimit) {
s.overflow = true
return
}
// A terminal event wins even if a duplicate create is delivered later.
if old.ID != "" && old.Status != protocol.ChatShellRunning {
return
}
s.changes[shell.ID] = *shell
s.bytes = n
}
func (s *ShellSnapshot) Merge(shells []protocol.ChatShell) ([]protocol.ChatShell, error) {
if s.overflow {
return nil, fmt.Errorf("shell reconciliation capacity exceeded")
}
byID := make(map[string]protocol.ChatShell)
for _, shell := range shells {
if shell.ID != "" && shell.Status == protocol.ChatShellRunning {
byID[shell.ID] = shell
}
}
for id, shell := range s.changes {
if shell.Status == protocol.ChatShellRunning {
byID[id] = shell
} else {
delete(byID, id)
}
}
if len(byID) > chatShellIdentityLimit {
return nil, fmt.Errorf("shell reconciliation capacity exceeded")
}
ids := make([]string, 0, len(byID))
for id := range byID {
ids = append(ids, id)
}
sort.Strings(ids)
out := make([]protocol.ChatShell, 0, len(ids))
for _, id := range ids {
out = append(out, byID[id])
}
return out, nil
}
+42
View File
@@ -395,3 +395,45 @@ func TestOpenChatRejectsUnrunnableCLI(t *testing.T) {
t.Fatal("a missing CLI must fail detection, not open an endpoint")
}
}
// TestChatShellReadbackSeedsReplacementStream proves an authoritative shell
// readback survives a later stream replacement: shell.exited has no session id,
// so it must still clear the verified shell after reconnect.
func TestChatShellReadbackSeedsReplacementStream(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
switch r.URL.Path {
case "/api/shell":
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte(`{"data":[{"id":"sh_1","status":"running","command":"sleep 30","metadata":{"sessionID":"ses_1"}},{"id":"sh_other","status":"running","command":"sleep 30","metadata":{"sessionID":"ses_other"}}]}`))
case "/api/event":
w.Header().Set("Content-Type", "text/event-stream")
_, _ = w.Write([]byte("data: {\"type\":\"shell.exited\",\"data\":{\"id\":\"sh_1\",\"status\":\"exited\",\"exit\":0}}\n\n"))
}
}))
defer srv.Close()
chat := NewChat(&Connection{Backend: "v2", URL: srv.URL})
shells, err := chat.Shells(context.Background(), "ses_1")
if err != nil || len(shells) != 1 || shells[0].ID != "sh_1" {
t.Fatalf("Shells = %+v, %v", shells, err)
}
var got []protocol.ChatEventFrame
if err := chat.Stream(context.Background(), "ses_1", nil, func(frame protocol.ChatEventFrame) error { got = append(got, frame); return nil }); err != nil {
t.Fatal(err)
}
if len(got) != 1 || got[0].Shell == nil || got[0].Shell.ID != "sh_1" || got[0].Shell.Status != protocol.ChatShellExited {
t.Fatalf("exit after readback = %+v", got)
}
}
func TestChatReleaseShellsBoundsDisconnectedState(t *testing.T) {
chat := NewChat(&Connection{})
state := chat.streamState("ses_1")
state.markShell("sh_1")
chat.ReleaseShells("ses_1")
chat.shellMu.Lock()
_, retained := chat.shells["ses_1"]
chat.shellMu.Unlock()
if retained {
t.Fatal("released session shell state was retained")
}
}
+14 -1
View File
@@ -10,7 +10,7 @@ import (
)
func decodeControl(text string, value any) error {
// Both control blocks are flat objects. Reject duplicate keys before decoding.
// Control blocks are flat objects. Reject duplicate keys before decoding.
keys := json.NewDecoder(strings.NewReader(text))
if token, err := keys.Token(); err != nil || token != json.Delim('{') {
return fmt.Errorf("expected object")
@@ -44,6 +44,12 @@ func decodeControl(text string, value any) error {
}
func parseControl(s *Stage, fields map[string]string) error {
if text, ok := fields["resources"]; ok {
if err := decodeControl(text, &s.Resources); err != nil || s.Resources == nil || !ValidResourceKey(s.Resources.Exclusive) {
return fmt.Errorf("resources must contain exactly one exclusive key (1–128 ASCII letters, digits, _ or -; start with a letter)")
}
delete(fields, "resources")
}
if text, ok := fields["limits"]; ok {
if err := decodeControl(text, &s.Limits); err != nil || s.Limits == nil {
return fmt.Errorf("limits must be an object with max_visits and optional on outcomes")
@@ -76,6 +82,9 @@ func parseControl(s *Stage, fields map[string]string) error {
}
func validateControl(s Stage, stages map[string]bool) error {
if s.Resources != nil && s.Type == "human" {
return fmt.Errorf("stage %s: human stages cannot declare resources", s.Key)
}
if s.Limits != nil {
for _, outcome := range s.Limits.On {
target, exists := s.Edges[outcome]
@@ -95,3 +104,7 @@ func validateControl(s Stage, stages map[string]bool) error {
}
return nil
}
func ValidResourceKey(key string) bool {
return len(key) <= 128 && identifier.MatchString(key)
}
+277
View File
@@ -0,0 +1,277 @@
package pipeline
import (
"context"
"crypto/rand"
"database/sql"
"encoding/hex"
"encoding/json"
"fmt"
"time"
"git.a-wan.net/alan/pipelines/internal/protocol"
)
// ResourceVisit identifies a pending visit, before it consumes execution accounting.
type ResourceVisit struct {
Resource string
RunID int64
Stage string
Sequence int
}
// ResourceLease is an opaque capability. Its credential is deliberately not exported
// or serializable, and must never be put in model input or public wire metadata.
type ResourceLease struct {
ResourceVisit
token string
}
// RequestResource acquires immediately or registers one FIFO waiter/channel. The
// mutex covers only short DB transactions and channel bookkeeping, never execution.
func (s *Store) RequestResource(ctx context.Context, visit ResourceVisit) (ResourceLease, <-chan ResourceLease, *protocol.ResourceWait, error) {
s.resourceMu.Lock()
defer s.resourceMu.Unlock()
if !ValidResourceKey(visit.Resource) || visit.RunID <= 0 || visit.Sequence <= 0 || !identifier.MatchString(visit.Stage) {
return ResourceLease{}, nil, nil, fmt.Errorf("invalid resource visit")
}
if ch := s.resourceWaiters[visit]; ch != nil {
wait, err := s.resourceWait(ctx, visit.RunID)
return ResourceLease{}, ch, wait, err
}
credential := make([]byte, 32)
if _, err := rand.Read(credential); err != nil {
return ResourceLease{}, nil, nil, err
}
lease := ResourceLease{ResourceVisit: visit, token: hex.EncodeToString(credential)}
tx, err := s.db.BeginTx(ctx, nil)
if err != nil {
return ResourceLease{}, nil, nil, err
}
defer tx.Rollback()
var status string
if err := tx.QueryRowContext(ctx, "SELECT status FROM pipeline_runs WHERE id=?", visit.RunID).Scan(&status); err != nil {
return ResourceLease{}, nil, nil, err
}
if status != protocol.RunStatusQueued && status != protocol.RunStatusRunning {
return ResourceLease{}, nil, nil, fmt.Errorf("run is not active")
}
var existing ResourceLease
err = tx.QueryRowContext(ctx, "SELECT resource_key,run_id,stage_key,sequence,token FROM resource_owners WHERE run_id=?", visit.RunID).Scan(&existing.Resource, &existing.RunID, &existing.Stage, &existing.Sequence, &existing.token)
if err == nil {
if existing.ResourceVisit != visit {
return ResourceLease{}, nil, nil, fmt.Errorf("run already owns another resource visit")
}
return existing, nil, nil, nil
}
if err != sql.ErrNoRows {
return ResourceLease{}, nil, nil, err
}
result, err := tx.ExecContext(ctx, `INSERT INTO resource_owners(resource_key,run_id,stage_key,sequence,token)
VALUES(?,?,?,?,?) ON CONFLICT(resource_key) DO NOTHING`, visit.Resource, visit.RunID, visit.Stage, visit.Sequence, lease.token)
if err != nil {
return ResourceLease{}, nil, nil, err
}
n, err := result.RowsAffected()
if err != nil {
return ResourceLease{}, nil, nil, err
}
if n == 1 {
return lease, nil, nil, tx.Commit()
}
wait := &protocol.ResourceWait{Resource: visit.Resource, Stage: visit.Stage, Sequence: visit.Sequence, QueuedSince: time.Now().UTC().Format(time.RFC3339Nano)}
encoded, err := json.Marshal(wait)
if err != nil {
return ResourceLease{}, nil, nil, err
}
if _, err := tx.ExecContext(ctx, `INSERT INTO resource_waiters(resource_key,run_id,stage_key,sequence,token,queued_at) VALUES(?,?,?,?,?,?)`, visit.Resource, visit.RunID, visit.Stage, visit.Sequence, lease.token, wait.QueuedSince); err != nil {
return ResourceLease{}, nil, nil, err
}
if err := affected(tx.ExecContext(ctx, "UPDATE pipeline_runs SET status='queued',current_stage=?,resource_wait=? WHERE id=? AND status IN ('queued','running')", visit.Stage, string(encoded), visit.RunID)); err != nil {
return ResourceLease{}, nil, nil, err
}
if err := tx.Commit(); err != nil {
return ResourceLease{}, nil, nil, err
}
if s.resourceWaiters == nil {
s.resourceWaiters = make(map[ResourceVisit]chan ResourceLease)
}
ch := make(chan ResourceLease, 1)
s.resourceWaiters[visit] = ch
return ResourceLease{}, ch, wait, nil
}
func (s *Store) resourceWait(ctx context.Context, id int64) (*protocol.ResourceWait, error) {
var wait protocol.ResourceWait
err := s.db.QueryRowContext(ctx, "SELECT resource_key,stage_key,sequence,queued_at FROM resource_waiters WHERE run_id=?", id).Scan(&wait.Resource, &wait.Stage, &wait.Sequence, &wait.QueuedSince)
if err == sql.ErrNoRows {
return nil, nil
}
return &wait, err
}
// CancelResource removes a queued visit, or returns a concurrently reserved grant
// to the queue before any execution is allowed. It cannot cancel another visit.
func (s *Store) CancelResource(ctx context.Context, visit ResourceVisit) error {
s.resourceMu.Lock()
defer s.resourceMu.Unlock()
defer delete(s.resourceWaiters, visit)
tx, err := s.db.BeginTx(ctx, nil)
if err != nil {
return err
}
defer tx.Rollback()
deleted, err := tx.ExecContext(ctx, "DELETE FROM resource_waiters WHERE resource_key=? AND run_id=? AND stage_key=? AND sequence=?", visit.Resource, visit.RunID, visit.Stage, visit.Sequence)
if err != nil {
return err
}
n, err := deleted.RowsAffected()
if err != nil {
return err
}
if n > 0 {
if _, err := tx.ExecContext(ctx, "UPDATE pipeline_runs SET resource_wait='' WHERE id=?", visit.RunID); err != nil {
return err
}
}
ch := s.resourceWaiters[visit]
// A grant is buffered while this registry entry exists. Cancellation can
// reclaim only that not-yet-accepted capability, never an executing owner.
var grant ResourceLease
if ch != nil {
select {
case grant = <-ch:
default:
}
}
if err := tx.Commit(); err != nil {
return err
}
if grant.token != "" {
if err := s.SetResourceCleanup(ctx, grant, true); err != nil {
return err
}
return s.releaseResourceLocked(ctx, grant)
}
return nil
}
// AcceptResource removes the channel exactly once. Cancellation is rechecked by
// the runner before BeginStep; a canceled accepted grant is released as unused.
func (s *Store) AcceptResource(lease ResourceLease) {
s.resourceMu.Lock()
delete(s.resourceWaiters, lease.ResourceVisit)
s.resourceMu.Unlock()
}
// HasResourceOwner is used only for orphan/restart reconciliation, never for a
// resource-free stage visit.
func (s *Store) HasResourceOwner(ctx context.Context, runID int64) (bool, error) {
var exists bool
err := s.db.QueryRowContext(ctx, "SELECT EXISTS(SELECT 1 FROM resource_owners WHERE run_id=?)", runID).Scan(&exists)
return exists, err
}
func (s *Store) ConfirmRecoveredSession(ctx context.Context, owner SessionOwner) error {
_, err := s.db.ExecContext(ctx, "UPDATE run_steps SET agent_active=0 WHERE run_id=? AND session_id=?", owner.RunID, owner.ID)
return err
}
// SetResourceCleanup records positive quiescence durably before release. False
// quarantines the owner; neither age nor another run can remove the lease.
func (s *Store) SetResourceCleanup(ctx context.Context, lease ResourceLease, confirmed bool) error {
diagnostic := ""
if !confirmed {
diagnostic = "Cleanup/restoration unconfirmed; resource quarantined pending inspected reconciliation"
}
return affected(s.db.ExecContext(ctx, `UPDATE resource_owners SET cleanup_confirmed=?,diagnostic=? WHERE resource_key=? AND run_id=? AND stage_key=? AND sequence=? AND token=?`, confirmed, diagnostic, lease.Resource, lease.RunID, lease.Stage, lease.Sequence, lease.token))
}
// ReleaseResource requires the complete capability AND positive cleanup evidence.
// Mismatches and repeated releases are harmless, including after transfer.
func (s *Store) ReleaseResource(ctx context.Context, lease ResourceLease) error {
s.resourceMu.Lock()
defer s.resourceMu.Unlock()
return s.releaseResourceLocked(ctx, lease)
}
func (s *Store) releaseResourceLocked(ctx context.Context, lease ResourceLease) error {
tx, err := s.db.BeginTx(ctx, nil)
if err != nil {
return err
}
defer tx.Rollback()
n, err := deleteResourceOwner(ctx, tx, lease)
if err != nil {
return err
}
if n == 0 {
return nil
}
var next ResourceLease
next.Resource = lease.Resource
err = tx.QueryRowContext(ctx, `SELECT run_id,stage_key,sequence,token FROM resource_waiters WHERE resource_key=? ORDER BY queue_order LIMIT 1`, lease.Resource).Scan(&next.RunID, &next.Stage, &next.Sequence, &next.token)
if err != nil && err != sql.ErrNoRows {
return err
}
if err == nil {
if s.resourceWaiters[next.ResourceVisit] == nil {
return fmt.Errorf("resource waiter has no live runner; refusing handoff")
}
if _, err := tx.ExecContext(ctx, `INSERT INTO resource_owners(resource_key,run_id,stage_key,sequence,token) VALUES(?,?,?,?,?)`, next.Resource, next.RunID, next.Stage, next.Sequence, next.token); err != nil {
return err
}
if _, err := tx.ExecContext(ctx, "DELETE FROM resource_waiters WHERE run_id=?", next.RunID); err != nil {
return err
}
if _, err := tx.ExecContext(ctx, "UPDATE pipeline_runs SET resource_wait='' WHERE id=?", next.RunID); err != nil {
return err
}
}
if err := tx.Commit(); err != nil {
return err
}
if next.token != "" {
s.resourceWaiters[next.ResourceVisit] <- next
}
return nil
}
func deleteResourceOwner(ctx context.Context, tx *sql.Tx, lease ResourceLease) (int64, error) {
result, err := tx.ExecContext(ctx, `DELETE FROM resource_owners WHERE resource_key=? AND run_id=? AND stage_key=? AND sequence=? AND token=? AND cleanup_confirmed=1`, lease.Resource, lease.RunID, lease.Stage, lease.Sequence, lease.token)
if err != nil {
return 0, err
}
return result.RowsAffected()
}
// recoverResourceOwners runs only at startup, after pending waiters have been
// interrupted and removed. Every reclamation uses the persisted capability;
// absent cleanup evidence is never replaced with a timestamp-based expiry.
func recoverResourceOwners(ctx context.Context, tx *sql.Tx) error {
rows, err := tx.QueryContext(ctx, "SELECT resource_key,run_id,stage_key,sequence,token FROM resource_owners WHERE cleanup_confirmed=1")
if err != nil {
return err
}
var leases []ResourceLease
for rows.Next() {
var lease ResourceLease
if err := rows.Scan(&lease.Resource, &lease.RunID, &lease.Stage, &lease.Sequence, &lease.token); err != nil {
rows.Close()
return err
}
leases = append(leases, lease)
}
err = rows.Err()
rows.Close()
if err != nil {
return err
}
for _, lease := range leases {
if _, err := deleteResourceOwner(ctx, tx, lease); err != nil {
return err
}
}
_, err = tx.ExecContext(ctx, "UPDATE resource_owners SET diagnostic='Server stopped; resource quarantined pending inspected cleanup/restoration'")
return err
}
+288
View File
@@ -0,0 +1,288 @@
package pipeline
import (
"context"
"encoding/json"
"fmt"
"path/filepath"
"strings"
"sync"
"testing"
"time"
)
func TestResourceDeclaration(t *testing.T) {
base := Template([]string{"work"}, true)
for _, value := range []string{`{"exclusive":"phone-X"}`, `{ "exclusive" : "A_1" }`} {
graph, err := Parse(base + "```resources\n" + value + "\n```\n")
if err != nil || graph.Stages[0].Resources == nil {
t.Fatalf("%s: %v", value, err)
}
if _, ok := StageDefinition(graph.Stages[0]).Blocks["resources"]; ok {
t.Fatal("runner control reached executor")
}
encoded, _ := json.Marshal(graph)
if !strings.Contains(string(encoded), `"resources"`) {
t.Fatal("resource lost in persisted graph")
}
}
for _, value := range []string{`{}`, `null`, `[]`, `{"exclusive":""}`, `{"exclusive":"a b"}`, `{"exclusive":"a","exclusive":"b"}`, `{"exclusive":"a","extra":1}`, `{"exclusive":["a","b"]}`, fmt.Sprintf(`{"exclusive":%q}`, strings.Repeat("a", 129))} {
if _, err := Parse(base + "```resources\n" + value + "\n```\n"); err == nil {
t.Fatalf("accepted %s", value)
}
}
human := "# stage gate\n```prompt\nChoose\n```\n```configs\ntype=human\n```\n```edges\ngo -> #success\n```\n```resources\n{\"exclusive\":\"phone\"}\n```\n"
if _, err := Parse(human); err == nil {
t.Fatal("human acquired a resource")
}
}
func resourceStore(t *testing.T) (*Store, func(string) ResourceVisit) {
t.Helper()
ctx := context.Background()
s, err := Open(filepath.Join(t.TempDir(), "resources.db"))
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { s.Close() })
if _, err := s.Submit(ctx, "resource", Template([]string{"work"}, true)); err != nil {
t.Fatal(err)
}
items, _ := s.List(ctx)
d, _ := s.Get(ctx, items[0].ID, 1)
count := 0
return s, func(resource string) ResourceVisit {
count++
id, _, err := s.CreateRun(ctx, d, fmt.Sprint(count), "workspace", false, nil, nil)
if err != nil {
t.Fatal(err)
}
return ResourceVisit{Resource: resource, RunID: id, Stage: "work", Sequence: 1}
}
}
func acquireTestResource(t *testing.T, s *Store, visit ResourceVisit) ResourceLease {
t.Helper()
l, ch, _, err := s.RequestResource(context.Background(), visit)
if err != nil || ch != nil {
t.Fatalf("acquire: %v %v", ch, err)
}
return l
}
func releaseTestResource(t *testing.T, s *Store, lease ResourceLease) {
t.Helper()
if err := s.SetResourceCleanup(context.Background(), lease, true); err != nil {
t.Fatal(err)
}
if err := s.ReleaseResource(context.Background(), lease); err != nil {
t.Fatal(err)
}
}
func TestResourceFIFOOwnershipCancellationAndRegistry(t *testing.T) {
s, create := resourceStore(t)
ctx := context.Background()
a := acquireTestResource(t, s, create("X"))
duplicate := acquireTestResource(t, s, a.ResourceVisit)
if duplicate != a {
t.Fatal("duplicate acquisition changed capability")
}
var visits []ResourceVisit
var channels []<-chan ResourceLease
for i := 0; i < 4; i++ {
v := create("X")
_, ch, wait, err := s.RequestResource(ctx, v)
if err != nil || ch == nil || wait == nil {
t.Fatal("missing waiter", err)
}
_, again, _, err := s.RequestResource(ctx, v)
if err != nil || again != ch {
t.Fatal("duplicate waiter registered", err)
}
visits = append(visits, v)
channels = append(channels, ch)
}
if err := s.ReleaseResource(ctx, a); err != nil {
t.Fatal(err)
} // no cleanup receipt
for _, ch := range channels {
select {
case <-ch:
t.Fatal("unconfirmed release granted")
default:
}
}
if err := s.SetResourceCleanup(ctx, a, true); err != nil {
t.Fatal(err)
}
for _, corrupt := range []func(*ResourceLease){func(l *ResourceLease) { l.RunID++ }, func(l *ResourceLease) { l.Stage = "other" }, func(l *ResourceLease) { l.Sequence++ }, func(l *ResourceLease) { l.token = "wrong" }} {
bad := a
corrupt(&bad)
if err := s.ReleaseResource(ctx, bad); err != nil {
t.Fatal(err)
}
}
for _, ch := range channels {
select {
case <-ch:
t.Fatal("unconfirmed/mismatched release granted")
default:
}
}
if err := s.CancelResource(ctx, visits[1]); err != nil {
t.Fatal(err)
}
releaseTestResource(t, s, a)
for _, i := range []int{0, 2, 3} {
var grant ResourceLease
select {
case grant = <-channels[i]:
case <-time.After(time.Second):
t.Fatal("FIFO grant not delivered")
}
s.AcceptResource(grant)
if grant.ResourceVisit != visits[i] {
t.Fatal("wrong FIFO owner")
}
// An old owner's repeated release cannot release the next owner.
if err := s.ReleaseResource(ctx, a); err != nil {
t.Fatal(err)
}
for _, j := range []int{2, 3} {
if j > i {
select {
case <-channels[j]:
t.Fatal("multiple grants")
default:
}
}
}
releaseTestResource(t, s, grant)
}
if len(s.resourceWaiters) != 0 {
t.Fatal("channel registry leaked")
}
for i := 0; i < 30; i++ {
owner := acquireTestResource(t, s, create("X"))
v := create("X")
_, _, _, err := s.RequestResource(ctx, v)
if err != nil {
t.Fatal(err)
}
releaseTestResource(t, s, owner) // race window: granted but not accepted
if err := s.CancelResource(ctx, v); err != nil {
t.Fatal(err)
}
if len(s.resourceWaiters) != 0 {
t.Fatal("grant/cancel leaked channel")
}
}
for i := 0; i < 30; i++ {
owner := acquireTestResource(t, s, create("X"))
v := create("X")
if _, _, _, err := s.RequestResource(ctx, v); err != nil {
t.Fatal(err)
}
if err := s.SetResourceCleanup(ctx, owner, true); err != nil {
t.Fatal(err)
}
var wg sync.WaitGroup
wg.Add(2)
go func() {
defer wg.Done()
if err := s.ReleaseResource(ctx, owner); err != nil {
t.Error(err)
}
}()
go func() {
defer wg.Done()
if err := s.CancelResource(ctx, v); err != nil {
t.Error(err)
}
}()
wg.Wait()
if len(s.resourceWaiters) != 0 {
t.Fatal("concurrent grant/cancel leaked channel")
}
}
var owners, waiters int
s.db.QueryRow("SELECT COUNT(*) FROM resource_owners").Scan(&owners)
s.db.QueryRow("SELECT COUNT(*) FROM resource_waiters").Scan(&waiters)
if owners != 0 || waiters != 0 {
t.Fatalf("stranded owners/waiters: %d/%d", owners, waiters)
}
}
func TestResourceCancellationPersistenceFailureRemovesChannel(t *testing.T) {
s, create := resourceStore(t)
acquireTestResource(t, s, create("X"))
v := create("X")
if _, _, _, err := s.RequestResource(context.Background(), v); err != nil {
t.Fatal(err)
}
if err := s.Close(); err != nil {
t.Fatal(err)
}
if err := s.CancelResource(context.Background(), v); err == nil {
t.Fatal("closed DB cancellation claimed success")
}
if len(s.resourceWaiters) != 0 {
t.Fatal("persistence failure leaked channel")
}
}
func TestResourceRestartQuarantineAndRetention(t *testing.T) {
for _, confirmed := range []bool{false, true} {
t.Run(fmt.Sprint(confirmed), func(t *testing.T) {
s, create := resourceStore(t)
ctx := context.Background()
a := acquireTestResource(t, s, create("X"))
b := create("X")
if _, _, _, err := s.RequestResource(ctx, b); err != nil {
t.Fatal(err)
}
if confirmed {
if err := s.SetResourceCleanup(ctx, a, true); err != nil {
t.Fatal(err)
}
}
path := s.path
if err := s.Close(); err != nil {
t.Fatal(err)
}
reopened, err := Open(path)
if err != nil {
t.Fatal(err)
}
defer reopened.Close()
if err := reopened.RecoverRuns(ctx); err != nil {
t.Fatal(err)
}
if err := reopened.RecoverRuns(ctx); err != nil {
t.Fatal(err)
}
detail, err := reopened.GetRun(ctx, b.RunID)
if err != nil || detail.Status != "interrupted" || detail.ResourceWait != nil || len(detail.Steps) != 0 || !strings.Contains(detail.Diagnostic, "resource") {
t.Fatal("queued visit replayed/lost diagnostic", detail, err)
}
exists, err := reopened.HasResourceOwner(ctx, a.RunID)
if err != nil || exists == confirmed {
t.Fatal("incorrect recovery disposition", exists, err)
}
report, err := reopened.RetentionReport(ctx, RetentionPolicy{Keep: 0}, "", time.Now())
if err != nil {
t.Fatal(err)
}
protected := false
for _, p := range report.Protected {
if p.ID == a.RunID {
protected = true
}
}
if protected == confirmed {
t.Fatal("quarantined evidence not protected")
}
})
}
}
+22 -7
View File
@@ -130,11 +130,12 @@ type retentionQuerier interface {
// retentionRunInfo is one pipeline_runs row plus the step aggregates protection
// needs, read in the same statement so the report never issues a per-run query.
type retentionRunInfo struct {
id int64
status string
createdAt time.Time
waitingSteps int
activeSteps int
id int64
status string
createdAt time.Time
waitingSteps int
activeSteps int
resourceOwners int
}
// protectionReasons lists why a run must not be removed for a reason intrinsic
@@ -155,6 +156,9 @@ func (r retentionRunInfo) protectionReasons() []string {
if r.activeSteps > 0 {
reasons = append(reasons, "cleanup agent still active (agent_active=1)")
}
if r.resourceOwners > 0 {
reasons = append(reasons, "resource ownership retained (possibly quarantined)")
}
return reasons
}
@@ -171,9 +175,20 @@ type retentionSelection struct {
// report is a bounded number of round trips rather than one query per run.
func selectRetentionRuns(ctx context.Context, q retentionQuerier, policy RetentionPolicy, now time.Time) (retentionSelection, error) {
var sel retentionSelection
// Maintenance is deliberately non-migrating and may inspect a database
// whose server predates resource support.
var resources bool
if err := q.QueryRowContext(ctx, "SELECT EXISTS(SELECT 1 FROM sqlite_master WHERE type='table' AND name='resource_owners')").Scan(&resources); err != nil {
return sel, err
}
owners := "0"
if resources {
owners = "(SELECT COUNT(*) FROM resource_owners o WHERE o.run_id=r.id)"
}
rows, err := q.QueryContext(ctx, `SELECT r.id,r.status,strftime('%Y-%m-%dT%H:%M:%SZ',r.created_at),
COALESCE((SELECT COUNT(*) FROM run_steps s WHERE s.run_id=r.id AND s.status=?),0),
COALESCE((SELECT COUNT(*) FROM run_steps s WHERE s.run_id=r.id AND s.agent_active=1),0) `+
COALESCE((SELECT COUNT(*) FROM run_steps s WHERE s.run_id=r.id AND s.agent_active=1),0),
`+owners+` `+
runFrom+` ORDER BY r.id DESC`, protocol.StepStatusWaiting)
if err != nil {
return sel, err
@@ -182,7 +197,7 @@ COALESCE((SELECT COUNT(*) FROM run_steps s WHERE s.run_id=r.id AND s.agent_activ
for rows.Next() {
var run retentionRunInfo
var created string
if err := rows.Scan(&run.id, &run.status, &created, &run.waitingSteps, &run.activeSteps); err != nil {
if err := rows.Scan(&run.id, &run.status, &created, &run.waitingSteps, &run.activeSteps, &run.resourceOwners); err != nil {
return sel, err
}
run.createdAt = parseRetentionTime(created)
+8 -3
View File
@@ -21,14 +21,19 @@ const runStamps = `strftime('%Y-%m-%dT%H:%M:%SZ',r.created_at),COALESCE(strftime
const runSelect = `SELECT r.id,r.pipeline_id,p.name,r.version,v.content_hash,r.status,r.outcome,r.current_stage,
` + runStamps + `,r.diagnostic,
CASE WHEN r.status='succeeded' THEN r.output_json ELSE COALESCE((SELECT output_json FROM run_steps WHERE run_id=r.id AND status='completed' ORDER BY sequence DESC LIMIT 1),'{}') END,
(SELECT COUNT(*) FROM run_steps WHERE run_id=r.id AND status='completed'),r.workspace,r.title,r.model,r.stage_models
(SELECT COUNT(*) FROM run_steps WHERE run_id=r.id AND status='completed'),r.workspace,r.title,r.model,r.stage_models,r.resource_wait
` + runFrom + ` JOIN pipeline_versions v ON v.pipeline_id=r.pipeline_id AND v.version=r.version`
func scanRun(row interface{ Scan(...any) error }, r *protocol.RunSummary) error {
var output, stageModels string
if err := row.Scan(&r.ID, &r.PipelineID, &r.Name, &r.Version, &r.ContentHash, &r.Status, &r.Outcome, &r.CurrentStage, &r.CreatedAt, &r.FinishedAt, &r.Diagnostic, &output, &r.CompletedSteps, &r.Workspace, &r.Title, &r.Model, &stageModels); err != nil {
var output, stageModels, resourceWait string
if err := row.Scan(&r.ID, &r.PipelineID, &r.Name, &r.Version, &r.ContentHash, &r.Status, &r.Outcome, &r.CurrentStage, &r.CreatedAt, &r.FinishedAt, &r.Diagnostic, &output, &r.CompletedSteps, &r.Workspace, &r.Title, &r.Model, &stageModels, &resourceWait); err != nil {
return err
}
if resourceWait != "" {
if err := json.Unmarshal([]byte(resourceWait), &r.ResourceWait); err != nil {
return err
}
}
if stageModels != "" && stageModels != "{}" {
if err := json.Unmarshal([]byte(stageModels), &r.StageModels); err != nil {
return err
+2 -2
View File
@@ -44,7 +44,7 @@ func (s *Store) BeginStep(ctx context.Context, id int64, sequence int, key strin
return err
}
defer tx.Rollback()
if err := affected(tx.ExecContext(ctx, "UPDATE pipeline_runs SET status='running',current_stage=? WHERE id=? AND status IN ('queued','running')", key, id)); err != nil {
if err := affected(tx.ExecContext(ctx, "UPDATE pipeline_runs SET status='running',current_stage=?,resource_wait='' WHERE id=? AND status IN ('queued','running')", key, id)); err != nil {
return err
}
if _, err := tx.ExecContext(ctx, "INSERT INTO run_steps(run_id,sequence,stage_key,input_json) VALUES (?,?,?,?)", id, sequence, key, string(encoded)); err != nil {
@@ -310,5 +310,5 @@ func (s *Store) FinishRun(ctx context.Context, id int64, status, outcome, diagno
if err != nil {
return err
}
return affected(s.db.ExecContext(ctx, "UPDATE pipeline_runs SET status=?,outcome=?,diagnostic=?,output_json=?,finished_at=CURRENT_TIMESTAMP WHERE id=? AND status IN ('queued','running')", status, outcome, diagnostic, string(data), id))
return affected(s.db.ExecContext(ctx, "UPDATE pipeline_runs SET status=?,outcome=?,diagnostic=?,output_json=?,finished_at=CURRENT_TIMESTAMP,resource_wait='' WHERE id=? AND status IN ('queued','running')", status, outcome, diagnostic, string(data), id))
}
+30 -2
View File
@@ -51,13 +51,28 @@ CREATE TABLE IF NOT EXISTS run_session_owners (
run_id INTEGER NOT NULL, sequence INTEGER NOT NULL, backend TEXT NOT NULL, reference TEXT NOT NULL DEFAULT '',
PRIMARY KEY(run_id,sequence), FOREIGN KEY(run_id,sequence) REFERENCES run_steps(run_id,sequence));
CREATE TABLE IF NOT EXISTS run_retries (
run_id INTEGER PRIMARY KEY REFERENCES pipeline_runs(id), original_run_id INTEGER NOT NULL REFERENCES pipeline_runs(id));`)
run_id INTEGER PRIMARY KEY REFERENCES pipeline_runs(id), original_run_id INTEGER NOT NULL REFERENCES pipeline_runs(id));
CREATE TABLE IF NOT EXISTS resource_owners (
resource_key TEXT PRIMARY KEY, run_id INTEGER NOT NULL REFERENCES pipeline_runs(id),
stage_key TEXT NOT NULL, sequence INTEGER NOT NULL, token TEXT NOT NULL,
cleanup_confirmed INTEGER NOT NULL DEFAULT 0, diagnostic TEXT NOT NULL DEFAULT '',
acquired_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP);
CREATE INDEX IF NOT EXISTS resource_owner_run ON resource_owners(run_id);
CREATE UNIQUE INDEX IF NOT EXISTS resource_owner_visit ON resource_owners(run_id);
CREATE TABLE IF NOT EXISTS resource_waiters (
queue_order INTEGER PRIMARY KEY AUTOINCREMENT, resource_key TEXT NOT NULL,
run_id INTEGER NOT NULL UNIQUE REFERENCES pipeline_runs(id), stage_key TEXT NOT NULL,
sequence INTEGER NOT NULL, token TEXT NOT NULL, queued_at TEXT NOT NULL);
CREATE INDEX IF NOT EXISTS resource_wait_fifo ON resource_waiters(resource_key,queue_order);`)
if err != nil {
return err
}
if err := s.migrateRunTitles(); err != nil {
return err
}
if err := s.addColumnIfMissing("pipeline_runs", "resource_wait", "TEXT NOT NULL DEFAULT ''"); err != nil {
return err
}
return s.migrateRunModels()
}
@@ -122,16 +137,29 @@ func (s *Store) addColumnIfMissing(table, column, definition string) error {
}
func (s *Store) RecoverRuns(ctx context.Context) error {
s.resourceMu.Lock()
defer s.resourceMu.Unlock()
tx, err := s.db.BeginTx(ctx, nil)
if err != nil {
return err
}
defer tx.Rollback()
if _, err := tx.ExecContext(ctx, "UPDATE run_steps SET status='interrupted',agent_active=0,diagnostic='Server stopped; stage was not resumed',finished_at=CURRENT_TIMESTAMP WHERE (status='running' OR agent_active=1) AND status<>?", protocol.StepStatusWaiting); err != nil {
// No replay: pending visits never began. Uncertain owners survive recovery,
// including the crash window before BeginStep/session registration.
if _, err := tx.ExecContext(ctx, "UPDATE pipeline_runs SET diagnostic='Server stopped while queued for a resource; pending visit was not resumed',status='interrupted',finished_at=CURRENT_TIMESTAMP,resource_wait='' WHERE id IN (SELECT run_id FROM resource_waiters); DELETE FROM resource_waiters"); err != nil {
return err
}
if err := recoverResourceOwners(ctx, tx); err != nil {
return err
}
if _, err := tx.ExecContext(ctx, "UPDATE run_steps SET status='interrupted',agent_active=CASE WHEN EXISTS(SELECT 1 FROM resource_owners o WHERE o.run_id=run_steps.run_id AND o.sequence=run_steps.sequence) THEN agent_active ELSE 0 END,diagnostic='Server stopped; stage was not resumed',finished_at=CURRENT_TIMESTAMP WHERE (status='running' OR agent_active=1) AND status<>?", protocol.StepStatusWaiting); err != nil {
return err
}
if _, err := tx.ExecContext(ctx, "UPDATE pipeline_runs SET status='interrupted',diagnostic='Server stopped; run was not resumed',finished_at=CURRENT_TIMESTAMP WHERE status IN ('queued','running') AND status<>?", protocol.RunStatusWaiting); err != nil {
return err
}
if _, err := tx.ExecContext(ctx, "UPDATE pipeline_runs SET diagnostic='Server stopped; resource ownership quarantined: inspect retained session, tool termination and restoration before reconciliation' WHERE id IN (SELECT run_id FROM resource_owners)"); err != nil {
return err
}
return tx.Commit()
}
+1
View File
@@ -18,6 +18,7 @@ func StageDefinition(s Stage) stage.Definition {
delete(blocks, "input") // Retired contracts in historical graph JSON are ignored.
delete(blocks, "limits")
delete(blocks, "memory")
delete(blocks, "resources")
for name, value := range map[string]string{"output": s.Output, "prompt": s.Prompt} {
if value != "" {
blocks[name] = value
+6 -4
View File
@@ -18,10 +18,12 @@ import (
)
type Store struct {
db *sql.DB
dir string
path string
mu sync.Mutex
db *sql.DB
dir string
path string
mu sync.Mutex
resourceMu sync.Mutex
resourceWaiters map[ResourceVisit]chan ResourceLease
}
// openDB opens one SQLite connection pool for a DSN. Every open path shares it
+6
View File
@@ -19,6 +19,7 @@ const (
)
type Stage struct {
Resources *StageResources `json:"resources,omitempty"`
Limits *StageLimits `json:"limits,omitempty"`
Memory map[string]string `json:"memory,omitempty"`
Notify *NotificationPolicy `json:"notify,omitempty"`
@@ -33,6 +34,11 @@ type Stage struct {
Blocks map[string]string `json:"blocks,omitempty"`
}
// StageResources declares one capacity-one resource, local to this store authority.
type StageResources struct {
Exclusive string `json:"exclusive"`
}
// Limits are enforced by the runner and never included in the model prompt.
type StageLimits struct {
MaxVisits int `json:"max_visits"`
+29
View File
@@ -0,0 +1,29 @@
package protocol
import (
"encoding/json"
"strings"
"testing"
)
func TestResourceWaitOlderClientCompatibility(t *testing.T) {
data, err := json.Marshal(RunSummary{ID: 1, Status: RunStatusQueued, ResourceWait: &ResourceWait{Resource: "X", Stage: "review", Sequence: 2, QueuedSince: "now"}})
if err != nil {
t.Fatal(err)
}
var old struct {
ID int64 `json:"id"`
Status string `json:"status"`
}
if err := json.Unmarshal(data, &old); err != nil || old.Status != "queued" {
t.Fatal("old client no longer waits", old, err)
}
clear, err := json.Marshal(RunEventFrame{RunID: 1, ResourceWaitSet: true})
if err != nil || !strings.Contains(string(clear), `"resource_wait_set":true`) {
t.Fatal("no explicit clear", string(clear), err)
}
var current RunSummary
if err := json.Unmarshal([]byte(`{"id":2,"status":"running"}`), &current); err != nil || current.ResourceWait != nil {
t.Fatal("missing field changed old payload", err)
}
}
+5 -2
View File
@@ -24,8 +24,11 @@ const (
// from the frame instead of re-reading the whole list. They repeat on every
// frame (pipeline identity never changes mid-run).
type RunEventFrame struct {
RunID int64 `json:"run_id"`
Workspace string `json:"workspace,omitempty"`
// ResourceWaitSet makes a nil ResourceWait an explicit cache clear.
ResourceWaitSet bool `json:"resource_wait_set,omitempty"`
ResourceWait *ResourceWait `json:"resource_wait,omitempty"`
RunID int64 `json:"run_id"`
Workspace string `json:"workspace,omitempty"`
// PipelineID, Name, Title and Version identify the run's pipeline and its
// operator-facing title, so a new run's row can be built without a fetch.
PipelineID int64 `json:"pipeline_id,omitempty"`
+25 -3
View File
@@ -118,9 +118,10 @@ type SessionExport struct {
}
type RunSummary struct {
ID int64 `json:"id"`
PipelineID int64 `json:"pipeline_id"`
Name string `json:"name"`
ResourceWait *ResourceWait `json:"resource_wait,omitempty"`
ID int64 `json:"id"`
PipelineID int64 `json:"pipeline_id"`
Name string `json:"name"`
// Title is the operator-facing, bounded single-line description of what the
// run is for, generated from its prompt. It is never empty for a new run;
// historical rows predating the title column may omit it.
@@ -145,6 +146,19 @@ type RunSummary struct {
StageModels map[string]string `json:"stage_models,omitempty"`
}
// ResourceWait is bounded public queue metadata. Ownership credentials stay private.
type ResourceWait struct {
Resource string `json:"resource"`
Stage string `json:"stage"`
Sequence int `json:"sequence"`
QueuedSince string `json:"queued_since"`
}
// ResourceCleanupConfirmedKey lets a stage report failed restoration without
// pretending that tool idleness also proves the resource is ready. An explicit
// false (or malformed value) vetoes release; it never authorizes release alone.
const ResourceCleanupConfirmedKey = "resource_cleanup_confirmed"
// Label is the operator-facing name of a run: the generated Title when the
// server stored one, otherwise the pipeline Name (historical rows predate
// titles). It is the single definition shared by the CLI and TUI.
@@ -155,6 +169,14 @@ func (r RunSummary) Label() string {
return r.Name
}
// StatusLabel separates resource contention from a human decision everywhere.
func (r RunSummary) StatusLabel() string {
if r.ResourceWait != nil {
return "Waiting for resource " + r.ResourceWait.Resource
}
return r.Status
}
type RunStep struct {
Control *ControlEvidence `json:"control,omitempty"`
FileChanges *StageFileChanges `json:"file_changes,omitempty"`
+55
View File
@@ -105,6 +105,61 @@ func TestArtifactsScanMissingDirectoryIsEmpty(t *testing.T) {
}
}
func TestArtifactAccessUsesOnlyCurrentAndEarlierVisits(t *testing.T) {
root := t.TempDir()
r := &runner{artifactsRoot: root}
for _, directory := range []string{"7/1", "7/2", "7/3", "8/1"} {
if err := os.MkdirAll(filepath.Join(root, directory), 0700); err != nil {
t.Fatal(err)
}
}
access := r.artifactAccess(7, 2, []string{"1", "3", "../8/1"}, filepath.Join(root, "7", "2"))
current, _ := filepath.EvalSymlinks(filepath.Join(root, "7", "2"))
prior, _ := filepath.EvalSymlinks(filepath.Join(root, "7", "1"))
if access.Write != current || len(access.Read) != 2 || access.Read[0] != access.Write || access.Read[1] != prior {
t.Fatalf("artifact scope = %#v", access)
}
if got := r.artifactAccess(7, 2, nil, filepath.Join(root, "8", "1")); got.Write != "" || len(got.Read) != 0 {
t.Fatalf("other-run current scope accepted: %#v", got)
}
if got := r.artifactAccess(7, 2, nil, filepath.Join(root, "7", "3")); got.Write != "" || len(got.Read) != 0 {
t.Fatalf("sibling current scope accepted: %#v", got)
}
if err := os.WriteFile(filepath.Join(root, "7", "4"), []byte("not a directory"), 0600); err != nil {
t.Fatal(err)
}
if got := r.artifactAccess(7, 4, nil, filepath.Join(root, "7", "4")); got.Write != "" || len(got.Read) != 0 {
t.Fatalf("file current scope accepted: %#v", got)
}
if err := os.Symlink(filepath.Join(root, "7", "2"), filepath.Join(root, "7", "5")); err != nil {
t.Fatal(err)
}
if got := r.artifactAccess(7, 5, nil, filepath.Join(root, "7", "5")); got.Write != "" || len(got.Read) != 0 {
t.Fatalf("symlinked current scope accepted: %#v", got)
}
}
func TestRunnerPassesDerivedArtifactAccessToAgentClient(t *testing.T) {
root := t.TempDir()
for _, directory := range []string{"7/1", "7/2"} {
if err := os.MkdirAll(filepath.Join(root, directory), 0700); err != nil {
t.Fatal(err)
}
}
r := &runner{artifactsRoot: root}
client := r.agentClient(7, 2, []string{"1", "3"}, t.TempDir(), filepath.Join(root, "7", "2"), &stage.OperationLog{}, nil, []string{"read", "edit"})
if client.ArtifactAccess.Write == "" || len(client.ArtifactAccess.Read) != 2 {
t.Fatalf("runner did not pass scoped artifact access: %#v", client.ArtifactAccess)
}
if client.ArtifactAccess.Read[1] == filepath.Join(root, "7", "3") {
t.Fatal("runner passed a future visit to the client")
}
piClient := r.piClient(7, 2, []string{"1", "3"}, "pi", t.TempDir(), filepath.Join(root, "7", "2"), nil, []string{"read", "edit"})
if piClient.ArtifactAccess.Write != client.ArtifactAccess.Write || len(piClient.ArtifactAccess.Read) != len(client.ArtifactAccess.Read) || piClient.ArtifactAccess.Read[1] != client.ArtifactAccess.Read[1] {
t.Fatalf("runner passed different artifact scope to Pi client: %#v", piClient.ArtifactAccess)
}
}
// runWithArtifact executes one real run whose stage writes a file into the
// runner-provided artifacts directory. It returns the live handler and the
// recorded step so tests can serve the artifact over the real HTTP endpoint.
+12
View File
@@ -32,9 +32,21 @@ type chatBroker interface {
// Permissions lists the session's pending OpenCode approval requests so the
// handler can push chat_permission frames.
Permissions(ctx context.Context, sessionID string) ([]protocol.ChatPermissionFrame, error)
// Shells returns only shell operations whose ownership is authoritatively
// verified for sessionID. It is the reconnect/readback source.
Shells(ctx context.Context, sessionID string) ([]protocol.ChatShell, error)
Reply(ctx context.Context, request protocol.ChatReplyRequest) error
}
// chatShellReleaser is implemented by brokers that retain verified shell ids for
// attributing exit events. The hub calls it only after the last observer leaves
// a room; a later attach obtains a fresh authoritative snapshot.
// This is a bounded local-cache operation: it must not perform I/O or call the
// hub, which serializes it with room replacement under its mutex.
type chatShellReleaser interface {
ReleaseShells(sessionID string)
}
// operatorRegistry is the server-side operator-session seam. *pipeline.Store
// satisfies it. It is a table of its own, separate from the pipeline stage
// ownership (SessionOwner/run_session_owners) that settles a run: chat_sessions
+11 -1
View File
@@ -133,6 +133,13 @@ func (h *chatHub) apply(frame protocol.ChatEventFrame) {
}
h.mu.Lock()
defer h.mu.Unlock()
h.applyLocked(frame)
}
func (h *chatHub) applyLocked(frame protocol.ChatEventFrame) {
if frame.SessionID == "" {
return
}
turn := h.turnLocked(frame.SessionID)
switch frame.Kind {
case protocol.ChatEventStep:
@@ -144,6 +151,9 @@ func (h *chatHub) apply(frame protocol.ChatEventFrame) {
case protocol.ChatEventShell:
// A shell frame carries no message id (it outlives the tool call that
// started it), so it is folded before the message-id gate below.
if room := h.rooms[frame.SessionID]; room != nil {
room.shellReadback.Record(frame.Shell)
}
turn.applyShell(frame.Shell)
return
}
@@ -240,7 +250,7 @@ func (t *chatTurn) applyShell(shell *protocol.ChatShell) {
if shell == nil || shell.ID == "" {
return
}
if shell.Status == protocol.ChatShellExited {
if shell.Status != protocol.ChatShellRunning {
old, ok := t.shells[shell.ID]
if !ok {
return
+2
View File
@@ -82,6 +82,8 @@ func (c *channelChat) Session(context.Context, string) (protocol.ChatSession, bo
return protocol.ChatSession{}, false, nil
}
func (c *channelChat) Shells(context.Context, string) ([]protocol.ChatShell, error) { return nil, nil }
func (c *channelChat) Permissions(context.Context, string) ([]protocol.ChatPermissionFrame, error) {
c.mu.Lock()
defer c.mu.Unlock()
+181 -5
View File
@@ -106,6 +106,11 @@ type chatQueuedPrompt struct {
acked bool
}
type chatObservation struct {
done chan struct{}
err error // published by closing done; belongs to this attempt, not the next
}
// chatRoom is one session's shared live channel. Every field is guarded by the
// owning chatHub's mutex.
type chatRoom struct {
@@ -117,6 +122,11 @@ type chatRoom struct {
queue []*chatQueuedPrompt
running bool
streaming bool
// observing is non-nil while one caller is attaching the shared broker
// stream. It prevents concurrent subscribers from opening duplicate global
// event streams before streaming can be published.
observing *chatObservation
shellReadback *opencode.ShellSnapshot
// buffering holds the turn's frames until the prompting client's ack has been
// written, so the ack always precedes the frames it acknowledges. It is set
// only while the very first turn subscribes to the firehose.
@@ -160,6 +170,81 @@ func (h *chatHub) roomLocked(sessionID string) *chatRoom {
return room
}
// ensureObserving starts the firehose even for a read-only subscriber, then
// reconciles retained shells from the endpoint's authoritative list. A bounded
// journal folds concurrent lifecycle events over that snapshot atomically. All
// observers share this reconciliation, with no broker I/O under the hub mutex.
func (h *chatHub) ensureObserving(room *chatRoom, broker chatBroker) error {
if broker == nil {
return errors.New("chat is not configured on this server")
}
h.mu.Lock()
if err := room.ctx.Err(); err != nil {
h.mu.Unlock()
return err
}
if waiting := room.observing; waiting != nil {
h.mu.Unlock()
select {
case <-waiting.done:
return waiting.err
case <-room.ctx.Done():
return room.ctx.Err()
}
}
if room.streaming {
h.mu.Unlock()
return nil
}
room.observing = &chatObservation{done: make(chan struct{})}
waiting := room.observing
room.shellReadback = &opencode.ShellSnapshot{}
h.mu.Unlock()
var observeErr error
defer func() {
h.mu.Lock()
room.shellReadback = nil
waiting.err = observeErr
room.observing = nil
close(waiting.done)
h.mu.Unlock()
}()
if err := h.startStream(room, broker); err != nil {
observeErr = err
return err
}
shells, err := broker.Shells(room.ctx, room.sessionID)
h.mu.Lock()
if room.ctx.Err() != nil || h.rooms[room.sessionID] != room {
h.mu.Unlock()
observeErr = context.Canceled
return observeErr
}
if err == nil {
shells, err = room.shellReadback.Merge(shells)
}
if err != nil {
// The firehose remains useful when its authoritative shell readback is
// temporarily unavailable. Do not turn that read failure into a chat
// failure (or claim the last retained snapshot is current): clear the
// replay cache and tell attached clients that shell visibility is limited.
// Preserve events already folded during this read, but clear stale replay.
live, mergeErr := room.shellReadback.Merge(nil)
if mergeErr == nil {
h.replaceShellsLocked(room.sessionID, live)
}
h.mu.Unlock()
// Never log broker bodies/URLs: they may contain credentials or private output.
log.Printf("chat shell readback session=%s failed (%T)", room.sessionID, err)
h.fanout(room.sessionID, protocol.Message{Type: protocol.Error, Error: "background shell visibility is limited"}, "")
return nil
}
h.replaceShellsLocked(room.sessionID, shells)
room.shellReadback = nil
h.mu.Unlock()
return nil
}
// attach registers write as one client's subscription to sessionID's stream and
// returns it, or the existing subscription when the same connection has already
// attached. It is idempotent so a client may prompt and subscribe without
@@ -209,6 +294,11 @@ func (h *chatHub) detach(sessionID string, sub *chatSub) {
if len(room.subs) == 0 {
delete(h.rooms, sessionID)
room.cancel()
// ReleaseShells is a local identity-cache operation (no broker I/O).
// Complete it before a reconnect can acquire this session's new room.
if releaser, ok := room.broker.(chatShellReleaser); ok {
releaser.ReleaseShells(sessionID)
}
}
h.mu.Unlock()
}
@@ -265,9 +355,17 @@ func (h *chatHub) broadcastUser(frame protocol.ChatEventFrame, exceptClientID st
// once it is written. A step-ended frame finishes the turn, letting the queue
// advance.
func (h *chatHub) emit(frame protocol.ChatEventFrame) {
h.apply(frame)
h.emitRoom(nil, frame)
}
func (h *chatHub) emitRoom(source *chatRoom, frame protocol.ChatEventFrame) {
h.mu.Lock()
room := h.rooms[frame.SessionID]
if source != nil && (source != room || source.ctx.Err() != nil) {
h.mu.Unlock()
return
}
h.applyLocked(frame)
buffered := false
var targets []*chatSub
if room != nil {
@@ -283,6 +381,22 @@ func (h *chatHub) emit(frame protocol.ChatEventFrame) {
if room != nil && frame.Kind == protocol.ChatEventStep && frame.Step != nil && frame.Step.Status == protocol.ChatStepEnded {
h.finishTurnLocked(room)
}
// Shell replay and live lifecycle delivery must enqueue in the same order
// as folding. Otherwise an exit can overtake an older running replay.
if frame.Kind == protocol.ChatEventShell && !buffered {
msg := protocol.Message{Type: protocol.ChatEvent, ChatEvent: &frame}
var slow []*chatSub
for _, sub := range targets {
if !enqueueChatLocked(sub, msg) {
slow = append(slow, sub)
}
}
h.mu.Unlock()
for _, sub := range slow {
sub.close()
}
return
}
h.mu.Unlock()
if buffered {
return
@@ -293,6 +407,62 @@ func (h *chatHub) emit(frame protocol.ChatEventFrame) {
}
}
// enqueueChatLocked never blocks or calls detach under the hub mutex.
func enqueueChatLocked(sub *chatSub, msg protocol.Message) bool {
select {
case <-sub.done:
return true
default:
}
select {
case sub.ch <- msg:
return true
default:
return false
}
}
func (h *chatHub) replayShells(sub *chatSub) {
h.mu.Lock()
slow := false
if h.rooms[sub.room.sessionID] == sub.room && sub.room.subs[sub.clientID] == sub {
turn := h.turnLocked(sub.room.sessionID)
for _, id := range turn.shellOrder {
shell := turn.shells[id]
frame := protocol.ChatEventFrame{SessionID: sub.room.sessionID, Kind: protocol.ChatEventShell, Shell: &shell}
if !enqueueChatLocked(sub, protocol.Message{Type: protocol.ChatEvent, ChatEvent: &frame}) {
slow = true
break
}
}
}
h.mu.Unlock()
if slow {
sub.close()
}
}
// replaceShells reconciles a successful authoritative readback. It drops stale
// retained entries (including exits missed while disconnected) and upserts the
// verified running set without fabricating a lifecycle event for foreign shells.
func (h *chatHub) replaceShells(sessionID string, shells []protocol.ChatShell) {
h.mu.Lock()
defer h.mu.Unlock()
h.replaceShellsLocked(sessionID, shells)
}
func (h *chatHub) replaceShellsLocked(sessionID string, shells []protocol.ChatShell) {
turn := h.turnLocked(sessionID)
for _, shell := range turn.shells {
turn.shrink(len(shell.Command))
}
turn.shells = nil
turn.shellOrder = nil
for i := range shells {
turn.applyShell(&shells[i])
}
}
// finishTurnLocked marks the room's turn finished and releases the turn runner's
// wait. Callers must hold h.mu.
func (h *chatHub) finishTurnLocked(room *chatRoom) {
@@ -331,14 +501,20 @@ func (h *chatHub) flushBuffered(sessionID string) {
targets = append(targets, sub)
}
}
h.mu.Unlock()
var slow []*chatSub
for i := range pending {
frame := pending[i]
msg := protocol.Message{Type: protocol.ChatEvent, ChatEvent: &frame}
for _, sub := range targets {
sub.send(msg)
if !enqueueChatLocked(sub, msg) {
slow = append(slow, sub)
}
}
}
h.mu.Unlock()
for _, sub := range slow {
sub.close()
}
}
// submitPrompt accepts one prompt: it starts the turn runner immediately when no
@@ -476,7 +652,7 @@ func (h *chatHub) runTurn(room *chatRoom, p *chatQueuedPrompt, done chan struct{
return
}
if first {
if err := h.startStream(room, broker); err != nil {
if err := h.ensureObserving(room, broker); err != nil {
h.failPrompt(room, p, err)
h.endRoomTurn(room)
return
@@ -550,7 +726,7 @@ func (h *chatHub) startStream(room *chatRoom, broker chatBroker) error {
result <- broker.Stream(ctx, room.sessionID, func() {
once.Do(func() { close(attached) })
}, func(frame protocol.ChatEventFrame) error {
h.emit(frame)
h.emitRoom(room, frame)
return nil
})
}()
@@ -0,0 +1,261 @@
package server
import (
"context"
"net/http/httptest"
"strings"
"sync/atomic"
"testing"
"time"
"git.a-wan.net/alan/pipelines/internal/protocol"
"github.com/coder/websocket"
"github.com/coder/websocket/wsjson"
)
type shellBarrierBroker struct {
*fakeChat
read chan struct{}
apply chan struct{}
events chan shellBarrierEvent
stopped chan struct{}
reads atomic.Int32
snapshot []protocol.ChatShell
}
// Exercise the real websocket handler on a loopback scratch server and a fresh
// SQLite database. A marker on the shared stream bounds the replay assertion
// without sleeps, polling or a negative-read timeout.
func TestChatShellReconciliationScratchServer(t *testing.T) {
b := newShellBarrierBroker()
b.snapshot = []protocol.ChatShell{{ID: "sh_old", Status: protocol.ChatShellRunning}}
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
srv := httptest.NewServer(HandlerWithOptions(ctx, Options{Chat: b}, newOperatorStore(t)))
defer srv.Close()
conn, _, err := websocket.Dial(ctx, "ws"+strings.TrimPrefix(srv.URL, "http")+"/connect", nil)
if err != nil {
t.Fatal(err)
}
defer conn.CloseNow()
subscribe := func(id string) {
t.Helper()
if err := wsjson.Write(ctx, conn, protocol.Message{Type: protocol.ChatSubscribe, ID: id, ChatSubscribe: &protocol.ChatQuery{SessionID: "ses_1"}}); err != nil {
t.Fatal(err)
}
}
read := func() protocol.Message {
t.Helper()
var m protocol.Message
if err := wsjson.Read(ctx, conn, &m); err != nil {
t.Fatal(err)
}
return m
}
if m := read(); m.Type != protocol.PipelineList {
t.Fatalf("server greeting = %s", m.Type)
}
subscribe("first")
if m := read(); m.ID != "first" || m.Type != protocol.ChatSubscribe {
t.Fatalf("ack = %+v", m)
}
shellBarrierWait(t, b.read)
done := make(chan struct{})
b.events <- shellBarrierEvent{protocol.ChatShell{ID: "sh_old", Status: protocol.ChatShellExited}, done}
shellBarrierWait(t, done)
if m := read(); m.ChatEvent == nil || m.ChatEvent.Shell.Status != protocol.ChatShellExited {
t.Fatalf("exit not visible: %+v", m)
}
close(b.apply)
subscribe("second")
for {
m := read()
if m.ChatEvent != nil && m.ChatEvent.Shell.Status == protocol.ChatShellRunning {
t.Fatalf("stale replay: %+v", m)
}
if m.ID == "second" {
break
}
}
done = make(chan struct{})
b.events <- shellBarrierEvent{protocol.ChatShell{ID: "sh_marker", Status: protocol.ChatShellExited}, done}
shellBarrierWait(t, done)
for {
m := read()
if m.ChatEvent == nil {
continue
}
if m.ChatEvent.Shell.Status == protocol.ChatShellRunning {
t.Fatalf("ghost visible after apply: %+v", m)
}
if m.ChatEvent.Shell.ID == "sh_marker" {
break
}
}
if b.reads.Load() != 1 {
t.Fatalf("shared observer read count = %d", b.reads.Load())
}
conn.CloseNow()
shellBarrierWait(t, b.stopped)
}
type shellBarrierEvent struct {
shell protocol.ChatShell
done chan struct{}
}
func newShellBarrierBroker() *shellBarrierBroker {
return &shellBarrierBroker{fakeChat: scriptedChat(), read: make(chan struct{}, 8), apply: make(chan struct{}), events: make(chan shellBarrierEvent), stopped: make(chan struct{})}
}
func (b *shellBarrierBroker) Shells(ctx context.Context, _ string) ([]protocol.ChatShell, error) {
b.reads.Add(1)
snapshot := append([]protocol.ChatShell(nil), b.snapshot...)
b.read <- struct{}{}
select {
case <-b.apply:
return snapshot, nil
case <-ctx.Done():
return nil, ctx.Err()
}
}
func (b *shellBarrierBroker) Stream(ctx context.Context, id string, ready func(), emit func(protocol.ChatEventFrame) error) error {
defer close(b.stopped)
ready()
for {
select {
case <-ctx.Done():
return ctx.Err()
case event := <-b.events:
if err := emit(protocol.ChatEventFrame{SessionID: id, Kind: protocol.ChatEventShell, Shell: &event.shell}); err != nil {
return err
}
close(event.done)
}
}
}
func shellBarrierWait(t *testing.T, ch <-chan struct{}) {
t.Helper()
select {
case <-ch:
case <-time.After(3 * time.Second):
t.Fatal("barrier timed out")
}
}
func TestChatShellReconciliationBarrier(t *testing.T) {
for _, status := range []string{protocol.ChatShellExited, protocol.ChatShellRunning, "timeout", "killed"} {
t.Run(status, func(t *testing.T) {
b := newShellBarrierBroker()
b.snapshot = []protocol.ChatShell{{ID: "sh_old", Status: protocol.ChatShellRunning}}
h := newChatHub()
messages := make(chan protocol.Message, 32)
sub := h.attach("ses_1", "one", "", b, func(m protocol.Message) error { messages <- m; return nil })
t.Cleanup(sub.close)
result := make(chan error, 2)
go func() { result <- h.ensureObserving(sub.room, b) }()
shellBarrierWait(t, b.read)
// A second observer must share the in-flight read, not apply an
// independently completing older/newer snapshot out of order.
go func() { result <- h.ensureObserving(sub.room, b) }()
shell := protocol.ChatShell{ID: "sh_old", Status: status}
if status == protocol.ChatShellRunning {
shell.ID = "sh_new"
shell.Command = "sleep 10"
}
done := make(chan struct{})
select {
case b.events <- shellBarrierEvent{shell, done}:
case <-time.After(time.Second):
t.Fatal("event blocked behind readback")
}
shellBarrierWait(t, done)
select {
case m := <-messages:
if m.ChatEvent == nil || m.ChatEvent.Shell == nil || m.ChatEvent.Shell.ID != shell.ID || m.ChatEvent.Shell.Status != status {
t.Fatalf("live shell = %+v", m)
}
case <-time.After(time.Second):
t.Fatal("live event not visible during read")
}
close(b.apply)
for range 2 {
select {
case err := <-result:
if err != nil {
t.Fatal(err)
}
case <-time.After(time.Second):
t.Fatal("observer stuck")
}
}
if b.reads.Load() != 1 {
t.Fatalf("redundant/out-of-order reads: %d", b.reads.Load())
}
snapshot := h.shellSnapshot("ses_1")
if status == protocol.ChatShellRunning {
if len(snapshot) != 2 {
t.Fatalf("new create lost: %+v", snapshot)
}
} else if len(snapshot) != 0 {
t.Fatalf("terminal shell resurrected: %+v", snapshot)
}
// Assert the same retained state is replayed to the client.
h.replayShells(sub)
for range len(snapshot) {
select {
case m := <-messages:
if m.ChatEvent == nil || m.ChatEvent.Shell.Status != protocol.ChatShellRunning {
t.Fatalf("replay = %+v", m)
}
case <-time.After(time.Second):
t.Fatal("replay missing")
}
}
sub.close()
shellBarrierWait(t, b.stopped)
})
}
}
func TestChatShellReconciliationCancelReconnectBarrier(t *testing.T) {
b := newShellBarrierBroker()
h := newChatHub()
sub := h.attach("ses_1", "one", "", b, func(protocol.Message) error { return nil })
results := make(chan error, 2)
go func() { results <- h.ensureObserving(sub.room, b) }()
shellBarrierWait(t, b.read)
go func() { results <- h.ensureObserving(sub.room, b) }()
sub.close()
for range 2 {
select {
case err := <-results:
if err == nil {
t.Fatal("canceled observer succeeded")
}
case <-time.After(time.Second):
t.Fatal("canceled waiter stuck")
}
}
shellBarrierWait(t, b.stopped)
next := newShellBarrierBroker()
next.snapshot = []protocol.ChatShell{{ID: "sh_new", Status: protocol.ChatShellRunning}}
close(next.apply)
newSub := h.attach("ses_1", "two", "", next, func(protocol.Message) error { return nil })
t.Cleanup(newSub.close)
if err := h.ensureObserving(newSub.room, next); err != nil {
t.Fatal(err)
}
// A late callback from the canceled stream cannot poison the new room,
// nor can a foreign-session callback from the current stream.
h.emitRoom(sub.room, protocol.ChatEventFrame{SessionID: "ses_1", Kind: protocol.ChatEventShell, Shell: &protocol.ChatShell{ID: "sh_ghost", Status: protocol.ChatShellRunning}})
h.emitRoom(newSub.room, protocol.ChatEventFrame{SessionID: "ses_foreign", Kind: protocol.ChatEventShell, Shell: &protocol.ChatShell{ID: "sh_foreign", Status: protocol.ChatShellRunning}})
got := h.shellSnapshot("ses_1")
if len(got) != 1 || got[0].ID != "sh_new" || len(h.shellSnapshot("ses_foreign")) != 0 {
t.Fatalf("reconnect state = %+v", got)
}
newSub.close()
shellBarrierWait(t, next.stopped)
}
+8 -5
View File
@@ -111,7 +111,14 @@ func handleChatSubscribe(request protocol.Message, broker chatBroker, hub *chatH
_ = write(protocol.Message{Type: protocol.Error, ID: request.ID, Error: "could not attach to session " + query.SessionID})
return
}
// Acknowledge before the stream is allowed to fan out: otherwise an immediately
// available retained shell can overtake the subscribe reply on this connection.
_ = write(protocol.Message{Type: protocol.ChatSubscribe, ID: request.ID})
if err := hub.ensureObserving(sub.room, broker); err != nil {
sub.close()
_ = write(protocol.Message{Type: protocol.Error, ID: request.ID, Error: "could not observe session " + query.SessionID})
return
}
// Replay every still-unresolved approval request the room holds, not only the
// most recent one, so a client that attaches after two requests were raised
// still sees both cards.
@@ -124,9 +131,5 @@ func handleChatSubscribe(request protocol.Message, broker chatBroker, hub *chatH
// shows them rather than nothing until they exit. The message snapshot cannot
// carry a shell — a shell outlives the turn and is not a turn — so this is the
// shell half of the resume.
for _, shell := range hub.shellSnapshot(query.SessionID) {
running := shell
frame := protocol.ChatEventFrame{SessionID: query.SessionID, Kind: protocol.ChatEventShell, Shell: &running}
sub.send(protocol.Message{Type: protocol.ChatEvent, ChatEvent: &frame})
}
hub.replayShells(sub)
}
+96 -3
View File
@@ -3,6 +3,7 @@ package server
import (
"context"
"encoding/json"
"errors"
"fmt"
"net/http"
"net/http/httptest"
@@ -52,6 +53,8 @@ type fakeChat struct {
permissionSessionID string
permissions []protocol.ChatPermissionFrame
permissionErr error
shells []protocol.ChatShell
shellErr error
sessionCalls int
sessions []protocol.ChatSession
sessionErr error
@@ -62,6 +65,8 @@ type fakeChat struct {
started chan struct{}
done chan struct{}
startOnce sync.Once
streamCalls int
readyGate <-chan struct{}
block bool
subscribed bool
promptBeforeSubscribe bool
@@ -91,13 +96,18 @@ func (f *fakeChat) Prompt(context.Context, string, string) error {
func (f *fakeChat) Stream(ctx context.Context, _ string, ready func(), emit func(protocol.ChatEventFrame) error) error {
f.mu.Lock()
f.subscribed = true
f.streamCalls++
readyGate := f.readyGate
f.mu.Unlock()
if ready != nil {
ready()
}
if f.started != nil {
f.startOnce.Do(func() { close(f.started) })
}
if readyGate != nil {
<-readyGate
}
if ready != nil {
ready()
}
if f.block {
<-ctx.Done()
if f.done != nil {
@@ -116,6 +126,42 @@ func (f *fakeChat) Stream(ctx context.Context, _ string, ready func(), emit func
return nil
}
// TestChatHubConcurrentObserversShareStream pins the start-up handoff: two
// clients that subscribe before the broker says ready still share one firehose.
func TestChatHubConcurrentObserversShareStream(t *testing.T) {
broker := scriptedChat()
broker.block = true
broker.started = make(chan struct{})
gate := make(chan struct{})
broker.readyGate = gate
hub := newChatHub()
hub.mu.Lock()
room := hub.roomLocked("ses_1")
hub.mu.Unlock()
errs := make(chan error, 2)
go func() { errs <- hub.ensureObserving(room, broker) }()
select {
case <-broker.started:
case <-time.After(time.Second):
t.Fatal("first observer did not start the broker stream")
}
go func() { errs <- hub.ensureObserving(room, broker) }()
close(gate)
for range 2 {
if err := <-errs; err != nil {
t.Fatalf("ensureObserving: %v", err)
}
}
broker.mu.Lock()
calls := broker.streamCalls
broker.mu.Unlock()
if calls != 1 {
t.Fatalf("broker Stream calls = %d, want 1", calls)
}
room.cancel()
}
func (f *fakeChat) Interrupt(context.Context, string) error {
f.mu.Lock()
defer f.mu.Unlock()
@@ -151,6 +197,12 @@ func (f *fakeChat) Reply(_ context.Context, request protocol.ChatReplyRequest) e
// Permissions returns the seeded pending requests, or blocks until the caller's
// context is canceled when permBlock is set (used to prove the poller stops on
// disconnect). Every call is counted so a test can assert polling happened.
func (f *fakeChat) Shells(context.Context, string) ([]protocol.ChatShell, error) {
f.mu.Lock()
defer f.mu.Unlock()
return append([]protocol.ChatShell(nil), f.shells...), f.shellErr
}
func (f *fakeChat) Permissions(ctx context.Context, sessionID string) ([]protocol.ChatPermissionFrame, error) {
f.mu.Lock()
f.permissionCalls++
@@ -682,6 +734,9 @@ func TestChatInterruptEndsRunningPrompt(t *testing.T) {
case r.Method == http.MethodGet && r.URL.Path == "/api/permission/request":
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte(`{"location":{"directory":"/work"},"data":[]}`))
case r.Method == http.MethodGet && r.URL.Path == "/api/shell":
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte(`{"data":[]}`))
case r.Method == http.MethodGet && r.URL.Path == "/api/event":
flusher, ok := w.(http.Flusher)
if !ok {
@@ -995,6 +1050,7 @@ func TestChatPromptStreamsShellFrame(t *testing.T) {
// shows the background process instead of nothing until it exits.
func TestChatSubscribeReplaysRunningShell(t *testing.T) {
broker := scriptedChat()
broker.shells = []protocol.ChatShell{{ID: "sh_1", Status: protocol.ChatShellRunning, Command: "sleep 120"}}
broker.script = append([]protocol.ChatEventFrame{
{SessionID: "ses_1", Kind: protocol.ChatEventShell, Shell: &protocol.ChatShell{ID: "sh_1", Status: protocol.ChatShellRunning, Command: "sleep 120"}},
}, broker.script...)
@@ -1039,6 +1095,40 @@ func TestChatSubscribeReplaysRunningShell(t *testing.T) {
}
}
// TestChatSubscribeKeepsStreamWhenShellReadbackFails ensures a temporary
// authoritative readback failure does not make chat unavailable or replay a
// stale process as running. The subscriber receives an explicit limitation
// while its broker stream remains attached for future lifecycle events.
func TestChatSubscribeKeepsStreamWhenShellReadbackFails(t *testing.T) {
broker := scriptedChat()
broker.block = true
broker.shellErr = errors.New("shell endpoint unavailable: https://user:secret@example.invalid private response body")
ctx, _, conn := dialChat(t, broker)
if err := wsjson.Write(ctx, conn, protocol.Message{Type: protocol.ChatSubscribe, ID: "sub1", ChatSubscribe: &protocol.ChatQuery{SessionID: "ses_1"}}); err != nil {
t.Fatal(err)
}
var ack protocol.Message
if err := wsjson.Read(ctx, conn, &ack); err != nil {
t.Fatal(err)
}
if ack.Type != protocol.ChatSubscribe || ack.ID != "sub1" {
t.Fatalf("subscribe ack = %+v", ack)
}
var limited protocol.Message
if err := wsjson.Read(ctx, conn, &limited); err != nil {
t.Fatal(err)
}
if limited.Type != protocol.Error || limited.Error != "background shell visibility is limited" {
t.Fatalf("limited visibility notice = %+v", limited)
}
broker.mu.Lock()
calls := broker.streamCalls
broker.mu.Unlock()
if calls != 1 {
t.Fatalf("broker Stream calls = %d, want 1", calls)
}
}
// TestChatPromptSubscribesBeforePrompt pins the subscribe-before-prompt order:
// the firehose has no replay cursor, so the subscription must be attached before
// the prompt is posted or deltas emitted in between are lost.
@@ -1079,6 +1169,9 @@ func realAdapterChat(t *testing.T, frames []string) []protocol.ChatEventFrame {
case r.Method == http.MethodGet && r.URL.Path == "/api/permission/request":
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte(`{"location":{"directory":"/work"},"data":[]}`))
case r.Method == http.MethodGet && r.URL.Path == "/api/shell":
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte(`{"data":[]}`))
case r.Method == http.MethodGet && r.URL.Path == "/api/event":
streams.Add(1)
w.Header().Set("Content-Type", "text/event-stream")
+185
View File
@@ -0,0 +1,185 @@
package server
import (
"bytes"
"context"
"errors"
"fmt"
"io"
"os"
"os/exec"
"path/filepath"
"strings"
"syscall"
"time"
"git.a-wan.net/alan/pipelines/internal/stage"
)
// verifyDelivery proves that the stage landed a complete, committed tree before
// its success can settle the run. It re-derives the capability after the stage:
// a stage result and its input cannot select either the landing path or checks.
const maxDeliveryCheckOutput = 64 << 10
const deliveryCleanupTimeout = 10 * time.Second
// verifyDelivery verifies the committed item tree in a disposable worktree
// before fast-forwarding canonical main. Settlement deliberately performs only
// non-mutating repository checks after that mutation: a declared check must
// never be the first signal that the landed tree is known-red.
func verifyDelivery(ctx context.Context, workspace string, access stage.LandingAccess, checks []string) error {
lock, err := acquireDeliveryLock(ctx, access.LockPath)
if err != nil {
return err
}
defer lock.Close()
current, err := landingAccess(workspace)
if err != nil {
return fmt.Errorf("delivery access changed: %w", err)
}
if current != access {
return fmt.Errorf("delivery access changed during stage")
}
for _, directory := range []string{workspace, access.MainWorktree} {
status, err := gitOutput(directory, "status", "--porcelain=v1", "--untracked-files=all")
if err != nil {
return fmt.Errorf("delivery checkout status: %w", err)
}
if status != "" {
return fmt.Errorf("delivery checkout is not clean: %s", strings.Split(status, "\n")[0])
}
}
itemHead, err := gitOutput(workspace, "rev-parse", "HEAD")
if err != nil {
return fmt.Errorf("delivery item commit: %w", err)
}
if err := runDeliveryCandidate(ctx, workspace, itemHead, checks); err != nil {
return err
}
mainHead, err := gitOutput(access.MainWorktree, "rev-parse", "HEAD")
if err != nil {
return fmt.Errorf("delivery main commit: %w", err)
}
if err := exec.CommandContext(ctx, "git", "-C", workspace, "merge-base", "--is-ancestor", mainHead, itemHead).Run(); err != nil {
return fmt.Errorf("delivery item commit cannot fast-forward canonical main")
}
if output, err := exec.CommandContext(ctx, "git", "-C", access.MainWorktree, "merge", "--ff-only", itemHead).CombinedOutput(); err != nil {
return fmt.Errorf("delivery fast-forward canonical main: %w: %s", err, strings.TrimSpace(string(output)))
}
for _, directory := range []string{workspace, access.MainWorktree} {
status, err := gitOutput(directory, "status", "--porcelain=v1", "--untracked-files=all")
if err != nil {
return fmt.Errorf("delivery settlement status: %w", err)
}
if status != "" {
return fmt.Errorf("delivery checkout is not clean after checks: %s", strings.Split(status, "\n")[0])
}
}
// Re-derive at the settlement boundary. A success means exactly the checked
// item commit is now reachable from the same canonical main worktree.
current, err = landingAccess(workspace)
if err != nil || current != access {
return fmt.Errorf("delivery access changed during settlement")
}
mainHead, err = gitOutput(access.MainWorktree, "rev-parse", "HEAD")
if err != nil {
return fmt.Errorf("delivery main commit: %w", err)
}
if err := exec.CommandContext(ctx, "git", "-C", workspace, "merge-base", "--is-ancestor", itemHead, mainHead).Run(); err != nil {
return fmt.Errorf("delivery item commit is not an ancestor of canonical main")
}
return nil
}
func acquireDeliveryLock(ctx context.Context, path string) (*os.File, error) {
file, err := os.OpenFile(path, os.O_CREATE|os.O_RDWR, 0600)
if err != nil {
return nil, fmt.Errorf("delivery lock: %w", err)
}
for {
err := syscall.Flock(int(file.Fd()), syscall.LOCK_EX|syscall.LOCK_NB)
if err == nil {
return file, nil
}
if !errors.Is(err, syscall.EWOULDBLOCK) && !errors.Is(err, syscall.EAGAIN) {
_ = file.Close()
return nil, fmt.Errorf("delivery lock: %w", err)
}
select {
case <-ctx.Done():
_ = file.Close()
return nil, fmt.Errorf("delivery lock: %w", ctx.Err())
case <-time.After(20 * time.Millisecond):
}
}
}
func runDeliveryCandidate(ctx context.Context, workspace, itemHead string, checks []string) error {
candidate, err := os.MkdirTemp(filepath.Dir(workspace), ".pipelines-delivery-")
if err != nil {
return fmt.Errorf("create delivery candidate: %w", err)
}
if err := os.Remove(candidate); err != nil {
return fmt.Errorf("prepare delivery candidate: %w", err)
}
if output, err := boundedCommandOutput(ctx, workspace, "worktree", "add", "--detach", candidate, itemHead); err != nil {
cleanupDeliveryCandidate(workspace, candidate)
return fmt.Errorf("create delivery candidate: %w: %s", err, strings.TrimSpace(string(output)))
}
defer cleanupDeliveryCandidate(workspace, candidate)
for _, check := range checks {
if err := runDeliveryCheck(ctx, candidate, check); err != nil {
return err
}
}
return nil
}
// cleanupDeliveryCandidate always uses its own bounded context: the stage context
// may already be canceled when a check fails or times out.
func cleanupDeliveryCandidate(workspace, candidate string) {
ctx, cancel := context.WithTimeout(context.Background(), deliveryCleanupTimeout)
defer cancel()
_ = exec.CommandContext(ctx, "git", "-C", workspace, "worktree", "remove", "--force", candidate).Run()
_ = os.RemoveAll(candidate)
}
func boundedCommandOutput(ctx context.Context, directory string, args ...string) ([]byte, error) {
cmd := exec.CommandContext(ctx, "git", append([]string{"-C", directory}, args...)...)
var output boundedDeliveryOutput
cmd.Stdout = &output
cmd.Stderr = &output
err := cmd.Run()
return []byte(output.String()), err
}
func runDeliveryCheck(ctx context.Context, directory, check string) error {
cmd := exec.CommandContext(ctx, "sh", "-c", check)
cmd.Dir = directory
var output boundedDeliveryOutput
cmd.Stdout = &output
cmd.Stderr = &output
if err := cmd.Run(); err != nil {
return fmt.Errorf("delivery check %q failed: %w: %s", check, err, output.String())
}
return nil
}
type boundedDeliveryOutput struct{ bytes.Buffer }
func (b *boundedDeliveryOutput) Write(p []byte) (int, error) {
remaining := maxDeliveryCheckOutput - b.Len()
if remaining > 0 {
_, _ = b.Buffer.Write(p[:min(remaining, len(p))])
}
return len(p), nil
}
func (b *boundedDeliveryOutput) String() string {
text := strings.TrimSpace(b.Buffer.String())
if b.Len() == maxDeliveryCheckOutput {
return text + " (output truncated)"
}
return text
}
var _ io.Writer = (*boundedDeliveryOutput)(nil)
+130
View File
@@ -0,0 +1,130 @@
package server
import (
"bytes"
"fmt"
"os"
"os/exec"
"path/filepath"
"strings"
"git.a-wan.net/alan/pipelines/internal/stage"
)
// landingAccess derives delivery paths from Git's verified worktree metadata,
// never from stage input. A missing canonical main worktree fails closed.
func landingAccess(workspace string) (stage.LandingAccess, error) {
workspace, err := canonicalDirectory(workspace)
if err != nil {
return stage.LandingAccess{}, fmt.Errorf("delivery workspace: %w", err)
}
gitDir, err := gitOutput(workspace, "rev-parse", "--git-dir")
if err != nil {
return stage.LandingAccess{}, fmt.Errorf("delivery workspace is not a Git worktree: %w", err)
}
commonDir, err := gitOutput(workspace, "rev-parse", "--git-common-dir")
if err != nil {
return stage.LandingAccess{}, fmt.Errorf("delivery repository metadata: %w", err)
}
gitDir, err = canonicalDirectory(resolveGitPath(workspace, gitDir))
if err != nil {
return stage.LandingAccess{}, fmt.Errorf("delivery Git metadata: %w", err)
}
commonDir, err = canonicalDirectory(resolveGitPath(workspace, commonDir))
if err != nil {
return stage.LandingAccess{}, fmt.Errorf("delivery common Git metadata: %w", err)
}
main, err := mainWorktree(workspace, commonDir)
if err != nil {
return stage.LandingAccess{}, err
}
// Delivery originates in a linked item worktree. Its metadata must be under
// the repository's canonical common directory, never a standalone or foreign
// checkout. Git's common directory is the bounded capability needed for refs,
// worktree state, and the landing lock.
if !pathWithin(filepath.Join(commonDir, "worktrees"), gitDir) {
return stage.LandingAccess{}, fmt.Errorf("delivery workspace is not a linked Git worktree")
}
return stage.LandingAccess{MainWorktree: main, GitMetadata: commonDir, LockPath: filepath.Join(commonDir, "pipelines-landing.lock")}, nil
}
func mainWorktree(workspace, commonDir string) (string, error) {
list, err := gitOutput(workspace, "worktree", "list", "--porcelain")
if err != nil {
return "", fmt.Errorf("list repository worktrees: %w", err)
}
for _, record := range strings.Split(strings.TrimSpace(list), "\n\n") {
lines := strings.Split(record, "\n")
if len(lines) == 0 || !strings.HasPrefix(lines[0], "worktree ") || !worktreeHasBranch(lines, "refs/heads/main") {
continue
}
candidate, err := canonicalDirectory(strings.TrimPrefix(lines[0], "worktree "))
if err != nil || candidate == workspace {
continue
}
candidateGitDir, err := gitOutput(candidate, "rev-parse", "--git-dir")
if err != nil {
continue
}
candidateGitDir, err = canonicalDirectory(resolveGitPath(candidate, candidateGitDir))
if err != nil || candidateGitDir != commonDir {
continue
}
candidateCommon, err := gitOutput(candidate, "rev-parse", "--git-common-dir")
if err != nil {
continue
}
candidateCommon, err = canonicalDirectory(resolveGitPath(candidate, candidateCommon))
if err == nil && candidateCommon == commonDir {
return candidate, nil
}
}
return "", fmt.Errorf("no canonical main worktree in the delivery repository")
}
func worktreeHasBranch(lines []string, branch string) bool {
for _, line := range lines[1:] {
if line == "branch "+branch {
return true
}
}
return false
}
func canonicalDirectory(path string) (string, error) {
resolved, err := filepath.EvalSymlinks(path)
if err != nil {
return "", err
}
info, err := os.Stat(resolved)
if err != nil {
return "", err
}
if !info.IsDir() {
return "", fmt.Errorf("not a directory")
}
return filepath.Clean(resolved), nil
}
func resolveGitPath(workspace, value string) string {
if filepath.IsAbs(value) {
return value
}
return filepath.Join(workspace, value)
}
func pathWithin(parent, child string) bool {
rel, err := filepath.Rel(parent, child)
return err == nil && rel != "." && rel != ".." && !strings.HasPrefix(rel, ".."+string(filepath.Separator))
}
func gitOutput(directory string, args ...string) (string, error) {
cmd := exec.Command("git", append([]string{"-C", directory}, args...)...)
var stderr bytes.Buffer
cmd.Stderr = &stderr
data, err := cmd.Output()
if err != nil {
return "", fmt.Errorf("git %s: %w: %s", strings.Join(args, " "), err, strings.TrimSpace(stderr.String()))
}
return strings.TrimSpace(string(data)), nil
}
+304
View File
@@ -0,0 +1,304 @@
package server
import (
"context"
"os"
"os/exec"
"path/filepath"
"strings"
"testing"
"git.a-wan.net/alan/pipelines/internal/pipeline"
"git.a-wan.net/alan/pipelines/internal/protocol"
"git.a-wan.net/alan/pipelines/internal/stage"
)
func TestLandingAccessAcceptsOnlyVerifiedSameRepositoryMainWorktree(t *testing.T) {
root := t.TempDir()
main := filepath.Join(root, "main")
runGit(t, root, "init", "-b", "main", main)
runGit(t, main, "config", "user.email", "test@example.invalid")
runGit(t, main, "config", "user.name", "test")
if err := os.WriteFile(filepath.Join(main, "README"), []byte("base\n"), 0600); err != nil {
t.Fatal(err)
}
runGit(t, main, "add", "README")
runGit(t, main, "commit", "-m", "base")
item := filepath.Join(root, "item")
runGit(t, main, "worktree", "add", "-b", "item", item)
access, err := landingAccess(item)
if err != nil {
t.Fatal(err)
}
canonicalMain, err := canonicalDirectory(main)
if err != nil {
t.Fatal(err)
}
if access.MainWorktree != canonicalMain || access.GitMetadata != filepath.Join(canonicalMain, ".git") || access.LockPath != filepath.Join(canonicalMain, ".git", "pipelines-landing.lock") {
t.Fatalf("unexpected access: %+v", access)
}
foreign := filepath.Join(root, "foreign")
runGit(t, root, "init", "-b", "main", foreign)
if _, err := landingAccess(foreign); err == nil {
t.Fatal("foreign repository without an item worktree gained landing access")
}
if _, err := landingAccess(main); err == nil {
t.Fatal("main worktree gained linked-worktree delivery access")
}
if err := os.RemoveAll(item); err != nil {
t.Fatal(err)
}
runGit(t, main, "worktree", "prune")
if _, err := landingAccess(main); err == nil {
t.Fatal("repository without a linked delivery worktree gained landing access")
}
secondItem := filepath.Join(root, "second-item")
runGit(t, main, "worktree", "add", "-b", "second-item", secondItem)
runGit(t, main, "branch", "-m", "main", "trunk")
if _, err := landingAccess(secondItem); err == nil {
t.Fatal("repository without a canonical main worktree gained landing access")
}
}
func TestVerifyDeliveryRequiresCleanLandedMainAndPassingChecks(t *testing.T) {
root := t.TempDir()
main := filepath.Join(root, "main")
runGit(t, root, "init", "-b", "main", main)
runGit(t, main, "config", "user.email", "test@example.invalid")
runGit(t, main, "config", "user.name", "test")
if err := os.WriteFile(filepath.Join(main, "README"), []byte("base\n"), 0600); err != nil {
t.Fatal(err)
}
runGit(t, main, "add", "README")
runGit(t, main, "commit", "-m", "base")
item := filepath.Join(root, "item")
runGit(t, main, "worktree", "add", "-b", "item", item)
if err := os.WriteFile(filepath.Join(item, "delivered"), []byte("complete\n"), 0600); err != nil {
t.Fatal(err)
}
runGit(t, item, "add", "delivered")
runGit(t, item, "commit", "-m", "delivery")
runGit(t, main, "merge", "--ff-only", "item")
access, err := landingAccess(item)
if err != nil {
t.Fatal(err)
}
if err := verifyDelivery(context.Background(), item, access, []string{"test -f delivered"}); err != nil {
t.Fatalf("complete landed delivery rejected: %v", err)
}
if err := os.WriteFile(filepath.Join(item, "missing-helper"), []byte("dirty\n"), 0600); err != nil {
t.Fatal(err)
}
if err := verifyDelivery(context.Background(), item, access, []string{"true"}); err == nil {
t.Fatal("dirty task-owned helper was accepted")
}
if err := os.Remove(filepath.Join(item, "missing-helper")); err != nil {
t.Fatal(err)
}
if err := verifyDelivery(context.Background(), item, access, []string{"false"}); err == nil {
t.Fatal("failed required check was accepted")
}
if err := os.WriteFile(filepath.Join(item, "unlanded"), []byte("no\n"), 0600); err != nil {
t.Fatal(err)
}
runGit(t, item, "add", "unlanded")
runGit(t, item, "commit", "-m", "unlanded")
if err := verifyDelivery(context.Background(), item, access, []string{"test -f unlanded"}); err != nil {
t.Fatalf("complete unlanded item was not landed after candidate verification: %v", err)
}
mainHead := runGitOutput(t, main, "rev-parse", "HEAD")
itemHead := runGitOutput(t, item, "rev-parse", "HEAD")
if mainHead != itemHead {
t.Fatal("verified item was not fast-forwarded into canonical main")
}
}
func TestVerifyDeliveryFailingCandidateLeavesMainUnchanged(t *testing.T) {
root := t.TempDir()
main := filepath.Join(root, "main")
runGit(t, root, "init", "-b", "main", main)
runGit(t, main, "config", "user.email", "test@example.invalid")
runGit(t, main, "config", "user.name", "test")
if err := os.WriteFile(filepath.Join(main, "README"), []byte("base\n"), 0600); err != nil {
t.Fatal(err)
}
runGit(t, main, "add", "README")
runGit(t, main, "commit", "-m", "base")
item := filepath.Join(root, "item")
runGit(t, main, "worktree", "add", "-b", "item", item)
if err := os.WriteFile(filepath.Join(item, "candidate"), []byte("candidate\n"), 0600); err != nil {
t.Fatal(err)
}
runGit(t, item, "add", "candidate")
runGit(t, item, "commit", "-m", "candidate")
access, err := landingAccess(item)
if err != nil {
t.Fatal(err)
}
before := runGitOutput(t, main, "rev-parse", "HEAD")
if err := verifyDelivery(context.Background(), item, access, []string{"test -f candidate", "false"}); err == nil {
t.Fatal("failing candidate check landed main")
}
if after := runGitOutput(t, main, "rev-parse", "HEAD"); after != before {
t.Fatal("failing candidate check changed canonical main")
}
}
func TestRunnerSettlesSuccessfulDeliveryAfterCandidateLanding(t *testing.T) {
ctx := context.Background()
root := t.TempDir()
main := filepath.Join(root, "main")
runGit(t, root, "init", "-b", "main", main)
runGit(t, main, "config", "user.email", "test@example.invalid")
runGit(t, main, "config", "user.name", "test")
if err := os.WriteFile(filepath.Join(main, "README"), []byte("base\n"), 0600); err != nil {
t.Fatal(err)
}
runGit(t, main, "add", "README")
runGit(t, main, "commit", "-m", "base")
item := filepath.Join(root, "item")
runGit(t, main, "worktree", "add", "-b", "item", item)
if err := os.WriteFile(filepath.Join(item, "unlanded"), []byte("no\n"), 0600); err != nil {
t.Fatal(err)
}
runGit(t, item, "add", "unlanded")
runGit(t, item, "commit", "-m", "unlanded")
store, err := pipeline.Open(filepath.Join(root, "runner.db"))
if err != nil {
t.Fatal(err)
}
defer store.Close()
source := pipeline.Template([]string{"committer"}, true)
source = strings.Replace(source, "```configs", "```checks\ntrue\n```\n\n```configs", 1)
source = strings.Replace(source, "timeout=1h", "timeout=1h\ndelivery=true", 1)
if _, err := store.Submit(ctx, "delivery", source); err != nil {
t.Fatal(err)
}
items, err := store.List(ctx)
if err != nil {
t.Fatal(err)
}
r, err := newRunner(ctx, store)
if err != nil {
t.Fatal(err)
}
r.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) {
return stage.Result{Outcome: "success", Output: stage.Values{}}, nil
}
run, err := r.start(protocol.Message{PipelineID: items[0].ID, Version: 1, RequestKey: "unlanded", Workspace: item, Input: map[string]any{"prompt": "task"}})
if err != nil {
t.Fatal(err)
}
r.wg.Wait()
detail, err := store.GetRun(ctx, run.ID)
if err != nil {
t.Fatal(err)
}
if detail.Status != "succeeded" || len(detail.Steps) != 1 {
t.Fatalf("complete delivery did not settle successfully: %+v", detail)
}
if runGitOutput(t, main, "rev-parse", "HEAD") != runGitOutput(t, item, "rev-parse", "HEAD") {
t.Fatal("runner did not land the verified item commit")
}
}
func TestRunnerRejectsWrongBackendVerificationEvidence(t *testing.T) {
ctx := context.Background()
source := pipeline.Template([]string{"worker"}, true)
source = strings.Replace(source, "```configs", "```verification\nbackend=branch-api\nrevision=abc123\ncapability=controlled-state\nfixture=reconnect-foreground\n```\n\n```configs", 1)
r, store, id := submitRunner(t, ctx, source)
defer store.Close()
r.verifyTarget = func(context.Context, stage.VerificationTarget) (stage.VerificationEvidence, error) {
return stage.VerificationEvidence{Backend: "old-api", Revision: "abc123", Capability: "controlled-state", Fixture: "reconnect-foreground"}, nil
}
r.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) {
return stage.Result{Outcome: "success", Output: stage.Values{}}, nil
}
run := startRun(t, r, id, "wrong-backend-evidence")
detail, err := store.GetRun(ctx, run.ID)
if err != nil {
t.Fatal(err)
}
if detail.Status != "failed" || len(detail.Steps) != 1 || !strings.Contains(detail.Steps[0].Diagnostic, "backend") {
t.Fatalf("wrong-backend evidence settled as success: %+v", detail)
}
}
func TestRunnerIgnoresModelSuppliedVerificationEvidence(t *testing.T) {
ctx := context.Background()
source := pipeline.Template([]string{"worker"}, true)
source = strings.Replace(source, "```configs", "```verification\nbackend=branch-api\nrevision=abc123\ncapability=controlled-state\nfixture=reconnect-foreground\n```\n\n```configs", 1)
r, store, id := submitRunner(t, ctx, source)
defer store.Close()
r.verifyTarget = func(context.Context, stage.VerificationTarget) (stage.VerificationEvidence, error) {
return stage.VerificationEvidence{Backend: "branch-api", Revision: "abc123", Capability: "controlled-state", Fixture: "reconnect-foreground"}, nil
}
r.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) {
return stage.Result{Outcome: "success", Output: stage.Values{"verification": map[string]any{"backend": "old-api", "revision": "stale", "capability": "invented", "fixture": "other"}}}, nil
}
run := startRun(t, r, id, "model-supplied-evidence")
detail, err := store.GetRun(ctx, run.ID)
if err != nil {
t.Fatal(err)
}
if detail.Status != "succeeded" || len(detail.Steps) != 1 {
t.Fatalf("controlled verification did not settle: %+v", detail)
}
}
func TestMainWorktreeRejectsNearMatchAndLinkedMain(t *testing.T) {
for _, test := range []struct {
name string
branch string
}{
{name: "mainline is not main", branch: "mainline"},
{name: "linked main is not canonical", branch: "main"},
} {
t.Run(test.name, func(t *testing.T) {
root := t.TempDir()
primary := filepath.Join(root, "primary")
runGit(t, root, "init", "-b", "trunk", primary)
runGit(t, primary, "config", "user.email", "test@example.invalid")
runGit(t, primary, "config", "user.name", "test")
if err := os.WriteFile(filepath.Join(primary, "README"), []byte("base\n"), 0600); err != nil {
t.Fatal(err)
}
runGit(t, primary, "add", "README")
runGit(t, primary, "commit", "-m", "base")
item := filepath.Join(root, "item")
runGit(t, primary, "worktree", "add", "-b", "item", item)
candidate := filepath.Join(root, test.branch)
runGit(t, primary, "worktree", "add", "-b", test.branch, candidate)
commonDir, err := gitOutput(item, "rev-parse", "--git-common-dir")
if err != nil {
t.Fatal(err)
}
commonDir, err = canonicalDirectory(resolveGitPath(item, commonDir))
if err != nil {
t.Fatal(err)
}
if _, err := mainWorktree(item, commonDir); err == nil {
t.Fatalf("%s gained canonical main access", test.name)
}
})
}
}
func runGit(t *testing.T, directory string, args ...string) {
t.Helper()
cmd := exec.Command("git", append([]string{"-C", directory}, args...)...)
if output, err := cmd.CombinedOutput(); err != nil {
t.Fatalf("git %v: %v: %s", args, err, output)
}
}
func runGitOutput(t *testing.T, directory string, args ...string) string {
t.Helper()
output, err := exec.Command("git", append([]string{"-C", directory}, args...)...).Output()
if err != nil {
t.Fatalf("git %v: %v", args, err)
}
return strings.TrimSpace(string(output))
}
+154
View File
@@ -0,0 +1,154 @@
package server
import (
"bytes"
"context"
"crypto/sha256"
"encoding/json"
"fmt"
"net/http/httptest"
"os"
"path/filepath"
"reflect"
"strings"
"testing"
"time"
"git.a-wan.net/alan/pipelines/internal/cli"
"git.a-wan.net/alan/pipelines/internal/pipeline"
"git.a-wan.net/alan/pipelines/internal/protocol"
"git.a-wan.net/alan/pipelines/internal/stage"
)
// This is a real CLI/WebSocket/scratch-SQLite exercise with an explicitly FAKE
// executor. It never calls inference, a phone, or a deployed server.
func TestResourceScratchCLIWaitingThenProgress(t *testing.T) {
t.Setenv("PIPELINES_HARNESS_CONFIG", filepath.Join(t.TempDir(), "harnesses.json"))
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
store, err := pipeline.Open(filepath.Join(t.TempDir(), "scratch.db"))
if err != nil {
t.Fatal(err)
}
defer store.Close()
source := pipeline.Template([]string{"work"}, true) + "```resources\n{\"exclusive\":\"fake-phone\"}\n```\n"
if _, err := store.Submit(ctx, "scratch-resource", source); err != nil {
t.Fatal(err)
}
handler := Handler(ctx, store).(*runHandler)
started, release := make(chan struct{}), make(chan struct{})
handler.runner.execute = func(ctx context.Context, _ stage.Definition, input stage.Values) (stage.Result, error) {
if input["prompt"] == "A" {
close(started)
select {
case <-release:
case <-ctx.Done():
return stage.Result{}, ctx.Err()
}
}
return stage.Result{Outcome: "success", Output: stage.Values{"execution": "FAKE executor; no device or inference"}}, nil
}
handler.runner.events.interval = time.Millisecond
events := make(chan protocol.RunEventFrame, 64)
sub := handler.runner.events.subscribe("cli-evidence", "", nil, func(m protocol.Message) error {
if m.RunEvent != nil {
events <- *m.RunEvent
}
return nil
})
sub.ready()
defer sub.close()
srv := httptest.NewServer(handler)
defer srv.Close()
defer func() { cancel(); WaitRuns(handler) }()
endpoint := "ws" + strings.TrimPrefix(srv.URL, "http") + "/connect"
run := func(args ...string) string {
var out bytes.Buffer
app := cli.App{Out: &out, Err: &out, In: strings.NewReader("")}
if err := app.Run(ctx, append([]string{"--url", endpoint}, args...)); err != nil {
t.Fatalf("CLI %v: %v\n%s", args, err, out.String())
}
t.Logf("pipelines %s\n%s", strings.Join(args, " "), out.String())
return out.String()
}
parseRun := func(output string) int64 {
var m protocol.Message
if err := json.Unmarshal([]byte(output), &m); err != nil || m.RunDetail == nil {
t.Fatal("CLI run response", output, err)
}
return m.RunDetail.ID
}
run("validate", "testdata/android-resource-draft.md", "--json")
a := parseRun(run("run", "scratch-resource", "A", "--workspace", t.TempDir(), "--yes", "--json", "--request-key", "a"))
<-started
b := parseRun(run("run", "scratch-resource", "B", "--workspace", t.TempDir(), "--yes", "--json", "--request-key", "b"))
awaitResourceEvent(t, events, b, func(e protocol.RunEventFrame) bool { return e.ResourceWait != nil })
var detail protocol.RunDetail
if err := json.Unmarshal([]byte(run("inspect", fmt.Sprint(b), "--json")), &detail); err != nil || detail.ResourceWait == nil || detail.Status != "queued" || len(detail.Steps) != 0 {
t.Fatal("CLI did not observe queue", err)
}
if output := run("runs"); !strings.Contains(output, "Waiting for resource fake-phone") {
t.Fatal("list omitted resource status", output)
}
var snapshot *protocol.RunEventFrame
handleRunEvents(protocol.Message{Type: protocol.Subscribe, RunIDs: []int64{b}}, handler.runner.events, handler.runner, "snapshot-evidence", func(m protocol.Message) error {
if m.RunEvent != nil {
snapshot = m.RunEvent
}
return nil
})
handler.runner.events.unsubscribeClient("snapshot-evidence")
if snapshot == nil || snapshot.ResourceWait == nil || !snapshot.ResourceWaitSet {
t.Fatal("subscription snapshot omitted resource wait")
}
close(release)
run("wait", fmt.Sprint(a), fmt.Sprint(b), "--json")
detail = protocol.RunDetail{}
if err := json.Unmarshal([]byte(run("inspect", fmt.Sprint(b), "--json")), &detail); err != nil || detail.Status != "succeeded" || detail.ResourceWait != nil || len(detail.Steps) != 1 {
t.Fatal("automatic progress not observed", err)
}
t.Log("Observed queued resource wait followed by automatic progress; FAKE executor only, no phone/inference/deployment.")
}
func TestResourceAndroidDraftPublishedV3Provenance(t *testing.T) {
baseline, err := os.ReadFile("testdata/android-v3-published.md")
if err != nil {
t.Fatal(err)
}
if hash := fmt.Sprintf("%x", sha256.Sum256(baseline)); hash != "4a3a4b8b947eb20c115c5470e6e378f117d5fa291a7b6c8ae966a3745bc826de" {
t.Fatal("baseline is not retrieved Android v3", hash)
}
draft, err := os.ReadFile("testdata/android-resource-draft.md")
if err != nil {
t.Fatal(err)
}
old, err := pipeline.Parse(string(baseline))
if err != nil {
t.Fatal(err)
}
updated, err := pipeline.Parse(string(draft))
if err != nil {
t.Fatal(err)
}
if len(old.Stages) != len(updated.Stages) {
t.Fatal("draft changed graph shape")
}
for i := range old.Stages {
a, b := old.Stages[i], updated.Stages[i]
if b.Configs["provider"] != "openai" || b.Configs["model"] != a.Configs["model"] {
t.Fatal("draft changed approved model")
}
if b.Key == "ui_reviewer" {
if b.Resources == nil || b.Resources.Exclusive != "phone-47241FDAS00DLK" {
t.Fatal("reviewer resource missing")
}
} else if b.Resources != nil {
t.Fatal("non-UI stage acquired phone")
}
a.Prompt, b.Prompt = "", ""
a.Resources, b.Resources = nil, nil
if !reflect.DeepEqual(a, b) {
t.Fatal("draft changed fields outside approved role split/resource", a.Key)
}
}
}
+60
View File
@@ -0,0 +1,60 @@
package server
import (
"context"
"errors"
"time"
"git.a-wan.net/alan/pipelines/internal/pipeline"
"git.a-wan.net/alan/pipelines/internal/protocol"
"git.a-wan.net/alan/pipelines/internal/stage"
)
func (r *runner) acquireResource(ctx context.Context, visit pipeline.ResourceVisit, workspace string, completed int) (pipeline.ResourceLease, error) {
lease, ready, wait, err := r.store.RequestResource(ctx, visit)
if err != nil {
return lease, err
}
if ready != nil {
r.emitRun(protocol.RunEventFrame{RunID: visit.RunID, Workspace: workspace, Status: protocol.RunStatusQueued, CurrentStage: visit.Stage, CompletedSteps: completed, ResourceWaitSet: true, ResourceWait: wait}, false)
select {
case lease = <-ready:
r.store.AcceptResource(lease)
case <-ctx.Done():
cleanup, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
if err := r.store.CancelResource(cleanup, visit); err != nil {
return lease, errors.Join(ctx.Err(), err)
}
return lease, ctx.Err()
}
}
if ctx.Err() != nil {
err := r.releaseResource(lease, true) // Never launched, including a concurrent grant.
return pipeline.ResourceLease{}, errors.Join(ctx.Err(), err)
}
r.emitRun(protocol.RunEventFrame{RunID: visit.RunID, Workspace: workspace, ResourceWaitSet: true}, false)
return lease, nil
}
func (r *runner) releaseResource(lease pipeline.ResourceLease, confirmed bool) error {
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
if err := r.store.SetResourceCleanup(ctx, lease, confirmed); err != nil {
return err
}
if confirmed {
return r.store.ReleaseResource(ctx, lease)
}
return nil
}
func (r *runner) resourceQuiescent(id int64, executionErr error, panicked bool) bool {
if panicked || errors.Is(executionErr, stage.ErrStageCleanup) {
return false
}
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
sessions, err := r.store.ActiveAgentSessionsForRun(ctx, id)
return err == nil && len(sessions) == 0
}
+327
View File
@@ -0,0 +1,327 @@
package server
import (
"context"
"database/sql"
"errors"
"fmt"
"path/filepath"
"strings"
"sync/atomic"
"testing"
"time"
"git.a-wan.net/alan/pipelines/internal/pipeline"
"git.a-wan.net/alan/pipelines/internal/protocol"
"git.a-wan.net/alan/pipelines/internal/stage"
)
func resourceRunner(t *testing.T) (*runner, <-chan protocol.RunEventFrame, func(string, string, string) *protocol.RunDetail) {
t.Helper()
ctx, cancel := context.WithCancel(context.Background())
s, err := pipeline.Open(filepath.Join(t.TempDir(), "resources.db"))
if err != nil {
t.Fatal(err)
}
r, err := newRunner(ctx, s)
if err != nil {
t.Fatal(err)
}
r.events = newRunEventHub()
r.events.interval = time.Millisecond
events := make(chan protocol.RunEventFrame, 128)
sub := r.events.subscribe("resource-test", "", nil, func(m protocol.Message) error {
if m.RunEvent != nil {
events <- *m.RunEvent
}
return nil
})
sub.ready()
t.Cleanup(func() { cancel(); r.wg.Wait(); sub.close(); s.Close() })
count := 0
return r, events, func(resource, timeout, prompt string) *protocol.RunDetail {
count++
source := strings.Replace(pipeline.Template([]string{"work"}, true), "timeout=1h", "timeout="+timeout, 1)
if resource != "" {
source += "```resources\n{\"exclusive\":\"" + resource + "\"}\n```\n"
}
name := fmt.Sprint("resource-", count)
if _, err := s.Submit(ctx, name, source); err != nil {
t.Fatal(err)
}
items, _ := s.List(ctx)
var pid int64
for _, p := range items {
if p.Name == name {
pid = p.ID
}
}
d, err := r.start(protocol.Message{PipelineID: pid, Version: 1, RequestKey: name, Workspace: t.TempDir(), Input: map[string]any{"prompt": prompt}})
if err != nil {
t.Fatal(err)
}
return d
}
}
func awaitResourceEvent(t *testing.T, events <-chan protocol.RunEventFrame, id int64, predicate func(protocol.RunEventFrame) bool) protocol.RunEventFrame {
t.Helper()
timer := time.NewTimer(3 * time.Second)
defer timer.Stop()
for {
select {
case event := <-events:
if event.RunID == id && predicate(event) {
return event
}
case <-timer.C:
t.Fatalf("resource event missing for run %d", id)
return protocol.RunEventFrame{}
}
}
}
func TestResourcePauseBeyondTimeoutAndAutomaticResume(t *testing.T) {
r, events, start := resourceRunner(t)
aStarted, releaseA := make(chan struct{}), make(chan struct{})
bStarted := make(chan struct{}, 1)
var bCalls atomic.Int32
r.execute = func(ctx context.Context, _ stage.Definition, input stage.Values) (stage.Result, error) {
switch input["prompt"] {
case "A":
close(aStarted)
select {
case <-releaseA:
case <-ctx.Done():
return stage.Result{}, ctx.Err()
}
case "B":
bCalls.Add(1)
deadline, ok := ctx.Deadline()
if !ok || time.Until(deadline) < 15*time.Millisecond {
return stage.Result{}, errors.New("execution timeout was consumed in queue")
}
bStarted <- struct{}{}
}
return stage.Result{Outcome: "success", Output: stage.Values{}}, nil
}
a := start("X", "2s", "A")
<-aStarted
b := start("X", "30ms", "B")
awaitResourceEvent(t, events, b.ID, func(e protocol.RunEventFrame) bool { return e.ResourceWait != nil })
<-time.After(70 * time.Millisecond) // Deliberately longer than B's whole execution timeout.
d, err := r.store.GetRun(context.Background(), b.ID)
if err != nil || d.Status != protocol.RunStatusQueued || d.ResourceWait == nil || len(d.Steps) != 0 || bCalls.Load() != 0 {
t.Fatal("queued execution consumed accounting", d, err, bCalls.Load())
}
if _, err := r.decide(protocol.Message{RunID: b.ID, Option: "success"}); err == nil {
t.Fatal("resource wait exposed a human decision")
}
c := start("Y", "1s", "C")
awaitResourceEvent(t, events, c.ID, func(e protocol.RunEventFrame) bool { return e.Status == "succeeded" })
select {
case <-bStarted:
t.Fatal("B ran while A held X")
default:
}
owner, err := r.store.GetRun(context.Background(), a.ID)
if err != nil || owner.Status != "running" {
t.Fatal("A did not remain owner", err)
}
close(releaseA)
awaitResourceEvent(t, events, b.ID, func(e protocol.RunEventFrame) bool { return e.Status == "succeeded" })
r.wg.Wait()
d, err = r.store.GetRun(context.Background(), b.ID)
if err != nil || bCalls.Load() != 1 || d.ResourceWait != nil || len(d.Steps) != 1 || d.Steps[0].Control.Visit != 1 || d.Steps[0].Sequence != 1 {
t.Fatal("resume did not execute exactly one fresh visit", d, err)
}
}
func TestResourceCancellationWaitsForOwnerCleanup(t *testing.T) {
r, events, start := resourceRunner(t)
started, cleaning, cleanupDone := make(chan struct{}), make(chan struct{}), make(chan struct{})
var waitingCalls atomic.Int32
r.execute = func(ctx context.Context, _ stage.Definition, input stage.Values) (stage.Result, error) {
if input["prompt"] == "owner" {
close(started)
<-ctx.Done()
close(cleaning)
<-cleanupDone
return stage.Result{}, ctx.Err()
}
waitingCalls.Add(1)
return stage.Result{Outcome: "success", Output: stage.Values{}}, nil
}
a := start("X", "2s", "owner")
<-started
b := start("X", "30ms", "waiter")
awaitResourceEvent(t, events, b.ID, func(e protocol.RunEventFrame) bool { return e.ResourceWait != nil })
if err := r.cancel(a.ID); err != nil {
t.Fatal(err)
}
<-cleaning
select {
case <-time.After(50 * time.Millisecond):
if waitingCalls.Load() != 0 {
t.Fatal("handoff before cleanup")
}
}
close(cleanupDone)
awaitResourceEvent(t, events, b.ID, func(e protocol.RunEventFrame) bool { return e.Status == "succeeded" })
if waitingCalls.Load() != 1 {
t.Fatal("waiter did not resume exactly once")
}
}
func TestResourceQuarantineAndQueuedCancellation(t *testing.T) {
for _, failure := range []string{"cleanup", "panic", "restoration"} {
t.Run(failure, func(t *testing.T) {
r, events, start := resourceRunner(t)
started, release := make(chan struct{}), make(chan struct{})
var calls atomic.Int32
r.execute = func(ctx context.Context, _ stage.Definition, input stage.Values) (stage.Result, error) {
if input["prompt"] == "owner" {
close(started)
select {
case <-release:
case <-ctx.Done():
}
if failure == "panic" {
panic("unknown cleanup")
}
if failure == "restoration" {
return stage.Result{Outcome: "success", Output: stage.Values{protocol.ResourceCleanupConfirmedKey: false}}, nil
}
return stage.Result{}, stage.ErrStageCleanup
}
calls.Add(1)
return stage.Result{Outcome: "success", Output: stage.Values{}}, nil
}
a := start("X", "1s", "owner")
<-started
b := start("X", "30ms", "waiter")
awaitResourceEvent(t, events, b.ID, func(e protocol.RunEventFrame) bool { return e.ResourceWait != nil })
close(release)
awaitResourceEvent(t, events, a.ID, func(e protocol.RunEventFrame) bool { return e.Status == "failed" })
if err := r.cancel(b.ID); err != nil {
t.Fatal(err)
}
awaitResourceEvent(t, events, b.ID, func(e protocol.RunEventFrame) bool {
return e.Status == "canceled" && e.ResourceWaitSet && e.ResourceWait == nil
})
r.wg.Wait()
if calls.Load() != 0 {
t.Fatal("quarantined resource handed off")
}
exists, err := r.store.HasResourceOwner(context.Background(), a.ID)
if err != nil || !exists {
t.Fatal("quarantine evidence lost", err)
}
d, err := r.store.GetRun(context.Background(), b.ID)
if err != nil || d.ResourceWait != nil || len(d.Steps) != 0 {
t.Fatal("canceled queue consumed step", err)
}
})
}
}
func TestResourceEventClearCoalescing(t *testing.T) {
frame := protocol.RunEventFrame{ResourceWaitSet: true, ResourceWait: &protocol.ResourceWait{Resource: "X"}}
mergeRunEventFrame(&frame, &protocol.RunEventFrame{ResourceWaitSet: true})
if !frame.ResourceWaitSet || frame.ResourceWait != nil {
t.Fatal("coalescer retained stale resource wait")
}
}
func TestResourceReleaseOnBeginAndPersistenceFailure(t *testing.T) {
for _, failure := range []string{"begin", "persist"} {
t.Run(failure, func(t *testing.T) {
r, events, start := resourceRunner(t)
db, err := sql.Open("sqlite", filepath.Join(r.store.Directory(), "resources.db"))
if err != nil {
t.Fatal(err)
}
defer db.Close()
trigger := `CREATE TRIGGER resource_fail BEFORE INSERT ON run_steps WHEN json_extract(NEW.input_json,'$.prompt')='fail' BEGIN SELECT RAISE(FAIL,'test BeginStep failure'); END`
if failure == "persist" {
trigger = `CREATE TRIGGER resource_fail BEFORE UPDATE OF output_json ON run_steps WHEN json_extract(OLD.input_json,'$.prompt')='fail' BEGIN SELECT RAISE(FAIL,'test persistence failure'); END`
}
if _, err := db.Exec(trigger); err != nil {
t.Fatal(err)
}
var calls atomic.Int32
r.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) {
calls.Add(1)
return stage.Result{Outcome: "success", Output: stage.Values{}}, nil
}
a := start("X", "1s", "fail")
awaitResourceEvent(t, events, a.ID, func(e protocol.RunEventFrame) bool { return e.Status == "failed" })
r.wg.Wait()
if exists, err := r.store.HasResourceOwner(context.Background(), a.ID); err != nil || exists {
t.Fatal("safe failure stranded resource", err)
}
if failure == "begin" && calls.Load() != 0 {
t.Fatal("BeginStep failure executed work")
}
b := start("X", "1s", "pass")
awaitResourceEvent(t, events, b.ID, func(e protocol.RunEventFrame) bool { return e.Status == "succeeded" })
})
}
}
func TestResourceCleanupQueryFailureQuarantines(t *testing.T) {
r, events, start := resourceRunner(t)
r.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) {
if err := r.store.Close(); err != nil {
return stage.Result{}, err
}
return stage.Result{Outcome: "success", Output: stage.Values{}}, nil
}
a := start("X", "1s", "query-failure")
awaitResourceEvent(t, events, a.ID, func(e protocol.RunEventFrame) bool { return e.Status == "failed" })
r.wg.Wait()
reopened, err := pipeline.Open(filepath.Join(r.store.Directory(), "resources.db"))
if err != nil {
t.Fatal(err)
}
defer reopened.Close()
if err := reopened.RecoverRuns(context.Background()); err != nil {
t.Fatal(err)
}
if exists, err := reopened.HasResourceOwner(context.Background(), a.ID); err != nil || !exists {
t.Fatal("query failure released uncertain owner", err)
}
}
func TestResourceSkippedVisitDoesNotAcquire(t *testing.T) {
r, events, _ := resourceRunner(t)
ctx := context.Background()
source := strings.Replace(pipeline.Template([]string{"review"}, true), "success -> #success", "success -> #success\nskipped -> #success", 1) + "```resources\n{\"exclusive\":\"X\"}\n```\n"
if _, err := r.store.Submit(ctx, "skip", source); err != nil {
t.Fatal(err)
}
items, _ := r.store.List(ctx)
d, _ := r.store.Get(ctx, items[0].ID, 1)
owner, _, err := r.store.CreateRun(ctx, d, "held", t.TempDir(), false, nil, nil)
if err != nil {
t.Fatal(err)
}
lease, ch, _, err := r.store.RequestResource(ctx, pipeline.ResourceVisit{Resource: "X", RunID: owner, Stage: "review", Sequence: 1})
if err != nil || ch != nil {
t.Fatal("owner fixture failed", err)
}
defer r.releaseResource(lease, true)
r.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) {
t.Error("skipped stage executed")
return stage.Result{}, nil
}
run, err := r.start(protocol.Message{PipelineID: d.ID, Version: 1, RequestKey: "skip", Workspace: t.TempDir(), Input: map[string]any{"prompt": "no UI"}})
if err != nil {
t.Fatal(err)
}
awaitResourceEvent(t, events, run.ID, func(e protocol.RunEventFrame) bool { return e.Status == "succeeded" })
detail, err := r.store.GetRun(ctx, run.ID)
if err != nil || len(detail.Steps) != 1 || detail.Steps[0].Outcome != "skipped" {
t.Fatal("skip not preserved", detail, err)
}
}
+1
View File
@@ -29,6 +29,7 @@ func TestRetryCreatesFreshOpenCodeSessionAndRetainsFailedSession(t *testing.T) {
if [ "$1" = "--help" ]; then echo ' api'; exit 0; fi
printf '%%s %%s %%s\n' "$PWD" "$2" "$3" >> %q
case "$3" in
/api/session/active) echo '{"data":{}}' ;;
/api/session)
n=0; if [ -f %q ]; then n=$(cat %q); fi
n=$((n+1)); echo "$n" > %q
+17 -12
View File
@@ -424,6 +424,9 @@ func (s *runSubscription) close() {
}
func mergeRunEventFrame(into, in *protocol.RunEventFrame) {
if in.ResourceWaitSet {
into.ResourceWaitSet, into.ResourceWait = true, in.ResourceWait
}
if in.Workspace != "" {
into.Workspace = in.Workspace
}
@@ -500,18 +503,20 @@ func handleRunEvents(request protocol.Message, events *runEventHub, runs *runner
continue
}
frame := protocol.RunEventFrame{
RunID: summary.ID,
Workspace: summary.Workspace,
PipelineID: summary.PipelineID,
Name: summary.Name,
Title: summary.Title,
Version: summary.Version,
Status: summary.Status,
Outcome: summary.Outcome,
CurrentStage: summary.CurrentStage,
CompletedSteps: summary.CompletedSteps,
CreatedAt: summary.CreatedAt,
FinishedAt: summary.FinishedAt,
ResourceWaitSet: true,
ResourceWait: summary.ResourceWait,
RunID: summary.ID,
Workspace: summary.Workspace,
PipelineID: summary.PipelineID,
Name: summary.Name,
Title: summary.Title,
Version: summary.Version,
Status: summary.Status,
Outcome: summary.Outcome,
CurrentStage: summary.CurrentStage,
CompletedSteps: summary.CompletedSteps,
CreatedAt: summary.CreatedAt,
FinishedAt: summary.FinishedAt,
}
if !sub.writeSnapshot(protocol.Message{Type: protocol.RunEvent, RunEvent: &frame}) {
// The connection write failed; the subscription is closed and the
+210 -27
View File
@@ -58,8 +58,11 @@ type runner struct {
// directory. It defaults to stage.ExportSession; a failure is best-effort and
// never changes the stage's status, outcome, target or result validation.
exportSession func(context.Context, stage.SessionExportRequest) (stage.SessionExportResult, error)
wg sync.WaitGroup
schedulerWG sync.WaitGroup
// verifyTarget is installed only by server wiring for a controlled fixture
// workflow. Leaving it nil fails closed for target-aware definitions.
verifyTarget stage.VerificationRunner
wg sync.WaitGroup
schedulerWG sync.WaitGroup
// attentionMu guards attentionSent, the per-run set of run-attention events
// already delivered — by a stage's own notify policy or by the run-level
// default — so neither republishes the same genuine event.
@@ -78,12 +81,19 @@ func newRunner(ctx context.Context, s *pipeline.Store) (*runner, error) {
if owner.Backend == "pi" {
err = stage.InterruptPi(stop, owner.Reference)
} else {
err = (stage.OpenCodeClient{Backend: owner.Backend, Directory: owner.Directory, Reference: owner.Reference}).Interrupt(stop, owner.ID)
err = (stage.OpenCodeClient{Backend: owner.Backend, Directory: owner.Directory, Reference: owner.Reference}).InterruptAndConfirm(stop, owner.ID)
}
cancel()
if err != nil {
if resource, checkErr := s.HasResourceOwner(ctx, owner.RunID); checkErr == nil && resource {
log.Printf("resource owner run=%d remains quarantined: cleanup not confirmed", owner.RunID)
continue
}
return nil, fmt.Errorf("cannot interrupt previously active agent %s: %w", owner.ID, err)
}
if err := s.ConfirmRecoveredSession(ctx, owner); err != nil {
return nil, err
}
}
if err := s.RecoverRuns(ctx); err != nil {
return nil, err
@@ -213,6 +223,9 @@ func (r *runner) start(request protocol.Message, retryOf ...int64) (*protocol.Ru
// workspace) must not block new work.
for _, owner := range owners {
if _, ok := r.active[owner.RunID]; !ok {
if resource, checkErr := r.store.HasResourceOwner(r.ctx, owner.RunID); checkErr == nil && resource && owner.Directory != workspace {
continue
}
r.mu.Unlock()
return nil, fmt.Errorf("an agent may still be active; restart recovery must interrupt it before new work")
}
@@ -580,7 +593,8 @@ type runState struct {
capturedSeen map[string]bool
// completed counts the run's finished steps, so a delta can report the
// running total without re-reading the store.
completed int
completed int
artifactDirs []string
}
func newRunState(key string, input stage.Values) *runState {
@@ -606,10 +620,67 @@ func rebuildRunState(detail *protocol.RunDetail, decided protocol.RunStep) *runS
// Seed the UI-surface accumulator from the persisted evidence so a
// resumed run reaches the same skip decision with no extra query.
state.recordChanges(step.FileChanges, step.Artifacts)
// Only completed, recorded visits are eligible for a later stage's
// review capability. The path is reconstructed by the runner below.
if step.Status == "completed" {
state.artifactDirs = append(state.artifactDirs, filepath.Join("", strconv.Itoa(step.Sequence)))
}
}
return state
}
// artifactAccess derives exact visit directories; it never consumes a path from
// a definition or model result. An unreadable or symlinked root fails closed.
func (r *runner) artifactAccess(runID int64, sequence int, prior []string, current string) stage.ArtifactAccess {
root, err := filepath.EvalSymlinks(r.artifactsRoot)
if err != nil {
return stage.ArtifactAccess{}
}
runRoot := filepath.Join(root, strconv.FormatInt(runID, 10))
visitDirectory := func(visit int) (string, bool) {
if visit <= 0 {
return "", false
}
directory := filepath.Join(runRoot, strconv.Itoa(visit))
resolved, err := filepath.EvalSymlinks(directory)
if err != nil || resolved != directory {
return "", false
}
info, err := os.Stat(resolved)
if err != nil || !info.IsDir() {
return "", false
}
return resolved, true
}
access := stage.ArtifactAccess{}
if directory, ok := visitDirectory(sequence); ok {
resolved, err := filepath.EvalSymlinks(current)
if err != nil || resolved != directory {
return access
}
access.Write = directory
access.Read = append(access.Read, directory)
}
for _, priorSequence := range prior {
visit, err := strconv.Atoi(priorSequence)
if err != nil || visit <= 0 || visit >= sequence {
continue
}
if directory, ok := visitDirectory(visit); ok {
access.Read = append(access.Read, directory)
}
}
return access
}
func (r *runner) agentClient(runID int64, sequence int, prior []string, workspace, artifactsDir string, operations *stage.OperationLog, writePaths, tools []string) *stage.AgentClient {
return &stage.AgentClient{OpenCodeClient: stage.OpenCodeClient{Directory: workspace, Operations: operations, ArtifactsDir: artifactsDir}, WritePaths: writePaths, Tools: tools, ArtifactAccess: r.artifactAccess(runID, sequence, prior, artifactsDir)}
}
func (r *runner) piClient(runID int64, sequence int, prior []string, executable, workspace, artifactsDir string, writePaths, tools []string) *stage.PiClient {
return &stage.PiClient{Executable: executable, Directory: workspace, Tools: tools, WritePaths: writePaths, ArtifactsDir: artifactsDir, ArtifactAccess: r.artifactAccess(runID, sequence, prior, artifactsDir)}
}
// serverChangedFilesField is the reserved run-input memory field under which the
// runner publishes the server-captured changed paths. A stage selects it with
// `"server_changed_files":"input.server_changed_files"`, alongside another
@@ -717,13 +788,14 @@ func (r *runner) finishRunState(id int64, name, workspace, status, outcome, diag
// Push the terminal delta before the attention notification, so a slow ntfy
// never delays the live update the app folds.
r.emitRun(protocol.RunEventFrame{
RunID: id,
Workspace: workspace,
Status: status,
Outcome: outcome,
CurrentStage: currentStage,
CompletedSteps: completed,
FinishedAt: time.Now().UTC().Format(time.RFC3339),
RunID: id,
Workspace: workspace,
Status: status,
ResourceWaitSet: true,
Outcome: outcome,
CurrentStage: currentStage,
CompletedSteps: completed,
FinishedAt: time.Now().UTC().Format(time.RFC3339),
}, true)
r.notifyRun(id, name, status, sequence, currentStage, diagnostic)
// The run is terminal: nothing more can notify for it.
@@ -791,6 +863,21 @@ func (r *runner) finishStage(id int64, name, workspace string, selected protocol
}
func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, name, workspace string, request protocol.Message, state *runState) {
var held *pipeline.ResourceLease
resourceQuiet := true
release := func() error {
if held != nil {
err := r.releaseResource(*held, resourceQuiet)
held = nil
return err
}
return nil
}
defer func() {
if err := release(); err != nil {
log.Printf("resource cleanup run=%d: %v", id, err)
}
}()
finish := func(status, outcome, diagnostic string) {
r.finishRunState(id, name, workspace, status, outcome, diagnostic, state.completed, state.nextSequence-1, state.key, state.input)
}
@@ -819,7 +906,6 @@ func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, na
effectiveModels = resolved
}
for sequence := state.nextSequence; sequence <= 30; sequence++ {
state.nextSequence = sequence + 1
if ctx.Err() != nil {
finish("canceled", "", ctx.Err().Error())
return
@@ -851,24 +937,42 @@ func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, na
}
}
isHuman := stage.EffectiveType(def.Type) == stage.HumanType
state.visits[key]++
maxVisits := 5
if selected.Limits != nil {
maxVisits = selected.Limits.MaxVisits
}
// A human stage pauses for a person; neither the global visit budget nor
// the per-stage max_visits may kill it. The shared 30-step run budget
// still bounds every run.
// Validate the next visit without consuming it. An already-exhausted
// stage must not queue for a resource it can never execute.
if !isHuman {
if state.visits[key] > 5 {
if state.visits[key]+1 > 5 {
finish("failed", "error", "stage visit budget exceeded")
return
}
if state.visits[key] > maxVisits {
if state.visits[key]+1 > maxVisits {
finish("failed", "error", fmt.Sprintf("stage %s visit limit reached (%d)", key, maxVisits))
return
}
}
_, declaresSkip := def.Edges["skipped"]
skip := !isHuman && declaresSkip && !state.uiSurface
if selected.Resources != nil && !skip {
if isHuman || !pipeline.ValidResourceKey(selected.Resources.Exclusive) {
finish("failed", "error", "invalid persisted resource declaration")
return
}
lease, err := r.acquireResource(ctx, pipeline.ResourceVisit{Resource: selected.Resources.Exclusive, RunID: id, Stage: key, Sequence: sequence}, workspace, state.completed)
if err != nil {
if ctx.Err() != nil {
finish("canceled", "", err.Error())
} else {
finish("failed", "error", err.Error())
}
return
}
held, resourceQuiet = &lease, true
}
state.nextSequence = sequence + 1
state.visits[key]++
// Publish the run's server-captured changed paths into the input memory
// value before resolving bindings, so a stage selecting
// input.server_changed_files always resolves (an empty list when nothing
@@ -890,12 +994,13 @@ func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, na
}
// queued -> running: report the new current stage and the step start.
r.emitRun(protocol.RunEventFrame{
RunID: id,
Workspace: workspace,
Status: protocol.RunStatusRunning,
CurrentStage: key,
CompletedSteps: state.completed,
Step: &protocol.RunStepDelta{Sequence: sequence, Stage: key, Status: protocol.StepStatusRunning},
RunID: id,
Workspace: workspace,
Status: protocol.RunStatusRunning,
CurrentStage: key,
ResourceWaitSet: true,
CompletedSteps: state.completed,
Step: &protocol.RunStepDelta{Sequence: sequence, Stage: key, Status: protocol.StepStatusRunning},
}, false)
if isHuman {
// A human step is O(1): one BeginStep plus one MarkWaiting, no
@@ -929,7 +1034,7 @@ func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, na
// the model call and its capture work, record the step as completed and
// route along the declared edge. Only a declared route is honoured, and no
// other stage or outcome routing changes.
if _, declaresSkip := def.Edges["skipped"]; declaresSkip && !state.uiSurface {
if skip {
reason := skipReasonLine(state)
step := protocol.RunStep{
Sequence: sequence,
@@ -977,7 +1082,22 @@ func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, na
}
}
operations := &stage.OperationLog{}
client := &stage.AgentClient{OpenCodeClient: stage.OpenCodeClient{Directory: workspace, Operations: operations, ArtifactsDir: artifactsDir}, WritePaths: request.WritePaths, Tools: tools}
var landing stage.LandingAccess
delivery, deliveryErr := stage.DeliveryEnabled(def)
if deliveryErr != nil {
finish("failed", "error", deliveryErr.Error())
return
}
if delivery {
access, accessErr := landingAccess(workspace)
if accessErr != nil {
finish("failed", "error", "delivery access denied: "+accessErr.Error())
return
}
landing = access
}
client := r.agentClient(id, sequence, state.artifactDirs, workspace, artifactsDir, operations, request.WritePaths, tools)
client.LandingAccess = landing
client.OnSession = func(session string, active bool) error {
c, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
@@ -1010,6 +1130,9 @@ func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, na
snapshotCancel()
var stageSession string
panicked := false
if held != nil {
resourceQuiet = false
}
func() {
defer func() {
if recover() != nil {
@@ -1017,6 +1140,10 @@ func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, na
err = fmt.Errorf("stage execution panicked")
}
}()
if ctx.Err() != nil {
err = ctx.Err()
return
}
if r.execute != nil {
timeout, _ := stage.StageTimeout(def)
executeCtx, cancel := context.WithTimeout(stageCtx, timeout)
@@ -1033,7 +1160,7 @@ func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, na
err = fmt.Errorf("Pi is not configured on execution server")
return
}
piClient = &stage.PiClient{Executable: profile.Executable, Directory: workspace, Tools: tools, WritePaths: request.WritePaths, ArtifactsDir: artifactsDir}
piClient = r.piClient(id, sequence, state.artifactDirs, profile.Executable, workspace, artifactsDir, request.WritePaths, tools)
piClient.OnOwnership = func(session, reference string, active bool) error {
c, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
@@ -1061,6 +1188,12 @@ func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, na
result, err = stage.DefaultRegistry().Run(ctx, def, input)
}
}()
if held != nil {
resourceQuiet = r.resourceQuiescent(id, err, panicked)
if confirmed, reported := result.Output[protocol.ResourceCleanupConfirmedKey]; reported && confirmed != true {
resourceQuiet = false
}
}
// Capture after cleanup, even when the stage's context was canceled.
snapshotCtx, snapshotCancel = context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second)
after := filediff.Capture(snapshotCtx, workspace, before)
@@ -1107,6 +1240,45 @@ func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, na
step.Output["salvage"] = result.Output
}
}
if err == nil && result.Outcome == "success" {
target, targetErr := stage.VerificationTargetFor(def)
if targetErr != nil {
err = targetErr
} else if target != nil {
if r.verifyTarget == nil {
err = fmt.Errorf("verification target %q requires a server-controlled fixture runner", target.Fixture)
} else {
evidence, verificationErr := r.verifyTarget(stageCtx, *target)
if verificationErr != nil {
err = verificationErr
} else {
err = stage.VerifyTargetEvidence(*target, evidence)
}
}
}
if err != nil {
step.Status = "failed"
step.Diagnostic = err.Error()
step.Outcome = "error"
step.Output = stage.Values{"error": err.Error(), "stage": key, "input": input}
}
}
if err == nil && delivery && result.Outcome == "success" {
checks, checksErr := stage.DeliveryChecks(def)
if checksErr != nil {
err = checksErr
} else {
gateCtx, gateCancel := context.WithTimeout(stageCtx, modelConfig.Timeout)
err = verifyDelivery(gateCtx, workspace, landing, checks)
gateCancel()
}
if err != nil {
step.Status = "failed"
step.Diagnostic = err.Error()
step.Outcome = "error"
step.Output = stage.Values{"error": err.Error(), "stage": key, "input": input}
}
}
// A failed or timed-out step carries the session pointer even when the
// bounded transcript would otherwise be only its stage_limits envelope.
if err != nil && stageSession != "" {
@@ -1148,6 +1320,9 @@ func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, na
state.recordChanges(step.FileChanges, step.Artifacts)
if step.Status == "completed" {
state.completed++
if artifactsDir != "" {
state.artifactDirs = append(state.artifactDirs, strconv.Itoa(sequence))
}
}
// Step finished: report its outcome, routing target, diagnostic and the
// artifacts recorded for the visit, plus the run's running step count.
@@ -1175,6 +1350,14 @@ func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, na
return
}
}
if held != nil && !resourceQuiet {
finish("failed", "error", "resource cleanup/restoration not confirmed; ownership quarantined pending inspection")
return
}
if err := release(); err != nil {
finish("failed", "error", "resource release failed; retained ownership needs reconciliation: "+err.Error())
return
}
if ctx.Err() != nil {
finish("canceled", "", ctx.Err().Error())
return
+138
View File
@@ -0,0 +1,138 @@
package server
import (
"bytes"
"context"
"encoding/json"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"strings"
"testing"
"git.a-wan.net/alan/pipelines/internal/cli"
"git.a-wan.net/alan/pipelines/internal/pipeline"
"git.a-wan.net/alan/pipelines/internal/protocol"
)
// Exercise the real V2 CLI/poll path, runner, SQLite retention and websocket CLI
// against an isolated loopback server/database, never the configured deployment.
func TestStageDeadlineDiagnosticThroughCLIAndWebsocket(t *testing.T) {
ntfy := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
_, _ = w.Write([]byte(`{"id":"scratch-receipt","event":"message"}`))
}))
defer ntfy.Close()
t.Setenv("PIPELINES_NTFY_URL", ntfy.URL+"/scratch")
t.Setenv("PIPELINES_NTFY_TOKEN", "")
t.Setenv("PIPELINES_NTFY_TOKEN_FILE", "")
for _, mode := range []string{"active", "http", "connection"} {
t.Run(mode, func(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
dir := t.TempDir()
executable := filepath.Join(dir, "opencode")
script := `#!/bin/sh
if [ "$1" = "--help" ]; then echo ' api'; exit 0; fi
printf '%s %s\n' "$2" "$3" >> "@STATE@/calls"
case "$3" in
/api/session) echo '{"data":{"id":"ses_deadline"}}' ;;
*/prompt) echo 'edited once' >> "@STATE@/edited"; echo '{}' ;;
/api/session/active)
if [ -f "@STATE@/interrupted" ]; then echo '{"data":{}}'; exit 0; fi
case "@MODE@" in
active) echo '{"data":{"ses_deadline":{"type":"running"}}}' ;;
http) echo 'HTTP 503 token=private-token' >&2; exit 1 ;;
connection) echo 'connection refused token=private-token' >&2; exit 1 ;;
esac ;;
*/interrupt) touch "@STATE@/interrupted"; echo '{}' ;;
*/message*) echo '{"data":[{"content":[{"type":"text","text":"still working"}]}]}' ;;
/api/session/ses_deadline) echo '{"data":{"id":"ses_deadline"}}' ;;
*) echo '{}' ;;
esac
`
script = strings.ReplaceAll(strings.ReplaceAll(script, "@STATE@", dir), "@MODE@", mode)
if err := os.WriteFile(executable, []byte(script), 0700); err != nil {
t.Fatal(err)
}
t.Setenv("PIPELINES_OPENCODE", executable)
store, err := pipeline.Open(filepath.Join(dir, "runs.db"))
if err != nil {
t.Fatal(err)
}
defer store.Close()
source := pipeline.Template([]string{"worker"}, true)
source = strings.Replace(source, "```tools\n```", "```tools\nedit\n```", 1)
source = strings.Replace(source, "timeout=1h", "timeout=500ms", 1)
if _, err := store.Submit(ctx, "deadline", source); err != nil {
t.Fatal(err)
}
handler := Handler(ctx, store)
defer func() { cancel(); WaitRuns(handler) }()
srv := httptest.NewServer(handler)
defer srv.Close()
url := "ws" + strings.TrimPrefix(srv.URL, "http") + "/connect"
var out, stderr bytes.Buffer
app := cli.App{Out: &out, Err: &stderr, In: strings.NewReader("")}
if err := app.Run(ctx, []string{"--url", url, "run", "deadline", "task", "--yes", "--json", "--wait", "--timeout", "10s", "--workspace", dir}); err == nil || !strings.Contains(err.Error(), "run ended failed") {
t.Fatalf("run not failed: %v, %s, %s", err, out.String(), stderr.String())
}
var response protocol.Message
if err := json.Unmarshal(out.Bytes(), &response); err != nil {
t.Fatal(err, out.String())
}
detail := response.RunDetail
if detail == nil || detail.Status != "failed" || len(detail.Steps) != 1 {
t.Fatal("missing failed run/step", out.String())
}
step := detail.Steps[0]
if step.SessionID != "ses_deadline" || step.AgentActive || step.Output["error"] != step.Diagnostic || step.Status != "failed" || detail.Diagnostic != step.Diagnostic {
t.Fatalf("incorrect retained failure: %+v", detail)
}
wants := []string{"HARNESS TRANSPORT", "get /api/session/active", "CLI exit code 1"}
if mode == "active" {
wants = []string{"STAGE DEADLINE", "timeout 500ms", "waiting for session idle", "ses_deadline", "stage had not finished"}
if strings.Contains(detail.Diagnostic, "HARNESS TRANSPORT") || strings.Contains(detail.Diagnostic, "response lost") {
t.Fatal("active stage overrun reported as transport", detail.Diagnostic)
}
if strings.Contains(detail.Diagnostic, "\n") {
t.Fatal("duplicate context error appended to bounded stage diagnostic", detail.Diagnostic)
}
for _, want := range []string{"opencode_failures", "wait for session idle", "timeout 500ms", "retained_session", "still working"} {
if !strings.Contains(step.Transcript, want) {
t.Fatalf("missing retained evidence %q: %s", want, step.Transcript)
}
}
} else {
if !strings.Contains(step.Transcript, "[REDACTED]") {
t.Fatal("request failure evidence missing", step.Transcript)
}
if mode == "http" {
wants = append(wants, "HTTP 503")
} else if !strings.Contains(step.Transcript, "connection refused") {
t.Fatal("underlying connection evidence missing", step.Transcript)
}
}
for _, want := range wants {
if !strings.Contains(detail.Diagnostic, want) {
t.Fatalf("missing diagnostic %q: %s", want, detail.Diagnostic)
}
}
if strings.Contains(detail.Diagnostic+step.Transcript, "private-token") {
t.Fatal("credential leaked in retained diagnostic/evidence")
}
edited, err := os.ReadFile(filepath.Join(dir, "edited"))
if err != nil || string(edited) != "edited once\n" {
t.Fatalf("admitted tool work replayed or lost: %q, %v", edited, err)
}
if _, err := os.Stat(filepath.Join(dir, "interrupted")); err != nil {
t.Fatal("cleanup did not interrupt", err)
}
stored, err := store.GetRun(ctx, detail.ID)
if err != nil || stored.Diagnostic != detail.Diagnostic || stored.Steps[0].Transcript != step.Transcript {
t.Fatal("diagnostic/evidence not persisted in scratch database", err)
}
t.Logf("scratch run %s: %s", mode, stored.Diagnostic)
})
}
}
+312
View File
@@ -0,0 +1,312 @@
# stage implementer
```limits
{"max_visits":5,"on":["blocked_retryable"]}
```
```memory
{"task": "input.prompt", "code_findings": "reviewer.findings?", "ui_findings": "ui_reviewer.findings?"}
```
```output
{ task, acceptance_checks, changed_files, implementation_summary, new_helpers }
```
```prompt
Implement input.prompt in the run directory using the repository's actual languages and
file types; create files as needed.
Read android/AGENTS.md first. It is the map of this project - how to build, test, run and capture here, where
the pieces live, and the conventions that apply. Do not re-derive what it tells you and do not go
looking for what it already names; if it is wrong or out of date, say so in your output.
Read the surrounding code before editing so you extend the existing helpers and patterns rather than
duplicating them. Keep each change as small as the task allows and follow the repository's own
conventions and error handling.
This is the Android app. Build and test it with the project's own Gradle invocation as the map
gives it, and treat the map's geometry tokens as the only source of dimensions - a UI change that
adds a raw value fails the repository's own check. Perform code/build/unit tests only;
no adb, install, capture, device-setting changes or restoring main's build. The UI reviewer
alone verifies rendered surfaces under its declared runner resource lease.
Record the acceptance checks in acceptance_checks, the actual changes in changed_files, and any new
shared helper in new_helpers. Verify with the checks this repository uses - as the map gives them -
and report the exact commands and observed output. Address any concrete findings the runner supplies
and note the actual fixes. Return blocked if the required scope or tools are unavailable. Do not
commit or push; the committer stage lands the work.
```
```tools
read
glob
grep
edit
shell
```
```configs
provider=openai
model=gpt-5.6-terra
retries=1
timeout=30m
```
```edges
success -> reviewer
blocked_retryable -> implementer
blocked -> #failure
error -> #failure
```
# stage reviewer
```limits
{"max_visits": 3, "on": ["rejected"]}
```
```memory
{"task": "input.prompt", "acceptance_checks": "implementer.acceptance_checks", "changed_files": "implementer.changed_files", "implementation": "implementer.implementation_summary?"}
```
```output
{ summary, changed_files, verification_evidence, duplication_review, performance_review, findings }
```
```prompt
Verify with builds, unit tests and source review as android/AGENTS.md gives them.
Do not use adb, install, capture, change device settings or restore main's build.
Device verification belongs exclusively to the UI reviewer under its runner resource lease.
Enforce the map's geometry-token rule: a raw dimension outside the token file is a must-fix finding.
Review the actual changes against the task in input.prompt and the acceptance
checks the implementer recorded. Run the repository's relevant verification checks
with the stage tools (for example build, lint, tests or type checks) and report the
exact commands and observed results in verification_evidence.
Independently assess code quality:
- Duplication: search the run directory for repeated or near-identical functions,
blocks and copy-pasted logic introduced by the change; identify code that should be
shared and record concrete locations in duplication_review.
- Performance and resources: flag redundant computation, unnecessary dependencies,
and repeated I/O or database/filesystem work that wastes time, CPU or memory;
record it in performance_review.
- Workarounds: flag hacks, suppressed errors and shortcuts that trade reliability for
speed in findings.
Keep verification proportionate to the change: exercise the files and screens the diff
actually touches, capture at most one screenshot per changed surface plus one regression
screen, and do not repeat sweeps or measurements that an earlier visit already recorded.
Return rejected ONLY when at least one must-fix finding exists: a defect that blocks
acceptance and that you can locate concretely (file and line), with evidence and a
remedy. Do not return rejected for advisory-only, cosmetic or nit items. When there
is no must-fix finding but you have non-blocking notes (duplication, performance or
workaround observations), record them in findings and return advisory, which
continues the run without another implementer visit. Return success only when the
observed checks pass and you have nothing further to record. Do not edit files or
claim unexecuted checks.
```
```tools
read
glob
grep
shell
```
```configs
provider=openai
model=gpt-5.6-luna
retries=1
timeout=45m
```
```edges
success -> ui_reviewer
advisory -> ui_reviewer
rejected -> implementer
error -> #failure
```
# stage ui_reviewer
```resources
{"exclusive":"phone-47241FDAS00DLK"}
```
```limits
{"max_visits": 3, "on": ["rejected"]}
```
```memory
{"task": "input.prompt", "acceptance_checks": "implementer.acceptance_checks", "changed_files": "implementer.changed_files", "implementation": "implementer.implementation_summary?"}
```
```output
{ summary, changed_files, ui_applicable, screenshots, ui_review, platform_conventions, findings }
```
```prompt
Decide whether this change touches a user-facing UI. If it does not (a backend,
library, CLI, config or data-only change with no rendered surface), set
ui_applicable=false, explain briefly in summary, and return skipped. Do not invent
screenshots or visual work.
When the change does affect a user-facing UI, verify it visually:
Read android/AGENTS.md first. You alone perform phone work while the runner holds
this stage's declared resource lease. Use controlled fixtures for changed states,
including human-decision sheets, on a scratch server/database and throwaway session.
Never mutate an operator session or restart deployed services. Record original
settings and connection/workspace; use the configured authoritative endpoint, not a
hardcoded scratch default. Keep every shell/ADB operation foreground and awaited.
Restore original settings and main's build before successful return, under the same
lease. Report capture/restoration failures explicitly; do not approve incomplete
verification or advertise readiness when cleanup/restoration is uncertain.
Include resource_cleanup_confirmed in output: true only after confirmed restoration;
false on any uncertainty (a release veto that quarantines ownership even if tools are idle).
- Build and launch the application the way the repository actually does it, then
capture screenshots of every relevant screen, state and viewport (default, empty,
loading, error, long content, small and large sizes) with the target platform's
screenshot tooling through shell. Prefer a headless or scripted capture when the
project supports one. Save the images under the run directory and list the exact
capture commands and image paths in screenshots.
- Read the captured images and review them as a UI/UX designer: layout, spacing,
alignment, typographic hierarchy, color and contrast, iconography, interaction
and focus states, loading/empty/error states, accessibility, and consistency.
- Judge the UI against established industry patterns and, above all, the native
conventions of the platform the app targets (operating system, toolkit or web
standards). Record the platform you reviewed against in platform_conventions,
and any deviations and concrete fixes in findings. Infer the target platform from
the task and the repository's actual UI toolkit when it is unstated.
- Approve only when the screenshots show the task and the acceptance checks are met
and the UI is correct, consistent and native to the target platform; otherwise
return rejected with specific, actionable findings the implementer can fix.
Keep the capture pass proportionate: capture each changed state once (for example one
collapsed and one expanded section, one run state, one stuck header) instead of every
screen at multiple viewports or pan directions, and reuse what this run already captured
rather than re-doing it.
If the change is UI-facing but screenshots genuinely cannot be captured
(for example no runnable UI or no display), return blocked rather than approving.
Do not edit files, and do not approve a UI you did not actually see.
Return rejected only for a must-fix visual defect with a concrete location, evidence
and remedy; record non-blocking polish, duplication or convention notes in findings
and return advisory, which continues the run without another implementer visit.
```
```tools
read
glob
grep
shell
```
```configs
provider=openai
model=gpt-5.6-luna
retries=1
timeout=45m
```
```edges
skipped -> committer
approved -> committer
advisory -> committer
rejected -> implementer
blocked -> #failure
error -> #failure
```
# stage committer
```memory
{"task": "input.prompt", "changed_files": "implementer.changed_files", "review": "reviewer.summary?", "ui_review": "ui_reviewer.summary?"}
```
```output
{ commit, files, conflicts_resolved, summary }
```
```prompt
Commit the work this run produced, and resolve any conflict that is present. This stage
runs after the reviews have approved and is the only stage allowed to commit.
Before committing, check that the change is reflected where it belongs: the packaged
templates, the packaged
authoring skill references. Update anything the change makes stale, and state in the
commit body what you checked, saying explicitly when none of them needs updating.
Stage exactly the files this run changed - the paths in changed_files that still exist -
using the repository's own version control commands, and commit them as one commit whose
subject says what the change does and whose body summarises it. Do not stage unrelated
files, generated or ignored files, or anything another change owns. Do not invent work
that was not done; describe what is actually in the diff.
If the workspace is in a conflicted state - unmerged paths, or a merge, rebase or
cherry-pick left unfinished - resolve each conflict: keep both intents where they are
compatible, and choose the result that satisfies the task where they are not, then
complete the operation so the tree is clean and the commit is finished. Never commit
conflict markers, and never discard the other side's work silently; record what you
resolved.
Do not push, tag, revert, or rewrite unrelated history. If there is nothing to commit,
report that instead of creating an empty commit. Report the commit identifier, the files
committed, whatever you resolved, and anything you deliberately left uncommitted.
Finally, bring this branch up to date with main, in this same stage. After committing the work
above, merge main into this branch here, resolve any conflict here - keeping both intents where they
are compatible, and choosing the result that satisfies this task where they are not - run the
repository's checks again after resolving, and commit the completed merge with a message naming what
was combined. The point is that this branch can later be merged back without conflict: leave the
tree clean and the merge finished. Never leave conflict markers, never discard the other side's work
silently. If the merge genuinely cannot be completed, say so with the path and the reason instead of
leaving the merge half-finished.
Then land it: this branch is the work and main is where it belongs, so merge this branch into main
as the last act of this stage. Find the repository's main working tree with `git worktree list` (the
entry checked out on main, not this one) and merge this branch into it there with `--no-ff`, with a
message saying what the change does. Two lanes can land at the same moment, so take a lock first -
`mkdir` a lock directory beside main's `git` directory in a bounded retry loop, and remove it when
done - and retry briefly if another lane holds it. Then verify with
`git merge-base --is-ancestor <this branch> main` and report the commit main now has. If main cannot
be updated, say exactly why and leave this branch committed and clean rather than leaving main
half-merged. There is no separate merge pipeline and nothing to hand this to: if it cannot be landed
here, it is reported here.
Do not edit README or other shared documentation files: an item records what it changed and which documentation is now stale in its own commit body, and the shared documents are folded by a single writer.
```
```tools
read
glob
grep
edit
shell
```
```configs
provider=openai
model=gpt-5.6-terra
retries=1
timeout=30m
```
```edges
success -> #success
blocked -> #failure
error -> #failure
```
+296
View File
@@ -0,0 +1,296 @@
# stage implementer
```limits
{"max_visits":5,"on":["blocked_retryable"]}
```
```memory
{"task": "input.prompt", "code_findings": "reviewer.findings?", "ui_findings": "ui_reviewer.findings?"}
```
```output
{ task, acceptance_checks, changed_files, implementation_summary, new_helpers }
```
```prompt
Implement input.prompt in the run directory using the repository's actual languages and
file types; create files as needed.
Read android/AGENTS.md first. It is the map of this project - how to build, test, run and capture here, where
the pieces live, and the conventions that apply. Do not re-derive what it tells you and do not go
looking for what it already names; if it is wrong or out of date, say so in your output.
Read the surrounding code before editing so you extend the existing helpers and patterns rather than
duplicating them. Keep each change as small as the task allows and follow the repository's own
conventions and error handling.
This is the Android app. Build and test it with the project's own Gradle invocation as the map
gives it, and treat the map's geometry tokens as the only source of dimensions - a UI change that
adds a raw value fails the repository's own check. A change with a rendered surface is verified on
the device the map names (never an emulator), capturing what the map says to capture.
Record the acceptance checks in acceptance_checks, the actual changes in changed_files, and any new
shared helper in new_helpers. Verify with the checks this repository uses - as the map gives them -
and report the exact commands and observed output. Address any concrete findings the runner supplies
and note the actual fixes. Return blocked if the required scope or tools are unavailable. Do not
commit or push; the committer stage lands the work.
```
```tools
read
glob
grep
edit
shell
```
```configs
provider=openai
model=gpt-6.1-sol-fast
retries=1
timeout=30m
```
```edges
success -> reviewer
blocked_retryable -> implementer
blocked -> #failure
error -> #failure
```
# stage reviewer
```limits
{"max_visits": 3, "on": ["rejected"]}
```
```memory
{"task": "input.prompt", "acceptance_checks": "implementer.acceptance_checks", "changed_files": "implementer.changed_files", "implementation": "implementer.implementation_summary?"}
```
```output
{ summary, changed_files, verification_evidence, duplication_review, performance_review, findings }
```
```prompt
Verify with the build and the unit tests as android/AGENTS.md gives them, and for a
change with a rendered surface check the captures against the running app on the device the map
names. Enforce the map's geometry-token rule: a raw dimension outside the token file is a must-fix
finding.
Review the actual changes against the task in input.prompt and the acceptance
checks the implementer recorded. Run the repository's relevant verification checks
with the stage tools (for example build, lint, tests or type checks) and report the
exact commands and observed results in verification_evidence.
Independently assess code quality:
- Duplication: search the run directory for repeated or near-identical functions,
blocks and copy-pasted logic introduced by the change; identify code that should be
shared and record concrete locations in duplication_review.
- Performance and resources: flag redundant computation, unnecessary dependencies,
and repeated I/O or database/filesystem work that wastes time, CPU or memory;
record it in performance_review.
- Workarounds: flag hacks, suppressed errors and shortcuts that trade reliability for
speed in findings.
Keep verification proportionate to the change: exercise the files and screens the diff
actually touches, capture at most one screenshot per changed surface plus one regression
screen, and do not repeat sweeps or measurements that an earlier visit already recorded.
Return rejected ONLY when at least one must-fix finding exists: a defect that blocks
acceptance and that you can locate concretely (file and line), with evidence and a
remedy. Do not return rejected for advisory-only, cosmetic or nit items. When there
is no must-fix finding but you have non-blocking notes (duplication, performance or
workaround observations), record them in findings and return advisory, which
continues the run without another implementer visit. Return success only when the
observed checks pass and you have nothing further to record. Do not edit files or
claim unexecuted checks.
```
```tools
read
glob
grep
shell
```
```configs
provider=openai
model=gpt-6.1-sol-fast
retries=1
timeout=45m
```
```edges
success -> ui_reviewer
advisory -> ui_reviewer
rejected -> implementer
error -> #failure
```
# stage ui_reviewer
```limits
{"max_visits": 3, "on": ["rejected"]}
```
```memory
{"task": "input.prompt", "acceptance_checks": "implementer.acceptance_checks", "changed_files": "implementer.changed_files", "implementation": "implementer.implementation_summary?"}
```
```output
{ summary, changed_files, ui_applicable, screenshots, ui_review, platform_conventions, findings }
```
```prompt
Decide whether this change touches a user-facing UI. If it does not (a backend,
library, CLI, config or data-only change with no rendered surface), set
ui_applicable=false, explain briefly in summary, and return skipped. Do not invent
screenshots or visual work.
When the change does affect a user-facing UI, verify it visually:
- Build and launch the application the way the repository actually does it, then
capture screenshots of every relevant screen, state and viewport (default, empty,
loading, error, long content, small and large sizes) with the target platform's
screenshot tooling through shell. Prefer a headless or scripted capture when the
project supports one. Save the images under the run directory and list the exact
capture commands and image paths in screenshots.
- Read the captured images and review them as a UI/UX designer: layout, spacing,
alignment, typographic hierarchy, color and contrast, iconography, interaction
and focus states, loading/empty/error states, accessibility, and consistency.
- Judge the UI against established industry patterns and, above all, the native
conventions of the platform the app targets (operating system, toolkit or web
standards). Record the platform you reviewed against in platform_conventions,
and any deviations and concrete fixes in findings. Infer the target platform from
the task and the repository's actual UI toolkit when it is unstated.
- Approve only when the screenshots show the task and the acceptance checks are met
and the UI is correct, consistent and native to the target platform; otherwise
return rejected with specific, actionable findings the implementer can fix.
Keep the capture pass proportionate: capture each changed state once (for example one
collapsed and one expanded section, one run state, one stuck header) instead of every
screen at multiple viewports or pan directions, and reuse what this run already captured
rather than re-doing it.
If the change is UI-facing but screenshots genuinely cannot be captured
(for example no runnable UI or no display), return blocked rather than approving.
Do not edit files, and do not approve a UI you did not actually see.
Return rejected only for a must-fix visual defect with a concrete location, evidence
and remedy; record non-blocking polish, duplication or convention notes in findings
and return advisory, which continues the run without another implementer visit.
```
```tools
read
glob
grep
shell
```
```configs
provider=openai
model=gpt-6.1-sol-fast
retries=1
timeout=45m
```
```edges
skipped -> committer
approved -> committer
advisory -> committer
rejected -> implementer
blocked -> #failure
error -> #failure
```
# stage committer
```memory
{"task": "input.prompt", "changed_files": "implementer.changed_files", "review": "reviewer.summary?", "ui_review": "ui_reviewer.summary?"}
```
```output
{ commit, files, conflicts_resolved, summary }
```
```prompt
Commit the work this run produced, and resolve any conflict that is present. This stage
runs after the reviews have approved and is the only stage allowed to commit.
Before committing, check that the change is reflected where it belongs: the packaged
templates, the packaged
authoring skill references. Update anything the change makes stale, and state in the
commit body what you checked, saying explicitly when none of them needs updating.
Stage exactly the files this run changed - the paths in changed_files that still exist -
using the repository's own version control commands, and commit them as one commit whose
subject says what the change does and whose body summarises it. Do not stage unrelated
files, generated or ignored files, or anything another change owns. Do not invent work
that was not done; describe what is actually in the diff.
If the workspace is in a conflicted state - unmerged paths, or a merge, rebase or
cherry-pick left unfinished - resolve each conflict: keep both intents where they are
compatible, and choose the result that satisfies the task where they are not, then
complete the operation so the tree is clean and the commit is finished. Never commit
conflict markers, and never discard the other side's work silently; record what you
resolved.
Do not push, tag, revert, or rewrite unrelated history. If there is nothing to commit,
report that instead of creating an empty commit. Report the commit identifier, the files
committed, whatever you resolved, and anything you deliberately left uncommitted.
Finally, bring this branch up to date with main, in this same stage. After committing the work
above, merge main into this branch here, resolve any conflict here - keeping both intents where they
are compatible, and choosing the result that satisfies this task where they are not - run the
repository's checks again after resolving, and commit the completed merge with a message naming what
was combined. The point is that this branch can later be merged back without conflict: leave the
tree clean and the merge finished. Never leave conflict markers, never discard the other side's work
silently. If the merge genuinely cannot be completed, say so with the path and the reason instead of
leaving the merge half-finished.
Then land it: this branch is the work and main is where it belongs, so merge this branch into main
as the last act of this stage. Find the repository's main working tree with `git worktree list` (the
entry checked out on main, not this one) and merge this branch into it there with `--no-ff`, with a
message saying what the change does. Two lanes can land at the same moment, so take a lock first -
`mkdir` a lock directory beside main's `git` directory in a bounded retry loop, and remove it when
done - and retry briefly if another lane holds it. Then verify with
`git merge-base --is-ancestor <this branch> main` and report the commit main now has. If main cannot
be updated, say exactly why and leave this branch committed and clean rather than leaving main
half-merged. There is no separate merge pipeline and nothing to hand this to: if it cannot be landed
here, it is reported here.
Do not edit README or other shared documentation files: an item records what it changed and which documentation is now stale in its own commit body, and the shared documents are folded by a single writer.
```
```tools
read
glob
grep
edit
shell
```
```configs
provider=openai
model=gpt-6.1-sol-fast
retries=1
timeout=30m
```
```edges
success -> #success
blocked -> #failure
error -> #failure
```
+296
View File
@@ -0,0 +1,296 @@
# stage implementer
```limits
{"max_visits":5,"on":["blocked_retryable"]}
```
```memory
{"task": "input.prompt", "code_findings": "reviewer.findings?", "ui_findings": "ui_reviewer.findings?"}
```
```output
{ task, acceptance_checks, changed_files, implementation_summary, new_helpers }
```
```prompt
Implement input.prompt in the run directory using the repository's actual languages and
file types; create files as needed.
Read android/AGENTS.md first. It is the map of this project - how to build, test, run and capture here, where
the pieces live, and the conventions that apply. Do not re-derive what it tells you and do not go
looking for what it already names; if it is wrong or out of date, say so in your output.
Read the surrounding code before editing so you extend the existing helpers and patterns rather than
duplicating them. Keep each change as small as the task allows and follow the repository's own
conventions and error handling.
This is the Android app. Build and test it with the project's own Gradle invocation as the map
gives it, and treat the map's geometry tokens as the only source of dimensions - a UI change that
adds a raw value fails the repository's own check. A change with a rendered surface is verified on
the device the map names (never an emulator), capturing what the map says to capture.
Record the acceptance checks in acceptance_checks, the actual changes in changed_files, and any new
shared helper in new_helpers. Verify with the checks this repository uses - as the map gives them -
and report the exact commands and observed output. Address any concrete findings the runner supplies
and note the actual fixes. Return blocked if the required scope or tools are unavailable. Do not
commit or push; the committer stage lands the work.
```
```tools
read
glob
grep
edit
shell
```
```configs
provider=openai
model=gpt-5.6-terra
retries=1
timeout=30m
```
```edges
success -> reviewer
blocked_retryable -> implementer
blocked -> #failure
error -> #failure
```
# stage reviewer
```limits
{"max_visits": 3, "on": ["rejected"]}
```
```memory
{"task": "input.prompt", "acceptance_checks": "implementer.acceptance_checks", "changed_files": "implementer.changed_files", "implementation": "implementer.implementation_summary?"}
```
```output
{ summary, changed_files, verification_evidence, duplication_review, performance_review, findings }
```
```prompt
Verify with the build and the unit tests as android/AGENTS.md gives them, and for a
change with a rendered surface check the captures against the running app on the device the map
names. Enforce the map's geometry-token rule: a raw dimension outside the token file is a must-fix
finding.
Review the actual changes against the task in input.prompt and the acceptance
checks the implementer recorded. Run the repository's relevant verification checks
with the stage tools (for example build, lint, tests or type checks) and report the
exact commands and observed results in verification_evidence.
Independently assess code quality:
- Duplication: search the run directory for repeated or near-identical functions,
blocks and copy-pasted logic introduced by the change; identify code that should be
shared and record concrete locations in duplication_review.
- Performance and resources: flag redundant computation, unnecessary dependencies,
and repeated I/O or database/filesystem work that wastes time, CPU or memory;
record it in performance_review.
- Workarounds: flag hacks, suppressed errors and shortcuts that trade reliability for
speed in findings.
Keep verification proportionate to the change: exercise the files and screens the diff
actually touches, capture at most one screenshot per changed surface plus one regression
screen, and do not repeat sweeps or measurements that an earlier visit already recorded.
Return rejected ONLY when at least one must-fix finding exists: a defect that blocks
acceptance and that you can locate concretely (file and line), with evidence and a
remedy. Do not return rejected for advisory-only, cosmetic or nit items. When there
is no must-fix finding but you have non-blocking notes (duplication, performance or
workaround observations), record them in findings and return advisory, which
continues the run without another implementer visit. Return success only when the
observed checks pass and you have nothing further to record. Do not edit files or
claim unexecuted checks.
```
```tools
read
glob
grep
shell
```
```configs
provider=openai
model=gpt-5.6-luna
retries=1
timeout=45m
```
```edges
success -> ui_reviewer
advisory -> ui_reviewer
rejected -> implementer
error -> #failure
```
# stage ui_reviewer
```limits
{"max_visits": 3, "on": ["rejected"]}
```
```memory
{"task": "input.prompt", "acceptance_checks": "implementer.acceptance_checks", "changed_files": "implementer.changed_files", "implementation": "implementer.implementation_summary?"}
```
```output
{ summary, changed_files, ui_applicable, screenshots, ui_review, platform_conventions, findings }
```
```prompt
Decide whether this change touches a user-facing UI. If it does not (a backend,
library, CLI, config or data-only change with no rendered surface), set
ui_applicable=false, explain briefly in summary, and return skipped. Do not invent
screenshots or visual work.
When the change does affect a user-facing UI, verify it visually:
- Build and launch the application the way the repository actually does it, then
capture screenshots of every relevant screen, state and viewport (default, empty,
loading, error, long content, small and large sizes) with the target platform's
screenshot tooling through shell. Prefer a headless or scripted capture when the
project supports one. Save the images under the run directory and list the exact
capture commands and image paths in screenshots.
- Read the captured images and review them as a UI/UX designer: layout, spacing,
alignment, typographic hierarchy, color and contrast, iconography, interaction
and focus states, loading/empty/error states, accessibility, and consistency.
- Judge the UI against established industry patterns and, above all, the native
conventions of the platform the app targets (operating system, toolkit or web
standards). Record the platform you reviewed against in platform_conventions,
and any deviations and concrete fixes in findings. Infer the target platform from
the task and the repository's actual UI toolkit when it is unstated.
- Approve only when the screenshots show the task and the acceptance checks are met
and the UI is correct, consistent and native to the target platform; otherwise
return rejected with specific, actionable findings the implementer can fix.
Keep the capture pass proportionate: capture each changed state once (for example one
collapsed and one expanded section, one run state, one stuck header) instead of every
screen at multiple viewports or pan directions, and reuse what this run already captured
rather than re-doing it.
If the change is UI-facing but screenshots genuinely cannot be captured
(for example no runnable UI or no display), return blocked rather than approving.
Do not edit files, and do not approve a UI you did not actually see.
Return rejected only for a must-fix visual defect with a concrete location, evidence
and remedy; record non-blocking polish, duplication or convention notes in findings
and return advisory, which continues the run without another implementer visit.
```
```tools
read
glob
grep
shell
```
```configs
provider=openai
model=gpt-5.6-luna
retries=1
timeout=45m
```
```edges
skipped -> committer
approved -> committer
advisory -> committer
rejected -> implementer
blocked -> #failure
error -> #failure
```
# stage committer
```memory
{"task": "input.prompt", "changed_files": "implementer.changed_files", "review": "reviewer.summary?", "ui_review": "ui_reviewer.summary?"}
```
```output
{ commit, files, conflicts_resolved, summary }
```
```prompt
Commit the work this run produced, and resolve any conflict that is present. This stage
runs after the reviews have approved and is the only stage allowed to commit.
Before committing, check that the change is reflected where it belongs: the packaged
templates, the packaged
authoring skill references. Update anything the change makes stale, and state in the
commit body what you checked, saying explicitly when none of them needs updating.
Stage exactly the files this run changed - the paths in changed_files that still exist -
using the repository's own version control commands, and commit them as one commit whose
subject says what the change does and whose body summarises it. Do not stage unrelated
files, generated or ignored files, or anything another change owns. Do not invent work
that was not done; describe what is actually in the diff.
If the workspace is in a conflicted state - unmerged paths, or a merge, rebase or
cherry-pick left unfinished - resolve each conflict: keep both intents where they are
compatible, and choose the result that satisfies the task where they are not, then
complete the operation so the tree is clean and the commit is finished. Never commit
conflict markers, and never discard the other side's work silently; record what you
resolved.
Do not push, tag, revert, or rewrite unrelated history. If there is nothing to commit,
report that instead of creating an empty commit. Report the commit identifier, the files
committed, whatever you resolved, and anything you deliberately left uncommitted.
Finally, bring this branch up to date with main, in this same stage. After committing the work
above, merge main into this branch here, resolve any conflict here - keeping both intents where they
are compatible, and choosing the result that satisfies this task where they are not - run the
repository's checks again after resolving, and commit the completed merge with a message naming what
was combined. The point is that this branch can later be merged back without conflict: leave the
tree clean and the merge finished. Never leave conflict markers, never discard the other side's work
silently. If the merge genuinely cannot be completed, say so with the path and the reason instead of
leaving the merge half-finished.
Then land it: this branch is the work and main is where it belongs, so merge this branch into main
as the last act of this stage. Find the repository's main working tree with `git worktree list` (the
entry checked out on main, not this one) and merge this branch into it there with `--no-ff`, with a
message saying what the change does. Two lanes can land at the same moment, so take a lock first -
`mkdir` a lock directory beside main's `git` directory in a bounded retry loop, and remove it when
done - and retry briefly if another lane holds it. Then verify with
`git merge-base --is-ancestor <this branch> main` and report the commit main now has. If main cannot
be updated, say exactly why and leave this branch committed and clean rather than leaving main
half-merged. There is no separate merge pipeline and nothing to hand this to: if it cannot be landed
here, it is reported here.
Do not edit README or other shared documentation files: an item records what it changed and which documentation is now stale in its own commit body, and the shared documents are folded by a single writer.
```
```tools
read
glob
grep
edit
shell
```
```configs
provider=openai
model=gpt-5.6-terra
retries=1
timeout=30m
```
```edges
success -> #success
blocked -> #failure
error -> #failure
```
+135
View File
@@ -0,0 +1,135 @@
package server
import (
"context"
"encoding/json"
"fmt"
"io"
"net/http"
"net/url"
"os"
"strings"
"time"
"git.a-wan.net/alan/pipelines/internal/stage"
)
const (
verificationFixtureID = "reconnect-foreground"
verificationFixtureResponseLimit int64 = 64 << 10
verificationFixtureTimeout = 10 * time.Second
)
// VerificationFixture is a server-owned, explicitly registered probe. Its
// endpoint and expected provenance come from server configuration, never a
// stage definition or result.
type VerificationFixture struct {
ID, Endpoint, Backend, Revision, Capability, State string
}
// VerificationFixturesFromEnvironment returns the single supported fixture
// registration when its endpoint is explicitly configured. There is no default
// endpoint: an unconfigured server fails target-aware verification closed.
func VerificationFixturesFromEnvironment(getenv func(string) string) []VerificationFixture {
if getenv == nil {
getenv = os.Getenv
}
endpoint := strings.TrimSpace(getenv("PIPELINES_VERIFICATION_RECONNECT_FOREGROUND_URL"))
if endpoint == "" {
return nil
}
return []VerificationFixture{{
ID: verificationFixtureID,
Endpoint: endpoint,
Backend: strings.TrimSpace(getenv("PIPELINES_VERIFICATION_RECONNECT_FOREGROUND_BACKEND")),
Revision: strings.TrimSpace(getenv("PIPELINES_VERIFICATION_RECONNECT_FOREGROUND_REVISION")),
Capability: strings.TrimSpace(getenv("PIPELINES_VERIFICATION_RECONNECT_FOREGROUND_CAPABILITY")),
State: strings.TrimSpace(getenv("PIPELINES_VERIFICATION_RECONNECT_FOREGROUND_STATE")),
}}
}
type verificationFixtureRegistry struct {
fixtures map[string]VerificationFixture
err error
}
func newVerificationFixtureRegistry(fixtures []VerificationFixture) *verificationFixtureRegistry {
r := &verificationFixtureRegistry{fixtures: make(map[string]VerificationFixture, len(fixtures))}
for _, fixture := range fixtures {
if err := validVerificationFixture(fixture); err != nil {
r.err = err
return r
}
if _, exists := r.fixtures[fixture.ID]; exists {
r.err = fmt.Errorf("duplicate verification fixture %q", fixture.ID)
return r
}
r.fixtures[fixture.ID] = fixture
}
return r
}
func validVerificationFixture(fixture VerificationFixture) error {
if fixture.ID != verificationFixtureID {
return fmt.Errorf("unsupported verification fixture %q", fixture.ID)
}
for _, value := range []string{fixture.Endpoint, fixture.Backend, fixture.Revision, fixture.Capability, fixture.State} {
if value == "" || len(value) > 256 {
return fmt.Errorf("verification fixture %q has incomplete configuration", fixture.ID)
}
}
u, err := url.Parse(fixture.Endpoint)
if err != nil || u.Host == "" || (u.Scheme != "http" && u.Scheme != "https") {
return fmt.Errorf("verification fixture %q has invalid endpoint", fixture.ID)
}
return nil
}
func (r *verificationFixtureRegistry) verify(ctx context.Context, target stage.VerificationTarget) (stage.VerificationEvidence, error) {
if r.err != nil {
return stage.VerificationEvidence{}, r.err
}
fixture, exists := r.fixtures[target.Fixture]
if !exists {
return stage.VerificationEvidence{}, fmt.Errorf("unknown verification fixture %q", target.Fixture)
}
if fixture.Backend != target.Backend || fixture.Revision != target.Revision || fixture.Capability != target.Capability {
return stage.VerificationEvidence{}, fmt.Errorf("verification target does not match registered fixture %q", target.Fixture)
}
probeCtx, cancel := context.WithTimeout(ctx, verificationFixtureTimeout)
defer cancel()
req, err := http.NewRequestWithContext(probeCtx, http.MethodGet, fixture.Endpoint, nil)
if err != nil {
return stage.VerificationEvidence{}, fmt.Errorf("verification fixture %q request: %w", fixture.ID, err)
}
client := &http.Client{Timeout: verificationFixtureTimeout, CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }}
response, err := client.Do(req)
if err != nil {
return stage.VerificationEvidence{}, fmt.Errorf("verification fixture %q probe: %w", fixture.ID, err)
}
defer response.Body.Close()
if response.StatusCode != http.StatusOK {
return stage.VerificationEvidence{}, fmt.Errorf("verification fixture %q probe returned %s", fixture.ID, response.Status)
}
var observed struct {
Backend string `json:"backend"`
Revision string `json:"revision"`
Capability string `json:"capability"`
Fixture string `json:"fixture"`
State string `json:"state"`
}
body, err := io.ReadAll(io.LimitReader(response.Body, verificationFixtureResponseLimit+1))
if err != nil {
return stage.VerificationEvidence{}, fmt.Errorf("verification fixture %q response: %w", fixture.ID, err)
}
if int64(len(body)) > verificationFixtureResponseLimit {
return stage.VerificationEvidence{}, fmt.Errorf("verification fixture %q response exceeds %d bytes", fixture.ID, verificationFixtureResponseLimit)
}
if err := json.Unmarshal(body, &observed); err != nil {
return stage.VerificationEvidence{}, fmt.Errorf("verification fixture %q response: %w", fixture.ID, err)
}
if observed.Backend != fixture.Backend || observed.Revision != fixture.Revision || observed.Capability != fixture.Capability || observed.Fixture != fixture.ID || observed.State != fixture.State {
return stage.VerificationEvidence{}, fmt.Errorf("verification fixture %q returned incompatible backend, revision, capability, fixture, or state", fixture.ID)
}
return stage.VerificationEvidence{Backend: observed.Backend, Revision: observed.Revision, Capability: observed.Capability, Fixture: observed.Fixture}, nil
}
@@ -0,0 +1,148 @@
package server
import (
"context"
"encoding/json"
"net/http"
"net/http/httptest"
"path/filepath"
"strings"
"sync"
"testing"
"time"
"git.a-wan.net/alan/pipelines/internal/pipeline"
"git.a-wan.net/alan/pipelines/internal/protocol"
"git.a-wan.net/alan/pipelines/internal/stage"
)
func TestHandlerRunsRegisteredVerificationFixture(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
var mu sync.Mutex
response := map[string]string{"backend": "branch-api", "revision": "abc123", "capability": "controlled-state", "fixture": verificationFixtureID, "state": "ready"}
calls := 0
fixture := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
mu.Lock()
defer mu.Unlock()
calls++
_ = json.NewEncoder(w).Encode(response)
}))
defer fixture.Close()
store, err := pipeline.Open(filepath.Join(t.TempDir(), "fixture.db"))
if err != nil {
t.Fatal(err)
}
defer store.Close()
source := pipeline.Template([]string{"worker"}, true)
source = strings.Replace(source, "```configs", "```verification\nbackend=branch-api\nrevision=abc123\ncapability=controlled-state\nfixture="+verificationFixtureID+"\n```\n\n```configs", 1)
if _, err := store.Submit(ctx, "controlled-fixture", source); err != nil {
t.Fatal(err)
}
items, err := store.List(ctx)
if err != nil {
t.Fatal(err)
}
handler := HandlerWithOptions(ctx, Options{VerificationFixtures: []VerificationFixture{{ID: verificationFixtureID, Endpoint: fixture.URL, Backend: "branch-api", Revision: "abc123", Capability: "controlled-state", State: "ready"}}}, store).(*runHandler)
defer func() {
cancel()
WaitRuns(handler)
}()
r := handler.runner
if r.verifyTarget == nil {
t.Fatal("normal server construction did not wire controlled fixture verification")
}
r.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) {
// These forged model labels must not influence fixture provenance.
return stage.Result{Outcome: "success", Output: stage.Values{"verification": map[string]any{"backend": "old-api", "revision": "stale", "capability": "forged", "fixture": "other"}}}, nil
}
start := func(key string) *protocol.RunDetail {
run, err := r.start(protocol.Message{PipelineID: items[0].ID, Version: 1, RequestKey: key, Workspace: t.TempDir(), Input: map[string]any{"prompt": "verify"}})
if err != nil {
t.Fatal(err)
}
r.wg.Wait()
detail, err := store.GetRun(ctx, run.ID)
if err != nil {
t.Fatal(err)
}
return detail
}
if detail := start("compatible"); detail.Status != "succeeded" {
t.Fatalf("compatible controlled fixture did not settle: %+v", detail)
}
mu.Lock()
response["revision"] = "old-build"
mu.Unlock()
if detail := start("stale"); detail.Status != "failed" || !strings.Contains(detail.Steps[0].Diagnostic, "incompatible") {
t.Fatalf("stale fixture provenance settled as success: %+v", detail)
}
mu.Lock()
if calls != 2 {
t.Fatalf("fixture probes = %d, want one per target-aware stage", calls)
}
mu.Unlock()
}
func TestVerificationFixtureCancellationClosesProbe(t *testing.T) {
started := make(chan struct{})
canceled := make(chan struct{})
fixture := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
close(started)
<-r.Context().Done()
close(canceled)
}))
defer fixture.Close()
registry := newVerificationFixtureRegistry([]VerificationFixture{{ID: verificationFixtureID, Endpoint: fixture.URL, Backend: "branch-api", Revision: "abc123", Capability: "controlled-state", State: "ready"}})
ctx, cancel := context.WithCancel(context.Background())
done := make(chan error, 1)
go func() {
_, err := registry.verify(ctx, stage.VerificationTarget{Backend: "branch-api", Revision: "abc123", Capability: "controlled-state", Fixture: verificationFixtureID})
done <- err
}()
select {
case <-started:
case <-time.After(time.Second):
t.Fatal("fixture probe did not start")
}
cancel()
select {
case err := <-done:
if err == nil {
t.Fatal("canceled fixture probe succeeded")
}
case <-time.After(time.Second):
t.Fatal("canceled fixture probe did not return")
}
select {
case <-canceled:
case <-time.After(time.Second):
t.Fatal("canceled fixture request was not cleaned up")
}
}
func TestVerificationFixtureRejectsUnknownAndMismatchedProvenance(t *testing.T) {
base := map[string]string{"backend": "branch-api", "revision": "abc123", "capability": "controlled-state", "fixture": verificationFixtureID, "state": "ready"}
for field, wrong := range map[string]string{"backend": "old-api", "revision": "old-build", "capability": "other-state", "state": "stale"} {
t.Run(field, func(t *testing.T) {
response := make(map[string]string, len(base))
for key, value := range base {
response[key] = value
}
response[field] = wrong
fixture := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
_ = json.NewEncoder(w).Encode(response)
}))
defer fixture.Close()
registry := newVerificationFixtureRegistry([]VerificationFixture{{ID: verificationFixtureID, Endpoint: fixture.URL, Backend: base["backend"], Revision: base["revision"], Capability: base["capability"], State: base["state"]}})
_, err := registry.verify(context.Background(), stage.VerificationTarget{Backend: base["backend"], Revision: base["revision"], Capability: base["capability"], Fixture: verificationFixtureID})
if err == nil || !strings.Contains(err.Error(), "incompatible") {
t.Fatalf("mismatched %s provenance was accepted: %v", field, err)
}
})
}
registry := newVerificationFixtureRegistry(nil)
if _, err := registry.verify(context.Background(), stage.VerificationTarget{Backend: "branch-api", Revision: "abc123", Capability: "controlled-state", Fixture: "unknown"}); err == nil || !strings.Contains(err.Error(), "unknown") {
t.Fatalf("unknown fixture was accepted: %v", err)
}
}
+8
View File
@@ -49,6 +49,10 @@ type Options struct {
// read time with no model call, which keeps tests and unconfigured servers
// deterministic.
StepVerdicts func(context.Context, map[string]any) string
// VerificationFixtures are explicit server-owned HTTP fixtures for optional
// target-aware verification. Definitions can name only a registered ID; they
// cannot select an endpoint or supply provenance.
VerificationFixtures []VerificationFixture
}
// Handler serves the persistent connection with default options. Canceling ctx
@@ -81,6 +85,10 @@ func HandlerWithOptions(ctx context.Context, options Options, stores ...*pipelin
if runErr == nil && options.StepVerdicts != nil {
runs.verdict = options.StepVerdicts
}
if runErr == nil {
fixtures := newVerificationFixtureRegistry(options.VerificationFixtures)
runs.verifyTarget = fixtures.verify
}
if runErr == nil {
runs.schedulerWG.Add(1)
go runs.scheduleLoop()
+34 -9
View File
@@ -14,16 +14,28 @@ import (
// before any prompt is admitted, so restart recovery can interrupt the agent.
type AgentClient struct {
OpenCodeClient
WritePaths []string
Tools []string // nil is the legacy manual approval bundle; empty denies all
OnSession func(string, bool) error
SessionID string
Transcript string
Policy ToolPolicy
WorkStarted bool
WritePaths []string
Tools []string // nil is the legacy manual approval bundle; empty denies all
// ArtifactAccess is supplied only by the runner. Write is this visit's
// directory; Read contains this and completed earlier visits in the same run.
ArtifactAccess ArtifactAccess
LandingAccess LandingAccess
OnSession func(string, bool) error
SessionID string
Transcript string
Policy ToolPolicy
WorkStarted bool
}
func (c OpenCodeClient) Interrupt(ctx context.Context, id string) error {
return c.interrupt(ctx, id, false)
}
func (c OpenCodeClient) InterruptAndConfirm(ctx context.Context, id string) error {
return c.interrupt(ctx, id, true)
}
func (c OpenCodeClient) interrupt(ctx context.Context, id string, confirm bool) error {
if !sessionIDPattern.MatchString(id) {
return fmt.Errorf("invalid session ID")
}
@@ -43,7 +55,20 @@ func (c OpenCodeClient) Interrupt(ctx context.Context, id string) error {
c.Backend = connection.Backend
}
_, err := c.call(ctx, "post", "/api/session/"+id+"/interrupt", map[string]any{})
return err
if err != nil || !confirm {
return err
}
// An interrupt acknowledgement is not a cleanup receipt. Keep ownership
// until the harness positively reports settlement, bounded by the caller.
if c.Backend == "v1" {
_, err := c.call(ctx, "post", "/api/experimental/session/"+id+"/wait", nil)
return err
}
// The failed stage wait already records deadline evidence. Cleanup settlement
// must still be confirmed, but must not add a second duplicate operation entry.
confirmed := c
confirmed.Operations = nil
return confirmed.waitForSessionIdle(ctx, id)
}
func (c *AgentClient) Generate(ctx context.Context, config ModelConfig, prompt string) (text string, err error) {
@@ -97,7 +122,7 @@ func (c *AgentClient) Generate(ctx context.Context, config ModelConfig, prompt s
defer cancel()
if err != nil {
c.captureTimedOutEvidence(ctx, err, config, watch)
if stopErr := c.Interrupt(cleanup, c.SessionID); stopErr != nil {
if stopErr := c.InterruptAndConfirm(cleanup, c.SessionID); stopErr != nil {
err = fmt.Errorf("agent interrupt failed; retained session %s needs inspection: %w", c.SessionID, errors.Join(err, stopErr, ErrStageCleanup))
return
}
+2 -2
View File
@@ -52,8 +52,8 @@ func TestAgentOwnershipPermissionsAndInterrupt(t *testing.T) {
t.Fatal("unexpected call", args)
return nil, nil
}
if _, err := client.Generate(context.Background(), ModelConfig{}, "This call has no tools."); err == nil || !interrupted || owned {
t.Fatal("failed generation did not stop and release ownership")
if _, err := client.Generate(context.Background(), ModelConfig{}, "This call has no tools."); !errors.Is(err, ErrStageCleanup) || !interrupted || !owned {
t.Fatal("interrupt acknowledgement without idle confirmation must retain ownership")
}
}
+6
View File
@@ -82,6 +82,9 @@ func TestBudgetsAccumulateAcrossRetriesAndBlockFurtherGeneration(t *testing.T) {
var sessions, generations atomic.Int64
client := &OpenCodeClient{command: func(ctx context.Context, _ string, args ...string) ([]byte, error) {
path := args[2]
if path == "/api/session/active" {
return []byte(`{"data":{}}`), nil
}
if path == "/api/session" {
return []byte(fmt.Sprintf(`{"data":{"id":"ses_attempt_%d"}}`, sessions.Add(1))), nil
}
@@ -147,6 +150,9 @@ func TestLiveAgentBudgetInterruptsAndDoesNotReplay(t *testing.T) {
return []byte(`{"data":{"id":"ses_agent_budget"}}`), nil
}
if path == "/api/session/active" {
if interrupts.Load() > 0 {
return []byte(`{"data":{}}`), nil
}
<-ctx.Done()
return nil, ctx.Err()
}
+14 -2
View File
@@ -59,7 +59,7 @@ func (ModelStage) Spec(def Definition) (ModelSpec, error) {
}
for block := range def.Blocks {
switch block {
case "output", "prompt", "edges", "configs", "tools", "permissions":
case "output", "prompt", "edges", "configs", "tools", "permissions", "checks", "verification":
default:
return ModelSpec{}, fmt.Errorf("unexpected block %q for model stage", block)
}
@@ -70,6 +70,9 @@ func (ModelStage) Spec(def Definition) (ModelSpec, error) {
if _, err := StagePermissions(def); err != nil {
return ModelSpec{}, err
}
if _, err := VerificationTargetFor(def); err != nil {
return ModelSpec{}, err
}
for _, key := range []string{"provider", "model"} {
if def.Configs[key] == "" {
return ModelSpec{}, fmt.Errorf("missing config %s", key)
@@ -77,7 +80,7 @@ func (ModelStage) Spec(def Definition) (ModelSpec, error) {
}
for key := range def.Configs {
switch key {
case "provider", "model", "retries", "timeout", "budget_usd", "budget_tokens", "harness":
case "provider", "model", "retries", "timeout", "budget_usd", "budget_tokens", "harness", "delivery":
default:
return ModelSpec{}, fmt.Errorf("unknown config %q", key)
}
@@ -92,6 +95,15 @@ func (ModelStage) Spec(def Definition) (ModelSpec, error) {
if backend != "opencode" && backend != "pi" {
return ModelSpec{}, fmt.Errorf("harness must be opencode or pi")
}
if delivery := def.Configs["delivery"]; delivery != "" && delivery != "true" {
return ModelSpec{}, fmt.Errorf("delivery must be true when set")
}
if def.Configs["delivery"] == "true" && backend != "opencode" {
return ModelSpec{}, fmt.Errorf("delivery is supported only by the opencode harness")
}
if _, present := def.Blocks["checks"]; present && def.Configs["delivery"] != "true" {
return ModelSpec{}, fmt.Errorf("checks block is supported only by delivery stages")
}
if backend == "pi" {
tools, _, _ := ToolNames(def)
if _, err := piTools(tools); err != nil {
+2 -2
View File
@@ -36,8 +36,8 @@ func TestModelFailureCategoriesAreSanitizedAndDistinct(t *testing.T) {
if typed := wrapModelStageFailure("worker", &openCodeError{operation: "get /api/session", status: 503}); !strings.Contains(typed.Error(), "HARNESS TRANSPORT") || !errors.As(typed, new(*openCodeError)) {
t.Fatalf("typed harness error was not classified: %v", typed)
}
if deadline := wrapModelStageFailure("worker", context.DeadlineExceeded); !strings.Contains(deadline.Error(), "HARNESS TRANSPORT") || !errors.Is(deadline, context.DeadlineExceeded) {
t.Fatalf("context deadline was not classified as harness: %v", deadline)
if deadline := wrapModelStageFailure("worker", context.DeadlineExceeded); strings.Contains(deadline.Error(), "HARNESS TRANSPORT") || !errors.Is(deadline, context.DeadlineExceeded) {
t.Fatalf("bare context deadline was misclassified or lost: %v", deadline)
}
if modelFailureCategory(nil) != "" || wrapModelStageFailure("worker", nil) != nil {
t.Fatal("nil failure was classified")
+26 -1
View File
@@ -4,6 +4,7 @@ import (
"context"
"errors"
"fmt"
"time"
)
// Retained model-stage failure categories. A strict final-result validation
@@ -13,8 +14,27 @@ import (
const (
finalResultValidation = "FINAL RESULT VALIDATION"
harnessTransport = "HARNESS TRANSPORT"
stageDeadline = "STAGE DEADLINE"
)
// stageTimeoutCause identifies this attempt's configured timer, not a parent
// cancellation or a request's own deadline. It is carried through budget contexts.
type stageTimeoutCause struct{ timeout time.Duration }
func (e *stageTimeoutCause) Error() string { return "configured stage timeout reached" }
func (e *stageTimeoutCause) Unwrap() error { return context.DeadlineExceeded }
type stageDeadlineError struct {
timeout time.Duration
session string
}
func (e *stageDeadlineError) Error() string {
return fmt.Sprintf("configured stage deadline reached (timeout %s) while waiting for session idle; retained OpenCode session %.128s was still active at the last poll; stage had not finished", e.timeout, RedactEvidence(e.session))
}
func (e *stageDeadlineError) Unwrap() error { return context.DeadlineExceeded }
// finalResultGuidance is a fixed sanitized sentence appended only to validation
// diagnostics. It states the retained side effects and the cost of a full retry.
const finalResultGuidance = "completed edits and tool side effects are retained and not undone; a full retry repeats work"
@@ -51,11 +71,14 @@ func (c *PiClient) workAdmitted() bool { return c.WorkStarted }
// the label stays sanitized and bounded. Validation is checked first: a
// structured-output failure is never reported as a transport failure and vice versa.
func modelFailureCategory(err error) string {
var deadline *stageDeadlineError
switch {
case errors.Is(err, ErrInvalidModelResult):
return finalResultValidation
case harnessTransportFailure(err), errors.Is(err, context.DeadlineExceeded):
case harnessTransportFailure(err):
return harnessTransport
case errors.As(err, &deadline):
return stageDeadline
default:
return ""
}
@@ -73,6 +96,8 @@ func wrapModelStageFailure(key string, err error) error {
return fmt.Errorf("model stage %s: %s: %w; %s", key, finalResultValidation, err, finalResultGuidance)
case harnessTransport:
return fmt.Errorf("model stage %s: %s: %w", key, harnessTransport, err)
case stageDeadline:
return fmt.Errorf("model stage %s: %s: %w", key, stageDeadline, err)
default:
return fmt.Errorf("model stage %s: %w", key, err)
}
+7 -2
View File
@@ -61,7 +61,7 @@ func (m ModelStage) Run(ctx context.Context, def Definition, input Values) (Resu
return Result{}, err
}
spec.Config.Budget.Attempt()
attemptCtx, cancel := context.WithTimeout(ctx, spec.Config.Timeout)
attemptCtx, cancel := context.WithTimeoutCause(ctx, spec.Config.Timeout, &stageTimeoutCause{timeout: spec.Config.Timeout})
text, generateErr := client.Generate(attemptCtx, spec.Config, prompt)
if generateErr == nil && attemptCtx.Err() != nil {
generateErr = attemptCtx.Err()
@@ -96,11 +96,16 @@ func (m ModelStage) Run(ctx context.Context, def Definition, input Values) (Resu
}
} else {
err = generateErr
if attemptCtx.Err() != nil && !errors.Is(err, ErrBudgetExceeded) && !errors.Is(err, ErrBudgetAccounting) {
// Do not append the same context error twice: the wait/transport
// error may already preserve it, with a more precise diagnostic.
if attemptCtx.Err() != nil && !errors.Is(err, attemptCtx.Err()) && !errors.Is(err, ErrBudgetExceeded) && !errors.Is(err, ErrBudgetAccounting) {
err = errors.Join(err, attemptCtx.Err())
}
cancel()
if ctx.Err() != nil {
if harnessTransportFailure(err) {
return Result{}, wrapModelStageFailure(def.Key, err)
}
return Result{}, ctx.Err()
}
if errors.Is(err, ErrBudgetExceeded) || errors.Is(err, ErrBudgetAccounting) || errors.Is(err, ErrStageCleanup) {
+8 -3
View File
@@ -217,6 +217,11 @@ func (c OpenCodeClient) waitForSessionIdle(ctx context.Context, sessionID string
select {
case <-ctx.Done():
timer.Stop()
waitErr := fmt.Errorf("waiting for session idle; retained OpenCode session %.128s; stage had not finished: %w", RedactEvidence(sessionID), ctx.Err())
var timeout *stageTimeoutCause
if errors.Is(ctx.Err(), context.DeadlineExceeded) && errors.As(context.Cause(ctx), &timeout) {
waitErr = &stageDeadlineError{timeout: timeout.timeout, session: sessionID}
}
// A settled session is the only success signal; retain one bounded
// sanitized record so a timed-out wait is not an empty transcript.
c.Operations.record(OperationEvidence{
@@ -225,9 +230,9 @@ func (c OpenCodeClient) waitForSessionIdle(ctx context.Context, sessionID string
Executable: opencode.Executable(),
Directory: c.Directory,
ExitCode: -1,
Diagnostic: "stage deadline reached while waiting for the agent session to become idle",
Diagnostic: waitErr.Error(),
}, started)
return ctx.Err()
return waitErr
case <-timer.C:
}
}
@@ -316,7 +321,7 @@ func (c *OpenCodeClient) Generate(ctx context.Context, config ModelConfig, promp
cleanup, cancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second)
defer cancel()
if err != nil {
if stopErr := c.Interrupt(cleanup, session.Data.ID); stopErr != nil {
if stopErr := c.InterruptAndConfirm(cleanup, session.Data.ID); stopErr != nil {
err = fmt.Errorf("retained OpenCode session %s; interrupt failed: %w", session.Data.ID, errors.Join(err, stopErr, ErrStageCleanup))
return
}
+15 -3
View File
@@ -28,6 +28,7 @@ type PiClient struct {
// ArtifactsDir is exported to the spawned Pi process as
// PIPELINES_ARTIFACTS_DIR so its tools can write stage artifacts.
ArtifactsDir string
ArtifactAccess ArtifactAccess
Policy ToolPolicy
OnSession func(string, bool) error
OnOwnership func(string, string, bool) error
@@ -128,7 +129,7 @@ func (c *PiClient) Generate(ctx context.Context, config ModelConfig, prompt stri
if err != nil {
return "", err
}
policy := map[string]any{"directory": c.Directory, "tools": tools, "paths": c.WritePaths, "edit": c.Policy.Edit, "shell": c.Policy.Shell}
policy := map[string]any{"directory": c.Directory, "tools": tools, "paths": c.WritePaths, "edit": c.Policy.Edit, "shell": c.Policy.Shell, "artifact_write": c.ArtifactAccess.Write, "artifact_read": c.ArtifactAccess.Read}
if c.WritePaths == nil {
policy["paths"] = []string{}
}
@@ -237,6 +238,10 @@ func (c *PiClient) Generate(ctx context.Context, config ModelConfig, prompt stri
}
close(done)
close(heartbeatDone)
// agent_end/abort acknowledgement and the parent's exit alone do not
// prove its shell descendants exited. A surviving process group retains
// ownership rather than handing the resource to another stage.
quiet = quiet && errors.Is(syscall.Kill(-c.cmd.Process.Pid, 0), syscall.ESRCH)
if quiet {
_ = os.WriteFile(filepath.Join(c.Reference, "quiescent"), []byte("quiescent"), 0600)
}
@@ -531,9 +536,16 @@ func InterruptPi(ctx context.Context, reference string) error {
if !filepath.IsAbs(reference) {
return fmt.Errorf("invalid retained Pi reference")
}
if _, err := os.Stat(filepath.Join(reference, "owner.json")); err != nil {
data, err := os.ReadFile(filepath.Join(reference, "owner.json"))
if err != nil {
return errors.Join(err, ErrStageCleanup)
}
var owner struct {
PID int `json:"pid"`
}
if json.Unmarshal(data, &owner) != nil || owner.PID <= 1 {
return errors.Join(fmt.Errorf("invalid retained Pi owner"), ErrStageCleanup)
}
old := time.Unix(0, 0)
if err := os.Chtimes(filepath.Join(reference, "lease"), old, old); err != nil {
return err
@@ -541,7 +553,7 @@ func InterruptPi(ctx context.Context, reference string) error {
ticker := time.NewTicker(100 * time.Millisecond)
defer ticker.Stop()
for {
if data, err := os.ReadFile(filepath.Join(reference, "quiescent")); err == nil && string(data) == "quiescent" {
if data, err := os.ReadFile(filepath.Join(reference, "quiescent")); err == nil && string(data) == "quiescent" && errors.Is(syscall.Kill(owner.PID, 0), syscall.ESRCH) && errors.Is(syscall.Kill(-owner.PID, 0), syscall.ESRCH) {
return nil
}
select {
+33 -2
View File
@@ -8,6 +8,24 @@ export function permit(policy, tool, args) {
if (policy.shell !== null && !policy.shell.includes(args.command)) return "Shell command is not permitted";
return null;
}
const artifact = (absolute, write) => {
const permitted = write ? [policy.artifact_write] : policy.artifact_read;
for (const directory of permitted) {
if (typeof directory !== "string" || !path.isAbsolute(directory)) continue;
let parent = absolute;
while (!fs.existsSync(parent)) {
const next = path.dirname(parent);
if (next === parent) break;
parent = next;
}
if (!fs.existsSync(parent)) continue;
const resolved = fs.realpathSync(parent);
const target = path.join(resolved, path.relative(parent, absolute));
const root = fs.realpathSync(directory);
if (target === root || target.startsWith(root + path.sep)) return true;
}
return false;
};
if (tool === "edit" || tool === "write") {
if (typeof args.path !== "string") return "Missing file path";
if (args.path.startsWith("~") || args.path.startsWith("@")) return "Use a literal path, not a harness path shortcut";
@@ -20,7 +38,10 @@ export function permit(policy, tool, args) {
}
const resolved = fs.realpathSync(parent);
const root = fs.realpathSync(policy.directory);
if (resolved !== root && !resolved.startsWith(root + path.sep)) return "Edit escapes run directory";
if (resolved !== root && !resolved.startsWith(root + path.sep)) {
if (!artifact(absolute, true)) return "Edit escapes run directory";
return null;
}
const relative = path.relative(path.resolve(policy.directory), absolute).split(path.sep).join("/");
const canonical = path.relative(root, path.join(resolved, path.relative(parent, absolute))).split(path.sep).join("/");
if (relative.startsWith("../") || path.isAbsolute(relative)) return "Edit escapes run directory";
@@ -31,9 +52,19 @@ export function permit(policy, tool, args) {
});
if (policy.edit !== null && (!matches(relative) || !matches(canonical))) return "Edit is outside stage patterns";
}
if (tool === "read" && typeof args.path === "string") {
if (["read", "grep", "find"].includes(tool) && typeof args.path === "string") {
const name = path.basename(args.path);
if (/^(.*\.env(?:\..*)?|pipelines\.db.*)$/.test(name)) return "Protected file cannot be read";
const absolute = path.resolve(policy.directory, args.path);
let parent = absolute;
while (!fs.existsSync(parent)) {
const next = path.dirname(parent);
if (next === parent) return "Cannot resolve read path";
parent = next;
}
const resolved = fs.realpathSync(parent);
const root = fs.realpathSync(policy.directory);
if (resolved !== root && !resolved.startsWith(root + path.sep) && !artifact(absolute, false)) return "Read escapes run directory";
}
return null;
}
+72
View File
@@ -0,0 +1,72 @@
package stage
import (
"encoding/json"
"os"
"os/exec"
"path/filepath"
"testing"
)
func TestPiPolicyArtifactScope(t *testing.T) {
if _, err := exec.LookPath("node"); err != nil {
t.Skip("node is required to exercise the Pi policy extension")
}
root := t.TempDir()
workspace := filepath.Join(root, "workspace")
current := filepath.Join(root, "artifacts [7]", "2")
prior := filepath.Join(root, "artifacts [7]", "1")
sibling := filepath.Join(root, "artifacts [7]", "3")
for _, directory := range []string{workspace, current, prior, sibling} {
if err := os.MkdirAll(directory, 0700); err != nil {
t.Fatal(err)
}
}
policy := map[string]any{
"directory": workspace,
"tools": []string{"read", "grep", "find", "edit"},
"paths": []string{},
"edit": nil,
"shell": nil,
"artifact_write": current,
"artifact_read": []string{current, prior},
}
policyJSON, err := json.Marshal(policy)
if err != nil {
t.Fatal(err)
}
policyFile := filepath.Join(root, "policy.mjs")
if err := os.WriteFile(policyFile, piPolicy, 0600); err != nil {
t.Fatal(err)
}
script := `const { permit } = await import(process.argv[1]);
const policy = JSON.parse(process.argv[2]);
const checks = JSON.parse(process.argv[3]);
console.log(JSON.stringify(checks.map(([tool, path]) => permit(policy, tool, {path}))));`
checks := [][2]string{
{"edit", filepath.Join(current, "report.md")},
{"edit", filepath.Join(prior, "report.md")},
{"read", filepath.Join(prior, "report.md")},
{"grep", filepath.Join(prior, "report.md")},
{"find", filepath.Join(current, "reports")},
{"read", filepath.Join(sibling, "report.md")},
{"grep", filepath.Join(root, "other-run", "1", "report.md")},
{"find", filepath.Join(current, "..", "3", "report.md")},
{"read", filepath.Join(prior, ".env")},
}
checksJSON, err := json.Marshal(checks)
if err != nil {
t.Fatal(err)
}
output, err := exec.Command("node", "--input-type=module", "--eval", script, "file://"+policyFile, string(policyJSON), string(checksJSON)).Output()
if err != nil {
t.Fatal(err)
}
var got []any
if err := json.Unmarshal(output, &got); err != nil {
t.Fatal(err)
}
if len(got) != len(checks) || got[0] != nil || got[1] == nil || got[2] != nil || got[3] != nil || got[4] != nil || got[5] == nil || got[6] == nil || got[7] == nil || got[8] == nil {
t.Fatalf("Pi artifact policy results = %#v", got)
}
}
+44
View File
@@ -0,0 +1,44 @@
package stage
import (
"context"
"encoding/json"
"errors"
"os"
"os/exec"
"path/filepath"
"syscall"
"testing"
"time"
)
func TestResourcePiReceiptRequiresProcessExit(t *testing.T) {
// A test-owned process only: no Pi provider, shell/ADB command or phone.
cmd := exec.Command("sleep", "5")
cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
if err := cmd.Start(); err != nil {
t.Fatal(err)
}
defer func() { cmd.Process.Kill(); cmd.Wait() }()
dir := t.TempDir()
owner, _ := json.Marshal(map[string]any{"pid": cmd.Process.Pid})
for name, data := range map[string][]byte{"owner.json": owner, "lease": []byte("lease"), "quiescent": []byte("quiescent")} {
if err := os.WriteFile(filepath.Join(dir, name), data, 0600); err != nil {
t.Fatal(err)
}
}
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Millisecond)
defer cancel()
if err := InterruptPi(ctx, dir); !errors.Is(err, ErrStageCleanup) {
t.Fatal("receipt authorized handoff while process lived", err)
}
if err := cmd.Process.Kill(); err != nil {
t.Fatal(err)
}
cmd.Wait()
ctx2, cancel2 := context.WithTimeout(context.Background(), time.Second)
defer cancel2()
if err := InterruptPi(ctx2, dir); err != nil {
t.Fatal("confirmed process exit not accepted", err)
}
}
+1 -1
View File
@@ -251,7 +251,7 @@ func TestWaitForSessionIdleRecordsOneTimeoutEvidence(t *testing.T) {
t.Fatalf("expected exactly one timeout evidence, got %d", len(log.Failures))
}
entry := log.Failures[0]
if entry.Operation != "wait for session idle" || !strings.Contains(entry.Diagnostic, "deadline reached") {
if entry.Operation != "wait for session idle" || !strings.Contains(entry.Diagnostic, "context deadline exceeded") {
t.Fatalf("timeout evidence not sanitized as expected: %+v", entry)
}
clean := &OperationLog{}
+140
View File
@@ -0,0 +1,140 @@
package stage
import (
"context"
"errors"
"net/http"
"net/http/httptest"
"strings"
"testing"
"time"
"git.a-wan.net/alan/pipelines/internal/opencode"
)
func TestConfiguredStageDeadlineWhileSessionActive(t *testing.T) {
def := modelDefinition()
def.Blocks["tools"] = "edit"
def.Configs["timeout"] = "20ms"
def.Configs["retries"] = "2"
log := &OperationLog{}
creates, prompts, interrupts := 0, 0, 0
agent := &AgentClient{OpenCodeClient: OpenCodeClient{Backend: "v2", Operations: log, sessionPollInterval: time.Hour}}
agent.command = func(_ context.Context, _ string, args ...string) ([]byte, error) {
switch args[2] {
case "/api/session":
creates++
return []byte(`{"data":{"id":"ses_deadline"}}`), nil
case "/api/session/ses_deadline/prompt":
prompts++
return []byte(`{}`), nil
case "/api/session/active":
return []byte(`{"data":{"ses_deadline":{"type":"running"}}}`), nil
case "/api/session/ses_deadline/interrupt":
interrupts++
return []byte(`{}`), nil
default:
// Best-effort timeout readback/usage cannot convert the failure to success.
return []byte(`{}`), nil
}
}
result, err := (ModelStage{Client: agent}).Run(context.Background(), def, nil)
if !errors.Is(err, context.DeadlineExceeded) || modelFailureCategory(err) != stageDeadline || errors.Is(err, ErrHarnessTransport) {
t.Fatalf("deadline identity/category: %v", err)
}
for _, want := range []string{"STAGE DEADLINE", "worker", "timeout 20ms", "waiting for session idle", "ses_deadline", "stage had not finished"} {
if !strings.Contains(err.Error(), want) {
t.Fatalf("missing %q: %v", want, err)
}
}
if result.Output != nil || creates != 1 || prompts != 1 || interrupts != 1 || !agent.WorkStarted {
t.Fatalf("work replayed, cleanup skipped or incomplete turn accepted: %+v creates=%d prompts=%d interrupts=%d", result, creates, prompts, interrupts)
}
if len(log.Failures) != 1 || !strings.Contains(log.Failures[0].Diagnostic, "timeout 20ms") {
t.Fatalf("deadline evidence missing: %+v", log.Failures)
}
}
func TestWaitCancellationIsNotConfiguredStageDeadline(t *testing.T) {
for _, cause := range []error{context.Canceled, ErrBudgetExceeded, ErrBudgetAccounting} {
t.Run(cause.Error(), func(t *testing.T) {
attempt, stop := context.WithTimeoutCause(context.Background(), time.Hour, &stageTimeoutCause{timeout: time.Hour})
defer stop()
ctx, cancel := context.WithCancelCause(attempt)
log := &OperationLog{}
client := OpenCodeClient{Operations: log, command: func(context.Context, string, ...string) ([]byte, error) {
cancel(cause)
return []byte(`{"data":{"ses_wait":{"type":"running"}}}`), nil
}}
err := client.waitForSessionIdle(ctx, "ses_wait")
if !errors.Is(err, context.Canceled) || modelFailureCategory(err) != "" || len(log.Failures) != 1 {
t.Fatalf("cancellation misclassified: %v, %+v", err, log.Failures)
}
if strings.Contains(log.Failures[0].Diagnostic, "deadline") {
t.Fatalf("cancellation recorded as deadline: %+v", log.Failures)
}
})
}
}
func TestDeadlineWithTransportRetainsUnderlyingIdentity(t *testing.T) {
transport := &openCodeError{operation: "get /api/session/active", status: 503, cause: errors.New("request failed")}
err := wrapModelStageFailure("worker", errors.Join(&stageDeadlineError{timeout: time.Second, session: "ses_wait"}, transport))
var got *openCodeError
if modelFailureCategory(err) != harnessTransport || !errors.Is(err, context.DeadlineExceeded) || !errors.As(err, &got) || got != transport || !errors.Is(err, transport.cause) {
t.Fatalf("combined failure lost transport/deadline identity: %v", err)
}
if !strings.Contains(err.Error(), "HTTP 503") || !strings.Contains(err.Error(), "get /api/session/active") {
t.Fatalf("combined failure lost operation/status: %v", err)
}
bounded := (&stageDeadlineError{timeout: time.Hour, session: strings.Repeat("s", 10000) + " Bearer private-token"}).Error()
if len(bounded) > 512 || strings.Contains(bounded, "private-token") {
t.Fatal("deadline diagnostic was not bounded/redacted")
}
}
func TestParentDeadlinePreservesModelTransportFailure(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Millisecond)
defer cancel()
transport := &openCodeError{operation: "get /api/session/active", status: 504, cause: errors.New("request failed")}
client := generatorFunc(func(ctx context.Context, _ ModelConfig, _ string) (string, error) {
<-ctx.Done()
return "", errors.Join(ctx.Err(), transport)
})
_, err := (ModelStage{Client: client}).Run(ctx, modelDefinition(), nil)
var got *openCodeError
if !errors.Is(err, context.DeadlineExceeded) || !errors.As(err, &got) || got != transport || modelFailureCategory(err) != harnessTransport {
t.Fatalf("parent deadline erased underlying request failure: %v", err)
}
}
func TestNativeHTTPAndConnectionFailuresRemainTransport(t *testing.T) {
for _, mode := range []string{"http", "connection"} {
t.Run(mode, func(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
w.WriteHeader(http.StatusServiceUnavailable)
_, _ = w.Write([]byte(`{"message":"service unavailable","token":"private-token"}`))
}))
defer srv.Close()
if mode == "connection" {
srv.Close()
}
log := &OperationLog{}
client := OpenCodeClient{Backend: "v1", Operations: log, connection: &opencode.Connection{Backend: "v1", URL: srv.URL}}
_, err := client.call(context.Background(), "get", "/api/session", nil)
var typed *openCodeError
if !errors.As(err, &typed) || modelFailureCategory(err) != harnessTransport || len(log.Failures) != 1 {
t.Fatalf("real request failure misclassified: %v %+v", err, log.Failures)
}
if mode == "http" && (typed.status != 503 || log.Failures[0].HTTPStatus != 503 || !strings.Contains(log.Failures[0].Stdout, "service unavailable")) {
t.Fatalf("HTTP status/evidence lost: %v %+v", err, log.Failures)
}
if mode == "connection" && (log.Failures[0].Diagnostic != "OpenCode native API request failed" || log.Failures[0].NativeOperation != "GET /session") {
t.Fatalf("connection evidence lost: %+v", log.Failures)
}
if strings.Contains(err.Error()+log.Transcript(""), "private-token") {
t.Fatal("credential leaked in transport diagnostic/evidence")
}
})
}
}
+109
View File
@@ -7,6 +7,84 @@ import (
"strings"
)
// ArtifactAccess is runtime-only, server-derived access to artifact visits.
// Empty fields deliberately grant nothing outside the workspace.
type ArtifactAccess struct {
Write string
Read []string
}
// LandingAccess is the server-derived external capability for a verified
// delivery operation. Empty or malformed values grant nothing.
type LandingAccess struct {
MainWorktree string
GitMetadata string
LockPath string
}
func (a LandingAccess) valid() bool {
for _, path := range []string{a.MainWorktree, a.GitMetadata, a.LockPath} {
if path == "" || !filepath.IsAbs(path) || filepath.Clean(path) != path || strings.ContainsAny(path, "*?\\") {
return false
}
}
// The server grants the canonical repository metadata directory as OpenCode
// V2's smallest external-directory unit. Keep the lock inside that directory
// so a caller cannot turn a valid metadata capability into access elsewhere.
return a.GitMetadata == filepath.Join(a.MainWorktree, ".git") &&
a.LockPath == filepath.Join(a.GitMetadata, "pipelines-landing.lock")
}
// DeliveryEnabled is the narrow, declarative capability check used before the
// runner supplies any landing paths. Only a validated model definition can ask
// for it; runtime paths are still derived by the server.
func DeliveryEnabled(def Definition) (bool, error) {
delivery := def.Configs["delivery"]
if delivery == "" {
return false, nil
}
if delivery != "true" {
return false, fmt.Errorf("delivery must be true when set")
}
if EffectiveType(def.Type) != ModelType {
return false, fmt.Errorf("delivery is supported only by model stages")
}
if _, err := (ModelStage{}).Spec(def); err != nil {
return false, fmt.Errorf("invalid delivery model stage: %w", err)
}
if _, err := DeliveryChecks(def); err != nil {
return false, err
}
return true, nil
}
// DeliveryChecks returns the bounded, definition-declared checks that the
// runner executes against a committed delivery candidate before canonical main
// is changed. They
// are intentionally a block rather than a generic runner setting: each
// workflow owns its repository-specific verification commands.
func DeliveryChecks(def Definition) ([]string, error) {
text := strings.TrimSpace(def.Blocks["checks"])
if text == "" {
return nil, fmt.Errorf("delivery stages require a nonempty checks block")
}
var checks []string
for _, line := range strings.Split(text, "\n") {
check := strings.TrimSpace(line)
if check == "" {
continue
}
if len(check) > 4096 {
return nil, fmt.Errorf("delivery check exceeds 4096 bytes")
}
checks = append(checks, check)
}
if len(checks) == 0 || len(checks) > 20 {
return nil, fmt.Errorf("delivery stages require 1 to 20 checks")
}
return checks, nil
}
// ToolNames validates model-stage permissions, not general graph semantics.
// A present empty block explicitly denies all tools.
func ToolNames(def Definition) ([]string, bool, error) {
@@ -128,12 +206,43 @@ func (c *AgentClient) permissionRules() []map[string]string {
}
}
}
// OpenCode V2 resolves an external file to its containing directory and
// checks external_directory against <directory>/*. Do not grant a parent:
// each visit is deliberately an independent capability.
readArtifacts := false
writeArtifacts := false
for _, tool := range tools {
readArtifacts = readArtifacts || tool == "read" || tool == "glob" || tool == "grep"
writeArtifacts = writeArtifacts || tool == "edit"
}
if readArtifacts {
for _, directory := range c.ArtifactAccess.Read {
if artifactDirectory(directory) {
allow("external_directory", filepath.ToSlash(filepath.Join(directory, "*")))
}
}
}
if writeArtifacts && artifactDirectory(c.ArtifactAccess.Write) {
allow("external_directory", filepath.ToSlash(filepath.Join(c.ArtifactAccess.Write, "*")))
allow("edit", filepath.ToSlash(filepath.Join(c.ArtifactAccess.Write, "*")))
}
if c.LandingAccess.valid() {
allow("external_directory", filepath.ToSlash(filepath.Join(c.LandingAccess.MainWorktree, "*")))
allow("external_directory", filepath.ToSlash(filepath.Join(c.LandingAccess.GitMetadata, "*")))
}
for _, pattern := range []string{"../*", "*/../*"} {
rules = append(rules, map[string]string{"action": "external_directory", "resource": pattern, "effect": "deny"})
}
for _, pattern := range []string{"*.env", "*.env.*", "pipelines.db*"} {
rules = append(rules, map[string]string{"action": "read", "resource": pattern, "effect": "deny"})
}
return rules
}
func artifactDirectory(directory string) bool {
return directory != "" && filepath.IsAbs(directory) && filepath.Clean(directory) == directory && !strings.ContainsAny(directory, "*?\\")
}
func matchesEditPolicy(patterns []string, path string) bool {
for _, pattern := range patterns {
// OpenCode resource wildcards match across directory separators.
+139
View File
@@ -139,3 +139,142 @@ func TestDefaultEditScopeIsLanguageIndependentRunDirectory(t *testing.T) {
t.Fatal("external directories were enabled")
}
}
func TestArtifactPermissionsAreExactAndToolScoped(t *testing.T) {
current := "/artifacts/run [7]/2"
prior := "/artifacts/run [7]/1"
sibling := "/artifacts/run 7/3"
otherRun := "/artifacts/run 8/1"
client := &AgentClient{OpenCodeClient: OpenCodeClient{Directory: "/repo"}, Tools: []string{"read", "edit"}, ArtifactAccess: ArtifactAccess{Write: current, Read: []string{current, prior}}}
rules := client.permissionRules()
for _, path := range []string{current + "/report.md", prior + "/report.md"} {
if permissionEffect(rules, "external_directory", path) != "allow" {
t.Fatalf("artifact directory denied: %q", path)
}
}
if permissionEffect(rules, "edit", current+"/report.md") != "allow" || permissionEffect(rules, "edit", prior+"/report.md") != "deny" {
t.Fatal("artifact writes were not limited to the current visit")
}
for _, path := range []string{sibling + "/report.md", otherRun + "/report.md", current + "/../3/report.md", "/artifacts/run 7/20/report.md"} {
if permissionEffect(rules, "external_directory", path) != "deny" {
t.Fatalf("unapproved artifact scope allowed: %q", path)
}
}
client.Tools = []string{}
if permissionEffect(client.permissionRules(), "external_directory", current+"/report.md") != "deny" {
t.Fatal("no-tool stage gained artifact access")
}
client.Tools = []string{"read"}
rules = client.permissionRules()
if permissionEffect(rules, "external_directory", prior+"/report.md") != "allow" || permissionEffect(rules, "edit", current+"/report.md") != "deny" {
t.Fatal("read-only artifact stage gained write access")
}
client.ArtifactAccess = ArtifactAccess{}
if permissionEffect(client.permissionRules(), "external_directory", current+"/report.md") != "deny" {
t.Fatal("missing artifact scope enabled an external directory")
}
}
func TestLandingPermissionsAreExactAndServerDerived(t *testing.T) {
access := LandingAccess{MainWorktree: "/repos/pipelines", GitMetadata: "/repos/pipelines/.git", LockPath: "/repos/pipelines/.git/pipelines-landing.lock"}
client := &AgentClient{Tools: []string{"read", "shell"}, LandingAccess: access}
rules := client.permissionRules()
for _, path := range []string{"/repos/pipelines/go.mod", "/repos/pipelines/.git/worktrees/item/HEAD", "/repos/pipelines/.git/pipelines-landing.lock"} {
if permissionEffect(rules, "external_directory", path) != "allow" {
t.Fatalf("verified landing path denied: %q", path)
}
}
for _, path := range []string{"/repos/other/go.mod", "/repos/pipelines/../other/go.mod", "/Users/alan/.git/config", "/.git/config"} {
if permissionEffect(rules, "external_directory", path) != "deny" {
t.Fatalf("unverified landing path allowed: %q", path)
}
}
client.LandingAccess.MainWorktree = "/repos/pipelines/../other"
if permissionEffect(client.permissionRules(), "external_directory", "/repos/other/go.mod") != "deny" {
t.Fatal("non-canonical landing capability allowed")
}
client.LandingAccess = LandingAccess{MainWorktree: "/repos/pipelines", GitMetadata: "/repos/pipelines/.git", LockPath: "/tmp/pipelines-landing.lock"}
if permissionEffect(client.permissionRules(), "external_directory", "/tmp/pipelines-landing.lock") != "deny" {
t.Fatal("landing lock outside verified metadata allowed")
}
client.LandingAccess = LandingAccess{MainWorktree: "/repos/pipelines", GitMetadata: "/repos/other/.git", LockPath: "/repos/other/.git/pipelines-landing.lock"}
if permissionEffect(client.permissionRules(), "external_directory", "/repos/other/.git/config") != "deny" {
t.Fatal("unrelated Git metadata allowed")
}
}
func TestDeliveryEnabledRequiresValidatedModelStage(t *testing.T) {
def := modelDefinition()
def.Configs["delivery"] = "true"
def.Blocks["checks"] = "go test ./..."
if enabled, err := DeliveryEnabled(def); err != nil || !enabled {
t.Fatalf("valid delivery stage = %v, %v", enabled, err)
}
for name, change := range map[string]func(*Definition){
"malformed value": func(d *Definition) { d.Configs["delivery"] = "yes" },
"non-model": func(d *Definition) { d.Type = HumanType },
"invalid model": func(d *Definition) { d.Configs["unexpected"] = "value" },
} {
t.Run(name, func(t *testing.T) {
copy := def
copy.Configs = map[string]string{}
for key, value := range def.Configs {
copy.Configs[key] = value
}
change(&copy)
if enabled, err := DeliveryEnabled(copy); err == nil || enabled {
t.Fatalf("invalid delivery stage = %v, %v", enabled, err)
}
})
}
}
func TestDeliveryRejectsPiBeforeAgentExecution(t *testing.T) {
def := modelDefinition()
def.Configs["delivery"] = "true"
def.Configs["harness"] = "pi"
def.Blocks["checks"] = "true"
if err := (ModelStage{}).Validate(def); err == nil || !strings.Contains(err.Error(), "opencode harness") {
t.Fatal("Pi delivery was accepted", err)
}
delete(def.Configs, "delivery")
delete(def.Blocks, "checks")
if err := (ModelStage{}).Validate(def); err != nil {
t.Fatal("Pi non-delivery was changed", err)
}
}
func TestDeliveryChecksRequireBoundedDeliveryOnlyDeclaration(t *testing.T) {
def := modelDefinition()
def.Configs["delivery"] = "true"
for name, mutate := range map[string]func(*Definition){
"missing": func(d *Definition) {},
"empty": func(d *Definition) { d.Blocks["checks"] = "\n" },
"non delivery": func(d *Definition) { d.Blocks["checks"] = "go test ./..." },
"too many checks": func(d *Definition) { d.Blocks["checks"] = strings.Repeat("true\n", 21) },
"oversized check": func(d *Definition) { d.Blocks["checks"] = strings.Repeat("x", 4097) },
} {
t.Run(name, func(t *testing.T) {
copy := def
copy.Blocks = map[string]string{}
copy.Configs = map[string]string{}
for key, value := range def.Blocks {
copy.Blocks[key] = value
}
for key, value := range def.Configs {
copy.Configs[key] = value
}
mutate(&copy)
if name == "non delivery" {
delete(copy.Configs, "delivery")
if _, err := (ModelStage{}).Spec(copy); err == nil {
t.Fatal("non-delivery checks block was accepted")
}
return
}
if _, err := DeliveryEnabled(copy); err == nil {
t.Fatal("invalid delivery checks were accepted")
}
})
}
}
+97
View File
@@ -0,0 +1,97 @@
package stage
import (
"context"
"fmt"
"strings"
)
// VerificationTarget declares the controlled fixture that can substantiate a
// target-specific behavior claim. It is intentionally data-only: the runner
// does not execute fixture input supplied by a stage result.
type VerificationTarget struct {
Backend string
Revision string
Capability string
Fixture string
}
// VerificationEvidence is returned by a server-controlled fixture workflow,
// never parsed from a model's result.
type VerificationEvidence struct {
Backend string
Revision string
Capability string
Fixture string
}
// VerificationRunner is the narrow server seam for a controlled fixture or
// workflow. The runner selects it during server setup; model output cannot
// select a backend, revision, fixture, or executable.
type VerificationRunner func(context.Context, VerificationTarget) (VerificationEvidence, error)
// VerificationTargetFor parses an optional, bounded verification block. A
// declared target requires all provenance fields so an old or other-backend
// fixture cannot be presented as evidence for the current target.
func VerificationTargetFor(def Definition) (*VerificationTarget, error) {
text, declared := def.Blocks["verification"]
if !declared {
return nil, nil
}
if len(text) > 4096 {
return nil, fmt.Errorf("verification block exceeds 4096 bytes")
}
values := map[string]string{}
for _, line := range strings.Split(text, "\n") {
line = strings.TrimSpace(line)
if line == "" {
continue
}
key, value, ok := strings.Cut(line, "=")
key, value = strings.TrimSpace(key), strings.TrimSpace(value)
if !ok || value == "" || len(value) > 256 {
return nil, fmt.Errorf("invalid verification entry %q", line)
}
switch key {
case "backend", "revision", "capability", "fixture":
default:
return nil, fmt.Errorf("unknown verification entry %q", key)
}
if _, exists := values[key]; exists {
return nil, fmt.Errorf("duplicate verification entry %q", key)
}
values[key] = value
}
target := &VerificationTarget{Backend: values["backend"], Revision: values["revision"], Capability: values["capability"], Fixture: values["fixture"]}
if target.Backend == "" || target.Revision == "" || target.Capability == "" || target.Fixture == "" {
return nil, fmt.Errorf("verification requires backend, revision, capability, and fixture")
}
return target, nil
}
// VerifyTargetEvidence compares server-collected controlled-fixture provenance
// with the workflow declaration. It deliberately accepts no model result: a
// stage cannot substantiate its own target-specific behavior claim.
func VerifyTargetEvidence(target VerificationTarget, evidence VerificationEvidence) error {
for key, want := range map[string]string{
"backend": target.Backend, "revision": target.Revision, "capability": target.Capability, "fixture": target.Fixture,
} {
got := map[string]string{
"backend": evidence.Backend, "revision": evidence.Revision, "capability": evidence.Capability, "fixture": evidence.Fixture,
}[key]
if got != want {
return fmt.Errorf("verification evidence %s does not match declared target", key)
}
}
return nil
}
// VerifyTargetResult is retained for callers that only need declaration
// validation. Result-provided provenance is intentionally not evidence.
func VerifyTargetResult(def Definition, _ Values) error {
target, err := VerificationTargetFor(def)
if err != nil || target == nil {
return err
}
return fmt.Errorf("verification target %q requires a server-controlled fixture runner", target.Fixture)
}
+27
View File
@@ -0,0 +1,27 @@
package stage
import "testing"
func TestTargetVerificationRejectsWrongBackendEvidence(t *testing.T) {
def := modelDefinition()
def.Blocks["verification"] = "backend=branch-api\nrevision=abc123\ncapability=controlled-state\nfixture=reconnect-foreground"
if err := (ModelStage{}).Validate(def); err != nil {
t.Fatal(err)
}
target, err := VerificationTargetFor(def)
if err != nil {
t.Fatal(err)
}
good := VerificationEvidence{Backend: "branch-api", Revision: "abc123", Capability: "controlled-state", Fixture: "reconnect-foreground"}
if err := VerifyTargetEvidence(*target, good); err != nil {
t.Fatalf("matching fixture evidence rejected: %v", err)
}
wrongBackend := VerificationEvidence{Backend: "old-api", Revision: "abc123", Capability: "controlled-state", Fixture: "reconnect-foreground"}
if err := VerifyTargetEvidence(*target, wrongBackend); err == nil {
t.Fatal("wrong-backend fixture evidence accepted")
}
def.Blocks["verification"] = "backend=branch-api\nrevision=abc123"
if err := (ModelStage{}).Validate(def); err == nil {
t.Fatal("incomplete target verification declaration accepted")
}
}
+43
View File
@@ -0,0 +1,43 @@
package tui
import (
"fmt"
"strings"
"testing"
"git.a-wan.net/alan/pipelines/internal/protocol"
"github.com/charmbracelet/lipgloss"
"github.com/charmbracelet/x/ansi"
"github.com/muesli/termenv"
)
func TestResourceWaitFrames(t *testing.T) {
previous := lipgloss.ColorProfile()
defer lipgloss.SetColorProfile(previous)
for _, width := range []int{60, 120} {
for _, plain := range []bool{false, true} {
t.Run(fmt.Sprintf("width-%d-plain-%t", width, plain), func(t *testing.T) {
if plain {
lipgloss.SetColorProfile(termenv.Ascii)
} else {
lipgloss.SetColorProfile(termenv.TrueColor)
}
m := decisionModel()
m.width = width
m.runDetail.Status = protocol.RunStatusQueued
m.runDetail.Steps = nil
m.runDetail.ResourceWait = &protocol.ResourceWait{Resource: "phone-X", Stage: "review", Sequence: 1, QueuedSince: "2026-10-18T10:00:00Z"}
frame := ansi.Strip(m.runsDetailsView(24))
if !strings.Contains(frame, "Waiting for resource phone-X") || strings.Contains(frame, "c decide") || strings.Contains(frame, "Waiting for a person") {
t.Fatal("wrong resource wait frame:\n", frame)
}
for _, line := range strings.Split(frame, "\n") {
if ansi.StringWidth(line) > width {
t.Fatalf("overflow: %q", line)
}
}
t.Log("Rendered resource wait (no phone):\n" + frame)
})
}
}
}
+4 -1
View File
@@ -11,6 +11,9 @@ import (
)
func resultLabel(r protocol.RunSummary) string {
if r.ResourceWait != nil {
return r.StatusLabel() + " · stage " + r.ResourceWait.Stage
}
preview := ansi.Strip(r.ResultPreview)
if r.Status == "succeeded" {
if preview != "" {
@@ -55,7 +58,7 @@ func (m Model) pipelineResultLines() []string {
if r == nil {
return []string{"Results · This version has not run yet. No accomplishments recorded."}
}
lines := []string{fmt.Sprintf("Results · run #%d · %s · %s · %d completed visits · %s", r.ID, r.Label(), ansi.Strip(m.progress(r.Status)), r.CompletedSteps, runDuration(r.CreatedAt, r.FinishedAt)), resultLabel(*r)}
lines := []string{fmt.Sprintf("Results · run #%d · %s · %s · %d completed visits · %s", r.ID, r.Label(), ansi.Strip(m.runSummaryStatus(*r)), r.CompletedSteps, runDuration(r.CreatedAt, r.FinishedAt)), resultLabel(*r)}
if r.Status != "succeeded" && r.ResultPreview != "" && r.Diagnostic != "" {
lines = append(lines, "Partial output: "+ansi.Strip(r.ResultPreview))
}
+10 -2
View File
@@ -275,6 +275,13 @@ func runStatus(status string) string {
return lipgloss.NewStyle().Foreground(color).Render(status)
}
func (m Model) runSummaryStatus(run protocol.RunSummary) string {
if run.ResourceWait != nil {
return run.StatusLabel()
}
return m.progress(run.Status)
}
func runDuration(start, end string) string {
first, err := time.Parse(time.RFC3339, start)
if err != nil {
@@ -687,7 +694,7 @@ func (m Model) runsDetailsView(height int) string {
start, end := selectionWindow(len(runs), m.runIndex, max(1, (height-4-len(rows))/2))
for i := start; i < end; i++ {
r := runs[i]
text := fmt.Sprintf("#%d %s · v%d · %s · %s", r.ID, r.Label(), r.Version, m.progress(r.Status), dateLabel(r.FinishedAt))
text := fmt.Sprintf("#%d %s · v%d · %s · %s", r.ID, r.Label(), r.Version, m.runSummaryStatus(r), dateLabel(r.FinishedAt))
rows = append(rows, menuOption(text, i == m.runIndex))
rows = append(rows, " "+ansi.Truncate(resultLabel(r), max(1, m.width-6), "…"))
}
@@ -710,7 +717,8 @@ func (m Model) runsDetailsView(height int) string {
return panel(m.width, height, []string{bar(title, fmt.Sprint(len(runs)), max(1, m.width-4), true)}, rows, footer, true)
}
r := m.runDetail
header := bar(fmt.Sprintf("#%d %s · v%d", r.ID, r.Label(), r.Version), m.progress(r.Status)+" · "+runDuration(r.CreatedAt, r.FinishedAt), m.width, false)
status := m.runSummaryStatus(r.RunSummary)
header := bar(fmt.Sprintf("#%d %s · v%d", r.ID, r.Label(), r.Version), status+" · "+runDuration(r.CreatedAt, r.FinishedAt), m.width, false)
if height <= 6 {
return fitBlock([]string{header, runHighlight(m.runSelectionLabel(), m.width), m.runActions() + " · Tab pane · Esc back"}, m.width, height)
}