Merge main into pill-run-decision for Android recovery
This commit is contained in:
+26
-4
@@ -20,16 +20,37 @@ time while iterating — the full suite is for the end.
|
||||
|
||||
## The dev device
|
||||
|
||||
Only the **UI reviewer**, while its stage holds a declared runner-exclusive resource
|
||||
for the selected phone, may use the device. Implementers and code reviewers do code,
|
||||
builds, unit tests and source review only: **no adb, installs, captures, device-setting
|
||||
changes or main-build restoration**. Never acquire a phone using sleep/check loops.
|
||||
Do not use the phone until compatible resource scheduling is deployed and every device
|
||||
user has opted in or has been reconciled by the operator. Existing pinned runs are not
|
||||
retroactively protected. The resource for the device below is `phone-47241FDAS00DLK`.
|
||||
|
||||
The only device is the operator's **Pixel 9 Pro XL, serial `47241FDAS00DLK`**. The operator's
|
||||
Pixel 10 Pro Fold (`58281FDCG000JF`) is theirs: use it only where an item's brief explicitly allows
|
||||
it, and never make it a default. **Never start or use an emulator.**
|
||||
|
||||
Before any capture: `svc power stayon usb`, `settings put global stay_on_while_plugged_in 7`,
|
||||
The UI reviewer records original app connection/workspace, display size and power/settings
|
||||
before changing them. Before any capture: `svc power stayon usb`, `settings put global stay_on_while_plugged_in 7`,
|
||||
`input keyevent KEYCODE_WAKEUP`, `wm dismiss-keyguard`. Capture with
|
||||
`adb -s <serial> shell screencap -p /sdcard/shot.png` then `adb pull` — never `exec-out > file`.
|
||||
Wide layout: `wm size 2000x2244`; the fold's unfolded shape is `wm size 2076x2152`; `wm size reset`
|
||||
afterwards either way.
|
||||
|
||||
Create controlled fixtures for changed screens/states (including human-decision sheets)
|
||||
on a scratch server/database and throwaway session; never depend on incidental production
|
||||
state, change a real operator session, or restart a deployed service. Use the configured
|
||||
authoritative endpoint, never a scratch URL as a project default. Keep all shell/ADB work
|
||||
foreground and awaited: no detached/background processes may outlive the lease.
|
||||
UI review includes restoring the captured original settings and main's build before its
|
||||
successful return. Confirm every tool has exited. If capture, restoration or cleanup fails,
|
||||
report the failure explicitly; do not claim verification or readiness. Uncertain termination
|
||||
must fail closed and keep ownership quarantined, not delete a lock or expire it by age.
|
||||
Return `output.resource_cleanup_confirmed=false` if restoration/readiness is uncertain;
|
||||
that generic release veto retains the runner lease even if the harness reports idle.
|
||||
|
||||
## Geometry is tokenised, and a test enforces it
|
||||
|
||||
Every dimension comes from `ui/theme/Dimens.kt` (plus the layout itself) — no new hardcoded `dp`, no
|
||||
@@ -53,13 +74,14 @@ genuinely needs a new token goes there, named for its purpose.
|
||||
## Publishing a build
|
||||
|
||||
Copy the APK into the running run's artifacts directory as `pipelines-debug-wip-<short-HEAD>.apk` and
|
||||
report the LAN URL. After any install, leave the operator's phone on **main's** build:
|
||||
report the LAN URL. The UI reviewer alone, under the same lease, restores **main's** build after an install:
|
||||
`/private/var/folders/jp/q0qvjyg54fl_c5vhhk1ylz2m0000gn/T/opencode/deploy-app.sh main`.
|
||||
|
||||
## Traps
|
||||
|
||||
- A build made in a worktree that is behind main is missing features main has — sync before building,
|
||||
and say which revision you built.
|
||||
- Changing the app's server URL for a test (a stub) must be undone: `ws://10.0.0.75:8080/connect`,
|
||||
workspace `/Users/alan/repos/pipelines`, then force-stop and relaunch.
|
||||
- Restore the app's captured original connection and workspace after a fixture test,
|
||||
using the configured authoritative endpoint; then force-stop and relaunch under the lease.
|
||||
Never substitute a hardcoded or disposable scratch endpoint for the original configuration.
|
||||
- `mStayOn`/screen timeout are already set; do not "fix" the device by locking or sleeping it.
|
||||
|
||||
@@ -58,6 +58,7 @@ dependencies {
|
||||
implementation(libs.androidx.compose.material.icons.extended)
|
||||
implementation(libs.androidx.navigation.compose)
|
||||
implementation(libs.androidx.lifecycle.runtime.compose)
|
||||
implementation(libs.androidx.lifecycle.process)
|
||||
implementation(libs.androidx.lifecycle.viewmodel.compose)
|
||||
implementation(libs.androidx.lifecycle.viewmodel.ktx)
|
||||
implementation(libs.kotlinx.coroutines.android)
|
||||
|
||||
@@ -1,12 +1,15 @@
|
||||
package net.awan.pipelines
|
||||
|
||||
import android.app.Application
|
||||
import androidx.lifecycle.ProcessLifecycleOwner
|
||||
import kotlinx.coroutines.CoroutineScope
|
||||
import kotlinx.coroutines.Dispatchers
|
||||
import kotlinx.coroutines.SupervisorJob
|
||||
import net.awan.pipelines.data.ForegroundReconnector
|
||||
import net.awan.pipelines.data.PipelinesRepository
|
||||
import net.awan.pipelines.data.SettingsRepository
|
||||
import net.awan.pipelines.data.transport.PipelinesClient
|
||||
import net.awan.pipelines.data.transport.ServerUrl
|
||||
import net.awan.pipelines.notify.RunNotifier
|
||||
|
||||
/**
|
||||
@@ -30,8 +33,39 @@ class PipelinesApplication : Application() {
|
||||
*/
|
||||
val runNotifier: RunNotifier by lazy { RunNotifier(this) }
|
||||
|
||||
/**
|
||||
* The one owner of return-to-foreground recovery: when the process comes back
|
||||
* to the front it restores a connection the background took away, with no
|
||||
* interaction. It is created once with the Application and observes the
|
||||
* process lifecycle, so every Activity and ViewModel recreation leaves it
|
||||
* untouched. The liveness check is the existing `ping`; the reconnect is the
|
||||
* same shared-client connect the app's own Connect/Reconnect uses, so the
|
||||
* policy lives only in `foregroundAction` and the client's backoff stays the
|
||||
* only reconnect mechanism.
|
||||
*/
|
||||
private val foregroundReconnector: ForegroundReconnector by lazy {
|
||||
ForegroundReconnector(
|
||||
state = { repository.connectionState.value },
|
||||
confirmAlive = {
|
||||
// The existing ping is the positive check: on a dead socket it
|
||||
// throws (or never answers) instead of returning, so reaching the
|
||||
// next line is the proof of life.
|
||||
repository.ping()
|
||||
true
|
||||
},
|
||||
reconnect = {
|
||||
// A saved URL that no longer validates is left for Settings to
|
||||
// report; there is nothing to reconnect to.
|
||||
val validated = ServerUrl.validate(settings.serverUrl.value)
|
||||
if (validated.valid) repository.recover(validated)
|
||||
},
|
||||
scope = applicationScope,
|
||||
)
|
||||
}
|
||||
|
||||
override fun onCreate() {
|
||||
super.onCreate()
|
||||
runNotifier.ensureChannel()
|
||||
ProcessLifecycleOwner.get().lifecycle.addObserver(foregroundReconnector)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,81 @@
|
||||
package net.awan.pipelines.data
|
||||
|
||||
import androidx.lifecycle.DefaultLifecycleObserver
|
||||
import androidx.lifecycle.LifecycleOwner
|
||||
import kotlinx.coroutines.CancellationException
|
||||
import kotlinx.coroutines.CoroutineScope
|
||||
import kotlinx.coroutines.launch
|
||||
import kotlinx.coroutines.withTimeoutOrNull
|
||||
import net.awan.pipelines.data.transport.ConnectionState
|
||||
import net.awan.pipelines.data.transport.ForegroundAction
|
||||
import net.awan.pipelines.data.transport.foregroundAction
|
||||
|
||||
/**
|
||||
* The one lifecycle-aware owner of the app's return-to-foreground recovery.
|
||||
*
|
||||
* While another app is in front Android freezes — and often kills — this process,
|
||||
* so the socket it holds dies or goes quiet and the client's own bounded backoff
|
||||
* either never starts or starts too late for the operator, who comes back and
|
||||
* taps Connect. The client's backoff (see `PipelinesClient.scheduleReconnect`) is
|
||||
* still the only reconnect mechanism; this owner only decides *when* to drive it,
|
||||
* and does so the moment the app becomes visible again.
|
||||
*
|
||||
* It observes the whole process lifecycle (a `ProcessLifecycleOwner`), never the
|
||||
* Activity or the ViewModel, both of which Android recreates: the owner is
|
||||
* created once with the Application and survives every recreation. The decision
|
||||
* itself is the one pure [foregroundAction] policy; this class supplies only the
|
||||
* current state, a positive liveness check and the reconnect effect, so the
|
||||
* policy is unit-testable with no device, socket or framework.
|
||||
*/
|
||||
class ForegroundReconnector(
|
||||
private val state: () -> ConnectionState,
|
||||
private val confirmAlive: suspend () -> Boolean,
|
||||
private val reconnect: () -> Unit,
|
||||
private val scope: CoroutineScope,
|
||||
private val confirmTimeoutMillis: Long = CONFIRM_TIMEOUT_MILLIS,
|
||||
) : DefaultLifecycleObserver {
|
||||
|
||||
/**
|
||||
* The app became visible again. A state the policy calls reconnecting is
|
||||
* reconnected at once — the pending backoff delay is cancelled by the
|
||||
* reconnect effect, so the operator never waits it out. A socket the state
|
||||
* still calls Connected is first confirmed to answer, so a silent
|
||||
* "Connected" is never trusted; the state is re-read afterwards, because the
|
||||
* socket's own failure callback may have run while the check was in flight.
|
||||
*/
|
||||
override fun onStart(owner: LifecycleOwner) {
|
||||
val before = state()
|
||||
if (before !is ConnectionState.Connected) {
|
||||
if (foregroundAction(before, confirmedAlive = false) == ForegroundAction.Reconnect) reconnect()
|
||||
return
|
||||
}
|
||||
scope.launch {
|
||||
val alive = confirm()
|
||||
if (foregroundAction(state(), alive) == ForegroundAction.Reconnect) reconnect()
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Confirms a supposedly-connected socket still answers, bounded so a dead one
|
||||
* cannot hold recovery: the check runs off the main thread on the app scope
|
||||
* and a probe that fails, throws or outlives its budget counts as not alive.
|
||||
*/
|
||||
private suspend fun confirm(): Boolean =
|
||||
withTimeoutOrNull(confirmTimeoutMillis) {
|
||||
try {
|
||||
confirmAlive()
|
||||
} catch (t: Throwable) {
|
||||
if (t is CancellationException) throw t
|
||||
false
|
||||
}
|
||||
} ?: false
|
||||
|
||||
companion object {
|
||||
/**
|
||||
* Bounds the liveness check. A healthy round trip on the trusted LAN/VPN
|
||||
* is far shorter, so a socket that cannot answer is noticed promptly
|
||||
* instead of holding the "Connected" the operator is looking at.
|
||||
*/
|
||||
const val CONFIRM_TIMEOUT_MILLIS = 3_000L
|
||||
}
|
||||
}
|
||||
@@ -66,6 +66,9 @@ class PipelinesRepository(
|
||||
/** Opens the single shared connection to a validated server URL. */
|
||||
fun connect(validated: ServerUrl.Validated) = client.connect(validated)
|
||||
|
||||
/** Recovers a lost connection without reversing an explicit Disconnect. */
|
||||
fun recover(validated: ServerUrl.Validated) = client.recover(validated)
|
||||
|
||||
/** Explicit disconnect; the client then stays Idle. */
|
||||
fun disconnect() = client.disconnect()
|
||||
|
||||
|
||||
@@ -315,6 +315,7 @@ data class SessionExport(
|
||||
|
||||
@Serializable
|
||||
data class RunSummary(
|
||||
@SerialName("resource_wait") val resourceWait: ResourceWait? = null,
|
||||
val id: Long = 0,
|
||||
@SerialName("pipeline_id") val pipelineId: Long = 0,
|
||||
val name: String = "",
|
||||
@@ -370,6 +371,7 @@ data class RunStep(
|
||||
// declared directly with the exact internal/protocol/runs.go wire names.
|
||||
@Serializable
|
||||
data class RunDetail(
|
||||
@SerialName("resource_wait") val resourceWait: ResourceWait? = null,
|
||||
val id: Long = 0,
|
||||
@SerialName("pipeline_id") val pipelineId: Long = 0,
|
||||
val name: String = "",
|
||||
@@ -405,6 +407,8 @@ data class RunDetail(
|
||||
*/
|
||||
@Serializable
|
||||
data class RunEventFrame(
|
||||
@SerialName("resource_wait") val resourceWait: ResourceWait? = null,
|
||||
@SerialName("resource_wait_set") val resourceWaitSet: Boolean = false,
|
||||
@SerialName("run_id") val runId: Long = 0,
|
||||
val workspace: String = "",
|
||||
// The run's pipeline identity and stored title, repeated on every frame so a
|
||||
@@ -428,6 +432,16 @@ data class RunEventFrame(
|
||||
val step: RunStepDelta? = null,
|
||||
)
|
||||
|
||||
@Serializable
|
||||
data class ResourceWait(
|
||||
val resource: String = "",
|
||||
val stage: String = "",
|
||||
val sequence: Int = 0,
|
||||
@SerialName("queued_since") val queuedSince: String = "",
|
||||
) {
|
||||
fun statusLabel(): String = "Waiting for resource $resource"
|
||||
}
|
||||
|
||||
/**
|
||||
* The compact change of one step within a [RunEventFrame]
|
||||
* (internal/protocol/runevents.go RunStepDelta): a delta, not a RunStep, so it
|
||||
|
||||
@@ -0,0 +1,40 @@
|
||||
package net.awan.pipelines.data.transport
|
||||
|
||||
/** What the app does with the connection the moment it becomes visible again. */
|
||||
enum class ForegroundAction {
|
||||
/** Leave the connection exactly as it is. */
|
||||
LeaveAlone,
|
||||
|
||||
/** Re-establish the connection now, cancelling any pending reconnect delay. */
|
||||
Reconnect,
|
||||
}
|
||||
|
||||
/**
|
||||
* The app's return-to-foreground recovery policy, in one pure place.
|
||||
*
|
||||
* Android freezes and often kills a backgrounded app's process, so the socket
|
||||
* the app holds dies or goes quiet while another app is in front and nothing in
|
||||
* the process notices in time. Coming back must restore that connection without
|
||||
* the operator tapping Connect — but an explicit Disconnect is the operator's
|
||||
* own decision and stays authoritative. A deliberate Disconnect is the only way
|
||||
* the client reaches [ConnectionState.Idle], so Idle is left alone; every other
|
||||
* state reconnects.
|
||||
*
|
||||
* A still [ConnectionState.Connected] socket is not trusted on return: a frozen
|
||||
* process can leave it claiming Connected while it answers nothing, which is
|
||||
* exactly the stale state this replaces. The caller therefore checks it with a
|
||||
* positive probe (the existing ping) and passes the verdict as [confirmedAlive];
|
||||
* only a connection just confirmed to answer is left alone.
|
||||
*
|
||||
* @param state the connection state at the moment the app became visible again.
|
||||
* @param confirmedAlive whether a [ConnectionState.Connected] socket has just
|
||||
* been confirmed to answer. It is consulted only for Connected; every other
|
||||
* state carries its own answer.
|
||||
*/
|
||||
fun foregroundAction(state: ConnectionState, confirmedAlive: Boolean): ForegroundAction = when (state) {
|
||||
ConnectionState.Idle -> ForegroundAction.LeaveAlone
|
||||
is ConnectionState.Connected ->
|
||||
if (confirmedAlive) ForegroundAction.LeaveAlone else ForegroundAction.Reconnect
|
||||
is ConnectionState.Connecting -> ForegroundAction.Reconnect
|
||||
is ConnectionState.Disconnected -> ForegroundAction.Reconnect
|
||||
}
|
||||
@@ -78,11 +78,33 @@ class PipelinesClient(
|
||||
@Volatile
|
||||
private var manualClose = false
|
||||
|
||||
/** Serializes socket replacement with callbacks arriving on OkHttp threads. */
|
||||
private val connectionLock = Any()
|
||||
|
||||
private var reconnectJob: Job? = null
|
||||
private var attempt = 0
|
||||
|
||||
/** Opens (or re-opens) the connection to a validated server URL. */
|
||||
fun connect(validated: ServerUrl.Validated) {
|
||||
synchronized(connectionLock) {
|
||||
connectLocked(validated)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Re-opens a connection the transport lost without reversing an explicit
|
||||
* [disconnect]. The check and connection replacement share one lock, so a
|
||||
* Disconnect racing a foreground liveness probe wins even after that probe
|
||||
* has decided recovery is needed.
|
||||
*/
|
||||
fun recover(validated: ServerUrl.Validated) {
|
||||
synchronized(connectionLock) {
|
||||
if (manualClose) return
|
||||
connectLocked(validated)
|
||||
}
|
||||
}
|
||||
|
||||
private fun connectLocked(validated: ServerUrl.Validated) {
|
||||
val httpUrl = validated.httpUrl ?: throw IllegalArgumentException("server URL is not valid")
|
||||
endpoint = httpUrl
|
||||
manualClose = false
|
||||
@@ -94,14 +116,16 @@ class PipelinesClient(
|
||||
|
||||
/** Explicit user disconnect: stops reconnection and closes the socket. */
|
||||
fun disconnect() {
|
||||
manualClose = true
|
||||
reconnectJob?.cancel()
|
||||
reconnectJob = null
|
||||
val socket = webSocket
|
||||
webSocket = null
|
||||
socket?.close(NORMAL_CLOSURE, "client closing")
|
||||
failPending("Disconnected")
|
||||
_state.value = ConnectionState.Idle
|
||||
synchronized(connectionLock) {
|
||||
manualClose = true
|
||||
reconnectJob?.cancel()
|
||||
reconnectJob = null
|
||||
val socket = webSocket
|
||||
webSocket = null
|
||||
socket?.close(NORMAL_CLOSURE, "client closing")
|
||||
failPending("Disconnected")
|
||||
_state.value = ConnectionState.Idle
|
||||
}
|
||||
}
|
||||
|
||||
fun close() {
|
||||
@@ -115,14 +139,20 @@ class PipelinesClient(
|
||||
*/
|
||||
suspend fun call(request: Message): Message = withContext(ioDispatcher) {
|
||||
sendLock.withLock {
|
||||
val socket = webSocket ?: throw NotConnectedException()
|
||||
if (_state.value !is ConnectionState.Connected) throw NotConnectedException()
|
||||
val id = "android-" + sequence.incrementAndGet()
|
||||
val deferred = CompletableDeferred<Message>()
|
||||
pending[id] = deferred
|
||||
try {
|
||||
val (id, deferred) = synchronized(connectionLock) {
|
||||
val socket = webSocket ?: throw NotConnectedException()
|
||||
if (_state.value !is ConnectionState.Connected) throw NotConnectedException()
|
||||
val id = "android-" + sequence.incrementAndGet()
|
||||
val deferred = CompletableDeferred<Message>()
|
||||
pending[id] = deferred
|
||||
val payload = json.encodeToString(Message.serializer(), request.copy(id = id))
|
||||
if (!socket.send(payload)) throw IOException("websocket send failed")
|
||||
if (!socket.send(payload)) {
|
||||
pending.remove(id)
|
||||
throw IOException("websocket send failed")
|
||||
}
|
||||
id to deferred
|
||||
}
|
||||
try {
|
||||
val response = withTimeout(requestTimeoutMillis) { deferred.await() }
|
||||
if (response.type == Protocol.ERROR) {
|
||||
throw ProtocolException(response.error?.ifBlank { "server error" } ?: "server error")
|
||||
@@ -144,12 +174,14 @@ class PipelinesClient(
|
||||
*/
|
||||
suspend fun sendOnly(request: Message): String = withContext(ioDispatcher) {
|
||||
sendLock.withLock {
|
||||
val socket = webSocket ?: throw NotConnectedException()
|
||||
if (_state.value !is ConnectionState.Connected) throw NotConnectedException()
|
||||
val id = "android-" + sequence.incrementAndGet()
|
||||
val payload = json.encodeToString(Message.serializer(), request.copy(id = id))
|
||||
if (!socket.send(payload)) throw IOException("websocket send failed")
|
||||
id
|
||||
synchronized(connectionLock) {
|
||||
val socket = webSocket ?: throw NotConnectedException()
|
||||
if (_state.value !is ConnectionState.Connected) throw NotConnectedException()
|
||||
val id = "android-" + sequence.incrementAndGet()
|
||||
val payload = json.encodeToString(Message.serializer(), request.copy(id = id))
|
||||
if (!socket.send(payload)) throw IOException("websocket send failed")
|
||||
id
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -166,14 +198,34 @@ class PipelinesClient(
|
||||
}
|
||||
}
|
||||
|
||||
private fun open() {
|
||||
private fun open() = synchronized(connectionLock) { openLocked() }
|
||||
|
||||
private fun openLocked() {
|
||||
val current = endpoint ?: return
|
||||
attempt += 1
|
||||
_state.value = ConnectionState.Connecting(attempt)
|
||||
val request = Request.Builder().url(current).build()
|
||||
// Replace the current socket, then close the one it superseded. Its
|
||||
// callbacks are ignored from then on (see [isCurrent]), so a failure the
|
||||
// old socket reports late — the common case when the process was frozen
|
||||
// while it died — can neither clear the new socket nor schedule a
|
||||
// reconnect that would undo it.
|
||||
val superseded = webSocket
|
||||
webSocket = httpClient.newWebSocket(request, listener)
|
||||
// A request sent over the superseded socket cannot receive a reply from
|
||||
// the replacement. Fail it now rather than leaving a screen waiting for
|
||||
// the ordinary request timeout after recovery.
|
||||
if (superseded != null) failPending("Connection replaced")
|
||||
superseded?.close(NORMAL_CLOSURE, "superseded")
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether [socket] is still the connection this client acts on. A socket this
|
||||
* client has already replaced (a deliberate reconnect, or the foreground
|
||||
* recovery re-establishing a frozen one) must never drive the state again.
|
||||
*/
|
||||
private fun isCurrent(socket: WebSocket): Boolean = webSocket === socket
|
||||
|
||||
private fun failPending(reason: String) {
|
||||
val snapshot = pending.keys.toList()
|
||||
for (key in snapshot) {
|
||||
@@ -181,7 +233,8 @@ class PipelinesClient(
|
||||
}
|
||||
}
|
||||
|
||||
private fun handleClosed(reason: String) {
|
||||
private fun handleClosed(socket: WebSocket, reason: String) {
|
||||
if (!isCurrent(socket)) return
|
||||
webSocket = null
|
||||
failPending(reason)
|
||||
if (manualClose) {
|
||||
@@ -194,52 +247,65 @@ class PipelinesClient(
|
||||
private fun scheduleReconnect(reason: String) {
|
||||
if (reconnectJob?.isActive == true) return
|
||||
_state.value = ConnectionState.Disconnected(reason, retrying = true)
|
||||
val nextAttempt = attempt
|
||||
val backoff = (BASE_BACKOFF_MILLIS shl (nextAttempt - 1).coerceAtMost(6))
|
||||
.coerceAtMost(MAX_BACKOFF_MILLIS)
|
||||
val backoff = reconnectBackoffMillis(attempt)
|
||||
val jitter = Random.nextLong(0, 400)
|
||||
reconnectJob = scope.launch {
|
||||
delay(backoff + jitter)
|
||||
if (!manualClose && endpoint != null) open()
|
||||
synchronized(connectionLock) {
|
||||
if (!manualClose && endpoint != null) open()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private val listener = object : WebSocketListener() {
|
||||
override fun onOpen(webSocket: WebSocket, response: Response) {
|
||||
attempt = 0
|
||||
_state.value = ConnectionState.Connected(build = null, serverTime = null)
|
||||
synchronized(connectionLock) {
|
||||
if (!isCurrent(webSocket)) return
|
||||
attempt = 0
|
||||
_state.value = ConnectionState.Connected(build = null, serverTime = null)
|
||||
}
|
||||
}
|
||||
|
||||
override fun onMessage(webSocket: WebSocket, text: String) {
|
||||
val message = try {
|
||||
json.decodeFromString(Message.serializer(), text)
|
||||
} catch (_: Exception) {
|
||||
// Malformed frame: never misattribute it to a request.
|
||||
return
|
||||
}
|
||||
val id = message.id
|
||||
val deferred = if (id != null) pending.remove(id) else null
|
||||
if (deferred != null) {
|
||||
deferred.complete(message)
|
||||
} else {
|
||||
// Unsolicited envelopes (the connection's initial pipeline_list,
|
||||
// server-pushed chat frames, stream errors) carry no matching id:
|
||||
// hand them to the event stream instead of dropping them.
|
||||
_events.tryEmit(message)
|
||||
synchronized(connectionLock) {
|
||||
if (!isCurrent(webSocket)) return
|
||||
val message = try {
|
||||
json.decodeFromString(Message.serializer(), text)
|
||||
} catch (_: Exception) {
|
||||
// Malformed frame: never misattribute it to a request.
|
||||
return
|
||||
}
|
||||
val id = message.id
|
||||
val deferred = if (id != null) pending.remove(id) else null
|
||||
if (deferred != null) {
|
||||
deferred.complete(message)
|
||||
} else {
|
||||
// Unsolicited envelopes (the connection's initial pipeline_list,
|
||||
// server-pushed chat frames, stream errors) carry no matching id:
|
||||
// hand them to the event stream instead of dropping them.
|
||||
_events.tryEmit(message)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
override fun onClosed(webSocket: WebSocket, code: Int, reason: String) {
|
||||
handleClosed("Connection closed (${code})")
|
||||
synchronized(connectionLock) {
|
||||
handleClosed(webSocket, "Connection closed (${code})")
|
||||
}
|
||||
}
|
||||
|
||||
override fun onClosing(webSocket: WebSocket, code: Int, reason: String) {
|
||||
webSocket.close(NORMAL_CLOSURE, null)
|
||||
synchronized(connectionLock) {
|
||||
if (!isCurrent(webSocket)) return
|
||||
webSocket.close(NORMAL_CLOSURE, null)
|
||||
}
|
||||
}
|
||||
|
||||
override fun onFailure(webSocket: WebSocket, t: Throwable, response: Response?) {
|
||||
val reason = t.message?.ifBlank { t.javaClass.simpleName } ?: "connection failed"
|
||||
handleClosed(reason)
|
||||
synchronized(connectionLock) {
|
||||
val reason = t.message?.ifBlank { t.javaClass.simpleName } ?: "connection failed"
|
||||
handleClosed(webSocket, reason)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -247,6 +313,16 @@ class PipelinesClient(
|
||||
private const val NORMAL_CLOSURE = 1000
|
||||
private const val BASE_BACKOFF_MILLIS = 1_000L
|
||||
private const val MAX_BACKOFF_MILLIS = 30_000L
|
||||
|
||||
/**
|
||||
* The successful open callback resets [attempt] to zero. A later socket
|
||||
* failure must therefore begin at the base delay, never shift by -1
|
||||
* (which Kotlin masks into a very large shift count).
|
||||
*/
|
||||
internal fun reconnectBackoffMillis(attempt: Int): Long {
|
||||
val exponent = (attempt - 1).coerceIn(0, 6)
|
||||
return (BASE_BACKOFF_MILLIS shl exponent).coerceAtMost(MAX_BACKOFF_MILLIS)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -101,6 +101,7 @@ internal fun foldRunEvent(runs: List<RunSummary>, frame: RunEventFrame): RunList
|
||||
internal fun RunEventFrame.newRunSummary(): RunSummary? {
|
||||
if (pipelineId <= 0 || name.isBlank()) return null
|
||||
return RunSummary(
|
||||
resourceWait = resourceWait,
|
||||
id = runId,
|
||||
pipelineId = pipelineId,
|
||||
name = name,
|
||||
@@ -124,10 +125,11 @@ internal fun RunEventFrame.newRunSummary(): RunSummary? {
|
||||
*/
|
||||
internal fun foldRunEvent(detail: RunDetail, frame: RunEventFrame): RunDetail {
|
||||
if (frame.runId != detail.id) return detail
|
||||
if (isStaleRunFrame(detail.status, frame.status)) return detail
|
||||
if (isStaleRunFrame(detail.status, frame.status, frame.resourceWaitSet)) return detail
|
||||
val merged = detail.fields().applying(frame)
|
||||
val steps = frame.step?.let { upsertStep(detail.steps, it) } ?: detail.steps
|
||||
val updated = detail.copy(
|
||||
resourceWait = merged.resourceWait,
|
||||
status = merged.status,
|
||||
outcome = merged.outcome,
|
||||
currentStage = merged.currentStage,
|
||||
@@ -169,8 +171,9 @@ private fun advanceRunStatus(current: String, incoming: String): String =
|
||||
* run already settled. Dropping every field (not just the status) is what keeps
|
||||
* a late `running` delta from also rewriting the stage or step of a settled run.
|
||||
*/
|
||||
private fun isStaleRunFrame(current: String, incoming: String): Boolean =
|
||||
isStaleTransition(current, incoming, ::runLifecycleRank)
|
||||
private fun isStaleRunFrame(current: String, incoming: String, resourceWaitSet: Boolean = false): Boolean =
|
||||
if (resourceWaitSet && incoming == "queued" && !isTerminalRunStatus(current)) false
|
||||
else isStaleTransition(current, incoming, ::runLifecycleRank)
|
||||
|
||||
/** The step counterpart of [isStaleRunFrame]: a settled visit drops a stale delta whole. */
|
||||
private fun isStaleStepDelta(current: String, incoming: String): Boolean =
|
||||
@@ -213,6 +216,7 @@ private fun advanceStatus(current: String, incoming: String, rank: (String) -> I
|
||||
|
||||
/** The run-state fields a frame advances, shared by the summary row and the detail. */
|
||||
private data class RunStateFields(
|
||||
val resourceWait: net.awan.pipelines.data.protocol.ResourceWait?,
|
||||
val status: String,
|
||||
val outcome: String,
|
||||
val currentStage: String,
|
||||
@@ -226,8 +230,10 @@ private data class RunStateFields(
|
||||
* detail can never apply a frame differently.
|
||||
*/
|
||||
fun applying(frame: RunEventFrame): RunStateFields {
|
||||
val status = advanceRunStatus(status, frame.status)
|
||||
val status = if (frame.resourceWaitSet && frame.status == "queued" && !isTerminalRunStatus(status))
|
||||
"queued" else advanceRunStatus(status, frame.status)
|
||||
return RunStateFields(
|
||||
resourceWait = if (isTerminalRunStatus(status)) null else if (frame.resourceWaitSet) frame.resourceWait else resourceWait,
|
||||
status = status,
|
||||
outcome = frame.outcome.ifBlank { outcome },
|
||||
currentStage = frame.currentStage.ifBlank { currentStage },
|
||||
@@ -240,6 +246,7 @@ private data class RunStateFields(
|
||||
}
|
||||
|
||||
private fun RunSummary.fields() = RunStateFields(
|
||||
resourceWait = resourceWait,
|
||||
status = status,
|
||||
outcome = outcome,
|
||||
currentStage = currentStage,
|
||||
@@ -250,6 +257,7 @@ private fun RunSummary.fields() = RunStateFields(
|
||||
)
|
||||
|
||||
private fun RunDetail.fields() = RunStateFields(
|
||||
resourceWait = resourceWait,
|
||||
status = status,
|
||||
outcome = outcome,
|
||||
currentStage = currentStage,
|
||||
@@ -267,11 +275,12 @@ private fun executingFor(status: String, current: Boolean): Boolean = when (stat
|
||||
}
|
||||
|
||||
private fun RunSummary.applying(frame: RunEventFrame): RunSummary {
|
||||
if (isStaleRunFrame(status, frame.status)) return this
|
||||
if (isStaleRunFrame(status, frame.status, frame.resourceWaitSet)) return this
|
||||
val current = fields()
|
||||
val merged = current.applying(frame)
|
||||
if (merged == current) return this
|
||||
return copy(
|
||||
resourceWait = merged.resourceWait,
|
||||
status = merged.status,
|
||||
outcome = merged.outcome,
|
||||
currentStage = merged.currentStage,
|
||||
|
||||
@@ -615,6 +615,7 @@ private fun VersionChip(version: String) {
|
||||
|
||||
/** Progress from the fields a summary actually has: completed visits + stage. */
|
||||
private fun progressText(run: RunSummary): String = buildString {
|
||||
run.resourceWait?.let { append(it.statusLabel()); append(" · ") }
|
||||
append(visitCount(run.completedSteps, "completed"))
|
||||
if (run.currentStage.isNotBlank()) append(" · stage ${run.currentStage}")
|
||||
}
|
||||
|
||||
@@ -135,6 +135,7 @@ internal fun RunSummaryTab(
|
||||
)
|
||||
KeyValueRow("Pipeline", "id ${run.pipelineId} · v${run.version}")
|
||||
KeyValueRow("Stage", run.currentStage)
|
||||
run.resourceWait?.let { Text(it.statusLabel(), style = text.meta) }
|
||||
KeyValueRow("Workspace", run.workspace)
|
||||
KeyValueRow("Content hash", run.contentHash, monospace = true)
|
||||
KeyValueRow("Started", formatLocalTimestamp(run.createdAt).orEmpty())
|
||||
|
||||
@@ -0,0 +1,138 @@
|
||||
package net.awan.pipelines.data
|
||||
|
||||
import androidx.lifecycle.Lifecycle
|
||||
import androidx.lifecycle.LifecycleOwner
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.TimeUnit
|
||||
import java.util.concurrent.atomic.AtomicInteger
|
||||
import kotlinx.coroutines.CoroutineScope
|
||||
import kotlinx.coroutines.Dispatchers
|
||||
import kotlinx.coroutines.delay
|
||||
import net.awan.pipelines.data.transport.ConnectionState
|
||||
import org.junit.Assert.assertEquals
|
||||
import org.junit.Test
|
||||
|
||||
/**
|
||||
* The one lifecycle-aware owner of return-to-foreground recovery: it applies the
|
||||
* pure `foregroundAction` policy to the state it reads, confirms a supposedly
|
||||
* connected socket before trusting it, and reconnects when — and only when — the
|
||||
* policy says so. The state, the liveness check and the reconnect are injected,
|
||||
* so the owner is exercised with no device, socket or Android framework. The
|
||||
* owner never touches the LifecycleOwner it is handed.
|
||||
*/
|
||||
class ForegroundReconnectorTest {
|
||||
|
||||
private val reconnects = AtomicInteger(0)
|
||||
private val probes = AtomicInteger(0)
|
||||
|
||||
private val owner = object : LifecycleOwner {
|
||||
override val lifecycle: Lifecycle get() = throw UnsupportedOperationException("not used")
|
||||
}
|
||||
|
||||
private val connected = ConnectionState.Connected(build = null, serverTime = null)
|
||||
|
||||
/**
|
||||
* A reconnector whose probe answers [alive] without suspending, so
|
||||
* `Dispatchers.Unconfined` runs the whole `onStart` decision inline and the
|
||||
* assertions below hold the moment it returns.
|
||||
*/
|
||||
private fun reconnector(
|
||||
state: () -> ConnectionState,
|
||||
alive: suspend () -> Boolean = { true },
|
||||
) = ForegroundReconnector(
|
||||
state = state,
|
||||
confirmAlive = { probes.incrementAndGet(); alive() },
|
||||
reconnect = { reconnects.incrementAndGet() },
|
||||
scope = CoroutineScope(Dispatchers.Unconfined),
|
||||
)
|
||||
|
||||
@Test
|
||||
fun aDeliberateDisconnectIsLeftAloneAndNotEvenProbed() {
|
||||
reconnector(state = { ConnectionState.Idle }).onStart(owner)
|
||||
assertEquals(0, probes.get())
|
||||
assertEquals(0, reconnects.get())
|
||||
}
|
||||
|
||||
@Test
|
||||
fun aDroppedConnectionReconnectsImmediatelyWithoutProbing() {
|
||||
reconnector(state = { ConnectionState.Disconnected("socket closed", retrying = true) }).onStart(owner)
|
||||
assertEquals("a dropped socket has nothing to confirm", 0, probes.get())
|
||||
assertEquals(1, reconnects.get())
|
||||
}
|
||||
|
||||
@Test
|
||||
fun aConfirmedConnectedSocketIsLeftAlone() {
|
||||
reconnector(state = { connected }).onStart(owner)
|
||||
assertEquals(1, probes.get())
|
||||
assertEquals(0, reconnects.get())
|
||||
}
|
||||
|
||||
@Test
|
||||
fun aConnectedSocketThatDoesNotAnswerReconnects() {
|
||||
reconnector(state = { connected }, alive = { false }).onStart(owner)
|
||||
assertEquals(1, probes.get())
|
||||
assertEquals(1, reconnects.get())
|
||||
}
|
||||
|
||||
@Test
|
||||
fun aProbeThatThrowsIsNotAlive() {
|
||||
reconnector(
|
||||
state = { connected },
|
||||
alive = { throw IllegalStateException("socket closed") },
|
||||
).onStart(owner)
|
||||
assertEquals(1, reconnects.get())
|
||||
}
|
||||
|
||||
@Test
|
||||
fun aSocketThatDiesWhileTheProbeRunsStillReconnects() {
|
||||
// The socket's own failure callback lands mid-check; the owner re-reads
|
||||
// the state and the policy then decides on what is actually there.
|
||||
var state: ConnectionState = connected
|
||||
reconnector(
|
||||
state = { state },
|
||||
alive = {
|
||||
state = ConnectionState.Disconnected("socket closed", retrying = true)
|
||||
false
|
||||
},
|
||||
).onStart(owner)
|
||||
assertEquals(1, reconnects.get())
|
||||
}
|
||||
|
||||
@Test
|
||||
fun aDisconnectWhileTheProbeRunsStaysDisconnected() {
|
||||
// The user can tap Disconnect while the bounded foreground probe is in
|
||||
// flight. Re-reading state after that probe keeps the explicit choice
|
||||
// authoritative rather than reconnecting over it.
|
||||
var state: ConnectionState = connected
|
||||
reconnector(
|
||||
state = { state },
|
||||
alive = {
|
||||
state = ConnectionState.Idle
|
||||
true
|
||||
},
|
||||
).onStart(owner)
|
||||
assertEquals(0, reconnects.get())
|
||||
}
|
||||
|
||||
@Test
|
||||
fun aProbeThatNeverAnswersIsNotAliveWhenItsBudgetExpires() {
|
||||
val done = CountDownLatch(1)
|
||||
val reconnector = ForegroundReconnector(
|
||||
state = { connected },
|
||||
confirmAlive = { delay(5_000); true },
|
||||
reconnect = {
|
||||
reconnects.incrementAndGet()
|
||||
done.countDown()
|
||||
},
|
||||
scope = CoroutineScope(Dispatchers.Unconfined),
|
||||
confirmTimeoutMillis = 20L,
|
||||
)
|
||||
reconnector.onStart(owner)
|
||||
assertEquals(
|
||||
"the bounded check must give up on its own rather than hold the UI's Connected",
|
||||
true,
|
||||
done.await(2, TimeUnit.SECONDS),
|
||||
)
|
||||
assertEquals(1, reconnects.get())
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,60 @@
|
||||
package net.awan.pipelines.data.transport
|
||||
|
||||
import org.junit.Assert.assertEquals
|
||||
import org.junit.Test
|
||||
|
||||
/**
|
||||
* The return-to-foreground policy, in the one pure place it lives: an explicit
|
||||
* Disconnect is never overridden, a dropped or in-flight connection is
|
||||
* re-established at once, and a still-Connected socket is trusted only after it
|
||||
* has been confirmed to answer.
|
||||
*/
|
||||
class ConnectionRecoveryTest {
|
||||
|
||||
@Test
|
||||
fun aDeliberateDisconnectIsNeverOverridden() {
|
||||
// Idle is only ever reached by a deliberate Disconnect (the client starts
|
||||
// Idle and never returns to it by itself), so returning must leave it.
|
||||
assertEquals(
|
||||
ForegroundAction.LeaveAlone,
|
||||
foregroundAction(ConnectionState.Idle, confirmedAlive = false),
|
||||
)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun aDroppedConnectionReconnectsAtOnce() {
|
||||
assertEquals(
|
||||
ForegroundAction.Reconnect,
|
||||
foregroundAction(ConnectionState.Disconnected("socket closed", retrying = true), confirmedAlive = false),
|
||||
)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun anOpenAlreadyInFlightIsRestartedRatherThanWaitedOn() {
|
||||
assertEquals(
|
||||
ForegroundAction.Reconnect,
|
||||
foregroundAction(ConnectionState.Connecting(3), confirmedAlive = false),
|
||||
)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun aConnectedSocketIsTrustedOnlyAfterItAnswers() {
|
||||
val connected = ConnectionState.Connected(build = null, serverTime = null)
|
||||
assertEquals(ForegroundAction.LeaveAlone, foregroundAction(connected, confirmedAlive = true))
|
||||
// The defect: a socket the process still calls Connected but that answers
|
||||
// nothing must be reconnected, not trusted.
|
||||
assertEquals(ForegroundAction.Reconnect, foregroundAction(connected, confirmedAlive = false))
|
||||
}
|
||||
|
||||
@Test
|
||||
fun aFailureAfterConnectedStartsAtTheBaseBackoff() {
|
||||
// onOpen resets the client's attempt counter to zero. The next failure
|
||||
// must not pass -1 to the shift used for exponential backoff.
|
||||
assertEquals(1_000L, PipelinesClient.reconnectBackoffMillis(attempt = 0))
|
||||
}
|
||||
|
||||
@Test
|
||||
fun reconnectBackoffRemainsBounded() {
|
||||
assertEquals(30_000L, PipelinesClient.reconnectBackoffMillis(attempt = Int.MAX_VALUE))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
package net.awan.pipelines.ui
|
||||
|
||||
import kotlinx.serialization.json.Json
|
||||
import net.awan.pipelines.data.protocol.ResourceWait
|
||||
import net.awan.pipelines.data.protocol.RunDetail
|
||||
import net.awan.pipelines.data.protocol.RunEventFrame
|
||||
import net.awan.pipelines.data.protocol.RunSummary
|
||||
import org.junit.Assert.*
|
||||
import org.junit.Test
|
||||
|
||||
class ResourceWaitTest {
|
||||
@Test fun optionalMetadataAndPauseResumeClearing() {
|
||||
val json = Json { ignoreUnknownKeys = true }
|
||||
assertNull(json.decodeFromString<RunDetail>("""{"id":1,"status":"queued"}""").resourceWait)
|
||||
val wait = ResourceWait("phone-X", "review", 2, "2026-10-18T10:00:00Z")
|
||||
val queued = RunEventFrame(runId = 1, status = "queued", resourceWaitSet = true, resourceWait = wait)
|
||||
val detail = foldRunEvent(RunDetail(id = 1, status = "running"), queued)
|
||||
assertEquals("queued", detail.status)
|
||||
assertEquals(wait, detail.resourceWait)
|
||||
assertEquals("Waiting for resource phone-X", wait.statusLabel())
|
||||
val rows = foldRunEvent(listOf(RunSummary(id = 1, status = "running")), queued).runs
|
||||
assertEquals(wait, rows.single().resourceWait)
|
||||
val clear = RunEventFrame(runId = 1, status = "running", resourceWaitSet = true)
|
||||
assertNull(foldRunEvent(detail, clear).resourceWait)
|
||||
assertNull(foldRunEvent(rows, clear).runs.single().resourceWait)
|
||||
assertNull(foldRunEvent(detail, RunEventFrame(runId = 1, status = "canceled")).resourceWait)
|
||||
assertEquals("succeeded", foldRunEvent(RunDetail(id = 1, status = "succeeded"), queued).status)
|
||||
val decoded = json.decodeFromString<RunDetail>("""{"id":1,"status":"queued","resource_wait":{"resource":"phone-X","stage":"review","sequence":2,"queued_since":"now"}}""")
|
||||
assertEquals("phone-X", decoded.resourceWait?.resource)
|
||||
assertFalse(isWaitingRunStatus(decoded.status))
|
||||
}
|
||||
}
|
||||
@@ -23,6 +23,7 @@ androidx-compose-material3 = { group = "androidx.compose.material3", name = "mat
|
||||
androidx-compose-material-icons-extended = { group = "androidx.compose.material", name = "material-icons-extended" }
|
||||
androidx-navigation-compose = { group = "androidx.navigation", name = "navigation-compose", version.ref = "navigationCompose" }
|
||||
androidx-lifecycle-runtime-compose = { group = "androidx.lifecycle", name = "lifecycle-runtime-compose", version.ref = "lifecycle" }
|
||||
androidx-lifecycle-process = { group = "androidx.lifecycle", name = "lifecycle-process", version.ref = "lifecycle" }
|
||||
androidx-lifecycle-viewmodel-compose = { group = "androidx.lifecycle", name = "lifecycle-viewmodel-compose", version.ref = "lifecycle" }
|
||||
androidx-lifecycle-viewmodel-ktx = { group = "androidx.lifecycle", name = "lifecycle-viewmodel-ktx", version.ref = "lifecycle" }
|
||||
kotlinx-coroutines-android = { group = "org.jetbrains.kotlinx", name = "kotlinx-coroutines-android", version.ref = "coroutines" }
|
||||
|
||||
@@ -47,7 +47,7 @@ func main() {
|
||||
// Each new run gets a bounded one-line title from the configured default
|
||||
// harness; the summariser falls back to the prompt's first line, so title
|
||||
// generation never blocks or fails run creation.
|
||||
options := server.Options{ArtifactsRoot: artifactsRoot, RunTitles: summarise.GenerateTitle, StepVerdicts: summarise.GenerateVerdictReason}
|
||||
options := server.Options{ArtifactsRoot: artifactsRoot, RunTitles: summarise.GenerateTitle, StepVerdicts: summarise.GenerateVerdictReason, VerificationFixtures: server.VerificationFixturesFromEnvironment(os.Getenv)}
|
||||
recorded := ""
|
||||
if value, err := store.OpenCodeEndpoint(ctx); err != nil {
|
||||
log.Printf("OpenCode endpoint setting unreadable: %v", err)
|
||||
|
||||
@@ -0,0 +1,37 @@
|
||||
# Background shell visibility
|
||||
|
||||
The chat server now opens the broker stream for read-only subscribers and reconciles verified running shell operations from OpenCode's authoritative `/api/shell` readback. The packaged OpenCode skill references and chat documentation should describe that shell visibility survives reconnects only when OpenCode reports explicit `metadata.sessionID` ownership; shared working directories are not ownership evidence.
|
||||
|
||||
## Current source behavior
|
||||
|
||||
Observers share one initial readback per room/stream attachment. The adapter and hub
|
||||
fold bounded lifecycle journals over each snapshot, so a stale read cannot resurrect
|
||||
an exited shell or erase a concurrent create. Shell replay and live delivery enqueue
|
||||
in fold order. Cancellation rejects old-room callbacks and releases local identity
|
||||
state before reconnect. Journals retain at most 256 identities and 1 MiB of id/command
|
||||
text; capacity/read failures surface stable limited-visibility text, never broker
|
||||
response bodies. No shell polling was added. OpenCode's terminal `timeout` and
|
||||
`killed` statuses are excluded from retained running operations as well as `exited`.
|
||||
|
||||
## Verification evidence — 2026-10-18
|
||||
|
||||
`opencode api get /openapi.json` authenticated through supported service discovery.
|
||||
The live schema advertises shell list/detail/output and generic `metadata`; it does
|
||||
not type `metadata.sessionID`. A throwaway session
|
||||
`ses_eb12ab7b3ffe0ZAle7PHpftHMG` and its own `sleep 1` shell
|
||||
`sh_14ed57134001e6fuYqvn62pKit` verified actual create/detail readback preserving
|
||||
explicit `metadata.sessionID`, status `running`, subsequent `exited`/exit 0, and
|
||||
absence from the running list. This verifies explicit API metadata, not automatic
|
||||
agent-tool ownership injection. The CLI event attempt yielded no captured events;
|
||||
live-provider event ordering is **not tested**. Forced ordering evidence comes from
|
||||
channel/barrier regressions, including a real WebSocket handler on a loopback scratch
|
||||
server with fresh SQLite, not from the deployed service.
|
||||
|
||||
## Documentation follow-up / deployment
|
||||
|
||||
`docs/chat-verification.md` and the chat subscribe/replay section of `docs/cli.md`
|
||||
are stale: docs-fold should describe authoritative ownership/readback, coalesced
|
||||
observation, bounded reconciliation and limited-visibility errors, distinguishing
|
||||
fixture ordering evidence from the live readback above. Shared docs and packaged
|
||||
skill references were not edited here. No deployment, installation, service restart
|
||||
or physical-phone/UI verification was performed; this changes no rendered layout.
|
||||
@@ -0,0 +1,47 @@
|
||||
# Delivery stabilization
|
||||
|
||||
Date: 2026-10-18.
|
||||
|
||||
The server now has draft-only `delivery=true` support for OpenCode model stages.
|
||||
At run time it derives the canonical `main` worktree and linked-worktree Git
|
||||
metadata from Git rather than accepting paths from a stage prompt. It verifies
|
||||
the committed item tree in a disposable candidate worktree before fast-forwarding
|
||||
canonical `main`; declared checks are bounded configuration, not proof that any
|
||||
deployed definition has used them. It is denied for Pi (until equivalent
|
||||
capability enforcement exists), foreign repositories, missing main worktrees,
|
||||
and malformed capabilities. This item does not publish or deploy a definition.
|
||||
|
||||
The source also supports an optional controlled-fixture verification declaration
|
||||
(`backend`, `revision`, `capability`, and `fixture`). Ordinary server setup wires
|
||||
an explicit, bounded `reconnect-foreground` HTTP fixture registry when
|
||||
`PIPELINES_VERIFICATION_RECONNECT_FOREGROUND_URL` and its expected `BACKEND`,
|
||||
`REVISION`, `CAPABILITY`, and `STATE` values are configured. There is no default
|
||||
remote URL. The server, not the model, probes that registered endpoint and
|
||||
collects its returned backend identity, build revision, capability, fixture ID,
|
||||
and state before comparing provenance. Unknown fixtures, incomplete setup,
|
||||
wrong backend/revision/capability/state, non-200 responses, cancellation, and
|
||||
oversized or malformed responses fail closed. A model result's labels are not
|
||||
evidence. This is controlled local/backend fixture infrastructure, not evidence
|
||||
of a rendered layout or a live Android device.
|
||||
Definitions must be scratch-validated against the deployed server only after
|
||||
the server support is deployed; no future delivery or verification definition
|
||||
is activated by this source change.
|
||||
|
||||
`internal/author/skill/references/format.md` and
|
||||
`internal/author/skill/references/queue.md` are stale: a later docs fold should
|
||||
describe `delivery=true`, OpenCode-only availability, candidate verification,
|
||||
and the required activation order (deploy server support before scratch-validating
|
||||
and then publishing a definition).
|
||||
OpenCode V2 resolves external resources at a containing directory, so delivery
|
||||
uses the verified repository metadata directory rather than an impossible
|
||||
single-lock-file scope. It does not grant a parent, home, foreign repository,
|
||||
or global policy. Unrestricted shell permissions are not an OS sandbox.
|
||||
|
||||
Activation remains draft-only: deploy compatible server support, configure the
|
||||
controlled fixture endpoint and matching provenance, scratch-validate the
|
||||
fixture, then publish a definition. No definition is activated by this source
|
||||
change.
|
||||
|
||||
Recovery checklist: retain `connection-reconnect` and unfinished feature
|
||||
branches; merge only complete committed trees that pass mapped checks; do not
|
||||
infer delivery from a dirty worktree or an old deployed-backend capture.
|
||||
@@ -0,0 +1,95 @@
|
||||
# Android: coming back to the app restores the connection
|
||||
|
||||
## What this item changed
|
||||
|
||||
Android app only.
|
||||
|
||||
- `data/transport/ConnectionRecovery.kt` (new): `ForegroundAction` and the one
|
||||
pure `foregroundAction(state, confirmedAlive)` policy — `Idle` → leave alone (a
|
||||
deliberate Disconnect is the only way the client reaches Idle), `Connected` with
|
||||
a confirmed-alive socket → leave alone, `Connected` unconfirmed → reconnect,
|
||||
`Connecting`/`Disconnected` → reconnect.
|
||||
- `data/ForegroundReconnector.kt` (new): the one lifecycle-aware owner. It is
|
||||
created once with the Application and observes `ProcessLifecycleOwner`, never
|
||||
the Activity or the ViewModel (both recreated). On `ON_START` it applies the
|
||||
policy: a non-Connected state reconnects at once, and the reconnect cancels the
|
||||
client's pending backoff delay, so the operator never waits it out; a state
|
||||
that still claims `Connected` is first confirmed with the existing `ping`,
|
||||
bounded to 3 s, and the state is re-read afterwards (the socket's own failure
|
||||
callback can land while the check is in flight).
|
||||
- `PipelinesApplication.kt`: wires the owner to the process lifecycle;
|
||||
`confirmAlive` is `repository.ping()`, `reconnect` is
|
||||
`ServerUrl.validate(...) + repository.recover(...)`. Recovery shares the
|
||||
client's state lock with Disconnect, so a Disconnect after the probe has made
|
||||
its decision but before its reconnect effect runs stays authoritative. Manual
|
||||
Connect/Reconnect continue to use `repository.connect(...)`; the client's
|
||||
bounded backoff stays the only reconnect mechanism and the policy lives in the
|
||||
one pure place.
|
||||
- `data/transport/PipelinesClient.kt`: `open()` now closes the socket it
|
||||
supersedes, and every `WebSocketListener` callback ignores a socket that is no
|
||||
longer the current one (`isCurrent`). A failure the replaced socket reports late
|
||||
— the common case when the process was frozen while it died — can then neither
|
||||
clear the new connection nor schedule a reconnect that would undo the recovery.
|
||||
- `app/build.gradle.kts` and `gradle/libs.versions.toml`:
|
||||
`androidx.lifecycle:lifecycle-process` 2.8.7 (already in the offline cache,
|
||||
with its `androidx.startup`/`androidx.tracing` transitives).
|
||||
- Tests: `data/transport/ConnectionRecoveryTest.kt` (5 cases) and
|
||||
`data/ForegroundReconnectorTest.kt` (8 cases).
|
||||
|
||||
## Shared documents this change makes stale
|
||||
|
||||
`android/README.md`.
|
||||
|
||||
- The Configuration section (`android/README.md:99-101`) — "Tap **Connect** to
|
||||
open the single shared WebSocket (probed with `ping`, showing the server
|
||||
build). Reconnect uses bounded exponential backoff; **Disconnect** stops it." —
|
||||
should record that coming back to the app restores the connection on its own:
|
||||
the process coming to the front reconnects immediately, cancelling the pending
|
||||
backoff delay, and when the last state still claims Connected it first confirms
|
||||
the socket with `ping` before trusting it. An explicit Disconnect stays
|
||||
authoritative and is never reconnected over; the manual Reconnect control
|
||||
stays, but normal recovery never needs it.
|
||||
- The Architecture block (`android/README.md:310-323`) lists `data/transport` and
|
||||
`data`. It should name the two new files — `data/transport/ConnectionRecovery.kt`
|
||||
(the pure return-to-foreground policy) and `data/ForegroundReconnector.kt` (the
|
||||
process-lifecycle owner) — and the new `androidx.lifecycle:lifecycle-process`
|
||||
dependency behind the owner.
|
||||
|
||||
## Not stale
|
||||
|
||||
- `android/AGENTS.md` names neither the presence nor the absence of a lifecycle
|
||||
observer; its "Where things are" summary stays correct (it could optionally add
|
||||
the new owner beside `data/PipelinesRepository.kt`).
|
||||
- `docs/app-integration-verification.md` — its connection mentions are about
|
||||
restoring the stub URL, which this item does not change.
|
||||
- No packaged OpenCode skill reference (`internal/author/skill/references/`)
|
||||
documents the app's connection lifecycle, so nothing there is stale.
|
||||
|
||||
## Verified on the device, and what was not
|
||||
|
||||
Built and captured on the operator's Pixel 9 Pro XL (`47241FDAS00DLK`) from
|
||||
`51a2ba4` + this change; after merging main the same tree (now `b93eee2`) was
|
||||
rebuilt and the full unit suite re-run (691 tests, 0 failures). The operator's
|
||||
`ws://10.0.0.75:8080/connect` and `/Users/alan/repos/pipelines` were used
|
||||
throughout and were not changed. The phone was left on main's build.
|
||||
|
||||
Captured, all with `android/scripts/capture_png.sh` (every image validated):
|
||||
|
||||
- a **live** socket on return: the server log shows a `ping`/`pong` on the same
|
||||
connection at the second of the return and no reconnect — the positive check
|
||||
runs and a healthy connection is left alone;
|
||||
- a **dead** socket on return: the reconnect and re-subscribe land in the same
|
||||
second as the return, with no interaction (the process was frozen, so the
|
||||
client's own backoff had not run);
|
||||
- **no network** on return: the banner reads "Disconnected: Failed to connect to
|
||||
…:8080 · retrying with backoff" — never a false Connected — and recovery
|
||||
follows on its own once the network returns;
|
||||
- an explicit **Disconnect**: still "Not connected. Enter the server URL in
|
||||
Settings, then connect." after returning, with no reconnect in the server log.
|
||||
|
||||
**Not verified live:** the stale-Connected probe branch. In every capture the
|
||||
frozen socket's own failure was delivered to the client before the lifecycle
|
||||
owner ran, so the client already read `Disconnected` and no probe was attempted
|
||||
(confirmed by the client's request-id sequence never skipping a value in the
|
||||
server log). That branch — `Connected` still claimed, probe fails, reconnect — is
|
||||
covered by unit tests only.
|
||||
@@ -0,0 +1,158 @@
|
||||
# Stage-exclusive resources — 2026-10-18, not deployed
|
||||
|
||||
This is source work for run 355, **not a deployment, installation or publication**.
|
||||
No physical phone, real operator session or deployed service was used or changed.
|
||||
The Android role split does not retrospectively gate pinned versions or existing runs.
|
||||
|
||||
## Contract and scope
|
||||
|
||||
An executable non-human stage can declare one runner-owned control block:
|
||||
|
||||
```resources
|
||||
{"exclusive":"phone-47241FDAS00DLK"}
|
||||
```
|
||||
|
||||
The key is 1–128 ASCII identifier bytes (letter first, then letters, digits, `_`, `-`).
|
||||
Duplicate/unknown fields, multiple resources and human-stage resources are rejected.
|
||||
One capacity-one resource per stage visit, one server/store authority; no expressions,
|
||||
nested acquisition, distributed scheduling, capacity settings or multi-resource requests.
|
||||
Resource-free definitions retain their execution path. Old servers reject this new block;
|
||||
do not publish to an incompatible server.
|
||||
|
||||
Acquisition follows selection and automatic UI-skip evaluation, before BeginStep,
|
||||
visits/sequences, artifacts, filediff, stage timeout, budget setup or executor calls.
|
||||
Queued runs retain workspace reservation, use legacy `queued` (never human `waiting`),
|
||||
and carry bounded `resource_wait` identity/stage/sequence/queued-since metadata.
|
||||
Events use `resource_wait_set=true` with absent `resource_wait` to clear caches.
|
||||
CLI lists and TUI/Android details show “Waiting for resource …”; human decisions and
|
||||
attention notifications remain separate. Older clients still see an unfinished run.
|
||||
|
||||
SQLite owners and FIFO waiters are indexed by resource/order and run identity.
|
||||
Private random capabilities attribute ownership to resource/run/stage/sequence and
|
||||
never enter wire data, model input, transcripts or artifacts. Only the full capability
|
||||
and positive cleanup evidence authorize release; mismatches and repeated release cannot
|
||||
release a subsequent owner. One buffered channel per live queued run, no resource
|
||||
polling, waiter tickers, model-based acquisition, queue driver or new daemon.
|
||||
|
||||
The gate holds through tool cleanup. OpenCode interruption now confirms session idle/
|
||||
V1 settlement before clearing session ownership. Pi recovery requires both its receipt
|
||||
and process/process-group exit, not merely abort acknowledgement. Executor panic,
|
||||
ErrStageCleanup, retained active sessions and failed cleanup queries quarantine the owner.
|
||||
An explicit `output.resource_cleanup_confirmed=false` (or malformed value) also vetoes
|
||||
release for failed restoration; true never substitutes for harness cleanup evidence.
|
||||
The generic runner does not inspect Android settings or magically verify restoration.
|
||||
The Android UI reviewer must keep all device work foreground/awaited, restore original
|
||||
settings/connection and main's build, and report uncertainty with the release veto.
|
||||
|
||||
## Restart and retention
|
||||
|
||||
Automatic FIFO progress is promised only within a live server process. Startup interrupts
|
||||
persisted queued resource visits, removes their waiter rows, and never replays an executor.
|
||||
Durably cleanup-confirmed owners can be reclaimed through the token-checked delete helper.
|
||||
Unconfirmed owners remain quarantined, including crashes before BeginStep, before session
|
||||
registration, or after cleanup but before its confirmation is persisted. Session interruption
|
||||
alone cannot prove resource-specific restoration. Age never authorizes reclamation.
|
||||
Quarantined ownership/session evidence survives recovery and protects even terminal runs
|
||||
from retention. Reconciliation requires inspected cleanup and restoration, not lock deletion.
|
||||
There is intentionally no force-unlock CLI or automatic uncertain-owner reclamation.
|
||||
|
||||
## Android draft, provenance and activation
|
||||
|
||||
Read-only retrieval: `pipelines show Android --version 2 --json` returned pipeline 20,
|
||||
v2, hash `8c93aeb9b76f1e6e7696415d6308d786ec08d805a1a346fbabe4d252d5d65032`.
|
||||
`python3` recomputed SHA-256 of the returned source and asserted equality.
|
||||
The exact source is retained as `internal/server/testdata/android-v2-published.md`;
|
||||
the proposed source is `internal/server/testdata/android-resource-draft.md`.
|
||||
The provenance test compares graph fields/outcomes unchanged, apart from the approved
|
||||
three stage prompts and the UI review resource; every stage keeps its published v3 model: Terra for implementer/committer and Luna for reviewer/UI reviewer. No unrelated published definition was edited.
|
||||
|
||||
Artifact handoff recovery: the validated v3 draft was copied to this run's retained
|
||||
artifact directory as `android-resource-draft.md`. The similarly named artifact from
|
||||
run 355 remains the old v2 baseline and must not be published. The proposed source in
|
||||
this workspace and this run's artifact have matching SHA-256 digests
|
||||
`a10d8b9fa3dc6a1875b10469061ed25fb4a7a404d463d1a2d304a309d55aa5d9`.
|
||||
|
||||
Activation, requiring separate explicit operator authorization:
|
||||
1. Review/land the source change and independently rerun the required checks.
|
||||
2. Arrange a compatible deployment separately; nothing here deploys or restarts it.
|
||||
3. Reconcile every existing phone user, pinned/live run and any quarantined owner.
|
||||
4. Transfer the **workspace proposed draft**, provenance and this note into the retained
|
||||
stage artifacts after resolving the artifact edit permission.
|
||||
5. Validate that retained draft against the configured authoritative compatible endpoint;
|
||||
only then explicitly publish it. Do not use a disposable endpoint as a project default.
|
||||
6. Opt in other device users explicitly before relying on mutual exclusion system-wide.
|
||||
|
||||
Read-only published inventory (`pipelines show <name> --json`, v2 for each):
|
||||
- UI sweep: `survey` and `ui_reviewer` request visual/screenshots work; phone work there
|
||||
needs explicit resource opt-in and restoration/fixture policy.
|
||||
- Generic: `reviewer` and `ui_reviewer` have visual/device verification wording; when
|
||||
targeting a phone, separate code review from leased device review first.
|
||||
- Plan and land: `implementer`, `reviewer`, `ui_reviewer` contain rendered-surface work;
|
||||
they need an explicit technology-specific role split/lease before phone use.
|
||||
- Probe: one harness smoke stage; no default phone declaration. Any device-specific
|
||||
probe needs explicit opt-in, never an inferred resource or Android stage-name special case.
|
||||
Server/TUI/other definitions were not changed. Inventory is not activation evidence.
|
||||
|
||||
## Verification evidence and remaining limits
|
||||
|
||||
Executed focused checks:
|
||||
- Run 358 recovery revalidation: `go build ./... && go vet ./... && go test ./...`
|
||||
passed; `go test -race ./internal/pipeline ./internal/server ./internal/stage -run
|
||||
'Resource|Scheduler|Concurrent|Human|Decision|RetryWaits' -count=1` passed; and the
|
||||
Android map invocation with `JAVA_HOME`/`ANDROID_HOME`, installed Gradle,
|
||||
`--offline --no-daemon --rerun-tasks :app:assembleDebug :app:testDebugUnitTest`
|
||||
executed all 41 tasks and reported `BUILD SUCCESSFUL`. These remain build/unit and
|
||||
fake-executor scheduler evidence,
|
||||
not physical-phone verification or screenshots.
|
||||
- Recovery after integrating main `88595eb`: `go build ./...`, `go vet ./...`,
|
||||
`go test ./internal/server -run Resource -count=1`,
|
||||
`go test ./internal/pipeline ./internal/protocol ./internal/tui -run Resource -count=1`,
|
||||
`go test ./internal/stage -count=1`,
|
||||
`go test -race ./internal/pipeline ./internal/server ./internal/stage -run 'Resource|Scheduler|Concurrent|Human|Decision|RetryWaits' -count=1`, and
|
||||
`go test ./...` all passed. `gofmt -l .` and `git diff --check` produced no output.
|
||||
- Recovery Android build/unit check with the map's offline, no-daemon Gradle invocation:
|
||||
`:app:assembleDebug :app:testDebugUnitTest` passed. This remains build/unit evidence,
|
||||
not phone verification or screenshots.
|
||||
- `go test ./internal/server ./internal/pipeline ./internal/protocol ./internal/tui -run Resource -count=1`:
|
||||
passed. Includes fake-executor queue >30ms timeout, independent-resource progress,
|
||||
FIFO/mismatch/duplicate/grant-cancel cycles, blocked cleanup, panic/quarantine,
|
||||
persistence/BeginStep/query failure, no-acquire skip, restart/retention, wire defaults,
|
||||
narrow/wide/plain TUI frames and real CLI/WebSocket scratch exercise.
|
||||
- `go test ./internal/stage -count=1`: passed; the later full suite also included
|
||||
`TestResourcePiReceiptRequiresProcessExit` (a test-owned sleep process, not a device).
|
||||
- Android map invocation with `JAVA_HOME`/`ANDROID_HOME` and installed Gradle,
|
||||
`--offline --no-daemon :app:assembleDebug :app:testDebugUnitTest`: BUILD SUCCESSFUL.
|
||||
These are build/unit checks, not device verification or screenshots.
|
||||
- After the Go tree settled: `go build ./...`, `go vet ./...`, `go test ./...`
|
||||
all exited zero (including author-package tests).
|
||||
- `go test -race ./internal/pipeline ./internal/server ./internal/stage -run 'Resource|Scheduler|Concurrent|Human|Decision|RetryWaits' -count=1`:
|
||||
all three packages passed, including concurrent grant/cancel cycles.
|
||||
- `gofmt -l .` and `git diff --check`: exited zero with no output.
|
||||
|
||||
The scratch CLI test uses the real CLI and WebSocket handler with temporary SQLite and
|
||||
an explicitly **FAKE executor**. It validates the proposed Android draft on compatible
|
||||
scratch code, observes B as queued with zero steps while A holds fake-phone, inspects
|
||||
subscription metadata, releases A, runs `wait`, and observes B succeeded automatically.
|
||||
It does not call a model or use a physical phone. Test log output is evidence, not a
|
||||
live-deployment claim; no retained CLI transcript was written to the denied artifact path.
|
||||
|
||||
Live OpenCode/Pi shell/ADB termination and Android visual/device restoration are **not
|
||||
tested** here. The checks prove scheduler behavior with controlled fakes, plus test-owned
|
||||
process-exit checks, not device readiness on the current deployment.
|
||||
Commit/merge/landing belong to the later committer stage; no commit or push here.
|
||||
|
||||
## Stale shared documentation for docs-fold (do not edit in this item)
|
||||
|
||||
- `docs/cli.md`: resource queue presentation, inspection metadata, unchanged wait/decide semantics.
|
||||
- `internal/author/skill/references/format.md`: optional resources block and validation/key bounds.
|
||||
- `internal/author/skill/references/control.md`: runner-owned declaration, timeout/accounting boundary,
|
||||
release veto and automatic skip placement.
|
||||
- `internal/author/skill/references/queue.md`: FIFO resource waits vs workspace exclusion and human waits.
|
||||
- `internal/author/skill/references/operations.md`: authority, restart interruption/quarantine,
|
||||
inspected recovery, retention and explicit compatible-deployment/publication prerequisites.
|
||||
- `internal/author/skill/references/verification.md`: fake-vs-device evidence, fixtures and leased restoration.
|
||||
- `internal/author/skill/references/orchestration.md`: no queue drivers/polling/extra agents; explicit
|
||||
opt-in of every device user, no retroactive gating or distributed guarantees.
|
||||
|
||||
No packaged skill/reference, shared README/topic documentation, pipeline registry,
|
||||
installation, deployment, provider setting or `internal/stage/model_failure.go` was edited.
|
||||
@@ -0,0 +1,20 @@
|
||||
# Scoped stage artifact access
|
||||
|
||||
## What this item changed
|
||||
|
||||
The server now derives one exact artifact capability for an OpenCode or Pi
|
||||
model-stage visit: its own directory is writable when the edit tool is present,
|
||||
and its own plus completed earlier visit directories in the same run are
|
||||
readable when a file-reading tool is present. No artifact root, sibling, future
|
||||
visit, other run, traversal path, or symlink escape is granted. The scope is
|
||||
empty if the runner cannot validate its artifact root.
|
||||
|
||||
Pi retains its lease and explicit extension policy; unrestricted shell remains
|
||||
outside this file-tool boundary and is not an OS sandbox.
|
||||
|
||||
## Shared documents this change makes stale
|
||||
|
||||
- `docs/stage-artifacts.md` and `internal/author/skill/references/operations.md`
|
||||
should explain that model-stage artifact directories are usable by the
|
||||
supported file tools only within the exact same-run visit scope, not as a
|
||||
general external-directory permission.
|
||||
@@ -0,0 +1,42 @@
|
||||
# Stage-timeout diagnostics
|
||||
|
||||
## Source behavior
|
||||
|
||||
The V2 active-session wait now reports `STAGE DEADLINE` when the configured
|
||||
attempt timer expires between successful polls of a running admitted session.
|
||||
Its bounded diagnostic names the idle-wait phase, configured timeout, retained
|
||||
session, and that the stage had not finished (active at the last poll). It does
|
||||
not claim that a completed response was lost. Cancellation is no longer recorded
|
||||
as a stage deadline. A bare deadline alone is not `HARNESS TRANSPORT`.
|
||||
|
||||
Actual request/HTTP/operation failures keep `HARNESS TRANSPORT`, their sanitized
|
||||
operation/status evidence and underlying error identities, including when joined
|
||||
with a deadline. A parent deadline no longer discards an accompanying transport
|
||||
failure. Final-result validation, budget accounting, cleanup, interrupts,
|
||||
side effects and replay policy are unchanged. No timeout was increased.
|
||||
|
||||
## Documentation to fold manually
|
||||
|
||||
`internal/author/skill/references/debugging.md` needs a separate stage-deadline
|
||||
playbook alongside its HARNESS TRANSPORT section: inspect the retained session,
|
||||
last activity, configured limits and unfinished final-result evidence before
|
||||
deciding what to do. Do not blindly retry an admitted mutating stage; edits/tool
|
||||
side effects remain. Explain the distinction between a configured idle-wait
|
||||
overrun, cancellation, an actual failed request with a deadline, budget failure,
|
||||
and final-result validation. Do not infer that all deadline errors are transport
|
||||
failures or that a successful last tool means the stage returned a result.
|
||||
|
||||
## Evidence and deployment
|
||||
|
||||
2026-10-18: installed server build metadata (`go version -m
|
||||
/Users/alan/.local/share/pipelines/bin/pipelines-server`) identifies revision
|
||||
`0d1ddc73a444fa24110ba5223646856c8f053d29`. Comparing that revision with this
|
||||
worktree's initial HEAD showed identical model-failure classification, model
|
||||
attempt timing, V2 polling and agent wait/cleanup paths. The runner difference
|
||||
was unrelated session-export HTML-only metadata.
|
||||
|
||||
Regression tests use isolated loopback servers and scratch SQLite databases,
|
||||
with a controlled V2 CLI fixture for the admitted-active-session deadline and
|
||||
real loopback HTTP/refused-connection checks for native transport evidence.
|
||||
This is not a live-provider timeout test. No deployment or deployed-service
|
||||
restart is part of this item; retained historical runs are not rewritten.
|
||||
+2
-2
@@ -774,7 +774,7 @@ func (a App) Run(ctx context.Context, args []string) error {
|
||||
}
|
||||
case "runs":
|
||||
for _, run := range r.Runs {
|
||||
fmt.Fprintf(a.Out, "%d\t%s\t%s\tv%d\t%s\n", run.ID, run.Label(), run.Name, run.Version, run.Status)
|
||||
fmt.Fprintf(a.Out, "%d\t%s\t%s\tv%d\t%s\n", run.ID, run.Label(), run.Name, run.Version, run.StatusLabel())
|
||||
}
|
||||
case "ping":
|
||||
fmt.Fprintln(a.Out, "Connected ·", r.Time)
|
||||
@@ -1484,7 +1484,7 @@ func (e *waitLimitError) Runs() string {
|
||||
// waitLine is the one-line terminal report `pipelines wait` prints: the run id,
|
||||
// its status and outcome, its stage and how long it took.
|
||||
func waitLine(d *protocol.RunDetail) string {
|
||||
line := fmt.Sprintf("run %d · %s", d.ID, d.Status)
|
||||
line := fmt.Sprintf("run %d · %s", d.ID, d.StatusLabel())
|
||||
if outcome := strings.TrimSpace(d.Outcome); outcome != "" {
|
||||
line += " (" + outcome + ")"
|
||||
}
|
||||
|
||||
+198
-11
@@ -28,7 +28,9 @@ var ErrPermissionUnsupported = errors.New("OpenCode permission approval is not i
|
||||
// semantics, the session-id shape and the 64 KiB / 1 MiB bounds of Connection so
|
||||
// no second transport or credential path is introduced.
|
||||
type Chat struct {
|
||||
conn *Connection
|
||||
conn *Connection
|
||||
shellMu sync.Mutex
|
||||
shells map[string]*ChatStreamState
|
||||
// mu serializes the adapter's HTTP calls. Connection records its last
|
||||
// operation and status on the shared connection, so two calls must not
|
||||
// overlap; the permission poller runs alongside the read loop's
|
||||
@@ -294,6 +296,114 @@ func decodeChatPermissions(sessionID string, data []byte) ([]protocol.ChatPermis
|
||||
return frames, nil
|
||||
}
|
||||
|
||||
// Shells reads the endpoint's authoritative shell list and returns only running
|
||||
// operations whose metadata explicitly names sessionID. The shell API is global;
|
||||
// filtering locally avoids guessing an unsupported query filter and never assigns
|
||||
// ownership from a shared working directory. A successful read also seeds the
|
||||
// stream state used to attribute later shell.exited events, which carry no session.
|
||||
func (c *Chat) Shells(ctx context.Context, sessionID string) ([]protocol.ChatShell, error) {
|
||||
if !ValidSessionID(sessionID) {
|
||||
return nil, fmt.Errorf("invalid OpenCode session id")
|
||||
}
|
||||
state, err := c.streamStateContext(ctx, sessionID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// One read per state; waiting and I/O never hold the event-state mutex.
|
||||
state.mu.Lock()
|
||||
if state.readGate == nil {
|
||||
state.readGate = make(chan struct{}, 1)
|
||||
}
|
||||
gate := state.readGate
|
||||
state.mu.Unlock()
|
||||
select {
|
||||
case gate <- struct{}{}:
|
||||
case <-ctx.Done():
|
||||
return nil, ctx.Err()
|
||||
}
|
||||
defer func() { <-gate }()
|
||||
state.mu.Lock()
|
||||
state.readback = &ShellSnapshot{}
|
||||
state.mu.Unlock()
|
||||
defer func() { state.mu.Lock(); state.readback = nil; state.mu.Unlock() }()
|
||||
data, err := c.call(ctx, "get", "/api/shell", nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var response struct {
|
||||
Data []chatShellInfo `json:"data"`
|
||||
}
|
||||
if err := json.Unmarshal(data, &response); err != nil || response.Data == nil {
|
||||
if err := json.Unmarshal(data, &response.Data); err != nil {
|
||||
return nil, fmt.Errorf("invalid OpenCode shell response")
|
||||
}
|
||||
}
|
||||
out := make([]protocol.ChatShell, 0, len(response.Data))
|
||||
for _, info := range response.Data {
|
||||
if info.ID == "" || info.shellSessionID() != sessionID || (info.Status != "" && info.Status != protocol.ChatShellRunning) {
|
||||
continue
|
||||
}
|
||||
status := info.Status
|
||||
if status == "" {
|
||||
status = protocol.ChatShellRunning
|
||||
}
|
||||
out = append(out, protocol.ChatShell{ID: info.ID, Status: status, Command: info.Command})
|
||||
}
|
||||
state.mu.Lock()
|
||||
defer state.mu.Unlock()
|
||||
out, err = state.readback.Merge(out)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ids := make([]string, 0, len(out))
|
||||
for _, shell := range out {
|
||||
ids = append(ids, shell.ID)
|
||||
}
|
||||
state.replaceShellsLocked(ids)
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// streamState returns the session's retained, verified shell identities. It is
|
||||
// deliberately kept across stream replacement so an exit after reconnect remains
|
||||
// attributable; Shells reconciles its contents from authoritative readback on
|
||||
// each attach rather than replacing the pointer a live scanner is using.
|
||||
func (c *Chat) streamState(sessionID string) *ChatStreamState {
|
||||
state, _ := c.streamStateContext(context.Background(), sessionID)
|
||||
return state
|
||||
}
|
||||
|
||||
func (c *Chat) streamStateContext(ctx context.Context, sessionID string) (*ChatStreamState, error) {
|
||||
c.shellMu.Lock()
|
||||
defer c.shellMu.Unlock()
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if c.shells == nil {
|
||||
c.shells = map[string]*ChatStreamState{}
|
||||
}
|
||||
state := c.shells[sessionID]
|
||||
if state == nil {
|
||||
state = &ChatStreamState{}
|
||||
c.shells[sessionID] = state
|
||||
}
|
||||
return state, nil
|
||||
}
|
||||
|
||||
// ReleaseShells discards verified shell identities after the last room observer
|
||||
// disconnects. A future attach calls Shells for a fresh authoritative snapshot,
|
||||
// so this bounds state without attributing a foreign shell during the gap.
|
||||
func (c *Chat) ReleaseShells(sessionID string) {
|
||||
if sessionID == "" {
|
||||
return
|
||||
}
|
||||
c.shellMu.Lock()
|
||||
delete(c.shells, sessionID)
|
||||
c.shellMu.Unlock()
|
||||
}
|
||||
|
||||
// Stream subscribes to the OpenCode event firehose and invokes emit for every
|
||||
// frame belonging to sessionID. It reads one bounded SSE frame at a time and
|
||||
// never polls the per-session message log; the global firehose is the only
|
||||
@@ -306,6 +416,12 @@ func (c *Chat) Stream(ctx context.Context, id string, ready func(), emit func(pr
|
||||
if !sessionID.MatchString(id) {
|
||||
return fmt.Errorf("invalid OpenCode session id")
|
||||
}
|
||||
// Pin state before opening/announcing the stream so a canceled scanner
|
||||
// cannot acquire a replacement room's identity map.
|
||||
state, err := c.streamStateContext(ctx, id)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
body, err := c.conn.EventStream(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -314,7 +430,7 @@ func (c *Chat) Stream(ctx context.Context, id string, ready func(), emit func(pr
|
||||
if ready != nil {
|
||||
ready()
|
||||
}
|
||||
return scanChatEvents(ctx, body, id, emit)
|
||||
return c.scanChatEventsWithState(ctx, body, id, state, emit)
|
||||
}
|
||||
|
||||
// scanChatEvents turns an SSE body into protocol frames, one bounded frame at a
|
||||
@@ -322,10 +438,13 @@ func (c *Chat) Stream(ctx context.Context, id string, ready func(), emit func(pr
|
||||
// malformed JSON fails the stream instead of producing a malformed frame. The
|
||||
// last assistant message id is retained so a later tool/step event that omits it
|
||||
// still attributes every frame to a stable message.
|
||||
func scanChatEvents(ctx context.Context, body io.Reader, sessionID string, emit func(protocol.ChatEventFrame) error) error {
|
||||
func (c *Chat) scanChatEvents(ctx context.Context, body io.Reader, sessionID string, emit func(protocol.ChatEventFrame) error) error {
|
||||
return c.scanChatEventsWithState(ctx, body, sessionID, c.streamState(sessionID), emit)
|
||||
}
|
||||
|
||||
func (c *Chat) scanChatEventsWithState(ctx context.Context, body io.Reader, sessionID string, state *ChatStreamState, emit func(protocol.ChatEventFrame) error) error {
|
||||
scanner := bufio.NewScanner(body)
|
||||
scanner.Buffer(make([]byte, 0, 64<<10), MaxResponse)
|
||||
state := &ChatStreamState{}
|
||||
for scanner.Scan() {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
@@ -346,7 +465,7 @@ func scanChatEvents(ctx context.Context, body io.Reader, sessionID string, emit
|
||||
continue
|
||||
}
|
||||
if frame.MessageID != "" {
|
||||
state.LastMessageID = frame.MessageID
|
||||
state.setLastMessageID(frame.MessageID)
|
||||
}
|
||||
if err := emit(frame); err != nil {
|
||||
return err
|
||||
@@ -449,8 +568,29 @@ func (i *chatShellInfo) shellSessionID() string {
|
||||
// what attributes the exit to its session: another session's shell is an
|
||||
// unknown id and is ignored.
|
||||
type ChatStreamState struct {
|
||||
mu sync.Mutex
|
||||
LastMessageID string
|
||||
Shells map[string]struct{}
|
||||
readGate chan struct{}
|
||||
readback *ShellSnapshot
|
||||
}
|
||||
|
||||
func (s *ChatStreamState) lastMessageID() string {
|
||||
if s == nil {
|
||||
return ""
|
||||
}
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
return s.LastMessageID
|
||||
}
|
||||
|
||||
func (s *ChatStreamState) setLastMessageID(id string) {
|
||||
if s == nil || id == "" {
|
||||
return
|
||||
}
|
||||
s.mu.Lock()
|
||||
s.LastMessageID = id
|
||||
s.mu.Unlock()
|
||||
}
|
||||
|
||||
// knowsShell reports whether id is a shell this stream has already surfaced for
|
||||
@@ -459,6 +599,8 @@ func (s *ChatStreamState) knowsShell(id string) bool {
|
||||
if s == nil || id == "" {
|
||||
return false
|
||||
}
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
_, ok := s.Shells[id]
|
||||
return ok
|
||||
}
|
||||
@@ -468,6 +610,8 @@ func (s *ChatStreamState) markShell(id string) {
|
||||
if s == nil || id == "" {
|
||||
return
|
||||
}
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if s.Shells == nil {
|
||||
s.Shells = map[string]struct{}{}
|
||||
}
|
||||
@@ -476,12 +620,37 @@ func (s *ChatStreamState) markShell(id string) {
|
||||
|
||||
// forgetShell drops id once its exit has been surfaced.
|
||||
func (s *ChatStreamState) forgetShell(id string) {
|
||||
if s == nil || s.Shells == nil {
|
||||
if s == nil {
|
||||
return
|
||||
}
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if s.Shells == nil {
|
||||
return
|
||||
}
|
||||
delete(s.Shells, id)
|
||||
}
|
||||
|
||||
// replaceShells applies one authoritative running-shell snapshot without
|
||||
// replacing the state object an already attached event scanner owns.
|
||||
func (s *ChatStreamState) replaceShells(ids []string) {
|
||||
if s == nil {
|
||||
return
|
||||
}
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
s.replaceShellsLocked(ids)
|
||||
}
|
||||
|
||||
func (s *ChatStreamState) replaceShellsLocked(ids []string) {
|
||||
s.Shells = make(map[string]struct{}, len(ids))
|
||||
for _, id := range ids {
|
||||
if id != "" {
|
||||
s.Shells[id] = struct{}{}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// NormalizeChatEvent maps one OpenCode firehose `data:` payload to a
|
||||
// protocol.ChatEventFrame, folding the small per-session context ([stream]) it
|
||||
// needs across frames. ok is false for an event type this adapter does not
|
||||
@@ -528,7 +697,7 @@ func NormalizeChatEvent(payload []byte, sessionID string, stream *ChatStreamStat
|
||||
messageID = data.MessageIDAlt
|
||||
}
|
||||
if messageID == "" && stream != nil {
|
||||
messageID = stream.LastMessageID
|
||||
messageID = stream.lastMessageID()
|
||||
}
|
||||
// Every pushed frame must carry a stable message id so the client can
|
||||
// attribute it; without one this is not a chat frame.
|
||||
@@ -590,20 +759,38 @@ func normalizeShellEvent(eventType string, data chatEvent, sessionID string, str
|
||||
if status == "" {
|
||||
status = protocol.ChatShellRunning
|
||||
}
|
||||
stream.markShell(data.Info.ID)
|
||||
shell := &protocol.ChatShell{ID: data.Info.ID, Status: status, Command: data.Info.Command}
|
||||
if stream != nil {
|
||||
stream.mu.Lock()
|
||||
stream.readback.Record(shell)
|
||||
if stream.Shells == nil {
|
||||
stream.Shells = map[string]struct{}{}
|
||||
}
|
||||
if status == protocol.ChatShellRunning && len(stream.Shells) < chatShellIdentityLimit {
|
||||
stream.Shells[shell.ID] = struct{}{}
|
||||
}
|
||||
stream.mu.Unlock()
|
||||
}
|
||||
return protocol.ChatEventFrame{
|
||||
SessionID: sessionID,
|
||||
Kind: protocol.ChatEventShell,
|
||||
Shell: &protocol.ChatShell{ID: data.Info.ID, Status: status, Command: data.Info.Command},
|
||||
Shell: shell,
|
||||
}, true, nil
|
||||
}
|
||||
if data.ID == "" {
|
||||
return protocol.ChatEventFrame{}, false, fmt.Errorf("malformed OpenCode shell.exited event")
|
||||
}
|
||||
if !stream.knowsShell(data.ID) {
|
||||
if stream == nil {
|
||||
return protocol.ChatEventFrame{}, false, nil
|
||||
}
|
||||
stream.mu.Lock()
|
||||
stream.readback.Record(&protocol.ChatShell{ID: data.ID, Status: protocol.ChatShellExited})
|
||||
_, known := stream.Shells[data.ID]
|
||||
delete(stream.Shells, data.ID)
|
||||
stream.mu.Unlock()
|
||||
if !known {
|
||||
return protocol.ChatEventFrame{}, false, nil
|
||||
}
|
||||
stream.forgetShell(data.ID)
|
||||
status := data.Status
|
||||
if status == "" {
|
||||
status = protocol.ChatShellExited
|
||||
|
||||
@@ -0,0 +1,116 @@
|
||||
package opencode
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.a-wan.net/alan/pipelines/internal/protocol"
|
||||
)
|
||||
|
||||
func TestChatShellReadbackBarrier(t *testing.T) {
|
||||
for _, known := range []bool{false, true} {
|
||||
t.Run(fmt.Sprint(known), func(t *testing.T) {
|
||||
read, apply := make(chan struct{}), make(chan struct{})
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
close(read)
|
||||
select {
|
||||
case <-apply:
|
||||
fmt.Fprint(w, `{"data":[{"id":"sh_old","status":"running","metadata":{"sessionID":"ses_1"}},{"id":"sh_foreign","status":"running","metadata":{"sessionID":"ses_other"}}]}`)
|
||||
case <-r.Context().Done():
|
||||
}
|
||||
}))
|
||||
defer srv.Close()
|
||||
c := NewChat(&Connection{Backend: "v2", URL: srv.URL})
|
||||
state := c.streamState("ses_1")
|
||||
if known {
|
||||
state.markShell("sh_old")
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
|
||||
defer cancel()
|
||||
result := make(chan []protocol.ChatShell, 1)
|
||||
errs := make(chan error, 1)
|
||||
go func() { shells, err := c.Shells(ctx, "ses_1"); result <- shells; errs <- err }()
|
||||
select {
|
||||
case <-read:
|
||||
case <-ctx.Done():
|
||||
t.Fatal("read did not reach barrier")
|
||||
}
|
||||
// Both a previously verified exit and an exit first learned while
|
||||
// reading must suppress the stale running snapshot.
|
||||
frame, ok, err := NormalizeChatEvent([]byte(`{"type":"shell.exited","data":{"id":"sh_old","status":"exited","exit":0}}`), "ses_1", state)
|
||||
if err != nil || ok != known || (ok && frame.Shell.Status != protocol.ChatShellExited) {
|
||||
t.Fatalf("exit = %+v %v %v", frame, ok, err)
|
||||
}
|
||||
frame, ok, err = NormalizeChatEvent([]byte(`{"type":"shell.created","data":{"info":{"id":"sh_new","status":"running","command":"sleep 10","metadata":{"sessionID":"ses_1"}}}}`), "ses_1", state)
|
||||
if err != nil || !ok || frame.Shell.ID != "sh_new" {
|
||||
t.Fatalf("create = %+v %v %v", frame, ok, err)
|
||||
}
|
||||
close(apply)
|
||||
got := <-result
|
||||
if err := <-errs; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(got) != 1 || got[0].ID != "sh_new" || got[0].Command != "sleep 10" {
|
||||
t.Fatalf("readback lost events/leaked ownership: %+v", got)
|
||||
}
|
||||
if state.knowsShell("sh_old") || state.knowsShell("sh_foreign") || !state.knowsShell("sh_new") {
|
||||
t.Fatal("adapter identity state regressed")
|
||||
}
|
||||
// User-visible subsequent exit remains attributable after reconciliation.
|
||||
frame, ok, err = NormalizeChatEvent([]byte(`{"type":"shell.exited","data":{"id":"sh_new","status":"exited","exit":0}}`), "ses_1", state)
|
||||
if err != nil || !ok || frame.Shell.ID != "sh_new" {
|
||||
t.Fatalf("new shell exit lost: %+v %v %v", frame, ok, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestChatShellReadbackCancelWaiter(t *testing.T) {
|
||||
read := make(chan struct{})
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { close(read); <-r.Context().Done() }))
|
||||
defer srv.Close()
|
||||
c := NewChat(&Connection{Backend: "v2", URL: srv.URL})
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
result := make(chan error, 2)
|
||||
go func() { _, err := c.Shells(ctx, "ses_1"); result <- err }()
|
||||
select {
|
||||
case <-read:
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("read stuck")
|
||||
}
|
||||
go func() { _, err := c.Shells(ctx, "ses_1"); result <- err }()
|
||||
cancel()
|
||||
for range 2 {
|
||||
select {
|
||||
case err := <-result:
|
||||
if err == nil {
|
||||
t.Fatal("canceled read succeeded")
|
||||
}
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("read or waiter stuck")
|
||||
}
|
||||
}
|
||||
old := c.streamState("ses_1")
|
||||
c.ReleaseShells("ses_1")
|
||||
if c.streamState("ses_1") == old {
|
||||
t.Fatal("reconnect reused released state")
|
||||
}
|
||||
}
|
||||
|
||||
func TestShellSnapshotBounded(t *testing.T) {
|
||||
var snapshot ShellSnapshot
|
||||
for i := 0; i <= chatShellIdentityLimit; i++ {
|
||||
snapshot.Record(&protocol.ChatShell{ID: fmt.Sprintf("sh_%d", i), Status: protocol.ChatShellExited})
|
||||
}
|
||||
if len(snapshot.changes) != chatShellIdentityLimit {
|
||||
t.Fatalf("unbounded journal: %d", len(snapshot.changes))
|
||||
}
|
||||
if _, err := snapshot.Merge([]protocol.ChatShell{{ID: "sh_0", Status: protocol.ChatShellRunning}}); err == nil {
|
||||
t.Fatal("overflow accepted stale snapshot")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,72 @@
|
||||
package opencode
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sort"
|
||||
|
||||
"git.a-wan.net/alan/pipelines/internal/protocol"
|
||||
)
|
||||
|
||||
// ShellSnapshot records changes while an authoritative read is in flight. Its
|
||||
// owner serializes access. Unknown exits may be recorded, but can only remove an
|
||||
// id: they never establish ownership. Overflow rejects the snapshot, not events.
|
||||
type ShellSnapshot struct {
|
||||
changes map[string]protocol.ChatShell
|
||||
bytes int
|
||||
overflow bool
|
||||
}
|
||||
|
||||
const chatShellIdentityLimit = 256
|
||||
|
||||
func (s *ShellSnapshot) Record(shell *protocol.ChatShell) {
|
||||
if s == nil || shell == nil || shell.ID == "" || s.overflow {
|
||||
return
|
||||
}
|
||||
if s.changes == nil {
|
||||
s.changes = make(map[string]protocol.ChatShell)
|
||||
}
|
||||
old := s.changes[shell.ID]
|
||||
n := s.bytes - len(old.ID) - len(old.Command) + len(shell.ID) + len(shell.Command)
|
||||
if n > MaxResponse || (old.ID == "" && len(s.changes) >= chatShellIdentityLimit) {
|
||||
s.overflow = true
|
||||
return
|
||||
}
|
||||
// A terminal event wins even if a duplicate create is delivered later.
|
||||
if old.ID != "" && old.Status != protocol.ChatShellRunning {
|
||||
return
|
||||
}
|
||||
s.changes[shell.ID] = *shell
|
||||
s.bytes = n
|
||||
}
|
||||
|
||||
func (s *ShellSnapshot) Merge(shells []protocol.ChatShell) ([]protocol.ChatShell, error) {
|
||||
if s.overflow {
|
||||
return nil, fmt.Errorf("shell reconciliation capacity exceeded")
|
||||
}
|
||||
byID := make(map[string]protocol.ChatShell)
|
||||
for _, shell := range shells {
|
||||
if shell.ID != "" && shell.Status == protocol.ChatShellRunning {
|
||||
byID[shell.ID] = shell
|
||||
}
|
||||
}
|
||||
for id, shell := range s.changes {
|
||||
if shell.Status == protocol.ChatShellRunning {
|
||||
byID[id] = shell
|
||||
} else {
|
||||
delete(byID, id)
|
||||
}
|
||||
}
|
||||
if len(byID) > chatShellIdentityLimit {
|
||||
return nil, fmt.Errorf("shell reconciliation capacity exceeded")
|
||||
}
|
||||
ids := make([]string, 0, len(byID))
|
||||
for id := range byID {
|
||||
ids = append(ids, id)
|
||||
}
|
||||
sort.Strings(ids)
|
||||
out := make([]protocol.ChatShell, 0, len(ids))
|
||||
for _, id := range ids {
|
||||
out = append(out, byID[id])
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
@@ -395,3 +395,45 @@ func TestOpenChatRejectsUnrunnableCLI(t *testing.T) {
|
||||
t.Fatal("a missing CLI must fail detection, not open an endpoint")
|
||||
}
|
||||
}
|
||||
|
||||
// TestChatShellReadbackSeedsReplacementStream proves an authoritative shell
|
||||
// readback survives a later stream replacement: shell.exited has no session id,
|
||||
// so it must still clear the verified shell after reconnect.
|
||||
func TestChatShellReadbackSeedsReplacementStream(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
switch r.URL.Path {
|
||||
case "/api/shell":
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{"data":[{"id":"sh_1","status":"running","command":"sleep 30","metadata":{"sessionID":"ses_1"}},{"id":"sh_other","status":"running","command":"sleep 30","metadata":{"sessionID":"ses_other"}}]}`))
|
||||
case "/api/event":
|
||||
w.Header().Set("Content-Type", "text/event-stream")
|
||||
_, _ = w.Write([]byte("data: {\"type\":\"shell.exited\",\"data\":{\"id\":\"sh_1\",\"status\":\"exited\",\"exit\":0}}\n\n"))
|
||||
}
|
||||
}))
|
||||
defer srv.Close()
|
||||
chat := NewChat(&Connection{Backend: "v2", URL: srv.URL})
|
||||
shells, err := chat.Shells(context.Background(), "ses_1")
|
||||
if err != nil || len(shells) != 1 || shells[0].ID != "sh_1" {
|
||||
t.Fatalf("Shells = %+v, %v", shells, err)
|
||||
}
|
||||
var got []protocol.ChatEventFrame
|
||||
if err := chat.Stream(context.Background(), "ses_1", nil, func(frame protocol.ChatEventFrame) error { got = append(got, frame); return nil }); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(got) != 1 || got[0].Shell == nil || got[0].Shell.ID != "sh_1" || got[0].Shell.Status != protocol.ChatShellExited {
|
||||
t.Fatalf("exit after readback = %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestChatReleaseShellsBoundsDisconnectedState(t *testing.T) {
|
||||
chat := NewChat(&Connection{})
|
||||
state := chat.streamState("ses_1")
|
||||
state.markShell("sh_1")
|
||||
chat.ReleaseShells("ses_1")
|
||||
chat.shellMu.Lock()
|
||||
_, retained := chat.shells["ses_1"]
|
||||
chat.shellMu.Unlock()
|
||||
if retained {
|
||||
t.Fatal("released session shell state was retained")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -10,7 +10,7 @@ import (
|
||||
)
|
||||
|
||||
func decodeControl(text string, value any) error {
|
||||
// Both control blocks are flat objects. Reject duplicate keys before decoding.
|
||||
// Control blocks are flat objects. Reject duplicate keys before decoding.
|
||||
keys := json.NewDecoder(strings.NewReader(text))
|
||||
if token, err := keys.Token(); err != nil || token != json.Delim('{') {
|
||||
return fmt.Errorf("expected object")
|
||||
@@ -44,6 +44,12 @@ func decodeControl(text string, value any) error {
|
||||
}
|
||||
|
||||
func parseControl(s *Stage, fields map[string]string) error {
|
||||
if text, ok := fields["resources"]; ok {
|
||||
if err := decodeControl(text, &s.Resources); err != nil || s.Resources == nil || !ValidResourceKey(s.Resources.Exclusive) {
|
||||
return fmt.Errorf("resources must contain exactly one exclusive key (1–128 ASCII letters, digits, _ or -; start with a letter)")
|
||||
}
|
||||
delete(fields, "resources")
|
||||
}
|
||||
if text, ok := fields["limits"]; ok {
|
||||
if err := decodeControl(text, &s.Limits); err != nil || s.Limits == nil {
|
||||
return fmt.Errorf("limits must be an object with max_visits and optional on outcomes")
|
||||
@@ -76,6 +82,9 @@ func parseControl(s *Stage, fields map[string]string) error {
|
||||
}
|
||||
|
||||
func validateControl(s Stage, stages map[string]bool) error {
|
||||
if s.Resources != nil && s.Type == "human" {
|
||||
return fmt.Errorf("stage %s: human stages cannot declare resources", s.Key)
|
||||
}
|
||||
if s.Limits != nil {
|
||||
for _, outcome := range s.Limits.On {
|
||||
target, exists := s.Edges[outcome]
|
||||
@@ -95,3 +104,7 @@ func validateControl(s Stage, stages map[string]bool) error {
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func ValidResourceKey(key string) bool {
|
||||
return len(key) <= 128 && identifier.MatchString(key)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,277 @@
|
||||
package pipeline
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"database/sql"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"git.a-wan.net/alan/pipelines/internal/protocol"
|
||||
)
|
||||
|
||||
// ResourceVisit identifies a pending visit, before it consumes execution accounting.
|
||||
type ResourceVisit struct {
|
||||
Resource string
|
||||
RunID int64
|
||||
Stage string
|
||||
Sequence int
|
||||
}
|
||||
|
||||
// ResourceLease is an opaque capability. Its credential is deliberately not exported
|
||||
// or serializable, and must never be put in model input or public wire metadata.
|
||||
type ResourceLease struct {
|
||||
ResourceVisit
|
||||
token string
|
||||
}
|
||||
|
||||
// RequestResource acquires immediately or registers one FIFO waiter/channel. The
|
||||
// mutex covers only short DB transactions and channel bookkeeping, never execution.
|
||||
func (s *Store) RequestResource(ctx context.Context, visit ResourceVisit) (ResourceLease, <-chan ResourceLease, *protocol.ResourceWait, error) {
|
||||
s.resourceMu.Lock()
|
||||
defer s.resourceMu.Unlock()
|
||||
if !ValidResourceKey(visit.Resource) || visit.RunID <= 0 || visit.Sequence <= 0 || !identifier.MatchString(visit.Stage) {
|
||||
return ResourceLease{}, nil, nil, fmt.Errorf("invalid resource visit")
|
||||
}
|
||||
if ch := s.resourceWaiters[visit]; ch != nil {
|
||||
wait, err := s.resourceWait(ctx, visit.RunID)
|
||||
return ResourceLease{}, ch, wait, err
|
||||
}
|
||||
credential := make([]byte, 32)
|
||||
if _, err := rand.Read(credential); err != nil {
|
||||
return ResourceLease{}, nil, nil, err
|
||||
}
|
||||
lease := ResourceLease{ResourceVisit: visit, token: hex.EncodeToString(credential)}
|
||||
tx, err := s.db.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
return ResourceLease{}, nil, nil, err
|
||||
}
|
||||
defer tx.Rollback()
|
||||
var status string
|
||||
if err := tx.QueryRowContext(ctx, "SELECT status FROM pipeline_runs WHERE id=?", visit.RunID).Scan(&status); err != nil {
|
||||
return ResourceLease{}, nil, nil, err
|
||||
}
|
||||
if status != protocol.RunStatusQueued && status != protocol.RunStatusRunning {
|
||||
return ResourceLease{}, nil, nil, fmt.Errorf("run is not active")
|
||||
}
|
||||
var existing ResourceLease
|
||||
err = tx.QueryRowContext(ctx, "SELECT resource_key,run_id,stage_key,sequence,token FROM resource_owners WHERE run_id=?", visit.RunID).Scan(&existing.Resource, &existing.RunID, &existing.Stage, &existing.Sequence, &existing.token)
|
||||
if err == nil {
|
||||
if existing.ResourceVisit != visit {
|
||||
return ResourceLease{}, nil, nil, fmt.Errorf("run already owns another resource visit")
|
||||
}
|
||||
return existing, nil, nil, nil
|
||||
}
|
||||
if err != sql.ErrNoRows {
|
||||
return ResourceLease{}, nil, nil, err
|
||||
}
|
||||
result, err := tx.ExecContext(ctx, `INSERT INTO resource_owners(resource_key,run_id,stage_key,sequence,token)
|
||||
VALUES(?,?,?,?,?) ON CONFLICT(resource_key) DO NOTHING`, visit.Resource, visit.RunID, visit.Stage, visit.Sequence, lease.token)
|
||||
if err != nil {
|
||||
return ResourceLease{}, nil, nil, err
|
||||
}
|
||||
n, err := result.RowsAffected()
|
||||
if err != nil {
|
||||
return ResourceLease{}, nil, nil, err
|
||||
}
|
||||
if n == 1 {
|
||||
return lease, nil, nil, tx.Commit()
|
||||
}
|
||||
wait := &protocol.ResourceWait{Resource: visit.Resource, Stage: visit.Stage, Sequence: visit.Sequence, QueuedSince: time.Now().UTC().Format(time.RFC3339Nano)}
|
||||
encoded, err := json.Marshal(wait)
|
||||
if err != nil {
|
||||
return ResourceLease{}, nil, nil, err
|
||||
}
|
||||
if _, err := tx.ExecContext(ctx, `INSERT INTO resource_waiters(resource_key,run_id,stage_key,sequence,token,queued_at) VALUES(?,?,?,?,?,?)`, visit.Resource, visit.RunID, visit.Stage, visit.Sequence, lease.token, wait.QueuedSince); err != nil {
|
||||
return ResourceLease{}, nil, nil, err
|
||||
}
|
||||
if err := affected(tx.ExecContext(ctx, "UPDATE pipeline_runs SET status='queued',current_stage=?,resource_wait=? WHERE id=? AND status IN ('queued','running')", visit.Stage, string(encoded), visit.RunID)); err != nil {
|
||||
return ResourceLease{}, nil, nil, err
|
||||
}
|
||||
if err := tx.Commit(); err != nil {
|
||||
return ResourceLease{}, nil, nil, err
|
||||
}
|
||||
if s.resourceWaiters == nil {
|
||||
s.resourceWaiters = make(map[ResourceVisit]chan ResourceLease)
|
||||
}
|
||||
ch := make(chan ResourceLease, 1)
|
||||
s.resourceWaiters[visit] = ch
|
||||
return ResourceLease{}, ch, wait, nil
|
||||
}
|
||||
|
||||
func (s *Store) resourceWait(ctx context.Context, id int64) (*protocol.ResourceWait, error) {
|
||||
var wait protocol.ResourceWait
|
||||
err := s.db.QueryRowContext(ctx, "SELECT resource_key,stage_key,sequence,queued_at FROM resource_waiters WHERE run_id=?", id).Scan(&wait.Resource, &wait.Stage, &wait.Sequence, &wait.QueuedSince)
|
||||
if err == sql.ErrNoRows {
|
||||
return nil, nil
|
||||
}
|
||||
return &wait, err
|
||||
}
|
||||
|
||||
// CancelResource removes a queued visit, or returns a concurrently reserved grant
|
||||
// to the queue before any execution is allowed. It cannot cancel another visit.
|
||||
func (s *Store) CancelResource(ctx context.Context, visit ResourceVisit) error {
|
||||
s.resourceMu.Lock()
|
||||
defer s.resourceMu.Unlock()
|
||||
defer delete(s.resourceWaiters, visit)
|
||||
tx, err := s.db.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer tx.Rollback()
|
||||
deleted, err := tx.ExecContext(ctx, "DELETE FROM resource_waiters WHERE resource_key=? AND run_id=? AND stage_key=? AND sequence=?", visit.Resource, visit.RunID, visit.Stage, visit.Sequence)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
n, err := deleted.RowsAffected()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if n > 0 {
|
||||
if _, err := tx.ExecContext(ctx, "UPDATE pipeline_runs SET resource_wait='' WHERE id=?", visit.RunID); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
ch := s.resourceWaiters[visit]
|
||||
// A grant is buffered while this registry entry exists. Cancellation can
|
||||
// reclaim only that not-yet-accepted capability, never an executing owner.
|
||||
var grant ResourceLease
|
||||
if ch != nil {
|
||||
select {
|
||||
case grant = <-ch:
|
||||
default:
|
||||
}
|
||||
}
|
||||
if err := tx.Commit(); err != nil {
|
||||
return err
|
||||
}
|
||||
if grant.token != "" {
|
||||
if err := s.SetResourceCleanup(ctx, grant, true); err != nil {
|
||||
return err
|
||||
}
|
||||
return s.releaseResourceLocked(ctx, grant)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// AcceptResource removes the channel exactly once. Cancellation is rechecked by
|
||||
// the runner before BeginStep; a canceled accepted grant is released as unused.
|
||||
func (s *Store) AcceptResource(lease ResourceLease) {
|
||||
s.resourceMu.Lock()
|
||||
delete(s.resourceWaiters, lease.ResourceVisit)
|
||||
s.resourceMu.Unlock()
|
||||
}
|
||||
|
||||
// HasResourceOwner is used only for orphan/restart reconciliation, never for a
|
||||
// resource-free stage visit.
|
||||
func (s *Store) HasResourceOwner(ctx context.Context, runID int64) (bool, error) {
|
||||
var exists bool
|
||||
err := s.db.QueryRowContext(ctx, "SELECT EXISTS(SELECT 1 FROM resource_owners WHERE run_id=?)", runID).Scan(&exists)
|
||||
return exists, err
|
||||
}
|
||||
|
||||
func (s *Store) ConfirmRecoveredSession(ctx context.Context, owner SessionOwner) error {
|
||||
_, err := s.db.ExecContext(ctx, "UPDATE run_steps SET agent_active=0 WHERE run_id=? AND session_id=?", owner.RunID, owner.ID)
|
||||
return err
|
||||
}
|
||||
|
||||
// SetResourceCleanup records positive quiescence durably before release. False
|
||||
// quarantines the owner; neither age nor another run can remove the lease.
|
||||
func (s *Store) SetResourceCleanup(ctx context.Context, lease ResourceLease, confirmed bool) error {
|
||||
diagnostic := ""
|
||||
if !confirmed {
|
||||
diagnostic = "Cleanup/restoration unconfirmed; resource quarantined pending inspected reconciliation"
|
||||
}
|
||||
return affected(s.db.ExecContext(ctx, `UPDATE resource_owners SET cleanup_confirmed=?,diagnostic=? WHERE resource_key=? AND run_id=? AND stage_key=? AND sequence=? AND token=?`, confirmed, diagnostic, lease.Resource, lease.RunID, lease.Stage, lease.Sequence, lease.token))
|
||||
}
|
||||
|
||||
// ReleaseResource requires the complete capability AND positive cleanup evidence.
|
||||
// Mismatches and repeated releases are harmless, including after transfer.
|
||||
func (s *Store) ReleaseResource(ctx context.Context, lease ResourceLease) error {
|
||||
s.resourceMu.Lock()
|
||||
defer s.resourceMu.Unlock()
|
||||
return s.releaseResourceLocked(ctx, lease)
|
||||
}
|
||||
|
||||
func (s *Store) releaseResourceLocked(ctx context.Context, lease ResourceLease) error {
|
||||
tx, err := s.db.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer tx.Rollback()
|
||||
n, err := deleteResourceOwner(ctx, tx, lease)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if n == 0 {
|
||||
return nil
|
||||
}
|
||||
var next ResourceLease
|
||||
next.Resource = lease.Resource
|
||||
err = tx.QueryRowContext(ctx, `SELECT run_id,stage_key,sequence,token FROM resource_waiters WHERE resource_key=? ORDER BY queue_order LIMIT 1`, lease.Resource).Scan(&next.RunID, &next.Stage, &next.Sequence, &next.token)
|
||||
if err != nil && err != sql.ErrNoRows {
|
||||
return err
|
||||
}
|
||||
if err == nil {
|
||||
if s.resourceWaiters[next.ResourceVisit] == nil {
|
||||
return fmt.Errorf("resource waiter has no live runner; refusing handoff")
|
||||
}
|
||||
if _, err := tx.ExecContext(ctx, `INSERT INTO resource_owners(resource_key,run_id,stage_key,sequence,token) VALUES(?,?,?,?,?)`, next.Resource, next.RunID, next.Stage, next.Sequence, next.token); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := tx.ExecContext(ctx, "DELETE FROM resource_waiters WHERE run_id=?", next.RunID); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := tx.ExecContext(ctx, "UPDATE pipeline_runs SET resource_wait='' WHERE id=?", next.RunID); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if err := tx.Commit(); err != nil {
|
||||
return err
|
||||
}
|
||||
if next.token != "" {
|
||||
s.resourceWaiters[next.ResourceVisit] <- next
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func deleteResourceOwner(ctx context.Context, tx *sql.Tx, lease ResourceLease) (int64, error) {
|
||||
result, err := tx.ExecContext(ctx, `DELETE FROM resource_owners WHERE resource_key=? AND run_id=? AND stage_key=? AND sequence=? AND token=? AND cleanup_confirmed=1`, lease.Resource, lease.RunID, lease.Stage, lease.Sequence, lease.token)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return result.RowsAffected()
|
||||
}
|
||||
|
||||
// recoverResourceOwners runs only at startup, after pending waiters have been
|
||||
// interrupted and removed. Every reclamation uses the persisted capability;
|
||||
// absent cleanup evidence is never replaced with a timestamp-based expiry.
|
||||
func recoverResourceOwners(ctx context.Context, tx *sql.Tx) error {
|
||||
rows, err := tx.QueryContext(ctx, "SELECT resource_key,run_id,stage_key,sequence,token FROM resource_owners WHERE cleanup_confirmed=1")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var leases []ResourceLease
|
||||
for rows.Next() {
|
||||
var lease ResourceLease
|
||||
if err := rows.Scan(&lease.Resource, &lease.RunID, &lease.Stage, &lease.Sequence, &lease.token); err != nil {
|
||||
rows.Close()
|
||||
return err
|
||||
}
|
||||
leases = append(leases, lease)
|
||||
}
|
||||
err = rows.Err()
|
||||
rows.Close()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, lease := range leases {
|
||||
if _, err := deleteResourceOwner(ctx, tx, lease); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
_, err = tx.ExecContext(ctx, "UPDATE resource_owners SET diagnostic='Server stopped; resource quarantined pending inspected cleanup/restoration'")
|
||||
return err
|
||||
}
|
||||
@@ -0,0 +1,288 @@
|
||||
package pipeline
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestResourceDeclaration(t *testing.T) {
|
||||
base := Template([]string{"work"}, true)
|
||||
for _, value := range []string{`{"exclusive":"phone-X"}`, `{ "exclusive" : "A_1" }`} {
|
||||
graph, err := Parse(base + "```resources\n" + value + "\n```\n")
|
||||
if err != nil || graph.Stages[0].Resources == nil {
|
||||
t.Fatalf("%s: %v", value, err)
|
||||
}
|
||||
if _, ok := StageDefinition(graph.Stages[0]).Blocks["resources"]; ok {
|
||||
t.Fatal("runner control reached executor")
|
||||
}
|
||||
encoded, _ := json.Marshal(graph)
|
||||
if !strings.Contains(string(encoded), `"resources"`) {
|
||||
t.Fatal("resource lost in persisted graph")
|
||||
}
|
||||
}
|
||||
for _, value := range []string{`{}`, `null`, `[]`, `{"exclusive":""}`, `{"exclusive":"a b"}`, `{"exclusive":"a","exclusive":"b"}`, `{"exclusive":"a","extra":1}`, `{"exclusive":["a","b"]}`, fmt.Sprintf(`{"exclusive":%q}`, strings.Repeat("a", 129))} {
|
||||
if _, err := Parse(base + "```resources\n" + value + "\n```\n"); err == nil {
|
||||
t.Fatalf("accepted %s", value)
|
||||
}
|
||||
}
|
||||
human := "# stage gate\n```prompt\nChoose\n```\n```configs\ntype=human\n```\n```edges\ngo -> #success\n```\n```resources\n{\"exclusive\":\"phone\"}\n```\n"
|
||||
if _, err := Parse(human); err == nil {
|
||||
t.Fatal("human acquired a resource")
|
||||
}
|
||||
}
|
||||
|
||||
func resourceStore(t *testing.T) (*Store, func(string) ResourceVisit) {
|
||||
t.Helper()
|
||||
ctx := context.Background()
|
||||
s, err := Open(filepath.Join(t.TempDir(), "resources.db"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { s.Close() })
|
||||
if _, err := s.Submit(ctx, "resource", Template([]string{"work"}, true)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
items, _ := s.List(ctx)
|
||||
d, _ := s.Get(ctx, items[0].ID, 1)
|
||||
count := 0
|
||||
return s, func(resource string) ResourceVisit {
|
||||
count++
|
||||
id, _, err := s.CreateRun(ctx, d, fmt.Sprint(count), "workspace", false, nil, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return ResourceVisit{Resource: resource, RunID: id, Stage: "work", Sequence: 1}
|
||||
}
|
||||
}
|
||||
|
||||
func acquireTestResource(t *testing.T, s *Store, visit ResourceVisit) ResourceLease {
|
||||
t.Helper()
|
||||
l, ch, _, err := s.RequestResource(context.Background(), visit)
|
||||
if err != nil || ch != nil {
|
||||
t.Fatalf("acquire: %v %v", ch, err)
|
||||
}
|
||||
return l
|
||||
}
|
||||
|
||||
func releaseTestResource(t *testing.T, s *Store, lease ResourceLease) {
|
||||
t.Helper()
|
||||
if err := s.SetResourceCleanup(context.Background(), lease, true); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := s.ReleaseResource(context.Background(), lease); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResourceFIFOOwnershipCancellationAndRegistry(t *testing.T) {
|
||||
s, create := resourceStore(t)
|
||||
ctx := context.Background()
|
||||
a := acquireTestResource(t, s, create("X"))
|
||||
duplicate := acquireTestResource(t, s, a.ResourceVisit)
|
||||
if duplicate != a {
|
||||
t.Fatal("duplicate acquisition changed capability")
|
||||
}
|
||||
var visits []ResourceVisit
|
||||
var channels []<-chan ResourceLease
|
||||
for i := 0; i < 4; i++ {
|
||||
v := create("X")
|
||||
_, ch, wait, err := s.RequestResource(ctx, v)
|
||||
if err != nil || ch == nil || wait == nil {
|
||||
t.Fatal("missing waiter", err)
|
||||
}
|
||||
_, again, _, err := s.RequestResource(ctx, v)
|
||||
if err != nil || again != ch {
|
||||
t.Fatal("duplicate waiter registered", err)
|
||||
}
|
||||
visits = append(visits, v)
|
||||
channels = append(channels, ch)
|
||||
}
|
||||
if err := s.ReleaseResource(ctx, a); err != nil {
|
||||
t.Fatal(err)
|
||||
} // no cleanup receipt
|
||||
for _, ch := range channels {
|
||||
select {
|
||||
case <-ch:
|
||||
t.Fatal("unconfirmed release granted")
|
||||
default:
|
||||
}
|
||||
}
|
||||
if err := s.SetResourceCleanup(ctx, a, true); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, corrupt := range []func(*ResourceLease){func(l *ResourceLease) { l.RunID++ }, func(l *ResourceLease) { l.Stage = "other" }, func(l *ResourceLease) { l.Sequence++ }, func(l *ResourceLease) { l.token = "wrong" }} {
|
||||
bad := a
|
||||
corrupt(&bad)
|
||||
if err := s.ReleaseResource(ctx, bad); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
for _, ch := range channels {
|
||||
select {
|
||||
case <-ch:
|
||||
t.Fatal("unconfirmed/mismatched release granted")
|
||||
default:
|
||||
}
|
||||
}
|
||||
if err := s.CancelResource(ctx, visits[1]); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
releaseTestResource(t, s, a)
|
||||
for _, i := range []int{0, 2, 3} {
|
||||
var grant ResourceLease
|
||||
select {
|
||||
case grant = <-channels[i]:
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("FIFO grant not delivered")
|
||||
}
|
||||
s.AcceptResource(grant)
|
||||
if grant.ResourceVisit != visits[i] {
|
||||
t.Fatal("wrong FIFO owner")
|
||||
}
|
||||
// An old owner's repeated release cannot release the next owner.
|
||||
if err := s.ReleaseResource(ctx, a); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, j := range []int{2, 3} {
|
||||
if j > i {
|
||||
select {
|
||||
case <-channels[j]:
|
||||
t.Fatal("multiple grants")
|
||||
default:
|
||||
}
|
||||
}
|
||||
}
|
||||
releaseTestResource(t, s, grant)
|
||||
}
|
||||
if len(s.resourceWaiters) != 0 {
|
||||
t.Fatal("channel registry leaked")
|
||||
}
|
||||
for i := 0; i < 30; i++ {
|
||||
owner := acquireTestResource(t, s, create("X"))
|
||||
v := create("X")
|
||||
_, _, _, err := s.RequestResource(ctx, v)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
releaseTestResource(t, s, owner) // race window: granted but not accepted
|
||||
if err := s.CancelResource(ctx, v); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(s.resourceWaiters) != 0 {
|
||||
t.Fatal("grant/cancel leaked channel")
|
||||
}
|
||||
}
|
||||
for i := 0; i < 30; i++ {
|
||||
owner := acquireTestResource(t, s, create("X"))
|
||||
v := create("X")
|
||||
if _, _, _, err := s.RequestResource(ctx, v); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := s.SetResourceCleanup(ctx, owner, true); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var wg sync.WaitGroup
|
||||
wg.Add(2)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
if err := s.ReleaseResource(ctx, owner); err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
}()
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
if err := s.CancelResource(ctx, v); err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
}()
|
||||
wg.Wait()
|
||||
if len(s.resourceWaiters) != 0 {
|
||||
t.Fatal("concurrent grant/cancel leaked channel")
|
||||
}
|
||||
}
|
||||
var owners, waiters int
|
||||
s.db.QueryRow("SELECT COUNT(*) FROM resource_owners").Scan(&owners)
|
||||
s.db.QueryRow("SELECT COUNT(*) FROM resource_waiters").Scan(&waiters)
|
||||
if owners != 0 || waiters != 0 {
|
||||
t.Fatalf("stranded owners/waiters: %d/%d", owners, waiters)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResourceCancellationPersistenceFailureRemovesChannel(t *testing.T) {
|
||||
s, create := resourceStore(t)
|
||||
acquireTestResource(t, s, create("X"))
|
||||
v := create("X")
|
||||
if _, _, _, err := s.RequestResource(context.Background(), v); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := s.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := s.CancelResource(context.Background(), v); err == nil {
|
||||
t.Fatal("closed DB cancellation claimed success")
|
||||
}
|
||||
if len(s.resourceWaiters) != 0 {
|
||||
t.Fatal("persistence failure leaked channel")
|
||||
}
|
||||
}
|
||||
|
||||
func TestResourceRestartQuarantineAndRetention(t *testing.T) {
|
||||
for _, confirmed := range []bool{false, true} {
|
||||
t.Run(fmt.Sprint(confirmed), func(t *testing.T) {
|
||||
s, create := resourceStore(t)
|
||||
ctx := context.Background()
|
||||
a := acquireTestResource(t, s, create("X"))
|
||||
b := create("X")
|
||||
if _, _, _, err := s.RequestResource(ctx, b); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if confirmed {
|
||||
if err := s.SetResourceCleanup(ctx, a, true); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
path := s.path
|
||||
if err := s.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
reopened, err := Open(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer reopened.Close()
|
||||
if err := reopened.RecoverRuns(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := reopened.RecoverRuns(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
detail, err := reopened.GetRun(ctx, b.RunID)
|
||||
if err != nil || detail.Status != "interrupted" || detail.ResourceWait != nil || len(detail.Steps) != 0 || !strings.Contains(detail.Diagnostic, "resource") {
|
||||
t.Fatal("queued visit replayed/lost diagnostic", detail, err)
|
||||
}
|
||||
exists, err := reopened.HasResourceOwner(ctx, a.RunID)
|
||||
if err != nil || exists == confirmed {
|
||||
t.Fatal("incorrect recovery disposition", exists, err)
|
||||
}
|
||||
report, err := reopened.RetentionReport(ctx, RetentionPolicy{Keep: 0}, "", time.Now())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
protected := false
|
||||
for _, p := range report.Protected {
|
||||
if p.ID == a.RunID {
|
||||
protected = true
|
||||
}
|
||||
}
|
||||
if protected == confirmed {
|
||||
t.Fatal("quarantined evidence not protected")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -130,11 +130,12 @@ type retentionQuerier interface {
|
||||
// retentionRunInfo is one pipeline_runs row plus the step aggregates protection
|
||||
// needs, read in the same statement so the report never issues a per-run query.
|
||||
type retentionRunInfo struct {
|
||||
id int64
|
||||
status string
|
||||
createdAt time.Time
|
||||
waitingSteps int
|
||||
activeSteps int
|
||||
id int64
|
||||
status string
|
||||
createdAt time.Time
|
||||
waitingSteps int
|
||||
activeSteps int
|
||||
resourceOwners int
|
||||
}
|
||||
|
||||
// protectionReasons lists why a run must not be removed for a reason intrinsic
|
||||
@@ -155,6 +156,9 @@ func (r retentionRunInfo) protectionReasons() []string {
|
||||
if r.activeSteps > 0 {
|
||||
reasons = append(reasons, "cleanup agent still active (agent_active=1)")
|
||||
}
|
||||
if r.resourceOwners > 0 {
|
||||
reasons = append(reasons, "resource ownership retained (possibly quarantined)")
|
||||
}
|
||||
return reasons
|
||||
}
|
||||
|
||||
@@ -171,9 +175,20 @@ type retentionSelection struct {
|
||||
// report is a bounded number of round trips rather than one query per run.
|
||||
func selectRetentionRuns(ctx context.Context, q retentionQuerier, policy RetentionPolicy, now time.Time) (retentionSelection, error) {
|
||||
var sel retentionSelection
|
||||
// Maintenance is deliberately non-migrating and may inspect a database
|
||||
// whose server predates resource support.
|
||||
var resources bool
|
||||
if err := q.QueryRowContext(ctx, "SELECT EXISTS(SELECT 1 FROM sqlite_master WHERE type='table' AND name='resource_owners')").Scan(&resources); err != nil {
|
||||
return sel, err
|
||||
}
|
||||
owners := "0"
|
||||
if resources {
|
||||
owners = "(SELECT COUNT(*) FROM resource_owners o WHERE o.run_id=r.id)"
|
||||
}
|
||||
rows, err := q.QueryContext(ctx, `SELECT r.id,r.status,strftime('%Y-%m-%dT%H:%M:%SZ',r.created_at),
|
||||
COALESCE((SELECT COUNT(*) FROM run_steps s WHERE s.run_id=r.id AND s.status=?),0),
|
||||
COALESCE((SELECT COUNT(*) FROM run_steps s WHERE s.run_id=r.id AND s.agent_active=1),0) `+
|
||||
COALESCE((SELECT COUNT(*) FROM run_steps s WHERE s.run_id=r.id AND s.agent_active=1),0),
|
||||
`+owners+` `+
|
||||
runFrom+` ORDER BY r.id DESC`, protocol.StepStatusWaiting)
|
||||
if err != nil {
|
||||
return sel, err
|
||||
@@ -182,7 +197,7 @@ COALESCE((SELECT COUNT(*) FROM run_steps s WHERE s.run_id=r.id AND s.agent_activ
|
||||
for rows.Next() {
|
||||
var run retentionRunInfo
|
||||
var created string
|
||||
if err := rows.Scan(&run.id, &run.status, &created, &run.waitingSteps, &run.activeSteps); err != nil {
|
||||
if err := rows.Scan(&run.id, &run.status, &created, &run.waitingSteps, &run.activeSteps, &run.resourceOwners); err != nil {
|
||||
return sel, err
|
||||
}
|
||||
run.createdAt = parseRetentionTime(created)
|
||||
|
||||
@@ -21,14 +21,19 @@ const runStamps = `strftime('%Y-%m-%dT%H:%M:%SZ',r.created_at),COALESCE(strftime
|
||||
const runSelect = `SELECT r.id,r.pipeline_id,p.name,r.version,v.content_hash,r.status,r.outcome,r.current_stage,
|
||||
` + runStamps + `,r.diagnostic,
|
||||
CASE WHEN r.status='succeeded' THEN r.output_json ELSE COALESCE((SELECT output_json FROM run_steps WHERE run_id=r.id AND status='completed' ORDER BY sequence DESC LIMIT 1),'{}') END,
|
||||
(SELECT COUNT(*) FROM run_steps WHERE run_id=r.id AND status='completed'),r.workspace,r.title,r.model,r.stage_models
|
||||
(SELECT COUNT(*) FROM run_steps WHERE run_id=r.id AND status='completed'),r.workspace,r.title,r.model,r.stage_models,r.resource_wait
|
||||
` + runFrom + ` JOIN pipeline_versions v ON v.pipeline_id=r.pipeline_id AND v.version=r.version`
|
||||
|
||||
func scanRun(row interface{ Scan(...any) error }, r *protocol.RunSummary) error {
|
||||
var output, stageModels string
|
||||
if err := row.Scan(&r.ID, &r.PipelineID, &r.Name, &r.Version, &r.ContentHash, &r.Status, &r.Outcome, &r.CurrentStage, &r.CreatedAt, &r.FinishedAt, &r.Diagnostic, &output, &r.CompletedSteps, &r.Workspace, &r.Title, &r.Model, &stageModels); err != nil {
|
||||
var output, stageModels, resourceWait string
|
||||
if err := row.Scan(&r.ID, &r.PipelineID, &r.Name, &r.Version, &r.ContentHash, &r.Status, &r.Outcome, &r.CurrentStage, &r.CreatedAt, &r.FinishedAt, &r.Diagnostic, &output, &r.CompletedSteps, &r.Workspace, &r.Title, &r.Model, &stageModels, &resourceWait); err != nil {
|
||||
return err
|
||||
}
|
||||
if resourceWait != "" {
|
||||
if err := json.Unmarshal([]byte(resourceWait), &r.ResourceWait); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if stageModels != "" && stageModels != "{}" {
|
||||
if err := json.Unmarshal([]byte(stageModels), &r.StageModels); err != nil {
|
||||
return err
|
||||
|
||||
@@ -44,7 +44,7 @@ func (s *Store) BeginStep(ctx context.Context, id int64, sequence int, key strin
|
||||
return err
|
||||
}
|
||||
defer tx.Rollback()
|
||||
if err := affected(tx.ExecContext(ctx, "UPDATE pipeline_runs SET status='running',current_stage=? WHERE id=? AND status IN ('queued','running')", key, id)); err != nil {
|
||||
if err := affected(tx.ExecContext(ctx, "UPDATE pipeline_runs SET status='running',current_stage=?,resource_wait='' WHERE id=? AND status IN ('queued','running')", key, id)); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := tx.ExecContext(ctx, "INSERT INTO run_steps(run_id,sequence,stage_key,input_json) VALUES (?,?,?,?)", id, sequence, key, string(encoded)); err != nil {
|
||||
@@ -310,5 +310,5 @@ func (s *Store) FinishRun(ctx context.Context, id int64, status, outcome, diagno
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return affected(s.db.ExecContext(ctx, "UPDATE pipeline_runs SET status=?,outcome=?,diagnostic=?,output_json=?,finished_at=CURRENT_TIMESTAMP WHERE id=? AND status IN ('queued','running')", status, outcome, diagnostic, string(data), id))
|
||||
return affected(s.db.ExecContext(ctx, "UPDATE pipeline_runs SET status=?,outcome=?,diagnostic=?,output_json=?,finished_at=CURRENT_TIMESTAMP,resource_wait='' WHERE id=? AND status IN ('queued','running')", status, outcome, diagnostic, string(data), id))
|
||||
}
|
||||
|
||||
@@ -51,13 +51,28 @@ CREATE TABLE IF NOT EXISTS run_session_owners (
|
||||
run_id INTEGER NOT NULL, sequence INTEGER NOT NULL, backend TEXT NOT NULL, reference TEXT NOT NULL DEFAULT '',
|
||||
PRIMARY KEY(run_id,sequence), FOREIGN KEY(run_id,sequence) REFERENCES run_steps(run_id,sequence));
|
||||
CREATE TABLE IF NOT EXISTS run_retries (
|
||||
run_id INTEGER PRIMARY KEY REFERENCES pipeline_runs(id), original_run_id INTEGER NOT NULL REFERENCES pipeline_runs(id));`)
|
||||
run_id INTEGER PRIMARY KEY REFERENCES pipeline_runs(id), original_run_id INTEGER NOT NULL REFERENCES pipeline_runs(id));
|
||||
CREATE TABLE IF NOT EXISTS resource_owners (
|
||||
resource_key TEXT PRIMARY KEY, run_id INTEGER NOT NULL REFERENCES pipeline_runs(id),
|
||||
stage_key TEXT NOT NULL, sequence INTEGER NOT NULL, token TEXT NOT NULL,
|
||||
cleanup_confirmed INTEGER NOT NULL DEFAULT 0, diagnostic TEXT NOT NULL DEFAULT '',
|
||||
acquired_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP);
|
||||
CREATE INDEX IF NOT EXISTS resource_owner_run ON resource_owners(run_id);
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS resource_owner_visit ON resource_owners(run_id);
|
||||
CREATE TABLE IF NOT EXISTS resource_waiters (
|
||||
queue_order INTEGER PRIMARY KEY AUTOINCREMENT, resource_key TEXT NOT NULL,
|
||||
run_id INTEGER NOT NULL UNIQUE REFERENCES pipeline_runs(id), stage_key TEXT NOT NULL,
|
||||
sequence INTEGER NOT NULL, token TEXT NOT NULL, queued_at TEXT NOT NULL);
|
||||
CREATE INDEX IF NOT EXISTS resource_wait_fifo ON resource_waiters(resource_key,queue_order);`)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := s.migrateRunTitles(); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := s.addColumnIfMissing("pipeline_runs", "resource_wait", "TEXT NOT NULL DEFAULT ''"); err != nil {
|
||||
return err
|
||||
}
|
||||
return s.migrateRunModels()
|
||||
}
|
||||
|
||||
@@ -122,16 +137,29 @@ func (s *Store) addColumnIfMissing(table, column, definition string) error {
|
||||
}
|
||||
|
||||
func (s *Store) RecoverRuns(ctx context.Context) error {
|
||||
s.resourceMu.Lock()
|
||||
defer s.resourceMu.Unlock()
|
||||
tx, err := s.db.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer tx.Rollback()
|
||||
if _, err := tx.ExecContext(ctx, "UPDATE run_steps SET status='interrupted',agent_active=0,diagnostic='Server stopped; stage was not resumed',finished_at=CURRENT_TIMESTAMP WHERE (status='running' OR agent_active=1) AND status<>?", protocol.StepStatusWaiting); err != nil {
|
||||
// No replay: pending visits never began. Uncertain owners survive recovery,
|
||||
// including the crash window before BeginStep/session registration.
|
||||
if _, err := tx.ExecContext(ctx, "UPDATE pipeline_runs SET diagnostic='Server stopped while queued for a resource; pending visit was not resumed',status='interrupted',finished_at=CURRENT_TIMESTAMP,resource_wait='' WHERE id IN (SELECT run_id FROM resource_waiters); DELETE FROM resource_waiters"); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := recoverResourceOwners(ctx, tx); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := tx.ExecContext(ctx, "UPDATE run_steps SET status='interrupted',agent_active=CASE WHEN EXISTS(SELECT 1 FROM resource_owners o WHERE o.run_id=run_steps.run_id AND o.sequence=run_steps.sequence) THEN agent_active ELSE 0 END,diagnostic='Server stopped; stage was not resumed',finished_at=CURRENT_TIMESTAMP WHERE (status='running' OR agent_active=1) AND status<>?", protocol.StepStatusWaiting); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := tx.ExecContext(ctx, "UPDATE pipeline_runs SET status='interrupted',diagnostic='Server stopped; run was not resumed',finished_at=CURRENT_TIMESTAMP WHERE status IN ('queued','running') AND status<>?", protocol.RunStatusWaiting); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := tx.ExecContext(ctx, "UPDATE pipeline_runs SET diagnostic='Server stopped; resource ownership quarantined: inspect retained session, tool termination and restoration before reconciliation' WHERE id IN (SELECT run_id FROM resource_owners)"); err != nil {
|
||||
return err
|
||||
}
|
||||
return tx.Commit()
|
||||
}
|
||||
|
||||
@@ -18,6 +18,7 @@ func StageDefinition(s Stage) stage.Definition {
|
||||
delete(blocks, "input") // Retired contracts in historical graph JSON are ignored.
|
||||
delete(blocks, "limits")
|
||||
delete(blocks, "memory")
|
||||
delete(blocks, "resources")
|
||||
for name, value := range map[string]string{"output": s.Output, "prompt": s.Prompt} {
|
||||
if value != "" {
|
||||
blocks[name] = value
|
||||
|
||||
@@ -18,10 +18,12 @@ import (
|
||||
)
|
||||
|
||||
type Store struct {
|
||||
db *sql.DB
|
||||
dir string
|
||||
path string
|
||||
mu sync.Mutex
|
||||
db *sql.DB
|
||||
dir string
|
||||
path string
|
||||
mu sync.Mutex
|
||||
resourceMu sync.Mutex
|
||||
resourceWaiters map[ResourceVisit]chan ResourceLease
|
||||
}
|
||||
|
||||
// openDB opens one SQLite connection pool for a DSN. Every open path shares it
|
||||
|
||||
@@ -19,6 +19,7 @@ const (
|
||||
)
|
||||
|
||||
type Stage struct {
|
||||
Resources *StageResources `json:"resources,omitempty"`
|
||||
Limits *StageLimits `json:"limits,omitempty"`
|
||||
Memory map[string]string `json:"memory,omitempty"`
|
||||
Notify *NotificationPolicy `json:"notify,omitempty"`
|
||||
@@ -33,6 +34,11 @@ type Stage struct {
|
||||
Blocks map[string]string `json:"blocks,omitempty"`
|
||||
}
|
||||
|
||||
// StageResources declares one capacity-one resource, local to this store authority.
|
||||
type StageResources struct {
|
||||
Exclusive string `json:"exclusive"`
|
||||
}
|
||||
|
||||
// Limits are enforced by the runner and never included in the model prompt.
|
||||
type StageLimits struct {
|
||||
MaxVisits int `json:"max_visits"`
|
||||
|
||||
@@ -0,0 +1,29 @@
|
||||
package protocol
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestResourceWaitOlderClientCompatibility(t *testing.T) {
|
||||
data, err := json.Marshal(RunSummary{ID: 1, Status: RunStatusQueued, ResourceWait: &ResourceWait{Resource: "X", Stage: "review", Sequence: 2, QueuedSince: "now"}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var old struct {
|
||||
ID int64 `json:"id"`
|
||||
Status string `json:"status"`
|
||||
}
|
||||
if err := json.Unmarshal(data, &old); err != nil || old.Status != "queued" {
|
||||
t.Fatal("old client no longer waits", old, err)
|
||||
}
|
||||
clear, err := json.Marshal(RunEventFrame{RunID: 1, ResourceWaitSet: true})
|
||||
if err != nil || !strings.Contains(string(clear), `"resource_wait_set":true`) {
|
||||
t.Fatal("no explicit clear", string(clear), err)
|
||||
}
|
||||
var current RunSummary
|
||||
if err := json.Unmarshal([]byte(`{"id":2,"status":"running"}`), ¤t); err != nil || current.ResourceWait != nil {
|
||||
t.Fatal("missing field changed old payload", err)
|
||||
}
|
||||
}
|
||||
@@ -24,8 +24,11 @@ const (
|
||||
// from the frame instead of re-reading the whole list. They repeat on every
|
||||
// frame (pipeline identity never changes mid-run).
|
||||
type RunEventFrame struct {
|
||||
RunID int64 `json:"run_id"`
|
||||
Workspace string `json:"workspace,omitempty"`
|
||||
// ResourceWaitSet makes a nil ResourceWait an explicit cache clear.
|
||||
ResourceWaitSet bool `json:"resource_wait_set,omitempty"`
|
||||
ResourceWait *ResourceWait `json:"resource_wait,omitempty"`
|
||||
RunID int64 `json:"run_id"`
|
||||
Workspace string `json:"workspace,omitempty"`
|
||||
// PipelineID, Name, Title and Version identify the run's pipeline and its
|
||||
// operator-facing title, so a new run's row can be built without a fetch.
|
||||
PipelineID int64 `json:"pipeline_id,omitempty"`
|
||||
|
||||
@@ -118,9 +118,10 @@ type SessionExport struct {
|
||||
}
|
||||
|
||||
type RunSummary struct {
|
||||
ID int64 `json:"id"`
|
||||
PipelineID int64 `json:"pipeline_id"`
|
||||
Name string `json:"name"`
|
||||
ResourceWait *ResourceWait `json:"resource_wait,omitempty"`
|
||||
ID int64 `json:"id"`
|
||||
PipelineID int64 `json:"pipeline_id"`
|
||||
Name string `json:"name"`
|
||||
// Title is the operator-facing, bounded single-line description of what the
|
||||
// run is for, generated from its prompt. It is never empty for a new run;
|
||||
// historical rows predating the title column may omit it.
|
||||
@@ -145,6 +146,19 @@ type RunSummary struct {
|
||||
StageModels map[string]string `json:"stage_models,omitempty"`
|
||||
}
|
||||
|
||||
// ResourceWait is bounded public queue metadata. Ownership credentials stay private.
|
||||
type ResourceWait struct {
|
||||
Resource string `json:"resource"`
|
||||
Stage string `json:"stage"`
|
||||
Sequence int `json:"sequence"`
|
||||
QueuedSince string `json:"queued_since"`
|
||||
}
|
||||
|
||||
// ResourceCleanupConfirmedKey lets a stage report failed restoration without
|
||||
// pretending that tool idleness also proves the resource is ready. An explicit
|
||||
// false (or malformed value) vetoes release; it never authorizes release alone.
|
||||
const ResourceCleanupConfirmedKey = "resource_cleanup_confirmed"
|
||||
|
||||
// Label is the operator-facing name of a run: the generated Title when the
|
||||
// server stored one, otherwise the pipeline Name (historical rows predate
|
||||
// titles). It is the single definition shared by the CLI and TUI.
|
||||
@@ -155,6 +169,14 @@ func (r RunSummary) Label() string {
|
||||
return r.Name
|
||||
}
|
||||
|
||||
// StatusLabel separates resource contention from a human decision everywhere.
|
||||
func (r RunSummary) StatusLabel() string {
|
||||
if r.ResourceWait != nil {
|
||||
return "Waiting for resource " + r.ResourceWait.Resource
|
||||
}
|
||||
return r.Status
|
||||
}
|
||||
|
||||
type RunStep struct {
|
||||
Control *ControlEvidence `json:"control,omitempty"`
|
||||
FileChanges *StageFileChanges `json:"file_changes,omitempty"`
|
||||
|
||||
@@ -105,6 +105,61 @@ func TestArtifactsScanMissingDirectoryIsEmpty(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestArtifactAccessUsesOnlyCurrentAndEarlierVisits(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
r := &runner{artifactsRoot: root}
|
||||
for _, directory := range []string{"7/1", "7/2", "7/3", "8/1"} {
|
||||
if err := os.MkdirAll(filepath.Join(root, directory), 0700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
access := r.artifactAccess(7, 2, []string{"1", "3", "../8/1"}, filepath.Join(root, "7", "2"))
|
||||
current, _ := filepath.EvalSymlinks(filepath.Join(root, "7", "2"))
|
||||
prior, _ := filepath.EvalSymlinks(filepath.Join(root, "7", "1"))
|
||||
if access.Write != current || len(access.Read) != 2 || access.Read[0] != access.Write || access.Read[1] != prior {
|
||||
t.Fatalf("artifact scope = %#v", access)
|
||||
}
|
||||
if got := r.artifactAccess(7, 2, nil, filepath.Join(root, "8", "1")); got.Write != "" || len(got.Read) != 0 {
|
||||
t.Fatalf("other-run current scope accepted: %#v", got)
|
||||
}
|
||||
if got := r.artifactAccess(7, 2, nil, filepath.Join(root, "7", "3")); got.Write != "" || len(got.Read) != 0 {
|
||||
t.Fatalf("sibling current scope accepted: %#v", got)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(root, "7", "4"), []byte("not a directory"), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := r.artifactAccess(7, 4, nil, filepath.Join(root, "7", "4")); got.Write != "" || len(got.Read) != 0 {
|
||||
t.Fatalf("file current scope accepted: %#v", got)
|
||||
}
|
||||
if err := os.Symlink(filepath.Join(root, "7", "2"), filepath.Join(root, "7", "5")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := r.artifactAccess(7, 5, nil, filepath.Join(root, "7", "5")); got.Write != "" || len(got.Read) != 0 {
|
||||
t.Fatalf("symlinked current scope accepted: %#v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunnerPassesDerivedArtifactAccessToAgentClient(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
for _, directory := range []string{"7/1", "7/2"} {
|
||||
if err := os.MkdirAll(filepath.Join(root, directory), 0700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
r := &runner{artifactsRoot: root}
|
||||
client := r.agentClient(7, 2, []string{"1", "3"}, t.TempDir(), filepath.Join(root, "7", "2"), &stage.OperationLog{}, nil, []string{"read", "edit"})
|
||||
if client.ArtifactAccess.Write == "" || len(client.ArtifactAccess.Read) != 2 {
|
||||
t.Fatalf("runner did not pass scoped artifact access: %#v", client.ArtifactAccess)
|
||||
}
|
||||
if client.ArtifactAccess.Read[1] == filepath.Join(root, "7", "3") {
|
||||
t.Fatal("runner passed a future visit to the client")
|
||||
}
|
||||
piClient := r.piClient(7, 2, []string{"1", "3"}, "pi", t.TempDir(), filepath.Join(root, "7", "2"), nil, []string{"read", "edit"})
|
||||
if piClient.ArtifactAccess.Write != client.ArtifactAccess.Write || len(piClient.ArtifactAccess.Read) != len(client.ArtifactAccess.Read) || piClient.ArtifactAccess.Read[1] != client.ArtifactAccess.Read[1] {
|
||||
t.Fatalf("runner passed different artifact scope to Pi client: %#v", piClient.ArtifactAccess)
|
||||
}
|
||||
}
|
||||
|
||||
// runWithArtifact executes one real run whose stage writes a file into the
|
||||
// runner-provided artifacts directory. It returns the live handler and the
|
||||
// recorded step so tests can serve the artifact over the real HTTP endpoint.
|
||||
|
||||
@@ -32,9 +32,21 @@ type chatBroker interface {
|
||||
// Permissions lists the session's pending OpenCode approval requests so the
|
||||
// handler can push chat_permission frames.
|
||||
Permissions(ctx context.Context, sessionID string) ([]protocol.ChatPermissionFrame, error)
|
||||
// Shells returns only shell operations whose ownership is authoritatively
|
||||
// verified for sessionID. It is the reconnect/readback source.
|
||||
Shells(ctx context.Context, sessionID string) ([]protocol.ChatShell, error)
|
||||
Reply(ctx context.Context, request protocol.ChatReplyRequest) error
|
||||
}
|
||||
|
||||
// chatShellReleaser is implemented by brokers that retain verified shell ids for
|
||||
// attributing exit events. The hub calls it only after the last observer leaves
|
||||
// a room; a later attach obtains a fresh authoritative snapshot.
|
||||
// This is a bounded local-cache operation: it must not perform I/O or call the
|
||||
// hub, which serializes it with room replacement under its mutex.
|
||||
type chatShellReleaser interface {
|
||||
ReleaseShells(sessionID string)
|
||||
}
|
||||
|
||||
// operatorRegistry is the server-side operator-session seam. *pipeline.Store
|
||||
// satisfies it. It is a table of its own, separate from the pipeline stage
|
||||
// ownership (SessionOwner/run_session_owners) that settles a run: chat_sessions
|
||||
|
||||
@@ -133,6 +133,13 @@ func (h *chatHub) apply(frame protocol.ChatEventFrame) {
|
||||
}
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
h.applyLocked(frame)
|
||||
}
|
||||
|
||||
func (h *chatHub) applyLocked(frame protocol.ChatEventFrame) {
|
||||
if frame.SessionID == "" {
|
||||
return
|
||||
}
|
||||
turn := h.turnLocked(frame.SessionID)
|
||||
switch frame.Kind {
|
||||
case protocol.ChatEventStep:
|
||||
@@ -144,6 +151,9 @@ func (h *chatHub) apply(frame protocol.ChatEventFrame) {
|
||||
case protocol.ChatEventShell:
|
||||
// A shell frame carries no message id (it outlives the tool call that
|
||||
// started it), so it is folded before the message-id gate below.
|
||||
if room := h.rooms[frame.SessionID]; room != nil {
|
||||
room.shellReadback.Record(frame.Shell)
|
||||
}
|
||||
turn.applyShell(frame.Shell)
|
||||
return
|
||||
}
|
||||
@@ -240,7 +250,7 @@ func (t *chatTurn) applyShell(shell *protocol.ChatShell) {
|
||||
if shell == nil || shell.ID == "" {
|
||||
return
|
||||
}
|
||||
if shell.Status == protocol.ChatShellExited {
|
||||
if shell.Status != protocol.ChatShellRunning {
|
||||
old, ok := t.shells[shell.ID]
|
||||
if !ok {
|
||||
return
|
||||
|
||||
@@ -82,6 +82,8 @@ func (c *channelChat) Session(context.Context, string) (protocol.ChatSession, bo
|
||||
return protocol.ChatSession{}, false, nil
|
||||
}
|
||||
|
||||
func (c *channelChat) Shells(context.Context, string) ([]protocol.ChatShell, error) { return nil, nil }
|
||||
|
||||
func (c *channelChat) Permissions(context.Context, string) ([]protocol.ChatPermissionFrame, error) {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
|
||||
@@ -106,6 +106,11 @@ type chatQueuedPrompt struct {
|
||||
acked bool
|
||||
}
|
||||
|
||||
type chatObservation struct {
|
||||
done chan struct{}
|
||||
err error // published by closing done; belongs to this attempt, not the next
|
||||
}
|
||||
|
||||
// chatRoom is one session's shared live channel. Every field is guarded by the
|
||||
// owning chatHub's mutex.
|
||||
type chatRoom struct {
|
||||
@@ -117,6 +122,11 @@ type chatRoom struct {
|
||||
queue []*chatQueuedPrompt
|
||||
running bool
|
||||
streaming bool
|
||||
// observing is non-nil while one caller is attaching the shared broker
|
||||
// stream. It prevents concurrent subscribers from opening duplicate global
|
||||
// event streams before streaming can be published.
|
||||
observing *chatObservation
|
||||
shellReadback *opencode.ShellSnapshot
|
||||
// buffering holds the turn's frames until the prompting client's ack has been
|
||||
// written, so the ack always precedes the frames it acknowledges. It is set
|
||||
// only while the very first turn subscribes to the firehose.
|
||||
@@ -160,6 +170,81 @@ func (h *chatHub) roomLocked(sessionID string) *chatRoom {
|
||||
return room
|
||||
}
|
||||
|
||||
// ensureObserving starts the firehose even for a read-only subscriber, then
|
||||
// reconciles retained shells from the endpoint's authoritative list. A bounded
|
||||
// journal folds concurrent lifecycle events over that snapshot atomically. All
|
||||
// observers share this reconciliation, with no broker I/O under the hub mutex.
|
||||
func (h *chatHub) ensureObserving(room *chatRoom, broker chatBroker) error {
|
||||
if broker == nil {
|
||||
return errors.New("chat is not configured on this server")
|
||||
}
|
||||
h.mu.Lock()
|
||||
if err := room.ctx.Err(); err != nil {
|
||||
h.mu.Unlock()
|
||||
return err
|
||||
}
|
||||
if waiting := room.observing; waiting != nil {
|
||||
h.mu.Unlock()
|
||||
select {
|
||||
case <-waiting.done:
|
||||
return waiting.err
|
||||
case <-room.ctx.Done():
|
||||
return room.ctx.Err()
|
||||
}
|
||||
}
|
||||
if room.streaming {
|
||||
h.mu.Unlock()
|
||||
return nil
|
||||
}
|
||||
room.observing = &chatObservation{done: make(chan struct{})}
|
||||
waiting := room.observing
|
||||
room.shellReadback = &opencode.ShellSnapshot{}
|
||||
h.mu.Unlock()
|
||||
var observeErr error
|
||||
defer func() {
|
||||
h.mu.Lock()
|
||||
room.shellReadback = nil
|
||||
waiting.err = observeErr
|
||||
room.observing = nil
|
||||
close(waiting.done)
|
||||
h.mu.Unlock()
|
||||
}()
|
||||
if err := h.startStream(room, broker); err != nil {
|
||||
observeErr = err
|
||||
return err
|
||||
}
|
||||
shells, err := broker.Shells(room.ctx, room.sessionID)
|
||||
h.mu.Lock()
|
||||
if room.ctx.Err() != nil || h.rooms[room.sessionID] != room {
|
||||
h.mu.Unlock()
|
||||
observeErr = context.Canceled
|
||||
return observeErr
|
||||
}
|
||||
if err == nil {
|
||||
shells, err = room.shellReadback.Merge(shells)
|
||||
}
|
||||
if err != nil {
|
||||
// The firehose remains useful when its authoritative shell readback is
|
||||
// temporarily unavailable. Do not turn that read failure into a chat
|
||||
// failure (or claim the last retained snapshot is current): clear the
|
||||
// replay cache and tell attached clients that shell visibility is limited.
|
||||
// Preserve events already folded during this read, but clear stale replay.
|
||||
live, mergeErr := room.shellReadback.Merge(nil)
|
||||
if mergeErr == nil {
|
||||
h.replaceShellsLocked(room.sessionID, live)
|
||||
}
|
||||
h.mu.Unlock()
|
||||
// Never log broker bodies/URLs: they may contain credentials or private output.
|
||||
log.Printf("chat shell readback session=%s failed (%T)", room.sessionID, err)
|
||||
h.fanout(room.sessionID, protocol.Message{Type: protocol.Error, Error: "background shell visibility is limited"}, "")
|
||||
return nil
|
||||
}
|
||||
h.replaceShellsLocked(room.sessionID, shells)
|
||||
room.shellReadback = nil
|
||||
h.mu.Unlock()
|
||||
return nil
|
||||
}
|
||||
|
||||
// attach registers write as one client's subscription to sessionID's stream and
|
||||
// returns it, or the existing subscription when the same connection has already
|
||||
// attached. It is idempotent so a client may prompt and subscribe without
|
||||
@@ -209,6 +294,11 @@ func (h *chatHub) detach(sessionID string, sub *chatSub) {
|
||||
if len(room.subs) == 0 {
|
||||
delete(h.rooms, sessionID)
|
||||
room.cancel()
|
||||
// ReleaseShells is a local identity-cache operation (no broker I/O).
|
||||
// Complete it before a reconnect can acquire this session's new room.
|
||||
if releaser, ok := room.broker.(chatShellReleaser); ok {
|
||||
releaser.ReleaseShells(sessionID)
|
||||
}
|
||||
}
|
||||
h.mu.Unlock()
|
||||
}
|
||||
@@ -265,9 +355,17 @@ func (h *chatHub) broadcastUser(frame protocol.ChatEventFrame, exceptClientID st
|
||||
// once it is written. A step-ended frame finishes the turn, letting the queue
|
||||
// advance.
|
||||
func (h *chatHub) emit(frame protocol.ChatEventFrame) {
|
||||
h.apply(frame)
|
||||
h.emitRoom(nil, frame)
|
||||
}
|
||||
|
||||
func (h *chatHub) emitRoom(source *chatRoom, frame protocol.ChatEventFrame) {
|
||||
h.mu.Lock()
|
||||
room := h.rooms[frame.SessionID]
|
||||
if source != nil && (source != room || source.ctx.Err() != nil) {
|
||||
h.mu.Unlock()
|
||||
return
|
||||
}
|
||||
h.applyLocked(frame)
|
||||
buffered := false
|
||||
var targets []*chatSub
|
||||
if room != nil {
|
||||
@@ -283,6 +381,22 @@ func (h *chatHub) emit(frame protocol.ChatEventFrame) {
|
||||
if room != nil && frame.Kind == protocol.ChatEventStep && frame.Step != nil && frame.Step.Status == protocol.ChatStepEnded {
|
||||
h.finishTurnLocked(room)
|
||||
}
|
||||
// Shell replay and live lifecycle delivery must enqueue in the same order
|
||||
// as folding. Otherwise an exit can overtake an older running replay.
|
||||
if frame.Kind == protocol.ChatEventShell && !buffered {
|
||||
msg := protocol.Message{Type: protocol.ChatEvent, ChatEvent: &frame}
|
||||
var slow []*chatSub
|
||||
for _, sub := range targets {
|
||||
if !enqueueChatLocked(sub, msg) {
|
||||
slow = append(slow, sub)
|
||||
}
|
||||
}
|
||||
h.mu.Unlock()
|
||||
for _, sub := range slow {
|
||||
sub.close()
|
||||
}
|
||||
return
|
||||
}
|
||||
h.mu.Unlock()
|
||||
if buffered {
|
||||
return
|
||||
@@ -293,6 +407,62 @@ func (h *chatHub) emit(frame protocol.ChatEventFrame) {
|
||||
}
|
||||
}
|
||||
|
||||
// enqueueChatLocked never blocks or calls detach under the hub mutex.
|
||||
func enqueueChatLocked(sub *chatSub, msg protocol.Message) bool {
|
||||
select {
|
||||
case <-sub.done:
|
||||
return true
|
||||
default:
|
||||
}
|
||||
select {
|
||||
case sub.ch <- msg:
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func (h *chatHub) replayShells(sub *chatSub) {
|
||||
h.mu.Lock()
|
||||
slow := false
|
||||
if h.rooms[sub.room.sessionID] == sub.room && sub.room.subs[sub.clientID] == sub {
|
||||
turn := h.turnLocked(sub.room.sessionID)
|
||||
for _, id := range turn.shellOrder {
|
||||
shell := turn.shells[id]
|
||||
frame := protocol.ChatEventFrame{SessionID: sub.room.sessionID, Kind: protocol.ChatEventShell, Shell: &shell}
|
||||
if !enqueueChatLocked(sub, protocol.Message{Type: protocol.ChatEvent, ChatEvent: &frame}) {
|
||||
slow = true
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
h.mu.Unlock()
|
||||
if slow {
|
||||
sub.close()
|
||||
}
|
||||
}
|
||||
|
||||
// replaceShells reconciles a successful authoritative readback. It drops stale
|
||||
// retained entries (including exits missed while disconnected) and upserts the
|
||||
// verified running set without fabricating a lifecycle event for foreign shells.
|
||||
func (h *chatHub) replaceShells(sessionID string, shells []protocol.ChatShell) {
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
h.replaceShellsLocked(sessionID, shells)
|
||||
}
|
||||
|
||||
func (h *chatHub) replaceShellsLocked(sessionID string, shells []protocol.ChatShell) {
|
||||
turn := h.turnLocked(sessionID)
|
||||
for _, shell := range turn.shells {
|
||||
turn.shrink(len(shell.Command))
|
||||
}
|
||||
turn.shells = nil
|
||||
turn.shellOrder = nil
|
||||
for i := range shells {
|
||||
turn.applyShell(&shells[i])
|
||||
}
|
||||
}
|
||||
|
||||
// finishTurnLocked marks the room's turn finished and releases the turn runner's
|
||||
// wait. Callers must hold h.mu.
|
||||
func (h *chatHub) finishTurnLocked(room *chatRoom) {
|
||||
@@ -331,14 +501,20 @@ func (h *chatHub) flushBuffered(sessionID string) {
|
||||
targets = append(targets, sub)
|
||||
}
|
||||
}
|
||||
h.mu.Unlock()
|
||||
var slow []*chatSub
|
||||
for i := range pending {
|
||||
frame := pending[i]
|
||||
msg := protocol.Message{Type: protocol.ChatEvent, ChatEvent: &frame}
|
||||
for _, sub := range targets {
|
||||
sub.send(msg)
|
||||
if !enqueueChatLocked(sub, msg) {
|
||||
slow = append(slow, sub)
|
||||
}
|
||||
}
|
||||
}
|
||||
h.mu.Unlock()
|
||||
for _, sub := range slow {
|
||||
sub.close()
|
||||
}
|
||||
}
|
||||
|
||||
// submitPrompt accepts one prompt: it starts the turn runner immediately when no
|
||||
@@ -476,7 +652,7 @@ func (h *chatHub) runTurn(room *chatRoom, p *chatQueuedPrompt, done chan struct{
|
||||
return
|
||||
}
|
||||
if first {
|
||||
if err := h.startStream(room, broker); err != nil {
|
||||
if err := h.ensureObserving(room, broker); err != nil {
|
||||
h.failPrompt(room, p, err)
|
||||
h.endRoomTurn(room)
|
||||
return
|
||||
@@ -550,7 +726,7 @@ func (h *chatHub) startStream(room *chatRoom, broker chatBroker) error {
|
||||
result <- broker.Stream(ctx, room.sessionID, func() {
|
||||
once.Do(func() { close(attached) })
|
||||
}, func(frame protocol.ChatEventFrame) error {
|
||||
h.emit(frame)
|
||||
h.emitRoom(room, frame)
|
||||
return nil
|
||||
})
|
||||
}()
|
||||
|
||||
@@ -0,0 +1,261 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.a-wan.net/alan/pipelines/internal/protocol"
|
||||
"github.com/coder/websocket"
|
||||
"github.com/coder/websocket/wsjson"
|
||||
)
|
||||
|
||||
type shellBarrierBroker struct {
|
||||
*fakeChat
|
||||
read chan struct{}
|
||||
apply chan struct{}
|
||||
events chan shellBarrierEvent
|
||||
stopped chan struct{}
|
||||
reads atomic.Int32
|
||||
snapshot []protocol.ChatShell
|
||||
}
|
||||
|
||||
// Exercise the real websocket handler on a loopback scratch server and a fresh
|
||||
// SQLite database. A marker on the shared stream bounds the replay assertion
|
||||
// without sleeps, polling or a negative-read timeout.
|
||||
func TestChatShellReconciliationScratchServer(t *testing.T) {
|
||||
b := newShellBarrierBroker()
|
||||
b.snapshot = []protocol.ChatShell{{ID: "sh_old", Status: protocol.ChatShellRunning}}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
srv := httptest.NewServer(HandlerWithOptions(ctx, Options{Chat: b}, newOperatorStore(t)))
|
||||
defer srv.Close()
|
||||
conn, _, err := websocket.Dial(ctx, "ws"+strings.TrimPrefix(srv.URL, "http")+"/connect", nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer conn.CloseNow()
|
||||
subscribe := func(id string) {
|
||||
t.Helper()
|
||||
if err := wsjson.Write(ctx, conn, protocol.Message{Type: protocol.ChatSubscribe, ID: id, ChatSubscribe: &protocol.ChatQuery{SessionID: "ses_1"}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
read := func() protocol.Message {
|
||||
t.Helper()
|
||||
var m protocol.Message
|
||||
if err := wsjson.Read(ctx, conn, &m); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return m
|
||||
}
|
||||
if m := read(); m.Type != protocol.PipelineList {
|
||||
t.Fatalf("server greeting = %s", m.Type)
|
||||
}
|
||||
subscribe("first")
|
||||
if m := read(); m.ID != "first" || m.Type != protocol.ChatSubscribe {
|
||||
t.Fatalf("ack = %+v", m)
|
||||
}
|
||||
shellBarrierWait(t, b.read)
|
||||
done := make(chan struct{})
|
||||
b.events <- shellBarrierEvent{protocol.ChatShell{ID: "sh_old", Status: protocol.ChatShellExited}, done}
|
||||
shellBarrierWait(t, done)
|
||||
if m := read(); m.ChatEvent == nil || m.ChatEvent.Shell.Status != protocol.ChatShellExited {
|
||||
t.Fatalf("exit not visible: %+v", m)
|
||||
}
|
||||
close(b.apply)
|
||||
subscribe("second")
|
||||
for {
|
||||
m := read()
|
||||
if m.ChatEvent != nil && m.ChatEvent.Shell.Status == protocol.ChatShellRunning {
|
||||
t.Fatalf("stale replay: %+v", m)
|
||||
}
|
||||
if m.ID == "second" {
|
||||
break
|
||||
}
|
||||
}
|
||||
done = make(chan struct{})
|
||||
b.events <- shellBarrierEvent{protocol.ChatShell{ID: "sh_marker", Status: protocol.ChatShellExited}, done}
|
||||
shellBarrierWait(t, done)
|
||||
for {
|
||||
m := read()
|
||||
if m.ChatEvent == nil {
|
||||
continue
|
||||
}
|
||||
if m.ChatEvent.Shell.Status == protocol.ChatShellRunning {
|
||||
t.Fatalf("ghost visible after apply: %+v", m)
|
||||
}
|
||||
if m.ChatEvent.Shell.ID == "sh_marker" {
|
||||
break
|
||||
}
|
||||
}
|
||||
if b.reads.Load() != 1 {
|
||||
t.Fatalf("shared observer read count = %d", b.reads.Load())
|
||||
}
|
||||
conn.CloseNow()
|
||||
shellBarrierWait(t, b.stopped)
|
||||
}
|
||||
|
||||
type shellBarrierEvent struct {
|
||||
shell protocol.ChatShell
|
||||
done chan struct{}
|
||||
}
|
||||
|
||||
func newShellBarrierBroker() *shellBarrierBroker {
|
||||
return &shellBarrierBroker{fakeChat: scriptedChat(), read: make(chan struct{}, 8), apply: make(chan struct{}), events: make(chan shellBarrierEvent), stopped: make(chan struct{})}
|
||||
}
|
||||
|
||||
func (b *shellBarrierBroker) Shells(ctx context.Context, _ string) ([]protocol.ChatShell, error) {
|
||||
b.reads.Add(1)
|
||||
snapshot := append([]protocol.ChatShell(nil), b.snapshot...)
|
||||
b.read <- struct{}{}
|
||||
select {
|
||||
case <-b.apply:
|
||||
return snapshot, nil
|
||||
case <-ctx.Done():
|
||||
return nil, ctx.Err()
|
||||
}
|
||||
}
|
||||
|
||||
func (b *shellBarrierBroker) Stream(ctx context.Context, id string, ready func(), emit func(protocol.ChatEventFrame) error) error {
|
||||
defer close(b.stopped)
|
||||
ready()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case event := <-b.events:
|
||||
if err := emit(protocol.ChatEventFrame{SessionID: id, Kind: protocol.ChatEventShell, Shell: &event.shell}); err != nil {
|
||||
return err
|
||||
}
|
||||
close(event.done)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func shellBarrierWait(t *testing.T, ch <-chan struct{}) {
|
||||
t.Helper()
|
||||
select {
|
||||
case <-ch:
|
||||
case <-time.After(3 * time.Second):
|
||||
t.Fatal("barrier timed out")
|
||||
}
|
||||
}
|
||||
|
||||
func TestChatShellReconciliationBarrier(t *testing.T) {
|
||||
for _, status := range []string{protocol.ChatShellExited, protocol.ChatShellRunning, "timeout", "killed"} {
|
||||
t.Run(status, func(t *testing.T) {
|
||||
b := newShellBarrierBroker()
|
||||
b.snapshot = []protocol.ChatShell{{ID: "sh_old", Status: protocol.ChatShellRunning}}
|
||||
h := newChatHub()
|
||||
messages := make(chan protocol.Message, 32)
|
||||
sub := h.attach("ses_1", "one", "", b, func(m protocol.Message) error { messages <- m; return nil })
|
||||
t.Cleanup(sub.close)
|
||||
result := make(chan error, 2)
|
||||
go func() { result <- h.ensureObserving(sub.room, b) }()
|
||||
shellBarrierWait(t, b.read)
|
||||
// A second observer must share the in-flight read, not apply an
|
||||
// independently completing older/newer snapshot out of order.
|
||||
go func() { result <- h.ensureObserving(sub.room, b) }()
|
||||
shell := protocol.ChatShell{ID: "sh_old", Status: status}
|
||||
if status == protocol.ChatShellRunning {
|
||||
shell.ID = "sh_new"
|
||||
shell.Command = "sleep 10"
|
||||
}
|
||||
done := make(chan struct{})
|
||||
select {
|
||||
case b.events <- shellBarrierEvent{shell, done}:
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("event blocked behind readback")
|
||||
}
|
||||
shellBarrierWait(t, done)
|
||||
select {
|
||||
case m := <-messages:
|
||||
if m.ChatEvent == nil || m.ChatEvent.Shell == nil || m.ChatEvent.Shell.ID != shell.ID || m.ChatEvent.Shell.Status != status {
|
||||
t.Fatalf("live shell = %+v", m)
|
||||
}
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("live event not visible during read")
|
||||
}
|
||||
close(b.apply)
|
||||
for range 2 {
|
||||
select {
|
||||
case err := <-result:
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("observer stuck")
|
||||
}
|
||||
}
|
||||
if b.reads.Load() != 1 {
|
||||
t.Fatalf("redundant/out-of-order reads: %d", b.reads.Load())
|
||||
}
|
||||
snapshot := h.shellSnapshot("ses_1")
|
||||
if status == protocol.ChatShellRunning {
|
||||
if len(snapshot) != 2 {
|
||||
t.Fatalf("new create lost: %+v", snapshot)
|
||||
}
|
||||
} else if len(snapshot) != 0 {
|
||||
t.Fatalf("terminal shell resurrected: %+v", snapshot)
|
||||
}
|
||||
// Assert the same retained state is replayed to the client.
|
||||
h.replayShells(sub)
|
||||
for range len(snapshot) {
|
||||
select {
|
||||
case m := <-messages:
|
||||
if m.ChatEvent == nil || m.ChatEvent.Shell.Status != protocol.ChatShellRunning {
|
||||
t.Fatalf("replay = %+v", m)
|
||||
}
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("replay missing")
|
||||
}
|
||||
}
|
||||
sub.close()
|
||||
shellBarrierWait(t, b.stopped)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestChatShellReconciliationCancelReconnectBarrier(t *testing.T) {
|
||||
b := newShellBarrierBroker()
|
||||
h := newChatHub()
|
||||
sub := h.attach("ses_1", "one", "", b, func(protocol.Message) error { return nil })
|
||||
results := make(chan error, 2)
|
||||
go func() { results <- h.ensureObserving(sub.room, b) }()
|
||||
shellBarrierWait(t, b.read)
|
||||
go func() { results <- h.ensureObserving(sub.room, b) }()
|
||||
sub.close()
|
||||
for range 2 {
|
||||
select {
|
||||
case err := <-results:
|
||||
if err == nil {
|
||||
t.Fatal("canceled observer succeeded")
|
||||
}
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("canceled waiter stuck")
|
||||
}
|
||||
}
|
||||
shellBarrierWait(t, b.stopped)
|
||||
next := newShellBarrierBroker()
|
||||
next.snapshot = []protocol.ChatShell{{ID: "sh_new", Status: protocol.ChatShellRunning}}
|
||||
close(next.apply)
|
||||
newSub := h.attach("ses_1", "two", "", next, func(protocol.Message) error { return nil })
|
||||
t.Cleanup(newSub.close)
|
||||
if err := h.ensureObserving(newSub.room, next); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// A late callback from the canceled stream cannot poison the new room,
|
||||
// nor can a foreign-session callback from the current stream.
|
||||
h.emitRoom(sub.room, protocol.ChatEventFrame{SessionID: "ses_1", Kind: protocol.ChatEventShell, Shell: &protocol.ChatShell{ID: "sh_ghost", Status: protocol.ChatShellRunning}})
|
||||
h.emitRoom(newSub.room, protocol.ChatEventFrame{SessionID: "ses_foreign", Kind: protocol.ChatEventShell, Shell: &protocol.ChatShell{ID: "sh_foreign", Status: protocol.ChatShellRunning}})
|
||||
got := h.shellSnapshot("ses_1")
|
||||
if len(got) != 1 || got[0].ID != "sh_new" || len(h.shellSnapshot("ses_foreign")) != 0 {
|
||||
t.Fatalf("reconnect state = %+v", got)
|
||||
}
|
||||
newSub.close()
|
||||
shellBarrierWait(t, next.stopped)
|
||||
}
|
||||
@@ -111,7 +111,14 @@ func handleChatSubscribe(request protocol.Message, broker chatBroker, hub *chatH
|
||||
_ = write(protocol.Message{Type: protocol.Error, ID: request.ID, Error: "could not attach to session " + query.SessionID})
|
||||
return
|
||||
}
|
||||
// Acknowledge before the stream is allowed to fan out: otherwise an immediately
|
||||
// available retained shell can overtake the subscribe reply on this connection.
|
||||
_ = write(protocol.Message{Type: protocol.ChatSubscribe, ID: request.ID})
|
||||
if err := hub.ensureObserving(sub.room, broker); err != nil {
|
||||
sub.close()
|
||||
_ = write(protocol.Message{Type: protocol.Error, ID: request.ID, Error: "could not observe session " + query.SessionID})
|
||||
return
|
||||
}
|
||||
// Replay every still-unresolved approval request the room holds, not only the
|
||||
// most recent one, so a client that attaches after two requests were raised
|
||||
// still sees both cards.
|
||||
@@ -124,9 +131,5 @@ func handleChatSubscribe(request protocol.Message, broker chatBroker, hub *chatH
|
||||
// shows them rather than nothing until they exit. The message snapshot cannot
|
||||
// carry a shell — a shell outlives the turn and is not a turn — so this is the
|
||||
// shell half of the resume.
|
||||
for _, shell := range hub.shellSnapshot(query.SessionID) {
|
||||
running := shell
|
||||
frame := protocol.ChatEventFrame{SessionID: query.SessionID, Kind: protocol.ChatEventShell, Shell: &running}
|
||||
sub.send(protocol.Message{Type: protocol.ChatEvent, ChatEvent: &frame})
|
||||
}
|
||||
hub.replayShells(sub)
|
||||
}
|
||||
|
||||
@@ -3,6 +3,7 @@ package server
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
@@ -52,6 +53,8 @@ type fakeChat struct {
|
||||
permissionSessionID string
|
||||
permissions []protocol.ChatPermissionFrame
|
||||
permissionErr error
|
||||
shells []protocol.ChatShell
|
||||
shellErr error
|
||||
sessionCalls int
|
||||
sessions []protocol.ChatSession
|
||||
sessionErr error
|
||||
@@ -62,6 +65,8 @@ type fakeChat struct {
|
||||
started chan struct{}
|
||||
done chan struct{}
|
||||
startOnce sync.Once
|
||||
streamCalls int
|
||||
readyGate <-chan struct{}
|
||||
block bool
|
||||
subscribed bool
|
||||
promptBeforeSubscribe bool
|
||||
@@ -91,13 +96,18 @@ func (f *fakeChat) Prompt(context.Context, string, string) error {
|
||||
func (f *fakeChat) Stream(ctx context.Context, _ string, ready func(), emit func(protocol.ChatEventFrame) error) error {
|
||||
f.mu.Lock()
|
||||
f.subscribed = true
|
||||
f.streamCalls++
|
||||
readyGate := f.readyGate
|
||||
f.mu.Unlock()
|
||||
if ready != nil {
|
||||
ready()
|
||||
}
|
||||
if f.started != nil {
|
||||
f.startOnce.Do(func() { close(f.started) })
|
||||
}
|
||||
if readyGate != nil {
|
||||
<-readyGate
|
||||
}
|
||||
if ready != nil {
|
||||
ready()
|
||||
}
|
||||
if f.block {
|
||||
<-ctx.Done()
|
||||
if f.done != nil {
|
||||
@@ -116,6 +126,42 @@ func (f *fakeChat) Stream(ctx context.Context, _ string, ready func(), emit func
|
||||
return nil
|
||||
}
|
||||
|
||||
// TestChatHubConcurrentObserversShareStream pins the start-up handoff: two
|
||||
// clients that subscribe before the broker says ready still share one firehose.
|
||||
func TestChatHubConcurrentObserversShareStream(t *testing.T) {
|
||||
broker := scriptedChat()
|
||||
broker.block = true
|
||||
broker.started = make(chan struct{})
|
||||
gate := make(chan struct{})
|
||||
broker.readyGate = gate
|
||||
hub := newChatHub()
|
||||
hub.mu.Lock()
|
||||
room := hub.roomLocked("ses_1")
|
||||
hub.mu.Unlock()
|
||||
|
||||
errs := make(chan error, 2)
|
||||
go func() { errs <- hub.ensureObserving(room, broker) }()
|
||||
select {
|
||||
case <-broker.started:
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("first observer did not start the broker stream")
|
||||
}
|
||||
go func() { errs <- hub.ensureObserving(room, broker) }()
|
||||
close(gate)
|
||||
for range 2 {
|
||||
if err := <-errs; err != nil {
|
||||
t.Fatalf("ensureObserving: %v", err)
|
||||
}
|
||||
}
|
||||
broker.mu.Lock()
|
||||
calls := broker.streamCalls
|
||||
broker.mu.Unlock()
|
||||
if calls != 1 {
|
||||
t.Fatalf("broker Stream calls = %d, want 1", calls)
|
||||
}
|
||||
room.cancel()
|
||||
}
|
||||
|
||||
func (f *fakeChat) Interrupt(context.Context, string) error {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
@@ -151,6 +197,12 @@ func (f *fakeChat) Reply(_ context.Context, request protocol.ChatReplyRequest) e
|
||||
// Permissions returns the seeded pending requests, or blocks until the caller's
|
||||
// context is canceled when permBlock is set (used to prove the poller stops on
|
||||
// disconnect). Every call is counted so a test can assert polling happened.
|
||||
func (f *fakeChat) Shells(context.Context, string) ([]protocol.ChatShell, error) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
return append([]protocol.ChatShell(nil), f.shells...), f.shellErr
|
||||
}
|
||||
|
||||
func (f *fakeChat) Permissions(ctx context.Context, sessionID string) ([]protocol.ChatPermissionFrame, error) {
|
||||
f.mu.Lock()
|
||||
f.permissionCalls++
|
||||
@@ -682,6 +734,9 @@ func TestChatInterruptEndsRunningPrompt(t *testing.T) {
|
||||
case r.Method == http.MethodGet && r.URL.Path == "/api/permission/request":
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{"location":{"directory":"/work"},"data":[]}`))
|
||||
case r.Method == http.MethodGet && r.URL.Path == "/api/shell":
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{"data":[]}`))
|
||||
case r.Method == http.MethodGet && r.URL.Path == "/api/event":
|
||||
flusher, ok := w.(http.Flusher)
|
||||
if !ok {
|
||||
@@ -995,6 +1050,7 @@ func TestChatPromptStreamsShellFrame(t *testing.T) {
|
||||
// shows the background process instead of nothing until it exits.
|
||||
func TestChatSubscribeReplaysRunningShell(t *testing.T) {
|
||||
broker := scriptedChat()
|
||||
broker.shells = []protocol.ChatShell{{ID: "sh_1", Status: protocol.ChatShellRunning, Command: "sleep 120"}}
|
||||
broker.script = append([]protocol.ChatEventFrame{
|
||||
{SessionID: "ses_1", Kind: protocol.ChatEventShell, Shell: &protocol.ChatShell{ID: "sh_1", Status: protocol.ChatShellRunning, Command: "sleep 120"}},
|
||||
}, broker.script...)
|
||||
@@ -1039,6 +1095,40 @@ func TestChatSubscribeReplaysRunningShell(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestChatSubscribeKeepsStreamWhenShellReadbackFails ensures a temporary
|
||||
// authoritative readback failure does not make chat unavailable or replay a
|
||||
// stale process as running. The subscriber receives an explicit limitation
|
||||
// while its broker stream remains attached for future lifecycle events.
|
||||
func TestChatSubscribeKeepsStreamWhenShellReadbackFails(t *testing.T) {
|
||||
broker := scriptedChat()
|
||||
broker.block = true
|
||||
broker.shellErr = errors.New("shell endpoint unavailable: https://user:secret@example.invalid private response body")
|
||||
ctx, _, conn := dialChat(t, broker)
|
||||
if err := wsjson.Write(ctx, conn, protocol.Message{Type: protocol.ChatSubscribe, ID: "sub1", ChatSubscribe: &protocol.ChatQuery{SessionID: "ses_1"}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var ack protocol.Message
|
||||
if err := wsjson.Read(ctx, conn, &ack); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if ack.Type != protocol.ChatSubscribe || ack.ID != "sub1" {
|
||||
t.Fatalf("subscribe ack = %+v", ack)
|
||||
}
|
||||
var limited protocol.Message
|
||||
if err := wsjson.Read(ctx, conn, &limited); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if limited.Type != protocol.Error || limited.Error != "background shell visibility is limited" {
|
||||
t.Fatalf("limited visibility notice = %+v", limited)
|
||||
}
|
||||
broker.mu.Lock()
|
||||
calls := broker.streamCalls
|
||||
broker.mu.Unlock()
|
||||
if calls != 1 {
|
||||
t.Fatalf("broker Stream calls = %d, want 1", calls)
|
||||
}
|
||||
}
|
||||
|
||||
// TestChatPromptSubscribesBeforePrompt pins the subscribe-before-prompt order:
|
||||
// the firehose has no replay cursor, so the subscription must be attached before
|
||||
// the prompt is posted or deltas emitted in between are lost.
|
||||
@@ -1079,6 +1169,9 @@ func realAdapterChat(t *testing.T, frames []string) []protocol.ChatEventFrame {
|
||||
case r.Method == http.MethodGet && r.URL.Path == "/api/permission/request":
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{"location":{"directory":"/work"},"data":[]}`))
|
||||
case r.Method == http.MethodGet && r.URL.Path == "/api/shell":
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{"data":[]}`))
|
||||
case r.Method == http.MethodGet && r.URL.Path == "/api/event":
|
||||
streams.Add(1)
|
||||
w.Header().Set("Content-Type", "text/event-stream")
|
||||
|
||||
@@ -0,0 +1,185 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"git.a-wan.net/alan/pipelines/internal/stage"
|
||||
)
|
||||
|
||||
// verifyDelivery proves that the stage landed a complete, committed tree before
|
||||
// its success can settle the run. It re-derives the capability after the stage:
|
||||
// a stage result and its input cannot select either the landing path or checks.
|
||||
const maxDeliveryCheckOutput = 64 << 10
|
||||
const deliveryCleanupTimeout = 10 * time.Second
|
||||
|
||||
// verifyDelivery verifies the committed item tree in a disposable worktree
|
||||
// before fast-forwarding canonical main. Settlement deliberately performs only
|
||||
// non-mutating repository checks after that mutation: a declared check must
|
||||
// never be the first signal that the landed tree is known-red.
|
||||
func verifyDelivery(ctx context.Context, workspace string, access stage.LandingAccess, checks []string) error {
|
||||
lock, err := acquireDeliveryLock(ctx, access.LockPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer lock.Close()
|
||||
current, err := landingAccess(workspace)
|
||||
if err != nil {
|
||||
return fmt.Errorf("delivery access changed: %w", err)
|
||||
}
|
||||
if current != access {
|
||||
return fmt.Errorf("delivery access changed during stage")
|
||||
}
|
||||
for _, directory := range []string{workspace, access.MainWorktree} {
|
||||
status, err := gitOutput(directory, "status", "--porcelain=v1", "--untracked-files=all")
|
||||
if err != nil {
|
||||
return fmt.Errorf("delivery checkout status: %w", err)
|
||||
}
|
||||
if status != "" {
|
||||
return fmt.Errorf("delivery checkout is not clean: %s", strings.Split(status, "\n")[0])
|
||||
}
|
||||
}
|
||||
itemHead, err := gitOutput(workspace, "rev-parse", "HEAD")
|
||||
if err != nil {
|
||||
return fmt.Errorf("delivery item commit: %w", err)
|
||||
}
|
||||
if err := runDeliveryCandidate(ctx, workspace, itemHead, checks); err != nil {
|
||||
return err
|
||||
}
|
||||
mainHead, err := gitOutput(access.MainWorktree, "rev-parse", "HEAD")
|
||||
if err != nil {
|
||||
return fmt.Errorf("delivery main commit: %w", err)
|
||||
}
|
||||
if err := exec.CommandContext(ctx, "git", "-C", workspace, "merge-base", "--is-ancestor", mainHead, itemHead).Run(); err != nil {
|
||||
return fmt.Errorf("delivery item commit cannot fast-forward canonical main")
|
||||
}
|
||||
if output, err := exec.CommandContext(ctx, "git", "-C", access.MainWorktree, "merge", "--ff-only", itemHead).CombinedOutput(); err != nil {
|
||||
return fmt.Errorf("delivery fast-forward canonical main: %w: %s", err, strings.TrimSpace(string(output)))
|
||||
}
|
||||
for _, directory := range []string{workspace, access.MainWorktree} {
|
||||
status, err := gitOutput(directory, "status", "--porcelain=v1", "--untracked-files=all")
|
||||
if err != nil {
|
||||
return fmt.Errorf("delivery settlement status: %w", err)
|
||||
}
|
||||
if status != "" {
|
||||
return fmt.Errorf("delivery checkout is not clean after checks: %s", strings.Split(status, "\n")[0])
|
||||
}
|
||||
}
|
||||
// Re-derive at the settlement boundary. A success means exactly the checked
|
||||
// item commit is now reachable from the same canonical main worktree.
|
||||
current, err = landingAccess(workspace)
|
||||
if err != nil || current != access {
|
||||
return fmt.Errorf("delivery access changed during settlement")
|
||||
}
|
||||
mainHead, err = gitOutput(access.MainWorktree, "rev-parse", "HEAD")
|
||||
if err != nil {
|
||||
return fmt.Errorf("delivery main commit: %w", err)
|
||||
}
|
||||
if err := exec.CommandContext(ctx, "git", "-C", workspace, "merge-base", "--is-ancestor", itemHead, mainHead).Run(); err != nil {
|
||||
return fmt.Errorf("delivery item commit is not an ancestor of canonical main")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func acquireDeliveryLock(ctx context.Context, path string) (*os.File, error) {
|
||||
file, err := os.OpenFile(path, os.O_CREATE|os.O_RDWR, 0600)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("delivery lock: %w", err)
|
||||
}
|
||||
for {
|
||||
err := syscall.Flock(int(file.Fd()), syscall.LOCK_EX|syscall.LOCK_NB)
|
||||
if err == nil {
|
||||
return file, nil
|
||||
}
|
||||
if !errors.Is(err, syscall.EWOULDBLOCK) && !errors.Is(err, syscall.EAGAIN) {
|
||||
_ = file.Close()
|
||||
return nil, fmt.Errorf("delivery lock: %w", err)
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
_ = file.Close()
|
||||
return nil, fmt.Errorf("delivery lock: %w", ctx.Err())
|
||||
case <-time.After(20 * time.Millisecond):
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func runDeliveryCandidate(ctx context.Context, workspace, itemHead string, checks []string) error {
|
||||
candidate, err := os.MkdirTemp(filepath.Dir(workspace), ".pipelines-delivery-")
|
||||
if err != nil {
|
||||
return fmt.Errorf("create delivery candidate: %w", err)
|
||||
}
|
||||
if err := os.Remove(candidate); err != nil {
|
||||
return fmt.Errorf("prepare delivery candidate: %w", err)
|
||||
}
|
||||
if output, err := boundedCommandOutput(ctx, workspace, "worktree", "add", "--detach", candidate, itemHead); err != nil {
|
||||
cleanupDeliveryCandidate(workspace, candidate)
|
||||
return fmt.Errorf("create delivery candidate: %w: %s", err, strings.TrimSpace(string(output)))
|
||||
}
|
||||
defer cleanupDeliveryCandidate(workspace, candidate)
|
||||
for _, check := range checks {
|
||||
if err := runDeliveryCheck(ctx, candidate, check); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// cleanupDeliveryCandidate always uses its own bounded context: the stage context
|
||||
// may already be canceled when a check fails or times out.
|
||||
func cleanupDeliveryCandidate(workspace, candidate string) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), deliveryCleanupTimeout)
|
||||
defer cancel()
|
||||
_ = exec.CommandContext(ctx, "git", "-C", workspace, "worktree", "remove", "--force", candidate).Run()
|
||||
_ = os.RemoveAll(candidate)
|
||||
}
|
||||
|
||||
func boundedCommandOutput(ctx context.Context, directory string, args ...string) ([]byte, error) {
|
||||
cmd := exec.CommandContext(ctx, "git", append([]string{"-C", directory}, args...)...)
|
||||
var output boundedDeliveryOutput
|
||||
cmd.Stdout = &output
|
||||
cmd.Stderr = &output
|
||||
err := cmd.Run()
|
||||
return []byte(output.String()), err
|
||||
}
|
||||
|
||||
func runDeliveryCheck(ctx context.Context, directory, check string) error {
|
||||
cmd := exec.CommandContext(ctx, "sh", "-c", check)
|
||||
cmd.Dir = directory
|
||||
var output boundedDeliveryOutput
|
||||
cmd.Stdout = &output
|
||||
cmd.Stderr = &output
|
||||
if err := cmd.Run(); err != nil {
|
||||
return fmt.Errorf("delivery check %q failed: %w: %s", check, err, output.String())
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type boundedDeliveryOutput struct{ bytes.Buffer }
|
||||
|
||||
func (b *boundedDeliveryOutput) Write(p []byte) (int, error) {
|
||||
remaining := maxDeliveryCheckOutput - b.Len()
|
||||
if remaining > 0 {
|
||||
_, _ = b.Buffer.Write(p[:min(remaining, len(p))])
|
||||
}
|
||||
return len(p), nil
|
||||
}
|
||||
|
||||
func (b *boundedDeliveryOutput) String() string {
|
||||
text := strings.TrimSpace(b.Buffer.String())
|
||||
if b.Len() == maxDeliveryCheckOutput {
|
||||
return text + " (output truncated)"
|
||||
}
|
||||
return text
|
||||
}
|
||||
|
||||
var _ io.Writer = (*boundedDeliveryOutput)(nil)
|
||||
@@ -0,0 +1,130 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"git.a-wan.net/alan/pipelines/internal/stage"
|
||||
)
|
||||
|
||||
// landingAccess derives delivery paths from Git's verified worktree metadata,
|
||||
// never from stage input. A missing canonical main worktree fails closed.
|
||||
func landingAccess(workspace string) (stage.LandingAccess, error) {
|
||||
workspace, err := canonicalDirectory(workspace)
|
||||
if err != nil {
|
||||
return stage.LandingAccess{}, fmt.Errorf("delivery workspace: %w", err)
|
||||
}
|
||||
gitDir, err := gitOutput(workspace, "rev-parse", "--git-dir")
|
||||
if err != nil {
|
||||
return stage.LandingAccess{}, fmt.Errorf("delivery workspace is not a Git worktree: %w", err)
|
||||
}
|
||||
commonDir, err := gitOutput(workspace, "rev-parse", "--git-common-dir")
|
||||
if err != nil {
|
||||
return stage.LandingAccess{}, fmt.Errorf("delivery repository metadata: %w", err)
|
||||
}
|
||||
gitDir, err = canonicalDirectory(resolveGitPath(workspace, gitDir))
|
||||
if err != nil {
|
||||
return stage.LandingAccess{}, fmt.Errorf("delivery Git metadata: %w", err)
|
||||
}
|
||||
commonDir, err = canonicalDirectory(resolveGitPath(workspace, commonDir))
|
||||
if err != nil {
|
||||
return stage.LandingAccess{}, fmt.Errorf("delivery common Git metadata: %w", err)
|
||||
}
|
||||
main, err := mainWorktree(workspace, commonDir)
|
||||
if err != nil {
|
||||
return stage.LandingAccess{}, err
|
||||
}
|
||||
// Delivery originates in a linked item worktree. Its metadata must be under
|
||||
// the repository's canonical common directory, never a standalone or foreign
|
||||
// checkout. Git's common directory is the bounded capability needed for refs,
|
||||
// worktree state, and the landing lock.
|
||||
if !pathWithin(filepath.Join(commonDir, "worktrees"), gitDir) {
|
||||
return stage.LandingAccess{}, fmt.Errorf("delivery workspace is not a linked Git worktree")
|
||||
}
|
||||
return stage.LandingAccess{MainWorktree: main, GitMetadata: commonDir, LockPath: filepath.Join(commonDir, "pipelines-landing.lock")}, nil
|
||||
}
|
||||
|
||||
func mainWorktree(workspace, commonDir string) (string, error) {
|
||||
list, err := gitOutput(workspace, "worktree", "list", "--porcelain")
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("list repository worktrees: %w", err)
|
||||
}
|
||||
for _, record := range strings.Split(strings.TrimSpace(list), "\n\n") {
|
||||
lines := strings.Split(record, "\n")
|
||||
if len(lines) == 0 || !strings.HasPrefix(lines[0], "worktree ") || !worktreeHasBranch(lines, "refs/heads/main") {
|
||||
continue
|
||||
}
|
||||
candidate, err := canonicalDirectory(strings.TrimPrefix(lines[0], "worktree "))
|
||||
if err != nil || candidate == workspace {
|
||||
continue
|
||||
}
|
||||
candidateGitDir, err := gitOutput(candidate, "rev-parse", "--git-dir")
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
candidateGitDir, err = canonicalDirectory(resolveGitPath(candidate, candidateGitDir))
|
||||
if err != nil || candidateGitDir != commonDir {
|
||||
continue
|
||||
}
|
||||
candidateCommon, err := gitOutput(candidate, "rev-parse", "--git-common-dir")
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
candidateCommon, err = canonicalDirectory(resolveGitPath(candidate, candidateCommon))
|
||||
if err == nil && candidateCommon == commonDir {
|
||||
return candidate, nil
|
||||
}
|
||||
}
|
||||
return "", fmt.Errorf("no canonical main worktree in the delivery repository")
|
||||
}
|
||||
|
||||
func worktreeHasBranch(lines []string, branch string) bool {
|
||||
for _, line := range lines[1:] {
|
||||
if line == "branch "+branch {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func canonicalDirectory(path string) (string, error) {
|
||||
resolved, err := filepath.EvalSymlinks(path)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
info, err := os.Stat(resolved)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if !info.IsDir() {
|
||||
return "", fmt.Errorf("not a directory")
|
||||
}
|
||||
return filepath.Clean(resolved), nil
|
||||
}
|
||||
|
||||
func resolveGitPath(workspace, value string) string {
|
||||
if filepath.IsAbs(value) {
|
||||
return value
|
||||
}
|
||||
return filepath.Join(workspace, value)
|
||||
}
|
||||
|
||||
func pathWithin(parent, child string) bool {
|
||||
rel, err := filepath.Rel(parent, child)
|
||||
return err == nil && rel != "." && rel != ".." && !strings.HasPrefix(rel, ".."+string(filepath.Separator))
|
||||
}
|
||||
|
||||
func gitOutput(directory string, args ...string) (string, error) {
|
||||
cmd := exec.Command("git", append([]string{"-C", directory}, args...)...)
|
||||
var stderr bytes.Buffer
|
||||
cmd.Stderr = &stderr
|
||||
data, err := cmd.Output()
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("git %s: %w: %s", strings.Join(args, " "), err, strings.TrimSpace(stderr.String()))
|
||||
}
|
||||
return strings.TrimSpace(string(data)), nil
|
||||
}
|
||||
@@ -0,0 +1,304 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.a-wan.net/alan/pipelines/internal/pipeline"
|
||||
"git.a-wan.net/alan/pipelines/internal/protocol"
|
||||
"git.a-wan.net/alan/pipelines/internal/stage"
|
||||
)
|
||||
|
||||
func TestLandingAccessAcceptsOnlyVerifiedSameRepositoryMainWorktree(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
main := filepath.Join(root, "main")
|
||||
runGit(t, root, "init", "-b", "main", main)
|
||||
runGit(t, main, "config", "user.email", "test@example.invalid")
|
||||
runGit(t, main, "config", "user.name", "test")
|
||||
if err := os.WriteFile(filepath.Join(main, "README"), []byte("base\n"), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
runGit(t, main, "add", "README")
|
||||
runGit(t, main, "commit", "-m", "base")
|
||||
item := filepath.Join(root, "item")
|
||||
runGit(t, main, "worktree", "add", "-b", "item", item)
|
||||
access, err := landingAccess(item)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
canonicalMain, err := canonicalDirectory(main)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if access.MainWorktree != canonicalMain || access.GitMetadata != filepath.Join(canonicalMain, ".git") || access.LockPath != filepath.Join(canonicalMain, ".git", "pipelines-landing.lock") {
|
||||
t.Fatalf("unexpected access: %+v", access)
|
||||
}
|
||||
foreign := filepath.Join(root, "foreign")
|
||||
runGit(t, root, "init", "-b", "main", foreign)
|
||||
if _, err := landingAccess(foreign); err == nil {
|
||||
t.Fatal("foreign repository without an item worktree gained landing access")
|
||||
}
|
||||
if _, err := landingAccess(main); err == nil {
|
||||
t.Fatal("main worktree gained linked-worktree delivery access")
|
||||
}
|
||||
if err := os.RemoveAll(item); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
runGit(t, main, "worktree", "prune")
|
||||
if _, err := landingAccess(main); err == nil {
|
||||
t.Fatal("repository without a linked delivery worktree gained landing access")
|
||||
}
|
||||
secondItem := filepath.Join(root, "second-item")
|
||||
runGit(t, main, "worktree", "add", "-b", "second-item", secondItem)
|
||||
runGit(t, main, "branch", "-m", "main", "trunk")
|
||||
if _, err := landingAccess(secondItem); err == nil {
|
||||
t.Fatal("repository without a canonical main worktree gained landing access")
|
||||
}
|
||||
}
|
||||
|
||||
func TestVerifyDeliveryRequiresCleanLandedMainAndPassingChecks(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
main := filepath.Join(root, "main")
|
||||
runGit(t, root, "init", "-b", "main", main)
|
||||
runGit(t, main, "config", "user.email", "test@example.invalid")
|
||||
runGit(t, main, "config", "user.name", "test")
|
||||
if err := os.WriteFile(filepath.Join(main, "README"), []byte("base\n"), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
runGit(t, main, "add", "README")
|
||||
runGit(t, main, "commit", "-m", "base")
|
||||
item := filepath.Join(root, "item")
|
||||
runGit(t, main, "worktree", "add", "-b", "item", item)
|
||||
if err := os.WriteFile(filepath.Join(item, "delivered"), []byte("complete\n"), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
runGit(t, item, "add", "delivered")
|
||||
runGit(t, item, "commit", "-m", "delivery")
|
||||
runGit(t, main, "merge", "--ff-only", "item")
|
||||
access, err := landingAccess(item)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := verifyDelivery(context.Background(), item, access, []string{"test -f delivered"}); err != nil {
|
||||
t.Fatalf("complete landed delivery rejected: %v", err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(item, "missing-helper"), []byte("dirty\n"), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := verifyDelivery(context.Background(), item, access, []string{"true"}); err == nil {
|
||||
t.Fatal("dirty task-owned helper was accepted")
|
||||
}
|
||||
if err := os.Remove(filepath.Join(item, "missing-helper")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := verifyDelivery(context.Background(), item, access, []string{"false"}); err == nil {
|
||||
t.Fatal("failed required check was accepted")
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(item, "unlanded"), []byte("no\n"), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
runGit(t, item, "add", "unlanded")
|
||||
runGit(t, item, "commit", "-m", "unlanded")
|
||||
if err := verifyDelivery(context.Background(), item, access, []string{"test -f unlanded"}); err != nil {
|
||||
t.Fatalf("complete unlanded item was not landed after candidate verification: %v", err)
|
||||
}
|
||||
mainHead := runGitOutput(t, main, "rev-parse", "HEAD")
|
||||
itemHead := runGitOutput(t, item, "rev-parse", "HEAD")
|
||||
if mainHead != itemHead {
|
||||
t.Fatal("verified item was not fast-forwarded into canonical main")
|
||||
}
|
||||
}
|
||||
|
||||
func TestVerifyDeliveryFailingCandidateLeavesMainUnchanged(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
main := filepath.Join(root, "main")
|
||||
runGit(t, root, "init", "-b", "main", main)
|
||||
runGit(t, main, "config", "user.email", "test@example.invalid")
|
||||
runGit(t, main, "config", "user.name", "test")
|
||||
if err := os.WriteFile(filepath.Join(main, "README"), []byte("base\n"), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
runGit(t, main, "add", "README")
|
||||
runGit(t, main, "commit", "-m", "base")
|
||||
item := filepath.Join(root, "item")
|
||||
runGit(t, main, "worktree", "add", "-b", "item", item)
|
||||
if err := os.WriteFile(filepath.Join(item, "candidate"), []byte("candidate\n"), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
runGit(t, item, "add", "candidate")
|
||||
runGit(t, item, "commit", "-m", "candidate")
|
||||
access, err := landingAccess(item)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
before := runGitOutput(t, main, "rev-parse", "HEAD")
|
||||
if err := verifyDelivery(context.Background(), item, access, []string{"test -f candidate", "false"}); err == nil {
|
||||
t.Fatal("failing candidate check landed main")
|
||||
}
|
||||
if after := runGitOutput(t, main, "rev-parse", "HEAD"); after != before {
|
||||
t.Fatal("failing candidate check changed canonical main")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunnerSettlesSuccessfulDeliveryAfterCandidateLanding(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
root := t.TempDir()
|
||||
main := filepath.Join(root, "main")
|
||||
runGit(t, root, "init", "-b", "main", main)
|
||||
runGit(t, main, "config", "user.email", "test@example.invalid")
|
||||
runGit(t, main, "config", "user.name", "test")
|
||||
if err := os.WriteFile(filepath.Join(main, "README"), []byte("base\n"), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
runGit(t, main, "add", "README")
|
||||
runGit(t, main, "commit", "-m", "base")
|
||||
item := filepath.Join(root, "item")
|
||||
runGit(t, main, "worktree", "add", "-b", "item", item)
|
||||
if err := os.WriteFile(filepath.Join(item, "unlanded"), []byte("no\n"), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
runGit(t, item, "add", "unlanded")
|
||||
runGit(t, item, "commit", "-m", "unlanded")
|
||||
|
||||
store, err := pipeline.Open(filepath.Join(root, "runner.db"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer store.Close()
|
||||
source := pipeline.Template([]string{"committer"}, true)
|
||||
source = strings.Replace(source, "```configs", "```checks\ntrue\n```\n\n```configs", 1)
|
||||
source = strings.Replace(source, "timeout=1h", "timeout=1h\ndelivery=true", 1)
|
||||
if _, err := store.Submit(ctx, "delivery", source); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
items, err := store.List(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
r, err := newRunner(ctx, store)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
r.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) {
|
||||
return stage.Result{Outcome: "success", Output: stage.Values{}}, nil
|
||||
}
|
||||
run, err := r.start(protocol.Message{PipelineID: items[0].ID, Version: 1, RequestKey: "unlanded", Workspace: item, Input: map[string]any{"prompt": "task"}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
r.wg.Wait()
|
||||
detail, err := store.GetRun(ctx, run.ID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if detail.Status != "succeeded" || len(detail.Steps) != 1 {
|
||||
t.Fatalf("complete delivery did not settle successfully: %+v", detail)
|
||||
}
|
||||
if runGitOutput(t, main, "rev-parse", "HEAD") != runGitOutput(t, item, "rev-parse", "HEAD") {
|
||||
t.Fatal("runner did not land the verified item commit")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunnerRejectsWrongBackendVerificationEvidence(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
source := pipeline.Template([]string{"worker"}, true)
|
||||
source = strings.Replace(source, "```configs", "```verification\nbackend=branch-api\nrevision=abc123\ncapability=controlled-state\nfixture=reconnect-foreground\n```\n\n```configs", 1)
|
||||
r, store, id := submitRunner(t, ctx, source)
|
||||
defer store.Close()
|
||||
r.verifyTarget = func(context.Context, stage.VerificationTarget) (stage.VerificationEvidence, error) {
|
||||
return stage.VerificationEvidence{Backend: "old-api", Revision: "abc123", Capability: "controlled-state", Fixture: "reconnect-foreground"}, nil
|
||||
}
|
||||
r.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) {
|
||||
return stage.Result{Outcome: "success", Output: stage.Values{}}, nil
|
||||
}
|
||||
run := startRun(t, r, id, "wrong-backend-evidence")
|
||||
detail, err := store.GetRun(ctx, run.ID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if detail.Status != "failed" || len(detail.Steps) != 1 || !strings.Contains(detail.Steps[0].Diagnostic, "backend") {
|
||||
t.Fatalf("wrong-backend evidence settled as success: %+v", detail)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunnerIgnoresModelSuppliedVerificationEvidence(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
source := pipeline.Template([]string{"worker"}, true)
|
||||
source = strings.Replace(source, "```configs", "```verification\nbackend=branch-api\nrevision=abc123\ncapability=controlled-state\nfixture=reconnect-foreground\n```\n\n```configs", 1)
|
||||
r, store, id := submitRunner(t, ctx, source)
|
||||
defer store.Close()
|
||||
r.verifyTarget = func(context.Context, stage.VerificationTarget) (stage.VerificationEvidence, error) {
|
||||
return stage.VerificationEvidence{Backend: "branch-api", Revision: "abc123", Capability: "controlled-state", Fixture: "reconnect-foreground"}, nil
|
||||
}
|
||||
r.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) {
|
||||
return stage.Result{Outcome: "success", Output: stage.Values{"verification": map[string]any{"backend": "old-api", "revision": "stale", "capability": "invented", "fixture": "other"}}}, nil
|
||||
}
|
||||
run := startRun(t, r, id, "model-supplied-evidence")
|
||||
detail, err := store.GetRun(ctx, run.ID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if detail.Status != "succeeded" || len(detail.Steps) != 1 {
|
||||
t.Fatalf("controlled verification did not settle: %+v", detail)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMainWorktreeRejectsNearMatchAndLinkedMain(t *testing.T) {
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
branch string
|
||||
}{
|
||||
{name: "mainline is not main", branch: "mainline"},
|
||||
{name: "linked main is not canonical", branch: "main"},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
primary := filepath.Join(root, "primary")
|
||||
runGit(t, root, "init", "-b", "trunk", primary)
|
||||
runGit(t, primary, "config", "user.email", "test@example.invalid")
|
||||
runGit(t, primary, "config", "user.name", "test")
|
||||
if err := os.WriteFile(filepath.Join(primary, "README"), []byte("base\n"), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
runGit(t, primary, "add", "README")
|
||||
runGit(t, primary, "commit", "-m", "base")
|
||||
item := filepath.Join(root, "item")
|
||||
runGit(t, primary, "worktree", "add", "-b", "item", item)
|
||||
candidate := filepath.Join(root, test.branch)
|
||||
runGit(t, primary, "worktree", "add", "-b", test.branch, candidate)
|
||||
|
||||
commonDir, err := gitOutput(item, "rev-parse", "--git-common-dir")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
commonDir, err = canonicalDirectory(resolveGitPath(item, commonDir))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := mainWorktree(item, commonDir); err == nil {
|
||||
t.Fatalf("%s gained canonical main access", test.name)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func runGit(t *testing.T, directory string, args ...string) {
|
||||
t.Helper()
|
||||
cmd := exec.Command("git", append([]string{"-C", directory}, args...)...)
|
||||
if output, err := cmd.CombinedOutput(); err != nil {
|
||||
t.Fatalf("git %v: %v: %s", args, err, output)
|
||||
}
|
||||
}
|
||||
|
||||
func runGitOutput(t *testing.T, directory string, args ...string) string {
|
||||
t.Helper()
|
||||
output, err := exec.Command("git", append([]string{"-C", directory}, args...)...).Output()
|
||||
if err != nil {
|
||||
t.Fatalf("git %v: %v", args, err)
|
||||
}
|
||||
return strings.TrimSpace(string(output))
|
||||
}
|
||||
@@ -0,0 +1,154 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.a-wan.net/alan/pipelines/internal/cli"
|
||||
"git.a-wan.net/alan/pipelines/internal/pipeline"
|
||||
"git.a-wan.net/alan/pipelines/internal/protocol"
|
||||
"git.a-wan.net/alan/pipelines/internal/stage"
|
||||
)
|
||||
|
||||
// This is a real CLI/WebSocket/scratch-SQLite exercise with an explicitly FAKE
|
||||
// executor. It never calls inference, a phone, or a deployed server.
|
||||
func TestResourceScratchCLIWaitingThenProgress(t *testing.T) {
|
||||
t.Setenv("PIPELINES_HARNESS_CONFIG", filepath.Join(t.TempDir(), "harnesses.json"))
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
store, err := pipeline.Open(filepath.Join(t.TempDir(), "scratch.db"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer store.Close()
|
||||
source := pipeline.Template([]string{"work"}, true) + "```resources\n{\"exclusive\":\"fake-phone\"}\n```\n"
|
||||
if _, err := store.Submit(ctx, "scratch-resource", source); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
handler := Handler(ctx, store).(*runHandler)
|
||||
started, release := make(chan struct{}), make(chan struct{})
|
||||
handler.runner.execute = func(ctx context.Context, _ stage.Definition, input stage.Values) (stage.Result, error) {
|
||||
if input["prompt"] == "A" {
|
||||
close(started)
|
||||
select {
|
||||
case <-release:
|
||||
case <-ctx.Done():
|
||||
return stage.Result{}, ctx.Err()
|
||||
}
|
||||
}
|
||||
return stage.Result{Outcome: "success", Output: stage.Values{"execution": "FAKE executor; no device or inference"}}, nil
|
||||
}
|
||||
handler.runner.events.interval = time.Millisecond
|
||||
events := make(chan protocol.RunEventFrame, 64)
|
||||
sub := handler.runner.events.subscribe("cli-evidence", "", nil, func(m protocol.Message) error {
|
||||
if m.RunEvent != nil {
|
||||
events <- *m.RunEvent
|
||||
}
|
||||
return nil
|
||||
})
|
||||
sub.ready()
|
||||
defer sub.close()
|
||||
srv := httptest.NewServer(handler)
|
||||
defer srv.Close()
|
||||
defer func() { cancel(); WaitRuns(handler) }()
|
||||
endpoint := "ws" + strings.TrimPrefix(srv.URL, "http") + "/connect"
|
||||
run := func(args ...string) string {
|
||||
var out bytes.Buffer
|
||||
app := cli.App{Out: &out, Err: &out, In: strings.NewReader("")}
|
||||
if err := app.Run(ctx, append([]string{"--url", endpoint}, args...)); err != nil {
|
||||
t.Fatalf("CLI %v: %v\n%s", args, err, out.String())
|
||||
}
|
||||
t.Logf("pipelines %s\n%s", strings.Join(args, " "), out.String())
|
||||
return out.String()
|
||||
}
|
||||
parseRun := func(output string) int64 {
|
||||
var m protocol.Message
|
||||
if err := json.Unmarshal([]byte(output), &m); err != nil || m.RunDetail == nil {
|
||||
t.Fatal("CLI run response", output, err)
|
||||
}
|
||||
return m.RunDetail.ID
|
||||
}
|
||||
run("validate", "testdata/android-resource-draft.md", "--json")
|
||||
a := parseRun(run("run", "scratch-resource", "A", "--workspace", t.TempDir(), "--yes", "--json", "--request-key", "a"))
|
||||
<-started
|
||||
b := parseRun(run("run", "scratch-resource", "B", "--workspace", t.TempDir(), "--yes", "--json", "--request-key", "b"))
|
||||
awaitResourceEvent(t, events, b, func(e protocol.RunEventFrame) bool { return e.ResourceWait != nil })
|
||||
var detail protocol.RunDetail
|
||||
if err := json.Unmarshal([]byte(run("inspect", fmt.Sprint(b), "--json")), &detail); err != nil || detail.ResourceWait == nil || detail.Status != "queued" || len(detail.Steps) != 0 {
|
||||
t.Fatal("CLI did not observe queue", err)
|
||||
}
|
||||
if output := run("runs"); !strings.Contains(output, "Waiting for resource fake-phone") {
|
||||
t.Fatal("list omitted resource status", output)
|
||||
}
|
||||
var snapshot *protocol.RunEventFrame
|
||||
handleRunEvents(protocol.Message{Type: protocol.Subscribe, RunIDs: []int64{b}}, handler.runner.events, handler.runner, "snapshot-evidence", func(m protocol.Message) error {
|
||||
if m.RunEvent != nil {
|
||||
snapshot = m.RunEvent
|
||||
}
|
||||
return nil
|
||||
})
|
||||
handler.runner.events.unsubscribeClient("snapshot-evidence")
|
||||
if snapshot == nil || snapshot.ResourceWait == nil || !snapshot.ResourceWaitSet {
|
||||
t.Fatal("subscription snapshot omitted resource wait")
|
||||
}
|
||||
close(release)
|
||||
run("wait", fmt.Sprint(a), fmt.Sprint(b), "--json")
|
||||
detail = protocol.RunDetail{}
|
||||
if err := json.Unmarshal([]byte(run("inspect", fmt.Sprint(b), "--json")), &detail); err != nil || detail.Status != "succeeded" || detail.ResourceWait != nil || len(detail.Steps) != 1 {
|
||||
t.Fatal("automatic progress not observed", err)
|
||||
}
|
||||
t.Log("Observed queued resource wait followed by automatic progress; FAKE executor only, no phone/inference/deployment.")
|
||||
}
|
||||
|
||||
func TestResourceAndroidDraftPublishedV3Provenance(t *testing.T) {
|
||||
baseline, err := os.ReadFile("testdata/android-v3-published.md")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if hash := fmt.Sprintf("%x", sha256.Sum256(baseline)); hash != "4a3a4b8b947eb20c115c5470e6e378f117d5fa291a7b6c8ae966a3745bc826de" {
|
||||
t.Fatal("baseline is not retrieved Android v3", hash)
|
||||
}
|
||||
draft, err := os.ReadFile("testdata/android-resource-draft.md")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
old, err := pipeline.Parse(string(baseline))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
updated, err := pipeline.Parse(string(draft))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(old.Stages) != len(updated.Stages) {
|
||||
t.Fatal("draft changed graph shape")
|
||||
}
|
||||
for i := range old.Stages {
|
||||
a, b := old.Stages[i], updated.Stages[i]
|
||||
if b.Configs["provider"] != "openai" || b.Configs["model"] != a.Configs["model"] {
|
||||
t.Fatal("draft changed approved model")
|
||||
}
|
||||
if b.Key == "ui_reviewer" {
|
||||
if b.Resources == nil || b.Resources.Exclusive != "phone-47241FDAS00DLK" {
|
||||
t.Fatal("reviewer resource missing")
|
||||
}
|
||||
} else if b.Resources != nil {
|
||||
t.Fatal("non-UI stage acquired phone")
|
||||
}
|
||||
a.Prompt, b.Prompt = "", ""
|
||||
a.Resources, b.Resources = nil, nil
|
||||
if !reflect.DeepEqual(a, b) {
|
||||
t.Fatal("draft changed fields outside approved role split/resource", a.Key)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,60 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"time"
|
||||
|
||||
"git.a-wan.net/alan/pipelines/internal/pipeline"
|
||||
"git.a-wan.net/alan/pipelines/internal/protocol"
|
||||
"git.a-wan.net/alan/pipelines/internal/stage"
|
||||
)
|
||||
|
||||
func (r *runner) acquireResource(ctx context.Context, visit pipeline.ResourceVisit, workspace string, completed int) (pipeline.ResourceLease, error) {
|
||||
lease, ready, wait, err := r.store.RequestResource(ctx, visit)
|
||||
if err != nil {
|
||||
return lease, err
|
||||
}
|
||||
if ready != nil {
|
||||
r.emitRun(protocol.RunEventFrame{RunID: visit.RunID, Workspace: workspace, Status: protocol.RunStatusQueued, CurrentStage: visit.Stage, CompletedSteps: completed, ResourceWaitSet: true, ResourceWait: wait}, false)
|
||||
select {
|
||||
case lease = <-ready:
|
||||
r.store.AcceptResource(lease)
|
||||
case <-ctx.Done():
|
||||
cleanup, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
if err := r.store.CancelResource(cleanup, visit); err != nil {
|
||||
return lease, errors.Join(ctx.Err(), err)
|
||||
}
|
||||
return lease, ctx.Err()
|
||||
}
|
||||
}
|
||||
if ctx.Err() != nil {
|
||||
err := r.releaseResource(lease, true) // Never launched, including a concurrent grant.
|
||||
return pipeline.ResourceLease{}, errors.Join(ctx.Err(), err)
|
||||
}
|
||||
r.emitRun(protocol.RunEventFrame{RunID: visit.RunID, Workspace: workspace, ResourceWaitSet: true}, false)
|
||||
return lease, nil
|
||||
}
|
||||
|
||||
func (r *runner) releaseResource(lease pipeline.ResourceLease, confirmed bool) error {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
if err := r.store.SetResourceCleanup(ctx, lease, confirmed); err != nil {
|
||||
return err
|
||||
}
|
||||
if confirmed {
|
||||
return r.store.ReleaseResource(ctx, lease)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *runner) resourceQuiescent(id int64, executionErr error, panicked bool) bool {
|
||||
if panicked || errors.Is(executionErr, stage.ErrStageCleanup) {
|
||||
return false
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
sessions, err := r.store.ActiveAgentSessionsForRun(ctx, id)
|
||||
return err == nil && len(sessions) == 0
|
||||
}
|
||||
@@ -0,0 +1,327 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"errors"
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.a-wan.net/alan/pipelines/internal/pipeline"
|
||||
"git.a-wan.net/alan/pipelines/internal/protocol"
|
||||
"git.a-wan.net/alan/pipelines/internal/stage"
|
||||
)
|
||||
|
||||
func resourceRunner(t *testing.T) (*runner, <-chan protocol.RunEventFrame, func(string, string, string) *protocol.RunDetail) {
|
||||
t.Helper()
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
s, err := pipeline.Open(filepath.Join(t.TempDir(), "resources.db"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
r, err := newRunner(ctx, s)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
r.events = newRunEventHub()
|
||||
r.events.interval = time.Millisecond
|
||||
events := make(chan protocol.RunEventFrame, 128)
|
||||
sub := r.events.subscribe("resource-test", "", nil, func(m protocol.Message) error {
|
||||
if m.RunEvent != nil {
|
||||
events <- *m.RunEvent
|
||||
}
|
||||
return nil
|
||||
})
|
||||
sub.ready()
|
||||
t.Cleanup(func() { cancel(); r.wg.Wait(); sub.close(); s.Close() })
|
||||
count := 0
|
||||
return r, events, func(resource, timeout, prompt string) *protocol.RunDetail {
|
||||
count++
|
||||
source := strings.Replace(pipeline.Template([]string{"work"}, true), "timeout=1h", "timeout="+timeout, 1)
|
||||
if resource != "" {
|
||||
source += "```resources\n{\"exclusive\":\"" + resource + "\"}\n```\n"
|
||||
}
|
||||
name := fmt.Sprint("resource-", count)
|
||||
if _, err := s.Submit(ctx, name, source); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
items, _ := s.List(ctx)
|
||||
var pid int64
|
||||
for _, p := range items {
|
||||
if p.Name == name {
|
||||
pid = p.ID
|
||||
}
|
||||
}
|
||||
d, err := r.start(protocol.Message{PipelineID: pid, Version: 1, RequestKey: name, Workspace: t.TempDir(), Input: map[string]any{"prompt": prompt}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return d
|
||||
}
|
||||
}
|
||||
|
||||
func awaitResourceEvent(t *testing.T, events <-chan protocol.RunEventFrame, id int64, predicate func(protocol.RunEventFrame) bool) protocol.RunEventFrame {
|
||||
t.Helper()
|
||||
timer := time.NewTimer(3 * time.Second)
|
||||
defer timer.Stop()
|
||||
for {
|
||||
select {
|
||||
case event := <-events:
|
||||
if event.RunID == id && predicate(event) {
|
||||
return event
|
||||
}
|
||||
case <-timer.C:
|
||||
t.Fatalf("resource event missing for run %d", id)
|
||||
return protocol.RunEventFrame{}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestResourcePauseBeyondTimeoutAndAutomaticResume(t *testing.T) {
|
||||
r, events, start := resourceRunner(t)
|
||||
aStarted, releaseA := make(chan struct{}), make(chan struct{})
|
||||
bStarted := make(chan struct{}, 1)
|
||||
var bCalls atomic.Int32
|
||||
r.execute = func(ctx context.Context, _ stage.Definition, input stage.Values) (stage.Result, error) {
|
||||
switch input["prompt"] {
|
||||
case "A":
|
||||
close(aStarted)
|
||||
select {
|
||||
case <-releaseA:
|
||||
case <-ctx.Done():
|
||||
return stage.Result{}, ctx.Err()
|
||||
}
|
||||
case "B":
|
||||
bCalls.Add(1)
|
||||
deadline, ok := ctx.Deadline()
|
||||
if !ok || time.Until(deadline) < 15*time.Millisecond {
|
||||
return stage.Result{}, errors.New("execution timeout was consumed in queue")
|
||||
}
|
||||
bStarted <- struct{}{}
|
||||
}
|
||||
return stage.Result{Outcome: "success", Output: stage.Values{}}, nil
|
||||
}
|
||||
a := start("X", "2s", "A")
|
||||
<-aStarted
|
||||
b := start("X", "30ms", "B")
|
||||
awaitResourceEvent(t, events, b.ID, func(e protocol.RunEventFrame) bool { return e.ResourceWait != nil })
|
||||
<-time.After(70 * time.Millisecond) // Deliberately longer than B's whole execution timeout.
|
||||
d, err := r.store.GetRun(context.Background(), b.ID)
|
||||
if err != nil || d.Status != protocol.RunStatusQueued || d.ResourceWait == nil || len(d.Steps) != 0 || bCalls.Load() != 0 {
|
||||
t.Fatal("queued execution consumed accounting", d, err, bCalls.Load())
|
||||
}
|
||||
if _, err := r.decide(protocol.Message{RunID: b.ID, Option: "success"}); err == nil {
|
||||
t.Fatal("resource wait exposed a human decision")
|
||||
}
|
||||
c := start("Y", "1s", "C")
|
||||
awaitResourceEvent(t, events, c.ID, func(e protocol.RunEventFrame) bool { return e.Status == "succeeded" })
|
||||
select {
|
||||
case <-bStarted:
|
||||
t.Fatal("B ran while A held X")
|
||||
default:
|
||||
}
|
||||
owner, err := r.store.GetRun(context.Background(), a.ID)
|
||||
if err != nil || owner.Status != "running" {
|
||||
t.Fatal("A did not remain owner", err)
|
||||
}
|
||||
close(releaseA)
|
||||
awaitResourceEvent(t, events, b.ID, func(e protocol.RunEventFrame) bool { return e.Status == "succeeded" })
|
||||
r.wg.Wait()
|
||||
d, err = r.store.GetRun(context.Background(), b.ID)
|
||||
if err != nil || bCalls.Load() != 1 || d.ResourceWait != nil || len(d.Steps) != 1 || d.Steps[0].Control.Visit != 1 || d.Steps[0].Sequence != 1 {
|
||||
t.Fatal("resume did not execute exactly one fresh visit", d, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResourceCancellationWaitsForOwnerCleanup(t *testing.T) {
|
||||
r, events, start := resourceRunner(t)
|
||||
started, cleaning, cleanupDone := make(chan struct{}), make(chan struct{}), make(chan struct{})
|
||||
var waitingCalls atomic.Int32
|
||||
r.execute = func(ctx context.Context, _ stage.Definition, input stage.Values) (stage.Result, error) {
|
||||
if input["prompt"] == "owner" {
|
||||
close(started)
|
||||
<-ctx.Done()
|
||||
close(cleaning)
|
||||
<-cleanupDone
|
||||
return stage.Result{}, ctx.Err()
|
||||
}
|
||||
waitingCalls.Add(1)
|
||||
return stage.Result{Outcome: "success", Output: stage.Values{}}, nil
|
||||
}
|
||||
a := start("X", "2s", "owner")
|
||||
<-started
|
||||
b := start("X", "30ms", "waiter")
|
||||
awaitResourceEvent(t, events, b.ID, func(e protocol.RunEventFrame) bool { return e.ResourceWait != nil })
|
||||
if err := r.cancel(a.ID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
<-cleaning
|
||||
select {
|
||||
case <-time.After(50 * time.Millisecond):
|
||||
if waitingCalls.Load() != 0 {
|
||||
t.Fatal("handoff before cleanup")
|
||||
}
|
||||
}
|
||||
close(cleanupDone)
|
||||
awaitResourceEvent(t, events, b.ID, func(e protocol.RunEventFrame) bool { return e.Status == "succeeded" })
|
||||
if waitingCalls.Load() != 1 {
|
||||
t.Fatal("waiter did not resume exactly once")
|
||||
}
|
||||
}
|
||||
|
||||
func TestResourceQuarantineAndQueuedCancellation(t *testing.T) {
|
||||
for _, failure := range []string{"cleanup", "panic", "restoration"} {
|
||||
t.Run(failure, func(t *testing.T) {
|
||||
r, events, start := resourceRunner(t)
|
||||
started, release := make(chan struct{}), make(chan struct{})
|
||||
var calls atomic.Int32
|
||||
r.execute = func(ctx context.Context, _ stage.Definition, input stage.Values) (stage.Result, error) {
|
||||
if input["prompt"] == "owner" {
|
||||
close(started)
|
||||
select {
|
||||
case <-release:
|
||||
case <-ctx.Done():
|
||||
}
|
||||
if failure == "panic" {
|
||||
panic("unknown cleanup")
|
||||
}
|
||||
if failure == "restoration" {
|
||||
return stage.Result{Outcome: "success", Output: stage.Values{protocol.ResourceCleanupConfirmedKey: false}}, nil
|
||||
}
|
||||
return stage.Result{}, stage.ErrStageCleanup
|
||||
}
|
||||
calls.Add(1)
|
||||
return stage.Result{Outcome: "success", Output: stage.Values{}}, nil
|
||||
}
|
||||
a := start("X", "1s", "owner")
|
||||
<-started
|
||||
b := start("X", "30ms", "waiter")
|
||||
awaitResourceEvent(t, events, b.ID, func(e protocol.RunEventFrame) bool { return e.ResourceWait != nil })
|
||||
close(release)
|
||||
awaitResourceEvent(t, events, a.ID, func(e protocol.RunEventFrame) bool { return e.Status == "failed" })
|
||||
if err := r.cancel(b.ID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
awaitResourceEvent(t, events, b.ID, func(e protocol.RunEventFrame) bool {
|
||||
return e.Status == "canceled" && e.ResourceWaitSet && e.ResourceWait == nil
|
||||
})
|
||||
r.wg.Wait()
|
||||
if calls.Load() != 0 {
|
||||
t.Fatal("quarantined resource handed off")
|
||||
}
|
||||
exists, err := r.store.HasResourceOwner(context.Background(), a.ID)
|
||||
if err != nil || !exists {
|
||||
t.Fatal("quarantine evidence lost", err)
|
||||
}
|
||||
d, err := r.store.GetRun(context.Background(), b.ID)
|
||||
if err != nil || d.ResourceWait != nil || len(d.Steps) != 0 {
|
||||
t.Fatal("canceled queue consumed step", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestResourceEventClearCoalescing(t *testing.T) {
|
||||
frame := protocol.RunEventFrame{ResourceWaitSet: true, ResourceWait: &protocol.ResourceWait{Resource: "X"}}
|
||||
mergeRunEventFrame(&frame, &protocol.RunEventFrame{ResourceWaitSet: true})
|
||||
if !frame.ResourceWaitSet || frame.ResourceWait != nil {
|
||||
t.Fatal("coalescer retained stale resource wait")
|
||||
}
|
||||
}
|
||||
|
||||
func TestResourceReleaseOnBeginAndPersistenceFailure(t *testing.T) {
|
||||
for _, failure := range []string{"begin", "persist"} {
|
||||
t.Run(failure, func(t *testing.T) {
|
||||
r, events, start := resourceRunner(t)
|
||||
db, err := sql.Open("sqlite", filepath.Join(r.store.Directory(), "resources.db"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer db.Close()
|
||||
trigger := `CREATE TRIGGER resource_fail BEFORE INSERT ON run_steps WHEN json_extract(NEW.input_json,'$.prompt')='fail' BEGIN SELECT RAISE(FAIL,'test BeginStep failure'); END`
|
||||
if failure == "persist" {
|
||||
trigger = `CREATE TRIGGER resource_fail BEFORE UPDATE OF output_json ON run_steps WHEN json_extract(OLD.input_json,'$.prompt')='fail' BEGIN SELECT RAISE(FAIL,'test persistence failure'); END`
|
||||
}
|
||||
if _, err := db.Exec(trigger); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var calls atomic.Int32
|
||||
r.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) {
|
||||
calls.Add(1)
|
||||
return stage.Result{Outcome: "success", Output: stage.Values{}}, nil
|
||||
}
|
||||
a := start("X", "1s", "fail")
|
||||
awaitResourceEvent(t, events, a.ID, func(e protocol.RunEventFrame) bool { return e.Status == "failed" })
|
||||
r.wg.Wait()
|
||||
if exists, err := r.store.HasResourceOwner(context.Background(), a.ID); err != nil || exists {
|
||||
t.Fatal("safe failure stranded resource", err)
|
||||
}
|
||||
if failure == "begin" && calls.Load() != 0 {
|
||||
t.Fatal("BeginStep failure executed work")
|
||||
}
|
||||
b := start("X", "1s", "pass")
|
||||
awaitResourceEvent(t, events, b.ID, func(e protocol.RunEventFrame) bool { return e.Status == "succeeded" })
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestResourceCleanupQueryFailureQuarantines(t *testing.T) {
|
||||
r, events, start := resourceRunner(t)
|
||||
r.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) {
|
||||
if err := r.store.Close(); err != nil {
|
||||
return stage.Result{}, err
|
||||
}
|
||||
return stage.Result{Outcome: "success", Output: stage.Values{}}, nil
|
||||
}
|
||||
a := start("X", "1s", "query-failure")
|
||||
awaitResourceEvent(t, events, a.ID, func(e protocol.RunEventFrame) bool { return e.Status == "failed" })
|
||||
r.wg.Wait()
|
||||
reopened, err := pipeline.Open(filepath.Join(r.store.Directory(), "resources.db"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer reopened.Close()
|
||||
if err := reopened.RecoverRuns(context.Background()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if exists, err := reopened.HasResourceOwner(context.Background(), a.ID); err != nil || !exists {
|
||||
t.Fatal("query failure released uncertain owner", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResourceSkippedVisitDoesNotAcquire(t *testing.T) {
|
||||
r, events, _ := resourceRunner(t)
|
||||
ctx := context.Background()
|
||||
source := strings.Replace(pipeline.Template([]string{"review"}, true), "success -> #success", "success -> #success\nskipped -> #success", 1) + "```resources\n{\"exclusive\":\"X\"}\n```\n"
|
||||
if _, err := r.store.Submit(ctx, "skip", source); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
items, _ := r.store.List(ctx)
|
||||
d, _ := r.store.Get(ctx, items[0].ID, 1)
|
||||
owner, _, err := r.store.CreateRun(ctx, d, "held", t.TempDir(), false, nil, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
lease, ch, _, err := r.store.RequestResource(ctx, pipeline.ResourceVisit{Resource: "X", RunID: owner, Stage: "review", Sequence: 1})
|
||||
if err != nil || ch != nil {
|
||||
t.Fatal("owner fixture failed", err)
|
||||
}
|
||||
defer r.releaseResource(lease, true)
|
||||
r.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) {
|
||||
t.Error("skipped stage executed")
|
||||
return stage.Result{}, nil
|
||||
}
|
||||
run, err := r.start(protocol.Message{PipelineID: d.ID, Version: 1, RequestKey: "skip", Workspace: t.TempDir(), Input: map[string]any{"prompt": "no UI"}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
awaitResourceEvent(t, events, run.ID, func(e protocol.RunEventFrame) bool { return e.Status == "succeeded" })
|
||||
detail, err := r.store.GetRun(ctx, run.ID)
|
||||
if err != nil || len(detail.Steps) != 1 || detail.Steps[0].Outcome != "skipped" {
|
||||
t.Fatal("skip not preserved", detail, err)
|
||||
}
|
||||
}
|
||||
@@ -29,6 +29,7 @@ func TestRetryCreatesFreshOpenCodeSessionAndRetainsFailedSession(t *testing.T) {
|
||||
if [ "$1" = "--help" ]; then echo ' api'; exit 0; fi
|
||||
printf '%%s %%s %%s\n' "$PWD" "$2" "$3" >> %q
|
||||
case "$3" in
|
||||
/api/session/active) echo '{"data":{}}' ;;
|
||||
/api/session)
|
||||
n=0; if [ -f %q ]; then n=$(cat %q); fi
|
||||
n=$((n+1)); echo "$n" > %q
|
||||
|
||||
@@ -424,6 +424,9 @@ func (s *runSubscription) close() {
|
||||
}
|
||||
|
||||
func mergeRunEventFrame(into, in *protocol.RunEventFrame) {
|
||||
if in.ResourceWaitSet {
|
||||
into.ResourceWaitSet, into.ResourceWait = true, in.ResourceWait
|
||||
}
|
||||
if in.Workspace != "" {
|
||||
into.Workspace = in.Workspace
|
||||
}
|
||||
@@ -500,18 +503,20 @@ func handleRunEvents(request protocol.Message, events *runEventHub, runs *runner
|
||||
continue
|
||||
}
|
||||
frame := protocol.RunEventFrame{
|
||||
RunID: summary.ID,
|
||||
Workspace: summary.Workspace,
|
||||
PipelineID: summary.PipelineID,
|
||||
Name: summary.Name,
|
||||
Title: summary.Title,
|
||||
Version: summary.Version,
|
||||
Status: summary.Status,
|
||||
Outcome: summary.Outcome,
|
||||
CurrentStage: summary.CurrentStage,
|
||||
CompletedSteps: summary.CompletedSteps,
|
||||
CreatedAt: summary.CreatedAt,
|
||||
FinishedAt: summary.FinishedAt,
|
||||
ResourceWaitSet: true,
|
||||
ResourceWait: summary.ResourceWait,
|
||||
RunID: summary.ID,
|
||||
Workspace: summary.Workspace,
|
||||
PipelineID: summary.PipelineID,
|
||||
Name: summary.Name,
|
||||
Title: summary.Title,
|
||||
Version: summary.Version,
|
||||
Status: summary.Status,
|
||||
Outcome: summary.Outcome,
|
||||
CurrentStage: summary.CurrentStage,
|
||||
CompletedSteps: summary.CompletedSteps,
|
||||
CreatedAt: summary.CreatedAt,
|
||||
FinishedAt: summary.FinishedAt,
|
||||
}
|
||||
if !sub.writeSnapshot(protocol.Message{Type: protocol.RunEvent, RunEvent: &frame}) {
|
||||
// The connection write failed; the subscription is closed and the
|
||||
|
||||
+210
-27
@@ -58,8 +58,11 @@ type runner struct {
|
||||
// directory. It defaults to stage.ExportSession; a failure is best-effort and
|
||||
// never changes the stage's status, outcome, target or result validation.
|
||||
exportSession func(context.Context, stage.SessionExportRequest) (stage.SessionExportResult, error)
|
||||
wg sync.WaitGroup
|
||||
schedulerWG sync.WaitGroup
|
||||
// verifyTarget is installed only by server wiring for a controlled fixture
|
||||
// workflow. Leaving it nil fails closed for target-aware definitions.
|
||||
verifyTarget stage.VerificationRunner
|
||||
wg sync.WaitGroup
|
||||
schedulerWG sync.WaitGroup
|
||||
// attentionMu guards attentionSent, the per-run set of run-attention events
|
||||
// already delivered — by a stage's own notify policy or by the run-level
|
||||
// default — so neither republishes the same genuine event.
|
||||
@@ -78,12 +81,19 @@ func newRunner(ctx context.Context, s *pipeline.Store) (*runner, error) {
|
||||
if owner.Backend == "pi" {
|
||||
err = stage.InterruptPi(stop, owner.Reference)
|
||||
} else {
|
||||
err = (stage.OpenCodeClient{Backend: owner.Backend, Directory: owner.Directory, Reference: owner.Reference}).Interrupt(stop, owner.ID)
|
||||
err = (stage.OpenCodeClient{Backend: owner.Backend, Directory: owner.Directory, Reference: owner.Reference}).InterruptAndConfirm(stop, owner.ID)
|
||||
}
|
||||
cancel()
|
||||
if err != nil {
|
||||
if resource, checkErr := s.HasResourceOwner(ctx, owner.RunID); checkErr == nil && resource {
|
||||
log.Printf("resource owner run=%d remains quarantined: cleanup not confirmed", owner.RunID)
|
||||
continue
|
||||
}
|
||||
return nil, fmt.Errorf("cannot interrupt previously active agent %s: %w", owner.ID, err)
|
||||
}
|
||||
if err := s.ConfirmRecoveredSession(ctx, owner); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
if err := s.RecoverRuns(ctx); err != nil {
|
||||
return nil, err
|
||||
@@ -213,6 +223,9 @@ func (r *runner) start(request protocol.Message, retryOf ...int64) (*protocol.Ru
|
||||
// workspace) must not block new work.
|
||||
for _, owner := range owners {
|
||||
if _, ok := r.active[owner.RunID]; !ok {
|
||||
if resource, checkErr := r.store.HasResourceOwner(r.ctx, owner.RunID); checkErr == nil && resource && owner.Directory != workspace {
|
||||
continue
|
||||
}
|
||||
r.mu.Unlock()
|
||||
return nil, fmt.Errorf("an agent may still be active; restart recovery must interrupt it before new work")
|
||||
}
|
||||
@@ -580,7 +593,8 @@ type runState struct {
|
||||
capturedSeen map[string]bool
|
||||
// completed counts the run's finished steps, so a delta can report the
|
||||
// running total without re-reading the store.
|
||||
completed int
|
||||
completed int
|
||||
artifactDirs []string
|
||||
}
|
||||
|
||||
func newRunState(key string, input stage.Values) *runState {
|
||||
@@ -606,10 +620,67 @@ func rebuildRunState(detail *protocol.RunDetail, decided protocol.RunStep) *runS
|
||||
// Seed the UI-surface accumulator from the persisted evidence so a
|
||||
// resumed run reaches the same skip decision with no extra query.
|
||||
state.recordChanges(step.FileChanges, step.Artifacts)
|
||||
// Only completed, recorded visits are eligible for a later stage's
|
||||
// review capability. The path is reconstructed by the runner below.
|
||||
if step.Status == "completed" {
|
||||
state.artifactDirs = append(state.artifactDirs, filepath.Join("", strconv.Itoa(step.Sequence)))
|
||||
}
|
||||
}
|
||||
return state
|
||||
}
|
||||
|
||||
// artifactAccess derives exact visit directories; it never consumes a path from
|
||||
// a definition or model result. An unreadable or symlinked root fails closed.
|
||||
func (r *runner) artifactAccess(runID int64, sequence int, prior []string, current string) stage.ArtifactAccess {
|
||||
root, err := filepath.EvalSymlinks(r.artifactsRoot)
|
||||
if err != nil {
|
||||
return stage.ArtifactAccess{}
|
||||
}
|
||||
runRoot := filepath.Join(root, strconv.FormatInt(runID, 10))
|
||||
visitDirectory := func(visit int) (string, bool) {
|
||||
if visit <= 0 {
|
||||
return "", false
|
||||
}
|
||||
directory := filepath.Join(runRoot, strconv.Itoa(visit))
|
||||
resolved, err := filepath.EvalSymlinks(directory)
|
||||
if err != nil || resolved != directory {
|
||||
return "", false
|
||||
}
|
||||
info, err := os.Stat(resolved)
|
||||
if err != nil || !info.IsDir() {
|
||||
return "", false
|
||||
}
|
||||
return resolved, true
|
||||
}
|
||||
access := stage.ArtifactAccess{}
|
||||
if directory, ok := visitDirectory(sequence); ok {
|
||||
resolved, err := filepath.EvalSymlinks(current)
|
||||
if err != nil || resolved != directory {
|
||||
return access
|
||||
}
|
||||
access.Write = directory
|
||||
access.Read = append(access.Read, directory)
|
||||
}
|
||||
for _, priorSequence := range prior {
|
||||
visit, err := strconv.Atoi(priorSequence)
|
||||
if err != nil || visit <= 0 || visit >= sequence {
|
||||
continue
|
||||
}
|
||||
if directory, ok := visitDirectory(visit); ok {
|
||||
access.Read = append(access.Read, directory)
|
||||
}
|
||||
}
|
||||
return access
|
||||
}
|
||||
|
||||
func (r *runner) agentClient(runID int64, sequence int, prior []string, workspace, artifactsDir string, operations *stage.OperationLog, writePaths, tools []string) *stage.AgentClient {
|
||||
return &stage.AgentClient{OpenCodeClient: stage.OpenCodeClient{Directory: workspace, Operations: operations, ArtifactsDir: artifactsDir}, WritePaths: writePaths, Tools: tools, ArtifactAccess: r.artifactAccess(runID, sequence, prior, artifactsDir)}
|
||||
}
|
||||
|
||||
func (r *runner) piClient(runID int64, sequence int, prior []string, executable, workspace, artifactsDir string, writePaths, tools []string) *stage.PiClient {
|
||||
return &stage.PiClient{Executable: executable, Directory: workspace, Tools: tools, WritePaths: writePaths, ArtifactsDir: artifactsDir, ArtifactAccess: r.artifactAccess(runID, sequence, prior, artifactsDir)}
|
||||
}
|
||||
|
||||
// serverChangedFilesField is the reserved run-input memory field under which the
|
||||
// runner publishes the server-captured changed paths. A stage selects it with
|
||||
// `"server_changed_files":"input.server_changed_files"`, alongside another
|
||||
@@ -717,13 +788,14 @@ func (r *runner) finishRunState(id int64, name, workspace, status, outcome, diag
|
||||
// Push the terminal delta before the attention notification, so a slow ntfy
|
||||
// never delays the live update the app folds.
|
||||
r.emitRun(protocol.RunEventFrame{
|
||||
RunID: id,
|
||||
Workspace: workspace,
|
||||
Status: status,
|
||||
Outcome: outcome,
|
||||
CurrentStage: currentStage,
|
||||
CompletedSteps: completed,
|
||||
FinishedAt: time.Now().UTC().Format(time.RFC3339),
|
||||
RunID: id,
|
||||
Workspace: workspace,
|
||||
Status: status,
|
||||
ResourceWaitSet: true,
|
||||
Outcome: outcome,
|
||||
CurrentStage: currentStage,
|
||||
CompletedSteps: completed,
|
||||
FinishedAt: time.Now().UTC().Format(time.RFC3339),
|
||||
}, true)
|
||||
r.notifyRun(id, name, status, sequence, currentStage, diagnostic)
|
||||
// The run is terminal: nothing more can notify for it.
|
||||
@@ -791,6 +863,21 @@ func (r *runner) finishStage(id int64, name, workspace string, selected protocol
|
||||
}
|
||||
|
||||
func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, name, workspace string, request protocol.Message, state *runState) {
|
||||
var held *pipeline.ResourceLease
|
||||
resourceQuiet := true
|
||||
release := func() error {
|
||||
if held != nil {
|
||||
err := r.releaseResource(*held, resourceQuiet)
|
||||
held = nil
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
defer func() {
|
||||
if err := release(); err != nil {
|
||||
log.Printf("resource cleanup run=%d: %v", id, err)
|
||||
}
|
||||
}()
|
||||
finish := func(status, outcome, diagnostic string) {
|
||||
r.finishRunState(id, name, workspace, status, outcome, diagnostic, state.completed, state.nextSequence-1, state.key, state.input)
|
||||
}
|
||||
@@ -819,7 +906,6 @@ func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, na
|
||||
effectiveModels = resolved
|
||||
}
|
||||
for sequence := state.nextSequence; sequence <= 30; sequence++ {
|
||||
state.nextSequence = sequence + 1
|
||||
if ctx.Err() != nil {
|
||||
finish("canceled", "", ctx.Err().Error())
|
||||
return
|
||||
@@ -851,24 +937,42 @@ func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, na
|
||||
}
|
||||
}
|
||||
isHuman := stage.EffectiveType(def.Type) == stage.HumanType
|
||||
state.visits[key]++
|
||||
maxVisits := 5
|
||||
if selected.Limits != nil {
|
||||
maxVisits = selected.Limits.MaxVisits
|
||||
}
|
||||
// A human stage pauses for a person; neither the global visit budget nor
|
||||
// the per-stage max_visits may kill it. The shared 30-step run budget
|
||||
// still bounds every run.
|
||||
// Validate the next visit without consuming it. An already-exhausted
|
||||
// stage must not queue for a resource it can never execute.
|
||||
if !isHuman {
|
||||
if state.visits[key] > 5 {
|
||||
if state.visits[key]+1 > 5 {
|
||||
finish("failed", "error", "stage visit budget exceeded")
|
||||
return
|
||||
}
|
||||
if state.visits[key] > maxVisits {
|
||||
if state.visits[key]+1 > maxVisits {
|
||||
finish("failed", "error", fmt.Sprintf("stage %s visit limit reached (%d)", key, maxVisits))
|
||||
return
|
||||
}
|
||||
}
|
||||
_, declaresSkip := def.Edges["skipped"]
|
||||
skip := !isHuman && declaresSkip && !state.uiSurface
|
||||
if selected.Resources != nil && !skip {
|
||||
if isHuman || !pipeline.ValidResourceKey(selected.Resources.Exclusive) {
|
||||
finish("failed", "error", "invalid persisted resource declaration")
|
||||
return
|
||||
}
|
||||
lease, err := r.acquireResource(ctx, pipeline.ResourceVisit{Resource: selected.Resources.Exclusive, RunID: id, Stage: key, Sequence: sequence}, workspace, state.completed)
|
||||
if err != nil {
|
||||
if ctx.Err() != nil {
|
||||
finish("canceled", "", err.Error())
|
||||
} else {
|
||||
finish("failed", "error", err.Error())
|
||||
}
|
||||
return
|
||||
}
|
||||
held, resourceQuiet = &lease, true
|
||||
}
|
||||
state.nextSequence = sequence + 1
|
||||
state.visits[key]++
|
||||
// Publish the run's server-captured changed paths into the input memory
|
||||
// value before resolving bindings, so a stage selecting
|
||||
// input.server_changed_files always resolves (an empty list when nothing
|
||||
@@ -890,12 +994,13 @@ func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, na
|
||||
}
|
||||
// queued -> running: report the new current stage and the step start.
|
||||
r.emitRun(protocol.RunEventFrame{
|
||||
RunID: id,
|
||||
Workspace: workspace,
|
||||
Status: protocol.RunStatusRunning,
|
||||
CurrentStage: key,
|
||||
CompletedSteps: state.completed,
|
||||
Step: &protocol.RunStepDelta{Sequence: sequence, Stage: key, Status: protocol.StepStatusRunning},
|
||||
RunID: id,
|
||||
Workspace: workspace,
|
||||
Status: protocol.RunStatusRunning,
|
||||
CurrentStage: key,
|
||||
ResourceWaitSet: true,
|
||||
CompletedSteps: state.completed,
|
||||
Step: &protocol.RunStepDelta{Sequence: sequence, Stage: key, Status: protocol.StepStatusRunning},
|
||||
}, false)
|
||||
if isHuman {
|
||||
// A human step is O(1): one BeginStep plus one MarkWaiting, no
|
||||
@@ -929,7 +1034,7 @@ func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, na
|
||||
// the model call and its capture work, record the step as completed and
|
||||
// route along the declared edge. Only a declared route is honoured, and no
|
||||
// other stage or outcome routing changes.
|
||||
if _, declaresSkip := def.Edges["skipped"]; declaresSkip && !state.uiSurface {
|
||||
if skip {
|
||||
reason := skipReasonLine(state)
|
||||
step := protocol.RunStep{
|
||||
Sequence: sequence,
|
||||
@@ -977,7 +1082,22 @@ func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, na
|
||||
}
|
||||
}
|
||||
operations := &stage.OperationLog{}
|
||||
client := &stage.AgentClient{OpenCodeClient: stage.OpenCodeClient{Directory: workspace, Operations: operations, ArtifactsDir: artifactsDir}, WritePaths: request.WritePaths, Tools: tools}
|
||||
var landing stage.LandingAccess
|
||||
delivery, deliveryErr := stage.DeliveryEnabled(def)
|
||||
if deliveryErr != nil {
|
||||
finish("failed", "error", deliveryErr.Error())
|
||||
return
|
||||
}
|
||||
if delivery {
|
||||
access, accessErr := landingAccess(workspace)
|
||||
if accessErr != nil {
|
||||
finish("failed", "error", "delivery access denied: "+accessErr.Error())
|
||||
return
|
||||
}
|
||||
landing = access
|
||||
}
|
||||
client := r.agentClient(id, sequence, state.artifactDirs, workspace, artifactsDir, operations, request.WritePaths, tools)
|
||||
client.LandingAccess = landing
|
||||
client.OnSession = func(session string, active bool) error {
|
||||
c, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
@@ -1010,6 +1130,9 @@ func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, na
|
||||
snapshotCancel()
|
||||
var stageSession string
|
||||
panicked := false
|
||||
if held != nil {
|
||||
resourceQuiet = false
|
||||
}
|
||||
func() {
|
||||
defer func() {
|
||||
if recover() != nil {
|
||||
@@ -1017,6 +1140,10 @@ func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, na
|
||||
err = fmt.Errorf("stage execution panicked")
|
||||
}
|
||||
}()
|
||||
if ctx.Err() != nil {
|
||||
err = ctx.Err()
|
||||
return
|
||||
}
|
||||
if r.execute != nil {
|
||||
timeout, _ := stage.StageTimeout(def)
|
||||
executeCtx, cancel := context.WithTimeout(stageCtx, timeout)
|
||||
@@ -1033,7 +1160,7 @@ func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, na
|
||||
err = fmt.Errorf("Pi is not configured on execution server")
|
||||
return
|
||||
}
|
||||
piClient = &stage.PiClient{Executable: profile.Executable, Directory: workspace, Tools: tools, WritePaths: request.WritePaths, ArtifactsDir: artifactsDir}
|
||||
piClient = r.piClient(id, sequence, state.artifactDirs, profile.Executable, workspace, artifactsDir, request.WritePaths, tools)
|
||||
piClient.OnOwnership = func(session, reference string, active bool) error {
|
||||
c, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
@@ -1061,6 +1188,12 @@ func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, na
|
||||
result, err = stage.DefaultRegistry().Run(ctx, def, input)
|
||||
}
|
||||
}()
|
||||
if held != nil {
|
||||
resourceQuiet = r.resourceQuiescent(id, err, panicked)
|
||||
if confirmed, reported := result.Output[protocol.ResourceCleanupConfirmedKey]; reported && confirmed != true {
|
||||
resourceQuiet = false
|
||||
}
|
||||
}
|
||||
// Capture after cleanup, even when the stage's context was canceled.
|
||||
snapshotCtx, snapshotCancel = context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second)
|
||||
after := filediff.Capture(snapshotCtx, workspace, before)
|
||||
@@ -1107,6 +1240,45 @@ func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, na
|
||||
step.Output["salvage"] = result.Output
|
||||
}
|
||||
}
|
||||
if err == nil && result.Outcome == "success" {
|
||||
target, targetErr := stage.VerificationTargetFor(def)
|
||||
if targetErr != nil {
|
||||
err = targetErr
|
||||
} else if target != nil {
|
||||
if r.verifyTarget == nil {
|
||||
err = fmt.Errorf("verification target %q requires a server-controlled fixture runner", target.Fixture)
|
||||
} else {
|
||||
evidence, verificationErr := r.verifyTarget(stageCtx, *target)
|
||||
if verificationErr != nil {
|
||||
err = verificationErr
|
||||
} else {
|
||||
err = stage.VerifyTargetEvidence(*target, evidence)
|
||||
}
|
||||
}
|
||||
}
|
||||
if err != nil {
|
||||
step.Status = "failed"
|
||||
step.Diagnostic = err.Error()
|
||||
step.Outcome = "error"
|
||||
step.Output = stage.Values{"error": err.Error(), "stage": key, "input": input}
|
||||
}
|
||||
}
|
||||
if err == nil && delivery && result.Outcome == "success" {
|
||||
checks, checksErr := stage.DeliveryChecks(def)
|
||||
if checksErr != nil {
|
||||
err = checksErr
|
||||
} else {
|
||||
gateCtx, gateCancel := context.WithTimeout(stageCtx, modelConfig.Timeout)
|
||||
err = verifyDelivery(gateCtx, workspace, landing, checks)
|
||||
gateCancel()
|
||||
}
|
||||
if err != nil {
|
||||
step.Status = "failed"
|
||||
step.Diagnostic = err.Error()
|
||||
step.Outcome = "error"
|
||||
step.Output = stage.Values{"error": err.Error(), "stage": key, "input": input}
|
||||
}
|
||||
}
|
||||
// A failed or timed-out step carries the session pointer even when the
|
||||
// bounded transcript would otherwise be only its stage_limits envelope.
|
||||
if err != nil && stageSession != "" {
|
||||
@@ -1148,6 +1320,9 @@ func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, na
|
||||
state.recordChanges(step.FileChanges, step.Artifacts)
|
||||
if step.Status == "completed" {
|
||||
state.completed++
|
||||
if artifactsDir != "" {
|
||||
state.artifactDirs = append(state.artifactDirs, strconv.Itoa(sequence))
|
||||
}
|
||||
}
|
||||
// Step finished: report its outcome, routing target, diagnostic and the
|
||||
// artifacts recorded for the visit, plus the run's running step count.
|
||||
@@ -1175,6 +1350,14 @@ func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, na
|
||||
return
|
||||
}
|
||||
}
|
||||
if held != nil && !resourceQuiet {
|
||||
finish("failed", "error", "resource cleanup/restoration not confirmed; ownership quarantined pending inspection")
|
||||
return
|
||||
}
|
||||
if err := release(); err != nil {
|
||||
finish("failed", "error", "resource release failed; retained ownership needs reconciliation: "+err.Error())
|
||||
return
|
||||
}
|
||||
if ctx.Err() != nil {
|
||||
finish("canceled", "", ctx.Err().Error())
|
||||
return
|
||||
|
||||
@@ -0,0 +1,138 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.a-wan.net/alan/pipelines/internal/cli"
|
||||
"git.a-wan.net/alan/pipelines/internal/pipeline"
|
||||
"git.a-wan.net/alan/pipelines/internal/protocol"
|
||||
)
|
||||
|
||||
// Exercise the real V2 CLI/poll path, runner, SQLite retention and websocket CLI
|
||||
// against an isolated loopback server/database, never the configured deployment.
|
||||
func TestStageDeadlineDiagnosticThroughCLIAndWebsocket(t *testing.T) {
|
||||
ntfy := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
_, _ = w.Write([]byte(`{"id":"scratch-receipt","event":"message"}`))
|
||||
}))
|
||||
defer ntfy.Close()
|
||||
t.Setenv("PIPELINES_NTFY_URL", ntfy.URL+"/scratch")
|
||||
t.Setenv("PIPELINES_NTFY_TOKEN", "")
|
||||
t.Setenv("PIPELINES_NTFY_TOKEN_FILE", "")
|
||||
for _, mode := range []string{"active", "http", "connection"} {
|
||||
t.Run(mode, func(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
dir := t.TempDir()
|
||||
executable := filepath.Join(dir, "opencode")
|
||||
script := `#!/bin/sh
|
||||
if [ "$1" = "--help" ]; then echo ' api'; exit 0; fi
|
||||
printf '%s %s\n' "$2" "$3" >> "@STATE@/calls"
|
||||
case "$3" in
|
||||
/api/session) echo '{"data":{"id":"ses_deadline"}}' ;;
|
||||
*/prompt) echo 'edited once' >> "@STATE@/edited"; echo '{}' ;;
|
||||
/api/session/active)
|
||||
if [ -f "@STATE@/interrupted" ]; then echo '{"data":{}}'; exit 0; fi
|
||||
case "@MODE@" in
|
||||
active) echo '{"data":{"ses_deadline":{"type":"running"}}}' ;;
|
||||
http) echo 'HTTP 503 token=private-token' >&2; exit 1 ;;
|
||||
connection) echo 'connection refused token=private-token' >&2; exit 1 ;;
|
||||
esac ;;
|
||||
*/interrupt) touch "@STATE@/interrupted"; echo '{}' ;;
|
||||
*/message*) echo '{"data":[{"content":[{"type":"text","text":"still working"}]}]}' ;;
|
||||
/api/session/ses_deadline) echo '{"data":{"id":"ses_deadline"}}' ;;
|
||||
*) echo '{}' ;;
|
||||
esac
|
||||
`
|
||||
script = strings.ReplaceAll(strings.ReplaceAll(script, "@STATE@", dir), "@MODE@", mode)
|
||||
if err := os.WriteFile(executable, []byte(script), 0700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Setenv("PIPELINES_OPENCODE", executable)
|
||||
store, err := pipeline.Open(filepath.Join(dir, "runs.db"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer store.Close()
|
||||
source := pipeline.Template([]string{"worker"}, true)
|
||||
source = strings.Replace(source, "```tools\n```", "```tools\nedit\n```", 1)
|
||||
source = strings.Replace(source, "timeout=1h", "timeout=500ms", 1)
|
||||
if _, err := store.Submit(ctx, "deadline", source); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
handler := Handler(ctx, store)
|
||||
defer func() { cancel(); WaitRuns(handler) }()
|
||||
srv := httptest.NewServer(handler)
|
||||
defer srv.Close()
|
||||
url := "ws" + strings.TrimPrefix(srv.URL, "http") + "/connect"
|
||||
var out, stderr bytes.Buffer
|
||||
app := cli.App{Out: &out, Err: &stderr, In: strings.NewReader("")}
|
||||
if err := app.Run(ctx, []string{"--url", url, "run", "deadline", "task", "--yes", "--json", "--wait", "--timeout", "10s", "--workspace", dir}); err == nil || !strings.Contains(err.Error(), "run ended failed") {
|
||||
t.Fatalf("run not failed: %v, %s, %s", err, out.String(), stderr.String())
|
||||
}
|
||||
var response protocol.Message
|
||||
if err := json.Unmarshal(out.Bytes(), &response); err != nil {
|
||||
t.Fatal(err, out.String())
|
||||
}
|
||||
detail := response.RunDetail
|
||||
if detail == nil || detail.Status != "failed" || len(detail.Steps) != 1 {
|
||||
t.Fatal("missing failed run/step", out.String())
|
||||
}
|
||||
step := detail.Steps[0]
|
||||
if step.SessionID != "ses_deadline" || step.AgentActive || step.Output["error"] != step.Diagnostic || step.Status != "failed" || detail.Diagnostic != step.Diagnostic {
|
||||
t.Fatalf("incorrect retained failure: %+v", detail)
|
||||
}
|
||||
wants := []string{"HARNESS TRANSPORT", "get /api/session/active", "CLI exit code 1"}
|
||||
if mode == "active" {
|
||||
wants = []string{"STAGE DEADLINE", "timeout 500ms", "waiting for session idle", "ses_deadline", "stage had not finished"}
|
||||
if strings.Contains(detail.Diagnostic, "HARNESS TRANSPORT") || strings.Contains(detail.Diagnostic, "response lost") {
|
||||
t.Fatal("active stage overrun reported as transport", detail.Diagnostic)
|
||||
}
|
||||
if strings.Contains(detail.Diagnostic, "\n") {
|
||||
t.Fatal("duplicate context error appended to bounded stage diagnostic", detail.Diagnostic)
|
||||
}
|
||||
for _, want := range []string{"opencode_failures", "wait for session idle", "timeout 500ms", "retained_session", "still working"} {
|
||||
if !strings.Contains(step.Transcript, want) {
|
||||
t.Fatalf("missing retained evidence %q: %s", want, step.Transcript)
|
||||
}
|
||||
}
|
||||
} else {
|
||||
if !strings.Contains(step.Transcript, "[REDACTED]") {
|
||||
t.Fatal("request failure evidence missing", step.Transcript)
|
||||
}
|
||||
if mode == "http" {
|
||||
wants = append(wants, "HTTP 503")
|
||||
} else if !strings.Contains(step.Transcript, "connection refused") {
|
||||
t.Fatal("underlying connection evidence missing", step.Transcript)
|
||||
}
|
||||
}
|
||||
for _, want := range wants {
|
||||
if !strings.Contains(detail.Diagnostic, want) {
|
||||
t.Fatalf("missing diagnostic %q: %s", want, detail.Diagnostic)
|
||||
}
|
||||
}
|
||||
if strings.Contains(detail.Diagnostic+step.Transcript, "private-token") {
|
||||
t.Fatal("credential leaked in retained diagnostic/evidence")
|
||||
}
|
||||
edited, err := os.ReadFile(filepath.Join(dir, "edited"))
|
||||
if err != nil || string(edited) != "edited once\n" {
|
||||
t.Fatalf("admitted tool work replayed or lost: %q, %v", edited, err)
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(dir, "interrupted")); err != nil {
|
||||
t.Fatal("cleanup did not interrupt", err)
|
||||
}
|
||||
stored, err := store.GetRun(ctx, detail.ID)
|
||||
if err != nil || stored.Diagnostic != detail.Diagnostic || stored.Steps[0].Transcript != step.Transcript {
|
||||
t.Fatal("diagnostic/evidence not persisted in scratch database", err)
|
||||
}
|
||||
t.Logf("scratch run %s: %s", mode, stored.Diagnostic)
|
||||
})
|
||||
}
|
||||
}
|
||||
+312
@@ -0,0 +1,312 @@
|
||||
# stage implementer
|
||||
|
||||
```limits
|
||||
{"max_visits":5,"on":["blocked_retryable"]}
|
||||
```
|
||||
|
||||
|
||||
```memory
|
||||
{"task": "input.prompt", "code_findings": "reviewer.findings?", "ui_findings": "ui_reviewer.findings?"}
|
||||
```
|
||||
|
||||
```output
|
||||
{ task, acceptance_checks, changed_files, implementation_summary, new_helpers }
|
||||
```
|
||||
|
||||
```prompt
|
||||
Implement input.prompt in the run directory using the repository's actual languages and
|
||||
file types; create files as needed.
|
||||
|
||||
Read android/AGENTS.md first. It is the map of this project - how to build, test, run and capture here, where
|
||||
the pieces live, and the conventions that apply. Do not re-derive what it tells you and do not go
|
||||
looking for what it already names; if it is wrong or out of date, say so in your output.
|
||||
|
||||
Read the surrounding code before editing so you extend the existing helpers and patterns rather than
|
||||
duplicating them. Keep each change as small as the task allows and follow the repository's own
|
||||
conventions and error handling.
|
||||
|
||||
This is the Android app. Build and test it with the project's own Gradle invocation as the map
|
||||
gives it, and treat the map's geometry tokens as the only source of dimensions - a UI change that
|
||||
adds a raw value fails the repository's own check. Perform code/build/unit tests only;
|
||||
no adb, install, capture, device-setting changes or restoring main's build. The UI reviewer
|
||||
alone verifies rendered surfaces under its declared runner resource lease.
|
||||
|
||||
Record the acceptance checks in acceptance_checks, the actual changes in changed_files, and any new
|
||||
shared helper in new_helpers. Verify with the checks this repository uses - as the map gives them -
|
||||
and report the exact commands and observed output. Address any concrete findings the runner supplies
|
||||
and note the actual fixes. Return blocked if the required scope or tools are unavailable. Do not
|
||||
commit or push; the committer stage lands the work.
|
||||
```
|
||||
|
||||
```tools
|
||||
read
|
||||
glob
|
||||
grep
|
||||
edit
|
||||
shell
|
||||
```
|
||||
|
||||
```configs
|
||||
provider=openai
|
||||
model=gpt-5.6-terra
|
||||
retries=1
|
||||
timeout=30m
|
||||
```
|
||||
|
||||
|
||||
```edges
|
||||
success -> reviewer
|
||||
blocked_retryable -> implementer
|
||||
blocked -> #failure
|
||||
error -> #failure
|
||||
```
|
||||
|
||||
|
||||
# stage reviewer
|
||||
|
||||
```limits
|
||||
{"max_visits": 3, "on": ["rejected"]}
|
||||
```
|
||||
|
||||
```memory
|
||||
{"task": "input.prompt", "acceptance_checks": "implementer.acceptance_checks", "changed_files": "implementer.changed_files", "implementation": "implementer.implementation_summary?"}
|
||||
```
|
||||
|
||||
|
||||
```output
|
||||
{ summary, changed_files, verification_evidence, duplication_review, performance_review, findings }
|
||||
```
|
||||
|
||||
```prompt
|
||||
Verify with builds, unit tests and source review as android/AGENTS.md gives them.
|
||||
Do not use adb, install, capture, change device settings or restore main's build.
|
||||
Device verification belongs exclusively to the UI reviewer under its runner resource lease.
|
||||
Enforce the map's geometry-token rule: a raw dimension outside the token file is a must-fix finding.
|
||||
|
||||
Review the actual changes against the task in input.prompt and the acceptance
|
||||
checks the implementer recorded. Run the repository's relevant verification checks
|
||||
with the stage tools (for example build, lint, tests or type checks) and report the
|
||||
exact commands and observed results in verification_evidence.
|
||||
|
||||
Independently assess code quality:
|
||||
- Duplication: search the run directory for repeated or near-identical functions,
|
||||
blocks and copy-pasted logic introduced by the change; identify code that should be
|
||||
shared and record concrete locations in duplication_review.
|
||||
- Performance and resources: flag redundant computation, unnecessary dependencies,
|
||||
and repeated I/O or database/filesystem work that wastes time, CPU or memory;
|
||||
record it in performance_review.
|
||||
- Workarounds: flag hacks, suppressed errors and shortcuts that trade reliability for
|
||||
speed in findings.
|
||||
|
||||
Keep verification proportionate to the change: exercise the files and screens the diff
|
||||
actually touches, capture at most one screenshot per changed surface plus one regression
|
||||
screen, and do not repeat sweeps or measurements that an earlier visit already recorded.
|
||||
|
||||
Return rejected ONLY when at least one must-fix finding exists: a defect that blocks
|
||||
acceptance and that you can locate concretely (file and line), with evidence and a
|
||||
remedy. Do not return rejected for advisory-only, cosmetic or nit items. When there
|
||||
is no must-fix finding but you have non-blocking notes (duplication, performance or
|
||||
workaround observations), record them in findings and return advisory, which
|
||||
continues the run without another implementer visit. Return success only when the
|
||||
observed checks pass and you have nothing further to record. Do not edit files or
|
||||
claim unexecuted checks.
|
||||
```
|
||||
|
||||
```tools
|
||||
read
|
||||
glob
|
||||
grep
|
||||
shell
|
||||
```
|
||||
|
||||
```configs
|
||||
provider=openai
|
||||
model=gpt-5.6-luna
|
||||
retries=1
|
||||
timeout=45m
|
||||
```
|
||||
|
||||
|
||||
```edges
|
||||
success -> ui_reviewer
|
||||
advisory -> ui_reviewer
|
||||
rejected -> implementer
|
||||
error -> #failure
|
||||
```
|
||||
|
||||
|
||||
# stage ui_reviewer
|
||||
|
||||
```resources
|
||||
{"exclusive":"phone-47241FDAS00DLK"}
|
||||
```
|
||||
|
||||
```limits
|
||||
{"max_visits": 3, "on": ["rejected"]}
|
||||
```
|
||||
|
||||
```memory
|
||||
{"task": "input.prompt", "acceptance_checks": "implementer.acceptance_checks", "changed_files": "implementer.changed_files", "implementation": "implementer.implementation_summary?"}
|
||||
```
|
||||
|
||||
|
||||
```output
|
||||
{ summary, changed_files, ui_applicable, screenshots, ui_review, platform_conventions, findings }
|
||||
```
|
||||
|
||||
```prompt
|
||||
Decide whether this change touches a user-facing UI. If it does not (a backend,
|
||||
library, CLI, config or data-only change with no rendered surface), set
|
||||
ui_applicable=false, explain briefly in summary, and return skipped. Do not invent
|
||||
screenshots or visual work.
|
||||
|
||||
When the change does affect a user-facing UI, verify it visually:
|
||||
Read android/AGENTS.md first. You alone perform phone work while the runner holds
|
||||
this stage's declared resource lease. Use controlled fixtures for changed states,
|
||||
including human-decision sheets, on a scratch server/database and throwaway session.
|
||||
Never mutate an operator session or restart deployed services. Record original
|
||||
settings and connection/workspace; use the configured authoritative endpoint, not a
|
||||
hardcoded scratch default. Keep every shell/ADB operation foreground and awaited.
|
||||
Restore original settings and main's build before successful return, under the same
|
||||
lease. Report capture/restoration failures explicitly; do not approve incomplete
|
||||
verification or advertise readiness when cleanup/restoration is uncertain.
|
||||
Include resource_cleanup_confirmed in output: true only after confirmed restoration;
|
||||
false on any uncertainty (a release veto that quarantines ownership even if tools are idle).
|
||||
- Build and launch the application the way the repository actually does it, then
|
||||
capture screenshots of every relevant screen, state and viewport (default, empty,
|
||||
loading, error, long content, small and large sizes) with the target platform's
|
||||
screenshot tooling through shell. Prefer a headless or scripted capture when the
|
||||
project supports one. Save the images under the run directory and list the exact
|
||||
capture commands and image paths in screenshots.
|
||||
- Read the captured images and review them as a UI/UX designer: layout, spacing,
|
||||
alignment, typographic hierarchy, color and contrast, iconography, interaction
|
||||
and focus states, loading/empty/error states, accessibility, and consistency.
|
||||
- Judge the UI against established industry patterns and, above all, the native
|
||||
conventions of the platform the app targets (operating system, toolkit or web
|
||||
standards). Record the platform you reviewed against in platform_conventions,
|
||||
and any deviations and concrete fixes in findings. Infer the target platform from
|
||||
the task and the repository's actual UI toolkit when it is unstated.
|
||||
- Approve only when the screenshots show the task and the acceptance checks are met
|
||||
and the UI is correct, consistent and native to the target platform; otherwise
|
||||
return rejected with specific, actionable findings the implementer can fix.
|
||||
|
||||
Keep the capture pass proportionate: capture each changed state once (for example one
|
||||
collapsed and one expanded section, one run state, one stuck header) instead of every
|
||||
screen at multiple viewports or pan directions, and reuse what this run already captured
|
||||
rather than re-doing it.
|
||||
|
||||
If the change is UI-facing but screenshots genuinely cannot be captured
|
||||
(for example no runnable UI or no display), return blocked rather than approving.
|
||||
Do not edit files, and do not approve a UI you did not actually see.
|
||||
|
||||
Return rejected only for a must-fix visual defect with a concrete location, evidence
|
||||
and remedy; record non-blocking polish, duplication or convention notes in findings
|
||||
and return advisory, which continues the run without another implementer visit.
|
||||
```
|
||||
|
||||
```tools
|
||||
read
|
||||
glob
|
||||
grep
|
||||
shell
|
||||
```
|
||||
|
||||
```configs
|
||||
provider=openai
|
||||
model=gpt-5.6-luna
|
||||
retries=1
|
||||
timeout=45m
|
||||
```
|
||||
|
||||
|
||||
```edges
|
||||
skipped -> committer
|
||||
approved -> committer
|
||||
advisory -> committer
|
||||
rejected -> implementer
|
||||
blocked -> #failure
|
||||
error -> #failure
|
||||
```
|
||||
|
||||
# stage committer
|
||||
|
||||
```memory
|
||||
{"task": "input.prompt", "changed_files": "implementer.changed_files", "review": "reviewer.summary?", "ui_review": "ui_reviewer.summary?"}
|
||||
```
|
||||
|
||||
```output
|
||||
{ commit, files, conflicts_resolved, summary }
|
||||
```
|
||||
|
||||
```prompt
|
||||
Commit the work this run produced, and resolve any conflict that is present. This stage
|
||||
runs after the reviews have approved and is the only stage allowed to commit.
|
||||
|
||||
Before committing, check that the change is reflected where it belongs: the packaged
|
||||
templates, the packaged
|
||||
authoring skill references. Update anything the change makes stale, and state in the
|
||||
commit body what you checked, saying explicitly when none of them needs updating.
|
||||
|
||||
Stage exactly the files this run changed - the paths in changed_files that still exist -
|
||||
using the repository's own version control commands, and commit them as one commit whose
|
||||
subject says what the change does and whose body summarises it. Do not stage unrelated
|
||||
files, generated or ignored files, or anything another change owns. Do not invent work
|
||||
that was not done; describe what is actually in the diff.
|
||||
|
||||
If the workspace is in a conflicted state - unmerged paths, or a merge, rebase or
|
||||
cherry-pick left unfinished - resolve each conflict: keep both intents where they are
|
||||
compatible, and choose the result that satisfies the task where they are not, then
|
||||
complete the operation so the tree is clean and the commit is finished. Never commit
|
||||
conflict markers, and never discard the other side's work silently; record what you
|
||||
resolved.
|
||||
|
||||
Do not push, tag, revert, or rewrite unrelated history. If there is nothing to commit,
|
||||
report that instead of creating an empty commit. Report the commit identifier, the files
|
||||
committed, whatever you resolved, and anything you deliberately left uncommitted.
|
||||
|
||||
|
||||
Finally, bring this branch up to date with main, in this same stage. After committing the work
|
||||
above, merge main into this branch here, resolve any conflict here - keeping both intents where they
|
||||
are compatible, and choosing the result that satisfies this task where they are not - run the
|
||||
repository's checks again after resolving, and commit the completed merge with a message naming what
|
||||
was combined. The point is that this branch can later be merged back without conflict: leave the
|
||||
tree clean and the merge finished. Never leave conflict markers, never discard the other side's work
|
||||
silently. If the merge genuinely cannot be completed, say so with the path and the reason instead of
|
||||
leaving the merge half-finished.
|
||||
|
||||
Then land it: this branch is the work and main is where it belongs, so merge this branch into main
|
||||
as the last act of this stage. Find the repository's main working tree with `git worktree list` (the
|
||||
entry checked out on main, not this one) and merge this branch into it there with `--no-ff`, with a
|
||||
message saying what the change does. Two lanes can land at the same moment, so take a lock first -
|
||||
`mkdir` a lock directory beside main's `git` directory in a bounded retry loop, and remove it when
|
||||
done - and retry briefly if another lane holds it. Then verify with
|
||||
`git merge-base --is-ancestor <this branch> main` and report the commit main now has. If main cannot
|
||||
be updated, say exactly why and leave this branch committed and clean rather than leaving main
|
||||
half-merged. There is no separate merge pipeline and nothing to hand this to: if it cannot be landed
|
||||
here, it is reported here.
|
||||
|
||||
|
||||
Do not edit README or other shared documentation files: an item records what it changed and which documentation is now stale in its own commit body, and the shared documents are folded by a single writer.
|
||||
|
||||
```
|
||||
|
||||
```tools
|
||||
read
|
||||
glob
|
||||
grep
|
||||
edit
|
||||
shell
|
||||
```
|
||||
|
||||
```configs
|
||||
provider=openai
|
||||
model=gpt-5.6-terra
|
||||
retries=1
|
||||
timeout=30m
|
||||
```
|
||||
|
||||
```edges
|
||||
success -> #success
|
||||
blocked -> #failure
|
||||
error -> #failure
|
||||
```
|
||||
+296
@@ -0,0 +1,296 @@
|
||||
# stage implementer
|
||||
|
||||
```limits
|
||||
{"max_visits":5,"on":["blocked_retryable"]}
|
||||
```
|
||||
|
||||
|
||||
```memory
|
||||
{"task": "input.prompt", "code_findings": "reviewer.findings?", "ui_findings": "ui_reviewer.findings?"}
|
||||
```
|
||||
|
||||
```output
|
||||
{ task, acceptance_checks, changed_files, implementation_summary, new_helpers }
|
||||
```
|
||||
|
||||
```prompt
|
||||
Implement input.prompt in the run directory using the repository's actual languages and
|
||||
file types; create files as needed.
|
||||
|
||||
Read android/AGENTS.md first. It is the map of this project - how to build, test, run and capture here, where
|
||||
the pieces live, and the conventions that apply. Do not re-derive what it tells you and do not go
|
||||
looking for what it already names; if it is wrong or out of date, say so in your output.
|
||||
|
||||
Read the surrounding code before editing so you extend the existing helpers and patterns rather than
|
||||
duplicating them. Keep each change as small as the task allows and follow the repository's own
|
||||
conventions and error handling.
|
||||
|
||||
This is the Android app. Build and test it with the project's own Gradle invocation as the map
|
||||
gives it, and treat the map's geometry tokens as the only source of dimensions - a UI change that
|
||||
adds a raw value fails the repository's own check. A change with a rendered surface is verified on
|
||||
the device the map names (never an emulator), capturing what the map says to capture.
|
||||
|
||||
Record the acceptance checks in acceptance_checks, the actual changes in changed_files, and any new
|
||||
shared helper in new_helpers. Verify with the checks this repository uses - as the map gives them -
|
||||
and report the exact commands and observed output. Address any concrete findings the runner supplies
|
||||
and note the actual fixes. Return blocked if the required scope or tools are unavailable. Do not
|
||||
commit or push; the committer stage lands the work.
|
||||
```
|
||||
|
||||
```tools
|
||||
read
|
||||
glob
|
||||
grep
|
||||
edit
|
||||
shell
|
||||
```
|
||||
|
||||
```configs
|
||||
provider=openai
|
||||
model=gpt-6.1-sol-fast
|
||||
retries=1
|
||||
timeout=30m
|
||||
```
|
||||
|
||||
|
||||
```edges
|
||||
success -> reviewer
|
||||
blocked_retryable -> implementer
|
||||
blocked -> #failure
|
||||
error -> #failure
|
||||
```
|
||||
|
||||
|
||||
# stage reviewer
|
||||
|
||||
```limits
|
||||
{"max_visits": 3, "on": ["rejected"]}
|
||||
```
|
||||
|
||||
```memory
|
||||
{"task": "input.prompt", "acceptance_checks": "implementer.acceptance_checks", "changed_files": "implementer.changed_files", "implementation": "implementer.implementation_summary?"}
|
||||
```
|
||||
|
||||
|
||||
```output
|
||||
{ summary, changed_files, verification_evidence, duplication_review, performance_review, findings }
|
||||
```
|
||||
|
||||
```prompt
|
||||
Verify with the build and the unit tests as android/AGENTS.md gives them, and for a
|
||||
change with a rendered surface check the captures against the running app on the device the map
|
||||
names. Enforce the map's geometry-token rule: a raw dimension outside the token file is a must-fix
|
||||
finding.
|
||||
|
||||
Review the actual changes against the task in input.prompt and the acceptance
|
||||
checks the implementer recorded. Run the repository's relevant verification checks
|
||||
with the stage tools (for example build, lint, tests or type checks) and report the
|
||||
exact commands and observed results in verification_evidence.
|
||||
|
||||
Independently assess code quality:
|
||||
- Duplication: search the run directory for repeated or near-identical functions,
|
||||
blocks and copy-pasted logic introduced by the change; identify code that should be
|
||||
shared and record concrete locations in duplication_review.
|
||||
- Performance and resources: flag redundant computation, unnecessary dependencies,
|
||||
and repeated I/O or database/filesystem work that wastes time, CPU or memory;
|
||||
record it in performance_review.
|
||||
- Workarounds: flag hacks, suppressed errors and shortcuts that trade reliability for
|
||||
speed in findings.
|
||||
|
||||
Keep verification proportionate to the change: exercise the files and screens the diff
|
||||
actually touches, capture at most one screenshot per changed surface plus one regression
|
||||
screen, and do not repeat sweeps or measurements that an earlier visit already recorded.
|
||||
|
||||
Return rejected ONLY when at least one must-fix finding exists: a defect that blocks
|
||||
acceptance and that you can locate concretely (file and line), with evidence and a
|
||||
remedy. Do not return rejected for advisory-only, cosmetic or nit items. When there
|
||||
is no must-fix finding but you have non-blocking notes (duplication, performance or
|
||||
workaround observations), record them in findings and return advisory, which
|
||||
continues the run without another implementer visit. Return success only when the
|
||||
observed checks pass and you have nothing further to record. Do not edit files or
|
||||
claim unexecuted checks.
|
||||
```
|
||||
|
||||
```tools
|
||||
read
|
||||
glob
|
||||
grep
|
||||
shell
|
||||
```
|
||||
|
||||
```configs
|
||||
provider=openai
|
||||
model=gpt-6.1-sol-fast
|
||||
retries=1
|
||||
timeout=45m
|
||||
```
|
||||
|
||||
|
||||
```edges
|
||||
success -> ui_reviewer
|
||||
advisory -> ui_reviewer
|
||||
rejected -> implementer
|
||||
error -> #failure
|
||||
```
|
||||
|
||||
|
||||
# stage ui_reviewer
|
||||
|
||||
```limits
|
||||
{"max_visits": 3, "on": ["rejected"]}
|
||||
```
|
||||
|
||||
```memory
|
||||
{"task": "input.prompt", "acceptance_checks": "implementer.acceptance_checks", "changed_files": "implementer.changed_files", "implementation": "implementer.implementation_summary?"}
|
||||
```
|
||||
|
||||
|
||||
```output
|
||||
{ summary, changed_files, ui_applicable, screenshots, ui_review, platform_conventions, findings }
|
||||
```
|
||||
|
||||
```prompt
|
||||
Decide whether this change touches a user-facing UI. If it does not (a backend,
|
||||
library, CLI, config or data-only change with no rendered surface), set
|
||||
ui_applicable=false, explain briefly in summary, and return skipped. Do not invent
|
||||
screenshots or visual work.
|
||||
|
||||
When the change does affect a user-facing UI, verify it visually:
|
||||
- Build and launch the application the way the repository actually does it, then
|
||||
capture screenshots of every relevant screen, state and viewport (default, empty,
|
||||
loading, error, long content, small and large sizes) with the target platform's
|
||||
screenshot tooling through shell. Prefer a headless or scripted capture when the
|
||||
project supports one. Save the images under the run directory and list the exact
|
||||
capture commands and image paths in screenshots.
|
||||
- Read the captured images and review them as a UI/UX designer: layout, spacing,
|
||||
alignment, typographic hierarchy, color and contrast, iconography, interaction
|
||||
and focus states, loading/empty/error states, accessibility, and consistency.
|
||||
- Judge the UI against established industry patterns and, above all, the native
|
||||
conventions of the platform the app targets (operating system, toolkit or web
|
||||
standards). Record the platform you reviewed against in platform_conventions,
|
||||
and any deviations and concrete fixes in findings. Infer the target platform from
|
||||
the task and the repository's actual UI toolkit when it is unstated.
|
||||
- Approve only when the screenshots show the task and the acceptance checks are met
|
||||
and the UI is correct, consistent and native to the target platform; otherwise
|
||||
return rejected with specific, actionable findings the implementer can fix.
|
||||
|
||||
Keep the capture pass proportionate: capture each changed state once (for example one
|
||||
collapsed and one expanded section, one run state, one stuck header) instead of every
|
||||
screen at multiple viewports or pan directions, and reuse what this run already captured
|
||||
rather than re-doing it.
|
||||
|
||||
If the change is UI-facing but screenshots genuinely cannot be captured
|
||||
(for example no runnable UI or no display), return blocked rather than approving.
|
||||
Do not edit files, and do not approve a UI you did not actually see.
|
||||
|
||||
Return rejected only for a must-fix visual defect with a concrete location, evidence
|
||||
and remedy; record non-blocking polish, duplication or convention notes in findings
|
||||
and return advisory, which continues the run without another implementer visit.
|
||||
```
|
||||
|
||||
```tools
|
||||
read
|
||||
glob
|
||||
grep
|
||||
shell
|
||||
```
|
||||
|
||||
```configs
|
||||
provider=openai
|
||||
model=gpt-6.1-sol-fast
|
||||
retries=1
|
||||
timeout=45m
|
||||
```
|
||||
|
||||
|
||||
```edges
|
||||
skipped -> committer
|
||||
approved -> committer
|
||||
advisory -> committer
|
||||
rejected -> implementer
|
||||
blocked -> #failure
|
||||
error -> #failure
|
||||
```
|
||||
|
||||
# stage committer
|
||||
|
||||
```memory
|
||||
{"task": "input.prompt", "changed_files": "implementer.changed_files", "review": "reviewer.summary?", "ui_review": "ui_reviewer.summary?"}
|
||||
```
|
||||
|
||||
```output
|
||||
{ commit, files, conflicts_resolved, summary }
|
||||
```
|
||||
|
||||
```prompt
|
||||
Commit the work this run produced, and resolve any conflict that is present. This stage
|
||||
runs after the reviews have approved and is the only stage allowed to commit.
|
||||
|
||||
Before committing, check that the change is reflected where it belongs: the packaged
|
||||
templates, the packaged
|
||||
authoring skill references. Update anything the change makes stale, and state in the
|
||||
commit body what you checked, saying explicitly when none of them needs updating.
|
||||
|
||||
Stage exactly the files this run changed - the paths in changed_files that still exist -
|
||||
using the repository's own version control commands, and commit them as one commit whose
|
||||
subject says what the change does and whose body summarises it. Do not stage unrelated
|
||||
files, generated or ignored files, or anything another change owns. Do not invent work
|
||||
that was not done; describe what is actually in the diff.
|
||||
|
||||
If the workspace is in a conflicted state - unmerged paths, or a merge, rebase or
|
||||
cherry-pick left unfinished - resolve each conflict: keep both intents where they are
|
||||
compatible, and choose the result that satisfies the task where they are not, then
|
||||
complete the operation so the tree is clean and the commit is finished. Never commit
|
||||
conflict markers, and never discard the other side's work silently; record what you
|
||||
resolved.
|
||||
|
||||
Do not push, tag, revert, or rewrite unrelated history. If there is nothing to commit,
|
||||
report that instead of creating an empty commit. Report the commit identifier, the files
|
||||
committed, whatever you resolved, and anything you deliberately left uncommitted.
|
||||
|
||||
|
||||
Finally, bring this branch up to date with main, in this same stage. After committing the work
|
||||
above, merge main into this branch here, resolve any conflict here - keeping both intents where they
|
||||
are compatible, and choosing the result that satisfies this task where they are not - run the
|
||||
repository's checks again after resolving, and commit the completed merge with a message naming what
|
||||
was combined. The point is that this branch can later be merged back without conflict: leave the
|
||||
tree clean and the merge finished. Never leave conflict markers, never discard the other side's work
|
||||
silently. If the merge genuinely cannot be completed, say so with the path and the reason instead of
|
||||
leaving the merge half-finished.
|
||||
|
||||
Then land it: this branch is the work and main is where it belongs, so merge this branch into main
|
||||
as the last act of this stage. Find the repository's main working tree with `git worktree list` (the
|
||||
entry checked out on main, not this one) and merge this branch into it there with `--no-ff`, with a
|
||||
message saying what the change does. Two lanes can land at the same moment, so take a lock first -
|
||||
`mkdir` a lock directory beside main's `git` directory in a bounded retry loop, and remove it when
|
||||
done - and retry briefly if another lane holds it. Then verify with
|
||||
`git merge-base --is-ancestor <this branch> main` and report the commit main now has. If main cannot
|
||||
be updated, say exactly why and leave this branch committed and clean rather than leaving main
|
||||
half-merged. There is no separate merge pipeline and nothing to hand this to: if it cannot be landed
|
||||
here, it is reported here.
|
||||
|
||||
|
||||
Do not edit README or other shared documentation files: an item records what it changed and which documentation is now stale in its own commit body, and the shared documents are folded by a single writer.
|
||||
|
||||
```
|
||||
|
||||
```tools
|
||||
read
|
||||
glob
|
||||
grep
|
||||
edit
|
||||
shell
|
||||
```
|
||||
|
||||
```configs
|
||||
provider=openai
|
||||
model=gpt-6.1-sol-fast
|
||||
retries=1
|
||||
timeout=30m
|
||||
```
|
||||
|
||||
```edges
|
||||
success -> #success
|
||||
blocked -> #failure
|
||||
error -> #failure
|
||||
```
|
||||
+296
@@ -0,0 +1,296 @@
|
||||
# stage implementer
|
||||
|
||||
```limits
|
||||
{"max_visits":5,"on":["blocked_retryable"]}
|
||||
```
|
||||
|
||||
|
||||
```memory
|
||||
{"task": "input.prompt", "code_findings": "reviewer.findings?", "ui_findings": "ui_reviewer.findings?"}
|
||||
```
|
||||
|
||||
```output
|
||||
{ task, acceptance_checks, changed_files, implementation_summary, new_helpers }
|
||||
```
|
||||
|
||||
```prompt
|
||||
Implement input.prompt in the run directory using the repository's actual languages and
|
||||
file types; create files as needed.
|
||||
|
||||
Read android/AGENTS.md first. It is the map of this project - how to build, test, run and capture here, where
|
||||
the pieces live, and the conventions that apply. Do not re-derive what it tells you and do not go
|
||||
looking for what it already names; if it is wrong or out of date, say so in your output.
|
||||
|
||||
Read the surrounding code before editing so you extend the existing helpers and patterns rather than
|
||||
duplicating them. Keep each change as small as the task allows and follow the repository's own
|
||||
conventions and error handling.
|
||||
|
||||
This is the Android app. Build and test it with the project's own Gradle invocation as the map
|
||||
gives it, and treat the map's geometry tokens as the only source of dimensions - a UI change that
|
||||
adds a raw value fails the repository's own check. A change with a rendered surface is verified on
|
||||
the device the map names (never an emulator), capturing what the map says to capture.
|
||||
|
||||
Record the acceptance checks in acceptance_checks, the actual changes in changed_files, and any new
|
||||
shared helper in new_helpers. Verify with the checks this repository uses - as the map gives them -
|
||||
and report the exact commands and observed output. Address any concrete findings the runner supplies
|
||||
and note the actual fixes. Return blocked if the required scope or tools are unavailable. Do not
|
||||
commit or push; the committer stage lands the work.
|
||||
```
|
||||
|
||||
```tools
|
||||
read
|
||||
glob
|
||||
grep
|
||||
edit
|
||||
shell
|
||||
```
|
||||
|
||||
```configs
|
||||
provider=openai
|
||||
model=gpt-5.6-terra
|
||||
retries=1
|
||||
timeout=30m
|
||||
```
|
||||
|
||||
|
||||
```edges
|
||||
success -> reviewer
|
||||
blocked_retryable -> implementer
|
||||
blocked -> #failure
|
||||
error -> #failure
|
||||
```
|
||||
|
||||
|
||||
# stage reviewer
|
||||
|
||||
```limits
|
||||
{"max_visits": 3, "on": ["rejected"]}
|
||||
```
|
||||
|
||||
```memory
|
||||
{"task": "input.prompt", "acceptance_checks": "implementer.acceptance_checks", "changed_files": "implementer.changed_files", "implementation": "implementer.implementation_summary?"}
|
||||
```
|
||||
|
||||
|
||||
```output
|
||||
{ summary, changed_files, verification_evidence, duplication_review, performance_review, findings }
|
||||
```
|
||||
|
||||
```prompt
|
||||
Verify with the build and the unit tests as android/AGENTS.md gives them, and for a
|
||||
change with a rendered surface check the captures against the running app on the device the map
|
||||
names. Enforce the map's geometry-token rule: a raw dimension outside the token file is a must-fix
|
||||
finding.
|
||||
|
||||
Review the actual changes against the task in input.prompt and the acceptance
|
||||
checks the implementer recorded. Run the repository's relevant verification checks
|
||||
with the stage tools (for example build, lint, tests or type checks) and report the
|
||||
exact commands and observed results in verification_evidence.
|
||||
|
||||
Independently assess code quality:
|
||||
- Duplication: search the run directory for repeated or near-identical functions,
|
||||
blocks and copy-pasted logic introduced by the change; identify code that should be
|
||||
shared and record concrete locations in duplication_review.
|
||||
- Performance and resources: flag redundant computation, unnecessary dependencies,
|
||||
and repeated I/O or database/filesystem work that wastes time, CPU or memory;
|
||||
record it in performance_review.
|
||||
- Workarounds: flag hacks, suppressed errors and shortcuts that trade reliability for
|
||||
speed in findings.
|
||||
|
||||
Keep verification proportionate to the change: exercise the files and screens the diff
|
||||
actually touches, capture at most one screenshot per changed surface plus one regression
|
||||
screen, and do not repeat sweeps or measurements that an earlier visit already recorded.
|
||||
|
||||
Return rejected ONLY when at least one must-fix finding exists: a defect that blocks
|
||||
acceptance and that you can locate concretely (file and line), with evidence and a
|
||||
remedy. Do not return rejected for advisory-only, cosmetic or nit items. When there
|
||||
is no must-fix finding but you have non-blocking notes (duplication, performance or
|
||||
workaround observations), record them in findings and return advisory, which
|
||||
continues the run without another implementer visit. Return success only when the
|
||||
observed checks pass and you have nothing further to record. Do not edit files or
|
||||
claim unexecuted checks.
|
||||
```
|
||||
|
||||
```tools
|
||||
read
|
||||
glob
|
||||
grep
|
||||
shell
|
||||
```
|
||||
|
||||
```configs
|
||||
provider=openai
|
||||
model=gpt-5.6-luna
|
||||
retries=1
|
||||
timeout=45m
|
||||
```
|
||||
|
||||
|
||||
```edges
|
||||
success -> ui_reviewer
|
||||
advisory -> ui_reviewer
|
||||
rejected -> implementer
|
||||
error -> #failure
|
||||
```
|
||||
|
||||
|
||||
# stage ui_reviewer
|
||||
|
||||
```limits
|
||||
{"max_visits": 3, "on": ["rejected"]}
|
||||
```
|
||||
|
||||
```memory
|
||||
{"task": "input.prompt", "acceptance_checks": "implementer.acceptance_checks", "changed_files": "implementer.changed_files", "implementation": "implementer.implementation_summary?"}
|
||||
```
|
||||
|
||||
|
||||
```output
|
||||
{ summary, changed_files, ui_applicable, screenshots, ui_review, platform_conventions, findings }
|
||||
```
|
||||
|
||||
```prompt
|
||||
Decide whether this change touches a user-facing UI. If it does not (a backend,
|
||||
library, CLI, config or data-only change with no rendered surface), set
|
||||
ui_applicable=false, explain briefly in summary, and return skipped. Do not invent
|
||||
screenshots or visual work.
|
||||
|
||||
When the change does affect a user-facing UI, verify it visually:
|
||||
- Build and launch the application the way the repository actually does it, then
|
||||
capture screenshots of every relevant screen, state and viewport (default, empty,
|
||||
loading, error, long content, small and large sizes) with the target platform's
|
||||
screenshot tooling through shell. Prefer a headless or scripted capture when the
|
||||
project supports one. Save the images under the run directory and list the exact
|
||||
capture commands and image paths in screenshots.
|
||||
- Read the captured images and review them as a UI/UX designer: layout, spacing,
|
||||
alignment, typographic hierarchy, color and contrast, iconography, interaction
|
||||
and focus states, loading/empty/error states, accessibility, and consistency.
|
||||
- Judge the UI against established industry patterns and, above all, the native
|
||||
conventions of the platform the app targets (operating system, toolkit or web
|
||||
standards). Record the platform you reviewed against in platform_conventions,
|
||||
and any deviations and concrete fixes in findings. Infer the target platform from
|
||||
the task and the repository's actual UI toolkit when it is unstated.
|
||||
- Approve only when the screenshots show the task and the acceptance checks are met
|
||||
and the UI is correct, consistent and native to the target platform; otherwise
|
||||
return rejected with specific, actionable findings the implementer can fix.
|
||||
|
||||
Keep the capture pass proportionate: capture each changed state once (for example one
|
||||
collapsed and one expanded section, one run state, one stuck header) instead of every
|
||||
screen at multiple viewports or pan directions, and reuse what this run already captured
|
||||
rather than re-doing it.
|
||||
|
||||
If the change is UI-facing but screenshots genuinely cannot be captured
|
||||
(for example no runnable UI or no display), return blocked rather than approving.
|
||||
Do not edit files, and do not approve a UI you did not actually see.
|
||||
|
||||
Return rejected only for a must-fix visual defect with a concrete location, evidence
|
||||
and remedy; record non-blocking polish, duplication or convention notes in findings
|
||||
and return advisory, which continues the run without another implementer visit.
|
||||
```
|
||||
|
||||
```tools
|
||||
read
|
||||
glob
|
||||
grep
|
||||
shell
|
||||
```
|
||||
|
||||
```configs
|
||||
provider=openai
|
||||
model=gpt-5.6-luna
|
||||
retries=1
|
||||
timeout=45m
|
||||
```
|
||||
|
||||
|
||||
```edges
|
||||
skipped -> committer
|
||||
approved -> committer
|
||||
advisory -> committer
|
||||
rejected -> implementer
|
||||
blocked -> #failure
|
||||
error -> #failure
|
||||
```
|
||||
|
||||
# stage committer
|
||||
|
||||
```memory
|
||||
{"task": "input.prompt", "changed_files": "implementer.changed_files", "review": "reviewer.summary?", "ui_review": "ui_reviewer.summary?"}
|
||||
```
|
||||
|
||||
```output
|
||||
{ commit, files, conflicts_resolved, summary }
|
||||
```
|
||||
|
||||
```prompt
|
||||
Commit the work this run produced, and resolve any conflict that is present. This stage
|
||||
runs after the reviews have approved and is the only stage allowed to commit.
|
||||
|
||||
Before committing, check that the change is reflected where it belongs: the packaged
|
||||
templates, the packaged
|
||||
authoring skill references. Update anything the change makes stale, and state in the
|
||||
commit body what you checked, saying explicitly when none of them needs updating.
|
||||
|
||||
Stage exactly the files this run changed - the paths in changed_files that still exist -
|
||||
using the repository's own version control commands, and commit them as one commit whose
|
||||
subject says what the change does and whose body summarises it. Do not stage unrelated
|
||||
files, generated or ignored files, or anything another change owns. Do not invent work
|
||||
that was not done; describe what is actually in the diff.
|
||||
|
||||
If the workspace is in a conflicted state - unmerged paths, or a merge, rebase or
|
||||
cherry-pick left unfinished - resolve each conflict: keep both intents where they are
|
||||
compatible, and choose the result that satisfies the task where they are not, then
|
||||
complete the operation so the tree is clean and the commit is finished. Never commit
|
||||
conflict markers, and never discard the other side's work silently; record what you
|
||||
resolved.
|
||||
|
||||
Do not push, tag, revert, or rewrite unrelated history. If there is nothing to commit,
|
||||
report that instead of creating an empty commit. Report the commit identifier, the files
|
||||
committed, whatever you resolved, and anything you deliberately left uncommitted.
|
||||
|
||||
|
||||
Finally, bring this branch up to date with main, in this same stage. After committing the work
|
||||
above, merge main into this branch here, resolve any conflict here - keeping both intents where they
|
||||
are compatible, and choosing the result that satisfies this task where they are not - run the
|
||||
repository's checks again after resolving, and commit the completed merge with a message naming what
|
||||
was combined. The point is that this branch can later be merged back without conflict: leave the
|
||||
tree clean and the merge finished. Never leave conflict markers, never discard the other side's work
|
||||
silently. If the merge genuinely cannot be completed, say so with the path and the reason instead of
|
||||
leaving the merge half-finished.
|
||||
|
||||
Then land it: this branch is the work and main is where it belongs, so merge this branch into main
|
||||
as the last act of this stage. Find the repository's main working tree with `git worktree list` (the
|
||||
entry checked out on main, not this one) and merge this branch into it there with `--no-ff`, with a
|
||||
message saying what the change does. Two lanes can land at the same moment, so take a lock first -
|
||||
`mkdir` a lock directory beside main's `git` directory in a bounded retry loop, and remove it when
|
||||
done - and retry briefly if another lane holds it. Then verify with
|
||||
`git merge-base --is-ancestor <this branch> main` and report the commit main now has. If main cannot
|
||||
be updated, say exactly why and leave this branch committed and clean rather than leaving main
|
||||
half-merged. There is no separate merge pipeline and nothing to hand this to: if it cannot be landed
|
||||
here, it is reported here.
|
||||
|
||||
|
||||
Do not edit README or other shared documentation files: an item records what it changed and which documentation is now stale in its own commit body, and the shared documents are folded by a single writer.
|
||||
|
||||
```
|
||||
|
||||
```tools
|
||||
read
|
||||
glob
|
||||
grep
|
||||
edit
|
||||
shell
|
||||
```
|
||||
|
||||
```configs
|
||||
provider=openai
|
||||
model=gpt-5.6-terra
|
||||
retries=1
|
||||
timeout=30m
|
||||
```
|
||||
|
||||
```edges
|
||||
success -> #success
|
||||
blocked -> #failure
|
||||
error -> #failure
|
||||
```
|
||||
@@ -0,0 +1,135 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.a-wan.net/alan/pipelines/internal/stage"
|
||||
)
|
||||
|
||||
const (
|
||||
verificationFixtureID = "reconnect-foreground"
|
||||
verificationFixtureResponseLimit int64 = 64 << 10
|
||||
verificationFixtureTimeout = 10 * time.Second
|
||||
)
|
||||
|
||||
// VerificationFixture is a server-owned, explicitly registered probe. Its
|
||||
// endpoint and expected provenance come from server configuration, never a
|
||||
// stage definition or result.
|
||||
type VerificationFixture struct {
|
||||
ID, Endpoint, Backend, Revision, Capability, State string
|
||||
}
|
||||
|
||||
// VerificationFixturesFromEnvironment returns the single supported fixture
|
||||
// registration when its endpoint is explicitly configured. There is no default
|
||||
// endpoint: an unconfigured server fails target-aware verification closed.
|
||||
func VerificationFixturesFromEnvironment(getenv func(string) string) []VerificationFixture {
|
||||
if getenv == nil {
|
||||
getenv = os.Getenv
|
||||
}
|
||||
endpoint := strings.TrimSpace(getenv("PIPELINES_VERIFICATION_RECONNECT_FOREGROUND_URL"))
|
||||
if endpoint == "" {
|
||||
return nil
|
||||
}
|
||||
return []VerificationFixture{{
|
||||
ID: verificationFixtureID,
|
||||
Endpoint: endpoint,
|
||||
Backend: strings.TrimSpace(getenv("PIPELINES_VERIFICATION_RECONNECT_FOREGROUND_BACKEND")),
|
||||
Revision: strings.TrimSpace(getenv("PIPELINES_VERIFICATION_RECONNECT_FOREGROUND_REVISION")),
|
||||
Capability: strings.TrimSpace(getenv("PIPELINES_VERIFICATION_RECONNECT_FOREGROUND_CAPABILITY")),
|
||||
State: strings.TrimSpace(getenv("PIPELINES_VERIFICATION_RECONNECT_FOREGROUND_STATE")),
|
||||
}}
|
||||
}
|
||||
|
||||
type verificationFixtureRegistry struct {
|
||||
fixtures map[string]VerificationFixture
|
||||
err error
|
||||
}
|
||||
|
||||
func newVerificationFixtureRegistry(fixtures []VerificationFixture) *verificationFixtureRegistry {
|
||||
r := &verificationFixtureRegistry{fixtures: make(map[string]VerificationFixture, len(fixtures))}
|
||||
for _, fixture := range fixtures {
|
||||
if err := validVerificationFixture(fixture); err != nil {
|
||||
r.err = err
|
||||
return r
|
||||
}
|
||||
if _, exists := r.fixtures[fixture.ID]; exists {
|
||||
r.err = fmt.Errorf("duplicate verification fixture %q", fixture.ID)
|
||||
return r
|
||||
}
|
||||
r.fixtures[fixture.ID] = fixture
|
||||
}
|
||||
return r
|
||||
}
|
||||
|
||||
func validVerificationFixture(fixture VerificationFixture) error {
|
||||
if fixture.ID != verificationFixtureID {
|
||||
return fmt.Errorf("unsupported verification fixture %q", fixture.ID)
|
||||
}
|
||||
for _, value := range []string{fixture.Endpoint, fixture.Backend, fixture.Revision, fixture.Capability, fixture.State} {
|
||||
if value == "" || len(value) > 256 {
|
||||
return fmt.Errorf("verification fixture %q has incomplete configuration", fixture.ID)
|
||||
}
|
||||
}
|
||||
u, err := url.Parse(fixture.Endpoint)
|
||||
if err != nil || u.Host == "" || (u.Scheme != "http" && u.Scheme != "https") {
|
||||
return fmt.Errorf("verification fixture %q has invalid endpoint", fixture.ID)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *verificationFixtureRegistry) verify(ctx context.Context, target stage.VerificationTarget) (stage.VerificationEvidence, error) {
|
||||
if r.err != nil {
|
||||
return stage.VerificationEvidence{}, r.err
|
||||
}
|
||||
fixture, exists := r.fixtures[target.Fixture]
|
||||
if !exists {
|
||||
return stage.VerificationEvidence{}, fmt.Errorf("unknown verification fixture %q", target.Fixture)
|
||||
}
|
||||
if fixture.Backend != target.Backend || fixture.Revision != target.Revision || fixture.Capability != target.Capability {
|
||||
return stage.VerificationEvidence{}, fmt.Errorf("verification target does not match registered fixture %q", target.Fixture)
|
||||
}
|
||||
probeCtx, cancel := context.WithTimeout(ctx, verificationFixtureTimeout)
|
||||
defer cancel()
|
||||
req, err := http.NewRequestWithContext(probeCtx, http.MethodGet, fixture.Endpoint, nil)
|
||||
if err != nil {
|
||||
return stage.VerificationEvidence{}, fmt.Errorf("verification fixture %q request: %w", fixture.ID, err)
|
||||
}
|
||||
client := &http.Client{Timeout: verificationFixtureTimeout, CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }}
|
||||
response, err := client.Do(req)
|
||||
if err != nil {
|
||||
return stage.VerificationEvidence{}, fmt.Errorf("verification fixture %q probe: %w", fixture.ID, err)
|
||||
}
|
||||
defer response.Body.Close()
|
||||
if response.StatusCode != http.StatusOK {
|
||||
return stage.VerificationEvidence{}, fmt.Errorf("verification fixture %q probe returned %s", fixture.ID, response.Status)
|
||||
}
|
||||
var observed struct {
|
||||
Backend string `json:"backend"`
|
||||
Revision string `json:"revision"`
|
||||
Capability string `json:"capability"`
|
||||
Fixture string `json:"fixture"`
|
||||
State string `json:"state"`
|
||||
}
|
||||
body, err := io.ReadAll(io.LimitReader(response.Body, verificationFixtureResponseLimit+1))
|
||||
if err != nil {
|
||||
return stage.VerificationEvidence{}, fmt.Errorf("verification fixture %q response: %w", fixture.ID, err)
|
||||
}
|
||||
if int64(len(body)) > verificationFixtureResponseLimit {
|
||||
return stage.VerificationEvidence{}, fmt.Errorf("verification fixture %q response exceeds %d bytes", fixture.ID, verificationFixtureResponseLimit)
|
||||
}
|
||||
if err := json.Unmarshal(body, &observed); err != nil {
|
||||
return stage.VerificationEvidence{}, fmt.Errorf("verification fixture %q response: %w", fixture.ID, err)
|
||||
}
|
||||
if observed.Backend != fixture.Backend || observed.Revision != fixture.Revision || observed.Capability != fixture.Capability || observed.Fixture != fixture.ID || observed.State != fixture.State {
|
||||
return stage.VerificationEvidence{}, fmt.Errorf("verification fixture %q returned incompatible backend, revision, capability, fixture, or state", fixture.ID)
|
||||
}
|
||||
return stage.VerificationEvidence{Backend: observed.Backend, Revision: observed.Revision, Capability: observed.Capability, Fixture: observed.Fixture}, nil
|
||||
}
|
||||
@@ -0,0 +1,148 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.a-wan.net/alan/pipelines/internal/pipeline"
|
||||
"git.a-wan.net/alan/pipelines/internal/protocol"
|
||||
"git.a-wan.net/alan/pipelines/internal/stage"
|
||||
)
|
||||
|
||||
func TestHandlerRunsRegisteredVerificationFixture(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
var mu sync.Mutex
|
||||
response := map[string]string{"backend": "branch-api", "revision": "abc123", "capability": "controlled-state", "fixture": verificationFixtureID, "state": "ready"}
|
||||
calls := 0
|
||||
fixture := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
calls++
|
||||
_ = json.NewEncoder(w).Encode(response)
|
||||
}))
|
||||
defer fixture.Close()
|
||||
|
||||
store, err := pipeline.Open(filepath.Join(t.TempDir(), "fixture.db"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer store.Close()
|
||||
source := pipeline.Template([]string{"worker"}, true)
|
||||
source = strings.Replace(source, "```configs", "```verification\nbackend=branch-api\nrevision=abc123\ncapability=controlled-state\nfixture="+verificationFixtureID+"\n```\n\n```configs", 1)
|
||||
if _, err := store.Submit(ctx, "controlled-fixture", source); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
items, err := store.List(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
handler := HandlerWithOptions(ctx, Options{VerificationFixtures: []VerificationFixture{{ID: verificationFixtureID, Endpoint: fixture.URL, Backend: "branch-api", Revision: "abc123", Capability: "controlled-state", State: "ready"}}}, store).(*runHandler)
|
||||
defer func() {
|
||||
cancel()
|
||||
WaitRuns(handler)
|
||||
}()
|
||||
r := handler.runner
|
||||
if r.verifyTarget == nil {
|
||||
t.Fatal("normal server construction did not wire controlled fixture verification")
|
||||
}
|
||||
r.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) {
|
||||
// These forged model labels must not influence fixture provenance.
|
||||
return stage.Result{Outcome: "success", Output: stage.Values{"verification": map[string]any{"backend": "old-api", "revision": "stale", "capability": "forged", "fixture": "other"}}}, nil
|
||||
}
|
||||
start := func(key string) *protocol.RunDetail {
|
||||
run, err := r.start(protocol.Message{PipelineID: items[0].ID, Version: 1, RequestKey: key, Workspace: t.TempDir(), Input: map[string]any{"prompt": "verify"}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
r.wg.Wait()
|
||||
detail, err := store.GetRun(ctx, run.ID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return detail
|
||||
}
|
||||
if detail := start("compatible"); detail.Status != "succeeded" {
|
||||
t.Fatalf("compatible controlled fixture did not settle: %+v", detail)
|
||||
}
|
||||
mu.Lock()
|
||||
response["revision"] = "old-build"
|
||||
mu.Unlock()
|
||||
if detail := start("stale"); detail.Status != "failed" || !strings.Contains(detail.Steps[0].Diagnostic, "incompatible") {
|
||||
t.Fatalf("stale fixture provenance settled as success: %+v", detail)
|
||||
}
|
||||
mu.Lock()
|
||||
if calls != 2 {
|
||||
t.Fatalf("fixture probes = %d, want one per target-aware stage", calls)
|
||||
}
|
||||
mu.Unlock()
|
||||
}
|
||||
|
||||
func TestVerificationFixtureCancellationClosesProbe(t *testing.T) {
|
||||
started := make(chan struct{})
|
||||
canceled := make(chan struct{})
|
||||
fixture := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
close(started)
|
||||
<-r.Context().Done()
|
||||
close(canceled)
|
||||
}))
|
||||
defer fixture.Close()
|
||||
registry := newVerificationFixtureRegistry([]VerificationFixture{{ID: verificationFixtureID, Endpoint: fixture.URL, Backend: "branch-api", Revision: "abc123", Capability: "controlled-state", State: "ready"}})
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
done := make(chan error, 1)
|
||||
go func() {
|
||||
_, err := registry.verify(ctx, stage.VerificationTarget{Backend: "branch-api", Revision: "abc123", Capability: "controlled-state", Fixture: verificationFixtureID})
|
||||
done <- err
|
||||
}()
|
||||
select {
|
||||
case <-started:
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("fixture probe did not start")
|
||||
}
|
||||
cancel()
|
||||
select {
|
||||
case err := <-done:
|
||||
if err == nil {
|
||||
t.Fatal("canceled fixture probe succeeded")
|
||||
}
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("canceled fixture probe did not return")
|
||||
}
|
||||
select {
|
||||
case <-canceled:
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("canceled fixture request was not cleaned up")
|
||||
}
|
||||
}
|
||||
|
||||
func TestVerificationFixtureRejectsUnknownAndMismatchedProvenance(t *testing.T) {
|
||||
base := map[string]string{"backend": "branch-api", "revision": "abc123", "capability": "controlled-state", "fixture": verificationFixtureID, "state": "ready"}
|
||||
for field, wrong := range map[string]string{"backend": "old-api", "revision": "old-build", "capability": "other-state", "state": "stale"} {
|
||||
t.Run(field, func(t *testing.T) {
|
||||
response := make(map[string]string, len(base))
|
||||
for key, value := range base {
|
||||
response[key] = value
|
||||
}
|
||||
response[field] = wrong
|
||||
fixture := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
_ = json.NewEncoder(w).Encode(response)
|
||||
}))
|
||||
defer fixture.Close()
|
||||
registry := newVerificationFixtureRegistry([]VerificationFixture{{ID: verificationFixtureID, Endpoint: fixture.URL, Backend: base["backend"], Revision: base["revision"], Capability: base["capability"], State: base["state"]}})
|
||||
_, err := registry.verify(context.Background(), stage.VerificationTarget{Backend: base["backend"], Revision: base["revision"], Capability: base["capability"], Fixture: verificationFixtureID})
|
||||
if err == nil || !strings.Contains(err.Error(), "incompatible") {
|
||||
t.Fatalf("mismatched %s provenance was accepted: %v", field, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
registry := newVerificationFixtureRegistry(nil)
|
||||
if _, err := registry.verify(context.Background(), stage.VerificationTarget{Backend: "branch-api", Revision: "abc123", Capability: "controlled-state", Fixture: "unknown"}); err == nil || !strings.Contains(err.Error(), "unknown") {
|
||||
t.Fatalf("unknown fixture was accepted: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -49,6 +49,10 @@ type Options struct {
|
||||
// read time with no model call, which keeps tests and unconfigured servers
|
||||
// deterministic.
|
||||
StepVerdicts func(context.Context, map[string]any) string
|
||||
// VerificationFixtures are explicit server-owned HTTP fixtures for optional
|
||||
// target-aware verification. Definitions can name only a registered ID; they
|
||||
// cannot select an endpoint or supply provenance.
|
||||
VerificationFixtures []VerificationFixture
|
||||
}
|
||||
|
||||
// Handler serves the persistent connection with default options. Canceling ctx
|
||||
@@ -81,6 +85,10 @@ func HandlerWithOptions(ctx context.Context, options Options, stores ...*pipelin
|
||||
if runErr == nil && options.StepVerdicts != nil {
|
||||
runs.verdict = options.StepVerdicts
|
||||
}
|
||||
if runErr == nil {
|
||||
fixtures := newVerificationFixtureRegistry(options.VerificationFixtures)
|
||||
runs.verifyTarget = fixtures.verify
|
||||
}
|
||||
if runErr == nil {
|
||||
runs.schedulerWG.Add(1)
|
||||
go runs.scheduleLoop()
|
||||
|
||||
+34
-9
@@ -14,16 +14,28 @@ import (
|
||||
// before any prompt is admitted, so restart recovery can interrupt the agent.
|
||||
type AgentClient struct {
|
||||
OpenCodeClient
|
||||
WritePaths []string
|
||||
Tools []string // nil is the legacy manual approval bundle; empty denies all
|
||||
OnSession func(string, bool) error
|
||||
SessionID string
|
||||
Transcript string
|
||||
Policy ToolPolicy
|
||||
WorkStarted bool
|
||||
WritePaths []string
|
||||
Tools []string // nil is the legacy manual approval bundle; empty denies all
|
||||
// ArtifactAccess is supplied only by the runner. Write is this visit's
|
||||
// directory; Read contains this and completed earlier visits in the same run.
|
||||
ArtifactAccess ArtifactAccess
|
||||
LandingAccess LandingAccess
|
||||
OnSession func(string, bool) error
|
||||
SessionID string
|
||||
Transcript string
|
||||
Policy ToolPolicy
|
||||
WorkStarted bool
|
||||
}
|
||||
|
||||
func (c OpenCodeClient) Interrupt(ctx context.Context, id string) error {
|
||||
return c.interrupt(ctx, id, false)
|
||||
}
|
||||
|
||||
func (c OpenCodeClient) InterruptAndConfirm(ctx context.Context, id string) error {
|
||||
return c.interrupt(ctx, id, true)
|
||||
}
|
||||
|
||||
func (c OpenCodeClient) interrupt(ctx context.Context, id string, confirm bool) error {
|
||||
if !sessionIDPattern.MatchString(id) {
|
||||
return fmt.Errorf("invalid session ID")
|
||||
}
|
||||
@@ -43,7 +55,20 @@ func (c OpenCodeClient) Interrupt(ctx context.Context, id string) error {
|
||||
c.Backend = connection.Backend
|
||||
}
|
||||
_, err := c.call(ctx, "post", "/api/session/"+id+"/interrupt", map[string]any{})
|
||||
return err
|
||||
if err != nil || !confirm {
|
||||
return err
|
||||
}
|
||||
// An interrupt acknowledgement is not a cleanup receipt. Keep ownership
|
||||
// until the harness positively reports settlement, bounded by the caller.
|
||||
if c.Backend == "v1" {
|
||||
_, err := c.call(ctx, "post", "/api/experimental/session/"+id+"/wait", nil)
|
||||
return err
|
||||
}
|
||||
// The failed stage wait already records deadline evidence. Cleanup settlement
|
||||
// must still be confirmed, but must not add a second duplicate operation entry.
|
||||
confirmed := c
|
||||
confirmed.Operations = nil
|
||||
return confirmed.waitForSessionIdle(ctx, id)
|
||||
}
|
||||
|
||||
func (c *AgentClient) Generate(ctx context.Context, config ModelConfig, prompt string) (text string, err error) {
|
||||
@@ -97,7 +122,7 @@ func (c *AgentClient) Generate(ctx context.Context, config ModelConfig, prompt s
|
||||
defer cancel()
|
||||
if err != nil {
|
||||
c.captureTimedOutEvidence(ctx, err, config, watch)
|
||||
if stopErr := c.Interrupt(cleanup, c.SessionID); stopErr != nil {
|
||||
if stopErr := c.InterruptAndConfirm(cleanup, c.SessionID); stopErr != nil {
|
||||
err = fmt.Errorf("agent interrupt failed; retained session %s needs inspection: %w", c.SessionID, errors.Join(err, stopErr, ErrStageCleanup))
|
||||
return
|
||||
}
|
||||
|
||||
@@ -52,8 +52,8 @@ func TestAgentOwnershipPermissionsAndInterrupt(t *testing.T) {
|
||||
t.Fatal("unexpected call", args)
|
||||
return nil, nil
|
||||
}
|
||||
if _, err := client.Generate(context.Background(), ModelConfig{}, "This call has no tools."); err == nil || !interrupted || owned {
|
||||
t.Fatal("failed generation did not stop and release ownership")
|
||||
if _, err := client.Generate(context.Background(), ModelConfig{}, "This call has no tools."); !errors.Is(err, ErrStageCleanup) || !interrupted || !owned {
|
||||
t.Fatal("interrupt acknowledgement without idle confirmation must retain ownership")
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -82,6 +82,9 @@ func TestBudgetsAccumulateAcrossRetriesAndBlockFurtherGeneration(t *testing.T) {
|
||||
var sessions, generations atomic.Int64
|
||||
client := &OpenCodeClient{command: func(ctx context.Context, _ string, args ...string) ([]byte, error) {
|
||||
path := args[2]
|
||||
if path == "/api/session/active" {
|
||||
return []byte(`{"data":{}}`), nil
|
||||
}
|
||||
if path == "/api/session" {
|
||||
return []byte(fmt.Sprintf(`{"data":{"id":"ses_attempt_%d"}}`, sessions.Add(1))), nil
|
||||
}
|
||||
@@ -147,6 +150,9 @@ func TestLiveAgentBudgetInterruptsAndDoesNotReplay(t *testing.T) {
|
||||
return []byte(`{"data":{"id":"ses_agent_budget"}}`), nil
|
||||
}
|
||||
if path == "/api/session/active" {
|
||||
if interrupts.Load() > 0 {
|
||||
return []byte(`{"data":{}}`), nil
|
||||
}
|
||||
<-ctx.Done()
|
||||
return nil, ctx.Err()
|
||||
}
|
||||
|
||||
+14
-2
@@ -59,7 +59,7 @@ func (ModelStage) Spec(def Definition) (ModelSpec, error) {
|
||||
}
|
||||
for block := range def.Blocks {
|
||||
switch block {
|
||||
case "output", "prompt", "edges", "configs", "tools", "permissions":
|
||||
case "output", "prompt", "edges", "configs", "tools", "permissions", "checks", "verification":
|
||||
default:
|
||||
return ModelSpec{}, fmt.Errorf("unexpected block %q for model stage", block)
|
||||
}
|
||||
@@ -70,6 +70,9 @@ func (ModelStage) Spec(def Definition) (ModelSpec, error) {
|
||||
if _, err := StagePermissions(def); err != nil {
|
||||
return ModelSpec{}, err
|
||||
}
|
||||
if _, err := VerificationTargetFor(def); err != nil {
|
||||
return ModelSpec{}, err
|
||||
}
|
||||
for _, key := range []string{"provider", "model"} {
|
||||
if def.Configs[key] == "" {
|
||||
return ModelSpec{}, fmt.Errorf("missing config %s", key)
|
||||
@@ -77,7 +80,7 @@ func (ModelStage) Spec(def Definition) (ModelSpec, error) {
|
||||
}
|
||||
for key := range def.Configs {
|
||||
switch key {
|
||||
case "provider", "model", "retries", "timeout", "budget_usd", "budget_tokens", "harness":
|
||||
case "provider", "model", "retries", "timeout", "budget_usd", "budget_tokens", "harness", "delivery":
|
||||
default:
|
||||
return ModelSpec{}, fmt.Errorf("unknown config %q", key)
|
||||
}
|
||||
@@ -92,6 +95,15 @@ func (ModelStage) Spec(def Definition) (ModelSpec, error) {
|
||||
if backend != "opencode" && backend != "pi" {
|
||||
return ModelSpec{}, fmt.Errorf("harness must be opencode or pi")
|
||||
}
|
||||
if delivery := def.Configs["delivery"]; delivery != "" && delivery != "true" {
|
||||
return ModelSpec{}, fmt.Errorf("delivery must be true when set")
|
||||
}
|
||||
if def.Configs["delivery"] == "true" && backend != "opencode" {
|
||||
return ModelSpec{}, fmt.Errorf("delivery is supported only by the opencode harness")
|
||||
}
|
||||
if _, present := def.Blocks["checks"]; present && def.Configs["delivery"] != "true" {
|
||||
return ModelSpec{}, fmt.Errorf("checks block is supported only by delivery stages")
|
||||
}
|
||||
if backend == "pi" {
|
||||
tools, _, _ := ToolNames(def)
|
||||
if _, err := piTools(tools); err != nil {
|
||||
|
||||
@@ -36,8 +36,8 @@ func TestModelFailureCategoriesAreSanitizedAndDistinct(t *testing.T) {
|
||||
if typed := wrapModelStageFailure("worker", &openCodeError{operation: "get /api/session", status: 503}); !strings.Contains(typed.Error(), "HARNESS TRANSPORT") || !errors.As(typed, new(*openCodeError)) {
|
||||
t.Fatalf("typed harness error was not classified: %v", typed)
|
||||
}
|
||||
if deadline := wrapModelStageFailure("worker", context.DeadlineExceeded); !strings.Contains(deadline.Error(), "HARNESS TRANSPORT") || !errors.Is(deadline, context.DeadlineExceeded) {
|
||||
t.Fatalf("context deadline was not classified as harness: %v", deadline)
|
||||
if deadline := wrapModelStageFailure("worker", context.DeadlineExceeded); strings.Contains(deadline.Error(), "HARNESS TRANSPORT") || !errors.Is(deadline, context.DeadlineExceeded) {
|
||||
t.Fatalf("bare context deadline was misclassified or lost: %v", deadline)
|
||||
}
|
||||
if modelFailureCategory(nil) != "" || wrapModelStageFailure("worker", nil) != nil {
|
||||
t.Fatal("nil failure was classified")
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Retained model-stage failure categories. A strict final-result validation
|
||||
@@ -13,8 +14,27 @@ import (
|
||||
const (
|
||||
finalResultValidation = "FINAL RESULT VALIDATION"
|
||||
harnessTransport = "HARNESS TRANSPORT"
|
||||
stageDeadline = "STAGE DEADLINE"
|
||||
)
|
||||
|
||||
// stageTimeoutCause identifies this attempt's configured timer, not a parent
|
||||
// cancellation or a request's own deadline. It is carried through budget contexts.
|
||||
type stageTimeoutCause struct{ timeout time.Duration }
|
||||
|
||||
func (e *stageTimeoutCause) Error() string { return "configured stage timeout reached" }
|
||||
func (e *stageTimeoutCause) Unwrap() error { return context.DeadlineExceeded }
|
||||
|
||||
type stageDeadlineError struct {
|
||||
timeout time.Duration
|
||||
session string
|
||||
}
|
||||
|
||||
func (e *stageDeadlineError) Error() string {
|
||||
return fmt.Sprintf("configured stage deadline reached (timeout %s) while waiting for session idle; retained OpenCode session %.128s was still active at the last poll; stage had not finished", e.timeout, RedactEvidence(e.session))
|
||||
}
|
||||
|
||||
func (e *stageDeadlineError) Unwrap() error { return context.DeadlineExceeded }
|
||||
|
||||
// finalResultGuidance is a fixed sanitized sentence appended only to validation
|
||||
// diagnostics. It states the retained side effects and the cost of a full retry.
|
||||
const finalResultGuidance = "completed edits and tool side effects are retained and not undone; a full retry repeats work"
|
||||
@@ -51,11 +71,14 @@ func (c *PiClient) workAdmitted() bool { return c.WorkStarted }
|
||||
// the label stays sanitized and bounded. Validation is checked first: a
|
||||
// structured-output failure is never reported as a transport failure and vice versa.
|
||||
func modelFailureCategory(err error) string {
|
||||
var deadline *stageDeadlineError
|
||||
switch {
|
||||
case errors.Is(err, ErrInvalidModelResult):
|
||||
return finalResultValidation
|
||||
case harnessTransportFailure(err), errors.Is(err, context.DeadlineExceeded):
|
||||
case harnessTransportFailure(err):
|
||||
return harnessTransport
|
||||
case errors.As(err, &deadline):
|
||||
return stageDeadline
|
||||
default:
|
||||
return ""
|
||||
}
|
||||
@@ -73,6 +96,8 @@ func wrapModelStageFailure(key string, err error) error {
|
||||
return fmt.Errorf("model stage %s: %s: %w; %s", key, finalResultValidation, err, finalResultGuidance)
|
||||
case harnessTransport:
|
||||
return fmt.Errorf("model stage %s: %s: %w", key, harnessTransport, err)
|
||||
case stageDeadline:
|
||||
return fmt.Errorf("model stage %s: %s: %w", key, stageDeadline, err)
|
||||
default:
|
||||
return fmt.Errorf("model stage %s: %w", key, err)
|
||||
}
|
||||
|
||||
@@ -61,7 +61,7 @@ func (m ModelStage) Run(ctx context.Context, def Definition, input Values) (Resu
|
||||
return Result{}, err
|
||||
}
|
||||
spec.Config.Budget.Attempt()
|
||||
attemptCtx, cancel := context.WithTimeout(ctx, spec.Config.Timeout)
|
||||
attemptCtx, cancel := context.WithTimeoutCause(ctx, spec.Config.Timeout, &stageTimeoutCause{timeout: spec.Config.Timeout})
|
||||
text, generateErr := client.Generate(attemptCtx, spec.Config, prompt)
|
||||
if generateErr == nil && attemptCtx.Err() != nil {
|
||||
generateErr = attemptCtx.Err()
|
||||
@@ -96,11 +96,16 @@ func (m ModelStage) Run(ctx context.Context, def Definition, input Values) (Resu
|
||||
}
|
||||
} else {
|
||||
err = generateErr
|
||||
if attemptCtx.Err() != nil && !errors.Is(err, ErrBudgetExceeded) && !errors.Is(err, ErrBudgetAccounting) {
|
||||
// Do not append the same context error twice: the wait/transport
|
||||
// error may already preserve it, with a more precise diagnostic.
|
||||
if attemptCtx.Err() != nil && !errors.Is(err, attemptCtx.Err()) && !errors.Is(err, ErrBudgetExceeded) && !errors.Is(err, ErrBudgetAccounting) {
|
||||
err = errors.Join(err, attemptCtx.Err())
|
||||
}
|
||||
cancel()
|
||||
if ctx.Err() != nil {
|
||||
if harnessTransportFailure(err) {
|
||||
return Result{}, wrapModelStageFailure(def.Key, err)
|
||||
}
|
||||
return Result{}, ctx.Err()
|
||||
}
|
||||
if errors.Is(err, ErrBudgetExceeded) || errors.Is(err, ErrBudgetAccounting) || errors.Is(err, ErrStageCleanup) {
|
||||
|
||||
@@ -217,6 +217,11 @@ func (c OpenCodeClient) waitForSessionIdle(ctx context.Context, sessionID string
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
timer.Stop()
|
||||
waitErr := fmt.Errorf("waiting for session idle; retained OpenCode session %.128s; stage had not finished: %w", RedactEvidence(sessionID), ctx.Err())
|
||||
var timeout *stageTimeoutCause
|
||||
if errors.Is(ctx.Err(), context.DeadlineExceeded) && errors.As(context.Cause(ctx), &timeout) {
|
||||
waitErr = &stageDeadlineError{timeout: timeout.timeout, session: sessionID}
|
||||
}
|
||||
// A settled session is the only success signal; retain one bounded
|
||||
// sanitized record so a timed-out wait is not an empty transcript.
|
||||
c.Operations.record(OperationEvidence{
|
||||
@@ -225,9 +230,9 @@ func (c OpenCodeClient) waitForSessionIdle(ctx context.Context, sessionID string
|
||||
Executable: opencode.Executable(),
|
||||
Directory: c.Directory,
|
||||
ExitCode: -1,
|
||||
Diagnostic: "stage deadline reached while waiting for the agent session to become idle",
|
||||
Diagnostic: waitErr.Error(),
|
||||
}, started)
|
||||
return ctx.Err()
|
||||
return waitErr
|
||||
case <-timer.C:
|
||||
}
|
||||
}
|
||||
@@ -316,7 +321,7 @@ func (c *OpenCodeClient) Generate(ctx context.Context, config ModelConfig, promp
|
||||
cleanup, cancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second)
|
||||
defer cancel()
|
||||
if err != nil {
|
||||
if stopErr := c.Interrupt(cleanup, session.Data.ID); stopErr != nil {
|
||||
if stopErr := c.InterruptAndConfirm(cleanup, session.Data.ID); stopErr != nil {
|
||||
err = fmt.Errorf("retained OpenCode session %s; interrupt failed: %w", session.Data.ID, errors.Join(err, stopErr, ErrStageCleanup))
|
||||
return
|
||||
}
|
||||
|
||||
+15
-3
@@ -28,6 +28,7 @@ type PiClient struct {
|
||||
// ArtifactsDir is exported to the spawned Pi process as
|
||||
// PIPELINES_ARTIFACTS_DIR so its tools can write stage artifacts.
|
||||
ArtifactsDir string
|
||||
ArtifactAccess ArtifactAccess
|
||||
Policy ToolPolicy
|
||||
OnSession func(string, bool) error
|
||||
OnOwnership func(string, string, bool) error
|
||||
@@ -128,7 +129,7 @@ func (c *PiClient) Generate(ctx context.Context, config ModelConfig, prompt stri
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
policy := map[string]any{"directory": c.Directory, "tools": tools, "paths": c.WritePaths, "edit": c.Policy.Edit, "shell": c.Policy.Shell}
|
||||
policy := map[string]any{"directory": c.Directory, "tools": tools, "paths": c.WritePaths, "edit": c.Policy.Edit, "shell": c.Policy.Shell, "artifact_write": c.ArtifactAccess.Write, "artifact_read": c.ArtifactAccess.Read}
|
||||
if c.WritePaths == nil {
|
||||
policy["paths"] = []string{}
|
||||
}
|
||||
@@ -237,6 +238,10 @@ func (c *PiClient) Generate(ctx context.Context, config ModelConfig, prompt stri
|
||||
}
|
||||
close(done)
|
||||
close(heartbeatDone)
|
||||
// agent_end/abort acknowledgement and the parent's exit alone do not
|
||||
// prove its shell descendants exited. A surviving process group retains
|
||||
// ownership rather than handing the resource to another stage.
|
||||
quiet = quiet && errors.Is(syscall.Kill(-c.cmd.Process.Pid, 0), syscall.ESRCH)
|
||||
if quiet {
|
||||
_ = os.WriteFile(filepath.Join(c.Reference, "quiescent"), []byte("quiescent"), 0600)
|
||||
}
|
||||
@@ -531,9 +536,16 @@ func InterruptPi(ctx context.Context, reference string) error {
|
||||
if !filepath.IsAbs(reference) {
|
||||
return fmt.Errorf("invalid retained Pi reference")
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(reference, "owner.json")); err != nil {
|
||||
data, err := os.ReadFile(filepath.Join(reference, "owner.json"))
|
||||
if err != nil {
|
||||
return errors.Join(err, ErrStageCleanup)
|
||||
}
|
||||
var owner struct {
|
||||
PID int `json:"pid"`
|
||||
}
|
||||
if json.Unmarshal(data, &owner) != nil || owner.PID <= 1 {
|
||||
return errors.Join(fmt.Errorf("invalid retained Pi owner"), ErrStageCleanup)
|
||||
}
|
||||
old := time.Unix(0, 0)
|
||||
if err := os.Chtimes(filepath.Join(reference, "lease"), old, old); err != nil {
|
||||
return err
|
||||
@@ -541,7 +553,7 @@ func InterruptPi(ctx context.Context, reference string) error {
|
||||
ticker := time.NewTicker(100 * time.Millisecond)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
if data, err := os.ReadFile(filepath.Join(reference, "quiescent")); err == nil && string(data) == "quiescent" {
|
||||
if data, err := os.ReadFile(filepath.Join(reference, "quiescent")); err == nil && string(data) == "quiescent" && errors.Is(syscall.Kill(owner.PID, 0), syscall.ESRCH) && errors.Is(syscall.Kill(-owner.PID, 0), syscall.ESRCH) {
|
||||
return nil
|
||||
}
|
||||
select {
|
||||
|
||||
@@ -8,6 +8,24 @@ export function permit(policy, tool, args) {
|
||||
if (policy.shell !== null && !policy.shell.includes(args.command)) return "Shell command is not permitted";
|
||||
return null;
|
||||
}
|
||||
const artifact = (absolute, write) => {
|
||||
const permitted = write ? [policy.artifact_write] : policy.artifact_read;
|
||||
for (const directory of permitted) {
|
||||
if (typeof directory !== "string" || !path.isAbsolute(directory)) continue;
|
||||
let parent = absolute;
|
||||
while (!fs.existsSync(parent)) {
|
||||
const next = path.dirname(parent);
|
||||
if (next === parent) break;
|
||||
parent = next;
|
||||
}
|
||||
if (!fs.existsSync(parent)) continue;
|
||||
const resolved = fs.realpathSync(parent);
|
||||
const target = path.join(resolved, path.relative(parent, absolute));
|
||||
const root = fs.realpathSync(directory);
|
||||
if (target === root || target.startsWith(root + path.sep)) return true;
|
||||
}
|
||||
return false;
|
||||
};
|
||||
if (tool === "edit" || tool === "write") {
|
||||
if (typeof args.path !== "string") return "Missing file path";
|
||||
if (args.path.startsWith("~") || args.path.startsWith("@")) return "Use a literal path, not a harness path shortcut";
|
||||
@@ -20,7 +38,10 @@ export function permit(policy, tool, args) {
|
||||
}
|
||||
const resolved = fs.realpathSync(parent);
|
||||
const root = fs.realpathSync(policy.directory);
|
||||
if (resolved !== root && !resolved.startsWith(root + path.sep)) return "Edit escapes run directory";
|
||||
if (resolved !== root && !resolved.startsWith(root + path.sep)) {
|
||||
if (!artifact(absolute, true)) return "Edit escapes run directory";
|
||||
return null;
|
||||
}
|
||||
const relative = path.relative(path.resolve(policy.directory), absolute).split(path.sep).join("/");
|
||||
const canonical = path.relative(root, path.join(resolved, path.relative(parent, absolute))).split(path.sep).join("/");
|
||||
if (relative.startsWith("../") || path.isAbsolute(relative)) return "Edit escapes run directory";
|
||||
@@ -31,9 +52,19 @@ export function permit(policy, tool, args) {
|
||||
});
|
||||
if (policy.edit !== null && (!matches(relative) || !matches(canonical))) return "Edit is outside stage patterns";
|
||||
}
|
||||
if (tool === "read" && typeof args.path === "string") {
|
||||
if (["read", "grep", "find"].includes(tool) && typeof args.path === "string") {
|
||||
const name = path.basename(args.path);
|
||||
if (/^(.*\.env(?:\..*)?|pipelines\.db.*)$/.test(name)) return "Protected file cannot be read";
|
||||
const absolute = path.resolve(policy.directory, args.path);
|
||||
let parent = absolute;
|
||||
while (!fs.existsSync(parent)) {
|
||||
const next = path.dirname(parent);
|
||||
if (next === parent) return "Cannot resolve read path";
|
||||
parent = next;
|
||||
}
|
||||
const resolved = fs.realpathSync(parent);
|
||||
const root = fs.realpathSync(policy.directory);
|
||||
if (resolved !== root && !resolved.startsWith(root + path.sep) && !artifact(absolute, false)) return "Read escapes run directory";
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,72 @@
|
||||
package stage
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestPiPolicyArtifactScope(t *testing.T) {
|
||||
if _, err := exec.LookPath("node"); err != nil {
|
||||
t.Skip("node is required to exercise the Pi policy extension")
|
||||
}
|
||||
root := t.TempDir()
|
||||
workspace := filepath.Join(root, "workspace")
|
||||
current := filepath.Join(root, "artifacts [7]", "2")
|
||||
prior := filepath.Join(root, "artifacts [7]", "1")
|
||||
sibling := filepath.Join(root, "artifacts [7]", "3")
|
||||
for _, directory := range []string{workspace, current, prior, sibling} {
|
||||
if err := os.MkdirAll(directory, 0700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
policy := map[string]any{
|
||||
"directory": workspace,
|
||||
"tools": []string{"read", "grep", "find", "edit"},
|
||||
"paths": []string{},
|
||||
"edit": nil,
|
||||
"shell": nil,
|
||||
"artifact_write": current,
|
||||
"artifact_read": []string{current, prior},
|
||||
}
|
||||
policyJSON, err := json.Marshal(policy)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
policyFile := filepath.Join(root, "policy.mjs")
|
||||
if err := os.WriteFile(policyFile, piPolicy, 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
script := `const { permit } = await import(process.argv[1]);
|
||||
const policy = JSON.parse(process.argv[2]);
|
||||
const checks = JSON.parse(process.argv[3]);
|
||||
console.log(JSON.stringify(checks.map(([tool, path]) => permit(policy, tool, {path}))));`
|
||||
checks := [][2]string{
|
||||
{"edit", filepath.Join(current, "report.md")},
|
||||
{"edit", filepath.Join(prior, "report.md")},
|
||||
{"read", filepath.Join(prior, "report.md")},
|
||||
{"grep", filepath.Join(prior, "report.md")},
|
||||
{"find", filepath.Join(current, "reports")},
|
||||
{"read", filepath.Join(sibling, "report.md")},
|
||||
{"grep", filepath.Join(root, "other-run", "1", "report.md")},
|
||||
{"find", filepath.Join(current, "..", "3", "report.md")},
|
||||
{"read", filepath.Join(prior, ".env")},
|
||||
}
|
||||
checksJSON, err := json.Marshal(checks)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
output, err := exec.Command("node", "--input-type=module", "--eval", script, "file://"+policyFile, string(policyJSON), string(checksJSON)).Output()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var got []any
|
||||
if err := json.Unmarshal(output, &got); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(got) != len(checks) || got[0] != nil || got[1] == nil || got[2] != nil || got[3] != nil || got[4] != nil || got[5] == nil || got[6] == nil || got[7] == nil || got[8] == nil {
|
||||
t.Fatalf("Pi artifact policy results = %#v", got)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,44 @@
|
||||
package stage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"syscall"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestResourcePiReceiptRequiresProcessExit(t *testing.T) {
|
||||
// A test-owned process only: no Pi provider, shell/ADB command or phone.
|
||||
cmd := exec.Command("sleep", "5")
|
||||
cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
|
||||
if err := cmd.Start(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() { cmd.Process.Kill(); cmd.Wait() }()
|
||||
dir := t.TempDir()
|
||||
owner, _ := json.Marshal(map[string]any{"pid": cmd.Process.Pid})
|
||||
for name, data := range map[string][]byte{"owner.json": owner, "lease": []byte("lease"), "quiescent": []byte("quiescent")} {
|
||||
if err := os.WriteFile(filepath.Join(dir, name), data, 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Millisecond)
|
||||
defer cancel()
|
||||
if err := InterruptPi(ctx, dir); !errors.Is(err, ErrStageCleanup) {
|
||||
t.Fatal("receipt authorized handoff while process lived", err)
|
||||
}
|
||||
if err := cmd.Process.Kill(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cmd.Wait()
|
||||
ctx2, cancel2 := context.WithTimeout(context.Background(), time.Second)
|
||||
defer cancel2()
|
||||
if err := InterruptPi(ctx2, dir); err != nil {
|
||||
t.Fatal("confirmed process exit not accepted", err)
|
||||
}
|
||||
}
|
||||
@@ -251,7 +251,7 @@ func TestWaitForSessionIdleRecordsOneTimeoutEvidence(t *testing.T) {
|
||||
t.Fatalf("expected exactly one timeout evidence, got %d", len(log.Failures))
|
||||
}
|
||||
entry := log.Failures[0]
|
||||
if entry.Operation != "wait for session idle" || !strings.Contains(entry.Diagnostic, "deadline reached") {
|
||||
if entry.Operation != "wait for session idle" || !strings.Contains(entry.Diagnostic, "context deadline exceeded") {
|
||||
t.Fatalf("timeout evidence not sanitized as expected: %+v", entry)
|
||||
}
|
||||
clean := &OperationLog{}
|
||||
|
||||
@@ -0,0 +1,140 @@
|
||||
package stage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.a-wan.net/alan/pipelines/internal/opencode"
|
||||
)
|
||||
|
||||
func TestConfiguredStageDeadlineWhileSessionActive(t *testing.T) {
|
||||
def := modelDefinition()
|
||||
def.Blocks["tools"] = "edit"
|
||||
def.Configs["timeout"] = "20ms"
|
||||
def.Configs["retries"] = "2"
|
||||
log := &OperationLog{}
|
||||
creates, prompts, interrupts := 0, 0, 0
|
||||
agent := &AgentClient{OpenCodeClient: OpenCodeClient{Backend: "v2", Operations: log, sessionPollInterval: time.Hour}}
|
||||
agent.command = func(_ context.Context, _ string, args ...string) ([]byte, error) {
|
||||
switch args[2] {
|
||||
case "/api/session":
|
||||
creates++
|
||||
return []byte(`{"data":{"id":"ses_deadline"}}`), nil
|
||||
case "/api/session/ses_deadline/prompt":
|
||||
prompts++
|
||||
return []byte(`{}`), nil
|
||||
case "/api/session/active":
|
||||
return []byte(`{"data":{"ses_deadline":{"type":"running"}}}`), nil
|
||||
case "/api/session/ses_deadline/interrupt":
|
||||
interrupts++
|
||||
return []byte(`{}`), nil
|
||||
default:
|
||||
// Best-effort timeout readback/usage cannot convert the failure to success.
|
||||
return []byte(`{}`), nil
|
||||
}
|
||||
}
|
||||
result, err := (ModelStage{Client: agent}).Run(context.Background(), def, nil)
|
||||
if !errors.Is(err, context.DeadlineExceeded) || modelFailureCategory(err) != stageDeadline || errors.Is(err, ErrHarnessTransport) {
|
||||
t.Fatalf("deadline identity/category: %v", err)
|
||||
}
|
||||
for _, want := range []string{"STAGE DEADLINE", "worker", "timeout 20ms", "waiting for session idle", "ses_deadline", "stage had not finished"} {
|
||||
if !strings.Contains(err.Error(), want) {
|
||||
t.Fatalf("missing %q: %v", want, err)
|
||||
}
|
||||
}
|
||||
if result.Output != nil || creates != 1 || prompts != 1 || interrupts != 1 || !agent.WorkStarted {
|
||||
t.Fatalf("work replayed, cleanup skipped or incomplete turn accepted: %+v creates=%d prompts=%d interrupts=%d", result, creates, prompts, interrupts)
|
||||
}
|
||||
if len(log.Failures) != 1 || !strings.Contains(log.Failures[0].Diagnostic, "timeout 20ms") {
|
||||
t.Fatalf("deadline evidence missing: %+v", log.Failures)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWaitCancellationIsNotConfiguredStageDeadline(t *testing.T) {
|
||||
for _, cause := range []error{context.Canceled, ErrBudgetExceeded, ErrBudgetAccounting} {
|
||||
t.Run(cause.Error(), func(t *testing.T) {
|
||||
attempt, stop := context.WithTimeoutCause(context.Background(), time.Hour, &stageTimeoutCause{timeout: time.Hour})
|
||||
defer stop()
|
||||
ctx, cancel := context.WithCancelCause(attempt)
|
||||
log := &OperationLog{}
|
||||
client := OpenCodeClient{Operations: log, command: func(context.Context, string, ...string) ([]byte, error) {
|
||||
cancel(cause)
|
||||
return []byte(`{"data":{"ses_wait":{"type":"running"}}}`), nil
|
||||
}}
|
||||
err := client.waitForSessionIdle(ctx, "ses_wait")
|
||||
if !errors.Is(err, context.Canceled) || modelFailureCategory(err) != "" || len(log.Failures) != 1 {
|
||||
t.Fatalf("cancellation misclassified: %v, %+v", err, log.Failures)
|
||||
}
|
||||
if strings.Contains(log.Failures[0].Diagnostic, "deadline") {
|
||||
t.Fatalf("cancellation recorded as deadline: %+v", log.Failures)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeadlineWithTransportRetainsUnderlyingIdentity(t *testing.T) {
|
||||
transport := &openCodeError{operation: "get /api/session/active", status: 503, cause: errors.New("request failed")}
|
||||
err := wrapModelStageFailure("worker", errors.Join(&stageDeadlineError{timeout: time.Second, session: "ses_wait"}, transport))
|
||||
var got *openCodeError
|
||||
if modelFailureCategory(err) != harnessTransport || !errors.Is(err, context.DeadlineExceeded) || !errors.As(err, &got) || got != transport || !errors.Is(err, transport.cause) {
|
||||
t.Fatalf("combined failure lost transport/deadline identity: %v", err)
|
||||
}
|
||||
if !strings.Contains(err.Error(), "HTTP 503") || !strings.Contains(err.Error(), "get /api/session/active") {
|
||||
t.Fatalf("combined failure lost operation/status: %v", err)
|
||||
}
|
||||
bounded := (&stageDeadlineError{timeout: time.Hour, session: strings.Repeat("s", 10000) + " Bearer private-token"}).Error()
|
||||
if len(bounded) > 512 || strings.Contains(bounded, "private-token") {
|
||||
t.Fatal("deadline diagnostic was not bounded/redacted")
|
||||
}
|
||||
}
|
||||
|
||||
func TestParentDeadlinePreservesModelTransportFailure(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Millisecond)
|
||||
defer cancel()
|
||||
transport := &openCodeError{operation: "get /api/session/active", status: 504, cause: errors.New("request failed")}
|
||||
client := generatorFunc(func(ctx context.Context, _ ModelConfig, _ string) (string, error) {
|
||||
<-ctx.Done()
|
||||
return "", errors.Join(ctx.Err(), transport)
|
||||
})
|
||||
_, err := (ModelStage{Client: client}).Run(ctx, modelDefinition(), nil)
|
||||
var got *openCodeError
|
||||
if !errors.Is(err, context.DeadlineExceeded) || !errors.As(err, &got) || got != transport || modelFailureCategory(err) != harnessTransport {
|
||||
t.Fatalf("parent deadline erased underlying request failure: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNativeHTTPAndConnectionFailuresRemainTransport(t *testing.T) {
|
||||
for _, mode := range []string{"http", "connection"} {
|
||||
t.Run(mode, func(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
w.WriteHeader(http.StatusServiceUnavailable)
|
||||
_, _ = w.Write([]byte(`{"message":"service unavailable","token":"private-token"}`))
|
||||
}))
|
||||
defer srv.Close()
|
||||
if mode == "connection" {
|
||||
srv.Close()
|
||||
}
|
||||
log := &OperationLog{}
|
||||
client := OpenCodeClient{Backend: "v1", Operations: log, connection: &opencode.Connection{Backend: "v1", URL: srv.URL}}
|
||||
_, err := client.call(context.Background(), "get", "/api/session", nil)
|
||||
var typed *openCodeError
|
||||
if !errors.As(err, &typed) || modelFailureCategory(err) != harnessTransport || len(log.Failures) != 1 {
|
||||
t.Fatalf("real request failure misclassified: %v %+v", err, log.Failures)
|
||||
}
|
||||
if mode == "http" && (typed.status != 503 || log.Failures[0].HTTPStatus != 503 || !strings.Contains(log.Failures[0].Stdout, "service unavailable")) {
|
||||
t.Fatalf("HTTP status/evidence lost: %v %+v", err, log.Failures)
|
||||
}
|
||||
if mode == "connection" && (log.Failures[0].Diagnostic != "OpenCode native API request failed" || log.Failures[0].NativeOperation != "GET /session") {
|
||||
t.Fatalf("connection evidence lost: %+v", log.Failures)
|
||||
}
|
||||
if strings.Contains(err.Error()+log.Transcript(""), "private-token") {
|
||||
t.Fatal("credential leaked in transport diagnostic/evidence")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -7,6 +7,84 @@ import (
|
||||
"strings"
|
||||
)
|
||||
|
||||
// ArtifactAccess is runtime-only, server-derived access to artifact visits.
|
||||
// Empty fields deliberately grant nothing outside the workspace.
|
||||
type ArtifactAccess struct {
|
||||
Write string
|
||||
Read []string
|
||||
}
|
||||
|
||||
// LandingAccess is the server-derived external capability for a verified
|
||||
// delivery operation. Empty or malformed values grant nothing.
|
||||
type LandingAccess struct {
|
||||
MainWorktree string
|
||||
GitMetadata string
|
||||
LockPath string
|
||||
}
|
||||
|
||||
func (a LandingAccess) valid() bool {
|
||||
for _, path := range []string{a.MainWorktree, a.GitMetadata, a.LockPath} {
|
||||
if path == "" || !filepath.IsAbs(path) || filepath.Clean(path) != path || strings.ContainsAny(path, "*?\\") {
|
||||
return false
|
||||
}
|
||||
}
|
||||
// The server grants the canonical repository metadata directory as OpenCode
|
||||
// V2's smallest external-directory unit. Keep the lock inside that directory
|
||||
// so a caller cannot turn a valid metadata capability into access elsewhere.
|
||||
return a.GitMetadata == filepath.Join(a.MainWorktree, ".git") &&
|
||||
a.LockPath == filepath.Join(a.GitMetadata, "pipelines-landing.lock")
|
||||
}
|
||||
|
||||
// DeliveryEnabled is the narrow, declarative capability check used before the
|
||||
// runner supplies any landing paths. Only a validated model definition can ask
|
||||
// for it; runtime paths are still derived by the server.
|
||||
func DeliveryEnabled(def Definition) (bool, error) {
|
||||
delivery := def.Configs["delivery"]
|
||||
if delivery == "" {
|
||||
return false, nil
|
||||
}
|
||||
if delivery != "true" {
|
||||
return false, fmt.Errorf("delivery must be true when set")
|
||||
}
|
||||
if EffectiveType(def.Type) != ModelType {
|
||||
return false, fmt.Errorf("delivery is supported only by model stages")
|
||||
}
|
||||
if _, err := (ModelStage{}).Spec(def); err != nil {
|
||||
return false, fmt.Errorf("invalid delivery model stage: %w", err)
|
||||
}
|
||||
if _, err := DeliveryChecks(def); err != nil {
|
||||
return false, err
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
|
||||
// DeliveryChecks returns the bounded, definition-declared checks that the
|
||||
// runner executes against a committed delivery candidate before canonical main
|
||||
// is changed. They
|
||||
// are intentionally a block rather than a generic runner setting: each
|
||||
// workflow owns its repository-specific verification commands.
|
||||
func DeliveryChecks(def Definition) ([]string, error) {
|
||||
text := strings.TrimSpace(def.Blocks["checks"])
|
||||
if text == "" {
|
||||
return nil, fmt.Errorf("delivery stages require a nonempty checks block")
|
||||
}
|
||||
var checks []string
|
||||
for _, line := range strings.Split(text, "\n") {
|
||||
check := strings.TrimSpace(line)
|
||||
if check == "" {
|
||||
continue
|
||||
}
|
||||
if len(check) > 4096 {
|
||||
return nil, fmt.Errorf("delivery check exceeds 4096 bytes")
|
||||
}
|
||||
checks = append(checks, check)
|
||||
}
|
||||
if len(checks) == 0 || len(checks) > 20 {
|
||||
return nil, fmt.Errorf("delivery stages require 1 to 20 checks")
|
||||
}
|
||||
return checks, nil
|
||||
}
|
||||
|
||||
// ToolNames validates model-stage permissions, not general graph semantics.
|
||||
// A present empty block explicitly denies all tools.
|
||||
func ToolNames(def Definition) ([]string, bool, error) {
|
||||
@@ -128,12 +206,43 @@ func (c *AgentClient) permissionRules() []map[string]string {
|
||||
}
|
||||
}
|
||||
}
|
||||
// OpenCode V2 resolves an external file to its containing directory and
|
||||
// checks external_directory against <directory>/*. Do not grant a parent:
|
||||
// each visit is deliberately an independent capability.
|
||||
readArtifacts := false
|
||||
writeArtifacts := false
|
||||
for _, tool := range tools {
|
||||
readArtifacts = readArtifacts || tool == "read" || tool == "glob" || tool == "grep"
|
||||
writeArtifacts = writeArtifacts || tool == "edit"
|
||||
}
|
||||
if readArtifacts {
|
||||
for _, directory := range c.ArtifactAccess.Read {
|
||||
if artifactDirectory(directory) {
|
||||
allow("external_directory", filepath.ToSlash(filepath.Join(directory, "*")))
|
||||
}
|
||||
}
|
||||
}
|
||||
if writeArtifacts && artifactDirectory(c.ArtifactAccess.Write) {
|
||||
allow("external_directory", filepath.ToSlash(filepath.Join(c.ArtifactAccess.Write, "*")))
|
||||
allow("edit", filepath.ToSlash(filepath.Join(c.ArtifactAccess.Write, "*")))
|
||||
}
|
||||
if c.LandingAccess.valid() {
|
||||
allow("external_directory", filepath.ToSlash(filepath.Join(c.LandingAccess.MainWorktree, "*")))
|
||||
allow("external_directory", filepath.ToSlash(filepath.Join(c.LandingAccess.GitMetadata, "*")))
|
||||
}
|
||||
for _, pattern := range []string{"../*", "*/../*"} {
|
||||
rules = append(rules, map[string]string{"action": "external_directory", "resource": pattern, "effect": "deny"})
|
||||
}
|
||||
for _, pattern := range []string{"*.env", "*.env.*", "pipelines.db*"} {
|
||||
rules = append(rules, map[string]string{"action": "read", "resource": pattern, "effect": "deny"})
|
||||
}
|
||||
return rules
|
||||
}
|
||||
|
||||
func artifactDirectory(directory string) bool {
|
||||
return directory != "" && filepath.IsAbs(directory) && filepath.Clean(directory) == directory && !strings.ContainsAny(directory, "*?\\")
|
||||
}
|
||||
|
||||
func matchesEditPolicy(patterns []string, path string) bool {
|
||||
for _, pattern := range patterns {
|
||||
// OpenCode resource wildcards match across directory separators.
|
||||
|
||||
@@ -139,3 +139,142 @@ func TestDefaultEditScopeIsLanguageIndependentRunDirectory(t *testing.T) {
|
||||
t.Fatal("external directories were enabled")
|
||||
}
|
||||
}
|
||||
|
||||
func TestArtifactPermissionsAreExactAndToolScoped(t *testing.T) {
|
||||
current := "/artifacts/run [7]/2"
|
||||
prior := "/artifacts/run [7]/1"
|
||||
sibling := "/artifacts/run 7/3"
|
||||
otherRun := "/artifacts/run 8/1"
|
||||
client := &AgentClient{OpenCodeClient: OpenCodeClient{Directory: "/repo"}, Tools: []string{"read", "edit"}, ArtifactAccess: ArtifactAccess{Write: current, Read: []string{current, prior}}}
|
||||
rules := client.permissionRules()
|
||||
for _, path := range []string{current + "/report.md", prior + "/report.md"} {
|
||||
if permissionEffect(rules, "external_directory", path) != "allow" {
|
||||
t.Fatalf("artifact directory denied: %q", path)
|
||||
}
|
||||
}
|
||||
if permissionEffect(rules, "edit", current+"/report.md") != "allow" || permissionEffect(rules, "edit", prior+"/report.md") != "deny" {
|
||||
t.Fatal("artifact writes were not limited to the current visit")
|
||||
}
|
||||
for _, path := range []string{sibling + "/report.md", otherRun + "/report.md", current + "/../3/report.md", "/artifacts/run 7/20/report.md"} {
|
||||
if permissionEffect(rules, "external_directory", path) != "deny" {
|
||||
t.Fatalf("unapproved artifact scope allowed: %q", path)
|
||||
}
|
||||
}
|
||||
client.Tools = []string{}
|
||||
if permissionEffect(client.permissionRules(), "external_directory", current+"/report.md") != "deny" {
|
||||
t.Fatal("no-tool stage gained artifact access")
|
||||
}
|
||||
client.Tools = []string{"read"}
|
||||
rules = client.permissionRules()
|
||||
if permissionEffect(rules, "external_directory", prior+"/report.md") != "allow" || permissionEffect(rules, "edit", current+"/report.md") != "deny" {
|
||||
t.Fatal("read-only artifact stage gained write access")
|
||||
}
|
||||
client.ArtifactAccess = ArtifactAccess{}
|
||||
if permissionEffect(client.permissionRules(), "external_directory", current+"/report.md") != "deny" {
|
||||
t.Fatal("missing artifact scope enabled an external directory")
|
||||
}
|
||||
}
|
||||
|
||||
func TestLandingPermissionsAreExactAndServerDerived(t *testing.T) {
|
||||
access := LandingAccess{MainWorktree: "/repos/pipelines", GitMetadata: "/repos/pipelines/.git", LockPath: "/repos/pipelines/.git/pipelines-landing.lock"}
|
||||
client := &AgentClient{Tools: []string{"read", "shell"}, LandingAccess: access}
|
||||
rules := client.permissionRules()
|
||||
for _, path := range []string{"/repos/pipelines/go.mod", "/repos/pipelines/.git/worktrees/item/HEAD", "/repos/pipelines/.git/pipelines-landing.lock"} {
|
||||
if permissionEffect(rules, "external_directory", path) != "allow" {
|
||||
t.Fatalf("verified landing path denied: %q", path)
|
||||
}
|
||||
}
|
||||
for _, path := range []string{"/repos/other/go.mod", "/repos/pipelines/../other/go.mod", "/Users/alan/.git/config", "/.git/config"} {
|
||||
if permissionEffect(rules, "external_directory", path) != "deny" {
|
||||
t.Fatalf("unverified landing path allowed: %q", path)
|
||||
}
|
||||
}
|
||||
client.LandingAccess.MainWorktree = "/repos/pipelines/../other"
|
||||
if permissionEffect(client.permissionRules(), "external_directory", "/repos/other/go.mod") != "deny" {
|
||||
t.Fatal("non-canonical landing capability allowed")
|
||||
}
|
||||
client.LandingAccess = LandingAccess{MainWorktree: "/repos/pipelines", GitMetadata: "/repos/pipelines/.git", LockPath: "/tmp/pipelines-landing.lock"}
|
||||
if permissionEffect(client.permissionRules(), "external_directory", "/tmp/pipelines-landing.lock") != "deny" {
|
||||
t.Fatal("landing lock outside verified metadata allowed")
|
||||
}
|
||||
client.LandingAccess = LandingAccess{MainWorktree: "/repos/pipelines", GitMetadata: "/repos/other/.git", LockPath: "/repos/other/.git/pipelines-landing.lock"}
|
||||
if permissionEffect(client.permissionRules(), "external_directory", "/repos/other/.git/config") != "deny" {
|
||||
t.Fatal("unrelated Git metadata allowed")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeliveryEnabledRequiresValidatedModelStage(t *testing.T) {
|
||||
def := modelDefinition()
|
||||
def.Configs["delivery"] = "true"
|
||||
def.Blocks["checks"] = "go test ./..."
|
||||
if enabled, err := DeliveryEnabled(def); err != nil || !enabled {
|
||||
t.Fatalf("valid delivery stage = %v, %v", enabled, err)
|
||||
}
|
||||
for name, change := range map[string]func(*Definition){
|
||||
"malformed value": func(d *Definition) { d.Configs["delivery"] = "yes" },
|
||||
"non-model": func(d *Definition) { d.Type = HumanType },
|
||||
"invalid model": func(d *Definition) { d.Configs["unexpected"] = "value" },
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
copy := def
|
||||
copy.Configs = map[string]string{}
|
||||
for key, value := range def.Configs {
|
||||
copy.Configs[key] = value
|
||||
}
|
||||
change(©)
|
||||
if enabled, err := DeliveryEnabled(copy); err == nil || enabled {
|
||||
t.Fatalf("invalid delivery stage = %v, %v", enabled, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeliveryRejectsPiBeforeAgentExecution(t *testing.T) {
|
||||
def := modelDefinition()
|
||||
def.Configs["delivery"] = "true"
|
||||
def.Configs["harness"] = "pi"
|
||||
def.Blocks["checks"] = "true"
|
||||
if err := (ModelStage{}).Validate(def); err == nil || !strings.Contains(err.Error(), "opencode harness") {
|
||||
t.Fatal("Pi delivery was accepted", err)
|
||||
}
|
||||
delete(def.Configs, "delivery")
|
||||
delete(def.Blocks, "checks")
|
||||
if err := (ModelStage{}).Validate(def); err != nil {
|
||||
t.Fatal("Pi non-delivery was changed", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeliveryChecksRequireBoundedDeliveryOnlyDeclaration(t *testing.T) {
|
||||
def := modelDefinition()
|
||||
def.Configs["delivery"] = "true"
|
||||
for name, mutate := range map[string]func(*Definition){
|
||||
"missing": func(d *Definition) {},
|
||||
"empty": func(d *Definition) { d.Blocks["checks"] = "\n" },
|
||||
"non delivery": func(d *Definition) { d.Blocks["checks"] = "go test ./..." },
|
||||
"too many checks": func(d *Definition) { d.Blocks["checks"] = strings.Repeat("true\n", 21) },
|
||||
"oversized check": func(d *Definition) { d.Blocks["checks"] = strings.Repeat("x", 4097) },
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
copy := def
|
||||
copy.Blocks = map[string]string{}
|
||||
copy.Configs = map[string]string{}
|
||||
for key, value := range def.Blocks {
|
||||
copy.Blocks[key] = value
|
||||
}
|
||||
for key, value := range def.Configs {
|
||||
copy.Configs[key] = value
|
||||
}
|
||||
mutate(©)
|
||||
if name == "non delivery" {
|
||||
delete(copy.Configs, "delivery")
|
||||
if _, err := (ModelStage{}).Spec(copy); err == nil {
|
||||
t.Fatal("non-delivery checks block was accepted")
|
||||
}
|
||||
return
|
||||
}
|
||||
if _, err := DeliveryEnabled(copy); err == nil {
|
||||
t.Fatal("invalid delivery checks were accepted")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,97 @@
|
||||
package stage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// VerificationTarget declares the controlled fixture that can substantiate a
|
||||
// target-specific behavior claim. It is intentionally data-only: the runner
|
||||
// does not execute fixture input supplied by a stage result.
|
||||
type VerificationTarget struct {
|
||||
Backend string
|
||||
Revision string
|
||||
Capability string
|
||||
Fixture string
|
||||
}
|
||||
|
||||
// VerificationEvidence is returned by a server-controlled fixture workflow,
|
||||
// never parsed from a model's result.
|
||||
type VerificationEvidence struct {
|
||||
Backend string
|
||||
Revision string
|
||||
Capability string
|
||||
Fixture string
|
||||
}
|
||||
|
||||
// VerificationRunner is the narrow server seam for a controlled fixture or
|
||||
// workflow. The runner selects it during server setup; model output cannot
|
||||
// select a backend, revision, fixture, or executable.
|
||||
type VerificationRunner func(context.Context, VerificationTarget) (VerificationEvidence, error)
|
||||
|
||||
// VerificationTargetFor parses an optional, bounded verification block. A
|
||||
// declared target requires all provenance fields so an old or other-backend
|
||||
// fixture cannot be presented as evidence for the current target.
|
||||
func VerificationTargetFor(def Definition) (*VerificationTarget, error) {
|
||||
text, declared := def.Blocks["verification"]
|
||||
if !declared {
|
||||
return nil, nil
|
||||
}
|
||||
if len(text) > 4096 {
|
||||
return nil, fmt.Errorf("verification block exceeds 4096 bytes")
|
||||
}
|
||||
values := map[string]string{}
|
||||
for _, line := range strings.Split(text, "\n") {
|
||||
line = strings.TrimSpace(line)
|
||||
if line == "" {
|
||||
continue
|
||||
}
|
||||
key, value, ok := strings.Cut(line, "=")
|
||||
key, value = strings.TrimSpace(key), strings.TrimSpace(value)
|
||||
if !ok || value == "" || len(value) > 256 {
|
||||
return nil, fmt.Errorf("invalid verification entry %q", line)
|
||||
}
|
||||
switch key {
|
||||
case "backend", "revision", "capability", "fixture":
|
||||
default:
|
||||
return nil, fmt.Errorf("unknown verification entry %q", key)
|
||||
}
|
||||
if _, exists := values[key]; exists {
|
||||
return nil, fmt.Errorf("duplicate verification entry %q", key)
|
||||
}
|
||||
values[key] = value
|
||||
}
|
||||
target := &VerificationTarget{Backend: values["backend"], Revision: values["revision"], Capability: values["capability"], Fixture: values["fixture"]}
|
||||
if target.Backend == "" || target.Revision == "" || target.Capability == "" || target.Fixture == "" {
|
||||
return nil, fmt.Errorf("verification requires backend, revision, capability, and fixture")
|
||||
}
|
||||
return target, nil
|
||||
}
|
||||
|
||||
// VerifyTargetEvidence compares server-collected controlled-fixture provenance
|
||||
// with the workflow declaration. It deliberately accepts no model result: a
|
||||
// stage cannot substantiate its own target-specific behavior claim.
|
||||
func VerifyTargetEvidence(target VerificationTarget, evidence VerificationEvidence) error {
|
||||
for key, want := range map[string]string{
|
||||
"backend": target.Backend, "revision": target.Revision, "capability": target.Capability, "fixture": target.Fixture,
|
||||
} {
|
||||
got := map[string]string{
|
||||
"backend": evidence.Backend, "revision": evidence.Revision, "capability": evidence.Capability, "fixture": evidence.Fixture,
|
||||
}[key]
|
||||
if got != want {
|
||||
return fmt.Errorf("verification evidence %s does not match declared target", key)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// VerifyTargetResult is retained for callers that only need declaration
|
||||
// validation. Result-provided provenance is intentionally not evidence.
|
||||
func VerifyTargetResult(def Definition, _ Values) error {
|
||||
target, err := VerificationTargetFor(def)
|
||||
if err != nil || target == nil {
|
||||
return err
|
||||
}
|
||||
return fmt.Errorf("verification target %q requires a server-controlled fixture runner", target.Fixture)
|
||||
}
|
||||
@@ -0,0 +1,27 @@
|
||||
package stage
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestTargetVerificationRejectsWrongBackendEvidence(t *testing.T) {
|
||||
def := modelDefinition()
|
||||
def.Blocks["verification"] = "backend=branch-api\nrevision=abc123\ncapability=controlled-state\nfixture=reconnect-foreground"
|
||||
if err := (ModelStage{}).Validate(def); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
target, err := VerificationTargetFor(def)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
good := VerificationEvidence{Backend: "branch-api", Revision: "abc123", Capability: "controlled-state", Fixture: "reconnect-foreground"}
|
||||
if err := VerifyTargetEvidence(*target, good); err != nil {
|
||||
t.Fatalf("matching fixture evidence rejected: %v", err)
|
||||
}
|
||||
wrongBackend := VerificationEvidence{Backend: "old-api", Revision: "abc123", Capability: "controlled-state", Fixture: "reconnect-foreground"}
|
||||
if err := VerifyTargetEvidence(*target, wrongBackend); err == nil {
|
||||
t.Fatal("wrong-backend fixture evidence accepted")
|
||||
}
|
||||
def.Blocks["verification"] = "backend=branch-api\nrevision=abc123"
|
||||
if err := (ModelStage{}).Validate(def); err == nil {
|
||||
t.Fatal("incomplete target verification declaration accepted")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,43 @@
|
||||
package tui
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.a-wan.net/alan/pipelines/internal/protocol"
|
||||
"github.com/charmbracelet/lipgloss"
|
||||
"github.com/charmbracelet/x/ansi"
|
||||
"github.com/muesli/termenv"
|
||||
)
|
||||
|
||||
func TestResourceWaitFrames(t *testing.T) {
|
||||
previous := lipgloss.ColorProfile()
|
||||
defer lipgloss.SetColorProfile(previous)
|
||||
for _, width := range []int{60, 120} {
|
||||
for _, plain := range []bool{false, true} {
|
||||
t.Run(fmt.Sprintf("width-%d-plain-%t", width, plain), func(t *testing.T) {
|
||||
if plain {
|
||||
lipgloss.SetColorProfile(termenv.Ascii)
|
||||
} else {
|
||||
lipgloss.SetColorProfile(termenv.TrueColor)
|
||||
}
|
||||
m := decisionModel()
|
||||
m.width = width
|
||||
m.runDetail.Status = protocol.RunStatusQueued
|
||||
m.runDetail.Steps = nil
|
||||
m.runDetail.ResourceWait = &protocol.ResourceWait{Resource: "phone-X", Stage: "review", Sequence: 1, QueuedSince: "2026-10-18T10:00:00Z"}
|
||||
frame := ansi.Strip(m.runsDetailsView(24))
|
||||
if !strings.Contains(frame, "Waiting for resource phone-X") || strings.Contains(frame, "c decide") || strings.Contains(frame, "Waiting for a person") {
|
||||
t.Fatal("wrong resource wait frame:\n", frame)
|
||||
}
|
||||
for _, line := range strings.Split(frame, "\n") {
|
||||
if ansi.StringWidth(line) > width {
|
||||
t.Fatalf("overflow: %q", line)
|
||||
}
|
||||
}
|
||||
t.Log("Rendered resource wait (no phone):\n" + frame)
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -11,6 +11,9 @@ import (
|
||||
)
|
||||
|
||||
func resultLabel(r protocol.RunSummary) string {
|
||||
if r.ResourceWait != nil {
|
||||
return r.StatusLabel() + " · stage " + r.ResourceWait.Stage
|
||||
}
|
||||
preview := ansi.Strip(r.ResultPreview)
|
||||
if r.Status == "succeeded" {
|
||||
if preview != "" {
|
||||
@@ -55,7 +58,7 @@ func (m Model) pipelineResultLines() []string {
|
||||
if r == nil {
|
||||
return []string{"Results · This version has not run yet. No accomplishments recorded."}
|
||||
}
|
||||
lines := []string{fmt.Sprintf("Results · run #%d · %s · %s · %d completed visits · %s", r.ID, r.Label(), ansi.Strip(m.progress(r.Status)), r.CompletedSteps, runDuration(r.CreatedAt, r.FinishedAt)), resultLabel(*r)}
|
||||
lines := []string{fmt.Sprintf("Results · run #%d · %s · %s · %d completed visits · %s", r.ID, r.Label(), ansi.Strip(m.runSummaryStatus(*r)), r.CompletedSteps, runDuration(r.CreatedAt, r.FinishedAt)), resultLabel(*r)}
|
||||
if r.Status != "succeeded" && r.ResultPreview != "" && r.Diagnostic != "" {
|
||||
lines = append(lines, "Partial output: "+ansi.Strip(r.ResultPreview))
|
||||
}
|
||||
|
||||
@@ -275,6 +275,13 @@ func runStatus(status string) string {
|
||||
return lipgloss.NewStyle().Foreground(color).Render(status)
|
||||
}
|
||||
|
||||
func (m Model) runSummaryStatus(run protocol.RunSummary) string {
|
||||
if run.ResourceWait != nil {
|
||||
return run.StatusLabel()
|
||||
}
|
||||
return m.progress(run.Status)
|
||||
}
|
||||
|
||||
func runDuration(start, end string) string {
|
||||
first, err := time.Parse(time.RFC3339, start)
|
||||
if err != nil {
|
||||
@@ -687,7 +694,7 @@ func (m Model) runsDetailsView(height int) string {
|
||||
start, end := selectionWindow(len(runs), m.runIndex, max(1, (height-4-len(rows))/2))
|
||||
for i := start; i < end; i++ {
|
||||
r := runs[i]
|
||||
text := fmt.Sprintf("#%d %s · v%d · %s · %s", r.ID, r.Label(), r.Version, m.progress(r.Status), dateLabel(r.FinishedAt))
|
||||
text := fmt.Sprintf("#%d %s · v%d · %s · %s", r.ID, r.Label(), r.Version, m.runSummaryStatus(r), dateLabel(r.FinishedAt))
|
||||
rows = append(rows, menuOption(text, i == m.runIndex))
|
||||
rows = append(rows, " "+ansi.Truncate(resultLabel(r), max(1, m.width-6), "…"))
|
||||
}
|
||||
@@ -710,7 +717,8 @@ func (m Model) runsDetailsView(height int) string {
|
||||
return panel(m.width, height, []string{bar(title, fmt.Sprint(len(runs)), max(1, m.width-4), true)}, rows, footer, true)
|
||||
}
|
||||
r := m.runDetail
|
||||
header := bar(fmt.Sprintf("#%d %s · v%d", r.ID, r.Label(), r.Version), m.progress(r.Status)+" · "+runDuration(r.CreatedAt, r.FinishedAt), m.width, false)
|
||||
status := m.runSummaryStatus(r.RunSummary)
|
||||
header := bar(fmt.Sprintf("#%d %s · v%d", r.ID, r.Label(), r.Version), status+" · "+runDuration(r.CreatedAt, r.FinishedAt), m.width, false)
|
||||
if height <= 6 {
|
||||
return fitBlock([]string{header, runHighlight(m.runSelectionLabel(), m.width), m.runActions() + " · Tab pane · Esc back"}, m.width, height)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user