Merge main into pill-run-decision for Android recovery

This commit is contained in:
2026-10-18 15:06:52 +01:00
85 changed files with 6293 additions and 178 deletions
@@ -1,12 +1,15 @@
package net.awan.pipelines
import android.app.Application
import androidx.lifecycle.ProcessLifecycleOwner
import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.SupervisorJob
import net.awan.pipelines.data.ForegroundReconnector
import net.awan.pipelines.data.PipelinesRepository
import net.awan.pipelines.data.SettingsRepository
import net.awan.pipelines.data.transport.PipelinesClient
import net.awan.pipelines.data.transport.ServerUrl
import net.awan.pipelines.notify.RunNotifier
/**
@@ -30,8 +33,39 @@ class PipelinesApplication : Application() {
*/
val runNotifier: RunNotifier by lazy { RunNotifier(this) }
/**
* The one owner of return-to-foreground recovery: when the process comes back
* to the front it restores a connection the background took away, with no
* interaction. It is created once with the Application and observes the
* process lifecycle, so every Activity and ViewModel recreation leaves it
* untouched. The liveness check is the existing `ping`; the reconnect is the
* same shared-client connect the app's own Connect/Reconnect uses, so the
* policy lives only in `foregroundAction` and the client's backoff stays the
* only reconnect mechanism.
*/
private val foregroundReconnector: ForegroundReconnector by lazy {
ForegroundReconnector(
state = { repository.connectionState.value },
confirmAlive = {
// The existing ping is the positive check: on a dead socket it
// throws (or never answers) instead of returning, so reaching the
// next line is the proof of life.
repository.ping()
true
},
reconnect = {
// A saved URL that no longer validates is left for Settings to
// report; there is nothing to reconnect to.
val validated = ServerUrl.validate(settings.serverUrl.value)
if (validated.valid) repository.recover(validated)
},
scope = applicationScope,
)
}
override fun onCreate() {
super.onCreate()
runNotifier.ensureChannel()
ProcessLifecycleOwner.get().lifecycle.addObserver(foregroundReconnector)
}
}
@@ -0,0 +1,81 @@
package net.awan.pipelines.data
import androidx.lifecycle.DefaultLifecycleObserver
import androidx.lifecycle.LifecycleOwner
import kotlinx.coroutines.CancellationException
import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.launch
import kotlinx.coroutines.withTimeoutOrNull
import net.awan.pipelines.data.transport.ConnectionState
import net.awan.pipelines.data.transport.ForegroundAction
import net.awan.pipelines.data.transport.foregroundAction
/**
* The one lifecycle-aware owner of the app's return-to-foreground recovery.
*
* While another app is in front Android freezes — and often kills — this process,
* so the socket it holds dies or goes quiet and the client's own bounded backoff
* either never starts or starts too late for the operator, who comes back and
* taps Connect. The client's backoff (see `PipelinesClient.scheduleReconnect`) is
* still the only reconnect mechanism; this owner only decides *when* to drive it,
* and does so the moment the app becomes visible again.
*
* It observes the whole process lifecycle (a `ProcessLifecycleOwner`), never the
* Activity or the ViewModel, both of which Android recreates: the owner is
* created once with the Application and survives every recreation. The decision
* itself is the one pure [foregroundAction] policy; this class supplies only the
* current state, a positive liveness check and the reconnect effect, so the
* policy is unit-testable with no device, socket or framework.
*/
class ForegroundReconnector(
private val state: () -> ConnectionState,
private val confirmAlive: suspend () -> Boolean,
private val reconnect: () -> Unit,
private val scope: CoroutineScope,
private val confirmTimeoutMillis: Long = CONFIRM_TIMEOUT_MILLIS,
) : DefaultLifecycleObserver {
/**
* The app became visible again. A state the policy calls reconnecting is
* reconnected at once — the pending backoff delay is cancelled by the
* reconnect effect, so the operator never waits it out. A socket the state
* still calls Connected is first confirmed to answer, so a silent
* "Connected" is never trusted; the state is re-read afterwards, because the
* socket's own failure callback may have run while the check was in flight.
*/
override fun onStart(owner: LifecycleOwner) {
val before = state()
if (before !is ConnectionState.Connected) {
if (foregroundAction(before, confirmedAlive = false) == ForegroundAction.Reconnect) reconnect()
return
}
scope.launch {
val alive = confirm()
if (foregroundAction(state(), alive) == ForegroundAction.Reconnect) reconnect()
}
}
/**
* Confirms a supposedly-connected socket still answers, bounded so a dead one
* cannot hold recovery: the check runs off the main thread on the app scope
* and a probe that fails, throws or outlives its budget counts as not alive.
*/
private suspend fun confirm(): Boolean =
withTimeoutOrNull(confirmTimeoutMillis) {
try {
confirmAlive()
} catch (t: Throwable) {
if (t is CancellationException) throw t
false
}
} ?: false
companion object {
/**
* Bounds the liveness check. A healthy round trip on the trusted LAN/VPN
* is far shorter, so a socket that cannot answer is noticed promptly
* instead of holding the "Connected" the operator is looking at.
*/
const val CONFIRM_TIMEOUT_MILLIS = 3_000L
}
}
@@ -66,6 +66,9 @@ class PipelinesRepository(
/** Opens the single shared connection to a validated server URL. */
fun connect(validated: ServerUrl.Validated) = client.connect(validated)
/** Recovers a lost connection without reversing an explicit Disconnect. */
fun recover(validated: ServerUrl.Validated) = client.recover(validated)
/** Explicit disconnect; the client then stays Idle. */
fun disconnect() = client.disconnect()
@@ -315,6 +315,7 @@ data class SessionExport(
@Serializable
data class RunSummary(
@SerialName("resource_wait") val resourceWait: ResourceWait? = null,
val id: Long = 0,
@SerialName("pipeline_id") val pipelineId: Long = 0,
val name: String = "",
@@ -370,6 +371,7 @@ data class RunStep(
// declared directly with the exact internal/protocol/runs.go wire names.
@Serializable
data class RunDetail(
@SerialName("resource_wait") val resourceWait: ResourceWait? = null,
val id: Long = 0,
@SerialName("pipeline_id") val pipelineId: Long = 0,
val name: String = "",
@@ -405,6 +407,8 @@ data class RunDetail(
*/
@Serializable
data class RunEventFrame(
@SerialName("resource_wait") val resourceWait: ResourceWait? = null,
@SerialName("resource_wait_set") val resourceWaitSet: Boolean = false,
@SerialName("run_id") val runId: Long = 0,
val workspace: String = "",
// The run's pipeline identity and stored title, repeated on every frame so a
@@ -428,6 +432,16 @@ data class RunEventFrame(
val step: RunStepDelta? = null,
)
@Serializable
data class ResourceWait(
val resource: String = "",
val stage: String = "",
val sequence: Int = 0,
@SerialName("queued_since") val queuedSince: String = "",
) {
fun statusLabel(): String = "Waiting for resource $resource"
}
/**
* The compact change of one step within a [RunEventFrame]
* (internal/protocol/runevents.go RunStepDelta): a delta, not a RunStep, so it
@@ -0,0 +1,40 @@
package net.awan.pipelines.data.transport
/** What the app does with the connection the moment it becomes visible again. */
enum class ForegroundAction {
/** Leave the connection exactly as it is. */
LeaveAlone,
/** Re-establish the connection now, cancelling any pending reconnect delay. */
Reconnect,
}
/**
* The app's return-to-foreground recovery policy, in one pure place.
*
* Android freezes and often kills a backgrounded app's process, so the socket
* the app holds dies or goes quiet while another app is in front and nothing in
* the process notices in time. Coming back must restore that connection without
* the operator tapping Connect — but an explicit Disconnect is the operator's
* own decision and stays authoritative. A deliberate Disconnect is the only way
* the client reaches [ConnectionState.Idle], so Idle is left alone; every other
* state reconnects.
*
* A still [ConnectionState.Connected] socket is not trusted on return: a frozen
* process can leave it claiming Connected while it answers nothing, which is
* exactly the stale state this replaces. The caller therefore checks it with a
* positive probe (the existing ping) and passes the verdict as [confirmedAlive];
* only a connection just confirmed to answer is left alone.
*
* @param state the connection state at the moment the app became visible again.
* @param confirmedAlive whether a [ConnectionState.Connected] socket has just
* been confirmed to answer. It is consulted only for Connected; every other
* state carries its own answer.
*/
fun foregroundAction(state: ConnectionState, confirmedAlive: Boolean): ForegroundAction = when (state) {
ConnectionState.Idle -> ForegroundAction.LeaveAlone
is ConnectionState.Connected ->
if (confirmedAlive) ForegroundAction.LeaveAlone else ForegroundAction.Reconnect
is ConnectionState.Connecting -> ForegroundAction.Reconnect
is ConnectionState.Disconnected -> ForegroundAction.Reconnect
}
@@ -78,11 +78,33 @@ class PipelinesClient(
@Volatile
private var manualClose = false
/** Serializes socket replacement with callbacks arriving on OkHttp threads. */
private val connectionLock = Any()
private var reconnectJob: Job? = null
private var attempt = 0
/** Opens (or re-opens) the connection to a validated server URL. */
fun connect(validated: ServerUrl.Validated) {
synchronized(connectionLock) {
connectLocked(validated)
}
}
/**
* Re-opens a connection the transport lost without reversing an explicit
* [disconnect]. The check and connection replacement share one lock, so a
* Disconnect racing a foreground liveness probe wins even after that probe
* has decided recovery is needed.
*/
fun recover(validated: ServerUrl.Validated) {
synchronized(connectionLock) {
if (manualClose) return
connectLocked(validated)
}
}
private fun connectLocked(validated: ServerUrl.Validated) {
val httpUrl = validated.httpUrl ?: throw IllegalArgumentException("server URL is not valid")
endpoint = httpUrl
manualClose = false
@@ -94,14 +116,16 @@ class PipelinesClient(
/** Explicit user disconnect: stops reconnection and closes the socket. */
fun disconnect() {
manualClose = true
reconnectJob?.cancel()
reconnectJob = null
val socket = webSocket
webSocket = null
socket?.close(NORMAL_CLOSURE, "client closing")
failPending("Disconnected")
_state.value = ConnectionState.Idle
synchronized(connectionLock) {
manualClose = true
reconnectJob?.cancel()
reconnectJob = null
val socket = webSocket
webSocket = null
socket?.close(NORMAL_CLOSURE, "client closing")
failPending("Disconnected")
_state.value = ConnectionState.Idle
}
}
fun close() {
@@ -115,14 +139,20 @@ class PipelinesClient(
*/
suspend fun call(request: Message): Message = withContext(ioDispatcher) {
sendLock.withLock {
val socket = webSocket ?: throw NotConnectedException()
if (_state.value !is ConnectionState.Connected) throw NotConnectedException()
val id = "android-" + sequence.incrementAndGet()
val deferred = CompletableDeferred<Message>()
pending[id] = deferred
try {
val (id, deferred) = synchronized(connectionLock) {
val socket = webSocket ?: throw NotConnectedException()
if (_state.value !is ConnectionState.Connected) throw NotConnectedException()
val id = "android-" + sequence.incrementAndGet()
val deferred = CompletableDeferred<Message>()
pending[id] = deferred
val payload = json.encodeToString(Message.serializer(), request.copy(id = id))
if (!socket.send(payload)) throw IOException("websocket send failed")
if (!socket.send(payload)) {
pending.remove(id)
throw IOException("websocket send failed")
}
id to deferred
}
try {
val response = withTimeout(requestTimeoutMillis) { deferred.await() }
if (response.type == Protocol.ERROR) {
throw ProtocolException(response.error?.ifBlank { "server error" } ?: "server error")
@@ -144,12 +174,14 @@ class PipelinesClient(
*/
suspend fun sendOnly(request: Message): String = withContext(ioDispatcher) {
sendLock.withLock {
val socket = webSocket ?: throw NotConnectedException()
if (_state.value !is ConnectionState.Connected) throw NotConnectedException()
val id = "android-" + sequence.incrementAndGet()
val payload = json.encodeToString(Message.serializer(), request.copy(id = id))
if (!socket.send(payload)) throw IOException("websocket send failed")
id
synchronized(connectionLock) {
val socket = webSocket ?: throw NotConnectedException()
if (_state.value !is ConnectionState.Connected) throw NotConnectedException()
val id = "android-" + sequence.incrementAndGet()
val payload = json.encodeToString(Message.serializer(), request.copy(id = id))
if (!socket.send(payload)) throw IOException("websocket send failed")
id
}
}
}
@@ -166,14 +198,34 @@ class PipelinesClient(
}
}
private fun open() {
private fun open() = synchronized(connectionLock) { openLocked() }
private fun openLocked() {
val current = endpoint ?: return
attempt += 1
_state.value = ConnectionState.Connecting(attempt)
val request = Request.Builder().url(current).build()
// Replace the current socket, then close the one it superseded. Its
// callbacks are ignored from then on (see [isCurrent]), so a failure the
// old socket reports late — the common case when the process was frozen
// while it died — can neither clear the new socket nor schedule a
// reconnect that would undo it.
val superseded = webSocket
webSocket = httpClient.newWebSocket(request, listener)
// A request sent over the superseded socket cannot receive a reply from
// the replacement. Fail it now rather than leaving a screen waiting for
// the ordinary request timeout after recovery.
if (superseded != null) failPending("Connection replaced")
superseded?.close(NORMAL_CLOSURE, "superseded")
}
/**
* Whether [socket] is still the connection this client acts on. A socket this
* client has already replaced (a deliberate reconnect, or the foreground
* recovery re-establishing a frozen one) must never drive the state again.
*/
private fun isCurrent(socket: WebSocket): Boolean = webSocket === socket
private fun failPending(reason: String) {
val snapshot = pending.keys.toList()
for (key in snapshot) {
@@ -181,7 +233,8 @@ class PipelinesClient(
}
}
private fun handleClosed(reason: String) {
private fun handleClosed(socket: WebSocket, reason: String) {
if (!isCurrent(socket)) return
webSocket = null
failPending(reason)
if (manualClose) {
@@ -194,52 +247,65 @@ class PipelinesClient(
private fun scheduleReconnect(reason: String) {
if (reconnectJob?.isActive == true) return
_state.value = ConnectionState.Disconnected(reason, retrying = true)
val nextAttempt = attempt
val backoff = (BASE_BACKOFF_MILLIS shl (nextAttempt - 1).coerceAtMost(6))
.coerceAtMost(MAX_BACKOFF_MILLIS)
val backoff = reconnectBackoffMillis(attempt)
val jitter = Random.nextLong(0, 400)
reconnectJob = scope.launch {
delay(backoff + jitter)
if (!manualClose && endpoint != null) open()
synchronized(connectionLock) {
if (!manualClose && endpoint != null) open()
}
}
}
private val listener = object : WebSocketListener() {
override fun onOpen(webSocket: WebSocket, response: Response) {
attempt = 0
_state.value = ConnectionState.Connected(build = null, serverTime = null)
synchronized(connectionLock) {
if (!isCurrent(webSocket)) return
attempt = 0
_state.value = ConnectionState.Connected(build = null, serverTime = null)
}
}
override fun onMessage(webSocket: WebSocket, text: String) {
val message = try {
json.decodeFromString(Message.serializer(), text)
} catch (_: Exception) {
// Malformed frame: never misattribute it to a request.
return
}
val id = message.id
val deferred = if (id != null) pending.remove(id) else null
if (deferred != null) {
deferred.complete(message)
} else {
// Unsolicited envelopes (the connection's initial pipeline_list,
// server-pushed chat frames, stream errors) carry no matching id:
// hand them to the event stream instead of dropping them.
_events.tryEmit(message)
synchronized(connectionLock) {
if (!isCurrent(webSocket)) return
val message = try {
json.decodeFromString(Message.serializer(), text)
} catch (_: Exception) {
// Malformed frame: never misattribute it to a request.
return
}
val id = message.id
val deferred = if (id != null) pending.remove(id) else null
if (deferred != null) {
deferred.complete(message)
} else {
// Unsolicited envelopes (the connection's initial pipeline_list,
// server-pushed chat frames, stream errors) carry no matching id:
// hand them to the event stream instead of dropping them.
_events.tryEmit(message)
}
}
}
override fun onClosed(webSocket: WebSocket, code: Int, reason: String) {
handleClosed("Connection closed (${code})")
synchronized(connectionLock) {
handleClosed(webSocket, "Connection closed (${code})")
}
}
override fun onClosing(webSocket: WebSocket, code: Int, reason: String) {
webSocket.close(NORMAL_CLOSURE, null)
synchronized(connectionLock) {
if (!isCurrent(webSocket)) return
webSocket.close(NORMAL_CLOSURE, null)
}
}
override fun onFailure(webSocket: WebSocket, t: Throwable, response: Response?) {
val reason = t.message?.ifBlank { t.javaClass.simpleName } ?: "connection failed"
handleClosed(reason)
synchronized(connectionLock) {
val reason = t.message?.ifBlank { t.javaClass.simpleName } ?: "connection failed"
handleClosed(webSocket, reason)
}
}
}
@@ -247,6 +313,16 @@ class PipelinesClient(
private const val NORMAL_CLOSURE = 1000
private const val BASE_BACKOFF_MILLIS = 1_000L
private const val MAX_BACKOFF_MILLIS = 30_000L
/**
* The successful open callback resets [attempt] to zero. A later socket
* failure must therefore begin at the base delay, never shift by -1
* (which Kotlin masks into a very large shift count).
*/
internal fun reconnectBackoffMillis(attempt: Int): Long {
val exponent = (attempt - 1).coerceIn(0, 6)
return (BASE_BACKOFF_MILLIS shl exponent).coerceAtMost(MAX_BACKOFF_MILLIS)
}
}
}
@@ -101,6 +101,7 @@ internal fun foldRunEvent(runs: List<RunSummary>, frame: RunEventFrame): RunList
internal fun RunEventFrame.newRunSummary(): RunSummary? {
if (pipelineId <= 0 || name.isBlank()) return null
return RunSummary(
resourceWait = resourceWait,
id = runId,
pipelineId = pipelineId,
name = name,
@@ -124,10 +125,11 @@ internal fun RunEventFrame.newRunSummary(): RunSummary? {
*/
internal fun foldRunEvent(detail: RunDetail, frame: RunEventFrame): RunDetail {
if (frame.runId != detail.id) return detail
if (isStaleRunFrame(detail.status, frame.status)) return detail
if (isStaleRunFrame(detail.status, frame.status, frame.resourceWaitSet)) return detail
val merged = detail.fields().applying(frame)
val steps = frame.step?.let { upsertStep(detail.steps, it) } ?: detail.steps
val updated = detail.copy(
resourceWait = merged.resourceWait,
status = merged.status,
outcome = merged.outcome,
currentStage = merged.currentStage,
@@ -169,8 +171,9 @@ private fun advanceRunStatus(current: String, incoming: String): String =
* run already settled. Dropping every field (not just the status) is what keeps
* a late `running` delta from also rewriting the stage or step of a settled run.
*/
private fun isStaleRunFrame(current: String, incoming: String): Boolean =
isStaleTransition(current, incoming, ::runLifecycleRank)
private fun isStaleRunFrame(current: String, incoming: String, resourceWaitSet: Boolean = false): Boolean =
if (resourceWaitSet && incoming == "queued" && !isTerminalRunStatus(current)) false
else isStaleTransition(current, incoming, ::runLifecycleRank)
/** The step counterpart of [isStaleRunFrame]: a settled visit drops a stale delta whole. */
private fun isStaleStepDelta(current: String, incoming: String): Boolean =
@@ -213,6 +216,7 @@ private fun advanceStatus(current: String, incoming: String, rank: (String) -> I
/** The run-state fields a frame advances, shared by the summary row and the detail. */
private data class RunStateFields(
val resourceWait: net.awan.pipelines.data.protocol.ResourceWait?,
val status: String,
val outcome: String,
val currentStage: String,
@@ -226,8 +230,10 @@ private data class RunStateFields(
* detail can never apply a frame differently.
*/
fun applying(frame: RunEventFrame): RunStateFields {
val status = advanceRunStatus(status, frame.status)
val status = if (frame.resourceWaitSet && frame.status == "queued" && !isTerminalRunStatus(status))
"queued" else advanceRunStatus(status, frame.status)
return RunStateFields(
resourceWait = if (isTerminalRunStatus(status)) null else if (frame.resourceWaitSet) frame.resourceWait else resourceWait,
status = status,
outcome = frame.outcome.ifBlank { outcome },
currentStage = frame.currentStage.ifBlank { currentStage },
@@ -240,6 +246,7 @@ private data class RunStateFields(
}
private fun RunSummary.fields() = RunStateFields(
resourceWait = resourceWait,
status = status,
outcome = outcome,
currentStage = currentStage,
@@ -250,6 +257,7 @@ private fun RunSummary.fields() = RunStateFields(
)
private fun RunDetail.fields() = RunStateFields(
resourceWait = resourceWait,
status = status,
outcome = outcome,
currentStage = currentStage,
@@ -267,11 +275,12 @@ private fun executingFor(status: String, current: Boolean): Boolean = when (stat
}
private fun RunSummary.applying(frame: RunEventFrame): RunSummary {
if (isStaleRunFrame(status, frame.status)) return this
if (isStaleRunFrame(status, frame.status, frame.resourceWaitSet)) return this
val current = fields()
val merged = current.applying(frame)
if (merged == current) return this
return copy(
resourceWait = merged.resourceWait,
status = merged.status,
outcome = merged.outcome,
currentStage = merged.currentStage,
@@ -615,6 +615,7 @@ private fun VersionChip(version: String) {
/** Progress from the fields a summary actually has: completed visits + stage. */
private fun progressText(run: RunSummary): String = buildString {
run.resourceWait?.let { append(it.statusLabel()); append(" · ") }
append(visitCount(run.completedSteps, "completed"))
if (run.currentStage.isNotBlank()) append(" · stage ${run.currentStage}")
}
@@ -135,6 +135,7 @@ internal fun RunSummaryTab(
)
KeyValueRow("Pipeline", "id ${run.pipelineId} · v${run.version}")
KeyValueRow("Stage", run.currentStage)
run.resourceWait?.let { Text(it.statusLabel(), style = text.meta) }
KeyValueRow("Workspace", run.workspace)
KeyValueRow("Content hash", run.contentHash, monospace = true)
KeyValueRow("Started", formatLocalTimestamp(run.createdAt).orEmpty())
@@ -0,0 +1,138 @@
package net.awan.pipelines.data
import androidx.lifecycle.Lifecycle
import androidx.lifecycle.LifecycleOwner
import java.util.concurrent.CountDownLatch
import java.util.concurrent.TimeUnit
import java.util.concurrent.atomic.AtomicInteger
import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.delay
import net.awan.pipelines.data.transport.ConnectionState
import org.junit.Assert.assertEquals
import org.junit.Test
/**
* The one lifecycle-aware owner of return-to-foreground recovery: it applies the
* pure `foregroundAction` policy to the state it reads, confirms a supposedly
* connected socket before trusting it, and reconnects when — and only when — the
* policy says so. The state, the liveness check and the reconnect are injected,
* so the owner is exercised with no device, socket or Android framework. The
* owner never touches the LifecycleOwner it is handed.
*/
class ForegroundReconnectorTest {
private val reconnects = AtomicInteger(0)
private val probes = AtomicInteger(0)
private val owner = object : LifecycleOwner {
override val lifecycle: Lifecycle get() = throw UnsupportedOperationException("not used")
}
private val connected = ConnectionState.Connected(build = null, serverTime = null)
/**
* A reconnector whose probe answers [alive] without suspending, so
* `Dispatchers.Unconfined` runs the whole `onStart` decision inline and the
* assertions below hold the moment it returns.
*/
private fun reconnector(
state: () -> ConnectionState,
alive: suspend () -> Boolean = { true },
) = ForegroundReconnector(
state = state,
confirmAlive = { probes.incrementAndGet(); alive() },
reconnect = { reconnects.incrementAndGet() },
scope = CoroutineScope(Dispatchers.Unconfined),
)
@Test
fun aDeliberateDisconnectIsLeftAloneAndNotEvenProbed() {
reconnector(state = { ConnectionState.Idle }).onStart(owner)
assertEquals(0, probes.get())
assertEquals(0, reconnects.get())
}
@Test
fun aDroppedConnectionReconnectsImmediatelyWithoutProbing() {
reconnector(state = { ConnectionState.Disconnected("socket closed", retrying = true) }).onStart(owner)
assertEquals("a dropped socket has nothing to confirm", 0, probes.get())
assertEquals(1, reconnects.get())
}
@Test
fun aConfirmedConnectedSocketIsLeftAlone() {
reconnector(state = { connected }).onStart(owner)
assertEquals(1, probes.get())
assertEquals(0, reconnects.get())
}
@Test
fun aConnectedSocketThatDoesNotAnswerReconnects() {
reconnector(state = { connected }, alive = { false }).onStart(owner)
assertEquals(1, probes.get())
assertEquals(1, reconnects.get())
}
@Test
fun aProbeThatThrowsIsNotAlive() {
reconnector(
state = { connected },
alive = { throw IllegalStateException("socket closed") },
).onStart(owner)
assertEquals(1, reconnects.get())
}
@Test
fun aSocketThatDiesWhileTheProbeRunsStillReconnects() {
// The socket's own failure callback lands mid-check; the owner re-reads
// the state and the policy then decides on what is actually there.
var state: ConnectionState = connected
reconnector(
state = { state },
alive = {
state = ConnectionState.Disconnected("socket closed", retrying = true)
false
},
).onStart(owner)
assertEquals(1, reconnects.get())
}
@Test
fun aDisconnectWhileTheProbeRunsStaysDisconnected() {
// The user can tap Disconnect while the bounded foreground probe is in
// flight. Re-reading state after that probe keeps the explicit choice
// authoritative rather than reconnecting over it.
var state: ConnectionState = connected
reconnector(
state = { state },
alive = {
state = ConnectionState.Idle
true
},
).onStart(owner)
assertEquals(0, reconnects.get())
}
@Test
fun aProbeThatNeverAnswersIsNotAliveWhenItsBudgetExpires() {
val done = CountDownLatch(1)
val reconnector = ForegroundReconnector(
state = { connected },
confirmAlive = { delay(5_000); true },
reconnect = {
reconnects.incrementAndGet()
done.countDown()
},
scope = CoroutineScope(Dispatchers.Unconfined),
confirmTimeoutMillis = 20L,
)
reconnector.onStart(owner)
assertEquals(
"the bounded check must give up on its own rather than hold the UI's Connected",
true,
done.await(2, TimeUnit.SECONDS),
)
assertEquals(1, reconnects.get())
}
}
@@ -0,0 +1,60 @@
package net.awan.pipelines.data.transport
import org.junit.Assert.assertEquals
import org.junit.Test
/**
* The return-to-foreground policy, in the one pure place it lives: an explicit
* Disconnect is never overridden, a dropped or in-flight connection is
* re-established at once, and a still-Connected socket is trusted only after it
* has been confirmed to answer.
*/
class ConnectionRecoveryTest {
@Test
fun aDeliberateDisconnectIsNeverOverridden() {
// Idle is only ever reached by a deliberate Disconnect (the client starts
// Idle and never returns to it by itself), so returning must leave it.
assertEquals(
ForegroundAction.LeaveAlone,
foregroundAction(ConnectionState.Idle, confirmedAlive = false),
)
}
@Test
fun aDroppedConnectionReconnectsAtOnce() {
assertEquals(
ForegroundAction.Reconnect,
foregroundAction(ConnectionState.Disconnected("socket closed", retrying = true), confirmedAlive = false),
)
}
@Test
fun anOpenAlreadyInFlightIsRestartedRatherThanWaitedOn() {
assertEquals(
ForegroundAction.Reconnect,
foregroundAction(ConnectionState.Connecting(3), confirmedAlive = false),
)
}
@Test
fun aConnectedSocketIsTrustedOnlyAfterItAnswers() {
val connected = ConnectionState.Connected(build = null, serverTime = null)
assertEquals(ForegroundAction.LeaveAlone, foregroundAction(connected, confirmedAlive = true))
// The defect: a socket the process still calls Connected but that answers
// nothing must be reconnected, not trusted.
assertEquals(ForegroundAction.Reconnect, foregroundAction(connected, confirmedAlive = false))
}
@Test
fun aFailureAfterConnectedStartsAtTheBaseBackoff() {
// onOpen resets the client's attempt counter to zero. The next failure
// must not pass -1 to the shift used for exponential backoff.
assertEquals(1_000L, PipelinesClient.reconnectBackoffMillis(attempt = 0))
}
@Test
fun reconnectBackoffRemainsBounded() {
assertEquals(30_000L, PipelinesClient.reconnectBackoffMillis(attempt = Int.MAX_VALUE))
}
}
@@ -0,0 +1,32 @@
package net.awan.pipelines.ui
import kotlinx.serialization.json.Json
import net.awan.pipelines.data.protocol.ResourceWait
import net.awan.pipelines.data.protocol.RunDetail
import net.awan.pipelines.data.protocol.RunEventFrame
import net.awan.pipelines.data.protocol.RunSummary
import org.junit.Assert.*
import org.junit.Test
class ResourceWaitTest {
@Test fun optionalMetadataAndPauseResumeClearing() {
val json = Json { ignoreUnknownKeys = true }
assertNull(json.decodeFromString<RunDetail>("""{"id":1,"status":"queued"}""").resourceWait)
val wait = ResourceWait("phone-X", "review", 2, "2026-10-18T10:00:00Z")
val queued = RunEventFrame(runId = 1, status = "queued", resourceWaitSet = true, resourceWait = wait)
val detail = foldRunEvent(RunDetail(id = 1, status = "running"), queued)
assertEquals("queued", detail.status)
assertEquals(wait, detail.resourceWait)
assertEquals("Waiting for resource phone-X", wait.statusLabel())
val rows = foldRunEvent(listOf(RunSummary(id = 1, status = "running")), queued).runs
assertEquals(wait, rows.single().resourceWait)
val clear = RunEventFrame(runId = 1, status = "running", resourceWaitSet = true)
assertNull(foldRunEvent(detail, clear).resourceWait)
assertNull(foldRunEvent(rows, clear).runs.single().resourceWait)
assertNull(foldRunEvent(detail, RunEventFrame(runId = 1, status = "canceled")).resourceWait)
assertEquals("succeeded", foldRunEvent(RunDetail(id = 1, status = "succeeded"), queued).status)
val decoded = json.decodeFromString<RunDetail>("""{"id":1,"status":"queued","resource_wait":{"resource":"phone-X","stage":"review","sequence":2,"queued_since":"now"}}""")
assertEquals("phone-X", decoded.resourceWait?.resource)
assertFalse(isWaitingRunStatus(decoded.status))
}
}