Guard socket recovery from stale callbacks

Serialize socket replacement and callbacks so a superseded socket cannot clear or regress the active connection, and make reconnect backoff safe after a successful open. Add focused recovery regression coverage.\n\nChecked packaged templates and packaged authoring skill references; neither needs updating for this Android transport-only change.
This commit is contained in:
2026-10-18 12:27:24 +01:00
parent b93eee234c
commit 269a8d65bb
2 changed files with 184 additions and 48 deletions
@@ -78,11 +78,33 @@ class PipelinesClient(
@Volatile
private var manualClose = false
/** Serializes socket replacement with callbacks arriving on OkHttp threads. */
private val connectionLock = Any()
private var reconnectJob: Job? = null
private var attempt = 0
/** Opens (or re-opens) the connection to a validated server URL. */
fun connect(validated: ServerUrl.Validated) {
synchronized(connectionLock) {
connectLocked(validated)
}
}
/**
* Re-opens a connection the transport lost without reversing an explicit
* [disconnect]. The check and connection replacement share one lock, so a
* Disconnect racing a foreground liveness probe wins even after that probe
* has decided recovery is needed.
*/
fun recover(validated: ServerUrl.Validated) {
synchronized(connectionLock) {
if (manualClose) return
connectLocked(validated)
}
}
private fun connectLocked(validated: ServerUrl.Validated) {
val httpUrl = validated.httpUrl ?: throw IllegalArgumentException("server URL is not valid")
endpoint = httpUrl
manualClose = false
@@ -94,14 +116,16 @@ class PipelinesClient(
/** Explicit user disconnect: stops reconnection and closes the socket. */
fun disconnect() {
manualClose = true
reconnectJob?.cancel()
reconnectJob = null
val socket = webSocket
webSocket = null
socket?.close(NORMAL_CLOSURE, "client closing")
failPending("Disconnected")
_state.value = ConnectionState.Idle
synchronized(connectionLock) {
manualClose = true
reconnectJob?.cancel()
reconnectJob = null
val socket = webSocket
webSocket = null
socket?.close(NORMAL_CLOSURE, "client closing")
failPending("Disconnected")
_state.value = ConnectionState.Idle
}
}
fun close() {
@@ -115,14 +139,20 @@ class PipelinesClient(
*/
suspend fun call(request: Message): Message = withContext(ioDispatcher) {
sendLock.withLock {
val socket = webSocket ?: throw NotConnectedException()
if (_state.value !is ConnectionState.Connected) throw NotConnectedException()
val id = "android-" + sequence.incrementAndGet()
val deferred = CompletableDeferred<Message>()
pending[id] = deferred
try {
val (id, deferred) = synchronized(connectionLock) {
val socket = webSocket ?: throw NotConnectedException()
if (_state.value !is ConnectionState.Connected) throw NotConnectedException()
val id = "android-" + sequence.incrementAndGet()
val deferred = CompletableDeferred<Message>()
pending[id] = deferred
val payload = json.encodeToString(Message.serializer(), request.copy(id = id))
if (!socket.send(payload)) throw IOException("websocket send failed")
if (!socket.send(payload)) {
pending.remove(id)
throw IOException("websocket send failed")
}
id to deferred
}
try {
val response = withTimeout(requestTimeoutMillis) { deferred.await() }
if (response.type == Protocol.ERROR) {
throw ProtocolException(response.error?.ifBlank { "server error" } ?: "server error")
@@ -144,12 +174,14 @@ class PipelinesClient(
*/
suspend fun sendOnly(request: Message): String = withContext(ioDispatcher) {
sendLock.withLock {
val socket = webSocket ?: throw NotConnectedException()
if (_state.value !is ConnectionState.Connected) throw NotConnectedException()
val id = "android-" + sequence.incrementAndGet()
val payload = json.encodeToString(Message.serializer(), request.copy(id = id))
if (!socket.send(payload)) throw IOException("websocket send failed")
id
synchronized(connectionLock) {
val socket = webSocket ?: throw NotConnectedException()
if (_state.value !is ConnectionState.Connected) throw NotConnectedException()
val id = "android-" + sequence.incrementAndGet()
val payload = json.encodeToString(Message.serializer(), request.copy(id = id))
if (!socket.send(payload)) throw IOException("websocket send failed")
id
}
}
}
@@ -166,14 +198,34 @@ class PipelinesClient(
}
}
private fun open() {
private fun open() = synchronized(connectionLock) { openLocked() }
private fun openLocked() {
val current = endpoint ?: return
attempt += 1
_state.value = ConnectionState.Connecting(attempt)
val request = Request.Builder().url(current).build()
// Replace the current socket, then close the one it superseded. Its
// callbacks are ignored from then on (see [isCurrent]), so a failure the
// old socket reports late — the common case when the process was frozen
// while it died — can neither clear the new socket nor schedule a
// reconnect that would undo it.
val superseded = webSocket
webSocket = httpClient.newWebSocket(request, listener)
// A request sent over the superseded socket cannot receive a reply from
// the replacement. Fail it now rather than leaving a screen waiting for
// the ordinary request timeout after recovery.
if (superseded != null) failPending("Connection replaced")
superseded?.close(NORMAL_CLOSURE, "superseded")
}
/**
* Whether [socket] is still the connection this client acts on. A socket this
* client has already replaced (a deliberate reconnect, or the foreground
* recovery re-establishing a frozen one) must never drive the state again.
*/
private fun isCurrent(socket: WebSocket): Boolean = webSocket === socket
private fun failPending(reason: String) {
val snapshot = pending.keys.toList()
for (key in snapshot) {
@@ -181,7 +233,8 @@ class PipelinesClient(
}
}
private fun handleClosed(reason: String) {
private fun handleClosed(socket: WebSocket, reason: String) {
if (!isCurrent(socket)) return
webSocket = null
failPending(reason)
if (manualClose) {
@@ -194,52 +247,65 @@ class PipelinesClient(
private fun scheduleReconnect(reason: String) {
if (reconnectJob?.isActive == true) return
_state.value = ConnectionState.Disconnected(reason, retrying = true)
val nextAttempt = attempt
val backoff = (BASE_BACKOFF_MILLIS shl (nextAttempt - 1).coerceAtMost(6))
.coerceAtMost(MAX_BACKOFF_MILLIS)
val backoff = reconnectBackoffMillis(attempt)
val jitter = Random.nextLong(0, 400)
reconnectJob = scope.launch {
delay(backoff + jitter)
if (!manualClose && endpoint != null) open()
synchronized(connectionLock) {
if (!manualClose && endpoint != null) open()
}
}
}
private val listener = object : WebSocketListener() {
override fun onOpen(webSocket: WebSocket, response: Response) {
attempt = 0
_state.value = ConnectionState.Connected(build = null, serverTime = null)
synchronized(connectionLock) {
if (!isCurrent(webSocket)) return
attempt = 0
_state.value = ConnectionState.Connected(build = null, serverTime = null)
}
}
override fun onMessage(webSocket: WebSocket, text: String) {
val message = try {
json.decodeFromString(Message.serializer(), text)
} catch (_: Exception) {
// Malformed frame: never misattribute it to a request.
return
}
val id = message.id
val deferred = if (id != null) pending.remove(id) else null
if (deferred != null) {
deferred.complete(message)
} else {
// Unsolicited envelopes (the connection's initial pipeline_list,
// server-pushed chat frames, stream errors) carry no matching id:
// hand them to the event stream instead of dropping them.
_events.tryEmit(message)
synchronized(connectionLock) {
if (!isCurrent(webSocket)) return
val message = try {
json.decodeFromString(Message.serializer(), text)
} catch (_: Exception) {
// Malformed frame: never misattribute it to a request.
return
}
val id = message.id
val deferred = if (id != null) pending.remove(id) else null
if (deferred != null) {
deferred.complete(message)
} else {
// Unsolicited envelopes (the connection's initial pipeline_list,
// server-pushed chat frames, stream errors) carry no matching id:
// hand them to the event stream instead of dropping them.
_events.tryEmit(message)
}
}
}
override fun onClosed(webSocket: WebSocket, code: Int, reason: String) {
handleClosed("Connection closed (${code})")
synchronized(connectionLock) {
handleClosed(webSocket, "Connection closed (${code})")
}
}
override fun onClosing(webSocket: WebSocket, code: Int, reason: String) {
webSocket.close(NORMAL_CLOSURE, null)
synchronized(connectionLock) {
if (!isCurrent(webSocket)) return
webSocket.close(NORMAL_CLOSURE, null)
}
}
override fun onFailure(webSocket: WebSocket, t: Throwable, response: Response?) {
val reason = t.message?.ifBlank { t.javaClass.simpleName } ?: "connection failed"
handleClosed(reason)
synchronized(connectionLock) {
val reason = t.message?.ifBlank { t.javaClass.simpleName } ?: "connection failed"
handleClosed(webSocket, reason)
}
}
}
@@ -247,6 +313,16 @@ class PipelinesClient(
private const val NORMAL_CLOSURE = 1000
private const val BASE_BACKOFF_MILLIS = 1_000L
private const val MAX_BACKOFF_MILLIS = 30_000L
/**
* The successful open callback resets [attempt] to zero. A later socket
* failure must therefore begin at the base delay, never shift by -1
* (which Kotlin masks into a very large shift count).
*/
internal fun reconnectBackoffMillis(attempt: Int): Long {
val exponent = (attempt - 1).coerceIn(0, 6)
return (BASE_BACKOFF_MILLIS shl exponent).coerceAtMost(MAX_BACKOFF_MILLIS)
}
}
}
@@ -0,0 +1,60 @@
package net.awan.pipelines.data.transport
import org.junit.Assert.assertEquals
import org.junit.Test
/**
* The return-to-foreground policy, in the one pure place it lives: an explicit
* Disconnect is never overridden, a dropped or in-flight connection is
* re-established at once, and a still-Connected socket is trusted only after it
* has been confirmed to answer.
*/
class ConnectionRecoveryTest {
@Test
fun aDeliberateDisconnectIsNeverOverridden() {
// Idle is only ever reached by a deliberate Disconnect (the client starts
// Idle and never returns to it by itself), so returning must leave it.
assertEquals(
ForegroundAction.LeaveAlone,
foregroundAction(ConnectionState.Idle, confirmedAlive = false),
)
}
@Test
fun aDroppedConnectionReconnectsAtOnce() {
assertEquals(
ForegroundAction.Reconnect,
foregroundAction(ConnectionState.Disconnected("socket closed", retrying = true), confirmedAlive = false),
)
}
@Test
fun anOpenAlreadyInFlightIsRestartedRatherThanWaitedOn() {
assertEquals(
ForegroundAction.Reconnect,
foregroundAction(ConnectionState.Connecting(3), confirmedAlive = false),
)
}
@Test
fun aConnectedSocketIsTrustedOnlyAfterItAnswers() {
val connected = ConnectionState.Connected(build = null, serverTime = null)
assertEquals(ForegroundAction.LeaveAlone, foregroundAction(connected, confirmedAlive = true))
// The defect: a socket the process still calls Connected but that answers
// nothing must be reconnected, not trusted.
assertEquals(ForegroundAction.Reconnect, foregroundAction(connected, confirmedAlive = false))
}
@Test
fun aFailureAfterConnectedStartsAtTheBaseBackoff() {
// onOpen resets the client's attempt counter to zero. The next failure
// must not pass -1 to the shift used for exponential backoff.
assertEquals(1_000L, PipelinesClient.reconnectBackoffMillis(attempt = 0))
}
@Test
fun reconnectBackoffRemainsBounded() {
assertEquals(30_000L, PipelinesClient.reconnectBackoffMillis(attempt = Int.MAX_VALUE))
}
}