Guard socket recovery from stale callbacks
Serialize socket replacement and callbacks so a superseded socket cannot clear or regress the active connection, and make reconnect backoff safe after a successful open. Add focused recovery regression coverage.\n\nChecked packaged templates and packaged authoring skill references; neither needs updating for this Android transport-only change.
This commit is contained in:
@@ -78,11 +78,33 @@ class PipelinesClient(
|
||||
@Volatile
|
||||
private var manualClose = false
|
||||
|
||||
/** Serializes socket replacement with callbacks arriving on OkHttp threads. */
|
||||
private val connectionLock = Any()
|
||||
|
||||
private var reconnectJob: Job? = null
|
||||
private var attempt = 0
|
||||
|
||||
/** Opens (or re-opens) the connection to a validated server URL. */
|
||||
fun connect(validated: ServerUrl.Validated) {
|
||||
synchronized(connectionLock) {
|
||||
connectLocked(validated)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Re-opens a connection the transport lost without reversing an explicit
|
||||
* [disconnect]. The check and connection replacement share one lock, so a
|
||||
* Disconnect racing a foreground liveness probe wins even after that probe
|
||||
* has decided recovery is needed.
|
||||
*/
|
||||
fun recover(validated: ServerUrl.Validated) {
|
||||
synchronized(connectionLock) {
|
||||
if (manualClose) return
|
||||
connectLocked(validated)
|
||||
}
|
||||
}
|
||||
|
||||
private fun connectLocked(validated: ServerUrl.Validated) {
|
||||
val httpUrl = validated.httpUrl ?: throw IllegalArgumentException("server URL is not valid")
|
||||
endpoint = httpUrl
|
||||
manualClose = false
|
||||
@@ -94,14 +116,16 @@ class PipelinesClient(
|
||||
|
||||
/** Explicit user disconnect: stops reconnection and closes the socket. */
|
||||
fun disconnect() {
|
||||
manualClose = true
|
||||
reconnectJob?.cancel()
|
||||
reconnectJob = null
|
||||
val socket = webSocket
|
||||
webSocket = null
|
||||
socket?.close(NORMAL_CLOSURE, "client closing")
|
||||
failPending("Disconnected")
|
||||
_state.value = ConnectionState.Idle
|
||||
synchronized(connectionLock) {
|
||||
manualClose = true
|
||||
reconnectJob?.cancel()
|
||||
reconnectJob = null
|
||||
val socket = webSocket
|
||||
webSocket = null
|
||||
socket?.close(NORMAL_CLOSURE, "client closing")
|
||||
failPending("Disconnected")
|
||||
_state.value = ConnectionState.Idle
|
||||
}
|
||||
}
|
||||
|
||||
fun close() {
|
||||
@@ -115,14 +139,20 @@ class PipelinesClient(
|
||||
*/
|
||||
suspend fun call(request: Message): Message = withContext(ioDispatcher) {
|
||||
sendLock.withLock {
|
||||
val socket = webSocket ?: throw NotConnectedException()
|
||||
if (_state.value !is ConnectionState.Connected) throw NotConnectedException()
|
||||
val id = "android-" + sequence.incrementAndGet()
|
||||
val deferred = CompletableDeferred<Message>()
|
||||
pending[id] = deferred
|
||||
try {
|
||||
val (id, deferred) = synchronized(connectionLock) {
|
||||
val socket = webSocket ?: throw NotConnectedException()
|
||||
if (_state.value !is ConnectionState.Connected) throw NotConnectedException()
|
||||
val id = "android-" + sequence.incrementAndGet()
|
||||
val deferred = CompletableDeferred<Message>()
|
||||
pending[id] = deferred
|
||||
val payload = json.encodeToString(Message.serializer(), request.copy(id = id))
|
||||
if (!socket.send(payload)) throw IOException("websocket send failed")
|
||||
if (!socket.send(payload)) {
|
||||
pending.remove(id)
|
||||
throw IOException("websocket send failed")
|
||||
}
|
||||
id to deferred
|
||||
}
|
||||
try {
|
||||
val response = withTimeout(requestTimeoutMillis) { deferred.await() }
|
||||
if (response.type == Protocol.ERROR) {
|
||||
throw ProtocolException(response.error?.ifBlank { "server error" } ?: "server error")
|
||||
@@ -144,12 +174,14 @@ class PipelinesClient(
|
||||
*/
|
||||
suspend fun sendOnly(request: Message): String = withContext(ioDispatcher) {
|
||||
sendLock.withLock {
|
||||
val socket = webSocket ?: throw NotConnectedException()
|
||||
if (_state.value !is ConnectionState.Connected) throw NotConnectedException()
|
||||
val id = "android-" + sequence.incrementAndGet()
|
||||
val payload = json.encodeToString(Message.serializer(), request.copy(id = id))
|
||||
if (!socket.send(payload)) throw IOException("websocket send failed")
|
||||
id
|
||||
synchronized(connectionLock) {
|
||||
val socket = webSocket ?: throw NotConnectedException()
|
||||
if (_state.value !is ConnectionState.Connected) throw NotConnectedException()
|
||||
val id = "android-" + sequence.incrementAndGet()
|
||||
val payload = json.encodeToString(Message.serializer(), request.copy(id = id))
|
||||
if (!socket.send(payload)) throw IOException("websocket send failed")
|
||||
id
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -166,14 +198,34 @@ class PipelinesClient(
|
||||
}
|
||||
}
|
||||
|
||||
private fun open() {
|
||||
private fun open() = synchronized(connectionLock) { openLocked() }
|
||||
|
||||
private fun openLocked() {
|
||||
val current = endpoint ?: return
|
||||
attempt += 1
|
||||
_state.value = ConnectionState.Connecting(attempt)
|
||||
val request = Request.Builder().url(current).build()
|
||||
// Replace the current socket, then close the one it superseded. Its
|
||||
// callbacks are ignored from then on (see [isCurrent]), so a failure the
|
||||
// old socket reports late — the common case when the process was frozen
|
||||
// while it died — can neither clear the new socket nor schedule a
|
||||
// reconnect that would undo it.
|
||||
val superseded = webSocket
|
||||
webSocket = httpClient.newWebSocket(request, listener)
|
||||
// A request sent over the superseded socket cannot receive a reply from
|
||||
// the replacement. Fail it now rather than leaving a screen waiting for
|
||||
// the ordinary request timeout after recovery.
|
||||
if (superseded != null) failPending("Connection replaced")
|
||||
superseded?.close(NORMAL_CLOSURE, "superseded")
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether [socket] is still the connection this client acts on. A socket this
|
||||
* client has already replaced (a deliberate reconnect, or the foreground
|
||||
* recovery re-establishing a frozen one) must never drive the state again.
|
||||
*/
|
||||
private fun isCurrent(socket: WebSocket): Boolean = webSocket === socket
|
||||
|
||||
private fun failPending(reason: String) {
|
||||
val snapshot = pending.keys.toList()
|
||||
for (key in snapshot) {
|
||||
@@ -181,7 +233,8 @@ class PipelinesClient(
|
||||
}
|
||||
}
|
||||
|
||||
private fun handleClosed(reason: String) {
|
||||
private fun handleClosed(socket: WebSocket, reason: String) {
|
||||
if (!isCurrent(socket)) return
|
||||
webSocket = null
|
||||
failPending(reason)
|
||||
if (manualClose) {
|
||||
@@ -194,52 +247,65 @@ class PipelinesClient(
|
||||
private fun scheduleReconnect(reason: String) {
|
||||
if (reconnectJob?.isActive == true) return
|
||||
_state.value = ConnectionState.Disconnected(reason, retrying = true)
|
||||
val nextAttempt = attempt
|
||||
val backoff = (BASE_BACKOFF_MILLIS shl (nextAttempt - 1).coerceAtMost(6))
|
||||
.coerceAtMost(MAX_BACKOFF_MILLIS)
|
||||
val backoff = reconnectBackoffMillis(attempt)
|
||||
val jitter = Random.nextLong(0, 400)
|
||||
reconnectJob = scope.launch {
|
||||
delay(backoff + jitter)
|
||||
if (!manualClose && endpoint != null) open()
|
||||
synchronized(connectionLock) {
|
||||
if (!manualClose && endpoint != null) open()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private val listener = object : WebSocketListener() {
|
||||
override fun onOpen(webSocket: WebSocket, response: Response) {
|
||||
attempt = 0
|
||||
_state.value = ConnectionState.Connected(build = null, serverTime = null)
|
||||
synchronized(connectionLock) {
|
||||
if (!isCurrent(webSocket)) return
|
||||
attempt = 0
|
||||
_state.value = ConnectionState.Connected(build = null, serverTime = null)
|
||||
}
|
||||
}
|
||||
|
||||
override fun onMessage(webSocket: WebSocket, text: String) {
|
||||
val message = try {
|
||||
json.decodeFromString(Message.serializer(), text)
|
||||
} catch (_: Exception) {
|
||||
// Malformed frame: never misattribute it to a request.
|
||||
return
|
||||
}
|
||||
val id = message.id
|
||||
val deferred = if (id != null) pending.remove(id) else null
|
||||
if (deferred != null) {
|
||||
deferred.complete(message)
|
||||
} else {
|
||||
// Unsolicited envelopes (the connection's initial pipeline_list,
|
||||
// server-pushed chat frames, stream errors) carry no matching id:
|
||||
// hand them to the event stream instead of dropping them.
|
||||
_events.tryEmit(message)
|
||||
synchronized(connectionLock) {
|
||||
if (!isCurrent(webSocket)) return
|
||||
val message = try {
|
||||
json.decodeFromString(Message.serializer(), text)
|
||||
} catch (_: Exception) {
|
||||
// Malformed frame: never misattribute it to a request.
|
||||
return
|
||||
}
|
||||
val id = message.id
|
||||
val deferred = if (id != null) pending.remove(id) else null
|
||||
if (deferred != null) {
|
||||
deferred.complete(message)
|
||||
} else {
|
||||
// Unsolicited envelopes (the connection's initial pipeline_list,
|
||||
// server-pushed chat frames, stream errors) carry no matching id:
|
||||
// hand them to the event stream instead of dropping them.
|
||||
_events.tryEmit(message)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
override fun onClosed(webSocket: WebSocket, code: Int, reason: String) {
|
||||
handleClosed("Connection closed (${code})")
|
||||
synchronized(connectionLock) {
|
||||
handleClosed(webSocket, "Connection closed (${code})")
|
||||
}
|
||||
}
|
||||
|
||||
override fun onClosing(webSocket: WebSocket, code: Int, reason: String) {
|
||||
webSocket.close(NORMAL_CLOSURE, null)
|
||||
synchronized(connectionLock) {
|
||||
if (!isCurrent(webSocket)) return
|
||||
webSocket.close(NORMAL_CLOSURE, null)
|
||||
}
|
||||
}
|
||||
|
||||
override fun onFailure(webSocket: WebSocket, t: Throwable, response: Response?) {
|
||||
val reason = t.message?.ifBlank { t.javaClass.simpleName } ?: "connection failed"
|
||||
handleClosed(reason)
|
||||
synchronized(connectionLock) {
|
||||
val reason = t.message?.ifBlank { t.javaClass.simpleName } ?: "connection failed"
|
||||
handleClosed(webSocket, reason)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -247,6 +313,16 @@ class PipelinesClient(
|
||||
private const val NORMAL_CLOSURE = 1000
|
||||
private const val BASE_BACKOFF_MILLIS = 1_000L
|
||||
private const val MAX_BACKOFF_MILLIS = 30_000L
|
||||
|
||||
/**
|
||||
* The successful open callback resets [attempt] to zero. A later socket
|
||||
* failure must therefore begin at the base delay, never shift by -1
|
||||
* (which Kotlin masks into a very large shift count).
|
||||
*/
|
||||
internal fun reconnectBackoffMillis(attempt: Int): Long {
|
||||
val exponent = (attempt - 1).coerceIn(0, 6)
|
||||
return (BASE_BACKOFF_MILLIS shl exponent).coerceAtMost(MAX_BACKOFF_MILLIS)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,60 @@
|
||||
package net.awan.pipelines.data.transport
|
||||
|
||||
import org.junit.Assert.assertEquals
|
||||
import org.junit.Test
|
||||
|
||||
/**
|
||||
* The return-to-foreground policy, in the one pure place it lives: an explicit
|
||||
* Disconnect is never overridden, a dropped or in-flight connection is
|
||||
* re-established at once, and a still-Connected socket is trusted only after it
|
||||
* has been confirmed to answer.
|
||||
*/
|
||||
class ConnectionRecoveryTest {
|
||||
|
||||
@Test
|
||||
fun aDeliberateDisconnectIsNeverOverridden() {
|
||||
// Idle is only ever reached by a deliberate Disconnect (the client starts
|
||||
// Idle and never returns to it by itself), so returning must leave it.
|
||||
assertEquals(
|
||||
ForegroundAction.LeaveAlone,
|
||||
foregroundAction(ConnectionState.Idle, confirmedAlive = false),
|
||||
)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun aDroppedConnectionReconnectsAtOnce() {
|
||||
assertEquals(
|
||||
ForegroundAction.Reconnect,
|
||||
foregroundAction(ConnectionState.Disconnected("socket closed", retrying = true), confirmedAlive = false),
|
||||
)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun anOpenAlreadyInFlightIsRestartedRatherThanWaitedOn() {
|
||||
assertEquals(
|
||||
ForegroundAction.Reconnect,
|
||||
foregroundAction(ConnectionState.Connecting(3), confirmedAlive = false),
|
||||
)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun aConnectedSocketIsTrustedOnlyAfterItAnswers() {
|
||||
val connected = ConnectionState.Connected(build = null, serverTime = null)
|
||||
assertEquals(ForegroundAction.LeaveAlone, foregroundAction(connected, confirmedAlive = true))
|
||||
// The defect: a socket the process still calls Connected but that answers
|
||||
// nothing must be reconnected, not trusted.
|
||||
assertEquals(ForegroundAction.Reconnect, foregroundAction(connected, confirmedAlive = false))
|
||||
}
|
||||
|
||||
@Test
|
||||
fun aFailureAfterConnectedStartsAtTheBaseBackoff() {
|
||||
// onOpen resets the client's attempt counter to zero. The next failure
|
||||
// must not pass -1 to the shift used for exponential backoff.
|
||||
assertEquals(1_000L, PipelinesClient.reconnectBackoffMillis(attempt = 0))
|
||||
}
|
||||
|
||||
@Test
|
||||
fun reconnectBackoffRemainsBounded() {
|
||||
assertEquals(30_000L, PipelinesClient.reconnectBackoffMillis(attempt = Int.MAX_VALUE))
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user