diff --git a/android/app/src/main/java/net/awan/pipelines/data/transport/PipelinesClient.kt b/android/app/src/main/java/net/awan/pipelines/data/transport/PipelinesClient.kt index 886d99e3..b649f361 100644 --- a/android/app/src/main/java/net/awan/pipelines/data/transport/PipelinesClient.kt +++ b/android/app/src/main/java/net/awan/pipelines/data/transport/PipelinesClient.kt @@ -78,11 +78,33 @@ class PipelinesClient( @Volatile private var manualClose = false + /** Serializes socket replacement with callbacks arriving on OkHttp threads. */ + private val connectionLock = Any() + private var reconnectJob: Job? = null private var attempt = 0 /** Opens (or re-opens) the connection to a validated server URL. */ fun connect(validated: ServerUrl.Validated) { + synchronized(connectionLock) { + connectLocked(validated) + } + } + + /** + * Re-opens a connection the transport lost without reversing an explicit + * [disconnect]. The check and connection replacement share one lock, so a + * Disconnect racing a foreground liveness probe wins even after that probe + * has decided recovery is needed. + */ + fun recover(validated: ServerUrl.Validated) { + synchronized(connectionLock) { + if (manualClose) return + connectLocked(validated) + } + } + + private fun connectLocked(validated: ServerUrl.Validated) { val httpUrl = validated.httpUrl ?: throw IllegalArgumentException("server URL is not valid") endpoint = httpUrl manualClose = false @@ -94,14 +116,16 @@ class PipelinesClient( /** Explicit user disconnect: stops reconnection and closes the socket. */ fun disconnect() { - manualClose = true - reconnectJob?.cancel() - reconnectJob = null - val socket = webSocket - webSocket = null - socket?.close(NORMAL_CLOSURE, "client closing") - failPending("Disconnected") - _state.value = ConnectionState.Idle + synchronized(connectionLock) { + manualClose = true + reconnectJob?.cancel() + reconnectJob = null + val socket = webSocket + webSocket = null + socket?.close(NORMAL_CLOSURE, "client closing") + failPending("Disconnected") + _state.value = ConnectionState.Idle + } } fun close() { @@ -115,14 +139,20 @@ class PipelinesClient( */ suspend fun call(request: Message): Message = withContext(ioDispatcher) { sendLock.withLock { - val socket = webSocket ?: throw NotConnectedException() - if (_state.value !is ConnectionState.Connected) throw NotConnectedException() - val id = "android-" + sequence.incrementAndGet() - val deferred = CompletableDeferred() - pending[id] = deferred - try { + val (id, deferred) = synchronized(connectionLock) { + val socket = webSocket ?: throw NotConnectedException() + if (_state.value !is ConnectionState.Connected) throw NotConnectedException() + val id = "android-" + sequence.incrementAndGet() + val deferred = CompletableDeferred() + pending[id] = deferred val payload = json.encodeToString(Message.serializer(), request.copy(id = id)) - if (!socket.send(payload)) throw IOException("websocket send failed") + if (!socket.send(payload)) { + pending.remove(id) + throw IOException("websocket send failed") + } + id to deferred + } + try { val response = withTimeout(requestTimeoutMillis) { deferred.await() } if (response.type == Protocol.ERROR) { throw ProtocolException(response.error?.ifBlank { "server error" } ?: "server error") @@ -144,12 +174,14 @@ class PipelinesClient( */ suspend fun sendOnly(request: Message): String = withContext(ioDispatcher) { sendLock.withLock { - val socket = webSocket ?: throw NotConnectedException() - if (_state.value !is ConnectionState.Connected) throw NotConnectedException() - val id = "android-" + sequence.incrementAndGet() - val payload = json.encodeToString(Message.serializer(), request.copy(id = id)) - if (!socket.send(payload)) throw IOException("websocket send failed") - id + synchronized(connectionLock) { + val socket = webSocket ?: throw NotConnectedException() + if (_state.value !is ConnectionState.Connected) throw NotConnectedException() + val id = "android-" + sequence.incrementAndGet() + val payload = json.encodeToString(Message.serializer(), request.copy(id = id)) + if (!socket.send(payload)) throw IOException("websocket send failed") + id + } } } @@ -166,14 +198,34 @@ class PipelinesClient( } } - private fun open() { + private fun open() = synchronized(connectionLock) { openLocked() } + + private fun openLocked() { val current = endpoint ?: return attempt += 1 _state.value = ConnectionState.Connecting(attempt) val request = Request.Builder().url(current).build() + // Replace the current socket, then close the one it superseded. Its + // callbacks are ignored from then on (see [isCurrent]), so a failure the + // old socket reports late — the common case when the process was frozen + // while it died — can neither clear the new socket nor schedule a + // reconnect that would undo it. + val superseded = webSocket webSocket = httpClient.newWebSocket(request, listener) + // A request sent over the superseded socket cannot receive a reply from + // the replacement. Fail it now rather than leaving a screen waiting for + // the ordinary request timeout after recovery. + if (superseded != null) failPending("Connection replaced") + superseded?.close(NORMAL_CLOSURE, "superseded") } + /** + * Whether [socket] is still the connection this client acts on. A socket this + * client has already replaced (a deliberate reconnect, or the foreground + * recovery re-establishing a frozen one) must never drive the state again. + */ + private fun isCurrent(socket: WebSocket): Boolean = webSocket === socket + private fun failPending(reason: String) { val snapshot = pending.keys.toList() for (key in snapshot) { @@ -181,7 +233,8 @@ class PipelinesClient( } } - private fun handleClosed(reason: String) { + private fun handleClosed(socket: WebSocket, reason: String) { + if (!isCurrent(socket)) return webSocket = null failPending(reason) if (manualClose) { @@ -194,52 +247,65 @@ class PipelinesClient( private fun scheduleReconnect(reason: String) { if (reconnectJob?.isActive == true) return _state.value = ConnectionState.Disconnected(reason, retrying = true) - val nextAttempt = attempt - val backoff = (BASE_BACKOFF_MILLIS shl (nextAttempt - 1).coerceAtMost(6)) - .coerceAtMost(MAX_BACKOFF_MILLIS) + val backoff = reconnectBackoffMillis(attempt) val jitter = Random.nextLong(0, 400) reconnectJob = scope.launch { delay(backoff + jitter) - if (!manualClose && endpoint != null) open() + synchronized(connectionLock) { + if (!manualClose && endpoint != null) open() + } } } private val listener = object : WebSocketListener() { override fun onOpen(webSocket: WebSocket, response: Response) { - attempt = 0 - _state.value = ConnectionState.Connected(build = null, serverTime = null) + synchronized(connectionLock) { + if (!isCurrent(webSocket)) return + attempt = 0 + _state.value = ConnectionState.Connected(build = null, serverTime = null) + } } override fun onMessage(webSocket: WebSocket, text: String) { - val message = try { - json.decodeFromString(Message.serializer(), text) - } catch (_: Exception) { - // Malformed frame: never misattribute it to a request. - return - } - val id = message.id - val deferred = if (id != null) pending.remove(id) else null - if (deferred != null) { - deferred.complete(message) - } else { - // Unsolicited envelopes (the connection's initial pipeline_list, - // server-pushed chat frames, stream errors) carry no matching id: - // hand them to the event stream instead of dropping them. - _events.tryEmit(message) + synchronized(connectionLock) { + if (!isCurrent(webSocket)) return + val message = try { + json.decodeFromString(Message.serializer(), text) + } catch (_: Exception) { + // Malformed frame: never misattribute it to a request. + return + } + val id = message.id + val deferred = if (id != null) pending.remove(id) else null + if (deferred != null) { + deferred.complete(message) + } else { + // Unsolicited envelopes (the connection's initial pipeline_list, + // server-pushed chat frames, stream errors) carry no matching id: + // hand them to the event stream instead of dropping them. + _events.tryEmit(message) + } } } override fun onClosed(webSocket: WebSocket, code: Int, reason: String) { - handleClosed("Connection closed (${code})") + synchronized(connectionLock) { + handleClosed(webSocket, "Connection closed (${code})") + } } override fun onClosing(webSocket: WebSocket, code: Int, reason: String) { - webSocket.close(NORMAL_CLOSURE, null) + synchronized(connectionLock) { + if (!isCurrent(webSocket)) return + webSocket.close(NORMAL_CLOSURE, null) + } } override fun onFailure(webSocket: WebSocket, t: Throwable, response: Response?) { - val reason = t.message?.ifBlank { t.javaClass.simpleName } ?: "connection failed" - handleClosed(reason) + synchronized(connectionLock) { + val reason = t.message?.ifBlank { t.javaClass.simpleName } ?: "connection failed" + handleClosed(webSocket, reason) + } } } @@ -247,6 +313,16 @@ class PipelinesClient( private const val NORMAL_CLOSURE = 1000 private const val BASE_BACKOFF_MILLIS = 1_000L private const val MAX_BACKOFF_MILLIS = 30_000L + + /** + * The successful open callback resets [attempt] to zero. A later socket + * failure must therefore begin at the base delay, never shift by -1 + * (which Kotlin masks into a very large shift count). + */ + internal fun reconnectBackoffMillis(attempt: Int): Long { + val exponent = (attempt - 1).coerceIn(0, 6) + return (BASE_BACKOFF_MILLIS shl exponent).coerceAtMost(MAX_BACKOFF_MILLIS) + } } } diff --git a/android/app/src/test/java/net/awan/pipelines/data/transport/ConnectionRecoveryTest.kt b/android/app/src/test/java/net/awan/pipelines/data/transport/ConnectionRecoveryTest.kt new file mode 100644 index 00000000..13e2bd1a --- /dev/null +++ b/android/app/src/test/java/net/awan/pipelines/data/transport/ConnectionRecoveryTest.kt @@ -0,0 +1,60 @@ +package net.awan.pipelines.data.transport + +import org.junit.Assert.assertEquals +import org.junit.Test + +/** + * The return-to-foreground policy, in the one pure place it lives: an explicit + * Disconnect is never overridden, a dropped or in-flight connection is + * re-established at once, and a still-Connected socket is trusted only after it + * has been confirmed to answer. + */ +class ConnectionRecoveryTest { + + @Test + fun aDeliberateDisconnectIsNeverOverridden() { + // Idle is only ever reached by a deliberate Disconnect (the client starts + // Idle and never returns to it by itself), so returning must leave it. + assertEquals( + ForegroundAction.LeaveAlone, + foregroundAction(ConnectionState.Idle, confirmedAlive = false), + ) + } + + @Test + fun aDroppedConnectionReconnectsAtOnce() { + assertEquals( + ForegroundAction.Reconnect, + foregroundAction(ConnectionState.Disconnected("socket closed", retrying = true), confirmedAlive = false), + ) + } + + @Test + fun anOpenAlreadyInFlightIsRestartedRatherThanWaitedOn() { + assertEquals( + ForegroundAction.Reconnect, + foregroundAction(ConnectionState.Connecting(3), confirmedAlive = false), + ) + } + + @Test + fun aConnectedSocketIsTrustedOnlyAfterItAnswers() { + val connected = ConnectionState.Connected(build = null, serverTime = null) + assertEquals(ForegroundAction.LeaveAlone, foregroundAction(connected, confirmedAlive = true)) + // The defect: a socket the process still calls Connected but that answers + // nothing must be reconnected, not trusted. + assertEquals(ForegroundAction.Reconnect, foregroundAction(connected, confirmedAlive = false)) + } + + @Test + fun aFailureAfterConnectedStartsAtTheBaseBackoff() { + // onOpen resets the client's attempt counter to zero. The next failure + // must not pass -1 to the shift used for exponential backoff. + assertEquals(1_000L, PipelinesClient.reconnectBackoffMillis(attempt = 0)) + } + + @Test + fun reconnectBackoffRemainsBounded() { + assertEquals(30_000L, PipelinesClient.reconnectBackoffMillis(attempt = Int.MAX_VALUE)) + } +}