diff --git a/app/lib/core/transport/pipelines_client.dart b/app/lib/core/transport/pipelines_client.dart index 5e2eaf8d..bc5c9b2a 100644 --- a/app/lib/core/transport/pipelines_client.dart +++ b/app/lib/core/transport/pipelines_client.dart @@ -17,18 +17,22 @@ class PipelinesClient implements ApiGateway { PipelinesClient({ ChannelFactory? channelFactory, this.requestTimeout = const Duration(seconds: 30), + this.connectionTimeout = const Duration(seconds: 5), this.backoffBase = const Duration(seconds: 1), this.backoffMax = const Duration(seconds: 30), }) : _factory = channelFactory ?? connectPipelinesChannel; final ChannelFactory _factory; - final Duration requestTimeout, backoffBase, backoffMax; + final Duration requestTimeout, connectionTimeout, backoffBase, backoffMax; final _frames = StreamController.broadcast(); final _states = StreamController.broadcast(); final Map> _pending = {}; WebSocketChannel? _channel; StreamSubscription? _subscription; Uri? _endpoint; + List _endpoints = const []; + int _endpointIndex = 0; + Completer? _openingCancelled; Timer? _reconnect; int _generation = 0, _sequence = 0, _attempt = 0; bool _manual = true, _disposed = false; @@ -40,13 +44,28 @@ class PipelinesClient implements ApiGateway { Stream get states => _states.stream; ConnectionState get state => _state; - /// Last validated endpoint, including changes made through native Settings. + /// Current attempted or connected address. Uri? get endpoint => _endpoint; - Future connect(Uri uri) async { + /// Immutable, normalized addresses for this logical server, in priority order. + List get endpoints => _endpoints; + + Future connect(Uri uri) => connectAny([uri]); + + /// Starts at the primary address; automatic retries rotate only within [uris]. + Future connectAny(List uris) async { if (_disposed) throw StateError('Client has been disposed'); - final endpoint = ServerUrl.validate(uri.toString()); - _endpoint = Uri.parse(endpoint.wsUrl); + if (uris.isEmpty) { + throw ArgumentError.value(uris, 'uris', 'Must not be empty'); + } + // Validate the entire replacement before changing a healthy connection. + final candidates = uris + .map((uri) => Uri.parse(ServerUrl.validate(uri.toString()).wsUrl)) + .toSet() + .toList(growable: false); + _endpoints = List.unmodifiable(candidates); + _endpointIndex = 0; + _endpoint = _endpoints.first; _manual = false; _attempt = 0; _reconnect?.cancel(); @@ -57,6 +76,7 @@ class PipelinesClient implements ApiGateway { final endpoint = _endpoint; if (endpoint == null || _disposed) return; final generation = ++_generation; + _cancelOpening(); _setState(ConnectionState(ConnectionKind.connecting, attempt: ++_attempt)); final old = _channel; _channel = null; @@ -66,6 +86,8 @@ class PipelinesClient implements ApiGateway { _subscription = null; await subscription?.cancel(); if (generation != _generation || _manual || _disposed) return; + final cancelled = Completer(); + _openingCancelled = cancelled; try { final channel = _factory(endpoint); _channel = channel; @@ -80,15 +102,26 @@ class PipelinesClient implements ApiGateway { if (generation == _generation) _closed('Connection closed'); }, ); - await channel.ready; + await Future.any([ + channel.ready, + cancelled.future, + ]).timeout(connectionTimeout); if (generation != _generation) return; _attempt = 0; _setState(const ConnectionState(ConnectionKind.connected)); } catch (error) { if (generation == _generation) _closed(error.toString()); + } finally { + if (identical(_openingCancelled, cancelled)) _openingCancelled = null; } } + void _cancelOpening() { + final cancelled = _openingCancelled; + _openingCancelled = null; + if (cancelled != null && !cancelled.isCompleted) cancelled.complete(); + } + void _setState(ConnectionState value) { _state = value; if (!_states.isClosed) _states.add(value); @@ -196,6 +229,7 @@ class PipelinesClient implements ApiGateway { void _closed(String reason) { _generation++; + _cancelOpening(); final channel = _channel; final subscription = _subscription; _channel = null; @@ -217,17 +251,20 @@ class PipelinesClient implements ApiGateway { ); final shift = (_attempt - 1).clamp(0, 6); final delay = backoffBase * (1 << shift); + _endpointIndex = (_endpointIndex + 1) % _endpoints.length; _reconnect?.cancel(); - _reconnect = Timer( - delay > backoffMax ? backoffMax : delay, - () => unawaited(_open()), - ); + _reconnect = Timer(delay > backoffMax ? backoffMax : delay, () { + if (_manual || _disposed) return; + _endpoint = _endpoints[_endpointIndex]; + unawaited(_open()); + }); } Future disconnect() async { _manual = true; _reconnect?.cancel(); final generation = ++_generation; + _cancelOpening(); final channel = _channel; _channel = null; final subscription = _subscription; diff --git a/app/test/core/transport/address_fallback_test.dart b/app/test/core/transport/address_fallback_test.dart new file mode 100644 index 00000000..08a75a57 --- /dev/null +++ b/app/test/core/transport/address_fallback_test.dart @@ -0,0 +1,262 @@ +import 'dart:convert'; + +import 'package:flutter_test/flutter_test.dart'; +import 'package:pipelines_app/core/transport/pipelines_client.dart'; + +import 'pipelines_client_test.dart' show FakeChannel; + +final primary = Uri.parse('ws://tailscale.test/connect'); +final backup = Uri.parse('ws://lan.test/connect'); +const retry = Duration(milliseconds: 5); + +Future connected(PipelinesClient client) => client.states + .firstWhere((state) => state.kind == ConnectionKind.connected) + .timeout(const Duration(seconds: 2)); +Future flush() => Future.delayed(Duration.zero); + +void main() { + test('failed primary uses only configured backup and stays there', () async { + final tried = []; + final channel = FakeChannel()..readyCompleter.complete(); + final client = PipelinesClient( + channelFactory: (uri) { + tried.add(uri); + if (uri == primary) throw StateError('unreachable'); + return channel; + }, + backoffBase: retry, + ); + final ready = connected(client); + await client.connectAny([primary, backup]); + expect(client.endpoint, primary); + expect(client.state.retrying, isTrue); + await ready; + expect(tried, [primary, backup]); + expect(client.endpoint, backup); + await Future.delayed(const Duration(milliseconds: 40)); + expect(tried, [primary, backup]); + await client.dispose(); + }); + + test( + 'stalled readiness times out; late success cannot replace backup', + () async { + final first = FakeChannel(); + final second = FakeChannel()..readyCompleter.complete(); + final tried = []; + final client = PipelinesClient( + channelFactory: (uri) { + tried.add(uri); + return uri == primary ? first : second; + }, + connectionTimeout: const Duration(milliseconds: 10), + backoffBase: retry, + ); + final ready = connected(client); + await client.connectAny([primary, backup]); + expect(client.state.reason, contains('TimeoutException')); + await ready; + first.readyCompleter.complete(); + await flush(); + expect(client.endpoint, backup); + expect(client.state.kind, ConnectionKind.connected); + expect(tried, [primary, backup]); + await client.dispose(); + }, + ); + + test('failed readiness advances to backup', () async { + final first = FakeChannel(); + final second = FakeChannel()..readyCompleter.complete(); + final client = PipelinesClient( + channelFactory: (uri) => uri == primary ? first : second, + backoffBase: retry, + ); + final ready = connected(client); + final connecting = client.connectAny([primary, backup]); + await flush(); + first.readyCompleter.completeError(StateError('handshake refused')); + await connecting; + await ready; + expect(client.endpoint, backup); + await client.dispose(); + }); + + test('closed primary advances; explicit connect restarts primary', () async { + final channels = List.generate( + 3, + (_) => FakeChannel()..readyCompleter.complete(), + ); + final tried = []; + final client = PipelinesClient( + channelFactory: (uri) { + tried.add(uri); + return channels[tried.length - 1]; + }, + backoffBase: retry, + ); + await client.connectAny([primary, backup]); + final ready = connected(client); + await channels.first.incoming.close(); + await ready; + expect(client.endpoint, backup); + await client.connectAny(client.endpoints); + expect(tried, [primary, backup, primary]); + expect(client.endpoint, primary); + await client.dispose(); + }); + + testWidgets( + 'all down wraps with exponential capped backoff; manual stop cancels', + (tester) async { + final tried = []; + final client = PipelinesClient( + channelFactory: (uri) { + tried.add(uri); + throw StateError('offline'); + }, + backoffBase: const Duration(milliseconds: 20), + backoffMax: const Duration(milliseconds: 50), + ); + await client.connectAny([primary, backup]); + await tester.pump(const Duration(milliseconds: 19)); + expect(tried, [primary]); + await tester.pump(const Duration(milliseconds: 1)); + expect(tried, [primary, backup]); + await tester.pump(const Duration(milliseconds: 39)); + expect(tried, hasLength(2)); + await tester.pump(const Duration(milliseconds: 1)); + expect(tried, [primary, backup, primary]); + await tester.pump(const Duration(milliseconds: 49)); + expect(tried, hasLength(3)); + await tester.pump(const Duration(milliseconds: 1)); + expect(tried, [primary, backup, primary, backup]); + expect(client.state.attempt, 4); + await client.disconnect(); + await tester.pump(const Duration(seconds: 10)); + expect(tried, hasLength(4)); + expect(client.state.kind, ConnectionKind.idle); + await tester.runAsync(client.dispose); + }, + ); + + for (final dispose in [false, true]) { + test( + '${dispose ? 'dispose' : 'disconnect'} cancels stalled open', + () async { + final channel = FakeChannel(); + var factories = 0; + final client = PipelinesClient( + channelFactory: (_) { + factories++; + return channel; + }, + connectionTimeout: const Duration(milliseconds: 10), + backoffBase: retry, + ); + final connecting = client.connectAny([primary, backup]); + await flush(); + if (dispose) { + await client.dispose(); + } else { + await client.disconnect(); + } + await connecting; + channel.readyCompleter.completeError(StateError('late failure')); + await Future.delayed(const Duration(milliseconds: 40)); + expect(factories, 1); + expect(client.state.kind, ConnectionKind.idle); + await client.dispose(); + }, + ); + } + + test( + 'replacement candidates cancel stale callbacks and retry target', + () async { + final stale = FakeChannel(); + final current = FakeChannel()..readyCompleter.complete(); + final next = FakeChannel()..readyCompleter.complete(); + final replacement = Uri.parse('wss://replacement.test/connect'); + final replacementBackup = Uri.parse('wss://replacement-lan.test/connect'); + final channels = [stale, current, next]; + final tried = []; + final client = PipelinesClient( + channelFactory: (uri) { + tried.add(uri); + return channels[tried.length - 1]; + }, + backoffBase: retry, + ); + final oldConnect = client.connectAny([primary, backup]); + await flush(); + stale.incoming.add(jsonEncode({'type': 'stale'})); + final frames = >[]; + client.frames.listen(frames.add); + await client.connectAny([replacement, replacementBackup]); + await oldConnect; + stale.readyCompleter.completeError(StateError('late ready error')); + await Future.delayed(const Duration(milliseconds: 30)); + expect(frames, isEmpty); + expect(client.endpoint, replacement); + expect(client.state.kind, ConnectionKind.connected); + final ready = connected(client); + current.incoming.addError(StateError('lost')); + await ready; + expect(tried, [primary, replacement, replacementBackup]); + expect(client.endpoints, [replacement, replacementBackup]); + await client.dispose(); + }, + ); + + test('failure never replays outstanding or queued mutations', () async { + final first = FakeChannel()..readyCompleter.complete(); + final second = FakeChannel()..readyCompleter.complete(); + final client = PipelinesClient( + channelFactory: (uri) => uri == primary ? first : second, + backoffBase: retry, + ); + await client.connectAny([primary, backup]); + final outstanding = expectLater(client.call('start_run'), throwsStateError); + final queued = expectLater(client.call('delete_run'), throwsStateError); + await flush(); + expect(first.outgoing, hasLength(1)); + final ready = connected(client); + first.incoming.addError(StateError('network drop')); + await Future.wait([outstanding, queued, ready]); + expect(second.outgoing, isEmpty); + expect(first.outgoing, hasLength(1)); + await client.dispose(); + }); + + test( + 'validation preserves live socket; normalized candidates are immutable', + () async { + final channel = FakeChannel()..readyCompleter.complete(); + var factories = 0; + final client = PipelinesClient( + channelFactory: (_) { + factories++; + return channel; + }, + ); + final supplied = [Uri.parse('ws://TAILSCALE.test/'), primary, backup]; + await client.connectAny(supplied); + supplied.clear(); + expect(client.endpoints, [primary, backup]); + expect(() => client.endpoints.add(primary), throwsUnsupportedError); + await expectLater(client.connectAny([]), throwsArgumentError); + await expectLater( + client.connectAny([backup, Uri.parse('https://invalid.test')]), + throwsFormatException, + ); + expect(client.endpoints, [primary, backup]); + expect(client.endpoint, primary); + expect(client.state.kind, ConnectionKind.connected); + expect(factories, 1); + client.sendOnly('ping'); + expect(channel.outgoing, hasLength(1)); + await client.dispose(); + }, + ); +}