Add ordered websocket address fallback

This commit is contained in:
2026-10-24 02:31:21 +01:00
parent 54f74ade33
commit b2e5606c84
2 changed files with 309 additions and 10 deletions
+47 -10
View File
@@ -17,18 +17,22 @@ class PipelinesClient implements ApiGateway {
PipelinesClient({
ChannelFactory? channelFactory,
this.requestTimeout = const Duration(seconds: 30),
this.connectionTimeout = const Duration(seconds: 5),
this.backoffBase = const Duration(seconds: 1),
this.backoffMax = const Duration(seconds: 30),
}) : _factory = channelFactory ?? connectPipelinesChannel;
final ChannelFactory _factory;
final Duration requestTimeout, backoffBase, backoffMax;
final Duration requestTimeout, connectionTimeout, backoffBase, backoffMax;
final _frames = StreamController<JsonMap>.broadcast();
final _states = StreamController<ConnectionState>.broadcast();
final Map<String, Completer<JsonMap>> _pending = {};
WebSocketChannel? _channel;
StreamSubscription<dynamic>? _subscription;
Uri? _endpoint;
List<Uri> _endpoints = const [];
int _endpointIndex = 0;
Completer<void>? _openingCancelled;
Timer? _reconnect;
int _generation = 0, _sequence = 0, _attempt = 0;
bool _manual = true, _disposed = false;
@@ -40,13 +44,28 @@ class PipelinesClient implements ApiGateway {
Stream<ConnectionState> get states => _states.stream;
ConnectionState get state => _state;
/// Last validated endpoint, including changes made through native Settings.
/// Current attempted or connected address.
Uri? get endpoint => _endpoint;
Future<void> connect(Uri uri) async {
/// Immutable, normalized addresses for this logical server, in priority order.
List<Uri> get endpoints => _endpoints;
Future<void> connect(Uri uri) => connectAny([uri]);
/// Starts at the primary address; automatic retries rotate only within [uris].
Future<void> connectAny(List<Uri> uris) async {
if (_disposed) throw StateError('Client has been disposed');
final endpoint = ServerUrl.validate(uri.toString());
_endpoint = Uri.parse(endpoint.wsUrl);
if (uris.isEmpty) {
throw ArgumentError.value(uris, 'uris', 'Must not be empty');
}
// Validate the entire replacement before changing a healthy connection.
final candidates = uris
.map((uri) => Uri.parse(ServerUrl.validate(uri.toString()).wsUrl))
.toSet()
.toList(growable: false);
_endpoints = List.unmodifiable(candidates);
_endpointIndex = 0;
_endpoint = _endpoints.first;
_manual = false;
_attempt = 0;
_reconnect?.cancel();
@@ -57,6 +76,7 @@ class PipelinesClient implements ApiGateway {
final endpoint = _endpoint;
if (endpoint == null || _disposed) return;
final generation = ++_generation;
_cancelOpening();
_setState(ConnectionState(ConnectionKind.connecting, attempt: ++_attempt));
final old = _channel;
_channel = null;
@@ -66,6 +86,8 @@ class PipelinesClient implements ApiGateway {
_subscription = null;
await subscription?.cancel();
if (generation != _generation || _manual || _disposed) return;
final cancelled = Completer<void>();
_openingCancelled = cancelled;
try {
final channel = _factory(endpoint);
_channel = channel;
@@ -80,15 +102,26 @@ class PipelinesClient implements ApiGateway {
if (generation == _generation) _closed('Connection closed');
},
);
await channel.ready;
await Future.any([
channel.ready,
cancelled.future,
]).timeout(connectionTimeout);
if (generation != _generation) return;
_attempt = 0;
_setState(const ConnectionState(ConnectionKind.connected));
} catch (error) {
if (generation == _generation) _closed(error.toString());
} finally {
if (identical(_openingCancelled, cancelled)) _openingCancelled = null;
}
}
void _cancelOpening() {
final cancelled = _openingCancelled;
_openingCancelled = null;
if (cancelled != null && !cancelled.isCompleted) cancelled.complete();
}
void _setState(ConnectionState value) {
_state = value;
if (!_states.isClosed) _states.add(value);
@@ -196,6 +229,7 @@ class PipelinesClient implements ApiGateway {
void _closed(String reason) {
_generation++;
_cancelOpening();
final channel = _channel;
final subscription = _subscription;
_channel = null;
@@ -217,17 +251,20 @@ class PipelinesClient implements ApiGateway {
);
final shift = (_attempt - 1).clamp(0, 6);
final delay = backoffBase * (1 << shift);
_endpointIndex = (_endpointIndex + 1) % _endpoints.length;
_reconnect?.cancel();
_reconnect = Timer(
delay > backoffMax ? backoffMax : delay,
() => unawaited(_open()),
);
_reconnect = Timer(delay > backoffMax ? backoffMax : delay, () {
if (_manual || _disposed) return;
_endpoint = _endpoints[_endpointIndex];
unawaited(_open());
});
}
Future<void> disconnect() async {
_manual = true;
_reconnect?.cancel();
final generation = ++_generation;
_cancelOpening();
final channel = _channel;
_channel = null;
final subscription = _subscription;
@@ -0,0 +1,262 @@
import 'dart:convert';
import 'package:flutter_test/flutter_test.dart';
import 'package:pipelines_app/core/transport/pipelines_client.dart';
import 'pipelines_client_test.dart' show FakeChannel;
final primary = Uri.parse('ws://tailscale.test/connect');
final backup = Uri.parse('ws://lan.test/connect');
const retry = Duration(milliseconds: 5);
Future<void> connected(PipelinesClient client) => client.states
.firstWhere((state) => state.kind == ConnectionKind.connected)
.timeout(const Duration(seconds: 2));
Future<void> flush() => Future<void>.delayed(Duration.zero);
void main() {
test('failed primary uses only configured backup and stays there', () async {
final tried = <Uri>[];
final channel = FakeChannel()..readyCompleter.complete();
final client = PipelinesClient(
channelFactory: (uri) {
tried.add(uri);
if (uri == primary) throw StateError('unreachable');
return channel;
},
backoffBase: retry,
);
final ready = connected(client);
await client.connectAny([primary, backup]);
expect(client.endpoint, primary);
expect(client.state.retrying, isTrue);
await ready;
expect(tried, [primary, backup]);
expect(client.endpoint, backup);
await Future<void>.delayed(const Duration(milliseconds: 40));
expect(tried, [primary, backup]);
await client.dispose();
});
test(
'stalled readiness times out; late success cannot replace backup',
() async {
final first = FakeChannel();
final second = FakeChannel()..readyCompleter.complete();
final tried = <Uri>[];
final client = PipelinesClient(
channelFactory: (uri) {
tried.add(uri);
return uri == primary ? first : second;
},
connectionTimeout: const Duration(milliseconds: 10),
backoffBase: retry,
);
final ready = connected(client);
await client.connectAny([primary, backup]);
expect(client.state.reason, contains('TimeoutException'));
await ready;
first.readyCompleter.complete();
await flush();
expect(client.endpoint, backup);
expect(client.state.kind, ConnectionKind.connected);
expect(tried, [primary, backup]);
await client.dispose();
},
);
test('failed readiness advances to backup', () async {
final first = FakeChannel();
final second = FakeChannel()..readyCompleter.complete();
final client = PipelinesClient(
channelFactory: (uri) => uri == primary ? first : second,
backoffBase: retry,
);
final ready = connected(client);
final connecting = client.connectAny([primary, backup]);
await flush();
first.readyCompleter.completeError(StateError('handshake refused'));
await connecting;
await ready;
expect(client.endpoint, backup);
await client.dispose();
});
test('closed primary advances; explicit connect restarts primary', () async {
final channels = List.generate(
3,
(_) => FakeChannel()..readyCompleter.complete(),
);
final tried = <Uri>[];
final client = PipelinesClient(
channelFactory: (uri) {
tried.add(uri);
return channels[tried.length - 1];
},
backoffBase: retry,
);
await client.connectAny([primary, backup]);
final ready = connected(client);
await channels.first.incoming.close();
await ready;
expect(client.endpoint, backup);
await client.connectAny(client.endpoints);
expect(tried, [primary, backup, primary]);
expect(client.endpoint, primary);
await client.dispose();
});
testWidgets(
'all down wraps with exponential capped backoff; manual stop cancels',
(tester) async {
final tried = <Uri>[];
final client = PipelinesClient(
channelFactory: (uri) {
tried.add(uri);
throw StateError('offline');
},
backoffBase: const Duration(milliseconds: 20),
backoffMax: const Duration(milliseconds: 50),
);
await client.connectAny([primary, backup]);
await tester.pump(const Duration(milliseconds: 19));
expect(tried, [primary]);
await tester.pump(const Duration(milliseconds: 1));
expect(tried, [primary, backup]);
await tester.pump(const Duration(milliseconds: 39));
expect(tried, hasLength(2));
await tester.pump(const Duration(milliseconds: 1));
expect(tried, [primary, backup, primary]);
await tester.pump(const Duration(milliseconds: 49));
expect(tried, hasLength(3));
await tester.pump(const Duration(milliseconds: 1));
expect(tried, [primary, backup, primary, backup]);
expect(client.state.attempt, 4);
await client.disconnect();
await tester.pump(const Duration(seconds: 10));
expect(tried, hasLength(4));
expect(client.state.kind, ConnectionKind.idle);
await tester.runAsync(client.dispose);
},
);
for (final dispose in [false, true]) {
test(
'${dispose ? 'dispose' : 'disconnect'} cancels stalled open',
() async {
final channel = FakeChannel();
var factories = 0;
final client = PipelinesClient(
channelFactory: (_) {
factories++;
return channel;
},
connectionTimeout: const Duration(milliseconds: 10),
backoffBase: retry,
);
final connecting = client.connectAny([primary, backup]);
await flush();
if (dispose) {
await client.dispose();
} else {
await client.disconnect();
}
await connecting;
channel.readyCompleter.completeError(StateError('late failure'));
await Future<void>.delayed(const Duration(milliseconds: 40));
expect(factories, 1);
expect(client.state.kind, ConnectionKind.idle);
await client.dispose();
},
);
}
test(
'replacement candidates cancel stale callbacks and retry target',
() async {
final stale = FakeChannel();
final current = FakeChannel()..readyCompleter.complete();
final next = FakeChannel()..readyCompleter.complete();
final replacement = Uri.parse('wss://replacement.test/connect');
final replacementBackup = Uri.parse('wss://replacement-lan.test/connect');
final channels = [stale, current, next];
final tried = <Uri>[];
final client = PipelinesClient(
channelFactory: (uri) {
tried.add(uri);
return channels[tried.length - 1];
},
backoffBase: retry,
);
final oldConnect = client.connectAny([primary, backup]);
await flush();
stale.incoming.add(jsonEncode({'type': 'stale'}));
final frames = <Map<String, dynamic>>[];
client.frames.listen(frames.add);
await client.connectAny([replacement, replacementBackup]);
await oldConnect;
stale.readyCompleter.completeError(StateError('late ready error'));
await Future<void>.delayed(const Duration(milliseconds: 30));
expect(frames, isEmpty);
expect(client.endpoint, replacement);
expect(client.state.kind, ConnectionKind.connected);
final ready = connected(client);
current.incoming.addError(StateError('lost'));
await ready;
expect(tried, [primary, replacement, replacementBackup]);
expect(client.endpoints, [replacement, replacementBackup]);
await client.dispose();
},
);
test('failure never replays outstanding or queued mutations', () async {
final first = FakeChannel()..readyCompleter.complete();
final second = FakeChannel()..readyCompleter.complete();
final client = PipelinesClient(
channelFactory: (uri) => uri == primary ? first : second,
backoffBase: retry,
);
await client.connectAny([primary, backup]);
final outstanding = expectLater(client.call('start_run'), throwsStateError);
final queued = expectLater(client.call('delete_run'), throwsStateError);
await flush();
expect(first.outgoing, hasLength(1));
final ready = connected(client);
first.incoming.addError(StateError('network drop'));
await Future.wait([outstanding, queued, ready]);
expect(second.outgoing, isEmpty);
expect(first.outgoing, hasLength(1));
await client.dispose();
});
test(
'validation preserves live socket; normalized candidates are immutable',
() async {
final channel = FakeChannel()..readyCompleter.complete();
var factories = 0;
final client = PipelinesClient(
channelFactory: (_) {
factories++;
return channel;
},
);
final supplied = [Uri.parse('ws://TAILSCALE.test/'), primary, backup];
await client.connectAny(supplied);
supplied.clear();
expect(client.endpoints, [primary, backup]);
expect(() => client.endpoints.add(primary), throwsUnsupportedError);
await expectLater(client.connectAny([]), throwsArgumentError);
await expectLater(
client.connectAny([backup, Uri.parse('https://invalid.test')]),
throwsFormatException,
);
expect(client.endpoints, [primary, backup]);
expect(client.endpoint, primary);
expect(client.state.kind, ConnectionKind.connected);
expect(factories, 1);
client.sendOnly('ping');
expect(channel.outgoing, hasLength(1));
await client.dispose();
},
);
}