Add ordered websocket address fallback
This commit is contained in:
@@ -17,18 +17,22 @@ class PipelinesClient implements ApiGateway {
|
||||
PipelinesClient({
|
||||
ChannelFactory? channelFactory,
|
||||
this.requestTimeout = const Duration(seconds: 30),
|
||||
this.connectionTimeout = const Duration(seconds: 5),
|
||||
this.backoffBase = const Duration(seconds: 1),
|
||||
this.backoffMax = const Duration(seconds: 30),
|
||||
}) : _factory = channelFactory ?? connectPipelinesChannel;
|
||||
|
||||
final ChannelFactory _factory;
|
||||
final Duration requestTimeout, backoffBase, backoffMax;
|
||||
final Duration requestTimeout, connectionTimeout, backoffBase, backoffMax;
|
||||
final _frames = StreamController<JsonMap>.broadcast();
|
||||
final _states = StreamController<ConnectionState>.broadcast();
|
||||
final Map<String, Completer<JsonMap>> _pending = {};
|
||||
WebSocketChannel? _channel;
|
||||
StreamSubscription<dynamic>? _subscription;
|
||||
Uri? _endpoint;
|
||||
List<Uri> _endpoints = const [];
|
||||
int _endpointIndex = 0;
|
||||
Completer<void>? _openingCancelled;
|
||||
Timer? _reconnect;
|
||||
int _generation = 0, _sequence = 0, _attempt = 0;
|
||||
bool _manual = true, _disposed = false;
|
||||
@@ -40,13 +44,28 @@ class PipelinesClient implements ApiGateway {
|
||||
Stream<ConnectionState> get states => _states.stream;
|
||||
ConnectionState get state => _state;
|
||||
|
||||
/// Last validated endpoint, including changes made through native Settings.
|
||||
/// Current attempted or connected address.
|
||||
Uri? get endpoint => _endpoint;
|
||||
|
||||
Future<void> connect(Uri uri) async {
|
||||
/// Immutable, normalized addresses for this logical server, in priority order.
|
||||
List<Uri> get endpoints => _endpoints;
|
||||
|
||||
Future<void> connect(Uri uri) => connectAny([uri]);
|
||||
|
||||
/// Starts at the primary address; automatic retries rotate only within [uris].
|
||||
Future<void> connectAny(List<Uri> uris) async {
|
||||
if (_disposed) throw StateError('Client has been disposed');
|
||||
final endpoint = ServerUrl.validate(uri.toString());
|
||||
_endpoint = Uri.parse(endpoint.wsUrl);
|
||||
if (uris.isEmpty) {
|
||||
throw ArgumentError.value(uris, 'uris', 'Must not be empty');
|
||||
}
|
||||
// Validate the entire replacement before changing a healthy connection.
|
||||
final candidates = uris
|
||||
.map((uri) => Uri.parse(ServerUrl.validate(uri.toString()).wsUrl))
|
||||
.toSet()
|
||||
.toList(growable: false);
|
||||
_endpoints = List.unmodifiable(candidates);
|
||||
_endpointIndex = 0;
|
||||
_endpoint = _endpoints.first;
|
||||
_manual = false;
|
||||
_attempt = 0;
|
||||
_reconnect?.cancel();
|
||||
@@ -57,6 +76,7 @@ class PipelinesClient implements ApiGateway {
|
||||
final endpoint = _endpoint;
|
||||
if (endpoint == null || _disposed) return;
|
||||
final generation = ++_generation;
|
||||
_cancelOpening();
|
||||
_setState(ConnectionState(ConnectionKind.connecting, attempt: ++_attempt));
|
||||
final old = _channel;
|
||||
_channel = null;
|
||||
@@ -66,6 +86,8 @@ class PipelinesClient implements ApiGateway {
|
||||
_subscription = null;
|
||||
await subscription?.cancel();
|
||||
if (generation != _generation || _manual || _disposed) return;
|
||||
final cancelled = Completer<void>();
|
||||
_openingCancelled = cancelled;
|
||||
try {
|
||||
final channel = _factory(endpoint);
|
||||
_channel = channel;
|
||||
@@ -80,15 +102,26 @@ class PipelinesClient implements ApiGateway {
|
||||
if (generation == _generation) _closed('Connection closed');
|
||||
},
|
||||
);
|
||||
await channel.ready;
|
||||
await Future.any([
|
||||
channel.ready,
|
||||
cancelled.future,
|
||||
]).timeout(connectionTimeout);
|
||||
if (generation != _generation) return;
|
||||
_attempt = 0;
|
||||
_setState(const ConnectionState(ConnectionKind.connected));
|
||||
} catch (error) {
|
||||
if (generation == _generation) _closed(error.toString());
|
||||
} finally {
|
||||
if (identical(_openingCancelled, cancelled)) _openingCancelled = null;
|
||||
}
|
||||
}
|
||||
|
||||
void _cancelOpening() {
|
||||
final cancelled = _openingCancelled;
|
||||
_openingCancelled = null;
|
||||
if (cancelled != null && !cancelled.isCompleted) cancelled.complete();
|
||||
}
|
||||
|
||||
void _setState(ConnectionState value) {
|
||||
_state = value;
|
||||
if (!_states.isClosed) _states.add(value);
|
||||
@@ -196,6 +229,7 @@ class PipelinesClient implements ApiGateway {
|
||||
|
||||
void _closed(String reason) {
|
||||
_generation++;
|
||||
_cancelOpening();
|
||||
final channel = _channel;
|
||||
final subscription = _subscription;
|
||||
_channel = null;
|
||||
@@ -217,17 +251,20 @@ class PipelinesClient implements ApiGateway {
|
||||
);
|
||||
final shift = (_attempt - 1).clamp(0, 6);
|
||||
final delay = backoffBase * (1 << shift);
|
||||
_endpointIndex = (_endpointIndex + 1) % _endpoints.length;
|
||||
_reconnect?.cancel();
|
||||
_reconnect = Timer(
|
||||
delay > backoffMax ? backoffMax : delay,
|
||||
() => unawaited(_open()),
|
||||
);
|
||||
_reconnect = Timer(delay > backoffMax ? backoffMax : delay, () {
|
||||
if (_manual || _disposed) return;
|
||||
_endpoint = _endpoints[_endpointIndex];
|
||||
unawaited(_open());
|
||||
});
|
||||
}
|
||||
|
||||
Future<void> disconnect() async {
|
||||
_manual = true;
|
||||
_reconnect?.cancel();
|
||||
final generation = ++_generation;
|
||||
_cancelOpening();
|
||||
final channel = _channel;
|
||||
_channel = null;
|
||||
final subscription = _subscription;
|
||||
|
||||
@@ -0,0 +1,262 @@
|
||||
import 'dart:convert';
|
||||
|
||||
import 'package:flutter_test/flutter_test.dart';
|
||||
import 'package:pipelines_app/core/transport/pipelines_client.dart';
|
||||
|
||||
import 'pipelines_client_test.dart' show FakeChannel;
|
||||
|
||||
final primary = Uri.parse('ws://tailscale.test/connect');
|
||||
final backup = Uri.parse('ws://lan.test/connect');
|
||||
const retry = Duration(milliseconds: 5);
|
||||
|
||||
Future<void> connected(PipelinesClient client) => client.states
|
||||
.firstWhere((state) => state.kind == ConnectionKind.connected)
|
||||
.timeout(const Duration(seconds: 2));
|
||||
Future<void> flush() => Future<void>.delayed(Duration.zero);
|
||||
|
||||
void main() {
|
||||
test('failed primary uses only configured backup and stays there', () async {
|
||||
final tried = <Uri>[];
|
||||
final channel = FakeChannel()..readyCompleter.complete();
|
||||
final client = PipelinesClient(
|
||||
channelFactory: (uri) {
|
||||
tried.add(uri);
|
||||
if (uri == primary) throw StateError('unreachable');
|
||||
return channel;
|
||||
},
|
||||
backoffBase: retry,
|
||||
);
|
||||
final ready = connected(client);
|
||||
await client.connectAny([primary, backup]);
|
||||
expect(client.endpoint, primary);
|
||||
expect(client.state.retrying, isTrue);
|
||||
await ready;
|
||||
expect(tried, [primary, backup]);
|
||||
expect(client.endpoint, backup);
|
||||
await Future<void>.delayed(const Duration(milliseconds: 40));
|
||||
expect(tried, [primary, backup]);
|
||||
await client.dispose();
|
||||
});
|
||||
|
||||
test(
|
||||
'stalled readiness times out; late success cannot replace backup',
|
||||
() async {
|
||||
final first = FakeChannel();
|
||||
final second = FakeChannel()..readyCompleter.complete();
|
||||
final tried = <Uri>[];
|
||||
final client = PipelinesClient(
|
||||
channelFactory: (uri) {
|
||||
tried.add(uri);
|
||||
return uri == primary ? first : second;
|
||||
},
|
||||
connectionTimeout: const Duration(milliseconds: 10),
|
||||
backoffBase: retry,
|
||||
);
|
||||
final ready = connected(client);
|
||||
await client.connectAny([primary, backup]);
|
||||
expect(client.state.reason, contains('TimeoutException'));
|
||||
await ready;
|
||||
first.readyCompleter.complete();
|
||||
await flush();
|
||||
expect(client.endpoint, backup);
|
||||
expect(client.state.kind, ConnectionKind.connected);
|
||||
expect(tried, [primary, backup]);
|
||||
await client.dispose();
|
||||
},
|
||||
);
|
||||
|
||||
test('failed readiness advances to backup', () async {
|
||||
final first = FakeChannel();
|
||||
final second = FakeChannel()..readyCompleter.complete();
|
||||
final client = PipelinesClient(
|
||||
channelFactory: (uri) => uri == primary ? first : second,
|
||||
backoffBase: retry,
|
||||
);
|
||||
final ready = connected(client);
|
||||
final connecting = client.connectAny([primary, backup]);
|
||||
await flush();
|
||||
first.readyCompleter.completeError(StateError('handshake refused'));
|
||||
await connecting;
|
||||
await ready;
|
||||
expect(client.endpoint, backup);
|
||||
await client.dispose();
|
||||
});
|
||||
|
||||
test('closed primary advances; explicit connect restarts primary', () async {
|
||||
final channels = List.generate(
|
||||
3,
|
||||
(_) => FakeChannel()..readyCompleter.complete(),
|
||||
);
|
||||
final tried = <Uri>[];
|
||||
final client = PipelinesClient(
|
||||
channelFactory: (uri) {
|
||||
tried.add(uri);
|
||||
return channels[tried.length - 1];
|
||||
},
|
||||
backoffBase: retry,
|
||||
);
|
||||
await client.connectAny([primary, backup]);
|
||||
final ready = connected(client);
|
||||
await channels.first.incoming.close();
|
||||
await ready;
|
||||
expect(client.endpoint, backup);
|
||||
await client.connectAny(client.endpoints);
|
||||
expect(tried, [primary, backup, primary]);
|
||||
expect(client.endpoint, primary);
|
||||
await client.dispose();
|
||||
});
|
||||
|
||||
testWidgets(
|
||||
'all down wraps with exponential capped backoff; manual stop cancels',
|
||||
(tester) async {
|
||||
final tried = <Uri>[];
|
||||
final client = PipelinesClient(
|
||||
channelFactory: (uri) {
|
||||
tried.add(uri);
|
||||
throw StateError('offline');
|
||||
},
|
||||
backoffBase: const Duration(milliseconds: 20),
|
||||
backoffMax: const Duration(milliseconds: 50),
|
||||
);
|
||||
await client.connectAny([primary, backup]);
|
||||
await tester.pump(const Duration(milliseconds: 19));
|
||||
expect(tried, [primary]);
|
||||
await tester.pump(const Duration(milliseconds: 1));
|
||||
expect(tried, [primary, backup]);
|
||||
await tester.pump(const Duration(milliseconds: 39));
|
||||
expect(tried, hasLength(2));
|
||||
await tester.pump(const Duration(milliseconds: 1));
|
||||
expect(tried, [primary, backup, primary]);
|
||||
await tester.pump(const Duration(milliseconds: 49));
|
||||
expect(tried, hasLength(3));
|
||||
await tester.pump(const Duration(milliseconds: 1));
|
||||
expect(tried, [primary, backup, primary, backup]);
|
||||
expect(client.state.attempt, 4);
|
||||
await client.disconnect();
|
||||
await tester.pump(const Duration(seconds: 10));
|
||||
expect(tried, hasLength(4));
|
||||
expect(client.state.kind, ConnectionKind.idle);
|
||||
await tester.runAsync(client.dispose);
|
||||
},
|
||||
);
|
||||
|
||||
for (final dispose in [false, true]) {
|
||||
test(
|
||||
'${dispose ? 'dispose' : 'disconnect'} cancels stalled open',
|
||||
() async {
|
||||
final channel = FakeChannel();
|
||||
var factories = 0;
|
||||
final client = PipelinesClient(
|
||||
channelFactory: (_) {
|
||||
factories++;
|
||||
return channel;
|
||||
},
|
||||
connectionTimeout: const Duration(milliseconds: 10),
|
||||
backoffBase: retry,
|
||||
);
|
||||
final connecting = client.connectAny([primary, backup]);
|
||||
await flush();
|
||||
if (dispose) {
|
||||
await client.dispose();
|
||||
} else {
|
||||
await client.disconnect();
|
||||
}
|
||||
await connecting;
|
||||
channel.readyCompleter.completeError(StateError('late failure'));
|
||||
await Future<void>.delayed(const Duration(milliseconds: 40));
|
||||
expect(factories, 1);
|
||||
expect(client.state.kind, ConnectionKind.idle);
|
||||
await client.dispose();
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
test(
|
||||
'replacement candidates cancel stale callbacks and retry target',
|
||||
() async {
|
||||
final stale = FakeChannel();
|
||||
final current = FakeChannel()..readyCompleter.complete();
|
||||
final next = FakeChannel()..readyCompleter.complete();
|
||||
final replacement = Uri.parse('wss://replacement.test/connect');
|
||||
final replacementBackup = Uri.parse('wss://replacement-lan.test/connect');
|
||||
final channels = [stale, current, next];
|
||||
final tried = <Uri>[];
|
||||
final client = PipelinesClient(
|
||||
channelFactory: (uri) {
|
||||
tried.add(uri);
|
||||
return channels[tried.length - 1];
|
||||
},
|
||||
backoffBase: retry,
|
||||
);
|
||||
final oldConnect = client.connectAny([primary, backup]);
|
||||
await flush();
|
||||
stale.incoming.add(jsonEncode({'type': 'stale'}));
|
||||
final frames = <Map<String, dynamic>>[];
|
||||
client.frames.listen(frames.add);
|
||||
await client.connectAny([replacement, replacementBackup]);
|
||||
await oldConnect;
|
||||
stale.readyCompleter.completeError(StateError('late ready error'));
|
||||
await Future<void>.delayed(const Duration(milliseconds: 30));
|
||||
expect(frames, isEmpty);
|
||||
expect(client.endpoint, replacement);
|
||||
expect(client.state.kind, ConnectionKind.connected);
|
||||
final ready = connected(client);
|
||||
current.incoming.addError(StateError('lost'));
|
||||
await ready;
|
||||
expect(tried, [primary, replacement, replacementBackup]);
|
||||
expect(client.endpoints, [replacement, replacementBackup]);
|
||||
await client.dispose();
|
||||
},
|
||||
);
|
||||
|
||||
test('failure never replays outstanding or queued mutations', () async {
|
||||
final first = FakeChannel()..readyCompleter.complete();
|
||||
final second = FakeChannel()..readyCompleter.complete();
|
||||
final client = PipelinesClient(
|
||||
channelFactory: (uri) => uri == primary ? first : second,
|
||||
backoffBase: retry,
|
||||
);
|
||||
await client.connectAny([primary, backup]);
|
||||
final outstanding = expectLater(client.call('start_run'), throwsStateError);
|
||||
final queued = expectLater(client.call('delete_run'), throwsStateError);
|
||||
await flush();
|
||||
expect(first.outgoing, hasLength(1));
|
||||
final ready = connected(client);
|
||||
first.incoming.addError(StateError('network drop'));
|
||||
await Future.wait([outstanding, queued, ready]);
|
||||
expect(second.outgoing, isEmpty);
|
||||
expect(first.outgoing, hasLength(1));
|
||||
await client.dispose();
|
||||
});
|
||||
|
||||
test(
|
||||
'validation preserves live socket; normalized candidates are immutable',
|
||||
() async {
|
||||
final channel = FakeChannel()..readyCompleter.complete();
|
||||
var factories = 0;
|
||||
final client = PipelinesClient(
|
||||
channelFactory: (_) {
|
||||
factories++;
|
||||
return channel;
|
||||
},
|
||||
);
|
||||
final supplied = [Uri.parse('ws://TAILSCALE.test/'), primary, backup];
|
||||
await client.connectAny(supplied);
|
||||
supplied.clear();
|
||||
expect(client.endpoints, [primary, backup]);
|
||||
expect(() => client.endpoints.add(primary), throwsUnsupportedError);
|
||||
await expectLater(client.connectAny([]), throwsArgumentError);
|
||||
await expectLater(
|
||||
client.connectAny([backup, Uri.parse('https://invalid.test')]),
|
||||
throwsFormatException,
|
||||
);
|
||||
expect(client.endpoints, [primary, backup]);
|
||||
expect(client.endpoint, primary);
|
||||
expect(client.state.kind, ConnectionKind.connected);
|
||||
expect(factories, 1);
|
||||
client.sendOnly('ping');
|
||||
expect(channel.outgoing, hasLength(1));
|
||||
await client.dispose();
|
||||
},
|
||||
);
|
||||
}
|
||||
Reference in New Issue
Block a user