Add ordered websocket address fallback
This commit is contained in:
@@ -17,18 +17,22 @@ class PipelinesClient implements ApiGateway {
|
|||||||
PipelinesClient({
|
PipelinesClient({
|
||||||
ChannelFactory? channelFactory,
|
ChannelFactory? channelFactory,
|
||||||
this.requestTimeout = const Duration(seconds: 30),
|
this.requestTimeout = const Duration(seconds: 30),
|
||||||
|
this.connectionTimeout = const Duration(seconds: 5),
|
||||||
this.backoffBase = const Duration(seconds: 1),
|
this.backoffBase = const Duration(seconds: 1),
|
||||||
this.backoffMax = const Duration(seconds: 30),
|
this.backoffMax = const Duration(seconds: 30),
|
||||||
}) : _factory = channelFactory ?? connectPipelinesChannel;
|
}) : _factory = channelFactory ?? connectPipelinesChannel;
|
||||||
|
|
||||||
final ChannelFactory _factory;
|
final ChannelFactory _factory;
|
||||||
final Duration requestTimeout, backoffBase, backoffMax;
|
final Duration requestTimeout, connectionTimeout, backoffBase, backoffMax;
|
||||||
final _frames = StreamController<JsonMap>.broadcast();
|
final _frames = StreamController<JsonMap>.broadcast();
|
||||||
final _states = StreamController<ConnectionState>.broadcast();
|
final _states = StreamController<ConnectionState>.broadcast();
|
||||||
final Map<String, Completer<JsonMap>> _pending = {};
|
final Map<String, Completer<JsonMap>> _pending = {};
|
||||||
WebSocketChannel? _channel;
|
WebSocketChannel? _channel;
|
||||||
StreamSubscription<dynamic>? _subscription;
|
StreamSubscription<dynamic>? _subscription;
|
||||||
Uri? _endpoint;
|
Uri? _endpoint;
|
||||||
|
List<Uri> _endpoints = const [];
|
||||||
|
int _endpointIndex = 0;
|
||||||
|
Completer<void>? _openingCancelled;
|
||||||
Timer? _reconnect;
|
Timer? _reconnect;
|
||||||
int _generation = 0, _sequence = 0, _attempt = 0;
|
int _generation = 0, _sequence = 0, _attempt = 0;
|
||||||
bool _manual = true, _disposed = false;
|
bool _manual = true, _disposed = false;
|
||||||
@@ -40,13 +44,28 @@ class PipelinesClient implements ApiGateway {
|
|||||||
Stream<ConnectionState> get states => _states.stream;
|
Stream<ConnectionState> get states => _states.stream;
|
||||||
ConnectionState get state => _state;
|
ConnectionState get state => _state;
|
||||||
|
|
||||||
/// Last validated endpoint, including changes made through native Settings.
|
/// Current attempted or connected address.
|
||||||
Uri? get endpoint => _endpoint;
|
Uri? get endpoint => _endpoint;
|
||||||
|
|
||||||
Future<void> connect(Uri uri) async {
|
/// Immutable, normalized addresses for this logical server, in priority order.
|
||||||
|
List<Uri> get endpoints => _endpoints;
|
||||||
|
|
||||||
|
Future<void> connect(Uri uri) => connectAny([uri]);
|
||||||
|
|
||||||
|
/// Starts at the primary address; automatic retries rotate only within [uris].
|
||||||
|
Future<void> connectAny(List<Uri> uris) async {
|
||||||
if (_disposed) throw StateError('Client has been disposed');
|
if (_disposed) throw StateError('Client has been disposed');
|
||||||
final endpoint = ServerUrl.validate(uri.toString());
|
if (uris.isEmpty) {
|
||||||
_endpoint = Uri.parse(endpoint.wsUrl);
|
throw ArgumentError.value(uris, 'uris', 'Must not be empty');
|
||||||
|
}
|
||||||
|
// Validate the entire replacement before changing a healthy connection.
|
||||||
|
final candidates = uris
|
||||||
|
.map((uri) => Uri.parse(ServerUrl.validate(uri.toString()).wsUrl))
|
||||||
|
.toSet()
|
||||||
|
.toList(growable: false);
|
||||||
|
_endpoints = List.unmodifiable(candidates);
|
||||||
|
_endpointIndex = 0;
|
||||||
|
_endpoint = _endpoints.first;
|
||||||
_manual = false;
|
_manual = false;
|
||||||
_attempt = 0;
|
_attempt = 0;
|
||||||
_reconnect?.cancel();
|
_reconnect?.cancel();
|
||||||
@@ -57,6 +76,7 @@ class PipelinesClient implements ApiGateway {
|
|||||||
final endpoint = _endpoint;
|
final endpoint = _endpoint;
|
||||||
if (endpoint == null || _disposed) return;
|
if (endpoint == null || _disposed) return;
|
||||||
final generation = ++_generation;
|
final generation = ++_generation;
|
||||||
|
_cancelOpening();
|
||||||
_setState(ConnectionState(ConnectionKind.connecting, attempt: ++_attempt));
|
_setState(ConnectionState(ConnectionKind.connecting, attempt: ++_attempt));
|
||||||
final old = _channel;
|
final old = _channel;
|
||||||
_channel = null;
|
_channel = null;
|
||||||
@@ -66,6 +86,8 @@ class PipelinesClient implements ApiGateway {
|
|||||||
_subscription = null;
|
_subscription = null;
|
||||||
await subscription?.cancel();
|
await subscription?.cancel();
|
||||||
if (generation != _generation || _manual || _disposed) return;
|
if (generation != _generation || _manual || _disposed) return;
|
||||||
|
final cancelled = Completer<void>();
|
||||||
|
_openingCancelled = cancelled;
|
||||||
try {
|
try {
|
||||||
final channel = _factory(endpoint);
|
final channel = _factory(endpoint);
|
||||||
_channel = channel;
|
_channel = channel;
|
||||||
@@ -80,15 +102,26 @@ class PipelinesClient implements ApiGateway {
|
|||||||
if (generation == _generation) _closed('Connection closed');
|
if (generation == _generation) _closed('Connection closed');
|
||||||
},
|
},
|
||||||
);
|
);
|
||||||
await channel.ready;
|
await Future.any([
|
||||||
|
channel.ready,
|
||||||
|
cancelled.future,
|
||||||
|
]).timeout(connectionTimeout);
|
||||||
if (generation != _generation) return;
|
if (generation != _generation) return;
|
||||||
_attempt = 0;
|
_attempt = 0;
|
||||||
_setState(const ConnectionState(ConnectionKind.connected));
|
_setState(const ConnectionState(ConnectionKind.connected));
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
if (generation == _generation) _closed(error.toString());
|
if (generation == _generation) _closed(error.toString());
|
||||||
|
} finally {
|
||||||
|
if (identical(_openingCancelled, cancelled)) _openingCancelled = null;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
void _cancelOpening() {
|
||||||
|
final cancelled = _openingCancelled;
|
||||||
|
_openingCancelled = null;
|
||||||
|
if (cancelled != null && !cancelled.isCompleted) cancelled.complete();
|
||||||
|
}
|
||||||
|
|
||||||
void _setState(ConnectionState value) {
|
void _setState(ConnectionState value) {
|
||||||
_state = value;
|
_state = value;
|
||||||
if (!_states.isClosed) _states.add(value);
|
if (!_states.isClosed) _states.add(value);
|
||||||
@@ -196,6 +229,7 @@ class PipelinesClient implements ApiGateway {
|
|||||||
|
|
||||||
void _closed(String reason) {
|
void _closed(String reason) {
|
||||||
_generation++;
|
_generation++;
|
||||||
|
_cancelOpening();
|
||||||
final channel = _channel;
|
final channel = _channel;
|
||||||
final subscription = _subscription;
|
final subscription = _subscription;
|
||||||
_channel = null;
|
_channel = null;
|
||||||
@@ -217,17 +251,20 @@ class PipelinesClient implements ApiGateway {
|
|||||||
);
|
);
|
||||||
final shift = (_attempt - 1).clamp(0, 6);
|
final shift = (_attempt - 1).clamp(0, 6);
|
||||||
final delay = backoffBase * (1 << shift);
|
final delay = backoffBase * (1 << shift);
|
||||||
|
_endpointIndex = (_endpointIndex + 1) % _endpoints.length;
|
||||||
_reconnect?.cancel();
|
_reconnect?.cancel();
|
||||||
_reconnect = Timer(
|
_reconnect = Timer(delay > backoffMax ? backoffMax : delay, () {
|
||||||
delay > backoffMax ? backoffMax : delay,
|
if (_manual || _disposed) return;
|
||||||
() => unawaited(_open()),
|
_endpoint = _endpoints[_endpointIndex];
|
||||||
);
|
unawaited(_open());
|
||||||
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
Future<void> disconnect() async {
|
Future<void> disconnect() async {
|
||||||
_manual = true;
|
_manual = true;
|
||||||
_reconnect?.cancel();
|
_reconnect?.cancel();
|
||||||
final generation = ++_generation;
|
final generation = ++_generation;
|
||||||
|
_cancelOpening();
|
||||||
final channel = _channel;
|
final channel = _channel;
|
||||||
_channel = null;
|
_channel = null;
|
||||||
final subscription = _subscription;
|
final subscription = _subscription;
|
||||||
|
|||||||
@@ -0,0 +1,262 @@
|
|||||||
|
import 'dart:convert';
|
||||||
|
|
||||||
|
import 'package:flutter_test/flutter_test.dart';
|
||||||
|
import 'package:pipelines_app/core/transport/pipelines_client.dart';
|
||||||
|
|
||||||
|
import 'pipelines_client_test.dart' show FakeChannel;
|
||||||
|
|
||||||
|
final primary = Uri.parse('ws://tailscale.test/connect');
|
||||||
|
final backup = Uri.parse('ws://lan.test/connect');
|
||||||
|
const retry = Duration(milliseconds: 5);
|
||||||
|
|
||||||
|
Future<void> connected(PipelinesClient client) => client.states
|
||||||
|
.firstWhere((state) => state.kind == ConnectionKind.connected)
|
||||||
|
.timeout(const Duration(seconds: 2));
|
||||||
|
Future<void> flush() => Future<void>.delayed(Duration.zero);
|
||||||
|
|
||||||
|
void main() {
|
||||||
|
test('failed primary uses only configured backup and stays there', () async {
|
||||||
|
final tried = <Uri>[];
|
||||||
|
final channel = FakeChannel()..readyCompleter.complete();
|
||||||
|
final client = PipelinesClient(
|
||||||
|
channelFactory: (uri) {
|
||||||
|
tried.add(uri);
|
||||||
|
if (uri == primary) throw StateError('unreachable');
|
||||||
|
return channel;
|
||||||
|
},
|
||||||
|
backoffBase: retry,
|
||||||
|
);
|
||||||
|
final ready = connected(client);
|
||||||
|
await client.connectAny([primary, backup]);
|
||||||
|
expect(client.endpoint, primary);
|
||||||
|
expect(client.state.retrying, isTrue);
|
||||||
|
await ready;
|
||||||
|
expect(tried, [primary, backup]);
|
||||||
|
expect(client.endpoint, backup);
|
||||||
|
await Future<void>.delayed(const Duration(milliseconds: 40));
|
||||||
|
expect(tried, [primary, backup]);
|
||||||
|
await client.dispose();
|
||||||
|
});
|
||||||
|
|
||||||
|
test(
|
||||||
|
'stalled readiness times out; late success cannot replace backup',
|
||||||
|
() async {
|
||||||
|
final first = FakeChannel();
|
||||||
|
final second = FakeChannel()..readyCompleter.complete();
|
||||||
|
final tried = <Uri>[];
|
||||||
|
final client = PipelinesClient(
|
||||||
|
channelFactory: (uri) {
|
||||||
|
tried.add(uri);
|
||||||
|
return uri == primary ? first : second;
|
||||||
|
},
|
||||||
|
connectionTimeout: const Duration(milliseconds: 10),
|
||||||
|
backoffBase: retry,
|
||||||
|
);
|
||||||
|
final ready = connected(client);
|
||||||
|
await client.connectAny([primary, backup]);
|
||||||
|
expect(client.state.reason, contains('TimeoutException'));
|
||||||
|
await ready;
|
||||||
|
first.readyCompleter.complete();
|
||||||
|
await flush();
|
||||||
|
expect(client.endpoint, backup);
|
||||||
|
expect(client.state.kind, ConnectionKind.connected);
|
||||||
|
expect(tried, [primary, backup]);
|
||||||
|
await client.dispose();
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
|
test('failed readiness advances to backup', () async {
|
||||||
|
final first = FakeChannel();
|
||||||
|
final second = FakeChannel()..readyCompleter.complete();
|
||||||
|
final client = PipelinesClient(
|
||||||
|
channelFactory: (uri) => uri == primary ? first : second,
|
||||||
|
backoffBase: retry,
|
||||||
|
);
|
||||||
|
final ready = connected(client);
|
||||||
|
final connecting = client.connectAny([primary, backup]);
|
||||||
|
await flush();
|
||||||
|
first.readyCompleter.completeError(StateError('handshake refused'));
|
||||||
|
await connecting;
|
||||||
|
await ready;
|
||||||
|
expect(client.endpoint, backup);
|
||||||
|
await client.dispose();
|
||||||
|
});
|
||||||
|
|
||||||
|
test('closed primary advances; explicit connect restarts primary', () async {
|
||||||
|
final channels = List.generate(
|
||||||
|
3,
|
||||||
|
(_) => FakeChannel()..readyCompleter.complete(),
|
||||||
|
);
|
||||||
|
final tried = <Uri>[];
|
||||||
|
final client = PipelinesClient(
|
||||||
|
channelFactory: (uri) {
|
||||||
|
tried.add(uri);
|
||||||
|
return channels[tried.length - 1];
|
||||||
|
},
|
||||||
|
backoffBase: retry,
|
||||||
|
);
|
||||||
|
await client.connectAny([primary, backup]);
|
||||||
|
final ready = connected(client);
|
||||||
|
await channels.first.incoming.close();
|
||||||
|
await ready;
|
||||||
|
expect(client.endpoint, backup);
|
||||||
|
await client.connectAny(client.endpoints);
|
||||||
|
expect(tried, [primary, backup, primary]);
|
||||||
|
expect(client.endpoint, primary);
|
||||||
|
await client.dispose();
|
||||||
|
});
|
||||||
|
|
||||||
|
testWidgets(
|
||||||
|
'all down wraps with exponential capped backoff; manual stop cancels',
|
||||||
|
(tester) async {
|
||||||
|
final tried = <Uri>[];
|
||||||
|
final client = PipelinesClient(
|
||||||
|
channelFactory: (uri) {
|
||||||
|
tried.add(uri);
|
||||||
|
throw StateError('offline');
|
||||||
|
},
|
||||||
|
backoffBase: const Duration(milliseconds: 20),
|
||||||
|
backoffMax: const Duration(milliseconds: 50),
|
||||||
|
);
|
||||||
|
await client.connectAny([primary, backup]);
|
||||||
|
await tester.pump(const Duration(milliseconds: 19));
|
||||||
|
expect(tried, [primary]);
|
||||||
|
await tester.pump(const Duration(milliseconds: 1));
|
||||||
|
expect(tried, [primary, backup]);
|
||||||
|
await tester.pump(const Duration(milliseconds: 39));
|
||||||
|
expect(tried, hasLength(2));
|
||||||
|
await tester.pump(const Duration(milliseconds: 1));
|
||||||
|
expect(tried, [primary, backup, primary]);
|
||||||
|
await tester.pump(const Duration(milliseconds: 49));
|
||||||
|
expect(tried, hasLength(3));
|
||||||
|
await tester.pump(const Duration(milliseconds: 1));
|
||||||
|
expect(tried, [primary, backup, primary, backup]);
|
||||||
|
expect(client.state.attempt, 4);
|
||||||
|
await client.disconnect();
|
||||||
|
await tester.pump(const Duration(seconds: 10));
|
||||||
|
expect(tried, hasLength(4));
|
||||||
|
expect(client.state.kind, ConnectionKind.idle);
|
||||||
|
await tester.runAsync(client.dispose);
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
|
for (final dispose in [false, true]) {
|
||||||
|
test(
|
||||||
|
'${dispose ? 'dispose' : 'disconnect'} cancels stalled open',
|
||||||
|
() async {
|
||||||
|
final channel = FakeChannel();
|
||||||
|
var factories = 0;
|
||||||
|
final client = PipelinesClient(
|
||||||
|
channelFactory: (_) {
|
||||||
|
factories++;
|
||||||
|
return channel;
|
||||||
|
},
|
||||||
|
connectionTimeout: const Duration(milliseconds: 10),
|
||||||
|
backoffBase: retry,
|
||||||
|
);
|
||||||
|
final connecting = client.connectAny([primary, backup]);
|
||||||
|
await flush();
|
||||||
|
if (dispose) {
|
||||||
|
await client.dispose();
|
||||||
|
} else {
|
||||||
|
await client.disconnect();
|
||||||
|
}
|
||||||
|
await connecting;
|
||||||
|
channel.readyCompleter.completeError(StateError('late failure'));
|
||||||
|
await Future<void>.delayed(const Duration(milliseconds: 40));
|
||||||
|
expect(factories, 1);
|
||||||
|
expect(client.state.kind, ConnectionKind.idle);
|
||||||
|
await client.dispose();
|
||||||
|
},
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
test(
|
||||||
|
'replacement candidates cancel stale callbacks and retry target',
|
||||||
|
() async {
|
||||||
|
final stale = FakeChannel();
|
||||||
|
final current = FakeChannel()..readyCompleter.complete();
|
||||||
|
final next = FakeChannel()..readyCompleter.complete();
|
||||||
|
final replacement = Uri.parse('wss://replacement.test/connect');
|
||||||
|
final replacementBackup = Uri.parse('wss://replacement-lan.test/connect');
|
||||||
|
final channels = [stale, current, next];
|
||||||
|
final tried = <Uri>[];
|
||||||
|
final client = PipelinesClient(
|
||||||
|
channelFactory: (uri) {
|
||||||
|
tried.add(uri);
|
||||||
|
return channels[tried.length - 1];
|
||||||
|
},
|
||||||
|
backoffBase: retry,
|
||||||
|
);
|
||||||
|
final oldConnect = client.connectAny([primary, backup]);
|
||||||
|
await flush();
|
||||||
|
stale.incoming.add(jsonEncode({'type': 'stale'}));
|
||||||
|
final frames = <Map<String, dynamic>>[];
|
||||||
|
client.frames.listen(frames.add);
|
||||||
|
await client.connectAny([replacement, replacementBackup]);
|
||||||
|
await oldConnect;
|
||||||
|
stale.readyCompleter.completeError(StateError('late ready error'));
|
||||||
|
await Future<void>.delayed(const Duration(milliseconds: 30));
|
||||||
|
expect(frames, isEmpty);
|
||||||
|
expect(client.endpoint, replacement);
|
||||||
|
expect(client.state.kind, ConnectionKind.connected);
|
||||||
|
final ready = connected(client);
|
||||||
|
current.incoming.addError(StateError('lost'));
|
||||||
|
await ready;
|
||||||
|
expect(tried, [primary, replacement, replacementBackup]);
|
||||||
|
expect(client.endpoints, [replacement, replacementBackup]);
|
||||||
|
await client.dispose();
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
|
test('failure never replays outstanding or queued mutations', () async {
|
||||||
|
final first = FakeChannel()..readyCompleter.complete();
|
||||||
|
final second = FakeChannel()..readyCompleter.complete();
|
||||||
|
final client = PipelinesClient(
|
||||||
|
channelFactory: (uri) => uri == primary ? first : second,
|
||||||
|
backoffBase: retry,
|
||||||
|
);
|
||||||
|
await client.connectAny([primary, backup]);
|
||||||
|
final outstanding = expectLater(client.call('start_run'), throwsStateError);
|
||||||
|
final queued = expectLater(client.call('delete_run'), throwsStateError);
|
||||||
|
await flush();
|
||||||
|
expect(first.outgoing, hasLength(1));
|
||||||
|
final ready = connected(client);
|
||||||
|
first.incoming.addError(StateError('network drop'));
|
||||||
|
await Future.wait([outstanding, queued, ready]);
|
||||||
|
expect(second.outgoing, isEmpty);
|
||||||
|
expect(first.outgoing, hasLength(1));
|
||||||
|
await client.dispose();
|
||||||
|
});
|
||||||
|
|
||||||
|
test(
|
||||||
|
'validation preserves live socket; normalized candidates are immutable',
|
||||||
|
() async {
|
||||||
|
final channel = FakeChannel()..readyCompleter.complete();
|
||||||
|
var factories = 0;
|
||||||
|
final client = PipelinesClient(
|
||||||
|
channelFactory: (_) {
|
||||||
|
factories++;
|
||||||
|
return channel;
|
||||||
|
},
|
||||||
|
);
|
||||||
|
final supplied = [Uri.parse('ws://TAILSCALE.test/'), primary, backup];
|
||||||
|
await client.connectAny(supplied);
|
||||||
|
supplied.clear();
|
||||||
|
expect(client.endpoints, [primary, backup]);
|
||||||
|
expect(() => client.endpoints.add(primary), throwsUnsupportedError);
|
||||||
|
await expectLater(client.connectAny([]), throwsArgumentError);
|
||||||
|
await expectLater(
|
||||||
|
client.connectAny([backup, Uri.parse('https://invalid.test')]),
|
||||||
|
throwsFormatException,
|
||||||
|
);
|
||||||
|
expect(client.endpoints, [primary, backup]);
|
||||||
|
expect(client.endpoint, primary);
|
||||||
|
expect(client.state.kind, ConnectionKind.connected);
|
||||||
|
expect(factories, 1);
|
||||||
|
client.sendOnly('ping');
|
||||||
|
expect(channel.outgoing, hasLength(1));
|
||||||
|
await client.dispose();
|
||||||
|
},
|
||||||
|
);
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user