Merge branch 'worktree-agent-a5eba3669bc0e09ad' into worktree-agent-ac54c8919e940163a

This commit is contained in:
2026-11-10 01:02:08 +00:00
17 changed files with 1010 additions and 26 deletions
+12
View File
@@ -47,12 +47,18 @@ class AppUpdate {
);
/// Reads `version.json`; the APK path is relative to [base].
///
/// A [signed] feed (its signature verified with a key the build trusts)
/// must name the artifact's sha256 and, for Android, its signing
/// certificate: an unsigned build tolerates their absence, a signed one
/// does not.
static AppUpdate? parse(
String body,
Uri base, {
String platform = 'android',
String architecture = 'arm64',
String? installedSignature,
bool signed = false,
}) {
try {
final json = jsonDecode(body);
@@ -67,6 +73,7 @@ class AppUpdate {
return null;
}
final digest = '${json['sha256'] ?? ''}';
if (signed && digest.isEmpty) return null;
if (digest.isNotEmpty && !RegExp(r'^[a-fA-F0-9]{64}$').hasMatch(digest)) {
return null;
}
@@ -76,6 +83,11 @@ class AppUpdate {
!RegExp(r'^[a-fA-F0-9]{64}$').hasMatch(signer))) {
return null;
}
if (signed &&
platform == 'android' &&
(signer is! String || installedSignature == null)) {
return null;
}
if (platform == 'android' &&
signer is String &&
installedSignature != null &&
+45 -11
View File
@@ -11,6 +11,7 @@ import 'package:path_provider/path_provider.dart';
import '../../core/session/server_session.dart';
import 'app_updates.dart';
import 'linux_self_update.dart';
import 'update_feed_signature.dart';
bool get supported =>
Platform.isAndroid || Platform.isLinux || Platform.isMacOS;
@@ -57,10 +58,42 @@ Future<HttpClientResponse> _get(
Future<AppUpdate> fetchLatest(Uri base) =>
fetchLatestScoped(base, ServerSession.headersFor);
Future<AppUpdate> fetchLatestScoped(
Uri base,
/// Reads at most [limit] bytes of [response].
Future<List<int>> _body(HttpClientResponse response, int limit) async {
final chunks = <int>[];
await for (final chunk in response.timeout(const Duration(seconds: 10))) {
if (chunks.length + chunk.length > limit) {
throw const FormatException('Update metadata is too large.');
}
chunks.addAll(chunk);
}
return chunks;
}
/// The detached feed signature, or null when the server offers none.
Future<String?> _signature(
HttpClient client,
Uri feed,
Map<String, String> Function(Uri) headers,
) async {
try {
final response = await _get(
client,
feed.resolve('version.json.sig'),
headers,
);
return utf8.decode(await _body(response, 4096));
} on HttpException {
return null;
}
}
Future<AppUpdate> fetchLatestScoped(
Uri base,
Map<String, String> Function(Uri) headers, {
List<List<int>>? trustedKeys,
}) async {
final trusted = trustedKeys ?? trustedUpdateKeys();
final platform = nativePlatform;
if (platform == 'linux' && nativeArchitecture != 'x64') {
throw const FormatException(
@@ -73,15 +106,15 @@ Future<AppUpdate> fetchLatestScoped(
final client = HttpClient()..connectionTimeout = const Duration(seconds: 10);
try {
final response = await _get(client, feed.resolve('version.json'), headers);
var bytes = 0;
final chunks = <int>[];
await for (final chunk in response.timeout(const Duration(seconds: 10))) {
bytes += chunk.length;
if (bytes > 65536) {
throw const FormatException('Update metadata is too large.');
}
chunks.addAll(chunk);
}
final chunks = await _body(response, 65536);
// The signature is checked before anything in the feed (the artifact's
// sha256 above all) is believed.
final problem = feedSignatureProblem(
chunks,
trusted.isEmpty ? null : await _signature(client, feed, headers),
trusted,
);
if (problem != null) throw FormatException(problem);
final update = AppUpdate.parse(
utf8.decode(chunks),
feed,
@@ -90,6 +123,7 @@ Future<AppUpdate> fetchLatestScoped(
installedSignature: platform == 'android'
? (await PackageInfo.fromPlatform()).buildSignature
: null,
signed: trusted.isNotEmpty,
);
if (update == null) {
throw const FormatException('No compatible build is offered.');
+140
View File
@@ -0,0 +1,140 @@
import 'dart:typed_data';
import 'package:crypto/crypto.dart' as crypto;
/// Ed25519 signature verification (RFC 8032 section 5.1.7), verify only.
///
/// The update feed is checked once per update, so plain [BigInt] arithmetic
/// is fast enough and keeps this small and auditable; it never handles a
/// private key, so constant-time arithmetic is not required. Tested against
/// the RFC 8032 vectors in `test/features/updates/ed25519_test.dart`.
bool ed25519Verify({
required List<int> publicKey,
required List<int> message,
required List<int> signature,
}) {
if (publicKey.length != 32 || signature.length != 64) return false;
final a = _decode(publicKey);
final r = _decode(signature.sublist(0, 32));
if (a == null || r == null) return false;
final s = _littleEndian(signature.sublist(32));
if (s >= _l) return false;
final digest = crypto.sha512.convert([
...signature.sublist(0, 32),
...publicKey,
...message,
]);
final k = _littleEndian(digest.bytes) % _l;
final left = _multiply(_base, s);
final right = _add(r, _multiply(a, k));
return _equal(_encode(left), _encode(right));
}
final BigInt _p = (BigInt.one << 255) - BigInt.from(19);
final BigInt _l =
(BigInt.one << 252) +
BigInt.parse('27742317777372353535851937790883648493');
final BigInt _d =
(BigInt.from(-121665) * BigInt.from(121666).modInverse(_p)) % _p;
final BigInt _sqrtMinusOne = BigInt.two.modPow((_p - BigInt.one) >> 2, _p);
final _Point _base = _decode(_baseEncoding)!;
// The base point: y = 4/5, x even.
final List<int> _baseEncoding = _bytes(
(BigInt.from(4) * BigInt.from(5).modInverse(_p)) % _p,
);
/// Extended coordinates (X:Y:Z:T) with x = X/Z, y = Y/Z, x*y = T/Z.
class _Point {
const _Point(this.x, this.y, this.z, this.t);
final BigInt x, y, z, t;
}
final _Point _identity = _Point(
BigInt.zero,
BigInt.one,
BigInt.one,
BigInt.zero,
);
BigInt _littleEndian(List<int> bytes) {
var value = BigInt.zero;
for (var i = bytes.length - 1; i >= 0; i--) {
value = (value << 8) | BigInt.from(bytes[i]);
}
return value;
}
List<int> _bytes(BigInt value) {
final out = Uint8List(32);
var rest = value;
for (var i = 0; i < 32; i++) {
out[i] = (rest & BigInt.from(0xff)).toInt();
rest >>= 8;
}
return out;
}
_Point? _decode(List<int> encoded) {
final bytes = List<int>.of(encoded);
final sign = bytes[31] >> 7;
bytes[31] &= 0x7f;
final y = _littleEndian(bytes);
if (y >= _p) return null;
final y2 = y * y % _p;
final u = (y2 - BigInt.one) % _p;
final v = (_d * y2 + BigInt.one) % _p;
final v3 = v * v % _p * v % _p;
final v7 = v3 * v3 % _p * v % _p;
var x =
u * v3 % _p * (u * v7 % _p).modPow((_p - BigInt.from(5)) >> 3, _p) % _p;
final check = v * x % _p * x % _p;
if (check == u) {
// x is a square root of u/v.
} else if (check == (_p - u) % _p) {
x = x * _sqrtMinusOne % _p;
} else {
return null;
}
if (x == BigInt.zero && sign == 1) return null;
if (x.isOdd != (sign == 1)) x = _p - x;
return _Point(x, y, BigInt.one, x * y % _p);
}
List<int> _encode(_Point point) {
final inverse = point.z.modInverse(_p);
final x = point.x * inverse % _p;
final y = point.y * inverse % _p;
final out = _bytes(y);
if (x.isOdd) out[31] |= 0x80;
return out;
}
// The unified addition law (RFC 8032 section 5.1.4); it also doubles.
_Point _add(_Point p, _Point q) {
final a = (p.y - p.x) * (q.y - q.x) % _p;
final b = (p.y + p.x) * (q.y + q.x) % _p;
final c = p.t * BigInt.two * _d % _p * q.t % _p;
final d = p.z * BigInt.two * q.z % _p;
final e = (b - a) % _p, f = (d - c) % _p, g = (d + c) % _p, h = (b + a) % _p;
return _Point(e * f % _p, g * h % _p, f * g % _p, e * h % _p);
}
_Point _multiply(_Point point, BigInt scalar) {
var result = _identity;
var addend = point;
var rest = scalar;
while (rest > BigInt.zero) {
if (rest.isOdd) result = _add(result, addend);
addend = _add(addend, addend);
rest >>= 1;
}
return result;
}
bool _equal(List<int> a, List<int> b) {
if (a.length != b.length) return false;
for (var i = 0; i < a.length; i++) {
if (a[i] != b[i]) return false;
}
return true;
}
@@ -0,0 +1,79 @@
import 'dart:convert';
import 'ed25519.dart';
/// The Ed25519 public keys this build trusts to sign update feeds, base64
/// and comma separated (`--dart-define=PIPELINES_UPDATE_PUBLIC_KEY`). The
/// installer embeds its own key and any listed in
/// `~/.config/pipelines/update-trusted-keys`; see
/// docs/notes/network-and-update-signing.md.
const embeddedUpdateKeys = String.fromEnvironment(
'PIPELINES_UPDATE_PUBLIC_KEY',
);
/// The trusted keys in [configured]; a malformed entry is dropped (the
/// installer refuses to build with one), and an empty result means this is
/// an unsigned development build.
List<List<int>> trustedUpdateKeys([String configured = embeddedUpdateKeys]) {
final keys = <List<int>>[];
for (final entry in configured.split(',')) {
final text = entry.trim();
if (text.isEmpty) continue;
try {
final key = base64.decode(text);
if (key.length == 32) keys.add(key);
} on FormatException {
continue;
}
}
return keys;
}
/// Why the [feed] bytes cannot be trusted, or null when they can.
///
/// [signatureDocument] is the detached `version.json.sig`
/// (`{"algorithm":"ed25519","publicKey":…,"signature":…}`), null when the
/// server offers none. A build with no trusted keys accepts any feed, as
/// before signing existed; a build with keys requires a valid signature by
/// one of them.
String? feedSignatureProblem(
List<int> feed,
String? signatureDocument,
List<List<int>> trusted,
) {
if (trusted.isEmpty) return null;
if (signatureDocument == null) {
return 'The update feed is not signed, so it was not trusted.';
}
final Object? document;
try {
document = jsonDecode(signatureDocument);
} on FormatException {
return 'The update feed signature is unreadable.';
}
if (document is! Map || document['algorithm'] != 'ed25519') {
return 'The update feed signature is unreadable.';
}
final List<int> key, signature;
try {
key = base64.decode('${document['publicKey'] ?? ''}');
signature = base64.decode('${document['signature'] ?? ''}');
} on FormatException {
return 'The update feed signature is unreadable.';
}
if (!trusted.any((candidate) => _same(candidate, key))) {
return 'The update feed is signed by a key this app does not trust.';
}
if (!ed25519Verify(publicKey: key, message: feed, signature: signature)) {
return 'The update feed signature does not match; it was not trusted.';
}
return null;
}
bool _same(List<int> a, List<int> b) {
if (a.length != b.length) return false;
for (var i = 0; i < a.length; i++) {
if (a[i] != b[i]) return false;
}
return true;
}
+102
View File
@@ -0,0 +1,102 @@
import 'package:flutter_test/flutter_test.dart';
import 'package:pipelines_app/features/updates/ed25519.dart';
List<int> hex(String text) => [
for (var i = 0; i < text.length; i += 2)
int.parse(text.substring(i, i + 2), radix: 16),
];
// RFC 8032 section 7.1, tests 1-3.
const vectors = [
(
'd75a980182b10ab7d54bfed3c964073a0ee172f3daa62325af021a68f707511a',
'',
'e5564300c360ac729086e2cc806e828a84877f1eb8e5d974d873e06522490155'
'5fb8821590a33bacc61e39701cf9b46bd25bf5f0595bbe24655141438e7a100b',
),
(
'3d4017c3e843895a92b70aa74d1b7ebc9c982ccf2ec4968cc0cd55f12af4660c',
'72',
'92a009a9f0d4cab8720e820b5f642540a2b27b5416503f8fb3762223ebdb69da'
'085ac1e43e15996e458f3613d0f11d8c387b2eaeb4302aeeb00d291612bb0c00',
),
(
'fc51cd8e6218a1a38da47ed00230f0580816ed13ba3303ac5deb911548908025',
'af82',
'6291d657deec24024827e69c3abe01a30ce548a284743a445e3680d7db5ac3ac'
'18ff9b538d16f290ae67f760984dc6594a7c15e9716ed28dc027beceea1ec40a',
),
];
void main() {
test('RFC 8032 vectors verify', () {
for (final (key, message, signature) in vectors) {
expect(
ed25519Verify(
publicKey: hex(key),
message: hex(message),
signature: hex(signature),
),
isTrue,
reason: key,
);
}
});
test('a changed message, signature or key is refused', () {
final (key, message, signature) = vectors[2];
bool verify(List<int> k, List<int> m, List<int> s) =>
ed25519Verify(publicKey: k, message: m, signature: s);
expect(verify(hex(key), hex('af83'), hex(signature)), isFalse);
expect(verify(hex(key), hex(''), hex(signature)), isFalse);
for (final index in [0, 31, 32, 63]) {
final changed = hex(signature)..[index] ^= 1;
expect(verify(hex(key), hex(message), changed), isFalse);
}
final otherKey = hex(vectors[1].$1);
expect(verify(otherKey, hex(message), hex(signature)), isFalse);
final changedKey = hex(key)..[0] ^= 1;
expect(verify(changedKey, hex(message), hex(signature)), isFalse);
});
test('malformed inputs are refused rather than thrown', () {
final (key, message, signature) = vectors[1];
expect(
ed25519Verify(
publicKey: hex(key).sublist(1),
message: hex(message),
signature: hex(signature),
),
isFalse,
);
expect(
ed25519Verify(
publicKey: hex(key),
message: hex(message),
signature: hex(signature).sublist(1),
),
isFalse,
);
// S must be below the group order L (no malleable second form).
final high = hex(signature)..fillRange(32, 64, 0xff);
expect(
ed25519Verify(
publicKey: hex(key),
message: hex(message),
signature: high,
),
isFalse,
);
// y = p (non-canonical) does not decode as a key.
final invalidKey = List<int>.filled(32, 0xff)..[0] = 0xed;
invalidKey[31] = 0x7f;
expect(
ed25519Verify(
publicKey: invalidKey,
message: hex(message),
signature: hex(signature),
),
isFalse,
);
});
}
@@ -0,0 +1 @@
6Jo+xjitfGvKp28mdXIziIyDfs5UUP4xzFl92+xrAAU=
@@ -0,0 +1 @@
{"versionCode": 29400000, "versionName": "0.1.29400000", "platform": "linux", "architectures": ["x64"], "file": "pipelines-linux-x64-29400000-0123abcd.tar.gz", "sha256": "abababababababababababababababababababababababababababababababab", "notes": "fixture"}
@@ -0,0 +1 @@
{"algorithm": "ed25519", "publicKey": "6Jo+xjitfGvKp28mdXIziIyDfs5UUP4xzFl92+xrAAU=", "signature": "TpyvCl63pcKcO2Nu4uVI3tCLEN7w/SZe/yQWrs82OqY8C/2cP/o/uFWZ6aRJCo0yaoyTa30wQPSExhUJ08swDg=="}
@@ -0,0 +1,157 @@
import 'dart:convert';
import 'dart:io';
import 'package:flutter_test/flutter_test.dart';
import 'package:pipelines_app/features/updates/app_updates.dart';
import 'package:pipelines_app/features/updates/app_updates_io.dart' as io;
import 'package:pipelines_app/features/updates/update_feed_signature.dart';
// Written by scripts/update_signing.py publish_feed with a throwaway key;
// scripts/test_update_signing.py checks the same files with OpenSSL.
const fixture = 'test/features/updates/fixtures/signed_feed';
void main() {
final feed = File('$fixture/version.json').readAsBytesSync();
final signature = File('$fixture/version.json.sig').readAsStringSync();
final key = File('$fixture/public_key.txt').readAsStringSync().trim();
final trusted = trustedUpdateKeys(key);
final otherKey = base64.encode(List<int>.generate(32, (i) => i));
test('a feed signed by the Python publisher verifies', () {
expect(trusted, hasLength(1));
expect(feedSignatureProblem(feed, signature, trusted), isNull);
// Trusted among several keys (this server's and another's).
expect(
feedSignatureProblem(
feed,
signature,
trustedUpdateKeys('$otherKey, $key'),
),
isNull,
);
});
test('a tampered feed or signature is refused', () {
final tampered = utf8.encode(utf8.decode(feed).replaceFirst('"ab', '"cd'));
expect(
feedSignatureProblem(tampered, signature, trusted),
contains('does not match'),
);
final document = jsonDecode(signature) as Map;
final bytes = base64.decode(document['signature'] as String)..[5] ^= 1;
final badSignature = jsonEncode({
...document,
'signature': base64.encode(bytes),
});
expect(
feedSignatureProblem(feed, badSignature, trusted),
contains('does not match'),
);
expect(
feedSignatureProblem(feed, '{"algorithm":"rsa"}', trusted),
contains('unreadable'),
);
expect(
feedSignatureProblem(feed, 'not json', trusted),
contains('unreadable'),
);
});
test('a missing signature is refused when the app has keys', () {
expect(feedSignatureProblem(feed, null, trusted), contains('not signed'));
// A development build without embedded keys keeps the old behaviour.
expect(feedSignatureProblem(feed, null, trustedUpdateKeys('')), isNull);
});
test('a signature by a key the app does not trust is refused', () {
expect(
feedSignatureProblem(feed, signature, trustedUpdateKeys(otherKey)),
contains('does not trust'),
);
// Swapping in the attacker's own key in the document does not help.
final document = jsonDecode(signature) as Map;
final swapped = jsonEncode({...document, 'publicKey': otherKey});
expect(
feedSignatureProblem(feed, swapped, trustedUpdateKeys('$key,$otherKey')),
contains('does not match'),
);
expect(trustedUpdateKeys('short, ${base64.encode([1])}'), isEmpty);
});
test('a signed feed must carry the artifact digest and Android signer', () {
final base = Uri.parse('https://mac.test/updates/');
final signer = 'a' * 64;
String android({bool digest = true, bool certificate = true}) =>
jsonEncode({
'versionCode': 9,
'apk': 'pipelines.apk',
if (digest) 'sha256': 'b' * 64,
if (certificate) 'signingCertificateSha256': signer,
});
AppUpdate? parse(String body, {bool signed = true}) =>
AppUpdate.parse(body, base, installedSignature: signer, signed: signed);
expect(parse(android()), isNotNull);
expect(parse(android(certificate: false)), isNull);
expect(parse(android(digest: false)), isNull);
expect(
AppUpdate.parse(android(), base, signed: true),
isNull,
reason: 'the installed signature is required to compare',
);
// Unsigned builds keep accepting legacy feeds; the OS installer checks.
expect(
parse(android(certificate: false, digest: false), signed: false),
isNotNull,
);
final linux = AppUpdate.parse(
utf8.decode(feed),
base,
platform: 'linux',
architecture: 'x64',
signed: true,
);
expect(linux?.sha256, 'ab' * 32);
});
test(
'the feed reader refuses unsigned and tampered feeds over HTTP',
() async {
final previous = HttpOverrides.current;
HttpOverrides.global = null;
final server = await HttpServer.bind(InternetAddress.loopbackIPv4, 0);
var body = feed;
String? served = signature;
server.listen((request) async {
final path = request.uri.path;
if (path.endsWith('/version.json')) {
request.response.add(body);
} else if (path.endsWith('/version.json.sig') && served != null) {
request.response.write(served);
} else {
request.response.statusCode = 404;
}
await request.response.close();
});
final base = Uri.parse('http://127.0.0.1:${server.port}/updates/');
Future<Object?> read() => io
.fetchLatestScoped(base, (_) => {}, trustedKeys: trusted)
.then<Object?>((update) => update, onError: (Object error) => error);
try {
served = null;
expect('${await read()}', contains('not signed'));
served = signature;
body = utf8.encode(utf8.decode(feed).replaceFirst('"ab', '"cd'));
expect('${await read()}', contains('does not match'));
body = feed;
// A genuine feed passes the signature; the host's platform may then
// still find the fixture's Linux build incompatible.
final result = await read();
expect('$result', isNot(contains('sign')));
expect('$result', isNot(contains('trust')));
} finally {
await server.close(force: true);
HttpOverrides.global = previous;
}
},
);
}
+121
View File
@@ -0,0 +1,121 @@
# Network defaults and update-feed signing
## Update feed signing
Before this change the app trusted whatever `/updates/` served: the Linux
self-update installed and relaunched an archive whose sha256 came from the same
`version.json`, Android skipped its signer check when the feed omitted the
field, and the macOS zip had no check beyond that sha256. A server (or anyone
able to write its update directory, or answer for its address) could hand every
client another build.
### What is signed
- Publishing (`./install --apk`, `./install --linux|--macos --publish`) writes
`version.json` and, beside it, a detached `version.json.sig`:
`{"algorithm":"ed25519","publicKey":"<base64 32 bytes>","signature":"<base64 64 bytes>"}`.
The signature covers the exact bytes of `version.json`, which carry the
artifact `sha256` (and, for Android, `signingCertificateSha256`). The
signature is written first, then the feed, both atomically; a check racing a
publish fails once and succeeds on the next check.
- Code: `scripts/update_signing.py` (keys, signing, `publish_feed`),
`scripts/app_build.py` (`build_apk`, web and desktop defines),
`scripts/app_desktop_build.py` (`publish_desktop(..., signer=...)`).
### Key management
- Each installation has one Ed25519 key in
`~/.config/pipelines/update-signing-key.pem` (mode 0600, directory 0700),
generated once on the first `./install` or app build/publish and never
replaced. `PIPELINES_UPDATE_SIGNING_KEY` names another path. A key file that
is a symlink or readable by others is refused. It is never in the repository.
`scripts/uninstall.py` (full reset) removes `~/.config/pipelines`, and so
the key; see "Lost or replaced key" below.
- Cryptography is the OpenSSL 3 command line (`genpkey`, `pkey`,
`pkeyutl -sign -rawin`). The installer's Python has no `cryptography`
package, and macOS's `/usr/bin/openssl` is LibreSSL 3.3.6, which has no
Ed25519 (`unable to load Private Key`). The tool is chosen as
`PIPELINES_OPENSSL`, then `/opt/homebrew/opt/openssl@3/bin/openssl`,
`/usr/local/opt/openssl@3/bin/openssl`, then `openssl` on PATH, taking the
first that can generate an Ed25519 key. Without one the install stops with
`Update signing needs OpenSSL 3 with Ed25519 ...` (`brew install openssl@3`).
On this Mac `openssl` on PATH is Homebrew OpenSSL 3.6.4.
- Print this installation's public key:
`python3 scripts/update_signing.py`.
### Trust model
- Every app build embeds the keys it trusts as
`--dart-define=PIPELINES_UPDATE_PUBLIC_KEY=<key>[,<key>...]` (apk, linux,
macos and web builds; the define is part of the build cache key, so a key
change rebuilds). The list is this installation's own key first, then each
base64 key in `~/.config/pipelines/update-trusted-keys` (one per line, `#`
comments allowed; a malformed line stops the build).
- Each server signs its own feed with its own key. A client accepts a feed
only when `version.json.sig` names an embedded key and the signature over
the served bytes verifies with it; the client checks the signature before it
believes anything in the feed. So an app built on the Mac updates from tank
only if tank's key was listed in the Mac's `update-trusted-keys` when that
app was built (and vice versa). There is no trust-on-first-use pinning:
trust comes only from the build, which keeps the model to one rule.
- With keys embedded, the app refuses: an unsigned feed (no `.sig`), an
unreadable signature, a key it does not embed, a bad signature, a feed
without `sha256`, and an Android feed without `signingCertificateSha256`
(or when the installed signature is unknown). The Android certificate must
still match the installed app's, as before.
- A build with no embedded key (a developer `flutter run`/`flutter build`
outside the installer) keeps the previous unsigned behaviour. Installer
builds always embed a key.
- Verification in the app is pure Dart (`app/lib/features/updates/ed25519.dart`,
RFC 8032 verify only, BigInt arithmetic, checked against RFC 8032 test
vectors 1-3; no new package) and
`app/lib/features/updates/update_feed_signature.dart`; the reader is
`fetchLatestScoped` in `app_updates_io.dart`. The web Settings "Android
download" link is a browser download the operator installs by hand and is
unchanged.
### Migration (the live Mac install and tank)
Not done by this change; nothing was deployed. In order, on each machine:
1. `./install` (or the next `./install --apk` / desktop publish) generates
`~/.config/pipelines/update-signing-key.pem`. Check:
`ls -l ~/.config/pipelines/update-signing-key.pem` shows `-rw-------`.
2. To let one machine's apps update from the other, exchange keys before
building: on tank run `python3 scripts/update_signing.py` and append the
line to the Mac's `~/.config/pipelines/update-trusted-keys`; do the
reverse on tank. Skip this if each machine's apps only update from it.
3. Rebuild and publish: `./install --apk`, `./install --macos --publish
--flutter <flutter>`, `./install --linux --publish`. Each feed directory
then holds `version.json.sig`.
4. Installed apps from before this change do not check signatures; they take
the next signed release normally (the `.sig` is ignored), and from then on
they refuse unsigned or foreign feeds. Phones whose feed came from the other
machine need step 2 before step 3.
5. Lost or replaced key (for example after a full uninstall): apps built with
the old key refuse the new feeds. Install the new build once by hand (APK
download from Settings in the browser, or the desktop archive), or restore
the old key file from backup.
### Verification output
- `cd scripts && python3 -m unittest test_update_signing test_app_build
test_app_desktop_build test_setup_native`: `Ran 81 tests ... OK`. Covers key
generation once at 0600 in a scratch HOME, refusal of readable/symlinked
keys, LibreSSL-style tools skipped, signatures checked independently with
`openssl pkeyutl -verify` (and a tampered message refused), feed plus `.sig`
publication, trusted-key list parsing, APK publication signed with the
generated key, desktop publication passing the exact feed bytes to the
signer, and the committed Dart fixture being a valid OpenSSL signature.
- `cd app && tool/memwatch/full.sh 'test/features/updates test/features/settings'`:
`memwatch: PASS: 187 passed, 0 failed, 0 skipped`. New:
`test/features/updates/ed25519_test.dart` (RFC 8032 vectors; changed
message, signature, key; short inputs; S >= L; non-canonical key) and
`test/features/updates/update_feed_signature_test.dart` (the
Python-signed fixture `test/features/updates/fixtures/signed_feed/`
verifies; tampered feed and signature refused; missing signature refused
with keys, accepted without; untrusted and swapped keys refused; signed
feeds require sha256 and the Android signer; and the HTTP reader
`fetchLatestScoped` against a loopback server refuses an unsigned and a
tampered feed and lets the genuine one past the signature check).
`test/core/session`: `PASS: 18 passed`. `tool/analyze.sh`: `analyze: clean`.
+13 -6
View File
@@ -27,6 +27,9 @@ from urllib.parse import urlsplit
REPO = Path(__file__).resolve().parents[1]
APP = REPO / "app"
DEFAULT_PREFIX = Path.home() / ".local/share/pipelines"
_signing_spec = importlib.util.spec_from_file_location("pipelines_update_signing", Path(__file__).with_name("update_signing.py"))
update_signing = importlib.util.module_from_spec(_signing_spec)
_signing_spec.loader.exec_module(update_signing)
def default_cache():
return Path(os.environ.get("PIPELINES_APP_CACHE") or Path.home() / ".cache/pipelines/app-builds")
@@ -346,7 +349,10 @@ def build_apk(prefix, server_url=None, notes="", runner=subprocess.run, log=prin
server = server_url or phone_server_url(manifest)
# The build number is the source key's build date, so the same sources
# reuse one APK and every rebuild is newer than the last.
arguments = (f"--dart-define=PIPELINES_SERVER_URL={server}",)
# The feed is signed and the app embeds the keys it trusts (see
# scripts/update_signing.py), so a server cannot hand phones another APK.
signer = update_signing.Signer()
arguments = (f"--dart-define=PIPELINES_SERVER_URL={server}", *update_signing.dart_defines())
host_flutter, sdk = native_flutter(flutter), android_sdk_dir()
use_native = native if native is not None else bool(host_flutter and sdk)
if use_native:
@@ -365,9 +371,9 @@ def build_apk(prefix, server_url=None, notes="", runner=subprocess.run, log=prin
target.mkdir(parents=True, exist_ok=True)
shutil.copyfile(output / "pipelines.apk", target / "pipelines.apk.next")
os.replace(target / "pipelines.apk.next", target / "pipelines.apk")
(target / "version.json").write_text(json.dumps({
update_signing.publish_feed(target, {
"versionCode": code, "versionName": f"0.1.{code}", "apk": "pipelines.apk", "notes": notes,
"signingCertificateSha256": fingerprint, "sha256": apk_sha256}) + "\n")
"signingCertificateSha256": fingerprint, "sha256": apk_sha256}, signer)
log(f"APK ready: {target / 'pipelines.apk'} (build {code}, server {server}).")
log("Phones get it from Settings → App updates → Check for updates, or download "
f"{server.replace('wss://', 'https://').replace('ws://', 'http://').removesuffix('/connect')}/updates/pipelines.apk")
@@ -417,13 +423,14 @@ def main(argv=None):
bundle, archive = desktop.build_desktop(args.kind, app=APP, cache=default_cache(), runner=subprocess.run,
source_digest=digest, docker_build=build, docker_ready=docker_available,
server_url=args.server_url, flutter=args.flutter, output=args.output,
prune_cache=prune)
prune_cache=prune, update_keys=update_signing.dart_defines())
if args.publish:
manifest = load_manifest(args.prefix)
updates = (manifest.get("web") or {}).get("updates")
if not updates:
raise ValueError("This installation has no app update directory; rerun ./install")
desktop.publish_desktop(args.kind, bundle, archive, Path(updates), notes=args.notes)
desktop.publish_desktop(args.kind, bundle, archive, Path(updates), notes=args.notes,
signer=update_signing.Signer())
return
if args.kind == "apk":
build_apk(args.prefix, args.server_url, args.notes, flutter=args.flutter, native=args.native)
@@ -435,7 +442,7 @@ def main(argv=None):
web = manifest.get("web") or {}
if not web.get("root"):
raise ValueError("This installation does not serve the app; rerun ./install")
install_tree(build("web"), Path(web["root"]))
install_tree(build("web", arguments=update_signing.dart_defines()), Path(web["root"]))
print(f"Web app installed at {web['root']}; reload the page.")
print_web_urls(manifest)
+10 -5
View File
@@ -120,8 +120,8 @@ def _macos_build(*, app, cache, source_digest, runner, arguments, flutter, log):
def build_desktop(kind, *, app, cache, runner, source_digest, docker_build,
docker_ready, server_url=None, flutter=None, output=None, log=print,
prune_cache=None):
arguments = _arguments(server_url)
prune_cache=None, update_keys=()):
arguments = _arguments(server_url) + tuple(update_keys)
if kind == "macos":
entry, bundle, archive = _macos_build(app=app, cache=cache, runner=runner,
source_digest=source_digest, arguments=arguments, flutter=flutter, log=log)
@@ -154,8 +154,10 @@ def build_desktop(kind, *, app, cache, runner, source_digest, docker_build,
return bundle, archive
def publish_desktop(kind, bundle, archive, updates, *, notes="", log=print):
"""Publish archive before its feed; versioned names preserve in-flight reads."""
def publish_desktop(kind, bundle, archive, updates, *, signer, notes="", log=print):
"""Publish archive before its feed; versioned names preserve in-flight reads.
[signer] returns the detached `version.json.sig` for the feed bytes."""
entry = bundle.parent if kind == "macos" else bundle
complete = json.loads((entry / ".complete").read_text())
code = int(complete["built"]) // 60
@@ -176,8 +178,11 @@ def publish_desktop(kind, bundle, archive, updates, *, notes="", log=print):
previous = json.loads(feed.read_text())
if previous.get("versionCode", 0) > code:
raise ValueError("Refusing to publish an older build over a newer update")
data = (json.dumps(metadata) + "\n").encode()
# The signature lands before the feed it covers, after the archive.
for destination, content in ((target / filename, archive.read_bytes()),
(feed, (json.dumps(metadata) + "\n").encode())):
(target / "version.json.sig", signer(data)),
(feed, data)):
descriptor, temporary = tempfile.mkstemp(prefix=destination.name + ".", dir=target)
try:
with os.fdopen(descriptor, "wb") as output:
+2 -1
View File
@@ -403,7 +403,8 @@ class Setup:
raise ValueError("Docker is not running, so the web app cannot be built. The server is never "
"installed without it; start Docker and rerun ./install.")
try:
built = app_build.build("web", runner=self.runner)
built = app_build.build("web", runner=self.runner,
arguments=app_build.update_signing.dart_defines(self.environ))
except RuntimeError as error:
raise ValueError(f"The web app build failed ({error}); nothing was installed") from None
if not (Path(built) / "index.html").is_file():
+5
View File
@@ -83,6 +83,11 @@ class AppBuildTests(unittest.TestCase):
feed = json.loads((updates / "version.json").read_text())
self.assertEqual(feed["signingCertificateSha256"], "ab" * 32)
self.assertEqual(feed["sha256"], app_build.hashlib.sha256((updates / "pipelines.apk").read_bytes()).hexdigest())
# Signed with the installation key generated (0600) in the scratch home.
key = Path(self.home.name) / ".config/pipelines/update-signing-key.pem"
self.assertEqual(key.stat().st_mode & 0o777, 0o600)
signature = json.loads((updates / "version.json.sig").read_text())
self.assertEqual(signature["publicKey"], app_build.update_signing.public_key(key))
(output / "signing-certificate-sha256.txt").write_text("invalid")
with patch.object(app_build, "load_manifest", return_value=manifest), patch.object(app_build, "build", return_value=output):
with self.assertRaisesRegex(ValueError, "exactly one"):
+15 -3
View File
@@ -160,24 +160,36 @@ class DesktopBuildTests(unittest.TestCase):
def test_linux_publication_has_matching_hash_and_rejects_downgrade(self):
bundle, archive = self.build("linux")
updates = self.root / "updates"
metadata = desktop.publish_desktop("linux", bundle, archive, updates, log=lambda *_: None)
signed = []
signer = lambda data: signed.append(data) or b"signature document\n"
metadata = desktop.publish_desktop("linux", bundle, archive, updates, signer=signer, log=lambda *_: None)
target = updates / "linux-x64"
self.assertEqual(json.loads((target / "version.json").read_text()), metadata)
# The detached signature covers the exact published feed bytes.
self.assertEqual(signed, [(target / "version.json").read_bytes()])
self.assertEqual((target / "version.json.sig").read_bytes(), b"signature document\n")
self.assertEqual(metadata["architectures"], ["x64"])
self.assertEqual(metadata["sha256"], desktop.hashlib.sha256((target / metadata["file"]).read_bytes()).hexdigest())
metadata["versionCode"] += 1
(target / "version.json").write_text(json.dumps(metadata))
with self.assertRaisesRegex(ValueError, "older build"):
desktop.publish_desktop("linux", bundle, archive, updates)
desktop.publish_desktop("linux", bundle, archive, updates, signer=signer)
@patch.object(desktop.platform, "system", return_value="Darwin")
def test_macos_feed_uses_actual_architectures_and_native_build_stamp(self, _):
bundle, archive = self.build("macos", flutter="/fixture/flutter")
metadata = desktop.publish_desktop("macos", bundle, archive, self.root / "updates", log=lambda *_: None)
metadata = desktop.publish_desktop("macos", bundle, archive, self.root / "updates",
signer=lambda data: b"{}\n", log=lambda *_: None)
self.assertTrue((self.root / "updates/macos/version.json.sig").is_file())
self.assertEqual(metadata["architectures"], ["arm64", "x64"])
command = next(c for c in self.runner.calls if c[1:3] == ["build", "macos"])
self.assertIn(f"--dart-define=PIPELINES_NATIVE_BUILD_CODE={metadata['versionCode']}", command)
def test_desktop_builds_embed_the_trusted_update_keys(self):
self.build("linux", update_keys=("--dart-define=PIPELINES_UPDATE_PUBLIC_KEY=abc",))
run = next(c for c in self.runner.calls if c[:2] == ["docker", "run"])
self.assertIn("--dart-define=PIPELINES_UPDATE_PUBLIC_KEY=abc", run)
def test_desktop_server_prefill_rejects_embedded_credentials(self):
with self.assertRaisesRegex(ValueError, "embedded credentials"):
self.build("linux", server_url="wss://secret@example.com/connect")
+117
View File
@@ -0,0 +1,117 @@
import base64
import importlib.util
import json
import os
from pathlib import Path
import subprocess
import tempfile
import unittest
from unittest.mock import patch
def load(name):
spec = importlib.util.spec_from_file_location(name, Path(__file__).with_name(name + ".py"))
result = importlib.util.module_from_spec(spec)
spec.loader.exec_module(result)
return result
signing = load("update_signing")
FIXTURE = Path(__file__).resolve().parents[1] / "app/test/features/updates/fixtures/signed_feed"
def verify(tool, public, data, signature):
"""Independent check with `openssl pkeyutl -verify` against the raw key."""
with tempfile.TemporaryDirectory() as directory:
root = Path(directory)
(root / "pub.der").write_bytes(signing.SPKI_PREFIX + base64.b64decode(public))
(root / "data").write_bytes(data)
(root / "sig").write_bytes(signature)
result = subprocess.run([tool, "pkeyutl", "-verify", "-pubin", "-keyform", "DER", "-inkey",
str(root / "pub.der"), "-rawin", "-in", str(root / "data"),
"-sigfile", str(root / "sig")], capture_output=True)
return result.returncode == 0
class UpdateSigningTests(unittest.TestCase):
def setUp(self):
self.home = tempfile.TemporaryDirectory()
self.addCleanup(self.home.cleanup)
patcher = patch.object(signing.Path, "home", return_value=Path(self.home.name))
patcher.start()
self.addCleanup(patcher.stop)
self.environ = {}
self.tool = signing.openssl(self.environ)
def test_key_is_generated_once_private_under_the_config_directory(self):
key = signing.ensure_key(self.environ)
self.assertEqual(key, Path(self.home.name) / ".config/pipelines/update-signing-key.pem")
self.assertEqual(key.stat().st_mode & 0o777, 0o600)
self.assertEqual(key.parent.stat().st_mode & 0o777, 0o700)
before = key.read_bytes()
self.assertEqual(signing.ensure_key(self.environ), key)
self.assertEqual(key.read_bytes(), before)
self.assertEqual([p.name for p in key.parent.iterdir()], [key.name])
def test_readable_or_symlinked_key_is_refused(self):
key = signing.ensure_key(self.environ)
os.chmod(key, 0o644)
with self.assertRaisesRegex(ValueError, "chmod 600"):
signing.ensure_key(self.environ)
os.chmod(key, 0o600)
link = key.with_name("link.pem")
link.symlink_to(key)
with self.assertRaisesRegex(ValueError, "regular file"):
signing.ensure_key({"PIPELINES_UPDATE_SIGNING_KEY": str(link)})
def test_openssl_without_ed25519_is_skipped_or_reported(self):
missing = {"PIPELINES_OPENSSL": "/nonexistent/openssl"}
self.assertEqual(signing.openssl(missing), self.tool)
with patch.object(signing, "CANDIDATES", ()), \
patch.object(signing.shutil, "which", return_value="/usr/bin/false"):
with self.assertRaisesRegex(ValueError, "OpenSSL 3"):
signing.openssl({})
def test_signature_verifies_and_tampering_does_not(self):
key = signing.ensure_key(self.environ)
data = b'{"versionCode": 7}\n'
document = json.loads(signing.signature_document(data, key, self.environ))
self.assertEqual(document["algorithm"], "ed25519")
public = document["publicKey"]
signature = base64.b64decode(document["signature"])
self.assertEqual((len(base64.b64decode(public)), len(signature)), (32, 64))
self.assertTrue(verify(self.tool, public, data, signature))
self.assertFalse(verify(self.tool, public, b'{"versionCode": 8}\n', signature))
def test_publish_writes_feed_and_detached_signature(self):
target = Path(self.home.name) / "updates"
target.mkdir()
data = signing.publish_feed(target, {"versionCode": 3}, signing.Signer(environ=self.environ))
self.assertEqual((target / "version.json").read_bytes(), data)
document = json.loads((target / "version.json.sig").read_text())
self.assertTrue(verify(self.tool, document["publicKey"], data, base64.b64decode(document["signature"])))
self.assertEqual((target / "version.json").stat().st_mode & 0o777, 0o644)
def test_build_embeds_own_key_first_then_listed_trusted_keys(self):
own = signing.public_key(signing.ensure_key(self.environ))
other = base64.b64encode(bytes(range(32))).decode()
listed = Path(self.home.name) / ".config/pipelines/update-trusted-keys"
listed.write_text(f"# tank\n{other}\n\n{own}\n")
self.assertEqual(signing.dart_defines(self.environ),
(f"--dart-define=PIPELINES_UPDATE_PUBLIC_KEY={own},{other}",))
listed.write_text("not-a-key\n")
with self.assertRaisesRegex(ValueError, "update-trusted-keys:1"):
signing.trusted_keys(self.environ)
def test_committed_dart_fixture_is_a_valid_python_signature(self):
# The Dart verifier test reads this fixture; publish_feed produced it
# with a throwaway key that was never kept.
public = (FIXTURE / "public_key.txt").read_text().strip()
document = json.loads((FIXTURE / "version.json.sig").read_text())
self.assertEqual(document["publicKey"], public)
self.assertTrue(verify(self.tool, public, (FIXTURE / "version.json").read_bytes(),
base64.b64decode(document["signature"])))
if __name__ == "__main__":
unittest.main()
+189
View File
@@ -0,0 +1,189 @@
"""Ed25519 signing for the app update feed.
Each installation owns one private key, generated once into a 0600 file under
~/.config/pipelines/ (never in the repository). App builds embed the public key
(`--dart-define=PIPELINES_UPDATE_PUBLIC_KEY`, comma separated) together with any
other servers' keys listed one per line in
~/.config/pipelines/update-trusted-keys. Publishing writes `version.json` and a
detached `version.json.sig` over its exact bytes; the app verifies that
signature with an embedded key before it trusts the artifact sha256 inside.
The OpenSSL 3 command line does the cryptography (the system Python has no
`cryptography` package and macOS's LibreSSL has no Ed25519), so the tool is
looked up explicitly: PIPELINES_OPENSSL, Homebrew's openssl@3, then PATH.
"""
import base64
import json
import os
from pathlib import Path
import shutil
import subprocess
import tempfile
KEY_NAME = "update-signing-key.pem"
TRUSTED_NAME = "update-trusted-keys"
SIGNATURE_SUFFIX = ".sig"
DEFINE = "PIPELINES_UPDATE_PUBLIC_KEY"
CANDIDATES = ("/opt/homebrew/opt/openssl@3/bin/openssl", "/usr/local/opt/openssl@3/bin/openssl")
# DER SubjectPublicKeyInfo for Ed25519 is this 12-byte prefix plus the raw key.
SPKI_PREFIX = bytes.fromhex("302a300506032b6570032100")
def config_dir():
return Path.home() / ".config/pipelines"
def key_path(environ=None):
environ = os.environ if environ is None else environ
explicit = environ.get("PIPELINES_UPDATE_SIGNING_KEY")
return Path(explicit).expanduser() if explicit else config_dir() / KEY_NAME
def _supports_ed25519(tool):
try:
result = subprocess.run([tool, "genpkey", "-algorithm", "ed25519"],
capture_output=True, timeout=30)
except (OSError, subprocess.TimeoutExpired):
return False
return result.returncode == 0 and b"PRIVATE KEY" in result.stdout
def openssl(environ=None):
"""An OpenSSL executable with Ed25519 support, or a clear error."""
environ = os.environ if environ is None else environ
found = [environ.get("PIPELINES_OPENSSL"), *CANDIDATES, shutil.which("openssl")]
for tool in dict.fromkeys(t for t in found if t):
if _supports_ed25519(tool):
return tool
raise ValueError("Update signing needs OpenSSL 3 with Ed25519 (LibreSSL lacks it): "
"install openssl@3 (brew install openssl@3) or set PIPELINES_OPENSSL")
def _check_private(path):
if path.is_symlink() or not path.is_file():
raise ValueError(f"Update signing key must be a regular file: {path}")
if path.stat().st_mode & 0o077:
raise ValueError(f"Update signing key must be private (chmod 600): {path}")
def ensure_key(environ=None):
"""The installation's signing key, generated once (0600) when absent."""
path = key_path(environ)
if path.exists() or path.is_symlink():
_check_private(path)
return path
tool = openssl(environ)
path.parent.mkdir(mode=0o700, parents=True, exist_ok=True)
descriptor, temporary = tempfile.mkstemp(prefix=".update-key-", dir=path.parent)
os.close(descriptor)
try:
os.chmod(temporary, 0o600)
result = subprocess.run([tool, "genpkey", "-algorithm", "ed25519", "-out", temporary],
capture_output=True, text=True, timeout=30)
if result.returncode:
raise ValueError("Could not generate the update signing key: " + result.stderr.strip())
os.chmod(temporary, 0o600)
# Never replace a key another process created meanwhile.
os.link(temporary, path)
finally:
Path(temporary).unlink(missing_ok=True)
_check_private(path)
return path
def public_key(key, environ=None):
"""The raw 32-byte public key, base64 encoded."""
result = subprocess.run([openssl(environ), "pkey", "-in", str(key), "-pubout", "-outform", "DER"],
capture_output=True, timeout=30)
der = result.stdout
if result.returncode or len(der) != 44 or not der.startswith(SPKI_PREFIX):
raise ValueError(f"Could not read the Ed25519 public key from {key}")
return base64.b64encode(der[len(SPKI_PREFIX):]).decode()
def valid_public_key(text):
try:
return len(base64.b64decode(text, validate=True)) == 32
except ValueError:
return False
def trusted_keys(environ=None):
"""This installation's key first, then other servers' keys it trusts."""
keys = [public_key(ensure_key(environ), environ)]
listed = config_dir() / TRUSTED_NAME
if listed.is_file():
for number, line in enumerate(listed.read_text().splitlines(), 1):
line = line.split("#", 1)[0].strip()
if not line:
continue
if not valid_public_key(line):
raise ValueError(f"{listed}:{number} is not a base64 Ed25519 public key")
keys.append(line)
return list(dict.fromkeys(keys))
def dart_defines(environ=None):
"""The build argument embedding the trusted update keys."""
return (f"--dart-define={DEFINE}={','.join(trusted_keys(environ))}",)
def sign(data, key, environ=None):
"""A raw 64-byte Ed25519 signature over [data]."""
with tempfile.NamedTemporaryFile(prefix="pipelines-feed-") as message:
message.write(data)
message.flush()
result = subprocess.run([openssl(environ), "pkeyutl", "-sign", "-rawin", "-inkey", str(key),
"-in", message.name], capture_output=True, timeout=30)
if result.returncode or len(result.stdout) != 64:
raise ValueError("Could not sign the update feed: " + result.stderr.decode(errors="replace").strip())
return result.stdout
def signature_document(data, key, environ=None):
"""The detached `version.json.sig` content for feed bytes [data]."""
return (json.dumps({"algorithm": "ed25519", "publicKey": public_key(key, environ),
"signature": base64.b64encode(sign(data, key, environ)).decode()}) + "\n").encode()
class Signer:
"""Signs feeds with one key file; the installer's key unless given."""
def __init__(self, key=None, environ=None):
self.environ = environ
self.key = Path(key) if key else ensure_key(environ)
_check_private(self.key)
def __call__(self, data):
return signature_document(data, self.key, self.environ)
def write_atomic(path, content, mode=0o644):
descriptor, temporary = tempfile.mkstemp(prefix=path.name + ".", dir=path.parent)
try:
with os.fdopen(descriptor, "wb") as output:
output.write(content)
output.flush()
os.fsync(output.fileno())
os.chmod(temporary, mode)
os.replace(temporary, path)
finally:
Path(temporary).unlink(missing_ok=True)
def publish_feed(directory, metadata, signer):
"""Writes the signature, then the feed it covers, both atomically.
A reader racing a publish may see a new signature with the old feed for a
moment; that verification fails and the next check succeeds."""
data = (json.dumps(metadata) + "\n").encode()
signature = signer(data)
write_atomic(directory / ("version.json" + SIGNATURE_SUFFIX), signature)
write_atomic(directory / "version.json", data)
return data
if __name__ == "__main__":
# Prints this installation's public key, for another server's
# update-trusted-keys file (generating the key on first use).
print(public_key(ensure_key()))