diff --git a/scripts/app_build.py b/scripts/app_build.py index d6f93fb7..11a9faef 100644 --- a/scripts/app_build.py +++ b/scripts/app_build.py @@ -38,6 +38,10 @@ SOURCES = { "apk": ("lib", "android", "assets", "pubspec.yaml", "pubspec.lock", ".flutter-version", "docker"), } SKIPPED = {"build", ".dart_tool", ".gradle", ".cxx", "local.properties", ".DS_Store"} +# This is part of the web cache key so entries made before release metadata was +# generated cannot be mistaken for complete web releases. +WEB_CACHE_SCHEMA = "web-release-metadata-v1" +WEB_RELEASE_SCHEMA = 1 def flutter_version(app=APP): @@ -48,6 +52,8 @@ def digest(kind, app=APP, extra=()): """The build key: every source byte and path, plus [extra] arguments.""" hasher = hashlib.sha256() hasher.update(kind.encode()) + if kind == "web": + hasher.update(b"\0cache-schema\0" + WEB_CACHE_SCHEMA.encode()) for item in extra: hasher.update(b"\0arg\0" + item.encode()) for name in SOURCES[kind]: @@ -62,6 +68,27 @@ def digest(kind, app=APP, extra=()): return hasher.hexdigest()[:20] +def web_build_id(key): + """Stable identity embedded in the web app and published beside it.""" + return f"web-{key}" + + +def web_release_metadata(key): + return {"schema": WEB_RELEASE_SCHEMA, "platform": "web", "buildId": web_build_id(key)} + + +def web_cache_complete(entry): + """Whether [entry] contains metadata from the current web build schema.""" + marker = entry / "release.json" + if not (entry / ".complete").is_file() or not marker.is_file(): + return False + try: + complete = json.loads((entry / ".complete").read_text()) + return json.loads(marker.read_text()) == web_release_metadata(complete["key"]) + except (KeyError, TypeError, ValueError, json.JSONDecodeError): + return False + + def docker(runner, *args, quiet=False): result = runner(["docker", *args], capture_output=quiet, text=True) if result.returncode: @@ -112,7 +139,8 @@ def build(kind, runner=subprocess.run, app=APP, cache=None, arguments=(), log=pr cache = Path(cache) if cache else default_cache() key = digest(kind, app, arguments) entry = cache / f"{kind}-{key}" - if (entry / ".complete").is_file(): + usable = web_cache_complete(entry) if kind == "web" else (entry / ".complete").is_file() + if usable: log(f"Reusing the {kind} build {key} (app sources unchanged).") return entry log(f"Building the app ({kind}) in Docker with Flutter {flutter_version(app)}; " @@ -131,13 +159,21 @@ def build(kind, runner=subprocess.run, app=APP, cache=None, arguments=(), log=pr "-v", f"{signing_dir()}:/home/builder/.android"] built = int(time.time()) stamp = () + build_arguments = arguments + if kind == "web": + # The define is derived from the source key, not included in it: + # this avoids circular hashing while keeping the compiled app and + # its published metadata on the same identity. + build_arguments += (f"--dart-define=PIPELINES_WEB_BUILD_ID={web_build_id(key)}",) if kind == "apk": # Android installs only a higher build number over the last one, so # each build is stamped when it is made (minutes since the epoch); # the stamp is not part of the reuse key. code = built // 60 stamp = (f"--build-number={code}", f"--build-name=0.1.{code}") - docker(runner, *command, tag, "pipelines-app-build", kind, *arguments, *stamp) + docker(runner, *command, tag, "pipelines-app-build", kind, *build_arguments, *stamp) + if kind == "web": + (work / "release.json").write_text(json.dumps(web_release_metadata(key)) + "\n") (work / ".complete").write_text(json.dumps({"key": key, "built": built}) + "\n") if entry.exists(): shutil.rmtree(entry) diff --git a/scripts/test_app_build.py b/scripts/test_app_build.py index 8607d552..6fd18770 100644 --- a/scripts/test_app_build.py +++ b/scripts/test_app_build.py @@ -71,10 +71,34 @@ class AppBuildTests(unittest.TestCase): def test_web_build_has_no_build_number(self): runner = Runner() - app_build.build("web", runner=runner, cache=Path(self.home.name) / "cache", log=lambda *_: None) + output = app_build.build("web", runner=runner, cache=Path(self.home.name) / "cache", log=lambda *_: None) run = next(call for call in runner.calls if call[:2] == ["docker", "run"]) self.assertFalse(any(part.startswith("--build-number") for part in run)) self.assertNotIn("linux/amd64", run) + identity = app_build.web_build_id(json.loads((output / ".complete").read_text())["key"]) + self.assertIn(f"--dart-define=PIPELINES_WEB_BUILD_ID={identity}", run) + self.assertEqual(json.loads((output / "release.json").read_text()), { + "schema": 1, "platform": "web", "buildId": identity}) + + def test_old_web_cache_without_release_metadata_is_rebuilt(self): + runner = Runner() + cache = Path(self.home.name) / "cache" + first = app_build.build("web", runner=runner, cache=cache, log=lambda *_: None) + (first / "release.json").unlink() + runner.calls.clear() + rebuilt = app_build.build("web", runner=runner, cache=cache, log=lambda *_: None) + self.assertEqual(rebuilt, first) + self.assertTrue((rebuilt / "release.json").is_file()) + self.assertTrue(any(call[:2] == ["docker", "run"] for call in runner.calls)) + + def test_web_identity_changes_when_sources_change(self): + app = Path(self.home.name) / "app" + (app / "lib").mkdir(parents=True) + (app / ".flutter-version").write_text("3.0") + before = app_build.digest("web", app) + (app / "lib/main.dart").write_text("changed") + after = app_build.digest("web", app) + self.assertNotEqual(app_build.web_build_id(before), app_build.web_build_id(after)) if __name__ == "__main__":