diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000000000..ade159ddabcb9 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,43 @@ +name: ci + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +on: + pull_request: + push: + branches: + - main + +jobs: + test: + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest, windows-latest] + + runs-on: ${{ matrix.os }} + + name: ๐Ÿงช Test + + env: + FLUTTER_STORAGE_BASE_URL: https://download.shorebird.dev + + steps: + - name: ๐Ÿ“š Git Checkout + uses: actions/checkout@v4 + with: + # Fetch all branches and tags to ensure that Flutter can determine its version + fetch-depth: 0 + + - name: ๐ŸŽฏ Setup Dart + uses: dart-lang/setup-dart@v1 + + - name: ๐Ÿ“ฆ Install Dependencies + run: | + dart pub get -C ./dev/bots + dart pub get -C ./dev/tools + + - name: ๐Ÿงช Run Tests + run: dart ./dev/bots/test.dart diff --git a/.github/workflows/freeze.yml b/.github/workflows/freeze.yml index 11c904b149de7..d3f5355f82d4c 100644 --- a/.github/workflows/freeze.yml +++ b/.github/workflows/freeze.yml @@ -20,16 +20,14 @@ jobs: name: Check Code Freeze runs-on: ubuntu-latest if: ${{ github.repository == 'flutter/flutter' }} - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: ${{ github.event.pull_request.head.sha }} + steps: - name: Check for changes in frozen folders + if: github.event_name != 'merge_group' uses: dorny/paths-filter@v3 id: filter with: + token: ${{ github.token }} filters: | frozen: - 'packages/flutter/lib/src/material/**' @@ -46,27 +44,9 @@ jobs: - 'packages/flutter/test_fixes/cupertino/**' - name: Fail on frozen changes - # This step only runs if the 'frozen' filter matched AND the override label is NOT present - if: | - steps.filter.outputs.frozen == 'true' && - !contains(github.event.pull_request.labels.*.name, 'override: code freeze') + if: github.event_name != 'merge_group' && steps.filter.outputs.frozen == 'true' && !contains(github.event.pull_request.labels.*.name, 'override code freeze') run: | echo "Error: Code changes detected during the current code freeze." - echo "The following paths are currently frozen:" - echo " - packages/flutter/lib/src/material/" - echo " - packages/flutter/lib/src/cupertino/" - echo " - (and associated tests/examples)" - echo "" - echo "If this is a critical fix that must land during the freeze, please file an issue for team-design." - echo "Information on this code freeze: https://github.com/flutter/flutter/issues/184093" + echo "If this is a critical fix that must land, please file an issue for team-design." + echo "Info: https://github.com/flutter/flutter/issues/184093" exit 1 - - - name: Pass if overridden - if: | - steps.filter.outputs.frozen == 'true' && - contains(github.event.pull_request.labels.*.name, 'override: code freeze') - run: echo "Code freeze overridden by override: code freeze label." - - - name: Pass if no frozen changes - if: steps.filter.outputs.frozen == 'false' - run: echo "No changes to frozen code." diff --git a/.github/workflows/shorebird_ci.yml b/.github/workflows/shorebird_ci.yml new file mode 100644 index 0000000000000..2d2905405521d --- /dev/null +++ b/.github/workflows/shorebird_ci.yml @@ -0,0 +1,130 @@ +name: shorebird_ci + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +on: + pull_request: + push: + branches: + - shorebird/dev + +jobs: + # NOTE: Windows is not included because shorebird_tests depends on + # flutter_flavorizr which requires Xcode. Additionally, flutter_tools + # tests on Windows would need additional setup work. + + flutter-tools-tests: + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest] + + runs-on: ${{ matrix.os }} + + name: ๐Ÿ› ๏ธ Flutter Tools Tests (${{ matrix.os }}) + + steps: + - name: ๐Ÿ“š Git Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: ๐ŸŽฏ Setup Dart + uses: dart-lang/setup-dart@v1 + + - name: ๐Ÿฆ Run Flutter Tools Tests + run: ../../bin/flutter test test/general.shard + working-directory: packages/flutter_tools + + shorebird-android-tests: + # Android tests run on Ubuntu (faster runners, no macOS needed) + runs-on: ubuntu-latest + + name: ๐Ÿค– Shorebird Android Tests + + steps: + - name: ๐Ÿ“š Git Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: ๐ŸŽฏ Setup Dart + uses: dart-lang/setup-dart@v1 + + - uses: actions/setup-java@v4 + with: + distribution: "zulu" + java-version: "17" + + - name: ๐Ÿ“ฆ Cache Gradle + uses: actions/cache@v4 + with: + path: | + ~/.gradle/caches + ~/.gradle/wrapper + key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }} + restore-keys: | + ${{ runner.os }}-gradle- + + - name: ๐Ÿค– Run Android Tests + run: dart test test/base_test.dart test/android_test.dart + working-directory: packages/shorebird_tests + + shorebird-ios-tests: + # iOS tests require macOS for Xcode + runs-on: macos-latest + + name: ๐ŸŽ Shorebird iOS + Android Smoke Tests + + steps: + - name: ๐Ÿ“š Git Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: ๐ŸŽฏ Setup Dart + uses: dart-lang/setup-dart@v1 + + - uses: actions/setup-java@v4 + with: + distribution: "zulu" + java-version: "17" + + - name: ๐Ÿ“ฆ Cache Gradle + uses: actions/cache@v4 + with: + path: | + ~/.gradle/caches + ~/.gradle/wrapper + key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }} + restore-keys: | + ${{ runner.os }}-gradle- + + - name: ๐ŸŽ Run iOS Tests + run: dart test test/base_test.dart test/ios_test.dart + working-directory: packages/shorebird_tests + + - name: ๐Ÿค– Run Android Smoke Test (macOS) + # Quick sanity check that Android builds work on macOS too + run: dart test test/android_test.dart --name "can build an apk" + working-directory: packages/shorebird_tests + env: + # Enables streaming subprocess output for debugging timeouts. + VERBOSE: "1" + + # Aggregator required by branch protection on shorebird/dev so the rule can + # list a single context ("ci") instead of every matrix-expanded job above. + ci: + if: always() + needs: + - flutter-tools-tests + - shorebird-android-tests + - shorebird-ios-tests + runs-on: ubuntu-latest + steps: + - if: ${{ contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') }} + run: | + echo "One or more upstream jobs failed or were cancelled." + exit 1 diff --git a/DEPS b/DEPS index e0e46d3e86d2a..52c2d7eb2ddf8 100644 --- a/DEPS +++ b/DEPS @@ -16,6 +16,14 @@ vars = { 'skia_git': 'https://skia.googlesource.com', 'llvm_git': 'https://llvm.googlesource.com', 'skia_revision': 'e9ed4fc9f1544c58d8a9347c1fc9471d8dd7c465', + # Our dart-sdk fork revision. Used both for the third_party/dart source + # clone (gen_snapshot etc.) and to key the macos-arm64 prebuilt object in + # GCS, so source and prebuilt stay the same revision. A published prebuilt + # must exist for this sha (see the macos-arm64 dart-sdk gcs dep below). + "dart_sdk_revision": "db98bdaa9d8f8e2250ff83d24abcaf775807244c", + "dart_sdk_git": "git@github.com:shorebirdtech/dart-sdk.git", + "updater_git": "https://github.com/shorebirdtech/updater.git", + "updater_rev": "1f85c4ab1ee5b540269b9859c75e1bffbb9050c7", # WARNING: DO NOT EDIT canvaskit_cipd_instance MANUALLY # See `lib/web_ui/README.md` for how to roll CanvasKit to a new version. @@ -284,7 +292,7 @@ deps = { 'https://boringssl.googlesource.com/boringssl.git' + '@' + Var('dart_boringssl_rev'), 'engine/src/flutter/third_party/dart': - Var('dart_git') + '/sdk.git' + '@' + Var('dart_revision'), + Var('dart_sdk_git') + '@' + Var('dart_sdk_revision'), # WARNING: Unused Dart dependencies in the list below till "WARNING:" marker are removed automatically - see create_updated_flutter_deps.py. @@ -396,14 +404,27 @@ deps = { 'dep_type': 'cipd', 'condition': 'host_os == "mac" and download_dart_sdk' }, + # Consume Shorebird's own published Dart SDK (built from + # shorebirdtech/dart-sdk) from GCS instead of Google's CIPD prebuilt, so + # the engine builds against our Dart fork and skips rebuilding it from + # source (see shards/macos.json: mac-arm64 drops --no-prebuilt-dart-sdk). + # The object is keyed by Var('dart_sdk_revision') so it tracks the same + # fork revision as the source clone above; sha256sum/size_bytes pin the + # specific artifact's content and must be updated whenever that revision + # bumps. tar.gz (not zip) so gclient's extraction preserves the executable + # bit on bin/dart etc. The bot reads this private bucket via a keyless-WIF + # reader SA (shorebirdtech/_build_engine sync.yaml). 'engine/src/flutter/prebuilts/macos-arm64/dart-sdk': { - 'packages': [ + 'bucket': 'shorebird-dart-sdk-prebuilt', + 'objects': [ { - 'package': 'flutter/dart-sdk/mac-arm64', - 'version': 'git_revision:'+Var('dart_revision') + 'object_name': Var('dart_sdk_revision') + '/dart-sdk-darwin-arm64.tar.gz', + 'sha256sum': '7fd7de2d929d6729975bed3cb18ded754fcc023b35f00c61295fb772f63add6c', + 'size_bytes': 205231858, + 'generation': 1781242258444863, } ], - 'dep_type': 'cipd', + 'dep_type': 'gcs', 'condition': 'host_os == "mac" and download_dart_sdk' }, 'engine/src/flutter/prebuilts/windows-x64/dart-sdk': { @@ -481,6 +502,9 @@ deps = { 'engine/src/flutter/third_party/ocmock': Var('flutter_git') + '/third_party/ocmock' + '@' + Var('ocmock_rev'), + 'engine/src/flutter/third_party/updater': + Var('updater_git') + '@' + Var('updater_rev'), + 'engine/src/flutter/third_party/libjpeg-turbo/src': Var('flutter_git') + '/third_party/libjpeg-turbo' + '@' + '0fb821f3b2e570b2783a94ccd9a2fb1f4916ae9f', diff --git a/bin/internal/engine.version b/bin/internal/engine.version index 6ec72f90abd60..5d6ef34464a25 100644 --- a/bin/internal/engine.version +++ b/bin/internal/engine.version @@ -1 +1 @@ -69c8c61792f04cc809dfef0c910414fb9afc06cd +e1eaecbcac6d9a32cb5590c646e21cf21252cf19 diff --git a/bin/internal/update_dart_sdk.ps1 b/bin/internal/update_dart_sdk.ps1 index 318ef62add853..a588137834eb6 100644 --- a/bin/internal/update_dart_sdk.ps1 +++ b/bin/internal/update_dart_sdk.ps1 @@ -41,7 +41,7 @@ if ((Test-Path $engineStamp) -and ($engineVersion -eq (Get-Content $engineStamp) $dartSdkBaseUrl = $Env:FLUTTER_STORAGE_BASE_URL if (-not $dartSdkBaseUrl) { - $dartSdkBaseUrl = "https://storage.googleapis.com" + $dartSdkBaseUrl = "https://download.shorebird.dev" } if ($engineRealm) { $dartSdkBaseUrl = "$dartSdkBaseUrl/$engineRealm" diff --git a/bin/internal/update_dart_sdk.sh b/bin/internal/update_dart_sdk.sh index 41f419112e1f4..03c56c9e8079c 100755 --- a/bin/internal/update_dart_sdk.sh +++ b/bin/internal/update_dart_sdk.sh @@ -127,7 +127,7 @@ if [ ! -f "$ENGINE_STAMP" ] || [ "$ENGINE_VERSION" != "$(< "$ENGINE_STAMP")" ]; FIND=find fi - DART_SDK_BASE_URL="${FLUTTER_STORAGE_BASE_URL:-https://storage.googleapis.com}${ENGINE_REALM:+/$ENGINE_REALM}" + DART_SDK_BASE_URL="${FLUTTER_STORAGE_BASE_URL:-https://download.shorebird.dev}${ENGINE_REALM:+/$ENGINE_REALM}" DART_SDK_URL="$DART_SDK_BASE_URL/flutter_infra_release/flutter/$ENGINE_VERSION/$DART_ZIP_NAME" # if the sdk path exists, copy it to a temporary location diff --git a/dev/bots/post_process_docs.dart b/dev/bots/post_process_docs.dart index bfb05542e55e8..c6dffc4e41ad3 100644 --- a/dev/bots/post_process_docs.dart +++ b/dev/bots/post_process_docs.dart @@ -38,7 +38,7 @@ Future postProcess() async { await runProcessWithValidations([ 'curl', '-L', - 'https://storage.googleapis.com/flutter_infra_release/flutter/$revision/api_docs.zip', + 'https://download.shorebird.dev/flutter_infra_release/flutter/$revision/api_docs.zip', '--output', zipDestination, '--fail', diff --git a/dev/bots/unpublish_package.dart b/dev/bots/unpublish_package.dart index 022111bd16ed9..03cc3a24d2fab 100644 --- a/dev/bots/unpublish_package.dart +++ b/dev/bots/unpublish_package.dart @@ -23,7 +23,7 @@ import 'package:process/process.dart'; const String gsBase = 'gs://flutter_infra_release'; const String releaseFolder = '/releases'; const String gsReleaseFolder = '$gsBase$releaseFolder'; -const String baseUrl = 'https://storage.googleapis.com/flutter_infra_release'; +const String baseUrl = 'https://download.shorebird.dev/flutter_infra_release'; /// Exception class for when a process fails to run, so we can catch /// it and provide something more readable than a stack trace. diff --git a/dev/integration_tests/pure_android_host_apps/android_host_app_v2_embedding/settings.gradle b/dev/integration_tests/pure_android_host_apps/android_host_app_v2_embedding/settings.gradle index 5368feb7ecde7..66364cdf4a190 100644 --- a/dev/integration_tests/pure_android_host_apps/android_host_app_v2_embedding/settings.gradle +++ b/dev/integration_tests/pure_android_host_apps/android_host_app_v2_embedding/settings.gradle @@ -7,7 +7,7 @@ dependencyResolutionManagement { repositories { google() mavenCentral() - def flutterStorageUrl = System.getenv("FLUTTER_STORAGE_BASE_URL") ?: "https://storage.googleapis.com" + def flutterStorageUrl = System.getenv("FLUTTER_STORAGE_BASE_URL") ?: "https://download.shorebird.dev" maven { url = uri("$flutterStorageUrl/download.flutter.io") } diff --git a/dev/integration_tests/pure_android_host_apps/host_app_kotlin_gradle_dsl/settings.gradle.kts b/dev/integration_tests/pure_android_host_apps/host_app_kotlin_gradle_dsl/settings.gradle.kts index f6d75bce11757..da25b49a46f7f 100644 --- a/dev/integration_tests/pure_android_host_apps/host_app_kotlin_gradle_dsl/settings.gradle.kts +++ b/dev/integration_tests/pure_android_host_apps/host_app_kotlin_gradle_dsl/settings.gradle.kts @@ -16,7 +16,7 @@ dependencyResolutionManagement { repositories { google() mavenCentral() - val flutterStorageUrl = System.getenv("FLUTTER_STORAGE_BASE_URL") ?: "https://storage.googleapis.com" + val flutterStorageUrl = System.getenv("FLUTTER_STORAGE_BASE_URL") ?: "https://download.shorebird.dev" maven("$flutterStorageUrl/download.flutter.io") } } diff --git a/dev/tools/create_api_docs.dart b/dev/tools/create_api_docs.dart index 47d336587ca0a..6c65430bf6973 100644 --- a/dev/tools/create_api_docs.dart +++ b/dev/tools/create_api_docs.dart @@ -923,8 +923,8 @@ class PlatformDocGenerator { for (final String platform in kPlatformDocs.keys) { final String zipFile = kPlatformDocs[platform]!.zipName; - final url = - 'https://storage.googleapis.com/${realm}flutter_infra_release/flutter/$engineRevision/$zipFile'; + final String url = + 'https://download.shorebird.dev/${realm}flutter_infra_release/flutter/$engineRevision/$zipFile'; await _extractDocs(url, platform, kPlatformDocs[platform]!, outputDir); } } diff --git a/engine/src/build/config/compiler/BUILD.gn b/engine/src/build/config/compiler/BUILD.gn index e06cf625abe8f..a7f5a3ae35bb9 100644 --- a/engine/src/build/config/compiler/BUILD.gn +++ b/engine/src/build/config/compiler/BUILD.gn @@ -443,7 +443,10 @@ config("compiler") { # Example PR: https://github.com/dart-lang/native/pull/1615 ldflags += [ "-Wl,--no-undefined", - "-Wl,--exclude-libs,ALL", + + # TODO: Terrible hack, but otherwise libupdater.a symbols can't + # be exported from libflutter.so, even when added to android_exports.lst. + # "-Wl,--exclude-libs,ALL", # Enable identical code folding to reduce size. "-Wl,--icf=all", @@ -646,6 +649,8 @@ config("runtime_library") { ldflags += [ "-Wl,--warn-shared-textrel" ] libs += [ + # Rust requires libunwind. + "unwind", "c", "dl", "m", @@ -660,6 +665,9 @@ config("runtime_library") { } else if (current_cpu == "x86") { current_android_cpu = "i686" } + # libunwind.a is located in the respective android cpu subdirectories. + # The clang version needs to match the version in the lib_dirs line above. + lib_dirs += [ "${android_toolchain_root}/lib/clang/19/lib/linux/${current_android_cpu}/" ] libs += [ "clang_rt.builtins-${current_android_cpu}-android" ] } diff --git a/engine/src/build/toolchain/win/BUILD.gn b/engine/src/build/toolchain/win/BUILD.gn index 45a98b1ecd64b..b5073fb22277b 100644 --- a/engine/src/build/toolchain/win/BUILD.gn +++ b/engine/src/build/toolchain/win/BUILD.gn @@ -204,8 +204,11 @@ template("msvc_toolchain") { expname = "${dllname}.exp" pdbname = "${dllname}.pdb" rspfile = "${dllname}.rsp" + # .def files are used to export symbols from the DLL. This arg will be + # removed by the python tool wrapper if the .def file doesn't exist. + deffile = "${dllname}.def" - link_command = "\"$python_path\" $tool_wrapper_path link-wrapper $env False link.exe /nologo /IMPLIB:$libname /DLL /OUT:$dllname /PDB:${dllname}.pdb @$rspfile" + link_command = "\"$python_path\" $tool_wrapper_path link-wrapper $env False link.exe /nologo /IMPLIB:$libname /DLL /OUT:$dllname /PDB:${dllname}.pdb /DEF:$deffile @$rspfile" # TODO(brettw) support manifests #manifest_command = "\"$python_path\" $tool_wrapper_path manifest-wrapper $env mt.exe -nologo -manifest $manifests -out:${dllname}.manifest" diff --git a/engine/src/build/toolchain/win/tool_wrapper.py b/engine/src/build/toolchain/win/tool_wrapper.py index b4fc8485ffa17..7866c6122d832 100644 --- a/engine/src/build/toolchain/win/tool_wrapper.py +++ b/engine/src/build/toolchain/win/tool_wrapper.py @@ -121,6 +121,17 @@ def ExecLinkWrapper(self, arch, use_separate_mspdbsrv, *args): self._UseSeparateMspdbsrv(env, args) if sys.platform == 'win32': args = list(args) # *args is a tuple by default, which is read-only. + + # Remove the /DEF arg if not provided. We would ideally be able to do this + # in build\toolchain\win\BUILD.gn, but there doesn't seem to be a way to + # conditionally add args to the command line based on whether a file exists + # or not, so we do it here instead. + def_arg_prefix = "/DEF:" + for arg in args: + if arg.startswith(def_arg_prefix): + def_file = arg[len(def_arg_prefix):] + if not os.path.exists(def_file): + args.remove(arg) args[0] = args[0].replace('/', '\\') # https://docs.python.org/2/library/subprocess.html: # "On Unix with shell=True [...] if args is a sequence, the first item diff --git a/engine/src/flutter/BUILD.gn b/engine/src/flutter/BUILD.gn index 7a4d1eaab385b..d3ca9a9b38f39 100644 --- a/engine/src/flutter/BUILD.gn +++ b/engine/src/flutter/BUILD.gn @@ -119,6 +119,9 @@ group("flutter") { # path_ops "//flutter/tools/path_ops", + + # Built alongside gen_snapshot arm64 targets. + "$dart_src/runtime/bin:analyze_snapshot", ] if (host_os == "linux" || host_os == "mac") { @@ -128,13 +131,6 @@ group("flutter") { ] } - if (host_os == "linux") { - public_deps += [ - # Built alongside gen_snapshot for 64 bit targets - "$dart_src/runtime/bin:analyze_snapshot", - ] - } - if (full_dart_sdk) { public_deps += [ "//flutter/web_sdk" ] } @@ -214,6 +210,7 @@ group("unittests") { "//flutter/runtime:no_dart_plugin_registrant_unittests", "//flutter/runtime:runtime_unittests", "//flutter/shell/common:shell_unittests", + "//flutter/shell/common/shorebird:shorebird_unittests", "//flutter/shell/geometry:geometry_unittests", "//flutter/shell/gpu:gpu_surface_unittests", "//flutter/shell/platform/embedder:embedder_a11y_unittests", @@ -345,3 +342,19 @@ if (host_os == "win") { outputs = [ "$root_build_dir/gen_snapshot/gen_snapshot.exe" ] } } + +# A top-level target for analyze_snapshot, modeled after the gen_snapshot +# target above. +if (host_os == "win") { + _analyze_snapshot_target = + "$dart_src/runtime/bin:analyze_snapshot($host_toolchain)" + + copy("analyze_snapshot") { + deps = [ _analyze_snapshot_target ] + + analyze_snapshot_out_dir = + get_label_info(_analyze_snapshot_target, "root_out_dir") + sources = [ "$analyze_snapshot_out_dir/analyze_snapshot.exe" ] + outputs = [ "$root_build_dir/analyze_snapshot/analyze_snapshot.exe" ] + } +} diff --git a/engine/src/flutter/build/archives/BUILD.gn b/engine/src/flutter/build/archives/BUILD.gn index 4f902010d21dc..55700433541db 100644 --- a/engine/src/flutter/build/archives/BUILD.gn +++ b/engine/src/flutter/build/archives/BUILD.gn @@ -45,6 +45,7 @@ generated_file("artifacts_entitlement_config") { if (build_engine_artifacts) { zip_bundle("artifacts") { deps = [ + "$dart_src/runtime/bin:analyze_snapshot", "$dart_src/runtime/bin:gen_snapshot", "//flutter/flutter_frontend_server:frontend_server", "//flutter/impeller/compiler:impellerc", @@ -142,6 +143,10 @@ if (build_engine_artifacts) { if (host_os == "mac") { deps += [ ":artifacts_entitlement_config" ] files += [ + { + source = "$root_out_dir/analyze_snapshot$exe" + destination = "analyze_snapshot$exe" + }, { source = "$target_gen_dir/entitlements.txt" destination = "entitlements.txt" @@ -320,14 +325,25 @@ if (is_mac) { } if (host_os == "win") { + # This rule archives both gen_snapshot *and* analyze_snapshot. The name is + # misleading. We (shorebird) have updated this rule to include + # analyze_snapshot but did not update the name because it is referenced + # elsewhere in the tooling. zip_bundle("archive_win_gen_snapshot") { - deps = [ "//flutter:gen_snapshot" ] + deps = [ + "//flutter:analyze_snapshot", + "//flutter:gen_snapshot", + ] output = "$full_target_platform_name-$flutter_runtime_mode/windows-x64.zip" files = [ { source = "$root_out_dir/gen_snapshot/gen_snapshot.exe" destination = "gen_snapshot.exe" }, + { + source = "$root_out_dir/analyze_snapshot/analyze_snapshot.exe" + destination = "analyze_snapshot.exe" + }, ] } } diff --git a/engine/src/flutter/build/dart/tools/dart_pkg.py b/engine/src/flutter/build/dart/tools/dart_pkg.py index c60e3e02431e4..5b3e0a8b5f3ee 100755 --- a/engine/src/flutter/build/dart/tools/dart_pkg.py +++ b/engine/src/flutter/build/dart/tools/dart_pkg.py @@ -163,7 +163,10 @@ def main(): for source in args.package_sources: relative_source = os.path.relpath(source, common_source_prefix) target = os.path.join(target_dir, relative_source) - copy(source, target) + try: + copy(source, target) + except shutil.SameFileError: + pass # Copy sdk-ext sources into pkg directory sdk_ext_dir = os.path.join(target_dir, 'sdk_ext') @@ -179,7 +182,10 @@ def main(): for source in args.sdk_ext_files: relative_source = os.path.relpath(source, common_source_prefix) target = os.path.join(sdk_ext_dir, relative_source) - copy(source, target) + try: + copy(source, target) + except shutil.SameFileError: + pass # Write stamp file. with open(args.stamp_file, 'w'): diff --git a/engine/src/flutter/build/zip_bundle.gni b/engine/src/flutter/build/zip_bundle.gni index 51e72df0ad854..707ab046d3637 100644 --- a/engine/src/flutter/build/zip_bundle.gni +++ b/engine/src/flutter/build/zip_bundle.gni @@ -55,7 +55,7 @@ template("zip_bundle") { license_path = rebase_path("//flutter/sky/packages/sky_engine/LICENSE", "//flutter") git_url = "https://github.com/flutter/engine/tree/$engine_version" - sky_engine_url = "https://storage.googleapis.com/flutter_infra_release/flutter/$engine_version/sky_engine.zip" + sky_engine_url = "https://download.shorebird.dev/flutter_infra_release/flutter/$engine_version/sky_engine.zip" outputs = [ license_readme ] contents = [ "# $target_name", diff --git a/engine/src/flutter/common/config.gni b/engine/src/flutter/common/config.gni index 318d305bd37fc..35ec1c7e058dc 100644 --- a/engine/src/flutter/common/config.gni +++ b/engine/src/flutter/common/config.gni @@ -67,6 +67,14 @@ if (slimpeller) { feature_defines_list += [ "SLIMPELLER=1" ] } +if (is_android || is_ios || is_linux || is_mac || is_win) { + feature_defines_list += [ "SHOREBIRD_PLATFORM_SUPPORTED=1" ] +} + +if (is_ios) { + feature_defines_list += [ "SHOREBIRD_USE_INTERPRETER=1" ] +} + if (is_ios || is_mac) { flutter_cflags_objc = [ "-Werror=overriding-method-mismatch", diff --git a/engine/src/flutter/impeller/toolkit/interop/README.md b/engine/src/flutter/impeller/toolkit/interop/README.md index 6e75a53a11775..279379217e143 100644 --- a/engine/src/flutter/impeller/toolkit/interop/README.md +++ b/engine/src/flutter/impeller/toolkit/interop/README.md @@ -27,7 +27,7 @@ A single-header C API for 2D graphics and text rendering. [Impeller](../../READM Users may plug in a custom toolchain into the Flutter Engine build system to build the `libimpeller.so` dynamic library. However, for the common platforms, the CI bots upload a tarball containing the library and headers. This URL for the SDK tarball for a particular platform can be constructed as follows: ```sh -https://storage.googleapis.com/flutter_infra_release/flutter/$FLUTTER_SHA/$PLATFORM_ARCH/impeller_sdk.zip +https://download.shorebird.dev/flutter_infra_release/flutter/$FLUTTER_SHA/$PLATFORM_ARCH/impeller_sdk.zip ``` The `$FLUTTER_SHA` is the Git hash in the [Flutter repository](https://github.com/flutter/flutter). The `$PLATFORM_ARCH` can be determined from the table below. diff --git a/engine/src/flutter/lib/snapshot/BUILD.gn b/engine/src/flutter/lib/snapshot/BUILD.gn index e4b52cac21985..e495383162a89 100644 --- a/engine/src/flutter/lib/snapshot/BUILD.gn +++ b/engine/src/flutter/lib/snapshot/BUILD.gn @@ -35,7 +35,10 @@ group("generate_snapshot_bins") { if (host_os == "mac" && (target_os == "mac" || target_os == "ios" || target_os == "android")) { # For macOS target builds: needed for both target CPUs (arm64, x64). - public_deps += [ ":create_macos_gen_snapshots" ] + public_deps += [ + ":create_macos_analyze_snapshots", + ":create_macos_gen_snapshots", + ] } else if (host_os == "mac" && (target_cpu == "arm" || target_cpu == "arm64")) { # For iOS, Android target builds: all AOT target CPUs are arm/arm64. @@ -46,9 +49,11 @@ group("generate_snapshot_bins") { public_deps = [ "$dart_src/runtime/bin:gen_snapshot($host_toolchain)" ] } - # Build analyze_snapshot for 64-bit target CPUs. - if (host_os == "linux" && (target_cpu == "x64" || target_cpu == "arm64" || - target_cpu == "riscv64")) { + # Build analyze_snapshot for 64-bit target CPUs on linux. + # Or always targeting arm64 for Shorebird builds. + if ((host_os == "linux" && + (target_cpu == "x64" || target_cpu == "riscv64")) || + target_cpu == "arm64") { public_deps += [ "$dart_src/runtime/bin:analyze_snapshot($host_toolchain)" ] } } @@ -257,6 +262,69 @@ if (host_os == "mac" && ":create_macos_gen_snapshot_x64${gen_snapshot_suffix}", ] } + + # Added by shorebird. + # analyze_snapshot targets below were copied from the gen_snapshot targets + # above to allow us to include analyze_snapshot in the artifacts generated + # for create_ios_framework.py. + template("build_mac_analyze_snapshot") { + assert(defined(invoker.host_arch)) + host_cpu = invoker.host_arch + + build_toolchain = "//build/toolchain/mac:clang_$host_cpu" + analyze_snapshot_target_name = "analyze_snapshot" + + # At this point, the gen_snapshot equivalent changes + # gen_ snapshot_target_name to "gen_snapshot_host_targeting_host". There is + # no equivalent for analyze_snapshot, so we don't do that here. + # + # It's unclear whether we need to do so now, but we didn't previously, so + # we're not doing it now until we have a reason to. + + analyze_snapshot_target = + "$dart_src/runtime/bin:$analyze_snapshot_target_name($build_toolchain)" + + copy(target_name) { + # The toolchain-specific output directory. For cross-compiles, this is a + # clang-x64 or clang-arm64 subdirectory of the top-level build directory. + output_dir = get_label_info(analyze_snapshot_target, "root_out_dir") + + sources = [ "${output_dir}/${analyze_snapshot_target_name}" ] + outputs = [ + "${root_out_dir}/artifacts_$host_cpu/analyze_snapshot_${target_cpu}", + ] + deps = [ analyze_snapshot_target ] + } + } + + build_mac_analyze_snapshot( + "create_macos_analyze_snapshot_arm64_${target_cpu}") { + host_arch = "arm64" + } + + build_mac_analyze_snapshot( + "create_macos_analyze_snapshot_x64_${target_cpu}") { + host_arch = "x64" + } + + action("create_macos_analyze_snapshots") { + script = "//flutter/sky/tools/create_macos_binary.py" + outputs = [ "${root_out_dir}/analyze_snapshot_${target_cpu}" ] + args = [ + "--in-arm64", + rebase_path( + "${root_out_dir}/artifacts_arm64/analyze_snapshot_${target_cpu}"), + "--in-x64", + rebase_path( + "${root_out_dir}/artifacts_x64/analyze_snapshot_${target_cpu}"), + "--out", + rebase_path("${root_out_dir}/analyze_snapshot_${target_cpu}"), + ] + deps = [ + ":create_macos_analyze_snapshot_arm64_${target_cpu}", + ":create_macos_analyze_snapshot_x64_${target_cpu}", + ] + } } source_set("snapshot") { diff --git a/engine/src/flutter/lib/web_ui/dev/steps/copy_artifacts_step.dart b/engine/src/flutter/lib/web_ui/dev/steps/copy_artifacts_step.dart index e95e0b3f0a4b3..5d94008deed48 100644 --- a/engine/src/flutter/lib/web_ui/dev/steps/copy_artifacts_step.dart +++ b/engine/src/flutter/lib/web_ui/dev/steps/copy_artifacts_step.dart @@ -55,8 +55,8 @@ class CopyArtifactsStep implements PipelineStep { 'Could not generate artifact bucket url for unknown realm.', ), }; - final url = Uri.https( - 'storage.googleapis.com', + final Uri url = Uri.https( + 'download.shorebird.dev', '${realmComponent}flutter_infra_release/flutter/${realm == LuciRealm.Try ? gitRevision : contentHash}/flutter-web-sdk.zip', ); final http.Response response = await http.Client().get(url); diff --git a/engine/src/flutter/runtime/BUILD.gn b/engine/src/flutter/runtime/BUILD.gn index 2f7aea3d0d107..259ad3b4b697f 100644 --- a/engine/src/flutter/runtime/BUILD.gn +++ b/engine/src/flutter/runtime/BUILD.gn @@ -114,10 +114,15 @@ source_set("runtime") { "//flutter/fml", "//flutter/lib/io", "//flutter/shell/common:display", + "//flutter/shell/common/shorebird:updater", "//flutter/skia", "//flutter/third_party/tonic", "//flutter/txt", ] + + if (is_ios) { + deps += [ "//flutter/runtime/shorebird:patch_cache" ] + } } if (enable_unittests) { diff --git a/engine/src/flutter/runtime/dart_isolate.cc b/engine/src/flutter/runtime/dart_isolate.cc index 29fa000131c8e..4fcacccd3ad26 100644 --- a/engine/src/flutter/runtime/dart_isolate.cc +++ b/engine/src/flutter/runtime/dart_isolate.cc @@ -1108,7 +1108,9 @@ Dart_Isolate DartIsolate::DartIsolateGroupCreateCallback( advisory_script_entrypoint, parent_group_data.GetChildIsolatePreparer(), parent_group_data.GetIsolateCreateCallback(), - parent_group_data.GetIsolateShutdownCallback()))); + parent_group_data.GetIsolateShutdownCallback(), + nullptr // native_assets_manager + ))); TaskRunners null_task_runners(advisory_script_uri, /* platform= */ nullptr, diff --git a/engine/src/flutter/runtime/dart_snapshot.cc b/engine/src/flutter/runtime/dart_snapshot.cc index 198a2e75a7edc..fc70f63a346f2 100644 --- a/engine/src/flutter/runtime/dart_snapshot.cc +++ b/engine/src/flutter/runtime/dart_snapshot.cc @@ -6,6 +6,7 @@ #include +#include #include "flutter/fml/native_library.h" #include "flutter/fml/paths.h" #include "flutter/fml/trace_event.h" @@ -13,6 +14,11 @@ #include "flutter/runtime/dart_vm.h" #include "third_party/dart/runtime/include/dart_api.h" +#if SHOREBIRD_USE_INTERPRETER +#include "flutter/runtime/shorebird/patch_cache.h" // nogncheck +#endif +#include "flutter/shell/common/shorebird/updater.h" // nogncheck + namespace flutter { const char* DartSnapshot::kVMDataSymbol = "kDartVmSnapshotData"; @@ -145,7 +151,20 @@ static std::shared_ptr ResolveIsolateData( nullptr, // release_func true // dontneed_safe ); -#else // DART_SNAPSHOT_STATIC_LINK +#else // DART_SNAPSHOT_STATIC_LINK + // Tell the Rust updater we're booting from whatever patch it selected. + // This copies next_boot โ†’ current_boot in the Rust state. The call is + // guarded inside Updater to execute at most once per process โ€” see the + // Updater class comment for why this matters in add-to-app and + // FlutterEngineGroup scenarios. + shorebird::Updater::Instance().ReportLaunchStart(); +#if SHOREBIRD_USE_INTERPRETER + // Try loading from a Shorebird patch first. + if (auto mapping = TryLoadFromPatch(settings.application_library_paths, + DartSnapshot::kIsolateDataSymbol)) { + return mapping; + } +#endif // SHOREBIRD_USE_INTERPRETER return SearchMapping( settings.isolate_snapshot_data, // embedder_mapping_callback settings.isolate_snapshot_data_path, // file_path @@ -165,7 +184,15 @@ static std::shared_ptr ResolveIsolateInstructions( nullptr, // release_func true // dontneed_safe ); -#else // DART_SNAPSHOT_STATIC_LINK +#else // DART_SNAPSHOT_STATIC_LINK +#if SHOREBIRD_USE_INTERPRETER + // Try loading from a Shorebird patch first. + if (auto mapping = + TryLoadFromPatch(settings.application_library_paths, + DartSnapshot::kIsolateInstructionsSymbol)) { + return mapping; + } +#endif // SHOREBIRD_USE_INTERPRETER return SearchMapping( settings.isolate_snapshot_instr, // embedder_mapping_callback settings.isolate_snapshot_instr_path, // file_path diff --git a/engine/src/flutter/runtime/runtime_controller.cc b/engine/src/flutter/runtime/runtime_controller.cc index 53f1684eea6ae..91cb3fee6dd4a 100644 --- a/engine/src/flutter/runtime/runtime_controller.cc +++ b/engine/src/flutter/runtime/runtime_controller.cc @@ -123,6 +123,24 @@ std::unique_ptr RuntimeController::Clone() const { ); } +// Shorebird: |Clone|, except the new controller launches isolates from +// |isolate_snapshot| instead of this controller's snapshot. Used by the +// production hot restart to relaunch the root isolate from a newly +// installed patch; see shell/common/shorebird/hot_restart.h. +std::unique_ptr RuntimeController::CloneWithSnapshot( + fml::RefPtr isolate_snapshot) const { + return std::make_unique(client_, // + vm_, // + std::move(isolate_snapshot), // + idle_notification_callback_, // + platform_data_, // + isolate_create_callback_, // + isolate_shutdown_callback_, // + persistent_isolate_data_, // + context_ // + ); +} + bool RuntimeController::FlushRuntimeStateToIsolate() { FML_DCHECK(!has_flushed_runtime_state_) << "FlushRuntimeStateToIsolate is called more than once somehow."; diff --git a/engine/src/flutter/runtime/runtime_controller.h b/engine/src/flutter/runtime/runtime_controller.h index 38a279bf3f511..d0cf5ff6d1039 100644 --- a/engine/src/flutter/runtime/runtime_controller.h +++ b/engine/src/flutter/runtime/runtime_controller.h @@ -184,6 +184,18 @@ class RuntimeController : public PlatformConfigurationClient, /// std::unique_ptr Clone() const; + //---------------------------------------------------------------------------- + /// @brief Shorebird: like |Clone|, but the cloned controller launches + /// isolates from |isolate_snapshot| instead of this + /// controller's snapshot. Used by the production hot restart + /// to relaunch the root isolate from a newly installed patch. + /// + /// @return A clone of the existing runtime controller with the given + /// isolate snapshot. + /// + std::unique_ptr CloneWithSnapshot( + fml::RefPtr isolate_snapshot) const; + //---------------------------------------------------------------------------- /// @brief Notify the isolate that a new view is available. /// diff --git a/engine/src/flutter/runtime/shorebird/BUILD.gn b/engine/src/flutter/runtime/shorebird/BUILD.gn new file mode 100644 index 0000000000000..1f856e36d3f48 --- /dev/null +++ b/engine/src/flutter/runtime/shorebird/BUILD.gn @@ -0,0 +1,16 @@ +import("//flutter/common/config.gni") + +source_set("patch_cache") { + sources = [ + "patch_cache.cc", + "patch_cache.h", + "patch_mapping.cc", + "patch_mapping.h", + ] + + deps = [ + "//flutter/fml", + "//flutter/runtime:libdart", + "//flutter/shell/common/shorebird:updater", + ] +} diff --git a/engine/src/flutter/runtime/shorebird/patch_cache.cc b/engine/src/flutter/runtime/shorebird/patch_cache.cc new file mode 100644 index 0000000000000..8c5cc3a83f182 --- /dev/null +++ b/engine/src/flutter/runtime/shorebird/patch_cache.cc @@ -0,0 +1,165 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "flutter/runtime/shorebird/patch_cache.h" + +#include + +#include "flutter/fml/logging.h" +#include "flutter/fml/mapping.h" +#include "flutter/runtime/shorebird/patch_mapping.h" +#include "third_party/dart/runtime/include/dart_api.h" + +namespace flutter { + +namespace { + +// These symbol names match the constants in dart_snapshot.cc. +// We duplicate them here rather than extracting them into a header. +// They are actually defined down in Dart and will never change. +constexpr const char* kIsolateDataSymbol = "kDartIsolateSnapshotData"; +constexpr const char* kIsolateInstructionsSymbol = + "kDartIsolateSnapshotInstructions"; + +} // namespace + +// PatchCacheEntry implementation + +std::shared_ptr PatchCacheEntry::Create( + const std::string& path) { + // vmcode files currently use ELF internally after a prefix of a Shorebird + // linker header. + auto elf_mapping = fml::FileMapping::CreateReadOnly(path); + if (!elf_mapping) { + FML_LOG(ERROR) << "Failed to map file: " << path; + return nullptr; + } + + int elf_file_offset = Shorebird_ReadLinkHeader(elf_mapping->GetMapping(), + elf_mapping->GetSize()); + + const char* error = nullptr; + // The VM Snapshot is identical for all binaries produced by a given version + // of Dart. Our linker checks this and will fail to link if ever the VM + // snapshot changes. We ignore the VM data/instrs here. + const uint8_t* ignored_vm_data = nullptr; + const uint8_t* ignored_vm_instrs = nullptr; + const uint8_t* isolate_data = nullptr; + const uint8_t* isolate_instrs = nullptr; + + Dart_LoadedElf* elf = Dart_LoadELF( + path.c_str(), elf_file_offset, &error, &ignored_vm_data, + &ignored_vm_instrs, &isolate_data, &isolate_instrs, dart::bin::kReadOnly); + + if (elf == nullptr) { + FML_LOG(ERROR) << "Failed to load patch at " << path << " error: " << error; + return nullptr; + } + + FML_LOG(INFO) << "Loaded patch from " << path; + + return std::shared_ptr( + new PatchCacheEntry(path, elf, isolate_data, isolate_instrs)); +} + +PatchCacheEntry::PatchCacheEntry(const std::string& path, + Dart_LoadedElf* elf, + const uint8_t* isolate_data, + const uint8_t* isolate_instrs) + : path_(path), + elf_(elf), + isolate_data_(isolate_data), + isolate_instrs_(isolate_instrs) {} + +PatchCacheEntry::~PatchCacheEntry() { + if (elf_ != nullptr) { + FML_LOG(INFO) << "Unloading patch from " << path_; + Dart_UnloadELF(elf_); + elf_ = nullptr; + } +} + +PatchCache& PatchCache::Instance() { + static PatchCache instance; + return instance; +} + +std::shared_ptr PatchCache::GetOrLoad( + const std::string& path) { + std::lock_guard lock(mutex_); + + // Check if we have a cached entry that's still alive + auto it = cache_.find(path); + if (it != cache_.end()) { + if (auto entry = it->second.lock()) { + FML_LOG(INFO) << "PatchCache hit for " << path; + return entry; + } + // Entry expired, remove it + cache_.erase(it); + } + + // Load a new entry + auto entry = PatchCacheEntry::Create(path); + if (entry) { + cache_[path] = entry; // Store weak_ptr + } + + return entry; +} + +void PatchCache::PruneExpired() { + std::lock_guard lock(mutex_); + + for (auto it = cache_.begin(); it != cache_.end();) { + if (it->second.expired()) { + it = cache_.erase(it); + } else { + ++it; + } + } +} + +std::shared_ptr TryLoadFromPatch( + const std::vector& native_library_paths, + const char* symbol_name) { + if (native_library_paths.empty()) { + return nullptr; + } + + // Check if the first path is a Shorebird patch (.vmcode file) + const auto& patch_path = native_library_paths.front(); + bool is_patch = patch_path.find(".vmcode") != std::string::npos; + if (!is_patch) { + return nullptr; + } + + // Patches only contain isolate data/instructions, not VM data/instructions. + // Return nullptr for VM symbols to allow fallback to the base app. + std::string symbol(symbol_name); + if (symbol != kIsolateDataSymbol && symbol != kIsolateInstructionsSymbol) { + return nullptr; + } + + // Load the patch using the cache. + auto cache_entry = PatchCache::Instance().GetOrLoad(patch_path); + if (!cache_entry) { + FML_LOG(FATAL) << "Failed to load symbol from patch at " << patch_path; + return nullptr; + } + + FML_LOG(INFO) << "Loading symbol from patch: " << symbol_name; + + // ReportLaunchStart is now called from ResolveIsolateData in + // dart_snapshot.cc, which runs before TryLoadFromPatch on all platforms. + + if (symbol == kIsolateDataSymbol) { + return PatchMapping::CreateIsolateData(cache_entry); + } else { + FML_CHECK(symbol == kIsolateInstructionsSymbol); + return PatchMapping::CreateIsolateInstructions(cache_entry); + } +} + +} // namespace flutter diff --git a/engine/src/flutter/runtime/shorebird/patch_cache.h b/engine/src/flutter/runtime/shorebird/patch_cache.h new file mode 100644 index 0000000000000..1f2e14fab711d --- /dev/null +++ b/engine/src/flutter/runtime/shorebird/patch_cache.h @@ -0,0 +1,104 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef FLUTTER_RUNTIME_SHOREBIRD_PATCH_CACHE_H_ +#define FLUTTER_RUNTIME_SHOREBIRD_PATCH_CACHE_H_ + +#include +#include +#include +#include +#include + +#include + +#include "flutter/fml/macros.h" +#include "flutter/fml/mapping.h" + +namespace flutter { + +/// A cache entry that holds a loaded patch file and its extracted snapshot +/// pointers. The patch is automatically unloaded when the last reference to +/// this entry is released. +class PatchCacheEntry { + public: + /// Creates a new cache entry by loading the patch file at the given path. + /// Returns nullptr if loading fails. + static std::shared_ptr Create(const std::string& path); + + ~PatchCacheEntry(); + + /// Returns the isolate snapshot data pointer. + const uint8_t* isolate_data() const { return isolate_data_; } + + /// Returns the isolate snapshot instructions pointer. + const uint8_t* isolate_instructions() const { return isolate_instrs_; } + + /// Returns the path this entry was loaded from. + const std::string& path() const { return path_; } + + private: + PatchCacheEntry(const std::string& path, + Dart_LoadedElf* elf, + const uint8_t* isolate_data, + const uint8_t* isolate_instrs); + + std::string path_; + Dart_LoadedElf* elf_; + const uint8_t* isolate_data_; + const uint8_t* isolate_instrs_; + + FML_DISALLOW_COPY_AND_ASSIGN(PatchCacheEntry); +}; + +/// A thread-safe cache for loaded patch files. Cache entries are automatically +/// removed when all references to them are released. +class PatchCache { + public: + /// Returns the singleton instance of the cache. + static PatchCache& Instance(); + + /// Gets or loads a patch file at the given path. If the file is already + /// cached and the entry is still alive, returns the existing entry. + /// Otherwise, loads the file and creates a new cache entry. + /// Returns nullptr if loading fails. + std::shared_ptr GetOrLoad(const std::string& path); + + /// Removes expired entries from the cache. This is called automatically + /// by GetOrLoad, but can also be called explicitly. + void PruneExpired(); + + private: + PatchCache() = default; + ~PatchCache() = default; + + std::mutex mutex_; + // We store weak references so entries are automatically cleaned up when + // all ElfMapping instances release their references. + std::map> cache_; + + FML_DISALLOW_COPY_AND_ASSIGN(PatchCache); +}; + +/// Checks if the first path in native_library_paths is a Shorebird patch +/// (.vmcode file) and if so, attempts to load the requested symbol from +/// the patch. +/// +/// @param native_library_paths The list of library paths to check. The first +/// path is checked for the .vmcode extension. +/// @param symbol_name The symbol to load (kIsolateDataSymbol or +/// kIsolateInstructionsSymbol). +/// @return A mapping for the requested symbol if this is a patch and the +/// symbol is available in the patch, nullptr otherwise. +/// +/// Note: Patches only contain isolate data/instructions, not VM +/// data/instructions. For VM symbols, this will always return nullptr, +/// allowing the caller to fall back to loading from the base app. +std::shared_ptr TryLoadFromPatch( + const std::vector& native_library_paths, + const char* symbol_name); + +} // namespace flutter + +#endif // FLUTTER_RUNTIME_SHOREBIRD_PATCH_CACHE_H_ diff --git a/engine/src/flutter/runtime/shorebird/patch_mapping.cc b/engine/src/flutter/runtime/shorebird/patch_mapping.cc new file mode 100644 index 0000000000000..d444cda817c34 --- /dev/null +++ b/engine/src/flutter/runtime/shorebird/patch_mapping.cc @@ -0,0 +1,51 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "flutter/runtime/shorebird/patch_mapping.h" + +#include "third_party/dart/runtime/include/dart_native_api.h" + +namespace flutter { + +std::shared_ptr PatchMapping::CreateIsolateData( + std::shared_ptr entry) { + if (!entry) { + return nullptr; + } + const uint8_t* data = entry->isolate_data(); + size_t size = Dart_SnapshotDataSize(data); + return std::shared_ptr(new PatchMapping(entry, data, size)); +} + +std::shared_ptr PatchMapping::CreateIsolateInstructions( + std::shared_ptr entry) { + if (!entry) { + return nullptr; + } + const uint8_t* data = entry->isolate_instructions(); + size_t size = Dart_SnapshotInstrSize(data); + return std::shared_ptr(new PatchMapping(entry, data, size)); +} + +PatchMapping::PatchMapping(std::shared_ptr entry, + const uint8_t* data, + size_t size) + : cache_entry_(std::move(entry)), data_(data), size_(size) {} + +PatchMapping::~PatchMapping() = default; + +size_t PatchMapping::GetSize() const { + return size_; +} + +const uint8_t* PatchMapping::GetMapping() const { + return data_; +} + +bool PatchMapping::IsDontNeedSafe() const { + // Patch mappings are file-backed and safe for madvise(DONTNEED). + return true; +} + +} // namespace flutter diff --git a/engine/src/flutter/runtime/shorebird/patch_mapping.h b/engine/src/flutter/runtime/shorebird/patch_mapping.h new file mode 100644 index 0000000000000..8c2e494a9004c --- /dev/null +++ b/engine/src/flutter/runtime/shorebird/patch_mapping.h @@ -0,0 +1,55 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef FLUTTER_RUNTIME_SHOREBIRD_PATCH_MAPPING_H_ +#define FLUTTER_RUNTIME_SHOREBIRD_PATCH_MAPPING_H_ + +#include + +#include "flutter/fml/macros.h" +#include "flutter/fml/mapping.h" +#include "flutter/runtime/shorebird/patch_cache.h" + +namespace flutter { + +/// A Mapping implementation that references data from a cached patch file. +/// Holding a reference to this mapping keeps the underlying patch loaded. +class PatchMapping final : public fml::Mapping { + public: + /// Creates a mapping for the isolate snapshot data from the given cache + /// entry. + static std::shared_ptr CreateIsolateData( + std::shared_ptr entry); + + /// Creates a mapping for the isolate snapshot instructions from the given + /// cache entry. + static std::shared_ptr CreateIsolateInstructions( + std::shared_ptr entry); + + ~PatchMapping() override; + + // |fml::Mapping| + size_t GetSize() const override; + + // |fml::Mapping| + const uint8_t* GetMapping() const override; + + // |fml::Mapping| + bool IsDontNeedSafe() const override; + + private: + PatchMapping(std::shared_ptr entry, + const uint8_t* data, + size_t size); + + std::shared_ptr cache_entry_; + const uint8_t* data_; + size_t size_; + + FML_DISALLOW_COPY_AND_ASSIGN(PatchMapping); +}; + +} // namespace flutter + +#endif // FLUTTER_RUNTIME_SHOREBIRD_PATCH_MAPPING_H_ diff --git a/engine/src/flutter/shell/common/BUILD.gn b/engine/src/flutter/shell/common/BUILD.gn index e1aba738f17be..e6ab962ff4106 100644 --- a/engine/src/flutter/shell/common/BUILD.gn +++ b/engine/src/flutter/shell/common/BUILD.gn @@ -114,6 +114,8 @@ source_set("common") { "shell.h", "shell_io_manager.cc", "shell_io_manager.h", + "shorebird/hot_restart.cc", + "shorebird/hot_restart.h", "skia_event_tracer_impl.cc", "skia_event_tracer_impl.h", "snapshot_controller.cc", @@ -153,6 +155,7 @@ source_set("common") { "//flutter/lib/ui", "//flutter/runtime", "//flutter/shell/common:base64", + "//flutter/shell/common/shorebird:updater", "//flutter/shell/geometry", "//flutter/shell/profiling", "//flutter/skia", @@ -342,6 +345,7 @@ if (enable_unittests) { "//flutter/common/graphics", "//flutter/display_list/testing:display_list_testing", "//flutter/shell/common:base64", + "//flutter/shell/common/shorebird:updater", "//flutter/shell/profiling:profiling_unittests", "//flutter/shell/version", "//flutter/testing:fixture_test", diff --git a/engine/src/flutter/shell/common/engine.cc b/engine/src/flutter/shell/common/engine.cc index fd0915c1accb6..0285f32e55aff 100644 --- a/engine/src/flutter/shell/common/engine.cc +++ b/engine/src/flutter/shell/common/engine.cc @@ -214,6 +214,24 @@ bool Engine::Restart(RunConfiguration configuration) { return Run(std::move(configuration)) == Engine::RunStatus::Success; } +// Shorebird: |Restart|, but the relaunched isolate boots from +// |isolate_snapshot| (a newly installed patch) instead of the snapshot this +// engine booted with. +bool Engine::RestartWithSnapshot( + RunConfiguration configuration, + fml::RefPtr isolate_snapshot) { + TRACE_EVENT0("flutter", "Engine::RestartWithSnapshot"); + if (!configuration.IsValid()) { + FML_LOG(ERROR) << "Engine run configuration was invalid."; + return false; + } + delegate_.OnPreEngineRestart(); + runtime_controller_ = + runtime_controller_->CloneWithSnapshot(std::move(isolate_snapshot)); + UpdateAssetManager(nullptr); + return Run(std::move(configuration)) == Engine::RunStatus::Success; +} + Engine::RunStatus Engine::Run(RunConfiguration configuration) { if (!configuration.IsValid()) { FML_LOG(ERROR) << "Engine run configuration was invalid."; @@ -222,8 +240,11 @@ Engine::RunStatus Engine::Run(RunConfiguration configuration) { last_entry_point_ = configuration.GetEntrypoint(); last_entry_point_library_ = configuration.GetEntrypointLibrary(); -#if (FLUTTER_RUNTIME_MODE == FLUTTER_RUNTIME_MODE_DEBUG) - // This is only used to support restart. +// This is only used to support restart. Shorebird: also retained in release +// on supported platforms, where the production hot restart relaunches the +// isolate with the same entrypoint arguments. +#if (FLUTTER_RUNTIME_MODE == FLUTTER_RUNTIME_MODE_DEBUG) || \ + SHOREBIRD_PLATFORM_SUPPORTED last_entry_point_args_ = configuration.GetEntrypointArgs(); #endif diff --git a/engine/src/flutter/shell/common/engine.h b/engine/src/flutter/shell/common/engine.h index e07f7d4f5697c..fe41a8c34542d 100644 --- a/engine/src/flutter/shell/common/engine.h +++ b/engine/src/flutter/shell/common/engine.h @@ -505,6 +505,20 @@ class Engine final : public RuntimeDelegate, PointerDataDispatcher::Delegate { /// [[nodiscard]] bool Restart(RunConfiguration configuration); + //---------------------------------------------------------------------------- + /// @brief Shorebird: like |Restart|, but relaunches the root isolate + /// from the given isolate snapshot instead of the one this + /// engine booted with. Used by the production hot restart to + /// boot a newly installed patch; see + /// shell/common/shorebird/hot_restart.h. + /// + /// @attention Like |Restart|, a non-successful restart still tears down + /// the existing root isolate. + /// + [[nodiscard]] bool RestartWithSnapshot( + RunConfiguration configuration, + fml::RefPtr isolate_snapshot); + //---------------------------------------------------------------------------- /// @brief Setup default font manager according to specific platform. /// diff --git a/engine/src/flutter/shell/common/shell.cc b/engine/src/flutter/shell/common/shell.cc index 4e920cbbb5115..c129c81ff9a1a 100644 --- a/engine/src/flutter/shell/common/shell.cc +++ b/engine/src/flutter/shell/common/shell.cc @@ -47,6 +47,8 @@ #include "third_party/skia/include/core/SkGraphics.h" #include "third_party/tonic/common/log.h" +#include "flutter/shell/common/shorebird/updater.h" + namespace flutter { constexpr char kSkiaChannel[] = "flutter/skia"; @@ -522,6 +524,18 @@ Shell::Shell(DartVMRef vm, is_gpu_disabled_sync_switch_(new fml::SyncSwitch(is_gpu_disabled)), weak_factory_gpu_(nullptr), weak_factory_(this) { + // Report launch outcome to the Shorebird updater for crash recovery. + // If the VM failed to start, we report failure so the updater can roll + // back the patch. These calls are guarded inside Updater to execute at + // most once per process โ€” only the first Shell's outcome is reported. + // In add-to-app, subsequent engines are silently ignored since they + // boot from the same snapshot that was already reported on. + // On unsupported platforms, NoOpUpdater handles these calls gracefully. + if (!vm_) { + shorebird::Updater::Instance().ReportLaunchFailure(); + } else { + shorebird::Updater::Instance().ReportLaunchSuccess(); + } FML_CHECK(!settings.enable_software_rendering || !settings.enable_impeller) << "Software rendering is incompatible with Impeller."; if (!settings.enable_impeller && settings.warn_on_impeller_opt_out) { @@ -603,6 +617,10 @@ Shell::Shell(DartVMRef vm, } Shell::~Shell() { + // Shorebird: stop servicing hot restart requests. Blocks until any + // in-flight request against this shell has finished scheduling. + ShorebirdUnregisterRestartHost(); + #if !SLIMPELLER PersistentCache::GetCacheForProcess()->RemoveWorkerTaskRunner( task_runners_.GetIOTaskRunner()); @@ -903,6 +921,10 @@ bool Shell::Setup(std::unique_ptr platform_view, is_set_up_ = true; + // Shorebird: allow package:shorebird_code_push to hot restart this shell + // in production. No-op unless eligible; see hot_restart.cc. + ShorebirdRegisterRestartHost(); + #if !SLIMPELLER PersistentCache::GetCacheForProcess()->AddWorkerTaskRunner( task_runners_.GetIOTaskRunner()); diff --git a/engine/src/flutter/shell/common/shell.h b/engine/src/flutter/shell/common/shell.h index a43f3ecda4cea..9bedcf570e571 100644 --- a/engine/src/flutter/shell/common/shell.h +++ b/engine/src/flutter/shell/common/shell.h @@ -445,6 +445,13 @@ class Shell final : public PlatformView::Delegate, static std::pair> InferVmInitDataFromSettings(Settings& settings); + // Shorebird: hot restart host registration. Declared here because the + // implementations need this shell's settings, VM, and engine; defined in + // shell/common/shorebird/hot_restart.cc to keep the logic out of this + // file. + void ShorebirdRegisterRestartHost(); + void ShorebirdUnregisterRestartHost(); + private: using ServiceProtocolHandler = std::function(); + auto* mock_ptr = mock.get(); + shorebird::Updater::SetInstanceForTesting(std::move(mock)); + shorebird::Updater::ResetLaunchStateForTesting(); + + auto settings = CreateSettingsForFixture(); + auto task_runners = GetTaskRunnersForFixture(); + auto shell = CreateShell(settings, task_runners); + ASSERT_TRUE(shell); + + const auto& log = mock_ptr->call_log(); + ASSERT_EQ(log.size(), 2u); + EXPECT_EQ(log[0], "ReportLaunchStart"); + EXPECT_EQ(log[1], "ReportLaunchSuccess"); + + DestroyShell(std::move(shell), task_runners); + shorebird::Updater::ResetLaunchStateForTesting(); + shorebird::Updater::ResetInstanceForTesting(); +} + +// In add-to-app, multiple engines may be created within a single process. +// Only the first engine should report launch start/success to the Rust +// updater. This prevents the updater from promoting a newly-downloaded patch +// to "current_boot" when subsequent engines are still running the original +// snapshot that was selected at process init time. +TEST_F(ShellTest, ShorebirdUpdaterReportsOnlyOnceForMultipleShells) { + auto mock = std::make_unique(); + auto* mock_ptr = mock.get(); + shorebird::Updater::SetInstanceForTesting(std::move(mock)); + shorebird::Updater::ResetLaunchStateForTesting(); + + auto settings = CreateSettingsForFixture(); + + // Create first shell โ€” gets Start + Success + auto task_runners1 = GetTaskRunnersForFixture(); + auto shell1 = CreateShell(settings, task_runners1); + ASSERT_TRUE(shell1); + EXPECT_EQ(mock_ptr->launch_start_count(), 1); + EXPECT_EQ(mock_ptr->launch_success_count(), 1); + + // Create second shell โ€” guarded, no additional Start or Success calls. + auto task_runners2 = GetTaskRunnersForFixture(); + auto shell2 = CreateShell(settings, task_runners2); + ASSERT_TRUE(shell2); + EXPECT_EQ(mock_ptr->launch_start_count(), 1); + EXPECT_EQ(mock_ptr->launch_success_count(), 1); + + // Only one Start+Success pair in the call log. + const auto& log = mock_ptr->call_log(); + ASSERT_EQ(log.size(), 2u); + EXPECT_EQ(log[0], "ReportLaunchStart"); + EXPECT_EQ(log[1], "ReportLaunchSuccess"); + + DestroyShell(std::move(shell1), task_runners1); + DestroyShell(std::move(shell2), task_runners2); + + shorebird::Updater::ResetLaunchStateForTesting(); + shorebird::Updater::ResetInstanceForTesting(); +} + } // namespace testing } // namespace flutter diff --git a/engine/src/flutter/shell/common/shorebird/BUILD.gn b/engine/src/flutter/shell/common/shorebird/BUILD.gn new file mode 100644 index 0000000000000..83d433b59872e --- /dev/null +++ b/engine/src/flutter/shell/common/shorebird/BUILD.gn @@ -0,0 +1,165 @@ +import("//flutter/common/config.gni") +import("//flutter/shell/common/shorebird/build_rust_updater.gni") +import("//flutter/testing/testing.gni") + +if (is_ios || is_mac) { + import("//build/config/darwin/darwin_sdk.gni") +} + +source_set("snapshots_data_handle") { + sources = [ + "snapshots_data_handle.cc", + "snapshots_data_handle.h", + ] + + deps = [ + "//flutter/fml", + "//flutter/runtime", + "//flutter/runtime:libdart", + "//flutter/shell/common", + ] +} + +if (shorebird_updater_supported) { + action("build_rust_updater") { + script = "//flutter/shell/common/shorebird/build_rust_updater.py" + + # The stamp file gives Ninja a deterministic output to depend on, but + # we also have to declare the actual static library so consumers + # referencing it via `libs = [ shorebird_updater_output_lib ]` find a + # rule that produces it. Without the lib in `outputs`, Ninja errors + # with `'cargo_target/.../libupdater.a' missing and no known rule to + # make it` once the cargo target dir lives inside root_out_dir (#124). + _stamp = + "$target_gen_dir/rust_updater_${shorebird_updater_rust_target}.stamp" + outputs = [ + _stamp, + shorebird_updater_output_lib, + ] + + args = [ + "--rust-target", + shorebird_updater_rust_target, + "--manifest-dir", + rebase_path("$shorebird_updater_dir", root_build_dir), + "--target-dir", + rebase_path("$shorebird_updater_target_dir", root_build_dir), + "--output-lib", + rebase_path("$shorebird_updater_output_lib", root_build_dir), + "--stamp", + rebase_path(_stamp, root_build_dir), + ] + + if (is_android) { + args += [ + "--ndk-path", + rebase_path(android_ndk_root, root_build_dir), + "--android-api-level", + "$android_api_level", + ] + } + + if (is_ios) { + # Thread the engine's iOS deployment target into the cargo build so + # the cc crate (compiling C deps like zstd-sys) and rustc's cdylib + # link step both target the same iOS version as the C++ engine. + # Without this, cargo defaults to whatever rustc/cc bake in (rustc's + # built-in target spec for aarch64-apple-ios is iOS 10/11, while cc + # picks up the host SDK), which causes link warnings and unresolved + # symbols like ___chkstk_darwin. + args += [ + "--ios-deployment-target", + ios_deployment_target, + ] + } + + if (is_mac) { + args += [ + "--mac-deployment-target", + mac_deployment_target, + ] + } + + # Declare inputs so Ninja knows when to re-run the action. + inputs = [ + "$shorebird_updater_dir/Cargo.toml", + "$shorebird_updater_dir/Cargo.lock", + "$shorebird_updater_dir/library/Cargo.toml", + "$shorebird_updater_dir/library/build.rs", + "$shorebird_updater_dir/library/cbindgen_dart.toml", + "$shorebird_updater_dir/library/cbindgen_engine.toml", + "$shorebird_updater_dir/library/.cargo/config.toml", + ] + inputs += shorebird_updater_rs_sources + } +} + +# C++ wrapper around the Rust updater C API. +# This provides a testable abstraction layer that can be mocked for testing. +source_set("updater") { + sources = [ + "updater.cc", + "updater.h", + ] + + deps = [ "//flutter/fml" ] + + # For the Rust updater C API (shorebird_report_launch_start, etc.) + include_dirs = [ "//flutter" ] + + if (shorebird_updater_supported) { + deps += [ ":build_rust_updater" ] + libs = [ shorebird_updater_output_lib ] + + if (is_win) { + libs += [ "userenv.lib" ] + } + } +} + +source_set("shorebird") { + sources = [ + "shorebird.cc", + "shorebird.h", + ] + + deps = [ + ":snapshots_data_handle", + ":updater", + "//flutter/fml", + "//flutter/runtime", + "//flutter/runtime:libdart", + "//flutter/shell/common", + "//flutter/shell/platform/embedder:embedder_headers", + ] +} + +if (enable_unittests) { + test_fixtures("shorebird_fixtures") { + fixtures = [] + } + + executable("shorebird_unittests") { + testonly = true + + sources = [ + "hot_restart_unittests.cc", + "patch_cache_unittests.cc", + "shorebird_unittests.cc", + "snapshots_data_handle_unittests.cc", + "updater_unittests.cc", + ] + + deps = [ + ":shorebird", + ":shorebird_fixtures", + ":snapshots_data_handle", + ":updater", + "//flutter/runtime", + "//flutter/runtime/shorebird:patch_cache", + "//flutter/shell/common", + "//flutter/testing", + "//flutter/testing:fixture_test", + ] + } +} diff --git a/engine/src/flutter/shell/common/shorebird/build_rust_updater.gni b/engine/src/flutter/shell/common/shorebird/build_rust_updater.gni new file mode 100644 index 0000000000000..08b72424af7b6 --- /dev/null +++ b/engine/src/flutter/shell/common/shorebird/build_rust_updater.gni @@ -0,0 +1,86 @@ +# Copyright 2024 The Shorebird Authors. All rights reserved. +# Use of this source code is governed by a MIT-style license that can be +# found in the LICENSE file. + +# Computes variables needed to build and link the Rust updater library. +# +# Exported variables: +# shorebird_updater_supported - Whether the current platform is supported. +# shorebird_updater_output_lib - Path to the built static library. +# shorebird_updater_rust_target - Rust target triple for the current platform. +# shorebird_updater_rs_sources - List of .rs source files (for GN inputs). + +if (is_android) { + import("//build/config/android/config.gni") +} + +shorebird_updater_dir = "//flutter/third_party/updater" + +# Compute the Rust target triple from the GN target_os/target_cpu. GN +# evaluates BUILD files under every toolchain in use, not just the +# top-level one, so we get called for host/sub toolchains too. Any +# (os, cpu) combo we don't explicitly handle leaves the rust target +# empty and marks the updater unsupported for that toolchain, which is +# the correct behavior โ€” the updater only needs to be built for +# toolchains whose final binary is the engine. +shorebird_updater_rust_target = "" + +if (is_android) { + if (target_cpu == "arm") { + shorebird_updater_rust_target = "armv7-linux-androideabi" + } else if (target_cpu == "arm64") { + shorebird_updater_rust_target = "aarch64-linux-android" + } else if (target_cpu == "x64") { + shorebird_updater_rust_target = "x86_64-linux-android" + } else if (target_cpu == "x86") { + shorebird_updater_rust_target = "i686-linux-android" + } +} else if (is_ios) { + if (target_cpu == "arm64") { + shorebird_updater_rust_target = "aarch64-apple-ios" + } else if (target_cpu == "x64") { + shorebird_updater_rust_target = "x86_64-apple-ios" + } +} else if (is_mac) { + if (target_cpu == "arm64") { + shorebird_updater_rust_target = "aarch64-apple-darwin" + } else if (target_cpu == "x64") { + shorebird_updater_rust_target = "x86_64-apple-darwin" + } +} else if (is_win) { + if (target_cpu == "x64") { + shorebird_updater_rust_target = "x86_64-pc-windows-msvc" + } +} else if (is_linux) { + if (target_cpu == "x64") { + shorebird_updater_rust_target = "x86_64-unknown-linux-gnu" + } +} + +shorebird_updater_supported = shorebird_updater_rust_target != "" + +if (shorebird_updater_supported) { + if (is_win) { + _updater_lib_name = "updater.lib" + } else { + _updater_lib_name = "libupdater.a" + } + + # Cargo target dir lives inside the GN build output dir, not inside the + # source tree. This means: + # - `rm -rf out/` clobbers compiled rust artifacts the same + # way it clobbers compiled C++ artifacts; no special-case cleanup + # in checkout.sh. + # - Each GN config (debug/release/ios/android/etc.) gets its own + # cargo target dir, so they don't step on each other's rlibs. + # - The source checkout stays clean, which is what GN/ninja expects. + shorebird_updater_target_dir = "$root_out_dir/cargo_target" + shorebird_updater_output_lib = "$shorebird_updater_target_dir/$shorebird_updater_rust_target/release/$_updater_lib_name" + + # Glob all .rs source files so the input list stays in sync automatically. + shorebird_updater_rs_sources = + exec_script("//flutter/shell/common/shorebird/list_rust_files.py", + [ rebase_path("$shorebird_updater_dir/library/src", + root_build_dir) ], + "list lines") +} diff --git a/engine/src/flutter/shell/common/shorebird/build_rust_updater.py b/engine/src/flutter/shell/common/shorebird/build_rust_updater.py new file mode 100644 index 0000000000000..aa4d66fe3aed3 --- /dev/null +++ b/engine/src/flutter/shell/common/shorebird/build_rust_updater.py @@ -0,0 +1,183 @@ +#!/usr/bin/env python3 +# Copyright 2024 The Shorebird Authors. All rights reserved. +# Use of this source code is governed by a MIT-style license that can be +# found in the LICENSE file. + +"""Build the Rust updater library via cargo, invoked as a GN action.""" + +import argparse +import os +import subprocess +import sys + + +def main(): + parser = argparse.ArgumentParser(description='Build the Rust updater static library.') + parser.add_argument( + '--rust-target', required=True, help='Rust target triple (e.g. aarch64-linux-android)' + ) + parser.add_argument( + '--manifest-dir', required=True, help='Directory containing the workspace Cargo.toml' + ) + parser.add_argument( + '--target-dir', + required=True, + help='Cargo target directory; should live inside the GN build output dir', + ) + parser.add_argument('--output-lib', required=True, help='Expected output library path') + parser.add_argument('--stamp', required=True, help='Stamp file to write on success') + parser.add_argument('--ndk-path', help='Path to the Android NDK (required for Android targets)') + parser.add_argument( + '--android-api-level', type=int, help='Android API level (required for Android targets)' + ) + parser.add_argument( + '--ios-deployment-target', + help='iOS deployment target (e.g. 13.0); required for *-apple-ios targets', + ) + parser.add_argument( + '--mac-deployment-target', + help='macOS deployment target (e.g. 10.15); required for *-apple-darwin targets', + ) + args = parser.parse_args() + + env = os.environ.copy() + + is_android = 'android' in args.rust_target + is_apple_ios = 'apple-ios' in args.rust_target + is_apple_darwin = 'apple-darwin' in args.rust_target + is_msvc = 'pc-windows-msvc' in args.rust_target + + if is_android: + if not args.ndk_path or not args.android_api_level: + print( + 'ERROR: --ndk-path and --android-api-level are required for ' + 'Android targets.', + file=sys.stderr + ) + return 1 + _configure_android_env(env, args.rust_target, args.ndk_path, args.android_api_level) + + if is_apple_ios: + if not args.ios_deployment_target: + print( + 'ERROR: --ios-deployment-target is required for *-apple-ios targets.', + file=sys.stderr, + ) + return 1 + # Setting IPHONEOS_DEPLOYMENT_TARGET makes both the cc crate (compiling + # transitive C deps like zstd-sys) and rustc's cdylib link step honor + # the engine's iOS deployment target instead of falling back to their + # respective defaults (host SDK for cc, target-spec default for rustc). + env['IPHONEOS_DEPLOYMENT_TARGET'] = args.ios_deployment_target + + if is_apple_darwin: + if not args.mac_deployment_target: + print( + 'ERROR: --mac-deployment-target is required for *-apple-darwin targets.', + file=sys.stderr, + ) + return 1 + env['MACOSX_DEPLOYMENT_TARGET'] = args.mac_deployment_target + + if is_msvc: + _configure_msvc_env(env, args.rust_target) + + # GN passes paths relative to the build output dir (which is cwd when + # Ninja runs the action). Resolve them to absolute paths so they work + # regardless of cargo's working directory. + manifest_path = os.path.abspath(os.path.join(args.manifest_dir, 'Cargo.toml')) + target_dir = os.path.abspath(args.target_dir) + output_lib = os.path.abspath(args.output_lib) + + cmd = [ + 'cargo', + 'build', + '--release', + '--target', + args.rust_target, + '--manifest-path', + manifest_path, + '--target-dir', + target_dir, + '-p', + 'updater', + ] + + print(f'Running: {" ".join(cmd)}', flush=True) + result = subprocess.run(cmd, env=env) + if result.returncode != 0: + print(f'ERROR: cargo build failed with exit code {result.returncode}', file=sys.stderr) + return result.returncode + + if not os.path.exists(output_lib): + print(f'ERROR: Expected output library not found: {output_lib}', file=sys.stderr) + return 1 + + # Write stamp file to signal success to Ninja. + with open(args.stamp, 'w') as f: + f.write('') + + return 0 + + +def _configure_msvc_env(env, rust_target): + """Force the cc crate to compile transitive C deps with the static CRT. + + The engine's Windows build uses the static CRT (/MT). The updater's + .cargo/config.toml sets `-C target-feature=+crt-static` so the rlib is + compiled to expect static-CRT linkage. However, cargo populates + CARGO_CFG_TARGET_FEATURE from the rustc target spec's default features, + not from user rustflags, so +crt-static is invisible to build scripts. + The cc crate (used by transitive *-sys deps like zstd-sys to compile + their C sources) therefore falls back to /MD, producing .obj files + full of __imp_* references that the engine's /MT link cannot resolve + (e.g. __imp_clock, __imp__wassert, __imp_qsort_s). + + Force /MT via the per-target CFLAGS env that the cc crate honors. cc + appends these flags at the end of its command line, so cl.exe sees + `/MD ... /MT` and emits warning D9025 ("overriding '/MD' with '/MT'") + and uses the last one -- /MT wins. + """ + triple_env = rust_target.replace('-', '_') + env[f'CFLAGS_{triple_env}'] = '/MT' + env[f'CXXFLAGS_{triple_env}'] = '/MT' + + +def _configure_android_env(env, rust_target, ndk_path, api_level): + """Set environment variables so cargo can cross-compile for Android.""" + # GN passes paths relative to the build output dir. Resolve to absolute + # so that cargo and the cc crate can find the NDK tools. + ndk_path = os.path.abspath(ndk_path) + + # Determine the host platform tag for NDK toolchain paths. + if sys.platform.startswith('linux'): + host_tag = 'linux-x86_64' + elif sys.platform == 'darwin': + host_tag = 'darwin-x86_64' + elif sys.platform == 'win32': + host_tag = 'windows-x86_64' + else: + raise RuntimeError(f'Unsupported host platform: {sys.platform}') + + toolchain_bin = os.path.join(ndk_path, 'toolchains', 'llvm', 'prebuilt', host_tag, 'bin') + + # The NDK clang binary uses a slightly different prefix for armv7. + # Rust target: armv7-linux-androideabi + # NDK prefix: armv7a-linux-androideabi + clang_prefix = rust_target + if rust_target.startswith('armv7-'): + clang_prefix = 'armv7a-' + rust_target[len('armv7-'):] + + clang = os.path.join(toolchain_bin, f'{clang_prefix}{api_level}-clang') + ar = os.path.join(toolchain_bin, 'llvm-ar') + + # Cargo looks for CARGO_TARGET__LINKER where the triple is + # upper-cased with hyphens replaced by underscores. + triple_env = rust_target.upper().replace('-', '_') + env[f'CARGO_TARGET_{triple_env}_LINKER'] = clang + env['CC'] = clang + env['AR'] = ar + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/engine/src/flutter/shell/common/shorebird/hot_restart.cc b/engine/src/flutter/shell/common/shorebird/hot_restart.cc new file mode 100644 index 0000000000000..6ce18d072d078 --- /dev/null +++ b/engine/src/flutter/shell/common/shorebird/hot_restart.cc @@ -0,0 +1,224 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "flutter/shell/common/shorebird/hot_restart.h" + +#include +#include + +#include "flutter/assets/asset_manager.h" +#include "flutter/fml/logging.h" +#include "flutter/runtime/dart_snapshot.h" +#include "flutter/runtime/dart_vm.h" +#include "flutter/runtime/isolate_configuration.h" +#include "flutter/shell/common/engine.h" +#include "flutter/shell/common/run_configuration.h" +#include "flutter/shell/common/shell.h" +#include "flutter/shell/common/shorebird/updater.h" +#include "third_party/dart/runtime/include/dart_native_api.h" + +namespace flutter { +namespace shorebird { + +namespace { +#if SHOREBIRD_USE_INTERPRETER +constexpr bool kUseInterpreter = true; +#else +constexpr bool kUseInterpreter = false; +#endif +} // namespace + +std::vector LibraryPathsForNextBoot( + const std::string& next_boot_patch_path, + const std::vector& original_libapp_paths, + bool use_interpreter) { + if (next_boot_patch_path.empty()) { + return original_libapp_paths; + } + if (use_interpreter) { + // Mirrors the iOS branch of ConfigureShorebird: the patch is prepended + // so the base library remains visible for VM snapshot resolution. + std::vector paths = {next_boot_patch_path}; + paths.insert(paths.end(), original_libapp_paths.begin(), + original_libapp_paths.end()); + return paths; + } + // Mirrors the Android branch of ConfigureShorebird: the patch library + // provides all snapshot symbols. + return {next_boot_patch_path}; +} + +bool VMSnapshotsIdentical(const DartSnapshot& running_vm_snapshot, + const DartSnapshot& next_vm_snapshot) { + const uint8_t* running_data = running_vm_snapshot.GetDataMapping(); + const uint8_t* next_data = next_vm_snapshot.GetDataMapping(); + if (running_data == nullptr || next_data == nullptr) { + return false; + } + // Same mapping โ€” e.g. both resolved to the App.framework symbols already + // loaded in this process. + if (running_data != next_data) { + int64_t running_data_size = Dart_SnapshotDataSize(running_data); + if (running_data_size <= 0 || + running_data_size != Dart_SnapshotDataSize(next_data)) { + return false; + } + if (memcmp(running_data, next_data, + static_cast(running_data_size)) != 0) { + return false; + } + } + + const uint8_t* running_instrs = running_vm_snapshot.GetInstructionsMapping(); + const uint8_t* next_instrs = next_vm_snapshot.GetInstructionsMapping(); + if (running_instrs == next_instrs) { + return true; + } + if (running_instrs == nullptr || next_instrs == nullptr) { + return false; + } + int64_t running_instrs_size = Dart_SnapshotInstrSize(running_instrs); + if (running_instrs_size <= 0 || + running_instrs_size != Dart_SnapshotInstrSize(next_instrs)) { + return false; + } + return memcmp(running_instrs, next_instrs, + static_cast(running_instrs_size)) == 0; +} + +bool HotRestartEngine(Engine& engine, + const Settings& settings, + const DartSnapshot& running_vm_snapshot) { + auto& updater = Updater::Instance(); + + // Two attempts: first the requested next-boot patch; if relaunching it + // fails, ReportLaunchFailure rolls the updater back, and the second + // attempt boots whatever it fell back to (the last good patch or the base + // release) so the app recovers. + for (int attempt = 0; attempt < 2; ++attempt) { + updater.ValidateNextBootPatch(); + const std::string patch_path = updater.NextBootPatchPath(); + FML_LOG(INFO) << "Shorebird hot restart: relaunching from " + << (patch_path.empty() ? std::string("the base release") + : patch_path); + + Settings restart_settings = settings; + restart_settings.application_library_paths = LibraryPathsForNextBoot( + patch_path, updater.GetAppConfig().original_libapp_paths, + kUseInterpreter); + + // The relaunched isolate keeps running against the VM snapshot this + // process booted with, so the next boot's VM snapshot must be + // identical. It always is for correctly built patches (same + // Flutter/Dart version as the release); this guards against mismatched + // artifacts. Refusing here reports nothing to the updater: the patch + // stays installed and boots normally (with its own VM snapshot) on the + // next app launch. + auto next_vm_snapshot = + DartSnapshot::VMSnapshotFromSettings(restart_settings); + if (!next_vm_snapshot || + !VMSnapshotsIdentical(running_vm_snapshot, *next_vm_snapshot)) { + FML_LOG(ERROR) + << "Shorebird hot restart: the next boot's VM snapshot does not " + "match the running VM snapshot; the patch will instead take " + "effect on the next app launch."; + return false; + } + + // Begin a fresh launch cycle: ReportLaunchStart is re-armed and fires + // during isolate snapshot resolution below (in ResolveIsolateData), + // exactly as it does on a cold boot. If the process dies mid-restart, + // the next cold boot's crash recovery rolls the patch back. + Updater::BeginNewLaunchCycle(); + auto isolate_snapshot = + DartSnapshot::IsolateSnapshotFromSettings(restart_settings); + if (!isolate_snapshot) { + FML_LOG(ERROR) + << "Shorebird hot restart: failed to resolve the isolate snapshot."; + updater.ReportLaunchFailure(); + continue; + } + + auto isolate_configuration = + IsolateConfiguration::InferFromSettings(restart_settings); + RunConfiguration configuration(std::move(isolate_configuration)); + configuration.SetEntrypointAndLibrary(engine.GetLastEntrypoint(), + engine.GetLastEntrypointLibrary()); + configuration.SetEntrypointArgs(engine.GetLastEntrypointArgs()); + configuration.SetEngineId(engine.GetLastEngineId()); + + // Carry over the existing asset resolvers (APK / bundle assets don't + // change across a hot restart) โ€” the same carry-over the debug hot + // restart does in Shell::OnServiceProtocolRunInView. + auto old_asset_manager = engine.GetAssetManager(); + if (old_asset_manager != nullptr) { + for (auto& resolver : old_asset_manager->TakeResolvers()) { + if (resolver->IsValidAfterAssetManagerChange()) { + configuration.AddAssetResolver(std::move(resolver)); + } + } + } + + if (engine.RestartWithSnapshot(std::move(configuration), + std::move(isolate_snapshot))) { + updater.ReportLaunchSuccess(); + FML_LOG(INFO) << "Shorebird hot restart succeeded."; + return true; + } + + FML_LOG(ERROR) << "Shorebird hot restart: relaunch failed."; + updater.ReportLaunchFailure(); + } + + FML_LOG(ERROR) << "Shorebird hot restart could not relaunch the app; the " + "app may be unresponsive until relaunched by the user."; + return false; +} + +} // namespace shorebird + +// Shell's hot restart host registration. Defined here (not in shell.cc) to +// keep the Shorebird-specific logic out of upstream files; these are +// declared as Shell members in shell.h because they need the Shell's +// settings, VM, and engine weak pointer. + +void Shell::ShorebirdRegisterRestartHost() { + // Hot restart is production-only (debug/JIT builds have the development + // hot restart) and only supported for engines initialized through the + // mobile ConfigureShorebird path โ€” see Updater::SetHotRestartSupported. + if (!DartVM::IsRunningPrecompiledCode()) { + return; + } + if (!shorebird::Updater::HotRestartSupported()) { + return; + } + + shorebird::Updater::RegisterRestartHost( + reinterpret_cast(this), + [ui_task_runner = task_runners_.GetUITaskRunner(), + weak_engine = weak_engine_, settings = settings_, + vm_data = vm_->GetVMData()]() { + // Invoked on an arbitrary thread (whichever thread ran the Dart FFI + // call). Restart asynchronously on the UI thread โ€” the same thread + // the debug hot restart runs on โ€” and report the request as + // accepted. + ui_task_runner->PostTask([weak_engine, settings, vm_data]() { + if (!weak_engine) { + FML_LOG(WARNING) + << "Shorebird hot restart: engine was destroyed before the " + "restart could run."; + return; + } + shorebird::HotRestartEngine(*weak_engine.get(), settings, + vm_data->GetVMSnapshot()); + }); + return true; + }); +} + +void Shell::ShorebirdUnregisterRestartHost() { + shorebird::Updater::UnregisterRestartHost(reinterpret_cast(this)); +} + +} // namespace flutter diff --git a/engine/src/flutter/shell/common/shorebird/hot_restart.h b/engine/src/flutter/shell/common/shorebird/hot_restart.h new file mode 100644 index 0000000000000..193098aa467c3 --- /dev/null +++ b/engine/src/flutter/shell/common/shorebird/hot_restart.h @@ -0,0 +1,70 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef FLUTTER_SHELL_COMMON_SHOREBIRD_HOT_RESTART_H_ +#define FLUTTER_SHELL_COMMON_SHOREBIRD_HOT_RESTART_H_ + +#include +#include + +#include "flutter/common/settings.h" +#include "flutter/fml/memory/ref_ptr.h" + +namespace flutter { + +class Engine; +class DartSnapshot; + +namespace shorebird { + +/// Hot restart of production (AOT) apps. +/// +/// A hot restart applies a downloaded patch without killing the OS process: +/// the Shell tears down the running root isolate and relaunches it from the +/// current next-boot patch, mirroring what a full app relaunch would boot. +/// +/// Flow: package:shorebird_code_push calls shorebird_restart_app over FFI โ†’ +/// the Rust updater invokes the engine's registered handler +/// (Updater::RequestRestart) โ†’ the single registered host (the Shell) posts +/// HotRestartEngine to its UI task runner. Everything below runs on the UI +/// thread, like the debug-mode hot restart in +/// Shell::OnServiceProtocolRunInView. + +/// The Settings::application_library_paths list to resolve the next boot's +/// snapshots from, mirroring how ConfigureShorebird builds the list at cold +/// boot. `next_boot_patch_path` may be empty (no patch: boot the base +/// release via `original_libapp_paths`). When `use_interpreter` (iOS), the +/// patch is prepended so the base library remains visible for VM snapshot +/// resolution; otherwise (Android) the patch library alone provides all +/// symbols. +std::vector LibraryPathsForNextBoot( + const std::string& next_boot_patch_path, + const std::vector& original_libapp_paths, + bool use_interpreter); + +/// Whether `next_vm_snapshot` is identical to the VM snapshot the running +/// process booted with. A relaunched isolate keeps running against the +/// original VM snapshot, so the patch's isolate snapshot must have been +/// generated against an identical one. Identical Flutter/Dart versions +/// produce identical VM snapshots (Shorebird requires patches to be built +/// with the release's Flutter version); this guards against mismatched +/// artifacts reaching a device. Exposed for testing. +bool VMSnapshotsIdentical(const DartSnapshot& running_vm_snapshot, + const DartSnapshot& next_vm_snapshot); + +/// Tears down `engine`'s root isolate and relaunches it from the current +/// next-boot patch. Must run on the UI thread. `settings` is the Shell's +/// settings; `running_vm_snapshot` is the VM snapshot the process booted +/// with. On failure, reports launch failure to the updater (rolling the +/// patch back) and retries once with the rolled-back selection so the app +/// recovers onto a known-good state. Returns whether an isolate is running +/// when done. +bool HotRestartEngine(Engine& engine, + const Settings& settings, + const DartSnapshot& running_vm_snapshot); + +} // namespace shorebird +} // namespace flutter + +#endif // FLUTTER_SHELL_COMMON_SHOREBIRD_HOT_RESTART_H_ diff --git a/engine/src/flutter/shell/common/shorebird/hot_restart_unittests.cc b/engine/src/flutter/shell/common/shorebird/hot_restart_unittests.cc new file mode 100644 index 0000000000000..50cd0aa354b44 --- /dev/null +++ b/engine/src/flutter/shell/common/shorebird/hot_restart_unittests.cc @@ -0,0 +1,80 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "flutter/shell/common/shorebird/hot_restart.h" + +#include "flutter/fml/mapping.h" +#include "flutter/runtime/dart_snapshot.h" +#include "gtest/gtest.h" + +namespace flutter { +namespace shorebird { +namespace testing { + +// The library-path lists must mirror how ConfigureShorebird builds them at +// cold boot, so a hot restart resolves the same snapshots a full app +// relaunch would. + +TEST(HotRestart, LibraryPathsWithoutPatchAreTheOriginalPaths) { + const std::vector originals = {"/data/app/lib/libapp.so"}; + EXPECT_EQ(LibraryPathsForNextBoot("", originals, /*use_interpreter=*/false), + originals); + EXPECT_EQ(LibraryPathsForNextBoot("", originals, /*use_interpreter=*/true), + originals); +} + +TEST(HotRestart, LibraryPathsWithPatchReplaceOriginals) { + // Mirrors the Android branch of ConfigureShorebird: the patch library + // provides all snapshot symbols. + const std::vector originals = {"/data/app/lib/libapp.so"}; + const std::vector expected = {"/data/patches/1/dlc.vmcode"}; + EXPECT_EQ(LibraryPathsForNextBoot("/data/patches/1/dlc.vmcode", originals, + /*use_interpreter=*/false), + expected); +} + +TEST(HotRestart, LibraryPathsWithPatchAndInterpreterPrependPatch) { + // Mirrors the iOS branch of ConfigureShorebird: the base library stays in + // the list so VM snapshot resolution can still find it. + const std::vector originals = {"/app/Frameworks/App"}; + const std::vector expected = {"/patches/1/dlc.vmcode", + "/app/Frameworks/App"}; + EXPECT_EQ(LibraryPathsForNextBoot("/patches/1/dlc.vmcode", originals, + /*use_interpreter=*/true), + expected); +} + +TEST(HotRestart, LibraryPathsWithoutPatchOrOriginalsAreEmpty) { + EXPECT_TRUE( + LibraryPathsForNextBoot("", {}, /*use_interpreter=*/false).empty()); +} + +// VMSnapshotsIdentical must accept the same underlying buffers (e.g. both +// snapshots resolved to the App.framework symbols already loaded in this +// process) without parsing them. The byte-comparison path requires real +// snapshot headers and is exercised by device tests. +TEST(HotRestart, VMSnapshotsIdenticalForSameBuffers) { + static const uint8_t data[16] = {0}; + static const uint8_t instructions[16] = {0}; + + auto make_snapshot = [] { + return DartSnapshot::IsolateSnapshotFromMappings( + std::make_shared(data, sizeof(data)), + std::make_shared(instructions, + sizeof(instructions))); + }; + + auto running = make_snapshot(); + auto next = make_snapshot(); + ASSERT_TRUE(running); + ASSERT_TRUE(next); + + // Distinct DartSnapshot objects over the same buffers are identical. + EXPECT_TRUE(VMSnapshotsIdentical(*running, *next)); + EXPECT_TRUE(VMSnapshotsIdentical(*running, *running)); +} + +} // namespace testing +} // namespace shorebird +} // namespace flutter diff --git a/engine/src/flutter/shell/common/shorebird/list_rust_files.py b/engine/src/flutter/shell/common/shorebird/list_rust_files.py new file mode 100644 index 0000000000000..f478d7bd8e176 --- /dev/null +++ b/engine/src/flutter/shell/common/shorebird/list_rust_files.py @@ -0,0 +1,28 @@ +#!/usr/bin/env python3 +# Copyright 2024 The Shorebird Authors. All rights reserved. +# Use of this source code is governed by a MIT-style license that can be +# found in the LICENSE file. + +"""List Rust source files for GN input tracking. + +Walks a directory tree and prints all .rs files as absolute paths. Used by +GN's exec_script to generate an input list for the Rust updater build action. + +Usage: + python3 list_rust_files.py +""" + +import os +import sys + + +def main(): + directory = os.path.abspath(sys.argv[1]) + for root, _, files in os.walk(directory): + for filename in sorted(files): + if filename.endswith('.rs'): + print(os.path.join(root, filename).replace(os.sep, '/')) + + +if __name__ == '__main__': + main() diff --git a/engine/src/flutter/shell/common/shorebird/patch_cache_unittests.cc b/engine/src/flutter/shell/common/shorebird/patch_cache_unittests.cc new file mode 100644 index 0000000000000..51fc20f9ca562 --- /dev/null +++ b/engine/src/flutter/shell/common/shorebird/patch_cache_unittests.cc @@ -0,0 +1,70 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "flutter/runtime/shorebird/patch_cache.h" + +#include +#include + +#include "gtest/gtest.h" + +namespace flutter { +namespace testing { + +TEST(PatchCache, InstanceReturnsSameInstance) { + PatchCache& instance1 = PatchCache::Instance(); + PatchCache& instance2 = PatchCache::Instance(); + EXPECT_EQ(&instance1, &instance2); +} + +TEST(TryLoadFromPatch, ReturnsNullptrForEmptyPaths) { + std::vector empty_paths; + auto result = TryLoadFromPatch(empty_paths, "kDartIsolateSnapshotData"); + EXPECT_EQ(result, nullptr); +} + +TEST(TryLoadFromPatch, ReturnsNullptrForNonVmcodePath) { + std::vector paths = {"/path/to/some/file.so"}; + auto result = TryLoadFromPatch(paths, "kDartIsolateSnapshotData"); + EXPECT_EQ(result, nullptr); +} + +TEST(TryLoadFromPatch, ReturnsNullptrForVmSymbol) { + // Even with a .vmcode path, VM symbols should return nullptr + // (we can't actually load the file, but we can verify the symbol check) + std::vector paths = {"/path/to/patch.vmcode"}; + + // VM data symbol should return nullptr (patches don't contain VM snapshots) + auto result_vm_data = TryLoadFromPatch(paths, "kDartVmSnapshotData"); + EXPECT_EQ(result_vm_data, nullptr); + + // VM instructions symbol should return nullptr + auto result_vm_instrs = + TryLoadFromPatch(paths, "kDartVmSnapshotInstructions"); + EXPECT_EQ(result_vm_instrs, nullptr); +} + +TEST(TryLoadFromPatch, ReturnsNullptrForUnknownSymbol) { + std::vector paths = {"/path/to/patch.vmcode"}; + auto result = TryLoadFromPatch(paths, "kSomeUnknownSymbol"); + EXPECT_EQ(result, nullptr); +} + +TEST(TryLoadFromPatch, ChecksOnlyFirstPath) { + // Only the first path should be checked for .vmcode extension + std::vector paths = {"/path/to/regular.so", + "/path/to/patch.vmcode"}; + auto result = TryLoadFromPatch(paths, "kDartIsolateSnapshotData"); + // Should return nullptr because first path is not .vmcode + EXPECT_EQ(result, nullptr); +} + +TEST(PatchCache, GetOrLoadReturnsNullptrForNonexistentFile) { + auto result = + PatchCache::Instance().GetOrLoad("/nonexistent/path/to/file.vmcode"); + EXPECT_EQ(result, nullptr); +} + +} // namespace testing +} // namespace flutter diff --git a/engine/src/flutter/shell/common/shorebird/shorebird.cc b/engine/src/flutter/shell/common/shorebird/shorebird.cc new file mode 100644 index 0000000000000..d3a67a3aae960 --- /dev/null +++ b/engine/src/flutter/shell/common/shorebird/shorebird.cc @@ -0,0 +1,338 @@ + +#include "flutter/shell/common/shorebird/shorebird.h" + +#include +#include +#include +#include +#include + +#include "flutter/fml/command_line.h" +#include "flutter/fml/file.h" +#include "flutter/fml/macros.h" +#include "flutter/fml/mapping.h" +#include "flutter/fml/message_loop.h" +#include "flutter/fml/native_library.h" +#include "flutter/fml/paths.h" +#include "flutter/lib/ui/plugins/callback_cache.h" +#include "flutter/runtime/dart_snapshot.h" +#include "flutter/runtime/dart_vm.h" +#include "flutter/shell/common/shell.h" +#include "flutter/shell/common/shorebird/snapshots_data_handle.h" +#include "flutter/shell/common/shorebird/updater.h" +#include "flutter/shell/common/switches.h" +#include "fml/logging.h" +#include "shell/platform/embedder/embedder.h" +#include "third_party/dart/runtime/include/dart_native_api.h" +#include "third_party/dart/runtime/include/dart_tools_api.h" + +// Namespaced to avoid Google style warnings. +namespace flutter { + +// Old Android versions (e.g. the v16 ndk Flutter uses) don't always include a +// getauxval symbol, but the Rust ring crate assumes it exists: +// https://github.com/briansmith/ring/blob/fa25bf3a7403c9fe6458cb87bd8427be41225ca2/src/cpu/arm.rs#L22 +// It uses it to determine if the CPU supports AES instructions. +// Making this a weak symbol allows the linker to use a real version instead +// if it can find one. +// BoringSSL just reads from procfs instead, which is what we would do if +// we needed to implement this ourselves. Implementation looks straightforward: +// https://lwn.net/Articles/519085/ +// https://github.com/google/boringssl/blob/6ab4f0ae7f2db96d240eb61a5a8b4724e5a09b2f/crypto/cpu_arm_linux.c +#if defined(__ANDROID__) && defined(__arm__) +extern "C" __attribute__((weak)) unsigned long getauxval(unsigned long type) { + return 0; +} +#endif + +#if SHOREBIRD_USE_INTERPRETER +// Global references to the base (unpatched) snapshots from the App.framework. +// These are process-global because: +// 1. The Shorebird updater library is a process-global singleton with its own +// internal state. FileCallbacksImpl provides it access to the base snapshot +// data for patch generation/validation. +// 2. The base snapshots are immutable (baked into the IPA) so sharing them +// across isolate groups is safe. +// +// Note: This design doesn't support multiple engines with different base +// snapshots, but I'm not aware of any use cases for that on iOS. +static fml::RefPtr vm_snapshot; +static fml::RefPtr isolate_snapshot; + +void SetBaseSnapshot(Settings& settings) { + // These mappings happen to be to static data in the App.framework, but + // we still need to seem to hold onto the DartSnapshot objects to keep + // the mappings alive. + vm_snapshot = DartSnapshot::VMSnapshotFromSettings(settings); + isolate_snapshot = DartSnapshot::IsolateSnapshotFromSettings(settings); + + // Per-mapping observability for the base snapshot. Logged at every app boot + // so customer syslogs include the exact sizes the on-device base reader + // exposes โ€” directly comparable to the host's `aot_tools dump_blobs` + // extraction the patch was generated against. + const uint8_t* vm_data_ptr = vm_snapshot->GetDataMapping(); + const uint8_t* iso_data_ptr = isolate_snapshot->GetDataMapping(); + const uint8_t* vm_insns_ptr = vm_snapshot->GetInstructionsMapping(); + const uint8_t* iso_insns_ptr = isolate_snapshot->GetInstructionsMapping(); + intptr_t vm_data_size = vm_data_ptr ? Dart_SnapshotDataSize(vm_data_ptr) : -1; + intptr_t iso_data_size = + iso_data_ptr ? Dart_SnapshotDataSize(iso_data_ptr) : -1; + intptr_t vm_insns_size = + vm_insns_ptr ? Dart_SnapshotInstrSize(vm_insns_ptr) : -1; + intptr_t iso_insns_size = + iso_insns_ptr ? Dart_SnapshotInstrSize(iso_insns_ptr) : -1; + FML_LOG(INFO) << "[shorebird] SetBaseSnapshot mappings: " + << "vm_data_size=" << vm_data_size + << " iso_data_size=" << iso_data_size + << " vm_insns_size=" << vm_insns_size + << " iso_insns_size=" << iso_insns_size << " total=" + << (vm_data_size + iso_data_size + vm_insns_size + + iso_insns_size); + + Shorebird_SetBaseSnapshots(isolate_snapshot->GetDataMapping(), + isolate_snapshot->GetInstructionsMapping(), + vm_snapshot->GetDataMapping(), + vm_snapshot->GetInstructionsMapping()); +} +#endif // SHOREBIRD_USE_INTERPRETER + +class FileCallbacksImpl { + public: + static void* Open(); + static uintptr_t Read(void* file, uint8_t* buffer, uintptr_t length); + static int64_t Seek(void* file, int64_t offset, int32_t whence); + static void Close(void* file); +}; + +shorebird::FileCallbacks ShorebirdFileCallbacks() { + return { + .open = FileCallbacksImpl::Open, + .read = FileCallbacksImpl::Read, + .seek = FileCallbacksImpl::Seek, + .close = FileCallbacksImpl::Close, + }; +} + +// Given the contents of a yaml file, return the given value if it exists, +// otherwise return an empty string. +// Does not support nested keys. +std::string GetValueFromYaml(const std::string& yaml, const std::string& key) { + std::stringstream ss(yaml); + std::string line; + std::string prefix = key + ":"; + while (std::getline(ss, line, '\n')) { + if (line.find(prefix) != std::string::npos) { + auto ret = line.substr(line.find(prefix) + prefix.size()); + + // Remove leading and trailing spaces + while (!ret.empty() && std::isspace(ret.front())) { + ret.erase(0, 1); + } + while (!ret.empty() && std::isspace(ret.back())) { + ret.pop_back(); + } + return ret; + } + } + return ""; +} + +/// Newer api, used by Desktop implementations. +/// Does not directly manipulate Settings. +// TODO(eseidel): Consolidate this with the other ConfigureShorebird() API. +bool ConfigureShorebird(const ShorebirdConfigArgs& args, + std::string& patch_path) { + patch_path = args.release_app_library_path; + auto shorebird_updater_dir_name = "shorebird_updater"; + + // Parse app id from shorebird.yaml + std::string app_id = GetValueFromYaml(args.shorebird_yaml, "app_id"); + if (app_id.empty()) { + FML_LOG(ERROR) << "Shorebird updater: appid not found in shorebird.yaml"; + return false; + } + + auto code_cache_dir = fml::paths::JoinPaths( + {std::move(args.code_cache_path), shorebird_updater_dir_name, app_id}); + auto app_storage_dir = fml::paths::JoinPaths( + {std::move(args.app_storage_path), shorebird_updater_dir_name, app_id}); + + fml::CreateDirectory(fml::paths::GetCachesDirectory(), + {shorebird_updater_dir_name}, + fml::FilePermission::kReadWrite); + + // Combine version and version_code into a single string. + // We could also pass these separately through to the updater if needed. + auto release_version = args.release_version.version; + if (!args.release_version.build_number.empty()) { + release_version += "+" + args.release_version.build_number; + } + + shorebird::AppConfig config; + config.release_version = release_version; + config.original_libapp_paths = {args.release_app_library_path}; + config.app_storage_dir = app_storage_dir; + config.code_cache_dir = code_cache_dir; + config.file_callbacks = ShorebirdFileCallbacks(); + config.yaml_config = args.shorebird_yaml; + + bool init_result = shorebird::Updater::Instance().Init(config); + + // We do not support synchronous updates on launch, it's a terrible UX. + // Users can implement custom check-for-updates using + // package:shorebird_code_push. + // https://github.com/shorebirdtech/shorebird/issues/950 + + FML_LOG(INFO) << "Checking for active patch"; + shorebird::Updater::Instance().ValidateNextBootPatch(); + std::string active_path = shorebird::Updater::Instance().NextBootPatchPath(); + if (!active_path.empty()) { + patch_path = active_path; + FML_LOG(INFO) << "Shorebird updater: patch path: " << patch_path; + } else { + FML_LOG(INFO) << "Shorebird updater: no active patch."; + } + + // Note: shorebird_report_launch_start() is now called from TryLoadFromPatch() + // in runtime/shorebird/patch_cache.cc, right before the patched snapshot is + // actually loaded. This fixes issues with FlutterEngineGroup and other cases + // where ConfigureShorebird() is called but no Shell is created. + if (!init_result) { + return false; + } + + if (shorebird::Updater::Instance().ShouldAutoUpdate()) { + FML_LOG(INFO) << "Starting Shorebird update"; + shorebird::Updater::Instance().StartUpdateThread(); + } else { + FML_LOG(INFO) + << "Shorebird auto_update disabled, not checking for updates."; + } + + return true; +} + +/// Older api used by iOS and Android, directly manipulates Settings. +// TODO(eseidel): Consolidate this with the other ConfigureShorebird() API. +void ConfigureShorebird(std::string code_cache_path, + std::string app_storage_path, + Settings& settings, + const std::string& shorebird_yaml, + const std::string& version, + const std::string& version_code) { + // If you are crashing here, you probably are running Shorebird in a Debug + // config, where the AOT snapshot won't be linked into the process, and thus + // lookups will fail. Change your Scheme to Release to fix: + // https://github.com/flutter/flutter/wiki/Debugging-the-engine#debugging-ios-builds-with-xcode + FML_CHECK(DartSnapshot::VMSnapshotFromSettings(settings)) + << "XCode Scheme must be set to Release to use Shorebird"; + + auto shorebird_updater_dir_name = "shorebird_updater"; + + auto code_cache_dir = fml::paths::JoinPaths( + {std::move(code_cache_path), shorebird_updater_dir_name}); + auto app_storage_dir = fml::paths::JoinPaths( + {std::move(app_storage_path), shorebird_updater_dir_name}); + + fml::CreateDirectory(fml::paths::GetCachesDirectory(), + {shorebird_updater_dir_name}, + fml::FilePermission::kReadWrite); + + // Combine version and version_code into a single string. + // We could also pass these separately through to the updater if needed. + shorebird::AppConfig config; + config.release_version = version + "+" + version_code; + config.original_libapp_paths = settings.application_library_paths; + config.app_storage_dir = app_storage_dir; + config.code_cache_dir = code_cache_dir; + config.file_callbacks = ShorebirdFileCallbacks(); + config.yaml_config = shorebird_yaml; + + bool init_result = shorebird::Updater::Instance().Init(config); + + // On the mobile path, snapshots resolve through + // Settings::application_library_paths, which hot restart can re-resolve to + // pick up a newly installed patch. Desktop embedders (the other + // ConfigureShorebird overload) wire AOT data through the embedder API, + // which a restart would not refresh, so they don't enable this. + if (init_result) { + shorebird::Updater::SetHotRestartSupported(true); + } + + // We do not support synchronous updates on launch, it's a terrible UX. + // Users can implement custom check-for-updates using + // package:shorebird_code_push. + // https://github.com/shorebirdtech/shorebird/issues/950 + + // We only set the base snapshot on iOS for now. +#if SHOREBIRD_USE_INTERPRETER + SetBaseSnapshot(settings); +#endif + + shorebird::Updater::Instance().ValidateNextBootPatch(); + std::string active_path = shorebird::Updater::Instance().NextBootPatchPath(); + if (!active_path.empty()) { + FML_LOG(INFO) << "Shorebird updater: active path: " << active_path; + +#if SHOREBIRD_USE_INTERPRETER + // On iOS we add the patch to the front of the list instead of clearing + // the list, to allow dart_snapshot.cc to still find the base snapshot + // for the vm isolate. + settings.application_library_paths.insert( + settings.application_library_paths.begin(), active_path); +#else + settings.application_library_paths.clear(); + settings.application_library_paths.emplace_back(active_path); +#endif + } else { + FML_LOG(INFO) << "Shorebird updater: no active patch."; + } + + // Note: shorebird_report_launch_start() is now called from TryLoadFromPatch() + // in runtime/shorebird/patch_cache.cc, right before the patched snapshot is + // actually loaded. This fixes issues with FlutterEngineGroup and other cases + // where ConfigureShorebird() is called but no Shell is created. + + if (!init_result) { + return; + } + + if (shorebird::Updater::Instance().ShouldAutoUpdate()) { + FML_LOG(INFO) << "Starting Shorebird update"; + shorebird::Updater::Instance().StartUpdateThread(); + } else { + FML_LOG(INFO) + << "Shorebird auto_update disabled, not checking for updates."; + } +} + +void* FileCallbacksImpl::Open() { +#if SHOREBIRD_USE_INTERPRETER + return SnapshotsDataHandle::createForSnapshots(*vm_snapshot, + *isolate_snapshot) + .release(); +#else + // SnapshotsDataHandle exists on all platforms (for testing) but is only used + // on iOS. iOS patches are generated from just the Dart parts of the snapshot, + // excluding the Mach-O specific headers which contain dates and paths that + // make them change on every build. + return nullptr; +#endif // SHOREBIRD_USE_INTERPRETER +} + +uintptr_t FileCallbacksImpl::Read(void* file, + uint8_t* buffer, + uintptr_t length) { + return reinterpret_cast(file)->Read(buffer, length); +} + +int64_t FileCallbacksImpl::Seek(void* file, int64_t offset, int32_t whence) { + // Currently we only support blob handles. + return reinterpret_cast(file)->Seek(offset, whence); +} + +void FileCallbacksImpl::Close(void* file) { + delete reinterpret_cast(file); +} + +} // namespace flutter \ No newline at end of file diff --git a/engine/src/flutter/shell/common/shorebird/shorebird.h b/engine/src/flutter/shell/common/shorebird/shorebird.h new file mode 100644 index 0000000000000..ab5c9162ea0f2 --- /dev/null +++ b/engine/src/flutter/shell/common/shorebird/shorebird.h @@ -0,0 +1,59 @@ +#ifndef FLUTTER_SHELL_COMMON_SHOREBIRD_SHOREBIRD_H_ +#define FLUTTER_SHELL_COMMON_SHOREBIRD_SHOREBIRD_H_ + +#include "flutter/common/settings.h" +#include "flutter/fml/memory/ref_ptr.h" +#include "shell/platform/embedder/embedder.h" + +namespace flutter { + +class DartSnapshot; + +/// Version and build number of the release. +/// Used by ShorebirdConfigArgs. +struct ReleaseVersion { + std::string version; + std::string build_number; +}; + +/// Arguments for ConfigureShorebird. +/// Used by Desktop implementations. +struct ShorebirdConfigArgs { + std::string code_cache_path; + std::string app_storage_path; + std::string release_app_library_path; + std::string shorebird_yaml; + ReleaseVersion release_version; + + ShorebirdConfigArgs(std::string code_cache_path, + std::string app_storage_path, + std::string release_app_library_path, + std::string shorebird_yaml, + ReleaseVersion release_version) + : code_cache_path(code_cache_path), + app_storage_path(app_storage_path), + release_app_library_path(release_app_library_path), + shorebird_yaml(shorebird_yaml), + release_version(release_version) {} +}; + +/// Newer api, used by Desktop implementations. +/// Does not directly manipulate Settings. +bool ConfigureShorebird(const ShorebirdConfigArgs& args, + std::string& patch_path); + +/// Older api used by iOS and Android, directly manipulates Settings. +void ConfigureShorebird(std::string code_cache_path, + std::string app_storage_path, + Settings& settings, + const std::string& shorebird_yaml, + const std::string& version, + const std::string& version_code); + +/// Used for reading app_id from shorebird.yaml. +/// Exposed for testing. +std::string GetValueFromYaml(const std::string& yaml, const std::string& key); + +} // namespace flutter + +#endif // FLUTTER_SHELL_COMMON_SHOREBIRD_SHOREBIRD_H_ diff --git a/engine/src/flutter/shell/common/shorebird/shorebird_unittests.cc b/engine/src/flutter/shell/common/shorebird/shorebird_unittests.cc new file mode 100644 index 0000000000000..7c108ac1e6231 --- /dev/null +++ b/engine/src/flutter/shell/common/shorebird/shorebird_unittests.cc @@ -0,0 +1,21 @@ +#include "flutter/shell/common/shorebird/shorebird.h" + +#include "gtest/gtest.h" + +namespace flutter { +namespace testing { +TEST(Shorebird, GetValueFromYamlValueExists) { + std::string yaml = "appid: com.example.app\nversion: 1.0.0\n"; + std::string key = "appid"; + std::string value = GetValueFromYaml(yaml, key); + EXPECT_EQ(value, "com.example.app"); +} + +TEST(Shorebird, GetValueFromYamlValueDoesNotExist) { + std::string yaml = "appid: com.example.app\nversion: 1.0.0\n"; + std::string key = "appid2"; + std::string value = GetValueFromYaml(yaml, key); + EXPECT_EQ(value, ""); +} +} // namespace testing +} // namespace flutter \ No newline at end of file diff --git a/engine/src/flutter/shell/common/shorebird/snapshots_data_handle.cc b/engine/src/flutter/shell/common/shorebird/snapshots_data_handle.cc new file mode 100644 index 0000000000000..3f8c7f6763794 --- /dev/null +++ b/engine/src/flutter/shell/common/shorebird/snapshots_data_handle.cc @@ -0,0 +1,163 @@ +#include "flutter/shell/common/shorebird/snapshots_data_handle.h" + +#include "third_party/dart/runtime/include/dart_native_api.h" + +namespace flutter { + +static std::unique_ptr DataMapping(const DartSnapshot& snapshot) { + auto ptr = snapshot.GetDataMapping(); + return std::make_unique(ptr, + Dart_SnapshotDataSize(ptr)); +} + +static std::unique_ptr InstructionsMapping( + const DartSnapshot& snapshot) { + auto ptr = snapshot.GetInstructionsMapping(); + return std::make_unique(ptr, + Dart_SnapshotInstrSize(ptr)); +} + +// The size of the snapshot data is the sum of the sizes of the blobs. +size_t SnapshotsDataHandle::FullSize() const { + size_t size = 0; + for (const auto& blob : blobs_) { + size += blob->GetSize(); + } + return size; +} + +// The offset into the snapshots data blobs as though they were a single +// contiguous buffer. +size_t SnapshotsDataHandle::AbsoluteOffsetForIndex(BlobsIndex index) { + if (index.blob >= blobs_.size()) { + FML_LOG(WARNING) << "Blob index " << index.blob + << " is larger than the number of blobs (" << blobs_.size() + << "). Returning full size (" << FullSize() << ")"; + return FullSize(); + } + if (index.offset > blobs_[index.blob]->GetSize()) { + FML_LOG(WARNING) << "Offset for blob " << index.blob << " (" << index.offset + << ") is larger than the blob size (" + << blobs_[index.blob]->GetSize() + << "). Returning index start of next blob"; + return AbsoluteOffsetForIndex({index.blob + 1, 0}); + } + size_t offset = 0; + for (size_t i = 0; i < index.blob; i++) { + offset += blobs_[i]->GetSize(); + } + offset += index.offset; + return offset; +} + +BlobsIndex SnapshotsDataHandle::IndexForAbsoluteOffset(int64_t offset, + BlobsIndex start_index) { + size_t start_offset = AbsoluteOffsetForIndex(start_index); + if (offset < 0) { + if ((size_t)abs(offset) > start_offset) { + FML_LOG(WARNING) + << "Offset is before the beginning of SnapshotsData. Returning 0, 0"; + return {0, 0}; + } + } else if (offset + start_offset >= FullSize()) { + FML_LOG(WARNING) << "Target offset is past the end of SnapshotsData (" + << offset + start_offset << ", blobs size:" << FullSize() + << "). Returning last blob index and offset"; + return {blobs_.size(), blobs_.back()->GetSize()}; + } + + size_t dest_offset = start_offset + offset; + BlobsIndex index = {0, 0}; + for (const auto& blob : blobs_) { + if (dest_offset < blob->GetSize()) { + // The remaining offset is within this blob. + index.offset = dest_offset; + break; + } + + index.blob++; + dest_offset -= blob->GetSize(); + } + return index; +} + +std::unique_ptr SnapshotsDataHandle::createForSnapshots( + const DartSnapshot& vm_snapshot, + const DartSnapshot& isolate_snapshot) { + // This needs to match the order in which the blobs are written out in + // analyze_snapshot --dump_blobs + auto vm_data = DataMapping(vm_snapshot); + auto iso_data = DataMapping(isolate_snapshot); + auto vm_insns = InstructionsMapping(vm_snapshot); + auto iso_insns = InstructionsMapping(isolate_snapshot); + + // Per-blob observability for the base byte stream the updater is about to + // patch against. Logged at every patch-apply attempt so customer syslogs + // include the exact sizes the bipatch state machine sees โ€” directly + // comparable to the host's `aot_tools dump_blobs` extraction. + FML_LOG(INFO) << "[shorebird] SnapshotsDataHandle blob sizes: vm_data=" + << vm_data->GetSize() << "b iso_data=" << iso_data->GetSize() + << "b vm_instructions=" << vm_insns->GetSize() + << "b iso_instructions=" << iso_insns->GetSize() << "b total=" + << (vm_data->GetSize() + iso_data->GetSize() + + vm_insns->GetSize() + iso_insns->GetSize()) + << "b"; + + std::vector> blobs; + blobs.push_back(std::move(vm_data)); + blobs.push_back(std::move(iso_data)); + blobs.push_back(std::move(vm_insns)); + blobs.push_back(std::move(iso_insns)); + return std::make_unique(std::move(blobs)); +} + +uintptr_t SnapshotsDataHandle::Read(uint8_t* buffer, uintptr_t length) { + uintptr_t bytes_read = 0; + // Copy current blob from current offset and possibly into the next blob + // until we have read length bytes. + while (bytes_read < length) { + if (current_index_.blob >= blobs_.size()) { + // We have read all blobs. + break; + } + intptr_t remaining_blob_length = + blobs_[current_index_.blob]->GetSize() - current_index_.offset; + if (remaining_blob_length <= 0) { + // We have read all bytes in this blob. + current_index_.blob++; + current_index_.offset = 0; + continue; + } + intptr_t bytes_to_read = fmin(length - bytes_read, remaining_blob_length); + memcpy(buffer + bytes_read, + blobs_[current_index_.blob]->GetMapping() + current_index_.offset, + bytes_to_read); + bytes_read += bytes_to_read; + current_index_.offset += bytes_to_read; + } + + return bytes_read; +} + +int64_t SnapshotsDataHandle::Seek(int64_t offset, int32_t whence) { + BlobsIndex start_index; + switch (whence) { + case SEEK_CUR: + start_index = current_index_; + break; + case SEEK_SET: + start_index = {0, 0}; + break; + case SEEK_END: + start_index = {blobs_.size(), blobs_.back()->GetSize()}; + break; + default: + FML_CHECK(false) << "Unrecognized whence value in Seek: " << whence; + } + current_index_ = IndexForAbsoluteOffset(offset, start_index); + // Per POSIX/Rust io::Seek contract, return the new absolute position from + // the start of the stream, not the offset within the current blob. + return static_cast(AbsoluteOffsetForIndex(current_index_)); +} + +} // namespace flutter \ No newline at end of file diff --git a/engine/src/flutter/shell/common/shorebird/snapshots_data_handle.h b/engine/src/flutter/shell/common/shorebird/snapshots_data_handle.h new file mode 100644 index 0000000000000..50c4b8179b412 --- /dev/null +++ b/engine/src/flutter/shell/common/shorebird/snapshots_data_handle.h @@ -0,0 +1,48 @@ +#ifndef FLUTTER_SHELL_COMMON_SHOREBIRD_SNAPSHOTS_DATA_HANDLE_H_ +#define FLUTTER_SHELL_COMMON_SHOREBIRD_SNAPSHOTS_DATA_HANDLE_H_ + +#include +#include "flutter/fml/file.h" +#include "flutter/runtime/dart_snapshot.h" +#include "third_party/dart/runtime/include/dart_tools_api.h" + +namespace flutter { + +// An offset into an indexed collection of buffers. blob is the index of the +// buffer, and offset is the offset into that buffer. +struct BlobsIndex { + size_t blob; + size_t offset; +}; + +// Implements a POSIX file I/O interface which allows us to provide the four +// data blobs of a Dart snapshot (vm_data, vm_instructions, isolate_data, +// isolate_instructions) to Rust as though it were a single piece of memory. +class SnapshotsDataHandle { + public: + // This would ideally be private, but we need to be able to call it from the + // static createForSnapshots method. + explicit SnapshotsDataHandle(std::vector> blobs) + : blobs_(std::move(blobs)) {} + + static std::unique_ptr createForSnapshots( + const DartSnapshot& vm_snapshot, + const DartSnapshot& isolate_snapshot); + + uintptr_t Read(uint8_t* buffer, uintptr_t length); + int64_t Seek(int64_t offset, int32_t whence); + + // The sum of all the blobs' sizes. + size_t FullSize() const; + + private: + size_t AbsoluteOffsetForIndex(BlobsIndex index); + BlobsIndex IndexForAbsoluteOffset(int64_t offset, BlobsIndex startIndex); + + BlobsIndex current_index_ = {0, 0}; + std::vector> blobs_; +}; + +} // namespace flutter + +#endif // FLUTTER_SHELL_COMMON_SHOREBIRD_SNAPSHOTS_DATA_HANDLE_H_ diff --git a/engine/src/flutter/shell/common/shorebird/snapshots_data_handle_unittests.cc b/engine/src/flutter/shell/common/shorebird/snapshots_data_handle_unittests.cc new file mode 100644 index 0000000000000..2acf44a7b8973 --- /dev/null +++ b/engine/src/flutter/shell/common/shorebird/snapshots_data_handle_unittests.cc @@ -0,0 +1,177 @@ +#include +#include +#include + +#include "flutter/shell/common/shorebird/snapshots_data_handle.h" + +#include "flutter/fml/mapping.h" +#include "flutter/runtime/dart_snapshot.h" +#include "flutter/testing/testing.h" +#include "gmock/gmock.h" +#include "gtest/gtest.h" +#include "testing/fixture_test.h" + +namespace flutter { +namespace testing { + +std::unique_ptr MakeHandle( + std::vector& blobs) { + // Map the strings into non-owned mappings: + std::vector> mappings = {}; + for (auto& blob : blobs) { + std::unique_ptr mapping = + std::make_unique( + reinterpret_cast(blob.data()), blob.size()); + mappings.push_back(std::move(mapping)); + } + auto handle = + std::make_unique(std::move(mappings)); + return handle; +} + +TEST(SnapshotsDataHandle, Read) { + std::vector blobs = {"abc", "def", "ghi", "jkl"}; + std::unique_ptr blobs_handle = MakeHandle(blobs); + + const size_t buffer_size = 12; + uint8_t buffer[buffer_size]; + std::fill(buffer, buffer + buffer_size, 0); + blobs_handle->Read(buffer, 6); + + EXPECT_EQ(buffer[0], 'a'); + EXPECT_EQ(buffer[1], 'b'); + EXPECT_EQ(buffer[2], 'c'); + EXPECT_EQ(buffer[3], 'd'); + EXPECT_EQ(buffer[4], 'e'); + EXPECT_EQ(buffer[5], 'f'); + + // Only the first 6 bytes should have been read. + EXPECT_EQ(buffer[6], 0); +} + +TEST(SnapshotsDataHandle, ReadAfterSeekWithPositiveOffset) { + std::vector blobs = {"abc", "def", "ghi", "jkl"}; + std::unique_ptr blobs_handle = MakeHandle(blobs); + + const size_t buffer_size = 20; + uint8_t buffer[buffer_size]; + std::fill(buffer, buffer + buffer_size, 0); + + blobs_handle->Seek(4, SEEK_CUR); + blobs_handle->Read(buffer, 6); + + EXPECT_EQ(buffer[0], 'e'); + EXPECT_EQ(buffer[1], 'f'); + EXPECT_EQ(buffer[2], 'g'); + EXPECT_EQ(buffer[3], 'h'); + EXPECT_EQ(buffer[4], 'i'); + EXPECT_EQ(buffer[5], 'j'); + + // Only the first 6 bytes should have been read. + EXPECT_EQ(buffer[6], 0); +} + +TEST(SnapshotsDataHandle, ReadAfterSeekWithNegativeOffset) { + std::vector blobs = {"abc", "def", "ghi", "jkl"}; + std::unique_ptr blobs_handle = MakeHandle(blobs); + + const size_t buffer_size = 20; + uint8_t buffer[buffer_size]; + std::fill(buffer, buffer + buffer_size, 0); + + blobs_handle->Read(buffer, 5); + EXPECT_EQ(buffer[0], 'a'); + EXPECT_EQ(buffer[1], 'b'); + EXPECT_EQ(buffer[2], 'c'); + EXPECT_EQ(buffer[3], 'd'); + EXPECT_EQ(buffer[4], 'e'); + EXPECT_EQ(buffer[5], 0); + + // Reset buffer + std::fill(buffer, buffer + buffer_size, 0); + + // Read 5, seeked back 4, should start reading at offset 1 ('b') + blobs_handle->Seek(-4, SEEK_CUR); + blobs_handle->Read(buffer, 6); + + EXPECT_EQ(buffer[0], 'b'); + EXPECT_EQ(buffer[1], 'c'); + EXPECT_EQ(buffer[2], 'd'); + EXPECT_EQ(buffer[3], 'e'); + EXPECT_EQ(buffer[4], 'f'); + EXPECT_EQ(buffer[5], 'g'); + EXPECT_EQ(buffer[6], 0); +} + +TEST(SnapshotsDataHandle, SeekPastEnd) { + std::vector blobs = {"abc", "def", "ghi", "jkl"}; + std::unique_ptr blobs_handle = MakeHandle(blobs); + + const size_t buffer_size = 20; + uint8_t buffer[buffer_size]; + std::fill(buffer, buffer + buffer_size, 0); + + // Seek 1 past the end + blobs_handle->Seek(blobs_handle->FullSize() + 1, SEEK_CUR); + + // Seek back 2 bytes and read 2 bytes + blobs_handle->Seek(-2, SEEK_CUR); + blobs_handle->Read(buffer, 2); + + EXPECT_EQ(buffer[0], 'k'); + EXPECT_EQ(buffer[1], 'l'); +} + +TEST(SnapshotsDataHandle, SeekBeforeBeginning) { + std::vector blobs = {"abc", "def", "ghi", "jkl"}; + std::unique_ptr blobs_handle = MakeHandle(blobs); + + const size_t buffer_size = 20; + uint8_t buffer[buffer_size]; + std::fill(buffer, buffer + buffer_size, 0); + + // Seek before the start of the blobs and read the first 2 bytes. + blobs_handle->Seek(-2, SEEK_CUR); + blobs_handle->Read(buffer, 2); + + EXPECT_EQ(buffer[0], 'a'); + EXPECT_EQ(buffer[1], 'b'); +} + +TEST(SnapshotsDataHandle, SeekFromBeginning) { + std::vector blobs = {"abc", "def", "ghi", "jkl"}; + std::unique_ptr blobs_handle = MakeHandle(blobs); + + const size_t buffer_size = 20; + uint8_t buffer[buffer_size]; + std::fill(buffer, buffer + buffer_size, 0); + + // Seek 10 bytes from current (the beginning) + blobs_handle->Seek(10, SEEK_CUR); + + // Seek 2 bytes from the beginning and read 2 bytes + blobs_handle->Seek(2, SEEK_SET); + blobs_handle->Read(buffer, 2); + + EXPECT_EQ(buffer[0], 'c'); + EXPECT_EQ(buffer[1], 'd'); +} + +TEST(SnapshotsDataHandle, SeekFromEnd) { + std::vector blobs = {"abc", "def", "ghi", "jkl"}; + std::unique_ptr blobs_handle = MakeHandle(blobs); + + const size_t buffer_size = 20; + uint8_t buffer[buffer_size]; + std::fill(buffer, buffer + buffer_size, 0); + + // Seek 2 bytes from the end and read 2 bytes + blobs_handle->Seek(-2, SEEK_END); + blobs_handle->Read(buffer, 2); + + EXPECT_EQ(buffer[0], 'k'); + EXPECT_EQ(buffer[1], 'l'); +} + +} // namespace testing +} // namespace flutter diff --git a/engine/src/flutter/shell/common/shorebird/updater.cc b/engine/src/flutter/shell/common/shorebird/updater.cc new file mode 100644 index 0000000000000..a6a059c189b6e --- /dev/null +++ b/engine/src/flutter/shell/common/shorebird/updater.cc @@ -0,0 +1,270 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "flutter/shell/common/shorebird/updater.h" + +#include "flutter/fml/logging.h" + +#if SHOREBIRD_PLATFORM_SUPPORTED +#include "third_party/updater/library/include/updater_engine.h" +#endif + +namespace flutter { +namespace shorebird { + +// Static member definitions +std::unique_ptr Updater::instance_; +std::mutex Updater::instance_mutex_; +std::atomic Updater::launch_started_{false}; +std::atomic Updater::launch_completed_{false}; +std::mutex Updater::restart_hosts_mutex_; +std::map> Updater::restart_hosts_; +std::atomic Updater::hot_restart_supported_{false}; + +Updater& Updater::Instance() { + std::lock_guard lock(instance_mutex_); + if (!instance_) { +#if SHOREBIRD_PLATFORM_SUPPORTED + instance_ = std::make_unique(); +#else + instance_ = std::make_unique(); +#endif + } + return *instance_; +} + +void Updater::SetInstanceForTesting(std::unique_ptr instance) { + std::lock_guard lock(instance_mutex_); + instance_ = std::move(instance); +} + +void Updater::ResetInstanceForTesting() { + std::lock_guard lock(instance_mutex_); + instance_.reset(); +} + +void Updater::ResetLaunchStateForTesting() { + BeginNewLaunchCycle(); +} + +bool Updater::Init(const AppConfig& config) { + app_config_ = config; + return DoInit(config); +} + +void Updater::BeginNewLaunchCycle() { + launch_started_.store(false); + launch_completed_.store(false); +} + +void Updater::RegisterRestartHost(uintptr_t host_id, + std::function request_restart) { + std::lock_guard lock(restart_hosts_mutex_); + restart_hosts_[host_id] = std::move(request_restart); +} + +void Updater::UnregisterRestartHost(uintptr_t host_id) { + // Blocks while RequestRestart is invoking this host's callback, so a + // callback never runs against a destroyed host. Host callbacks must only + // schedule work (never restart synchronously), both to keep this window + // short and because the request arrives on an arbitrary thread. + std::lock_guard lock(restart_hosts_mutex_); + restart_hosts_.erase(host_id); +} + +bool Updater::RequestRestart() { + std::lock_guard lock(restart_hosts_mutex_); + if (restart_hosts_.empty()) { + FML_LOG(WARNING) << "Shorebird restart requested, but no engine capable " + "of hot restart is running."; + return false; + } + if (restart_hosts_.size() > 1) { + FML_LOG(WARNING) + << "Shorebird restart requested, but multiple Flutter engines are " + "running in this process. Hot restart is not supported with " + "multiple engines; the patch will boot on the next app launch."; + return false; + } + return restart_hosts_.begin()->second(); +} + +void Updater::SetHotRestartSupported(bool supported) { + hot_restart_supported_.store(supported); +} + +bool Updater::HotRestartSupported() { + return hot_restart_supported_.load(); +} + +void Updater::ReportLaunchStart() { + // Guard: only the first engine in a process should promote next_boot โ†’ + // current_boot in the Rust updater. See class-level comment for rationale. + bool expected = false; + if (!launch_started_.compare_exchange_strong(expected, true)) { + return; + } + DoReportLaunchStart(); +} + +void Updater::ReportLaunchSuccess() { + // Guard: only report success once per process. Subsequent engines reuse + // the same patch and don't need to re-confirm the boot. + bool expected = false; + if (!launch_completed_.compare_exchange_strong(expected, true)) { + return; + } + DoReportLaunchSuccess(); +} + +void Updater::ReportLaunchFailure() { + // Guard: only report failure once per process. + bool expected = false; + if (!launch_completed_.compare_exchange_strong(expected, true)) { + return; + } + DoReportLaunchFailure(); +} + +#if SHOREBIRD_PLATFORM_SUPPORTED +// RealUpdater implementation - wraps the Rust C API + +namespace { +// Registered with the Rust updater; invoked (on an arbitrary thread) when +// Dart calls shorebird_restart_app. +bool RestartHandlerTrampoline() { + return Updater::RequestRestart(); +} +} // namespace + +bool RealUpdater::DoInit(const AppConfig& config) { + // Convert paths to C strings + std::vector c_paths; + c_paths.reserve(config.original_libapp_paths.size()); + for (const auto& path : config.original_libapp_paths) { + c_paths.push_back(path.c_str()); + } + + AppParameters params; + params.release_version = config.release_version.c_str(); + params.original_libapp_paths = c_paths.data(); + params.original_libapp_paths_size = static_cast(c_paths.size()); + params.app_storage_dir = config.app_storage_dir.c_str(); + params.code_cache_dir = config.code_cache_dir.c_str(); + + // Convert our FileCallbacks to the Rust struct + ::FileCallbacks rust_callbacks; + rust_callbacks.open = config.file_callbacks.open; + rust_callbacks.read = config.file_callbacks.read; + rust_callbacks.seek = config.file_callbacks.seek; + rust_callbacks.close = config.file_callbacks.close; + + bool result = + shorebird_init(¶ms, rust_callbacks, config.yaml_config.c_str()); + if (result) { + // Let package:shorebird_code_push request hot restarts via + // shorebird_restart_app. Whether a restart can actually be serviced is + // decided per-request by RequestRestart (a single registered host). + shorebird_set_restart_handler(&RestartHandlerTrampoline); + } + return result; +} + +void RealUpdater::ValidateNextBootPatch() { + shorebird_validate_next_boot_patch(); +} + +std::string RealUpdater::NextBootPatchPath() { + char* c_path = shorebird_next_boot_patch_path(); + if (c_path == nullptr) { + return ""; + } + std::string path(c_path); + shorebird_free_string(c_path); + return path; +} + +void RealUpdater::DoReportLaunchStart() { + shorebird_report_launch_start(); +} + +void RealUpdater::DoReportLaunchSuccess() { + shorebird_report_launch_success(); +} + +void RealUpdater::DoReportLaunchFailure() { + shorebird_report_launch_failure(); +} + +bool RealUpdater::ShouldAutoUpdate() { + return shorebird_should_auto_update(); +} + +void RealUpdater::StartUpdateThread() { + shorebird_start_update_thread(); +} +#endif // SHOREBIRD_PLATFORM_SUPPORTED + +// MockUpdater implementation - for testing + +bool MockUpdater::DoInit(const AppConfig& config) { + init_count_++; + last_release_version_ = config.release_version; + last_yaml_config_ = config.yaml_config; + call_log_.push_back("Init"); + return init_result_; +} + +void MockUpdater::ValidateNextBootPatch() { + validate_count_++; + call_log_.push_back("ValidateNextBootPatch"); +} + +std::string MockUpdater::NextBootPatchPath() { + call_log_.push_back("NextBootPatchPath"); + return next_boot_patch_path_; +} + +void MockUpdater::DoReportLaunchStart() { + launch_start_count_++; + call_log_.push_back("ReportLaunchStart"); +} + +void MockUpdater::DoReportLaunchSuccess() { + launch_success_count_++; + call_log_.push_back("ReportLaunchSuccess"); +} + +void MockUpdater::DoReportLaunchFailure() { + launch_failure_count_++; + call_log_.push_back("ReportLaunchFailure"); +} + +bool MockUpdater::ShouldAutoUpdate() { + call_log_.push_back("ShouldAutoUpdate"); + return should_auto_update_; +} + +void MockUpdater::StartUpdateThread() { + start_update_thread_count_++; + call_log_.push_back("StartUpdateThread"); +} + +void MockUpdater::Reset() { + init_count_ = 0; + validate_count_ = 0; + launch_start_count_ = 0; + launch_success_count_ = 0; + launch_failure_count_ = 0; + start_update_thread_count_ = 0; + init_result_ = true; + should_auto_update_ = false; + next_boot_patch_path_.clear(); + last_release_version_.clear(); + last_yaml_config_.clear(); + call_log_.clear(); +} + +} // namespace shorebird +} // namespace flutter diff --git a/engine/src/flutter/shell/common/shorebird/updater.h b/engine/src/flutter/shell/common/shorebird/updater.h new file mode 100644 index 0000000000000..db0f44895aabb --- /dev/null +++ b/engine/src/flutter/shell/common/shorebird/updater.h @@ -0,0 +1,279 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef FLUTTER_SHELL_COMMON_SHOREBIRD_UPDATER_H_ +#define FLUTTER_SHELL_COMMON_SHOREBIRD_UPDATER_H_ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace flutter { +namespace shorebird { + +/// File callbacks for iOS patch loading. +/// Mirrors the FileCallbacks struct from the Rust updater. +struct FileCallbacks { + void* (*open)(void); + uintptr_t (*read)(void* file_handle, uint8_t* buffer, uintptr_t count); + int64_t (*seek)(void* file_handle, int64_t offset, int32_t whence); + void (*close)(void* file_handle); +}; + +/// Configuration for initializing the Shorebird updater. +struct AppConfig { + /// Version string for this release (e.g., "1.0.0+1"). + std::string release_version; + + /// Paths to the original AOT libraries (libapp.so on Android, App.framework + /// on iOS). + std::vector original_libapp_paths; + + /// Directory for persistent updater state (survives app updates). + std::string app_storage_dir; + + /// Directory for cached artifacts (cleared on app updates). + std::string code_cache_dir; + + /// Callbacks for iOS patch file access (can be null callbacks on Android). + FileCallbacks file_callbacks; + + /// YAML configuration from shorebird.yaml. + std::string yaml_config; +}; + +/// Abstract interface for the Shorebird updater. +/// +/// This abstraction allows for: +/// 1. Mocking in tests without requiring the real Rust library +/// 2. Future migration from Rust to C++ implementation +/// 3. Test instrumentation (call counting, logging) +/// +/// ## Launch lifecycle (start/success/failure) +/// +/// The Rust updater uses a start/success/failure protocol to detect crashes: +/// - `ReportLaunchStart` copies `next_boot` โ†’ `current_boot` in the Rust +/// state. If the app crashes before `ReportLaunchSuccess`, the updater +/// assumes the patch caused the crash and rolls back on the next launch. +/// +/// These calls are guarded to execute at most once per process because: +/// 1. The Rust updater is a process-global singleton โ€” calling +/// `report_launch_start` multiple times would repeatedly copy `next_boot` +/// โ†’ `current_boot`, which could promote a newly-downloaded (but not yet +/// booted) patch to "current" even though the running engine loaded the +/// old snapshot. +/// 2. In add-to-app, multiple FlutterEngines may be created and destroyed +/// within a single process. Each engine creation resolves snapshots and +/// constructs a Shell, but we must only report launch start/success once +/// โ€” for the first engine that actually boots. Without this guard, a +/// background update that completes between engine creations would get +/// promoted to "current" by the second engine's `ReportLaunchStart`, +/// even though that engine is still running the old snapshot. +/// +/// Tests can call `ResetLaunchStateForTesting()` to re-enable the guards. +class Updater { + public: + virtual ~Updater() = default; + + /// Initialize the updater with configuration. + /// Retains a copy of `config` (see `GetAppConfig`) and calls the + /// implementation's `DoInit`. + /// @param config Configuration containing release version, paths, and + /// callbacks + /// @return true if initialization succeeded + bool Init(const AppConfig& config); + + /// The configuration passed to the last successful `Init` call. Hot + /// restart uses `original_libapp_paths` to fall back to the base release + /// when the running patch has been rolled back. + const AppConfig& GetAppConfig() const { return app_config_; } + + /// Validate the next boot patch. If invalid, falls back to last good state. + virtual void ValidateNextBootPatch() = 0; + + /// Get the path to the patch that will boot on next run. + /// @return Path to patch, or empty string if no patch available + virtual std::string NextBootPatchPath() = 0; + + // Boot lifecycle methods โ€” guarded to run at most once per launch cycle. + // Callers may call these freely; subsequent calls after the first are + // silently ignored. A process normally has exactly one launch cycle; hot + // restart begins another via `BeginNewLaunchCycle`. + void ReportLaunchStart(); + void ReportLaunchSuccess(); + void ReportLaunchFailure(); + + /// Re-arms the launch lifecycle guards so a hot restart can report a new + /// launch cycle (start, then success/failure) to the Rust updater โ€” + /// exactly like a fresh process boot. Only the hot restart host may call + /// this, immediately before re-resolving the isolate snapshot. + static void BeginNewLaunchCycle(); + + // Hot restart hosts. + // + // A host is a Shell that can tear down and relaunch its Dart isolate from + // the current next-boot patch. Hosts register at Shell setup and + // unregister at Shell destruction. `RequestRestart` is invoked (on an + // arbitrary thread) when Dart calls `shorebird_restart_app`; the host's + // callback must schedule the restart asynchronously and return whether it + // was scheduled. Restarting is only supported with exactly one live host: + // with multiple engines in one process (add-to-app, FlutterEngineGroup) + // there is no safe way to restart them all against a shared updater + // state, so the request is rejected. + static void RegisterRestartHost(uintptr_t host_id, + std::function request_restart); + static void UnregisterRestartHost(uintptr_t host_id); + static bool RequestRestart(); + + /// Marks hot restart as supported for this process. Set by the mobile + /// (Android/iOS) ConfigureShorebird path, where snapshot re-resolution via + /// `Settings::application_library_paths` picks up a newly installed patch. + /// Desktop embedders resolve AOT data through the embedder API instead; a + /// restart there would silently relaunch the old snapshot, so they do not + /// set this. + static void SetHotRestartSupported(bool supported); + static bool HotRestartSupported(); + + // Update checking + virtual bool ShouldAutoUpdate() = 0; + virtual void StartUpdateThread() = 0; + + // Singleton access + static Updater& Instance(); + + // Test support - allows injecting a mock implementation + static void SetInstanceForTesting(std::unique_ptr instance); + static void ResetInstanceForTesting(); + + /// Resets the once-per-launch-cycle guards so tests can verify + /// start/success/failure calls on fresh Updater instances. + static void ResetLaunchStateForTesting(); + + protected: + Updater() = default; + + // Subclass hooks โ€” called by the public guarded methods above. + virtual bool DoInit(const AppConfig& config) = 0; + virtual void DoReportLaunchStart() = 0; + virtual void DoReportLaunchSuccess() = 0; + virtual void DoReportLaunchFailure() = 0; + + private: + AppConfig app_config_; + + static std::unique_ptr instance_; + static std::mutex instance_mutex_; + + // Once-per-launch-cycle guards. See `BeginNewLaunchCycle`. + static std::atomic launch_started_; + static std::atomic launch_completed_; + + // Hot restart host registry. + static std::mutex restart_hosts_mutex_; + static std::map> restart_hosts_; + static std::atomic hot_restart_supported_; +}; + +/// No-op implementation for unsupported platforms. +/// All methods are safe to call but do nothing. +class NoOpUpdater : public Updater { + public: + NoOpUpdater() = default; + ~NoOpUpdater() override = default; + + bool DoInit(const AppConfig& config) override { return true; } + void ValidateNextBootPatch() override {} + std::string NextBootPatchPath() override { return ""; } + void DoReportLaunchStart() override {} + void DoReportLaunchSuccess() override {} + void DoReportLaunchFailure() override {} + bool ShouldAutoUpdate() override { return false; } + void StartUpdateThread() override {} +}; + +#if SHOREBIRD_PLATFORM_SUPPORTED +/// Production implementation that wraps the Rust updater C API. +/// Only available on supported platforms (Android, iOS, macOS, Windows, Linux). +class RealUpdater : public Updater { + public: + RealUpdater() = default; + ~RealUpdater() override = default; + + bool DoInit(const AppConfig& config) override; + void ValidateNextBootPatch() override; + std::string NextBootPatchPath() override; + void DoReportLaunchStart() override; + void DoReportLaunchSuccess() override; + void DoReportLaunchFailure() override; + bool ShouldAutoUpdate() override; + void StartUpdateThread() override; +}; +#endif // SHOREBIRD_PLATFORM_SUPPORTED + +/// Mock implementation for testing. +/// Tracks call counts and can be queried to verify behavior. +class MockUpdater : public Updater { + public: + MockUpdater() = default; + ~MockUpdater() override = default; + + bool DoInit(const AppConfig& config) override; + void ValidateNextBootPatch() override; + std::string NextBootPatchPath() override; + void DoReportLaunchStart() override; + void DoReportLaunchSuccess() override; + void DoReportLaunchFailure() override; + bool ShouldAutoUpdate() override; + void StartUpdateThread() override; + + // Test accessors + int init_count() const { return init_count_; } + int validate_count() const { return validate_count_; } + int launch_start_count() const { return launch_start_count_; } + int launch_success_count() const { return launch_success_count_; } + int launch_failure_count() const { return launch_failure_count_; } + int start_update_thread_count() const { return start_update_thread_count_; } + const std::vector& call_log() const { return call_log_; } + + // Last init parameters (for verification) + const std::string& last_release_version() const { + return last_release_version_; + } + const std::string& last_yaml_config() const { return last_yaml_config_; } + + // Test configuration + void set_init_result(bool value) { init_result_ = value; } + void set_should_auto_update(bool value) { should_auto_update_ = value; } + void set_next_boot_patch_path(const std::string& path) { + next_boot_patch_path_ = path; + } + + // Reset all counters and logs + void Reset(); + + private: + int init_count_ = 0; + int validate_count_ = 0; + int launch_start_count_ = 0; + int launch_success_count_ = 0; + int launch_failure_count_ = 0; + int start_update_thread_count_ = 0; + bool init_result_ = true; + bool should_auto_update_ = false; + std::string next_boot_patch_path_; + std::string last_release_version_; + std::string last_yaml_config_; + std::vector call_log_; +}; + +} // namespace shorebird +} // namespace flutter + +#endif // FLUTTER_SHELL_COMMON_SHOREBIRD_UPDATER_H_ diff --git a/engine/src/flutter/shell/common/shorebird/updater_unittests.cc b/engine/src/flutter/shell/common/shorebird/updater_unittests.cc new file mode 100644 index 0000000000000..70b110dcbef72 --- /dev/null +++ b/engine/src/flutter/shell/common/shorebird/updater_unittests.cc @@ -0,0 +1,294 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "flutter/shell/common/shorebird/updater.h" + +#include "gtest/gtest.h" + +namespace flutter { +namespace shorebird { +namespace testing { + +class UpdaterTest : public ::testing::Test { + protected: + void SetUp() override { + // Install a mock for each test and reset the once-per-process guards + // so each test starts with a clean slate. + auto mock = std::make_unique(); + mock_ = mock.get(); + Updater::SetInstanceForTesting(std::move(mock)); + Updater::ResetLaunchStateForTesting(); + } + + void TearDown() override { + mock_ = nullptr; + Updater::ResetInstanceForTesting(); + Updater::ResetLaunchStateForTesting(); + } + + MockUpdater* mock_ = nullptr; +}; + +// ReportLaunchStart is guarded to run at most once per process. +// The second call should be silently ignored. +TEST_F(UpdaterTest, ReportLaunchStartOnlyCallsOnce) { + EXPECT_EQ(mock_->launch_start_count(), 0); + + Updater::Instance().ReportLaunchStart(); + EXPECT_EQ(mock_->launch_start_count(), 1); + + // Second call is a no-op due to the once-per-process guard. + Updater::Instance().ReportLaunchStart(); + EXPECT_EQ(mock_->launch_start_count(), 1); +} + +TEST_F(UpdaterTest, ReportLaunchSuccessOnlyCallsOnce) { + EXPECT_EQ(mock_->launch_success_count(), 0); + + Updater::Instance().ReportLaunchSuccess(); + EXPECT_EQ(mock_->launch_success_count(), 1); + + // Second call is a no-op. + Updater::Instance().ReportLaunchSuccess(); + EXPECT_EQ(mock_->launch_success_count(), 1); +} + +TEST_F(UpdaterTest, ReportLaunchFailureOnlyCallsOnce) { + EXPECT_EQ(mock_->launch_failure_count(), 0); + + Updater::Instance().ReportLaunchFailure(); + EXPECT_EQ(mock_->launch_failure_count(), 1); + + // Second call is a no-op. + Updater::Instance().ReportLaunchFailure(); + EXPECT_EQ(mock_->launch_failure_count(), 1); +} + +TEST_F(UpdaterTest, MockUpdaterTracksShouldAutoUpdate) { + mock_->set_should_auto_update(false); + EXPECT_FALSE(Updater::Instance().ShouldAutoUpdate()); + + mock_->set_should_auto_update(true); + EXPECT_TRUE(Updater::Instance().ShouldAutoUpdate()); +} + +TEST_F(UpdaterTest, MockUpdaterTracksStartUpdateThreadCalls) { + EXPECT_EQ(mock_->start_update_thread_count(), 0); + + Updater::Instance().StartUpdateThread(); + EXPECT_EQ(mock_->start_update_thread_count(), 1); +} + +TEST_F(UpdaterTest, MockUpdaterCallLogRecordsSequence) { + EXPECT_TRUE(mock_->call_log().empty()); + + Updater::Instance().ReportLaunchStart(); + Updater::Instance().ShouldAutoUpdate(); + Updater::Instance().ReportLaunchSuccess(); + + const auto& log = mock_->call_log(); + ASSERT_EQ(log.size(), 3u); + EXPECT_EQ(log[0], "ReportLaunchStart"); + EXPECT_EQ(log[1], "ShouldAutoUpdate"); + EXPECT_EQ(log[2], "ReportLaunchSuccess"); +} + +TEST_F(UpdaterTest, MockUpdaterResetClearsState) { + Updater::Instance().ReportLaunchStart(); + Updater::Instance().ReportLaunchSuccess(); + mock_->set_should_auto_update(true); + + EXPECT_EQ(mock_->launch_start_count(), 1); + EXPECT_EQ(mock_->launch_success_count(), 1); + EXPECT_TRUE(mock_->ShouldAutoUpdate()); + + mock_->Reset(); + + EXPECT_EQ(mock_->launch_start_count(), 0); + EXPECT_EQ(mock_->launch_success_count(), 0); + // Check call_log before ShouldAutoUpdate() since the method adds to call_log + EXPECT_TRUE(mock_->call_log().empty()); + EXPECT_FALSE(mock_->ShouldAutoUpdate()); +} + +// ReportLaunchStart and ReportLaunchSuccess are paired once per process. +// The Rust updater no-ops both when no patch is booting. +TEST_F(UpdaterTest, LaunchStartAndSuccessArePairedOncePerProcess) { + Updater::Instance().ReportLaunchStart(); + Updater::Instance().ReportLaunchSuccess(); + + EXPECT_EQ(mock_->launch_start_count(), 1); + EXPECT_EQ(mock_->launch_success_count(), 1); + const auto& log = mock_->call_log(); + ASSERT_EQ(log.size(), 2u); + EXPECT_EQ(log[0], "ReportLaunchStart"); + EXPECT_EQ(log[1], "ReportLaunchSuccess"); +} + +// ReportLaunchStart and ReportLaunchFailure are paired once per process. +TEST_F(UpdaterTest, LaunchStartAndFailureArePairedOncePerProcess) { + Updater::Instance().ReportLaunchStart(); + Updater::Instance().ReportLaunchFailure(); + + EXPECT_EQ(mock_->launch_start_count(), 1); + EXPECT_EQ(mock_->launch_failure_count(), 1); + const auto& log = mock_->call_log(); + ASSERT_EQ(log.size(), 2u); + EXPECT_EQ(log[0], "ReportLaunchStart"); + EXPECT_EQ(log[1], "ReportLaunchFailure"); +} + +// Simulates the add-to-app scenario: multiple engines call ReportLaunchStart +// and ReportLaunchSuccess, but only the first should actually reach the +// updater. This prevents the Rust updater from promoting a newly-downloaded +// patch to "current_boot" when subsequent engines are still running the +// original snapshot. +TEST_F(UpdaterTest, MultipleEnginesOnlyReportOnce) { + // First engine boots. + Updater::Instance().ReportLaunchStart(); + Updater::Instance().ReportLaunchSuccess(); + + // Second engine boots โ€” these should be no-ops. + Updater::Instance().ReportLaunchStart(); + Updater::Instance().ReportLaunchSuccess(); + + EXPECT_EQ(mock_->launch_start_count(), 1); + EXPECT_EQ(mock_->launch_success_count(), 1); + + const auto& log = mock_->call_log(); + ASSERT_EQ(log.size(), 2u); + EXPECT_EQ(log[0], "ReportLaunchStart"); + EXPECT_EQ(log[1], "ReportLaunchSuccess"); +} + +// ResetLaunchStateForTesting re-enables the guards, allowing tests to +// verify launch calls on a fresh state. +TEST_F(UpdaterTest, ResetLaunchStateReenablesGuards) { + Updater::Instance().ReportLaunchStart(); + Updater::Instance().ReportLaunchSuccess(); + EXPECT_EQ(mock_->launch_start_count(), 1); + EXPECT_EQ(mock_->launch_success_count(), 1); + + Updater::ResetLaunchStateForTesting(); + + Updater::Instance().ReportLaunchStart(); + Updater::Instance().ReportLaunchSuccess(); + EXPECT_EQ(mock_->launch_start_count(), 2); + EXPECT_EQ(mock_->launch_success_count(), 2); +} + +// BeginNewLaunchCycle re-arms the launch guards so a hot restart can report +// a full second launch cycle (start, then success/failure) โ€” exactly like a +// fresh process boot. This is the engine-side half of the hot restart +// launch-cycle contract (the Rust half is covered by hot_restart_tests in +// the updater repo). +TEST_F(UpdaterTest, BeginNewLaunchCycleAllowsSecondLaunchCycle) { + // Cold boot. + Updater::Instance().ReportLaunchStart(); + Updater::Instance().ReportLaunchSuccess(); + + // Hot restart begins a new launch cycle. + Updater::BeginNewLaunchCycle(); + Updater::Instance().ReportLaunchStart(); + Updater::Instance().ReportLaunchFailure(); + + EXPECT_EQ(mock_->launch_start_count(), 2); + EXPECT_EQ(mock_->launch_success_count(), 1); + EXPECT_EQ(mock_->launch_failure_count(), 1); + + const auto& log = mock_->call_log(); + ASSERT_EQ(log.size(), 4u); + EXPECT_EQ(log[0], "ReportLaunchStart"); + EXPECT_EQ(log[1], "ReportLaunchSuccess"); + EXPECT_EQ(log[2], "ReportLaunchStart"); + EXPECT_EQ(log[3], "ReportLaunchFailure"); +} + +// Init retains the configuration so the hot restart can fall back to the +// original libapp paths when no patch is installed. +TEST_F(UpdaterTest, InitRetainsAppConfig) { + AppConfig config; + config.release_version = "1.2.3+4"; + config.original_libapp_paths = {"/data/app/libapp.so"}; + config.yaml_config = "app_id: foo"; + + EXPECT_TRUE(Updater::Instance().Init(config)); + + const AppConfig& stored = Updater::Instance().GetAppConfig(); + EXPECT_EQ(stored.release_version, "1.2.3+4"); + ASSERT_EQ(stored.original_libapp_paths.size(), 1u); + EXPECT_EQ(stored.original_libapp_paths[0], "/data/app/libapp.so"); + EXPECT_EQ(mock_->init_count(), 1); +} + +class RestartHostTest : public ::testing::Test { + protected: + void TearDown() override { + // The registry is process-global; leave it empty for other tests. + Updater::UnregisterRestartHost(1); + Updater::UnregisterRestartHost(2); + Updater::SetHotRestartSupported(false); + } +}; + +// With no registered host (no shell capable of restarting), a restart +// request is rejected. +TEST_F(RestartHostTest, RequestRestartWithoutHostIsRejected) { + EXPECT_FALSE(Updater::RequestRestart()); +} + +// With exactly one host, the request is forwarded and its result relayed. +TEST_F(RestartHostTest, RequestRestartForwardsToSingleHost) { + int requests = 0; + Updater::RegisterRestartHost(1, [&requests]() { + requests++; + return true; + }); + EXPECT_TRUE(Updater::RequestRestart()); + EXPECT_EQ(requests, 1); + + Updater::RegisterRestartHost(1, []() { return false; }); + EXPECT_FALSE(Updater::RequestRestart()); + EXPECT_EQ(requests, 1); +} + +// With more than one live host (add-to-app / FlutterEngineGroup), restart +// is unsupported and the request is rejected without invoking any host. +TEST_F(RestartHostTest, RequestRestartWithMultipleHostsIsRejected) { + int requests = 0; + auto host = [&requests]() { + requests++; + return true; + }; + Updater::RegisterRestartHost(1, host); + Updater::RegisterRestartHost(2, host); + EXPECT_FALSE(Updater::RequestRestart()); + EXPECT_EQ(requests, 0); + + // Back to one host (e.g. the other engine was destroyed): supported again. + Updater::UnregisterRestartHost(2); + EXPECT_TRUE(Updater::RequestRestart()); + EXPECT_EQ(requests, 1); +} + +// Unregistering an unknown host is a no-op (a shell that never registered +// still unregisters in its destructor). +TEST_F(RestartHostTest, UnregisterUnknownHostIsNoOp) { + Updater::UnregisterRestartHost(42); + EXPECT_FALSE(Updater::RequestRestart()); +} + +// The hot-restart-supported flag is process-global, set only by the mobile +// ConfigureShorebird path. +TEST_F(RestartHostTest, HotRestartSupportedFlag) { + EXPECT_FALSE(Updater::HotRestartSupported()); + Updater::SetHotRestartSupported(true); + EXPECT_TRUE(Updater::HotRestartSupported()); + Updater::SetHotRestartSupported(false); + EXPECT_FALSE(Updater::HotRestartSupported()); +} + +} // namespace testing +} // namespace shorebird +} // namespace flutter diff --git a/engine/src/flutter/shell/platform/android/BUILD.gn b/engine/src/flutter/shell/platform/android/BUILD.gn index c15ac362d7d8a..37ea5b0430e44 100644 --- a/engine/src/flutter/shell/platform/android/BUILD.gn +++ b/engine/src/flutter/shell/platform/android/BUILD.gn @@ -171,6 +171,7 @@ source_set("flutter_shell_native_src") { "//flutter/runtime", "//flutter/runtime:libdart", "//flutter/shell/common", + "//flutter/shell/common/shorebird", "//flutter/shell/platform/android/context", "//flutter/shell/platform/android/external_view_embedder", "//flutter/shell/platform/android/jni", @@ -809,8 +810,10 @@ if (target_cpu != "x86") { } } -if (host_os == "linux" && - (target_cpu == "x64" || target_cpu == "arm64" || target_cpu == "riscv64")) { +# Build analyze_snapshot for 64-bit target CPUs on linux. +# Or always targeting arm64 for Shorebird builds. +if ((host_os == "linux" && (target_cpu == "x64" || target_cpu == "riscv64")) || + target_cpu == "arm64") { zip_bundle("analyze_snapshot") { deps = [ "$dart_src/runtime/bin:analyze_snapshot($host_toolchain)" ] diff --git a/engine/src/flutter/shell/platform/android/android_exports.lst b/engine/src/flutter/shell/platform/android/android_exports.lst index 198bff773dd74..349b25e04930f 100644 --- a/engine/src/flutter/shell/platform/android/android_exports.lst +++ b/engine/src/flutter/shell/platform/android/android_exports.lst @@ -11,6 +11,15 @@ _binary_icudtl_dat_size; InternalFlutterGpu*; kInternalFlutterGpu*; + shorebird_init; + shorebird_free_string; + shorebird_free_update_result; + shorebird_check_for_downloadable_update; + shorebird_update_with_result; + shorebird_next_boot_patch_number; + shorebird_current_boot_patch_number; + shorebird_restart_app; + shorebird_validate_next_boot_patch; local: *; }; diff --git a/engine/src/flutter/shell/platform/android/flutter_main.cc b/engine/src/flutter/shell/platform/android/flutter_main.cc index d14ac40029645..45e369f5b3a9d 100644 --- a/engine/src/flutter/shell/platform/android/flutter_main.cc +++ b/engine/src/flutter/shell/platform/android/flutter_main.cc @@ -20,6 +20,7 @@ #include "flutter/fml/platform/android/paths_android.h" #include "flutter/lib/ui/plugins/callback_cache.h" #include "flutter/runtime/dart_vm.h" +#include "flutter/shell/common/shorebird/shorebird.h" #include "flutter/shell/common/switches.h" #include "flutter/shell/platform/android/android_context_vk_impeller.h" #include "flutter/shell/platform/android/android_rendering_selector.h" @@ -93,6 +94,9 @@ void FlutterMain::Init(JNIEnv* env, jstring kernelPath, jstring appStoragePath, jstring engineCachesPath, + jstring shorebirdYaml, + jstring version, + jstring versionCode, jlong initTimeMillis, jint api_level) { std::vector args; @@ -151,8 +155,18 @@ void FlutterMain::Init(JNIEnv* env, flutter::DartCallbackCache::SetCachePath( fml::jni::JavaStringToString(env, appStoragePath)); - fml::paths::InitializeAndroidCachesPath( - fml::jni::JavaStringToString(env, engineCachesPath)); + auto code_cache_path = fml::jni::JavaStringToString(env, engineCachesPath); + auto app_storage_path = fml::jni::JavaStringToString(env, appStoragePath); + fml::paths::InitializeAndroidCachesPath(code_cache_path); + +#if FLUTTER_RELEASE + std::string shorebird_yaml = fml::jni::JavaStringToString(env, shorebirdYaml); + std::string version_string = fml::jni::JavaStringToString(env, version); + std::string version_code_string = + fml::jni::JavaStringToString(env, versionCode); + ConfigureShorebird(code_cache_path, app_storage_path, settings, + shorebird_yaml, version_string, version_code_string); +#endif flutter::DartCallbackCache::LoadCacheFromDisk(); @@ -245,6 +259,7 @@ bool FlutterMain::Register(JNIEnv* env) { { .name = "nativeInit", .signature = "(Landroid/content/Context;[Ljava/lang/String;Ljava/" + "lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/" "lang/String;Ljava/lang/String;Ljava/lang/String;JI)V", .fnPtr = reinterpret_cast(&Init), }, diff --git a/engine/src/flutter/shell/platform/android/flutter_main.h b/engine/src/flutter/shell/platform/android/flutter_main.h index dda959801bc32..cf03f889ec30e 100644 --- a/engine/src/flutter/shell/platform/android/flutter_main.h +++ b/engine/src/flutter/shell/platform/android/flutter_main.h @@ -44,6 +44,9 @@ class FlutterMain { jstring kernelPath, jstring appStoragePath, jstring engineCachesPath, + jstring shorebirdYaml, + jstring version, + jstring versionCode, jlong initTimeMillis, jint api_level); diff --git a/engine/src/flutter/shell/platform/android/io/flutter/embedding/engine/FlutterJNI.java b/engine/src/flutter/shell/platform/android/io/flutter/embedding/engine/FlutterJNI.java index 59dc281909838..8570bad6d8b95 100644 --- a/engine/src/flutter/shell/platform/android/io/flutter/embedding/engine/FlutterJNI.java +++ b/engine/src/flutter/shell/platform/android/io/flutter/embedding/engine/FlutterJNI.java @@ -8,6 +8,8 @@ import android.annotation.SuppressLint; import android.content.Context; +import android.content.pm.PackageInfo; +import android.content.pm.PackageManager; import android.content.res.AssetManager; import android.graphics.Bitmap; import android.graphics.SurfaceTexture; @@ -42,6 +44,10 @@ import io.flutter.view.AccessibilityBridge; import io.flutter.view.FlutterCallbackInformation; import io.flutter.view.TextureRegistry; +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; import java.lang.ref.WeakReference; import java.nio.ByteBuffer; import java.util.ArrayList; @@ -178,6 +184,9 @@ private static native void nativeInit( @Nullable String bundlePath, @NonNull String appStoragePath, @NonNull String engineCachesPath, + @Nullable String shorebirdYaml, + @Nullable String version, + @Nullable String versionCode, long initTimeMillis, int apiLevel); @@ -206,8 +215,47 @@ public void init( Log.w(TAG, "FlutterJNI.init called more than once"); } + String version = null; + String versionCode = null; + try { + PackageInfo packageInfo = + context.getPackageManager().getPackageInfo(context.getPackageName(), 0); + version = packageInfo.versionName; + if (Build.VERSION.SDK_INT >= API_LEVELS.API_28) { + versionCode = String.valueOf(packageInfo.getLongVersionCode()); + } else { + versionCode = String.valueOf(packageInfo.versionCode); + } + } catch (PackageManager.NameNotFoundException e) { + Log.e(TAG, "Failed to read app version. Shorebird updater can't run.", e); + } + + String shorebirdYaml = null; + try { + InputStream yaml = context.getAssets().open("flutter_assets/shorebird.yaml"); + BufferedReader r = new BufferedReader(new InputStreamReader(yaml)); + StringBuilder total = new StringBuilder(); + for (String line; (line = r.readLine()) != null; ) { + total.append(line).append('\n'); + } + shorebirdYaml = total.toString(); + Log.d(TAG, "shorebird.yaml: " + shorebirdYaml); + } catch (IOException e) { + Log.e(TAG, "Failed to load shorebird.yaml", e); + Log.e(TAG, "Did you remember to include shorebird.yaml in your pubspec.yaml's assets?"); + } + FlutterJNI.nativeInit( - context, args, bundlePath, appStoragePath, engineCachesPath, initTimeMillis, apiLevel); + context, + args, + bundlePath, + appStoragePath, + engineCachesPath, + shorebirdYaml, + version, + versionCode, + initTimeMillis, + apiLevel); FlutterJNI.initCalled = true; } diff --git a/engine/src/flutter/shell/platform/darwin/ios/BUILD.gn b/engine/src/flutter/shell/platform/darwin/ios/BUILD.gn index 2930099a5fe05..a50b691edffe2 100644 --- a/engine/src/flutter/shell/platform/darwin/ios/BUILD.gn +++ b/engine/src/flutter/shell/platform/darwin/ios/BUILD.gn @@ -195,12 +195,14 @@ source_set("flutter_framework_source") { deps = [ ":ios_gpu_configuration", + "$dart_src/runtime/bin:elf_loader", "//flutter/common", "//flutter/common/graphics", "//flutter/fml", "//flutter/lib/ui", "//flutter/runtime", "//flutter/shell/common", + "//flutter/shell/common/shorebird", "//flutter/shell/platform/common:common_cpp_input", "//flutter/shell/platform/darwin/common", "//flutter/shell/platform/darwin/common:framework_common", diff --git a/engine/src/flutter/shell/platform/darwin/ios/framework/Source/FlutterDartProject.mm b/engine/src/flutter/shell/platform/darwin/ios/framework/Source/FlutterDartProject.mm index 86418369f9979..1d81b5f3cbf67 100644 --- a/engine/src/flutter/shell/platform/darwin/ios/framework/Source/FlutterDartProject.mm +++ b/engine/src/flutter/shell/platform/darwin/ios/framework/Source/FlutterDartProject.mm @@ -13,6 +13,8 @@ #include "flutter/common/constants.h" #include "flutter/fml/build_config.h" +#include "flutter/fml/paths.h" +#include "flutter/shell/common/shorebird/shorebird.h" #include "flutter/shell/common/switches.h" #import "flutter/shell/platform/darwin/common/InternalFlutterSwiftCommon/InternalFlutterSwiftCommon.h" #include "flutter/shell/platform/darwin/common/command_line.h" @@ -92,10 +94,12 @@ static BOOL DoesHardwareSupportWideGamut() { } if (flutter::DartVM::IsRunningPrecompiledCode()) { + NSLog(@"SANITY CHECK: Running precompiled code."); if (hasExplicitBundle) { NSString* executablePath = bundle.executablePath; if ([[NSFileManager defaultManager] fileExistsAtPath:executablePath]) { settings.application_library_paths.push_back(executablePath.UTF8String); + NSLog(@"Using precompiled library from %@", executablePath); } } @@ -107,6 +111,7 @@ static BOOL DoesHardwareSupportWideGamut() { NSString* executablePath = [NSBundle bundleWithPath:libraryPath].executablePath; if (executablePath.length > 0) { settings.application_library_paths.push_back(executablePath.UTF8String); + NSLog(@"Using library from %@", libraryPath); } } } @@ -121,6 +126,7 @@ static BOOL DoesHardwareSupportWideGamut() { [NSBundle bundleWithPath:applicationFrameworkPath].executablePath; if (executablePath.length > 0) { settings.application_library_paths.push_back(executablePath.UTF8String); + NSLog(@"Using App.framework from %@", applicationFrameworkPath); } } } @@ -152,6 +158,33 @@ static BOOL DoesHardwareSupportWideGamut() { } } + NSString* assetsPath = [NSString stringWithUTF8String:settings.assets_path.c_str()]; + NSLog(@"ASSET PATH %@", assetsPath); + + // FIXME: This may not be the correct path (e.g., should it include the organization id?) + // See + // https://developer.apple.com/library/archive/documentation/FileManagement/Conceptual/FileSystemProgrammingGuide/FileSystemOverview/FileSystemOverview.html#//apple_ref/doc/uid/TP40010672-CH2-SW13 + // /private/var/mobile/Containers/Data/Application/264477BF-6E38-47C9-AAD9-532BB842F197/Library/Application + // Support/shorebird/shorebird_updater + std::string cache_path = + fml::paths::JoinPaths({getenv("HOME"), "Library/Application Support/shorebird"}); + NSURL* shorebirdYamlPath = [NSURL URLWithString:@"shorebird.yaml" + relativeToURL:[NSURL fileURLWithPath:assetsPath]]; + NSString* appVersion = [mainBundle objectForInfoDictionaryKey:@"CFBundleShortVersionString"]; + NSString* appBuildNumber = [mainBundle objectForInfoDictionaryKey:@"CFBundleVersion"]; + NSString* shorebirdYamlContents = [NSString stringWithContentsOfURL:shorebirdYamlPath + encoding:NSUTF8StringEncoding + error:nil]; + if (shorebirdYamlContents != nil) { + // Note: we intentionally pass cache_path twice. We provide two different directories + // to ConfigureShorebird because Android differentiates between data that persists + // between releases and data that does not. iOS does not make this distinction. + flutter::ConfigureShorebird(cache_path, cache_path, settings, shorebirdYamlContents.UTF8String, + appVersion.UTF8String, appBuildNumber.UTF8String); + } else { + NSLog(@"Failed to find shorebird.yaml, not starting updater."); + } + // Domain network configuration // Disabled in https://github.com/flutter/flutter/issues/72723. // Re-enable in https://github.com/flutter/flutter/issues/54448. diff --git a/engine/src/flutter/shell/platform/darwin/ios/framework/Source/FlutterViewController.mm b/engine/src/flutter/shell/platform/darwin/ios/framework/Source/FlutterViewController.mm index 831568de7759c..64a3dcbfb2268 100644 --- a/engine/src/flutter/shell/platform/darwin/ios/framework/Source/FlutterViewController.mm +++ b/engine/src/flutter/shell/platform/darwin/ios/framework/Source/FlutterViewController.mm @@ -39,6 +39,7 @@ #import "flutter/third_party/spring_animation/spring_animation.h" FLUTTER_ASSERT_ARC +#import static constexpr int kMicrosecondsPerSecond = 1000 * 1000; static constexpr CGFloat kScrollViewContentSize = 2.0; diff --git a/engine/src/flutter/shell/platform/darwin/macos/BUILD.gn b/engine/src/flutter/shell/platform/darwin/macos/BUILD.gn index ea069e2674648..d37ae587a8000 100644 --- a/engine/src/flutter/shell/platform/darwin/macos/BUILD.gn +++ b/engine/src/flutter/shell/platform/darwin/macos/BUILD.gn @@ -145,12 +145,14 @@ source_set("flutter_framework_source") { ":macos_gpu_configuration", "//flutter/flow", "//flutter/fml", + "//flutter/shell/common/shorebird", "//flutter/shell/platform/common:common_cpp_accessibility", "//flutter/shell/platform/common:common_cpp_core", "//flutter/shell/platform/common:common_cpp_enums", "//flutter/shell/platform/common:common_cpp_input", "//flutter/shell/platform/common:common_cpp_isolate_scope", "//flutter/shell/platform/common:common_cpp_switches", + "//flutter/shell/platform/darwin/common", "//flutter/shell/platform/darwin/common:availability_version_check", "//flutter/shell/platform/darwin/common:framework_common", "//flutter/shell/platform/darwin/graphics", diff --git a/engine/src/flutter/shell/platform/darwin/macos/framework/Source/FlutterEngine.mm b/engine/src/flutter/shell/platform/darwin/macos/framework/Source/FlutterEngine.mm index 6854413b14296..f4be157c5ef74 100644 --- a/engine/src/flutter/shell/platform/darwin/macos/framework/Source/FlutterEngine.mm +++ b/engine/src/flutter/shell/platform/darwin/macos/framework/Source/FlutterEngine.mm @@ -11,6 +11,8 @@ #include #include "flutter/common/constants.h" +#include "flutter/fml/paths.h" +#include "flutter/shell/common/shorebird/shorebird.h" #include "flutter/shell/platform/common/app_lifecycle_state.h" #include "flutter/shell/platform/common/engine_switches.h" #include "flutter/shell/platform/embedder/embedder.h" @@ -656,6 +658,40 @@ - (void)onFocusChangeRequest:(const FlutterViewFocusChangeRequest*)request { } } +- (BOOL)configureShorebird:(NSString**)patchPath { + NSLog(@"[shorebird] setting up non-linker shorebird"); + NSString* bundlePath = + [[NSBundle bundleWithURL:[NSBundle.mainBundle.privateFrameworksURL + URLByAppendingPathComponent:@"App.framework"]] bundlePath]; + bundlePath = [bundlePath stringByAppendingString:@"/App"]; + NSString* assetsPath = _project.assetsPath; + NSURL* shorebirdYamlPath = [NSURL URLWithString:@"shorebird.yaml" + relativeToURL:[NSURL fileURLWithPath:assetsPath]]; + NSString* shorebirdYamlContents = [NSString stringWithContentsOfURL:shorebirdYamlPath + encoding:NSUTF8StringEncoding + error:nil]; + NSString* appVersion = + [NSBundle.mainBundle objectForInfoDictionaryKey:@"CFBundleShortVersionString"]; + NSString* appBuildNumber = [NSBundle.mainBundle objectForInfoDictionaryKey:@"CFBundleVersion"]; + std::string cache_path = + fml::paths::JoinPaths({getenv("HOME"), "Library", "Application Support", "shorebird"}); + flutter::ReleaseVersion release_version = {appVersion.UTF8String, appBuildNumber.UTF8String}; + flutter::ShorebirdConfigArgs shorebird_args(cache_path, cache_path, bundlePath.UTF8String, + shorebirdYamlContents.UTF8String, release_version); + NSLog(@"[shorebird] calling ConfigureShorebird"); + std::string patch_path; + auto res = flutter::ConfigureShorebird(shorebird_args, patch_path); + if (!res) { + NSLog(@"[shorebird] ConfigureShorebird failed"); + return NO; + } + + NSLog(@"[shorebird] ConfigureShorebird success!"); + *patchPath = [NSString stringWithUTF8String:patch_path.c_str()]; + NSLog(@"[shorebird] patchPath: %@", *patchPath); + return YES; +} + - (BOOL)runWithEntrypoint:(NSString*)entrypoint { if (self.running) { return NO; @@ -759,7 +795,19 @@ - (BOOL)runWithEntrypoint:(NSString*)entrypoint { }; flutterArguments.custom_task_runners = &custom_task_runners; - [self loadAOTData:_project.assetsPath]; + NSString* elfPath; + BOOL configureShorebirdRes = [self configureShorebird:&elfPath]; + if (!configureShorebirdRes) { + // No patch exists, or we failed to configure shorebird. This is a fallback. + // Upstream, this code lives in -(void)loadAOTData:. + // + // This is the location where the test fixture places the snapshot file. + // For applications built by Flutter tool, this is in "App.framework". + elfPath = [NSString pathWithComponents:@[ _project.assetsPath, @"app_elf_snapshot.so" ]]; + } + + [self loadAOTData:elfPath]; + if (_aotData) { flutterArguments.aot_data = _aotData; } @@ -783,6 +831,7 @@ - (BOOL)runWithEntrypoint:(NSString*)entrypoint { }; FlutterRendererConfig rendererConfig = [_renderer createRendererConfig]; + FlutterEngineResult result = _embedderAPI.Initialize( FLUTTER_ENGINE_VERSION, &rendererConfig, &flutterArguments, (__bridge void*)(self), &_engine); if (result != kSuccess) { @@ -812,7 +861,7 @@ - (BOOL)runWithEntrypoint:(NSString*)entrypoint { return YES; } -- (void)loadAOTData:(NSString*)assetsDir { +- (void)loadAOTData:(NSString*)elfPath { if (!_embedderAPI.RunsAOTCompiledDartCode()) { return; } @@ -820,11 +869,8 @@ - (void)loadAOTData:(NSString*)assetsDir { BOOL isDirOut = false; // required for NSFileManager fileExistsAtPath. NSFileManager* fileManager = [NSFileManager defaultManager]; - // This is the location where the test fixture places the snapshot file. - // For applications built by Flutter tool, this is in "App.framework". - NSString* elfPath = [NSString pathWithComponents:@[ assetsDir, @"app_elf_snapshot.so" ]]; - if (![fileManager fileExistsAtPath:elfPath isDirectory:&isDirOut]) { + FML_LOG(INFO) << "in loadAOTData, elfPath does not exist: " << elfPath.UTF8String; return; } diff --git a/engine/src/flutter/shell/platform/linux/BUILD.gn b/engine/src/flutter/shell/platform/linux/BUILD.gn index d64d76019c3e5..979949848c4f7 100644 --- a/engine/src/flutter/shell/platform/linux/BUILD.gn +++ b/engine/src/flutter/shell/platform/linux/BUILD.gn @@ -158,6 +158,7 @@ source_set("flutter_linux_sources") { "fl_settings_channel.cc", "fl_settings_handler.cc", "fl_settings_portal.cc", + "fl_shorebird.cc", "fl_socket_accessible.cc", "fl_standard_message_codec.cc", "fl_standard_method_codec.cc", @@ -187,12 +188,14 @@ source_set("flutter_linux_sources") { deps = [ "$dart_src/runtime:dart_api", "//flutter/fml", + "//flutter/shell/common/shorebird", "//flutter/shell/platform/common:common_cpp_enums", "//flutter/shell/platform/common:common_cpp_input", "//flutter/shell/platform/common:common_cpp_isolate_scope", "//flutter/shell/platform/common:common_cpp_switches", "//flutter/shell/platform/embedder:embedder_headers", "//flutter/third_party/rapidjson", + "//flutter/third_party/tonic", ] } diff --git a/engine/src/flutter/shell/platform/linux/fl_engine.cc b/engine/src/flutter/shell/platform/linux/fl_engine.cc index 6ccdc39d51f83..62c302cdec509 100644 --- a/engine/src/flutter/shell/platform/linux/fl_engine.cc +++ b/engine/src/flutter/shell/platform/linux/fl_engine.cc @@ -10,6 +10,7 @@ #include #include "flutter/common/constants.h" +#include "flutter/fml/logging.h" #include "flutter/shell/platform/common/engine_switches.h" #include "flutter/shell/platform/embedder/embedder.h" #include "flutter/shell/platform/linux/fl_accessibility_handler.h" @@ -24,6 +25,7 @@ #include "flutter/shell/platform/linux/fl_platform_handler.h" #include "flutter/shell/platform/linux/fl_plugin_registrar_private.h" #include "flutter/shell/platform/linux/fl_settings_handler.h" +#include "flutter/shell/platform/linux/fl_shorebird.h" #include "flutter/shell/platform/linux/fl_texture_gl_private.h" #include "flutter/shell/platform/linux/fl_texture_registrar_private.h" #include "flutter/shell/platform/linux/public/flutter_linux/fl_plugin_registry.h" @@ -815,6 +817,9 @@ gboolean fl_engine_start(FlEngine* self, GError** error) { g_autoptr(GPtrArray) command_line_args = g_ptr_array_new_with_free_func(g_free); + // FlutterProjectArgs expects a full argv, so when processing it for flags + // the first item is treated as the executable and ignored. Add a dummy + // value so that all switches are used. g_ptr_array_insert(command_line_args, 0, g_strdup("flutter")); for (const auto& env_switch : flutter::GetSwitchesFromEnvironment()) { g_ptr_array_add(command_line_args, g_strdup(env_switch.c_str())); @@ -852,9 +857,22 @@ gboolean fl_engine_start(FlEngine* self, GError** error) { args.compositor = &compositor; if (self->embedder_api.RunsAOTCompiledDartCode()) { + // This struct contains raw C strings and needs to have its lifetime scoped + // to this block. FlutterEngineAOTDataSource source = {}; source.type = kFlutterEngineAOTDataSourceTypeElfPath; - source.elf_path = fl_dart_project_get_aot_library_path(self->project); + std::string patch_path; + auto setup_shorebird_result = + flutter::SetUpShorebird(args.assets_path, patch_path); + if (setup_shorebird_result) { + // If we have a patch installed, we replace the default AOT library path + // with the patch path here. + source.elf_path = patch_path.c_str(); + } else { + FML_LOG(ERROR) << "Failed to configure Shorebird."; + source.elf_path = fl_dart_project_get_aot_library_path(self->project); + } + if (self->embedder_api.CreateAOTData(&source, &self->aot_data) != kSuccess) { g_set_error(error, fl_engine_error_quark(), FL_ENGINE_ERROR_FAILED, diff --git a/engine/src/flutter/shell/platform/linux/fl_shorebird.cc b/engine/src/flutter/shell/platform/linux/fl_shorebird.cc new file mode 100644 index 0000000000000..eb913ad720c61 --- /dev/null +++ b/engine/src/flutter/shell/platform/linux/fl_shorebird.cc @@ -0,0 +1,56 @@ +#include "flutter/shell/platform/linux/fl_shorebird.h" + +#include +#include +#include + +#include "flutter/fml/file.h" +#include "flutter/fml/logging.h" +#include "flutter/fml/paths.h" +#include "flutter/shell/common/shorebird/shorebird.h" +#include "rapidjson/document.h" +#include "third_party/tonic/filesystem/filesystem/file.h" + +// Namespaced to avoid Google style warnings. +namespace flutter { + +gboolean SetUpShorebird(const char* assets_path, std::string& patch_path) { + auto shorebird_yaml_path = + fml::paths::JoinPaths({assets_path, "shorebird.yaml"}); + std::string shorebird_yaml_contents(""); + if (!filesystem::ReadFileToString(shorebird_yaml_path, + &shorebird_yaml_contents)) { + FML_LOG(ERROR) << "Failed to read shorebird.yaml."; + return false; + } + + std::string code_cache_path = + fml::paths::JoinPaths({g_get_home_dir(), ".shorebird_cache"}); + auto executable_location = fml::paths::GetExecutableDirectoryPath().second; + auto app_path = + fml::paths::JoinPaths({executable_location, "lib", "libapp.so"}); + auto version_json_path = fml::paths::JoinPaths({assets_path, "version.json"}); + std::ifstream input(version_json_path); + if (!input) { + return false; + } + std::string json_contents{std::istreambuf_iterator(input), + std::istreambuf_iterator()}; + + rapidjson::Document json_doc; + json_doc.Parse(json_contents.c_str()); + if (json_doc.HasParseError()) { + // Could not parse version file, aborting. + return false; + } + + const auto version_map = json_doc.GetObject(); + ReleaseVersion release_version{version_map["version"].GetString(), + version_map["build_number"].GetString()}; + + ShorebirdConfigArgs shorebird_args(code_cache_path, code_cache_path, app_path, + shorebird_yaml_contents, release_version); + return ConfigureShorebird(shorebird_args, patch_path); +} + +} // namespace flutter diff --git a/engine/src/flutter/shell/platform/linux/fl_shorebird.h b/engine/src/flutter/shell/platform/linux/fl_shorebird.h new file mode 100644 index 0000000000000..e38965776ac97 --- /dev/null +++ b/engine/src/flutter/shell/platform/linux/fl_shorebird.h @@ -0,0 +1,13 @@ +#ifndef FLUTTER_SHELL_PLATFORM_LINUX_FL_SHOREBIRD_H_ +#define FLUTTER_SHELL_PLATFORM_LINUX_FL_SHOREBIRD_H_ + +#include +#include + +namespace flutter { + +gboolean SetUpShorebird(const char* assets_path, std::string& patch_path); + +} // namespace flutter + +#endif // FLUTTER_SHELL_PLATFORM_LINUX_FL_SHOREBIRD_H_ diff --git a/engine/src/flutter/shell/platform/windows/BUILD.gn b/engine/src/flutter/shell/platform/windows/BUILD.gn index 09b52a96fb0b1..55318930541b3 100644 --- a/engine/src/flutter/shell/platform/windows/BUILD.gn +++ b/engine/src/flutter/shell/platform/windows/BUILD.gn @@ -174,6 +174,7 @@ source_set("flutter_windows_source") { ":flutter_windows_headers", "//flutter/fml", "//flutter/impeller/renderer/backend/gles", + "//flutter/shell/common/shorebird", "//flutter/shell/geometry", "//flutter/shell/platform/common:common_cpp", "//flutter/shell/platform/common:common_cpp_input", @@ -189,6 +190,7 @@ source_set("flutter_windows_source") { "//flutter/third_party/angle:libEGL_static", "//flutter/third_party/angle:libGLESv2_static", "//flutter/third_party/rapidjson", + "//flutter/third_party/tonic", ] } @@ -200,6 +202,11 @@ copy("publish_headers_windows") { deps = [ "//flutter/shell/platform/common:publish_headers" ] } +copy("updater_exports_windows") { + sources = [ "flutter_windows.dll.def" ] + outputs = [ "$root_out_dir/{{source_file_part}}" ] +} + shared_library("flutter_windows") { deps = [ ":flutter_windows_source" ] @@ -315,6 +322,7 @@ group("windows") { deps = [ ":flutter_windows", ":publish_headers_windows", + ":updater_exports_windows", "//flutter/shell/platform/windows/client_wrapper:publish_wrapper_windows", ] diff --git a/engine/src/flutter/shell/platform/windows/flutter_project_bundle.h b/engine/src/flutter/shell/platform/windows/flutter_project_bundle.h index ec465208ae8f1..b0417568ac19e 100644 --- a/engine/src/flutter/shell/platform/windows/flutter_project_bundle.h +++ b/engine/src/flutter/shell/platform/windows/flutter_project_bundle.h @@ -62,6 +62,10 @@ class FlutterProjectBundle { // Sets engine switches. void SetSwitches(const std::vector& switches); + void SetAotLibraryPath(const std::filesystem::path& aot_library_path) { + aot_library_path_ = aot_library_path; + } + // Attempts to load AOT data for this bundle. The returned data must be // retained until any engine instance it is passed to has been shut down. // diff --git a/engine/src/flutter/shell/platform/windows/flutter_windows.dll.def b/engine/src/flutter/shell/platform/windows/flutter_windows.dll.def new file mode 100644 index 0000000000000..c1325428fa1d8 --- /dev/null +++ b/engine/src/flutter/shell/platform/windows/flutter_windows.dll.def @@ -0,0 +1,15 @@ +EXPORTS + shorebird_check_for_downloadable_update = shorebird_check_for_downloadable_update + shorebird_current_boot_patch_number = shorebird_current_boot_patch_number + shorebird_free_string = shorebird_free_string + shorebird_free_update_result = shorebird_free_update_result + shorebird_init = shorebird_init + shorebird_next_boot_patch_number = shorebird_next_boot_patch_number + shorebird_next_boot_patch_path = shorebird_next_boot_patch_path + shorebird_validate_next_boot_patch = shorebird_validate_next_boot_patch + shorebird_report_launch_failure = shorebird_report_launch_failure + shorebird_report_launch_start = shorebird_report_launch_start + shorebird_report_launch_success = shorebird_report_launch_success + shorebird_should_auto_update = shorebird_should_auto_update + shorebird_start_update_thread = shorebird_start_update_thread + shorebird_update_with_result = shorebird_update_with_result \ No newline at end of file diff --git a/engine/src/flutter/shell/platform/windows/flutter_windows_engine.cc b/engine/src/flutter/shell/platform/windows/flutter_windows_engine.cc index 785a83d642da6..5dfaabcc3078b 100644 --- a/engine/src/flutter/shell/platform/windows/flutter_windows_engine.cc +++ b/engine/src/flutter/shell/platform/windows/flutter_windows_engine.cc @@ -5,15 +5,20 @@ #include "flutter/shell/platform/windows/flutter_windows_engine.h" #include +#include +#include +#include #include #include #include +#include #include "flutter/fml/logging.h" #include "flutter/fml/paths.h" #include "flutter/fml/platform/win/wstring_conversion.h" #include "flutter/fml/synchronization/waitable_event.h" +#include "flutter/shell/common/shorebird/shorebird.h" #include "flutter/shell/platform/common/client_wrapper/binary_messenger_impl.h" #include "flutter/shell/platform/common/client_wrapper/include/flutter/standard_message_codec.h" #include "flutter/shell/platform/common/path_utils.h" @@ -29,6 +34,7 @@ #include "flutter/shell/platform/windows/window_manager.h" #include "flutter/third_party/accessibility/ax/ax_node.h" #include "shell/platform/windows/flutter_project_bundle.h" +#include "third_party/tonic/filesystem/filesystem/file.h" // winbase.h defines GetCurrentTime as a macro. #undef GetCurrentTime @@ -269,6 +275,125 @@ bool FlutterWindowsEngine::Run() { return Run(""); } +int GetReleaseVersionAndBuildNumber(ReleaseVersion* release_version) { + char module_path[MAX_PATH]; + // Get the full path of the currently running executable. The return value is + // the size of the string that was copied to the buffer, with -1 indicating + // failure. + if (GetModuleFileNameA(NULL, module_path, MAX_PATH) == -1) { + return -1; + } + + // Get the size of the version information + DWORD handle = -1; + DWORD version_info_size = GetFileVersionInfoSizeA(module_path, &handle); + if (version_info_size == -1) { + return -1; + } + + // Allocate memory for version info + std::unique_ptr version_data(new char[version_info_size]); + if (!GetFileVersionInfoA(module_path, handle, version_info_size, + version_data.get())) { + return -1; + } + + // Adopted from + // https://learn.microsoft.com/en-us/windows/win32/api/winver/nf-winver-verqueryvaluea + // Get the translation table + struct LANGANDCODEPAGE { + WORD wLanguage; + WORD wCodePage; + }* lpTranslate; + + UINT cbTranslate = 0; + if (!VerQueryValueA(version_data.get(), "\\VarFileInfo\\Translation", + (LPVOID*)&lpTranslate, &cbTranslate)) { + FML_LOG(ERROR) << "Error: Unable to get translation info."; + return -1; + } + + // Construct the query string using the first translation found + char subBlock[64]; + sprintf_s(subBlock, "\\StringFileInfo\\%04x%04x\\ProductVersion", + lpTranslate[0].wLanguage, lpTranslate[0].wCodePage); + + LPSTR versionString = nullptr; + UINT size = 0; + if (!VerQueryValueA(version_data.get(), subBlock, (LPVOID*)&versionString, + &size)) { + return -1; + } + + if (!versionString) { + return -1; + } + + // The version string is in the format of "1.0.0+1", with the label ("+1") + // being optional. + auto version = std::string(versionString); + auto plusPos = version.find("+"); + if (plusPos != std::string::npos) { + auto semVer = version.substr(0, plusPos); + auto patch = version.substr(plusPos + 1, version.length()); + release_version->version = semVer; + release_version->build_number = patch; + } else { + release_version->version = version; + } + + return kSuccess; +} + +bool GetLocalAppDataPath(std::string& outPath) { + PWSTR path = nullptr; + HRESULT result = SHGetKnownFolderPath(FOLDERID_LocalAppData, 0, NULL, &path); + if (!SUCCEEDED(result)) { + return false; + } + + std::wstring widePath(path); + std::string localAppDataPath(widePath.begin(), widePath.end()); + // The calling process is responsible for freeing this resource + // https://learn.microsoft.com/en-us/windows/win32/api/shlobj_core/nf-shlobj_core-shgetknownfolderpath + CoTaskMemFree(path); + outPath = localAppDataPath; + return true; +} + +bool SetUpShorebird(std::string assets_path_string, std::string& patch_path) { + auto shorebird_yaml_path = + fml::paths::JoinPaths({assets_path_string, "shorebird.yaml"}); + std::string shorebird_yaml_contents(""); + if (!filesystem::ReadFileToString(shorebird_yaml_path, + &shorebird_yaml_contents)) { + FML_LOG(ERROR) << "Failed to read shorebird.yaml."; + return false; + } + + std::string code_cache_path; + if (!GetLocalAppDataPath(code_cache_path)) { + FML_LOG(ERROR) << "Failed to retrieve the local AppData directory."; + return false; + } + + auto executable_location = fml::paths::GetExecutableDirectoryPath().second; + auto app_path = + fml::paths::JoinPaths({executable_location, "data", "app.so"}); + ReleaseVersion release_version; + auto release_version_result = + GetReleaseVersionAndBuildNumber(&release_version); + if (release_version_result != kSuccess) { + FML_LOG(ERROR) + << "Failed to retrieve the release version and build number."; + return false; + } + + ShorebirdConfigArgs shorebird_args(code_cache_path, code_cache_path, app_path, + shorebird_yaml_contents, release_version); + return ConfigureShorebird(shorebird_args, patch_path); +} + bool FlutterWindowsEngine::Run(std::string_view entrypoint) { if (!project_->HasValidPaths()) { FML_LOG(ERROR) << "Missing or unresolvable paths to assets."; @@ -276,6 +401,19 @@ bool FlutterWindowsEngine::Run(std::string_view entrypoint) { } std::string assets_path_string = fml::PathToUtf8(project_->assets_path()); std::string icu_path_string = fml::PathToUtf8(project_->icu_path()); + + std::string patch_path; + auto setup_shorebird_result = SetUpShorebird(assets_path_string, patch_path); + if (setup_shorebird_result) { + // If we have a patch installed, we replace the default AOT library path + // with the patch path here. + FML_LOG(INFO) << "Setting project patch path: " << patch_path; + project_->SetAotLibraryPath(patch_path); + } else { + FML_LOG(ERROR) << "Failed to configure Shorebird."; + } + + // This loads AOT data from the project_'s aot_library_path_. if (embedder_api_.RunsAOTCompiledDartCode()) { aot_data_ = project_->LoadAotData(embedder_api_); if (!aot_data_) { @@ -406,6 +544,15 @@ bool FlutterWindowsEngine::Run(std::string_view entrypoint) { host->root_isolate_create_callback_(); } }; + // Copied from shell\platform\darwin\macos\framework\Source\FlutterEngine.mm + // Writes log messages to stdout. + args.log_message_callback = [](const char* tag, const char* message, + void* user_data) { + if (tag && tag[0]) { + std::cout << tag << ": "; + } + std::cout << message << std::endl; + }; args.channel_update_callback = [](const FlutterChannelUpdate* update, void* user_data) { auto host = static_cast(user_data); diff --git a/engine/src/flutter/sky/tools/create_ios_framework.py b/engine/src/flutter/sky/tools/create_ios_framework.py index d87e0f91bff49..0ead267e9609e 100644 --- a/engine/src/flutter/sky/tools/create_ios_framework.py +++ b/engine/src/flutter/sky/tools/create_ios_framework.py @@ -19,7 +19,8 @@ def main(): parser = argparse.ArgumentParser( description=( 'Creates Flutter.framework, Flutter.xcframework and ' - 'copies architecture-dependent gen_snapshot binaries to output dir' + 'copies architecture-dependent analyze_snapshot and gen_snapshot ' + 'binaries to output dir' ) ) @@ -89,13 +90,17 @@ def main(): '%s_extension_safe' % simulator_x64_out_dir, '%s_extension_safe' % simulator_arm64_out_dir ) - # Copy gen_snapshot binary to destination directory. + # Copy analyze_snapshot and gen_snapshot binaries to destination directory. if arm64_out_dir: gen_snapshot = os.path.join(arm64_out_dir, 'universal', 'gen_snapshot_arm64') + analyze_snapshot = os.path.join(arm64_out_dir, 'analyze_snapshot_arm64') sky_utils.copy_binary(gen_snapshot, os.path.join(dst, 'gen_snapshot_arm64')) + sky_utils.copy_binary(analyze_snapshot, os.path.join(dst, 'analyze_snapshot_arm64')) if x64_out_dir: gen_snapshot = os.path.join(x64_out_dir, 'universal', 'gen_snapshot_x64') + analyze_snapshot = os.path.join(x64_out_dir, 'analyze_snapshot_x64') sky_utils.copy_binary(gen_snapshot, os.path.join(dst, 'gen_snapshot_x64')) + sky_utils.copy_binary(analyze_snapshot, os.path.join(dst, 'analyze_snapshot_x64')) zip_archive(dst, args) return 0 @@ -177,7 +182,7 @@ def zip_archive(dst, args): # See: https://github.com/flutter/flutter/blob/62382c7b83a16b3f48dc06c19a47f6b8667005a5/dev/bots/suite_runners/run_verify_binaries_codesigned_tests.dart#L82-L130 # Binaries that must be codesigned and require entitlements for particular APIs. - with_entitlements = ['gen_snapshot_arm64'] + with_entitlements = ['analyze_snapshot_arm64', 'gen_snapshot_arm64'] with_entitlements_file = os.path.join(dst, 'entitlements.txt') sky_utils.write_codesign_config(with_entitlements_file, with_entitlements) @@ -211,6 +216,7 @@ def zip_archive(dst, args): # pylint: enable=line-too-long zip_contents = [ + 'analyze_snapshot_arm64', 'gen_snapshot_arm64', 'Flutter.xcframework', 'entitlements.txt', diff --git a/engine/src/flutter/testing/run_tests.py b/engine/src/flutter/testing/run_tests.py index ad378f7ea6b9e..bbf729bf63eb7 100755 --- a/engine/src/flutter/testing/run_tests.py +++ b/engine/src/flutter/testing/run_tests.py @@ -485,6 +485,7 @@ def make_test( make_test('platform_view_android_delegate_unittests'), # https://github.com/flutter/flutter/issues/36295 make_test('shell_unittests'), + make_test('shorebird_unittests'), ] if is_windows(): diff --git a/packages/flutter_tools/bin/xcode_backend.dart b/packages/flutter_tools/bin/xcode_backend.dart index 2142ba29bf1c2..9f6c8983a39df 100644 --- a/packages/flutter_tools/bin/xcode_backend.dart +++ b/packages/flutter_tools/bin/xcode_backend.dart @@ -692,6 +692,14 @@ class Context { ); } + // Shorebird-specific build-trace plumbing: mac.dart sets + // SHOREBIRD_TRACE_FILE in the Xcode build environment; here (running + // as an Xcode build phase script) we forward it to flutter assemble. + final String? shorebirdTraceFile = environment['SHOREBIRD_TRACE_FILE']; + if (shorebirdTraceFile != null && shorebirdTraceFile.isNotEmpty) { + flutterArgs.add('--shorebird-trace-file=$shorebirdTraceFile'); + } + if (environment['CODE_SIZE_DIRECTORY'] != null && environment['CODE_SIZE_DIRECTORY']!.isNotEmpty) { flutterArgs.add('-dCodeSizeDirectory=${environment['CODE_SIZE_DIRECTORY']}'); diff --git a/packages/flutter_tools/gradle/aar_init_script.gradle b/packages/flutter_tools/gradle/aar_init_script.gradle index b1bcfeef403b1..283e0343c28d6 100644 --- a/packages/flutter_tools/gradle/aar_init_script.gradle +++ b/packages/flutter_tools/gradle/aar_init_script.gradle @@ -42,7 +42,7 @@ void configureProject(Project project, String outputDir) { return } - String storageUrl = System.getenv('FLUTTER_STORAGE_BASE_URL') ?: "https://storage.googleapis.com" + String storageUrl = System.getenv('FLUTTER_STORAGE_BASE_URL') ?: "https://download.shorebird.dev" String engineRealm = Paths.get(getFlutterRoot(project), "bin", "cache", "engine.realm") .toFile().text.trim() diff --git a/packages/flutter_tools/gradle/build.gradle.kts b/packages/flutter_tools/gradle/build.gradle.kts index 43f5e8368bc41..9b4516d4ebc5f 100644 --- a/packages/flutter_tools/gradle/build.gradle.kts +++ b/packages/flutter_tools/gradle/build.gradle.kts @@ -66,6 +66,7 @@ dependencies { // * ndkVersion in FlutterExtension in packages/flutter_tools/gradle/src/main/kotlin/FlutterExtension.kt compileOnly("com.android.tools.build:gradle:8.11.1") + implementation("org.yaml:snakeyaml:2.0") testImplementation(kotlin("test")) testImplementation("com.android.tools.build:gradle:8.11.1") testImplementation("org.mockito:mockito-core:5.8.0") diff --git a/packages/flutter_tools/gradle/shorebird_trace_init.gradle b/packages/flutter_tools/gradle/shorebird_trace_init.gradle new file mode 100644 index 0000000000000..8f469f87397bf --- /dev/null +++ b/packages/flutter_tools/gradle/shorebird_trace_init.gradle @@ -0,0 +1,159 @@ +// Emits Chrome Trace Event Format events for every Gradle task so the +// Shorebird build trace (`flutter build ... --shorebird-trace`) can show +// per-task timings alongside flutter tool + flutter assemble spans. +// +// This file is Shorebird-specific โ€” it exists in the Shorebird fork of +// Flutter and has no equivalent upstream. Activated by passing +// `-I=` together with `-Pshorebird.gradle-trace-file=`. +// Output file is consumed by Flutter's gradle.dart and merged into the +// main trace. +// +// Events land on tid=4 (flutter tool = 1, native build outer = 2, flutter +// assemble = 3) so Perfetto shows a clean per-tier layout. + +import groovy.json.JsonOutput +import org.gradle.api.Task +import org.gradle.api.execution.TaskExecutionListener +import org.gradle.api.tasks.TaskState + +def traceFilePath = gradle.startParameter.projectProperties['shorebird.gradle-trace-file'] +if (!traceFilePath) { + traceFilePath = System.getProperty('shorebird.gradle-trace-file') +} +if (!traceFilePath) { + return +} + +// Flow id shared with flutter_tool so Perfetto draws an arrow from the +// parent "gradle " span into our first per-task event. Missing +// when flutter_tool is older than the flow-id plumbing; that's fine, +// the init script just skips the flow-end event in that case. +def gradleFlowIdRaw = gradle.startParameter.projectProperties['shorebird.gradle-trace-flow-id'] +def gradleFlowId = gradleFlowIdRaw ? gradleFlowIdRaw.toLong() : null + +// Real pid of this Gradle JVM โ€” Gradle tasks belong to this process, so +// emitting spans on it matches the process topology. Named via a +// `process_name` metadata event below so Perfetto labels the row +// "gradle" rather than showing a bare number. +def gradlePid = ProcessHandle.current().pid() +def gradleTid = 1 + +def events = Collections.synchronizedList([ + [ + name: 'process_name', + ph: 'M', + pid: gradlePid, + args: [name: 'gradle'], + ], + [ + name: 'thread_name', + ph: 'M', + pid: gradlePid, + tid: gradleTid, + args: [name: 'gradle tasks'], + ], +]) +def firstTaskFlowEmitted = new java.util.concurrent.atomic.AtomicBoolean(false) +def startTimesMicros = Collections.synchronizedMap([:]) + +gradle.addListener(new TaskExecutionListener() { + @Override + void beforeExecute(Task task) { + startTimesMicros[task.path] = System.currentTimeMillis() * 1000L + } + + @Override + void afterExecute(Task task, TaskState state) { + def start = startTimesMicros.remove(task.path) + if (start == null) return + def end = System.currentTimeMillis() * 1000L + def dur = Math.max(0L, end - start) + // Classify the task path into a small set of "kinds" so downstream + // summarizers can bucket without retaining names. Match against the + // task's simple name (last colon segment) so plugin names like + // `:package_info_plus:...` don't accidentally trigger the `packaging` + // bucket. + // + // The kind strings below are a wire-format contract with + // shorebird_cli (which buckets its summary by these values). + // KEEP IN SYNC with the `GradleTaskKind` enum in the + // `shorebird_build_trace` package; adding a new kind means + // landing the matching enum value there AND teaching + // shorebird_cli to bucket it, otherwise it silently falls into + // `other`. + def path = task.path + def lastColon = path.lastIndexOf(':') + def simpleName = (lastColon >= 0 ? path.substring(lastColon + 1) : path).toLowerCase() + + // Order matters: the first regex that matches wins. `kotlin_compile` + // must precede `java_compile` (both match `compile*`). `java_compile` + // must precede the `scaffold` bucket's `prepare*` because AGP's + // javaPreCompileRelease is scaffolding-shaped but semantically javac. + // `flutter_gradle_plugin` must precede `packaging` so tasks like + // `packageFlutterBuild` don't fall through to the `package*` bucket. + def kindRules = [ + [~/^compile.*kotlin.*/, 'kotlin_compile'], + [~/(^compile.*(java|jni).*|.*precompile.*)/, 'java_compile'], + [~/.*aidl.*/, 'aidl'], + [~/(^minify.*|.*r8.*)/, 'r8_minify'], + [~/.*dex.*/, 'dex'], + [~/^lint.*/, 'lint'], + [~/(.*jnilibs.*|.*nativelibs.*|.*link.*native.*)/, 'native_link'], + [~/.*(processresources|mergeresources|mergemanifest|rfile|parserelease|mergerelease|generaterelease).*/, 'resources'], + [~/(^compileflutter.*|.*flutterbuild.*|^flutter.*)/, 'flutter_gradle_plugin'], + [~/^package.*/, 'packaging'], + [~/.*bundle.*/, 'bundle'], + [~/.*transform.*/, 'transform'], + // Catch-all for Gradle's per-plugin / per-variant scaffolding + // (metadata files, proguard rule export, pre-/post-compile + // bookkeeping). Individually small but the long tail adds up on + // plugin-heavy apps. + [~/(.*aarmetadata.*|.*proguard.*|.*validate.*|^check.*|^prepare.*|^generate.*|^copy.*)/, 'gradle_scaffold'], + ] + def kind = kindRules.find { simpleName ==~ it[0] }?.getAt(1) ?: 'other' + + // The task "owner" is the first colon-separated segment. For + // subproject tasks this is typically the plugin name (e.g. + // `:camera_android`). Kept for local debug โ€” the privacy-safe + // summary that Shorebird writes aggregates this out. + def owner = path.startsWith(':') ? path.substring(1).split(':')[0] : '' + + // Emit the flow-end event on the first task, connecting back to + // flutter_tool's "gradle " span. Atomic so we emit at most + // one across concurrent task callbacks. + if (gradleFlowId != null && firstTaskFlowEmitted.compareAndSet(false, true)) { + events.add([ + ph: 'f', + name: 'spawn', + cat: 'flow', + id: gradleFlowId, + ts: start, + pid: gradlePid, + tid: gradleTid, + bp: 'e', + ]) + } + events.add([ + name: path, + cat: 'gradle_task', + ph: 'X', + ts: start, + dur: dur, + pid: gradlePid, + tid: gradleTid, + args: [ + kind: kind, + owner: owner, + skipped: state.skipped, + upToDate: state.upToDate, + fromCache: state.skipMessage == 'FROM-CACHE', + ], + ]) + } +}) + +gradle.buildFinished { + def f = new File(traceFilePath) + f.parentFile?.mkdirs() + f.text = JsonOutput.toJson(events) +} diff --git a/packages/flutter_tools/gradle/src/main/kotlin/FlutterPlugin.kt b/packages/flutter_tools/gradle/src/main/kotlin/FlutterPlugin.kt index 35edcec55254b..7cf22e36a7ab0 100644 --- a/packages/flutter_tools/gradle/src/main/kotlin/FlutterPlugin.kt +++ b/packages/flutter_tools/gradle/src/main/kotlin/FlutterPlugin.kt @@ -645,6 +645,8 @@ class FlutterPlugin : Plugin { val dartDefinesValue: String? = project.findProperty("dart-defines")?.toString() val performanceMeasurementFileValue: String? = project.findProperty("performance-measurement-file")?.toString() + val shorebirdTraceFileValue: String? = + project.findProperty("shorebird-trace-file")?.toString() val codeSizeDirectoryValue: String? = project.findProperty("code-size-directory")?.toString() val deferredComponentsValue: Boolean = @@ -735,6 +737,7 @@ class FlutterPlugin : Plugin { dartObfuscation = dartObfuscationValue dartDefines = dartDefinesValue performanceMeasurementFile = performanceMeasurementFileValue + shorebirdTraceFile = shorebirdTraceFileValue codeSizeDirectory = codeSizeDirectoryValue deferredComponents = deferredComponentsValue validateDeferredComponents = validateDeferredComponentsValue diff --git a/packages/flutter_tools/gradle/src/main/kotlin/FlutterPluginConstants.kt b/packages/flutter_tools/gradle/src/main/kotlin/FlutterPluginConstants.kt index 13eb2be26758b..61e8e3c17c006 100644 --- a/packages/flutter_tools/gradle/src/main/kotlin/FlutterPluginConstants.kt +++ b/packages/flutter_tools/gradle/src/main/kotlin/FlutterPluginConstants.kt @@ -21,7 +21,7 @@ object FlutterPluginConstants { const val INTERMEDIATES_DIR = "intermediates" const val FLUTTER_STORAGE_BASE_URL = "FLUTTER_STORAGE_BASE_URL" - const val DEFAULT_MAVEN_HOST = "https://storage.googleapis.com" + const val DEFAULT_MAVEN_HOST = "https://download.shorebird.dev" /** Maps platforms to ABI architectures. */ @JvmStatic val PLATFORM_ARCH_MAP = diff --git a/packages/flutter_tools/gradle/src/main/kotlin/tasks/BaseFlutterTask.kt b/packages/flutter_tools/gradle/src/main/kotlin/tasks/BaseFlutterTask.kt index 1858a139d0e96..96331ef87c94d 100644 --- a/packages/flutter_tools/gradle/src/main/kotlin/tasks/BaseFlutterTask.kt +++ b/packages/flutter_tools/gradle/src/main/kotlin/tasks/BaseFlutterTask.kt @@ -111,6 +111,12 @@ open class BaseFlutterTask : DefaultTask() { @Input var performanceMeasurementFile: String? = null + // Shorebird-specific: path for the Shorebird build trace. Prefixed so + // it stands out as fork-only surface in this otherwise upstream file. + @Optional + @Input + var shorebirdTraceFile: String? = null + @Optional @Input var deferredComponents: Boolean? = null diff --git a/packages/flutter_tools/gradle/src/main/kotlin/tasks/BaseFlutterTaskHelper.kt b/packages/flutter_tools/gradle/src/main/kotlin/tasks/BaseFlutterTaskHelper.kt index 2d16640d324fd..2f5fac2de3a76 100644 --- a/packages/flutter_tools/gradle/src/main/kotlin/tasks/BaseFlutterTaskHelper.kt +++ b/packages/flutter_tools/gradle/src/main/kotlin/tasks/BaseFlutterTaskHelper.kt @@ -110,6 +110,9 @@ object BaseFlutterTaskHelper { baseFlutterTask.performanceMeasurementFile?.let { args("--performance-measurement-file=$it") } + baseFlutterTask.shorebirdTraceFile?.let { + args("--shorebird-trace-file=$it") + } args("-dTargetFile=${baseFlutterTask.targetPath}") args("-dTargetPlatform=android") args("-dBuildMode=${baseFlutterTask.buildMode}") diff --git a/packages/flutter_tools/lib/src/android/gradle.dart b/packages/flutter_tools/lib/src/android/gradle.dart index 068732f6efa19..1c786709848be 100644 --- a/packages/flutter_tools/lib/src/android/gradle.dart +++ b/packages/flutter_tools/lib/src/android/gradle.dart @@ -29,6 +29,7 @@ import '../convert.dart'; import '../flutter_manifest.dart'; import '../globals.dart' as globals; import '../project.dart'; +import '../shorebird/android_build_trace_session.dart'; import 'android_builder.dart'; import 'android_studio.dart'; import 'gradle_errors.dart'; @@ -278,6 +279,7 @@ class AndroidGradleBuilder implements AndroidBuilder { VoidCallback? postRunTask, int? maxRetries, _OutputParser? outputParser, + void Function(Process process)? onStart, }) async { final bool usesAndroidX = isAppUsingAndroidX(project.android.hostAppGradleRoot); final String? agpVersion = gradle.getAgpVersion( @@ -354,6 +356,7 @@ class AndroidGradleBuilder implements AndroidBuilder { allowReentrantFlutter: true, environment: _java?.environment, mapFunction: consumeLog, + onStart: onStart, ); } on ProcessException catch (exception) { consumeLog(exception.toString()); @@ -482,7 +485,12 @@ class AndroidGradleBuilder implements AndroidBuilder { return; } - // Assembly work starts here. + final AndroidBuildTraceSession? traceSession = AndroidBuildTraceSession.maybeStart( + androidBuildInfo, + _fileSystem, + project.android.buildDirectory, + ); + final BuildInfo buildInfo = androidBuildInfo.buildInfo; final String assembleTask = isBuildingBundle ? getBundleTaskFor(buildInfo) @@ -559,6 +567,7 @@ class AndroidGradleBuilder implements AndroidBuilder { } } options.addAll(androidBuildInfo.buildInfo.toGradleConfig()); + options.addAll(traceSession?.extraGradleOptions(assembleTask) ?? const []); if (buildInfo.fileSystemRoots.isNotEmpty) { options.add('-Pfilesystem-roots=${buildInfo.fileSystemRoots.join('|')}'); } @@ -572,8 +581,10 @@ class AndroidGradleBuilder implements AndroidBuilder { final int exitCode = await _runGradleTask( assembleTask, preRunTask: () { + traceSession?.onGradleAboutToStart(); sw = Stopwatch()..start(); }, + onStart: (process) => traceSession?.onGradleSpawn(process), postRunTask: () { final Duration elapsedDuration = sw.elapsed; _analytics.send( @@ -591,13 +602,21 @@ class AndroidGradleBuilder implements AndroidBuilder { gradleExecutablePath: gradleExecutablePath, ); + traceSession?.onGradleFinished(assembleTask); + if (exitCode != 0) { + traceSession?.abortOnGradleFailure(); throwToolExit( 'Gradle task $assembleTask failed with exit code $exitCode', exitCode: exitCode, ); } + traceSession?.finish( + buildTarget: isBuildingBundle ? 'appbundle' : 'apk', + printStatus: _logger.printStatus, + ); + if (isBuildingBundle) { final File bundleFile = findBundleFile(project, buildInfo, _logger, _analytics); @@ -719,10 +738,26 @@ class AndroidGradleBuilder implements AndroidBuilder { } if (!(result.stdout.contains('libapp.so.sym') || result.stdout.contains('libapp.so.dbg'))) { - _logger.printTrace( - 'libapp.so.sym or libapp.so.dbg not present when checking final appbundle for debug symbols.', + // Shorebird-specific: demote upstream's fatal check to a warning. + // + // Upstream Flutter PR #181275 (3.44) inverted libapp.so strip + // responsibility โ€” Flutter stopped stripping it, AGP is expected to, + // and this check fails the build if `.sym/.dbg` is absent. Shorebird + // flows that either keep libapp.so unstripped (legacy fixtures with + // `keepDebugSymbols.add("**/libapp.so")`) or pre-strip via + // `--extra-gen-snapshot-options=--strip` (obfuscated builds) both + // produce AABs without `libapp.so.sym`. Returning `true` here lets the + // build succeed; the trade-off is that Play Console won't get Dart + // crash symbols for these AABs. New apps should remove the legacy + // `keepDebugSymbols` line to re-enable AGP-side stripping. + _logger.printWarning( + 'libapp.so.sym or libapp.so.dbg not present when checking final ' + 'appbundle for debug symbols. Play Console will not receive Dart ' + 'crash symbols for this build. To enable, remove ' + '`packaging.jniLibs.keepDebugSymbols.add("**/libapp.so")` from ' + 'android/app/build.gradle.kts.', ); - return false; + return true; } return true; diff --git a/packages/flutter_tools/lib/src/base/build.dart b/packages/flutter_tools/lib/src/base/build.dart index fa7ffb45912ee..ec06f9ee3efe0 100644 --- a/packages/flutter_tools/lib/src/base/build.dart +++ b/packages/flutter_tools/lib/src/base/build.dart @@ -2,13 +2,14 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. +import 'dart:io' show Platform; + import 'package:process/process.dart'; import '../artifacts.dart'; import '../build_info.dart'; import '../darwin/darwin.dart'; import '../macos/xcode.dart'; - import 'file_system.dart'; import 'logger.dart'; import 'process.dart'; @@ -30,7 +31,9 @@ class GenSnapshot { required Artifacts artifacts, required ProcessManager processManager, required Logger logger, + required FileSystem fileSystem, }) : _artifacts = artifacts, + _fileSystem = fileSystem, _processUtils = ProcessUtils(logger: logger, processManager: processManager); final Artifacts _artifacts; @@ -55,6 +58,75 @@ class GenSnapshot { ' See dartbug.com/30524 for more information.', }; + /// Returns the path to an analyze_snapshot binary that matches gen_snapshot's + /// SDK version, or null if no matching binary can be found. + /// + /// gen_snapshot and analyze_snapshot share a snapshot-format hash baked into + /// each binary; if the hashes disagree, analyze_snapshot rejects gen_snapshot's + /// output with "Wrong full snapshot version" at parse time. Pre-existing binary + /// layouts (e.g. an older `universal/analyze_snapshot_` left behind from + /// a previous BUILD.gn revision) can shadow the freshly built one. Resolve by + /// probing every candidate location and returning only the one whose + /// `--sdk_version` matches gen_snapshot's `--version` output. + String? getAnalyzeSnapshotPath(SnapshotType snapshotType, DarwinArch? darwinArch) { + final Artifact genSnapshotArtifact; + final String analyzeName; + if (snapshotType.platform == TargetPlatform.ios || + snapshotType.platform == TargetPlatform.darwin) { + if (darwinArch == DarwinArch.arm64) { + genSnapshotArtifact = Artifact.genSnapshotArm64; + analyzeName = 'analyze_snapshot_arm64'; + } else { + genSnapshotArtifact = Artifact.genSnapshotX64; + analyzeName = 'analyze_snapshot_x64'; + } + } else { + genSnapshotArtifact = Artifact.genSnapshot; + analyzeName = 'analyze_snapshot'; + } + final String genSnapshotPath = getSnapshotterPath(snapshotType, genSnapshotArtifact); + final String? genVersion = _probeSdkVersion(genSnapshotPath, '--version'); + final String dir = _fileSystem.path.dirname(genSnapshotPath); + // Cached SDK layout puts analyze_snapshot alongside gen_snapshot. Local + // engine layout resolves gen_snapshot via .../universal/gen_snapshot_arm64 + // while analyze_snapshot lives one level up at the build-dir root. Probe + // both. If we successfully read gen_snapshot's version, accept only a + // candidate whose --sdk_version matches; otherwise fall back to the first + // existing candidate (better than failing closed when the version probe + // itself misbehaves). + for (final candidate in [ + _fileSystem.path.join(dir, analyzeName), + _fileSystem.path.join(dir, '..', analyzeName), + ]) { + if (!_fileSystem.file(candidate).existsSync()) { + continue; + } + if (genVersion == null) { + return candidate; + } + final String? candidateVersion = _probeSdkVersion(candidate, '--sdk_version'); + if (candidateVersion == genVersion) { + return candidate; + } + } + return null; + } + + /// Runs [binary] with [versionFlag] and returns the trimmed stderr output, + /// or null if the binary couldn't be invoked or printed nothing. + /// gen_snapshot and analyze_snapshot both write their version line to stderr. + String? _probeSdkVersion(String binary, String versionFlag) { + try { + final RunResult result = _processUtils.runSync([binary, versionFlag]); + final String stderr = result.stderr.trim(); + return stderr.isEmpty ? null : stderr; + } on Exception { + return null; + } + } + + final FileSystem _fileSystem; + Future run({ required SnapshotType snapshotType, DarwinArch? darwinArch, @@ -96,15 +168,18 @@ class AOTSnapshotter { }) : _logger = logger, _fileSystem = fileSystem, _xcode = xcode, + _processUtils = ProcessUtils(logger: logger, processManager: processManager), _genSnapshot = GenSnapshot( artifacts: artifacts, processManager: processManager, logger: logger, + fileSystem: fileSystem, ); final Logger _logger; final FileSystem _fileSystem; final Xcode _xcode; + final ProcessUtils _processUtils; final GenSnapshot _genSnapshot; /// Builds an architecture-specific ahead-of-time compiled snapshot of the specified script. @@ -130,7 +205,27 @@ class AOTSnapshotter { final Directory outputDir = _fileSystem.directory(outputPath); outputDir.createSync(recursive: true); - final genSnapshotArgs = ['--deterministic']; + // Currently we only use the linker on iOS, but we will eventually split out + // the concept of "optimizes patch snapshot" from "uses linker" and probably + // only uses the linker on iOS, but optimize patch snapshots everywhere. + // TODO(eseidel): TargetPlatform.darwin doesn't use the linker. + bool usesLinker = (platform == TargetPlatform.ios || platform == TargetPlatform.darwin); + final dumpLinkInfoArgs = [ + // Shorebird dumps the class table information during snapshot compilation which is later used during linking. + '--print_class_table_link_debug_info_to=${_fileSystem.path.join(outputDir.parent.path, 'App.class_table.json')}', + '--print_class_table_link_info_to=${_fileSystem.path.join(outputDir.parent.path, 'App.ct.link')}', + '--print_field_table_link_debug_info_to=${_fileSystem.path.join(outputDir.parent.path, 'App.field_table.json')}', + '--print_field_table_link_info_to=${_fileSystem.path.join(outputDir.parent.path, 'App.ft.link')}', + '--print_dispatch_table_link_debug_info_to=${_fileSystem.path.join(outputDir.parent.path, 'App.dispatch_table.json')}', + '--print_dispatch_table_link_info_to=${_fileSystem.path.join(outputDir.parent.path, 'App.dt.link')}', + ]; + + final genSnapshotArgs = [ + // Shorebird uses --deterministic to improve snapshot stability and increase linking. + '--deterministic', + // Only save LinkInfo if we're using the linker. + if (usesLinker) ...dumpLinkInfoArgs, + ]; final bool targetingApplePlatform = platform == TargetPlatform.ios || platform == TargetPlatform.darwin; @@ -140,13 +235,6 @@ class AOTSnapshotter { buildMode == BuildMode.profile || buildMode == BuildMode.release; _logger.printTrace('extractAppleDebugSymbols = $extractAppleDebugSymbols'); - final bool targetingAndroidPlatform = - platform == TargetPlatform.android || - platform == TargetPlatform.android_arm || - platform == TargetPlatform.android_arm64 || - platform == TargetPlatform.android_x64; - _logger.printTrace('targetingAndroidPlatform = $targetingAndroidPlatform'); - // We strip snapshot by default, but allow to suppress this behavior // by supplying --no-strip in extraGenSnapshotOptions. var shouldStrip = true; @@ -161,37 +249,11 @@ class AOTSnapshotter { } } - String aotSharedLibrary = _fileSystem.path.join(outputDir.path, 'app.so'); - String? frameworkPath; + final String assembly = _fileSystem.path.join(outputDir.path, 'snapshot_assembly.S'); if (targetingApplePlatform) { - // On iOS and macOS, we use Xcode to compile the snapshot into a dynamic - // library that the end-developer can link into their app. - const frameworkName = 'App.framework'; - if (!quiet) { - final String targetArch = darwinArch!.name; - _logger.printStatus('Building $frameworkName for $targetArch...'); - } - frameworkPath = _fileSystem.path.join(outputPath, frameworkName); - _fileSystem.directory(frameworkPath).createSync(recursive: true); - - const frameworkSnapshotName = 'App'; - aotSharedLibrary = _fileSystem.path.join(frameworkPath, frameworkSnapshotName); - final String relocatableObject = _fileSystem.path.join(outputPath, 'app.o'); - // When the minimum version is updated, remember to update - // template MinimumOSVersion. - // https://github.com/flutter/flutter/pull/62902 - final minOSVersion = platform == TargetPlatform.ios - ? FlutterDarwinPlatform.ios.deploymentTarget().toString() - : FlutterDarwinPlatform.macos.deploymentTarget().toString(); - genSnapshotArgs.addAll([ - '--snapshot_kind=app-aot-macho-dylib', - '--macho=$aotSharedLibrary', - '--macho-object=$relocatableObject', - '--macho-min-os-version=$minOSVersion', - '--macho-rpath=@executable_path/Frameworks,@loader_path/Frameworks', - '--macho-install-name=@rpath/$frameworkName/$frameworkSnapshotName', - ]); + genSnapshotArgs.addAll(['--snapshot_kind=app-aot-assembly', '--assembly=$assembly']); } else { + final String aotSharedLibrary = _fileSystem.path.join(outputDir.path, 'app.so'); genSnapshotArgs.addAll(['--snapshot_kind=app-aot-elf', '--elf=$aotSharedLibrary']); } @@ -203,9 +265,6 @@ class AOTSnapshotter { if (stripAfterBuild) { _logger.printTrace('Will strip AOT snapshot manually after build and dSYM generation.'); } - } else if (targetingAndroidPlatform) { - stripAfterBuild = false; - // When building for Android, we let AGP handle stripping of debug symbols. } else { stripAfterBuild = false; if (shouldStrip) { @@ -246,6 +305,39 @@ class AOTSnapshotter { genSnapshotArgs.add(mainPath); final snapshotType = SnapshotType(platform, buildMode); + + final int ddMaxBytes = _readDdMaxBytes(); + // DD pass requires: + // 1. an analyze_snapshot binary whose --sdk_version matches gen_snapshot's + // --version, and + // 2. a single non-racing ELF output path per build invocation. + // + // macOS builds fail both. (1) The shipped `analyze_snapshot` is a single + // host-arch binary; when targeting x64 from an arm64 host, gen_snapshot_x64 + // reports `macos_simx64` while analyze_snapshot reports `macos_arm64` and + // the version-equality check rejects it. (2) Flutter spawns + // gen_snapshot_arm64 and gen_snapshot_x64 in parallel and both DD passes + // race on the same `App_dd_analysis.so` path under + // `.dart_tool/flutter_build//`. + // + // Skip DD on macOS until that work is done. Patches built without DD + // activation will compute DD on the fly when applied, which is the + // pre-1.6.99 behavior โ€” slightly worse link percentage but functional. + final bool ddPassSupported = platform != TargetPlatform.darwin; + if (ddMaxBytes > 0 && usesLinker && ddPassSupported) { + final int pass1Exit = await _runDdAnalysisPass( + snapshotType: snapshotType, + darwinArch: darwinArch, + outputDir: outputDir, + baseGenSnapshotArgs: genSnapshotArgs, + mainPath: mainPath, + ddMaxBytes: ddMaxBytes, + ); + if (pass1Exit != 0) { + return pass1Exit; + } + } + final int genSnapshotExitCode = await _genSnapshot.run( snapshotType: snapshotType, additionalArgs: genSnapshotArgs, @@ -256,38 +348,256 @@ class AOTSnapshotter { return genSnapshotExitCode; } + // On iOS and macOS, we use Xcode to compile the snapshot into a dynamic library that the + // end-developer can link into their app. if (targetingApplePlatform) { - if (extractAppleDebugSymbols) { - final RunResult dsymResult = await _xcode.dsymutil([ - '-o', - '$frameworkPath.dSYM', - aotSharedLibrary, - ]); - if (dsymResult.exitCode != 0) { + return _buildFramework( + appleArch: darwinArch!, + isIOS: platform == TargetPlatform.ios, + sdkRoot: sdkRoot, + assemblyPath: assembly, + outputPath: outputDir.path, + quiet: quiet, + stripAfterBuild: stripAfterBuild, + extractAppleDebugSymbols: extractAppleDebugSymbols, + ); + } else { + return 0; + } + } + + /// Reads the SHOREBIRD_DD_MAX_BYTES env var (preferring `dart-define` over + /// the process environment). Returns 0 if unset, malformed, or non-positive, + /// which signals "no DD pass." + int _readDdMaxBytes() { + const fromDefine = String.fromEnvironment('SHOREBIRD_DD_MAX_BYTES'); + final String? raw = fromDefine.isNotEmpty + ? fromDefine + : Platform.environment['SHOREBIRD_DD_MAX_BYTES']; + return int.tryParse(raw ?? '') ?? 0; + } + + /// Runs the DD analysis pass: gen_snapshot โ†’ ELF + DD identity, then + /// analyze_snapshot to compute the DD table, caller links, and slot mapping. + /// On success, mutates [baseGenSnapshotArgs] to add `--dd_slot_mapping=...` + /// before [mainPath] so the subsequent gen_snapshot run picks it up. + /// Returns the gen_snapshot pass-1 exit code (0 on success). + Future _runDdAnalysisPass({ + required SnapshotType snapshotType, + required DarwinArch? darwinArch, + required Directory outputDir, + required List baseGenSnapshotArgs, + required String mainPath, + required int ddMaxBytes, + }) async { + _logger.printTrace('DD 2-pass build: dd_max_bytes=$ddMaxBytes'); + + String linkPath(String name) => _fileSystem.path.join(outputDir.parent.path, name); + final String elfForAnalysis = linkPath('App_dd_analysis.so'); + final String ddIdentityPath = linkPath('App.dd_identity.link'); + final String ddTablePath = linkPath('App.dd.link'); + final String ddCallerLinksPath = linkPath('App.dd_callers.link'); + final String ddSlotMappingPath = linkPath('App.dd_slots.link'); + final String ddResolutionPath = linkPath('App.dd_resolution.tsv'); + + // Pass 1: build ELF for analysis + DD identity. Strip the existing snapshot + // kind/output args from the base set; mainPath must remain at the end. + final elfArgs = [ + ...baseGenSnapshotArgs.where( + (String a) => + a != mainPath && + !a.startsWith('--snapshot_kind=') && + !a.startsWith('--assembly=') && + !a.startsWith('--elf='), + ), + '--snapshot_kind=app-aot-elf', + '--elf=$elfForAnalysis', + '--print_dd_function_identity_to=$ddIdentityPath', + mainPath, + ]; + final int pass1Exit = await _genSnapshot.run( + snapshotType: snapshotType, + additionalArgs: elfArgs, + darwinArch: darwinArch, + ); + if (pass1Exit != 0) { + _logger.printError('DD pass 1 (ELF for analysis) failed with exit code $pass1Exit'); + return pass1Exit; + } + + final String? analyzeSnapshotPath = _genSnapshot.getAnalyzeSnapshotPath( + snapshotType, + darwinArch, + ); + if (analyzeSnapshotPath == null) { + _logger.printError( + 'DD pass: could not find an analyze_snapshot binary whose --sdk_version ' + 'matches gen_snapshot. The release will ship without DD activation; ' + 'patches against it will fall back to on-the-fly DD computation and ' + 'produce a structurally divergent snapshot (devastating link percentage). ' + 'Aborting the build instead.', + ); + _fileSystem.file(elfForAnalysis).deleteSync(); + return 1; + } + + final int ddTableExit = await _processUtils.stream([ + analyzeSnapshotPath, + '--compute_dd_table=$ddTablePath', + '--dd_caller_links=$ddCallerLinksPath', + '--dd_max_bytes=$ddMaxBytes', + elfForAnalysis, + ]); + if (ddTableExit != 0) { + _logger.printError( + 'DD pass: analyze_snapshot --compute_dd_table failed with exit code ' + '$ddTableExit. App.dd.link will not be produced and the release would ' + 'ship without DD activation.', + ); + _fileSystem.file(elfForAnalysis).deleteSync(); + return ddTableExit; + } + + final int ddSlotMappingExit = await _processUtils.stream([ + analyzeSnapshotPath, + '--compute_dd_slot_mapping=$ddSlotMappingPath', + '--dd_table_data=$ddTablePath', + '--dd_caller_links=$ddCallerLinksPath', + '--dd_function_identity=$ddIdentityPath', + elfForAnalysis, + ]); + if (ddSlotMappingExit != 0) { + _logger.printError( + 'DD pass: analyze_snapshot --compute_dd_slot_mapping failed with exit ' + 'code $ddSlotMappingExit. The DD slot mapping will not be available and ' + 'gen_snapshot pass 2 would emit a no-DD snapshot.', + ); + _fileSystem.file(elfForAnalysis).deleteSync(); + return ddSlotMappingExit; + } + + if (!_fileSystem.file(ddSlotMappingPath).existsSync()) { + _logger.printError( + 'DD pass: analyze_snapshot --compute_dd_slot_mapping reported success ' + 'but $ddSlotMappingPath was not produced.', + ); + _fileSystem.file(elfForAnalysis).deleteSync(); + return 1; + } + final int mainPathIndex = baseGenSnapshotArgs.indexOf(mainPath); + baseGenSnapshotArgs.insertAll(mainPathIndex, [ + '--dd_slot_mapping=$ddSlotMappingPath', + // Per-slot resolution outcome dump (TSV: slot, outcome, rewritten, + // name). Diagnostic for analyzing which slots got dropped during + // the resolver's run; ends up alongside the other supplement files + // and gets carried into the patch debug bundle. + '--print_dd_resolution_to=$ddResolutionPath', + ]); + _logger.printTrace('DD 2-pass build: added --dd_slot_mapping and --print_dd_resolution_to'); + + _fileSystem.file(elfForAnalysis).deleteSync(); + return 0; + } + + /// Builds an iOS or macOS framework at [outputPath]/App.framework from the assembly + /// source at [assemblyPath]. + Future _buildFramework({ + required DarwinArch appleArch, + required bool isIOS, + String? sdkRoot, + required String assemblyPath, + required String outputPath, + required bool quiet, + required bool stripAfterBuild, + required bool extractAppleDebugSymbols, + }) async { + final String targetArch = appleArch.name; + if (!quiet) { + _logger.printStatus('Building App.framework for $targetArch...'); + } + + final commonBuildOptions = [ + '-arch', + targetArch, + if (isIOS) + // When the minimum version is updated, remember to update + // template MinimumOSVersion. + // https://github.com/flutter/flutter/pull/62902 + '-miphoneos-version-min=${FlutterDarwinPlatform.ios.deploymentTarget()}', + if (sdkRoot != null) ...['-isysroot', sdkRoot], + ]; + + final String assemblyO = _fileSystem.path.join(outputPath, 'snapshot_assembly.o'); + + final RunResult compileResult = await _xcode.cc([ + ...commonBuildOptions, + '-c', + assemblyPath, + '-o', + assemblyO, + ]); + if (compileResult.exitCode != 0) { + _logger.printError( + 'Failed to compile AOT snapshot. Compiler terminated with exit code ${compileResult.exitCode}', + ); + return compileResult.exitCode; + } + + final String frameworkDir = _fileSystem.path.join(outputPath, 'App.framework'); + _fileSystem.directory(frameworkDir).createSync(recursive: true); + final String appLib = _fileSystem.path.join(frameworkDir, 'App'); + final linkArgs = [ + ...commonBuildOptions, + '-dynamiclib', + '-Xlinker', + '-rpath', + '-Xlinker', + '@executable_path/Frameworks', + '-Xlinker', + '-rpath', + '-Xlinker', + '@loader_path/Frameworks', + '-fapplication-extension', + '-install_name', + '@rpath/App.framework/App', + '-o', + appLib, + assemblyO, + ]; + + final RunResult linkResult = await _xcode.clang(linkArgs); + if (linkResult.exitCode != 0) { + _logger.printError( + 'Failed to link AOT snapshot. Linker terminated with exit code ${linkResult.exitCode}', + ); + return linkResult.exitCode; + } + + if (extractAppleDebugSymbols) { + final RunResult dsymResult = await _xcode.dsymutil([ + '-o', + '$frameworkDir.dSYM', + appLib, + ]); + if (dsymResult.exitCode != 0) { + _logger.printError( + 'Failed to generate dSYM - dsymutil terminated with exit code ${dsymResult.exitCode}', + ); + return dsymResult.exitCode; + } + + if (stripAfterBuild) { + // See https://www.unix.com/man-page/osx/1/strip/ for arguments + final RunResult stripResult = await _xcode.strip(['-x', appLib, '-o', appLib]); + if (stripResult.exitCode != 0) { _logger.printError( - 'Failed to generate dSYM - dsymutil terminated with exit code ${dsymResult.exitCode}', + 'Failed to strip debugging symbols from the generated AOT snapshot - strip terminated with exit code ${stripResult.exitCode}', ); - return dsymResult.exitCode; - } - - if (stripAfterBuild) { - // See https://www.unix.com/man-page/osx/1/strip/ for arguments - final RunResult stripResult = await _xcode.strip([ - '-x', - aotSharedLibrary, - '-o', - aotSharedLibrary, - ]); - if (stripResult.exitCode != 0) { - _logger.printError( - 'Failed to strip debugging symbols from the generated AOT snapshot - strip terminated with exit code ${stripResult.exitCode}', - ); - return stripResult.exitCode; - } + return stripResult.exitCode; } - } else { - assert(!stripAfterBuild); } + } else { + assert(!stripAfterBuild); } return 0; @@ -305,7 +615,6 @@ class AOTSnapshotter { TargetPlatform.darwin, TargetPlatform.linux_x64, TargetPlatform.linux_arm64, - TargetPlatform.linux_riscv64, TargetPlatform.windows_x64, TargetPlatform.windows_arm64, ].contains(platform); diff --git a/packages/flutter_tools/lib/src/base/net.dart b/packages/flutter_tools/lib/src/base/net.dart index 248824909cf5e..93f0584cdf071 100644 --- a/packages/flutter_tools/lib/src/base/net.dart +++ b/packages/flutter_tools/lib/src/base/net.dart @@ -7,6 +7,7 @@ import 'dart:async'; import 'package:meta/meta.dart'; import '../convert.dart'; +import '../shorebird/network_trace_span.dart'; import 'common.dart'; import 'file_system.dart'; import 'io.dart'; @@ -87,6 +88,8 @@ class Net { Future _attempt(Uri url, {IOSink? destSink, bool onlyHeaders = false}) async { assert(onlyHeaders || destSink != null); _logger.printTrace('Downloading: $url'); + final traceSpan = NetworkTraceSpan.start(url: url, onlyHeaders: onlyHeaders); + final HttpClient httpClient = _httpClientFactory(); HttpClientRequest request; HttpClientResponse? response; @@ -98,6 +101,7 @@ class Net { } response = await request.close(); } on ArgumentError catch (error) { + traceSpan.record(errorKind: 'ArgumentError'); final String? overrideUrl = _platform.environment[kFlutterStorageBaseUrl]; if (overrideUrl != null && url.toString().contains(overrideUrl)) { _logger.printError(error.toString()); @@ -112,6 +116,7 @@ class Net { _logger.printError(error.toString()); rethrow; } on HandshakeException catch (error) { + traceSpan.record(errorKind: 'HandshakeException'); _logger.printTrace(error.toString()); throwToolExit( 'Could not authenticate download server. You may be experiencing a man-in-the-middle attack,\n' @@ -120,29 +125,35 @@ class Net { exitCode: kNetworkProblemExitCode, ); } on SocketException catch (error) { + traceSpan.record(errorKind: 'SocketException'); _logger.printTrace('Download error: $error'); return false; } on HttpException catch (error) { + traceSpan.record(errorKind: 'HttpException'); _logger.printTrace('Download error: $error'); return false; } + final int statusCode = response.statusCode; + // If we're making a HEAD request, we're only checking to see if the URL is // valid. if (onlyHeaders) { - return response.statusCode == HttpStatus.ok; + traceSpan.record(statusCode: statusCode); + return statusCode == HttpStatus.ok; } - if (response.statusCode != HttpStatus.ok) { - if (response.statusCode > 0 && response.statusCode < 500) { + if (statusCode != HttpStatus.ok) { + traceSpan.record(statusCode: statusCode); + if (statusCode > 0 && statusCode < 500) { throwToolExit( 'Download failed.\n' 'URL: $url\n' - 'Error: ${response.statusCode} ${response.reasonPhrase}', + 'Error: $statusCode ${response.reasonPhrase}', exitCode: kNetworkProblemExitCode, ); } // 5xx errors are server errors and we can try again - _logger.printTrace('Download error: ${response.statusCode} ${response.reasonPhrase}'); + _logger.printTrace('Download error: $statusCode ${response.reasonPhrase}'); return false; } _logger.printTrace('Received response from server, collecting bytes...'); @@ -156,6 +167,9 @@ class Net { } finally { await destSink?.flush(); await destSink?.close(); + // Record after the body has been fully drained so `dur` reflects + // end-to-end download time, not just time-to-first-byte. + traceSpan.record(statusCode: statusCode); } } } diff --git a/packages/flutter_tools/lib/src/base/process.dart b/packages/flutter_tools/lib/src/base/process.dart index 187f5bfa8b728..35813f51983c3 100644 --- a/packages/flutter_tools/lib/src/base/process.dart +++ b/packages/flutter_tools/lib/src/base/process.dart @@ -238,6 +238,7 @@ abstract class ProcessUtils { RegExp? stdoutErrorMatcher, StringConverter? mapFunction, Map? environment, + void Function(Process process)? onStart, }); bool exitsHappySync(List cli, {Map? environment}); @@ -554,6 +555,7 @@ class _DefaultProcessUtils implements ProcessUtils { RegExp? stdoutErrorMatcher, StringConverter? mapFunction, Map? environment, + void Function(Process process)? onStart, }) async { final Process process = await start( cmd, @@ -561,6 +563,7 @@ class _DefaultProcessUtils implements ProcessUtils { allowReentrantFlutter: allowReentrantFlutter, environment: environment, ); + onStart?.call(process); final StreamSubscription stdoutSubscription = process.stdout .transform(utf8LineDecoder) .where((String line) => filter == null || filter.hasMatch(line)) diff --git a/packages/flutter_tools/lib/src/build_info.dart b/packages/flutter_tools/lib/src/build_info.dart index 29ef12f7bb891..cdb4290180e9c 100644 --- a/packages/flutter_tools/lib/src/build_info.dart +++ b/packages/flutter_tools/lib/src/build_info.dart @@ -44,6 +44,7 @@ class BuildInfo { List? dartExperiments, required this.treeShakeIcons, this.performanceMeasurementFile, + this.shorebirdTraceFilePath, required this.packageConfigPath, this.codeSizeDirectory, this.androidGradleDaemon = true, @@ -179,6 +180,12 @@ class BuildInfo { /// rerun tasks. final String? performanceMeasurementFile; + /// Path to the Shorebird build-trace output file. When non-null, the + /// Shorebird-specific build-trace plumbing emits a Chrome Trace Event + /// Format JSON file here. Named `shorebird` to keep the fork's + /// surface area visibly distinct from upstream Flutter. + final String? shorebirdTraceFilePath; + /// If provided, an output directory where one or more v8-style heap snapshots /// will be written for code size profiling. final String? codeSizeDirectory; @@ -375,6 +382,7 @@ class BuildInfo { kFileSystemScheme: ?fileSystemScheme, kBuildName: ?buildName, kBuildNumber: ?buildNumber, + kFlavor: ?flavor, if (useLocalCanvasKit) kUseLocalCanvasKitFlag: useLocalCanvasKit.toString(), }; } diff --git a/packages/flutter_tools/lib/src/build_system/build_system.dart b/packages/flutter_tools/lib/src/build_system/build_system.dart index 9abcda6d39c55..d6bc3becca308 100644 --- a/packages/flutter_tools/lib/src/build_system/build_system.dart +++ b/packages/flutter_tools/lib/src/build_system/build_system.dart @@ -882,6 +882,7 @@ class _BuildInstance { Future _invokeInternal(Node node) async { final PoolResource resource = await resourcePool.request(); + final int startTimeMicroseconds = DateTime.now().microsecondsSinceEpoch; final stopwatch = Stopwatch()..start(); var succeeded = true; var skipped = false; @@ -982,6 +983,7 @@ class _BuildInstance { skipped: skipped, succeeded: succeeded, analyticsName: node.target.analyticsName, + startTimeMicroseconds: startTimeMicroseconds, ); } return succeeded; @@ -1011,6 +1013,7 @@ class PerformanceMeasurement { required this.skipped, required this.succeeded, required this.analyticsName, + required this.startTimeMicroseconds, }); final int elapsedMilliseconds; @@ -1018,6 +1021,9 @@ class PerformanceMeasurement { final bool skipped; final bool succeeded; final String analyticsName; + + /// Wall-clock start time in microseconds since epoch. + final int startTimeMicroseconds; } /// Check if there are any dependency cycles in the target. diff --git a/packages/flutter_tools/lib/src/build_system/targets/assets.dart b/packages/flutter_tools/lib/src/build_system/targets/assets.dart index 89b0be24ed3fc..89bd961508363 100644 --- a/packages/flutter_tools/lib/src/build_system/targets/assets.dart +++ b/packages/flutter_tools/lib/src/build_system/targets/assets.dart @@ -13,6 +13,7 @@ import '../../dart/package_map.dart'; import '../../devfs.dart'; import '../../flutter_manifest.dart'; import '../../isolated/native_assets/dart_hook_result.dart'; +import '../../shorebird/shorebird_yaml.dart'; import '../build_system.dart'; import '../depfile.dart'; import '../exceptions.dart'; @@ -186,6 +187,17 @@ Future copyAssets( } if (doCopy) { await (content.file as File).copy(file.path); + if (file.basename == 'shorebird.yaml') { + try { + updateShorebirdYaml( + environment.defines[kFlavor], + file.path, + environment: environment.platform.environment, + ); + } on Exception catch (error) { + throw Exception('Failed to generate shorebird configuration. Error: $error'); + } + } } } else { await file.writeAsBytes(await entry.value.content.contentsAsBytes()); diff --git a/packages/flutter_tools/lib/src/build_system/targets/common.dart b/packages/flutter_tools/lib/src/build_system/targets/common.dart index 6e54a2ddecb7f..8f34ee062cead 100644 --- a/packages/flutter_tools/lib/src/build_system/targets/common.dart +++ b/packages/flutter_tools/lib/src/build_system/targets/common.dart @@ -509,3 +509,48 @@ abstract final class Lipo { } } } + +/// For managing the supplementary linking files for Shorebird. +abstract final class LinkSupplement { + static Future create( + Environment environment, { + required String inputBuildDir, + required String outputBuildDir, + }) async { + // If the shorebird directory exists, delete it first. + final Directory shorebirdDir = environment.fileSystem.directory( + environment.fileSystem.path.join(outputBuildDir, 'shorebird'), + ); + if (shorebirdDir.existsSync()) { + shorebirdDir.deleteSync(recursive: true); + } + + void maybeCopy(String name) { + final path = environment.fileSystem.path.join(inputBuildDir, name); + final File file = environment.fileSystem.file(path); + if (file.existsSync()) { + file.copySync(environment.fileSystem.path.join(shorebirdDir.path, name)); + environment.logger.printTrace('LinkSupplement: copied $name'); + } else { + environment.logger.printTrace('LinkSupplement: missing $name at $path'); + } + } + + // Copy the link information (generated by gen_snapshot) + // into the shorebird directory. + maybeCopy('App.ct.link'); + maybeCopy('App.class_table.json'); + maybeCopy('App.dt.link'); + maybeCopy('App.dispatch_table.json'); + maybeCopy('App.ft.link'); + maybeCopy('App.field_table.json'); + // DD table files for cascade limiter (produced by 2-pass release build + // when SHOREBIRD_DD_MAX_BYTES is set). + maybeCopy('App.dd.link'); + maybeCopy('App.dd_callers.link'); + maybeCopy('App.dd_identity.link'); + maybeCopy('App.dd_slots.link'); + // Per-slot DD resolution outcome diagnostic (TSV). + maybeCopy('App.dd_resolution.tsv'); + } +} diff --git a/packages/flutter_tools/lib/src/build_system/targets/ios.dart b/packages/flutter_tools/lib/src/build_system/targets/ios.dart index f3f9e0dee1de4..9f861d037adc6 100644 --- a/packages/flutter_tools/lib/src/build_system/targets/ios.dart +++ b/packages/flutter_tools/lib/src/build_system/targets/ios.dart @@ -143,6 +143,12 @@ abstract class AotAssemblyBase extends Target { // Don't fail if the dSYM wasn't created (i.e. during a debug build). skipMissingInputs: true, ); + + await LinkSupplement.create( + environment, + inputBuildDir: buildOutputPath, + outputBuildDir: getIosBuildDirectory(), + ); } } diff --git a/packages/flutter_tools/lib/src/build_system/targets/macos.dart b/packages/flutter_tools/lib/src/build_system/targets/macos.dart index f3fb4a878cf5f..52129e44511c5 100644 --- a/packages/flutter_tools/lib/src/build_system/targets/macos.dart +++ b/packages/flutter_tools/lib/src/build_system/targets/macos.dart @@ -353,6 +353,12 @@ class CompileMacOSFramework extends Target { // Don't fail if the dSYM wasn't created (i.e. during a debug build). skipMissingInputs: true, ); + + await LinkSupplement.create( + environment, + inputBuildDir: buildOutputPath, + outputBuildDir: getMacOSBuildDirectory(), + ); } @override diff --git a/packages/flutter_tools/lib/src/cache.dart b/packages/flutter_tools/lib/src/cache.dart index 49e35e71706fb..6b3cdb62a7dfc 100644 --- a/packages/flutter_tools/lib/src/cache.dart +++ b/packages/flutter_tools/lib/src/cache.dart @@ -37,6 +37,7 @@ import 'base/utils.dart' show getElapsedAsSeconds, getSizeAsPlatformMB; import 'convert.dart'; import 'features.dart'; +const kShorebirdStorageUrl = 'https://download.shorebird.dev'; const kFlutterRootEnvironmentVariableName = 'FLUTTER_ROOT'; // should point to //flutter/ (root of flutter/flutter repo) const kFlutterEngineEnvironmentVariableName = @@ -526,7 +527,7 @@ class Cache { /// The base for URLs that store Flutter engine artifacts that are fetched /// during the installation of the Flutter SDK. /// - /// By default the base URL is https://storage.googleapis.com. However, if + /// By default the base URL is https://download.shorebird.dev. However, if /// `FLUTTER_STORAGE_BASE_URL` environment variable ([kFlutterStorageBaseUrl]) /// is provided, the environment variable value is returned instead. /// @@ -538,9 +539,13 @@ class Cache { String? overrideUrl = _platform.environment[kFlutterStorageBaseUrl]; if (overrideUrl == null) { return storageRealm.isEmpty - ? 'https://storage.googleapis.com' + ? 'https://download.shorebird.dev' : 'https://storage.googleapis.com/$storageRealm'; } + // Shorebird's artifact proxy is a trusted source. + if (overrideUrl == kShorebirdStorageUrl) { + return overrideUrl; + } // verify that this is a valid URI. overrideUrl = storageRealm.isEmpty ? overrideUrl : '$overrideUrl/$storageRealm'; try { diff --git a/packages/flutter_tools/lib/src/commands/assemble.dart b/packages/flutter_tools/lib/src/commands/assemble.dart index 27c0c18461ecf..9064e92b929a2 100644 --- a/packages/flutter_tools/lib/src/commands/assemble.dart +++ b/packages/flutter_tools/lib/src/commands/assemble.dart @@ -4,6 +4,7 @@ import 'package:args/args.dart'; import 'package:meta/meta.dart'; +import 'package:shorebird_build_trace/shorebird_build_trace.dart'; import 'package:unified_analytics/unified_analytics.dart'; import '../artifacts.dart'; @@ -116,6 +117,13 @@ class AssembleCommand extends FlutterCommand { 'performance-measurement-file', help: 'Output individual target performance to a JSON file.', ); + argParser.addOption( + 'shorebird-trace-file', + help: + 'Output a Shorebird build trace in Chrome Trace Event Format ' + 'JSON. Shorebird-specific; used by the build-trace plumbing in ' + 'gradle.dart / mac.dart to collect flutter-assemble timings.', + ); argParser.addMultiOption( 'input', abbr: 'i', @@ -398,6 +406,10 @@ class AssembleCommand extends FlutterCommand { final File outFile = globals.fs.file(argumentResults['performance-measurement-file']); writePerformanceData(result.performance.values, outFile); } + if (argumentResults.wasParsed('shorebird-trace-file')) { + final File outFile = globals.fs.file(argumentResults['shorebird-trace-file']); + writeTraceData(result.performance.values, outFile); + } if (argumentResults.wasParsed('depfile')) { final File depfileFile = globals.fs.file(stringArg('depfile')); final depfile = Depfile(result.inputFiles, result.outputFiles); @@ -444,3 +456,35 @@ void writePerformanceData(Iterable measurements, File ou } outFile.writeAsStringSync(json.encode(jsonData)); } + +/// Output build trace data in Chrome Trace Event Format in [outFile]. +/// `flutter assemble` is re-entered as its own process by Gradle / +/// xcode_backend, so its events get their own Perfetto row named +/// `flutter assemble`. Spans for each [PerformanceMeasurement] are +/// emitted using its wall-clock [PerformanceMeasurement.startTimeMicroseconds] +/// (converted to a DateTime; not the stopwatch duration alone) so the +/// timeline aligns with the parent flutter tool's trace when merged. +@visibleForTesting +void writeTraceData(Iterable measurements, File outFile) { + final int pid = currentProcessId(); + final tracer = BuildTracer() + ..addProcessNameMetadata(pid: pid, name: 'flutter assemble') + ..addThreadNameMetadata(pid: pid, tid: 1, name: 'flutter assemble'); + for (final measurement in measurements) { + final start = DateTime.fromMicrosecondsSinceEpoch(measurement.startTimeMicroseconds); + tracer.addCompleteEvent( + name: measurement.analyticsName, + cat: TraceCategory.assemble.wireName, + pid: pid, + tid: 1, + start: start, + end: start.add(Duration(milliseconds: measurement.elapsedMilliseconds)), + args: { + 'target': measurement.target, + 'skipped': measurement.skipped, + 'succeeded': measurement.succeeded, + }, + ); + } + tracer.writeToFile(outFile); +} diff --git a/packages/flutter_tools/lib/src/commands/build_apk.dart b/packages/flutter_tools/lib/src/commands/build_apk.dart index 184a68be58d90..231b6650e18ac 100644 --- a/packages/flutter_tools/lib/src/commands/build_apk.dart +++ b/packages/flutter_tools/lib/src/commands/build_apk.dart @@ -31,6 +31,7 @@ class BuildApkCommand extends BuildSubCommand { usesExtraDartFlagOptions(verboseHelp: verboseHelp); addEnableExperimentation(hide: !verboseHelp); addBuildPerformanceFile(hide: !verboseHelp); + usesShorebirdTraceOption(hide: !verboseHelp); usesAnalyzeSizeFlag(); addAndroidSpecificBuildOptions(hide: !verboseHelp); addIgnoreDeprecationOption(); diff --git a/packages/flutter_tools/lib/src/commands/build_appbundle.dart b/packages/flutter_tools/lib/src/commands/build_appbundle.dart index af58cbe68fa11..32ab81b8bab68 100644 --- a/packages/flutter_tools/lib/src/commands/build_appbundle.dart +++ b/packages/flutter_tools/lib/src/commands/build_appbundle.dart @@ -33,6 +33,7 @@ class BuildAppBundleCommand extends BuildSubCommand { usesDartDefineOption(); usesExtraDartFlagOptions(verboseHelp: verboseHelp); addBuildPerformanceFile(hide: !verboseHelp); + usesShorebirdTraceOption(hide: !verboseHelp); usesTrackWidgetCreation(verboseHelp: verboseHelp); addEnableExperimentation(hide: !verboseHelp); usesAnalyzeSizeFlag(); diff --git a/packages/flutter_tools/lib/src/commands/build_ios.dart b/packages/flutter_tools/lib/src/commands/build_ios.dart index e783f793b9e7a..6cfb5b75dbc8a 100644 --- a/packages/flutter_tools/lib/src/commands/build_ios.dart +++ b/packages/flutter_tools/lib/src/commands/build_ios.dart @@ -907,6 +907,7 @@ abstract class _BuildIOSSubCommand extends BuildSubCommand { usesExtraDartFlagOptions(verboseHelp: verboseHelp); addEnableExperimentation(hide: !verboseHelp); addBuildPerformanceFile(hide: !verboseHelp); + usesShorebirdTraceOption(hide: !verboseHelp); usesAnalyzeSizeFlag(); argParser.addFlag( 'codesign', diff --git a/packages/flutter_tools/lib/src/http_host_validator.dart b/packages/flutter_tools/lib/src/http_host_validator.dart index 9702f83bffd08..8363221166398 100644 --- a/packages/flutter_tools/lib/src/http_host_validator.dart +++ b/packages/flutter_tools/lib/src/http_host_validator.dart @@ -11,7 +11,7 @@ import 'doctor_validator.dart'; import 'features.dart'; /// Common Flutter HTTP hosts. -const kCloudHost = 'https://storage.googleapis.com/'; +const kCloudHost = 'https://download.shorebird.dev/'; const kCocoaPods = 'https://cocoapods.org/'; const kGitHub = 'https://github.com/'; const kMaven = 'https://maven.google.com/'; diff --git a/packages/flutter_tools/lib/src/ios/application_package.dart b/packages/flutter_tools/lib/src/ios/application_package.dart index f1c8f44849548..0bc7bb5a45851 100644 --- a/packages/flutter_tools/lib/src/ios/application_package.dart +++ b/packages/flutter_tools/lib/src/ios/application_package.dart @@ -130,6 +130,17 @@ class BuildableIOSApp extends IOSApp { @override String? get name => _appProductName; + String get shorebirdYamlPath => globals.fs.path.join( + archiveBundleOutputPath, + 'Products', + 'Applications', + _appProductName != null ? '$_appProductName.app' : 'Runner.app', + 'Frameworks', + 'App.framework', + 'flutter_assets', + 'shorebird.yaml', + ); + @override String get simulatorBundlePath => _buildAppPath(XcodeSdk.IPhoneSimulator.platformName); diff --git a/packages/flutter_tools/lib/src/ios/mac.dart b/packages/flutter_tools/lib/src/ios/mac.dart index dce46d7edc605..ed8a1b1743d26 100644 --- a/packages/flutter_tools/lib/src/ios/mac.dart +++ b/packages/flutter_tools/lib/src/ios/mac.dart @@ -37,6 +37,7 @@ import '../migrations/xcode_script_build_phase_migration.dart'; import '../migrations/xcode_thin_binary_build_phase_input_paths_migration.dart'; import '../plugins.dart'; import '../project.dart'; +import '../shorebird/ios_build_trace_session.dart'; import 'application_package.dart'; import 'code_signing.dart'; import 'migrations/host_app_info_plist_migration.dart'; @@ -359,11 +360,24 @@ Future buildXcodeProject({ ); } } + // Created early enough to capture pod install, which can be minutes + // on plugin-heavy apps โ€” otherwise it's an invisible gap between + // "flutter build ios" starting and the xcode archive span. + final IosBuildTraceSession? traceSession = IosBuildTraceSession.maybeStart( + shorebirdTraceFilePath: buildInfo.shorebirdTraceFilePath, + fileSystem: globals.fs, + buildDirectoryPath: globals.fs.path.join(getBuildDirectory(), 'ios'), + ); + + traceSession?.onBeforePodInstall(); await processPodsIfNeeded(project.ios, buildDirectoryPath, buildInfo.mode); + traceSession?.onAfterPodInstall(); if (configOnly) { return XcodeBuildResult(success: true); } + buildCommands.addAll(traceSession?.extraBuildCommands() ?? const []); + if (globals.logger.isVerbose) { // An environment variable to be passed to xcode_backend.sh determining // whether to echo back executed commands. @@ -542,6 +556,8 @@ Future buildXcodeProject({ ]); } + traceSession?.onXcodeAboutToStart(); + final sw = Stopwatch()..start(); initialBuildStatus = globals.logger.startProgress('Running Xcode build...'); @@ -566,6 +582,13 @@ Future buildXcodeProject({ ), ); + await traceSession?.onXcodeFinished( + buildActionName: xcodeBuildActionToString(buildAction), + resultBundleDirectory: resultBundleDirectory, + runXcresultTool: (List args) => globals.processManager.run(args), + ); + traceSession?.finish(printStatus: globals.printStatus); + if (tempDir.existsSync()) { // Display additional warning and error message from xcresult bundle. final Directory resultBundle = tempDir.childDirectory(_kResultBundlePath); diff --git a/packages/flutter_tools/lib/src/macos/cocoapods.dart b/packages/flutter_tools/lib/src/macos/cocoapods.dart index 7245c7636aae0..5ab6f80a81916 100644 --- a/packages/flutter_tools/lib/src/macos/cocoapods.dart +++ b/packages/flutter_tools/lib/src/macos/cocoapods.dart @@ -2,8 +2,11 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. +import 'dart:convert'; + import 'package:file/file.dart'; import 'package:process/process.dart'; +import 'package:shorebird_build_trace/shorebird_build_trace.dart'; import 'package:unified_analytics/unified_analytics.dart'; import '../base/common.dart'; @@ -348,16 +351,28 @@ class CocoaPods { Future _runPodInstall(XcodeBasedProject xcodeProject, BuildMode buildMode) async { final Status status = _logger.startProgress('Running pod install...'); - final ProcessResult result = await _processManager.run( - ['pod', 'install', '--verbose'], - workingDirectory: _fileSystem.path.dirname(xcodeProject.podfile.path), - environment: { - // See https://github.com/flutter/flutter/issues/10873. - // CocoaPods analytics adds a lot of latency. - 'COCOAPODS_DISABLE_STATS': 'true', - 'LANG': 'en_US.UTF-8', - }, - ); + + // Shorebird-specific: when a build trace is active, stream pod install + // output so we can timestamp phase transitions. On plugin-heavy apps + // pod install can be minutes, and a single opaque span hides where the + // time goes (dependency analysis vs downloads vs project generation vs + // integration). + final BuildTracer? tracer = BuildTracer.current; + final ProcessResult result; + if (tracer == null) { + result = await _processManager.run( + ['pod', 'install', '--verbose'], + workingDirectory: _fileSystem.path.dirname(xcodeProject.podfile.path), + environment: { + // See https://github.com/flutter/flutter/issues/10873. + // CocoaPods analytics adds a lot of latency. + 'COCOAPODS_DISABLE_STATS': 'true', + 'LANG': 'en_US.UTF-8', + }, + ); + } else { + result = await _runPodInstallStreamed(tracer, xcodeProject); + } status.stop(); if (_logger.isVerbose || result.exitCode != 0) { final stdout = result.stdout as String; @@ -386,6 +401,60 @@ class CocoaPods { } } + /// Runs `pod install --verbose` with live stdout streaming so phase + /// transitions can be timestamped into [tracer]. Returns a [ProcessResult] + /// shaped the same as [ProcessManager.run] would have returned, so + /// downstream error handling keeps working unchanged. + Future _runPodInstallStreamed( + BuildTracer tracer, + XcodeBasedProject xcodeProject, + ) async { + final Process process = await _processManager.start( + ['pod', 'install', '--verbose'], + workingDirectory: _fileSystem.path.dirname(xcodeProject.podfile.path), + environment: {'COCOAPODS_DISABLE_STATS': 'true', 'LANG': 'en_US.UTF-8'}, + ); + // Real pid of the `pod` process. Phase spans land on this pid + + // tid=1, with a `process_name` metadata event so Perfetto groups + // pod install work into its own row. + final int podPid = process.pid; + tracer + ..addProcessNameMetadata(pid: podPid, name: TraceNames.podInstallSpanName) + ..addThreadNameMetadata(pid: podPid, tid: 1, name: TraceNames.podInstallSpanName); + final phases = PhaseTracker( + tracer: tracer, + pid: podPid, + tid: 1, + namePrefix: TraceNames.podInstallNamePrefix, + ); + + final stdoutBuf = StringBuffer(); + final stderrBuf = StringBuffer(); + + final Future stdoutDone = process.stdout + .transform(utf8.decoder) + .transform(const LineSplitter()) + .listen((String line) { + stdoutBuf + ..write(line) + ..write('\n'); + final PodInstallPhase? phase = _podPhaseForLogLine(line); + if (phase != null) phases.transitionTo(phase.wireName); + }) + .asFuture(); + + final Future stderrDone = process.stderr + .transform(utf8.decoder) + .listen(stderrBuf.write) + .asFuture(); + + final int exitCode = await process.exitCode; + await Future.wait(>[stdoutDone, stderrDone]); + phases.end(); + + return ProcessResult(process.pid, exitCode, stdoutBuf.toString(), stderrBuf.toString()); + } + Future _diagnosePodInstallFailure( ProcessResult result, XcodeBasedProject xcodeProject, @@ -641,3 +710,27 @@ class CocoaPods { } } } + +/// CocoaPods verbose-mode log markers, in the order they print during a +/// normal `pod install`. A match means "from now on, the process is in +/// this phase" โ€” the `PhaseTracker` closes the prior phase and opens a +/// new one. +/// +/// These substrings have been stable across recent CocoaPods releases; +/// if CocoaPods renames one, the worst case is we miss a phase in the +/// trace โ€” pod install still succeeds. +const _podPhaseMarkers = { + 'Analyzing dependencies': PodInstallPhase.analyzing, + 'Downloading dependencies': PodInstallPhase.downloading, + 'Generating Pods project': PodInstallPhase.generating, + 'Integrating client project': PodInstallPhase.integrating, +}; + +/// Returns the phase a `pod install --verbose` log [line] transitions +/// into, or null if the line doesn't contain any of the known markers. +PodInstallPhase? _podPhaseForLogLine(String line) { + for (final MapEntry entry in _podPhaseMarkers.entries) { + if (line.contains(entry.key)) return entry.value; + } + return null; +} diff --git a/packages/flutter_tools/lib/src/runner/flutter_command.dart b/packages/flutter_tools/lib/src/runner/flutter_command.dart index b2a593186aa9d..473967a49656d 100644 --- a/packages/flutter_tools/lib/src/runner/flutter_command.dart +++ b/packages/flutter_tools/lib/src/runner/flutter_command.dart @@ -131,6 +131,10 @@ abstract final class FlutterOptions { static const kDartDefineFromFileOption = 'dart-define-from-file'; static const kWebDefinesOption = 'web-define'; static const kPerformanceMeasurementFile = 'performance-measurement-file'; + // Shorebird-specific build-trace option. Prefixed so the flag name and + // generated env vars / Gradle properties don't squat on identifiers + // upstream Flutter might want later. + static const kShorebirdTrace = 'shorebird-trace'; static const kDeviceUser = 'device-user'; static const kDeviceTimeout = 'device-timeout'; static const kDeviceConnection = 'device-connection'; @@ -1064,6 +1068,19 @@ abstract class FlutterCommand extends Command { ); } + void usesShorebirdTraceOption({bool hide = false}) { + argParser.addOption( + FlutterOptions.kShorebirdTrace, + help: + 'Output a Chrome Trace Event Format JSON file for build ' + 'profiling. The resulting file can be viewed at ' + 'https://ui.perfetto.dev. Shorebird-specific; named to avoid ' + 'colliding with any future upstream trace option.', + hide: hide, + valueHelp: 'path/to/shorebird-trace.json', + ); + } + void addAndroidSpecificBuildOptions({bool hide = false}) { argParser.addFlag( FlutterOptions.kAndroidGradleDaemon, @@ -1466,6 +1483,11 @@ abstract class FlutterCommand extends Command { ? stringArg(FlutterOptions.kPerformanceMeasurementFile) : null; + final String? shorebirdTraceFilePath = + argParser.options.containsKey(FlutterOptions.kShorebirdTrace) + ? stringArg(FlutterOptions.kShorebirdTrace) + : null; + final Map defineConfigJsonMap = extractDartDefineConfigJsonMap(); final List dartDefines = extractDartDefines(defineConfigJsonMap: defineConfigJsonMap); @@ -1517,6 +1539,7 @@ abstract class FlutterCommand extends Command { dartDefines: dartDefines, dartExperiments: experiments, performanceMeasurementFile: performanceMeasurementFile, + shorebirdTraceFilePath: shorebirdTraceFilePath, packageConfigPath: packagesPath ?? packageConfigFile.path, codeSizeDirectory: codeSizeDirectory, androidGradleDaemon: androidGradleDaemon, diff --git a/packages/flutter_tools/lib/src/shorebird/android_build_trace_session.dart b/packages/flutter_tools/lib/src/shorebird/android_build_trace_session.dart new file mode 100644 index 0000000000000..5b01afa30fbc5 --- /dev/null +++ b/packages/flutter_tools/lib/src/shorebird/android_build_trace_session.dart @@ -0,0 +1,210 @@ +// Shorebird-specific. Keeps the build-trace plumbing for +// `flutter build apk` / `flutter build appbundle` out of gradle.dart +// so the Shorebird fork's diff against upstream stays small and the +// build flow reads the same as upstream. + +import 'dart:io' show Process; + +import 'package:shorebird_build_trace/shorebird_build_trace.dart'; + +import '../base/file_system.dart'; +import '../build_info.dart'; +import '../cache.dart'; +import 'network_trace_span.dart'; + +/// Wraps the lifecycle of a Shorebird build trace across one +/// `buildGradleApp` call. Returned by [maybeStart] only when +/// `--shorebird-trace=` was passed; the constructor installs +/// [BuildTracer.current] and [finish] / [abortOnGradleFailure] +/// clear it, so gradle.dart itself never touches the static. +/// +/// Manual start/stop (rather than a body-wrapping closure) because +/// wrapping the ~200-line `buildGradleApp` body would force a reindent +/// that balloons our diff against upstream flutter and makes every +/// subsequent merge of that method harder. +class AndroidBuildTraceSession { + AndroidBuildTraceSession._({ + required BuildTracer tracer, + required FileSystem fileSystem, + required String tracePath, + required Directory buildDirectory, + }) : _tracer = tracer, + _fs = fileSystem, + _tracePath = tracePath, + _buildDirectory = buildDirectory, + _flutterPid = currentProcessId(), + _buildStart = DateTime.now() { + BuildTracer.start(_tracer); + _tracer + ..addProcessNameMetadata(pid: _flutterPid, name: 'flutter_tool') + ..addThreadNameMetadata(pid: _flutterPid, tid: _flutterToolTid, name: 'flutter tool') + ..addThreadNameMetadata(pid: _flutterPid, tid: _gradleWaitTid, name: 'gradle (wait)') + ..addThreadNameMetadata(pid: _flutterPid, tid: networkTid, name: 'network'); + } + + /// Returns a session when a trace path is configured for this build, + /// null otherwise. + static AndroidBuildTraceSession? maybeStart( + AndroidBuildInfo androidBuildInfo, + FileSystem fileSystem, + Directory buildDirectory, + ) { + final String? path = androidBuildInfo.buildInfo.shorebirdTraceFilePath; + if (path == null) { + return null; + } + return AndroidBuildTraceSession._( + tracer: BuildTracer(), + fileSystem: fileSystem, + tracePath: path, + buildDirectory: buildDirectory, + ); + } + + final BuildTracer _tracer; + final FileSystem _fs; + final String _tracePath; + final Directory _buildDirectory; + final int _flutterPid; + final DateTime _buildStart; + + static const int _flutterToolTid = 1; + static const int _gradleWaitTid = 2; + + String? _assembleTraceFilePath; + String? _gradleTaskTraceFilePath; + DateTime? _gradleStart; + DateTime? _gradleEnd; + + /// Returns the `-P`/`-I=` flags to thread trace plumbing into Gradle. + /// Safe to splat into the options list unconditionally. + /// + /// [assembleTask] carries the variant + build type (e.g. + /// `assembleFooRelease` / `bundleRelease`); the intermediate trace + /// paths embed it so a single Gradle invocation running multiple + /// variants in parallel can't have per-variant FlutterTask instances + /// stomp on each other's trace. + List extraGradleOptions(String assembleTask) { + final String assembleTrace = _fs.path.join( + _buildDirectory.path, + 'intermediates', + 'shorebird', + 'shorebird_assemble_trace_$assembleTask.json', + ); + final String gradleTaskTrace = _fs.path.join( + _buildDirectory.path, + 'intermediates', + 'shorebird', + 'shorebird_gradle_task_trace_$assembleTask.json', + ); + final String initScript = _fs.path.join( + Cache.flutterRoot!, + 'packages', + 'flutter_tools', + 'gradle', + 'shorebird_trace_init.gradle', + ); + _assembleTraceFilePath = assembleTrace; + _gradleTaskTraceFilePath = gradleTaskTrace; + return [ + '-Pshorebird-trace-file=$assembleTrace', + '-I=$initScript', + '-Pshorebird.gradle-trace-file=$gradleTaskTrace', + ]; + } + + /// Hook for `_runGradleTask`'s `preRunTask` โ€” records when the Gradle + /// wrapper is about to run so [finish] can compute the outer build + /// span. + void onGradleAboutToStart() { + final preGradleEnd = DateTime.now(); + _tracer.addCompleteEvent( + name: 'pre-gradle setup', + cat: TraceCategory.flutter.wireName, + pid: _flutterPid, + tid: _flutterToolTid, + start: _buildStart, + end: preGradleEnd, + ); + _gradleStart = DateTime.now(); + } + + /// Hook for `_runGradleTask`'s `onStart` โ€” emits a flow-start tied to + /// Gradle's real pid so Perfetto draws an arrow from our + /// `gradle ` span into the first per-task event the init script + /// emits. + void onGradleSpawn(Process process) { + _tracer.addFlowStart( + id: process.pid, + pid: _flutterPid, + tid: _gradleWaitTid, + at: DateTime.now(), + ); + } + + /// Call after `_runGradleTask` returns, before the exit-code check. + /// Records the outer `gradle ` span and merges the intermediate + /// trace files into the main tracer. + void onGradleFinished(String assembleTask) { + final gradleEnd = DateTime.now(); + _gradleEnd = gradleEnd; + _tracer.addCompleteEvent( + name: '${TraceNames.gradleSpanPrefix}$assembleTask', + cat: TraceCategory.gradle.wireName, + pid: _flutterPid, + tid: _gradleWaitTid, + start: _gradleStart ?? _buildStart, + end: gradleEnd, + ); + final String? assembleTraceFilePath = _assembleTraceFilePath; + if (assembleTraceFilePath != null) { + _tracer.mergeEventsFromFile(_fs.file(assembleTraceFilePath)); + } + final String? gradleTaskTraceFilePath = _gradleTaskTraceFilePath; + if (gradleTaskTraceFilePath != null) { + _tracer.mergeEventsFromFile(_fs.file(gradleTaskTraceFilePath)); + } + } + + /// Call right before `throwToolExit` when Gradle returned non-zero โ€” + /// clears [BuildTracer.current] so the caught ToolExit doesn't leak + /// tracer state into any later code that reads [BuildTracer.current]. + void abortOnGradleFailure() { + BuildTracer.stop(); + } + + /// Writes the merged trace to disk and clears [BuildTracer.current]. + /// Records post-gradle + outer "flutter build" spans first. + /// + /// [buildTarget] is the target suffix (e.g. `apk`, `appbundle`); the + /// outer span name is assembled as + /// `TraceNames.flutterBuildSpanPrefix + buildTarget`. + /// [printStatus] is called once with a user-facing "trace written" + /// message so callers don't have to wire the logger through. + void finish({required String buildTarget, required void Function(String) printStatus}) { + final postGradleEnd = DateTime.now(); + _tracer + ..addCompleteEvent( + name: 'post-gradle processing', + cat: TraceCategory.flutter.wireName, + pid: _flutterPid, + tid: _flutterToolTid, + start: _gradleEnd ?? postGradleEnd, + end: postGradleEnd, + ) + ..addCompleteEvent( + name: '${TraceNames.flutterBuildSpanPrefix}$buildTarget', + cat: TraceCategory.flutter.wireName, + pid: _flutterPid, + tid: _flutterToolTid, + start: _buildStart, + end: postGradleEnd, + ) + ..writeToFile(_fs.file(_tracePath)); + printStatus( + 'Shorebird build trace written to $_tracePath. ' + 'View at https://ui.perfetto.dev', + ); + BuildTracer.stop(); + } +} diff --git a/packages/flutter_tools/lib/src/shorebird/ios_build_trace_session.dart b/packages/flutter_tools/lib/src/shorebird/ios_build_trace_session.dart new file mode 100644 index 0000000000000..d8a7ea3da8e55 --- /dev/null +++ b/packages/flutter_tools/lib/src/shorebird/ios_build_trace_session.dart @@ -0,0 +1,281 @@ +// Shorebird-specific. Keeps the build-trace plumbing for +// `flutter build ios` out of mac.dart so the Shorebird fork's diff +// against upstream stays small and the build flow reads the same as +// upstream. + +import 'dart:async'; +import 'dart:convert'; +import 'dart:io' show ProcessResult; + +import 'package:shorebird_build_trace/shorebird_build_trace.dart'; + +import '../base/file_system.dart'; +import 'network_trace_span.dart'; + +/// Wraps the lifecycle of a Shorebird build trace across one iOS +/// build. Returned by [maybeStart] only when `--shorebird-trace=` +/// was passed; the constructor installs [BuildTracer.current] and +/// [finish] / [abortOnFailure] clear it, so mac.dart itself never +/// touches the static. +/// +/// Manual start/stop (rather than a body-wrapping closure) because +/// wrapping the ~300-line `buildXcodeProject` body would force a +/// reindent that balloons our diff against upstream flutter and makes +/// every subsequent merge of that method harder. +class IosBuildTraceSession { + IosBuildTraceSession._({ + required BuildTracer tracer, + required FileSystem fileSystem, + required String tracePath, + required String buildDirectoryPath, + }) : _tracer = tracer, + _fs = fileSystem, + _tracePath = tracePath, + _buildDirectoryPath = buildDirectoryPath, + _flutterPid = currentProcessId(), + _buildStart = DateTime.now() { + BuildTracer.start(_tracer); + _tracer + ..addProcessNameMetadata(pid: _flutterPid, name: 'flutter_tool') + ..addThreadNameMetadata(pid: _flutterPid, tid: _flutterToolTid, name: 'flutter tool') + ..addThreadNameMetadata(pid: _flutterPid, tid: _xcodeWaitTid, name: 'xcode (wait)') + ..addThreadNameMetadata(pid: _flutterPid, tid: networkTid, name: 'network') + ..addProcessNameMetadata(pid: _xcodeSubsectionPid, name: 'xcodebuild') + ..addThreadNameMetadata(pid: _xcodeSubsectionPid, tid: 1, name: 'xcode phases'); + } + + /// Returns a session when a trace path is configured for this build, + /// null otherwise. [buildDirectoryPath] is the iOS build directory + /// (relative to the project root) where the intermediate assemble + /// trace file is written. + static IosBuildTraceSession? maybeStart({ + required String? shorebirdTraceFilePath, + required FileSystem fileSystem, + required String buildDirectoryPath, + }) { + if (shorebirdTraceFilePath == null) { + return null; + } + return IosBuildTraceSession._( + tracer: BuildTracer(), + fileSystem: fileSystem, + tracePath: shorebirdTraceFilePath, + buildDirectoryPath: buildDirectoryPath, + ); + } + + final BuildTracer _tracer; + final FileSystem _fs; + final String _tracePath; + final String _buildDirectoryPath; + final int _flutterPid; + final DateTime _buildStart; + + static const int _flutterToolTid = 1; + static const int _xcodeWaitTid = 2; + + /// Synthetic pid for Xcode subsection events parsed from xcresult. + /// xcresult doesn't surface xcodebuild's pid, so events end up on + /// this fixed id named via a `process_name` metadata event. + static const int _xcodeSubsectionPid = 0x78636f; // ASCII 'xco' + + DateTime? _podInstallStart; + DateTime? _xcodeStart; + DateTime? _xcodeEnd; + String? _assembleTraceFilePath; + + /// Call immediately before `processPodsIfNeeded` so the resulting + /// span covers its full wall-clock. + void onBeforePodInstall() { + _podInstallStart = DateTime.now(); + } + + /// Call immediately after `processPodsIfNeeded` returns. Records the + /// `pod install` complete event using the start timestamp captured + /// by [onBeforePodInstall]. + void onAfterPodInstall() { + final DateTime? start = _podInstallStart; + if (start == null) { + return; + } + _tracer.addCompleteEvent( + name: TraceNames.podInstallSpanName, + cat: TraceCategory.subprocess.wireName, + pid: _flutterPid, + tid: _flutterToolTid, + start: start, + end: DateTime.now(), + ); + } + + /// Returns the extra xcodebuild arguments that thread trace plumbing + /// into the build phase script. Safe to splat into the command list + /// unconditionally. + List extraBuildCommands() { + final String assembleTrace = _fs.path.join( + _fs.currentDirectory.path, + _buildDirectoryPath, + 'shorebird_assemble_trace.json', + ); + _assembleTraceFilePath = assembleTrace; + // Naming the env var SHOREBIRD_TRACE_FILE avoids squatting on an + // Xcode-reserved prefix. Read by xcode_backend.dart. + return ['SHOREBIRD_TRACE_FILE=$assembleTrace']; + } + + /// Call right before `_runBuildWithRetries` so the outer build span + /// can compute the pre-xcode setup interval. + void onXcodeAboutToStart() { + final xcodeStart = DateTime.now(); + _xcodeStart = xcodeStart; + _tracer.addCompleteEvent( + name: 'pre-xcode setup', + cat: TraceCategory.flutter.wireName, + pid: _flutterPid, + tid: _flutterToolTid, + start: _buildStart, + end: xcodeStart, + ); + } + + /// Call after `_runBuildWithRetries` returns. Records the outer + /// `xcode ` span, then emits per-subsection events pulled + /// from xcresulttool, then merges the intermediate assemble trace. + /// + /// [runXcresultTool] is injected so callers can route through their + /// own `ProcessManager`; the session avoids a direct `globals.*` + /// dependency. + Future onXcodeFinished({ + required String buildActionName, + required Directory resultBundleDirectory, + required Future Function(List) runXcresultTool, + }) async { + final xcodeEnd = DateTime.now(); + _xcodeEnd = xcodeEnd; + _tracer.addCompleteEvent( + name: '${TraceNames.xcodeSpanPrefix}$buildActionName', + cat: TraceCategory.xcode.wireName, + pid: _flutterPid, + tid: _xcodeWaitTid, + start: _xcodeStart ?? _buildStart, + end: xcodeEnd, + ); + await _emitXcodeSubsectionEvents( + resultBundleDirectory: resultBundleDirectory, + runXcresultTool: runXcresultTool, + ); + final String? assembleTraceFilePath = _assembleTraceFilePath; + if (assembleTraceFilePath != null) { + _tracer.mergeEventsFromFile(_fs.file(assembleTraceFilePath)); + } + } + + /// Clears [BuildTracer.current] without writing the trace. Use on + /// error paths where [finish] won't run. + void abortOnFailure() { + BuildTracer.stop(); + } + + /// Writes the merged trace to disk and clears [BuildTracer.current]. + /// Records the post-xcode + outer `flutter build ios` spans first. + void finish({required void Function(String) printStatus}) { + final buildEnd = DateTime.now(); + _tracer + ..addCompleteEvent( + name: 'post-xcode processing', + cat: TraceCategory.flutter.wireName, + pid: _flutterPid, + tid: _flutterToolTid, + start: _xcodeEnd ?? buildEnd, + end: buildEnd, + ) + ..addCompleteEvent( + name: '${TraceNames.flutterBuildSpanPrefix}ios', + cat: TraceCategory.flutter.wireName, + pid: _flutterPid, + tid: _flutterToolTid, + start: _buildStart, + end: buildEnd, + ) + ..writeToFile(_fs.file(_tracePath)); + printStatus( + 'Shorebird build trace written to $_tracePath. ' + 'View at https://ui.perfetto.dev', + ); + BuildTracer.stop(); + } + + /// Ask xcresulttool for the structured build log of the archive + /// action and emit one Chrome Trace Event per top-level subsection + /// (each Xcode target build). Best-effort: silently returns if the + /// bundle can't be parsed (xcresulttool output format drifts across + /// Xcode versions). + Future _emitXcodeSubsectionEvents({ + required Directory resultBundleDirectory, + required Future Function(List) runXcresultTool, + }) async { + if (!resultBundleDirectory.existsSync()) { + return; + } + final ProcessResult result; + try { + result = await runXcresultTool([ + 'xcrun', + 'xcresulttool', + 'get', + 'log', + '--type', + 'build', + '--path', + resultBundleDirectory.path, + ]); + } on Exception { + return; + } + if (result.exitCode != 0) { + return; + } + + final Object? decoded; + try { + decoded = json.decode(result.stdout as String); + } on FormatException { + return; + } + if (decoded is! Map) { + return; + } + + final Object? subsections = decoded['subsections']; + if (subsections is! List) { + return; + } + for (final Object? sub in subsections) { + if (sub is! Map) { + continue; + } + final String title = (sub['title'] as String?) ?? ''; + final double? startTime = (sub['startTime'] as num?)?.toDouble(); + final double? duration = (sub['duration'] as num?)?.toDouble(); + if (startTime == null || duration == null || duration <= 0) { + continue; + } + final start = DateTime.fromMicrosecondsSinceEpoch((startTime * 1000000).round()); + final end = DateTime.fromMicrosecondsSinceEpoch(((startTime + duration) * 1000000).round()); + _tracer.addCompleteEvent( + name: title, + cat: TraceCategory.xcodeSubsection.wireName, + // Synthetic pid so subsections display on their own row in + // Perfetto, labelled 'xcodebuild' via the process_name + // metadata emitted at tracer setup. xcresult doesn't surface + // xcodebuild's real pid, and compile sub-subsections are done + // by clang/swiftc/ld children we never saw โ€” synthetic is + // honest. + pid: _xcodeSubsectionPid, + tid: 1, + start: start, + end: end, + ); + } + } +} diff --git a/packages/flutter_tools/lib/src/shorebird/network_trace_span.dart b/packages/flutter_tools/lib/src/shorebird/network_trace_span.dart new file mode 100644 index 0000000000000..2f08277ae7f81 --- /dev/null +++ b/packages/flutter_tools/lib/src/shorebird/network_trace_span.dart @@ -0,0 +1,54 @@ +// Shorebird-specific. Keeps the build-trace plumbing for HTTP fetches +// out of net.dart so the Shorebird fork's diff against upstream stays +// small. + +import 'package:shorebird_build_trace/shorebird_build_trace.dart'; + +/// Perfetto row id for Flutter-tool HTTP artifact fetches. Local to +/// the flutter tool's pid; picked to sit alongside the flutter-tool +/// and assemble rows without overlapping either. +const int networkTid = 5; + +/// One HTTP request's contribution to the Shorebird build trace. +/// +/// Construct immediately before issuing the request and call [record] +/// once, after the request has completed (success, error, or body +/// fully drained). Records a Chrome Trace Event Format `X` event on +/// the network row only when a [BuildTracer] is installed; a no-op +/// otherwise, so net.dart can drop one of these in every return path +/// without branching on tracing state. +class NetworkTraceSpan { + NetworkTraceSpan.start({required Uri url, required bool onlyHeaders}) + : _url = url, + _onlyHeaders = onlyHeaders, + _start = DateTime.now(); + + final Uri _url; + final bool _onlyHeaders; + final DateTime _start; + + /// Emits the span. Safe to call whether or not a tracer is active. + /// [statusCode] is recorded on normal completion; [errorKind] on + /// IO/SSL/argument errors. Both optional so callers can record a + /// span before either is known (e.g. connection-level failures). + void record({int? statusCode, String? errorKind}) { + final BuildTracer? tracer = BuildTracer.current; + if (tracer == null) { + return; + } + tracer.addCompleteEvent( + name: '${_onlyHeaders ? 'HEAD' : 'GET'} ${_url.host}', + cat: TraceCategory.network.wireName, + pid: currentProcessId(), + tid: networkTid, + start: _start, + end: DateTime.now(), + args: { + 'method': _onlyHeaders ? 'HEAD' : 'GET', + 'host': _url.host, + if (statusCode != null) 'status': statusCode, + if (errorKind != null) 'error': errorKind, + }, + ); + } +} diff --git a/packages/flutter_tools/lib/src/shorebird/shorebird_yaml.dart b/packages/flutter_tools/lib/src/shorebird/shorebird_yaml.dart new file mode 100644 index 0000000000000..24d76f84ed257 --- /dev/null +++ b/packages/flutter_tools/lib/src/shorebird/shorebird_yaml.dart @@ -0,0 +1,76 @@ +// Copyright 2024 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'package:yaml/yaml.dart'; +import 'package:yaml_edit/yaml_edit.dart'; + +import '../base/file_system.dart'; +import '../globals.dart' as globals; + +void updateShorebirdYaml( + String? flavor, + String shorebirdYamlPath, { + required Map environment, +}) { + final File shorebirdYaml = globals.fs.file(shorebirdYamlPath); + if (!shorebirdYaml.existsSync()) { + throw Exception('shorebird.yaml not found at $shorebirdYamlPath'); + } + final YamlDocument input = loadYamlDocument(shorebirdYaml.readAsStringSync()); + final YamlMap yamlMap = input.contents as YamlMap; + final Map compiled = compileShorebirdYaml( + yamlMap, + flavor: flavor, + environment: environment, + ); + // Currently we write out over the same yaml file, we should fix this to + // write to a new .json file instead and avoid naming confusion between the + // input and compiled files. + final YamlEditor yamlEditor = YamlEditor(''); + yamlEditor.update([], compiled); + shorebirdYaml.writeAsStringSync(yamlEditor.toString(), flush: true); +} + +String appIdForFlavor(YamlMap yamlMap, {required String? flavor}) { + if (flavor == null || flavor.isEmpty) { + final String? defaultAppId = yamlMap['app_id'] as String?; + if (defaultAppId == null || defaultAppId.isEmpty) { + throw Exception('Cannot find "app_id" in shorebird.yaml'); + } + return defaultAppId; + } + + final YamlMap? yamlFlavors = yamlMap['flavors'] as YamlMap?; + if (yamlFlavors == null) { + throw Exception('Cannot find "flavors" in shorebird.yaml.'); + } + final String? flavorAppId = yamlFlavors[flavor] as String?; + if (flavorAppId == null || flavorAppId.isEmpty) { + throw Exception('Cannot find "app_id" for $flavor in shorebird.yaml'); + } + return flavorAppId; +} + +Map compileShorebirdYaml( + YamlMap yamlMap, { + required String? flavor, + required Map environment, +}) { + final String appId = appIdForFlavor(yamlMap, flavor: flavor); + final Map compiled = {'app_id': appId}; + void copyIfSet(String key) { + if (yamlMap[key] != null) { + compiled[key] = yamlMap[key]; + } + } + + copyIfSet('base_url'); + copyIfSet('auto_update'); + copyIfSet('patch_verification'); + final String? shorebirdPublicKeyEnvVar = environment['SHOREBIRD_PUBLIC_KEY']; + if (shorebirdPublicKeyEnvVar != null) { + compiled['patch_public_key'] = shorebirdPublicKeyEnvVar; + } + return compiled; +} diff --git a/packages/flutter_tools/lib/src/version.dart b/packages/flutter_tools/lib/src/version.dart index 50d1508fba0d7..f1c9b087af997 100644 --- a/packages/flutter_tools/lib/src/version.dart +++ b/packages/flutter_tools/lib/src/version.dart @@ -1059,6 +1059,28 @@ class GitTagVersion { } } + // Check if running on a Shorebird release branch. + final String shorebirdFlutterReleases = git + .runSync([ + 'for-each-ref', + '--contains', + gitRef, + '--format', + '%(refname:short)', + 'refs/remotes/origin/flutter_release/*', + ], workingDirectory: workingDirectory) + .stdout + .trim(); + final stableVersionPattern = RegExp(r'^\d+\.\d+\.\d+$'); + final String? shorebirdFlutterVersion = LineSplitter.split(shorebirdFlutterReleases) + .map((e) => e.replaceFirst('origin/flutter_release/', '')) + .where((e) => stableVersionPattern.hasMatch(e)) + .toList() + .firstOrNull; + if (shorebirdFlutterVersion != null) { + return parse(shorebirdFlutterVersion); + } + // If we don't exist in a tag, use git to find the latest tag. return _useNewestTagAndCommitsPastFallback( git: git, diff --git a/packages/flutter_tools/pubspec.yaml b/packages/flutter_tools/pubspec.yaml index 182009dc14a59..81abc7d7284c6 100644 --- a/packages/flutter_tools/pubspec.yaml +++ b/packages/flutter_tools/pubspec.yaml @@ -58,6 +58,16 @@ dependencies: pubspec_parse: 1.5.0 graphs: 2.3.2 + + # Shorebird-specific: shared build-trace library. Pinned to a specific + # commit of the shorebird monorepo. Roll this ref together with any + # change to the shared library. Subdir path points at the package + # within the monorepo. + shorebird_build_trace: + git: + url: https://github.com/shorebirdtech/shorebird.git + ref: cff436f66534e14618cddcaf93f0d0e9862c937a + path: packages/shorebird_build_trace hooks_runner: 1.1.1 hooks: 1.0.2 code_assets: 1.0.0 diff --git a/packages/flutter_tools/test/commands.shard/hermetic/assemble_test.dart b/packages/flutter_tools/test/commands.shard/hermetic/assemble_test.dart index f3d7dbe5038b7..43e58726c551f 100644 --- a/packages/flutter_tools/test/commands.shard/hermetic/assemble_test.dart +++ b/packages/flutter_tools/test/commands.shard/hermetic/assemble_test.dart @@ -28,13 +28,9 @@ void main() { final StackTrace stackTrace = StackTrace.current; late FakeAnalytics fakeAnalytics; - late MemoryFileSystem fileSystem; - setUp(() { - fileSystem = MemoryFileSystem.test(); - fileSystem.file('pubspec.yaml').createSync(); fakeAnalytics = getInitializedFakeAnalyticsInstance( - fs: fileSystem, + fs: MemoryFileSystem.test(), fakeFlutterVersion: FakeFlutterVersion(), ); }); @@ -55,7 +51,7 @@ void main() { }, overrides: { Cache: () => Cache.test(processManager: FakeProcessManager.any()), - FileSystem: () => fileSystem, + FileSystem: () => MemoryFileSystem.test(), ProcessManager: () => FakeProcessManager.any(), }, ); @@ -84,7 +80,7 @@ void main() { }, overrides: { Cache: () => Cache.test(processManager: FakeProcessManager.any()), - FileSystem: () => fileSystem, + FileSystem: () => MemoryFileSystem.test(), ProcessManager: () => FakeProcessManager.any(), }, ); @@ -113,7 +109,7 @@ void main() { }, overrides: { Cache: () => Cache.test(processManager: FakeProcessManager.any()), - FileSystem: () => fileSystem, + FileSystem: () => MemoryFileSystem.test(), ProcessManager: () => FakeProcessManager.any(), }, ); @@ -142,7 +138,7 @@ void main() { }, overrides: { Cache: () => Cache.test(processManager: FakeProcessManager.any()), - FileSystem: () => fileSystem, + FileSystem: () => MemoryFileSystem.test(), ProcessManager: () => FakeProcessManager.any(), }, ); @@ -164,7 +160,7 @@ void main() { }, overrides: { Cache: () => Cache.test(processManager: FakeProcessManager.any()), - FileSystem: () => fileSystem, + FileSystem: () => MemoryFileSystem.test(), ProcessManager: () => FakeProcessManager.any(), FeatureFlags: () => TestFeatureFlags(isMacOSEnabled: true), }, @@ -196,7 +192,7 @@ void main() { overrides: { Analytics: () => fakeAnalytics, Cache: () => Cache.test(processManager: FakeProcessManager.any()), - FileSystem: () => fileSystem, + FileSystem: () => MemoryFileSystem.test(), ProcessManager: () => FakeProcessManager.any(), }, ); @@ -228,7 +224,7 @@ void main() { }, overrides: { Cache: () => Cache.test(processManager: FakeProcessManager.any()), - FileSystem: () => fileSystem, + FileSystem: () => MemoryFileSystem.test(), ProcessManager: () => FakeProcessManager.any(), FeatureFlags: () => TestFeatureFlags(isMacOSEnabled: true), Analytics: () => fakeAnalytics, @@ -249,7 +245,7 @@ void main() { }, overrides: { Cache: () => Cache.test(processManager: FakeProcessManager.any()), - FileSystem: () => fileSystem, + FileSystem: () => MemoryFileSystem.test(), ProcessManager: () => FakeProcessManager.any(), }, ); @@ -273,7 +269,7 @@ void main() { }, overrides: { Cache: () => Cache.test(processManager: FakeProcessManager.any()), - FileSystem: () => fileSystem, + FileSystem: () => MemoryFileSystem.test(), ProcessManager: () => FakeProcessManager.any(), }, ); @@ -285,56 +281,24 @@ void main() { AssembleCommand(buildSystem: TestBuildSystem.all(BuildResult(success: true))), ); - const invalidDartDefines = [ - 'flutter.inspector.structuredErrors%3Dtrue', - '///', - '@@@@', - "'", - '"', - '`', - r'\', - r'$', - ';', - '/*', - '*/', - '//', - '\n', - '\r', - '<', - '>', - '{', - '}', - '[', - ']', - '(', - ')', - '%', - '=', - '&', - '?', - '#', + final command = [ + 'assemble', + '--output', + 'Output', + '-DartDefines=flutter.inspector.structuredErrors%3Dtrue', + 'debug_macos_bundle_flutter_assets', ]; - for (final invalidDartDefine in invalidDartDefines) { - final command = [ - 'assemble', - '--output', - 'Output', - '-DartDefines=$invalidDartDefine', - 'debug_macos_bundle_flutter_assets', - ]; - expect( - commandRunner.run(command), - throwsToolExit( - message: - 'Error parsing assemble command: The -Pdart-defines argument contains non-base64 encoded data. ' - 'Check your build command and try again.', - ), - ); - } + expect( + commandRunner.run(command), + throwsToolExit( + message: + 'Error parsing assemble command: your generated configuration may be out of date', + ), + ); }, overrides: { Cache: () => Cache.test(processManager: FakeProcessManager.any()), - FileSystem: () => fileSystem, + FileSystem: () => MemoryFileSystem.test(), ProcessManager: () => FakeProcessManager.any(), }, ); @@ -350,7 +314,7 @@ void main() { }, overrides: { Cache: () => Cache.test(processManager: FakeProcessManager.any()), - FileSystem: () => fileSystem, + FileSystem: () => MemoryFileSystem.test(), ProcessManager: () => FakeProcessManager.any(), }, ); @@ -380,7 +344,7 @@ void main() { }, overrides: { Cache: () => Cache.test(processManager: FakeProcessManager.any()), - FileSystem: () => fileSystem, + FileSystem: () => MemoryFileSystem.test(), ProcessManager: () => FakeProcessManager.any(), }, ); @@ -400,6 +364,7 @@ void main() { elapsedMilliseconds: 123, skipped: false, succeeded: true, + startTimeMicroseconds: 0, ), }, ), @@ -422,7 +387,7 @@ void main() { }, overrides: { Cache: () => Cache.test(processManager: FakeProcessManager.any()), - FileSystem: () => fileSystem, + FileSystem: () => MemoryFileSystem.test(), ProcessManager: () => FakeProcessManager.any(), }, ); @@ -452,7 +417,7 @@ void main() { localEngineHost: 'out/host_release', ), Cache: () => Cache.test(processManager: FakeProcessManager.any()), - FileSystem: () => fileSystem, + FileSystem: () => MemoryFileSystem.test(), ProcessManager: () => FakeProcessManager.any(), }, ); @@ -525,7 +490,7 @@ void main() { }, overrides: { Cache: () => Cache.test(processManager: FakeProcessManager.any()), - FileSystem: () => fileSystem, + FileSystem: () => MemoryFileSystem.test(), ProcessManager: () => FakeProcessManager.any(), }, ); @@ -538,6 +503,7 @@ void main() { skipped: false, succeeded: true, elapsedMilliseconds: 123, + startTimeMicroseconds: 0, ), ]; final FileSystem fileSystem = MemoryFileSystem.test(); @@ -582,23 +548,56 @@ void main() { expect(testLogger.statusText, contains('assemble')); }); - testUsingContext( - 'flutter assemble fails if pubspec.yaml is missing', - () async { - final CommandRunner commandRunner = createTestCommandRunner( - AssembleCommand(buildSystem: TestBuildSystem.error(null)), - ); + testWithoutContext('writeTraceData outputs Chrome Trace Event Format', () { + final measurements = [ + PerformanceMeasurement( + analyticsName: 'KernelSnapshot', + target: 'kernel_snapshot', + skipped: false, + succeeded: true, + elapsedMilliseconds: 500, + startTimeMicroseconds: 1000000, + ), + ]; + final FileSystem fileSystem = MemoryFileSystem.test(); + final outFile = fileSystem.currentDirectory.childDirectory('foo').childFile('trace.json'); - await expectLater( - commandRunner.run(['assemble', '-o Output', 'debug_macos_bundle_flutter_assets']), - throwsToolExit(message: 'No pubspec.yaml file found'), - ); - }, - overrides: { - FileSystem: () => MemoryFileSystem.test(), - ProcessManager: () => FakeProcessManager.any(), - }, - ); + writeTraceData(measurements, outFile); + + expect(outFile, exists); + final events = json.decode(outFile.readAsStringSync()) as List; + // 2 metadata events (process_name, thread_name) + 1 complete span. + expect(events, hasLength(3)); + + // Metadata events name the process + thread so Perfetto labels the + // row rather than showing a bare pid. + final processMeta = events[0]! as Map; + expect(processMeta['ph'], 'M'); + expect(processMeta['name'], 'process_name'); + expect((processMeta['args']! as Map)['name'], 'flutter assemble'); + + final threadMeta = events[1]! as Map; + expect(threadMeta['ph'], 'M'); + expect(threadMeta['name'], 'thread_name'); + expect((threadMeta['args']! as Map)['name'], 'flutter assemble'); + + final event = events[2]! as Map; + expect(event['ph'], 'X'); + expect(event['name'], 'KernelSnapshot'); + expect(event['cat'], 'assemble'); + expect(event['ts'], 1000000); + expect(event['dur'], 500000); + // pid is the real pid of the test process; just assert it's present + // and an int, and matches between metadata + span. + expect(event['pid'], isA()); + expect(event['pid'], processMeta['pid']); + expect(event['tid'], 1); + expect(event['args'], { + 'target': 'kernel_snapshot', + 'skipped': false, + 'succeeded': true, + }); + }); } final class _StubCommand extends FlutterCommand { diff --git a/packages/flutter_tools/test/commands.shard/hermetic/build_macos_test.dart b/packages/flutter_tools/test/commands.shard/hermetic/build_macos_test.dart index c38ae0cf970eb..1004ab255551b 100644 --- a/packages/flutter_tools/test/commands.shard/hermetic/build_macos_test.dart +++ b/packages/flutter_tools/test/commands.shard/hermetic/build_macos_test.dart @@ -81,6 +81,14 @@ final macosPlatformCustomEnv = FakePlatform( environment: {'FLUTTER_ROOT': '/', 'HOME': '/'}, ); +final Platform macosPlatformWithShorebirdPublicKey = FakePlatform( + operatingSystem: 'macos', + environment: { + 'FLUTTER_ROOT': '/', + 'HOME': '/', + 'SHOREBIRD_PUBLIC_KEY': 'my_public_key', + }, +); final Platform notMacosPlatform = FakePlatform(environment: {'FLUTTER_ROOT': '/'}); void main() { @@ -1407,4 +1415,43 @@ STDERR STUFF OperatingSystemUtils: () => FakeOperatingSystemUtils(hostPlatform: HostPlatform.darwin_x64), }, ); + + testUsingContext( + 'macOS build outputs path and size when successful', + () async { + final BuildCommand command = BuildCommand( + androidSdk: FakeAndroidSdk(), + buildSystem: TestBuildSystem.all(BuildResult(success: true)), + fileSystem: MemoryFileSystem.test(), + logger: BufferLogger.test(), + osUtils: FakeOperatingSystemUtils(), + ); + createMinimalMockProjectFiles(); + final File shorebirdYamlFile = + fileSystem.file( + 'build/macos/Build/Products/Release/example.app/Contents/Frameworks/App.framework/Resources/flutter_assets/shorebird.yaml', + ) + ..createSync(recursive: true) + ..writeAsStringSync('app_id: my-app-id'); + + await createTestCommandRunner(command).run(const ['build', 'macos', '--no-pub']); + + final String updatedYaml = shorebirdYamlFile.readAsStringSync(); + expect(updatedYaml, contains('app_id: my-app-id')); + expect(updatedYaml, contains('patch_public_key: my_public_key')); + }, + overrides: { + FileSystem: () => fileSystem, + ProcessManager: () => + FakeProcessManager.list([setUpFakeXcodeBuildHandler('Release')]), + Platform: () => macosPlatformWithShorebirdPublicKey, + FeatureFlags: () => TestFeatureFlags(isMacOSEnabled: true), + OperatingSystemUtils: () => FakeOperatingSystemUtils(hostPlatform: HostPlatform.darwin_x64), + }, + // SHOREBIRD_PUBLIC_KEY โ†’ shorebird.yaml injection happens inside the + // assets build target, which TestBuildSystem.all stubs out entirely. + // Exercising it needs a real build system or a direct updateShorebirdYaml + // call; skip until that rework lands. + skip: true, + ); } diff --git a/packages/flutter_tools/test/commands.shard/hermetic/build_windows_test.dart b/packages/flutter_tools/test/commands.shard/hermetic/build_windows_test.dart index 11c574670713a..b037587b9aa4e 100644 --- a/packages/flutter_tools/test/commands.shard/hermetic/build_windows_test.dart +++ b/packages/flutter_tools/test/commands.shard/hermetic/build_windows_test.dart @@ -38,6 +38,15 @@ final Platform windowsPlatform = FakePlatform( 'USERPROFILE': '/', }, ); +final Platform windowsPlatformWithPublicKey = FakePlatform( + operatingSystem: 'windows', + environment: { + 'PROGRAMFILES(X86)': r'C:\Program Files (x86)\', + 'FLUTTER_ROOT': flutterRoot, + 'USERPROFILE': '/', + 'SHOREBIRD_PUBLIC_KEY': 'my_public_key', + }, +); final Platform notWindowsPlatform = FakePlatform( environment: {'FLUTTER_ROOT': flutterRoot}, ); @@ -1213,6 +1222,46 @@ No file or variants found for asset: images/a_dot_burr.jpeg. FeatureFlags: () => TestFeatureFlags(isWindowsEnabled: true), }, ); + + testUsingContext( + 'shorebird.yaml is updated when SHOREBIRD_PUBLIC_KEY env var is set', + () async { + final FakeVisualStudio fakeVisualStudio = FakeVisualStudio(); + final BuildWindowsCommand command = BuildWindowsCommand( + logger: BufferLogger.test(), + operatingSystemUtils: FakeOperatingSystemUtils(), + )..visualStudioOverride = fakeVisualStudio; + setUpMockProjectFilesForBuild(); + final File shorebirdYamlFile = + fileSystem.file(r'build\windows\x64\runner\Release\data\flutter_assets\shorebird.yaml') + ..createSync(recursive: true) + ..writeAsStringSync('app_id: my-app-id'); + + processManager = FakeProcessManager.list([ + cmakeGenerationCommand(), + buildCommand('Release'), + ]); + + await createTestCommandRunner( + command, + ).run(const ['windows', '--release', '--no-pub']); + + final String updatedYaml = shorebirdYamlFile.readAsStringSync(); + expect(updatedYaml, contains('app_id: my-app-id')); + expect(updatedYaml, contains('patch_public_key: my_public_key')); + }, + overrides: { + FileSystem: () => fileSystem, + ProcessManager: () => processManager, + Platform: () => windowsPlatformWithPublicKey, + FeatureFlags: () => TestFeatureFlags(isWindowsEnabled: true), + }, + // SHOREBIRD_PUBLIC_KEY โ†’ shorebird.yaml injection happens inside the + // assets build target, which the mocked FakeProcessManager build + // commands don't drive. Same situation as the macOS analogue; skip + // until we test updateShorebirdYaml directly. + skip: true, + ); } class FakeVisualStudio extends Fake implements VisualStudio { diff --git a/packages/flutter_tools/test/general.shard/base/build_test.dart b/packages/flutter_tools/test/general.shard/base/build_test.dart index 577a1fea13f18..0f1bbd14e3bad 100644 --- a/packages/flutter_tools/test/general.shard/base/build_test.dart +++ b/packages/flutter_tools/test/general.shard/base/build_test.dart @@ -16,6 +16,38 @@ const kWhichSysctlCommand = FakeCommand(command: ['which', 'sysctl']); const kARMCheckCommand = FakeCommand(command: ['sysctl', 'hw.optional.arm64'], exitCode: 1); +const kDefaultClang = [ + '-miphoneos-version-min=13.0', + '-isysroot', + 'path/to/sdk', + '-dynamiclib', + '-Xlinker', + '-rpath', + '-Xlinker', + '@executable_path/Frameworks', + '-Xlinker', + '-rpath', + '-Xlinker', + '@loader_path/Frameworks', + '-fapplication-extension', + '-install_name', + '@rpath/App.framework/App', + '-o', + 'build/foo/App.framework/App', + 'build/foo/snapshot_assembly.o', +]; + +// Shorebird link info arguments added for iOS/macOS builds. +// These correspond to the dumpLinkInfoArgs in AOTSnapshotter.build(). +const kLinkInfoArgs = [ + '--print_class_table_link_debug_info_to=build/App.class_table.json', + '--print_class_table_link_info_to=build/App.ct.link', + '--print_field_table_link_debug_info_to=build/App.field_table.json', + '--print_field_table_link_info_to=build/App.ft.link', + '--print_dispatch_table_link_debug_info_to=build/App.dispatch_table.json', + '--print_dispatch_table_link_info_to=build/App.dt.link', +]; + void main() { group('GenSnapshot', () { late GenSnapshot genSnapshot; @@ -31,6 +63,7 @@ void main() { artifacts: artifacts, logger: logger, processManager: processManager, + fileSystem: MemoryFileSystem.test(), ); }); @@ -170,6 +203,7 @@ void main() { testWithoutContext('builds iOS snapshot with dwarfStackTraces', () async { final String outputPath = fileSystem.path.join('build', 'foo'); + final String assembly = fileSystem.path.join(outputPath, 'snapshot_assembly.S'); final String debugPath = fileSystem.path.join('foo', 'app.ios-arm64.symbols'); final String genSnapshotPath = artifacts.getArtifactPath( Artifact.genSnapshotArm64, @@ -181,12 +215,9 @@ void main() { command: [ genSnapshotPath, '--deterministic', - '--snapshot_kind=app-aot-macho-dylib', - '--macho=$outputPath/App.framework/App', - '--macho-object=$outputPath/app.o', - '--macho-min-os-version=13.0', - '--macho-rpath=@executable_path/Frameworks,@loader_path/Frameworks', - '--macho-install-name=@rpath/App.framework/App', + ...kLinkInfoArgs, + '--snapshot_kind=app-aot-assembly', + '--assembly=$assembly', '--dwarf-stack-traces', '--resolve-dwarf-paths', '--save-debugging-info=$debugPath', @@ -195,23 +226,39 @@ void main() { ), kWhichSysctlCommand, kARMCheckCommand, - FakeCommand( + const FakeCommand( + command: [ + 'xcrun', + 'cc', + '-arch', + 'arm64', + '-miphoneos-version-min=13.0', + '-isysroot', + 'path/to/sdk', + '-c', + 'build/foo/snapshot_assembly.S', + '-o', + 'build/foo/snapshot_assembly.o', + ], + ), + const FakeCommand(command: ['xcrun', 'clang', '-arch', 'arm64', ...kDefaultClang]), + const FakeCommand( command: [ 'xcrun', 'dsymutil', '-o', - '$outputPath/App.framework.dSYM', - '$outputPath/App.framework/App', + 'build/foo/App.framework.dSYM', + 'build/foo/App.framework/App', ], ), - FakeCommand( + const FakeCommand( command: [ 'xcrun', 'strip', '-x', - '$outputPath/App.framework/App', + 'build/foo/App.framework/App', '-o', - '$outputPath/App.framework/App', + 'build/foo/App.framework/App', ], ), ]); @@ -233,6 +280,7 @@ void main() { testWithoutContext('builds iOS snapshot with obfuscate', () async { final String outputPath = fileSystem.path.join('build', 'foo'); + final String assembly = fileSystem.path.join(outputPath, 'snapshot_assembly.S'); final String genSnapshotPath = artifacts.getArtifactPath( Artifact.genSnapshotArm64, platform: TargetPlatform.ios, @@ -243,35 +291,48 @@ void main() { command: [ genSnapshotPath, '--deterministic', - '--snapshot_kind=app-aot-macho-dylib', - '--macho=$outputPath/App.framework/App', - '--macho-object=$outputPath/app.o', - '--macho-min-os-version=13.0', - '--macho-rpath=@executable_path/Frameworks,@loader_path/Frameworks', - '--macho-install-name=@rpath/App.framework/App', + ...kLinkInfoArgs, + '--snapshot_kind=app-aot-assembly', + '--assembly=$assembly', '--obfuscate', 'main.dill', ], ), kWhichSysctlCommand, kARMCheckCommand, - FakeCommand( + const FakeCommand( + command: [ + 'xcrun', + 'cc', + '-arch', + 'arm64', + '-miphoneos-version-min=13.0', + '-isysroot', + 'path/to/sdk', + '-c', + 'build/foo/snapshot_assembly.S', + '-o', + 'build/foo/snapshot_assembly.o', + ], + ), + const FakeCommand(command: ['xcrun', 'clang', '-arch', 'arm64', ...kDefaultClang]), + const FakeCommand( command: [ 'xcrun', 'dsymutil', '-o', - '$outputPath/App.framework.dSYM', - '$outputPath/App.framework/App', + 'build/foo/App.framework.dSYM', + 'build/foo/App.framework/App', ], ), - FakeCommand( + const FakeCommand( command: [ 'xcrun', 'strip', '-x', - '$outputPath/App.framework/App', + 'build/foo/App.framework/App', '-o', - '$outputPath/App.framework/App', + 'build/foo/App.framework/App', ], ), ]); @@ -302,34 +363,47 @@ void main() { command: [ genSnapshotPath, '--deterministic', - '--snapshot_kind=app-aot-macho-dylib', - '--macho=$outputPath/App.framework/App', - '--macho-object=$outputPath/app.o', - '--macho-min-os-version=13.0', - '--macho-rpath=@executable_path/Frameworks,@loader_path/Frameworks', - '--macho-install-name=@rpath/App.framework/App', + ...kLinkInfoArgs, + '--snapshot_kind=app-aot-assembly', + '--assembly=${fileSystem.path.join(outputPath, 'snapshot_assembly.S')}', 'main.dill', ], ), kWhichSysctlCommand, kARMCheckCommand, - FakeCommand( + const FakeCommand( + command: [ + 'xcrun', + 'cc', + '-arch', + 'arm64', + '-miphoneos-version-min=13.0', + '-isysroot', + 'path/to/sdk', + '-c', + 'build/foo/snapshot_assembly.S', + '-o', + 'build/foo/snapshot_assembly.o', + ], + ), + const FakeCommand(command: ['xcrun', 'clang', '-arch', 'arm64', ...kDefaultClang]), + const FakeCommand( command: [ 'xcrun', 'dsymutil', '-o', - '$outputPath/App.framework.dSYM', - '$outputPath/App.framework/App', + 'build/foo/App.framework.dSYM', + 'build/foo/App.framework/App', ], ), - FakeCommand( + const FakeCommand( command: [ 'xcrun', 'strip', '-x', - '$outputPath/App.framework/App', + 'build/foo/App.framework/App', '-o', - '$outputPath/App.framework/App', + 'build/foo/App.framework/App', ], ), ]); @@ -361,6 +435,7 @@ void main() { '--deterministic', '--snapshot_kind=app-aot-elf', '--elf=build/foo/app.so', + '--strip', '--no-sim-use-hardfp', '--no-use-integer-division', 'main.dill', @@ -394,6 +469,7 @@ void main() { '--deterministic', '--snapshot_kind=app-aot-elf', '--elf=build/foo/app.so', + '--strip', '--no-sim-use-hardfp', '--no-use-integer-division', '--dwarf-stack-traces', @@ -430,6 +506,7 @@ void main() { '--deterministic', '--snapshot_kind=app-aot-elf', '--elf=build/foo/app.so', + '--strip', '--no-sim-use-hardfp', '--no-use-integer-division', '--obfuscate', @@ -465,6 +542,7 @@ void main() { '--deterministic', '--snapshot_kind=app-aot-elf', '--elf=build/foo/app.so', + '--strip', '--no-sim-use-hardfp', '--no-use-integer-division', 'main.dill', @@ -499,6 +577,7 @@ void main() { '--deterministic', '--snapshot_kind=app-aot-elf', '--elf=build/foo/app.so', + '--strip', 'main.dill', ], ), diff --git a/packages/flutter_tools/test/general.shard/build_info_test.dart b/packages/flutter_tools/test/general.shard/build_info_test.dart index f744f3cd2d50b..ebc13483c7330 100644 --- a/packages/flutter_tools/test/general.shard/build_info_test.dart +++ b/packages/flutter_tools/test/general.shard/build_info_test.dart @@ -224,7 +224,7 @@ void main() { testWithoutContext('toBuildSystemEnvironment encoding of standard values', () { const buildInfo = BuildInfo( BuildMode.debug, - '', + 'strawberry', treeShakeIcons: true, trackWidgetCreation: true, dartDefines: ['foo=2', 'bar=2'], @@ -256,9 +256,21 @@ void main() { 'FileSystemScheme': 'scheme', 'BuildName': '122', 'BuildNumber': '22', + 'Flavor': 'strawberry', }); }); + testWithoutContext('toBuildSystemEnvironment omits Flavor when flavor is null', () { + const buildInfo = BuildInfo( + BuildMode.release, + null, + treeShakeIcons: false, + packageConfigPath: '.dart_tool/package_config.json', + ); + + expect(buildInfo.toBuildSystemEnvironment().containsKey('Flavor'), isFalse); + }); + testWithoutContext('toEnvironmentConfig encoding of standard values', () { const buildInfo = BuildInfo( BuildMode.debug, diff --git a/packages/flutter_tools/test/general.shard/build_system/build_trace_test.dart b/packages/flutter_tools/test/general.shard/build_system/build_trace_test.dart new file mode 100644 index 0000000000000..c1b61aa9488bd --- /dev/null +++ b/packages/flutter_tools/test/general.shard/build_system/build_trace_test.dart @@ -0,0 +1,239 @@ +// Copyright 2014 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'package:file/memory.dart'; +import 'package:flutter_tools/src/base/file_system.dart'; +import 'package:shorebird_build_trace/shorebird_build_trace.dart'; +import 'package:flutter_tools/src/convert.dart'; + +import '../../src/common.dart'; + +void main() { + group('BuildTraceEvent', () { + testWithoutContext('toJson produces Chrome Trace Event Format', () { + final event = BuildTraceEvent( + name: 'test_target', + cat: 'assemble', + start: DateTime.fromMicrosecondsSinceEpoch(1000000), + duration: const Duration(microseconds: 500000), + pid: 1, + tid: 3, + args: {'key': 'value'}, + ); + + final result = event.toJson(); + + expect(result['ph'], 'X'); + expect(result['name'], 'test_target'); + expect(result['cat'], 'assemble'); + expect(result['ts'], 1000000); + expect(result['dur'], 500000); + expect(result['pid'], 1); + expect(result['tid'], 3); + expect(result['args'], {'key': 'value'}); + }); + + testWithoutContext('toJson omits args when null', () { + final event = BuildTraceEvent( + name: 'test', + cat: 'flutter', + start: DateTime.fromMicrosecondsSinceEpoch(0), + duration: const Duration(microseconds: 100), + pid: 1, + tid: 1, + ); + + final result = event.toJson(); + + expect(result.containsKey('args'), isFalse); + }); + + testWithoutContext('fromJson round-trips correctly', () { + final original = { + 'ph': 'X', + 'name': 'test', + 'cat': 'flutter', + 'ts': 1000, + 'dur': 500, + 'pid': 1, + 'tid': 2, + 'args': {'foo': 'bar'}, + }; + + final event = BuildTraceEvent.fromJson(original); + final result = event.toJson(); + + expect(result['name'], 'test'); + expect(result['cat'], 'flutter'); + expect(result['ts'], 1000); + expect(result['dur'], 500); + expect(result['pid'], 1); + expect(result['tid'], 2); + }); + }); + + group('BuildTracer', () { + late FileSystem fileSystem; + + setUp(() { + fileSystem = MemoryFileSystem.test(); + }); + + testWithoutContext('addCompleteEvent adds event with correct duration', () { + final tracer = BuildTracer(); + + tracer.addCompleteEvent( + name: 'gradle build', + cat: 'gradle', + pid: 1, + tid: 2, + start: DateTime.fromMicrosecondsSinceEpoch(1000), + end: DateTime.fromMicrosecondsSinceEpoch(5000), + ); + + final outFile = fileSystem.file('trace.json'); + tracer.writeToFile(outFile); + + final events = json.decode(outFile.readAsStringSync()) as List; + expect(events, hasLength(1)); + + final event = events.first! as Map; + expect(event['name'], 'gradle build'); + expect(event['cat'], 'gradle'); + expect(event['tid'], 2); + expect(event['ts'], 1000); + expect(event['dur'], 4000); + expect(event['ph'], 'X'); + expect(event['pid'], 1); + }); + + testWithoutContext('mergeEventsFromFile reads and appends events', () { + final tracer = BuildTracer(); + + // Write a trace file to merge. + final sourceFile = fileSystem.file('source_trace.json'); + sourceFile.writeAsStringSync( + json.encode(>[ + { + 'ph': 'X', + 'name': 'KernelSnapshot', + 'cat': 'assemble', + 'ts': 2000, + 'dur': 1000, + 'pid': 1, + 'tid': 3, + }, + ]), + ); + + tracer.addCompleteEvent( + name: 'gradle build', + cat: 'gradle', + pid: 1, + tid: 2, + start: DateTime.fromMicrosecondsSinceEpoch(1000), + end: DateTime.fromMicrosecondsSinceEpoch(5000), + ); + + tracer.mergeEventsFromFile(sourceFile); + + final outFile = fileSystem.file('trace.json'); + tracer.writeToFile(outFile); + + final events = json.decode(outFile.readAsStringSync()) as List; + expect(events, hasLength(2)); + expect((events[0]! as Map)['name'], 'gradle build'); + expect((events[1]! as Map)['name'], 'KernelSnapshot'); + }); + + testWithoutContext('mergeEventsFromFile does nothing for non-existent file', () { + final tracer = BuildTracer(); + + tracer.addCompleteEvent( + name: 'test', + cat: 'flutter', + pid: 1, + tid: 1, + start: DateTime.fromMicrosecondsSinceEpoch(0), + end: DateTime.fromMicrosecondsSinceEpoch(100), + ); + + // Should not throw. + tracer.mergeEventsFromFile(fileSystem.file('does_not_exist.json')); + + final outFile = fileSystem.file('trace.json'); + tracer.writeToFile(outFile); + + final events = json.decode(outFile.readAsStringSync()) as List; + expect(events, hasLength(1)); + }); + + testWithoutContext('writeToFile creates parent directories', () { + final tracer = BuildTracer(); + tracer.addCompleteEvent( + name: 'test', + cat: 'flutter', + pid: 1, + tid: 1, + start: DateTime.fromMicrosecondsSinceEpoch(0), + end: DateTime.fromMicrosecondsSinceEpoch(100), + ); + + final outFile = fileSystem.file('/a/b/c/trace.json'); + tracer.writeToFile(outFile); + + expect(outFile.existsSync(), isTrue); + }); + + testWithoutContext('output is valid Chrome Trace Event Format', () { + final tracer = BuildTracer(); + + tracer.addCompleteEvent( + name: 'flutter build apk', + cat: 'flutter', + pid: 1, + tid: 1, + start: DateTime.fromMicrosecondsSinceEpoch(0), + end: DateTime.fromMicrosecondsSinceEpoch(15000000), + ); + tracer.addCompleteEvent( + name: 'gradle assembleRelease', + cat: 'gradle', + pid: 1, + tid: 2, + start: DateTime.fromMicrosecondsSinceEpoch(500000), + end: DateTime.fromMicrosecondsSinceEpoch(12500000), + ); + tracer.addCompleteEvent( + name: 'KernelSnapshot', + cat: 'assemble', + pid: 2, + tid: 1, + start: DateTime.fromMicrosecondsSinceEpoch(2000000), + end: DateTime.fromMicrosecondsSinceEpoch(5000000), + args: {'skipped': false}, + ); + + final outFile = fileSystem.file('trace.json'); + tracer.writeToFile(outFile); + + // Verify it's a valid JSON array. + final events = json.decode(outFile.readAsStringSync()) as List; + expect(events, hasLength(3)); + + // Verify all required fields are present in each event. + for (final item in events) { + final event = item! as Map; + expect(event.containsKey('ph'), isTrue); + expect(event.containsKey('name'), isTrue); + expect(event.containsKey('cat'), isTrue); + expect(event.containsKey('ts'), isTrue); + expect(event.containsKey('dur'), isTrue); + expect(event.containsKey('pid'), isTrue); + expect(event.containsKey('tid'), isTrue); + expect(event['ph'], 'X'); + } + }); + }); +} diff --git a/packages/flutter_tools/test/general.shard/build_system/targets/android_test.dart b/packages/flutter_tools/test/general.shard/build_system/targets/android_test.dart index 9ccb733787282..691e5c5857c8b 100644 --- a/packages/flutter_tools/test/general.shard/build_system/targets/android_test.dart +++ b/packages/flutter_tools/test/general.shard/build_system/targets/android_test.dart @@ -180,6 +180,7 @@ void main() { '--deterministic', '--snapshot_kind=app-aot-elf', '--elf=${environment.buildDir.childDirectory('arm64-v8a').childFile('app.so').path}', + '--strip', environment.buildDir.childFile('app.dill').path, ], ), @@ -218,6 +219,7 @@ void main() { '--trace-precompiler-to=code_size_1/trace.arm64-v8a.json', '--snapshot_kind=app-aot-elf', '--elf=${environment.buildDir.childDirectory('arm64-v8a').childFile('app.so').path}', + '--strip', environment.buildDir.childFile('app.dill').path, ], ), @@ -261,6 +263,7 @@ void main() { 'baz=2', '--snapshot_kind=app-aot-elf', '--elf=${environment.buildDir.childDirectory('arm64-v8a').childFile('app.so').path}', + '--strip', environment.buildDir.childFile('app.dill').path, ], ), @@ -303,6 +306,7 @@ void main() { 'bar', '--snapshot_kind=app-aot-elf', '--elf=${environment.buildDir.childDirectory('arm64-v8a').childFile('app.so').path}', + '--strip', environment.buildDir.childFile('app.dill').path, ], ), diff --git a/packages/flutter_tools/test/general.shard/build_system/targets/assets_test.dart b/packages/flutter_tools/test/general.shard/build_system/targets/assets_test.dart index fe482aeed9547..120ddb1c1efe8 100644 --- a/packages/flutter_tools/test/general.shard/build_system/targets/assets_test.dart +++ b/packages/flutter_tools/test/general.shard/build_system/targets/assets_test.dart @@ -244,183 +244,66 @@ flutter: }, ); - group( - "Only compiles shaders with a flavor if the shaders' flavor matches the flavor in the environment", - () { - testUsingContext( - 'When the environment does not have a flavor defined', - () async { - // Re-create the environment with the correct process manager for this test. - environment = Environment.test( - fileSystem.currentDirectory, - processManager: globals.processManager, - artifacts: Artifacts.test(), - fileSystem: fileSystem, - logger: BufferLogger.test(), - platform: FakePlatform(), - defines: {kBuildMode: BuildMode.debug.cliName}, - ); - - final artifacts = Artifacts.test(); - final String impellercPath = artifacts.getHostArtifact(HostArtifact.impellerc).path; - fileSystem.file(impellercPath).createSync(recursive: true); - - fileSystem.file('pubspec.yaml') - ..createSync() - ..writeAsStringSync(''' -name: example -flutter: - shaders: - - shaders/common.frag - - path: shaders/premium.frag - flavors: - - premium -'''); - writePackageConfigFiles(directory: globals.fs.currentDirectory, mainLibName: 'example'); - - fileSystem.file('shaders/common.frag').createSync(recursive: true); - fileSystem.file('shaders/premium.frag').createSync(recursive: true); - - // Manually create expected output files since FakeProcessManager.any() won't create them - fileSystem - .file('${environment.buildDir.path}/flutter_assets/shaders/common.frag') - .createSync(recursive: true); - - await const CopyAssets().build(environment); - - expect( - fileSystem.file('${environment.buildDir.path}/flutter_assets/shaders/common.frag'), - exists, - ); - expect( - fileSystem.file('${environment.buildDir.path}/flutter_assets/shaders/premium.frag'), - isNot(exists), - ); - }, - overrides: { - FileSystem: () => fileSystem, - ProcessManager: () => FakeProcessManager.any(), - }, - ); - - testUsingContext( - 'When the environment has a matching flavor defined', - () async { - // Re-create the environment with the correct process manager for this test. - environment = Environment.test( - fileSystem.currentDirectory, - processManager: globals.processManager, - artifacts: Artifacts.test(), - fileSystem: fileSystem, - logger: BufferLogger.test(), - platform: FakePlatform(), - defines: {kBuildMode: BuildMode.debug.cliName}, - ); - - final artifacts = Artifacts.test(); - final String impellercPath = artifacts.getHostArtifact(HostArtifact.impellerc).path; - fileSystem.file(impellercPath).createSync(recursive: true); + group('CopyAssets compiles shorebird.yaml using the environment flavor', () { + const shorebirdYamlContents = ''' +app_id: base-app-id +flavors: + internal: internal-app-id + stable: stable-app-id +'''; - environment.defines[kFlavor] = 'premium'; - fileSystem.file('pubspec.yaml') - ..createSync() - ..writeAsStringSync(''' + void writeProjectWithShorebirdYaml() { + fileSystem.file('pubspec.yaml') + ..createSync() + ..writeAsStringSync(''' name: example flutter: - shaders: - - shaders/common.frag - - path: shaders/premium.frag - flavors: - - premium + assets: + - shorebird.yaml '''); - writePackageConfigFiles(directory: globals.fs.currentDirectory, mainLibName: 'example'); - - fileSystem.file('shaders/common.frag').createSync(recursive: true); - fileSystem.file('shaders/premium.frag').createSync(recursive: true); - - // Manually create expected output files since FakeProcessManager.any() won't create them - fileSystem - .file('${environment.buildDir.path}/flutter_assets/shaders/common.frag') - .createSync(recursive: true); - fileSystem - .file('${environment.buildDir.path}/flutter_assets/shaders/premium.frag') - .createSync(recursive: true); - - await const CopyAssets().build(environment); - - expect( - fileSystem.file('${environment.buildDir.path}/flutter_assets/shaders/common.frag'), - exists, - ); - expect( - fileSystem.file('${environment.buildDir.path}/flutter_assets/shaders/premium.frag'), - exists, - ); - }, - overrides: { - FileSystem: () => fileSystem, - ProcessManager: () => FakeProcessManager.any(), - }, - ); - - testUsingContext( - 'When the environment has a mismatching flavor defined', - () async { - // Re-create the environment with the correct process manager for this test. - environment = Environment.test( - fileSystem.currentDirectory, - processManager: globals.processManager, - artifacts: Artifacts.test(), - fileSystem: fileSystem, - logger: BufferLogger.test(), - platform: FakePlatform(), - defines: {kBuildMode: BuildMode.debug.cliName}, - ); + fileSystem.file('shorebird.yaml') + ..createSync() + ..writeAsStringSync(shorebirdYamlContents); + writePackageConfigFiles(directory: globals.fs.currentDirectory, mainLibName: 'example'); + } - final artifacts = Artifacts.test(); - final String impellercPath = artifacts.getHostArtifact(HostArtifact.impellerc).path; - fileSystem.file(impellercPath).createSync(recursive: true); + testUsingContext( + 'falls back to top-level app_id when no flavor is set', + () async { + writeProjectWithShorebirdYaml(); - environment.defines[kFlavor] = 'free'; // Mismatch - fileSystem.file('pubspec.yaml') - ..createSync() - ..writeAsStringSync(''' -name: example -flutter: - shaders: - - shaders/common.frag - - path: shaders/premium.frag - flavors: - - premium -'''); - writePackageConfigFiles(directory: globals.fs.currentDirectory, mainLibName: 'example'); + await const CopyAssets().build(environment); - fileSystem.file('shaders/common.frag').createSync(recursive: true); - fileSystem.file('shaders/premium.frag').createSync(recursive: true); + final File compiled = fileSystem.file( + '${environment.buildDir.path}/flutter_assets/shorebird.yaml', + ); + expect(compiled.readAsStringSync(), 'app_id: base-app-id'); + }, + overrides: { + FileSystem: () => fileSystem, + ProcessManager: () => FakeProcessManager.any(), + }, + ); - // Manually create expected output files for the ones that SHOULD exist - fileSystem - .file('${environment.buildDir.path}/flutter_assets/shaders/common.frag') - .createSync(recursive: true); + testUsingContext( + 'uses the flavor app_id when flavor is set', + () async { + environment.defines[kFlavor] = 'internal'; + writeProjectWithShorebirdYaml(); - await const CopyAssets().build(environment); + await const CopyAssets().build(environment); - expect( - fileSystem.file('${environment.buildDir.path}/flutter_assets/shaders/common.frag'), - exists, - ); - expect( - fileSystem.file('${environment.buildDir.path}/flutter_assets/shaders/premium.frag'), - isNot(exists), - ); - }, - overrides: { - FileSystem: () => fileSystem, - ProcessManager: () => FakeProcessManager.any(), - }, - ); - }, - ); + final File compiled = fileSystem.file( + '${environment.buildDir.path}/flutter_assets/shorebird.yaml', + ); + expect(compiled.readAsStringSync(), 'app_id: internal-app-id'); + }, + overrides: { + FileSystem: () => fileSystem, + ProcessManager: () => FakeProcessManager.any(), + }, + ); + }); testUsingContext( 'transforms assets declared with transformers', @@ -509,124 +392,6 @@ flutter: }, ); - testUsingContext( - 'transforms shaders declared with transformers before compilation', - () async { - Cache.flutterRoot = Cache.defaultFlutterRoot( - platform: globals.platform, - fileSystem: fileSystem, - userMessages: UserMessages(), - ); - - final environment = Environment.test( - fileSystem.currentDirectory, - processManager: globals.processManager, - artifacts: Artifacts.test(), - fileSystem: fileSystem, - logger: logger, - platform: globals.platform, - defines: {kBuildMode: BuildMode.debug.cliName}, - ); - - final artifacts = Artifacts.test(); - final String impellercPath = artifacts.getHostArtifact(HostArtifact.impellerc).path; - fileSystem.file(impellercPath).createSync(recursive: true); - - fileSystem.file('pubspec.yaml') - ..createSync() - ..writeAsStringSync(''' -name: example -flutter: - shaders: - - path: shaders/test.frag - transformers: - - package: my_capitalizer_transformer -'''); - - writePackageConfigFiles(directory: globals.fs.currentDirectory, mainLibName: 'example'); - - fileSystem.file('shaders/test.frag') - ..createSync(recursive: true) - ..writeAsStringSync('abc'); - - await const CopyAssets().build(environment); - - expect(logger.errorText, isEmpty); - expect(globals.processManager, hasNoRemainingExpectations); - expect( - fileSystem.file('${environment.buildDir.path}/flutter_assets/shaders/test.frag'), - exists, - ); - }, - overrides: { - Logger: () => logger, - FileSystem: () => fileSystem, - Platform: () => FakePlatform(), - ProcessManager: () { - final artifacts = Artifacts.test(); - - return FakeProcessManager.list([ - FakeCommand( - command: [ - artifacts.getArtifactPath(Artifact.engineDartBinary), - 'run', - 'my_capitalizer_transformer', - RegExp('--input=.*'), - RegExp('--output=.*'), - ], - onRun: (List args) { - final ArgResults parsedArgs = - (ArgParser() - ..addOption('input') - ..addOption('output')) - .parse(args); - - final File input = fileSystem.file(parsedArgs['input'] as String); - expect(input, exists); - expect(input.readAsStringSync(), 'abc'); - - fileSystem.file(parsedArgs['output']) - ..createSync(recursive: true) - ..writeAsStringSync('ABC'); - }, - ), - FakeCommand( - command: [ - RegExp(r'.*impellerc.*'), - // Match the 11 arguments generated by ShaderCompiler.compileShader. - ...List.filled(11, RegExp(r'.*')), - ], - onRun: (List args) { - String? inputPath; - String? outputPath; - String? outputSpirvPath; - for (final arg in args) { - if (arg.startsWith('--input=')) { - inputPath = arg.substring('--input='.length); - } else if (arg.startsWith('--sl=')) { - outputPath = arg.substring('--sl='.length); - } else if (arg.startsWith('--spirv=')) { - outputSpirvPath = arg.substring('--spirv='.length); - } - } - - expect(inputPath, isNotNull); - expect(outputPath, isNotNull); - expect(outputSpirvPath, isNotNull); - - final File input = fileSystem.file(inputPath); - expect(input, exists); - expect(input.readAsStringSync(), 'ABC'); - - fileSystem.file(outputPath).createSync(recursive: true); - fileSystem.file(outputSpirvPath).createSync(recursive: true); - }, - ), - ]); - }, - }, - ); - testUsingContext( 'exits tool if an asset transformation fails', () async { diff --git a/packages/flutter_tools/test/general.shard/build_system/targets/common_test.dart b/packages/flutter_tools/test/general.shard/build_system/targets/common_test.dart index 707312f172316..18424a444742a 100644 --- a/packages/flutter_tools/test/general.shard/build_system/targets/common_test.dart +++ b/packages/flutter_tools/test/general.shard/build_system/targets/common_test.dart @@ -24,7 +24,17 @@ import '../../../src/fake_process_manager.dart'; const kBoundaryKey = '4d2d9609-c662-4571-afde-31410f96caa6'; const kElfAot = '--snapshot_kind=app-aot-elf'; -const kMachoDylibAot = '--snapshot_kind=app-aot-macho-dylib'; + +/// Generate Shorebird link info arguments for iOS/macOS AOT builds. +/// The [buildPath] should be the build directory path (outputDir.parent.path). +List linkInfoArgsFor(String buildPath) => [ + '--print_class_table_link_debug_info_to=$buildPath/App.class_table.json', + '--print_class_table_link_info_to=$buildPath/App.ct.link', + '--print_field_table_link_debug_info_to=$buildPath/App.field_table.json', + '--print_field_table_link_info_to=$buildPath/App.ft.link', + '--print_dispatch_table_link_debug_info_to=$buildPath/App.dispatch_table.json', + '--print_dispatch_table_link_info_to=$buildPath/App.dt.link', +]; final Platform macPlatform = FakePlatform( operatingSystem: 'macos', @@ -681,6 +691,7 @@ void main() { '--deterministic', kElfAot, '--elf=$build/app.so', + '--strip', '--no-sim-use-hardfp', '--no-use-integer-division', '$build/app.dill', @@ -711,6 +722,7 @@ void main() { '--trace-precompiler-to=code_size_1/trace.android-arm.json', kElfAot, '--elf=$build/app.so', + '--strip', '--no-sim-use-hardfp', '--no-use-integer-division', '$build/app.dill', @@ -802,17 +814,55 @@ void main() { // This path is not known by the cache due to the iOS gen_snapshot split. 'Artifact.genSnapshotArm64.TargetPlatform.ios.profile', '--deterministic', + ...linkInfoArgsFor(build), '--write-v8-snapshot-profile-to=code_size_1/snapshot.arm64.json', '--trace-precompiler-to=code_size_1/trace.arm64.json', - kMachoDylibAot, - '--macho=$build/arm64/App.framework/App', - '--macho-object=$build/arm64/app.o', - '--macho-min-os-version=13.0', - '--macho-rpath=@executable_path/Frameworks,@loader_path/Frameworks', - '--macho-install-name=@rpath/App.framework/App', + '--snapshot_kind=app-aot-assembly', + '--assembly=$build/arm64/snapshot_assembly.S', '$build/app.dill', ], ), + FakeCommand( + command: [ + 'xcrun', + 'cc', + '-arch', + 'arm64', + '-miphoneos-version-min=13.0', + '-isysroot', + 'path/to/iPhoneOS.sdk', + '-c', + '$build/arm64/snapshot_assembly.S', + '-o', + '$build/arm64/snapshot_assembly.o', + ], + ), + FakeCommand( + command: [ + 'xcrun', + 'clang', + '-arch', + 'arm64', + '-miphoneos-version-min=13.0', + '-isysroot', + 'path/to/iPhoneOS.sdk', + '-dynamiclib', + '-Xlinker', + '-rpath', + '-Xlinker', + '@executable_path/Frameworks', + '-Xlinker', + '-rpath', + '-Xlinker', + '@loader_path/Frameworks', + '-fapplication-extension', + '-install_name', + '@rpath/App.framework/App', + '-o', + '$build/arm64/App.framework/App', + '$build/arm64/snapshot_assembly.o', + ], + ), FakeCommand( command: [ 'xcrun', @@ -873,6 +923,7 @@ void main() { 'baz=2', kElfAot, '--elf=$build/app.so', + '--strip', '--no-sim-use-hardfp', '--no-use-integer-division', '$build/app.dill', diff --git a/packages/flutter_tools/test/general.shard/build_system/targets/ios_test.dart b/packages/flutter_tools/test/general.shard/build_system/targets/ios_test.dart index 743e6ffb37a84..b65c23b64ae80 100644 --- a/packages/flutter_tools/test/general.shard/build_system/targets/ios_test.dart +++ b/packages/flutter_tools/test/general.shard/build_system/targets/ios_test.dart @@ -771,6 +771,75 @@ void main() { }, ); + testUsingContext( + 'ReleaseIosApplicationBundle compiles shorebird.yaml using the flavor app_id', + () async { + environment.defines[kBuildMode] = 'release'; + environment.defines[kXcodeAction] = 'install'; + environment.defines[kFlavor] = 'internal'; + + fileSystem.file('pubspec.yaml') + ..createSync() + ..writeAsStringSync(''' +name: example +flutter: + assets: + - shorebird.yaml +'''); + + fileSystem.file('shorebird.yaml') + ..createSync() + ..writeAsStringSync(''' +app_id: base-app-id +flavors: + internal: internal-app-id + stable: stable-app-id +'''); + + writePackageConfigFiles(directory: fileSystem.currentDirectory, mainLibName: 'example'); + + fileSystem + .file(fileSystem.path.join('ios', 'Flutter', 'AppFrameworkInfo.plist')) + .createSync(recursive: true); + environment.buildDir + .childDirectory('App.framework') + .childFile('App') + .createSync(recursive: true); + environment.buildDir.childFile('native_assets.json').createSync(); + + final Directory frameworkDirectory = environment.outputDir.childDirectory('App.framework'); + final File frameworkDirectoryBinary = frameworkDirectory.childFile('App'); + final File infoPlist = frameworkDirectory.childFile('Info.plist'); + processManager.addCommands([ + createPlutilFakeCommand(infoPlist), + FakeCommand( + command: [ + 'xattr', + '-r', + '-d', + 'com.apple.FinderInfo', + frameworkDirectoryBinary.path, + ], + ), + FakeCommand( + command: ['codesign', '--force', '--sign', '-', frameworkDirectoryBinary.path], + ), + ]); + + await const ReleaseIosApplicationBundle().build(environment); + + final File compiledYaml = frameworkDirectory + .childDirectory('flutter_assets') + .childFile('shorebird.yaml'); + expect(compiledYaml.readAsStringSync(), 'app_id: internal-app-id'); + }, + overrides: { + FileSystem: () => fileSystem, + ProcessManager: () => processManager, + Platform: () => macPlatform, + }, + ); + testUsingContext( 'AotAssemblyRelease throws exception if asked to build for simulator', () async { diff --git a/packages/flutter_tools/test/general.shard/build_system/targets/macos_test.dart b/packages/flutter_tools/test/general.shard/build_system/targets/macos_test.dart index 15e6dbe62cba7..ed40c4dc93186 100644 --- a/packages/flutter_tools/test/general.shard/build_system/targets/macos_test.dart +++ b/packages/flutter_tools/test/general.shard/build_system/targets/macos_test.dart @@ -7,11 +7,9 @@ import 'package:file_testing/file_testing.dart'; import 'package:flutter_tools/src/artifacts.dart'; import 'package:flutter_tools/src/base/file_system.dart'; import 'package:flutter_tools/src/base/logger.dart'; -import 'package:flutter_tools/src/base/version.dart'; import 'package:flutter_tools/src/build_info.dart'; import 'package:flutter_tools/src/build_system/build_system.dart'; import 'package:flutter_tools/src/build_system/targets/macos.dart'; -import 'package:flutter_tools/src/features.dart'; import 'package:flutter_tools/src/ios/xcodeproj.dart'; import 'package:flutter_tools/src/project.dart'; import 'package:test/fake.dart'; @@ -23,6 +21,17 @@ import '../../../src/fake_process_manager.dart'; import '../../../src/fakes.dart'; import '../../../src/package_config.dart'; +/// Generate Shorebird link info arguments for iOS/macOS AOT builds. +/// The [buildPath] should be the build directory path (outputDir.parent.path). +List linkInfoArgsFor(String buildPath) => [ + '--print_class_table_link_debug_info_to=$buildPath/App.class_table.json', + '--print_class_table_link_info_to=$buildPath/App.ct.link', + '--print_field_table_link_debug_info_to=$buildPath/App.field_table.json', + '--print_field_table_link_info_to=$buildPath/App.ft.link', + '--print_dispatch_table_link_debug_info_to=$buildPath/App.dispatch_table.json', + '--print_dispatch_table_link_info_to=$buildPath/App.dt.link', +]; + void main() { late Environment environment; late MemoryFileSystem fileSystem; @@ -756,6 +765,73 @@ void main() { }, ); + testUsingContext( + 'ReleaseMacOSBundleFlutterAssets updates shorebird.yaml if present', + () async { + environment.defines[kBuildMode] = 'release'; + environment.defines[kXcodeAction] = 'install'; + environment.defines[kFlavor] = 'internal'; + + // Set up engine artifacts + fileSystem + .file('bin/cache/artifacts/engine/darwin-x64/vm_isolate_snapshot.bin') + .createSync(recursive: true); + fileSystem + .file('bin/cache/artifacts/engine/darwin-x64/isolate_snapshot.bin') + .createSync(recursive: true); + + // Set up App.framework binary + fileSystem + .file(fileSystem.path.join(environment.buildDir.path, 'App.framework', 'App')) + .createSync(recursive: true); + + // Set up native_assets.json (required by MacOSBundleFlutterAssets) + environment.buildDir.childFile('native_assets.json').createSync(); + + // Set up pubspec.yaml with shorebird.yaml as an asset + fileSystem.file('pubspec.yaml') + ..createSync() + ..writeAsStringSync(''' +name: example +flutter: + assets: + - shorebird.yaml +'''); + + // Create the shorebird.yaml asset file + fileSystem.file('shorebird.yaml') + ..createSync() + ..writeAsStringSync(''' +# Some other text that should be removed +app_id: base-app-id +flavors: + internal: internal-app-id + stable: stable-app-id +'''); + + // Set up package config + writePackageConfigFiles(directory: fileSystem.currentDirectory, mainLibName: 'example'); + + await const ReleaseMacOSBundleFlutterAssets().build(environment); + + // The output is in environment.outputDir, not buildDir + final String shorebirdYamlPath = fileSystem.path.join( + environment.outputDir.path, + 'App.framework', + 'Versions', + 'A', + 'Resources', + 'flutter_assets', + 'shorebird.yaml', + ); + expect(fileSystem.file(shorebirdYamlPath).readAsStringSync(), 'app_id: internal-app-id'); + }, + overrides: { + FileSystem: () => fileSystem, + ProcessManager: () => processManager, + }, + ); + testUsingContext( 'DebugMacOSFramework creates universal binary', () async { @@ -813,33 +889,107 @@ void main() { .childFile('x86_64/App.framework.dSYM/Contents/Resources/DWARF/App') .createSync(recursive: true); + final String build = environment.buildDir.path; + // Both archs go through the same code path now (the DD-pass is gated on + // SHOREBIRD_DD_MAX_BYTES which isn't set in this test), so neither + // architecture has an extra async hop. With concurrent Future.wait, the + // archs reach gen_snapshot in iteration order: arm64 first + // (`kDarwinArchs = 'arm64 x86_64'`), x86_64 second. They then + // interleave at each subsequent await point in the same order. processManager.addCommands([ + // arm64 gen_snapshot runs first (iteration order). FakeCommand( command: [ 'Artifact.genSnapshotArm64.TargetPlatform.darwin.release', '--deterministic', - '--snapshot_kind=app-aot-macho-dylib', - '--macho=${environment.buildDir.childFile('arm64/App.framework/App').path}', - '--macho-object=${environment.buildDir.childFile('arm64/app.o').path}', - '--macho-min-os-version=10.15', - '--macho-rpath=@executable_path/Frameworks,@loader_path/Frameworks', - '--macho-install-name=@rpath/App.framework/App', + ...linkInfoArgsFor(build), + '--snapshot_kind=app-aot-assembly', + '--assembly=${environment.buildDir.childFile('arm64/snapshot_assembly.S').path}', environment.buildDir.childFile('app.dill').path, ], ), + // x86_64 gen_snapshot runs next. FakeCommand( command: [ 'Artifact.genSnapshotX64.TargetPlatform.darwin.release', '--deterministic', - '--snapshot_kind=app-aot-macho-dylib', - '--macho=${environment.buildDir.childFile('x86_64/App.framework/App').path}', - '--macho-object=${environment.buildDir.childFile('x86_64/app.o').path}', - '--macho-min-os-version=10.15', - '--macho-rpath=@executable_path/Frameworks,@loader_path/Frameworks', - '--macho-install-name=@rpath/App.framework/App', + ...linkInfoArgsFor(build), + '--snapshot_kind=app-aot-assembly', + '--assembly=${environment.buildDir.childFile('x86_64/snapshot_assembly.S').path}', environment.buildDir.childFile('app.dill').path, ], ), + // From here on the two builds interleave: arm64 then x86_64 at each step. + FakeCommand( + command: [ + 'xcrun', + 'cc', + '-arch', + 'arm64', + '-c', + environment.buildDir.childFile('arm64/snapshot_assembly.S').path, + '-o', + environment.buildDir.childFile('arm64/snapshot_assembly.o').path, + ], + ), + FakeCommand( + command: [ + 'xcrun', + 'cc', + '-arch', + 'x86_64', + '-c', + environment.buildDir.childFile('x86_64/snapshot_assembly.S').path, + '-o', + environment.buildDir.childFile('x86_64/snapshot_assembly.o').path, + ], + ), + FakeCommand( + command: [ + 'xcrun', + 'clang', + '-arch', + 'arm64', + '-dynamiclib', + '-Xlinker', + '-rpath', + '-Xlinker', + '@executable_path/Frameworks', + '-Xlinker', + '-rpath', + '-Xlinker', + '@loader_path/Frameworks', + '-fapplication-extension', + '-install_name', + '@rpath/App.framework/App', + '-o', + environment.buildDir.childFile('arm64/App.framework/App').path, + environment.buildDir.childFile('arm64/snapshot_assembly.o').path, + ], + ), + FakeCommand( + command: [ + 'xcrun', + 'clang', + '-arch', + 'x86_64', + '-dynamiclib', + '-Xlinker', + '-rpath', + '-Xlinker', + '@executable_path/Frameworks', + '-Xlinker', + '-rpath', + '-Xlinker', + '@loader_path/Frameworks', + '-fapplication-extension', + '-install_name', + '@rpath/App.framework/App', + '-o', + environment.buildDir.childFile('x86_64/App.framework/App').path, + environment.buildDir.childFile('x86_64/snapshot_assembly.o').path, + ], + ), FakeCommand( command: [ 'xcrun', @@ -912,45 +1062,14 @@ void main() { ProcessManager: () => processManager, }, ); - - group('FlutterMacOS output', () { - late MemoryFileSystem testFileSystem; - - setUp(() { - testFileSystem = MemoryFileSystem.test(); - testFileSystem.file('pubspec.yaml').createSync(recursive: true); - testFileSystem.directory('macos').createSync(recursive: true); - testFileSystem - .directory('macos/Flutter/ephemeral/Packages/.packages/FlutterFramework') - .createSync(recursive: true); - }); - testUsingContext( - 'included when not using SwiftPM', - () async { - const Target target = ReleaseUnpackMacOS(); - expect(target.outputs.contains(kFlutterMacOSFrameworkBinarySource), isTrue); - }, - overrides: { - FeatureFlags: () => TestFeatureFlags(), - XcodeProjectInterpreter: () => FakeXcodeProjectInterpreter(version: Version(15, 0, 0)), - }, - ); - }); } class FakeXcodeProjectInterpreter extends Fake implements XcodeProjectInterpreter { - FakeXcodeProjectInterpreter({ - this.isInstalled = true, - this.version, - this.schemes = const ['Runner'], - }); + FakeXcodeProjectInterpreter({this.isInstalled = true, this.schemes = const ['Runner']}); @override final bool isInstalled; - @override - final Version? version; - List schemes; @override diff --git a/packages/flutter_tools/test/general.shard/cache_test.dart b/packages/flutter_tools/test/general.shard/cache_test.dart index 3ee1942488e52..7073da49c315e 100644 --- a/packages/flutter_tools/test/general.shard/cache_test.dart +++ b/packages/flutter_tools/test/general.shard/cache_test.dart @@ -12,7 +12,6 @@ import 'package:flutter_tools/src/base/io.dart'; import 'package:flutter_tools/src/base/logger.dart'; import 'package:flutter_tools/src/base/os.dart'; import 'package:flutter_tools/src/base/platform.dart'; -import 'package:flutter_tools/src/base/terminal.dart'; import 'package:flutter_tools/src/cache.dart'; import 'package:flutter_tools/src/dart/pub.dart'; import 'package:flutter_tools/src/flutter_cache.dart'; @@ -22,7 +21,6 @@ import 'package:test/fake.dart'; import '../src/common.dart'; import '../src/context.dart'; -import '../src/fake_http_client.dart'; import '../src/fakes.dart'; const unameCommandForX64 = FakeCommand(command: ['uname', '-m'], stdout: 'x86_64'); @@ -369,468 +367,6 @@ void main() { ); }); - testWithoutContext('ArtifactSet.displayName defaults to name', () { - final cache = Cache.test(processManager: FakeProcessManager.any()); - final artifact = FakeSimpleArtifact(cache); - - expect(artifact.name, 'fake'); - expect(artifact.displayName, 'fake'); - }); - - testWithoutContext('ArtifactSet.downloadCount defaults to 1', () { - final cache = Cache.test(processManager: FakeProcessManager.any()); - final artifact = FakeSimpleArtifact(cache); - - expect(artifact.downloadCount, 1); - }); - - testWithoutContext( - 'EngineCachedArtifact.downloadCount returns sum of package and binary dirs', - () { - final FileSystem fileSystem = MemoryFileSystem.test(); - final Directory artifactDir = fileSystem.systemTempDirectory.createTempSync( - 'flutter_cache_test_artifact.', - ); - final cache = FakeSecondaryCache()..artifactDirectory = artifactDir; - - final artifact = FakeCachedArtifact( - cache: cache, - binaryDirs: >[ - ['dir1', 'url1'], - ['dir2', 'url2'], - ], - packageDirs: ['pkg1', 'pkg2', 'pkg3'], - requiredArtifacts: DevelopmentArtifact.universal, - ); - - expect(artifact.downloadCount, 5); - }, - ); - - group('ArtifactUpdater progress formatting', () { - late FileSystem fileSystem; - late Directory tempStorage; - late ArtifactUpdater artifactUpdater; - - setUp(() { - fileSystem = MemoryFileSystem.test(); - tempStorage = fileSystem.systemTempDirectory.createTempSync('temp.'); - artifactUpdater = ArtifactUpdater( - operatingSystemUtils: FakeOperatingSystemUtils(), - logger: BufferLogger.test(), - fileSystem: fileSystem, - tempStorage: tempStorage, - httpClient: FakeHttpClient.any(), - platform: FakePlatform(), - allowedBaseUrls: ['https://storage.googleapis.com'], - ); - }); - - testWithoutContext('formats single download with artifact index', () { - artifactUpdater.setProgressContext(artifactIndex: 2, artifactTotal: 5, downloadTotal: 1); - - expect(artifactUpdater.formatProgressMessage('Test Artifact'), '[2/5] Test Artifact'); - }); - - testWithoutContext('formats multiple downloads with tree prefix', () { - // First download gets โ”œโ”€ prefix - artifactUpdater.setProgressContext(artifactIndex: 1, artifactTotal: 3, downloadTotal: 4); - expect(artifactUpdater.formatProgressMessage('first'), ' โ”œโ”€ [1/4] first'); - - // Middle downloads also get โ”œโ”€ prefix - artifactUpdater.setProgressContext( - artifactIndex: 1, - artifactTotal: 3, - downloadTotal: 4, - downloadIndex: 1, - ); - expect(artifactUpdater.formatProgressMessage('second'), ' โ”œโ”€ [2/4] second'); - - artifactUpdater.setProgressContext( - artifactIndex: 1, - artifactTotal: 3, - downloadTotal: 4, - downloadIndex: 2, - ); - expect(artifactUpdater.formatProgressMessage('third'), ' โ”œโ”€ [3/4] third'); - - // Last download gets โ””โ”€ prefix - artifactUpdater.setProgressContext( - artifactIndex: 1, - artifactTotal: 3, - downloadTotal: 4, - downloadIndex: 3, - ); - expect(artifactUpdater.formatProgressMessage('fourth'), ' โ””โ”€ [4/4] fourth'); - }); - - testWithoutContext('resetProgressContext resets download index', () { - artifactUpdater.setProgressContext(artifactIndex: 2, artifactTotal: 5, downloadTotal: 1); - expect(artifactUpdater.formatProgressMessage('first'), '[2/5] first'); - - artifactUpdater.resetProgressContext(); - artifactUpdater.setProgressContext(artifactIndex: 1, artifactTotal: 3, downloadTotal: 1); - - // After reset, index should start fresh - expect(artifactUpdater.formatProgressMessage('new'), '[1/3] new'); - }); - }); - - group('DownloadProgress', () { - testWithoutContext('fraction and percent with known total', () { - final progress = DownloadProgress()..totalBytes = 1000; - progress.addBytesReceived(250); - - expect(progress.fractionReceived, 0.25); - expect(progress.percentReceived, 25); - }); - - testWithoutContext('fraction and percent with unknown total', () { - final progress = DownloadProgress(); - progress.addBytesReceived(500); - - expect(progress.fractionReceived, 0.0); - expect(progress.percentReceived, 0); - expect(progress.hasKnownSize, false); - }); - - testWithoutContext('fraction clamps to 1.0 when bytesReceived exceeds total', () { - final progress = DownloadProgress()..totalBytes = 100; - progress.addBytesReceived(150); - - expect(progress.fractionReceived, 1.0); - expect(progress.percentReceived, 100); - }); - - testWithoutContext('speed calculation', () { - final progress = DownloadProgress(); - progress.addBytesReceived(5000000); - const elapsed = Duration(seconds: 2); - - expect(progress.speedBytesPerSecond(elapsed), 2500000.0); - }); - - testWithoutContext('speed is zero when elapsed is zero', () { - final progress = DownloadProgress(); - progress.addBytesReceived(5000000); - - expect(progress.speedBytesPerSecond(Duration.zero), 0.0); - }); - - testWithoutContext('timeRemaining with known total', () { - final progress = DownloadProgress()..totalBytes = 10000000; - progress.addBytesReceived(5000000); - const elapsed = Duration(seconds: 2); - - // 5MB received in 2s = 2.5MB/s, 5MB timeRemaining = 2s timeRemaining - final Duration? rem = progress.timeRemaining(elapsed); - expect(rem, isNotNull); - expect(rem!.inSeconds, 2); - }); - - testWithoutContext('timeRemaining is null with unknown total', () { - final progress = DownloadProgress(); - progress.addBytesReceived(5000000); - - expect(progress.timeRemaining(const Duration(seconds: 2)), isNull); - }); - - testWithoutContext('timeRemaining is null when speed is zero', () { - final progress = DownloadProgress()..totalBytes = 10000000; - progress.addBytesReceived(5000000); - - expect(progress.timeRemaining(Duration.zero), isNull); - }); - - testWithoutContext('renderProgressBar at 50%', () { - final progress = DownloadProgress()..totalBytes = 100; - progress.addBytesReceived(50); - - final String bar = progress.renderProgressBar(20); - expect(bar.length, 20); - expect(bar, '${'โ–ˆ' * 10}${' ' * 10}'); - }); - - testWithoutContext('renderProgressBar uses sub-character partial block', () { - // 46% of 20 cells: totalEighths = round(0.46 * 20 * 8) = 74 - // fullBlocks=9, remainder=2 โ†’ partial='โ–Ž', emptyBlocks=10 - final progress = DownloadProgress()..totalBytes = 100; - progress.addBytesReceived(46); - - final String bar = progress.renderProgressBar(20); - expect(bar.length, 20); - expect(bar, '${'โ–ˆ' * 9}โ–Ž${' ' * 10}'); - }); - - testWithoutContext('renderProgressBar returns empty with unknown total', () { - final progress = DownloadProgress(); - progress.addBytesReceived(50); - - expect(progress.renderProgressBar(20), ''); - }); - - testWithoutContext('renderProgressBar returns empty with zero width', () { - final progress = DownloadProgress()..totalBytes = 100; - progress.addBytesReceived(50); - - expect(progress.renderProgressBar(0), ''); - }); - - testWithoutContext('formatBytes with known total', () { - final progress = DownloadProgress()..totalBytes = 10000000; - progress.addBytesReceived(5000000); - - expect(progress.formatBytes(), matches(r'^\d+\.\d+MB/\d+\.\d+MB$')); - }); - - testWithoutContext('formatBytes with unknown total', () { - final progress = DownloadProgress(); - progress.addBytesReceived(5000000); - - expect(progress.formatBytes(), matches(r'^\d+\.\d+MB$')); - }); - - testWithoutContext('formatSpeed', () { - final progress = DownloadProgress(); - progress.addBytesReceived(2000000); - - expect(progress.formatSpeed(const Duration(seconds: 1)), matches(r'^\d+\.\d+MB/s$')); - }); - - testWithoutContext('formatRemaining with known total and speed', () { - final progress = DownloadProgress()..totalBytes = 10000000; - progress.addBytesReceived(5000000); - - final String eta = progress.formatRemaining(const Duration(seconds: 2)); - expect(eta, startsWith('ETA ')); - expect(eta, contains('s')); - }); - - testWithoutContext('formatRemaining is empty with unknown total', () { - final progress = DownloadProgress(); - progress.addBytesReceived(5000000); - - expect(progress.formatRemaining(const Duration(seconds: 2)), ''); - }); - - testWithoutContext('formatProgressLine includes progress bar with known total', () { - final progress = DownloadProgress()..totalBytes = 10000000; - progress.addBytesReceived(5000000); - - final String line = progress.formatProgressLine( - elapsed: const Duration(seconds: 2), - terminalWidth: 80, - ); - - expect(line, contains('50%')); - expect(line, matches(r'\d+\.\d+MB/\d+\.\d+MB')); - expect(line, contains('/s')); - expect(line, contains('โ–ˆ')); - expect(line, contains('โ–')); - expect(line, contains('โ–•')); - }); - - testWithoutContext('formatProgressLine has fixed bar width and right-aligns info', () { - final progress = DownloadProgress()..totalBytes = 10000000; - progress.addBytesReceived(5000000); - - final String line = progress.formatProgressLine( - elapsed: const Duration(seconds: 2), - terminalWidth: 80, - ); - - // Bar is fixed at 28 inner chars + 2 brackets = 30 visual chars. - // Line total must equal terminalWidth. - expect(line.length, 80); - // Info is right-aligned: the line must end with the info, no trailing spaces. - expect(line, endsWith('ETA 2.0s')); - }); - - testWithoutContext('formatProgressLine omits bar with unknown total', () { - final progress = DownloadProgress(); - progress.addBytesReceived(5000000); - - final String line = progress.formatProgressLine( - elapsed: const Duration(seconds: 2), - terminalWidth: 80, - ); - - expect(line, isNot(contains('โ–ˆ'))); - expect(line, matches(r'\d+\.\d+MB')); - expect(line, contains('/s')); - }); - - testWithoutContext('formatProgressLine omits bar when terminal too narrow', () { - final progress = DownloadProgress()..totalBytes = 10000000; - progress.addBytesReceived(5000000); - - final String line = progress.formatProgressLine( - elapsed: const Duration(seconds: 2), - terminalWidth: 30, - ); - - expect(line, isNot(contains('โ–ˆ'))); - expect(line, contains('50%')); - }); - - testWithoutContext('formatProgressLine never exceeds terminalWidth', () { - final progress = DownloadProgress()..totalBytes = 10000000; - progress.addBytesReceived(5000000); - - final String line = progress.formatProgressLine( - elapsed: const Duration(seconds: 2), - terminalWidth: 30, - ); - - expect(line.length, lessThanOrEqualTo(30)); - }); - - testWithoutContext( - 'formatProgressLine does not have trailing whitespace with unknown total', - () { - final progress = DownloadProgress(); - progress.addBytesReceived(5000000); - - final String line = progress.formatProgressLine( - elapsed: const Duration(seconds: 2), - terminalWidth: 80, - ); - - expect(line, isNot(endsWith(' '))); - }, - ); - - testWithoutContext('formatCompletionSummary', () { - final progress = DownloadProgress(); - progress.addBytesReceived(21100000); - - expect( - progress.formatCompletionSummary(const Duration(seconds: 5)), - matches(r'^\(\d+\.\d+MB in \d+\.\d+s\)$'), - ); - }); - }); - - group('ArtifactUpdater download progress display', () { - late FileSystem fileSystem; - late Directory tempStorage; - late FakeStdio stdio; - late BufferLogger logger; - - setUp(() { - fileSystem = MemoryFileSystem.test(); - tempStorage = fileSystem.systemTempDirectory.createTempSync('temp.'); - stdio = FakeStdio(); - logger = BufferLogger.test(terminal: Terminal.test(supportsColor: true)); - }); - - testWithoutContext( - 'shows ANSI progress when stdio is provided and color is supported', - () async { - final body = List.filled(1000, 0); - final updater = ArtifactUpdater( - operatingSystemUtils: FakeOperatingSystemUtils(), - logger: logger, - fileSystem: fileSystem, - tempStorage: tempStorage, - httpClient: FakeHttpClient.list([ - FakeRequest( - Uri.parse('https://storage.googleapis.com/test-artifact.zip'), - response: FakeResponse(body: body), - ), - ]), - platform: FakePlatform(), - allowedBaseUrls: ['https://storage.googleapis.com'], - stdio: stdio, - ); - - updater.setProgressContext(artifactIndex: 1, artifactTotal: 1, downloadTotal: 1); - - await updater.downloadZipArchive( - 'Test Artifact', - Uri.parse('https://storage.googleapis.com/test-artifact.zip'), - fileSystem.directory('output')..createSync(), - ); - - final String allOutput = stdio.writtenToStdout.join(); - - // Should have written the status message followed by newline - expect(allOutput, contains('[1/1] Test Artifact')); - - // Should contain completion summary with size and time - expect(allOutput, contains('MB in')); - expect(allOutput, contains('s)')); - - // Should contain ANSI clear codes for cleanup - expect(allOutput, contains('\x1B[K')); // clear to end of line - expect(allOutput, contains('\x1B[1A')); // cursor up - - // Completion line should be right-aligned to terminal width (80). - final String completionLine = stdio.writtenToStdout.last.replaceAll('\n', ''); - expect(completionLine.length, 80); - }, - ); - - testWithoutContext('falls back to logger progress when stdio is not provided', () async { - final updater = ArtifactUpdater( - operatingSystemUtils: FakeOperatingSystemUtils(), - logger: logger, - fileSystem: fileSystem, - tempStorage: tempStorage, - httpClient: FakeHttpClient.list([ - FakeRequest( - Uri.parse('https://storage.googleapis.com/test-artifact.zip'), - response: const FakeResponse(body: [0, 0, 0]), - ), - ]), - platform: FakePlatform(), - allowedBaseUrls: ['https://storage.googleapis.com'], - ); - - updater.setProgressContext(artifactIndex: 1, artifactTotal: 1, downloadTotal: 1); - - await updater.downloadZipArchive( - 'Test Artifact', - Uri.parse('https://storage.googleapis.com/test-artifact.zip'), - fileSystem.directory('output')..createSync(), - ); - - // Without stdio, nothing should be written to stdout directly - expect(stdio.writtenToStdout, isEmpty); - }); - - testWithoutContext('falls back to logger progress when color is not supported', () async { - final noColorLogger = BufferLogger.test(); - - final updater = ArtifactUpdater( - operatingSystemUtils: FakeOperatingSystemUtils(), - logger: noColorLogger, - fileSystem: fileSystem, - tempStorage: tempStorage, - httpClient: FakeHttpClient.list([ - FakeRequest( - Uri.parse('https://storage.googleapis.com/test-artifact.zip'), - response: const FakeResponse(body: [0, 0, 0]), - ), - ]), - platform: FakePlatform(), - allowedBaseUrls: ['https://storage.googleapis.com'], - stdio: stdio, - ); - - updater.setProgressContext(artifactIndex: 1, artifactTotal: 1, downloadTotal: 1); - - await updater.downloadZipArchive( - 'Test Artifact', - Uri.parse('https://storage.googleapis.com/test-artifact.zip'), - fileSystem.directory('output')..createSync(), - ); - - // With stdio but no color support, should not write progress to stdout - expect(stdio.writtenToStdout, isEmpty); - }); - }); - testWithoutContext( 'EngineCachedArtifact makes binary dirs readable and executable by all', () async { @@ -1348,7 +884,7 @@ void main() { expect(messages, ['Web SDK']); expect(downloads, [ - 'https://storage.googleapis.com/flutter_infra_release/flutter/hijklmnop/flutter-web-sdk.zip', + 'https://download.shorebird.dev/flutter_infra_release/flutter/hijklmnop/flutter-web-sdk.zip', ]); expect(locations, ['/bin/cache/flutter_web_sdk']); @@ -1469,7 +1005,7 @@ void main() { expect(messages, ['Engine Information']); expect(downloads, [ - 'https://storage.googleapis.com/flutter_infra_release/flutter/hijklmnop/engine_stamp.json', + 'https://download.shorebird.dev/flutter_infra_release/flutter/hijklmnop/engine_stamp.json', ]); expect(locations, ['/bin/cache']); // file copy is done by the real uploader; not the fake. @@ -1902,33 +1438,22 @@ class FakeArtifactUpdater extends Fake implements ArtifactUpdater { void Function(String, Uri, Directory)? onDownloadFile; @override - Future downloadZippedTarball(String artifactName, Uri url, Directory location) async { - onDownloadZipTarball?.call(artifactName, url, location); + Future downloadZippedTarball(String message, Uri url, Directory location) async { + onDownloadZipTarball?.call(message, url, location); } @override - Future downloadZipArchive(String artifactName, Uri url, Directory location) async { - onDownloadZipArchive?.call(artifactName, url, location); + Future downloadZipArchive(String message, Uri url, Directory location) async { + onDownloadZipArchive?.call(message, url, location); } @override - Future downloadFile(String artifactName, Uri url, Directory location) async { - onDownloadFile?.call(artifactName, url, location); + Future downloadFile(String message, Uri url, Directory location) async { + onDownloadFile?.call(message, url, location); } @override void removeDownloadedFiles() {} - - @override - void setProgressContext({ - required int artifactIndex, - required int artifactTotal, - required int downloadTotal, - int downloadIndex = 0, - }) {} - - @override - void resetProgressContext() {} } class FakeArtifactUpdaterDownload extends ArtifactUpdater { diff --git a/packages/flutter_tools/test/general.shard/shorebird/shorebird_yaml_test.dart b/packages/flutter_tools/test/general.shard/shorebird/shorebird_yaml_test.dart new file mode 100644 index 0000000000000..240ae45f5bb18 --- /dev/null +++ b/packages/flutter_tools/test/general.shard/shorebird/shorebird_yaml_test.dart @@ -0,0 +1,113 @@ +// Copyright 2024 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'dart:io'; + +import 'package:flutter_tools/src/shorebird/shorebird_yaml.dart'; +import 'package:test/test.dart'; +import 'package:yaml/yaml.dart'; + +void main() { + group('ShorebirdYaml', () { + test('yaml ignores comments', () { + const String yamlContents = ''' +# This file is used to configure the Shorebird updater used by your app. +app_id: 6160a7d8-cc18-4928-1233-05b51c0bb02c + +# auto_update controls if Shorebird should automatically update in the background on launch. +auto_update: false +'''; + final YamlDocument input = loadYamlDocument(yamlContents); + final YamlMap yamlMap = input.contents as YamlMap; + final Map compiled = compileShorebirdYaml( + yamlMap, + flavor: null, + environment: {}, + ); + expect(compiled, { + 'app_id': '6160a7d8-cc18-4928-1233-05b51c0bb02c', + 'auto_update': false, + }); + }); + test('flavors', () { + // These are invalid app_ids but make for easy testing. + const String yamlContents = ''' +app_id: 1-a +auto_update: false +flavors: + foo: 2-a + bar: 3-a +'''; + final YamlDocument input = loadYamlDocument(yamlContents); + final YamlMap yamlMap = input.contents as YamlMap; + expect(appIdForFlavor(yamlMap, flavor: null), '1-a'); + expect(appIdForFlavor(yamlMap, flavor: 'foo'), '2-a'); + expect(appIdForFlavor(yamlMap, flavor: 'bar'), '3-a'); + expect(() => appIdForFlavor(yamlMap, flavor: 'unknown'), throwsException); + }); + test('all values', () { + // These are invalid app_ids but make for easy testing. + const String yamlContents = ''' +app_id: 1-a +auto_update: false +flavors: + foo: 2-a + bar: 3-a +base_url: https://example.com +patch_verification: strict +'''; + final YamlDocument input = loadYamlDocument(yamlContents); + final YamlMap yamlMap = input.contents as YamlMap; + final Map compiled1 = compileShorebirdYaml( + yamlMap, + flavor: null, + environment: {}, + ); + expect(compiled1, { + 'app_id': '1-a', + 'auto_update': false, + 'base_url': 'https://example.com', + 'patch_verification': 'strict', + }); + final Map compiled2 = compileShorebirdYaml( + yamlMap, + flavor: 'foo', + environment: {'SHOREBIRD_PUBLIC_KEY': '4-a'}, + ); + expect(compiled2, { + 'app_id': '2-a', + 'auto_update': false, + 'base_url': 'https://example.com', + 'patch_verification': 'strict', + 'patch_public_key': '4-a', + }); + }); + test('edit in place', () { + const String yamlContents = ''' +app_id: 1-a +auto_update: false +flavors: + foo: 2-a + bar: 3-a +base_url: https://example.com +'''; + // Make a temporary file to test editing in place. + final Directory tempDir = Directory.systemTemp.createTempSync('shorebird_yaml_test.'); + final File tempFile = File('${tempDir.path}/shorebird.yaml'); + tempFile.writeAsStringSync(yamlContents); + updateShorebirdYaml( + 'foo', + tempFile.path, + environment: {'SHOREBIRD_PUBLIC_KEY': '4-a'}, + ); + final String updatedContents = tempFile.readAsStringSync(); + // Order is not guaranteed, so parse as YAML to compare. + final YamlDocument updated = loadYamlDocument(updatedContents); + final YamlMap yamlMap = updated.contents as YamlMap; + expect(yamlMap['app_id'], '2-a'); + expect(yamlMap['auto_update'], false); + expect(yamlMap['base_url'], 'https://example.com'); + }); + }); +} diff --git a/packages/flutter_tools/test/general.shard/version_test.dart b/packages/flutter_tools/test/general.shard/version_test.dart index 1d8360c14866c..90681d938b5b0 100644 --- a/packages/flutter_tools/test/general.shard/version_test.dart +++ b/packages/flutter_tools/test/general.shard/version_test.dart @@ -67,6 +67,20 @@ void main() { required int commitsBetweenRefs, }) { return [ + // Shorebird release branch check (returns empty to fall through to + // regular version lookup). + FakeCommand( + command: [ + 'git', + 'for-each-ref', + '--contains', + headRef, + '--format', + '%(refname:short)', + 'refs/remotes/origin/flutter_release/*', + ], + stdout: '', + ), FakeCommand( command: const [ 'git', @@ -1524,6 +1538,123 @@ void main() { expect(processManager, hasNoRemainingExpectations); }, overrides: {Git: () => git}); + testUsingContext('determine resolves version from shorebird flutter_release branch', () { + processManager.addCommands([ + const FakeCommand( + command: ['git', 'tag', '--points-at', 'HEAD'], + // No tag at HEAD. + ), + const FakeCommand( + command: [ + 'git', + 'for-each-ref', + '--contains', + 'HEAD', + '--format', + '%(refname:short)', + 'refs/remotes/origin/flutter_release/*', + ], + stdout: 'origin/flutter_release/3.41.4', + ), + ]); + final platform = FakePlatform(); + + final GitTagVersion gitTagVersion = GitTagVersion.determine( + platform, + git: git, + workingDirectory: '.', + ); + expect(gitTagVersion.frameworkVersionFor('abcd1234'), '3.41.4'); + expect(processManager, hasNoRemainingExpectations); + }); + + testUsingContext('determine skips non-stable shorebird flutter_release branches', () { + const headRevision = 'abcd1234'; + processManager.addCommands([ + const FakeCommand( + command: ['git', 'tag', '--points-at', 'HEAD'], + // No tag at HEAD. + ), + // Shorebird release branch check returns only a non-stable branch. + const FakeCommand( + command: [ + 'git', + 'for-each-ref', + '--contains', + 'HEAD', + '--format', + '%(refname:short)', + 'refs/remotes/origin/flutter_release/*', + ], + stdout: 'origin/flutter_release/3.41.4-rc2', + ), + // Falls through to tag-based fallback. + const FakeCommand( + command: [ + 'git', + 'for-each-ref', + '--sort=-v:refname', + '--count=1', + '--format=%(refname:short)', + 'refs/tags/[0-9]*.*.*', + ], + stdout: '3.41.4', + ), + const FakeCommand( + command: ['git', 'merge-base', 'HEAD', '3.41.4'], + stdout: headRevision, + ), + const FakeCommand( + command: ['git', 'rev-list', '--count', '$headRevision..HEAD'], + stdout: '48', + ), + ]); + final platform = FakePlatform(); + + final GitTagVersion gitTagVersion = GitTagVersion.determine( + platform, + git: git, + workingDirectory: '.', + ); + // Should NOT be 0.0.0-unknown; the tag fallback should produce a valid version. + expect(gitTagVersion.frameworkVersionFor(headRevision), '3.41.5-0.0.pre-48'); + expect(processManager, hasNoRemainingExpectations); + }); + + testUsingContext( + 'determine picks stable branch over rc branch from shorebird flutter_release', + () { + processManager.addCommands([ + const FakeCommand( + command: ['git', 'tag', '--points-at', 'HEAD'], + // No tag at HEAD. + ), + const FakeCommand( + command: [ + 'git', + 'for-each-ref', + '--contains', + 'HEAD', + '--format', + '%(refname:short)', + 'refs/remotes/origin/flutter_release/*', + ], + // Both stable and rc branches contain this commit. + stdout: 'origin/flutter_release/3.41.4\norigin/flutter_release/3.41.4-rc2', + ), + ]); + final platform = FakePlatform(); + + final GitTagVersion gitTagVersion = GitTagVersion.determine( + platform, + git: git, + workingDirectory: '.', + ); + expect(gitTagVersion.frameworkVersionFor('abcd1234'), '3.41.4'); + expect(processManager, hasNoRemainingExpectations); + }, + ); + group('$FlutterEngineStampFromFile', () { late FileSystem fs; const flutterRoot = '/path/to/flutter'; diff --git a/packages/shorebird_tests/.gitignore b/packages/shorebird_tests/.gitignore new file mode 100644 index 0000000000000..3a85790408401 --- /dev/null +++ b/packages/shorebird_tests/.gitignore @@ -0,0 +1,3 @@ +# https://dart.dev/guides/libraries/private-files +# Created by `dart pub` +.dart_tool/ diff --git a/packages/shorebird_tests/README.md b/packages/shorebird_tests/README.md new file mode 100644 index 0000000000000..b87fb24a480da --- /dev/null +++ b/packages/shorebird_tests/README.md @@ -0,0 +1,2 @@ +A dart project that includes tests that perform asserts in the modifications +made on the Flutter framework by the Shorebird team. diff --git a/packages/shorebird_tests/analysis_options.yaml b/packages/shorebird_tests/analysis_options.yaml new file mode 100644 index 0000000000000..a767d79d7f4b1 --- /dev/null +++ b/packages/shorebird_tests/analysis_options.yaml @@ -0,0 +1,2 @@ +# This file configures the static analysis results for your project (errors, +include: package:lints/recommended.yaml diff --git a/packages/shorebird_tests/pubspec.yaml b/packages/shorebird_tests/pubspec.yaml new file mode 100644 index 0000000000000..d0fba0c1fa95d --- /dev/null +++ b/packages/shorebird_tests/pubspec.yaml @@ -0,0 +1,16 @@ +name: shorebird_tests +description: Shorebird's Flutter customizations tests +version: 1.0.0 + +environment: + sdk: ^3.3.4 + +dependencies: + archive: ^3.5.1 + path: ^1.9.0 + yaml: ^3.1.2 + +dev_dependencies: + lints: ^3.0.0 + meta: ^1.15.0 + test: ^1.24.0 diff --git a/packages/shorebird_tests/test/android_test.dart b/packages/shorebird_tests/test/android_test.dart new file mode 100644 index 0000000000000..1e0a6da4f3824 --- /dev/null +++ b/packages/shorebird_tests/test/android_test.dart @@ -0,0 +1,94 @@ +import 'package:test/test.dart'; + +import 'shorebird_tests.dart'; + +void main() { + setUpAll(warmUpTemplateProject); + + group('shorebird android projects', () { + testWithShorebirdProject('can build an apk', (projectDirectory) async { + await projectDirectory.runFlutterBuildApk(); + + expect(projectDirectory.apkFile().existsSync(), isTrue); + expect(projectDirectory.shorebirdFile.existsSync(), isTrue); + expect(projectDirectory.getGeneratedAndroidShorebirdYaml(), completes); + }); + + group('when passing the public key through the environment variable', () { + testWithShorebirdProject( + 'adds the public key on top of the original file', + (projectDirectory) async { + final originalYaml = projectDirectory.shorebirdYaml; + + const base64PublicKey = 'public_123'; + await projectDirectory.runFlutterBuildApk( + environment: { + 'SHOREBIRD_PUBLIC_KEY': base64PublicKey, + }, + ); + + final generatedYaml = + await projectDirectory.getGeneratedAndroidShorebirdYaml(); + + expect( + generatedYaml.keys, + containsAll(originalYaml.keys), + ); + + expect( + generatedYaml['patch_public_key'], + equals(base64PublicKey), + ); + }, + ); + }); + + group('when building with a flavor', () { + testWithShorebirdProject( + 'correctly changes the app id', + (projectDirectory) async { + await projectDirectory.addProjectFlavors(); + projectDirectory.addShorebirdFlavors(); + + await projectDirectory.runFlutterBuildApk(flavor: 'internal'); + + final generatedYaml = + await projectDirectory.getGeneratedAndroidShorebirdYaml( + flavor: 'internal', + ); + + expect(generatedYaml['app_id'], equals('internal_123')); + }, + ); + + group('when public key passed through environment variable', () { + testWithShorebirdProject( + 'correctly changes the app id and adds the public key', + (projectDirectory) async { + const base64PublicKey = 'public_123'; + await projectDirectory.addProjectFlavors(); + projectDirectory.addShorebirdFlavors(); + + await projectDirectory.runFlutterBuildApk( + flavor: 'internal', + environment: { + 'SHOREBIRD_PUBLIC_KEY': base64PublicKey, + }, + ); + + final generatedYaml = + await projectDirectory.getGeneratedAndroidShorebirdYaml( + flavor: 'internal', + ); + + expect(generatedYaml['app_id'], equals('internal_123')); + expect( + generatedYaml['patch_public_key'], + equals(base64PublicKey), + ); + }, + ); + }); + }); + }); +} diff --git a/packages/shorebird_tests/test/base_test.dart b/packages/shorebird_tests/test/base_test.dart new file mode 100644 index 0000000000000..ce9b0564b1396 --- /dev/null +++ b/packages/shorebird_tests/test/base_test.dart @@ -0,0 +1,17 @@ +import 'package:test/test.dart'; + +import 'shorebird_tests.dart'; + +void main() { + setUpAll(warmUpTemplateProject); + + group('shorebird helpers', () { + testWithShorebirdProject('can build a base project', + (projectDirectory) async { + expect(projectDirectory.existsSync(), isTrue); + + expect(projectDirectory.pubspecFile.existsSync(), isTrue); + expect(projectDirectory.shorebirdFile.existsSync(), isTrue); + }); + }); +} diff --git a/packages/shorebird_tests/test/ios_test.dart b/packages/shorebird_tests/test/ios_test.dart new file mode 100644 index 0000000000000..bb694405ae946 --- /dev/null +++ b/packages/shorebird_tests/test/ios_test.dart @@ -0,0 +1,93 @@ +import 'package:test/test.dart'; + +import 'shorebird_tests.dart'; + +void main() { + setUpAll(warmUpTemplateProject); + + group( + 'shorebird ios projects', + () { + testWithShorebirdProject('can build', (projectDirectory) async { + await projectDirectory.runFlutterBuildIos(); + + expect(projectDirectory.iosArchiveFile().existsSync(), isTrue); + expect(projectDirectory.getGeneratedIosShorebirdYaml(), completes); + }); + + group('when passing the public key through the environment variable', () { + testWithShorebirdProject( + 'adds the public key on top of the original file', + (projectDirectory) async { + final originalYaml = projectDirectory.shorebirdYaml; + + const base64PublicKey = 'public_123'; + await projectDirectory.runFlutterBuildIos( + environment: { + 'SHOREBIRD_PUBLIC_KEY': base64PublicKey, + }, + ); + + final generatedYaml = + await projectDirectory.getGeneratedIosShorebirdYaml(); + + expect( + generatedYaml.keys, + containsAll(originalYaml.keys), + ); + + expect( + generatedYaml['patch_public_key'], + equals(base64PublicKey), + ); + }, + ); + }); + + group('when building with a flavor', () { + testWithShorebirdProject( + 'correctly changes the app id', + (projectDirectory) async { + await projectDirectory.addProjectFlavors(); + projectDirectory.addShorebirdFlavors(); + + await projectDirectory.runFlutterBuildIos(flavor: 'internal'); + + final generatedYaml = + await projectDirectory.getGeneratedIosShorebirdYaml(); + + expect(generatedYaml['app_id'], equals('internal_123')); + }, + ); + + group('when public key passed through environment variable', () { + testWithShorebirdProject( + 'correctly changes the app id and adds the public key', + (projectDirectory) async { + const base64PublicKey = 'public_123'; + await projectDirectory.addProjectFlavors(); + projectDirectory.addShorebirdFlavors(); + + await projectDirectory.runFlutterBuildIos( + flavor: 'internal', + environment: { + 'SHOREBIRD_PUBLIC_KEY': base64PublicKey, + }, + ); + + final generatedYaml = + await projectDirectory.getGeneratedIosShorebirdYaml(); + + expect(generatedYaml['app_id'], equals('internal_123')); + expect( + generatedYaml['patch_public_key'], + equals(base64PublicKey), + ); + }, + ); + }); + }); + }, + testOn: 'mac-os', + ); +} diff --git a/packages/shorebird_tests/test/shorebird_tests.dart b/packages/shorebird_tests/test/shorebird_tests.dart new file mode 100644 index 0000000000000..cfcda4c5b4603 --- /dev/null +++ b/packages/shorebird_tests/test/shorebird_tests.dart @@ -0,0 +1,424 @@ +import 'dart:async'; +import 'dart:convert'; +import 'dart:io'; + +import 'package:archive/archive_io.dart'; +import 'package:path/path.dart' as path; + +import 'package:meta/meta.dart'; +import 'package:test/test.dart'; +import 'package:yaml/yaml.dart'; + +/// This will be the path to the flutter binary housed in this flutter repository. +/// +/// Which since we are running the tests from this inner package , we need to go up two directories +/// in order to find the flutter binary in the bin folder. +File get _flutterBinaryFile => File( + path.join( + Directory.current.path, + '..', + '..', + 'bin', + 'flutter${Platform.isWindows ? '.bat' : ''}', + ), + ); + +/// Whether to print line-by-line subprocess output. +/// +/// Set the `VERBOSE` environment variable to enable streaming output, +/// which is useful for debugging timeouts in CI. +final bool _verbose = Platform.environment.containsKey('VERBOSE'); + +/// Runs a flutter command using the correct binary ([_flutterBinaryFile]) with the given arguments. +/// +/// Streams stdout and stderr to the test output in real time when [_verbose] +/// is true, so CI logs show progress even if the process hangs or times out. +Future _runFlutterCommand( + List arguments, { + required Directory workingDirectory, + Map? environment, +}) async { + final String command = 'flutter ${arguments.join(' ')}'; + print('[$command] starting...'); + final stopwatch = Stopwatch()..start(); + + final Process process = await Process.start( + _flutterBinaryFile.absolute.path, + arguments, + workingDirectory: workingDirectory.path, + environment: { + 'FLUTTER_STORAGE_BASE_URL': 'https://download.shorebird.dev', + if (environment != null) ...environment, + }, + ); + + final StringBuffer stdoutBuffer = StringBuffer(); + final StringBuffer stderrBuffer = StringBuffer(); + + process.stdout.transform(utf8.decoder).listen((String data) { + stdoutBuffer.write(data); + if (_verbose) { + for (final String line in data.split('\n')) { + if (line.isNotEmpty) { + print(' [$command] $line'); + } + } + } + }); + + process.stderr.transform(utf8.decoder).listen((String data) { + stderrBuffer.write(data); + if (_verbose) { + for (final String line in data.split('\n')) { + if (line.isNotEmpty) { + print(' [$command] (stderr) $line'); + } + } + } + }); + + final int exitCode = await process.exitCode; + stopwatch.stop(); + print('[$command] completed in ${stopwatch.elapsed} ' + '(exit code $exitCode)'); + if (exitCode != 0 && !_verbose) { + print('[$command] stdout:\n$stdoutBuffer'); + print('[$command] stderr:\n$stderrBuffer'); + } + + return ProcessResult( + process.pid, + exitCode, + stdoutBuffer.toString(), + stderrBuffer.toString(), + ); +} + +Future _createFlutterProject(Directory projectDirectory) async { + final result = await _runFlutterCommand( + ['create', '--empty', '.'], + workingDirectory: projectDirectory, + ); + if (result.exitCode != 0) { + throw Exception('Failed to create Flutter project: ${result.stderr}'); + } +} + +/// Cached template project directory, created once and reused across tests. +/// +/// This avoids running `flutter create` for every test, which saves +/// significant time (especially the first Gradle/SDK download). +Directory? _templateProject; + +/// Creates (or returns the cached) template Flutter project with +/// shorebird.yaml configured. The first call runs `flutter create` and +/// `flutter build apk` to warm up Gradle caches. +/// +/// Call this from `setUpAll` so the expensive setup runs outside per-test +/// timeouts. +Future warmUpTemplateProject() => _getTemplateProject(); + +Future _getTemplateProject() async { + if (_templateProject != null) { + return _templateProject!; + } + + final Directory templateDir = Directory( + path.join(Directory.systemTemp.createTempSync().path, 'shorebird_template'), + )..createSync(); + + await _createFlutterProject(templateDir); + + templateDir.pubspecFile.writeAsStringSync(''' +${templateDir.pubspecFile.readAsStringSync()} + assets: + - shorebird.yaml +'''); + + File( + path.join(templateDir.path, 'shorebird.yaml'), + ).writeAsStringSync(''' +app_id: "123" +'''); + + // Warm up the Gradle cache with a throwaway build so subsequent + // per-test builds are fast and don't hit the per-test timeout. + // Skip if Gradle cache is already populated (e.g., from GHA cache restore). + final Directory gradleCache = Directory( + path.join(Platform.environment['HOME'] ?? '', '.gradle', 'caches'), + ); + final bool hasGradleCache = + gradleCache.existsSync() && gradleCache.listSync().isNotEmpty; + if (hasGradleCache) { + print('[warmup] Gradle cache exists, skipping warm-up build'); + } else { + await _runFlutterCommand( + ['build', 'apk'], + workingDirectory: templateDir, + ); + } + + _templateProject = templateDir; + return templateDir; +} + +/// Copies the template project to a fresh directory for test isolation. +Future _copyTemplateProject() async { + final Directory template = await _getTemplateProject(); + final Directory testDir = Directory( + path.join(Directory.systemTemp.createTempSync().path, 'shorebird_test'), + ); + + // Use platform copy to preserve the full directory tree efficiently. + final ProcessResult result; + if (Platform.isWindows) { + result = await Process.run('xcopy', [ + template.path, + testDir.path, + '/E', + '/I', + '/Q', + ]); + } else { + result = await Process.run('cp', ['-R', template.path, testDir.path]); + } + if (result.exitCode != 0) { + throw Exception('Failed to copy template project: ${result.stderr}'); + } + + return testDir; +} + +@isTest +Future testWithShorebirdProject(String name, + FutureOr Function(Directory projectDirectory) testFn) async { + test( + name, + () async { + final Directory projectDirectory = await _copyTemplateProject(); + + try { + await testFn(projectDirectory); + } finally { + projectDirectory.deleteSync(recursive: true); + } + }, + timeout: Timeout( + // Per-test timeout can be shorter now since the template project + // creation and Gradle warm-up happen outside the test timeout. + Duration(minutes: 6), + ), + ); +} + +extension ShorebirdProjectDirectoryOnDirectory on Directory { + File get pubspecFile => File( + path.join(this.path, 'pubspec.yaml'), + ); + + File get shorebirdFile => File( + path.join(this.path, 'shorebird.yaml'), + ); + + YamlMap get shorebirdYaml => + loadYaml(shorebirdFile.readAsStringSync()) as YamlMap; + + File get appGradleFile => File( + path.join(this.path, 'android', 'app', 'build.gradle'), + ); + + Future addPubDependency(String name, {bool dev = false}) async { + final result = await _runFlutterCommand( + ['pub', 'add', if (dev) '--dev', name], + workingDirectory: this, + ); + if (result.exitCode != 0) { + throw Exception( + 'Failed to run `flutter pub add $name`: ${result.stderr}'); + } + } + + Future addProjectFlavors() async { + await addPubDependency( + // Published flutter_flavorizr. We previously pinned a fork for Flutter + // 3.29 support (AngeloAvv/flutter_flavorizr#291, never merged); the + // published package has since adopted 3.29/AGP-8 support (>=2.3.0), so + // the fork is obsolete. + // + // Pinned to 2.4.2 (not the latest 2.5.x): 2.5.0 replaced the Ruby + // xcodeproj gem with dart_xcodeproj, and its generated .pbxproj breaks + // `flutter build ipa --no-codesign --flavor` (the unsigned flavor + // archive demands a Development Team). 2.4.2 is the last Ruby-xcodeproj + // release and matches the generation path the old fork used, while + // still carrying the AGP-8 resValues fix the fork lacked. + 'dev:flutter_flavorizr:2.4.2', + ); + + await File( + path.join( + this.path, + 'flavorizr.yaml', + ), + ).writeAsString(''' +flavors: + playStore: + app: + name: "App" + + android: + applicationId: "com.example.shorebird_test" + ios: + bundleId: "com.example.shorebird_test" + internal: + app: + name: "App (Internal)" + + android: + applicationId: "com.example.shorebird_test.internal" + ios: + bundleId: "com.example.shorebird_test.internal" + global: + app: + name: "App (Global)" + + android: + applicationId: "com.example.shorebird_test.global" + ios: + bundleId: "com.example.shorebird_test.global" +'''); + + final result = await _runFlutterCommand( + // -f/--force skips flutter_flavorizr's interactive "proceed? (Y/n)" + // prompt, which throws "No terminal attached to stdout" under CI. + ['pub', 'run', 'flutter_flavorizr', '-f'], + workingDirectory: this, + ); + if (result.exitCode != 0) { + throw Exception( + 'Failed to run `flutter pub run flutter_flavorizr`: ${result.stderr}'); + } + + // flutter_flavorizr 2.4.2 emits per-flavor `resValue` entries in + // build.gradle. AGP 8 (Flutter 3.44+) gates resValues behind an opt-in + // build feature, so the flavored `flutter build apk` fails with "contains + // custom resource values, but the feature is disabled" unless we enable + // it. (Published flavorizr only enables this on 2.5.x, which we can't use: + // 2.5.0's dart_xcodeproj rewrite breaks `flutter build ipa --no-codesign + // --flavor`. So we stay on the last Ruby-xcodeproj release and toggle the + // build feature ourselves.) + final gradleProperties = + File(path.join(this.path, 'android', 'gradle.properties')); + await gradleProperties.writeAsString( + '\nandroid.defaults.buildfeatures.resvalues=true\n', + mode: FileMode.append, + ); + } + + void addShorebirdFlavors() { + const flavors = ''' +flavors: + global: global_123 + internal: internal_123 + playStore: playStore_123 +'''; + + final currentShorebirdContent = shorebirdFile.readAsStringSync(); + shorebirdFile.writeAsStringSync( + ''' +$currentShorebirdContent +$flavors +''', + ); + } + + Future runFlutterBuildApk({ + String? flavor, + Map? environment, + }) async { + final result = await _runFlutterCommand( + [ + 'build', + 'apk', + if (flavor != null) '--flavor=$flavor', + ], + workingDirectory: this, + environment: environment, + ); + if (result.exitCode != 0) { + throw Exception('Failed to run `flutter build apk`: ${result.stderr}'); + } + } + + Future runFlutterBuildIos({ + Map? environment, + String? flavor, + }) async { + final result = await _runFlutterCommand( + // The projects used to test are generated on spot, to make it simpler we don't + // configure any apple accounts on it, so we skip code signing here. + ['build', 'ipa', '--no-codesign', if (flavor != null) '--flavor=$flavor'], + workingDirectory: this, + environment: environment, + ); + + if (result.exitCode != 0) { + throw Exception('Failed to run `flutter build ios`: ${result.stderr}'); + } + } + + File apkFile({String? flavor}) => File( + path.join( + this.path, + 'build', + 'app', + 'outputs', + 'flutter-apk', + 'app-${flavor != null ? '$flavor-' : ''}release.apk', + ), + ); + + Directory iosArchiveFile() => Directory( + path.join( + this.path, + 'build', + 'ios', + 'archive', + 'Runner.xcarchive', + ), + ); + + Future getGeneratedAndroidShorebirdYaml({String? flavor}) async { + final decodedBytes = + ZipDecoder().decodeBytes(apkFile(flavor: flavor).readAsBytesSync()); + + await extractArchiveToDisk( + decodedBytes, path.join(this.path, 'apk-extracted')); + + final yamlString = File( + path.join( + this.path, + 'apk-extracted', + 'assets', + 'flutter_assets', + 'shorebird.yaml', + ), + ).readAsStringSync(); + return loadYaml(yamlString) as YamlMap; + } + + Future getGeneratedIosShorebirdYaml() async { + final yamlString = File( + path.join( + iosArchiveFile().path, + 'Products', + 'Applications', + 'Runner.app', + 'Frameworks', + 'App.framework', + 'flutter_assets', + 'shorebird.yaml', + ), + ).readAsStringSync(); + return loadYaml(yamlString) as YamlMap; + } +} diff --git a/shorebird/ci/artifacts_manifest.template.yaml b/shorebird/ci/artifacts_manifest.template.yaml new file mode 100644 index 0000000000000..33223fced2de8 --- /dev/null +++ b/shorebird/ci/artifacts_manifest.template.yaml @@ -0,0 +1,65 @@ +# Template for artifacts_manifest.yaml +# This file is processed by shard_runner:finalize +# Variable substitution: {{flutter_engine_revision}} is replaced at generation time +# The $engine placeholder is kept as-is for Shorebird's artifact proxy + +flutter_engine_revision: {{flutter_engine_revision}} +storage_bucket: download.shorebird.dev +artifact_overrides: + # Android release artifacts + - flutter_infra_release/flutter/$engine/android-arm64-release/artifacts.zip + - flutter_infra_release/flutter/$engine/android-arm64-release/darwin-x64.zip + - flutter_infra_release/flutter/$engine/android-arm64-release/linux-x64.zip + - flutter_infra_release/flutter/$engine/android-arm64-release/symbols.zip + - flutter_infra_release/flutter/$engine/android-arm64-release/windows-x64.zip + + - flutter_infra_release/flutter/$engine/android-arm-release/artifacts.zip + - flutter_infra_release/flutter/$engine/android-arm-release/darwin-x64.zip + - flutter_infra_release/flutter/$engine/android-arm-release/linux-x64.zip + - flutter_infra_release/flutter/$engine/android-arm-release/symbols.zip + - flutter_infra_release/flutter/$engine/android-arm-release/windows-x64.zip + + - flutter_infra_release/flutter/$engine/android-x64-release/artifacts.zip + - flutter_infra_release/flutter/$engine/android-x64-release/darwin-x64.zip + - flutter_infra_release/flutter/$engine/android-x64-release/linux-x64.zip + - flutter_infra_release/flutter/$engine/android-x64-release/symbols.zip + - flutter_infra_release/flutter/$engine/android-x64-release/windows-x64.zip + + # engine_stamp.json + - flutter_infra_release/flutter/$engine/engine_stamp.json + + # Dart SDK + - flutter_infra_release/flutter/$engine/dart-sdk-darwin-arm64.zip + - flutter_infra_release/flutter/$engine/dart-sdk-darwin-x64.zip + - flutter_infra_release/flutter/$engine/dart-sdk-linux-x64.zip + - flutter_infra_release/flutter/$engine/dart-sdk-windows-x64.zip + + # Maven artifacts + - download.flutter.io/io/flutter/flutter_embedding_release/1.0.0-$engine/flutter_embedding_release-1.0.0-$engine.pom + - download.flutter.io/io/flutter/flutter_embedding_release/1.0.0-$engine/flutter_embedding_release-1.0.0-$engine.jar + - download.flutter.io/io/flutter/arm64_v8a_release/1.0.0-$engine/arm64_v8a_release-1.0.0-$engine.pom + - download.flutter.io/io/flutter/arm64_v8a_release/1.0.0-$engine/arm64_v8a_release-1.0.0-$engine.jar + - download.flutter.io/io/flutter/armeabi_v7a_release/1.0.0-$engine/armeabi_v7a_release-1.0.0-$engine.pom + - download.flutter.io/io/flutter/armeabi_v7a_release/1.0.0-$engine/armeabi_v7a_release-1.0.0-$engine.jar + - download.flutter.io/io/flutter/x86_64_release/1.0.0-$engine/x86_64_release-1.0.0-$engine.pom + - download.flutter.io/io/flutter/x86_64_release/1.0.0-$engine/x86_64_release-1.0.0-$engine.jar + + # Common release artifacts + - flutter_infra_release/flutter/$engine/flutter_patched_sdk_product.zip + + # iOS release artifacts + - flutter_infra_release/flutter/$engine/ios-release/artifacts.zip + - flutter_infra_release/flutter/$engine/ios-release/Flutter.framework.dSYM.zip + + # Linux release artifacts + - flutter_infra_release/flutter/$engine/linux-x64/artifacts.zip + - flutter_infra_release/flutter/$engine/linux-x64-release/linux-x64-flutter-gtk.zip + + # macOS release artifacts + - flutter_infra_release/flutter/$engine/darwin-x64-release/artifacts.zip + - flutter_infra_release/flutter/$engine/darwin-x64-release/framework.zip + - flutter_infra_release/flutter/$engine/darwin-x64-release/gen_snapshot.zip + + # Windows release artifacts + - flutter_infra_release/flutter/$engine/windows-x64/artifacts.zip + - flutter_infra_release/flutter/$engine/windows-x64-release/windows-x64-flutter.zip diff --git a/shorebird/ci/compose.json b/shorebird/ci/compose.json new file mode 100644 index 0000000000000..494ad46cc0f23 --- /dev/null +++ b/shorebird/ci/compose.json @@ -0,0 +1,42 @@ +{ + "ios-framework": { + "requires": ["ios-release", "ios-sim-x64", "ios-sim-arm64"], + "script": "flutter/sky/tools/create_ios_framework.py", + "flags": ["--dsym", "--strip", "--no-extension-safe-frameworks"], + "path_args": { + "--arm64-out-dir": "ios_release", + "--simulator-x64-out-dir": "ios_debug_sim", + "--simulator-arm64-out-dir": "ios_debug_sim_arm64" + }, + "artifacts": [ + {"src": "artifacts.zip", "dst": "flutter_infra_release/flutter/$engine/ios-release/artifacts.zip"}, + {"src": "Flutter.framework.dSYM.zip", "dst": "flutter_infra_release/flutter/$engine/ios-release/Flutter.framework.dSYM.zip"} + ] + }, + "macos-framework": { + "requires": ["mac-arm64", "mac-x64"], + "script": "flutter/sky/tools/create_macos_framework.py", + "flags": ["--dsym", "--strip", "--zip"], + "path_args": { + "--arm64-out-dir": "mac_release_arm64", + "--x64-out-dir": "mac_release" + }, + "artifacts": [ + {"src": "framework.zip", "dst": "flutter_infra_release/flutter/$engine/darwin-x64-release/framework.zip"}, + {"src": "FlutterMacOS.framework.dSYM.zip", "dst": "flutter_infra_release/flutter/$engine/darwin-x64/FlutterMacOS.framework.dSYM.zip"} + ] + }, + "macos-gen-snapshot": { + "requires": ["mac-arm64", "mac-x64"], + "script": "flutter/sky/tools/create_macos_gen_snapshots.py", + "flags": ["--zip"], + "path_args": { + "--arm64-path": "mac_release_arm64/universal/gen_snapshot_arm64", + "--x64-path": "mac_release/universal/gen_snapshot_x64" + }, + "dst": "release/snapshot", + "artifacts": [ + {"src": "snapshot/gen_snapshot.zip", "dst": "flutter_infra_release/flutter/$engine/darwin-x64-release/gen_snapshot.zip"} + ] + } +} diff --git a/shorebird/ci/internal/generate_manifest.sh b/shorebird/ci/internal/generate_manifest.sh new file mode 100755 index 0000000000000..ee9ef7ddfdf67 --- /dev/null +++ b/shorebird/ci/internal/generate_manifest.sh @@ -0,0 +1,93 @@ +#!/bin/bash + +# This script outputs an artifact_manifest.yaml mapping +# a shorebird engine revision to a flutter engine revision. +# Usage: +# ./generate_manifest.sh > artifact_manifest.yaml + +set -e + +# NOTE: If you edit this file you also may need to edit the global list +# of all known artifacts in the artifact_proxy's config.dart + +if [ "$#" -ne 1 ]; then + echo "Usage: ./generate_manifest.sh " + exit 1 +fi + +FLUTTER_ENGINE_REVISION=$1 + +cat < $MANIFEST_FILE + +# FIXME: This should not be in shell, it's too complicated/repetitive. +# Only need the libflutter.so (and flutter.jar) artifacts +# Artifact list: https://github.com/shorebirdtech/shorebird/blob/main/packages/artifact_proxy/lib/config.dart + +HOST_ARCH='darwin-x64' +ARM64_HOST_ARCH='darwin-arm64' + +INFRA_ROOT="gs://$STORAGE_BUCKET/flutter_infra_release/flutter/$ENGINE_HASH" + +# engine_stamp.json +ENGINE_STAMP_FILE=$ENGINE_OUT/engine_stamp.json +gsutil cp $ENGINE_STAMP_FILE $INFRA_ROOT/engine_stamp.json + +# Dart SDK +# This gets uploaded to flutter_infra_release/flutter/\$engine/dart-sdk-$HOST_ARCH.zip +# We also upload to the content-aware hash path to support local development branches. +CONTENT_INFRA_ROOT="gs://$STORAGE_BUCKET/flutter_infra_release/flutter/$CONTENT_HASH" + +# x64 Dart SDK +HOST_RELEASE=$ENGINE_OUT/host_release +DART_ZIP_FILE=dart-sdk-$HOST_ARCH.zip +( + cd $HOST_RELEASE; + zip -r $DART_ZIP_FILE dart-sdk +) +ZIPS_DEST=$INFRA_ROOT/$DART_ZIP_FILE +gsutil cp $HOST_RELEASE/$DART_ZIP_FILE $ZIPS_DEST +# Also upload to content-aware hash path +gsutil cp $HOST_RELEASE/$DART_ZIP_FILE $CONTENT_INFRA_ROOT/$DART_ZIP_FILE + +# arm64 Dart SDK +HOST_RELEASE_ARM64=$ENGINE_OUT/host_release_arm64 +DART_ZIP_FILE=dart-sdk-$ARM64_HOST_ARCH.zip +( + cd $HOST_RELEASE_ARM64; + zip -r $DART_ZIP_FILE dart-sdk +) +ZIPS_DEST=$INFRA_ROOT/$DART_ZIP_FILE +gsutil cp $HOST_RELEASE_ARM64/$DART_ZIP_FILE $ZIPS_DEST +# Also upload to content-aware hash path +gsutil cp $HOST_RELEASE_ARM64/$DART_ZIP_FILE $CONTENT_INFRA_ROOT/$DART_ZIP_FILE + +# We want to build flutter/tools/font_subset, but that doesn't work with +# --no-prebuilt-dart-sdk. +# https://github.com/flutter/flutter/issues/164531 +# # mac x64 host_release font_subset (ConstFinder) +# ARCH_OUT=$ENGINE_OUT/host_release +# ZIPS_OUT=$ARCH_OUT/zip_archives/$HOST_ARCH +# ZIPS_DEST=$INFRA_ROOT/darwin-x64-release +# gsutil cp $ZIPS_OUT/font-subset.zip $ZIPS_DEST/font-subset.zip + +# # mac arm64 host_release font_subset (ConstFinder) +# ARCH_OUT=$ENGINE_OUT/host_release_arm64 +# ZIPS_OUT=$ARCH_OUT/zip_archives/$HOST_ARCH +# ZIPS_DEST=$INFRA_ROOT/darwin-arm64-release +# gsutil cp $ZIPS_OUT/font-subset.zip $ZIPS_DEST/font-subset.zip + +# Android Arm64 release gen_snapshot +ARCH_OUT=$ENGINE_OUT/android_release_arm64 +ZIPS_OUT=$ARCH_OUT/zip_archives/android-arm64-release +ZIPS_DEST=$INFRA_ROOT/android-arm64-release +gsutil cp $ZIPS_OUT/$HOST_ARCH.zip $ZIPS_DEST/$HOST_ARCH.zip + +# Android Arm32 release gen_snapshot +ARCH_OUT=$ENGINE_OUT/android_release +ZIPS_OUT=$ARCH_OUT/zip_archives/android-arm-release +ZIPS_DEST=$INFRA_ROOT/android-arm-release +gsutil cp $ZIPS_OUT/$HOST_ARCH.zip $ZIPS_DEST/$HOST_ARCH.zip + +# Android x64 release gen_snapshot +ARCH_OUT=$ENGINE_OUT/android_release_x64 +ZIPS_OUT=$ARCH_OUT/zip_archives/android-x64-release +ZIPS_DEST=$INFRA_ROOT/android-x64-release +gsutil cp $ZIPS_OUT/$HOST_ARCH.zip $ZIPS_DEST/$HOST_ARCH.zip + +# Match the upload pattern from iOS: +# https://github.com/flutter/engine/commit/1d7f0c66c316a37105601b13136f890f6595aebc + +# iOS release Flutter artifacts +ARCH_OUT=$ENGINE_OUT/release +ZIPS_OUT=$ARCH_OUT +ZIPS_DEST=$INFRA_ROOT/ios-release +gsutil cp $ZIPS_OUT/artifacts.zip $ZIPS_DEST/artifacts.zip + +# iOS dSYM +gsutil cp $ZIPS_OUT/Flutter.framework.dSYM.zip $ZIPS_DEST/Flutter.framework.dSYM.zip + +# macOS framework +ARCH_OUT=$ENGINE_OUT/release +ZIPS_OUT=$ARCH_OUT/framework +ZIPS_DEST=$INFRA_ROOT/darwin-x64-release +gsutil cp $ZIPS_OUT/framework.zip $ZIPS_DEST/framework.zip + +# macOS gen_snapshot +ARCH_OUT=$ENGINE_OUT/release +ZIPS_OUT=$ARCH_OUT/snapshot +ZIPS_DEST=$INFRA_ROOT/darwin-x64-release +gsutil cp $ZIPS_OUT/gen_snapshot.zip $ZIPS_DEST/gen_snapshot.zip + +# FIXME: these should go where we're putting the arm64 macOS artifacts +# (darwin-x64-release), however, arm macs use darwin-x64-release and we +# currently only support those. We need to find a way to support both. +# macOS x64 release artifacts +# ARCH_OUT=$ENGINE_OUT/mac_release +# ZIPS_OUT=$ARCH_OUT/zip_archives/darwin-x64-release +# ZIPS_DEST=$INFRA_ROOT/darwin-x64-release +# gsutil cp $ZIPS_OUT/artifacts.zip $ZIPS_DEST/artifacts.zip + +# macOS arm64 release artifacts +ARCH_OUT=$ENGINE_OUT/mac_release_arm64 +ZIPS_OUT=$ARCH_OUT/zip_archives/darwin-arm64-release +# This looks wrong - why are we putting arm64 artifacts in darwin-x64-release +# instead of darwin-arm64-release? This is because arm macs use darwin-x64-release +# and we need to use the artifacts we've built for arm64 macs. +ZIPS_DEST=$INFRA_ROOT/darwin-x64-release +gsutil cp $ZIPS_OUT/artifacts.zip $ZIPS_DEST/artifacts.zip + +# macOS dSYM (used for symbolication, not by Flutter) +ARCH_OUT=$ENGINE_OUT/release +ZIPS_OUT=$ARCH_OUT/framework +ZIPS_DEST=$INFRA_ROOT/darwin-x64 +gsutil cp $ZIPS_OUT/FlutterMacOS.framework.dSYM.zip $ZIPS_DEST/FlutterMacOS.framework.dSYM.zip + +TMP_DIR=$(mktemp -d) + +PATCH_VERSION=0.3.0 +GH_RELEASE=https://github.com/shorebirdtech/updater/releases/download/patch-v$PATCH_VERSION/ +cd $TMP_DIR +curl -L $GH_RELEASE/patch-x86_64-apple-darwin.zip -o patch-x86_64-apple-darwin.zip +curl -L $GH_RELEASE/patch-aarch64-apple-darwin.zip -o patch-aarch64-apple-darwin.zip +curl -L $GH_RELEASE/patch-x86_64-pc-windows-msvc.zip -o patch-x86_64-pc-windows-msvc.zip +curl -L $GH_RELEASE/patch-x86_64-unknown-linux-musl.zip -o patch-x86_64-unknown-linux-musl.zip + +gsutil cp patch-x86_64-apple-darwin.zip $SHOREBIRD_ROOT/patch-darwin-x64.zip +gsutil cp patch-aarch64-apple-darwin.zip $SHOREBIRD_ROOT/patch-darwin-arm64.zip +gsutil cp patch-x86_64-pc-windows-msvc.zip $SHOREBIRD_ROOT/patch-windows-x64.zip +gsutil cp patch-x86_64-unknown-linux-musl.zip $SHOREBIRD_ROOT/patch-linux-x64.zip + +gsutil cp $MANIFEST_FILE $SHOREBIRD_ROOT/artifacts_manifest.yaml diff --git a/shorebird/ci/internal/win_build.sh b/shorebird/ci/internal/win_build.sh new file mode 100755 index 0000000000000..9c47369707d34 --- /dev/null +++ b/shorebird/ci/internal/win_build.sh @@ -0,0 +1,63 @@ +#!/bin/bash -e + +# Usage: +# ./win_build.sh engine_path + +ENGINE_ROOT=$1 + +ENGINE_SRC=$ENGINE_ROOT/src +ENGINE_OUT=$ENGINE_SRC/out +UPDATER_SRC=$ENGINE_SRC/flutter/third_party/updater +HOST_ARCH='windows-x64' + +# The Rust updater library is now built as part of the GN/Ninja engine +# build โ€” see //flutter/shell/common/shorebird/BUILD.gn's +# build_rust_updater action. Any engine target that depends (transitively) +# on //flutter/shell/common/shorebird:updater pulls in updater.lib +# automatically. + +# Compile the engine using the steps here: +# https://github.com/flutter/flutter/wiki/Compiling-the-engine#compiling-for-android-from-macos-or-linux +cd $ENGINE_SRC + +NINJA="ninja" +GN=./flutter/tools/gn +# We could probably use our own prebuilt dart SDK, by modifying the gn files. +GN_ARGS="--no-rbe --no-enable-unittests" + +# Windows only needs gen_snapshot for each Android CPU type. +# See https://github.com/flutter/engine/blob/e590b24f3962fda3ec9144dcee3f7565b195839a/ci/builders/windows_android_aot_engine.json + +TARGETS="archive_win_gen_snapshot" + +# Build host_release +$GN $GN_ARGS --runtime-mode=release --no-prebuilt-dart-sdk +$NINJA -C out/host_release dart_sdk +# We want to build flutter/tools/font_subset, but that doesn't work with +# --no-prebuilt-dart-sdk. +# https://github.com/flutter/flutter/issues/164531 + +# Build windows desktop targets +$GN $GN_ARGS --runtime-mode=release --no-prebuilt-dart-sdk +$NINJA -C ./out/host_release flutter/build/archives:windows_flutter gen_snapshot windows flutter/build/archives:artifacts + +# Build debug Windows artifacts +# These are output to the `windows-x64` directory in host_debug, and are used +# by `flutter build windows --release`. +$GN $GN_ARGS --no-prebuilt-dart-sdk +$NINJA -C ./out/host_debug flutter/build/archives:artifacts + +# If this gives you trouble, try using VS2019 instead. I had trouble with 2022. +# Android arm64 release +$GN $GN_ARGS --android --android-cpu=arm64 --runtime-mode=release +$NINJA -C ./out/android_release_arm64 $TARGETS + +# Android arm32 release +$GN $GN_ARGS --runtime-mode=release --android +$NINJA -C out/android_release $TARGETS + +# Android x64 release +$GN $GN_ARGS --android --android-cpu=x64 --runtime-mode=release +$NINJA -C ./out/android_release_x64 $TARGETS + +# We could also build the `patch` tool for Windows here. diff --git a/shorebird/ci/internal/win_setup.sh b/shorebird/ci/internal/win_setup.sh new file mode 100755 index 0000000000000..971135644f66b --- /dev/null +++ b/shorebird/ci/internal/win_setup.sh @@ -0,0 +1,8 @@ +#!/bin/bash -e + +# Usage: +# ./windows_setup.sh + +# Add the MSVC toolchain to Rust. +rustup target add \ + x86_64-pc-windows-msvc diff --git a/shorebird/ci/internal/win_upload.sh b/shorebird/ci/internal/win_upload.sh new file mode 100755 index 0000000000000..952457109e40c --- /dev/null +++ b/shorebird/ci/internal/win_upload.sh @@ -0,0 +1,90 @@ +#!/bin/bash -e + +# Usage: +# ./win_upload.sh engine_path git_hash +ENGINE_ROOT=$1 +ENGINE_HASH=$2 + +STORAGE_BUCKET="download.shorebird.dev" +SHOREBIRD_ROOT=gs://$STORAGE_BUCKET/shorebird/$ENGINE_HASH + +ENGINE_SRC=$ENGINE_ROOT/src +ENGINE_OUT=$ENGINE_SRC/out +ENGINE_FLUTTER=$ENGINE_SRC/flutter +# FLUTTER_ROOT is the Flutter monorepo root (parent of engine/) +FLUTTER_ROOT=$(dirname $ENGINE_ROOT) + +cd $FLUTTER_ROOT + +# Compute the content-aware hash for the Dart SDK. +# This allows Flutter checkouts that haven't changed engine content to share +# the same pre-built Dart SDK, even if they have different git commit SHAs. +CONTENT_HASH=$($FLUTTER_ROOT/bin/internal/content_aware_hash.sh) + +# We do not generate a manifest file, we assume another builder did that. + +# TODO(eseidel): This should not be in shell, it's too complicated/repetitive. + +HOST_ARCH='windows-x64' + +INFRA_ROOT="gs://$STORAGE_BUCKET/flutter_infra_release/flutter/$ENGINE_HASH" + +# Dart SDK +# This gets uploaded to flutter_infra_release/flutter/\$engine/dart-sdk-$HOST_ARCH.zip +# We also upload to the content-aware hash path to support local development branches. +CONTENT_INFRA_ROOT="gs://$STORAGE_BUCKET/flutter_infra_release/flutter/$CONTENT_HASH" + +DART_SDK_DIR=$ENGINE_OUT/host_release/dart-sdk +DART_ZIP_FILE=dart-sdk-$HOST_ARCH.zip + +# Use 7zip to compress the Dart SDK, as zip isn't available on Windows and +# Powershell, which we would normally use in the form of +# `powershell Compress-Archive dart-sdk dart-sdk.zip`, doesn't play nicely +# with git bash paths (e.g. /c/Users/... instead of C:/Users/...) +/c/Program\ Files/7-Zip/7z a $DART_ZIP_FILE $DART_SDK_DIR +ZIPS_DEST=$INFRA_ROOT/$DART_ZIP_FILE +gsutil cp $DART_ZIP_FILE $ZIPS_DEST +# Also upload to content-aware hash path +gsutil cp $DART_ZIP_FILE $CONTENT_INFRA_ROOT/$DART_ZIP_FILE + +# Android Arm64 release gen_snapshot +ARCH_OUT=$ENGINE_OUT/android_release_arm64 +ZIPS_OUT=$ARCH_OUT/zip_archives/android-arm64-release +ZIPS_DEST=$INFRA_ROOT/android-arm64-release +gsutil cp $ZIPS_OUT/$HOST_ARCH.zip $ZIPS_DEST/$HOST_ARCH.zip + +# Android Arm32 release gen_snapshot +ARCH_OUT=$ENGINE_OUT/android_release +ZIPS_OUT=$ARCH_OUT/zip_archives/android-arm-release +ZIPS_DEST=$INFRA_ROOT/android-arm-release +gsutil cp $ZIPS_OUT/$HOST_ARCH.zip $ZIPS_DEST/$HOST_ARCH.zip + +# Android x64 release gen_snapshot +ARCH_OUT=$ENGINE_OUT/android_release_x64 +ZIPS_OUT=$ARCH_OUT/zip_archives/android-x64-release +ZIPS_DEST=$INFRA_ROOT/android-x64-release +gsutil cp $ZIPS_OUT/$HOST_ARCH.zip $ZIPS_DEST/$HOST_ARCH.zip + +# We could upload patch if we built it here. +# gsutil cp $ENGINE_OUT/host_release/patch.zip $SHOREBIRD_ROOT/patch-win-x64.zip + +# Engine release artifacts +ARCH_OUT=$ENGINE_OUT/host_release +ZIPS_OUT=$ARCH_OUT/zip_archives/$HOST_ARCH-release +ZIPS_DEST=$INFRA_ROOT/$HOST_ARCH-release +gsutil cp $ZIPS_OUT/$HOST_ARCH-flutter.zip $ZIPS_DEST/$HOST_ARCH-flutter.zip + +# We want to build flutter/tools/font_subset, but that doesn't work with +# --no-prebuilt-dart-sdk. +# https://github.com/flutter/flutter/issues/164531 +# # Windows x64 host_release font_subset (ConstFinder) +# ARCH_OUT=$ENGINE_OUT/host_release +# ZIPS_OUT=$ARCH_OUT/zip_archives/$HOST_ARCH +# ZIPS_DEST=$INFRA_ROOT/windows-x64-release +# gsutil cp $ZIPS_OUT/font-subset.zip $ZIPS_DEST/font-subset.zip + +# Engine debug artifacts (not sure why this is needed?) +ARCH_OUT=$ENGINE_OUT/host_debug +ZIPS_OUT=$ARCH_OUT/zip_archives/$HOST_ARCH +ZIPS_DEST=$INFRA_ROOT/$HOST_ARCH +gsutil cp $ZIPS_OUT/artifacts.zip $ZIPS_DEST/artifacts.zip diff --git a/shorebird/ci/linux_build_and_upload.sh b/shorebird/ci/linux_build_and_upload.sh new file mode 100755 index 0000000000000..9657e14178695 --- /dev/null +++ b/shorebird/ci/linux_build_and_upload.sh @@ -0,0 +1,32 @@ +#!/bin/bash -e + +# Usage: +# ./linux_build_and_upload.sh flutter_root engine_hash +# +# This is the main entrypoint for building and uploading Linux engine artifacts. +# It is called from the _build_engine repository's CI scripts. + +if [ "$#" -ne 2 ]; then + echo "Usage: $0 flutter_root engine_hash" + exit 1 +fi + +FLUTTER_ROOT=$1 +ENGINE_HASH=$2 +ENGINE_ROOT=$FLUTTER_ROOT/engine + +# Get the absolute path to the directory of this script. +SCRIPT_DIR=$(cd $(dirname $0) && pwd) + +echo "Building engine at $ENGINE_ROOT and uploading to gs://download.shorebird.dev" + +cd $SCRIPT_DIR + +# Run the setup script. +./internal/linux_setup.sh + +# Then run the build. +./internal/linux_build.sh $ENGINE_ROOT + +# Copy Shorebird engine artifacts to Google Cloud Storage. +./internal/linux_upload.sh $ENGINE_ROOT $ENGINE_HASH diff --git a/shorebird/ci/mac_build_and_upload.sh b/shorebird/ci/mac_build_and_upload.sh new file mode 100755 index 0000000000000..9ee302620d207 --- /dev/null +++ b/shorebird/ci/mac_build_and_upload.sh @@ -0,0 +1,32 @@ +#!/bin/bash -e + +# Usage: +# ./mac_build_and_upload.sh flutter_root engine_hash +# +# This is the main entrypoint for building and uploading macOS engine artifacts. +# It is called from the _build_engine repository's CI scripts. + +if [ "$#" -ne 2 ]; then + echo "Usage: $0 flutter_root engine_hash" + exit 1 +fi + +FLUTTER_ROOT=$1 +ENGINE_HASH=$2 +ENGINE_ROOT=$FLUTTER_ROOT/engine + +# Get the absolute path to the directory of this script. +SCRIPT_DIR=$(cd $(dirname $0) && pwd) + +echo "Building engine at $ENGINE_ROOT and uploading to gs://download.shorebird.dev" + +cd $SCRIPT_DIR + +# Run the setup script. +./internal/mac_setup.sh + +# Then run the build. +./internal/mac_build.sh $ENGINE_ROOT + +# Copy Shorebird engine artifacts to Google Cloud Storage. +./internal/mac_upload.sh $ENGINE_ROOT $ENGINE_HASH diff --git a/shorebird/ci/shards/linux.json b/shorebird/ci/shards/linux.json new file mode 100644 index 0000000000000..ffd1b4982dbda --- /dev/null +++ b/shorebird/ci/shards/linux.json @@ -0,0 +1,108 @@ +{ + "android-arm64": { + "steps": [ + { + "type": "rust", + "targets": ["aarch64-linux-android"] + }, + { + "type": "gn_ninja", + "gn_args": ["--android", "--android-cpu=arm64", "--runtime-mode=release"], + "ninja_targets": ["default", "gen_snapshot"], + "out_dir": "android_release_arm64" + } + ], + "artifacts": [ + {"src": "android_release_arm64/zip_archives/android-arm64-release/artifacts.zip", "dst": "flutter_infra_release/flutter/$engine/android-arm64-release/artifacts.zip"}, + {"src": "android_release_arm64/zip_archives/android-arm64-release/linux-x64.zip", "dst": "flutter_infra_release/flutter/$engine/android-arm64-release/linux-x64.zip"}, + {"src": "android_release_arm64/zip_archives/android-arm64-release/symbols.zip", "dst": "flutter_infra_release/flutter/$engine/android-arm64-release/symbols.zip"}, + {"src": "android_release_arm64/arm64_v8a_release.pom", "dst": "download.flutter.io/io/flutter/arm64_v8a_release/1.0.0-$engine/arm64_v8a_release-1.0.0-$engine.pom"}, + {"src": "android_release_arm64/arm64_v8a_release.jar", "dst": "download.flutter.io/io/flutter/arm64_v8a_release/1.0.0-$engine/arm64_v8a_release-1.0.0-$engine.jar"}, + {"src": "android_release_arm64/arm64_v8a_release.maven-metadata.xml", "dst": "download.flutter.io/io/flutter/arm64_v8a_release/1.0.0-$engine/arm64_v8a_release-1.0.0-$engine.maven-metadata.xml"} + ] + }, + "android-arm32": { + "steps": [ + { + "type": "rust", + "targets": ["armv7-linux-androideabi"] + }, + { + "type": "gn_ninja", + "gn_args": ["--android", "--runtime-mode=release"], + "ninja_targets": ["default", "gen_snapshot"], + "out_dir": "android_release" + } + ], + "artifacts": [ + {"src": "android_release/zip_archives/android-arm-release/artifacts.zip", "dst": "flutter_infra_release/flutter/$engine/android-arm-release/artifacts.zip"}, + {"src": "android_release/zip_archives/android-arm-release/linux-x64.zip", "dst": "flutter_infra_release/flutter/$engine/android-arm-release/linux-x64.zip"}, + {"src": "android_release/zip_archives/android-arm-release/symbols.zip", "dst": "flutter_infra_release/flutter/$engine/android-arm-release/symbols.zip"}, + {"src": "android_release/armeabi_v7a_release.pom", "dst": "download.flutter.io/io/flutter/armeabi_v7a_release/1.0.0-$engine/armeabi_v7a_release-1.0.0-$engine.pom"}, + {"src": "android_release/armeabi_v7a_release.jar", "dst": "download.flutter.io/io/flutter/armeabi_v7a_release/1.0.0-$engine/armeabi_v7a_release-1.0.0-$engine.jar"}, + {"src": "android_release/armeabi_v7a_release.maven-metadata.xml", "dst": "download.flutter.io/io/flutter/armeabi_v7a_release/1.0.0-$engine/armeabi_v7a_release-1.0.0-$engine.maven-metadata.xml"}, + {"src": "android_release/flutter_embedding_release.pom", "dst": "download.flutter.io/io/flutter/flutter_embedding_release/1.0.0-$engine/flutter_embedding_release-1.0.0-$engine.pom"}, + {"src": "android_release/flutter_embedding_release.jar", "dst": "download.flutter.io/io/flutter/flutter_embedding_release/1.0.0-$engine/flutter_embedding_release-1.0.0-$engine.jar"}, + {"src": "android_release/flutter_embedding_release.maven-metadata.xml", "dst": "download.flutter.io/io/flutter/flutter_embedding_release/1.0.0-$engine/flutter_embedding_release-1.0.0-$engine.maven-metadata.xml"} + ] + }, + "android-x64": { + "steps": [ + { + "type": "rust", + "targets": ["x86_64-linux-android"] + }, + { + "type": "gn_ninja", + "gn_args": ["--android", "--android-cpu=x64", "--runtime-mode=release"], + "ninja_targets": ["default", "gen_snapshot"], + "out_dir": "android_release_x64" + } + ], + "artifacts": [ + {"src": "android_release_x64/zip_archives/android-x64-release/artifacts.zip", "dst": "flutter_infra_release/flutter/$engine/android-x64-release/artifacts.zip"}, + {"src": "android_release_x64/zip_archives/android-x64-release/linux-x64.zip", "dst": "flutter_infra_release/flutter/$engine/android-x64-release/linux-x64.zip"}, + {"src": "android_release_x64/zip_archives/android-x64-release/symbols.zip", "dst": "flutter_infra_release/flutter/$engine/android-x64-release/symbols.zip"}, + {"src": "android_release_x64/x86_64_release.pom", "dst": "download.flutter.io/io/flutter/x86_64_release/1.0.0-$engine/x86_64_release-1.0.0-$engine.pom"}, + {"src": "android_release_x64/x86_64_release.jar", "dst": "download.flutter.io/io/flutter/x86_64_release/1.0.0-$engine/x86_64_release-1.0.0-$engine.jar"}, + {"src": "android_release_x64/x86_64_release.maven-metadata.xml", "dst": "download.flutter.io/io/flutter/x86_64_release/1.0.0-$engine/x86_64_release-1.0.0-$engine.maven-metadata.xml"} + ] + }, + "host": { + "steps": [ + { + "type": "rust", + "targets": [ + "armv7-linux-androideabi", + "aarch64-linux-android", + "i686-linux-android", + "x86_64-linux-android", + "x86_64-unknown-linux-gnu" + ] + }, + { + "type": "gn_ninja", + "gn_args": ["--runtime-mode=release", "--no-prebuilt-dart-sdk"], + "ninja_targets": ["dart_sdk", "flutter/shell/platform/linux:flutter_gtk", "flutter/build/archives:flutter_patched_sdk"], + "out_dir": "host_release" + }, + { + "type": "gn_ninja", + "gn_args": ["--no-prebuilt-dart-sdk"], + "ninja_targets": ["flutter/build/archives:artifacts"], + "out_dir": "host_debug" + }, + { + "type": "shell", + "command": "mkdir -p out/host_release/aot_tools && out/host_release/dart-sdk/bin/dart pub get --offline --directory=flutter/third_party/dart/pkg/aot_tools && out/host_release/dart-sdk/bin/dart compile kernel flutter/third_party/dart/pkg/aot_tools/bin/aot_tools.dart -o out/host_release/aot_tools/aot-tools.dill" + } + ], + "artifacts": [ + {"src": "host_release/dart-sdk", "dst": "flutter_infra_release/flutter/$engine/dart-sdk-linux-x64.zip", "zip": true, "content_hash": true}, + {"src": "host_release/zip_archives/flutter_patched_sdk_product.zip", "dst": "flutter_infra_release/flutter/$engine/flutter_patched_sdk_product.zip"}, + {"src": "host_release/zip_archives/linux-x64-release/linux-x64-flutter-gtk.zip", "dst": "flutter_infra_release/flutter/$engine/linux-x64-release/linux-x64-flutter-gtk.zip"}, + {"src": "host_debug/zip_archives/linux-x64/artifacts.zip", "dst": "flutter_infra_release/flutter/$engine/linux-x64/artifacts.zip"}, + {"src": "host_release/aot_tools/aot-tools.dill", "dst": "shorebird/$engine/aot-tools.dill"} + ] + } +} diff --git a/shorebird/ci/shards/macos.json b/shorebird/ci/shards/macos.json new file mode 100644 index 0000000000000..0d1336c2be69f --- /dev/null +++ b/shorebird/ci/shards/macos.json @@ -0,0 +1,131 @@ +{ + "android": { + "steps": [ + { + "type": "rust", + "targets": [ + "aarch64-apple-darwin", + "x86_64-apple-darwin" + ] + }, + { + "type": "gn_ninja", + "gn_args": ["--android", "--android-cpu=arm64", "--runtime-mode=release", "--gn-args=host_cpu=\"x64\""], + "ninja_targets": ["flutter/shell/platform/android:gen_snapshot"], + "out_dir": "android_release_arm64" + }, + { + "type": "gn_ninja", + "gn_args": ["--android", "--runtime-mode=release", "--gn-args=host_cpu=\"x64\""], + "ninja_targets": ["flutter/shell/platform/android:gen_snapshot"], + "out_dir": "android_release" + }, + { + "type": "gn_ninja", + "gn_args": ["--android", "--android-cpu=x64", "--runtime-mode=release", "--gn-args=host_cpu=\"x64\""], + "ninja_targets": ["flutter/shell/platform/android:gen_snapshot"], + "out_dir": "android_release_x64" + } + ], + "artifacts": [ + {"src": "android_release_arm64/zip_archives/android-arm64-release/darwin-x64.zip", "dst": "flutter_infra_release/flutter/$engine/android-arm64-release/darwin-x64.zip"}, + {"src": "android_release/zip_archives/android-arm-release/darwin-x64.zip", "dst": "flutter_infra_release/flutter/$engine/android-arm-release/darwin-x64.zip"}, + {"src": "android_release_x64/zip_archives/android-x64-release/darwin-x64.zip", "dst": "flutter_infra_release/flutter/$engine/android-x64-release/darwin-x64.zip"} + ] + }, + "ios-release": { + "steps": [ + { + "type": "rust", + "targets": ["aarch64-apple-ios"] + }, + { + "type": "gn_ninja", + "gn_args": ["--ios", "--runtime-mode=release", "--gn-arg=shorebird_runtime=true"], + "ninja_targets": ["flutter/shell/platform/darwin/ios:flutter_framework", "flutter/lib/snapshot:generate_snapshot_bins"], + "out_dir": "ios_release" + } + ], + "compose_input": "ios-framework" + }, + "ios-sim-x64": { + "steps": [ + { + "type": "rust", + "targets": ["x86_64-apple-ios"] + }, + { + "type": "gn_ninja", + "gn_args": ["--ios", "--runtime-mode=debug", "--simulator"], + "ninja_targets": ["flutter/shell/platform/darwin/ios:flutter_framework", "flutter/lib/snapshot:generate_snapshot_bins"], + "out_dir": "ios_debug_sim" + } + ], + "compose_input": "ios-framework" + }, + "ios-sim-arm64": { + "steps": [ + { + "type": "gn_ninja", + "gn_args": ["--ios", "--runtime-mode=debug", "--simulator", "--simulator-cpu=arm64"], + "ninja_targets": ["flutter/shell/platform/darwin/ios:flutter_framework", "flutter/lib/snapshot:generate_snapshot_bins"], + "out_dir": "ios_debug_sim_arm64" + } + ], + "compose_input": "ios-framework" + }, + "mac-arm64": { + "steps": [ + { + "type": "rust", + "targets": ["aarch64-apple-darwin"] + }, + { + "type": "gn_ninja", + "gn_args": ["--runtime-mode=release", "--mac-cpu=arm64", "--no-prebuilt-dart-sdk"], + "ninja_targets": ["dart_sdk"], + "out_dir": "host_release_arm64" + }, + { + "type": "gn_ninja", + "gn_args": ["--runtime-mode=release", "--mac", "--mac-cpu=arm64"], + "ninja_targets": ["flutter/shell/platform/darwin/macos:zip_macos_flutter_framework", "flutter/lib/snapshot:generate_snapshot_bins", "flutter/build/archives:artifacts"], + "out_dir": "mac_release_arm64" + } + ], + "compose_input": "macos-framework", + "artifacts": [ + {"src": "host_release_arm64/dart-sdk", "dst": "flutter_infra_release/flutter/$engine/dart-sdk-darwin-arm64.zip", "zip": true, "content_hash": true}, + {"src": "mac_release_arm64/zip_archives/darwin-arm64-release/artifacts.zip", "dst": "flutter_infra_release/flutter/$engine/darwin-x64-release/artifacts.zip"} + ] + }, + "mac-x64": { + "steps": [ + { + "type": "rust", + "targets": ["x86_64-apple-darwin"] + }, + { + "type": "gn_ninja", + "gn_args": ["--runtime-mode=release", "--mac-cpu=x64", "--no-prebuilt-dart-sdk"], + "ninja_targets": ["dart_sdk"], + "out_dir": "host_release" + }, + { + "type": "gn_ninja", + "gn_args": ["--runtime-mode=release", "--mac", "--mac-cpu=x64"], + "ninja_targets": ["flutter/shell/platform/darwin/macos:zip_macos_flutter_framework", "flutter/lib/snapshot:generate_snapshot_bins", "flutter/build/archives:artifacts"], + "out_dir": "mac_release" + }, + { + "type": "shell", + "command": "flutter/bin/et stamp && cp out/engine_stamp.json out/mac_release/engine_stamp.json" + } + ], + "compose_input": "macos-framework", + "artifacts": [ + {"src": "host_release/dart-sdk", "dst": "flutter_infra_release/flutter/$engine/dart-sdk-darwin-x64.zip", "zip": true, "content_hash": true}, + {"src": "mac_release/engine_stamp.json", "dst": "flutter_infra_release/flutter/$engine/engine_stamp.json"} + ] + } +} diff --git a/shorebird/ci/shards/windows.json b/shorebird/ci/shards/windows.json new file mode 100644 index 0000000000000..4e5c47c71f0b5 --- /dev/null +++ b/shorebird/ci/shards/windows.json @@ -0,0 +1,76 @@ +{ + "android-arm64": { + "steps": [ + { + "type": "gn_ninja", + "gn_args": ["--android", "--android-cpu=arm64", "--runtime-mode=release"], + "ninja_targets": ["archive_win_gen_snapshot"], + "out_dir": "android_release_arm64" + } + ], + "artifacts": [ + {"src": "android_release_arm64/zip_archives/android-arm64-release/windows-x64.zip", "dst": "flutter_infra_release/flutter/$engine/android-arm64-release/windows-x64.zip"} + ] + }, + "android-arm32": { + "steps": [ + { + "type": "gn_ninja", + "gn_args": ["--android", "--runtime-mode=release"], + "ninja_targets": ["archive_win_gen_snapshot"], + "out_dir": "android_release" + } + ], + "artifacts": [ + {"src": "android_release/zip_archives/android-arm-release/windows-x64.zip", "dst": "flutter_infra_release/flutter/$engine/android-arm-release/windows-x64.zip"} + ] + }, + "android-x64": { + "steps": [ + { + "type": "gn_ninja", + "gn_args": ["--android", "--android-cpu=x64", "--runtime-mode=release"], + "ninja_targets": ["archive_win_gen_snapshot"], + "out_dir": "android_release_x64" + } + ], + "artifacts": [ + {"src": "android_release_x64/zip_archives/android-x64-release/windows-x64.zip", "dst": "flutter_infra_release/flutter/$engine/android-x64-release/windows-x64.zip"} + ] + }, + "host-release": { + "steps": [ + { + "type": "rust", + "targets": ["x86_64-pc-windows-msvc"] + }, + { + "type": "gn_ninja", + "gn_args": ["--runtime-mode=release", "--no-prebuilt-dart-sdk"], + "ninja_targets": ["dart_sdk", "flutter/build/archives:windows_flutter", "gen_snapshot", "windows", "flutter/build/archives:artifacts"], + "out_dir": "host_release" + } + ], + "artifacts": [ + {"src": "host_release/dart-sdk", "dst": "flutter_infra_release/flutter/$engine/dart-sdk-windows-x64.zip", "zip": true, "content_hash": true}, + {"src": "host_release/zip_archives/windows-x64-release/windows-x64-flutter.zip", "dst": "flutter_infra_release/flutter/$engine/windows-x64-release/windows-x64-flutter.zip"} + ] + }, + "host-debug": { + "steps": [ + { + "type": "rust", + "targets": ["x86_64-pc-windows-msvc"] + }, + { + "type": "gn_ninja", + "gn_args": ["--no-prebuilt-dart-sdk"], + "ninja_targets": ["flutter/build/archives:artifacts"], + "out_dir": "host_debug" + } + ], + "artifacts": [ + {"src": "host_debug/zip_archives/windows-x64/artifacts.zip", "dst": "flutter_infra_release/flutter/$engine/windows-x64/artifacts.zip"} + ] + } +} diff --git a/shorebird/ci/win_build_and_upload.sh b/shorebird/ci/win_build_and_upload.sh new file mode 100755 index 0000000000000..bbda403ff2c67 --- /dev/null +++ b/shorebird/ci/win_build_and_upload.sh @@ -0,0 +1,32 @@ +#!/bin/bash -e + +# Usage: +# ./win_build_and_upload.sh flutter_root engine_hash +# +# This is the main entrypoint for building and uploading Windows engine artifacts. +# It is called from the _build_engine repository's CI scripts. + +if [ "$#" -ne 2 ]; then + echo "Usage: $0 flutter_root engine_hash" + exit 1 +fi + +FLUTTER_ROOT=$1 +ENGINE_HASH=$2 +ENGINE_ROOT=$FLUTTER_ROOT/engine + +# Get the absolute path to the directory of this script. +SCRIPT_DIR=$(cd $(dirname $0) && pwd) + +echo "Building engine at $ENGINE_ROOT and uploading to gs://download.shorebird.dev" + +cd $SCRIPT_DIR + +# Run the setup script. +./internal/win_setup.sh + +# Then run the build. +./internal/win_build.sh $ENGINE_ROOT + +# Copy Shorebird engine artifacts to Google Cloud Storage. +./internal/win_upload.sh $ENGINE_ROOT $ENGINE_HASH diff --git a/shorebird/docs/BUILDING.md b/shorebird/docs/BUILDING.md new file mode 100644 index 0000000000000..dfd3c655a1324 --- /dev/null +++ b/shorebird/docs/BUILDING.md @@ -0,0 +1,44 @@ +# Building the Shorebird Engine Locally + +This document explains how to build the Shorebird engine locally for different platforms. + +All commands assume you are running from the `engine/src` directory. + +## Prerequisites + +- Follow the standard Flutter engine setup instructions +- Ensure you have the necessary toolchains installed for your target platform + +## macOS + +### iOS (arm64) + +```bash +./flutter/tools/gn --no-rbe --no-enable-unittests --runtime-mode=release --ios --gn-arg='shorebird_runtime=true' +ninja -C out/ios_release flutter/shell/platform/darwin/ios:flutter_framework flutter/lib/snapshot:generate_snapshot_bins +``` + +### Android arm64 + +```bash +./flutter/tools/gn --no-rbe --no-enable-unittests --android --android-cpu=arm64 --runtime-mode=release --gn-args='host_cpu="x64"' +ninja -C out/android_release_arm64 flutter/shell/platform/android:gen_snapshot +``` + +## Windows + +### Android arm64 + +```bash +./flutter/tools/gn --no-rbe --no-enable-unittests --android --android-cpu=arm64 --runtime-mode=release +ninja -C out/android_release_arm64 archive_win_gen_snapshot +``` + +## Linux + +### Android arm64 + +```bash +./flutter/tools/gn --no-rbe --no-enable-unittests --android --android-cpu=arm64 --runtime-mode=release +ninja -C out/android_release_arm64 default gen_snapshot +```