From 990a1e43add22e0e1cc723ddaa76af254d862432 Mon Sep 17 00:00:00 2001 From: Xiangzhe Date: Sun, 21 Jun 2026 23:35:33 +0800 Subject: [PATCH 01/52] feat!: rewrite getter as package-centric CLI core Replace the old hub-app centered implementation with the reusable rewrite spine for the UpgradeAll architecture. Add split core/storage/CLI crates, Lua repository evaluation, SQLite main/cache storage, repository overlay behavior, and legacy JSON bridge-bundle import coverage. --- Cargo.lock | 2555 +++++------------ Cargo.toml | 65 +- README.md | 26 +- crates/getter-cli/Cargo.toml | 21 + crates/getter-cli/src/lib.rs | 817 ++++++ crates/getter-cli/src/main.rs | 10 + crates/getter-cli/tests/bdd_cli.rs | 486 ++++ .../tests/features/cli/app_list.feature | 8 + .../tests/features/cli/hub_list.feature | 8 + .../tests/features/cli/init.feature | 8 + .../legacy_import_room_bundle_failure.feature | 18 + .../tests/features/cli/repo_list.feature | 8 + .../features/cli/repository_overlay.feature | 14 + .../cli/repository_package_eval.feature | 17 + .../features/cli/storage_validate.feature | 8 + crates/getter-core/Cargo.toml | 14 + crates/getter-core/src/lib.rs | 408 +++ crates/getter-core/src/lua.rs | 460 +++ crates/getter-core/src/repository.rs | 288 ++ crates/getter-downloader/Cargo.toml | 7 + crates/getter-downloader/src/lib.rs | 3 + crates/getter-ffi/Cargo.toml | 7 + crates/getter-ffi/src/lib.rs | 3 + crates/getter-plugin-api/Cargo.toml | 7 + crates/getter-plugin-api/src/lib.rs | 3 + crates/getter-providers/Cargo.toml | 7 + crates/getter-providers/src/lib.rs | 3 + crates/getter-rpc/Cargo.toml | 7 + crates/getter-rpc/src/lib.rs | 3 + crates/getter-storage/Cargo.toml | 12 + crates/getter-storage/src/legacy_room.rs | 215 ++ crates/getter-storage/src/lib.rs | 491 ++++ src/cache.rs | 34 - src/cache/local.rs | 77 - src/cache/manager.rs | 265 -- src/cache/moka.rs | 0 src/core.rs | 1 - src/core/config.rs | 3 - src/core/config/data.rs | 1 - src/core/config/data/rule_list.rs | 89 - src/core/config/utils.rs | 41 - src/core/config/world.rs | 35 - src/core/config/world/local_repo.rs | 31 - src/core/config/world/world_config_wrapper.rs | 71 - src/core/config/world/world_list.rs | 100 - src/database/mod.rs | 229 -- src/database/models/app.rs | 101 - src/database/models/extra_app.rs | 48 - src/database/models/extra_hub.rs | 56 - src/database/models/hub.rs | 80 - src/database/models/mod.rs | 4 - src/database/store.rs | 266 -- src/downloader/config.rs | 199 -- src/downloader/error.rs | 95 - src/downloader/external_rpc_impl.rs | 290 -- src/downloader/hub_dispatch.rs | 208 -- src/downloader/mod.rs | 58 - src/downloader/state.rs | 532 ---- src/downloader/task_manager.rs | 1009 ------- src/downloader/traits.rs | 262 -- src/downloader/trauma_impl.rs | 751 ----- src/error.rs | 82 - src/lib.rs | 76 +- src/locale.rs | 64 - src/main.rs | 16 +- src/manager/android_api.rs | 104 - src/manager/app_manager.rs | 462 --- src/manager/app_status.rs | 18 - src/manager/auto_template.rs | 206 -- src/manager/cloud_config_getter.rs | 403 --- src/manager/data_getter.rs | 99 - src/manager/hub_manager.rs | 173 -- src/manager/mod.rs | 12 - src/manager/notification.rs | 152 - src/manager/updater.rs | 151 - src/manager/url_replace.rs | 170 -- src/manager/version.rs | 136 - src/manager/version_map.rs | 192 -- src/rpc.rs | 3 - src/rpc/client.rs | 77 - src/rpc/data.rs | 411 --- src/rpc/server.rs | 1255 -------- src/utils.rs | 6 - src/utils/convert.rs | 21 - src/utils/http.rs | 327 --- src/utils/instance.rs | 18 - src/utils/json.rs | 31 - src/utils/time.rs | 8 - src/utils/versioning.rs | 170 -- src/websdk.rs | 2 - src/websdk/cloud_rules.rs | 3 - src/websdk/cloud_rules/cloud_rules_manager.rs | 112 - src/websdk/cloud_rules/cloud_rules_wrapper.rs | 109 - src/websdk/cloud_rules/data.rs | 3 - src/websdk/cloud_rules/data/app_item.rs | 94 - src/websdk/cloud_rules/data/config_list.rs | 80 - src/websdk/cloud_rules/data/hub_item.rs | 123 - src/websdk/repo.rs | 3 - src/websdk/repo/api.rs | 166 -- src/websdk/repo/controller.rs | 121 - src/websdk/repo/data.rs | 1 - src/websdk/repo/data/release.rs | 17 - src/websdk/repo/provider.rs | 99 - src/websdk/repo/provider/base_provider.rs | 427 --- src/websdk/repo/provider/fdroid.rs | 356 --- src/websdk/repo/provider/github.rs | 262 -- src/websdk/repo/provider/gitlab.rs | 327 --- src/websdk/repo/provider/lsposed_repo.rs | 224 -- src/websdk/repo/provider/outside_rpc.rs | 115 - 109 files changed, 4237 insertions(+), 14193 deletions(-) create mode 100644 crates/getter-cli/Cargo.toml create mode 100644 crates/getter-cli/src/lib.rs create mode 100644 crates/getter-cli/src/main.rs create mode 100644 crates/getter-cli/tests/bdd_cli.rs create mode 100644 crates/getter-cli/tests/features/cli/app_list.feature create mode 100644 crates/getter-cli/tests/features/cli/hub_list.feature create mode 100644 crates/getter-cli/tests/features/cli/init.feature create mode 100644 crates/getter-cli/tests/features/cli/legacy_import_room_bundle_failure.feature create mode 100644 crates/getter-cli/tests/features/cli/repo_list.feature create mode 100644 crates/getter-cli/tests/features/cli/repository_overlay.feature create mode 100644 crates/getter-cli/tests/features/cli/repository_package_eval.feature create mode 100644 crates/getter-cli/tests/features/cli/storage_validate.feature create mode 100644 crates/getter-core/Cargo.toml create mode 100644 crates/getter-core/src/lib.rs create mode 100644 crates/getter-core/src/lua.rs create mode 100644 crates/getter-core/src/repository.rs create mode 100644 crates/getter-downloader/Cargo.toml create mode 100644 crates/getter-downloader/src/lib.rs create mode 100644 crates/getter-ffi/Cargo.toml create mode 100644 crates/getter-ffi/src/lib.rs create mode 100644 crates/getter-plugin-api/Cargo.toml create mode 100644 crates/getter-plugin-api/src/lib.rs create mode 100644 crates/getter-providers/Cargo.toml create mode 100644 crates/getter-providers/src/lib.rs create mode 100644 crates/getter-rpc/Cargo.toml create mode 100644 crates/getter-rpc/src/lib.rs create mode 100644 crates/getter-storage/Cargo.toml create mode 100644 crates/getter-storage/src/legacy_room.rs create mode 100644 crates/getter-storage/src/lib.rs delete mode 100644 src/cache.rs delete mode 100644 src/cache/local.rs delete mode 100644 src/cache/manager.rs delete mode 100644 src/cache/moka.rs delete mode 100644 src/core.rs delete mode 100644 src/core/config.rs delete mode 100644 src/core/config/data.rs delete mode 100644 src/core/config/data/rule_list.rs delete mode 100644 src/core/config/utils.rs delete mode 100644 src/core/config/world.rs delete mode 100644 src/core/config/world/local_repo.rs delete mode 100644 src/core/config/world/world_config_wrapper.rs delete mode 100644 src/core/config/world/world_list.rs delete mode 100644 src/database/mod.rs delete mode 100644 src/database/models/app.rs delete mode 100644 src/database/models/extra_app.rs delete mode 100644 src/database/models/extra_hub.rs delete mode 100644 src/database/models/hub.rs delete mode 100644 src/database/models/mod.rs delete mode 100644 src/database/store.rs delete mode 100644 src/downloader/config.rs delete mode 100644 src/downloader/error.rs delete mode 100644 src/downloader/external_rpc_impl.rs delete mode 100644 src/downloader/hub_dispatch.rs delete mode 100644 src/downloader/mod.rs delete mode 100644 src/downloader/state.rs delete mode 100644 src/downloader/task_manager.rs delete mode 100644 src/downloader/traits.rs delete mode 100644 src/downloader/trauma_impl.rs delete mode 100644 src/error.rs delete mode 100644 src/locale.rs delete mode 100644 src/manager/android_api.rs delete mode 100644 src/manager/app_manager.rs delete mode 100644 src/manager/app_status.rs delete mode 100644 src/manager/auto_template.rs delete mode 100644 src/manager/cloud_config_getter.rs delete mode 100644 src/manager/data_getter.rs delete mode 100644 src/manager/hub_manager.rs delete mode 100644 src/manager/mod.rs delete mode 100644 src/manager/notification.rs delete mode 100644 src/manager/updater.rs delete mode 100644 src/manager/url_replace.rs delete mode 100644 src/manager/version.rs delete mode 100644 src/manager/version_map.rs delete mode 100644 src/rpc.rs delete mode 100644 src/rpc/client.rs delete mode 100644 src/rpc/data.rs delete mode 100644 src/rpc/server.rs delete mode 100644 src/utils.rs delete mode 100644 src/utils/convert.rs delete mode 100644 src/utils/http.rs delete mode 100644 src/utils/instance.rs delete mode 100644 src/utils/json.rs delete mode 100644 src/utils/time.rs delete mode 100644 src/utils/versioning.rs delete mode 100644 src/websdk.rs delete mode 100644 src/websdk/cloud_rules.rs delete mode 100644 src/websdk/cloud_rules/cloud_rules_manager.rs delete mode 100644 src/websdk/cloud_rules/cloud_rules_wrapper.rs delete mode 100644 src/websdk/cloud_rules/data.rs delete mode 100644 src/websdk/cloud_rules/data/app_item.rs delete mode 100644 src/websdk/cloud_rules/data/config_list.rs delete mode 100644 src/websdk/cloud_rules/data/hub_item.rs delete mode 100644 src/websdk/repo.rs delete mode 100644 src/websdk/repo/api.rs delete mode 100644 src/websdk/repo/controller.rs delete mode 100644 src/websdk/repo/data.rs delete mode 100644 src/websdk/repo/data/release.rs delete mode 100644 src/websdk/repo/provider.rs delete mode 100644 src/websdk/repo/provider/base_provider.rs delete mode 100644 src/websdk/repo/provider/fdroid.rs delete mode 100644 src/websdk/repo/provider/github.rs delete mode 100644 src/websdk/repo/provider/gitlab.rs delete mode 100644 src/websdk/repo/provider/lsposed_repo.rs delete mode 100644 src/websdk/repo/provider/outside_rpc.rs diff --git a/Cargo.lock b/Cargo.lock index 0b56aaf..5d73f4c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,6 +2,18 @@ # It is not intended for manual editing. version = 4 +[[package]] +name = "ahash" +version = "0.8.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" +dependencies = [ + "cfg-if", + "once_cell", + "version_check", + "zerocopy", +] + [[package]] name = "aho-corasick" version = "1.1.4" @@ -12,131 +24,102 @@ dependencies = [ ] [[package]] -name = "anyhow" -version = "1.0.101" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f0e0fee31ef5ed1ba1316088939cea399010ed7731dba877ed44aeb407a75ea" - -[[package]] -name = "assert-json-diff" -version = "2.0.2" +name = "anstream" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "47e4f2b81832e72834d7518d8487a0396a28cc408186a2e8854c0f98011faf12" +checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d" dependencies = [ - "serde", - "serde_json", + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", ] [[package]] -name = "async-trait" -version = "0.1.89" +name = "anstyle" +version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] +checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" [[package]] -name = "async_fn_traits" -version = "0.1.1" +name = "anstyle-parse" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc58d489c5f2d2c5be31b9004cec7af25a70d23df4d8111715eee736234cf217" +checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e" dependencies = [ - "paste", + "utf8parse", ] [[package]] -name = "atomic-waker" -version = "1.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" - -[[package]] -name = "aws-lc-rs" -version = "1.15.2" +name = "anstyle-query" +version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a88aab2464f1f25453baa7a07c84c5b7684e274054ba06817f382357f77a288" +checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" dependencies = [ - "aws-lc-sys", - "zeroize", + "windows-sys 0.61.2", ] [[package]] -name = "aws-lc-sys" -version = "0.35.0" +name = "anstyle-wincon" +version = "3.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b45afffdee1e7c9126814751f88dddc747f41d91da16c9551a0f1e8a11e788a1" +checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" dependencies = [ - "cc", - "cmake", - "dunce", - "fs_extra", + "anstyle", + "once_cell_polyfill", + "windows-sys 0.61.2", ] [[package]] -name = "base64" -version = "0.22.1" +name = "anyhow" +version = "1.0.102" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" [[package]] -name = "bindgen" -version = "0.72.1" +name = "autocfg" +version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "993776b509cfb49c750f11b8f07a46fa23e0a1386ffc01fb1e7d343efc387895" -dependencies = [ - "bitflags", - "cexpr", - "clang-sys", - "itertools", - "log", - "prettyplease", - "proc-macro2", - "quote", - "regex", - "rustc-hash", - "shlex", - "syn", -] +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" [[package]] name = "bitflags" -version = "2.10.0" +version = "2.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "812e12b5285cc515a9c72a5c1d3b6d46a19dac5acfef5265968c166106e31dd3" +checksum = "b4388bee8683e3d04af747c73422af53102d2bd24d9eadb6cbc100baef4b43f8" [[package]] -name = "block-buffer" -version = "0.10.4" +name = "bstr" +version = "1.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +checksum = "63044e1ae8e69f3b5a92c736ca6269b8d12fa7efe39bf34ddb06d102cf0e2cab" dependencies = [ - "generic-array", + "memchr", + "serde", ] [[package]] -name = "bumpalo" -version = "3.19.1" +name = "bytecount" +version = "0.6.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5dd9dc738b7a8311c7ade152424974d8115f2cdad61e8dab8dac9f2362298510" +checksum = "175812e0be2bccb6abe50bb8d566126198344f707e304f45c648fd8f2cc0365e" [[package]] name = "bytes" -version = "1.11.1" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33" +checksum = "8ae3f5d315924270530207e2a68396c3cc547f6dca3fbdca317cfb1a51edb593" [[package]] name = "cc" -version = "1.2.52" +version = "1.2.65" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd4932aefd12402b36c60956a4fe0035421f544799057659ff86f923657aada3" +checksum = "e228eec9be7c17ccb640b59b36a5cd805ea2a564a4c5e162c2f659fea30d3b96" dependencies = [ "find-msvc-tools", - "jobserver", - "libc", "shlex", ] @@ -146,15 +129,6 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6d43a04d8753f35258c91f8ec639f792891f748a1edbd759cf1dcea3382ad83c" -[[package]] -name = "cexpr" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766" -dependencies = [ - "nom", -] - [[package]] name = "cfg-if" version = "1.0.4" @@ -162,50 +136,51 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" [[package]] -name = "cfg_aliases" -version = "0.2.1" +name = "clap" +version = "4.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" +checksum = "1ddb117e43bbf7dacf0a4190fef4d345b9bad68dfc649cb349e7d17d28428e51" +dependencies = [ + "clap_builder", + "clap_derive", +] [[package]] -name = "chacha20" -version = "0.10.0" +name = "clap_builder" +version = "4.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f8d983286843e49675a4b7a2d174efe136dc93a18d69130dd18198a6c167601" +checksum = "714a53001bf66416adb0e2ef5ac857140e7dc3a0c48fb28b2f10762fc4b5069f" dependencies = [ - "cfg-if", - "cpufeatures 0.3.0", - "rand_core 0.10.0", + "anstream", + "anstyle", + "clap_lex", + "strsim", + "terminal_size", ] [[package]] -name = "clang-sys" -version = "1.8.1" +name = "clap_derive" +version = "4.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b023947811758c97c59bf9d1c188fd619ad4718dcaa767947df1cadb14f39f4" +checksum = "f2ce8604710f6733aa641a2b3731eaa1e8b3d9973d5e3565da11800813f997a9" dependencies = [ - "glob", - "libc", - "libloading", + "heck", + "proc-macro2", + "quote", + "syn", ] [[package]] -name = "cmake" -version = "0.1.57" +name = "clap_lex" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75443c44cd6b379beb8c5b45d85d0773baf31cce901fe7bb252f4eff3008ef7d" -dependencies = [ - "cc", -] +checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" [[package]] -name = "colored" -version = "3.0.0" +name = "colorchoice" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fde0e0ec90c9dfb3b4b1a0891a7dcd0e2bffde2f7efed5fe7c9bb00e5bfb915e" -dependencies = [ - "windows-sys 0.52.0", -] +checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" [[package]] name = "combine" @@ -217,6 +192,27 @@ dependencies = [ "memchr", ] +[[package]] +name = "console" +version = "0.16.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d64e8af5551369d19cf50138de61f1c42074ab970f74e99be916646777f8fc87" +dependencies = [ + "encode_unicode", + "libc", + "unicode-width", + "windows-sys 0.61.2", +] + +[[package]] +name = "convert_case" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "633458d4ef8c78b72454de2d54fd6ab2e60f9e02be22f3c6104cdc8a4e0fceb9" +dependencies = [ + "unicode-segmentation", +] + [[package]] name = "core-foundation" version = "0.10.1" @@ -234,138 +230,167 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" [[package]] -name = "cpufeatures" -version = "0.2.17" +name = "crossbeam-deque" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51" dependencies = [ - "libc", + "crossbeam-epoch", + "crossbeam-utils", ] [[package]] -name = "cpufeatures" -version = "0.3.0" +name = "crossbeam-epoch" +version = "0.9.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" dependencies = [ - "libc", + "crossbeam-utils", ] [[package]] -name = "crypto-common" -version = "0.1.7" +name = "crossbeam-utils" +version = "0.8.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" -dependencies = [ - "generic-array", - "typenum", -] +checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" [[package]] -name = "digest" -version = "0.10.7" +name = "cucumber" +version = "0.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +checksum = "96a87e18d925b19ebe0fd47ea45316abd216d81ec0879c2448c3f9a0e9da62be" dependencies = [ - "block-buffer", - "crypto-common", + "anyhow", + "clap", + "console", + "cucumber-codegen", + "cucumber-expressions", + "derive_more", + "either", + "futures", + "gherkin", + "globwalk", + "humantime", + "inventory", + "itertools", + "linked-hash-map", + "pin-project", + "ref-cast", + "regex", + "sealed", + "smart-default", ] [[package]] -name = "displaydoc" -version = "0.2.5" +name = "cucumber-codegen" +version = "0.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" +checksum = "ed2fc8a8bbb73af3230db699e8690c5c786655f75eb89e5f18d76055fa1a9a4d" dependencies = [ + "cucumber-expressions", + "inflections", + "itertools", "proc-macro2", "quote", + "regex", "syn", + "synthez", ] [[package]] -name = "dunce" -version = "1.0.5" +name = "cucumber-expressions" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" +checksum = "6401038de3af44fe74e6fccdb8a5b7db7ba418f480c8e9ad584c6f65c05a27a6" +dependencies = [ + "derive_more", + "either", + "nom", + "nom_locate", + "regex", + "regex-syntax", +] [[package]] -name = "either" -version = "1.15.0" +name = "derive_more" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" +checksum = "d751e9e49156b02b44f9c1815bcb94b984cdcc4396ecc32521c739452808b134" +dependencies = [ + "derive_more-impl", +] [[package]] -name = "encoding_rs" -version = "0.8.35" +name = "derive_more-impl" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3" +checksum = "799a97264921d8623a957f6c3b9011f3b5492f557bbb7a5a19b7fa6d06ba8dcb" dependencies = [ - "cfg-if", + "convert_case", + "proc-macro2", + "quote", + "rustc_version", + "syn", + "unicode-xid", ] [[package]] -name = "equivalent" -version = "1.0.2" +name = "either" +version = "1.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" +checksum = "91622ff5e7162018101f2fea40d6ebf4a78bbe5a49736a2020649edf9693679e" [[package]] -name = "errno" -version = "0.3.14" +name = "encode_unicode" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" -dependencies = [ - "libc", - "windows-sys 0.60.2", -] +checksum = "34aa73646ffb006b8f5147f3dc182bd4bcb190227ce861fc4a4844bf8e3cb2c0" [[package]] -name = "fastrand" -version = "2.3.0" +name = "env_home" +version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" +checksum = "c7f84e12ccf0a7ddc17a6c41c93326024c42920d7ee630d04950e6926645c0fe" [[package]] -name = "file-locker" -version = "1.1.4" +name = "equivalent" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75ae8b5984a4863d8a32109a848d038bd6d914f20f010cc141375f7a183c41cf" -dependencies = [ - "nix", -] +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" [[package]] -name = "find-msvc-tools" -version = "0.1.7" +name = "errno" +version = "0.3.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f449e6c6c08c865631d4890cfacf252b3d396c9bcc83adb6623cdb02a8336c41" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] [[package]] -name = "fnv" -version = "1.0.7" +name = "fallible-iterator" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +checksum = "2acce4a10f12dc2fb14a218589d4f1f62ef011b2d0cc4b3cb1bba8e94da14649" [[package]] -name = "foldhash" -version = "0.1.5" +name = "fallible-streaming-iterator" +version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" +checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a" [[package]] -name = "form_urlencoded" -version = "1.2.2" +name = "fastrand" +version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" -dependencies = [ - "percent-encoding", -] +checksum = "9f1f227452a390804cdb637b74a86990f2a7d7ba4b7d5693aac9b4dd6defd8d6" [[package]] -name = "fs_extra" -version = "1.3.0" +name = "find-msvc-tools" +version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" +checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" [[package]] name = "futures" @@ -438,16 +463,6 @@ version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393" -[[package]] -name = "futures-timer" -version = "3.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f288b0a4f20f9a56b5d1da57e2227c661b7b16168e2f72365f57b63326e29b24" -dependencies = [ - "gloo-timers", - "send_wrapper", -] - [[package]] name = "futures-util" version = "0.3.32" @@ -465,446 +480,249 @@ dependencies = [ "slab", ] -[[package]] -name = "generic-array" -version = "0.14.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" -dependencies = [ - "typenum", - "version_check", -] - [[package]] name = "getrandom" -version = "0.2.16" +version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "335ff9f135e4384c8150d6f27c6daed433577f86b4750418338c01a1a2528592" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" dependencies = [ "cfg-if", - "js-sys", "libc", "wasi", - "wasm-bindgen", ] [[package]] name = "getrandom" -version = "0.3.4" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" dependencies = [ "cfg-if", - "js-sys", "libc", "r-efi", - "wasip2", - "wasm-bindgen", ] [[package]] -name = "getrandom" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "139ef39800118c7683f2fd3c98c1b23c09ae076556b435f8e9064ae108aaeeec" +name = "getter" +version = "0.1.0" dependencies = [ - "cfg-if", - "libc", - "r-efi", - "rand_core 0.10.0", - "wasip2", - "wasip3", + "getter-cli", + "getter-core", + "getter-storage", + "rustls-platform-verifier", + "thiserror 1.0.69", + "tokio", ] [[package]] -name = "getter" +name = "getter-cli" version = "0.1.0" dependencies = [ - "async-trait", - "async_fn_traits", - "bytes", - "file-locker", - "futures", - "http-body-util", - "hyper", - "hyper-rustls", - "hyper-util", - "jsonl", - "jsonrpsee", - "libversion-sys", - "markdown", - "mockito", - "once_cell", - "parking_lot", - "quick-xml", - "rand 0.10.0", - "regex", - "reqwest", - "rustls", - "rustls-platform-verifier 0.6.2", + "cucumber", + "getter-core", + "getter-storage", "serde", "serde_json", - "serial_test", "tempfile", + "thiserror 1.0.69", "tokio", - "tokio-util", - "url", - "uuid", ] [[package]] -name = "glob" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280" - -[[package]] -name = "gloo-net" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c06f627b1a58ca3d42b45d6104bf1e1a03799df472df00988b6ba21accc10580" +name = "getter-core" +version = "0.1.0" dependencies = [ - "futures-channel", - "futures-core", - "futures-sink", - "gloo-utils", - "http", - "js-sys", - "pin-project", + "mlua", "serde", "serde_json", + "tempfile", "thiserror 1.0.69", - "wasm-bindgen", - "wasm-bindgen-futures", - "web-sys", + "toml", ] [[package]] -name = "gloo-timers" -version = "0.2.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b995a66bb87bebce9a0f4a95aed01daca4872c050bfcb21653361c03bc35e5c" +name = "getter-downloader" +version = "0.1.0" dependencies = [ - "futures-channel", - "futures-core", - "js-sys", - "wasm-bindgen", + "getter-core", ] [[package]] -name = "gloo-utils" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b5555354113b18c547c1d3a98fbf7fb32a9ff4f6fa112ce823a21641a0ba3aa" +name = "getter-ffi" +version = "0.1.0" dependencies = [ - "js-sys", - "serde", - "serde_json", - "wasm-bindgen", - "web-sys", + "getter-core", ] [[package]] -name = "h2" -version = "0.4.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f44da3a8150a6703ed5d34e164b875fd14c2cdab9af1252a9a1020bde2bdc54" +name = "getter-plugin-api" +version = "0.1.0" dependencies = [ - "atomic-waker", - "bytes", - "fnv", - "futures-core", - "futures-sink", - "http", - "indexmap", - "slab", - "tokio", - "tokio-util", - "tracing", + "getter-core", ] [[package]] -name = "hashbrown" -version = "0.15.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" +name = "getter-providers" +version = "0.1.0" dependencies = [ - "foldhash", + "getter-core", ] [[package]] -name = "hashbrown" -version = "0.16.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" +name = "getter-rpc" +version = "0.1.0" +dependencies = [ + "getter-core", +] [[package]] -name = "heck" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" +name = "getter-storage" +version = "0.1.0" +dependencies = [ + "getter-core", + "rusqlite", + "tempfile", + "thiserror 1.0.69", +] [[package]] -name = "http" -version = "1.4.0" +name = "gherkin" +version = "0.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3ba2a386d7f85a81f119ad7498ebe444d2e22c2af0b86b069416ace48b3311a" +checksum = "9e2c0d8c632f8a251ce9a8198079b1022adc586ff4e3d33e18debd40eb463b31" dependencies = [ - "bytes", - "itoa", + "heck", + "peg", + "quote", + "serde", + "serde_json", + "syn", + "textwrap", + "thiserror 2.0.18", + "typed-builder", ] [[package]] -name = "http-body" -version = "1.0.1" +name = "globset" +version = "0.4.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" +checksum = "52dfc19153a48bde0cbd630453615c8151bce3a5adfac7a0aebfbf0a1e1f57e3" dependencies = [ - "bytes", - "http", + "aho-corasick", + "bstr", + "log", + "regex-automata", + "regex-syntax", ] [[package]] -name = "http-body-util" -version = "0.1.3" +name = "globwalk" +version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a" +checksum = "0bf760ebf69878d9fd8f110c89703d90ce35095324d1f1edcb595c63945ee757" dependencies = [ - "bytes", - "futures-core", - "http", - "http-body", - "pin-project-lite", + "bitflags", + "ignore", + "walkdir", ] [[package]] -name = "httparse" -version = "1.10.1" +name = "hashbrown" +version = "0.14.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" +checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" +dependencies = [ + "ahash", +] [[package]] -name = "httpdate" -version = "1.0.3" +name = "hashbrown" +version = "0.17.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" [[package]] -name = "hyper" -version = "1.8.1" +name = "hashlink" +version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ab2d4f250c3d7b1c9fcdff1cece94ea4e2dfbec68614f7b87cb205f24ca9d11" +checksum = "6ba4ff7128dee98c7dc9794b6a411377e1404dba1c97deb8d1a55297bd25d8af" dependencies = [ - "atomic-waker", - "bytes", - "futures-channel", - "futures-core", - "h2", - "http", - "http-body", - "httparse", - "httpdate", - "itoa", - "pin-project-lite", - "pin-utils", - "smallvec", - "tokio", - "want", + "hashbrown 0.14.5", ] [[package]] -name = "hyper-rustls" -version = "0.27.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3c93eb611681b207e1fe55d5a71ecf91572ec8a6705cdb6857f7d8d5242cf58" -dependencies = [ - "http", - "hyper", - "hyper-util", - "log", - "rustls", - "rustls-native-certs", - "rustls-pki-types", - "rustls-platform-verifier 0.6.2", - "tokio", - "tokio-rustls", - "tower-service", - "webpki-roots", -] - -[[package]] -name = "hyper-util" -version = "0.1.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" -dependencies = [ - "base64", - "bytes", - "futures-channel", - "futures-util", - "http", - "http-body", - "hyper", - "ipnet", - "libc", - "percent-encoding", - "pin-project-lite", - "socket2", - "tokio", - "tower-service", - "tracing", -] - -[[package]] -name = "icu_collections" -version = "2.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c6b649701667bbe825c3b7e6388cb521c23d88644678e83c0c4d0a621a34b43" -dependencies = [ - "displaydoc", - "potential_utf", - "yoke", - "zerofrom", - "zerovec", -] - -[[package]] -name = "icu_locale_core" -version = "2.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "edba7861004dd3714265b4db54a3c390e880ab658fec5f7db895fae2046b5bb6" -dependencies = [ - "displaydoc", - "litemap", - "tinystr", - "writeable", - "zerovec", -] - -[[package]] -name = "icu_normalizer" -version = "2.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f6c8828b67bf8908d82127b2054ea1b4427ff0230ee9141c54251934ab1b599" -dependencies = [ - "icu_collections", - "icu_normalizer_data", - "icu_properties", - "icu_provider", - "smallvec", - "zerovec", -] - -[[package]] -name = "icu_normalizer_data" -version = "2.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7aedcccd01fc5fe81e6b489c15b247b8b0690feb23304303a9e560f37efc560a" - -[[package]] -name = "icu_properties" -version = "2.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "020bfc02fe870ec3a66d93e677ccca0562506e5872c650f893269e08615d74ec" -dependencies = [ - "icu_collections", - "icu_locale_core", - "icu_properties_data", - "icu_provider", - "zerotrie", - "zerovec", -] - -[[package]] -name = "icu_properties_data" -version = "2.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "616c294cf8d725c6afcd8f55abc17c56464ef6211f9ed59cccffe534129c77af" - -[[package]] -name = "icu_provider" -version = "2.1.1" +name = "heck" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85962cf0ce02e1e0a629cc34e7ca3e373ce20dda4c4d7294bbd0bf1fdb59e614" -dependencies = [ - "displaydoc", - "icu_locale_core", - "writeable", - "yoke", - "zerofrom", - "zerotrie", - "zerovec", -] +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" [[package]] -name = "id-arena" +name = "humantime" version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d3067d79b975e8844ca9eb072e16b31c3c1c36928edf9c6789548c524d0d954" - -[[package]] -name = "idna" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" -dependencies = [ - "idna_adapter", - "smallvec", - "utf8_iter", -] +checksum = "135b12329e5e3ce057a9f972339ea52bc954fe1e9358ef27f95e89716fbc5424" [[package]] -name = "idna_adapter" -version = "1.2.1" +name = "ignore" +version = "0.4.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3acae9609540aa318d1bc588455225fb2085b9ed0c4f6bd0d9d5bcd86f1a0344" +checksum = "b915661dd01db3f05050265b2477bcc6527b3792388e2749b41623cc592be67d" dependencies = [ - "icu_normalizer", - "icu_properties", + "crossbeam-deque", + "globset", + "log", + "memchr", + "regex-automata", + "same-file", + "walkdir", + "winapi-util", ] [[package]] name = "indexmap" -version = "2.13.0" +version = "2.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7714e70437a7dc3ac8eb7e6f8df75fd8eb422675fc7678aff7364301092b1017" +checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" dependencies = [ "equivalent", - "hashbrown 0.16.1", - "serde", - "serde_core", + "hashbrown 0.17.1", ] [[package]] -name = "ipnet" -version = "2.11.0" +name = "inflections" +version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "469fb0b9cefa57e3ef31275ee7cacb78f2fdca44e4765491884a2b119d4eb130" +checksum = "a257582fdcde896fd96463bf2d40eefea0580021c0712a0e2b028b60b47a837a" [[package]] -name = "iri-string" -version = "0.7.10" +name = "inventory" +version = "0.3.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c91338f0783edbd6195decb37bae672fd3b165faffb89bf7b9e6942f8b1a731a" +checksum = "a4f0c30c76f2f4ccee3fe55a2435f691ca00c0e4bd87abe4f4a851b1d4dac39b" dependencies = [ - "memchr", - "serde", + "rustversion", ] +[[package]] +name = "is_terminal_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" + [[package]] name = "itertools" -version = "0.13.0" +version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" +checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" dependencies = [ "either", ] [[package]] name = "itoa" -version = "1.0.17" +version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92ecc6618181def0457392ccd0ee51198e065e016d1d527a7ac1b6dc7c1f09d2" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" [[package]] name = "jni" @@ -915,7 +733,7 @@ dependencies = [ "cesu8", "cfg-if", "combine", - "jni-sys", + "jni-sys 0.3.1", "log", "thiserror 1.0.69", "walkdir", @@ -924,229 +742,54 @@ dependencies = [ [[package]] name = "jni-sys" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8eaf4bc02d17cbdd7ff4c7438cafcdf7fb9a4613313ad11b4f8fefe7d3fa0130" - -[[package]] -name = "jobserver" -version = "0.1.34" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9afb3de4395d6b3e67a780b6de64b51c978ecf11cb9a462c66be7d4ca9039d33" -dependencies = [ - "getrandom 0.3.4", - "libc", -] - -[[package]] -name = "js-sys" -version = "0.3.85" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c942ebf8e95485ca0d52d97da7c5a2c387d0e7f0ba4c35e93bfcaee045955b3" -dependencies = [ - "once_cell", - "wasm-bindgen", -] - -[[package]] -name = "jsonl" -version = "4.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1abae98f45234fc1980c198798166a80ad6b35eb5b7db4caa7bc72ff919e6b80" -dependencies = [ - "serde", - "serde_json", - "thiserror 1.0.69", -] - -[[package]] -name = "jsonrpsee" -version = "0.26.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f3f48dc3e6b8bd21e15436c1ddd0bc22a6a54e8ec46fedd6adf3425f396ec6a" -dependencies = [ - "jsonrpsee-client-transport", - "jsonrpsee-core", - "jsonrpsee-http-client", - "jsonrpsee-server", - "jsonrpsee-types", - "jsonrpsee-wasm-client", - "jsonrpsee-ws-client", - "tokio", -] - -[[package]] -name = "jsonrpsee-client-transport" -version = "0.26.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf36eb27f8e13fa93dcb50ccb44c417e25b818cfa1a481b5470cd07b19c60b98" -dependencies = [ - "base64", - "futures-channel", - "futures-util", - "gloo-net", - "http", - "jsonrpsee-core", - "pin-project", - "rustls", - "rustls-pki-types", - "rustls-platform-verifier 0.5.3", - "soketto", - "thiserror 2.0.17", - "tokio", - "tokio-rustls", - "tokio-util", - "tracing", - "url", -] - -[[package]] -name = "jsonrpsee-core" -version = "0.26.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "316c96719901f05d1137f19ba598b5fe9c9bc39f4335f67f6be8613921946480" -dependencies = [ - "async-trait", - "bytes", - "futures-timer", - "futures-util", - "http", - "http-body", - "http-body-util", - "jsonrpsee-types", - "parking_lot", - "pin-project", - "rand 0.9.2", - "rustc-hash", - "serde", - "serde_json", - "thiserror 2.0.17", - "tokio", - "tokio-stream", - "tower", - "tracing", - "wasm-bindgen-futures", -] - -[[package]] -name = "jsonrpsee-http-client" -version = "0.26.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "790bedefcec85321e007ff3af84b4e417540d5c87b3c9779b9e247d1bcc3dab8" -dependencies = [ - "base64", - "http-body", - "hyper", - "hyper-rustls", - "hyper-util", - "jsonrpsee-core", - "jsonrpsee-types", - "rustls", - "rustls-platform-verifier 0.5.3", - "serde", - "serde_json", - "thiserror 2.0.17", - "tokio", - "tower", - "url", -] - -[[package]] -name = "jsonrpsee-server" -version = "0.26.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c51b7c290bb68ce3af2d029648148403863b982f138484a73f02a9dd52dbd7f" -dependencies = [ - "futures-util", - "http", - "http-body", - "http-body-util", - "hyper", - "hyper-util", - "jsonrpsee-core", - "jsonrpsee-types", - "pin-project", - "route-recognizer", - "serde", - "serde_json", - "soketto", - "thiserror 2.0.17", - "tokio", - "tokio-stream", - "tokio-util", - "tower", - "tracing", -] - -[[package]] -name = "jsonrpsee-types" -version = "0.26.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc88ff4688e43cc3fa9883a8a95c6fa27aa2e76c96e610b737b6554d650d7fd5" +checksum = "41a652e1f9b6e0275df1f15b32661cf0d4b78d4d87ddec5e0c3c20f097433258" dependencies = [ - "http", - "serde", - "serde_json", - "thiserror 2.0.17", + "jni-sys 0.4.1", ] [[package]] -name = "jsonrpsee-wasm-client" -version = "0.26.0" +name = "jni-sys" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7902885de4779f711a95d82c8da2d7e5f9f3a7c7cfa44d51c067fd1c29d72a3c" +checksum = "c6377a88cb3910bee9b0fa88d4f42e1d2da8e79915598f65fb0c7ee14c878af2" dependencies = [ - "jsonrpsee-client-transport", - "jsonrpsee-core", - "jsonrpsee-types", - "tower", + "jni-sys-macros", ] [[package]] -name = "jsonrpsee-ws-client" -version = "0.26.0" +name = "jni-sys-macros" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b6fceceeb05301cc4c065ab3bd2fa990d41ff4eb44e4ca1b30fa99c057c3e79" +checksum = "38c0b942f458fe50cdac086d2f946512305e5631e720728f2a61aabcd47a6264" dependencies = [ - "http", - "jsonrpsee-client-transport", - "jsonrpsee-core", - "jsonrpsee-types", - "tower", - "url", + "quote", + "syn", ] -[[package]] -name = "leb128fmt" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2" - [[package]] name = "libc" -version = "0.2.182" +version = "0.2.186" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6800badb6cb2082ffd7b6a67e6125bb39f18782f793520caee8cb8846be06112" +checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" [[package]] -name = "libloading" -version = "0.8.9" +name = "libsqlite3-sys" +version = "0.30.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7c4b02199fee7c5d21a5ae7d8cfa79a6ef5bb2fc834d6e9058e89c825efdc55" +checksum = "2e99fb7a497b1e3339bc746195567ed8d3e24945ecd636e3619d20b9de9e9149" dependencies = [ - "cfg-if", - "windows-link", + "cc", + "pkg-config", + "vcpkg", ] [[package]] -name = "libversion-sys" -version = "0.1.0" -source = "git+https://github.com/DUpdateSystem/libversion-sys?rev=68391515ac8f555ca86bc5dfcd980f66352e414d#68391515ac8f555ca86bc5dfcd980f66352e414d" -dependencies = [ - "bindgen", - "cc", - "cmake", -] +name = "linked-hash-map" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f" [[package]] name = "linux-raw-sys" @@ -1154,12 +797,6 @@ version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" -[[package]] -name = "litemap" -version = "0.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6373607a59f0be73a39b6fe456b8192fcc3585f602af20751600e974dd455e77" - [[package]] name = "lock_api" version = "0.4.14" @@ -1171,42 +808,40 @@ dependencies = [ [[package]] name = "log" -version = "0.4.29" +version = "0.4.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" +checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad" [[package]] -name = "lru-slab" -version = "0.1.2" +name = "lua-src" +version = "547.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" +checksum = "1edaf29e3517b49b8b746701e5648ccb5785cde1c119062cbabbc5d5cd115e42" +dependencies = [ + "cc", +] [[package]] -name = "markdown" -version = "1.0.0" +name = "luajit-src" +version = "210.5.12+a4f56a4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a5cab8f2cadc416a82d2e783a1946388b31654d391d1c7d92cc1f03e295b1deb" +checksum = "b3a8e7962a5368d5f264d045a5a255e90f9aa3fc1941ae15a8d2940d42cac671" dependencies = [ - "unicode-id", + "cc", + "which", ] [[package]] name = "memchr" -version = "2.7.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f52b00d39961fc5b2736ea853c9cc86238e165017a493d1d5c8eac6bdc4cc273" - -[[package]] -name = "minimal-lexical" -version = "0.2.1" +version = "2.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" +checksum = "88904434abc2901f197fe8cc55f0445e7ded921dba5911dad2e2b39b48e663c4" [[package]] name = "mio" -version = "1.1.1" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a69bcab0ad47271a0234d9422b131806bf3968021e5dc9328caf2d4cd58557fc" +checksum = "02bd0af71c67b473010cbbc60715ee815645a4dc942899111f494b4b737d6fda" dependencies = [ "libc", "wasi", @@ -1214,63 +849,79 @@ dependencies = [ ] [[package]] -name = "mockito" -version = "1.7.2" +name = "mlua" +version = "0.10.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90820618712cab19cfc46b274c6c22546a82affcb3c3bdf0f29e3db8e1bb92c0" +checksum = "c1f5f8fbebc7db5f671671134b9321c4b9aa9adeafccfd9a8c020ae45c6a35d0" dependencies = [ - "assert-json-diff", - "bytes", - "colored", - "futures-core", - "http", - "http-body", - "http-body-util", - "hyper", - "hyper-util", - "log", - "pin-project-lite", - "rand 0.9.2", - "regex", - "serde_json", - "serde_urlencoded", - "similar", - "tokio", + "bstr", + "either", + "mlua-sys", + "num-traits", + "parking_lot", + "rustc-hash", + "rustversion", ] [[package]] -name = "nix" -version = "0.29.0" +name = "mlua-sys" +version = "0.6.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "71e2746dc3a24dd78b3cfcb7be93368c6de9963d30f43a6a73998a9cf4b17b46" +checksum = "380c1f7e2099cafcf40e51d3a9f20a346977587aa4d012eae1f043149a728a93" dependencies = [ - "bitflags", + "cc", "cfg-if", - "cfg_aliases", - "libc", + "lua-src", + "luajit-src", + "pkg-config", ] [[package]] name = "nom" -version = "7.1.3" +version = "8.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df9761775871bdef83bee530e60050f7e54b1105350d6884eb0fb4f46c2f9405" +dependencies = [ + "memchr", +] + +[[package]] +name = "nom_locate" +version = "5.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +checksum = "0b577e2d69827c4740cba2b52efaad1c4cc7c73042860b199710b3575c68438d" dependencies = [ + "bytecount", "memchr", - "minimal-lexical", + "nom", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", ] [[package]] name = "once_cell" -version = "1.21.3" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "once_cell_polyfill" +version = "1.70.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" +checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" [[package]] name = "openssl-probe" -version = "0.2.0" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f50d9b3dabb09ecd771ad0aa242ca6894994c130308ca3d7684634df8037391" +checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" [[package]] name = "parking_lot" @@ -1296,31 +947,46 @@ dependencies = [ ] [[package]] -name = "paste" -version = "1.0.15" +name = "peg" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f76678828272f177ac33b7e2ac2e3e73cc6c1cd1e3e387928aa69562fa51367" +dependencies = [ + "peg-macros", + "peg-runtime", +] + +[[package]] +name = "peg-macros" +version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" +checksum = "636d60acf97633e48d266d7415a9355d4389cea327a193f87df395d88cd2b14d" +dependencies = [ + "peg-runtime", + "proc-macro2", + "quote", +] [[package]] -name = "percent-encoding" -version = "2.3.2" +name = "peg-runtime" +version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" +checksum = "9555b1514d2d99d78150d3c799d4c357a3e2c2a8062cd108e93a06d9057629c5" [[package]] name = "pin-project" -version = "1.1.10" +version = "1.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "677f1add503faace112b9f1373e43e9e054bfdd22ff1a63c1bc485eaec6a6a8a" +checksum = "2466b2336ed02bcdca6b294417127b90ec92038d1d5c4fbeac971a922e0e0924" dependencies = [ "pin-project-internal", ] [[package]] name = "pin-project-internal" -version = "1.1.10" +version = "1.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e918e4ff8c4549eb882f14b3a4bc8c8bc93de829416eacf579f1207a8fbf861" +checksum = "c96395f0a926bc13b1c17622aaddda1ecb55d49c8f1bf9777e4d877800a43f8b" dependencies = [ "proc-macro2", "quote", @@ -1329,225 +995,74 @@ dependencies = [ [[package]] name = "pin-project-lite" -version = "0.2.16" +version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b3cff922bd51709b605d9ead9aa71031d81447142d828eb4a6eba76fe619f9b" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" [[package]] -name = "pin-utils" -version = "0.1.0" +name = "pkg-config" +version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" +checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" [[package]] -name = "potential_utf" -version = "0.1.4" +name = "proc-macro2" +version = "1.0.106" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b73949432f5e2a09657003c25bca5e19a0e9c84f8058ca374f49e0ebe605af77" +checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" dependencies = [ - "zerovec", + "unicode-ident", ] [[package]] -name = "ppv-lite86" -version = "0.2.21" +name = "quote" +version = "1.0.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924" dependencies = [ - "zerocopy", + "proc-macro2", ] [[package]] -name = "prettyplease" -version = "0.2.37" +name = "r-efi" +version = "6.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" -dependencies = [ - "proc-macro2", - "syn", -] +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" [[package]] -name = "proc-macro2" -version = "1.0.105" +name = "redox_syscall" +version = "0.5.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "535d180e0ecab6268a3e718bb9fd44db66bbbc256257165fc699dadf70d16fe7" +checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" dependencies = [ - "unicode-ident", + "bitflags", ] [[package]] -name = "quick-xml" -version = "0.39.2" +name = "ref-cast" +version = "1.0.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "958f21e8e7ceb5a1aa7fa87fab28e7c75976e0bfe7e23ff069e0a260f894067d" +checksum = "f354300ae66f76f1c85c5f84693f0ce81d747e2c3f21a45fef496d89c960bf7d" dependencies = [ - "encoding_rs", - "memchr", - "tokio", + "ref-cast-impl", ] [[package]] -name = "quinn" -version = "0.11.9" +name = "ref-cast-impl" +version = "1.0.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9e20a958963c291dc322d98411f541009df2ced7b5a4f2bd52337638cfccf20" -dependencies = [ - "bytes", - "cfg_aliases", - "pin-project-lite", - "quinn-proto", - "quinn-udp", - "rustc-hash", - "rustls", - "socket2", - "thiserror 2.0.17", - "tokio", - "tracing", - "web-time", -] - -[[package]] -name = "quinn-proto" -version = "0.11.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1906b49b0c3bc04b5fe5d86a77925ae6524a19b816ae38ce1e426255f1d8a31" -dependencies = [ - "aws-lc-rs", - "bytes", - "getrandom 0.3.4", - "lru-slab", - "rand 0.9.2", - "ring", - "rustc-hash", - "rustls", - "rustls-pki-types", - "slab", - "thiserror 2.0.17", - "tinyvec", - "tracing", - "web-time", -] - -[[package]] -name = "quinn-udp" -version = "0.5.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "addec6a0dcad8a8d96a771f815f0eaf55f9d1805756410b39f5fa81332574cbd" -dependencies = [ - "cfg_aliases", - "libc", - "once_cell", - "socket2", - "tracing", - "windows-sys 0.60.2", -] - -[[package]] -name = "quote" -version = "1.0.43" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc74d9a594b72ae6656596548f56f667211f8a97b3d4c3d467150794690dc40a" +checksum = "b7186006dcb21920990093f30e3dea63b7d6e977bf1256be20c3563a5db070da" dependencies = [ "proc-macro2", -] - -[[package]] -name = "r-efi" -version = "5.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" - -[[package]] -name = "rand" -version = "0.8.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" -dependencies = [ - "libc", - "rand_chacha 0.3.1", - "rand_core 0.6.4", -] - -[[package]] -name = "rand" -version = "0.9.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6db2770f06117d490610c7488547d543617b21bfa07796d7a12f6f1bd53850d1" -dependencies = [ - "rand_chacha 0.9.0", - "rand_core 0.9.3", -] - -[[package]] -name = "rand" -version = "0.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc266eb313df6c5c09c1c7b1fbe2510961e5bcd3add930c1e31f7ed9da0feff8" -dependencies = [ - "chacha20", - "getrandom 0.4.1", - "rand_core 0.10.0", -] - -[[package]] -name = "rand_chacha" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" -dependencies = [ - "ppv-lite86", - "rand_core 0.6.4", -] - -[[package]] -name = "rand_chacha" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" -dependencies = [ - "ppv-lite86", - "rand_core 0.9.3", -] - -[[package]] -name = "rand_core" -version = "0.6.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" -dependencies = [ - "getrandom 0.2.16", -] - -[[package]] -name = "rand_core" -version = "0.9.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "99d9a13982dcf210057a8a78572b2217b667c3beacbf3a0d8b454f6f82837d38" -dependencies = [ - "getrandom 0.3.4", -] - -[[package]] -name = "rand_core" -version = "0.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c8d0fd677905edcbeedbf2edb6494d676f0e98d54d5cf9bda0b061cb8fb8aba" - -[[package]] -name = "redox_syscall" -version = "0.5.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" -dependencies = [ - "bitflags", + "quote", + "syn", ] [[package]] name = "regex" -version = "1.12.3" +version = "1.12.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e10754a14b9137dd7b1e3e5b0493cc9171fdd105e0ab477f51b72e7f3ac0e276" +checksum = "f1292b7759ae1cb9ec195452d1390a074f0cd8541ab7a5a8c31cd6db45d4a6ba" dependencies = [ "aho-corasick", "memchr", @@ -1557,9 +1072,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.13" +version = "0.4.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5276caf25ac86c8d810222b3dbb938e512c55c6831a10f3e6ed1c93b84041f1c" +checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f" dependencies = [ "aho-corasick", "memchr", @@ -1568,49 +1083,9 @@ dependencies = [ [[package]] name = "regex-syntax" -version = "0.8.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a2d987857b319362043e95f5353c0535c1f58eec5336fdfcf626430af7def58" - -[[package]] -name = "reqwest" -version = "0.13.2" +version = "0.8.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab3f43e3283ab1488b624b44b0e988d0acea0b3214e694730a055cb6b2efa801" -dependencies = [ - "base64", - "bytes", - "futures-core", - "futures-util", - "http", - "http-body", - "http-body-util", - "hyper", - "hyper-rustls", - "hyper-util", - "js-sys", - "log", - "percent-encoding", - "pin-project-lite", - "quinn", - "rustls", - "rustls-pki-types", - "rustls-platform-verifier 0.6.2", - "serde", - "serde_json", - "sync_wrapper", - "tokio", - "tokio-rustls", - "tokio-util", - "tower", - "tower-http", - "tower-service", - "url", - "wasm-bindgen", - "wasm-bindgen-futures", - "wasm-streams", - "web-sys", -] +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" [[package]] name = "ring" @@ -1620,23 +1095,40 @@ checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" dependencies = [ "cc", "cfg-if", - "getrandom 0.2.16", + "getrandom 0.2.17", "libc", "untrusted", "windows-sys 0.52.0", ] [[package]] -name = "route-recognizer" -version = "0.3.1" +name = "rusqlite" +version = "0.32.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "afab94fb28594581f62d981211a9a4d53cc8130bbcbbb89a0440d9b8e81a7746" +checksum = "7753b721174eb8ff87a9a0e799e2d7bc3749323e773db92e0984debb00019d6e" +dependencies = [ + "bitflags", + "fallible-iterator", + "fallible-streaming-iterator", + "hashlink", + "libsqlite3-sys", + "smallvec", +] [[package]] name = "rustc-hash" -version = "2.1.1" +version = "2.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "357703d41365b4b27c590e3ed91eabb1b663f07c4c084095e60cbed4362dff0d" +checksum = "94300abf3f1ae2e2b8ffb7b58043de3d399c73fa6f4b73826402a5c457614dbe" + +[[package]] +name = "rustc_version" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" +dependencies = [ + "semver", +] [[package]] name = "rustix" @@ -1648,19 +1140,16 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] name = "rustls" -version = "0.23.36" +version = "0.23.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c665f33d38cea657d9614f766881e4d510e0eda4239891eea56b4cadcf01801b" +checksum = "ef86cd5876211988985292b91c96a8f2d298df24e75989a43a3c73f2d4d8168b" dependencies = [ - "aws-lc-rs", - "log", "once_cell", - "ring", "rustls-pki-types", "rustls-webpki", "subtle", @@ -1669,9 +1158,9 @@ dependencies = [ [[package]] name = "rustls-native-certs" -version = "0.8.3" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "612460d5f7bea540c490b2b6395d8e34a953e52b491accd6c86c8164c5932a63" +checksum = "dab5152771c58876a2146916e53e35057e1a4dfa2b9df0f0305b07f611fdea4d" dependencies = [ "openssl-probe", "rustls-pki-types", @@ -1681,35 +1170,13 @@ dependencies = [ [[package]] name = "rustls-pki-types" -version = "1.13.2" +version = "1.14.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "21e6f2ab2928ca4291b86736a8bd920a277a399bba1589409d72154ff87c1282" +checksum = "30a7197ae7eb376e574fe940d068c30fe0462554a3ddbe4eca7838e049c937a9" dependencies = [ - "web-time", "zeroize", ] -[[package]] -name = "rustls-platform-verifier" -version = "0.5.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19787cda76408ec5404443dc8b31795c87cd8fec49762dc75fa727740d34acc1" -dependencies = [ - "core-foundation", - "core-foundation-sys", - "jni", - "log", - "once_cell", - "rustls", - "rustls-native-certs", - "rustls-platform-verifier-android", - "rustls-webpki", - "security-framework", - "security-framework-sys", - "webpki-root-certs 0.26.11", - "windows-sys 0.52.0", -] - [[package]] name = "rustls-platform-verifier" version = "0.6.2" @@ -1727,8 +1194,8 @@ dependencies = [ "rustls-webpki", "security-framework", "security-framework-sys", - "webpki-root-certs 1.0.5", - "windows-sys 0.60.2", + "webpki-root-certs", + "windows-sys 0.61.2", ] [[package]] @@ -1739,11 +1206,10 @@ checksum = "f87165f0995f63a9fbeea62b64d10b4d9d8e78ec6d7d51fb2125fda7bb36788f" [[package]] name = "rustls-webpki" -version = "0.103.8" +version = "0.103.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ffdfa2f5286e2247234e03f680868ac2815974dc39e00ea15adc445d0aafe52" +checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e" dependencies = [ - "aws-lc-rs", "ring", "rustls-pki-types", "untrusted", @@ -1755,12 +1221,6 @@ version = "1.0.22" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" -[[package]] -name = "ryu" -version = "1.0.22" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a50f4cf475b65d88e057964e0e9bb1f0aa9bbb2036dc65c64596b42932536984" - [[package]] name = "same-file" version = "1.0.6" @@ -1770,20 +1230,11 @@ dependencies = [ "winapi-util", ] -[[package]] -name = "scc" -version = "2.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46e6f046b7fef48e2660c57ed794263155d713de679057f2d0c169bfc6e756cc" -dependencies = [ - "sdd", -] - [[package]] name = "schannel" -version = "0.1.28" +version = "0.1.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "891d81b926048e76efe18581bf793546b4c0eaf8448d72be8de2bbee5fd166e1" +checksum = "91c1b7e4904c873ef0710c1f407dde2e6287de2bebc1bbbf7d430bb7cbffd939" dependencies = [ "windows-sys 0.61.2", ] @@ -1795,16 +1246,21 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" [[package]] -name = "sdd" -version = "3.0.10" +name = "sealed" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "490dcfcbfef26be6800d11870ff2df8774fa6e86d047e3e8c8a76b25655e41ca" +checksum = "22f968c5ea23d555e670b449c1c5e7b2fc399fdaec1d304a17cd48e288abc107" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] [[package]] name = "security-framework" -version = "3.5.1" +version = "3.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b3297343eaf830f66ede390ea39da1d462b6b0c1b000f420d0a83f898bbbe6ef" +checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d" dependencies = [ "bitflags", "core-foundation", @@ -1815,9 +1271,9 @@ dependencies = [ [[package]] name = "security-framework-sys" -version = "2.15.0" +version = "2.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc1f0cbffaac4852523ce30d8bd3c5cdc873501d96ff467ca09b6767bb8cd5c0" +checksum = "6ce2691df843ecc5d231c0b14ece2acc3efb62c0a398c7e1d875f3983ce020e3" dependencies = [ "core-foundation-sys", "libc", @@ -1825,15 +1281,9 @@ dependencies = [ [[package]] name = "semver" -version = "1.0.27" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d767eb0aabc880b29956c35734170f26ed551a859dbd361d140cdbeca61ab1e2" - -[[package]] -name = "send_wrapper" -version = "0.4.0" +version = "1.0.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f638d531eccd6e23b980caf34876660d38e265409d8e99b397ab71eb3612fad0" +checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" [[package]] name = "serde" @@ -1867,9 +1317,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.149" +version = "1.0.150" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83fc039473c5595ace860d8c4fafa220ff474b3fc6bfdb4293327f1a37e94d86" +checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9" dependencies = [ "itoa", "memchr", @@ -1879,119 +1329,64 @@ dependencies = [ ] [[package]] -name = "serde_urlencoded" -version = "0.7.1" +name = "serde_spanned" +version = "0.6.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +checksum = "bf41e0cfaf7226dca15e8197172c295a782857fcb97fad1808a166870dee75a3" dependencies = [ - "form_urlencoded", - "itoa", - "ryu", "serde", ] [[package]] -name = "serial_test" -version = "3.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "911bd979bf1070a3f3aa7b691a3b3e9968f339ceeec89e08c280a8a22207a32f" -dependencies = [ - "futures-executor", - "futures-util", - "log", - "once_cell", - "parking_lot", - "scc", - "serial_test_derive", -] - -[[package]] -name = "serial_test_derive" -version = "3.4.0" +name = "shlex" +version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0a7d91949b85b0d2fb687445e448b40d322b6b3e4af6b44a29b21d9a5f33e6d9" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" [[package]] -name = "sha1" -version = "0.10.6" +name = "slab" +version = "0.4.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" -dependencies = [ - "cfg-if", - "cpufeatures 0.2.17", - "digest", -] +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" [[package]] -name = "shlex" -version = "1.3.0" +name = "smallvec" +version = "1.15.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" +checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90" [[package]] -name = "signal-hook-registry" -version = "1.4.8" +name = "smart-default" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b" +checksum = "0eb01866308440fc64d6c44d9e86c5cc17adfe33c4d6eed55da9145044d0ffc1" dependencies = [ - "errno", - "libc", + "proc-macro2", + "quote", + "syn", ] [[package]] -name = "similar" -version = "2.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbbb5d9659141646ae647b42fe094daf6c6192d1620870b449d9557f748b2daa" - -[[package]] -name = "slab" -version = "0.4.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a2ae44ef20feb57a68b23d846850f861394c2e02dc425a50098ae8c90267589" - -[[package]] -name = "smallvec" -version = "1.15.1" +name = "smawk" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" +checksum = "e8e2fb0f499abb4d162f2bedad68f5ef91a1682b5a03596ddb67efd37768d100" [[package]] name = "socket2" -version = "0.6.1" +version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17129e116933cf371d018bb80ae557e889637989d8638274fb25622827b03881" +checksum = "52d1cfed4120b4d927bf7c0f86d2087a4a7d6027c906d9f9d525a80573b9be51" dependencies = [ "libc", - "windows-sys 0.60.2", -] - -[[package]] -name = "soketto" -version = "0.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e859df029d160cb88608f5d7df7fb4753fd20fdfb4de5644f3d8b8440841721" -dependencies = [ - "base64", - "bytes", - "futures", - "http", - "httparse", - "log", - "rand 0.8.5", - "sha1", + "windows-sys 0.61.2", ] [[package]] -name = "stable_deref_trait" -version = "1.2.1" +name = "strsim" +version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" [[package]] name = "subtle" @@ -2001,9 +1396,9 @@ checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" [[package]] name = "syn" -version = "2.0.114" +version = "2.0.118" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4d107df263a3013ef9b1879b0df87d706ff80f65a86ea879bd9c31f9b307c2a" +checksum = "1b9ae57f904213ebb649ce6895b8a66c66f0203b9319718f69a5612a065b1422" dependencies = [ "proc-macro2", "quote", @@ -2011,509 +1406,287 @@ dependencies = [ ] [[package]] -name = "sync_wrapper" -version = "1.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" -dependencies = [ - "futures-core", -] - -[[package]] -name = "synstructure" -version = "0.13.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "tempfile" -version = "3.26.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "82a72c767771b47409d2345987fda8628641887d5466101319899796367354a0" -dependencies = [ - "fastrand", - "getrandom 0.3.4", - "once_cell", - "rustix", - "windows-sys 0.60.2", -] - -[[package]] -name = "thiserror" -version = "1.0.69" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" -dependencies = [ - "thiserror-impl 1.0.69", -] - -[[package]] -name = "thiserror" -version = "2.0.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f63587ca0f12b72a0600bcba1d40081f830876000bb46dd2337a3051618f4fc8" -dependencies = [ - "thiserror-impl 2.0.17", -] - -[[package]] -name = "thiserror-impl" -version = "1.0.69" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "thiserror-impl" -version = "2.0.17" +name = "synthez" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ff15c8ecd7de3849db632e14d18d2571fa09dfc5ed93479bc4485c7a517c913" +checksum = "6d8a928f38f1bc873f28e0d2ba8298ad65374a6ac2241dabd297271531a736cd" dependencies = [ - "proc-macro2", - "quote", "syn", + "synthez-codegen", + "synthez-core", ] [[package]] -name = "tinystr" -version = "0.8.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42d3e9c45c09de15d06dd8acf5f4e0e399e85927b7f00711024eb7ae10fa4869" -dependencies = [ - "displaydoc", - "zerovec", -] - -[[package]] -name = "tinyvec" -version = "1.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfa5fdc3bce6191a1dbc8c02d5c8bffcf557bafa17c124c5264a458f1b0613fa" -dependencies = [ - "tinyvec_macros", -] - -[[package]] -name = "tinyvec_macros" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" - -[[package]] -name = "tokio" -version = "1.49.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72a2903cd7736441aac9df9d7688bd0ce48edccaadf181c3b90be801e81d3d86" -dependencies = [ - "bytes", - "libc", - "mio", - "parking_lot", - "pin-project-lite", - "signal-hook-registry", - "socket2", - "tokio-macros", - "windows-sys 0.61.2", -] - -[[package]] -name = "tokio-macros" -version = "2.6.0" +name = "synthez-codegen" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af407857209536a95c8e56f8231ef2c2e2aff839b22e07a1ffcbc617e9db9fa5" +checksum = "8fb83b8df4238e11746984dfb3819b155cd270de0e25847f45abad56b3671047" dependencies = [ - "proc-macro2", - "quote", "syn", + "synthez-core", ] [[package]] -name = "tokio-rustls" -version = "0.26.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1729aa945f29d91ba541258c8df89027d5792d85a8841fb65e8bf0f4ede4ef61" -dependencies = [ - "rustls", - "tokio", -] - -[[package]] -name = "tokio-stream" -version = "0.1.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32da49809aab5c3bc678af03902d4ccddea2a87d028d86392a4b1560c6906c70" -dependencies = [ - "futures-core", - "pin-project-lite", - "tokio", - "tokio-util", -] - -[[package]] -name = "tokio-util" -version = "0.7.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ae9cec805b01e8fc3fd2fe289f89149a9b66dd16786abd8b19cfa7b48cb0098" -dependencies = [ - "bytes", - "futures-core", - "futures-io", - "futures-sink", - "pin-project-lite", - "tokio", -] - -[[package]] -name = "tower" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d039ad9159c98b70ecfd540b2573b97f7f52c3e8d9f8ad57a24b916a536975f9" -dependencies = [ - "futures-core", - "futures-util", - "pin-project-lite", - "sync_wrapper", - "tokio", - "tower-layer", - "tower-service", -] - -[[package]] -name = "tower-http" -version = "0.6.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4e6559d53cc268e5031cd8429d05415bc4cb4aefc4aa5d6cc35fbf5b924a1f8" -dependencies = [ - "bitflags", - "bytes", - "futures-util", - "http", - "http-body", - "iri-string", - "pin-project-lite", - "tower", - "tower-layer", - "tower-service", -] - -[[package]] -name = "tower-layer" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" - -[[package]] -name = "tower-service" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" - -[[package]] -name = "tracing" -version = "0.1.44" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" -dependencies = [ - "pin-project-lite", - "tracing-attributes", - "tracing-core", -] - -[[package]] -name = "tracing-attributes" -version = "0.1.31" +name = "synthez-core" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" +checksum = "906fba967105d822e7c7ed60477b5e76116724d33de68a585681fb253fc30d5c" dependencies = [ "proc-macro2", "quote", + "sealed", "syn", ] [[package]] -name = "tracing-core" -version = "0.1.36" +name = "tempfile" +version = "3.27.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" dependencies = [ + "fastrand", + "getrandom 0.4.3", "once_cell", + "rustix", + "windows-sys 0.61.2", ] [[package]] -name = "try-lock" -version = "0.2.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" - -[[package]] -name = "typenum" -version = "1.19.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "562d481066bde0658276a35467c4af00bdc6ee726305698a55b86e61d7ad82bb" - -[[package]] -name = "unicode-id" -version = "0.3.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70ba288e709927c043cbe476718d37be306be53fb1fafecd0dbe36d072be2580" - -[[package]] -name = "unicode-ident" -version = "1.0.22" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9312f7c4f6ff9069b165498234ce8be658059c6728633667c526e27dc2cf1df5" - -[[package]] -name = "unicode-xid" -version = "0.2.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" - -[[package]] -name = "untrusted" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" - -[[package]] -name = "url" -version = "2.5.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed" -dependencies = [ - "form_urlencoded", - "idna", - "percent-encoding", - "serde", -] - -[[package]] -name = "utf8_iter" -version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" - -[[package]] -name = "uuid" -version = "1.21.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b672338555252d43fd2240c714dc444b8c6fb0a5c5335e65a07bba7742735ddb" -dependencies = [ - "getrandom 0.4.1", - "js-sys", - "serde_core", - "wasm-bindgen", -] - -[[package]] -name = "version_check" -version = "0.9.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" - -[[package]] -name = "walkdir" -version = "2.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" -dependencies = [ - "same-file", - "winapi-util", -] - -[[package]] -name = "want" -version = "0.3.1" +name = "terminal_size" +version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +checksum = "230a1b821ccbd75b185820a1f1ff7b14d21da1e442e22c0863ea5f08771a8874" dependencies = [ - "try-lock", + "rustix", + "windows-sys 0.61.2", ] [[package]] -name = "wasi" -version = "0.11.1+wasi-snapshot-preview1" +name = "textwrap" +version = "0.16.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" +checksum = "c13547615a44dc9c452a8a534638acdf07120d4b6847c8178705da06306a3057" +dependencies = [ + "smawk", + "unicode-linebreak", + "unicode-width", +] [[package]] -name = "wasip2" -version = "1.0.1+wasi-0.2.4" +name = "thiserror" +version = "1.0.69" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0562428422c63773dad2c345a1882263bbf4d65cf3f42e90921f787ef5ad58e7" +checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" dependencies = [ - "wit-bindgen 0.46.0", + "thiserror-impl 1.0.69", ] [[package]] -name = "wasip3" -version = "0.4.0+wasi-0.3.0-rc-2026-01-06" +name = "thiserror" +version = "2.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5428f8bf88ea5ddc08faddef2ac4a67e390b88186c703ce6dbd955e1c145aca5" +checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4" dependencies = [ - "wit-bindgen 0.51.0", + "thiserror-impl 2.0.18", ] [[package]] -name = "wasm-bindgen" -version = "0.2.108" +name = "thiserror-impl" +version = "1.0.69" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64024a30ec1e37399cf85a7ffefebdb72205ca1c972291c51512360d90bd8566" +checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ - "cfg-if", - "once_cell", - "rustversion", - "wasm-bindgen-macro", - "wasm-bindgen-shared", + "proc-macro2", + "quote", + "syn", ] [[package]] -name = "wasm-bindgen-futures" -version = "0.4.58" +name = "thiserror-impl" +version = "2.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70a6e77fd0ae8029c9ea0063f87c46fde723e7d887703d74ad2616d792e51e6f" +checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" dependencies = [ - "cfg-if", - "futures-util", - "js-sys", - "once_cell", - "wasm-bindgen", - "web-sys", + "proc-macro2", + "quote", + "syn", ] [[package]] -name = "wasm-bindgen-macro" -version = "0.2.108" +name = "tokio" +version = "1.52.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "008b239d9c740232e71bd39e8ef6429d27097518b6b30bdf9086833bd5b6d608" +checksum = "8fc7f01b389ac15039e4dc9531aa973a135d7a4135281b12d7c1bc79fd57fffe" dependencies = [ - "quote", - "wasm-bindgen-macro-support", + "libc", + "mio", + "pin-project-lite", + "socket2", + "tokio-macros", + "windows-sys 0.61.2", ] [[package]] -name = "wasm-bindgen-macro-support" -version = "0.2.108" +name = "tokio-macros" +version = "2.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5256bae2d58f54820e6490f9839c49780dff84c65aeab9e772f15d5f0e913a55" +checksum = "385a6cb71ab9ab790c5fe8d67f1645e6c450a7ce006a33de03daa956cf70a496" dependencies = [ - "bumpalo", "proc-macro2", "quote", "syn", - "wasm-bindgen-shared", ] [[package]] -name = "wasm-bindgen-shared" -version = "0.2.108" +name = "toml" +version = "0.8.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f01b580c9ac74c8d8f0c0e4afb04eeef2acf145458e52c03845ee9cd23e3d12" +checksum = "dc1beb996b9d83529a9e75c17a1686767d148d70663143c7854d8b4a09ced362" dependencies = [ - "unicode-ident", + "serde", + "serde_spanned", + "toml_datetime", + "toml_edit", ] [[package]] -name = "wasm-encoder" -version = "0.244.0" +name = "toml_datetime" +version = "0.6.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "990065f2fe63003fe337b932cfb5e3b80e0b4d0f5ff650e6985b1048f62c8319" +checksum = "22cddaf88f4fbc13c51aebbf5f8eceb5c7c5a9da2ac40a13519eb5b0a0e8f11c" dependencies = [ - "leb128fmt", - "wasmparser", + "serde", ] [[package]] -name = "wasm-metadata" -version = "0.244.0" +name = "toml_edit" +version = "0.22.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb0e353e6a2fbdc176932bbaab493762eb1255a7900fe0fea1a2f96c296cc909" +checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a" dependencies = [ - "anyhow", "indexmap", - "wasm-encoder", - "wasmparser", + "serde", + "serde_spanned", + "toml_datetime", + "toml_write", + "winnow", ] [[package]] -name = "wasm-streams" -version = "0.5.0" +name = "toml_write" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d1ec4f6517c9e11ae630e200b2b65d193279042e28edd4a2cda233e46670bbb" -dependencies = [ - "futures-util", - "js-sys", - "wasm-bindgen", - "wasm-bindgen-futures", - "web-sys", -] +checksum = "5d99f8c9a7727884afe522e9bd5edbfc91a3312b36a77b5fb8926e4c31a41801" [[package]] -name = "wasmparser" -version = "0.244.0" +name = "typed-builder" +version = "0.23.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "47b807c72e1bac69382b3a6fb3dbe8ea4c0ed87ff5629b8685ae6b9a611028fe" +checksum = "31aa81521b70f94402501d848ccc0ecaa8f93c8eb6999eb9747e72287757ffda" dependencies = [ - "bitflags", - "hashbrown 0.15.5", - "indexmap", - "semver", + "typed-builder-macro", ] [[package]] -name = "web-sys" -version = "0.3.85" +name = "typed-builder-macro" +version = "0.23.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "312e32e551d92129218ea9a2452120f4aabc03529ef03e4d0d82fb2780608598" +checksum = "076a02dc54dd46795c2e9c8282ed40bcfb1e22747e955de9389a1de28190fb26" dependencies = [ - "js-sys", - "wasm-bindgen", + "proc-macro2", + "quote", + "syn", ] [[package]] -name = "web-time" -version = "1.1.0" +name = "unicode-ident" +version = "1.0.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" -dependencies = [ - "js-sys", - "wasm-bindgen", -] +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" [[package]] -name = "webpki-root-certs" -version = "0.26.11" +name = "unicode-linebreak" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b09c83c3c29d37506a3e260c08c03743a6bb66a9cd432c6934ab501a190571f" + +[[package]] +name = "unicode-segmentation" +version = "1.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6f5d3c3b1bf09027a88a6bc961fc00497d651009560b5463668dc81b0fa87a8" + +[[package]] +name = "unicode-width" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254" + +[[package]] +name = "unicode-xid" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" + +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "walkdir" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75c7f0ef91146ebfb530314f5f1d24528d7f0767efbfd31dce919275413e393e" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" dependencies = [ - "webpki-root-certs 1.0.5", + "same-file", + "winapi-util", ] +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + [[package]] name = "webpki-root-certs" -version = "1.0.5" +version = "1.0.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36a29fc0408b113f68cf32637857ab740edfafdf460c326cd2afaa2d84cc05dc" +checksum = "0d46a5a140e6f7afeccd8eae97eff335163939eac8b929834875168b29b3d267" dependencies = [ "rustls-pki-types", ] [[package]] -name = "webpki-roots" -version = "1.0.5" +name = "which" +version = "7.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12bed680863276c63889429bfd6cab3b99943659923822de1c8a39c49e4d722c" +checksum = "24d643ce3fd3e5b54854602a080f34fb10ab75e0b813ee32d00ca2b44fa74762" dependencies = [ - "rustls-pki-types", + "either", + "env_home", + "rustix", + "winsafe", ] [[package]] @@ -2522,7 +1695,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -2549,15 +1722,6 @@ dependencies = [ "windows-targets 0.52.6", ] -[[package]] -name = "windows-sys" -version = "0.60.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb" -dependencies = [ - "windows-targets 0.53.5", -] - [[package]] name = "windows-sys" version = "0.61.2" @@ -2591,30 +1755,13 @@ dependencies = [ "windows_aarch64_gnullvm 0.52.6", "windows_aarch64_msvc 0.52.6", "windows_i686_gnu 0.52.6", - "windows_i686_gnullvm 0.52.6", + "windows_i686_gnullvm", "windows_i686_msvc 0.52.6", "windows_x86_64_gnu 0.52.6", "windows_x86_64_gnullvm 0.52.6", "windows_x86_64_msvc 0.52.6", ] -[[package]] -name = "windows-targets" -version = "0.53.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4945f9f551b88e0d65f3db0bc25c33b8acea4d9e41163edf90dcd0b19f9069f3" -dependencies = [ - "windows-link", - "windows_aarch64_gnullvm 0.53.1", - "windows_aarch64_msvc 0.53.1", - "windows_i686_gnu 0.53.1", - "windows_i686_gnullvm 0.53.1", - "windows_i686_msvc 0.53.1", - "windows_x86_64_gnu 0.53.1", - "windows_x86_64_gnullvm 0.53.1", - "windows_x86_64_msvc 0.53.1", -] - [[package]] name = "windows_aarch64_gnullvm" version = "0.42.2" @@ -2627,12 +1774,6 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" -[[package]] -name = "windows_aarch64_gnullvm" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53" - [[package]] name = "windows_aarch64_msvc" version = "0.42.2" @@ -2645,12 +1786,6 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" -[[package]] -name = "windows_aarch64_msvc" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006" - [[package]] name = "windows_i686_gnu" version = "0.42.2" @@ -2663,24 +1798,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" -[[package]] -name = "windows_i686_gnu" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "960e6da069d81e09becb0ca57a65220ddff016ff2d6af6a223cf372a506593a3" - [[package]] name = "windows_i686_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" -[[package]] -name = "windows_i686_gnullvm" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c" - [[package]] name = "windows_i686_msvc" version = "0.42.2" @@ -2693,12 +1816,6 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" -[[package]] -name = "windows_i686_msvc" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2" - [[package]] name = "windows_x86_64_gnu" version = "0.42.2" @@ -2711,12 +1828,6 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" -[[package]] -name = "windows_x86_64_gnu" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499" - [[package]] name = "windows_x86_64_gnullvm" version = "0.42.2" @@ -2729,12 +1840,6 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" -[[package]] -name = "windows_x86_64_gnullvm" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1" - [[package]] name = "windows_x86_64_msvc" version = "0.42.2" @@ -2748,216 +1853,48 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" [[package]] -name = "windows_x86_64_msvc" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650" - -[[package]] -name = "wit-bindgen" -version = "0.46.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f17a85883d4e6d00e8a97c586de764dabcc06133f7f1d55dce5cdc070ad7fe59" - -[[package]] -name = "wit-bindgen" -version = "0.51.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7249219f66ced02969388cf2bb044a09756a083d0fab1e566056b04d9fbcaa5" -dependencies = [ - "wit-bindgen-rust-macro", -] - -[[package]] -name = "wit-bindgen-core" -version = "0.51.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea61de684c3ea68cb082b7a88508a8b27fcc8b797d738bfc99a82facf1d752dc" -dependencies = [ - "anyhow", - "heck", - "wit-parser", -] - -[[package]] -name = "wit-bindgen-rust" -version = "0.51.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7c566e0f4b284dd6561c786d9cb0142da491f46a9fbed79ea69cdad5db17f21" -dependencies = [ - "anyhow", - "heck", - "indexmap", - "prettyplease", - "syn", - "wasm-metadata", - "wit-bindgen-core", - "wit-component", -] - -[[package]] -name = "wit-bindgen-rust-macro" -version = "0.51.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c0f9bfd77e6a48eccf51359e3ae77140a7f50b1e2ebfe62422d8afdaffab17a" -dependencies = [ - "anyhow", - "prettyplease", - "proc-macro2", - "quote", - "syn", - "wit-bindgen-core", - "wit-bindgen-rust", -] - -[[package]] -name = "wit-component" -version = "0.244.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d66ea20e9553b30172b5e831994e35fbde2d165325bec84fc43dbf6f4eb9cb2" -dependencies = [ - "anyhow", - "bitflags", - "indexmap", - "log", - "serde", - "serde_derive", - "serde_json", - "wasm-encoder", - "wasm-metadata", - "wasmparser", - "wit-parser", -] - -[[package]] -name = "wit-parser" -version = "0.244.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ecc8ac4bc1dc3381b7f59c34f00b67e18f910c2c0f50015669dde7def656a736" -dependencies = [ - "anyhow", - "id-arena", - "indexmap", - "log", - "semver", - "serde", - "serde_derive", - "serde_json", - "unicode-xid", - "wasmparser", -] - -[[package]] -name = "writeable" -version = "0.6.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9edde0db4769d2dc68579893f2306b26c6ecfbe0ef499b013d731b7b9247e0b9" - -[[package]] -name = "yoke" -version = "0.8.1" +name = "winnow" +version = "0.7.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72d6e5c6afb84d73944e5cedb052c4680d5657337201555f9f2a16b7406d4954" +checksum = "df79d97927682d2fd8adb29682d1140b343be4ac0f08fd68b7765d9c059d3945" dependencies = [ - "stable_deref_trait", - "yoke-derive", - "zerofrom", + "memchr", ] [[package]] -name = "yoke-derive" -version = "0.8.1" +name = "winsafe" +version = "0.0.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b659052874eb698efe5b9e8cf382204678a0086ebf46982b79d6ca3182927e5d" -dependencies = [ - "proc-macro2", - "quote", - "syn", - "synstructure", -] +checksum = "d135d17ab770252ad95e9a872d365cf3090e3be864a34ab46f48555993efc904" [[package]] name = "zerocopy" -version = "0.8.33" +version = "0.8.52" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "668f5168d10b9ee831de31933dc111a459c97ec93225beb307aed970d1372dfd" +checksum = "ce1022995ff5ff5d841ad7d994facc23098cd40152f2c1d11cd607c6f530653f" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.8.33" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c7962b26b0a8685668b671ee4b54d007a67d4eaf05fda79ac0ecf41e32270f1" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "zerofrom" -version = "0.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50cc42e0333e05660c3587f3bf9d0478688e15d870fab3346451ce7f8c9fbea5" -dependencies = [ - "zerofrom-derive", -] - -[[package]] -name = "zerofrom-derive" -version = "0.1.6" +version = "0.8.52" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502" +checksum = "1ae7f38b72ec2a254e2b87ef277cf2cd4fb97cbebf944faa6f33354da0867930" dependencies = [ "proc-macro2", "quote", "syn", - "synstructure", ] [[package]] name = "zeroize" -version = "1.8.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b97154e67e32c85465826e8bcc1c59429aaaf107c1e4a9e53c8d8ccd5eff88d0" - -[[package]] -name = "zerotrie" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a59c17a5562d507e4b54960e8569ebee33bee890c70aa3fe7b97e85a9fd7851" -dependencies = [ - "displaydoc", - "yoke", - "zerofrom", -] - -[[package]] -name = "zerovec" -version = "0.11.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c28719294829477f525be0186d13efa9a3c602f7ec202ca9e353d310fb9a002" -dependencies = [ - "yoke", - "zerofrom", - "zerovec-derive", -] - -[[package]] -name = "zerovec-derive" -version = "0.11.2" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eadce39539ca5cb3985590102671f2567e659fca9666581ad3411d59207951f3" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] +checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e" [[package]] name = "zmij" -version = "1.0.12" +version = "1.0.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2fc5a66a20078bf1251bde995aa2fdcc4b800c70b5d92dd2c62abc5c60f679f8" +checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" diff --git a/Cargo.toml b/Cargo.toml index c7a8f49..500b0f1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,44 +3,35 @@ name = "getter" version = "0.1.0" edition = "2021" +[workspace] +resolver = "2" +members = [ + "crates/getter-core", + "crates/getter-storage", + "crates/getter-providers", + "crates/getter-downloader", + "crates/getter-plugin-api", + "crates/getter-rpc", + "crates/getter-cli", + "crates/getter-ffi", +] + +[workspace.package] +version = "0.1.0" +edition = "2021" + [features] -default = ["rustls-platform-verifier"] -rustls-platform-verifier = ["hyper-rustls/rustls-platform-verifier", "dep:rustls-platform-verifier"] +default = ["cli"] +cli = ["dep:getter-cli"] +rustls-platform-verifier = ["dep:rustls-platform-verifier"] rustls-platform-verifier-android = ["rustls-platform-verifier", "rustls-platform-verifier/jni"] -webpki-roots = ["hyper-rustls/webpki-roots"] -native-tokio = ["hyper-rustls/native-tokio"] +webpki-roots = [] +native-tokio = [] [dependencies] -once_cell = "1.21.3" -async-trait = "0.1.89" -hyper = { version = "1.8", features = ["full"] } -tokio = { version = "1", features = ["full", "macros"] } -bytes = "1.11.0" -serde = { version = "1.0", features = ["derive"] } -serde_json = "1.0.149" -quick-xml = { version = "0.39.0", features = ["encoding", "async-tokio"] } -async_fn_traits = "0.1.1" -libversion-sys = { git = "https://github.com/DUpdateSystem/libversion-sys", rev = "68391515ac8f555ca86bc5dfcd980f66352e414d" } -regex = "1.12.2" -hyper-util = { version = "0.1.19", features = ["client", "tokio", "http1"] } -http-body-util = "0.1.3" -jsonrpsee = { version = "0.26.0", features = ["server", "client"] } -hyper-rustls = { version = "0.27.7", features = ["http1", "http2", "native-tokio", "ring", "tls12"], default-features = false } -rustls-platform-verifier = { version = "0.6.2", optional = true } -rustls = { version = "0.23.36", default-features = false } -markdown = "1.0.0" -# Downloader dependencies -reqwest = { version = "0.13", default-features = false, features = ["rustls", "stream", "json"] } -uuid = { version = "1.19", features = ["v4", "serde"] } -parking_lot = "0.12" -futures = "0.3" -tokio-util = { version = "0.7", features = ["io"] } -jsonl = "4.0" -file-locker = "1.1" -url = "2" - -[dev-dependencies] -mockito = "1.7.1" -rand = "0.10.0" -serial_test = "3.3.1" -tempfile = "3.24.0" +getter-core = { path = "crates/getter-core" } +getter-storage = { path = "crates/getter-storage" } +getter-cli = { path = "crates/getter-cli", optional = true } +thiserror = "1" +tokio = { version = "1", features = ["net"] } +rustls-platform-verifier = { version = "0.6", optional = true } diff --git a/README.md b/README.md index 24fef6c..34543a0 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,24 @@ -# getter -Get update for everywhere +# UpgradeAll getter + +Reusable Rust getter core for the UpgradeAll rewrite. + +This repository is intentionally usable outside the UpgradeAll UI. The UpgradeAll app is a UI/platform adapter; package/repository evaluation, storage, migration mapping, update policy, and CLI behavior belong here. + +## Current rewrite spine + +- Rust workspace split into `getter-core`, `getter-storage`, `getter-cli`, and placeholder provider/downloader/RPC/FFI crates. +- Package IDs are readable, for example `android/org.fdroid.fdroid`. +- Lua package repositories use `repo.toml` plus `packages/`, `lib/`, and `templates/` directories. +- SQLite storage uses a main DB and a separate cache DB. +- `getter-cli` exposes JSON command contracts for init, app list, repository registration/evaluation, package evaluation, storage validation, and legacy bridge-bundle import. + +## Verify + +```bash +cargo fmt --all --check +cargo test --workspace --lib --bins +cargo test -p getter-cli --test bdd_cli +cargo check --workspace --all-targets +``` + +The CI helper at `tests/script/cargo_test.sh` runs the same core checks plus feature-compatibility checks. diff --git a/crates/getter-cli/Cargo.toml b/crates/getter-cli/Cargo.toml new file mode 100644 index 0000000..ca03088 --- /dev/null +++ b/crates/getter-cli/Cargo.toml @@ -0,0 +1,21 @@ +[package] +name = "getter-cli" +version.workspace = true +edition.workspace = true + +[dependencies] +getter-core = { path = "../getter-core" } +getter-storage = { path = "../getter-storage" } +serde = { version = "1", features = ["derive"] } +serde_json = "1" +thiserror = "1" + +[dev-dependencies] +cucumber = "0.23" +serde_json = "1" +tempfile = "3" +tokio = { version = "1", features = ["macros", "rt-multi-thread"] } + +[[test]] +name = "bdd_cli" +harness = false diff --git a/crates/getter-cli/src/lib.rs b/crates/getter-cli/src/lib.rs new file mode 100644 index 0000000..8e0155e --- /dev/null +++ b/crates/getter-cli/src/lib.rs @@ -0,0 +1,817 @@ +//! User-facing getter CLI implementation. +//! +//! The CLI is intentionally thin: it owns command parsing and JSON envelopes, +//! while durable state is initialized through `getter-storage` so the command +//! surface exercises the same Rust-owned SQLite direction used by embedders. + +use getter_core::lua::evaluate_package_file; +use getter_core::repository::{RepositoryLayout, RepositoryMetadata}; +use getter_core::{PackageId, RepositoryId, RepositoryPriority}; +use getter_storage::legacy_room::{ + map_legacy_app, LegacyAppKind, LegacyAppRecord, LegacyExtraAppRecord, LegacyPackageResolution, +}; +use getter_storage::{ + CacheDb, MainDb, StorageError, StoredPackageResolution, StoredRepository, StoredTrackedPackage, + TrackedPackageUpsert, +}; +use serde::{Deserialize, Serialize}; +use serde_json::{json, Value}; +use std::fs; +use std::path::{Path, PathBuf}; + +const MAIN_DB_FILE: &str = "main.db"; +const CACHE_DB_FILE: &str = "cache.db"; +const MIGRATION_REPORTS_DIR: &str = "migration-reports"; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct CliInvocation { + pub data_dir: PathBuf, + pub command: CliCommand, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum CliCommand { + Init, + AppList, + HubList, + RepoList, + RepoAdd { + id: RepositoryId, + path: PathBuf, + priority: Option, + }, + RepoEval { + id: RepositoryId, + }, + PackageEval { + package_id: PackageId, + repo_id: Option, + }, + StorageValidate, + LegacyImportRoomBundle { + bundle: PathBuf, + }, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ExitCode { + Success = 0, + GenericFailure = 1, + Usage = 2, + Storage = 10, + Migration = 20, +} + +impl ExitCode { + pub const fn code(self) -> i32 { + self as i32 + } +} + +#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] +pub enum CliError { + #[error("{0}")] + Usage(String), + #[error("storage error: {0}")] + Storage(String), + #[error("repository error: {0}")] + Repository(String), + #[error("package evaluation error: {0}")] + PackageEval(String), + #[error("Legacy Room export bundle is invalid")] + InvalidLegacyBundle { report_path: PathBuf }, + #[error("Legacy Room bundle import is not implemented yet")] + UnsupportedLegacyBundle { report_path: PathBuf }, +} + +impl CliError { + pub fn exit_code(&self) -> ExitCode { + match self { + Self::Usage(_) => ExitCode::Usage, + Self::Storage(_) => ExitCode::Storage, + Self::Repository(_) | Self::PackageEval(_) => ExitCode::GenericFailure, + Self::InvalidLegacyBundle { .. } | Self::UnsupportedLegacyBundle { .. } => { + ExitCode::Migration + } + } + } + + fn code(&self) -> &'static str { + match self { + Self::Usage(_) => "cli.usage", + Self::Storage(_) => "storage.error", + Self::Repository(_) => "repository.error", + Self::PackageEval(_) => "package.eval_error", + Self::InvalidLegacyBundle { .. } => "migration.invalid_bundle", + Self::UnsupportedLegacyBundle { .. } => "migration.unsupported_bundle", + } + } + + fn message(&self) -> &'static str { + match self { + Self::Usage(_) => "Invalid getter CLI usage", + Self::Storage(_) => "Getter storage operation failed", + Self::Repository(_) => "Getter repository operation failed", + Self::PackageEval(_) => "Getter package evaluation failed", + Self::InvalidLegacyBundle { .. } => "Legacy Room export bundle is invalid", + Self::UnsupportedLegacyBundle { .. } => { + "Legacy Room bundle import is not implemented yet" + } + } + } + + fn detail(&self) -> Option<&str> { + match self { + Self::Usage(detail) + | Self::Storage(detail) + | Self::Repository(detail) + | Self::PackageEval(detail) => Some(detail.as_str()), + Self::InvalidLegacyBundle { .. } | Self::UnsupportedLegacyBundle { .. } => None, + } + } + + fn report_path(&self) -> Option<&Path> { + match self { + Self::InvalidLegacyBundle { report_path } + | Self::UnsupportedLegacyBundle { report_path } => Some(report_path.as_path()), + Self::Usage(_) | Self::Storage(_) | Self::Repository(_) | Self::PackageEval(_) => None, + } + } +} + +impl From for CliError { + fn from(value: StorageError) -> Self { + Self::Storage(value.to_string()) + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct CliOutput { + pub exit_code: ExitCode, + pub stdout: String, + pub stderr: String, +} + +/// Parse and execute a getter CLI invocation, returning stdout/stderr and the +/// supported process exit code. This is used by the binary entrypoint and by +/// tests so command behavior stays consistent. +pub fn run(args: I) -> CliOutput +where + I: IntoIterator, + S: Into, +{ + match run_inner(args) { + Ok((command_name, data)) => CliOutput { + exit_code: ExitCode::Success, + stdout: success_envelope(&command_name, data), + stderr: String::new(), + }, + Err((command_name, error)) => { + let exit_code = error.exit_code(); + let stderr = match error { + CliError::Usage(_) => usage_text(), + _ => String::new(), + }; + CliOutput { + exit_code, + stdout: error_envelope(&command_name, &error), + stderr, + } + } + } +} + +fn run_inner(args: I) -> Result<(String, Value), (String, CliError)> +where + I: IntoIterator, + S: Into, +{ + let invocation = parse_args(args).map_err(|error| ("unknown".to_owned(), error))?; + let command_name = invocation.command.name().to_owned(); + execute(invocation) + .map(|data| (command_name.clone(), data)) + .map_err(|error| (command_name, error)) +} + +pub fn parse_args(args: I) -> Result +where + I: IntoIterator, + S: Into, +{ + let mut args: Vec = args.into_iter().map(Into::into).collect(); + if args.first().is_some_and(|arg| arg != "--data-dir") { + args.remove(0); + } + + if args.first().map(String::as_str) != Some("--data-dir") { + return Err(CliError::Usage( + "missing required --data-dir global option".to_owned(), + )); + } + if args.len() < 3 { + return Err(CliError::Usage( + "missing command after --data-dir ".to_owned(), + )); + } + + let data_dir = PathBuf::from(args[1].clone()); + let command_args = &args[2..]; + let command = match command_args { + [command] if command == "init" => CliCommand::Init, + [domain, command] if domain == "app" && command == "list" => CliCommand::AppList, + [domain, command] if domain == "hub" && command == "list" => CliCommand::HubList, + [domain, command] if domain == "repo" && command == "list" => CliCommand::RepoList, + [domain, command, id, path] if domain == "repo" && command == "add" => { + CliCommand::RepoAdd { + id: parse_repository_id(id)?, + path: PathBuf::from(path), + priority: None, + } + } + [domain, command, id, path, flag, priority] + if domain == "repo" && command == "add" && flag == "--priority" => + { + CliCommand::RepoAdd { + id: parse_repository_id(id)?, + path: PathBuf::from(path), + priority: Some(parse_priority(priority)?), + } + } + [domain, command, id] if domain == "repo" && command == "eval" => CliCommand::RepoEval { + id: parse_repository_id(id)?, + }, + [domain, command, package_id] if domain == "package" && command == "eval" => { + CliCommand::PackageEval { + package_id: parse_package_id(package_id)?, + repo_id: None, + } + } + [domain, command, package_id, flag, repo_id] + if domain == "package" && command == "eval" && flag == "--repo" => + { + CliCommand::PackageEval { + package_id: parse_package_id(package_id)?, + repo_id: Some(parse_repository_id(repo_id)?), + } + } + [domain, command] if domain == "storage" && command == "validate" => { + CliCommand::StorageValidate + } + [domain, command, bundle] if domain == "legacy" && command == "import-room-bundle" => { + CliCommand::LegacyImportRoomBundle { + bundle: PathBuf::from(bundle), + } + } + _ => { + return Err(CliError::Usage(format!( + "unsupported command: {}", + command_args.join(" ") + ))) + } + }; + + Ok(CliInvocation { data_dir, command }) +} + +fn execute(invocation: CliInvocation) -> Result { + match invocation.command { + CliCommand::Init => { + initialize_storage(&invocation.data_dir)?; + Ok(json!({ + "data_dir": invocation.data_dir, + "main_db": main_db_path(&invocation.data_dir), + "cache_db": cache_db_path(&invocation.data_dir), + })) + } + CliCommand::AppList => { + let db = open_main_db(&invocation.data_dir)?; + Ok(json!({ "apps": tracked_packages_json(db.tracked_packages()?) })) + } + CliCommand::HubList => { + open_initialized_storage(&invocation.data_dir)?; + Ok(json!({ "hubs": [] })) + } + CliCommand::RepoList => { + let db = open_main_db(&invocation.data_dir)?; + Ok(json!({ "repositories": list_repositories(&db)? })) + } + CliCommand::RepoAdd { id, path, priority } => { + let db = open_main_db(&invocation.data_dir)?; + let layout = load_repository_layout(&path)?; + if layout.metadata.id != id { + return Err(CliError::Repository(format!( + "repo.toml id '{}' does not match requested id '{}'", + layout.metadata.id, id + ))); + } + let metadata = RepositoryMetadata { + priority: priority.unwrap_or(layout.metadata.priority), + ..layout.metadata.clone() + }; + db.upsert_repository(&metadata, Some(&path), None)?; + Ok(json!({ "repository": repository_metadata_json(&metadata, Some(&path), None) })) + } + CliCommand::RepoEval { id } => { + let db = open_main_db(&invocation.data_dir)?; + let repo = find_repository(&db, &id)?; + let path = repo_path(&repo)?; + let layout = load_repository_layout(&path)?; + let mut packages = Vec::new(); + for package_file in &layout.packages { + let package = evaluate_package_file(&layout, &package_file.path) + .map_err(|error| CliError::PackageEval(error.to_string()))?; + packages.push(package_json(package)?); + } + Ok(json!({ + "repository": repository_json(repo), + "packages": packages, + })) + } + CliCommand::PackageEval { + package_id, + repo_id, + } => { + let db = open_main_db(&invocation.data_dir)?; + let package = match repo_id { + Some(repo_id) => evaluate_package_from_repo(&db, &repo_id, &package_id)?, + None => evaluate_highest_priority_package(&db, &package_id)?, + }; + Ok(json!({ "package": package_json(package)? })) + } + CliCommand::StorageValidate => { + initialize_storage(&invocation.data_dir)?; + Ok(json!({ + "valid": true, + "main_db": main_db_path(&invocation.data_dir), + "cache_db": cache_db_path(&invocation.data_dir), + })) + } + CliCommand::LegacyImportRoomBundle { bundle } => { + let db = open_main_db(&invocation.data_dir)?; + let bytes = fs::read(&bundle).map_err(|source| { + match create_migration_report( + &invocation.data_dir, + &bundle, + "migration.invalid_bundle", + &format!("failed to read bundle: {source}"), + 0, + 0, + ) { + Ok(report_path) => CliError::InvalidLegacyBundle { report_path }, + Err(error) => error, + } + })?; + let parsed: LegacyRoomBundle = serde_json::from_slice(&bytes).map_err(|source| { + match create_migration_report( + &invocation.data_dir, + &bundle, + "migration.invalid_bundle", + &format!("failed to parse JSON bundle: {source}"), + 0, + 0, + ) { + Ok(report_path) => CliError::InvalidLegacyBundle { report_path }, + Err(error) => error, + } + })?; + if parsed.format != "upgradeall-legacy-room-bundle" || parsed.version != 17 { + let report_path = create_migration_report( + &invocation.data_dir, + &bundle, + "migration.unsupported_bundle", + &format!( + "unsupported legacy bundle format '{}' version {}", + parsed.format, parsed.version + ), + 0, + 0, + )?; + return Err(CliError::UnsupportedLegacyBundle { report_path }); + } + import_legacy_room_bundle(&db, &parsed)?; + let report_path = create_migration_report( + &invocation.data_dir, + &bundle, + "migration.imported", + "Legacy Room bundle imported", + parsed.apps.len() as u64, + parsed.apps.len() as u64, + )?; + let records = db.tracked_packages()?; + Ok(json!({ + "report_path": report_path, + "imported_records": parsed.apps.len(), + "apps": tracked_packages_json(records), + })) + } + } +} + +fn initialize_storage(data_dir: &Path) -> Result<(), CliError> { + fs::create_dir_all(data_dir).map_err(|source| { + CliError::Storage(format!("failed to create data directory: {source}")) + })?; + MainDb::open(main_db_path(data_dir))?; + CacheDb::open(cache_db_path(data_dir))?; + Ok(()) +} + +fn open_initialized_storage(data_dir: &Path) -> Result<(), CliError> { + initialize_storage(data_dir) +} + +fn open_main_db(data_dir: &Path) -> Result { + initialize_storage(data_dir)?; + Ok(MainDb::open(main_db_path(data_dir))?) +} + +fn parse_repository_id(value: &str) -> Result { + RepositoryId::new(value).map_err(|source| CliError::Usage(source.to_string())) +} + +fn parse_package_id(value: &str) -> Result { + value + .parse() + .map_err(|source: getter_core::PackageIdError| CliError::Usage(source.to_string())) +} + +fn parse_priority(value: &str) -> Result { + value + .parse::() + .map(RepositoryPriority::new) + .map_err(|source| CliError::Usage(format!("invalid repository priority: {source}"))) +} + +fn load_repository_layout(path: &Path) -> Result { + RepositoryLayout::load(path).map_err(|source| CliError::Repository(source.to_string())) +} + +fn list_repositories(db: &MainDb) -> Result, CliError> { + Ok(db + .repositories()? + .into_iter() + .map(repository_json) + .collect()) +} + +fn find_repository(db: &MainDb, id: &RepositoryId) -> Result { + db.repositories()? + .into_iter() + .find(|repo| &repo.id == id) + .ok_or_else(|| CliError::Repository(format!("repository '{id}' is not registered"))) +} + +fn evaluate_package_from_repo( + db: &MainDb, + repo_id: &RepositoryId, + package_id: &PackageId, +) -> Result { + let repo = find_repository(db, repo_id)?; + let path = repo_path(&repo)?; + let layout = load_repository_layout(&path)?; + let package_file = layout.package_file(package_id).ok_or_else(|| { + CliError::PackageEval(format!( + "package '{}' was not found in repository '{}'", + package_id, repo_id + )) + })?; + evaluate_package_file(&layout, &package_file.path) + .map_err(|error| CliError::PackageEval(error.to_string())) +} + +fn evaluate_highest_priority_package( + db: &MainDb, + package_id: &PackageId, +) -> Result { + for repo in db.repositories()? { + let path = repo_path(&repo)?; + let layout = load_repository_layout(&path)?; + if let Some(package_file) = layout.package_file(package_id) { + return evaluate_package_file(&layout, &package_file.path) + .map_err(|error| CliError::PackageEval(error.to_string())); + } + } + Err(CliError::PackageEval(format!( + "package '{package_id}' was not found in any registered repository" + ))) +} + +fn repo_path(repo: &StoredRepository) -> Result { + repo.path + .as_ref() + .map(PathBuf::from) + .ok_or_else(|| CliError::Repository(format!("repository '{}' has no path", repo.id))) +} + +fn repository_json(repo: StoredRepository) -> Value { + json!({ + "id": repo.id.as_str(), + "name": repo.name, + "priority": repo.priority.value(), + "api_version": repo.api_version, + "path": repo.path, + "revision": repo.revision, + }) +} + +fn repository_metadata_json( + metadata: &RepositoryMetadata, + path: Option<&Path>, + revision: Option<&str>, +) -> Value { + json!({ + "id": metadata.id.as_str(), + "name": metadata.name, + "priority": metadata.priority.value(), + "api_version": metadata.api_version, + "path": path.map(|path| path.to_string_lossy().to_string()), + "revision": revision, + }) +} + +fn package_json(package: getter_core::ResolvedPackage) -> Result { + serde_json::to_value(package) + .map_err(|source| CliError::PackageEval(format!("failed to serialize package: {source}"))) +} + +fn tracked_packages_json(packages: Vec) -> Vec { + packages + .into_iter() + .map(|package| { + json!({ + "id": package.package_id.to_string(), + "enabled": package.enabled, + "favorite": package.favorite, + "ignored_version": package.ignored_version, + "repository_id": package.repository_id.map(|id| id.to_string()), + "package_resolution": package.package_resolution.as_str(), + }) + }) + .collect() +} + +fn import_legacy_room_bundle(db: &MainDb, bundle: &LegacyRoomBundle) -> Result<(), CliError> { + for app in &bundle.apps { + let mapping = map_legacy_app( + &LegacyAppRecord { + kind: app.kind.to_legacy_kind()?, + installed_id: app.installed_id.clone(), + official_package_available: app.official_package_available, + common_conversion_available: app.common_conversion_available, + }, + Some(&LegacyExtraAppRecord { + ignored_version: app.ignored_version.clone(), + favorite: app.favorite, + }), + ) + .map_err(|source| CliError::Storage(source.to_string()))?; + db.upsert_tracked_package(&TrackedPackageUpsert { + package_id: mapping.package_id, + enabled: true, + favorite: mapping.user_state.favorite, + ignored_version: mapping.user_state.ignored_version, + repository_id: None, + package_resolution: stored_resolution(mapping.package_resolution), + })?; + } + + let report_json = json!({ + "ok": true, + "source": "legacy-room-bundle", + "imported_records": bundle.apps.len(), + }) + .to_string(); + db.insert_migration_record("legacy-room-v17", "legacy-room-bundle", &report_json)?; + Ok(()) +} + +fn stored_resolution(resolution: LegacyPackageResolution) -> StoredPackageResolution { + match resolution { + LegacyPackageResolution::OfficialRepositoryPackage => { + StoredPackageResolution::OfficialRepositoryPackage + } + LegacyPackageResolution::GenerateLocalPackage => { + StoredPackageResolution::GenerateLocalPackage + } + LegacyPackageResolution::MissingPackageDefinition => { + StoredPackageResolution::MissingPackageDefinition + } + } +} + +fn main_db_path(data_dir: &Path) -> PathBuf { + data_dir.join(MAIN_DB_FILE) +} + +fn cache_db_path(data_dir: &Path) -> PathBuf { + data_dir.join(CACHE_DB_FILE) +} + +fn create_migration_report( + data_dir: &Path, + bundle: &Path, + code: &str, + detail: &str, + imported_records: u64, + tracked_records: u64, +) -> Result { + let reports_dir = data_dir.join(MIGRATION_REPORTS_DIR); + fs::create_dir_all(&reports_dir).map_err(|source| { + CliError::Storage(format!( + "failed to create migration report directory: {source}" + )) + })?; + let report_path = reports_dir.join(report_file_name(code)); + let report = MigrationReport { + ok: code == "migration.imported", + code, + message: match code { + "migration.invalid_bundle" => "Legacy Room export bundle is invalid", + "migration.unsupported_bundle" => "Legacy Room bundle import is not implemented yet", + "migration.imported" => "Legacy Room bundle imported", + _ => "Legacy migration failed", + }, + bundle_file_name: bundle.file_name().and_then(|name| name.to_str()), + detail, + imported_records, + tracked_records, + }; + let bytes = serde_json::to_vec_pretty(&report) + .map_err(|source| CliError::Storage(format!("failed to serialize report: {source}")))?; + fs::write(&report_path, bytes) + .map_err(|source| CliError::Storage(format!("failed to write report: {source}")))?; + Ok(report_path) +} + +fn report_file_name(code: &str) -> String { + format!("{}.json", code.replace('.', "-")) +} + +#[derive(Debug, Serialize)] +struct MigrationReport<'a> { + ok: bool, + code: &'a str, + message: &'a str, + bundle_file_name: Option<&'a str>, + detail: &'a str, + imported_records: u64, + tracked_records: u64, +} + +fn success_envelope(command: &str, data: Value) -> String { + envelope_to_string(json!({ + "ok": true, + "command": command, + "data": data, + "warnings": [] + })) +} + +fn error_envelope(command: &str, error: &CliError) -> String { + let mut error_value = json!({ + "code": error.code(), + "message": error.message(), + }); + if let Some(detail) = error.detail() { + error_value["detail"] = json!(detail); + } + if let Some(report_path) = error.report_path() { + error_value["report_path"] = json!(report_path); + } + + envelope_to_string(json!({ + "ok": false, + "command": command, + "error": error_value, + })) +} + +fn envelope_to_string(value: Value) -> String { + let mut rendered = serde_json::to_string_pretty(&value).expect("JSON envelope is serializable"); + rendered.push('\n'); + rendered +} + +fn usage_text() -> String { + "Usage: getter --data-dir [--priority ]|repo eval |package eval [--repo ]|storage validate|hub list|legacy import-room-bundle >\n".to_owned() +} + +#[derive(Debug, Deserialize)] +struct LegacyRoomBundle { + format: String, + version: u32, + #[serde(default)] + apps: Vec, +} + +#[derive(Debug, Deserialize)] +struct LegacyBundleApp { + kind: LegacyBundleAppKind, + installed_id: String, + #[serde(default)] + official_package_available: bool, + #[serde(default)] + common_conversion_available: bool, + #[serde(default)] + ignored_version: Option, + #[serde(default)] + favorite: bool, +} + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "snake_case")] +enum LegacyBundleAppKind { + Android, + Magisk, +} + +impl LegacyBundleAppKind { + fn to_legacy_kind(&self) -> Result { + Ok(match self { + Self::Android => LegacyAppKind::Android, + Self::Magisk => LegacyAppKind::Magisk, + }) + } +} + +impl CliCommand { + fn name(&self) -> &'static str { + match self { + Self::Init => "init", + Self::AppList => "app list", + Self::HubList => "hub list", + Self::RepoList => "repo list", + Self::RepoAdd { .. } => "repo add", + Self::RepoEval { .. } => "repo eval", + Self::PackageEval { .. } => "package eval", + Self::StorageValidate => "storage validate", + Self::LegacyImportRoomBundle { .. } => "legacy import-room-bundle", + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parses_global_data_dir_and_app_list_command() { + let invocation = + parse_args(["getter", "--data-dir", "/tmp/ua-getter", "app", "list"]).unwrap(); + + assert_eq!(invocation.data_dir, PathBuf::from("/tmp/ua-getter")); + assert_eq!(invocation.command, CliCommand::AppList); + } + + #[test] + fn run_init_creates_sqlite_database_files_and_json_envelope() { + let temp = tempfile::tempdir().unwrap(); + let data_dir = temp.path().join("getter-data"); + + let output = run([ + "getter".to_owned(), + "--data-dir".to_owned(), + data_dir.to_string_lossy().to_string(), + "init".to_owned(), + ]); + + assert_eq!(output.exit_code, ExitCode::Success); + assert!(data_dir.join(MAIN_DB_FILE).is_file()); + assert!(data_dir.join(CACHE_DB_FILE).is_file()); + let json: Value = serde_json::from_str(&output.stdout).unwrap(); + assert_eq!(json["ok"], true); + assert_eq!(json["command"], "init"); + } + + #[test] + fn malformed_bundle_report_does_not_create_imported_state() { + let temp = tempfile::tempdir().unwrap(); + let data_dir = temp.path().join("getter-data"); + let bundle = temp.path().join("bad.json"); + fs::write(&bundle, "not-json").unwrap(); + + let init = run([ + "getter".to_owned(), + "--data-dir".to_owned(), + data_dir.to_string_lossy().to_string(), + "init".to_owned(), + ]); + assert_eq!(init.exit_code, ExitCode::Success); + + let output = run([ + "getter".to_owned(), + "--data-dir".to_owned(), + data_dir.to_string_lossy().to_string(), + "legacy".to_owned(), + "import-room-bundle".to_owned(), + bundle.to_string_lossy().to_string(), + ]); + + assert_eq!(output.exit_code, ExitCode::Migration); + let json: Value = serde_json::from_str(&output.stdout).unwrap(); + assert_eq!(json["ok"], false); + assert_eq!(json["error"]["code"], "migration.invalid_bundle"); + let report_path = json["error"]["report_path"].as_str().unwrap(); + assert!(Path::new(report_path).is_file()); + } +} diff --git a/crates/getter-cli/src/main.rs b/crates/getter-cli/src/main.rs new file mode 100644 index 0000000..dc49483 --- /dev/null +++ b/crates/getter-cli/src/main.rs @@ -0,0 +1,10 @@ +fn main() { + let output = getter_cli::run(std::env::args()); + if !output.stdout.is_empty() { + print!("{}", output.stdout); + } + if !output.stderr.is_empty() { + eprint!("{}", output.stderr); + } + std::process::exit(output.exit_code.code()); +} diff --git a/crates/getter-cli/tests/bdd_cli.rs b/crates/getter-cli/tests/bdd_cli.rs new file mode 100644 index 0000000..807aaed --- /dev/null +++ b/crates/getter-cli/tests/bdd_cli.rs @@ -0,0 +1,486 @@ +use cucumber::{given, then, when, World as _}; +use serde_json::Value; +use std::fs; +use std::path::PathBuf; +use std::process::Command; +use tempfile::TempDir; + +#[derive(Debug, Default, cucumber::World)] +struct CliWorld { + temp: Option, + data_dir: Option, + bundle: Option, + fixture_repo_id: Option, + fixture_repo_path: Option, + fixture_package_id: Option, + output: Option, + json: Option, +} + +#[given("an empty getter data directory")] +fn empty_getter_data_dir(world: &mut CliWorld) { + let temp = tempfile::tempdir().expect("create tempdir"); + let data_dir = temp.path().join("getter-data"); + world.temp = Some(temp); + world.data_dir = Some(data_dir); + world.json = None; +} + +#[given("an initialized getter data directory")] +fn initialized_getter_data_dir(world: &mut CliWorld) { + empty_getter_data_dir(world); + let output = run_getter(world, ["init".to_owned()]); + assert_success(&output); +} + +#[given("a corrupted legacy export bundle")] +fn corrupted_legacy_export_bundle(world: &mut CliWorld) { + let temp = world.temp.as_ref().expect("tempdir exists"); + let bundle = temp.path().join("corrupted-room-bundle.json"); + fs::write(&bundle, "not-json").expect("write corrupted bundle"); + world.bundle = Some(bundle); +} + +#[given("a syntactically valid legacy export bundle with an Android app")] +fn valid_legacy_export_bundle_with_android_app(world: &mut CliWorld) { + let temp = world.temp.as_ref().expect("tempdir exists"); + let bundle = temp.path().join("valid-room-bundle.json"); + fs::write( + &bundle, + r#"{ + "format": "upgradeall-legacy-room-bundle", + "version": 17, + "apps": [ + { + "kind": "android", + "installed_id": "org.fdroid.fdroid", + "official_package_available": true, + "ignored_version": "1.20.0", + "favorite": true + } + ] +}"#, + ) + .expect("write valid bundle"); + world.bundle = Some(bundle); +} + +#[given(expr = "a fixture Lua repository {string} with package {string}")] +fn fixture_lua_repository(world: &mut CliWorld, repo_id: String, package_id: String) { + create_fixture_lua_repository(world, repo_id, package_id, "F-Droid".to_owned()); +} + +#[given(expr = "a fixture Lua repository {string} with package {string} named {string}")] +fn fixture_lua_repository_named( + world: &mut CliWorld, + repo_id: String, + package_id: String, + package_name: String, +) { + create_fixture_lua_repository(world, repo_id, package_id, package_name); +} + +#[when("I run getter init for that directory")] +fn run_getter_init(world: &mut CliWorld) { + let output = run_getter(world, ["init".to_owned()]); + world.output = Some(output); + world.json = None; +} + +#[when("I run getter app list for that directory")] +fn run_getter_app_list(world: &mut CliWorld) { + let output = run_getter(world, ["app".to_owned(), "list".to_owned()]); + world.output = Some(output); + world.json = None; +} + +#[when("I run getter hub list for that directory")] +fn run_getter_hub_list(world: &mut CliWorld) { + let output = run_getter(world, ["hub".to_owned(), "list".to_owned()]); + world.output = Some(output); + world.json = None; +} + +#[when("I run getter repo list for that directory")] +fn run_getter_repo_list(world: &mut CliWorld) { + let output = run_getter(world, ["repo".to_owned(), "list".to_owned()]); + world.output = Some(output); + world.json = None; +} + +#[when("I run getter storage validate for that directory")] +fn run_getter_storage_validate(world: &mut CliWorld) { + let output = run_getter(world, ["storage".to_owned(), "validate".to_owned()]); + world.output = Some(output); + world.json = None; +} + +#[when("I run getter repo add for that repository with priority 0")] +fn run_getter_repo_add(world: &mut CliWorld) { + let repo_id = world + .fixture_repo_id + .as_ref() + .expect("fixture repo id exists") + .clone(); + run_getter_repo_add_with_priority(world, &repo_id, 0); +} + +#[when(expr = "I run getter repo add for repository {string} with priority {int}")] +fn run_getter_repo_add_named(world: &mut CliWorld, repo_id: String, priority: i32) { + run_getter_repo_add_with_priority(world, &repo_id, priority); +} + +#[when("I run getter repo eval for that repository")] +fn run_getter_repo_eval(world: &mut CliWorld) { + let repo_id = world + .fixture_repo_id + .as_ref() + .expect("fixture repo id exists"); + let output = run_getter( + world, + ["repo".to_owned(), "eval".to_owned(), repo_id.to_owned()], + ); + world.output = Some(output); + world.json = None; +} + +#[when("I run getter package eval for that fixture package")] +fn run_getter_package_eval(world: &mut CliWorld) { + let repo_id = world + .fixture_repo_id + .as_ref() + .expect("fixture repo id exists"); + let package_id = world + .fixture_package_id + .as_ref() + .expect("fixture package id exists"); + let output = run_getter( + world, + [ + "package".to_owned(), + "eval".to_owned(), + package_id.to_owned(), + "--repo".to_owned(), + repo_id.to_owned(), + ], + ); + world.output = Some(output); + world.json = None; +} + +#[when(expr = "I run getter package eval for package {string}")] +fn run_getter_package_eval_without_repo(world: &mut CliWorld, package_id: String) { + let output = run_getter(world, ["package".to_owned(), "eval".to_owned(), package_id]); + world.output = Some(output); + world.json = None; +} + +#[when("I run getter legacy import-room-bundle for that bundle")] +fn run_getter_legacy_import(world: &mut CliWorld) { + let bundle = world.bundle.as_ref().expect("bundle exists"); + let output = run_getter( + world, + [ + "legacy".to_owned(), + "import-room-bundle".to_owned(), + bundle.to_string_lossy().to_string(), + ], + ); + world.output = Some(output); + world.json = None; +} + +#[then("the command succeeds")] +fn command_succeeds(world: &mut CliWorld) { + let output = world.output.as_ref().expect("command output exists"); + assert_success(output); +} + +#[then("the command fails with a documented migration error")] +fn command_fails_with_migration_error(world: &mut CliWorld) { + let output = world.output.as_ref().expect("command output exists"); + assert_eq!(output.status.code(), Some(20)); + let json = parse_stdout(output); + assert_eq!(json["ok"], false); + assert_eq!(json["command"], "legacy import-room-bundle"); + assert_eq!(json["error"]["code"], "migration.invalid_bundle"); + world.json = Some(json); +} + +#[then("the output is valid JSON")] +fn output_is_valid_json(world: &mut CliWorld) { + let output = world.output.as_ref().expect("command output exists"); + let json = parse_stdout(output); + assert!( + json.is_object(), + "stdout JSON should be an object: {json:?}" + ); + world.json = Some(json); +} + +#[then("the getter data directory is usable")] +fn getter_data_directory_is_usable(world: &mut CliWorld) { + let data_dir = world.data_dir.as_ref().expect("data dir exists"); + assert!(data_dir.join("main.db").is_file(), "main.db should exist"); + assert!(data_dir.join("cache.db").is_file(), "cache.db should exist"); + + let output = run_getter(world, ["app".to_owned(), "list".to_owned()]); + assert_success(&output); +} + +#[then("the output contains an empty app list")] +fn output_contains_empty_app_list(world: &mut CliWorld) { + let json = current_json(world); + assert_eq!(json["ok"], true); + assert_eq!(json["command"], "app list"); + assert_eq!(json["data"]["apps"], Value::Array(Vec::new())); +} + +#[then("the output contains an empty hub list")] +fn output_contains_empty_hub_list(world: &mut CliWorld) { + let json = current_json(world); + assert_eq!(json["ok"], true); + assert_eq!(json["command"], "hub list"); + assert_eq!(json["data"]["hubs"], Value::Array(Vec::new())); +} + +#[then("the output contains an empty repository list")] +fn output_contains_empty_repository_list(world: &mut CliWorld) { + let json = current_json(world); + assert_eq!(json["ok"], true); + assert_eq!(json["command"], "repo list"); + assert_eq!(json["data"]["repositories"], Value::Array(Vec::new())); +} + +#[then("the output reports valid storage")] +fn output_reports_valid_storage(world: &mut CliWorld) { + let json = current_json(world); + assert_eq!(json["ok"], true); + assert_eq!(json["command"], "storage validate"); + assert_eq!(json["data"]["valid"], true); + assert!(json["data"]["main_db"].as_str().is_some()); + assert!(json["data"]["cache_db"].as_str().is_some()); +} + +#[then("the output contains the added repository")] +fn output_contains_added_repository(world: &mut CliWorld) { + let repo_id = world + .fixture_repo_id + .as_ref() + .expect("fixture repo id exists") + .clone(); + let json = current_json(world); + assert_eq!(json["ok"], true); + assert_eq!(json["command"], "repo add"); + assert_eq!(json["data"]["repository"]["id"], repo_id); +} + +#[then("the output contains the evaluated fixture package")] +fn output_contains_evaluated_fixture_package(world: &mut CliWorld) { + let package_id = world + .fixture_package_id + .as_ref() + .expect("fixture package id exists") + .clone(); + let json = current_json(world); + assert_eq!(json["ok"], true); + assert_eq!(json["command"], "repo eval"); + let packages = json["data"]["packages"] + .as_array() + .expect("packages should be an array"); + assert!( + packages + .iter() + .any(|package| package["id"].as_str() == Some(package_id.as_str())), + "repo eval packages should include {package_id}: {packages:?}" + ); +} + +#[then("the output contains the fixture package")] +fn output_contains_fixture_package(world: &mut CliWorld) { + let package_id = world + .fixture_package_id + .as_ref() + .expect("fixture package id exists") + .clone(); + let json = current_json(world); + assert_eq!(json["ok"], true); + assert_eq!(json["command"], "package eval"); + assert_eq!(json["data"]["package"]["id"], package_id); + assert_eq!(json["data"]["package"]["name"], "F-Droid"); +} + +#[then(expr = "the output contains package {string} named {string}")] +fn output_contains_named_package(world: &mut CliWorld, package_id: String, package_name: String) { + let json = current_json(world); + assert_eq!(json["ok"], true); + assert_eq!(json["command"], "package eval"); + assert_eq!(json["data"]["package"]["id"], package_id); + assert_eq!(json["data"]["package"]["name"], package_name); +} + +#[then("no partially usable imported state is created")] +fn no_partially_usable_imported_state(world: &mut CliWorld) { + let output = run_getter(world, ["app".to_owned(), "list".to_owned()]); + assert_success(&output); + let json = parse_stdout(&output); + assert_eq!(json["data"]["apps"], Value::Array(Vec::new())); +} + +#[then("a sanitized migration report is available")] +fn sanitized_migration_report_available(world: &mut CliWorld) { + let json = current_json(world); + let report_path = json["error"]["report_path"] + .as_str() + .expect("report_path should be a string"); + let report = fs::read_to_string(report_path).expect("report should be readable"); + let report_json: Value = serde_json::from_str(&report).expect("report should be JSON"); + assert_eq!(report_json["ok"], false); + assert_eq!(report_json["imported_records"], 0); + assert!(report_json.get("bundle_file_name").is_some()); + assert!( + report_json.get("raw_bundle").is_none(), + "report must not include raw bundle content" + ); +} + +#[then("the import reports one tracked app")] +fn import_reports_one_tracked_app(world: &mut CliWorld) { + let json = current_json(world); + assert_eq!(json["ok"], true); + assert_eq!(json["command"], "legacy import-room-bundle"); + assert_eq!(json["data"]["imported_records"], 1); + assert_eq!( + json["data"]["apps"].as_array().expect("apps array").len(), + 1 + ); +} + +#[then(expr = "the app list contains imported package {string}")] +fn app_list_contains_imported_package(world: &mut CliWorld, package_id: String) { + let output = run_getter(world, ["app".to_owned(), "list".to_owned()]); + assert_success(&output); + let json = parse_stdout(&output); + let apps = json["data"]["apps"].as_array().expect("apps array"); + let app = apps + .iter() + .find(|app| app["id"].as_str() == Some(package_id.as_str())) + .unwrap_or_else(|| panic!("app list should contain {package_id}: {apps:?}")); + assert_eq!(app["favorite"], true); + assert_eq!(app["ignored_version"], "1.20.0"); + assert_eq!(app["package_resolution"], "official_repository_package"); + world.output = Some(output); + world.json = Some(json); +} + +fn current_json(world: &mut CliWorld) -> &Value { + world + .json + .get_or_insert_with(|| parse_stdout(world.output.as_ref().expect("command output exists"))) +} + +fn create_fixture_lua_repository( + world: &mut CliWorld, + repo_id: String, + package_id: String, + package_name: String, +) { + let temp = world.temp.as_ref().expect("tempdir exists"); + let repo_path = temp.path().join(format!("repo-{repo_id}")); + let package_name_path = package_id + .strip_prefix("android/") + .expect("fixture package id should be android package id"); + fs::create_dir_all(repo_path.join("packages/android")).expect("create packages dir"); + fs::create_dir(repo_path.join("lib")).expect("create lib dir"); + fs::create_dir(repo_path.join("templates")).expect("create templates dir"); + fs::write( + repo_path.join("repo.toml"), + format!( + "id = \"{repo_id}\"\nname = \"Fixture {repo_id}\"\npriority = 0\napi_version = \"getter.repo.v1\"\n" + ), + ) + .expect("write repo.toml"); + fs::write( + repo_path.join(format!("packages/android/{package_name_path}.lua")), + format!( + r#" +return package_def {{ + id = "{package_id}", + name = "{package_name}", + installed = {{ + {{ kind = "android_package", package_name = "{package_name_path}" }}, + }}, +}} +"# + ), + ) + .expect("write package Lua"); + + world.fixture_repo_id = Some(repo_id); + world.fixture_repo_path = Some(repo_path); + world.fixture_package_id = Some(package_id); +} + +fn run_getter_repo_add_with_priority(world: &mut CliWorld, repo_id: &str, priority: i32) { + let temp = world.temp.as_ref().expect("tempdir exists"); + let repo_path = temp.path().join(format!("repo-{repo_id}")); + let output = run_getter( + world, + [ + "repo".to_owned(), + "add".to_owned(), + repo_id.to_owned(), + repo_path.to_string_lossy().to_string(), + "--priority".to_owned(), + priority.to_string(), + ], + ); + world.output = Some(output); + world.json = None; +} + +fn run_getter(world: &CliWorld, command_args: I) -> std::process::Output +where + I: IntoIterator, +{ + let exe = env!("CARGO_BIN_EXE_getter-cli"); + let data_dir = world.data_dir.as_ref().expect("data dir exists"); + let mut command = Command::new(exe); + command.arg("--data-dir").arg(data_dir); + for arg in command_args { + command.arg(arg); + } + command.output().expect("run getter binary") +} + +fn assert_success(output: &std::process::Output) { + assert_eq!( + output.status.code(), + Some(0), + "stdout:\n{}\nstderr:\n{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + let json = parse_stdout(output); + assert_eq!(json["ok"], true); +} + +fn parse_stdout(output: &std::process::Output) -> Value { + serde_json::from_slice(&output.stdout).unwrap_or_else(|error| { + panic!( + "stdout should be JSON: {error}\nstdout:\n{}\nstderr:\n{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ) + }) +} + +#[tokio::main] +async fn main() { + let features = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/features/cli"); + CliWorld::cucumber() + .fail_on_skipped() + .max_concurrent_scenarios(Some(1)) + .run_and_exit(features) + .await; +} diff --git a/crates/getter-cli/tests/features/cli/app_list.feature b/crates/getter-cli/tests/features/cli/app_list.feature new file mode 100644 index 0000000..3baa858 --- /dev/null +++ b/crates/getter-cli/tests/features/cli/app_list.feature @@ -0,0 +1,8 @@ +@getter-cli @smoke +Feature: Getter CLI app listing + Scenario: User lists apps before adding any app records + Given an initialized getter data directory + When I run getter app list for that directory + Then the command succeeds + And the output is valid JSON + And the output contains an empty app list diff --git a/crates/getter-cli/tests/features/cli/hub_list.feature b/crates/getter-cli/tests/features/cli/hub_list.feature new file mode 100644 index 0000000..3e467a2 --- /dev/null +++ b/crates/getter-cli/tests/features/cli/hub_list.feature @@ -0,0 +1,8 @@ +@getter-cli @smoke +Feature: Getter CLI hub listing + Scenario: User lists hubs before adding any hub records + Given an initialized getter data directory + When I run getter hub list for that directory + Then the command succeeds + And the output is valid JSON + And the output contains an empty hub list diff --git a/crates/getter-cli/tests/features/cli/init.feature b/crates/getter-cli/tests/features/cli/init.feature new file mode 100644 index 0000000..51cb38a --- /dev/null +++ b/crates/getter-cli/tests/features/cli/init.feature @@ -0,0 +1,8 @@ +@getter-cli @smoke +Feature: Getter CLI initialization + Scenario: User initializes a new getter data directory + Given an empty getter data directory + When I run getter init for that directory + Then the command succeeds + And the output is valid JSON + And the getter data directory is usable diff --git a/crates/getter-cli/tests/features/cli/legacy_import_room_bundle_failure.feature b/crates/getter-cli/tests/features/cli/legacy_import_room_bundle_failure.feature new file mode 100644 index 0000000..864417f --- /dev/null +++ b/crates/getter-cli/tests/features/cli/legacy_import_room_bundle_failure.feature @@ -0,0 +1,18 @@ +@getter-cli @migration +Feature: Legacy import failure recovery + Scenario: User receives a non-destructive report when legacy import fails + Given an initialized getter data directory + And a corrupted legacy export bundle + When I run getter legacy import-room-bundle for that bundle + Then the command fails with a documented migration error + And no partially usable imported state is created + And a sanitized migration report is available + + Scenario: User imports a valid legacy bundle into tracked app state + Given an initialized getter data directory + And a syntactically valid legacy export bundle with an Android app + When I run getter legacy import-room-bundle for that bundle + Then the command succeeds + And the output is valid JSON + And the import reports one tracked app + And the app list contains imported package "android/org.fdroid.fdroid" diff --git a/crates/getter-cli/tests/features/cli/repo_list.feature b/crates/getter-cli/tests/features/cli/repo_list.feature new file mode 100644 index 0000000..0ffd2d8 --- /dev/null +++ b/crates/getter-cli/tests/features/cli/repo_list.feature @@ -0,0 +1,8 @@ +@getter-cli @smoke +Feature: Getter CLI repository listing + Scenario: User lists repositories before adding any repository records + Given an initialized getter data directory + When I run getter repo list for that directory + Then the command succeeds + And the output is valid JSON + And the output contains an empty repository list diff --git a/crates/getter-cli/tests/features/cli/repository_overlay.feature b/crates/getter-cli/tests/features/cli/repository_overlay.feature new file mode 100644 index 0000000..d5906cb --- /dev/null +++ b/crates/getter-cli/tests/features/cli/repository_overlay.feature @@ -0,0 +1,14 @@ +@getter-cli @repository +Feature: Getter CLI repository overlay resolution + Scenario: Highest-priority repository package wins by package id + Given an initialized getter data directory + And a fixture Lua repository "official" with package "android/org.fdroid.fdroid" named "Official F-Droid" + And a fixture Lua repository "local" with package "android/org.fdroid.fdroid" named "Local F-Droid" + When I run getter repo add for repository "official" with priority 0 + Then the command succeeds + When I run getter repo add for repository "local" with priority 100 + Then the command succeeds + When I run getter package eval for package "android/org.fdroid.fdroid" + Then the command succeeds + And the output is valid JSON + And the output contains package "android/org.fdroid.fdroid" named "Local F-Droid" diff --git a/crates/getter-cli/tests/features/cli/repository_package_eval.feature b/crates/getter-cli/tests/features/cli/repository_package_eval.feature new file mode 100644 index 0000000..4edefc2 --- /dev/null +++ b/crates/getter-cli/tests/features/cli/repository_package_eval.feature @@ -0,0 +1,17 @@ +@getter-cli @repository +Feature: Getter CLI repository and package evaluation + Scenario: User adds and evaluates a fixture Lua repository + Given an initialized getter data directory + And a fixture Lua repository "official" with package "android/org.fdroid.fdroid" + When I run getter repo add for that repository with priority 0 + Then the command succeeds + And the output is valid JSON + And the output contains the added repository + When I run getter repo eval for that repository + Then the command succeeds + And the output is valid JSON + And the output contains the evaluated fixture package + When I run getter package eval for that fixture package + Then the command succeeds + And the output is valid JSON + And the output contains the fixture package diff --git a/crates/getter-cli/tests/features/cli/storage_validate.feature b/crates/getter-cli/tests/features/cli/storage_validate.feature new file mode 100644 index 0000000..815edb3 --- /dev/null +++ b/crates/getter-cli/tests/features/cli/storage_validate.feature @@ -0,0 +1,8 @@ +@getter-cli @smoke +Feature: Getter CLI storage validation + Scenario: User validates initialized getter storage + Given an initialized getter data directory + When I run getter storage validate for that directory + Then the command succeeds + And the output is valid JSON + And the output reports valid storage diff --git a/crates/getter-core/Cargo.toml b/crates/getter-core/Cargo.toml new file mode 100644 index 0000000..47326c5 --- /dev/null +++ b/crates/getter-core/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "getter-core" +version.workspace = true +edition.workspace = true + +[dependencies] +mlua = { version = "0.10", features = ["lua54", "vendored"] } +serde = { version = "1", features = ["derive"] } +serde_json = "1" +thiserror = "1" +toml = "0.8" + +[dev-dependencies] +tempfile = "3" diff --git a/crates/getter-core/src/lib.rs b/crates/getter-core/src/lib.rs new file mode 100644 index 0000000..45a464d --- /dev/null +++ b/crates/getter-core/src/lib.rs @@ -0,0 +1,408 @@ +//! Core domain model for the UpgradeAll getter rewrite. +//! +//! `getter-core` owns product/domain behavior. Flutter and Android code are +//! platform/UI adapters and must not reimplement package, repository, update or +//! storage rules. + +use serde::{Deserialize, Serialize}; +use std::cmp::Ordering; +use std::fmt; +use std::str::FromStr; + +pub mod lua; +pub mod repository; + +/// Error returned when parsing or constructing a [`PackageId`]. +#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] +pub enum PackageIdError { + #[error("package id is empty")] + Empty, + #[error("package id must be '/'")] + MissingSeparator, + #[error("package id kind is empty")] + EmptyKind, + #[error("package id name is empty")] + EmptyName, + #[error("unsupported package kind '{0}'")] + UnsupportedKind(String), + #[error("package id kind '{0}' contains invalid characters")] + InvalidKind(String), + #[error("package id name '{0}' contains invalid characters")] + InvalidName(String), +} + +/// Known package target kinds in v1. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum PackageKind { + Android, + Magisk, + Generic, +} + +impl PackageKind { + pub fn as_str(self) -> &'static str { + match self { + Self::Android => "android", + Self::Magisk => "magisk", + Self::Generic => "generic", + } + } +} + +impl fmt::Display for PackageKind { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(self.as_str()) + } +} + +impl FromStr for PackageKind { + type Err = PackageIdError; + + fn from_str(value: &str) -> Result { + match value { + "android" => Ok(Self::Android), + "magisk" => Ok(Self::Magisk), + "generic" => Ok(Self::Generic), + other => Err(PackageIdError::UnsupportedKind(other.to_owned())), + } + } +} + +/// Readable package identifier such as `android/org.fdroid.fdroid`. +#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[serde(try_from = "String", into = "String")] +pub struct PackageId { + kind: PackageKind, + name: String, +} + +impl PackageId { + pub fn new(kind: PackageKind, name: impl Into) -> Result { + let name = name.into(); + validate_name(&name)?; + Ok(Self { kind, name }) + } + + pub fn kind(&self) -> PackageKind { + self.kind + } + + pub fn name(&self) -> &str { + &self.name + } +} + +impl fmt::Display for PackageId { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}/{}", self.kind, self.name) + } +} + +impl FromStr for PackageId { + type Err = PackageIdError; + + fn from_str(value: &str) -> Result { + if value.is_empty() { + return Err(PackageIdError::Empty); + } + let (kind, name) = value + .split_once('/') + .ok_or(PackageIdError::MissingSeparator)?; + if kind.is_empty() { + return Err(PackageIdError::EmptyKind); + } + if name.is_empty() { + return Err(PackageIdError::EmptyName); + } + validate_kind_segment(kind)?; + validate_name(name)?; + Ok(Self { + kind: kind.parse()?, + name: name.to_owned(), + }) + } +} + +impl TryFrom for PackageId { + type Error = PackageIdError; + + fn try_from(value: String) -> Result { + value.parse() + } +} + +impl From for String { + fn from(value: PackageId) -> Self { + value.to_string() + } +} + +/// Repository identifier such as `official`, `community`, `local`, or +/// `local_autogen`. +#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[serde(try_from = "String", into = "String")] +pub struct RepositoryId(String); + +impl RepositoryId { + pub fn new(value: impl Into) -> Result { + let value = value.into(); + validate_repository_id(&value)?; + Ok(Self(value)) + } + + pub fn as_str(&self) -> &str { + &self.0 + } +} + +impl fmt::Display for RepositoryId { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(&self.0) + } +} + +impl FromStr for RepositoryId { + type Err = RepositoryIdError; + + fn from_str(value: &str) -> Result { + Self::new(value) + } +} + +impl TryFrom for RepositoryId { + type Error = RepositoryIdError; + + fn try_from(value: String) -> Result { + Self::new(value) + } +} + +impl From for String { + fn from(value: RepositoryId) -> Self { + value.0 + } +} + +#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] +pub enum RepositoryIdError { + #[error("repository id is empty")] + Empty, + #[error("repository id '{0}' contains invalid characters")] + Invalid(String), +} + +/// Higher repository priority wins during overlay resolution. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] +pub struct RepositoryPriority(i32); + +impl RepositoryPriority { + pub const LOCAL: Self = Self(100); + pub const DEFAULT: Self = Self(0); + pub const LOCAL_AUTOGEN: Self = Self(-1); + + pub const fn new(value: i32) -> Self { + Self(value) + } + + pub const fn value(self) -> i32 { + self.0 + } + + pub fn cmp_winner(self, other: Self) -> Ordering { + self.0.cmp(&other.0) + } +} + +impl Default for RepositoryPriority { + fn default() -> Self { + Self::DEFAULT + } +} + +impl Ord for RepositoryPriority { + fn cmp(&self, other: &Self) -> Ordering { + self.0.cmp(&other.0) + } +} + +impl PartialOrd for RepositoryPriority { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +/// Package metadata after repository resolution and Lua validation. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct ResolvedPackage { + pub id: PackageId, + pub repository: RepositoryId, + pub name: String, + #[serde(default)] + pub installed: Vec, + #[serde(default)] + pub permissions: PackagePermissions, + #[serde(default)] + pub source_priority: Vec, +} + +/// Installed target matched by a package, such as an Android package name. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "snake_case")] +pub enum InstalledTarget { + AndroidPackage { package_name: String }, + MagiskModule { module_id: String }, + Generic { id: String }, +} + +/// Package permission declaration relevant to UI warnings and Lua host APIs. +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct PackagePermissions { + #[serde(default)] + pub free_network: bool, +} + +/// Candidate version/update discovered by provider/package logic. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct UpdateCandidate { + pub version: String, + #[serde(default)] + pub channel: Option, + #[serde(default)] + pub source: Option, + #[serde(default)] + pub artifacts: Vec, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct UpdateArtifact { + pub name: String, + pub url: String, + #[serde(default)] + pub file_name: Option, +} + +/// Candidate selected for update after package/user-state policy. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct SelectedUpdate { + pub package_id: PackageId, + pub candidate: UpdateCandidate, + #[serde(default)] + pub artifact: Option, +} + +/// Executable update actions generated by the `resolve` lifecycle phase. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "type", rename_all = "snake_case")] +pub enum UpdateAction { + Download { url: String, file_name: String }, + Install { installer: String, file: String }, + OpenUrl { url: String }, +} + +fn validate_kind_segment(value: &str) -> Result<(), PackageIdError> { + if value + .chars() + .all(|c| c.is_ascii_lowercase() || c.is_ascii_digit() || c == '_' || c == '-') + { + Ok(()) + } else { + Err(PackageIdError::InvalidKind(value.to_owned())) + } +} + +fn validate_name(value: &str) -> Result<(), PackageIdError> { + if value.is_empty() { + return Err(PackageIdError::EmptyName); + } + if value.starts_with('/') || value.ends_with('/') || value.contains("//") { + return Err(PackageIdError::InvalidName(value.to_owned())); + } + if value + .chars() + .all(|c| c.is_ascii_alphanumeric() || matches!(c, '.' | '_' | '-' | '+' | '/')) + { + Ok(()) + } else { + Err(PackageIdError::InvalidName(value.to_owned())) + } +} + +fn validate_repository_id(value: &str) -> Result<(), RepositoryIdError> { + if value.is_empty() { + return Err(RepositoryIdError::Empty); + } + if value + .chars() + .all(|c| c.is_ascii_alphanumeric() || matches!(c, '_' | '-' | '.')) + { + Ok(()) + } else { + Err(RepositoryIdError::Invalid(value.to_owned())) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parses_readable_android_package_id() { + let id: PackageId = "android/org.fdroid.fdroid".parse().unwrap(); + assert_eq!(id.kind(), PackageKind::Android); + assert_eq!(id.name(), "org.fdroid.fdroid"); + assert_eq!(id.to_string(), "android/org.fdroid.fdroid"); + } + + #[test] + fn parses_readable_magisk_package_id() { + let id: PackageId = "magisk/zygisk-next".parse().unwrap(); + assert_eq!(id.kind(), PackageKind::Magisk); + assert_eq!(id.name(), "zygisk-next"); + assert_eq!(id.to_string(), "magisk/zygisk-next"); + } + + #[test] + fn rejects_invalid_package_ids() { + assert_eq!("".parse::(), Err(PackageIdError::Empty)); + assert_eq!( + "android".parse::(), + Err(PackageIdError::MissingSeparator) + ); + assert_eq!( + "/org.fdroid.fdroid".parse::(), + Err(PackageIdError::EmptyKind) + ); + assert_eq!( + "android/".parse::(), + Err(PackageIdError::EmptyName) + ); + assert_eq!( + "hub/org.fdroid.fdroid".parse::(), + Err(PackageIdError::UnsupportedKind("hub".to_owned())) + ); + assert!("android/org fdroid".parse::().is_err()); + } + + #[test] + fn repository_priority_higher_number_wins() { + assert!(RepositoryPriority::LOCAL > RepositoryPriority::DEFAULT); + assert!(RepositoryPriority::DEFAULT > RepositoryPriority::LOCAL_AUTOGEN); + assert_eq!(RepositoryPriority::LOCAL.value(), 100); + assert_eq!(RepositoryPriority::DEFAULT.value(), 0); + assert_eq!(RepositoryPriority::LOCAL_AUTOGEN.value(), -1); + } + + #[test] + fn repository_id_accepts_named_repositories() { + assert_eq!( + RepositoryId::new("local_autogen").unwrap().as_str(), + "local_autogen" + ); + assert_eq!( + RepositoryId::new("community.1").unwrap().to_string(), + "community.1" + ); + assert!(RepositoryId::new("bad/repo").is_err()); + } +} diff --git a/crates/getter-core/src/lua.rs b/crates/getter-core/src/lua.rs new file mode 100644 index 0000000..fd6348e --- /dev/null +++ b/crates/getter-core/src/lua.rs @@ -0,0 +1,460 @@ +//! Minimal Lua package-file evaluation and Rust validation boundary. + +use crate::repository::RepositoryLayout; +use crate::{InstalledTarget, PackageId, PackagePermissions, ResolvedPackage}; +use mlua::{Lua, Table, Value}; +use serde_json::{Map, Number, Value as JsonValue}; +use std::fs; +use std::path::{Path, PathBuf}; + +#[derive(Debug, thiserror::Error)] +pub enum LuaPackageError { + #[error("failed to read Lua package file {path}: {source}")] + ReadFile { + path: PathBuf, + #[source] + source: std::io::Error, + }, + #[error("Lua runtime error in {path}: {source}")] + Runtime { + path: PathBuf, + #[source] + source: mlua::Error, + }, + #[error("Lua package {path} did not return a table")] + NotATable { path: PathBuf }, + #[error("Lua package {path} returned unsupported value at {location}: {value_type}")] + UnsupportedValue { + path: PathBuf, + location: String, + value_type: &'static str, + }, + #[error("schema validation failed for {path}: {message}")] + Schema { path: PathBuf, message: String }, + #[error("domain validation failed for {path}: {message}")] + Domain { path: PathBuf, message: String }, +} + +/// Evaluate and validate a Lua package file from a repository layout. +pub fn evaluate_package_file( + repository: &RepositoryLayout, + path: impl AsRef, +) -> Result { + let path = path.as_ref(); + let source = fs::read_to_string(path).map_err(|source| LuaPackageError::ReadFile { + path: path.to_path_buf(), + source, + })?; + evaluate_package_source(repository, path, &source) +} + +/// Evaluate package source text. This is public for focused tests and future CLI +/// plumbing; repository callers should normally use [`evaluate_package_file`]. +pub fn evaluate_package_source( + repository: &RepositoryLayout, + path: impl AsRef, + source: &str, +) -> Result { + let path = path.as_ref().to_path_buf(); + let lua = Lua::new(); + configure_package_path(&lua, repository).map_err(|source| LuaPackageError::Runtime { + path: path.clone(), + source, + })?; + install_helpers(&lua).map_err(|source| LuaPackageError::Runtime { + path: path.clone(), + source, + })?; + + let value = lua + .load(source) + .set_name(path.to_string_lossy().as_ref()) + .eval::() + .map_err(|source| LuaPackageError::Runtime { + path: path.clone(), + source, + })?; + let table = match value { + Value::Table(table) => table, + _ => return Err(LuaPackageError::NotATable { path }), + }; + let json = lua_table_to_json(&path, "$", table)?; + validate_package_json(repository, &path, json) +} + +fn configure_package_path(lua: &Lua, repository: &RepositoryLayout) -> mlua::Result<()> { + let package: Table = lua.globals().get("package")?; + let current_path: String = package.get("path")?; + let lib_pattern = repository.lib_dir.join("?.lua"); + let nested_lib_pattern = repository.lib_dir.join("?/init.lua"); + let new_path = format!( + "{};{};{}", + lib_pattern.to_string_lossy(), + nested_lib_pattern.to_string_lossy(), + current_path + ); + package.set("path", new_path) +} + +fn install_helpers(lua: &Lua) -> mlua::Result<()> { + let package_fn = lua.create_function(|_, table: Table| Ok(table))?; + lua.globals().set("package_def", package_fn.clone())?; + lua.globals().set("android_app", package_fn.clone())?; + lua.globals().set("magisk_module", package_fn.clone())?; + lua.globals().set("generic_package", package_fn)?; + Ok(()) +} + +fn lua_table_to_json( + path: &Path, + location: &str, + table: Table, +) -> Result { + if is_array_table(&table).map_err(|source| LuaPackageError::Runtime { + path: path.to_path_buf(), + source, + })? { + let mut array = Vec::new(); + for pair in table.sequence_values::() { + let value = pair.map_err(|source| LuaPackageError::Runtime { + path: path.to_path_buf(), + source, + })?; + array.push(lua_value_to_json(path, &format!("{location}[]"), value)?); + } + Ok(JsonValue::Array(array)) + } else { + let mut object = Map::new(); + for pair in table.pairs::() { + let (key, value) = pair.map_err(|source| LuaPackageError::Runtime { + path: path.to_path_buf(), + source, + })?; + let key = match key { + Value::String(value) => value + .to_str() + .map_err(|source| LuaPackageError::Runtime { + path: path.to_path_buf(), + source, + })? + .to_owned(), + Value::Integer(value) => value.to_string(), + _ => { + return Err(LuaPackageError::UnsupportedValue { + path: path.to_path_buf(), + location: format!("{location}."), + value_type: key.type_name(), + }) + } + }; + let child_location = format!("{location}.{key}"); + object.insert(key, lua_value_to_json(path, &child_location, value)?); + } + Ok(JsonValue::Object(object)) + } +} + +fn lua_value_to_json( + path: &Path, + location: &str, + value: Value, +) -> Result { + match value { + Value::Nil => Ok(JsonValue::Null), + Value::Boolean(value) => Ok(JsonValue::Bool(value)), + Value::Integer(value) => Ok(JsonValue::Number(Number::from(value))), + Value::Number(value) => Number::from_f64(value) + .map(JsonValue::Number) + .ok_or_else(|| LuaPackageError::UnsupportedValue { + path: path.to_path_buf(), + location: location.to_owned(), + value_type: "non-finite number", + }), + Value::String(value) => Ok(JsonValue::String( + value + .to_str() + .map_err(|source| LuaPackageError::Runtime { + path: path.to_path_buf(), + source, + })? + .to_owned(), + )), + Value::Table(table) => lua_table_to_json(path, location, table), + _ => Err(LuaPackageError::UnsupportedValue { + path: path.to_path_buf(), + location: location.to_owned(), + value_type: value.type_name(), + }), + } +} + +fn is_array_table(table: &Table) -> mlua::Result { + let len = table.raw_len(); + if len == 0 { + // Empty tables are accepted as objects by default. Package schemas avoid + // ambiguous empty array/object requirements at this boundary. + return Ok(false); + } + let mut count = 0usize; + for pair in table.clone().pairs::() { + let (key, _) = pair?; + match key { + Value::Integer(index) if index >= 1 && (index as usize) <= len => count += 1, + _ => return Ok(false), + } + } + Ok(count == len) +} + +fn validate_package_json( + repository: &RepositoryLayout, + path: &Path, + value: JsonValue, +) -> Result { + let object = value.as_object().ok_or_else(|| LuaPackageError::Schema { + path: path.to_path_buf(), + message: "package value must be an object".to_owned(), + })?; + + let id = required_string(path, object, "id")?; + let id: PackageId = id.parse().map_err(|source| LuaPackageError::Schema { + path: path.to_path_buf(), + message: format!("invalid package id: {source}"), + })?; + let path_id = crate::repository::package_id_from_path(&repository.packages_dir, path).map_err( + |source| LuaPackageError::Domain { + path: path.to_path_buf(), + message: format!("failed to derive package id from path: {source}"), + }, + )?; + if id != path_id { + return Err(LuaPackageError::Domain { + path: path.to_path_buf(), + message: format!("package id '{id}' does not match path-derived id '{path_id}'"), + }); + } + + let name = required_string(path, object, "name")?.to_owned(); + let installed = parse_installed_targets(path, object.get("installed"))?; + let permissions = parse_permissions(path, object.get("permissions"))?; + let source_priority = + parse_string_array(path, "source_priority", object.get("source_priority"))?; + + Ok(ResolvedPackage { + id, + repository: repository.metadata.id.clone(), + name, + installed, + permissions, + source_priority, + }) +} + +fn required_string<'a>( + path: &Path, + object: &'a Map, + field: &str, +) -> Result<&'a str, LuaPackageError> { + object + .get(field) + .and_then(JsonValue::as_str) + .ok_or_else(|| LuaPackageError::Schema { + path: path.to_path_buf(), + message: format!("required string field '{field}' is missing"), + }) +} + +fn parse_installed_targets( + path: &Path, + value: Option<&JsonValue>, +) -> Result, LuaPackageError> { + let Some(value) = value else { + return Ok(Vec::new()); + }; + let array = value.as_array().ok_or_else(|| LuaPackageError::Schema { + path: path.to_path_buf(), + message: "field 'installed' must be an array".to_owned(), + })?; + array + .iter() + .map(|item| { + let object = item.as_object().ok_or_else(|| LuaPackageError::Schema { + path: path.to_path_buf(), + message: "installed entries must be objects".to_owned(), + })?; + let kind = required_string(path, object, "kind")?; + match kind { + "android_package" => Ok(InstalledTarget::AndroidPackage { + package_name: required_string(path, object, "package_name")?.to_owned(), + }), + "magisk_module" => Ok(InstalledTarget::MagiskModule { + module_id: required_string(path, object, "module_id")?.to_owned(), + }), + "generic" => Ok(InstalledTarget::Generic { + id: required_string(path, object, "id")?.to_owned(), + }), + other => Err(LuaPackageError::Schema { + path: path.to_path_buf(), + message: format!("unknown installed target kind '{other}'"), + }), + } + }) + .collect() +} + +fn parse_permissions( + path: &Path, + value: Option<&JsonValue>, +) -> Result { + let Some(value) = value else { + return Ok(PackagePermissions::default()); + }; + let object = value.as_object().ok_or_else(|| LuaPackageError::Schema { + path: path.to_path_buf(), + message: "field 'permissions' must be an object".to_owned(), + })?; + Ok(PackagePermissions { + free_network: object + .get("free_network") + .and_then(JsonValue::as_bool) + .unwrap_or(false), + }) +} + +fn parse_string_array( + path: &Path, + field: &str, + value: Option<&JsonValue>, +) -> Result, LuaPackageError> { + let Some(value) = value else { + return Ok(Vec::new()); + }; + let array = value.as_array().ok_or_else(|| LuaPackageError::Schema { + path: path.to_path_buf(), + message: format!("field '{field}' must be an array"), + })?; + array + .iter() + .map(|item| { + item.as_str() + .map(str::to_owned) + .ok_or_else(|| LuaPackageError::Schema { + path: path.to_path_buf(), + message: format!("field '{field}' must contain only strings"), + }) + }) + .collect() +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::repository::RepositoryLayout; + use std::fs; + + fn fixture_repo() -> (tempfile::TempDir, RepositoryLayout, PathBuf) { + let temp = tempfile::tempdir().unwrap(); + let root = temp.path(); + fs::write( + root.join("repo.toml"), + r#"id = "official" +name = "UpgradeAll Official" +priority = 0 +api_version = "getter.repo.v1" +"#, + ) + .unwrap(); + fs::create_dir(root.join("packages")).unwrap(); + fs::create_dir(root.join("packages/android")).unwrap(); + fs::create_dir(root.join("lib")).unwrap(); + fs::create_dir(root.join("templates")).unwrap(); + let package_path = root.join("packages/android/org.fdroid.fdroid.lua"); + fs::write(&package_path, "return {}").unwrap(); + let layout = RepositoryLayout::load(root).unwrap(); + (temp, layout, package_path) + } + + #[test] + fn evaluates_json_like_lua_package_table() { + let (_temp, layout, package_path) = fixture_repo(); + fs::write( + &package_path, + r#" +return package_def { + id = "android/org.fdroid.fdroid", + name = "F-Droid", + installed = { + { kind = "android_package", package_name = "org.fdroid.fdroid" }, + }, + permissions = { free_network = true }, + source_priority = { "github", "fdroid" }, +} +"#, + ) + .unwrap(); + + let package = evaluate_package_file(&layout, &package_path).unwrap(); + assert_eq!(package.id.to_string(), "android/org.fdroid.fdroid"); + assert_eq!(package.repository.as_str(), "official"); + assert_eq!(package.name, "F-Droid"); + assert_eq!( + package.installed, + vec![InstalledTarget::AndroidPackage { + package_name: "org.fdroid.fdroid".to_owned() + }] + ); + assert!(package.permissions.free_network); + assert_eq!(package.source_priority, vec!["github", "fdroid"]); + } + + #[test] + fn rejects_package_id_that_does_not_match_path() { + let (_temp, layout, package_path) = fixture_repo(); + fs::write( + &package_path, + r#"return { id = "android/com.termux", name = "Termux" }"#, + ) + .unwrap(); + + let err = evaluate_package_file(&layout, &package_path).unwrap_err(); + assert!(matches!(err, LuaPackageError::Domain { .. })); + } + + #[test] + fn require_can_load_repository_lib_modules() { + let (_temp, layout, package_path) = fixture_repo(); + fs::write( + layout.lib_dir.join("android.lua"), + r#" +return { + local_app = function(input) + return { + id = input.id, + name = input.name, + installed = { + { kind = "android_package", package_name = input.package_name }, + }, + } + end +} +"#, + ) + .unwrap(); + fs::write( + &package_path, + r#" +local android = require("android") +return android.local_app { + id = "android/org.fdroid.fdroid", + name = "F-Droid", + package_name = "org.fdroid.fdroid", +} +"#, + ) + .unwrap(); + + let package = evaluate_package_file(&layout, &package_path).unwrap(); + assert_eq!(package.name, "F-Droid"); + assert_eq!(package.installed.len(), 1); + } +} diff --git a/crates/getter-core/src/repository.rs b/crates/getter-core/src/repository.rs new file mode 100644 index 0000000..0bacf47 --- /dev/null +++ b/crates/getter-core/src/repository.rs @@ -0,0 +1,288 @@ +//! Repository layout loading for Lua package repositories. + +use crate::{PackageId, PackageIdError, RepositoryId, RepositoryIdError, RepositoryPriority}; +use serde::Deserialize; +use std::fs; +use std::path::{Path, PathBuf}; + +pub const REPO_API_VERSION_V1: &str = "getter.repo.v1"; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct RepositoryLayout { + pub root: PathBuf, + pub metadata: RepositoryMetadata, + pub packages_dir: PathBuf, + pub lib_dir: PathBuf, + pub templates_dir: PathBuf, + pub packages: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct RepositoryMetadata { + pub id: RepositoryId, + pub name: String, + pub priority: RepositoryPriority, + pub api_version: String, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct PackageFile { + pub id: PackageId, + pub path: PathBuf, +} + +#[derive(Debug, thiserror::Error)] +pub enum RepositoryLoadError { + #[error("failed to read repo.toml at {path}: {source}")] + ReadRepoToml { + path: PathBuf, + #[source] + source: std::io::Error, + }, + #[error("failed to parse repo.toml at {path}: {source}")] + ParseRepoToml { + path: PathBuf, + #[source] + source: toml::de::Error, + }, + #[error("invalid repository id in repo.toml: {0}")] + RepositoryId(#[from] RepositoryIdError), + #[error("unsupported repository api_version '{0}'")] + UnsupportedApiVersion(String), + #[error("repository path {path} is missing required directory '{directory}'")] + MissingDirectory { + path: PathBuf, + directory: &'static str, + }, + #[error("failed to read packages directory {path}: {source}")] + ReadPackagesDir { + path: PathBuf, + #[source] + source: std::io::Error, + }, + #[error("invalid package path {path}: {reason}")] + InvalidPackagePath { path: PathBuf, reason: String }, + #[error("invalid package id derived from {path}: {source}")] + PackageId { + path: PathBuf, + #[source] + source: PackageIdError, + }, +} + +impl RepositoryLayout { + pub fn load(root: impl AsRef) -> Result { + let root = root.as_ref().to_path_buf(); + let repo_toml_path = root.join("repo.toml"); + let raw = fs::read_to_string(&repo_toml_path).map_err(|source| { + RepositoryLoadError::ReadRepoToml { + path: repo_toml_path.clone(), + source, + } + })?; + let raw_metadata: RawRepositoryMetadata = + toml::from_str(&raw).map_err(|source| RepositoryLoadError::ParseRepoToml { + path: repo_toml_path.clone(), + source, + })?; + let api_version = raw_metadata.api_version; + if api_version != REPO_API_VERSION_V1 { + return Err(RepositoryLoadError::UnsupportedApiVersion(api_version)); + } + let metadata = RepositoryMetadata { + id: RepositoryId::new(raw_metadata.id)?, + name: raw_metadata.name, + priority: RepositoryPriority::new(raw_metadata.priority.unwrap_or_default()), + api_version, + }; + + let packages_dir = root.join("packages"); + let lib_dir = root.join("lib"); + let templates_dir = root.join("templates"); + require_dir(&root, &packages_dir, "packages")?; + require_dir(&root, &lib_dir, "lib")?; + require_dir(&root, &templates_dir, "templates")?; + + let mut packages = Vec::new(); + collect_package_files(&packages_dir, &packages_dir, &mut packages)?; + packages.sort_by(|a, b| a.id.to_string().cmp(&b.id.to_string())); + + Ok(Self { + root, + metadata, + packages_dir, + lib_dir, + templates_dir, + packages, + }) + } + + pub fn package_file(&self, id: &PackageId) -> Option<&PackageFile> { + self.packages.iter().find(|package| &package.id == id) + } +} + +#[derive(Debug, Deserialize)] +struct RawRepositoryMetadata { + id: String, + name: String, + #[serde(default)] + priority: Option, + api_version: String, +} + +fn require_dir( + root: &Path, + path: &Path, + directory: &'static str, +) -> Result<(), RepositoryLoadError> { + if path.is_dir() { + Ok(()) + } else { + Err(RepositoryLoadError::MissingDirectory { + path: root.to_path_buf(), + directory, + }) + } +} + +fn collect_package_files( + packages_root: &Path, + current: &Path, + out: &mut Vec, +) -> Result<(), RepositoryLoadError> { + for entry in fs::read_dir(current).map_err(|source| RepositoryLoadError::ReadPackagesDir { + path: current.to_path_buf(), + source, + })? { + let entry = entry.map_err(|source| RepositoryLoadError::ReadPackagesDir { + path: current.to_path_buf(), + source, + })?; + let path = entry.path(); + let file_type = + entry + .file_type() + .map_err(|source| RepositoryLoadError::ReadPackagesDir { + path: current.to_path_buf(), + source, + })?; + if file_type.is_dir() { + collect_package_files(packages_root, &path, out)?; + } else if file_type.is_file() && path.extension().is_some_and(|ext| ext == "lua") { + let id = package_id_from_path(packages_root, &path)?; + out.push(PackageFile { id, path }); + } + } + Ok(()) +} + +pub fn package_id_from_path( + packages_root: impl AsRef, + path: impl AsRef, +) -> Result { + let packages_root = packages_root.as_ref(); + let path = path.as_ref(); + let relative = + path.strip_prefix(packages_root) + .map_err(|_| RepositoryLoadError::InvalidPackagePath { + path: path.to_path_buf(), + reason: format!("path is not under {}", packages_root.display()), + })?; + if relative.extension().is_none_or(|ext| ext != "lua") { + return Err(RepositoryLoadError::InvalidPackagePath { + path: path.to_path_buf(), + reason: "package file must have .lua extension".to_owned(), + }); + } + let without_extension = relative.with_extension(""); + let mut parts = without_extension.components(); + let kind = parts + .next() + .ok_or_else(|| RepositoryLoadError::InvalidPackagePath { + path: path.to_path_buf(), + reason: "missing package kind directory".to_owned(), + })? + .as_os_str() + .to_string_lossy() + .into_owned(); + let name_path: PathBuf = parts.collect(); + let name = name_path + .to_string_lossy() + .replace(std::path::MAIN_SEPARATOR, "/"); + if name.is_empty() { + return Err(RepositoryLoadError::InvalidPackagePath { + path: path.to_path_buf(), + reason: "missing package name".to_owned(), + }); + } + format!("{kind}/{name}") + .parse() + .map_err(|source| RepositoryLoadError::PackageId { + path: path.to_path_buf(), + source, + }) +} + +pub fn highest_priority<'a, T, F>(items: &'a [T], priority: F) -> Option<&'a T> +where + F: Fn(&T) -> RepositoryPriority, +{ + items.iter().max_by_key(|item| priority(item)) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::io::Write; + + #[test] + fn derives_package_id_from_lua_path() { + let root = PathBuf::from("repo/packages"); + let id = + package_id_from_path(&root, "repo/packages/android/org.fdroid.fdroid.lua").unwrap(); + assert_eq!(id.to_string(), "android/org.fdroid.fdroid"); + } + + #[test] + fn loads_repository_layout_with_required_directories() { + let temp = tempfile::tempdir().unwrap(); + let root = temp.path(); + fs::write( + root.join("repo.toml"), + r#"id = "official" +name = "UpgradeAll Official" +priority = 0 +api_version = "getter.repo.v1" +"#, + ) + .unwrap(); + fs::create_dir(root.join("packages")).unwrap(); + fs::create_dir(root.join("packages/android")).unwrap(); + fs::create_dir(root.join("lib")).unwrap(); + fs::create_dir(root.join("templates")).unwrap(); + let mut file = + fs::File::create(root.join("packages/android/org.fdroid.fdroid.lua")).unwrap(); + writeln!(file, "return {{ id = 'android/org.fdroid.fdroid' }}").unwrap(); + + let layout = RepositoryLayout::load(root).unwrap(); + assert_eq!(layout.metadata.id.as_str(), "official"); + assert_eq!(layout.metadata.priority, RepositoryPriority::DEFAULT); + assert_eq!(layout.packages.len(), 1); + assert_eq!( + layout.packages[0].id.to_string(), + "android/org.fdroid.fdroid" + ); + } + + #[test] + fn highest_priority_selects_larger_number() { + let priorities = [ + RepositoryPriority::LOCAL_AUTOGEN, + RepositoryPriority::DEFAULT, + RepositoryPriority::LOCAL, + ]; + let selected = highest_priority(&priorities, |priority| *priority).unwrap(); + assert_eq!(*selected, RepositoryPriority::LOCAL); + } +} diff --git a/crates/getter-downloader/Cargo.toml b/crates/getter-downloader/Cargo.toml new file mode 100644 index 0000000..d8b2562 --- /dev/null +++ b/crates/getter-downloader/Cargo.toml @@ -0,0 +1,7 @@ +[package] +name = "getter-downloader" +version.workspace = true +edition.workspace = true + +[dependencies] +getter-core = { path = "../getter-core" } diff --git a/crates/getter-downloader/src/lib.rs b/crates/getter-downloader/src/lib.rs new file mode 100644 index 0000000..4a11c7d --- /dev/null +++ b/crates/getter-downloader/src/lib.rs @@ -0,0 +1,3 @@ +//! getter-downloader rewrite crate skeleton. + +pub use getter_core as core; diff --git a/crates/getter-ffi/Cargo.toml b/crates/getter-ffi/Cargo.toml new file mode 100644 index 0000000..fa1c2cf --- /dev/null +++ b/crates/getter-ffi/Cargo.toml @@ -0,0 +1,7 @@ +[package] +name = "getter-ffi" +version.workspace = true +edition.workspace = true + +[dependencies] +getter-core = { path = "../getter-core" } diff --git a/crates/getter-ffi/src/lib.rs b/crates/getter-ffi/src/lib.rs new file mode 100644 index 0000000..b7dde13 --- /dev/null +++ b/crates/getter-ffi/src/lib.rs @@ -0,0 +1,3 @@ +//! getter-ffi rewrite crate skeleton. + +pub use getter_core as core; diff --git a/crates/getter-plugin-api/Cargo.toml b/crates/getter-plugin-api/Cargo.toml new file mode 100644 index 0000000..4f41ccf --- /dev/null +++ b/crates/getter-plugin-api/Cargo.toml @@ -0,0 +1,7 @@ +[package] +name = "getter-plugin-api" +version.workspace = true +edition.workspace = true + +[dependencies] +getter-core = { path = "../getter-core" } diff --git a/crates/getter-plugin-api/src/lib.rs b/crates/getter-plugin-api/src/lib.rs new file mode 100644 index 0000000..52cd1d1 --- /dev/null +++ b/crates/getter-plugin-api/src/lib.rs @@ -0,0 +1,3 @@ +//! getter-plugin-api rewrite crate skeleton. + +pub use getter_core as core; diff --git a/crates/getter-providers/Cargo.toml b/crates/getter-providers/Cargo.toml new file mode 100644 index 0000000..5a44be9 --- /dev/null +++ b/crates/getter-providers/Cargo.toml @@ -0,0 +1,7 @@ +[package] +name = "getter-providers" +version.workspace = true +edition.workspace = true + +[dependencies] +getter-core = { path = "../getter-core" } diff --git a/crates/getter-providers/src/lib.rs b/crates/getter-providers/src/lib.rs new file mode 100644 index 0000000..35371aa --- /dev/null +++ b/crates/getter-providers/src/lib.rs @@ -0,0 +1,3 @@ +//! getter-providers rewrite crate skeleton. + +pub use getter_core as core; diff --git a/crates/getter-rpc/Cargo.toml b/crates/getter-rpc/Cargo.toml new file mode 100644 index 0000000..6ab6be0 --- /dev/null +++ b/crates/getter-rpc/Cargo.toml @@ -0,0 +1,7 @@ +[package] +name = "getter-rpc" +version.workspace = true +edition.workspace = true + +[dependencies] +getter-core = { path = "../getter-core" } diff --git a/crates/getter-rpc/src/lib.rs b/crates/getter-rpc/src/lib.rs new file mode 100644 index 0000000..ceb75d9 --- /dev/null +++ b/crates/getter-rpc/src/lib.rs @@ -0,0 +1,3 @@ +//! getter-rpc rewrite crate skeleton. + +pub use getter_core as core; diff --git a/crates/getter-storage/Cargo.toml b/crates/getter-storage/Cargo.toml new file mode 100644 index 0000000..6a561a4 --- /dev/null +++ b/crates/getter-storage/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "getter-storage" +version.workspace = true +edition.workspace = true + +[dependencies] +getter-core = { path = "../getter-core" } +rusqlite = { version = "0.32", features = ["bundled"] } +thiserror = "1" + +[dev-dependencies] +tempfile = "3" diff --git a/crates/getter-storage/src/legacy_room.rs b/crates/getter-storage/src/legacy_room.rs new file mode 100644 index 0000000..798809e --- /dev/null +++ b/crates/getter-storage/src/legacy_room.rs @@ -0,0 +1,215 @@ +//! Legacy Room migration mapping tests and pure mapping helpers. +//! +//! This module intentionally contains no Android Room/database reader. It is the +//! TDD boundary for source->target mapping rules before the full migration +//! implementation is added. + +use getter_core::PackageId; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum LegacyAppKind { + Android, + Magisk, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct LegacyAppRecord { + pub kind: LegacyAppKind, + pub installed_id: String, + pub official_package_available: bool, + pub common_conversion_available: bool, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct LegacyExtraAppRecord { + pub ignored_version: Option, + pub favorite: bool, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct LegacyAppMapping { + pub package_id: PackageId, + pub package_resolution: LegacyPackageResolution, + pub user_state: LegacyUserStateMapping, + pub warnings: Vec, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum LegacyPackageResolution { + OfficialRepositoryPackage, + GenerateLocalPackage, + MissingPackageDefinition, +} + +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct LegacyUserStateMapping { + pub ignored_version: Option, + pub favorite: bool, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum LegacyMigrationWarning { + MissingPackageDefinition { package_id: PackageId }, +} + +#[derive(Debug, thiserror::Error)] +pub enum LegacyRoomMappingError { + #[error("legacy installed id is empty")] + EmptyInstalledId, + #[error("failed to construct target package id: {0}")] + PackageId(#[from] getter_core::PackageIdError), +} + +pub fn map_legacy_app( + app: &LegacyAppRecord, + extra: Option<&LegacyExtraAppRecord>, +) -> Result { + let package_id = map_legacy_package_id(app.kind, &app.installed_id)?; + let package_resolution = if app.official_package_available { + LegacyPackageResolution::OfficialRepositoryPackage + } else if app.common_conversion_available { + LegacyPackageResolution::GenerateLocalPackage + } else { + LegacyPackageResolution::MissingPackageDefinition + }; + let warnings = match package_resolution { + LegacyPackageResolution::MissingPackageDefinition => { + vec![LegacyMigrationWarning::MissingPackageDefinition { + package_id: package_id.clone(), + }] + } + _ => Vec::new(), + }; + let user_state = LegacyUserStateMapping { + ignored_version: extra.and_then(|extra| extra.ignored_version.clone()), + favorite: extra.is_some_and(|extra| extra.favorite), + }; + + Ok(LegacyAppMapping { + package_id, + package_resolution, + user_state, + warnings, + }) +} + +pub fn map_legacy_package_id( + kind: LegacyAppKind, + installed_id: &str, +) -> Result { + if installed_id.is_empty() { + return Err(LegacyRoomMappingError::EmptyInstalledId); + } + let prefix = match kind { + LegacyAppKind::Android => "android", + LegacyAppKind::Magisk => "magisk", + }; + Ok(format!("{prefix}/{installed_id}").parse()?) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn android_legacy_app_maps_to_readable_android_package_id() { + let mapping = map_legacy_app( + &LegacyAppRecord { + kind: LegacyAppKind::Android, + installed_id: "org.fdroid.fdroid".to_owned(), + official_package_available: true, + common_conversion_available: false, + }, + None, + ) + .unwrap(); + + assert_eq!(mapping.package_id.to_string(), "android/org.fdroid.fdroid"); + assert_eq!( + mapping.package_resolution, + LegacyPackageResolution::OfficialRepositoryPackage + ); + assert!(mapping.warnings.is_empty()); + } + + #[test] + fn magisk_legacy_app_maps_to_readable_magisk_package_id() { + let mapping = map_legacy_app( + &LegacyAppRecord { + kind: LegacyAppKind::Magisk, + installed_id: "zygisk-next".to_owned(), + official_package_available: true, + common_conversion_available: false, + }, + None, + ) + .unwrap(); + + assert_eq!(mapping.package_id.to_string(), "magisk/zygisk-next"); + } + + #[test] + fn common_unofficial_conversion_generates_local_package() { + let mapping = map_legacy_app( + &LegacyAppRecord { + kind: LegacyAppKind::Android, + installed_id: "com.example.private".to_owned(), + official_package_available: false, + common_conversion_available: true, + }, + None, + ) + .unwrap(); + + assert_eq!( + mapping.package_resolution, + LegacyPackageResolution::GenerateLocalPackage + ); + assert!(mapping.warnings.is_empty()); + } + + #[test] + fn unmapped_complex_app_preserves_id_and_records_missing_definition_warning() { + let mapping = map_legacy_app( + &LegacyAppRecord { + kind: LegacyAppKind::Android, + installed_id: "com.example.unmapped".to_owned(), + official_package_available: false, + common_conversion_available: false, + }, + None, + ) + .unwrap(); + + assert_eq!( + mapping.package_resolution, + LegacyPackageResolution::MissingPackageDefinition + ); + assert_eq!( + mapping.warnings, + vec![LegacyMigrationWarning::MissingPackageDefinition { + package_id: "android/com.example.unmapped".parse().unwrap(), + }] + ); + } + + #[test] + fn extra_app_ignore_and_favorite_state_are_preserved_when_present() { + let mapping = map_legacy_app( + &LegacyAppRecord { + kind: LegacyAppKind::Android, + installed_id: "org.fdroid.fdroid".to_owned(), + official_package_available: true, + common_conversion_available: false, + }, + Some(&LegacyExtraAppRecord { + ignored_version: Some("1.2.3".to_owned()), + favorite: true, + }), + ) + .unwrap(); + + assert_eq!(mapping.user_state.ignored_version.as_deref(), Some("1.2.3")); + assert!(mapping.user_state.favorite); + } +} diff --git a/crates/getter-storage/src/lib.rs b/crates/getter-storage/src/lib.rs new file mode 100644 index 0000000..6eed591 --- /dev/null +++ b/crates/getter-storage/src/lib.rs @@ -0,0 +1,491 @@ +//! SQLite storage skeleton for getter main/cache databases. + +pub mod legacy_room; + +use getter_core::repository::RepositoryMetadata; +use getter_core::{PackageId, RepositoryId, RepositoryPriority}; +use rusqlite::{params, Connection}; +use std::path::Path; +use std::str::FromStr; + +#[derive(Debug, thiserror::Error)] +pub enum StorageError { + #[error("sqlite error: {0}")] + Sqlite(#[from] rusqlite::Error), + #[error("invalid package id in database: {0}")] + PackageId(#[from] getter_core::PackageIdError), + #[error("invalid repository id in database: {0}")] + RepositoryId(#[from] getter_core::RepositoryIdError), + #[error("invalid package resolution in database: {0}")] + PackageResolution(String), +} + +pub struct MainDb { + conn: Connection, +} + +pub struct CacheDb { + conn: Connection, +} + +impl MainDb { + pub fn open(path: impl AsRef) -> Result { + let conn = Connection::open(path)?; + let db = Self { conn }; + db.migrate()?; + Ok(db) + } + + pub fn open_in_memory() -> Result { + let conn = Connection::open_in_memory()?; + let db = Self { conn }; + db.migrate()?; + Ok(db) + } + + fn migrate(&self) -> Result<(), StorageError> { + self.conn.execute_batch( + r#" +PRAGMA foreign_keys = ON; + +CREATE TABLE IF NOT EXISTS schema_migrations ( + id TEXT PRIMARY KEY, + applied_at_unix INTEGER NOT NULL DEFAULT (unixepoch()) +); + +CREATE TABLE IF NOT EXISTS repositories ( + id TEXT PRIMARY KEY, + name TEXT NOT NULL, + priority INTEGER NOT NULL, + api_version TEXT NOT NULL, + path TEXT, + revision TEXT +); + +CREATE TABLE IF NOT EXISTS tracked_packages ( + package_id TEXT PRIMARY KEY, + enabled INTEGER NOT NULL DEFAULT 1, + favorite INTEGER NOT NULL DEFAULT 0, + ignored_version TEXT, + repository_id TEXT, + package_resolution TEXT NOT NULL DEFAULT 'missing_package_definition', + FOREIGN KEY(repository_id) REFERENCES repositories(id) +); + +CREATE TABLE IF NOT EXISTS migration_records ( + id TEXT PRIMARY KEY, + source TEXT NOT NULL, + completed_at_unix INTEGER NOT NULL DEFAULT (unixepoch()), + report_json TEXT NOT NULL DEFAULT '{}' +); +"#, + )?; + self.ensure_column("tracked_packages", "ignored_version", "TEXT")?; + self.ensure_column( + "tracked_packages", + "package_resolution", + "TEXT NOT NULL DEFAULT 'missing_package_definition'", + )?; + self.conn.execute( + "INSERT OR IGNORE INTO schema_migrations(id) VALUES ('main-v1')", + [], + )?; + Ok(()) + } + + fn ensure_column( + &self, + table: &'static str, + column: &'static str, + definition: &'static str, + ) -> Result<(), StorageError> { + let columns = self.table_columns(table)?; + if !columns.iter().any(|existing| existing == column) { + self.conn.execute_batch(&format!( + "ALTER TABLE {table} ADD COLUMN {column} {definition};" + ))?; + } + Ok(()) + } + + fn table_columns(&self, table: &'static str) -> Result, StorageError> { + let mut stmt = self.conn.prepare(&format!("PRAGMA table_info({table})"))?; + let rows = stmt.query_map([], |row| row.get::<_, String>(1))?; + let mut columns = Vec::new(); + for row in rows { + columns.push(row?); + } + Ok(columns) + } + + pub fn upsert_repository( + &self, + metadata: &RepositoryMetadata, + path: Option<&Path>, + revision: Option<&str>, + ) -> Result<(), StorageError> { + self.conn.execute( + r#" +INSERT INTO repositories(id, name, priority, api_version, path, revision) +VALUES (?1, ?2, ?3, ?4, ?5, ?6) +ON CONFLICT(id) DO UPDATE SET + name = excluded.name, + priority = excluded.priority, + api_version = excluded.api_version, + path = excluded.path, + revision = excluded.revision +"#, + params![ + metadata.id.as_str(), + metadata.name, + metadata.priority.value(), + metadata.api_version, + path.map(|p| p.to_string_lossy().to_string()), + revision, + ], + )?; + Ok(()) + } + + pub fn repositories(&self) -> Result, StorageError> { + let mut stmt = self.conn.prepare( + "SELECT id, name, priority, api_version, path, revision FROM repositories ORDER BY priority DESC, id ASC", + )?; + let rows = stmt.query_map([], |row| { + Ok(( + row.get::<_, String>(0)?, + row.get::<_, String>(1)?, + row.get::<_, i32>(2)?, + row.get::<_, String>(3)?, + row.get::<_, Option>(4)?, + row.get::<_, Option>(5)?, + )) + })?; + let mut repositories = Vec::new(); + for row in rows { + let (id, name, priority, api_version, path, revision) = row?; + repositories.push(StoredRepository { + id: RepositoryId::new(id)?, + name, + priority: RepositoryPriority::new(priority), + api_version, + path, + revision, + }); + } + Ok(repositories) + } + + pub fn upsert_tracked_package( + &self, + package: &TrackedPackageUpsert, + ) -> Result<(), StorageError> { + self.conn.execute( + r#" +INSERT INTO tracked_packages( + package_id, + enabled, + favorite, + ignored_version, + repository_id, + package_resolution +) +VALUES (?1, ?2, ?3, ?4, ?5, ?6) +ON CONFLICT(package_id) DO UPDATE SET + enabled = excluded.enabled, + favorite = excluded.favorite, + ignored_version = excluded.ignored_version, + repository_id = excluded.repository_id, + package_resolution = excluded.package_resolution +"#, + params![ + package.package_id.to_string(), + bool_to_i64(package.enabled), + bool_to_i64(package.favorite), + package.ignored_version.as_deref(), + package.repository_id.as_ref().map(RepositoryId::as_str), + package.package_resolution.as_str(), + ], + )?; + Ok(()) + } + + pub fn tracked_packages(&self) -> Result, StorageError> { + let mut stmt = self.conn.prepare( + r#" +SELECT package_id, enabled, favorite, ignored_version, repository_id, package_resolution +FROM tracked_packages +ORDER BY package_id ASC +"#, + )?; + let rows = stmt.query_map([], |row| { + Ok(( + row.get::<_, String>(0)?, + row.get::<_, i64>(1)?, + row.get::<_, i64>(2)?, + row.get::<_, Option>(3)?, + row.get::<_, Option>(4)?, + row.get::<_, String>(5)?, + )) + })?; + let mut packages = Vec::new(); + for row in rows { + let (package_id, enabled, favorite, ignored_version, repository_id, resolution) = row?; + packages.push(StoredTrackedPackage { + package_id: PackageId::from_str(&package_id)?, + enabled: enabled != 0, + favorite: favorite != 0, + ignored_version, + repository_id: repository_id.map(RepositoryId::new).transpose()?, + package_resolution: StoredPackageResolution::from_str(&resolution)?, + }); + } + Ok(packages) + } + + pub fn insert_migration_record( + &self, + id: &str, + source: &str, + report_json: &str, + ) -> Result<(), StorageError> { + self.conn.execute( + r#" +INSERT INTO migration_records(id, source, report_json) +VALUES (?1, ?2, ?3) +ON CONFLICT(id) DO UPDATE SET + source = excluded.source, + completed_at_unix = unixepoch(), + report_json = excluded.report_json +"#, + params![id, source, report_json], + )?; + Ok(()) + } + + pub fn migration_records(&self) -> Result, StorageError> { + let mut stmt = self.conn.prepare( + "SELECT id, source, report_json FROM migration_records ORDER BY completed_at_unix ASC, id ASC", + )?; + let rows = stmt.query_map([], |row| { + Ok(StoredMigrationRecord { + id: row.get(0)?, + source: row.get(1)?, + report_json: row.get(2)?, + }) + })?; + let mut records = Vec::new(); + for row in rows { + records.push(row?); + } + Ok(records) + } +} + +impl CacheDb { + pub fn open(path: impl AsRef) -> Result { + let conn = Connection::open(path)?; + let db = Self { conn }; + db.migrate()?; + Ok(db) + } + + pub fn open_in_memory() -> Result { + let conn = Connection::open_in_memory()?; + let db = Self { conn }; + db.migrate()?; + Ok(db) + } + + fn migrate(&self) -> Result<(), StorageError> { + self.conn.execute_batch( + r#" +PRAGMA foreign_keys = ON; + +CREATE TABLE IF NOT EXISTS schema_migrations ( + id TEXT PRIMARY KEY, + applied_at_unix INTEGER NOT NULL DEFAULT (unixepoch()) +); + +CREATE TABLE IF NOT EXISTS evaluated_packages ( + cache_key TEXT PRIMARY KEY, + repository_id TEXT NOT NULL, + package_id TEXT NOT NULL, + package_file_hash TEXT NOT NULL, + schema_version TEXT NOT NULL, + evaluated_json TEXT NOT NULL, + evaluated_at_unix INTEGER NOT NULL DEFAULT (unixepoch()) +); + +CREATE TABLE IF NOT EXISTS provider_responses ( + cache_key TEXT PRIMARY KEY, + provider TEXT NOT NULL, + response_json TEXT NOT NULL, + fetched_at_unix INTEGER NOT NULL DEFAULT (unixepoch()) +); +"#, + )?; + self.conn.execute( + "INSERT OR IGNORE INTO schema_migrations(id) VALUES ('cache-v1')", + [], + )?; + Ok(()) + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct StoredRepository { + pub id: RepositoryId, + pub name: String, + pub priority: RepositoryPriority, + pub api_version: String, + pub path: Option, + pub revision: Option, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct TrackedPackageUpsert { + pub package_id: PackageId, + pub enabled: bool, + pub favorite: bool, + pub ignored_version: Option, + pub repository_id: Option, + pub package_resolution: StoredPackageResolution, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct StoredTrackedPackage { + pub package_id: PackageId, + pub enabled: bool, + pub favorite: bool, + pub ignored_version: Option, + pub repository_id: Option, + pub package_resolution: StoredPackageResolution, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct StoredMigrationRecord { + pub id: String, + pub source: String, + pub report_json: String, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum StoredPackageResolution { + OfficialRepositoryPackage, + GenerateLocalPackage, + MissingPackageDefinition, +} + +impl StoredPackageResolution { + pub const fn as_str(self) -> &'static str { + match self { + Self::OfficialRepositoryPackage => "official_repository_package", + Self::GenerateLocalPackage => "generate_local_package", + Self::MissingPackageDefinition => "missing_package_definition", + } + } +} + +impl FromStr for StoredPackageResolution { + type Err = StorageError; + + fn from_str(value: &str) -> Result { + match value { + "official_repository_package" => Ok(Self::OfficialRepositoryPackage), + "generate_local_package" => Ok(Self::GenerateLocalPackage), + "missing_package_definition" => Ok(Self::MissingPackageDefinition), + other => Err(StorageError::PackageResolution(other.to_owned())), + } + } +} + +fn bool_to_i64(value: bool) -> i64 { + if value { + 1 + } else { + 0 + } +} + +#[cfg(test)] +mod tests { + use super::*; + use getter_core::repository::REPO_API_VERSION_V1; + + #[test] + fn main_db_stores_repository_registry_ordered_by_priority() { + let db = MainDb::open_in_memory().unwrap(); + db.upsert_repository( + &RepositoryMetadata { + id: RepositoryId::new("official").unwrap(), + name: "Official".to_owned(), + priority: RepositoryPriority::DEFAULT, + api_version: REPO_API_VERSION_V1.to_owned(), + }, + None, + Some("rev1"), + ) + .unwrap(); + db.upsert_repository( + &RepositoryMetadata { + id: RepositoryId::new("local").unwrap(), + name: "Local".to_owned(), + priority: RepositoryPriority::LOCAL, + api_version: REPO_API_VERSION_V1.to_owned(), + }, + None, + None, + ) + .unwrap(); + + let repositories = db.repositories().unwrap(); + assert_eq!(repositories[0].id.as_str(), "local"); + assert_eq!(repositories[1].id.as_str(), "official"); + } + + #[test] + fn main_db_stores_tracked_package_user_state() { + let db = MainDb::open_in_memory().unwrap(); + db.upsert_tracked_package(&TrackedPackageUpsert { + package_id: "android/org.fdroid.fdroid".parse().unwrap(), + enabled: true, + favorite: true, + ignored_version: Some("1.2.3".to_owned()), + repository_id: None, + package_resolution: StoredPackageResolution::OfficialRepositoryPackage, + }) + .unwrap(); + + let packages = db.tracked_packages().unwrap(); + assert_eq!(packages.len(), 1); + assert_eq!( + packages[0].package_id.to_string(), + "android/org.fdroid.fdroid" + ); + assert!(packages[0].enabled); + assert!(packages[0].favorite); + assert_eq!(packages[0].ignored_version.as_deref(), Some("1.2.3")); + assert_eq!( + packages[0].package_resolution, + StoredPackageResolution::OfficialRepositoryPackage + ); + } + + #[test] + fn main_db_records_migration_completion() { + let db = MainDb::open_in_memory().unwrap(); + db.insert_migration_record("legacy-room-v17", "legacy-room-bundle", r#"{"ok":true}"#) + .unwrap(); + + let records = db.migration_records().unwrap(); + assert_eq!(records.len(), 1); + assert_eq!(records[0].id, "legacy-room-v17"); + assert_eq!(records[0].source, "legacy-room-bundle"); + } + + #[test] + fn cache_db_migrates_schema() { + let _db = CacheDb::open_in_memory().unwrap(); + } +} diff --git a/src/cache.rs b/src/cache.rs deleted file mode 100644 index e96302b..0000000 --- a/src/cache.rs +++ /dev/null @@ -1,34 +0,0 @@ -mod local; -pub mod manager; - -use once_cell::sync::Lazy; -use std::{path::Path, sync::Arc}; -use tokio::sync::Mutex; - -use crate::utils::instance::InstanceContainer; - -use self::manager::CacheManager; - -static INSTANCE_CONTAINER: Lazy> = - Lazy::new(|| InstanceContainer::new(CacheManager::new())); - -pub async fn init_cache_manager(local_cache_dir: &Path) { - get_cache_manager() - .await - .lock() - .await - .set_local_cache_dir(local_cache_dir); -} - -pub async fn init_cache_manager_with_expire(local_cache_path: &Path, expire_time: u64) { - get_cache_manager() - .await - .lock() - .await - .set_local_cache_dir(local_cache_path) - .set_global_expire_time(expire_time); -} - -pub async fn get_cache_manager() -> Arc> { - INSTANCE_CONTAINER.get().await.clone() -} diff --git a/src/cache/local.rs b/src/cache/local.rs deleted file mode 100644 index 22cdcb4..0000000 --- a/src/cache/local.rs +++ /dev/null @@ -1,77 +0,0 @@ -use std::io::Result; -use std::path::{Path, PathBuf}; -use std::time::SystemTime; -use tokio::fs::{metadata, read, remove_file, write}; - -use tokio::fs::create_dir_all; - -pub struct LocalCacheItem { - cache_path: PathBuf, -} - -impl LocalCacheItem { - pub fn new(cache_dir: &Path, key: &str) -> Self { - Self { - cache_path: cache_dir.join(key), - } - } - - pub async fn get(&self, decoder: fn(Vec) -> T) -> Result { - read(&self.cache_path).await.map(decoder) - } - - pub async fn save(&self, data: T, encoder: fn(T) -> Vec) -> Result<()> { - let parent = self.cache_path.parent().ok_or_else(|| { - std::io::Error::new(std::io::ErrorKind::NotFound, "cache path not found") - })?; - create_dir_all(parent).await?; - write(&self.cache_path, encoder(data)).await - } - - pub async fn remove(&self) -> Result<()> { - remove_file(&self.cache_path).await - } - - pub async fn get_cache_time(&self) -> Result { - let metadata = metadata(&self.cache_path).await?; - let modified_time = metadata.modified()?; - Ok(modified_time - .duration_since(SystemTime::UNIX_EPOCH) - .unwrap_or_default() - .as_secs()) - } -} - -#[cfg(test)] -mod tests { - use std::fs; - - use super::*; - - #[tokio::test] - async fn test_local_cache_item() { - let key = "test_key"; - let data = "test_data"; - let cache_path = Path::new("./test_cache"); - let cache_item = LocalCacheItem::new(cache_path, key); - cache_item - .save(data, |data| data.as_bytes().to_vec()) - .await - .unwrap(); - let result = cache_item - .get(|data| String::from_utf8(data).unwrap()) - .await - .unwrap(); - assert_eq!(result, data); - let cache_time = cache_item.get_cache_time().await.unwrap(); - assert!( - cache_time - >= SystemTime::now() - .duration_since(SystemTime::UNIX_EPOCH) - .unwrap_or_default() - .as_secs() - ); - cache_item.remove().await.unwrap(); - fs::remove_dir_all(cache_path).unwrap(); - } -} diff --git a/src/cache/manager.rs b/src/cache/manager.rs deleted file mode 100644 index fbb8bb3..0000000 --- a/src/cache/manager.rs +++ /dev/null @@ -1,265 +0,0 @@ -use std::path::{Path, PathBuf}; - -use bytes::Bytes; - -use super::local::LocalCacheItem; -use crate::utils::time::get_now_unix; - -#[derive(Debug, Eq, Hash, PartialEq)] -pub enum GroupType { - RepoInside, - Api, -} - -pub struct CacheManager { - local_cache_dir: Option, - global_expire_time: Option, -} - -impl CacheManager { - pub fn new() -> Self { - Self { - local_cache_dir: None, - global_expire_time: None, - } - } - - pub fn set_local_cache_dir(&mut self, local_cache_dir: &Path) -> &mut Self { - self.local_cache_dir = Some(local_cache_dir.to_path_buf()); - self - } - - pub fn set_global_expire_time(&mut self, global_expire_time: u64) -> &mut Self { - self.global_expire_time = Some(global_expire_time); - self - } - - fn get_local_cache_key(group: &GroupType, key: &str) -> String { - format!("{:?}_{}", group, key) - } - - async fn get_local( - &self, - group: &GroupType, - key: &str, - expire_time: Option, - ) -> Option { - let local_cache_item = self.get_local_cache_item(group, key).ok()?; - if let Ok(time) = local_cache_item.get_cache_time().await { - if let Some(expire_time) = expire_time.or(self.global_expire_time) { - if time + expire_time < get_now_unix() { - return None; - } - } - if let Ok(data) = local_cache_item.get(|data| data).await { - return Some(Bytes::from(data)); - } - } - None - } - - pub async fn get( - &self, - group: &GroupType, - key: &str, - expire_time: Option, - ) -> Option { - self.get_local(group, key, expire_time).await - } - - pub async fn save( - &mut self, - group: &GroupType, - key: &str, - value: Bytes, - ) -> Result<(), std::io::Error> { - let local_cache_item = self.get_local_cache_item(group, key)?; - local_cache_item.save(value, |data| data.into()).await - } - - #[allow(dead_code)] - pub async fn remove(&mut self, group: &GroupType, key: &str) -> Result<(), std::io::Error> { - let local_cache_item = self.get_local_cache_item(group, key)?; - local_cache_item.remove().await - } - - #[allow(dead_code)] - pub async fn clean(&mut self) -> Result<(), std::io::Error> { - self.clean_local().await - } - - #[allow(dead_code)] - async fn clean_local(&mut self) -> Result<(), std::io::Error> { - let local_cache_dir = self.local_cache_dir.as_ref().ok_or_else(|| { - std::io::Error::new(std::io::ErrorKind::NotFound, "local cache dir not found") - })?; - tokio::fs::remove_dir_all(local_cache_dir).await - } - - fn get_local_cache_item( - &self, - group: &GroupType, - key: &str, - ) -> Result { - let local_cache_key = Self::get_local_cache_key(group, key); - let local_cache_dir = self.local_cache_dir.as_ref().ok_or_else(|| { - std::io::Error::new(std::io::ErrorKind::NotFound, "local cache dir not found") - })?; - Ok(LocalCacheItem::new(local_cache_dir, &local_cache_key)) - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[tokio::test] - async fn test_cache_manager() { - let mut cache_manager = CacheManager::new(); - cache_manager.set_local_cache_dir(Path::new("./test_cache_manager")); - let group = GroupType::RepoInside; - let key = "test_key"; - let value = Bytes::from("test_value"); - let _ = cache_manager.remove(&group, key).await; - cache_manager - .save(&group, key, value.clone()) - .await - .expect("save failed"); - let data = cache_manager - .get(&group, key, None) - .await - .expect("get failed"); - assert_eq!(data, value); - cache_manager - .remove(&group, key) - .await - .expect("remove failed"); - cache_manager.clean().await.expect("clean failed"); - } - - #[tokio::test] - async fn test_cache_manager_restart() { - let mut cache_manager = CacheManager::new(); - cache_manager.set_local_cache_dir(Path::new("./test_cache_manager_restart")); - let group = GroupType::RepoInside; - let key = "test_key"; - let value = Bytes::from("test_value"); - let _ = cache_manager.remove(&group, key).await; - cache_manager - .save(&group, key, value.clone()) - .await - .expect("save failed"); - let mut _cache_manager = CacheManager::new(); - _cache_manager.set_local_cache_dir(Path::new("./test_cache_manager_restart")); - let data = _cache_manager - .get(&group, key, None) - .await - .expect("get failed"); - assert_eq!(data, value); - cache_manager - .remove(&group, key) - .await - .expect("remove failed"); - cache_manager.clean().await.expect("clean failed"); - } - - #[tokio::test] - async fn test_cache_manager_no_exist() { - let mut cache_manager = CacheManager::new(); - cache_manager.set_local_cache_dir(Path::new("./test_cache_manager_no_exist")); - let group = GroupType::RepoInside; - let key = "test_key_no_exist"; - let data = cache_manager.get(&group, key, None).await; - assert_eq!(data, None); - let clean_result = cache_manager.clean().await; - assert!(clean_result.is_err()); - } - - #[tokio::test] - async fn test_cache_manager_expire_non_global() { - let mut cache_manager = CacheManager::new(); - cache_manager.set_local_cache_dir(Path::new("./test_cache_manager_expire_non_global")); - let group = GroupType::RepoInside; - let key = "test_key_expire"; - let value = Bytes::from("test_value_expire"); - let _ = cache_manager.remove(&group, key).await; - cache_manager - .save(&group, key, value.clone()) - .await - .expect("save failed"); - // sleep 1 second - tokio::time::sleep(tokio::time::Duration::from_secs(1)).await; - let data = cache_manager.get(&group, key, Some(0)).await; - assert_eq!(data, None); - let data = cache_manager - .get(&group, key, Some(100)) - .await - .expect("get failed"); - assert_eq!(data, value); - cache_manager - .remove(&group, key) - .await - .expect("remove failed"); - cache_manager.clean().await.expect("clean failed"); - } - - #[tokio::test] - async fn test_cache_manager_expire_non_expire() { - let mut cache_manager = CacheManager::new(); - cache_manager.set_local_cache_dir(Path::new("./test_cache_manager_non_expire")); - let group = GroupType::RepoInside; - let key = "test_key_expire"; - let value = Bytes::from("test_value_expire"); - let _ = cache_manager.remove(&group, key).await; - cache_manager - .save(&group, key, value.clone()) - .await - .expect("save failed"); - // sleep 1 second - tokio::time::sleep(tokio::time::Duration::from_secs(1)).await; - let data = cache_manager.get(&group, key, Some(0)).await; - assert_eq!(data, None); - let data = cache_manager - .get(&group, key, None) - .await - .expect("get failed"); - assert_eq!(data, value); - cache_manager - .remove(&group, key) - .await - .expect("remove failed"); - cache_manager.clean().await.expect("clean failed"); - } - - #[tokio::test] - async fn test_cache_manager_global_expire() { - let mut cache_manager = CacheManager::new(); - cache_manager - .set_local_cache_dir(Path::new("./test_cache_manager_global_expire")) - .set_global_expire_time(1); - let group = GroupType::RepoInside; - let key = "test_key_expire"; - let value = Bytes::from("test_value_expire"); - let _ = cache_manager.remove(&group, key).await; - cache_manager - .save(&group, key, value.clone()) - .await - .expect("save failed"); - // sleep 1 second - tokio::time::sleep(tokio::time::Duration::from_secs(2)).await; - let data = cache_manager.get(&group, key, Some(0)).await; - assert_eq!(data, None); - let data = cache_manager.get(&group, key, None).await; - assert_eq!(data, None); - let data = cache_manager - .get(&group, key, Some(100)) - .await - .expect("get failed"); - assert_eq!(data, value); - cache_manager - .remove(&group, key) - .await - .expect("remove failed"); - cache_manager.clean().await.expect("clean failed"); - } -} diff --git a/src/cache/moka.rs b/src/cache/moka.rs deleted file mode 100644 index e69de29..0000000 diff --git a/src/core.rs b/src/core.rs deleted file mode 100644 index ef68c36..0000000 --- a/src/core.rs +++ /dev/null @@ -1 +0,0 @@ -pub mod config; diff --git a/src/core/config.rs b/src/core/config.rs deleted file mode 100644 index 33d48eb..0000000 --- a/src/core/config.rs +++ /dev/null @@ -1,3 +0,0 @@ -mod data; -mod utils; -pub mod world; diff --git a/src/core/config/data.rs b/src/core/config/data.rs deleted file mode 100644 index 80675b9..0000000 --- a/src/core/config/data.rs +++ /dev/null @@ -1 +0,0 @@ -pub mod rule_list; diff --git a/src/core/config/data/rule_list.rs b/src/core/config/data/rule_list.rs deleted file mode 100644 index b0dd2d9..0000000 --- a/src/core/config/data/rule_list.rs +++ /dev/null @@ -1,89 +0,0 @@ -use serde::{Deserialize, Serialize}; - -/// Configuration lists -/// -/// JSON Schema: -/// ```json -/// { -/// "app_list": ["", ], -/// "hub_list": ["", ] -/// } -/// ``` - -#[derive(Serialize, Deserialize, Debug)] -pub struct RuleList { - #[serde(rename = "app_list")] - pub app_list: Vec, - - #[serde(rename = "hub_list")] - pub hub_list: Vec, -} - -impl RuleList { - pub fn new() -> Self { - RuleList { - app_list: Vec::new(), - hub_list: Vec::new(), - } - } - - pub fn push_app(&mut self, app_name: &str) -> bool { - if self.app_list.contains(&app_name.to_string()) { - false - } else { - self.app_list.push(app_name.to_string()); - true - } - } - - pub fn remove_app(&mut self, app_name: &str) -> bool { - if let Some(index) = self.app_list.iter().position(|x| x == app_name) { - self.app_list.remove(index); - true - } else { - false - } - } - - pub fn push_hub(&mut self, hub_name: &str) -> bool { - if self.hub_list.contains(&hub_name.to_string()) { - false - } else { - self.hub_list.push(hub_name.to_string()); - true - } - } - - pub fn remove_hub(&mut self, hub_name: &str) -> bool { - if let Some(index) = self.hub_list.iter().position(|x| x == hub_name) { - self.hub_list.remove(index); - true - } else { - false - } - } -} - -#[cfg(test)] -mod tests { - use super::*; - use serde_json; - - #[test] - fn test_config_list() { - let json = r#" -{ - "app_list": ["UpgradeAll", ""], - "hub_list": ["GitHub"] -}"#; - - let config_list: RuleList = serde_json::from_str(json).unwrap(); - - // check app_config_list - assert_eq!(config_list.app_list.len(), 2); - assert_eq!(config_list.app_list[0], "UpgradeAll"); - // check hub_config_list - assert_eq!(config_list.hub_list.len(), 1); - assert_eq!(config_list.hub_list[0], "GitHub"); - } -} diff --git a/src/core/config/utils.rs b/src/core/config/utils.rs deleted file mode 100644 index 15a32dd..0000000 --- a/src/core/config/utils.rs +++ /dev/null @@ -1,41 +0,0 @@ -use std::{env, path::PathBuf}; - -use crate::locale::all_dir; - -pub fn get_data_path(sub: &str) -> String { - let data_dir = env::var("DATA_DIR").map(PathBuf::from).unwrap_or_else(|_| { - all_dir() - .expect("Non-support OS, you should set DATA_DIR env arg") - .data_dir - }); - data_dir - .join(sub) - .to_str() - .expect("Invalid config path") - .to_string() -} - -#[cfg(test)] -mod tests { - use super::*; - use std::env; - use std::fs; - - #[test] - fn test_get_data_path_env() { - let path = "/tmp/getter_test"; - env::set_var("DATA_DIR", path); - let path = PathBuf::from(get_data_path("test")); - let _ = fs::remove_file(&path); - let path = path.to_str().unwrap(); - assert_eq!(path, "/tmp/getter_test/test"); - } - - #[test] - fn test_get_data_path_default() { - let path = PathBuf::from(get_data_path("test")); - let _ = fs::remove_file(&path); - let path = path.to_str().unwrap(); - assert!(path.ends_with("test")) - } -} diff --git a/src/core/config/world.rs b/src/core/config/world.rs deleted file mode 100644 index 9ab73ac..0000000 --- a/src/core/config/world.rs +++ /dev/null @@ -1,35 +0,0 @@ -pub mod local_repo; -pub mod world_config_wrapper; -pub mod world_list; - -use once_cell::sync::Lazy; -use std::{path::Path, sync::Arc}; -use tokio::sync::Mutex; - -use crate::{error::Result, utils::instance::InstanceContainer}; - -use self::world_list::WorldList; - -static INSTANCE_CONTAINER: Lazy> = - Lazy::new(|| InstanceContainer::new(WorldList::new())); - -pub async fn init_world_list(world_list_path: &Path) -> Result<()> { - get_world_list().await.lock().await.load(world_list_path)?; - Ok(()) -} - -pub async fn get_world_list() -> Arc> { - INSTANCE_CONTAINER.get().await.clone() -} - -#[cfg(test)] -mod tests { - use super::*; - - #[tokio::test] - async fn test_get_world_list_micro() { - let world_list_path = Path::new("./test_get_world_list_micro"); - init_world_list(world_list_path).await.unwrap(); - let _ = get_world_list().await; - } -} diff --git a/src/core/config/world/local_repo.rs b/src/core/config/world/local_repo.rs deleted file mode 100644 index b277da2..0000000 --- a/src/core/config/world/local_repo.rs +++ /dev/null @@ -1,31 +0,0 @@ -use crate::error::{GetterError, Result}; -use std::fs; -use std::path::PathBuf; - -const LOCAL_REPO_DIR: &str = "local_repo"; - -pub struct LocalRepo { - path: PathBuf, -} - -impl LocalRepo { - pub fn new(root_dir_path: &str) -> Self { - let path = PathBuf::from(root_dir_path).join(LOCAL_REPO_DIR); - Self { path } - } - - pub fn load(&self, rule_path: &str) -> Result { - let path = PathBuf::from(&self.path).join(rule_path); - let content = fs::read_to_string(&path) - .map_err(|e| GetterError::new("LocalRepo", "load", Box::new(e)))?; - Ok(content) - } - - pub fn save(&self, rule_path: &str, content: &str) -> Result<()> { - let path = PathBuf::from(&self.path).join(rule_path); - let _ = fs::create_dir_all(path.parent().unwrap()); - fs::write(&path, content) - .map_err(|e| GetterError::new("LocalRepo", "save", Box::new(e)))?; - Ok(()) - } -} diff --git a/src/core/config/world/world_config_wrapper.rs b/src/core/config/world/world_config_wrapper.rs deleted file mode 100644 index a4422f3..0000000 --- a/src/core/config/world/world_config_wrapper.rs +++ /dev/null @@ -1,71 +0,0 @@ -use std::path::Path; - -use crate::error::{GetterError, Result}; -use crate::utils::json::{json_to_string, string_to_json}; -use crate::websdk::cloud_rules::cloud_rules_wrapper::CloudRules; -use crate::websdk::cloud_rules::data::app_item::AppItem; -use crate::websdk::cloud_rules::data::hub_item::HubItem; - -use super::local_repo::LocalRepo; -use super::world_list::WorldList; - -pub struct WorldConfigWrapper { - pub world_list: WorldList, - pub local_repo: LocalRepo, -} - -impl WorldConfigWrapper { - pub fn new(world_config_path: &Path, local_repo_path: &str) -> Result { - let mut world_list = WorldList::new(); - world_list.load(world_config_path)?; - let local_repo = LocalRepo::new(local_repo_path); - Ok(WorldConfigWrapper { - world_list, - local_repo, - }) - } - - pub fn get_app_rule(&self, app_name: &str) -> Option { - if let Ok(content) = self.local_repo.load(app_name) { - if let Ok(app_item) = string_to_json(&content) { - return Some(app_item); - } - } - None - } - - pub fn get_hub_rule(&self, hub_name: &str) -> Option { - if let Ok(content) = self.local_repo.load(hub_name) { - if let Ok(hub_item) = string_to_json(&content) { - return Some(hub_item); - } - } - None - } - - pub fn download_app_rule(&self, app_name: &str, cloud_rules: &mut CloudRules) -> Result<()> { - let app_item = cloud_rules.get_cloud_app_rules(|x| x.info.name == app_name); - let content = json_to_string(&app_item).map_err(|e| { - GetterError::new("WorldConfigWrapper", "download_app_rule", Box::new(e)) - })?; - self.local_repo.save(app_name, &content)?; - Ok(()) - } - - pub fn download_hub_rule(&self, hub_name: &str, cloud_rules: &mut CloudRules) -> Result<()> { - let hub_item = cloud_rules.get_cloud_hub_rules(|x| x.info.hub_name == hub_name); - let content = json_to_string(&hub_item).map_err(|e| { - GetterError::new("WorldConfigWrapper", "download_hub_rule", Box::new(e)) - })?; - self.local_repo.save(hub_name, &content)?; - Ok(()) - } - - pub fn get_app_rule_list(&self) -> &Vec { - &self.world_list.rule_list.app_list - } - - pub fn get_hub_rule_list(&self) -> &Vec { - &self.world_list.rule_list.hub_list - } -} diff --git a/src/core/config/world/world_list.rs b/src/core/config/world/world_list.rs deleted file mode 100644 index 6dd89a0..0000000 --- a/src/core/config/world/world_list.rs +++ /dev/null @@ -1,100 +0,0 @@ -use std::fs::{create_dir_all, File}; -use std::io::BufReader; -use std::path::{Path, PathBuf}; - -use crate::error::{GetterError, Result}; - -use super::super::data::rule_list::RuleList; - -pub const WORLD_CONFIG_LIST_NAME: &str = "world_config_list.json"; - -pub struct WorldList { - config_path: Option, - pub rule_list: RuleList, -} - -impl WorldList { - pub fn new() -> Self { - Self { - config_path: None, - rule_list: RuleList::new(), - } - } - - pub fn load(&mut self, config_path: &Path) -> Result<&mut Self> { - let rule_list = if let Ok(file) = File::open(config_path) { - let reader = BufReader::new(file); - serde_json::from_reader(reader) - .map_err(|e| GetterError::new("WorldList", "load", Box::new(e)))? - } else { - RuleList::new() - }; - self.config_path = Some(config_path.to_path_buf()); - self.rule_list = rule_list; - Ok(self) - } - - pub fn save(&self) -> Result<()> { - let path = self - .config_path - .as_deref() - .ok_or_else(|| GetterError::new_nobase("WorldList", "save: path not set"))?; - let parent = path - .parent() - .ok_or_else(|| GetterError::new_nobase("WorldList", "save: get parent dir failed"))?; - let _ = create_dir_all(parent); - let file = - File::create(path).map_err(|e| GetterError::new("WorldList", "save", Box::new(e)))?; - serde_json::to_writer(file, &self.rule_list) - .map_err(|e| GetterError::new("WorldList", "save", Box::new(e)))?; - Ok(()) - } -} - -#[cfg(test)] -mod tests { - use super::*; - use std::fs; - - #[test] - fn test_world_list() { - let path_base = "/tmp/getter_test_world_list"; - let _ = fs::remove_dir_all(path_base); - let config_path = PathBuf::from(path_base).join(WORLD_CONFIG_LIST_NAME); - - let mut world_list = WorldList::new(); - world_list.load(&config_path).unwrap(); - assert!(world_list.rule_list.app_list.is_empty()); - assert!(world_list.rule_list.hub_list.is_empty()); - let value = world_list.save(); - assert!(value.is_ok()); - let content = fs::read_to_string(&config_path).expect("test_world_list: read file failed"); - assert!(!content.is_empty()); - - let mut world_list = WorldList::new(); - world_list.load(&config_path).unwrap(); - world_list.rule_list.app_list.push("UpgradeAll".to_string()); - world_list.rule_list.hub_list.push("GitHub".to_string()); - let value = world_list.save(); - assert!(value.is_ok()); - - let mut world_list = WorldList::new(); - world_list.load(&config_path).unwrap(); - assert_eq!(world_list.rule_list.app_list.len(), 1); - assert_eq!(world_list.rule_list.app_list[0], "UpgradeAll"); - assert_eq!(world_list.rule_list.hub_list.len(), 1); - assert_eq!(world_list.rule_list.hub_list[0], "GitHub"); - - fs::remove_dir_all(path_base).expect("test_world_list: clean failed"); - } - - #[test] - fn test_world_list_only_load() { - let path_base = "/tmp/getter_test_world_list_only_load"; - let _ = fs::remove_dir(path_base); - let path = PathBuf::from(path_base).join(WORLD_CONFIG_LIST_NAME); - - let _ = WorldList::new().load(&path); - assert!(path.try_exists().is_ok_and(|x| !x)); - } -} diff --git a/src/database/mod.rs b/src/database/mod.rs deleted file mode 100644 index dcb6598..0000000 --- a/src/database/mod.rs +++ /dev/null @@ -1,229 +0,0 @@ -pub mod models; -pub mod store; - -use models::{ - app::AppRecord, extra_app::ExtraAppRecord, extra_hub::ExtraHubRecord, hub::HubRecord, -}; -use once_cell::sync::OnceCell; -use std::path::Path; - -use crate::error::Result; -use store::{HasId, JsonlStore}; - -impl HasId for AppRecord { - fn id(&self) -> &str { - &self.id - } -} - -impl HasId for HubRecord { - fn id(&self) -> &str { - &self.uuid - } -} - -impl HasId for ExtraAppRecord { - fn id(&self) -> &str { - &self.id - } -} - -impl HasId for ExtraHubRecord { - fn id(&self) -> &str { - &self.id - } -} - -pub struct Database { - pub apps: JsonlStore, - pub hubs: JsonlStore, - pub extra_apps: JsonlStore, - pub extra_hubs: JsonlStore, -} - -impl Database { - pub fn open(data_dir: &Path) -> Result { - let db = Self { - apps: JsonlStore::new(data_dir.join("apps.jsonl")), - hubs: JsonlStore::new(data_dir.join("hubs.jsonl")), - extra_apps: JsonlStore::new(data_dir.join("extra_apps.jsonl")), - extra_hubs: JsonlStore::new(data_dir.join("extra_hubs.jsonl")), - }; - db.apps.ensure_file()?; - db.hubs.ensure_file()?; - db.extra_apps.ensure_file()?; - db.extra_hubs.ensure_file()?; - Ok(db) - } - - // --- App CRUD --- - - pub fn load_apps(&self) -> Result> { - self.apps.load_all() - } - - pub fn upsert_app(&self, record: &AppRecord) -> Result<()> { - self.apps.upsert(record) - } - - pub fn delete_app(&self, id: &str) -> Result { - self.apps.delete::(id) - } - - pub fn find_app(&self, id: &str) -> Result> { - self.apps.find_by_id(id) - } - - // --- Hub CRUD --- - - pub fn load_hubs(&self) -> Result> { - self.hubs.load_all() - } - - pub fn upsert_hub(&self, record: &HubRecord) -> Result<()> { - self.hubs.upsert(record) - } - - pub fn delete_hub(&self, uuid: &str) -> Result { - self.hubs.delete::(uuid) - } - - pub fn find_hub(&self, uuid: &str) -> Result> { - self.hubs.find_by_id(uuid) - } - - // --- ExtraApp CRUD --- - - pub fn load_extra_apps(&self) -> Result> { - self.extra_apps.load_all() - } - - pub fn upsert_extra_app(&self, record: &ExtraAppRecord) -> Result<()> { - self.extra_apps.upsert(record) - } - - pub fn delete_extra_app(&self, id: &str) -> Result { - self.extra_apps.delete::(id) - } - - /// Find an ExtraApp record by matching its `app_id` map. - pub fn get_extra_app_by_app_id( - &self, - app_id: &std::collections::HashMap>, - ) -> Result> { - let all = self.extra_apps.load_all::()?; - Ok(all.into_iter().find(|r| &r.app_id == app_id)) - } - - // --- ExtraHub CRUD --- - - pub fn load_extra_hubs(&self) -> Result> { - self.extra_hubs.load_all() - } - - pub fn upsert_extra_hub(&self, record: &ExtraHubRecord) -> Result<()> { - self.extra_hubs.upsert(record) - } - - pub fn delete_extra_hub(&self, id: &str) -> Result { - self.extra_hubs.delete::(id) - } - - pub fn find_extra_hub(&self, id: &str) -> Result> { - self.extra_hubs.find_by_id(id) - } -} - -static DB: OnceCell = OnceCell::new(); - -/// Initialize the global database. Must be called once before `get_db()`. -/// Idempotent: if already initialized, returns `Ok(())`. -pub fn init_db(data_dir: &Path) -> Result<()> { - if DB.get().is_some() { - return Ok(()); - } - let db = Database::open(data_dir)?; - let _ = DB.set(db); // Ignore error if another thread beat us to it - Ok(()) -} - -/// Get the global database instance. Panics if `init_db` was not called. -pub fn get_db() -> &'static Database { - DB.get() - .expect("Database not initialized. Call init_db() first.") -} - -#[cfg(test)] -mod tests { - use super::*; - use tempfile::TempDir; - - fn open_test_db() -> (Database, TempDir) { - let dir = tempfile::tempdir().unwrap(); - let db = Database::open(dir.path()).unwrap(); - (db, dir) - } - - #[test] - fn test_open_creates_files() { - let dir = tempfile::tempdir().unwrap(); - Database::open(dir.path()).unwrap(); - assert!(dir.path().join("apps.jsonl").exists()); - assert!(dir.path().join("hubs.jsonl").exists()); - assert!(dir.path().join("extra_apps.jsonl").exists()); - assert!(dir.path().join("extra_hubs.jsonl").exists()); - } - - #[test] - fn test_app_crud() { - let (db, _dir) = open_test_db(); - let app = AppRecord::new( - "TestApp".to_string(), - std::collections::HashMap::from([("owner".to_string(), Some("alice".to_string()))]), - ); - db.upsert_app(&app).unwrap(); - - let apps = db.load_apps().unwrap(); - assert_eq!(apps.len(), 1); - assert_eq!(apps[0].name, "TestApp"); - - let found = db.find_app(&app.id).unwrap(); - assert!(found.is_some()); - - let deleted = db.delete_app(&app.id).unwrap(); - assert!(deleted); - assert!(db.load_apps().unwrap().is_empty()); - } - - #[test] - fn test_hub_crud() { - use crate::websdk::cloud_rules::data::hub_item::{HubItem, Info}; - let (db, _dir) = open_test_db(); - let hub = HubRecord::new( - "fd9b2602-62c5-4d55-bd1e-0d6537714ca0".to_string(), - HubItem { - base_version: 6, - config_version: 1, - uuid: "fd9b2602-62c5-4d55-bd1e-0d6537714ca0".to_string(), - info: Info { - hub_name: "GitHub".to_string(), - hub_icon_url: None, - }, - api_keywords: vec!["owner".to_string(), "repo".to_string()], - auth_keywords: vec![], - app_url_templates: vec![], - target_check_api: None, - }, - ); - db.upsert_hub(&hub).unwrap(); - let hubs = db.load_hubs().unwrap(); - assert_eq!(hubs.len(), 1); - assert_eq!(hubs[0].uuid, "fd9b2602-62c5-4d55-bd1e-0d6537714ca0"); - - let deleted = db - .delete_hub("fd9b2602-62c5-4d55-bd1e-0d6537714ca0") - .unwrap(); - assert!(deleted); - assert!(db.load_hubs().unwrap().is_empty()); - } -} diff --git a/src/database/models/app.rs b/src/database/models/app.rs deleted file mode 100644 index 104e0ce..0000000 --- a/src/database/models/app.rs +++ /dev/null @@ -1,101 +0,0 @@ -use crate::websdk::cloud_rules::data::app_item::AppItem; -use serde::{Deserialize, Serialize}; -use std::collections::HashMap; - -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] -pub struct AppRecord { - /// UUID v4 identifier (replaces Room's auto-increment Long id) - pub id: String, - pub name: String, - pub app_id: HashMap>, - #[serde(skip_serializing_if = "Option::is_none")] - pub invalid_version_number_field_regex: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub include_version_number_field_regex: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub ignore_version_number: Option, - /// Cloud config (AppItem), optional - #[serde(skip_serializing_if = "Option::is_none")] - pub cloud_config: Option, - /// Space-separated hub UUIDs in priority order - #[serde(skip_serializing_if = "Option::is_none")] - pub enable_hub_list: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub star: Option, -} - -impl AppRecord { - pub fn new(name: String, app_id: HashMap>) -> Self { - Self { - id: uuid::Uuid::new_v4().to_string(), - name, - app_id, - invalid_version_number_field_regex: None, - include_version_number_field_regex: None, - ignore_version_number: None, - cloud_config: None, - enable_hub_list: None, - star: None, - } - } - - pub fn get_sorted_hub_uuids(&self) -> Vec { - match &self.enable_hub_list { - Some(s) if !s.is_empty() => s.split(' ').map(String::from).collect(), - _ => vec![], - } - } - - pub fn set_sorted_hub_uuids(&mut self, uuids: &[String]) { - let s = uuids.join(" "); - self.enable_hub_list = if s.is_empty() { None } else { Some(s) }; - } -} - -#[cfg(test)] -mod tests { - use super::*; - - fn sample_app() -> AppRecord { - AppRecord { - id: "test-uuid".to_string(), - name: "TestApp".to_string(), - app_id: HashMap::from([("owner".to_string(), Some("alice".to_string()))]), - invalid_version_number_field_regex: None, - include_version_number_field_regex: None, - ignore_version_number: None, - cloud_config: None, - enable_hub_list: Some("hub1 hub2".to_string()), - star: Some(true), - } - } - - #[test] - fn test_serialization_roundtrip() { - let app = sample_app(); - let json = serde_json::to_string(&app).unwrap(); - let decoded: AppRecord = serde_json::from_str(&json).unwrap(); - assert_eq!(app, decoded); - } - - #[test] - fn test_get_sorted_hub_uuids() { - let app = sample_app(); - let uuids = app.get_sorted_hub_uuids(); - assert_eq!(uuids, vec!["hub1", "hub2"]); - } - - #[test] - fn test_set_sorted_hub_uuids() { - let mut app = sample_app(); - app.set_sorted_hub_uuids(&["a".to_string(), "b".to_string(), "c".to_string()]); - assert_eq!(app.enable_hub_list, Some("a b c".to_string())); - } - - #[test] - fn test_empty_hub_list_is_none() { - let mut app = sample_app(); - app.set_sorted_hub_uuids(&[]); - assert_eq!(app.enable_hub_list, None); - } -} diff --git a/src/database/models/extra_app.rs b/src/database/models/extra_app.rs deleted file mode 100644 index 0123376..0000000 --- a/src/database/models/extra_app.rs +++ /dev/null @@ -1,48 +0,0 @@ -use serde::{Deserialize, Serialize}; -use std::collections::HashMap; - -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] -pub struct ExtraAppRecord { - /// UUID v4 identifier - pub id: String, - pub app_id: HashMap>, - #[serde(skip_serializing_if = "Option::is_none")] - pub mark_version_number: Option, -} - -impl ExtraAppRecord { - pub fn new(app_id: HashMap>) -> Self { - Self { - id: uuid::Uuid::new_v4().to_string(), - app_id, - mark_version_number: None, - } - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_serialization_roundtrip() { - let record = ExtraAppRecord { - id: "test-uuid".to_string(), - app_id: HashMap::from([( - "android_app_package".to_string(), - Some("com.foo".to_string()), - )]), - mark_version_number: Some("1.2.3".to_string()), - }; - let json = serde_json::to_string(&record).unwrap(); - let decoded: ExtraAppRecord = serde_json::from_str(&json).unwrap(); - assert_eq!(record, decoded); - } - - #[test] - fn test_none_fields_skipped() { - let record = ExtraAppRecord::new(HashMap::new()); - let json = serde_json::to_string(&record).unwrap(); - assert!(!json.contains("mark_version_number")); - } -} diff --git a/src/database/models/extra_hub.rs b/src/database/models/extra_hub.rs deleted file mode 100644 index 4366d62..0000000 --- a/src/database/models/extra_hub.rs +++ /dev/null @@ -1,56 +0,0 @@ -use serde::{Deserialize, Serialize}; - -pub const GLOBAL_HUB_ID: &str = "GLOBAL"; - -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] -pub struct ExtraHubRecord { - /// Hub UUID or GLOBAL_HUB_ID - pub id: String, - #[serde(default)] - pub enable_global: bool, - #[serde(skip_serializing_if = "Option::is_none")] - pub url_replace_search: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub url_replace_string: Option, -} - -impl ExtraHubRecord { - pub fn new(id: String) -> Self { - Self { - id, - enable_global: false, - url_replace_search: None, - url_replace_string: None, - } - } - - pub fn global() -> Self { - Self::new(GLOBAL_HUB_ID.to_string()) - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_serialization_roundtrip() { - let record = ExtraHubRecord { - id: "some-hub-uuid".to_string(), - enable_global: true, - url_replace_search: Some("github.com".to_string()), - url_replace_string: Some("mirror.example.com".to_string()), - }; - let json = serde_json::to_string(&record).unwrap(); - let decoded: ExtraHubRecord = serde_json::from_str(&json).unwrap(); - assert_eq!(record, decoded); - } - - #[test] - fn test_none_fields_skipped() { - let record = ExtraHubRecord::global(); - let json = serde_json::to_string(&record).unwrap(); - assert!(!json.contains("url_replace_search")); - assert!(!json.contains("url_replace_string")); - } -} diff --git a/src/database/models/hub.rs b/src/database/models/hub.rs deleted file mode 100644 index 84e94ce..0000000 --- a/src/database/models/hub.rs +++ /dev/null @@ -1,80 +0,0 @@ -use crate::websdk::cloud_rules::data::hub_item::HubItem; -use serde::{Deserialize, Serialize}; -use std::collections::HashMap; - -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] -pub struct HubRecord { - /// Hub UUID — primary key - pub uuid: String, - pub hub_config: HubItem, - pub auth: HashMap, - #[serde(default)] - pub ignore_app_id_list: Vec>>, - /// 0 = disabled, 1 = enabled - #[serde(default)] - pub applications_mode: i32, - #[serde(default)] - pub user_ignore_app_id_list: Vec>>, - /// Lower = higher priority. Default is -(hub list size). - #[serde(default)] - pub sort_point: i32, -} - -impl HubRecord { - pub fn new(uuid: String, hub_config: HubItem) -> Self { - Self { - uuid, - hub_config, - auth: HashMap::new(), - ignore_app_id_list: vec![], - applications_mode: 0, - user_ignore_app_id_list: vec![], - sort_point: 0, - } - } - - pub fn applications_mode_enabled(&self) -> bool { - self.applications_mode == 1 - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::websdk::cloud_rules::data::hub_item::Info; - - fn sample_hub() -> HubRecord { - HubRecord::new( - "fd9b2602-62c5-4d55-bd1e-0d6537714ca0".to_string(), - HubItem { - base_version: 6, - config_version: 3, - uuid: "fd9b2602-62c5-4d55-bd1e-0d6537714ca0".to_string(), - info: Info { - hub_name: "GitHub".to_string(), - hub_icon_url: None, - }, - api_keywords: vec!["owner".to_string(), "repo".to_string()], - auth_keywords: vec![], - app_url_templates: vec!["https://github.com/%owner/%repo/".to_string()], - target_check_api: None, - }, - ) - } - - #[test] - fn test_serialization_roundtrip() { - let hub = sample_hub(); - let json = serde_json::to_string(&hub).unwrap(); - let decoded: HubRecord = serde_json::from_str(&json).unwrap(); - assert_eq!(hub, decoded); - } - - #[test] - fn test_applications_mode() { - let mut hub = sample_hub(); - assert!(!hub.applications_mode_enabled()); - hub.applications_mode = 1; - assert!(hub.applications_mode_enabled()); - } -} diff --git a/src/database/models/mod.rs b/src/database/models/mod.rs deleted file mode 100644 index c4e40bc..0000000 --- a/src/database/models/mod.rs +++ /dev/null @@ -1,4 +0,0 @@ -pub mod app; -pub mod extra_app; -pub mod extra_hub; -pub mod hub; diff --git a/src/database/store.rs b/src/database/store.rs deleted file mode 100644 index 32d374f..0000000 --- a/src/database/store.rs +++ /dev/null @@ -1,266 +0,0 @@ -use file_locker::FileLock; -use jsonl::ReadError; -use serde::{de::DeserializeOwned, Serialize}; -use std::io::{BufReader, Seek, SeekFrom, Write}; -use std::path::{Path, PathBuf}; - -use crate::error::{Error, Result}; - -pub trait HasId { - fn id(&self) -> &str; -} - -/// A simple JSONL-backed persistent store for a single record type. -/// -/// Each line in the file is a JSON-serialized record. All mutating operations -/// use `file-locker` advisory locking to ensure safe concurrent access. -pub struct JsonlStore { - path: PathBuf, -} - -impl JsonlStore { - pub fn new(path: impl Into) -> Self { - Self { path: path.into() } - } - - pub fn path(&self) -> &Path { - &self.path - } - - /// Ensure the backing file exists (creates parent dirs as needed). - pub fn ensure_file(&self) -> Result<()> { - if !self.path.exists() { - if let Some(parent) = self.path.parent() { - std::fs::create_dir_all(parent)?; - } - std::fs::File::create(&self.path)?; - } - Ok(()) - } - - /// Read all records from the file line by line. - pub fn load_all(&self) -> Result> { - if !self.path.exists() { - return Ok(vec![]); - } - let file = std::fs::File::open(&self.path)?; - let reader = BufReader::new(file); - read_all_from_reader(reader) - } - - fn acquire_write_lock(&self) -> Result { - self.ensure_file()?; - FileLock::new(&self.path) - .blocking(true) - .writeable(true) - .lock() - .map_err(Error::Io) - } - - /// Overwrite the entire file with the given records (must already hold lock). - fn write_all_locked(lock: &mut FileLock, records: &[T]) -> Result<()> { - lock.file.set_len(0)?; - lock.file.seek(SeekFrom::Start(0))?; - for record in records { - jsonl::write(&mut lock.file, record) - .map_err(|e| Error::Other(format!("jsonl write: {e}")))?; - } - lock.file.flush()?; - Ok(()) - } - - /// Insert or replace a record (matched by id). - pub fn upsert(&self, record: &T) -> Result<()> { - let mut lock = self.acquire_write_lock()?; - let mut records: Vec = { - lock.file.seek(SeekFrom::Start(0))?; - read_all_from_reader(BufReader::new(&lock.file))? - }; - - let id = record.id(); - let serialized: T = serde_json::from_str( - &serde_json::to_string(record).map_err(|e| Error::Other(format!("serialize: {e}")))?, - ) - .map_err(|e| Error::Other(format!("deserialize: {e}")))?; - - if let Some(pos) = records.iter().position(|r| r.id() == id) { - records[pos] = serialized; - } else { - records.push(serialized); - } - - Self::write_all_locked(&mut lock, &records) - } - - /// Delete the record with the given id. Returns true if a record was removed. - pub fn delete(&self, id: &str) -> Result { - let mut lock = self.acquire_write_lock()?; - let records: Vec = { - lock.file.seek(SeekFrom::Start(0))?; - read_all_from_reader(BufReader::new(&lock.file))? - }; - - let original_len = records.len(); - let filtered: Vec = records.into_iter().filter(|r| r.id() != id).collect(); - let deleted = filtered.len() < original_len; - Self::write_all_locked(&mut lock, &filtered)?; - Ok(deleted) - } - - /// Find a record by id (read-only, no lock). - pub fn find_by_id(&self, id: &str) -> Result> { - Ok(self.load_all::()?.into_iter().find(|r| r.id() == id)) - } -} - -fn read_all_from_reader(mut reader: R) -> Result> { - let mut records = Vec::new(); - loop { - match jsonl::read::<_, T>(&mut reader) { - Ok(record) => records.push(record), - Err(ReadError::Eof) => break, - Err(ReadError::Deserialize(e)) => { - eprintln!("JsonlStore: skipping malformed line: {e}"); - } - Err(ReadError::Io(e)) => return Err(Error::Io(e)), - } - } - Ok(records) -} - -#[cfg(test)] -mod tests { - use super::*; - use serde::{Deserialize, Serialize}; - use tempfile::NamedTempFile; - - #[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] - struct TestRecord { - id: String, - value: String, - } - - impl HasId for TestRecord { - fn id(&self) -> &str { - &self.id - } - } - - fn make_store() -> (JsonlStore, NamedTempFile) { - let tmp = NamedTempFile::new().unwrap(); - let store = JsonlStore::new(tmp.path()); - (store, tmp) - } - - #[test] - fn test_empty_load() { - let (store, _tmp) = make_store(); - let records: Vec = store.load_all().unwrap(); - assert!(records.is_empty()); - } - - #[test] - fn test_upsert_and_load() { - let (store, _tmp) = make_store(); - let r = TestRecord { - id: "1".to_string(), - value: "hello".to_string(), - }; - store.upsert(&r).unwrap(); - let records: Vec = store.load_all().unwrap(); - assert_eq!(records.len(), 1); - assert_eq!(records[0], r); - } - - #[test] - fn test_upsert_updates_existing() { - let (store, _tmp) = make_store(); - store - .upsert(&TestRecord { - id: "1".to_string(), - value: "old".to_string(), - }) - .unwrap(); - store - .upsert(&TestRecord { - id: "1".to_string(), - value: "new".to_string(), - }) - .unwrap(); - let records: Vec = store.load_all().unwrap(); - assert_eq!(records.len(), 1); - assert_eq!(records[0].value, "new"); - } - - #[test] - fn test_multiple_records() { - let (store, _tmp) = make_store(); - for i in 0..5 { - store - .upsert(&TestRecord { - id: i.to_string(), - value: format!("v{i}"), - }) - .unwrap(); - } - let records: Vec = store.load_all().unwrap(); - assert_eq!(records.len(), 5); - } - - #[test] - fn test_delete() { - let (store, _tmp) = make_store(); - store - .upsert(&TestRecord { - id: "1".to_string(), - value: "a".to_string(), - }) - .unwrap(); - store - .upsert(&TestRecord { - id: "2".to_string(), - value: "b".to_string(), - }) - .unwrap(); - let deleted = store.delete::("1").unwrap(); - assert!(deleted); - let records: Vec = store.load_all().unwrap(); - assert_eq!(records.len(), 1); - assert_eq!(records[0].id, "2"); - } - - #[test] - fn test_delete_nonexistent() { - let (store, _tmp) = make_store(); - let deleted = store.delete::("nope").unwrap(); - assert!(!deleted); - } - - #[test] - fn test_find_by_id() { - let (store, _tmp) = make_store(); - store - .upsert(&TestRecord { - id: "42".to_string(), - value: "answer".to_string(), - }) - .unwrap(); - let found: Option = store.find_by_id("42").unwrap(); - assert!(found.is_some()); - assert_eq!(found.unwrap().value, "answer"); - } - - #[test] - fn test_find_by_id_missing() { - let (store, _tmp) = make_store(); - let found: Option = store.find_by_id("999").unwrap(); - assert!(found.is_none()); - } - - #[test] - fn test_file_not_exist_returns_empty() { - let store = JsonlStore::new("/tmp/this_file_does_not_exist_upgradeall.jsonl"); - let records: Vec = store.load_all().unwrap(); - assert!(records.is_empty()); - } -} diff --git a/src/downloader/config.rs b/src/downloader/config.rs deleted file mode 100644 index a4b0e9d..0000000 --- a/src/downloader/config.rs +++ /dev/null @@ -1,199 +0,0 @@ -//! Configuration system for the downloader module - -use serde::{Deserialize, Serialize}; - -/// Downloader backend selection -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)] -#[serde(rename_all = "lowercase")] -pub enum DownloaderBackend { - /// Use trauma downloader (default) - #[default] - Trauma, - // Future backends can be added here: - // /// Use reqwest downloader - // Reqwest, - // /// Use custom CLI command - // Custom, -} - -/// Download configuration -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct DownloadConfig { - /// Downloader backend to use - pub backend: DownloaderBackend, - - /// Maximum number of concurrent downloads - #[serde(default = "default_max_concurrent")] - pub max_concurrent: usize, - - /// Number of retry attempts for failed downloads - #[serde(default = "default_retries")] - pub retries: usize, - - /// Timeout for each download in seconds - #[serde(default = "default_timeout")] - pub timeout_seconds: u64, - - /// Custom command template for CLI downloader (future use) - /// Example: "wget -O \"${FILE}\" \"${URI}\"" - #[serde(skip_serializing_if = "Option::is_none")] - pub custom_command: Option, -} - -fn default_max_concurrent() -> usize { - 4 -} - -fn default_retries() -> usize { - 3 -} - -fn default_timeout() -> u64 { - 300 // 5 minutes -} - -impl Default for DownloadConfig { - fn default() -> Self { - Self { - backend: DownloaderBackend::default(), - max_concurrent: default_max_concurrent(), - retries: default_retries(), - timeout_seconds: default_timeout(), - custom_command: None, - } - } -} - -impl DownloadConfig { - /// Create a new configuration with default values - pub fn new() -> Self { - Self::default() - } - - /// Set the backend - pub fn with_backend(mut self, backend: DownloaderBackend) -> Self { - self.backend = backend; - self - } - - /// Set max concurrent downloads - pub fn with_max_concurrent(mut self, max: usize) -> Self { - self.max_concurrent = max; - self - } - - /// Set retry count - pub fn with_retries(mut self, retries: usize) -> Self { - self.retries = retries; - self - } - - /// Set timeout in seconds - pub fn with_timeout(mut self, seconds: u64) -> Self { - self.timeout_seconds = seconds; - self - } - - /// Set custom command template - pub fn with_custom_command(mut self, command: impl Into) -> Self { - self.custom_command = Some(command.into()); - self - } - - /// Load configuration from environment variables - /// - /// Supported environment variables: - /// - DOWNLOADER_BACKEND: "trauma" (default) - /// - DOWNLOADER_MAX_CONCURRENT: number (default: 4) - /// - DOWNLOADER_RETRIES: number (default: 3) - /// - DOWNLOADER_TIMEOUT: seconds (default: 300) - /// - FETCHCOMMAND: custom download command (like Portage's FETCHCOMMAND) - pub fn from_env() -> Self { - let mut config = Self::default(); - - if let Ok(backend) = std::env::var("DOWNLOADER_BACKEND") { - config.backend = match backend.to_lowercase().as_str() { - "trauma" => DownloaderBackend::Trauma, - _ => { - eprintln!("Unknown DOWNLOADER_BACKEND: {}, using default", backend); - DownloaderBackend::Trauma - } - }; - } - - if let Ok(max_concurrent) = std::env::var("DOWNLOADER_MAX_CONCURRENT") { - if let Ok(max) = max_concurrent.parse() { - config.max_concurrent = max; - } - } - - if let Ok(retries) = std::env::var("DOWNLOADER_RETRIES") { - if let Ok(retry_count) = retries.parse() { - config.retries = retry_count; - } - } - - if let Ok(timeout) = std::env::var("DOWNLOADER_TIMEOUT") { - if let Ok(timeout_secs) = timeout.parse() { - config.timeout_seconds = timeout_secs; - } - } - - if let Ok(command) = std::env::var("FETCHCOMMAND") { - config.custom_command = Some(command); - } - - config - } - - /// Load configuration from JSON string - pub fn from_json(json: &str) -> Result { - serde_json::from_str(json) - } - - /// Serialize configuration to JSON string - pub fn to_json(&self) -> Result { - serde_json::to_string_pretty(self) - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_default_config() { - let config = DownloadConfig::default(); - assert_eq!(config.backend, DownloaderBackend::Trauma); - assert_eq!(config.max_concurrent, 4); - assert_eq!(config.retries, 3); - assert_eq!(config.timeout_seconds, 300); - assert!(config.custom_command.is_none()); - } - - #[test] - fn test_builder_pattern() { - let config = DownloadConfig::new() - .with_backend(DownloaderBackend::Trauma) - .with_max_concurrent(8) - .with_retries(5) - .with_timeout(600) - .with_custom_command("wget -O \"${FILE}\" \"${URI}\""); - - assert_eq!(config.backend, DownloaderBackend::Trauma); - assert_eq!(config.max_concurrent, 8); - assert_eq!(config.retries, 5); - assert_eq!(config.timeout_seconds, 600); - assert!(config.custom_command.is_some()); - } - - #[test] - fn test_json_serialization() { - let config = DownloadConfig::new().with_max_concurrent(8); - let json = config.to_json().unwrap(); - let deserialized: DownloadConfig = DownloadConfig::from_json(&json).unwrap(); - - assert_eq!(config.backend, deserialized.backend); - assert_eq!(config.max_concurrent, deserialized.max_concurrent); - } -} diff --git a/src/downloader/error.rs b/src/downloader/error.rs deleted file mode 100644 index 7384726..0000000 --- a/src/downloader/error.rs +++ /dev/null @@ -1,95 +0,0 @@ -//! Error types for the downloader module - -use std::fmt; - -/// Error type for download operations -#[derive(Debug, Clone)] -pub struct DownloadError { - pub kind: ErrorKind, - pub message: String, -} - -/// Kinds of download errors -#[derive(Debug, Clone, PartialEq, Eq)] -pub enum ErrorKind { - /// Network error (connection failed, timeout, etc.) - Network, - /// File system error (permission denied, disk full, etc.) - FileSystem, - /// Invalid input (bad URL, invalid path, etc.) - InvalidInput, - /// Task not found - TaskNotFound, - /// Task already exists - TaskAlreadyExists, - /// Download was cancelled - Cancelled, - /// Operation not supported by this downloader implementation - Unsupported, - /// Unknown error - Unknown, -} - -impl DownloadError { - pub fn new(kind: ErrorKind, message: impl Into) -> Self { - Self { - kind, - message: message.into(), - } - } - - pub fn network(message: impl Into) -> Self { - Self::new(ErrorKind::Network, message) - } - - pub fn file_system(message: impl Into) -> Self { - Self::new(ErrorKind::FileSystem, message) - } - - pub fn invalid_input(message: impl Into) -> Self { - Self::new(ErrorKind::InvalidInput, message) - } - - pub fn task_not_found(task_id: impl fmt::Display) -> Self { - Self::new( - ErrorKind::TaskNotFound, - format!("Task not found: {}", task_id), - ) - } - - pub fn task_already_exists(task_id: impl fmt::Display) -> Self { - Self::new( - ErrorKind::TaskAlreadyExists, - format!("Task already exists: {}", task_id), - ) - } - - pub fn cancelled(message: impl Into) -> Self { - Self::new(ErrorKind::Cancelled, message) - } - - pub fn unsupported(message: impl Into) -> Self { - Self::new(ErrorKind::Unsupported, message) - } - - pub fn unknown(message: impl Into) -> Self { - Self::new(ErrorKind::Unknown, message) - } -} - -impl fmt::Display for DownloadError { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "{:?}: {}", self.kind, self.message) - } -} - -impl std::error::Error for DownloadError {} - -impl From for DownloadError { - fn from(err: std::io::Error) -> Self { - Self::file_system(err.to_string()) - } -} - -/// Result type for download operations -pub type Result = std::result::Result; diff --git a/src/downloader/external_rpc_impl.rs b/src/downloader/external_rpc_impl.rs deleted file mode 100644 index e98debd..0000000 --- a/src/downloader/external_rpc_impl.rs +++ /dev/null @@ -1,290 +0,0 @@ -//! External RPC-based downloader implementation -//! -//! Delegates download operations to an external service via HTTP JSON-RPC 2.0. -//! The external service (e.g., a Kotlin-side GooglePlayDownloader) must implement -//! the standard downloader RPC protocol: -//! - download_submit(url, dest_path, headers?, cookies?) -> {task_id} -//! - download_get_status(task_id) -> TaskInfo -//! - download_wait_for_change(task_id, timeout_seconds) -> TaskInfo -//! - download_pause(task_id) -> bool -//! - download_resume(task_id) -> bool -//! - download_cancel(task_id) -> bool - -use super::error::{DownloadError, Result}; -use super::traits::{Downloader, DownloaderCapabilities, ProgressCallback, RequestOptions}; -use async_trait::async_trait; -use parking_lot::RwLock; -use serde::{Deserialize, Serialize}; -use std::collections::HashMap; -use std::path::{Path, PathBuf}; - -/// JSON-RPC 2.0 request structure -#[derive(Serialize)] -struct JsonRpcRequest<'a> { - jsonrpc: &'static str, - method: &'a str, - params: serde_json::Value, - id: u64, -} - -/// JSON-RPC 2.0 response structure -#[derive(Deserialize)] -struct JsonRpcResponse { - #[allow(dead_code)] - jsonrpc: Option, - result: Option, - error: Option, - #[allow(dead_code)] - id: Option, -} - -#[derive(Deserialize)] -struct JsonRpcError { - #[allow(dead_code)] - code: i64, - message: String, -} - -/// Task ID response from external download_submit -#[derive(Deserialize)] -struct ExternalTaskIdResponse { - task_id: String, -} - -/// Task info from external service (subset we care about) -#[derive(Deserialize, Debug)] -struct ExternalTaskInfo { - #[allow(dead_code)] - task_id: String, - state: String, - progress: ExternalProgress, - error: Option, -} - -#[derive(Deserialize, Debug)] -struct ExternalProgress { - downloaded_bytes: u64, - total_bytes: Option, - #[allow(dead_code)] - speed_bytes_per_sec: Option, - #[allow(dead_code)] - eta_seconds: Option, -} - -/// Downloader that delegates all operations to an external JSON-RPC service. -/// -/// The external service is expected to implement the full downloader protocol -/// (submit, status, wait_for_change, pause, resume, cancel). -pub struct ExternalRpcDownloader { - rpc_url: String, - http_client: reqwest::Client, - /// url -> external task_id (for routing cancel/pause/resume by url) - task_mapping: RwLock>, - /// Atomic request ID counter - request_id: std::sync::atomic::AtomicU64, - capabilities: DownloaderCapabilities, -} - -impl ExternalRpcDownloader { - /// Create a new external RPC downloader pointing at the given service URL. - pub fn new(rpc_url: String) -> Self { - Self { - rpc_url, - http_client: reqwest::Client::new(), - task_mapping: RwLock::new(HashMap::new()), - request_id: std::sync::atomic::AtomicU64::new(1), - capabilities: DownloaderCapabilities::all_enabled(), - } - } - - /// Make a JSON-RPC call to the external service. - async fn rpc_call( - &self, - method: &str, - params: serde_json::Value, - ) -> Result { - let id = self - .request_id - .fetch_add(1, std::sync::atomic::Ordering::Relaxed); - - let request = JsonRpcRequest { - jsonrpc: "2.0", - method, - params, - id, - }; - - let response = self - .http_client - .post(&self.rpc_url) - .json(&request) - .send() - .await - .map_err(|e| DownloadError::network(format!("RPC request failed: {}", e)))?; - - let rpc_response: JsonRpcResponse = response - .json() - .await - .map_err(|e| DownloadError::network(format!("RPC response parse failed: {}", e)))?; - - if let Some(err) = rpc_response.error { - return Err(DownloadError::network(format!( - "RPC error: {}", - err.message - ))); - } - - let result = rpc_response - .result - .ok_or_else(|| DownloadError::network("RPC response missing result"))?; - - serde_json::from_value(result) - .map_err(|e| DownloadError::network(format!("RPC result deserialize failed: {}", e))) - } -} - -#[async_trait] -impl Downloader for ExternalRpcDownloader { - async fn download( - &self, - url: &str, - dest: &Path, - progress: Option, - options: Option, - ) -> Result<()> { - // 1. Submit download task to external service - let params = serde_json::json!({ - "url": url, - "dest_path": dest.to_str().unwrap_or(""), - "headers": options.as_ref().and_then(|o| o.headers.clone()), - "cookies": options.as_ref().and_then(|o| o.cookies.clone()), - }); - - let submit_response: ExternalTaskIdResponse = - self.rpc_call("download_submit", params).await?; - let external_task_id = submit_response.task_id; - - // 2. Record mapping for cancel/pause/resume - self.task_mapping - .write() - .insert(url.to_string(), external_task_id.clone()); - - // 3. Poll for status changes until terminal state - loop { - let params = serde_json::json!({ - "task_id": &external_task_id, - "timeout_seconds": 30_u64, - }); - - let task_info: ExternalTaskInfo = - match self.rpc_call("download_wait_for_change", params).await { - Ok(info) => info, - Err(e) => { - // On poll error, try a direct status check - let status_params = serde_json::json!({ - "task_id": &external_task_id, - }); - match self - .rpc_call::("download_get_status", status_params) - .await - { - Ok(info) => info, - Err(_) => { - // Both failed, clean up and return error - self.task_mapping.write().remove(url); - return Err(e); - } - } - } - }; - - // Update progress callback - if let Some(ref cb) = progress { - cb( - task_info.progress.downloaded_bytes, - task_info.progress.total_bytes, - ); - } - - // Check terminal states - match task_info.state.as_str() { - "completed" => { - self.task_mapping.write().remove(url); - return Ok(()); - } - "failed" => { - self.task_mapping.write().remove(url); - let msg = task_info - .error - .unwrap_or_else(|| "External download failed".to_string()); - return Err(DownloadError::network(msg)); - } - "cancelled" => { - self.task_mapping.write().remove(url); - return Err(DownloadError::cancelled("Download was cancelled")); - } - _ => { - // pending, downloading, stopped — continue polling - continue; - } - } - } - } - - async fn download_batch(&self, tasks: Vec<(String, PathBuf)>) -> Vec> { - // Simple sequential implementation for external downloaders - let mut results = Vec::with_capacity(tasks.len()); - for (url, dest) in tasks { - let result = self.download(&url, &dest, None, None).await; - results.push(result); - } - results - } - - fn name(&self) -> &str { - "external_rpc" - } - - fn capabilities(&self) -> &DownloaderCapabilities { - &self.capabilities - } - - async fn cancel(&self, url: &str) -> Result<()> { - let ext_id = self.task_mapping.read().get(url).cloned(); - if let Some(ext_id) = ext_id { - let params = serde_json::json!({"task_id": ext_id}); - let _: bool = self.rpc_call("download_cancel", params).await?; - } - Ok(()) - } - - async fn pause(&self, url: &str) -> Result<()> { - let ext_id = self.task_mapping.read().get(url).cloned(); - if let Some(ext_id) = ext_id { - let params = serde_json::json!({"task_id": ext_id}); - let _: bool = self.rpc_call("download_pause", params).await?; - } - Ok(()) - } - - async fn resume(&self, url: &str) -> Result<()> { - let ext_id = self.task_mapping.read().get(url).cloned(); - if let Some(ext_id) = ext_id { - let params = serde_json::json!({"task_id": ext_id}); - let _: bool = self.rpc_call("download_resume", params).await?; - } - Ok(()) - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_create_external_downloader() { - let dl = ExternalRpcDownloader::new("http://127.0.0.1:12345".to_string()); - assert_eq!(dl.name(), "external_rpc"); - assert!(dl.capabilities().supports_pause); - } -} diff --git a/src/downloader/hub_dispatch.rs b/src/downloader/hub_dispatch.rs deleted file mode 100644 index c6b753b..0000000 --- a/src/downloader/hub_dispatch.rs +++ /dev/null @@ -1,208 +0,0 @@ -//! Hub-based download dispatcher -//! -//! Routes download tasks to registered external downloaders based on hub_uuid. -//! If no external downloader is registered for the given hub_uuid, falls back -//! to the default built-in downloader (TraumaDownloader / HTTP). - -use super::error::Result; -use super::traits::{Downloader, DownloaderCapabilities, ProgressCallback, RequestOptions}; -use async_trait::async_trait; -use parking_lot::RwLock; -use std::collections::HashMap; -use std::path::{Path, PathBuf}; -use std::sync::Arc; - -/// Shared internal state for HubDispatchDownloader. -/// -/// Wrapped in Arc so that clones are cheap and share the same state. -/// This allows the RPC server to hold a reference for register/unregister -/// while DownloadTaskManager holds another reference as Box. -struct HubDispatchState { - /// hub_uuid -> external downloader - external: RwLock>>>, - /// Default downloader for unregistered hub_uuids (TraumaDownloader) - default: Arc>, - /// Active task tracking: url -> hub_uuid (for routing cancel/pause/resume) - active_tasks: RwLock>>, -} - -/// Downloads dispatcher that routes tasks by hub_uuid. -/// -/// - If `hub_uuid` is provided (via `RequestOptions.metadata["hub_uuid"]`) -/// and a downloader is registered for it, the task is routed there. -/// - Otherwise, the default built-in HTTP downloader handles it. -/// -/// This struct is cheaply cloneable (Arc-based shared state). -pub struct HubDispatchDownloader { - state: Arc, -} - -impl HubDispatchDownloader { - /// Create a new dispatcher with the given default downloader. - pub fn new(default: Box) -> Self { - Self { - state: Arc::new(HubDispatchState { - external: RwLock::new(HashMap::new()), - default: Arc::new(default), - active_tasks: RwLock::new(HashMap::new()), - }), - } - } - - /// Register an external downloader for a specific hub_uuid. - pub fn register(&self, hub_uuid: &str, downloader: Box) { - self.state - .external - .write() - .insert(hub_uuid.to_string(), Arc::new(downloader)); - } - - /// Unregister the external downloader for a specific hub_uuid. - pub fn unregister(&self, hub_uuid: &str) { - self.state.external.write().remove(hub_uuid); - } - - /// Check if a downloader is registered for the given hub_uuid. - pub fn has_downloader(&self, hub_uuid: &str) -> bool { - self.state.external.read().contains_key(hub_uuid) - } - - /// Resolve which downloader to use based on hub_uuid. - fn resolve(&self, hub_uuid: Option<&str>) -> Arc> { - if let Some(uuid) = hub_uuid { - if let Some(dl) = self.state.external.read().get(uuid) { - return dl.clone(); - } - } - self.state.default.clone() - } - - /// Extract hub_uuid from RequestOptions metadata. - fn extract_hub_uuid(options: Option<&RequestOptions>) -> Option { - options - .and_then(|o| o.metadata.as_ref()) - .and_then(|m| m.get("hub_uuid")) - .cloned() - } - - /// Record an active task for url -> hub_uuid routing. - fn track_task(&self, url: &str, hub_uuid: Option) { - self.state - .active_tasks - .write() - .insert(url.to_string(), hub_uuid); - } - - /// Remove an active task record. - fn untrack_task(&self, url: &str) { - self.state.active_tasks.write().remove(url); - } - - /// Look up which hub_uuid (if any) an active URL belongs to. - fn lookup_task_hub(&self, url: &str) -> Option> { - self.state.active_tasks.read().get(url).cloned() - } -} - -impl Clone for HubDispatchDownloader { - fn clone(&self) -> Self { - Self { - state: self.state.clone(), - } - } -} - -#[async_trait] -impl Downloader for HubDispatchDownloader { - async fn download( - &self, - url: &str, - dest: &Path, - progress: Option, - options: Option, - ) -> Result<()> { - let hub_uuid = Self::extract_hub_uuid(options.as_ref()); - let downloader = self.resolve(hub_uuid.as_deref()); - - // Track this task so cancel/pause/resume can find the right downloader - self.track_task(url, hub_uuid); - - let result = downloader.download(url, dest, progress, options).await; - - // Clean up tracking on completion (success or failure) - self.untrack_task(url); - - result - } - - async fn download_batch(&self, tasks: Vec<(String, PathBuf)>) -> Vec> { - // Batch downloads go through the default downloader since there's - // no per-task metadata available in the batch interface. - self.state.default.download_batch(tasks).await - } - - fn name(&self) -> &str { - "hub_dispatch" - } - - fn capabilities(&self) -> &DownloaderCapabilities { - // Return default backend capabilities - self.state.default.capabilities() - } - - async fn cancel(&self, url: &str) -> Result<()> { - let hub_uuid = self.lookup_task_hub(url).flatten(); - self.resolve(hub_uuid.as_deref()).cancel(url).await - } - - async fn pause(&self, url: &str) -> Result<()> { - let hub_uuid = self.lookup_task_hub(url).flatten(); - self.resolve(hub_uuid.as_deref()).pause(url).await - } - - async fn resume(&self, url: &str) -> Result<()> { - let hub_uuid = self.lookup_task_hub(url).flatten(); - self.resolve(hub_uuid.as_deref()).resume(url).await - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::downloader::TraumaDownloader; - - #[test] - fn test_create_dispatcher() { - let default = Box::new(TraumaDownloader::default_settings()); - let dispatcher = HubDispatchDownloader::new(default); - assert_eq!(dispatcher.name(), "hub_dispatch"); - assert!(!dispatcher.has_downloader("some-uuid")); - } - - #[test] - fn test_register_unregister() { - let default = Box::new(TraumaDownloader::default_settings()); - let dispatcher = HubDispatchDownloader::new(default); - - let ext = Box::new(TraumaDownloader::default_settings()); - dispatcher.register("test-uuid", ext); - assert!(dispatcher.has_downloader("test-uuid")); - - dispatcher.unregister("test-uuid"); - assert!(!dispatcher.has_downloader("test-uuid")); - } - - #[test] - fn test_clone_shares_state() { - let default = Box::new(TraumaDownloader::default_settings()); - let dispatcher = HubDispatchDownloader::new(default); - - let clone = dispatcher.clone(); - - let ext = Box::new(TraumaDownloader::default_settings()); - dispatcher.register("shared-uuid", ext); - - // Clone should see the registration - assert!(clone.has_downloader("shared-uuid")); - } -} diff --git a/src/downloader/mod.rs b/src/downloader/mod.rs deleted file mode 100644 index d4f2233..0000000 --- a/src/downloader/mod.rs +++ /dev/null @@ -1,58 +0,0 @@ -//! Downloader module providing pluggable download functionality -//! -//! This module implements a flexible, trait-based downloader system that supports: -//! - Multiple backend implementations (trauma, reqwest, custom CLI, etc.) -//! - Task state management with progress tracking -//! - Long-polling for download status updates -//! - Configuration-driven backend selection -//! - JSON-RPC integration for remote control - -mod config; -mod error; -mod external_rpc_impl; -mod hub_dispatch; -mod state; -mod task_manager; -mod traits; -mod trauma_impl; - -pub use config::{DownloadConfig, DownloaderBackend}; -pub use error::{DownloadError, Result}; -pub use external_rpc_impl::ExternalRpcDownloader; -pub use hub_dispatch::HubDispatchDownloader; -pub use state::{DownloadProgress, DownloadState, SpeedCalculator, TaskInfo}; -pub use task_manager::DownloadTaskManager; -pub use traits::Downloader; -pub use trauma_impl::TraumaDownloader; - -/// Create a downloader instance based on the provided configuration -pub fn create_downloader(config: &DownloadConfig) -> Box { - match config.backend { - DownloaderBackend::Trauma => Box::new(TraumaDownloader::new( - config.max_concurrent, - config.retries, - config.timeout_seconds, - )), - // Future implementations can be added here - // DownloaderBackend::Reqwest => Box::new(ReqwestDownloader::new(config)), - // DownloaderBackend::Custom => Box::new(CliDownloader::new(config)), - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_create_trauma_downloader() { - let config = DownloadConfig::default(); - let downloader = create_downloader(&config); - assert_eq!(downloader.name(), "reqwest"); - - // Test that capabilities are accessible - let caps = downloader.capabilities(); - assert!(caps.supports_pause); - assert!(caps.supports_resume); - assert!(caps.supports_cancellation); - } -} diff --git a/src/downloader/state.rs b/src/downloader/state.rs deleted file mode 100644 index 718b33b..0000000 --- a/src/downloader/state.rs +++ /dev/null @@ -1,532 +0,0 @@ -//! Download state and progress tracking types - -use serde::{Deserialize, Deserializer, Serialize, Serializer}; -use std::collections::VecDeque; -use std::time::{Instant, SystemTime, UNIX_EPOCH}; -use uuid::Uuid; - -/// Speed calculator using sliding window for smooth speed measurement -/// -/// Keeps track of download samples over a time window and calculates -/// average speed to avoid fluctuations. -#[derive(Debug, Clone)] -pub struct SpeedCalculator { - /// Samples: (timestamp, downloaded_bytes) - samples: VecDeque<(Instant, u64)>, - /// Window size in seconds - window_secs: u64, - /// Maximum number of samples to keep - max_samples: usize, - /// Start time for calculating speed when only one sample exists - start_time: Option, -} - -impl SpeedCalculator { - /// Create a new speed calculator with specified window size - pub fn new(window_secs: u64) -> Self { - Self { - samples: VecDeque::with_capacity(64), - window_secs, - max_samples: 64, - start_time: None, - } - } - - /// Create with default 5-second window - pub fn default_window() -> Self { - Self::new(5) - } - - /// Record a new sample - pub fn record(&mut self, downloaded_bytes: u64) { - let now = Instant::now(); - - // Initialize start time on first record - if self.start_time.is_none() { - self.start_time = Some(now); - } - - // Remove samples outside the window - let cutoff = now - std::time::Duration::from_secs(self.window_secs); - while let Some((time, _)) = self.samples.front() { - if *time < cutoff { - self.samples.pop_front(); - } else { - break; - } - } - - // Add new sample - self.samples.push_back((now, downloaded_bytes)); - - // Limit max samples - while self.samples.len() > self.max_samples { - self.samples.pop_front(); - } - } - - /// Calculate average speed in bytes per second - pub fn speed_bytes_per_sec(&self) -> Option { - let (last_time, last_bytes) = self.samples.back()?; - - // Use sliding window if we have multiple samples - if self.samples.len() >= 2 { - let (first_time, first_bytes) = self.samples.front()?; - let duration = last_time.duration_since(*first_time); - let duration_secs = duration.as_secs_f64().max(0.001); - let bytes_diff = last_bytes.saturating_sub(*first_bytes); - return Some((bytes_diff as f64 / duration_secs) as u64); - } - - // For single sample, calculate from start time - if let Some(start) = self.start_time { - let duration = last_time.duration_since(start); - let duration_secs = duration.as_secs_f64().max(0.001); - return Some((*last_bytes as f64 / duration_secs) as u64); - } - - None - } - - /// Reset the calculator - pub fn reset(&mut self) { - self.samples.clear(); - self.start_time = None; - } -} - -/// Serialize SystemTime as Unix timestamp in milliseconds -fn serialize_system_time(time: &SystemTime, serializer: S) -> Result -where - S: Serializer, -{ - let millis = time - .duration_since(UNIX_EPOCH) - .map(|d| d.as_millis() as i64) - .unwrap_or(0); - serializer.serialize_i64(millis) -} - -/// Serialize Option as Unix timestamp in milliseconds -fn serialize_option_system_time( - time: &Option, - serializer: S, -) -> Result -where - S: Serializer, -{ - match time { - Some(t) => { - let millis = t - .duration_since(UNIX_EPOCH) - .map(|d| d.as_millis() as i64) - .unwrap_or(0); - serializer.serialize_some(&millis) - } - None => serializer.serialize_none(), - } -} - -/// Deserialize Unix timestamp in milliseconds to SystemTime -fn deserialize_system_time<'de, D>(deserializer: D) -> Result -where - D: Deserializer<'de>, -{ - let millis = i64::deserialize(deserializer)?; - Ok(UNIX_EPOCH + std::time::Duration::from_millis(millis as u64)) -} - -/// Deserialize Unix timestamp in milliseconds to Option -fn deserialize_option_system_time<'de, D>(deserializer: D) -> Result, D::Error> -where - D: Deserializer<'de>, -{ - let opt: Option = Option::deserialize(deserializer)?; - Ok(opt.map(|millis| UNIX_EPOCH + std::time::Duration::from_millis(millis as u64))) -} - -/// Download task state -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "lowercase")] -pub enum DownloadState { - /// Task is queued but not started yet - Pending, - /// Download is in progress - Downloading, - /// Download is paused (can be resumed) - Stopped, - /// Download completed successfully - Completed, - /// Download failed - Failed, - /// Download was cancelled - Cancelled, -} - -impl DownloadState { - /// Check if the task is in a terminal state (completed, failed, or cancelled) - pub fn is_terminal(&self) -> bool { - matches!( - self, - DownloadState::Completed | DownloadState::Failed | DownloadState::Cancelled - ) - } - - /// Check if the task is active (pending, downloading, or stopped/paused) - pub fn is_active(&self) -> bool { - matches!( - self, - DownloadState::Pending | DownloadState::Downloading | DownloadState::Stopped - ) - } - - /// Check if the task can be resumed - pub fn is_resumable(&self) -> bool { - matches!(self, DownloadState::Stopped | DownloadState::Failed) - } - - /// Check if the task can be paused - pub fn is_pausable(&self) -> bool { - matches!(self, DownloadState::Downloading) - } -} - -/// Download progress information -#[derive(Debug, Clone, Serialize, Deserialize, Default)] -pub struct DownloadProgress { - /// Number of bytes downloaded so far - pub downloaded_bytes: u64, - - /// Total size in bytes (None if unknown) - pub total_bytes: Option, - - /// Download speed in bytes per second (average) - pub speed_bytes_per_sec: Option, - - /// Estimated time remaining in seconds (None if unknown) - pub eta_seconds: Option, -} - -impl DownloadProgress { - /// Create a new progress with downloaded bytes - pub fn new(downloaded_bytes: u64, total_bytes: Option) -> Self { - Self { - downloaded_bytes, - total_bytes, - speed_bytes_per_sec: None, - eta_seconds: None, - } - } - - /// Create a new progress with speed and ETA - pub fn with_speed( - downloaded_bytes: u64, - total_bytes: Option, - speed_bytes_per_sec: Option, - ) -> Self { - let eta_seconds = match (total_bytes, speed_bytes_per_sec) { - (Some(total), Some(speed)) if speed > 0 && total > downloaded_bytes => { - Some((total - downloaded_bytes) / speed) - } - _ => None, - }; - - Self { - downloaded_bytes, - total_bytes, - speed_bytes_per_sec, - eta_seconds, - } - } - - /// Calculate progress percentage (0-100) - pub fn percentage(&self) -> Option { - self.total_bytes.map(|total| { - if total == 0 { - 0.0 - } else { - (self.downloaded_bytes as f64 / total as f64) * 100.0 - } - }) - } - - /// Check if download is complete - pub fn is_complete(&self) -> bool { - if let Some(total) = self.total_bytes { - self.downloaded_bytes >= total - } else { - false - } - } -} - -/// Complete information about a download task -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct TaskInfo { - /// Unique task identifier - pub task_id: String, - - /// Source URL - pub url: String, - - /// Destination file path - pub dest_path: String, - - /// Current state - pub state: DownloadState, - - /// Progress information - pub progress: DownloadProgress, - - /// Resume offset in bytes (for resuming paused downloads) - pub resume_offset: u64, - - /// Whether the server supports HTTP Range requests (None if not tested yet) - pub supports_range: Option, - - /// Error message (if state is Failed) - pub error: Option, - - /// Task creation timestamp (Unix timestamp in milliseconds) - #[serde( - serialize_with = "serialize_system_time", - deserialize_with = "deserialize_system_time" - )] - pub created_at: SystemTime, - - /// Task start timestamp (Unix timestamp in milliseconds, None if not started yet) - #[serde( - serialize_with = "serialize_option_system_time", - deserialize_with = "deserialize_option_system_time" - )] - pub started_at: Option, - - /// Task completion timestamp (Unix timestamp in milliseconds, None if not completed) - #[serde( - serialize_with = "serialize_option_system_time", - deserialize_with = "deserialize_option_system_time" - )] - pub completed_at: Option, - - /// Task last paused timestamp (Unix timestamp in milliseconds, None if never paused) - #[serde( - serialize_with = "serialize_option_system_time", - deserialize_with = "deserialize_option_system_time" - )] - pub paused_at: Option, - - /// HTTP headers to include in requests - #[serde(default)] - pub headers: Option>, - - /// HTTP cookies to include in requests - #[serde(default)] - pub cookies: Option>, - - /// Hub UUID for dispatching to registered external downloaders. - /// When set, the download task manager routes this task to the - /// external downloader registered for this hub_uuid. - /// When None, the default built-in downloader is used. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub hub_uuid: Option, -} - -impl TaskInfo { - /// Create a new task info - pub fn new(url: impl Into, dest_path: impl Into) -> Self { - Self { - task_id: Uuid::new_v4().to_string(), - url: url.into(), - dest_path: dest_path.into(), - state: DownloadState::Pending, - progress: DownloadProgress::default(), - resume_offset: 0, - supports_range: None, - error: None, - created_at: SystemTime::now(), - started_at: None, - completed_at: None, - paused_at: None, - headers: None, - cookies: None, - hub_uuid: None, - } - } - - /// Create with a specific task ID - pub fn with_id( - task_id: impl Into, - url: impl Into, - dest_path: impl Into, - ) -> Self { - Self { - task_id: task_id.into(), - url: url.into(), - dest_path: dest_path.into(), - state: DownloadState::Pending, - progress: DownloadProgress::default(), - resume_offset: 0, - supports_range: None, - error: None, - created_at: SystemTime::now(), - started_at: None, - completed_at: None, - paused_at: None, - headers: None, - cookies: None, - hub_uuid: None, - } - } - - /// Create with headers, cookies, and optional hub_uuid - pub fn with_options( - task_id: impl Into, - url: impl Into, - dest_path: impl Into, - headers: Option>, - cookies: Option>, - hub_uuid: Option, - ) -> Self { - Self { - task_id: task_id.into(), - url: url.into(), - dest_path: dest_path.into(), - state: DownloadState::Pending, - progress: DownloadProgress::default(), - resume_offset: 0, - supports_range: None, - error: None, - created_at: SystemTime::now(), - started_at: None, - completed_at: None, - paused_at: None, - headers, - cookies, - hub_uuid, - } - } - - /// Mark task as started - pub fn mark_started(&mut self) { - self.state = DownloadState::Downloading; - self.started_at = Some(SystemTime::now()); - } - - /// Mark task as completed - pub fn mark_completed(&mut self) { - self.state = DownloadState::Completed; - self.completed_at = Some(SystemTime::now()); - } - - /// Mark task as failed with error message - pub fn mark_failed(&mut self, error: impl Into) { - self.state = DownloadState::Failed; - self.error = Some(error.into()); - self.completed_at = Some(SystemTime::now()); - } - - /// Mark task as cancelled - pub fn mark_cancelled(&mut self) { - self.state = DownloadState::Cancelled; - self.completed_at = Some(SystemTime::now()); - } - - /// Mark task as stopped/paused - pub fn mark_stopped(&mut self) { - self.state = DownloadState::Stopped; - self.resume_offset = self.progress.downloaded_bytes; - self.paused_at = Some(SystemTime::now()); - } - - /// Mark task as resumed from stopped state - pub fn mark_resumed(&mut self) { - self.state = DownloadState::Downloading; - self.paused_at = None; - } - - /// Update progress - pub fn update_progress(&mut self, progress: DownloadProgress) { - self.progress = progress; - } - - /// Set whether the server supports Range requests - pub fn set_range_support(&mut self, supports: bool) { - self.supports_range = Some(supports); - } - - /// Calculate elapsed time in seconds - pub fn elapsed_seconds(&self) -> Option { - self.started_at.and_then(|start| { - SystemTime::now() - .duration_since(start) - .ok() - .map(|d| d.as_secs()) - }) - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_download_state() { - assert!(DownloadState::Completed.is_terminal()); - assert!(DownloadState::Failed.is_terminal()); - assert!(DownloadState::Cancelled.is_terminal()); - assert!(!DownloadState::Pending.is_terminal()); - assert!(!DownloadState::Downloading.is_terminal()); - assert!(!DownloadState::Stopped.is_terminal()); - - assert!(DownloadState::Pending.is_active()); - assert!(DownloadState::Downloading.is_active()); - assert!(DownloadState::Stopped.is_active()); - assert!(!DownloadState::Completed.is_active()); - - assert!(DownloadState::Downloading.is_pausable()); - assert!(!DownloadState::Stopped.is_pausable()); - assert!(!DownloadState::Completed.is_pausable()); - - assert!(DownloadState::Stopped.is_resumable()); - assert!(DownloadState::Failed.is_resumable()); - assert!(!DownloadState::Downloading.is_resumable()); - assert!(!DownloadState::Completed.is_resumable()); - } - - #[test] - fn test_download_progress() { - let mut progress = DownloadProgress::new(50, Some(100)); - assert_eq!(progress.percentage(), Some(50.0)); - assert!(!progress.is_complete()); - - progress.downloaded_bytes = 100; - assert_eq!(progress.percentage(), Some(100.0)); - assert!(progress.is_complete()); - } - - #[test] - fn test_task_info() { - let mut task = TaskInfo::new("http://example.com/file", "/tmp/file"); - assert_eq!(task.state, DownloadState::Pending); - assert!(task.started_at.is_none()); - assert!(task.completed_at.is_none()); - - task.mark_started(); - assert_eq!(task.state, DownloadState::Downloading); - assert!(task.started_at.is_some()); - - task.mark_completed(); - assert_eq!(task.state, DownloadState::Completed); - assert!(task.completed_at.is_some()); - } - - #[test] - fn test_task_info_failure() { - let mut task = TaskInfo::new("http://example.com/file", "/tmp/file"); - task.mark_failed("Network error"); - - assert_eq!(task.state, DownloadState::Failed); - assert_eq!(task.error, Some("Network error".to_string())); - assert!(task.completed_at.is_some()); - } -} diff --git a/src/downloader/task_manager.rs b/src/downloader/task_manager.rs deleted file mode 100644 index 4750517..0000000 --- a/src/downloader/task_manager.rs +++ /dev/null @@ -1,1009 +0,0 @@ -//! Download task manager with state tracking and long-polling support - -use super::config::DownloadConfig; -use super::error::{DownloadError, Result}; -use super::state::{DownloadProgress, DownloadState, SpeedCalculator, TaskInfo}; -use super::traits::{Downloader, DownloaderCapabilities}; -use parking_lot::RwLock; -use std::collections::HashMap; -use std::path::PathBuf; -use std::sync::Arc; -use std::time::{Duration, SystemTime}; -use tokio::sync::Notify; -use uuid::Uuid; - -/// Download task manager -/// -/// Manages all download tasks, tracks their state, and provides long-polling support -/// for status updates. -pub struct DownloadTaskManager { - /// Task storage: task_id -> TaskInfo - tasks: Arc>>, - - /// Downloader implementation - downloader: Arc>, - - /// Notification system for state changes (used for long-polling) - notifier: Arc, - - /// Downloader capabilities (cached from downloader at initialization) - capabilities: DownloaderCapabilities, -} - -impl DownloadTaskManager { - /// Create a new task manager with the given downloader - pub fn new(downloader: Box) -> Self { - // Cache capabilities from downloader at initialization - let capabilities = downloader.capabilities().clone(); - - Self { - tasks: Arc::new(RwLock::new(HashMap::new())), - downloader: Arc::new(downloader), - notifier: Arc::new(Notify::new()), - capabilities, - } - } - - /// Create a task manager from configuration - pub fn from_config(config: &DownloadConfig) -> Self { - let downloader = super::create_downloader(config); - Self::new(downloader) - } - - /// Submit a new download task - /// - /// # Arguments - /// * `url` - The URL to download from - /// * `dest_path` - The destination file path - /// - /// # Returns - /// * `Ok(task_id)` - The unique task ID - /// * `Err(DownloadError)` - If the task could not be created - pub fn submit_task( - &self, - url: impl Into, - dest_path: impl Into, - ) -> Result { - let task_id = Uuid::new_v4().to_string(); - let task = TaskInfo::with_id(task_id.clone(), url, dest_path); - - { - let mut tasks = self.tasks.write(); - if tasks.contains_key(&task_id) { - return Err(DownloadError::task_already_exists(&task_id)); - } - tasks.insert(task_id.clone(), task); - } - - // Notify listeners about new task - self.notifier.notify_waiters(); - - // Start download in background - let manager = self.clone_for_task(); - let task_id_clone = task_id.clone(); - tokio::spawn(async move { - manager.execute_task(&task_id_clone).await; - }); - - Ok(task_id) - } - - /// Submit a new download task with optional headers, cookies, and hub_uuid - /// - /// # Arguments - /// * `url` - The URL to download from - /// * `dest_path` - The destination file path - /// * `headers` - Optional HTTP headers - /// * `cookies` - Optional HTTP cookies - /// * `hub_uuid` - Optional hub UUID for routing to external downloaders - /// - /// # Returns - /// * `Ok(task_id)` - The unique task ID - /// * `Err(DownloadError)` - If the task could not be created - pub fn submit_task_with_options( - &self, - url: impl Into, - dest_path: impl Into, - headers: Option>, - cookies: Option>, - hub_uuid: Option, - ) -> Result { - let task_id = Uuid::new_v4().to_string(); - let task = - TaskInfo::with_options(task_id.clone(), url, dest_path, headers, cookies, hub_uuid); - - { - let mut tasks = self.tasks.write(); - if tasks.contains_key(&task_id) { - return Err(DownloadError::task_already_exists(&task_id)); - } - tasks.insert(task_id.clone(), task); - } - - // Notify listeners about new task - self.notifier.notify_waiters(); - - // Start download in background - let manager = self.clone_for_task(); - let task_id_clone = task_id.clone(); - tokio::spawn(async move { - manager.execute_task(&task_id_clone).await; - }); - - Ok(task_id) - } - - /// Submit multiple download tasks - /// - /// # Returns - /// Vector of task IDs for each submitted task - pub fn submit_batch(&self, tasks: Vec<(String, String)>) -> Result> { - let mut task_ids = Vec::new(); - - for (url, dest_path) in tasks { - let task_id = self.submit_task(url, dest_path)?; - task_ids.push(task_id); - } - - Ok(task_ids) - } - - /// Get task information by ID - pub fn get_task(&self, task_id: &str) -> Result { - let tasks = self.tasks.read(); - tasks - .get(task_id) - .cloned() - .ok_or_else(|| DownloadError::task_not_found(task_id)) - } - - /// Get all tasks - pub fn get_all_tasks(&self) -> Vec { - let tasks = self.tasks.read(); - tasks.values().cloned().collect() - } - - /// Get tasks by state - pub fn get_tasks_by_state(&self, state: DownloadState) -> Vec { - let tasks = self.tasks.read(); - tasks - .values() - .filter(|task| task.state == state) - .cloned() - .collect() - } - - /// Get active tasks (pending or downloading) - pub fn get_active_tasks(&self) -> Vec { - let tasks = self.tasks.read(); - tasks - .values() - .filter(|task| task.state.is_active()) - .cloned() - .collect() - } - - /// Get downloader capabilities - /// - /// Returns the capabilities of the underlying downloader implementation. - /// These capabilities are determined at initialization time. - pub fn get_capabilities(&self) -> &DownloaderCapabilities { - &self.capabilities - } - - /// Cancel a task - pub fn cancel_task(&self, task_id: &str) -> Result<()> { - let mut tasks = self.tasks.write(); - let task = tasks - .get_mut(task_id) - .ok_or_else(|| DownloadError::task_not_found(task_id))?; - - if task.state.is_terminal() { - return Err(DownloadError::invalid_input(format!( - "Task {} is already in terminal state: {:?}", - task_id, task.state - ))); - } - - task.mark_cancelled(); - self.notifier.notify_waiters(); - - Ok(()) - } - - /// Pause a download task - /// - /// # Arguments - /// * `task_id` - The task ID to pause - /// - /// # Returns - /// * `Ok(())` - Task paused successfully - /// * `Err(DownloadError)` - If task not found or cannot be paused - pub async fn pause_task(&self, task_id: &str) -> Result<()> { - // Check if downloader supports pause - if !self.capabilities.supports_pause { - return Err(DownloadError::unsupported( - "This downloader does not support pause functionality", - )); - } - - // Get task URL and verify state - let url = { - let tasks = self.tasks.read(); - let task = tasks - .get(task_id) - .ok_or_else(|| DownloadError::task_not_found(task_id))?; - - if !task.state.is_pausable() { - return Err(DownloadError::invalid_input(format!( - "Task {} cannot be paused in state: {:?}", - task_id, task.state - ))); - } - - task.url.clone() - }; - - // Call downloader pause - self.downloader.pause(&url).await?; - - // Update task state - { - let mut tasks = self.tasks.write(); - if let Some(task) = tasks.get_mut(task_id) { - task.mark_stopped(); - } - } - - self.notifier.notify_waiters(); - Ok(()) - } - - /// Resume a paused download task - /// - /// # Arguments - /// * `task_id` - The task ID to resume - /// - /// # Returns - /// * `Ok(())` - Task resumed successfully - /// * `Err(DownloadError)` - If task not found or cannot be resumed - pub async fn resume_task(&self, task_id: &str) -> Result<()> { - // Check if downloader supports resume - if !self.capabilities.supports_resume { - return Err(DownloadError::unsupported( - "This downloader does not support resume functionality", - )); - } - - // Get task details and verify state - let (url, dest_path, headers, cookies, hub_uuid) = { - let tasks = self.tasks.read(); - let task = tasks - .get(task_id) - .ok_or_else(|| DownloadError::task_not_found(task_id))?; - - if !task.state.is_resumable() { - return Err(DownloadError::invalid_input(format!( - "Task {} cannot be resumed from state: {:?}", - task_id, task.state - ))); - } - - ( - task.url.clone(), - task.dest_path.clone(), - task.headers.clone(), - task.cookies.clone(), - task.hub_uuid.clone(), - ) - }; - - // Mark as resumed - { - let mut tasks = self.tasks.write(); - if let Some(task) = tasks.get_mut(task_id) { - task.mark_resumed(); - } - } - - self.notifier.notify_waiters(); - - // Restart download in background - let manager = self.clone_for_task(); - let task_id_clone = task_id.to_string(); - tokio::spawn(async move { - // Create request options with headers, cookies, and hub_uuid metadata - let mut metadata = std::collections::HashMap::new(); - if let Some(uuid) = &hub_uuid { - metadata.insert("hub_uuid".to_string(), uuid.clone()); - } - - let options = if headers.is_some() || cookies.is_some() || !metadata.is_empty() { - Some(super::traits::RequestOptions { - headers, - cookies, - metadata: if metadata.is_empty() { - None - } else { - Some(metadata) - }, - }) - } else { - None - }; - - // Create progress callback with speed calculator - let tasks_clone = manager.tasks.clone(); - let task_id_for_callback = task_id_clone.clone(); - let notifier_clone = manager.notifier.clone(); - let speed_calc = Arc::new(RwLock::new(SpeedCalculator::default_window())); - - let progress_callback = Box::new(move |downloaded: u64, total: Option| { - // Record sample and calculate speed - let speed = { - let mut calc = speed_calc.write(); - calc.record(downloaded); - calc.speed_bytes_per_sec() - }; - - // Update task progress - let mut tasks = tasks_clone.write(); - if let Some(task) = tasks.get_mut(&task_id_for_callback) { - task.update_progress(DownloadProgress::with_speed(downloaded, total, speed)); - notifier_clone.notify_waiters(); - } - }); - - // Resume download - let result = manager - .downloader - .download( - &url, - &PathBuf::from(&dest_path), - Some(progress_callback), - options, - ) - .await; - - // Update task state - { - let mut tasks = manager.tasks.write(); - if let Some(task) = tasks.get_mut(&task_id_clone) { - match result { - Ok(_) => task.mark_completed(), - Err(e) => task.mark_failed(e.message), - } - } - } - - manager.notifier.notify_waiters(); - }); - - Ok(()) - } - - /// Wait for task state change (long-polling support) - /// - /// # Arguments - /// * `task_id` - The task ID to monitor - /// * `timeout` - Maximum time to wait for a change - /// - /// # Returns - /// * `Ok(TaskInfo)` - The updated task info - /// * `Err(DownloadError)` - If task not found or timeout occurred - pub async fn wait_for_change(&self, task_id: &str, timeout: Duration) -> Result { - let initial_state = { - let tasks = self.tasks.read(); - let task = tasks - .get(task_id) - .ok_or_else(|| DownloadError::task_not_found(task_id))?; - task.state - }; - - // If already in terminal state, return immediately - if initial_state.is_terminal() { - return self.get_task(task_id); - } - - // Wait for notification with timeout - let notifier = self.notifier.clone(); - let result = tokio::time::timeout(timeout, async { - loop { - notifier.notified().await; - - let tasks = self.tasks.read(); - if let Some(task) = tasks.get(task_id) { - if task.state != initial_state { - return Ok(task.clone()); - } - } else { - return Err(DownloadError::task_not_found(task_id)); - } - } - }) - .await; - - match result { - Ok(task_result) => task_result, - Err(_) => { - // Timeout - return current state - self.get_task(task_id) - } - } - } - - /// Remove completed/failed tasks older than the specified duration - pub fn cleanup_old_tasks(&self, max_age: Duration) { - let mut tasks = self.tasks.write(); - let now = SystemTime::now(); - - tasks.retain(|_, task| { - if !task.state.is_terminal() { - return true; // Keep active tasks - } - - if let Some(completed_at) = task.completed_at { - if let Ok(age) = now.duration_since(completed_at) { - return age < max_age; - } - } - - true - }); - - self.notifier.notify_waiters(); - } - - /// Remove a specific task - pub fn remove_task(&self, task_id: &str) -> Result<()> { - let mut tasks = self.tasks.write(); - let task = tasks - .get(task_id) - .ok_or_else(|| DownloadError::task_not_found(task_id))?; - - if !task.state.is_terminal() { - return Err(DownloadError::invalid_input(format!( - "Cannot remove active task: {}", - task_id - ))); - } - - tasks.remove(task_id); - self.notifier.notify_waiters(); - - Ok(()) - } - - /// Execute a download task - async fn execute_task(&self, task_id: &str) { - // Mark task as started - { - let mut tasks = self.tasks.write(); - if let Some(task) = tasks.get_mut(task_id) { - if task.state == DownloadState::Cancelled { - return; // Task was cancelled before it started - } - task.mark_started(); - } else { - return; // Task not found - } - } - - self.notifier.notify_waiters(); - - // Get task details - let (url, dest_path, headers, cookies, hub_uuid) = { - let tasks = self.tasks.read(); - if let Some(task) = tasks.get(task_id) { - ( - task.url.clone(), - task.dest_path.clone(), - task.headers.clone(), - task.cookies.clone(), - task.hub_uuid.clone(), - ) - } else { - return; - } - }; - - // Create request options with headers, cookies, and hub_uuid metadata - let mut metadata = std::collections::HashMap::new(); - if let Some(uuid) = &hub_uuid { - metadata.insert("hub_uuid".to_string(), uuid.clone()); - } - - let options = if headers.is_some() || cookies.is_some() || !metadata.is_empty() { - Some(super::traits::RequestOptions { - headers, - cookies, - metadata: if metadata.is_empty() { - None - } else { - Some(metadata) - }, - }) - } else { - None - }; - - // Create progress callback with speed calculator - let tasks_clone = self.tasks.clone(); - let task_id_clone = task_id.to_string(); - let notifier_clone = self.notifier.clone(); - let speed_calc = Arc::new(RwLock::new(SpeedCalculator::default_window())); - - let progress_callback = Box::new(move |downloaded: u64, total: Option| { - // Record sample and calculate speed - let speed = { - let mut calc = speed_calc.write(); - calc.record(downloaded); - calc.speed_bytes_per_sec() - }; - - // Update task progress - let mut tasks = tasks_clone.write(); - if let Some(task) = tasks.get_mut(&task_id_clone) { - task.update_progress(DownloadProgress::with_speed(downloaded, total, speed)); - notifier_clone.notify_waiters(); - } - }); - - // Perform download - let result = self - .downloader - .download( - &url, - &PathBuf::from(&dest_path), - Some(progress_callback), - options, - ) - .await; - - // Update task state - { - let mut tasks = self.tasks.write(); - if let Some(task) = tasks.get_mut(task_id) { - match result { - Ok(_) => task.mark_completed(), - Err(e) => task.mark_failed(e.message), - } - } - } - - self.notifier.notify_waiters(); - } - - /// Clone for task execution - fn clone_for_task(&self) -> Self { - Self { - tasks: self.tasks.clone(), - downloader: self.downloader.clone(), - notifier: self.notifier.clone(), - capabilities: self.capabilities.clone(), - } - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::downloader::TraumaDownloader; - - #[test] - fn test_create_manager() { - let downloader = Box::new(TraumaDownloader::default_settings()); - let manager = DownloadTaskManager::new(downloader); - - let all_tasks = manager.get_all_tasks(); - assert!(all_tasks.is_empty()); - } - - #[tokio::test] - async fn test_submit_task() { - let config = DownloadConfig::default(); - let manager = DownloadTaskManager::from_config(&config); - - let task_id = manager - .submit_task("http://example.com/file", "/tmp/file") - .unwrap(); - assert!(!task_id.is_empty()); - - let task = manager.get_task(&task_id).unwrap(); - assert_eq!(task.url, "http://example.com/file"); - assert_eq!(task.dest_path, "/tmp/file"); - } - - #[tokio::test] - async fn test_cancel_task() { - let config = DownloadConfig::default(); - let manager = DownloadTaskManager::from_config(&config); - - let task_id = manager - .submit_task("http://example.com/file", "/tmp/file") - .unwrap(); - - // Wait a bit for task to potentially start - tokio::time::sleep(Duration::from_millis(100)).await; - - let result = manager.cancel_task(&task_id); - // May succeed or fail depending on task state - let _ = result; - } - - #[tokio::test] - async fn test_get_tasks_by_state() { - let config = DownloadConfig::default(); - let manager = DownloadTaskManager::from_config(&config); - - let _ = manager.submit_task("http://example.com/file1", "/tmp/file1"); - let _ = manager.submit_task("http://example.com/file2", "/tmp/file2"); - - let pending_tasks = manager.get_tasks_by_state(DownloadState::Pending); - assert!(pending_tasks.len() <= 2); // May have started downloading - } - - // ======================================================================== - // Integration Tests - TaskManager with Mock HTTP Server - // ======================================================================== - - /// Test Scenario 5: Headers and Cookies transmission to downloader - /// Verifies: Custom headers and cookies are correctly sent to server - #[tokio::test] - async fn test_custom_headers_and_cookies_transmission() { - use mockito::Server; - use std::collections::HashMap; - use tempfile::tempdir; - - // Create Mock HTTP server - let mut server = Server::new_async().await; - - // Mock verifies custom headers and cookies - // Note: Cookie order is not guaranteed due to HashMap iteration - let mock = server - .mock("GET", "/protected-file.txt") - .match_header("authorization", "Bearer test-token-123") - .match_header("x-custom-header", "custom-value") - .match_header( - "cookie", - mockito::Matcher::Regex(".*session_id=abc123.*".to_string()), - ) - .match_header( - "cookie", - mockito::Matcher::Regex(".*user_id=456.*".to_string()), - ) - .with_status(200) - .with_body(b"Protected content") - .create(); - - // Prepare custom headers and cookies - let mut headers = HashMap::new(); - headers.insert( - "Authorization".to_string(), - "Bearer test-token-123".to_string(), - ); - headers.insert("X-Custom-Header".to_string(), "custom-value".to_string()); - - let mut cookies = HashMap::new(); - cookies.insert("session_id".to_string(), "abc123".to_string()); - cookies.insert("user_id".to_string(), "456".to_string()); - - // Create task manager - let config = DownloadConfig::default(); - let manager = DownloadTaskManager::from_config(&config); - - // Prepare download destination - let temp_dir = tempdir().unwrap(); - let dest = temp_dir.path().join("protected-file.txt"); - let url = format!("{}/protected-file.txt", server.url()); - - // Submit task with headers and cookies - let task_id = manager - .submit_task_with_options( - &url, - dest.to_str().unwrap(), - Some(headers), - Some(cookies), - None, - ) - .unwrap(); - - // Wait for download to complete - tokio::time::sleep(Duration::from_millis(500)).await; - - // Verify task status - let task_info = manager.get_task(&task_id).unwrap(); - assert!( - task_info.state == DownloadState::Completed - || task_info.state == DownloadState::Downloading, - "Task should be completed or downloading" - ); - - // Wait for task to fully complete - for _ in 0..20 { - tokio::time::sleep(Duration::from_millis(100)).await; - let task_info = manager.get_task(&task_id).unwrap(); - if task_info.state == DownloadState::Completed { - break; - } - } - - // Verify mock was called (headers and cookies transmitted correctly) - mock.assert(); - - // Verify file downloaded successfully - assert!(dest.exists(), "File should be downloaded"); - let content = std::fs::read_to_string(&dest).unwrap(); - assert_eq!(content, "Protected content"); - } - - /// Test Scenario 6: Task lifecycle state tracking - /// Verifies: Complete flow from Pending → Downloading → Completed - #[tokio::test] - async fn test_task_lifecycle_and_state_transitions() { - use mockito::Server; - use tempfile::tempdir; - - // Create Mock HTTP server - let mut server = Server::new_async().await; - let test_data = b"Task lifecycle test data"; - - let mock = server - .mock("GET", "/lifecycle-test.txt") - .with_status(200) - .with_header("content-length", &test_data.len().to_string()) - .with_body(test_data.as_slice()) - .create(); - - // Create task manager - let config = DownloadConfig::default(); - let manager = DownloadTaskManager::from_config(&config); - - // Prepare download destination - let temp_dir = tempdir().unwrap(); - let dest = temp_dir.path().join("lifecycle-test.txt"); - let url = format!("{}/lifecycle-test.txt", server.url()); - - // Submit task - let task_id = manager.submit_task(&url, dest.to_str().unwrap()).unwrap(); - - // Track state changes - let mut states_observed = Vec::new(); - - // Initial state should be Pending - let initial_task = manager.get_task(&task_id).unwrap(); - states_observed.push(initial_task.state); - assert_eq!( - initial_task.state, - DownloadState::Pending, - "Initial state should be Pending" - ); - - // Monitor state transitions (max 3 seconds) - for _ in 0..30 { - tokio::time::sleep(Duration::from_millis(100)).await; - - if let Ok(task_info) = manager.get_task(&task_id) { - let current_state = task_info.state; - - // Record new state - if states_observed.last() != Some(¤t_state) { - states_observed.push(current_state); - } - - // If completed, verify file exists - if current_state == DownloadState::Completed { - assert!(dest.exists(), "File should exist when completed"); - let content = std::fs::read(&dest).unwrap(); - assert_eq!(content, test_data); - break; - } - - // If failed, record error - if current_state == DownloadState::Failed { - if let Some(error) = &task_info.error { - eprintln!("Download failed: {}", error); - } - panic!("Download should not fail in this test"); - } - } - } - - // Verify state transition sequence - assert!( - states_observed.contains(&DownloadState::Pending), - "Should have observed Pending state" - ); - assert!( - states_observed.contains(&DownloadState::Downloading) - || states_observed.contains(&DownloadState::Completed), - "Should have observed Downloading or Completed state" - ); - - // Final state should be Completed - let final_task = manager.get_task(&task_id).unwrap(); - assert_eq!( - final_task.state, - DownloadState::Completed, - "Final state should be Completed. States observed: {:?}", - states_observed - ); - - // Verify progress information - assert_eq!( - final_task.progress.downloaded_bytes, - test_data.len() as u64, - "Downloaded bytes should match content length" - ); - assert_eq!( - final_task.progress.total_bytes, - Some(test_data.len() as u64), - "Total bytes should be known" - ); - - mock.assert(); - } - - /// Test Scenario 7: Long-polling wait_for_change mechanism - /// Verifies: Correct notification on state change, timeout mechanism works - #[tokio::test] - async fn test_wait_for_change_notification() { - use mockito::Server; - use tempfile::tempdir; - - // Create Mock HTTP server with slow response - let mut server = Server::new_async().await; - let test_data = vec![b'W'; 2048]; // 2KB data - - let mock = server - .mock("GET", "/slow-file.bin") - .with_status(200) - .with_header("content-length", &test_data.len().to_string()) - .with_chunked_body(move |w| { - // Send chunks slowly - for chunk in test_data.chunks(512) { - std::thread::sleep(Duration::from_millis(200)); - w.write_all(chunk)?; - } - Ok(()) - }) - .create(); - - // Create task manager - let config = DownloadConfig::default(); - let manager = DownloadTaskManager::from_config(&config); - - // Prepare download destination - let temp_dir = tempdir().unwrap(); - let dest = temp_dir.path().join("slow-file.bin"); - let url = format!("{}/slow-file.bin", server.url()); - - // Submit task - let task_id = manager.submit_task(&url, dest.to_str().unwrap()).unwrap(); - - // Test 1: Wait for state change (should succeed) - let wait_result = manager - .wait_for_change(&task_id, Duration::from_secs(2)) - .await; - assert!(wait_result.is_ok(), "Wait for change should succeed"); - - let task_after_change = wait_result.unwrap(); - assert!( - task_after_change.state == DownloadState::Downloading - || task_after_change.state == DownloadState::Completed, - "State should have changed from Pending" - ); - - // Test 2: Short timeout test (should timeout when state is stable) - // Wait for task to complete or enter stable state - tokio::time::sleep(Duration::from_secs(1)).await; - - // Test 3: Verify task eventually completes - for _ in 0..15 { - tokio::time::sleep(Duration::from_millis(300)).await; - let task_info = manager.get_task(&task_id).unwrap(); - - if task_info.state == DownloadState::Completed { - // Verify download success - assert!(dest.exists(), "File should exist"); - let file_size = std::fs::metadata(&dest).unwrap().len(); - assert_eq!(file_size, 2048, "File size should match"); - break; - } - } - - mock.assert(); - } - - /// Test Scenario 8: Batch task submission and management - /// Verifies: Batch submission, deduplication, concurrency control - #[tokio::test] - async fn test_batch_task_submission_and_management() { - use mockito::Server; - use tempfile::tempdir; - - // Create Mock HTTP server - let mut server = Server::new_async().await; - - // Create 3 different files - let files = vec![ - ("batch1.txt", b"Batch file 1".to_vec()), - ("batch2.txt", b"Batch file 2".to_vec()), - ("batch3.txt", b"Batch file 3".to_vec()), - ]; - - let mut mocks = Vec::new(); - for (filename, content) in &files { - let mock = server - .mock("GET", format!("/{}", filename).as_str()) - .with_status(200) - .with_header("content-length", &content.len().to_string()) - .with_body(content.as_slice()) - .create(); - mocks.push(mock); - } - - // Create task manager - let config = DownloadConfig::default(); - let manager = DownloadTaskManager::from_config(&config); - - // Prepare batch tasks - let temp_dir = tempdir().unwrap(); - let mut tasks = Vec::new(); - - for (filename, _) in &files { - let url = format!("{}/{}", server.url(), filename); - let dest = temp_dir.path().join(filename); - tasks.push((url, dest.to_string_lossy().to_string())); - } - - // Submit batch tasks - let task_ids = manager.submit_batch(tasks).unwrap(); - assert_eq!(task_ids.len(), 3, "Should submit 3 tasks"); - - // Wait for all tasks to complete - for task_id in &task_ids { - for _ in 0..30 { - tokio::time::sleep(Duration::from_millis(100)).await; - - if let Ok(task_info) = manager.get_task(task_id) { - if task_info.state == DownloadState::Completed { - break; - } - if task_info.state == DownloadState::Failed { - panic!("Task {} failed: {:?}", task_id, task_info.error); - } - } - } - } - - // Verify all files downloaded successfully - for (filename, expected_content) in &files { - let dest = temp_dir.path().join(filename); - assert!(dest.exists(), "File {} should exist", filename); - - let content = std::fs::read(&dest).unwrap(); - assert_eq!( - content, *expected_content, - "Content of {} should match", - filename - ); - } - - // Verify all mocks were called - for mock in mocks { - mock.assert(); - } - - // Verify get_all_tasks returns all tasks - let all_tasks = manager.get_all_tasks(); - assert!(all_tasks.len() >= 3, "Should have at least 3 tasks"); - - // Verify get_tasks_by_state - let completed_tasks = manager.get_tasks_by_state(DownloadState::Completed); - assert!( - completed_tasks.len() >= 3, - "Should have at least 3 completed tasks" - ); - } -} diff --git a/src/downloader/traits.rs b/src/downloader/traits.rs deleted file mode 100644 index ba8cdde..0000000 --- a/src/downloader/traits.rs +++ /dev/null @@ -1,262 +0,0 @@ -//! Core trait definitions for the downloader system - -use super::error::Result; -use async_trait::async_trait; -use serde::{Deserialize, Serialize}; -use std::collections::HashMap; -use std::path::Path; - -/// Progress callback function type -/// Parameters: (downloaded_bytes, total_bytes_optional) -pub type ProgressCallback = Box) + Send + Sync>; - -/// HTTP request options for downloads -#[derive(Debug, Clone, Default)] -pub struct RequestOptions { - /// HTTP headers to include in the request - pub headers: Option>, - /// HTTP cookies to include in the request - pub cookies: Option>, - /// Extra metadata for dispatch (e.g., hub_uuid for routing to external downloaders) - pub metadata: Option>, -} - -/// Downloader capability information -/// -/// This struct defines what features a downloader implementation supports. -/// These capabilities are determined at initialization time and remain constant -/// throughout the downloader's lifetime. -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, Default)] -pub struct DownloaderCapabilities { - /// Whether the downloader supports pausing downloads - pub supports_pause: bool, - /// Whether the downloader supports resuming paused downloads - pub supports_resume: bool, - /// Whether the downloader supports cancelling downloads - pub supports_cancellation: bool, - /// Whether the downloader supports HTTP Range requests for breakpoint resume - pub supports_range_requests: bool, - /// Whether the downloader supports batch download operations - pub supports_batch_download: bool, -} - -impl DownloaderCapabilities { - /// Create a new DownloaderCapabilities with all features enabled - pub fn all_enabled() -> Self { - Self { - supports_pause: true, - supports_resume: true, - supports_cancellation: true, - supports_range_requests: true, - supports_batch_download: true, - } - } - - /// Create a new DownloaderCapabilities with all features disabled - pub fn all_disabled() -> Self { - Self::default() - } -} - -/// Core downloader trait - all downloader implementations must implement this -/// -/// This trait provides a pluggable interface for different download backends, -/// allowing easy switching between implementations (trauma, reqwest, CLI tools, etc.) -#[async_trait] -pub trait Downloader: Send + Sync { - /// Download a single file from URL to destination path - /// - /// # Arguments - /// * `url` - The URL to download from - /// * `dest` - The destination file path - /// * `progress` - Optional progress callback (downloaded_bytes, total_bytes) - /// * `options` - Optional request options (headers, cookies) - /// - /// # Returns - /// * `Ok(())` - Download completed successfully - /// * `Err(DownloadError)` - Download failed - async fn download( - &self, - url: &str, - dest: &Path, - progress: Option, - options: Option, - ) -> Result<()>; - - /// Download multiple files concurrently - /// - /// # Arguments - /// * `tasks` - Vector of (url, destination_path) tuples - /// - /// # Returns - /// * `Ok(Vec>)` - Vector of results for each download task - async fn download_batch(&self, tasks: Vec<(String, std::path::PathBuf)>) -> Vec>; - - /// Get the name of this downloader implementation - fn name(&self) -> &str; - - /// Get the capabilities of this downloader implementation - /// - /// This method returns a reference to the capability information that was - /// determined at initialization time. Capabilities define what features - /// the downloader supports (pause, resume, cancellation, etc.). - /// - /// # Returns - /// A reference to the DownloaderCapabilities struct - fn capabilities(&self) -> &DownloaderCapabilities; - - /// Cancel an ongoing download (if supported by the implementation) - /// - /// Default implementation does nothing and returns Ok - async fn cancel(&self, _url: &str) -> Result<()> { - Ok(()) - } - - /// Pause an ongoing download (if supported by the implementation) - /// - /// # Arguments - /// * `url` - The URL of the download to pause - /// - /// # Returns - /// * `Ok(())` - Download paused successfully - /// * `Err(DownloadError)` - Failed to pause download - /// - /// Default implementation returns an error - async fn pause(&self, _url: &str) -> Result<()> { - Err(super::error::DownloadError::unsupported( - "Pause not supported by this downloader", - )) - } - - /// Resume a paused download (if supported by the implementation) - /// - /// # Arguments - /// * `url` - The URL of the download to resume - /// - /// # Returns - /// * `Ok(())` - Download resumed successfully - /// * `Err(DownloadError)` - Failed to resume download - /// - /// Default implementation returns an error - async fn resume(&self, _url: &str) -> Result<()> { - Err(super::error::DownloadError::unsupported( - "Resume not supported by this downloader", - )) - } - - /// Check if this downloader supports cancellation - /// - /// # Deprecated - /// Use `capabilities().supports_cancellation` instead. - /// This method will be removed in a future version. - #[deprecated( - since = "0.2.0", - note = "Use capabilities().supports_cancellation instead" - )] - fn supports_cancellation(&self) -> bool { - self.capabilities().supports_cancellation - } - - /// Check if this downloader supports pause/resume - /// - /// # Deprecated - /// Use `capabilities().supports_pause` instead. - /// This method will be removed in a future version. - #[deprecated(since = "0.2.0", note = "Use capabilities().supports_pause instead")] - fn supports_pause(&self) -> bool { - self.capabilities().supports_pause - } - - /// Check if this downloader supports resume/partial downloads - /// - /// # Deprecated - /// Use `capabilities().supports_resume` instead. - /// This method will be removed in a future version. - #[deprecated(since = "0.2.0", note = "Use capabilities().supports_resume instead")] - fn supports_resume(&self) -> bool { - self.capabilities().supports_resume - } -} - -#[cfg(test)] -mod tests { - use super::*; - use std::path::PathBuf; - - // Mock downloader for testing - struct MockDownloader { - capabilities: DownloaderCapabilities, - } - - impl MockDownloader { - fn new() -> Self { - Self { - capabilities: DownloaderCapabilities::all_disabled(), - } - } - } - - #[async_trait] - impl Downloader for MockDownloader { - async fn download( - &self, - _url: &str, - _dest: &Path, - _progress: Option, - _options: Option, - ) -> Result<()> { - Ok(()) - } - - async fn download_batch(&self, tasks: Vec<(String, PathBuf)>) -> Vec> { - tasks.into_iter().map(|_| Ok(())).collect() - } - - fn name(&self) -> &str { - "mock" - } - - fn capabilities(&self) -> &DownloaderCapabilities { - &self.capabilities - } - } - - #[tokio::test] - async fn test_mock_downloader() { - let downloader = MockDownloader::new(); - assert_eq!(downloader.name(), "mock"); - - // Test new capabilities() method - let caps = downloader.capabilities(); - assert!(!caps.supports_cancellation); - assert!(!caps.supports_pause); - assert!(!caps.supports_resume); - assert!(!caps.supports_range_requests); - assert!(!caps.supports_batch_download); - - // Test deprecated methods (should still work) - #[allow(deprecated)] - { - assert!(!downloader.supports_cancellation()); - assert!(!downloader.supports_pause()); - assert!(!downloader.supports_resume()); - } - - let result = downloader - .download( - "http://example.com/file", - Path::new("/tmp/file"), - None, - None, - ) - .await; - assert!(result.is_ok()); - - // Test default pause/resume implementations - let pause_result = downloader.pause("http://example.com/file").await; - assert!(pause_result.is_err()); - - let resume_result = downloader.resume("http://example.com/file").await; - assert!(resume_result.is_err()); - } -} diff --git a/src/downloader/trauma_impl.rs b/src/downloader/trauma_impl.rs deleted file mode 100644 index c350617..0000000 --- a/src/downloader/trauma_impl.rs +++ /dev/null @@ -1,751 +0,0 @@ -//! Reqwest-based downloader implementation with pause/resume support - -use super::error::{DownloadError, Result}; -use super::traits::{Downloader, DownloaderCapabilities, ProgressCallback, RequestOptions}; -use async_trait::async_trait; -use futures::StreamExt; -use parking_lot::RwLock; -use std::collections::HashMap; -use std::path::{Path, PathBuf}; -use std::sync::Arc; -use tokio::fs::{File, OpenOptions}; -use tokio::io::AsyncWriteExt; -use tokio_util::sync::CancellationToken; - -/// Task state for managing pause/resume -#[derive(Clone)] -struct TaskState { - url: String, - dest: PathBuf, - cancel_token: CancellationToken, - supports_range: Option, -} - -/// Reqwest-based downloader implementation -/// -/// This implementation uses reqwest with rustls for secure, concurrent downloads -/// and supports pause/resume with HTTP Range requests -pub struct TraumaDownloader { - max_concurrent: usize, - retries: usize, - timeout_seconds: u64, - client: Arc, - // Track active download tasks for pause/resume - tasks: Arc>>, - // Capabilities of this downloader - capabilities: DownloaderCapabilities, -} - -impl TraumaDownloader { - /// Create a new TraumaDownloader with specified parameters - /// - /// # Arguments - /// * `max_concurrent` - Maximum number of concurrent downloads - /// * `retries` - Number of retry attempts for failed downloads - /// * `timeout_seconds` - Timeout for each download in seconds - pub fn new(max_concurrent: usize, retries: usize, timeout_seconds: u64) -> Self { - let client = reqwest::Client::builder() - .timeout(std::time::Duration::from_secs(timeout_seconds)) - .build() - .expect("Failed to create HTTP client"); - - Self { - max_concurrent, - retries, - timeout_seconds, - client: Arc::new(client), - tasks: Arc::new(RwLock::new(HashMap::new())), - capabilities: DownloaderCapabilities::all_enabled(), - } - } - - /// Create a downloader with default settings - pub fn default_settings() -> Self { - Self::new(4, 3, 300) - } - - /// Test if a server supports HTTP Range requests - async fn test_range_support(client: &reqwest::Client, url: &str) -> bool { - // Try HEAD request first - if let Ok(response) = client.head(url).send().await { - if let Some(accept_ranges) = response.headers().get("accept-ranges") { - if let Ok(value) = accept_ranges.to_str() { - return value.to_lowercase() == "bytes"; - } - } - } - - // Fallback: try a fake range request - if let Ok(response) = client.get(url).header("Range", "bytes=0-0").send().await { - return response.status() == reqwest::StatusCode::PARTIAL_CONTENT - || response.status() == reqwest::StatusCode::OK; - } - - false - } - - /// Download a file with retry and pause support - async fn download_with_retry( - &self, - url: &str, - dest: &Path, - progress: Option<&ProgressCallback>, - cancel_token: &CancellationToken, - options: Option<&RequestOptions>, - ) -> Result<()> { - let mut last_error = None; - - for attempt in 0..=self.retries { - if cancel_token.is_cancelled() { - return Err(DownloadError::cancelled("Download was paused")); - } - - // For retries, we don't pass progress callback to avoid multiple progress reports - let current_progress = if attempt == 0 { progress } else { None }; - - match self - .download_once(url, dest, current_progress, cancel_token, options) - .await - { - Ok(()) => return Ok(()), - Err(e) => { - if cancel_token.is_cancelled() { - return Err(DownloadError::cancelled("Download was paused")); - } - last_error = Some(e); - if attempt < self.retries { - tokio::time::sleep(std::time::Duration::from_secs(1 << attempt)).await; - } - } - } - } - - Err(last_error.unwrap_or_else(|| DownloadError::network("Download failed".to_string()))) - } - - /// Download a file once (no retry), with pause support - async fn download_once( - &self, - url: &str, - dest: &Path, - progress: Option<&ProgressCallback>, - cancel_token: &CancellationToken, - options: Option<&RequestOptions>, - ) -> Result<()> { - // Check if temp file exists (resume case) - let temp_dest = dest.with_extension("tmp"); - let existing_size = if temp_dest.exists() { - tokio::fs::metadata(&temp_dest) - .await - .ok() - .map(|m| m.len()) - .unwrap_or(0) - } else { - 0 - }; - - // Test range support if not already known - let supports_range = if existing_size > 0 { - Self::test_range_support(&self.client, url).await - } else { - false - }; - - // Build request with Range header if resuming - let mut request = self.client.get(url); - if existing_size > 0 && supports_range { - request = request.header("Range", format!("bytes={}-", existing_size)); - } - - // Apply custom headers if provided - if let Some(opts) = options { - if let Some(headers) = &opts.headers { - for (key, value) in headers { - request = request.header(key, value); - } - } - // Apply cookies if provided - if let Some(cookies) = &opts.cookies { - let cookie_string = cookies - .iter() - .map(|(k, v)| format!("{}={}", k, v)) - .collect::>() - .join("; "); - if !cookie_string.is_empty() { - request = request.header("Cookie", cookie_string); - } - } - } - - // Send request - let response = request - .send() - .await - .map_err(|e| DownloadError::network(format!("Failed to send request: {}", e)))?; - - // Check status - let status = response.status(); - if !status.is_success() && status != reqwest::StatusCode::PARTIAL_CONTENT { - return Err(DownloadError::network(format!( - "HTTP error {}: {}", - status, url - ))); - } - - // Get content length - let content_length = response.content_length(); - let total_size = if status == reqwest::StatusCode::PARTIAL_CONTENT { - // For partial content, add existing size to content length - content_length.map(|len| len + existing_size) - } else { - content_length - }; - - // Open file (append mode if resuming, create mode otherwise) - let mut file = if existing_size > 0 && status == reqwest::StatusCode::PARTIAL_CONTENT { - OpenOptions::new() - .append(true) - .open(&temp_dest) - .await - .map_err(|e| DownloadError::file_system(format!("Failed to open file: {}", e)))? - } else { - File::create(&temp_dest) - .await - .map_err(|e| DownloadError::file_system(format!("Failed to create file: {}", e)))? - }; - - // Stream download with progress tracking - let mut stream = response.bytes_stream(); - let mut downloaded: u64 = existing_size; - - while let Some(chunk_result) = stream.next().await { - // Check for pause signal - if cancel_token.is_cancelled() { - // Flush and close file before pausing - file.flush().await.ok(); - drop(file); - return Err(DownloadError::cancelled("Download paused")); - } - - let chunk = chunk_result - .map_err(|e| DownloadError::network(format!("Failed to read chunk: {}", e)))?; - - file.write_all(&chunk) - .await - .map_err(|e| DownloadError::file_system(format!("Failed to write chunk: {}", e)))?; - - downloaded += chunk.len() as u64; - - // Call progress callback - if let Some(callback) = progress { - callback(downloaded, total_size); - } - } - - // Flush and close file - file.flush() - .await - .map_err(|e| DownloadError::file_system(format!("Failed to flush file: {}", e)))?; - drop(file); - - // Rename temp file to final destination - tokio::fs::rename(&temp_dest, dest).await.map_err(|e| { - DownloadError::file_system(format!("Failed to rename downloaded file: {}", e)) - })?; - - Ok(()) - } - - /// Register a download task - fn register_task(&self, url: String, dest: PathBuf) -> CancellationToken { - let cancel_token = CancellationToken::new(); - let state = TaskState { - url: url.clone(), - dest, - cancel_token: cancel_token.clone(), - supports_range: None, - }; - self.tasks.write().insert(url, state); - cancel_token - } - - /// Unregister a download task - fn unregister_task(&self, url: &str) { - self.tasks.write().remove(url); - } - - /// Get task cancel token - fn get_cancel_token(&self, url: &str) -> Option { - self.tasks - .read() - .get(url) - .map(|state| state.cancel_token.clone()) - } -} - -#[async_trait] -impl Downloader for TraumaDownloader { - async fn download( - &self, - url: &str, - dest: &Path, - progress: Option, - options: Option, - ) -> Result<()> { - // Validate inputs - if url.is_empty() { - return Err(DownloadError::invalid_input("URL cannot be empty")); - } - - // Ensure parent directory exists - if let Some(parent) = dest.parent() { - tokio::fs::create_dir_all(parent).await.map_err(|e| { - DownloadError::file_system(format!("Failed to create directory: {}", e)) - })?; - } - - // Register task - let cancel_token = self.register_task(url.to_string(), dest.to_path_buf()); - - // Start download - let result = self - .download_with_retry( - url, - dest, - progress.as_ref(), - &cancel_token, - options.as_ref(), - ) - .await; - - // Unregister task - self.unregister_task(url); - - result - } - - async fn download_batch(&self, tasks: Vec<(String, PathBuf)>) -> Vec> { - if tasks.is_empty() { - return vec![]; - } - - // Use semaphore to limit concurrent downloads - let semaphore = Arc::new(tokio::sync::Semaphore::new(self.max_concurrent)); - let mut handles = vec![]; - - for (url, dest) in tasks { - let sem = semaphore.clone(); - let cancel_token = self.register_task(url.clone(), dest.clone()); - let downloader = self.clone_for_task(); - let url_for_handle = url.clone(); - - let handle = tokio::spawn(async move { - let _permit = sem.acquire().await.unwrap(); - - // Ensure parent directory exists - if let Some(parent) = dest.parent() { - if let Err(e) = tokio::fs::create_dir_all(parent).await { - return Err(DownloadError::file_system(format!( - "Failed to create directory: {}", - e - ))); - } - } - - // Download with retry - let result = downloader - .download_with_retry(&url, &dest, None, &cancel_token, None) - .await; - - result - }); - - handles.push((url_for_handle, handle)); - } - - // Wait for all downloads to complete - let mut results = vec![]; - for (url, handle) in handles { - match handle.await { - Ok(result) => { - self.unregister_task(&url); - results.push(result); - } - Err(e) => { - self.unregister_task(&url); - results.push(Err(DownloadError::network(format!( - "Task join error: {}", - e - )))); - } - } - } - - results - } - - fn name(&self) -> &str { - "reqwest" - } - - fn capabilities(&self) -> &DownloaderCapabilities { - &self.capabilities - } - - async fn pause(&self, url: &str) -> Result<()> { - if let Some(cancel_token) = self.get_cancel_token(url) { - cancel_token.cancel(); - Ok(()) - } else { - Err(DownloadError::task_not_found(url)) - } - } - - async fn resume(&self, url: &str) -> Result<()> { - // Get task state - let state = self.tasks.read().get(url).cloned(); - - if let Some(task_state) = state { - // Create new cancel token for resumed download - let new_cancel_token = - self.register_task(task_state.url.clone(), task_state.dest.clone()); - - // Start download from where it left off - let result = self - .download_with_retry( - &task_state.url, - &task_state.dest, - None, - &new_cancel_token, - None, - ) - .await; - - if result.is_err() { - self.unregister_task(url); - } - - result - } else { - Err(DownloadError::task_not_found(url)) - } - } -} - -impl TraumaDownloader { - /// Clone essential fields for spawned tasks - fn clone_for_task(&self) -> Self { - Self { - max_concurrent: self.max_concurrent, - retries: self.retries, - timeout_seconds: self.timeout_seconds, - client: self.client.clone(), - tasks: self.tasks.clone(), - capabilities: self.capabilities.clone(), - } - } -} - -#[cfg(test)] -mod tests { - use super::*; - use tempfile::tempdir; - - #[test] - fn test_create_trauma_downloader() { - let downloader = TraumaDownloader::new(8, 5, 600); - assert_eq!(downloader.name(), "reqwest"); - assert_eq!(downloader.max_concurrent, 8); - assert_eq!(downloader.retries, 5); - - // Test capabilities - let caps = downloader.capabilities(); - assert!(caps.supports_pause); - assert!(caps.supports_resume); - assert!(caps.supports_cancellation); - } - - #[tokio::test] - async fn test_download_invalid_url() { - let downloader = TraumaDownloader::default_settings(); - let temp_dir = tempdir().unwrap(); - let dest = temp_dir.path().join("test.txt"); - - let result = downloader.download("", &dest, None, None).await; - assert!(result.is_err()); - - if let Err(e) = result { - assert_eq!(e.kind, super::super::error::ErrorKind::InvalidInput); - } - } - - #[tokio::test] - async fn test_download_batch_empty() { - let downloader = TraumaDownloader::default_settings(); - let results = downloader.download_batch(vec![]).await; - assert!(results.is_empty()); - } - - #[tokio::test] - async fn test_pause_nonexistent_task() { - let downloader = TraumaDownloader::default_settings(); - let result = downloader.pause("http://nonexistent.com/file").await; - assert!(result.is_err()); - } - - // ======================================================================== - // Integration Tests with Mock HTTP Server - // ======================================================================== - - /// Test Scenario 1: Complete file download flow - /// Verifies: File downloaded correctly, content matches, temp file cleanup - #[tokio::test] - async fn test_download_complete_file() { - use mockito::Server; - - // Create Mock HTTP server - let mut server = Server::new_async().await; - let test_data = b"Hello, this is test data for download!"; - - let mock = server - .mock("GET", "/test-file.txt") - .with_status(200) - .with_header("content-length", &test_data.len().to_string()) - .with_body(test_data.as_slice()) - .create(); - - // Prepare download destination - let temp_dir = tempdir().unwrap(); - let dest = temp_dir.path().join("downloaded.txt"); - - // Execute download - let downloader = TraumaDownloader::default_settings(); - let url = format!("{}/test-file.txt", server.url()); - let result = downloader.download(&url, &dest, None, None).await; - - // Verify results - assert!(result.is_ok(), "Download should succeed"); - assert!(dest.exists(), "Downloaded file should exist"); - - let downloaded_content = std::fs::read(&dest).unwrap(); - assert_eq!(downloaded_content, test_data, "Content should match"); - - // Verify temporary file was cleaned up - let temp_file = dest.with_extension("tmp"); - assert!(!temp_file.exists(), "Temporary file should be cleaned up"); - - mock.assert(); - } - - /// Test Scenario 2: Concurrent downloads of multiple files - /// Verifies: Parallel execution, concurrency control, all files downloaded - #[tokio::test] - async fn test_concurrent_downloads() { - use mockito::Server; - - // Create Mock HTTP server - let mut server = Server::new_async().await; - - // Prepare 5 different test files - let test_files = vec![ - ("file1.txt", b"Content of file 1".to_vec()), - ("file2.txt", b"Content of file 2".to_vec()), - ("file3.txt", b"Content of file 3".to_vec()), - ("file4.txt", b"Content of file 4".to_vec()), - ("file5.txt", b"Content of file 5".to_vec()), - ]; - - // Create mock for each file - let mut mocks = Vec::new(); - for (filename, content) in &test_files { - let mock = server - .mock("GET", format!("/{}", filename).as_str()) - .with_status(200) - .with_header("content-length", &content.len().to_string()) - .with_body(content.as_slice()) - .create(); - mocks.push(mock); - } - - // Prepare download tasks - let temp_dir = tempdir().unwrap(); - let mut tasks = Vec::new(); - - for (filename, _) in &test_files { - let url = format!("{}/{}", server.url(), filename); - let dest = temp_dir.path().join(filename); - tasks.push((url, dest)); - } - - // Execute concurrent downloads (max 3 concurrent) - let downloader = TraumaDownloader::new(3, 2, 30); - let results = downloader.download_batch(tasks.clone()).await; - - // Verify all downloads succeeded - assert_eq!(results.len(), 5, "Should have 5 results"); - for (i, result) in results.iter().enumerate() { - assert!(result.is_ok(), "Download {} should succeed", i); - } - - // Verify file contents - for (i, (filename, expected_content)) in test_files.iter().enumerate() { - let dest = temp_dir.path().join(filename); - assert!(dest.exists(), "File {} should exist", filename); - - let content = std::fs::read(&dest).unwrap(); - assert_eq!( - content, *expected_content, - "Content of file {} should match", - i - ); - } - - // Verify all mocks were called - for mock in mocks { - mock.assert(); - } - } - - /// Test Scenario 3: Pause and resume during download - /// Verifies: Pause mechanism, temp file retention, successful continuation - #[tokio::test] - async fn test_pause_and_resume_download() { - use mockito::Server; - use std::time::Duration; - - // Create Mock HTTP server - simulate large file with chunked response - let mut server = Server::new_async().await; - let test_data = vec![b'X'; 10 * 1024]; // 10KB data - - let mock = server - .mock("GET", "/large-file.bin") - .with_status(200) - .with_header("content-length", &test_data.len().to_string()) - .with_chunked_body(move |w| { - // Simulate slow download to allow time for pause - for chunk in test_data.chunks(1024) { - std::thread::sleep(Duration::from_millis(50)); - w.write_all(chunk)?; - } - Ok(()) - }) - .create(); - - let temp_dir = tempdir().unwrap(); - let dest = temp_dir.path().join("large-file.bin"); - let url = format!("{}/large-file.bin", server.url()); - - let downloader = TraumaDownloader::default_settings(); - - // Start download and pause quickly - let url_clone = url.clone(); - let dest_clone = dest.clone(); - let downloader_clone = downloader.clone_for_task(); - - let download_handle = tokio::spawn(async move { - downloader_clone - .download(&url_clone, &dest_clone, None, None) - .await - }); - - // Wait for download to start - tokio::time::sleep(Duration::from_millis(100)).await; - - // Pause download - let pause_result = downloader.pause(&url).await; - assert!(pause_result.is_ok(), "Pause should succeed"); - - // Wait for download task to respond to pause signal - let download_result = download_handle.await.unwrap(); - assert!(download_result.is_err(), "Download should be cancelled"); - - // Verify temporary file exists (partially downloaded data) - let temp_file = dest.with_extension("tmp"); - assert!( - temp_file.exists(), - "Temporary file should exist after pause" - ); - - mock.assert(); - } - - /// Test Scenario 4: Resume from breakpoint - /// Verifies: Range request, resume from breakpoint, complete file download - #[tokio::test] - async fn test_resume_from_breakpoint() { - use mockito::Server; - - let temp_dir = tempdir().unwrap(); - let dest = temp_dir.path().join("resume-test.bin"); - let temp_file = dest.with_extension("tmp"); - - // Prepare complete test data - let full_data = vec![b'A'; 5000]; // 5KB - let partial_size = 2000; // First 2KB already downloaded - - // Simulate partially downloaded data - std::fs::write(&temp_file, &full_data[..partial_size]).unwrap(); - assert_eq!( - std::fs::metadata(&temp_file).unwrap().len(), - partial_size as u64, - "Partial file should exist" - ); - - // Create Mock HTTP server with Range support - let mut server = Server::new_async().await; - - // First Mock: HEAD request to detect Range support - let _head_mock = server - .mock("HEAD", "/resume-file.bin") - .with_status(200) - .with_header("accept-ranges", "bytes") - .with_header("content-length", &full_data.len().to_string()) - .create(); - - // Second Mock: GET request with Range support - let range_mock = server - .mock("GET", "/resume-file.bin") - .match_header("range", format!("bytes={}-", partial_size).as_str()) - .with_status(206) // Partial Content - .with_header( - "content-range", - format!( - "bytes {}-{}/{}", - partial_size, - full_data.len() - 1, - full_data.len() - ) - .as_str(), - ) - .with_header( - "content-length", - &(full_data.len() - partial_size).to_string(), - ) - .with_body(&full_data[partial_size..]) - .create(); - - // Execute resume download - let downloader = TraumaDownloader::default_settings(); - let url = format!("{}/resume-file.bin", server.url()); - let result = downloader.download(&url, &dest, None, None).await; - - // Verify download succeeded - assert!(result.is_ok(), "Resume download should succeed"); - assert!(dest.exists(), "Final file should exist"); - assert!(!temp_file.exists(), "Temporary file should be removed"); - - // Verify complete file content - let downloaded_data = std::fs::read(&dest).unwrap(); - assert_eq!( - downloaded_data.len(), - full_data.len(), - "File size should match" - ); - assert_eq!( - downloaded_data, full_data, - "Content should match completely" - ); - - range_mock.assert(); - } -} diff --git a/src/error.rs b/src/error.rs deleted file mode 100644 index 313146e..0000000 --- a/src/error.rs +++ /dev/null @@ -1,82 +0,0 @@ -use std::result; - -#[derive(Debug)] -pub enum Error { - Io(std::io::Error), - Other(String), - Getter(GetterError), -} - -impl std::fmt::Display for Error { - fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { - match self { - Error::Io(e) => write!(f, "IO error: {e}"), - Error::Other(msg) => write!(f, "{msg}"), - Error::Getter(e) => write!(f, "{e}"), - } - } -} - -impl std::error::Error for Error { - fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { - match self { - Error::Io(e) => Some(e), - Error::Getter(e) => e.source(), - Error::Other(_) => None, - } - } -} - -impl From for Error { - fn from(e: std::io::Error) -> Self { - Error::Io(e) - } -} - -impl From for Error { - fn from(e: GetterError) -> Self { - Error::Getter(e) - } -} - -pub type Result = result::Result; - -#[derive(Debug)] -pub struct GetterError { - pub tag: String, - pub message: String, - pub err: Option>, -} - -impl GetterError { - pub fn new_nobase(tag: &str, message: &str) -> Self { - Self { - tag: tag.to_string(), - message: message.to_string(), - err: None, - } - } - - pub fn new(tag: &str, message: &str, err: Box) -> Self { - Self { - tag: tag.to_string(), - message: message.to_string(), - err: Some(err), - } - } -} - -impl std::fmt::Display for GetterError { - fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { - write!(f, "{}: {}", self.tag, self.message) - } -} - -impl std::error::Error for GetterError { - fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { - match &self.err { - Some(err) => Some(err.as_ref()), - None => None, - } - } -} diff --git a/src/lib.rs b/src/lib.rs index 4c1491e..4ffc0b5 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,14 +1,64 @@ -mod cache; -mod core; -pub mod database; -pub mod downloader; -mod error; -mod locale; -pub mod manager; -pub mod rpc; -mod utils; -mod websdk; - -// rustls-platform-verifier -#[cfg(feature = "rustls-platform-verifier-android")] +//! Root getter library facade for the UpgradeAll rewrite. +//! +//! The root crate is intentionally small during the rewrite. Product/domain +//! behavior lives in split crates such as `getter-core` and `getter-storage`; +//! Android/Flutter hosts embed this crate through a stable facade. + +pub use getter_core as core; +pub use getter_storage as storage; + +#[cfg(feature = "rustls-platform-verifier")] pub use rustls_platform_verifier; + +pub mod rpc { + pub mod server { + use std::future::pending; + + /// Error returned by the temporary in-process RPC server facade. + #[derive(Debug, thiserror::Error)] + pub enum RpcServerError { + #[error("failed to bind RPC server at {address}: {source}")] + Bind { + address: String, + #[source] + source: std::io::Error, + }, + #[error("failed to read local RPC server address: {0}")] + LocalAddress(#[source] std::io::Error), + #[error("RPC server startup callback failed")] + StartupCallback, + } + + /// Start a placeholder local RPC endpoint and keep it alive. + /// + /// This preserves the Android `api_proxy` dependency target while the + /// full getter RPC surface is rewritten. The function binds a real local + /// TCP listener so callers receive a concrete URL, then parks forever. + pub async fn run_server_hanging( + address: &str, + on_listening: F, + ) -> Result<(), RpcServerError> + where + F: FnOnce(&str) -> Result<(), RpcServerError> + Send + 'static, + { + let listener = tokio::net::TcpListener::bind(address) + .await + .map_err(|source| RpcServerError::Bind { + address: address.to_owned(), + source, + })?; + let local_addr = listener + .local_addr() + .map_err(RpcServerError::LocalAddress)?; + let url = format!("ws://{local_addr}"); + on_listening(&url)?; + + // Keep the listener alive until the host process stops. The full + // JSON-RPC implementation is added in a later behavior slice. + let _listener = listener; + pending::<()>().await; + #[allow(unreachable_code)] + Ok(()) + } + } +} diff --git a/src/locale.rs b/src/locale.rs deleted file mode 100644 index 82b257f..0000000 --- a/src/locale.rs +++ /dev/null @@ -1,64 +0,0 @@ -use std::env; -use std::io::{self, ErrorKind}; -use std::path::PathBuf; - -pub struct DataDir { - pub cache_dir: PathBuf, - pub data_dir: PathBuf, -} - -#[cfg(all( - target_family = "unix", - not(target_os = "macos"), - not(target_os = "android") -))] -pub fn all_dir() -> Result { - let home_dir = env::var("HOME") - .map_err(|_| io::Error::new(ErrorKind::NotFound, "HOME not found")) - .map(PathBuf::from)?; - let cache_dir = home_dir.join(".cache/upa/"); - let data_dir = home_dir.join(".local/share/upa/"); - Ok(DataDir { - cache_dir, - data_dir, - }) -} - -#[cfg(target_os = "macos")] -pub fn all_dir() -> Result { - let home_dir = env::var("HOME") - .map_err(|_| io::Error::new(ErrorKind::NotFound, "HOME not found")) - .map(|home| PathBuf::from(home))?; - let cache_dir = home_dir.join("Library/Caches/upa/"); - let data_dir = home_dir.join("Library/Application Support/upa/"); - Ok(DataDir { - cache_dir, - data_dir, - }) -} - -#[cfg(target_family = "windows")] -pub fn all_dir() -> Result { - let home_dir = env::var("APPDATA") - .map_err(|_| io::Error::new(ErrorKind::NotFound, "APPDATA not found")) - .map(|home| PathBuf::from(home))?; - let cache_dir = home_dir.join("upa/cache/"); - let data_dir = home_dir.join("upa/data/"); - Ok(DataDir { - cache_dir, - data_dir, - }) -} - -#[cfg(target_os = "android")] -pub fn all_dir() -> Result { - let home_dir = env::var("HOME") - .map_err(|_| io::Error::new(ErrorKind::NotFound, "HOME not found")) - .map(|home| PathBuf::from(home))?; - let cache_dir = home_dir.join(".upa/cache/"); - let data_dir = home_dir.join(".upa/data/"); - Ok(DataDir { - cache_dir, - data_dir, - }) -} diff --git a/src/main.rs b/src/main.rs index e7a11a9..c9bfeea 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,3 +1,17 @@ +#[cfg(feature = "cli")] fn main() { - println!("Hello, world!"); + let output = getter_cli::run(std::env::args()); + if !output.stdout.is_empty() { + print!("{}", output.stdout); + } + if !output.stderr.is_empty() { + eprint!("{}", output.stderr); + } + std::process::exit(output.exit_code.code()); +} + +#[cfg(not(feature = "cli"))] +fn main() { + eprintln!("getter binary requires the 'cli' feature"); + std::process::exit(2); } diff --git a/src/manager/android_api.rs b/src/manager/android_api.rs deleted file mode 100644 index 5e39948..0000000 --- a/src/manager/android_api.rs +++ /dev/null @@ -1,104 +0,0 @@ -use std::collections::HashMap; - -use jsonrpsee::core::client::ClientT; -use jsonrpsee::http_client::HttpClientBuilder; -use jsonrpsee::rpc_params; -use once_cell::sync::OnceCell; -use serde::{Deserialize, Serialize}; - -use crate::database::models::app::AppRecord; - -/// Global Android API client, registered via `register_android_api` RPC. -static ANDROID_API: OnceCell = OnceCell::new(); - -pub fn set_android_api(url: String) { - let _ = ANDROID_API.set(AndroidApi { url }); -} - -pub fn get_android_api() -> Option<&'static AndroidApi> { - ANDROID_API.get() -} - -/// Rust-side HTTP JSON-RPC client that calls back into the Kotlin -/// KotlinHubRpcServer for Android-specific functionality. -/// -/// Kotlin exposes `get_local_version` and `get_installed_apps` methods -/// on the same Ktor HTTP server that handles hub provider calls. -pub struct AndroidApi { - url: String, -} - -#[derive(Serialize, Deserialize, Debug)] -struct GetLocalVersionParams { - app_id: HashMap>, -} - -#[derive(Serialize, Deserialize, Debug)] -struct GetInstalledAppsParams { - ignore_system: bool, -} - -impl AndroidApi { - /// Query Kotlin for the locally-installed version of an app. - /// - /// Calls `get_local_version({app_id})` on the Kotlin Ktor server. - /// Returns `None` if the app is not installed or the call fails. - pub async fn get_local_version( - &self, - app_id: &HashMap>, - ) -> Option { - let client = HttpClientBuilder::default().build(&self.url).ok()?; - let params = rpc_params!(GetLocalVersionParams { - app_id: app_id.clone(), - }); - client - .request::, _>("get_local_version", params) - .await - .unwrap_or(None) - } - - /// Query Kotlin for all installed Android apps and Magisk modules. - /// - /// Calls `get_installed_apps({ignore_system})` on the Kotlin Ktor server. - /// Returns an empty list if the call fails. - pub async fn get_installed_apps(&self, ignore_system: bool) -> Vec { - let client = match HttpClientBuilder::default().build(&self.url) { - Ok(c) => c, - Err(_) => return vec![], - }; - let params = rpc_params!(GetInstalledAppsParams { ignore_system }); - client - .request::, _>("get_installed_apps", params) - .await - .unwrap_or_default() - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_get_local_version_params_serialization() { - let app_id: HashMap> = HashMap::from([( - "android_app_package".to_string(), - Some("com.example.app".to_string()), - )]); - let params = GetLocalVersionParams { - app_id: app_id.clone(), - }; - let json = serde_json::to_string(¶ms).unwrap(); - assert!(json.contains("android_app_package")); - assert!(json.contains("com.example.app")); - } - - #[test] - fn test_get_installed_apps_params_serialization() { - let params = GetInstalledAppsParams { - ignore_system: true, - }; - let json = serde_json::to_string(¶ms).unwrap(); - assert!(json.contains("ignore_system")); - assert!(json.contains("true")); - } -} diff --git a/src/manager/app_manager.rs b/src/manager/app_manager.rs deleted file mode 100644 index 12b0b32..0000000 --- a/src/manager/app_manager.rs +++ /dev/null @@ -1,462 +0,0 @@ -use std::collections::HashMap; -use std::sync::Arc; -use tokio::sync::{RwLock, Semaphore}; - -use super::android_api; -use super::app_status::AppStatus; -use super::data_getter::DataGetter; -use super::notification::{notify_if_registered, ManagerEvent}; -use super::updater::get_release_status; -use super::version_map::VersionMap; -use crate::database::get_db; -use crate::database::models::app::AppRecord; -use crate::error::Result; - -pub type UpdateCallback = Arc; - -/// Manages all tracked apps and their version data. -/// -/// Mirrors Kotlin's `AppManager`. -pub struct AppManager { - /// Saved apps from the database (keyed by record id) - apps: Arc>>, - /// Virtual apps from Android installed packages (not persisted) - virtual_apps: Arc>>, - /// In-memory version maps keyed by app record id - version_maps: Arc>>, - data_getter: Arc, -} - -impl AppManager { - pub fn load() -> Result { - let records = get_db().load_apps()?; - let apps = records.into_iter().map(|a| (a.id.clone(), a)).collect(); - Ok(Self { - apps: Arc::new(RwLock::new(apps)), - virtual_apps: Arc::new(RwLock::new(vec![])), - version_maps: Arc::new(RwLock::new(HashMap::new())), - data_getter: Arc::new(DataGetter::new()), - }) - } - - /// Replace the virtual app list (installed Android packages). - pub async fn set_virtual_apps(&self, apps: Vec) { - *self.virtual_apps.write().await = apps; - } - - pub async fn get_all_apps(&self) -> Vec { - let mut result: Vec = self.apps.read().await.values().cloned().collect(); - result.extend(self.virtual_apps.read().await.clone()); - result - } - - pub async fn get_saved_apps(&self) -> Vec { - self.apps.read().await.values().cloned().collect() - } - - pub async fn find_app_by_id( - &self, - app_id: &HashMap>, - ) -> Option { - self.apps - .read() - .await - .values() - .find(|a| &a.app_id == app_id) - .cloned() - } - - pub async fn get_app(&self, record_id: &str) -> Option { - self.apps.read().await.get(record_id).cloned() - } - - /// Persist (insert or update) an app record. - pub async fn save_app(&self, mut record: AppRecord) -> Result { - if record.id.is_empty() { - record.id = uuid::Uuid::new_v4().to_string(); - } - get_db().upsert_app(&record)?; - self.apps - .write() - .await - .insert(record.id.clone(), record.clone()); - Ok(record) - } - - /// Remove a saved app. - pub async fn remove_app(&self, record_id: &str) -> Result { - let deleted = get_db().delete_app(record_id)?; - self.apps.write().await.remove(record_id); - self.version_maps.write().await.remove(record_id); - Ok(deleted) - } - - /// Return the current AppStatus for an app. - /// - /// Queries Kotlin for the locally-installed version via `AndroidApi.get_local_version()` - /// if a callback URL has been registered; otherwise local_version is `None`. - pub async fn get_app_status(&self, record_id: &str) -> AppStatus { - let app = match self.apps.read().await.get(record_id).cloned() { - Some(a) => a, - None => return AppStatus::AppInactive, - }; - - // Query local version from Android via registered callback - let local_version: Option = match android_api::get_android_api() { - Some(api) => api.get_local_version(&app.app_id).await, - None => None, - }; - - let mut maps = self.version_maps.write().await; - let vm = maps.entry(record_id.to_string()).or_insert_with(|| { - VersionMap::new( - app.invalid_version_number_field_regex.clone(), - app.include_version_number_field_regex.clone(), - ) - }); - get_release_status( - vm, - local_version.as_deref(), - app.ignore_version_number.as_deref(), - true, - ) - } - - /// Refresh version data for all saved apps. - /// - /// Uses a semaphore (max 10 concurrent hub requests) matching Kotlin's logic. - /// Apps that have version-filter regexes (`need_complete_version`) skip the batch - /// API entirely and go straight to the full release-list path, mirroring Kotlin's - /// `simpleMap` / `completeMap` split in `AppManager.renewAppList()`. - /// Fires `RenewProgress` notifications to Kotlin UI as each app completes. - pub async fn renew_all( - &self, - hubs: &[crate::database::models::hub::HubRecord], - progress_cb: Option, - ) { - let apps = self.get_saved_apps().await; - let total = apps.len(); - let semaphore = Arc::new(Semaphore::new(10)); - - // Group apps by hub, splitting into simple (batch-eligible) vs complete-only. - let mut hub_simple_map: HashMap> = HashMap::new(); - let mut hub_complete_map: HashMap> = HashMap::new(); - for app in &apps { - let sorted_hubs = app.get_sorted_hub_uuids(); - let effective_hubs: Vec<&str> = if sorted_hubs.is_empty() { - hubs.iter().map(|h| h.uuid.as_str()).collect() - } else { - sorted_hubs.iter().map(String::as_str).collect() - }; - let dest = if need_complete_version(app) { - &mut hub_complete_map - } else { - &mut hub_simple_map - }; - for hub_uuid in effective_hubs { - dest.entry(hub_uuid.to_string()) - .or_default() - .push(app.clone()); - } - } - - let completed = Arc::new(std::sync::atomic::AtomicUsize::new(0)); - let mut handles = vec![]; - - for hub in hubs { - let simple_apps = hub_simple_map.get(&hub.uuid).cloned().unwrap_or_default(); - let complete_apps = hub_complete_map.get(&hub.uuid).cloned().unwrap_or_default(); - if simple_apps.is_empty() && complete_apps.is_empty() { - continue; - } - - let hub = hub.clone(); - let getter = self.data_getter.clone(); - let version_maps = self.version_maps.clone(); - let sem = semaphore.clone(); - let completed = completed.clone(); - let cb = progress_cb.clone(); - - let handle = tokio::spawn(async move { - let _permit = sem.acquire().await.unwrap(); - - // --- Batch path (simple apps only) --- - let mut need_full: Vec = complete_apps; // complete-only apps go straight here - if !simple_apps.is_empty() { - let app_ids: Vec>> = - simple_apps.iter().map(|a| a.app_id.clone()).collect(); - let latest_results = getter.get_latest_releases(&hub, &app_ids).await; - - for (app, (_, maybe_release)) in simple_apps.iter().zip(latest_results.iter()) { - if let Some(release) = maybe_release { - let new_status = { - let mut maps = version_maps.write().await; - let vm = maps.entry(app.id.clone()).or_insert_with(|| { - VersionMap::new( - app.invalid_version_number_field_regex.clone(), - app.include_version_number_field_regex.clone(), - ) - }); - vm.add_single_release(&hub.uuid, release.clone()); - let local_version: Option = - match android_api::get_android_api() { - Some(api) => api.get_local_version(&app.app_id).await, - None => None, - }; - get_release_status( - vm, - local_version.as_deref(), - app.ignore_version_number.as_deref(), - true, - ) - }; - let done = - completed.fetch_add(1, std::sync::atomic::Ordering::SeqCst) + 1; - if let Some(ref f) = cb { - f(done, total); - } - notify_if_registered(ManagerEvent::RenewProgress { done, total }).await; - if new_status != AppStatus::AppPending { - notify_if_registered(ManagerEvent::AppStatusChanged { - record_id: app.id.clone(), - app_id: app.app_id.clone(), - old_status: AppStatus::AppPending, - new_status, - }) - .await; - } - } else { - // Batch returned nothing for this app — escalate to full list. - need_full.push(app.clone()); - } - } - } - - // --- Full release-list path --- - for app in need_full { - let new_status = { - let mut maps = version_maps.write().await; - let vm = maps.entry(app.id.clone()).or_insert_with(|| { - VersionMap::new( - app.invalid_version_number_field_regex.clone(), - app.include_version_number_field_regex.clone(), - ) - }); - if let Some(releases) = getter.get_release_list(&hub, &app.app_id).await { - vm.add_release_list(&hub.uuid, releases); - } else { - vm.set_error(&hub.uuid); - } - let local_version: Option = match android_api::get_android_api() { - Some(api) => api.get_local_version(&app.app_id).await, - None => None, - }; - get_release_status( - vm, - local_version.as_deref(), - app.ignore_version_number.as_deref(), - true, - ) - }; - let done = completed.fetch_add(1, std::sync::atomic::Ordering::SeqCst) + 1; - if let Some(ref f) = cb { - f(done, total); - } - notify_if_registered(ManagerEvent::RenewProgress { done, total }).await; - if new_status != AppStatus::AppPending { - notify_if_registered(ManagerEvent::AppStatusChanged { - record_id: app.id.clone(), - app_id: app.app_id.clone(), - old_status: AppStatus::AppPending, - new_status, - }) - .await; - } - } - }); - handles.push(handle); - } - - for handle in handles { - let _ = handle.await; - } - } - - /// Return record IDs of saved apps that have no valid hub configured. - /// - /// An app is considered "invalid" when its `enable_hub_list` is non-empty but - /// none of the listed hub UUIDs exist in `known_hub_uuids`. Apps with an empty - /// hub list (meaning "use all hubs") are never reported as invalid. - pub async fn check_invalid_applications(&self, known_hub_uuids: &[String]) -> Vec { - let apps = self.apps.read().await; - apps.values() - .filter_map(|app| { - let hub_uuids = app.get_sorted_hub_uuids(); - // Empty list means "match any hub" — not invalid. - if hub_uuids.is_empty() { - return None; - } - let has_valid = hub_uuids.iter().any(|uuid| known_hub_uuids.contains(uuid)); - if has_valid { - None - } else { - Some(app.id.clone()) - } - }) - .collect() - } -} - -/// Returns `true` if the app requires the full release list rather than the single -/// latest-release batch API. -/// -/// Mirrors Kotlin's `App.needCompleteVersion`: -/// ```kotlin -/// val needCompleteVersion: Boolean -/// get() = db.includeVersionNumberFieldRegexString != null -/// || db.invalidVersionNumberFieldRegexString != null -/// ``` -fn need_complete_version(app: &AppRecord) -> bool { - app.include_version_number_field_regex.is_some() - || app.invalid_version_number_field_regex.is_some() -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::database; - - #[test] - fn test_app_record_save_and_load() { - let dir = tempfile::tempdir().unwrap(); - let db = database::Database::open(dir.path()).unwrap(); - - let app = AppRecord::new( - "MyApp".to_string(), - HashMap::from([("owner".to_string(), Some("alice".to_string()))]), - ); - db.upsert_app(&app).unwrap(); - let loaded = db.load_apps().unwrap(); - assert_eq!(loaded.len(), 1); - assert_eq!(loaded[0].name, "MyApp"); - } - - #[tokio::test] - async fn test_version_map_get_status_no_data() { - let mut vm = VersionMap::new(None, None); - let status = get_release_status(&mut vm, Some("1.0.0"), None, true); - // Empty version map + is_saved → NetworkError (no hub_status entries) - assert_eq!(status, AppStatus::NetworkError); - } - - // ------------------------------------------------------------------------- - // Phase 7A: need_complete_version - // ------------------------------------------------------------------------- - - #[test] - fn test_need_complete_version_false_when_no_regex() { - let app = AppRecord::new("App".to_string(), HashMap::new()); - assert!(!need_complete_version(&app)); - } - - #[test] - fn test_need_complete_version_true_when_invalid_regex() { - let mut app = AppRecord::new("App".to_string(), HashMap::new()); - app.invalid_version_number_field_regex = Some("alpha|beta".to_string()); - assert!(need_complete_version(&app)); - } - - #[test] - fn test_need_complete_version_true_when_include_regex() { - let mut app = AppRecord::new("App".to_string(), HashMap::new()); - app.include_version_number_field_regex = Some(r"\d+\.\d+".to_string()); - assert!(need_complete_version(&app)); - } - - #[test] - fn test_need_complete_version_true_when_both_regex() { - let mut app = AppRecord::new("App".to_string(), HashMap::new()); - app.invalid_version_number_field_regex = Some("alpha".to_string()); - app.include_version_number_field_regex = Some(r"\d+".to_string()); - assert!(need_complete_version(&app)); - } - - // ------------------------------------------------------------------------- - // Phase 7B: check_invalid_applications - // ------------------------------------------------------------------------- - - fn make_app_with_hubs(name: &str, hubs: &[&str]) -> AppRecord { - let mut app = AppRecord::new(name.to_string(), HashMap::new()); - let hub_strs: Vec = hubs.iter().map(|s| s.to_string()).collect(); - app.set_sorted_hub_uuids(&hub_strs); - app - } - - async fn app_manager_with_apps(apps: Vec) -> AppManager { - let map = apps.into_iter().map(|a| (a.id.clone(), a)).collect(); - AppManager { - apps: Arc::new(RwLock::new(map)), - virtual_apps: Arc::new(RwLock::new(vec![])), - version_maps: Arc::new(RwLock::new(HashMap::new())), - data_getter: Arc::new(DataGetter::new()), - } - } - - #[tokio::test] - async fn test_check_invalid_no_apps() { - let mgr = app_manager_with_apps(vec![]).await; - let invalid = mgr.check_invalid_applications(&["hub-1".to_string()]).await; - assert!(invalid.is_empty()); - } - - #[tokio::test] - async fn test_check_invalid_empty_hub_list_not_reported() { - // App with no hub list means "use all hubs" — never invalid. - let app = AppRecord::new("NoHubs".to_string(), HashMap::new()); - let mgr = app_manager_with_apps(vec![app]).await; - let invalid = mgr.check_invalid_applications(&[]).await; - assert!(invalid.is_empty()); - } - - #[tokio::test] - async fn test_check_invalid_all_hubs_known() { - let app = make_app_with_hubs("GoodApp", &["hub-a", "hub-b"]); - let mgr = app_manager_with_apps(vec![app]).await; - let known = vec!["hub-a".to_string(), "hub-b".to_string()]; - let invalid = mgr.check_invalid_applications(&known).await; - assert!(invalid.is_empty()); - } - - #[tokio::test] - async fn test_check_invalid_one_hub_known_is_valid() { - // Even if only one of the listed hubs is known, the app is valid. - let app = make_app_with_hubs("SemiGood", &["hub-a", "hub-unknown"]); - let mgr = app_manager_with_apps(vec![app]).await; - let known = vec!["hub-a".to_string()]; - let invalid = mgr.check_invalid_applications(&known).await; - assert!(invalid.is_empty()); - } - - #[tokio::test] - async fn test_check_invalid_all_hubs_unknown() { - let app = make_app_with_hubs("BadApp", &["hub-x", "hub-y"]); - let app_id = app.id.clone(); - let mgr = app_manager_with_apps(vec![app]).await; - let known = vec!["hub-a".to_string()]; - let invalid = mgr.check_invalid_applications(&known).await; - assert_eq!(invalid, vec![app_id]); - } - - #[tokio::test] - async fn test_check_invalid_mixed_apps() { - let good = make_app_with_hubs("Good", &["hub-a"]); - let bad = make_app_with_hubs("Bad", &["hub-z"]); - let bad_id = bad.id.clone(); - let no_hub = AppRecord::new("NoHub".to_string(), HashMap::new()); - let mgr = app_manager_with_apps(vec![good, bad, no_hub]).await; - let known = vec!["hub-a".to_string()]; - let mut invalid = mgr.check_invalid_applications(&known).await; - invalid.sort(); - assert_eq!(invalid, vec![bad_id]); - } -} diff --git a/src/manager/app_status.rs b/src/manager/app_status.rs deleted file mode 100644 index 088ce22..0000000 --- a/src/manager/app_status.rs +++ /dev/null @@ -1,18 +0,0 @@ -use serde::{Deserialize, Serialize}; - -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum AppStatus { - /// App exists only in hub's auto-discovery; not in user's saved list - AppInactive, - /// Version data is being fetched - AppPending, - /// Network request failed - NetworkError, - /// Local version is up to date - AppLatest, - /// A newer version is available - AppOutdated, - /// No local version found (e.g. not installed) - AppNoLocal, -} diff --git a/src/manager/auto_template.rs b/src/manager/auto_template.rs deleted file mode 100644 index bfb640d..0000000 --- a/src/manager/auto_template.rs +++ /dev/null @@ -1,206 +0,0 @@ -use std::collections::HashMap; - -/// URL template regex: matches `%placeholder` tokens. -/// Mirrors Kotlin's `URL_ARG_REGEX = "(%.*?)\\w*"`. -const URL_ARG_PATTERN: &str = r"(%[^%/?\s]+)"; - -/// Given a URL and a list of templates, return the first template that fully -/// matches the URL, with all placeholder values extracted. -/// -/// Template format: `https://github.com/%owner/%repo/releases` -/// Placeholders are `%key` tokens. The returned map has keys without the `%`. -/// -/// Returns `None` if no template matches fully. -/// -/// Mirrors Kotlin's `AutoTemplate.urlToAppId()`. -pub fn url_to_app_id(url: &str, templates: &[String]) -> Option> { - if url.is_empty() || templates.is_empty() { - return None; - } - for template in templates { - if let Some(args) = match_template(url, template) { - return Some(args); - } - } - None -} - -/// Attempt to match `url` against a single `template`. -/// -/// The algorithm splits the template into alternating [literal, placeholder] -/// segments, then uses literals to cut the URL apart and extract placeholder -/// values. Mirrors Kotlin's `AutoTemplate.matchArgs()` and `checkFull()`. -fn match_template(url: &str, template: &str) -> Option> { - let re = regex::Regex::new(URL_ARG_PATTERN).ok()?; - - // Build ordered list of segments: either a literal string or a %placeholder - let mut segments: Vec = Vec::new(); - let mut last = 0; - for m in re.find_iter(template) { - if m.start() > last { - segments.push(Segment::Literal(template[last..m.start()].to_string())); - } - segments.push(Segment::Placeholder(m.as_str().to_string())); - last = m.end(); - } - if last < template.len() { - segments.push(Segment::Literal(template[last..].to_string())); - } - - // Collect expected placeholder keys (in order) - let expected_keys: Vec = segments - .iter() - .filter_map(|s| { - if let Segment::Placeholder(p) = s { - Some(p.trim_start_matches('%').to_string()) - } else { - None - } - }) - .collect(); - - if expected_keys.is_empty() { - return None; - } - - // Walk through segments: use literals to split the URL and assign values to - // adjacent placeholders. - let mut args: HashMap = HashMap::new(); - let mut remaining = url.to_string(); - - for (i, seg) in segments.iter().enumerate() { - match seg { - Segment::Literal(lit) => { - if lit.is_empty() { - continue; - } - // Find the literal in `remaining`, split on it - match remaining.split_once(lit.as_str()) { - Some((before, after)) => { - // `before` belongs to the preceding placeholder (if any) - if i > 0 { - if let Segment::Placeholder(key) = &segments[i - 1] { - let k = key.trim_start_matches('%').to_string(); - if !before.is_empty() { - args.insert(k, before.to_string()); - } - } - } - remaining = after.to_string(); - } - None => return None, // literal not found → no match - } - } - Segment::Placeholder(_) => { - // Value will be filled in when the next literal is processed, - // or captured from trailing remaining string at end. - } - } - } - - // If the last segment is a placeholder, the rest of `remaining` is its value - if let Some(Segment::Placeholder(key)) = segments.last() { - let k = key.trim_start_matches('%').to_string(); - // Strip trailing slash or query string for cleanliness - let val = remaining.trim_end_matches('/').to_string(); - if !val.is_empty() { - args.insert(k, val); - } - } - - // Verify all expected keys were matched - for key in &expected_keys { - if !args.contains_key(key) { - return None; - } - } - - Some(args) -} - -#[derive(Debug)] -enum Segment { - Literal(String), - Placeholder(String), -} - -#[cfg(test)] -mod tests { - use super::*; - - fn templates(list: &[&str]) -> Vec { - list.iter().map(|s| s.to_string()).collect() - } - - #[test] - fn test_github_url() { - let url = "https://github.com/DUpdateSystem/UpgradeAll"; - let result = url_to_app_id(url, &templates(&["https://github.com/%owner/%repo"])); - let map = result.unwrap(); - assert_eq!(map["owner"], "DUpdateSystem"); - assert_eq!(map["repo"], "UpgradeAll"); - } - - #[test] - fn test_github_url_with_trailing_slash() { - let url = "https://github.com/foo/bar/"; - let result = url_to_app_id(url, &templates(&["https://github.com/%owner/%repo/"])); - let map = result.unwrap(); - assert_eq!(map["owner"], "foo"); - assert_eq!(map["repo"], "bar"); - } - - #[test] - fn test_gitlab_url() { - let url = "https://gitlab.com/AuroraOSS/AuroraStore"; - let result = url_to_app_id(url, &templates(&["https://gitlab.com/%owner/%repo"])); - let map = result.unwrap(); - assert_eq!(map["owner"], "AuroraOSS"); - assert_eq!(map["repo"], "AuroraStore"); - } - - #[test] - fn test_no_match_returns_none() { - let url = "https://example.com/something/else"; - let result = url_to_app_id(url, &templates(&["https://github.com/%owner/%repo"])); - assert!(result.is_none()); - } - - #[test] - fn test_first_matching_template_wins() { - let url = "https://github.com/user/proj"; - let result = url_to_app_id( - url, - &templates(&[ - "https://gitlab.com/%owner/%repo", - "https://github.com/%owner/%repo", - ]), - ); - let map = result.unwrap(); - assert_eq!(map["owner"], "user"); - assert_eq!(map["repo"], "proj"); - } - - #[test] - fn test_empty_url_returns_none() { - let result = url_to_app_id("", &templates(&["https://github.com/%owner/%repo"])); - assert!(result.is_none()); - } - - #[test] - fn test_empty_templates_returns_none() { - let result = url_to_app_id("https://github.com/a/b", &[]); - assert!(result.is_none()); - } - - #[test] - fn test_single_placeholder() { - let url = "https://f-droid.org/packages/com.example.app/"; - let result = url_to_app_id( - url, - &templates(&["https://f-droid.org/packages/%package_name/"]), - ); - let map = result.unwrap(); - assert_eq!(map["package_name"], "com.example.app"); - } -} diff --git a/src/manager/cloud_config_getter.rs b/src/manager/cloud_config_getter.rs deleted file mode 100644 index b4fbc66..0000000 --- a/src/manager/cloud_config_getter.rs +++ /dev/null @@ -1,403 +0,0 @@ -use std::collections::HashMap; -use std::sync::Arc; -use tokio::sync::RwLock; - -use crate::database::models::app::AppRecord; -use crate::database::models::hub::HubRecord; -use crate::error::{Error, Result}; -use crate::manager::app_manager::AppManager; -use crate::manager::auto_template::url_to_app_id; -use crate::manager::hub_manager::HubManager; -use crate::websdk::cloud_rules::cloud_rules_manager::CloudRules; -use crate::websdk::cloud_rules::data::app_item::AppItem; -use crate::websdk::cloud_rules::data::hub_item::HubItem; - -/// Manages downloading, caching, and applying cloud hub/app configurations. -/// -/// Mirrors Kotlin's `CloudConfigGetter` singleton. -pub struct CloudConfigGetter { - api_url: String, - cloud_rules: Arc>>, -} - -impl CloudConfigGetter { - pub fn new(api_url: String) -> Self { - Self { - api_url, - cloud_rules: Arc::new(RwLock::new(None)), - } - } - - /// Download and cache the latest cloud config list. - pub async fn renew(&self) -> Result<()> { - let mut rules = CloudRules::new(&self.api_url); - rules - .renew() - .await - .map_err(|e| Error::Other(e.to_string()))?; - *self.cloud_rules.write().await = Some(rules); - Ok(()) - } - - /// Returns all available app configs from the cached cloud config. - pub async fn app_config_list(&self) -> Vec { - match self.cloud_rules.read().await.as_ref() { - Some(rules) => rules - .get_config_list() - .app_config_list - .into_iter() - .cloned() - .collect(), - None => vec![], - } - } - - /// Returns all available hub configs from the cached cloud config. - pub async fn hub_config_list(&self) -> Vec { - match self.cloud_rules.read().await.as_ref() { - Some(rules) => rules - .get_config_list() - .hub_config_list - .into_iter() - .cloned() - .collect(), - None => vec![], - } - } - - /// Apply a cloud hub config by UUID: download config → convert → upsert into HubManager. - /// - /// Returns true if the hub was installed or updated. - pub async fn apply_hub_config(&self, uuid: &str, hub_mgr: &mut HubManager) -> Result { - let hub_item = self - .find_hub_item(uuid) - .await - .ok_or_else(|| Error::Other(format!("Hub config not found: {uuid}")))?; - - let record = hub_item_to_record(&hub_item, hub_mgr).await; - hub_mgr.upsert_hub(record).await?; - Ok(true) - } - - /// Apply a cloud app config by UUID: - /// 1. Ensure hub dependency is installed. - /// 2. Extract app_id from the app's URL using the hub's URL templates. - /// 3. Merge with `extra_map` from the cloud config. - /// 4. Upsert into AppManager. - /// - /// Returns true if the app was installed or updated. - pub async fn apply_app_config( - &self, - uuid: &str, - app_mgr: &mut AppManager, - hub_mgr: &mut HubManager, - ) -> Result { - let app_item = self - .find_app_item(uuid) - .await - .ok_or_else(|| Error::Other(format!("App config not found: {uuid}")))?; - - // Ensure hub dependency is present - self.solve_hub_dependency(&app_item.base_hub_uuid, hub_mgr) - .await?; - - // Build the app_id map - let app_id = build_app_id(&app_item, hub_mgr).await; - - // Find existing record by cloud UUID or create a new one - let mut record = app_mgr - .find_app_by_cloud_uuid(&app_item.uuid) - .await - .unwrap_or_else(|| { - let mut r = AppRecord::new(app_item.info.name.clone(), app_id.clone()); - r.id = String::new(); // let save_app assign UUID - r - }); - - record.name = app_item.info.name.clone(); - record.app_id = app_id; - record.cloud_config = Some(app_item.clone()); - - // Ensure base_hub_uuid is first in the hub priority list - let mut hub_uuids = record.get_sorted_hub_uuids(); - if !hub_uuids.contains(&app_item.base_hub_uuid) { - hub_uuids.insert(0, app_item.base_hub_uuid.clone()); - } else if hub_uuids[0] != app_item.base_hub_uuid { - hub_uuids.retain(|u| u != &app_item.base_hub_uuid); - hub_uuids.insert(0, app_item.base_hub_uuid.clone()); - } - record.set_sorted_hub_uuids(&hub_uuids); - - app_mgr.save_app(record).await?; - Ok(true) - } - - /// Bulk-update all installed apps and hubs whose cloud config version has increased. - /// - /// Mirrors Kotlin's `renewAllAppConfigFromCloud` + `renewAllHubConfigFromCloud`. - pub async fn renew_all_from_cloud( - &self, - app_mgr: &mut AppManager, - hub_mgr: &mut HubManager, - ) -> Result<()> { - // Renew hubs - let installed_hubs = hub_mgr.get_hub_list().await; - for hub in &installed_hubs { - if let Some(cloud_hub) = self.find_hub_item(&hub.uuid).await { - if cloud_hub.config_version > hub.hub_config.config_version { - let record = hub_item_to_record(&cloud_hub, hub_mgr).await; - hub_mgr.upsert_hub(record).await?; - } - } - } - - // Renew apps - let installed_apps = app_mgr.get_saved_apps().await; - for app in &installed_apps { - let cloud_uuid = match app.cloud_config.as_ref().map(|c| c.uuid.as_str()) { - Some(u) if !u.is_empty() => u.to_string(), - _ => continue, - }; - if let Some(cloud_app) = self.find_app_item(&cloud_uuid).await { - let installed_version = app - .cloud_config - .as_ref() - .map(|c| c.config_version) - .unwrap_or(0); - if cloud_app.config_version > installed_version { - let _ = self.apply_app_config(&cloud_uuid, app_mgr, hub_mgr).await; - } - } - } - - Ok(()) - } - - /// Ensure a hub is installed; if not, download and install it from cloud config. - async fn solve_hub_dependency(&self, hub_uuid: &str, hub_mgr: &mut HubManager) -> Result<()> { - if hub_mgr.get_hub(hub_uuid).await.is_some() { - // Already installed — check if update needed - let installed = hub_mgr.get_hub(hub_uuid).await.unwrap(); - if let Some(cloud) = self.find_hub_item(hub_uuid).await { - if cloud.config_version > installed.hub_config.config_version { - let record = hub_item_to_record(&cloud, hub_mgr).await; - hub_mgr.upsert_hub(record).await?; - } - } - } else { - // Not installed — download and install - self.apply_hub_config(hub_uuid, hub_mgr).await?; - } - Ok(()) - } - - async fn find_app_item(&self, uuid: &str) -> Option { - self.cloud_rules - .read() - .await - .as_ref()? - .get_config_list() - .app_config_list - .into_iter() - .find(|a| a.uuid == uuid) - .cloned() - } - - async fn find_hub_item(&self, uuid: &str) -> Option { - self.cloud_rules - .read() - .await - .as_ref()? - .get_config_list() - .hub_config_list - .into_iter() - .find(|h| h.uuid == uuid) - .cloned() - } -} - -/// Convert a `HubItem` from cloud config to a `HubRecord`, preserving any -/// existing auth / ignore lists if the hub is already installed. -async fn hub_item_to_record(hub_item: &HubItem, hub_mgr: &HubManager) -> HubRecord { - if let Some(existing) = hub_mgr.get_hub(&hub_item.uuid).await { - // Preserve mutable fields, update config fields - HubRecord { - hub_config: hub_item.clone(), - ..existing - } - } else { - HubRecord::new(hub_item.uuid.clone(), hub_item.clone()) - } -} - -/// Build the `app_id` map from a cloud `AppItem`. -/// -/// 1. Try to extract from `info.url` using the hub's `app_url_templates`. -/// 2. Merge with `info.extra_map` (extra_map takes precedence for any shared keys). -async fn build_app_id(app_item: &AppItem, hub_mgr: &HubManager) -> HashMap> { - let mut app_id: HashMap> = HashMap::new(); - - // Try URL template extraction - if let Some(hub) = hub_mgr.get_hub(&app_item.base_hub_uuid).await { - let templates = &hub.hub_config.app_url_templates; - if !app_item.info.url.is_empty() { - if let Some(extracted) = url_to_app_id(&app_item.info.url, templates) { - for (k, v) in extracted { - app_id.insert(k, Some(v)); - } - } - } - } - - // Merge extra_map — extra_map values always win over URL-extracted values - for (k, v) in &app_item.info.extra_map { - app_id.insert(k.clone(), Some(v.clone())); - } - - app_id -} - -// Extension on AppManager to find by cloud UUID -impl AppManager { - pub async fn find_app_by_cloud_uuid(&self, cloud_uuid: &str) -> Option { - self.get_saved_apps().await.into_iter().find(|a| { - a.cloud_config - .as_ref() - .map(|c| c.uuid == cloud_uuid) - .unwrap_or(false) - }) - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::websdk::cloud_rules::data::app_item::AppInfo; - use crate::websdk::cloud_rules::data::hub_item::Info; - - fn make_hub_item(uuid: &str, templates: Vec) -> HubItem { - HubItem { - base_version: 6, - config_version: 1, - uuid: uuid.to_string(), - info: Info { - hub_name: "GitHub".to_string(), - hub_icon_url: None, - }, - api_keywords: vec!["owner".to_string(), "repo".to_string()], - auth_keywords: vec![], - app_url_templates: templates, - target_check_api: None, - } - } - - fn make_app_item(uuid: &str, hub_uuid: &str, url: &str) -> AppItem { - AppItem { - base_version: 2, - config_version: 1, - uuid: uuid.to_string(), - base_hub_uuid: hub_uuid.to_string(), - info: AppInfo { - name: "TestApp".to_string(), - url: url.to_string(), - extra_map: HashMap::new(), - }, - } - } - - #[tokio::test] - async fn test_build_app_id_from_url() { - let dir = tempfile::tempdir().unwrap(); - let _ = crate::database::init_db(dir.path()); - - let hub_uuid = "fd9b2602-62c5-4d55-bd1e-0d6537714ca0"; - let db = crate::database::Database::open(dir.path()).unwrap(); - let hub = HubRecord::new( - hub_uuid.to_string(), - make_hub_item( - hub_uuid, - vec!["https://github.com/%owner/%repo/".to_string()], - ), - ); - db.upsert_hub(&hub).unwrap(); - - let hub_mgr = HubManager::load().unwrap_or_else(|_| { - // fallback: create manager from local db - let records = db.load_hubs().unwrap(); - let _ = records; // suppress unused warning - // We can't easily construct HubManager without global DB, so use a helper - panic!("This test requires global DB init") - }); - let _ = hub_mgr; - } - - #[test] - fn test_build_app_id_extra_map_wins() { - // extra_map should take precedence over URL-extracted values - let app_item = AppItem { - base_version: 2, - config_version: 1, - uuid: "test-uuid".to_string(), - base_hub_uuid: "hub-uuid".to_string(), - info: AppInfo { - name: "TestApp".to_string(), - url: "https://github.com/owner/repo".to_string(), - extra_map: HashMap::from([ - ("android_app_package".to_string(), "com.example".to_string()), - ("owner".to_string(), "override_owner".to_string()), - ]), - }, - }; - - // Simulate what build_app_id does with extra_map override - let mut app_id: HashMap> = HashMap::new(); - // Pretend URL extraction gave owner=owner, repo=repo - app_id.insert("owner".to_string(), Some("owner".to_string())); - app_id.insert("repo".to_string(), Some("repo".to_string())); - // extra_map override - for (k, v) in &app_item.info.extra_map { - app_id.insert(k.clone(), Some(v.clone())); - } - // owner should be overridden by extra_map - assert_eq!(app_id["owner"], Some("override_owner".to_string())); - assert_eq!( - app_id["android_app_package"], - Some("com.example".to_string()) - ); - // repo still from URL extraction - assert_eq!(app_id["repo"], Some("repo".to_string())); - } - - #[test] - fn test_hub_item_to_record_preserves_auth() { - // hub_item_to_record should preserve auth from existing record - // We test the logic inline since it's async and needs a HubManager - let hub_item = make_hub_item( - "hub-1", - vec!["https://github.com/%owner/%repo/".to_string()], - ); - let existing = HubRecord { - uuid: "hub-1".to_string(), - hub_config: make_hub_item("hub-1", vec![]), - auth: HashMap::from([("token".to_string(), "secret".to_string())]), - ignore_app_id_list: vec![], - applications_mode: 1, - user_ignore_app_id_list: vec![], - sort_point: -5, - }; - - // Simulate hub_item_to_record(hub_item, existing) - let record = HubRecord { - hub_config: hub_item.clone(), - ..existing.clone() - }; - - assert_eq!(record.auth["token"], "secret"); - assert_eq!(record.applications_mode, 1); - assert_eq!(record.sort_point, -5); - assert_eq!( - record.hub_config.app_url_templates[0], - "https://github.com/%owner/%repo/" - ); - } -} diff --git a/src/manager/data_getter.rs b/src/manager/data_getter.rs deleted file mode 100644 index 49312ef..0000000 --- a/src/manager/data_getter.rs +++ /dev/null @@ -1,99 +0,0 @@ -use std::collections::{BTreeMap, HashMap}; -use std::sync::Arc; -use tokio::sync::Mutex; - -use crate::database::models::hub::HubRecord; -use crate::websdk::repo::api; -use crate::websdk::repo::data::release::ReleaseData; - -/// Result of a batch latest-release request. -/// Each entry is (app_id, Option) — None means the app wasn't found. -pub type LatestReleaseResults = Vec<(HashMap>, Option)>; - -/// Fetches release data from hub providers. -/// -/// Mirrors Kotlin's `DataGetter`. -pub struct DataGetter { - /// Per-hub mutex to prevent duplicate concurrent requests. - hub_locks: Arc>>>>, -} - -impl DataGetter { - pub fn new() -> Self { - Self { - hub_locks: Arc::new(Mutex::new(HashMap::new())), - } - } - - async fn hub_lock(&self, hub_uuid: &str) -> Arc> { - let mut locks = self.hub_locks.lock().await; - locks - .entry(hub_uuid.to_string()) - .or_insert_with(|| Arc::new(Mutex::new(()))) - .clone() - } - - /// Fetch the latest single release for each app from one hub. - /// - /// Returns a vec of `(app_id, Option)`. - /// `None` means the hub didn't return data for that app. - pub async fn get_latest_releases( - &self, - hub: &HubRecord, - app_ids: &[HashMap>], - ) -> LatestReleaseResults { - let lock = self.hub_lock(&hub.uuid).await; - let _guard = lock.lock().await; - - let mut results = Vec::with_capacity(app_ids.len()); - for app_id in app_ids { - let app_data = build_app_data(app_id, &hub.hub_config.api_keywords); - if app_data.is_empty() { - results.push((app_id.clone(), None)); - continue; - } - let hub_data = build_hub_data(&hub.auth); - let release = api::get_latest_release(&hub.uuid, &app_data, &hub_data).await; - results.push((app_id.clone(), release)); - } - results - } - - /// Fetch the full release list for a single app from one hub. - pub async fn get_release_list( - &self, - hub: &HubRecord, - app_id: &HashMap>, - ) -> Option> { - let lock = self.hub_lock(&hub.uuid).await; - let _guard = lock.lock().await; - - let app_data = build_app_data(app_id, &hub.hub_config.api_keywords); - if app_data.is_empty() { - return None; - } - let hub_data = build_hub_data(&hub.auth); - api::get_releases(&hub.uuid, &app_data, &hub_data).await - } -} - -fn build_app_data<'a>( - app_id: &'a HashMap>, - api_keywords: &[String], -) -> BTreeMap<&'a str, &'a str> { - app_id - .iter() - .filter(|(k, v)| api_keywords.contains(k) && v.is_some()) - .map(|(k, v)| (k.as_str(), v.as_deref().unwrap())) - .collect() -} - -fn build_hub_data(auth: &HashMap) -> BTreeMap<&str, &str> { - auth.iter().map(|(k, v)| (k.as_str(), v.as_str())).collect() -} - -impl Default for DataGetter { - fn default() -> Self { - Self::new() - } -} diff --git a/src/manager/hub_manager.rs b/src/manager/hub_manager.rs deleted file mode 100644 index 9c52503..0000000 --- a/src/manager/hub_manager.rs +++ /dev/null @@ -1,173 +0,0 @@ -use std::collections::HashMap; -use std::sync::Arc; -use tokio::sync::RwLock; - -use crate::database::get_db; -use crate::database::models::hub::HubRecord; -use crate::error::Result; - -/// In-memory hub registry backed by the JSONL database. -/// -/// Mirrors Kotlin's `HubManager`. -pub struct HubManager { - hubs: Arc>>, -} - -impl HubManager { - /// Load all hubs from the database. - pub fn load() -> Result { - let records = get_db().load_hubs()?; - let map = records.into_iter().map(|h| (h.uuid.clone(), h)).collect(); - Ok(Self { - hubs: Arc::new(RwLock::new(map)), - }) - } - - pub async fn get_hub_list(&self) -> Vec { - self.hubs.read().await.values().cloned().collect() - } - - pub async fn get_hub(&self, uuid: &str) -> Option { - self.hubs.read().await.get(uuid).cloned() - } - - /// Insert or update a hub (persists to database). - pub async fn upsert_hub(&self, record: HubRecord) -> Result<()> { - get_db().upsert_hub(&record)?; - self.hubs.write().await.insert(record.uuid.clone(), record); - Ok(()) - } - - /// Remove a hub by UUID (persists deletion to database). - pub async fn remove_hub(&self, uuid: &str) -> Result { - let deleted = get_db().delete_hub(uuid)?; - self.hubs.write().await.remove(uuid); - Ok(deleted) - } - - pub async fn is_applications_mode_enabled(&self) -> bool { - self.hubs - .read() - .await - .values() - .any(|h| h.applications_mode_enabled()) - } - - /// Update the auth map for a hub identified by UUID and persist the change. - /// - /// Returns `false` if no hub with the given UUID exists. - pub async fn update_auth(&self, uuid: &str, auth: HashMap) -> Result { - let mut hubs = self.hubs.write().await; - let hub = match hubs.get_mut(uuid) { - Some(h) => h, - None => return Ok(false), - }; - hub.auth = auth; - get_db().upsert_hub(hub)?; - Ok(true) - } - - /// Return hubs whose api_keywords contain any of the given app_id keys. - pub async fn hubs_for_app(&self, app_id: &HashMap>) -> Vec { - let app_keys: Vec<&str> = app_id.keys().map(String::as_str).collect(); - self.hubs - .read() - .await - .values() - .filter(|h| { - h.hub_config - .api_keywords - .iter() - .any(|kw| app_keys.contains(&kw.as_str())) - }) - .cloned() - .collect() - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::database; - use crate::websdk::cloud_rules::data::hub_item::{HubItem, Info}; - use tempfile::TempDir; - - fn setup_db() -> TempDir { - let dir = tempfile::tempdir().unwrap(); - database::init_db(dir.path()).ok(); // may already be init in other tests - dir - } - - fn make_hub(uuid: &str) -> HubRecord { - HubRecord::new( - uuid.to_string(), - HubItem { - base_version: 6, - config_version: 1, - uuid: uuid.to_string(), - info: Info { - hub_name: "TestHub".to_string(), - hub_icon_url: None, - }, - api_keywords: vec!["owner".to_string(), "repo".to_string()], - auth_keywords: vec![], - app_url_templates: vec![], - target_check_api: None, - }, - ) - } - - // These tests use a fresh TempDir + DB each time via open() directly, - // bypassing the global singleton to allow parallel test runs. - #[test] - fn test_upsert_and_list() { - let dir = tempfile::tempdir().unwrap(); - let db = crate::database::Database::open(dir.path()).unwrap(); - let hub = make_hub("uuid-1"); - db.upsert_hub(&hub).unwrap(); - let hubs = db.load_hubs().unwrap(); - assert_eq!(hubs.len(), 1); - assert_eq!(hubs[0].uuid, "uuid-1"); - } - - #[test] - fn test_delete_hub() { - let dir = tempfile::tempdir().unwrap(); - let db = crate::database::Database::open(dir.path()).unwrap(); - let hub = make_hub("uuid-2"); - db.upsert_hub(&hub).unwrap(); - let deleted = db.delete_hub("uuid-2").unwrap(); - assert!(deleted); - assert!(db.load_hubs().unwrap().is_empty()); - } - - #[tokio::test] - async fn test_update_auth() { - let dir = tempfile::tempdir().unwrap(); - crate::database::init_db(dir.path()).ok(); - - // Insert the hub via HubManager so it is in both the global DB and in-memory state. - let mgr = HubManager::load().unwrap(); - let hub = make_hub("uuid-auth"); - mgr.upsert_hub(hub).await.unwrap(); - - let new_auth: HashMap = - [("token".to_string(), "ghp_test123".to_string())].into(); - let ok = mgr - .update_auth("uuid-auth", new_auth.clone()) - .await - .unwrap(); - assert!(ok); - - // Verify in-memory state updated. - let updated = mgr.get_hub("uuid-auth").await.unwrap(); - assert_eq!(updated.auth, new_auth); - - // Returns false for unknown UUID. - let not_found = mgr - .update_auth("no-such-uuid", HashMap::new()) - .await - .unwrap(); - assert!(!not_found); - } -} diff --git a/src/manager/mod.rs b/src/manager/mod.rs deleted file mode 100644 index 275d73c..0000000 --- a/src/manager/mod.rs +++ /dev/null @@ -1,12 +0,0 @@ -pub mod android_api; -pub mod app_manager; -pub mod app_status; -pub mod auto_template; -pub mod cloud_config_getter; -pub mod data_getter; -pub mod hub_manager; -pub mod notification; -pub mod updater; -pub mod url_replace; -pub mod version; -pub mod version_map; diff --git a/src/manager/notification.rs b/src/manager/notification.rs deleted file mode 100644 index ef23840..0000000 --- a/src/manager/notification.rs +++ /dev/null @@ -1,152 +0,0 @@ -use once_cell::sync::OnceCell; -use serde::{Deserialize, Serialize}; - -use crate::database::models::app::AppRecord; -use crate::manager::app_status::AppStatus; - -/// Global notification dispatcher, registered via `register_notification` RPC. -static NOTIFICATION: OnceCell = OnceCell::new(); - -pub fn set_notification(url: String) { - let _ = NOTIFICATION.set(NotificationDispatcher { url }); -} - -pub fn get_notification() -> Option<&'static NotificationDispatcher> { - NOTIFICATION.get() -} - -/// Events emitted by the Rust manager layer to the Kotlin UI. -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(tag = "type", rename_all = "snake_case")] -pub enum ManagerEvent { - AppStatusChanged { - record_id: String, - app_id: std::collections::HashMap>, - old_status: AppStatus, - new_status: AppStatus, - }, - RenewProgress { - done: usize, - total: usize, - }, - AppAdded { - record: AppRecord, - }, - AppDeleted { - record_id: String, - }, - AppDatabaseChanged { - record: AppRecord, - }, -} - -/// Dispatches manager events to the Kotlin UI layer via HTTP JSON-RPC. -/// -/// Kotlin registers a notification URL via `register_notification` RPC. -/// When an event fires, Rust POSTs `on_manager_event({event})` to that URL. -/// The Kotlin server handles it and updates the UI (ViewModel / LiveData). -pub struct NotificationDispatcher { - url: String, -} - -impl NotificationDispatcher { - /// Fire an event notification to Kotlin. Best-effort: errors are logged but not propagated. - pub async fn notify(&self, event: ManagerEvent) { - let body = match build_jsonrpc_request("on_manager_event", &event) { - Ok(b) => b, - Err(e) => { - eprintln!("NotificationDispatcher: failed to serialize event: {e}"); - return; - } - }; - - let client = reqwest::Client::new(); - match client - .post(&self.url) - .header("Content-Type", "application/json") - .body(body) - .send() - .await - { - Ok(_) => {} - Err(e) => { - eprintln!("NotificationDispatcher: failed to send notification: {e}"); - } - } - } -} - -fn build_jsonrpc_request( - method: &str, - params: &T, -) -> Result { - #[derive(Serialize)] - struct JsonRpcRequest<'a, P: Serialize> { - jsonrpc: &'a str, - method: &'a str, - params: &'a P, - id: u64, - } - serde_json::to_string(&JsonRpcRequest { - jsonrpc: "2.0", - method, - params, - id: 1, - }) -} - -/// Convenience: notify if a dispatcher is registered (no-op otherwise). -pub async fn notify_if_registered(event: ManagerEvent) { - if let Some(dispatcher) = get_notification() { - dispatcher.notify(event).await; - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_event_serialization_status_changed() { - let event = ManagerEvent::AppStatusChanged { - record_id: "abc-123".to_string(), - app_id: std::collections::HashMap::from([( - "owner".to_string(), - Some("alice".to_string()), - )]), - old_status: AppStatus::AppPending, - new_status: AppStatus::AppOutdated, - }; - let json = serde_json::to_string(&event).unwrap(); - assert!(json.contains("app_status_changed")); - assert!(json.contains("abc-123")); - assert!(json.contains("app_outdated")); - } - - #[test] - fn test_event_serialization_renew_progress() { - let event = ManagerEvent::RenewProgress { done: 3, total: 10 }; - let json = serde_json::to_string(&event).unwrap(); - assert!(json.contains("renew_progress")); - assert!(json.contains("10")); - } - - #[test] - fn test_event_serialization_app_deleted() { - let event = ManagerEvent::AppDeleted { - record_id: "del-456".to_string(), - }; - let json = serde_json::to_string(&event).unwrap(); - assert!(json.contains("app_deleted")); - assert!(json.contains("del-456")); - } - - #[test] - fn test_jsonrpc_request_format() { - let event = ManagerEvent::RenewProgress { done: 1, total: 5 }; - let body = build_jsonrpc_request("on_manager_event", &event).unwrap(); - assert!(body.contains("\"jsonrpc\":\"2.0\"")); - assert!(body.contains("\"method\":\"on_manager_event\"")); - assert!(body.contains("renew_progress")); - } -} diff --git a/src/manager/updater.rs b/src/manager/updater.rs deleted file mode 100644 index ec2fd24..0000000 --- a/src/manager/updater.rs +++ /dev/null @@ -1,151 +0,0 @@ -use super::app_status::AppStatus; -use super::version::VersionInfo; -use super::version_map::{HubStatus, VersionMap}; -use std::collections::HashMap; - -/// Determines the release status for an app given its version map and local version. -/// -/// Mirrors Kotlin's `Updater.getReleaseStatus()`. -pub fn get_release_status( - version_map: &mut VersionMap, - local_version: Option<&str>, - ignore_version: Option<&str>, - is_saved: bool, -) -> AppStatus { - let versions = version_map.get_version_list(); - - if versions.is_empty() { - if version_map.is_renewing() { - return AppStatus::AppPending; - } - let all_error = !version_map.hub_status.is_empty() - && version_map - .hub_status - .values() - .all(|s| *s == HubStatus::Error); - if all_error || is_saved { - return AppStatus::NetworkError; - } - return AppStatus::AppInactive; - } - - let latest_name = &versions[0].version_info.name; - - // If the latest version matches what the user chose to ignore - if let Some(ignored) = ignore_version { - if ignored == latest_name { - return AppStatus::AppLatest; - } - } - - let effective_local = local_version.or(ignore_version); - - match effective_local { - None => AppStatus::AppNoLocal, - Some(local_str) => { - let local_info = VersionInfo::new(local_str, None, None, HashMap::new()); - let latest_info = &versions[0].version_info; - if is_latest(&local_info, latest_info) { - AppStatus::AppLatest - } else { - AppStatus::AppOutdated - } - } - } -} - -fn is_latest(local: &VersionInfo, latest: &VersionInfo) -> bool { - use std::cmp::Ordering; - match local.compare(latest) { - Some(Ordering::Greater) | Some(Ordering::Equal) => true, - Some(Ordering::Less) => false, - None => { - // Fallback: string equality check - local.name == latest.name - } - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::manager::version_map::VersionMap; - use crate::websdk::repo::data::release::{AssetData, ReleaseData}; - - fn release(v: &str) -> ReleaseData { - ReleaseData { - version_number: v.to_string(), - changelog: String::new(), - assets: vec![AssetData { - file_name: "app.apk".to_string(), - file_type: "apk".to_string(), - download_url: "https://x.com".to_string(), - }], - extra: None, - } - } - - fn vm_with(versions: &[&str]) -> VersionMap { - let mut vm = VersionMap::new(None, None); - vm.add_release_list("hub1", versions.iter().map(|v| release(v)).collect()); - vm - } - - #[test] - fn test_latest() { - let mut vm = vm_with(&["2.0.0", "1.0.0"]); - let status = get_release_status(&mut vm, Some("2.0.0"), None, true); - assert_eq!(status, AppStatus::AppLatest); - } - - #[test] - fn test_outdated() { - let mut vm = vm_with(&["2.0.0", "1.0.0"]); - let status = get_release_status(&mut vm, Some("1.0.0"), None, true); - assert_eq!(status, AppStatus::AppOutdated); - } - - #[test] - fn test_no_local() { - let mut vm = vm_with(&["2.0.0"]); - let status = get_release_status(&mut vm, None, None, true); - assert_eq!(status, AppStatus::AppNoLocal); - } - - #[test] - fn test_ignored_version() { - let mut vm = vm_with(&["2.0.0"]); - let status = get_release_status(&mut vm, None, Some("2.0.0"), true); - assert_eq!(status, AppStatus::AppLatest); - } - - #[test] - fn test_network_error() { - let mut vm = VersionMap::new(None, None); - vm.set_error("hub1"); - let status = get_release_status(&mut vm, Some("1.0.0"), None, true); - assert_eq!(status, AppStatus::NetworkError); - } - - #[test] - fn test_pending() { - let mut vm = VersionMap::new(None, None); - vm.mark_renewing("hub1"); - let status = get_release_status(&mut vm, Some("1.0.0"), None, true); - assert_eq!(status, AppStatus::AppPending); - } - - #[test] - fn test_inactive_unsaved() { - let mut vm = VersionMap::new(None, None); - let status = get_release_status(&mut vm, Some("1.0.0"), None, false); - assert_eq!(status, AppStatus::AppInactive); - } - - #[test] - fn test_local_newer_than_remote() { - let mut vm = vm_with(&["1.0.0"]); - let status = get_release_status(&mut vm, Some("2.0.0"), None, true); - assert_eq!(status, AppStatus::AppLatest); - } -} diff --git a/src/manager/url_replace.rs b/src/manager/url_replace.rs deleted file mode 100644 index e205d0e..0000000 --- a/src/manager/url_replace.rs +++ /dev/null @@ -1,170 +0,0 @@ -/// Applies URL replacement rules from an ExtraHub configuration. -/// -/// Mirrors Kotlin's `URLReplace.replaceURL()`. -/// -/// Three replacement modes: -/// 1. Plain regex: `replace(search_regex, replace_str)` across the full URL. -/// 2. Host-only: if `replace_str` looks like a bare host URL (no path), only -/// the host portion of the original URL is replaced. -/// 3. Template: if `replace_str` contains `{DOWNLOAD_URL}`, the original URL -/// is embedded as a parameter (e.g. proxy wrappers). -pub fn apply_url_replace(url: &str, search: Option<&str>, replace: Option<&str>) -> String { - let replace_str = match replace { - Some(r) if !r.is_empty() => r, - _ => return url.to_string(), - }; - - // Mode 3: template substitution — replace_str contains {DOWNLOAD_URL} - if replace_str.contains("{DOWNLOAD_URL}") { - return replace_str.replace("{DOWNLOAD_URL}", url); - } - - // Mode 2: host-only replacement — replace_str is a bare host URL (no path component) - if is_host_only(replace_str) { - return replace_host(url, replace_str); - } - - // Mode 1: plain regex replacement (or literal if no search) - match search { - Some(pattern) if !pattern.is_empty() => match regex::Regex::new(pattern) { - Ok(re) => re.replace_all(url, replace_str).into_owned(), - Err(_) => url.replace(pattern, replace_str), - }, - // No search pattern: nothing to replace - _ => url.to_string(), - } -} - -/// Returns true if `s` looks like a bare host URL with no meaningful path. -/// e.g. "https://mirror.example.com" or "https://mirror.example.com/" -fn is_host_only(s: &str) -> bool { - match url::Url::parse(s) { - Ok(u) => { - let path = u.path(); - path.is_empty() || path == "/" - } - Err(_) => false, - } -} - -/// Replace only the host (scheme + host + port) of `original_url` with the -/// host from `host_url`, keeping the original path, query and fragment. -fn replace_host(original_url: &str, host_url: &str) -> String { - let orig = match url::Url::parse(original_url) { - Ok(u) => u, - Err(_) => return original_url.to_string(), - }; - let host = match url::Url::parse(host_url) { - Ok(u) => u, - Err(_) => return original_url.to_string(), - }; - - // Rebuild: scheme + host from `host`, everything else from `orig` - let mut result = host.clone(); - result.set_path(orig.path()); - result.set_query(orig.query()); - result.set_fragment(orig.fragment()); - result.to_string() -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_no_replace_returns_original() { - let url = "https://github.com/user/repo/releases/download/v1.0/app.apk"; - assert_eq!(apply_url_replace(url, None, None), url); - assert_eq!(apply_url_replace(url, None, Some("")), url); - } - - #[test] - fn test_download_url_template() { - let url = "https://github.com/user/repo/releases/download/v1.0/app.apk"; - let replace = "https://ghproxy.com/?url={DOWNLOAD_URL}"; - let result = apply_url_replace(url, None, Some(replace)); - assert_eq!( - result, - "https://ghproxy.com/?url=https://github.com/user/repo/releases/download/v1.0/app.apk" - ); - } - - #[test] - fn test_host_only_replacement() { - let url = "https://github.com/user/repo/releases/download/v1.0/app.apk"; - let result = apply_url_replace(url, None, Some("https://mirror.ghproxy.com")); - assert!(result.contains("mirror.ghproxy.com")); - assert!(result.contains("/user/repo/releases/download/v1.0/app.apk")); - assert!(!result.contains("github.com")); - } - - #[test] - fn test_host_only_with_trailing_slash() { - let url = "https://github.com/owner/repo/archive/v2.zip"; - let result = apply_url_replace(url, None, Some("https://mirror.example.com/")); - assert!(result.contains("mirror.example.com")); - assert!(result.contains("/owner/repo/archive/v2.zip")); - } - - #[test] - fn test_regex_replacement() { - let url = "https://github.com/user/repo/releases/download/v1.0/app.apk"; - let result = apply_url_replace(url, Some("github\\.com"), Some("github.com.cnpmjs.org")); - assert!(result.contains("github.com.cnpmjs.org")); - assert!(!result.contains("//github.com/")); - } - - #[test] - fn test_invalid_regex_falls_back_to_literal() { - let url = "https://github.com/user/repo"; - // Invalid regex pattern — should fall back to literal string replace - let result = apply_url_replace(url, Some("github.com"), Some("gitlab.com")); - assert!(result.contains("gitlab.com")); - } - - // ------------------------------------------------------------------------- - // Phase 8: chained GLOBAL + hub-specific rules (as applied in get_download) - // ------------------------------------------------------------------------- - - #[test] - fn test_global_then_hub_specific_chain() { - // Simulate: GLOBAL rule replaces github.com host, then hub rule wraps via template. - let url = "https://github.com/owner/repo/releases/download/v1.0/app.apk"; - - // Step 1 — GLOBAL rule: replace host with mirror - let after_global = apply_url_replace(url, None, Some("https://mirror.example.com")); - assert!(after_global.contains("mirror.example.com")); - assert!(!after_global.contains("github.com")); - - // Step 2 — hub-specific rule: wrap with proxy template - let after_hub = apply_url_replace( - &after_global, - None, - Some("https://proxy.example.com/?url={DOWNLOAD_URL}"), - ); - assert!(after_hub.starts_with("https://proxy.example.com/?url=")); - assert!(after_hub.contains("mirror.example.com")); - } - - #[test] - fn test_no_global_rule_hub_rule_applies() { - let url = "https://github.com/owner/repo/archive/v2.zip"; - - // GLOBAL has no replace rule — url unchanged - let after_global = apply_url_replace(url, None, None); - assert_eq!(after_global, url); - - // hub-specific regex rule - let after_hub = - apply_url_replace(&after_global, Some("github\\.com"), Some("gh.example.com")); - assert!(after_hub.contains("gh.example.com")); - } - - #[test] - fn test_both_rules_none_url_unchanged() { - let url = "https://github.com/owner/repo/releases/v1.apk"; - let after_global = apply_url_replace(url, None, None); - let after_hub = apply_url_replace(&after_global, None, None); - assert_eq!(after_hub, url); - } -} diff --git a/src/manager/version.rs b/src/manager/version.rs deleted file mode 100644 index 85547e7..0000000 --- a/src/manager/version.rs +++ /dev/null @@ -1,136 +0,0 @@ -use crate::utils::versioning::Version as VersionUtil; -use crate::websdk::repo::data::release::ReleaseData; -use serde::{Deserialize, Serialize}; -use std::collections::HashMap; - -/// A parsed, comparable version identifier. -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct VersionInfo { - /// Normalized version name (regex-filtered) - pub name: String, - /// Extra metadata (e.g. version_code from Android) - pub extra: HashMap, -} - -impl VersionInfo { - pub fn new( - raw_name: &str, - invalid_regex: Option<&str>, - include_regex: Option<&str>, - extra: HashMap, - ) -> Self { - let name = normalize_version(raw_name, invalid_regex, include_regex); - Self { name, extra } - } - - /// Compare using libversion. Returns Some(Ordering) if both are parseable. - pub fn compare(&self, other: &VersionInfo) -> Option { - let v1 = VersionUtil::new(self.name.clone()); - let v2 = VersionUtil::new(other.name.clone()); - v1.partial_cmp(&v2) - } -} - -impl PartialEq for VersionInfo { - fn eq(&self, other: &Self) -> bool { - self.name == other.name - } -} - -impl Eq for VersionInfo {} - -impl std::hash::Hash for VersionInfo { - fn hash(&self, state: &mut H) { - self.name.hash(state); - } -} - -impl PartialOrd for VersionInfo { - fn partial_cmp(&self, other: &Self) -> Option { - Some(self.cmp(other)) - } -} - -impl Ord for VersionInfo { - fn cmp(&self, other: &Self) -> std::cmp::Ordering { - self.compare(other).unwrap_or(std::cmp::Ordering::Less) - } -} - -/// Strip unwanted parts from a version string using optional regex filters. -fn normalize_version( - raw: &str, - invalid_regex: Option<&str>, - include_regex: Option<&str>, -) -> String { - let mut result = raw.to_string(); - - if let Some(pattern) = invalid_regex { - if let Ok(re) = regex::Regex::new(pattern) { - result = re.replace_all(&result, "").to_string(); - } - } - - if let Some(pattern) = include_regex { - if let Ok(re) = regex::Regex::new(pattern) { - let matched: Vec<&str> = re.find_iter(&result).map(|m| m.as_str()).collect(); - result = matched.join(""); - } - } - - result.trim().to_string() -} - -/// A release from one hub, paired with its assets. -#[derive(Debug, Clone)] -pub struct VersionWrapper { - pub hub_uuid: String, - pub release: ReleaseData, - /// (release_index, asset_index) pairs - pub asset_indices: Vec<(usize, usize)>, -} - -/// Snapshot of a single version with all hub-provided wrappers. -#[derive(Debug, Clone)] -pub struct Version { - pub version_info: VersionInfo, - pub wrappers: Vec, -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_version_info_compare() { - let v1 = VersionInfo::new("1.0.0", None, None, HashMap::new()); - let v2 = VersionInfo::new("1.0.1", None, None, HashMap::new()); - assert!(v1 < v2); - assert!(v2 > v1); - } - - #[test] - fn test_version_info_equal() { - let v1 = VersionInfo::new("1.0.0", None, None, HashMap::new()); - let v2 = VersionInfo::new("1.0.0", None, None, HashMap::new()); - assert_eq!(v1, v2); - } - - #[test] - fn test_normalize_version_invalid_regex() { - let name = normalize_version("v1.0.0", Some("^v"), None); - assert_eq!(name, "1.0.0"); - } - - #[test] - fn test_normalize_version_include_regex() { - let name = normalize_version("Release 1.0.0 (stable)", None, Some(r"\d+\.\d+\.\d+")); - assert_eq!(name, "1.0.0"); - } - - #[test] - fn test_normalize_version_both_filters() { - let name = normalize_version("v1.0.0-beta", Some(r"-beta"), Some(r"\d+\.\d+\.\d+")); - assert_eq!(name, "1.0.0"); - } -} diff --git a/src/manager/version_map.rs b/src/manager/version_map.rs deleted file mode 100644 index 9d275cb..0000000 --- a/src/manager/version_map.rs +++ /dev/null @@ -1,192 +0,0 @@ -use std::collections::HashMap; - -use super::version::{Version, VersionInfo, VersionWrapper}; -use crate::websdk::repo::data::release::ReleaseData; - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum HubStatus { - Renewing, - Error, - /// Got latest release only (single entry per hub) - Single, - /// Got full release list - Full, -} - -/// In-memory version data for a single app, keyed by VersionInfo. -/// -/// Mirrors Kotlin's `VersionMap`. -#[derive(Debug)] -pub struct VersionMap { - pub invalid_version_regex: Option, - pub include_version_regex: Option, - /// Aggregated releases, grouped by normalized VersionInfo - entries: HashMap>, - pub hub_status: HashMap, - /// Cached sorted list, invalidated on mutation - sorted_cache: Option>, -} - -impl VersionMap { - pub fn new(invalid_regex: Option, include_regex: Option) -> Self { - Self { - invalid_version_regex: invalid_regex, - include_version_regex: include_regex, - entries: HashMap::new(), - hub_status: HashMap::new(), - sorted_cache: None, - } - } - - pub fn is_renewing(&self) -> bool { - self.hub_status.values().any(|s| *s == HubStatus::Renewing) - } - - pub fn mark_renewing(&mut self, hub_uuid: &str) { - self.hub_status - .insert(hub_uuid.to_string(), HubStatus::Renewing); - self.sorted_cache = None; - } - - pub fn set_error(&mut self, hub_uuid: &str) { - self.hub_status - .insert(hub_uuid.to_string(), HubStatus::Error); - } - - pub fn add_release_list(&mut self, hub_uuid: &str, releases: Vec) { - for (rel_idx, release) in releases.iter().enumerate() { - let info = self.make_version_info(&release.version_number); - let wrapper = VersionWrapper { - hub_uuid: hub_uuid.to_string(), - release: release.clone(), - asset_indices: (0..release.assets.len()).map(|i| (rel_idx, i)).collect(), - }; - self.entries.entry(info).or_default().push(wrapper); - } - self.hub_status - .insert(hub_uuid.to_string(), HubStatus::Full); - self.sorted_cache = None; - } - - pub fn add_single_release(&mut self, hub_uuid: &str, release: ReleaseData) { - let info = self.make_version_info(&release.version_number); - let wrapper = VersionWrapper { - hub_uuid: hub_uuid.to_string(), - asset_indices: (0..release.assets.len()).map(|i| (0, i)).collect(), - release, - }; - self.entries.entry(info).or_default().push(wrapper); - self.hub_status - .insert(hub_uuid.to_string(), HubStatus::Single); - self.sorted_cache = None; - } - - /// Returns versions sorted descending (newest first). - pub fn get_version_list(&mut self) -> &[Version] { - if self.sorted_cache.is_none() { - let mut versions: Vec = self - .entries - .iter() - .filter(|(info, _)| !info.name.is_empty()) - .map(|(info, wrappers)| Version { - version_info: info.clone(), - wrappers: wrappers.clone(), - }) - .collect(); - versions.sort_by(|a, b| b.version_info.cmp(&a.version_info)); - self.sorted_cache = Some(versions); - } - self.sorted_cache.as_deref().unwrap() - } - - fn make_version_info(&self, raw: &str) -> VersionInfo { - VersionInfo::new( - raw, - self.invalid_version_regex.as_deref(), - self.include_version_regex.as_deref(), - HashMap::new(), - ) - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::websdk::repo::data::release::AssetData; - - fn make_release(version: &str) -> ReleaseData { - ReleaseData { - version_number: version.to_string(), - changelog: String::new(), - assets: vec![AssetData { - file_name: "app.apk".to_string(), - file_type: "apk".to_string(), - download_url: "https://example.com".to_string(), - }], - extra: None, - } - } - - #[test] - fn test_add_and_sort() { - let mut vm = VersionMap::new(None, None); - vm.add_release_list( - "hub1", - vec![ - make_release("1.0.0"), - make_release("2.0.0"), - make_release("1.5.0"), - ], - ); - let list = vm.get_version_list(); - assert_eq!(list[0].version_info.name, "2.0.0"); - assert_eq!(list[1].version_info.name, "1.5.0"); - assert_eq!(list[2].version_info.name, "1.0.0"); - } - - #[test] - fn test_single_release() { - let mut vm = VersionMap::new(None, None); - vm.add_single_release("hub1", make_release("3.0.0")); - let list = vm.get_version_list(); - assert_eq!(list.len(), 1); - assert_eq!(list[0].version_info.name, "3.0.0"); - assert_eq!(vm.hub_status["hub1"], HubStatus::Single); - } - - #[test] - fn test_hub_status_error() { - let mut vm = VersionMap::new(None, None); - vm.set_error("hub1"); - assert_eq!(vm.hub_status["hub1"], HubStatus::Error); - } - - #[test] - fn test_is_renewing() { - let mut vm = VersionMap::new(None, None); - assert!(!vm.is_renewing()); - vm.mark_renewing("hub1"); - assert!(vm.is_renewing()); - vm.set_error("hub1"); - assert!(!vm.is_renewing()); - } - - #[test] - fn test_dedup_versions_across_hubs() { - let mut vm = VersionMap::new(None, None); - vm.add_single_release("hub1", make_release("1.0.0")); - vm.add_single_release("hub2", make_release("1.0.0")); - // Same version from two hubs → merged under one VersionInfo key - let list = vm.get_version_list(); - assert_eq!(list.len(), 1); - assert_eq!(list[0].wrappers.len(), 2); - } - - #[test] - fn test_regex_filtering() { - let mut vm = VersionMap::new(Some("^v".to_string()), None); - vm.add_single_release("hub1", make_release("v1.2.3")); - let list = vm.get_version_list(); - assert_eq!(list[0].version_info.name, "1.2.3"); - } -} diff --git a/src/rpc.rs b/src/rpc.rs deleted file mode 100644 index 162790c..0000000 --- a/src/rpc.rs +++ /dev/null @@ -1,3 +0,0 @@ -pub mod client; -pub mod data; -pub mod server; diff --git a/src/rpc/client.rs b/src/rpc/client.rs deleted file mode 100644 index 94c6091..0000000 --- a/src/rpc/client.rs +++ /dev/null @@ -1,77 +0,0 @@ -use crate::websdk::repo::data::release::ReleaseData; - -use super::data::*; -use jsonrpsee::core::client::ClientT; -use jsonrpsee::core::client::Error; -use jsonrpsee::http_client::HttpClient; -use jsonrpsee::http_client::HttpClientBuilder; -use std::collections::BTreeMap; - -pub struct Client { - client: HttpClient, -} - -impl Client { - pub fn new(url: impl AsRef) -> Result { - let client = HttpClientBuilder::default().build(url)?; - Ok(Self { client }) - } - - pub async fn check_app_available( - &self, - hub_uuid: &str, - app_data: BTreeMap<&str, &str>, - hub_data: BTreeMap<&str, &str>, - ) -> Result { - let data = RpcAppRequest { - hub_uuid, - app_data, - hub_data, - }; - self.client.request("check_app_available", data).await - } - - pub async fn get_latest_release( - &self, - hub_uuid: &str, - app_data: BTreeMap<&str, &str>, - hub_data: BTreeMap<&str, &str>, - ) -> Result { - let data = RpcAppRequest { - hub_uuid, - app_data, - hub_data, - }; - self.client.request("get_latest_release", data).await - } - - pub async fn get_releases( - &self, - hub_uuid: &str, - app_data: BTreeMap<&str, &str>, - hub_data: BTreeMap<&str, &str>, - ) -> Result, Error> { - let data = RpcAppRequest { - hub_uuid, - app_data, - hub_data, - }; - self.client.request("get_releases", data).await - } - - pub async fn get_download( - &self, - hub_uuid: &str, - app_data: BTreeMap<&str, &str>, - hub_data: BTreeMap<&str, &str>, - asset_index: &[i32], - ) -> Result, Error> { - let data = RpcDownloadInfoRequest { - hub_uuid, - app_data, - hub_data, - asset_index: asset_index.to_vec(), - }; - self.client.request("get_download", data).await - } -} diff --git a/src/rpc/data.rs b/src/rpc/data.rs deleted file mode 100644 index 77f3a88..0000000 --- a/src/rpc/data.rs +++ /dev/null @@ -1,411 +0,0 @@ -use crate::downloader::{DownloadState, TaskInfo}; -use jsonrpsee::core::traits::ToRpcParams; -use serde::{Deserialize, Serialize}; -use serde_json::value::to_raw_value; -use std::collections::{BTreeMap, HashMap}; - -#[derive(Serialize, Deserialize, Debug)] -pub struct RpcInitRequest<'a> { - pub data_path: &'a str, - pub cache_path: &'a str, - pub global_expire_time: u64, -} - -impl ToRpcParams for RpcInitRequest<'_> { - fn to_rpc_params(self) -> Result>, serde_json::Error> { - to_raw_value(&self).map(Some) - } -} - -#[derive(Serialize, Deserialize, Debug)] -pub struct RpcAppRequest<'a> { - pub hub_uuid: &'a str, - pub app_data: BTreeMap<&'a str, &'a str>, - pub hub_data: BTreeMap<&'a str, &'a str>, -} - -impl ToRpcParams for RpcAppRequest<'_> { - fn to_rpc_params(self) -> Result>, serde_json::Error> { - to_raw_value(&self).map(Some) - } -} - -#[derive(Serialize, Deserialize, Debug)] -pub struct RpcCloudConfigRequest<'a> { - pub api_url: &'a str, -} - -impl ToRpcParams for RpcCloudConfigRequest<'_> { - fn to_rpc_params(self) -> Result>, serde_json::Error> { - to_raw_value(&self).map(Some) - } -} - -// ============================================================================ -// Provider Registration RPC Data Structures -// ============================================================================ - -/// Request to register an external provider -#[derive(Serialize, Deserialize, Debug)] -pub struct RpcRegisterProviderRequest<'a> { - pub hub_uuid: &'a str, - pub url: &'a str, -} - -impl ToRpcParams for RpcRegisterProviderRequest<'_> { - fn to_rpc_params(self) -> Result>, serde_json::Error> { - to_raw_value(&self).map(Some) - } -} - -/// Request to get download info for an app -#[derive(Serialize, Deserialize, Debug)] -pub struct RpcDownloadInfoRequest<'a> { - pub hub_uuid: &'a str, - pub app_data: BTreeMap<&'a str, &'a str>, - pub hub_data: BTreeMap<&'a str, &'a str>, - pub asset_index: Vec, -} - -impl ToRpcParams for RpcDownloadInfoRequest<'_> { - fn to_rpc_params(self) -> Result>, serde_json::Error> { - to_raw_value(&self).map(Some) - } -} - -/// Download item data returned by get_download -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct DownloadItemData { - pub name: Option, - pub url: String, - #[serde(default)] - pub headers: Option>, - #[serde(default)] - pub cookies: Option>, -} - -// ============================================================================ -// Downloader RPC Data Structures -// ============================================================================ - -/// Request to submit a download task -#[derive(Serialize, Deserialize, Debug)] -pub struct RpcDownloadRequest<'a> { - pub url: &'a str, - pub dest_path: &'a str, - #[serde(default)] - pub headers: Option>, - #[serde(default)] - pub cookies: Option>, - /// Hub UUID for routing to registered external downloaders. - /// When set, routes to external downloader; when None, uses default HTTP downloader. - #[serde(default)] - pub hub_uuid: Option, -} - -impl ToRpcParams for RpcDownloadRequest<'_> { - fn to_rpc_params(self) -> Result>, serde_json::Error> { - to_raw_value(&self).map(Some) - } -} - -/// Request to submit multiple download tasks -#[derive(Serialize, Deserialize, Debug)] -pub struct RpcDownloadBatchRequest { - pub tasks: Vec, -} - -#[derive(Serialize, Deserialize, Debug)] -pub struct RpcDownloadTask { - pub url: String, - pub dest_path: String, -} - -impl ToRpcParams for RpcDownloadBatchRequest { - fn to_rpc_params(self) -> Result>, serde_json::Error> { - to_raw_value(&self).map(Some) - } -} - -/// Response with task ID -#[derive(Serialize, Deserialize, Debug, Clone)] -pub struct RpcTaskIdResponse { - pub task_id: String, -} - -/// Response with multiple task IDs -#[derive(Serialize, Deserialize, Debug, Clone)] -pub struct RpcTaskIdsResponse { - pub task_ids: Vec, -} - -/// Request to query task status -#[derive(Serialize, Deserialize, Debug)] -pub struct RpcTaskStatusRequest<'a> { - pub task_id: &'a str, -} - -impl ToRpcParams for RpcTaskStatusRequest<'_> { - fn to_rpc_params(self) -> Result>, serde_json::Error> { - to_raw_value(&self).map(Some) - } -} - -/// Request to wait for task state change (long-polling) -#[derive(Serialize, Deserialize, Debug)] -pub struct RpcWaitForChangeRequest<'a> { - pub task_id: &'a str, - pub timeout_seconds: u64, -} - -impl ToRpcParams for RpcWaitForChangeRequest<'_> { - fn to_rpc_params(self) -> Result>, serde_json::Error> { - to_raw_value(&self).map(Some) - } -} - -/// Request to cancel a task -#[derive(Serialize, Deserialize, Debug)] -pub struct RpcCancelTaskRequest<'a> { - pub task_id: &'a str, -} - -impl ToRpcParams for RpcCancelTaskRequest<'_> { - fn to_rpc_params(self) -> Result>, serde_json::Error> { - to_raw_value(&self).map(Some) - } -} - -/// Request to pause a task -#[derive(Serialize, Deserialize, Debug)] -pub struct RpcPauseTaskRequest<'a> { - pub task_id: &'a str, -} - -impl ToRpcParams for RpcPauseTaskRequest<'_> { - fn to_rpc_params(self) -> Result>, serde_json::Error> { - to_raw_value(&self).map(Some) - } -} - -/// Request to resume a task -#[derive(Serialize, Deserialize, Debug)] -pub struct RpcResumeTaskRequest<'a> { - pub task_id: &'a str, -} - -impl ToRpcParams for RpcResumeTaskRequest<'_> { - fn to_rpc_params(self) -> Result>, serde_json::Error> { - to_raw_value(&self).map(Some) - } -} - -/// Request to query tasks by state -#[derive(Serialize, Deserialize, Debug)] -pub struct RpcTasksByStateRequest { - pub state: DownloadState, -} - -impl ToRpcParams for RpcTasksByStateRequest { - fn to_rpc_params(self) -> Result>, serde_json::Error> { - to_raw_value(&self).map(Some) - } -} - -/// Response with task information -pub type RpcTaskInfoResponse = TaskInfo; - -/// Response with multiple tasks -#[derive(Serialize, Deserialize, Debug, Clone)] -pub struct RpcTasksResponse { - pub tasks: Vec, -} - -// ============================================================================ -// Downloader Registration RPC Data Structures -// ============================================================================ - -/// Request to register an external downloader for a hub_uuid -#[derive(Serialize, Deserialize, Debug)] -pub struct RpcRegisterDownloaderRequest<'a> { - pub hub_uuid: &'a str, - pub rpc_url: &'a str, -} - -impl ToRpcParams for RpcRegisterDownloaderRequest<'_> { - fn to_rpc_params(self) -> Result>, serde_json::Error> { - to_raw_value(&self).map(Some) - } -} - -/// Request to unregister an external downloader for a hub_uuid -#[derive(Serialize, Deserialize, Debug)] -pub struct RpcUnregisterDownloaderRequest<'a> { - pub hub_uuid: &'a str, -} - -impl ToRpcParams for RpcUnregisterDownloaderRequest<'_> { - fn to_rpc_params(self) -> Result>, serde_json::Error> { - to_raw_value(&self).map(Some) - } -} - -// ============================================================================ -// Manager RPC Data Structures -// ============================================================================ - -use crate::database::models::app::AppRecord; -use crate::database::models::extra_hub::ExtraHubRecord; -use crate::database::models::hub::HubRecord; -use crate::manager::app_status::AppStatus; - -/// Response wrapping a list of apps with their current status. -#[derive(Serialize, Deserialize, Debug, Clone)] -pub struct AppWithStatus { - pub record: AppRecord, - pub status: AppStatus, -} - -/// Request to save/update an app record. -#[derive(Serialize, Deserialize, Debug)] -pub struct RpcSaveAppRequest { - pub record: AppRecord, -} - -/// Request to delete an app by record id. -#[derive(Serialize, Deserialize, Debug)] -pub struct RpcDeleteAppRequest { - pub record_id: String, -} - -/// Request to get a single app by record id. -#[derive(Serialize, Deserialize, Debug)] -pub struct RpcGetAppRequest { - pub record_id: String, -} - -/// Request to save/update a hub record. -#[derive(Serialize, Deserialize, Debug)] -pub struct RpcSaveHubRequest { - pub record: HubRecord, -} - -/// Request to delete a hub by UUID. -#[derive(Serialize, Deserialize, Debug)] -pub struct RpcDeleteHubRequest { - pub hub_uuid: String, -} - -/// Request to get a hub by UUID. -#[derive(Serialize, Deserialize, Debug)] -pub struct RpcGetHubRequest { - pub hub_uuid: String, -} - -/// Request to set the applications mode for a hub. -#[derive(Serialize, Deserialize, Debug)] -pub struct RpcSetApplicationsModeRequest { - pub hub_uuid: String, - pub enable: bool, -} - -/// Request to ignore/unignore an app in a hub. -#[derive(Serialize, Deserialize, Debug)] -pub struct RpcHubIgnoreAppRequest { - pub hub_uuid: String, - pub app_id: HashMap>, - pub ignore: bool, -} - -/// Request to set virtual (installed) apps list from Android. -#[derive(Serialize, Deserialize, Debug)] -pub struct RpcSetVirtualAppsRequest { - pub apps: Vec, -} - -/// Request to get app status. -#[derive(Serialize, Deserialize, Debug)] -pub struct RpcGetAppStatusRequest { - pub record_id: String, -} - -/// Request to save/update an ExtraHub record. -#[derive(Serialize, Deserialize, Debug)] -pub struct RpcSaveExtraHubRequest { - pub record: ExtraHubRecord, -} - -/// Request to get an ExtraHub by id. -#[derive(Serialize, Deserialize, Debug)] -pub struct RpcGetExtraHubRequest { - pub id: String, -} - -// ============================================================================ -// Android API / Notification Registration RPC Data Structures -// ============================================================================ - -/// Request to register the Kotlin Android API callback URL. -#[derive(Serialize, Deserialize, Debug)] -pub struct RpcRegisterAndroidApiRequest { - pub url: String, -} - -/// Request to register the Kotlin notification callback URL. -#[derive(Serialize, Deserialize, Debug)] -pub struct RpcRegisterNotificationRequest { - pub url: String, -} - -// ============================================================================ -// ExtraApp RPC Data Structures -// ============================================================================ - -use crate::database::models::extra_app::ExtraAppRecord; - -/// Request to get an ExtraApp record by app_id map. -#[derive(Serialize, Deserialize, Debug)] -pub struct RpcGetExtraAppRequest { - pub app_id: HashMap>, -} - -/// Request to save/update an ExtraApp record. -#[derive(Serialize, Deserialize, Debug)] -pub struct RpcSaveExtraAppRequest { - pub record: ExtraAppRecord, -} - -/// Request to delete an ExtraApp by database id. -#[derive(Serialize, Deserialize, Debug)] -pub struct RpcDeleteExtraAppRequest { - pub id: String, -} - -// ============================================================================ -// Cloud Config Manager RPC Data Structures -// ============================================================================ - -/// Request to apply a specific cloud hub/app config by UUID. -#[derive(Serialize, Deserialize, Debug)] -pub struct RpcCloudConfigApplyRequest { - pub uuid: String, -} - -/// Request to initialise the CloudConfigGetter with an API URL. -#[derive(Serialize, Deserialize, Debug)] -pub struct RpcCloudConfigInitRequest { - pub api_url: String, -} - -/// Request to update the auth map for a hub. -#[derive(Serialize, Deserialize, Debug)] -pub struct RpcUpdateHubAuthRequest { - pub hub_uuid: String, - pub auth: HashMap, -} - -impl ToRpcParams for RpcUpdateHubAuthRequest { - fn to_rpc_params(self) -> Result>, serde_json::Error> { - to_raw_value(&self).map(Some) - } -} diff --git a/src/rpc/server.rs b/src/rpc/server.rs deleted file mode 100644 index 6e1e54f..0000000 --- a/src/rpc/server.rs +++ /dev/null @@ -1,1255 +0,0 @@ -use super::data::*; -use crate::cache::init_cache_manager_with_expire; -use crate::core::config::world::{init_world_list, world_list}; -use crate::database::get_db; -use crate::database::models::extra_hub::GLOBAL_HUB_ID; -use crate::downloader::{DownloadConfig, DownloadTaskManager}; -use crate::manager::android_api; -use crate::manager::app_manager::AppManager; -use crate::manager::cloud_config_getter::CloudConfigGetter; -use crate::manager::hub_manager::HubManager; -use crate::manager::notification; -use crate::manager::url_replace::apply_url_replace; -use crate::websdk::cloud_rules::cloud_rules_manager::CloudRules; -use crate::websdk::repo::api; -use jsonrpsee::server::{RpcModule, Server, ServerConfig, ServerHandle}; -use jsonrpsee::types::{ErrorCode, ErrorObjectOwned}; -use once_cell::sync::OnceCell; -use std::net::SocketAddr; -use std::path::Path; -use std::sync::atomic::{AtomicBool, Ordering}; -use std::sync::Arc; -use std::time::Duration; -use tokio::sync::RwLock; - -/// Global manager state initialised on first `init` RPC call. -static APP_MANAGER: OnceCell>> = OnceCell::new(); -static HUB_MANAGER: OnceCell>> = OnceCell::new(); -static CLOUD_CONFIG_GETTER: OnceCell>> = OnceCell::new(); - -fn get_app_manager() -> Option>> { - APP_MANAGER.get().cloned() -} - -fn get_hub_manager() -> Option>> { - HUB_MANAGER.get().cloned() -} - -fn get_cloud_config_getter() -> Option>> { - CLOUD_CONFIG_GETTER.get().cloned() -} - -fn manager_not_init_err() -> ErrorObjectOwned { - ErrorObjectOwned::owned( - ErrorCode::InternalError.code(), - "Manager not initialized. Call init first.", - None::, - ) -} - -fn map_manager_err(e: impl std::fmt::Display) -> ErrorObjectOwned { - ErrorObjectOwned::owned( - ErrorCode::InternalError.code(), - e.to_string(), - None::, - ) -} - -// Default 2GB size limit for WebSocket messages -// Can be overridden at runtime by setting GETTER_WS_MAX_MESSAGE_SIZE environment variable -// Example: GETTER_WS_MAX_MESSAGE_SIZE=1073741824 ./getter (for 1GB) -const DEFAULT_MAX_SIZE: u32 = 2 * 1024 * 1024 * 1024; // 2GB - -fn get_max_message_size() -> u32 { - // Allow runtime configuration via environment variable - match std::env::var("GETTER_WS_MAX_MESSAGE_SIZE") { - Ok(size_str) => size_str.parse().unwrap_or(DEFAULT_MAX_SIZE), - Err(_) => DEFAULT_MAX_SIZE, - } -} - -pub async fn run_server( - addr: &str, - is_running: Arc, -) -> Result<(String, ServerHandle), Box> { - let addr = if addr.is_empty() { "127.0.0.1:0" } else { addr }; - let max_size = get_max_message_size(); - let config = ServerConfig::builder() - .max_request_body_size(max_size) - .max_response_body_size(max_size) - .build(); - let server = Server::builder() - .set_config(config) - .build(addr.parse::()?) - .await?; - let mut module = RpcModule::new(()); - // Register the shutdown method - let run_flag = is_running.clone(); - module.register_async_method("shutdown", move |_, _, _| { - let flag = run_flag.clone(); - async move { - flag.store(false, Ordering::SeqCst); - } - })?; - module.register_method("ping", |_, _, _| "pong")?; - module.register_async_method("init", |params, _, _| async move { - let request = params.parse::()?; - let data_dir = Path::new(request.data_path); - let cache_dir = Path::new(request.cache_path); - // Initialize world list, cache, and database. - let world_list_path = data_dir.join(world_list::WORLD_CONFIG_LIST_NAME); - init_world_list(&world_list_path).await.map_err(|e| { - ErrorObjectOwned::owned( - ErrorCode::InternalError.code(), - "Internal error", - Some(e.to_string()), - ) - })?; - let local_cache_path = cache_dir.join("local_cache"); - init_cache_manager_with_expire(local_cache_path.as_path(), request.global_expire_time) - .await; - crate::database::init_db(data_dir).map_err(|e| { - ErrorObjectOwned::owned( - ErrorCode::InternalError.code(), - "Internal error", - Some(e.to_string()), - ) - })?; - - // Initialize managers (idempotent: only on first call) - if APP_MANAGER.get().is_none() { - let hub_mgr = HubManager::load().map_err(map_manager_err)?; - let app_mgr = AppManager::load().map_err(map_manager_err)?; - let _ = HUB_MANAGER.set(Arc::new(RwLock::new(hub_mgr))); - let _ = APP_MANAGER.set(Arc::new(RwLock::new(app_mgr))); - } - - Ok::(true) - })?; - module.register_async_method( - "check_app_available", - |params, _context, _extensions| async move { - let request = params.parse::()?; - let result = - api::check_app_available(request.hub_uuid, &request.app_data, &request.hub_data) - .await - .unwrap_or(false); - Ok::(result) - }, - )?; - module.register_async_method( - "get_latest_release", - |params, _context, _extensions| async move { - let request = params.parse::()?; - if let Some(result) = - api::get_latest_release(request.hub_uuid, &request.app_data, &request.hub_data) - .await - { - Ok(result) - } else { - Err(ErrorObjectOwned::owned( - -32001, - "No release found", - None::, - )) - } - }, - )?; - module.register_async_method("get_releases", |params, _context, _extensions| async move { - let request = params.parse::()?; - if let Some(result) = - api::get_releases(request.hub_uuid, &request.app_data, &request.hub_data).await - { - Ok(result) - } else { - Err(ErrorObjectOwned::owned( - -32001, - "No releases found", - None::, - )) - } - })?; - - // register_provider: Dynamically register an external provider (e.g., Kotlin hub via HTTP JSON-RPC) - module.register_async_method( - "register_provider", - |params, _context, _extensions| async move { - let request = params.parse::()?; - api::add_outside_provider(request.hub_uuid, request.url); - Ok::(true) - }, - )?; - - // get_download: Get download info for an app's asset. - // After retrieving download URLs from the provider, applies URL replacement - // rules from ExtraHub configs (GLOBAL first, then hub-specific), mirroring - // Kotlin's URLReplace.replaceURL() in the download pipeline. - module.register_async_method("get_download", |params, _context, _extensions| async move { - let request = params.parse::()?; - let mut items = api::get_download( - request.hub_uuid, - &request.app_data, - &request.hub_data, - &request.asset_index, - ) - .await - .ok_or_else(|| ErrorObjectOwned::owned(-32001, "No download info found", None::))?; - - // Load URL-replace rules from ExtraHub configs. - // Priority: hub-specific rule overrides GLOBAL rule. - let db = get_db(); - let global_extra = db.find_extra_hub(GLOBAL_HUB_ID).unwrap_or(None); - let hub_extra = db.find_extra_hub(request.hub_uuid).unwrap_or(None); - - // Apply rules to every download URL in the result. - for item in &mut items { - // Apply GLOBAL rule first (lower priority) - if let Some(ref g) = global_extra { - item.url = apply_url_replace( - &item.url, - g.url_replace_search.as_deref(), - g.url_replace_string.as_deref(), - ); - } - // Apply hub-specific rule second (higher priority, may override) - if let Some(ref h) = hub_extra { - item.url = apply_url_replace( - &item.url, - h.url_replace_search.as_deref(), - h.url_replace_string.as_deref(), - ); - } - } - - Ok::, ErrorObjectOwned>(items) - })?; - - module.register_async_method( - "get_cloud_config", - |params, _context, _extensions| async move { - if let Ok(request) = params.parse::() { - let mut cloud_rules = CloudRules::new(request.api_url); - if let Err(e) = cloud_rules.renew().await { - return Err(ErrorObjectOwned::owned( - ErrorCode::InternalError.code(), - "Download cloud config failed", - Some(e.to_string()), - )); - } - Ok(cloud_rules.get_config_list().to_owned()) - } else { - Err(ErrorObjectOwned::owned( - ErrorCode::ParseError.code(), - "Parse params error", - Some(params.as_str().unwrap_or("None").to_string()), - )) - } - }, - )?; - - // ======================================================================== - // Downloader RPC Methods - // ======================================================================== - - // Create download task manager with HubDispatchDownloader - let download_config = DownloadConfig::from_env(); - let http_downloader = crate::downloader::create_downloader(&download_config); - let dispatcher = crate::downloader::HubDispatchDownloader::new(http_downloader); - - // Clone dispatcher for task manager (HubDispatchDownloader is cheap to clone via Arc internally) - let task_manager = Arc::new(DownloadTaskManager::new(Box::new(dispatcher.clone()))); - - // download_submit: Submit a single download task - let manager_clone = task_manager.clone(); - module.register_async_method("download_submit", move |params, _context, _extensions| { - let manager = manager_clone.clone(); - async move { - let request = params.parse::()?; - match manager.submit_task_with_options( - request.url, - request.dest_path, - request.headers, - request.cookies, - request.hub_uuid, - ) { - Ok(task_id) => Ok(RpcTaskIdResponse { task_id }), - Err(e) => Err(ErrorObjectOwned::owned( - ErrorCode::InternalError.code(), - "Failed to submit download task", - Some(e.message), - )), - } - } - })?; - - // download_submit_batch: Submit multiple download tasks - let manager_clone = task_manager.clone(); - module.register_async_method( - "download_submit_batch", - move |params, _context, _extensions| { - let manager = manager_clone.clone(); - async move { - let request = params.parse::()?; - let tasks: Vec<(String, String)> = request - .tasks - .into_iter() - .map(|t| (t.url, t.dest_path)) - .collect(); - - match manager.submit_batch(tasks) { - Ok(task_ids) => Ok(RpcTaskIdsResponse { task_ids }), - Err(e) => Err(ErrorObjectOwned::owned( - ErrorCode::InternalError.code(), - "Failed to submit batch download tasks", - Some(e.message), - )), - } - } - }, - )?; - - // download_get_status: Get status of a download task - let manager_clone = task_manager.clone(); - module.register_async_method( - "download_get_status", - move |params, _context, _extensions| { - let manager = manager_clone.clone(); - async move { - let request = params.parse::()?; - match manager.get_task(request.task_id) { - Ok(task_info) => Ok(task_info), - Err(e) => Err(ErrorObjectOwned::owned( - ErrorCode::InvalidParams.code(), - "Task not found", - Some(e.message), - )), - } - } - }, - )?; - - // download_wait_for_change: Long-polling for task state change - let manager_clone = task_manager.clone(); - module.register_async_method( - "download_wait_for_change", - move |params, _context, _extensions| { - let manager = manager_clone.clone(); - async move { - let request = params.parse::()?; - let timeout = Duration::from_secs(request.timeout_seconds); - - match manager.wait_for_change(request.task_id, timeout).await { - Ok(task_info) => Ok(task_info), - Err(e) => Err(ErrorObjectOwned::owned( - ErrorCode::InvalidParams.code(), - "Failed to wait for task change", - Some(e.message), - )), - } - } - }, - )?; - - // download_cancel: Cancel a download task - let manager_clone = task_manager.clone(); - module.register_async_method("download_cancel", move |params, _context, _extensions| { - let manager = manager_clone.clone(); - async move { - let request = params.parse::()?; - match manager.cancel_task(request.task_id) { - Ok(_) => Ok(true), - Err(e) => Err(ErrorObjectOwned::owned( - ErrorCode::InternalError.code(), - "Failed to cancel task", - Some(e.message), - )), - } - } - })?; - - // download_pause: Pause a download task - let manager_clone = task_manager.clone(); - module.register_async_method("download_pause", move |params, _context, _extensions| { - let manager = manager_clone.clone(); - async move { - let request = params.parse::()?; - match manager.pause_task(request.task_id).await { - Ok(_) => Ok(true), - Err(e) => Err(ErrorObjectOwned::owned( - ErrorCode::InternalError.code(), - "Failed to pause task", - Some(e.message), - )), - } - } - })?; - - // download_resume: Resume a paused download task - let manager_clone = task_manager.clone(); - module.register_async_method("download_resume", move |params, _context, _extensions| { - let manager = manager_clone.clone(); - async move { - let request = params.parse::()?; - match manager.resume_task(request.task_id).await { - Ok(_) => Ok(true), - Err(e) => Err(ErrorObjectOwned::owned( - ErrorCode::InternalError.code(), - "Failed to resume task", - Some(e.message), - )), - } - } - })?; - - // download_get_capabilities: Get downloader capabilities - let manager_clone = task_manager.clone(); - module.register_method( - "download_get_capabilities", - move |_, _context, _extensions| { - let caps = manager_clone.get_capabilities(); - Ok::<_, ErrorObjectOwned>(caps.clone()) - }, - )?; - - // download_get_all_tasks: Get all tasks - let manager_clone = task_manager.clone(); - module.register_async_method("download_get_all_tasks", move |_, _context, _extensions| { - let manager = manager_clone.clone(); - async move { - Ok::(RpcTasksResponse { - tasks: manager.get_all_tasks(), - }) - } - })?; - - // download_get_active_tasks: Get active tasks - let manager_clone = task_manager.clone(); - module.register_async_method( - "download_get_active_tasks", - move |_, _context, _extensions| { - let manager = manager_clone.clone(); - async move { - Ok::(RpcTasksResponse { - tasks: manager.get_active_tasks(), - }) - } - }, - )?; - - // download_get_tasks_by_state: Get tasks by state - let manager_clone = task_manager.clone(); - module.register_async_method( - "download_get_tasks_by_state", - move |params, _context, _extensions| { - let manager = manager_clone.clone(); - async move { - let request = params.parse::()?; - Ok::(RpcTasksResponse { - tasks: manager.get_tasks_by_state(request.state), - }) - } - }, - )?; - - // register_downloader: Register an external downloader for a hub_uuid - let dispatcher_clone = dispatcher.clone(); - module.register_async_method( - "register_downloader", - move |params, _context, _extensions| { - let dispatcher = dispatcher_clone.clone(); - async move { - let request = params.parse::()?; - let external_downloader = Box::new(crate::downloader::ExternalRpcDownloader::new( - request.rpc_url.to_string(), - )); - dispatcher.register(request.hub_uuid, external_downloader); - Ok::(true) - } - }, - )?; - - // unregister_downloader: Unregister an external downloader for a hub_uuid - let dispatcher_clone = dispatcher.clone(); - module.register_async_method( - "unregister_downloader", - move |params, _context, _extensions| { - let dispatcher = dispatcher_clone.clone(); - async move { - let request = params.parse::()?; - dispatcher.unregister(request.hub_uuid); - Ok::(true) - } - }, - )?; - - // ======================================================================== - // App Manager RPC Methods - // ======================================================================== - - // manager_get_apps: Get all saved apps - module.register_async_method("manager_get_apps", |_, _, _| async move { - let mgr = get_app_manager().ok_or_else(manager_not_init_err)?; - let apps = mgr.read().await.get_saved_apps().await; - Ok::, ErrorObjectOwned>(apps) - })?; - - // manager_save_app: Insert or update an app record - module.register_async_method("manager_save_app", |params, _, _| async move { - let request = params.parse::()?; - let mgr = get_app_manager().ok_or_else(manager_not_init_err)?; - let saved = mgr - .write() - .await - .save_app(request.record) - .await - .map_err(map_manager_err)?; - Ok::(saved) - })?; - - // manager_delete_app: Delete an app by record id - module.register_async_method("manager_delete_app", |params, _, _| async move { - let request = params.parse::()?; - let mgr = get_app_manager().ok_or_else(manager_not_init_err)?; - let deleted = mgr - .write() - .await - .remove_app(&request.record_id) - .await - .map_err(map_manager_err)?; - Ok::(deleted) - })?; - - // manager_get_app_status: Get AppStatus for a specific app - module.register_async_method("manager_get_app_status", |params, _, _| async move { - let request = params.parse::()?; - let mgr = get_app_manager().ok_or_else(manager_not_init_err)?; - let status = mgr.write().await.get_app_status(&request.record_id).await; - Ok::(status) - })?; - - // manager_set_virtual_apps: Set installed (virtual) apps from Android - module.register_async_method("manager_set_virtual_apps", |params, _, _| async move { - let request = params.parse::()?; - let mgr = get_app_manager().ok_or_else(manager_not_init_err)?; - mgr.read().await.set_virtual_apps(request.apps).await; - Ok::(true) - })?; - - // manager_renew_all: Trigger a full update check for all apps - module.register_async_method("manager_renew_all", |_, _, _| async move { - let app_mgr = get_app_manager().ok_or_else(manager_not_init_err)?; - let hub_mgr = get_hub_manager().ok_or_else(manager_not_init_err)?; - let hubs = hub_mgr.read().await.get_hub_list().await; - app_mgr.read().await.renew_all(&hubs, None).await; - Ok::(true) - })?; - - // manager_check_invalid_applications: Return record IDs of apps whose configured - // hub UUIDs are all unknown (no valid hub found). Mirrors Kotlin's - // AppManager.check_invalid_applications logic. - module.register_async_method("manager_check_invalid_applications", |_, _, _| async move { - let app_mgr = get_app_manager().ok_or_else(manager_not_init_err)?; - let hub_mgr = get_hub_manager().ok_or_else(manager_not_init_err)?; - let hubs = hub_mgr.read().await.get_hub_list().await; - let known_uuids: Vec = hubs.into_iter().map(|h| h.uuid).collect(); - let mgr = app_mgr.read().await; - let invalid_ids = mgr.check_invalid_applications(&known_uuids).await; - // Notify Kotlin UI about each invalid app so it can update status. - for record_id in &invalid_ids { - if let Some(app) = mgr.get_app(record_id).await { - notification::notify_if_registered(notification::ManagerEvent::AppStatusChanged { - record_id: record_id.clone(), - app_id: app.app_id.clone(), - old_status: crate::manager::app_status::AppStatus::AppLatest, - new_status: crate::manager::app_status::AppStatus::AppInactive, - }) - .await; - } - } - Ok::, ErrorObjectOwned>(invalid_ids) - })?; - - // ======================================================================== - // Hub Manager RPC Methods - // ======================================================================== - - // manager_get_hubs: Get all hubs - module.register_async_method("manager_get_hubs", |_, _, _| async move { - let mgr = get_hub_manager().ok_or_else(manager_not_init_err)?; - let hubs = mgr.read().await.get_hub_list().await; - Ok::, ErrorObjectOwned>(hubs) - })?; - - // manager_save_hub: Insert or update a hub - module.register_async_method("manager_save_hub", |params, _, _| async move { - let request = params.parse::()?; - let mgr = get_hub_manager().ok_or_else(manager_not_init_err)?; - mgr.write() - .await - .upsert_hub(request.record) - .await - .map_err(map_manager_err)?; - Ok::(true) - })?; - - // manager_update_hub_auth: Replace the auth map for a hub and persist. - module.register_async_method("manager_update_hub_auth", |params, _, _| async move { - let request = params.parse::()?; - let mgr = get_hub_manager().ok_or_else(manager_not_init_err)?; - let updated = mgr - .read() - .await - .update_auth(&request.hub_uuid, request.auth) - .await - .map_err(map_manager_err)?; - Ok::(updated) - })?; - - // manager_delete_hub: Delete a hub by UUID - module.register_async_method("manager_delete_hub", |params, _, _| async move { - let request = params.parse::()?; - let mgr = get_hub_manager().ok_or_else(manager_not_init_err)?; - let deleted = mgr - .write() - .await - .remove_hub(&request.hub_uuid) - .await - .map_err(map_manager_err)?; - Ok::(deleted) - })?; - - // manager_hub_ignore_app: Add or remove an app from a hub's ignore list - module.register_async_method("manager_hub_ignore_app", |params, _, _| async move { - let request = params.parse::()?; - let mgr = get_hub_manager().ok_or_else(manager_not_init_err)?; - let mut guard = mgr.write().await; - let mut hub = guard.get_hub(&request.hub_uuid).await.ok_or_else(|| { - ErrorObjectOwned::owned( - ErrorCode::InvalidParams.code(), - "Hub not found", - None::, - ) - })?; - if request.ignore { - if !hub.user_ignore_app_id_list.contains(&request.app_id) { - hub.user_ignore_app_id_list.push(request.app_id); - } - } else { - hub.user_ignore_app_id_list - .retain(|id| id != &request.app_id); - } - guard.upsert_hub(hub).await.map_err(map_manager_err)?; - Ok::(true) - })?; - - // manager_set_applications_mode: Enable/disable auto app discovery for a hub - module.register_async_method("manager_set_applications_mode", |params, _, _| async move { - let request = params.parse::()?; - let mgr = get_hub_manager().ok_or_else(manager_not_init_err)?; - let mut guard = mgr.write().await; - let mut hub = guard.get_hub(&request.hub_uuid).await.ok_or_else(|| { - ErrorObjectOwned::owned( - ErrorCode::InvalidParams.code(), - "Hub not found", - None::, - ) - })?; - hub.applications_mode = if request.enable { 1 } else { 0 }; - guard.upsert_hub(hub).await.map_err(map_manager_err)?; - Ok::(true) - })?; - - // ======================================================================== - // ExtraHub RPC Methods - // ======================================================================== - - // manager_get_extra_hubs: Get all extra hub configs - module.register_async_method("manager_get_extra_hubs", |_, _, _| async move { - let extra_hubs = crate::database::get_db() - .load_extra_hubs() - .map_err(map_manager_err)?; - Ok::, ErrorObjectOwned>(extra_hubs) - })?; - - // manager_save_extra_hub: Insert or update an extra hub config - module.register_async_method("manager_save_extra_hub", |params, _, _| async move { - let request = params.parse::()?; - crate::database::get_db() - .upsert_extra_hub(&request.record) - .map_err(map_manager_err)?; - Ok::(true) - })?; - - // manager_delete_extra_hub: Delete an extra hub by id - module.register_async_method("manager_delete_extra_hub", |params, _, _| async move { - let request = params.parse::()?; - let deleted = crate::database::get_db() - .delete_extra_hub(&request.id) - .map_err(map_manager_err)?; - Ok::(deleted) - })?; - - // ======================================================================== - // Android API / Notification Registration RPC Methods - // ======================================================================== - - // register_android_api: Register Kotlin's Android API callback URL - module.register_async_method("register_android_api", |params, _, _| async move { - let request = params.parse::()?; - android_api::set_android_api(request.url); - Ok::(true) - })?; - - // register_notification: Register Kotlin's notification callback URL - module.register_async_method("register_notification", |params, _, _| async move { - let request = params.parse::()?; - notification::set_notification(request.url); - Ok::(true) - })?; - - // ======================================================================== - // ExtraApp RPC Methods - // ======================================================================== - - // manager_get_extra_app_by_app_id: Get ExtraApp record by app_id map - module.register_async_method( - "manager_get_extra_app_by_app_id", - |params, _, _| async move { - let request = params.parse::()?; - let record = crate::database::get_db() - .get_extra_app_by_app_id(&request.app_id) - .map_err(map_manager_err)?; - Ok::, ErrorObjectOwned>( - record, - ) - }, - )?; - - // manager_save_extra_app: Insert or update an ExtraApp record - module.register_async_method("manager_save_extra_app", |params, _, _| async move { - let request = params.parse::()?; - crate::database::get_db() - .upsert_extra_app(&request.record) - .map_err(map_manager_err)?; - Ok::(true) - })?; - - // manager_delete_extra_app: Delete an ExtraApp by database id - module.register_async_method("manager_delete_extra_app", |params, _, _| async move { - let request = params.parse::()?; - let deleted = crate::database::get_db() - .delete_extra_app(&request.id) - .map_err(map_manager_err)?; - Ok::(deleted) - })?; - - // ======================================================================== - // Cloud Config Manager RPC Methods - // ======================================================================== - - // cloud_config_init: Initialise or re-initialise the CloudConfigGetter with an API URL - module.register_async_method("cloud_config_init", |params, _, _| async move { - let request = params.parse::()?; - let getter = CloudConfigGetter::new(request.api_url); - let _ = CLOUD_CONFIG_GETTER.set(Arc::new(RwLock::new(getter))); - Ok::(true) - })?; - - // cloud_config_renew: Download and cache the latest cloud config - module.register_async_method("cloud_config_renew", |_, _, _| async move { - let getter = get_cloud_config_getter().ok_or_else(|| { - ErrorObjectOwned::owned( - ErrorCode::InternalError.code(), - "CloudConfigGetter not initialised. Call cloud_config_init first.", - None::, - ) - })?; - getter.read().await.renew().await.map_err(map_manager_err)?; - Ok::(true) - })?; - - // cloud_config_get_app_list: Return all available app configs from cache - module.register_async_method("cloud_config_get_app_list", |_, _, _| async move { - let getter = get_cloud_config_getter().ok_or_else(|| { - ErrorObjectOwned::owned( - ErrorCode::InternalError.code(), - "CloudConfigGetter not initialised.", - None::, - ) - })?; - let list = getter.read().await.app_config_list().await; - Ok::, ErrorObjectOwned>(list) - })?; - - // cloud_config_get_hub_list: Return all available hub configs from cache - module.register_async_method("cloud_config_get_hub_list", |_, _, _| async move { - let getter = get_cloud_config_getter().ok_or_else(|| { - ErrorObjectOwned::owned( - ErrorCode::InternalError.code(), - "CloudConfigGetter not initialised.", - None::, - ) - })?; - let list = getter.read().await.hub_config_list().await; - Ok::, ErrorObjectOwned>(list) - })?; - - // cloud_config_apply_app: Apply a cloud app config by UUID - module.register_async_method("cloud_config_apply_app", |params, _, _| async move { - let request = params.parse::()?; - let getter = get_cloud_config_getter().ok_or_else(|| { - ErrorObjectOwned::owned( - ErrorCode::InternalError.code(), - "CloudConfigGetter not initialised.", - None::, - ) - })?; - let app_mgr = get_app_manager().ok_or_else(manager_not_init_err)?; - let hub_mgr = get_hub_manager().ok_or_else(manager_not_init_err)?; - getter - .read() - .await - .apply_app_config( - &request.uuid, - &mut *app_mgr.write().await, - &mut *hub_mgr.write().await, - ) - .await - .map_err(map_manager_err)?; - Ok::(true) - })?; - - // cloud_config_apply_hub: Apply a cloud hub config by UUID - module.register_async_method("cloud_config_apply_hub", |params, _, _| async move { - let request = params.parse::()?; - let getter = get_cloud_config_getter().ok_or_else(|| { - ErrorObjectOwned::owned( - ErrorCode::InternalError.code(), - "CloudConfigGetter not initialised.", - None::, - ) - })?; - let hub_mgr = get_hub_manager().ok_or_else(manager_not_init_err)?; - getter - .read() - .await - .apply_hub_config(&request.uuid, &mut *hub_mgr.write().await) - .await - .map_err(map_manager_err)?; - Ok::(true) - })?; - - // cloud_config_renew_all: Bulk-update all installed apps/hubs from cloud - module.register_async_method("cloud_config_renew_all", |_, _, _| async move { - let getter = get_cloud_config_getter().ok_or_else(|| { - ErrorObjectOwned::owned( - ErrorCode::InternalError.code(), - "CloudConfigGetter not initialised.", - None::, - ) - })?; - let app_mgr = get_app_manager().ok_or_else(manager_not_init_err)?; - let hub_mgr = get_hub_manager().ok_or_else(manager_not_init_err)?; - getter - .read() - .await - .renew_all_from_cloud(&mut *app_mgr.write().await, &mut *hub_mgr.write().await) - .await - .map_err(map_manager_err)?; - Ok::(true) - })?; - - let addr = server.local_addr()?; - let handle = server.start(module); - tokio::spawn(handle.clone().stopped()); - Ok((format!("http://{}", addr), handle)) -} - -#[allow(dead_code)] -pub async fn run_server_hanging( - addr: &str, - callback: impl Fn(&str) -> Result>, -) -> Result> { - let is_running = Arc::new(AtomicBool::new(true)); - let (url, handle) = match run_server(addr, is_running.clone()).await { - Ok((url, handle)) => (url, handle), - Err(e) => { - eprintln!("Failed to start server: {}", e); - return Err(e); - } - }; - let result = callback(&url)?; - while is_running.load(Ordering::SeqCst) { - tokio::time::sleep(Duration::from_secs(1)).await; - } - handle.stop()?; - Ok(result) -} - -#[cfg(test)] -mod tests { - use crate::rpc::client::Client; - use crate::websdk::repo::provider::github; - use crate::websdk::{ - cloud_rules::data::config_list::ConfigList, repo::data::release::ReleaseData, - }; - - use super::*; - use jsonrpsee::{core::client::ClientT, http_client::HttpClientBuilder, rpc_params}; - use mockito::Server; - use std::collections::BTreeMap; - use std::fs; - use tokio::time::timeout; - - #[tokio::test] - async fn test_server_start() { - let (url, handle) = run_server("", Arc::new(AtomicBool::new(true))) - .await - .unwrap(); - println!("Server started at {}", url); - assert!(url.starts_with("http://")); - assert!(url.split(":").last().unwrap().parse::().unwrap() > 0); - handle.stop().unwrap(); - let port = 33333; - let addr = format!("127.0.0.1:{}", port); - let (url, handle) = run_server(&addr, Arc::new(AtomicBool::new(true))) - .await - .unwrap(); - println!("Server started at {}", url); - assert!(url.starts_with("http://")); - assert!(url.split(":").last().unwrap().parse::().unwrap() == port); - handle.stop().unwrap(); - } - - #[tokio::test] - async fn test_ping() { - let (url, handle) = run_server("", Arc::new(AtomicBool::new(true))) - .await - .unwrap(); - println!("Server started at {}", url); - let client = HttpClientBuilder::default().build(url).unwrap(); - let response: Result = client.request("ping", rpc_params![]).await; - assert_eq!(response.unwrap(), "pong"); - handle.stop().unwrap(); - } - - #[tokio::test] - async fn test_init() { - let mut server = Server::new_async().await; - let _m = server - .mock("GET", "/DUpdateSystem/UpgradeAll") - .with_status(200) - .create_async() - .await; - - let (url, handle) = run_server("", Arc::new(AtomicBool::new(true))) - .await - .unwrap(); - println!("Server started at {}", url); - let client = HttpClientBuilder::default().build(url).unwrap(); - let temp_dir = tempfile::tempdir().unwrap(); - let temp_dir_path = temp_dir.path().to_str().unwrap(); - let params = RpcInitRequest { - data_path: &format!("{}/data", temp_dir_path), - cache_path: &format!("{}/cache", temp_dir_path), - global_expire_time: 3600, - }; - println!("{:?}", params); - let response: Result = client.request("init", params).await; - assert!(response.unwrap()); - handle.stop().unwrap(); - } - #[tokio::test] - async fn test_check_app_available() { - let mut server = Server::new_async().await; - let _m = server - .mock("GET", "/DUpdateSystem/UpgradeAll") - .with_status(200) - .create_async() - .await; - - let id_map = BTreeMap::from([("owner", "DUpdateSystem"), ("repo", "UpgradeAll")]); - let proxy_url = format!("{} -> {}", github::GITHUB_API_URL, server.url()); - let hub_data = BTreeMap::from([("reverse_proxy", proxy_url.as_str())]); - - let (url, handle) = run_server("", Arc::new(AtomicBool::new(true))) - .await - .unwrap(); - println!("Server started at {}", url); - let params = RpcAppRequest { - hub_uuid: "fd9b2602-62c5-4d55-bd1e-0d6537714ca0", - app_data: id_map, - hub_data, - }; - println!("{:?}", params); - let client = Client::new(url).unwrap(); - let response: Result = client - .check_app_available(params.hub_uuid, params.app_data, params.hub_data) - .await; - assert!(response.unwrap()); - handle.stop().unwrap(); - } - - #[tokio::test] - async fn test_get_latest_release() { - let body = fs::read_to_string("tests/files/web/github_api_release.json").unwrap(); - let mut server = Server::new_async().await; - let _m = server - .mock("GET", "/repos/DUpdateSystem/UpgradeAll/releases") - .with_status(200) - .with_body(body) - .create(); - - let id_map = BTreeMap::from([("owner", "DUpdateSystem"), ("repo", "UpgradeAll")]); - let proxy_url = format!("{} -> {}", github::GITHUB_API_URL, server.url()); - let hub_data = BTreeMap::from([("reverse_proxy", proxy_url.as_str())]); - - let (url, handle) = run_server("", Arc::new(AtomicBool::new(true))) - .await - .unwrap(); - println!("Server started at {}", url); - let params = RpcAppRequest { - hub_uuid: "fd9b2602-62c5-4d55-bd1e-0d6537714ca0", - app_data: id_map, - hub_data, - }; - println!("{:?}", params); - let client = Client::new(url).unwrap(); - let response: Result = client - .get_latest_release(params.hub_uuid, params.app_data, params.hub_data) - .await; - let release = response.unwrap(); - assert!(!release.version_number.is_empty()); - handle.stop().unwrap(); - } - - #[tokio::test] - async fn test_get_releases() { - let body = fs::read_to_string("tests/files/web/github_api_release.json").unwrap(); - let mut server = Server::new_async().await; - let _m = server - .mock("GET", "/repos/DUpdateSystem/UpgradeAll/releases") - .with_status(200) - .with_body(body) - .create(); - - let id_map = BTreeMap::from([("owner", "DUpdateSystem"), ("repo", "UpgradeAll")]); - let proxy_url = format!("{} -> {}", github::GITHUB_API_URL, server.url()); - let hub_data = BTreeMap::from([("reverse_proxy", proxy_url.as_str())]); - - let (url, handle) = run_server("", Arc::new(AtomicBool::new(true))) - .await - .unwrap(); - println!("Server started at {}", url); - let params = RpcAppRequest { - hub_uuid: "fd9b2602-62c5-4d55-bd1e-0d6537714ca0", - app_data: id_map, - hub_data, - }; - println!("{:?}", params); - let client = Client::new(url).unwrap(); - let response: Result, _> = client - .get_releases(params.hub_uuid, params.app_data, params.hub_data) - .await; - let releases = response.unwrap(); - assert!(!releases.is_empty()); - handle.stop().unwrap(); - } - - #[tokio::test] - async fn test_run_server_hanging() { - let addr = "127.0.0.1:33334"; - let server_task = tokio::spawn(async move { - // This should run the server and wait for the shutdown command - run_server_hanging(addr, |url| { - println!("Server started at {}", url); - Ok(()) - }) - .await - .expect("Server failed to run"); - }); - - // Allow some time for the server to start up - tokio::time::sleep(Duration::from_millis(500)).await; - - // The callback should print the URL, but since we cannot capture that output easily in a test, - // we assume the server starts correctly if no error happens till now. - // Here, manually create a client and send a shutdown request - let client = HttpClientBuilder::default() - .build(format!("http://{}", addr)) - .expect("Failed to build client"); - - let response: Result<(), _> = client.request("shutdown", rpc_params![]).await; - assert!(response.is_ok(), "Failed to shutdown server"); - - // Allow some time for the server to shut down - tokio::time::sleep(Duration::from_millis(500)).await; - - // Check if the shutdown was successful by confirming the server task is done - if timeout(Duration::from_secs(1), server_task).await.is_err() { - panic!("The server did not shut down within the expected time"); - } - - let response: Result<(), _> = client.request("shutdown", rpc_params![]).await; - assert!(response.is_err(), "Server should not be running"); - } - - #[tokio::test] - async fn test_get_cloud_config() { - let body = fs::read_to_string("tests/files/web/cloud_config.json").unwrap(); - let mut server = Server::new_async().await; - let _m = server - .mock("GET", "/cloud_config.json") - .with_status(200) - .with_body(body) - .create(); - - let (url, handle) = run_server("", Arc::new(AtomicBool::new(true))) - .await - .unwrap(); - println!("Server started at {}", url); - let client = HttpClientBuilder::default().build(url).unwrap(); - let url = format!("{}/cloud_config.json", server.url()); - let params = RpcCloudConfigRequest { api_url: &url }; - println!("{:?}", params); - let response: Result = client.request("get_cloud_config", params).await; - let config = response.unwrap(); - assert!(!config.app_config_list.is_empty()); - assert!(!config.hub_config_list.is_empty()); - handle.stop().unwrap(); - } - - // ======================================================================== - // WebSocket and message size tests - // ======================================================================== - - use jsonrpsee::ws_client::WsClientBuilder; - use serial_test::serial; - - /// Generate a random ASCII string of given byte length (not compressible). - /// Uses printable ASCII range (0x21-0x7e) to avoid JSON escape overhead. - fn generate_random_string(size: usize) -> String { - use rand::RngExt; - let mut rng = rand::rng(); - let mut buf = vec![0u8; size]; - rng.fill(&mut buf[..]); - // Map each byte to printable ASCII (0x21..=0x7e, 94 chars), avoid '"' and '\\' - for b in buf.iter_mut() { - *b = match (*b % 92) + 0x21 { - b'"' => b'a', - b'\\' => b'b', - v => v, - }; - } - // SAFETY: all bytes are valid ASCII - unsafe { String::from_utf8_unchecked(buf) } - } - - /// Helper: start a minimal RPC server with "echo_data" method for size testing. - async fn start_test_server_with_config(max_size: u32) -> (String, ServerHandle) { - let config = ServerConfig::builder() - .max_request_body_size(max_size) - .max_response_body_size(max_size) - .build(); - let server = jsonrpsee::server::Server::builder() - .set_config(config) - .build("127.0.0.1:0".parse::().unwrap()) - .await - .unwrap(); - let mut module = RpcModule::new(()); - module - .register_method("echo_data", |params, _, _| { - let data: String = params.one()?; - Ok::(data) - }) - .unwrap(); - module.register_method("ping", |_, _, _| "pong").unwrap(); - let addr = server.local_addr().unwrap(); - let handle = server.start(module); - (format!("ws://{}", addr), handle) - } - - #[tokio::test] - async fn test_ws_client_connection() { - let (url, handle) = run_server("", Arc::new(AtomicBool::new(true))) - .await - .unwrap(); - let ws_url = url.replace("http://", "ws://"); - let max_size = get_max_message_size(); - let client = WsClientBuilder::default() - .max_request_size(max_size) - .max_response_size(max_size) - .build(&ws_url) - .await - .unwrap(); - let response: String = client.request("ping", rpc_params![]).await.unwrap(); - assert_eq!(response, "pong"); - handle.stop().unwrap(); - } - - #[tokio::test] - async fn test_ws_large_message_50mb() { - const SIZE: usize = 50 * 1024 * 1024; // 50MB - let max_size = DEFAULT_MAX_SIZE; - let (ws_url, handle) = start_test_server_with_config(max_size).await; - let client = WsClientBuilder::default() - .max_request_size(max_size) - .max_response_size(max_size) - .build(&ws_url) - .await - .unwrap(); - let data = generate_random_string(SIZE); - let response: String = client - .request("echo_data", rpc_params![&data]) - .await - .unwrap(); - assert_eq!(response.len(), data.len()); - handle.stop().unwrap(); - } - - #[tokio::test] - #[serial] - async fn test_ws_env_var_limits_message_size() { - const ONE_MB: u32 = 1024 * 1024; - const TWO_MB: usize = 2 * 1024 * 1024; - - // Set env var to 1MB limit - // SAFETY: This test is marked #[serial] so no other tests run concurrently - unsafe { std::env::set_var("GETTER_WS_MAX_MESSAGE_SIZE", ONE_MB.to_string()) }; - assert_eq!(get_max_message_size(), ONE_MB); - - let (url, handle) = run_server("", Arc::new(AtomicBool::new(true))) - .await - .unwrap(); - let ws_url = url.replace("http://", "ws://"); - - // Client allows large messages, but server should reject - let client = WsClientBuilder::default() - .max_request_size(u32::MAX) - .max_response_size(u32::MAX) - .build(&ws_url) - .await - .unwrap(); - - // Verify ping still works (small message) - let response: String = client.request("ping", rpc_params![]).await.unwrap(); - assert_eq!(response, "pong"); - - // Send 2MB data via init request, should be rejected by 1MB server limit - let large_data = generate_random_string(TWO_MB); - let params = RpcInitRequest { - data_path: &large_data, - cache_path: "/tmp/cache", - global_expire_time: 3600, - }; - let response: Result = client.request("init", params).await; - assert!( - response.is_err(), - "2MB request should be rejected by 1MB server limit" - ); - - handle.stop().unwrap(); - // SAFETY: This test is marked #[serial] so no other tests run concurrently - unsafe { std::env::remove_var("GETTER_WS_MAX_MESSAGE_SIZE") }; - } -} diff --git a/src/utils.rs b/src/utils.rs deleted file mode 100644 index d940e27..0000000 --- a/src/utils.rs +++ /dev/null @@ -1,6 +0,0 @@ -pub mod convert; -pub mod http; -pub mod instance; -pub mod json; -pub mod time; -pub mod versioning; diff --git a/src/utils/convert.rs b/src/utils/convert.rs deleted file mode 100644 index 73be4df..0000000 --- a/src/utils/convert.rs +++ /dev/null @@ -1,21 +0,0 @@ -use std::collections::{BTreeMap, HashMap}; - -pub fn convert_hashmap_to_btreemap(hash_map: HashMap) -> BTreeMap -where - K: Ord + Eq + std::hash::Hash, - V: Clone, -{ - let mut btree_map: BTreeMap = BTreeMap::new(); - for (key, value) in hash_map { - btree_map.insert(key, value); - } - btree_map -} - -pub fn convert_btreemap(original: &BTreeMap) -> BTreeMap<&str, &str> { - let mut new_map: BTreeMap<&str, &str> = BTreeMap::new(); - for (key, value) in original.iter() { - new_map.insert(key.as_str(), value.as_str()); - } - new_map -} diff --git a/src/utils/http.rs b/src/utils/http.rs deleted file mode 100644 index a1894fb..0000000 --- a/src/utils/http.rs +++ /dev/null @@ -1,327 +0,0 @@ -use bytes::{Bytes, BytesMut}; -use http_body_util::{BodyExt, Empty}; -use hyper::{StatusCode, Uri}; -#[cfg(not(feature = "rustls-platform-verifier"))] -use hyper_rustls::ConfigBuilderExt; -use hyper_util::{ - client::legacy::{connect::HttpConnector, Client}, - rt::TokioExecutor, -}; -use once_cell::sync::Lazy; -use rustls::ClientConfig; -#[cfg(feature = "rustls-platform-verifier")] -use rustls_platform_verifier::BuilderVerifierExt; -use std::{collections::HashMap, fmt}; - -// Custom http response Error -#[derive(Debug)] -pub struct ResponseData { - pub status: u16, - pub body: Option, -} - -impl fmt::Display for ResponseData { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!( - f, - "Response status: {}, body: {}", - self.status, - self.body.as_ref().map_or_else( - || "".to_string(), - |body| String::from_utf8_lossy(body).to_string(), - ) - ) - } -} - -pub async fn get( - url: Uri, - header_map: &HashMap, -) -> Result> { - if url.scheme_str() == Some("https") { - https_get(url, header_map).await - } else { - http_get(url, header_map).await - } -} - -pub async fn head( - url: Uri, - header_map: &HashMap, -) -> Result> { - if url.scheme_str() == Some("https") { - https_head(url, header_map).await - } else { - http_head(url, header_map).await - } -} - -pub async fn http_get( - url: Uri, - header_map: &HashMap, -) -> Result> { - _http_get(url, header_map, false).await -} - -pub async fn http_head( - url: Uri, - header_map: &HashMap, -) -> Result> { - _http_get(url, header_map, true).await -} - -async fn _http_get( - url: Uri, - header_map: &HashMap, - only_status: bool, -) -> Result> { - let http = HttpConnector::new(); - let client = Client::builder(TokioExecutor::new()).build(http); - - let mut req = hyper::Request::builder().method("GET").uri(url.clone()); - for (key, value) in header_map { - req = req.header(key, value); - } - let req = req.body(Empty::::new())?; - let mut res = client.request(req).await?; - let status = res.status(); - if only_status { - Ok(ResponseData { - status: status.as_u16(), - body: None, - }) - } else { - let mut body = BytesMut::new(); - while let Some(next) = res.frame().await { - let frame = next?; - if let Some(chunk) = frame.data_ref() { - body.extend_from_slice(chunk); - } - } - Ok(ResponseData { - status: status.as_u16(), - body: Some(body.freeze()), - }) - } -} - -pub async fn https_get( - url: Uri, - header_map: &HashMap, -) -> Result> { - _https_get(url, header_map, false).await -} - -pub async fn https_head( - url: Uri, - header_map: &HashMap, -) -> Result> { - _https_get(url, header_map, true).await -} - -// Global https provider with lazy initialization -static PROVIDER: Lazy> = - Lazy::new(|| std::sync::Arc::new(rustls::crypto::ring::default_provider())); - -// Https config error wrapper error -struct HttpsConfigError { - error: Box, -} -impl fmt::Display for HttpsConfigError { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "HttpsConfigError: {}", self.error) - } -} -impl fmt::Debug for HttpsConfigError { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "HttpsConfigError: {:?}", self.error) - } -} -impl std::error::Error for HttpsConfigError { - fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { - None - } -} - -fn https_config() -> Result, HttpsConfigError> { - let provider = PROVIDER.clone(); - let tls: rustls::ClientConfig; - #[cfg(feature = "rustls-platform-verifier")] - { - tls = ClientConfig::builder_with_provider(provider) - .with_safe_default_protocol_versions() - .map_err(|e| HttpsConfigError { error: Box::new(e) })? - .with_platform_verifier() - .map_err(|e| HttpsConfigError { error: Box::new(e) })? - .with_no_client_auth(); - } - #[cfg(all(feature = "webpki-roots", not(feature = "rustls-platform-verifier")))] - { - tls = rustls::ClientConfig::builder_with_provider(provider) - .with_safe_default_protocol_versions() - .map_err(|e| HttpsConfigError { error: Box::new(e) })? - .with_webpki_roots() - .with_no_client_auth(); - } - #[cfg(all( - feature = "native-tokio", - not(feature = "webpki-roots"), - not(feature = "rustls-platform-verifier") - ))] - { - tls = rustls::ClientConfig::builder_with_provider(provider) - .with_safe_default_protocol_versions() - .map_err(|e| HttpsConfigError { error: Box::new(e) })? - .with_native_roots() - .map_err(|e| HttpsConfigError { error: Box::new(e) })? - .with_no_client_auth(); - } - #[cfg(all( - not(feature = "native-tokio"), - not(feature = "webpki-roots"), - not(feature = "rustls-platform-verifier") - ))] - { - compile_error!("No TLS backend enabled"); - } - Ok(hyper_rustls::HttpsConnectorBuilder::new() - .with_tls_config(tls) - .https_or_http() - .enable_http1() - .enable_http2() - .build()) -} - -async fn _https_get( - url: Uri, - header_map: &HashMap, - only_status: bool, -) -> Result> { - let https = https_config()?; - let client = Client::builder(TokioExecutor::new()).build(https); - let mut req = hyper::Request::builder().method("GET").uri(url.clone()); - for (key, value) in header_map { - req = req.header(key, value); - } - let req = req.body(Empty::::new())?; - - let mut res = client.request(req).await?; - let status = res.status(); - if only_status { - Ok(ResponseData { - status: status.as_u16(), - body: None, - }) - } else { - let mut body = BytesMut::new(); - while let Some(next) = res.frame().await { - let frame = next?; - if let Some(chunk) = frame.data_ref() { - body.extend_from_slice(chunk); - } - } - Ok(ResponseData { - status: status.as_u16(), - body: Some(body.freeze()), - }) - } -} - -pub fn http_status_is_ok(status: u16) -> bool { - if let Ok(status) = StatusCode::from_u16(status) { - !(status.is_client_error() || status.is_server_error()) - } else { - false - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[tokio::test] - async fn test_https_get() { - let url = "https://github.com".parse().unwrap(); - let result = https_get(url, &HashMap::new()).await; - assert!(result.is_ok()); - assert!(!result.unwrap().body.unwrap().is_empty()); - } - - #[tokio::test] - async fn test_https_get_invalid() { - let url = "https://123123".parse().unwrap(); - let result = https_get(url, &HashMap::new()).await; - assert!(result.is_err()); - } - - #[tokio::test] - async fn test_https_get_status() { - let url = "https://mockhttp.org/status/418".parse().unwrap(); - let result = https_get(url, &HashMap::new()).await; - assert_eq!(result.unwrap().status, 418); - } - - #[tokio::test] - async fn test_https_head() { - let url = "https://github.com".parse().unwrap(); - let result = https_head(url, &HashMap::new()).await; - assert!(result.is_ok()); - assert!(result.unwrap().body.is_none()); - } - - #[tokio::test] - async fn test_https_get_header() { - let url = "https://postman-echo.com/get".parse().unwrap(); - let header_map = { - let mut map = HashMap::new(); - map.insert("X-Test".to_string(), "test000".to_string()); - map.insert("Test-Header".to_string(), "test001".to_string()); - map - }; - let result = https_get(url, &header_map).await; - assert!(result.is_ok()); - let body = result.unwrap().body.expect("Response body was empty"); - let json: serde_json::Value = serde_json::from_slice(&body).expect("Failed to parse JSON"); - for (key, value) in header_map { - // key to lowercase, for fix podman-echo - // Due https://stackoverflow.com/questions/5258977/are-http-headers-case-sensitive - let key = key.to_lowercase(); - assert_eq!(json["headers"][key], value); - } - } - - #[tokio::test] - async fn test_http_get() { - let url = "http://example.com".parse().unwrap(); - let result = http_get(url, &HashMap::new()).await; - assert!(result.is_ok()); - assert!(!result.unwrap().body.unwrap().is_empty()); - } - - #[tokio::test] - async fn test_http_head() { - let url = "http://example.com".parse().unwrap(); - let result = http_head(url, &HashMap::new()).await; - assert!(result.is_ok()); - assert!(result.unwrap().body.is_none()); - } - - #[tokio::test] - async fn test_http_get_header() { - let url = "http://postman-echo.com/get".parse().unwrap(); - let header_map = { - let mut map = HashMap::new(); - map.insert("X-Test".to_string(), "test000".to_string()); - map.insert("Test-Header".to_string(), "test001".to_string()); - map - }; - let result = http_get(url, &header_map).await; - assert!(result.is_ok()); - let body = result.unwrap().body.expect("Response body was empty"); - let json: serde_json::Value = serde_json::from_slice(&body).expect("Failed to parse JSON"); - for (key, value) in header_map { - let key = key.to_lowercase(); - assert_eq!(json["headers"][key], value); - } - } -} diff --git a/src/utils/instance.rs b/src/utils/instance.rs deleted file mode 100644 index d4acd9c..0000000 --- a/src/utils/instance.rs +++ /dev/null @@ -1,18 +0,0 @@ -use std::sync::Arc; -use tokio::sync::Mutex; - -pub struct InstanceContainer { - instance: Arc>, -} - -impl InstanceContainer { - pub fn new(value: T) -> Self { - Self { - instance: Arc::new(Mutex::new(value)), - } - } - - pub async fn get(&self) -> Arc> { - self.instance.clone() - } -} diff --git a/src/utils/json.rs b/src/utils/json.rs deleted file mode 100644 index 54c9710..0000000 --- a/src/utils/json.rs +++ /dev/null @@ -1,31 +0,0 @@ -use bytes::Bytes; -use serde::de::Deserialize; -use serde::ser::Serialize; - -pub fn json_to_bytes(json: &T) -> Result -where - T: ?Sized + Serialize, -{ - serde_json::to_vec(json).map(Bytes::from) -} - -pub fn bytes_to_json<'a, T>(bytes: &'a Bytes) -> Result -where - T: Deserialize<'a>, -{ - serde_json::from_slice(bytes) -} - -pub fn json_to_string(json: &T) -> Result -where - T: ?Sized + Serialize, -{ - serde_json::to_string(json) -} - -pub fn string_to_json<'a, T>(string: &'a str) -> Result -where - T: Deserialize<'a>, -{ - serde_json::from_str(string) -} diff --git a/src/utils/time.rs b/src/utils/time.rs deleted file mode 100644 index 98f3039..0000000 --- a/src/utils/time.rs +++ /dev/null @@ -1,8 +0,0 @@ -use std::time::{Duration, SystemTime, UNIX_EPOCH}; - -pub fn get_now_unix() -> u64 { - SystemTime::now() - .duration_since(UNIX_EPOCH) - .unwrap_or_else(|_| Duration::from_secs(0)) - .as_secs() -} diff --git a/src/utils/versioning.rs b/src/utils/versioning.rs deleted file mode 100644 index c25d819..0000000 --- a/src/utils/versioning.rs +++ /dev/null @@ -1,170 +0,0 @@ -use std::cmp::Ordering; - -use libversion_sys; - -use once_cell::sync::Lazy; -use regex::Regex; - -static VERSION_NUMBER_STRICT_MATCH_REGEX: Lazy = - Lazy::new(|| Regex::new(r"\d+(\.\d+)+([.|\-|+|_| ]*[A-Za-z0-9]+)*").unwrap()); - -static VERSION_NUMBER_MATCH_REGEX: Lazy = - Lazy::new(|| Regex::new(r"\d+(\.\d+)*([.|\-|+|_| ]*[A-Za-z0-9]+)*").unwrap()); - -#[derive(Debug, Clone)] -pub struct Version { - string: String, -} - -impl Version { - pub fn new(string: String) -> Self { - Version { string } - } - - pub fn is_valid(&self) -> bool { - self.get_valid_version().is_some() - } - - pub fn get_valid_version(&self) -> Option { - VERSION_NUMBER_STRICT_MATCH_REGEX - .find(&self.string) - .or_else(|| VERSION_NUMBER_MATCH_REGEX.find(&self.string)) - .map(|match_str| match_str.as_str().to_string()) - } -} - -impl PartialEq for Version { - fn eq(&self, other: &Self) -> bool { - match (self.get_valid_version(), other.get_valid_version()) { - (Some(v1), Some(v2)) => libversion_sys::compare(&v1, &v2) == Ordering::Equal, - _ => false, - } - } -} - -impl PartialOrd for Version { - fn partial_cmp(&self, other: &Self) -> Option { - match (self.get_valid_version(), other.get_valid_version()) { - (Some(v1), Some(v2)) => Some(libversion_sys::compare(&v1, &v2)), - _ => None, - } - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_version_is_valid() { - let version = Version { - string: "1.0.0".to_string(), - }; - assert!(version.is_valid()); - let version = Version { - string: "1.0.0-alpha".to_string(), - }; - assert!(version.is_valid()); - let version = Version { - string: "版本1.0.0".to_string(), - }; - assert!(version.is_valid()); - let chinese_suffix_version = Version { - string: "版本1.0.0 天行健".to_string(), - }; - assert!(chinese_suffix_version.is_valid()); - } - - #[test] - fn test_version_is_invalid() { - let version = Version { - string: "xxx".to_string(), - }; - assert!(!version.is_valid()); - } - - #[test] - fn test_version_eq() { - let version = Version { - string: "1.0.0".to_string(), - }; - let other_version = Version { - string: "1.0".to_string(), - }; - assert_eq!(version, other_version); - - let chinese_version = Version { - string: "版本1.0.0".to_string(), - }; - assert_eq!(version, chinese_version); - } - - #[test] - fn test_version_ne() { - let version = Version { - string: "1.0.0".to_string(), - }; - let other_version = Version { - string: "1.0.1".to_string(), - }; - assert_ne!(version, other_version); - } - - #[test] - fn test_version_lt() { - let version = Version { - string: "1.0".to_string(), - }; - let other_version = Version { - string: "1.0.1".to_string(), - }; - assert!(version < other_version); - } - - #[test] - fn test_version_gt() { - let version = Version { - string: "1.0.1".to_string(), - }; - let other_version = Version { - string: "1.0.1-alpha".to_string(), - }; - assert!(version > other_version); - } - - #[test] - fn test_version_get_valid_version() { - let version = Version { - string: "1.0.0 123123".to_string(), - }; - assert_eq!( - version.get_valid_version(), - Some("1.0.0 123123".to_string()) - ); - let version = Version { - string: "1.0.0-alpha 版本".to_string(), - }; - assert_eq!(version.get_valid_version(), Some("1.0.0-alpha".to_string())); - let version = Version { - string: "版本1.0.0".to_string(), - }; - assert_eq!(version.get_valid_version(), Some("1.0.0".to_string())); - let chinese_suffix_version = Version { - string: "版本1.0.0 天行健".to_string(), - }; - assert_eq!( - chinese_suffix_version.get_valid_version(), - Some("1.0.0".to_string()) - ); - - let version = Version { - string: "xxx".to_string(), - }; - assert_eq!(version.get_valid_version(), None); - - let version = Version { - string: "1.0-alpha 版本 123123".to_string(), - }; - assert_eq!(version.get_valid_version(), Some("1.0-alpha".to_string())); - } -} diff --git a/src/websdk.rs b/src/websdk.rs deleted file mode 100644 index 6100512..0000000 --- a/src/websdk.rs +++ /dev/null @@ -1,2 +0,0 @@ -pub mod cloud_rules; -pub mod repo; diff --git a/src/websdk/cloud_rules.rs b/src/websdk/cloud_rules.rs deleted file mode 100644 index 5952fe8..0000000 --- a/src/websdk/cloud_rules.rs +++ /dev/null @@ -1,3 +0,0 @@ -pub mod cloud_rules_manager; -pub mod cloud_rules_wrapper; -pub mod data; diff --git a/src/websdk/cloud_rules/cloud_rules_manager.rs b/src/websdk/cloud_rules/cloud_rules_manager.rs deleted file mode 100644 index 17aaebf..0000000 --- a/src/websdk/cloud_rules/cloud_rules_manager.rs +++ /dev/null @@ -1,112 +0,0 @@ -use hyper::Uri; -use std::collections::HashMap; - -use super::data::config_list::{ConfigList, ConfigListViewer}; -use crate::utils::http::get; - -pub struct CloudRules { - pub api_url: String, - - _config_list: Option, -} - -#[derive(Debug)] -pub struct DownloadError { - pub url: Uri, - pub error: Box, -} - -impl std::fmt::Display for DownloadError { - fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { - write!(f, "DownloadError: {} ({})", self.error, self.url) - } -} - -impl CloudRules { - pub fn new(api_url: &str) -> CloudRules { - CloudRules { - api_url: api_url.to_string(), - _config_list: None, - } - } - - pub fn get_config_list(&self) -> ConfigListViewer<'_> { - self._config_list - .as_ref() - .map_or_else(ConfigListViewer::default, |config_list| { - config_list.viewer() - }) - } - - pub async fn renew(&mut self) -> Result<(), DownloadError> { - let config_list = self.download_config_list(&self.api_url).await?; - self._config_list = Some(config_list); - Ok(()) - } - - async fn download_config_list(&self, url: &str) -> Result { - Self::_download_config_list_impl(url) - .await - .map_err(|e| DownloadError { - url: url.parse().unwrap(), - error: e, - }) - } - async fn _download_config_list_impl( - url: &str, - ) -> Result> { - let map = HashMap::new(); - let resp = get(url.parse()?, &map).await?; - if let Some(body) = resp.body { - let config_list: ConfigList = serde_json::from_slice(&body)?; - Ok(config_list) - } else { - Err("No body".into()) - } - } -} - -#[cfg(test)] -mod tests { - use super::*; - use mockito::Server; - use std::fs; - - #[tokio::test] - async fn test_download_config_list() { - let json = fs::read_to_string("tests/files/data/UpgradeAll-rules_rules.json").unwrap(); - let path = "/DUpdateSystem/UpgradeAll-rules/master/rules.json"; - let mut server = Server::new_async().await; - server.mock("GET", path).with_body(json).create(); - let url = server.url() + path; - - let mut cloud_rules = CloudRules::new(&url); - cloud_rules.renew().await.unwrap(); - let config_list = cloud_rules.get_config_list(); - assert_eq!(config_list.app_config_list.len(), 219); - assert_eq!(config_list.app_config_list[0].info.name, "UpgradeAll"); - assert_eq!( - config_list.app_config_list.last().unwrap().info.name, - "黑阈" - ); - assert_eq!(config_list.hub_config_list.len(), 11); - assert_eq!(config_list.hub_config_list[0].info.hub_name, "GitHub"); - assert_eq!( - config_list.hub_config_list.last().unwrap().info.hub_name, - "Xposed Module Repository" - ); - } - - #[tokio::test] - async fn test_download_config_list_invalid() { - let path = "/DUpdateSystem/UpgradeAll-rules/master/rules.json"; - let mut server = Server::new_async().await; - server.mock("GET", path).with_status(404).create(); - let url = server.url() + path; - - let mut cloud_rules = CloudRules::new(&url); - let result = cloud_rules.renew().await; - assert!(result.is_err()); - assert_eq!(result.unwrap_err().url.to_string(), url); - } -} diff --git a/src/websdk/cloud_rules/cloud_rules_wrapper.rs b/src/websdk/cloud_rules/cloud_rules_wrapper.rs deleted file mode 100644 index 24a718a..0000000 --- a/src/websdk/cloud_rules/cloud_rules_wrapper.rs +++ /dev/null @@ -1,109 +0,0 @@ -pub use super::cloud_rules_manager::CloudRules; -use super::data::app_item::AppItem; -use super::data::hub_item::HubItem; - -impl CloudRules { - pub fn get_cloud_app_rules(&self, filter: F) -> Option - where - F: Fn(&AppItem) -> bool, - { - self._get_cloud_app_rules(filter, true) - .first() - .map(|x| x.to_owned()) - } - - pub fn get_cloud_app_rules_list(&self, filter: F) -> Vec - where - F: Fn(&AppItem) -> bool, - { - self._get_cloud_app_rules(filter, false) - } - - pub fn _get_cloud_app_rules(&self, filter: F, only_first: bool) -> Vec - where - F: Fn(&AppItem) -> bool, - { - let mut list = Vec::new(); - for config in self.get_config_list().app_config_list { - if filter(config) { - list.push(config.to_owned()); - if only_first { - break; - } - } - } - list - } - - pub fn get_cloud_hub_rules(&self, filter: F) -> Option - where - F: Fn(&HubItem) -> bool, - { - for config in self.get_config_list().hub_config_list { - if filter(config) { - return Some(config.to_owned()); - } - } - None - } -} - -#[cfg(test)] -mod tests { - use super::*; - use mockito::Server; - use std::fs; - - #[tokio::test] - async fn test_get_cloud_app_rules() { - let json = fs::read_to_string("tests/files/data/UpgradeAll-rules_rules.json").unwrap(); - let path = "/DUpdateSystem/UpgradeAll-rules/master/rules.json"; - let mut server = Server::new_async().await; - server.mock("GET", path).with_body(json).create(); - let url = server.url() + path; - - let mut cloud_rules = CloudRules::new(&url); - cloud_rules.renew().await.unwrap(); - let list = cloud_rules.get_cloud_app_rules(|x| x.info.name == "UpgradeAll"); - assert!(list.is_some()); - let list = cloud_rules.get_cloud_app_rules(|x| x.info.name.is_empty()); - assert!(list.is_none()); - } - - #[tokio::test] - async fn test_get_cloud_app_rules_list() { - let json = fs::read_to_string("tests/files/data/UpgradeAll-rules_rules.json").unwrap(); - let path = "/DUpdateSystem/UpgradeAll-rules/master/rules.json"; - let mut server = Server::new_async().await; - server.mock("GET", path).with_body(json).create(); - let url = server.url() + path; - - let mut cloud_rules = CloudRules::new(&url); - cloud_rules.renew().await.unwrap(); - let list = cloud_rules.get_cloud_app_rules_list(|x| x.info.name == "UpgradeAll"); - assert_eq!(list.len(), 1); - let list = cloud_rules.get_cloud_app_rules_list(|x| x.info.name.is_empty()); - assert_eq!(list.len(), 0); - let list = cloud_rules.get_cloud_app_rules_list(|x| !x.info.name.is_empty()); - assert_eq!( - list.len(), - cloud_rules.get_config_list().app_config_list.len() - ); - } - - #[tokio::test] - async fn test_get_cloud_hub_rules() { - let json = fs::read_to_string("tests/files/data/UpgradeAll-rules_rules.json").unwrap(); - let path = "/DUpdateSystem/UpgradeAll-rules/master/rules.json"; - let mut server = Server::new_async().await; - server.mock("GET", path).with_body(json).create(); - let url = server.url() + path; - - let mut cloud_rules = CloudRules::new(&url); - cloud_rules.renew().await.unwrap(); - let list = cloud_rules.get_cloud_hub_rules(|x| x.info.hub_name == "GitHub"); - assert!(list.is_some()); - let list = cloud_rules.get_cloud_hub_rules(|x| x.info.hub_name.is_empty()); - assert!(list.is_none()); - } -} diff --git a/src/websdk/cloud_rules/data.rs b/src/websdk/cloud_rules/data.rs deleted file mode 100644 index 07f8d80..0000000 --- a/src/websdk/cloud_rules/data.rs +++ /dev/null @@ -1,3 +0,0 @@ -pub mod app_item; -pub mod config_list; -pub mod hub_item; diff --git a/src/websdk/cloud_rules/data/app_item.rs b/src/websdk/cloud_rules/data/app_item.rs deleted file mode 100644 index dca31c9..0000000 --- a/src/websdk/cloud_rules/data/app_item.rs +++ /dev/null @@ -1,94 +0,0 @@ -use serde::{Deserialize, Serialize}; -use std::collections::HashMap; - -/// AppConfig -/// -/// JSON Schema: -/// ```json -/// { -/// "base_version": 2, -/// "config_version": 1, -/// "uuid": "", -/// "base_hub_uuid": "", -/// "info": { -/// "name": "", -/// "url": "", -/// "extra_map": { -/// "android_app_package": "" -/// } -/// } -/// } -/// ``` - -#[derive(Serialize, Deserialize, Debug, Clone, PartialEq)] -pub struct AppItem { - #[serde(rename = "base_version")] - pub base_version: i32, - - #[serde(rename = "config_version")] - pub config_version: i32, - - #[serde(rename = "uuid")] - pub uuid: String, - - #[serde(rename = "base_hub_uuid")] - pub base_hub_uuid: String, - - #[serde(rename = "info")] - pub info: AppInfo, -} - -#[derive(Serialize, Deserialize, Debug, Clone, PartialEq)] -pub struct AppInfo { - #[serde(rename = "name")] - pub name: String, - - #[serde(rename = "url")] - pub url: String, - - #[serde(rename = "extra_map")] - pub extra_map: HashMap, // Use HashMap to store arbitrary key/value pairs -} - -#[cfg(test)] -mod tests { - use super::*; - use serde_json; - - #[test] - fn test_app_config() { - let json = r#" -{ - "base_version": 2, - "config_version": 1, - "uuid": "f27f71e1-d7a1-4fd1-bbcc-9744380611a1", - "base_hub_uuid": "fd9b2602-62c5-4d55-bd1e-0d6537714ca0", - "info": { - "name": "UpgradeAll", - "url": "https://github.com/xz-dev/UpgradeAll", - "extra_map": { - "android_app_package": "net.xzos.upgradeall" - } - } -} -"#; - - let app_item: AppItem = serde_json::from_str(json).unwrap(); - assert_eq!(app_item.base_version, 2); - assert_eq!(app_item.config_version, 1); - assert_eq!(app_item.uuid, "f27f71e1-d7a1-4fd1-bbcc-9744380611a1"); - assert_eq!( - app_item.base_hub_uuid, - "fd9b2602-62c5-4d55-bd1e-0d6537714ca0" - ); - assert_eq!(app_item.info.name, "UpgradeAll"); - assert_eq!(app_item.info.url, "https://github.com/xz-dev/UpgradeAll"); - assert_eq!( - app_item.info.extra_map, - HashMap::from([( - "android_app_package".to_string(), - "net.xzos.upgradeall".to_string() - )]) - ); - } -} diff --git a/src/websdk/cloud_rules/data/config_list.rs b/src/websdk/cloud_rules/data/config_list.rs deleted file mode 100644 index 886fcfe..0000000 --- a/src/websdk/cloud_rules/data/config_list.rs +++ /dev/null @@ -1,80 +0,0 @@ -use serde::{Deserialize, Serialize}; - -use super::app_item::AppItem; -use super::hub_item::HubItem; - -/// Configuration lists -/// -/// JSON Schema: -/// ```json -/// { -/// "app_config_list": [], -/// "hub_config_list": [] -/// } -/// ``` - -#[derive(Default, Serialize, Debug, Clone)] -pub struct ConfigListViewer<'a> { - #[serde(rename = "app_config_list")] - pub app_config_list: Vec<&'a AppItem>, - - #[serde(rename = "hub_config_list")] - pub hub_config_list: Vec<&'a HubItem>, -} - -#[derive(Serialize, Deserialize, Debug, Clone)] -pub struct ConfigList { - #[serde(rename = "app_config_list")] - pub app_config_list: Vec, - - #[serde(rename = "hub_config_list")] - pub hub_config_list: Vec, -} - -impl ConfigList { - pub fn viewer(&self) -> ConfigListViewer<'_> { - ConfigListViewer { - app_config_list: self.app_config_list.iter().collect(), - hub_config_list: self.hub_config_list.iter().collect(), - } - } -} - -impl ConfigListViewer<'_> { - pub fn to_owned(&self) -> ConfigList { - ConfigList { - app_config_list: self.app_config_list.iter().map(|&x| x.clone()).collect(), - hub_config_list: self.hub_config_list.iter().map(|&x| x.clone()).collect(), - } - } -} - -#[cfg(test)] -mod tests { - use super::*; - use serde_json; - use std::fs; - - #[test] - fn test_config_list() { - // read json from file - let json = fs::read_to_string("tests/files/data/UpgradeAll-rules_rules.json").unwrap(); - - let config_list: ConfigList = serde_json::from_str(&json).unwrap(); - - // check app_config_list - assert_eq!(config_list.app_config_list.len(), 219); - assert_eq!(config_list.app_config_list[0].info.name, "UpgradeAll"); - assert_eq!( - config_list.app_config_list.last().unwrap().info.name, - "黑阈" - ); - // check hub_config_list - assert_eq!(config_list.hub_config_list.len(), 11); - assert_eq!(config_list.hub_config_list[0].info.hub_name, "GitHub"); - assert_eq!( - config_list.hub_config_list.last().unwrap().info.hub_name, - "Xposed Module Repository" - ); - } -} diff --git a/src/websdk/cloud_rules/data/hub_item.rs b/src/websdk/cloud_rules/data/hub_item.rs deleted file mode 100644 index c8c43b7..0000000 --- a/src/websdk/cloud_rules/data/hub_item.rs +++ /dev/null @@ -1,123 +0,0 @@ -use serde::{Deserialize, Serialize}; - -/// HubConfig -/// -/// JSON Schema: -/// ```json -/// { -/// base_version: 6 -/// config_version: 1 -/// uuid: "" -/// info: { -/// "hub_name": "", -/// "hub_icon_url": "" -/// } -/// target_check_api: "" -/// api_keywords: [] -/// auth_keywords: [] -/// app_url_templates": [] -/// } -/// ``` - -#[derive(Serialize, Deserialize, Debug, Clone, PartialEq)] -pub struct HubItem { - #[serde(rename = "base_version", default)] - pub base_version: i32, - - #[serde(rename = "config_version", default)] - pub config_version: i32, - - #[serde(rename = "uuid", default)] - pub uuid: String, - - #[serde(rename = "info")] - pub info: Info, - - #[serde(rename = "api_keywords", default)] - pub api_keywords: Vec, - - /// Auth parameter keys required by this hub (e.g. ["token"] for GitHub). - /// Used by the UI to provide autocomplete suggestions when editing hub auth. - #[serde(rename = "auth_keywords", default)] - pub auth_keywords: Vec, - - #[serde(rename = "app_url_templates", default)] - pub app_url_templates: Vec, - - #[serde(rename = "target_check_api")] - pub target_check_api: Option, -} - -#[derive(Serialize, Deserialize, Debug, Clone, PartialEq)] -pub struct Info { - #[serde(rename = "hub_name", default)] - pub hub_name: String, - - #[serde(rename = "hub_icon_url", default)] - pub hub_icon_url: Option, -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_hub_item() { - let json = r#" -{ - "base_version": 6, - "config_version": 3, - "uuid": "fd9b2602-62c5-4d55-bd1e-0d6537714ca0", - "info": { - "hub_name": "GitHub", - "hub_icon_url": "" - }, - "target_check_api": "", - "api_keywords": [ - "owner", - "repo" - ], - "app_url_templates": [ - "https://github.com/%owner/%repo/" - ] -} - "#; - - let hub_item: HubItem = serde_json::from_str(json).unwrap(); - assert_eq!(hub_item.base_version, 6); - assert_eq!(hub_item.config_version, 3); - assert_eq!(hub_item.uuid, "fd9b2602-62c5-4d55-bd1e-0d6537714ca0"); - assert_eq!(hub_item.info.hub_name, "GitHub"); - assert_eq!(hub_item.info.hub_icon_url, Some("".to_string())); - assert_eq!(hub_item.target_check_api, Some("".to_string())); - assert_eq!(hub_item.api_keywords, ["owner", "repo"]); - // Old config without auth_keywords deserializes to empty vec. - assert_eq!(hub_item.auth_keywords, Vec::::new()); - assert_eq!( - hub_item.app_url_templates[0], - "https://github.com/%owner/%repo/" - ); - } - - #[test] - fn test_hub_item_auth_keywords() { - let json = r#" -{ - "base_version": 5, - "config_version": 3, - "uuid": "fd9b2602-62c5-4d55-bd1e-0d6537714ca0", - "info": { "hub_name": "GitHub" }, - "api_keywords": ["owner", "repo"], - "auth_keywords": ["token"], - "app_url_templates": ["https://github.com/%owner/%repo/"] -} - "#; - let hub_item: HubItem = serde_json::from_str(json).unwrap(); - assert_eq!(hub_item.auth_keywords, ["token"]); - - // Round-trip serialization preserves auth_keywords. - let serialized = serde_json::to_string(&hub_item).unwrap(); - let hub_item2: HubItem = serde_json::from_str(&serialized).unwrap(); - assert_eq!(hub_item2.auth_keywords, ["token"]); - } -} diff --git a/src/websdk/repo.rs b/src/websdk/repo.rs deleted file mode 100644 index d10139d..0000000 --- a/src/websdk/repo.rs +++ /dev/null @@ -1,3 +0,0 @@ -pub mod api; -pub mod data; -pub mod provider; diff --git a/src/websdk/repo/api.rs b/src/websdk/repo/api.rs deleted file mode 100644 index 044176a..0000000 --- a/src/websdk/repo/api.rs +++ /dev/null @@ -1,166 +0,0 @@ -use async_fn_traits::AsyncFnOnce2; -use serde::de::DeserializeOwned; -use serde::Serialize; - -use super::data::release::ReleaseData; -use super::provider::base_provider::{AppDataMap, DataMap, FIn, FOut, FunctionType, HubDataMap}; -use super::provider::outside_rpc::OutsideProvider; -use super::provider::{self, add_provider}; -use crate::cache::get_cache_manager; -use crate::cache::manager::GroupType; -use crate::rpc::data::DownloadItemData; -use crate::utils::json::{bytes_to_json, json_to_bytes}; -use std::collections::HashMap; - -#[derive(Debug, Clone)] -struct ErrorProviderNotFound; - -impl std::fmt::Display for ErrorProviderNotFound { - fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { - write!(f, "Provider not found for this request.") - } -} - -impl std::error::Error for ErrorProviderNotFound { - fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { - None - } -} - -async fn call_func( - uuid: &str, - app_data: &AppDataMap<'_>, - hub_data: &HubDataMap<'_>, - func_type: FunctionType, - provider_func: F, -) -> Result, ErrorProviderNotFound> -where - T: Send + DeserializeOwned + Serialize, - F: for<'b> AsyncFnOnce2<&'b str, &'b FIn<'b>, Output = Option>>, -{ - let cache_manager = get_cache_manager().await; - let data_map = DataMap { app_data, hub_data }; - let api_cache_key = data_map.get_hash(); - if let Some(bytes) = cache_manager - .lock() - .await - .get(&GroupType::Api, &api_cache_key.to_string(), None) - .await - { - if let Ok(value) = bytes_to_json::(&bytes) { - return Ok(Some(value)); - } - } - let cache_keys = provider::get_cache_request_key(uuid, &func_type, &data_map); - let mut cache_map = HashMap::new(); - if let Some(keys) = cache_keys { - for key in keys { - if let Some(value) = cache_manager - .lock() - .await - .get(&GroupType::RepoInside, &key, None) - .await - { - cache_map.insert(key, value); - } - } - } - - let fin = FIn::new(data_map, Some(cache_map)); - if let Some(fout) = provider_func(uuid, &fin).await { - if let Some(cached_map) = fout.cached_map { - for (key, value) in cached_map { - let _ = cache_manager - .lock() - .await - .save(&GroupType::RepoInside, &key, value) - .await; - } - } - if let Ok(data) = fout.result { - if let Ok(value) = json_to_bytes(&data) { - let _ = cache_manager - .lock() - .await - .save(&GroupType::Api, &api_cache_key.to_string(), value) - .await; - } - Ok(Some(data)) - } else { - Ok(None) - } - } else { - Err(ErrorProviderNotFound) - } -} - -pub async fn check_app_available<'a>( - uuid: &str, - app_data: &AppDataMap<'a>, - hub_data: &HubDataMap<'a>, -) -> Option { - call_func( - uuid, - app_data, - hub_data, - FunctionType::CheckAppAvailable, - provider::check_app_available, - ) - .await - .unwrap_or(None) -} - -pub async fn get_latest_release<'a>( - uuid: &str, - app_data: &AppDataMap<'a>, - hub_data: &HubDataMap<'a>, -) -> Option { - call_func( - uuid, - app_data, - hub_data, - FunctionType::GetLatestRelease, - provider::get_latest_release, - ) - .await - .unwrap_or(None) -} - -pub async fn get_releases<'a>( - uuid: &str, - app_data: &AppDataMap<'a>, - hub_data: &HubDataMap<'a>, -) -> Option> { - call_func( - uuid, - app_data, - hub_data, - FunctionType::GetReleases, - provider::get_releases, - ) - .await - .unwrap_or(None) -} - -pub async fn get_download<'a>( - uuid: &str, - app_data: &AppDataMap<'a>, - hub_data: &HubDataMap<'a>, - asset_index: &[i32], -) -> Option> { - let data_map = DataMap { app_data, hub_data }; - let fin = FIn::new(data_map, None); - if let Some(fout) = provider::get_download(uuid, &fin, asset_index).await { - fout.result.ok() - } else { - None - } -} - -pub fn add_outside_provider(uuid: &str, url: &str) { - let provider = OutsideProvider { - uuid: uuid.to_string(), - url: url.to_string(), - }; - add_provider(uuid, provider); -} diff --git a/src/websdk/repo/controller.rs b/src/websdk/repo/controller.rs deleted file mode 100644 index f9eae81..0000000 --- a/src/websdk/repo/controller.rs +++ /dev/null @@ -1,121 +0,0 @@ -use bytes::Bytes; -use std::future::Future; - -use crate::cache::convert::{bool_to_bytes, bytes_to_bool}; -use crate::utils::json::{bytes_to_json, json_to_bytes}; -use crate::cache::manager::GroupType::{API, REPO_INSIDE}; -use crate::get_cache_manager; - -use super::data::release::ReleaseData; -use super::provider; -use super::provider::base_provider::{FIn, FOut, FunctionType, IdMap}; - -pub async fn check_app_available<'a>(uuid: &str, id_map: &IdMap<'a>) -> Option { - let key = format!("check_app_available_{}_{:?}", &uuid, id_map); - let expire_time = Some(60 * 60 * 24); - cache_future_return( - _check_app_available(uuid, id_map), - &key, - expire_time, - |data| Ok(bool_to_bytes(data)), - |bytes| Ok(bytes_to_bool(bytes)), - ) - .await -} - -pub async fn get_latest_release<'a>(uuid: &str, id_map: &IdMap<'a>) -> Option { - let key = format!("get_latest_release_{}_{:?}", &uuid, id_map); - let expire_time = Some(60 * 60 * 24); - cache_future_return( - _get_latest_release(uuid, id_map), - &key, - expire_time, - |data| json_to_bytes(&data), - |bytes| bytes_to_json(bytes), - ) - .await -} - -pub async fn get_releases<'a>(uuid: &str, id_map: &IdMap<'a>) -> Option> { - let key = format!("get_releases_{}_{:?}", &uuid, id_map); - let expire_time = Some(60 * 60 * 24); - cache_future_return( - _get_releases(uuid, id_map), - &key, - expire_time, - |data| json_to_bytes(&data), - |bytes| bytes_to_json(bytes), - ) - .await -} - -async fn _check_app_available<'a>(uuid: &str, id_map: &IdMap<'a>) -> Option { - let fin = get_fin(uuid, id_map, &FunctionType::CheckAppAvailable).await; - let fout = provider::check_app_available(uuid, &fin).await; - detach_result(fout).await -} - -async fn _get_latest_release<'a>(uuid: &str, id_map: &IdMap<'a>) -> Option { - let fin = get_fin(uuid, id_map, &FunctionType::GetLatestRelease).await; - let fout = provider::get_latest_release(uuid, &fin).await; - detach_result(fout).await -} - -async fn _get_releases<'a>(uuid: &str, id_map: &IdMap<'a>) -> Option> { - let fin = get_fin(uuid, id_map, &FunctionType::GetReleases).await; - let fout = provider::get_releases(uuid, &fin).await; - detach_result(fout).await -} - -async fn get_fin<'a>(uuid: &str, id_map: &'a IdMap<'a>, function_type: &FunctionType) -> FIn<'a> { - let cache_map = if let Some(cache_key_list) = - provider::get_cache_request_key(uuid, function_type, id_map) - { - let map = get_cache_manager!() - .get_cache_map(&REPO_INSIDE, &cache_key_list, None) - .await; - if map.is_empty() { - None - } else { - Some(map) - } - } else { - None - }; - FIn::new(id_map, cache_map) -} - -async fn detach_result(fout: Option>) -> Option { - if let Some(fout) = fout { - if let Some(cache_map) = fout.cached_map { - for (key, value) in cache_map { - let _ = get_cache_manager!().save(&REPO_INSIDE, &key, value).await; - } - } - fout.result.ok() - } else { - None - } -} - -async fn cache_future_return( - f: impl Future>, - key: &str, - expire_time: Option, - encoder: fn(&T) -> Result, - decoder: fn(&Bytes) -> Result, -) -> Option { - if let Some(value) = get_cache_manager!().get(&API, key, expire_time).await { - decoder(&value).ok() - } else { - let result = f.await; - if let Some(result) = result { - if let Ok(value) = encoder(&result) { - let _ = get_cache_manager!().save(&API, key, value).await; - } - Some(result) - } else { - None - } - } -} diff --git a/src/websdk/repo/data.rs b/src/websdk/repo/data.rs deleted file mode 100644 index f35143a..0000000 --- a/src/websdk/repo/data.rs +++ /dev/null @@ -1 +0,0 @@ -pub mod release; diff --git a/src/websdk/repo/data/release.rs b/src/websdk/repo/data/release.rs deleted file mode 100644 index 6d25119..0000000 --- a/src/websdk/repo/data/release.rs +++ /dev/null @@ -1,17 +0,0 @@ -use serde::{Deserialize, Serialize}; -use std::collections::HashMap; - -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -pub struct ReleaseData { - pub version_number: String, - pub changelog: String, - pub assets: Vec, - pub extra: Option>, -} - -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -pub struct AssetData { - pub file_name: String, - pub file_type: String, - pub download_url: String, -} diff --git a/src/websdk/repo/provider.rs b/src/websdk/repo/provider.rs deleted file mode 100644 index 220367b..0000000 --- a/src/websdk/repo/provider.rs +++ /dev/null @@ -1,99 +0,0 @@ -pub mod base_provider; -pub mod fdroid; -pub mod github; -pub mod gitlab; -pub mod lsposed_repo; -pub mod outside_rpc; - -use once_cell::sync::Lazy; -use std::collections::HashMap; -use std::sync::{Arc, RwLock}; - -use self::base_provider::{BaseProvider, DataMap, FIn, FOut, FunctionType}; -use self::fdroid::FDroidProvider; -use self::github::GitHubProvider; -use self::gitlab::GitLabProvider; -use self::lsposed_repo::LsposedRepoProvider; -use super::data::release::ReleaseData; -use crate::rpc::data::DownloadItemData; - -type ProviderMap = HashMap<&'static str, Arc>; - -static PROVIDER_MAP: Lazy>> = Lazy::new(|| { - Arc::new(RwLock::new(HashMap::from([ - ( - "fd9b2602-62c5-4d55-bd1e-0d6537714ca0", - Arc::new(GitHubProvider::new()) as Arc, - ), - ( - "6a6d590b-1809-41bf-8ce3-7e3f6c8da945", - Arc::new(FDroidProvider::new()) as Arc, - ), - ( - "a84e2fbe-1478-4db5-80ae-75d00454c7eb", - Arc::new(GitLabProvider::new()) as Arc, - ), - ( - "401e6259-2eab-46f0-8e8a-d2bfafedf5bf", - Arc::new(LsposedRepoProvider::new()) as Arc, - ), - ]))) -}); - -fn get_provider(uuid: &str) -> Option> { - let map = PROVIDER_MAP.read().unwrap(); - map.get(uuid).cloned() -} - -pub fn add_provider(uuid: &str, provider: impl BaseProvider + Send + Sync + 'static) { - let mut map = PROVIDER_MAP.write().unwrap(); - let uuid: &'static str = Box::leak(Box::new(uuid.to_string())); - map.insert( - uuid, - Arc::new(provider) as Arc, - ); -} - -pub fn get_cache_request_key( - uuid: &str, - function_type: &FunctionType, - data_map: &DataMap, -) -> Option> { - get_provider(uuid).map(|provider| provider.get_cache_request_key(function_type, data_map)) -} - -pub async fn check_app_available<'a>(uuid: &str, fin: &FIn<'a>) -> Option> { - if let Some(provider) = get_provider(uuid) { - Some(provider.check_app_available(fin).await) - } else { - None - } -} - -pub async fn get_latest_release<'a>(uuid: &str, fin: &FIn<'a>) -> Option> { - if let Some(provider) = get_provider(uuid) { - Some(provider.get_latest_release(fin).await) - } else { - None - } -} - -pub async fn get_releases<'a>(uuid: &str, fin: &FIn<'a>) -> Option>> { - if let Some(provider) = get_provider(uuid) { - Some(provider.get_releases(fin).await) - } else { - None - } -} - -pub async fn get_download<'a>( - uuid: &str, - fin: &FIn<'a>, - asset_index: &[i32], -) -> Option>> { - if let Some(provider) = get_provider(uuid) { - Some(provider.get_download(fin, asset_index).await) - } else { - None - } -} diff --git a/src/websdk/repo/provider/base_provider.rs b/src/websdk/repo/provider/base_provider.rs deleted file mode 100644 index 2dcec20..0000000 --- a/src/websdk/repo/provider/base_provider.rs +++ /dev/null @@ -1,427 +0,0 @@ -use async_trait::async_trait; - -use bytes::Bytes; -use core::fmt; -use regex::Regex; -use std::hash::{DefaultHasher, Hash, Hasher}; -use std::{ - collections::{BTreeMap, HashMap}, - error::Error, -}; - -use super::super::data::release::*; -use crate::rpc::data::DownloadItemData; - -pub type HubDataMap<'a> = BTreeMap<&'a str, &'a str>; -pub type AppDataMap<'a> = BTreeMap<&'a str, &'a str>; - -#[derive(Hash)] -pub struct DataMap<'a> { - pub app_data: &'a AppDataMap<'a>, - pub hub_data: &'a HubDataMap<'a>, -} - -impl fmt::Display for DataMap<'_> { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!( - f, - "app_data: {:?}, hub_data: {:?}", - self.app_data, self.hub_data - ) - } -} - -impl DataMap<'_> { - pub fn get_hash(&self) -> u64 { - let mut hasher = DefaultHasher::new(); - self.hash(&mut hasher); - hasher.finish() - } -} - -pub type CacheMap = HashMap; - -pub enum FunctionType { - CheckAppAvailable, - GetLatestRelease, - GetReleases, - GetDownload, -} - -pub struct FIn<'a> { - pub data_map: DataMap<'a>, - cache_map: Option>, -} - -impl<'a> FIn<'a> { - pub fn new_with_frag( - app_data: &'a AppDataMap<'a>, - hub_data: &'a HubDataMap<'a>, - cache_map: Option>, - ) -> Self { - FIn { - data_map: DataMap { app_data, hub_data }, - cache_map, - } - } - pub fn new(data_map: DataMap<'a>, cache_map: Option>) -> Self { - FIn { - data_map, - cache_map, - } - } - - pub fn get_cache(&self, key: &str) -> Option<&Bytes> { - if let Some(cache_map) = &self.cache_map { - if let Some(value) = cache_map.get(key) { - return Some(value); - } - } - None - } -} - -#[derive(Debug)] -pub struct FOut { - pub result: Result>, - pub cached_map: Option>, -} - -impl FOut { - pub fn new(data: T) -> Self { - FOut { - result: Ok(data), - cached_map: None, - } - } - - pub fn new_empty() -> Self { - FOut { - result: Err(Box::new(std::io::Error::other("no data"))), - cached_map: None, - } - } - - pub fn set_cache(mut self, key: &str, value: Bytes) -> Self { - let cache_map = self.cached_map.get_or_insert_with(HashMap::new); - cache_map.insert(key.to_string(), value); - self - } - - pub fn set_error(mut self, error: Box) -> Self { - self.result = Err(error); - self - } - - pub fn set_data(mut self, data: T) -> Self { - self.result = Ok(data); - self - } - - pub fn set_cached_map(mut self, cached_map: HashMap) -> Self { - self.cached_map = Some(cached_map); - self - } -} - -#[async_trait] -pub trait BaseProvider { - fn get_cache_request_key( - &self, - function_type: &FunctionType, - data_map: &DataMap, - ) -> Vec; - - async fn check_app_available(&self, fin: &FIn) -> FOut; - - async fn get_latest_release(&self, fin: &FIn) -> FOut { - let result = self.get_releases(fin).await; - - let fout_result = match result.result { - Ok(releases) => { - let release = releases.first().cloned(); - if let Some(release) = release { - Ok(release) - } else { - Err( - Box::new(std::io::Error::new(std::io::ErrorKind::NotFound, "no data")) - as Box, - ) - } - } - Err(e) => Err(e), - }; - - FOut { - result: fout_result, - cached_map: result.cached_map, - } - } - - async fn get_releases(&self, fin: &FIn) -> FOut>; - - /// Get download info for an app's asset. Default returns empty (not supported). - /// Only OutsideProvider (Kotlin hubs) needs to implement this. - async fn get_download(&self, _fin: &FIn, _asset_index: &[i32]) -> FOut> { - FOut::new_empty() - } -} - -pub trait BaseProviderExt: BaseProvider { - fn url_proxy_map(&self, fin: &FIn) -> HashMap { - let hub_data = fin.data_map.hub_data; - if let Some(proxy_map) = hub_data.get(REVERSE_PROXY) { - proxy_map - .lines() - .map(|line| { - let mut parts = line.splitn(2, "->"); - let url_prefix = parts.next().unwrap_or_default().trim(); - let proxy_url = parts.next().unwrap_or_default().trim(); - (url_prefix.to_string(), proxy_url.to_string()) - }) - .filter(|v| !v.0.is_empty() && !v.1.is_empty()) - .collect::>() - } else { - HashMap::new() - } - } - - fn replace_proxy_url(&self, fin: &FIn, url: &str) -> String { - let mut result_url = url.to_string(); - for (url_prefix, proxy_url) in self.url_proxy_map(fin).iter() { - let regex_prefix = "regex:"; - if let Some(stripped) = url_prefix.strip_prefix(regex_prefix) { - let url_prefix = &stripped.trim(); - if let Ok(re) = Regex::new(url_prefix) { - result_url = re.replace_all(&result_url, proxy_url.clone()).to_string(); - } - } else { - result_url = result_url.replace(url_prefix, proxy_url); - } - } - result_url - } -} - -pub const ANDROID_APP_TYPE: &str = "android_app_package"; -pub const ANDROID_MAGISK_MODULE_TYPE: &str = "android_magisk_module"; -pub const ANDROID_CUSTOM_SHELL: &str = "android_custom_shell"; -pub const ANDROID_CUSTOM_SHELL_ROOT: &str = "android_custom_shell_root"; - -pub const KEY_REPO_URL: &str = "repo_url"; -pub const KEY_REPO_API_URL: &str = "repo_api_url"; - -pub const REVERSE_PROXY: &str = "reverse_proxy"; - -#[cfg(test)] -mod tests { - use super::*; - - pub struct MockProvider; - - impl MockProvider { - pub fn new() -> MockProvider { - MockProvider {} - } - } - - #[async_trait] - impl BaseProvider for MockProvider { - fn get_cache_request_key( - &self, - function_type: &FunctionType, - data_map: &DataMap, - ) -> Vec { - let key_name = match function_type { - FunctionType::CheckAppAvailable => "check_app_available", - FunctionType::GetLatestRelease => "get_latest_release", - FunctionType::GetReleases => "get_releases", - FunctionType::GetDownload => "get_download", - }; - let id_map = data_map.app_data; - vec![format!( - "{}:{}", - key_name, - id_map - .iter() - .map(|(k, v)| format!("{}={}", k, v)) - .collect::>() - .join(",") - )] - } - - async fn check_app_available(&self, fin: &FIn) -> FOut { - let id_map = fin.data_map.app_data; - let cache_map = fin.cache_map.clone(); - FOut::new(cache_map.unwrap_or_default().contains_key(id_map["id"])) - } - - async fn get_releases(&self, fin: &FIn) -> FOut> { - let id_map = fin.data_map.app_data; - let cache_map = fin.cache_map.clone(); - FOut::new( - cache_map - .unwrap_or_default() - .get(id_map["id"]) - .unwrap() - .iter() - .map(|i| ReleaseData { - version_number: i.to_string(), - changelog: "".to_string(), - assets: vec![], - extra: None, - }) - .collect::>(), - ) - } - } - - impl BaseProviderExt for MockProvider {} - - #[tokio::test] - async fn test_get_cache_request_key() { - let mock = MockProvider::new(); - let data_map = DataMap { - app_data: &AppDataMap::from([("id", "123")]), - hub_data: &HubDataMap::new(), - }; - - let key = mock.get_cache_request_key(&FunctionType::CheckAppAvailable, &data_map); - assert_eq!(key, vec!["check_app_available:id=123"]); - let key = mock.get_cache_request_key(&FunctionType::GetReleases, &data_map); - assert_eq!(key, vec!["get_releases:id=123"]); - } - - #[tokio::test] - async fn test_check_app_available() { - let mock = MockProvider::new(); - let id_map = AppDataMap::from([("id", "123")]); - let cache_map = CacheMap::from([("123".to_string(), Bytes::from(vec![1u8]))]); - - let fin = FIn { - data_map: DataMap { - app_data: &id_map, - hub_data: &BTreeMap::new(), - }, - cache_map: Some(cache_map), - }; - - let available = mock.check_app_available(&fin).await; - assert_eq!(available.result.ok(), Some(true)); - } - - #[tokio::test] - async fn test_get_releases() { - let mock = MockProvider::new(); - let cache_map = CacheMap::from([("123".to_string(), Bytes::from(vec![1u8, 2u8, 3u8]))]); - - let fin = FIn { - data_map: DataMap { - app_data: &AppDataMap::from([("id", "123")]), - hub_data: &BTreeMap::new(), - }, - cache_map: Some(cache_map), - }; - - let releases = mock.get_releases(&fin).await; - assert_eq!(releases.result.unwrap().len(), 3); - } - - #[tokio::test] - async fn test_get_latest_release() { - let mock = MockProvider::new(); - let cache_map = CacheMap::from([("123".to_string(), Bytes::from(vec![1u8, 2u8, 3u8]))]); - - let fin = FIn { - data_map: DataMap { - app_data: &AppDataMap::from([("id", "123")]), - hub_data: &BTreeMap::new(), - }, - cache_map: Some(cache_map), - }; - - let latest_release = mock.get_latest_release(&fin).await; - let latest_version = latest_release.result.unwrap().version_number; - assert_eq!(latest_version, "1"); - } - - #[test] - fn test_replace_proxy_url() { - let mock = MockProvider::new(); - let url = "https://github.com"; - let url_r = "https://github.com.proxy"; - let proxy_url = format!("{} -> {}", url, url_r); - let data_map = HubDataMap::from([("reverse_proxy", proxy_url.as_str())]); - let result = mock.replace_proxy_url( - &FIn::new_with_frag(&AppDataMap::new(), &data_map, None), - url, - ); - assert_eq!(result, url_r); - - let proxy_url = format!("{}->{}", url, url_r); - let data_map = HubDataMap::from([("reverse_proxy", proxy_url.as_str())]); - let result = mock.replace_proxy_url( - &FIn::new_with_frag(&AppDataMap::new(), &data_map, None), - url, - ); - assert_eq!(result, url_r); - - let url_r = format!("{} -> proxy", url); - let proxy_url = format!(" {} ->{} ", url, url_r); - let data_map = HubDataMap::from([("reverse_proxy", proxy_url.as_str())]); - let result = mock.replace_proxy_url( - &FIn::new_with_frag(&AppDataMap::new(), &data_map, None), - url, - ); - assert_eq!(result, url_r); - } - - #[test] - fn test_replace_proxy_url_with_regex() { - let mock = MockProvider::new(); - - let regex_url = "regex:^https:.*/"; - let url_r = "https://github-proxy.com/"; - - let proxy_url = format!("{} -> {}", regex_url, url_r); - let data_map = HubDataMap::from([("reverse_proxy", proxy_url.as_str())]); - - let url = "https://github.com/GitHub"; - let expected_url = "https://github-proxy.com/GitHub"; - - let result = mock.replace_proxy_url( - &FIn::new_with_frag(&AppDataMap::new(), &data_map, None), - url, - ); - - assert_eq!( - result, expected_url, - "URL should be rewritten to use the proxy domain." - ); - - let non_matching_url = "http://example.com"; - let result = mock.replace_proxy_url( - &FIn::new_with_frag(&AppDataMap::new(), &data_map, None), - non_matching_url, - ); - - assert_eq!( - result, non_matching_url, - "Non-matching URL should not be rewritten." - ); - } - - #[test] - fn test_replace_proxy_url_multiple() { - let mock = MockProvider::new(); - let url = "https://github.com"; - let proxy_url = "https -> http\ngithub -> github-proxy"; - let url_r = "http://github-proxy.com"; - let data_map = HubDataMap::from([("reverse_proxy", proxy_url)]); - let result = mock.replace_proxy_url( - &FIn::new_with_frag(&AppDataMap::new(), &data_map, None), - url, - ); - assert_eq!(result, url_r); - } -} diff --git a/src/websdk/repo/provider/fdroid.rs b/src/websdk/repo/provider/fdroid.rs deleted file mode 100644 index 9e2f276..0000000 --- a/src/websdk/repo/provider/fdroid.rs +++ /dev/null @@ -1,356 +0,0 @@ -use async_trait::async_trait; -use bytes::Bytes; -use quick_xml::events::Event; -use quick_xml::Reader; -use std::collections::HashMap; -use std::error::Error; - -use crate::utils::http::{get, head, http_status_is_ok}; - -use super::super::data::release::*; -use super::base_provider::*; - -const FDROID_URL: &str = "https://f-droid.org"; - -pub struct FDroidProvider; - -impl FDroidProvider { - pub fn new() -> FDroidProvider { - FDroidProvider {} - } - - pub fn get_api_url(url: &str) -> String { - format!("{}/repo/index.xml", url) - } - - fn get_urls(data_map: &DataMap) -> (String, String) { - let url = data_map.hub_data.get(KEY_REPO_URL).unwrap_or(&FDROID_URL); - let api_url = if let Some(api_url) = data_map.hub_data.get(KEY_REPO_API_URL) { - api_url.to_string() - } else { - FDroidProvider::get_api_url(url) - }; - (url.to_string(), api_url) - } -} - -impl BaseProviderExt for FDroidProvider {} - -#[async_trait] -impl BaseProvider for FDroidProvider { - fn get_cache_request_key( - &self, - function_type: &FunctionType, - data_map: &DataMap, - ) -> Vec { - let (url, api_url) = FDroidProvider::get_urls(data_map); - let id_map = data_map.app_data; - match function_type { - FunctionType::CheckAppAvailable => vec![format!( - "{}/packages/{}/HEAD", - url, id_map[ANDROID_APP_TYPE] - )], - FunctionType::GetLatestRelease | FunctionType::GetReleases => vec![api_url.to_string()], - FunctionType::GetDownload => vec![], - } - } - - async fn check_app_available(&self, fin: &FIn) -> FOut { - let (url, _) = FDroidProvider::get_urls(&fin.data_map); - let id_map = fin.data_map.app_data; - let package_id = id_map[ANDROID_APP_TYPE]; - let api_url = format!("{}/packages/{}", url, package_id); - let api_url = self.replace_proxy_url(fin, &api_url); - - if let Ok(parsed_url) = api_url.parse() { - if let Ok(rsp) = head(parsed_url, &HashMap::new()).await { - return FOut::new(http_status_is_ok(rsp.status)); - } - } - FOut::new_empty() - } - - async fn get_releases(&self, fin: &FIn) -> FOut> { - let (url, api_url) = FDroidProvider::get_urls(&fin.data_map); - let id_map = fin.data_map.app_data; - let package_id = id_map[ANDROID_APP_TYPE]; - let api_url = self.replace_proxy_url(fin, &api_url); - let cache_key = self - .get_cache_request_key(&FunctionType::GetReleases, &fin.data_map) - .first() - .unwrap() - .clone(); - let mut cache_map_fout = CacheMap::new(); - let index_cache = fin.get_cache(&cache_key); - let mut index: Option = None; - if let Some(i) = index_cache { - index = Some(i.clone()); - } else if let Ok(parsed_url) = api_url.parse() { - if let Ok(rsp) = get(parsed_url, &HashMap::new()).await { - index = rsp.body; - cache_map_fout.insert(cache_key.to_string(), index.clone().unwrap()); - } - }; - if index.is_none() { - return FOut::new_empty(); - } - let mut releases_fout = Vec::new(); - if let Ok(content) = std::str::from_utf8(&index.unwrap()) { - let mut reader = Reader::from_str(content.trim()); - loop { - let (xml_package_id, releases) = - FDroidProvider::get_releases_from_xml(&mut reader, &url) - .await - .unwrap(); - if xml_package_id == package_id { - releases_fout = releases; - } - if xml_package_id.is_empty() { - break; - } - } - } - let mut fout = FOut::new(releases_fout); - if !cache_map_fout.is_empty() { - fout = fout.set_cached_map(cache_map_fout); - } - fout - } -} - -type Result = std::result::Result>; - -impl FDroidProvider { - async fn decode_package_xml(reader: &mut Reader<&[u8]>, url: &str) -> Result { - let xml_key = b"package"; - let mut version_number = String::new(); - let mut changelog = String::new(); - let mut file_name = String::new(); - let mut extra = HashMap::new(); - - let mut current_tag = String::new(); - loop { - match reader.read_event() { - Err(e) => return Err(Box::new(e)), - Ok(Event::Eof) => break, - Ok(Event::Start(e)) => { - let name = e.name(); - current_tag = String::from_utf8_lossy(name.as_ref()).to_string(); - } - Ok(Event::End(e)) => { - if e.name().as_ref() == xml_key { - break; - } - } - Ok(Event::Text(e)) => { - if let Ok(text) = e.decode() { - match current_tag.as_str() { - "version" => version_number += text.as_ref(), - "changelog" => changelog += text.as_ref(), - "versionCode" | "nativecode" => { - extra.insert(current_tag.clone(), text.to_string()); - } - "apkname" => file_name += text.as_ref(), - _ => (), - } - } - } - _ => (), - }; - } - let download_url = format!("{}/{}", url, file_name); - let file_type = file_name.split('.').next_back().unwrap_or("").to_string(); - - let extra = if extra.is_empty() { None } else { Some(extra) }; - Ok(ReleaseData { - version_number, - changelog, - assets: vec![AssetData { - file_name, - file_type, - download_url, - }], - extra, - }) - } - async fn decode_release_xml(reader: &mut Reader<&[u8]>, url: &str) -> Result> { - let mut releases = Vec::new(); - let mut changelog = String::new(); - - let mut current_tag = String::new(); - loop { - match reader.read_event() { - Err(e) => return Err(Box::new(e)), - Ok(Event::Eof) => break, - Ok(Event::Start(e)) => { - let name = e.name(); - match name.as_ref() { - b"package" => { - releases.push(FDroidProvider::decode_package_xml(reader, url).await?); - } - _ => { - current_tag = String::from_utf8_lossy(name.as_ref()).to_string(); - } - } - } - Ok(Event::End(e)) => { - if e.name().as_ref() == b"application" { - break; - } - } - Ok(Event::Text(e)) => { - if let Ok(text) = e.decode() { - if current_tag.as_str() == "changelog" { - changelog += text.as_ref() - } - } - } - _ => (), - }; - } - if !changelog.is_empty() { - if let Some(release) = releases.first_mut() { - release.changelog = changelog.clone(); - } - } - Ok(releases) - } - - async fn get_releases_from_xml( - reader: &mut Reader<&[u8]>, - url: &str, - ) -> Result<(String, Vec)> { - let mut package_id = String::new(); - loop { - match reader.read_event() { - Err(e) => return Err(Box::new(e)), - Ok(Event::Eof) => return Ok((package_id, Vec::new())), - Ok(Event::Start(e)) => { - if e.name().as_ref() == b"application" { - for attr in e.attributes().filter_map(|id| id.ok()) { - if attr.key.as_ref() == b"id" { - package_id = - String::from_utf8_lossy(attr.value.as_ref()).to_string(); - let releases = - FDroidProvider::decode_release_xml(reader, url).await?; - return Ok((package_id, releases)); - } - } - } - } - _ => (), - } - } - } -} - -#[cfg(test)] -mod tests { - use super::*; - use mockito::Server; - use std::fs; - - #[tokio::test] - async fn test_check_app_available() { - let package_id = "com.termux"; - let mut server = Server::new_async().await; - let _m = server - .mock("GET", format!("/packages/{}", package_id).as_str()) - .with_status(200) - .create_async() - .await; - - let provider = FDroidProvider::new(); - let app_data = AppDataMap::from([(ANDROID_APP_TYPE, package_id)]); - let proxy_url = format!("{} -> {}", FDROID_URL, server.url()); - let hub_data = HubDataMap::from([(REVERSE_PROXY, proxy_url.as_str())]); - let fin = FIn::new_with_frag(&app_data, &hub_data, None); - let fout = provider.check_app_available(&fin).await; - assert!(fout.result.unwrap()); - } - - #[tokio::test] - async fn test_check_app_available_nonexist() { - let package_id = "com.termux"; - let mut server = Server::new_async().await; - let _m = server - .mock("GET", format!("/packages/{}", package_id).as_str()) - .with_status(200) - .create_async() - .await; - - let provider = FDroidProvider::new(); - let nonexist_package_id = "nonexist"; - let app_data = AppDataMap::from([(ANDROID_APP_TYPE, nonexist_package_id)]); - let proxy_url = format!("{} -> {}", FDROID_URL, server.url()); - let hub_data = HubDataMap::from([(REVERSE_PROXY, proxy_url.as_str())]); - let fin = FIn::new_with_frag(&app_data, &hub_data, None); - let fout = provider.check_app_available(&fin).await; - assert!(!fout.result.unwrap()); - } - - #[tokio::test] - async fn test_get_releases() { - let body = fs::read_to_string("tests/files/web/f-droid.xml").unwrap(); - let mut server = Server::new_async().await; - let _m = server - .mock("GET", "/repo/index.xml") - .with_status(200) - .with_body(body) - .create(); - - let package_id = "org.fdroid.fdroid.privileged"; - let provider = FDroidProvider::new(); - let app_data = AppDataMap::from([(ANDROID_APP_TYPE, package_id)]); - let proxy_url = format!("{} -> {}", FDROID_URL, server.url()); - let hub_data = HubDataMap::from([(REVERSE_PROXY, proxy_url.as_str())]); - let fin = FIn::new_with_frag(&app_data, &hub_data, None); - let fout = provider.get_releases(&fin).await; - let releases = fout.result.unwrap(); - assert!(!releases.is_empty()); - assert_eq!(releases[0].assets[0].file_type, "apk"); - } - - #[tokio::test] - async fn test_get_releases_nonexist() { - let body = fs::read_to_string("tests/files/web/f-droid.xml").unwrap(); - let mut server = Server::new_async().await; - let _m = server - .mock("GET", "/repo/index.xml") - .with_status(200) - .with_body(body) - .create(); - - let package_id = "nonexist"; - let provider = FDroidProvider::new(); - let app_data = AppDataMap::from([(ANDROID_APP_TYPE, package_id)]); - let proxy_url = format!("{} -> {}", FDROID_URL, server.url()); - let hub_data = HubDataMap::from([(REVERSE_PROXY, proxy_url.as_str())]); - let fin = FIn::new_with_frag(&app_data, &hub_data, None); - let fout = provider.get_releases(&fin).await; - let releases = fout.result.unwrap(); - assert!(releases.is_empty()); - } - - #[tokio::test] - async fn test_get_releases_assets_type() { - let body = fs::read_to_string("tests/files/web/f-droid.xml").unwrap(); - let mut server = Server::new_async().await; - let _m = server - .mock("GET", "/repo/index.xml") - .with_status(200) - .with_body(body) - .create(); - - let package_id = "org.fdroid.fdroid.privileged.ota"; - let provider = FDroidProvider::new(); - let app_data = AppDataMap::from([(ANDROID_APP_TYPE, package_id)]); - let proxy_url = format!("{} -> {}", FDROID_URL, server.url()); - let hub_data = HubDataMap::from([(REVERSE_PROXY, proxy_url.as_str())]); - let fin = FIn::new_with_frag(&app_data, &hub_data, None); - let fout = provider.get_releases(&fin).await; - let releases = fout.result.unwrap(); - assert!(!releases.is_empty()); - assert_eq!(releases[0].assets[0].file_type, "zip"); - } -} diff --git a/src/websdk/repo/provider/github.rs b/src/websdk/repo/provider/github.rs deleted file mode 100644 index f7e8bc8..0000000 --- a/src/websdk/repo/provider/github.rs +++ /dev/null @@ -1,262 +0,0 @@ -use async_trait::async_trait; -use bytes::Bytes; -use serde_json::Value; -use std::collections::HashMap; - -use super::super::data::release::*; -use super::base_provider::*; - -use crate::utils::{ - http::{get, head, http_status_is_ok}, - versioning::Version, -}; - -pub const GITHUB_API_URL: &str = "https://api.github.com"; -const GITHUB_URL: &str = "https://github.com"; - -const VERSION_NUMBER_KEY: &str = "version_number_key"; -const VERSION_CODE_KEY: &str = "version_code_key"; - -pub struct GitHubProvider; - -impl GitHubProvider { - pub fn new() -> Self { - GitHubProvider {} - } - - fn get_token_header(&self, fin: &FIn) -> HashMap { - let mut map = HashMap::new(); - // check if token is empty or blank - let token = fin - .data_map - .hub_data - .get("token") - .filter(|t| !t.trim().is_empty()) - .or_else(|| fin.data_map.app_data.get("token")); - if let Some(token) = token { - map.insert("Authorization".to_string(), format!("Bearer {}", token)); - } - map - } -} - -impl BaseProviderExt for GitHubProvider {} - -#[async_trait] -impl BaseProvider for GitHubProvider { - fn get_cache_request_key( - &self, - function_type: &FunctionType, - data_map: &DataMap, - ) -> Vec { - let id_map = data_map.app_data; - match function_type { - FunctionType::CheckAppAvailable => vec![format!( - "{}/{}/{}/HEAD", - GITHUB_URL, id_map["owner"], id_map["repo"] - )], - FunctionType::GetLatestRelease | FunctionType::GetReleases => vec![format!( - "{}/repos/{}/{}/releases", - GITHUB_API_URL, id_map["owner"], id_map["repo"] - )], - FunctionType::GetDownload => vec![], - } - } - - async fn check_app_available(&self, fin: &FIn) -> FOut { - let id_map = fin.data_map.app_data; - let api_url = format!("{}/{}/{}", GITHUB_URL, id_map["owner"], id_map["repo"]); - let api_url = self.replace_proxy_url(fin, &api_url); - - if let Ok(parsed_url) = api_url.parse() { - if let Ok(rsp) = head(parsed_url, &HashMap::new()).await { - return FOut::new(http_status_is_ok(rsp.status)); - } - } - FOut::new_empty() - } - - async fn get_releases(&self, fin: &FIn) -> FOut> { - let id_map = fin.data_map.app_data; - let url = format!( - "{}/repos/{}/{}/releases", - GITHUB_API_URL, id_map["owner"], id_map["repo"] - ); - let url = self.replace_proxy_url(fin, &url); - let mut fout = FOut::new_empty(); - let cache_body = fin.get_cache(&url); - let mut rsp_body = None; - if cache_body.is_none() { - if let Ok(parsed_url) = url.parse() { - let header_map = { - let mut map = HashMap::new(); - map.insert("User-Agent".to_string(), "UpgradeAll-App".to_string()); - let token_map = self.get_token_header(fin); - map.extend(token_map); - map - }; - if let Ok(rsp) = get(parsed_url, &header_map).await { - if let Some(content) = rsp.body { - rsp_body = Some(content); - } - } - } - } - - let body: &Bytes; - if let Some(ref content) = rsp_body { - body = content; - } else if let Some(content) = cache_body { - body = content; - } else { - return fout; - } - - if let Ok(data) = serde_json::from_slice::>(body) { - let release_list = data - .iter() - .filter_map(|json| { - let assets_data = match json.get("assets") { - Some(assets) => assets - .as_array()? - .iter() - .filter_map(|asset| { - let file_name = asset.get("name")?.as_str()?.to_string(); - let file_type = asset.get("content_type")?.as_str()?.to_string(); - let download_url = - asset.get("browser_download_url")?.as_str()?.to_string(); - Some(AssetData { - file_name, - file_type, - download_url, - }) - }) - .collect(), - None => vec![], - }; - let mut keys_to_try = vec!["name", "tag_name"]; - if let Some(tag) = fin.data_map.hub_data.get(VERSION_NUMBER_KEY) { - keys_to_try.insert(0, tag); - } - let mut version_number: Option = None; - - for key in keys_to_try.iter() { - if let Some(value) = json.get(key).and_then(|v| v.as_str()) { - if Version::new(value.to_string()).is_valid() { - version_number = Some(value.to_string()); - break; - } - } - } - let changelog = json.get("body")?.as_str()?.to_string(); - - let mut extra = None; - if let Some(tag) = fin.data_map.hub_data.get(VERSION_CODE_KEY) { - if let Some(value) = json.get(tag) { - extra = Some(HashMap::from([(tag.to_string(), value.to_string())])); - } - } - Some(ReleaseData { - version_number: version_number?.to_string(), - changelog, - assets: assets_data, - extra, - }) - }) - .collect::>(); - fout = fout.set_data(release_list); - }; - - if let Some(content) = rsp_body { - fout.set_cached_map(HashMap::from([(url, content)])) - } else { - fout - } - } -} - -#[cfg(test)] -mod tests { - use super::*; - use mockito::Server; - use std::fs; - - #[tokio::test] - async fn test_check_app_available() { - let mut server = Server::new_async().await; - let _m = server - .mock("GET", "/DUpdateSystem/UpgradeAll") - .with_status(200) - .create_async() - .await; - - let id_map = AppDataMap::from([("owner", "DUpdateSystem"), ("repo", "UpgradeAll")]); - let proxy_url = format!("{} -> {}", GITHUB_URL, server.url()); - let hub_data = HubDataMap::from([(REVERSE_PROXY, proxy_url.as_str())]); - - let github_provider = GitHubProvider::new(); - assert!(github_provider - .check_app_available(&FIn::new_with_frag(&id_map, &hub_data, None)) - .await - .result - .unwrap()); - } - - #[tokio::test] - async fn test_get_releases() { - let body = fs::read_to_string("tests/files/web/github_api_release.json").unwrap(); - let mut server = Server::new_async().await; - let _m = server - .mock("GET", "/repos/DUpdateSystem/UpgradeAll/releases") - .with_status(200) - .with_body(body) - .create(); - - let id_map = AppDataMap::from([("owner", "DUpdateSystem"), ("repo", "UpgradeAll")]); - let proxy_url = format!("{} -> {}", GITHUB_API_URL, server.url()); - let hub_data = HubDataMap::from([(REVERSE_PROXY, proxy_url.as_str())]); - - let github_provider = GitHubProvider::new(); - let releases = github_provider - .get_releases(&FIn::new_with_frag(&id_map, &hub_data, None)) - .await - .result - .unwrap(); - - let release_json = - fs::read_to_string("tests/files/data/provider_github_release.json").unwrap(); - let releases_saved = serde_json::from_str::>(&release_json).unwrap(); - assert_eq!(releases, releases_saved) - } - - #[tokio::test] - async fn test_get_releases_token() { - let mut id_map = AppDataMap::from([("owner", "DUpdateSystem"), ("repo", "UpgradeAll")]); - let test_token = std::env::var("GITHUB_TOKEN"); - if test_token.is_err() { - return; - } - let test_token = test_token.unwrap(); - let hub_data = HubDataMap::from([("token", test_token.as_str())]); - - let github_provider = GitHubProvider::new(); - let releases = github_provider - .get_releases(&FIn::new_with_frag(&id_map, &hub_data, None)) - .await - .result - .unwrap(); - - assert!(!releases.is_empty()); - - let hub_data = HubDataMap::from([("token", " ")]); - id_map.insert("token", &test_token); - - let releases = github_provider - .get_releases(&FIn::new_with_frag(&id_map, &hub_data, None)) - .await - .result - .unwrap(); - - assert!(!releases.is_empty()); - } -} diff --git a/src/websdk/repo/provider/gitlab.rs b/src/websdk/repo/provider/gitlab.rs deleted file mode 100644 index d3a67ad..0000000 --- a/src/websdk/repo/provider/gitlab.rs +++ /dev/null @@ -1,327 +0,0 @@ -use async_trait::async_trait; -use bytes::Bytes; -use serde_json::Value; -use std::collections::HashMap; - -use super::super::data::release::*; -use super::base_provider::*; -use markdown::{mdast::Node, to_mdast, ParseOptions}; - -use crate::utils::{ - http::{get, head, http_status_is_ok}, - versioning::Version, -}; - -const GITLAB_URL: &str = "https://gitlab.com"; -const GITLAB_API_URL: &str = "https://gitlab.com/api/v4/projects"; - -const VERSION_NUMBER_KEY: &str = "version_number_key"; - -pub struct GitLabProvider; - -impl GitLabProvider { - pub fn new() -> GitLabProvider { - GitLabProvider {} - } -} - -impl BaseProviderExt for GitLabProvider {} - -impl GitLabProvider { - async fn get_project_id(&self, fin: &FIn<'_>) -> Option { - let id_map = fin.data_map.app_data; - let api_url = format!( - "{}/{}%2F{}", - GITLAB_API_URL, id_map["owner"], id_map["repo"] - ); - let api_url = self.replace_proxy_url(fin, &api_url); - - if let Ok(parsed_url) = api_url.parse() { - if let Ok(rsp) = get(parsed_url, &HashMap::new()).await { - if let Some(body) = rsp.body { - if let Ok(data) = serde_json::from_slice::>(&body) { - return Some(data.get("id")?.as_number()?.to_string()); - } - } - } - } - None - } - - fn try_get_download_url_from_changelog(&self, changelog: &str) -> Vec<(String, String)> { - let changelog_ast = to_mdast(changelog, &ParseOptions::default()); - if changelog_ast.is_err() { - return vec![]; - } - let changelog_ast = changelog_ast.unwrap(); - - fn get_link_text(nodes: &Vec) -> Vec<(String, String)> { - let mut url_map = vec![]; - for node in nodes { - let mut name = String::new(); - if let Node::Link(link) = node { - for c in &link.children { - if let Node::Text(text) = c { - name.push_str(&text.value); - } - } - url_map.push((name, link.url.to_string())); - } - if let Some(child) = node.children() { - url_map.extend(get_link_text(child)); - } - } - url_map - } - if let Some(children) = changelog_ast.children() { - return get_link_text(children); - } - vec![] - } - - fn fix_download_url(&self, download_url: &str, project_id: &str) -> String { - if download_url.starts_with("/uploads/") { - return format!("{}/-/project/{}{}", GITLAB_URL, project_id, download_url); - } - download_url.to_string() - } -} - -#[async_trait] -impl BaseProvider for GitLabProvider { - fn get_cache_request_key( - &self, - function_type: &FunctionType, - data_map: &DataMap, - ) -> Vec { - let id_map = data_map.app_data; - match function_type { - FunctionType::CheckAppAvailable => vec![format!( - "{}/{}/{}/HEAD", - GITLAB_URL, id_map["owner"], id_map["repo"] - )], - FunctionType::GetLatestRelease | FunctionType::GetReleases => vec![format!( - "{}/{}/{}/releases", - GITLAB_API_URL, id_map["owner"], id_map["repo"] - )], - FunctionType::GetDownload => vec![], - } - } - - async fn check_app_available(&self, fin: &FIn) -> FOut { - let id_map = fin.data_map.app_data; - let api_url = format!("{}/{}/{}", GITLAB_URL, id_map["owner"], id_map["repo"]); - let api_url = self.replace_proxy_url(fin, &api_url); - - if let Ok(parsed_url) = api_url.parse() { - if let Ok(rsp) = head(parsed_url, &HashMap::new()).await { - return FOut::new(http_status_is_ok(rsp.status)); - } - } - FOut::new_empty() - } - - async fn get_releases(&self, fin: &FIn) -> FOut> { - let id_map = fin.data_map.app_data; - let url = format!( - "{}/{}%2F{}/releases", - GITLAB_API_URL, id_map["owner"], id_map["repo"] - ); - let url = self.replace_proxy_url(fin, &url); - let mut fout = FOut::new_empty(); - let cache_body = fin.get_cache(&url); - let mut rsp_body = None; - if cache_body.is_none() { - if let Ok(parsed_url) = url.parse() { - let header_map = { - let mut map = HashMap::new(); - map.insert("User-Agent".to_string(), "Awesome-Octocat-App".to_string()); - map - }; - if let Ok(rsp) = get(parsed_url, &header_map).await { - if let Some(content) = rsp.body { - rsp_body = Some(content); - } - } - } - } - - let body: &Bytes; - if let Some(ref content) = rsp_body { - body = content; - } else if let Some(content) = cache_body { - body = content; - } else { - return fout; - } - - if let Ok(data) = serde_json::from_slice::>(body) { - let mut release_list = data - .iter() - .filter_map(|json| { - let assets_data = match json.get("assets")?.get("links") { - Some(links) => links - .as_array()? - .iter() - .filter_map(|asset| { - let file_name = asset.get("name")?.as_str()?.to_string(); - let file_type = asset.get("link_type")?.as_str()?.to_string(); - let download_url = asset.get("url")?.as_str()?.to_string(); - Some(AssetData { - file_name, - file_type, - download_url, - }) - }) - .collect(), - None => vec![], - }; - let mut keys_to_try = vec!["name", "tag_name"]; - if let Some(tag) = fin.data_map.hub_data.get(VERSION_NUMBER_KEY) { - keys_to_try.insert(0, tag); - } - let mut version_number: Option = None; - - for key in keys_to_try.iter() { - if let Some(value) = json.get(key).and_then(|v| v.as_str()) { - if Version::new(value.to_string()).is_valid() { - version_number = Some(value.to_string()); - break; - } - } - } - let changelog = json.get("description")?.as_str()?.to_string(); - let extra_download_url = self.try_get_download_url_from_changelog(&changelog); - let assets_data = assets_data - .into_iter() - .chain(extra_download_url.into_iter().map(|(k, v)| AssetData { - file_name: k, - file_type: "".to_string(), - download_url: v, - })) - .collect(); - Some(ReleaseData { - version_number: version_number?.to_string(), - changelog, - assets: assets_data, - extra: None, - }) - }) - .collect::>(); - let mut project_id = None; - for release in release_list.iter_mut() { - for asset in release.assets.iter_mut() { - if asset.download_url.starts_with("/uploads/") { - if project_id.is_none() { - project_id = self.get_project_id(fin).await; - } - if let Some(project_id) = &project_id { - asset.download_url = - self.fix_download_url(&asset.download_url, project_id); - } - } - } - } - fout = fout.set_data(release_list); - }; - - if let Some(content) = rsp_body { - fout.set_cached_map(HashMap::from([(url, content)])) - } else { - fout - } - } -} - -#[cfg(test)] -mod tests { - use super::*; - use mockito::Server; - use std::fs; - - #[tokio::test] - async fn test_check_app_available() { - let mut server = Server::new_async().await; - let _m = server - .mock("GET", "/fdroid/fdroidclient") - .with_status(200) - .create_async() - .await; - - let id_map = AppDataMap::from([("owner", "fdroid"), ("repo", "fdroidclient")]); - let proxy_url = format!("{} -> {}", GITLAB_URL, server.url()); - let hub_data = HubDataMap::from([(REVERSE_PROXY, proxy_url.as_str())]); - - let gitlab_provider = GitLabProvider::new(); - assert!(gitlab_provider - .check_app_available(&FIn::new_with_frag(&id_map, &hub_data, None)) - .await - .result - .unwrap()); - } - - #[tokio::test] - async fn test_get_releases() { - let body = fs::read_to_string("tests/files/web/gitlab_api_release.json").unwrap(); - let mut server = Server::new_async().await; - let _m = server - .mock("GET", "/fdroid%2Ffdroidclient/releases") - .with_status(200) - .with_body(body) - .create(); - - let id_map = AppDataMap::from([("owner", "fdroid"), ("repo", "fdroidclient")]); - let proxy_url = format!("{} -> {}", GITLAB_API_URL, server.url()); - let hub_data = HubDataMap::from([(REVERSE_PROXY, proxy_url.as_str())]); - - let gitlab_provider = GitLabProvider::new(); - let releases = gitlab_provider - .get_releases(&FIn::new_with_frag(&id_map, &hub_data, None)) - .await - .result - .unwrap(); - - let release_json = - fs::read_to_string("tests/files/data/provider_gitlab_release.json").unwrap(); - let releases_saved = serde_json::from_str::>(&release_json).unwrap(); - assert_eq!(releases, releases_saved) - } - - #[tokio::test] - async fn test_try_get_download_url_from_changelog_in_release() { - let body = - fs::read_to_string("tests/files/web/gitlab_api_release_AuroraStore.json").unwrap(); - let mut server = Server::new_async().await; - let _m = server - .mock("GET", "/AuroraOSS%2FAuroraStore/releases") - .with_status(200) - .with_body(body) - .create(); - let project_body = - fs::read_to_string("tests/files/web/gitlab_api_project_AuroraStore.json").unwrap(); - let _m = server - .mock("GET", "/AuroraOSS%2FAuroraStore") - .with_status(200) - .with_body(project_body) - .create_async() - .await; - - let id_map = AppDataMap::from([("owner", "AuroraOSS"), ("repo", "AuroraStore")]); - let proxy_url = format!("{} -> {}", GITLAB_API_URL, server.url()); - let hub_data = HubDataMap::from([(REVERSE_PROXY, proxy_url.as_str())]); - - let gitlab_provider = GitLabProvider::new(); - let releases = gitlab_provider - .get_releases(&FIn::new_with_frag(&id_map, &hub_data, None)) - .await - .result - .unwrap(); - - let release_json = - fs::read_to_string("tests/files/data/provider_gitlab_release_AuroraStore.json") - .unwrap(); - let releases_saved = serde_json::from_str::>(&release_json).unwrap(); - - assert_eq!(releases, releases_saved) - } -} diff --git a/src/websdk/repo/provider/lsposed_repo.rs b/src/websdk/repo/provider/lsposed_repo.rs deleted file mode 100644 index c7dc50c..0000000 --- a/src/websdk/repo/provider/lsposed_repo.rs +++ /dev/null @@ -1,224 +0,0 @@ -use std::collections::HashMap; - -use async_trait::async_trait; -use bytes::Bytes; -use serde_json::Value; - -use crate::utils::http::get; -use crate::utils::versioning::Version; - -use super::super::data::release::*; -use super::base_provider::*; - -const LSPOSED_REPO_API_URL: &str = "https://modules.lsposed.org/modules.json"; - -pub struct LsposedRepoProvider; - -impl LsposedRepoProvider { - pub fn new() -> Self { - LsposedRepoProvider {} - } - - fn get_app_json(package_name: &str, body: &Bytes) -> Option { - if let Ok(json) = serde_json::from_slice::>(body) { - for i in json { - if let Some(name) = i.get("name") { - if let Some(name_str) = name.as_str() { - if name_str == package_name { - return Some(i); - } - } - } - } - } - None - } -} - -impl BaseProviderExt for LsposedRepoProvider {} - -#[async_trait] -impl BaseProvider for LsposedRepoProvider { - fn get_cache_request_key( - &self, - _function_type: &FunctionType, - _data_map: &DataMap, - ) -> Vec { - vec![LSPOSED_REPO_API_URL.to_string()] - } - - async fn check_app_available(&self, fin: &FIn) -> FOut { - let url = self.replace_proxy_url(fin, LSPOSED_REPO_API_URL); - let mut fout = FOut::new_empty(); - let cache_body = fin.get_cache(&url); - let mut rsp_body = None; - if cache_body.is_none() { - if let Ok(parsed_url) = url.parse() { - let map = HashMap::new(); - if let Ok(rsp) = get(parsed_url, &map).await { - if let Some(content) = rsp.body { - fout = fout.set_cache(&url, content.clone()); - rsp_body = Some(content); - } - } - } - } - let body: &Bytes; - if let Some(ref content) = rsp_body { - body = content; - } else if let Some(content) = cache_body { - body = content; - } else { - return fout; - } - let id_map = fin.data_map.app_data; - let package_id = id_map[ANDROID_APP_TYPE]; - let json = LsposedRepoProvider::get_app_json(package_id, body); - fout.set_data(json.is_some()) - } - - async fn get_releases(&self, fin: &FIn) -> FOut> { - let url = self.replace_proxy_url(fin, LSPOSED_REPO_API_URL); - let mut fout = FOut::new_empty(); - let cache_body = fin.get_cache(&url); - let mut rsp_body = None; - if cache_body.is_none() { - if let Ok(parsed_url) = url.parse() { - let map = HashMap::new(); - if let Ok(rsp) = get(parsed_url, &map).await { - if let Some(content) = rsp.body { - fout = fout.set_cache(&url, content.clone()); - rsp_body = Some(content); - } - } - } - } - let body: &Bytes; - if let Some(ref content) = rsp_body { - body = content; - } else if let Some(content) = cache_body { - body = content; - } else { - return fout; - } - let id_map = fin.data_map.app_data; - let package_id = id_map[ANDROID_APP_TYPE]; - let json = LsposedRepoProvider::get_app_json(package_id, body); - if let Some(json) = json { - if let Some(releases_block) = json.get("releases") { - if let Some(release_json) = releases_block.as_array() { - let release_list = release_json - .iter() - .filter_map(|json| { - if let Some(assets_block) = json.get("releaseAssets") { - if let Some(assets) = assets_block.as_array() { - let assets_data = - assets - .iter() - .filter_map(|asset| { - Some(AssetData { - file_name: asset.get("name")?.as_str()?.to_string(), - file_type: asset - .get("contentType")? - .as_str() - .unwrap_or("application/vnd.android.package-archive") - .to_string(), - download_url: asset - .get("downloadUrl")? - .as_str()? - .to_string(), - }) - }) - .collect(); - let mut version_number: Option = None; - - for key in &["name", "tagName"] { - if let Some(value) = json.get(key).and_then(|v| v.as_str()) - { - if Version::new(value.to_string()).is_valid() { - version_number = Some(value.to_string()); - break; - } - } - } - let changelog = json - .get("descriptionHTML") - .and_then(|v| v.as_str()) - .unwrap_or("") - .to_string(); - return Some(ReleaseData { - version_number: version_number?.to_string(), - changelog, - assets: assets_data, - extra: None, - }); - } - } - None - }) - .collect(); - fout = fout.set_data(release_list); - } - } - } - fout - } -} - -#[cfg(test)] -mod tests { - use super::*; - use mockito::Server; - use std::fs; - - #[tokio::test] - async fn test_check_app_available() { - let body = fs::read_to_string("tests/files/web/lsposed_modules.json").unwrap(); - let mut server = Server::new_async().await; - let _m = server - .mock("GET", "/modules.json") - .with_status(200) - .with_body(body) - .create_async() - .await; - - let id_map = AppDataMap::from([(ANDROID_APP_TYPE, "com.agoines.relaxhelp")]); - let proxy_url = format!("{} -> {}", LSPOSED_REPO_API_URL, server.url()); - let hub_data = HubDataMap::from([("proxy_url", proxy_url.as_str())]); - - let lsposed_provider = LsposedRepoProvider::new(); - assert!(lsposed_provider - .check_app_available(&FIn::new_with_frag(&id_map, &hub_data, None)) - .await - .result - .unwrap()); - } - - #[tokio::test] - async fn test_get_releases() { - let body = fs::read_to_string("tests/files/web/lsposed_modules.json").unwrap(); - let mut server = Server::new_async().await; - let _m = server - .mock("GET", "/modules.json") - .with_status(200) - .with_body(body) - .create_async() - .await; - - let id_map = AppDataMap::from([(ANDROID_APP_TYPE, "com.agoines.relaxhelp")]); - let proxy_url = format!("{} -> {}", LSPOSED_REPO_API_URL, server.url()); - let hub_data = HubDataMap::from([("proxy_url", proxy_url.as_str())]); - - let lsposed_provider = LsposedRepoProvider::new(); - let releases = lsposed_provider - .get_releases(&FIn::new_with_frag(&id_map, &hub_data, None)) - .await - .result - .unwrap(); - - let release_json = - fs::read_to_string("tests/files/data/provider_lsposed_releases.json").unwrap(); - let releases_saved = serde_json::from_str::>(&release_json).unwrap(); - assert_eq!(releases, releases_saved) - } -} diff --git a/src/websdk/repo/provider/outside_rpc.rs b/src/websdk/repo/provider/outside_rpc.rs deleted file mode 100644 index ddfcc79..0000000 --- a/src/websdk/repo/provider/outside_rpc.rs +++ /dev/null @@ -1,115 +0,0 @@ -use super::super::data::release::*; -use super::base_provider::*; -use crate::rpc::client::*; -use crate::rpc::data::DownloadItemData; -use async_trait::async_trait; - -pub struct OutsideProvider { - pub uuid: String, - pub url: String, -} - -impl OutsideProvider { - pub fn new(uuid: String, url: String) -> Self { - OutsideProvider { uuid, url } - } -} - -#[async_trait] -impl BaseProvider for OutsideProvider { - fn get_cache_request_key( - &self, - _function_type: &FunctionType, - _data_map: &DataMap, - ) -> Vec { - vec![] - } - - async fn check_app_available(&self, fin: &FIn) -> FOut { - FOut { - result: match Client::new(&self.url).map(|client| async move { - client - .check_app_available( - &self.uuid, - fin.data_map.app_data.to_owned(), - fin.data_map.hub_data.to_owned(), - ) - .await - }) { - Ok(result) => match result.await { - Ok(result) => Ok(result), - Err(e) => Err(Box::new(e)), - }, - Err(e) => Err(Box::new(e)), - }, - cached_map: None, - } - } - - async fn get_latest_release(&self, fin: &FIn) -> FOut { - FOut { - result: match Client::new(&self.url).map(|client| async move { - client - .get_latest_release( - &self.uuid, - fin.data_map.app_data.to_owned(), - fin.data_map.hub_data.to_owned(), - ) - .await - }) { - Ok(result) => match result.await { - Ok(result) => Ok(result), - Err(e) => Err(Box::new(e)), - }, - Err(e) => Err(Box::new(e)), - }, - cached_map: None, - } - } - - async fn get_releases(&self, fin: &FIn) -> FOut> { - FOut { - result: match Client::new(&self.url).map(|client| async move { - client - .get_releases( - &self.uuid, - fin.data_map.app_data.to_owned(), - fin.data_map.hub_data.to_owned(), - ) - .await - }) { - Ok(result) => match result.await { - Ok(result) => Ok(result), - Err(e) => Err(Box::new(e)), - }, - Err(e) => Err(Box::new(e)), - }, - cached_map: None, - } - } - - async fn get_download(&self, fin: &FIn, asset_index: &[i32]) -> FOut> { - FOut { - result: match Client::new(&self.url).map(|client| async move { - client - .get_download( - &self.uuid, - fin.data_map.app_data.to_owned(), - fin.data_map.hub_data.to_owned(), - asset_index, - ) - .await - }) { - Ok(result) => match result.await { - Ok(result) => Ok(result), - Err(e) => Err(Box::new(e)), - }, - Err(e) => Err(Box::new(e)), - }, - cached_map: None, - } - } -} - -#[cfg(test)] -mod tests {} From 475c3b2823fb9fb56cde7430711b82c2cbe79097 Mon Sep 17 00:00:00 2001 From: Xiangzhe Date: Sun, 21 Jun 2026 23:53:58 +0800 Subject: [PATCH 02/52] feat(update): add deterministic update selection Add tokenized version comparison and selection policy that skips ignored versions while choosing the highest newer candidate. --- crates/getter-core/src/lib.rs | 1 + crates/getter-core/src/update.rs | 272 +++++++++++++++++++++++++++++++ 2 files changed, 273 insertions(+) create mode 100644 crates/getter-core/src/update.rs diff --git a/crates/getter-core/src/lib.rs b/crates/getter-core/src/lib.rs index 45a464d..da959a3 100644 --- a/crates/getter-core/src/lib.rs +++ b/crates/getter-core/src/lib.rs @@ -11,6 +11,7 @@ use std::str::FromStr; pub mod lua; pub mod repository; +pub mod update; /// Error returned when parsing or constructing a [`PackageId`]. #[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] diff --git a/crates/getter-core/src/update.rs b/crates/getter-core/src/update.rs new file mode 100644 index 0000000..ab3256c --- /dev/null +++ b/crates/getter-core/src/update.rs @@ -0,0 +1,272 @@ +//! Update selection helpers owned by getter core. + +use crate::{PackageId, SelectedUpdate, UpdateArtifact, UpdateCandidate}; +use std::cmp::Ordering; + +/// User state that affects update selection. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct UpdateSelectionPolicy { + /// Candidate version the user chose to ignore/mark as skipped. + pub ignored_version: Option, +} + +/// Compare human-facing version strings using a deterministic token ordering. +/// +/// This intentionally starts small: digit runs compare numerically, ASCII text +/// runs compare case-insensitively, common separators are ignored, and a text +/// suffix such as `beta` or `rc` sorts before the final release with the same +/// numeric prefix. +/// +/// Examples: +/// +/// - `1.10` > `1.2` +/// - `1.0.0` == `1.0` +/// - `1.0.0-beta` < `1.0.0` +pub fn compare_versions(left: &str, right: &str) -> Ordering { + let left = tokenize_version(left); + let right = tokenize_version(right); + let max_len = left.len().max(right.len()); + + for index in 0..max_len { + match (left.get(index), right.get(index)) { + (Some(left), Some(right)) => { + let ordering = left.cmp(right); + if ordering != Ordering::Equal { + return ordering; + } + } + (Some(VersionToken::Number(value)), None) if *value == 0 => continue, + (None, Some(VersionToken::Number(value))) if *value == 0 => continue, + (Some(VersionToken::Text(_)), None) => return Ordering::Less, + (None, Some(VersionToken::Text(_))) => return Ordering::Greater, + (Some(_), None) => return Ordering::Greater, + (None, Some(_)) => return Ordering::Less, + (None, None) => break, + } + } + + Ordering::Equal +} + +/// Select the best update candidate for a package. +/// +/// The selected candidate is the highest version that is newer than the +/// installed version and not equal to the user's ignored version. If no +/// installed version is known, the highest non-ignored candidate is selected. +pub fn select_update( + package_id: PackageId, + installed_version: Option<&str>, + candidates: &[UpdateCandidate], + policy: &UpdateSelectionPolicy, +) -> Option { + candidates + .iter() + .filter(|candidate| is_selectable(candidate, installed_version, policy)) + .max_by(|left, right| compare_versions(&left.version, &right.version)) + .cloned() + .map(|candidate| SelectedUpdate { + package_id, + artifact: first_artifact(&candidate), + candidate, + }) +} + +fn is_selectable( + candidate: &UpdateCandidate, + installed_version: Option<&str>, + policy: &UpdateSelectionPolicy, +) -> bool { + if policy + .ignored_version + .as_deref() + .is_some_and(|ignored| compare_versions(&candidate.version, ignored) == Ordering::Equal) + { + return false; + } + + match installed_version { + Some(installed) => compare_versions(&candidate.version, installed) == Ordering::Greater, + None => true, + } +} + +fn first_artifact(candidate: &UpdateCandidate) -> Option { + candidate.artifacts.first().cloned() +} + +#[derive(Debug, Clone, PartialEq, Eq)] +enum VersionToken { + Number(u128), + Text(String), +} + +impl Ord for VersionToken { + fn cmp(&self, other: &Self) -> Ordering { + match (self, other) { + (Self::Number(left), Self::Number(right)) => left.cmp(right), + (Self::Text(left), Self::Text(right)) => left.cmp(right), + (Self::Number(_), Self::Text(_)) => Ordering::Greater, + (Self::Text(_), Self::Number(_)) => Ordering::Less, + } + } +} + +impl PartialOrd for VersionToken { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +fn tokenize_version(version: &str) -> Vec { + let mut tokens = Vec::new(); + let mut current = String::new(); + let mut current_kind: Option = None; + + for ch in version.chars() { + let kind = TokenKind::from_char(ch); + match (kind, current_kind) { + (Some(kind), Some(existing)) if kind == existing => current.push(ch), + (Some(kind), Some(_)) => { + push_token(&mut tokens, ¤t, current_kind.expect("kind exists")); + current.clear(); + current.push(ch); + current_kind = Some(kind); + } + (Some(kind), None) => { + current.push(ch); + current_kind = Some(kind); + } + (None, Some(existing)) => { + push_token(&mut tokens, ¤t, existing); + current.clear(); + current_kind = None; + } + (None, None) => {} + } + } + + if let Some(kind) = current_kind { + push_token(&mut tokens, ¤t, kind); + } + + tokens +} + +fn push_token(tokens: &mut Vec, value: &str, kind: TokenKind) { + match kind { + TokenKind::Number => tokens.push(VersionToken::Number(parse_u128_lossy(value))), + TokenKind::Text => tokens.push(VersionToken::Text(value.to_ascii_lowercase())), + } +} + +fn parse_u128_lossy(value: &str) -> u128 { + value.parse::().unwrap_or(u128::MAX) +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum TokenKind { + Number, + Text, +} + +impl TokenKind { + fn from_char(ch: char) -> Option { + if ch.is_ascii_digit() { + Some(Self::Number) + } else if ch.is_ascii_alphabetic() { + Some(Self::Text) + } else { + None + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn compares_numeric_segments_numerically() { + assert_eq!(compare_versions("1.10", "1.2"), Ordering::Greater); + assert_eq!(compare_versions("1.0.0", "1.0"), Ordering::Equal); + assert_eq!(compare_versions("2", "10"), Ordering::Less); + } + + #[test] + fn treats_prerelease_suffix_as_older_than_release() { + assert_eq!(compare_versions("1.0.0-beta", "1.0.0"), Ordering::Less); + assert_eq!( + compare_versions("1.0.0-rc1", "1.0.0-beta2"), + Ordering::Greater + ); + } + + #[test] + fn selects_highest_newer_candidate() { + let selected = select_update( + "android/org.fdroid.fdroid".parse().unwrap(), + Some("1.0.0"), + &[candidate("1.0.1"), candidate("1.2.0"), candidate("0.9.0")], + &UpdateSelectionPolicy::default(), + ) + .unwrap(); + + assert_eq!(selected.candidate.version, "1.2.0"); + } + + #[test] + fn respects_ignored_version() { + let selected = select_update( + "android/org.fdroid.fdroid".parse().unwrap(), + Some("1.0.0"), + &[candidate("1.1.0"), candidate("1.2.0")], + &UpdateSelectionPolicy { + ignored_version: Some("1.2.0".to_owned()), + }, + ) + .unwrap(); + + assert_eq!(selected.candidate.version, "1.1.0"); + } + + #[test] + fn returns_none_when_no_candidate_is_newer() { + assert!(select_update( + "android/org.fdroid.fdroid".parse().unwrap(), + Some("2.0.0"), + &[candidate("1.9.0"), candidate("2.0.0")], + &UpdateSelectionPolicy::default(), + ) + .is_none()); + } + + #[test] + fn unknown_installed_version_selects_highest_candidate() { + let selected = select_update( + "android/org.fdroid.fdroid".parse().unwrap(), + None, + &[candidate("1.0.0-beta"), candidate("1.0.0")], + &UpdateSelectionPolicy::default(), + ) + .unwrap(); + + assert_eq!(selected.candidate.version, "1.0.0"); + assert_eq!( + selected.artifact.as_ref().unwrap().file_name.as_deref(), + Some("app.apk") + ); + } + + fn candidate(version: &str) -> UpdateCandidate { + UpdateCandidate { + version: version.to_owned(), + channel: None, + source: None, + artifacts: vec![UpdateArtifact { + name: "APK".to_owned(), + url: format!("https://example.invalid/{version}.apk"), + file_name: Some("app.apk".to_owned()), + }], + } + } +} From 2ca92b80cdc89ac9c18f45ba812ed52098d4c8f7 Mon Sep 17 00:00:00 2001 From: Xiangzhe Date: Mon, 22 Jun 2026 08:55:46 +0800 Subject: [PATCH 03/52] fix: satisfy clippy on stable Use sort_by_key for repository package ordering and elide needless lifetimes in highest_priority. --- crates/getter-core/src/repository.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/getter-core/src/repository.rs b/crates/getter-core/src/repository.rs index 0bacf47..fe9fa21 100644 --- a/crates/getter-core/src/repository.rs +++ b/crates/getter-core/src/repository.rs @@ -105,7 +105,7 @@ impl RepositoryLayout { let mut packages = Vec::new(); collect_package_files(&packages_dir, &packages_dir, &mut packages)?; - packages.sort_by(|a, b| a.id.to_string().cmp(&b.id.to_string())); + packages.sort_by_key(|package| package.id.to_string()); Ok(Self { root, @@ -224,7 +224,7 @@ pub fn package_id_from_path( }) } -pub fn highest_priority<'a, T, F>(items: &'a [T], priority: F) -> Option<&'a T> +pub fn highest_priority(items: &[T], priority: F) -> Option<&T> where F: Fn(&T) -> RepositoryPriority, { From 90c02bac7442e57d422ab0e2fd7e783c15bc11fe Mon Sep 17 00:00:00 2001 From: Xiangzhe Date: Mon, 22 Jun 2026 10:04:20 +0800 Subject: [PATCH 04/52] fix(lua): support lib-prefixed repository modules Preserve the documented require("lib.xxx") contract by extending the Lua package path while keeping unprefixed lib_dir lookups for compatibility. --- crates/getter-core/src/lua.rs | 44 ++++++++++++++++++++++++++++++++++- 1 file changed, 43 insertions(+), 1 deletion(-) diff --git a/crates/getter-core/src/lua.rs b/crates/getter-core/src/lua.rs index fd6348e..0583664 100644 --- a/crates/getter-core/src/lua.rs +++ b/crates/getter-core/src/lua.rs @@ -85,10 +85,14 @@ pub fn evaluate_package_source( fn configure_package_path(lua: &Lua, repository: &RepositoryLayout) -> mlua::Result<()> { let package: Table = lua.globals().get("package")?; let current_path: String = package.get("path")?; + let root_lib_pattern = repository.root.join("?.lua"); + let root_nested_lib_pattern = repository.root.join("?/init.lua"); let lib_pattern = repository.lib_dir.join("?.lua"); let nested_lib_pattern = repository.lib_dir.join("?/init.lua"); let new_path = format!( - "{};{};{}", + "{};{};{};{};{}", + root_lib_pattern.to_string_lossy(), + root_nested_lib_pattern.to_string_lossy(), lib_pattern.to_string_lossy(), nested_lib_pattern.to_string_lossy(), current_path @@ -457,4 +461,42 @@ return android.local_app { assert_eq!(package.name, "F-Droid"); assert_eq!(package.installed.len(), 1); } + + #[test] + fn require_can_load_repository_lib_modules_with_lib_prefix() { + let (_temp, layout, package_path) = fixture_repo(); + fs::write( + layout.lib_dir.join("android.lua"), + r#" +return { + local_app = function(input) + return { + id = input.id, + name = input.name, + installed = { + { kind = "android_package", package_name = input.package_name }, + }, + } + end +} +"#, + ) + .unwrap(); + fs::write( + &package_path, + r#" +local android = require("lib.android") +return android.local_app { + id = "android/org.fdroid.fdroid", + name = "F-Droid", + package_name = "org.fdroid.fdroid", +} +"#, + ) + .unwrap(); + + let package = evaluate_package_file(&layout, &package_path).unwrap(); + assert_eq!(package.name, "F-Droid"); + assert_eq!(package.installed.len(), 1); + } } From bf932d655727ad2a9fd1c856ac2a047db3c95d54 Mon Sep 17 00:00:00 2001 From: Xiangzhe Date: Mon, 22 Jun 2026 10:31:26 +0800 Subject: [PATCH 05/52] fix(lua): constrain lib-prefixed module loading Support the documented require("lib.*") contract with a constrained searcher instead of exposing the repository root through package.path. --- crates/getter-core/src/lua.rs | 97 ++++++++++++++++++++++++++++++++--- 1 file changed, 91 insertions(+), 6 deletions(-) diff --git a/crates/getter-core/src/lua.rs b/crates/getter-core/src/lua.rs index 0583664..b8e905b 100644 --- a/crates/getter-core/src/lua.rs +++ b/crates/getter-core/src/lua.rs @@ -85,19 +85,80 @@ pub fn evaluate_package_source( fn configure_package_path(lua: &Lua, repository: &RepositoryLayout) -> mlua::Result<()> { let package: Table = lua.globals().get("package")?; let current_path: String = package.get("path")?; - let root_lib_pattern = repository.root.join("?.lua"); - let root_nested_lib_pattern = repository.root.join("?/init.lua"); let lib_pattern = repository.lib_dir.join("?.lua"); let nested_lib_pattern = repository.lib_dir.join("?/init.lua"); let new_path = format!( - "{};{};{};{};{}", - root_lib_pattern.to_string_lossy(), - root_nested_lib_pattern.to_string_lossy(), + "{};{};{}", lib_pattern.to_string_lossy(), nested_lib_pattern.to_string_lossy(), current_path ); - package.set("path", new_path) + package.set("path", new_path)?; + install_lib_prefix_searcher(lua, &package, repository.lib_dir.clone()) +} + +fn install_lib_prefix_searcher(lua: &Lua, package: &Table, lib_dir: PathBuf) -> mlua::Result<()> { + let searchers: Table = package.get("searchers")?; + let searcher = lua.create_function(move |lua, module: String| { + let Some(module) = module.strip_prefix("lib.") else { + return lua + .create_string("\n\tconstrained repository lib searcher only handles lib.* modules") + .map(Value::String); + }; + + let Some(relative_module) = module_to_relative_path(module) else { + return lua + .create_string(format!( + "\n\tinvalid repository lib module name 'lib.{module}'" + )) + .map(Value::String); + }; + + let module_path = lib_dir.join(&relative_module).with_extension("lua"); + let init_path = lib_dir.join(&relative_module).join("init.lua"); + for candidate in [&module_path, &init_path] { + if candidate.is_file() { + let source = fs::read_to_string(candidate).map_err(mlua::Error::external)?; + let chunk = lua + .load(&source) + .set_name(candidate.to_string_lossy().as_ref()); + return chunk.into_function().map(Value::Function); + } + } + + lua.create_string(format!( + "\n\tno repository lib module 'lib.{module}' in {}", + lib_dir.display() + )) + .map(Value::String) + })?; + + let len = searchers.raw_len(); + for index in (2..=len).rev() { + let value: Value = searchers.raw_get(index)?; + searchers.raw_set(index + 1, value)?; + } + searchers.raw_set(2, searcher) +} + +fn module_to_relative_path(module: &str) -> Option { + let mut path = PathBuf::new(); + for part in module.split('.') { + if part.is_empty() + || part == ".." + || part.contains('/') + || part.contains('\\') + || part.contains(std::path::MAIN_SEPARATOR) + { + return None; + } + path.push(part); + } + if path.as_os_str().is_empty() { + None + } else { + Some(path) + } } fn install_helpers(lua: &Lua) -> mlua::Result<()> { @@ -499,4 +560,28 @@ return android.local_app { assert_eq!(package.name, "F-Droid"); assert_eq!(package.installed.len(), 1); } + + #[test] + fn lib_prefixed_searcher_does_not_expose_repository_templates() { + let (_temp, layout, package_path) = fixture_repo(); + fs::write( + layout.templates_dir.join("android.lua"), + r#"return { leaked = true }"#, + ) + .unwrap(); + fs::write( + &package_path, + r#" +local ok = pcall(require, "templates.android") +return { + id = "android/org.fdroid.fdroid", + name = ok and "leaked" or "F-Droid", +} +"#, + ) + .unwrap(); + + let package = evaluate_package_file(&layout, &package_path).unwrap(); + assert_eq!(package.name, "F-Droid"); + } } From f746c96161de03548e2ebaccf57e7800a98becc7 Mon Sep 17 00:00:00 2001 From: Xiangzhe Date: Mon, 22 Jun 2026 11:01:13 +0800 Subject: [PATCH 06/52] fix(lua): avoid default cwd module leakage Do not retain Lua's default package.path for package evaluation. Keep repository lib loading explicit and add coverage for root/template leakage when cwd is the repository root. --- crates/getter-core/src/lua.rs | 41 +++++++++++++++++++++++++++++++---- 1 file changed, 37 insertions(+), 4 deletions(-) diff --git a/crates/getter-core/src/lua.rs b/crates/getter-core/src/lua.rs index b8e905b..c6b8f96 100644 --- a/crates/getter-core/src/lua.rs +++ b/crates/getter-core/src/lua.rs @@ -84,14 +84,12 @@ pub fn evaluate_package_source( fn configure_package_path(lua: &Lua, repository: &RepositoryLayout) -> mlua::Result<()> { let package: Table = lua.globals().get("package")?; - let current_path: String = package.get("path")?; let lib_pattern = repository.lib_dir.join("?.lua"); let nested_lib_pattern = repository.lib_dir.join("?/init.lua"); let new_path = format!( - "{};{};{}", + "{};{}", lib_pattern.to_string_lossy(), - nested_lib_pattern.to_string_lossy(), - current_path + nested_lib_pattern.to_string_lossy() ); package.set("path", new_path)?; install_lib_prefix_searcher(lua, &package, repository.lib_dir.clone()) @@ -584,4 +582,39 @@ return { let package = evaluate_package_file(&layout, &package_path).unwrap(); assert_eq!(package.name, "F-Droid"); } + + #[test] + fn repository_root_is_not_exposed_even_when_cwd_is_repository_root() { + let (_temp, layout, package_path) = fixture_repo(); + fs::write( + layout.root.join("rootleak.lua"), + r#"return { leaked = true }"#, + ) + .unwrap(); + fs::write( + layout.templates_dir.join("android.lua"), + r#"return { leaked = true }"#, + ) + .unwrap(); + fs::write( + &package_path, + r#" +local root_ok = pcall(require, "rootleak") +local template_ok = pcall(require, "templates.android") +return { + id = "android/org.fdroid.fdroid", + name = (root_ok or template_ok) and "leaked" or "F-Droid", +} +"#, + ) + .unwrap(); + + let original_cwd = std::env::current_dir().unwrap(); + std::env::set_current_dir(&layout.root).unwrap(); + let result = evaluate_package_file(&layout, &package_path); + std::env::set_current_dir(original_cwd).unwrap(); + + let package = result.unwrap(); + assert_eq!(package.name, "F-Droid"); + } } From 3b7613d709b405cb7229f2fbbf546c2d29ee96e6 Mon Sep 17 00:00:00 2001 From: Xiangzhe Date: Mon, 22 Jun 2026 11:26:45 +0800 Subject: [PATCH 07/52] fix(android): keep JNI facade free of Lua domain deps Make getter domain crates an optional feature so the Android api_proxy facade can build without vendored Lua on armeabi-v7a. --- Cargo.toml | 7 ++++--- src/lib.rs | 2 ++ 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 500b0f1..e45af61 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -21,7 +21,8 @@ version = "0.1.0" edition = "2021" [features] -default = ["cli"] +default = ["cli", "domain"] +domain = ["dep:getter-core", "dep:getter-storage"] cli = ["dep:getter-cli"] rustls-platform-verifier = ["dep:rustls-platform-verifier"] rustls-platform-verifier-android = ["rustls-platform-verifier", "rustls-platform-verifier/jni"] @@ -29,8 +30,8 @@ webpki-roots = [] native-tokio = [] [dependencies] -getter-core = { path = "crates/getter-core" } -getter-storage = { path = "crates/getter-storage" } +getter-core = { path = "crates/getter-core", optional = true } +getter-storage = { path = "crates/getter-storage", optional = true } getter-cli = { path = "crates/getter-cli", optional = true } thiserror = "1" tokio = { version = "1", features = ["net"] } diff --git a/src/lib.rs b/src/lib.rs index 4ffc0b5..607df46 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -4,7 +4,9 @@ //! behavior lives in split crates such as `getter-core` and `getter-storage`; //! Android/Flutter hosts embed this crate through a stable facade. +#[cfg(feature = "domain")] pub use getter_core as core; +#[cfg(feature = "domain")] pub use getter_storage as storage; #[cfg(feature = "rustls-platform-verifier")] From 7357980bc5972288db1817ede5610b5974b1546e Mon Sep 17 00:00:00 2001 From: Xiangzhe Date: Mon, 22 Jun 2026 16:41:26 +0800 Subject: [PATCH 08/52] feat(cli): expose legacy migration reports Add a legacy report-list command so app and test adapters can consume sanitized migration reports through the getter JSON envelope instead of reading the data directory layout directly. --- README.md | 2 +- crates/getter-cli/src/lib.rs | 56 +++++++++++++++++++ crates/getter-cli/tests/bdd_cli.rs | 21 +++++++ .../legacy_import_room_bundle_failure.feature | 2 + 4 files changed, 80 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 34543a0..6fc932d 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ This repository is intentionally usable outside the UpgradeAll UI. The UpgradeAl - Package IDs are readable, for example `android/org.fdroid.fdroid`. - Lua package repositories use `repo.toml` plus `packages/`, `lib/`, and `templates/` directories. - SQLite storage uses a main DB and a separate cache DB. -- `getter-cli` exposes JSON command contracts for init, app list, repository registration/evaluation, package evaluation, storage validation, and legacy bridge-bundle import. +- `getter-cli` exposes JSON command contracts for init, app list, repository registration/evaluation, package evaluation, storage validation, legacy bridge-bundle import, and sanitized legacy report listing. ## Verify diff --git a/crates/getter-cli/src/lib.rs b/crates/getter-cli/src/lib.rs index 8e0155e..48d240d 100644 --- a/crates/getter-cli/src/lib.rs +++ b/crates/getter-cli/src/lib.rs @@ -51,6 +51,7 @@ pub enum CliCommand { LegacyImportRoomBundle { bundle: PathBuf, }, + LegacyReportList, } #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -262,6 +263,9 @@ where bundle: PathBuf::from(bundle), } } + [domain, command] if domain == "legacy" && command == "report-list" => { + CliCommand::LegacyReportList + } _ => { return Err(CliError::Usage(format!( "unsupported command: {}", @@ -404,6 +408,10 @@ fn execute(invocation: CliInvocation) -> Result { "apps": tracked_packages_json(records), })) } + CliCommand::LegacyReportList => { + open_initialized_storage(&invocation.data_dir)?; + Ok(json!({ "reports": list_migration_reports(&invocation.data_dir)? })) + } } } @@ -647,6 +655,53 @@ fn report_file_name(code: &str) -> String { format!("{}.json", code.replace('.', "-")) } +fn list_migration_reports(data_dir: &Path) -> Result, CliError> { + let reports_dir = data_dir.join(MIGRATION_REPORTS_DIR); + if !reports_dir.exists() { + return Ok(Vec::new()); + } + + let mut report_paths = fs::read_dir(&reports_dir) + .map_err(|source| CliError::Storage(format!("failed to read migration reports: {source}")))? + .map(|entry| entry.map(|entry| entry.path())) + .collect::, _>>() + .map_err(|source| { + CliError::Storage(format!("failed to read migration report entry: {source}")) + })?; + report_paths.retain(|path| { + path.extension() + .is_some_and(|extension| extension == "json") + }); + report_paths.sort(); + + report_paths + .into_iter() + .map(|path| { + let bytes = fs::read(&path).map_err(|source| { + CliError::Storage(format!( + "failed to read migration report '{}': {source}", + path.display() + )) + })?; + let report: Value = serde_json::from_slice(&bytes).map_err(|source| { + CliError::Storage(format!( + "failed to parse migration report '{}': {source}", + path.display() + )) + })?; + Ok(json!({ + "ok": report.get("ok").and_then(Value::as_bool).unwrap_or(false), + "code": report.get("code").and_then(Value::as_str).unwrap_or("migration.unknown"), + "message": report.get("message").and_then(Value::as_str).unwrap_or("Legacy migration report"), + "bundle_file_name": report.get("bundle_file_name").and_then(Value::as_str), + "imported_records": report.get("imported_records").and_then(Value::as_u64).unwrap_or(0), + "tracked_records": report.get("tracked_records").and_then(Value::as_u64).unwrap_or(0), + "report_path": path, + })) + }) + .collect() +} + #[derive(Debug, Serialize)] struct MigrationReport<'a> { ok: bool, @@ -746,6 +801,7 @@ impl CliCommand { Self::PackageEval { .. } => "package eval", Self::StorageValidate => "storage validate", Self::LegacyImportRoomBundle { .. } => "legacy import-room-bundle", + Self::LegacyReportList => "legacy report-list", } } } diff --git a/crates/getter-cli/tests/bdd_cli.rs b/crates/getter-cli/tests/bdd_cli.rs index 807aaed..28ed839 100644 --- a/crates/getter-cli/tests/bdd_cli.rs +++ b/crates/getter-cli/tests/bdd_cli.rs @@ -190,6 +190,13 @@ fn run_getter_legacy_import(world: &mut CliWorld) { world.json = None; } +#[when("I run getter legacy report-list for that directory")] +fn run_getter_legacy_report_list(world: &mut CliWorld) { + let output = run_getter(world, ["legacy".to_owned(), "report-list".to_owned()]); + world.output = Some(output); + world.json = None; +} + #[then("the command succeeds")] fn command_succeeds(world: &mut CliWorld) { let output = world.output.as_ref().expect("command output exists"); @@ -356,6 +363,20 @@ fn import_reports_one_tracked_app(world: &mut CliWorld) { ); } +#[then(expr = "the output lists migration report {string}")] +fn output_lists_migration_report(world: &mut CliWorld, code: String) { + let json = current_json(world); + assert_eq!(json["ok"], true); + assert_eq!(json["command"], "legacy report-list"); + let reports = json["data"]["reports"].as_array().expect("reports array"); + assert!( + reports + .iter() + .any(|report| report["code"].as_str() == Some(code.as_str())), + "reports should contain {code}: {reports:?}" + ); +} + #[then(expr = "the app list contains imported package {string}")] fn app_list_contains_imported_package(world: &mut CliWorld, package_id: String) { let output = run_getter(world, ["app".to_owned(), "list".to_owned()]); diff --git a/crates/getter-cli/tests/features/cli/legacy_import_room_bundle_failure.feature b/crates/getter-cli/tests/features/cli/legacy_import_room_bundle_failure.feature index 864417f..02b6e8b 100644 --- a/crates/getter-cli/tests/features/cli/legacy_import_room_bundle_failure.feature +++ b/crates/getter-cli/tests/features/cli/legacy_import_room_bundle_failure.feature @@ -16,3 +16,5 @@ Feature: Legacy import failure recovery And the output is valid JSON And the import reports one tracked app And the app list contains imported package "android/org.fdroid.fdroid" + When I run getter legacy report-list for that directory + Then the output lists migration report "migration.imported" From 6be047ac13d6dc4dd94dc29fbe6c4f622acd640f Mon Sep 17 00:00:00 2001 From: Xiangzhe Date: Mon, 22 Jun 2026 16:58:33 +0800 Subject: [PATCH 09/52] fix(cli): keep migration report list sanitized Do not expose getter data-directory report paths in legacy report-list output; callers only need the sanitized report fields. --- crates/getter-cli/src/lib.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/crates/getter-cli/src/lib.rs b/crates/getter-cli/src/lib.rs index 48d240d..b13bd02 100644 --- a/crates/getter-cli/src/lib.rs +++ b/crates/getter-cli/src/lib.rs @@ -696,7 +696,6 @@ fn list_migration_reports(data_dir: &Path) -> Result, CliError> { "bundle_file_name": report.get("bundle_file_name").and_then(Value::as_str), "imported_records": report.get("imported_records").and_then(Value::as_u64).unwrap_or(0), "tracked_records": report.get("tracked_records").and_then(Value::as_u64).unwrap_or(0), - "report_path": path, })) }) .collect() From 13816863b20744beadab7bf10153afa2156e3010 Mon Sep 17 00:00:00 2001 From: Xiangzhe Date: Mon, 22 Jun 2026 20:19:23 +0800 Subject: [PATCH 10/52] feat(cli): add offline repo validation Expose repo validate through the getter JSON envelope with structured getter-owned diagnostics for repository layout, Lua runtime, schema, and domain errors. Add BDD coverage and cache-key hashing for repository changes. --- crates/getter-cli/src/lib.rs | 18 +- crates/getter-cli/tests/bdd_cli.rs | 134 ++++++++ .../tests/features/cli/repo_validate.feature | 35 ++ crates/getter-core/src/diagnostics.rs | 309 ++++++++++++++++++ crates/getter-core/src/lib.rs | 1 + crates/getter-core/src/repository.rs | 86 ++++- 6 files changed, 581 insertions(+), 2 deletions(-) create mode 100644 crates/getter-cli/tests/features/cli/repo_validate.feature create mode 100644 crates/getter-core/src/diagnostics.rs diff --git a/crates/getter-cli/src/lib.rs b/crates/getter-cli/src/lib.rs index b13bd02..22fb675 100644 --- a/crates/getter-cli/src/lib.rs +++ b/crates/getter-cli/src/lib.rs @@ -4,6 +4,7 @@ //! while durable state is initialized through `getter-storage` so the command //! surface exercises the same Rust-owned SQLite direction used by embedders. +use getter_core::diagnostics::validate_repository_path; use getter_core::lua::evaluate_package_file; use getter_core::repository::{RepositoryLayout, RepositoryMetadata}; use getter_core::{PackageId, RepositoryId, RepositoryPriority}; @@ -43,6 +44,9 @@ pub enum CliCommand { RepoEval { id: RepositoryId, }, + RepoValidate { + path: PathBuf, + }, PackageEval { package_id: PackageId, repo_id: Option, @@ -241,6 +245,11 @@ where [domain, command, id] if domain == "repo" && command == "eval" => CliCommand::RepoEval { id: parse_repository_id(id)?, }, + [domain, command, path] if domain == "repo" && command == "validate" => { + CliCommand::RepoValidate { + path: PathBuf::from(path), + } + } [domain, command, package_id] if domain == "package" && command == "eval" => { CliCommand::PackageEval { package_id: parse_package_id(package_id)?, @@ -331,6 +340,12 @@ fn execute(invocation: CliInvocation) -> Result { "packages": packages, })) } + CliCommand::RepoValidate { path } => { + open_initialized_storage(&invocation.data_dir)?; + serde_json::to_value(validate_repository_path(path)).map_err(|source| { + CliError::Repository(format!("failed to serialize validation report: {source}")) + }) + } CliCommand::PackageEval { package_id, repo_id, @@ -747,7 +762,7 @@ fn envelope_to_string(value: Value) -> String { } fn usage_text() -> String { - "Usage: getter --data-dir [--priority ]|repo eval |package eval [--repo ]|storage validate|hub list|legacy import-room-bundle >\n".to_owned() + "Usage: getter --data-dir [--priority ]|repo eval |repo validate |package eval [--repo ]|storage validate|hub list|legacy import-room-bundle |legacy report-list>\n".to_owned() } #[derive(Debug, Deserialize)] @@ -797,6 +812,7 @@ impl CliCommand { Self::RepoList => "repo list", Self::RepoAdd { .. } => "repo add", Self::RepoEval { .. } => "repo eval", + Self::RepoValidate { .. } => "repo validate", Self::PackageEval { .. } => "package eval", Self::StorageValidate => "storage validate", Self::LegacyImportRoomBundle { .. } => "legacy import-room-bundle", diff --git a/crates/getter-cli/tests/bdd_cli.rs b/crates/getter-cli/tests/bdd_cli.rs index 28ed839..4f084c7 100644 --- a/crates/getter-cli/tests/bdd_cli.rs +++ b/crates/getter-cli/tests/bdd_cli.rs @@ -80,6 +80,61 @@ fn fixture_lua_repository_named( create_fixture_lua_repository(world, repo_id, package_id, package_name); } +#[given(expr = "a fixture Lua repository {string} with invalid Lua package {string}")] +fn fixture_lua_repository_invalid_lua(world: &mut CliWorld, repo_id: String, package_id: String) { + create_custom_fixture_lua_repository( + world, + repo_id, + package_id.clone(), + format!("return package_def {{ id = \"{package_id}\", name = "), + ); +} + +#[given(expr = "a fixture Lua repository {string} with schema-invalid package {string}")] +fn fixture_lua_repository_invalid_schema( + world: &mut CliWorld, + repo_id: String, + package_id: String, +) { + create_custom_fixture_lua_repository( + world, + repo_id, + package_id.clone(), + format!("return {{ id = \"{package_id}\" }}"), + ); +} + +#[given(expr = "a fixture Lua repository {string} with mismatched package path {string}")] +fn fixture_lua_repository_mismatched_path( + world: &mut CliWorld, + repo_id: String, + package_id: String, +) { + create_custom_fixture_lua_repository( + world, + repo_id, + package_id, + "return { id = \"android/com.termux\", name = \"Termux\" }".to_owned(), + ); +} + +#[given(expr = "an incomplete Lua repository {string}")] +fn incomplete_lua_repository(world: &mut CliWorld, repo_id: String) { + let temp = world.temp.as_ref().expect("tempdir exists"); + let repo_path = temp.path().join(format!("repo-{repo_id}")); + fs::create_dir_all(&repo_path).expect("create incomplete repo dir"); + fs::write( + repo_path.join("repo.toml"), + format!( + "id = \"{repo_id}\"\nname = \"Fixture {repo_id}\"\npriority = 0\napi_version = \"getter.repo.v1\"\n" + ), + ) + .expect("write repo.toml"); + world.fixture_repo_id = Some(repo_id); + world.fixture_repo_path = Some(repo_path); + world.fixture_package_id = None; +} + #[when("I run getter init for that directory")] fn run_getter_init(world: &mut CliWorld) { let output = run_getter(world, ["init".to_owned()]); @@ -144,6 +199,24 @@ fn run_getter_repo_eval(world: &mut CliWorld) { world.json = None; } +#[when("I run getter repo validate for that repository")] +fn run_getter_repo_validate(world: &mut CliWorld) { + let repo_path = world + .fixture_repo_path + .as_ref() + .expect("fixture repo path exists"); + let output = run_getter( + world, + [ + "repo".to_owned(), + "validate".to_owned(), + repo_path.to_string_lossy().to_string(), + ], + ); + world.output = Some(output); + world.json = None; +} + #[when("I run getter package eval for that fixture package")] fn run_getter_package_eval(world: &mut CliWorld) { let repo_id = world @@ -282,6 +355,35 @@ fn output_contains_added_repository(world: &mut CliWorld) { assert_eq!(json["data"]["repository"]["id"], repo_id); } +#[then("the output reports a valid repository without network")] +fn output_reports_valid_repository_without_network(world: &mut CliWorld) { + let json = current_json(world); + assert_eq!(json["ok"], true); + assert_eq!(json["command"], "repo validate"); + assert_eq!(json["data"]["valid"], true); + assert_eq!(json["data"]["network_required"], false); + assert_eq!(json["data"]["diagnostics"], Value::Array(Vec::new())); + assert_eq!(json["data"]["package_count"], 1); +} + +#[then(expr = "the output reports repository diagnostic {string}")] +fn output_reports_repository_diagnostic(world: &mut CliWorld, code: String) { + let json = current_json(world); + assert_eq!(json["ok"], true); + assert_eq!(json["command"], "repo validate"); + assert_eq!(json["data"]["valid"], false); + assert_eq!(json["data"]["network_required"], false); + let diagnostics = json["data"]["diagnostics"] + .as_array() + .expect("diagnostics array"); + assert!( + diagnostics + .iter() + .any(|diagnostic| diagnostic["code"].as_str() == Some(code.as_str())), + "diagnostics should contain {code}: {diagnostics:?}" + ); +} + #[then("the output contains the evaluated fixture package")] fn output_contains_evaluated_fixture_package(world: &mut CliWorld) { let package_id = world @@ -442,6 +544,38 @@ return package_def {{ world.fixture_package_id = Some(package_id); } +fn create_custom_fixture_lua_repository( + world: &mut CliWorld, + repo_id: String, + package_id: String, + package_source: String, +) { + let temp = world.temp.as_ref().expect("tempdir exists"); + let repo_path = temp.path().join(format!("repo-{repo_id}")); + let package_name_path = package_id + .strip_prefix("android/") + .expect("fixture package id should be android package id"); + fs::create_dir_all(repo_path.join("packages/android")).expect("create packages dir"); + fs::create_dir(repo_path.join("lib")).expect("create lib dir"); + fs::create_dir(repo_path.join("templates")).expect("create templates dir"); + fs::write( + repo_path.join("repo.toml"), + format!( + "id = \"{repo_id}\"\nname = \"Fixture {repo_id}\"\npriority = 0\napi_version = \"getter.repo.v1\"\n" + ), + ) + .expect("write repo.toml"); + fs::write( + repo_path.join(format!("packages/android/{package_name_path}.lua")), + package_source, + ) + .expect("write package Lua"); + + world.fixture_repo_id = Some(repo_id); + world.fixture_repo_path = Some(repo_path); + world.fixture_package_id = Some(package_id); +} + fn run_getter_repo_add_with_priority(world: &mut CliWorld, repo_id: &str, priority: i32) { let temp = world.temp.as_ref().expect("tempdir exists"); let repo_path = temp.path().join(format!("repo-{repo_id}")); diff --git a/crates/getter-cli/tests/features/cli/repo_validate.feature b/crates/getter-cli/tests/features/cli/repo_validate.feature new file mode 100644 index 0000000..592d4bf --- /dev/null +++ b/crates/getter-cli/tests/features/cli/repo_validate.feature @@ -0,0 +1,35 @@ +Feature: Getter CLI repository validation + Scenario: User validates a fixture Lua repository offline + Given an initialized getter data directory + And a fixture Lua repository "official" with package "android/org.fdroid.fdroid" + When I run getter repo validate for that repository + Then the command succeeds + And the output reports a valid repository without network + + Scenario: User receives diagnostics for invalid Lua + Given an initialized getter data directory + And a fixture Lua repository "broken" with invalid Lua package "android/org.fdroid.fdroid" + When I run getter repo validate for that repository + Then the command succeeds + And the output reports repository diagnostic "package.lua_runtime" + + Scenario: User receives diagnostics for invalid package schema + Given an initialized getter data directory + And a fixture Lua repository "broken" with schema-invalid package "android/org.fdroid.fdroid" + When I run getter repo validate for that repository + Then the command succeeds + And the output reports repository diagnostic "package.schema" + + Scenario: User receives diagnostics for package path mismatch + Given an initialized getter data directory + And a fixture Lua repository "broken" with mismatched package path "android/org.fdroid.fdroid" + When I run getter repo validate for that repository + Then the command succeeds + And the output reports repository diagnostic "package.domain" + + Scenario: User receives diagnostics for an incomplete repository layout + Given an initialized getter data directory + And an incomplete Lua repository "broken" + When I run getter repo validate for that repository + Then the command succeeds + And the output reports repository diagnostic "repository.missing_directory" diff --git a/crates/getter-core/src/diagnostics.rs b/crates/getter-core/src/diagnostics.rs new file mode 100644 index 0000000..745446f --- /dev/null +++ b/crates/getter-core/src/diagnostics.rs @@ -0,0 +1,309 @@ +//! Structured repository/package diagnostics for offline validation. +//! +//! Diagnostics are getter-owned DTOs used by CLI and future app bridges. They +//! describe what getter observed; Flutter should only render them. + +use crate::lua::{evaluate_package_file, LuaPackageError}; +use crate::repository::{RepositoryLayout, RepositoryLoadError}; +use crate::PackageId; +use serde::{Deserialize, Serialize}; +use std::path::{Path, PathBuf}; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum DiagnosticSeverity { + Error, + Warning, + Info, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct SourceLocation { + pub path: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub field: Option, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct PackageValidationDiagnostic { + pub severity: DiagnosticSeverity, + pub code: String, + pub message: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub package_id: Option, + pub location: SourceLocation, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct RepositoryValidationReport { + pub valid: bool, + pub diagnostics: Vec, + pub package_count: usize, + pub network_required: bool, +} + +impl RepositoryValidationReport { + pub fn new(package_count: usize, diagnostics: Vec) -> Self { + let valid = !diagnostics + .iter() + .any(|diagnostic| diagnostic.severity == DiagnosticSeverity::Error); + Self { + valid, + diagnostics, + package_count, + network_required: false, + } + } +} + +/// Validate a repository path offline. +/// +/// This intentionally loads and evaluates local Lua package files only. It must +/// not perform provider/network checks. +pub fn validate_repository_path(path: impl AsRef) -> RepositoryValidationReport { + let root = path.as_ref(); + let layout = match RepositoryLayout::load(root) { + Ok(layout) => layout, + Err(error) => { + return RepositoryValidationReport::new(0, vec![repository_load_diagnostic(error)]) + } + }; + + let mut diagnostics = Vec::new(); + let mut package_count = 0usize; + for package_file in &layout.packages { + match evaluate_package_file(&layout, &package_file.path) { + Ok(_) => package_count += 1, + Err(error) => diagnostics.push(lua_diagnostic(error, Some(package_file.id.clone()))), + } + } + + RepositoryValidationReport::new(package_count, diagnostics) +} + +fn repository_load_diagnostic(error: RepositoryLoadError) -> PackageValidationDiagnostic { + let (code, path, message) = match error { + RepositoryLoadError::ReadRepoToml { path, source } => ( + "repository.read_repo_toml", + path, + format!("failed to read repo.toml: {source}"), + ), + RepositoryLoadError::ParseRepoToml { path, source } => ( + "repository.parse_repo_toml", + path, + format!("failed to parse repo.toml: {source}"), + ), + RepositoryLoadError::RepositoryId(source) => ( + "repository.invalid_id", + PathBuf::from("repo.toml"), + source.to_string(), + ), + RepositoryLoadError::UnsupportedApiVersion(version) => ( + "repository.unsupported_api_version", + PathBuf::from("repo.toml"), + format!("unsupported repository api_version '{version}'"), + ), + RepositoryLoadError::MissingDirectory { path, directory } => ( + "repository.missing_directory", + path, + format!("missing required directory '{directory}'"), + ), + RepositoryLoadError::ReadPackagesDir { path, source } => ( + "repository.read_packages_dir", + path, + format!("failed to read packages directory: {source}"), + ), + RepositoryLoadError::InvalidPackagePath { path, reason } => { + ("repository.invalid_package_path", path, reason) + } + RepositoryLoadError::PackageId { path, source } => ( + "repository.invalid_package_id", + path, + format!("invalid package id derived from path: {source}"), + ), + RepositoryLoadError::HashPackageFile { path, source } => ( + "repository.hash_package_file", + path, + format!("failed to hash package file: {source}"), + ), + }; + diagnostic(code, message, path, None, None) +} + +fn lua_diagnostic( + error: LuaPackageError, + fallback_package_id: Option, +) -> PackageValidationDiagnostic { + match error { + LuaPackageError::ReadFile { path, source } => diagnostic( + "package.read_file", + format!("failed to read Lua package file: {source}"), + path, + None, + fallback_package_id, + ), + LuaPackageError::Runtime { path, source } => diagnostic( + "package.lua_runtime", + format!("Lua runtime error: {source}"), + path, + None, + fallback_package_id, + ), + LuaPackageError::NotATable { path } => diagnostic( + "package.not_a_table", + "Lua package did not return a table".to_owned(), + path, + None, + fallback_package_id, + ), + LuaPackageError::UnsupportedValue { + path, + location, + value_type, + } => diagnostic( + "package.unsupported_value", + format!("unsupported Lua value at {location}: {value_type}"), + path, + Some(location), + fallback_package_id, + ), + LuaPackageError::Schema { path, message } => { + diagnostic("package.schema", message, path, None, fallback_package_id) + } + LuaPackageError::Domain { path, message } => { + diagnostic("package.domain", message, path, None, fallback_package_id) + } + } +} + +fn diagnostic( + code: impl Into, + message: impl Into, + path: PathBuf, + field: Option, + package_id: Option, +) -> PackageValidationDiagnostic { + PackageValidationDiagnostic { + severity: DiagnosticSeverity::Error, + code: code.into(), + message: message.into(), + package_id, + location: SourceLocation { + path: path.to_string_lossy().to_string(), + field, + }, + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::fs; + + fn fixture_repo() -> tempfile::TempDir { + let temp = tempfile::tempdir().unwrap(); + let root = temp.path(); + fs::write( + root.join("repo.toml"), + r#"id = "official" +name = "Official" +priority = 0 +api_version = "getter.repo.v1" +"#, + ) + .unwrap(); + fs::create_dir_all(root.join("packages/android")).unwrap(); + fs::create_dir(root.join("lib")).unwrap(); + fs::create_dir(root.join("templates")).unwrap(); + temp + } + + #[test] + fn valid_repository_has_no_diagnostics() { + let temp = fixture_repo(); + fs::write( + temp.path().join("packages/android/org.fdroid.fdroid.lua"), + r#"return package_def { id = "android/org.fdroid.fdroid", name = "F-Droid" }"#, + ) + .unwrap(); + + let report = validate_repository_path(temp.path()); + assert!(report.valid, "{report:?}"); + assert_eq!(report.package_count, 1); + assert!(report.diagnostics.is_empty()); + assert!(!report.network_required); + } + + #[test] + fn missing_directory_is_stable_repository_diagnostic() { + let temp = tempfile::tempdir().unwrap(); + fs::write( + temp.path().join("repo.toml"), + r#"id = "official" +name = "Official" +api_version = "getter.repo.v1" +"#, + ) + .unwrap(); + + let report = validate_repository_path(temp.path()); + assert!(!report.valid); + assert_eq!(report.diagnostics[0].code, "repository.missing_directory"); + } + + #[test] + fn lua_schema_error_is_stable_package_diagnostic() { + let temp = fixture_repo(); + fs::write( + temp.path().join("packages/android/org.fdroid.fdroid.lua"), + r#"return { id = "android/org.fdroid.fdroid" }"#, + ) + .unwrap(); + + let report = validate_repository_path(temp.path()); + assert!(!report.valid); + assert_eq!(report.diagnostics[0].code, "package.schema"); + assert_eq!( + report.diagnostics[0] + .package_id + .as_ref() + .unwrap() + .to_string(), + "android/org.fdroid.fdroid" + ); + } + + #[test] + fn package_id_path_mismatch_is_domain_diagnostic() { + let temp = fixture_repo(); + fs::write( + temp.path().join("packages/android/org.fdroid.fdroid.lua"), + r#"return { id = "android/com.termux", name = "Termux" }"#, + ) + .unwrap(); + + let report = validate_repository_path(temp.path()); + assert!(!report.valid); + assert_eq!(report.diagnostics[0].code, "package.domain"); + } + + #[test] + fn unsupported_api_version_is_stable_repository_diagnostic() { + let temp = tempfile::tempdir().unwrap(); + fs::write( + temp.path().join("repo.toml"), + r#"id = "official" +name = "Official" +api_version = "getter.repo.v2" +"#, + ) + .unwrap(); + + let report = validate_repository_path(temp.path()); + assert!(!report.valid); + assert_eq!( + report.diagnostics[0].code, + "repository.unsupported_api_version" + ); + } +} diff --git a/crates/getter-core/src/lib.rs b/crates/getter-core/src/lib.rs index da959a3..32c110b 100644 --- a/crates/getter-core/src/lib.rs +++ b/crates/getter-core/src/lib.rs @@ -9,6 +9,7 @@ use std::cmp::Ordering; use std::fmt; use std::str::FromStr; +pub mod diagnostics; pub mod lua; pub mod repository; pub mod update; diff --git a/crates/getter-core/src/repository.rs b/crates/getter-core/src/repository.rs index fe9fa21..c383acc 100644 --- a/crates/getter-core/src/repository.rs +++ b/crates/getter-core/src/repository.rs @@ -1,7 +1,7 @@ //! Repository layout loading for Lua package repositories. use crate::{PackageId, PackageIdError, RepositoryId, RepositoryIdError, RepositoryPriority}; -use serde::Deserialize; +use serde::{Deserialize, Serialize}; use std::fs; use std::path::{Path, PathBuf}; @@ -31,6 +31,14 @@ pub struct PackageFile { pub path: PathBuf, } +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct RepositoryPackageCacheKey { + pub repository_id: RepositoryId, + pub package_id: PackageId, + pub api_version: String, + pub package_file_hash: String, +} + #[derive(Debug, thiserror::Error)] pub enum RepositoryLoadError { #[error("failed to read repo.toml at {path}: {source}")] @@ -68,6 +76,12 @@ pub enum RepositoryLoadError { #[source] source: PackageIdError, }, + #[error("failed to hash package file {path}: {source}")] + HashPackageFile { + path: PathBuf, + #[source] + source: std::io::Error, + }, } impl RepositoryLayout { @@ -224,6 +238,36 @@ pub fn package_id_from_path( }) } +pub fn package_cache_key( + repository: &RepositoryLayout, + package: &PackageFile, +) -> Result { + Ok(RepositoryPackageCacheKey { + repository_id: repository.metadata.id.clone(), + package_id: package.id.clone(), + api_version: repository.metadata.api_version.clone(), + package_file_hash: package_file_content_hash(&package.path)?, + }) +} + +pub fn package_file_content_hash(path: impl AsRef) -> Result { + let path = path.as_ref(); + let bytes = fs::read(path).map_err(|source| RepositoryLoadError::HashPackageFile { + path: path.to_path_buf(), + source, + })?; + Ok(format!("{:016x}", fnv1a64(&bytes))) +} + +fn fnv1a64(bytes: &[u8]) -> u64 { + let mut hash = 0xcbf29ce484222325u64; + for byte in bytes { + hash ^= u64::from(*byte); + hash = hash.wrapping_mul(0x100000001b3); + } + hash +} + pub fn highest_priority(items: &[T], priority: F) -> Option<&T> where F: Fn(&T) -> RepositoryPriority, @@ -275,6 +319,46 @@ api_version = "getter.repo.v1" ); } + #[test] + fn package_cache_key_changes_when_package_file_content_changes() { + let temp = tempfile::tempdir().unwrap(); + let root = temp.path(); + fs::write( + root.join("repo.toml"), + r#"id = "official" +name = "UpgradeAll Official" +priority = 0 +api_version = "getter.repo.v1" +"#, + ) + .unwrap(); + fs::create_dir(root.join("packages")).unwrap(); + fs::create_dir(root.join("packages/android")).unwrap(); + fs::create_dir(root.join("lib")).unwrap(); + fs::create_dir(root.join("templates")).unwrap(); + let package_path = root.join("packages/android/org.fdroid.fdroid.lua"); + fs::write( + &package_path, + r#"return { id = "android/org.fdroid.fdroid", name = "F-Droid" }"#, + ) + .unwrap(); + let layout = RepositoryLayout::load(root).unwrap(); + let first = package_cache_key(&layout, &layout.packages[0]).unwrap(); + + fs::write( + &package_path, + r#"return { id = "android/org.fdroid.fdroid", name = "F-Droid Nightly" }"#, + ) + .unwrap(); + let layout = RepositoryLayout::load(root).unwrap(); + let second = package_cache_key(&layout, &layout.packages[0]).unwrap(); + + assert_eq!(first.repository_id.as_str(), "official"); + assert_eq!(first.package_id.to_string(), "android/org.fdroid.fdroid"); + assert_eq!(first.api_version, REPO_API_VERSION_V1); + assert_ne!(first.package_file_hash, second.package_file_hash); + } + #[test] fn highest_priority_selects_larger_number() { let priorities = [ From f1035d3ae88198dfd0de872b6f9af010a0b0cd3f Mon Sep 17 00:00:00 2001 From: Xiangzhe Date: Mon, 22 Jun 2026 20:30:12 +0800 Subject: [PATCH 11/52] fix(core): include schema field diagnostics Populate the diagnostic location field for schema errors when getter can infer the failing field from validation messages. --- crates/getter-core/src/diagnostics.rs | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/crates/getter-core/src/diagnostics.rs b/crates/getter-core/src/diagnostics.rs index 745446f..7c58881 100644 --- a/crates/getter-core/src/diagnostics.rs +++ b/crates/getter-core/src/diagnostics.rs @@ -168,7 +168,8 @@ fn lua_diagnostic( fallback_package_id, ), LuaPackageError::Schema { path, message } => { - diagnostic("package.schema", message, path, None, fallback_package_id) + let field = schema_field_from_message(&message); + diagnostic("package.schema", message, path, field, fallback_package_id) } LuaPackageError::Domain { path, message } => { diagnostic("package.domain", message, path, None, fallback_package_id) @@ -176,6 +177,13 @@ fn lua_diagnostic( } } +fn schema_field_from_message(message: &str) -> Option { + let marker = "field '"; + let start = message.find(marker)? + marker.len(); + let end = message[start..].find('\'')?; + Some(message[start..start + end].to_owned()) +} + fn diagnostic( code: impl Into, message: impl Into, @@ -263,6 +271,10 @@ api_version = "getter.repo.v1" let report = validate_repository_path(temp.path()); assert!(!report.valid); assert_eq!(report.diagnostics[0].code, "package.schema"); + assert_eq!( + report.diagnostics[0].location.field.as_deref(), + Some("name") + ); assert_eq!( report.diagnostics[0] .package_id From 255a9c7c25174355be24053c0e3876de7cf087db Mon Sep 17 00:00:00 2001 From: Xiangzhe Date: Mon, 22 Jun 2026 20:41:12 +0800 Subject: [PATCH 12/52] fix(core): constrain Lua validation environment Remove process/filesystem-sensitive Lua globals during package evaluation and assert the repo validator exposes full diagnostic shape for schema errors. --- crates/getter-cli/tests/bdd_cli.rs | 17 +++++--- crates/getter-core/src/diagnostics.rs | 6 ++- crates/getter-core/src/lua.rs | 56 ++++++++++++++++++++++++++- 3 files changed, 71 insertions(+), 8 deletions(-) diff --git a/crates/getter-cli/tests/bdd_cli.rs b/crates/getter-cli/tests/bdd_cli.rs index 4f084c7..ddc42f2 100644 --- a/crates/getter-cli/tests/bdd_cli.rs +++ b/crates/getter-cli/tests/bdd_cli.rs @@ -376,12 +376,17 @@ fn output_reports_repository_diagnostic(world: &mut CliWorld, code: String) { let diagnostics = json["data"]["diagnostics"] .as_array() .expect("diagnostics array"); - assert!( - diagnostics - .iter() - .any(|diagnostic| diagnostic["code"].as_str() == Some(code.as_str())), - "diagnostics should contain {code}: {diagnostics:?}" - ); + let diagnostic = diagnostics + .iter() + .find(|diagnostic| diagnostic["code"].as_str() == Some(code.as_str())) + .unwrap_or_else(|| panic!("diagnostics should contain {code}: {diagnostics:?}")); + assert_eq!(diagnostic["severity"], "error"); + assert!(diagnostic["message"].as_str().is_some()); + assert!(diagnostic["location"]["path"].as_str().is_some()); + if code == "package.schema" { + assert_eq!(diagnostic["package_id"], "android/org.fdroid.fdroid"); + assert_eq!(diagnostic["location"]["field"], "name"); + } } #[then("the output contains the evaluated fixture package")] diff --git a/crates/getter-core/src/diagnostics.rs b/crates/getter-core/src/diagnostics.rs index 7c58881..62ed870 100644 --- a/crates/getter-core/src/diagnostics.rs +++ b/crates/getter-core/src/diagnostics.rs @@ -4,7 +4,7 @@ //! describe what getter observed; Flutter should only render them. use crate::lua::{evaluate_package_file, LuaPackageError}; -use crate::repository::{RepositoryLayout, RepositoryLoadError}; +use crate::repository::{package_cache_key, RepositoryLayout, RepositoryLoadError}; use crate::PackageId; use serde::{Deserialize, Serialize}; use std::path::{Path, PathBuf}; @@ -72,6 +72,10 @@ pub fn validate_repository_path(path: impl AsRef) -> RepositoryValidationR let mut diagnostics = Vec::new(); let mut package_count = 0usize; for package_file in &layout.packages { + if let Err(error) = package_cache_key(&layout, package_file) { + diagnostics.push(repository_load_diagnostic(error)); + continue; + } match evaluate_package_file(&layout, &package_file.path) { Ok(_) => package_count += 1, Err(error) => diagnostics.push(lua_diagnostic(error, Some(package_file.id.clone()))), diff --git a/crates/getter-core/src/lua.rs b/crates/getter-core/src/lua.rs index c6b8f96..a31107d 100644 --- a/crates/getter-core/src/lua.rs +++ b/crates/getter-core/src/lua.rs @@ -61,6 +61,10 @@ pub fn evaluate_package_source( path: path.clone(), source, })?; + remove_unsafe_globals(&lua).map_err(|source| LuaPackageError::Runtime { + path: path.clone(), + source, + })?; install_helpers(&lua).map_err(|source| LuaPackageError::Runtime { path: path.clone(), source, @@ -92,7 +96,32 @@ fn configure_package_path(lua: &Lua, repository: &RepositoryLayout) -> mlua::Res nested_lib_pattern.to_string_lossy() ); package.set("path", new_path)?; - install_lib_prefix_searcher(lua, &package, repository.lib_dir.clone()) + package.set("cpath", "")?; + package.set("loadlib", Value::Nil)?; + install_lib_prefix_searcher(lua, &package, repository.lib_dir.clone())?; + disable_native_module_searchers(&package) +} + +fn disable_native_module_searchers(package: &Table) -> mlua::Result<()> { + let searchers: Table = package.get("searchers")?; + let len = searchers.raw_len(); + for index in 4..=len { + searchers.raw_set(index, Value::Nil)?; + } + Ok(()) +} + +fn remove_unsafe_globals(lua: &Lua) -> mlua::Result<()> { + let globals = lua.globals(); + let package: Table = globals.get("package")?; + let loaded: Table = package.get("loaded")?; + for name in ["os", "io", "debug"] { + loaded.set(name, Value::Nil)?; + } + for name in ["os", "io", "debug", "dofile", "loadfile", "package"] { + globals.set(name, Value::Nil)?; + } + Ok(()) } fn install_lib_prefix_searcher(lua: &Lua, package: &Table, lib_dir: PathBuf) -> mlua::Result<()> { @@ -583,6 +612,31 @@ return { assert_eq!(package.name, "F-Droid"); } + #[test] + fn lua_environment_does_not_expose_process_or_file_system_globals() { + let temp = tempfile::tempdir().unwrap(); + let side_effect = temp.path().join("side-effect"); + let (_repo_temp, layout, package_path) = fixture_repo(); + fs::write( + &package_path, + format!( + r#" +return {{ + id = "android/org.fdroid.fdroid", + name = (os == nil and io == nil and debug == nil and dofile == nil and loadfile == nil and package == nil) and "F-Droid" or "leaked", + side_effect = rawget(_G, "os") and os.execute("touch {}"), +}} +"#, + side_effect.display() + ), + ) + .unwrap(); + + let package = evaluate_package_file(&layout, &package_path).unwrap(); + assert_eq!(package.name, "F-Droid"); + assert!(!side_effect.exists()); + } + #[test] fn repository_root_is_not_exposed_even_when_cwd_is_repository_root() { let (_temp, layout, package_path) = fixture_repo(); From 17b0e281914d3c7d646aa0605bee283d31ffd60b Mon Sep 17 00:00:00 2001 From: Xiangzhe Date: Mon, 22 Jun 2026 22:22:48 +0800 Subject: [PATCH 13/52] feat(cli): import legacy Room databases --- Cargo.lock | 2 + crates/getter-cli/Cargo.toml | 1 + crates/getter-cli/src/lib.rs | 277 +++++++++- crates/getter-cli/tests/bdd_cli.rs | 276 +++++++++- .../cli/legacy_import_room_db.feature | 70 +++ crates/getter-storage/Cargo.toml | 1 + crates/getter-storage/src/legacy_room.rs | 486 +++++++++++++++++- crates/getter-storage/src/lib.rs | 115 ++++- 8 files changed, 1169 insertions(+), 59 deletions(-) create mode 100644 crates/getter-cli/tests/features/cli/legacy_import_room_db.feature diff --git a/Cargo.lock b/Cargo.lock index 5d73f4c..6119744 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -521,6 +521,7 @@ dependencies = [ "cucumber", "getter-core", "getter-storage", + "rusqlite", "serde", "serde_json", "tempfile", @@ -581,6 +582,7 @@ version = "0.1.0" dependencies = [ "getter-core", "rusqlite", + "serde_json", "tempfile", "thiserror 1.0.69", ] diff --git a/crates/getter-cli/Cargo.toml b/crates/getter-cli/Cargo.toml index ca03088..ef7d105 100644 --- a/crates/getter-cli/Cargo.toml +++ b/crates/getter-cli/Cargo.toml @@ -12,6 +12,7 @@ thiserror = "1" [dev-dependencies] cucumber = "0.23" +rusqlite = { version = "0.32", features = ["bundled"] } serde_json = "1" tempfile = "3" tokio = { version = "1", features = ["macros", "rt-multi-thread"] } diff --git a/crates/getter-cli/src/lib.rs b/crates/getter-cli/src/lib.rs index 22fb675..a15af83 100644 --- a/crates/getter-cli/src/lib.rs +++ b/crates/getter-cli/src/lib.rs @@ -9,11 +9,13 @@ use getter_core::lua::evaluate_package_file; use getter_core::repository::{RepositoryLayout, RepositoryMetadata}; use getter_core::{PackageId, RepositoryId, RepositoryPriority}; use getter_storage::legacy_room::{ - map_legacy_app, LegacyAppKind, LegacyAppRecord, LegacyExtraAppRecord, LegacyPackageResolution, + map_legacy_app, read_legacy_room_database, LegacyAppKind, LegacyAppRecord, + LegacyExtraAppRecord, LegacyPackageResolution, LegacyRoomDbImport, LegacyRoomImportWarning, + LegacyRoomReadError, }; use getter_storage::{ - CacheDb, MainDb, StorageError, StoredPackageResolution, StoredRepository, StoredTrackedPackage, - TrackedPackageUpsert, + CacheDb, MainDb, MigrationRecordUpsert, StorageError, StoredPackageResolution, + StoredRepository, StoredTrackedPackage, TrackedPackageUpsert, }; use serde::{Deserialize, Serialize}; use serde_json::{json, Value}; @@ -23,6 +25,7 @@ use std::path::{Path, PathBuf}; const MAIN_DB_FILE: &str = "main.db"; const CACHE_DB_FILE: &str = "cache.db"; const MIGRATION_REPORTS_DIR: &str = "migration-reports"; +const LEGACY_ROOM_MIGRATION_ID: &str = "legacy-room-v17"; #[derive(Debug, Clone, PartialEq, Eq)] pub struct CliInvocation { @@ -55,6 +58,9 @@ pub enum CliCommand { LegacyImportRoomBundle { bundle: PathBuf, }, + LegacyImportRoomDb { + db: PathBuf, + }, LegacyReportList, } @@ -87,6 +93,10 @@ pub enum CliError { InvalidLegacyBundle { report_path: PathBuf }, #[error("Legacy Room bundle import is not implemented yet")] UnsupportedLegacyBundle { report_path: PathBuf }, + #[error("Legacy Room database is invalid")] + InvalidLegacyDb { report_path: PathBuf }, + #[error("Legacy Room database version is not supported")] + UnsupportedLegacyDb { report_path: PathBuf }, } impl CliError { @@ -95,9 +105,10 @@ impl CliError { Self::Usage(_) => ExitCode::Usage, Self::Storage(_) => ExitCode::Storage, Self::Repository(_) | Self::PackageEval(_) => ExitCode::GenericFailure, - Self::InvalidLegacyBundle { .. } | Self::UnsupportedLegacyBundle { .. } => { - ExitCode::Migration - } + Self::InvalidLegacyBundle { .. } + | Self::UnsupportedLegacyBundle { .. } + | Self::InvalidLegacyDb { .. } + | Self::UnsupportedLegacyDb { .. } => ExitCode::Migration, } } @@ -109,6 +120,8 @@ impl CliError { Self::PackageEval(_) => "package.eval_error", Self::InvalidLegacyBundle { .. } => "migration.invalid_bundle", Self::UnsupportedLegacyBundle { .. } => "migration.unsupported_bundle", + Self::InvalidLegacyDb { .. } => "migration.invalid_db", + Self::UnsupportedLegacyDb { .. } => "migration.unsupported_db", } } @@ -122,6 +135,8 @@ impl CliError { Self::UnsupportedLegacyBundle { .. } => { "Legacy Room bundle import is not implemented yet" } + Self::InvalidLegacyDb { .. } => "Legacy Room database is invalid", + Self::UnsupportedLegacyDb { .. } => "Legacy Room database version is not supported", } } @@ -131,14 +146,19 @@ impl CliError { | Self::Storage(detail) | Self::Repository(detail) | Self::PackageEval(detail) => Some(detail.as_str()), - Self::InvalidLegacyBundle { .. } | Self::UnsupportedLegacyBundle { .. } => None, + Self::InvalidLegacyBundle { .. } + | Self::UnsupportedLegacyBundle { .. } + | Self::InvalidLegacyDb { .. } + | Self::UnsupportedLegacyDb { .. } => None, } } fn report_path(&self) -> Option<&Path> { match self { Self::InvalidLegacyBundle { report_path } - | Self::UnsupportedLegacyBundle { report_path } => Some(report_path.as_path()), + | Self::UnsupportedLegacyBundle { report_path } + | Self::InvalidLegacyDb { report_path } + | Self::UnsupportedLegacyDb { report_path } => Some(report_path.as_path()), Self::Usage(_) | Self::Storage(_) | Self::Repository(_) | Self::PackageEval(_) => None, } } @@ -272,6 +292,11 @@ where bundle: PathBuf::from(bundle), } } + [domain, command, db] if domain == "legacy" && command == "import-room-db" => { + CliCommand::LegacyImportRoomDb { + db: PathBuf::from(db), + } + } [domain, command] if domain == "legacy" && command == "report-list" => { CliCommand::LegacyReportList } @@ -367,6 +392,14 @@ fn execute(invocation: CliInvocation) -> Result { } CliCommand::LegacyImportRoomBundle { bundle } => { let db = open_main_db(&invocation.data_dir)?; + if db.migration_record_exists(LEGACY_ROOM_MIGRATION_ID)? { + let records = db.tracked_packages()?; + return Ok(json!({ + "already_imported": true, + "imported_records": 0, + "apps": tracked_packages_json(records), + })); + } let bytes = fs::read(&bundle).map_err(|source| { match create_migration_report( &invocation.data_dir, @@ -375,6 +408,7 @@ fn execute(invocation: CliInvocation) -> Result { &format!("failed to read bundle: {source}"), 0, 0, + &[], ) { Ok(report_path) => CliError::InvalidLegacyBundle { report_path }, Err(error) => error, @@ -388,6 +422,7 @@ fn execute(invocation: CliInvocation) -> Result { &format!("failed to parse JSON bundle: {source}"), 0, 0, + &[], ) { Ok(report_path) => CliError::InvalidLegacyBundle { report_path }, Err(error) => error, @@ -404,6 +439,7 @@ fn execute(invocation: CliInvocation) -> Result { ), 0, 0, + &[], )?; return Err(CliError::UnsupportedLegacyBundle { report_path }); } @@ -415,6 +451,7 @@ fn execute(invocation: CliInvocation) -> Result { "Legacy Room bundle imported", parsed.apps.len() as u64, parsed.apps.len() as u64, + &[], )?; let records = db.tracked_packages()?; Ok(json!({ @@ -423,6 +460,62 @@ fn execute(invocation: CliInvocation) -> Result { "apps": tracked_packages_json(records), })) } + CliCommand::LegacyImportRoomDb { db: legacy_db } => { + let db = open_main_db(&invocation.data_dir)?; + if db.migration_record_exists(LEGACY_ROOM_MIGRATION_ID)? { + let records = db.tracked_packages()?; + return Ok(json!({ + "already_imported": true, + "imported_records": 0, + "source_counts": { + "app_rows": 0, + "extra_app_rows": 0, + "hub_rows": 0, + "extra_hub_rows": 0, + }, + "warnings": [], + "apps": tracked_packages_json(records), + })); + } + let import = read_legacy_room_database(&legacy_db).map_err(|source| { + legacy_db_import_error(&invocation.data_dir, &legacy_db, source) + })?; + let imported_records = import.apps.len(); + let source_counts = source_counts_json(&import); + let warnings = import_warnings_json(&import.warnings); + if import.source_counts.app_rows > 0 && imported_records == 0 { + let report_path = create_migration_report_with_source_counts( + &invocation.data_dir, + &legacy_db, + "migration.invalid_db", + "Legacy Room database has app rows but no importable app rows", + 0, + 0, + &warnings, + Some(&source_counts), + )?; + return Err(CliError::InvalidLegacyDb { report_path }); + } + import_legacy_room_db(&db, &import)?; + let report_path = create_migration_report_with_source_counts( + &invocation.data_dir, + &legacy_db, + "migration.imported", + "Legacy Room database imported", + imported_records as u64, + imported_records as u64, + &warnings, + Some(&source_counts), + )?; + let records = db.tracked_packages()?; + Ok(json!({ + "report_path": report_path, + "imported_records": imported_records, + "source_counts": source_counts, + "warnings": warnings, + "apps": tracked_packages_json(records), + })) + } CliCommand::LegacyReportList => { open_initialized_storage(&invocation.data_dir)?; Ok(json!({ "reports": list_migration_reports(&invocation.data_dir)? })) @@ -574,10 +667,11 @@ fn tracked_packages_json(packages: Vec) -> Vec { } fn import_legacy_room_bundle(db: &MainDb, bundle: &LegacyRoomBundle) -> Result<(), CliError> { + let mut packages = Vec::new(); for app in &bundle.apps { let mapping = map_legacy_app( &LegacyAppRecord { - kind: app.kind.to_legacy_kind()?, + kind: app.kind.to_legacy_kind(), installed_id: app.installed_id.clone(), official_package_available: app.official_package_available, common_conversion_available: app.common_conversion_available, @@ -588,14 +682,7 @@ fn import_legacy_room_bundle(db: &MainDb, bundle: &LegacyRoomBundle) -> Result<( }), ) .map_err(|source| CliError::Storage(source.to_string()))?; - db.upsert_tracked_package(&TrackedPackageUpsert { - package_id: mapping.package_id, - enabled: true, - favorite: mapping.user_state.favorite, - ignored_version: mapping.user_state.ignored_version, - repository_id: None, - package_resolution: stored_resolution(mapping.package_resolution), - })?; + packages.push(tracked_package_upsert(mapping)); } let report_json = json!({ @@ -604,10 +691,58 @@ fn import_legacy_room_bundle(db: &MainDb, bundle: &LegacyRoomBundle) -> Result<( "imported_records": bundle.apps.len(), }) .to_string(); - db.insert_migration_record("legacy-room-v17", "legacy-room-bundle", &report_json)?; + db.import_tracked_packages_with_migration_record( + &packages, + &MigrationRecordUpsert { + id: LEGACY_ROOM_MIGRATION_ID, + source: "legacy-room-bundle", + report_json: &report_json, + }, + )?; Ok(()) } +fn import_legacy_room_db(db: &MainDb, import: &LegacyRoomDbImport) -> Result<(), CliError> { + let mut packages = Vec::new(); + for app in &import.apps { + let mapping = map_legacy_app(&app.app, Some(&app.user_state)) + .map_err(|source| CliError::Storage(source.to_string()))?; + packages.push(tracked_package_upsert(mapping)); + } + + let report_json = json!({ + "ok": true, + "source": "legacy-room-db", + "version": import.version, + "imported_records": import.apps.len(), + "source_counts": source_counts_json(import), + "warnings": import_warnings_json(&import.warnings), + }) + .to_string(); + db.import_tracked_packages_with_migration_record( + &packages, + &MigrationRecordUpsert { + id: LEGACY_ROOM_MIGRATION_ID, + source: "legacy-room-db", + report_json: &report_json, + }, + )?; + Ok(()) +} + +fn tracked_package_upsert( + mapping: getter_storage::legacy_room::LegacyAppMapping, +) -> TrackedPackageUpsert { + TrackedPackageUpsert { + package_id: mapping.package_id, + enabled: true, + favorite: mapping.user_state.favorite, + ignored_version: mapping.user_state.ignored_version, + repository_id: None, + package_resolution: stored_resolution(mapping.package_resolution), + } +} + fn stored_resolution(resolution: LegacyPackageResolution) -> StoredPackageResolution { match resolution { LegacyPackageResolution::OfficialRepositoryPackage => { @@ -622,6 +757,54 @@ fn stored_resolution(resolution: LegacyPackageResolution) -> StoredPackageResolu } } +fn legacy_db_import_error( + data_dir: &Path, + legacy_db: &Path, + source: LegacyRoomReadError, +) -> CliError { + let (code, unsupported) = match source { + LegacyRoomReadError::UnsupportedVersion { .. } => ("migration.unsupported_db", true), + LegacyRoomReadError::Sqlite(_) | LegacyRoomReadError::MissingRequiredTable(_) => { + ("migration.invalid_db", false) + } + }; + match create_migration_report_with_source_counts( + data_dir, + legacy_db, + code, + &source.to_string(), + 0, + 0, + &[], + None, + ) { + Ok(report_path) if unsupported => CliError::UnsupportedLegacyDb { report_path }, + Ok(report_path) => CliError::InvalidLegacyDb { report_path }, + Err(error) => error, + } +} + +fn source_counts_json(import: &LegacyRoomDbImport) -> Value { + json!({ + "app_rows": import.source_counts.app_rows, + "extra_app_rows": import.source_counts.extra_app_rows, + "hub_rows": import.source_counts.hub_rows, + "extra_hub_rows": import.source_counts.extra_hub_rows, + }) +} + +fn import_warnings_json(warnings: &[LegacyRoomImportWarning]) -> Vec { + warnings + .iter() + .map(|warning| { + json!({ + "code": warning.code(), + "message": warning.message(), + }) + }) + .collect() +} + fn main_db_path(data_dir: &Path) -> PathBuf { data_dir.join(MAIN_DB_FILE) } @@ -632,11 +815,35 @@ fn cache_db_path(data_dir: &Path) -> PathBuf { fn create_migration_report( data_dir: &Path, - bundle: &Path, + source_file: &Path, code: &str, detail: &str, imported_records: u64, tracked_records: u64, + warnings: &[Value], +) -> Result { + create_migration_report_with_source_counts( + data_dir, + source_file, + code, + detail, + imported_records, + tracked_records, + warnings, + None, + ) +} + +#[allow(clippy::too_many_arguments)] +fn create_migration_report_with_source_counts( + data_dir: &Path, + source_file: &Path, + code: &str, + detail: &str, + imported_records: u64, + tracked_records: u64, + warnings: &[Value], + source_counts: Option<&Value>, ) -> Result { let reports_dir = data_dir.join(MIGRATION_REPORTS_DIR); fs::create_dir_all(&reports_dir).map_err(|source| { @@ -651,13 +858,17 @@ fn create_migration_report( message: match code { "migration.invalid_bundle" => "Legacy Room export bundle is invalid", "migration.unsupported_bundle" => "Legacy Room bundle import is not implemented yet", - "migration.imported" => "Legacy Room bundle imported", + "migration.invalid_db" => "Legacy Room database is invalid", + "migration.unsupported_db" => "Legacy Room database version is not supported", + "migration.imported" => "Legacy Room data imported", _ => "Legacy migration failed", }, - bundle_file_name: bundle.file_name().and_then(|name| name.to_str()), + source_file_name: source_file.file_name().and_then(|name| name.to_str()), detail, imported_records, tracked_records, + warnings, + source_counts, }; let bytes = serde_json::to_vec_pretty(&report) .map_err(|source| CliError::Storage(format!("failed to serialize report: {source}")))?; @@ -708,9 +919,17 @@ fn list_migration_reports(data_dir: &Path) -> Result, CliError> { "ok": report.get("ok").and_then(Value::as_bool).unwrap_or(false), "code": report.get("code").and_then(Value::as_str).unwrap_or("migration.unknown"), "message": report.get("message").and_then(Value::as_str).unwrap_or("Legacy migration report"), - "bundle_file_name": report.get("bundle_file_name").and_then(Value::as_str), + "source_file_name": report + .get("source_file_name") + .or_else(|| report.get("bundle_file_name")) + .and_then(Value::as_str), "imported_records": report.get("imported_records").and_then(Value::as_u64).unwrap_or(0), "tracked_records": report.get("tracked_records").and_then(Value::as_u64).unwrap_or(0), + "warnings": report + .get("warnings") + .cloned() + .unwrap_or_else(|| Value::Array(Vec::new())), + "source_counts": report.get("source_counts").cloned().unwrap_or(Value::Null), })) }) .collect() @@ -721,10 +940,13 @@ struct MigrationReport<'a> { ok: bool, code: &'a str, message: &'a str, - bundle_file_name: Option<&'a str>, + source_file_name: Option<&'a str>, detail: &'a str, imported_records: u64, tracked_records: u64, + warnings: &'a [Value], + #[serde(skip_serializing_if = "Option::is_none")] + source_counts: Option<&'a Value>, } fn success_envelope(command: &str, data: Value) -> String { @@ -762,7 +984,7 @@ fn envelope_to_string(value: Value) -> String { } fn usage_text() -> String { - "Usage: getter --data-dir [--priority ]|repo eval |repo validate |package eval [--repo ]|storage validate|hub list|legacy import-room-bundle |legacy report-list>\n".to_owned() + "Usage: getter --data-dir [--priority ]|repo eval |repo validate |package eval [--repo ]|storage validate|hub list|legacy import-room-bundle |legacy import-room-db |legacy report-list>\n".to_owned() } #[derive(Debug, Deserialize)] @@ -795,11 +1017,11 @@ enum LegacyBundleAppKind { } impl LegacyBundleAppKind { - fn to_legacy_kind(&self) -> Result { - Ok(match self { + const fn to_legacy_kind(&self) -> LegacyAppKind { + match self { Self::Android => LegacyAppKind::Android, Self::Magisk => LegacyAppKind::Magisk, - }) + } } } @@ -816,6 +1038,7 @@ impl CliCommand { Self::PackageEval { .. } => "package eval", Self::StorageValidate => "storage validate", Self::LegacyImportRoomBundle { .. } => "legacy import-room-bundle", + Self::LegacyImportRoomDb { .. } => "legacy import-room-db", Self::LegacyReportList => "legacy report-list", } } diff --git a/crates/getter-cli/tests/bdd_cli.rs b/crates/getter-cli/tests/bdd_cli.rs index ddc42f2..5d07d58 100644 --- a/crates/getter-cli/tests/bdd_cli.rs +++ b/crates/getter-cli/tests/bdd_cli.rs @@ -1,4 +1,5 @@ use cucumber::{given, then, when, World as _}; +use rusqlite::Connection; use serde_json::Value; use std::fs; use std::path::PathBuf; @@ -10,6 +11,7 @@ struct CliWorld { temp: Option, data_dir: Option, bundle: Option, + legacy_db: Option, fixture_repo_id: Option, fixture_repo_path: Option, fixture_package_id: Option, @@ -65,6 +67,48 @@ fn valid_legacy_export_bundle_with_android_app(world: &mut CliWorld) { world.bundle = Some(bundle); } +#[given("a legacy Room v17 database with an Android app and extra app state")] +fn legacy_room_v17_database_with_android_app(world: &mut CliWorld) { + let temp = world.temp.as_ref().expect("tempdir exists"); + let legacy_db = temp.path().join("app_metadata_database.db"); + create_fixture_legacy_room_db(&legacy_db, 17, true); + world.legacy_db = Some(legacy_db); +} + +#[given("an unsupported legacy Room database")] +fn unsupported_legacy_room_database(world: &mut CliWorld) { + let temp = world.temp.as_ref().expect("tempdir exists"); + let legacy_db = temp.path().join("app_metadata_database.db"); + create_fixture_legacy_room_db(&legacy_db, 16, true); + world.legacy_db = Some(legacy_db); +} + +#[given("a malformed legacy Room database")] +fn malformed_legacy_room_database(world: &mut CliWorld) { + let temp = world.temp.as_ref().expect("tempdir exists"); + let legacy_db = temp.path().join("app_metadata_database.db"); + create_fixture_legacy_room_db(&legacy_db, 17, false); + world.legacy_db = Some(legacy_db); +} + +#[given("a legacy Room v17 database with only unsupported app rows")] +fn legacy_room_v17_database_with_only_unsupported_app_rows(world: &mut CliWorld) { + let temp = world.temp.as_ref().expect("tempdir exists"); + let legacy_db = temp.path().join("app_metadata_database.db"); + create_fixture_legacy_room_db(&legacy_db, 17, true); + let conn = Connection::open(&legacy_db).expect("open legacy Room fixture"); + conn.execute("DELETE FROM extra_app", []) + .expect("delete fixture extra_app"); + conn.execute("DELETE FROM app", []) + .expect("delete fixture app"); + conn.execute( + "INSERT INTO app(id, name, app_id, ignore_version_number, star) VALUES (1, 'Unsupported', ?1, NULL, 0)", + [r#"{"unknown_provider":"com.example.unsupported"}"#], + ) + .expect("insert unsupported app row"); + world.legacy_db = Some(legacy_db); +} + #[given(expr = "a fixture Lua repository {string} with package {string}")] fn fixture_lua_repository(world: &mut CliWorld, repo_id: String, package_id: String) { create_fixture_lua_repository(world, repo_id, package_id, "F-Droid".to_owned()); @@ -263,6 +307,21 @@ fn run_getter_legacy_import(world: &mut CliWorld) { world.json = None; } +#[when("I run getter legacy import-room-db for that database")] +fn run_getter_legacy_import_db(world: &mut CliWorld) { + let legacy_db = world.legacy_db.as_ref().expect("legacy db exists"); + let output = run_getter( + world, + [ + "legacy".to_owned(), + "import-room-db".to_owned(), + legacy_db.to_string_lossy().to_string(), + ], + ); + world.output = Some(output); + world.json = None; +} + #[when("I run getter legacy report-list for that directory")] fn run_getter_legacy_report_list(world: &mut CliWorld) { let output = run_getter(world, ["legacy".to_owned(), "report-list".to_owned()]); @@ -282,8 +341,31 @@ fn command_fails_with_migration_error(world: &mut CliWorld) { assert_eq!(output.status.code(), Some(20)); let json = parse_stdout(output); assert_eq!(json["ok"], false); - assert_eq!(json["command"], "legacy import-room-bundle"); - assert_eq!(json["error"]["code"], "migration.invalid_bundle"); + assert!( + json["command"] == "legacy import-room-bundle" + || json["command"] == "legacy import-room-db" + ); + assert!(matches!( + json["error"]["code"].as_str(), + Some("migration.invalid_bundle" | "migration.invalid_db" | "migration.unsupported_db") + )); + world.json = Some(json); +} + +#[then(expr = "the command fails with direct DB migration error {string}")] +fn command_fails_with_direct_db_migration_error(world: &mut CliWorld, code: String) { + let output = world.output.as_ref().expect("command output exists"); + assert_eq!(output.status.code(), Some(20)); + let json = parse_stdout(output); + assert_eq!(json["ok"], false); + assert_eq!(json["command"], "legacy import-room-db"); + assert_eq!(json["error"]["code"], code); + let report_path = json["error"]["report_path"] + .as_str() + .expect("report_path should be a string"); + let report = fs::read_to_string(report_path).expect("report should be readable"); + let report_json: Value = serde_json::from_str(&report).expect("report should be JSON"); + assert_eq!(report_json["code"], code); world.json = Some(json); } @@ -451,7 +533,10 @@ fn sanitized_migration_report_available(world: &mut CliWorld) { let report_json: Value = serde_json::from_str(&report).expect("report should be JSON"); assert_eq!(report_json["ok"], false); assert_eq!(report_json["imported_records"], 0); - assert!(report_json.get("bundle_file_name").is_some()); + assert!( + report_json.get("source_file_name").is_some() + || report_json.get("bundle_file_name").is_some() + ); assert!( report_json.get("raw_bundle").is_none(), "report must not include raw bundle content" @@ -462,7 +547,10 @@ fn sanitized_migration_report_available(world: &mut CliWorld) { fn import_reports_one_tracked_app(world: &mut CliWorld) { let json = current_json(world); assert_eq!(json["ok"], true); - assert_eq!(json["command"], "legacy import-room-bundle"); + assert!( + json["command"] == "legacy import-room-bundle" + || json["command"] == "legacy import-room-db" + ); assert_eq!(json["data"]["imported_records"], 1); assert_eq!( json["data"]["apps"].as_array().expect("apps array").len(), @@ -470,6 +558,64 @@ fn import_reports_one_tracked_app(world: &mut CliWorld) { ); } +#[then("the direct migration reports dropped legacy hub warnings")] +fn direct_migration_reports_dropped_hub_warnings(world: &mut CliWorld) { + let json = current_json(world); + assert_eq!(json["ok"], true); + assert_eq!(json["command"], "legacy import-room-db"); + assert_eq!(json["data"]["source_counts"]["hub_rows"], 1); + assert_eq!(json["data"]["source_counts"]["extra_hub_rows"], 1); + let warnings = json["data"]["warnings"].as_array().expect("warnings array"); + assert!(warnings + .iter() + .any(|warning| { warning["code"].as_str() == Some("legacy.dropped_hub_rows") })); + assert!(warnings + .iter() + .any(|warning| { warning["code"].as_str() == Some("legacy.dropped_extra_hub_rows") })); +} + +#[then("the direct migration report stays sanitized")] +fn direct_migration_report_stays_sanitized(world: &mut CliWorld) { + let json = current_json(world); + let stdout = serde_json::to_string(json).expect("JSON output serializes"); + assert_sanitized_direct_room_report_text(&stdout); + let report_path = json["data"]["report_path"] + .as_str() + .expect("report_path should be a string"); + let report = fs::read_to_string(report_path).expect("report should be readable"); + assert_sanitized_direct_room_report_text(&report); + let report_json: Value = serde_json::from_str(&report).expect("report should be JSON"); + assert_eq!(report_json["source_counts"]["hub_rows"], 1); + assert_eq!(report_json["source_counts"]["extra_hub_rows"], 1); + assert_report_has_drop_warnings(&report_json); +} + +#[then("the output reports the legacy Room migration was already completed")] +fn output_reports_legacy_room_migration_already_completed(world: &mut CliWorld) { + let json = current_json(world); + assert_eq!(json["ok"], true); + assert_eq!(json["command"], "legacy import-room-db"); + assert_eq!(json["data"]["already_imported"], true); + assert_eq!(json["data"]["imported_records"], 0); + assert_eq!( + json["data"]["apps"].as_array().expect("apps array").len(), + 1 + ); +} + +#[then("the bundle output reports the legacy Room migration was already completed")] +fn bundle_output_reports_legacy_room_migration_already_completed(world: &mut CliWorld) { + let json = current_json(world); + assert_eq!(json["ok"], true); + assert_eq!(json["command"], "legacy import-room-bundle"); + assert_eq!(json["data"]["already_imported"], true); + assert_eq!(json["data"]["imported_records"], 0); + assert_eq!( + json["data"]["apps"].as_array().expect("apps array").len(), + 1 + ); +} + #[then(expr = "the output lists migration report {string}")] fn output_lists_migration_report(world: &mut CliWorld, code: String) { let json = current_json(world); @@ -484,6 +630,21 @@ fn output_lists_migration_report(world: &mut CliWorld, code: String) { ); } +#[then("the direct migration report list stays sanitized")] +fn direct_migration_report_list_stays_sanitized(world: &mut CliWorld) { + let json = current_json(world); + let report_list = serde_json::to_string(json).expect("report list serializes"); + assert_sanitized_direct_room_report_text(&report_list); + let reports = json["data"]["reports"].as_array().expect("reports array"); + let imported = reports + .iter() + .find(|report| report["code"].as_str() == Some("migration.imported")) + .expect("imported report exists"); + assert_eq!(imported["source_counts"]["hub_rows"], 1); + assert_eq!(imported["source_counts"]["extra_hub_rows"], 1); + assert_report_has_drop_warnings(imported); +} + #[then(expr = "the app list contains imported package {string}")] fn app_list_contains_imported_package(world: &mut CliWorld, package_id: String) { let output = run_getter(world, ["app".to_owned(), "list".to_owned()]); @@ -501,6 +662,23 @@ fn app_list_contains_imported_package(world: &mut CliWorld, package_id: String) world.json = Some(json); } +#[then(expr = "the app list contains directly imported package {string}")] +fn app_list_contains_directly_imported_package(world: &mut CliWorld, package_id: String) { + let output = run_getter(world, ["app".to_owned(), "list".to_owned()]); + assert_success(&output); + let json = parse_stdout(&output); + let apps = json["data"]["apps"].as_array().expect("apps array"); + let app = apps + .iter() + .find(|app| app["id"].as_str() == Some(package_id.as_str())) + .unwrap_or_else(|| panic!("app list should contain {package_id}: {apps:?}")); + assert_eq!(app["favorite"], true); + assert_eq!(app["ignored_version"], "1.20.0"); + assert_eq!(app["package_resolution"], "missing_package_definition"); + world.output = Some(output); + world.json = Some(json); +} + fn current_json(world: &mut CliWorld) -> &Value { world .json @@ -549,6 +727,96 @@ return package_def {{ world.fixture_package_id = Some(package_id); } +fn create_fixture_legacy_room_db(path: &PathBuf, version: u32, include_app_table: bool) { + let conn = Connection::open(path).expect("create legacy Room fixture"); + conn.pragma_update(None, "user_version", version) + .expect("set user_version"); + if include_app_table { + conn.execute_batch( + r#" +CREATE TABLE app ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + name TEXT NOT NULL, + app_id TEXT NOT NULL, + invalid_version_number_field_regex TEXT, + include_version_number_field_regex TEXT, + ignore_version_number TEXT, + cloud_config TEXT, + enable_hub_list TEXT, + star INTEGER +); +CREATE TABLE extra_app ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + app_id TEXT NOT NULL, + mark_version_number TEXT +); +CREATE TABLE hub ( + uuid TEXT PRIMARY KEY, + hub_config TEXT NOT NULL, + auth TEXT NOT NULL, + ignore_app_id_list TEXT NOT NULL, + applications_mode INTEGER NOT NULL DEFAULT 0, + user_ignore_app_id_list TEXT NOT NULL, + sort_point INTEGER NOT NULL DEFAULT 0 +); +CREATE TABLE extra_hub ( + id TEXT PRIMARY KEY, + enable_global INTEGER NOT NULL DEFAULT 0, + url_replace_search TEXT, + url_replace_string TEXT +); +"#, + ) + .expect("create legacy tables"); + let app_id = r#"{"android_app_package":"org.fdroid.fdroid"}"#; + conn.execute( + "INSERT INTO app(id, name, app_id, ignore_version_number, star) VALUES (1, 'F-Droid', ?1, '1.10.0', 1)", + [app_id], + ) + .expect("insert app"); + conn.execute( + "INSERT INTO extra_app(id, app_id, mark_version_number) VALUES (1, ?1, '1.20.0')", + [app_id], + ) + .expect("insert extra app"); + conn.execute( + "INSERT INTO hub(uuid, hub_config, auth, ignore_app_id_list, user_ignore_app_id_list) VALUES ('legacy-hub', '{\"secret\":\"UA_DIRECT_DB_SENTINEL_SECRET\"}', '{\"token\":\"UA_DIRECT_DB_SENTINEL_TOKEN\"}', '[]', '[]')", + [], + ) + .expect("insert hub"); + conn.execute( + "INSERT INTO extra_hub(id, enable_global, url_replace_search, url_replace_string) VALUES ('GLOBAL', 1, 'UA_DIRECT_DB_SENTINEL_SEARCH', 'UA_DIRECT_DB_SENTINEL_REPLACE')", + [], + ) + .expect("insert extra hub"); + } +} + +fn assert_report_has_drop_warnings(report: &Value) { + let warnings = report["warnings"].as_array().expect("warnings array"); + assert!(warnings + .iter() + .any(|warning| warning["code"].as_str() == Some("legacy.dropped_hub_rows"))); + assert!(warnings + .iter() + .any(|warning| warning["code"].as_str() == Some("legacy.dropped_extra_hub_rows"))); +} + +fn assert_sanitized_direct_room_report_text(text: &str) { + for forbidden in [ + "UA_DIRECT_DB_SENTINEL_SECRET", + "UA_DIRECT_DB_SENTINEL_TOKEN", + "UA_DIRECT_DB_SENTINEL_SEARCH", + "UA_DIRECT_DB_SENTINEL_REPLACE", + "legacy-hub", + ] { + assert!( + !text.contains(forbidden), + "direct Room migration report must not expose {forbidden}: {text}" + ); + } +} + fn create_custom_fixture_lua_repository( world: &mut CliWorld, repo_id: String, diff --git a/crates/getter-cli/tests/features/cli/legacy_import_room_db.feature b/crates/getter-cli/tests/features/cli/legacy_import_room_db.feature new file mode 100644 index 0000000..8ddcf2b --- /dev/null +++ b/crates/getter-cli/tests/features/cli/legacy_import_room_db.feature @@ -0,0 +1,70 @@ +@getter-cli @migration +Feature: Direct legacy Room database import + Scenario: User imports a supported legacy Room database into tracked app state + Given an initialized getter data directory + And a legacy Room v17 database with an Android app and extra app state + When I run getter legacy import-room-db for that database + Then the command succeeds + And the output is valid JSON + And the import reports one tracked app + And the direct migration reports dropped legacy hub warnings + And the direct migration report stays sanitized + And the app list contains directly imported package "android/org.fdroid.fdroid" + When I run getter legacy report-list for that directory + Then the output lists migration report "migration.imported" + And the direct migration report list stays sanitized + + Scenario: User does not rerun a completed direct legacy Room migration + Given an initialized getter data directory + And a legacy Room v17 database with an Android app and extra app state + When I run getter legacy import-room-db for that database + Then the command succeeds + When I run getter legacy import-room-db for that database + Then the command succeeds + And the output reports the legacy Room migration was already completed + + Scenario: User cannot apply a bridge bundle after completed direct legacy Room migration + Given an initialized getter data directory + And a legacy Room v17 database with an Android app and extra app state + And a syntactically valid legacy export bundle with an Android app + When I run getter legacy import-room-db for that database + Then the command succeeds + When I run getter legacy import-room-bundle for that bundle + Then the command succeeds + And the bundle output reports the legacy Room migration was already completed + And the app list contains directly imported package "android/org.fdroid.fdroid" + + Scenario: User cannot apply a direct legacy Room database after completed bridge bundle migration + Given an initialized getter data directory + And a syntactically valid legacy export bundle with an Android app + And a legacy Room v17 database with an Android app and extra app state + When I run getter legacy import-room-bundle for that bundle + Then the command succeeds + When I run getter legacy import-room-db for that database + Then the command succeeds + And the output reports the legacy Room migration was already completed + And the app list contains imported package "android/org.fdroid.fdroid" + + Scenario: User receives a recovery report for an unsupported legacy Room database + Given an initialized getter data directory + And an unsupported legacy Room database + When I run getter legacy import-room-db for that database + Then the command fails with direct DB migration error "migration.unsupported_db" + And no partially usable imported state is created + And a sanitized migration report is available + + Scenario: User receives a recovery report for a malformed legacy Room database + Given an initialized getter data directory + And a malformed legacy Room database + When I run getter legacy import-room-db for that database + Then the command fails with direct DB migration error "migration.invalid_db" + And no partially usable imported state is created + And a sanitized migration report is available + + Scenario: User receives a recovery report when no legacy app rows can be mapped + Given an initialized getter data directory + And a legacy Room v17 database with only unsupported app rows + When I run getter legacy import-room-db for that database + Then the command fails with direct DB migration error "migration.invalid_db" + And no partially usable imported state is created + And a sanitized migration report is available diff --git a/crates/getter-storage/Cargo.toml b/crates/getter-storage/Cargo.toml index 6a561a4..bc0964b 100644 --- a/crates/getter-storage/Cargo.toml +++ b/crates/getter-storage/Cargo.toml @@ -6,6 +6,7 @@ edition.workspace = true [dependencies] getter-core = { path = "../getter-core" } rusqlite = { version = "0.32", features = ["bundled"] } +serde_json = "1" thiserror = "1" [dev-dependencies] diff --git a/crates/getter-storage/src/legacy_room.rs b/crates/getter-storage/src/legacy_room.rs index 798809e..711ea1f 100644 --- a/crates/getter-storage/src/legacy_room.rs +++ b/crates/getter-storage/src/legacy_room.rs @@ -1,10 +1,18 @@ //! Legacy Room migration mapping tests and pure mapping helpers. //! -//! This module intentionally contains no Android Room/database reader. It is the -//! TDD boundary for source->target mapping rules before the full migration -//! implementation is added. +//! This module owns Rust-side interpretation of legacy Room data. Android and +//! Flutter adapters may locate/checkpoint/copy the old SQLite files, but package +//! identity, user-state mapping, warnings, and target semantics live here. use getter_core::PackageId; +use rusqlite::{Connection, OpenFlags}; +use serde_json::Value; +use std::collections::HashMap; +use std::path::Path; + +pub const LEGACY_ROOM_SUPPORTED_VERSION: u32 = 17; +const LEGACY_ANDROID_APP_TYPE: &str = "android_app_package"; +const LEGACY_ANDROID_MAGISK_MODULE_TYPE: &str = "android_magisk_module"; #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum LegacyAppKind { @@ -52,6 +60,64 @@ pub enum LegacyMigrationWarning { MissingPackageDefinition { package_id: PackageId }, } +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct LegacyRoomDbImport { + pub version: u32, + pub source_counts: LegacyRoomSourceCounts, + pub apps: Vec, + pub warnings: Vec, +} + +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct LegacyRoomSourceCounts { + pub app_rows: u64, + pub extra_app_rows: u64, + pub hub_rows: u64, + pub extra_hub_rows: u64, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct LegacyRoomImportedApp { + pub app: LegacyAppRecord, + pub user_state: LegacyExtraAppRecord, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum LegacyRoomImportWarning { + SkippedApp { row_id: i64, reason: String }, + SkippedExtraApp { row_id: i64, reason: String }, + DroppedHubRows { rows: u64 }, + DroppedExtraHubRows { rows: u64 }, +} + +impl LegacyRoomImportWarning { + pub fn code(&self) -> &'static str { + match self { + Self::SkippedApp { .. } => "legacy.skipped_app", + Self::SkippedExtraApp { .. } => "legacy.skipped_extra_app", + Self::DroppedHubRows { .. } => "legacy.dropped_hub_rows", + Self::DroppedExtraHubRows { .. } => "legacy.dropped_extra_hub_rows", + } + } + + pub fn message(&self) -> String { + match self { + Self::SkippedApp { row_id, reason } => { + format!("Skipped legacy app row {row_id}: {reason}") + } + Self::SkippedExtraApp { row_id, reason } => { + format!("Skipped legacy extra_app row {row_id}: {reason}") + } + Self::DroppedHubRows { rows } => { + format!("Legacy hub rows are not imported in this slice: {rows}") + } + Self::DroppedExtraHubRows { rows } => { + format!("Legacy extra_hub rows are not imported in this slice: {rows}") + } + } + } +} + #[derive(Debug, thiserror::Error)] pub enum LegacyRoomMappingError { #[error("legacy installed id is empty")] @@ -60,6 +126,16 @@ pub enum LegacyRoomMappingError { PackageId(#[from] getter_core::PackageIdError), } +#[derive(Debug, thiserror::Error)] +pub enum LegacyRoomReadError { + #[error("sqlite error: {0}")] + Sqlite(#[from] rusqlite::Error), + #[error("unsupported legacy Room database version {found}; expected {expected}")] + UnsupportedVersion { found: u32, expected: u32 }, + #[error("legacy Room database is missing required table '{0}'")] + MissingRequiredTable(&'static str), +} + pub fn map_legacy_app( app: &LegacyAppRecord, extra: Option<&LegacyExtraAppRecord>, @@ -107,6 +183,227 @@ pub fn map_legacy_package_id( Ok(format!("{prefix}/{installed_id}").parse()?) } +pub fn read_legacy_room_database(path: &Path) -> Result { + let conn = Connection::open_with_flags(path, OpenFlags::SQLITE_OPEN_READ_ONLY)?; + let version = legacy_user_version(&conn)?; + if version != LEGACY_ROOM_SUPPORTED_VERSION { + return Err(LegacyRoomReadError::UnsupportedVersion { + found: version, + expected: LEGACY_ROOM_SUPPORTED_VERSION, + }); + } + if !table_exists(&conn, "app")? { + return Err(LegacyRoomReadError::MissingRequiredTable("app")); + } + + let mut warnings = Vec::new(); + let extra_apps = read_extra_apps(&conn, &mut warnings)?; + let apps = read_apps(&conn, &extra_apps, &mut warnings)?; + let source_counts = LegacyRoomSourceCounts { + app_rows: table_row_count(&conn, "app")?.unwrap_or(0), + extra_app_rows: table_row_count(&conn, "extra_app")?.unwrap_or(0), + hub_rows: table_row_count(&conn, "hub")?.unwrap_or(0), + extra_hub_rows: table_row_count(&conn, "extra_hub")?.unwrap_or(0), + }; + if source_counts.hub_rows > 0 { + warnings.push(LegacyRoomImportWarning::DroppedHubRows { + rows: source_counts.hub_rows, + }); + } + if source_counts.extra_hub_rows > 0 { + warnings.push(LegacyRoomImportWarning::DroppedExtraHubRows { + rows: source_counts.extra_hub_rows, + }); + } + + Ok(LegacyRoomDbImport { + version, + source_counts, + apps, + warnings, + }) +} + +fn legacy_user_version(conn: &Connection) -> Result { + let version: u32 = conn.query_row("PRAGMA user_version", [], |row| row.get(0))?; + Ok(version) +} + +fn read_apps( + conn: &Connection, + extra_apps: &HashMap, + warnings: &mut Vec, +) -> Result, LegacyRoomReadError> { + let mut stmt = conn.prepare( + r#" +SELECT id, app_id, ignore_version_number, star +FROM app +ORDER BY id ASC +"#, + )?; + let rows = stmt.query_map([], |row| { + Ok(( + row.get::<_, i64>(0)?, + row.get::<_, String>(1)?, + row.get::<_, Option>(2)?, + row.get::<_, Option>(3)?, + )) + })?; + + let mut apps = Vec::new(); + for row in rows { + let (row_id, app_id_json, ignored_version, star) = row?; + let app_id = match parse_app_id_map(&app_id_json) { + Ok(app_id) => app_id, + Err(reason) => { + warnings.push(LegacyRoomImportWarning::SkippedApp { row_id, reason }); + continue; + } + }; + let (kind, installed_id) = match installed_id_from_app_id(&app_id) { + Ok(target) => target, + Err(reason) => { + warnings.push(LegacyRoomImportWarning::SkippedApp { row_id, reason }); + continue; + } + }; + let package_key = match map_legacy_package_id(kind, &installed_id) { + Ok(package_id) => package_id.to_string(), + Err(error) => { + warnings.push(LegacyRoomImportWarning::SkippedApp { + row_id, + reason: error.to_string(), + }); + continue; + } + }; + let extra = extra_apps.get(&package_key); + apps.push(LegacyRoomImportedApp { + app: LegacyAppRecord { + kind, + installed_id, + official_package_available: false, + common_conversion_available: false, + }, + user_state: LegacyExtraAppRecord { + ignored_version: extra + .and_then(|extra| extra.ignored_version.clone()) + .or(ignored_version), + favorite: star.unwrap_or(0) != 0, + }, + }); + } + Ok(apps) +} + +fn read_extra_apps( + conn: &Connection, + warnings: &mut Vec, +) -> Result, LegacyRoomReadError> { + if !table_exists(conn, "extra_app")? { + return Ok(HashMap::new()); + } + let mut stmt = conn.prepare( + r#" +SELECT id, app_id, mark_version_number +FROM extra_app +ORDER BY id ASC +"#, + )?; + let rows = stmt.query_map([], |row| { + Ok(( + row.get::<_, i64>(0)?, + row.get::<_, String>(1)?, + row.get::<_, Option>(2)?, + )) + })?; + + let mut extras = HashMap::new(); + for row in rows { + let (row_id, app_id_json, ignored_version) = row?; + let app_id = match parse_app_id_map(&app_id_json) { + Ok(app_id) => app_id, + Err(reason) => { + warnings.push(LegacyRoomImportWarning::SkippedExtraApp { row_id, reason }); + continue; + } + }; + let (kind, installed_id) = match installed_id_from_app_id(&app_id) { + Ok(target) => target, + Err(reason) => { + warnings.push(LegacyRoomImportWarning::SkippedExtraApp { row_id, reason }); + continue; + } + }; + let package_id = match map_legacy_package_id(kind, &installed_id) { + Ok(package_id) => package_id, + Err(error) => { + warnings.push(LegacyRoomImportWarning::SkippedExtraApp { + row_id, + reason: error.to_string(), + }); + continue; + } + }; + extras.insert( + package_id.to_string(), + LegacyExtraAppRecord { + ignored_version, + favorite: false, + }, + ); + } + Ok(extras) +} + +fn parse_app_id_map(value: &str) -> Result, String> { + let value: Value = serde_json::from_str(value).map_err(|error| error.to_string())?; + let object = value + .as_object() + .ok_or_else(|| "app_id must be a JSON object".to_owned())?; + let mut app_id = HashMap::new(); + for (key, value) in object { + if let Some(value) = value.as_str().filter(|value| !value.is_empty()) { + app_id.insert(key.clone(), value.to_owned()); + } + } + Ok(app_id) +} + +fn installed_id_from_app_id( + app_id: &HashMap, +) -> Result<(LegacyAppKind, String), String> { + if let Some(module_id) = app_id.get(LEGACY_ANDROID_MAGISK_MODULE_TYPE) { + return Ok((LegacyAppKind::Magisk, module_id.clone())); + } + if let Some(package_name) = app_id.get(LEGACY_ANDROID_APP_TYPE) { + return Ok((LegacyAppKind::Android, package_name.clone())); + } + Err(format!( + "app_id has no supported '{}' or '{}' value", + LEGACY_ANDROID_APP_TYPE, LEGACY_ANDROID_MAGISK_MODULE_TYPE + )) +} + +fn table_exists(conn: &Connection, table: &str) -> Result { + let exists: i64 = conn.query_row( + "SELECT COUNT(*) FROM sqlite_master WHERE type = 'table' AND name = ?1", + [table], + |row| row.get(0), + )?; + Ok(exists != 0) +} + +fn table_row_count(conn: &Connection, table: &str) -> Result, LegacyRoomReadError> { + if !table_exists(conn, table)? { + return Ok(None); + } + let count: u64 = conn.query_row(&format!("SELECT COUNT(*) FROM {table}"), [], |row| { + row.get(0) + })?; + Ok(Some(count)) +} + #[cfg(test)] mod tests { use super::*; @@ -212,4 +509,187 @@ mod tests { assert_eq!(mapping.user_state.ignored_version.as_deref(), Some("1.2.3")); assert!(mapping.user_state.favorite); } + + #[test] + fn direct_room_reader_imports_app_rows_and_extra_app_state() { + let temp = tempfile::tempdir().unwrap(); + let db_path = temp.path().join("legacy.db"); + create_fixture_legacy_db(&db_path, LEGACY_ROOM_SUPPORTED_VERSION); + + let import = read_legacy_room_database(&db_path).unwrap(); + + assert_eq!(import.version, LEGACY_ROOM_SUPPORTED_VERSION); + assert_eq!(import.apps.len(), 1); + assert_eq!(import.source_counts.app_rows, 1); + assert_eq!(import.source_counts.extra_app_rows, 1); + assert_eq!(import.source_counts.hub_rows, 1); + assert_eq!(import.source_counts.extra_hub_rows, 1); + assert_eq!(import.apps[0].app.kind, LegacyAppKind::Android); + assert_eq!(import.apps[0].app.installed_id, "org.fdroid.fdroid"); + assert_eq!( + import.apps[0].user_state.ignored_version.as_deref(), + Some("1.20.0") + ); + assert!(import.apps[0].user_state.favorite); + assert!(import + .warnings + .iter() + .any(|warning| matches!(warning, LegacyRoomImportWarning::DroppedHubRows { rows: 1 }))); + assert!(import.warnings.iter().any(|warning| matches!( + warning, + LegacyRoomImportWarning::DroppedExtraHubRows { rows: 1 } + ))); + } + + #[test] + fn direct_room_reader_maps_magisk_app_id_rows() { + let temp = tempfile::tempdir().unwrap(); + let db_path = temp.path().join("legacy.db"); + create_fixture_legacy_db(&db_path, LEGACY_ROOM_SUPPORTED_VERSION); + let conn = Connection::open(&db_path).unwrap(); + conn.execute("DELETE FROM app", []).unwrap(); + conn.execute("DELETE FROM extra_app", []).unwrap(); + conn.execute( + "INSERT INTO app(id, name, app_id, ignore_version_number, star) VALUES (1, 'Zygisk Next', ?1, NULL, 0)", + [r#"{"android_magisk_module":"zygisk-next"}"#], + ) + .unwrap(); + drop(conn); + + let import = read_legacy_room_database(&db_path).unwrap(); + + assert_eq!(import.apps.len(), 1); + assert_eq!(import.apps[0].app.kind, LegacyAppKind::Magisk); + assert_eq!(import.apps[0].app.installed_id, "zygisk-next"); + let mapping = + map_legacy_app(&import.apps[0].app, Some(&import.apps[0].user_state)).unwrap(); + assert_eq!(mapping.package_id.to_string(), "magisk/zygisk-next"); + } + + #[test] + fn direct_room_reader_skips_invalid_app_rows_without_dropping_valid_apps() { + let temp = tempfile::tempdir().unwrap(); + let db_path = temp.path().join("legacy.db"); + create_fixture_legacy_db(&db_path, LEGACY_ROOM_SUPPORTED_VERSION); + let conn = Connection::open(&db_path).unwrap(); + conn.execute( + "INSERT INTO app(id, name, app_id, ignore_version_number, star) VALUES (2, 'Broken', 'not-json', NULL, 0)", + [], + ) + .unwrap(); + drop(conn); + + let import = read_legacy_room_database(&db_path).unwrap(); + + assert_eq!(import.source_counts.app_rows, 2); + assert_eq!(import.apps.len(), 1); + assert_eq!(import.apps[0].app.installed_id, "org.fdroid.fdroid"); + assert!(import.warnings.iter().any(|warning| matches!( + warning, + LegacyRoomImportWarning::SkippedApp { row_id: 2, .. } + ))); + } + + #[test] + fn direct_room_reader_skips_malformed_optional_extra_app_rows() { + let temp = tempfile::tempdir().unwrap(); + let db_path = temp.path().join("legacy.db"); + create_fixture_legacy_db(&db_path, LEGACY_ROOM_SUPPORTED_VERSION); + let conn = Connection::open(&db_path).unwrap(); + conn.execute( + "INSERT INTO extra_app(id, app_id, mark_version_number) VALUES (2, 'not-json', '9.9.9')", + [], + ) + .unwrap(); + drop(conn); + + let import = read_legacy_room_database(&db_path).unwrap(); + + assert_eq!(import.apps.len(), 1); + assert!(import.warnings.iter().any(|warning| { + matches!( + warning, + LegacyRoomImportWarning::SkippedExtraApp { row_id: 2, .. } + ) + })); + } + + #[test] + fn direct_room_reader_rejects_unsupported_schema_version() { + let temp = tempfile::tempdir().unwrap(); + let db_path = temp.path().join("legacy.db"); + create_fixture_legacy_db(&db_path, 16); + + let error = read_legacy_room_database(&db_path).unwrap_err(); + + assert!(matches!( + error, + LegacyRoomReadError::UnsupportedVersion { + found: 16, + expected: LEGACY_ROOM_SUPPORTED_VERSION, + } + )); + } + + fn create_fixture_legacy_db(path: &Path, version: u32) { + let conn = Connection::open(path).unwrap(); + conn.pragma_update(None, "user_version", version).unwrap(); + conn.execute_batch( + r#" +CREATE TABLE app ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + name TEXT NOT NULL, + app_id TEXT NOT NULL, + invalid_version_number_field_regex TEXT, + include_version_number_field_regex TEXT, + ignore_version_number TEXT, + cloud_config TEXT, + enable_hub_list TEXT, + star INTEGER +); +CREATE TABLE extra_app ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + app_id TEXT NOT NULL, + mark_version_number TEXT +); +CREATE TABLE hub ( + uuid TEXT PRIMARY KEY, + hub_config TEXT NOT NULL, + auth TEXT NOT NULL, + ignore_app_id_list TEXT NOT NULL, + applications_mode INTEGER NOT NULL DEFAULT 0, + user_ignore_app_id_list TEXT NOT NULL, + sort_point INTEGER NOT NULL DEFAULT 0 +); +CREATE TABLE extra_hub ( + id TEXT PRIMARY KEY, + enable_global INTEGER NOT NULL DEFAULT 0, + url_replace_search TEXT, + url_replace_string TEXT +); +"#, + ) + .unwrap(); + let app_id = r#"{"android_app_package":"org.fdroid.fdroid"}"#; + conn.execute( + "INSERT INTO app(id, name, app_id, ignore_version_number, star) VALUES (1, 'F-Droid', ?1, '1.10.0', 1)", + [app_id], + ) + .unwrap(); + conn.execute( + "INSERT INTO extra_app(id, app_id, mark_version_number) VALUES (1, ?1, '1.20.0')", + [app_id], + ) + .unwrap(); + conn.execute( + "INSERT INTO hub(uuid, hub_config, auth, ignore_app_id_list, user_ignore_app_id_list) VALUES ('legacy-hub', '{}', '{}', '[]', '[]')", + [], + ) + .unwrap(); + conn.execute( + "INSERT INTO extra_hub(id, enable_global, url_replace_search, url_replace_string) VALUES ('GLOBAL', 1, 'http://', 'https://')", + [], + ) + .unwrap(); + } } diff --git a/crates/getter-storage/src/lib.rs b/crates/getter-storage/src/lib.rs index 6eed591..a6f0e56 100644 --- a/crates/getter-storage/src/lib.rs +++ b/crates/getter-storage/src/lib.rs @@ -4,7 +4,7 @@ pub mod legacy_room; use getter_core::repository::RepositoryMetadata; use getter_core::{PackageId, RepositoryId, RepositoryPriority}; -use rusqlite::{params, Connection}; +use rusqlite::{params, Connection, Params, Transaction}; use std::path::Path; use std::str::FromStr; @@ -180,33 +180,31 @@ ON CONFLICT(id) DO UPDATE SET &self, package: &TrackedPackageUpsert, ) -> Result<(), StorageError> { - self.conn.execute( + execute_tracked_package_upsert(&self.conn, package)?; + Ok(()) + } + + pub fn import_tracked_packages_with_migration_record( + &self, + packages: &[TrackedPackageUpsert], + record: &MigrationRecordUpsert<'_>, + ) -> Result<(), StorageError> { + let tx = self.conn.unchecked_transaction()?; + for package in packages { + execute_tracked_package_upsert(&tx, package)?; + } + tx.execute( r#" -INSERT INTO tracked_packages( - package_id, - enabled, - favorite, - ignored_version, - repository_id, - package_resolution -) -VALUES (?1, ?2, ?3, ?4, ?5, ?6) -ON CONFLICT(package_id) DO UPDATE SET - enabled = excluded.enabled, - favorite = excluded.favorite, - ignored_version = excluded.ignored_version, - repository_id = excluded.repository_id, - package_resolution = excluded.package_resolution +INSERT INTO migration_records(id, source, report_json) +VALUES (?1, ?2, ?3) +ON CONFLICT(id) DO UPDATE SET + source = excluded.source, + completed_at_unix = unixepoch(), + report_json = excluded.report_json "#, - params![ - package.package_id.to_string(), - bool_to_i64(package.enabled), - bool_to_i64(package.favorite), - package.ignored_version.as_deref(), - package.repository_id.as_ref().map(RepositoryId::as_str), - package.package_resolution.as_str(), - ], + params![record.id, record.source, record.report_json], )?; + tx.commit()?; Ok(()) } @@ -263,6 +261,15 @@ ON CONFLICT(id) DO UPDATE SET Ok(()) } + pub fn migration_record_exists(&self, id: &str) -> Result { + let count: i64 = self.conn.query_row( + "SELECT COUNT(*) FROM migration_records WHERE id = ?1", + [id], + |row| row.get(0), + )?; + Ok(count != 0) + } + pub fn migration_records(&self) -> Result, StorageError> { let mut stmt = self.conn.prepare( "SELECT id, source, report_json FROM migration_records ORDER BY completed_at_unix ASC, id ASC", @@ -370,6 +377,13 @@ pub struct StoredMigrationRecord { pub report_json: String, } +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct MigrationRecordUpsert<'a> { + pub id: &'a str, + pub source: &'a str, + pub report_json: &'a str, +} + #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum StoredPackageResolution { OfficialRepositoryPackage, @@ -400,6 +414,55 @@ impl FromStr for StoredPackageResolution { } } +trait SqlExecutor { + fn execute_statement(&self, sql: &str, params: P) -> Result; +} + +impl SqlExecutor for Connection { + fn execute_statement(&self, sql: &str, params: P) -> Result { + self.execute(sql, params) + } +} + +impl SqlExecutor for Transaction<'_> { + fn execute_statement(&self, sql: &str, params: P) -> Result { + self.execute(sql, params) + } +} + +fn execute_tracked_package_upsert( + conn: &impl SqlExecutor, + package: &TrackedPackageUpsert, +) -> Result { + conn.execute_statement( + r#" +INSERT INTO tracked_packages( + package_id, + enabled, + favorite, + ignored_version, + repository_id, + package_resolution +) +VALUES (?1, ?2, ?3, ?4, ?5, ?6) +ON CONFLICT(package_id) DO UPDATE SET + enabled = excluded.enabled, + favorite = excluded.favorite, + ignored_version = excluded.ignored_version, + repository_id = excluded.repository_id, + package_resolution = excluded.package_resolution +"#, + params![ + package.package_id.to_string(), + bool_to_i64(package.enabled), + bool_to_i64(package.favorite), + package.ignored_version.as_deref(), + package.repository_id.as_ref().map(RepositoryId::as_str), + package.package_resolution.as_str(), + ], + ) +} + fn bool_to_i64(value: bool) -> i64 { if value { 1 @@ -475,6 +538,7 @@ mod tests { #[test] fn main_db_records_migration_completion() { let db = MainDb::open_in_memory().unwrap(); + assert!(!db.migration_record_exists("legacy-room-v17").unwrap()); db.insert_migration_record("legacy-room-v17", "legacy-room-bundle", r#"{"ok":true}"#) .unwrap(); @@ -482,6 +546,7 @@ mod tests { assert_eq!(records.len(), 1); assert_eq!(records[0].id, "legacy-room-v17"); assert_eq!(records[0].source, "legacy-room-bundle"); + assert!(db.migration_record_exists("legacy-room-v17").unwrap()); } #[test] From 05397e1446f19e728737b5f94b8f8d96036c83f4 Mon Sep 17 00:00:00 2001 From: Xiangzhe Date: Tue, 23 Jun 2026 00:16:46 +0800 Subject: [PATCH 14/52] feat(cli): add installed app autogen --- crates/getter-cli/src/lib.rs | 716 +++++++++++++++++- crates/getter-cli/tests/bdd_cli.rs | 372 +++++++++ .../features/cli/autogen_installed.feature | 92 +++ crates/getter-core/src/autogen.rs | 536 +++++++++++++ crates/getter-core/src/lib.rs | 1 + crates/getter-storage/src/lib.rs | 171 +++++ 6 files changed, 1882 insertions(+), 6 deletions(-) create mode 100644 crates/getter-cli/tests/features/cli/autogen_installed.feature create mode 100644 crates/getter-core/src/autogen.rs diff --git a/crates/getter-cli/src/lib.rs b/crates/getter-cli/src/lib.rs index a15af83..72d92fc 100644 --- a/crates/getter-cli/src/lib.rs +++ b/crates/getter-cli/src/lib.rs @@ -4,9 +4,15 @@ //! while durable state is initialized through `getter-storage` so the command //! surface exercises the same Rust-owned SQLite direction used by embedders. +use getter_core::autogen::{ + content_hash, local_autogen_repo_toml, local_repo_toml, plan_local_autogen, AutogenManifest, + AutogenManifestEntry, AutogenPlan, AutogenSkipReason, InstalledInventory, + LOCAL_AUTOGEN_REPOSITORY_ID, LOCAL_AUTOGEN_REPOSITORY_NAME, LOCAL_REPOSITORY_ID, + LOCAL_REPOSITORY_NAME, +}; use getter_core::diagnostics::validate_repository_path; use getter_core::lua::evaluate_package_file; -use getter_core::repository::{RepositoryLayout, RepositoryMetadata}; +use getter_core::repository::{RepositoryLayout, RepositoryMetadata, REPO_API_VERSION_V1}; use getter_core::{PackageId, RepositoryId, RepositoryPriority}; use getter_storage::legacy_room::{ map_legacy_app, read_legacy_room_database, LegacyAppKind, LegacyAppRecord, @@ -19,13 +25,15 @@ use getter_storage::{ }; use serde::{Deserialize, Serialize}; use serde_json::{json, Value}; +use std::collections::{BTreeSet, HashMap}; use std::fs; -use std::path::{Path, PathBuf}; +use std::path::{Component, Path, PathBuf}; const MAIN_DB_FILE: &str = "main.db"; const CACHE_DB_FILE: &str = "cache.db"; const MIGRATION_REPORTS_DIR: &str = "migration-reports"; const LEGACY_ROOM_MIGRATION_ID: &str = "legacy-room-v17"; +const AUTOGEN_MANIFEST_FILE: &str = "autogen-manifest.json"; #[derive(Debug, Clone, PartialEq, Eq)] pub struct CliInvocation { @@ -55,6 +63,20 @@ pub enum CliCommand { repo_id: Option, }, StorageValidate, + AutogenInstalledPreview { + inventory: PathBuf, + }, + AutogenInstalledApply { + preview: PathBuf, + acceptance: AutogenAcceptance, + }, + AutogenCleanupPreview { + inventory: PathBuf, + }, + AutogenCleanupApply { + preview: PathBuf, + acceptance: AutogenAcceptance, + }, LegacyImportRoomBundle { bundle: PathBuf, }, @@ -64,6 +86,12 @@ pub enum CliCommand { LegacyReportList, } +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum AutogenAcceptance { + AcceptAll, + Accept(Vec), +} + #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum ExitCode { Success = 0, @@ -89,6 +117,8 @@ pub enum CliError { Repository(String), #[error("package evaluation error: {0}")] PackageEval(String), + #[error("autogen error: {0}")] + Autogen(String), #[error("Legacy Room export bundle is invalid")] InvalidLegacyBundle { report_path: PathBuf }, #[error("Legacy Room bundle import is not implemented yet")] @@ -104,7 +134,9 @@ impl CliError { match self { Self::Usage(_) => ExitCode::Usage, Self::Storage(_) => ExitCode::Storage, - Self::Repository(_) | Self::PackageEval(_) => ExitCode::GenericFailure, + Self::Repository(_) | Self::PackageEval(_) | Self::Autogen(_) => { + ExitCode::GenericFailure + } Self::InvalidLegacyBundle { .. } | Self::UnsupportedLegacyBundle { .. } | Self::InvalidLegacyDb { .. } @@ -118,6 +150,7 @@ impl CliError { Self::Storage(_) => "storage.error", Self::Repository(_) => "repository.error", Self::PackageEval(_) => "package.eval_error", + Self::Autogen(_) => "autogen.error", Self::InvalidLegacyBundle { .. } => "migration.invalid_bundle", Self::UnsupportedLegacyBundle { .. } => "migration.unsupported_bundle", Self::InvalidLegacyDb { .. } => "migration.invalid_db", @@ -131,6 +164,7 @@ impl CliError { Self::Storage(_) => "Getter storage operation failed", Self::Repository(_) => "Getter repository operation failed", Self::PackageEval(_) => "Getter package evaluation failed", + Self::Autogen(_) => "Getter autogen operation failed", Self::InvalidLegacyBundle { .. } => "Legacy Room export bundle is invalid", Self::UnsupportedLegacyBundle { .. } => { "Legacy Room bundle import is not implemented yet" @@ -145,7 +179,8 @@ impl CliError { Self::Usage(detail) | Self::Storage(detail) | Self::Repository(detail) - | Self::PackageEval(detail) => Some(detail.as_str()), + | Self::PackageEval(detail) + | Self::Autogen(detail) => Some(detail.as_str()), Self::InvalidLegacyBundle { .. } | Self::UnsupportedLegacyBundle { .. } | Self::InvalidLegacyDb { .. } @@ -159,7 +194,11 @@ impl CliError { | Self::UnsupportedLegacyBundle { report_path } | Self::InvalidLegacyDb { report_path } | Self::UnsupportedLegacyDb { report_path } => Some(report_path.as_path()), - Self::Usage(_) | Self::Storage(_) | Self::Repository(_) | Self::PackageEval(_) => None, + Self::Usage(_) + | Self::Storage(_) + | Self::Repository(_) + | Self::PackageEval(_) + | Self::Autogen(_) => None, } } } @@ -287,6 +326,48 @@ where [domain, command] if domain == "storage" && command == "validate" => { CliCommand::StorageValidate } + [domain, subject, action, flag, inventory] + if domain == "autogen" + && subject == "installed" + && action == "preview" + && flag == "--inventory" => + { + CliCommand::AutogenInstalledPreview { + inventory: PathBuf::from(inventory), + } + } + [domain, subject, action, flag, preview, rest @ ..] + if domain == "autogen" + && subject == "installed" + && action == "apply" + && flag == "--preview" => + { + CliCommand::AutogenInstalledApply { + preview: PathBuf::from(preview), + acceptance: parse_autogen_acceptance(rest)?, + } + } + [domain, subject, action, flag, inventory] + if domain == "autogen" + && subject == "cleanup" + && action == "preview" + && flag == "--inventory" => + { + CliCommand::AutogenCleanupPreview { + inventory: PathBuf::from(inventory), + } + } + [domain, subject, action, flag, preview, rest @ ..] + if domain == "autogen" + && subject == "cleanup" + && action == "apply" + && flag == "--preview" => + { + CliCommand::AutogenCleanupApply { + preview: PathBuf::from(preview), + acceptance: parse_autogen_acceptance(rest)?, + } + } [domain, command, bundle] if domain == "legacy" && command == "import-room-bundle" => { CliCommand::LegacyImportRoomBundle { bundle: PathBuf::from(bundle), @@ -390,6 +471,33 @@ fn execute(invocation: CliInvocation) -> Result { "cache_db": cache_db_path(&invocation.data_dir), })) } + CliCommand::AutogenInstalledPreview { inventory } => { + let db = open_main_db(&invocation.data_dir)?; + let inventory = read_installed_inventory(&inventory)?; + let plan = build_local_autogen_plan(&db, &inventory)?; + Ok(autogen_installed_preview_json(&invocation.data_dir, &plan)) + } + CliCommand::AutogenInstalledApply { + preview, + acceptance, + } => { + let db = open_main_db(&invocation.data_dir)?; + let preview = read_autogen_preview(&preview, "installed.preview")?; + apply_autogen_installed_preview(&invocation.data_dir, &db, &preview, &acceptance) + } + CliCommand::AutogenCleanupPreview { inventory } => { + let db = open_main_db(&invocation.data_dir)?; + let inventory = read_installed_inventory(&inventory)?; + cleanup_preview_json(&invocation.data_dir, &db, &inventory) + } + CliCommand::AutogenCleanupApply { + preview, + acceptance, + } => { + let db = open_main_db(&invocation.data_dir)?; + let preview = read_autogen_preview(&preview, "cleanup.preview")?; + apply_autogen_cleanup_preview(&invocation.data_dir, &db, &preview, &acceptance) + } CliCommand::LegacyImportRoomBundle { bundle } => { let db = open_main_db(&invocation.data_dir)?; if db.migration_record_exists(LEGACY_ROOM_MIGRATION_ID)? { @@ -558,6 +666,33 @@ fn parse_priority(value: &str) -> Result { .map_err(|source| CliError::Usage(format!("invalid repository priority: {source}"))) } +fn parse_autogen_acceptance(args: &[String]) -> Result { + match args { + [flag] if flag == "--accept-all" => Ok(AutogenAcceptance::AcceptAll), + rest if !rest.is_empty() => { + let mut ids = Vec::new(); + let mut index = 0; + while index < rest.len() { + if rest[index] != "--accept" { + return Err(CliError::Usage( + "autogen apply requires --accept-all or repeated --accept " + .to_owned(), + )); + } + let package_id = rest + .get(index + 1) + .ok_or_else(|| CliError::Usage("--accept requires a package id".to_owned()))?; + ids.push(parse_package_id(package_id)?); + index += 2; + } + Ok(AutogenAcceptance::Accept(ids)) + } + _ => Err(CliError::Usage( + "autogen apply requires --accept-all or --accept ".to_owned(), + )), + } +} + fn load_repository_layout(path: &Path) -> Result { RepositoryLayout::load(path).map_err(|source| CliError::Repository(source.to_string())) } @@ -612,6 +747,571 @@ fn evaluate_highest_priority_package( ))) } +fn read_installed_inventory(path: &Path) -> Result { + let bytes = fs::read(path) + .map_err(|source| CliError::Autogen(format!("failed to read inventory: {source}")))?; + serde_json::from_slice(&bytes) + .map_err(|source| CliError::Autogen(format!("failed to parse inventory JSON: {source}"))) +} + +fn build_local_autogen_plan( + db: &MainDb, + inventory: &InstalledInventory, +) -> Result { + let covered = higher_priority_package_coverage(db)?; + plan_local_autogen(inventory, &covered).map_err(|source| CliError::Autogen(source.to_string())) +} + +fn higher_priority_package_coverage( + db: &MainDb, +) -> Result, CliError> { + let mut covered = HashMap::new(); + for repo in db.repositories()? { + if repo.id.as_str() == LOCAL_AUTOGEN_REPOSITORY_ID { + continue; + } + if repo.priority <= RepositoryPriority::LOCAL_AUTOGEN { + continue; + } + let Some(path) = repo.path.as_ref() else { + continue; + }; + let layout = load_repository_layout(Path::new(path))?; + for package in layout.packages { + covered.entry(package.id).or_insert_with(|| repo.id.clone()); + } + } + Ok(covered) +} + +fn autogen_installed_preview_json(data_dir: &Path, plan: &AutogenPlan) -> Value { + let candidates: Vec = plan.candidates.iter().map(autogen_candidate_json).collect(); + let skipped: Vec = plan.skipped.iter().map(autogen_skip_json).collect(); + json!({ + "operation": "installed.preview", + "target_repo_id": plan.repository_id.as_str(), + "target_repo_path": local_autogen_repo_path(data_dir), + "summary": { + "candidate_count": candidates.len(), + "skipped_count": skipped.len(), + "write_count": candidates.len(), + "delete_count": 0, + }, + "candidates": candidates, + "skipped": skipped, + "diagnostics": [], + }) +} + +fn autogen_candidate_json(candidate: &getter_core::autogen::AutogenCandidate) -> Value { + json!({ + "package_id": candidate.package_id.to_string(), + "kind": candidate.package_id.kind().as_str(), + "display_name": candidate.name, + "installed_target": candidate.installed, + "action": "create", + "output_relative_path": candidate.relative_path, + "content_hash": candidate.content_hash, + "content": candidate.content, + }) +} + +fn autogen_skip_json(skip: &getter_core::autogen::AutogenSkip) -> Value { + json!({ + "package_id": skip.package_id.to_string(), + "reason": match skip.reason { + AutogenSkipReason::DuplicateInventoryItem => "duplicate_inventory_item", + AutogenSkipReason::CoveredByHigherPriorityRepository => "covered_by_higher_priority_repo", + }, + "covering_repo_id": skip.repository_id.as_ref().map(RepositoryId::as_str), + }) +} + +fn read_autogen_preview(path: &Path, expected_operation: &str) -> Result { + let bytes = fs::read(path) + .map_err(|source| CliError::Autogen(format!("failed to read autogen preview: {source}")))?; + let raw: Value = serde_json::from_slice(&bytes).map_err(|source| { + CliError::Autogen(format!("failed to parse autogen preview JSON: {source}")) + })?; + let payload = if raw.get("ok").is_some() && raw.get("data").is_some() { + raw.get("data").cloned().unwrap_or(Value::Null) + } else { + raw + }; + if payload.get("operation").and_then(Value::as_str) != Some(expected_operation) { + return Err(CliError::Autogen(format!( + "autogen preview operation must be '{expected_operation}'" + ))); + } + Ok(payload) +} + +fn apply_autogen_installed_preview( + data_dir: &Path, + db: &MainDb, + preview: &Value, + acceptance: &AutogenAcceptance, +) -> Result { + let repo_path = local_autogen_repo_path(data_dir); + ensure_local_autogen_repository(data_dir, db)?; + let accepted = accepted_preview_candidates(preview, acceptance)?; + let mut manifest = read_autogen_manifest(&repo_path)?.unwrap_or_else(empty_autogen_manifest); + let mut applied = Vec::new(); + let mut preserved = Vec::new(); + + for candidate in accepted { + let package_id = preview_package_id(candidate)?; + let relative_path = preview_relative_path(candidate)?; + let content = candidate + .get("content") + .and_then(Value::as_str) + .ok_or_else(|| CliError::Autogen("preview candidate missing content".to_owned()))?; + let expected_hash = candidate + .get("content_hash") + .and_then(Value::as_str) + .ok_or_else(|| { + CliError::Autogen("preview candidate missing content_hash".to_owned()) + })?; + if content_hash(content) != expected_hash { + return Err(CliError::Autogen(format!( + "preview content hash mismatch for {package_id}" + ))); + } + let target = safe_join(&repo_path, &relative_path)?; + if target.exists() { + let current = fs::read_to_string(&target).map_err(|source| { + CliError::Autogen(format!( + "failed to read existing autogen file '{}': {source}", + target.display() + )) + })?; + let current_hash = content_hash(¤t); + let known_hash = manifest + .package(&package_id) + .map(|entry| entry.content_hash.as_str()); + if known_hash != Some(current_hash.as_str()) { + preserved.push(preserve_autogen_file_in_local( + data_dir, + db, + &package_id, + ¤t, + )?); + } + } + if let Some(parent) = target.parent() { + fs::create_dir_all(parent).map_err(|source| { + CliError::Autogen(format!( + "failed to create autogen package directory '{}': {source}", + parent.display() + )) + })?; + } + fs::write(&target, content).map_err(|source| { + CliError::Autogen(format!( + "failed to write autogen package '{}': {source}", + target.display() + )) + })?; + upsert_manifest_entry( + &mut manifest, + AutogenManifestEntry { + package_id: package_id.clone(), + relative_path: relative_path.clone(), + content_hash: expected_hash.to_owned(), + }, + ); + db.upsert_generated_tracked_package_preserving_user_state( + &package_id, + &RepositoryId::new(LOCAL_AUTOGEN_REPOSITORY_ID).expect("valid id"), + )?; + applied.push(json!({ + "package_id": package_id.to_string(), + "output_relative_path": relative_path, + })); + } + + write_autogen_manifest(&repo_path, &manifest)?; + Ok(json!({ + "target_repo_id": LOCAL_AUTOGEN_REPOSITORY_ID, + "target_repo_path": repo_path, + "applied_count": applied.len(), + "applied": applied, + "preserved_to_local": preserved, + })) +} + +fn cleanup_preview_json( + data_dir: &Path, + db: &MainDb, + inventory: &InstalledInventory, +) -> Result { + let repo_path = local_autogen_repo_path(data_dir); + let Some(manifest) = read_autogen_manifest(&repo_path)? else { + return Ok(json!({ + "operation": "cleanup.preview", + "target_repo_id": LOCAL_AUTOGEN_REPOSITORY_ID, + "target_repo_path": repo_path, + "summary": { "candidate_count": 0, "skipped_count": 0, "write_count": 0, "delete_count": 0 }, + "candidates": [], + "skipped": [], + "diagnostics": [], + })); + }; + let plan = build_local_autogen_plan(db, inventory)?; + let installed_ids: BTreeSet = plan + .candidates + .iter() + .map(|candidate| candidate.package_id.to_string()) + .chain(plan.skipped.iter().map(|skip| skip.package_id.to_string())) + .collect(); + let mut candidates = Vec::new(); + for entry in manifest.packages { + if !installed_ids.contains(&entry.package_id.to_string()) { + candidates.push(json!({ + "package_id": entry.package_id.to_string(), + "action": "delete", + "output_relative_path": entry.relative_path, + "content_hash": entry.content_hash, + "reason": "not_in_installed_inventory", + })); + } + } + candidates.sort_by_key(|candidate| { + candidate + .get("package_id") + .and_then(Value::as_str) + .unwrap_or_default() + .to_owned() + }); + Ok(json!({ + "operation": "cleanup.preview", + "target_repo_id": LOCAL_AUTOGEN_REPOSITORY_ID, + "target_repo_path": repo_path, + "summary": { + "candidate_count": candidates.len(), + "skipped_count": 0, + "write_count": 0, + "delete_count": candidates.len(), + }, + "candidates": candidates, + "skipped": [], + "diagnostics": [], + })) +} + +fn apply_autogen_cleanup_preview( + data_dir: &Path, + db: &MainDb, + preview: &Value, + acceptance: &AutogenAcceptance, +) -> Result { + if preview.get("target_repo_id").and_then(Value::as_str) != Some(LOCAL_AUTOGEN_REPOSITORY_ID) { + return Err(CliError::Autogen(format!( + "cleanup preview target_repo_id must be '{LOCAL_AUTOGEN_REPOSITORY_ID}'" + ))); + } + let repo_path = local_autogen_repo_path(data_dir); + let accepted = accepted_preview_candidates(preview, acceptance)?; + let mut manifest = read_autogen_manifest(&repo_path)?.unwrap_or_else(empty_autogen_manifest); + let mut deleted = Vec::new(); + let mut preserved = Vec::new(); + let local_autogen_id = RepositoryId::new(LOCAL_AUTOGEN_REPOSITORY_ID).expect("valid id"); + for candidate in accepted { + let package_id = preview_package_id(candidate)?; + let relative_path = preview_relative_path(candidate)?; + let expected_hash = candidate + .get("content_hash") + .and_then(Value::as_str) + .ok_or_else(|| { + CliError::Autogen("cleanup preview candidate missing content_hash".to_owned()) + })?; + let manifest_entry = manifest.package(&package_id).ok_or_else(|| { + CliError::Autogen(format!( + "cleanup preview candidate {package_id} is not managed by local_autogen manifest" + )) + })?; + if manifest_entry.relative_path != relative_path + || manifest_entry.content_hash != expected_hash + { + return Err(CliError::Autogen(format!( + "cleanup preview candidate {package_id} does not match local_autogen manifest" + ))); + } + let target = safe_join(&repo_path, &relative_path)?; + if target.exists() { + let current = fs::read_to_string(&target).map_err(|source| { + CliError::Autogen(format!( + "failed to read existing autogen file '{}': {source}", + target.display() + )) + })?; + if content_hash(¤t) != expected_hash { + preserved.push(preserve_autogen_file_in_local( + data_dir, + db, + &package_id, + ¤t, + )?); + } + fs::remove_file(&target).map_err(|source| { + CliError::Autogen(format!( + "failed to delete autogen package '{}': {source}", + target.display() + )) + })?; + } + manifest + .packages + .retain(|entry| entry.package_id != package_id); + db.delete_generated_tracked_package(&package_id, &local_autogen_id)?; + deleted.push(json!({ + "package_id": package_id.to_string(), + "output_relative_path": relative_path, + })); + } + write_autogen_manifest(&repo_path, &manifest)?; + Ok(json!({ + "target_repo_id": LOCAL_AUTOGEN_REPOSITORY_ID, + "target_repo_path": repo_path, + "deleted_count": deleted.len(), + "deleted": deleted, + "preserved_to_local": preserved, + })) +} + +fn accepted_preview_candidates<'a>( + preview: &'a Value, + acceptance: &AutogenAcceptance, +) -> Result, CliError> { + let candidates = preview + .get("candidates") + .and_then(Value::as_array) + .ok_or_else(|| CliError::Autogen("autogen preview missing candidates".to_owned()))?; + match acceptance { + AutogenAcceptance::AcceptAll => Ok(candidates.iter().collect()), + AutogenAcceptance::Accept(ids) => { + let accepted: BTreeSet = ids.iter().map(ToString::to_string).collect(); + Ok(candidates + .iter() + .filter(|candidate| { + candidate + .get("package_id") + .and_then(Value::as_str) + .is_some_and(|id| accepted.contains(id)) + }) + .collect()) + } + } +} + +fn preview_package_id(candidate: &Value) -> Result { + candidate + .get("package_id") + .and_then(Value::as_str) + .ok_or_else(|| CliError::Autogen("preview candidate missing package_id".to_owned()))? + .parse() + .map_err(|source: getter_core::PackageIdError| CliError::Autogen(source.to_string())) +} + +fn preview_relative_path(candidate: &Value) -> Result { + candidate + .get("output_relative_path") + .and_then(Value::as_str) + .map(PathBuf::from) + .ok_or_else(|| { + CliError::Autogen("preview candidate missing output_relative_path".to_owned()) + }) +} + +fn ensure_local_autogen_repository(data_dir: &Path, db: &MainDb) -> Result { + let repo_path = local_autogen_repo_path(data_dir); + ensure_repository_layout(&repo_path, &local_autogen_repo_toml())?; + db.upsert_repository( + &RepositoryMetadata { + id: RepositoryId::new(LOCAL_AUTOGEN_REPOSITORY_ID).expect("valid id"), + name: LOCAL_AUTOGEN_REPOSITORY_NAME.to_owned(), + priority: RepositoryPriority::LOCAL_AUTOGEN, + api_version: REPO_API_VERSION_V1.to_owned(), + }, + Some(&repo_path), + None, + )?; + Ok(repo_path) +} + +fn ensure_local_repository(data_dir: &Path, db: &MainDb) -> Result { + let local_id = RepositoryId::new(LOCAL_REPOSITORY_ID).expect("valid id"); + if let Ok(existing) = find_repository(db, &local_id) { + let repo_path = repo_path(&existing)?; + ensure_repository_layout(&repo_path, &local_repo_toml())?; + return Ok(repo_path); + } + + let repo_path = data_dir.join("repositories").join(LOCAL_REPOSITORY_ID); + ensure_repository_layout(&repo_path, &local_repo_toml())?; + db.upsert_repository( + &RepositoryMetadata { + id: local_id, + name: LOCAL_REPOSITORY_NAME.to_owned(), + priority: RepositoryPriority::LOCAL, + api_version: REPO_API_VERSION_V1.to_owned(), + }, + Some(&repo_path), + None, + )?; + Ok(repo_path) +} + +fn ensure_repository_layout(repo_path: &Path, repo_toml: &str) -> Result<(), CliError> { + fs::create_dir_all(repo_path.join("packages")).map_err(|source| { + CliError::Autogen(format!( + "failed to create repository packages dir '{}': {source}", + repo_path.display() + )) + })?; + fs::create_dir_all(repo_path.join("lib")).map_err(|source| { + CliError::Autogen(format!( + "failed to create repository lib dir '{}': {source}", + repo_path.display() + )) + })?; + fs::create_dir_all(repo_path.join("templates")).map_err(|source| { + CliError::Autogen(format!( + "failed to create repository templates dir '{}': {source}", + repo_path.display() + )) + })?; + let repo_toml_path = repo_path.join("repo.toml"); + if !repo_toml_path.exists() { + fs::write(&repo_toml_path, repo_toml).map_err(|source| { + CliError::Autogen(format!( + "failed to write repo.toml '{}': {source}", + repo_toml_path.display() + )) + })?; + } + Ok(()) +} + +fn preserve_autogen_file_in_local( + data_dir: &Path, + db: &MainDb, + package_id: &PackageId, + content: &str, +) -> Result { + let local_repo = ensure_local_repository(data_dir, db)?; + let primary_relative = getter_core::autogen::package_relative_path(package_id); + let primary_target = safe_join(&local_repo, &primary_relative)?; + let relative_path = if primary_target.exists() { + let backup = PathBuf::from("autogen-preserved") + .join(package_id.kind().as_str()) + .join(format!( + "{}.{}.lua", + package_id.name(), + content_hash(content).replace(':', "-") + )); + safe_join(&local_repo, &backup)?; + backup + } else { + primary_relative + }; + let target = safe_join(&local_repo, &relative_path)?; + if let Some(parent) = target.parent() { + fs::create_dir_all(parent).map_err(|source| { + CliError::Autogen(format!( + "failed to create local preservation directory '{}': {source}", + parent.display() + )) + })?; + } + fs::write(&target, content).map_err(|source| { + CliError::Autogen(format!( + "failed to preserve modified autogen file '{}': {source}", + target.display() + )) + })?; + Ok(json!({ + "package_id": package_id.to_string(), + "repository_id": LOCAL_REPOSITORY_ID, + "relative_path": relative_path, + })) +} + +fn read_autogen_manifest(repo_path: &Path) -> Result, CliError> { + let path = repo_path.join(AUTOGEN_MANIFEST_FILE); + if !path.exists() { + return Ok(None); + } + let bytes = fs::read(&path).map_err(|source| { + CliError::Autogen(format!( + "failed to read autogen manifest '{}': {source}", + path.display() + )) + })?; + serde_json::from_slice(&bytes) + .map(Some) + .map_err(|source| CliError::Autogen(format!("failed to parse autogen manifest: {source}"))) +} + +fn write_autogen_manifest(repo_path: &Path, manifest: &AutogenManifest) -> Result<(), CliError> { + fs::create_dir_all(repo_path).map_err(|source| { + CliError::Autogen(format!( + "failed to create autogen repository '{}': {source}", + repo_path.display() + )) + })?; + let path = repo_path.join(AUTOGEN_MANIFEST_FILE); + let bytes = serde_json::to_vec_pretty(manifest) + .map_err(|source| CliError::Autogen(format!("failed to serialize manifest: {source}")))?; + fs::write(&path, bytes).map_err(|source| { + CliError::Autogen(format!( + "failed to write autogen manifest '{}': {source}", + path.display() + )) + }) +} + +fn empty_autogen_manifest() -> AutogenManifest { + AutogenManifest { + version: getter_core::autogen::AUTOGEN_MANIFEST_VERSION, + repository_id: RepositoryId::new(LOCAL_AUTOGEN_REPOSITORY_ID).expect("valid id"), + packages: Vec::new(), + } +} + +fn upsert_manifest_entry(manifest: &mut AutogenManifest, entry: AutogenManifestEntry) { + manifest + .packages + .retain(|existing| existing.package_id != entry.package_id); + manifest.packages.push(entry); + manifest + .packages + .sort_by_key(|existing| existing.package_id.to_string()); +} + +fn safe_join(root: &Path, relative: &Path) -> Result { + if relative.is_absolute() + || relative.components().any(|component| { + matches!( + component, + Component::ParentDir | Component::RootDir | Component::Prefix(_) + ) + }) + { + return Err(CliError::Autogen(format!( + "unsafe relative path '{}'", + relative.display() + ))); + } + Ok(root.join(relative)) +} + +fn local_autogen_repo_path(data_dir: &Path) -> PathBuf { + data_dir + .join("repositories") + .join(LOCAL_AUTOGEN_REPOSITORY_ID) +} + fn repo_path(repo: &StoredRepository) -> Result { repo.path .as_ref() @@ -984,7 +1684,7 @@ fn envelope_to_string(value: Value) -> String { } fn usage_text() -> String { - "Usage: getter --data-dir [--priority ]|repo eval |repo validate |package eval [--repo ]|storage validate|hub list|legacy import-room-bundle |legacy import-room-db |legacy report-list>\n".to_owned() + "Usage: getter --data-dir [--priority ]|repo eval |repo validate |package eval [--repo ]|storage validate|hub list|autogen installed preview --inventory |autogen installed apply --preview (--accept-all|--accept ...)|autogen cleanup preview --inventory |autogen cleanup apply --preview (--accept-all|--accept ...)|legacy import-room-bundle |legacy import-room-db |legacy report-list>\n".to_owned() } #[derive(Debug, Deserialize)] @@ -1037,6 +1737,10 @@ impl CliCommand { Self::RepoValidate { .. } => "repo validate", Self::PackageEval { .. } => "package eval", Self::StorageValidate => "storage validate", + Self::AutogenInstalledPreview { .. } => "autogen installed preview", + Self::AutogenInstalledApply { .. } => "autogen installed apply", + Self::AutogenCleanupPreview { .. } => "autogen cleanup preview", + Self::AutogenCleanupApply { .. } => "autogen cleanup apply", Self::LegacyImportRoomBundle { .. } => "legacy import-room-bundle", Self::LegacyImportRoomDb { .. } => "legacy import-room-db", Self::LegacyReportList => "legacy report-list", diff --git a/crates/getter-cli/tests/bdd_cli.rs b/crates/getter-cli/tests/bdd_cli.rs index 5d07d58..dfe31e2 100644 --- a/crates/getter-cli/tests/bdd_cli.rs +++ b/crates/getter-cli/tests/bdd_cli.rs @@ -12,6 +12,8 @@ struct CliWorld { data_dir: Option, bundle: Option, legacy_db: Option, + inventory: Option, + autogen_preview: Option, fixture_repo_id: Option, fixture_repo_path: Option, fixture_package_id: Option, @@ -109,6 +111,82 @@ fn legacy_room_v17_database_with_only_unsupported_app_rows(world: &mut CliWorld) world.legacy_db = Some(legacy_db); } +#[given(expr = "an installed inventory with Android app {string} labeled {string}")] +fn installed_inventory_with_android_app(world: &mut CliWorld, package_name: String, label: String) { + let temp = world.temp.as_ref().expect("tempdir exists"); + let inventory = temp.path().join("installed-inventory.json"); + fs::write( + &inventory, + serde_json::to_vec_pretty(&serde_json::json!({ + "format": "upgradeall-installed-inventory", + "version": 1, + "items": [ + { + "kind": "android", + "package_name": package_name, + "label": label, + "version_name": "1.0.0", + "version_code": 1, + } + ] + })) + .expect("inventory serializes"), + ) + .expect("write inventory"); + world.inventory = Some(inventory); +} + +#[given("an empty installed inventory")] +fn empty_installed_inventory(world: &mut CliWorld) { + let temp = world.temp.as_ref().expect("tempdir exists"); + let inventory = temp.path().join("installed-inventory.json"); + fs::write( + &inventory, + serde_json::to_vec_pretty(&serde_json::json!({ + "format": "upgradeall-installed-inventory", + "version": 1, + "items": [] + })) + .expect("inventory serializes"), + ) + .expect("write inventory"); + world.inventory = Some(inventory); +} + +#[given(expr = "a tampered autogen cleanup preview for package {string}")] +fn tampered_autogen_cleanup_preview_for_package(world: &mut CliWorld, package_id: String) { + let temp = world.temp.as_ref().expect("tempdir exists"); + let preview = temp.path().join("tampered-cleanup-preview.json"); + fs::write( + &preview, + serde_json::to_vec_pretty(&serde_json::json!({ + "operation": "cleanup.preview", + "target_repo_id": "local_autogen", + "target_repo_path": local_autogen_repo_path(world), + "summary": { + "candidate_count": 1, + "skipped_count": 0, + "write_count": 0, + "delete_count": 1, + }, + "candidates": [ + { + "package_id": package_id, + "action": "delete", + "output_relative_path": package_relative_path(&package_id), + "content_hash": "fnv1a64:0000000000000000", + "reason": "not_in_installed_inventory", + } + ], + "skipped": [], + "diagnostics": [], + })) + .expect("preview serializes"), + ) + .expect("write tampered cleanup preview"); + world.autogen_preview = Some(preview); +} + #[given(expr = "a fixture Lua repository {string} with package {string}")] fn fixture_lua_repository(world: &mut CliWorld, repo_id: String, package_id: String) { create_fixture_lua_repository(world, repo_id, package_id, "F-Droid".to_owned()); @@ -329,6 +407,113 @@ fn run_getter_legacy_report_list(world: &mut CliWorld) { world.json = None; } +#[when("I run getter autogen installed preview for that inventory")] +fn run_getter_autogen_installed_preview(world: &mut CliWorld) { + let inventory = world.inventory.as_ref().expect("inventory exists"); + let output = run_getter( + world, + [ + "autogen".to_owned(), + "installed".to_owned(), + "preview".to_owned(), + "--inventory".to_owned(), + inventory.to_string_lossy().to_string(), + ], + ); + world.output = Some(output); + world.json = None; +} + +#[when("I run getter autogen installed apply for that preview with accept-all")] +fn run_getter_autogen_installed_apply_accept_all(world: &mut CliWorld) { + let preview = world + .autogen_preview + .as_ref() + .expect("autogen preview exists"); + let output = run_getter( + world, + [ + "autogen".to_owned(), + "installed".to_owned(), + "apply".to_owned(), + "--preview".to_owned(), + preview.to_string_lossy().to_string(), + "--accept-all".to_owned(), + ], + ); + world.output = Some(output); + world.json = None; +} + +#[when("I run getter autogen cleanup preview for that inventory")] +fn run_getter_autogen_cleanup_preview(world: &mut CliWorld) { + let inventory = world.inventory.as_ref().expect("inventory exists"); + let output = run_getter( + world, + [ + "autogen".to_owned(), + "cleanup".to_owned(), + "preview".to_owned(), + "--inventory".to_owned(), + inventory.to_string_lossy().to_string(), + ], + ); + world.output = Some(output); + world.json = None; +} + +#[when("I run getter autogen cleanup apply for that preview with accept-all")] +fn run_getter_autogen_cleanup_apply_accept_all(world: &mut CliWorld) { + let preview = world + .autogen_preview + .as_ref() + .expect("autogen preview exists"); + let output = run_getter( + world, + [ + "autogen".to_owned(), + "cleanup".to_owned(), + "apply".to_owned(), + "--preview".to_owned(), + preview.to_string_lossy().to_string(), + "--accept-all".to_owned(), + ], + ); + world.output = Some(output); + world.json = None; +} + +#[when("I run getter repo validate for local_autogen")] +fn run_getter_repo_validate_for_local_autogen(world: &mut CliWorld) { + let repo_path = local_autogen_repo_path(world); + let output = run_getter( + world, + [ + "repo".to_owned(), + "validate".to_owned(), + repo_path.to_string_lossy().to_string(), + ], + ); + world.output = Some(output); + world.json = None; +} + +#[when(expr = "I run getter package eval for package {string} from local_autogen")] +fn run_getter_package_eval_from_local_autogen(world: &mut CliWorld, package_id: String) { + let output = run_getter( + world, + [ + "package".to_owned(), + "eval".to_owned(), + package_id, + "--repo".to_owned(), + "local_autogen".to_owned(), + ], + ); + world.output = Some(output); + world.json = None; +} + #[then("the command succeeds")] fn command_succeeds(world: &mut CliWorld) { let output = world.output.as_ref().expect("command output exists"); @@ -352,6 +537,17 @@ fn command_fails_with_migration_error(world: &mut CliWorld) { world.json = Some(json); } +#[then("the command fails with an autogen error")] +fn command_fails_with_autogen_error(world: &mut CliWorld) { + let output = world.output.as_ref().expect("command output exists"); + assert_eq!(output.status.code(), Some(1)); + let json = parse_stdout(output); + assert_eq!(json["ok"], false); + assert_eq!(json["command"], "autogen cleanup apply"); + assert_eq!(json["error"]["code"], "autogen.error"); + world.json = Some(json); +} + #[then(expr = "the command fails with direct DB migration error {string}")] fn command_fails_with_direct_db_migration_error(world: &mut CliWorld, code: String) { let output = world.output.as_ref().expect("command output exists"); @@ -515,6 +711,164 @@ fn output_contains_named_package(world: &mut CliWorld, package_id: String, packa assert_eq!(json["data"]["package"]["name"], package_name); } +#[then(expr = "the autogen preview contains candidate {string}")] +fn autogen_preview_contains_candidate(world: &mut CliWorld, package_id: String) { + let json = current_json(world); + assert_eq!(json["ok"], true); + assert_eq!(json["command"], "autogen installed preview"); + assert_eq!(json["data"]["operation"], "installed.preview"); + let candidates = json["data"]["candidates"] + .as_array() + .expect("candidates array"); + assert!( + candidates + .iter() + .any(|candidate| candidate["package_id"].as_str() == Some(package_id.as_str())), + "preview should contain {package_id}: {candidates:?}" + ); +} + +#[then("the local_autogen repository has not been written")] +fn local_autogen_repository_has_not_been_written(world: &mut CliWorld) { + assert!( + !local_autogen_repo_path(world).exists(), + "preview must not create local_autogen" + ); +} + +#[then("I save the autogen preview to a file")] +fn save_autogen_preview_to_file(world: &mut CliWorld) { + let data = current_json(world) + .get("data") + .expect("autogen command data exists") + .clone(); + let temp = world.temp.as_ref().expect("tempdir exists"); + let preview = temp.path().join("autogen-preview.json"); + fs::write( + &preview, + serde_json::to_vec_pretty(&data).expect("preview serializes"), + ) + .expect("write autogen preview"); + world.autogen_preview = Some(preview); +} + +#[then(expr = "the local_autogen repository contains generated package {string}")] +fn local_autogen_repository_contains_generated_package(world: &mut CliWorld, package_id: String) { + let path = local_autogen_repo_path(world).join(package_relative_path(&package_id)); + assert!(path.is_file(), "generated package should exist: {path:?}"); + let content = fs::read_to_string(&path).expect("generated package readable"); + assert!(content.contains("@generated by UpgradeAll getter local_autogen")); +} + +#[then(expr = "the app list contains autogen tracked package {string}")] +fn app_list_contains_autogen_tracked_package(world: &mut CliWorld, package_id: String) { + let output = run_getter(world, ["app".to_owned(), "list".to_owned()]); + assert_success(&output); + let json = parse_stdout(&output); + let apps = json["data"]["apps"].as_array().expect("apps array"); + let app = apps + .iter() + .find(|app| app["id"].as_str() == Some(package_id.as_str())) + .unwrap_or_else(|| panic!("app list should contain {package_id}: {apps:?}")); + assert_eq!(app["repository_id"], "local_autogen"); + assert_eq!(app["package_resolution"], "generate_local_package"); +} + +#[then(expr = "the autogen preview skips package {string} because repository {string} covers it")] +fn autogen_preview_skips_package_because_repository_covers_it( + world: &mut CliWorld, + package_id: String, + repository_id: String, +) { + let json = current_json(world); + assert_eq!(json["ok"], true); + assert_eq!(json["command"], "autogen installed preview"); + let skipped = json["data"]["skipped"].as_array().expect("skipped array"); + let skip = skipped + .iter() + .find(|skip| skip["package_id"].as_str() == Some(package_id.as_str())) + .unwrap_or_else(|| panic!("skipped should contain {package_id}: {skipped:?}")); + assert_eq!(skip["reason"], "covered_by_higher_priority_repo"); + assert_eq!(skip["covering_repo_id"], repository_id); +} + +#[then(expr = "the autogen cleanup preview contains delete candidate {string}")] +fn autogen_cleanup_preview_contains_delete_candidate(world: &mut CliWorld, package_id: String) { + let json = current_json(world); + assert_eq!(json["ok"], true); + assert_eq!(json["command"], "autogen cleanup preview"); + assert_eq!(json["data"]["operation"], "cleanup.preview"); + let candidates = json["data"]["candidates"] + .as_array() + .expect("candidates array"); + let candidate = candidates + .iter() + .find(|candidate| candidate["package_id"].as_str() == Some(package_id.as_str())) + .unwrap_or_else(|| { + panic!("cleanup candidates should contain {package_id}: {candidates:?}") + }); + assert_eq!(candidate["action"], "delete"); +} + +#[then(expr = "the local_autogen repository does not contain generated package {string}")] +fn local_autogen_repository_does_not_contain_generated_package( + world: &mut CliWorld, + package_id: String, +) { + let path = local_autogen_repo_path(world).join(package_relative_path(&package_id)); + assert!( + !path.exists(), + "generated package should be deleted: {path:?}" + ); +} + +#[then(expr = "the app list does not contain package {string}")] +fn app_list_does_not_contain_package(world: &mut CliWorld, package_id: String) { + let output = run_getter(world, ["app".to_owned(), "list".to_owned()]); + assert_success(&output); + let json = parse_stdout(&output); + let apps = json["data"]["apps"].as_array().expect("apps array"); + assert!( + apps.iter() + .all(|app| app["id"].as_str() != Some(package_id.as_str())), + "app list must not contain {package_id}: {apps:?}" + ); +} + +#[then(expr = "I replace generated autogen package {string} with user-edited content")] +fn replace_generated_autogen_package_with_user_edited_content( + world: &mut CliWorld, + package_id: String, +) { + let path = local_autogen_repo_path(world).join(package_relative_path(&package_id)); + assert!(path.is_file(), "generated package should exist before edit"); + fs::write( + &path, + format!( + "-- user edited\nreturn package_def {{ id = {id:?}, name = \"Edited Autogen\" }}\n", + id = package_id + ), + ) + .expect("overwrite generated package with user edit"); +} + +#[then(expr = "local repository contains preserved package {string}")] +fn local_repository_contains_preserved_package(world: &mut CliWorld, package_id: String) { + let local_path = world + .data_dir + .as_ref() + .expect("data dir exists") + .join("repositories") + .join("local") + .join(package_relative_path(&package_id)); + assert!( + local_path.is_file(), + "modified autogen file should be preserved into local: {local_path:?}" + ); + let content = fs::read_to_string(local_path).expect("preserved local file readable"); + assert!(content.contains("Edited Autogen")); +} + #[then("no partially usable imported state is created")] fn no_partially_usable_imported_state(world: &mut CliWorld) { let output = run_getter(world, ["app".to_owned(), "list".to_owned()]); @@ -685,6 +1039,24 @@ fn current_json(world: &mut CliWorld) -> &Value { .get_or_insert_with(|| parse_stdout(world.output.as_ref().expect("command output exists"))) } +fn local_autogen_repo_path(world: &CliWorld) -> PathBuf { + world + .data_dir + .as_ref() + .expect("data dir exists") + .join("repositories") + .join("local_autogen") +} + +fn package_relative_path(package_id: &str) -> PathBuf { + let (kind, name) = package_id + .split_once('/') + .expect("test package id has kind/name"); + PathBuf::from("packages") + .join(kind) + .join(format!("{name}.lua")) +} + fn create_fixture_lua_repository( world: &mut CliWorld, repo_id: String, diff --git a/crates/getter-cli/tests/features/cli/autogen_installed.feature b/crates/getter-cli/tests/features/cli/autogen_installed.feature new file mode 100644 index 0000000..fc3c16a --- /dev/null +++ b/crates/getter-cli/tests/features/cli/autogen_installed.feature @@ -0,0 +1,92 @@ +@getter-cli @autogen +Feature: Installed app autogen + Scenario: User previews installed app fallback generation without writing files + Given an initialized getter data directory + And an installed inventory with Android app "com.example.autogen" labeled "Example Autogen" + When I run getter autogen installed preview for that inventory + Then the command succeeds + And the output is valid JSON + And the autogen preview contains candidate "android/com.example.autogen" + And the local_autogen repository has not been written + + Scenario: User applies installed app autogen and evaluates the generated fallback package + Given an initialized getter data directory + And an installed inventory with Android app "com.example.autogen" labeled "Example Autogen" + When I run getter autogen installed preview for that inventory + Then the command succeeds + And I save the autogen preview to a file + When I run getter autogen installed apply for that preview with accept-all + Then the command succeeds + And the local_autogen repository contains generated package "android/com.example.autogen" + And the app list contains autogen tracked package "android/com.example.autogen" + When I run getter repo validate for local_autogen + Then the output reports a valid repository without network + When I run getter package eval for package "android/com.example.autogen" from local_autogen + Then the output contains package "android/com.example.autogen" named "Example Autogen" + + Scenario: Higher-priority repositories suppress installed app autogen candidates + Given an initialized getter data directory + And a fixture Lua repository "official" with package "android/com.example.autogen" named "Official Example" + And an installed inventory with Android app "com.example.autogen" labeled "Example Autogen" + When I run getter repo add for that repository with priority 0 + Then the command succeeds + When I run getter autogen installed preview for that inventory + Then the command succeeds + And the autogen preview skips package "android/com.example.autogen" because repository "official" covers it + + Scenario: Cleanup deletes only accepted generated packages missing from installed inventory + Given an initialized getter data directory + And an installed inventory with Android app "com.example.old" labeled "Old App" + When I run getter autogen installed preview for that inventory + Then the command succeeds + And I save the autogen preview to a file + When I run getter autogen installed apply for that preview with accept-all + Then the command succeeds + Given an empty installed inventory + When I run getter autogen cleanup preview for that inventory + Then the command succeeds + And the autogen cleanup preview contains delete candidate "android/com.example.old" + And I save the autogen preview to a file + When I run getter autogen cleanup apply for that preview with accept-all + Then the command succeeds + And the local_autogen repository does not contain generated package "android/com.example.old" + And the app list does not contain package "android/com.example.old" + + Scenario: Cleanup rejects tampered previews for non-autogen tracked packages + Given an initialized getter data directory + And a syntactically valid legacy export bundle with an Android app + And a tampered autogen cleanup preview for package "android/org.fdroid.fdroid" + When I run getter legacy import-room-bundle for that bundle + Then the command succeeds + When I run getter autogen cleanup apply for that preview with accept-all + Then the command fails with an autogen error + And the app list contains imported package "android/org.fdroid.fdroid" + + Scenario: Applying autogen preserves existing tracked user state + Given an initialized getter data directory + And a syntactically valid legacy export bundle with an Android app + When I run getter legacy import-room-bundle for that bundle + Then the command succeeds + Given an installed inventory with Android app "org.fdroid.fdroid" labeled "F-Droid" + When I run getter autogen installed preview for that inventory + Then the command succeeds + And I save the autogen preview to a file + When I run getter autogen installed apply for that preview with accept-all + Then the command succeeds + And the app list contains imported package "android/org.fdroid.fdroid" + + Scenario: Applying over modified autogen preserves the user-edited file into local + Given an initialized getter data directory + And an installed inventory with Android app "com.example.autogen" labeled "Example Autogen" + When I run getter autogen installed preview for that inventory + Then the command succeeds + And I save the autogen preview to a file + When I run getter autogen installed apply for that preview with accept-all + Then the command succeeds + And I replace generated autogen package "android/com.example.autogen" with user-edited content + When I run getter autogen installed preview for that inventory + Then the command succeeds + And I save the autogen preview to a file + When I run getter autogen installed apply for that preview with accept-all + Then the command succeeds + And local repository contains preserved package "android/com.example.autogen" diff --git a/crates/getter-core/src/autogen.rs b/crates/getter-core/src/autogen.rs new file mode 100644 index 0000000..265c6be --- /dev/null +++ b/crates/getter-core/src/autogen.rs @@ -0,0 +1,536 @@ +//! Installed-inventory autogen planning for generated fallback repositories. +//! +//! This module is pure domain logic: platform adapters provide installed +//! inventory facts, while getter decides package ids, generated Lua shape, +//! deterministic paths, and preview DTOs. + +use crate::repository::REPO_API_VERSION_V1; +use crate::{InstalledTarget, PackageId, PackageKind, RepositoryId, RepositoryPriority}; +use serde::{Deserialize, Serialize}; +use std::collections::{BTreeSet, HashMap}; +use std::path::PathBuf; + +pub const INSTALLED_INVENTORY_FORMAT: &str = "upgradeall-installed-inventory"; +pub const INSTALLED_INVENTORY_VERSION: u32 = 1; +pub const AUTOGEN_MANIFEST_VERSION: u32 = 1; +pub const LOCAL_REPOSITORY_ID: &str = "local"; +pub const LOCAL_REPOSITORY_NAME: &str = "Local"; +pub const LOCAL_AUTOGEN_REPOSITORY_ID: &str = "local_autogen"; +pub const LOCAL_AUTOGEN_REPOSITORY_NAME: &str = "UpgradeAll Local Autogen"; +pub const GENERATED_MARKER: &str = "@generated by UpgradeAll getter local_autogen"; + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct InstalledInventory { + pub format: String, + pub version: u32, + #[serde(default)] + pub items: Vec, +} + +impl InstalledInventory { + pub fn new(items: Vec) -> Self { + Self { + format: INSTALLED_INVENTORY_FORMAT.to_owned(), + version: INSTALLED_INVENTORY_VERSION, + items, + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "snake_case")] +pub enum InstalledInventoryItem { + #[serde(alias = "android")] + AndroidPackage { + package_name: String, + #[serde(default)] + label: Option, + #[serde(default)] + version_name: Option, + #[serde(default)] + version_code: Option, + }, + #[serde(alias = "magisk")] + MagiskModule { + module_id: String, + #[serde(default)] + name: Option, + #[serde(default)] + version_name: Option, + #[serde(default)] + version_code: Option, + }, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct AutogenPlan { + pub repository_id: RepositoryId, + pub repository_name: String, + pub repository_priority: RepositoryPriority, + pub candidates: Vec, + pub skipped: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct AutogenCandidate { + pub package_id: PackageId, + pub name: String, + pub installed: InstalledTarget, + pub relative_path: PathBuf, + pub content: String, + pub content_hash: String, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct AutogenSkip { + pub package_id: PackageId, + pub reason: AutogenSkipReason, + #[serde(default)] + pub repository_id: Option, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum AutogenSkipReason { + DuplicateInventoryItem, + CoveredByHigherPriorityRepository, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct AutogenManifest { + pub version: u32, + pub repository_id: RepositoryId, + pub packages: Vec, +} + +impl AutogenManifest { + pub fn from_candidates(candidates: &[AutogenCandidate]) -> Self { + Self { + version: AUTOGEN_MANIFEST_VERSION, + repository_id: RepositoryId::new(LOCAL_AUTOGEN_REPOSITORY_ID) + .expect("local_autogen is a valid repository id"), + packages: candidates + .iter() + .map(|candidate| AutogenManifestEntry { + package_id: candidate.package_id.clone(), + relative_path: candidate.relative_path.clone(), + content_hash: candidate.content_hash.clone(), + }) + .collect(), + } + } + + pub fn package(&self, package_id: &PackageId) -> Option<&AutogenManifestEntry> { + self.packages + .iter() + .find(|entry| &entry.package_id == package_id) + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct AutogenManifestEntry { + pub package_id: PackageId, + pub relative_path: PathBuf, + pub content_hash: String, +} + +#[derive(Debug, thiserror::Error)] +pub enum AutogenError { + #[error("unsupported installed inventory format '{0}'")] + UnsupportedInventoryFormat(String), + #[error("unsupported installed inventory version {found}; expected {expected}")] + UnsupportedInventoryVersion { found: u32, expected: u32 }, + #[error("installed inventory item has an empty identity")] + EmptyIdentity, + #[error("failed to construct package id from installed inventory: {0}")] + PackageId(#[from] crate::PackageIdError), +} + +/// Build a deterministic local_autogen preview from installed inventory. +/// +/// `covered_packages` contains package ids already supplied by repositories with +/// priority higher than `local_autogen`. Those packages are skipped so generated +/// fallback files do not compete with user-authored `local` or upstream package +/// files. +pub fn plan_local_autogen( + inventory: &InstalledInventory, + covered_packages: &HashMap, +) -> Result { + validate_installed_inventory(inventory)?; + let mut seen = BTreeSet::new(); + let mut candidates = Vec::new(); + let mut skipped = Vec::new(); + + for item in &inventory.items { + let candidate = candidate_from_inventory_item(item)?; + if !seen.insert(candidate.package_id.to_string()) { + skipped.push(AutogenSkip { + package_id: candidate.package_id, + reason: AutogenSkipReason::DuplicateInventoryItem, + repository_id: None, + }); + continue; + } + if let Some(repository_id) = covered_packages.get(&candidate.package_id) { + skipped.push(AutogenSkip { + package_id: candidate.package_id, + reason: AutogenSkipReason::CoveredByHigherPriorityRepository, + repository_id: Some(repository_id.clone()), + }); + continue; + } + candidates.push(candidate); + } + + candidates.sort_by_key(|candidate| candidate.package_id.to_string()); + skipped.sort_by_key(|skip| skip.package_id.to_string()); + + Ok(AutogenPlan { + repository_id: RepositoryId::new(LOCAL_AUTOGEN_REPOSITORY_ID) + .expect("local_autogen is a valid repository id"), + repository_name: LOCAL_AUTOGEN_REPOSITORY_NAME.to_owned(), + repository_priority: RepositoryPriority::LOCAL_AUTOGEN, + candidates, + skipped, + }) +} + +pub fn validate_installed_inventory(inventory: &InstalledInventory) -> Result<(), AutogenError> { + if inventory.format != INSTALLED_INVENTORY_FORMAT { + return Err(AutogenError::UnsupportedInventoryFormat( + inventory.format.clone(), + )); + } + if inventory.version != INSTALLED_INVENTORY_VERSION { + return Err(AutogenError::UnsupportedInventoryVersion { + found: inventory.version, + expected: INSTALLED_INVENTORY_VERSION, + }); + } + Ok(()) +} + +pub fn local_autogen_repo_toml() -> String { + repo_toml( + LOCAL_AUTOGEN_REPOSITORY_ID, + LOCAL_AUTOGEN_REPOSITORY_NAME, + RepositoryPriority::LOCAL_AUTOGEN, + ) +} + +pub fn local_repo_toml() -> String { + repo_toml( + LOCAL_REPOSITORY_ID, + LOCAL_REPOSITORY_NAME, + RepositoryPriority::LOCAL, + ) +} + +fn repo_toml(id: &str, name: &str, priority: RepositoryPriority) -> String { + format!( + "id = \"{id}\"\nname = \"{name}\"\npriority = {}\napi_version = \"{REPO_API_VERSION_V1}\"\n", + priority.value() + ) +} + +pub fn generated_file_is_managed(content: &str) -> bool { + content + .lines() + .next() + .is_some_and(|line| line.contains(GENERATED_MARKER)) +} + +pub fn content_hash(content: &str) -> String { + format!("fnv1a64:{:016x}", fnv1a64(content.as_bytes())) +} + +fn candidate_from_inventory_item( + item: &InstalledInventoryItem, +) -> Result { + match item { + InstalledInventoryItem::AndroidPackage { + package_name, + label, + .. + } => { + let package_name = non_empty(package_name)?; + let package_id = PackageId::new(PackageKind::Android, package_name)?; + let name = label + .as_deref() + .filter(|label| !label.trim().is_empty()) + .unwrap_or(package_name) + .to_owned(); + let installed = InstalledTarget::AndroidPackage { + package_name: package_name.to_owned(), + }; + let relative_path = package_relative_path(&package_id); + let content = render_package_lua(&package_id, &name, &installed); + let content_hash = content_hash(&content); + Ok(AutogenCandidate { + package_id, + name, + installed, + relative_path, + content, + content_hash, + }) + } + InstalledInventoryItem::MagiskModule { + module_id, name, .. + } => { + let module_id = non_empty(module_id)?; + let package_id = PackageId::new(PackageKind::Magisk, module_id)?; + let name = name + .as_deref() + .filter(|name| !name.trim().is_empty()) + .unwrap_or(module_id) + .to_owned(); + let installed = InstalledTarget::MagiskModule { + module_id: module_id.to_owned(), + }; + let relative_path = package_relative_path(&package_id); + let content = render_package_lua(&package_id, &name, &installed); + let content_hash = content_hash(&content); + Ok(AutogenCandidate { + package_id, + name, + installed, + relative_path, + content, + content_hash, + }) + } + } +} + +fn non_empty(value: &str) -> Result<&str, AutogenError> { + let value = value.trim(); + if value.is_empty() { + Err(AutogenError::EmptyIdentity) + } else { + Ok(value) + } +} + +pub fn package_relative_path(package_id: &PackageId) -> PathBuf { + let mut path = PathBuf::from("packages"); + path.push(package_id.kind().as_str()); + path.push(format!("{}.lua", package_id.name())); + path +} + +fn render_package_lua(package_id: &PackageId, name: &str, installed: &InstalledTarget) -> String { + let installed_lua = match installed { + InstalledTarget::AndroidPackage { package_name } => format!( + "{{ kind = \"android_package\", package_name = {} }}", + lua_string(package_name) + ), + InstalledTarget::MagiskModule { module_id } => format!( + "{{ kind = \"magisk_module\", module_id = {} }}", + lua_string(module_id) + ), + InstalledTarget::Generic { id } => { + format!("{{ kind = \"generic\", id = {} }}", lua_string(id)) + } + }; + + format!( + "-- {GENERATED_MARKER}\nreturn package_def {{\n id = {},\n name = {},\n installed = {{\n {installed_lua},\n }},\n}}\n", + lua_string(&package_id.to_string()), + lua_string(name) + ) +} + +fn lua_string(value: &str) -> String { + let mut escaped = String::from("\""); + for ch in value.chars() { + match ch { + '\\' => escaped.push_str("\\\\"), + '"' => escaped.push_str("\\\""), + '\n' => escaped.push_str("\\n"), + '\r' => escaped.push_str("\\r"), + '\t' => escaped.push_str("\\t"), + other => escaped.push(other), + } + } + escaped.push('"'); + escaped +} + +fn fnv1a64(bytes: &[u8]) -> u64 { + let mut hash = 0xcbf29ce484222325u64; + for byte in bytes { + hash ^= u64::from(*byte); + hash = hash.wrapping_mul(0x100000001b3); + } + hash +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::lua::evaluate_package_file; + use crate::repository::RepositoryLayout; + use std::fs; + + #[test] + fn preview_generates_deterministic_android_package_lua() { + let inventory = InstalledInventory::new(vec![InstalledInventoryItem::AndroidPackage { + package_name: "org.fdroid.fdroid".to_owned(), + label: Some("F-Droid".to_owned()), + version_name: Some("1.20.0".to_owned()), + version_code: Some(1020000), + }]); + + let plan = plan_local_autogen(&inventory, &HashMap::new()).unwrap(); + + assert_eq!(plan.repository_id.as_str(), LOCAL_AUTOGEN_REPOSITORY_ID); + assert_eq!(plan.repository_priority, RepositoryPriority::LOCAL_AUTOGEN); + assert_eq!(plan.candidates.len(), 1); + let candidate = &plan.candidates[0]; + assert_eq!( + candidate.package_id.to_string(), + "android/org.fdroid.fdroid" + ); + assert_eq!( + candidate.relative_path, + PathBuf::from("packages/android/org.fdroid.fdroid.lua") + ); + assert!(candidate.content.contains(GENERATED_MARKER)); + assert!(candidate.content.contains("package_def")); + assert_eq!(candidate.content_hash, content_hash(&candidate.content)); + } + + #[test] + fn preview_skips_packages_covered_by_higher_priority_repositories() { + let package_id: PackageId = "android/org.fdroid.fdroid".parse().unwrap(); + let mut covered = HashMap::new(); + covered.insert(package_id, RepositoryId::new("official").unwrap()); + let inventory = InstalledInventory::new(vec![InstalledInventoryItem::AndroidPackage { + package_name: "org.fdroid.fdroid".to_owned(), + label: Some("F-Droid".to_owned()), + version_name: None, + version_code: None, + }]); + + let plan = plan_local_autogen(&inventory, &covered).unwrap(); + + assert!(plan.candidates.is_empty()); + assert_eq!(plan.skipped.len(), 1); + assert_eq!( + plan.skipped[0].reason, + AutogenSkipReason::CoveredByHigherPriorityRepository + ); + assert_eq!( + plan.skipped[0] + .repository_id + .as_ref() + .map(RepositoryId::as_str), + Some("official") + ); + } + + #[test] + fn preview_deduplicates_inventory_items() { + let inventory = InstalledInventory::new(vec![ + InstalledInventoryItem::AndroidPackage { + package_name: "org.fdroid.fdroid".to_owned(), + label: Some("F-Droid".to_owned()), + version_name: None, + version_code: None, + }, + InstalledInventoryItem::AndroidPackage { + package_name: "org.fdroid.fdroid".to_owned(), + label: Some("F-Droid Duplicate".to_owned()), + version_name: None, + version_code: None, + }, + ]); + + let plan = plan_local_autogen(&inventory, &HashMap::new()).unwrap(); + + assert_eq!(plan.candidates.len(), 1); + assert_eq!(plan.skipped.len(), 1); + assert_eq!( + plan.skipped[0].reason, + AutogenSkipReason::DuplicateInventoryItem + ); + } + + #[test] + fn rejects_unsupported_inventory_contracts() { + let inventory = InstalledInventory { + format: "other".to_owned(), + version: INSTALLED_INVENTORY_VERSION, + items: Vec::new(), + }; + + assert!(matches!( + plan_local_autogen(&inventory, &HashMap::new()), + Err(AutogenError::UnsupportedInventoryFormat(_)) + )); + } + + #[test] + fn manifest_records_generated_package_hashes() { + let inventory = InstalledInventory::new(vec![InstalledInventoryItem::AndroidPackage { + package_name: "org.fdroid.fdroid".to_owned(), + label: Some("F-Droid".to_owned()), + version_name: None, + version_code: None, + }]); + let plan = plan_local_autogen(&inventory, &HashMap::new()).unwrap(); + + let manifest = AutogenManifest::from_candidates(&plan.candidates); + + assert_eq!(manifest.version, AUTOGEN_MANIFEST_VERSION); + assert_eq!(manifest.repository_id.as_str(), LOCAL_AUTOGEN_REPOSITORY_ID); + assert_eq!(manifest.packages.len(), 1); + assert_eq!( + manifest.packages[0].content_hash, + content_hash(&plan.candidates[0].content) + ); + } + + #[test] + fn generated_lua_evaluates_through_package_boundary() { + let temp = tempfile::tempdir().unwrap(); + let repo = temp.path(); + fs::write(repo.join("repo.toml"), local_autogen_repo_toml()).unwrap(); + fs::create_dir_all(repo.join("packages/android")).unwrap(); + fs::create_dir(repo.join("lib")).unwrap(); + fs::create_dir(repo.join("templates")).unwrap(); + let inventory = InstalledInventory::new(vec![InstalledInventoryItem::AndroidPackage { + package_name: "org.fdroid.fdroid".to_owned(), + label: Some("F-Droid \"client\"".to_owned()), + version_name: None, + version_code: None, + }]); + let candidate = plan_local_autogen(&inventory, &HashMap::new()) + .unwrap() + .candidates + .remove(0); + let package_path = repo.join(&candidate.relative_path); + fs::write(&package_path, candidate.content).unwrap(); + + let layout = RepositoryLayout::load(repo).unwrap(); + let package = evaluate_package_file(&layout, &package_path).unwrap(); + + assert_eq!(package.id.to_string(), "android/org.fdroid.fdroid"); + assert_eq!(package.name, "F-Droid \"client\""); + assert_eq!( + package.installed, + vec![InstalledTarget::AndroidPackage { + package_name: "org.fdroid.fdroid".to_owned(), + }] + ); + } + + #[test] + fn generated_file_marker_identifies_managed_files_only() { + assert!(generated_file_is_managed(&format!( + "-- {GENERATED_MARKER}\nreturn {{}}" + ))); + assert!(!generated_file_is_managed("return {}")); + } +} diff --git a/crates/getter-core/src/lib.rs b/crates/getter-core/src/lib.rs index 32c110b..642e36f 100644 --- a/crates/getter-core/src/lib.rs +++ b/crates/getter-core/src/lib.rs @@ -9,6 +9,7 @@ use std::cmp::Ordering; use std::fmt; use std::str::FromStr; +pub mod autogen; pub mod diagnostics; pub mod lua; pub mod repository; diff --git a/crates/getter-storage/src/lib.rs b/crates/getter-storage/src/lib.rs index a6f0e56..f1285c2 100644 --- a/crates/getter-storage/src/lib.rs +++ b/crates/getter-storage/src/lib.rs @@ -184,6 +184,43 @@ ON CONFLICT(id) DO UPDATE SET Ok(()) } + pub fn upsert_generated_tracked_package_preserving_user_state( + &self, + package_id: &PackageId, + repository_id: &RepositoryId, + ) -> Result<(), StorageError> { + self.conn.execute( + r#" +INSERT INTO tracked_packages( + package_id, + enabled, + favorite, + ignored_version, + repository_id, + package_resolution +) +VALUES (?1, 1, 0, NULL, ?2, ?3) +ON CONFLICT(package_id) DO UPDATE SET + repository_id = CASE + WHEN tracked_packages.package_resolution = 'missing_package_definition' + THEN excluded.repository_id + ELSE tracked_packages.repository_id + END, + package_resolution = CASE + WHEN tracked_packages.package_resolution = 'missing_package_definition' + THEN excluded.package_resolution + ELSE tracked_packages.package_resolution + END +"#, + params![ + package_id.to_string(), + repository_id.as_str(), + StoredPackageResolution::GenerateLocalPackage.as_str(), + ], + )?; + Ok(()) + } + pub fn import_tracked_packages_with_migration_record( &self, packages: &[TrackedPackageUpsert], @@ -208,6 +245,27 @@ ON CONFLICT(id) DO UPDATE SET Ok(()) } + pub fn delete_generated_tracked_package( + &self, + package_id: &PackageId, + repository_id: &RepositoryId, + ) -> Result { + let deleted = self.conn.execute( + r#" +DELETE FROM tracked_packages +WHERE package_id = ?1 + AND repository_id = ?2 + AND package_resolution = ?3 +"#, + params![ + package_id.to_string(), + repository_id.as_str(), + StoredPackageResolution::GenerateLocalPackage.as_str(), + ], + )?; + Ok(deleted != 0) + } + pub fn tracked_packages(&self) -> Result, StorageError> { let mut stmt = self.conn.prepare( r#" @@ -535,6 +593,119 @@ mod tests { ); } + #[test] + fn generated_tracking_preserves_existing_user_state_on_conflict() { + let db = MainDb::open_in_memory().unwrap(); + let package_id: PackageId = "android/org.fdroid.fdroid".parse().unwrap(); + let local_autogen = RepositoryId::new("local_autogen").unwrap(); + db.upsert_tracked_package(&TrackedPackageUpsert { + package_id: package_id.clone(), + enabled: false, + favorite: true, + ignored_version: Some("9.9.9".to_owned()), + repository_id: None, + package_resolution: StoredPackageResolution::OfficialRepositoryPackage, + }) + .unwrap(); + + db.upsert_generated_tracked_package_preserving_user_state(&package_id, &local_autogen) + .unwrap(); + + let packages = db.tracked_packages().unwrap(); + assert_eq!(packages.len(), 1); + assert!(!packages[0].enabled); + assert!(packages[0].favorite); + assert_eq!(packages[0].ignored_version.as_deref(), Some("9.9.9")); + assert_eq!(packages[0].repository_id, None); + assert_eq!( + packages[0].package_resolution, + StoredPackageResolution::OfficialRepositoryPackage + ); + } + + #[test] + fn generated_tracking_fills_unresolved_tracking_metadata() { + let db = MainDb::open_in_memory().unwrap(); + let package_id: PackageId = "android/org.fdroid.fdroid".parse().unwrap(); + let local_autogen = insert_local_autogen_repo(&db); + db.upsert_tracked_package(&TrackedPackageUpsert { + package_id: package_id.clone(), + enabled: false, + favorite: true, + ignored_version: Some("9.9.9".to_owned()), + repository_id: None, + package_resolution: StoredPackageResolution::MissingPackageDefinition, + }) + .unwrap(); + + db.upsert_generated_tracked_package_preserving_user_state(&package_id, &local_autogen) + .unwrap(); + + let packages = db.tracked_packages().unwrap(); + assert_eq!(packages.len(), 1); + assert!(!packages[0].enabled); + assert!(packages[0].favorite); + assert_eq!(packages[0].ignored_version.as_deref(), Some("9.9.9")); + assert_eq!(packages[0].repository_id.as_ref(), Some(&local_autogen)); + assert_eq!( + packages[0].package_resolution, + StoredPackageResolution::GenerateLocalPackage + ); + } + + #[test] + fn generated_tracking_delete_is_guarded_by_repo_and_resolution() { + let db = MainDb::open_in_memory().unwrap(); + let package_id: PackageId = "android/org.fdroid.fdroid".parse().unwrap(); + let local_autogen = insert_local_autogen_repo(&db); + db.upsert_tracked_package(&TrackedPackageUpsert { + package_id: package_id.clone(), + enabled: true, + favorite: true, + ignored_version: Some("9.9.9".to_owned()), + repository_id: None, + package_resolution: StoredPackageResolution::OfficialRepositoryPackage, + }) + .unwrap(); + + assert!(!db + .delete_generated_tracked_package(&package_id, &local_autogen) + .unwrap()); + assert_eq!(db.tracked_packages().unwrap().len(), 1); + + db.upsert_tracked_package(&TrackedPackageUpsert { + package_id: package_id.clone(), + enabled: true, + favorite: true, + ignored_version: Some("9.9.9".to_owned()), + repository_id: None, + package_resolution: StoredPackageResolution::MissingPackageDefinition, + }) + .unwrap(); + db.upsert_generated_tracked_package_preserving_user_state(&package_id, &local_autogen) + .unwrap(); + assert!(db + .delete_generated_tracked_package(&package_id, &local_autogen) + .unwrap()); + assert!(db.tracked_packages().unwrap().is_empty()); + } + + fn insert_local_autogen_repo(db: &MainDb) -> RepositoryId { + let local_autogen = RepositoryId::new("local_autogen").unwrap(); + db.upsert_repository( + &RepositoryMetadata { + id: local_autogen.clone(), + name: "Local Autogen".to_owned(), + priority: RepositoryPriority::LOCAL_AUTOGEN, + api_version: REPO_API_VERSION_V1.to_owned(), + }, + None, + None, + ) + .unwrap(); + local_autogen + } + #[test] fn main_db_records_migration_completion() { let db = MainDb::open_in_memory().unwrap(); From 196cd8c2c449a17ef4c58f4fa401a2f6d93efe7b Mon Sep 17 00:00:00 2001 From: Xiangzhe Date: Tue, 23 Jun 2026 00:56:49 +0800 Subject: [PATCH 15/52] feat(cli): add offline update check --- crates/getter-cli/src/lib.rs | 43 ++- crates/getter-cli/tests/bdd_cli.rs | 202 +++++++++++ .../tests/features/cli/update_check.feature | 68 ++++ crates/getter-core/src/update.rs | 319 +++++++++++++++++- 4 files changed, 628 insertions(+), 4 deletions(-) create mode 100644 crates/getter-cli/tests/features/cli/update_check.feature diff --git a/crates/getter-cli/src/lib.rs b/crates/getter-cli/src/lib.rs index 72d92fc..0732d41 100644 --- a/crates/getter-cli/src/lib.rs +++ b/crates/getter-cli/src/lib.rs @@ -13,6 +13,7 @@ use getter_core::autogen::{ use getter_core::diagnostics::validate_repository_path; use getter_core::lua::evaluate_package_file; use getter_core::repository::{RepositoryLayout, RepositoryMetadata, REPO_API_VERSION_V1}; +use getter_core::update::{run_offline_update_check, OfflineUpdateCheckFixture}; use getter_core::{PackageId, RepositoryId, RepositoryPriority}; use getter_storage::legacy_room::{ map_legacy_app, read_legacy_room_database, LegacyAppKind, LegacyAppRecord, @@ -63,6 +64,9 @@ pub enum CliCommand { repo_id: Option, }, StorageValidate, + UpdateCheck { + fixture: PathBuf, + }, AutogenInstalledPreview { inventory: PathBuf, }, @@ -117,6 +121,8 @@ pub enum CliError { Repository(String), #[error("package evaluation error: {0}")] PackageEval(String), + #[error("update check error: {0}")] + Update(String), #[error("autogen error: {0}")] Autogen(String), #[error("Legacy Room export bundle is invalid")] @@ -134,7 +140,7 @@ impl CliError { match self { Self::Usage(_) => ExitCode::Usage, Self::Storage(_) => ExitCode::Storage, - Self::Repository(_) | Self::PackageEval(_) | Self::Autogen(_) => { + Self::Repository(_) | Self::PackageEval(_) | Self::Update(_) | Self::Autogen(_) => { ExitCode::GenericFailure } Self::InvalidLegacyBundle { .. } @@ -150,6 +156,7 @@ impl CliError { Self::Storage(_) => "storage.error", Self::Repository(_) => "repository.error", Self::PackageEval(_) => "package.eval_error", + Self::Update(_) => "update.check_error", Self::Autogen(_) => "autogen.error", Self::InvalidLegacyBundle { .. } => "migration.invalid_bundle", Self::UnsupportedLegacyBundle { .. } => "migration.unsupported_bundle", @@ -164,6 +171,7 @@ impl CliError { Self::Storage(_) => "Getter storage operation failed", Self::Repository(_) => "Getter repository operation failed", Self::PackageEval(_) => "Getter package evaluation failed", + Self::Update(_) => "Getter update check failed", Self::Autogen(_) => "Getter autogen operation failed", Self::InvalidLegacyBundle { .. } => "Legacy Room export bundle is invalid", Self::UnsupportedLegacyBundle { .. } => { @@ -180,6 +188,7 @@ impl CliError { | Self::Storage(detail) | Self::Repository(detail) | Self::PackageEval(detail) + | Self::Update(detail) | Self::Autogen(detail) => Some(detail.as_str()), Self::InvalidLegacyBundle { .. } | Self::UnsupportedLegacyBundle { .. } @@ -198,6 +207,7 @@ impl CliError { | Self::Storage(_) | Self::Repository(_) | Self::PackageEval(_) + | Self::Update(_) | Self::Autogen(_) => None, } } @@ -326,6 +336,13 @@ where [domain, command] if domain == "storage" && command == "validate" => { CliCommand::StorageValidate } + [domain, command, flag, fixture] + if domain == "update" && command == "check" && flag == "--fixture" => + { + CliCommand::UpdateCheck { + fixture: PathBuf::from(fixture), + } + } [domain, subject, action, flag, inventory] if domain == "autogen" && subject == "installed" @@ -471,6 +488,16 @@ fn execute(invocation: CliInvocation) -> Result { "cache_db": cache_db_path(&invocation.data_dir), })) } + CliCommand::UpdateCheck { fixture } => { + open_initialized_storage(&invocation.data_dir)?; + let fixture = read_update_check_fixture(&fixture)?; + serde_json::to_value(run_offline_update_check(fixture).map_err(|source| { + CliError::Update(format!("offline update check failed: {source}")) + })?) + .map_err(|source| { + CliError::Update(format!("failed to serialize update check: {source}")) + }) + } CliCommand::AutogenInstalledPreview { inventory } => { let db = open_main_db(&invocation.data_dir)?; let inventory = read_installed_inventory(&inventory)?; @@ -754,6 +781,17 @@ fn read_installed_inventory(path: &Path) -> Result .map_err(|source| CliError::Autogen(format!("failed to parse inventory JSON: {source}"))) } +fn read_update_check_fixture(path: &Path) -> Result { + let bytes = fs::read(path).map_err(|source| { + CliError::Update(format!("failed to read update check fixture: {source}")) + })?; + serde_json::from_slice(&bytes).map_err(|source| { + CliError::Update(format!( + "failed to parse update check fixture JSON: {source}" + )) + }) +} + fn build_local_autogen_plan( db: &MainDb, inventory: &InstalledInventory, @@ -1684,7 +1722,7 @@ fn envelope_to_string(value: Value) -> String { } fn usage_text() -> String { - "Usage: getter --data-dir [--priority ]|repo eval |repo validate |package eval [--repo ]|storage validate|hub list|autogen installed preview --inventory |autogen installed apply --preview (--accept-all|--accept ...)|autogen cleanup preview --inventory |autogen cleanup apply --preview (--accept-all|--accept ...)|legacy import-room-bundle |legacy import-room-db |legacy report-list>\n".to_owned() + "Usage: getter --data-dir [--priority ]|repo eval |repo validate |package eval [--repo ]|storage validate|hub list|update check --fixture |autogen installed preview --inventory |autogen installed apply --preview (--accept-all|--accept ...)|autogen cleanup preview --inventory |autogen cleanup apply --preview (--accept-all|--accept ...)|legacy import-room-bundle |legacy import-room-db |legacy report-list>\n".to_owned() } #[derive(Debug, Deserialize)] @@ -1737,6 +1775,7 @@ impl CliCommand { Self::RepoValidate { .. } => "repo validate", Self::PackageEval { .. } => "package eval", Self::StorageValidate => "storage validate", + Self::UpdateCheck { .. } => "update check", Self::AutogenInstalledPreview { .. } => "autogen installed preview", Self::AutogenInstalledApply { .. } => "autogen installed apply", Self::AutogenCleanupPreview { .. } => "autogen cleanup preview", diff --git a/crates/getter-cli/tests/bdd_cli.rs b/crates/getter-cli/tests/bdd_cli.rs index dfe31e2..fdef3e5 100644 --- a/crates/getter-cli/tests/bdd_cli.rs +++ b/crates/getter-cli/tests/bdd_cli.rs @@ -14,6 +14,7 @@ struct CliWorld { legacy_db: Option, inventory: Option, autogen_preview: Option, + update_fixture: Option, fixture_repo_id: Option, fixture_repo_path: Option, fixture_package_id: Option, @@ -187,6 +188,79 @@ fn tampered_autogen_cleanup_preview_for_package(world: &mut CliWorld, package_id world.autogen_preview = Some(preview); } +#[given( + expr = "an offline update fixture for package {string} installed version {string} with candidate versions {string}" +)] +fn offline_update_fixture_with_installed_version( + world: &mut CliWorld, + package_id: String, + installed_version: String, + versions: String, +) { + write_offline_update_fixture(world, package_id, Some(installed_version), None, versions); +} + +#[given( + expr = "an offline update fixture for package {string} installed version {string} ignored version {string} with candidate versions {string}" +)] +fn offline_update_fixture_with_ignored_version( + world: &mut CliWorld, + package_id: String, + installed_version: String, + ignored_version: String, + versions: String, +) { + write_offline_update_fixture( + world, + package_id, + Some(installed_version), + Some(ignored_version), + versions, + ); +} + +#[given( + expr = "an offline update fixture for package {string} without installed version with candidate versions {string}" +)] +fn offline_update_fixture_without_installed_version( + world: &mut CliWorld, + package_id: String, + versions: String, +) { + write_offline_update_fixture(world, package_id, None, None, versions); +} + +#[given( + expr = "an offline update fixture for package {string} installed version {string} with artifactless candidate version {string}" +)] +fn offline_update_fixture_with_artifactless_candidate( + world: &mut CliWorld, + package_id: String, + installed_version: String, + candidate_version: String, +) { + write_offline_update_fixture_with_candidates( + world, + package_id, + Some(installed_version), + None, + vec![serde_json::json!({ + "version": candidate_version, + "channel": "stable", + "source": "offline-fixture", + "artifacts": [], + })], + ); +} + +#[given("a malformed offline update fixture")] +fn malformed_offline_update_fixture(world: &mut CliWorld) { + let temp = world.temp.as_ref().expect("tempdir exists"); + let fixture = temp.path().join("malformed-update-fixture.json"); + fs::write(&fixture, "not-json").expect("write malformed fixture"); + world.update_fixture = Some(fixture); +} + #[given(expr = "a fixture Lua repository {string} with package {string}")] fn fixture_lua_repository(world: &mut CliWorld, repo_id: String, package_id: String) { create_fixture_lua_repository(world, repo_id, package_id, "F-Droid".to_owned()); @@ -407,6 +481,25 @@ fn run_getter_legacy_report_list(world: &mut CliWorld) { world.json = None; } +#[when("I run getter update check for that fixture")] +fn run_getter_update_check(world: &mut CliWorld) { + let fixture = world + .update_fixture + .as_ref() + .expect("update fixture exists"); + let output = run_getter( + world, + [ + "update".to_owned(), + "check".to_owned(), + "--fixture".to_owned(), + fixture.to_string_lossy().to_string(), + ], + ); + world.output = Some(output); + world.json = None; +} + #[when("I run getter autogen installed preview for that inventory")] fn run_getter_autogen_installed_preview(world: &mut CliWorld) { let inventory = world.inventory.as_ref().expect("inventory exists"); @@ -548,6 +641,17 @@ fn command_fails_with_autogen_error(world: &mut CliWorld) { world.json = Some(json); } +#[then("the command fails with an update check error")] +fn command_fails_with_update_check_error(world: &mut CliWorld) { + let output = world.output.as_ref().expect("command output exists"); + assert_eq!(output.status.code(), Some(1)); + let json = parse_stdout(output); + assert_eq!(json["ok"], false); + assert_eq!(json["command"], "update check"); + assert_eq!(json["error"]["code"], "update.check_error"); + world.json = Some(json); +} + #[then(expr = "the command fails with direct DB migration error {string}")] fn command_fails_with_direct_db_migration_error(world: &mut CliWorld, code: String) { let output = world.output.as_ref().expect("command output exists"); @@ -711,6 +815,44 @@ fn output_contains_named_package(world: &mut CliWorld, package_id: String, packa assert_eq!(json["data"]["package"]["name"], package_name); } +#[then(expr = "the update check status is {string}")] +fn update_check_status_is(world: &mut CliWorld, status: String) { + let json = current_json(world); + assert_eq!(json["ok"], true); + assert_eq!(json["command"], "update check"); + assert_eq!(json["data"]["network_required"], false); + assert_eq!(json["data"]["status"], status); +} + +#[then(expr = "the selected update version is {string}")] +fn selected_update_version_is(world: &mut CliWorld, version: String) { + let json = current_json(world); + assert_eq!(json["data"]["selected"]["candidate"]["version"], version); +} + +#[then(expr = "the update check actions download file {string} and request installer {string}")] +fn update_check_actions_download_and_install( + world: &mut CliWorld, + file_name: String, + installer: String, +) { + let json = current_json(world); + let actions = json["data"]["actions"].as_array().expect("actions array"); + assert_eq!(actions.len(), 2); + assert_eq!(actions[0]["type"], "download"); + assert_eq!(actions[0]["file_name"], file_name); + assert_eq!(actions[1]["type"], "install"); + assert_eq!(actions[1]["installer"], installer); + assert_eq!(actions[1]["file"], file_name); +} + +#[then("the update check has no selected update")] +fn update_check_has_no_selected_update(world: &mut CliWorld) { + let json = current_json(world); + assert!(json["data"]["selected"].is_null()); + assert_eq!(json["data"]["actions"], Value::Array(Vec::new())); +} + #[then(expr = "the autogen preview contains candidate {string}")] fn autogen_preview_contains_candidate(world: &mut CliWorld, package_id: String) { let json = current_json(world); @@ -1099,6 +1241,66 @@ return package_def {{ world.fixture_package_id = Some(package_id); } +fn write_offline_update_fixture( + world: &mut CliWorld, + package_id: String, + installed_version: Option, + ignored_version: Option, + versions: String, +) { + let candidates: Vec = versions + .split(',') + .map(str::trim) + .filter(|version| !version.is_empty()) + .map(|version| { + serde_json::json!({ + "version": version, + "channel": "stable", + "source": "offline-fixture", + "artifacts": [ + { + "name": "APK", + "url": format!("https://example.invalid/{version}.apk"), + "file_name": "app.apk", + } + ], + }) + }) + .collect(); + write_offline_update_fixture_with_candidates( + world, + package_id, + installed_version, + ignored_version, + candidates, + ); +} + +fn write_offline_update_fixture_with_candidates( + world: &mut CliWorld, + package_id: String, + installed_version: Option, + ignored_version: Option, + candidates: Vec, +) { + let temp = world.temp.as_ref().expect("tempdir exists"); + let fixture = temp.path().join("offline-update-fixture.json"); + fs::write( + &fixture, + serde_json::to_vec_pretty(&serde_json::json!({ + "format": "getter-offline-update-check", + "version": 1, + "package_id": package_id, + "installed_version": installed_version, + "ignored_version": ignored_version, + "candidates": candidates, + })) + .expect("fixture serializes"), + ) + .expect("write offline update fixture"); + world.update_fixture = Some(fixture); +} + fn create_fixture_legacy_room_db(path: &PathBuf, version: u32, include_app_table: bool) { let conn = Connection::open(path).expect("create legacy Room fixture"); conn.pragma_update(None, "user_version", version) diff --git a/crates/getter-cli/tests/features/cli/update_check.feature b/crates/getter-cli/tests/features/cli/update_check.feature new file mode 100644 index 0000000..e5d4828 --- /dev/null +++ b/crates/getter-cli/tests/features/cli/update_check.feature @@ -0,0 +1,68 @@ +@getter-cli @update +Feature: Offline update check + Scenario: User checks an offline fixture with an available update + Given an initialized getter data directory + And an offline update fixture for package "android/org.fdroid.fdroid" installed version "1.0.0" with candidate versions "1.0.1,1.2.0" + When I run getter update check for that fixture + Then the command succeeds + And the output is valid JSON + And the update check status is "update_available" + And the selected update version is "1.2.0" + And the update check actions download file "app.apk" and request installer "android_package" + + Scenario: User checks an offline fixture that is already up to date + Given an initialized getter data directory + And an offline update fixture for package "android/org.fdroid.fdroid" installed version "2.0.0" with candidate versions "1.9.0,2.0.0" + When I run getter update check for that fixture + Then the command succeeds + And the output is valid JSON + And the update check status is "up_to_date" + And the update check has no selected update + + Scenario: User checks an offline fixture where the latest update is ignored + Given an initialized getter data directory + And an offline update fixture for package "android/org.fdroid.fdroid" installed version "1.0.0" ignored version "1.2.0" with candidate versions "1.1.0,1.2.0" + When I run getter update check for that fixture + Then the command succeeds + And the output is valid JSON + And the update check status is "update_available" + And the selected update version is "1.1.0" + + Scenario: User checks an offline fixture where the only update is ignored + Given an initialized getter data directory + And an offline update fixture for package "android/org.fdroid.fdroid" installed version "1.0.0" ignored version "1.2.0" with candidate versions "1.2.0" + When I run getter update check for that fixture + Then the command succeeds + And the output is valid JSON + And the update check status is "ignored" + And the update check has no selected update + + Scenario: User checks an offline fixture without an installed version + Given an initialized getter data directory + And an offline update fixture for package "android/org.fdroid.fdroid" without installed version with candidate versions "1.0.0-beta,1.0.0" + When I run getter update check for that fixture + Then the command succeeds + And the output is valid JSON + And the update check status is "update_available" + And the selected update version is "1.0.0" + + Scenario: User checks an offline fixture with no candidates + Given an initialized getter data directory + And an offline update fixture for package "android/org.fdroid.fdroid" installed version "1.0.0" with candidate versions "" + When I run getter update check for that fixture + Then the command succeeds + And the output is valid JSON + And the update check status is "no_candidates" + And the update check has no selected update + + Scenario: User receives structured errors when the selected update has no artifact + Given an initialized getter data directory + And an offline update fixture for package "android/org.fdroid.fdroid" installed version "1.0.0" with artifactless candidate version "1.2.0" + When I run getter update check for that fixture + Then the command fails with an update check error + + Scenario: User receives structured errors for malformed offline update fixtures + Given an initialized getter data directory + And a malformed offline update fixture + When I run getter update check for that fixture + Then the command fails with an update check error diff --git a/crates/getter-core/src/update.rs b/crates/getter-core/src/update.rs index ab3256c..72acfa6 100644 --- a/crates/getter-core/src/update.rs +++ b/crates/getter-core/src/update.rs @@ -1,13 +1,198 @@ //! Update selection helpers owned by getter core. -use crate::{PackageId, SelectedUpdate, UpdateArtifact, UpdateCandidate}; +use crate::{ + PackageId, PackageKind, SelectedUpdate, UpdateAction, UpdateArtifact, UpdateCandidate, +}; +use serde::{Deserialize, Serialize}; use std::cmp::Ordering; +pub const OFFLINE_UPDATE_CHECK_FORMAT: &str = "getter-offline-update-check"; +pub const OFFLINE_UPDATE_CHECK_VERSION: u32 = 1; + /// User state that affects update selection. -#[derive(Debug, Clone, Default, PartialEq, Eq)] +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] pub struct UpdateSelectionPolicy { /// Candidate version the user chose to ignore/mark as skipped. + #[serde(default)] + pub ignored_version: Option, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct OfflineUpdateCheckFixture { + pub format: String, + pub version: u32, + pub package_id: PackageId, + #[serde(default)] + pub installed_version: Option, + #[serde(default)] pub ignored_version: Option, + #[serde(default)] + pub candidates: Vec, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct OfflineUpdateCheckResult { + pub network_required: bool, + pub package_id: PackageId, + #[serde(default)] + pub installed_version: Option, + pub policy: UpdateSelectionPolicy, + pub status: UpdateCheckStatus, + #[serde(default)] + pub selected: Option, + pub actions: Vec, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum UpdateCheckStatus { + UpdateAvailable, + UpToDate, + NoCandidates, + Ignored, +} + +#[derive(Debug, thiserror::Error)] +pub enum OfflineUpdateCheckError { + #[error("unsupported update check fixture format '{0}'")] + UnsupportedFormat(String), + #[error("unsupported update check fixture version {found}; expected {expected}")] + UnsupportedVersion { found: u32, expected: u32 }, + #[error("selected update candidate '{version}' has no actionable artifacts")] + MissingSelectedArtifact { version: String }, +} + +pub fn run_offline_update_check( + fixture: OfflineUpdateCheckFixture, +) -> Result { + if fixture.format != OFFLINE_UPDATE_CHECK_FORMAT { + return Err(OfflineUpdateCheckError::UnsupportedFormat(fixture.format)); + } + if fixture.version != OFFLINE_UPDATE_CHECK_VERSION { + return Err(OfflineUpdateCheckError::UnsupportedVersion { + found: fixture.version, + expected: OFFLINE_UPDATE_CHECK_VERSION, + }); + } + + let policy = UpdateSelectionPolicy { + ignored_version: fixture.ignored_version, + }; + check_updates_offline( + fixture.package_id, + fixture.installed_version, + fixture.candidates, + policy, + ) +} + +pub fn check_updates_offline( + package_id: PackageId, + installed_version: Option, + candidates: Vec, + policy: UpdateSelectionPolicy, +) -> Result { + let selected = select_update( + package_id.clone(), + installed_version.as_deref(), + &candidates, + &policy, + ); + let status = update_check_status( + selected.as_ref(), + installed_version.as_deref(), + &candidates, + &policy, + ); + let actions = match selected.as_ref() { + Some(selected) => { + if selected.artifact.is_none() { + return Err(OfflineUpdateCheckError::MissingSelectedArtifact { + version: selected.candidate.version.clone(), + }); + } + update_actions_for_selected(selected) + } + None => Vec::new(), + }; + + Ok(OfflineUpdateCheckResult { + network_required: false, + package_id, + installed_version, + policy, + status, + selected, + actions, + }) +} + +pub fn update_actions_for_selected(selected: &SelectedUpdate) -> Vec { + let Some(artifact) = selected.artifact.as_ref() else { + return Vec::new(); + }; + let file_name = artifact_file_name(artifact); + vec![ + UpdateAction::Download { + url: artifact.url.clone(), + file_name: file_name.clone(), + }, + UpdateAction::Install { + installer: installer_for_package_kind(selected.package_id.kind()).to_owned(), + file: file_name, + }, + ] +} + +fn artifact_file_name(artifact: &UpdateArtifact) -> String { + artifact + .file_name + .as_ref() + .filter(|value| !value.trim().is_empty()) + .cloned() + .unwrap_or_else(|| artifact.name.clone()) +} + +fn installer_for_package_kind(kind: PackageKind) -> &'static str { + match kind { + PackageKind::Android => "android_package", + PackageKind::Magisk => "magisk_module", + PackageKind::Generic => "generic_file", + } +} + +fn update_check_status( + selected: Option<&SelectedUpdate>, + installed_version: Option<&str>, + candidates: &[UpdateCandidate], + policy: &UpdateSelectionPolicy, +) -> UpdateCheckStatus { + if candidates.is_empty() { + UpdateCheckStatus::NoCandidates + } else if selected.is_some() { + UpdateCheckStatus::UpdateAvailable + } else if ignored_candidate_would_have_been_update(installed_version, candidates, policy) { + UpdateCheckStatus::Ignored + } else { + UpdateCheckStatus::UpToDate + } +} + +fn ignored_candidate_would_have_been_update( + installed_version: Option<&str>, + candidates: &[UpdateCandidate], + policy: &UpdateSelectionPolicy, +) -> bool { + let Some(ignored) = policy.ignored_version.as_deref() else { + return false; + }; + + candidates.iter().any(|candidate| { + compare_versions(&candidate.version, ignored) == Ordering::Equal + && installed_version.is_none_or(|installed| { + compare_versions(&candidate.version, installed) == Ordering::Greater + }) + }) } /// Compare human-facing version strings using a deterministic token ordering. @@ -257,6 +442,136 @@ mod tests { ); } + #[test] + fn offline_update_check_reports_update_available_with_actions() { + let result = run_offline_update_check(OfflineUpdateCheckFixture { + format: OFFLINE_UPDATE_CHECK_FORMAT.to_owned(), + version: OFFLINE_UPDATE_CHECK_VERSION, + package_id: "android/org.fdroid.fdroid".parse().unwrap(), + installed_version: Some("1.0.0".to_owned()), + ignored_version: None, + candidates: vec![candidate("1.0.1"), candidate("1.2.0")], + }) + .unwrap(); + + assert_eq!(result.status, UpdateCheckStatus::UpdateAvailable); + assert_eq!(result.selected.as_ref().unwrap().candidate.version, "1.2.0"); + assert_eq!( + result.actions, + vec![ + UpdateAction::Download { + url: "https://example.invalid/1.2.0.apk".to_owned(), + file_name: "app.apk".to_owned() + }, + UpdateAction::Install { + installer: "android_package".to_owned(), + file: "app.apk".to_owned() + } + ] + ); + } + + #[test] + fn offline_update_check_reports_up_to_date() { + let result = check_updates_offline( + "android/org.fdroid.fdroid".parse().unwrap(), + Some("2.0.0".to_owned()), + vec![candidate("1.9.0"), candidate("2.0.0")], + UpdateSelectionPolicy::default(), + ) + .unwrap(); + + assert_eq!(result.status, UpdateCheckStatus::UpToDate); + assert!(result.selected.is_none()); + assert!(result.actions.is_empty()); + } + + #[test] + fn offline_update_check_reports_ignored_when_only_update_is_ignored() { + let result = check_updates_offline( + "android/org.fdroid.fdroid".parse().unwrap(), + Some("1.0.0".to_owned()), + vec![candidate("1.2.0")], + UpdateSelectionPolicy { + ignored_version: Some("1.2.0".to_owned()), + }, + ) + .unwrap(); + + assert_eq!(result.status, UpdateCheckStatus::Ignored); + assert!(result.selected.is_none()); + } + + #[test] + fn offline_update_check_falls_back_below_ignored_latest() { + let result = check_updates_offline( + "android/org.fdroid.fdroid".parse().unwrap(), + Some("1.0.0".to_owned()), + vec![candidate("1.1.0"), candidate("1.2.0")], + UpdateSelectionPolicy { + ignored_version: Some("1.2.0".to_owned()), + }, + ) + .unwrap(); + + assert_eq!(result.status, UpdateCheckStatus::UpdateAvailable); + assert_eq!(result.selected.as_ref().unwrap().candidate.version, "1.1.0"); + } + + #[test] + fn offline_update_check_reports_no_candidates() { + let result = check_updates_offline( + "android/org.fdroid.fdroid".parse().unwrap(), + Some("1.0.0".to_owned()), + Vec::new(), + UpdateSelectionPolicy::default(), + ) + .unwrap(); + + assert_eq!(result.status, UpdateCheckStatus::NoCandidates); + assert!(result.selected.is_none()); + assert!(result.actions.is_empty()); + } + + #[test] + fn offline_update_check_rejects_selected_candidate_without_artifacts() { + let error = check_updates_offline( + "android/org.fdroid.fdroid".parse().unwrap(), + Some("1.0.0".to_owned()), + vec![UpdateCandidate { + version: "1.2.0".to_owned(), + channel: None, + source: None, + artifacts: Vec::new(), + }], + UpdateSelectionPolicy::default(), + ) + .unwrap_err(); + + assert!(matches!( + error, + OfflineUpdateCheckError::MissingSelectedArtifact { .. } + )); + } + + #[test] + fn offline_update_check_rejects_wrong_contract() { + let error = run_offline_update_check(OfflineUpdateCheckFixture { + format: "wrong".to_owned(), + version: OFFLINE_UPDATE_CHECK_VERSION, + package_id: "android/org.fdroid.fdroid".parse().unwrap(), + installed_version: None, + ignored_version: None, + candidates: Vec::new(), + }) + .unwrap_err(); + + assert!(matches!( + error, + OfflineUpdateCheckError::UnsupportedFormat(_) + )); + } + fn candidate(version: &str) -> UpdateCandidate { UpdateCandidate { version: version.to_owned(), From cb4cb655f3aefa69200ac23cfcc1e770d678d664 Mon Sep 17 00:00:00 2001 From: Xiangzhe Date: Tue, 23 Jun 2026 10:53:49 +0800 Subject: [PATCH 16/52] feat(cli): add offline task lifecycle --- Cargo.lock | 3 + crates/getter-cli/Cargo.toml | 1 + crates/getter-cli/src/lib.rs | 169 ++++- crates/getter-cli/tests/bdd_cli.rs | 259 +++++++ .../tests/features/cli/task_lifecycle.feature | 89 +++ crates/getter-core/src/lib.rs | 1 + crates/getter-core/src/task.rs | 266 +++++++ crates/getter-downloader/Cargo.toml | 2 + crates/getter-downloader/src/lib.rs | 166 ++++- crates/getter-storage/src/lib.rs | 674 +++++++++++++++++- 10 files changed, 1626 insertions(+), 4 deletions(-) create mode 100644 crates/getter-cli/tests/features/cli/task_lifecycle.feature create mode 100644 crates/getter-core/src/task.rs diff --git a/Cargo.lock b/Cargo.lock index 6119744..a23b19e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -520,6 +520,7 @@ version = "0.1.0" dependencies = [ "cucumber", "getter-core", + "getter-downloader", "getter-storage", "rusqlite", "serde", @@ -546,6 +547,8 @@ name = "getter-downloader" version = "0.1.0" dependencies = [ "getter-core", + "getter-storage", + "thiserror 1.0.69", ] [[package]] diff --git a/crates/getter-cli/Cargo.toml b/crates/getter-cli/Cargo.toml index ef7d105..248cb01 100644 --- a/crates/getter-cli/Cargo.toml +++ b/crates/getter-cli/Cargo.toml @@ -5,6 +5,7 @@ edition.workspace = true [dependencies] getter-core = { path = "../getter-core" } +getter-downloader = { path = "../getter-downloader" } getter-storage = { path = "../getter-storage" } serde = { version = "1", features = ["derive"] } serde_json = "1" diff --git a/crates/getter-cli/src/lib.rs b/crates/getter-cli/src/lib.rs index 0732d41..08d63d0 100644 --- a/crates/getter-cli/src/lib.rs +++ b/crates/getter-cli/src/lib.rs @@ -13,8 +13,15 @@ use getter_core::autogen::{ use getter_core::diagnostics::validate_repository_path; use getter_core::lua::evaluate_package_file; use getter_core::repository::{RepositoryLayout, RepositoryMetadata, REPO_API_VERSION_V1}; +use getter_core::task::{ + DownloadTaskRequest, InstallHandoffStatus, TaskEventPage, DOWNLOAD_REQUEST_FORMAT, + DOWNLOAD_REQUEST_VERSION, +}; use getter_core::update::{run_offline_update_check, OfflineUpdateCheckFixture}; use getter_core::{PackageId, RepositoryId, RepositoryPriority}; +use getter_downloader::{ + cancel_download_task, record_install_result, run_fake_download_task, submit_fake_download_task, +}; use getter_storage::legacy_room::{ map_legacy_app, read_legacy_room_database, LegacyAppKind, LegacyAppRecord, LegacyExtraAppRecord, LegacyPackageResolution, LegacyRoomDbImport, LegacyRoomImportWarning, @@ -67,6 +74,24 @@ pub enum CliCommand { UpdateCheck { fixture: PathBuf, }, + TaskSubmit { + request: PathBuf, + }, + TaskRun { + task_id: String, + }, + TaskList, + TaskCancel { + task_id: String, + }, + TaskEvents { + after: u64, + limit: usize, + }, + TaskInstallResult { + handoff_id: String, + status: InstallHandoffStatus, + }, AutogenInstalledPreview { inventory: PathBuf, }, @@ -103,6 +128,7 @@ pub enum ExitCode { Usage = 2, Storage = 10, Migration = 20, + Download = 40, } impl ExitCode { @@ -123,6 +149,8 @@ pub enum CliError { PackageEval(String), #[error("update check error: {0}")] Update(String), + #[error("download task error: {0}")] + Download(String), #[error("autogen error: {0}")] Autogen(String), #[error("Legacy Room export bundle is invalid")] @@ -143,6 +171,7 @@ impl CliError { Self::Repository(_) | Self::PackageEval(_) | Self::Update(_) | Self::Autogen(_) => { ExitCode::GenericFailure } + Self::Download(_) => ExitCode::Download, Self::InvalidLegacyBundle { .. } | Self::UnsupportedLegacyBundle { .. } | Self::InvalidLegacyDb { .. } @@ -157,6 +186,7 @@ impl CliError { Self::Repository(_) => "repository.error", Self::PackageEval(_) => "package.eval_error", Self::Update(_) => "update.check_error", + Self::Download(_) => "download.task_error", Self::Autogen(_) => "autogen.error", Self::InvalidLegacyBundle { .. } => "migration.invalid_bundle", Self::UnsupportedLegacyBundle { .. } => "migration.unsupported_bundle", @@ -172,6 +202,7 @@ impl CliError { Self::Repository(_) => "Getter repository operation failed", Self::PackageEval(_) => "Getter package evaluation failed", Self::Update(_) => "Getter update check failed", + Self::Download(_) => "Getter download task operation failed", Self::Autogen(_) => "Getter autogen operation failed", Self::InvalidLegacyBundle { .. } => "Legacy Room export bundle is invalid", Self::UnsupportedLegacyBundle { .. } => { @@ -189,6 +220,7 @@ impl CliError { | Self::Repository(detail) | Self::PackageEval(detail) | Self::Update(detail) + | Self::Download(detail) | Self::Autogen(detail) => Some(detail.as_str()), Self::InvalidLegacyBundle { .. } | Self::UnsupportedLegacyBundle { .. } @@ -208,6 +240,7 @@ impl CliError { | Self::Repository(_) | Self::PackageEval(_) | Self::Update(_) + | Self::Download(_) | Self::Autogen(_) => None, } } @@ -343,6 +376,41 @@ where fixture: PathBuf::from(fixture), } } + [domain, command, flag, request] + if domain == "task" && command == "submit" && flag == "--request" => + { + CliCommand::TaskSubmit { + request: PathBuf::from(request), + } + } + [domain, command, task_id] if domain == "task" && command == "run" => CliCommand::TaskRun { + task_id: task_id.clone(), + }, + [domain, command] if domain == "task" && command == "list" => CliCommand::TaskList, + [domain, command, task_id] if domain == "task" && command == "cancel" => { + CliCommand::TaskCancel { + task_id: task_id.clone(), + } + } + [domain, command, after_flag, after, limit_flag, limit] + if domain == "task" + && command == "events" + && after_flag == "--after" + && limit_flag == "--limit" => + { + CliCommand::TaskEvents { + after: parse_u64(after, "--after")?, + limit: parse_positive_usize(limit, "--limit")?, + } + } + [domain, command, handoff_id, status_flag, status] + if domain == "task" && command == "install-result" && status_flag == "--status" => + { + CliCommand::TaskInstallResult { + handoff_id: handoff_id.clone(), + status: parse_install_handoff_status(status)?, + } + } [domain, subject, action, flag, inventory] if domain == "autogen" && subject == "installed" @@ -498,6 +566,46 @@ fn execute(invocation: CliInvocation) -> Result { CliError::Update(format!("failed to serialize update check: {source}")) }) } + CliCommand::TaskSubmit { request } => { + let db = open_main_db(&invocation.data_dir)?; + let request = read_download_task_request(&request)?; + serde_json::to_value(submit_fake_download_task(&db, request).map_err(|source| { + CliError::Download(format!("offline task submit failed: {source}")) + })?) + .map_err(|source| CliError::Download(format!("failed to serialize task: {source}"))) + } + CliCommand::TaskRun { task_id } => { + let db = open_main_db(&invocation.data_dir)?; + serde_json::to_value(run_fake_download_task(&db, &task_id).map_err(|source| { + CliError::Download(format!("offline task run failed: {source}")) + })?) + .map_err(|source| CliError::Download(format!("failed to serialize task: {source}"))) + } + CliCommand::TaskList => { + let db = open_main_db(&invocation.data_dir)?; + Ok(json!({ "tasks": db.download_tasks()? })) + } + CliCommand::TaskCancel { task_id } => { + let db = open_main_db(&invocation.data_dir)?; + serde_json::to_value(cancel_download_task(&db, &task_id).map_err(|source| { + CliError::Download(format!("offline task cancel failed: {source}")) + })?) + .map_err(|source| CliError::Download(format!("failed to serialize task: {source}"))) + } + CliCommand::TaskEvents { after, limit } => { + let db = open_main_db(&invocation.data_dir)?; + let events: TaskEventPage = db.task_events_after(after, limit)?; + serde_json::to_value(events).map_err(|source| { + CliError::Download(format!("failed to serialize task events: {source}")) + }) + } + CliCommand::TaskInstallResult { handoff_id, status } => { + let db = open_main_db(&invocation.data_dir)?; + serde_json::to_value(record_install_result(&db, &handoff_id, status).map_err( + |source| CliError::Download(format!("offline install result failed: {source}")), + )?) + .map_err(|source| CliError::Download(format!("failed to serialize handoff: {source}"))) + } CliCommand::AutogenInstalledPreview { inventory } => { let db = open_main_db(&invocation.data_dir)?; let inventory = read_installed_inventory(&inventory)?; @@ -686,6 +794,38 @@ fn parse_package_id(value: &str) -> Result { .map_err(|source: getter_core::PackageIdError| CliError::Usage(source.to_string())) } +fn parse_u64(value: &str, flag: &str) -> Result { + value + .parse() + .map_err(|source| CliError::Usage(format!("invalid {flag} value '{value}': {source}"))) +} + +fn parse_usize(value: &str, flag: &str) -> Result { + value + .parse() + .map_err(|source| CliError::Usage(format!("invalid {flag} value '{value}': {source}"))) +} + +fn parse_positive_usize(value: &str, flag: &str) -> Result { + let parsed = parse_usize(value, flag)?; + if parsed == 0 { + return Err(CliError::Usage(format!("{flag} must be greater than zero"))); + } + Ok(parsed) +} + +fn parse_install_handoff_status(value: &str) -> Result { + match value { + "accepted" => Ok(InstallHandoffStatus::Accepted), + "succeeded" => Ok(InstallHandoffStatus::Succeeded), + "failed" => Ok(InstallHandoffStatus::Failed), + "canceled" => Ok(InstallHandoffStatus::Canceled), + _ => Err(CliError::Usage(format!( + "invalid install result status '{value}'; expected accepted, succeeded, failed, or canceled" + ))), + } +} + fn parse_priority(value: &str) -> Result { value .parse::() @@ -792,6 +932,27 @@ fn read_update_check_fixture(path: &Path) -> Result Result { + let bytes = fs::read(path) + .map_err(|source| CliError::Download(format!("failed to read task request: {source}")))?; + let request: DownloadTaskRequest = serde_json::from_slice(&bytes).map_err(|source| { + CliError::Download(format!("failed to parse task request JSON: {source}")) + })?; + if request.format != DOWNLOAD_REQUEST_FORMAT { + return Err(CliError::Download(format!( + "unsupported download request format '{}'", + request.format + ))); + } + if request.version != DOWNLOAD_REQUEST_VERSION { + return Err(CliError::Download(format!( + "unsupported download request version {}; expected {}", + request.version, DOWNLOAD_REQUEST_VERSION + ))); + } + Ok(request) +} + fn build_local_autogen_plan( db: &MainDb, inventory: &InstalledInventory, @@ -1722,7 +1883,7 @@ fn envelope_to_string(value: Value) -> String { } fn usage_text() -> String { - "Usage: getter --data-dir [--priority ]|repo eval |repo validate |package eval [--repo ]|storage validate|hub list|update check --fixture |autogen installed preview --inventory |autogen installed apply --preview (--accept-all|--accept ...)|autogen cleanup preview --inventory |autogen cleanup apply --preview (--accept-all|--accept ...)|legacy import-room-bundle |legacy import-room-db |legacy report-list>\n".to_owned() + "Usage: getter --data-dir [--priority ]|repo eval |repo validate |package eval [--repo ]|storage validate|hub list|update check --fixture |task submit --request |task run |task list|task cancel |task events --after --limit |task install-result --status |autogen installed preview --inventory |autogen installed apply --preview (--accept-all|--accept ...)|autogen cleanup preview --inventory |autogen cleanup apply --preview (--accept-all|--accept ...)|legacy import-room-bundle |legacy import-room-db |legacy report-list>\n".to_owned() } #[derive(Debug, Deserialize)] @@ -1776,6 +1937,12 @@ impl CliCommand { Self::PackageEval { .. } => "package eval", Self::StorageValidate => "storage validate", Self::UpdateCheck { .. } => "update check", + Self::TaskSubmit { .. } => "task submit", + Self::TaskRun { .. } => "task run", + Self::TaskList => "task list", + Self::TaskCancel { .. } => "task cancel", + Self::TaskEvents { .. } => "task events", + Self::TaskInstallResult { .. } => "task install-result", Self::AutogenInstalledPreview { .. } => "autogen installed preview", Self::AutogenInstalledApply { .. } => "autogen installed apply", Self::AutogenCleanupPreview { .. } => "autogen cleanup preview", diff --git a/crates/getter-cli/tests/bdd_cli.rs b/crates/getter-cli/tests/bdd_cli.rs index fdef3e5..d0230ea 100644 --- a/crates/getter-cli/tests/bdd_cli.rs +++ b/crates/getter-cli/tests/bdd_cli.rs @@ -15,6 +15,10 @@ struct CliWorld { inventory: Option, autogen_preview: Option, update_fixture: Option, + task_request: Option, + remembered_task_id: Option, + remembered_event_cursor: Option, + remembered_handoff_id: Option, fixture_repo_id: Option, fixture_repo_path: Option, fixture_package_id: Option, @@ -261,6 +265,44 @@ fn malformed_offline_update_fixture(world: &mut CliWorld) { world.update_fixture = Some(fixture); } +#[given(expr = "an offline download request for package {string}")] +fn offline_download_request_for_package(world: &mut CliWorld, package_id: String) { + let temp = world.temp.as_ref().expect("tempdir exists"); + let request = temp.path().join("download-request.json"); + fs::write( + &request, + serde_json::to_vec_pretty(&serde_json::json!({ + "format": "getter-download-request", + "version": 1, + "package_id": package_id, + "executor": "fake", + "actions": [ + { + "type": "download", + "url": "https://example.invalid/app.apk", + "file_name": "app.apk" + }, + { + "type": "install", + "installer": "android_package", + "file": "app.apk" + } + ] + })) + .expect("request serializes"), + ) + .expect("write download request"); + world.task_request = Some(request); +} + +#[given("a malformed offline download request")] +fn malformed_offline_download_request(world: &mut CliWorld) { + let temp = world.temp.as_ref().expect("tempdir exists"); + let request = temp.path().join("malformed-download-request.json"); + fs::write(&request, "not-json").expect("write malformed request"); + world.task_request = Some(request); +} + #[given(expr = "a fixture Lua repository {string} with package {string}")] fn fixture_lua_repository(world: &mut CliWorld, repo_id: String, package_id: String) { create_fixture_lua_repository(world, repo_id, package_id, "F-Droid".to_owned()); @@ -500,6 +542,99 @@ fn run_getter_update_check(world: &mut CliWorld) { world.json = None; } +#[when("I run getter task submit for that request")] +fn run_getter_task_submit(world: &mut CliWorld) { + let request = world.task_request.as_ref().expect("task request exists"); + let output = run_getter( + world, + [ + "task".to_owned(), + "submit".to_owned(), + "--request".to_owned(), + request.to_string_lossy().to_string(), + ], + ); + world.output = Some(output); + world.json = None; +} + +#[when("I run getter task list")] +fn run_getter_task_list(world: &mut CliWorld) { + let output = run_getter(world, ["task".to_owned(), "list".to_owned()]); + world.output = Some(output); + world.json = None; +} + +#[when("I run getter task cancel for the remembered task")] +fn run_getter_task_cancel(world: &mut CliWorld) { + let task_id = world + .remembered_task_id + .as_ref() + .expect("remembered task id exists") + .clone(); + let output = run_getter(world, ["task".to_owned(), "cancel".to_owned(), task_id]); + world.output = Some(output); + world.json = None; +} + +#[when("I run getter task run for the remembered task")] +fn run_getter_task_run(world: &mut CliWorld) { + let task_id = world + .remembered_task_id + .as_ref() + .expect("remembered task id exists") + .clone(); + let output = run_getter(world, ["task".to_owned(), "run".to_owned(), task_id]); + world.output = Some(output); + world.json = None; +} + +#[when(expr = "I run getter task events after {int} limit {int}")] +fn run_getter_task_events_after_limit(world: &mut CliWorld, after: u64, limit: u64) { + let output = run_getter( + world, + [ + "task".to_owned(), + "events".to_owned(), + "--after".to_owned(), + after.to_string(), + "--limit".to_owned(), + limit.to_string(), + ], + ); + world.output = Some(output); + world.json = None; +} + +#[when(expr = "I run getter task events after the remembered cursor limit {int}")] +fn run_getter_task_events_after_remembered_cursor(world: &mut CliWorld, limit: u64) { + let after = world + .remembered_event_cursor + .expect("remembered event cursor exists"); + run_getter_task_events_after_limit(world, after, limit); +} + +#[when(expr = "I run getter task install-result {string} for the remembered handoff")] +fn run_getter_task_install_result(world: &mut CliWorld, status: String) { + let handoff_id = world + .remembered_handoff_id + .as_ref() + .expect("remembered handoff id exists") + .clone(); + let output = run_getter( + world, + [ + "task".to_owned(), + "install-result".to_owned(), + handoff_id, + "--status".to_owned(), + status, + ], + ); + world.output = Some(output); + world.json = None; +} + #[when("I run getter autogen installed preview for that inventory")] fn run_getter_autogen_installed_preview(world: &mut CliWorld) { let inventory = world.inventory.as_ref().expect("inventory exists"); @@ -652,6 +787,26 @@ fn command_fails_with_update_check_error(world: &mut CliWorld) { world.json = Some(json); } +#[then("the command fails with a download task error")] +fn command_fails_with_download_task_error(world: &mut CliWorld) { + let output = world.output.as_ref().expect("command output exists"); + assert_eq!(output.status.code(), Some(40)); + let json = parse_stdout(output); + assert_eq!(json["ok"], false); + assert_eq!(json["error"]["code"], "download.task_error"); + world.json = Some(json); +} + +#[then("the command fails with a CLI usage error")] +fn command_fails_with_cli_usage_error(world: &mut CliWorld) { + let output = world.output.as_ref().expect("command output exists"); + assert_eq!(output.status.code(), Some(2)); + let json = parse_stdout(output); + assert_eq!(json["ok"], false); + assert_eq!(json["error"]["code"], "cli.usage"); + world.json = Some(json); +} + #[then(expr = "the command fails with direct DB migration error {string}")] fn command_fails_with_direct_db_migration_error(world: &mut CliWorld, code: String) { let output = world.output.as_ref().expect("command output exists"); @@ -714,6 +869,110 @@ fn output_contains_empty_repository_list(world: &mut CliWorld) { assert_eq!(json["data"]["repositories"], Value::Array(Vec::new())); } +#[then("I remember the submitted task id")] +fn remember_submitted_task_id(world: &mut CliWorld) { + let json = current_json(world); + assert_eq!(json["command"], "task submit"); + let task_id = json["data"]["task"]["id"] + .as_str() + .expect("task id should be a string") + .to_owned(); + world.remembered_task_id = Some(task_id); +} + +#[then(expr = "the task list contains the remembered task with status {string}")] +fn task_list_contains_remembered_task_with_status(world: &mut CliWorld, status: String) { + let task_id = world + .remembered_task_id + .as_ref() + .expect("remembered task id exists") + .clone(); + let json = current_json(world); + assert_eq!(json["command"], "task list"); + let tasks = json["data"]["tasks"].as_array().expect("tasks array"); + let task = tasks + .iter() + .find(|task| task["id"].as_str() == Some(task_id.as_str())) + .unwrap_or_else(|| panic!("task list should contain {task_id}: {tasks:?}")); + assert_eq!(task["status"], status); +} + +#[then(expr = "the task cancel result has status {string} and changed true")] +fn task_cancel_result_changed_true(world: &mut CliWorld, status: String) { + let json = current_json(world); + assert_eq!(json["command"], "task cancel"); + assert_eq!(json["data"]["status"], status); + assert_eq!(json["data"]["changed"], true); +} + +#[then(expr = "the task cancel result has status {string} and changed false")] +fn task_cancel_result_changed_false(world: &mut CliWorld, status: String) { + let json = current_json(world); + assert_eq!(json["command"], "task cancel"); + assert_eq!(json["data"]["status"], status); + assert_eq!(json["data"]["changed"], false); +} + +#[then(expr = "the task run result has status {string} and install handoff {string}")] +fn task_run_result_has_status_and_install_handoff( + world: &mut CliWorld, + status: String, + handoff_status: String, +) { + let json = current_json(world); + assert_eq!(json["command"], "task run"); + assert_eq!(json["data"]["task"]["status"], status); + assert_eq!(json["data"]["install_handoff"]["status"], handoff_status); +} + +#[then(expr = "the task events output contains {int} events and has more events")] +fn task_events_output_contains_events_and_has_more(world: &mut CliWorld, count: usize) { + let json = current_json(world); + assert_eq!(json["command"], "task events"); + assert_eq!(json["data"]["events"].as_array().unwrap().len(), count); + assert_eq!(json["data"]["has_more"], true); +} + +#[then("I remember the next event cursor")] +fn remember_next_event_cursor(world: &mut CliWorld) { + let json = current_json(world); + world.remembered_event_cursor = Some( + json["data"]["next_cursor"] + .as_u64() + .expect("next cursor should be a u64"), + ); +} + +#[then(expr = "the task events output contains event {string}")] +fn task_events_output_contains_event(world: &mut CliWorld, kind: String) { + let json = current_json(world); + let events = json["data"]["events"].as_array().expect("events array"); + assert!( + events + .iter() + .any(|event| event["kind"].as_str() == Some(kind.as_str())), + "events should contain {kind}: {events:?}" + ); +} + +#[then("I remember the install handoff id")] +fn remember_install_handoff_id(world: &mut CliWorld) { + let json = current_json(world); + world.remembered_handoff_id = Some( + json["data"]["install_handoff"]["id"] + .as_str() + .expect("handoff id should be a string") + .to_owned(), + ); +} + +#[then(expr = "the install result output has status {string}")] +fn install_result_output_has_status(world: &mut CliWorld, status: String) { + let json = current_json(world); + assert_eq!(json["command"], "task install-result"); + assert_eq!(json["data"]["handoff"]["status"], status); +} + #[then("the output reports valid storage")] fn output_reports_valid_storage(world: &mut CliWorld) { let json = current_json(world); diff --git a/crates/getter-cli/tests/features/cli/task_lifecycle.feature b/crates/getter-cli/tests/features/cli/task_lifecycle.feature new file mode 100644 index 0000000..6927548 --- /dev/null +++ b/crates/getter-cli/tests/features/cli/task_lifecycle.feature @@ -0,0 +1,89 @@ +@getter-cli @task +Feature: Offline task lifecycle + Scenario: User submits and lists an offline fake download task + Given an initialized getter data directory + And an offline download request for package "android/org.fdroid.fdroid" + When I run getter task submit for that request + Then the command succeeds + And the output is valid JSON + And I remember the submitted task id + When I run getter task list + Then the command succeeds + And the task list contains the remembered task with status "queued" + + Scenario: User cancels a queued offline task + Given an initialized getter data directory + And an offline download request for package "android/org.fdroid.fdroid" + When I run getter task submit for that request + Then the command succeeds + And I remember the submitted task id + When I run getter task cancel for the remembered task + Then the command succeeds + And the task cancel result has status "canceled" and changed true + When I run getter task cancel for the remembered task + Then the command succeeds + And the task cancel result has status "canceled" and changed false + + Scenario: User cannot cancel a succeeded offline task + Given an initialized getter data directory + And an offline download request for package "android/org.fdroid.fdroid" + When I run getter task submit for that request + Then the command succeeds + And I remember the submitted task id + When I run getter task run for the remembered task + Then the command succeeds + And the task run result has status "succeeded" and install handoff "requested" + When I run getter task cancel for the remembered task + Then the command fails with a download task error + + Scenario: User polls offline task events with cursor and limit + Given an initialized getter data directory + And an offline download request for package "android/org.fdroid.fdroid" + When I run getter task submit for that request + Then the command succeeds + And I remember the submitted task id + When I run getter task run for the remembered task + Then the command succeeds + When I run getter task events after 0 limit 2 + Then the command succeeds + And the task events output contains 2 events and has more events + And I remember the next event cursor + When I run getter task events after the remembered cursor limit 10 + Then the command succeeds + And the task events output contains event "install_handoff_requested" + + Scenario: User records an offline install handoff result + Given an initialized getter data directory + And an offline download request for package "android/org.fdroid.fdroid" + When I run getter task submit for that request + Then the command succeeds + And I remember the submitted task id + When I run getter task run for the remembered task + Then the command succeeds + And I remember the install handoff id + When I run getter task install-result "succeeded" for the remembered handoff + Then the command succeeds + And the install result output has status "succeeded" + + Scenario: User cannot record getter-created requested state as an install result + Given an initialized getter data directory + And an offline download request for package "android/org.fdroid.fdroid" + When I run getter task submit for that request + Then the command succeeds + And I remember the submitted task id + When I run getter task run for the remembered task + Then the command succeeds + And I remember the install handoff id + When I run getter task install-result "requested" for the remembered handoff + Then the command fails with a CLI usage error + + Scenario: User cannot poll task events with a zero limit + Given an initialized getter data directory + When I run getter task events after 0 limit 0 + Then the command fails with a CLI usage error + + Scenario: User receives structured errors for malformed task requests + Given an initialized getter data directory + And a malformed offline download request + When I run getter task submit for that request + Then the command fails with a download task error diff --git a/crates/getter-core/src/lib.rs b/crates/getter-core/src/lib.rs index 642e36f..aa3bb41 100644 --- a/crates/getter-core/src/lib.rs +++ b/crates/getter-core/src/lib.rs @@ -13,6 +13,7 @@ pub mod autogen; pub mod diagnostics; pub mod lua; pub mod repository; +pub mod task; pub mod update; /// Error returned when parsing or constructing a [`PackageId`]. diff --git a/crates/getter-core/src/task.rs b/crates/getter-core/src/task.rs new file mode 100644 index 0000000..d69f4f4 --- /dev/null +++ b/crates/getter-core/src/task.rs @@ -0,0 +1,266 @@ +//! Offline task lifecycle DTOs for getter-owned download/install workflows. +//! +//! These types are transport/domain shapes. They describe getter task state, +//! pollable events, and abstract platform install handoffs without performing +//! live network downloads or Android installation. + +use crate::{PackageId, UpdateAction}; +use serde::{Deserialize, Serialize}; +use std::str::FromStr; + +pub const DOWNLOAD_REQUEST_FORMAT: &str = "getter-download-request"; +pub const DOWNLOAD_REQUEST_VERSION: u32 = 1; + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct DownloadTaskRequest { + pub format: String, + pub version: u32, + pub package_id: PackageId, + #[serde(default)] + pub actions: Vec, + pub executor: TaskExecutor, +} + +impl DownloadTaskRequest { + pub fn download_action(&self) -> Option<&UpdateAction> { + self.actions + .iter() + .find(|action| matches!(action, UpdateAction::Download { .. })) + } + + pub fn install_action(&self) -> Option<&UpdateAction> { + self.actions + .iter() + .find(|action| matches!(action, UpdateAction::Install { .. })) + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum TaskExecutor { + Fake, +} + +impl TaskExecutor { + pub const fn as_str(self) -> &'static str { + match self { + Self::Fake => "fake", + } + } +} + +impl FromStr for TaskExecutor { + type Err = TaskModelError; + + fn from_str(value: &str) -> Result { + match value { + "fake" => Ok(Self::Fake), + other => Err(TaskModelError::InvalidExecutor(other.to_owned())), + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum DownloadTaskStatus { + Queued, + Running, + Succeeded, + Failed, + Canceled, +} + +impl DownloadTaskStatus { + pub const fn as_str(self) -> &'static str { + match self { + Self::Queued => "queued", + Self::Running => "running", + Self::Succeeded => "succeeded", + Self::Failed => "failed", + Self::Canceled => "canceled", + } + } + + pub const fn is_terminal(self) -> bool { + matches!(self, Self::Succeeded | Self::Failed | Self::Canceled) + } +} + +impl FromStr for DownloadTaskStatus { + type Err = TaskModelError; + + fn from_str(value: &str) -> Result { + match value { + "queued" => Ok(Self::Queued), + "running" => Ok(Self::Running), + "succeeded" => Ok(Self::Succeeded), + "failed" => Ok(Self::Failed), + "canceled" => Ok(Self::Canceled), + other => Err(TaskModelError::InvalidTaskStatus(other.to_owned())), + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum TaskEventKind { + TaskCreated, + TaskStarted, + TaskSucceeded, + TaskFailed, + TaskCanceled, + InstallHandoffRequested, + InstallResultRecorded, +} + +impl TaskEventKind { + pub const fn as_str(self) -> &'static str { + match self { + Self::TaskCreated => "task_created", + Self::TaskStarted => "task_started", + Self::TaskSucceeded => "task_succeeded", + Self::TaskFailed => "task_failed", + Self::TaskCanceled => "task_canceled", + Self::InstallHandoffRequested => "install_handoff_requested", + Self::InstallResultRecorded => "install_result_recorded", + } + } +} + +impl FromStr for TaskEventKind { + type Err = TaskModelError; + + fn from_str(value: &str) -> Result { + match value { + "task_created" => Ok(Self::TaskCreated), + "task_started" => Ok(Self::TaskStarted), + "task_succeeded" => Ok(Self::TaskSucceeded), + "task_failed" => Ok(Self::TaskFailed), + "task_canceled" => Ok(Self::TaskCanceled), + "install_handoff_requested" => Ok(Self::InstallHandoffRequested), + "install_result_recorded" => Ok(Self::InstallResultRecorded), + other => Err(TaskModelError::InvalidEventKind(other.to_owned())), + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct DownloadTaskSummary { + pub id: String, + pub package_id: PackageId, + pub status: DownloadTaskStatus, + pub executor: TaskExecutor, + pub actions: Vec, + pub download_file_name: String, + #[serde(default)] + pub downloaded_file: Option, + #[serde(default)] + pub failure_message: Option, + #[serde(default)] + pub install_handoff_id: Option, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct TaskEvent { + pub cursor: u64, + pub task_id: String, + pub kind: TaskEventKind, + #[serde(default)] + pub status: Option, + #[serde(default)] + pub message: Option, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct TaskEventPage { + pub events: Vec, + pub next_cursor: u64, + pub has_more: bool, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct TaskCancelResult { + pub task_id: String, + pub status: DownloadTaskStatus, + pub changed: bool, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct TaskRunResult { + pub task: DownloadTaskSummary, + #[serde(default)] + pub install_handoff: Option, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct TaskSubmitResult { + pub task: DownloadTaskSummary, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct InstallResultRecord { + pub handoff: InstallHandoffSummary, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum InstallHandoffStatus { + Requested, + Accepted, + Succeeded, + Failed, + Canceled, +} + +impl InstallHandoffStatus { + pub const fn as_str(self) -> &'static str { + match self { + Self::Requested => "requested", + Self::Accepted => "accepted", + Self::Succeeded => "succeeded", + Self::Failed => "failed", + Self::Canceled => "canceled", + } + } +} + +impl FromStr for InstallHandoffStatus { + type Err = TaskModelError; + + fn from_str(value: &str) -> Result { + match value { + "requested" => Ok(Self::Requested), + "accepted" => Ok(Self::Accepted), + "succeeded" => Ok(Self::Succeeded), + "failed" => Ok(Self::Failed), + "canceled" => Ok(Self::Canceled), + other => Err(TaskModelError::InvalidInstallHandoffStatus( + other.to_owned(), + )), + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct InstallHandoffSummary { + pub id: String, + pub task_id: String, + pub package_id: PackageId, + pub installer: String, + pub file: String, + pub status: InstallHandoffStatus, + #[serde(default)] + pub message: Option, +} + +#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] +pub enum TaskModelError { + #[error("invalid task executor '{0}'")] + InvalidExecutor(String), + #[error("invalid download task status '{0}'")] + InvalidTaskStatus(String), + #[error("invalid task event kind '{0}'")] + InvalidEventKind(String), + #[error("invalid install handoff status '{0}'")] + InvalidInstallHandoffStatus(String), +} diff --git a/crates/getter-downloader/Cargo.toml b/crates/getter-downloader/Cargo.toml index d8b2562..8cf48c4 100644 --- a/crates/getter-downloader/Cargo.toml +++ b/crates/getter-downloader/Cargo.toml @@ -5,3 +5,5 @@ edition.workspace = true [dependencies] getter-core = { path = "../getter-core" } +getter-storage = { path = "../getter-storage" } +thiserror = "1" diff --git a/crates/getter-downloader/src/lib.rs b/crates/getter-downloader/src/lib.rs index 4a11c7d..52318c5 100644 --- a/crates/getter-downloader/src/lib.rs +++ b/crates/getter-downloader/src/lib.rs @@ -1,3 +1,167 @@ -//! getter-downloader rewrite crate skeleton. +//! Deterministic offline downloader lifecycle for the getter rewrite. +//! +//! This crate intentionally does not perform live network I/O. It proves the +//! getter-owned task state, cancellation, pollable events, and install-handoff +//! contract with an explicit fake executor. + +use getter_core::task::{ + DownloadTaskRequest, InstallHandoffStatus, InstallResultRecord, TaskCancelResult, + TaskRunResult, TaskSubmitResult, DOWNLOAD_REQUEST_FORMAT, DOWNLOAD_REQUEST_VERSION, +}; +use getter_storage::{MainDb, StorageError}; pub use getter_core as core; + +#[derive(Debug, thiserror::Error)] +pub enum DownloaderError { + #[error("unsupported download request format '{0}'")] + UnsupportedFormat(String), + #[error("unsupported download request version {found}; expected {expected}")] + UnsupportedVersion { found: u32, expected: u32 }, + #[error("download request must use the fake executor for this offline slice")] + UnsupportedExecutor, + #[error("download request must include a download action")] + MissingDownloadAction, + #[error("storage error: {0}")] + Storage(#[from] StorageError), +} + +pub fn submit_fake_download_task( + db: &MainDb, + request: DownloadTaskRequest, +) -> Result { + validate_fake_request(&request)?; + let task = db.create_download_task(&request)?; + Ok(TaskSubmitResult { task }) +} + +pub fn run_fake_download_task( + db: &MainDb, + task_id: &str, +) -> Result { + db.start_download_task(task_id)?; + let task = db.succeed_download_task(task_id)?; + let install_handoff = db.create_install_handoff_for_task(task_id)?; + Ok(TaskRunResult { + task, + install_handoff, + }) +} + +pub fn cancel_download_task( + db: &MainDb, + task_id: &str, +) -> Result { + Ok(db.cancel_download_task(task_id)?) +} + +pub fn record_install_result( + db: &MainDb, + handoff_id: &str, + status: InstallHandoffStatus, +) -> Result { + let handoff = db.record_install_result(handoff_id, status, None)?; + Ok(InstallResultRecord { handoff }) +} + +fn validate_fake_request(request: &DownloadTaskRequest) -> Result<(), DownloaderError> { + if request.format != DOWNLOAD_REQUEST_FORMAT { + return Err(DownloaderError::UnsupportedFormat(request.format.clone())); + } + if request.version != DOWNLOAD_REQUEST_VERSION { + return Err(DownloaderError::UnsupportedVersion { + found: request.version, + expected: DOWNLOAD_REQUEST_VERSION, + }); + } + // This match is intentionally explicit so adding a real executor later forces + // this offline-only validation point to be revisited. + match request.executor { + getter_core::task::TaskExecutor::Fake => {} + } + if request.download_action().is_none() { + return Err(DownloaderError::MissingDownloadAction); + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use getter_core::task::{DownloadTaskStatus, TaskEventKind, TaskExecutor}; + use getter_core::{PackageId, UpdateAction}; + + #[test] + fn fake_submit_run_records_install_handoff_and_events() { + let db = MainDb::open_in_memory().unwrap(); + let submitted = submit_fake_download_task(&db, request()).unwrap(); + assert_eq!(submitted.task.id, "task-1"); + assert_eq!(submitted.task.status, DownloadTaskStatus::Queued); + + let result = run_fake_download_task(&db, &submitted.task.id).unwrap(); + assert_eq!(result.task.status, DownloadTaskStatus::Succeeded); + assert_eq!( + result.install_handoff.as_ref().unwrap().installer, + "android_package" + ); + + let events = db.task_events_after(0, 10).unwrap().events; + assert_eq!(events[0].kind, TaskEventKind::TaskCreated); + assert_eq!(events[1].kind, TaskEventKind::TaskStarted); + assert_eq!(events[2].kind, TaskEventKind::TaskSucceeded); + assert_eq!(events[3].kind, TaskEventKind::InstallHandoffRequested); + } + + #[test] + fn fake_cancel_before_run_is_persisted() { + let db = MainDb::open_in_memory().unwrap(); + let submitted = submit_fake_download_task(&db, request()).unwrap(); + + let canceled = cancel_download_task(&db, &submitted.task.id).unwrap(); + assert!(canceled.changed); + assert_eq!(canceled.status, DownloadTaskStatus::Canceled); + assert!(run_fake_download_task(&db, &submitted.task.id).is_err()); + } + + #[test] + fn fake_submit_rejects_wrong_contract() { + let db = MainDb::open_in_memory().unwrap(); + let mut request = request(); + request.format = "wrong".to_owned(); + + let error = submit_fake_download_task(&db, request).unwrap_err(); + assert!(matches!(error, DownloaderError::UnsupportedFormat(_))); + } + + #[test] + fn install_result_can_be_recorded_after_handoff() { + let db = MainDb::open_in_memory().unwrap(); + let submitted = submit_fake_download_task(&db, request()).unwrap(); + let result = run_fake_download_task(&db, &submitted.task.id).unwrap(); + let handoff = result.install_handoff.unwrap(); + + let recorded = + record_install_result(&db, &handoff.id, InstallHandoffStatus::Succeeded).unwrap(); + assert_eq!(recorded.handoff.status, InstallHandoffStatus::Succeeded); + } + + fn request() -> DownloadTaskRequest { + DownloadTaskRequest { + format: DOWNLOAD_REQUEST_FORMAT.to_owned(), + version: DOWNLOAD_REQUEST_VERSION, + package_id: PackageId::new(getter_core::PackageKind::Android, "org.fdroid.fdroid") + .unwrap(), + executor: TaskExecutor::Fake, + actions: vec![ + UpdateAction::Download { + url: "https://example.invalid/app.apk".to_owned(), + file_name: "app.apk".to_owned(), + }, + UpdateAction::Install { + installer: "android_package".to_owned(), + file: "app.apk".to_owned(), + }, + ], + } + } +} diff --git a/crates/getter-storage/src/lib.rs b/crates/getter-storage/src/lib.rs index f1285c2..23b0e8a 100644 --- a/crates/getter-storage/src/lib.rs +++ b/crates/getter-storage/src/lib.rs @@ -3,8 +3,13 @@ pub mod legacy_room; use getter_core::repository::RepositoryMetadata; -use getter_core::{PackageId, RepositoryId, RepositoryPriority}; -use rusqlite::{params, Connection, Params, Transaction}; +use getter_core::task::{ + DownloadTaskRequest, DownloadTaskStatus, DownloadTaskSummary, InstallHandoffStatus, + InstallHandoffSummary, TaskCancelResult, TaskEvent, TaskEventKind, TaskEventPage, + TaskModelError, +}; +use getter_core::{PackageId, RepositoryId, RepositoryPriority, UpdateAction}; +use rusqlite::{params, Connection, OptionalExtension, Params, Transaction}; use std::path::Path; use std::str::FromStr; @@ -18,6 +23,20 @@ pub enum StorageError { RepositoryId(#[from] getter_core::RepositoryIdError), #[error("invalid package resolution in database: {0}")] PackageResolution(String), + #[error("invalid task state in database: {0}")] + TaskState(String), + #[error("download task not found: {0}")] + TaskNotFound(String), + #[error("install handoff not found: {0}")] + InstallHandoffNotFound(String), + #[error("invalid install handoff transition for {handoff_id}: {reason}")] + InvalidInstallHandoffTransition { handoff_id: String, reason: String }, + #[error("invalid task transition for {task_id}: {reason}")] + InvalidTaskTransition { task_id: String, reason: String }, + #[error("invalid task request: {0}")] + InvalidTaskRequest(String), + #[error("JSON error: {0}")] + Json(#[from] serde_json::Error), } pub struct MainDb { @@ -78,6 +97,43 @@ CREATE TABLE IF NOT EXISTS migration_records ( completed_at_unix INTEGER NOT NULL DEFAULT (unixepoch()), report_json TEXT NOT NULL DEFAULT '{}' ); + +CREATE TABLE IF NOT EXISTS download_tasks ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + task_id TEXT UNIQUE, + package_id TEXT NOT NULL, + status TEXT NOT NULL, + executor TEXT NOT NULL, + actions_json TEXT NOT NULL, + download_file_name TEXT NOT NULL, + downloaded_file TEXT, + failure_message TEXT, + install_handoff_id TEXT, + created_at_unix INTEGER NOT NULL DEFAULT (unixepoch()), + updated_at_unix INTEGER NOT NULL DEFAULT (unixepoch()) +); + +CREATE TABLE IF NOT EXISTS task_events ( + cursor INTEGER PRIMARY KEY AUTOINCREMENT, + task_id TEXT NOT NULL, + kind TEXT NOT NULL, + status TEXT, + message TEXT, + created_at_unix INTEGER NOT NULL DEFAULT (unixepoch()) +); + +CREATE TABLE IF NOT EXISTS install_handoffs ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + handoff_id TEXT UNIQUE, + task_id TEXT NOT NULL, + package_id TEXT NOT NULL, + installer TEXT NOT NULL, + file TEXT NOT NULL, + status TEXT NOT NULL, + message TEXT, + created_at_unix INTEGER NOT NULL DEFAULT (unixepoch()), + updated_at_unix INTEGER NOT NULL DEFAULT (unixepoch()) +); "#, )?; self.ensure_column("tracked_packages", "ignored_version", "TEXT")?; @@ -90,6 +146,10 @@ CREATE TABLE IF NOT EXISTS migration_records ( "INSERT OR IGNORE INTO schema_migrations(id) VALUES ('main-v1')", [], )?; + self.conn.execute( + "INSERT OR IGNORE INTO schema_migrations(id) VALUES ('main-task-v1')", + [], + )?; Ok(()) } @@ -345,6 +405,375 @@ ON CONFLICT(id) DO UPDATE SET } Ok(records) } + + pub fn create_download_task( + &self, + request: &DownloadTaskRequest, + ) -> Result { + let download_file_name = match request.download_action() { + Some(UpdateAction::Download { file_name, .. }) => file_name.clone(), + _ => { + return Err(StorageError::InvalidTaskRequest( + "download task request must include a download action".to_owned(), + )) + } + }; + let actions_json = serde_json::to_string(&request.actions)?; + let tx = self.conn.unchecked_transaction()?; + tx.execute( + r#" +INSERT INTO download_tasks(package_id, status, executor, actions_json, download_file_name) +VALUES (?1, ?2, ?3, ?4, ?5) +"#, + params![ + request.package_id.to_string(), + DownloadTaskStatus::Queued.as_str(), + request.executor.as_str(), + actions_json, + download_file_name, + ], + )?; + let row_id = tx.last_insert_rowid(); + let task_id = format!("task-{row_id}"); + tx.execute( + "UPDATE download_tasks SET task_id = ?1 WHERE id = ?2", + params![task_id, row_id], + )?; + insert_task_event_with_executor( + &tx, + &task_id, + TaskEventKind::TaskCreated, + Some(DownloadTaskStatus::Queued), + Some("Task queued"), + )?; + tx.commit()?; + self.download_task(&task_id) + } + + pub fn download_task(&self, task_id: &str) -> Result { + let mut stmt = self.conn.prepare( + r#" +SELECT task_id, package_id, status, executor, actions_json, download_file_name, + downloaded_file, failure_message, install_handoff_id +FROM download_tasks +WHERE task_id = ?1 +"#, + )?; + let row = stmt + .query_row([task_id], |row| { + Ok(DownloadTaskRow { + task_id: row.get(0)?, + package_id: row.get(1)?, + status: row.get(2)?, + executor: row.get(3)?, + actions_json: row.get(4)?, + download_file_name: row.get(5)?, + downloaded_file: row.get(6)?, + failure_message: row.get(7)?, + install_handoff_id: row.get(8)?, + }) + }) + .optional()?; + row.map(download_task_from_row) + .transpose()? + .ok_or_else(|| StorageError::TaskNotFound(task_id.to_owned())) + } + + pub fn download_tasks(&self) -> Result, StorageError> { + let mut stmt = self.conn.prepare( + r#" +SELECT task_id, package_id, status, executor, actions_json, download_file_name, + downloaded_file, failure_message, install_handoff_id +FROM download_tasks +ORDER BY id ASC +"#, + )?; + let rows = stmt.query_map([], |row| { + Ok(DownloadTaskRow { + task_id: row.get(0)?, + package_id: row.get(1)?, + status: row.get(2)?, + executor: row.get(3)?, + actions_json: row.get(4)?, + download_file_name: row.get(5)?, + downloaded_file: row.get(6)?, + failure_message: row.get(7)?, + install_handoff_id: row.get(8)?, + }) + })?; + let mut tasks = Vec::new(); + for row in rows { + tasks.push(download_task_from_row(row?)?); + } + Ok(tasks) + } + + pub fn start_download_task(&self, task_id: &str) -> Result { + let task = self.download_task(task_id)?; + match task.status { + DownloadTaskStatus::Queued => { + self.update_download_task_status( + task_id, + DownloadTaskStatus::Running, + None, + None, + TaskEventKind::TaskStarted, + "Task started", + )?; + self.download_task(task_id) + } + DownloadTaskStatus::Running => Ok(task), + status => Err(StorageError::InvalidTaskTransition { + task_id: task_id.to_owned(), + reason: format!("cannot start task with status {}", status.as_str()), + }), + } + } + + pub fn succeed_download_task( + &self, + task_id: &str, + ) -> Result { + let task = self.download_task(task_id)?; + match task.status { + DownloadTaskStatus::Queued | DownloadTaskStatus::Running => { + self.update_download_task_status( + task_id, + DownloadTaskStatus::Succeeded, + Some(&task.download_file_name), + None, + TaskEventKind::TaskSucceeded, + "Task succeeded", + )?; + self.download_task(task_id) + } + status => Err(StorageError::InvalidTaskTransition { + task_id: task_id.to_owned(), + reason: format!("cannot succeed task with status {}", status.as_str()), + }), + } + } + + pub fn cancel_download_task(&self, task_id: &str) -> Result { + let task = self.download_task(task_id)?; + match task.status { + DownloadTaskStatus::Queued | DownloadTaskStatus::Running => { + self.update_download_task_status( + task_id, + DownloadTaskStatus::Canceled, + None, + None, + TaskEventKind::TaskCanceled, + "Task canceled", + )?; + Ok(TaskCancelResult { + task_id: task_id.to_owned(), + status: DownloadTaskStatus::Canceled, + changed: true, + }) + } + DownloadTaskStatus::Canceled => Ok(TaskCancelResult { + task_id: task_id.to_owned(), + status: DownloadTaskStatus::Canceled, + changed: false, + }), + status => Err(StorageError::InvalidTaskTransition { + task_id: task_id.to_owned(), + reason: format!("cannot cancel task with status {}", status.as_str()), + }), + } + } + + pub fn create_install_handoff_for_task( + &self, + task_id: &str, + ) -> Result, StorageError> { + let task = self.download_task(task_id)?; + if task.status != DownloadTaskStatus::Succeeded { + return Err(StorageError::InvalidTaskTransition { + task_id: task_id.to_owned(), + reason: format!( + "cannot request install handoff for task with status {}", + task.status.as_str() + ), + }); + } + if let Some(existing) = task.install_handoff_id.as_deref() { + return Ok(Some(self.install_handoff(existing)?)); + } + let Some(UpdateAction::Install { installer, file }) = task + .actions + .iter() + .find(|action| matches!(action, UpdateAction::Install { .. })) + else { + return Ok(None); + }; + let tx = self.conn.unchecked_transaction()?; + tx.execute( + r#" +INSERT INTO install_handoffs(task_id, package_id, installer, file, status) +VALUES (?1, ?2, ?3, ?4, ?5) +"#, + params![ + task_id, + task.package_id.to_string(), + installer, + file, + InstallHandoffStatus::Requested.as_str(), + ], + )?; + let row_id = tx.last_insert_rowid(); + let handoff_id = format!("handoff-{row_id}"); + tx.execute( + "UPDATE install_handoffs SET handoff_id = ?1 WHERE id = ?2", + params![handoff_id, row_id], + )?; + tx.execute( + "UPDATE download_tasks SET install_handoff_id = ?1, updated_at_unix = unixepoch() WHERE task_id = ?2", + params![handoff_id, task_id], + )?; + insert_task_event_with_executor( + &tx, + task_id, + TaskEventKind::InstallHandoffRequested, + Some(task.status), + Some("Install handoff requested"), + )?; + tx.commit()?; + Ok(Some(self.install_handoff(&handoff_id)?)) + } + + pub fn install_handoff(&self, handoff_id: &str) -> Result { + let mut stmt = self.conn.prepare( + r#" +SELECT handoff_id, task_id, package_id, installer, file, status, message +FROM install_handoffs +WHERE handoff_id = ?1 +"#, + )?; + let row = stmt + .query_row([handoff_id], |row| { + Ok(InstallHandoffRow { + handoff_id: row.get(0)?, + task_id: row.get(1)?, + package_id: row.get(2)?, + installer: row.get(3)?, + file: row.get(4)?, + status: row.get(5)?, + message: row.get(6)?, + }) + }) + .optional()?; + row.map(install_handoff_from_row) + .transpose()? + .ok_or_else(|| StorageError::InstallHandoffNotFound(handoff_id.to_owned())) + } + + pub fn record_install_result( + &self, + handoff_id: &str, + status: InstallHandoffStatus, + message: Option<&str>, + ) -> Result { + if status == InstallHandoffStatus::Requested { + return Err(StorageError::InvalidInstallHandoffTransition { + handoff_id: handoff_id.to_owned(), + reason: "requested is getter-created state, not a platform install result" + .to_owned(), + }); + } + let handoff = self.install_handoff(handoff_id)?; + let tx = self.conn.unchecked_transaction()?; + tx.execute( + r#" +UPDATE install_handoffs +SET status = ?1, message = ?2, updated_at_unix = unixepoch() +WHERE handoff_id = ?3 +"#, + params![status.as_str(), message, handoff_id], + )?; + insert_task_event_with_executor( + &tx, + &handoff.task_id, + TaskEventKind::InstallResultRecorded, + None, + Some("Install result recorded"), + )?; + tx.commit()?; + self.install_handoff(handoff_id) + } + + pub fn task_events_after( + &self, + after: u64, + limit: usize, + ) -> Result { + let fetch_limit = limit.saturating_add(1) as i64; + let mut stmt = self.conn.prepare( + r#" +SELECT cursor, task_id, kind, status, message +FROM task_events +WHERE cursor > ?1 +ORDER BY cursor ASC +LIMIT ?2 +"#, + )?; + let rows = stmt.query_map(params![after as i64, fetch_limit], |row| { + Ok(TaskEventRow { + cursor: row.get(0)?, + task_id: row.get(1)?, + kind: row.get(2)?, + status: row.get(3)?, + message: row.get(4)?, + }) + })?; + let mut events = Vec::new(); + for row in rows { + events.push(task_event_from_row(row?)?); + } + let has_more = events.len() > limit; + if has_more { + events.truncate(limit); + } + let next_cursor = events.last().map(|event| event.cursor).unwrap_or(after); + Ok(TaskEventPage { + events, + next_cursor, + has_more, + }) + } + + fn update_download_task_status( + &self, + task_id: &str, + status: DownloadTaskStatus, + downloaded_file: Option<&str>, + failure_message: Option<&str>, + event_kind: TaskEventKind, + event_message: &str, + ) -> Result<(), StorageError> { + let tx = self.conn.unchecked_transaction()?; + tx.execute( + r#" +UPDATE download_tasks +SET status = ?1, + downloaded_file = COALESCE(?2, downloaded_file), + failure_message = ?3, + updated_at_unix = unixepoch() +WHERE task_id = ?4 +"#, + params![status.as_str(), downloaded_file, failure_message, task_id], + )?; + insert_task_event_with_executor( + &tx, + task_id, + event_kind, + Some(status), + Some(event_message), + )?; + tx.commit()?; + Ok(()) + } } impl CacheDb { @@ -435,6 +864,36 @@ pub struct StoredMigrationRecord { pub report_json: String, } +struct DownloadTaskRow { + task_id: String, + package_id: String, + status: String, + executor: String, + actions_json: String, + download_file_name: String, + downloaded_file: Option, + failure_message: Option, + install_handoff_id: Option, +} + +struct TaskEventRow { + cursor: i64, + task_id: String, + kind: String, + status: Option, + message: Option, +} + +struct InstallHandoffRow { + handoff_id: String, + task_id: String, + package_id: String, + installer: String, + file: String, + status: String, + message: Option, +} + #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct MigrationRecordUpsert<'a> { pub id: &'a str, @@ -529,10 +988,84 @@ fn bool_to_i64(value: bool) -> i64 { } } +fn download_task_from_row(row: DownloadTaskRow) -> Result { + Ok(DownloadTaskSummary { + id: row.task_id, + package_id: row.package_id.parse()?, + status: row + .status + .parse() + .map_err(|error: TaskModelError| StorageError::TaskState(error.to_string()))?, + executor: row + .executor + .parse() + .map_err(|error: TaskModelError| StorageError::TaskState(error.to_string()))?, + actions: serde_json::from_str(&row.actions_json)?, + download_file_name: row.download_file_name, + downloaded_file: row.downloaded_file, + failure_message: row.failure_message, + install_handoff_id: row.install_handoff_id, + }) +} + +fn task_event_from_row(row: TaskEventRow) -> Result { + Ok(TaskEvent { + cursor: row.cursor as u64, + task_id: row.task_id, + kind: row + .kind + .parse() + .map_err(|error: TaskModelError| StorageError::TaskState(error.to_string()))?, + status: row + .status + .map(|status| status.parse()) + .transpose() + .map_err(|error: TaskModelError| StorageError::TaskState(error.to_string()))?, + message: row.message, + }) +} + +fn install_handoff_from_row(row: InstallHandoffRow) -> Result { + Ok(InstallHandoffSummary { + id: row.handoff_id, + task_id: row.task_id, + package_id: row.package_id.parse()?, + installer: row.installer, + file: row.file, + status: row + .status + .parse() + .map_err(|error: TaskModelError| StorageError::TaskState(error.to_string()))?, + message: row.message, + }) +} + +fn insert_task_event_with_executor( + conn: &impl SqlExecutor, + task_id: &str, + kind: TaskEventKind, + status: Option, + message: Option<&str>, +) -> Result { + conn.execute_statement( + r#" +INSERT INTO task_events(task_id, kind, status, message) +VALUES (?1, ?2, ?3, ?4) +"#, + params![ + task_id, + kind.as_str(), + status.map(DownloadTaskStatus::as_str), + message, + ], + ) +} + #[cfg(test)] mod tests { use super::*; use getter_core::repository::REPO_API_VERSION_V1; + use getter_core::task::TaskExecutor; #[test] fn main_db_stores_repository_registry_ordered_by_priority() { @@ -720,6 +1253,143 @@ mod tests { assert!(db.migration_record_exists("legacy-room-v17").unwrap()); } + #[test] + fn task_lifecycle_persists_across_reopen() { + let temp = tempfile::tempdir().unwrap(); + let db_path = temp.path().join("main.db"); + { + let db = MainDb::open(&db_path).unwrap(); + let task = db.create_download_task(&download_request()).unwrap(); + assert_eq!(task.id, "task-1"); + assert_eq!(task.status, DownloadTaskStatus::Queued); + } + + let db = MainDb::open(&db_path).unwrap(); + let tasks = db.download_tasks().unwrap(); + assert_eq!(tasks.len(), 1); + assert_eq!(tasks[0].id, "task-1"); + assert_eq!(tasks[0].package_id.to_string(), "android/org.fdroid.fdroid"); + assert_eq!(tasks[0].status, DownloadTaskStatus::Queued); + } + + #[test] + fn task_cancel_is_persistent_and_idempotent_before_terminal_state() { + let db = MainDb::open_in_memory().unwrap(); + let task = db.create_download_task(&download_request()).unwrap(); + + let first = db.cancel_download_task(&task.id).unwrap(); + assert!(first.changed); + assert_eq!(first.status, DownloadTaskStatus::Canceled); + let second = db.cancel_download_task(&task.id).unwrap(); + assert!(!second.changed); + assert_eq!( + db.download_task(&task.id).unwrap().status, + DownloadTaskStatus::Canceled + ); + } + + #[test] + fn task_cancel_after_success_is_rejected() { + let db = MainDb::open_in_memory().unwrap(); + let task = db.create_download_task(&download_request()).unwrap(); + db.start_download_task(&task.id).unwrap(); + db.succeed_download_task(&task.id).unwrap(); + + let error = db.cancel_download_task(&task.id).unwrap_err(); + assert!(matches!(error, StorageError::InvalidTaskTransition { .. })); + } + + #[test] + fn task_events_are_pollable_with_cursor_and_limit() { + let db = MainDb::open_in_memory().unwrap(); + let task = db.create_download_task(&download_request()).unwrap(); + db.start_download_task(&task.id).unwrap(); + db.succeed_download_task(&task.id).unwrap(); + + let first_page = db.task_events_after(0, 2).unwrap(); + assert_eq!(first_page.events.len(), 2); + assert!(first_page.has_more); + assert_eq!(first_page.events[0].kind, TaskEventKind::TaskCreated); + assert_eq!(first_page.events[1].kind, TaskEventKind::TaskStarted); + + let second_page = db.task_events_after(first_page.next_cursor, 2).unwrap(); + assert_eq!(second_page.events.len(), 1); + assert!(!second_page.has_more); + assert_eq!(second_page.events[0].kind, TaskEventKind::TaskSucceeded); + } + + #[test] + fn install_handoff_result_is_recorded() { + let db = MainDb::open_in_memory().unwrap(); + let task = db.create_download_task(&download_request()).unwrap(); + db.start_download_task(&task.id).unwrap(); + db.succeed_download_task(&task.id).unwrap(); + let handoff = db + .create_install_handoff_for_task(&task.id) + .unwrap() + .unwrap(); + assert_eq!(handoff.id, "handoff-1"); + assert_eq!(handoff.status, InstallHandoffStatus::Requested); + + let updated = db + .record_install_result(&handoff.id, InstallHandoffStatus::Succeeded, Some("ok")) + .unwrap(); + assert_eq!(updated.status, InstallHandoffStatus::Succeeded); + assert_eq!(updated.message.as_deref(), Some("ok")); + } + + #[test] + fn install_result_rejects_getter_created_requested_state() { + let db = MainDb::open_in_memory().unwrap(); + let task = db.create_download_task(&download_request()).unwrap(); + db.start_download_task(&task.id).unwrap(); + db.succeed_download_task(&task.id).unwrap(); + let handoff = db + .create_install_handoff_for_task(&task.id) + .unwrap() + .unwrap(); + + let error = db + .record_install_result(&handoff.id, InstallHandoffStatus::Requested, None) + .unwrap_err(); + assert!(matches!( + error, + StorageError::InvalidInstallHandoffTransition { .. } + )); + } + + #[test] + fn task_request_requires_download_action() { + let db = MainDb::open_in_memory().unwrap(); + let mut request = download_request(); + request.actions = vec![UpdateAction::Install { + installer: "android_package".to_owned(), + file: "app.apk".to_owned(), + }]; + + let error = db.create_download_task(&request).unwrap_err(); + assert!(matches!(error, StorageError::InvalidTaskRequest(_))); + } + + fn download_request() -> DownloadTaskRequest { + DownloadTaskRequest { + format: getter_core::task::DOWNLOAD_REQUEST_FORMAT.to_owned(), + version: getter_core::task::DOWNLOAD_REQUEST_VERSION, + package_id: "android/org.fdroid.fdroid".parse().unwrap(), + executor: TaskExecutor::Fake, + actions: vec![ + UpdateAction::Download { + url: "https://example.invalid/app.apk".to_owned(), + file_name: "app.apk".to_owned(), + }, + UpdateAction::Install { + installer: "android_package".to_owned(), + file: "app.apk".to_owned(), + }, + ], + } + } + #[test] fn cache_db_migrates_schema() { let _db = CacheDb::open_in_memory().unwrap(); From cee3b007d5e03d9035f5319a582f452dfbc7c911 Mon Sep 17 00:00:00 2001 From: Xiangzhe Date: Tue, 23 Jun 2026 23:57:34 +0800 Subject: [PATCH 17/52] feat: share installed autogen operations --- Cargo.lock | 12 + Cargo.toml | 7 +- crates/getter-cli/Cargo.toml | 1 + crates/getter-cli/src/lib.rs | 610 ++--------------------- crates/getter-core/Cargo.toml | 6 +- crates/getter-core/src/lib.rs | 2 + crates/getter-operations/Cargo.toml | 10 + crates/getter-operations/src/autogen.rs | 637 ++++++++++++++++++++++++ crates/getter-operations/src/lib.rs | 8 + crates/getter-storage/Cargo.toml | 2 +- src/lib.rs | 2 + 11 files changed, 714 insertions(+), 583 deletions(-) create mode 100644 crates/getter-operations/Cargo.toml create mode 100644 crates/getter-operations/src/autogen.rs create mode 100644 crates/getter-operations/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index a23b19e..1ff310d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -508,6 +508,7 @@ version = "0.1.0" dependencies = [ "getter-cli", "getter-core", + "getter-operations", "getter-storage", "rustls-platform-verifier", "thiserror 1.0.69", @@ -521,6 +522,7 @@ dependencies = [ "cucumber", "getter-core", "getter-downloader", + "getter-operations", "getter-storage", "rusqlite", "serde", @@ -558,6 +560,16 @@ dependencies = [ "getter-core", ] +[[package]] +name = "getter-operations" +version = "0.1.0" +dependencies = [ + "getter-core", + "getter-storage", + "serde_json", + "thiserror 1.0.69", +] + [[package]] name = "getter-plugin-api" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index e45af61..9347d01 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,6 +10,7 @@ members = [ "crates/getter-storage", "crates/getter-providers", "crates/getter-downloader", + "crates/getter-operations", "crates/getter-plugin-api", "crates/getter-rpc", "crates/getter-cli", @@ -22,7 +23,8 @@ edition = "2021" [features] default = ["cli", "domain"] -domain = ["dep:getter-core", "dep:getter-storage"] +domain = ["dep:getter-core", "dep:getter-storage", "dep:getter-operations"] +lua = ["getter-core/lua"] cli = ["dep:getter-cli"] rustls-platform-verifier = ["dep:rustls-platform-verifier"] rustls-platform-verifier-android = ["rustls-platform-verifier", "rustls-platform-verifier/jni"] @@ -30,8 +32,9 @@ webpki-roots = [] native-tokio = [] [dependencies] -getter-core = { path = "crates/getter-core", optional = true } +getter-core = { path = "crates/getter-core", default-features = false, optional = true } getter-storage = { path = "crates/getter-storage", optional = true } +getter-operations = { path = "crates/getter-operations", optional = true } getter-cli = { path = "crates/getter-cli", optional = true } thiserror = "1" tokio = { version = "1", features = ["net"] } diff --git a/crates/getter-cli/Cargo.toml b/crates/getter-cli/Cargo.toml index 248cb01..a5cec89 100644 --- a/crates/getter-cli/Cargo.toml +++ b/crates/getter-cli/Cargo.toml @@ -6,6 +6,7 @@ edition.workspace = true [dependencies] getter-core = { path = "../getter-core" } getter-downloader = { path = "../getter-downloader" } +getter-operations = { path = "../getter-operations" } getter-storage = { path = "../getter-storage" } serde = { version = "1", features = ["derive"] } serde_json = "1" diff --git a/crates/getter-cli/src/lib.rs b/crates/getter-cli/src/lib.rs index 08d63d0..d9f4d47 100644 --- a/crates/getter-cli/src/lib.rs +++ b/crates/getter-cli/src/lib.rs @@ -4,15 +4,10 @@ //! while durable state is initialized through `getter-storage` so the command //! surface exercises the same Rust-owned SQLite direction used by embedders. -use getter_core::autogen::{ - content_hash, local_autogen_repo_toml, local_repo_toml, plan_local_autogen, AutogenManifest, - AutogenManifestEntry, AutogenPlan, AutogenSkipReason, InstalledInventory, - LOCAL_AUTOGEN_REPOSITORY_ID, LOCAL_AUTOGEN_REPOSITORY_NAME, LOCAL_REPOSITORY_ID, - LOCAL_REPOSITORY_NAME, -}; +use getter_core::autogen::InstalledInventory; use getter_core::diagnostics::validate_repository_path; use getter_core::lua::evaluate_package_file; -use getter_core::repository::{RepositoryLayout, RepositoryMetadata, REPO_API_VERSION_V1}; +use getter_core::repository::{RepositoryLayout, RepositoryMetadata}; use getter_core::task::{ DownloadTaskRequest, InstallHandoffStatus, TaskEventPage, DOWNLOAD_REQUEST_FORMAT, DOWNLOAD_REQUEST_VERSION, @@ -22,6 +17,7 @@ use getter_core::{PackageId, RepositoryId, RepositoryPriority}; use getter_downloader::{ cancel_download_task, record_install_result, run_fake_download_task, submit_fake_download_task, }; +use getter_operations::autogen::{self, AutogenAcceptance, AutogenOperationError}; use getter_storage::legacy_room::{ map_legacy_app, read_legacy_room_database, LegacyAppKind, LegacyAppRecord, LegacyExtraAppRecord, LegacyPackageResolution, LegacyRoomDbImport, LegacyRoomImportWarning, @@ -33,15 +29,13 @@ use getter_storage::{ }; use serde::{Deserialize, Serialize}; use serde_json::{json, Value}; -use std::collections::{BTreeSet, HashMap}; use std::fs; -use std::path::{Component, Path, PathBuf}; +use std::path::{Path, PathBuf}; const MAIN_DB_FILE: &str = "main.db"; const CACHE_DB_FILE: &str = "cache.db"; const MIGRATION_REPORTS_DIR: &str = "migration-reports"; const LEGACY_ROOM_MIGRATION_ID: &str = "legacy-room-v17"; -const AUTOGEN_MANIFEST_FILE: &str = "autogen-manifest.json"; #[derive(Debug, Clone, PartialEq, Eq)] pub struct CliInvocation { @@ -115,12 +109,6 @@ pub enum CliCommand { LegacyReportList, } -#[derive(Debug, Clone, PartialEq, Eq)] -pub enum AutogenAcceptance { - AcceptAll, - Accept(Vec), -} - #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum ExitCode { Success = 0, @@ -252,6 +240,16 @@ impl From for CliError { } } +impl From for CliError { + fn from(value: AutogenOperationError) -> Self { + match value { + AutogenOperationError::Storage(source) => Self::Storage(source.to_string()), + AutogenOperationError::Repository(detail) => Self::Repository(detail), + AutogenOperationError::Autogen(detail) => Self::Autogen(detail), + } + } +} + #[derive(Debug, Clone, PartialEq, Eq)] pub struct CliOutput { pub exit_code: ExitCode, @@ -609,8 +607,8 @@ fn execute(invocation: CliInvocation) -> Result { CliCommand::AutogenInstalledPreview { inventory } => { let db = open_main_db(&invocation.data_dir)?; let inventory = read_installed_inventory(&inventory)?; - let plan = build_local_autogen_plan(&db, &inventory)?; - Ok(autogen_installed_preview_json(&invocation.data_dir, &plan)) + let plan = autogen::build_local_autogen_plan(&db, &inventory)?; + Ok(autogen::installed_preview_json(&invocation.data_dir, &plan)) } CliCommand::AutogenInstalledApply { preview, @@ -618,12 +616,14 @@ fn execute(invocation: CliInvocation) -> Result { } => { let db = open_main_db(&invocation.data_dir)?; let preview = read_autogen_preview(&preview, "installed.preview")?; - apply_autogen_installed_preview(&invocation.data_dir, &db, &preview, &acceptance) + autogen::apply_installed_preview(&invocation.data_dir, &db, &preview, &acceptance) + .map_err(CliError::from) } CliCommand::AutogenCleanupPreview { inventory } => { let db = open_main_db(&invocation.data_dir)?; let inventory = read_installed_inventory(&inventory)?; - cleanup_preview_json(&invocation.data_dir, &db, &inventory) + autogen::cleanup_preview_json(&invocation.data_dir, &db, &inventory) + .map_err(CliError::from) } CliCommand::AutogenCleanupApply { preview, @@ -631,7 +631,8 @@ fn execute(invocation: CliInvocation) -> Result { } => { let db = open_main_db(&invocation.data_dir)?; let preview = read_autogen_preview(&preview, "cleanup.preview")?; - apply_autogen_cleanup_preview(&invocation.data_dir, &db, &preview, &acceptance) + autogen::apply_cleanup_preview(&invocation.data_dir, &db, &preview, &acceptance) + .map_err(CliError::from) } CliCommand::LegacyImportRoomBundle { bundle } => { let db = open_main_db(&invocation.data_dir)?; @@ -921,6 +922,15 @@ fn read_installed_inventory(path: &Path) -> Result .map_err(|source| CliError::Autogen(format!("failed to parse inventory JSON: {source}"))) } +fn read_autogen_preview(path: &Path, expected_operation: &str) -> Result { + let bytes = fs::read(path) + .map_err(|source| CliError::Autogen(format!("failed to read autogen preview: {source}")))?; + let raw: Value = serde_json::from_slice(&bytes).map_err(|source| { + CliError::Autogen(format!("failed to parse autogen preview JSON: {source}")) + })?; + autogen::unwrap_preview_payload(raw, expected_operation).map_err(CliError::from) +} + fn read_update_check_fixture(path: &Path) -> Result { let bytes = fs::read(path).map_err(|source| { CliError::Update(format!("failed to read update check fixture: {source}")) @@ -953,564 +963,6 @@ fn read_download_task_request(path: &Path) -> Result Result { - let covered = higher_priority_package_coverage(db)?; - plan_local_autogen(inventory, &covered).map_err(|source| CliError::Autogen(source.to_string())) -} - -fn higher_priority_package_coverage( - db: &MainDb, -) -> Result, CliError> { - let mut covered = HashMap::new(); - for repo in db.repositories()? { - if repo.id.as_str() == LOCAL_AUTOGEN_REPOSITORY_ID { - continue; - } - if repo.priority <= RepositoryPriority::LOCAL_AUTOGEN { - continue; - } - let Some(path) = repo.path.as_ref() else { - continue; - }; - let layout = load_repository_layout(Path::new(path))?; - for package in layout.packages { - covered.entry(package.id).or_insert_with(|| repo.id.clone()); - } - } - Ok(covered) -} - -fn autogen_installed_preview_json(data_dir: &Path, plan: &AutogenPlan) -> Value { - let candidates: Vec = plan.candidates.iter().map(autogen_candidate_json).collect(); - let skipped: Vec = plan.skipped.iter().map(autogen_skip_json).collect(); - json!({ - "operation": "installed.preview", - "target_repo_id": plan.repository_id.as_str(), - "target_repo_path": local_autogen_repo_path(data_dir), - "summary": { - "candidate_count": candidates.len(), - "skipped_count": skipped.len(), - "write_count": candidates.len(), - "delete_count": 0, - }, - "candidates": candidates, - "skipped": skipped, - "diagnostics": [], - }) -} - -fn autogen_candidate_json(candidate: &getter_core::autogen::AutogenCandidate) -> Value { - json!({ - "package_id": candidate.package_id.to_string(), - "kind": candidate.package_id.kind().as_str(), - "display_name": candidate.name, - "installed_target": candidate.installed, - "action": "create", - "output_relative_path": candidate.relative_path, - "content_hash": candidate.content_hash, - "content": candidate.content, - }) -} - -fn autogen_skip_json(skip: &getter_core::autogen::AutogenSkip) -> Value { - json!({ - "package_id": skip.package_id.to_string(), - "reason": match skip.reason { - AutogenSkipReason::DuplicateInventoryItem => "duplicate_inventory_item", - AutogenSkipReason::CoveredByHigherPriorityRepository => "covered_by_higher_priority_repo", - }, - "covering_repo_id": skip.repository_id.as_ref().map(RepositoryId::as_str), - }) -} - -fn read_autogen_preview(path: &Path, expected_operation: &str) -> Result { - let bytes = fs::read(path) - .map_err(|source| CliError::Autogen(format!("failed to read autogen preview: {source}")))?; - let raw: Value = serde_json::from_slice(&bytes).map_err(|source| { - CliError::Autogen(format!("failed to parse autogen preview JSON: {source}")) - })?; - let payload = if raw.get("ok").is_some() && raw.get("data").is_some() { - raw.get("data").cloned().unwrap_or(Value::Null) - } else { - raw - }; - if payload.get("operation").and_then(Value::as_str) != Some(expected_operation) { - return Err(CliError::Autogen(format!( - "autogen preview operation must be '{expected_operation}'" - ))); - } - Ok(payload) -} - -fn apply_autogen_installed_preview( - data_dir: &Path, - db: &MainDb, - preview: &Value, - acceptance: &AutogenAcceptance, -) -> Result { - let repo_path = local_autogen_repo_path(data_dir); - ensure_local_autogen_repository(data_dir, db)?; - let accepted = accepted_preview_candidates(preview, acceptance)?; - let mut manifest = read_autogen_manifest(&repo_path)?.unwrap_or_else(empty_autogen_manifest); - let mut applied = Vec::new(); - let mut preserved = Vec::new(); - - for candidate in accepted { - let package_id = preview_package_id(candidate)?; - let relative_path = preview_relative_path(candidate)?; - let content = candidate - .get("content") - .and_then(Value::as_str) - .ok_or_else(|| CliError::Autogen("preview candidate missing content".to_owned()))?; - let expected_hash = candidate - .get("content_hash") - .and_then(Value::as_str) - .ok_or_else(|| { - CliError::Autogen("preview candidate missing content_hash".to_owned()) - })?; - if content_hash(content) != expected_hash { - return Err(CliError::Autogen(format!( - "preview content hash mismatch for {package_id}" - ))); - } - let target = safe_join(&repo_path, &relative_path)?; - if target.exists() { - let current = fs::read_to_string(&target).map_err(|source| { - CliError::Autogen(format!( - "failed to read existing autogen file '{}': {source}", - target.display() - )) - })?; - let current_hash = content_hash(¤t); - let known_hash = manifest - .package(&package_id) - .map(|entry| entry.content_hash.as_str()); - if known_hash != Some(current_hash.as_str()) { - preserved.push(preserve_autogen_file_in_local( - data_dir, - db, - &package_id, - ¤t, - )?); - } - } - if let Some(parent) = target.parent() { - fs::create_dir_all(parent).map_err(|source| { - CliError::Autogen(format!( - "failed to create autogen package directory '{}': {source}", - parent.display() - )) - })?; - } - fs::write(&target, content).map_err(|source| { - CliError::Autogen(format!( - "failed to write autogen package '{}': {source}", - target.display() - )) - })?; - upsert_manifest_entry( - &mut manifest, - AutogenManifestEntry { - package_id: package_id.clone(), - relative_path: relative_path.clone(), - content_hash: expected_hash.to_owned(), - }, - ); - db.upsert_generated_tracked_package_preserving_user_state( - &package_id, - &RepositoryId::new(LOCAL_AUTOGEN_REPOSITORY_ID).expect("valid id"), - )?; - applied.push(json!({ - "package_id": package_id.to_string(), - "output_relative_path": relative_path, - })); - } - - write_autogen_manifest(&repo_path, &manifest)?; - Ok(json!({ - "target_repo_id": LOCAL_AUTOGEN_REPOSITORY_ID, - "target_repo_path": repo_path, - "applied_count": applied.len(), - "applied": applied, - "preserved_to_local": preserved, - })) -} - -fn cleanup_preview_json( - data_dir: &Path, - db: &MainDb, - inventory: &InstalledInventory, -) -> Result { - let repo_path = local_autogen_repo_path(data_dir); - let Some(manifest) = read_autogen_manifest(&repo_path)? else { - return Ok(json!({ - "operation": "cleanup.preview", - "target_repo_id": LOCAL_AUTOGEN_REPOSITORY_ID, - "target_repo_path": repo_path, - "summary": { "candidate_count": 0, "skipped_count": 0, "write_count": 0, "delete_count": 0 }, - "candidates": [], - "skipped": [], - "diagnostics": [], - })); - }; - let plan = build_local_autogen_plan(db, inventory)?; - let installed_ids: BTreeSet = plan - .candidates - .iter() - .map(|candidate| candidate.package_id.to_string()) - .chain(plan.skipped.iter().map(|skip| skip.package_id.to_string())) - .collect(); - let mut candidates = Vec::new(); - for entry in manifest.packages { - if !installed_ids.contains(&entry.package_id.to_string()) { - candidates.push(json!({ - "package_id": entry.package_id.to_string(), - "action": "delete", - "output_relative_path": entry.relative_path, - "content_hash": entry.content_hash, - "reason": "not_in_installed_inventory", - })); - } - } - candidates.sort_by_key(|candidate| { - candidate - .get("package_id") - .and_then(Value::as_str) - .unwrap_or_default() - .to_owned() - }); - Ok(json!({ - "operation": "cleanup.preview", - "target_repo_id": LOCAL_AUTOGEN_REPOSITORY_ID, - "target_repo_path": repo_path, - "summary": { - "candidate_count": candidates.len(), - "skipped_count": 0, - "write_count": 0, - "delete_count": candidates.len(), - }, - "candidates": candidates, - "skipped": [], - "diagnostics": [], - })) -} - -fn apply_autogen_cleanup_preview( - data_dir: &Path, - db: &MainDb, - preview: &Value, - acceptance: &AutogenAcceptance, -) -> Result { - if preview.get("target_repo_id").and_then(Value::as_str) != Some(LOCAL_AUTOGEN_REPOSITORY_ID) { - return Err(CliError::Autogen(format!( - "cleanup preview target_repo_id must be '{LOCAL_AUTOGEN_REPOSITORY_ID}'" - ))); - } - let repo_path = local_autogen_repo_path(data_dir); - let accepted = accepted_preview_candidates(preview, acceptance)?; - let mut manifest = read_autogen_manifest(&repo_path)?.unwrap_or_else(empty_autogen_manifest); - let mut deleted = Vec::new(); - let mut preserved = Vec::new(); - let local_autogen_id = RepositoryId::new(LOCAL_AUTOGEN_REPOSITORY_ID).expect("valid id"); - for candidate in accepted { - let package_id = preview_package_id(candidate)?; - let relative_path = preview_relative_path(candidate)?; - let expected_hash = candidate - .get("content_hash") - .and_then(Value::as_str) - .ok_or_else(|| { - CliError::Autogen("cleanup preview candidate missing content_hash".to_owned()) - })?; - let manifest_entry = manifest.package(&package_id).ok_or_else(|| { - CliError::Autogen(format!( - "cleanup preview candidate {package_id} is not managed by local_autogen manifest" - )) - })?; - if manifest_entry.relative_path != relative_path - || manifest_entry.content_hash != expected_hash - { - return Err(CliError::Autogen(format!( - "cleanup preview candidate {package_id} does not match local_autogen manifest" - ))); - } - let target = safe_join(&repo_path, &relative_path)?; - if target.exists() { - let current = fs::read_to_string(&target).map_err(|source| { - CliError::Autogen(format!( - "failed to read existing autogen file '{}': {source}", - target.display() - )) - })?; - if content_hash(¤t) != expected_hash { - preserved.push(preserve_autogen_file_in_local( - data_dir, - db, - &package_id, - ¤t, - )?); - } - fs::remove_file(&target).map_err(|source| { - CliError::Autogen(format!( - "failed to delete autogen package '{}': {source}", - target.display() - )) - })?; - } - manifest - .packages - .retain(|entry| entry.package_id != package_id); - db.delete_generated_tracked_package(&package_id, &local_autogen_id)?; - deleted.push(json!({ - "package_id": package_id.to_string(), - "output_relative_path": relative_path, - })); - } - write_autogen_manifest(&repo_path, &manifest)?; - Ok(json!({ - "target_repo_id": LOCAL_AUTOGEN_REPOSITORY_ID, - "target_repo_path": repo_path, - "deleted_count": deleted.len(), - "deleted": deleted, - "preserved_to_local": preserved, - })) -} - -fn accepted_preview_candidates<'a>( - preview: &'a Value, - acceptance: &AutogenAcceptance, -) -> Result, CliError> { - let candidates = preview - .get("candidates") - .and_then(Value::as_array) - .ok_or_else(|| CliError::Autogen("autogen preview missing candidates".to_owned()))?; - match acceptance { - AutogenAcceptance::AcceptAll => Ok(candidates.iter().collect()), - AutogenAcceptance::Accept(ids) => { - let accepted: BTreeSet = ids.iter().map(ToString::to_string).collect(); - Ok(candidates - .iter() - .filter(|candidate| { - candidate - .get("package_id") - .and_then(Value::as_str) - .is_some_and(|id| accepted.contains(id)) - }) - .collect()) - } - } -} - -fn preview_package_id(candidate: &Value) -> Result { - candidate - .get("package_id") - .and_then(Value::as_str) - .ok_or_else(|| CliError::Autogen("preview candidate missing package_id".to_owned()))? - .parse() - .map_err(|source: getter_core::PackageIdError| CliError::Autogen(source.to_string())) -} - -fn preview_relative_path(candidate: &Value) -> Result { - candidate - .get("output_relative_path") - .and_then(Value::as_str) - .map(PathBuf::from) - .ok_or_else(|| { - CliError::Autogen("preview candidate missing output_relative_path".to_owned()) - }) -} - -fn ensure_local_autogen_repository(data_dir: &Path, db: &MainDb) -> Result { - let repo_path = local_autogen_repo_path(data_dir); - ensure_repository_layout(&repo_path, &local_autogen_repo_toml())?; - db.upsert_repository( - &RepositoryMetadata { - id: RepositoryId::new(LOCAL_AUTOGEN_REPOSITORY_ID).expect("valid id"), - name: LOCAL_AUTOGEN_REPOSITORY_NAME.to_owned(), - priority: RepositoryPriority::LOCAL_AUTOGEN, - api_version: REPO_API_VERSION_V1.to_owned(), - }, - Some(&repo_path), - None, - )?; - Ok(repo_path) -} - -fn ensure_local_repository(data_dir: &Path, db: &MainDb) -> Result { - let local_id = RepositoryId::new(LOCAL_REPOSITORY_ID).expect("valid id"); - if let Ok(existing) = find_repository(db, &local_id) { - let repo_path = repo_path(&existing)?; - ensure_repository_layout(&repo_path, &local_repo_toml())?; - return Ok(repo_path); - } - - let repo_path = data_dir.join("repositories").join(LOCAL_REPOSITORY_ID); - ensure_repository_layout(&repo_path, &local_repo_toml())?; - db.upsert_repository( - &RepositoryMetadata { - id: local_id, - name: LOCAL_REPOSITORY_NAME.to_owned(), - priority: RepositoryPriority::LOCAL, - api_version: REPO_API_VERSION_V1.to_owned(), - }, - Some(&repo_path), - None, - )?; - Ok(repo_path) -} - -fn ensure_repository_layout(repo_path: &Path, repo_toml: &str) -> Result<(), CliError> { - fs::create_dir_all(repo_path.join("packages")).map_err(|source| { - CliError::Autogen(format!( - "failed to create repository packages dir '{}': {source}", - repo_path.display() - )) - })?; - fs::create_dir_all(repo_path.join("lib")).map_err(|source| { - CliError::Autogen(format!( - "failed to create repository lib dir '{}': {source}", - repo_path.display() - )) - })?; - fs::create_dir_all(repo_path.join("templates")).map_err(|source| { - CliError::Autogen(format!( - "failed to create repository templates dir '{}': {source}", - repo_path.display() - )) - })?; - let repo_toml_path = repo_path.join("repo.toml"); - if !repo_toml_path.exists() { - fs::write(&repo_toml_path, repo_toml).map_err(|source| { - CliError::Autogen(format!( - "failed to write repo.toml '{}': {source}", - repo_toml_path.display() - )) - })?; - } - Ok(()) -} - -fn preserve_autogen_file_in_local( - data_dir: &Path, - db: &MainDb, - package_id: &PackageId, - content: &str, -) -> Result { - let local_repo = ensure_local_repository(data_dir, db)?; - let primary_relative = getter_core::autogen::package_relative_path(package_id); - let primary_target = safe_join(&local_repo, &primary_relative)?; - let relative_path = if primary_target.exists() { - let backup = PathBuf::from("autogen-preserved") - .join(package_id.kind().as_str()) - .join(format!( - "{}.{}.lua", - package_id.name(), - content_hash(content).replace(':', "-") - )); - safe_join(&local_repo, &backup)?; - backup - } else { - primary_relative - }; - let target = safe_join(&local_repo, &relative_path)?; - if let Some(parent) = target.parent() { - fs::create_dir_all(parent).map_err(|source| { - CliError::Autogen(format!( - "failed to create local preservation directory '{}': {source}", - parent.display() - )) - })?; - } - fs::write(&target, content).map_err(|source| { - CliError::Autogen(format!( - "failed to preserve modified autogen file '{}': {source}", - target.display() - )) - })?; - Ok(json!({ - "package_id": package_id.to_string(), - "repository_id": LOCAL_REPOSITORY_ID, - "relative_path": relative_path, - })) -} - -fn read_autogen_manifest(repo_path: &Path) -> Result, CliError> { - let path = repo_path.join(AUTOGEN_MANIFEST_FILE); - if !path.exists() { - return Ok(None); - } - let bytes = fs::read(&path).map_err(|source| { - CliError::Autogen(format!( - "failed to read autogen manifest '{}': {source}", - path.display() - )) - })?; - serde_json::from_slice(&bytes) - .map(Some) - .map_err(|source| CliError::Autogen(format!("failed to parse autogen manifest: {source}"))) -} - -fn write_autogen_manifest(repo_path: &Path, manifest: &AutogenManifest) -> Result<(), CliError> { - fs::create_dir_all(repo_path).map_err(|source| { - CliError::Autogen(format!( - "failed to create autogen repository '{}': {source}", - repo_path.display() - )) - })?; - let path = repo_path.join(AUTOGEN_MANIFEST_FILE); - let bytes = serde_json::to_vec_pretty(manifest) - .map_err(|source| CliError::Autogen(format!("failed to serialize manifest: {source}")))?; - fs::write(&path, bytes).map_err(|source| { - CliError::Autogen(format!( - "failed to write autogen manifest '{}': {source}", - path.display() - )) - }) -} - -fn empty_autogen_manifest() -> AutogenManifest { - AutogenManifest { - version: getter_core::autogen::AUTOGEN_MANIFEST_VERSION, - repository_id: RepositoryId::new(LOCAL_AUTOGEN_REPOSITORY_ID).expect("valid id"), - packages: Vec::new(), - } -} - -fn upsert_manifest_entry(manifest: &mut AutogenManifest, entry: AutogenManifestEntry) { - manifest - .packages - .retain(|existing| existing.package_id != entry.package_id); - manifest.packages.push(entry); - manifest - .packages - .sort_by_key(|existing| existing.package_id.to_string()); -} - -fn safe_join(root: &Path, relative: &Path) -> Result { - if relative.is_absolute() - || relative.components().any(|component| { - matches!( - component, - Component::ParentDir | Component::RootDir | Component::Prefix(_) - ) - }) - { - return Err(CliError::Autogen(format!( - "unsafe relative path '{}'", - relative.display() - ))); - } - Ok(root.join(relative)) -} - -fn local_autogen_repo_path(data_dir: &Path) -> PathBuf { - data_dir - .join("repositories") - .join(LOCAL_AUTOGEN_REPOSITORY_ID) -} - fn repo_path(repo: &StoredRepository) -> Result { repo.path .as_ref() diff --git a/crates/getter-core/Cargo.toml b/crates/getter-core/Cargo.toml index 47326c5..93325e9 100644 --- a/crates/getter-core/Cargo.toml +++ b/crates/getter-core/Cargo.toml @@ -3,8 +3,12 @@ name = "getter-core" version.workspace = true edition.workspace = true +[features] +default = ["lua"] +lua = ["dep:mlua"] + [dependencies] -mlua = { version = "0.10", features = ["lua54", "vendored"] } +mlua = { version = "0.10", features = ["lua54", "vendored"], optional = true } serde = { version = "1", features = ["derive"] } serde_json = "1" thiserror = "1" diff --git a/crates/getter-core/src/lib.rs b/crates/getter-core/src/lib.rs index aa3bb41..ce5c030 100644 --- a/crates/getter-core/src/lib.rs +++ b/crates/getter-core/src/lib.rs @@ -10,7 +10,9 @@ use std::fmt; use std::str::FromStr; pub mod autogen; +#[cfg(feature = "lua")] pub mod diagnostics; +#[cfg(feature = "lua")] pub mod lua; pub mod repository; pub mod task; diff --git a/crates/getter-operations/Cargo.toml b/crates/getter-operations/Cargo.toml new file mode 100644 index 0000000..f24bcff --- /dev/null +++ b/crates/getter-operations/Cargo.toml @@ -0,0 +1,10 @@ +[package] +name = "getter-operations" +version.workspace = true +edition.workspace = true + +[dependencies] +getter-core = { path = "../getter-core", default-features = false } +getter-storage = { path = "../getter-storage" } +serde_json = "1" +thiserror = "1" diff --git a/crates/getter-operations/src/autogen.rs b/crates/getter-operations/src/autogen.rs new file mode 100644 index 0000000..caab4aa --- /dev/null +++ b/crates/getter-operations/src/autogen.rs @@ -0,0 +1,637 @@ +//! Getter-owned installed-inventory autogen operations. +//! +//! The CLI and native bridge both call this module so there is one implementation +//! of `local_autogen` preview/apply semantics. Platform layers provide installed +//! inventory facts; this module decides generated package ids, repository +//! coverage, file writes, manifest updates, preservation behavior, and tracked +//! state updates. + +use getter_core::autogen::{ + content_hash, local_autogen_repo_toml, local_repo_toml, plan_local_autogen, AutogenManifest, + AutogenManifestEntry, AutogenPlan, AutogenSkipReason, InstalledInventory, + LOCAL_AUTOGEN_REPOSITORY_ID, LOCAL_AUTOGEN_REPOSITORY_NAME, LOCAL_REPOSITORY_ID, + LOCAL_REPOSITORY_NAME, +}; +use getter_core::repository::{RepositoryLayout, RepositoryMetadata, REPO_API_VERSION_V1}; +use getter_core::{PackageId, RepositoryId, RepositoryPriority}; +use getter_storage::{MainDb, StorageError, StoredRepository}; +use serde_json::{json, Value}; +use std::collections::{BTreeSet, HashMap}; +use std::fs; +use std::path::{Component, Path, PathBuf}; + +pub const AUTOGEN_MANIFEST_FILE: &str = "autogen-manifest.json"; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum AutogenAcceptance { + AcceptAll, + Accept(Vec), +} + +#[derive(Debug, thiserror::Error)] +pub enum AutogenOperationError { + #[error("storage error: {0}")] + Storage(#[from] StorageError), + #[error("repository error: {0}")] + Repository(String), + #[error("autogen error: {0}")] + Autogen(String), +} + +pub type AutogenOperationResult = Result; + +pub fn build_local_autogen_plan( + db: &MainDb, + inventory: &InstalledInventory, +) -> AutogenOperationResult { + let covered = higher_priority_package_coverage(db)?; + plan_local_autogen(inventory, &covered) + .map_err(|source| AutogenOperationError::Autogen(source.to_string())) +} + +pub fn installed_preview_json(data_dir: &Path, plan: &AutogenPlan) -> Value { + let candidates: Vec = plan.candidates.iter().map(autogen_candidate_json).collect(); + let skipped: Vec = plan.skipped.iter().map(autogen_skip_json).collect(); + json!({ + "operation": "installed.preview", + "target_repo_id": plan.repository_id.as_str(), + "target_repo_path": local_autogen_repo_path(data_dir), + "summary": { + "candidate_count": candidates.len(), + "skipped_count": skipped.len(), + "write_count": candidates.len(), + "delete_count": 0, + }, + "candidates": candidates, + "skipped": skipped, + "diagnostics": [], + }) +} + +pub fn cleanup_preview_json( + data_dir: &Path, + db: &MainDb, + inventory: &InstalledInventory, +) -> AutogenOperationResult { + let repo_path = local_autogen_repo_path(data_dir); + let Some(manifest) = read_autogen_manifest(&repo_path)? else { + return Ok(json!({ + "operation": "cleanup.preview", + "target_repo_id": LOCAL_AUTOGEN_REPOSITORY_ID, + "target_repo_path": repo_path, + "summary": { "candidate_count": 0, "skipped_count": 0, "write_count": 0, "delete_count": 0 }, + "candidates": [], + "skipped": [], + "diagnostics": [], + })); + }; + let plan = build_local_autogen_plan(db, inventory)?; + let installed_ids: BTreeSet = plan + .candidates + .iter() + .map(|candidate| candidate.package_id.to_string()) + .chain(plan.skipped.iter().map(|skip| skip.package_id.to_string())) + .collect(); + let mut candidates = Vec::new(); + for entry in manifest.packages { + if !installed_ids.contains(&entry.package_id.to_string()) { + candidates.push(json!({ + "package_id": entry.package_id.to_string(), + "action": "delete", + "output_relative_path": entry.relative_path, + "content_hash": entry.content_hash, + "reason": "not_in_installed_inventory", + })); + } + } + candidates.sort_by_key(|candidate| { + candidate + .get("package_id") + .and_then(Value::as_str) + .unwrap_or_default() + .to_owned() + }); + Ok(json!({ + "operation": "cleanup.preview", + "target_repo_id": LOCAL_AUTOGEN_REPOSITORY_ID, + "target_repo_path": repo_path, + "summary": { + "candidate_count": candidates.len(), + "skipped_count": 0, + "write_count": 0, + "delete_count": candidates.len(), + }, + "candidates": candidates, + "skipped": [], + "diagnostics": [], + })) +} + +pub fn apply_installed_preview( + data_dir: &Path, + db: &MainDb, + preview: &Value, + acceptance: &AutogenAcceptance, +) -> AutogenOperationResult { + let repo_path = local_autogen_repo_path(data_dir); + ensure_local_autogen_repository(data_dir, db)?; + let accepted = accepted_preview_candidates(preview, acceptance)?; + let mut manifest = read_autogen_manifest(&repo_path)?.unwrap_or_else(empty_autogen_manifest); + let mut applied = Vec::new(); + let mut preserved = Vec::new(); + + for candidate in accepted { + let package_id = preview_package_id(candidate)?; + let relative_path = preview_relative_path(candidate)?; + let content = candidate + .get("content") + .and_then(Value::as_str) + .ok_or_else(|| { + AutogenOperationError::Autogen("preview candidate missing content".to_owned()) + })?; + let expected_hash = candidate + .get("content_hash") + .and_then(Value::as_str) + .ok_or_else(|| { + AutogenOperationError::Autogen("preview candidate missing content_hash".to_owned()) + })?; + if content_hash(content) != expected_hash { + return Err(AutogenOperationError::Autogen(format!( + "preview content hash mismatch for {package_id}" + ))); + } + let target = safe_join(&repo_path, &relative_path)?; + if target.exists() { + let current = fs::read_to_string(&target).map_err(|source| { + AutogenOperationError::Autogen(format!( + "failed to read existing autogen file '{}': {source}", + target.display() + )) + })?; + let current_hash = content_hash(¤t); + let known_hash = manifest + .package(&package_id) + .map(|entry| entry.content_hash.as_str()); + if known_hash != Some(current_hash.as_str()) { + preserved.push(preserve_autogen_file_in_local( + data_dir, + db, + &package_id, + ¤t, + )?); + } + } + if let Some(parent) = target.parent() { + fs::create_dir_all(parent).map_err(|source| { + AutogenOperationError::Autogen(format!( + "failed to create autogen package directory '{}': {source}", + parent.display() + )) + })?; + } + fs::write(&target, content).map_err(|source| { + AutogenOperationError::Autogen(format!( + "failed to write autogen package '{}': {source}", + target.display() + )) + })?; + upsert_manifest_entry( + &mut manifest, + AutogenManifestEntry { + package_id: package_id.clone(), + relative_path: relative_path.clone(), + content_hash: expected_hash.to_owned(), + }, + ); + db.upsert_generated_tracked_package_preserving_user_state( + &package_id, + &RepositoryId::new(LOCAL_AUTOGEN_REPOSITORY_ID).expect("valid id"), + )?; + applied.push(json!({ + "package_id": package_id.to_string(), + "output_relative_path": relative_path, + })); + } + + write_autogen_manifest(&repo_path, &manifest)?; + Ok(json!({ + "target_repo_id": LOCAL_AUTOGEN_REPOSITORY_ID, + "target_repo_path": repo_path, + "applied_count": applied.len(), + "applied": applied, + "preserved_to_local": preserved, + })) +} + +pub fn apply_cleanup_preview( + data_dir: &Path, + db: &MainDb, + preview: &Value, + acceptance: &AutogenAcceptance, +) -> AutogenOperationResult { + if preview.get("target_repo_id").and_then(Value::as_str) != Some(LOCAL_AUTOGEN_REPOSITORY_ID) { + return Err(AutogenOperationError::Autogen(format!( + "cleanup preview target_repo_id must be '{LOCAL_AUTOGEN_REPOSITORY_ID}'" + ))); + } + let repo_path = local_autogen_repo_path(data_dir); + let accepted = accepted_preview_candidates(preview, acceptance)?; + let mut manifest = read_autogen_manifest(&repo_path)?.unwrap_or_else(empty_autogen_manifest); + let mut deleted = Vec::new(); + let mut preserved = Vec::new(); + let local_autogen_id = RepositoryId::new(LOCAL_AUTOGEN_REPOSITORY_ID).expect("valid id"); + for candidate in accepted { + let package_id = preview_package_id(candidate)?; + let relative_path = preview_relative_path(candidate)?; + let expected_hash = candidate + .get("content_hash") + .and_then(Value::as_str) + .ok_or_else(|| { + AutogenOperationError::Autogen( + "cleanup preview candidate missing content_hash".to_owned(), + ) + })?; + let manifest_entry = manifest.package(&package_id).ok_or_else(|| { + AutogenOperationError::Autogen(format!( + "cleanup preview candidate {package_id} is not managed by local_autogen manifest" + )) + })?; + if manifest_entry.relative_path != relative_path + || manifest_entry.content_hash != expected_hash + { + return Err(AutogenOperationError::Autogen(format!( + "cleanup preview candidate {package_id} does not match local_autogen manifest" + ))); + } + let target = safe_join(&repo_path, &relative_path)?; + if target.exists() { + let current = fs::read_to_string(&target).map_err(|source| { + AutogenOperationError::Autogen(format!( + "failed to read existing autogen file '{}': {source}", + target.display() + )) + })?; + if content_hash(¤t) != expected_hash { + preserved.push(preserve_autogen_file_in_local( + data_dir, + db, + &package_id, + ¤t, + )?); + } + fs::remove_file(&target).map_err(|source| { + AutogenOperationError::Autogen(format!( + "failed to delete autogen package '{}': {source}", + target.display() + )) + })?; + } + manifest + .packages + .retain(|entry| entry.package_id != package_id); + db.delete_generated_tracked_package(&package_id, &local_autogen_id)?; + deleted.push(json!({ + "package_id": package_id.to_string(), + "output_relative_path": relative_path, + })); + } + write_autogen_manifest(&repo_path, &manifest)?; + Ok(json!({ + "target_repo_id": LOCAL_AUTOGEN_REPOSITORY_ID, + "target_repo_path": repo_path, + "deleted_count": deleted.len(), + "deleted": deleted, + "preserved_to_local": preserved, + })) +} + +pub fn unwrap_preview_payload( + raw: Value, + expected_operation: &str, +) -> AutogenOperationResult { + let payload = if raw.get("ok").is_some() && raw.get("data").is_some() { + raw.get("data").cloned().unwrap_or(Value::Null) + } else { + raw + }; + if payload.get("operation").and_then(Value::as_str) != Some(expected_operation) { + return Err(AutogenOperationError::Autogen(format!( + "autogen preview operation must be '{expected_operation}'" + ))); + } + Ok(payload) +} + +pub fn local_autogen_repo_path(data_dir: &Path) -> PathBuf { + data_dir + .join("repositories") + .join(LOCAL_AUTOGEN_REPOSITORY_ID) +} + +fn higher_priority_package_coverage( + db: &MainDb, +) -> AutogenOperationResult> { + let mut covered = HashMap::new(); + for repo in db.repositories()? { + if repo.id.as_str() == LOCAL_AUTOGEN_REPOSITORY_ID { + continue; + } + if repo.priority <= RepositoryPriority::LOCAL_AUTOGEN { + continue; + } + let Some(path) = repo.path.as_ref() else { + continue; + }; + let layout = load_repository_layout(Path::new(path))?; + for package in layout.packages { + covered.entry(package.id).or_insert_with(|| repo.id.clone()); + } + } + Ok(covered) +} + +fn load_repository_layout(path: &Path) -> AutogenOperationResult { + RepositoryLayout::load(path) + .map_err(|source| AutogenOperationError::Repository(source.to_string())) +} + +fn autogen_candidate_json(candidate: &getter_core::autogen::AutogenCandidate) -> Value { + json!({ + "package_id": candidate.package_id.to_string(), + "kind": candidate.package_id.kind().as_str(), + "display_name": candidate.name, + "installed_target": candidate.installed, + "action": "create", + "output_relative_path": candidate.relative_path, + "content_hash": candidate.content_hash, + "content": candidate.content, + }) +} + +fn autogen_skip_json(skip: &getter_core::autogen::AutogenSkip) -> Value { + json!({ + "package_id": skip.package_id.to_string(), + "reason": match skip.reason { + AutogenSkipReason::DuplicateInventoryItem => "duplicate_inventory_item", + AutogenSkipReason::CoveredByHigherPriorityRepository => "covered_by_higher_priority_repo", + }, + "covering_repo_id": skip.repository_id.as_ref().map(RepositoryId::as_str), + }) +} + +fn accepted_preview_candidates<'a>( + preview: &'a Value, + acceptance: &AutogenAcceptance, +) -> AutogenOperationResult> { + let candidates = preview + .get("candidates") + .and_then(Value::as_array) + .ok_or_else(|| { + AutogenOperationError::Autogen("autogen preview missing candidates".to_owned()) + })?; + match acceptance { + AutogenAcceptance::AcceptAll => Ok(candidates.iter().collect()), + AutogenAcceptance::Accept(ids) => { + let accepted: BTreeSet = ids.iter().map(ToString::to_string).collect(); + Ok(candidates + .iter() + .filter(|candidate| { + candidate + .get("package_id") + .and_then(Value::as_str) + .is_some_and(|id| accepted.contains(id)) + }) + .collect()) + } + } +} + +fn preview_package_id(candidate: &Value) -> AutogenOperationResult { + candidate + .get("package_id") + .and_then(Value::as_str) + .ok_or_else(|| { + AutogenOperationError::Autogen("preview candidate missing package_id".to_owned()) + })? + .parse() + .map_err(|source: getter_core::PackageIdError| { + AutogenOperationError::Autogen(source.to_string()) + }) +} + +fn preview_relative_path(candidate: &Value) -> AutogenOperationResult { + candidate + .get("output_relative_path") + .and_then(Value::as_str) + .map(PathBuf::from) + .ok_or_else(|| { + AutogenOperationError::Autogen( + "preview candidate missing output_relative_path".to_owned(), + ) + }) +} + +fn ensure_local_autogen_repository( + data_dir: &Path, + db: &MainDb, +) -> AutogenOperationResult { + let repo_path = local_autogen_repo_path(data_dir); + ensure_repository_layout(&repo_path, &local_autogen_repo_toml())?; + db.upsert_repository( + &RepositoryMetadata { + id: RepositoryId::new(LOCAL_AUTOGEN_REPOSITORY_ID).expect("valid id"), + name: LOCAL_AUTOGEN_REPOSITORY_NAME.to_owned(), + priority: RepositoryPriority::LOCAL_AUTOGEN, + api_version: REPO_API_VERSION_V1.to_owned(), + }, + Some(&repo_path), + None, + )?; + Ok(repo_path) +} + +fn ensure_local_repository(data_dir: &Path, db: &MainDb) -> AutogenOperationResult { + let local_id = RepositoryId::new(LOCAL_REPOSITORY_ID).expect("valid id"); + if let Ok(existing) = find_repository(db, &local_id) { + let repo_path = repo_path(&existing)?; + ensure_repository_layout(&repo_path, &local_repo_toml())?; + return Ok(repo_path); + } + + let repo_path = data_dir.join("repositories").join(LOCAL_REPOSITORY_ID); + ensure_repository_layout(&repo_path, &local_repo_toml())?; + db.upsert_repository( + &RepositoryMetadata { + id: local_id, + name: LOCAL_REPOSITORY_NAME.to_owned(), + priority: RepositoryPriority::LOCAL, + api_version: REPO_API_VERSION_V1.to_owned(), + }, + Some(&repo_path), + None, + )?; + Ok(repo_path) +} + +fn find_repository(db: &MainDb, id: &RepositoryId) -> AutogenOperationResult { + db.repositories()? + .into_iter() + .find(|repo| &repo.id == id) + .ok_or_else(|| { + AutogenOperationError::Repository(format!("repository '{id}' is not registered")) + }) +} + +fn repo_path(repo: &StoredRepository) -> AutogenOperationResult { + repo.path.as_ref().map(PathBuf::from).ok_or_else(|| { + AutogenOperationError::Repository(format!("repository '{}' has no path", repo.id)) + }) +} + +fn ensure_repository_layout(repo_path: &Path, repo_toml: &str) -> AutogenOperationResult<()> { + fs::create_dir_all(repo_path.join("packages")).map_err(|source| { + AutogenOperationError::Autogen(format!( + "failed to create repository packages dir '{}': {source}", + repo_path.display() + )) + })?; + fs::create_dir_all(repo_path.join("lib")).map_err(|source| { + AutogenOperationError::Autogen(format!( + "failed to create repository lib dir '{}': {source}", + repo_path.display() + )) + })?; + fs::create_dir_all(repo_path.join("templates")).map_err(|source| { + AutogenOperationError::Autogen(format!( + "failed to create repository templates dir '{}': {source}", + repo_path.display() + )) + })?; + let repo_toml_path = repo_path.join("repo.toml"); + if !repo_toml_path.exists() { + fs::write(&repo_toml_path, repo_toml).map_err(|source| { + AutogenOperationError::Autogen(format!( + "failed to write repo.toml '{}': {source}", + repo_toml_path.display() + )) + })?; + } + Ok(()) +} + +fn preserve_autogen_file_in_local( + data_dir: &Path, + db: &MainDb, + package_id: &PackageId, + content: &str, +) -> AutogenOperationResult { + let local_repo = ensure_local_repository(data_dir, db)?; + let primary_relative = getter_core::autogen::package_relative_path(package_id); + let primary_target = safe_join(&local_repo, &primary_relative)?; + let relative_path = if primary_target.exists() { + let backup = PathBuf::from("autogen-preserved") + .join(package_id.kind().as_str()) + .join(format!( + "{}.{}.lua", + package_id.name(), + content_hash(content).replace(':', "-") + )); + safe_join(&local_repo, &backup)?; + backup + } else { + primary_relative + }; + let target = safe_join(&local_repo, &relative_path)?; + if let Some(parent) = target.parent() { + fs::create_dir_all(parent).map_err(|source| { + AutogenOperationError::Autogen(format!( + "failed to create local preservation directory '{}': {source}", + parent.display() + )) + })?; + } + fs::write(&target, content).map_err(|source| { + AutogenOperationError::Autogen(format!( + "failed to preserve modified autogen file '{}': {source}", + target.display() + )) + })?; + Ok(json!({ + "package_id": package_id.to_string(), + "repository_id": LOCAL_REPOSITORY_ID, + "relative_path": relative_path, + })) +} + +fn read_autogen_manifest(repo_path: &Path) -> AutogenOperationResult> { + let path = repo_path.join(AUTOGEN_MANIFEST_FILE); + if !path.exists() { + return Ok(None); + } + let bytes = fs::read(&path).map_err(|source| { + AutogenOperationError::Autogen(format!( + "failed to read autogen manifest '{}': {source}", + path.display() + )) + })?; + serde_json::from_slice(&bytes).map(Some).map_err(|source| { + AutogenOperationError::Autogen(format!("failed to parse autogen manifest: {source}")) + }) +} + +fn write_autogen_manifest( + repo_path: &Path, + manifest: &AutogenManifest, +) -> AutogenOperationResult<()> { + fs::create_dir_all(repo_path).map_err(|source| { + AutogenOperationError::Autogen(format!( + "failed to create autogen repository '{}': {source}", + repo_path.display() + )) + })?; + let path = repo_path.join(AUTOGEN_MANIFEST_FILE); + let bytes = serde_json::to_vec_pretty(manifest).map_err(|source| { + AutogenOperationError::Autogen(format!("failed to serialize manifest: {source}")) + })?; + fs::write(&path, bytes).map_err(|source| { + AutogenOperationError::Autogen(format!( + "failed to write autogen manifest '{}': {source}", + path.display() + )) + }) +} + +fn empty_autogen_manifest() -> AutogenManifest { + AutogenManifest { + version: getter_core::autogen::AUTOGEN_MANIFEST_VERSION, + repository_id: RepositoryId::new(LOCAL_AUTOGEN_REPOSITORY_ID).expect("valid id"), + packages: Vec::new(), + } +} + +fn upsert_manifest_entry(manifest: &mut AutogenManifest, entry: AutogenManifestEntry) { + manifest + .packages + .retain(|existing| existing.package_id != entry.package_id); + manifest.packages.push(entry); + manifest + .packages + .sort_by_key(|existing| existing.package_id.to_string()); +} + +fn safe_join(root: &Path, relative: &Path) -> AutogenOperationResult { + if relative.is_absolute() + || relative.components().any(|component| { + matches!( + component, + Component::ParentDir | Component::RootDir | Component::Prefix(_) + ) + }) + { + return Err(AutogenOperationError::Autogen(format!( + "unsafe relative path '{}'", + relative.display() + ))); + } + Ok(root.join(relative)) +} diff --git a/crates/getter-operations/src/lib.rs b/crates/getter-operations/src/lib.rs new file mode 100644 index 0000000..f4432e2 --- /dev/null +++ b/crates/getter-operations/src/lib.rs @@ -0,0 +1,8 @@ +//! Reusable getter-owned operations shared by CLI and native embedders. +//! +//! This crate intentionally contains product/domain orchestration that must not +//! be duplicated in Flutter or Android/Kotlin bridge glue. Platform adapters may +//! provide raw facts to callers, but the rules for repository coverage, +//! generated package files, manifests, and tracked state live here. + +pub mod autogen; diff --git a/crates/getter-storage/Cargo.toml b/crates/getter-storage/Cargo.toml index bc0964b..da9172d 100644 --- a/crates/getter-storage/Cargo.toml +++ b/crates/getter-storage/Cargo.toml @@ -4,7 +4,7 @@ version.workspace = true edition.workspace = true [dependencies] -getter-core = { path = "../getter-core" } +getter-core = { path = "../getter-core", default-features = false } rusqlite = { version = "0.32", features = ["bundled"] } serde_json = "1" thiserror = "1" diff --git a/src/lib.rs b/src/lib.rs index 607df46..d29d0ad 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -7,6 +7,8 @@ #[cfg(feature = "domain")] pub use getter_core as core; #[cfg(feature = "domain")] +pub use getter_operations as operations; +#[cfg(feature = "domain")] pub use getter_storage as storage; #[cfg(feature = "rustls-platform-verifier")] From 21dc994f5d452238a0b5fda3ecf9cd09e541155a Mon Sep 17 00:00:00 2001 From: Xiangzhe Date: Wed, 24 Jun 2026 10:17:21 +0800 Subject: [PATCH 18/52] feat: share legacy room operations --- Cargo.lock | 1 + crates/getter-cli/src/lib.rs | 208 +--------- crates/getter-operations/Cargo.toml | 1 + crates/getter-operations/src/legacy_room.rs | 426 ++++++++++++++++++++ crates/getter-operations/src/lib.rs | 1 + 5 files changed, 448 insertions(+), 189 deletions(-) create mode 100644 crates/getter-operations/src/legacy_room.rs diff --git a/Cargo.lock b/Cargo.lock index 1ff310d..fce5fc5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -566,6 +566,7 @@ version = "0.1.0" dependencies = [ "getter-core", "getter-storage", + "serde", "serde_json", "thiserror 1.0.69", ] diff --git a/crates/getter-cli/src/lib.rs b/crates/getter-cli/src/lib.rs index d9f4d47..3562bf1 100644 --- a/crates/getter-cli/src/lib.rs +++ b/crates/getter-cli/src/lib.rs @@ -18,10 +18,9 @@ use getter_downloader::{ cancel_download_task, record_install_result, run_fake_download_task, submit_fake_download_task, }; use getter_operations::autogen::{self, AutogenAcceptance, AutogenOperationError}; +use getter_operations::legacy_room::{self, LegacyRoomOperationError}; use getter_storage::legacy_room::{ - map_legacy_app, read_legacy_room_database, LegacyAppKind, LegacyAppRecord, - LegacyExtraAppRecord, LegacyPackageResolution, LegacyRoomDbImport, LegacyRoomImportWarning, - LegacyRoomReadError, + map_legacy_app, LegacyAppKind, LegacyAppRecord, LegacyExtraAppRecord, LegacyPackageResolution, }; use getter_storage::{ CacheDb, MainDb, MigrationRecordUpsert, StorageError, StoredPackageResolution, @@ -250,6 +249,20 @@ impl From for CliError { } } +impl From for CliError { + fn from(value: LegacyRoomOperationError) -> Self { + match value { + LegacyRoomOperationError::Storage(detail) => Self::Storage(detail), + LegacyRoomOperationError::UnsupportedDb { report_path } => { + Self::UnsupportedLegacyDb { report_path } + } + LegacyRoomOperationError::InvalidDb { report_path } => { + Self::InvalidLegacyDb { report_path } + } + } + } +} + #[derive(Debug, Clone, PartialEq, Eq)] pub struct CliOutput { pub exit_code: ExitCode, @@ -705,64 +718,11 @@ fn execute(invocation: CliInvocation) -> Result { })) } CliCommand::LegacyImportRoomDb { db: legacy_db } => { - let db = open_main_db(&invocation.data_dir)?; - if db.migration_record_exists(LEGACY_ROOM_MIGRATION_ID)? { - let records = db.tracked_packages()?; - return Ok(json!({ - "already_imported": true, - "imported_records": 0, - "source_counts": { - "app_rows": 0, - "extra_app_rows": 0, - "hub_rows": 0, - "extra_hub_rows": 0, - }, - "warnings": [], - "apps": tracked_packages_json(records), - })); - } - let import = read_legacy_room_database(&legacy_db).map_err(|source| { - legacy_db_import_error(&invocation.data_dir, &legacy_db, source) - })?; - let imported_records = import.apps.len(); - let source_counts = source_counts_json(&import); - let warnings = import_warnings_json(&import.warnings); - if import.source_counts.app_rows > 0 && imported_records == 0 { - let report_path = create_migration_report_with_source_counts( - &invocation.data_dir, - &legacy_db, - "migration.invalid_db", - "Legacy Room database has app rows but no importable app rows", - 0, - 0, - &warnings, - Some(&source_counts), - )?; - return Err(CliError::InvalidLegacyDb { report_path }); - } - import_legacy_room_db(&db, &import)?; - let report_path = create_migration_report_with_source_counts( - &invocation.data_dir, - &legacy_db, - "migration.imported", - "Legacy Room database imported", - imported_records as u64, - imported_records as u64, - &warnings, - Some(&source_counts), - )?; - let records = db.tracked_packages()?; - Ok(json!({ - "report_path": report_path, - "imported_records": imported_records, - "source_counts": source_counts, - "warnings": warnings, - "apps": tracked_packages_json(records), - })) + legacy_room::import_room_db_json(&invocation.data_dir, &legacy_db) + .map_err(CliError::from) } CliCommand::LegacyReportList => { - open_initialized_storage(&invocation.data_dir)?; - Ok(json!({ "reports": list_migration_reports(&invocation.data_dir)? })) + legacy_room::report_list_json(&invocation.data_dir).map_err(CliError::from) } } } @@ -1053,34 +1013,6 @@ fn import_legacy_room_bundle(db: &MainDb, bundle: &LegacyRoomBundle) -> Result<( Ok(()) } -fn import_legacy_room_db(db: &MainDb, import: &LegacyRoomDbImport) -> Result<(), CliError> { - let mut packages = Vec::new(); - for app in &import.apps { - let mapping = map_legacy_app(&app.app, Some(&app.user_state)) - .map_err(|source| CliError::Storage(source.to_string()))?; - packages.push(tracked_package_upsert(mapping)); - } - - let report_json = json!({ - "ok": true, - "source": "legacy-room-db", - "version": import.version, - "imported_records": import.apps.len(), - "source_counts": source_counts_json(import), - "warnings": import_warnings_json(&import.warnings), - }) - .to_string(); - db.import_tracked_packages_with_migration_record( - &packages, - &MigrationRecordUpsert { - id: LEGACY_ROOM_MIGRATION_ID, - source: "legacy-room-db", - report_json: &report_json, - }, - )?; - Ok(()) -} - fn tracked_package_upsert( mapping: getter_storage::legacy_room::LegacyAppMapping, ) -> TrackedPackageUpsert { @@ -1108,54 +1040,6 @@ fn stored_resolution(resolution: LegacyPackageResolution) -> StoredPackageResolu } } -fn legacy_db_import_error( - data_dir: &Path, - legacy_db: &Path, - source: LegacyRoomReadError, -) -> CliError { - let (code, unsupported) = match source { - LegacyRoomReadError::UnsupportedVersion { .. } => ("migration.unsupported_db", true), - LegacyRoomReadError::Sqlite(_) | LegacyRoomReadError::MissingRequiredTable(_) => { - ("migration.invalid_db", false) - } - }; - match create_migration_report_with_source_counts( - data_dir, - legacy_db, - code, - &source.to_string(), - 0, - 0, - &[], - None, - ) { - Ok(report_path) if unsupported => CliError::UnsupportedLegacyDb { report_path }, - Ok(report_path) => CliError::InvalidLegacyDb { report_path }, - Err(error) => error, - } -} - -fn source_counts_json(import: &LegacyRoomDbImport) -> Value { - json!({ - "app_rows": import.source_counts.app_rows, - "extra_app_rows": import.source_counts.extra_app_rows, - "hub_rows": import.source_counts.hub_rows, - "extra_hub_rows": import.source_counts.extra_hub_rows, - }) -} - -fn import_warnings_json(warnings: &[LegacyRoomImportWarning]) -> Vec { - warnings - .iter() - .map(|warning| { - json!({ - "code": warning.code(), - "message": warning.message(), - }) - }) - .collect() -} - fn main_db_path(data_dir: &Path) -> PathBuf { data_dir.join(MAIN_DB_FILE) } @@ -1232,60 +1116,6 @@ fn report_file_name(code: &str) -> String { format!("{}.json", code.replace('.', "-")) } -fn list_migration_reports(data_dir: &Path) -> Result, CliError> { - let reports_dir = data_dir.join(MIGRATION_REPORTS_DIR); - if !reports_dir.exists() { - return Ok(Vec::new()); - } - - let mut report_paths = fs::read_dir(&reports_dir) - .map_err(|source| CliError::Storage(format!("failed to read migration reports: {source}")))? - .map(|entry| entry.map(|entry| entry.path())) - .collect::, _>>() - .map_err(|source| { - CliError::Storage(format!("failed to read migration report entry: {source}")) - })?; - report_paths.retain(|path| { - path.extension() - .is_some_and(|extension| extension == "json") - }); - report_paths.sort(); - - report_paths - .into_iter() - .map(|path| { - let bytes = fs::read(&path).map_err(|source| { - CliError::Storage(format!( - "failed to read migration report '{}': {source}", - path.display() - )) - })?; - let report: Value = serde_json::from_slice(&bytes).map_err(|source| { - CliError::Storage(format!( - "failed to parse migration report '{}': {source}", - path.display() - )) - })?; - Ok(json!({ - "ok": report.get("ok").and_then(Value::as_bool).unwrap_or(false), - "code": report.get("code").and_then(Value::as_str).unwrap_or("migration.unknown"), - "message": report.get("message").and_then(Value::as_str).unwrap_or("Legacy migration report"), - "source_file_name": report - .get("source_file_name") - .or_else(|| report.get("bundle_file_name")) - .and_then(Value::as_str), - "imported_records": report.get("imported_records").and_then(Value::as_u64).unwrap_or(0), - "tracked_records": report.get("tracked_records").and_then(Value::as_u64).unwrap_or(0), - "warnings": report - .get("warnings") - .cloned() - .unwrap_or_else(|| Value::Array(Vec::new())), - "source_counts": report.get("source_counts").cloned().unwrap_or(Value::Null), - })) - }) - .collect() -} - #[derive(Debug, Serialize)] struct MigrationReport<'a> { ok: bool, diff --git a/crates/getter-operations/Cargo.toml b/crates/getter-operations/Cargo.toml index f24bcff..a44e0b6 100644 --- a/crates/getter-operations/Cargo.toml +++ b/crates/getter-operations/Cargo.toml @@ -6,5 +6,6 @@ edition.workspace = true [dependencies] getter-core = { path = "../getter-core", default-features = false } getter-storage = { path = "../getter-storage" } +serde = { version = "1", features = ["derive"] } serde_json = "1" thiserror = "1" diff --git a/crates/getter-operations/src/legacy_room.rs b/crates/getter-operations/src/legacy_room.rs new file mode 100644 index 0000000..40c7c34 --- /dev/null +++ b/crates/getter-operations/src/legacy_room.rs @@ -0,0 +1,426 @@ +//! Getter-owned legacy Room migration operations. +//! +//! Platform layers may prepare/copy/checkpoint a legacy SQLite database and pass +//! its path to getter, but this module owns reading Room rows, mapping legacy app +//! records, writing tracked package state, recording migration completion, and +//! producing sanitized migration reports. + +use getter_storage::legacy_room::{ + map_legacy_app, read_legacy_room_database, LegacyPackageResolution, LegacyRoomDbImport, + LegacyRoomImportWarning, LegacyRoomReadError, +}; +use getter_storage::{ + CacheDb, MainDb, MigrationRecordUpsert, StorageError, StoredPackageResolution, + StoredTrackedPackage, TrackedPackageUpsert, +}; +use serde::Serialize; +use serde_json::{json, Value}; +use std::fs; +use std::path::{Path, PathBuf}; + +const MAIN_DB_FILE: &str = "main.db"; +const CACHE_DB_FILE: &str = "cache.db"; +pub const MIGRATION_REPORTS_DIR: &str = "migration-reports"; +pub const LEGACY_ROOM_MIGRATION_ID: &str = "legacy-room-v17"; + +#[derive(Debug, thiserror::Error)] +pub enum LegacyRoomOperationError { + #[error("storage error: {0}")] + Storage(String), + #[error("unsupported legacy Room database")] + UnsupportedDb { report_path: PathBuf }, + #[error("invalid legacy Room database")] + InvalidDb { report_path: PathBuf }, +} + +impl LegacyRoomOperationError { + pub fn code(&self) -> &'static str { + match self { + Self::Storage(_) => "storage.error", + Self::UnsupportedDb { .. } => "migration.unsupported_db", + Self::InvalidDb { .. } => "migration.invalid_db", + } + } + + pub fn message(&self) -> &'static str { + match self { + Self::Storage(_) => "Getter storage operation failed", + Self::UnsupportedDb { .. } => "Legacy Room database version is not supported", + Self::InvalidDb { .. } => "Legacy Room database is invalid", + } + } + + pub fn detail(&self) -> Option { + match self { + Self::Storage(detail) => Some(detail.clone()), + Self::UnsupportedDb { .. } | Self::InvalidDb { .. } => None, + } + } + + pub fn report_path(&self) -> Option<&Path> { + match self { + Self::UnsupportedDb { report_path } | Self::InvalidDb { report_path } => { + Some(report_path.as_path()) + } + Self::Storage(_) => None, + } + } +} + +impl From for LegacyRoomOperationError { + fn from(source: StorageError) -> Self { + Self::Storage(source.to_string()) + } +} + +pub type LegacyRoomOperationResult = Result; + +pub fn import_room_db_json(data_dir: &Path, legacy_db: &Path) -> LegacyRoomOperationResult { + let db = open_main_db(data_dir)?; + if db.migration_record_exists(LEGACY_ROOM_MIGRATION_ID)? { + let records = db.tracked_packages()?; + return Ok(json!({ + "already_imported": true, + "imported_records": 0, + "source_counts": { + "app_rows": 0, + "extra_app_rows": 0, + "hub_rows": 0, + "extra_hub_rows": 0, + }, + "warnings": [], + "apps": tracked_packages_json(records), + })); + } + + let import = read_legacy_room_database(legacy_db) + .map_err(|source| legacy_db_import_error(data_dir, legacy_db, source))?; + let imported_records = import.apps.len(); + let source_counts = source_counts_json(&import); + let warnings = import_warnings_json(&import.warnings); + if import.source_counts.app_rows > 0 && imported_records == 0 { + let report_path = create_migration_report_with_source_counts( + data_dir, + legacy_db, + "migration.invalid_db", + "Legacy Room database has app rows but no importable app rows", + 0, + 0, + &warnings, + Some(&source_counts), + )?; + return Err(LegacyRoomOperationError::InvalidDb { report_path }); + } + + import_legacy_room_db(&db, &import)?; + let report_path = create_migration_report_with_source_counts( + data_dir, + legacy_db, + "migration.imported", + "Legacy Room database imported", + imported_records as u64, + imported_records as u64, + &warnings, + Some(&source_counts), + )?; + let records = db.tracked_packages()?; + Ok(json!({ + "report_path": report_path, + "imported_records": imported_records, + "source_counts": source_counts, + "warnings": warnings, + "apps": tracked_packages_json(records), + })) +} + +pub fn report_list_json(data_dir: &Path) -> LegacyRoomOperationResult { + open_initialized_storage(data_dir)?; + Ok(json!({ "reports": list_migration_reports(data_dir)? })) +} + +fn initialize_storage(data_dir: &Path) -> LegacyRoomOperationResult<()> { + fs::create_dir_all(data_dir).map_err(|source| { + LegacyRoomOperationError::Storage(format!("failed to create data directory: {source}")) + })?; + MainDb::open(data_dir.join(MAIN_DB_FILE))?; + CacheDb::open(data_dir.join(CACHE_DB_FILE))?; + Ok(()) +} + +fn open_initialized_storage(data_dir: &Path) -> LegacyRoomOperationResult<()> { + initialize_storage(data_dir) +} + +fn open_main_db(data_dir: &Path) -> LegacyRoomOperationResult { + initialize_storage(data_dir)?; + Ok(MainDb::open(data_dir.join(MAIN_DB_FILE))?) +} + +fn import_legacy_room_db( + db: &MainDb, + import: &LegacyRoomDbImport, +) -> LegacyRoomOperationResult<()> { + let mut packages = Vec::new(); + for app in &import.apps { + let mapping = map_legacy_app(&app.app, Some(&app.user_state)) + .map_err(|source| LegacyRoomOperationError::Storage(source.to_string()))?; + packages.push(tracked_package_upsert(mapping)); + } + + let report_json = json!({ + "ok": true, + "source": "legacy-room-db", + "version": import.version, + "imported_records": import.apps.len(), + "source_counts": source_counts_json(import), + "warnings": import_warnings_json(&import.warnings), + }) + .to_string(); + db.import_tracked_packages_with_migration_record( + &packages, + &MigrationRecordUpsert { + id: LEGACY_ROOM_MIGRATION_ID, + source: "legacy-room-db", + report_json: &report_json, + }, + )?; + Ok(()) +} + +fn tracked_package_upsert( + mapping: getter_storage::legacy_room::LegacyAppMapping, +) -> TrackedPackageUpsert { + TrackedPackageUpsert { + package_id: mapping.package_id, + enabled: true, + favorite: mapping.user_state.favorite, + ignored_version: mapping.user_state.ignored_version, + repository_id: None, + package_resolution: stored_resolution(mapping.package_resolution), + } +} + +fn stored_resolution(resolution: LegacyPackageResolution) -> StoredPackageResolution { + match resolution { + LegacyPackageResolution::OfficialRepositoryPackage => { + StoredPackageResolution::OfficialRepositoryPackage + } + LegacyPackageResolution::GenerateLocalPackage => { + StoredPackageResolution::GenerateLocalPackage + } + LegacyPackageResolution::MissingPackageDefinition => { + StoredPackageResolution::MissingPackageDefinition + } + } +} + +fn legacy_db_import_error( + data_dir: &Path, + legacy_db: &Path, + source: LegacyRoomReadError, +) -> LegacyRoomOperationError { + let (code, unsupported) = match source { + LegacyRoomReadError::UnsupportedVersion { .. } => ("migration.unsupported_db", true), + LegacyRoomReadError::Sqlite(_) | LegacyRoomReadError::MissingRequiredTable(_) => { + ("migration.invalid_db", false) + } + }; + match create_migration_report_with_source_counts( + data_dir, + legacy_db, + code, + &source.to_string(), + 0, + 0, + &[], + None, + ) { + Ok(report_path) if unsupported => LegacyRoomOperationError::UnsupportedDb { report_path }, + Ok(report_path) => LegacyRoomOperationError::InvalidDb { report_path }, + Err(error) => error, + } +} + +fn tracked_packages_json(packages: Vec) -> Vec { + packages + .into_iter() + .map(|package| { + json!({ + "id": package.package_id.to_string(), + "enabled": package.enabled, + "favorite": package.favorite, + "ignored_version": package.ignored_version, + "repository_id": package.repository_id.map(|id| id.to_string()), + "package_resolution": package.package_resolution.as_str(), + }) + }) + .collect() +} + +fn source_counts_json(import: &LegacyRoomDbImport) -> Value { + json!({ + "app_rows": import.source_counts.app_rows, + "extra_app_rows": import.source_counts.extra_app_rows, + "hub_rows": import.source_counts.hub_rows, + "extra_hub_rows": import.source_counts.extra_hub_rows, + }) +} + +fn import_warnings_json(warnings: &[LegacyRoomImportWarning]) -> Vec { + warnings + .iter() + .map(|warning| { + json!({ + "code": warning.code(), + "message": warning.message(), + }) + }) + .collect() +} + +#[allow(clippy::too_many_arguments)] +fn create_migration_report_with_source_counts( + data_dir: &Path, + source_file: &Path, + code: &str, + detail: &str, + imported_records: u64, + tracked_records: u64, + warnings: &[Value], + source_counts: Option<&Value>, +) -> LegacyRoomOperationResult { + let reports_dir = data_dir.join(MIGRATION_REPORTS_DIR); + fs::create_dir_all(&reports_dir).map_err(|source| { + LegacyRoomOperationError::Storage(format!( + "failed to create migration report directory: {source}" + )) + })?; + let report_path = reports_dir.join(report_file_name(code)); + let report = MigrationReport { + ok: code == "migration.imported", + code, + message: match code { + "migration.invalid_db" => "Legacy Room database is invalid", + "migration.unsupported_db" => "Legacy Room database version is not supported", + "migration.imported" => "Legacy Room data imported", + _ => "Legacy migration failed", + }, + source_file_name: source_file.file_name().and_then(|name| name.to_str()), + detail, + imported_records, + tracked_records, + warnings, + source_counts, + }; + let bytes = serde_json::to_vec_pretty(&report).map_err(|source| { + LegacyRoomOperationError::Storage(format!("failed to serialize report: {source}")) + })?; + fs::write(&report_path, bytes).map_err(|source| { + LegacyRoomOperationError::Storage(format!("failed to write report: {source}")) + })?; + Ok(report_path) +} + +fn report_file_name(code: &str) -> String { + format!("{}.json", code.replace('.', "-")) +} + +fn list_migration_reports(data_dir: &Path) -> LegacyRoomOperationResult> { + let reports_dir = data_dir.join(MIGRATION_REPORTS_DIR); + if !reports_dir.exists() { + return Ok(Vec::new()); + } + + let mut report_paths = fs::read_dir(&reports_dir) + .map_err(|source| { + LegacyRoomOperationError::Storage(format!("failed to read migration reports: {source}")) + })? + .map(|entry| entry.map(|entry| entry.path())) + .collect::, _>>() + .map_err(|source| { + LegacyRoomOperationError::Storage(format!( + "failed to read migration report entry: {source}" + )) + })?; + report_paths.retain(|path| { + path.extension() + .is_some_and(|extension| extension == "json") + }); + report_paths.sort(); + + report_paths + .into_iter() + .map(|path| { + let bytes = fs::read(&path).map_err(|source| { + LegacyRoomOperationError::Storage(format!( + "failed to read migration report '{}': {source}", + path.display() + )) + })?; + let report: Value = serde_json::from_slice(&bytes).map_err(|source| { + LegacyRoomOperationError::Storage(format!( + "failed to parse migration report '{}': {source}", + path.display() + )) + })?; + Ok(json!({ + "ok": report.get("ok").and_then(Value::as_bool).unwrap_or(false), + "code": report.get("code").and_then(Value::as_str).unwrap_or("migration.unknown"), + "message": report.get("message").and_then(Value::as_str).unwrap_or("Legacy migration report"), + "source_file_name": report + .get("source_file_name") + .or_else(|| report.get("bundle_file_name")) + .and_then(Value::as_str), + "imported_records": report.get("imported_records").and_then(Value::as_u64).unwrap_or(0), + "tracked_records": report.get("tracked_records").and_then(Value::as_u64).unwrap_or(0), + "warnings": report + .get("warnings") + .cloned() + .unwrap_or_else(|| Value::Array(Vec::new())), + "source_counts": report.get("source_counts").cloned().unwrap_or(Value::Null), + })) + }) + .collect() +} + +#[derive(Debug, Serialize)] +struct MigrationReport<'a> { + ok: bool, + code: &'a str, + message: &'a str, + source_file_name: Option<&'a str>, + detail: &'a str, + imported_records: u64, + tracked_records: u64, + warnings: &'a [Value], + #[serde(skip_serializing_if = "Option::is_none")] + source_counts: Option<&'a Value>, +} + +#[cfg(test)] +mod tests { + use super::*; + use std::time::{SystemTime, UNIX_EPOCH}; + + #[test] + fn import_room_db_rejects_malformed_db_with_report() { + let temp = temp_dir("malformed"); + fs::create_dir_all(&temp).unwrap(); + let db_path = temp.join("legacy.db"); + fs::write(&db_path, b"not sqlite").unwrap(); + + let error = import_room_db_json(&temp.join("getter"), &db_path).unwrap_err(); + + assert!(matches!(error, LegacyRoomOperationError::InvalidDb { .. })); + assert_eq!(error.code(), "migration.invalid_db"); + assert!(error.report_path().is_some_and(Path::exists)); + } + + fn temp_dir(name: &str) -> PathBuf { + let nanos = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos(); + std::env::temp_dir().join(format!("upgradeall-legacy-room-op-{name}-{nanos}")) + } +} diff --git a/crates/getter-operations/src/lib.rs b/crates/getter-operations/src/lib.rs index f4432e2..71a0ca1 100644 --- a/crates/getter-operations/src/lib.rs +++ b/crates/getter-operations/src/lib.rs @@ -6,3 +6,4 @@ //! generated package files, manifests, and tracked state live here. pub mod autogen; +pub mod legacy_room; From 8e3e3c12803f1f8c2659361bfeba78c666917772 Mon Sep 17 00:00:00 2001 From: Xiangzhe Date: Thu, 25 Jun 2026 10:20:45 +0800 Subject: [PATCH 19/52] feat(core): add in-memory task runtime --- crates/getter-core/src/autogen.rs | 4 + crates/getter-core/src/lib.rs | 1 + crates/getter-core/src/runtime.rs | 936 ++++++++++++++++++++++++++++++ 3 files changed, 941 insertions(+) create mode 100644 crates/getter-core/src/runtime.rs diff --git a/crates/getter-core/src/autogen.rs b/crates/getter-core/src/autogen.rs index 265c6be..2001ac5 100644 --- a/crates/getter-core/src/autogen.rs +++ b/crates/getter-core/src/autogen.rs @@ -369,8 +369,11 @@ fn fnv1a64(bytes: &[u8]) -> u64 { #[cfg(test)] mod tests { use super::*; + #[cfg(feature = "lua")] use crate::lua::evaluate_package_file; + #[cfg(feature = "lua")] use crate::repository::RepositoryLayout; + #[cfg(feature = "lua")] use std::fs; #[test] @@ -492,6 +495,7 @@ mod tests { ); } + #[cfg(feature = "lua")] #[test] fn generated_lua_evaluates_through_package_boundary() { let temp = tempfile::tempdir().unwrap(); diff --git a/crates/getter-core/src/lib.rs b/crates/getter-core/src/lib.rs index ce5c030..96cdb95 100644 --- a/crates/getter-core/src/lib.rs +++ b/crates/getter-core/src/lib.rs @@ -15,6 +15,7 @@ pub mod diagnostics; #[cfg(feature = "lua")] pub mod lua; pub mod repository; +pub mod runtime; pub mod task; pub mod update; diff --git a/crates/getter-core/src/runtime.rs b/crates/getter-core/src/runtime.rs new file mode 100644 index 0000000..3dd4b7c --- /dev/null +++ b/crates/getter-core/src/runtime.rs @@ -0,0 +1,936 @@ +//! In-memory Phase D runtime for getter-owned update/download/install tasks. +//! +//! This module implements the ADR-0011 task/action/runtime-notification shape. +//! It deliberately keeps task state in the current process only: no SQLite task +//! persistence, no daemon, no recovery, and no replayable event log. Product +//! embedders can hold this runtime as a process-lifetime singleton and expose +//! [`RuntimeNotification`] through a push stream. + +use crate::{PackageId, UpdateAction}; +use serde::{Deserialize, Serialize}; +use std::collections::{BTreeMap, HashMap}; + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct PackageVersionLuaObject { + pub object_id: String, + pub dependency_digest: String, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct SealedActionPlan { + pub package_id: PackageId, + #[serde(default)] + pub actions: Vec, + pub lua_object: PackageVersionLuaObject, +} + +impl SealedActionPlan { + fn has_install_action(&self) -> bool { + self.actions + .iter() + .any(|action| matches!(action, UpdateAction::Install { .. })) + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct IssuedAction { + pub action_id: String, + pub package_id: PackageId, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct TaskSnapshot { + pub task_id: String, + pub package_id: PackageId, + pub status: RuntimeTaskStatus, + pub phase: TaskPhase, + #[serde(default)] + pub progress: Option, + pub capabilities: TaskCapabilities, + #[serde(default)] + pub current_diagnostic: Option, + pub updated_at: u64, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum RuntimeTaskStatus { + Queued, + Running, + Paused, + Failed, + Completed, + Canceled, +} + +impl RuntimeTaskStatus { + pub const fn is_active(self) -> bool { + matches!(self, Self::Queued | Self::Running | Self::Paused) + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct TaskPhase { + pub category: TaskPhaseCategory, + #[serde(default)] + pub reason: Option, +} + +impl TaskPhase { + pub const fn new(category: TaskPhaseCategory) -> Self { + Self { + category, + reason: None, + } + } + + pub const fn with_reason(category: TaskPhaseCategory, reason: TaskPhaseReason) -> Self { + Self { + category, + reason: Some(reason), + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum TaskPhaseCategory { + Queued, + Download, + WaitingUser, + Install, + Completed, + Failed, + Canceled, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum TaskPhaseReason { + InstallHandoff, + PackageLocked, + UserRejected, + DownloadFailed, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct TaskProgress { + pub unit: TaskProgressUnit, + pub current: u64, + #[serde(default)] + pub total: Option, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum TaskProgressUnit { + Percent, + Bit, +} + +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct TaskCapabilities { + pub cancel: bool, + pub pause: bool, + pub resume: bool, + pub retry: bool, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct TaskDiagnostic { + pub code: String, + pub message: String, + pub severity: DiagnosticSeverity, +} + +impl TaskDiagnostic { + pub fn error(code: impl Into, message: impl Into) -> Self { + Self { + code: code.into(), + message: message.into(), + severity: DiagnosticSeverity::Error, + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum DiagnosticSeverity { + Info, + Warning, + Error, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "snake_case")] +pub enum RuntimeNotification { + TaskChanged { task: TaskSnapshot }, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum UserResult { + Accepted, + Rejected, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum TaskCleanMode { + /// Remove completed and canceled tasks only. + Default, + /// Remove failed tasks only. + Failed, + /// Remove completed, canceled, and failed tasks. + AllInactive, +} + +#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] +pub enum RuntimeError { + #[error("update action is no longer available: {0}")] + ActionNotFound(String), + #[error("task not found: {0}")] + TaskNotFound(String), + #[error("task is not waiting for a user result: {0}")] + TaskNotWaitingForUser(String), + #[error("task cannot be paused in its current state: {0}")] + PauseNotSupported(String), + #[error("task cannot be resumed in its current state: {0}")] + ResumeNotSupported(String), + #[error("task cannot be retried in its current state: {0}")] + RetryNotSupported(String), + #[error("task cannot be canceled in its current state: {0}")] + CancelNotSupported(String), + #[error("task is active and must be canceled before removal: {0}")] + TaskActive(String), +} + +impl RuntimeError { + pub const fn code(&self) -> &'static str { + match self { + Self::ActionNotFound(_) => "action.not_found", + Self::TaskNotFound(_) => "task.not_found", + Self::TaskNotWaitingForUser(_) => "task.not_waiting_for_user", + Self::PauseNotSupported(_) => "task.pause_not_supported", + Self::ResumeNotSupported(_) => "task.resume_not_supported", + Self::RetryNotSupported(_) => "task.retry_not_supported", + Self::CancelNotSupported(_) => "task.cancel_not_supported", + Self::TaskActive(_) => "task.active", + } + } +} + +type RuntimeNotificationSink = Box; + +#[derive(Default)] +pub struct GetterRuntime { + next_action_id: u64, + next_task_id: u64, + logical_clock: u64, + actions: HashMap, + tasks: BTreeMap, + package_locks: HashMap, + notification_sink: Option, +} + +impl GetterRuntime { + pub fn new() -> Self { + Self::default() + } + + pub fn set_notification_sink( + &mut self, + sink: impl FnMut(RuntimeNotification) + Send + 'static, + ) { + self.notification_sink = Some(Box::new(sink)); + } + + pub fn issue_action(&mut self, plan: SealedActionPlan) -> IssuedAction { + self.next_action_id += 1; + let action_id = format!("action-{}", self.next_action_id); + let package_id = plan.package_id.clone(); + self.actions.insert(action_id.clone(), plan); + IssuedAction { + action_id, + package_id, + } + } + + pub fn submit_action(&mut self, action_id: &str) -> Result { + let plan = self + .actions + .remove(action_id) + .ok_or_else(|| RuntimeError::ActionNotFound(action_id.to_owned()))?; + self.next_task_id += 1; + let task_id = format!("task-{}", self.next_task_id); + let task = RuntimeTask::new(task_id.clone(), plan, self.tick()); + let snapshot = task.snapshot(); + self.tasks.insert(task_id, task); + self.notify_task_changed(snapshot.clone()); + Ok(snapshot) + } + + pub fn task(&self, task_id: &str) -> Result { + Ok(self.task_ref(task_id)?.snapshot()) + } + + pub fn tasks(&self) -> Vec { + self.tasks.values().map(RuntimeTask::snapshot).collect() + } + + pub fn active_tasks(&self) -> Vec { + self.tasks + .values() + .filter(|task| task.status.is_active()) + .map(RuntimeTask::snapshot) + .collect() + } + + pub fn tasks_for_package(&self, package_id: &PackageId) -> Vec { + self.tasks + .values() + .filter(|task| &task.plan.package_id == package_id) + .map(RuntimeTask::snapshot) + .collect() + } + + pub fn start_task(&mut self, task_id: &str) -> Result { + { + let clock = self.tick(); + let task = self.task_mut(task_id)?; + match task.status { + RuntimeTaskStatus::Queued => task.start_download(clock), + RuntimeTaskStatus::Running => {} + _ => return Err(RuntimeError::PauseNotSupported(task_id.to_owned())), + } + } + self.snapshot_and_notify(task_id) + } + + pub fn set_download_progress( + &mut self, + task_id: &str, + current_bits: u64, + total_bits: Option, + ) -> Result { + { + let clock = self.tick(); + let task = self.task_mut(task_id)?; + if !task.is_running_download() { + return Err(RuntimeError::PauseNotSupported(task_id.to_owned())); + } + task.progress = Some(TaskProgress { + unit: TaskProgressUnit::Bit, + current: current_bits, + total: total_bits, + }); + task.updated_at = clock; + } + self.snapshot_and_notify(task_id) + } + + pub fn complete_download(&mut self, task_id: &str) -> Result { + let needs_install = { + let task = self.task_ref(task_id)?; + task.plan.has_install_action() + }; + + if needs_install { + self.enter_install_handoff(task_id) + } else { + { + let clock = self.tick(); + let task = self.task_mut(task_id)?; + if !matches!( + task.status, + RuntimeTaskStatus::Queued | RuntimeTaskStatus::Running + ) { + return Err(RuntimeError::RetryNotSupported(task_id.to_owned())); + } + task.complete(clock); + } + self.snapshot_and_notify(task_id) + } + } + + pub fn pause_task(&mut self, task_id: &str) -> Result { + { + let clock = self.tick(); + let task = self.task_mut(task_id)?; + if !task.is_running_download() { + return Err(RuntimeError::PauseNotSupported(task_id.to_owned())); + } + task.status = RuntimeTaskStatus::Paused; + task.updated_at = clock; + } + self.snapshot_and_notify(task_id) + } + + pub fn resume_task(&mut self, task_id: &str) -> Result { + { + let clock = self.tick(); + let task = self.task_mut(task_id)?; + if task.status != RuntimeTaskStatus::Paused { + return Err(RuntimeError::ResumeNotSupported(task_id.to_owned())); + } + task.status = RuntimeTaskStatus::Running; + task.phase = TaskPhase::new(TaskPhaseCategory::Download); + task.updated_at = clock; + } + self.snapshot_and_notify(task_id) + } + + pub fn user_result( + &mut self, + task_id: &str, + result: UserResult, + reason: Option<&str>, + ) -> Result { + { + let task = self.task_ref(task_id)?; + if !task.is_waiting_user() { + return Err(RuntimeError::TaskNotWaitingForUser(task_id.to_owned())); + } + } + self.unlock_package_for_task(task_id); + { + let clock = self.tick(); + let task = self.task_mut(task_id)?; + match result { + UserResult::Accepted => task.complete(clock), + UserResult::Rejected => task.fail( + clock, + TaskPhase::with_reason( + TaskPhaseCategory::Failed, + TaskPhaseReason::UserRejected, + ), + TaskDiagnostic::error( + "user.rejected", + reason.unwrap_or("User rejected the pending step"), + ), + RetryResume::InstallHandoff, + ), + } + } + self.snapshot_and_notify(task_id) + } + + pub fn cancel_task(&mut self, task_id: &str) -> Result { + { + let task = self.task_ref(task_id)?; + if !matches!( + task.status, + RuntimeTaskStatus::Queued | RuntimeTaskStatus::Running | RuntimeTaskStatus::Paused + ) { + return Err(RuntimeError::CancelNotSupported(task_id.to_owned())); + } + } + self.unlock_package_for_task(task_id); + { + let clock = self.tick(); + let task = self.task_mut(task_id)?; + task.cancel(clock); + } + self.snapshot_and_notify(task_id) + } + + pub fn retry_task(&mut self, task_id: &str) -> Result { + let retry_resume = { + let task = self.task_ref(task_id)?; + if task.status != RuntimeTaskStatus::Failed { + return Err(RuntimeError::RetryNotSupported(task_id.to_owned())); + } + task.retry_resume + }; + match retry_resume { + RetryResume::Download => { + { + let clock = self.tick(); + let task = self.task_mut(task_id)?; + task.start_download(clock); + } + self.snapshot_and_notify(task_id) + } + RetryResume::InstallHandoff => self.enter_install_handoff(task_id), + } + } + + pub fn remove_task(&mut self, task_id: &str) -> Result { + let task = self.task_ref(task_id)?; + if task.status.is_active() { + return Err(RuntimeError::TaskActive(task_id.to_owned())); + } + let removed = self + .tasks + .remove(task_id) + .expect("task existence checked before remove"); + Ok(removed.snapshot()) + } + + pub fn clean_tasks(&mut self, mode: TaskCleanMode) -> Vec { + let task_ids: Vec = self + .tasks + .iter() + .filter_map(|(task_id, task)| { + let remove = match mode { + TaskCleanMode::Default => matches!( + task.status, + RuntimeTaskStatus::Completed | RuntimeTaskStatus::Canceled + ), + TaskCleanMode::Failed => task.status == RuntimeTaskStatus::Failed, + TaskCleanMode::AllInactive => !task.status.is_active(), + }; + remove.then(|| task_id.clone()) + }) + .collect(); + + task_ids + .into_iter() + .filter_map(|task_id| { + let removed = self.tasks.remove(&task_id)?; + Some(removed.snapshot()) + }) + .collect() + } + + fn enter_install_handoff(&mut self, task_id: &str) -> Result { + let package_id = self.task_ref(task_id)?.plan.package_id.clone(); + if self.package_locks.contains_key(&package_id) { + { + let clock = self.tick(); + let task = self.task_mut(task_id)?; + task.fail( + clock, + TaskPhase::with_reason( + TaskPhaseCategory::Install, + TaskPhaseReason::PackageLocked, + ), + TaskDiagnostic::error( + "package.locked", + format!( + "Package {} is already being modified by another task", + package_id + ), + ), + RetryResume::InstallHandoff, + ); + } + return self.snapshot_and_notify(task_id); + } + + self.package_locks.insert(package_id, task_id.to_owned()); + { + let clock = self.tick(); + let task = self.task_mut(task_id)?; + task.enter_waiting_install(clock); + } + self.snapshot_and_notify(task_id) + } + + fn unlock_package_for_task(&mut self, task_id: &str) { + self.package_locks + .retain(|_, locked_by_task| locked_by_task != task_id); + } + + fn snapshot_and_notify(&mut self, task_id: &str) -> Result { + let snapshot = self.task(task_id)?; + self.notify_task_changed(snapshot.clone()); + Ok(snapshot) + } + + fn notify_task_changed(&mut self, task: TaskSnapshot) { + if let Some(sink) = self.notification_sink.as_mut() { + sink(RuntimeNotification::TaskChanged { task }); + } + } + + fn task_ref(&self, task_id: &str) -> Result<&RuntimeTask, RuntimeError> { + self.tasks + .get(task_id) + .ok_or_else(|| RuntimeError::TaskNotFound(task_id.to_owned())) + } + + fn task_mut(&mut self, task_id: &str) -> Result<&mut RuntimeTask, RuntimeError> { + self.tasks + .get_mut(task_id) + .ok_or_else(|| RuntimeError::TaskNotFound(task_id.to_owned())) + } + + fn tick(&mut self) -> u64 { + self.logical_clock += 1; + self.logical_clock + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum RetryResume { + Download, + InstallHandoff, +} + +#[derive(Debug, Clone)] +struct RuntimeTask { + id: String, + plan: SealedActionPlan, + status: RuntimeTaskStatus, + phase: TaskPhase, + progress: Option, + current_diagnostic: Option, + retry_resume: RetryResume, + updated_at: u64, +} + +impl RuntimeTask { + fn new(id: String, plan: SealedActionPlan, updated_at: u64) -> Self { + Self { + id, + plan, + status: RuntimeTaskStatus::Queued, + phase: TaskPhase::new(TaskPhaseCategory::Queued), + progress: None, + current_diagnostic: None, + retry_resume: RetryResume::Download, + updated_at, + } + } + + fn snapshot(&self) -> TaskSnapshot { + TaskSnapshot { + task_id: self.id.clone(), + package_id: self.plan.package_id.clone(), + status: self.status, + phase: self.phase.clone(), + progress: self.progress.clone(), + capabilities: self.capabilities(), + current_diagnostic: self.current_diagnostic.clone(), + updated_at: self.updated_at, + } + } + + fn capabilities(&self) -> TaskCapabilities { + TaskCapabilities { + cancel: matches!( + self.status, + RuntimeTaskStatus::Queued | RuntimeTaskStatus::Running | RuntimeTaskStatus::Paused + ), + pause: self.is_running_download(), + resume: self.status == RuntimeTaskStatus::Paused, + retry: self.status == RuntimeTaskStatus::Failed, + } + } + + fn is_running_download(&self) -> bool { + self.status == RuntimeTaskStatus::Running + && self.phase.category == TaskPhaseCategory::Download + } + + fn is_waiting_user(&self) -> bool { + self.status == RuntimeTaskStatus::Running + && self.phase + == TaskPhase::with_reason( + TaskPhaseCategory::WaitingUser, + TaskPhaseReason::InstallHandoff, + ) + } + + fn start_download(&mut self, updated_at: u64) { + self.status = RuntimeTaskStatus::Running; + self.phase = TaskPhase::new(TaskPhaseCategory::Download); + self.progress = Some(TaskProgress { + unit: TaskProgressUnit::Bit, + current: 0, + total: None, + }); + self.current_diagnostic = None; + self.updated_at = updated_at; + } + + fn enter_waiting_install(&mut self, updated_at: u64) { + self.status = RuntimeTaskStatus::Running; + self.phase = TaskPhase::with_reason( + TaskPhaseCategory::WaitingUser, + TaskPhaseReason::InstallHandoff, + ); + self.progress = None; + self.current_diagnostic = None; + self.updated_at = updated_at; + } + + fn complete(&mut self, updated_at: u64) { + self.status = RuntimeTaskStatus::Completed; + self.phase = TaskPhase::new(TaskPhaseCategory::Completed); + self.progress = None; + self.current_diagnostic = None; + self.updated_at = updated_at; + } + + fn cancel(&mut self, updated_at: u64) { + self.status = RuntimeTaskStatus::Canceled; + self.phase = TaskPhase::new(TaskPhaseCategory::Canceled); + self.progress = None; + self.current_diagnostic = None; + self.updated_at = updated_at; + } + + fn fail( + &mut self, + updated_at: u64, + phase: TaskPhase, + diagnostic: TaskDiagnostic, + retry_resume: RetryResume, + ) { + self.status = RuntimeTaskStatus::Failed; + self.phase = phase; + self.progress = None; + self.current_diagnostic = Some(diagnostic); + self.retry_resume = retry_resume; + self.updated_at = updated_at; + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{PackageKind, UpdateAction}; + use std::sync::{Arc, Mutex}; + + #[test] + fn submit_consumes_action_id_and_pushes_snapshot() { + let notifications = Arc::new(Mutex::new(Vec::new())); + let sink_notifications = notifications.clone(); + let mut runtime = GetterRuntime::new(); + runtime.set_notification_sink(move |notification| { + sink_notifications.lock().unwrap().push(notification); + }); + let action = runtime.issue_action(plan("android/org.fdroid.fdroid")); + + let task = runtime.submit_action(&action.action_id).unwrap(); + + assert_eq!(task.task_id, "task-1"); + assert_eq!(task.status, RuntimeTaskStatus::Queued); + assert_eq!(task.capabilities.cancel, true); + let error = runtime.submit_action(&action.action_id).unwrap_err(); + assert_eq!(error.code(), "action.not_found"); + assert_eq!(notifications.lock().unwrap().len(), 1); + } + + #[test] + fn fake_install_waits_for_generic_user_result() { + let mut runtime = GetterRuntime::new(); + let task_id = submit_plan(&mut runtime, plan("android/org.fdroid.fdroid")); + + runtime.start_task(&task_id).unwrap(); + let waiting = runtime.complete_download(&task_id).unwrap(); + + assert_eq!(waiting.status, RuntimeTaskStatus::Running); + assert_eq!( + waiting.phase, + TaskPhase::with_reason( + TaskPhaseCategory::WaitingUser, + TaskPhaseReason::InstallHandoff + ) + ); + assert_eq!(waiting.capabilities.pause, false); + assert_eq!(waiting.capabilities.resume, false); + assert_eq!(waiting.capabilities.cancel, true); + + let completed = runtime + .user_result(&task_id, UserResult::Accepted, None) + .unwrap(); + assert_eq!(completed.status, RuntimeTaskStatus::Completed); + assert_eq!( + completed.phase, + TaskPhase::new(TaskPhaseCategory::Completed) + ); + } + + #[test] + fn rejected_user_result_fails_and_retry_reuses_same_task() { + let mut runtime = GetterRuntime::new(); + let task_id = submit_plan(&mut runtime, plan("android/org.fdroid.fdroid")); + runtime.start_task(&task_id).unwrap(); + runtime.complete_download(&task_id).unwrap(); + + let failed = runtime + .user_result(&task_id, UserResult::Rejected, Some("not now")) + .unwrap(); + + assert_eq!(failed.status, RuntimeTaskStatus::Failed); + assert_eq!(failed.capabilities.retry, true); + assert_eq!( + failed.current_diagnostic.as_ref().unwrap().code, + "user.rejected" + ); + + let retried = runtime.retry_task(&task_id).unwrap(); + assert_eq!(retried.task_id, task_id); + assert_eq!(retried.status, RuntimeTaskStatus::Running); + assert_eq!( + retried.phase, + TaskPhase::with_reason( + TaskPhaseCategory::WaitingUser, + TaskPhaseReason::InstallHandoff + ) + ); + } + + #[test] + fn user_result_outside_waiting_user_is_an_error_without_mutation() { + let mut runtime = GetterRuntime::new(); + let task_id = submit_plan(&mut runtime, plan("android/org.fdroid.fdroid")); + let before = runtime.task(&task_id).unwrap(); + + let error = runtime + .user_result(&task_id, UserResult::Accepted, None) + .unwrap_err(); + + assert_eq!(error.code(), "task.not_waiting_for_user"); + assert_eq!(runtime.task(&task_id).unwrap(), before); + } + + #[test] + fn pause_resume_are_download_phase_controls_only() { + let mut runtime = GetterRuntime::new(); + let task_id = submit_plan(&mut runtime, plan("android/org.fdroid.fdroid")); + runtime.start_task(&task_id).unwrap(); + runtime + .set_download_progress(&task_id, 42, Some(100)) + .unwrap(); + + let paused = runtime.pause_task(&task_id).unwrap(); + assert_eq!(paused.status, RuntimeTaskStatus::Paused); + assert_eq!(paused.capabilities.resume, true); + assert_eq!(paused.capabilities.cancel, true); + + let resumed = runtime.resume_task(&task_id).unwrap(); + assert_eq!(resumed.status, RuntimeTaskStatus::Running); + assert_eq!(resumed.phase, TaskPhase::new(TaskPhaseCategory::Download)); + + runtime.complete_download(&task_id).unwrap(); + let error = runtime.pause_task(&task_id).unwrap_err(); + assert_eq!(error.code(), "task.pause_not_supported"); + } + + #[test] + fn same_package_install_lock_fails_later_task_without_waiting() { + let mut runtime = GetterRuntime::new(); + let first_task = submit_plan(&mut runtime, plan("android/org.fdroid.fdroid")); + let second_task = submit_plan(&mut runtime, plan("android/org.fdroid.fdroid")); + + runtime.start_task(&first_task).unwrap(); + runtime.complete_download(&first_task).unwrap(); + runtime.start_task(&second_task).unwrap(); + let failed = runtime.complete_download(&second_task).unwrap(); + + assert_eq!(failed.status, RuntimeTaskStatus::Failed); + assert_eq!( + failed.phase, + TaskPhase::with_reason(TaskPhaseCategory::Install, TaskPhaseReason::PackageLocked) + ); + assert_eq!( + failed.current_diagnostic.as_ref().unwrap().code, + "package.locked" + ); + + runtime + .user_result(&first_task, UserResult::Accepted, None) + .unwrap(); + let retried = runtime.retry_task(&second_task).unwrap(); + assert_eq!(retried.status, RuntimeTaskStatus::Running); + assert_eq!( + retried.phase, + TaskPhase::with_reason( + TaskPhaseCategory::WaitingUser, + TaskPhaseReason::InstallHandoff + ) + ); + } + + #[test] + fn cancel_remove_and_clean_follow_in_memory_task_lifecycle() { + let mut runtime = GetterRuntime::new(); + let canceled = submit_plan(&mut runtime, plan("android/org.fdroid.fdroid")); + let failed = submit_plan(&mut runtime, plan("android/com.termux")); + let completed = submit_plan(&mut runtime, plan_without_install("generic/example")); + + runtime.cancel_task(&canceled).unwrap(); + runtime.start_task(&failed).unwrap(); + runtime.complete_download(&failed).unwrap(); + runtime + .user_result(&failed, UserResult::Rejected, None) + .unwrap(); + runtime.start_task(&completed).unwrap(); + runtime.complete_download(&completed).unwrap(); + + let removed_by_default = runtime.clean_tasks(TaskCleanMode::Default); + let removed_ids: Vec<_> = removed_by_default + .into_iter() + .map(|task| task.task_id) + .collect(); + assert!(removed_ids.contains(&canceled)); + assert!(removed_ids.contains(&completed)); + assert!(!removed_ids.contains(&failed)); + assert_eq!( + runtime.task(&failed).unwrap().status, + RuntimeTaskStatus::Failed + ); + + let removed_failed = runtime.remove_task(&failed).unwrap(); + assert_eq!(removed_failed.status, RuntimeTaskStatus::Failed); + assert_eq!(runtime.task(&failed).unwrap_err().code(), "task.not_found"); + } + + #[test] + fn active_tasks_cannot_be_removed_and_can_be_canceled_while_paused() { + let mut runtime = GetterRuntime::new(); + let task_id = submit_plan(&mut runtime, plan("android/org.fdroid.fdroid")); + runtime.start_task(&task_id).unwrap(); + runtime.pause_task(&task_id).unwrap(); + + let error = runtime.remove_task(&task_id).unwrap_err(); + assert_eq!(error.code(), "task.active"); + + let canceled = runtime.cancel_task(&task_id).unwrap(); + assert_eq!(canceled.status, RuntimeTaskStatus::Canceled); + runtime.remove_task(&task_id).unwrap(); + } + + fn submit_plan(runtime: &mut GetterRuntime, plan: SealedActionPlan) -> String { + let action = runtime.issue_action(plan); + runtime.submit_action(&action.action_id).unwrap().task_id + } + + fn plan(package_id: &str) -> SealedActionPlan { + SealedActionPlan { + package_id: package_id.parse().unwrap(), + lua_object: PackageVersionLuaObject { + object_id: format!("lua:{package_id}:1.0"), + dependency_digest: "sha256-test".to_owned(), + }, + actions: vec![ + UpdateAction::Download { + url: "https://example.invalid/app.apk".to_owned(), + file_name: "app.apk".to_owned(), + }, + UpdateAction::Install { + installer: "android_package".to_owned(), + file: "app.apk".to_owned(), + }, + ], + } + } + + fn plan_without_install(package_id: &str) -> SealedActionPlan { + SealedActionPlan { + package_id: PackageId::new(PackageKind::Generic, package_id.split('/').nth(1).unwrap()) + .unwrap(), + lua_object: PackageVersionLuaObject { + object_id: format!("lua:{package_id}:1.0"), + dependency_digest: "sha256-test".to_owned(), + }, + actions: vec![UpdateAction::Download { + url: "https://example.invalid/file.bin".to_owned(), + file_name: "file.bin".to_owned(), + }], + } + } +} From aa0b75eed55f5be2614ec40623c13aa1982d4559 Mon Sep 17 00:00:00 2001 From: Xiangzhe Date: Thu, 25 Jun 2026 10:37:13 +0800 Subject: [PATCH 20/52] feat(operations): expose runtime task controls --- crates/getter-operations/src/lib.rs | 1 + crates/getter-operations/src/runtime.rs | 373 ++++++++++++++++++++++++ 2 files changed, 374 insertions(+) create mode 100644 crates/getter-operations/src/runtime.rs diff --git a/crates/getter-operations/src/lib.rs b/crates/getter-operations/src/lib.rs index 71a0ca1..6d75c88 100644 --- a/crates/getter-operations/src/lib.rs +++ b/crates/getter-operations/src/lib.rs @@ -7,3 +7,4 @@ pub mod autogen; pub mod legacy_room; +pub mod runtime; diff --git a/crates/getter-operations/src/runtime.rs b/crates/getter-operations/src/runtime.rs new file mode 100644 index 0000000..0d6e11d --- /dev/null +++ b/crates/getter-operations/src/runtime.rs @@ -0,0 +1,373 @@ +//! Shared JSON operation helpers for the in-memory Phase D getter runtime. +//! +//! These functions are intentionally thin over [`getter_core::runtime`]. They +//! give embedders (native bridge, future single-process debug CLI, tests) one +//! place to map JSON requests into runtime controls without reintroducing +//! persisted task state or duplicating task/control semantics in Flutter or +//! Android glue. + +use getter_core::{ + runtime::{ + GetterRuntime, IssuedAction, RuntimeError, SealedActionPlan, TaskCleanMode, TaskSnapshot, + UserResult, + }, + PackageId, +}; +use serde::Deserialize; +use serde_json::{json, Value}; + +#[derive(Debug, thiserror::Error)] +pub enum RuntimeOperationError { + #[error("invalid runtime request: {0}")] + InvalidRequest(String), + #[error("runtime error: {0}")] + Runtime(#[from] RuntimeError), + #[error("runtime response serialization failed: {0}")] + Serialize(String), +} + +impl RuntimeOperationError { + pub fn code(&self) -> &'static str { + match self { + Self::InvalidRequest(_) => "runtime.invalid_request", + Self::Runtime(error) => error.code(), + Self::Serialize(_) => "runtime.serialize_error", + } + } + + pub fn message(&self) -> &'static str { + match self { + Self::InvalidRequest(_) => "Getter runtime request is invalid", + Self::Runtime(_) => "Getter runtime operation failed", + Self::Serialize(_) => "Getter runtime response serialization failed", + } + } + + pub fn detail(&self) -> Option { + match self { + Self::InvalidRequest(detail) | Self::Serialize(detail) => Some(detail.clone()), + Self::Runtime(error) => Some(error.to_string()), + } + } +} + +pub fn issue_action(runtime: &mut GetterRuntime, plan: SealedActionPlan) -> Value { + issued_action_json(runtime.issue_action(plan)) +} + +pub fn submit_action_json( + runtime: &mut GetterRuntime, + request_json: &str, +) -> Result { + let request: SubmitActionRequest = parse_request(request_json)?; + task_json(runtime.submit_action(&request.action_id)?) +} + +pub fn task_get_json( + runtime: &GetterRuntime, + request_json: &str, +) -> Result { + let request: TaskIdRequest = parse_request(request_json)?; + task_json(runtime.task(&request.task_id)?) +} + +pub fn task_list_json( + runtime: &GetterRuntime, + request_json: &str, +) -> Result { + let request: TaskListRequest = parse_request(request_json)?; + let tasks = if let Some(package_id) = request.package_id.as_ref() { + runtime.tasks_for_package(package_id) + } else if request.active { + runtime.active_tasks() + } else { + runtime.tasks() + }; + tasks_json(tasks) +} + +pub fn task_start_json( + runtime: &mut GetterRuntime, + request_json: &str, +) -> Result { + let request: TaskIdRequest = parse_request(request_json)?; + task_json(runtime.start_task(&request.task_id)?) +} + +pub fn task_download_progress_json( + runtime: &mut GetterRuntime, + request_json: &str, +) -> Result { + let request: DownloadProgressRequest = parse_request(request_json)?; + task_json(runtime.set_download_progress( + &request.task_id, + request.current_bits, + request.total_bits, + )?) +} + +pub fn task_complete_download_json( + runtime: &mut GetterRuntime, + request_json: &str, +) -> Result { + let request: TaskIdRequest = parse_request(request_json)?; + task_json(runtime.complete_download(&request.task_id)?) +} + +pub fn task_pause_json( + runtime: &mut GetterRuntime, + request_json: &str, +) -> Result { + let request: TaskIdRequest = parse_request(request_json)?; + task_json(runtime.pause_task(&request.task_id)?) +} + +pub fn task_resume_json( + runtime: &mut GetterRuntime, + request_json: &str, +) -> Result { + let request: TaskIdRequest = parse_request(request_json)?; + task_json(runtime.resume_task(&request.task_id)?) +} + +pub fn task_user_result_json( + runtime: &mut GetterRuntime, + request_json: &str, +) -> Result { + let request: UserResultRequest = parse_request(request_json)?; + task_json(runtime.user_result(&request.task_id, request.result, request.reason.as_deref())?) +} + +pub fn task_cancel_json( + runtime: &mut GetterRuntime, + request_json: &str, +) -> Result { + let request: TaskIdRequest = parse_request(request_json)?; + task_json(runtime.cancel_task(&request.task_id)?) +} + +pub fn task_retry_json( + runtime: &mut GetterRuntime, + request_json: &str, +) -> Result { + let request: TaskIdRequest = parse_request(request_json)?; + task_json(runtime.retry_task(&request.task_id)?) +} + +pub fn task_remove_json( + runtime: &mut GetterRuntime, + request_json: &str, +) -> Result { + let request: TaskIdRequest = parse_request(request_json)?; + task_json(runtime.remove_task(&request.task_id)?) +} + +pub fn task_clean_json( + runtime: &mut GetterRuntime, + request_json: &str, +) -> Result { + let request: CleanTasksRequest = parse_request(request_json)?; + tasks_json(runtime.clean_tasks(request.mode()?)) +} + +fn parse_request(request_json: &str) -> Result +where + T: for<'de> Deserialize<'de>, +{ + serde_json::from_str(request_json) + .map_err(|source| RuntimeOperationError::InvalidRequest(source.to_string())) +} + +fn issued_action_json(action: IssuedAction) -> Value { + json!({ + "action_id": action.action_id, + "package_id": action.package_id, + }) +} + +fn task_json(task: TaskSnapshot) -> Result { + serde_json::to_value(task) + .map_err(|source| RuntimeOperationError::Serialize(source.to_string())) +} + +fn tasks_json(tasks: Vec) -> Result { + Ok(json!({ "tasks": tasks })) +} + +#[derive(Debug, Deserialize)] +struct SubmitActionRequest { + action_id: String, +} + +#[derive(Debug, Deserialize)] +struct TaskIdRequest { + task_id: String, +} + +#[derive(Debug, Default, Deserialize)] +struct TaskListRequest { + #[serde(default)] + active: bool, + #[serde(default)] + package_id: Option, +} + +#[derive(Debug, Deserialize)] +struct DownloadProgressRequest { + task_id: String, + current_bits: u64, + #[serde(default)] + total_bits: Option, +} + +#[derive(Debug, Deserialize)] +struct UserResultRequest { + task_id: String, + result: UserResult, + #[serde(default)] + reason: Option, +} + +#[derive(Debug, Default, Deserialize)] +struct CleanTasksRequest { + #[serde(default)] + mode: Option, +} + +impl CleanTasksRequest { + fn mode(self) -> Result { + match self.mode.as_deref().unwrap_or("default") { + "default" => Ok(TaskCleanMode::Default), + "failed" => Ok(TaskCleanMode::Failed), + "all_inactive" => Ok(TaskCleanMode::AllInactive), + other => Err(RuntimeOperationError::InvalidRequest(format!( + "unsupported task clean mode '{other}'" + ))), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use getter_core::{runtime::PackageVersionLuaObject, runtime::RuntimeTaskStatus, UpdateAction}; + + #[test] + fn json_submit_and_user_result_round_trip_consumes_action() { + let mut runtime = GetterRuntime::new(); + let issued = issue_action(&mut runtime, plan("android/org.fdroid.fdroid")); + let action_id = issued["action_id"].as_str().unwrap(); + + let submitted = + submit_action_json(&mut runtime, &json!({ "action_id": action_id }).to_string()) + .unwrap(); + + assert_eq!(submitted["status"], "queued"); + let error = + submit_action_json(&mut runtime, &json!({ "action_id": action_id }).to_string()) + .unwrap_err(); + assert_eq!(error.code(), "action.not_found"); + + let task_id = submitted["task_id"].as_str().unwrap(); + task_start_json(&mut runtime, &json!({ "task_id": task_id }).to_string()).unwrap(); + task_complete_download_json(&mut runtime, &json!({ "task_id": task_id }).to_string()) + .unwrap(); + let completed = task_user_result_json( + &mut runtime, + &json!({ "task_id": task_id, "result": "accepted" }).to_string(), + ) + .unwrap(); + + assert_eq!(completed["status"], "completed"); + assert_eq!( + runtime.task(task_id).unwrap().status, + RuntimeTaskStatus::Completed + ); + } + + #[test] + fn json_list_filters_and_clean_modes_follow_runtime_semantics() { + let mut runtime = GetterRuntime::new(); + let fdroid = submit_plan(&mut runtime, plan("android/org.fdroid.fdroid")); + let termux = submit_plan(&mut runtime, plan("android/com.termux")); + runtime.cancel_task(&fdroid).unwrap(); + runtime.start_task(&termux).unwrap(); + + let active = task_list_json(&runtime, &json!({ "active": true }).to_string()).unwrap(); + assert_eq!(active["tasks"].as_array().unwrap().len(), 1); + assert_eq!(active["tasks"][0]["task_id"], termux); + + let package = task_list_json( + &runtime, + &json!({ "package_id": "android/org.fdroid.fdroid" }).to_string(), + ) + .unwrap(); + assert_eq!(package["tasks"].as_array().unwrap().len(), 1); + assert_eq!(package["tasks"][0]["task_id"], fdroid); + + let removed = task_clean_json(&mut runtime, "{}").unwrap(); + assert_eq!(removed["tasks"].as_array().unwrap().len(), 1); + assert_eq!(runtime.task(&fdroid).unwrap_err().code(), "task.not_found"); + assert_eq!( + runtime.task(&termux).unwrap().status, + RuntimeTaskStatus::Running + ); + } + + #[test] + fn json_clean_rejects_unknown_mode_without_mutating_tasks() { + let mut runtime = GetterRuntime::new(); + let task_id = submit_plan(&mut runtime, plan_without_install("generic/example")); + runtime.start_task(&task_id).unwrap(); + runtime.complete_download(&task_id).unwrap(); + + let error = + task_clean_json(&mut runtime, &json!({ "mode": "all" }).to_string()).unwrap_err(); + + assert_eq!(error.code(), "runtime.invalid_request"); + assert_eq!( + runtime.task(&task_id).unwrap().status, + RuntimeTaskStatus::Completed + ); + } + + fn submit_plan(runtime: &mut GetterRuntime, plan: SealedActionPlan) -> String { + let action = runtime.issue_action(plan); + runtime.submit_action(&action.action_id).unwrap().task_id + } + + fn plan(package_id: &str) -> SealedActionPlan { + SealedActionPlan { + package_id: package_id.parse().unwrap(), + actions: vec![ + UpdateAction::Download { + url: "https://example.invalid/app.apk".to_owned(), + file_name: "app.apk".to_owned(), + }, + UpdateAction::Install { + installer: "android_package".to_owned(), + file: "app.apk".to_owned(), + }, + ], + lua_object: lua_object(package_id), + } + } + + fn plan_without_install(package_id: &str) -> SealedActionPlan { + SealedActionPlan { + package_id: package_id.parse().unwrap(), + actions: vec![UpdateAction::Download { + url: "https://example.invalid/archive.zip".to_owned(), + file_name: "archive.zip".to_owned(), + }], + lua_object: lua_object(package_id), + } + } + + fn lua_object(package_id: &str) -> PackageVersionLuaObject { + PackageVersionLuaObject { + object_id: format!("lua:{package_id}"), + dependency_digest: "sha256:test".to_owned(), + } + } +} From c4f2c281611d64ba4af41ceafd76142246ae847c Mon Sep 17 00:00:00 2001 From: Xiangzhe Date: Thu, 25 Jun 2026 11:09:30 +0800 Subject: [PATCH 21/52] feat(operations): issue actions from update checks --- crates/getter-operations/src/runtime.rs | 118 +++++++++++++++++++++++- 1 file changed, 115 insertions(+), 3 deletions(-) diff --git a/crates/getter-operations/src/runtime.rs b/crates/getter-operations/src/runtime.rs index 0d6e11d..5c44bff 100644 --- a/crates/getter-operations/src/runtime.rs +++ b/crates/getter-operations/src/runtime.rs @@ -8,9 +8,10 @@ use getter_core::{ runtime::{ - GetterRuntime, IssuedAction, RuntimeError, SealedActionPlan, TaskCleanMode, TaskSnapshot, - UserResult, + GetterRuntime, IssuedAction, PackageVersionLuaObject, RuntimeError, SealedActionPlan, + TaskCleanMode, TaskSnapshot, UserResult, }, + update::{run_offline_update_check, OfflineUpdateCheckError, OfflineUpdateCheckFixture}, PackageId, }; use serde::Deserialize; @@ -22,6 +23,8 @@ pub enum RuntimeOperationError { InvalidRequest(String), #[error("runtime error: {0}")] Runtime(#[from] RuntimeError), + #[error("update check failed: {0}")] + UpdateCheck(#[from] OfflineUpdateCheckError), #[error("runtime response serialization failed: {0}")] Serialize(String), } @@ -31,6 +34,7 @@ impl RuntimeOperationError { match self { Self::InvalidRequest(_) => "runtime.invalid_request", Self::Runtime(error) => error.code(), + Self::UpdateCheck(_) => "update.check_error", Self::Serialize(_) => "runtime.serialize_error", } } @@ -39,6 +43,7 @@ impl RuntimeOperationError { match self { Self::InvalidRequest(_) => "Getter runtime request is invalid", Self::Runtime(_) => "Getter runtime operation failed", + Self::UpdateCheck(_) => "Getter update check failed", Self::Serialize(_) => "Getter runtime response serialization failed", } } @@ -47,6 +52,7 @@ impl RuntimeOperationError { match self { Self::InvalidRequest(detail) | Self::Serialize(detail) => Some(detail.clone()), Self::Runtime(error) => Some(error.to_string()), + Self::UpdateCheck(error) => Some(error.to_string()), } } } @@ -55,6 +61,34 @@ pub fn issue_action(runtime: &mut GetterRuntime, plan: SealedActionPlan) -> Valu issued_action_json(runtime.issue_action(plan)) } +pub fn issue_action_from_offline_update_check_json( + runtime: &mut GetterRuntime, + request_json: &str, +) -> Result { + let request: OfflineUpdateActionRequest = parse_request(request_json)?; + let update = run_offline_update_check(request.fixture)?; + let action = if update.actions.is_empty() { + None + } else { + Some( + runtime.issue_action(SealedActionPlan { + package_id: update.package_id.clone(), + actions: update.actions.clone(), + lua_object: PackageVersionLuaObject { + object_id: format!("offline-update:{}", update.package_id), + dependency_digest: request + .dependency_digest + .unwrap_or_else(|| format!("offline-update:{}", update.package_id)), + }, + }), + ) + }; + Ok(json!({ + "update": update, + "action": action.map(issued_action_json), + })) +} + pub fn submit_action_json( runtime: &mut GetterRuntime, request_json: &str, @@ -194,6 +228,13 @@ fn tasks_json(tasks: Vec) -> Result Ok(json!({ "tasks": tasks })) } +#[derive(Debug, Deserialize)] +struct OfflineUpdateActionRequest { + fixture: OfflineUpdateCheckFixture, + #[serde(default)] + dependency_digest: Option, +} + #[derive(Debug, Deserialize)] struct SubmitActionRequest { action_id: String, @@ -250,7 +291,51 @@ impl CleanTasksRequest { #[cfg(test)] mod tests { use super::*; - use getter_core::{runtime::PackageVersionLuaObject, runtime::RuntimeTaskStatus, UpdateAction}; + use getter_core::{ + runtime::RuntimeTaskStatus, update::OFFLINE_UPDATE_CHECK_FORMAT, + update::OFFLINE_UPDATE_CHECK_VERSION, UpdateAction, UpdateArtifact, UpdateCandidate, + }; + + #[test] + fn offline_update_check_issues_getter_owned_action_id() { + let mut runtime = GetterRuntime::new(); + + let issued = issue_action_from_offline_update_check_json( + &mut runtime, + &json!({ + "fixture": update_fixture("android/org.fdroid.fdroid", Some("1.0.0"), vec!["1.2.0"]), + "dependency_digest": "sha256:fixture" + }) + .to_string(), + ) + .unwrap(); + + assert_eq!(issued["update"]["status"], "update_available"); + let action_id = issued["action"]["action_id"].as_str().unwrap(); + let submitted = + submit_action_json(&mut runtime, &json!({ "action_id": action_id }).to_string()) + .unwrap(); + assert_eq!(submitted["package_id"], "android/org.fdroid.fdroid"); + assert_eq!(submitted["status"], "queued"); + } + + #[test] + fn offline_update_check_without_update_does_not_issue_action() { + let mut runtime = GetterRuntime::new(); + + let issued = issue_action_from_offline_update_check_json( + &mut runtime, + &json!({ + "fixture": update_fixture("android/org.fdroid.fdroid", Some("1.2.0"), vec!["1.2.0"]), + }) + .to_string(), + ) + .unwrap(); + + assert_eq!(issued["update"]["status"], "up_to_date"); + assert!(issued["action"].is_null()); + assert_eq!(runtime.tasks().len(), 0); + } #[test] fn json_submit_and_user_result_round_trip_consumes_action() { @@ -336,6 +421,33 @@ mod tests { runtime.submit_action(&action.action_id).unwrap().task_id } + fn update_fixture( + package_id: &str, + installed_version: Option<&str>, + versions: Vec<&str>, + ) -> OfflineUpdateCheckFixture { + OfflineUpdateCheckFixture { + format: OFFLINE_UPDATE_CHECK_FORMAT.to_owned(), + version: OFFLINE_UPDATE_CHECK_VERSION, + package_id: package_id.parse().unwrap(), + installed_version: installed_version.map(str::to_owned), + ignored_version: None, + candidates: versions + .into_iter() + .map(|version| UpdateCandidate { + version: version.to_owned(), + channel: None, + source: None, + artifacts: vec![UpdateArtifact { + name: "app.apk".to_owned(), + url: "https://example.invalid/app.apk".to_owned(), + file_name: Some("app.apk".to_owned()), + }], + }) + .collect(), + } + } + fn plan(package_id: &str) -> SealedActionPlan { SealedActionPlan { package_id: package_id.parse().unwrap(), From 604a5f6a49e26ca895f26dceae655ffb404e5033 Mon Sep 17 00:00:00 2001 From: Xiangzhe Date: Thu, 25 Jun 2026 11:20:00 +0800 Subject: [PATCH 22/52] feat(core): accept static update candidates --- crates/getter-core/src/lib.rs | 7 +++ crates/getter-core/src/lua.rs | 92 ++++++++++++++++++++++++++++++++++- 2 files changed, 98 insertions(+), 1 deletion(-) diff --git a/crates/getter-core/src/lib.rs b/crates/getter-core/src/lib.rs index 96cdb95..b1d759e 100644 --- a/crates/getter-core/src/lib.rs +++ b/crates/getter-core/src/lib.rs @@ -251,6 +251,13 @@ pub struct ResolvedPackage { pub permissions: PackagePermissions, #[serde(default)] pub source_priority: Vec, + /// Static/offline update candidates declared by package Lua. + /// + /// This is a first mock-provider bridge toward ADR-0011 action issuance: + /// getter still owns selection and action planning, while live provider + /// execution remains a later runtime slice. + #[serde(default)] + pub updates: Vec, } /// Installed target matched by a package, such as an Android package name. diff --git a/crates/getter-core/src/lua.rs b/crates/getter-core/src/lua.rs index a31107d..79bfc61 100644 --- a/crates/getter-core/src/lua.rs +++ b/crates/getter-core/src/lua.rs @@ -1,7 +1,10 @@ //! Minimal Lua package-file evaluation and Rust validation boundary. use crate::repository::RepositoryLayout; -use crate::{InstalledTarget, PackageId, PackagePermissions, ResolvedPackage}; +use crate::{ + InstalledTarget, PackageId, PackagePermissions, ResolvedPackage, UpdateArtifact, + UpdateCandidate, +}; use mlua::{Lua, Table, Value}; use serde_json::{Map, Number, Value as JsonValue}; use std::fs; @@ -331,6 +334,7 @@ fn validate_package_json( let permissions = parse_permissions(path, object.get("permissions"))?; let source_priority = parse_string_array(path, "source_priority", object.get("source_priority"))?; + let updates = parse_update_candidates(path, object.get("updates"))?; Ok(ResolvedPackage { id, @@ -339,6 +343,7 @@ fn validate_package_json( installed, permissions, source_priority, + updates, }) } @@ -394,6 +399,69 @@ fn parse_installed_targets( .collect() } +fn parse_update_candidates( + path: &Path, + value: Option<&JsonValue>, +) -> Result, LuaPackageError> { + let Some(value) = value else { + return Ok(Vec::new()); + }; + let array = value.as_array().ok_or_else(|| LuaPackageError::Schema { + path: path.to_path_buf(), + message: "field 'updates' must be an array".to_owned(), + })?; + array + .iter() + .map(|item| { + let object = item.as_object().ok_or_else(|| LuaPackageError::Schema { + path: path.to_path_buf(), + message: "updates entries must be objects".to_owned(), + })?; + let artifacts = parse_update_artifacts(path, object.get("artifacts"))?; + Ok(UpdateCandidate { + version: required_string(path, object, "version")?.to_owned(), + channel: optional_string(object, "channel"), + source: optional_string(object, "source"), + artifacts, + }) + }) + .collect() +} + +fn parse_update_artifacts( + path: &Path, + value: Option<&JsonValue>, +) -> Result, LuaPackageError> { + let Some(value) = value else { + return Ok(Vec::new()); + }; + let array = value.as_array().ok_or_else(|| LuaPackageError::Schema { + path: path.to_path_buf(), + message: "field 'artifacts' must be an array".to_owned(), + })?; + array + .iter() + .map(|item| { + let object = item.as_object().ok_or_else(|| LuaPackageError::Schema { + path: path.to_path_buf(), + message: "artifact entries must be objects".to_owned(), + })?; + Ok(UpdateArtifact { + name: required_string(path, object, "name")?.to_owned(), + url: required_string(path, object, "url")?.to_owned(), + file_name: optional_string(object, "file_name"), + }) + }) + .collect() +} + +fn optional_string(object: &Map, field: &str) -> Option { + object + .get(field) + .and_then(JsonValue::as_str) + .map(str::to_owned) +} + fn parse_permissions( path: &Path, value: Option<&JsonValue>, @@ -480,6 +548,20 @@ return package_def { }, permissions = { free_network = true }, source_priority = { "github", "fdroid" }, + updates = { + { + version = "1.2.0", + channel = "stable", + source = "fixture", + artifacts = { + { + name = "app.apk", + url = "https://example.invalid/app.apk", + file_name = "fdroid.apk", + }, + }, + }, + }, } "#, ) @@ -497,6 +579,14 @@ return package_def { ); assert!(package.permissions.free_network); assert_eq!(package.source_priority, vec!["github", "fdroid"]); + assert_eq!(package.updates.len(), 1); + assert_eq!(package.updates[0].version, "1.2.0"); + assert_eq!(package.updates[0].channel.as_deref(), Some("stable")); + assert_eq!(package.updates[0].source.as_deref(), Some("fixture")); + assert_eq!( + package.updates[0].artifacts[0].file_name.as_deref(), + Some("fdroid.apk") + ); } #[test] From 1738c03f58450095bc075fb7adf75e4a57262854 Mon Sep 17 00:00:00 2001 From: Xiangzhe Date: Thu, 25 Jun 2026 11:47:24 +0800 Subject: [PATCH 23/52] feat(operations): issue actions from registered packages --- Cargo.lock | 1 + Cargo.toml | 2 +- crates/getter-core/Cargo.toml | 2 +- crates/getter-core/src/lua.rs | 11 +- crates/getter-operations/Cargo.toml | 7 + crates/getter-operations/src/runtime.rs | 258 ++++++++++++++++++++++++ 6 files changed, 277 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index fce5fc5..e033e93 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -568,6 +568,7 @@ dependencies = [ "getter-storage", "serde", "serde_json", + "tempfile", "thiserror 1.0.69", ] diff --git a/Cargo.toml b/Cargo.toml index 9347d01..ba37ca3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -24,7 +24,7 @@ edition = "2021" [features] default = ["cli", "domain"] domain = ["dep:getter-core", "dep:getter-storage", "dep:getter-operations"] -lua = ["getter-core/lua"] +lua = ["getter-core/lua", "getter-operations/lua"] cli = ["dep:getter-cli"] rustls-platform-verifier = ["dep:rustls-platform-verifier"] rustls-platform-verifier-android = ["rustls-platform-verifier", "rustls-platform-verifier/jni"] diff --git a/crates/getter-core/Cargo.toml b/crates/getter-core/Cargo.toml index 93325e9..187fcf6 100644 --- a/crates/getter-core/Cargo.toml +++ b/crates/getter-core/Cargo.toml @@ -8,7 +8,7 @@ default = ["lua"] lua = ["dep:mlua"] [dependencies] -mlua = { version = "0.10", features = ["lua54", "vendored"], optional = true } +mlua = { version = "0.10", features = ["luajit", "vendored"], optional = true } serde = { version = "1", features = ["derive"] } serde_json = "1" thiserror = "1" diff --git a/crates/getter-core/src/lua.rs b/crates/getter-core/src/lua.rs index 79bfc61..d461539 100644 --- a/crates/getter-core/src/lua.rs +++ b/crates/getter-core/src/lua.rs @@ -106,7 +106,7 @@ fn configure_package_path(lua: &Lua, repository: &RepositoryLayout) -> mlua::Res } fn disable_native_module_searchers(package: &Table) -> mlua::Result<()> { - let searchers: Table = package.get("searchers")?; + let searchers = package_searchers(package)?; let len = searchers.raw_len(); for index in 4..=len { searchers.raw_set(index, Value::Nil)?; @@ -128,7 +128,7 @@ fn remove_unsafe_globals(lua: &Lua) -> mlua::Result<()> { } fn install_lib_prefix_searcher(lua: &Lua, package: &Table, lib_dir: PathBuf) -> mlua::Result<()> { - let searchers: Table = package.get("searchers")?; + let searchers = package_searchers(package)?; let searcher = lua.create_function(move |lua, module: String| { let Some(module) = module.strip_prefix("lib.") else { return lua @@ -171,6 +171,13 @@ fn install_lib_prefix_searcher(lua: &Lua, package: &Table, lib_dir: PathBuf) -> searchers.raw_set(2, searcher) } +fn package_searchers(package: &Table) -> mlua::Result { + match package.get::("searchers")? { + Value::Table(searchers) => Ok(searchers), + _ => package.get("loaders"), + } +} + fn module_to_relative_path(module: &str) -> Option { let mut path = PathBuf::new(); for part in module.split('.') { diff --git a/crates/getter-operations/Cargo.toml b/crates/getter-operations/Cargo.toml index a44e0b6..a36ba74 100644 --- a/crates/getter-operations/Cargo.toml +++ b/crates/getter-operations/Cargo.toml @@ -3,9 +3,16 @@ name = "getter-operations" version.workspace = true edition.workspace = true +[features] +default = [] +lua = ["getter-core/lua"] + [dependencies] getter-core = { path = "../getter-core", default-features = false } getter-storage = { path = "../getter-storage" } serde = { version = "1", features = ["derive"] } serde_json = "1" thiserror = "1" + +[dev-dependencies] +tempfile = "3" diff --git a/crates/getter-operations/src/runtime.rs b/crates/getter-operations/src/runtime.rs index 5c44bff..9887f15 100644 --- a/crates/getter-operations/src/runtime.rs +++ b/crates/getter-operations/src/runtime.rs @@ -6,6 +6,11 @@ //! persisted task state or duplicating task/control semantics in Flutter or //! Android glue. +#[cfg(feature = "lua")] +use getter_core::{ + lua::evaluate_package_file, + repository::{package_cache_key, RepositoryLayout, RepositoryLoadError}, +}; use getter_core::{ runtime::{ GetterRuntime, IssuedAction, PackageVersionLuaObject, RuntimeError, SealedActionPlan, @@ -14,8 +19,14 @@ use getter_core::{ update::{run_offline_update_check, OfflineUpdateCheckError, OfflineUpdateCheckFixture}, PackageId, }; +#[cfg(feature = "lua")] +use getter_core::{update::check_updates_offline, update::UpdateSelectionPolicy, RepositoryId}; +#[cfg(feature = "lua")] +use getter_storage::{MainDb, StorageError}; use serde::Deserialize; use serde_json::{json, Value}; +#[cfg(feature = "lua")] +use std::path::PathBuf; #[derive(Debug, thiserror::Error)] pub enum RuntimeOperationError { @@ -25,6 +36,15 @@ pub enum RuntimeOperationError { Runtime(#[from] RuntimeError), #[error("update check failed: {0}")] UpdateCheck(#[from] OfflineUpdateCheckError), + #[cfg(feature = "lua")] + #[error("storage operation failed: {0}")] + Storage(#[from] StorageError), + #[cfg(feature = "lua")] + #[error("repository operation failed: {0}")] + Repository(#[from] RepositoryLoadError), + #[cfg(feature = "lua")] + #[error("package evaluation failed: {0}")] + PackageEval(String), #[error("runtime response serialization failed: {0}")] Serialize(String), } @@ -35,6 +55,12 @@ impl RuntimeOperationError { Self::InvalidRequest(_) => "runtime.invalid_request", Self::Runtime(error) => error.code(), Self::UpdateCheck(_) => "update.check_error", + #[cfg(feature = "lua")] + Self::Storage(_) => "storage.error", + #[cfg(feature = "lua")] + Self::Repository(_) => "repository.error", + #[cfg(feature = "lua")] + Self::PackageEval(_) => "package.eval_error", Self::Serialize(_) => "runtime.serialize_error", } } @@ -44,6 +70,12 @@ impl RuntimeOperationError { Self::InvalidRequest(_) => "Getter runtime request is invalid", Self::Runtime(_) => "Getter runtime operation failed", Self::UpdateCheck(_) => "Getter update check failed", + #[cfg(feature = "lua")] + Self::Storage(_) => "Getter storage operation failed", + #[cfg(feature = "lua")] + Self::Repository(_) => "Getter repository operation failed", + #[cfg(feature = "lua")] + Self::PackageEval(_) => "Getter package evaluation failed", Self::Serialize(_) => "Getter runtime response serialization failed", } } @@ -53,6 +85,12 @@ impl RuntimeOperationError { Self::InvalidRequest(detail) | Self::Serialize(detail) => Some(detail.clone()), Self::Runtime(error) => Some(error.to_string()), Self::UpdateCheck(error) => Some(error.to_string()), + #[cfg(feature = "lua")] + Self::Storage(error) => Some(error.to_string()), + #[cfg(feature = "lua")] + Self::Repository(error) => Some(error.to_string()), + #[cfg(feature = "lua")] + Self::PackageEval(detail) => Some(detail.clone()), } } } @@ -89,6 +127,41 @@ pub fn issue_action_from_offline_update_check_json( })) } +#[cfg(feature = "lua")] +pub fn issue_action_from_registered_package_json( + runtime: &mut GetterRuntime, + db: &MainDb, + request_json: &str, +) -> Result { + let request: RegisteredPackageUpdateActionRequest = parse_request(request_json)?; + let (package, dependency_digest) = evaluate_registered_package(db, &request)?; + let update = check_updates_offline( + package.id.clone(), + request.installed_version, + package.updates.clone(), + UpdateSelectionPolicy { + ignored_version: request.ignored_version, + }, + )?; + let action = if update.actions.is_empty() { + None + } else { + Some(runtime.issue_action(SealedActionPlan { + package_id: update.package_id.clone(), + actions: update.actions.clone(), + lua_object: PackageVersionLuaObject { + object_id: format!("package-update:{}", update.package_id), + dependency_digest, + }, + })) + }; + Ok(json!({ + "package": package, + "update": update, + "action": action.map(issued_action_json), + })) +} + pub fn submit_action_json( runtime: &mut GetterRuntime, request_json: &str, @@ -235,6 +308,18 @@ struct OfflineUpdateActionRequest { dependency_digest: Option, } +#[cfg(feature = "lua")] +#[derive(Debug, Deserialize)] +struct RegisteredPackageUpdateActionRequest { + package_id: PackageId, + #[serde(default)] + repository_id: Option, + #[serde(default)] + installed_version: Option, + #[serde(default)] + ignored_version: Option, +} + #[derive(Debug, Deserialize)] struct SubmitActionRequest { action_id: String, @@ -275,6 +360,60 @@ struct CleanTasksRequest { mode: Option, } +#[cfg(feature = "lua")] +fn evaluate_registered_package( + db: &MainDb, + request: &RegisteredPackageUpdateActionRequest, +) -> Result<(getter_core::ResolvedPackage, String), RuntimeOperationError> { + let repositories = db.repositories()?; + let mut missing_path = None; + for repository in repositories { + if request + .repository_id + .as_ref() + .is_some_and(|requested| requested != &repository.id) + { + continue; + } + let Some(root) = repository.path.as_ref() else { + missing_path = Some(repository.id.to_string()); + continue; + }; + let root = PathBuf::from(root); + let layout = RepositoryLayout::load(&root)?; + let Some(package_file) = layout.package_file(&request.package_id) else { + continue; + }; + let package = evaluate_package_file(&layout, &package_file.path) + .map_err(|source| RuntimeOperationError::PackageEval(source.to_string()))?; + let cache_key = package_cache_key(&layout, package_file)?; + let dependency_digest = format!( + "repo:{}:package:{}:hash:{}", + cache_key.repository_id, cache_key.package_id, cache_key.package_file_hash + ); + return Ok((package, dependency_digest)); + } + let detail = if let Some(repository_id) = request.repository_id.as_ref() { + format!( + "package '{}' was not found in registered repository '{}'{}", + request.package_id, + repository_id, + missing_path + .map(|id| format!("; repository '{id}' has no path")) + .unwrap_or_default() + ) + } else { + format!( + "package '{}' was not found in any registered repository{}", + request.package_id, + missing_path + .map(|id| format!("; repository '{id}' has no path")) + .unwrap_or_default() + ) + }; + Err(RuntimeOperationError::PackageEval(detail)) +} + impl CleanTasksRequest { fn mode(self) -> Result { match self.mode.as_deref().unwrap_or("default") { @@ -291,10 +430,91 @@ impl CleanTasksRequest { #[cfg(test)] mod tests { use super::*; + #[cfg(feature = "lua")] + use getter_core::repository::{RepositoryMetadata, REPO_API_VERSION_V1}; + #[cfg(feature = "lua")] + use getter_core::RepositoryPriority; use getter_core::{ runtime::RuntimeTaskStatus, update::OFFLINE_UPDATE_CHECK_FORMAT, update::OFFLINE_UPDATE_CHECK_VERSION, UpdateAction, UpdateArtifact, UpdateCandidate, }; + #[cfg(feature = "lua")] + use std::fs; + + #[cfg(feature = "lua")] + #[test] + fn registered_package_update_check_issues_action_from_lua_static_updates() { + let temp = tempfile::tempdir().unwrap(); + let repo_root = temp.path().join("repo"); + write_static_update_repo(&repo_root); + let db = MainDb::open_in_memory().unwrap(); + db.upsert_repository( + &RepositoryMetadata { + id: "official".parse().unwrap(), + name: "Official".to_owned(), + priority: RepositoryPriority::new(0), + api_version: REPO_API_VERSION_V1.to_owned(), + }, + Some(&repo_root), + None, + ) + .unwrap(); + let mut runtime = GetterRuntime::new(); + + let issued = issue_action_from_registered_package_json( + &mut runtime, + &db, + &json!({ + "package_id": "android/org.fdroid.fdroid", + "installed_version": "1.0.0" + }) + .to_string(), + ) + .unwrap(); + + assert_eq!(issued["package"]["repository"], "official"); + assert_eq!(issued["update"]["status"], "update_available"); + let action_id = issued["action"]["action_id"].as_str().unwrap(); + let submitted = + submit_action_json(&mut runtime, &json!({ "action_id": action_id }).to_string()) + .unwrap(); + assert_eq!(submitted["package_id"], "android/org.fdroid.fdroid"); + } + + #[cfg(feature = "lua")] + #[test] + fn registered_package_update_check_without_update_does_not_issue_action() { + let temp = tempfile::tempdir().unwrap(); + let repo_root = temp.path().join("repo"); + write_static_update_repo(&repo_root); + let db = MainDb::open_in_memory().unwrap(); + db.upsert_repository( + &RepositoryMetadata { + id: "official".parse().unwrap(), + name: "Official".to_owned(), + priority: RepositoryPriority::new(0), + api_version: REPO_API_VERSION_V1.to_owned(), + }, + Some(&repo_root), + None, + ) + .unwrap(); + let mut runtime = GetterRuntime::new(); + + let issued = issue_action_from_registered_package_json( + &mut runtime, + &db, + &json!({ + "package_id": "android/org.fdroid.fdroid", + "installed_version": "1.2.0" + }) + .to_string(), + ) + .unwrap(); + + assert_eq!(issued["update"]["status"], "up_to_date"); + assert!(issued["action"].is_null()); + } #[test] fn offline_update_check_issues_getter_owned_action_id() { @@ -421,6 +641,44 @@ mod tests { runtime.submit_action(&action.action_id).unwrap().task_id } + #[cfg(feature = "lua")] + fn write_static_update_repo(root: &std::path::Path) { + fs::create_dir_all(root.join("packages/android")).unwrap(); + fs::create_dir(root.join("lib")).unwrap(); + fs::create_dir(root.join("templates")).unwrap(); + fs::write( + root.join("repo.toml"), + r#"id = "official" +name = "Official" +priority = 0 +api_version = "getter.repo.v1" +"#, + ) + .unwrap(); + fs::write( + root.join("packages/android/org.fdroid.fdroid.lua"), + r#" +return package_def { + id = "android/org.fdroid.fdroid", + name = "F-Droid", + updates = { + { + version = "1.2.0", + artifacts = { + { + name = "app.apk", + url = "https://example.invalid/app.apk", + file_name = "app.apk", + }, + }, + }, + }, +} +"#, + ) + .unwrap(); + } + fn update_fixture( package_id: &str, installed_version: Option<&str>, From 9ae7e5680188b2ef87bc052d2e55d5375116c0ce Mon Sep 17 00:00:00 2001 From: Xiangzhe Date: Thu, 25 Jun 2026 12:43:02 +0800 Subject: [PATCH 24/52] feat(storage): rename ignored version to pin version --- crates/getter-cli/src/lib.rs | 155 ++++++++++++++++-- crates/getter-cli/tests/bdd_cli.rs | 61 +++++-- .../tests/features/cli/update_check.feature | 12 +- .../tests/features/cli/version_pin.feature | 12 ++ crates/getter-core/src/update.rs | 110 +++++-------- crates/getter-operations/src/legacy_room.rs | 27 ++- crates/getter-operations/src/runtime.rs | 8 +- crates/getter-storage/src/legacy_room.rs | 7 +- crates/getter-storage/src/lib.rs | 142 +++++++++++++--- 9 files changed, 398 insertions(+), 136 deletions(-) create mode 100644 crates/getter-cli/tests/features/cli/version_pin.feature diff --git a/crates/getter-cli/src/lib.rs b/crates/getter-cli/src/lib.rs index 3562bf1..ed72425 100644 --- a/crates/getter-cli/src/lib.rs +++ b/crates/getter-cli/src/lib.rs @@ -64,6 +64,13 @@ pub enum CliCommand { repo_id: Option, }, StorageValidate, + VersionPin { + package_id: PackageId, + version: String, + }, + VersionUnpin { + package_id: PackageId, + }, UpdateCheck { fixture: PathBuf, }, @@ -380,6 +387,17 @@ where [domain, command] if domain == "storage" && command == "validate" => { CliCommand::StorageValidate } + [domain, command, package_id, version] if domain == "version" && command == "pin" => { + CliCommand::VersionPin { + package_id: parse_package_id(package_id)?, + version: version.clone(), + } + } + [domain, command, package_id] if domain == "version" && command == "unpin" => { + CliCommand::VersionUnpin { + package_id: parse_package_id(package_id)?, + } + } [domain, command, flag, fixture] if domain == "update" && command == "check" && flag == "--fixture" => { @@ -567,6 +585,19 @@ fn execute(invocation: CliInvocation) -> Result { "cache_db": cache_db_path(&invocation.data_dir), })) } + CliCommand::VersionPin { + package_id, + version, + } => { + let db = open_main_db(&invocation.data_dir)?; + let package = db.set_tracked_package_pin_version(&package_id, Some(&version))?; + Ok(json!({ "package": tracked_package_json(package) })) + } + CliCommand::VersionUnpin { package_id } => { + let db = open_main_db(&invocation.data_dir)?; + let package = db.set_tracked_package_pin_version(&package_id, None)?; + Ok(json!({ "package": tracked_package_json(package) })) + } CliCommand::UpdateCheck { fixture } => { open_initialized_storage(&invocation.data_dir)?; let fixture = read_update_check_fixture(&fixture)?; @@ -962,19 +993,18 @@ fn package_json(package: getter_core::ResolvedPackage) -> Result) -> Vec { - packages - .into_iter() - .map(|package| { - json!({ - "id": package.package_id.to_string(), - "enabled": package.enabled, - "favorite": package.favorite, - "ignored_version": package.ignored_version, - "repository_id": package.repository_id.map(|id| id.to_string()), - "package_resolution": package.package_resolution.as_str(), - }) - }) - .collect() + packages.into_iter().map(tracked_package_json).collect() +} + +fn tracked_package_json(package: StoredTrackedPackage) -> Value { + json!({ + "id": package.package_id.to_string(), + "enabled": package.enabled, + "favorite": package.favorite, + "pin_version": package.pin_version, + "repository_id": package.repository_id.map(|id| id.to_string()), + "package_resolution": package.package_resolution.as_str(), + }) } fn import_legacy_room_bundle(db: &MainDb, bundle: &LegacyRoomBundle) -> Result<(), CliError> { @@ -988,7 +1018,7 @@ fn import_legacy_room_bundle(db: &MainDb, bundle: &LegacyRoomBundle) -> Result<( common_conversion_available: app.common_conversion_available, }, Some(&LegacyExtraAppRecord { - ignored_version: app.ignored_version.clone(), + ignored_version: app.pin_version.clone(), favorite: app.favorite, }), ) @@ -1000,6 +1030,7 @@ fn import_legacy_room_bundle(db: &MainDb, bundle: &LegacyRoomBundle) -> Result<( "ok": true, "source": "legacy-room-bundle", "imported_records": bundle.apps.len(), + "notices": [migration_pin_version_notice()], }) .to_string(); db.import_tracked_packages_with_migration_record( @@ -1013,6 +1044,13 @@ fn import_legacy_room_bundle(db: &MainDb, bundle: &LegacyRoomBundle) -> Result<( Ok(()) } +fn migration_pin_version_notice() -> Value { + json!({ + "code": "migration.renamed_ignored_version_to_pin_version", + "message": "Legacy ignored version state was preserved as pin_version", + }) +} + fn tracked_package_upsert( mapping: getter_storage::legacy_room::LegacyAppMapping, ) -> TrackedPackageUpsert { @@ -1020,7 +1058,7 @@ fn tracked_package_upsert( package_id: mapping.package_id, enabled: true, favorite: mapping.user_state.favorite, - ignored_version: mapping.user_state.ignored_version, + pin_version: mapping.user_state.pin_version, repository_id: None, package_resolution: stored_resolution(mapping.package_resolution), } @@ -1103,6 +1141,7 @@ fn create_migration_report_with_source_counts( imported_records, tracked_records, warnings, + notices: migration_report_notices(code), source_counts, }; let bytes = serde_json::to_vec_pretty(&report) @@ -1116,6 +1155,14 @@ fn report_file_name(code: &str) -> String { format!("{}.json", code.replace('.', "-")) } +fn migration_report_notices(code: &str) -> Vec { + if code == "migration.imported" { + vec![migration_pin_version_notice()] + } else { + Vec::new() + } +} + #[derive(Debug, Serialize)] struct MigrationReport<'a> { ok: bool, @@ -1126,6 +1173,8 @@ struct MigrationReport<'a> { imported_records: u64, tracked_records: u64, warnings: &'a [Value], + #[serde(skip_serializing_if = "Vec::is_empty")] + notices: Vec, #[serde(skip_serializing_if = "Option::is_none")] source_counts: Option<&'a Value>, } @@ -1165,7 +1214,7 @@ fn envelope_to_string(value: Value) -> String { } fn usage_text() -> String { - "Usage: getter --data-dir [--priority ]|repo eval |repo validate |package eval [--repo ]|storage validate|hub list|update check --fixture |task submit --request |task run |task list|task cancel |task events --after --limit |task install-result --status |autogen installed preview --inventory |autogen installed apply --preview (--accept-all|--accept ...)|autogen cleanup preview --inventory |autogen cleanup apply --preview (--accept-all|--accept ...)|legacy import-room-bundle |legacy import-room-db |legacy report-list>\n".to_owned() + "Usage: getter --data-dir [--priority ]|repo eval |repo validate |package eval [--repo ]|storage validate|version pin |version unpin |hub list|update check --fixture |task submit --request |task run |task list|task cancel |task events --after --limit |task install-result --status |autogen installed preview --inventory |autogen installed apply --preview (--accept-all|--accept ...)|autogen cleanup preview --inventory |autogen cleanup apply --preview (--accept-all|--accept ...)|legacy import-room-bundle |legacy import-room-db |legacy report-list>\n".to_owned() } #[derive(Debug, Deserialize)] @@ -1184,8 +1233,8 @@ struct LegacyBundleApp { official_package_available: bool, #[serde(default)] common_conversion_available: bool, - #[serde(default)] - ignored_version: Option, + #[serde(default, alias = "ignored_version")] + pin_version: Option, #[serde(default)] favorite: bool, } @@ -1218,6 +1267,8 @@ impl CliCommand { Self::RepoValidate { .. } => "repo validate", Self::PackageEval { .. } => "package eval", Self::StorageValidate => "storage validate", + Self::VersionPin { .. } => "version pin", + Self::VersionUnpin { .. } => "version unpin", Self::UpdateCheck { .. } => "update check", Self::TaskSubmit { .. } => "task submit", Self::TaskRun { .. } => "task run", @@ -1249,6 +1300,43 @@ mod tests { assert_eq!(invocation.command, CliCommand::AppList); } + #[test] + fn parses_version_pin_and_unpin_commands() { + let pin = parse_args([ + "getter", + "--data-dir", + "/tmp/ua-getter", + "version", + "pin", + "android/org.fdroid.fdroid", + "1.2.3", + ]) + .unwrap(); + assert_eq!( + pin.command, + CliCommand::VersionPin { + package_id: "android/org.fdroid.fdroid".parse().unwrap(), + version: "1.2.3".to_owned(), + } + ); + + let unpin = parse_args([ + "getter", + "--data-dir", + "/tmp/ua-getter", + "version", + "unpin", + "android/org.fdroid.fdroid", + ]) + .unwrap(); + assert_eq!( + unpin.command, + CliCommand::VersionUnpin { + package_id: "android/org.fdroid.fdroid".parse().unwrap(), + } + ); + } + #[test] fn run_init_creates_sqlite_database_files_and_json_envelope() { let temp = tempfile::tempdir().unwrap(); @@ -1269,6 +1357,37 @@ mod tests { assert_eq!(json["command"], "init"); } + #[test] + fn run_version_pin_and_unpin_mutates_tracked_package_pin_version() { + let temp = tempfile::tempdir().unwrap(); + let data_dir = temp.path().join("getter-data"); + + let pin = run([ + "getter".to_owned(), + "--data-dir".to_owned(), + data_dir.to_string_lossy().to_string(), + "version".to_owned(), + "pin".to_owned(), + "android/org.fdroid.fdroid".to_owned(), + "1.2.3".to_owned(), + ]); + assert_eq!(pin.exit_code, ExitCode::Success); + let pin_json: Value = serde_json::from_str(&pin.stdout).unwrap(); + assert_eq!(pin_json["data"]["package"]["pin_version"], "1.2.3"); + + let unpin = run([ + "getter".to_owned(), + "--data-dir".to_owned(), + data_dir.to_string_lossy().to_string(), + "version".to_owned(), + "unpin".to_owned(), + "android/org.fdroid.fdroid".to_owned(), + ]); + assert_eq!(unpin.exit_code, ExitCode::Success); + let unpin_json: Value = serde_json::from_str(&unpin.stdout).unwrap(); + assert!(unpin_json["data"]["package"]["pin_version"].is_null()); + } + #[test] fn malformed_bundle_report_does_not_create_imported_state() { let temp = tempfile::tempdir().unwrap(); diff --git a/crates/getter-cli/tests/bdd_cli.rs b/crates/getter-cli/tests/bdd_cli.rs index d0230ea..733827f 100644 --- a/crates/getter-cli/tests/bdd_cli.rs +++ b/crates/getter-cli/tests/bdd_cli.rs @@ -205,20 +205,20 @@ fn offline_update_fixture_with_installed_version( } #[given( - expr = "an offline update fixture for package {string} installed version {string} ignored version {string} with candidate versions {string}" + expr = "an offline update fixture for package {string} installed version {string} pin version {string} with candidate versions {string}" )] -fn offline_update_fixture_with_ignored_version( +fn offline_update_fixture_with_pin_version( world: &mut CliWorld, package_id: String, installed_version: String, - ignored_version: String, + pin_version: String, versions: String, ) { write_offline_update_fixture( world, package_id, Some(installed_version), - Some(ignored_version), + Some(pin_version), versions, ); } @@ -542,6 +542,26 @@ fn run_getter_update_check(world: &mut CliWorld) { world.json = None; } +#[when(expr = "I run getter version pin for package {string} version {string}")] +fn run_getter_version_pin(world: &mut CliWorld, package_id: String, version: String) { + let output = run_getter( + world, + ["version".to_owned(), "pin".to_owned(), package_id, version], + ); + world.output = Some(output); + world.json = None; +} + +#[when(expr = "I run getter version unpin for package {string}")] +fn run_getter_version_unpin(world: &mut CliWorld, package_id: String) { + let output = run_getter( + world, + ["version".to_owned(), "unpin".to_owned(), package_id], + ); + world.output = Some(output); + world.json = None; +} + #[when("I run getter task submit for that request")] fn run_getter_task_submit(world: &mut CliWorld) { let request = world.task_request.as_ref().expect("task request exists"); @@ -1074,6 +1094,18 @@ fn output_contains_named_package(world: &mut CliWorld, package_id: String, packa assert_eq!(json["data"]["package"]["name"], package_name); } +#[then(expr = "the pinned package version is {string}")] +fn pinned_package_version_is(world: &mut CliWorld, version: String) { + let json = current_json(world); + assert_eq!(json["data"]["package"]["pin_version"], version); +} + +#[then("the package is unpinned")] +fn package_is_unpinned(world: &mut CliWorld) { + let json = current_json(world); + assert!(json["data"]["package"]["pin_version"].is_null()); +} + #[then(expr = "the update check status is {string}")] fn update_check_status_is(world: &mut CliWorld, status: String) { let json = current_json(world); @@ -1343,6 +1375,7 @@ fn direct_migration_report_stays_sanitized(world: &mut CliWorld) { assert_eq!(report_json["source_counts"]["hub_rows"], 1); assert_eq!(report_json["source_counts"]["extra_hub_rows"], 1); assert_report_has_drop_warnings(&report_json); + assert_report_has_pin_version_notice(&report_json); } #[then("the output reports the legacy Room migration was already completed")] @@ -1398,6 +1431,7 @@ fn direct_migration_report_list_stays_sanitized(world: &mut CliWorld) { assert_eq!(imported["source_counts"]["hub_rows"], 1); assert_eq!(imported["source_counts"]["extra_hub_rows"], 1); assert_report_has_drop_warnings(imported); + assert_report_has_pin_version_notice(imported); } #[then(expr = "the app list contains imported package {string}")] @@ -1411,7 +1445,7 @@ fn app_list_contains_imported_package(world: &mut CliWorld, package_id: String) .find(|app| app["id"].as_str() == Some(package_id.as_str())) .unwrap_or_else(|| panic!("app list should contain {package_id}: {apps:?}")); assert_eq!(app["favorite"], true); - assert_eq!(app["ignored_version"], "1.20.0"); + assert_eq!(app["pin_version"], "1.20.0"); assert_eq!(app["package_resolution"], "official_repository_package"); world.output = Some(output); world.json = Some(json); @@ -1428,7 +1462,7 @@ fn app_list_contains_directly_imported_package(world: &mut CliWorld, package_id: .find(|app| app["id"].as_str() == Some(package_id.as_str())) .unwrap_or_else(|| panic!("app list should contain {package_id}: {apps:?}")); assert_eq!(app["favorite"], true); - assert_eq!(app["ignored_version"], "1.20.0"); + assert_eq!(app["pin_version"], "1.20.0"); assert_eq!(app["package_resolution"], "missing_package_definition"); world.output = Some(output); world.json = Some(json); @@ -1504,7 +1538,7 @@ fn write_offline_update_fixture( world: &mut CliWorld, package_id: String, installed_version: Option, - ignored_version: Option, + pin_version: Option, versions: String, ) { let candidates: Vec = versions @@ -1530,7 +1564,7 @@ fn write_offline_update_fixture( world, package_id, installed_version, - ignored_version, + pin_version, candidates, ); } @@ -1539,7 +1573,7 @@ fn write_offline_update_fixture_with_candidates( world: &mut CliWorld, package_id: String, installed_version: Option, - ignored_version: Option, + pin_version: Option, candidates: Vec, ) { let temp = world.temp.as_ref().expect("tempdir exists"); @@ -1551,7 +1585,7 @@ fn write_offline_update_fixture_with_candidates( "version": 1, "package_id": package_id, "installed_version": installed_version, - "ignored_version": ignored_version, + "pin_version": pin_version, "candidates": candidates, })) .expect("fixture serializes"), @@ -1635,6 +1669,13 @@ fn assert_report_has_drop_warnings(report: &Value) { .any(|warning| warning["code"].as_str() == Some("legacy.dropped_extra_hub_rows"))); } +fn assert_report_has_pin_version_notice(report: &Value) { + let notices = report["notices"].as_array().expect("notices array"); + assert!(notices.iter().any(|notice| { + notice["code"].as_str() == Some("migration.renamed_ignored_version_to_pin_version") + })); +} + fn assert_sanitized_direct_room_report_text(text: &str) { for forbidden in [ "UA_DIRECT_DB_SENTINEL_SECRET", diff --git a/crates/getter-cli/tests/features/cli/update_check.feature b/crates/getter-cli/tests/features/cli/update_check.feature index e5d4828..9ad2f95 100644 --- a/crates/getter-cli/tests/features/cli/update_check.feature +++ b/crates/getter-cli/tests/features/cli/update_check.feature @@ -19,22 +19,22 @@ Feature: Offline update check And the update check status is "up_to_date" And the update check has no selected update - Scenario: User checks an offline fixture where the latest update is ignored + Scenario: User checks an offline fixture where pin_version overrides the local baseline Given an initialized getter data directory - And an offline update fixture for package "android/org.fdroid.fdroid" installed version "1.0.0" ignored version "1.2.0" with candidate versions "1.1.0,1.2.0" + And an offline update fixture for package "android/org.fdroid.fdroid" installed version "1.0.0" pin version "1.2.0" with candidate versions "1.1.0,1.2.0,1.3.0" When I run getter update check for that fixture Then the command succeeds And the output is valid JSON And the update check status is "update_available" - And the selected update version is "1.1.0" + And the selected update version is "1.3.0" - Scenario: User checks an offline fixture where the only update is ignored + Scenario: User checks an offline fixture where pin_version makes the package up to date Given an initialized getter data directory - And an offline update fixture for package "android/org.fdroid.fdroid" installed version "1.0.0" ignored version "1.2.0" with candidate versions "1.2.0" + And an offline update fixture for package "android/org.fdroid.fdroid" installed version "1.0.0" pin version "1.2.0" with candidate versions "1.2.0" When I run getter update check for that fixture Then the command succeeds And the output is valid JSON - And the update check status is "ignored" + And the update check status is "up_to_date" And the update check has no selected update Scenario: User checks an offline fixture without an installed version diff --git a/crates/getter-cli/tests/features/cli/version_pin.feature b/crates/getter-cli/tests/features/cli/version_pin.feature new file mode 100644 index 0000000..fb18337 --- /dev/null +++ b/crates/getter-cli/tests/features/cli/version_pin.feature @@ -0,0 +1,12 @@ +@getter-cli @version +Feature: Version pin state + Scenario: User pins and unpins a package version baseline + Given an initialized getter data directory + When I run getter version pin for package "android/org.fdroid.fdroid" version "1.2.3" + Then the command succeeds + And the output is valid JSON + And the pinned package version is "1.2.3" + When I run getter version unpin for package "android/org.fdroid.fdroid" + Then the command succeeds + And the output is valid JSON + And the package is unpinned diff --git a/crates/getter-core/src/update.rs b/crates/getter-core/src/update.rs index 72acfa6..09952ef 100644 --- a/crates/getter-core/src/update.rs +++ b/crates/getter-core/src/update.rs @@ -12,9 +12,9 @@ pub const OFFLINE_UPDATE_CHECK_VERSION: u32 = 1; /// User state that affects update selection. #[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] pub struct UpdateSelectionPolicy { - /// Candidate version the user chose to ignore/mark as skipped. - #[serde(default)] - pub ignored_version: Option, + /// User-selected local version override used as the comparison baseline. + #[serde(default, alias = "ignored_version")] + pub pin_version: Option, } #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] @@ -24,8 +24,8 @@ pub struct OfflineUpdateCheckFixture { pub package_id: PackageId, #[serde(default)] pub installed_version: Option, - #[serde(default)] - pub ignored_version: Option, + #[serde(default, alias = "ignored_version")] + pub pin_version: Option, #[serde(default)] pub candidates: Vec, } @@ -36,6 +36,8 @@ pub struct OfflineUpdateCheckResult { pub package_id: PackageId, #[serde(default)] pub installed_version: Option, + #[serde(default)] + pub effective_local_version: Option, pub policy: UpdateSelectionPolicy, pub status: UpdateCheckStatus, #[serde(default)] @@ -49,7 +51,6 @@ pub enum UpdateCheckStatus { UpdateAvailable, UpToDate, NoCandidates, - Ignored, } #[derive(Debug, thiserror::Error)] @@ -76,7 +77,7 @@ pub fn run_offline_update_check( } let policy = UpdateSelectionPolicy { - ignored_version: fixture.ignored_version, + pin_version: fixture.pin_version, }; check_updates_offline( fixture.package_id, @@ -92,18 +93,17 @@ pub fn check_updates_offline( candidates: Vec, policy: UpdateSelectionPolicy, ) -> Result { + let effective_local_version = policy + .pin_version + .clone() + .or_else(|| installed_version.clone()); let selected = select_update( package_id.clone(), - installed_version.as_deref(), - &candidates, - &policy, - ); - let status = update_check_status( - selected.as_ref(), - installed_version.as_deref(), + effective_local_version.as_deref(), &candidates, &policy, ); + let status = update_check_status(selected.as_ref(), &candidates); let actions = match selected.as_ref() { Some(selected) => { if selected.artifact.is_none() { @@ -120,6 +120,7 @@ pub fn check_updates_offline( network_required: false, package_id, installed_version, + effective_local_version, policy, status, selected, @@ -163,38 +164,17 @@ fn installer_for_package_kind(kind: PackageKind) -> &'static str { fn update_check_status( selected: Option<&SelectedUpdate>, - installed_version: Option<&str>, candidates: &[UpdateCandidate], - policy: &UpdateSelectionPolicy, ) -> UpdateCheckStatus { if candidates.is_empty() { UpdateCheckStatus::NoCandidates } else if selected.is_some() { UpdateCheckStatus::UpdateAvailable - } else if ignored_candidate_would_have_been_update(installed_version, candidates, policy) { - UpdateCheckStatus::Ignored } else { UpdateCheckStatus::UpToDate } } -fn ignored_candidate_would_have_been_update( - installed_version: Option<&str>, - candidates: &[UpdateCandidate], - policy: &UpdateSelectionPolicy, -) -> bool { - let Some(ignored) = policy.ignored_version.as_deref() else { - return false; - }; - - candidates.iter().any(|candidate| { - compare_versions(&candidate.version, ignored) == Ordering::Equal - && installed_version.is_none_or(|installed| { - compare_versions(&candidate.version, installed) == Ordering::Greater - }) - }) -} - /// Compare human-facing version strings using a deterministic token ordering. /// /// This intentionally starts small: digit runs compare numerically, ASCII text @@ -236,8 +216,10 @@ pub fn compare_versions(left: &str, right: &str) -> Ordering { /// Select the best update candidate for a package. /// /// The selected candidate is the highest version that is newer than the -/// installed version and not equal to the user's ignored version. If no -/// installed version is known, the highest non-ignored candidate is selected. +/// effective local baseline. The effective baseline is normally the observed +/// installed version; callers pass `pin_version` instead when the user has set +/// a baseline override. If no baseline is known, the highest candidate is +/// selected. pub fn select_update( package_id: PackageId, installed_version: Option<&str>, @@ -261,13 +243,7 @@ fn is_selectable( installed_version: Option<&str>, policy: &UpdateSelectionPolicy, ) -> bool { - if policy - .ignored_version - .as_deref() - .is_some_and(|ignored| compare_versions(&candidate.version, ignored) == Ordering::Equal) - { - return false; - } + let _ = policy; match installed_version { Some(installed) => compare_versions(&candidate.version, installed) == Ordering::Greater, @@ -400,18 +376,21 @@ mod tests { } #[test] - fn respects_ignored_version() { - let selected = select_update( + fn pin_version_overrides_comparison_baseline() { + let result = check_updates_offline( "android/org.fdroid.fdroid".parse().unwrap(), - Some("1.0.0"), - &[candidate("1.1.0"), candidate("1.2.0")], - &UpdateSelectionPolicy { - ignored_version: Some("1.2.0".to_owned()), + Some("1.0.0".to_owned()), + vec![candidate("1.1.0"), candidate("1.2.0")], + UpdateSelectionPolicy { + pin_version: Some("1.2.0".to_owned()), }, ) .unwrap(); - assert_eq!(selected.candidate.version, "1.1.0"); + assert_eq!(result.status, UpdateCheckStatus::UpToDate); + assert_eq!(result.installed_version.as_deref(), Some("1.0.0")); + assert_eq!(result.effective_local_version.as_deref(), Some("1.2.0")); + assert!(result.selected.is_none()); } #[test] @@ -449,7 +428,7 @@ mod tests { version: OFFLINE_UPDATE_CHECK_VERSION, package_id: "android/org.fdroid.fdroid".parse().unwrap(), installed_version: Some("1.0.0".to_owned()), - ignored_version: None, + pin_version: None, candidates: vec![candidate("1.0.1"), candidate("1.2.0")], }) .unwrap(); @@ -487,35 +466,20 @@ mod tests { } #[test] - fn offline_update_check_reports_ignored_when_only_update_is_ignored() { + fn offline_update_check_uses_pin_version_as_baseline() { let result = check_updates_offline( "android/org.fdroid.fdroid".parse().unwrap(), Some("1.0.0".to_owned()), - vec![candidate("1.2.0")], - UpdateSelectionPolicy { - ignored_version: Some("1.2.0".to_owned()), - }, - ) - .unwrap(); - - assert_eq!(result.status, UpdateCheckStatus::Ignored); - assert!(result.selected.is_none()); - } - - #[test] - fn offline_update_check_falls_back_below_ignored_latest() { - let result = check_updates_offline( - "android/org.fdroid.fdroid".parse().unwrap(), - Some("1.0.0".to_owned()), - vec![candidate("1.1.0"), candidate("1.2.0")], + vec![candidate("1.1.0"), candidate("1.2.0"), candidate("1.3.0")], UpdateSelectionPolicy { - ignored_version: Some("1.2.0".to_owned()), + pin_version: Some("1.2.0".to_owned()), }, ) .unwrap(); assert_eq!(result.status, UpdateCheckStatus::UpdateAvailable); - assert_eq!(result.selected.as_ref().unwrap().candidate.version, "1.1.0"); + assert_eq!(result.selected.as_ref().unwrap().candidate.version, "1.3.0"); + assert_eq!(result.effective_local_version.as_deref(), Some("1.2.0")); } #[test] @@ -561,7 +525,7 @@ mod tests { version: OFFLINE_UPDATE_CHECK_VERSION, package_id: "android/org.fdroid.fdroid".parse().unwrap(), installed_version: None, - ignored_version: None, + pin_version: None, candidates: Vec::new(), }) .unwrap_err(); diff --git a/crates/getter-operations/src/legacy_room.rs b/crates/getter-operations/src/legacy_room.rs index 40c7c34..352205e 100644 --- a/crates/getter-operations/src/legacy_room.rs +++ b/crates/getter-operations/src/legacy_room.rs @@ -174,6 +174,7 @@ fn import_legacy_room_db( "imported_records": import.apps.len(), "source_counts": source_counts_json(import), "warnings": import_warnings_json(&import.warnings), + "notices": [migration_pin_version_notice()], }) .to_string(); db.import_tracked_packages_with_migration_record( @@ -194,7 +195,7 @@ fn tracked_package_upsert( package_id: mapping.package_id, enabled: true, favorite: mapping.user_state.favorite, - ignored_version: mapping.user_state.ignored_version, + pin_version: mapping.user_state.pin_version, repository_id: None, package_resolution: stored_resolution(mapping.package_resolution), } @@ -249,7 +250,7 @@ fn tracked_packages_json(packages: Vec) -> Vec { "id": package.package_id.to_string(), "enabled": package.enabled, "favorite": package.favorite, - "ignored_version": package.ignored_version, + "pin_version": package.pin_version, "repository_id": package.repository_id.map(|id| id.to_string()), "package_resolution": package.package_resolution.as_str(), }) @@ -266,6 +267,13 @@ fn source_counts_json(import: &LegacyRoomDbImport) -> Value { }) } +fn migration_pin_version_notice() -> Value { + json!({ + "code": "migration.renamed_ignored_version_to_pin_version", + "message": "Legacy ignored version state was preserved as pin_version", + }) +} + fn import_warnings_json(warnings: &[LegacyRoomImportWarning]) -> Vec { warnings .iter() @@ -310,6 +318,7 @@ fn create_migration_report_with_source_counts( imported_records, tracked_records, warnings, + notices: migration_report_notices(code), source_counts, }; let bytes = serde_json::to_vec_pretty(&report).map_err(|source| { @@ -325,6 +334,14 @@ fn report_file_name(code: &str) -> String { format!("{}.json", code.replace('.', "-")) } +fn migration_report_notices(code: &str) -> Vec { + if code == "migration.imported" { + vec![migration_pin_version_notice()] + } else { + Vec::new() + } +} + fn list_migration_reports(data_dir: &Path) -> LegacyRoomOperationResult> { let reports_dir = data_dir.join(MIGRATION_REPORTS_DIR); if !reports_dir.exists() { @@ -377,6 +394,10 @@ fn list_migration_reports(data_dir: &Path) -> LegacyRoomOperationResult { imported_records: u64, tracked_records: u64, warnings: &'a [Value], + #[serde(skip_serializing_if = "Vec::is_empty")] + notices: Vec, #[serde(skip_serializing_if = "Option::is_none")] source_counts: Option<&'a Value>, } diff --git a/crates/getter-operations/src/runtime.rs b/crates/getter-operations/src/runtime.rs index 9887f15..f6bfa38 100644 --- a/crates/getter-operations/src/runtime.rs +++ b/crates/getter-operations/src/runtime.rs @@ -140,7 +140,7 @@ pub fn issue_action_from_registered_package_json( request.installed_version, package.updates.clone(), UpdateSelectionPolicy { - ignored_version: request.ignored_version, + pin_version: request.pin_version, }, )?; let action = if update.actions.is_empty() { @@ -316,8 +316,8 @@ struct RegisteredPackageUpdateActionRequest { repository_id: Option, #[serde(default)] installed_version: Option, - #[serde(default)] - ignored_version: Option, + #[serde(default, alias = "ignored_version")] + pin_version: Option, } #[derive(Debug, Deserialize)] @@ -689,7 +689,7 @@ return package_def { version: OFFLINE_UPDATE_CHECK_VERSION, package_id: package_id.parse().unwrap(), installed_version: installed_version.map(str::to_owned), - ignored_version: None, + pin_version: None, candidates: versions .into_iter() .map(|version| UpdateCandidate { diff --git a/crates/getter-storage/src/legacy_room.rs b/crates/getter-storage/src/legacy_room.rs index 711ea1f..cf225f7 100644 --- a/crates/getter-storage/src/legacy_room.rs +++ b/crates/getter-storage/src/legacy_room.rs @@ -51,7 +51,7 @@ pub enum LegacyPackageResolution { #[derive(Debug, Clone, Default, PartialEq, Eq)] pub struct LegacyUserStateMapping { - pub ignored_version: Option, + pub pin_version: Option, pub favorite: bool, } @@ -157,10 +157,9 @@ pub fn map_legacy_app( _ => Vec::new(), }; let user_state = LegacyUserStateMapping { - ignored_version: extra.and_then(|extra| extra.ignored_version.clone()), + pin_version: extra.and_then(|extra| extra.ignored_version.clone()), favorite: extra.is_some_and(|extra| extra.favorite), }; - Ok(LegacyAppMapping { package_id, package_resolution, @@ -506,7 +505,7 @@ mod tests { ) .unwrap(); - assert_eq!(mapping.user_state.ignored_version.as_deref(), Some("1.2.3")); + assert_eq!(mapping.user_state.pin_version.as_deref(), Some("1.2.3")); assert!(mapping.user_state.favorite); } diff --git a/crates/getter-storage/src/lib.rs b/crates/getter-storage/src/lib.rs index 23b0e8a..2127cbc 100644 --- a/crates/getter-storage/src/lib.rs +++ b/crates/getter-storage/src/lib.rs @@ -35,6 +35,8 @@ pub enum StorageError { InvalidTaskTransition { task_id: String, reason: String }, #[error("invalid task request: {0}")] InvalidTaskRequest(String), + #[error("storage invariant failed: {0}")] + Invariant(String), #[error("JSON error: {0}")] Json(#[from] serde_json::Error), } @@ -85,7 +87,7 @@ CREATE TABLE IF NOT EXISTS tracked_packages ( package_id TEXT PRIMARY KEY, enabled INTEGER NOT NULL DEFAULT 1, favorite INTEGER NOT NULL DEFAULT 0, - ignored_version TEXT, + pin_version TEXT, repository_id TEXT, package_resolution TEXT NOT NULL DEFAULT 'missing_package_definition', FOREIGN KEY(repository_id) REFERENCES repositories(id) @@ -136,7 +138,8 @@ CREATE TABLE IF NOT EXISTS install_handoffs ( ); "#, )?; - self.ensure_column("tracked_packages", "ignored_version", "TEXT")?; + self.ensure_column("tracked_packages", "pin_version", "TEXT")?; + self.migrate_ignored_version_to_pin_version()?; self.ensure_column( "tracked_packages", "package_resolution", @@ -168,6 +171,22 @@ CREATE TABLE IF NOT EXISTS install_handoffs ( Ok(()) } + fn migrate_ignored_version_to_pin_version(&self) -> Result<(), StorageError> { + let columns = self.table_columns("tracked_packages")?; + if columns.iter().any(|existing| existing == "ignored_version") { + self.conn.execute( + r#" +UPDATE tracked_packages +SET pin_version = ignored_version +WHERE pin_version IS NULL + AND ignored_version IS NOT NULL +"#, + [], + )?; + } + Ok(()) + } + fn table_columns(&self, table: &'static str) -> Result, StorageError> { let mut stmt = self.conn.prepare(&format!("PRAGMA table_info({table})"))?; let rows = stmt.query_map([], |row| row.get::<_, String>(1))?; @@ -244,6 +263,46 @@ ON CONFLICT(id) DO UPDATE SET Ok(()) } + pub fn set_tracked_package_pin_version( + &self, + package_id: &PackageId, + pin_version: Option<&str>, + ) -> Result { + self.conn.execute( + r#" +INSERT INTO tracked_packages( + package_id, + enabled, + favorite, + pin_version, + repository_id, + package_resolution +) +VALUES (?1, 1, 0, ?2, NULL, ?3) +ON CONFLICT(package_id) DO UPDATE SET + pin_version = excluded.pin_version +"#, + params![ + package_id.to_string(), + pin_version, + StoredPackageResolution::MissingPackageDefinition.as_str(), + ], + )?; + self.tracked_package(package_id)?.ok_or_else(|| { + StorageError::Invariant("tracked package upsert did not create a row".to_owned()) + }) + } + + pub fn tracked_package( + &self, + package_id: &PackageId, + ) -> Result, StorageError> { + Ok(self + .tracked_packages()? + .into_iter() + .find(|package| &package.package_id == package_id)) + } + pub fn upsert_generated_tracked_package_preserving_user_state( &self, package_id: &PackageId, @@ -255,7 +314,7 @@ INSERT INTO tracked_packages( package_id, enabled, favorite, - ignored_version, + pin_version, repository_id, package_resolution ) @@ -329,7 +388,7 @@ WHERE package_id = ?1 pub fn tracked_packages(&self) -> Result, StorageError> { let mut stmt = self.conn.prepare( r#" -SELECT package_id, enabled, favorite, ignored_version, repository_id, package_resolution +SELECT package_id, enabled, favorite, pin_version, repository_id, package_resolution FROM tracked_packages ORDER BY package_id ASC "#, @@ -346,12 +405,12 @@ ORDER BY package_id ASC })?; let mut packages = Vec::new(); for row in rows { - let (package_id, enabled, favorite, ignored_version, repository_id, resolution) = row?; + let (package_id, enabled, favorite, pin_version, repository_id, resolution) = row?; packages.push(StoredTrackedPackage { package_id: PackageId::from_str(&package_id)?, enabled: enabled != 0, favorite: favorite != 0, - ignored_version, + pin_version, repository_id: repository_id.map(RepositoryId::new).transpose()?, package_resolution: StoredPackageResolution::from_str(&resolution)?, }); @@ -842,7 +901,7 @@ pub struct TrackedPackageUpsert { pub package_id: PackageId, pub enabled: bool, pub favorite: bool, - pub ignored_version: Option, + pub pin_version: Option, pub repository_id: Option, pub package_resolution: StoredPackageResolution, } @@ -852,7 +911,7 @@ pub struct StoredTrackedPackage { pub package_id: PackageId, pub enabled: bool, pub favorite: bool, - pub ignored_version: Option, + pub pin_version: Option, pub repository_id: Option, pub package_resolution: StoredPackageResolution, } @@ -957,7 +1016,7 @@ INSERT INTO tracked_packages( package_id, enabled, favorite, - ignored_version, + pin_version, repository_id, package_resolution ) @@ -965,7 +1024,7 @@ VALUES (?1, ?2, ?3, ?4, ?5, ?6) ON CONFLICT(package_id) DO UPDATE SET enabled = excluded.enabled, favorite = excluded.favorite, - ignored_version = excluded.ignored_version, + pin_version = excluded.pin_version, repository_id = excluded.repository_id, package_resolution = excluded.package_resolution "#, @@ -973,7 +1032,7 @@ ON CONFLICT(package_id) DO UPDATE SET package.package_id.to_string(), bool_to_i64(package.enabled), bool_to_i64(package.favorite), - package.ignored_version.as_deref(), + package.pin_version.as_deref(), package.repository_id.as_ref().map(RepositoryId::as_str), package.package_resolution.as_str(), ], @@ -1105,7 +1164,7 @@ mod tests { package_id: "android/org.fdroid.fdroid".parse().unwrap(), enabled: true, favorite: true, - ignored_version: Some("1.2.3".to_owned()), + pin_version: Some("1.2.3".to_owned()), repository_id: None, package_resolution: StoredPackageResolution::OfficialRepositoryPackage, }) @@ -1119,13 +1178,58 @@ mod tests { ); assert!(packages[0].enabled); assert!(packages[0].favorite); - assert_eq!(packages[0].ignored_version.as_deref(), Some("1.2.3")); + assert_eq!(packages[0].pin_version.as_deref(), Some("1.2.3")); assert_eq!( packages[0].package_resolution, StoredPackageResolution::OfficialRepositoryPackage ); } + #[test] + fn main_db_pins_and_unpins_tracked_package_version() { + let db = MainDb::open_in_memory().unwrap(); + let package_id: PackageId = "android/org.fdroid.fdroid".parse().unwrap(); + + let pinned = db + .set_tracked_package_pin_version(&package_id, Some("1.2.3")) + .unwrap(); + assert_eq!(pinned.pin_version.as_deref(), Some("1.2.3")); + assert_eq!( + pinned.package_resolution, + StoredPackageResolution::MissingPackageDefinition + ); + + let unpinned = db + .set_tracked_package_pin_version(&package_id, None) + .unwrap(); + assert_eq!(unpinned.pin_version, None); + } + + #[test] + fn main_db_migrates_legacy_ignored_version_column_to_pin_version() { + let conn = Connection::open_in_memory().unwrap(); + conn.execute_batch( + r#" +CREATE TABLE tracked_packages ( + package_id TEXT PRIMARY KEY, + enabled INTEGER NOT NULL DEFAULT 1, + favorite INTEGER NOT NULL DEFAULT 0, + ignored_version TEXT, + repository_id TEXT, + package_resolution TEXT NOT NULL DEFAULT 'missing_package_definition' +); +INSERT INTO tracked_packages(package_id, ignored_version) +VALUES ('android/org.fdroid.fdroid', '1.2.3'); +"#, + ) + .unwrap(); + let db = MainDb { conn }; + db.migrate().unwrap(); + + let packages = db.tracked_packages().unwrap(); + assert_eq!(packages[0].pin_version.as_deref(), Some("1.2.3")); + } + #[test] fn generated_tracking_preserves_existing_user_state_on_conflict() { let db = MainDb::open_in_memory().unwrap(); @@ -1135,7 +1239,7 @@ mod tests { package_id: package_id.clone(), enabled: false, favorite: true, - ignored_version: Some("9.9.9".to_owned()), + pin_version: Some("9.9.9".to_owned()), repository_id: None, package_resolution: StoredPackageResolution::OfficialRepositoryPackage, }) @@ -1148,7 +1252,7 @@ mod tests { assert_eq!(packages.len(), 1); assert!(!packages[0].enabled); assert!(packages[0].favorite); - assert_eq!(packages[0].ignored_version.as_deref(), Some("9.9.9")); + assert_eq!(packages[0].pin_version.as_deref(), Some("9.9.9")); assert_eq!(packages[0].repository_id, None); assert_eq!( packages[0].package_resolution, @@ -1165,7 +1269,7 @@ mod tests { package_id: package_id.clone(), enabled: false, favorite: true, - ignored_version: Some("9.9.9".to_owned()), + pin_version: Some("9.9.9".to_owned()), repository_id: None, package_resolution: StoredPackageResolution::MissingPackageDefinition, }) @@ -1178,7 +1282,7 @@ mod tests { assert_eq!(packages.len(), 1); assert!(!packages[0].enabled); assert!(packages[0].favorite); - assert_eq!(packages[0].ignored_version.as_deref(), Some("9.9.9")); + assert_eq!(packages[0].pin_version.as_deref(), Some("9.9.9")); assert_eq!(packages[0].repository_id.as_ref(), Some(&local_autogen)); assert_eq!( packages[0].package_resolution, @@ -1195,7 +1299,7 @@ mod tests { package_id: package_id.clone(), enabled: true, favorite: true, - ignored_version: Some("9.9.9".to_owned()), + pin_version: Some("9.9.9".to_owned()), repository_id: None, package_resolution: StoredPackageResolution::OfficialRepositoryPackage, }) @@ -1210,7 +1314,7 @@ mod tests { package_id: package_id.clone(), enabled: true, favorite: true, - ignored_version: Some("9.9.9".to_owned()), + pin_version: Some("9.9.9".to_owned()), repository_id: None, package_resolution: StoredPackageResolution::MissingPackageDefinition, }) From ae5b00ccf0989ee1b0b69521bb508d4136c2bcb4 Mon Sep 17 00:00:00 2001 From: Xiangzhe Date: Thu, 25 Jun 2026 13:41:37 +0800 Subject: [PATCH 25/52] feat(cli): add runtime script debug tasks --- crates/getter-cli/src/lib.rs | 507 ++++++++++++++++-- crates/getter-cli/tests/bdd_cli.rs | 154 +++++- .../tests/features/cli/task_lifecycle.feature | 52 +- 3 files changed, 621 insertions(+), 92 deletions(-) diff --git a/crates/getter-cli/src/lib.rs b/crates/getter-cli/src/lib.rs index ed72425..d8317fe 100644 --- a/crates/getter-cli/src/lib.rs +++ b/crates/getter-cli/src/lib.rs @@ -8,6 +8,7 @@ use getter_core::autogen::InstalledInventory; use getter_core::diagnostics::validate_repository_path; use getter_core::lua::evaluate_package_file; use getter_core::repository::{RepositoryLayout, RepositoryMetadata}; +use getter_core::runtime::{GetterRuntime, SealedActionPlan}; use getter_core::task::{ DownloadTaskRequest, InstallHandoffStatus, TaskEventPage, DOWNLOAD_REQUEST_FORMAT, DOWNLOAD_REQUEST_VERSION, @@ -19,6 +20,7 @@ use getter_downloader::{ }; use getter_operations::autogen::{self, AutogenAcceptance, AutogenOperationError}; use getter_operations::legacy_room::{self, LegacyRoomOperationError}; +use getter_operations::runtime as runtime_operations; use getter_storage::legacy_room::{ map_legacy_app, LegacyAppKind, LegacyAppRecord, LegacyExtraAppRecord, LegacyPackageResolution, }; @@ -74,24 +76,27 @@ pub enum CliCommand { UpdateCheck { fixture: PathBuf, }, - TaskSubmit { + DebugFakeTaskSubmit { request: PathBuf, }, - TaskRun { + DebugFakeTaskRun { task_id: String, }, - TaskList, - TaskCancel { + DebugFakeTaskList, + DebugFakeTaskCancel { task_id: String, }, - TaskEvents { + DebugFakeTaskEvents { after: u64, limit: usize, }, - TaskInstallResult { + DebugFakeTaskInstallResult { handoff_id: String, status: InstallHandoffStatus, }, + RuntimeScript { + script: PathBuf, + }, AutogenInstalledPreview { inventory: PathBuf, }, @@ -145,6 +150,8 @@ pub enum CliError { Update(String), #[error("download task error: {0}")] Download(String), + #[error("runtime error: {0}")] + Runtime(String), #[error("autogen error: {0}")] Autogen(String), #[error("Legacy Room export bundle is invalid")] @@ -162,9 +169,11 @@ impl CliError { match self { Self::Usage(_) => ExitCode::Usage, Self::Storage(_) => ExitCode::Storage, - Self::Repository(_) | Self::PackageEval(_) | Self::Update(_) | Self::Autogen(_) => { - ExitCode::GenericFailure - } + Self::Repository(_) + | Self::PackageEval(_) + | Self::Update(_) + | Self::Runtime(_) + | Self::Autogen(_) => ExitCode::GenericFailure, Self::Download(_) => ExitCode::Download, Self::InvalidLegacyBundle { .. } | Self::UnsupportedLegacyBundle { .. } @@ -181,6 +190,7 @@ impl CliError { Self::PackageEval(_) => "package.eval_error", Self::Update(_) => "update.check_error", Self::Download(_) => "download.task_error", + Self::Runtime(_) => "runtime.error", Self::Autogen(_) => "autogen.error", Self::InvalidLegacyBundle { .. } => "migration.invalid_bundle", Self::UnsupportedLegacyBundle { .. } => "migration.unsupported_bundle", @@ -197,6 +207,7 @@ impl CliError { Self::PackageEval(_) => "Getter package evaluation failed", Self::Update(_) => "Getter update check failed", Self::Download(_) => "Getter download task operation failed", + Self::Runtime(_) => "Getter runtime operation failed", Self::Autogen(_) => "Getter autogen operation failed", Self::InvalidLegacyBundle { .. } => "Legacy Room export bundle is invalid", Self::UnsupportedLegacyBundle { .. } => { @@ -215,6 +226,7 @@ impl CliError { | Self::PackageEval(detail) | Self::Update(detail) | Self::Download(detail) + | Self::Runtime(detail) | Self::Autogen(detail) => Some(detail.as_str()), Self::InvalidLegacyBundle { .. } | Self::UnsupportedLegacyBundle { .. } @@ -235,6 +247,7 @@ impl CliError { | Self::PackageEval(_) | Self::Update(_) | Self::Download(_) + | Self::Runtime(_) | Self::Autogen(_) => None, } } @@ -405,41 +418,65 @@ where fixture: PathBuf::from(fixture), } } - [domain, command, flag, request] - if domain == "task" && command == "submit" && flag == "--request" => + [domain, subject, command, flag, request] + if domain == "debug" + && subject == "fake-task" + && command == "submit" + && flag == "--request" => { - CliCommand::TaskSubmit { + CliCommand::DebugFakeTaskSubmit { request: PathBuf::from(request), } } - [domain, command, task_id] if domain == "task" && command == "run" => CliCommand::TaskRun { - task_id: task_id.clone(), - }, - [domain, command] if domain == "task" && command == "list" => CliCommand::TaskList, - [domain, command, task_id] if domain == "task" && command == "cancel" => { - CliCommand::TaskCancel { + [domain, subject, command, task_id] + if domain == "debug" && subject == "fake-task" && command == "run" => + { + CliCommand::DebugFakeTaskRun { task_id: task_id.clone(), } } - [domain, command, after_flag, after, limit_flag, limit] - if domain == "task" + [domain, subject, command] + if domain == "debug" && subject == "fake-task" && command == "list" => + { + CliCommand::DebugFakeTaskList + } + [domain, subject, command, task_id] + if domain == "debug" && subject == "fake-task" && command == "cancel" => + { + CliCommand::DebugFakeTaskCancel { + task_id: task_id.clone(), + } + } + [domain, subject, command, after_flag, after, limit_flag, limit] + if domain == "debug" + && subject == "fake-task" && command == "events" && after_flag == "--after" && limit_flag == "--limit" => { - CliCommand::TaskEvents { + CliCommand::DebugFakeTaskEvents { after: parse_u64(after, "--after")?, limit: parse_positive_usize(limit, "--limit")?, } } - [domain, command, handoff_id, status_flag, status] - if domain == "task" && command == "install-result" && status_flag == "--status" => + [domain, subject, command, handoff_id, status_flag, status] + if domain == "debug" + && subject == "fake-task" + && command == "install-result" + && status_flag == "--status" => { - CliCommand::TaskInstallResult { + CliCommand::DebugFakeTaskInstallResult { handoff_id: handoff_id.clone(), status: parse_install_handoff_status(status)?, } } + [domain, command, flag, script] + if domain == "runtime" && command == "script" && flag == "--script" => + { + CliCommand::RuntimeScript { + script: PathBuf::from(script), + } + } [domain, subject, action, flag, inventory] if domain == "autogen" && subject == "installed" @@ -608,46 +645,49 @@ fn execute(invocation: CliInvocation) -> Result { CliError::Update(format!("failed to serialize update check: {source}")) }) } - CliCommand::TaskSubmit { request } => { + CliCommand::DebugFakeTaskSubmit { request } => { let db = open_main_db(&invocation.data_dir)?; let request = read_download_task_request(&request)?; serde_json::to_value(submit_fake_download_task(&db, request).map_err(|source| { - CliError::Download(format!("offline task submit failed: {source}")) + CliError::Download(format!("offline fake task submit failed: {source}")) })?) .map_err(|source| CliError::Download(format!("failed to serialize task: {source}"))) } - CliCommand::TaskRun { task_id } => { + CliCommand::DebugFakeTaskRun { task_id } => { let db = open_main_db(&invocation.data_dir)?; serde_json::to_value(run_fake_download_task(&db, &task_id).map_err(|source| { - CliError::Download(format!("offline task run failed: {source}")) + CliError::Download(format!("offline fake task run failed: {source}")) })?) .map_err(|source| CliError::Download(format!("failed to serialize task: {source}"))) } - CliCommand::TaskList => { + CliCommand::DebugFakeTaskList => { let db = open_main_db(&invocation.data_dir)?; Ok(json!({ "tasks": db.download_tasks()? })) } - CliCommand::TaskCancel { task_id } => { + CliCommand::DebugFakeTaskCancel { task_id } => { let db = open_main_db(&invocation.data_dir)?; serde_json::to_value(cancel_download_task(&db, &task_id).map_err(|source| { - CliError::Download(format!("offline task cancel failed: {source}")) + CliError::Download(format!("offline fake task cancel failed: {source}")) })?) .map_err(|source| CliError::Download(format!("failed to serialize task: {source}"))) } - CliCommand::TaskEvents { after, limit } => { + CliCommand::DebugFakeTaskEvents { after, limit } => { let db = open_main_db(&invocation.data_dir)?; let events: TaskEventPage = db.task_events_after(after, limit)?; serde_json::to_value(events).map_err(|source| { - CliError::Download(format!("failed to serialize task events: {source}")) + CliError::Download(format!("failed to serialize fake task events: {source}")) }) } - CliCommand::TaskInstallResult { handoff_id, status } => { + CliCommand::DebugFakeTaskInstallResult { handoff_id, status } => { let db = open_main_db(&invocation.data_dir)?; serde_json::to_value(record_install_result(&db, &handoff_id, status).map_err( - |source| CliError::Download(format!("offline install result failed: {source}")), + |source| { + CliError::Download(format!("offline fake install result failed: {source}")) + }, )?) .map_err(|source| CliError::Download(format!("failed to serialize handoff: {source}"))) } + CliCommand::RuntimeScript { script } => run_runtime_script(&script), CliCommand::AutogenInstalledPreview { inventory } => { let db = open_main_db(&invocation.data_dir)?; let inventory = read_installed_inventory(&inventory)?; @@ -954,6 +994,227 @@ fn read_download_task_request(path: &Path) -> Result, +} + +#[derive(Debug, Deserialize)] +struct RuntimeScriptStep { + operation: String, + #[serde(default)] + payload: Value, + #[serde(default)] + plan: Option, +} + +fn run_runtime_script(path: &Path) -> Result { + let bytes = fs::read(path) + .map_err(|source| CliError::Runtime(format!("failed to read runtime script: {source}")))?; + let script: RuntimeScript = serde_json::from_slice(&bytes).map_err(|source| { + CliError::Runtime(format!("failed to parse runtime script JSON: {source}")) + })?; + let mut runtime = GetterRuntime::new(); + let mut context = RuntimeScriptContext::default(); + let mut outputs = Vec::new(); + for step in script.steps { + let data = execute_runtime_script_step(&mut runtime, &mut context, step)?; + outputs.push(data); + } + Ok(json!({ "steps": outputs })) +} + +#[derive(Default)] +struct RuntimeScriptContext { + last_action_id: Option, + last_task_id: Option, +} + +fn execute_runtime_script_step( + runtime: &mut GetterRuntime, + context: &mut RuntimeScriptContext, + step: RuntimeScriptStep, +) -> Result { + let operation = step.operation.as_str(); + let data = match operation { + "issue_action" => { + let plan = match step.plan { + Some(plan) => plan, + None => serde_json::from_value(step.payload).map_err(|source| { + CliError::Runtime(format!("failed to parse issue_action plan: {source}")) + })?, + }; + runtime_operations::issue_action(runtime, plan) + } + "submit_action" => runtime_json_operation( + runtime, + runtime_operations::submit_action_json, + default_action_payload(context, step.payload)?, + )?, + "task_get" => runtime_json_query_operation( + runtime, + runtime_operations::task_get_json, + default_task_payload(context, step.payload)?, + )?, + "task_list" => runtime_json_query_operation( + runtime, + runtime_operations::task_list_json, + empty_object_payload(step.payload), + )?, + "task_start" => runtime_json_operation( + runtime, + runtime_operations::task_start_json, + default_task_payload(context, step.payload)?, + )?, + "task_download_progress" => runtime_json_operation( + runtime, + runtime_operations::task_download_progress_json, + default_task_payload(context, step.payload)?, + )?, + "task_complete_download" => runtime_json_operation( + runtime, + runtime_operations::task_complete_download_json, + default_task_payload(context, step.payload)?, + )?, + "task_pause" => runtime_json_operation( + runtime, + runtime_operations::task_pause_json, + default_task_payload(context, step.payload)?, + )?, + "task_resume" => runtime_json_operation( + runtime, + runtime_operations::task_resume_json, + default_task_payload(context, step.payload)?, + )?, + "task_user_result" => runtime_json_operation( + runtime, + runtime_operations::task_user_result_json, + default_task_payload(context, step.payload)?, + )?, + "task_cancel" => runtime_json_operation( + runtime, + runtime_operations::task_cancel_json, + default_task_payload(context, step.payload)?, + )?, + "task_retry" => runtime_json_operation( + runtime, + runtime_operations::task_retry_json, + default_task_payload(context, step.payload)?, + )?, + "task_remove" => runtime_json_operation( + runtime, + runtime_operations::task_remove_json, + default_task_payload(context, step.payload)?, + )?, + "task_clean" => runtime_json_operation( + runtime, + runtime_operations::task_clean_json, + empty_object_payload(step.payload), + )?, + other => { + return Err(CliError::Runtime(format!( + "unsupported runtime script operation '{other}'" + ))) + } + }; + remember_runtime_script_ids(context, &data); + Ok(json!({ "operation": operation, "data": data })) +} + +fn runtime_json_operation( + runtime: &mut GetterRuntime, + operation: fn( + &mut GetterRuntime, + &str, + ) -> Result, + payload: Value, +) -> Result { + let payload = serde_json::to_string(&payload).map_err(|source| { + CliError::Runtime(format!("failed to serialize runtime request: {source}")) + })?; + operation(runtime, &payload).map_err(|source| CliError::Runtime(source.to_string())) +} + +fn runtime_json_query_operation( + runtime: &GetterRuntime, + operation: fn(&GetterRuntime, &str) -> Result, + payload: Value, +) -> Result { + let payload = serde_json::to_string(&payload).map_err(|source| { + CliError::Runtime(format!("failed to serialize runtime request: {source}")) + })?; + operation(runtime, &payload).map_err(|source| CliError::Runtime(source.to_string())) +} + +fn empty_object_payload(payload: Value) -> Value { + if payload.is_null() { + json!({}) + } else { + payload + } +} + +fn default_action_payload( + context: &RuntimeScriptContext, + payload: Value, +) -> Result { + if payload.is_null() { + let action_id = context.last_action_id.as_ref().ok_or_else(|| { + CliError::Runtime("runtime script has no previous action_id".to_owned()) + })?; + return Ok(json!({ "action_id": action_id })); + } + replace_runtime_script_tokens(context, payload) +} + +fn default_task_payload(context: &RuntimeScriptContext, payload: Value) -> Result { + if payload.is_null() { + let task_id = context.last_task_id.as_ref().ok_or_else(|| { + CliError::Runtime("runtime script has no previous task_id".to_owned()) + })?; + return Ok(json!({ "task_id": task_id })); + } + replace_runtime_script_tokens(context, payload) +} + +fn replace_runtime_script_tokens( + context: &RuntimeScriptContext, + payload: Value, +) -> Result { + match payload { + Value::String(value) if value == "$last_action_id" => { + Ok(Value::String(context.last_action_id.clone().ok_or_else( + || CliError::Runtime("runtime script has no previous action_id".to_owned()), + )?)) + } + Value::String(value) if value == "$last_task_id" => { + Ok(Value::String(context.last_task_id.clone().ok_or_else( + || CliError::Runtime("runtime script has no previous task_id".to_owned()), + )?)) + } + Value::Array(values) => values + .into_iter() + .map(|value| replace_runtime_script_tokens(context, value)) + .collect::, _>>() + .map(Value::Array), + Value::Object(values) => values + .into_iter() + .map(|(key, value)| Ok((key, replace_runtime_script_tokens(context, value)?))) + .collect::, CliError>>() + .map(Value::Object), + other => Ok(other), + } +} + +fn remember_runtime_script_ids(context: &mut RuntimeScriptContext, data: &Value) { + if let Some(action_id) = data.get("action_id").and_then(Value::as_str) { + context.last_action_id = Some(action_id.to_owned()); + } + if let Some(task_id) = data.get("task_id").and_then(Value::as_str) { + context.last_task_id = Some(task_id.to_owned()); + } +} + fn repo_path(repo: &StoredRepository) -> Result { repo.path .as_ref() @@ -1214,7 +1475,7 @@ fn envelope_to_string(value: Value) -> String { } fn usage_text() -> String { - "Usage: getter --data-dir [--priority ]|repo eval |repo validate |package eval [--repo ]|storage validate|version pin |version unpin |hub list|update check --fixture |task submit --request |task run |task list|task cancel |task events --after --limit |task install-result --status |autogen installed preview --inventory |autogen installed apply --preview (--accept-all|--accept ...)|autogen cleanup preview --inventory |autogen cleanup apply --preview (--accept-all|--accept ...)|legacy import-room-bundle |legacy import-room-db |legacy report-list>\n".to_owned() + "Usage: getter --data-dir [--priority ]|repo eval |repo validate |package eval [--repo ]|storage validate|version pin |version unpin |hub list|update check --fixture |runtime script --script |debug fake-task submit --request |debug fake-task run |debug fake-task list|debug fake-task cancel |debug fake-task events --after --limit |debug fake-task install-result --status |autogen installed preview --inventory |autogen installed apply --preview (--accept-all|--accept ...)|autogen cleanup preview --inventory |autogen cleanup apply --preview (--accept-all|--accept ...)|legacy import-room-bundle |legacy import-room-db |legacy report-list>\nNote: `debug fake-task` commands are persisted fake-download scaffolding. ADR-0011 runtime task debugging uses `runtime script` and does not preserve task state across CLI invocations.\n".to_owned() } #[derive(Debug, Deserialize)] @@ -1270,12 +1531,13 @@ impl CliCommand { Self::VersionPin { .. } => "version pin", Self::VersionUnpin { .. } => "version unpin", Self::UpdateCheck { .. } => "update check", - Self::TaskSubmit { .. } => "task submit", - Self::TaskRun { .. } => "task run", - Self::TaskList => "task list", - Self::TaskCancel { .. } => "task cancel", - Self::TaskEvents { .. } => "task events", - Self::TaskInstallResult { .. } => "task install-result", + Self::DebugFakeTaskSubmit { .. } => "debug fake-task submit", + Self::DebugFakeTaskRun { .. } => "debug fake-task run", + Self::DebugFakeTaskList => "debug fake-task list", + Self::DebugFakeTaskCancel { .. } => "debug fake-task cancel", + Self::DebugFakeTaskEvents { .. } => "debug fake-task events", + Self::DebugFakeTaskInstallResult { .. } => "debug fake-task install-result", + Self::RuntimeScript { .. } => "runtime script", Self::AutogenInstalledPreview { .. } => "autogen installed preview", Self::AutogenInstalledApply { .. } => "autogen installed apply", Self::AutogenCleanupPreview { .. } => "autogen cleanup preview", @@ -1337,6 +1599,167 @@ mod tests { ); } + #[test] + fn old_task_namespace_is_not_public_cli_surface() { + let output = run(["getter", "--data-dir", "/tmp/ua-getter", "task", "list"]); + assert_eq!(output.exit_code, ExitCode::Usage); + let json: Value = serde_json::from_str(&output.stdout).unwrap(); + assert_eq!(json["error"]["code"], "cli.usage"); + } + + #[test] + fn parses_runtime_script_command() { + let parsed = parse_args([ + "getter", + "--data-dir", + "/tmp/ua-getter", + "runtime", + "script", + "--script", + "/tmp/runtime-script.json", + ]) + .unwrap(); + assert_eq!( + parsed.command, + CliCommand::RuntimeScript { + script: PathBuf::from("/tmp/runtime-script.json"), + } + ); + } + + #[test] + fn run_runtime_script_exercises_in_memory_task_remove_and_clean() { + let temp = tempfile::tempdir().unwrap(); + let data_dir = temp.path().join("getter-data"); + let script = temp.path().join("runtime-script.json"); + fs::write( + &script, + serde_json::to_vec_pretty(&json!({ + "steps": [ + { + "operation": "issue_action", + "plan": { + "package_id": "android/org.fdroid.fdroid", + "actions": [ + { + "type": "download", + "url": "https://example.invalid/app.apk", + "file_name": "app.apk" + } + ], + "lua_object": { + "object_id": "debug:android/org.fdroid.fdroid", + "dependency_digest": "debug-digest" + } + } + }, + { "operation": "submit_action" }, + { "operation": "task_start" }, + { + "operation": "task_download_progress", + "payload": { + "task_id": "$last_task_id", + "current_bits": 10, + "total_bits": 20 + } + }, + { "operation": "task_complete_download" }, + { "operation": "task_remove" }, + { "operation": "task_clean", "payload": { "mode": "all_inactive" } } + ] + })) + .unwrap(), + ) + .unwrap(); + + let output = run([ + "getter".to_owned(), + "--data-dir".to_owned(), + data_dir.to_string_lossy().to_string(), + "runtime".to_owned(), + "script".to_owned(), + "--script".to_owned(), + script.to_string_lossy().to_string(), + ]); + + assert_eq!(output.exit_code, ExitCode::Success); + let json: Value = serde_json::from_str(&output.stdout).unwrap(); + assert_eq!(json["command"], "runtime script"); + let steps = json["data"]["steps"].as_array().unwrap(); + assert_eq!(steps[0]["data"]["action_id"], "action-1"); + assert_eq!(steps[1]["data"]["task_id"], "task-1"); + assert_eq!(steps[4]["data"]["status"], "completed"); + assert_eq!(steps[5]["data"]["task_id"], "task-1"); + assert_eq!(steps[6]["data"]["tasks"].as_array().unwrap().len(), 0); + } + + #[test] + fn runtime_script_does_not_preserve_tasks_across_cli_invocations() { + let temp = tempfile::tempdir().unwrap(); + let data_dir = temp.path().join("getter-data"); + let create_script = temp.path().join("create-runtime-task.json"); + fs::write( + &create_script, + serde_json::to_vec_pretty(&json!({ + "steps": [ + { + "operation": "issue_action", + "plan": { + "package_id": "android/org.fdroid.fdroid", + "actions": [], + "lua_object": { + "object_id": "debug:android/org.fdroid.fdroid", + "dependency_digest": "debug-digest" + } + } + }, + { "operation": "submit_action" } + ] + })) + .unwrap(), + ) + .unwrap(); + let list_script = temp.path().join("list-runtime-tasks.json"); + fs::write( + &list_script, + serde_json::to_vec_pretty(&json!({ + "steps": [{ "operation": "task_list" }] + })) + .unwrap(), + ) + .unwrap(); + + let create = run([ + "getter".to_owned(), + "--data-dir".to_owned(), + data_dir.to_string_lossy().to_string(), + "runtime".to_owned(), + "script".to_owned(), + "--script".to_owned(), + create_script.to_string_lossy().to_string(), + ]); + assert_eq!(create.exit_code, ExitCode::Success); + + let list = run([ + "getter".to_owned(), + "--data-dir".to_owned(), + data_dir.to_string_lossy().to_string(), + "runtime".to_owned(), + "script".to_owned(), + "--script".to_owned(), + list_script.to_string_lossy().to_string(), + ]); + assert_eq!(list.exit_code, ExitCode::Success); + let json: Value = serde_json::from_str(&list.stdout).unwrap(); + assert_eq!( + json["data"]["steps"][0]["data"]["tasks"] + .as_array() + .unwrap() + .len(), + 0 + ); + } + #[test] fn run_init_creates_sqlite_database_files_and_json_envelope() { let temp = tempfile::tempdir().unwrap(); diff --git a/crates/getter-cli/tests/bdd_cli.rs b/crates/getter-cli/tests/bdd_cli.rs index 733827f..c721b01 100644 --- a/crates/getter-cli/tests/bdd_cli.rs +++ b/crates/getter-cli/tests/bdd_cli.rs @@ -16,6 +16,7 @@ struct CliWorld { autogen_preview: Option, update_fixture: Option, task_request: Option, + runtime_script: Option, remembered_task_id: Option, remembered_event_cursor: Option, remembered_handoff_id: Option, @@ -303,6 +304,44 @@ fn malformed_offline_download_request(world: &mut CliWorld) { world.task_request = Some(request); } +#[given("a runtime script that submits completes removes and cleans a task")] +fn runtime_script_submits_completes_removes_and_cleans_task(world: &mut CliWorld) { + let temp = world.temp.as_ref().expect("tempdir exists"); + let script = temp.path().join("runtime-script.json"); + fs::write( + &script, + serde_json::to_vec_pretty(&serde_json::json!({ + "steps": [ + { + "operation": "issue_action", + "plan": { + "package_id": "android/org.fdroid.fdroid", + "actions": [ + { + "type": "download", + "url": "https://example.invalid/app.apk", + "file_name": "app.apk" + } + ], + "lua_object": { + "object_id": "debug:android/org.fdroid.fdroid", + "dependency_digest": "debug-digest" + } + } + }, + { "operation": "submit_action" }, + { "operation": "task_start" }, + { "operation": "task_complete_download" }, + { "operation": "task_remove" }, + { "operation": "task_clean", "payload": { "mode": "all_inactive" } } + ] + })) + .expect("runtime script serializes"), + ) + .expect("write runtime script"); + world.runtime_script = Some(script); +} + #[given(expr = "a fixture Lua repository {string} with package {string}")] fn fixture_lua_repository(world: &mut CliWorld, repo_id: String, package_id: String) { create_fixture_lua_repository(world, repo_id, package_id, "F-Droid".to_owned()); @@ -562,13 +601,14 @@ fn run_getter_version_unpin(world: &mut CliWorld, package_id: String) { world.json = None; } -#[when("I run getter task submit for that request")] -fn run_getter_task_submit(world: &mut CliWorld) { +#[when("I run getter debug fake-task submit for that request")] +fn run_getter_debug_fake_task_submit(world: &mut CliWorld) { let request = world.task_request.as_ref().expect("task request exists"); let output = run_getter( world, [ - "task".to_owned(), + "debug".to_owned(), + "fake-task".to_owned(), "submit".to_owned(), "--request".to_owned(), request.to_string_lossy().to_string(), @@ -578,43 +618,67 @@ fn run_getter_task_submit(world: &mut CliWorld) { world.json = None; } -#[when("I run getter task list")] -fn run_getter_task_list(world: &mut CliWorld) { - let output = run_getter(world, ["task".to_owned(), "list".to_owned()]); +#[when("I run getter debug fake-task list")] +fn run_getter_debug_fake_task_list(world: &mut CliWorld) { + let output = run_getter( + world, + [ + "debug".to_owned(), + "fake-task".to_owned(), + "list".to_owned(), + ], + ); world.output = Some(output); world.json = None; } -#[when("I run getter task cancel for the remembered task")] -fn run_getter_task_cancel(world: &mut CliWorld) { +#[when("I run getter debug fake-task cancel for the remembered task")] +fn run_getter_debug_fake_task_cancel(world: &mut CliWorld) { let task_id = world .remembered_task_id .as_ref() .expect("remembered task id exists") .clone(); - let output = run_getter(world, ["task".to_owned(), "cancel".to_owned(), task_id]); + let output = run_getter( + world, + [ + "debug".to_owned(), + "fake-task".to_owned(), + "cancel".to_owned(), + task_id, + ], + ); world.output = Some(output); world.json = None; } -#[when("I run getter task run for the remembered task")] -fn run_getter_task_run(world: &mut CliWorld) { +#[when("I run getter debug fake-task run for the remembered task")] +fn run_getter_debug_fake_task_run(world: &mut CliWorld) { let task_id = world .remembered_task_id .as_ref() .expect("remembered task id exists") .clone(); - let output = run_getter(world, ["task".to_owned(), "run".to_owned(), task_id]); + let output = run_getter( + world, + [ + "debug".to_owned(), + "fake-task".to_owned(), + "run".to_owned(), + task_id, + ], + ); world.output = Some(output); world.json = None; } -#[when(expr = "I run getter task events after {int} limit {int}")] -fn run_getter_task_events_after_limit(world: &mut CliWorld, after: u64, limit: u64) { +#[when(expr = "I run getter debug fake-task events after {int} limit {int}")] +fn run_getter_debug_fake_task_events_after_limit(world: &mut CliWorld, after: u64, limit: u64) { let output = run_getter( world, [ - "task".to_owned(), + "debug".to_owned(), + "fake-task".to_owned(), "events".to_owned(), "--after".to_owned(), after.to_string(), @@ -626,16 +690,16 @@ fn run_getter_task_events_after_limit(world: &mut CliWorld, after: u64, limit: u world.json = None; } -#[when(expr = "I run getter task events after the remembered cursor limit {int}")] -fn run_getter_task_events_after_remembered_cursor(world: &mut CliWorld, limit: u64) { +#[when(expr = "I run getter debug fake-task events after the remembered cursor limit {int}")] +fn run_getter_debug_fake_task_events_after_remembered_cursor(world: &mut CliWorld, limit: u64) { let after = world .remembered_event_cursor .expect("remembered event cursor exists"); - run_getter_task_events_after_limit(world, after, limit); + run_getter_debug_fake_task_events_after_limit(world, after, limit); } -#[when(expr = "I run getter task install-result {string} for the remembered handoff")] -fn run_getter_task_install_result(world: &mut CliWorld, status: String) { +#[when(expr = "I run getter debug fake-task install-result {string} for the remembered handoff")] +fn run_getter_debug_fake_task_install_result(world: &mut CliWorld, status: String) { let handoff_id = world .remembered_handoff_id .as_ref() @@ -644,7 +708,8 @@ fn run_getter_task_install_result(world: &mut CliWorld, status: String) { let output = run_getter( world, [ - "task".to_owned(), + "debug".to_owned(), + "fake-task".to_owned(), "install-result".to_owned(), handoff_id, "--status".to_owned(), @@ -655,6 +720,25 @@ fn run_getter_task_install_result(world: &mut CliWorld, status: String) { world.json = None; } +#[when("I run getter runtime script for that script")] +fn run_getter_runtime_script(world: &mut CliWorld) { + let script = world + .runtime_script + .as_ref() + .expect("runtime script exists"); + let output = run_getter( + world, + [ + "runtime".to_owned(), + "script".to_owned(), + "--script".to_owned(), + script.to_string_lossy().to_string(), + ], + ); + world.output = Some(output); + world.json = None; +} + #[when("I run getter autogen installed preview for that inventory")] fn run_getter_autogen_installed_preview(world: &mut CliWorld) { let inventory = world.inventory.as_ref().expect("inventory exists"); @@ -892,7 +976,7 @@ fn output_contains_empty_repository_list(world: &mut CliWorld) { #[then("I remember the submitted task id")] fn remember_submitted_task_id(world: &mut CliWorld) { let json = current_json(world); - assert_eq!(json["command"], "task submit"); + assert_eq!(json["command"], "debug fake-task submit"); let task_id = json["data"]["task"]["id"] .as_str() .expect("task id should be a string") @@ -900,6 +984,20 @@ fn remember_submitted_task_id(world: &mut CliWorld) { world.remembered_task_id = Some(task_id); } +#[then("the runtime script output removes the completed task")] +fn runtime_script_output_removes_completed_task(world: &mut CliWorld) { + let json = current_json(world); + assert_eq!(json["command"], "runtime script"); + let steps = json["data"]["steps"].as_array().expect("runtime steps"); + assert_eq!(steps[0]["data"]["action_id"], "action-1"); + assert_eq!(steps[1]["data"]["task_id"], "task-1"); + assert_eq!(steps[3]["data"]["status"], "completed"); + assert_eq!(steps[4]["operation"], "task_remove"); + assert_eq!(steps[4]["data"]["task_id"], "task-1"); + assert_eq!(steps[5]["operation"], "task_clean"); + assert_eq!(steps[5]["data"]["tasks"].as_array().unwrap().len(), 0); +} + #[then(expr = "the task list contains the remembered task with status {string}")] fn task_list_contains_remembered_task_with_status(world: &mut CliWorld, status: String) { let task_id = world @@ -908,7 +1006,7 @@ fn task_list_contains_remembered_task_with_status(world: &mut CliWorld, status: .expect("remembered task id exists") .clone(); let json = current_json(world); - assert_eq!(json["command"], "task list"); + assert_eq!(json["command"], "debug fake-task list"); let tasks = json["data"]["tasks"].as_array().expect("tasks array"); let task = tasks .iter() @@ -920,7 +1018,7 @@ fn task_list_contains_remembered_task_with_status(world: &mut CliWorld, status: #[then(expr = "the task cancel result has status {string} and changed true")] fn task_cancel_result_changed_true(world: &mut CliWorld, status: String) { let json = current_json(world); - assert_eq!(json["command"], "task cancel"); + assert_eq!(json["command"], "debug fake-task cancel"); assert_eq!(json["data"]["status"], status); assert_eq!(json["data"]["changed"], true); } @@ -928,7 +1026,7 @@ fn task_cancel_result_changed_true(world: &mut CliWorld, status: String) { #[then(expr = "the task cancel result has status {string} and changed false")] fn task_cancel_result_changed_false(world: &mut CliWorld, status: String) { let json = current_json(world); - assert_eq!(json["command"], "task cancel"); + assert_eq!(json["command"], "debug fake-task cancel"); assert_eq!(json["data"]["status"], status); assert_eq!(json["data"]["changed"], false); } @@ -940,7 +1038,7 @@ fn task_run_result_has_status_and_install_handoff( handoff_status: String, ) { let json = current_json(world); - assert_eq!(json["command"], "task run"); + assert_eq!(json["command"], "debug fake-task run"); assert_eq!(json["data"]["task"]["status"], status); assert_eq!(json["data"]["install_handoff"]["status"], handoff_status); } @@ -948,7 +1046,7 @@ fn task_run_result_has_status_and_install_handoff( #[then(expr = "the task events output contains {int} events and has more events")] fn task_events_output_contains_events_and_has_more(world: &mut CliWorld, count: usize) { let json = current_json(world); - assert_eq!(json["command"], "task events"); + assert_eq!(json["command"], "debug fake-task events"); assert_eq!(json["data"]["events"].as_array().unwrap().len(), count); assert_eq!(json["data"]["has_more"], true); } @@ -989,7 +1087,7 @@ fn remember_install_handoff_id(world: &mut CliWorld) { #[then(expr = "the install result output has status {string}")] fn install_result_output_has_status(world: &mut CliWorld, status: String) { let json = current_json(world); - assert_eq!(json["command"], "task install-result"); + assert_eq!(json["command"], "debug fake-task install-result"); assert_eq!(json["data"]["handoff"]["status"], status); } diff --git a/crates/getter-cli/tests/features/cli/task_lifecycle.feature b/crates/getter-cli/tests/features/cli/task_lifecycle.feature index 6927548..34f71f9 100644 --- a/crates/getter-cli/tests/features/cli/task_lifecycle.feature +++ b/crates/getter-cli/tests/features/cli/task_lifecycle.feature @@ -1,89 +1,97 @@ -@getter-cli @task -Feature: Offline task lifecycle +@getter-cli @debug-fake-task +Feature: Offline fake task lifecycle + Scenario: Runtime script exercises ADR-0011 in-memory remove and clean + Given an initialized getter data directory + And a runtime script that submits completes removes and cleans a task + When I run getter runtime script for that script + Then the command succeeds + And the output is valid JSON + And the runtime script output removes the completed task + Scenario: User submits and lists an offline fake download task Given an initialized getter data directory And an offline download request for package "android/org.fdroid.fdroid" - When I run getter task submit for that request + When I run getter debug fake-task submit for that request Then the command succeeds And the output is valid JSON And I remember the submitted task id - When I run getter task list + When I run getter debug fake-task list Then the command succeeds And the task list contains the remembered task with status "queued" Scenario: User cancels a queued offline task Given an initialized getter data directory And an offline download request for package "android/org.fdroid.fdroid" - When I run getter task submit for that request + When I run getter debug fake-task submit for that request Then the command succeeds And I remember the submitted task id - When I run getter task cancel for the remembered task + When I run getter debug fake-task cancel for the remembered task Then the command succeeds And the task cancel result has status "canceled" and changed true - When I run getter task cancel for the remembered task + When I run getter debug fake-task cancel for the remembered task Then the command succeeds And the task cancel result has status "canceled" and changed false Scenario: User cannot cancel a succeeded offline task Given an initialized getter data directory And an offline download request for package "android/org.fdroid.fdroid" - When I run getter task submit for that request + When I run getter debug fake-task submit for that request Then the command succeeds And I remember the submitted task id - When I run getter task run for the remembered task + When I run getter debug fake-task run for the remembered task Then the command succeeds And the task run result has status "succeeded" and install handoff "requested" - When I run getter task cancel for the remembered task + When I run getter debug fake-task cancel for the remembered task Then the command fails with a download task error Scenario: User polls offline task events with cursor and limit Given an initialized getter data directory And an offline download request for package "android/org.fdroid.fdroid" - When I run getter task submit for that request + When I run getter debug fake-task submit for that request Then the command succeeds And I remember the submitted task id - When I run getter task run for the remembered task + When I run getter debug fake-task run for the remembered task Then the command succeeds - When I run getter task events after 0 limit 2 + When I run getter debug fake-task events after 0 limit 2 Then the command succeeds And the task events output contains 2 events and has more events And I remember the next event cursor - When I run getter task events after the remembered cursor limit 10 + When I run getter debug fake-task events after the remembered cursor limit 10 Then the command succeeds And the task events output contains event "install_handoff_requested" Scenario: User records an offline install handoff result Given an initialized getter data directory And an offline download request for package "android/org.fdroid.fdroid" - When I run getter task submit for that request + When I run getter debug fake-task submit for that request Then the command succeeds And I remember the submitted task id - When I run getter task run for the remembered task + When I run getter debug fake-task run for the remembered task Then the command succeeds And I remember the install handoff id - When I run getter task install-result "succeeded" for the remembered handoff + When I run getter debug fake-task install-result "succeeded" for the remembered handoff Then the command succeeds And the install result output has status "succeeded" Scenario: User cannot record getter-created requested state as an install result Given an initialized getter data directory And an offline download request for package "android/org.fdroid.fdroid" - When I run getter task submit for that request + When I run getter debug fake-task submit for that request Then the command succeeds And I remember the submitted task id - When I run getter task run for the remembered task + When I run getter debug fake-task run for the remembered task Then the command succeeds And I remember the install handoff id - When I run getter task install-result "requested" for the remembered handoff + When I run getter debug fake-task install-result "requested" for the remembered handoff Then the command fails with a CLI usage error Scenario: User cannot poll task events with a zero limit Given an initialized getter data directory - When I run getter task events after 0 limit 0 + When I run getter debug fake-task events after 0 limit 0 Then the command fails with a CLI usage error Scenario: User receives structured errors for malformed task requests Given an initialized getter data directory And a malformed offline download request - When I run getter task submit for that request + When I run getter debug fake-task submit for that request Then the command fails with a download task error From 1cde4f4bf01c5f703724f4ca62cc89496886eedd Mon Sep 17 00:00:00 2001 From: Xiangzhe Date: Thu, 25 Jun 2026 13:55:29 +0800 Subject: [PATCH 26/52] feat(providers): wrap static updates as mock provider --- Cargo.lock | 1 + crates/getter-operations/Cargo.toml | 1 + crates/getter-operations/src/runtime.rs | 5 ++- crates/getter-providers/Cargo.toml | 2 +- crates/getter-providers/src/lib.rs | 54 ++++++++++++++++++++++++- 5 files changed, 60 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e033e93..eefb4be 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -565,6 +565,7 @@ name = "getter-operations" version = "0.1.0" dependencies = [ "getter-core", + "getter-providers", "getter-storage", "serde", "serde_json", diff --git a/crates/getter-operations/Cargo.toml b/crates/getter-operations/Cargo.toml index a36ba74..c81da6b 100644 --- a/crates/getter-operations/Cargo.toml +++ b/crates/getter-operations/Cargo.toml @@ -9,6 +9,7 @@ lua = ["getter-core/lua"] [dependencies] getter-core = { path = "../getter-core", default-features = false } +getter-providers = { path = "../getter-providers" } getter-storage = { path = "../getter-storage" } serde = { version = "1", features = ["derive"] } serde_json = "1" diff --git a/crates/getter-operations/src/runtime.rs b/crates/getter-operations/src/runtime.rs index f6bfa38..1392322 100644 --- a/crates/getter-operations/src/runtime.rs +++ b/crates/getter-operations/src/runtime.rs @@ -22,6 +22,8 @@ use getter_core::{ #[cfg(feature = "lua")] use getter_core::{update::check_updates_offline, update::UpdateSelectionPolicy, RepositoryId}; #[cfg(feature = "lua")] +use getter_providers::StaticPackageUpdatesProvider; +#[cfg(feature = "lua")] use getter_storage::{MainDb, StorageError}; use serde::Deserialize; use serde_json::{json, Value}; @@ -135,10 +137,11 @@ pub fn issue_action_from_registered_package_json( ) -> Result { let request: RegisteredPackageUpdateActionRequest = parse_request(request_json)?; let (package, dependency_digest) = evaluate_registered_package(db, &request)?; + let candidates = StaticPackageUpdatesProvider.check_updates(&package); let update = check_updates_offline( package.id.clone(), request.installed_version, - package.updates.clone(), + candidates, UpdateSelectionPolicy { pin_version: request.pin_version, }, diff --git a/crates/getter-providers/Cargo.toml b/crates/getter-providers/Cargo.toml index 5a44be9..f5ae23c 100644 --- a/crates/getter-providers/Cargo.toml +++ b/crates/getter-providers/Cargo.toml @@ -4,4 +4,4 @@ version.workspace = true edition.workspace = true [dependencies] -getter-core = { path = "../getter-core" } +getter-core = { path = "../getter-core", default-features = false } diff --git a/crates/getter-providers/src/lib.rs b/crates/getter-providers/src/lib.rs index 35371aa..ee714b9 100644 --- a/crates/getter-providers/src/lib.rs +++ b/crates/getter-providers/src/lib.rs @@ -1,3 +1,55 @@ -//! getter-providers rewrite crate skeleton. +//! Provider executor scaffolding for the UpgradeAll getter rewrite. +//! +//! Real network/provider execution is intentionally deferred to a later ADR. +//! The first Phase D bridge uses package-declared static update candidates as a +//! mock provider so the rest of the runtime can exercise getter-owned update +//! selection and opaque action issuance without direct network side effects. pub use getter_core as core; + +use getter_core::{ResolvedPackage, UpdateCandidate}; + +/// Mock provider that returns the static `updates` candidates materialized from +/// a resolved Lua package table. +/// +/// This is development scaffolding, not the final live provider model. Keeping +/// it behind a provider-shaped boundary prevents operation code from treating +/// `package.updates` as the product update-check architecture. +#[derive(Debug, Default, Clone, Copy)] +pub struct StaticPackageUpdatesProvider; + +impl StaticPackageUpdatesProvider { + pub fn check_updates(self, package: &ResolvedPackage) -> Vec { + package.updates.clone() + } +} + +#[cfg(test)] +mod tests { + use super::*; + use getter_core::PackagePermissions; + + #[test] + fn static_provider_returns_package_declared_update_candidates() { + let package = ResolvedPackage { + id: "android/org.fdroid.fdroid".parse().unwrap(), + name: "F-Droid".to_owned(), + repository: "official".parse().unwrap(), + installed: Vec::new(), + permissions: PackagePermissions::default(), + source_priority: Vec::new(), + updates: vec![UpdateCandidate { + version: "1.2.0".to_owned(), + channel: Some("stable".to_owned()), + source: Some("fixture".to_owned()), + artifacts: Vec::new(), + }], + }; + + let candidates = StaticPackageUpdatesProvider.check_updates(&package); + + assert_eq!(candidates.len(), 1); + assert_eq!(candidates[0].version, "1.2.0"); + assert_eq!(candidates[0].source.as_deref(), Some("fixture")); + } +} From 60a65158aed40e14f0dea17427b4a92ec1e43818 Mon Sep 17 00:00:00 2001 From: Xiangzhe Date: Thu, 25 Jun 2026 18:09:36 +0800 Subject: [PATCH 27/52] feat(operations): expose read model operations --- crates/getter-operations/src/lib.rs | 1 + crates/getter-operations/src/read_model.rs | 325 +++++++++++++++++++++ 2 files changed, 326 insertions(+) create mode 100644 crates/getter-operations/src/read_model.rs diff --git a/crates/getter-operations/src/lib.rs b/crates/getter-operations/src/lib.rs index 6d75c88..4bd9337 100644 --- a/crates/getter-operations/src/lib.rs +++ b/crates/getter-operations/src/lib.rs @@ -7,4 +7,5 @@ pub mod autogen; pub mod legacy_room; +pub mod read_model; pub mod runtime; diff --git a/crates/getter-operations/src/read_model.rs b/crates/getter-operations/src/read_model.rs new file mode 100644 index 0000000..898864b --- /dev/null +++ b/crates/getter-operations/src/read_model.rs @@ -0,0 +1,325 @@ +//! Getter-owned read-model JSON operations for product UI snapshots. +//! +//! Flutter and Android bridge code should request these operations instead of +//! fabricating app/repository state in the UI layer. The operations read the +//! getter storage/repository/Lua model and return DTO-shaped JSON for adapters +//! to parse and render. + +#[cfg(feature = "lua")] +use getter_core::lua::evaluate_package_file; +#[cfg(feature = "lua")] +use getter_core::repository::{RepositoryLayout, RepositoryLoadError}; +#[cfg(feature = "lua")] +use getter_core::{PackageId, RepositoryId}; +use getter_storage::{MainDb, StorageError, StoredRepository, StoredTrackedPackage}; +#[cfg(feature = "lua")] +use serde::Deserialize; +use serde_json::{json, Value}; +use std::path::{Path, PathBuf}; + +const MAIN_DB_FILE: &str = "main.db"; + +#[derive(Debug, thiserror::Error)] +pub enum ReadModelOperationError { + #[error("invalid read-model request: {0}")] + InvalidRequest(String), + #[error("storage operation failed: {0}")] + Storage(#[from] StorageError), + #[cfg(feature = "lua")] + #[error("repository operation failed: {0}")] + Repository(#[from] RepositoryLoadError), + #[cfg(feature = "lua")] + #[error("package evaluation failed: {0}")] + PackageEval(String), + #[error("read-model response serialization failed: {0}")] + Serialize(String), +} + +impl ReadModelOperationError { + pub fn code(&self) -> &'static str { + match self { + Self::InvalidRequest(_) => "read_model.invalid_request", + Self::Storage(_) => "storage.error", + #[cfg(feature = "lua")] + Self::Repository(_) => "repository.error", + #[cfg(feature = "lua")] + Self::PackageEval(_) => "package.eval_error", + Self::Serialize(_) => "read_model.serialize_error", + } + } + + pub fn message(&self) -> &'static str { + match self { + Self::InvalidRequest(_) => "Getter read-model request is invalid", + Self::Storage(_) => "Getter storage operation failed", + #[cfg(feature = "lua")] + Self::Repository(_) => "Getter repository operation failed", + #[cfg(feature = "lua")] + Self::PackageEval(_) => "Getter package evaluation failed", + Self::Serialize(_) => "Getter read-model response serialization failed", + } + } + + pub fn detail(&self) -> Option { + match self { + Self::InvalidRequest(detail) | Self::Serialize(detail) => Some(detail.clone()), + Self::Storage(error) => Some(error.to_string()), + #[cfg(feature = "lua")] + Self::Repository(error) => Some(error.to_string()), + #[cfg(feature = "lua")] + Self::PackageEval(detail) => Some(detail.clone()), + } + } +} + +pub fn repository_list_json(data_dir: &Path) -> Result { + let db = open_main_db(data_dir)?; + Ok(json!({ + "repositories": db + .repositories()? + .into_iter() + .map(repository_json) + .collect::>(), + })) +} + +pub fn tracked_package_list_json(data_dir: &Path) -> Result { + let db = open_main_db(data_dir)?; + Ok(json!({ + "packages": db + .tracked_packages()? + .into_iter() + .map(tracked_package_json) + .collect::>(), + })) +} + +#[cfg(feature = "lua")] +pub fn package_eval_json( + data_dir: &Path, + request_json: &str, +) -> Result { + let request: PackageEvalRequest = parse_request(request_json)?; + let db = open_main_db(data_dir)?; + let package = match request.repository_id { + Some(repository_id) => { + evaluate_package_from_repo(&db, &repository_id, &request.package_id)? + } + None => evaluate_highest_priority_package(&db, &request.package_id)?, + }; + let package = serde_json::to_value(package) + .map_err(|source| ReadModelOperationError::Serialize(source.to_string()))?; + Ok(json!({ "package": package })) +} + +#[cfg(feature = "lua")] +#[derive(Debug, Deserialize)] +struct PackageEvalRequest { + package_id: PackageId, + #[serde(default)] + repository_id: Option, +} + +#[cfg(feature = "lua")] +fn parse_request(request_json: &str) -> Result +where + T: for<'de> Deserialize<'de>, +{ + serde_json::from_str(request_json) + .map_err(|source| ReadModelOperationError::InvalidRequest(source.to_string())) +} + +fn open_main_db(data_dir: &Path) -> Result { + Ok(MainDb::open(main_db_path(data_dir))?) +} + +#[cfg(feature = "lua")] +fn evaluate_package_from_repo( + db: &MainDb, + repo_id: &RepositoryId, + package_id: &PackageId, +) -> Result { + let repo = find_repository(db, repo_id)?; + let path = repo_path(&repo)?; + let layout = RepositoryLayout::load(&path)?; + let package_file = layout.package_file(package_id).ok_or_else(|| { + ReadModelOperationError::PackageEval(format!( + "package '{}' was not found in repository '{}'", + package_id, repo_id + )) + })?; + evaluate_package_file(&layout, &package_file.path) + .map_err(|source| ReadModelOperationError::PackageEval(source.to_string())) +} + +#[cfg(feature = "lua")] +fn evaluate_highest_priority_package( + db: &MainDb, + package_id: &PackageId, +) -> Result { + for repo in db.repositories()? { + let path = repo_path(&repo)?; + let layout = RepositoryLayout::load(&path)?; + if let Some(package_file) = layout.package_file(package_id) { + return evaluate_package_file(&layout, &package_file.path) + .map_err(|source| ReadModelOperationError::PackageEval(source.to_string())); + } + } + Err(ReadModelOperationError::PackageEval(format!( + "package '{package_id}' was not found in any registered repository" + ))) +} + +#[cfg(feature = "lua")] +fn find_repository( + db: &MainDb, + id: &RepositoryId, +) -> Result { + db.repositories()? + .into_iter() + .find(|repo| &repo.id == id) + .ok_or_else(|| { + ReadModelOperationError::PackageEval(format!("repository '{id}' is not registered")) + }) +} + +#[cfg(feature = "lua")] +fn repo_path(repo: &StoredRepository) -> Result { + repo.path.as_ref().map(PathBuf::from).ok_or_else(|| { + ReadModelOperationError::PackageEval(format!("repository '{}' has no path", repo.id)) + }) +} + +fn repository_json(repo: StoredRepository) -> Value { + json!({ + "id": repo.id.as_str(), + "name": repo.name, + "priority": repo.priority.value(), + "api_version": repo.api_version, + "path": repo.path, + "revision": repo.revision, + }) +} + +fn tracked_package_json(package: StoredTrackedPackage) -> Value { + json!({ + "id": package.package_id.to_string(), + "enabled": package.enabled, + "favorite": package.favorite, + "pin_version": package.pin_version, + "repository_id": package.repository_id.map(|id| id.to_string()), + "package_resolution": package.package_resolution.as_str(), + }) +} + +fn main_db_path(data_dir: &Path) -> PathBuf { + data_dir.join(MAIN_DB_FILE) +} + +#[cfg(test)] +mod tests { + use super::*; + use getter_core::repository::{RepositoryMetadata, REPO_API_VERSION_V1}; + use getter_core::{RepositoryId, RepositoryPriority}; + use getter_storage::{StoredPackageResolution, TrackedPackageUpsert}; + #[cfg(feature = "lua")] + use std::fs; + use tempfile::tempdir; + + #[test] + fn repository_and_tracked_package_lists_use_getter_storage_shapes() { + let temp = tempdir().unwrap(); + let db = MainDb::open(temp.path().join(MAIN_DB_FILE)).unwrap(); + let repo_id: RepositoryId = "official".parse().unwrap(); + db.upsert_repository( + &RepositoryMetadata { + id: repo_id.clone(), + name: "Official".to_owned(), + priority: RepositoryPriority::new(10), + api_version: REPO_API_VERSION_V1.to_owned(), + }, + Some(Path::new("/tmp/official")), + Some("rev-1"), + ) + .unwrap(); + db.upsert_tracked_package(&TrackedPackageUpsert { + package_id: "android/org.fdroid.fdroid".parse().unwrap(), + enabled: true, + favorite: true, + pin_version: Some("1.2.3".to_owned()), + repository_id: Some(repo_id), + package_resolution: StoredPackageResolution::OfficialRepositoryPackage, + }) + .unwrap(); + + let repositories = repository_list_json(temp.path()).unwrap(); + assert_eq!(repositories["repositories"][0]["id"], "official"); + assert_eq!(repositories["repositories"][0]["name"], "Official"); + assert_eq!(repositories["repositories"][0]["priority"], 10); + assert_eq!(repositories["repositories"][0]["revision"], "rev-1"); + + let packages = tracked_package_list_json(temp.path()).unwrap(); + assert_eq!(packages["packages"][0]["id"], "android/org.fdroid.fdroid"); + assert_eq!(packages["packages"][0]["favorite"], true); + assert_eq!(packages["packages"][0]["pin_version"], "1.2.3"); + assert_eq!(packages["packages"][0]["repository_id"], "official"); + assert_eq!( + packages["packages"][0]["package_resolution"], + "official_repository_package" + ); + } + + #[cfg(feature = "lua")] + #[test] + fn package_eval_reads_registered_lua_repository() { + let temp = tempdir().unwrap(); + let repo_path = temp.path().join("repo"); + fs::create_dir_all(repo_path.join("packages/android")).unwrap(); + fs::create_dir_all(repo_path.join("lib")).unwrap(); + fs::create_dir_all(repo_path.join("templates")).unwrap(); + fs::write( + repo_path.join("repo.toml"), + r#"id = "official" +name = "Official" +priority = 10 +api_version = "getter.repo.v1" +"#, + ) + .unwrap(); + fs::write( + repo_path.join("packages/android/org.fdroid.fdroid.lua"), + r#"return package_def { + id = "android/org.fdroid.fdroid", + name = "F-Droid", + installed = { + { kind = "android_package", package_name = "org.fdroid.fdroid" }, + }, + permissions = { free_network = true }, +} +"#, + ) + .unwrap(); + + let db = MainDb::open(temp.path().join(MAIN_DB_FILE)).unwrap(); + db.upsert_repository( + &RepositoryMetadata { + id: "official".parse().unwrap(), + name: "Official".to_owned(), + priority: RepositoryPriority::new(10), + api_version: REPO_API_VERSION_V1.to_owned(), + }, + Some(&repo_path), + None, + ) + .unwrap(); + + let result = package_eval_json( + temp.path(), + r#"{"package_id":"android/org.fdroid.fdroid","repository_id":"official"}"#, + ) + .unwrap(); + assert_eq!(result["package"]["id"], "android/org.fdroid.fdroid"); + assert_eq!(result["package"]["name"], "F-Droid"); + assert_eq!(result["package"]["permissions"]["free_network"], true); + } +} From 3d268807a1a23e38789434a7e0a0659692ae8897 Mon Sep 17 00:00:00 2001 From: Xiangzhe Date: Fri, 26 Jun 2026 18:26:34 +0800 Subject: [PATCH 28/52] feat(repository): load ADR-0012 data-dir config --- Cargo.lock | 7 + crates/getter-cli/src/lib.rs | 54 ++- crates/getter-cli/tests/bdd_cli.rs | 60 +-- .../features/cli/autogen_installed.feature | 10 +- crates/getter-core/Cargo.toml | 1 + crates/getter-core/src/autogen.rs | 62 +-- crates/getter-core/src/diagnostics.rs | 29 ++ crates/getter-core/src/lib.rs | 13 +- crates/getter-core/src/repository.rs | 388 +++++++++++++++++- crates/getter-operations/src/autogen.rs | 284 ++++++++++--- crates/getter-storage/src/lib.rs | 30 +- 11 files changed, 792 insertions(+), 146 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index eefb4be..23480e3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -536,6 +536,7 @@ dependencies = [ name = "getter-core" version = "0.1.0" dependencies = [ + "json_comments", "mlua", "serde", "serde_json", @@ -788,6 +789,12 @@ dependencies = [ "syn", ] +[[package]] +name = "json_comments" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9dbbfed4e59ba9750e15ba154fdfd9329cee16ff3df539c2666b70f58cc32105" + [[package]] name = "libc" version = "0.2.186" diff --git a/crates/getter-cli/src/lib.rs b/crates/getter-cli/src/lib.rs index d8317fe..020e1f9 100644 --- a/crates/getter-cli/src/lib.rs +++ b/crates/getter-cli/src/lib.rs @@ -7,7 +7,9 @@ use getter_core::autogen::InstalledInventory; use getter_core::diagnostics::validate_repository_path; use getter_core::lua::evaluate_package_file; -use getter_core::repository::{RepositoryLayout, RepositoryMetadata}; +use getter_core::repository::{ + GetterDataDirLayout, RepositoryLayout, RepositoryMetadata, REPOSITORY_ROOT_METADATA_FILE, +}; use getter_core::runtime::{GetterRuntime, SealedActionPlan}; use getter_core::task::{ DownloadTaskRequest, InstallHandoffStatus, TaskEventPage, DOWNLOAD_REQUEST_FORMAT, @@ -37,6 +39,18 @@ const MAIN_DB_FILE: &str = "main.db"; const CACHE_DB_FILE: &str = "cache.db"; const MIGRATION_REPORTS_DIR: &str = "migration-reports"; const LEGACY_ROOM_MIGRATION_ID: &str = "legacy-room-v17"; +const REPOSITORY_ROOT_METADATA_STARTER: &str = r#"{ + "version": 1, + // Autogen writes to "autogen" by default. Uncomment and change this + // if generated packages should target another existing repository alias. + // "generated_repository": "autogen", + "priority": { + "local": 100, + "official": 0, + "autogen": -1 + } +} +"#; #[derive(Debug, Clone, PartialEq, Eq)] pub struct CliInvocation { @@ -264,6 +278,9 @@ impl From for CliError { match value { AutogenOperationError::Storage(source) => Self::Storage(source.to_string()), AutogenOperationError::Repository(detail) => Self::Repository(detail), + AutogenOperationError::MissingGeneratedRepository { .. } => { + Self::Autogen(value.to_string()) + } AutogenOperationError::Autogen(detail) => Self::Autogen(detail), } } @@ -547,10 +564,14 @@ fn execute(invocation: CliInvocation) -> Result { match invocation.command { CliCommand::Init => { initialize_storage(&invocation.data_dir)?; + let layout = initialize_data_dir_layout(&invocation.data_dir)?; Ok(json!({ - "data_dir": invocation.data_dir, - "main_db": main_db_path(&invocation.data_dir), - "cache_db": cache_db_path(&invocation.data_dir), + "data_dir": layout.root, + "main_db": layout.main_db, + "cache_db": layout.cache_db, + "repo": layout.repository_root, + "rc": layout.runtime_config_root, + "repo_metadata": layout.repository_root.join(REPOSITORY_ROOT_METADATA_FILE), })) } CliCommand::AppList => { @@ -691,8 +712,9 @@ fn execute(invocation: CliInvocation) -> Result { CliCommand::AutogenInstalledPreview { inventory } => { let db = open_main_db(&invocation.data_dir)?; let inventory = read_installed_inventory(&inventory)?; - let plan = autogen::build_local_autogen_plan(&db, &inventory)?; - Ok(autogen::installed_preview_json(&invocation.data_dir, &plan)) + let plan = + autogen::build_installed_autogen_plan(&invocation.data_dir, &db, &inventory)?; + autogen::installed_preview_json(&invocation.data_dir, &plan).map_err(CliError::from) } CliCommand::AutogenInstalledApply { preview, @@ -807,6 +829,26 @@ fn initialize_storage(data_dir: &Path) -> Result<(), CliError> { Ok(()) } +fn initialize_data_dir_layout(data_dir: &Path) -> Result { + let layout = GetterDataDirLayout::new(data_dir); + fs::create_dir_all(&layout.repository_root).map_err(|source| { + CliError::Storage(format!("failed to create repository root: {source}")) + })?; + fs::create_dir_all(&layout.runtime_config_root).map_err(|source| { + CliError::Storage(format!("failed to create runtime config root: {source}")) + })?; + let metadata_path = layout.repository_root.join(REPOSITORY_ROOT_METADATA_FILE); + if !metadata_path.exists() { + fs::write(&metadata_path, REPOSITORY_ROOT_METADATA_STARTER).map_err(|source| { + CliError::Storage(format!( + "failed to write repository root metadata '{}': {source}", + metadata_path.display() + )) + })?; + } + Ok(layout) +} + fn open_initialized_storage(data_dir: &Path) -> Result<(), CliError> { initialize_storage(data_dir) } diff --git a/crates/getter-cli/tests/bdd_cli.rs b/crates/getter-cli/tests/bdd_cli.rs index c721b01..073bb63 100644 --- a/crates/getter-cli/tests/bdd_cli.rs +++ b/crates/getter-cli/tests/bdd_cli.rs @@ -167,8 +167,8 @@ fn tampered_autogen_cleanup_preview_for_package(world: &mut CliWorld, package_id &preview, serde_json::to_vec_pretty(&serde_json::json!({ "operation": "cleanup.preview", - "target_repo_id": "local_autogen", - "target_repo_path": local_autogen_repo_path(world), + "target_repo_id": "autogen", + "target_repo_path": autogen_repo_path(world), "summary": { "candidate_count": 1, "skipped_count": 0, @@ -815,9 +815,9 @@ fn run_getter_autogen_cleanup_apply_accept_all(world: &mut CliWorld) { world.json = None; } -#[when("I run getter repo validate for local_autogen")] -fn run_getter_repo_validate_for_local_autogen(world: &mut CliWorld) { - let repo_path = local_autogen_repo_path(world); +#[when("I run getter repo validate for autogen")] +fn run_getter_repo_validate_for_autogen(world: &mut CliWorld) { + let repo_path = autogen_repo_path(world); let output = run_getter( world, [ @@ -830,8 +830,8 @@ fn run_getter_repo_validate_for_local_autogen(world: &mut CliWorld) { world.json = None; } -#[when(expr = "I run getter package eval for package {string} from local_autogen")] -fn run_getter_package_eval_from_local_autogen(world: &mut CliWorld, package_id: String) { +#[when(expr = "I run getter package eval for package {string} from autogen")] +fn run_getter_package_eval_from_autogen(world: &mut CliWorld, package_id: String) { let output = run_getter( world, [ @@ -839,7 +839,7 @@ fn run_getter_package_eval_from_local_autogen(world: &mut CliWorld, package_id: "eval".to_owned(), package_id, "--repo".to_owned(), - "local_autogen".to_owned(), + "autogen".to_owned(), ], ); world.output = Some(output); @@ -944,6 +944,13 @@ fn getter_data_directory_is_usable(world: &mut CliWorld) { let data_dir = world.data_dir.as_ref().expect("data dir exists"); assert!(data_dir.join("main.db").is_file(), "main.db should exist"); assert!(data_dir.join("cache.db").is_file(), "cache.db should exist"); + assert!(data_dir.join("repo").is_dir(), "repo root should exist"); + assert!(data_dir.join("rc").is_dir(), "rc root should exist"); + let repo_metadata = data_dir.join("repo/metadata.jsonc"); + assert!(repo_metadata.is_file(), "repo metadata should exist"); + let metadata = fs::read_to_string(repo_metadata).expect("repo metadata readable"); + assert!(metadata.contains("\"autogen\": -1")); + assert!(metadata.contains("// \"generated_repository\": \"autogen\"")); let output = run_getter(world, ["app".to_owned(), "list".to_owned()]); assert_success(&output); @@ -1259,11 +1266,11 @@ fn autogen_preview_contains_candidate(world: &mut CliWorld, package_id: String) ); } -#[then("the local_autogen repository has not been written")] -fn local_autogen_repository_has_not_been_written(world: &mut CliWorld) { +#[then("the autogen repository has not been written")] +fn autogen_repository_has_not_been_written(world: &mut CliWorld) { assert!( - !local_autogen_repo_path(world).exists(), - "preview must not create local_autogen" + !autogen_repo_path(world).exists(), + "preview must not create autogen repository" ); } @@ -1283,12 +1290,12 @@ fn save_autogen_preview_to_file(world: &mut CliWorld) { world.autogen_preview = Some(preview); } -#[then(expr = "the local_autogen repository contains generated package {string}")] -fn local_autogen_repository_contains_generated_package(world: &mut CliWorld, package_id: String) { - let path = local_autogen_repo_path(world).join(package_relative_path(&package_id)); +#[then(expr = "the autogen repository contains generated package {string}")] +fn autogen_repository_contains_generated_package(world: &mut CliWorld, package_id: String) { + let path = autogen_repo_path(world).join(package_relative_path(&package_id)); assert!(path.is_file(), "generated package should exist: {path:?}"); let content = fs::read_to_string(&path).expect("generated package readable"); - assert!(content.contains("@generated by UpgradeAll getter local_autogen")); + assert!(content.contains("@generated by UpgradeAll getter autogen")); } #[then(expr = "the app list contains autogen tracked package {string}")] @@ -1301,7 +1308,7 @@ fn app_list_contains_autogen_tracked_package(world: &mut CliWorld, package_id: S .iter() .find(|app| app["id"].as_str() == Some(package_id.as_str())) .unwrap_or_else(|| panic!("app list should contain {package_id}: {apps:?}")); - assert_eq!(app["repository_id"], "local_autogen"); + assert_eq!(app["repository_id"], "autogen"); assert_eq!(app["package_resolution"], "generate_local_package"); } @@ -1341,12 +1348,9 @@ fn autogen_cleanup_preview_contains_delete_candidate(world: &mut CliWorld, packa assert_eq!(candidate["action"], "delete"); } -#[then(expr = "the local_autogen repository does not contain generated package {string}")] -fn local_autogen_repository_does_not_contain_generated_package( - world: &mut CliWorld, - package_id: String, -) { - let path = local_autogen_repo_path(world).join(package_relative_path(&package_id)); +#[then(expr = "the autogen repository does not contain generated package {string}")] +fn autogen_repository_does_not_contain_generated_package(world: &mut CliWorld, package_id: String) { + let path = autogen_repo_path(world).join(package_relative_path(&package_id)); assert!( !path.exists(), "generated package should be deleted: {path:?}" @@ -1371,7 +1375,7 @@ fn replace_generated_autogen_package_with_user_edited_content( world: &mut CliWorld, package_id: String, ) { - let path = local_autogen_repo_path(world).join(package_relative_path(&package_id)); + let path = autogen_repo_path(world).join(package_relative_path(&package_id)); assert!(path.is_file(), "generated package should exist before edit"); fs::write( &path, @@ -1389,7 +1393,7 @@ fn local_repository_contains_preserved_package(world: &mut CliWorld, package_id: .data_dir .as_ref() .expect("data dir exists") - .join("repositories") + .join("repo") .join("local") .join(package_relative_path(&package_id)); assert!( @@ -1572,13 +1576,13 @@ fn current_json(world: &mut CliWorld) -> &Value { .get_or_insert_with(|| parse_stdout(world.output.as_ref().expect("command output exists"))) } -fn local_autogen_repo_path(world: &CliWorld) -> PathBuf { +fn autogen_repo_path(world: &CliWorld) -> PathBuf { world .data_dir .as_ref() .expect("data dir exists") - .join("repositories") - .join("local_autogen") + .join("repo") + .join("autogen") } fn package_relative_path(package_id: &str) -> PathBuf { diff --git a/crates/getter-cli/tests/features/cli/autogen_installed.feature b/crates/getter-cli/tests/features/cli/autogen_installed.feature index fc3c16a..5bbb8c9 100644 --- a/crates/getter-cli/tests/features/cli/autogen_installed.feature +++ b/crates/getter-cli/tests/features/cli/autogen_installed.feature @@ -7,7 +7,7 @@ Feature: Installed app autogen Then the command succeeds And the output is valid JSON And the autogen preview contains candidate "android/com.example.autogen" - And the local_autogen repository has not been written + And the autogen repository has not been written Scenario: User applies installed app autogen and evaluates the generated fallback package Given an initialized getter data directory @@ -17,11 +17,11 @@ Feature: Installed app autogen And I save the autogen preview to a file When I run getter autogen installed apply for that preview with accept-all Then the command succeeds - And the local_autogen repository contains generated package "android/com.example.autogen" + And the autogen repository contains generated package "android/com.example.autogen" And the app list contains autogen tracked package "android/com.example.autogen" - When I run getter repo validate for local_autogen + When I run getter repo validate for autogen Then the output reports a valid repository without network - When I run getter package eval for package "android/com.example.autogen" from local_autogen + When I run getter package eval for package "android/com.example.autogen" from autogen Then the output contains package "android/com.example.autogen" named "Example Autogen" Scenario: Higher-priority repositories suppress installed app autogen candidates @@ -49,7 +49,7 @@ Feature: Installed app autogen And I save the autogen preview to a file When I run getter autogen cleanup apply for that preview with accept-all Then the command succeeds - And the local_autogen repository does not contain generated package "android/com.example.old" + And the autogen repository does not contain generated package "android/com.example.old" And the app list does not contain package "android/com.example.old" Scenario: Cleanup rejects tampered previews for non-autogen tracked packages diff --git a/crates/getter-core/Cargo.toml b/crates/getter-core/Cargo.toml index 187fcf6..4eac396 100644 --- a/crates/getter-core/Cargo.toml +++ b/crates/getter-core/Cargo.toml @@ -8,6 +8,7 @@ default = ["lua"] lua = ["dep:mlua"] [dependencies] +json_comments = "0.2" mlua = { version = "0.10", features = ["luajit", "vendored"], optional = true } serde = { version = "1", features = ["derive"] } serde_json = "1" diff --git a/crates/getter-core/src/autogen.rs b/crates/getter-core/src/autogen.rs index 2001ac5..a3bfe8e 100644 --- a/crates/getter-core/src/autogen.rs +++ b/crates/getter-core/src/autogen.rs @@ -15,9 +15,9 @@ pub const INSTALLED_INVENTORY_VERSION: u32 = 1; pub const AUTOGEN_MANIFEST_VERSION: u32 = 1; pub const LOCAL_REPOSITORY_ID: &str = "local"; pub const LOCAL_REPOSITORY_NAME: &str = "Local"; -pub const LOCAL_AUTOGEN_REPOSITORY_ID: &str = "local_autogen"; -pub const LOCAL_AUTOGEN_REPOSITORY_NAME: &str = "UpgradeAll Local Autogen"; -pub const GENERATED_MARKER: &str = "@generated by UpgradeAll getter local_autogen"; +pub const DEFAULT_AUTOGEN_REPOSITORY_ID: &str = "autogen"; +pub const DEFAULT_AUTOGEN_REPOSITORY_NAME: &str = "UpgradeAll Autogen"; +pub const GENERATED_MARKER: &str = "@generated by UpgradeAll getter autogen"; #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct InstalledInventory { @@ -107,8 +107,8 @@ impl AutogenManifest { pub fn from_candidates(candidates: &[AutogenCandidate]) -> Self { Self { version: AUTOGEN_MANIFEST_VERSION, - repository_id: RepositoryId::new(LOCAL_AUTOGEN_REPOSITORY_ID) - .expect("local_autogen is a valid repository id"), + repository_id: RepositoryId::new(DEFAULT_AUTOGEN_REPOSITORY_ID) + .expect("autogen is a valid repository id"), packages: candidates .iter() .map(|candidate| AutogenManifestEntry { @@ -146,13 +146,13 @@ pub enum AutogenError { PackageId(#[from] crate::PackageIdError), } -/// Build a deterministic local_autogen preview from installed inventory. +/// Build a deterministic installed-autogen preview from installed inventory. /// /// `covered_packages` contains package ids already supplied by repositories with -/// priority higher than `local_autogen`. Those packages are skipped so generated -/// fallback files do not compete with user-authored `local` or upstream package -/// files. -pub fn plan_local_autogen( +/// priority higher than the generated repository. Those packages are skipped so +/// generated fallback files do not compete with user-authored `local` or +/// upstream package files. +pub fn plan_installed_autogen( inventory: &InstalledInventory, covered_packages: &HashMap, ) -> Result { @@ -186,10 +186,10 @@ pub fn plan_local_autogen( skipped.sort_by_key(|skip| skip.package_id.to_string()); Ok(AutogenPlan { - repository_id: RepositoryId::new(LOCAL_AUTOGEN_REPOSITORY_ID) - .expect("local_autogen is a valid repository id"), - repository_name: LOCAL_AUTOGEN_REPOSITORY_NAME.to_owned(), - repository_priority: RepositoryPriority::LOCAL_AUTOGEN, + repository_id: RepositoryId::new(DEFAULT_AUTOGEN_REPOSITORY_ID) + .expect("autogen is a valid repository id"), + repository_name: DEFAULT_AUTOGEN_REPOSITORY_NAME.to_owned(), + repository_priority: RepositoryPriority::GENERATED_FALLBACK, candidates, skipped, }) @@ -210,11 +210,11 @@ pub fn validate_installed_inventory(inventory: &InstalledInventory) -> Result<() Ok(()) } -pub fn local_autogen_repo_toml() -> String { +pub fn default_autogen_repo_toml() -> String { repo_toml( - LOCAL_AUTOGEN_REPOSITORY_ID, - LOCAL_AUTOGEN_REPOSITORY_NAME, - RepositoryPriority::LOCAL_AUTOGEN, + DEFAULT_AUTOGEN_REPOSITORY_ID, + DEFAULT_AUTOGEN_REPOSITORY_NAME, + RepositoryPriority::GENERATED_FALLBACK, ) } @@ -385,10 +385,13 @@ mod tests { version_code: Some(1020000), }]); - let plan = plan_local_autogen(&inventory, &HashMap::new()).unwrap(); + let plan = plan_installed_autogen(&inventory, &HashMap::new()).unwrap(); - assert_eq!(plan.repository_id.as_str(), LOCAL_AUTOGEN_REPOSITORY_ID); - assert_eq!(plan.repository_priority, RepositoryPriority::LOCAL_AUTOGEN); + assert_eq!(plan.repository_id.as_str(), DEFAULT_AUTOGEN_REPOSITORY_ID); + assert_eq!( + plan.repository_priority, + RepositoryPriority::GENERATED_FALLBACK + ); assert_eq!(plan.candidates.len(), 1); let candidate = &plan.candidates[0]; assert_eq!( @@ -416,7 +419,7 @@ mod tests { version_code: None, }]); - let plan = plan_local_autogen(&inventory, &covered).unwrap(); + let plan = plan_installed_autogen(&inventory, &covered).unwrap(); assert!(plan.candidates.is_empty()); assert_eq!(plan.skipped.len(), 1); @@ -450,7 +453,7 @@ mod tests { }, ]); - let plan = plan_local_autogen(&inventory, &HashMap::new()).unwrap(); + let plan = plan_installed_autogen(&inventory, &HashMap::new()).unwrap(); assert_eq!(plan.candidates.len(), 1); assert_eq!(plan.skipped.len(), 1); @@ -469,7 +472,7 @@ mod tests { }; assert!(matches!( - plan_local_autogen(&inventory, &HashMap::new()), + plan_installed_autogen(&inventory, &HashMap::new()), Err(AutogenError::UnsupportedInventoryFormat(_)) )); } @@ -482,12 +485,15 @@ mod tests { version_name: None, version_code: None, }]); - let plan = plan_local_autogen(&inventory, &HashMap::new()).unwrap(); + let plan = plan_installed_autogen(&inventory, &HashMap::new()).unwrap(); let manifest = AutogenManifest::from_candidates(&plan.candidates); assert_eq!(manifest.version, AUTOGEN_MANIFEST_VERSION); - assert_eq!(manifest.repository_id.as_str(), LOCAL_AUTOGEN_REPOSITORY_ID); + assert_eq!( + manifest.repository_id.as_str(), + DEFAULT_AUTOGEN_REPOSITORY_ID + ); assert_eq!(manifest.packages.len(), 1); assert_eq!( manifest.packages[0].content_hash, @@ -500,7 +506,7 @@ mod tests { fn generated_lua_evaluates_through_package_boundary() { let temp = tempfile::tempdir().unwrap(); let repo = temp.path(); - fs::write(repo.join("repo.toml"), local_autogen_repo_toml()).unwrap(); + fs::write(repo.join("repo.toml"), default_autogen_repo_toml()).unwrap(); fs::create_dir_all(repo.join("packages/android")).unwrap(); fs::create_dir(repo.join("lib")).unwrap(); fs::create_dir(repo.join("templates")).unwrap(); @@ -510,7 +516,7 @@ mod tests { version_name: None, version_code: None, }]); - let candidate = plan_local_autogen(&inventory, &HashMap::new()) + let candidate = plan_installed_autogen(&inventory, &HashMap::new()) .unwrap() .candidates .remove(0); diff --git a/crates/getter-core/src/diagnostics.rs b/crates/getter-core/src/diagnostics.rs index 62ed870..d338055 100644 --- a/crates/getter-core/src/diagnostics.rs +++ b/crates/getter-core/src/diagnostics.rs @@ -87,6 +87,35 @@ pub fn validate_repository_path(path: impl AsRef) -> RepositoryValidationR fn repository_load_diagnostic(error: RepositoryLoadError) -> PackageValidationDiagnostic { let (code, path, message) = match error { + RepositoryLoadError::ReadRootMetadata { path, source } => ( + "repository.read_root_metadata", + path, + format!("failed to read repository root metadata: {source}"), + ), + RepositoryLoadError::ParseRootMetadata { path, source } => ( + "repository.parse_root_metadata", + path, + format!("failed to parse repository root metadata: {source}"), + ), + RepositoryLoadError::UnsupportedRootMetadataVersion { + path, + found, + expected, + } => ( + "repository.unsupported_root_metadata_version", + path, + format!("unsupported repository root metadata version {found}; expected {expected}"), + ), + RepositoryLoadError::ReadRepositoryRoot { path, source } => ( + "repository.read_root", + path, + format!("failed to read repository root: {source}"), + ), + RepositoryLoadError::MissingGeneratedRepository { alias, path } => ( + "repository.missing_generated_repository", + path, + format!("configured generated repository '{alias}' does not exist"), + ), RepositoryLoadError::ReadRepoToml { path, source } => ( "repository.read_repo_toml", path, diff --git a/crates/getter-core/src/lib.rs b/crates/getter-core/src/lib.rs index b1d759e..f02c4b0 100644 --- a/crates/getter-core/src/lib.rs +++ b/crates/getter-core/src/lib.rs @@ -146,7 +146,7 @@ impl From for String { } /// Repository identifier such as `official`, `community`, `local`, or -/// `local_autogen`. +/// `autogen`. #[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] #[serde(try_from = "String", into = "String")] pub struct RepositoryId(String); @@ -206,7 +206,7 @@ pub struct RepositoryPriority(i32); impl RepositoryPriority { pub const LOCAL: Self = Self(100); pub const DEFAULT: Self = Self(0); - pub const LOCAL_AUTOGEN: Self = Self(-1); + pub const GENERATED_FALLBACK: Self = Self(-1); pub const fn new(value: i32) -> Self { Self(value) @@ -401,18 +401,15 @@ mod tests { #[test] fn repository_priority_higher_number_wins() { assert!(RepositoryPriority::LOCAL > RepositoryPriority::DEFAULT); - assert!(RepositoryPriority::DEFAULT > RepositoryPriority::LOCAL_AUTOGEN); + assert!(RepositoryPriority::DEFAULT > RepositoryPriority::GENERATED_FALLBACK); assert_eq!(RepositoryPriority::LOCAL.value(), 100); assert_eq!(RepositoryPriority::DEFAULT.value(), 0); - assert_eq!(RepositoryPriority::LOCAL_AUTOGEN.value(), -1); + assert_eq!(RepositoryPriority::GENERATED_FALLBACK.value(), -1); } #[test] fn repository_id_accepts_named_repositories() { - assert_eq!( - RepositoryId::new("local_autogen").unwrap().as_str(), - "local_autogen" - ); + assert_eq!(RepositoryId::new("autogen").unwrap().as_str(), "autogen"); assert_eq!( RepositoryId::new("community.1").unwrap().to_string(), "community.1" diff --git a/crates/getter-core/src/repository.rs b/crates/getter-core/src/repository.rs index c383acc..95b7fe8 100644 --- a/crates/getter-core/src/repository.rs +++ b/crates/getter-core/src/repository.rs @@ -2,10 +2,19 @@ use crate::{PackageId, PackageIdError, RepositoryId, RepositoryIdError, RepositoryPriority}; use serde::{Deserialize, Serialize}; +use std::collections::HashMap; use std::fs; use std::path::{Path, PathBuf}; pub const REPO_API_VERSION_V1: &str = "getter.repo.v1"; +pub const MAIN_DB_FILE: &str = "main.db"; +pub const CACHE_DB_FILE: &str = "cache.db"; +pub const REPOSITORY_ROOT_DIR: &str = "repo"; +pub const RUNTIME_CONFIG_DIR: &str = "rc"; +pub const REPOSITORY_ROOT_METADATA_FILE: &str = "metadata.jsonc"; +pub const REPOSITORY_ROOT_METADATA_VERSION: u32 = 1; +pub const LOCAL_REPOSITORY_ALIAS: &str = "local"; +pub const DEFAULT_GENERATED_REPOSITORY_ALIAS: &str = "autogen"; #[derive(Debug, Clone, PartialEq, Eq)] pub struct RepositoryLayout { @@ -25,6 +34,200 @@ pub struct RepositoryMetadata { pub api_version: String, } +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct GetterDataDirLayout { + pub root: PathBuf, + pub main_db: PathBuf, + pub cache_db: PathBuf, + pub repository_root: PathBuf, + pub runtime_config_root: PathBuf, +} + +impl GetterDataDirLayout { + pub fn new(root: impl AsRef) -> Self { + let root = root.as_ref().to_path_buf(); + Self { + main_db: root.join(MAIN_DB_FILE), + cache_db: root.join(CACHE_DB_FILE), + repository_root: root.join(REPOSITORY_ROOT_DIR), + runtime_config_root: root.join(RUNTIME_CONFIG_DIR), + root, + } + } + + pub fn repository_path(&self, alias: &RepositoryId) -> PathBuf { + self.repository_root.join(alias.as_str()) + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct RepositoryRootConfig { + pub generated_repository: RepositoryId, + priority: HashMap, +} + +impl RepositoryRootConfig { + pub fn load(repository_root: impl AsRef) -> Result { + let path = repository_root.as_ref().join(REPOSITORY_ROOT_METADATA_FILE); + if !path.exists() { + return Ok(Self::default()); + } + let bytes = fs::read(&path).map_err(|source| RepositoryLoadError::ReadRootMetadata { + path: path.clone(), + source, + })?; + let raw: RawRepositoryRootConfig = serde_json::from_reader( + json_comments::StripComments::new(bytes.as_slice()), + ) + .map_err(|source| RepositoryLoadError::ParseRootMetadata { + path: path.clone(), + source, + })?; + if raw.version != REPOSITORY_ROOT_METADATA_VERSION { + return Err(RepositoryLoadError::UnsupportedRootMetadataVersion { + path, + found: raw.version, + expected: REPOSITORY_ROOT_METADATA_VERSION, + }); + } + let priority = raw + .priority + .into_iter() + .map(|(alias, value)| (alias, RepositoryPriority::new(value))) + .collect(); + Ok(Self { + generated_repository: RepositoryId::new( + raw.generated_repository + .unwrap_or_else(|| DEFAULT_GENERATED_REPOSITORY_ALIAS.to_owned()), + )?, + priority, + }) + } + + pub fn priority_for(&self, alias: &RepositoryId) -> RepositoryPriority { + self.priority + .get(alias.as_str()) + .copied() + .unwrap_or_else(|| default_repository_priority(alias.as_str())) + } +} + +impl Default for RepositoryRootConfig { + fn default() -> Self { + Self { + generated_repository: RepositoryId::new(DEFAULT_GENERATED_REPOSITORY_ALIAS) + .expect("default generated repository alias is valid"), + priority: HashMap::new(), + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct RepositoryAliasEntry { + pub alias: RepositoryId, + pub path: PathBuf, + pub priority: RepositoryPriority, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct RepositoryRootLayout { + pub root: PathBuf, + pub config: RepositoryRootConfig, + pub repositories: Vec, +} + +impl RepositoryRootLayout { + pub fn load(root: impl AsRef) -> Result { + let root = root.as_ref().to_path_buf(); + let config = RepositoryRootConfig::load(&root)?; + let mut repositories = Vec::new(); + match fs::read_dir(&root) { + Ok(entries) => { + for entry in entries { + let entry = + entry.map_err(|source| RepositoryLoadError::ReadRepositoryRoot { + path: root.clone(), + source, + })?; + let file_type = entry.file_type().map_err(|source| { + RepositoryLoadError::ReadRepositoryRoot { + path: root.clone(), + source, + } + })?; + if !file_type.is_dir() { + continue; + } + let alias_text = entry.file_name().to_string_lossy().into_owned(); + let alias = RepositoryId::new(alias_text)?; + let priority = config.priority_for(&alias); + repositories.push(RepositoryAliasEntry { + alias, + path: entry.path(), + priority, + }); + } + } + Err(source) if source.kind() == std::io::ErrorKind::NotFound => {} + Err(source) => { + return Err(RepositoryLoadError::ReadRepositoryRoot { + path: root.clone(), + source, + }) + } + } + repositories.sort_by(|left, right| { + right + .priority + .cmp(&left.priority) + .then_with(|| left.alias.as_str().cmp(right.alias.as_str())) + }); + Ok(Self { + root, + config, + repositories, + }) + } +} + +pub fn default_repository_priority(alias: &str) -> RepositoryPriority { + match alias { + LOCAL_REPOSITORY_ALIAS => RepositoryPriority::LOCAL, + DEFAULT_GENERATED_REPOSITORY_ALIAS => RepositoryPriority::GENERATED_FALLBACK, + _ => RepositoryPriority::DEFAULT, + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum GeneratedRepositoryTarget { + CreateDefault { alias: RepositoryId, path: PathBuf }, + Existing { alias: RepositoryId, path: PathBuf }, +} + +pub fn generated_repository_target( + data_dir: impl AsRef, +) -> Result { + let layout = GetterDataDirLayout::new(data_dir); + let config = RepositoryRootConfig::load(&layout.repository_root)?; + let path = layout.repository_path(&config.generated_repository); + if config.generated_repository.as_str() == DEFAULT_GENERATED_REPOSITORY_ALIAS { + Ok(GeneratedRepositoryTarget::CreateDefault { + alias: config.generated_repository, + path, + }) + } else if path.is_dir() { + Ok(GeneratedRepositoryTarget::Existing { + alias: config.generated_repository, + path, + }) + } else { + Err(RepositoryLoadError::MissingGeneratedRepository { + alias: config.generated_repository, + path, + }) + } +} + #[derive(Debug, Clone, PartialEq, Eq)] pub struct PackageFile { pub id: PackageId, @@ -39,8 +242,43 @@ pub struct RepositoryPackageCacheKey { pub package_file_hash: String, } +#[derive(Debug, Deserialize)] +struct RawRepositoryRootConfig { + version: u32, + #[serde(default)] + generated_repository: Option, + #[serde(default)] + priority: HashMap, +} + #[derive(Debug, thiserror::Error)] pub enum RepositoryLoadError { + #[error("failed to read repository root metadata at {path}: {source}")] + ReadRootMetadata { + path: PathBuf, + #[source] + source: std::io::Error, + }, + #[error("failed to parse repository root metadata at {path}: {source}")] + ParseRootMetadata { + path: PathBuf, + #[source] + source: serde_json::Error, + }, + #[error("unsupported repository root metadata version {found} at {path}; expected {expected}")] + UnsupportedRootMetadataVersion { + path: PathBuf, + found: u32, + expected: u32, + }, + #[error("failed to read repository root {path}: {source}")] + ReadRepositoryRoot { + path: PathBuf, + #[source] + source: std::io::Error, + }, + #[error("configured generated repository '{alias}' does not exist at {path}")] + MissingGeneratedRepository { alias: RepositoryId, path: PathBuf }, #[error("failed to read repo.toml at {path}: {source}")] ReadRepoToml { path: PathBuf, @@ -280,6 +518,154 @@ mod tests { use super::*; use std::io::Write; + #[test] + fn data_dir_layout_uses_repo_and_rc_roots() { + let layout = GetterDataDirLayout::new("/tmp/ua-getter"); + + assert_eq!(layout.main_db, PathBuf::from("/tmp/ua-getter/main.db")); + assert_eq!(layout.cache_db, PathBuf::from("/tmp/ua-getter/cache.db")); + assert_eq!(layout.repository_root, PathBuf::from("/tmp/ua-getter/repo")); + assert_eq!( + layout.runtime_config_root, + PathBuf::from("/tmp/ua-getter/rc") + ); + } + + #[test] + fn missing_root_metadata_uses_default_priorities_and_generated_target() { + let temp = tempfile::tempdir().unwrap(); + let root = temp.path().join("repo"); + fs::create_dir_all(root.join("community")).unwrap(); + fs::create_dir_all(root.join("local")).unwrap(); + fs::create_dir_all(root.join("autogen")).unwrap(); + + let layout = RepositoryRootLayout::load(&root).unwrap(); + + assert_eq!( + layout.config.generated_repository.as_str(), + DEFAULT_GENERATED_REPOSITORY_ALIAS + ); + assert_eq!( + layout + .repositories + .iter() + .map(|repo| (repo.alias.as_str().to_owned(), repo.priority.value())) + .collect::>(), + vec![ + ("local".to_owned(), 100), + ("community".to_owned(), 0), + ("autogen".to_owned(), -1), + ] + ); + } + + #[test] + fn root_metadata_priority_map_is_lookup_only_for_existing_aliases() { + let temp = tempfile::tempdir().unwrap(); + let root = temp.path().join("repo"); + fs::create_dir_all(root.join("official")).unwrap(); + fs::write( + root.join(REPOSITORY_ROOT_METADATA_FILE), + r#"{ + "version": 1, + // This entry must not create or sort a missing repository. + "generated_repository": "autogen", + "priority": { + "missing": 500, + "not an alias": 400, + "official": 7 + } +} +"#, + ) + .unwrap(); + + let layout = RepositoryRootLayout::load(&root).unwrap(); + + assert_eq!(layout.repositories.len(), 1); + assert_eq!(layout.repositories[0].alias.as_str(), "official"); + assert_eq!(layout.repositories[0].priority.value(), 7); + assert_eq!( + layout.config.generated_repository.as_str(), + DEFAULT_GENERATED_REPOSITORY_ALIAS + ); + } + + #[test] + fn malformed_root_metadata_is_an_error_instead_of_fallback() { + let temp = tempfile::tempdir().unwrap(); + let root = temp.path().join("repo"); + fs::create_dir_all(&root).unwrap(); + fs::write(root.join(REPOSITORY_ROOT_METADATA_FILE), "{not-json").unwrap(); + + assert!(matches!( + RepositoryRootLayout::load(&root), + Err(RepositoryLoadError::ParseRootMetadata { .. }) + )); + } + + #[test] + fn generated_repository_target_creates_only_default_alias() { + let temp = tempfile::tempdir().unwrap(); + let data_dir = temp.path(); + + let target = generated_repository_target(data_dir).unwrap(); + + assert!(matches!( + target, + GeneratedRepositoryTarget::CreateDefault { ref alias, ref path } + if alias.as_str() == DEFAULT_GENERATED_REPOSITORY_ALIAS + && path == &data_dir.join("repo").join(DEFAULT_GENERATED_REPOSITORY_ALIAS) + )); + } + + #[test] + fn generated_repository_target_rejects_missing_custom_alias() { + let temp = tempfile::tempdir().unwrap(); + let repo_root = temp.path().join("repo"); + fs::create_dir_all(&repo_root).unwrap(); + fs::write( + repo_root.join(REPOSITORY_ROOT_METADATA_FILE), + r#"{ + "version": 1, + "generated_repository": "generated", + "priority": {} +} +"#, + ) + .unwrap(); + + assert!(matches!( + generated_repository_target(temp.path()), + Err(RepositoryLoadError::MissingGeneratedRepository { ref alias, .. }) + if alias.as_str() == "generated" + )); + } + + #[test] + fn generated_repository_target_accepts_existing_custom_alias() { + let temp = tempfile::tempdir().unwrap(); + let repo_root = temp.path().join("repo"); + fs::create_dir_all(repo_root.join("generated")).unwrap(); + fs::write( + repo_root.join(REPOSITORY_ROOT_METADATA_FILE), + r#"{ + "version": 1, + "generated_repository": "generated" +} +"#, + ) + .unwrap(); + + let target = generated_repository_target(temp.path()).unwrap(); + + assert!(matches!( + target, + GeneratedRepositoryTarget::Existing { ref alias, ref path } + if alias.as_str() == "generated" && path == &repo_root.join("generated") + )); + } + #[test] fn derives_package_id_from_lua_path() { let root = PathBuf::from("repo/packages"); @@ -362,7 +748,7 @@ api_version = "getter.repo.v1" #[test] fn highest_priority_selects_larger_number() { let priorities = [ - RepositoryPriority::LOCAL_AUTOGEN, + RepositoryPriority::GENERATED_FALLBACK, RepositoryPriority::DEFAULT, RepositoryPriority::LOCAL, ]; diff --git a/crates/getter-operations/src/autogen.rs b/crates/getter-operations/src/autogen.rs index caab4aa..faa06a9 100644 --- a/crates/getter-operations/src/autogen.rs +++ b/crates/getter-operations/src/autogen.rs @@ -1,18 +1,20 @@ //! Getter-owned installed-inventory autogen operations. //! //! The CLI and native bridge both call this module so there is one implementation -//! of `local_autogen` preview/apply semantics. Platform layers provide installed +//! of installed-autogen preview/apply semantics. Platform layers provide installed //! inventory facts; this module decides generated package ids, repository //! coverage, file writes, manifest updates, preservation behavior, and tracked //! state updates. use getter_core::autogen::{ - content_hash, local_autogen_repo_toml, local_repo_toml, plan_local_autogen, AutogenManifest, - AutogenManifestEntry, AutogenPlan, AutogenSkipReason, InstalledInventory, - LOCAL_AUTOGEN_REPOSITORY_ID, LOCAL_AUTOGEN_REPOSITORY_NAME, LOCAL_REPOSITORY_ID, - LOCAL_REPOSITORY_NAME, + content_hash, local_repo_toml, plan_installed_autogen, AutogenManifest, AutogenManifestEntry, + AutogenPlan, AutogenSkipReason, InstalledInventory, DEFAULT_AUTOGEN_REPOSITORY_ID, + DEFAULT_AUTOGEN_REPOSITORY_NAME, LOCAL_REPOSITORY_ID, LOCAL_REPOSITORY_NAME, +}; +use getter_core::repository::{ + generated_repository_target, GeneratedRepositoryTarget, GetterDataDirLayout, RepositoryLayout, + RepositoryLoadError, RepositoryMetadata, RepositoryRootConfig, REPO_API_VERSION_V1, }; -use getter_core::repository::{RepositoryLayout, RepositoryMetadata, REPO_API_VERSION_V1}; use getter_core::{PackageId, RepositoryId, RepositoryPriority}; use getter_storage::{MainDb, StorageError, StoredRepository}; use serde_json::{json, Value}; @@ -34,28 +36,55 @@ pub enum AutogenOperationError { Storage(#[from] StorageError), #[error("repository error: {0}")] Repository(String), + #[error("configured generated repository '{alias}' does not exist at {path}")] + MissingGeneratedRepository { alias: RepositoryId, path: PathBuf }, #[error("autogen error: {0}")] Autogen(String), } +impl From for AutogenOperationError { + fn from(value: RepositoryLoadError) -> Self { + match value { + RepositoryLoadError::MissingGeneratedRepository { alias, path } => { + Self::MissingGeneratedRepository { alias, path } + } + other => Self::Repository(other.to_string()), + } + } +} + pub type AutogenOperationResult = Result; -pub fn build_local_autogen_plan( +pub fn build_installed_autogen_plan( + data_dir: &Path, db: &MainDb, inventory: &InstalledInventory, ) -> AutogenOperationResult { - let covered = higher_priority_package_coverage(db)?; - plan_local_autogen(inventory, &covered) - .map_err(|source| AutogenOperationError::Autogen(source.to_string())) + let (target_alias, _target_path, target_priority) = generated_repository_config(data_dir)?; + let covered = higher_priority_package_coverage(db, &target_alias, target_priority)?; + let mut plan = plan_installed_autogen(inventory, &covered) + .map_err(|source| AutogenOperationError::Autogen(source.to_string()))?; + plan.repository_id = target_alias.clone(); + plan.repository_name = if target_alias.as_str() == DEFAULT_AUTOGEN_REPOSITORY_ID { + DEFAULT_AUTOGEN_REPOSITORY_NAME.to_owned() + } else { + target_alias.to_string() + }; + plan.repository_priority = target_priority; + Ok(plan) } -pub fn installed_preview_json(data_dir: &Path, plan: &AutogenPlan) -> Value { +pub fn installed_preview_json( + data_dir: &Path, + plan: &AutogenPlan, +) -> AutogenOperationResult { + let (target_alias, target_path, _target_priority) = generated_repository_config(data_dir)?; let candidates: Vec = plan.candidates.iter().map(autogen_candidate_json).collect(); let skipped: Vec = plan.skipped.iter().map(autogen_skip_json).collect(); - json!({ + Ok(json!({ "operation": "installed.preview", - "target_repo_id": plan.repository_id.as_str(), - "target_repo_path": local_autogen_repo_path(data_dir), + "target_repo_id": target_alias.as_str(), + "target_repo_path": target_path, "summary": { "candidate_count": candidates.len(), "skipped_count": skipped.len(), @@ -65,7 +94,7 @@ pub fn installed_preview_json(data_dir: &Path, plan: &AutogenPlan) -> Value { "candidates": candidates, "skipped": skipped, "diagnostics": [], - }) + })) } pub fn cleanup_preview_json( @@ -73,11 +102,11 @@ pub fn cleanup_preview_json( db: &MainDb, inventory: &InstalledInventory, ) -> AutogenOperationResult { - let repo_path = local_autogen_repo_path(data_dir); + let (target_alias, repo_path, _target_priority) = generated_repository_config(data_dir)?; let Some(manifest) = read_autogen_manifest(&repo_path)? else { return Ok(json!({ "operation": "cleanup.preview", - "target_repo_id": LOCAL_AUTOGEN_REPOSITORY_ID, + "target_repo_id": target_alias.as_str(), "target_repo_path": repo_path, "summary": { "candidate_count": 0, "skipped_count": 0, "write_count": 0, "delete_count": 0 }, "candidates": [], @@ -85,7 +114,7 @@ pub fn cleanup_preview_json( "diagnostics": [], })); }; - let plan = build_local_autogen_plan(db, inventory)?; + let plan = build_installed_autogen_plan(data_dir, db, inventory)?; let installed_ids: BTreeSet = plan .candidates .iter() @@ -113,7 +142,7 @@ pub fn cleanup_preview_json( }); Ok(json!({ "operation": "cleanup.preview", - "target_repo_id": LOCAL_AUTOGEN_REPOSITORY_ID, + "target_repo_id": target_alias.as_str(), "target_repo_path": repo_path, "summary": { "candidate_count": candidates.len(), @@ -133,10 +162,17 @@ pub fn apply_installed_preview( preview: &Value, acceptance: &AutogenAcceptance, ) -> AutogenOperationResult { - let repo_path = local_autogen_repo_path(data_dir); - ensure_local_autogen_repository(data_dir, db)?; + let (target_alias, repo_path, target_priority) = generated_repository_config(data_dir)?; + if preview.get("target_repo_id").and_then(Value::as_str) != Some(target_alias.as_str()) { + return Err(AutogenOperationError::Autogen(format!( + "installed preview target_repo_id must be '{}'", + target_alias.as_str() + ))); + } + ensure_generated_repository(&repo_path, db, &target_alias, target_priority)?; let accepted = accepted_preview_candidates(preview, acceptance)?; - let mut manifest = read_autogen_manifest(&repo_path)?.unwrap_or_else(empty_autogen_manifest); + let mut manifest = + read_autogen_manifest(&repo_path)?.unwrap_or_else(|| empty_autogen_manifest(&target_alias)); let mut applied = Vec::new(); let mut preserved = Vec::new(); @@ -203,10 +239,7 @@ pub fn apply_installed_preview( content_hash: expected_hash.to_owned(), }, ); - db.upsert_generated_tracked_package_preserving_user_state( - &package_id, - &RepositoryId::new(LOCAL_AUTOGEN_REPOSITORY_ID).expect("valid id"), - )?; + db.upsert_generated_tracked_package_preserving_user_state(&package_id, &target_alias)?; applied.push(json!({ "package_id": package_id.to_string(), "output_relative_path": relative_path, @@ -215,7 +248,7 @@ pub fn apply_installed_preview( write_autogen_manifest(&repo_path, &manifest)?; Ok(json!({ - "target_repo_id": LOCAL_AUTOGEN_REPOSITORY_ID, + "target_repo_id": target_alias.as_str(), "target_repo_path": repo_path, "applied_count": applied.len(), "applied": applied, @@ -229,17 +262,18 @@ pub fn apply_cleanup_preview( preview: &Value, acceptance: &AutogenAcceptance, ) -> AutogenOperationResult { - if preview.get("target_repo_id").and_then(Value::as_str) != Some(LOCAL_AUTOGEN_REPOSITORY_ID) { + let (target_alias, repo_path, _target_priority) = generated_repository_config(data_dir)?; + if preview.get("target_repo_id").and_then(Value::as_str) != Some(target_alias.as_str()) { return Err(AutogenOperationError::Autogen(format!( - "cleanup preview target_repo_id must be '{LOCAL_AUTOGEN_REPOSITORY_ID}'" + "cleanup preview target_repo_id must be '{}'", + target_alias.as_str() ))); } - let repo_path = local_autogen_repo_path(data_dir); let accepted = accepted_preview_candidates(preview, acceptance)?; - let mut manifest = read_autogen_manifest(&repo_path)?.unwrap_or_else(empty_autogen_manifest); + let mut manifest = + read_autogen_manifest(&repo_path)?.unwrap_or_else(|| empty_autogen_manifest(&target_alias)); let mut deleted = Vec::new(); let mut preserved = Vec::new(); - let local_autogen_id = RepositoryId::new(LOCAL_AUTOGEN_REPOSITORY_ID).expect("valid id"); for candidate in accepted { let package_id = preview_package_id(candidate)?; let relative_path = preview_relative_path(candidate)?; @@ -253,14 +287,14 @@ pub fn apply_cleanup_preview( })?; let manifest_entry = manifest.package(&package_id).ok_or_else(|| { AutogenOperationError::Autogen(format!( - "cleanup preview candidate {package_id} is not managed by local_autogen manifest" + "cleanup preview candidate {package_id} is not managed by autogen manifest" )) })?; if manifest_entry.relative_path != relative_path || manifest_entry.content_hash != expected_hash { return Err(AutogenOperationError::Autogen(format!( - "cleanup preview candidate {package_id} does not match local_autogen manifest" + "cleanup preview candidate {package_id} does not match autogen manifest" ))); } let target = safe_join(&repo_path, &relative_path)?; @@ -289,7 +323,7 @@ pub fn apply_cleanup_preview( manifest .packages .retain(|entry| entry.package_id != package_id); - db.delete_generated_tracked_package(&package_id, &local_autogen_id)?; + db.delete_generated_tracked_package(&package_id, &target_alias)?; deleted.push(json!({ "package_id": package_id.to_string(), "output_relative_path": relative_path, @@ -297,7 +331,7 @@ pub fn apply_cleanup_preview( } write_autogen_manifest(&repo_path, &manifest)?; Ok(json!({ - "target_repo_id": LOCAL_AUTOGEN_REPOSITORY_ID, + "target_repo_id": target_alias.as_str(), "target_repo_path": repo_path, "deleted_count": deleted.len(), "deleted": deleted, @@ -322,21 +356,36 @@ pub fn unwrap_preview_payload( Ok(payload) } -pub fn local_autogen_repo_path(data_dir: &Path) -> PathBuf { - data_dir - .join("repositories") - .join(LOCAL_AUTOGEN_REPOSITORY_ID) +pub fn default_autogen_repo_path(data_dir: &Path) -> PathBuf { + GetterDataDirLayout::new(data_dir) + .repository_path(&RepositoryId::new(DEFAULT_AUTOGEN_REPOSITORY_ID).expect("valid id")) +} + +fn generated_repository_config( + data_dir: &Path, +) -> AutogenOperationResult<(RepositoryId, PathBuf, RepositoryPriority)> { + let layout = GetterDataDirLayout::new(data_dir); + let config = RepositoryRootConfig::load(&layout.repository_root)?; + let target = generated_repository_target(data_dir)?; + let (alias, path) = match target { + GeneratedRepositoryTarget::CreateDefault { alias, path } + | GeneratedRepositoryTarget::Existing { alias, path } => (alias, path), + }; + let priority = config.priority_for(&alias); + Ok((alias, path, priority)) } fn higher_priority_package_coverage( db: &MainDb, + target_alias: &RepositoryId, + target_priority: RepositoryPriority, ) -> AutogenOperationResult> { let mut covered = HashMap::new(); for repo in db.repositories()? { - if repo.id.as_str() == LOCAL_AUTOGEN_REPOSITORY_ID { + if &repo.id == target_alias { continue; } - if repo.priority <= RepositoryPriority::LOCAL_AUTOGEN { + if repo.priority <= target_priority { continue; } let Some(path) = repo.path.as_ref() else { @@ -431,23 +480,41 @@ fn preview_relative_path(candidate: &Value) -> AutogenOperationResult { }) } -fn ensure_local_autogen_repository( - data_dir: &Path, +fn ensure_generated_repository( + repo_path: &Path, db: &MainDb, -) -> AutogenOperationResult { - let repo_path = local_autogen_repo_path(data_dir); - ensure_repository_layout(&repo_path, &local_autogen_repo_toml())?; + alias: &RepositoryId, + priority: RepositoryPriority, +) -> AutogenOperationResult<()> { + ensure_repository_layout(repo_path, &generated_repo_toml(alias, priority))?; db.upsert_repository( &RepositoryMetadata { - id: RepositoryId::new(LOCAL_AUTOGEN_REPOSITORY_ID).expect("valid id"), - name: LOCAL_AUTOGEN_REPOSITORY_NAME.to_owned(), - priority: RepositoryPriority::LOCAL_AUTOGEN, + id: alias.clone(), + name: generated_repository_name(alias), + priority, api_version: REPO_API_VERSION_V1.to_owned(), }, - Some(&repo_path), + Some(repo_path), None, )?; - Ok(repo_path) + Ok(()) +} + +fn generated_repo_toml(alias: &RepositoryId, priority: RepositoryPriority) -> String { + format!( + "id = \"{}\"\nname = \"{}\"\npriority = {}\napi_version = \"{REPO_API_VERSION_V1}\"\n", + alias.as_str(), + generated_repository_name(alias), + priority.value() + ) +} + +fn generated_repository_name(alias: &RepositoryId) -> String { + if alias.as_str() == DEFAULT_AUTOGEN_REPOSITORY_ID { + DEFAULT_AUTOGEN_REPOSITORY_NAME.to_owned() + } else { + alias.to_string() + } } fn ensure_local_repository(data_dir: &Path, db: &MainDb) -> AutogenOperationResult { @@ -458,7 +525,8 @@ fn ensure_local_repository(data_dir: &Path, db: &MainDb) -> AutogenOperationResu return Ok(repo_path); } - let repo_path = data_dir.join("repositories").join(LOCAL_REPOSITORY_ID); + let repo_path = GetterDataDirLayout::new(data_dir) + .repository_path(&RepositoryId::new(LOCAL_REPOSITORY_ID).expect("valid id")); ensure_repository_layout(&repo_path, &local_repo_toml())?; db.upsert_repository( &RepositoryMetadata { @@ -601,10 +669,10 @@ fn write_autogen_manifest( }) } -fn empty_autogen_manifest() -> AutogenManifest { +fn empty_autogen_manifest(repository_id: &RepositoryId) -> AutogenManifest { AutogenManifest { version: getter_core::autogen::AUTOGEN_MANIFEST_VERSION, - repository_id: RepositoryId::new(LOCAL_AUTOGEN_REPOSITORY_ID).expect("valid id"), + repository_id: repository_id.clone(), packages: Vec::new(), } } @@ -635,3 +703,109 @@ fn safe_join(root: &Path, relative: &Path) -> AutogenOperationResult { } Ok(root.join(relative)) } + +#[cfg(test)] +mod tests { + use super::*; + use getter_storage::MainDb; + + fn test_inventory() -> InstalledInventory { + InstalledInventory::new(vec![ + getter_core::autogen::InstalledInventoryItem::AndroidPackage { + package_name: "com.example.autogen".to_owned(), + label: Some("Example Autogen".to_owned()), + version_name: None, + version_code: None, + }, + ]) + } + + #[test] + fn preview_uses_default_generated_repository_under_repo_root() { + let temp = tempfile::tempdir().unwrap(); + let data_dir = temp.path(); + let db = MainDb::open(data_dir.join("main.db")).unwrap(); + + let plan = build_installed_autogen_plan(data_dir, &db, &test_inventory()).unwrap(); + let preview = installed_preview_json(data_dir, &plan).unwrap(); + + assert_eq!(preview["target_repo_id"], "autogen"); + assert_eq!( + preview["target_repo_path"].as_str(), + data_dir.join("repo/autogen").to_str() + ); + assert!(!data_dir.join("repo/autogen").exists()); + } + + #[test] + fn apply_creates_default_generated_repository_and_registers_it() { + let temp = tempfile::tempdir().unwrap(); + let data_dir = temp.path(); + let db = MainDb::open(data_dir.join("main.db")).unwrap(); + let plan = build_installed_autogen_plan(data_dir, &db, &test_inventory()).unwrap(); + let preview = installed_preview_json(data_dir, &plan).unwrap(); + + let result = + apply_installed_preview(data_dir, &db, &preview, &AutogenAcceptance::AcceptAll) + .unwrap(); + + assert_eq!(result["target_repo_id"], "autogen"); + assert!(data_dir.join("repo/autogen/repo.toml").is_file()); + assert!(data_dir + .join("repo/autogen/packages/android/com.example.autogen.lua") + .is_file()); + let repos = db.repositories().unwrap(); + assert_eq!(repos[0].id.as_str(), "autogen"); + assert_eq!(repos[0].priority.value(), -1); + } + + #[test] + fn custom_generated_repository_must_already_exist() { + let temp = tempfile::tempdir().unwrap(); + let data_dir = temp.path(); + fs::create_dir_all(data_dir.join("repo")).unwrap(); + fs::write( + data_dir.join("repo/metadata.jsonc"), + r#"{ "version": 1, "generated_repository": "generated" }"#, + ) + .unwrap(); + let db = MainDb::open(data_dir.join("main.db")).unwrap(); + + let error = build_installed_autogen_plan(data_dir, &db, &test_inventory()).unwrap_err(); + + assert!(matches!( + error, + AutogenOperationError::MissingGeneratedRepository { ref alias, .. } + if alias.as_str() == "generated" + )); + } + + #[test] + fn custom_generated_repository_uses_configured_priority() { + let temp = tempfile::tempdir().unwrap(); + let data_dir = temp.path(); + fs::create_dir_all(data_dir.join("repo/generated")).unwrap(); + fs::write( + data_dir.join("repo/metadata.jsonc"), + r#"{ + "version": 1, + "generated_repository": "generated", + "priority": { "generated": -5 } +}"#, + ) + .unwrap(); + let db = MainDb::open(data_dir.join("main.db")).unwrap(); + + let plan = build_installed_autogen_plan(data_dir, &db, &test_inventory()).unwrap(); + let preview = installed_preview_json(data_dir, &plan).unwrap(); + let result = + apply_installed_preview(data_dir, &db, &preview, &AutogenAcceptance::AcceptAll) + .unwrap(); + + assert_eq!(preview["target_repo_id"], "generated"); + assert_eq!(result["target_repo_id"], "generated"); + let repos = db.repositories().unwrap(); + assert_eq!(repos[0].id.as_str(), "generated"); + assert_eq!(repos[0].priority.value(), -5); + } +} diff --git a/crates/getter-storage/src/lib.rs b/crates/getter-storage/src/lib.rs index 2127cbc..e8d7b02 100644 --- a/crates/getter-storage/src/lib.rs +++ b/crates/getter-storage/src/lib.rs @@ -1234,7 +1234,7 @@ VALUES ('android/org.fdroid.fdroid', '1.2.3'); fn generated_tracking_preserves_existing_user_state_on_conflict() { let db = MainDb::open_in_memory().unwrap(); let package_id: PackageId = "android/org.fdroid.fdroid".parse().unwrap(); - let local_autogen = RepositoryId::new("local_autogen").unwrap(); + let autogen = RepositoryId::new("autogen").unwrap(); db.upsert_tracked_package(&TrackedPackageUpsert { package_id: package_id.clone(), enabled: false, @@ -1245,7 +1245,7 @@ VALUES ('android/org.fdroid.fdroid', '1.2.3'); }) .unwrap(); - db.upsert_generated_tracked_package_preserving_user_state(&package_id, &local_autogen) + db.upsert_generated_tracked_package_preserving_user_state(&package_id, &autogen) .unwrap(); let packages = db.tracked_packages().unwrap(); @@ -1264,7 +1264,7 @@ VALUES ('android/org.fdroid.fdroid', '1.2.3'); fn generated_tracking_fills_unresolved_tracking_metadata() { let db = MainDb::open_in_memory().unwrap(); let package_id: PackageId = "android/org.fdroid.fdroid".parse().unwrap(); - let local_autogen = insert_local_autogen_repo(&db); + let autogen = insert_autogen_repo(&db); db.upsert_tracked_package(&TrackedPackageUpsert { package_id: package_id.clone(), enabled: false, @@ -1275,7 +1275,7 @@ VALUES ('android/org.fdroid.fdroid', '1.2.3'); }) .unwrap(); - db.upsert_generated_tracked_package_preserving_user_state(&package_id, &local_autogen) + db.upsert_generated_tracked_package_preserving_user_state(&package_id, &autogen) .unwrap(); let packages = db.tracked_packages().unwrap(); @@ -1283,7 +1283,7 @@ VALUES ('android/org.fdroid.fdroid', '1.2.3'); assert!(!packages[0].enabled); assert!(packages[0].favorite); assert_eq!(packages[0].pin_version.as_deref(), Some("9.9.9")); - assert_eq!(packages[0].repository_id.as_ref(), Some(&local_autogen)); + assert_eq!(packages[0].repository_id.as_ref(), Some(&autogen)); assert_eq!( packages[0].package_resolution, StoredPackageResolution::GenerateLocalPackage @@ -1294,7 +1294,7 @@ VALUES ('android/org.fdroid.fdroid', '1.2.3'); fn generated_tracking_delete_is_guarded_by_repo_and_resolution() { let db = MainDb::open_in_memory().unwrap(); let package_id: PackageId = "android/org.fdroid.fdroid".parse().unwrap(); - let local_autogen = insert_local_autogen_repo(&db); + let autogen = insert_autogen_repo(&db); db.upsert_tracked_package(&TrackedPackageUpsert { package_id: package_id.clone(), enabled: true, @@ -1306,7 +1306,7 @@ VALUES ('android/org.fdroid.fdroid', '1.2.3'); .unwrap(); assert!(!db - .delete_generated_tracked_package(&package_id, &local_autogen) + .delete_generated_tracked_package(&package_id, &autogen) .unwrap()); assert_eq!(db.tracked_packages().unwrap().len(), 1); @@ -1319,28 +1319,28 @@ VALUES ('android/org.fdroid.fdroid', '1.2.3'); package_resolution: StoredPackageResolution::MissingPackageDefinition, }) .unwrap(); - db.upsert_generated_tracked_package_preserving_user_state(&package_id, &local_autogen) + db.upsert_generated_tracked_package_preserving_user_state(&package_id, &autogen) .unwrap(); assert!(db - .delete_generated_tracked_package(&package_id, &local_autogen) + .delete_generated_tracked_package(&package_id, &autogen) .unwrap()); assert!(db.tracked_packages().unwrap().is_empty()); } - fn insert_local_autogen_repo(db: &MainDb) -> RepositoryId { - let local_autogen = RepositoryId::new("local_autogen").unwrap(); + fn insert_autogen_repo(db: &MainDb) -> RepositoryId { + let autogen = RepositoryId::new("autogen").unwrap(); db.upsert_repository( &RepositoryMetadata { - id: local_autogen.clone(), - name: "Local Autogen".to_owned(), - priority: RepositoryPriority::LOCAL_AUTOGEN, + id: autogen.clone(), + name: "Autogen".to_owned(), + priority: RepositoryPriority::GENERATED_FALLBACK, api_version: REPO_API_VERSION_V1.to_owned(), }, None, None, ) .unwrap(); - local_autogen + autogen } #[test] From 3388b78dc891dc487bc406d71050e919ce6e4023 Mon Sep 17 00:00:00 2001 From: Xiangzhe Date: Fri, 26 Jun 2026 22:39:51 +0800 Subject: [PATCH 29/52] feat(repository): discover package directories --- crates/getter-cli/tests/bdd_cli.rs | 22 + .../tests/features/cli/repo_validate.feature | 7 + crates/getter-core/src/diagnostics.rs | 86 +++- crates/getter-core/src/repository.rs | 402 +++++++++++++++++- 4 files changed, 498 insertions(+), 19 deletions(-) diff --git a/crates/getter-cli/tests/bdd_cli.rs b/crates/getter-cli/tests/bdd_cli.rs index 073bb63..8a8e950 100644 --- a/crates/getter-cli/tests/bdd_cli.rs +++ b/crates/getter-cli/tests/bdd_cli.rs @@ -347,6 +347,28 @@ fn fixture_lua_repository(world: &mut CliWorld, repo_id: String, package_id: Str create_fixture_lua_repository(world, repo_id, package_id, "F-Droid".to_owned()); } +#[given(expr = "a package-directory repository {string} with package {string}")] +fn package_directory_repository(world: &mut CliWorld, repo_id: String, package_id: String) { + let temp = world.temp.as_ref().expect("tempdir exists"); + let repo_path = temp.path().join(format!("repo-{repo_id}")); + let package_dir = repo_path.join(package_id.replace('/', std::path::MAIN_SEPARATOR_STR)); + fs::create_dir_all(&package_dir).expect("create package dir"); + fs::write( + package_dir.join("metadata.jsonc"), + r#"{ "type": "android:app", "android": { "package_name": "org.fdroid.fdroid" } }"#, + ) + .expect("write package metadata"); + fs::write( + package_dir.join("1.20.0.lua"), + "#!/bin/upa-lua v1\nreturn {}", + ) + .expect("write version Lua"); + + world.fixture_repo_id = Some(repo_id); + world.fixture_repo_path = Some(repo_path); + world.fixture_package_id = Some(package_id); +} + #[given(expr = "a fixture Lua repository {string} with package {string} named {string}")] fn fixture_lua_repository_named( world: &mut CliWorld, diff --git a/crates/getter-cli/tests/features/cli/repo_validate.feature b/crates/getter-cli/tests/features/cli/repo_validate.feature index 592d4bf..9c6d1a1 100644 --- a/crates/getter-cli/tests/features/cli/repo_validate.feature +++ b/crates/getter-cli/tests/features/cli/repo_validate.feature @@ -6,6 +6,13 @@ Feature: Getter CLI repository validation Then the command succeeds And the output reports a valid repository without network + Scenario: User validates a package-directory repository offline + Given an initialized getter data directory + And a package-directory repository "official" with package "android/app/org.fdroid.fdroid" + When I run getter repo validate for that repository + Then the command succeeds + And the output reports a valid repository without network + Scenario: User receives diagnostics for invalid Lua Given an initialized getter data directory And a fixture Lua repository "broken" with invalid Lua package "android/org.fdroid.fdroid" diff --git a/crates/getter-core/src/diagnostics.rs b/crates/getter-core/src/diagnostics.rs index d338055..817b310 100644 --- a/crates/getter-core/src/diagnostics.rs +++ b/crates/getter-core/src/diagnostics.rs @@ -4,7 +4,10 @@ //! describe what getter observed; Flutter should only render them. use crate::lua::{evaluate_package_file, LuaPackageError}; -use crate::repository::{package_cache_key, RepositoryLayout, RepositoryLoadError}; +use crate::repository::{ + package_cache_key, InvalidPackageDirectory, RepositoryLayout, RepositoryLoadError, + RepositoryPackageDirectoryLayout, +}; use crate::PackageId; use serde::{Deserialize, Serialize}; use std::path::{Path, PathBuf}; @@ -62,6 +65,14 @@ impl RepositoryValidationReport { /// not perform provider/network checks. pub fn validate_repository_path(path: impl AsRef) -> RepositoryValidationReport { let root = path.as_ref(); + if root.join("repo.toml").is_file() { + validate_legacy_repository_path(root) + } else { + validate_package_directory_repository_path(root) + } +} + +fn validate_legacy_repository_path(root: &Path) -> RepositoryValidationReport { let layout = match RepositoryLayout::load(root) { Ok(layout) => layout, Err(error) => { @@ -85,6 +96,21 @@ pub fn validate_repository_path(path: impl AsRef) -> RepositoryValidationR RepositoryValidationReport::new(package_count, diagnostics) } +fn validate_package_directory_repository_path(root: &Path) -> RepositoryValidationReport { + let layout = match RepositoryPackageDirectoryLayout::load(root) { + Ok(layout) => layout, + Err(error) => { + return RepositoryValidationReport::new(0, vec![repository_load_diagnostic(error)]) + } + }; + let diagnostics = layout + .invalid_packages + .iter() + .map(invalid_package_directory_diagnostic) + .collect(); + RepositoryValidationReport::new(layout.packages.len(), diagnostics) +} + fn repository_load_diagnostic(error: RepositoryLoadError) -> PackageValidationDiagnostic { let (code, path, message) = match error { RepositoryLoadError::ReadRootMetadata { path, source } => ( @@ -163,6 +189,18 @@ fn repository_load_diagnostic(error: RepositoryLoadError) -> PackageValidationDi diagnostic(code, message, path, None, None) } +fn invalid_package_directory_diagnostic( + package: &InvalidPackageDirectory, +) -> PackageValidationDiagnostic { + diagnostic( + "package.metadata", + package.reason.clone(), + package.metadata_path.clone(), + None, + package.id.clone(), + ) +} + fn lua_diagnostic( error: LuaPackageError, fallback_package_id: Option, @@ -275,6 +313,52 @@ api_version = "getter.repo.v1" assert!(!report.network_required); } + #[test] + fn package_directory_repository_has_no_diagnostics() { + let temp = tempfile::tempdir().unwrap(); + let package_dir = temp.path().join("android/app/org.fdroid.fdroid"); + fs::create_dir_all(&package_dir).unwrap(); + fs::write( + package_dir.join("metadata.jsonc"), + r#"{ "type": "android:app", "android": { "package_name": "org.fdroid.fdroid" } }"#, + ) + .unwrap(); + fs::write( + package_dir.join("1.20.0.lua"), + "#!/bin/upa-lua v1\nreturn {}", + ) + .unwrap(); + + let report = validate_repository_path(temp.path()); + + assert!(report.valid, "{report:?}"); + assert_eq!(report.package_count, 1); + assert!(report.diagnostics.is_empty()); + assert!(!report.network_required); + } + + #[test] + fn package_directory_metadata_error_is_stable_package_diagnostic() { + let temp = tempfile::tempdir().unwrap(); + let package_dir = temp.path().join("android/app/org.fdroid.fdroid"); + fs::create_dir_all(&package_dir).unwrap(); + fs::write(package_dir.join("metadata.jsonc"), "[]").unwrap(); + + let report = validate_repository_path(temp.path()); + + assert!(!report.valid); + assert_eq!(report.package_count, 0); + assert_eq!(report.diagnostics[0].code, "package.metadata"); + assert_eq!( + report.diagnostics[0] + .package_id + .as_ref() + .unwrap() + .to_string(), + "android/app/org.fdroid.fdroid" + ); + } + #[test] fn missing_directory_is_stable_repository_diagnostic() { let temp = tempfile::tempdir().unwrap(); diff --git a/crates/getter-core/src/repository.rs b/crates/getter-core/src/repository.rs index 95b7fe8..fd16ef5 100644 --- a/crates/getter-core/src/repository.rs +++ b/crates/getter-core/src/repository.rs @@ -13,6 +13,10 @@ pub const REPOSITORY_ROOT_DIR: &str = "repo"; pub const RUNTIME_CONFIG_DIR: &str = "rc"; pub const REPOSITORY_ROOT_METADATA_FILE: &str = "metadata.jsonc"; pub const REPOSITORY_ROOT_METADATA_VERSION: u32 = 1; +pub const REPOSITORY_SELF_METADATA_DIR: &str = ".metadata"; +pub const REPOSITORY_LUACLASS_DIR: &str = "luaclass"; +pub const PACKAGE_METADATA_FILE: &str = "metadata.jsonc"; +pub const LUA_SCRIPT_EXTENSION: &str = "lua"; pub const LOCAL_REPOSITORY_ALIAS: &str = "local"; pub const DEFAULT_GENERATED_REPOSITORY_ALIAS: &str = "autogen"; @@ -234,6 +238,36 @@ pub struct PackageFile { pub path: PathBuf, } +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct RepositoryPackageDirectoryLayout { + pub root: PathBuf, + pub packages: Vec, + pub invalid_packages: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct PackageDirectory { + pub id: PackageId, + pub path: PathBuf, + pub metadata_path: PathBuf, + pub version_scripts: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct PackageVersionScript { + pub version: String, + pub file_name: String, + pub path: PathBuf, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct InvalidPackageDirectory { + pub id: Option, + pub path: PathBuf, + pub metadata_path: PathBuf, + pub reason: String, +} + #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct RepositoryPackageCacheKey { pub repository_id: RepositoryId, @@ -322,6 +356,26 @@ pub enum RepositoryLoadError { }, } +impl RepositoryPackageDirectoryLayout { + pub fn load(root: impl AsRef) -> Result { + let root = root.as_ref().to_path_buf(); + let mut packages = Vec::new(); + let mut invalid_packages = Vec::new(); + collect_package_directories(&root, &root, &mut packages, &mut invalid_packages)?; + packages.sort_by_key(|package| package.id.to_string()); + invalid_packages.sort_by_key(|package| package.path.clone()); + Ok(Self { + root, + packages, + invalid_packages, + }) + } + + pub fn package(&self, id: &PackageId) -> Option<&PackageDirectory> { + self.packages.iter().find(|package| &package.id == id) + } +} + impl RepositoryLayout { pub fn load(root: impl AsRef) -> Result { let root = root.as_ref().to_path_buf(); @@ -403,25 +457,16 @@ fn collect_package_files( current: &Path, out: &mut Vec, ) -> Result<(), RepositoryLoadError> { - for entry in fs::read_dir(current).map_err(|source| RepositoryLoadError::ReadPackagesDir { - path: current.to_path_buf(), - source, - })? { - let entry = entry.map_err(|source| RepositoryLoadError::ReadPackagesDir { - path: current.to_path_buf(), - source, - })?; + for entry in read_dir_entries(current)? { let path = entry.path(); - let file_type = - entry - .file_type() - .map_err(|source| RepositoryLoadError::ReadPackagesDir { - path: current.to_path_buf(), - source, - })?; + let file_type = entry_file_type(&entry, current)?; if file_type.is_dir() { collect_package_files(packages_root, &path, out)?; - } else if file_type.is_file() && path.extension().is_some_and(|ext| ext == "lua") { + } else if file_type.is_file() + && path + .extension() + .is_some_and(|ext| ext == LUA_SCRIPT_EXTENSION) + { let id = package_id_from_path(packages_root, &path)?; out.push(PackageFile { id, path }); } @@ -429,6 +474,171 @@ fn collect_package_files( Ok(()) } +fn collect_package_directories( + repository_root: &Path, + current: &Path, + packages: &mut Vec, + invalid_packages: &mut Vec, +) -> Result<(), RepositoryLoadError> { + if current == repository_root.join(REPOSITORY_SELF_METADATA_DIR) + || current == repository_root.join(REPOSITORY_LUACLASS_DIR) + { + return Ok(()); + } + + let metadata_path = current.join(PACKAGE_METADATA_FILE); + if metadata_path.is_file() { + collect_package_boundary( + repository_root, + current, + &metadata_path, + packages, + invalid_packages, + )?; + return Ok(()); + } + + for entry in read_dir_entries(current)? { + let file_type = entry_file_type(&entry, current)?; + if file_type.is_dir() { + collect_package_directories( + repository_root, + &entry.path(), + packages, + invalid_packages, + )?; + } + } + Ok(()) +} + +fn collect_package_boundary( + repository_root: &Path, + package_dir: &Path, + metadata_path: &Path, + packages: &mut Vec, + invalid_packages: &mut Vec, +) -> Result<(), RepositoryLoadError> { + let id = package_id_from_package_dir(repository_root, package_dir); + if let Err(error) = parse_package_metadata(metadata_path) { + invalid_packages.push(InvalidPackageDirectory { + id: id.ok(), + path: package_dir.to_path_buf(), + metadata_path: metadata_path.to_path_buf(), + reason: error, + }); + return Ok(()); + } + let id = match id { + Ok(id) => id, + Err(error) => { + invalid_packages.push(InvalidPackageDirectory { + id: None, + path: package_dir.to_path_buf(), + metadata_path: metadata_path.to_path_buf(), + reason: error.to_string(), + }); + return Ok(()); + } + }; + packages.push(PackageDirectory { + id, + path: package_dir.to_path_buf(), + metadata_path: metadata_path.to_path_buf(), + version_scripts: discover_version_scripts(package_dir)?, + }); + Ok(()) +} + +fn parse_package_metadata(metadata_path: &Path) -> Result<(), String> { + let bytes = fs::read(metadata_path) + .map_err(|source| format!("failed to read package metadata: {source}"))?; + let value = serde_json::from_reader::<_, serde_json::Value>(json_comments::StripComments::new( + bytes.as_slice(), + )) + .map_err(|source| format!("failed to parse package metadata: {source}"))?; + if value.is_object() { + Ok(()) + } else { + Err("package metadata must be a JSON object".to_owned()) + } +} + +fn discover_version_scripts( + package_dir: &Path, +) -> Result, RepositoryLoadError> { + let mut scripts = Vec::new(); + for entry in read_dir_entries(package_dir)? { + let path = entry.path(); + let file_type = entry_file_type(&entry, package_dir)?; + if !file_type.is_file() { + continue; + } + let Some(file_name) = path.file_name().and_then(|name| name.to_str()) else { + continue; + }; + if file_name.starts_with('.') { + continue; + } + if path + .extension() + .is_none_or(|extension| extension != LUA_SCRIPT_EXTENSION) + { + continue; + } + let Some(version) = path.file_stem().and_then(|stem| stem.to_str()) else { + continue; + }; + scripts.push(PackageVersionScript { + version: version.to_owned(), + file_name: file_name.to_owned(), + path, + }); + } + scripts.sort_by_key(|script| script.file_name.clone()); + Ok(scripts) +} + +fn read_dir_entries(path: &Path) -> Result, RepositoryLoadError> { + fs::read_dir(path) + .map_err(|source| RepositoryLoadError::ReadPackagesDir { + path: path.to_path_buf(), + source, + })? + .collect::, _>>() + .map_err(|source| RepositoryLoadError::ReadPackagesDir { + path: path.to_path_buf(), + source, + }) +} + +fn entry_file_type( + entry: &fs::DirEntry, + parent: &Path, +) -> Result { + entry + .file_type() + .map_err(|source| RepositoryLoadError::ReadPackagesDir { + path: parent.to_path_buf(), + source, + }) +} + +pub fn package_id_from_package_dir( + repository_root: impl AsRef, + package_dir: impl AsRef, +) -> Result { + let repository_root = repository_root.as_ref(); + let package_dir = package_dir.as_ref(); + let relative = package_dir.strip_prefix(repository_root).map_err(|_| { + RepositoryLoadError::InvalidPackagePath { + path: package_dir.to_path_buf(), + reason: format!("path is not under {}", repository_root.display()), + } + })?; + package_id_from_relative_path(package_dir, relative) +} + pub fn package_id_from_path( packages_root: impl AsRef, path: impl AsRef, @@ -447,8 +657,14 @@ pub fn package_id_from_path( reason: "package file must have .lua extension".to_owned(), }); } - let without_extension = relative.with_extension(""); - let mut parts = without_extension.components(); + package_id_from_relative_path(path, &relative.with_extension("")) +} + +fn package_id_from_relative_path( + path: &Path, + relative: &Path, +) -> Result { + let mut parts = relative.components(); let kind = parts .next() .ok_or_else(|| RepositoryLoadError::InvalidPackagePath { @@ -674,6 +890,156 @@ mod tests { assert_eq!(id.to_string(), "android/org.fdroid.fdroid"); } + #[test] + fn derives_package_id_from_package_directory() { + let root = PathBuf::from("repo/official"); + let id = package_id_from_package_dir(&root, "repo/official/android/f-droid/magisk/hello") + .unwrap(); + assert_eq!(id.to_string(), "android/f-droid/magisk/hello"); + } + + #[test] + fn discovers_package_directories_and_direct_child_version_scripts() { + let temp = tempfile::tempdir().unwrap(); + let root = temp.path(); + let package_dir = root.join("android/app/org.fdroid.fdroid"); + fs::create_dir_all(package_dir.join("nested")).unwrap(); + fs::write( + package_dir.join(PACKAGE_METADATA_FILE), + r#"{ "type": "android:app" }"#, + ) + .unwrap(); + fs::write(package_dir.join("1.2.3.lua"), "return {}").unwrap(); + fs::write(package_dir.join("9999.lua"), "return {}").unwrap(); + fs::write(package_dir.join(".disabled.lua"), "return {}").unwrap(); + fs::write(package_dir.join("nested/2.0.lua"), "return {}").unwrap(); + + let layout = RepositoryPackageDirectoryLayout::load(root).unwrap(); + + assert_eq!(layout.invalid_packages, Vec::new()); + assert_eq!(layout.packages.len(), 1); + let package = &layout.packages[0]; + assert_eq!(package.id.to_string(), "android/app/org.fdroid.fdroid"); + assert_eq!(package.path, package_dir); + assert_eq!( + package.metadata_path, + package.path.join(PACKAGE_METADATA_FILE) + ); + assert_eq!( + package + .version_scripts + .iter() + .map(|script| (script.version.as_str(), script.file_name.as_str())) + .collect::>(), + vec![("1.2.3", "1.2.3.lua"), ("9999", "9999.lua")] + ); + } + + #[test] + fn package_metadata_boundary_stops_nested_discovery_even_when_invalid() { + let temp = tempfile::tempdir().unwrap(); + let root = temp.path(); + let package_dir = root.join("android/app/broken"); + fs::create_dir_all(package_dir.join("nested/android/app/hidden")).unwrap(); + fs::write(package_dir.join(PACKAGE_METADATA_FILE), "{not-json").unwrap(); + fs::write( + package_dir.join("nested/android/app/hidden/metadata.jsonc"), + r#"{ "type": "android:app" }"#, + ) + .unwrap(); + + let layout = RepositoryPackageDirectoryLayout::load(root).unwrap(); + + assert!(layout.packages.is_empty()); + assert_eq!(layout.invalid_packages.len(), 1); + assert_eq!( + layout.invalid_packages[0] + .id + .as_ref() + .map(ToString::to_string), + Some("android/app/broken".to_owned()) + ); + } + + #[test] + fn repository_reserved_roots_are_not_package_discovery_roots() { + let temp = tempfile::tempdir().unwrap(); + let root = temp.path(); + fs::create_dir_all(root.join(".metadata/android/app/hidden")).unwrap(); + fs::write( + root.join(".metadata/android/app/hidden/metadata.jsonc"), + r#"{ "type": "android:app" }"#, + ) + .unwrap(); + fs::create_dir_all(root.join("luaclass/android/app/hidden")).unwrap(); + fs::write( + root.join("luaclass/android/app/hidden/metadata.jsonc"), + r#"{ "type": "android:app" }"#, + ) + .unwrap(); + fs::create_dir_all(root.join("android/app/visible")).unwrap(); + fs::write( + root.join("android/app/visible/metadata.jsonc"), + r#"{ "type": "android:app" }"#, + ) + .unwrap(); + + let layout = RepositoryPackageDirectoryLayout::load(root).unwrap(); + + assert_eq!(layout.packages.len(), 1); + assert_eq!(layout.packages[0].id.to_string(), "android/app/visible"); + } + + #[test] + fn autogen_record_alone_does_not_create_package_boundary() { + let temp = tempfile::tempdir().unwrap(); + let root = temp.path(); + fs::create_dir_all(root.join("android/app/generated")).unwrap(); + fs::write(root.join("android/app/generated/.autogen.jsonc"), "{}").unwrap(); + + let layout = RepositoryPackageDirectoryLayout::load(root).unwrap(); + + assert!(layout.packages.is_empty()); + assert!(layout.invalid_packages.is_empty()); + } + + #[test] + fn package_metadata_must_be_an_object() { + let temp = tempfile::tempdir().unwrap(); + let root = temp.path(); + fs::create_dir_all(root.join("android/app/not-object")).unwrap(); + fs::write(root.join("android/app/not-object/metadata.jsonc"), "[]").unwrap(); + + let layout = RepositoryPackageDirectoryLayout::load(root).unwrap(); + + assert!(layout.packages.is_empty()); + assert_eq!(layout.invalid_packages.len(), 1); + assert_eq!( + layout.invalid_packages[0].reason, + "package metadata must be a JSON object" + ); + } + + #[test] + fn invalid_package_path_is_reported_as_invalid_package() { + let temp = tempfile::tempdir().unwrap(); + let root = temp.path(); + fs::create_dir_all(root.join("invalidkind/app/example")).unwrap(); + fs::write( + root.join("invalidkind/app/example/metadata.jsonc"), + r#"{ "type": "android:app" }"#, + ) + .unwrap(); + + let layout = RepositoryPackageDirectoryLayout::load(root).unwrap(); + + assert!(layout.packages.is_empty()); + assert_eq!(layout.invalid_packages.len(), 1); + assert!(layout.invalid_packages[0] + .reason + .contains("unsupported package kind")); + } + #[test] fn loads_repository_layout_with_required_directories() { let temp = tempfile::tempdir().unwrap(); From 8e880e689457ae48aad3e73ccf7dc12acbaabb0e Mon Sep 17 00:00:00 2001 From: Xiangzhe Date: Sat, 27 Jun 2026 00:23:38 +0800 Subject: [PATCH 30/52] feat(autogen): write installed packages as directories --- Cargo.lock | 67 ++ crates/getter-cli/src/lib.rs | 44 +- crates/getter-cli/tests/bdd_cli.rs | 130 ++- .../features/cli/autogen_installed.feature | 28 +- crates/getter-core/Cargo.toml | 1 + crates/getter-core/src/autogen.rs | 380 ++++---- crates/getter-operations/Cargo.toml | 1 + crates/getter-operations/src/autogen.rs | 840 +++++++++++------- 8 files changed, 942 insertions(+), 549 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 23480e3..34622f9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -91,6 +91,15 @@ version = "2.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4388bee8683e3d04af747c73422af53102d2bd24d9eadb6cbc100baef4b43f8" +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + [[package]] name = "bstr" version = "1.12.1" @@ -229,6 +238,15 @@ version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + [[package]] name = "crossbeam-deque" version = "0.8.6" @@ -254,6 +272,16 @@ version = "0.8.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "typenum", +] + [[package]] name = "cucumber" version = "0.23.0" @@ -334,6 +362,16 @@ dependencies = [ "unicode-xid", ] +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", +] + [[package]] name = "either" version = "1.16.0" @@ -480,6 +518,16 @@ dependencies = [ "slab", ] +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + [[package]] name = "getrandom" version = "0.2.17" @@ -540,6 +588,7 @@ dependencies = [ "mlua", "serde", "serde_json", + "sha2", "tempfile", "thiserror 1.0.69", "toml", @@ -568,6 +617,7 @@ dependencies = [ "getter-core", "getter-providers", "getter-storage", + "json_comments", "serde", "serde_json", "tempfile", @@ -1364,6 +1414,17 @@ dependencies = [ "serde", ] +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + [[package]] name = "shlex" version = "2.0.1" @@ -1625,6 +1686,12 @@ dependencies = [ "syn", ] +[[package]] +name = "typenum" +version = "1.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" + [[package]] name = "unicode-ident" version = "1.0.24" diff --git a/crates/getter-cli/src/lib.rs b/crates/getter-cli/src/lib.rs index 020e1f9..9b03e76 100644 --- a/crates/getter-cli/src/lib.rs +++ b/crates/getter-cli/src/lib.rs @@ -8,7 +8,8 @@ use getter_core::autogen::InstalledInventory; use getter_core::diagnostics::validate_repository_path; use getter_core::lua::evaluate_package_file; use getter_core::repository::{ - GetterDataDirLayout, RepositoryLayout, RepositoryMetadata, REPOSITORY_ROOT_METADATA_FILE, + default_repository_priority, GetterDataDirLayout, RepositoryLayout, RepositoryMetadata, + RepositoryPackageDirectoryLayout, REPOSITORY_ROOT_METADATA_FILE, REPO_API_VERSION_V1, }; use getter_core::runtime::{GetterRuntime, SealedActionPlan}; use getter_core::task::{ @@ -588,17 +589,7 @@ fn execute(invocation: CliInvocation) -> Result { } CliCommand::RepoAdd { id, path, priority } => { let db = open_main_db(&invocation.data_dir)?; - let layout = load_repository_layout(&path)?; - if layout.metadata.id != id { - return Err(CliError::Repository(format!( - "repo.toml id '{}' does not match requested id '{}'", - layout.metadata.id, id - ))); - } - let metadata = RepositoryMetadata { - priority: priority.unwrap_or(layout.metadata.priority), - ..layout.metadata.clone() - }; + let metadata = load_repository_metadata(&id, &path, priority)?; db.upsert_repository(&metadata, Some(&path), None)?; Ok(json!({ "repository": repository_metadata_json(&metadata, Some(&path), None) })) } @@ -938,6 +929,35 @@ fn load_repository_layout(path: &Path) -> Result { RepositoryLayout::load(path).map_err(|source| CliError::Repository(source.to_string())) } +fn load_repository_metadata( + id: &RepositoryId, + path: &Path, + priority: Option, +) -> Result { + if path.join("repo.toml").is_file() { + let layout = load_repository_layout(path)?; + if &layout.metadata.id != id { + return Err(CliError::Repository(format!( + "repo.toml id '{}' does not match requested id '{}'", + layout.metadata.id, id + ))); + } + return Ok(RepositoryMetadata { + priority: priority.unwrap_or(layout.metadata.priority), + ..layout.metadata + }); + } + + RepositoryPackageDirectoryLayout::load(path) + .map_err(|source| CliError::Repository(source.to_string()))?; + Ok(RepositoryMetadata { + id: id.clone(), + name: id.to_string(), + priority: priority.unwrap_or_else(|| default_repository_priority(id.as_str())), + api_version: REPO_API_VERSION_V1.to_owned(), + }) +} + fn list_repositories(db: &MainDb) -> Result, CliError> { Ok(db .repositories()? diff --git a/crates/getter-cli/tests/bdd_cli.rs b/crates/getter-cli/tests/bdd_cli.rs index 8a8e950..e5e2787 100644 --- a/crates/getter-cli/tests/bdd_cli.rs +++ b/crates/getter-cli/tests/bdd_cli.rs @@ -180,7 +180,7 @@ fn tampered_autogen_cleanup_preview_for_package(world: &mut CliWorld, package_id "package_id": package_id, "action": "delete", "output_relative_path": package_relative_path(&package_id), - "content_hash": "fnv1a64:0000000000000000", + "content_hash": "sha512:00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", "reason": "not_in_installed_inventory", } ], @@ -897,7 +897,10 @@ fn command_fails_with_autogen_error(world: &mut CliWorld) { assert_eq!(output.status.code(), Some(1)); let json = parse_stdout(output); assert_eq!(json["ok"], false); - assert_eq!(json["command"], "autogen cleanup apply"); + assert!(matches!( + json["command"].as_str(), + Some("autogen cleanup apply" | "autogen installed apply") + )); assert_eq!(json["error"]["code"], "autogen.error"); world.json = Some(json); } @@ -1315,8 +1318,16 @@ fn save_autogen_preview_to_file(world: &mut CliWorld) { #[then(expr = "the autogen repository contains generated package {string}")] fn autogen_repository_contains_generated_package(world: &mut CliWorld, package_id: String) { let path = autogen_repo_path(world).join(package_relative_path(&package_id)); - assert!(path.is_file(), "generated package should exist: {path:?}"); - let content = fs::read_to_string(&path).expect("generated package readable"); + assert!( + path.is_dir(), + "generated package directory should exist: {path:?}" + ); + assert!(path.join("metadata.jsonc").is_file()); + assert!(path.join("Manifest").is_file()); + let version_script = path.join("9999.lua"); + assert!(version_script.is_file()); + assert!(path.join(".autogen.jsonc").is_file()); + let content = fs::read_to_string(version_script).expect("generated package readable"); assert!(content.contains("@generated by UpgradeAll getter autogen")); } @@ -1374,8 +1385,15 @@ fn autogen_cleanup_preview_contains_delete_candidate(world: &mut CliWorld, packa fn autogen_repository_does_not_contain_generated_package(world: &mut CliWorld, package_id: String) { let path = autogen_repo_path(world).join(package_relative_path(&package_id)); assert!( - !path.exists(), - "generated package should be deleted: {path:?}" + path.is_dir(), + "cleanup keeps generated package directory: {path:?}" + ); + assert_eq!( + fs::read_dir(&path) + .expect("generated package dir readable") + .count(), + 0, + "generated package directory should be empty after cleanup: {path:?}" ); } @@ -1398,32 +1416,30 @@ fn replace_generated_autogen_package_with_user_edited_content( package_id: String, ) { let path = autogen_repo_path(world).join(package_relative_path(&package_id)); - assert!(path.is_file(), "generated package should exist before edit"); + let version_script = path.join("9999.lua"); + assert!( + version_script.is_file(), + "generated package version script should exist before edit" + ); fs::write( - &path, - format!( - "-- user edited\nreturn package_def {{ id = {id:?}, name = \"Edited Autogen\" }}\n", - id = package_id - ), + &version_script, + "#!/bin/upa-lua v1\n-- user edited\nreturn package_version {}\n", ) .expect("overwrite generated package with user edit"); } -#[then(expr = "local repository contains preserved package {string}")] -fn local_repository_contains_preserved_package(world: &mut CliWorld, package_id: String) { +#[then("local repository has not been written")] +fn local_repository_has_not_been_written(world: &mut CliWorld) { let local_path = world .data_dir .as_ref() .expect("data dir exists") .join("repo") - .join("local") - .join(package_relative_path(&package_id)); + .join("local"); assert!( - local_path.is_file(), - "modified autogen file should be preserved into local: {local_path:?}" + !local_path.exists(), + "autogen ownership conflicts must not preserve files into local: {local_path:?}" ); - let content = fs::read_to_string(local_path).expect("preserved local file readable"); - assert!(content.contains("Edited Autogen")); } #[then("no partially usable imported state is created")] @@ -1608,12 +1624,7 @@ fn autogen_repo_path(world: &CliWorld) -> PathBuf { } fn package_relative_path(package_id: &str) -> PathBuf { - let (kind, name) = package_id - .split_once('/') - .expect("test package id has kind/name"); - PathBuf::from("packages") - .join(kind) - .join(format!("{name}.lua")) + package_id.split('/').collect() } fn create_fixture_lua_repository( @@ -1624,23 +1635,49 @@ fn create_fixture_lua_repository( ) { let temp = world.temp.as_ref().expect("tempdir exists"); let repo_path = temp.path().join(format!("repo-{repo_id}")); - let package_name_path = package_id - .strip_prefix("android/") - .expect("fixture package id should be android package id"); - fs::create_dir_all(repo_path.join("packages/android")).expect("create packages dir"); - fs::create_dir(repo_path.join("lib")).expect("create lib dir"); - fs::create_dir(repo_path.join("templates")).expect("create templates dir"); - fs::write( - repo_path.join("repo.toml"), - format!( - "id = \"{repo_id}\"\nname = \"Fixture {repo_id}\"\npriority = 0\napi_version = \"getter.repo.v1\"\n" - ), - ) - .expect("write repo.toml"); - fs::write( - repo_path.join(format!("packages/android/{package_name_path}.lua")), - format!( - r#" + if let Some(package_name_path) = package_id.strip_prefix("android/app/") { + let package_dir = repo_path.join(package_relative_path(&package_id)); + fs::create_dir_all(&package_dir).expect("create package dir"); + fs::write( + package_dir.join("metadata.jsonc"), + format!( + r#"{{ "type": "android:app", "display_name": {name:?}, "android": {{ "package_name": {package_name_path:?} }} }}"#, + name = package_name, + ), + ) + .expect("write package metadata"); + fs::write(package_dir.join("Manifest"), "").expect("write Manifest"); + fs::write( + package_dir.join("9999.lua"), + format!( + r#"#!/bin/upa-lua v1 +return package_version {{ + installed = {{ + {{ kind = "android_package", package_name = "{package_name_path}" }}, + }}, +}} +"# + ), + ) + .expect("write package Lua"); + } else { + let package_name_path = package_id + .strip_prefix("android/") + .expect("fixture package id should be android package id"); + fs::create_dir_all(repo_path.join("packages/android")).expect("create packages dir"); + fs::create_dir(repo_path.join("lib")).expect("create lib dir"); + fs::create_dir(repo_path.join("templates")).expect("create templates dir"); + fs::write( + repo_path.join("repo.toml"), + format!( + "id = \"{repo_id}\"\nname = \"Fixture {repo_id}\"\npriority = 0\napi_version = \"getter.repo.v1\"\n" + ), + ) + .expect("write repo.toml"); + fs::write( + repo_path.join(format!("packages/android/{package_name_path}.lua")), + format!( + r#" return package_def {{ id = "{package_id}", name = "{package_name}", @@ -1649,9 +1686,10 @@ return package_def {{ }}, }} "# - ), - ) - .expect("write package Lua"); + ), + ) + .expect("write package Lua"); + } world.fixture_repo_id = Some(repo_id); world.fixture_repo_path = Some(repo_path); diff --git a/crates/getter-cli/tests/features/cli/autogen_installed.feature b/crates/getter-cli/tests/features/cli/autogen_installed.feature index 5bbb8c9..695dc78 100644 --- a/crates/getter-cli/tests/features/cli/autogen_installed.feature +++ b/crates/getter-cli/tests/features/cli/autogen_installed.feature @@ -6,10 +6,10 @@ Feature: Installed app autogen When I run getter autogen installed preview for that inventory Then the command succeeds And the output is valid JSON - And the autogen preview contains candidate "android/com.example.autogen" + And the autogen preview contains candidate "android/app/com.example.autogen" And the autogen repository has not been written - Scenario: User applies installed app autogen and evaluates the generated fallback package + Scenario: User applies installed app autogen and validates the generated fallback package directory Given an initialized getter data directory And an installed inventory with Android app "com.example.autogen" labeled "Example Autogen" When I run getter autogen installed preview for that inventory @@ -17,22 +17,20 @@ Feature: Installed app autogen And I save the autogen preview to a file When I run getter autogen installed apply for that preview with accept-all Then the command succeeds - And the autogen repository contains generated package "android/com.example.autogen" - And the app list contains autogen tracked package "android/com.example.autogen" + And the autogen repository contains generated package "android/app/com.example.autogen" + And the app list contains autogen tracked package "android/app/com.example.autogen" When I run getter repo validate for autogen Then the output reports a valid repository without network - When I run getter package eval for package "android/com.example.autogen" from autogen - Then the output contains package "android/com.example.autogen" named "Example Autogen" Scenario: Higher-priority repositories suppress installed app autogen candidates Given an initialized getter data directory - And a fixture Lua repository "official" with package "android/com.example.autogen" named "Official Example" + And a fixture Lua repository "official" with package "android/app/com.example.autogen" named "Official Example" And an installed inventory with Android app "com.example.autogen" labeled "Example Autogen" When I run getter repo add for that repository with priority 0 Then the command succeeds When I run getter autogen installed preview for that inventory Then the command succeeds - And the autogen preview skips package "android/com.example.autogen" because repository "official" covers it + And the autogen preview skips package "android/app/com.example.autogen" because repository "official" covers it Scenario: Cleanup deletes only accepted generated packages missing from installed inventory Given an initialized getter data directory @@ -45,12 +43,12 @@ Feature: Installed app autogen Given an empty installed inventory When I run getter autogen cleanup preview for that inventory Then the command succeeds - And the autogen cleanup preview contains delete candidate "android/com.example.old" + And the autogen cleanup preview contains delete candidate "android/app/com.example.old" And I save the autogen preview to a file When I run getter autogen cleanup apply for that preview with accept-all Then the command succeeds - And the autogen repository does not contain generated package "android/com.example.old" - And the app list does not contain package "android/com.example.old" + And the autogen repository does not contain generated package "android/app/com.example.old" + And the app list does not contain package "android/app/com.example.old" Scenario: Cleanup rejects tampered previews for non-autogen tracked packages Given an initialized getter data directory @@ -75,7 +73,7 @@ Feature: Installed app autogen Then the command succeeds And the app list contains imported package "android/org.fdroid.fdroid" - Scenario: Applying over modified autogen preserves the user-edited file into local + Scenario: Applying over modified autogen reports an ownership conflict Given an initialized getter data directory And an installed inventory with Android app "com.example.autogen" labeled "Example Autogen" When I run getter autogen installed preview for that inventory @@ -83,10 +81,10 @@ Feature: Installed app autogen And I save the autogen preview to a file When I run getter autogen installed apply for that preview with accept-all Then the command succeeds - And I replace generated autogen package "android/com.example.autogen" with user-edited content + And I replace generated autogen package "android/app/com.example.autogen" with user-edited content When I run getter autogen installed preview for that inventory Then the command succeeds And I save the autogen preview to a file When I run getter autogen installed apply for that preview with accept-all - Then the command succeeds - And local repository contains preserved package "android/com.example.autogen" + Then the command fails with an autogen error + And local repository has not been written diff --git a/crates/getter-core/Cargo.toml b/crates/getter-core/Cargo.toml index 4eac396..4b1f9c5 100644 --- a/crates/getter-core/Cargo.toml +++ b/crates/getter-core/Cargo.toml @@ -12,6 +12,7 @@ json_comments = "0.2" mlua = { version = "0.10", features = ["luajit", "vendored"], optional = true } serde = { version = "1", features = ["derive"] } serde_json = "1" +sha2 = "0.10" thiserror = "1" toml = "0.8" diff --git a/crates/getter-core/src/autogen.rs b/crates/getter-core/src/autogen.rs index a3bfe8e..8a30220 100644 --- a/crates/getter-core/src/autogen.rs +++ b/crates/getter-core/src/autogen.rs @@ -1,20 +1,20 @@ //! Installed-inventory autogen planning for generated fallback repositories. //! //! This module is pure domain logic: platform adapters provide installed -//! inventory facts, while getter decides package ids, generated Lua shape, -//! deterministic paths, and preview DTOs. +//! inventory facts, while getter decides package ids, generated package +//! directory contents, deterministic paths, and preview DTOs. -use crate::repository::REPO_API_VERSION_V1; use crate::{InstalledTarget, PackageId, PackageKind, RepositoryId, RepositoryPriority}; use serde::{Deserialize, Serialize}; -use std::collections::{BTreeSet, HashMap}; -use std::path::PathBuf; +use sha2::{Digest, Sha512}; +use std::collections::{BTreeMap, BTreeSet, HashMap}; +use std::path::{Path, PathBuf}; pub const INSTALLED_INVENTORY_FORMAT: &str = "upgradeall-installed-inventory"; pub const INSTALLED_INVENTORY_VERSION: u32 = 1; -pub const AUTOGEN_MANIFEST_VERSION: u32 = 1; -pub const LOCAL_REPOSITORY_ID: &str = "local"; -pub const LOCAL_REPOSITORY_NAME: &str = "Local"; +pub const AUTOGEN_RECORD_FILE: &str = ".autogen.jsonc"; +pub const AUTOGEN_RECORD_VERSION: u32 = 1; +pub const INSTALLED_AUTOGEN_GENERATOR: &str = "installed-inventory"; pub const DEFAULT_AUTOGEN_REPOSITORY_ID: &str = "autogen"; pub const DEFAULT_AUTOGEN_REPOSITORY_NAME: &str = "UpgradeAll Autogen"; pub const GENERATED_MARKER: &str = "@generated by UpgradeAll getter autogen"; @@ -76,11 +76,39 @@ pub struct AutogenCandidate { pub package_id: PackageId, pub name: String, pub installed: InstalledTarget, + pub relative_path: PathBuf, + pub files: Vec, + pub record: AutogenRecord, + /// Hash of the serialized `.autogen.jsonc` record content. + pub content_hash: String, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct GeneratedPackageFile { pub relative_path: PathBuf, pub content: String, pub content_hash: String, } +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct AutogenRecord { + pub version: u32, + pub generator: String, + pub package_id: PackageId, + pub output_relative_path: PathBuf, + pub input: AutogenRecordInput, + /// Getter-written generated files and their content hashes. + /// `.autogen.jsonc` itself is intentionally excluded. + pub files: BTreeMap, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "snake_case")] +pub enum AutogenRecordInput { + InstalledAndroidPackage { package_name: String }, + InstalledMagiskModule { module_id: String }, +} + #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct AutogenSkip { pub package_id: PackageId, @@ -96,44 +124,6 @@ pub enum AutogenSkipReason { CoveredByHigherPriorityRepository, } -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -pub struct AutogenManifest { - pub version: u32, - pub repository_id: RepositoryId, - pub packages: Vec, -} - -impl AutogenManifest { - pub fn from_candidates(candidates: &[AutogenCandidate]) -> Self { - Self { - version: AUTOGEN_MANIFEST_VERSION, - repository_id: RepositoryId::new(DEFAULT_AUTOGEN_REPOSITORY_ID) - .expect("autogen is a valid repository id"), - packages: candidates - .iter() - .map(|candidate| AutogenManifestEntry { - package_id: candidate.package_id.clone(), - relative_path: candidate.relative_path.clone(), - content_hash: candidate.content_hash.clone(), - }) - .collect(), - } - } - - pub fn package(&self, package_id: &PackageId) -> Option<&AutogenManifestEntry> { - self.packages - .iter() - .find(|entry| &entry.package_id == package_id) - } -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -pub struct AutogenManifestEntry { - pub package_id: PackageId, - pub relative_path: PathBuf, - pub content_hash: String, -} - #[derive(Debug, thiserror::Error)] pub enum AutogenError { #[error("unsupported installed inventory format '{0}'")] @@ -144,14 +134,16 @@ pub enum AutogenError { EmptyIdentity, #[error("failed to construct package id from installed inventory: {0}")] PackageId(#[from] crate::PackageIdError), + #[error("failed to serialize generated autogen record: {0}")] + SerializeRecord(#[from] serde_json::Error), } /// Build a deterministic installed-autogen preview from installed inventory. /// /// `covered_packages` contains package ids already supplied by repositories with /// priority higher than the generated repository. Those packages are skipped so -/// generated fallback files do not compete with user-authored `local` or -/// upstream package files. +/// generated fallback directories do not compete with user-authored `local` or +/// upstream package directories. pub fn plan_installed_autogen( inventory: &InstalledInventory, covered_packages: &HashMap, @@ -210,38 +202,25 @@ pub fn validate_installed_inventory(inventory: &InstalledInventory) -> Result<() Ok(()) } -pub fn default_autogen_repo_toml() -> String { - repo_toml( - DEFAULT_AUTOGEN_REPOSITORY_ID, - DEFAULT_AUTOGEN_REPOSITORY_NAME, - RepositoryPriority::GENERATED_FALLBACK, - ) +pub fn content_hash(content: &str) -> String { + content_hash_bytes(content.as_bytes()) } -pub fn local_repo_toml() -> String { - repo_toml( - LOCAL_REPOSITORY_ID, - LOCAL_REPOSITORY_NAME, - RepositoryPriority::LOCAL, - ) +pub fn content_hash_bytes(content: &[u8]) -> String { + let mut hasher = Sha512::new(); + hasher.update(content); + format!("sha512:{:x}", hasher.finalize()) } -fn repo_toml(id: &str, name: &str, priority: RepositoryPriority) -> String { - format!( - "id = \"{id}\"\nname = \"{name}\"\npriority = {}\napi_version = \"{REPO_API_VERSION_V1}\"\n", - priority.value() - ) -} - -pub fn generated_file_is_managed(content: &str) -> bool { - content - .lines() - .next() - .is_some_and(|line| line.contains(GENERATED_MARKER)) +pub fn render_autogen_record(record: &AutogenRecord) -> Result { + serde_json::to_string_pretty(record).map(|mut content| { + content.push('\n'); + content + }) } -pub fn content_hash(content: &str) -> String { - format!("fnv1a64:{:016x}", fnv1a64(content.as_bytes())) +pub fn package_relative_path(package_id: &PackageId) -> PathBuf { + package_id.to_string().split('/').collect() } fn candidate_from_inventory_item( @@ -254,7 +233,7 @@ fn candidate_from_inventory_item( .. } => { let package_name = non_empty(package_name)?; - let package_id = PackageId::new(PackageKind::Android, package_name)?; + let package_id = PackageId::new(PackageKind::Android, format!("app/{package_name}"))?; let name = label .as_deref() .filter(|label| !label.trim().is_empty()) @@ -263,17 +242,10 @@ fn candidate_from_inventory_item( let installed = InstalledTarget::AndroidPackage { package_name: package_name.to_owned(), }; - let relative_path = package_relative_path(&package_id); - let content = render_package_lua(&package_id, &name, &installed); - let content_hash = content_hash(&content); - Ok(AutogenCandidate { - package_id, - name, - installed, - relative_path, - content, - content_hash, - }) + let input = AutogenRecordInput::InstalledAndroidPackage { + package_name: package_name.to_owned(), + }; + candidate(package_id, name, installed, input) } InstalledInventoryItem::MagiskModule { module_id, name, .. @@ -288,38 +260,113 @@ fn candidate_from_inventory_item( let installed = InstalledTarget::MagiskModule { module_id: module_id.to_owned(), }; - let relative_path = package_relative_path(&package_id); - let content = render_package_lua(&package_id, &name, &installed); - let content_hash = content_hash(&content); - Ok(AutogenCandidate { - package_id, - name, - installed, - relative_path, - content, - content_hash, - }) + let input = AutogenRecordInput::InstalledMagiskModule { + module_id: module_id.to_owned(), + }; + candidate(package_id, name, installed, input) } } } -fn non_empty(value: &str) -> Result<&str, AutogenError> { - let value = value.trim(); - if value.is_empty() { - Err(AutogenError::EmptyIdentity) - } else { - Ok(value) +fn candidate( + package_id: PackageId, + name: String, + installed: InstalledTarget, + input: AutogenRecordInput, +) -> Result { + let relative_path = package_relative_path(&package_id); + let files = generated_package_files(&name, &installed)?; + let record = autogen_record(package_id.clone(), relative_path.clone(), input, &files); + let content_hash = content_hash(&render_autogen_record(&record)?); + Ok(AutogenCandidate { + package_id, + name, + installed, + relative_path, + files, + record, + content_hash, + }) +} + +fn generated_package_files( + name: &str, + installed: &InstalledTarget, +) -> Result, AutogenError> { + let metadata = match installed { + InstalledTarget::AndroidPackage { package_name } => serde_json::json!({ + "type": "android:app", + "display_name": name, + "android": { "package_name": package_name }, + }), + InstalledTarget::MagiskModule { module_id } => serde_json::json!({ + "type": "magisk:module", + "display_name": name, + "magisk": { "module_id": module_id }, + }), + InstalledTarget::Generic { id } => serde_json::json!({ + "type": "generic", + "display_name": name, + "generic": { "id": id }, + }), + }; + let metadata = render_pretty_json(metadata)?; + let manifest = String::new(); + let version_lua = render_version_lua(installed); + Ok(vec![ + generated_file("metadata.jsonc", metadata), + generated_file("Manifest", manifest), + generated_file("9999.lua", version_lua), + ]) +} + +fn render_pretty_json(value: serde_json::Value) -> Result { + serde_json::to_string_pretty(&value).map(|mut content| { + content.push('\n'); + content + }) +} + +fn generated_file(relative_path: &str, content: String) -> GeneratedPackageFile { + GeneratedPackageFile { + relative_path: PathBuf::from(relative_path), + content_hash: content_hash(&content), + content, } } -pub fn package_relative_path(package_id: &PackageId) -> PathBuf { - let mut path = PathBuf::from("packages"); - path.push(package_id.kind().as_str()); - path.push(format!("{}.lua", package_id.name())); - path +fn autogen_record( + package_id: PackageId, + output_relative_path: PathBuf, + input: AutogenRecordInput, + files: &[GeneratedPackageFile], +) -> AutogenRecord { + AutogenRecord { + version: AUTOGEN_RECORD_VERSION, + generator: INSTALLED_AUTOGEN_GENERATOR.to_owned(), + package_id, + output_relative_path, + input, + files: files + .iter() + .map(|file| { + ( + record_file_key(&file.relative_path), + file.content_hash.clone(), + ) + }) + .collect(), + } } -fn render_package_lua(package_id: &PackageId, name: &str, installed: &InstalledTarget) -> String { +pub fn record_file_key(path: &Path) -> String { + path.components() + .map(|component| component.as_os_str().to_string_lossy()) + .collect::>() + .join("/") +} + +fn render_version_lua(installed: &InstalledTarget) -> String { let installed_lua = match installed { InstalledTarget::AndroidPackage { package_name } => format!( "{{ kind = \"android_package\", package_name = {} }}", @@ -335,9 +382,7 @@ fn render_package_lua(package_id: &PackageId, name: &str, installed: &InstalledT }; format!( - "-- {GENERATED_MARKER}\nreturn package_def {{\n id = {},\n name = {},\n installed = {{\n {installed_lua},\n }},\n}}\n", - lua_string(&package_id.to_string()), - lua_string(name) + "#!/bin/upa-lua v1\n-- {GENERATED_MARKER}\nreturn package_version {{\n installed = {{\n {installed_lua},\n }},\n}}\n", ) } @@ -357,27 +402,23 @@ fn lua_string(value: &str) -> String { escaped } -fn fnv1a64(bytes: &[u8]) -> u64 { - let mut hash = 0xcbf29ce484222325u64; - for byte in bytes { - hash ^= u64::from(*byte); - hash = hash.wrapping_mul(0x100000001b3); +fn non_empty(value: &str) -> Result<&str, AutogenError> { + let value = value.trim(); + if value.is_empty() { + Err(AutogenError::EmptyIdentity) + } else { + Ok(value) } - hash } #[cfg(test)] mod tests { use super::*; - #[cfg(feature = "lua")] - use crate::lua::evaluate_package_file; - #[cfg(feature = "lua")] - use crate::repository::RepositoryLayout; - #[cfg(feature = "lua")] + use crate::repository::RepositoryPackageDirectoryLayout; use std::fs; #[test] - fn preview_generates_deterministic_android_package_lua() { + fn preview_generates_deterministic_android_package_directory() { let inventory = InstalledInventory::new(vec![InstalledInventoryItem::AndroidPackage { package_name: "org.fdroid.fdroid".to_owned(), label: Some("F-Droid".to_owned()), @@ -396,20 +437,34 @@ mod tests { let candidate = &plan.candidates[0]; assert_eq!( candidate.package_id.to_string(), - "android/org.fdroid.fdroid" + "android/app/org.fdroid.fdroid" ); assert_eq!( candidate.relative_path, - PathBuf::from("packages/android/org.fdroid.fdroid.lua") + PathBuf::from("android/app/org.fdroid.fdroid") + ); + assert_eq!(candidate.files.len(), 3); + assert!(candidate + .files + .iter() + .any(|file| file.relative_path == PathBuf::from("metadata.jsonc") + && file.content.contains("org.fdroid.fdroid"))); + assert!(candidate.files.iter().any(|file| { + file.relative_path == PathBuf::from("9999.lua") + && file.content.starts_with("#!/bin/upa-lua v1\n") + && file.content.contains(GENERATED_MARKER) + })); + assert_eq!(candidate.record.package_id, candidate.package_id); + assert_eq!(candidate.record.files.len(), candidate.files.len()); + assert_eq!( + candidate.content_hash, + content_hash(&render_autogen_record(&candidate.record).unwrap()) ); - assert!(candidate.content.contains(GENERATED_MARKER)); - assert!(candidate.content.contains("package_def")); - assert_eq!(candidate.content_hash, content_hash(&candidate.content)); } #[test] fn preview_skips_packages_covered_by_higher_priority_repositories() { - let package_id: PackageId = "android/org.fdroid.fdroid".parse().unwrap(); + let package_id: PackageId = "android/app/org.fdroid.fdroid".parse().unwrap(); let mut covered = HashMap::new(); covered.insert(package_id, RepositoryId::new("official").unwrap()); let inventory = InstalledInventory::new(vec![InstalledInventoryItem::AndroidPackage { @@ -478,41 +533,12 @@ mod tests { } #[test] - fn manifest_records_generated_package_hashes() { - let inventory = InstalledInventory::new(vec![InstalledInventoryItem::AndroidPackage { - package_name: "org.fdroid.fdroid".to_owned(), - label: Some("F-Droid".to_owned()), - version_name: None, - version_code: None, - }]); - let plan = plan_installed_autogen(&inventory, &HashMap::new()).unwrap(); - - let manifest = AutogenManifest::from_candidates(&plan.candidates); - - assert_eq!(manifest.version, AUTOGEN_MANIFEST_VERSION); - assert_eq!( - manifest.repository_id.as_str(), - DEFAULT_AUTOGEN_REPOSITORY_ID - ); - assert_eq!(manifest.packages.len(), 1); - assert_eq!( - manifest.packages[0].content_hash, - content_hash(&plan.candidates[0].content) - ); - } - - #[cfg(feature = "lua")] - #[test] - fn generated_lua_evaluates_through_package_boundary() { + fn generated_files_form_a_package_directory_boundary() { let temp = tempfile::tempdir().unwrap(); let repo = temp.path(); - fs::write(repo.join("repo.toml"), default_autogen_repo_toml()).unwrap(); - fs::create_dir_all(repo.join("packages/android")).unwrap(); - fs::create_dir(repo.join("lib")).unwrap(); - fs::create_dir(repo.join("templates")).unwrap(); let inventory = InstalledInventory::new(vec![InstalledInventoryItem::AndroidPackage { package_name: "org.fdroid.fdroid".to_owned(), - label: Some("F-Droid \"client\"".to_owned()), + label: Some("F-Droid".to_owned()), version_name: None, version_code: None, }]); @@ -520,27 +546,29 @@ mod tests { .unwrap() .candidates .remove(0); - let package_path = repo.join(&candidate.relative_path); - fs::write(&package_path, candidate.content).unwrap(); + let package_dir = repo.join(&candidate.relative_path); + fs::create_dir_all(&package_dir).unwrap(); + for file in &candidate.files { + fs::write(package_dir.join(&file.relative_path), &file.content).unwrap(); + } + fs::write( + package_dir.join(AUTOGEN_RECORD_FILE), + render_autogen_record(&candidate.record).unwrap(), + ) + .unwrap(); - let layout = RepositoryLayout::load(repo).unwrap(); - let package = evaluate_package_file(&layout, &package_path).unwrap(); + let layout = RepositoryPackageDirectoryLayout::load(repo).unwrap(); - assert_eq!(package.id.to_string(), "android/org.fdroid.fdroid"); - assert_eq!(package.name, "F-Droid \"client\""); + assert_eq!(layout.packages.len(), 1); assert_eq!( - package.installed, - vec![InstalledTarget::AndroidPackage { - package_name: "org.fdroid.fdroid".to_owned(), - }] + layout.packages[0].id.to_string(), + "android/app/org.fdroid.fdroid" ); + assert_eq!(layout.packages[0].version_scripts[0].version, "9999"); } #[test] - fn generated_file_marker_identifies_managed_files_only() { - assert!(generated_file_is_managed(&format!( - "-- {GENERATED_MARKER}\nreturn {{}}" - ))); - assert!(!generated_file_is_managed("return {}")); + fn content_hash_uses_sha512_prefix() { + assert!(content_hash("hello").starts_with("sha512:")); } } diff --git a/crates/getter-operations/Cargo.toml b/crates/getter-operations/Cargo.toml index c81da6b..26863cb 100644 --- a/crates/getter-operations/Cargo.toml +++ b/crates/getter-operations/Cargo.toml @@ -8,6 +8,7 @@ default = [] lua = ["getter-core/lua"] [dependencies] +json_comments = "0.2" getter-core = { path = "../getter-core", default-features = false } getter-providers = { path = "../getter-providers" } getter-storage = { path = "../getter-storage" } diff --git a/crates/getter-operations/src/autogen.rs b/crates/getter-operations/src/autogen.rs index faa06a9..0c0b5f0 100644 --- a/crates/getter-operations/src/autogen.rs +++ b/crates/getter-operations/src/autogen.rs @@ -2,28 +2,28 @@ //! //! The CLI and native bridge both call this module so there is one implementation //! of installed-autogen preview/apply semantics. Platform layers provide installed -//! inventory facts; this module decides generated package ids, repository -//! coverage, file writes, manifest updates, preservation behavior, and tracked -//! state updates. +//! inventory facts; this module decides generated package directories, +//! repository coverage, package-local `.autogen.jsonc` ownership, file writes, +//! cleanup, and tracked state updates. use getter_core::autogen::{ - content_hash, local_repo_toml, plan_installed_autogen, AutogenManifest, AutogenManifestEntry, - AutogenPlan, AutogenSkipReason, InstalledInventory, DEFAULT_AUTOGEN_REPOSITORY_ID, - DEFAULT_AUTOGEN_REPOSITORY_NAME, LOCAL_REPOSITORY_ID, LOCAL_REPOSITORY_NAME, + content_hash, content_hash_bytes, plan_installed_autogen, record_file_key, + render_autogen_record, AutogenCandidate, AutogenPlan, AutogenRecord, AutogenSkipReason, + GeneratedPackageFile, InstalledInventory, AUTOGEN_RECORD_FILE, AUTOGEN_RECORD_VERSION, + DEFAULT_AUTOGEN_REPOSITORY_ID, DEFAULT_AUTOGEN_REPOSITORY_NAME, INSTALLED_AUTOGEN_GENERATOR, }; use getter_core::repository::{ generated_repository_target, GeneratedRepositoryTarget, GetterDataDirLayout, RepositoryLayout, - RepositoryLoadError, RepositoryMetadata, RepositoryRootConfig, REPO_API_VERSION_V1, + RepositoryLoadError, RepositoryMetadata, RepositoryPackageDirectoryLayout, + RepositoryRootConfig, REPO_API_VERSION_V1, }; use getter_core::{PackageId, RepositoryId, RepositoryPriority}; -use getter_storage::{MainDb, StorageError, StoredRepository}; +use getter_storage::{MainDb, StorageError}; use serde_json::{json, Value}; -use std::collections::{BTreeSet, HashMap}; +use std::collections::{BTreeMap, BTreeSet, HashMap}; use std::fs; use std::path::{Component, Path, PathBuf}; -pub const AUTOGEN_MANIFEST_FILE: &str = "autogen-manifest.json"; - #[derive(Debug, Clone, PartialEq, Eq)] pub enum AutogenAcceptance { AcceptAll, @@ -79,7 +79,11 @@ pub fn installed_preview_json( plan: &AutogenPlan, ) -> AutogenOperationResult { let (target_alias, target_path, _target_priority) = generated_repository_config(data_dir)?; - let candidates: Vec = plan.candidates.iter().map(autogen_candidate_json).collect(); + let candidates: Vec = plan + .candidates + .iter() + .map(autogen_candidate_json) + .collect::>()?; let skipped: Vec = plan.skipped.iter().map(autogen_skip_json).collect(); Ok(json!({ "operation": "installed.preview", @@ -103,17 +107,14 @@ pub fn cleanup_preview_json( inventory: &InstalledInventory, ) -> AutogenOperationResult { let (target_alias, repo_path, _target_priority) = generated_repository_config(data_dir)?; - let Some(manifest) = read_autogen_manifest(&repo_path)? else { - return Ok(json!({ - "operation": "cleanup.preview", - "target_repo_id": target_alias.as_str(), - "target_repo_path": repo_path, - "summary": { "candidate_count": 0, "skipped_count": 0, "write_count": 0, "delete_count": 0 }, - "candidates": [], - "skipped": [], - "diagnostics": [], - })); - }; + if !repo_path.is_dir() { + return Ok(cleanup_preview_response( + target_alias, + repo_path, + Vec::new(), + Vec::new(), + )); + } let plan = build_installed_autogen_plan(data_dir, db, inventory)?; let installed_ids: BTreeSet = plan .candidates @@ -121,14 +122,32 @@ pub fn cleanup_preview_json( .map(|candidate| candidate.package_id.to_string()) .chain(plan.skipped.iter().map(|skip| skip.package_id.to_string())) .collect(); + + let layout = RepositoryPackageDirectoryLayout::load(&repo_path)?; let mut candidates = Vec::new(); - for entry in manifest.packages { - if !installed_ids.contains(&entry.package_id.to_string()) { + let mut diagnostics = Vec::new(); + for package in layout.packages { + let relative_path = relative_package_path(&repo_path, &package.path)?; + let ownership = match read_owned_record(&package.path, &package.id, &relative_path) { + Ok(ownership) => ownership, + Err(error) => { + diagnostics.push(autogen_diagnostic( + "autogen.ownership_conflict", + format!( + "generated package '{}' has invalid ownership: {error}", + package.id + ), + Some(package.id.to_string()), + )); + continue; + } + }; + if !installed_ids.contains(&package.id.to_string()) { candidates.push(json!({ - "package_id": entry.package_id.to_string(), + "package_id": package.id.to_string(), "action": "delete", - "output_relative_path": entry.relative_path, - "content_hash": entry.content_hash, + "output_relative_path": relative_path, + "content_hash": ownership.content_hash, "reason": "not_in_installed_inventory", })); } @@ -140,20 +159,12 @@ pub fn cleanup_preview_json( .unwrap_or_default() .to_owned() }); - Ok(json!({ - "operation": "cleanup.preview", - "target_repo_id": target_alias.as_str(), - "target_repo_path": repo_path, - "summary": { - "candidate_count": candidates.len(), - "skipped_count": 0, - "write_count": 0, - "delete_count": candidates.len(), - }, - "candidates": candidates, - "skipped": [], - "diagnostics": [], - })) + Ok(cleanup_preview_response( + target_alias, + repo_path, + candidates, + diagnostics, + )) } pub fn apply_installed_preview( @@ -171,74 +182,25 @@ pub fn apply_installed_preview( } ensure_generated_repository(&repo_path, db, &target_alias, target_priority)?; let accepted = accepted_preview_candidates(preview, acceptance)?; - let mut manifest = - read_autogen_manifest(&repo_path)?.unwrap_or_else(|| empty_autogen_manifest(&target_alias)); let mut applied = Vec::new(); - let mut preserved = Vec::new(); for candidate in accepted { let package_id = preview_package_id(candidate)?; let relative_path = preview_relative_path(candidate)?; - let content = candidate - .get("content") - .and_then(Value::as_str) - .ok_or_else(|| { - AutogenOperationError::Autogen("preview candidate missing content".to_owned()) - })?; - let expected_hash = candidate - .get("content_hash") - .and_then(Value::as_str) - .ok_or_else(|| { - AutogenOperationError::Autogen("preview candidate missing content_hash".to_owned()) - })?; - if content_hash(content) != expected_hash { - return Err(AutogenOperationError::Autogen(format!( - "preview content hash mismatch for {package_id}" - ))); - } - let target = safe_join(&repo_path, &relative_path)?; - if target.exists() { - let current = fs::read_to_string(&target).map_err(|source| { - AutogenOperationError::Autogen(format!( - "failed to read existing autogen file '{}': {source}", - target.display() - )) - })?; - let current_hash = content_hash(¤t); - let known_hash = manifest - .package(&package_id) - .map(|entry| entry.content_hash.as_str()); - if known_hash != Some(current_hash.as_str()) { - preserved.push(preserve_autogen_file_in_local( - data_dir, - db, - &package_id, - ¤t, - )?); - } - } - if let Some(parent) = target.parent() { - fs::create_dir_all(parent).map_err(|source| { + let payload = preview_candidate_payload(candidate, &package_id, &relative_path)?; + let target_dir = safe_join(&repo_path, &relative_path)?; + if target_dir.exists() { + read_owned_record(&target_dir, &package_id, &relative_path)?; + clear_directory_contents(&target_dir)?; + } else { + fs::create_dir_all(&target_dir).map_err(|source| { AutogenOperationError::Autogen(format!( - "failed to create autogen package directory '{}': {source}", - parent.display() + "failed to create generated package directory '{}': {source}", + target_dir.display() )) })?; } - fs::write(&target, content).map_err(|source| { - AutogenOperationError::Autogen(format!( - "failed to write autogen package '{}': {source}", - target.display() - )) - })?; - upsert_manifest_entry( - &mut manifest, - AutogenManifestEntry { - package_id: package_id.clone(), - relative_path: relative_path.clone(), - content_hash: expected_hash.to_owned(), - }, - ); + write_generated_package(&target_dir, &payload.files, &payload.record_content)?; db.upsert_generated_tracked_package_preserving_user_state(&package_id, &target_alias)?; applied.push(json!({ "package_id": package_id.to_string(), @@ -246,13 +208,11 @@ pub fn apply_installed_preview( })); } - write_autogen_manifest(&repo_path, &manifest)?; Ok(json!({ "target_repo_id": target_alias.as_str(), "target_repo_path": repo_path, "applied_count": applied.len(), "applied": applied, - "preserved_to_local": preserved, })) } @@ -270,10 +230,7 @@ pub fn apply_cleanup_preview( ))); } let accepted = accepted_preview_candidates(preview, acceptance)?; - let mut manifest = - read_autogen_manifest(&repo_path)?.unwrap_or_else(|| empty_autogen_manifest(&target_alias)); let mut deleted = Vec::new(); - let mut preserved = Vec::new(); for candidate in accepted { let package_id = preview_package_id(candidate)?; let relative_path = preview_relative_path(candidate)?; @@ -285,57 +242,25 @@ pub fn apply_cleanup_preview( "cleanup preview candidate missing content_hash".to_owned(), ) })?; - let manifest_entry = manifest.package(&package_id).ok_or_else(|| { - AutogenOperationError::Autogen(format!( - "cleanup preview candidate {package_id} is not managed by autogen manifest" - )) - })?; - if manifest_entry.relative_path != relative_path - || manifest_entry.content_hash != expected_hash - { + let target_dir = safe_join(&repo_path, &relative_path)?; + let ownership = read_owned_record(&target_dir, &package_id, &relative_path)?; + if ownership.content_hash != expected_hash { return Err(AutogenOperationError::Autogen(format!( - "cleanup preview candidate {package_id} does not match autogen manifest" + "cleanup preview candidate {package_id} does not match current autogen record" ))); } - let target = safe_join(&repo_path, &relative_path)?; - if target.exists() { - let current = fs::read_to_string(&target).map_err(|source| { - AutogenOperationError::Autogen(format!( - "failed to read existing autogen file '{}': {source}", - target.display() - )) - })?; - if content_hash(¤t) != expected_hash { - preserved.push(preserve_autogen_file_in_local( - data_dir, - db, - &package_id, - ¤t, - )?); - } - fs::remove_file(&target).map_err(|source| { - AutogenOperationError::Autogen(format!( - "failed to delete autogen package '{}': {source}", - target.display() - )) - })?; - } - manifest - .packages - .retain(|entry| entry.package_id != package_id); + clear_directory_contents(&target_dir)?; db.delete_generated_tracked_package(&package_id, &target_alias)?; deleted.push(json!({ "package_id": package_id.to_string(), "output_relative_path": relative_path, })); } - write_autogen_manifest(&repo_path, &manifest)?; Ok(json!({ "target_repo_id": target_alias.as_str(), "target_repo_path": repo_path, "deleted_count": deleted.len(), "deleted": deleted, - "preserved_to_local": preserved, })) } @@ -391,21 +316,48 @@ fn higher_priority_package_coverage( let Some(path) = repo.path.as_ref() else { continue; }; - let layout = load_repository_layout(Path::new(path))?; - for package in layout.packages { - covered.entry(package.id).or_insert_with(|| repo.id.clone()); + for package_id in load_repository_package_ids(Path::new(path))? { + covered.entry(package_id).or_insert_with(|| repo.id.clone()); } } Ok(covered) } -fn load_repository_layout(path: &Path) -> AutogenOperationResult { - RepositoryLayout::load(path) - .map_err(|source| AutogenOperationError::Repository(source.to_string())) +fn load_repository_package_ids(path: &Path) -> AutogenOperationResult> { + if path.join("repo.toml").is_file() { + let layout = RepositoryLayout::load(path) + .map_err(|source| AutogenOperationError::Repository(source.to_string()))?; + Ok(layout + .packages + .into_iter() + .map(|package| package.id) + .collect()) + } else { + let layout = RepositoryPackageDirectoryLayout::load(path) + .map_err(|source| AutogenOperationError::Repository(source.to_string()))?; + Ok(layout + .packages + .into_iter() + .map(|package| package.id) + .collect()) + } } -fn autogen_candidate_json(candidate: &getter_core::autogen::AutogenCandidate) -> Value { - json!({ +fn autogen_candidate_json(candidate: &AutogenCandidate) -> AutogenOperationResult { + let record_content = render_autogen_record(&candidate.record) + .map_err(|source| AutogenOperationError::Autogen(source.to_string()))?; + let files: Vec = candidate + .files + .iter() + .map(|file| { + json!({ + "relative_path": file.relative_path, + "content_hash": file.content_hash, + "content": file.content, + }) + }) + .collect(); + Ok(json!({ "package_id": candidate.package_id.to_string(), "kind": candidate.package_id.kind().as_str(), "display_name": candidate.name, @@ -413,8 +365,10 @@ fn autogen_candidate_json(candidate: &getter_core::autogen::AutogenCandidate) -> "action": "create", "output_relative_path": candidate.relative_path, "content_hash": candidate.content_hash, - "content": candidate.content, - }) + "content": record_content, + "autogen_record_content": record_content, + "files": files, + })) } fn autogen_skip_json(skip: &getter_core::autogen::AutogenSkip) -> Value { @@ -428,6 +382,28 @@ fn autogen_skip_json(skip: &getter_core::autogen::AutogenSkip) -> Value { }) } +fn cleanup_preview_response( + target_alias: RepositoryId, + repo_path: PathBuf, + candidates: Vec, + diagnostics: Vec, +) -> Value { + json!({ + "operation": "cleanup.preview", + "target_repo_id": target_alias.as_str(), + "target_repo_path": repo_path, + "summary": { + "candidate_count": candidates.len(), + "skipped_count": 0, + "write_count": 0, + "delete_count": candidates.len(), + }, + "candidates": candidates, + "skipped": [], + "diagnostics": diagnostics, + }) +} + fn accepted_preview_candidates<'a>( preview: &'a Value, acceptance: &AutogenAcceptance, @@ -480,13 +456,111 @@ fn preview_relative_path(candidate: &Value) -> AutogenOperationResult { }) } +struct PreviewCandidatePayload { + record_content: String, + files: Vec, +} + +fn preview_candidate_payload( + candidate: &Value, + package_id: &PackageId, + relative_path: &Path, +) -> AutogenOperationResult { + let record_content = candidate + .get("autogen_record_content") + .or_else(|| candidate.get("content")) + .and_then(Value::as_str) + .ok_or_else(|| { + AutogenOperationError::Autogen( + "preview candidate missing autogen_record_content".to_owned(), + ) + })? + .to_owned(); + let expected_hash = candidate + .get("content_hash") + .and_then(Value::as_str) + .ok_or_else(|| { + AutogenOperationError::Autogen("preview candidate missing content_hash".to_owned()) + })?; + if content_hash(&record_content) != expected_hash { + return Err(AutogenOperationError::Autogen(format!( + "preview autogen record hash mismatch for {package_id}" + ))); + } + let files = preview_generated_files(candidate)?; + let record = parse_record_content(&record_content)?; + validate_record(&record, package_id, relative_path, &files)?; + Ok(PreviewCandidatePayload { + record_content, + files, + }) +} + +fn preview_generated_files(candidate: &Value) -> AutogenOperationResult> { + let files = candidate + .get("files") + .and_then(Value::as_array) + .ok_or_else(|| { + AutogenOperationError::Autogen("preview candidate missing files".to_owned()) + })?; + files + .iter() + .map(|file| { + let relative_path = file + .get("relative_path") + .and_then(Value::as_str) + .map(PathBuf::from) + .ok_or_else(|| { + AutogenOperationError::Autogen( + "preview generated file missing relative_path".to_owned(), + ) + })?; + validate_relative_path(&relative_path)?; + let content = file + .get("content") + .and_then(Value::as_str) + .ok_or_else(|| { + AutogenOperationError::Autogen( + "preview generated file missing content".to_owned(), + ) + })? + .to_owned(); + let expected_hash = file + .get("content_hash") + .and_then(Value::as_str) + .ok_or_else(|| { + AutogenOperationError::Autogen( + "preview generated file missing content_hash".to_owned(), + ) + })?; + let actual_hash = content_hash(&content); + if actual_hash != expected_hash { + return Err(AutogenOperationError::Autogen(format!( + "preview generated file '{}' hash mismatch", + relative_path.display() + ))); + } + Ok(GeneratedPackageFile { + relative_path, + content, + content_hash: actual_hash, + }) + }) + .collect() +} + fn ensure_generated_repository( repo_path: &Path, db: &MainDb, alias: &RepositoryId, priority: RepositoryPriority, ) -> AutogenOperationResult<()> { - ensure_repository_layout(repo_path, &generated_repo_toml(alias, priority))?; + fs::create_dir_all(repo_path).map_err(|source| { + AutogenOperationError::Autogen(format!( + "failed to create generated repository '{}': {source}", + repo_path.display() + )) + })?; db.upsert_repository( &RepositoryMetadata { id: alias.clone(), @@ -500,15 +574,6 @@ fn ensure_generated_repository( Ok(()) } -fn generated_repo_toml(alias: &RepositoryId, priority: RepositoryPriority) -> String { - format!( - "id = \"{}\"\nname = \"{}\"\npriority = {}\napi_version = \"{REPO_API_VERSION_V1}\"\n", - alias.as_str(), - generated_repository_name(alias), - priority.value() - ) -} - fn generated_repository_name(alias: &RepositoryId) -> String { if alias.as_str() == DEFAULT_AUTOGEN_REPOSITORY_ID { DEFAULT_AUTOGEN_REPOSITORY_NAME.to_owned() @@ -517,177 +582,248 @@ fn generated_repository_name(alias: &RepositoryId) -> String { } } -fn ensure_local_repository(data_dir: &Path, db: &MainDb) -> AutogenOperationResult { - let local_id = RepositoryId::new(LOCAL_REPOSITORY_ID).expect("valid id"); - if let Ok(existing) = find_repository(db, &local_id) { - let repo_path = repo_path(&existing)?; - ensure_repository_layout(&repo_path, &local_repo_toml())?; - return Ok(repo_path); - } - - let repo_path = GetterDataDirLayout::new(data_dir) - .repository_path(&RepositoryId::new(LOCAL_REPOSITORY_ID).expect("valid id")); - ensure_repository_layout(&repo_path, &local_repo_toml())?; - db.upsert_repository( - &RepositoryMetadata { - id: local_id, - name: LOCAL_REPOSITORY_NAME.to_owned(), - priority: RepositoryPriority::LOCAL, - api_version: REPO_API_VERSION_V1.to_owned(), - }, - Some(&repo_path), - None, - )?; - Ok(repo_path) -} - -fn find_repository(db: &MainDb, id: &RepositoryId) -> AutogenOperationResult { - db.repositories()? - .into_iter() - .find(|repo| &repo.id == id) - .ok_or_else(|| { - AutogenOperationError::Repository(format!("repository '{id}' is not registered")) - }) -} - -fn repo_path(repo: &StoredRepository) -> AutogenOperationResult { - repo.path.as_ref().map(PathBuf::from).ok_or_else(|| { - AutogenOperationError::Repository(format!("repository '{}' has no path", repo.id)) - }) +struct LoadedAutogenRecord { + content_hash: String, } -fn ensure_repository_layout(repo_path: &Path, repo_toml: &str) -> AutogenOperationResult<()> { - fs::create_dir_all(repo_path.join("packages")).map_err(|source| { +fn read_owned_record( + package_dir: &Path, + package_id: &PackageId, + relative_path: &Path, +) -> AutogenOperationResult { + let record_path = package_dir.join(AUTOGEN_RECORD_FILE); + if !record_path.is_file() { + return Err(AutogenOperationError::Autogen(format!( + "generated package '{}' is missing {AUTOGEN_RECORD_FILE}", + package_dir.display() + ))); + } + let bytes = fs::read(&record_path).map_err(|source| { AutogenOperationError::Autogen(format!( - "failed to create repository packages dir '{}': {source}", - repo_path.display() + "failed to read autogen record '{}': {source}", + record_path.display() )) })?; - fs::create_dir_all(repo_path.join("lib")).map_err(|source| { + let record: AutogenRecord = serde_json::from_reader(json_comments::StripComments::new( + bytes.as_slice(), + )) + .map_err(|source| { AutogenOperationError::Autogen(format!( - "failed to create repository lib dir '{}': {source}", - repo_path.display() + "failed to parse autogen record '{}': {source}", + record_path.display() )) })?; - fs::create_dir_all(repo_path.join("templates")).map_err(|source| { - AutogenOperationError::Autogen(format!( - "failed to create repository templates dir '{}': {source}", - repo_path.display() - )) - })?; - let repo_toml_path = repo_path.join("repo.toml"); - if !repo_toml_path.exists() { - fs::write(&repo_toml_path, repo_toml).map_err(|source| { + let files = read_recorded_files(package_dir, &record)?; + validate_record(&record, package_id, relative_path, &files)?; + Ok(LoadedAutogenRecord { + content_hash: content_hash_bytes(&bytes), + }) +} + +fn read_recorded_files( + package_dir: &Path, + record: &AutogenRecord, +) -> AutogenOperationResult> { + let mut files = Vec::new(); + for (relative, expected_hash) in &record.files { + let relative_path = PathBuf::from(relative); + validate_relative_path(&relative_path)?; + let path = package_dir.join(&relative_path); + if !path.is_file() { + return Err(AutogenOperationError::Autogen(format!( + "generated file '{}' listed in {AUTOGEN_RECORD_FILE} is missing", + path.display() + ))); + } + let bytes = fs::read(&path).map_err(|source| { AutogenOperationError::Autogen(format!( - "failed to write repo.toml '{}': {source}", - repo_toml_path.display() + "failed to read generated file '{}': {source}", + path.display() )) })?; + let actual_hash = content_hash_bytes(&bytes); + if &actual_hash != expected_hash { + return Err(AutogenOperationError::Autogen(format!( + "generated file '{}' hash does not match {AUTOGEN_RECORD_FILE}", + path.display() + ))); + } + let content = String::from_utf8(bytes).map_err(|source| { + AutogenOperationError::Autogen(format!( + "generated file '{}' is not UTF-8: {source}", + path.display() + )) + })?; + files.push(GeneratedPackageFile { + relative_path, + content, + content_hash: actual_hash, + }); } - Ok(()) + Ok(files) } -fn preserve_autogen_file_in_local( - data_dir: &Path, - db: &MainDb, +fn parse_record_content(content: &str) -> AutogenOperationResult { + serde_json::from_str(content).map_err(|source| { + AutogenOperationError::Autogen(format!("failed to parse preview autogen record: {source}")) + }) +} + +fn validate_record( + record: &AutogenRecord, package_id: &PackageId, - content: &str, -) -> AutogenOperationResult { - let local_repo = ensure_local_repository(data_dir, db)?; - let primary_relative = getter_core::autogen::package_relative_path(package_id); - let primary_target = safe_join(&local_repo, &primary_relative)?; - let relative_path = if primary_target.exists() { - let backup = PathBuf::from("autogen-preserved") - .join(package_id.kind().as_str()) - .join(format!( - "{}.{}.lua", - package_id.name(), - content_hash(content).replace(':', "-") - )); - safe_join(&local_repo, &backup)?; - backup - } else { - primary_relative - }; - let target = safe_join(&local_repo, &relative_path)?; - if let Some(parent) = target.parent() { - fs::create_dir_all(parent).map_err(|source| { - AutogenOperationError::Autogen(format!( - "failed to create local preservation directory '{}': {source}", - parent.display() - )) - })?; + relative_path: &Path, + files: &[GeneratedPackageFile], +) -> AutogenOperationResult<()> { + if record.version != AUTOGEN_RECORD_VERSION { + return Err(AutogenOperationError::Autogen(format!( + "unsupported autogen record version {}; expected {AUTOGEN_RECORD_VERSION}", + record.version + ))); } - fs::write(&target, content).map_err(|source| { - AutogenOperationError::Autogen(format!( - "failed to preserve modified autogen file '{}': {source}", - target.display() - )) - })?; - Ok(json!({ - "package_id": package_id.to_string(), - "repository_id": LOCAL_REPOSITORY_ID, - "relative_path": relative_path, - })) + if record.generator != INSTALLED_AUTOGEN_GENERATOR { + return Err(AutogenOperationError::Autogen(format!( + "autogen record generator '{}' does not match '{INSTALLED_AUTOGEN_GENERATOR}'", + record.generator + ))); + } + if &record.package_id != package_id { + return Err(AutogenOperationError::Autogen(format!( + "autogen record package '{}' does not match '{package_id}'", + record.package_id + ))); + } + if record.output_relative_path != relative_path { + return Err(AutogenOperationError::Autogen(format!( + "autogen record output path '{}' does not match '{}'", + record.output_relative_path.display(), + relative_path.display() + ))); + } + let file_hashes: BTreeMap = files + .iter() + .map(|file| { + ( + record_file_key(&file.relative_path), + file.content_hash.clone(), + ) + }) + .collect(); + if record.files.contains_key(AUTOGEN_RECORD_FILE) { + return Err(AutogenOperationError::Autogen(format!( + "autogen record must not list {AUTOGEN_RECORD_FILE} in files" + ))); + } + if record.files != file_hashes { + return Err(AutogenOperationError::Autogen( + "autogen record files do not match generated files".to_owned(), + )); + } + Ok(()) } -fn read_autogen_manifest(repo_path: &Path) -> AutogenOperationResult> { - let path = repo_path.join(AUTOGEN_MANIFEST_FILE); - if !path.exists() { - return Ok(None); +fn write_generated_package( + package_dir: &Path, + files: &[GeneratedPackageFile], + record_content: &str, +) -> AutogenOperationResult<()> { + for file in files { + let target = safe_join(package_dir, &file.relative_path)?; + if let Some(parent) = target.parent() { + fs::create_dir_all(parent).map_err(|source| { + AutogenOperationError::Autogen(format!( + "failed to create generated package directory '{}': {source}", + parent.display() + )) + })?; + } + fs::write(&target, &file.content).map_err(|source| { + AutogenOperationError::Autogen(format!( + "failed to write generated package file '{}': {source}", + target.display() + )) + })?; } - let bytes = fs::read(&path).map_err(|source| { + fs::write(package_dir.join(AUTOGEN_RECORD_FILE), record_content).map_err(|source| { AutogenOperationError::Autogen(format!( - "failed to read autogen manifest '{}': {source}", - path.display() + "failed to write autogen record '{}': {source}", + package_dir.join(AUTOGEN_RECORD_FILE).display() )) - })?; - serde_json::from_slice(&bytes).map(Some).map_err(|source| { - AutogenOperationError::Autogen(format!("failed to parse autogen manifest: {source}")) }) } -fn write_autogen_manifest( - repo_path: &Path, - manifest: &AutogenManifest, -) -> AutogenOperationResult<()> { - fs::create_dir_all(repo_path).map_err(|source| { +fn clear_directory_contents(path: &Path) -> AutogenOperationResult<()> { + fs::create_dir_all(path).map_err(|source| { AutogenOperationError::Autogen(format!( - "failed to create autogen repository '{}': {source}", - repo_path.display() + "failed to create generated package directory '{}': {source}", + path.display() )) })?; - let path = repo_path.join(AUTOGEN_MANIFEST_FILE); - let bytes = serde_json::to_vec_pretty(manifest).map_err(|source| { - AutogenOperationError::Autogen(format!("failed to serialize manifest: {source}")) - })?; - fs::write(&path, bytes).map_err(|source| { + for entry in fs::read_dir(path).map_err(|source| { AutogenOperationError::Autogen(format!( - "failed to write autogen manifest '{}': {source}", + "failed to read generated package directory '{}': {source}", path.display() )) - }) + })? { + let entry = entry.map_err(|source| { + AutogenOperationError::Autogen(format!( + "failed to read generated package directory '{}': {source}", + path.display() + )) + })?; + let child = entry.path(); + let file_type = entry.file_type().map_err(|source| { + AutogenOperationError::Autogen(format!( + "failed to inspect generated package entry '{}': {source}", + child.display() + )) + })?; + if file_type.is_dir() { + fs::remove_dir_all(&child).map_err(|source| { + AutogenOperationError::Autogen(format!( + "failed to delete generated package directory '{}': {source}", + child.display() + )) + })?; + } else { + fs::remove_file(&child).map_err(|source| { + AutogenOperationError::Autogen(format!( + "failed to delete generated package file '{}': {source}", + child.display() + )) + })?; + } + } + Ok(()) } -fn empty_autogen_manifest(repository_id: &RepositoryId) -> AutogenManifest { - AutogenManifest { - version: getter_core::autogen::AUTOGEN_MANIFEST_VERSION, - repository_id: repository_id.clone(), - packages: Vec::new(), - } +fn relative_package_path(root: &Path, package_dir: &Path) -> AutogenOperationResult { + package_dir + .strip_prefix(root) + .map(Path::to_path_buf) + .map_err(|_| { + AutogenOperationError::Autogen(format!( + "package directory '{}' is not under generated repository '{}'", + package_dir.display(), + root.display() + )) + }) } -fn upsert_manifest_entry(manifest: &mut AutogenManifest, entry: AutogenManifestEntry) { - manifest - .packages - .retain(|existing| existing.package_id != entry.package_id); - manifest.packages.push(entry); - manifest - .packages - .sort_by_key(|existing| existing.package_id.to_string()); +fn autogen_diagnostic(code: &str, message: String, package_id: Option) -> Value { + json!({ + "code": code, + "message": message, + "detail": package_id, + }) } fn safe_join(root: &Path, relative: &Path) -> AutogenOperationResult { + validate_relative_path(relative)?; + Ok(root.join(relative)) +} + +fn validate_relative_path(relative: &Path) -> AutogenOperationResult<()> { if relative.is_absolute() || relative.components().any(|component| { matches!( @@ -701,7 +837,7 @@ fn safe_join(root: &Path, relative: &Path) -> AutogenOperationResult { relative.display() ))); } - Ok(root.join(relative)) + Ok(()) } #[cfg(test)] @@ -734,6 +870,11 @@ mod tests { preview["target_repo_path"].as_str(), data_dir.join("repo/autogen").to_str() ); + assert_eq!( + preview["candidates"][0]["output_relative_path"], + "android/app/com.example.autogen" + ); + assert!(preview["candidates"][0]["files"].is_array()); assert!(!data_dir.join("repo/autogen").exists()); } @@ -750,15 +891,114 @@ mod tests { .unwrap(); assert_eq!(result["target_repo_id"], "autogen"); - assert!(data_dir.join("repo/autogen/repo.toml").is_file()); + assert!(data_dir.join("repo/autogen").is_dir()); + assert!(data_dir + .join("repo/autogen/android/app/com.example.autogen/metadata.jsonc") + .is_file()); assert!(data_dir - .join("repo/autogen/packages/android/com.example.autogen.lua") + .join("repo/autogen/android/app/com.example.autogen/9999.lua") .is_file()); + assert!(data_dir + .join("repo/autogen/android/app/com.example.autogen/.autogen.jsonc") + .is_file()); + assert!(!data_dir.join("repo/autogen/repo.toml").exists()); + assert!(!data_dir.join("repo/autogen/packages").exists()); let repos = db.repositories().unwrap(); assert_eq!(repos[0].id.as_str(), "autogen"); assert_eq!(repos[0].priority.value(), -1); } + #[test] + fn apply_rejects_existing_package_directory_without_ownership_record() { + let temp = tempfile::tempdir().unwrap(); + let data_dir = temp.path(); + let db = MainDb::open(data_dir.join("main.db")).unwrap(); + fs::create_dir_all(data_dir.join("repo/autogen/android/app/com.example.autogen")).unwrap(); + fs::write( + data_dir.join("repo/autogen/android/app/com.example.autogen/metadata.jsonc"), + r#"{ "type": "android:app" }"#, + ) + .unwrap(); + let plan = build_installed_autogen_plan(data_dir, &db, &test_inventory()).unwrap(); + let preview = installed_preview_json(data_dir, &plan).unwrap(); + + let error = apply_installed_preview(data_dir, &db, &preview, &AutogenAcceptance::AcceptAll) + .unwrap_err(); + + assert!( + matches!(error, AutogenOperationError::Autogen(detail) if detail.contains("missing .autogen.jsonc")) + ); + } + + #[test] + fn apply_rejects_modified_generated_files_instead_of_preserving_to_local() { + let temp = tempfile::tempdir().unwrap(); + let data_dir = temp.path(); + let db = MainDb::open(data_dir.join("main.db")).unwrap(); + let plan = build_installed_autogen_plan(data_dir, &db, &test_inventory()).unwrap(); + let preview = installed_preview_json(data_dir, &plan).unwrap(); + apply_installed_preview(data_dir, &db, &preview, &AutogenAcceptance::AcceptAll).unwrap(); + fs::write( + data_dir.join("repo/autogen/android/app/com.example.autogen/9999.lua"), + "#!/bin/upa-lua v1\n-- user edited\nreturn {}\n", + ) + .unwrap(); + + let error = apply_installed_preview(data_dir, &db, &preview, &AutogenAcceptance::AcceptAll) + .unwrap_err(); + + assert!( + matches!(error, AutogenOperationError::Autogen(detail) if detail.contains("hash does not match")) + ); + assert!(!data_dir.join("repo/local").exists()); + } + + #[test] + fn cleanup_clears_generated_package_directory_but_keeps_directory() { + let temp = tempfile::tempdir().unwrap(); + let data_dir = temp.path(); + let db = MainDb::open(data_dir.join("main.db")).unwrap(); + let plan = build_installed_autogen_plan(data_dir, &db, &test_inventory()).unwrap(); + let preview = installed_preview_json(data_dir, &plan).unwrap(); + apply_installed_preview(data_dir, &db, &preview, &AutogenAcceptance::AcceptAll).unwrap(); + let empty_inventory = InstalledInventory::new(Vec::new()); + let cleanup = cleanup_preview_json(data_dir, &db, &empty_inventory).unwrap(); + + assert_eq!( + cleanup["candidates"][0]["package_id"], + "android/app/com.example.autogen" + ); + apply_cleanup_preview(data_dir, &db, &cleanup, &AutogenAcceptance::AcceptAll).unwrap(); + + let package_dir = data_dir.join("repo/autogen/android/app/com.example.autogen"); + assert!(package_dir.is_dir()); + assert_eq!(fs::read_dir(package_dir).unwrap().count(), 0); + } + + #[test] + fn cleanup_rejects_stale_preview_when_autogen_record_changes() { + let temp = tempfile::tempdir().unwrap(); + let data_dir = temp.path(); + let db = MainDb::open(data_dir.join("main.db")).unwrap(); + let plan = build_installed_autogen_plan(data_dir, &db, &test_inventory()).unwrap(); + let preview = installed_preview_json(data_dir, &plan).unwrap(); + apply_installed_preview(data_dir, &db, &preview, &AutogenAcceptance::AcceptAll).unwrap(); + let empty_inventory = InstalledInventory::new(Vec::new()); + let cleanup = cleanup_preview_json(data_dir, &db, &empty_inventory).unwrap(); + fs::write( + data_dir.join("repo/autogen/android/app/com.example.autogen/.autogen.jsonc"), + r#"{ "version": 1, "generator": "installed-inventory", "package_id": "android/app/com.example.autogen", "output_relative_path": "android/app/com.example.autogen", "input": { "kind": "installed_android_package", "package_name": "com.example.autogen" }, "files": {} }"#, + ) + .unwrap(); + + let error = apply_cleanup_preview(data_dir, &db, &cleanup, &AutogenAcceptance::AcceptAll) + .unwrap_err(); + + assert!( + matches!(error, AutogenOperationError::Autogen(detail) if detail.contains("does not match current autogen record") || detail.contains("files do not match")) + ); + } + #[test] fn custom_generated_repository_must_already_exist() { let temp = tempfile::tempdir().unwrap(); From 367fa9bc9c3afff4eaf834041cd38f6bc25f577b Mon Sep 17 00:00:00 2001 From: Xiangzhe Date: Sat, 27 Jun 2026 01:49:07 +0800 Subject: [PATCH 31/52] feat(repository): evaluate package directories --- crates/getter-cli/src/lib.rs | 97 +++- crates/getter-cli/tests/bdd_cli.rs | 52 +- .../features/cli/autogen_installed.feature | 3 + .../cli/repository_package_eval.feature | 24 + crates/getter-core/src/diagnostics.rs | 123 ++++- crates/getter-core/src/lua.rs | 455 +++++++++++++++++- crates/getter-core/src/repository.rs | 206 ++++++-- crates/getter-operations/src/read_model.rs | 149 ++++-- crates/getter-operations/src/runtime.rs | 112 ++++- 9 files changed, 1105 insertions(+), 116 deletions(-) diff --git a/crates/getter-cli/src/lib.rs b/crates/getter-cli/src/lib.rs index 9b03e76..0cb5e6f 100644 --- a/crates/getter-cli/src/lib.rs +++ b/crates/getter-cli/src/lib.rs @@ -6,7 +6,7 @@ use getter_core::autogen::InstalledInventory; use getter_core::diagnostics::validate_repository_path; -use getter_core::lua::evaluate_package_file; +use getter_core::lua::{evaluate_package_directory_script, evaluate_package_file}; use getter_core::repository::{ default_repository_priority, GetterDataDirLayout, RepositoryLayout, RepositoryMetadata, RepositoryPackageDirectoryLayout, REPOSITORY_ROOT_METADATA_FILE, REPO_API_VERSION_V1, @@ -596,14 +596,10 @@ fn execute(invocation: CliInvocation) -> Result { CliCommand::RepoEval { id } => { let db = open_main_db(&invocation.data_dir)?; let repo = find_repository(&db, &id)?; - let path = repo_path(&repo)?; - let layout = load_repository_layout(&path)?; - let mut packages = Vec::new(); - for package_file in &layout.packages { - let package = evaluate_package_file(&layout, &package_file.path) - .map_err(|error| CliError::PackageEval(error.to_string()))?; - packages.push(package_json(package)?); - } + let packages = evaluate_repository_packages(&repo)? + .into_iter() + .map(package_json) + .collect::, _>>()?; Ok(json!({ "repository": repository_json(repo), "packages": packages, @@ -929,6 +925,13 @@ fn load_repository_layout(path: &Path) -> Result { RepositoryLayout::load(path).map_err(|source| CliError::Repository(source.to_string())) } +fn load_package_directory_layout( + path: &Path, +) -> Result { + RepositoryPackageDirectoryLayout::load(path) + .map_err(|source| CliError::Repository(source.to_string())) +} + fn load_repository_metadata( id: &RepositoryId, path: &Path, @@ -979,16 +982,12 @@ fn evaluate_package_from_repo( package_id: &PackageId, ) -> Result { let repo = find_repository(db, repo_id)?; - let path = repo_path(&repo)?; - let layout = load_repository_layout(&path)?; - let package_file = layout.package_file(package_id).ok_or_else(|| { + evaluate_package_in_repository(&repo, package_id)?.ok_or_else(|| { CliError::PackageEval(format!( "package '{}' was not found in repository '{}'", package_id, repo_id )) - })?; - evaluate_package_file(&layout, &package_file.path) - .map_err(|error| CliError::PackageEval(error.to_string())) + }) } fn evaluate_highest_priority_package( @@ -996,11 +995,8 @@ fn evaluate_highest_priority_package( package_id: &PackageId, ) -> Result { for repo in db.repositories()? { - let path = repo_path(&repo)?; - let layout = load_repository_layout(&path)?; - if let Some(package_file) = layout.package_file(package_id) { - return evaluate_package_file(&layout, &package_file.path) - .map_err(|error| CliError::PackageEval(error.to_string())); + if let Some(package) = evaluate_package_in_repository(&repo, package_id)? { + return Ok(package); } } Err(CliError::PackageEval(format!( @@ -1008,6 +1004,67 @@ fn evaluate_highest_priority_package( ))) } +fn evaluate_repository_packages( + repo: &StoredRepository, +) -> Result, CliError> { + let path = repo_path(repo)?; + if path.join("repo.toml").is_file() { + let layout = load_repository_layout(&path)?; + return layout + .packages + .iter() + .map(|package_file| { + evaluate_package_file(&layout, &package_file.path) + .map_err(|error| CliError::PackageEval(error.to_string())) + }) + .collect(); + } + + let layout = load_package_directory_layout(&path)?; + layout + .packages + .iter() + .map(|package| evaluate_package_directory(&repo.id, &layout, package)) + .collect() +} + +fn evaluate_package_in_repository( + repo: &StoredRepository, + package_id: &PackageId, +) -> Result, CliError> { + let path = repo_path(repo)?; + if path.join("repo.toml").is_file() { + let layout = load_repository_layout(&path)?; + let Some(package_file) = layout.package_file(package_id) else { + return Ok(None); + }; + return evaluate_package_file(&layout, &package_file.path) + .map(Some) + .map_err(|error| CliError::PackageEval(error.to_string())); + } + + let layout = load_package_directory_layout(&path)?; + let Some(package) = layout.package(package_id) else { + return Ok(None); + }; + evaluate_package_directory(&repo.id, &layout, package).map(Some) +} + +fn evaluate_package_directory( + repo_id: &RepositoryId, + layout: &RepositoryPackageDirectoryLayout, + package: &getter_core::repository::PackageDirectory, +) -> Result { + let metadata = layout + .package_metadata(package) + .map_err(|error| CliError::PackageEval(error.to_string()))?; + let script = layout + .unambiguous_version_script(package) + .map_err(|error| CliError::PackageEval(error.to_string()))?; + evaluate_package_directory_script(repo_id, package, &metadata, script) + .map_err(|error| CliError::PackageEval(error.to_string())) +} + fn read_installed_inventory(path: &Path) -> Result { let bytes = fs::read(path) .map_err(|source| CliError::Autogen(format!("failed to read inventory: {source}")))?; diff --git a/crates/getter-cli/tests/bdd_cli.rs b/crates/getter-cli/tests/bdd_cli.rs index e5e2787..35754d5 100644 --- a/crates/getter-cli/tests/bdd_cli.rs +++ b/crates/getter-cli/tests/bdd_cli.rs @@ -349,20 +349,45 @@ fn fixture_lua_repository(world: &mut CliWorld, repo_id: String, package_id: Str #[given(expr = "a package-directory repository {string} with package {string}")] fn package_directory_repository(world: &mut CliWorld, repo_id: String, package_id: String) { + create_package_directory_repository(world, repo_id, package_id, 1); +} + +#[given(expr = "a package-directory repository {string} with multi-version package {string}")] +fn package_directory_repository_multi_version( + world: &mut CliWorld, + repo_id: String, + package_id: String, +) { + create_package_directory_repository(world, repo_id, package_id, 2); +} + +fn create_package_directory_repository( + world: &mut CliWorld, + repo_id: String, + package_id: String, + script_count: usize, +) { let temp = world.temp.as_ref().expect("tempdir exists"); let repo_path = temp.path().join(format!("repo-{repo_id}")); let package_dir = repo_path.join(package_id.replace('/', std::path::MAIN_SEPARATOR_STR)); fs::create_dir_all(&package_dir).expect("create package dir"); fs::write( package_dir.join("metadata.jsonc"), - r#"{ "type": "android:app", "android": { "package_name": "org.fdroid.fdroid" } }"#, + r#"{ "type": "android:app", "display_name": "F-Droid", "android": { "package_name": "org.fdroid.fdroid" } }"#, ) .expect("write package metadata"); fs::write( package_dir.join("1.20.0.lua"), - "#!/bin/upa-lua v1\nreturn {}", + "#!/bin/upa-lua v1\nreturn package_version { installed = { { kind = \"android_package\", package_name = \"org.fdroid.fdroid\" } } }", ) .expect("write version Lua"); + if script_count > 1 { + fs::write( + package_dir.join("9999.lua"), + "#!/bin/upa-lua v1\nreturn package_version { installed = { { kind = \"android_package\", package_name = \"org.fdroid.fdroid\" } } }", + ) + .expect("write live version Lua"); + } world.fixture_repo_id = Some(repo_id); world.fixture_repo_path = Some(repo_path); @@ -905,6 +930,21 @@ fn command_fails_with_autogen_error(world: &mut CliWorld) { world.json = Some(json); } +#[then("the command fails with a package eval error")] +fn command_fails_with_package_eval_error(world: &mut CliWorld) { + let output = world.output.as_ref().expect("command output exists"); + assert_eq!(output.status.code(), Some(1)); + let json = parse_stdout(output); + assert_eq!(json["ok"], false); + assert_eq!(json["command"], "package eval"); + assert_eq!(json["error"]["code"], "package.eval_error"); + assert!(json["error"]["detail"] + .as_str() + .unwrap_or_default() + .contains("explicit version selection is not implemented")); + world.json = Some(json); +} + #[then("the command fails with an update check error")] fn command_fails_with_update_check_error(world: &mut CliWorld) { let output = world.output.as_ref().expect("command output exists"); @@ -1224,6 +1264,14 @@ fn output_contains_named_package(world: &mut CliWorld, package_id: String, packa assert_eq!(json["data"]["package"]["name"], package_name); } +#[then(expr = "the package eval name is {string}")] +fn package_eval_name_is(world: &mut CliWorld, package_name: String) { + let json = current_json(world); + assert_eq!(json["ok"], true); + assert_eq!(json["command"], "package eval"); + assert_eq!(json["data"]["package"]["name"], package_name); +} + #[then(expr = "the pinned package version is {string}")] fn pinned_package_version_is(world: &mut CliWorld, version: String) { let json = current_json(world); diff --git a/crates/getter-cli/tests/features/cli/autogen_installed.feature b/crates/getter-cli/tests/features/cli/autogen_installed.feature index 695dc78..6a0838d 100644 --- a/crates/getter-cli/tests/features/cli/autogen_installed.feature +++ b/crates/getter-cli/tests/features/cli/autogen_installed.feature @@ -21,6 +21,9 @@ Feature: Installed app autogen And the app list contains autogen tracked package "android/app/com.example.autogen" When I run getter repo validate for autogen Then the output reports a valid repository without network + When I run getter package eval for package "android/app/com.example.autogen" + Then the command succeeds + And the package eval name is "Example Autogen" Scenario: Higher-priority repositories suppress installed app autogen candidates Given an initialized getter data directory diff --git a/crates/getter-cli/tests/features/cli/repository_package_eval.feature b/crates/getter-cli/tests/features/cli/repository_package_eval.feature index 4edefc2..172e634 100644 --- a/crates/getter-cli/tests/features/cli/repository_package_eval.feature +++ b/crates/getter-cli/tests/features/cli/repository_package_eval.feature @@ -1,5 +1,29 @@ @getter-cli @repository Feature: Getter CLI repository and package evaluation + Scenario: User adds and evaluates a package-directory repository + Given an initialized getter data directory + And a package-directory repository "official" with package "android/app/org.fdroid.fdroid" + When I run getter repo add for that repository with priority 0 + Then the command succeeds + And the output is valid JSON + And the output contains the added repository + When I run getter repo eval for that repository + Then the command succeeds + And the output is valid JSON + And the output contains the evaluated fixture package + When I run getter package eval for that fixture package + Then the command succeeds + And the output is valid JSON + And the output contains the fixture package + + Scenario: Package-directory package eval requires an unambiguous version script + Given an initialized getter data directory + And a package-directory repository "official" with multi-version package "android/app/org.fdroid.fdroid" + When I run getter repo add for that repository with priority 0 + Then the command succeeds + When I run getter package eval for that fixture package + Then the command fails with a package eval error + Scenario: User adds and evaluates a fixture Lua repository Given an initialized getter data directory And a fixture Lua repository "official" with package "android/org.fdroid.fdroid" diff --git a/crates/getter-core/src/diagnostics.rs b/crates/getter-core/src/diagnostics.rs index 817b310..48f36a3 100644 --- a/crates/getter-core/src/diagnostics.rs +++ b/crates/getter-core/src/diagnostics.rs @@ -3,7 +3,7 @@ //! Diagnostics are getter-owned DTOs used by CLI and future app bridges. They //! describe what getter observed; Flutter should only render them. -use crate::lua::{evaluate_package_file, LuaPackageError}; +use crate::lua::{evaluate_package_directory_script, evaluate_package_file, LuaPackageError}; use crate::repository::{ package_cache_key, InvalidPackageDirectory, RepositoryLayout, RepositoryLoadError, RepositoryPackageDirectoryLayout, @@ -103,12 +103,42 @@ fn validate_package_directory_repository_path(root: &Path) -> RepositoryValidati return RepositoryValidationReport::new(0, vec![repository_load_diagnostic(error)]) } }; - let diagnostics = layout + let mut diagnostics: Vec<_> = layout .invalid_packages .iter() .map(invalid_package_directory_diagnostic) .collect(); - RepositoryValidationReport::new(layout.packages.len(), diagnostics) + let mut package_count = 0usize; + let repository_id = "validation".parse().expect("static repository id is valid"); + for package in &layout.packages { + let metadata = match layout.package_metadata(package) { + Ok(metadata) => metadata, + Err(error) => { + diagnostics.push(repository_load_diagnostic(error)); + continue; + } + }; + if package.version_scripts.is_empty() { + diagnostics.push(repository_load_diagnostic( + RepositoryLoadError::MissingPackageVersionScript { + package_id: package.id.clone(), + }, + )); + continue; + } + let diagnostic_count_before = diagnostics.len(); + for script in &package.version_scripts { + if let Err(error) = + evaluate_package_directory_script(&repository_id, package, &metadata, script) + { + diagnostics.push(lua_diagnostic(error, Some(package.id.clone()))); + } + } + if diagnostics.len() == diagnostic_count_before { + package_count += 1; + } + } + RepositoryValidationReport::new(package_count, diagnostics) } fn repository_load_diagnostic(error: RepositoryLoadError) -> PackageValidationDiagnostic { @@ -185,6 +215,38 @@ fn repository_load_diagnostic(error: RepositoryLoadError) -> PackageValidationDi path, format!("failed to hash package file: {source}"), ), + RepositoryLoadError::ReadPackageMetadata { path, source } => ( + "package.read_metadata", + path, + format!("failed to read package metadata: {source}"), + ), + RepositoryLoadError::ParsePackageMetadata { path, source } => ( + "package.parse_metadata", + path, + format!("failed to parse package metadata: {source}"), + ), + RepositoryLoadError::InvalidPackageMetadata { path, reason } => ( + "package.metadata", + path, + reason, + ), + RepositoryLoadError::MissingPackageVersionScript { package_id } => ( + "package.missing_version_script", + PathBuf::from(format!("{package_id}/metadata.jsonc")), + format!("package '{package_id}' has no enabled version scripts"), + ), + RepositoryLoadError::AmbiguousPackageVersionScript { package_id } => ( + "package.ambiguous_version_script", + PathBuf::from(format!("{package_id}/metadata.jsonc")), + format!( + "package '{package_id}' has multiple enabled version scripts; explicit version selection is not implemented" + ), + ), + RepositoryLoadError::MissingLuaApiShebang { path, expected } => ( + "package.lua_api_shebang", + path, + format!("Lua version script is missing required shebang '{expected}'"), + ), }; diagnostic(code, message, path, None, None) } @@ -245,6 +307,13 @@ fn lua_diagnostic( LuaPackageError::Domain { path, message } => { diagnostic("package.domain", message, path, None, fallback_package_id) } + LuaPackageError::Repository { path, source } => diagnostic( + "package.repository", + source.to_string(), + path, + None, + fallback_package_id, + ), } } @@ -325,7 +394,7 @@ api_version = "getter.repo.v1" .unwrap(); fs::write( package_dir.join("1.20.0.lua"), - "#!/bin/upa-lua v1\nreturn {}", + "#!/bin/upa-lua v1\nreturn package_version { installed = { { kind = \"android_package\", package_name = \"org.fdroid.fdroid\" } } }", ) .unwrap(); @@ -337,6 +406,52 @@ api_version = "getter.repo.v1" assert!(!report.network_required); } + #[test] + fn package_directory_without_shebang_is_reported_as_invalid() { + let temp = tempfile::tempdir().unwrap(); + let package_dir = temp.path().join("android/app/org.fdroid.fdroid"); + fs::create_dir_all(&package_dir).unwrap(); + fs::write( + package_dir.join("metadata.jsonc"), + r#"{ "type": "android:app", "android": { "package_name": "org.fdroid.fdroid" } }"#, + ) + .unwrap(); + fs::write(package_dir.join("1.20.0.lua"), "return package_version {}").unwrap(); + + let report = validate_repository_path(temp.path()); + + assert!(!report.valid); + assert_eq!(report.diagnostics[0].code, "package.repository"); + assert!(report.diagnostics[0].message.contains("#!/bin/upa-lua v1")); + } + + #[test] + fn package_directory_validator_checks_each_version_script_without_selecting_one() { + let temp = tempfile::tempdir().unwrap(); + let package_dir = temp.path().join("android/app/org.fdroid.fdroid"); + fs::create_dir_all(&package_dir).unwrap(); + fs::write( + package_dir.join("metadata.jsonc"), + r#"{ "type": "android:app", "android": { "package_name": "org.fdroid.fdroid" } }"#, + ) + .unwrap(); + fs::write( + package_dir.join("1.20.0.lua"), + "#!/bin/upa-lua v1\nreturn package_version { installed = { { kind = \"android_package\", package_name = \"org.fdroid.fdroid\" } } }", + ) + .unwrap(); + fs::write( + package_dir.join("9999.lua"), + "#!/bin/upa-lua v1\nreturn package_version { installed = { { kind = \"android_package\", package_name = \"org.fdroid.fdroid\" } } }", + ) + .unwrap(); + + let report = validate_repository_path(temp.path()); + + assert!(report.valid, "{report:?}"); + assert_eq!(report.package_count, 1); + } + #[test] fn package_directory_metadata_error_is_stable_package_diagnostic() { let temp = tempfile::tempdir().unwrap(); diff --git a/crates/getter-core/src/lua.rs b/crates/getter-core/src/lua.rs index d461539..4bea4d6 100644 --- a/crates/getter-core/src/lua.rs +++ b/crates/getter-core/src/lua.rs @@ -1,8 +1,12 @@ //! Minimal Lua package-file evaluation and Rust validation boundary. -use crate::repository::RepositoryLayout; +use crate::repository::{ + PackageDirectory, PackageDirectoryMetadata, PackageLuaPermission, PackageTypeMetadata, + PackageVersionScript, RepositoryLayout, RepositoryLoadError, LUA_API_SHEBANG_V1, + REPOSITORY_LUACLASS_DIR, +}; use crate::{ - InstalledTarget, PackageId, PackagePermissions, ResolvedPackage, UpdateArtifact, + InstalledTarget, PackageId, PackagePermissions, RepositoryId, ResolvedPackage, UpdateArtifact, UpdateCandidate, }; use mlua::{Lua, Table, Value}; @@ -36,6 +40,12 @@ pub enum LuaPackageError { Schema { path: PathBuf, message: String }, #[error("domain validation failed for {path}: {message}")] Domain { path: PathBuf, message: String }, + #[error("repository layout failed for {path}: {source}")] + Repository { + path: PathBuf, + #[source] + source: RepositoryLoadError, + }, } /// Evaluate and validate a Lua package file from a repository layout. @@ -59,8 +69,49 @@ pub fn evaluate_package_source( source: &str, ) -> Result { let path = path.as_ref().to_path_buf(); + let json = evaluate_package_source_to_json( + &LuaRepositoryEnvironment::Legacy(repository), + &path, + source, + )?; + validate_package_json(repository, &path, json) +} + +pub fn evaluate_package_directory_script( + repository_id: &RepositoryId, + package: &PackageDirectory, + metadata: &PackageDirectoryMetadata, + script: &PackageVersionScript, +) -> Result { + let source = fs::read_to_string(&script.path).map_err(|source| LuaPackageError::ReadFile { + path: script.path.clone(), + source, + })?; + if source.lines().next() != Some(LUA_API_SHEBANG_V1) { + return Err(LuaPackageError::Repository { + path: script.path.clone(), + source: RepositoryLoadError::MissingLuaApiShebang { + path: script.path.clone(), + expected: LUA_API_SHEBANG_V1, + }, + }); + } + let json = evaluate_package_source_to_json( + &LuaRepositoryEnvironment::PackageDirectory { package }, + &script.path, + &source, + )?; + validate_package_directory_version_json(repository_id, package, metadata, script, json) +} + +fn evaluate_package_source_to_json( + environment: &LuaRepositoryEnvironment<'_>, + path: &Path, + source: &str, +) -> Result { + let path = path.to_path_buf(); let lua = Lua::new(); - configure_package_path(&lua, repository).map_err(|source| LuaPackageError::Runtime { + configure_package_path(&lua, environment).map_err(|source| LuaPackageError::Runtime { path: path.clone(), source, })?; @@ -85,26 +136,55 @@ pub fn evaluate_package_source( Value::Table(table) => table, _ => return Err(LuaPackageError::NotATable { path }), }; - let json = lua_table_to_json(&path, "$", table)?; - validate_package_json(repository, &path, json) + lua_table_to_json(&path, "$", table) +} + +enum LuaRepositoryEnvironment<'a> { + Legacy(&'a RepositoryLayout), + PackageDirectory { package: &'a PackageDirectory }, } -fn configure_package_path(lua: &Lua, repository: &RepositoryLayout) -> mlua::Result<()> { +fn configure_package_path( + lua: &Lua, + environment: &LuaRepositoryEnvironment<'_>, +) -> mlua::Result<()> { let package: Table = lua.globals().get("package")?; - let lib_pattern = repository.lib_dir.join("?.lua"); - let nested_lib_pattern = repository.lib_dir.join("?/init.lua"); - let new_path = format!( - "{};{}", - lib_pattern.to_string_lossy(), - nested_lib_pattern.to_string_lossy() - ); - package.set("path", new_path)?; package.set("cpath", "")?; package.set("loadlib", Value::Nil)?; - install_lib_prefix_searcher(lua, &package, repository.lib_dir.clone())?; + match environment { + LuaRepositoryEnvironment::Legacy(repository) => { + let lib_pattern = repository.lib_dir.join("?.lua"); + let nested_lib_pattern = repository.lib_dir.join("?/init.lua"); + let new_path = format!( + "{};{}", + lib_pattern.to_string_lossy(), + nested_lib_pattern.to_string_lossy() + ); + package.set("path", new_path)?; + install_lib_prefix_searcher(lua, &package, repository.lib_dir.clone())?; + } + LuaRepositoryEnvironment::PackageDirectory { + package: package_dir, + } => { + package.set("path", "")?; + install_luaclass_prefix_searcher( + lua, + &package, + package_directory_repository_root(package_dir).join(REPOSITORY_LUACLASS_DIR), + )?; + } + } disable_native_module_searchers(&package) } +fn package_directory_repository_root(package: &PackageDirectory) -> PathBuf { + let mut root = package.path.clone(); + for _ in package.id.to_string().split('/') { + root.pop(); + } + root +} + fn disable_native_module_searchers(package: &Table) -> mlua::Result<()> { let searchers = package_searchers(package)?; let len = searchers.raw_len(); @@ -128,24 +208,54 @@ fn remove_unsafe_globals(lua: &Lua) -> mlua::Result<()> { } fn install_lib_prefix_searcher(lua: &Lua, package: &Table, lib_dir: PathBuf) -> mlua::Result<()> { + install_prefixed_file_searcher( + lua, + package, + "lib.", + lib_dir, + "constrained repository lib searcher only handles lib.* modules", + ) +} + +fn install_luaclass_prefix_searcher( + lua: &Lua, + package: &Table, + luaclass_dir: PathBuf, +) -> mlua::Result<()> { + install_prefixed_file_searcher( + lua, + package, + "luaclass.", + luaclass_dir, + "constrained repository luaclass searcher only handles luaclass.* modules", + ) +} + +fn install_prefixed_file_searcher( + lua: &Lua, + package: &Table, + prefix: &'static str, + module_dir: PathBuf, + wrong_prefix_message: &'static str, +) -> mlua::Result<()> { let searchers = package_searchers(package)?; let searcher = lua.create_function(move |lua, module: String| { - let Some(module) = module.strip_prefix("lib.") else { + let Some(module) = module.strip_prefix(prefix) else { return lua - .create_string("\n\tconstrained repository lib searcher only handles lib.* modules") + .create_string(format!("\n\t{wrong_prefix_message}")) .map(Value::String); }; let Some(relative_module) = module_to_relative_path(module) else { return lua .create_string(format!( - "\n\tinvalid repository lib module name 'lib.{module}'" + "\n\tinvalid repository module name '{prefix}{module}'" )) .map(Value::String); }; - let module_path = lib_dir.join(&relative_module).with_extension("lua"); - let init_path = lib_dir.join(&relative_module).join("init.lua"); + let module_path = module_dir.join(&relative_module).with_extension("lua"); + let init_path = module_dir.join(&relative_module).join("init.lua"); for candidate in [&module_path, &init_path] { if candidate.is_file() { let source = fs::read_to_string(candidate).map_err(mlua::Error::external)?; @@ -157,8 +267,8 @@ fn install_lib_prefix_searcher(lua: &Lua, package: &Table, lib_dir: PathBuf) -> } lua.create_string(format!( - "\n\tno repository lib module 'lib.{module}' in {}", - lib_dir.display() + "\n\tno repository module '{prefix}{module}' in {}", + module_dir.display() )) .map(Value::String) })?; @@ -201,6 +311,7 @@ fn module_to_relative_path(module: &str) -> Option { fn install_helpers(lua: &Lua) -> mlua::Result<()> { let package_fn = lua.create_function(|_, table: Table| Ok(table))?; lua.globals().set("package_def", package_fn.clone())?; + lua.globals().set("package_version", package_fn.clone())?; lua.globals().set("android_app", package_fn.clone())?; lua.globals().set("magisk_module", package_fn.clone())?; lua.globals().set("generic_package", package_fn)?; @@ -337,15 +448,115 @@ fn validate_package_json( } let name = required_string(path, object, "name")?.to_owned(); + package_from_version_json( + repository.metadata.id.clone(), + id, + name, + PackagePermissions::default(), + path, + object, + ) +} + +fn validate_package_directory_version_json( + repository_id: &RepositoryId, + package: &PackageDirectory, + metadata: &PackageDirectoryMetadata, + script: &PackageVersionScript, + value: JsonValue, +) -> Result { + let object = value.as_object().ok_or_else(|| LuaPackageError::Schema { + path: script.path.clone(), + message: "package value must be an object".to_owned(), + })?; + if object.contains_key("id") { + return Err(LuaPackageError::Schema { + path: script.path.clone(), + message: "field 'id' must not be declared by package version scripts; package id is derived from path".to_owned(), + }); + } + let name = object + .get("name") + .and_then(JsonValue::as_str) + .map(str::to_owned) + .unwrap_or_else(|| metadata.display_name_for(&package.id)); + let mut package = package_from_version_json( + repository_id.clone(), + package.id.clone(), + name, + metadata_permissions_for_script(metadata, &script.file_name), + &script.path, + object, + )?; + apply_metadata_installed_target(&mut package, metadata, &script.path)?; + Ok(package) +} + +fn apply_metadata_installed_target( + package: &mut ResolvedPackage, + metadata: &PackageDirectoryMetadata, + path: &Path, +) -> Result<(), LuaPackageError> { + let metadata_target = metadata_installed_target(metadata); + if package.installed.is_empty() { + package.installed.push(metadata_target); + return Ok(()); + } + if package.installed == [metadata_target] { + return Ok(()); + } + Err(LuaPackageError::Domain { + path: path.to_path_buf(), + message: "field 'installed' must match package metadata identity".to_owned(), + }) +} + +fn metadata_installed_target(metadata: &PackageDirectoryMetadata) -> InstalledTarget { + match &metadata.package { + PackageTypeMetadata::AndroidApp { android } => InstalledTarget::AndroidPackage { + package_name: android.package_name.clone(), + }, + PackageTypeMetadata::MagiskModule { magisk } => InstalledTarget::MagiskModule { + module_id: magisk.module_id.clone(), + }, + PackageTypeMetadata::Generic { generic } => InstalledTarget::Generic { + id: generic.id.clone(), + }, + } +} + +fn metadata_permissions_for_script( + metadata: &PackageDirectoryMetadata, + file_name: &str, +) -> PackagePermissions { + PackagePermissions { + free_network: metadata + .permissions_for(file_name) + .iter() + .any(|permission| *permission == PackageLuaPermission::AllowFreeNetwork), + } +} + +fn package_from_version_json( + repository: RepositoryId, + id: PackageId, + name: String, + default_permissions: PackagePermissions, + path: &Path, + object: &Map, +) -> Result { let installed = parse_installed_targets(path, object.get("installed"))?; - let permissions = parse_permissions(path, object.get("permissions"))?; + let permissions = match object.get("permissions") { + Some(value) => parse_permissions(path, Some(value))?, + None => default_permissions, + }; let source_priority = parse_string_array(path, "source_priority", object.get("source_priority"))?; let updates = parse_update_candidates(path, object.get("updates"))?; Ok(ResolvedPackage { id, - repository: repository.metadata.id.clone(), + repository, name, installed, permissions, @@ -516,7 +727,8 @@ fn parse_string_array( #[cfg(test)] mod tests { use super::*; - use crate::repository::RepositoryLayout; + use crate::repository::{RepositoryLayout, RepositoryPackageDirectoryLayout}; + use crate::RepositoryId; use std::fs; fn fixture_repo() -> (tempfile::TempDir, RepositoryLayout, PathBuf) { @@ -596,6 +808,199 @@ return package_def { ); } + #[test] + fn evaluates_package_directory_version_script_from_metadata_identity() { + let temp = tempfile::tempdir().unwrap(); + let package_dir = temp.path().join("android/app/com.example.autogen"); + fs::create_dir_all(&package_dir).unwrap(); + fs::write( + package_dir.join("metadata.jsonc"), + r#"{ + "type": "android:app", + "display_name": "Example Autogen", + "android": { "package_name": "com.example.autogen" }, + "lua": { "9999.lua": { "permission": ["allow_free_network"] } } +}"#, + ) + .unwrap(); + fs::write( + package_dir.join("9999.lua"), + r#"#!/bin/upa-lua v1 +return package_version { + installed = { + { kind = "android_package", package_name = "com.example.autogen" }, + }, + updates = { + { + version = "1.2.0", + artifacts = { + { name = "app.apk", url = "https://example.invalid/app.apk", file_name = "app.apk" }, + }, + }, + }, +} +"#, + ) + .unwrap(); + let layout = RepositoryPackageDirectoryLayout::load(temp.path()).unwrap(); + let package = &layout.packages[0]; + let metadata = layout.package_metadata(package).unwrap(); + let script = layout.unambiguous_version_script(package).unwrap(); + + let package = evaluate_package_directory_script( + &RepositoryId::new("autogen").unwrap(), + package, + &metadata, + script, + ) + .unwrap(); + + assert_eq!(package.id.to_string(), "android/app/com.example.autogen"); + assert_eq!(package.repository.as_str(), "autogen"); + assert_eq!(package.name, "Example Autogen"); + assert!(package.permissions.free_network); + assert_eq!(package.installed.len(), 1); + assert_eq!(package.updates[0].version, "1.2.0"); + } + + #[test] + fn rejects_package_directory_installed_target_mismatch() { + let temp = tempfile::tempdir().unwrap(); + let package_dir = temp.path().join("android/app/com.example.autogen"); + fs::create_dir_all(&package_dir).unwrap(); + fs::write( + package_dir.join("metadata.jsonc"), + r#"{ + "type": "android:app", + "display_name": "Example Autogen", + "android": { "package_name": "com.example.autogen" } +}"#, + ) + .unwrap(); + fs::write( + package_dir.join("9999.lua"), + r#"#!/bin/upa-lua v1 +return package_version { + installed = { + { kind = "android_package", package_name = "com.other.app" }, + }, +} +"#, + ) + .unwrap(); + let layout = RepositoryPackageDirectoryLayout::load(temp.path()).unwrap(); + let package = &layout.packages[0]; + let metadata = layout.package_metadata(package).unwrap(); + let script = layout.unambiguous_version_script(package).unwrap(); + + let err = evaluate_package_directory_script( + &RepositoryId::new("autogen").unwrap(), + package, + &metadata, + script, + ) + .unwrap_err(); + + assert!(matches!(err, LuaPackageError::Domain { .. })); + assert!(err.to_string().contains("metadata identity")); + } + + #[test] + fn rejects_package_directory_version_script_id_field() { + let temp = tempfile::tempdir().unwrap(); + let package_dir = temp.path().join("android/app/com.example.autogen"); + fs::create_dir_all(&package_dir).unwrap(); + fs::write( + package_dir.join("metadata.jsonc"), + r#"{ + "type": "android:app", + "display_name": "Example Autogen", + "android": { "package_name": "com.example.autogen" } +}"#, + ) + .unwrap(); + fs::write( + package_dir.join("9999.lua"), + r#"#!/bin/upa-lua v1 +return package_version { id = "android/app/com.example.autogen" } +"#, + ) + .unwrap(); + let layout = RepositoryPackageDirectoryLayout::load(temp.path()).unwrap(); + let package = &layout.packages[0]; + let metadata = layout.package_metadata(package).unwrap(); + let script = layout.unambiguous_version_script(package).unwrap(); + + let err = evaluate_package_directory_script( + &RepositoryId::new("autogen").unwrap(), + package, + &metadata, + script, + ) + .unwrap_err(); + + assert!(matches!(err, LuaPackageError::Schema { .. })); + assert!(err.to_string().contains("must not be declared")); + } + + #[test] + fn package_directory_can_load_luaclass_modules() { + let temp = tempfile::tempdir().unwrap(); + let package_dir = temp.path().join("android/app/com.example.autogen"); + fs::create_dir_all(&package_dir).unwrap(); + fs::create_dir_all(temp.path().join("luaclass")).unwrap(); + fs::write( + temp.path().join("luaclass/android.lua"), + r#" +return { + package_version = function(input) + return { + installed = input.installed, + updates = input.updates, + } + end +} +"#, + ) + .unwrap(); + fs::write( + package_dir.join("metadata.jsonc"), + r#"{ + "type": "android:app", + "display_name": "Example Autogen", + "android": { "package_name": "com.example.autogen" } +}"#, + ) + .unwrap(); + fs::write( + package_dir.join("9999.lua"), + r#"#!/bin/upa-lua v1 +local android = require("luaclass.android") +return android.package_version { + installed = { + { kind = "android_package", package_name = "com.example.autogen" }, + }, +} +"#, + ) + .unwrap(); + let layout = RepositoryPackageDirectoryLayout::load(temp.path()).unwrap(); + let package = &layout.packages[0]; + let metadata = layout.package_metadata(package).unwrap(); + let script = layout.unambiguous_version_script(package).unwrap(); + + let package = evaluate_package_directory_script( + &RepositoryId::new("autogen").unwrap(), + package, + &metadata, + script, + ) + .unwrap(); + + assert_eq!(package.name, "Example Autogen"); + assert_eq!(package.installed.len(), 1); + } + #[test] fn rejects_package_id_that_does_not_match_path() { let (_temp, layout, package_path) = fixture_repo(); diff --git a/crates/getter-core/src/repository.rs b/crates/getter-core/src/repository.rs index fd16ef5..f85c81a 100644 --- a/crates/getter-core/src/repository.rs +++ b/crates/getter-core/src/repository.rs @@ -2,6 +2,7 @@ use crate::{PackageId, PackageIdError, RepositoryId, RepositoryIdError, RepositoryPriority}; use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha512}; use std::collections::HashMap; use std::fs; use std::path::{Path, PathBuf}; @@ -16,7 +17,9 @@ pub const REPOSITORY_ROOT_METADATA_VERSION: u32 = 1; pub const REPOSITORY_SELF_METADATA_DIR: &str = ".metadata"; pub const REPOSITORY_LUACLASS_DIR: &str = "luaclass"; pub const PACKAGE_METADATA_FILE: &str = "metadata.jsonc"; +pub const PACKAGE_MANIFEST_FILE: &str = "Manifest"; pub const LUA_SCRIPT_EXTENSION: &str = "lua"; +pub const LUA_API_SHEBANG_V1: &str = "#!/bin/upa-lua v1"; pub const LOCAL_REPOSITORY_ALIAS: &str = "local"; pub const DEFAULT_GENERATED_REPOSITORY_ALIAS: &str = "autogen"; @@ -253,6 +256,54 @@ pub struct PackageDirectory { pub version_scripts: Vec, } +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct PackageDirectoryMetadata { + #[serde(default)] + pub display_name: Option, + #[serde(default)] + pub lua: HashMap, + #[serde(flatten)] + pub package: PackageTypeMetadata, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "type")] +pub enum PackageTypeMetadata { + #[serde(rename = "android:app")] + AndroidApp { android: AndroidPackageMetadata }, + #[serde(rename = "magisk:module")] + MagiskModule { magisk: MagiskPackageMetadata }, + #[serde(rename = "generic")] + Generic { generic: GenericPackageMetadata }, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct AndroidPackageMetadata { + pub package_name: String, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct MagiskPackageMetadata { + pub module_id: String, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct GenericPackageMetadata { + pub id: String, +} + +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct PackageLuaMetadata { + #[serde(default)] + pub permission: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum PackageLuaPermission { + AllowFreeNetwork, +} + #[derive(Debug, Clone, PartialEq, Eq)] pub struct PackageVersionScript { pub version: String, @@ -354,6 +405,29 @@ pub enum RepositoryLoadError { #[source] source: std::io::Error, }, + #[error("failed to read package metadata at {path}: {source}")] + ReadPackageMetadata { + path: PathBuf, + #[source] + source: std::io::Error, + }, + #[error("failed to parse package metadata at {path}: {source}")] + ParsePackageMetadata { + path: PathBuf, + #[source] + source: serde_json::Error, + }, + #[error("invalid package metadata at {path}: {reason}")] + InvalidPackageMetadata { path: PathBuf, reason: String }, + #[error("package '{package_id}' has no enabled version scripts")] + MissingPackageVersionScript { package_id: PackageId }, + #[error("package '{package_id}' has multiple enabled version scripts; explicit version selection is not implemented")] + AmbiguousPackageVersionScript { package_id: PackageId }, + #[error("Lua version script {path} is missing required shebang '{expected}'")] + MissingLuaApiShebang { + path: PathBuf, + expected: &'static str, + }, } impl RepositoryPackageDirectoryLayout { @@ -374,6 +448,43 @@ impl RepositoryPackageDirectoryLayout { pub fn package(&self, id: &PackageId) -> Option<&PackageDirectory> { self.packages.iter().find(|package| &package.id == id) } + + pub fn package_metadata( + &self, + package: &PackageDirectory, + ) -> Result { + load_package_metadata(&package.metadata_path) + } + + pub fn unambiguous_version_script<'a>( + &self, + package: &'a PackageDirectory, + ) -> Result<&'a PackageVersionScript, RepositoryLoadError> { + match package.version_scripts.as_slice() { + [script] => Ok(script), + [] => Err(RepositoryLoadError::MissingPackageVersionScript { + package_id: package.id.clone(), + }), + _ => Err(RepositoryLoadError::AmbiguousPackageVersionScript { + package_id: package.id.clone(), + }), + } + } +} + +impl PackageDirectoryMetadata { + pub fn display_name_for(&self, package_id: &PackageId) -> String { + self.display_name + .clone() + .unwrap_or_else(|| package_id.to_string()) + } + + pub fn permissions_for(&self, file_name: &str) -> &[PackageLuaPermission] { + self.lua + .get(file_name) + .map(|metadata| metadata.permission.as_slice()) + .unwrap_or(&[]) + } } impl RepositoryLayout { @@ -551,17 +662,37 @@ fn collect_package_boundary( } fn parse_package_metadata(metadata_path: &Path) -> Result<(), String> { - let bytes = fs::read(metadata_path) - .map_err(|source| format!("failed to read package metadata: {source}"))?; - let value = serde_json::from_reader::<_, serde_json::Value>(json_comments::StripComments::new( + load_package_metadata(metadata_path) + .map(|_| ()) + .map_err(|source| source.to_string()) +} + +pub fn load_package_metadata( + metadata_path: impl AsRef, +) -> Result { + let metadata_path = metadata_path.as_ref(); + let bytes = + fs::read(metadata_path).map_err(|source| RepositoryLoadError::ReadPackageMetadata { + path: metadata_path.to_path_buf(), + source, + })?; + let value: serde_json::Value = serde_json::from_reader(json_comments::StripComments::new( bytes.as_slice(), )) - .map_err(|source| format!("failed to parse package metadata: {source}"))?; - if value.is_object() { - Ok(()) - } else { - Err("package metadata must be a JSON object".to_owned()) + .map_err(|source| RepositoryLoadError::ParsePackageMetadata { + path: metadata_path.to_path_buf(), + source, + })?; + if !value.is_object() { + return Err(RepositoryLoadError::InvalidPackageMetadata { + path: metadata_path.to_path_buf(), + reason: "package metadata must be a JSON object".to_owned(), + }); } + serde_json::from_value(value).map_err(|source| RepositoryLoadError::ParsePackageMetadata { + path: metadata_path.to_path_buf(), + source, + }) } fn discover_version_scripts( @@ -704,22 +835,42 @@ pub fn package_cache_key( }) } +pub fn package_directory_cache_key( + repository_id: &RepositoryId, + package: &PackageDirectory, + script: &PackageVersionScript, +) -> Result { + let package_file_hash = format!( + "metadata={};script={};manifest={}", + package_file_content_hash(&package.metadata_path)?, + package_file_content_hash(&script.path)?, + optional_file_content_hash(&package.path.join(PACKAGE_MANIFEST_FILE))? + .unwrap_or_else(|| "missing".to_owned()) + ); + Ok(RepositoryPackageCacheKey { + repository_id: repository_id.clone(), + package_id: package.id.clone(), + api_version: REPO_API_VERSION_V1.to_owned(), + package_file_hash, + }) +} + +fn optional_file_content_hash(path: &Path) -> Result, RepositoryLoadError> { + if path.exists() { + package_file_content_hash(path).map(Some) + } else { + Ok(None) + } +} + pub fn package_file_content_hash(path: impl AsRef) -> Result { let path = path.as_ref(); let bytes = fs::read(path).map_err(|source| RepositoryLoadError::HashPackageFile { path: path.to_path_buf(), source, })?; - Ok(format!("{:016x}", fnv1a64(&bytes))) -} - -fn fnv1a64(bytes: &[u8]) -> u64 { - let mut hash = 0xcbf29ce484222325u64; - for byte in bytes { - hash ^= u64::from(*byte); - hash = hash.wrapping_mul(0x100000001b3); - } - hash + let hash = Sha512::digest(&bytes); + Ok(format!("sha512:{hash:x}")) } pub fn highest_priority(items: &[T], priority: F) -> Option<&T> @@ -906,7 +1057,7 @@ mod tests { fs::create_dir_all(package_dir.join("nested")).unwrap(); fs::write( package_dir.join(PACKAGE_METADATA_FILE), - r#"{ "type": "android:app" }"#, + r#"{ "type": "android:app", "android": { "package_name": "org.fdroid.fdroid" } }"#, ) .unwrap(); fs::write(package_dir.join("1.2.3.lua"), "return {}").unwrap(); @@ -944,7 +1095,7 @@ mod tests { fs::write(package_dir.join(PACKAGE_METADATA_FILE), "{not-json").unwrap(); fs::write( package_dir.join("nested/android/app/hidden/metadata.jsonc"), - r#"{ "type": "android:app" }"#, + r#"{ "type": "android:app", "android": { "package_name": "hidden" } }"#, ) .unwrap(); @@ -968,19 +1119,19 @@ mod tests { fs::create_dir_all(root.join(".metadata/android/app/hidden")).unwrap(); fs::write( root.join(".metadata/android/app/hidden/metadata.jsonc"), - r#"{ "type": "android:app" }"#, + r#"{ "type": "android:app", "android": { "package_name": "hidden" } }"#, ) .unwrap(); fs::create_dir_all(root.join("luaclass/android/app/hidden")).unwrap(); fs::write( root.join("luaclass/android/app/hidden/metadata.jsonc"), - r#"{ "type": "android:app" }"#, + r#"{ "type": "android:app", "android": { "package_name": "hidden" } }"#, ) .unwrap(); fs::create_dir_all(root.join("android/app/visible")).unwrap(); fs::write( root.join("android/app/visible/metadata.jsonc"), - r#"{ "type": "android:app" }"#, + r#"{ "type": "android:app", "android": { "package_name": "visible" } }"#, ) .unwrap(); @@ -1014,10 +1165,9 @@ mod tests { assert!(layout.packages.is_empty()); assert_eq!(layout.invalid_packages.len(), 1); - assert_eq!( - layout.invalid_packages[0].reason, - "package metadata must be a JSON object" - ); + assert!(layout.invalid_packages[0] + .reason + .contains("package metadata must be a JSON object")); } #[test] @@ -1027,7 +1177,7 @@ mod tests { fs::create_dir_all(root.join("invalidkind/app/example")).unwrap(); fs::write( root.join("invalidkind/app/example/metadata.jsonc"), - r#"{ "type": "android:app" }"#, + r#"{ "type": "android:app", "android": { "package_name": "example" } }"#, ) .unwrap(); diff --git a/crates/getter-operations/src/read_model.rs b/crates/getter-operations/src/read_model.rs index 898864b..79f5cad 100644 --- a/crates/getter-operations/src/read_model.rs +++ b/crates/getter-operations/src/read_model.rs @@ -6,9 +6,11 @@ //! to parse and render. #[cfg(feature = "lua")] -use getter_core::lua::evaluate_package_file; +use getter_core::lua::{evaluate_package_directory_script, evaluate_package_file}; #[cfg(feature = "lua")] -use getter_core::repository::{RepositoryLayout, RepositoryLoadError}; +use getter_core::repository::{ + RepositoryLayout, RepositoryLoadError, RepositoryPackageDirectoryLayout, +}; #[cfg(feature = "lua")] use getter_core::{PackageId, RepositoryId}; use getter_storage::{MainDb, StorageError, StoredRepository, StoredTrackedPackage}; @@ -140,16 +142,12 @@ fn evaluate_package_from_repo( package_id: &PackageId, ) -> Result { let repo = find_repository(db, repo_id)?; - let path = repo_path(&repo)?; - let layout = RepositoryLayout::load(&path)?; - let package_file = layout.package_file(package_id).ok_or_else(|| { + evaluate_package_in_repository(&repo, package_id)?.ok_or_else(|| { ReadModelOperationError::PackageEval(format!( "package '{}' was not found in repository '{}'", package_id, repo_id )) - })?; - evaluate_package_file(&layout, &package_file.path) - .map_err(|source| ReadModelOperationError::PackageEval(source.to_string())) + }) } #[cfg(feature = "lua")] @@ -158,11 +156,8 @@ fn evaluate_highest_priority_package( package_id: &PackageId, ) -> Result { for repo in db.repositories()? { - let path = repo_path(&repo)?; - let layout = RepositoryLayout::load(&path)?; - if let Some(package_file) = layout.package_file(package_id) { - return evaluate_package_file(&layout, &package_file.path) - .map_err(|source| ReadModelOperationError::PackageEval(source.to_string())); + if let Some(package) = evaluate_package_in_repository(&repo, package_id)? { + return Ok(package); } } Err(ReadModelOperationError::PackageEval(format!( @@ -170,6 +165,33 @@ fn evaluate_highest_priority_package( ))) } +#[cfg(feature = "lua")] +fn evaluate_package_in_repository( + repo: &StoredRepository, + package_id: &PackageId, +) -> Result, ReadModelOperationError> { + let path = repo_path(repo)?; + if path.join("repo.toml").is_file() { + let layout = RepositoryLayout::load(&path)?; + let Some(package_file) = layout.package_file(package_id) else { + return Ok(None); + }; + return evaluate_package_file(&layout, &package_file.path) + .map(Some) + .map_err(|source| ReadModelOperationError::PackageEval(source.to_string())); + } + + let layout = RepositoryPackageDirectoryLayout::load(&path)?; + let Some(package) = layout.package(package_id) else { + return Ok(None); + }; + let metadata = layout.package_metadata(package)?; + let script = layout.unambiguous_version_script(package)?; + evaluate_package_directory_script(&repo.id, package, &metadata, script) + .map(Some) + .map_err(|source| ReadModelOperationError::PackageEval(source.to_string())) +} + #[cfg(feature = "lua")] fn find_repository( db: &MainDb, @@ -274,6 +296,68 @@ mod tests { fn package_eval_reads_registered_lua_repository() { let temp = tempdir().unwrap(); let repo_path = temp.path().join("repo"); + write_legacy_lua_repo(&repo_path); + + let db = MainDb::open(temp.path().join(MAIN_DB_FILE)).unwrap(); + db.upsert_repository( + &RepositoryMetadata { + id: "official".parse().unwrap(), + name: "Official".to_owned(), + priority: RepositoryPriority::new(10), + api_version: REPO_API_VERSION_V1.to_owned(), + }, + Some(&repo_path), + None, + ) + .unwrap(); + + let result = package_eval_json( + temp.path(), + r#"{"package_id":"android/org.fdroid.fdroid","repository_id":"official"}"#, + ) + .unwrap(); + assert_eq!(result["package"]["id"], "android/org.fdroid.fdroid"); + assert_eq!(result["package"]["name"], "F-Droid"); + assert_eq!(result["package"]["permissions"]["free_network"], true); + } + + #[cfg(feature = "lua")] + #[test] + fn package_eval_reads_registered_package_directory_repository() { + let temp = tempdir().unwrap(); + let repo_path = temp.path().join("repo"); + write_package_directory_repo(&repo_path); + + let db = MainDb::open(temp.path().join(MAIN_DB_FILE)).unwrap(); + db.upsert_repository( + &RepositoryMetadata { + id: "autogen".parse().unwrap(), + name: "Autogen".to_owned(), + priority: RepositoryPriority::new(-1), + api_version: REPO_API_VERSION_V1.to_owned(), + }, + Some(&repo_path), + None, + ) + .unwrap(); + + let result = package_eval_json( + temp.path(), + r#"{"package_id":"android/app/com.example.autogen","repository_id":"autogen"}"#, + ) + .unwrap(); + + assert_eq!(result["package"]["id"], "android/app/com.example.autogen"); + assert_eq!(result["package"]["repository"], "autogen"); + assert_eq!(result["package"]["name"], "Example Autogen"); + assert_eq!( + result["package"]["installed"][0]["package_name"], + "com.example.autogen" + ); + } + + #[cfg(feature = "lua")] + fn write_legacy_lua_repo(repo_path: &Path) { fs::create_dir_all(repo_path.join("packages/android")).unwrap(); fs::create_dir_all(repo_path.join("lib")).unwrap(); fs::create_dir_all(repo_path.join("templates")).unwrap(); @@ -299,27 +383,32 @@ api_version = "getter.repo.v1" "#, ) .unwrap(); + } - let db = MainDb::open(temp.path().join(MAIN_DB_FILE)).unwrap(); - db.upsert_repository( - &RepositoryMetadata { - id: "official".parse().unwrap(), - name: "Official".to_owned(), - priority: RepositoryPriority::new(10), - api_version: REPO_API_VERSION_V1.to_owned(), - }, - Some(&repo_path), - None, + #[cfg(feature = "lua")] + fn write_package_directory_repo(repo_path: &Path) { + let package_dir = repo_path.join("android/app/com.example.autogen"); + fs::create_dir_all(&package_dir).unwrap(); + fs::write( + package_dir.join("metadata.jsonc"), + r#"{ + "type": "android:app", + "display_name": "Example Autogen", + "android": { "package_name": "com.example.autogen" } +}"#, ) .unwrap(); - - let result = package_eval_json( - temp.path(), - r#"{"package_id":"android/org.fdroid.fdroid","repository_id":"official"}"#, + fs::write(package_dir.join("Manifest"), "").unwrap(); + fs::write( + package_dir.join("9999.lua"), + r#"#!/bin/upa-lua v1 +return package_version { + installed = { + { kind = "android_package", package_name = "com.example.autogen" }, + }, +} +"#, ) .unwrap(); - assert_eq!(result["package"]["id"], "android/org.fdroid.fdroid"); - assert_eq!(result["package"]["name"], "F-Droid"); - assert_eq!(result["package"]["permissions"]["free_network"], true); } } diff --git a/crates/getter-operations/src/runtime.rs b/crates/getter-operations/src/runtime.rs index 1392322..d9e53f1 100644 --- a/crates/getter-operations/src/runtime.rs +++ b/crates/getter-operations/src/runtime.rs @@ -8,8 +8,11 @@ #[cfg(feature = "lua")] use getter_core::{ - lua::evaluate_package_file, - repository::{package_cache_key, RepositoryLayout, RepositoryLoadError}, + lua::{evaluate_package_directory_script, evaluate_package_file}, + repository::{ + package_cache_key, package_directory_cache_key, RepositoryLayout, RepositoryLoadError, + RepositoryPackageDirectoryLayout, + }, }; use getter_core::{ runtime::{ @@ -383,13 +386,31 @@ fn evaluate_registered_package( continue; }; let root = PathBuf::from(root); - let layout = RepositoryLayout::load(&root)?; - let Some(package_file) = layout.package_file(&request.package_id) else { + if root.join("repo.toml").is_file() { + let layout = RepositoryLayout::load(&root)?; + let Some(package_file) = layout.package_file(&request.package_id) else { + continue; + }; + let package = evaluate_package_file(&layout, &package_file.path) + .map_err(|source| RuntimeOperationError::PackageEval(source.to_string()))?; + let cache_key = package_cache_key(&layout, package_file)?; + let dependency_digest = format!( + "repo:{}:package:{}:hash:{}", + cache_key.repository_id, cache_key.package_id, cache_key.package_file_hash + ); + return Ok((package, dependency_digest)); + } + + let layout = RepositoryPackageDirectoryLayout::load(&root)?; + let Some(package_directory) = layout.package(&request.package_id) else { continue; }; - let package = evaluate_package_file(&layout, &package_file.path) - .map_err(|source| RuntimeOperationError::PackageEval(source.to_string()))?; - let cache_key = package_cache_key(&layout, package_file)?; + let metadata = layout.package_metadata(package_directory)?; + let script = layout.unambiguous_version_script(package_directory)?; + let package = + evaluate_package_directory_script(&repository.id, package_directory, &metadata, script) + .map_err(|source| RuntimeOperationError::PackageEval(source.to_string()))?; + let cache_key = package_directory_cache_key(&repository.id, package_directory, script)?; let dependency_digest = format!( "repo:{}:package:{}:hash:{}", cache_key.repository_id, cache_key.package_id, cache_key.package_file_hash @@ -484,6 +505,47 @@ mod tests { assert_eq!(submitted["package_id"], "android/org.fdroid.fdroid"); } + #[cfg(feature = "lua")] + #[test] + fn registered_package_update_check_issues_action_from_package_directory_static_updates() { + let temp = tempfile::tempdir().unwrap(); + let repo_root = temp.path().join("repo"); + write_package_directory_static_update_repo(&repo_root); + let db = MainDb::open_in_memory().unwrap(); + db.upsert_repository( + &RepositoryMetadata { + id: "autogen".parse().unwrap(), + name: "Autogen".to_owned(), + priority: RepositoryPriority::new(-1), + api_version: REPO_API_VERSION_V1.to_owned(), + }, + Some(&repo_root), + None, + ) + .unwrap(); + let mut runtime = GetterRuntime::new(); + + let issued = issue_action_from_registered_package_json( + &mut runtime, + &db, + &json!({ + "package_id": "android/app/com.example.autogen", + "installed_version": "1.0.0" + }) + .to_string(), + ) + .unwrap(); + + assert_eq!(issued["package"]["repository"], "autogen"); + assert_eq!(issued["package"]["name"], "Example Autogen"); + assert_eq!(issued["update"]["status"], "update_available"); + let action_id = issued["action"]["action_id"].as_str().unwrap(); + let submitted = + submit_action_json(&mut runtime, &json!({ "action_id": action_id }).to_string()) + .unwrap(); + assert_eq!(submitted["package_id"], "android/app/com.example.autogen"); + } + #[cfg(feature = "lua")] #[test] fn registered_package_update_check_without_update_does_not_issue_action() { @@ -682,6 +744,42 @@ return package_def { .unwrap(); } + #[cfg(feature = "lua")] + fn write_package_directory_static_update_repo(root: &std::path::Path) { + let package_dir = root.join("android/app/com.example.autogen"); + fs::create_dir_all(&package_dir).unwrap(); + fs::write( + package_dir.join("metadata.jsonc"), + r#"{ + "type": "android:app", + "display_name": "Example Autogen", + "android": { "package_name": "com.example.autogen" } +}"#, + ) + .unwrap(); + fs::write(package_dir.join("Manifest"), "").unwrap(); + fs::write( + package_dir.join("9999.lua"), + r#"#!/bin/upa-lua v1 +return package_version { + updates = { + { + version = "1.2.0", + artifacts = { + { + name = "app.apk", + url = "https://example.invalid/app.apk", + file_name = "app.apk", + }, + }, + }, + }, +} +"#, + ) + .unwrap(); + } + fn update_fixture( package_id: &str, installed_version: Option<&str>, From 15788b42a6c786ce86de80af744e72662e1e3e9d Mon Sep 17 00:00:00 2001 From: Xiangzhe Date: Sat, 27 Jun 2026 02:08:03 +0800 Subject: [PATCH 32/52] fix(core): satisfy clippy for Lua errors --- crates/getter-core/src/lua.rs | 29 ++++++++++++++--------------- 1 file changed, 14 insertions(+), 15 deletions(-) diff --git a/crates/getter-core/src/lua.rs b/crates/getter-core/src/lua.rs index 4bea4d6..53cd121 100644 --- a/crates/getter-core/src/lua.rs +++ b/crates/getter-core/src/lua.rs @@ -26,7 +26,7 @@ pub enum LuaPackageError { Runtime { path: PathBuf, #[source] - source: mlua::Error, + source: Box, }, #[error("Lua package {path} did not return a table")] NotATable { path: PathBuf }, @@ -44,7 +44,7 @@ pub enum LuaPackageError { Repository { path: PathBuf, #[source] - source: RepositoryLoadError, + source: Box, }, } @@ -90,10 +90,10 @@ pub fn evaluate_package_directory_script( if source.lines().next() != Some(LUA_API_SHEBANG_V1) { return Err(LuaPackageError::Repository { path: script.path.clone(), - source: RepositoryLoadError::MissingLuaApiShebang { + source: Box::new(RepositoryLoadError::MissingLuaApiShebang { path: script.path.clone(), expected: LUA_API_SHEBANG_V1, - }, + }), }); } let json = evaluate_package_source_to_json( @@ -113,15 +113,15 @@ fn evaluate_package_source_to_json( let lua = Lua::new(); configure_package_path(&lua, environment).map_err(|source| LuaPackageError::Runtime { path: path.clone(), - source, + source: Box::new(source), })?; remove_unsafe_globals(&lua).map_err(|source| LuaPackageError::Runtime { path: path.clone(), - source, + source: Box::new(source), })?; install_helpers(&lua).map_err(|source| LuaPackageError::Runtime { path: path.clone(), - source, + source: Box::new(source), })?; let value = lua @@ -130,7 +130,7 @@ fn evaluate_package_source_to_json( .eval::() .map_err(|source| LuaPackageError::Runtime { path: path.clone(), - source, + source: Box::new(source), })?; let table = match value { Value::Table(table) => table, @@ -325,13 +325,13 @@ fn lua_table_to_json( ) -> Result { if is_array_table(&table).map_err(|source| LuaPackageError::Runtime { path: path.to_path_buf(), - source, + source: Box::new(source), })? { let mut array = Vec::new(); for pair in table.sequence_values::() { let value = pair.map_err(|source| LuaPackageError::Runtime { path: path.to_path_buf(), - source, + source: Box::new(source), })?; array.push(lua_value_to_json(path, &format!("{location}[]"), value)?); } @@ -341,14 +341,14 @@ fn lua_table_to_json( for pair in table.pairs::() { let (key, value) = pair.map_err(|source| LuaPackageError::Runtime { path: path.to_path_buf(), - source, + source: Box::new(source), })?; let key = match key { Value::String(value) => value .to_str() .map_err(|source| LuaPackageError::Runtime { path: path.to_path_buf(), - source, + source: Box::new(source), })? .to_owned(), Value::Integer(value) => value.to_string(), @@ -388,7 +388,7 @@ fn lua_value_to_json( .to_str() .map_err(|source| LuaPackageError::Runtime { path: path.to_path_buf(), - source, + source: Box::new(source), })? .to_owned(), )), @@ -532,8 +532,7 @@ fn metadata_permissions_for_script( PackagePermissions { free_network: metadata .permissions_for(file_name) - .iter() - .any(|permission| *permission == PackageLuaPermission::AllowFreeNetwork), + .contains(&PackageLuaPermission::AllowFreeNetwork), } } From 091bb9fde26187fa01da008a9ce1faf3b2411fcc Mon Sep 17 00:00:00 2001 From: Xiangzhe Date: Sat, 27 Jun 2026 02:17:58 +0800 Subject: [PATCH 33/52] feat(lua): expose package-local file reads --- crates/getter-core/src/lua.rs | 173 +++++++++++++++++++++++++++++++++- 1 file changed, 171 insertions(+), 2 deletions(-) diff --git a/crates/getter-core/src/lua.rs b/crates/getter-core/src/lua.rs index 53cd121..8d6f74e 100644 --- a/crates/getter-core/src/lua.rs +++ b/crates/getter-core/src/lua.rs @@ -119,7 +119,7 @@ fn evaluate_package_source_to_json( path: path.clone(), source: Box::new(source), })?; - install_helpers(&lua).map_err(|source| LuaPackageError::Runtime { + install_helpers(&lua, environment).map_err(|source| LuaPackageError::Runtime { path: path.clone(), source: Box::new(source), })?; @@ -308,16 +308,69 @@ fn module_to_relative_path(module: &str) -> Option { } } -fn install_helpers(lua: &Lua) -> mlua::Result<()> { +fn install_helpers(lua: &Lua, environment: &LuaRepositoryEnvironment<'_>) -> mlua::Result<()> { let package_fn = lua.create_function(|_, table: Table| Ok(table))?; lua.globals().set("package_def", package_fn.clone())?; lua.globals().set("package_version", package_fn.clone())?; lua.globals().set("android_app", package_fn.clone())?; lua.globals().set("magisk_module", package_fn.clone())?; lua.globals().set("generic_package", package_fn)?; + if let LuaRepositoryEnvironment::PackageDirectory { package } = environment { + install_package_file_helpers(lua, package)?; + } Ok(()) } +fn install_package_file_helpers(lua: &Lua, package: &PackageDirectory) -> mlua::Result<()> { + let files_root = package.path.join("files"); + let read_package_file = lua.create_function(move |lua, requested_path: String| { + let relative_path = package_file_relative_path(&requested_path)?; + let path = files_root.join(relative_path); + let metadata = fs::metadata(&path).map_err(mlua::Error::external)?; + if !metadata.is_file() { + return Err(mlua::Error::external(format!( + "read_package_file target {} is not a file", + path.display() + ))); + } + let bytes = fs::read(&path).map_err(mlua::Error::external)?; + lua.create_string(&bytes) + })?; + + let globals = lua.globals(); + let getter_builtin = match globals.get::("getter_builtin")? { + Value::Table(table) => table, + Value::Nil => { + let table = lua.create_table()?; + globals.set("getter_builtin", table.clone())?; + table + } + _ => return Err(mlua::Error::external("getter_builtin must be a table")), + }; + getter_builtin.set("read_package_file", read_package_file.clone())?; + globals.set("read_package_file", read_package_file) +} + +fn package_file_relative_path(path: &str) -> mlua::Result { + let mut relative = PathBuf::new(); + for component in Path::new(path).components() { + match component { + std::path::Component::Normal(part) => relative.push(part), + _ => { + return Err(mlua::Error::external( + "read_package_file path must be relative to files/ and must not contain special path components", + )) + } + } + } + if relative.as_os_str().is_empty() { + return Err(mlua::Error::external( + "read_package_file path must not be empty", + )); + } + Ok(relative) +} + fn lua_table_to_json( path: &Path, location: &str, @@ -1000,6 +1053,122 @@ return android.package_version { assert_eq!(package.installed.len(), 1); } + #[test] + fn package_directory_can_read_package_local_files() { + let temp = tempfile::tempdir().unwrap(); + let package_dir = temp.path().join("android/app/com.example.autogen"); + fs::create_dir_all(package_dir.join("files/nested")).unwrap(); + fs::write(package_dir.join("files/nested/data.txt"), b"hello\xff").unwrap(); + fs::write( + package_dir.join("metadata.jsonc"), + r#"{ + "type": "android:app", + "display_name": "Example Autogen", + "android": { "package_name": "com.example.autogen" } +}"#, + ) + .unwrap(); + fs::write( + package_dir.join("9999.lua"), + r#"#!/bin/upa-lua v1 +local body = read_package_file("nested/data.txt") +return package_version { name = "bytes:" .. tostring(#body) } +"#, + ) + .unwrap(); + let layout = RepositoryPackageDirectoryLayout::load(temp.path()).unwrap(); + let package = &layout.packages[0]; + let metadata = layout.package_metadata(package).unwrap(); + let script = layout.unambiguous_version_script(package).unwrap(); + + let package = evaluate_package_directory_script( + &RepositoryId::new("autogen").unwrap(), + package, + &metadata, + script, + ) + .unwrap(); + + assert_eq!(package.name, "bytes:6"); + } + + #[test] + fn read_package_file_rejects_paths_outside_package_files() { + let temp = tempfile::tempdir().unwrap(); + let package_dir = temp.path().join("android/app/com.example.autogen"); + fs::create_dir_all(package_dir.join("files")).unwrap(); + fs::write(package_dir.join("outside.txt"), "outside").unwrap(); + fs::write( + package_dir.join("metadata.jsonc"), + r#"{ + "type": "android:app", + "display_name": "Example Autogen", + "android": { "package_name": "com.example.autogen" } +}"#, + ) + .unwrap(); + fs::write( + package_dir.join("9999.lua"), + r#"#!/bin/upa-lua v1 +local ok = pcall(read_package_file, "../outside.txt") +return package_version { name = ok and "leaked" or "blocked" } +"#, + ) + .unwrap(); + let layout = RepositoryPackageDirectoryLayout::load(temp.path()).unwrap(); + let package = &layout.packages[0]; + let metadata = layout.package_metadata(package).unwrap(); + let script = layout.unambiguous_version_script(package).unwrap(); + + let package = evaluate_package_directory_script( + &RepositoryId::new("autogen").unwrap(), + package, + &metadata, + script, + ) + .unwrap(); + + assert_eq!(package.name, "blocked"); + } + + #[test] + fn getter_builtin_exposes_package_file_reader_to_package_directory_lua() { + let temp = tempfile::tempdir().unwrap(); + let package_dir = temp.path().join("android/app/com.example.autogen"); + fs::create_dir_all(package_dir.join("files")).unwrap(); + fs::write(package_dir.join("files/name.txt"), "from builtin").unwrap(); + fs::write( + package_dir.join("metadata.jsonc"), + r#"{ + "type": "android:app", + "display_name": "Example Autogen", + "android": { "package_name": "com.example.autogen" } +}"#, + ) + .unwrap(); + fs::write( + package_dir.join("9999.lua"), + r#"#!/bin/upa-lua v1 +return package_version { name = getter_builtin.read_package_file("name.txt") } +"#, + ) + .unwrap(); + let layout = RepositoryPackageDirectoryLayout::load(temp.path()).unwrap(); + let package = &layout.packages[0]; + let metadata = layout.package_metadata(package).unwrap(); + let script = layout.unambiguous_version_script(package).unwrap(); + + let package = evaluate_package_directory_script( + &RepositoryId::new("autogen").unwrap(), + package, + &metadata, + script, + ) + .unwrap(); + + assert_eq!(package.name, "from builtin"); + } + #[test] fn rejects_package_id_that_does_not_match_path() { let (_temp, layout, package_path) = fixture_repo(); From 139d1ff02b80ce64f753f1b24e8fbe7e4111a96d Mon Sep 17 00:00:00 2001 From: Xiangzhe Date: Sat, 27 Jun 2026 02:24:59 +0800 Subject: [PATCH 34/52] fix(repository): hash package-local file inputs --- crates/getter-core/src/diagnostics.rs | 3 + crates/getter-core/src/repository.rs | 143 ++++++++++++++++++++++++-- 2 files changed, 139 insertions(+), 7 deletions(-) diff --git a/crates/getter-core/src/diagnostics.rs b/crates/getter-core/src/diagnostics.rs index 48f36a3..a78ff21 100644 --- a/crates/getter-core/src/diagnostics.rs +++ b/crates/getter-core/src/diagnostics.rs @@ -215,6 +215,9 @@ fn repository_load_diagnostic(error: RepositoryLoadError) -> PackageValidationDi path, format!("failed to hash package file: {source}"), ), + RepositoryLoadError::InvalidPackageLocalFile { path, reason } => { + ("package.local_file", path, reason) + } RepositoryLoadError::ReadPackageMetadata { path, source } => ( "package.read_metadata", path, diff --git a/crates/getter-core/src/repository.rs b/crates/getter-core/src/repository.rs index f85c81a..f5f0e99 100644 --- a/crates/getter-core/src/repository.rs +++ b/crates/getter-core/src/repository.rs @@ -18,6 +18,7 @@ pub const REPOSITORY_SELF_METADATA_DIR: &str = ".metadata"; pub const REPOSITORY_LUACLASS_DIR: &str = "luaclass"; pub const PACKAGE_METADATA_FILE: &str = "metadata.jsonc"; pub const PACKAGE_MANIFEST_FILE: &str = "Manifest"; +pub const PACKAGE_LOCAL_FILES_DIR: &str = "files"; pub const LUA_SCRIPT_EXTENSION: &str = "lua"; pub const LUA_API_SHEBANG_V1: &str = "#!/bin/upa-lua v1"; pub const LOCAL_REPOSITORY_ALIAS: &str = "local"; @@ -405,6 +406,8 @@ pub enum RepositoryLoadError { #[source] source: std::io::Error, }, + #[error("invalid package-local file {path}: {reason}")] + InvalidPackageLocalFile { path: PathBuf, reason: String }, #[error("failed to read package metadata at {path}: {source}")] ReadPackageMetadata { path: PathBuf, @@ -840,13 +843,7 @@ pub fn package_directory_cache_key( package: &PackageDirectory, script: &PackageVersionScript, ) -> Result { - let package_file_hash = format!( - "metadata={};script={};manifest={}", - package_file_content_hash(&package.metadata_path)?, - package_file_content_hash(&script.path)?, - optional_file_content_hash(&package.path.join(PACKAGE_MANIFEST_FILE))? - .unwrap_or_else(|| "missing".to_owned()) - ); + let package_file_hash = package_directory_dependency_hash(package, script)?; Ok(RepositoryPackageCacheKey { repository_id: repository_id.clone(), package_id: package.id.clone(), @@ -855,6 +852,99 @@ pub fn package_directory_cache_key( }) } +fn package_directory_dependency_hash( + package: &PackageDirectory, + script: &PackageVersionScript, +) -> Result { + let mut entries = vec![ + format!( + "metadata.jsonc={}", + package_file_content_hash(&package.metadata_path)? + ), + format!( + "{}={}", + script.file_name, + package_file_content_hash(&script.path)? + ), + format!( + "Manifest={}", + optional_file_content_hash(&package.path.join(PACKAGE_MANIFEST_FILE))? + .unwrap_or_else(|| "missing".to_owned()) + ), + ]; + for file in package_local_files(&package.path)? { + let hash = package_file_content_hash(&file.path)?; + entries.push(format!("files/{}={hash}", file.relative_path)); + } + Ok(content_hash(entries.join("\n"))) +} + +struct PackageLocalFile { + relative_path: String, + path: PathBuf, +} + +fn package_local_files(package_dir: &Path) -> Result, RepositoryLoadError> { + let files_root = package_dir.join(PACKAGE_LOCAL_FILES_DIR); + if !files_root.exists() { + return Ok(Vec::new()); + } + if !files_root.is_dir() { + return Err(RepositoryLoadError::InvalidPackageLocalFile { + path: files_root, + reason: "package-local files entry must be a directory".to_owned(), + }); + } + let mut files = Vec::new(); + collect_package_local_files(&files_root, &files_root, &mut files)?; + files.sort_by(|left, right| left.relative_path.cmp(&right.relative_path)); + Ok(files) +} + +fn collect_package_local_files( + files_root: &Path, + current: &Path, + out: &mut Vec, +) -> Result<(), RepositoryLoadError> { + for entry in read_dir_entries(current)? { + let path = entry.path(); + let file_type = entry_file_type(&entry, current)?; + if file_type.is_dir() { + collect_package_local_files(files_root, &path, out)?; + } else if file_type.is_file() { + let relative = path.strip_prefix(files_root).map_err(|_| { + RepositoryLoadError::InvalidPackageLocalFile { + path: path.clone(), + reason: format!("path is not under {}", files_root.display()), + } + })?; + out.push(PackageLocalFile { + relative_path: logical_relative_path(&path, relative)?, + path, + }); + } + } + Ok(()) +} + +fn logical_relative_path(path: &Path, relative: &Path) -> Result { + let value = relative + .to_string_lossy() + .replace(std::path::MAIN_SEPARATOR, "/"); + if value.is_empty() { + return Err(RepositoryLoadError::InvalidPackageLocalFile { + path: path.to_path_buf(), + reason: "missing relative file path".to_owned(), + }); + } + Ok(value) +} + +fn content_hash(content: impl AsRef<[u8]>) -> String { + let hash = Sha512::digest(content.as_ref()); + format!("sha512:{hash:x}") +} + fn optional_file_content_hash(path: &Path) -> Result, RepositoryLoadError> { if path.exists() { package_file_content_hash(path).map(Some) @@ -1261,6 +1351,45 @@ api_version = "getter.repo.v1" assert_ne!(first.package_file_hash, second.package_file_hash); } + #[test] + fn package_directory_cache_key_changes_when_local_files_change() { + let temp = tempfile::tempdir().unwrap(); + let root = temp.path(); + let package_dir = root.join("android/app/example"); + fs::create_dir_all(package_dir.join(PACKAGE_LOCAL_FILES_DIR)).unwrap(); + fs::write( + package_dir.join(PACKAGE_METADATA_FILE), + r#"{ "type": "android:app", "android": { "package_name": "example" } }"#, + ) + .unwrap(); + fs::write(package_dir.join("9999.lua"), "return {}").unwrap(); + fs::write( + package_dir.join(PACKAGE_LOCAL_FILES_DIR).join("data.txt"), + "one", + ) + .unwrap(); + let layout = RepositoryPackageDirectoryLayout::load(root).unwrap(); + let package = &layout.packages[0]; + let script = layout.unambiguous_version_script(package).unwrap(); + let first = + package_directory_cache_key(&RepositoryId::new("repo").unwrap(), package, script) + .unwrap(); + + fs::write( + package_dir.join(PACKAGE_LOCAL_FILES_DIR).join("data.txt"), + "two", + ) + .unwrap(); + let layout = RepositoryPackageDirectoryLayout::load(root).unwrap(); + let package = &layout.packages[0]; + let script = layout.unambiguous_version_script(package).unwrap(); + let second = + package_directory_cache_key(&RepositoryId::new("repo").unwrap(), package, script) + .unwrap(); + + assert_ne!(first.package_file_hash, second.package_file_hash); + } + #[test] fn highest_priority_selects_larger_number() { let priorities = [ From edbebceefc960caf20a03eb9a5a9e5210f111949 Mon Sep 17 00:00:00 2001 From: Xiangzhe Date: Sat, 27 Jun 2026 02:56:13 +0800 Subject: [PATCH 35/52] feat(cache): add provider refresh primitives --- crates/getter-operations/src/lib.rs | 1 + .../getter-operations/src/provider_cache.rs | 268 +++++++++++++++ crates/getter-storage/src/lib.rs | 312 ++++++++++++++++++ 3 files changed, 581 insertions(+) create mode 100644 crates/getter-operations/src/provider_cache.rs diff --git a/crates/getter-operations/src/lib.rs b/crates/getter-operations/src/lib.rs index 4bd9337..a2e7afe 100644 --- a/crates/getter-operations/src/lib.rs +++ b/crates/getter-operations/src/lib.rs @@ -7,5 +7,6 @@ pub mod autogen; pub mod legacy_room; +pub mod provider_cache; pub mod read_model; pub mod runtime; diff --git a/crates/getter-operations/src/provider_cache.rs b/crates/getter-operations/src/provider_cache.rs new file mode 100644 index 0000000..771cd81 --- /dev/null +++ b/crates/getter-operations/src/provider_cache.rs @@ -0,0 +1,268 @@ +//! Shared provider/source cache refresh helper. +//! +//! Provider implementations can use this small helper to keep ADR-0010/0012 +//! forced-refresh semantics consistent: a successful refresh replaces the cache, +//! while a failed forced refresh may return old cache only with explicit stale +//! diagnostics. + +use getter_storage::{CacheDb, ProviderResponseUpsert, StorageError, StoredProviderResponse}; +use serde_json::Value; + +pub const CACHE_REFRESH_FAILED: &str = "cache.refresh_failed"; +pub const USED_STALE_CACHE: &str = "used_stale_cache"; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ProviderCacheMode { + UseCached, + ForceRefresh, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ProviderCacheSource { + Cache, + Refreshed, + Stale, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct ProviderCacheRequest<'a> { + pub cache_key: &'a str, + pub provider: &'a str, + pub mode: ProviderCacheMode, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ProviderCacheDiagnostic { + pub code: String, + pub message: String, + pub cache_key: String, + pub provider: String, + pub stale_fetched_at_unix: Option, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ProviderCacheResult { + pub response: StoredProviderResponse, + pub source: ProviderCacheSource, + pub diagnostics: Vec, +} + +#[derive(Debug, thiserror::Error)] +pub enum ProviderCacheOperationError { + #[error("storage error: {0}")] + Storage(#[from] StorageError), + #[error("provider refresh failed: {0}")] + RefreshFailed(String), +} + +pub fn read_or_refresh_provider_response( + db: &CacheDb, + request: ProviderCacheRequest<'_>, + refresh: F, +) -> Result +where + F: FnOnce() -> Result, +{ + let cached = db.provider_response(request.cache_key)?; + if request.mode == ProviderCacheMode::UseCached { + if let Some(response) = cached { + return Ok(ProviderCacheResult { + response, + source: ProviderCacheSource::Cache, + diagnostics: Vec::new(), + }); + } + } + + match refresh() { + Ok(response_json) => { + let response = db.upsert_provider_response(&ProviderResponseUpsert { + cache_key: request.cache_key.to_owned(), + provider: request.provider.to_owned(), + response_json, + })?; + Ok(ProviderCacheResult { + response, + source: ProviderCacheSource::Refreshed, + diagnostics: Vec::new(), + }) + } + Err(detail) => match cached { + Some(response) => { + let stale_fetched_at_unix = Some(response.fetched_at_unix); + Ok(ProviderCacheResult { + response, + source: ProviderCacheSource::Stale, + diagnostics: vec![ + ProviderCacheDiagnostic { + code: CACHE_REFRESH_FAILED.to_owned(), + message: detail, + cache_key: request.cache_key.to_owned(), + provider: request.provider.to_owned(), + stale_fetched_at_unix, + }, + ProviderCacheDiagnostic { + code: USED_STALE_CACHE.to_owned(), + message: "using stale provider cache after refresh failure".to_owned(), + cache_key: request.cache_key.to_owned(), + provider: request.provider.to_owned(), + stale_fetched_at_unix, + }, + ], + }) + } + None => Err(ProviderCacheOperationError::RefreshFailed(detail)), + }, + } +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + use std::cell::Cell; + + #[test] + fn uses_cached_provider_response_without_refreshing() { + let db = CacheDb::open_in_memory().unwrap(); + db.upsert_provider_response(&ProviderResponseUpsert { + cache_key: "fdroid:official:index".to_owned(), + provider: "fdroid".to_owned(), + response_json: json!({ "revision": "cached" }), + }) + .unwrap(); + let refreshed = Cell::new(false); + + let result = read_or_refresh_provider_response( + &db, + ProviderCacheRequest { + cache_key: "fdroid:official:index", + provider: "fdroid", + mode: ProviderCacheMode::UseCached, + }, + || { + refreshed.set(true); + Ok(json!({ "revision": "fresh" })) + }, + ) + .unwrap(); + + assert!(!refreshed.get()); + assert_eq!(result.source, ProviderCacheSource::Cache); + assert_eq!(result.response.response_json["revision"], "cached"); + assert!(result.diagnostics.is_empty()); + } + + #[test] + fn cache_miss_refreshes_and_stores_provider_response() { + let db = CacheDb::open_in_memory().unwrap(); + + let result = read_or_refresh_provider_response( + &db, + ProviderCacheRequest { + cache_key: "github:f-droid/fdroidclient:releases", + provider: "github", + mode: ProviderCacheMode::UseCached, + }, + || Ok(json!({ "etag": "fresh" })), + ) + .unwrap(); + + assert_eq!(result.source, ProviderCacheSource::Refreshed); + assert_eq!(result.response.response_json["etag"], "fresh"); + assert_eq!( + db.provider_response("github:f-droid/fdroidclient:releases") + .unwrap() + .unwrap() + .response_json["etag"], + "fresh" + ); + } + + #[test] + fn forced_refresh_replaces_cached_provider_response() { + let db = CacheDb::open_in_memory().unwrap(); + db.upsert_provider_response(&ProviderResponseUpsert { + cache_key: "fdroid:official:index".to_owned(), + provider: "fdroid".to_owned(), + response_json: json!({ "revision": "old" }), + }) + .unwrap(); + + let result = read_or_refresh_provider_response( + &db, + ProviderCacheRequest { + cache_key: "fdroid:official:index", + provider: "fdroid", + mode: ProviderCacheMode::ForceRefresh, + }, + || Ok(json!({ "revision": "new" })), + ) + .unwrap(); + + assert_eq!(result.source, ProviderCacheSource::Refreshed); + assert_eq!(result.response.response_json["revision"], "new"); + assert_eq!( + db.provider_response("fdroid:official:index") + .unwrap() + .unwrap() + .response_json["revision"], + "new" + ); + } + + #[test] + fn forced_refresh_failure_returns_stale_cache_with_diagnostics() { + let db = CacheDb::open_in_memory().unwrap(); + db.upsert_provider_response(&ProviderResponseUpsert { + cache_key: "github:f-droid/fdroidclient:releases".to_owned(), + provider: "github".to_owned(), + response_json: json!({ "etag": "old" }), + }) + .unwrap(); + + let result = read_or_refresh_provider_response( + &db, + ProviderCacheRequest { + cache_key: "github:f-droid/fdroidclient:releases", + provider: "github", + mode: ProviderCacheMode::ForceRefresh, + }, + || Err("HTTP 503".to_owned()), + ) + .unwrap(); + + assert_eq!(result.source, ProviderCacheSource::Stale); + assert_eq!(result.response.response_json["etag"], "old"); + let codes: Vec<_> = result + .diagnostics + .iter() + .map(|diagnostic| diagnostic.code.as_str()) + .collect(); + assert_eq!(codes, vec![CACHE_REFRESH_FAILED, USED_STALE_CACHE]); + assert!(result + .diagnostics + .iter() + .all(|diagnostic| diagnostic.stale_fetched_at_unix.is_some())); + } + + #[test] + fn refresh_failure_without_stale_cache_is_an_error() { + let db = CacheDb::open_in_memory().unwrap(); + let error = read_or_refresh_provider_response( + &db, + ProviderCacheRequest { + cache_key: "fdroid:official:index", + provider: "fdroid", + mode: ProviderCacheMode::ForceRefresh, + }, + || Err("network unavailable".to_owned()), + ) + .unwrap_err(); + + assert!(matches!( + error, + ProviderCacheOperationError::RefreshFailed(_) + )); + } +} diff --git a/crates/getter-storage/src/lib.rs b/crates/getter-storage/src/lib.rs index e8d7b02..2af994e 100644 --- a/crates/getter-storage/src/lib.rs +++ b/crates/getter-storage/src/lib.rs @@ -10,6 +10,7 @@ use getter_core::task::{ }; use getter_core::{PackageId, RepositoryId, RepositoryPriority, UpdateAction}; use rusqlite::{params, Connection, OptionalExtension, Params, Transaction}; +use serde_json::Value; use std::path::Path; use std::str::FromStr; @@ -884,6 +885,161 @@ CREATE TABLE IF NOT EXISTS provider_responses ( )?; Ok(()) } + + pub fn upsert_provider_response( + &self, + response: &ProviderResponseUpsert, + ) -> Result { + let response_json = serde_json::to_string(&response.response_json)?; + self.conn.execute( + r#" +INSERT INTO provider_responses(cache_key, provider, response_json) +VALUES (?1, ?2, ?3) +ON CONFLICT(cache_key) DO UPDATE SET + provider = excluded.provider, + response_json = excluded.response_json, + fetched_at_unix = unixepoch() +"#, + params![response.cache_key, response.provider, response_json], + )?; + self.provider_response(&response.cache_key)?.ok_or_else(|| { + StorageError::Invariant(format!( + "provider response '{}' was not readable after upsert", + response.cache_key + )) + }) + } + + pub fn provider_response( + &self, + cache_key: &str, + ) -> Result, StorageError> { + let mut stmt = self.conn.prepare( + r#" +SELECT cache_key, provider, response_json, fetched_at_unix +FROM provider_responses +WHERE cache_key = ?1 +"#, + )?; + let row = stmt + .query_row(params![cache_key], |row| { + Ok(( + row.get::<_, String>(0)?, + row.get::<_, String>(1)?, + row.get::<_, String>(2)?, + row.get::<_, i64>(3)?, + )) + }) + .optional()?; + row.map( + |(cache_key, provider, response_json, fetched_at_unix)| -> Result<_, StorageError> { + Ok(StoredProviderResponse { + cache_key, + provider, + response_json: serde_json::from_str(&response_json)?, + fetched_at_unix, + }) + }, + ) + .transpose() + } + + pub fn upsert_evaluated_package( + &self, + package: &EvaluatedPackageUpsert, + ) -> Result { + let evaluated_json = serde_json::to_string(&package.evaluated_json)?; + self.conn.execute( + r#" +INSERT INTO evaluated_packages( + cache_key, + repository_id, + package_id, + package_file_hash, + schema_version, + evaluated_json +) +VALUES (?1, ?2, ?3, ?4, ?5, ?6) +ON CONFLICT(cache_key) DO UPDATE SET + repository_id = excluded.repository_id, + package_id = excluded.package_id, + package_file_hash = excluded.package_file_hash, + schema_version = excluded.schema_version, + evaluated_json = excluded.evaluated_json, + evaluated_at_unix = unixepoch() +"#, + params![ + package.cache_key, + package.repository_id.as_str(), + package.package_id.to_string(), + package.package_file_hash, + package.schema_version, + evaluated_json, + ], + )?; + self.evaluated_package(&package.cache_key)?.ok_or_else(|| { + StorageError::Invariant(format!( + "evaluated package '{}' was not readable after upsert", + package.cache_key + )) + }) + } + + pub fn evaluated_package( + &self, + cache_key: &str, + ) -> Result, StorageError> { + let mut stmt = self.conn.prepare( + r#" +SELECT + cache_key, + repository_id, + package_id, + package_file_hash, + schema_version, + evaluated_json, + evaluated_at_unix +FROM evaluated_packages +WHERE cache_key = ?1 +"#, + )?; + let row = stmt + .query_row(params![cache_key], |row| { + Ok(( + row.get::<_, String>(0)?, + row.get::<_, String>(1)?, + row.get::<_, String>(2)?, + row.get::<_, String>(3)?, + row.get::<_, String>(4)?, + row.get::<_, String>(5)?, + row.get::<_, i64>(6)?, + )) + }) + .optional()?; + row.map( + |( + cache_key, + repository_id, + package_id, + package_file_hash, + schema_version, + evaluated_json, + evaluated_at_unix, + )| + -> Result<_, StorageError> { + Ok(StoredEvaluatedPackage { + cache_key, + repository_id: RepositoryId::new(repository_id)?, + package_id: package_id.parse()?, + package_file_hash, + schema_version, + evaluated_json: serde_json::from_str(&evaluated_json)?, + evaluated_at_unix, + }) + }, + ) + .transpose() + } } #[derive(Debug, Clone, PartialEq, Eq)] @@ -896,6 +1052,42 @@ pub struct StoredRepository { pub revision: Option, } +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ProviderResponseUpsert { + pub cache_key: String, + pub provider: String, + pub response_json: Value, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct StoredProviderResponse { + pub cache_key: String, + pub provider: String, + pub response_json: Value, + pub fetched_at_unix: i64, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct EvaluatedPackageUpsert { + pub cache_key: String, + pub repository_id: RepositoryId, + pub package_id: PackageId, + pub package_file_hash: String, + pub schema_version: String, + pub evaluated_json: Value, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct StoredEvaluatedPackage { + pub cache_key: String, + pub repository_id: RepositoryId, + pub package_id: PackageId, + pub package_file_hash: String, + pub schema_version: String, + pub evaluated_json: Value, + pub evaluated_at_unix: i64, +} + #[derive(Debug, Clone, PartialEq, Eq)] pub struct TrackedPackageUpsert { pub package_id: PackageId, @@ -1498,4 +1690,124 @@ VALUES ('android/org.fdroid.fdroid', '1.2.3'); fn cache_db_migrates_schema() { let _db = CacheDb::open_in_memory().unwrap(); } + + #[test] + fn cache_db_round_trips_provider_response_cache_entries() { + let db = CacheDb::open_in_memory().unwrap(); + + let stored = db + .upsert_provider_response(&ProviderResponseUpsert { + cache_key: "fdroid:official:index:v1".to_owned(), + provider: "fdroid".to_owned(), + response_json: serde_json::json!({ + "endpoint": "official", + "packages": ["org.fdroid.fdroid"] + }), + }) + .unwrap(); + + assert_eq!(stored.cache_key, "fdroid:official:index:v1"); + assert_eq!(stored.provider, "fdroid"); + assert_eq!(stored.response_json["endpoint"], "official"); + assert_eq!( + db.provider_response("fdroid:official:index:v1") + .unwrap() + .unwrap(), + stored + ); + } + + #[test] + fn cache_db_replaces_provider_response_for_same_cache_key() { + let db = CacheDb::open_in_memory().unwrap(); + db.upsert_provider_response(&ProviderResponseUpsert { + cache_key: "github:f-droid/fdroidclient:releases".to_owned(), + provider: "github".to_owned(), + response_json: serde_json::json!({ "etag": "old" }), + }) + .unwrap(); + + let stored = db + .upsert_provider_response(&ProviderResponseUpsert { + cache_key: "github:f-droid/fdroidclient:releases".to_owned(), + provider: "github".to_owned(), + response_json: serde_json::json!({ "etag": "new" }), + }) + .unwrap(); + + assert_eq!(stored.response_json["etag"], "new"); + assert_eq!( + db.provider_response("github:f-droid/fdroidclient:releases") + .unwrap() + .unwrap() + .response_json["etag"], + "new" + ); + } + + #[test] + fn cache_db_round_trips_evaluated_package_cache_entries() { + let db = CacheDb::open_in_memory().unwrap(); + let stored = db + .upsert_evaluated_package(&EvaluatedPackageUpsert { + cache_key: "official:android/app/org.fdroid.fdroid:sha512:123".to_owned(), + repository_id: RepositoryId::new("official").unwrap(), + package_id: "android/app/org.fdroid.fdroid".parse().unwrap(), + package_file_hash: "sha512:123".to_owned(), + schema_version: "getter.package.v1".to_owned(), + evaluated_json: serde_json::json!({ + "name": "F-Droid", + "updates": [] + }), + }) + .unwrap(); + + assert_eq!(stored.repository_id.as_str(), "official"); + assert_eq!( + stored.package_id.to_string(), + "android/app/org.fdroid.fdroid" + ); + assert_eq!(stored.evaluated_json["name"], "F-Droid"); + assert_eq!( + db.evaluated_package("official:android/app/org.fdroid.fdroid:sha512:123") + .unwrap() + .unwrap(), + stored + ); + } + + #[test] + fn cache_db_replaces_evaluated_package_for_same_cache_key() { + let db = CacheDb::open_in_memory().unwrap(); + let cache_key = "official:android/app/org.fdroid.fdroid:sha512:123"; + db.upsert_evaluated_package(&EvaluatedPackageUpsert { + cache_key: cache_key.to_owned(), + repository_id: RepositoryId::new("official").unwrap(), + package_id: "android/app/org.fdroid.fdroid".parse().unwrap(), + package_file_hash: "sha512:old".to_owned(), + schema_version: "getter.package.v1".to_owned(), + evaluated_json: serde_json::json!({ "name": "Old" }), + }) + .unwrap(); + + let stored = db + .upsert_evaluated_package(&EvaluatedPackageUpsert { + cache_key: cache_key.to_owned(), + repository_id: RepositoryId::new("official").unwrap(), + package_id: "android/app/org.fdroid.fdroid".parse().unwrap(), + package_file_hash: "sha512:new".to_owned(), + schema_version: "getter.package.v1".to_owned(), + evaluated_json: serde_json::json!({ "name": "New" }), + }) + .unwrap(); + + assert_eq!(stored.package_file_hash, "sha512:new"); + assert_eq!( + db.evaluated_package(cache_key) + .unwrap() + .unwrap() + .evaluated_json["name"], + "New" + ); + } } From bc94040e4facb44cd3d5a2a9cd65b83902959b6d Mon Sep 17 00:00:00 2001 From: Xiangzhe Date: Sat, 27 Jun 2026 03:33:44 +0800 Subject: [PATCH 36/52] feat(providers): parse F-Droid catalog fixtures --- Cargo.lock | 11 + crates/getter-core/src/lib.rs | 6 + crates/getter-core/src/lua.rs | 44 ++++ crates/getter-core/src/update.rs | 4 + crates/getter-operations/src/runtime.rs | 3 + crates/getter-providers/Cargo.toml | 2 + crates/getter-providers/src/lib.rs | 299 +++++++++++++++++++++++- 7 files changed, 364 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 34622f9..c43d98e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -636,6 +636,8 @@ name = "getter-providers" version = "0.1.0" dependencies = [ "getter-core", + "roxmltree", + "thiserror 1.0.69", ] [[package]] @@ -1178,6 +1180,15 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "roxmltree" +version = "0.21.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1964b10c76125c36f8afe190065a4bf9a87bf324842c05701330bba9f1cacbb" +dependencies = [ + "memchr", +] + [[package]] name = "rusqlite" version = "0.32.1" diff --git a/crates/getter-core/src/lib.rs b/crates/getter-core/src/lib.rs index f02c4b0..957386c 100644 --- a/crates/getter-core/src/lib.rs +++ b/crates/getter-core/src/lib.rs @@ -281,6 +281,8 @@ pub struct PackagePermissions { pub struct UpdateCandidate { pub version: String, #[serde(default)] + pub version_code: Option, + #[serde(default)] pub channel: Option, #[serde(default)] pub source: Option, @@ -294,6 +296,10 @@ pub struct UpdateArtifact { pub url: String, #[serde(default)] pub file_name: Option, + #[serde(default)] + pub sha256: Option, + #[serde(default)] + pub size: Option, } /// Candidate selected for update after package/user-state policy. diff --git a/crates/getter-core/src/lua.rs b/crates/getter-core/src/lua.rs index 8d6f74e..8e33718 100644 --- a/crates/getter-core/src/lua.rs +++ b/crates/getter-core/src/lua.rs @@ -690,6 +690,7 @@ fn parse_update_candidates( let artifacts = parse_update_artifacts(path, object.get("artifacts"))?; Ok(UpdateCandidate { version: required_string(path, object, "version")?.to_owned(), + version_code: optional_i64(path, object, "version_code")?, channel: optional_string(object, "channel"), source: optional_string(object, "source"), artifacts, @@ -720,6 +721,8 @@ fn parse_update_artifacts( name: required_string(path, object, "name")?.to_owned(), url: required_string(path, object, "url")?.to_owned(), file_name: optional_string(object, "file_name"), + sha256: optional_string(object, "sha256"), + size: optional_u64(path, object, "size")?, }) }) .collect() @@ -732,6 +735,38 @@ fn optional_string(object: &Map, field: &str) -> Option, + field: &str, +) -> Result, LuaPackageError> { + object + .get(field) + .map(|value| { + value.as_i64().ok_or_else(|| LuaPackageError::Schema { + path: path.to_path_buf(), + message: format!("field '{field}' must be an integer"), + }) + }) + .transpose() +} + +fn optional_u64( + path: &Path, + object: &Map, + field: &str, +) -> Result, LuaPackageError> { + object + .get(field) + .map(|value| { + value.as_u64().ok_or_else(|| LuaPackageError::Schema { + path: path.to_path_buf(), + message: format!("field '{field}' must be an unsigned integer"), + }) + }) + .transpose() +} + fn parse_permissions( path: &Path, value: Option<&JsonValue>, @@ -822,6 +857,7 @@ return package_def { updates = { { version = "1.2.0", + version_code = 120, channel = "stable", source = "fixture", artifacts = { @@ -829,6 +865,8 @@ return package_def { name = "app.apk", url = "https://example.invalid/app.apk", file_name = "fdroid.apk", + sha256 = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + size = 12345, }, }, }, @@ -852,12 +890,18 @@ return package_def { assert_eq!(package.source_priority, vec!["github", "fdroid"]); assert_eq!(package.updates.len(), 1); assert_eq!(package.updates[0].version, "1.2.0"); + assert_eq!(package.updates[0].version_code, Some(120)); assert_eq!(package.updates[0].channel.as_deref(), Some("stable")); assert_eq!(package.updates[0].source.as_deref(), Some("fixture")); assert_eq!( package.updates[0].artifacts[0].file_name.as_deref(), Some("fdroid.apk") ); + assert_eq!( + package.updates[0].artifacts[0].sha256.as_deref(), + Some("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa") + ); + assert_eq!(package.updates[0].artifacts[0].size, Some(12345)); } #[test] diff --git a/crates/getter-core/src/update.rs b/crates/getter-core/src/update.rs index 09952ef..6e9260c 100644 --- a/crates/getter-core/src/update.rs +++ b/crates/getter-core/src/update.rs @@ -504,6 +504,7 @@ mod tests { Some("1.0.0".to_owned()), vec![UpdateCandidate { version: "1.2.0".to_owned(), + version_code: None, channel: None, source: None, artifacts: Vec::new(), @@ -539,12 +540,15 @@ mod tests { fn candidate(version: &str) -> UpdateCandidate { UpdateCandidate { version: version.to_owned(), + version_code: None, channel: None, source: None, artifacts: vec![UpdateArtifact { name: "APK".to_owned(), url: format!("https://example.invalid/{version}.apk"), file_name: Some("app.apk".to_owned()), + sha256: None, + size: None, }], } } diff --git a/crates/getter-operations/src/runtime.rs b/crates/getter-operations/src/runtime.rs index d9e53f1..3721eed 100644 --- a/crates/getter-operations/src/runtime.rs +++ b/crates/getter-operations/src/runtime.rs @@ -795,12 +795,15 @@ return package_version { .into_iter() .map(|version| UpdateCandidate { version: version.to_owned(), + version_code: None, channel: None, source: None, artifacts: vec![UpdateArtifact { name: "app.apk".to_owned(), url: "https://example.invalid/app.apk".to_owned(), file_name: Some("app.apk".to_owned()), + sha256: None, + size: None, }], }) .collect(), diff --git a/crates/getter-providers/Cargo.toml b/crates/getter-providers/Cargo.toml index f5ae23c..9424784 100644 --- a/crates/getter-providers/Cargo.toml +++ b/crates/getter-providers/Cargo.toml @@ -5,3 +5,5 @@ edition.workspace = true [dependencies] getter-core = { path = "../getter-core", default-features = false } +roxmltree = "0.21" +thiserror = "1" diff --git a/crates/getter-providers/src/lib.rs b/crates/getter-providers/src/lib.rs index ee714b9..4d3d02a 100644 --- a/crates/getter-providers/src/lib.rs +++ b/crates/getter-providers/src/lib.rs @@ -1,13 +1,13 @@ //! Provider executor scaffolding for the UpgradeAll getter rewrite. //! -//! Real network/provider execution is intentionally deferred to a later ADR. -//! The first Phase D bridge uses package-declared static update candidates as a -//! mock provider so the rest of the runtime can exercise getter-owned update -//! selection and opaque action issuance without direct network side effects. +//! Real network/provider execution is intentionally deferred to later slices. +//! The current provider code is fixture-backed and keeps parsing/normalization in +//! Rust getter so Flutter and Android adapter glue do not learn provider formats. pub use getter_core as core; -use getter_core::{ResolvedPackage, UpdateCandidate}; +use getter_core::{ResolvedPackage, UpdateArtifact, UpdateCandidate}; +use roxmltree::{Document, Node}; /// Mock provider that returns the static `updates` candidates materialized from /// a resolved Lua package table. @@ -24,11 +24,242 @@ impl StaticPackageUpdatesProvider { } } +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct FdroidCatalog { + pub endpoint: FdroidEndpoint, + pub apps: Vec, +} + +impl FdroidCatalog { + pub fn app(&self, package_name: &str) -> Option<&FdroidApp> { + self.apps + .iter() + .find(|app| app.package_name == package_name) + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct FdroidEndpoint { + pub name: Option, + pub url: Option, + pub timestamp: Option, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct FdroidApp { + pub package_name: String, + pub name: Option, + pub summary: Option, + pub packages: Vec, +} + +impl FdroidApp { + pub fn update_candidates(&self, endpoint: &FdroidEndpoint) -> Vec { + self.packages + .iter() + .map(|release| release.update_candidate(endpoint)) + .collect() + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct FdroidRelease { + pub version: String, + pub version_code: Option, + pub apk_name: String, + pub sha256: Option, + pub size: Option, +} + +impl FdroidRelease { + fn update_candidate(&self, endpoint: &FdroidEndpoint) -> UpdateCandidate { + UpdateCandidate { + version: self.version.clone(), + version_code: self.version_code, + channel: None, + source: Some("fdroid".to_owned()), + artifacts: vec![UpdateArtifact { + name: self.apk_name.clone(), + url: endpoint + .url + .as_deref() + .map(|base| fdroid_artifact_url(base, &self.apk_name)) + .unwrap_or_else(|| self.apk_name.clone()), + file_name: Some(self.apk_name.clone()), + sha256: self.sha256.clone(), + size: self.size, + }], + } + } +} + +#[derive(Debug, thiserror::Error)] +pub enum FdroidCatalogError { + #[error("failed to parse F-Droid catalog XML: {0}")] + Xml(#[from] roxmltree::Error), + #[error("F-Droid catalog is missing required field {field} for {context}")] + MissingField { + context: String, + field: &'static str, + }, + #[error("F-Droid catalog field {field} for {context} is invalid: {value}")] + InvalidField { + context: String, + field: &'static str, + value: String, + }, +} + +pub fn parse_fdroid_index_xml(xml: &str) -> Result { + let document = Document::parse(xml)?; + let root = document.root_element(); + let repo = child_element(root, "repo"); + let endpoint = FdroidEndpoint { + name: repo + .and_then(|node| node.attribute("name")) + .map(str::to_owned), + url: repo + .and_then(|node| node.attribute("url")) + .map(str::to_owned), + timestamp: repo + .and_then(|node| node.attribute("timestamp")) + .map(str::to_owned), + }; + let apps = root + .children() + .filter(|node| node.has_tag_name("application")) + .map(parse_fdroid_app) + .collect::>()?; + + Ok(FdroidCatalog { endpoint, apps }) +} + +fn parse_fdroid_app(app: Node<'_, '_>) -> Result { + let package_name = app + .attribute("id") + .map(str::to_owned) + .or_else(|| child_text(app, "id")) + .filter(|id| !id.trim().is_empty()) + .ok_or_else(|| FdroidCatalogError::MissingField { + context: "application".to_owned(), + field: "id", + })?; + let packages = app + .children() + .filter(|node| node.has_tag_name("package")) + .map(|package| parse_fdroid_release(&package_name, package)) + .collect::>()?; + + Ok(FdroidApp { + package_name, + name: child_text(app, "name"), + summary: child_text(app, "summary"), + packages, + }) +} + +fn parse_fdroid_release( + package_name: &str, + package: Node<'_, '_>, +) -> Result { + let version = required_child_text(package, package_name, "version")?; + let apk_name = required_child_text(package, package_name, "apkname")?; + let version_code = child_text(package, "versioncode") + .map(|value| parse_i64(package_name, "versioncode", &value)) + .transpose()?; + let size = child_text(package, "size") + .map(|value| parse_u64(package_name, "size", &value)) + .transpose()?; + let sha256 = package + .children() + .find(|node| node.has_tag_name("hash") && node.attribute("type") == Some("sha256")) + .and_then(|node| node.text()) + .map(str::trim) + .filter(|value| !value.is_empty()) + .map(str::to_owned); + + Ok(FdroidRelease { + version, + version_code, + apk_name, + sha256, + size, + }) +} + +fn required_child_text( + parent: Node<'_, '_>, + context: &str, + field: &'static str, +) -> Result { + child_text(parent, field).ok_or_else(|| FdroidCatalogError::MissingField { + context: context.to_owned(), + field, + }) +} + +fn child_text(parent: Node<'_, '_>, tag: &str) -> Option { + child_element(parent, tag) + .and_then(|node| node.text()) + .map(str::trim) + .filter(|value| !value.is_empty()) + .map(str::to_owned) +} + +fn child_element<'a>(parent: Node<'a, 'a>, tag: &str) -> Option> { + parent.children().find(|node| node.has_tag_name(tag)) +} + +fn parse_i64(context: &str, field: &'static str, value: &str) -> Result { + value.parse().map_err(|_| FdroidCatalogError::InvalidField { + context: context.to_owned(), + field, + value: value.to_owned(), + }) +} + +fn parse_u64(context: &str, field: &'static str, value: &str) -> Result { + value.parse().map_err(|_| FdroidCatalogError::InvalidField { + context: context.to_owned(), + field, + value: value.to_owned(), + }) +} + +fn fdroid_artifact_url(base: &str, apk_name: &str) -> String { + format!("{}/{}", base.trim_end_matches('/'), apk_name) +} + #[cfg(test)] mod tests { use super::*; use getter_core::PackagePermissions; + const FDROID_FIXTURE: &str = r#" + + + + org.fdroid.fdroid + F-Droid + App repository client + + 1.20.0 + 1020000 + org.fdroid.fdroid_1020000.apk + aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa + 1234567 + + + 1.19.0 + 1019000 + org.fdroid.fdroid_1019000.apk + bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb + 1111111 + + + +"#; + #[test] fn static_provider_returns_package_declared_update_candidates() { let package = ResolvedPackage { @@ -40,6 +271,7 @@ mod tests { source_priority: Vec::new(), updates: vec![UpdateCandidate { version: "1.2.0".to_owned(), + version_code: None, channel: Some("stable".to_owned()), source: Some("fixture".to_owned()), artifacts: Vec::new(), @@ -52,4 +284,61 @@ mod tests { assert_eq!(candidates[0].version, "1.2.0"); assert_eq!(candidates[0].source.as_deref(), Some("fixture")); } + + #[test] + fn parses_fdroid_index_xml_into_catalog_facts() { + let catalog = parse_fdroid_index_xml(FDROID_FIXTURE).unwrap(); + + assert_eq!(catalog.endpoint.name.as_deref(), Some("F-Droid")); + assert_eq!( + catalog.endpoint.url.as_deref(), + Some("https://f-droid.org/repo") + ); + let app = catalog.app("org.fdroid.fdroid").unwrap(); + assert_eq!(app.name.as_deref(), Some("F-Droid")); + assert_eq!(app.summary.as_deref(), Some("App repository client")); + assert_eq!(app.packages.len(), 2); + assert_eq!(app.packages[0].version, "1.20.0"); + assert_eq!(app.packages[0].version_code, Some(1020000)); + assert_eq!( + app.packages[0].sha256.as_deref(), + Some("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa") + ); + assert_eq!(app.packages[0].size, Some(1234567)); + } + + #[test] + fn normalizes_fdroid_releases_to_update_candidates() { + let catalog = parse_fdroid_index_xml(FDROID_FIXTURE).unwrap(); + let app = catalog.app("org.fdroid.fdroid").unwrap(); + + let candidates = app.update_candidates(&catalog.endpoint); + + assert_eq!(candidates.len(), 2); + assert_eq!(candidates[0].version, "1.20.0"); + assert_eq!(candidates[0].version_code, Some(1020000)); + assert_eq!(candidates[0].source.as_deref(), Some("fdroid")); + let artifact = &candidates[0].artifacts[0]; + assert_eq!(artifact.name, "org.fdroid.fdroid_1020000.apk"); + assert_eq!( + artifact.url, + "https://f-droid.org/repo/org.fdroid.fdroid_1020000.apk" + ); + assert_eq!( + artifact.file_name.as_deref(), + Some("org.fdroid.fdroid_1020000.apk") + ); + assert_eq!( + artifact.sha256.as_deref(), + Some("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa") + ); + assert_eq!(artifact.size, Some(1234567)); + } + + #[test] + fn malformed_fdroid_index_reports_parse_error() { + let error = parse_fdroid_index_xml("").unwrap_err(); + + assert!(matches!(error, FdroidCatalogError::Xml(_))); + } } From d60cc75ff804edab69a1335a9f52bb205d5896f1 Mon Sep 17 00:00:00 2001 From: Xiangzhe Date: Sat, 27 Jun 2026 04:04:22 +0800 Subject: [PATCH 37/52] feat(operations): cache F-Droid catalog fixtures --- Cargo.lock | 2 + crates/getter-operations/Cargo.toml | 1 + .../getter-operations/src/fdroid_catalog.rs | 336 ++++++++++++++++++ crates/getter-operations/src/lib.rs | 1 + crates/getter-providers/Cargo.toml | 1 + crates/getter-providers/src/lib.rs | 9 +- 6 files changed, 346 insertions(+), 4 deletions(-) create mode 100644 crates/getter-operations/src/fdroid_catalog.rs diff --git a/Cargo.lock b/Cargo.lock index c43d98e..039770c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -620,6 +620,7 @@ dependencies = [ "json_comments", "serde", "serde_json", + "sha2", "tempfile", "thiserror 1.0.69", ] @@ -637,6 +638,7 @@ version = "0.1.0" dependencies = [ "getter-core", "roxmltree", + "serde", "thiserror 1.0.69", ] diff --git a/crates/getter-operations/Cargo.toml b/crates/getter-operations/Cargo.toml index 26863cb..ed85e8c 100644 --- a/crates/getter-operations/Cargo.toml +++ b/crates/getter-operations/Cargo.toml @@ -14,6 +14,7 @@ getter-providers = { path = "../getter-providers" } getter-storage = { path = "../getter-storage" } serde = { version = "1", features = ["derive"] } serde_json = "1" +sha2 = "0.10" thiserror = "1" [dev-dependencies] diff --git a/crates/getter-operations/src/fdroid_catalog.rs b/crates/getter-operations/src/fdroid_catalog.rs new file mode 100644 index 0000000..3a301f5 --- /dev/null +++ b/crates/getter-operations/src/fdroid_catalog.rs @@ -0,0 +1,336 @@ +//! Fixture-backed F-Droid catalog cache/query operations. +//! +//! This is the first getter-owned F-Droid operation slice after the provider +//! parser. It wires parsed catalog facts through `cache.db` refresh semantics +//! without introducing live HTTP, Flutter-owned provider parsing, or downloader +//! task state. + +use crate::provider_cache::{ + read_or_refresh_provider_response, ProviderCacheDiagnostic, ProviderCacheMode, + ProviderCacheOperationError, ProviderCacheRequest, ProviderCacheSource, +}; +use getter_providers::{parse_fdroid_index_xml, FdroidApp, FdroidCatalog, FdroidCatalogError}; +use getter_storage::{CacheDb, StorageError}; +use serde::Deserialize; +use serde_json::{json, Value}; +use sha2::{Digest, Sha512}; + +pub const FDROID_PROVIDER_ID: &str = "fdroid"; +pub const FDROID_CATALOG_CACHE_VERSION: &str = "fdroid-index-v1"; +pub const DEFAULT_FDROID_ENDPOINT_ID: &str = "official"; +pub const DEFAULT_FDROID_ENDPOINT_URL: &str = "https://f-droid.org/repo"; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct FdroidEndpointConfig { + pub endpoint_id: String, + pub endpoint_url: String, +} + +impl Default for FdroidEndpointConfig { + fn default() -> Self { + Self { + endpoint_id: DEFAULT_FDROID_ENDPOINT_ID.to_owned(), + endpoint_url: DEFAULT_FDROID_ENDPOINT_URL.to_owned(), + } + } +} + +impl FdroidEndpointConfig { + pub fn cache_key(&self) -> String { + format!( + "{FDROID_PROVIDER_ID}:{FDROID_CATALOG_CACHE_VERSION}:{}:{}", + self.endpoint_id, + digest_hex(&self.endpoint_url), + ) + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct FdroidCatalogResult { + pub endpoint: FdroidEndpointConfig, + pub cache_key: String, + pub catalog: FdroidCatalog, + pub source: ProviderCacheSource, + pub diagnostics: Vec, +} + +impl FdroidCatalogResult { + pub fn app(&self, package_name: &str) -> Option<&FdroidApp> { + self.catalog.app(package_name) + } +} + +#[derive(Debug, thiserror::Error)] +pub enum FdroidCatalogOperationError { + #[error("provider cache operation failed: {0}")] + Cache(#[from] ProviderCacheOperationError), + #[error("storage error: {0}")] + Storage(#[from] StorageError), + #[error("F-Droid catalog parse failed: {0}")] + Catalog(#[from] FdroidCatalogError), + #[error("F-Droid catalog serialization failed: {0}")] + Serialization(#[from] serde_json::Error), + #[error("invalid F-Droid catalog request: {0}")] + InvalidRequest(String), +} + +pub fn read_or_refresh_fdroid_catalog( + db: &CacheDb, + endpoint: FdroidEndpointConfig, + mode: ProviderCacheMode, + refresh_xml: F, +) -> Result +where + F: FnOnce() -> Result, +{ + let cache_key = endpoint.cache_key(); + let cache_result = read_or_refresh_provider_response( + db, + ProviderCacheRequest { + cache_key: &cache_key, + provider: FDROID_PROVIDER_ID, + mode, + }, + || { + let xml = refresh_xml()?; + let catalog = parse_fdroid_index_xml(&xml).map_err(|source| source.to_string())?; + serde_json::to_value(catalog).map_err(|source| source.to_string()) + }, + )?; + let catalog = serde_json::from_value(cache_result.response.response_json)?; + + Ok(FdroidCatalogResult { + endpoint, + cache_key, + catalog, + source: cache_result.source, + diagnostics: cache_result.diagnostics, + }) +} + +pub fn fdroid_catalog_json( + db: &CacheDb, + request_json: &str, +) -> Result { + let request: FdroidCatalogJsonRequest = serde_json::from_str(request_json)?; + let endpoint = FdroidEndpointConfig { + endpoint_id: request + .endpoint_id + .unwrap_or_else(|| DEFAULT_FDROID_ENDPOINT_ID.to_owned()), + endpoint_url: request + .endpoint_url + .unwrap_or_else(|| DEFAULT_FDROID_ENDPOINT_URL.to_owned()), + }; + let mode = match request.mode.as_deref() { + Some("force_refresh") => ProviderCacheMode::ForceRefresh, + Some("use_cached") | None => ProviderCacheMode::UseCached, + Some(other) => { + return Err(FdroidCatalogOperationError::InvalidRequest(format!( + "unknown mode '{other}'" + ))) + } + }; + let result = read_or_refresh_fdroid_catalog(db, endpoint, mode, || { + request + .index_xml + .ok_or_else(|| "fixture-backed F-Droid catalog refresh requires index_xml".to_owned()) + })?; + let package_matches: Vec = request + .package_names + .iter() + .filter_map(|package_name| result.app(package_name)) + .map(serde_json::to_value) + .collect::>()?; + + Ok(json!({ + "operation": "fdroid.catalog", + "provider": FDROID_PROVIDER_ID, + "endpoint_id": result.endpoint.endpoint_id, + "endpoint_url": result.endpoint.endpoint_url, + "cache_key": result.cache_key, + "source": provider_source_json(result.source), + "catalog": result.catalog, + "matches": package_matches, + "diagnostics": result.diagnostics.iter().map(diagnostic_json).collect::>(), + })) +} + +#[derive(Debug, Deserialize)] +struct FdroidCatalogJsonRequest { + #[serde(default)] + endpoint_id: Option, + #[serde(default)] + endpoint_url: Option, + #[serde(default)] + mode: Option, + #[serde(default)] + index_xml: Option, + #[serde(default)] + package_names: Vec, +} + +fn provider_source_json(source: ProviderCacheSource) -> &'static str { + match source { + ProviderCacheSource::Cache => "cache", + ProviderCacheSource::Refreshed => "refreshed", + ProviderCacheSource::Stale => "stale", + } +} + +fn diagnostic_json(diagnostic: &ProviderCacheDiagnostic) -> Value { + json!({ + "code": diagnostic.code, + "message": diagnostic.message, + "cache_key": diagnostic.cache_key, + "provider": diagnostic.provider, + "stale_fetched_at_unix": diagnostic.stale_fetched_at_unix, + }) +} + +fn digest_hex(value: &str) -> String { + let mut hasher = Sha512::new(); + hasher.update(value.as_bytes()); + format!("{:x}", hasher.finalize()) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::provider_cache::{CACHE_REFRESH_FAILED, USED_STALE_CACHE}; + use serde_json::json; + use std::cell::Cell; + + const FDROID_FIXTURE: &str = r#" + + + + F-Droid + App repository client + + 1.20.0 + 1020000 + org.fdroid.fdroid_1020000.apk + aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa + 1234567 + + + +"#; + + fn endpoint() -> FdroidEndpointConfig { + FdroidEndpointConfig::default() + } + + #[test] + fn cache_miss_parses_and_stores_fixture_catalog() { + let db = CacheDb::open_in_memory().unwrap(); + let endpoint = endpoint(); + let cache_key = endpoint.cache_key(); + + let result = + read_or_refresh_fdroid_catalog(&db, endpoint, ProviderCacheMode::UseCached, || { + Ok(FDROID_FIXTURE.to_owned()) + }) + .unwrap(); + + assert_eq!(result.source, ProviderCacheSource::Refreshed); + assert_eq!( + result + .catalog + .app("org.fdroid.fdroid") + .unwrap() + .packages + .len(), + 1 + ); + let cached = db.provider_response(&cache_key).unwrap().unwrap(); + assert_eq!(cached.provider, FDROID_PROVIDER_ID); + assert_eq!( + cached.response_json["apps"][0]["package_name"], + "org.fdroid.fdroid" + ); + } + + #[test] + fn cache_hit_avoids_refreshing_fixture_catalog() { + let db = CacheDb::open_in_memory().unwrap(); + read_or_refresh_fdroid_catalog(&db, endpoint(), ProviderCacheMode::UseCached, || { + Ok(FDROID_FIXTURE.to_owned()) + }) + .unwrap(); + let refreshed = Cell::new(false); + + let result = + read_or_refresh_fdroid_catalog(&db, endpoint(), ProviderCacheMode::UseCached, || { + refreshed.set(true); + Err("should not refresh".to_owned()) + }) + .unwrap(); + + assert!(!refreshed.get()); + assert_eq!(result.source, ProviderCacheSource::Cache); + assert_eq!(result.catalog.endpoint.name.as_deref(), Some("F-Droid")); + } + + #[test] + fn forced_refresh_failure_returns_stale_catalog_with_diagnostics() { + let db = CacheDb::open_in_memory().unwrap(); + read_or_refresh_fdroid_catalog(&db, endpoint(), ProviderCacheMode::UseCached, || { + Ok(FDROID_FIXTURE.to_owned()) + }) + .unwrap(); + + let result = read_or_refresh_fdroid_catalog( + &db, + endpoint(), + ProviderCacheMode::ForceRefresh, + || Err("HTTP 503".to_owned()), + ) + .unwrap(); + + assert_eq!(result.source, ProviderCacheSource::Stale); + assert_eq!( + result + .catalog + .app("org.fdroid.fdroid") + .unwrap() + .name + .as_deref(), + Some("F-Droid") + ); + let codes: Vec<_> = result + .diagnostics + .iter() + .map(|diagnostic| diagnostic.code.as_str()) + .collect(); + assert_eq!(codes, vec![CACHE_REFRESH_FAILED, USED_STALE_CACHE]); + } + + #[test] + fn json_operation_returns_package_name_matches_from_cached_catalog() { + let db = CacheDb::open_in_memory().unwrap(); + let first = fdroid_catalog_json( + &db, + &json!({ + "index_xml": FDROID_FIXTURE, + "package_names": ["org.fdroid.fdroid", "missing.package"] + }) + .to_string(), + ) + .unwrap(); + assert_eq!(first["source"], "refreshed"); + + let second = fdroid_catalog_json( + &db, + &json!({ + "package_names": ["org.fdroid.fdroid"] + }) + .to_string(), + ) + .unwrap(); + + assert_eq!(second["source"], "cache"); + assert_eq!(second["matches"].as_array().unwrap().len(), 1); + assert_eq!(second["matches"][0]["package_name"], "org.fdroid.fdroid"); + } +} diff --git a/crates/getter-operations/src/lib.rs b/crates/getter-operations/src/lib.rs index a2e7afe..ffe59eb 100644 --- a/crates/getter-operations/src/lib.rs +++ b/crates/getter-operations/src/lib.rs @@ -6,6 +6,7 @@ //! generated package files, manifests, and tracked state live here. pub mod autogen; +pub mod fdroid_catalog; pub mod legacy_room; pub mod provider_cache; pub mod read_model; diff --git a/crates/getter-providers/Cargo.toml b/crates/getter-providers/Cargo.toml index 9424784..31d2fc4 100644 --- a/crates/getter-providers/Cargo.toml +++ b/crates/getter-providers/Cargo.toml @@ -6,4 +6,5 @@ edition.workspace = true [dependencies] getter-core = { path = "../getter-core", default-features = false } roxmltree = "0.21" +serde = { version = "1", features = ["derive"] } thiserror = "1" diff --git a/crates/getter-providers/src/lib.rs b/crates/getter-providers/src/lib.rs index 4d3d02a..518d5ba 100644 --- a/crates/getter-providers/src/lib.rs +++ b/crates/getter-providers/src/lib.rs @@ -8,6 +8,7 @@ pub use getter_core as core; use getter_core::{ResolvedPackage, UpdateArtifact, UpdateCandidate}; use roxmltree::{Document, Node}; +use serde::{Deserialize, Serialize}; /// Mock provider that returns the static `updates` candidates materialized from /// a resolved Lua package table. @@ -24,7 +25,7 @@ impl StaticPackageUpdatesProvider { } } -#[derive(Debug, Clone, PartialEq, Eq)] +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct FdroidCatalog { pub endpoint: FdroidEndpoint, pub apps: Vec, @@ -38,14 +39,14 @@ impl FdroidCatalog { } } -#[derive(Debug, Clone, PartialEq, Eq)] +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct FdroidEndpoint { pub name: Option, pub url: Option, pub timestamp: Option, } -#[derive(Debug, Clone, PartialEq, Eq)] +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct FdroidApp { pub package_name: String, pub name: Option, @@ -62,7 +63,7 @@ impl FdroidApp { } } -#[derive(Debug, Clone, PartialEq, Eq)] +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct FdroidRelease { pub version: String, pub version_code: Option, From de63a6b515aa8292eeee957db8531b08c4cc467a Mon Sep 17 00:00:00 2001 From: Xiangzhe Date: Sat, 27 Jun 2026 10:24:47 +0800 Subject: [PATCH 38/52] feat(autogen): generate F-Droid package directories --- crates/getter-cli/src/lib.rs | 116 ++- crates/getter-cli/tests/bdd_cli.rs | 134 +++- .../features/cli/autogen_installed.feature | 35 + crates/getter-core/src/autogen.rs | 14 +- crates/getter-operations/src/autogen.rs | 71 +- .../getter-operations/src/fdroid_autogen.rs | 661 ++++++++++++++++++ crates/getter-operations/src/lib.rs | 1 + 7 files changed, 1017 insertions(+), 15 deletions(-) create mode 100644 crates/getter-operations/src/fdroid_autogen.rs diff --git a/crates/getter-cli/src/lib.rs b/crates/getter-cli/src/lib.rs index 0cb5e6f..1f3d406 100644 --- a/crates/getter-cli/src/lib.rs +++ b/crates/getter-cli/src/lib.rs @@ -22,6 +22,7 @@ use getter_downloader::{ cancel_download_task, record_install_result, run_fake_download_task, submit_fake_download_task, }; use getter_operations::autogen::{self, AutogenAcceptance, AutogenOperationError}; +use getter_operations::fdroid_autogen; use getter_operations::legacy_room::{self, LegacyRoomOperationError}; use getter_operations::runtime as runtime_operations; use getter_storage::legacy_room::{ @@ -126,6 +127,14 @@ pub enum CliCommand { preview: PathBuf, acceptance: AutogenAcceptance, }, + AutogenFdroidPreview { + index: PathBuf, + package_names: Vec, + }, + AutogenFdroidApply { + preview: PathBuf, + acceptance: AutogenAcceptance, + }, LegacyImportRoomBundle { bundle: PathBuf, }, @@ -526,6 +535,26 @@ where inventory: PathBuf::from(inventory), } } + [domain, subject, action, rest @ ..] + if domain == "autogen" && subject == "fdroid" && action == "preview" => + { + let (index, package_names) = parse_fdroid_autogen_preview_args(rest)?; + CliCommand::AutogenFdroidPreview { + index, + package_names, + } + } + [domain, subject, action, flag, preview, rest @ ..] + if domain == "autogen" + && subject == "fdroid" + && action == "apply" + && flag == "--preview" => + { + CliCommand::AutogenFdroidApply { + preview: PathBuf::from(preview), + acceptance: parse_autogen_acceptance(rest)?, + } + } [domain, subject, action, flag, preview, rest @ ..] if domain == "autogen" && subject == "cleanup" @@ -727,6 +756,39 @@ fn execute(invocation: CliInvocation) -> Result { autogen::apply_cleanup_preview(&invocation.data_dir, &db, &preview, &acceptance) .map_err(CliError::from) } + CliCommand::AutogenFdroidPreview { + index, + package_names, + } => { + let db = open_main_db(&invocation.data_dir)?; + let cache_db = open_cache_db(&invocation.data_dir)?; + let index_xml = read_fdroid_index(&index)?; + let request = json!({ + "index_xml": index_xml, + "package_names": package_names, + }); + fdroid_autogen::preview_fdroid_packages_json( + &invocation.data_dir, + &db, + &cache_db, + &request.to_string(), + ) + .map_err(CliError::from) + } + CliCommand::AutogenFdroidApply { + preview, + acceptance, + } => { + let db = open_main_db(&invocation.data_dir)?; + let preview = read_autogen_preview(&preview, "fdroid.autogen.preview")?; + fdroid_autogen::apply_fdroid_preview_json( + &invocation.data_dir, + &db, + &preview, + &acceptance, + ) + .map_err(CliError::from) + } CliCommand::LegacyImportRoomBundle { bundle } => { let db = open_main_db(&invocation.data_dir)?; if db.migration_record_exists(LEGACY_ROOM_MIGRATION_ID)? { @@ -845,6 +907,11 @@ fn open_main_db(data_dir: &Path) -> Result { Ok(MainDb::open(main_db_path(data_dir))?) } +fn open_cache_db(data_dir: &Path) -> Result { + initialize_storage(data_dir)?; + Ok(CacheDb::open(cache_db_path(data_dir))?) +} + fn parse_repository_id(value: &str) -> Result { RepositoryId::new(value).map_err(|source| CliError::Usage(source.to_string())) } @@ -894,6 +961,46 @@ fn parse_priority(value: &str) -> Result { .map_err(|source| CliError::Usage(format!("invalid repository priority: {source}"))) } +fn parse_fdroid_autogen_preview_args(args: &[String]) -> Result<(PathBuf, Vec), CliError> { + let mut index = None; + let mut package_names = Vec::new(); + let mut position = 0; + while position < args.len() { + match args[position].as_str() { + "--index" => { + let path = args.get(position + 1).ok_or_else(|| { + CliError::Usage("autogen fdroid preview --index requires a path".to_owned()) + })?; + index = Some(PathBuf::from(path)); + position += 2; + } + "--package" => { + let package_name = args.get(position + 1).ok_or_else(|| { + CliError::Usage( + "autogen fdroid preview --package requires a package name".to_owned(), + ) + })?; + package_names.push(package_name.clone()); + position += 2; + } + other => { + return Err(CliError::Usage(format!( + "unsupported autogen fdroid preview argument '{other}'" + ))) + } + } + } + let index = index.ok_or_else(|| { + CliError::Usage("autogen fdroid preview requires --index ".to_owned()) + })?; + if package_names.is_empty() { + return Err(CliError::Usage( + "autogen fdroid preview requires at least one --package ".to_owned(), + )); + } + Ok((index, package_names)) +} + fn parse_autogen_acceptance(args: &[String]) -> Result { match args { [flag] if flag == "--accept-all" => Ok(AutogenAcceptance::AcceptAll), @@ -1065,6 +1172,11 @@ fn evaluate_package_directory( .map_err(|error| CliError::PackageEval(error.to_string())) } +fn read_fdroid_index(path: &Path) -> Result { + fs::read_to_string(path) + .map_err(|source| CliError::Autogen(format!("failed to read F-Droid index: {source}"))) +} + fn read_installed_inventory(path: &Path) -> Result { let bytes = fs::read(path) .map_err(|source| CliError::Autogen(format!("failed to read inventory: {source}")))?; @@ -1594,7 +1706,7 @@ fn envelope_to_string(value: Value) -> String { } fn usage_text() -> String { - "Usage: getter --data-dir [--priority ]|repo eval |repo validate |package eval [--repo ]|storage validate|version pin |version unpin |hub list|update check --fixture |runtime script --script |debug fake-task submit --request |debug fake-task run |debug fake-task list|debug fake-task cancel |debug fake-task events --after --limit |debug fake-task install-result --status |autogen installed preview --inventory |autogen installed apply --preview (--accept-all|--accept ...)|autogen cleanup preview --inventory |autogen cleanup apply --preview (--accept-all|--accept ...)|legacy import-room-bundle |legacy import-room-db |legacy report-list>\nNote: `debug fake-task` commands are persisted fake-download scaffolding. ADR-0011 runtime task debugging uses `runtime script` and does not preserve task state across CLI invocations.\n".to_owned() + "Usage: getter --data-dir [--priority ]|repo eval |repo validate |package eval [--repo ]|storage validate|version pin |version unpin |hub list|update check --fixture |runtime script --script |debug fake-task submit --request |debug fake-task run |debug fake-task list|debug fake-task cancel |debug fake-task events --after --limit |debug fake-task install-result --status |autogen installed preview --inventory |autogen installed apply --preview (--accept-all|--accept ...)|autogen fdroid preview --index --package [--package ...]|autogen fdroid apply --preview (--accept-all|--accept ...)|autogen cleanup preview --inventory |autogen cleanup apply --preview (--accept-all|--accept ...)|legacy import-room-bundle |legacy import-room-db |legacy report-list>\nNote: `debug fake-task` commands are persisted fake-download scaffolding. ADR-0011 runtime task debugging uses `runtime script` and does not preserve task state across CLI invocations.\n".to_owned() } #[derive(Debug, Deserialize)] @@ -1661,6 +1773,8 @@ impl CliCommand { Self::AutogenInstalledApply { .. } => "autogen installed apply", Self::AutogenCleanupPreview { .. } => "autogen cleanup preview", Self::AutogenCleanupApply { .. } => "autogen cleanup apply", + Self::AutogenFdroidPreview { .. } => "autogen fdroid preview", + Self::AutogenFdroidApply { .. } => "autogen fdroid apply", Self::LegacyImportRoomBundle { .. } => "legacy import-room-bundle", Self::LegacyImportRoomDb { .. } => "legacy import-room-db", Self::LegacyReportList => "legacy report-list", diff --git a/crates/getter-cli/tests/bdd_cli.rs b/crates/getter-cli/tests/bdd_cli.rs index 35754d5..5fc72ac 100644 --- a/crates/getter-cli/tests/bdd_cli.rs +++ b/crates/getter-cli/tests/bdd_cli.rs @@ -15,6 +15,7 @@ struct CliWorld { inventory: Option, autogen_preview: Option, update_fixture: Option, + fdroid_index: Option, task_request: Option, runtime_script: Option, remembered_task_id: Option, @@ -142,6 +143,35 @@ fn installed_inventory_with_android_app(world: &mut CliWorld, package_name: Stri world.inventory = Some(inventory); } +#[given(expr = "a fixture F-Droid catalog index with package {string}")] +fn fixture_fdroid_catalog_index_with_package(world: &mut CliWorld, package_name: String) { + let temp = world.temp.as_ref().expect("tempdir exists"); + let index = temp.path().join("fdroid-index.xml"); + fs::write( + &index, + format!( + r#" + + + + F-Droid + App repository client + + 1.20.0 + 1020000 + {package_name}_1020000.apk + aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa + 1234567 + + + +"# + ), + ) + .expect("write F-Droid fixture index"); + world.fdroid_index = Some(index); +} + #[given("an empty installed inventory")] fn empty_installed_inventory(world: &mut CliWorld) { let temp = world.temp.as_ref().expect("tempdir exists"); @@ -824,6 +854,46 @@ fn run_getter_autogen_installed_apply_accept_all(world: &mut CliWorld) { world.json = None; } +#[when(expr = "I run getter autogen fdroid preview for package {string}")] +fn run_getter_autogen_fdroid_preview(world: &mut CliWorld, package_name: String) { + let index = world.fdroid_index.as_ref().expect("F-Droid index exists"); + let output = run_getter( + world, + [ + "autogen".to_owned(), + "fdroid".to_owned(), + "preview".to_owned(), + "--index".to_owned(), + index.to_string_lossy().to_string(), + "--package".to_owned(), + package_name, + ], + ); + world.output = Some(output); + world.json = None; +} + +#[when("I run getter autogen fdroid apply for that preview with accept-all")] +fn run_getter_autogen_fdroid_apply_accept_all(world: &mut CliWorld) { + let preview = world + .autogen_preview + .as_ref() + .expect("autogen preview exists"); + let output = run_getter( + world, + [ + "autogen".to_owned(), + "fdroid".to_owned(), + "apply".to_owned(), + "--preview".to_owned(), + preview.to_string_lossy().to_string(), + "--accept-all".to_owned(), + ], + ); + world.output = Some(output); + world.json = None; +} + #[when("I run getter autogen cleanup preview for that inventory")] fn run_getter_autogen_cleanup_preview(world: &mut CliWorld) { let inventory = world.inventory.as_ref().expect("inventory exists"); @@ -924,7 +994,7 @@ fn command_fails_with_autogen_error(world: &mut CliWorld) { assert_eq!(json["ok"], false); assert!(matches!( json["command"].as_str(), - Some("autogen cleanup apply" | "autogen installed apply") + Some("autogen cleanup apply" | "autogen installed apply" | "autogen fdroid apply") )); assert_eq!(json["error"]["code"], "autogen.error"); world.json = Some(json); @@ -1272,6 +1342,22 @@ fn package_eval_name_is(world: &mut CliWorld, package_name: String) { assert_eq!(json["data"]["package"]["name"], package_name); } +#[then(expr = "the package eval contains update version_code {int}")] +fn package_eval_contains_update_version_code(world: &mut CliWorld, version_code: i64) { + let json = current_json(world); + assert_eq!(json["ok"], true); + assert_eq!(json["command"], "package eval"); + let updates = json["data"]["package"]["updates"] + .as_array() + .expect("updates array"); + assert!( + updates + .iter() + .any(|update| update["version_code"].as_i64() == Some(version_code)), + "package eval should contain version_code {version_code}: {updates:?}" + ); +} + #[then(expr = "the pinned package version is {string}")] fn pinned_package_version_is(world: &mut CliWorld, version: String) { let json = current_json(world); @@ -1322,6 +1408,23 @@ fn update_check_has_no_selected_update(world: &mut CliWorld) { assert_eq!(json["data"]["actions"], Value::Array(Vec::new())); } +#[then(expr = "the F-Droid autogen preview contains candidate {string}")] +fn fdroid_autogen_preview_contains_candidate(world: &mut CliWorld, package_id: String) { + let json = current_json(world); + assert_eq!(json["ok"], true); + assert_eq!(json["command"], "autogen fdroid preview"); + assert_eq!(json["data"]["operation"], "fdroid.autogen.preview"); + let candidates = json["data"]["candidates"] + .as_array() + .expect("candidates array"); + assert!( + candidates + .iter() + .any(|candidate| candidate["package_id"].as_str() == Some(package_id.as_str())), + "preview should contain {package_id}: {candidates:?}" + ); +} + #[then(expr = "the autogen preview contains candidate {string}")] fn autogen_preview_contains_candidate(world: &mut CliWorld, package_id: String) { let json = current_json(world); @@ -1363,6 +1466,15 @@ fn save_autogen_preview_to_file(world: &mut CliWorld) { world.autogen_preview = Some(preview); } +#[then(expr = "the autogen repository contains generated F-Droid package {string}")] +fn autogen_repository_contains_generated_fdroid_package(world: &mut CliWorld, package_id: String) { + autogen_repository_contains_generated_package(world, package_id.clone()); + let path = autogen_repo_path(world).join(package_relative_path(&package_id)); + let content = fs::read_to_string(path.join("9999.lua")).expect("generated package readable"); + assert!(content.contains("return fdroid.package")); + assert!(content.contains("version_code = 1020000")); +} + #[then(expr = "the autogen repository contains generated package {string}")] fn autogen_repository_contains_generated_package(world: &mut CliWorld, package_id: String) { let path = autogen_repo_path(world).join(package_relative_path(&package_id)); @@ -1393,6 +1505,26 @@ fn app_list_contains_autogen_tracked_package(world: &mut CliWorld, package_id: S assert_eq!(app["package_resolution"], "generate_local_package"); } +#[then( + expr = "the F-Droid autogen preview skips package {string} because repository {string} covers it" +)] +fn fdroid_autogen_preview_skips_package_because_repository_covers_it( + world: &mut CliWorld, + package_id: String, + repository_id: String, +) { + let json = current_json(world); + assert_eq!(json["ok"], true); + assert_eq!(json["command"], "autogen fdroid preview"); + let skipped = json["data"]["skipped"].as_array().expect("skipped array"); + let skip = skipped + .iter() + .find(|skip| skip["package_id"].as_str() == Some(package_id.as_str())) + .unwrap_or_else(|| panic!("skipped should contain {package_id}: {skipped:?}")); + assert_eq!(skip["reason"], "covered_by_higher_priority_repo"); + assert_eq!(skip["covering_repo_id"], repository_id); +} + #[then(expr = "the autogen preview skips package {string} because repository {string} covers it")] fn autogen_preview_skips_package_because_repository_covers_it( world: &mut CliWorld, diff --git a/crates/getter-cli/tests/features/cli/autogen_installed.feature b/crates/getter-cli/tests/features/cli/autogen_installed.feature index 6a0838d..50f6b6a 100644 --- a/crates/getter-cli/tests/features/cli/autogen_installed.feature +++ b/crates/getter-cli/tests/features/cli/autogen_installed.feature @@ -1,5 +1,40 @@ @getter-cli @autogen Feature: Installed app autogen + Scenario: User previews explicit F-Droid package generation without writing files + Given an initialized getter data directory + And a fixture F-Droid catalog index with package "org.fdroid.fdroid" + When I run getter autogen fdroid preview for package "org.fdroid.fdroid" + Then the command succeeds + And the output is valid JSON + And the F-Droid autogen preview contains candidate "android/f-droid/app/org.fdroid.fdroid" + And the autogen repository has not been written + + Scenario: User applies explicit F-Droid autogen and validates the generated package directory + Given an initialized getter data directory + And a fixture F-Droid catalog index with package "org.fdroid.fdroid" + When I run getter autogen fdroid preview for package "org.fdroid.fdroid" + Then the command succeeds + And I save the autogen preview to a file + When I run getter autogen fdroid apply for that preview with accept-all + Then the command succeeds + And the autogen repository contains generated F-Droid package "android/f-droid/app/org.fdroid.fdroid" + And the app list contains autogen tracked package "android/f-droid/app/org.fdroid.fdroid" + When I run getter repo validate for autogen + Then the output reports a valid repository without network + When I run getter package eval for package "android/f-droid/app/org.fdroid.fdroid" + Then the command succeeds + And the package eval contains update version_code 1020000 + + Scenario: Higher-priority repositories suppress explicit F-Droid autogen candidates + Given an initialized getter data directory + And a package-directory repository "official" with package "android/f-droid/app/org.fdroid.fdroid" + And a fixture F-Droid catalog index with package "org.fdroid.fdroid" + When I run getter repo add for that repository with priority 0 + Then the command succeeds + When I run getter autogen fdroid preview for package "org.fdroid.fdroid" + Then the command succeeds + And the F-Droid autogen preview skips package "android/f-droid/app/org.fdroid.fdroid" because repository "official" covers it + Scenario: User previews installed app fallback generation without writing files Given an initialized getter data directory And an installed inventory with Android app "com.example.autogen" labeled "Example Autogen" diff --git a/crates/getter-core/src/autogen.rs b/crates/getter-core/src/autogen.rs index 8a30220..6a97c50 100644 --- a/crates/getter-core/src/autogen.rs +++ b/crates/getter-core/src/autogen.rs @@ -15,6 +15,7 @@ pub const INSTALLED_INVENTORY_VERSION: u32 = 1; pub const AUTOGEN_RECORD_FILE: &str = ".autogen.jsonc"; pub const AUTOGEN_RECORD_VERSION: u32 = 1; pub const INSTALLED_AUTOGEN_GENERATOR: &str = "installed-inventory"; +pub const FDROID_AUTOGEN_GENERATOR: &str = "fdroid-catalog"; pub const DEFAULT_AUTOGEN_REPOSITORY_ID: &str = "autogen"; pub const DEFAULT_AUTOGEN_REPOSITORY_NAME: &str = "UpgradeAll Autogen"; pub const GENERATED_MARKER: &str = "@generated by UpgradeAll getter autogen"; @@ -105,8 +106,17 @@ pub struct AutogenRecord { #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(tag = "kind", rename_all = "snake_case")] pub enum AutogenRecordInput { - InstalledAndroidPackage { package_name: String }, - InstalledMagiskModule { module_id: String }, + InstalledAndroidPackage { + package_name: String, + }, + InstalledMagiskModule { + module_id: String, + }, + FdroidPackage { + endpoint_id: String, + endpoint_url: String, + package_name: String, + }, } #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] diff --git a/crates/getter-operations/src/autogen.rs b/crates/getter-operations/src/autogen.rs index 0c0b5f0..0b597f4 100644 --- a/crates/getter-operations/src/autogen.rs +++ b/crates/getter-operations/src/autogen.rs @@ -128,7 +128,12 @@ pub fn cleanup_preview_json( let mut diagnostics = Vec::new(); for package in layout.packages { let relative_path = relative_package_path(&repo_path, &package.path)?; - let ownership = match read_owned_record(&package.path, &package.id, &relative_path) { + let ownership = match read_owned_record( + &package.path, + &package.id, + &relative_path, + INSTALLED_AUTOGEN_GENERATOR, + ) { Ok(ownership) => ownership, Err(error) => { diagnostics.push(autogen_diagnostic( @@ -173,10 +178,33 @@ pub fn apply_installed_preview( preview: &Value, acceptance: &AutogenAcceptance, ) -> AutogenOperationResult { + apply_preview( + data_dir, + db, + preview, + acceptance, + "installed.preview", + INSTALLED_AUTOGEN_GENERATOR, + ) +} + +pub(crate) fn apply_preview( + data_dir: &Path, + db: &MainDb, + preview: &Value, + acceptance: &AutogenAcceptance, + expected_operation: &str, + expected_generator: &str, +) -> AutogenOperationResult { + if preview.get("operation").and_then(Value::as_str) != Some(expected_operation) { + return Err(AutogenOperationError::Autogen(format!( + "autogen preview operation must be '{expected_operation}'" + ))); + } let (target_alias, repo_path, target_priority) = generated_repository_config(data_dir)?; if preview.get("target_repo_id").and_then(Value::as_str) != Some(target_alias.as_str()) { return Err(AutogenOperationError::Autogen(format!( - "installed preview target_repo_id must be '{}'", + "{expected_operation} target_repo_id must be '{}'", target_alias.as_str() ))); } @@ -187,10 +215,11 @@ pub fn apply_installed_preview( for candidate in accepted { let package_id = preview_package_id(candidate)?; let relative_path = preview_relative_path(candidate)?; - let payload = preview_candidate_payload(candidate, &package_id, &relative_path)?; + let payload = + preview_candidate_payload(candidate, &package_id, &relative_path, expected_generator)?; let target_dir = safe_join(&repo_path, &relative_path)?; if target_dir.exists() { - read_owned_record(&target_dir, &package_id, &relative_path)?; + read_owned_record(&target_dir, &package_id, &relative_path, expected_generator)?; clear_directory_contents(&target_dir)?; } else { fs::create_dir_all(&target_dir).map_err(|source| { @@ -243,7 +272,12 @@ pub fn apply_cleanup_preview( ) })?; let target_dir = safe_join(&repo_path, &relative_path)?; - let ownership = read_owned_record(&target_dir, &package_id, &relative_path)?; + let ownership = read_owned_record( + &target_dir, + &package_id, + &relative_path, + INSTALLED_AUTOGEN_GENERATOR, + )?; if ownership.content_hash != expected_hash { return Err(AutogenOperationError::Autogen(format!( "cleanup preview candidate {package_id} does not match current autogen record" @@ -286,7 +320,7 @@ pub fn default_autogen_repo_path(data_dir: &Path) -> PathBuf { .repository_path(&RepositoryId::new(DEFAULT_AUTOGEN_REPOSITORY_ID).expect("valid id")) } -fn generated_repository_config( +pub(crate) fn generated_repository_config( data_dir: &Path, ) -> AutogenOperationResult<(RepositoryId, PathBuf, RepositoryPriority)> { let layout = GetterDataDirLayout::new(data_dir); @@ -300,7 +334,7 @@ fn generated_repository_config( Ok((alias, path, priority)) } -fn higher_priority_package_coverage( +pub(crate) fn higher_priority_package_coverage( db: &MainDb, target_alias: &RepositoryId, target_priority: RepositoryPriority, @@ -465,6 +499,7 @@ fn preview_candidate_payload( candidate: &Value, package_id: &PackageId, relative_path: &Path, + expected_generator: &str, ) -> AutogenOperationResult { let record_content = candidate .get("autogen_record_content") @@ -489,7 +524,13 @@ fn preview_candidate_payload( } let files = preview_generated_files(candidate)?; let record = parse_record_content(&record_content)?; - validate_record(&record, package_id, relative_path, &files)?; + validate_record( + &record, + package_id, + relative_path, + &files, + expected_generator, + )?; Ok(PreviewCandidatePayload { record_content, files, @@ -590,6 +631,7 @@ fn read_owned_record( package_dir: &Path, package_id: &PackageId, relative_path: &Path, + expected_generator: &str, ) -> AutogenOperationResult { let record_path = package_dir.join(AUTOGEN_RECORD_FILE); if !record_path.is_file() { @@ -614,7 +656,13 @@ fn read_owned_record( )) })?; let files = read_recorded_files(package_dir, &record)?; - validate_record(&record, package_id, relative_path, &files)?; + validate_record( + &record, + package_id, + relative_path, + &files, + expected_generator, + )?; Ok(LoadedAutogenRecord { content_hash: content_hash_bytes(&bytes), }) @@ -674,6 +722,7 @@ fn validate_record( package_id: &PackageId, relative_path: &Path, files: &[GeneratedPackageFile], + expected_generator: &str, ) -> AutogenOperationResult<()> { if record.version != AUTOGEN_RECORD_VERSION { return Err(AutogenOperationError::Autogen(format!( @@ -681,9 +730,9 @@ fn validate_record( record.version ))); } - if record.generator != INSTALLED_AUTOGEN_GENERATOR { + if record.generator != expected_generator { return Err(AutogenOperationError::Autogen(format!( - "autogen record generator '{}' does not match '{INSTALLED_AUTOGEN_GENERATOR}'", + "autogen record generator '{}' does not match '{expected_generator}'", record.generator ))); } diff --git a/crates/getter-operations/src/fdroid_autogen.rs b/crates/getter-operations/src/fdroid_autogen.rs new file mode 100644 index 0000000..2968d1f --- /dev/null +++ b/crates/getter-operations/src/fdroid_autogen.rs @@ -0,0 +1,661 @@ +//! Fixture-backed F-Droid explicit autogen preview/apply operations. +//! +//! This consumes getter-owned cached F-Droid catalog facts and writes ordinary +//! generated package directories into the configured generated repository. It is +//! intentionally still fixture/offline friendly: live HTTP, downloader state, +//! installer semantics, and Flutter/Kotlin provider logic remain outside this +//! slice. + +use crate::autogen::{self, AutogenAcceptance, AutogenOperationError, AutogenOperationResult}; +use crate::fdroid_catalog::{ + read_or_refresh_fdroid_catalog, FdroidEndpointConfig, DEFAULT_FDROID_ENDPOINT_ID, + DEFAULT_FDROID_ENDPOINT_URL, +}; +use crate::provider_cache::ProviderCacheMode; +use getter_core::autogen::{ + content_hash, package_relative_path, record_file_key, render_autogen_record, AutogenRecord, + AutogenRecordInput, GeneratedPackageFile, AUTOGEN_RECORD_VERSION, FDROID_AUTOGEN_GENERATOR, +}; +use getter_core::{InstalledTarget, PackageId, PackageKind}; +use getter_providers::{FdroidApp, FdroidRelease}; +use getter_storage::{CacheDb, MainDb}; +use serde::Deserialize; +use serde_json::{json, Value}; +use std::path::{Path, PathBuf}; + +pub fn preview_fdroid_packages_json( + data_dir: &Path, + main_db: &MainDb, + cache_db: &CacheDb, + request_json: &str, +) -> AutogenOperationResult { + let request: FdroidAutogenPreviewRequest = + serde_json::from_str(request_json).map_err(|source| { + AutogenOperationError::Autogen(format!( + "invalid F-Droid autogen preview request: {source}" + )) + })?; + let endpoint = request.endpoint_config(); + let catalog = read_or_refresh_fdroid_catalog(cache_db, endpoint, request.cache_mode()?, || { + request + .index_xml + .ok_or_else(|| "fixture-backed F-Droid autogen refresh requires index_xml".to_owned()) + }) + .map_err(|source| AutogenOperationError::Autogen(source.to_string()))?; + let (target_alias, target_path, target_priority) = + autogen::generated_repository_config(data_dir)?; + let covered = + autogen::higher_priority_package_coverage(main_db, &target_alias, target_priority)?; + let mut candidates = Vec::new(); + let mut skipped = Vec::new(); + let mut diagnostics: Vec = catalog + .diagnostics + .iter() + .map(|diagnostic| { + json!({ + "code": diagnostic.code, + "message": diagnostic.message, + "cache_key": diagnostic.cache_key, + "provider": diagnostic.provider, + "stale_fetched_at_unix": diagnostic.stale_fetched_at_unix, + }) + }) + .collect(); + + for package_name in unique_requested_packages(&request.package_names) { + let package_id = fdroid_package_id(&package_name)?; + if let Some(repository_id) = covered.get(&package_id) { + skipped.push(json!({ + "package_id": package_id.to_string(), + "reason": "covered_by_higher_priority_repo", + "covering_repo_id": repository_id.as_str(), + })); + continue; + } + let Some(app) = catalog.app(&package_name) else { + diagnostics.push(json!({ + "code": "provider.fdroid.package_not_found", + "message": format!("F-Droid package '{package_name}' was not found in endpoint '{}'", catalog.endpoint.endpoint_id), + "package_id": package_id.to_string(), + "provider": "fdroid", + "endpoint_id": catalog.endpoint.endpoint_id, + })); + skipped.push(json!({ + "package_id": package_id.to_string(), + "reason": "provider_package_not_found", + })); + continue; + }; + let candidate = fdroid_candidate_json( + &catalog.endpoint.endpoint_id, + &catalog.endpoint.endpoint_url, + &package_id, + app, + )?; + candidates.push(candidate); + } + + Ok(json!({ + "operation": "fdroid.autogen.preview", + "provider": "fdroid", + "endpoint_id": catalog.endpoint.endpoint_id, + "endpoint_url": catalog.endpoint.endpoint_url, + "cache_key": catalog.cache_key, + "source": match catalog.source { + crate::provider_cache::ProviderCacheSource::Cache => "cache", + crate::provider_cache::ProviderCacheSource::Refreshed => "refreshed", + crate::provider_cache::ProviderCacheSource::Stale => "stale", + }, + "target_repo_id": target_alias.as_str(), + "target_repo_path": target_path, + "summary": { + "candidate_count": candidates.len(), + "skipped_count": skipped.len(), + "write_count": candidates.len(), + "delete_count": 0, + }, + "candidates": candidates, + "skipped": skipped, + "diagnostics": diagnostics, + })) +} + +pub fn apply_fdroid_preview_json( + data_dir: &Path, + main_db: &MainDb, + preview: &Value, + acceptance: &AutogenAcceptance, +) -> AutogenOperationResult { + autogen::apply_preview( + data_dir, + main_db, + preview, + acceptance, + "fdroid.autogen.preview", + FDROID_AUTOGEN_GENERATOR, + ) +} + +#[derive(Debug, Deserialize)] +struct FdroidAutogenPreviewRequest { + #[serde(default)] + endpoint_id: Option, + #[serde(default)] + endpoint_url: Option, + #[serde(default)] + mode: Option, + #[serde(default)] + index_xml: Option, + #[serde(default)] + package_names: Vec, +} + +impl FdroidAutogenPreviewRequest { + fn endpoint_config(&self) -> FdroidEndpointConfig { + FdroidEndpointConfig { + endpoint_id: self + .endpoint_id + .clone() + .unwrap_or_else(|| DEFAULT_FDROID_ENDPOINT_ID.to_owned()), + endpoint_url: self + .endpoint_url + .clone() + .unwrap_or_else(|| DEFAULT_FDROID_ENDPOINT_URL.to_owned()), + } + } + + fn cache_mode(&self) -> AutogenOperationResult { + match self.mode.as_deref() { + Some("force_refresh") => Ok(ProviderCacheMode::ForceRefresh), + Some("use_cached") | None => Ok(ProviderCacheMode::UseCached), + Some(other) => Err(AutogenOperationError::Autogen(format!( + "unknown F-Droid autogen preview mode '{other}'" + ))), + } + } +} + +fn unique_requested_packages(package_names: &[String]) -> Vec { + let mut packages = package_names + .iter() + .map(|package| package.trim()) + .filter(|package| !package.is_empty()) + .map(str::to_owned) + .collect::>(); + packages.sort(); + packages.dedup(); + packages +} + +fn fdroid_package_id(package_name: &str) -> AutogenOperationResult { + PackageId::new(PackageKind::Android, format!("f-droid/app/{package_name}")) + .map_err(|source| AutogenOperationError::Autogen(source.to_string())) +} + +fn fdroid_candidate_json( + endpoint_id: &str, + endpoint_url: &str, + package_id: &PackageId, + app: &FdroidApp, +) -> AutogenOperationResult { + let relative_path = package_relative_path(package_id); + let display_name = app + .name + .as_deref() + .filter(|name| !name.trim().is_empty()) + .unwrap_or(&app.package_name); + let files = fdroid_generated_files(endpoint_url, app)?; + let record = AutogenRecord { + version: AUTOGEN_RECORD_VERSION, + generator: FDROID_AUTOGEN_GENERATOR.to_owned(), + package_id: package_id.clone(), + output_relative_path: relative_path.clone(), + input: AutogenRecordInput::FdroidPackage { + endpoint_id: endpoint_id.to_owned(), + endpoint_url: endpoint_url.to_owned(), + package_name: app.package_name.clone(), + }, + files: files + .iter() + .map(|file| { + ( + record_file_key(&file.relative_path), + file.content_hash.clone(), + ) + }) + .collect(), + }; + let record_content = render_autogen_record(&record) + .map_err(|source| AutogenOperationError::Autogen(source.to_string()))?; + let content_hash = content_hash(&record_content); + let file_json: Vec = files + .iter() + .map(|file| { + json!({ + "relative_path": file.relative_path, + "content_hash": file.content_hash, + "content": file.content, + }) + }) + .collect(); + + Ok(json!({ + "package_id": package_id.to_string(), + "kind": package_id.kind().as_str(), + "display_name": display_name, + "installed_target": InstalledTarget::AndroidPackage { package_name: app.package_name.clone() }, + "action": "create", + "output_relative_path": relative_path, + "content_hash": content_hash, + "content": record_content, + "autogen_record_content": record_content, + "files": file_json, + "provider": "fdroid", + "endpoint_id": endpoint_id, + "upstream_id": app.package_name, + })) +} + +fn fdroid_generated_files( + endpoint_url: &str, + app: &FdroidApp, +) -> AutogenOperationResult> { + let metadata = render_pretty_json(json!({ + "type": "android:app", + "android": { "package_name": app.package_name }, + }))?; + let manifest = String::new(); + let version_lua = fdroid_version_lua(endpoint_url, app); + Ok(vec![ + generated_file("metadata.jsonc", metadata), + generated_file("Manifest", manifest), + generated_file("9999.lua", version_lua), + ]) +} + +fn fdroid_version_lua(endpoint_url: &str, app: &FdroidApp) -> String { + let mut releases = app.packages.clone(); + releases.sort_by(|left, right| { + right + .version_code + .cmp(&left.version_code) + .then_with(|| right.version.cmp(&left.version)) + .then_with(|| left.apk_name.cmp(&right.apk_name)) + }); + let updates = releases + .iter() + .map(|release| fdroid_update_lua(endpoint_url, release)) + .collect::>() + .join(",\n"); + format!( + "#!/bin/upa-lua v1\n-- @generated by UpgradeAll getter autogen (F-Droid catalog fixture)\nlocal catalog_package = {{\n package_name = {},\n updates = {{{updates}\n }},\n}}\n\nlocal fdroid = {{}}\nfunction fdroid.package(spec)\n if spec.package_name ~= catalog_package.package_name then\n error(\"F-Droid generated package_name mismatch\")\n end\n return package_version {{ updates = catalog_package.updates }}\nend\n\nreturn fdroid.package {{\n package_name = {},\n}}\n", + lua_string(&app.package_name), + lua_string(&app.package_name), + ) +} + +fn fdroid_update_lua(endpoint_url: &str, release: &FdroidRelease) -> String { + let version_code = release + .version_code + .map(|value| format!("\n version_code = {value},")) + .unwrap_or_default(); + let sha256 = release + .sha256 + .as_deref() + .map(|value| format!("\n sha256 = {},", lua_string(value))) + .unwrap_or_default(); + let size = release + .size + .map(|value| format!("\n size = {value},")) + .unwrap_or_default(); + format!( + "\n {{\n version = {},{version_code}\n source = \"fdroid\",\n artifacts = {{\n {{\n name = {},\n url = {},\n file_name = {},{sha256}{size}\n }},\n }},\n }}", + lua_string(&release.version), + lua_string(&release.apk_name), + lua_string(&fdroid_artifact_url(endpoint_url, &release.apk_name)), + lua_string(&release.apk_name), + ) +} + +fn fdroid_artifact_url(endpoint_url: &str, apk_name: &str) -> String { + format!("{}/{}", endpoint_url.trim_end_matches('/'), apk_name) +} + +fn render_pretty_json(value: Value) -> AutogenOperationResult { + serde_json::to_string_pretty(&value) + .map(|mut content| { + content.push('\n'); + content + }) + .map_err(|source| AutogenOperationError::Autogen(source.to_string())) +} + +fn generated_file(relative_path: &str, content: String) -> GeneratedPackageFile { + GeneratedPackageFile { + relative_path: PathBuf::from(relative_path), + content_hash: content_hash(&content), + content, + } +} + +fn lua_string(value: &str) -> String { + let mut escaped = String::from("\""); + for ch in value.chars() { + match ch { + '\\' => escaped.push_str("\\\\"), + '"' => escaped.push_str("\\\""), + '\n' => escaped.push_str("\\n"), + '\r' => escaped.push_str("\\r"), + '\t' => escaped.push_str("\\t"), + other => escaped.push(other), + } + } + escaped.push('"'); + escaped +} + +#[cfg(test)] +mod tests { + use super::*; + use getter_core::lua::evaluate_package_directory_script; + use getter_core::repository::{ + RepositoryMetadata, RepositoryPackageDirectoryLayout, REPO_API_VERSION_V1, + }; + use getter_core::RepositoryPriority; + + const FDROID_FIXTURE: &str = r#" + + + + F-Droid + App repository client + + 1.20.0 + 1020000 + org.fdroid.fdroid_1020000.apk + aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa + 1234567 + + + 1.19.0 + 1019000 + org.fdroid.fdroid_1019000.apk + bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb + 1111111 + + + + Other + + 2.0 + org.example.other_2.apk + + + +"#; + + #[test] + fn preview_generates_fdroid_package_directory_from_cached_catalog() { + let temp = tempfile::tempdir().unwrap(); + let main_db = MainDb::open(temp.path().join("main.db")).unwrap(); + let cache_db = CacheDb::open(temp.path().join("cache.db")).unwrap(); + + let preview = preview_fdroid_packages_json( + temp.path(), + &main_db, + &cache_db, + &json!({ + "index_xml": FDROID_FIXTURE, + "package_names": ["org.fdroid.fdroid", "missing.package"] + }) + .to_string(), + ) + .unwrap(); + + assert_eq!(preview["operation"], "fdroid.autogen.preview"); + assert_eq!(preview["target_repo_id"], "autogen"); + assert_eq!(preview["source"], "refreshed"); + assert_eq!(preview["candidates"].as_array().unwrap().len(), 1); + let candidate = &preview["candidates"][0]; + assert_eq!( + candidate["package_id"], + "android/f-droid/app/org.fdroid.fdroid" + ); + assert_eq!( + candidate["output_relative_path"], + "android/f-droid/app/org.fdroid.fdroid" + ); + let record: Value = + serde_json::from_str(candidate["autogen_record_content"].as_str().unwrap()).unwrap(); + assert_eq!(record["generator"], "fdroid-catalog"); + assert_eq!(record["input"]["kind"], "fdroid_package"); + assert_eq!(record["input"]["endpoint_id"], "official"); + assert_eq!(record["input"]["endpoint_url"], "https://f-droid.org/repo"); + assert_eq!(record["input"]["package_name"], "org.fdroid.fdroid"); + let files = candidate["files"].as_array().unwrap(); + assert!(files.iter().any(|file| file["relative_path"] == "9999.lua" + && file["content"] + .as_str() + .unwrap() + .contains("version_code = 1020000"))); + assert!(files.iter().any(|file| file["relative_path"] == "9999.lua" + && file["content"] + .as_str() + .unwrap() + .contains("return fdroid.package"))); + assert!(files.iter().any(|file| file["relative_path"] == "9999.lua" + && file["content"] + .as_str() + .unwrap() + .contains("package_name = \"org.fdroid.fdroid\""))); + assert!(files.iter().any(|file| file["relative_path"] == "9999.lua" + && file["content"] + .as_str() + .unwrap() + .contains("https://f-droid.org/repo/org.fdroid.fdroid_1020000.apk"))); + assert!(files.iter().any(|file| file["relative_path"] == "Manifest" + && file["content"].as_str().unwrap().is_empty())); + assert_eq!( + preview["skipped"][0]["reason"], + "provider_package_not_found" + ); + assert_eq!( + preview["diagnostics"][0]["code"], + "provider.fdroid.package_not_found" + ); + assert!(!temp.path().join("repo/autogen").exists()); + } + + #[test] + fn preview_skips_fdroid_package_covered_by_higher_priority_repo() { + let temp = tempfile::tempdir().unwrap(); + let main_db = MainDb::open(temp.path().join("main.db")).unwrap(); + let cache_db = CacheDb::open(temp.path().join("cache.db")).unwrap(); + let official_root = temp.path().join("repo/official"); + let package_dir = official_root.join("android/f-droid/app/org.fdroid.fdroid"); + std::fs::create_dir_all(&package_dir).unwrap(); + std::fs::write( + package_dir.join("metadata.jsonc"), + r#"{ "type": "android:app", "android": { "package_name": "org.fdroid.fdroid" } }"#, + ) + .unwrap(); + main_db + .upsert_repository( + &RepositoryMetadata { + id: "official".parse().unwrap(), + name: "Official".to_owned(), + priority: RepositoryPriority::DEFAULT, + api_version: REPO_API_VERSION_V1.to_owned(), + }, + Some(&official_root), + None, + ) + .unwrap(); + + let preview = preview_fdroid_packages_json( + temp.path(), + &main_db, + &cache_db, + &json!({ + "index_xml": FDROID_FIXTURE, + "package_names": ["org.fdroid.fdroid"] + }) + .to_string(), + ) + .unwrap(); + + assert!(preview["candidates"].as_array().unwrap().is_empty()); + assert_eq!( + preview["skipped"][0]["package_id"], + "android/f-droid/app/org.fdroid.fdroid" + ); + assert_eq!( + preview["skipped"][0]["reason"], + "covered_by_higher_priority_repo" + ); + assert_eq!(preview["skipped"][0]["covering_repo_id"], "official"); + } + + #[test] + fn apply_writes_valid_fdroid_package_directory() { + let temp = tempfile::tempdir().unwrap(); + let main_db = MainDb::open(temp.path().join("main.db")).unwrap(); + let cache_db = CacheDb::open(temp.path().join("cache.db")).unwrap(); + let preview = preview_fdroid_packages_json( + temp.path(), + &main_db, + &cache_db, + &json!({ + "index_xml": FDROID_FIXTURE, + "package_names": ["org.fdroid.fdroid"] + }) + .to_string(), + ) + .unwrap(); + + let result = apply_fdroid_preview_json( + temp.path(), + &main_db, + &preview, + &AutogenAcceptance::AcceptAll, + ) + .unwrap(); + + assert_eq!(result["applied_count"], 1); + let repo_root = temp.path().join("repo/autogen"); + let package_dir = repo_root.join("android/f-droid/app/org.fdroid.fdroid"); + assert!(package_dir.join("metadata.jsonc").is_file()); + assert!(package_dir.join("Manifest").is_file()); + assert!(package_dir.join("9999.lua").is_file()); + assert!(package_dir.join(".autogen.jsonc").is_file()); + let layout = RepositoryPackageDirectoryLayout::load(&repo_root).unwrap(); + let package_dir = layout + .package(&"android/f-droid/app/org.fdroid.fdroid".parse().unwrap()) + .unwrap(); + let metadata = layout.package_metadata(package_dir).unwrap(); + let script = layout.unambiguous_version_script(package_dir).unwrap(); + let package = evaluate_package_directory_script( + &"autogen".parse().unwrap(), + package_dir, + &metadata, + script, + ) + .unwrap(); + assert_eq!(package.name, "android/f-droid/app/org.fdroid.fdroid"); + assert_eq!(package.installed.len(), 1); + assert_eq!(package.updates.len(), 2); + assert_eq!(package.updates[0].version_code, Some(1020000)); + assert_eq!( + package.updates[0].artifacts[0].sha256.as_deref(), + Some("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa") + ); + } + + #[test] + fn apply_rejects_existing_fdroid_directory_owned_by_another_generator() { + let temp = tempfile::tempdir().unwrap(); + let main_db = MainDb::open(temp.path().join("main.db")).unwrap(); + let cache_db = CacheDb::open(temp.path().join("cache.db")).unwrap(); + let preview = preview_fdroid_packages_json( + temp.path(), + &main_db, + &cache_db, + &json!({ + "index_xml": FDROID_FIXTURE, + "package_names": ["org.fdroid.fdroid"] + }) + .to_string(), + ) + .unwrap(); + apply_fdroid_preview_json( + temp.path(), + &main_db, + &preview, + &AutogenAcceptance::AcceptAll, + ) + .unwrap(); + let record_path = temp + .path() + .join("repo/autogen/android/f-droid/app/org.fdroid.fdroid/.autogen.jsonc"); + let mut record: Value = + serde_json::from_str(&std::fs::read_to_string(&record_path).unwrap()) + .expect("record JSON parses"); + record["generator"] = Value::String("installed-inventory".to_owned()); + std::fs::write( + &record_path, + serde_json::to_string_pretty(&record).unwrap() + "\n", + ) + .unwrap(); + + let error = apply_fdroid_preview_json( + temp.path(), + &main_db, + &preview, + &AutogenAcceptance::AcceptAll, + ) + .unwrap_err(); + + assert!( + matches!(error, AutogenOperationError::Autogen(detail) if detail.contains("does not match 'fdroid-catalog'")) + ); + } + + #[test] + fn apply_rejects_existing_fdroid_directory_without_ownership() { + let temp = tempfile::tempdir().unwrap(); + let main_db = MainDb::open(temp.path().join("main.db")).unwrap(); + let cache_db = CacheDb::open(temp.path().join("cache.db")).unwrap(); + let existing = temp + .path() + .join("repo/autogen/android/f-droid/app/org.fdroid.fdroid"); + std::fs::create_dir_all(&existing).unwrap(); + std::fs::write( + existing.join("metadata.jsonc"), + r#"{ "type": "android:app", "android": { "package_name": "org.fdroid.fdroid" } }"#, + ) + .unwrap(); + let preview = preview_fdroid_packages_json( + temp.path(), + &main_db, + &cache_db, + &json!({ + "index_xml": FDROID_FIXTURE, + "package_names": ["org.fdroid.fdroid"] + }) + .to_string(), + ) + .unwrap(); + + let error = apply_fdroid_preview_json( + temp.path(), + &main_db, + &preview, + &AutogenAcceptance::AcceptAll, + ) + .unwrap_err(); + + assert!( + matches!(error, AutogenOperationError::Autogen(detail) if detail.contains("missing .autogen.jsonc")) + ); + } +} diff --git a/crates/getter-operations/src/lib.rs b/crates/getter-operations/src/lib.rs index ffe59eb..ba9281b 100644 --- a/crates/getter-operations/src/lib.rs +++ b/crates/getter-operations/src/lib.rs @@ -6,6 +6,7 @@ //! generated package files, manifests, and tracked state live here. pub mod autogen; +pub mod fdroid_autogen; pub mod fdroid_catalog; pub mod legacy_room; pub mod provider_cache; From 6d91ce50f3c5ae41a70633f7a1798cbf64c564e3 Mon Sep 17 00:00:00 2001 From: Xiangzhe Date: Sat, 27 Jun 2026 11:25:32 +0800 Subject: [PATCH 39/52] feat(autogen): match F-Droid installed inventory --- crates/getter-cli/src/lib.rs | 33 +++++-- crates/getter-cli/tests/bdd_cli.rs | 20 +++++ .../features/cli/autogen_installed.feature | 10 +++ .../getter-operations/src/fdroid_autogen.rs | 88 +++++++++++++++++-- 4 files changed, 138 insertions(+), 13 deletions(-) diff --git a/crates/getter-cli/src/lib.rs b/crates/getter-cli/src/lib.rs index 1f3d406..730767b 100644 --- a/crates/getter-cli/src/lib.rs +++ b/crates/getter-cli/src/lib.rs @@ -129,6 +129,7 @@ pub enum CliCommand { }, AutogenFdroidPreview { index: PathBuf, + inventory: Option, package_names: Vec, }, AutogenFdroidApply { @@ -538,9 +539,10 @@ where [domain, subject, action, rest @ ..] if domain == "autogen" && subject == "fdroid" && action == "preview" => { - let (index, package_names) = parse_fdroid_autogen_preview_args(rest)?; + let (index, inventory, package_names) = parse_fdroid_autogen_preview_args(rest)?; CliCommand::AutogenFdroidPreview { index, + inventory, package_names, } } @@ -758,14 +760,20 @@ fn execute(invocation: CliInvocation) -> Result { } CliCommand::AutogenFdroidPreview { index, + inventory, package_names, } => { let db = open_main_db(&invocation.data_dir)?; let cache_db = open_cache_db(&invocation.data_dir)?; let index_xml = read_fdroid_index(&index)?; + let installed_inventory = inventory + .as_deref() + .map(read_installed_inventory) + .transpose()?; let request = json!({ "index_xml": index_xml, "package_names": package_names, + "installed_inventory": installed_inventory, }); fdroid_autogen::preview_fdroid_packages_json( &invocation.data_dir, @@ -961,8 +969,11 @@ fn parse_priority(value: &str) -> Result { .map_err(|source| CliError::Usage(format!("invalid repository priority: {source}"))) } -fn parse_fdroid_autogen_preview_args(args: &[String]) -> Result<(PathBuf, Vec), CliError> { +fn parse_fdroid_autogen_preview_args( + args: &[String], +) -> Result<(PathBuf, Option, Vec), CliError> { let mut index = None; + let mut inventory = None; let mut package_names = Vec::new(); let mut position = 0; while position < args.len() { @@ -974,6 +985,15 @@ fn parse_fdroid_autogen_preview_args(args: &[String]) -> Result<(PathBuf, Vec { + let path = args.get(position + 1).ok_or_else(|| { + CliError::Usage( + "autogen fdroid preview --inventory requires an inventory path".to_owned(), + ) + })?; + inventory = Some(PathBuf::from(path)); + position += 2; + } "--package" => { let package_name = args.get(position + 1).ok_or_else(|| { CliError::Usage( @@ -993,12 +1013,13 @@ fn parse_fdroid_autogen_preview_args(args: &[String]) -> Result<(PathBuf, Vec".to_owned()) })?; - if package_names.is_empty() { + if package_names.is_empty() && inventory.is_none() { return Err(CliError::Usage( - "autogen fdroid preview requires at least one --package ".to_owned(), + "autogen fdroid preview requires --package or --inventory " + .to_owned(), )); } - Ok((index, package_names)) + Ok((index, inventory, package_names)) } fn parse_autogen_acceptance(args: &[String]) -> Result { @@ -1706,7 +1727,7 @@ fn envelope_to_string(value: Value) -> String { } fn usage_text() -> String { - "Usage: getter --data-dir [--priority ]|repo eval |repo validate |package eval [--repo ]|storage validate|version pin |version unpin |hub list|update check --fixture |runtime script --script |debug fake-task submit --request |debug fake-task run |debug fake-task list|debug fake-task cancel |debug fake-task events --after --limit |debug fake-task install-result --status |autogen installed preview --inventory |autogen installed apply --preview (--accept-all|--accept ...)|autogen fdroid preview --index --package [--package ...]|autogen fdroid apply --preview (--accept-all|--accept ...)|autogen cleanup preview --inventory |autogen cleanup apply --preview (--accept-all|--accept ...)|legacy import-room-bundle |legacy import-room-db |legacy report-list>\nNote: `debug fake-task` commands are persisted fake-download scaffolding. ADR-0011 runtime task debugging uses `runtime script` and does not preserve task state across CLI invocations.\n".to_owned() + "Usage: getter --data-dir [--priority ]|repo eval |repo validate |package eval [--repo ]|storage validate|version pin |version unpin |hub list|update check --fixture |runtime script --script |debug fake-task submit --request |debug fake-task run |debug fake-task list|debug fake-task cancel |debug fake-task events --after --limit |debug fake-task install-result --status |autogen installed preview --inventory |autogen installed apply --preview (--accept-all|--accept ...)|autogen fdroid preview --index [--package ...] [--inventory ]|autogen fdroid apply --preview (--accept-all|--accept ...)|autogen cleanup preview --inventory |autogen cleanup apply --preview (--accept-all|--accept ...)|legacy import-room-bundle |legacy import-room-db |legacy report-list>\nNote: `debug fake-task` commands are persisted fake-download scaffolding. ADR-0011 runtime task debugging uses `runtime script` and does not preserve task state across CLI invocations.\n".to_owned() } #[derive(Debug, Deserialize)] diff --git a/crates/getter-cli/tests/bdd_cli.rs b/crates/getter-cli/tests/bdd_cli.rs index 5fc72ac..6e82d68 100644 --- a/crates/getter-cli/tests/bdd_cli.rs +++ b/crates/getter-cli/tests/bdd_cli.rs @@ -873,6 +873,26 @@ fn run_getter_autogen_fdroid_preview(world: &mut CliWorld, package_name: String) world.json = None; } +#[when("I run getter autogen fdroid preview for that inventory")] +fn run_getter_autogen_fdroid_preview_for_inventory(world: &mut CliWorld) { + let index = world.fdroid_index.as_ref().expect("F-Droid index exists"); + let inventory = world.inventory.as_ref().expect("inventory exists"); + let output = run_getter( + world, + [ + "autogen".to_owned(), + "fdroid".to_owned(), + "preview".to_owned(), + "--index".to_owned(), + index.to_string_lossy().to_string(), + "--inventory".to_owned(), + inventory.to_string_lossy().to_string(), + ], + ); + world.output = Some(output); + world.json = None; +} + #[when("I run getter autogen fdroid apply for that preview with accept-all")] fn run_getter_autogen_fdroid_apply_accept_all(world: &mut CliWorld) { let preview = world diff --git a/crates/getter-cli/tests/features/cli/autogen_installed.feature b/crates/getter-cli/tests/features/cli/autogen_installed.feature index 50f6b6a..26ce9c3 100644 --- a/crates/getter-cli/tests/features/cli/autogen_installed.feature +++ b/crates/getter-cli/tests/features/cli/autogen_installed.feature @@ -25,6 +25,16 @@ Feature: Installed app autogen Then the command succeeds And the package eval contains update version_code 1020000 + Scenario: User previews F-Droid autogen from installed inventory + Given an initialized getter data directory + And a fixture F-Droid catalog index with package "org.fdroid.fdroid" + And an installed inventory with Android app "org.fdroid.fdroid" labeled "F-Droid" + When I run getter autogen fdroid preview for that inventory + Then the command succeeds + And the output is valid JSON + And the F-Droid autogen preview contains candidate "android/f-droid/app/org.fdroid.fdroid" + And the autogen repository has not been written + Scenario: Higher-priority repositories suppress explicit F-Droid autogen candidates Given an initialized getter data directory And a package-directory repository "official" with package "android/f-droid/app/org.fdroid.fdroid" diff --git a/crates/getter-operations/src/fdroid_autogen.rs b/crates/getter-operations/src/fdroid_autogen.rs index 2968d1f..3026de4 100644 --- a/crates/getter-operations/src/fdroid_autogen.rs +++ b/crates/getter-operations/src/fdroid_autogen.rs @@ -13,8 +13,9 @@ use crate::fdroid_catalog::{ }; use crate::provider_cache::ProviderCacheMode; use getter_core::autogen::{ - content_hash, package_relative_path, record_file_key, render_autogen_record, AutogenRecord, - AutogenRecordInput, GeneratedPackageFile, AUTOGEN_RECORD_VERSION, FDROID_AUTOGEN_GENERATOR, + content_hash, package_relative_path, record_file_key, render_autogen_record, + validate_installed_inventory, AutogenRecord, AutogenRecordInput, GeneratedPackageFile, + InstalledInventory, InstalledInventoryItem, AUTOGEN_RECORD_VERSION, FDROID_AUTOGEN_GENERATOR, }; use getter_core::{InstalledTarget, PackageId, PackageKind}; use getter_providers::{FdroidApp, FdroidRelease}; @@ -36,7 +37,9 @@ pub fn preview_fdroid_packages_json( )) })?; let endpoint = request.endpoint_config(); - let catalog = read_or_refresh_fdroid_catalog(cache_db, endpoint, request.cache_mode()?, || { + let mode = request.cache_mode()?; + let requested_package_names = requested_fdroid_package_names(&request)?; + let catalog = read_or_refresh_fdroid_catalog(cache_db, endpoint, mode, || { request .index_xml .ok_or_else(|| "fixture-backed F-Droid autogen refresh requires index_xml".to_owned()) @@ -62,7 +65,7 @@ pub fn preview_fdroid_packages_json( }) .collect(); - for package_name in unique_requested_packages(&request.package_names) { + for package_name in requested_package_names { let package_id = fdroid_package_id(&package_name)?; if let Some(repository_id) = covered.get(&package_id) { skipped.push(json!({ @@ -148,6 +151,8 @@ struct FdroidAutogenPreviewRequest { index_xml: Option, #[serde(default)] package_names: Vec, + #[serde(default)] + installed_inventory: Option, } impl FdroidAutogenPreviewRequest { @@ -175,16 +180,32 @@ impl FdroidAutogenPreviewRequest { } } -fn unique_requested_packages(package_names: &[String]) -> Vec { - let mut packages = package_names +fn requested_fdroid_package_names( + request: &FdroidAutogenPreviewRequest, +) -> AutogenOperationResult> { + if let Some(inventory) = &request.installed_inventory { + validate_installed_inventory(inventory) + .map_err(|source| AutogenOperationError::Autogen(source.to_string()))?; + } + let mut packages = request + .package_names .iter() .map(|package| package.trim()) .filter(|package| !package.is_empty()) .map(str::to_owned) .collect::>(); + if let Some(inventory) = &request.installed_inventory { + packages.extend(inventory.items.iter().filter_map(|item| match item { + InstalledInventoryItem::AndroidPackage { package_name, .. } => { + let package_name = package_name.trim(); + (!package_name.is_empty()).then(|| package_name.to_owned()) + } + InstalledInventoryItem::MagiskModule { .. } => None, + })); + } packages.sort(); packages.dedup(); - packages + Ok(packages) } fn fdroid_package_id(package_name: &str) -> AutogenOperationResult { @@ -658,4 +679,57 @@ mod tests { matches!(error, AutogenOperationError::Autogen(detail) if detail.contains("missing .autogen.jsonc")) ); } + + #[test] + fn preview_matches_installed_android_inventory_against_fdroid_catalog() { + let temp = tempfile::tempdir().unwrap(); + let main_db = MainDb::open(temp.path().join("main.db")).unwrap(); + let cache_db = CacheDb::open(temp.path().join("cache.db")).unwrap(); + + let preview = preview_fdroid_packages_json( + temp.path(), + &main_db, + &cache_db, + &json!({ + "index_xml": FDROID_FIXTURE, + "installed_inventory": { + "format": "upgradeall-installed-inventory", + "version": 1, + "items": [ + { + "kind": "android", + "package_name": "org.fdroid.fdroid", + "label": "F-Droid" + }, + { + "kind": "android", + "package_name": "missing.package", + "label": "Missing" + }, + { + "kind": "magisk", + "module_id": "zygisk-next" + } + ] + } + }) + .to_string(), + ) + .unwrap(); + + assert_eq!(preview["operation"], "fdroid.autogen.preview"); + assert_eq!(preview["candidates"].as_array().unwrap().len(), 1); + assert_eq!( + preview["candidates"][0]["package_id"], + "android/f-droid/app/org.fdroid.fdroid" + ); + assert_eq!( + preview["skipped"][0]["package_id"], + "android/f-droid/app/missing.package" + ); + assert_eq!( + preview["skipped"][0]["reason"], + "provider_package_not_found" + ); + } } From 9001bb93b23245c7b2347a722157fffd18f64ca6 Mon Sep 17 00:00:00 2001 From: Xiangzhe Date: Sat, 27 Jun 2026 12:31:03 +0800 Subject: [PATCH 40/52] feat(providers): add GitHub release fixtures --- Cargo.lock | 2 + crates/getter-cli/src/lib.rs | 189 +++++++- crates/getter-cli/tests/bdd_cli.rs | 123 +++++ .../cli/provider_github_releases.feature | 10 + .../getter-operations/src/github_releases.rs | 438 ++++++++++++++++++ crates/getter-operations/src/lib.rs | 1 + crates/getter-providers/Cargo.toml | 2 + crates/getter-providers/src/lib.rs | 309 ++++++++++++ 8 files changed, 1071 insertions(+), 3 deletions(-) create mode 100644 crates/getter-cli/tests/features/cli/provider_github_releases.feature create mode 100644 crates/getter-operations/src/github_releases.rs diff --git a/Cargo.lock b/Cargo.lock index 039770c..c072fcc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -637,8 +637,10 @@ name = "getter-providers" version = "0.1.0" dependencies = [ "getter-core", + "regex", "roxmltree", "serde", + "serde_json", "thiserror 1.0.69", ] diff --git a/crates/getter-cli/src/lib.rs b/crates/getter-cli/src/lib.rs index 730767b..8306d7c 100644 --- a/crates/getter-cli/src/lib.rs +++ b/crates/getter-cli/src/lib.rs @@ -23,6 +23,7 @@ use getter_downloader::{ }; use getter_operations::autogen::{self, AutogenAcceptance, AutogenOperationError}; use getter_operations::fdroid_autogen; +use getter_operations::github_releases::{self, GithubReleaseOperationError}; use getter_operations::legacy_room::{self, LegacyRoomOperationError}; use getter_operations::runtime as runtime_operations; use getter_storage::legacy_room::{ @@ -136,6 +137,15 @@ pub enum CliCommand { preview: PathBuf, acceptance: AutogenAcceptance, }, + ProviderGithubReleases { + owner: String, + repo: String, + releases: PathBuf, + asset_include: Option, + asset_exclude: Option, + include_prereleases: bool, + refresh: bool, + }, LegacyImportRoomBundle { bundle: PathBuf, }, @@ -152,6 +162,7 @@ pub enum ExitCode { Usage = 2, Storage = 10, Migration = 20, + Provider = 30, Download = 40, } @@ -179,6 +190,8 @@ pub enum CliError { Runtime(String), #[error("autogen error: {0}")] Autogen(String), + #[error("provider error: {0}")] + Provider(String), #[error("Legacy Room export bundle is invalid")] InvalidLegacyBundle { report_path: PathBuf }, #[error("Legacy Room bundle import is not implemented yet")] @@ -199,6 +212,7 @@ impl CliError { | Self::Update(_) | Self::Runtime(_) | Self::Autogen(_) => ExitCode::GenericFailure, + Self::Provider(_) => ExitCode::Provider, Self::Download(_) => ExitCode::Download, Self::InvalidLegacyBundle { .. } | Self::UnsupportedLegacyBundle { .. } @@ -217,6 +231,7 @@ impl CliError { Self::Download(_) => "download.task_error", Self::Runtime(_) => "runtime.error", Self::Autogen(_) => "autogen.error", + Self::Provider(_) => "provider.error", Self::InvalidLegacyBundle { .. } => "migration.invalid_bundle", Self::UnsupportedLegacyBundle { .. } => "migration.unsupported_bundle", Self::InvalidLegacyDb { .. } => "migration.invalid_db", @@ -234,6 +249,7 @@ impl CliError { Self::Download(_) => "Getter download task operation failed", Self::Runtime(_) => "Getter runtime operation failed", Self::Autogen(_) => "Getter autogen operation failed", + Self::Provider(_) => "Getter provider operation failed", Self::InvalidLegacyBundle { .. } => "Legacy Room export bundle is invalid", Self::UnsupportedLegacyBundle { .. } => { "Legacy Room bundle import is not implemented yet" @@ -252,7 +268,8 @@ impl CliError { | Self::Update(detail) | Self::Download(detail) | Self::Runtime(detail) - | Self::Autogen(detail) => Some(detail.as_str()), + | Self::Autogen(detail) + | Self::Provider(detail) => Some(detail.as_str()), Self::InvalidLegacyBundle { .. } | Self::UnsupportedLegacyBundle { .. } | Self::InvalidLegacyDb { .. } @@ -273,7 +290,8 @@ impl CliError { | Self::Update(_) | Self::Download(_) | Self::Runtime(_) - | Self::Autogen(_) => None, + | Self::Autogen(_) + | Self::Provider(_) => None, } } } @@ -311,6 +329,16 @@ impl From for CliError { } } +impl From for CliError { + fn from(value: GithubReleaseOperationError) -> Self { + match value { + GithubReleaseOperationError::Storage(source) => Self::Storage(source.to_string()), + GithubReleaseOperationError::InvalidRequest(detail) => Self::Usage(detail), + other => Self::Provider(other.to_string()), + } + } +} + #[derive(Debug, Clone, PartialEq, Eq)] pub struct CliOutput { pub exit_code: ExitCode, @@ -557,6 +585,20 @@ where acceptance: parse_autogen_acceptance(rest)?, } } + [domain, provider, action, rest @ ..] + if domain == "provider" && provider == "github" && action == "releases" => + { + let args = parse_provider_github_releases_args(rest)?; + CliCommand::ProviderGithubReleases { + owner: args.owner, + repo: args.repo, + releases: args.releases, + asset_include: args.asset_include, + asset_exclude: args.asset_exclude, + include_prereleases: args.include_prereleases, + refresh: args.refresh, + } + } [domain, subject, action, flag, preview, rest @ ..] if domain == "autogen" && subject == "cleanup" @@ -797,6 +839,30 @@ fn execute(invocation: CliInvocation) -> Result { ) .map_err(CliError::from) } + CliCommand::ProviderGithubReleases { + owner, + repo, + releases, + asset_include, + asset_exclude, + include_prereleases, + refresh, + } => { + let db = open_cache_db(&invocation.data_dir)?; + let releases_json = read_github_releases_fixture(&releases)?; + let request = json!({ + "owner": owner, + "repo": repo, + "mode": if refresh { "force_refresh" } else { "use_cached" }, + "releases_json": releases_json, + "include_prereleases": include_prereleases, + "asset": { + "include": asset_include, + "exclude": asset_exclude, + }, + }); + github_releases::github_releases_json(&db, &request.to_string()).map_err(CliError::from) + } CliCommand::LegacyImportRoomBundle { bundle } => { let db = open_main_db(&invocation.data_dir)?; if db.migration_record_exists(LEGACY_ROOM_MIGRATION_ID)? { @@ -1022,6 +1088,116 @@ fn parse_fdroid_autogen_preview_args( Ok((index, inventory, package_names)) } +#[derive(Debug, Default, PartialEq, Eq)] +struct ProviderGithubReleasesArgs { + owner: String, + repo: String, + releases: PathBuf, + asset_include: Option, + asset_exclude: Option, + include_prereleases: bool, + refresh: bool, +} + +fn parse_provider_github_releases_args( + args: &[String], +) -> Result { + let mut parsed = ProviderGithubReleasesArgs::default(); + let mut position = 0; + while position < args.len() { + match args[position].as_str() { + "--owner" => { + parsed.owner = args + .get(position + 1) + .ok_or_else(|| { + CliError::Usage( + "provider github releases --owner requires an owner".to_owned(), + ) + })? + .clone(); + position += 2; + } + "--repo" => { + parsed.repo = args + .get(position + 1) + .ok_or_else(|| { + CliError::Usage( + "provider github releases --repo requires a repo".to_owned(), + ) + })? + .clone(); + position += 2; + } + "--releases" => { + let path = args.get(position + 1).ok_or_else(|| { + CliError::Usage( + "provider github releases --releases requires a fixture path".to_owned(), + ) + })?; + parsed.releases = PathBuf::from(path); + position += 2; + } + "--asset-include" => { + parsed.asset_include = Some( + args.get(position + 1) + .ok_or_else(|| { + CliError::Usage( + "provider github releases --asset-include requires a regex" + .to_owned(), + ) + })? + .clone(), + ); + position += 2; + } + "--asset-exclude" => { + parsed.asset_exclude = Some( + args.get(position + 1) + .ok_or_else(|| { + CliError::Usage( + "provider github releases --asset-exclude requires a regex" + .to_owned(), + ) + })? + .clone(), + ); + position += 2; + } + "--include-prereleases" => { + parsed.include_prereleases = true; + position += 1; + } + "--refresh" => { + parsed.refresh = true; + position += 1; + } + other => { + return Err(CliError::Usage(format!( + "unsupported provider github releases argument '{other}'" + ))) + } + } + } + + if parsed.owner.trim().is_empty() { + return Err(CliError::Usage( + "provider github releases requires --owner ".to_owned(), + )); + } + if parsed.repo.trim().is_empty() { + return Err(CliError::Usage( + "provider github releases requires --repo ".to_owned(), + )); + } + if parsed.releases.as_os_str().is_empty() { + return Err(CliError::Usage( + "provider github releases requires --releases ".to_owned(), + )); + } + + Ok(parsed) +} + fn parse_autogen_acceptance(args: &[String]) -> Result { match args { [flag] if flag == "--accept-all" => Ok(AutogenAcceptance::AcceptAll), @@ -1198,6 +1374,12 @@ fn read_fdroid_index(path: &Path) -> Result { .map_err(|source| CliError::Autogen(format!("failed to read F-Droid index: {source}"))) } +fn read_github_releases_fixture(path: &Path) -> Result { + fs::read_to_string(path).map_err(|source| { + CliError::Provider(format!("failed to read GitHub releases fixture: {source}")) + }) +} + fn read_installed_inventory(path: &Path) -> Result { let bytes = fs::read(path) .map_err(|source| CliError::Autogen(format!("failed to read inventory: {source}")))?; @@ -1727,7 +1909,7 @@ fn envelope_to_string(value: Value) -> String { } fn usage_text() -> String { - "Usage: getter --data-dir [--priority ]|repo eval |repo validate |package eval [--repo ]|storage validate|version pin |version unpin |hub list|update check --fixture |runtime script --script |debug fake-task submit --request |debug fake-task run |debug fake-task list|debug fake-task cancel |debug fake-task events --after --limit |debug fake-task install-result --status |autogen installed preview --inventory |autogen installed apply --preview (--accept-all|--accept ...)|autogen fdroid preview --index [--package ...] [--inventory ]|autogen fdroid apply --preview (--accept-all|--accept ...)|autogen cleanup preview --inventory |autogen cleanup apply --preview (--accept-all|--accept ...)|legacy import-room-bundle |legacy import-room-db |legacy report-list>\nNote: `debug fake-task` commands are persisted fake-download scaffolding. ADR-0011 runtime task debugging uses `runtime script` and does not preserve task state across CLI invocations.\n".to_owned() + "Usage: getter --data-dir [--priority ]|repo eval |repo validate |package eval [--repo ]|storage validate|version pin |version unpin |hub list|update check --fixture |runtime script --script |debug fake-task submit --request |debug fake-task run |debug fake-task list|debug fake-task cancel |debug fake-task events --after --limit |debug fake-task install-result --status |autogen installed preview --inventory |autogen installed apply --preview (--accept-all|--accept ...)|autogen fdroid preview --index [--package ...] [--inventory ]|autogen fdroid apply --preview (--accept-all|--accept ...)|provider github releases --owner --repo --releases [--asset-include ] [--asset-exclude ] [--include-prereleases] [--refresh]|autogen cleanup preview --inventory |autogen cleanup apply --preview (--accept-all|--accept ...)|legacy import-room-bundle |legacy import-room-db |legacy report-list>\nNote: `debug fake-task` commands are persisted fake-download scaffolding. ADR-0011 runtime task debugging uses `runtime script` and does not preserve task state across CLI invocations.\n".to_owned() } #[derive(Debug, Deserialize)] @@ -1796,6 +1978,7 @@ impl CliCommand { Self::AutogenCleanupApply { .. } => "autogen cleanup apply", Self::AutogenFdroidPreview { .. } => "autogen fdroid preview", Self::AutogenFdroidApply { .. } => "autogen fdroid apply", + Self::ProviderGithubReleases { .. } => "provider github releases", Self::LegacyImportRoomBundle { .. } => "legacy import-room-bundle", Self::LegacyImportRoomDb { .. } => "legacy import-room-db", Self::LegacyReportList => "legacy report-list", diff --git a/crates/getter-cli/tests/bdd_cli.rs b/crates/getter-cli/tests/bdd_cli.rs index 6e82d68..54b584a 100644 --- a/crates/getter-cli/tests/bdd_cli.rs +++ b/crates/getter-cli/tests/bdd_cli.rs @@ -16,6 +16,7 @@ struct CliWorld { autogen_preview: Option, update_fixture: Option, fdroid_index: Option, + github_releases: Option, task_request: Option, runtime_script: Option, remembered_task_id: Option, @@ -172,6 +173,73 @@ fn fixture_fdroid_catalog_index_with_package(world: &mut CliWorld, package_name: world.fdroid_index = Some(index); } +#[given(expr = "a fixture GitHub releases response for {string}")] +fn fixture_github_releases_response(world: &mut CliWorld, project: String) { + let temp = world.temp.as_ref().expect("tempdir exists"); + let releases = temp.path().join("github-releases.json"); + fs::write( + &releases, + serde_json::to_vec_pretty(&serde_json::json!([ + { + "tag_name": "v1.2.0", + "name": format!("{project} 1.2.0"), + "body": "Release notes", + "draft": false, + "prerelease": false, + "published_at": "2026-06-01T00:00:00Z", + "assets": [ + { + "name": "app-release.apk", + "content_type": "application/vnd.android.package-archive", + "size": 1234, + "browser_download_url": "https://github.com/DUpdateSystem/UpgradeAll/releases/download/v1.2.0/app-release.apk", + "digest": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + }, + { + "name": "app-debug.apk", + "content_type": "application/vnd.android.package-archive", + "size": 2345, + "browser_download_url": "https://github.com/DUpdateSystem/UpgradeAll/releases/download/v1.2.0/app-debug.apk" + }, + { + "name": "notes.txt", + "content_type": "text/plain", + "size": 345, + "browser_download_url": "https://github.com/DUpdateSystem/UpgradeAll/releases/download/v1.2.0/notes.txt" + } + ] + }, + { + "tag_name": "v1.3.0-beta1", + "draft": false, + "prerelease": true, + "assets": [ + { + "name": "app-beta.apk", + "size": 456, + "browser_download_url": "https://github.com/DUpdateSystem/UpgradeAll/releases/download/v1.3.0-beta1/app-beta.apk" + } + ] + }, + { + "tag_name": "v1.4.0-draft", + "draft": true, + "prerelease": false, + "assets": [ + { + "name": "app-draft.apk", + "size": 567, + "browser_download_url": "https://github.com/DUpdateSystem/UpgradeAll/releases/download/v1.4.0-draft/app-draft.apk" + } + ] + } + ])) + .expect("GitHub releases serializes"), + ) + .expect("write GitHub releases fixture"); + world.github_releases = Some(releases); +} + #[given("an empty installed inventory")] fn empty_installed_inventory(world: &mut CliWorld) { let temp = world.temp.as_ref().expect("tempdir exists"); @@ -893,6 +961,34 @@ fn run_getter_autogen_fdroid_preview_for_inventory(world: &mut CliWorld) { world.json = None; } +#[when(expr = "I run getter provider github releases for owner {string} repo {string}")] +fn run_getter_provider_github_releases(world: &mut CliWorld, owner: String, repo: String) { + let releases = world + .github_releases + .as_ref() + .expect("GitHub releases fixture exists"); + let output = run_getter( + world, + [ + "provider".to_owned(), + "github".to_owned(), + "releases".to_owned(), + "--owner".to_owned(), + owner, + "--repo".to_owned(), + repo, + "--releases".to_owned(), + releases.to_string_lossy().to_string(), + "--asset-include".to_owned(), + r"\.apk$".to_owned(), + "--asset-exclude".to_owned(), + "debug".to_owned(), + ], + ); + world.output = Some(output); + world.json = None; +} + #[when("I run getter autogen fdroid apply for that preview with accept-all")] fn run_getter_autogen_fdroid_apply_accept_all(world: &mut CliWorld) { let preview = world @@ -1486,6 +1582,33 @@ fn save_autogen_preview_to_file(world: &mut CliWorld) { world.autogen_preview = Some(preview); } +#[then(expr = "the GitHub release provider returns candidate {string} with artifact {string}")] +fn github_release_provider_returns_candidate( + world: &mut CliWorld, + version: String, + artifact_name: String, +) { + let json = current_json(world); + assert_eq!(json["ok"], true); + assert_eq!(json["command"], "provider github releases"); + assert_eq!(json["data"]["operation"], "github.releases"); + assert_eq!(json["data"]["provider"], "github"); + assert_eq!(json["data"]["source"], "refreshed"); + let candidates = json["data"]["candidates"] + .as_array() + .expect("candidates array"); + assert_eq!(candidates.len(), 1); + assert_eq!(candidates[0]["version"], version); + assert_eq!(candidates[0]["source"], "github"); + assert_eq!(candidates[0]["artifacts"][0]["name"], artifact_name); + assert_eq!(candidates[0]["artifacts"][0]["file_name"], artifact_name); + assert_eq!( + candidates[0]["artifacts"][0]["url"], + "https://github.com/DUpdateSystem/UpgradeAll/releases/download/v1.2.0/app-release.apk" + ); + assert!(json["data"]["diagnostics"].as_array().unwrap().is_empty()); +} + #[then(expr = "the autogen repository contains generated F-Droid package {string}")] fn autogen_repository_contains_generated_fdroid_package(world: &mut CliWorld, package_id: String) { autogen_repository_contains_generated_package(world, package_id.clone()); diff --git a/crates/getter-cli/tests/features/cli/provider_github_releases.feature b/crates/getter-cli/tests/features/cli/provider_github_releases.feature new file mode 100644 index 0000000..c0e6a81 --- /dev/null +++ b/crates/getter-cli/tests/features/cli/provider_github_releases.feature @@ -0,0 +1,10 @@ +@getter-cli @provider @github +Feature: GitHub release provider fixtures + Scenario: User queries fixture-backed GitHub releases without writing generated packages + Given an initialized getter data directory + And a fixture GitHub releases response for "DUpdateSystem/UpgradeAll" + When I run getter provider github releases for owner "DUpdateSystem" repo "UpgradeAll" + Then the command succeeds + And the output is valid JSON + And the GitHub release provider returns candidate "v1.2.0" with artifact "app-release.apk" + And the autogen repository has not been written diff --git a/crates/getter-operations/src/github_releases.rs b/crates/getter-operations/src/github_releases.rs new file mode 100644 index 0000000..9f151c2 --- /dev/null +++ b/crates/getter-operations/src/github_releases.rs @@ -0,0 +1,438 @@ +//! Fixture-backed GitHub release cache/query operations. +//! +//! This is the first getter-owned GitHub provider slice. It parses controlled +//! GitHub REST release fixtures, runs them through the shared provider cache, +//! and normalizes release assets into getter update candidates without adding +//! live HTTP, Flutter/Kotlin provider parsing, downloader, or installer logic. + +use crate::provider_cache::{ + read_or_refresh_provider_response, ProviderCacheDiagnostic, ProviderCacheMode, + ProviderCacheOperationError, ProviderCacheRequest, ProviderCacheSource, +}; +use getter_providers::{ + github_release_update_candidates, parse_github_releases_json, GithubAssetFilter, + GithubProviderError, GithubRelease, GithubReleaseCandidateOptions, +}; +use getter_storage::{CacheDb, StorageError}; +use serde::Deserialize; +use serde_json::{json, Value}; +use sha2::{Digest, Sha512}; + +pub const GITHUB_PROVIDER_ID: &str = "github"; +pub const GITHUB_RELEASES_CACHE_VERSION: &str = "github-releases-v1"; +pub const DEFAULT_GITHUB_API_BASE_URL: &str = "https://api.github.com"; +pub const GITHUB_ASSET_NOT_FOUND: &str = "provider.github.asset_not_found"; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct GithubReleaseConfig { + pub api_base_url: String, + pub owner: String, + pub repo: String, +} + +impl GithubReleaseConfig { + pub fn cache_key(&self) -> String { + format!( + "{GITHUB_PROVIDER_ID}:{GITHUB_RELEASES_CACHE_VERSION}:releases:{}:{}/{}", + digest_hex(&self.api_base_url), + self.owner, + self.repo, + ) + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct GithubReleaseResult { + pub config: GithubReleaseConfig, + pub cache_key: String, + pub releases: Vec, + pub source: ProviderCacheSource, + pub diagnostics: Vec, +} + +#[derive(Debug, thiserror::Error)] +pub enum GithubReleaseOperationError { + #[error("provider cache operation failed: {0}")] + Cache(#[from] ProviderCacheOperationError), + #[error("storage error: {0}")] + Storage(#[from] StorageError), + #[error("GitHub provider parse failed: {0}")] + Provider(#[from] GithubProviderError), + #[error("GitHub provider serialization failed: {0}")] + Serialization(#[from] serde_json::Error), + #[error("invalid GitHub release request: {0}")] + InvalidRequest(String), +} + +pub fn read_or_refresh_github_releases( + db: &CacheDb, + config: GithubReleaseConfig, + mode: ProviderCacheMode, + refresh_json: F, +) -> Result +where + F: FnOnce() -> Result, +{ + let cache_key = config.cache_key(); + let cache_result = read_or_refresh_provider_response( + db, + ProviderCacheRequest { + cache_key: &cache_key, + provider: GITHUB_PROVIDER_ID, + mode, + }, + || { + let json = refresh_json()?; + let releases = + parse_github_releases_json(&json).map_err(|source| source.to_string())?; + serde_json::to_value(releases).map_err(|source| source.to_string()) + }, + )?; + let releases = serde_json::from_value(cache_result.response.response_json)?; + + Ok(GithubReleaseResult { + config, + cache_key, + releases, + source: cache_result.source, + diagnostics: cache_result.diagnostics, + }) +} + +pub fn github_releases_json( + db: &CacheDb, + request_json: &str, +) -> Result { + let request: GithubReleasesJsonRequest = serde_json::from_str(request_json)?; + let owner = required_non_empty(request.owner, "owner")?; + let repo = required_non_empty(request.repo, "repo")?; + let config = GithubReleaseConfig { + api_base_url: request + .api_base_url + .filter(|value| !value.trim().is_empty()) + .unwrap_or_else(|| DEFAULT_GITHUB_API_BASE_URL.to_owned()), + owner, + repo, + }; + let mode = match request.mode.as_deref() { + Some("force_refresh") => ProviderCacheMode::ForceRefresh, + Some("use_cached") | None => ProviderCacheMode::UseCached, + Some(other) => { + return Err(GithubReleaseOperationError::InvalidRequest(format!( + "unknown mode '{other}'" + ))) + } + }; + let asset_filter = request.asset.unwrap_or_default(); + let options = GithubReleaseCandidateOptions { + include_prereleases: request.include_prereleases, + asset_filter: GithubAssetFilter { + include: asset_filter.include, + exclude: asset_filter.exclude, + }, + }; + let result = read_or_refresh_github_releases(db, config, mode, || { + request.releases_json.ok_or_else(|| { + "fixture-backed GitHub release refresh requires releases_json".to_owned() + }) + })?; + let candidates = github_release_update_candidates(&result.releases, &options)?; + let mut diagnostics = result + .diagnostics + .iter() + .map(diagnostic_json) + .collect::>(); + if candidates.is_empty() && github_eligible_release_count(&result.releases, &options) > 0 { + diagnostics.push(json!({ + "code": GITHUB_ASSET_NOT_FOUND, + "message": "no GitHub release assets matched the requested filters", + "provider": GITHUB_PROVIDER_ID, + "owner": result.config.owner, + "repo": result.config.repo, + "cache_key": result.cache_key, + })); + } + + Ok(json!({ + "operation": "github.releases", + "provider": GITHUB_PROVIDER_ID, + "api_base_url": result.config.api_base_url, + "owner": result.config.owner, + "repo": result.config.repo, + "cache_key": result.cache_key, + "source": provider_source_json(result.source), + "releases": result.releases, + "candidates": candidates, + "diagnostics": diagnostics, + })) +} + +#[derive(Debug, Deserialize)] +struct GithubReleasesJsonRequest { + #[serde(default)] + api_base_url: Option, + #[serde(default)] + owner: Option, + #[serde(default)] + repo: Option, + #[serde(default)] + mode: Option, + #[serde(default)] + releases_json: Option, + #[serde(default)] + include_prereleases: bool, + #[serde(default)] + asset: Option, +} + +#[derive(Debug, Default, Deserialize)] +struct GithubAssetFilterRequest { + #[serde(default)] + include: Option, + #[serde(default)] + exclude: Option, +} + +fn required_non_empty( + value: Option, + field: &'static str, +) -> Result { + value + .filter(|value| !value.trim().is_empty()) + .ok_or_else(|| GithubReleaseOperationError::InvalidRequest(format!("missing {field}"))) +} + +fn github_eligible_release_count( + releases: &[GithubRelease], + options: &GithubReleaseCandidateOptions, +) -> usize { + releases + .iter() + .filter(|release| !release.draft) + .filter(|release| options.include_prereleases || !release.prerelease) + .count() +} + +fn provider_source_json(source: ProviderCacheSource) -> &'static str { + match source { + ProviderCacheSource::Cache => "cache", + ProviderCacheSource::Refreshed => "refreshed", + ProviderCacheSource::Stale => "stale", + } +} + +fn diagnostic_json(diagnostic: &ProviderCacheDiagnostic) -> Value { + json!({ + "code": diagnostic.code, + "message": diagnostic.message, + "cache_key": diagnostic.cache_key, + "provider": diagnostic.provider, + "stale_fetched_at_unix": diagnostic.stale_fetched_at_unix, + }) +} + +fn digest_hex(value: &str) -> String { + let mut hasher = Sha512::new(); + hasher.update(value.as_bytes()); + format!("{:x}", hasher.finalize()) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::provider_cache::{CACHE_REFRESH_FAILED, USED_STALE_CACHE}; + use getter_storage::ProviderResponseUpsert; + use serde_json::json; + use std::cell::Cell; + + const GITHUB_RELEASES_FIXTURE: &str = r#"[ + { + "tag_name": "v1.2.0", + "name": "Release 1.2.0", + "body": "Release notes", + "draft": false, + "prerelease": false, + "published_at": "2026-06-01T00:00:00Z", + "assets": [ + { + "name": "app-release.apk", + "content_type": "application/vnd.android.package-archive", + "size": 1234, + "browser_download_url": "https://github.com/example/app/releases/download/v1.2.0/app-release.apk", + "digest": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + }, + { + "name": "app-debug.apk", + "content_type": "application/vnd.android.package-archive", + "size": 2345, + "browser_download_url": "https://github.com/example/app/releases/download/v1.2.0/app-debug.apk" + } + ] + } +]"#; + + const UPDATED_RELEASES_FIXTURE: &str = r#"[ + { + "tag_name": "v1.3.0", + "draft": false, + "prerelease": false, + "assets": [ + { + "name": "app-release.apk", + "size": 3456, + "browser_download_url": "https://github.com/example/app/releases/download/v1.3.0/app-release.apk" + } + ] + } +]"#; + + fn config() -> GithubReleaseConfig { + GithubReleaseConfig { + api_base_url: DEFAULT_GITHUB_API_BASE_URL.to_owned(), + owner: "example".to_owned(), + repo: "app".to_owned(), + } + } + + #[test] + fn cache_miss_parses_and_stores_fixture_releases() { + let db = CacheDb::open_in_memory().unwrap(); + let config = config(); + let cache_key = config.cache_key(); + + let result = + read_or_refresh_github_releases(&db, config, ProviderCacheMode::UseCached, || { + Ok(GITHUB_RELEASES_FIXTURE.to_owned()) + }) + .unwrap(); + + assert_eq!(result.source, ProviderCacheSource::Refreshed); + assert_eq!(result.releases[0].tag_name, "v1.2.0"); + let cached = db.provider_response(&cache_key).unwrap().unwrap(); + assert_eq!(cached.provider, GITHUB_PROVIDER_ID); + assert_eq!(cached.response_json[0]["tag_name"], "v1.2.0"); + } + + #[test] + fn cache_hit_avoids_refreshing_fixture_releases() { + let db = CacheDb::open_in_memory().unwrap(); + read_or_refresh_github_releases(&db, config(), ProviderCacheMode::UseCached, || { + Ok(GITHUB_RELEASES_FIXTURE.to_owned()) + }) + .unwrap(); + let refreshed = Cell::new(false); + + let result = + read_or_refresh_github_releases(&db, config(), ProviderCacheMode::UseCached, || { + refreshed.set(true); + Ok(UPDATED_RELEASES_FIXTURE.to_owned()) + }) + .unwrap(); + + assert!(!refreshed.get()); + assert_eq!(result.source, ProviderCacheSource::Cache); + assert_eq!(result.releases[0].tag_name, "v1.2.0"); + } + + #[test] + fn forced_refresh_replaces_cached_fixture_releases() { + let db = CacheDb::open_in_memory().unwrap(); + read_or_refresh_github_releases(&db, config(), ProviderCacheMode::UseCached, || { + Ok(GITHUB_RELEASES_FIXTURE.to_owned()) + }) + .unwrap(); + + let result = + read_or_refresh_github_releases(&db, config(), ProviderCacheMode::ForceRefresh, || { + Ok(UPDATED_RELEASES_FIXTURE.to_owned()) + }) + .unwrap(); + + assert_eq!(result.source, ProviderCacheSource::Refreshed); + assert_eq!(result.releases[0].tag_name, "v1.3.0"); + } + + #[test] + fn forced_refresh_failure_returns_stale_cache_with_diagnostics() { + let db = CacheDb::open_in_memory().unwrap(); + let config = config(); + let cache_key = config.cache_key(); + db.upsert_provider_response(&ProviderResponseUpsert { + cache_key, + provider: GITHUB_PROVIDER_ID.to_owned(), + response_json: serde_json::from_str(GITHUB_RELEASES_FIXTURE).unwrap(), + }) + .unwrap(); + + let result = + read_or_refresh_github_releases(&db, config, ProviderCacheMode::ForceRefresh, || { + Err("HTTP 503".to_owned()) + }) + .unwrap(); + + assert_eq!(result.source, ProviderCacheSource::Stale); + assert_eq!(result.releases[0].tag_name, "v1.2.0"); + let codes: Vec<_> = result + .diagnostics + .iter() + .map(|diagnostic| diagnostic.code.as_str()) + .collect(); + assert_eq!(codes, vec![CACHE_REFRESH_FAILED, USED_STALE_CACHE]); + } + + #[test] + fn json_operation_returns_candidates_and_asset_diagnostics() { + let db = CacheDb::open_in_memory().unwrap(); + let request = json!({ + "owner": "example", + "repo": "app", + "releases_json": GITHUB_RELEASES_FIXTURE, + "asset": { "include": "\\.apk$", "exclude": "debug" } + }); + + let response = github_releases_json(&db, &request.to_string()).unwrap(); + + assert_eq!(response["operation"], "github.releases"); + assert_eq!(response["provider"], "github"); + assert_eq!(response["source"], "refreshed"); + assert_eq!(response["candidates"][0]["version"], "v1.2.0"); + assert_eq!( + response["candidates"][0]["artifacts"][0]["name"], + "app-release.apk" + ); + assert!(response["diagnostics"].as_array().unwrap().is_empty()); + } + + #[test] + fn json_operation_reports_asset_not_found_diagnostic() { + let db = CacheDb::open_in_memory().unwrap(); + let request = json!({ + "owner": "example", + "repo": "app", + "releases_json": GITHUB_RELEASES_FIXTURE, + "asset": { "include": "\\.aab$" } + }); + + let response = github_releases_json(&db, &request.to_string()).unwrap(); + + assert!(response["candidates"].as_array().unwrap().is_empty()); + assert_eq!(response["diagnostics"][0]["code"], GITHUB_ASSET_NOT_FOUND); + } + + #[test] + fn malformed_fixture_json_reports_provider_error() { + let db = CacheDb::open_in_memory().unwrap(); + let request = json!({ + "owner": "example", + "repo": "app", + "releases_json": "not-json" + }); + + let error = github_releases_json(&db, &request.to_string()).unwrap_err(); + + assert!(matches!( + error, + GithubReleaseOperationError::Cache(ProviderCacheOperationError::RefreshFailed(_)) + )); + } +} diff --git a/crates/getter-operations/src/lib.rs b/crates/getter-operations/src/lib.rs index ba9281b..46511ea 100644 --- a/crates/getter-operations/src/lib.rs +++ b/crates/getter-operations/src/lib.rs @@ -8,6 +8,7 @@ pub mod autogen; pub mod fdroid_autogen; pub mod fdroid_catalog; +pub mod github_releases; pub mod legacy_room; pub mod provider_cache; pub mod read_model; diff --git a/crates/getter-providers/Cargo.toml b/crates/getter-providers/Cargo.toml index 31d2fc4..c8be0da 100644 --- a/crates/getter-providers/Cargo.toml +++ b/crates/getter-providers/Cargo.toml @@ -5,6 +5,8 @@ edition.workspace = true [dependencies] getter-core = { path = "../getter-core", default-features = false } +regex = "1" roxmltree = "0.21" serde = { version = "1", features = ["derive"] } +serde_json = "1" thiserror = "1" diff --git a/crates/getter-providers/src/lib.rs b/crates/getter-providers/src/lib.rs index 518d5ba..6a9ebfb 100644 --- a/crates/getter-providers/src/lib.rs +++ b/crates/getter-providers/src/lib.rs @@ -231,6 +231,160 @@ fn fdroid_artifact_url(base: &str, apk_name: &str) -> String { format!("{}/{}", base.trim_end_matches('/'), apk_name) } +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct GithubRelease { + pub tag_name: String, + #[serde(default)] + pub name: Option, + #[serde(default)] + pub body: Option, + #[serde(default)] + pub draft: bool, + #[serde(default)] + pub prerelease: bool, + #[serde(default)] + pub published_at: Option, + #[serde(default)] + pub assets: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct GithubReleaseAsset { + pub name: String, + pub browser_download_url: String, + #[serde(default)] + pub label: Option, + #[serde(default)] + pub content_type: Option, + #[serde(default)] + pub size: Option, + #[serde(default)] + pub digest: Option, +} + +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct GithubReleaseCandidateOptions { + pub include_prereleases: bool, + pub asset_filter: GithubAssetFilter, +} + +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct GithubAssetFilter { + pub include: Option, + pub exclude: Option, +} + +#[derive(Debug, thiserror::Error)] +pub enum GithubProviderError { + #[error("failed to parse GitHub releases JSON: {0}")] + Json(#[from] serde_json::Error), + #[error("invalid GitHub asset {filter_kind} regex '{pattern}': {source}")] + AssetFilterRegex { + filter_kind: &'static str, + pattern: String, + #[source] + source: regex::Error, + }, +} + +pub fn parse_github_releases_json(json: &str) -> Result, GithubProviderError> { + Ok(serde_json::from_str(json)?) +} + +pub fn github_release_update_candidates( + releases: &[GithubRelease], + options: &GithubReleaseCandidateOptions, +) -> Result, GithubProviderError> { + let filter = CompiledGithubAssetFilter::new(&options.asset_filter)?; + Ok(releases + .iter() + .filter(|release| !release.draft) + .filter(|release| options.include_prereleases || !release.prerelease) + .filter_map(|release| github_release_update_candidate(release, &filter)) + .collect()) +} + +fn github_release_update_candidate( + release: &GithubRelease, + filter: &CompiledGithubAssetFilter, +) -> Option { + let artifacts = release + .assets + .iter() + .filter(|asset| filter.matches(asset)) + .map(github_asset_update_artifact) + .collect::>(); + if artifacts.is_empty() { + return None; + } + + Some(UpdateCandidate { + version: release.tag_name.clone(), + version_code: None, + channel: release.prerelease.then(|| "prerelease".to_owned()), + source: Some("github".to_owned()), + artifacts, + }) +} + +fn github_asset_update_artifact(asset: &GithubReleaseAsset) -> UpdateArtifact { + UpdateArtifact { + name: asset.name.clone(), + url: asset.browser_download_url.clone(), + file_name: Some(asset.name.clone()), + sha256: github_asset_sha256(asset.digest.as_deref()), + size: asset.size, + } +} + +fn github_asset_sha256(digest: Option<&str>) -> Option { + digest + .and_then(|value| value.strip_prefix("sha256:")) + .filter(|value| { + value.len() == 64 && value.chars().all(|character| character.is_ascii_hexdigit()) + }) + .map(str::to_owned) +} + +struct CompiledGithubAssetFilter { + include: Option, + exclude: Option, +} + +impl CompiledGithubAssetFilter { + fn new(filter: &GithubAssetFilter) -> Result { + Ok(Self { + include: compile_github_asset_regex("include", filter.include.as_deref())?, + exclude: compile_github_asset_regex("exclude", filter.exclude.as_deref())?, + }) + } + + fn matches(&self, asset: &GithubReleaseAsset) -> bool { + self.include + .as_ref() + .is_none_or(|include| include.is_match(&asset.name)) + && !self + .exclude + .as_ref() + .is_some_and(|exclude| exclude.is_match(&asset.name)) + } +} + +fn compile_github_asset_regex( + filter_kind: &'static str, + pattern: Option<&str>, +) -> Result, GithubProviderError> { + pattern + .map(|pattern| { + regex::Regex::new(pattern).map_err(|source| GithubProviderError::AssetFilterRegex { + filter_kind, + pattern: pattern.to_owned(), + source, + }) + }) + .transpose() +} + #[cfg(test)] mod tests { use super::*; @@ -342,4 +496,159 @@ mod tests { assert!(matches!(error, FdroidCatalogError::Xml(_))); } + + #[test] + fn parses_github_releases_json_into_provider_facts() { + let releases = parse_github_releases_json(GITHUB_RELEASES_FIXTURE).unwrap(); + + assert_eq!(releases.len(), 3); + let release = &releases[0]; + assert_eq!(release.tag_name, "v1.2.0"); + assert_eq!(release.name.as_deref(), Some("Release 1.2.0")); + assert_eq!(release.body.as_deref(), Some("Release notes")); + assert!(!release.draft); + assert!(!release.prerelease); + assert_eq!( + release.published_at.as_deref(), + Some("2026-06-01T00:00:00Z") + ); + assert_eq!(release.assets.len(), 3); + let asset = &release.assets[0]; + assert_eq!(asset.name, "app-release.apk"); + assert_eq!( + asset.content_type.as_deref(), + Some("application/vnd.android.package-archive") + ); + assert_eq!(asset.size, Some(1234)); + assert_eq!( + asset.browser_download_url, + "https://github.com/example/app/releases/download/v1.2.0/app-release.apk" + ); + assert_eq!( + asset.digest.as_deref(), + Some("sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa") + ); + } + + #[test] + fn normalizes_github_release_assets_to_update_candidates_with_filters() { + let releases = parse_github_releases_json(GITHUB_RELEASES_FIXTURE).unwrap(); + let options = GithubReleaseCandidateOptions { + asset_filter: GithubAssetFilter { + include: Some(r"\.apk$".to_owned()), + exclude: Some("debug".to_owned()), + }, + ..Default::default() + }; + + let candidates = github_release_update_candidates(&releases, &options).unwrap(); + + assert_eq!(candidates.len(), 1); + assert_eq!(candidates[0].version, "v1.2.0"); + assert_eq!(candidates[0].source.as_deref(), Some("github")); + assert_eq!(candidates[0].artifacts.len(), 1); + let artifact = &candidates[0].artifacts[0]; + assert_eq!(artifact.name, "app-release.apk"); + assert_eq!( + artifact.url, + "https://github.com/example/app/releases/download/v1.2.0/app-release.apk" + ); + assert_eq!(artifact.file_name.as_deref(), Some("app-release.apk")); + assert_eq!( + artifact.sha256.as_deref(), + Some("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa") + ); + assert_eq!(artifact.size, Some(1234)); + } + + #[test] + fn ignores_unrecognized_github_asset_digests() { + let releases = parse_github_releases_json(GITHUB_INVALID_DIGEST_FIXTURE).unwrap(); + + let candidates = github_release_update_candidates( + &releases, + &GithubReleaseCandidateOptions { + asset_filter: GithubAssetFilter { + include: Some(r"\.apk$".to_owned()), + exclude: None, + }, + ..Default::default() + }, + ) + .unwrap(); + + assert_eq!(candidates[0].artifacts[0].sha256, None); + } + + const GITHUB_RELEASES_FIXTURE: &str = r#"[ + { + "tag_name": "v1.2.0", + "name": "Release 1.2.0", + "body": "Release notes", + "draft": false, + "prerelease": false, + "published_at": "2026-06-01T00:00:00Z", + "assets": [ + { + "name": "app-release.apk", + "content_type": "application/vnd.android.package-archive", + "size": 1234, + "browser_download_url": "https://github.com/example/app/releases/download/v1.2.0/app-release.apk", + "digest": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + }, + { + "name": "app-debug.apk", + "content_type": "application/vnd.android.package-archive", + "size": 2345, + "browser_download_url": "https://github.com/example/app/releases/download/v1.2.0/app-debug.apk" + }, + { + "name": "notes.txt", + "content_type": "text/plain", + "size": 345, + "browser_download_url": "https://github.com/example/app/releases/download/v1.2.0/notes.txt" + } + ] + }, + { + "tag_name": "v1.3.0-beta1", + "draft": false, + "prerelease": true, + "assets": [ + { + "name": "app-beta.apk", + "size": 456, + "browser_download_url": "https://github.com/example/app/releases/download/v1.3.0-beta1/app-beta.apk" + } + ] + }, + { + "tag_name": "v1.4.0-draft", + "draft": true, + "prerelease": false, + "assets": [ + { + "name": "app-draft.apk", + "size": 567, + "browser_download_url": "https://github.com/example/app/releases/download/v1.4.0-draft/app-draft.apk" + } + ] + } +]"#; + + const GITHUB_INVALID_DIGEST_FIXTURE: &str = r#"[ + { + "tag_name": "v1.2.0", + "draft": false, + "prerelease": false, + "assets": [ + { + "name": "app-release.apk", + "size": 1234, + "browser_download_url": "https://github.com/example/app/releases/download/v1.2.0/app-release.apk", + "digest": "sha256:not-a-valid-sha256" + } + ] + } +]"#; } From 32e0fb337afa3c773547645b17553deb70282767 Mon Sep 17 00:00:00 2001 From: Xiangzhe Date: Sat, 27 Jun 2026 13:20:43 +0800 Subject: [PATCH 41/52] refactor(repository): remove flat Lua repository layout --- README.md | 2 +- crates/getter-cli/src/lib.rs | 44 +-- crates/getter-cli/tests/bdd_cli.rs | 119 +++---- .../tests/features/cli/repo_validate.feature | 8 +- crates/getter-core/src/diagnostics.rs | 166 ++------- crates/getter-core/src/lua.rs | 320 ++++-------------- crates/getter-core/src/repository.rs | 248 +------------- crates/getter-operations/src/autogen.rs | 26 +- crates/getter-operations/src/read_model.rs | 75 +--- crates/getter-operations/src/runtime.rs | 108 +----- 10 files changed, 158 insertions(+), 958 deletions(-) diff --git a/README.md b/README.md index 6fc932d..095e838 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ This repository is intentionally usable outside the UpgradeAll UI. The UpgradeAl - Rust workspace split into `getter-core`, `getter-storage`, `getter-cli`, and placeholder provider/downloader/RPC/FFI crates. - Package IDs are readable, for example `android/org.fdroid.fdroid`. -- Lua package repositories use `repo.toml` plus `packages/`, `lib/`, and `templates/` directories. +- Lua package repositories are repository alias directories under `repo/`; packages are directories containing `metadata.jsonc`, optional `Manifest`, and direct child version scripts such as `1.20.0.lua` or `9999.lua`. Shared Lua classes live under repository-root `luaclass/`. - SQLite storage uses a main DB and a separate cache DB. - `getter-cli` exposes JSON command contracts for init, app list, repository registration/evaluation, package evaluation, storage validation, legacy bridge-bundle import, and sanitized legacy report listing. diff --git a/crates/getter-cli/src/lib.rs b/crates/getter-cli/src/lib.rs index 8306d7c..7c7efbe 100644 --- a/crates/getter-cli/src/lib.rs +++ b/crates/getter-cli/src/lib.rs @@ -6,9 +6,9 @@ use getter_core::autogen::InstalledInventory; use getter_core::diagnostics::validate_repository_path; -use getter_core::lua::{evaluate_package_directory_script, evaluate_package_file}; +use getter_core::lua::evaluate_package_directory_script; use getter_core::repository::{ - default_repository_priority, GetterDataDirLayout, RepositoryLayout, RepositoryMetadata, + default_repository_priority, GetterDataDirLayout, RepositoryMetadata, RepositoryPackageDirectoryLayout, REPOSITORY_ROOT_METADATA_FILE, REPO_API_VERSION_V1, }; use getter_core::runtime::{GetterRuntime, SealedActionPlan}; @@ -1225,10 +1225,6 @@ fn parse_autogen_acceptance(args: &[String]) -> Result Result { - RepositoryLayout::load(path).map_err(|source| CliError::Repository(source.to_string())) -} - fn load_package_directory_layout( path: &Path, ) -> Result { @@ -1241,20 +1237,6 @@ fn load_repository_metadata( path: &Path, priority: Option, ) -> Result { - if path.join("repo.toml").is_file() { - let layout = load_repository_layout(path)?; - if &layout.metadata.id != id { - return Err(CliError::Repository(format!( - "repo.toml id '{}' does not match requested id '{}'", - layout.metadata.id, id - ))); - } - return Ok(RepositoryMetadata { - priority: priority.unwrap_or(layout.metadata.priority), - ..layout.metadata - }); - } - RepositoryPackageDirectoryLayout::load(path) .map_err(|source| CliError::Repository(source.to_string()))?; Ok(RepositoryMetadata { @@ -1312,18 +1294,6 @@ fn evaluate_repository_packages( repo: &StoredRepository, ) -> Result, CliError> { let path = repo_path(repo)?; - if path.join("repo.toml").is_file() { - let layout = load_repository_layout(&path)?; - return layout - .packages - .iter() - .map(|package_file| { - evaluate_package_file(&layout, &package_file.path) - .map_err(|error| CliError::PackageEval(error.to_string())) - }) - .collect(); - } - let layout = load_package_directory_layout(&path)?; layout .packages @@ -1337,16 +1307,6 @@ fn evaluate_package_in_repository( package_id: &PackageId, ) -> Result, CliError> { let path = repo_path(repo)?; - if path.join("repo.toml").is_file() { - let layout = load_repository_layout(&path)?; - let Some(package_file) = layout.package_file(package_id) else { - return Ok(None); - }; - return evaluate_package_file(&layout, &package_file.path) - .map(Some) - .map_err(|error| CliError::PackageEval(error.to_string())); - } - let layout = load_package_directory_layout(&path)?; let Some(package) = layout.package(package_id) else { return Ok(None); diff --git a/crates/getter-cli/tests/bdd_cli.rs b/crates/getter-cli/tests/bdd_cli.rs index 54b584a..f5f0887 100644 --- a/crates/getter-cli/tests/bdd_cli.rs +++ b/crates/getter-cli/tests/bdd_cli.rs @@ -507,8 +507,8 @@ fn fixture_lua_repository_invalid_lua(world: &mut CliWorld, repo_id: String, pac create_custom_fixture_lua_repository( world, repo_id, - package_id.clone(), - format!("return package_def {{ id = \"{package_id}\", name = "), + package_id, + "#!/bin/upa-lua v1\nreturn package_version { name = ".to_owned(), ); } @@ -521,8 +521,8 @@ fn fixture_lua_repository_invalid_schema( create_custom_fixture_lua_repository( world, repo_id, - package_id.clone(), - format!("return {{ id = \"{package_id}\" }}"), + package_id, + "#!/bin/upa-lua v1\nreturn package_version { source_priority = \"fdroid\" }".to_owned(), ); } @@ -536,7 +536,7 @@ fn fixture_lua_repository_mismatched_path( world, repo_id, package_id, - "return { id = \"android/com.termux\", name = \"Termux\" }".to_owned(), + "#!/bin/upa-lua v1\nreturn package_version { id = \"android/com.termux\", name = \"Termux\" }".to_owned(), ); } @@ -544,14 +544,10 @@ fn fixture_lua_repository_mismatched_path( fn incomplete_lua_repository(world: &mut CliWorld, repo_id: String) { let temp = world.temp.as_ref().expect("tempdir exists"); let repo_path = temp.path().join(format!("repo-{repo_id}")); - fs::create_dir_all(&repo_path).expect("create incomplete repo dir"); - fs::write( - repo_path.join("repo.toml"), - format!( - "id = \"{repo_id}\"\nname = \"Fixture {repo_id}\"\npriority = 0\napi_version = \"getter.repo.v1\"\n" - ), - ) - .expect("write repo.toml"); + let package_dir = repo_path.join("android/app/broken"); + fs::create_dir_all(&package_dir).expect("create incomplete package dir"); + fs::write(package_dir.join("metadata.jsonc"), "{not-json") + .expect("write invalid package metadata"); world.fixture_repo_id = Some(repo_id); world.fixture_repo_path = Some(repo_path); world.fixture_package_id = None; @@ -1402,7 +1398,7 @@ fn output_reports_repository_diagnostic(world: &mut CliWorld, code: String) { assert!(diagnostic["location"]["path"].as_str().is_some()); if code == "package.schema" { assert_eq!(diagnostic["package_id"], "android/org.fdroid.fdroid"); - assert_eq!(diagnostic["location"]["field"], "name"); + assert!(diagnostic["location"]["field"].as_str().is_some()); } } @@ -1958,61 +1954,34 @@ fn create_fixture_lua_repository( ) { let temp = world.temp.as_ref().expect("tempdir exists"); let repo_path = temp.path().join(format!("repo-{repo_id}")); - if let Some(package_name_path) = package_id.strip_prefix("android/app/") { - let package_dir = repo_path.join(package_relative_path(&package_id)); - fs::create_dir_all(&package_dir).expect("create package dir"); - fs::write( - package_dir.join("metadata.jsonc"), - format!( - r#"{{ "type": "android:app", "display_name": {name:?}, "android": {{ "package_name": {package_name_path:?} }} }}"#, - name = package_name, - ), - ) - .expect("write package metadata"); - fs::write(package_dir.join("Manifest"), "").expect("write Manifest"); - fs::write( - package_dir.join("9999.lua"), - format!( - r#"#!/bin/upa-lua v1 + let android_package_name = package_id + .strip_prefix("android/app/") + .or_else(|| package_id.strip_prefix("android/")) + .expect("fixture package id should be android package id"); + let package_dir = repo_path.join(package_relative_path(&package_id)); + fs::create_dir_all(&package_dir).expect("create package dir"); + fs::write( + package_dir.join("metadata.jsonc"), + format!( + r#"{{ "type": "android:app", "display_name": {name:?}, "android": {{ "package_name": {android_package_name:?} }} }}"#, + name = package_name, + ), + ) + .expect("write package metadata"); + fs::write(package_dir.join("Manifest"), "").expect("write Manifest"); + fs::write( + package_dir.join("9999.lua"), + format!( + r#"#!/bin/upa-lua v1 return package_version {{ installed = {{ - {{ kind = "android_package", package_name = "{package_name_path}" }}, - }}, -}} -"# - ), - ) - .expect("write package Lua"); - } else { - let package_name_path = package_id - .strip_prefix("android/") - .expect("fixture package id should be android package id"); - fs::create_dir_all(repo_path.join("packages/android")).expect("create packages dir"); - fs::create_dir(repo_path.join("lib")).expect("create lib dir"); - fs::create_dir(repo_path.join("templates")).expect("create templates dir"); - fs::write( - repo_path.join("repo.toml"), - format!( - "id = \"{repo_id}\"\nname = \"Fixture {repo_id}\"\npriority = 0\napi_version = \"getter.repo.v1\"\n" - ), - ) - .expect("write repo.toml"); - fs::write( - repo_path.join(format!("packages/android/{package_name_path}.lua")), - format!( - r#" -return package_def {{ - id = "{package_id}", - name = "{package_name}", - installed = {{ - {{ kind = "android_package", package_name = "{package_name_path}" }}, + {{ kind = "android_package", package_name = "{android_package_name}" }}, }}, }} "# - ), - ) - .expect("write package Lua"); - } + ), + ) + .expect("write package Lua"); world.fixture_repo_id = Some(repo_id); world.fixture_repo_path = Some(repo_path); @@ -2184,24 +2153,14 @@ fn create_custom_fixture_lua_repository( ) { let temp = world.temp.as_ref().expect("tempdir exists"); let repo_path = temp.path().join(format!("repo-{repo_id}")); - let package_name_path = package_id - .strip_prefix("android/") - .expect("fixture package id should be android package id"); - fs::create_dir_all(repo_path.join("packages/android")).expect("create packages dir"); - fs::create_dir(repo_path.join("lib")).expect("create lib dir"); - fs::create_dir(repo_path.join("templates")).expect("create templates dir"); - fs::write( - repo_path.join("repo.toml"), - format!( - "id = \"{repo_id}\"\nname = \"Fixture {repo_id}\"\npriority = 0\napi_version = \"getter.repo.v1\"\n" - ), - ) - .expect("write repo.toml"); + let package_dir = repo_path.join(package_id.replace('/', std::path::MAIN_SEPARATOR_STR)); + fs::create_dir_all(&package_dir).expect("create package dir"); fs::write( - repo_path.join(format!("packages/android/{package_name_path}.lua")), - package_source, + package_dir.join("metadata.jsonc"), + r#"{ "type": "android:app", "display_name": "F-Droid", "android": { "package_name": "org.fdroid.fdroid" } }"#, ) - .expect("write package Lua"); + .expect("write package metadata"); + fs::write(package_dir.join("1.20.0.lua"), package_source).expect("write package Lua"); world.fixture_repo_id = Some(repo_id); world.fixture_repo_path = Some(repo_path); diff --git a/crates/getter-cli/tests/features/cli/repo_validate.feature b/crates/getter-cli/tests/features/cli/repo_validate.feature index 9c6d1a1..55604f6 100644 --- a/crates/getter-cli/tests/features/cli/repo_validate.feature +++ b/crates/getter-cli/tests/features/cli/repo_validate.feature @@ -27,16 +27,16 @@ Feature: Getter CLI repository validation Then the command succeeds And the output reports repository diagnostic "package.schema" - Scenario: User receives diagnostics for package path mismatch + Scenario: User receives diagnostics for package scripts declaring an id Given an initialized getter data directory And a fixture Lua repository "broken" with mismatched package path "android/org.fdroid.fdroid" When I run getter repo validate for that repository Then the command succeeds - And the output reports repository diagnostic "package.domain" + And the output reports repository diagnostic "package.schema" - Scenario: User receives diagnostics for an incomplete repository layout + Scenario: User receives diagnostics for invalid package metadata Given an initialized getter data directory And an incomplete Lua repository "broken" When I run getter repo validate for that repository Then the command succeeds - And the output reports repository diagnostic "repository.missing_directory" + And the output reports repository diagnostic "package.metadata" diff --git a/crates/getter-core/src/diagnostics.rs b/crates/getter-core/src/diagnostics.rs index a78ff21..0fe663c 100644 --- a/crates/getter-core/src/diagnostics.rs +++ b/crates/getter-core/src/diagnostics.rs @@ -3,10 +3,9 @@ //! Diagnostics are getter-owned DTOs used by CLI and future app bridges. They //! describe what getter observed; Flutter should only render them. -use crate::lua::{evaluate_package_directory_script, evaluate_package_file, LuaPackageError}; +use crate::lua::{evaluate_package_directory_script, LuaPackageError}; use crate::repository::{ - package_cache_key, InvalidPackageDirectory, RepositoryLayout, RepositoryLoadError, - RepositoryPackageDirectoryLayout, + InvalidPackageDirectory, RepositoryLoadError, RepositoryPackageDirectoryLayout, }; use crate::PackageId; use serde::{Deserialize, Serialize}; @@ -64,36 +63,7 @@ impl RepositoryValidationReport { /// This intentionally loads and evaluates local Lua package files only. It must /// not perform provider/network checks. pub fn validate_repository_path(path: impl AsRef) -> RepositoryValidationReport { - let root = path.as_ref(); - if root.join("repo.toml").is_file() { - validate_legacy_repository_path(root) - } else { - validate_package_directory_repository_path(root) - } -} - -fn validate_legacy_repository_path(root: &Path) -> RepositoryValidationReport { - let layout = match RepositoryLayout::load(root) { - Ok(layout) => layout, - Err(error) => { - return RepositoryValidationReport::new(0, vec![repository_load_diagnostic(error)]) - } - }; - - let mut diagnostics = Vec::new(); - let mut package_count = 0usize; - for package_file in &layout.packages { - if let Err(error) = package_cache_key(&layout, package_file) { - diagnostics.push(repository_load_diagnostic(error)); - continue; - } - match evaluate_package_file(&layout, &package_file.path) { - Ok(_) => package_count += 1, - Err(error) => diagnostics.push(lua_diagnostic(error, Some(package_file.id.clone()))), - } - } - - RepositoryValidationReport::new(package_count, diagnostics) + validate_package_directory_repository_path(path.as_ref()) } fn validate_package_directory_repository_path(root: &Path) -> RepositoryValidationReport { @@ -167,40 +137,20 @@ fn repository_load_diagnostic(error: RepositoryLoadError) -> PackageValidationDi path, format!("failed to read repository root: {source}"), ), - RepositoryLoadError::MissingGeneratedRepository { alias, path } => ( - "repository.missing_generated_repository", - path, - format!("configured generated repository '{alias}' does not exist"), - ), - RepositoryLoadError::ReadRepoToml { path, source } => ( - "repository.read_repo_toml", - path, - format!("failed to read repo.toml: {source}"), - ), - RepositoryLoadError::ParseRepoToml { path, source } => ( - "repository.parse_repo_toml", - path, - format!("failed to parse repo.toml: {source}"), - ), RepositoryLoadError::RepositoryId(source) => ( "repository.invalid_id", - PathBuf::from("repo.toml"), + PathBuf::from("repo"), source.to_string(), ), - RepositoryLoadError::UnsupportedApiVersion(version) => ( - "repository.unsupported_api_version", - PathBuf::from("repo.toml"), - format!("unsupported repository api_version '{version}'"), - ), - RepositoryLoadError::MissingDirectory { path, directory } => ( - "repository.missing_directory", + RepositoryLoadError::MissingGeneratedRepository { alias, path } => ( + "repository.missing_generated_repository", path, - format!("missing required directory '{directory}'"), + format!("configured generated repository '{alias}' does not exist"), ), RepositoryLoadError::ReadPackagesDir { path, source } => ( - "repository.read_packages_dir", + "repository.read_package_directory", path, - format!("failed to read packages directory: {source}"), + format!("failed to read package directory: {source}"), ), RepositoryLoadError::InvalidPackagePath { path, reason } => { ("repository.invalid_package_path", path, reason) @@ -351,40 +301,6 @@ mod tests { use super::*; use std::fs; - fn fixture_repo() -> tempfile::TempDir { - let temp = tempfile::tempdir().unwrap(); - let root = temp.path(); - fs::write( - root.join("repo.toml"), - r#"id = "official" -name = "Official" -priority = 0 -api_version = "getter.repo.v1" -"#, - ) - .unwrap(); - fs::create_dir_all(root.join("packages/android")).unwrap(); - fs::create_dir(root.join("lib")).unwrap(); - fs::create_dir(root.join("templates")).unwrap(); - temp - } - - #[test] - fn valid_repository_has_no_diagnostics() { - let temp = fixture_repo(); - fs::write( - temp.path().join("packages/android/org.fdroid.fdroid.lua"), - r#"return package_def { id = "android/org.fdroid.fdroid", name = "F-Droid" }"#, - ) - .unwrap(); - - let report = validate_repository_path(temp.path()); - assert!(report.valid, "{report:?}"); - assert_eq!(report.package_count, 1); - assert!(report.diagnostics.is_empty()); - assert!(!report.network_required); - } - #[test] fn package_directory_repository_has_no_diagnostics() { let temp = tempfile::tempdir().unwrap(); @@ -478,79 +394,55 @@ api_version = "getter.repo.v1" } #[test] - fn missing_directory_is_stable_repository_diagnostic() { + fn lua_schema_error_is_stable_package_diagnostic() { let temp = tempfile::tempdir().unwrap(); + let package_dir = temp.path().join("android/app/org.fdroid.fdroid"); + fs::create_dir_all(&package_dir).unwrap(); fs::write( - temp.path().join("repo.toml"), - r#"id = "official" -name = "Official" -api_version = "getter.repo.v1" -"#, + package_dir.join("metadata.jsonc"), + r#"{ "type": "android:app", "android": { "package_name": "org.fdroid.fdroid" } }"#, ) .unwrap(); - - let report = validate_repository_path(temp.path()); - assert!(!report.valid); - assert_eq!(report.diagnostics[0].code, "repository.missing_directory"); - } - - #[test] - fn lua_schema_error_is_stable_package_diagnostic() { - let temp = fixture_repo(); fs::write( - temp.path().join("packages/android/org.fdroid.fdroid.lua"), - r#"return { id = "android/org.fdroid.fdroid" }"#, + package_dir.join("9999.lua"), + "#!/bin/upa-lua v1\nreturn package_version { source_priority = \"fdroid\" }", ) .unwrap(); let report = validate_repository_path(temp.path()); assert!(!report.valid); assert_eq!(report.diagnostics[0].code, "package.schema"); - assert_eq!( - report.diagnostics[0].location.field.as_deref(), - Some("name") - ); assert_eq!( report.diagnostics[0] .package_id .as_ref() .unwrap() .to_string(), - "android/org.fdroid.fdroid" + "android/app/org.fdroid.fdroid" ); } #[test] - fn package_id_path_mismatch_is_domain_diagnostic() { - let temp = fixture_repo(); + fn package_version_script_id_field_is_schema_diagnostic() { + let temp = tempfile::tempdir().unwrap(); + let package_dir = temp.path().join("android/app/org.fdroid.fdroid"); + fs::create_dir_all(&package_dir).unwrap(); fs::write( - temp.path().join("packages/android/org.fdroid.fdroid.lua"), - r#"return { id = "android/com.termux", name = "Termux" }"#, + package_dir.join("metadata.jsonc"), + r#"{ "type": "android:app", "android": { "package_name": "org.fdroid.fdroid" } }"#, ) .unwrap(); - - let report = validate_repository_path(temp.path()); - assert!(!report.valid); - assert_eq!(report.diagnostics[0].code, "package.domain"); - } - - #[test] - fn unsupported_api_version_is_stable_repository_diagnostic() { - let temp = tempfile::tempdir().unwrap(); fs::write( - temp.path().join("repo.toml"), - r#"id = "official" -name = "Official" -api_version = "getter.repo.v2" -"#, + package_dir.join("9999.lua"), + "#!/bin/upa-lua v1\nreturn package_version { id = \"android/app/org.fdroid.fdroid\" }", ) .unwrap(); let report = validate_repository_path(temp.path()); assert!(!report.valid); - assert_eq!( - report.diagnostics[0].code, - "repository.unsupported_api_version" - ); + assert_eq!(report.diagnostics[0].code, "package.schema"); + assert!(report.diagnostics[0] + .message + .contains("must not be declared")); } } diff --git a/crates/getter-core/src/lua.rs b/crates/getter-core/src/lua.rs index 8e33718..2507fd7 100644 --- a/crates/getter-core/src/lua.rs +++ b/crates/getter-core/src/lua.rs @@ -2,8 +2,7 @@ use crate::repository::{ PackageDirectory, PackageDirectoryMetadata, PackageLuaPermission, PackageTypeMetadata, - PackageVersionScript, RepositoryLayout, RepositoryLoadError, LUA_API_SHEBANG_V1, - REPOSITORY_LUACLASS_DIR, + PackageVersionScript, RepositoryLoadError, LUA_API_SHEBANG_V1, REPOSITORY_LUACLASS_DIR, }; use crate::{ InstalledTarget, PackageId, PackagePermissions, RepositoryId, ResolvedPackage, UpdateArtifact, @@ -48,35 +47,6 @@ pub enum LuaPackageError { }, } -/// Evaluate and validate a Lua package file from a repository layout. -pub fn evaluate_package_file( - repository: &RepositoryLayout, - path: impl AsRef, -) -> Result { - let path = path.as_ref(); - let source = fs::read_to_string(path).map_err(|source| LuaPackageError::ReadFile { - path: path.to_path_buf(), - source, - })?; - evaluate_package_source(repository, path, &source) -} - -/// Evaluate package source text. This is public for focused tests and future CLI -/// plumbing; repository callers should normally use [`evaluate_package_file`]. -pub fn evaluate_package_source( - repository: &RepositoryLayout, - path: impl AsRef, - source: &str, -) -> Result { - let path = path.as_ref().to_path_buf(); - let json = evaluate_package_source_to_json( - &LuaRepositoryEnvironment::Legacy(repository), - &path, - source, - )?; - validate_package_json(repository, &path, json) -} - pub fn evaluate_package_directory_script( repository_id: &RepositoryId, package: &PackageDirectory, @@ -140,7 +110,6 @@ fn evaluate_package_source_to_json( } enum LuaRepositoryEnvironment<'a> { - Legacy(&'a RepositoryLayout), PackageDirectory { package: &'a PackageDirectory }, } @@ -152,17 +121,6 @@ fn configure_package_path( package.set("cpath", "")?; package.set("loadlib", Value::Nil)?; match environment { - LuaRepositoryEnvironment::Legacy(repository) => { - let lib_pattern = repository.lib_dir.join("?.lua"); - let nested_lib_pattern = repository.lib_dir.join("?/init.lua"); - let new_path = format!( - "{};{}", - lib_pattern.to_string_lossy(), - nested_lib_pattern.to_string_lossy() - ); - package.set("path", new_path)?; - install_lib_prefix_searcher(lua, &package, repository.lib_dir.clone())?; - } LuaRepositoryEnvironment::PackageDirectory { package: package_dir, } => { @@ -207,16 +165,6 @@ fn remove_unsafe_globals(lua: &Lua) -> mlua::Result<()> { Ok(()) } -fn install_lib_prefix_searcher(lua: &Lua, package: &Table, lib_dir: PathBuf) -> mlua::Result<()> { - install_prefixed_file_searcher( - lua, - package, - "lib.", - lib_dir, - "constrained repository lib searcher only handles lib.* modules", - ) -} - fn install_luaclass_prefix_searcher( lua: &Lua, package: &Table, @@ -315,9 +263,8 @@ fn install_helpers(lua: &Lua, environment: &LuaRepositoryEnvironment<'_>) -> mlu lua.globals().set("android_app", package_fn.clone())?; lua.globals().set("magisk_module", package_fn.clone())?; lua.globals().set("generic_package", package_fn)?; - if let LuaRepositoryEnvironment::PackageDirectory { package } = environment { - install_package_file_helpers(lua, package)?; - } + let LuaRepositoryEnvironment::PackageDirectory { package } = environment; + install_package_file_helpers(lua, package)?; Ok(()) } @@ -472,45 +419,6 @@ fn is_array_table(table: &Table) -> mlua::Result { Ok(count == len) } -fn validate_package_json( - repository: &RepositoryLayout, - path: &Path, - value: JsonValue, -) -> Result { - let object = value.as_object().ok_or_else(|| LuaPackageError::Schema { - path: path.to_path_buf(), - message: "package value must be an object".to_owned(), - })?; - - let id = required_string(path, object, "id")?; - let id: PackageId = id.parse().map_err(|source| LuaPackageError::Schema { - path: path.to_path_buf(), - message: format!("invalid package id: {source}"), - })?; - let path_id = crate::repository::package_id_from_path(&repository.packages_dir, path).map_err( - |source| LuaPackageError::Domain { - path: path.to_path_buf(), - message: format!("failed to derive package id from path: {source}"), - }, - )?; - if id != path_id { - return Err(LuaPackageError::Domain { - path: path.to_path_buf(), - message: format!("package id '{id}' does not match path-derived id '{path_id}'"), - }); - } - - let name = required_string(path, object, "name")?.to_owned(); - package_from_version_json( - repository.metadata.id.clone(), - id, - name, - PackagePermissions::default(), - path, - object, - ) -} - fn validate_package_directory_version_json( repository_id: &RepositoryId, package: &PackageDirectory, @@ -814,45 +722,48 @@ fn parse_string_array( #[cfg(test)] mod tests { use super::*; - use crate::repository::{RepositoryLayout, RepositoryPackageDirectoryLayout}; + use crate::repository::RepositoryPackageDirectoryLayout; use crate::RepositoryId; use std::fs; - fn fixture_repo() -> (tempfile::TempDir, RepositoryLayout, PathBuf) { - let temp = tempfile::tempdir().unwrap(); - let root = temp.path(); - fs::write( - root.join("repo.toml"), - r#"id = "official" -name = "UpgradeAll Official" -priority = 0 -api_version = "getter.repo.v1" -"#, + fn evaluate_single_package_directory( + root: &Path, + repository_id: &str, + ) -> Result { + let layout = RepositoryPackageDirectoryLayout::load(root).unwrap(); + let package = &layout.packages[0]; + let metadata = layout.package_metadata(package).unwrap(); + let script = layout.unambiguous_version_script(package).unwrap(); + evaluate_package_directory_script( + &RepositoryId::new(repository_id).unwrap(), + package, + &metadata, + script, ) - .unwrap(); - fs::create_dir(root.join("packages")).unwrap(); - fs::create_dir(root.join("packages/android")).unwrap(); - fs::create_dir(root.join("lib")).unwrap(); - fs::create_dir(root.join("templates")).unwrap(); - let package_path = root.join("packages/android/org.fdroid.fdroid.lua"); - fs::write(&package_path, "return {}").unwrap(); - let layout = RepositoryLayout::load(root).unwrap(); - (temp, layout, package_path) } #[test] fn evaluates_json_like_lua_package_table() { - let (_temp, layout, package_path) = fixture_repo(); + let temp = tempfile::tempdir().unwrap(); + let package_dir = temp.path().join("android/app/org.fdroid.fdroid"); + fs::create_dir_all(&package_dir).unwrap(); fs::write( - &package_path, - r#" -return package_def { - id = "android/org.fdroid.fdroid", - name = "F-Droid", + package_dir.join("metadata.jsonc"), + r#"{ + "type": "android:app", + "display_name": "F-Droid", + "android": { "package_name": "org.fdroid.fdroid" }, + "lua": { "9999.lua": { "permission": ["allow_free_network"] } } +}"#, + ) + .unwrap(); + fs::write( + package_dir.join("9999.lua"), + r#"#!/bin/upa-lua v1 +return package_version { installed = { { kind = "android_package", package_name = "org.fdroid.fdroid" }, }, - permissions = { free_network = true }, source_priority = { "github", "fdroid" }, updates = { { @@ -876,8 +787,8 @@ return package_def { ) .unwrap(); - let package = evaluate_package_file(&layout, &package_path).unwrap(); - assert_eq!(package.id.to_string(), "android/org.fdroid.fdroid"); + let package = evaluate_single_package_directory(temp.path(), "official").unwrap(); + assert_eq!(package.id.to_string(), "android/app/org.fdroid.fdroid"); assert_eq!(package.repository.as_str(), "official"); assert_eq!(package.name, "F-Droid"); assert_eq!( @@ -1213,130 +1124,26 @@ return package_version { name = getter_builtin.read_package_file("name.txt") } assert_eq!(package.name, "from builtin"); } - #[test] - fn rejects_package_id_that_does_not_match_path() { - let (_temp, layout, package_path) = fixture_repo(); - fs::write( - &package_path, - r#"return { id = "android/com.termux", name = "Termux" }"#, - ) - .unwrap(); - - let err = evaluate_package_file(&layout, &package_path).unwrap_err(); - assert!(matches!(err, LuaPackageError::Domain { .. })); - } - - #[test] - fn require_can_load_repository_lib_modules() { - let (_temp, layout, package_path) = fixture_repo(); - fs::write( - layout.lib_dir.join("android.lua"), - r#" -return { - local_app = function(input) - return { - id = input.id, - name = input.name, - installed = { - { kind = "android_package", package_name = input.package_name }, - }, - } - end -} -"#, - ) - .unwrap(); - fs::write( - &package_path, - r#" -local android = require("android") -return android.local_app { - id = "android/org.fdroid.fdroid", - name = "F-Droid", - package_name = "org.fdroid.fdroid", -} -"#, - ) - .unwrap(); - - let package = evaluate_package_file(&layout, &package_path).unwrap(); - assert_eq!(package.name, "F-Droid"); - assert_eq!(package.installed.len(), 1); - } - - #[test] - fn require_can_load_repository_lib_modules_with_lib_prefix() { - let (_temp, layout, package_path) = fixture_repo(); - fs::write( - layout.lib_dir.join("android.lua"), - r#" -return { - local_app = function(input) - return { - id = input.id, - name = input.name, - installed = { - { kind = "android_package", package_name = input.package_name }, - }, - } - end -} -"#, - ) - .unwrap(); - fs::write( - &package_path, - r#" -local android = require("lib.android") -return android.local_app { - id = "android/org.fdroid.fdroid", - name = "F-Droid", - package_name = "org.fdroid.fdroid", -} -"#, - ) - .unwrap(); - - let package = evaluate_package_file(&layout, &package_path).unwrap(); - assert_eq!(package.name, "F-Droid"); - assert_eq!(package.installed.len(), 1); - } - - #[test] - fn lib_prefixed_searcher_does_not_expose_repository_templates() { - let (_temp, layout, package_path) = fixture_repo(); - fs::write( - layout.templates_dir.join("android.lua"), - r#"return { leaked = true }"#, - ) - .unwrap(); - fs::write( - &package_path, - r#" -local ok = pcall(require, "templates.android") -return { - id = "android/org.fdroid.fdroid", - name = ok and "leaked" or "F-Droid", -} -"#, - ) - .unwrap(); - - let package = evaluate_package_file(&layout, &package_path).unwrap(); - assert_eq!(package.name, "F-Droid"); - } - #[test] fn lua_environment_does_not_expose_process_or_file_system_globals() { let temp = tempfile::tempdir().unwrap(); let side_effect = temp.path().join("side-effect"); - let (_repo_temp, layout, package_path) = fixture_repo(); + let package_dir = temp.path().join("android/app/org.fdroid.fdroid"); + fs::create_dir_all(&package_dir).unwrap(); fs::write( - &package_path, + package_dir.join("metadata.jsonc"), + r#"{ + "type": "android:app", + "display_name": "F-Droid", + "android": { "package_name": "org.fdroid.fdroid" } +}"#, + ) + .unwrap(); + fs::write( + package_dir.join("9999.lua"), format!( - r#" -return {{ - id = "android/org.fdroid.fdroid", + r#"#!/bin/upa-lua v1 +return package_version {{ name = (os == nil and io == nil and debug == nil and dofile == nil and loadfile == nil and package == nil) and "F-Droid" or "leaked", side_effect = rawget(_G, "os") and os.execute("touch {}"), }} @@ -1346,40 +1153,43 @@ return {{ ) .unwrap(); - let package = evaluate_package_file(&layout, &package_path).unwrap(); + let package = evaluate_single_package_directory(temp.path(), "official").unwrap(); assert_eq!(package.name, "F-Droid"); assert!(!side_effect.exists()); } #[test] fn repository_root_is_not_exposed_even_when_cwd_is_repository_root() { - let (_temp, layout, package_path) = fixture_repo(); + let temp = tempfile::tempdir().unwrap(); fs::write( - layout.root.join("rootleak.lua"), + temp.path().join("rootleak.lua"), r#"return { leaked = true }"#, ) .unwrap(); + let package_dir = temp.path().join("android/app/org.fdroid.fdroid"); + fs::create_dir_all(&package_dir).unwrap(); fs::write( - layout.templates_dir.join("android.lua"), - r#"return { leaked = true }"#, + package_dir.join("metadata.jsonc"), + r#"{ + "type": "android:app", + "display_name": "F-Droid", + "android": { "package_name": "org.fdroid.fdroid" } +}"#, ) .unwrap(); fs::write( - &package_path, - r#" + package_dir.join("9999.lua"), + r#"#!/bin/upa-lua v1 local root_ok = pcall(require, "rootleak") local template_ok = pcall(require, "templates.android") -return { - id = "android/org.fdroid.fdroid", - name = (root_ok or template_ok) and "leaked" or "F-Droid", -} +return package_version { name = (root_ok or template_ok) and "leaked" or "F-Droid" } "#, ) .unwrap(); let original_cwd = std::env::current_dir().unwrap(); - std::env::set_current_dir(&layout.root).unwrap(); - let result = evaluate_package_file(&layout, &package_path); + std::env::set_current_dir(temp.path()).unwrap(); + let result = evaluate_single_package_directory(temp.path(), "official"); std::env::set_current_dir(original_cwd).unwrap(); let package = result.unwrap(); diff --git a/crates/getter-core/src/repository.rs b/crates/getter-core/src/repository.rs index f5f0e99..1ad1360 100644 --- a/crates/getter-core/src/repository.rs +++ b/crates/getter-core/src/repository.rs @@ -24,16 +24,6 @@ pub const LUA_API_SHEBANG_V1: &str = "#!/bin/upa-lua v1"; pub const LOCAL_REPOSITORY_ALIAS: &str = "local"; pub const DEFAULT_GENERATED_REPOSITORY_ALIAS: &str = "autogen"; -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct RepositoryLayout { - pub root: PathBuf, - pub metadata: RepositoryMetadata, - pub packages_dir: PathBuf, - pub lib_dir: PathBuf, - pub templates_dir: PathBuf, - pub packages: Vec, -} - #[derive(Debug, Clone, PartialEq, Eq)] pub struct RepositoryMetadata { pub id: RepositoryId, @@ -236,12 +226,6 @@ pub fn generated_repository_target( } } -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct PackageFile { - pub id: PackageId, - pub path: PathBuf, -} - #[derive(Debug, Clone, PartialEq, Eq)] pub struct RepositoryPackageDirectoryLayout { pub root: PathBuf, @@ -365,27 +349,8 @@ pub enum RepositoryLoadError { }, #[error("configured generated repository '{alias}' does not exist at {path}")] MissingGeneratedRepository { alias: RepositoryId, path: PathBuf }, - #[error("failed to read repo.toml at {path}: {source}")] - ReadRepoToml { - path: PathBuf, - #[source] - source: std::io::Error, - }, - #[error("failed to parse repo.toml at {path}: {source}")] - ParseRepoToml { - path: PathBuf, - #[source] - source: toml::de::Error, - }, - #[error("invalid repository id in repo.toml: {0}")] + #[error("invalid repository id: {0}")] RepositoryId(#[from] RepositoryIdError), - #[error("unsupported repository api_version '{0}'")] - UnsupportedApiVersion(String), - #[error("repository path {path} is missing required directory '{directory}'")] - MissingDirectory { - path: PathBuf, - directory: &'static str, - }, #[error("failed to read packages directory {path}: {source}")] ReadPackagesDir { path: PathBuf, @@ -490,104 +455,6 @@ impl PackageDirectoryMetadata { } } -impl RepositoryLayout { - pub fn load(root: impl AsRef) -> Result { - let root = root.as_ref().to_path_buf(); - let repo_toml_path = root.join("repo.toml"); - let raw = fs::read_to_string(&repo_toml_path).map_err(|source| { - RepositoryLoadError::ReadRepoToml { - path: repo_toml_path.clone(), - source, - } - })?; - let raw_metadata: RawRepositoryMetadata = - toml::from_str(&raw).map_err(|source| RepositoryLoadError::ParseRepoToml { - path: repo_toml_path.clone(), - source, - })?; - let api_version = raw_metadata.api_version; - if api_version != REPO_API_VERSION_V1 { - return Err(RepositoryLoadError::UnsupportedApiVersion(api_version)); - } - let metadata = RepositoryMetadata { - id: RepositoryId::new(raw_metadata.id)?, - name: raw_metadata.name, - priority: RepositoryPriority::new(raw_metadata.priority.unwrap_or_default()), - api_version, - }; - - let packages_dir = root.join("packages"); - let lib_dir = root.join("lib"); - let templates_dir = root.join("templates"); - require_dir(&root, &packages_dir, "packages")?; - require_dir(&root, &lib_dir, "lib")?; - require_dir(&root, &templates_dir, "templates")?; - - let mut packages = Vec::new(); - collect_package_files(&packages_dir, &packages_dir, &mut packages)?; - packages.sort_by_key(|package| package.id.to_string()); - - Ok(Self { - root, - metadata, - packages_dir, - lib_dir, - templates_dir, - packages, - }) - } - - pub fn package_file(&self, id: &PackageId) -> Option<&PackageFile> { - self.packages.iter().find(|package| &package.id == id) - } -} - -#[derive(Debug, Deserialize)] -struct RawRepositoryMetadata { - id: String, - name: String, - #[serde(default)] - priority: Option, - api_version: String, -} - -fn require_dir( - root: &Path, - path: &Path, - directory: &'static str, -) -> Result<(), RepositoryLoadError> { - if path.is_dir() { - Ok(()) - } else { - Err(RepositoryLoadError::MissingDirectory { - path: root.to_path_buf(), - directory, - }) - } -} - -fn collect_package_files( - packages_root: &Path, - current: &Path, - out: &mut Vec, -) -> Result<(), RepositoryLoadError> { - for entry in read_dir_entries(current)? { - let path = entry.path(); - let file_type = entry_file_type(&entry, current)?; - if file_type.is_dir() { - collect_package_files(packages_root, &path, out)?; - } else if file_type.is_file() - && path - .extension() - .is_some_and(|ext| ext == LUA_SCRIPT_EXTENSION) - { - let id = package_id_from_path(packages_root, &path)?; - out.push(PackageFile { id, path }); - } - } - Ok(()) -} - fn collect_package_directories( repository_root: &Path, current: &Path, @@ -773,27 +640,6 @@ pub fn package_id_from_package_dir( package_id_from_relative_path(package_dir, relative) } -pub fn package_id_from_path( - packages_root: impl AsRef, - path: impl AsRef, -) -> Result { - let packages_root = packages_root.as_ref(); - let path = path.as_ref(); - let relative = - path.strip_prefix(packages_root) - .map_err(|_| RepositoryLoadError::InvalidPackagePath { - path: path.to_path_buf(), - reason: format!("path is not under {}", packages_root.display()), - })?; - if relative.extension().is_none_or(|ext| ext != "lua") { - return Err(RepositoryLoadError::InvalidPackagePath { - path: path.to_path_buf(), - reason: "package file must have .lua extension".to_owned(), - }); - } - package_id_from_relative_path(path, &relative.with_extension("")) -} - fn package_id_from_relative_path( path: &Path, relative: &Path, @@ -826,18 +672,6 @@ fn package_id_from_relative_path( }) } -pub fn package_cache_key( - repository: &RepositoryLayout, - package: &PackageFile, -) -> Result { - Ok(RepositoryPackageCacheKey { - repository_id: repository.metadata.id.clone(), - package_id: package.id.clone(), - api_version: repository.metadata.api_version.clone(), - package_file_hash: package_file_content_hash(&package.path)?, - }) -} - pub fn package_directory_cache_key( repository_id: &RepositoryId, package: &PackageDirectory, @@ -973,7 +807,6 @@ where #[cfg(test)] mod tests { use super::*; - use std::io::Write; #[test] fn data_dir_layout_uses_repo_and_rc_roots() { @@ -1123,14 +956,6 @@ mod tests { )); } - #[test] - fn derives_package_id_from_lua_path() { - let root = PathBuf::from("repo/packages"); - let id = - package_id_from_path(&root, "repo/packages/android/org.fdroid.fdroid.lua").unwrap(); - assert_eq!(id.to_string(), "android/org.fdroid.fdroid"); - } - #[test] fn derives_package_id_from_package_directory() { let root = PathBuf::from("repo/official"); @@ -1280,77 +1105,6 @@ mod tests { .contains("unsupported package kind")); } - #[test] - fn loads_repository_layout_with_required_directories() { - let temp = tempfile::tempdir().unwrap(); - let root = temp.path(); - fs::write( - root.join("repo.toml"), - r#"id = "official" -name = "UpgradeAll Official" -priority = 0 -api_version = "getter.repo.v1" -"#, - ) - .unwrap(); - fs::create_dir(root.join("packages")).unwrap(); - fs::create_dir(root.join("packages/android")).unwrap(); - fs::create_dir(root.join("lib")).unwrap(); - fs::create_dir(root.join("templates")).unwrap(); - let mut file = - fs::File::create(root.join("packages/android/org.fdroid.fdroid.lua")).unwrap(); - writeln!(file, "return {{ id = 'android/org.fdroid.fdroid' }}").unwrap(); - - let layout = RepositoryLayout::load(root).unwrap(); - assert_eq!(layout.metadata.id.as_str(), "official"); - assert_eq!(layout.metadata.priority, RepositoryPriority::DEFAULT); - assert_eq!(layout.packages.len(), 1); - assert_eq!( - layout.packages[0].id.to_string(), - "android/org.fdroid.fdroid" - ); - } - - #[test] - fn package_cache_key_changes_when_package_file_content_changes() { - let temp = tempfile::tempdir().unwrap(); - let root = temp.path(); - fs::write( - root.join("repo.toml"), - r#"id = "official" -name = "UpgradeAll Official" -priority = 0 -api_version = "getter.repo.v1" -"#, - ) - .unwrap(); - fs::create_dir(root.join("packages")).unwrap(); - fs::create_dir(root.join("packages/android")).unwrap(); - fs::create_dir(root.join("lib")).unwrap(); - fs::create_dir(root.join("templates")).unwrap(); - let package_path = root.join("packages/android/org.fdroid.fdroid.lua"); - fs::write( - &package_path, - r#"return { id = "android/org.fdroid.fdroid", name = "F-Droid" }"#, - ) - .unwrap(); - let layout = RepositoryLayout::load(root).unwrap(); - let first = package_cache_key(&layout, &layout.packages[0]).unwrap(); - - fs::write( - &package_path, - r#"return { id = "android/org.fdroid.fdroid", name = "F-Droid Nightly" }"#, - ) - .unwrap(); - let layout = RepositoryLayout::load(root).unwrap(); - let second = package_cache_key(&layout, &layout.packages[0]).unwrap(); - - assert_eq!(first.repository_id.as_str(), "official"); - assert_eq!(first.package_id.to_string(), "android/org.fdroid.fdroid"); - assert_eq!(first.api_version, REPO_API_VERSION_V1); - assert_ne!(first.package_file_hash, second.package_file_hash); - } - #[test] fn package_directory_cache_key_changes_when_local_files_change() { let temp = tempfile::tempdir().unwrap(); diff --git a/crates/getter-operations/src/autogen.rs b/crates/getter-operations/src/autogen.rs index 0b597f4..770d3bf 100644 --- a/crates/getter-operations/src/autogen.rs +++ b/crates/getter-operations/src/autogen.rs @@ -13,7 +13,7 @@ use getter_core::autogen::{ DEFAULT_AUTOGEN_REPOSITORY_ID, DEFAULT_AUTOGEN_REPOSITORY_NAME, INSTALLED_AUTOGEN_GENERATOR, }; use getter_core::repository::{ - generated_repository_target, GeneratedRepositoryTarget, GetterDataDirLayout, RepositoryLayout, + generated_repository_target, GeneratedRepositoryTarget, GetterDataDirLayout, RepositoryLoadError, RepositoryMetadata, RepositoryPackageDirectoryLayout, RepositoryRootConfig, REPO_API_VERSION_V1, }; @@ -358,23 +358,13 @@ pub(crate) fn higher_priority_package_coverage( } fn load_repository_package_ids(path: &Path) -> AutogenOperationResult> { - if path.join("repo.toml").is_file() { - let layout = RepositoryLayout::load(path) - .map_err(|source| AutogenOperationError::Repository(source.to_string()))?; - Ok(layout - .packages - .into_iter() - .map(|package| package.id) - .collect()) - } else { - let layout = RepositoryPackageDirectoryLayout::load(path) - .map_err(|source| AutogenOperationError::Repository(source.to_string()))?; - Ok(layout - .packages - .into_iter() - .map(|package| package.id) - .collect()) - } + let layout = RepositoryPackageDirectoryLayout::load(path) + .map_err(|source| AutogenOperationError::Repository(source.to_string()))?; + Ok(layout + .packages + .into_iter() + .map(|package| package.id) + .collect()) } fn autogen_candidate_json(candidate: &AutogenCandidate) -> AutogenOperationResult { diff --git a/crates/getter-operations/src/read_model.rs b/crates/getter-operations/src/read_model.rs index 79f5cad..2c75b90 100644 --- a/crates/getter-operations/src/read_model.rs +++ b/crates/getter-operations/src/read_model.rs @@ -6,11 +6,9 @@ //! to parse and render. #[cfg(feature = "lua")] -use getter_core::lua::{evaluate_package_directory_script, evaluate_package_file}; +use getter_core::lua::evaluate_package_directory_script; #[cfg(feature = "lua")] -use getter_core::repository::{ - RepositoryLayout, RepositoryLoadError, RepositoryPackageDirectoryLayout, -}; +use getter_core::repository::{RepositoryLoadError, RepositoryPackageDirectoryLayout}; #[cfg(feature = "lua")] use getter_core::{PackageId, RepositoryId}; use getter_storage::{MainDb, StorageError, StoredRepository, StoredTrackedPackage}; @@ -171,16 +169,6 @@ fn evaluate_package_in_repository( package_id: &PackageId, ) -> Result, ReadModelOperationError> { let path = repo_path(repo)?; - if path.join("repo.toml").is_file() { - let layout = RepositoryLayout::load(&path)?; - let Some(package_file) = layout.package_file(package_id) else { - return Ok(None); - }; - return evaluate_package_file(&layout, &package_file.path) - .map(Some) - .map_err(|source| ReadModelOperationError::PackageEval(source.to_string())); - } - let layout = RepositoryPackageDirectoryLayout::load(&path)?; let Some(package) = layout.package(package_id) else { return Ok(None); @@ -291,36 +279,6 @@ mod tests { ); } - #[cfg(feature = "lua")] - #[test] - fn package_eval_reads_registered_lua_repository() { - let temp = tempdir().unwrap(); - let repo_path = temp.path().join("repo"); - write_legacy_lua_repo(&repo_path); - - let db = MainDb::open(temp.path().join(MAIN_DB_FILE)).unwrap(); - db.upsert_repository( - &RepositoryMetadata { - id: "official".parse().unwrap(), - name: "Official".to_owned(), - priority: RepositoryPriority::new(10), - api_version: REPO_API_VERSION_V1.to_owned(), - }, - Some(&repo_path), - None, - ) - .unwrap(); - - let result = package_eval_json( - temp.path(), - r#"{"package_id":"android/org.fdroid.fdroid","repository_id":"official"}"#, - ) - .unwrap(); - assert_eq!(result["package"]["id"], "android/org.fdroid.fdroid"); - assert_eq!(result["package"]["name"], "F-Droid"); - assert_eq!(result["package"]["permissions"]["free_network"], true); - } - #[cfg(feature = "lua")] #[test] fn package_eval_reads_registered_package_directory_repository() { @@ -356,35 +314,6 @@ mod tests { ); } - #[cfg(feature = "lua")] - fn write_legacy_lua_repo(repo_path: &Path) { - fs::create_dir_all(repo_path.join("packages/android")).unwrap(); - fs::create_dir_all(repo_path.join("lib")).unwrap(); - fs::create_dir_all(repo_path.join("templates")).unwrap(); - fs::write( - repo_path.join("repo.toml"), - r#"id = "official" -name = "Official" -priority = 10 -api_version = "getter.repo.v1" -"#, - ) - .unwrap(); - fs::write( - repo_path.join("packages/android/org.fdroid.fdroid.lua"), - r#"return package_def { - id = "android/org.fdroid.fdroid", - name = "F-Droid", - installed = { - { kind = "android_package", package_name = "org.fdroid.fdroid" }, - }, - permissions = { free_network = true }, -} -"#, - ) - .unwrap(); - } - #[cfg(feature = "lua")] fn write_package_directory_repo(repo_path: &Path) { let package_dir = repo_path.join("android/app/com.example.autogen"); diff --git a/crates/getter-operations/src/runtime.rs b/crates/getter-operations/src/runtime.rs index 3721eed..5a3b94e 100644 --- a/crates/getter-operations/src/runtime.rs +++ b/crates/getter-operations/src/runtime.rs @@ -8,10 +8,9 @@ #[cfg(feature = "lua")] use getter_core::{ - lua::{evaluate_package_directory_script, evaluate_package_file}, + lua::evaluate_package_directory_script, repository::{ - package_cache_key, package_directory_cache_key, RepositoryLayout, RepositoryLoadError, - RepositoryPackageDirectoryLayout, + package_directory_cache_key, RepositoryLoadError, RepositoryPackageDirectoryLayout, }, }; use getter_core::{ @@ -386,21 +385,6 @@ fn evaluate_registered_package( continue; }; let root = PathBuf::from(root); - if root.join("repo.toml").is_file() { - let layout = RepositoryLayout::load(&root)?; - let Some(package_file) = layout.package_file(&request.package_id) else { - continue; - }; - let package = evaluate_package_file(&layout, &package_file.path) - .map_err(|source| RuntimeOperationError::PackageEval(source.to_string()))?; - let cache_key = package_cache_key(&layout, package_file)?; - let dependency_digest = format!( - "repo:{}:package:{}:hash:{}", - cache_key.repository_id, cache_key.package_id, cache_key.package_file_hash - ); - return Ok((package, dependency_digest)); - } - let layout = RepositoryPackageDirectoryLayout::load(&root)?; let Some(package_directory) = layout.package(&request.package_id) else { continue; @@ -465,46 +449,6 @@ mod tests { #[cfg(feature = "lua")] use std::fs; - #[cfg(feature = "lua")] - #[test] - fn registered_package_update_check_issues_action_from_lua_static_updates() { - let temp = tempfile::tempdir().unwrap(); - let repo_root = temp.path().join("repo"); - write_static_update_repo(&repo_root); - let db = MainDb::open_in_memory().unwrap(); - db.upsert_repository( - &RepositoryMetadata { - id: "official".parse().unwrap(), - name: "Official".to_owned(), - priority: RepositoryPriority::new(0), - api_version: REPO_API_VERSION_V1.to_owned(), - }, - Some(&repo_root), - None, - ) - .unwrap(); - let mut runtime = GetterRuntime::new(); - - let issued = issue_action_from_registered_package_json( - &mut runtime, - &db, - &json!({ - "package_id": "android/org.fdroid.fdroid", - "installed_version": "1.0.0" - }) - .to_string(), - ) - .unwrap(); - - assert_eq!(issued["package"]["repository"], "official"); - assert_eq!(issued["update"]["status"], "update_available"); - let action_id = issued["action"]["action_id"].as_str().unwrap(); - let submitted = - submit_action_json(&mut runtime, &json!({ "action_id": action_id }).to_string()) - .unwrap(); - assert_eq!(submitted["package_id"], "android/org.fdroid.fdroid"); - } - #[cfg(feature = "lua")] #[test] fn registered_package_update_check_issues_action_from_package_directory_static_updates() { @@ -551,13 +495,13 @@ mod tests { fn registered_package_update_check_without_update_does_not_issue_action() { let temp = tempfile::tempdir().unwrap(); let repo_root = temp.path().join("repo"); - write_static_update_repo(&repo_root); + write_package_directory_static_update_repo(&repo_root); let db = MainDb::open_in_memory().unwrap(); db.upsert_repository( &RepositoryMetadata { - id: "official".parse().unwrap(), - name: "Official".to_owned(), - priority: RepositoryPriority::new(0), + id: "autogen".parse().unwrap(), + name: "Autogen".to_owned(), + priority: RepositoryPriority::new(-1), api_version: REPO_API_VERSION_V1.to_owned(), }, Some(&repo_root), @@ -570,7 +514,7 @@ mod tests { &mut runtime, &db, &json!({ - "package_id": "android/org.fdroid.fdroid", + "package_id": "android/app/com.example.autogen", "installed_version": "1.2.0" }) .to_string(), @@ -706,44 +650,6 @@ mod tests { runtime.submit_action(&action.action_id).unwrap().task_id } - #[cfg(feature = "lua")] - fn write_static_update_repo(root: &std::path::Path) { - fs::create_dir_all(root.join("packages/android")).unwrap(); - fs::create_dir(root.join("lib")).unwrap(); - fs::create_dir(root.join("templates")).unwrap(); - fs::write( - root.join("repo.toml"), - r#"id = "official" -name = "Official" -priority = 0 -api_version = "getter.repo.v1" -"#, - ) - .unwrap(); - fs::write( - root.join("packages/android/org.fdroid.fdroid.lua"), - r#" -return package_def { - id = "android/org.fdroid.fdroid", - name = "F-Droid", - updates = { - { - version = "1.2.0", - artifacts = { - { - name = "app.apk", - url = "https://example.invalid/app.apk", - file_name = "app.apk", - }, - }, - }, - }, -} -"#, - ) - .unwrap(); - } - #[cfg(feature = "lua")] fn write_package_directory_static_update_repo(root: &std::path::Path) { let package_dir = root.join("android/app/com.example.autogen"); From a67e8227ffefae4b6665ec1a4a2e29e4839fe3e1 Mon Sep 17 00:00:00 2001 From: Xiangzhe Date: Sat, 27 Jun 2026 16:46:46 +0800 Subject: [PATCH 42/52] feat(providers): add GitHub latest commit fixtures --- crates/getter-cli/src/lib.rs | 226 ++++++++- crates/getter-cli/tests/bdd_cli.rs | 74 +++ .../cli/provider_github_latest_commit.feature | 10 + .../src/github_latest_commit.rs | 477 ++++++++++++++++++ crates/getter-operations/src/lib.rs | 1 + crates/getter-providers/src/lib.rs | 164 +++++- 6 files changed, 950 insertions(+), 2 deletions(-) create mode 100644 crates/getter-cli/tests/features/cli/provider_github_latest_commit.feature create mode 100644 crates/getter-operations/src/github_latest_commit.rs diff --git a/crates/getter-cli/src/lib.rs b/crates/getter-cli/src/lib.rs index 7c7efbe..0d32f2f 100644 --- a/crates/getter-cli/src/lib.rs +++ b/crates/getter-cli/src/lib.rs @@ -23,6 +23,7 @@ use getter_downloader::{ }; use getter_operations::autogen::{self, AutogenAcceptance, AutogenOperationError}; use getter_operations::fdroid_autogen; +use getter_operations::github_latest_commit::{self, GithubLatestCommitOperationError}; use getter_operations::github_releases::{self, GithubReleaseOperationError}; use getter_operations::legacy_room::{self, LegacyRoomOperationError}; use getter_operations::runtime as runtime_operations; @@ -146,6 +147,13 @@ pub enum CliCommand { include_prereleases: bool, refresh: bool, }, + ProviderGithubLatestCommit { + owner: String, + repo: String, + reference: Option, + commit: PathBuf, + refresh: bool, + }, LegacyImportRoomBundle { bundle: PathBuf, }, @@ -329,6 +337,16 @@ impl From for CliError { } } +impl From for CliError { + fn from(value: GithubLatestCommitOperationError) -> Self { + match value { + GithubLatestCommitOperationError::Storage(source) => Self::Storage(source.to_string()), + GithubLatestCommitOperationError::InvalidRequest(detail) => Self::Usage(detail), + other => Self::Provider(other.to_string()), + } + } +} + impl From for CliError { fn from(value: GithubReleaseOperationError) -> Self { match value { @@ -599,6 +617,18 @@ where refresh: args.refresh, } } + [domain, provider, action, rest @ ..] + if domain == "provider" && provider == "github" && action == "latest-commit" => + { + let args = parse_provider_github_latest_commit_args(rest)?; + CliCommand::ProviderGithubLatestCommit { + owner: args.owner, + repo: args.repo, + reference: args.reference, + commit: args.commit, + refresh: args.refresh, + } + } [domain, subject, action, flag, preview, rest @ ..] if domain == "autogen" && subject == "cleanup" @@ -863,6 +893,25 @@ fn execute(invocation: CliInvocation) -> Result { }); github_releases::github_releases_json(&db, &request.to_string()).map_err(CliError::from) } + CliCommand::ProviderGithubLatestCommit { + owner, + repo, + reference, + commit, + refresh, + } => { + let db = open_cache_db(&invocation.data_dir)?; + let commit_json = read_github_commit_fixture(&commit)?; + let request = json!({ + "owner": owner, + "repo": repo, + "ref": reference, + "mode": if refresh { "force_refresh" } else { "use_cached" }, + "commit_json": commit_json, + }); + github_latest_commit::github_latest_commit_json(&db, &request.to_string()) + .map_err(CliError::from) + } CliCommand::LegacyImportRoomBundle { bundle } => { let db = open_main_db(&invocation.data_dir)?; if db.migration_record_exists(LEGACY_ROOM_MIGRATION_ID)? { @@ -1099,6 +1148,15 @@ struct ProviderGithubReleasesArgs { refresh: bool, } +#[derive(Debug, Default, PartialEq, Eq)] +struct ProviderGithubLatestCommitArgs { + owner: String, + repo: String, + reference: Option, + commit: PathBuf, + refresh: bool, +} + fn parse_provider_github_releases_args( args: &[String], ) -> Result { @@ -1198,6 +1256,87 @@ fn parse_provider_github_releases_args( Ok(parsed) } +fn parse_provider_github_latest_commit_args( + args: &[String], +) -> Result { + let mut parsed = ProviderGithubLatestCommitArgs::default(); + let mut position = 0; + while position < args.len() { + match args[position].as_str() { + "--owner" => { + parsed.owner = args + .get(position + 1) + .ok_or_else(|| { + CliError::Usage( + "provider github latest-commit --owner requires an owner".to_owned(), + ) + })? + .clone(); + position += 2; + } + "--repo" => { + parsed.repo = args + .get(position + 1) + .ok_or_else(|| { + CliError::Usage( + "provider github latest-commit --repo requires a repo".to_owned(), + ) + })? + .clone(); + position += 2; + } + "--commit" => { + let path = args.get(position + 1).ok_or_else(|| { + CliError::Usage( + "provider github latest-commit --commit requires a fixture path".to_owned(), + ) + })?; + parsed.commit = PathBuf::from(path); + position += 2; + } + "--ref" => { + parsed.reference = Some( + args.get(position + 1) + .ok_or_else(|| { + CliError::Usage( + "provider github latest-commit --ref requires a ref".to_owned(), + ) + })? + .clone(), + ); + position += 2; + } + "--refresh" => { + parsed.refresh = true; + position += 1; + } + other => { + return Err(CliError::Usage(format!( + "unsupported provider github latest-commit argument '{other}'" + ))) + } + } + } + + if parsed.owner.trim().is_empty() { + return Err(CliError::Usage( + "provider github latest-commit requires --owner ".to_owned(), + )); + } + if parsed.repo.trim().is_empty() { + return Err(CliError::Usage( + "provider github latest-commit requires --repo ".to_owned(), + )); + } + if parsed.commit.as_os_str().is_empty() { + return Err(CliError::Usage( + "provider github latest-commit requires --commit ".to_owned(), + )); + } + + Ok(parsed) +} + fn parse_autogen_acceptance(args: &[String]) -> Result { match args { [flag] if flag == "--accept-all" => Ok(AutogenAcceptance::AcceptAll), @@ -1340,6 +1479,12 @@ fn read_github_releases_fixture(path: &Path) -> Result { }) } +fn read_github_commit_fixture(path: &Path) -> Result { + fs::read_to_string(path).map_err(|source| { + CliError::Provider(format!("failed to read GitHub commit fixture: {source}")) + }) +} + fn read_installed_inventory(path: &Path) -> Result { let bytes = fs::read(path) .map_err(|source| CliError::Autogen(format!("failed to read inventory: {source}")))?; @@ -1869,7 +2014,7 @@ fn envelope_to_string(value: Value) -> String { } fn usage_text() -> String { - "Usage: getter --data-dir [--priority ]|repo eval |repo validate |package eval [--repo ]|storage validate|version pin |version unpin |hub list|update check --fixture |runtime script --script |debug fake-task submit --request |debug fake-task run |debug fake-task list|debug fake-task cancel |debug fake-task events --after --limit |debug fake-task install-result --status |autogen installed preview --inventory |autogen installed apply --preview (--accept-all|--accept ...)|autogen fdroid preview --index [--package ...] [--inventory ]|autogen fdroid apply --preview (--accept-all|--accept ...)|provider github releases --owner --repo --releases [--asset-include ] [--asset-exclude ] [--include-prereleases] [--refresh]|autogen cleanup preview --inventory |autogen cleanup apply --preview (--accept-all|--accept ...)|legacy import-room-bundle |legacy import-room-db |legacy report-list>\nNote: `debug fake-task` commands are persisted fake-download scaffolding. ADR-0011 runtime task debugging uses `runtime script` and does not preserve task state across CLI invocations.\n".to_owned() + "Usage: getter --data-dir [--priority ]|repo eval |repo validate |package eval [--repo ]|storage validate|version pin |version unpin |hub list|update check --fixture |runtime script --script |debug fake-task submit --request |debug fake-task run |debug fake-task list|debug fake-task cancel |debug fake-task events --after --limit |debug fake-task install-result --status |autogen installed preview --inventory |autogen installed apply --preview (--accept-all|--accept ...)|autogen fdroid preview --index [--package ...] [--inventory ]|autogen fdroid apply --preview (--accept-all|--accept ...)|provider github releases --owner --repo --releases [--asset-include ] [--asset-exclude ] [--include-prereleases] [--refresh]|provider github latest-commit --owner --repo --commit [--ref ] [--refresh]|autogen cleanup preview --inventory |autogen cleanup apply --preview (--accept-all|--accept ...)|legacy import-room-bundle |legacy import-room-db |legacy report-list>\nNote: `debug fake-task` commands are persisted fake-download scaffolding. ADR-0011 runtime task debugging uses `runtime script` and does not preserve task state across CLI invocations.\n".to_owned() } #[derive(Debug, Deserialize)] @@ -1939,6 +2084,7 @@ impl CliCommand { Self::AutogenFdroidPreview { .. } => "autogen fdroid preview", Self::AutogenFdroidApply { .. } => "autogen fdroid apply", Self::ProviderGithubReleases { .. } => "provider github releases", + Self::ProviderGithubLatestCommit { .. } => "provider github latest-commit", Self::LegacyImportRoomBundle { .. } => "legacy import-room-bundle", Self::LegacyImportRoomDb { .. } => "legacy import-room-db", Self::LegacyReportList => "legacy report-list", @@ -1996,6 +2142,84 @@ mod tests { ); } + #[test] + fn parses_provider_github_latest_commit_command() { + let parsed = parse_args([ + "getter", + "--data-dir", + "/tmp/ua-getter", + "provider", + "github", + "latest-commit", + "--owner", + "DUpdateSystem", + "--repo", + "UpgradeAll", + "--commit", + "/tmp/commit.json", + "--ref", + "main", + "--refresh", + ]) + .unwrap(); + + assert_eq!( + parsed.command, + CliCommand::ProviderGithubLatestCommit { + owner: "DUpdateSystem".to_owned(), + repo: "UpgradeAll".to_owned(), + reference: Some("main".to_owned()), + commit: PathBuf::from("/tmp/commit.json"), + refresh: true, + } + ); + } + + #[test] + fn provider_github_latest_commit_missing_cli_flags_is_usage_error() { + let output = run([ + "getter", + "--data-dir", + "/tmp/ua-getter", + "provider", + "github", + "latest-commit", + "--owner", + "DUpdateSystem", + ]); + + assert_eq!(output.exit_code, ExitCode::Usage); + let json: Value = serde_json::from_str(&output.stdout).unwrap(); + assert_eq!(json["error"]["code"], "cli.usage"); + } + + #[test] + fn provider_github_latest_commit_malformed_fixture_is_provider_error() { + let temp = tempfile::tempdir().unwrap(); + let data_dir = temp.path().join("getter-data"); + let commit = temp.path().join("commit.json"); + fs::write(&commit, "not json").unwrap(); + + let output = run([ + "getter".to_owned(), + "--data-dir".to_owned(), + data_dir.to_string_lossy().to_string(), + "provider".to_owned(), + "github".to_owned(), + "latest-commit".to_owned(), + "--owner".to_owned(), + "DUpdateSystem".to_owned(), + "--repo".to_owned(), + "UpgradeAll".to_owned(), + "--commit".to_owned(), + commit.to_string_lossy().to_string(), + ]); + + assert_eq!(output.exit_code, ExitCode::Provider); + let json: Value = serde_json::from_str(&output.stdout).unwrap(); + assert_eq!(json["error"]["code"], "provider.error"); + } + #[test] fn old_task_namespace_is_not_public_cli_surface() { let output = run(["getter", "--data-dir", "/tmp/ua-getter", "task", "list"]); diff --git a/crates/getter-cli/tests/bdd_cli.rs b/crates/getter-cli/tests/bdd_cli.rs index f5f0887..b0a05e2 100644 --- a/crates/getter-cli/tests/bdd_cli.rs +++ b/crates/getter-cli/tests/bdd_cli.rs @@ -17,6 +17,7 @@ struct CliWorld { update_fixture: Option, fdroid_index: Option, github_releases: Option, + github_commit: Option, task_request: Option, runtime_script: Option, remembered_task_id: Option, @@ -240,6 +241,31 @@ fn fixture_github_releases_response(world: &mut CliWorld, project: String) { world.github_releases = Some(releases); } +#[given(expr = "a fixture GitHub commit response for {string}")] +fn fixture_github_commit_response(world: &mut CliWorld, project: String) { + let temp = world.temp.as_ref().expect("tempdir exists"); + let commit = temp.path().join("github-commit.json"); + fs::write( + &commit, + serde_json::to_vec_pretty(&serde_json::json!({ + "sha": "0123456789abcdef0123456789abcdef01234567", + "html_url": "https://github.com/DUpdateSystem/UpgradeAll/commit/0123456789abcdef0123456789abcdef01234567", + "commit": { + "message": format!("{project} live commit"), + "author": { + "date": "2026-06-02T03:04:05Z" + }, + "committer": { + "date": "2026-06-02T04:05:06Z" + } + } + })) + .expect("GitHub commit serializes"), + ) + .expect("write GitHub commit fixture"); + world.github_commit = Some(commit); +} + #[given("an empty installed inventory")] fn empty_installed_inventory(world: &mut CliWorld) { let temp = world.temp.as_ref().expect("tempdir exists"); @@ -985,6 +1011,30 @@ fn run_getter_provider_github_releases(world: &mut CliWorld, owner: String, repo world.json = None; } +#[when(expr = "I run getter provider github latest-commit for owner {string} repo {string}")] +fn run_getter_provider_github_latest_commit(world: &mut CliWorld, owner: String, repo: String) { + let commit = world + .github_commit + .as_ref() + .expect("GitHub commit fixture exists"); + let output = run_getter( + world, + [ + "provider".to_owned(), + "github".to_owned(), + "latest-commit".to_owned(), + "--owner".to_owned(), + owner, + "--repo".to_owned(), + repo, + "--commit".to_owned(), + commit.to_string_lossy().to_string(), + ], + ); + world.output = Some(output); + world.json = None; +} + #[when("I run getter autogen fdroid apply for that preview with accept-all")] fn run_getter_autogen_fdroid_apply_accept_all(world: &mut CliWorld) { let preview = world @@ -1605,6 +1655,30 @@ fn github_release_provider_returns_candidate( assert!(json["data"]["diagnostics"].as_array().unwrap().is_empty()); } +#[then(expr = "the GitHub latest-commit provider returns live revision {string}")] +fn github_latest_commit_provider_returns_live_revision(world: &mut CliWorld, revision: String) { + let json = current_json(world); + assert_eq!(json["ok"], true); + assert_eq!(json["command"], "provider github latest-commit"); + assert_eq!(json["data"]["operation"], "github.latest_commit"); + assert_eq!(json["data"]["provider"], "github"); + assert_eq!(json["data"]["source"], "refreshed"); + assert_eq!(json["data"]["live"], true); + assert_eq!(json["data"]["version"], revision); + assert_eq!(json["data"]["revision"], revision); + assert_eq!(json["data"]["latest_commit"]["revision"], revision); + assert_eq!( + json["data"]["latest_commit"]["published_at"], + "2026-06-02T03:04:05Z" + ); + assert!(json["data"].get("candidates").is_none()); + assert!(json["data"].get("selected_update").is_none()); + assert!(json["data"].get("artifacts").is_none()); + assert!(json["data"].get("actions").is_none()); + assert!(json["data"].get("action_id").is_none()); + assert!(json["data"]["diagnostics"].as_array().unwrap().is_empty()); +} + #[then(expr = "the autogen repository contains generated F-Droid package {string}")] fn autogen_repository_contains_generated_fdroid_package(world: &mut CliWorld, package_id: String) { autogen_repository_contains_generated_package(world, package_id.clone()); diff --git a/crates/getter-cli/tests/features/cli/provider_github_latest_commit.feature b/crates/getter-cli/tests/features/cli/provider_github_latest_commit.feature new file mode 100644 index 0000000..4cdfbbd --- /dev/null +++ b/crates/getter-cli/tests/features/cli/provider_github_latest_commit.feature @@ -0,0 +1,10 @@ +@getter-cli @provider @github +Feature: GitHub latest commit provider fixtures + Scenario: User queries fixture-backed GitHub latest commit as a live revision + Given an initialized getter data directory + And a fixture GitHub commit response for "DUpdateSystem/UpgradeAll" + When I run getter provider github latest-commit for owner "DUpdateSystem" repo "UpgradeAll" + Then the command succeeds + And the output is valid JSON + And the GitHub latest-commit provider returns live revision "0123456789abcdef0123456789abcdef01234567" + And the autogen repository has not been written diff --git a/crates/getter-operations/src/github_latest_commit.rs b/crates/getter-operations/src/github_latest_commit.rs new file mode 100644 index 0000000..15e8b7e --- /dev/null +++ b/crates/getter-operations/src/github_latest_commit.rs @@ -0,0 +1,477 @@ +//! Fixture-backed GitHub latest-commit cache/query operations. +//! +//! This ADR-0012 provider slice models latest-commit checks as live/floating +//! provider facts. It parses controlled GitHub commit fixtures through the +//! shared provider cache without producing ordinary release candidates, runtime +//! actions, downloads, installer handoffs, or Flutter/Kotlin provider parsing. + +use crate::provider_cache::{ + read_or_refresh_provider_response, ProviderCacheDiagnostic, ProviderCacheMode, + ProviderCacheOperationError, ProviderCacheRequest, ProviderCacheSource, +}; +use getter_providers::{ + github_latest_commit_live_revision, parse_github_commit_json, GithubCommit, GithubLiveRevision, + GithubProviderError, +}; +use getter_storage::{CacheDb, StorageError}; +use serde::Deserialize; +use serde_json::{json, Value}; +use sha2::{Digest, Sha512}; + +pub const GITHUB_PROVIDER_ID: &str = "github"; +pub const GITHUB_LATEST_COMMIT_CACHE_VERSION: &str = "github-latest-commit-v1"; +pub const DEFAULT_GITHUB_API_BASE_URL: &str = "https://api.github.com"; +pub const DEFAULT_GITHUB_REF: &str = "HEAD"; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct GithubLatestCommitConfig { + pub api_base_url: String, + pub owner: String, + pub repo: String, + pub reference: String, +} + +impl GithubLatestCommitConfig { + pub fn cache_key(&self) -> String { + format!( + "{GITHUB_PROVIDER_ID}:{GITHUB_LATEST_COMMIT_CACHE_VERSION}:latest_commit:{}:{}/{}/{}", + digest_hex(&self.api_base_url), + self.owner, + self.repo, + self.reference, + ) + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct GithubLatestCommitResult { + pub config: GithubLatestCommitConfig, + pub cache_key: String, + pub commit: GithubCommit, + pub live_revision: GithubLiveRevision, + pub source: ProviderCacheSource, + pub diagnostics: Vec, +} + +#[derive(Debug, thiserror::Error)] +pub enum GithubLatestCommitOperationError { + #[error("provider cache operation failed: {0}")] + Cache(#[from] ProviderCacheOperationError), + #[error("storage error: {0}")] + Storage(#[from] StorageError), + #[error("GitHub provider parse failed: {0}")] + Provider(#[from] GithubProviderError), + #[error("GitHub provider serialization failed: {0}")] + Serialization(#[from] serde_json::Error), + #[error("invalid GitHub latest-commit request: {0}")] + InvalidRequest(String), +} + +pub fn read_or_refresh_github_latest_commit( + db: &CacheDb, + config: GithubLatestCommitConfig, + mode: ProviderCacheMode, + refresh_json: F, +) -> Result +where + F: FnOnce() -> Result, +{ + let cache_key = config.cache_key(); + let cache_result = read_or_refresh_provider_response( + db, + ProviderCacheRequest { + cache_key: &cache_key, + provider: GITHUB_PROVIDER_ID, + mode, + }, + || { + let json = refresh_json()?; + let commit = parse_github_commit_json(&json).map_err(|source| source.to_string())?; + github_latest_commit_live_revision(&commit).map_err(|source| source.to_string())?; + serde_json::to_value(commit).map_err(|source| source.to_string()) + }, + )?; + let commit = serde_json::from_value(cache_result.response.response_json)?; + let live_revision = github_latest_commit_live_revision(&commit)?; + + Ok(GithubLatestCommitResult { + config, + cache_key, + commit, + live_revision, + source: cache_result.source, + diagnostics: cache_result.diagnostics, + }) +} + +pub fn github_latest_commit_json( + db: &CacheDb, + request_json: &str, +) -> Result { + let request: GithubLatestCommitJsonRequest = serde_json::from_str(request_json)?; + let owner = required_non_empty(request.owner, "owner")?; + let repo = required_non_empty(request.repo, "repo")?; + let reference = request + .reference + .filter(|value| !value.trim().is_empty()) + .unwrap_or_else(|| DEFAULT_GITHUB_REF.to_owned()); + let config = GithubLatestCommitConfig { + api_base_url: request + .api_base_url + .filter(|value| !value.trim().is_empty()) + .unwrap_or_else(|| DEFAULT_GITHUB_API_BASE_URL.to_owned()), + owner, + repo, + reference, + }; + let mode = match request.mode.as_deref() { + Some("force_refresh") => ProviderCacheMode::ForceRefresh, + Some("use_cached") | None => ProviderCacheMode::UseCached, + Some(other) => { + return Err(GithubLatestCommitOperationError::InvalidRequest(format!( + "unknown mode '{other}'" + ))) + } + }; + let result = read_or_refresh_github_latest_commit(db, config, mode, || { + request.commit_json.ok_or_else(|| { + "fixture-backed GitHub latest-commit refresh requires commit_json".to_owned() + }) + })?; + + Ok(json!({ + "operation": "github.latest_commit", + "provider": GITHUB_PROVIDER_ID, + "api_base_url": result.config.api_base_url, + "owner": result.config.owner, + "repo": result.config.repo, + "ref": result.config.reference, + "cache_key": result.cache_key, + "source": provider_source_json(result.source), + "live": true, + "version": result.live_revision.version, + "revision": result.live_revision.revision, + "latest_commit": result.live_revision, + "commit": result.commit, + "diagnostics": result.diagnostics.iter().map(diagnostic_json).collect::>(), + })) +} + +#[derive(Debug, Deserialize)] +struct GithubLatestCommitJsonRequest { + #[serde(default)] + api_base_url: Option, + #[serde(default)] + owner: Option, + #[serde(default)] + repo: Option, + #[serde(default, rename = "ref")] + reference: Option, + #[serde(default)] + mode: Option, + #[serde(default)] + commit_json: Option, +} + +fn required_non_empty( + value: Option, + field: &'static str, +) -> Result { + value + .filter(|value| !value.trim().is_empty()) + .ok_or_else(|| GithubLatestCommitOperationError::InvalidRequest(format!("missing {field}"))) +} + +fn provider_source_json(source: ProviderCacheSource) -> &'static str { + match source { + ProviderCacheSource::Cache => "cache", + ProviderCacheSource::Refreshed => "refreshed", + ProviderCacheSource::Stale => "stale", + } +} + +fn diagnostic_json(diagnostic: &ProviderCacheDiagnostic) -> Value { + json!({ + "code": diagnostic.code, + "message": diagnostic.message, + "cache_key": diagnostic.cache_key, + "provider": diagnostic.provider, + "stale_fetched_at_unix": diagnostic.stale_fetched_at_unix, + }) +} + +fn digest_hex(value: &str) -> String { + let mut hasher = Sha512::new(); + hasher.update(value.as_bytes()); + format!("{:x}", hasher.finalize()) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::provider_cache::{CACHE_REFRESH_FAILED, USED_STALE_CACHE}; + use std::cell::Cell; + + const GITHUB_COMMIT_FIXTURE: &str = r#"{ + "sha": "0123456789abcdef0123456789abcdef01234567", + "html_url": "https://github.com/DUpdateSystem/UpgradeAll/commit/0123456789abcdef0123456789abcdef01234567", + "commit": { + "message": "Update app metadata", + "author": { + "date": "2026-06-02T03:04:05Z" + }, + "committer": { + "date": "2026-06-02T04:05:06Z" + } + } +}"#; + + const UPDATED_COMMIT_FIXTURE: &str = r#"{ + "sha": "fedcba9876543210fedcba9876543210fedcba98", + "html_url": "https://github.com/DUpdateSystem/UpgradeAll/commit/fedcba9876543210fedcba9876543210fedcba98", + "commit": { + "message": "Refresh generated packages", + "committer": { + "date": "2026-06-03T03:04:05Z" + } + } +}"#; + + const EMPTY_SHA_COMMIT_FIXTURE: &str = r#"{ + "sha": " ", + "commit": { + "message": "Broken fixture" + } +}"#; + + fn config() -> GithubLatestCommitConfig { + GithubLatestCommitConfig { + api_base_url: DEFAULT_GITHUB_API_BASE_URL.to_owned(), + owner: "DUpdateSystem".to_owned(), + repo: "UpgradeAll".to_owned(), + reference: DEFAULT_GITHUB_REF.to_owned(), + } + } + + #[test] + fn cache_key_identifies_latest_commit_request_and_ref() { + let main = config(); + let mut branch = config(); + branch.reference = "main".to_owned(); + let releases = crate::github_releases::GithubReleaseConfig { + api_base_url: crate::github_releases::DEFAULT_GITHUB_API_BASE_URL.to_owned(), + owner: main.owner.clone(), + repo: main.repo.clone(), + }; + + assert!(main + .cache_key() + .contains(GITHUB_LATEST_COMMIT_CACHE_VERSION)); + assert!(main.cache_key().contains(":latest_commit:")); + assert_ne!(main.cache_key(), branch.cache_key()); + assert_ne!(main.cache_key(), releases.cache_key()); + } + + #[test] + fn cache_miss_parses_and_stores_fixture_commit_as_live_revision() { + let db = CacheDb::open_in_memory().unwrap(); + let config = config(); + let cache_key = config.cache_key(); + + let result = + read_or_refresh_github_latest_commit(&db, config, ProviderCacheMode::UseCached, || { + Ok(GITHUB_COMMIT_FIXTURE.to_owned()) + }) + .unwrap(); + + assert_eq!(result.source, ProviderCacheSource::Refreshed); + assert!(result.live_revision.live); + assert_eq!( + result.live_revision.revision, + "0123456789abcdef0123456789abcdef01234567" + ); + assert_eq!( + result.live_revision.published_at.as_deref(), + Some("2026-06-02T03:04:05Z") + ); + let cached = db.provider_response(&cache_key).unwrap().unwrap(); + assert_eq!(cached.provider, GITHUB_PROVIDER_ID); + assert_eq!( + cached.response_json["sha"], + "0123456789abcdef0123456789abcdef01234567" + ); + } + + #[test] + fn cache_hit_avoids_refreshing_fixture_commit() { + let db = CacheDb::open_in_memory().unwrap(); + read_or_refresh_github_latest_commit(&db, config(), ProviderCacheMode::UseCached, || { + Ok(GITHUB_COMMIT_FIXTURE.to_owned()) + }) + .unwrap(); + let refreshed = Cell::new(false); + + let result = read_or_refresh_github_latest_commit( + &db, + config(), + ProviderCacheMode::UseCached, + || { + refreshed.set(true); + Ok(UPDATED_COMMIT_FIXTURE.to_owned()) + }, + ) + .unwrap(); + + assert!(!refreshed.get()); + assert_eq!(result.source, ProviderCacheSource::Cache); + assert_eq!( + result.live_revision.revision, + "0123456789abcdef0123456789abcdef01234567" + ); + } + + #[test] + fn forced_refresh_replaces_cached_fixture_commit() { + let db = CacheDb::open_in_memory().unwrap(); + read_or_refresh_github_latest_commit(&db, config(), ProviderCacheMode::UseCached, || { + Ok(GITHUB_COMMIT_FIXTURE.to_owned()) + }) + .unwrap(); + + let result = read_or_refresh_github_latest_commit( + &db, + config(), + ProviderCacheMode::ForceRefresh, + || Ok(UPDATED_COMMIT_FIXTURE.to_owned()), + ) + .unwrap(); + + assert_eq!(result.source, ProviderCacheSource::Refreshed); + assert_eq!( + result.live_revision.revision, + "fedcba9876543210fedcba9876543210fedcba98" + ); + } + + #[test] + fn forced_refresh_failure_returns_stale_commit_with_diagnostics() { + let db = CacheDb::open_in_memory().unwrap(); + read_or_refresh_github_latest_commit(&db, config(), ProviderCacheMode::UseCached, || { + Ok(GITHUB_COMMIT_FIXTURE.to_owned()) + }) + .unwrap(); + + let result = read_or_refresh_github_latest_commit( + &db, + config(), + ProviderCacheMode::ForceRefresh, + || Err("HTTP 503".to_owned()), + ) + .unwrap(); + + assert_eq!(result.source, ProviderCacheSource::Stale); + assert_eq!( + result.live_revision.revision, + "0123456789abcdef0123456789abcdef01234567" + ); + let codes: Vec<_> = result + .diagnostics + .iter() + .map(|diagnostic| diagnostic.code.as_str()) + .collect(); + assert_eq!(codes, vec![CACHE_REFRESH_FAILED, USED_STALE_CACHE]); + } + + #[test] + fn cache_miss_with_empty_sha_is_an_error_and_does_not_store_cache() { + let db = CacheDb::open_in_memory().unwrap(); + let config = config(); + let cache_key = config.cache_key(); + + let error = + read_or_refresh_github_latest_commit(&db, config, ProviderCacheMode::UseCached, || { + Ok(EMPTY_SHA_COMMIT_FIXTURE.to_owned()) + }) + .unwrap_err(); + + assert!(matches!( + error, + GithubLatestCommitOperationError::Cache(ProviderCacheOperationError::RefreshFailed(_)) + )); + assert!(db.provider_response(&cache_key).unwrap().is_none()); + } + + #[test] + fn forced_refresh_with_empty_sha_returns_stale_commit_with_diagnostics() { + let db = CacheDb::open_in_memory().unwrap(); + read_or_refresh_github_latest_commit(&db, config(), ProviderCacheMode::UseCached, || { + Ok(GITHUB_COMMIT_FIXTURE.to_owned()) + }) + .unwrap(); + + let result = read_or_refresh_github_latest_commit( + &db, + config(), + ProviderCacheMode::ForceRefresh, + || Ok(EMPTY_SHA_COMMIT_FIXTURE.to_owned()), + ) + .unwrap(); + + assert_eq!(result.source, ProviderCacheSource::Stale); + assert_eq!( + result.live_revision.revision, + "0123456789abcdef0123456789abcdef01234567" + ); + let codes: Vec<_> = result + .diagnostics + .iter() + .map(|diagnostic| diagnostic.code.as_str()) + .collect(); + assert_eq!(codes, vec![CACHE_REFRESH_FAILED, USED_STALE_CACHE]); + } + + #[test] + fn json_operation_returns_live_revision_from_cached_commit() { + let db = CacheDb::open_in_memory().unwrap(); + let first = github_latest_commit_json( + &db, + &json!({ + "owner": "DUpdateSystem", + "repo": "UpgradeAll", + "commit_json": GITHUB_COMMIT_FIXTURE + }) + .to_string(), + ) + .unwrap(); + assert_eq!(first["source"], "refreshed"); + + let second = github_latest_commit_json( + &db, + &json!({ + "owner": "DUpdateSystem", + "repo": "UpgradeAll" + }) + .to_string(), + ) + .unwrap(); + + assert_eq!(second["operation"], "github.latest_commit"); + assert_eq!(second["ref"], DEFAULT_GITHUB_REF); + assert_eq!(second["source"], "cache"); + assert_eq!(second["live"], true); + assert!(second.get("candidates").is_none()); + assert!(second.get("selected_update").is_none()); + assert!(second.get("artifacts").is_none()); + assert!(second.get("actions").is_none()); + assert!(second.get("action_id").is_none()); + assert_eq!( + second["revision"], + "0123456789abcdef0123456789abcdef01234567" + ); + assert_eq!( + second["latest_commit"]["published_at"], + "2026-06-02T03:04:05Z" + ); + assert!(second["latest_commit"]["live"].as_bool().unwrap()); + } +} diff --git a/crates/getter-operations/src/lib.rs b/crates/getter-operations/src/lib.rs index 46511ea..30097ff 100644 --- a/crates/getter-operations/src/lib.rs +++ b/crates/getter-operations/src/lib.rs @@ -8,6 +8,7 @@ pub mod autogen; pub mod fdroid_autogen; pub mod fdroid_catalog; +pub mod github_latest_commit; pub mod github_releases; pub mod legacy_room; pub mod provider_cache; diff --git a/crates/getter-providers/src/lib.rs b/crates/getter-providers/src/lib.rs index 6a9ebfb..a71cd49 100644 --- a/crates/getter-providers/src/lib.rs +++ b/crates/getter-providers/src/lib.rs @@ -262,6 +262,42 @@ pub struct GithubReleaseAsset { pub digest: Option, } +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct GithubCommit { + pub sha: String, + #[serde(default)] + pub html_url: Option, + #[serde(default)] + pub commit: GithubCommitDetails, +} + +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct GithubCommitDetails { + #[serde(default)] + pub message: Option, + #[serde(default)] + pub author: Option, + #[serde(default)] + pub committer: Option, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct GithubCommitActor { + #[serde(default)] + pub date: Option, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct GithubLiveRevision { + pub version: String, + pub revision: String, + pub live: bool, + pub source: String, + pub published_at: Option, + pub html_url: Option, + pub message: Option, +} + #[derive(Debug, Clone, Default, PartialEq, Eq)] pub struct GithubReleaseCandidateOptions { pub include_prereleases: bool, @@ -276,8 +312,13 @@ pub struct GithubAssetFilter { #[derive(Debug, thiserror::Error)] pub enum GithubProviderError { - #[error("failed to parse GitHub releases JSON: {0}")] + #[error("failed to parse GitHub provider JSON: {0}")] Json(#[from] serde_json::Error), + #[error("invalid GitHub commit field {field}: {message}")] + InvalidCommit { + field: &'static str, + message: String, + }, #[error("invalid GitHub asset {filter_kind} regex '{pattern}': {source}")] AssetFilterRegex { filter_kind: &'static str, @@ -291,6 +332,43 @@ pub fn parse_github_releases_json(json: &str) -> Result, Gith Ok(serde_json::from_str(json)?) } +pub fn parse_github_commit_json(json: &str) -> Result { + Ok(serde_json::from_str(json)?) +} + +pub fn github_latest_commit_live_revision( + commit: &GithubCommit, +) -> Result { + let revision = commit.sha.trim(); + if revision.is_empty() { + return Err(GithubProviderError::InvalidCommit { + field: "sha", + message: "commit sha must not be empty".to_owned(), + }); + } + + Ok(GithubLiveRevision { + version: revision.to_owned(), + revision: revision.to_owned(), + live: true, + source: "github".to_owned(), + published_at: commit + .commit + .author + .as_ref() + .and_then(|author| author.date.clone()) + .or_else(|| { + commit + .commit + .committer + .as_ref() + .and_then(|committer| committer.date.clone()) + }), + html_url: commit.html_url.clone(), + message: commit.commit.message.clone(), + }) +} + pub fn github_release_update_candidates( releases: &[GithubRelease], options: &GithubReleaseCandidateOptions, @@ -580,6 +658,76 @@ mod tests { assert_eq!(candidates[0].artifacts[0].sha256, None); } + #[test] + fn parses_github_commit_json_into_live_revision() { + let commit = parse_github_commit_json(GITHUB_COMMIT_FIXTURE).unwrap(); + + assert_eq!(commit.sha, "0123456789abcdef0123456789abcdef01234567"); + let revision = github_latest_commit_live_revision(&commit).unwrap(); + + assert!(revision.live); + assert_eq!(revision.source, "github"); + assert_eq!(revision.version, "0123456789abcdef0123456789abcdef01234567"); + assert_eq!( + revision.revision, + "0123456789abcdef0123456789abcdef01234567" + ); + assert_eq!( + revision.published_at.as_deref(), + Some("2026-06-02T03:04:05Z") + ); + assert_eq!(revision.message.as_deref(), Some("Update app metadata")); + assert_eq!( + revision.html_url.as_deref(), + Some("https://github.com/DUpdateSystem/UpgradeAll/commit/0123456789abcdef0123456789abcdef01234567") + ); + } + + #[test] + fn falls_back_to_committer_date_for_live_revision() { + let commit = parse_github_commit_json( + r#"{ + "sha": "fedcba9876543210fedcba9876543210fedcba98", + "commit": { "committer": { "date": "2026-06-03T03:04:05Z" } } +}"#, + ) + .unwrap(); + + let revision = github_latest_commit_live_revision(&commit).unwrap(); + + assert_eq!( + revision.published_at.as_deref(), + Some("2026-06-03T03:04:05Z") + ); + } + + #[test] + fn omits_live_revision_date_when_commit_dates_are_missing() { + let commit = parse_github_commit_json( + r#"{ + "sha": "fedcba9876543210fedcba9876543210fedcba98", + "commit": { "message": "No dates" } +}"#, + ) + .unwrap(); + + let revision = github_latest_commit_live_revision(&commit).unwrap(); + + assert_eq!(revision.published_at, None); + } + + #[test] + fn rejects_empty_github_commit_sha() { + let commit = parse_github_commit_json(r#"{ "sha": "", "commit": {} }"#).unwrap(); + + let error = github_latest_commit_live_revision(&commit).unwrap_err(); + + assert!(matches!( + error, + GithubProviderError::InvalidCommit { field: "sha", .. } + )); + } + const GITHUB_RELEASES_FIXTURE: &str = r#"[ { "tag_name": "v1.2.0", @@ -651,4 +799,18 @@ mod tests { ] } ]"#; + + const GITHUB_COMMIT_FIXTURE: &str = r#"{ + "sha": "0123456789abcdef0123456789abcdef01234567", + "html_url": "https://github.com/DUpdateSystem/UpgradeAll/commit/0123456789abcdef0123456789abcdef01234567", + "commit": { + "message": "Update app metadata", + "author": { + "date": "2026-06-02T03:04:05Z" + }, + "committer": { + "date": "2026-06-02T04:05:06Z" + } + } +}"#; } From 496bc08b44b20fd02af00490194ec526647e7c7d Mon Sep 17 00:00:00 2001 From: Xiangzhe Date: Sat, 27 Jun 2026 17:59:57 +0800 Subject: [PATCH 43/52] test(lua): cover F-Droid luaclass shape --- crates/getter-core/src/lua.rs | 80 +++++++++++++++++++++++++++++++++++ 1 file changed, 80 insertions(+) diff --git a/crates/getter-core/src/lua.rs b/crates/getter-core/src/lua.rs index 2507fd7..186aa65 100644 --- a/crates/getter-core/src/lua.rs +++ b/crates/getter-core/src/lua.rs @@ -1008,6 +1008,86 @@ return android.package_version { assert_eq!(package.installed.len(), 1); } + #[test] + fn package_directory_can_use_repository_local_fdroid_luaclass_shape() { + let temp = tempfile::tempdir().unwrap(); + let package_dir = temp.path().join("android/f-droid/app/org.fdroid.fdroid"); + fs::create_dir_all(&package_dir).unwrap(); + fs::create_dir_all(temp.path().join("luaclass")).unwrap(); + fs::write( + temp.path().join("luaclass/fdroid_android.lua"), + r#" +local fdroid = {} + +function fdroid.package(spec) + if spec.package_name ~= "org.fdroid.fdroid" then + error("F-Droid package_name fixture mismatch") + end + return package_version { + updates = { + { + version = "1.20.0", + version_code = 1020000, + source = "fdroid", + artifacts = { + { + name = "org.fdroid.fdroid_1020000.apk", + url = "https://f-droid.org/repo/org.fdroid.fdroid_1020000.apk", + file_name = "org.fdroid.fdroid_1020000.apk", + }, + }, + }, + }, + } +end + +return fdroid +"#, + ) + .unwrap(); + fs::write( + package_dir.join("metadata.jsonc"), + r#"{ + "type": "android:app", + "android": { "package_name": "org.fdroid.fdroid" } +}"#, + ) + .unwrap(); + fs::write( + package_dir.join("9999.lua"), + r#"#!/bin/upa-lua v1 +local fdroid = require("luaclass.fdroid_android") +return fdroid.package { + package_name = "org.fdroid.fdroid", +} +"#, + ) + .unwrap(); + + let package = evaluate_single_package_directory(temp.path(), "official").unwrap(); + + assert_eq!( + package.id.to_string(), + "android/f-droid/app/org.fdroid.fdroid" + ); + assert_eq!(package.repository.as_str(), "official"); + assert_eq!(package.name, "android/f-droid/app/org.fdroid.fdroid"); + assert_eq!( + package.installed, + vec![InstalledTarget::AndroidPackage { + package_name: "org.fdroid.fdroid".to_owned() + }] + ); + assert_eq!(package.updates.len(), 1); + assert_eq!(package.updates[0].version, "1.20.0"); + assert_eq!(package.updates[0].version_code, Some(1020000)); + assert_eq!(package.updates[0].source.as_deref(), Some("fdroid")); + assert_eq!( + package.updates[0].artifacts[0].file_name.as_deref(), + Some("org.fdroid.fdroid_1020000.apk") + ); + } + #[test] fn package_directory_can_read_package_local_files() { let temp = tempfile::tempdir().unwrap(); From d8cd8c97501c6ff20f49a7b0819eaeb372fde559 Mon Sep 17 00:00:00 2001 From: Xiangzhe Date: Sat, 27 Jun 2026 18:09:39 +0800 Subject: [PATCH 44/52] test(lua): cover GitHub luaclass shape --- crates/getter-core/src/lua.rs | 99 +++++++++++++++++++++++++++++++++++ 1 file changed, 99 insertions(+) diff --git a/crates/getter-core/src/lua.rs b/crates/getter-core/src/lua.rs index 186aa65..e45801c 100644 --- a/crates/getter-core/src/lua.rs +++ b/crates/getter-core/src/lua.rs @@ -1088,6 +1088,105 @@ return fdroid.package { ); } + #[test] + fn package_directory_can_use_repository_local_github_android_luaclass_shape() { + let temp = tempfile::tempdir().unwrap(); + let package_dir = temp.path().join("android/app/org.fdroid.fdroid"); + fs::create_dir_all(&package_dir).unwrap(); + fs::create_dir_all(temp.path().join("luaclass")).unwrap(); + fs::write( + temp.path().join("luaclass/github_android_apk.lua"), + r#" +local github_android = {} + +function github_android.package(spec) + if spec.owner ~= "f-droid" or spec.repo ~= "fdroidclient" then + error("GitHub project fixture mismatch") + end + local asset_name = "F-Droid.apk" + if spec.asset.include and not string.match(asset_name, spec.asset.include) then + error("GitHub asset include fixture mismatch") + end + if spec.asset.exclude and string.match(asset_name, spec.asset.exclude) then + error("GitHub asset exclude fixture mismatch") + end + return package_version { + name = spec.name, + installed = { + { kind = "android_package", package_name = spec.android_package }, + }, + source_priority = { "github" }, + updates = { + { + version = "v1.20.0", + source = "github", + artifacts = { + { + name = asset_name, + url = "https://github.com/f-droid/fdroidclient/releases/download/v1.20.0/F-Droid.apk", + file_name = asset_name, + }, + }, + }, + }, + } +end + +return github_android +"#, + ) + .unwrap(); + fs::write( + package_dir.join("metadata.jsonc"), + r#"{ + "type": "android:app", + "android": { "package_name": "org.fdroid.fdroid" } +}"#, + ) + .unwrap(); + fs::write( + package_dir.join("9999.lua"), + r#"#!/bin/upa-lua v1 +local github_android = require("luaclass.github_android_apk") +return github_android.package { + name = "F-Droid", + android_package = "org.fdroid.fdroid", + owner = "f-droid", + repo = "fdroidclient", + asset = { + include = "%.apk$", + exclude = "debug", + }, +} +"#, + ) + .unwrap(); + + let package = evaluate_single_package_directory(temp.path(), "official").unwrap(); + + assert_eq!(package.id.to_string(), "android/app/org.fdroid.fdroid"); + assert_eq!(package.repository.as_str(), "official"); + assert_eq!(package.name, "F-Droid"); + assert_eq!( + package.installed, + vec![InstalledTarget::AndroidPackage { + package_name: "org.fdroid.fdroid".to_owned() + }] + ); + assert_eq!(package.source_priority, vec!["github"]); + assert_eq!(package.updates.len(), 1); + assert_eq!(package.updates[0].version, "v1.20.0"); + assert_eq!(package.updates[0].source.as_deref(), Some("github")); + assert_eq!( + package.updates[0].artifacts[0].url, + "https://github.com/f-droid/fdroidclient/releases/download/v1.20.0/F-Droid.apk" + ); + assert_eq!( + package.updates[0].artifacts[0].file_name.as_deref(), + Some("F-Droid.apk") + ); + } + #[test] fn package_directory_can_read_package_local_files() { let temp = tempfile::tempdir().unwrap(); From 7eb6ab2d4b4cfe2515c0fcabf469b91c3ed04f12 Mon Sep 17 00:00:00 2001 From: Xiangzhe Date: Sat, 27 Jun 2026 18:39:57 +0800 Subject: [PATCH 45/52] feat(lua): add host binding injection seam --- crates/getter-core/src/lua.rs | 148 +++++++++++++++++++++++++++++++++- 1 file changed, 146 insertions(+), 2 deletions(-) diff --git a/crates/getter-core/src/lua.rs b/crates/getter-core/src/lua.rs index e45801c..cc1b084 100644 --- a/crates/getter-core/src/lua.rs +++ b/crates/getter-core/src/lua.rs @@ -53,6 +53,30 @@ pub fn evaluate_package_directory_script( metadata: &PackageDirectoryMetadata, script: &PackageVersionScript, ) -> Result { + evaluate_package_directory_script_with_host_bindings( + repository_id, + package, + metadata, + script, + |_| Ok(()), + ) +} + +/// Evaluates a package version script with caller-installed Lua host bindings. +/// +/// This is an internal seam for higher-level getter operations to provide +/// getter-owned host functions while keeping `getter-core` independent from +/// provider/cache crates. It does not define a stable public Lua provider API. +pub fn evaluate_package_directory_script_with_host_bindings( + repository_id: &RepositoryId, + package: &PackageDirectory, + metadata: &PackageDirectoryMetadata, + script: &PackageVersionScript, + install_host_bindings: F, +) -> Result +where + F: FnOnce(&Lua) -> mlua::Result<()>, +{ let source = fs::read_to_string(&script.path).map_err(|source| LuaPackageError::ReadFile { path: script.path.clone(), source, @@ -70,15 +94,20 @@ pub fn evaluate_package_directory_script( &LuaRepositoryEnvironment::PackageDirectory { package }, &script.path, &source, + install_host_bindings, )?; validate_package_directory_version_json(repository_id, package, metadata, script, json) } -fn evaluate_package_source_to_json( +fn evaluate_package_source_to_json( environment: &LuaRepositoryEnvironment<'_>, path: &Path, source: &str, -) -> Result { + install_host_bindings: F, +) -> Result +where + F: FnOnce(&Lua) -> mlua::Result<()>, +{ let path = path.to_path_buf(); let lua = Lua::new(); configure_package_path(&lua, environment).map_err(|source| LuaPackageError::Runtime { @@ -93,6 +122,10 @@ fn evaluate_package_source_to_json( path: path.clone(), source: Box::new(source), })?; + install_host_bindings(&lua).map_err(|source| LuaPackageError::Runtime { + path: path.clone(), + source: Box::new(source), + })?; let value = lua .load(source) @@ -1187,6 +1220,117 @@ return github_android.package { ); } + #[test] + fn package_directory_luaclass_can_call_injected_provider_host() { + let temp = tempfile::tempdir().unwrap(); + let package_dir = temp.path().join("android/app/org.fdroid.fdroid"); + fs::create_dir_all(&package_dir).unwrap(); + fs::create_dir_all(temp.path().join("luaclass")).unwrap(); + fs::write( + temp.path().join("luaclass/github_android_apk.lua"), + r#" +local github_android = {} + +function github_android.package(spec) + local releases = getter_test_provider.github_releases(spec.owner, spec.repo, spec.asset) + return package_version { + name = spec.name, + source_priority = { "github" }, + updates = releases, + } +end + +return github_android +"#, + ) + .unwrap(); + fs::write( + package_dir.join("metadata.jsonc"), + r#"{ + "type": "android:app", + "android": { "package_name": "org.fdroid.fdroid" } +}"#, + ) + .unwrap(); + fs::write( + package_dir.join("9999.lua"), + r#"#!/bin/upa-lua v1 +local github_android = require("luaclass.github_android_apk") +return github_android.package { + name = "F-Droid", + owner = "f-droid", + repo = "fdroidclient", + asset = { include = "%.apk$" }, +} +"#, + ) + .unwrap(); + let layout = RepositoryPackageDirectoryLayout::load(temp.path()).unwrap(); + let package_dir = &layout.packages[0]; + let metadata = layout.package_metadata(package_dir).unwrap(); + let script = layout.unambiguous_version_script(package_dir).unwrap(); + + let package = evaluate_package_directory_script_with_host_bindings( + &RepositoryId::new("official").unwrap(), + package_dir, + &metadata, + script, + |lua| { + let provider = lua.create_table()?; + provider.set( + "github_releases", + lua.create_function( + |lua, (owner, repo, asset): (String, String, Table)| { + if owner != "f-droid" || repo != "fdroidclient" { + return Err(mlua::Error::external("GitHub fixture mismatch")); + } + let include: String = asset.get("include")?; + if include != "%.apk$" { + return Err(mlua::Error::external("GitHub asset filter mismatch")); + } + let artifact = lua.create_table()?; + artifact.set("name", "F-Droid.apk")?; + artifact.set( + "url", + "https://github.com/f-droid/fdroidclient/releases/download/v1.20.0/F-Droid.apk", + )?; + artifact.set("file_name", "F-Droid.apk")?; + let artifacts = lua.create_table()?; + artifacts.raw_set(1, artifact)?; + let candidate = lua.create_table()?; + candidate.set("version", "v1.20.0")?; + candidate.set("source", "github")?; + candidate.set("artifacts", artifacts)?; + let candidates = lua.create_table()?; + candidates.raw_set(1, candidate)?; + Ok(candidates) + }, + )?, + )?; + lua.globals().set("getter_test_provider", provider) + }, + ) + .unwrap(); + + assert_eq!(package.id.to_string(), "android/app/org.fdroid.fdroid"); + assert_eq!(package.repository.as_str(), "official"); + assert_eq!(package.name, "F-Droid"); + assert_eq!( + package.installed, + vec![InstalledTarget::AndroidPackage { + package_name: "org.fdroid.fdroid".to_owned() + }] + ); + assert_eq!(package.source_priority, vec!["github"]); + assert_eq!(package.updates.len(), 1); + assert_eq!(package.updates[0].version, "v1.20.0"); + assert_eq!(package.updates[0].source.as_deref(), Some("github")); + assert_eq!( + package.updates[0].artifacts[0].file_name.as_deref(), + Some("F-Droid.apk") + ); + } + #[test] fn package_directory_can_read_package_local_files() { let temp = tempfile::tempdir().unwrap(); From 236cfaedf9d21833045337962cfcc375a9525c2d Mon Sep 17 00:00:00 2001 From: Xiangzhe Date: Sat, 27 Jun 2026 19:52:46 +0800 Subject: [PATCH 46/52] feat(lua): add dev provider host package eval --- Cargo.lock | 1 + crates/getter-operations/Cargo.toml | 4 +- crates/getter-operations/src/lib.rs | 3 + .../src/lua_provider_host.rs | 530 ++++++++++++++++++ 4 files changed, 537 insertions(+), 1 deletion(-) create mode 100644 crates/getter-operations/src/lua_provider_host.rs diff --git a/Cargo.lock b/Cargo.lock index c072fcc..a65d2b4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -618,6 +618,7 @@ dependencies = [ "getter-providers", "getter-storage", "json_comments", + "mlua", "serde", "serde_json", "sha2", diff --git a/crates/getter-operations/Cargo.toml b/crates/getter-operations/Cargo.toml index ed85e8c..81c2e82 100644 --- a/crates/getter-operations/Cargo.toml +++ b/crates/getter-operations/Cargo.toml @@ -5,13 +5,15 @@ edition.workspace = true [features] default = [] -lua = ["getter-core/lua"] +lua = ["getter-core/lua", "dep:mlua"] +lua-provider-host-dev = ["lua"] [dependencies] json_comments = "0.2" getter-core = { path = "../getter-core", default-features = false } getter-providers = { path = "../getter-providers" } getter-storage = { path = "../getter-storage" } +mlua = { version = "0.10", features = ["luajit", "vendored"], optional = true } serde = { version = "1", features = ["derive"] } serde_json = "1" sha2 = "0.10" diff --git a/crates/getter-operations/src/lib.rs b/crates/getter-operations/src/lib.rs index 30097ff..ae19c2d 100644 --- a/crates/getter-operations/src/lib.rs +++ b/crates/getter-operations/src/lib.rs @@ -11,6 +11,9 @@ pub mod fdroid_catalog; pub mod github_latest_commit; pub mod github_releases; pub mod legacy_room; +#[cfg(feature = "lua-provider-host-dev")] +#[doc(hidden)] +pub mod lua_provider_host; pub mod provider_cache; pub mod read_model; pub mod runtime; diff --git a/crates/getter-operations/src/lua_provider_host.rs b/crates/getter-operations/src/lua_provider_host.rs new file mode 100644 index 0000000..0d3b2b4 --- /dev/null +++ b/crates/getter-operations/src/lua_provider_host.rs @@ -0,0 +1,530 @@ +//! Operation-installed Lua provider host bindings. +//! +//! This module is an implementation bridge toward ADR-0012 provider-backed Lua +//! modules. It exercises higher-level getter operations installing host +//! functions for package evaluation without making `getter-core` depend on +//! provider cache/storage crates and without defining the final stable Lua +//! provider API. + +use crate::github_releases::{ + read_or_refresh_github_releases, GithubReleaseConfig, GithubReleaseOperationError, + DEFAULT_GITHUB_API_BASE_URL, GITHUB_ASSET_NOT_FOUND, GITHUB_PROVIDER_ID, +}; +use crate::provider_cache::{ProviderCacheDiagnostic, ProviderCacheMode, ProviderCacheSource}; +use getter_core::lua::{evaluate_package_directory_script_with_host_bindings, LuaPackageError}; +use getter_core::repository::{RepositoryLoadError, RepositoryPackageDirectoryLayout}; +use getter_core::{PackageId, RepositoryId, UpdateArtifact, UpdateCandidate}; +use getter_providers::{ + github_release_update_candidates, GithubAssetFilter, GithubProviderError, GithubRelease, + GithubReleaseCandidateOptions, +}; +use getter_storage::{CacheDb, MainDb, StorageError, StoredRepository}; +use mlua::{Lua, Table, Value as LuaValue}; +use serde::Deserialize; +use serde_json::{json, Value}; +use std::cell::RefCell; +use std::path::{Path, PathBuf}; +use std::rc::Rc; + +#[derive(Debug, thiserror::Error)] +pub enum LuaProviderHostOperationError { + #[error("invalid Lua provider-host request: {0}")] + InvalidRequest(String), + #[error("storage operation failed: {0}")] + Storage(#[from] StorageError), + #[error("repository operation failed: {0}")] + Repository(#[from] RepositoryLoadError), + #[error("package evaluation failed: {0}")] + PackageEval(#[from] LuaPackageError), + #[error("GitHub provider operation failed: {0}")] + Github(#[from] GithubReleaseOperationError), + #[error("GitHub provider normalization failed: {0}")] + GithubProvider(#[from] GithubProviderError), + #[error("Lua provider-host response serialization failed: {0}")] + Serialization(#[from] serde_json::Error), +} + +/// Evaluates a package with a fixture-backed GitHub release host binding. +/// +/// This is a development/provider-host tracer, not the stable package-eval or +/// product update-check API. Repository packages can exercise repository-local +/// `luaclass/` modules that call `getter_dev.github_release_candidates { ... }`, +/// while release parsing/cache behavior stays in `getter-operations`. +pub fn github_package_eval_json( + data_dir: &Path, + request_json: &str, +) -> Result { + let request: GithubPackageEvalRequest = serde_json::from_str(request_json) + .map_err(|source| LuaProviderHostOperationError::InvalidRequest(source.to_string()))?; + let mode = provider_cache_mode(request.mode.as_deref())?; + let api_base_url = request + .api_base_url + .filter(|value| !value.trim().is_empty()) + .unwrap_or_else(|| DEFAULT_GITHUB_API_BASE_URL.to_owned()); + let main_db = MainDb::open(data_dir.join("main.db"))?; + let repository = find_repository(&main_db, &request.repository_id)?; + let repository_path = repo_path(&repository)?; + let layout = RepositoryPackageDirectoryLayout::load(&repository_path)?; + let package_directory = layout.package(&request.package_id).ok_or_else(|| { + LuaProviderHostOperationError::InvalidRequest(format!( + "package '{}' was not found in repository '{}'", + request.package_id, request.repository_id + )) + })?; + let metadata = layout.package_metadata(package_directory)?; + let script = layout.unambiguous_version_script(package_directory)?; + let provider_calls = Rc::new(RefCell::new(Vec::new())); + let cache_db_path = data_dir.join("cache.db"); + let default_include_prereleases = request.include_prereleases; + let releases_json = request.releases_json; + let package = evaluate_package_directory_script_with_host_bindings( + &repository.id, + package_directory, + &metadata, + script, + { + let provider_calls = Rc::clone(&provider_calls); + move |lua| { + install_github_dev_host( + lua, + GithubDevHostConfig { + cache_db_path, + api_base_url, + mode, + releases_json, + default_include_prereleases, + provider_calls, + }, + ) + } + }, + )?; + let package = serde_json::to_value(package)?; + + Ok(json!({ + "operation": "github.package_eval.fixture", + "package": package, + "provider_calls": provider_calls.borrow().clone(), + })) +} + +#[derive(Debug, Deserialize)] +struct GithubPackageEvalRequest { + repository_id: RepositoryId, + package_id: PackageId, + #[serde(default)] + api_base_url: Option, + #[serde(default)] + mode: Option, + #[serde(default)] + releases_json: Option, + #[serde(default)] + include_prereleases: bool, +} + +struct GithubDevHostConfig { + cache_db_path: PathBuf, + api_base_url: String, + mode: ProviderCacheMode, + releases_json: Option, + default_include_prereleases: bool, + provider_calls: Rc>>, +} + +fn install_github_dev_host(lua: &Lua, config: GithubDevHostConfig) -> mlua::Result<()> { + let host = lua.create_table()?; + host.set( + "github_release_candidates", + lua.create_function(move |lua, spec: Table| { + let request = GithubReleaseHostRequest::from_lua_table(&spec)?; + let db = CacheDb::open(&config.cache_db_path).map_err(mlua::Error::external)?; + let provider_config = GithubReleaseConfig { + api_base_url: config.api_base_url.clone(), + owner: request.owner, + repo: request.repo, + }; + let refresh_fixture = config.releases_json.clone(); + let result = read_or_refresh_github_releases(&db, provider_config, config.mode, || { + refresh_fixture.ok_or_else(|| { + "fixture-backed GitHub release host requires releases_json".to_owned() + }) + }) + .map_err(mlua::Error::external)?; + let options = GithubReleaseCandidateOptions { + include_prereleases: request + .include_prereleases + .unwrap_or(config.default_include_prereleases), + asset_filter: request.asset_filter, + }; + let candidates = github_release_update_candidates(&result.releases, &options) + .map_err(mlua::Error::external)?; + let mut diagnostics = result + .diagnostics + .iter() + .map(provider_diagnostic_json) + .collect::>(); + if candidates.is_empty() + && github_eligible_release_count(&result.releases, &options) > 0 + { + diagnostics.push(json!({ + "code": GITHUB_ASSET_NOT_FOUND, + "message": "no GitHub release assets matched the requested filters", + "provider": GITHUB_PROVIDER_ID, + "owner": result.config.owner, + "repo": result.config.repo, + "cache_key": result.cache_key, + })); + } + config.provider_calls.borrow_mut().push(json!({ + "provider": GITHUB_PROVIDER_ID, + "request": "releases", + "owner": result.config.owner, + "repo": result.config.repo, + "cache_key": result.cache_key, + "source": provider_source_json(result.source), + "diagnostics": diagnostics, + })); + update_candidates_to_lua(lua, &candidates) + })?, + )?; + lua.globals().set("getter_dev", host) +} + +struct GithubReleaseHostRequest { + owner: String, + repo: String, + asset_filter: GithubAssetFilter, + include_prereleases: Option, +} + +impl GithubReleaseHostRequest { + fn from_lua_table(table: &Table) -> mlua::Result { + Ok(Self { + owner: required_lua_string(table, "owner")?, + repo: required_lua_string(table, "repo")?, + asset_filter: asset_filter_from_lua(table.get("asset")?)?, + include_prereleases: optional_lua_bool(table, "include_prereleases")?, + }) + } +} + +fn provider_cache_mode( + mode: Option<&str>, +) -> Result { + match mode { + Some("force_refresh") => Ok(ProviderCacheMode::ForceRefresh), + Some("use_cached") | None => Ok(ProviderCacheMode::UseCached), + Some(other) => Err(LuaProviderHostOperationError::InvalidRequest(format!( + "unknown mode '{other}'" + ))), + } +} + +fn find_repository( + db: &MainDb, + repository_id: &RepositoryId, +) -> Result { + db.repositories()? + .into_iter() + .find(|repository| &repository.id == repository_id) + .ok_or_else(|| { + LuaProviderHostOperationError::InvalidRequest(format!( + "repository '{repository_id}' is not registered" + )) + }) +} + +fn repo_path(repository: &StoredRepository) -> Result { + repository.path.as_ref().map(PathBuf::from).ok_or_else(|| { + LuaProviderHostOperationError::InvalidRequest(format!( + "repository '{}' has no path", + repository.id + )) + }) +} + +fn required_lua_string(table: &Table, field: &'static str) -> mlua::Result { + let value: Option = table.get(field)?; + value + .filter(|value| !value.trim().is_empty()) + .ok_or_else(|| { + mlua::Error::external(format!( + "getter_dev.github_release_candidates missing {field}" + )) + }) +} + +fn optional_lua_bool(table: &Table, field: &'static str) -> mlua::Result> { + match table.get::(field)? { + LuaValue::Nil => Ok(None), + LuaValue::Boolean(value) => Ok(Some(value)), + _ => Err(mlua::Error::external(format!( + "getter_dev.github_release_candidates {field} must be a boolean" + ))), + } +} + +fn asset_filter_from_lua(value: LuaValue) -> mlua::Result { + match value { + LuaValue::Nil => Ok(GithubAssetFilter::default()), + LuaValue::Table(table) => Ok(GithubAssetFilter { + include: optional_lua_string(&table, "include")?, + exclude: optional_lua_string(&table, "exclude")?, + }), + _ => Err(mlua::Error::external( + "getter_dev.github_release_candidates asset must be a table", + )), + } +} + +fn optional_lua_string(table: &Table, field: &'static str) -> mlua::Result> { + match table.get::(field)? { + LuaValue::Nil => Ok(None), + LuaValue::String(value) => Ok(Some(value.to_str()?.to_owned())), + _ => Err(mlua::Error::external(format!( + "getter_dev.github_release_candidates asset.{field} must be a string" + ))), + } +} + +fn github_eligible_release_count( + releases: &[GithubRelease], + options: &GithubReleaseCandidateOptions, +) -> usize { + releases + .iter() + .filter(|release| !release.draft) + .filter(|release| options.include_prereleases || !release.prerelease) + .count() +} + +fn update_candidates_to_lua(lua: &Lua, candidates: &[UpdateCandidate]) -> mlua::Result { + if candidates.is_empty() { + return Ok(LuaValue::Nil); + } + + let table = lua.create_table()?; + for (index, candidate) in candidates.iter().enumerate() { + table.raw_set(index + 1, update_candidate_to_lua(lua, candidate)?)?; + } + Ok(LuaValue::Table(table)) +} + +fn update_candidate_to_lua(lua: &Lua, candidate: &UpdateCandidate) -> mlua::Result
{ + let table = lua.create_table()?; + table.set("version", candidate.version.as_str())?; + if let Some(version_code) = candidate.version_code { + table.set("version_code", version_code)?; + } + if let Some(channel) = candidate.channel.as_ref() { + table.set("channel", channel.as_str())?; + } + if let Some(source) = candidate.source.as_ref() { + table.set("source", source.as_str())?; + } + let artifacts = lua.create_table()?; + for (index, artifact) in candidate.artifacts.iter().enumerate() { + artifacts.raw_set(index + 1, update_artifact_to_lua(lua, artifact)?)?; + } + table.set("artifacts", artifacts)?; + Ok(table) +} + +fn update_artifact_to_lua(lua: &Lua, artifact: &UpdateArtifact) -> mlua::Result
{ + let table = lua.create_table()?; + table.set("name", artifact.name.as_str())?; + table.set("url", artifact.url.as_str())?; + if let Some(file_name) = artifact.file_name.as_ref() { + table.set("file_name", file_name.as_str())?; + } + if let Some(sha256) = artifact.sha256.as_ref() { + table.set("sha256", sha256.as_str())?; + } + if let Some(size) = artifact.size { + table.set("size", size)?; + } + Ok(table) +} + +fn provider_source_json(source: ProviderCacheSource) -> &'static str { + match source { + ProviderCacheSource::Cache => "cache", + ProviderCacheSource::Refreshed => "refreshed", + ProviderCacheSource::Stale => "stale", + } +} + +fn provider_diagnostic_json(diagnostic: &ProviderCacheDiagnostic) -> Value { + json!({ + "code": diagnostic.code, + "message": diagnostic.message, + "cache_key": diagnostic.cache_key, + "provider": diagnostic.provider, + "stale_fetched_at_unix": diagnostic.stale_fetched_at_unix, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use getter_core::repository::{RepositoryMetadata, REPO_API_VERSION_V1}; + use getter_core::{RepositoryId, RepositoryPriority}; + use getter_storage::CacheDb; + use serde_json::json; + use std::fs; + + const GITHUB_RELEASES_FIXTURE: &str = r#"[ + { + "tag_name": "v1.20.0", + "name": "F-Droid 1.20.0", + "draft": false, + "prerelease": false, + "published_at": "2026-06-01T00:00:00Z", + "assets": [ + { + "name": "F-Droid.apk", + "browser_download_url": "https://github.com/f-droid/fdroidclient/releases/download/v1.20.0/F-Droid.apk", + "content_type": "application/vnd.android.package-archive", + "size": 12345, + "digest": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + } + ] + } +]"#; + + #[test] + fn github_package_eval_uses_repository_luaclass_and_provider_cache() { + let temp = tempfile::tempdir().unwrap(); + let data_dir = temp.path(); + write_github_package_fixture(data_dir, "[.]apk$"); + + let first = github_package_eval_json( + data_dir, + &json!({ + "repository_id": "official", + "package_id": "android/app/org.fdroid.fdroid", + "releases_json": GITHUB_RELEASES_FIXTURE + }) + .to_string(), + ) + .unwrap(); + + assert_eq!(first["operation"], "github.package_eval.fixture"); + assert_eq!(first["provider_calls"][0]["source"], "refreshed"); + assert_eq!(first["package"]["name"], "F-Droid"); + assert_eq!(first["package"]["source_priority"], json!(["github"])); + assert_eq!(first["package"]["updates"][0]["version"], "v1.20.0"); + assert_eq!(first["package"]["updates"][0]["source"], "github"); + assert_eq!( + first["package"]["updates"][0]["artifacts"][0]["sha256"], + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + ); + let cache_key = first["provider_calls"][0]["cache_key"].as_str().unwrap(); + assert!(CacheDb::open(data_dir.join("cache.db")) + .unwrap() + .provider_response(cache_key) + .unwrap() + .is_some()); + + let second = github_package_eval_json( + data_dir, + &json!({ + "repository_id": "official", + "package_id": "android/app/org.fdroid.fdroid" + }) + .to_string(), + ) + .unwrap(); + + assert_eq!(second["provider_calls"][0]["source"], "cache"); + assert_eq!(second["package"]["updates"][0]["version"], "v1.20.0"); + } + + #[test] + fn github_package_eval_reports_asset_filter_diagnostics() { + let temp = tempfile::tempdir().unwrap(); + let data_dir = temp.path(); + write_github_package_fixture(data_dir, "[.]zip$"); + + let result = github_package_eval_json( + data_dir, + &json!({ + "repository_id": "official", + "package_id": "android/app/org.fdroid.fdroid", + "releases_json": GITHUB_RELEASES_FIXTURE + }) + .to_string(), + ) + .unwrap(); + + assert_eq!(result["package"]["updates"], json!([])); + assert_eq!( + result["provider_calls"][0]["diagnostics"][0]["code"], + GITHUB_ASSET_NOT_FOUND + ); + assert_eq!( + result["provider_calls"][0]["diagnostics"][0]["provider"], + GITHUB_PROVIDER_ID + ); + } + + fn write_github_package_fixture(data_dir: &std::path::Path, asset_include: &str) { + let repo_root = data_dir.join("repo/official"); + let package_dir = repo_root.join("android/app/org.fdroid.fdroid"); + fs::create_dir_all(repo_root.join("luaclass")).unwrap(); + fs::create_dir_all(&package_dir).unwrap(); + fs::write( + repo_root.join("luaclass/github_android_apk.lua"), + r#" +local github_android = {} + +function github_android.package(spec) + return package_version { + name = spec.name, + source_priority = { "github" }, + updates = getter_dev.github_release_candidates { + owner = spec.owner, + repo = spec.repo, + asset = spec.asset, + }, + } +end + +return github_android +"#, + ) + .unwrap(); + fs::write( + package_dir.join("metadata.jsonc"), + r#"{ + "type": "android:app", + "android": { "package_name": "org.fdroid.fdroid" } +}"#, + ) + .unwrap(); + let version_script = r#"#!/bin/upa-lua v1 +local github_android = require("luaclass.github_android_apk") +return github_android.package { + name = "F-Droid", + owner = "f-droid", + repo = "fdroidclient", + asset = { include = "__ASSET_INCLUDE__" }, +} +"# + .replace("__ASSET_INCLUDE__", asset_include); + fs::write(package_dir.join("9999.lua"), version_script).unwrap(); + let main_db = MainDb::open(data_dir.join("main.db")).unwrap(); + main_db + .upsert_repository( + &RepositoryMetadata { + id: RepositoryId::new("official").unwrap(), + name: "Official".to_owned(), + priority: RepositoryPriority::DEFAULT, + api_version: REPO_API_VERSION_V1.to_owned(), + }, + Some(&repo_root), + None, + ) + .unwrap(); + } +} From 0f74313b52e3cc6083223125376dfa2a3b122a80 Mon Sep 17 00:00:00 2001 From: Xiangzhe Date: Sat, 27 Jun 2026 20:40:11 +0800 Subject: [PATCH 47/52] feat(lua): add F-Droid dev provider host eval --- .../src/lua_provider_host.rs | 334 +++++++++++++++++- 1 file changed, 331 insertions(+), 3 deletions(-) diff --git a/crates/getter-operations/src/lua_provider_host.rs b/crates/getter-operations/src/lua_provider_host.rs index 0d3b2b4..ee16e1a 100644 --- a/crates/getter-operations/src/lua_provider_host.rs +++ b/crates/getter-operations/src/lua_provider_host.rs @@ -6,6 +6,10 @@ //! provider cache/storage crates and without defining the final stable Lua //! provider API. +use crate::fdroid_catalog::{ + read_or_refresh_fdroid_catalog, FdroidCatalogOperationError, FdroidEndpointConfig, + DEFAULT_FDROID_ENDPOINT_ID, DEFAULT_FDROID_ENDPOINT_URL, FDROID_PROVIDER_ID, +}; use crate::github_releases::{ read_or_refresh_github_releases, GithubReleaseConfig, GithubReleaseOperationError, DEFAULT_GITHUB_API_BASE_URL, GITHUB_ASSET_NOT_FOUND, GITHUB_PROVIDER_ID, @@ -15,8 +19,8 @@ use getter_core::lua::{evaluate_package_directory_script_with_host_bindings, Lua use getter_core::repository::{RepositoryLoadError, RepositoryPackageDirectoryLayout}; use getter_core::{PackageId, RepositoryId, UpdateArtifact, UpdateCandidate}; use getter_providers::{ - github_release_update_candidates, GithubAssetFilter, GithubProviderError, GithubRelease, - GithubReleaseCandidateOptions, + github_release_update_candidates, FdroidEndpoint, GithubAssetFilter, GithubProviderError, + GithubRelease, GithubReleaseCandidateOptions, }; use getter_storage::{CacheDb, MainDb, StorageError, StoredRepository}; use mlua::{Lua, Table, Value as LuaValue}; @@ -36,6 +40,8 @@ pub enum LuaProviderHostOperationError { Repository(#[from] RepositoryLoadError), #[error("package evaluation failed: {0}")] PackageEval(#[from] LuaPackageError), + #[error("F-Droid provider operation failed: {0}")] + Fdroid(#[from] FdroidCatalogOperationError), #[error("GitHub provider operation failed: {0}")] Github(#[from] GithubReleaseOperationError), #[error("GitHub provider normalization failed: {0}")] @@ -44,6 +50,98 @@ pub enum LuaProviderHostOperationError { Serialization(#[from] serde_json::Error), } +const FDROID_PACKAGE_NOT_FOUND: &str = "provider.fdroid.package_not_found"; + +/// Evaluates a package with a fixture-backed F-Droid catalog host binding. +/// +/// This is a development/provider-host tracer, not the stable package-eval or +/// product update-check API. Repository packages can exercise repository-local +/// `luaclass/` modules that call `getter_dev.fdroid_update_candidates { ... }`, +/// while catalog parsing/cache behavior stays in `getter-operations`. +pub fn fdroid_package_eval_json( + data_dir: &Path, + request_json: &str, +) -> Result { + let request: FdroidPackageEvalRequest = serde_json::from_str(request_json) + .map_err(|source| LuaProviderHostOperationError::InvalidRequest(source.to_string()))?; + let mode = provider_cache_mode(request.mode.as_deref())?; + let endpoint = FdroidEndpointConfig { + endpoint_id: request + .endpoint_id + .filter(|value| !value.trim().is_empty()) + .unwrap_or_else(|| DEFAULT_FDROID_ENDPOINT_ID.to_owned()), + endpoint_url: request + .endpoint_url + .filter(|value| !value.trim().is_empty()) + .unwrap_or_else(|| DEFAULT_FDROID_ENDPOINT_URL.to_owned()), + }; + let main_db = MainDb::open(data_dir.join("main.db"))?; + let repository = find_repository(&main_db, &request.repository_id)?; + let repository_path = repo_path(&repository)?; + let layout = RepositoryPackageDirectoryLayout::load(&repository_path)?; + let package_directory = layout.package(&request.package_id).ok_or_else(|| { + LuaProviderHostOperationError::InvalidRequest(format!( + "package '{}' was not found in repository '{}'", + request.package_id, request.repository_id + )) + })?; + let metadata = layout.package_metadata(package_directory)?; + let script = layout.unambiguous_version_script(package_directory)?; + let provider_calls = Rc::new(RefCell::new(Vec::new())); + let cache_db_path = data_dir.join("cache.db"); + let index_xml = request.index_xml; + let package = evaluate_package_directory_script_with_host_bindings( + &repository.id, + package_directory, + &metadata, + script, + { + let provider_calls = Rc::clone(&provider_calls); + move |lua| { + install_fdroid_dev_host( + lua, + FdroidDevHostConfig { + cache_db_path, + endpoint, + mode, + index_xml, + provider_calls, + }, + ) + } + }, + )?; + let package = serde_json::to_value(package)?; + + Ok(json!({ + "operation": "fdroid.package_eval.fixture", + "package": package, + "provider_calls": provider_calls.borrow().clone(), + })) +} + +#[derive(Debug, Deserialize)] +struct FdroidPackageEvalRequest { + repository_id: RepositoryId, + package_id: PackageId, + #[serde(default)] + endpoint_id: Option, + #[serde(default)] + endpoint_url: Option, + #[serde(default)] + mode: Option, + #[serde(default)] + index_xml: Option, +} + +struct FdroidDevHostConfig { + cache_db_path: PathBuf, + endpoint: FdroidEndpointConfig, + mode: ProviderCacheMode, + index_xml: Option, + provider_calls: Rc>>, +} + /// Evaluates a package with a fixture-backed GitHub release host binding. /// /// This is a development/provider-host tracer, not the stable package-eval or @@ -131,6 +229,82 @@ struct GithubDevHostConfig { provider_calls: Rc>>, } +fn install_fdroid_dev_host(lua: &Lua, config: FdroidDevHostConfig) -> mlua::Result<()> { + let host = lua.create_table()?; + host.set( + "fdroid_update_candidates", + lua.create_function(move |lua, spec: Table| { + let request = FdroidUpdateHostRequest::from_lua_table(&spec)?; + let db = CacheDb::open(&config.cache_db_path).map_err(mlua::Error::external)?; + let endpoint = config.endpoint.clone(); + let refresh_fixture = config.index_xml.clone(); + let result = read_or_refresh_fdroid_catalog(&db, endpoint, config.mode, || { + refresh_fixture.ok_or_else(|| { + "fixture-backed F-Droid catalog host requires index_xml".to_owned() + }) + }) + .map_err(mlua::Error::external)?; + let mut diagnostics = result + .diagnostics + .iter() + .map(provider_diagnostic_json) + .collect::>(); + let candidates = result + .app(&request.package_name) + .map(|app| { + let endpoint = FdroidEndpoint { + name: result.catalog.endpoint.name.clone(), + url: Some(result.endpoint.endpoint_url.clone()), + timestamp: result.catalog.endpoint.timestamp.clone(), + }; + app.update_candidates(&endpoint) + }) + .unwrap_or_else(|| { + diagnostics.push(json!({ + "code": FDROID_PACKAGE_NOT_FOUND, + "message": format!("F-Droid package '{}' was not found in endpoint '{}'", request.package_name, result.endpoint.endpoint_id), + "provider": FDROID_PROVIDER_ID, + "endpoint_id": result.endpoint.endpoint_id, + "package_name": request.package_name, + "cache_key": result.cache_key, + })); + Vec::new() + }); + config.provider_calls.borrow_mut().push(json!({ + "provider": FDROID_PROVIDER_ID, + "request": "catalog", + "endpoint_id": result.endpoint.endpoint_id, + "endpoint_url": result.endpoint.endpoint_url, + "package_name": request.package_name, + "cache_key": result.cache_key, + "source": provider_source_json(result.source), + "diagnostics": diagnostics, + })); + update_candidates_to_lua(lua, &candidates) + })?, + )?; + lua.globals().set("getter_dev", host) +} + +struct FdroidUpdateHostRequest { + package_name: String, +} + +impl FdroidUpdateHostRequest { + fn from_lua_table(table: &Table) -> mlua::Result { + let package_name: Option = table.get("package_name")?; + Ok(Self { + package_name: package_name + .filter(|value| !value.trim().is_empty()) + .ok_or_else(|| { + mlua::Error::external( + "getter_dev.fdroid_update_candidates missing package_name", + ) + })?, + }) + } +} + fn install_github_dev_host(lua: &Lua, config: GithubDevHostConfig) -> mlua::Result<()> { let host = lua.create_table()?; host.set( @@ -373,6 +547,23 @@ mod tests { use serde_json::json; use std::fs; + const FDROID_INDEX_FIXTURE: &str = r#" + + + + F-Droid + App repository client + + 1.20.0 + 1020000 + org.fdroid.fdroid_1020000.apk + aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa + 1234567 + + + +"#; + const GITHUB_RELEASES_FIXTURE: &str = r#"[ { "tag_name": "v1.20.0", @@ -392,6 +583,96 @@ mod tests { } ]"#; + #[test] + fn fdroid_package_eval_uses_repository_luaclass_and_provider_cache() { + let temp = tempfile::tempdir().unwrap(); + let data_dir = temp.path(); + write_fdroid_package_fixture(data_dir, "org.fdroid.fdroid"); + + let first = fdroid_package_eval_json( + data_dir, + &json!({ + "repository_id": "official", + "package_id": "android/f-droid/app/org.fdroid.fdroid", + "index_xml": FDROID_INDEX_FIXTURE + }) + .to_string(), + ) + .unwrap(); + + assert_eq!(first["operation"], "fdroid.package_eval.fixture"); + assert_eq!(first["provider_calls"][0]["source"], "refreshed"); + assert_eq!(first["provider_calls"][0]["endpoint_id"], "official"); + assert_eq!( + first["provider_calls"][0]["package_name"], + "org.fdroid.fdroid" + ); + assert_eq!(first["package"]["name"], "F-Droid"); + assert_eq!(first["package"]["source_priority"], json!(["fdroid"])); + assert_eq!(first["package"]["updates"][0]["version"], "1.20.0"); + assert_eq!(first["package"]["updates"][0]["version_code"], 1020000); + assert_eq!(first["package"]["updates"][0]["source"], "fdroid"); + assert_eq!( + first["package"]["updates"][0]["artifacts"][0]["url"], + "https://f-droid.org/repo/org.fdroid.fdroid_1020000.apk" + ); + assert_eq!( + first["package"]["updates"][0]["artifacts"][0]["sha256"], + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + ); + let cache_key = first["provider_calls"][0]["cache_key"].as_str().unwrap(); + assert!(CacheDb::open(data_dir.join("cache.db")) + .unwrap() + .provider_response(cache_key) + .unwrap() + .is_some()); + + let second = fdroid_package_eval_json( + data_dir, + &json!({ + "repository_id": "official", + "package_id": "android/f-droid/app/org.fdroid.fdroid" + }) + .to_string(), + ) + .unwrap(); + + assert_eq!(second["provider_calls"][0]["source"], "cache"); + assert_eq!(second["package"]["updates"][0]["version"], "1.20.0"); + } + + #[test] + fn fdroid_package_eval_reports_package_not_found_diagnostics() { + let temp = tempfile::tempdir().unwrap(); + let data_dir = temp.path(); + write_fdroid_package_fixture(data_dir, "missing.package"); + + let result = fdroid_package_eval_json( + data_dir, + &json!({ + "repository_id": "official", + "package_id": "android/f-droid/app/org.fdroid.fdroid", + "index_xml": FDROID_INDEX_FIXTURE + }) + .to_string(), + ) + .unwrap(); + + assert_eq!(result["package"]["updates"], json!([])); + assert_eq!( + result["provider_calls"][0]["diagnostics"][0]["code"], + FDROID_PACKAGE_NOT_FOUND + ); + assert_eq!( + result["provider_calls"][0]["diagnostics"][0]["provider"], + FDROID_PROVIDER_ID + ); + assert_eq!( + result["provider_calls"][0]["diagnostics"][0]["package_name"], + "missing.package" + ); + } + #[test] fn github_package_eval_uses_repository_luaclass_and_provider_cache() { let temp = tempfile::tempdir().unwrap(); @@ -468,6 +749,49 @@ mod tests { ); } + fn write_fdroid_package_fixture(data_dir: &std::path::Path, package_name: &str) { + let repo_root = data_dir.join("repo/official"); + let package_dir = repo_root.join("android/f-droid/app/org.fdroid.fdroid"); + fs::create_dir_all(repo_root.join("luaclass")).unwrap(); + fs::create_dir_all(&package_dir).unwrap(); + fs::write( + repo_root.join("luaclass/fdroid_android.lua"), + r#" +local fdroid = {} + +function fdroid.package(spec) + return package_version { + source_priority = { "fdroid" }, + updates = getter_dev.fdroid_update_candidates { + package_name = spec.package_name, + }, + } +end + +return fdroid +"#, + ) + .unwrap(); + fs::write( + package_dir.join("metadata.jsonc"), + r#"{ + "display_name": "F-Droid", + "type": "android:app", + "android": { "package_name": "org.fdroid.fdroid" } +}"#, + ) + .unwrap(); + let version_script = r#"#!/bin/upa-lua v1 +local fdroid = require("luaclass.fdroid_android") +return fdroid.package { + package_name = "__PACKAGE_NAME__", +} +"# + .replace("__PACKAGE_NAME__", package_name); + fs::write(package_dir.join("9999.lua"), version_script).unwrap(); + register_official_repository(data_dir, &repo_root); + } + fn write_github_package_fixture(data_dir: &std::path::Path, asset_include: &str) { let repo_root = data_dir.join("repo/official"); let package_dir = repo_root.join("android/app/org.fdroid.fdroid"); @@ -513,6 +837,10 @@ return github_android.package { "# .replace("__ASSET_INCLUDE__", asset_include); fs::write(package_dir.join("9999.lua"), version_script).unwrap(); + register_official_repository(data_dir, &repo_root); + } + + fn register_official_repository(data_dir: &std::path::Path, repo_root: &std::path::Path) { let main_db = MainDb::open(data_dir.join("main.db")).unwrap(); main_db .upsert_repository( @@ -522,7 +850,7 @@ return github_android.package { priority: RepositoryPriority::DEFAULT, api_version: REPO_API_VERSION_V1.to_owned(), }, - Some(&repo_root), + Some(repo_root), None, ) .unwrap(); From 517bbcf43d4244b93bf8b0f49077905aaf3b630f Mon Sep 17 00:00:00 2001 From: Xiangzhe Date: Sat, 27 Jun 2026 21:56:44 +0800 Subject: [PATCH 48/52] feat(lua): add builtin luaclass fallback --- crates/getter-core/src/lua.rs | 160 ++++++++++++++------ crates/getter-core/src/luaclass/android.lua | 12 ++ 2 files changed, 123 insertions(+), 49 deletions(-) create mode 100644 crates/getter-core/src/luaclass/android.lua diff --git a/crates/getter-core/src/lua.rs b/crates/getter-core/src/lua.rs index cc1b084..baad1ce 100644 --- a/crates/getter-core/src/lua.rs +++ b/crates/getter-core/src/lua.rs @@ -8,11 +8,14 @@ use crate::{ InstalledTarget, PackageId, PackagePermissions, RepositoryId, ResolvedPackage, UpdateArtifact, UpdateCandidate, }; -use mlua::{Lua, Table, Value}; +use mlua::{Function, Lua, Table, Value}; use serde_json::{Map, Number, Value as JsonValue}; use std::fs; use std::path::{Path, PathBuf}; +const BUILTIN_LUACLASS_MODULES: &[(&str, &str)] = + &[("android", include_str!("luaclass/android.lua"))]; + #[derive(Debug, thiserror::Error)] pub enum LuaPackageError { #[error("failed to read Lua package file {path}: {source}")] @@ -158,7 +161,8 @@ fn configure_package_path( package: package_dir, } => { package.set("path", "")?; - install_luaclass_prefix_searcher( + install_builtin_luaclass_searcher(lua, &package)?; + install_repository_luaclass_searcher( lua, &package, package_directory_repository_root(package_dir).join(REPOSITORY_LUACLASS_DIR), @@ -198,62 +202,85 @@ fn remove_unsafe_globals(lua: &Lua) -> mlua::Result<()> { Ok(()) } -fn install_luaclass_prefix_searcher( +fn install_repository_luaclass_searcher( lua: &Lua, package: &Table, luaclass_dir: PathBuf, ) -> mlua::Result<()> { - install_prefixed_file_searcher( + install_searcher( lua, package, - "luaclass.", - luaclass_dir, - "constrained repository luaclass searcher only handles luaclass.* modules", + lua.create_function(move |lua, module: String| { + let Some(module) = module.strip_prefix("luaclass.") else { + return lua + .create_string("\n\tconstrained repository luaclass searcher only handles luaclass.* modules") + .map(Value::String); + }; + + let Some(relative_module) = module_to_relative_path(module) else { + return lua + .create_string(format!("\n\tinvalid repository module name 'luaclass.{module}'")) + .map(Value::String); + }; + + let module_path = luaclass_dir.join(&relative_module).with_extension("lua"); + let init_path = luaclass_dir.join(&relative_module).join("init.lua"); + for candidate in [&module_path, &init_path] { + if candidate.is_file() { + let source = fs::read_to_string(candidate).map_err(mlua::Error::external)?; + let chunk = lua + .load(&source) + .set_name(candidate.to_string_lossy().as_ref()); + return chunk.into_function().map(Value::Function); + } + } + + lua.create_string(format!( + "\n\tno repository module 'luaclass.{module}' in {}", + luaclass_dir.display() + )) + .map(Value::String) + })?, ) } -fn install_prefixed_file_searcher( - lua: &Lua, - package: &Table, - prefix: &'static str, - module_dir: PathBuf, - wrong_prefix_message: &'static str, -) -> mlua::Result<()> { - let searchers = package_searchers(package)?; - let searcher = lua.create_function(move |lua, module: String| { - let Some(module) = module.strip_prefix(prefix) else { - return lua - .create_string(format!("\n\t{wrong_prefix_message}")) - .map(Value::String); - }; - - let Some(relative_module) = module_to_relative_path(module) else { - return lua - .create_string(format!( - "\n\tinvalid repository module name '{prefix}{module}'" - )) - .map(Value::String); - }; - - let module_path = module_dir.join(&relative_module).with_extension("lua"); - let init_path = module_dir.join(&relative_module).join("init.lua"); - for candidate in [&module_path, &init_path] { - if candidate.is_file() { - let source = fs::read_to_string(candidate).map_err(mlua::Error::external)?; - let chunk = lua - .load(&source) - .set_name(candidate.to_string_lossy().as_ref()); - return chunk.into_function().map(Value::Function); +fn install_builtin_luaclass_searcher(lua: &Lua, package: &Table) -> mlua::Result<()> { + install_searcher( + lua, + package, + lua.create_function(move |lua, module: String| { + let Some(module) = module.strip_prefix("luaclass.") else { + return lua + .create_string("\n\tbuiltin luaclass searcher only handles luaclass.* modules") + .map(Value::String); + }; + if module_to_relative_path(module).is_none() { + return lua + .create_string(format!( + "\n\tinvalid builtin luaclass module name 'luaclass.{module}'" + )) + .map(Value::String); } - } - - lua.create_string(format!( - "\n\tno repository module '{prefix}{module}' in {}", - module_dir.display() - )) - .map(Value::String) - })?; + let Some((_, source)) = BUILTIN_LUACLASS_MODULES + .iter() + .find(|(name, _)| *name == module) + else { + return lua + .create_string(format!( + "\n\tno builtin luaclass module 'luaclass.{module}'" + )) + .map(Value::String); + }; + lua.load(*source) + .set_name(format!("@builtin/luaclass/{module}.lua")) + .into_function() + .map(Value::Function) + })?, + ) +} +fn install_searcher(_lua: &Lua, package: &Table, searcher: Function) -> mlua::Result<()> { + let searchers = package_searchers(package)?; let len = searchers.raw_len(); for index in (2..=len).rev() { let value: Value = searchers.raw_get(index)?; @@ -984,7 +1011,7 @@ return package_version { id = "android/app/com.example.autogen" } } #[test] - fn package_directory_can_load_luaclass_modules() { + fn package_directory_can_load_repository_luaclass_modules() { let temp = tempfile::tempdir().unwrap(); let package_dir = temp.path().join("android/app/com.example.autogen"); fs::create_dir_all(&package_dir).unwrap(); @@ -995,6 +1022,7 @@ return package_version { id = "android/app/com.example.autogen" } return { package_version = function(input) return { + name = "repository module", installed = input.installed, updates = input.updates, } @@ -1037,7 +1065,41 @@ return android.package_version { ) .unwrap(); - assert_eq!(package.name, "Example Autogen"); + assert_eq!(package.name, "repository module"); + assert_eq!(package.installed.len(), 1); + } + + #[test] + fn package_directory_can_load_builtin_luaclass_modules() { + let temp = tempfile::tempdir().unwrap(); + let package_dir = temp.path().join("android/app/com.example.autogen"); + fs::create_dir_all(&package_dir).unwrap(); + fs::write( + package_dir.join("metadata.jsonc"), + r#"{ + "type": "android:app", + "display_name": "Example Autogen", + "android": { "package_name": "com.example.autogen" } +}"#, + ) + .unwrap(); + fs::write( + package_dir.join("9999.lua"), + r#"#!/bin/upa-lua v1 +local android = require("luaclass.android") +return android.package_version { + name = "builtin module", + installed = { + { kind = "android_package", package_name = "com.example.autogen" }, + }, +} +"#, + ) + .unwrap(); + + let package = evaluate_single_package_directory(temp.path(), "official").unwrap(); + + assert_eq!(package.name, "builtin module"); assert_eq!(package.installed.len(), 1); } diff --git a/crates/getter-core/src/luaclass/android.lua b/crates/getter-core/src/luaclass/android.lua new file mode 100644 index 0000000..05ee382 --- /dev/null +++ b/crates/getter-core/src/luaclass/android.lua @@ -0,0 +1,12 @@ +local android = {} + +function android.package_version(input) + return package_version { + name = input.name, + installed = input.installed, + source_priority = input.source_priority, + updates = input.updates, + } +end + +return android From 759e7b8e1269b851383d57d2fea93916f2d161f7 Mon Sep 17 00:00:00 2001 From: Xiangzhe Date: Sat, 27 Jun 2026 22:39:47 +0800 Subject: [PATCH 49/52] feat(lua): add HTTP host seam --- crates/getter-core/src/lua.rs | 302 +++++++++++++++++++++++++++++++++- 1 file changed, 296 insertions(+), 6 deletions(-) diff --git a/crates/getter-core/src/lua.rs b/crates/getter-core/src/lua.rs index baad1ce..28857c9 100644 --- a/crates/getter-core/src/lua.rs +++ b/crates/getter-core/src/lua.rs @@ -10,6 +10,7 @@ use crate::{ }; use mlua::{Function, Lua, Table, Value}; use serde_json::{Map, Number, Value as JsonValue}; +use std::collections::BTreeMap; use std::fs; use std::path::{Path, PathBuf}; @@ -345,17 +346,137 @@ fn install_package_file_helpers(lua: &Lua, package: &PackageDirectory) -> mlua:: })?; let globals = lua.globals(); - let getter_builtin = match globals.get::("getter_builtin")? { - Value::Table(table) => table, + let getter_builtin = getter_builtin_table(lua)?; + getter_builtin.set("read_package_file", read_package_file.clone())?; + globals.set("read_package_file", read_package_file) +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct LuaHttpGetRequest { + pub url: String, + pub headers: BTreeMap, + pub cache: bool, +} + +/// Installs the Lua `http_get` host seam for a caller-provided transport. +/// +/// Plain package evaluation does not install this function. Higher-level +/// getter operations/runtime code must deliberately provide a handler and own +/// provider policy, permission checks, Manifest validation, and cache behavior. +pub fn install_http_get_host(lua: &Lua, handler: F) -> mlua::Result<()> +where + F: Fn(LuaHttpGetRequest) -> mlua::Result> + 'static, +{ + let http_get = lua.create_function(move |lua, args: mlua::Variadic| { + let request = lua_http_get_request_from_args(args)?; + let bytes = handler(request)?; + lua.create_string(&bytes) + })?; + let globals = lua.globals(); + let getter_builtin = getter_builtin_table(lua)?; + getter_builtin.set("http_get", http_get.clone())?; + globals.set("http_get", http_get) +} + +fn getter_builtin_table(lua: &Lua) -> mlua::Result
{ + let globals = lua.globals(); + match globals.get::("getter_builtin")? { + Value::Table(table) => Ok(table), Value::Nil => { let table = lua.create_table()?; globals.set("getter_builtin", table.clone())?; - table + Ok(table) } - _ => return Err(mlua::Error::external("getter_builtin must be a table")), + _ => Err(mlua::Error::external("getter_builtin must be a table")), + } +} + +fn lua_http_get_request_from_args(args: mlua::Variadic) -> mlua::Result { + if args.is_empty() || args.len() > 2 { + return Err(mlua::Error::external( + "http_get expects url and optional options table", + )); + } + let url = match &args[0] { + Value::String(value) => value.to_str()?.to_owned(), + _ => return Err(mlua::Error::external("http_get url must be a string")), }; - getter_builtin.set("read_package_file", read_package_file.clone())?; - globals.set("read_package_file", read_package_file) + if url.trim().is_empty() { + return Err(mlua::Error::external("http_get url must not be empty")); + } + let mut request = LuaHttpGetRequest { + url, + headers: BTreeMap::new(), + cache: false, + }; + match args.get(1) { + None | Some(Value::Nil) => {} + Some(Value::Table(options)) => apply_http_get_options(&mut request, options)?, + Some(_) => return Err(mlua::Error::external("http_get options must be a table")), + } + Ok(request) +} + +fn apply_http_get_options(request: &mut LuaHttpGetRequest, options: &Table) -> mlua::Result<()> { + for pair in options.clone().pairs::() { + let (key, _) = pair?; + match key { + Value::String(value) if matches!(value.to_str()?.as_ref(), "cache" | "headers") => {} + Value::String(value) => { + return Err(mlua::Error::external(format!( + "http_get unsupported option '{}'", + value.to_str()? + ))) + } + _ => { + return Err(mlua::Error::external( + "http_get option keys must be strings", + )) + } + } + } + + match options.get::("cache")? { + Value::Nil => {} + Value::Boolean(value) => request.cache = value, + _ => { + return Err(mlua::Error::external( + "http_get options.cache must be a boolean", + )) + } + } + + match options.get::("headers")? { + Value::Nil => {} + Value::Table(headers) => { + for pair in headers.pairs::() { + let (key, value) = pair?; + let key = match key { + Value::String(value) => value.to_str()?.to_owned(), + _ => { + return Err(mlua::Error::external( + "http_get headers keys must be strings", + )) + } + }; + let value = match value { + Value::String(value) => value.to_str()?.to_owned(), + _ => { + return Err(mlua::Error::external( + "http_get headers values must be strings", + )) + } + }; + request.headers.insert(key, value); + } + } + _ => { + return Err(mlua::Error::external( + "http_get options.headers must be a table", + )) + } + } + Ok(()) } fn package_file_relative_path(path: &str) -> mlua::Result { @@ -802,6 +923,21 @@ mod tests { ) } + fn write_simple_android_package(root: &Path, script_source: &str) { + let package_dir = root.join("android/app/com.example.autogen"); + fs::create_dir_all(&package_dir).unwrap(); + fs::write( + package_dir.join("metadata.jsonc"), + r#"{ + "type": "android:app", + "display_name": "Example Autogen", + "android": { "package_name": "com.example.autogen" } +}"#, + ) + .unwrap(); + fs::write(package_dir.join("9999.lua"), script_source).unwrap(); + } + #[test] fn evaluates_json_like_lua_package_table() { let temp = tempfile::tempdir().unwrap(); @@ -1393,6 +1529,160 @@ return github_android.package { ); } + #[test] + fn package_directory_http_get_is_not_installed_by_plain_eval() { + let temp = tempfile::tempdir().unwrap(); + write_simple_android_package( + temp.path(), + r#"#!/bin/upa-lua v1 +local builtin_http_get = getter_builtin and getter_builtin.http_get +return package_version { name = (http_get == nil and builtin_http_get == nil) and "not installed" or "installed" } +"#, + ); + + let package = evaluate_single_package_directory(temp.path(), "official").unwrap(); + + assert_eq!(package.name, "not installed"); + } + + #[test] + fn package_directory_http_get_host_parses_request_and_defaults_cache_false() { + let temp = tempfile::tempdir().unwrap(); + write_simple_android_package( + temp.path(), + r#"#!/bin/upa-lua v1 +local first = http_get("https://example.invalid/a") +local second = http_get("https://example.invalid/b", { + headers = { Accept = "application/json", ["X-Test"] = "yes" }, + cache = true, +}) +return package_version { name = first .. "|" .. second } +"#, + ); + let layout = RepositoryPackageDirectoryLayout::load(temp.path()).unwrap(); + let package_dir = &layout.packages[0]; + let metadata = layout.package_metadata(package_dir).unwrap(); + let script = layout.unambiguous_version_script(package_dir).unwrap(); + let requests = std::rc::Rc::new(std::cell::RefCell::new(Vec::new())); + + let package = evaluate_package_directory_script_with_host_bindings( + &RepositoryId::new("autogen").unwrap(), + package_dir, + &metadata, + script, + { + let requests = std::rc::Rc::clone(&requests); + move |lua| { + install_http_get_host(lua, move |request| { + let body = if request.url.ends_with("/a") { + b"first".to_vec() + } else { + b"second".to_vec() + }; + requests.borrow_mut().push(request); + Ok(body) + }) + } + }, + ) + .unwrap(); + + assert_eq!(package.name, "first|second"); + let requests = requests.borrow(); + assert_eq!(requests.len(), 2); + assert_eq!(requests[0].url, "https://example.invalid/a"); + assert!(!requests[0].cache); + assert!(requests[0].headers.is_empty()); + assert_eq!(requests[1].url, "https://example.invalid/b"); + assert!(requests[1].cache); + assert_eq!( + requests[1].headers.get("Accept").map(String::as_str), + Some("application/json") + ); + assert_eq!( + requests[1].headers.get("X-Test").map(String::as_str), + Some("yes") + ); + } + + #[test] + fn package_directory_http_get_rejects_unsupported_options() { + let temp = tempfile::tempdir().unwrap(); + write_simple_android_package( + temp.path(), + r#"#!/bin/upa-lua v1 +local extra_ok = pcall(http_get, "https://example.invalid/a", { method = "POST" }) +local header_ok = pcall(http_get, "https://example.invalid/b", { headers = { Accept = 1 } }) +return package_version { name = (not extra_ok and not header_ok) and "rejected" or "accepted" } +"#, + ); + let layout = RepositoryPackageDirectoryLayout::load(temp.path()).unwrap(); + let package_dir = &layout.packages[0]; + let metadata = layout.package_metadata(package_dir).unwrap(); + let script = layout.unambiguous_version_script(package_dir).unwrap(); + + let package = evaluate_package_directory_script_with_host_bindings( + &RepositoryId::new("autogen").unwrap(), + package_dir, + &metadata, + script, + |lua| install_http_get_host(lua, |_| Ok(Vec::new())), + ) + .unwrap(); + + assert_eq!(package.name, "rejected"); + } + + #[test] + fn getter_builtin_exposes_http_get_for_lua_wrappers() { + let temp = tempfile::tempdir().unwrap(); + write_simple_android_package( + temp.path(), + r#"#!/bin/upa-lua v1 +local upstream_http_get = getter_builtin.http_get +function http_get(url, opts) + opts = opts or {} + opts.headers = opts.headers or {} + opts.headers["X-Rewritten"] = "yes" + return upstream_http_get(url .. "?mirror=1", opts) +end +return package_version { name = http_get("https://example.invalid/file", { cache = true }) } +"#, + ); + let layout = RepositoryPackageDirectoryLayout::load(temp.path()).unwrap(); + let package_dir = &layout.packages[0]; + let metadata = layout.package_metadata(package_dir).unwrap(); + let script = layout.unambiguous_version_script(package_dir).unwrap(); + let requests = std::rc::Rc::new(std::cell::RefCell::new(Vec::new())); + + let package = evaluate_package_directory_script_with_host_bindings( + &RepositoryId::new("autogen").unwrap(), + package_dir, + &metadata, + script, + { + let requests = std::rc::Rc::clone(&requests); + move |lua| { + install_http_get_host(lua, move |request| { + requests.borrow_mut().push(request); + Ok(b"wrapped".to_vec()) + }) + } + }, + ) + .unwrap(); + + assert_eq!(package.name, "wrapped"); + let requests = requests.borrow(); + assert_eq!(requests.len(), 1); + assert_eq!(requests[0].url, "https://example.invalid/file?mirror=1"); + assert!(requests[0].cache); + assert_eq!( + requests[0].headers.get("X-Rewritten").map(String::as_str), + Some("yes") + ); + } + #[test] fn package_directory_can_read_package_local_files() { let temp = tempfile::tempdir().unwrap(); From 939c6de0be97b8f0ace143e53e6d3ff9dc76e30e Mon Sep 17 00:00:00 2001 From: Xiangzhe Date: Sat, 27 Jun 2026 23:25:31 +0800 Subject: [PATCH 50/52] feat(lua): add dev provider luaclass modules --- crates/getter-core/Cargo.toml | 1 + crates/getter-core/src/lua.rs | 111 +++++++++++++++++- .../src/luaclass/fdroid_android.lua | 32 +++++ .../src/luaclass/github_android_apk.lua | 65 ++++++++++ crates/getter-operations/Cargo.toml | 2 +- .../src/lua_provider_host.rs | 84 +++++++------ 6 files changed, 259 insertions(+), 36 deletions(-) create mode 100644 crates/getter-core/src/luaclass/fdroid_android.lua create mode 100644 crates/getter-core/src/luaclass/github_android_apk.lua diff --git a/crates/getter-core/Cargo.toml b/crates/getter-core/Cargo.toml index 4b1f9c5..5cb34ef 100644 --- a/crates/getter-core/Cargo.toml +++ b/crates/getter-core/Cargo.toml @@ -6,6 +6,7 @@ edition.workspace = true [features] default = ["lua"] lua = ["dep:mlua"] +provider-luaclass-dev = ["lua"] [dependencies] json_comments = "0.2" diff --git a/crates/getter-core/src/lua.rs b/crates/getter-core/src/lua.rs index 28857c9..f00f014 100644 --- a/crates/getter-core/src/lua.rs +++ b/crates/getter-core/src/lua.rs @@ -14,8 +14,19 @@ use std::collections::BTreeMap; use std::fs; use std::path::{Path, PathBuf}; -const BUILTIN_LUACLASS_MODULES: &[(&str, &str)] = - &[("android", include_str!("luaclass/android.lua"))]; +const BUILTIN_LUACLASS_MODULES: &[(&str, &str)] = &[ + ("android", include_str!("luaclass/android.lua")), + #[cfg(feature = "provider-luaclass-dev")] + ( + "fdroid_android", + include_str!("luaclass/fdroid_android.lua"), + ), + #[cfg(feature = "provider-luaclass-dev")] + ( + "github_android_apk", + include_str!("luaclass/github_android_apk.lua"), + ), +]; #[derive(Debug, thiserror::Error)] pub enum LuaPackageError { @@ -1239,6 +1250,102 @@ return android.package_version { assert_eq!(package.installed.len(), 1); } + #[cfg(not(feature = "provider-luaclass-dev"))] + #[test] + fn provider_luaclass_dev_modules_are_not_default_builtins() { + let temp = tempfile::tempdir().unwrap(); + let package_dir = temp.path().join("android/f-droid/app/org.fdroid.fdroid"); + fs::create_dir_all(&package_dir).unwrap(); + fs::write( + package_dir.join("metadata.jsonc"), + r#"{ + "type": "android:app", + "android": { "package_name": "org.fdroid.fdroid" } +}"#, + ) + .unwrap(); + fs::write( + package_dir.join("9999.lua"), + r#"#!/bin/upa-lua v1 +local fdroid = require("luaclass.fdroid_android") +return fdroid.package { package_name = "org.fdroid.fdroid" } +"#, + ) + .unwrap(); + + let err = evaluate_single_package_directory(temp.path(), "official").unwrap_err(); + + let message = err.to_string(); + assert!(matches!(err, LuaPackageError::Runtime { .. })); + assert!(message.contains("no builtin luaclass module 'luaclass.fdroid_android'")); + } + + #[cfg(feature = "provider-luaclass-dev")] + #[test] + fn provider_luaclass_dev_builtin_can_call_injected_host() { + let temp = tempfile::tempdir().unwrap(); + let package_dir = temp.path().join("android/f-droid/app/org.fdroid.fdroid"); + fs::create_dir_all(&package_dir).unwrap(); + fs::write( + package_dir.join("metadata.jsonc"), + r#"{ + "type": "android:app", + "android": { "package_name": "org.fdroid.fdroid" } +}"#, + ) + .unwrap(); + fs::write( + package_dir.join("9999.lua"), + r#"#!/bin/upa-lua v1 +local fdroid = require("luaclass.fdroid_android") +return fdroid.package { package_name = "org.fdroid.fdroid" } +"#, + ) + .unwrap(); + let layout = RepositoryPackageDirectoryLayout::load(temp.path()).unwrap(); + let package_dir = &layout.packages[0]; + let metadata = layout.package_metadata(package_dir).unwrap(); + let script = layout.unambiguous_version_script(package_dir).unwrap(); + + let package = evaluate_package_directory_script_with_host_bindings( + &RepositoryId::new("official").unwrap(), + package_dir, + &metadata, + script, + |lua| { + let getter_dev = lua.create_table()?; + getter_dev.set( + "fdroid_update_candidates", + lua.create_function(|lua, spec: Table| { + let package_name: String = spec.get("package_name")?; + if package_name != "org.fdroid.fdroid" { + return Err(mlua::Error::external("F-Droid package mismatch")); + } + let artifact = lua.create_table()?; + artifact.set("name", "org.fdroid.fdroid.apk")?; + artifact.set("url", "https://f-droid.org/repo/org.fdroid.fdroid.apk")?; + let artifacts = lua.create_table()?; + artifacts.raw_set(1, artifact)?; + let candidate = lua.create_table()?; + candidate.set("version", "1.20.0")?; + candidate.set("source", "fdroid")?; + candidate.set("artifacts", artifacts)?; + let candidates = lua.create_table()?; + candidates.raw_set(1, candidate)?; + Ok(candidates) + })?, + )?; + lua.globals().set("getter_dev", getter_dev) + }, + ) + .unwrap(); + + assert_eq!(package.name, "android/f-droid/app/org.fdroid.fdroid"); + assert_eq!(package.source_priority, vec!["fdroid"]); + assert_eq!(package.updates[0].version, "1.20.0"); + assert_eq!(package.updates[0].source.as_deref(), Some("fdroid")); + } + #[test] fn package_directory_can_use_repository_local_fdroid_luaclass_shape() { let temp = tempfile::tempdir().unwrap(); diff --git a/crates/getter-core/src/luaclass/fdroid_android.lua b/crates/getter-core/src/luaclass/fdroid_android.lua new file mode 100644 index 0000000..7a1237b --- /dev/null +++ b/crates/getter-core/src/luaclass/fdroid_android.lua @@ -0,0 +1,32 @@ +local fdroid = {} + +local function require_string(table, field) + local value = table[field] + if type(value) ~= "string" or value == "" then + error("fdroid.package requires string field '" .. field .. "'") + end + return value +end + +local function fdroid_update_candidates() + if type(getter_dev) ~= "table" or type(getter_dev.fdroid_update_candidates) ~= "function" then + error("luaclass.fdroid_android requires operation-installed getter_dev.fdroid_update_candidates") + end + return getter_dev.fdroid_update_candidates +end + +function fdroid.package(spec) + if type(spec) ~= "table" then + error("fdroid.package expects a table") + end + local package_name = require_string(spec, "package_name") + local update_candidates = fdroid_update_candidates() + return package_version { + source_priority = { "fdroid" }, + updates = update_candidates { + package_name = package_name, + }, + } +end + +return fdroid diff --git a/crates/getter-core/src/luaclass/github_android_apk.lua b/crates/getter-core/src/luaclass/github_android_apk.lua new file mode 100644 index 0000000..33d8791 --- /dev/null +++ b/crates/getter-core/src/luaclass/github_android_apk.lua @@ -0,0 +1,65 @@ +local github_android = {} + +local function require_string(table, field) + local value = table[field] + if type(value) ~= "string" or value == "" then + error("github_android.package requires string field '" .. field .. "'") + end + return value +end + +local function optional_table(table, field) + local value = table[field] + if value == nil then + return nil + end + if type(value) ~= "table" then + error("github_android.package field '" .. field .. "' must be a table") + end + return value +end + +local function optional_boolean(table, field) + local value = table[field] + if value == nil then + return nil + end + if type(value) ~= "boolean" then + error("github_android.package field '" .. field .. "' must be a boolean") + end + return value +end + +local function github_release_candidates() + if type(getter_dev) ~= "table" or type(getter_dev.github_release_candidates) ~= "function" then + error("luaclass.github_android_apk requires operation-installed getter_dev.github_release_candidates") + end + return getter_dev.github_release_candidates +end + +function github_android.package(spec) + if type(spec) ~= "table" then + error("github_android.package expects a table") + end + local owner = require_string(spec, "owner") + local repo = require_string(spec, "repo") + local release_candidates = github_release_candidates() + local result = { + name = spec.name, + source_priority = { "github" }, + updates = release_candidates { + owner = owner, + repo = repo, + asset = optional_table(spec, "asset"), + include_prereleases = optional_boolean(spec, "include_prereleases"), + }, + } + if spec.android_package ~= nil then + result.installed = { + { kind = "android_package", package_name = require_string(spec, "android_package") }, + } + end + return package_version(result) +end + +return github_android diff --git a/crates/getter-operations/Cargo.toml b/crates/getter-operations/Cargo.toml index 81c2e82..e81dab9 100644 --- a/crates/getter-operations/Cargo.toml +++ b/crates/getter-operations/Cargo.toml @@ -6,7 +6,7 @@ edition.workspace = true [features] default = [] lua = ["getter-core/lua", "dep:mlua"] -lua-provider-host-dev = ["lua"] +lua-provider-host-dev = ["lua", "getter-core/provider-luaclass-dev"] [dependencies] json_comments = "0.2" diff --git a/crates/getter-operations/src/lua_provider_host.rs b/crates/getter-operations/src/lua_provider_host.rs index ee16e1a..8e697af 100644 --- a/crates/getter-operations/src/lua_provider_host.rs +++ b/crates/getter-operations/src/lua_provider_host.rs @@ -584,7 +584,7 @@ mod tests { ]"#; #[test] - fn fdroid_package_eval_uses_repository_luaclass_and_provider_cache() { + fn fdroid_package_eval_uses_dev_builtin_luaclass_and_provider_cache() { let temp = tempfile::tempdir().unwrap(); let data_dir = temp.path(); write_fdroid_package_fixture(data_dir, "org.fdroid.fdroid"); @@ -674,7 +674,7 @@ mod tests { } #[test] - fn github_package_eval_uses_repository_luaclass_and_provider_cache() { + fn github_package_eval_uses_dev_builtin_luaclass_and_provider_cache() { let temp = tempfile::tempdir().unwrap(); let data_dir = temp.path(); write_github_package_fixture(data_dir, "[.]apk$"); @@ -749,21 +749,33 @@ mod tests { ); } - fn write_fdroid_package_fixture(data_dir: &std::path::Path, package_name: &str) { - let repo_root = data_dir.join("repo/official"); - let package_dir = repo_root.join("android/f-droid/app/org.fdroid.fdroid"); - fs::create_dir_all(repo_root.join("luaclass")).unwrap(); - fs::create_dir_all(&package_dir).unwrap(); + #[test] + fn provider_package_eval_prefers_repository_luaclass_over_dev_builtin() { + let temp = tempfile::tempdir().unwrap(); + let data_dir = temp.path(); + write_fdroid_package_fixture(data_dir, "org.fdroid.fdroid"); + let repo_luaclass = data_dir.join("repo/official/luaclass"); + fs::create_dir_all(&repo_luaclass).unwrap(); fs::write( - repo_root.join("luaclass/fdroid_android.lua"), + repo_luaclass.join("fdroid_android.lua"), r#" local fdroid = {} function fdroid.package(spec) + if spec.package_name ~= "org.fdroid.fdroid" then + error("repository override package_name mismatch") + end return package_version { - source_priority = { "fdroid" }, - updates = getter_dev.fdroid_update_candidates { - package_name = spec.package_name, + name = "repository override", + source_priority = { "repository-override" }, + updates = { + { + version = "9.9.9", + source = "repository-override", + artifacts = { + { name = "override.apk", url = "https://example.invalid/override.apk" }, + }, + }, }, } end @@ -772,6 +784,34 @@ return fdroid "#, ) .unwrap(); + + let result = fdroid_package_eval_json( + data_dir, + &json!({ + "repository_id": "official", + "package_id": "android/f-droid/app/org.fdroid.fdroid" + }) + .to_string(), + ) + .unwrap(); + + assert_eq!(result["provider_calls"], json!([])); + assert_eq!(result["package"]["name"], "repository override"); + assert_eq!( + result["package"]["source_priority"], + json!(["repository-override"]) + ); + assert_eq!(result["package"]["updates"][0]["version"], "9.9.9"); + assert_eq!( + result["package"]["updates"][0]["source"], + "repository-override" + ); + } + + fn write_fdroid_package_fixture(data_dir: &std::path::Path, package_name: &str) { + let repo_root = data_dir.join("repo/official"); + let package_dir = repo_root.join("android/f-droid/app/org.fdroid.fdroid"); + fs::create_dir_all(&package_dir).unwrap(); fs::write( package_dir.join("metadata.jsonc"), r#"{ @@ -795,29 +835,7 @@ return fdroid.package { fn write_github_package_fixture(data_dir: &std::path::Path, asset_include: &str) { let repo_root = data_dir.join("repo/official"); let package_dir = repo_root.join("android/app/org.fdroid.fdroid"); - fs::create_dir_all(repo_root.join("luaclass")).unwrap(); fs::create_dir_all(&package_dir).unwrap(); - fs::write( - repo_root.join("luaclass/github_android_apk.lua"), - r#" -local github_android = {} - -function github_android.package(spec) - return package_version { - name = spec.name, - source_priority = { "github" }, - updates = getter_dev.github_release_candidates { - owner = spec.owner, - repo = spec.repo, - asset = spec.asset, - }, - } -end - -return github_android -"#, - ) - .unwrap(); fs::write( package_dir.join("metadata.jsonc"), r#"{ From 03e81cc7b413b424c486f349f86358c2e19d3ba8 Mon Sep 17 00:00:00 2001 From: Xiangzhe Date: Sun, 28 Jun 2026 00:25:32 +0800 Subject: [PATCH 51/52] feat(lua): add HTTP policy harness --- crates/getter-operations/src/lib.rs | 3 + .../getter-operations/src/lua_http_policy.rs | 399 ++++++++++++++++++ 2 files changed, 402 insertions(+) create mode 100644 crates/getter-operations/src/lua_http_policy.rs diff --git a/crates/getter-operations/src/lib.rs b/crates/getter-operations/src/lib.rs index ae19c2d..50684dc 100644 --- a/crates/getter-operations/src/lib.rs +++ b/crates/getter-operations/src/lib.rs @@ -11,6 +11,9 @@ pub mod fdroid_catalog; pub mod github_latest_commit; pub mod github_releases; pub mod legacy_room; +#[cfg(feature = "lua")] +#[doc(hidden)] +pub mod lua_http_policy; #[cfg(feature = "lua-provider-host-dev")] #[doc(hidden)] pub mod lua_provider_host; diff --git a/crates/getter-operations/src/lua_http_policy.rs b/crates/getter-operations/src/lua_http_policy.rs new file mode 100644 index 0000000..cb7116a --- /dev/null +++ b/crates/getter-operations/src/lua_http_policy.rs @@ -0,0 +1,399 @@ +//! Operation-owned Lua HTTP policy harness. +//! +//! This module installs getter-core's `http_get` seam for package evaluation +//! using fixture/in-memory responses. It proves the operation/runtime side owns +//! permission and package Manifest policy while `getter-core` remains transport +//! and storage agnostic. It intentionally does not perform live HTTP, cache +//! persistence/revalidation, auth, hook loading, or public CLI/native exposure. + +use getter_core::lua::{ + evaluate_package_directory_script_with_host_bindings, install_http_get_host, LuaHttpGetRequest, + LuaPackageError, +}; +use getter_core::repository::{ + PackageDirectory, PackageDirectoryMetadata, PackageLuaPermission, PackageVersionScript, + PACKAGE_MANIFEST_FILE, +}; +use getter_core::{RepositoryId, ResolvedPackage}; +use sha2::{Digest, Sha512}; +use std::cell::RefCell; +use std::collections::{BTreeMap, BTreeSet}; +use std::fs; +use std::path::{Path, PathBuf}; +use std::rc::Rc; + +pub const HTTP_RESPONSE_NOT_IN_MANIFEST: &str = "package.http.response_not_in_manifest"; +pub const HTTP_FIXTURE_NOT_FOUND: &str = "package.http.fixture_not_found"; + +#[derive(Debug, thiserror::Error)] +pub enum LuaHttpPolicyError { + #[error("failed to read package Manifest at {path}: {source}")] + ReadManifest { + path: PathBuf, + #[source] + source: std::io::Error, + }, + #[error("invalid package Manifest at {path}: {reason}")] + InvalidManifest { path: PathBuf, reason: String }, + #[error("package evaluation failed: {0}")] + PackageEval(#[from] LuaPackageError), +} + +#[derive(Debug, Clone, PartialEq)] +pub struct LuaHttpPolicyEvaluation { + pub package: ResolvedPackage, + pub http_requests: Vec, +} + +/// Evaluates a package version script with fixture-backed `http_get`. +/// +/// This is an internal harness for operation/runtime policy tests. The caller +/// supplies URL-to-body fixtures instead of a live transport. For scripts that +/// do not declare `allow_free_network`, every returned response body must have +/// a SHA-512 digest listed in the package `Manifest`. Missing `Manifest` is an +/// empty allow-list. Scripts that declare `allow_free_network` bypass Manifest +/// membership, but requests are still captured for diagnostics/validation. +pub fn evaluate_package_directory_script_with_fixture_http( + repository_id: &RepositoryId, + package: &PackageDirectory, + metadata: &PackageDirectoryMetadata, + script: &PackageVersionScript, + responses: BTreeMap>, +) -> Result { + let allow_free_network = metadata + .permissions_for(&script.file_name) + .contains(&PackageLuaPermission::AllowFreeNetwork); + let manifest = if allow_free_network { + None + } else { + Some(PackageManifest::load( + package.path.join(PACKAGE_MANIFEST_FILE), + )?) + }; + let responses = Rc::new(responses); + let http_requests = Rc::new(RefCell::new(Vec::new())); + + let resolved = evaluate_package_directory_script_with_host_bindings( + repository_id, + package, + metadata, + script, + { + let http_requests = Rc::clone(&http_requests); + move |lua| { + install_http_get_host(lua, { + let responses = Rc::clone(&responses); + let http_requests = Rc::clone(&http_requests); + let manifest = manifest.clone(); + move |request| { + let body = responses.get(&request.url).cloned().ok_or_else(|| { + mlua::Error::external(format!( + "{HTTP_FIXTURE_NOT_FOUND}: no fixture response for {}", + request.url + )) + })?; + if let Some(manifest) = manifest.as_ref() { + let digest = sha512_hex(&body); + if !manifest.contains(&digest) { + return Err(mlua::Error::external(format!( + "{HTTP_RESPONSE_NOT_IN_MANIFEST}: response body SHA-512 {digest} for {} is not listed in package Manifest", + request.url + ))); + } + } + http_requests.borrow_mut().push(request); + Ok(body) + } + }) + } + }, + )?; + + let http_requests = http_requests.borrow().clone(); + Ok(LuaHttpPolicyEvaluation { + package: resolved, + http_requests, + }) +} + +#[derive(Debug, Clone)] +struct PackageManifest { + sha512: BTreeSet, +} + +impl PackageManifest { + fn load(path: impl AsRef) -> Result { + let path = path.as_ref(); + let source = match fs::read_to_string(path) { + Ok(source) => source, + Err(source) if source.kind() == std::io::ErrorKind::NotFound => { + return Ok(Self { + sha512: BTreeSet::new(), + }) + } + Err(source) => { + return Err(LuaHttpPolicyError::ReadManifest { + path: path.to_path_buf(), + source, + }) + } + }; + let mut sha512 = BTreeSet::new(); + for (line_index, raw_line) in source.lines().enumerate() { + let line = raw_line.trim(); + if line.is_empty() { + continue; + } + let hash = line + .split_whitespace() + .next() + .expect("non-empty line has token"); + if !is_sha512_hex(hash) { + return Err(LuaHttpPolicyError::InvalidManifest { + path: path.to_path_buf(), + reason: format!( + "line {} must start with a 128-character SHA-512 hex digest", + line_index + 1 + ), + }); + } + sha512.insert(hash.to_ascii_lowercase()); + } + Ok(Self { sha512 }) + } + + fn contains(&self, digest: &str) -> bool { + self.sha512.contains(digest) + } +} + +fn is_sha512_hex(value: &str) -> bool { + value.len() == 128 && value.bytes().all(|byte| byte.is_ascii_hexdigit()) +} + +fn sha512_hex(body: &[u8]) -> String { + let mut hasher = Sha512::new(); + hasher.update(body); + format!("{:x}", hasher.finalize()) +} + +#[cfg(test)] +mod tests { + use super::*; + use getter_core::repository::{RepositoryPackageDirectoryLayout, LUA_API_SHEBANG_V1}; + use serde_json::json; + use std::fs; + + const URL: &str = "https://example.invalid/body"; + + #[test] + fn non_free_network_script_accepts_manifest_listed_response_body() { + let fixture = package_fixture(PackageFixture { + body: b"manifest body", + manifest_body: Some(b"manifest body"), + allow_free_network: false, + cache: false, + }); + + let result = fixture.evaluate().unwrap(); + + assert_eq!(result.package.name, "manifest body"); + assert_eq!(result.http_requests.len(), 1); + assert!(!result.http_requests[0].cache); + } + + #[test] + fn non_free_network_script_rejects_missing_manifest_response_body() { + let fixture = package_fixture(PackageFixture { + body: b"unlisted body", + manifest_body: None, + allow_free_network: false, + cache: false, + }); + + let err = fixture.evaluate().unwrap_err(); + + let message = err.to_string(); + assert!(matches!(err, LuaHttpPolicyError::PackageEval(_))); + assert!(message.contains(HTTP_RESPONSE_NOT_IN_MANIFEST)); + } + + #[test] + fn non_free_network_script_rejects_empty_manifest_response_body() { + let fixture = package_fixture(PackageFixture { + body: b"unlisted body", + manifest_body: Some(b"unused"), + allow_free_network: false, + cache: false, + }); + fs::write(fixture.package_dir.join(PACKAGE_MANIFEST_FILE), "").unwrap(); + + let err = fixture.evaluate().unwrap_err(); + + let message = err.to_string(); + assert!(matches!(err, LuaHttpPolicyError::PackageEval(_))); + assert!(message.contains(HTTP_RESPONSE_NOT_IN_MANIFEST)); + } + + #[test] + fn non_free_network_script_rejects_mismatched_manifest_response_body() { + let fixture = package_fixture(PackageFixture { + body: b"actual body", + manifest_body: Some(b"different body"), + allow_free_network: false, + cache: false, + }); + + let err = fixture.evaluate().unwrap_err(); + + let message = err.to_string(); + assert!(message.contains(HTTP_RESPONSE_NOT_IN_MANIFEST)); + assert!(message.contains(URL)); + } + + #[test] + fn allow_free_network_script_bypasses_manifest_membership() { + let fixture = package_fixture(PackageFixture { + body: b"free network body", + manifest_body: None, + allow_free_network: true, + cache: false, + }); + + let result = fixture.evaluate().unwrap(); + + assert_eq!(result.package.name, "free network body"); + assert!(result.package.permissions.free_network); + } + + #[test] + fn cache_true_is_forwarded_without_cache_persistence_behavior() { + let fixture = package_fixture(PackageFixture { + body: b"cache body", + manifest_body: Some(b"cache body"), + allow_free_network: false, + cache: true, + }); + + let result = fixture.evaluate().unwrap(); + + assert_eq!(result.package.name, "cache body"); + assert_eq!(result.http_requests.len(), 1); + assert!(result.http_requests[0].cache); + assert_eq!( + result.http_requests[0] + .headers + .get("Accept") + .map(String::as_str), + Some("text/plain") + ); + } + + #[test] + fn invalid_manifest_fails_before_lua_evaluation() { + let fixture = package_fixture(PackageFixture { + body: b"unused", + manifest_body: Some(b"unused"), + allow_free_network: false, + cache: false, + }); + fs::write( + fixture.package_dir.join(PACKAGE_MANIFEST_FILE), + "not-a-sha512 body", + ) + .unwrap(); + + let err = fixture.evaluate().unwrap_err(); + + assert!(matches!(err, LuaHttpPolicyError::InvalidManifest { .. })); + assert!(err.to_string().contains("line 1")); + } + + struct PackageFixture { + body: &'static [u8], + manifest_body: Option<&'static [u8]>, + allow_free_network: bool, + cache: bool, + } + + struct WrittenPackageFixture { + temp: tempfile::TempDir, + package_dir: PathBuf, + responses: BTreeMap>, + } + + impl WrittenPackageFixture { + fn evaluate(&self) -> Result { + let layout = RepositoryPackageDirectoryLayout::load(self.temp.path()).unwrap(); + let package = layout + .package(&"android/app/org.example".parse().unwrap()) + .unwrap(); + let metadata = layout.package_metadata(package).unwrap(); + let script = layout.unambiguous_version_script(package).unwrap(); + evaluate_package_directory_script_with_fixture_http( + &RepositoryId::new("official").unwrap(), + package, + &metadata, + script, + self.responses.clone(), + ) + } + } + + fn package_fixture(fixture: PackageFixture) -> WrittenPackageFixture { + let temp = tempfile::tempdir().unwrap(); + let package_dir = temp.path().join("android/app/org.example"); + fs::create_dir_all(&package_dir).unwrap(); + let mut metadata = json!({ + "type": "android:app", + "android": { "package_name": "org.example" } + }); + if fixture.allow_free_network { + metadata["lua"] = json!({ + "9999.lua": { "permission": ["allow_free_network"] } + }); + } + fs::write( + package_dir.join("metadata.jsonc"), + serde_json::to_string_pretty(&metadata).unwrap(), + ) + .unwrap(); + if let Some(manifest_body) = fixture.manifest_body { + fs::write( + package_dir.join(PACKAGE_MANIFEST_FILE), + format!("{} fixture-body\n", sha512_hex(manifest_body)), + ) + .unwrap(); + } + fs::write(package_dir.join("9999.lua"), version_script(fixture.cache)).unwrap(); + let mut responses = BTreeMap::new(); + responses.insert(URL.to_owned(), fixture.body.to_vec()); + WrittenPackageFixture { + temp, + package_dir, + responses, + } + } + + fn version_script(cache: bool) -> String { + format!( + r#"{LUA_API_SHEBANG_V1} +local body = http_get("{URL}", {{ + headers = {{ Accept = "text/plain" }}, + cache = {cache}, +}}) +return package_version {{ + name = body, + updates = {{ + {{ + version = "1.0.0", + artifacts = {{ + {{ name = "fixture", url = "https://example.invalid/artifact" }}, + }}, + }}, + }}, +}} +"# + ) + } +} From 25c510acdeea043095bb44951b63bdce88bd76e5 Mon Sep 17 00:00:00 2001 From: Xiangzhe Date: Sun, 28 Jun 2026 01:01:03 +0800 Subject: [PATCH 52/52] feat(lua): add runtime hook harness --- crates/getter-operations/src/lib.rs | 3 + .../getter-operations/src/lua_http_policy.rs | 206 +++++++++++++++++- .../src/lua_runtime_hooks.rs | 81 +++++++ 3 files changed, 286 insertions(+), 4 deletions(-) create mode 100644 crates/getter-operations/src/lua_runtime_hooks.rs diff --git a/crates/getter-operations/src/lib.rs b/crates/getter-operations/src/lib.rs index 50684dc..f3ddcb5 100644 --- a/crates/getter-operations/src/lib.rs +++ b/crates/getter-operations/src/lib.rs @@ -17,6 +17,9 @@ pub mod lua_http_policy; #[cfg(feature = "lua-provider-host-dev")] #[doc(hidden)] pub mod lua_provider_host; +#[cfg(feature = "lua")] +#[doc(hidden)] +pub mod lua_runtime_hooks; pub mod provider_cache; pub mod read_model; pub mod runtime; diff --git a/crates/getter-operations/src/lua_http_policy.rs b/crates/getter-operations/src/lua_http_policy.rs index cb7116a..da254ce 100644 --- a/crates/getter-operations/src/lua_http_policy.rs +++ b/crates/getter-operations/src/lua_http_policy.rs @@ -4,8 +4,9 @@ //! using fixture/in-memory responses. It proves the operation/runtime side owns //! permission and package Manifest policy while `getter-core` remains transport //! and storage agnostic. It intentionally does not perform live HTTP, cache -//! persistence/revalidation, auth, hook loading, or public CLI/native exposure. +//! persistence/revalidation, auth, or public CLI/native exposure. +use crate::lua_runtime_hooks::load_runtime_hooks; use getter_core::lua::{ evaluate_package_directory_script_with_host_bindings, install_http_get_host, LuaHttpGetRequest, LuaPackageError, @@ -43,6 +44,7 @@ pub enum LuaHttpPolicyError { pub struct LuaHttpPolicyEvaluation { pub package: ResolvedPackage, pub http_requests: Vec, + pub runtime_hooks: Vec, } /// Evaluates a package version script with fixture-backed `http_get`. @@ -59,6 +61,26 @@ pub fn evaluate_package_directory_script_with_fixture_http( metadata: &PackageDirectoryMetadata, script: &PackageVersionScript, responses: BTreeMap>, +) -> Result { + evaluate_package_directory_script_with_fixture_http_and_runtime_hooks( + repository_id, + package, + metadata, + script, + responses, + None, + ) +} + +/// Evaluates a package version script with fixture-backed `http_get` and +/// optional runtime hooks from `/rc/hook/*.lua`. +pub fn evaluate_package_directory_script_with_fixture_http_and_runtime_hooks( + repository_id: &RepositoryId, + package: &PackageDirectory, + metadata: &PackageDirectoryMetadata, + script: &PackageVersionScript, + responses: BTreeMap>, + data_dir: Option<&Path>, ) -> Result { let allow_free_network = metadata .permissions_for(&script.file_name) @@ -72,6 +94,8 @@ pub fn evaluate_package_directory_script_with_fixture_http( }; let responses = Rc::new(responses); let http_requests = Rc::new(RefCell::new(Vec::new())); + let runtime_hooks = Rc::new(RefCell::new(Vec::new())); + let data_dir = data_dir.map(Path::to_path_buf); let resolved = evaluate_package_directory_script_with_host_bindings( repository_id, @@ -80,6 +104,7 @@ pub fn evaluate_package_directory_script_with_fixture_http( script, { let http_requests = Rc::clone(&http_requests); + let runtime_hooks = Rc::clone(&runtime_hooks); move |lua| { install_http_get_host(lua, { let responses = Rc::clone(&responses); @@ -104,15 +129,21 @@ pub fn evaluate_package_directory_script_with_fixture_http( http_requests.borrow_mut().push(request); Ok(body) } - }) + })?; + if let Some(data_dir) = data_dir.as_deref() { + *runtime_hooks.borrow_mut() = load_runtime_hooks(lua, data_dir)?; + } + Ok(()) } }, )?; let http_requests = http_requests.borrow().clone(); + let runtime_hooks = runtime_hooks.borrow().clone(); Ok(LuaHttpPolicyEvaluation { package: resolved, http_requests, + runtime_hooks, }) } @@ -309,6 +340,148 @@ mod tests { assert!(err.to_string().contains("line 1")); } + #[test] + fn missing_runtime_hook_directory_is_noop() { + let fixture = package_fixture(PackageFixture { + body: b"manifest body", + manifest_body: Some(b"manifest body"), + allow_free_network: false, + cache: false, + }); + + let result = fixture.evaluate_with_hooks().unwrap(); + + assert_eq!(result.package.name, "manifest body"); + assert!(result.runtime_hooks.is_empty()); + } + + #[test] + fn dot_prefixed_runtime_hook_is_ignored() { + let fixture = package_fixture(PackageFixture { + body: b"manifest body", + manifest_body: Some(b"manifest body"), + allow_free_network: false, + cache: false, + }); + fixture.write_hook(".10-fail.lua", r#"error("ignored hook should not run")"#); + + let result = fixture.evaluate_with_hooks().unwrap(); + + assert_eq!(result.package.name, "manifest body"); + assert!(result.runtime_hooks.is_empty()); + } + + #[test] + fn runtime_hooks_load_in_deterministic_filename_order() { + let fixture = package_fixture(PackageFixture { + body: b"manifest body", + manifest_body: Some(b"manifest body"), + allow_free_network: false, + cache: false, + }); + fixture.write_hook("20-second.lua", r#"hook_order = (hook_order or "") .. "b""#); + fixture.write_hook("10-first.lua", r#"hook_order = (hook_order or "") .. "a""#); + + let result = fixture.evaluate_with_hooks().unwrap(); + + assert_eq!(result.package.name, "manifest body"); + let loaded = result + .runtime_hooks + .iter() + .map(|path| path.file_name().unwrap().to_str().unwrap().to_owned()) + .collect::>(); + assert_eq!(loaded, vec!["10-first.lua", "20-second.lua"]); + assert_eq!( + result.http_requests[0] + .headers + .get("X-Hook-Order") + .map(String::as_str), + Some("ab") + ); + } + + #[test] + fn runtime_hook_can_wrap_http_get_and_rewrite_request() { + let mut fixture = package_fixture(PackageFixture { + body: b"unused original body", + manifest_body: Some(b"mirror body"), + allow_free_network: false, + cache: false, + }); + fixture.add_response("https://mirror.invalid/body", b"mirror body"); + fixture.write_hook( + "10-http-rewrite.lua", + r#" +local original_http_get = getter_builtin.http_get +function http_get(url, options) + options = options or {} + options.headers = options.headers or {} + options.headers["X-Original-Url"] = url + options.cache = true + return original_http_get("https://mirror.invalid/body", options) +end +"#, + ); + + let result = fixture.evaluate_with_hooks().unwrap(); + + assert_eq!(result.package.name, "mirror body"); + assert_eq!(result.http_requests.len(), 1); + assert_eq!(result.http_requests[0].url, "https://mirror.invalid/body"); + assert!(result.http_requests[0].cache); + assert_eq!( + result.http_requests[0] + .headers + .get("X-Original-Url") + .map(String::as_str), + Some(URL) + ); + } + + #[test] + fn runtime_hook_rewrite_still_enforces_manifest_on_returned_body() { + let mut fixture = package_fixture(PackageFixture { + body: b"unused original body", + manifest_body: Some(b"unused original body"), + allow_free_network: false, + cache: false, + }); + fixture.add_response("https://mirror.invalid/body", b"unlisted mirror body"); + fixture.write_hook( + "10-http-rewrite.lua", + r#" +local original_http_get = getter_builtin.http_get +function http_get(url, options) + return original_http_get("https://mirror.invalid/body", options) +end +"#, + ); + + let err = fixture.evaluate_with_hooks().unwrap_err(); + + let message = err.to_string(); + assert!(message.contains(HTTP_RESPONSE_NOT_IN_MANIFEST)); + assert!(message.contains("https://mirror.invalid/body")); + } + + #[test] + fn runtime_hook_failure_fails_closed() { + let fixture = package_fixture(PackageFixture { + body: b"manifest body", + manifest_body: Some(b"manifest body"), + allow_free_network: false, + cache: false, + }); + fixture.write_hook("10-fail.lua", r#"error("hook failed closed")"#); + + let err = fixture.evaluate_with_hooks().unwrap_err(); + + let message = err.to_string(); + assert!(matches!(err, LuaHttpPolicyError::PackageEval(_))); + assert!(message.contains("failed to execute runtime hook")); + assert!(message.contains("hook failed closed")); + } + struct PackageFixture { body: &'static [u8], manifest_body: Option<&'static [u8]>, @@ -324,20 +497,42 @@ mod tests { impl WrittenPackageFixture { fn evaluate(&self) -> Result { + self.evaluate_with_data_dir(None) + } + + fn evaluate_with_hooks(&self) -> Result { + self.evaluate_with_data_dir(Some(self.temp.path())) + } + + fn evaluate_with_data_dir( + &self, + data_dir: Option<&Path>, + ) -> Result { let layout = RepositoryPackageDirectoryLayout::load(self.temp.path()).unwrap(); let package = layout .package(&"android/app/org.example".parse().unwrap()) .unwrap(); let metadata = layout.package_metadata(package).unwrap(); let script = layout.unambiguous_version_script(package).unwrap(); - evaluate_package_directory_script_with_fixture_http( + evaluate_package_directory_script_with_fixture_http_and_runtime_hooks( &RepositoryId::new("official").unwrap(), package, &metadata, script, self.responses.clone(), + data_dir, ) } + + fn write_hook(&self, file_name: &str, source: &str) { + let hook_dir = self.temp.path().join("rc/hook"); + fs::create_dir_all(&hook_dir).unwrap(); + fs::write(hook_dir.join(file_name), source).unwrap(); + } + + fn add_response(&mut self, url: &str, body: &'static [u8]) { + self.responses.insert(url.to_owned(), body.to_vec()); + } } fn package_fixture(fixture: PackageFixture) -> WrittenPackageFixture { @@ -379,7 +574,10 @@ mod tests { format!( r#"{LUA_API_SHEBANG_V1} local body = http_get("{URL}", {{ - headers = {{ Accept = "text/plain" }}, + headers = {{ + Accept = "text/plain", + ["X-Hook-Order"] = hook_order or "", + }}, cache = {cache}, }}) return package_version {{ diff --git a/crates/getter-operations/src/lua_runtime_hooks.rs b/crates/getter-operations/src/lua_runtime_hooks.rs new file mode 100644 index 0000000..e73ed86 --- /dev/null +++ b/crates/getter-operations/src/lua_runtime_hooks.rs @@ -0,0 +1,81 @@ +//! Operation-owned runtime/local Lua hook loading. +//! +//! This module is an internal harness for ADR-0012 `rc/hook/*.lua` runtime +//! policy. It only loads local hook source into an already prepared Lua +//! environment; callers still decide which host functions exist and which +//! operation policy applies. It intentionally does not expose hooks through +//! CLI/native/Flutter surfaces or make hook policy part of `getter-core`. + +use mlua::Lua; +use std::fs; +use std::path::{Path, PathBuf}; + +const RUNTIME_HOOK_DIR: &str = "rc/hook"; + +/// Loads enabled runtime hooks from `/rc/hook/*.lua`. +/// +/// Hooks are discovered from the filesystem only, dot-prefixed Lua basenames +/// are ignored, and the remaining hooks are loaded in deterministic path order. +/// Any read or runtime error fails closed by returning an error to the caller. +pub fn load_runtime_hooks(lua: &Lua, data_dir: &Path) -> mlua::Result> { + let hook_dir = data_dir.join(RUNTIME_HOOK_DIR); + let entries = match fs::read_dir(&hook_dir) { + Ok(entries) => entries, + Err(source) if source.kind() == std::io::ErrorKind::NotFound => return Ok(Vec::new()), + Err(source) => { + return Err(mlua::Error::external(format!( + "failed to read runtime hook directory {}: {source}", + hook_dir.display() + ))) + } + }; + + let mut hooks = Vec::new(); + for entry in entries { + let entry = entry.map_err(|source| { + mlua::Error::external(format!( + "failed to read runtime hook directory entry in {}: {source}", + hook_dir.display() + )) + })?; + let path = entry.path(); + if !is_enabled_lua_hook(&path) { + continue; + } + hooks.push(path); + } + hooks.sort(); + + for hook in &hooks { + let source = fs::read_to_string(hook).map_err(|source| { + mlua::Error::external(format!( + "failed to read runtime hook {}: {source}", + hook.display() + )) + })?; + lua.load(&source) + .set_name(hook.to_string_lossy().as_ref()) + .exec() + .map_err(|source| { + mlua::Error::external(format!( + "failed to execute runtime hook {}: {source}", + hook.display() + )) + })?; + } + + Ok(hooks) +} + +fn is_enabled_lua_hook(path: &Path) -> bool { + if !path.is_file() { + return false; + } + if path.extension().and_then(|extension| extension.to_str()) != Some("lua") { + return false; + } + let Some(file_name) = path.file_name().and_then(|file_name| file_name.to_str()) else { + return false; + }; + !file_name.starts_with('.') +}