diff --git a/.github/actions/setup-vcpkg/action.yml b/.github/actions/setup-vcpkg/action.yml new file mode 100644 index 000000000..eb7d691d1 --- /dev/null +++ b/.github/actions/setup-vcpkg/action.yml @@ -0,0 +1,56 @@ +# Copyright 2026 Valve Corporation +# Copyright 2026 LunarG, Inc. +# +# SPDX-License-Identifier: Apache-2.0 + +name: 'Setup vcpkg' +description: >- + Restores this job's vcpkg binary cache and bootstraps vcpkg. Every CI job + that configures with the vcpkg toolchain file should use this action + instead of repeating the cache + lukka/run-vcpkg steps by hand, so the + vcpkg caching setup only has to be understood and fixed in one place. + Requires actions/checkout to have already run. + +inputs: + job-index: + description: >- + Pass the calling job's strategy.job-index value here. The matrix and + strategy contexts aren't visible inside a composite action, so the + index has to be forwarded explicitly to keep each matrix leg's cache + entry separate from its siblings. + required: true + run-vcpkg-install: + description: >- + Set to 'true' to have this step also run 'vcpkg install' immediately. + Only the codegen job needs this, to materialize the Vulkan-Headers + registry before its CMake-less verification step runs. Every other + job installs its packages later, implicitly, when CMake configures + via the vcpkg toolchain file. + required: false + default: 'false' + vcpkg-installed-dir: + description: >- + Overrides VCPKG_INSTALLED_DIR for the install above. Only meaningful + together with run-vcpkg-install: 'true'. + required: false + default: '' + +runs: + using: 'composite' + steps: + - name: Create vcpkg binary cache directory + shell: bash + run: mkdir -p "$VCPKG_CACHE_DIR" + - name: Cache vcpkg binary packages + uses: actions/cache@v5 + with: + path: ${{ env.VCPKG_CACHE_DIR }} + key: vcpkg-binary-cache-${{ runner.os }}-${{ github.job }}-${{ inputs.job-index }}-${{ hashFiles('vcpkg.json', 'vcpkg-configuration.json') }} + restore-keys: | + vcpkg-binary-cache-${{ runner.os }}-${{ github.job }}-${{ inputs.job-index }}- + - uses: lukka/run-vcpkg@v11 + with: + vcpkgJsonGlob: 'vcpkg.json' + runVcpkgInstall: ${{ inputs.run-vcpkg-install }} + env: + VCPKG_INSTALLED_DIR: ${{ inputs.vcpkg-installed-dir }} diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 8abd4cb11..e5ffb863d 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -33,6 +33,15 @@ on: permissions: read-all +env: + # vcpkg's 'files' binary cache writes here; actions/cache persists it + # across runs so jobs don't rebuild every dependency from source. + # VCPKG_BINARY_SOURCES is pinned explicitly because vcpkg dropped the + # GitHub Actions cache backend ('x-gha'): + # https://github.com/lukka/run-vcpkg/issues/251 + VCPKG_CACHE_DIR: ${{ github.workspace }}/vcpkg-binary-cache + VCPKG_BINARY_SOURCES: clear;files,${{ github.workspace }}/vcpkg-binary-cache,readwrite + jobs: linux: needs: codegen @@ -45,6 +54,9 @@ jobs: os: [ ubuntu-22.04, ubuntu-24.04 ] steps: - uses: actions/checkout@v7 + - uses: ./.github/actions/setup-vcpkg + with: + job-index: ${{ strategy.job-index }} - uses: actions/setup-python@v6 with: python-version: '3.11' @@ -64,7 +76,8 @@ jobs: cmake -S. -B build \ -D CMAKE_BUILD_TYPE=${{ matrix.config }} \ -D BUILD_TESTS=ON \ - -D UPDATE_DEPS=ON \ + -D VCPKG_MANIFEST_FEATURES=tests \ + -D "CMAKE_TOOLCHAIN_FILE=${{ env.VCPKG_ROOT }}/scripts/buildsystems/vcpkg.cmake" \ -D LOADER_ENABLE_ADDRESS_SANITIZER=ON \ -D BUILD_WERROR=ON \ -D CMAKE_CXX_COMPILER=${{ matrix.compiler.cxx }} \ @@ -78,8 +91,12 @@ jobs: timeout-minutes: 30 steps: - uses: actions/checkout@v7 - - run: scripts/update_deps.py --dir ext --no-build - - run: scripts/generate_source.py --verify ext/Vulkan-Headers/registry/ + - uses: ./.github/actions/setup-vcpkg + with: + job-index: ${{ strategy.job-index }} + run-vcpkg-install: 'true' + vcpkg-installed-dir: ${{ github.workspace }}/vcpkg_installed + - run: scripts/generate_source.py --verify vcpkg_installed/*/share/vulkan/registry linux-no-asm: needs: codegen @@ -87,13 +104,17 @@ jobs: timeout-minutes: 30 steps: - uses: actions/checkout@v7 + - uses: ./.github/actions/setup-vcpkg + with: + job-index: ${{ strategy.job-index }} - run: sudo apt update - run: sudo apt install --yes --no-install-recommends libwayland-dev libxrandr-dev - run: | cmake -S. -B build \ -D CMAKE_BUILD_TYPE=Release \ -D BUILD_TESTS=ON \ - -D UPDATE_DEPS=ON \ + -D VCPKG_MANIFEST_FEATURES=tests \ + -D "CMAKE_TOOLCHAIN_FILE=${{ env.VCPKG_ROOT }}/scripts/buildsystems/vcpkg.cmake" \ -D BUILD_WERROR=ON \ -D USE_GAS=OFF \ -D CMAKE_C_COMPILER=clang \ @@ -106,11 +127,22 @@ jobs: needs: codegen runs-on: ubuntu-24.04 timeout-minutes: 30 + # This job builds 32-bit binaries on a 64-bit runner, so vcpkg needs two + # different triplets: one for the packages the loader links against + # (32-bit) and one for the runner vcpkg itself runs on (64-bit). The + # cmake invocation below sets the first explicitly via + # VCPKG_TARGET_TRIPLET; these env vars cover the second. + env: + VCPKG_DEFAULT_TRIPLET: x86-linux + VCPKG_DEFAULT_HOST_TRIPLET: x64-linux strategy: matrix: config: [ Debug, Release ] steps: - uses: actions/checkout@v7 + - uses: ./.github/actions/setup-vcpkg + with: + job-index: ${{ strategy.job-index }} - uses: actions/setup-python@v6 with: python-version: '3.11' @@ -128,7 +160,9 @@ jobs: cmake -S. -B build \ -D CMAKE_BUILD_TYPE=${{matrix.config}} \ -D BUILD_TESTS=ON \ - -D UPDATE_DEPS=ON \ + -D VCPKG_MANIFEST_FEATURES=tests \ + -D "CMAKE_TOOLCHAIN_FILE=${{ env.VCPKG_ROOT }}/scripts/buildsystems/vcpkg.cmake" \ + -D VCPKG_TARGET_TRIPLET=x86-linux \ -D BUILD_WERROR=ON \ -D SYSCONFDIR=/etc/not_vulkan \ -D CMAKE_CXX_FLAGS=-m32 \ @@ -145,8 +179,15 @@ jobs: needs: codegen runs-on: ubuntu-24.04 timeout-minutes: 30 + # See the linux-32 job above for why both triplets are needed. + env: + VCPKG_DEFAULT_TRIPLET: x86-linux + VCPKG_DEFAULT_HOST_TRIPLET: x64-linux steps: - uses: actions/checkout@v7 + - uses: ./.github/actions/setup-vcpkg + with: + job-index: ${{ strategy.job-index }} - uses: actions/setup-python@v6 with: python-version: '3.11' @@ -164,7 +205,9 @@ jobs: cmake -S. -B build \ -D CMAKE_BUILD_TYPE=Release \ -D BUILD_TESTS=ON \ - -D UPDATE_DEPS=ON \ + -D VCPKG_MANIFEST_FEATURES=tests \ + -D "CMAKE_TOOLCHAIN_FILE=${{ env.VCPKG_ROOT }}/scripts/buildsystems/vcpkg.cmake" \ + -D VCPKG_TARGET_TRIPLET=x86-linux \ -D BUILD_WERROR=ON \ -D USE_GAS=OFF \ -D CMAKE_CXX_FLAGS=-m32 \ @@ -182,6 +225,9 @@ jobs: timeout-minutes: 30 steps: - uses: actions/checkout@v7 + - uses: ./.github/actions/setup-vcpkg + with: + job-index: ${{ strategy.job-index }} - uses: actions/setup-python@v6 with: python-version: '3.11' @@ -195,7 +241,8 @@ jobs: cmake -S. -B build \ -D CMAKE_BUILD_TYPE=Debug \ -D BUILD_TESTS=ON \ - -D UPDATE_DEPS=ON \ + -D VCPKG_MANIFEST_FEATURES=tests \ + -D "CMAKE_TOOLCHAIN_FILE=${{ env.VCPKG_ROOT }}/scripts/buildsystems/vcpkg.cmake" \ -D LOADER_ENABLE_ADDRESS_SANITIZER=ON \ -D BUILD_WERROR=ON - run: cmake --build build @@ -208,6 +255,9 @@ jobs: timeout-minutes: 30 steps: - uses: actions/checkout@v7 + - uses: ./.github/actions/setup-vcpkg + with: + job-index: ${{ strategy.job-index }} - uses: actions/setup-python@v6 with: python-version: '3.11' @@ -221,7 +271,8 @@ jobs: cmake -S. -B build \ -D CMAKE_BUILD_TYPE=Debug \ -D BUILD_TESTS=ON \ - -D UPDATE_DEPS=ON \ + -D VCPKG_MANIFEST_FEATURES=tests \ + -D "CMAKE_TOOLCHAIN_FILE=${{ env.VCPKG_ROOT }}/scripts/buildsystems/vcpkg.cmake" \ -D LOADER_ENABLE_THREAD_SANITIZER=ON \ -D BUILD_WERROR=ON - run: cmake --build build @@ -239,10 +290,14 @@ jobs: config: [ Debug, Release ] steps: - uses: actions/checkout@v7 + - uses: ./.github/actions/setup-vcpkg + with: + job-index: ${{ strategy.job-index }} - run: | cmake -S. -B build ` -D BUILD_TESTS=ON ` - -D UPDATE_DEPS=ON ` + -D VCPKG_MANIFEST_FEATURES=tests ` + -D "CMAKE_TOOLCHAIN_FILE=${{ env.VCPKG_ROOT }}/scripts/buildsystems/vcpkg.cmake" ` -D CMAKE_BUILD_TYPE=${{matrix.config}} ` -A ${{ matrix.arch }} ` -D BUILD_WERROR=ON @@ -260,10 +315,14 @@ jobs: arch: [ Win32, x64 ] steps: - uses: actions/checkout@v7 + - uses: ./.github/actions/setup-vcpkg + with: + job-index: ${{ strategy.job-index }} - run: | cmake -S. -B build ` -D BUILD_TESTS=ON ` - -D UPDATE_DEPS=ON ` + -D VCPKG_MANIFEST_FEATURES=tests ` + -D "CMAKE_TOOLCHAIN_FILE=${{ env.VCPKG_ROOT }}/scripts/buildsystems/vcpkg.cmake" ` -D USE_MASM=OFF ` -D CMAKE_BUILD_TYPE=Release ` -A ${{ matrix.arch }} ` @@ -281,15 +340,28 @@ jobs: matrix: arch: [ arm64, arm64ec ] config: [ Debug, Release ] + include: + - arch: arm64 + triplet: arm64-windows + - arch: arm64ec + triplet: arm64ec-windows steps: - uses: actions/checkout@v7 + - uses: ./.github/actions/setup-vcpkg + with: + job-index: ${{ strategy.job-index }} - uses: actions/setup-python@v6 with: python-version: '3.x' + # CMake's -A only selects the MSBuild platform; it does not + # influence vcpkg's target triplet, which must be set explicitly + # or arm64ec ends up linking against a plain-arm64 detours.lib. - run: | cmake -S. -B build ` -D BUILD_TESTS=ON ` - -D UPDATE_DEPS=ON ` + -D VCPKG_MANIFEST_FEATURES=tests ` + -D "CMAKE_TOOLCHAIN_FILE=${{ env.VCPKG_ROOT }}/scripts/buildsystems/vcpkg.cmake" ` + -D VCPKG_TARGET_TRIPLET=${{ matrix.triplet }} ` -D CMAKE_BUILD_TYPE=${{matrix.config}} ` -A ${{ matrix.arch }} ` -D BUILD_WERROR=ON @@ -300,31 +372,35 @@ jobs: windows_arm64x: # Native Windows on Arm: Tests building a genuine arm64x binary and validates that an x64 test suite passes # windows is 2x expensive to run on GitHub machines, so only run if we know something else simple passed as well - # needs: windows_arm + needs: linux-no-asm runs-on: windows-11-arm timeout-minutes: 30 steps: - uses: actions/checkout@v7 + - uses: ./.github/actions/setup-vcpkg + with: + job-index: ${{ strategy.job-index }} - uses: actions/setup-python@v6 with: python-version: '3.x' - - run: python scripts/update_deps.py --arch arm64 --api vulkan --dir external --clean-build --clean-install - run: | cmake -S. -B build_arm64x ` -A arm64 ` -D BUILD_AS_ARM64X=ARM64 ` -D CMAKE_BUILD_TYPE=Release ` -D BUILD_WERROR=ON ` - -C external/helper.cmake + -D "CMAKE_TOOLCHAIN_FILE=${{ env.VCPKG_ROOT }}/scripts/buildsystems/vcpkg.cmake" - run: cmake --build build_arm64x/ --config Release - run: | cmake -S. -B build_arm64xec ` -A arm64ec ` -D BUILD_AS_ARM64X=ARM64EC ` -D BUILD_TESTS=ON ` + -D VCPKG_MANIFEST_FEATURES=tests ` -D CMAKE_BUILD_TYPE=Release ` -D BUILD_WERROR=ON ` - -C external/helper.cmake + -D "CMAKE_TOOLCHAIN_FILE=${{ env.VCPKG_ROOT }}/scripts/buildsystems/vcpkg.cmake" ` + -D VCPKG_TARGET_TRIPLET=arm64ec-windows - run: cmake --build build_arm64xec/ --config Release - run: cmake --install build_arm64xec --prefix build/install --config Release - run: ctest --parallel --output-on-failure -C Release --test-dir build_arm64xec/ @@ -332,9 +408,11 @@ jobs: cmake -S. -B build_x64 ` -A x64 ` -D BUILD_TESTS=ON ` + -D VCPKG_MANIFEST_FEATURES=tests ` -D CMAKE_BUILD_TYPE=Release ` -D BUILD_WERROR=ON ` - -C external/helper.cmake + -D "CMAKE_TOOLCHAIN_FILE=${{ env.VCPKG_ROOT }}/scripts/buildsystems/vcpkg.cmake" ` + -D VCPKG_TARGET_TRIPLET=x64-windows - run: cmake --build build_x64/ --config Release - run: ctest --parallel --output-on-failure -C Release --test-dir build_x64/ env: @@ -352,15 +430,22 @@ jobs: config: [ Debug, Release ] steps: - uses: actions/checkout@v7 + - uses: ./.github/actions/setup-vcpkg + with: + job-index: ${{ strategy.job-index }} - uses: ilammy/msvc-dev-cmd@v1 + # ilammy/msvc-dev-cmd sets its own VCPKG_ROOT (VS's bundled vcpkg), + # clobbering the one lukka/run-vcpkg exported. RUNVCPKG_VCPKG_ROOT + # is a distinct output var that action doesn't touch. - run: | cmake -S. -B build ` -D CMAKE_C_COMPILER=${{matrix.compiler}} ` -D CMAKE_CXX_COMPILER=${{matrix.compiler}} ` - -D UPDATE_DEPS=ON ` + -D "CMAKE_TOOLCHAIN_FILE=${{ env.RUNVCPKG_VCPKG_ROOT }}/scripts/buildsystems/vcpkg.cmake" ` -D CMAKE_BUILD_TYPE=${{matrix.config}} ` -D BUILD_WERROR=ON ` -D BUILD_TESTS=ON ` + -D VCPKG_MANIFEST_FEATURES=tests ` -G Ninja - run: cmake --build build/ - run: ctest --parallel --output-on-failure --test-dir build/ @@ -377,6 +462,9 @@ jobs: static_build: [ APPLE_STATIC_LOADER=ON, APPLE_STATIC_LOADER=OFF ] steps: - uses: actions/checkout@v7 + - uses: ./.github/actions/setup-vcpkg + with: + job-index: ${{ strategy.job-index }} - uses: actions/setup-python@v6 with: python-version: '3.11' @@ -385,7 +473,8 @@ jobs: -D CMAKE_BUILD_TYPE=${{matrix.config}} \ -D ${{matrix.static_build}} \ -D BUILD_TESTS=ON \ - -D UPDATE_DEPS=ON \ + -D VCPKG_MANIFEST_FEATURES=tests \ + -D "CMAKE_TOOLCHAIN_FILE=${{ env.VCPKG_ROOT }}/scripts/buildsystems/vcpkg.cmake" \ -D BUILD_WERROR=ON \ -D LOADER_ENABLE_ADDRESS_SANITIZER=ON \ -G Ninja @@ -407,6 +496,9 @@ jobs: CMAKE_SYSTEM_NAME: [ iOS, tvOS ] steps: - uses: actions/checkout@v7 + - uses: ./.github/actions/setup-vcpkg + with: + job-index: ${{ strategy.job-index }} - uses: actions/setup-python@v6 with: python-version: '3.11' @@ -415,7 +507,7 @@ jobs: -D CMAKE_SYSTEM_NAME=${{ matrix.CMAKE_SYSTEM_NAME }} \ "-D CMAKE_OSX_ARCHITECTURES=arm64;x86_64" \ -D CMAKE_BUILD_TYPE=Debug \ - -D UPDATE_DEPS=ON \ + -D "CMAKE_TOOLCHAIN_FILE=${{ env.VCPKG_ROOT }}/scripts/buildsystems/vcpkg.cmake" \ -D BUILD_WERROR=ON \ -G Ninja env: @@ -441,16 +533,27 @@ jobs: generator: [ Ninja, Xcode ] steps: - uses: actions/checkout@v7 + - uses: ./.github/actions/setup-vcpkg + with: + job-index: ${{ strategy.job-index }} - uses: actions/setup-python@v6 with: python-version: '3.11' + # vcpkg triplets are normally single-architecture, so gtest would + # only ever be installed for arm64 or x86_64, not both, and + # couldn't link into this job's universal (multi-arch) binaries. + # universal-osx is a custom triplet (see its comment) that gets a + # fat gtest instead, so tests can build and run here like anywhere else. - run: | cmake -S. -B build \ -D CMAKE_BUILD_TYPE=Release \ -D APPLE_STATIC_LOADER=${{matrix.static}} \ "-D CMAKE_OSX_ARCHITECTURES=arm64;x86_64" \ -D BUILD_TESTS=ON \ - -D UPDATE_DEPS=ON \ + -D VCPKG_MANIFEST_FEATURES=tests \ + -D VCPKG_TARGET_TRIPLET=universal-osx \ + -D VCPKG_OVERLAY_TRIPLETS=${{ github.workspace }}/vcpkg-overlays/triplets \ + -D "CMAKE_TOOLCHAIN_FILE=${{ env.VCPKG_ROOT }}/scripts/buildsystems/vcpkg.cmake" \ -D BUILD_WERROR=ON \ -G ${{ matrix.generator }} env: @@ -482,6 +585,9 @@ jobs: shell: bash steps: - uses: actions/checkout@v7 + - uses: ./.github/actions/setup-vcpkg + with: + job-index: ${{ strategy.job-index }} - uses: actions/setup-python@v6 with: python-version: '3.11' @@ -493,7 +599,7 @@ jobs: run: uasm -? - run: | cmake -S. -B build \ - -D UPDATE_DEPS=ON \ + -D "CMAKE_TOOLCHAIN_FILE=${{ env.VCPKG_ROOT }}/scripts/buildsystems/vcpkg.cmake" \ -D CMAKE_BUILD_TYPE=Release \ -D BUILD_WERROR=ON \ -G Ninja @@ -510,12 +616,15 @@ jobs: shell: bash steps: - uses: actions/checkout@v7 + - uses: ./.github/actions/setup-vcpkg + with: + job-index: ${{ strategy.job-index }} - uses: actions/setup-python@v6 with: python-version: '3.11' - run: | cmake -S. -B build \ - -D UPDATE_DEPS=ON \ + -D "CMAKE_TOOLCHAIN_FILE=${{ env.VCPKG_ROOT }}/scripts/buildsystems/vcpkg.cmake" \ -D CMAKE_BUILD_TYPE=Release \ -D BUILD_WERROR=ON \ -D USE_GAS=ON \ @@ -533,13 +642,16 @@ jobs: shell: bash steps: - uses: actions/checkout@v7 + - uses: ./.github/actions/setup-vcpkg + with: + job-index: ${{ strategy.job-index }} - uses: actions/setup-python@v6 with: python-version: '3.11' # Make sure this doesn't fail even without explicitly setting '-D USE_MASM=OFF' and without uasm - run: | cmake -S. -B build \ - -D UPDATE_DEPS=ON \ + -D "CMAKE_TOOLCHAIN_FILE=${{ env.VCPKG_ROOT }}/scripts/buildsystems/vcpkg.cmake" \ -D CMAKE_BUILD_TYPE=Release \ -D BUILD_WERROR=ON \ -G Ninja @@ -556,9 +668,12 @@ jobs: shell: bash steps: - uses: actions/checkout@v7 + - uses: ./.github/actions/setup-vcpkg + with: + job-index: ${{ strategy.job-index }} - run: | cmake -S. -B build \ - -D UPDATE_DEPS=ON \ + -D "CMAKE_TOOLCHAIN_FILE=${{ env.VCPKG_ROOT }}/scripts/buildsystems/vcpkg.cmake" \ -D CMAKE_BUILD_TYPE=Release \ -D BUILD_WERROR=ON \ -D USE_MASM=OFF \ diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 80ba87e53..89bb3b04b 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -28,6 +28,11 @@ on: permissions: {} +env: + # Same vcpkg binary cache setup as build.yml; see there for details. + VCPKG_CACHE_DIR: ${{ github.workspace }}/vcpkg-binary-cache + VCPKG_BINARY_SOURCES: clear;files,${{ github.workspace }}/vcpkg-binary-cache,readwrite + jobs: analyze: name: Analyze @@ -81,9 +86,14 @@ jobs: sudo apt update sudo apt install --yes --no-install-recommends libwayland-dev libxrandr-dev + - uses: ./.github/actions/setup-vcpkg + if: matrix.language == 'cpp' + with: + job-index: ${{ strategy.job-index }} + - name: Generate build files if: matrix.language == 'cpp' - run: cmake -S. -B build -D CMAKE_BUILD_TYPE=Release -D UPDATE_DEPS=ON + run: cmake -S. -B build -D CMAKE_BUILD_TYPE=Release -D CMAKE_TOOLCHAIN_FILE=${{ env.VCPKG_ROOT }}/scripts/buildsystems/vcpkg.cmake env: CC: gcc CXX: g++ diff --git a/.gitignore b/.gitignore index ae5e0cd38..27afe58a4 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,9 @@ # By default we install dependencies to the external directory /external/* +# vcpkg manifest-mode install directory (when not nested under a build/ dir) +/vcpkg_installed/ + # Ignore any build directories *build*/ diff --git a/BUILD.md b/BUILD.md index d5623ddfa..64609f3e9 100644 --- a/BUILD.md +++ b/BUILD.md @@ -17,10 +17,7 @@ Instructions for building this repository on Linux, Windows, and MacOS. - [Test Dependencies](#test-dependencies) - [Warnings as errors off by default!](#warnings-as-errors-off-by-default) - [Build and Install Directory Locations](#build-and-install-directory-locations) - - [Building Dependent Repositories with Known-Good Revisions](#building-dependent-repositories-with-known-good-revisions) - - [Automatically](#automatically) - - [Manually](#manually) - - [Notes About the Manual Option](#notes-about-the-manual-option) + - [Getting Dependencies via vcpkg](#getting-dependencies-via-vcpkg) - [Generated source code](#generated-source-code) - [Build Options](#build-options) - [Building On Windows](#building-on-windows) @@ -82,6 +79,7 @@ indicated by *install_dir*: 1. `C99` capable compiler 2. `CMake` version 3.22.1 or greater 3. `Git` +4. `vcpkg` (see [Getting Dependencies via vcpkg](#getting-dependencies-via-vcpkg) below) ### Building with Code Generation Requirements @@ -104,16 +102,9 @@ some other suitable source if you intend to run Vulkan applications. ### Repository Dependencies -This repository attempts to resolve some of its dependencies by using -components found from the following places, in this order: - -1. CMake or Environment variable overrides (e.g., -D VULKAN_HEADERS_INSTALL_DIR) -2. System-installed packages, mostly applicable on Linux - -Dependencies that cannot be resolved by the SDK or installed packages must be -resolved with the "install directory" override and are listed below. The -"install directory" override can also be used to force the use of a specific -version of that dependency. +This repository resolves its dependencies using [vcpkg](https://github.com/microsoft/vcpkg) +in manifest mode. See [Getting Dependencies via vcpkg](#getting-dependencies-via-vcpkg) below +for setup instructions. #### Vulkan-Headers @@ -125,12 +116,9 @@ required to build the loader. The loader tests depend on the [Google Test](https://github.com/google/googletest) library and on Windows platforms depends on the [Microsoft Detours](https://github.com/microsoft/Detours) library. - -To build the tests, pass both `-D UPDATE_DEPS=ON` and `-D BUILD_TESTS=ON` options when generating the project: -```bash -cmake ... -D UPDATE_DEPS=ON -D BUILD_TESTS=ON ... -``` -This will ensure googletest and detours is downloaded and the appropriate version is used. +Both are declared under the `tests` feature in [`vcpkg.json`](vcpkg.json). vcpkg only installs +feature dependencies when explicitly requested, so building the tests requires passing both +`-D BUILD_TESTS=ON` and `-D VCPKG_MANIFEST_FEATURES=tests` to CMake. ### Warnings as errors off by default! @@ -149,67 +137,58 @@ the repository and place the `install` directory as a child of the `build` directory. The remainder of these instructions follow this convention, although you can place these directories in any location. -### Building Dependent Repositories with Known-Good Revisions +### Getting Dependencies via vcpkg -There is a Python utility script, `scripts/update_deps.py`, that you can use -to gather and build the dependent repositories mentioned above. -This program uses information stored in the `scripts/known-good.json` file -to checkout dependent repository revisions that are known to be compatible with -the revision of this repository that you currently have checked out. +This repository resolves its dependencies (Vulkan-Headers and, when building tests, +googletest and detours) using [vcpkg](https://github.com/microsoft/vcpkg) in manifest +mode. The [`vcpkg.json`](vcpkg.json) file at the root of the repository pins a +`builtin-baseline` commit of vcpkg, so that everyone building the repository resolves +the same dependency versions. -You can choose to do this automatically or manually. -The first step to either is cloning the Vulkan-Loader repo and stepping into -that newly cloned folder: +First, clone and bootstrap vcpkg somewhere on your machine (this only needs to be done +once, and can be shared across other repositories that also use vcpkg): ``` - git clone git@github.com:KhronosGroup/Vulkan-Loader.git - cd Vulkan-Loader + git clone https://github.com/microsoft/vcpkg.git + ./vcpkg/bootstrap-vcpkg.sh # bootstrap-vcpkg.bat on Windows ``` -#### Automatically +Then set the `VCPKG_ROOT` environment variable to point at that clone. This is the +standard vcpkg convention, and lets you reference vcpkg's toolchain file below without +hardcoding its path (it's also what this repository's CI does, via +[`lukka/run-vcpkg`](https://github.com/lukka/run-vcpkg)): -On the other hand, if you choose to let the CMake scripts do all the -heavy-lifting, you may just trigger the following CMake commands: +``` + export VCPKG_ROOT= # on Windows: setx VCPKG_ROOT +``` + +Then, when configuring the Vulkan-Loader build, point CMake at vcpkg's toolchain file: ``` - cmake -S . -B build -D UPDATE_DEPS=On + git clone git@github.com:KhronosGroup/Vulkan-Loader.git + cd Vulkan-Loader + cmake -S . -B build -D CMAKE_TOOLCHAIN_FILE=$VCPKG_ROOT/scripts/buildsystems/vcpkg.cmake cmake --build build ``` -#### Manually +vcpkg installs the dependencies declared in `vcpkg.json` into `build/vcpkg_installed` +automatically the first time CMake configures. To also build the tests, pass both +`-D BUILD_TESTS=ON` and `-D VCPKG_MANIFEST_FEATURES=tests`; the latter tells vcpkg to +additionally install googletest (and, on Windows, detours) via the `tests` feature. -To manually update the dependencies you now must create the build folder, and -run the update deps script followed by the necessary CMake build commands: +> Note: `CMAKE_TOOLCHAIN_FILE` must be set on the initial `cmake` configure invocation; +> it can't be set from inside `CMakeLists.txt` after `project()` has already run. On +> Windows, `setx` only takes effect in new shells opened after running it. -``` - mkdir build - cd build - python ../scripts/update_deps.py - cmake -C helper.cmake .. - cmake --build . -``` +CI does not check in a vcpkg clone; it bootstraps and caches vcpkg via the +[`lukka/run-vcpkg`](https://github.com/lukka/run-vcpkg) GitHub Action, which exposes the +resulting checkout to later steps as `VCPKG_ROOT` (see `.github/workflows/build.yml`). -##### Notes About the Manual Option - -- You may need to adjust some of the CMake options based on your platform. See - the platform-specific sections later in this document. -- The `update_deps.py` script fetches and builds the dependent repositories in - the current directory when it is invoked. -- The `--dir` option for `update_deps.py` can be used to relocate the - dependent repositories to another arbitrary directory using an absolute or - relative path. -- The `update_deps.py` script generates a file named `helper.cmake` and places - it in the same directory as the dependent repositories. - This file contains CMake commands to set the CMake `*_INSTALL_DIR` variables - that are used to point to the install artifacts of the dependent repositories. - The `-C helper.cmake` option is used to set these variables when you generate - the build files. -- If using "MINGW" (Git For Windows), you may wish to run - `winpty update_deps.py` in order to avoid buffering all of the script's - "print" output until the end and to retain the ability to interrupt script - execution. -- Please use `update_deps.py --help` to list additional options and read the - internal documentation in `update_deps.py` for further information. +This repository also ships a `vulkan-headers` [overlay port](vcpkg-overlays/vulkan-headers) +(registered in [`vcpkg-configuration.json`](vcpkg-configuration.json)) that pins Vulkan-Headers +to the exact upstream tag this repository's generated code was built against. It is needed +because vcpkg's registry port for `vulkan-headers` can lag behind the latest upstream release; +once the registry catches up, the overlay can be removed in favor of the registry port. ### Generated source code @@ -304,7 +283,7 @@ Open a developer command prompt and enter: cd Vulkan-Loader mkdir build cd build - cmake -A x64 -D UPDATE_DEPS=ON .. + cmake -A x64 -D CMAKE_TOOLCHAIN_FILE=%VCPKG_ROOT%/scripts/buildsystems/vcpkg.cmake .. cmake --build . The above commands instruct CMake to find and use the default Visual Studio @@ -320,7 +299,7 @@ create a build directory and generate the Visual Studio project files: cd Vulkan-Loader mkdir build cd build - cmake -D UPDATE_DEPS=ON -G "Visual Studio 16 2019" -A x64 .. + cmake -D CMAKE_TOOLCHAIN_FILE=%VCPKG_ROOT%/scripts/buildsystems/vcpkg.cmake -G "Visual Studio 16 2019" -A x64 .. > Note: The `..` parameter tells `cmake` the location of the top of the > repository. If you place your build directory someplace else, you'll need to @@ -332,15 +311,11 @@ Supported Visual Studio generators: * `Visual Studio 17 2022` * `Visual Studio 16 2019` -The `-A` option is used to select either the "Win32", "x64", or "ARM64 architecture. +The `-A` option is used to select either the "Win32", "x64", or "ARM64" architecture. -When generating the project files, the absolute path to a Vulkan-Headers -install directory must be provided. This can be done automatically by the -`-D UPDATE_DEPS=ON` option, by directly setting the -`VULKAN_HEADERS_INSTALL_DIR` environment variable, or by setting the -`VULKAN_HEADERS_INSTALL_DIR` CMake variable with the `-D` CMake option. In -either case, the variable should point to the installation directory of a -Vulkan-Headers repository built with the install target. +When generating the project files, `CMAKE_TOOLCHAIN_FILE` must point at +`/scripts/buildsystems/vcpkg.cmake` so that vcpkg can resolve the +Vulkan-Headers dependency; see [Getting Dependencies via vcpkg](#getting-dependencies-via-vcpkg). The above steps create a Windows solution file named `Vulkan-Loader.sln` in the build directory. @@ -409,7 +384,7 @@ CMake with the `--build` option or `make` to build from the command line. cd Vulkan-Loader mkdir build cd build - cmake -D UPDATE_DEPS=ON .. + cmake -D CMAKE_TOOLCHAIN_FILE=$VCPKG_ROOT/scripts/buildsystems/vcpkg.cmake .. make See below for the details. @@ -423,7 +398,7 @@ create a build directory and generate the make files. mkdir build cd build cmake -D CMAKE_BUILD_TYPE=Debug \ - -D VULKAN_HEADERS_INSTALL_DIR=absolute_path_to_install_dir \ + -D CMAKE_TOOLCHAIN_FILE=$VCPKG_ROOT/scripts/buildsystems/vcpkg.cmake \ -D CMAKE_INSTALL_PREFIX=install .. > Note: The `..` parameter tells `cmake` the location of the top of the @@ -432,13 +407,9 @@ create a build directory and generate the make files. Use `-D CMAKE_BUILD_TYPE` to specify a Debug or Release build. -When generating the project files, the absolute path to a Vulkan-Headers -install directory must be provided. This can be done automatically by the -`-D UPDATE_DEPS=ON` option, by directly setting the `VULKAN_HEADERS_INSTALL_DIR` -environment variable, or by setting the `VULKAN_HEADERS_INSTALL_DIR` CMake -variable with the `-D` CMake option. -In either case, the variable should point to the installation directory of a -Vulkan-Headers repository built with the install target. +When generating the project files, `CMAKE_TOOLCHAIN_FILE` must point at +`/scripts/buildsystems/vcpkg.cmake` so that vcpkg can resolve the +Vulkan-Headers dependency; see [Getting Dependencies via vcpkg](#getting-dependencies-via-vcpkg). > Note: For Linux, the default value for `CMAKE_INSTALL_PREFIX` is > `/usr/local`, which would be used if you do not specify @@ -543,7 +514,7 @@ https://gitlab.kitware.com/cmake/cmake/-/issues/25317 Your PKG_CONFIG configuration may be different, depending on your distribution. -You can the `PKG_CONFIG_PATH` environment variable to address this issue. +You can use the `PKG_CONFIG_PATH` environment variable to address this issue. Finally, build the repository normally as explained above. @@ -577,17 +548,13 @@ Clone the Vulkan-Loader repository: This generator is the default generator. -When generating the project files, the absolute path to a Vulkan-Headers -install directory must be provided. This can be done automatically by the -`-D UPDATE_DEPS=ON` option, by directly setting the -`VULKAN_HEADERS_INSTALL_DIR` environment variable, or by setting the -`VULKAN_HEADERS_INSTALL_DIR` CMake variable with the `-D` CMake option. In -either case, the variable should point to the installation directory of a -Vulkan-Headers repository built with the install target. +When generating the project files, `CMAKE_TOOLCHAIN_FILE` must point at +`/scripts/buildsystems/vcpkg.cmake` so that vcpkg can resolve the +Vulkan-Headers dependency; see [Getting Dependencies via vcpkg](#getting-dependencies-via-vcpkg). mkdir build cd build - cmake -D UPDATE_DEPS=ON -D VULKAN_HEADERS_INSTALL_DIR=absolute_path_to_install_dir -D CMAKE_BUILD_TYPE=Debug .. + cmake -D CMAKE_TOOLCHAIN_FILE=$VCPKG_ROOT/scripts/buildsystems/vcpkg.cmake -D CMAKE_BUILD_TYPE=Debug .. make To speed up the build on a multi-core machine, use the `-j` option for `make` @@ -651,8 +618,9 @@ No guarantees are made about the use of the fallback code paths. ## Tests -To build tests, make sure that the `BUILD_TESTS` option is set to true. Using -the command line, this looks like `-D BUILD_TESTS=ON`. +To build tests, make sure that the `BUILD_TESTS` option is set to true, and that the vcpkg +`tests` manifest feature is enabled so vcpkg installs the test dependencies. Using the command +line, this looks like `-D BUILD_TESTS=ON -D VCPKG_MANIFEST_FEATURES=tests`. This project is configured to run with `ctest`, which makes it easy to run the tests. To run the tests, change the directory to that of the build direction, and diff --git a/CMakeLists.txt b/CMakeLists.txt index c02db9d87..8d1b9a308 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -31,8 +31,6 @@ endif() # variant (e.g. Vulkan SC) set(API_TYPE "vulkan") -add_subdirectory(scripts) - set(CMAKE_C_STANDARD 99) set(CMAKE_C_STANDARD_REQUIRED ON) set(CMAKE_C_EXTENSIONS OFF) @@ -294,9 +292,12 @@ endif() option(LOADER_CODEGEN "Enable vulkan loader code generation") if(LOADER_CODEGEN) find_package(Python3 REQUIRED) + # VulkanHeaders_DIR is /share/cmake/VulkanHeaders (set by find_package(... CONFIG) above); + # walk back up to since the package config doesn't expose the registry path directly. + get_filename_component(vulkan_headers_prefix "${VulkanHeaders_DIR}/../../.." ABSOLUTE) add_custom_target(loader_codegen COMMAND Python3::Interpreter ${PROJECT_SOURCE_DIR}/scripts/generate_source.py - "${VULKAN_HEADERS_INSTALL_DIR}/${CMAKE_INSTALL_DATADIR}/vulkan/registry" + "${vulkan_headers_prefix}/${CMAKE_INSTALL_DATADIR}/vulkan/registry" --generated-version ${VulkanHeaders_VERSION} --incremental --api ${API_TYPE} ) endif() diff --git a/REUSE.toml b/REUSE.toml index 45aa84b1e..4c78b704a 100644 --- a/REUSE.toml +++ b/REUSE.toml @@ -95,19 +95,6 @@ SPDX-FileCopyrightText = [ ] SPDX-License-Identifier = "Apache-2.0" -[[annotations]] -path = [ - "scripts/update_deps.py", -] -precedence = "closest" -SPDX-FileCopyrightText = [ - "The Glslang Authors.", - "Valve Corporation", - "LunarG, Inc.", - "RasterGrid Kft.", -] -SPDX-License-Identifier = "Apache-2.0" - [[annotations]] path = [ "loader/asm_test_aarch32.S", diff --git a/scripts/CMakeLists.txt b/scripts/CMakeLists.txt deleted file mode 100644 index f52f3dcd6..000000000 --- a/scripts/CMakeLists.txt +++ /dev/null @@ -1,146 +0,0 @@ -# ~~~ -# Copyright (c) 2023 LunarG, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ~~~ - -option(UPDATE_DEPS "Run update_deps.py for user") -if (UPDATE_DEPS) - find_package(Python3 REQUIRED QUIET) - - set(update_dep_py "${CMAKE_CURRENT_LIST_DIR}/update_deps.py") - set(known_good_json "${CMAKE_CURRENT_LIST_DIR}/known_good.json") - - set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS ${update_dep_py} ${known_good_json}) - - list(APPEND update_dep_command "${update_dep_py}") - list(APPEND update_dep_command "--generator") - list(APPEND update_dep_command "${CMAKE_GENERATOR}") - - if (CMAKE_GENERATOR_PLATFORM) - list(APPEND update_dep_command "--arch") - list(APPEND update_dep_command "${CMAKE_GENERATOR_PLATFORM}") - endif() - - if ("${CMAKE_OSX_ARCHITECTURES}" MATCHES ";") - list(APPEND update_dep_command "--osx-archs") - string(REPLACE ";" ":" osx_archs "${CMAKE_OSX_ARCHITECTURES}") - list(APPEND update_dep_command "${osx_archs}") - - set(APPLE_UNIVERSAL_BINARY ON) - endif() - - if (NOT CMAKE_BUILD_TYPE) - message(WARNING "CMAKE_BUILD_TYPE not set. Using Debug for dependency build type") - set(_build_type Debug) - else() - set(_build_type ${CMAKE_BUILD_TYPE}) - endif() - list(APPEND update_dep_command "--config") - list(APPEND update_dep_command "${_build_type}") - list(APPEND update_dep_command "--api") - list(APPEND update_dep_command "${API_TYPE}") - - set(UPDATE_DEPS_DIR_SUFFIX "${_build_type}") - if (CMAKE_CROSSCOMPILING) - if (APPLE_UNIVERSAL_BINARY) - set(UPDATE_DEPS_DIR_SUFFIX "${CMAKE_SYSTEM_NAME}/${UPDATE_DEPS_DIR_SUFFIX}/universal") - else() - set(UPDATE_DEPS_DIR_SUFFIX "${CMAKE_SYSTEM_NAME}/${UPDATE_DEPS_DIR_SUFFIX}/${CMAKE_SYSTEM_PROCESSOR}") - endif() - else() - if (APPLE_UNIVERSAL_BINARY) - set(UPDATE_DEPS_DIR_SUFFIX "${UPDATE_DEPS_DIR_SUFFIX}/universal") - else() - math(EXPR bitness "8 * ${CMAKE_SIZEOF_VOID_P}") - set(UPDATE_DEPS_DIR_SUFFIX "${UPDATE_DEPS_DIR_SUFFIX}/${bitness}") - endif() - endif() - set(UPDATE_DEPS_DIR "${PROJECT_SOURCE_DIR}/external/${UPDATE_DEPS_DIR_SUFFIX}" CACHE PATH "Location where update_deps.py installs packages") - list(APPEND update_dep_command "--dir" ) - list(APPEND update_dep_command "${UPDATE_DEPS_DIR}") - - if (NOT BUILD_TESTS) - list(APPEND update_dep_command "--optional=tests") - endif() - - if (UPDATE_DEPS_SKIP_EXISTING_INSTALL) - list(APPEND update_dep_command "--skip-existing-install") - endif() - - list(APPEND cmake_vars "CMAKE_TOOLCHAIN_FILE") - - # Avoids manually setting CMAKE_SYSTEM_NAME unless it's needed: - # https://cmake.org/cmake/help/latest/variable/CMAKE_CROSSCOMPILING.html - if (NOT "${CMAKE_SYSTEM_NAME}" STREQUAL "${CMAKE_HOST_SYSTEM_NAME}") - list(APPEND cmake_vars "CMAKE_SYSTEM_NAME") - endif() - if (APPLE) - list(APPEND cmake_vars "CMAKE_OSX_DEPLOYMENT_TARGET") - endif() - if (NOT MSVC_IDE) - list(APPEND cmake_vars "CMAKE_CXX_COMPILER" "CMAKE_C_COMPILER" "CMAKE_ASM_COMPILER") - endif() - if (ANDROID) - list(APPEND cmake_vars "ANDROID_PLATFORM" "CMAKE_ANDROID_ARCH_ABI" "CMAKE_ANDROID_STL_TYPE" "CMAKE_ANDROID_RTTI" "CMAKE_ANDROID_EXCEPTIONS" "ANDROID_USE_LEGACY_TOOLCHAIN_FILE") - endif() - - set(cmake_var) - foreach(var IN LISTS cmake_vars) - if (DEFINED ${var}) - list(APPEND update_dep_command "--cmake_var") - list(APPEND update_dep_command "${var}=${${var}}") - endif() - endforeach() - - if (cmake_var) - list(APPEND update_dep_command "${cmake_var}") - endif() - - file(TIMESTAMP ${update_dep_py} timestamp_1) - file(TIMESTAMP ${known_good_json} timestamp_2) - - string("MD5" md5_hash "${timestamp_1}-${timestamp_2}-${update_dep_command}") - - set(UPDATE_DEPS_HASH "0" CACHE STRING "Default value until we run update_deps.py") - mark_as_advanced(UPDATE_DEPS_HASH) - - if ("${UPDATE_DEPS_HASH}" STREQUAL "0") - list(APPEND update_dep_command "--clean-build") - list(APPEND update_dep_command "--clean-install") - endif() - - if ("${md5_hash}" STREQUAL $CACHE{UPDATE_DEPS_HASH}) - message(DEBUG "update_deps.py: no work to do.") - else() - execute_process( - COMMAND ${Python3_EXECUTABLE} ${update_dep_command} - RESULT_VARIABLE _update_deps_result - ) - if (NOT (${_update_deps_result} EQUAL 0)) - message(FATAL_ERROR "Could not run update_deps.py which is necessary to download dependencies.") - endif() - set(UPDATE_DEPS_HASH ${md5_hash} CACHE STRING "Ensure we only run update_deps.py when we need to." FORCE) - include("${UPDATE_DEPS_DIR}/helper.cmake") - endif() -endif() -if (VULKAN_HEADERS_INSTALL_DIR) - list(APPEND CMAKE_PREFIX_PATH ${VULKAN_HEADERS_INSTALL_DIR}) - set(CMAKE_REQUIRE_FIND_PACKAGE_VulkanHeaders TRUE PARENT_SCOPE) -endif() - -set(CMAKE_PREFIX_PATH ${CMAKE_PREFIX_PATH} PARENT_SCOPE) - -if (CMAKE_CROSSCOMPILING) - set(CMAKE_FIND_ROOT_PATH ${CMAKE_FIND_ROOT_PATH} ${CMAKE_PREFIX_PATH} PARENT_SCOPE) -endif() diff --git a/scripts/gn/DEPS b/scripts/gn/DEPS index 8cf931a82..408b51cdb 100644 --- a/scripts/gn/DEPS +++ b/scripts/gn/DEPS @@ -34,6 +34,11 @@ deps = { 'url': '{chromium_git}/chromium/src/tools/clang.git@566877f1ff1a5fa6beaca3ab4b47bd0b92eb614f', }, + # Kept in sync with the vulkan-headers version in vcpkg.json; this isn't automated, but the "chromium" CI job will fail to build if the two drift far enough apart to matter. + 'external/Vulkan-Headers': { + 'url': 'https://github.com/KhronosGroup/Vulkan-Headers.git@e613b8277f4840b3f3292867bbd3639950676c37', + }, + 'third_party/ninja': { 'packages': [ { diff --git a/scripts/gn/gn.py b/scripts/gn/gn.py index 52b20c580..c41b70e36 100755 --- a/scripts/gn/gn.py +++ b/scripts/gn/gn.py @@ -17,25 +17,25 @@ def BuildGn(): if not os.path.exists(RepoRelative("depot_tools")): print("Cloning Chromium depot_tools\n", flush=True) clone_cmd = 'git clone https://chromium.googlesource.com/chromium/tools/depot_tools.git depot_tools'.split(" ") - subprocess.call(clone_cmd) + subprocess.run(clone_cmd, check=True) os.environ['PATH'] = os.environ.get('PATH') + ":" + RepoRelative("depot_tools") print("Updating Repo Dependencies and GN Toolchain\n", flush=True) update_cmd = './scripts/gn/update_deps.sh' - subprocess.call(update_cmd) + subprocess.run(update_cmd, check=True) print("Checking Header Dependencies\n", flush=True) gn_check_cmd = 'gn gen --check out/Debug'.split(" ") - subprocess.call(gn_check_cmd) + subprocess.run(gn_check_cmd, check=True) print("Generating Ninja Files\n", flush=True) gn_gen_cmd = 'gn gen out/Debug'.split(" ") - subprocess.call(gn_gen_cmd) + subprocess.run(gn_gen_cmd, check=True) print("Running Ninja Build\n", flush=True) ninja_build_cmd = 'ninja -C out/Debug'.split(" ") - subprocess.call(ninja_build_cmd) + subprocess.run(ninja_build_cmd, check=True) # # Module Entrypoint diff --git a/scripts/gn/update_deps.sh b/scripts/gn/update_deps.sh index 958992495..a722f02e6 100755 --- a/scripts/gn/update_deps.sh +++ b/scripts/gn/update_deps.sh @@ -17,9 +17,6 @@ # Execute at repo root cd "$(dirname $0)/../../" -# Use update_deps.py to update source dependencies from /scripts/known_good.json -scripts/update_deps.py --dir="external" --no-build - cat << EOF > .gn buildconfig = "//build/config/BUILDCONFIG.gn" secondary_source = "//scripts/gn/secondary/" diff --git a/scripts/known_good.json b/scripts/known_good.json deleted file mode 100644 index 79628a152..000000000 --- a/scripts/known_good.json +++ /dev/null @@ -1,45 +0,0 @@ -{ - "repos": [ - { - "name": "Vulkan-Headers", - "api": "vulkan", - "url": "https://github.com/KhronosGroup/Vulkan-Headers.git", - "sub_dir": "Vulkan-Headers", - "build_dir": "Vulkan-Headers/build", - "install_dir": "Vulkan-Headers/build/install", - "commit": "v1.4.356" - }, - { - "name": "googletest", - "url": "https://github.com/google/googletest.git", - "sub_dir": "googletest", - "build_dir": "googletest", - "install_dir": "googletest", - "build_step": "skip", - "commit": "v1.14.0", - "optional": [ - "tests" - ] - }, - { - "name": "detours", - "url": "https://github.com/microsoft/Detours.git", - "sub_dir": "detours", - "build_dir": "detours", - "install_dir": "detours", - "build_step": "skip", - "commit": "4b8c659f549b0ab21cf649377c7a84eb708f5e68", - "optional": [ - "tests" - ], - "build_platforms": [ - "windows" - ] - } - ], - "install_names": { - "Vulkan-Headers": "VULKAN_HEADERS_INSTALL_DIR", - "googletest": "GOOGLETEST_INSTALL_DIR", - "detours": "DETOURS_INSTALL_DIR" - } -} diff --git a/scripts/update_deps.py b/scripts/update_deps.py deleted file mode 100755 index acddd844a..000000000 --- a/scripts/update_deps.py +++ /dev/null @@ -1,824 +0,0 @@ -#!/usr/bin/env python3 - -# Copyright 2017 The Glslang Authors. All rights reserved. -# Copyright (c) 2018-2023 Valve Corporation -# Copyright (c) 2018-2023 LunarG, Inc. -# Copyright (c) 2023-2023 RasterGrid Kft. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# This script was heavily leveraged from KhronosGroup/glslang -# update_glslang_sources.py. -"""update_deps.py - -Get and build dependent repositories using known-good commits. - -Purpose -------- - -This program is intended to assist a developer of this repository -(the "home" repository) by gathering and building the repositories that -this home repository depend on. It also checks out each dependent -repository at a "known-good" commit in order to provide stability in -the dependent repositories. - -Known-Good JSON Database ------------------------- - -This program expects to find a file named "known-good.json" in the -same directory as the program file. This JSON file is tailored for -the needs of the home repository by including its dependent repositories. - -Program Options ---------------- - -See the help text (update_deps.py --help) for a complete list of options. - -Program Operation ------------------ - -The program uses the user's current directory at the time of program -invocation as the location for fetching and building the dependent -repositories. The user can override this by using the "--dir" option. - -For example, a directory named "build" in the repository's root directory -is a good place to put the dependent repositories because that directory -is not tracked by Git. (See the .gitignore file.) The "external" directory -may also be a suitable location. -A user can issue: - -$ cd My-Repo -$ mkdir build -$ cd build -$ ../scripts/update_deps.py - -or, to do the same thing, but using the --dir option: - -$ cd My-Repo -$ mkdir build -$ scripts/update_deps.py --dir=build - -With these commands, the "build" directory is considered the "top" -directory where the program clones the dependent repositories. The -JSON file configures the build and install working directories to be -within this "top" directory. - -Note that the "dir" option can also specify an absolute path: - -$ cd My-Repo -$ scripts/update_deps.py --dir=/tmp/deps - -The "top" dir is then /tmp/deps (Linux filesystem example) and is -where this program will clone and build the dependent repositories. - -Helper CMake Config File ------------------------- - -When the program finishes building the dependencies, it writes a file -named "helper.cmake" to the "top" directory that contains CMake commands -for setting CMake variables for locating the dependent repositories. -This helper file can be used to set up the CMake build files for this -"home" repository. - -A complete sequence might look like: - -$ git clone git@github.com:My-Group/My-Repo.git -$ cd My-Repo -$ mkdir build -$ cd build -$ ../scripts/update_deps.py -$ cmake -C helper.cmake .. -$ cmake --build . - -JSON File Schema ----------------- - -There's no formal schema for the "known-good" JSON file, but here is -a description of its elements. All elements are required except those -marked as optional. Please see the "known_good.json" file for -examples of all of these elements. - -- name - -The name of the dependent repository. This field can be referenced -by the "deps.repo_name" structure to record a dependency. - -- api - -The name of the API the dependency is specific to (e.g. "vulkan"). - -- url - -Specifies the URL of the repository. -Example: https://github.com/KhronosGroup/Vulkan-Loader.git - -- sub_dir - -The directory where the program clones the repository, relative to -the "top" directory. - -- build_dir - -The directory used to build the repository, relative to the "top" -directory. - -- install_dir - -The directory used to store the installed build artifacts, relative -to the "top" directory. - -- commit - -The commit used to checkout the repository. This can be a SHA-1 -object name or a refname used with the remote name "origin". - -- deps (optional) - -An array of pairs consisting of a CMake variable name and a -repository name to specify a dependent repo and a "link" to -that repo's install artifacts. For example: - -"deps" : [ - { - "var_name" : "VULKAN_HEADERS_INSTALL_DIR", - "repo_name" : "Vulkan-Headers" - } -] - -which represents that this repository depends on the Vulkan-Headers -repository and uses the VULKAN_HEADERS_INSTALL_DIR CMake variable to -specify the location where it expects to find the Vulkan-Headers install -directory. -Note that the "repo_name" element must match the "name" element of some -other repository in the JSON file. - -- prebuild (optional) -- prebuild_linux (optional) (For Linux and MacOS) -- prebuild_windows (optional) - -A list of commands to execute before building a dependent repository. -This is useful for repositories that require the execution of some -sort of "update" script or need to clone an auxillary repository like -googletest. - -The commands listed in "prebuild" are executed first, and then the -commands for the specific platform are executed. - -- custom_build (optional) - -A list of commands to execute as a custom build instead of using -the built in CMake way of building. Requires "build_step" to be -set to "custom" - -You can insert the following keywords into the commands listed in -"custom_build" if they require runtime information (like whether the -build config is "Debug" or "Release"). - -Keywords: -{0} reference to a dictionary of repos and their attributes -{1} reference to the command line arguments set before start -{2} reference to the CONFIG_MAP value of config. - -Example: -{2} returns the CONFIG_MAP value of config e.g. debug -> Debug -{1}.config returns the config variable set when you ran update_dep.py -{0}[Vulkan-Headers][repo_root] returns the repo_root variable from - the Vulkan-Headers GoodRepo object. - -- cmake_options (optional) - -A list of options to pass to CMake during the generation phase. - -- ci_only (optional) - -A list of environment variables where one must be set to "true" -(case-insensitive) in order for this repo to be fetched and built. -This list can be used to specify repos that should be built only in CI. - -- build_step (optional) - -Specifies if the dependent repository should be built or not. This can -have a value of 'build', 'custom', or 'skip'. The dependent repositories are -built by default. - -- build_platforms (optional) - -A list of platforms the repository will be built on. -Legal options include: -"windows" -"linux" -"darwin" -"android" - -Builds on all platforms by default. - -Note ----- - -The "sub_dir", "build_dir", and "install_dir" elements are all relative -to the effective "top" directory. Specifying absolute paths is not -supported. However, the "top" directory specified with the "--dir" -option can be a relative or absolute path. - -""" - -import argparse -import json -import os -import os.path -import subprocess -import sys -import platform -import multiprocessing -import shlex -import shutil -import stat -import time - -KNOWN_GOOD_FILE_NAME = 'known_good.json' - -CONFIG_MAP = { - 'debug': 'Debug', - 'release': 'Release', - 'relwithdebinfo': 'RelWithDebInfo', - 'minsizerel': 'MinSizeRel' -} - -# NOTE: CMake also uses the VERBOSE environment variable. This is intentional. -VERBOSE = os.getenv("VERBOSE") - -DEVNULL = open(os.devnull, 'wb') - - -def on_rm_error( func, path, exc_info): - """Error handler for recursively removing a directory. The - shutil.rmtree function can fail on Windows due to read-only files. - This handler will change the permissions for the file and continue. - """ - os.chmod( path, stat.S_IWRITE ) - os.unlink( path ) - -def make_or_exist_dirs(path): - "Wrapper for os.makedirs that tolerates the directory already existing" - # Could use os.makedirs(path, exist_ok=True) if we drop python2 - if not os.path.isdir(path): - os.makedirs(path) - -def command_output(cmd, directory): - # Runs a command in a directory and returns its standard output stream. - # Captures the standard error stream and prints it an error occurs. - # Raises a RuntimeError if the command fails to launch or otherwise fails. - if VERBOSE: - print('In {d}: {cmd}'.format(d=directory, cmd=cmd)) - - result = subprocess.run(cmd, cwd=directory, capture_output=True, text=True) - - if result.returncode != 0: - print(f'{result.stderr}', file=sys.stderr) - raise RuntimeError(f'Failed to run {cmd} in {directory}') - - if VERBOSE: - print(result.stdout) - return result.stdout - -def run_cmake_command(cmake_cmd): - # NOTE: Because CMake is an exectuable that runs executables - # stdout/stderr are mixed together. So this combines the outputs - # and prints them properly in case there is a non-zero exit code. - result = subprocess.run(cmake_cmd, - stdout = subprocess.PIPE, - stderr = subprocess.STDOUT, - text = True - ) - - if VERBOSE: - print(result.stdout) - print(f"CMake command: {cmake_cmd} ", flush=True) - - if result.returncode != 0: - print(result.stdout, file=sys.stderr) - sys.exit(result.returncode) - -def escape(path): - return path.replace('\\', '/') - -class GoodRepo(object): - """Represents a repository at a known-good commit.""" - - def __init__(self, json, args): - """Initializes this good repo object. - - Args: - 'json': A fully populated JSON object describing the repo. - 'args': Results from ArgumentParser - """ - self._json = json - self._args = args - # Required JSON elements - self.name = json['name'] - self.url = json['url'] - self.sub_dir = json['sub_dir'] - self.commit = json['commit'] - # Optional JSON elements - self.build_dir = None - self.install_dir = None - if json.get('build_dir'): - self.build_dir = os.path.normpath(json['build_dir']) - if json.get('install_dir'): - self.install_dir = os.path.normpath(json['install_dir']) - self.deps = json['deps'] if ('deps' in json) else [] - self.prebuild = json['prebuild'] if ('prebuild' in json) else [] - self.prebuild_linux = json['prebuild_linux'] if ( - 'prebuild_linux' in json) else [] - self.prebuild_windows = json['prebuild_windows'] if ( - 'prebuild_windows' in json) else [] - self.custom_build = json['custom_build'] if ('custom_build' in json) else [] - self.cmake_options = json['cmake_options'] if ( - 'cmake_options' in json) else [] - self.ci_only = json['ci_only'] if ('ci_only' in json) else [] - self.build_step = json['build_step'] if ('build_step' in json) else 'build' - self.build_platforms = json['build_platforms'] if ('build_platforms' in json) else [] - self.optional = set(json.get('optional', [])) - self.api = json['api'] if ('api' in json) else None - # Absolute paths for a repo's directories - dir_top = os.path.abspath(args.dir) - self.repo_dir = os.path.join(dir_top, self.sub_dir) - if self.build_dir: - self.build_dir = os.path.join(dir_top, self.build_dir) - if self.install_dir: - self.install_dir = os.path.join(dir_top, self.install_dir) - - # By default the target platform is the host platform. - target_platform = platform.system().lower() - # However, we need to account for cross-compiling. - for cmake_var in self._args.cmake_var: - if "android.toolchain.cmake" in cmake_var: - target_platform = 'android' - - self.on_build_platform = False - if self.build_platforms == [] or target_platform in self.build_platforms: - self.on_build_platform = True - - def Clone(self, retries=10, retry_seconds=60): - if VERBOSE: - print('Cloning {n} into {d}'.format(n=self.name, d=self.repo_dir)) - for retry in range(retries): - make_or_exist_dirs(self.repo_dir) - try: - command_output(['git', 'clone', self.url, '.'], self.repo_dir) - # If we get here, we didn't raise an error - return - except RuntimeError as e: - print("Error cloning on iteration {}/{}: {}".format(retry + 1, retries, e)) - if retry + 1 < retries: - if retry_seconds > 0: - print("Waiting {} seconds before trying again".format(retry_seconds)) - time.sleep(retry_seconds) - if os.path.isdir(self.repo_dir): - print("Removing old tree {}".format(self.repo_dir)) - shutil.rmtree(self.repo_dir, onerror=on_rm_error) - continue - - # If we get here, we've exhausted our retries. - print("Failed to clone {} on all retries.".format(self.url)) - raise e - - def Fetch(self, retries=10, retry_seconds=60): - for retry in range(retries): - try: - command_output(['git', 'fetch', 'origin'], self.repo_dir) - # if we get here, we didn't raise an error, and we're done - return - except RuntimeError as e: - print("Error fetching on iteration {}/{}: {}".format(retry + 1, retries, e)) - if retry + 1 < retries: - if retry_seconds > 0: - print("Waiting {} seconds before trying again".format(retry_seconds)) - time.sleep(retry_seconds) - continue - - # If we get here, we've exhausted our retries. - print("Failed to fetch {} on all retries.".format(self.url)) - raise e - - def Checkout(self): - if VERBOSE: - print('Checking out {n} in {d}'.format(n=self.name, d=self.repo_dir)) - - if os.path.exists(os.path.join(self.repo_dir, '.git')): - url_changed = command_output(['git', 'config', '--get', 'remote.origin.url'], self.repo_dir).strip() != self.url - else: - url_changed = False - - if self._args.do_clean_repo or url_changed: - if os.path.isdir(self.repo_dir): - if VERBOSE: - print('Clearing directory {d}'.format(d=self.repo_dir)) - shutil.rmtree(self.repo_dir, onerror = on_rm_error) - if not os.path.exists(os.path.join(self.repo_dir, '.git')): - self.Clone() - self.Fetch() - if len(self._args.ref): - command_output(['git', 'checkout', self._args.ref], self.repo_dir) - else: - command_output(['git', 'checkout', self.commit], self.repo_dir) - - if VERBOSE: - print(command_output(['git', 'status'], self.repo_dir)) - - def CustomPreProcess(self, cmd_str, repo_dict): - return cmd_str.format(repo_dict, self._args, CONFIG_MAP[self._args.config]) - - def PreBuild(self): - """Execute any prebuild steps from the repo root""" - for p in self.prebuild: - command_output(shlex.split(p), self.repo_dir) - if platform.system() == 'Linux' or platform.system() == 'Darwin': - for p in self.prebuild_linux: - command_output(shlex.split(p), self.repo_dir) - if platform.system() == 'Windows': - for p in self.prebuild_windows: - command_output(shlex.split(p), self.repo_dir) - - def CustomBuild(self, repo_dict): - """Execute any custom_build steps from the repo root""" - - # It's not uncommon for builds to not support universal binaries - if self._args.OSX_ARCHITECTURES: - print("Universal Binaries not supported for custom builds", file=sys.stderr) - exit(-1) - - for p in self.custom_build: - cmd = self.CustomPreProcess(p, repo_dict) - command_output(shlex.split(cmd), self.repo_dir) - - def CMakeConfig(self, repos): - """Build CMake command for the configuration phase and execute it""" - if self._args.do_clean_build: - if os.path.isdir(self.build_dir): - shutil.rmtree(self.build_dir, onerror=on_rm_error) - if self._args.do_clean_install: - if os.path.isdir(self.install_dir): - shutil.rmtree(self.install_dir, onerror=on_rm_error) - - # Create and change to build directory - make_or_exist_dirs(self.build_dir) - os.chdir(self.build_dir) - - cmake_cmd = [ - 'cmake', self.repo_dir, - '-DCMAKE_INSTALL_PREFIX=' + self.install_dir - ] - - # Allow users to pass in arbitrary cache variables - for cmake_var in self._args.cmake_var: - pieces = cmake_var.split('=', 1) - cmake_cmd.append('-D{}={}'.format(pieces[0], pieces[1])) - - # For each repo this repo depends on, generate a CMake variable - # definitions for "...INSTALL_DIR" that points to that dependent - # repo's install dir. - for d in self.deps: - dep_commit = [r for r in repos if r.name == d['repo_name']] - if len(dep_commit) and dep_commit[0].on_build_platform: - cmake_cmd.append('-D{var_name}={install_dir}'.format( - var_name=d['var_name'], - install_dir=dep_commit[0].install_dir)) - - # Add any CMake options - for option in self.cmake_options: - cmake_cmd.append(escape(option.format(**self.__dict__))) - - # Set build config for single-configuration generators (this is a no-op on multi-config generators) - cmake_cmd.append(f'-D CMAKE_BUILD_TYPE={CONFIG_MAP[self._args.config]}') - - if self._args.OSX_ARCHITECTURES: - # CMAKE_OSX_ARCHITECTURES must be a semi-colon seperated list - cmake_osx_archs = self._args.OSX_ARCHITECTURES.replace(':', ';') - cmake_cmd.append(f'-D CMAKE_OSX_ARCHITECTURES={cmake_osx_archs}') - - # Use the CMake -A option to select the platform architecture - # without needing a Visual Studio generator. - if platform.system() == 'Windows' and self._args.generator != "Ninja": - cmake_cmd.append('-A') - if self._args.arch.lower() == '64' or self._args.arch == 'x64' or self._args.arch == 'win64': - cmake_cmd.append('x64') - elif self._args.arch == 'arm64': - cmake_cmd.append('arm64') - elif self._args.arch == 'arm64ec': - cmake_cmd.append('arm64ec') - elif self._args.arch == 'arm': - cmake_cmd.append('arm') - else: - cmake_cmd.append('Win32') - - # Apply a generator, if one is specified. This can be used to supply - # a specific generator for the dependent repositories to match - # that of the main repository. - if self._args.generator is not None: - cmake_cmd.extend(['-G', self._args.generator]) - - # Removes warnings related to unused CLI - # EX: Setting CMAKE_CXX_COMPILER for a C project - if not VERBOSE: - cmake_cmd.append("--no-warn-unused-cli") - - run_cmake_command(cmake_cmd) - - def CMakeBuild(self): - """Build CMake command for the build phase and execute it""" - cmake_cmd = ['cmake', '--build', self.build_dir, '--target', 'install', '--config', CONFIG_MAP[self._args.config]] - if self._args.do_clean: - cmake_cmd.append('--clean-first') - - # Xcode / Ninja are parallel by default. - if self._args.generator != "Ninja" or self._args.generator != "Xcode": - cmake_cmd.append('--parallel') - cmake_cmd.append(format(multiprocessing.cpu_count())) - - run_cmake_command(cmake_cmd) - - def Build(self, repos, repo_dict): - """Build the dependent repo and time how long it took""" - if VERBOSE: - print('Building {n} in {d}'.format(n=self.name, d=self.repo_dir)) - print('Build dir = {b}'.format(b=self.build_dir)) - print('Install dir = {i}\n'.format(i=self.install_dir)) - - start = time.time() - - self.PreBuild() - - if self.build_step == 'custom': - self.CustomBuild(repo_dict) - else: - self.CMakeConfig(repos) - self.CMakeBuild() - - total_time = time.time() - start - - print(f"Installed {self.name} ({self.commit}) in {total_time} seconds", flush=True) - - def IsOptional(self, opts): - return len(self.optional.intersection(opts)) > 0 - -def GetGoodRepos(args): - """Returns the latest list of GoodRepo objects. - - The known-good file is expected to be in the same - directory as this script unless overridden by the 'known_good_dir' - parameter. - """ - if args.known_good_dir: - known_good_file = os.path.join( os.path.abspath(args.known_good_dir), - KNOWN_GOOD_FILE_NAME) - else: - known_good_file = os.path.join( - os.path.dirname(os.path.abspath(__file__)), KNOWN_GOOD_FILE_NAME) - with open(known_good_file) as known_good: - return [ - GoodRepo(repo, args) - for repo in json.loads(known_good.read())['repos'] - ] - - -def GetInstallNames(args): - """Returns the install names list. - - The known-good file is expected to be in the same - directory as this script unless overridden by the 'known_good_dir' - parameter. - """ - if args.known_good_dir: - known_good_file = os.path.join(os.path.abspath(args.known_good_dir), - KNOWN_GOOD_FILE_NAME) - else: - known_good_file = os.path.join( - os.path.dirname(os.path.abspath(__file__)), KNOWN_GOOD_FILE_NAME) - with open(known_good_file) as known_good: - install_info = json.loads(known_good.read()) - if install_info.get('install_names'): - return install_info['install_names'] - else: - return None - - -def CreateHelper(args, repos, filename): - """Create a CMake config helper file. - - The helper file is intended to be used with 'cmake -C ' - to build this home repo using the dependencies built by this script. - - The install_names dictionary represents the CMake variables used by the - home repo to locate the install dirs of the dependent repos. - This information is baked into the CMake files of the home repo and so - this dictionary is kept with the repo via the json file. - """ - install_names = GetInstallNames(args) - with open(filename, 'w') as helper_file: - for repo in repos: - # If the repo has an API tag and that does not match - # the target API then skip it - if repo.api is not None and repo.api != args.api: - continue - if install_names and repo.name in install_names and repo.on_build_platform: - helper_file.write('set({var} "{dir}" CACHE STRING "")\n' - .format( - var=install_names[repo.name], - dir=escape(repo.install_dir))) - - -def main(): - parser = argparse.ArgumentParser( - description='Get and build dependent repos at known-good commits') - parser.add_argument( - '--known_good_dir', - dest='known_good_dir', - help="Specify directory for known_good.json file.") - parser.add_argument( - '--dir', - dest='dir', - default='.', - help="Set target directory for repository roots. Default is \'.\'.") - parser.add_argument( - '--ref', - dest='ref', - default='', - help="Override 'commit' with git reference. E.g., 'origin/main'") - parser.add_argument( - '--no-build', - dest='do_build', - action='store_false', - help= - "Clone/update repositories and generate build files without performing compilation", - default=True) - parser.add_argument( - '--clean', - dest='do_clean', - action='store_true', - help="Clean files generated by compiler and linker before building", - default=False) - parser.add_argument( - '--clean-repo', - dest='do_clean_repo', - action='store_true', - help="Delete repository directory before building", - default=False) - parser.add_argument( - '--clean-build', - dest='do_clean_build', - action='store_true', - help="Delete build directory before building", - default=False) - parser.add_argument( - '--clean-install', - dest='do_clean_install', - action='store_true', - help="Delete install directory before building", - default=False) - parser.add_argument( - '--skip-existing-install', - dest='skip_existing_install', - action='store_true', - help="Skip build if install directory exists", - default=False) - parser.add_argument( - '--arch', - dest='arch', - choices=['32', '64', 'x86', 'x64', 'win32', 'win64', 'arm', 'arm64', 'arm64ec'], - type=str.lower, - help="Set build files architecture (Visual Studio Generator Only)", - default='64') - parser.add_argument( - '--config', - dest='config', - choices=['debug', 'release', 'relwithdebinfo', 'minsizerel'], - type=str.lower, - help="Set build files configuration", - default='debug') - parser.add_argument( - '--api', - dest='api', - default='vulkan', - choices=['vulkan'], - help="Target API") - parser.add_argument( - '--generator', - dest='generator', - help="Set the CMake generator", - default=None) - parser.add_argument( - '--optional', - dest='optional', - type=lambda a: set(a.lower().split(',')), - help="Comma-separated list of 'optional' resources that may be skipped. Only 'tests' is currently supported as 'optional'", - default=set()) - parser.add_argument( - '--cmake_var', - dest='cmake_var', - action='append', - metavar='VAR[=VALUE]', - help="Add CMake command line option -D'VAR'='VALUE' to the CMake generation command line; may be used multiple times", - default=[]) - parser.add_argument( - '--osx-archs', - dest='OSX_ARCHITECTURES', - help="Architectures when building a universal binary. Takes a colon seperated list. Ex: arm64:x86_64", - type=str, - default=None) - - args = parser.parse_args() - save_cwd = os.getcwd() - - if args.OSX_ARCHITECTURES: - print(f"Building dependencies as universal binaries targeting {args.OSX_ARCHITECTURES}") - - # Create working "top" directory if needed - make_or_exist_dirs(args.dir) - abs_top_dir = os.path.abspath(args.dir) - - repos = GetGoodRepos(args) - repo_dict = {} - - print('Starting builds in {d}'.format(d=abs_top_dir)) - for repo in repos: - # If the repo has an API tag and that does not match - # the target API then skip it - if repo.api is not None and repo.api != args.api: - continue - - # If the repo has a platform whitelist, skip the repo - # unless we are building on a whitelisted platform. - if not repo.on_build_platform: - continue - - # Skip building the repo if its install directory already exists - # and requested via an option. This is useful for cases where the - # install directory is restored from a cache that is known to be up - # to date. - if args.skip_existing_install and os.path.isdir(repo.install_dir): - print('Skipping build for repo {n} due to existing install directory'.format(n=repo.name)) - continue - - # Skip test-only repos if the --tests option was not passed in - if repo.IsOptional(args.optional): - continue - - field_list = ('url', - 'sub_dir', - 'commit', - 'build_dir', - 'install_dir', - 'deps', - 'prebuild', - 'prebuild_linux', - 'prebuild_windows', - 'custom_build', - 'cmake_options', - 'ci_only', - 'build_step', - 'build_platforms', - 'repo_dir', - 'on_build_platform') - repo_dict[repo.name] = {field: getattr(repo, field) for field in field_list} - - # If the repo has a CI whitelist, skip the repo unless - # one of the CI's environment variable is set to true. - if len(repo.ci_only): - do_build = False - for env in repo.ci_only: - if env not in os.environ: - continue - if os.environ[env].lower() == 'true': - do_build = True - break - if not do_build: - continue - - # Clone/update the repository - repo.Checkout() - - # Build the repository - if args.do_build and repo.build_step != 'skip': - repo.Build(repos, repo_dict) - - # Need to restore original cwd in order for CreateHelper to find json file - os.chdir(save_cwd) - CreateHelper(args, repos, os.path.join(abs_top_dir, 'helper.cmake')) - - sys.exit(0) - - -if __name__ == '__main__': - main() - diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 939ccadfc..9350f1cfa 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -25,47 +25,16 @@ set(CMAKE_CXX_VISIBILITY_PRESET "hidden") # Make sure tests uses the dynamic runtime instead set(CMAKE_MSVC_RUNTIME_LIBRARY "MultiThreaded$<$:Debug>DLL") -if (IS_DIRECTORY "${GOOGLETEST_INSTALL_DIR}/googletest") - set(BUILD_GTEST ON) - set(BUILD_GMOCK OFF) - set(gtest_force_shared_crt ON) - set(INSTALL_GTEST OFF) - add_subdirectory("${GOOGLETEST_INSTALL_DIR}" ${CMAKE_CURRENT_BINARY_DIR}/gtest) -else() - message(FATAL_ERROR "Could not find googletest directory. See BUILD.md") -endif() +find_package(GTest CONFIG REQUIRED) if (WIN32) - if(NOT IS_DIRECTORY ${DETOURS_INSTALL_DIR}) - message(FATAL_ERROR "Could not find detours! See BUILD.md") - endif() - add_library(detours STATIC - ${DETOURS_INSTALL_DIR}/src/creatwth.cpp - ${DETOURS_INSTALL_DIR}/src/detours.cpp - ${DETOURS_INSTALL_DIR}/src/detours.h - ${DETOURS_INSTALL_DIR}/src/detver.h - ${DETOURS_INSTALL_DIR}/src/disasm.cpp - ${DETOURS_INSTALL_DIR}/src/disolarm.cpp - ${DETOURS_INSTALL_DIR}/src/disolarm64.cpp - ${DETOURS_INSTALL_DIR}/src/disolia64.cpp - ${DETOURS_INSTALL_DIR}/src/disolx64.cpp - ${DETOURS_INSTALL_DIR}/src/disolx86.cpp - ${DETOURS_INSTALL_DIR}/src/image.cpp - ${DETOURS_INSTALL_DIR}/src/modules.cpp - ) - target_include_directories(detours PUBLIC ${DETOURS_INSTALL_DIR}/src) - - target_compile_definitions(detours PUBLIC WIN32_LEAN_AND_MEAN) - - if(MSVC) - target_compile_definitions(detours PUBLIC "_CRT_SECURE_NO_WARNINGS=1") - set_target_properties(detours PROPERTIES COMPILE_FLAGS /EHsc) - endif() + # vcpkg's detours port ships a prebuilt static lib + headers, not a CMake package config. + find_path(DETOURS_INCLUDE_DIRS "detours/detours.h" REQUIRED) + find_library(DETOURS_LIBRARY detours REQUIRED) - # Silence errors found in clang-cl - if(CMAKE_C_COMPILER_ID STREQUAL "Clang" AND "${CMAKE_C_SIMULATE_ID}" MATCHES "MSVC") - target_compile_options(detours PRIVATE -Wno-sizeof-pointer-memaccess -Wno-microsoft-goto -Wno-microsoft-cast) - endif() + add_library(detours INTERFACE) + target_include_directories(detours SYSTEM INTERFACE ${DETOURS_INCLUDE_DIRS}/detours) + target_link_libraries(detours INTERFACE ${DETOURS_LIBRARY}) endif() option(ENABLE_LIVE_VERIFICATION_TESTS "Enable tests which expect to run on live drivers. Meant for manual verification only" OFF) diff --git a/tests/framework/CMakeLists.txt b/tests/framework/CMakeLists.txt index 0e6bff12a..c28c6c158 100644 --- a/tests/framework/CMakeLists.txt +++ b/tests/framework/CMakeLists.txt @@ -48,6 +48,6 @@ add_subdirectory(layer) add_library(testing_dependencies STATIC test_environment.cpp test_environment.h) target_link_libraries(testing_dependencies - PUBLIC gtest Vulkan::Headers testing_framework_util shim-library) + PUBLIC GTest::gtest Vulkan::Headers testing_framework_util shim-library) target_include_directories(testing_dependencies PUBLIC ${CMAKE_CURRENT_BINARY_DIR}) diff --git a/tests/framework/README.md b/tests/framework/README.md index 20e1209ae..293a66c72 100644 --- a/tests/framework/README.md +++ b/tests/framework/README.md @@ -18,9 +18,9 @@ By default the Vulkan-Loader repo doesn't enable testing. To turn on building of the tests, set `BUILD_TESTS=ON` in the CMake configuration. -Use the CMake configuration `UPDATE_DEPS=ON` to automatically get all required test dependencies. -Or Ensure that `googletest` is in the `external` directory. -And on Windows only, ensure that the `Detours` library is in the `external` directory. +Test dependencies (`googletest`, and on Windows, `Detours`) are resolved automatically via +vcpkg when configuring with `-D CMAKE_TOOLCHAIN_FILE=/scripts/buildsystems/vcpkg.cmake`; +see [BUILD.md](../../BUILD.md#getting-dependencies-via-vcpkg). Linux & macOS only: The CMake Configuration `LOADER_ENABLE_ADDRESS_SANITIZER` can be used to enable Address Sanitizer. diff --git a/tests/framework/util/CMakeLists.txt b/tests/framework/util/CMakeLists.txt index ec263fa4a..bebfbdf66 100644 --- a/tests/framework/util/CMakeLists.txt +++ b/tests/framework/util/CMakeLists.txt @@ -43,7 +43,7 @@ add_library(testing_framework_util STATIC wide_char_handling.cpp wide_char_handling.h ) -target_link_libraries(testing_framework_util PUBLIC loader_common_options Vulkan::Headers gtest) +target_link_libraries(testing_framework_util PUBLIC loader_common_options Vulkan::Headers GTest::gtest) if(UNIX OR APPLE) target_link_libraries(testing_framework_util PUBLIC ${CMAKE_DL_LIBS}) @@ -69,8 +69,6 @@ if (UNIX) if (LOADER_ENABLE_THREAD_SANITIZER) target_compile_options(testing_framework_util PUBLIC -fsanitize=thread) target_link_options(testing_framework_util PUBLIC -fsanitize=thread) - target_compile_options(gtest PUBLIC -fsanitize=thread) - target_link_options(gtest PUBLIC -fsanitize=thread) endif() endif() diff --git a/vcpkg-configuration.json b/vcpkg-configuration.json new file mode 100644 index 000000000..3e779432e --- /dev/null +++ b/vcpkg-configuration.json @@ -0,0 +1,6 @@ +{ + "overlay-ports": [ + "./vcpkg-overlays/vulkan-headers", + "./vcpkg-overlays/detours" + ] +} diff --git a/vcpkg-overlays/detours/portfile.cmake b/vcpkg-overlays/detours/portfile.cmake new file mode 100644 index 000000000..159bd9c84 --- /dev/null +++ b/vcpkg-overlays/detours/portfile.cmake @@ -0,0 +1,49 @@ +vcpkg_check_linkage(ONLY_STATIC_LIBRARY) + +vcpkg_from_github( + OUT_SOURCE_PATH SOURCE_PATH + REPO microsoft/Detours + REF 9764cebcb1a75940e68fa83d6730ffaf0f669401 + SHA512 30f689a7f7dd3d762f1194ad8d7e05517678b754d6c0db297220f946485a8c8ec8a07cf5f3f893aabcc5623f64c81ee358e2a1c3ba23ba1fbd5856f6b3dd9eb7 + HEAD_REF main +) + +# Upstream's portfile passes PROCESSOR_ARCHITECTURE, but Detours' system.mak +# only consults that macro as a last resort: if VSCMD_ARG_TGT_ARCH is set it +# takes priority (system.mak lines 15-17). The ARM64EC toolset is invoked from +# a plain ARM64 dev environment (VSCMD_ARG_TGT_ARCH=arm64) plus the /arm64EC +# compiler switch, so PROCESSOR_ARCHITECTURE=arm64ec never wins and Detours +# builds as plain ARM64, producing a LNK1392 arch-mismatch when linked into +# the rest of the arm64ec-windows triplet's objects. Passing +# DETOURS_TARGET_PROCESSOR directly is checked first and short-circuits that +# fallback chain. It must be pre-uppercased: nmake macros set on the command +# line take precedence over same-named macro assignments inside the makefile, +# so system.mak's own "uppercase DETOURS_TARGET_PROCESSOR" conversion chain +# (which works by plain `MACRO=$(MACRO:x=X)` reassignment) never runs on a +# value supplied this way, and the Makefile's comparisons are all uppercase. +string(TOUPPER "${VCPKG_TARGET_ARCHITECTURE}" DETOURS_TARGET_PROCESSOR) + +vcpkg_build_nmake( + SOURCE_PATH "${SOURCE_PATH}" + PROJECT_SUBPATH "src" + PROJECT_NAME "Makefile" + OPTIONS "DETOURS_TARGET_PROCESSOR=${DETOURS_TARGET_PROCESSOR}" + OPTIONS_RELEASE "DETOURS_CONFIG=Release" + OPTIONS_DEBUG "DETOURS_CONFIG=Debug" +) + +if(NOT DEFINED VCPKG_BUILD_TYPE OR VCPKG_BUILD_TYPE STREQUAL "release") + file(INSTALL "${CURRENT_BUILDTREES_DIR}/${TARGET_TRIPLET}-rel/lib.${VCPKG_TARGET_ARCHITECTURE}Release/" DESTINATION "${CURRENT_PACKAGES_DIR}/lib") +endif() +if(NOT DEFINED VCPKG_BUILD_TYPE OR VCPKG_BUILD_TYPE STREQUAL "debug") + file(INSTALL "${CURRENT_BUILDTREES_DIR}/${TARGET_TRIPLET}-dbg/lib.${VCPKG_TARGET_ARCHITECTURE}Debug/" DESTINATION "${CURRENT_PACKAGES_DIR}/debug/lib") +endif() + +if(NOT DEFINED VCPKG_BUILD_TYPE OR VCPKG_BUILD_TYPE STREQUAL "release") + file(INSTALL "${CURRENT_BUILDTREES_DIR}/${TARGET_TRIPLET}-rel/include/" DESTINATION "${CURRENT_PACKAGES_DIR}/include" RENAME detours) +else() + file(INSTALL "${CURRENT_BUILDTREES_DIR}/${TARGET_TRIPLET}-dbg/include/" DESTINATION "${CURRENT_PACKAGES_DIR}/include" RENAME detours) +endif() + +file(INSTALL "${SOURCE_PATH}/LICENSE.md" DESTINATION "${CURRENT_PACKAGES_DIR}/share/${PORT}" RENAME copyright) +file(INSTALL "${CMAKE_CURRENT_LIST_DIR}/usage" DESTINATION "${CURRENT_PACKAGES_DIR}/share/${PORT}") diff --git a/vcpkg-overlays/detours/usage b/vcpkg-overlays/detours/usage new file mode 100644 index 000000000..00f978fb6 --- /dev/null +++ b/vcpkg-overlays/detours/usage @@ -0,0 +1,7 @@ +detours can be used from CMake via: + + find_path(DETOURS_INCLUDE_DIRS "detours/detours.h") + find_library(DETOURS_LIBRARY detours REQUIRED) + + target_include_directories(main PRIVATE ${DETOURS_INCLUDE_DIRS}) + target_link_libraries(main PRIVATE ${DETOURS_LIBRARY}) diff --git a/vcpkg-overlays/detours/vcpkg.json b/vcpkg-overlays/detours/vcpkg.json new file mode 100644 index 000000000..d1affdb84 --- /dev/null +++ b/vcpkg-overlays/detours/vcpkg.json @@ -0,0 +1,8 @@ +{ + "name": "detours", + "version-date": "2025-06-20", + "description": "Detours is a software package for monitoring and instrumenting API calls on Windows.", + "homepage": "https://github.com/microsoft/Detours", + "license": "MIT", + "supports": "windows & !uwp" +} diff --git a/vcpkg-overlays/triplets/universal-osx.cmake b/vcpkg-overlays/triplets/universal-osx.cmake new file mode 100644 index 000000000..85271cad7 --- /dev/null +++ b/vcpkg-overlays/triplets/universal-osx.cmake @@ -0,0 +1,12 @@ +# Only used by the mac-univeral CI job, to get a fat (arm64 + x86_64) gtest +# that can link into that job's universal libvulkan.dylib/test binaries. +# vcpkg forwards VCPKG_OSX_ARCHITECTURES straight to CMake's +# CMAKE_OSX_ARCHITECTURES, so this only works for CMake-based ports (like +# gtest) that already support multi-arch builds; it would not work for a +# port built with autotools or a hand-rolled makefile. +set(VCPKG_TARGET_ARCHITECTURE arm64) +set(VCPKG_CRT_LINKAGE dynamic) +set(VCPKG_LIBRARY_LINKAGE static) + +set(VCPKG_CMAKE_SYSTEM_NAME Darwin) +set(VCPKG_OSX_ARCHITECTURES "arm64;x86_64") diff --git a/vcpkg-overlays/vulkan-headers/portfile.cmake b/vcpkg-overlays/vulkan-headers/portfile.cmake new file mode 100644 index 000000000..17988126d --- /dev/null +++ b/vcpkg-overlays/vulkan-headers/portfile.cmake @@ -0,0 +1,25 @@ +# Ideally in the future we should be using the vcpkg port of vulkan-headers instead of this overlay. +# This requires updating the vcpkg port of vulkan-headers first in the vcpkg repo, and then updating +# the builtin-baseline in vcpkg.json. +# +# IMHO this is less work moving forward than trying to maintain a separate overlay for vulkan-headers, +# and it will also make it easier to use the vcpkg port of vulkan-headers in other projects. +vcpkg_from_github( + OUT_SOURCE_PATH SOURCE_PATH + REPO KhronosGroup/Vulkan-Headers + REF "v${VERSION}" + SHA512 2c336c163cc184f1aad147f8829dedabdffe0a801e1636c1ae8445feca76a64fc71815d63a69a85db6f5c485ecb6ff3197d94aef63b06f7f9daac0260f7853a0 + HEAD_REF main +) + +set(VCPKG_BUILD_TYPE release) # header-only port + +vcpkg_cmake_configure(SOURCE_PATH "${SOURCE_PATH}" + OPTIONS + -DVULKAN_HEADERS_ENABLE_MODULE=OFF + -DVULKAN_HEADERS_ENABLE_TESTS=OFF +) +vcpkg_cmake_install() + +vcpkg_install_copyright(FILE_LIST "${SOURCE_PATH}/LICENSE.md") +file(INSTALL "${CMAKE_CURRENT_LIST_DIR}/usage" DESTINATION "${CURRENT_PACKAGES_DIR}/share/${PORT}") diff --git a/vcpkg-overlays/vulkan-headers/usage b/vcpkg-overlays/vulkan-headers/usage new file mode 100644 index 000000000..5b461a6d4 --- /dev/null +++ b/vcpkg-overlays/vulkan-headers/usage @@ -0,0 +1,4 @@ +Vulkan-Headers provides official find_package support: + + find_package(VulkanHeaders CONFIG) + target_link_libraries(main PRIVATE Vulkan::Headers) diff --git a/vcpkg-overlays/vulkan-headers/vcpkg.json b/vcpkg-overlays/vulkan-headers/vcpkg.json new file mode 100644 index 000000000..dab7be3b2 --- /dev/null +++ b/vcpkg-overlays/vulkan-headers/vcpkg.json @@ -0,0 +1,18 @@ +{ + "name": "vulkan-headers", + "version": "1.4.356", + "description": "Vulkan header files and API registry", + "homepage": "https://github.com/KhronosGroup/Vulkan-Headers", + "license": "Apache-2.0 OR MIT", + "supports": "!uwp & !xbox", + "dependencies": [ + { + "name": "vcpkg-cmake", + "host": true + }, + { + "name": "vcpkg-cmake-config", + "host": true + } + ] +} diff --git a/vcpkg.json b/vcpkg.json new file mode 100644 index 000000000..627904a8d --- /dev/null +++ b/vcpkg.json @@ -0,0 +1,20 @@ +{ + "name": "vulkan-loader", + "version": "1.4.356", + "builtin-baseline": "97a332dee1b4940a12cbc8feaa30570f41338950", + "dependencies": [ + "vulkan-headers" + ], + "features": { + "tests": { + "description": "Build and run the loader test suite", + "dependencies": [ + "gtest", + { + "name": "detours", + "platform": "windows" + } + ] + } + } +}