diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 413b5e9..bf33c31 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -192,6 +192,109 @@ jobs: - run: mix dialyzer + # Nothing in CI ever selected the CUDA backend, which is how a NIF that could + # not resolve `cuMemCreate` reached a release. Runners have no GPU, so this + # cannot execute a kernel; it proves the two things that were actually broken: + # the toolkit is found without nvcc on PATH, and the link line resolves every + # CUDA symbol ggml-cuda leaves undefined. + cuda-link: + name: CUDA link (cu${{ matrix.major }}) + runs-on: ubuntu-22.04 + needs: [setup] + strategy: + fail-fast: false + matrix: + include: + - toolkit: '12-9' + major: '12' + - toolkit: '13-0' + major: '13' + steps: + - uses: actions/checkout@v6 + with: + submodules: recursive + + # Before setup-beam, which installs into the tool cache this would remove. + - name: Free disk space for the CUDA toolkit + run: | + sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /usr/local/share/boost + df -h / + + - uses: erlef/setup-beam@v1 + with: + otp-version: ${{ env.OTP_VERSION }} + elixir-version: ${{ env.ELIXIR_VERSION }} + + - name: Install build dependencies + run: sudo apt-get update && sudo apt-get install -y cmake libgomp1 + + - name: Install the CUDA toolkit + run: | + curl -fsSLO https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64/cuda-keyring_1.1-1_all.deb + sudo dpkg -i cuda-keyring_1.1-1_all.deb + sudo apt-get update + sudo apt-get install -y --no-install-recommends cuda-toolkit-${{ matrix.toolkit }} + # Deliberately NOT added to PATH. A toolkit that is installed but not + # on PATH is the normal case on real machines (DGX OS ships nvcc via + # /etc/profile.d, which no non-login shell sources) and it is the case + # that used to produce a silent CPU-only build. + ls -d /usr/local/cuda-* + + - run: mix deps.get + + - name: Build the NIF against CUDA + run: mix compile + env: + LLAMA_BACKEND: cuda + LLAMA_PORTABLE: '1' + # One architecture. This job is about finding the toolkit and + # resolving the link; compiling the full release architecture list + # would add an hour and prove nothing more. + LLAMA_CMAKE_ARGS: -DCMAKE_CUDA_ARCHITECTURES=86-real + + - name: The NIF must declare and resolve its CUDA dependencies + run: | + set -eu + # -L is required: mix makes _build//lib//priv a symlink to + # the project's priv/, and find does not descend into symlinked + # directories, so a plain find reports nothing here. + so=$(find -L _build -name llama_cpp_ex_nif.so | head -1) + test -n "$so" || { echo "::error::no NIF was built"; exit 1; } + readelf -d "$so" | grep NEEDED + + for lib in libcudart.so.${{ matrix.major }} libcublas.so.${{ matrix.major }} libcuda.so.1; do + readelf -d "$so" | grep -q "\[$lib\]" \ + || { echo "::error::$so does not link $lib"; exit 1; } + done + + # Full symbol resolution, which is the check that catches this class of + # bug. `ldd -r` needs libcuda.so.1 and a runner has no driver to supply + # it -- but the toolkit stub exports exactly the driver API under that + # same soname, so pointing the loader at it resolves the real symbols + # rather than faking them. + # Globs, not `find`: `lib64` is a symlink to targets//lib and + # find will not descend through it, which is why a `-path '*/lib64/*'` + # search comes back empty on a perfectly good toolkit. + stub=$(ls -1 /usr/local/cuda*/lib64/stubs/libcuda.so \ + /usr/local/cuda*/targets/*/lib/stubs/libcuda.so \ + 2>/dev/null | head -1) + test -n "$stub" || { echo "::error::no libcuda stub in the toolkit"; exit 1; } + echo "driver stub: $stub" + mkdir -p /tmp/driver + ln -sf "$stub" /tmp/driver/libcuda.so.1 + LD_LIBRARY_PATH=/tmp/driver ldd -r "$so" > /tmp/ldd.txt 2>&1 || true + cat /tmp/ldd.txt + + # enif_* is the NIF API. The BEAM exports it from the running emulator + # and resolves it when it dlopens the library, so it is undefined here + # by design and is the one thing this gate must ignore. Everything else + # unresolved is a library the link line forgot -- how `cuMemCreate` and + # then `ncclAllReduce` each shipped a NIF that could not load. + if grep -iE 'undefined symbol|not found' /tmp/ldd.txt | grep -v 'undefined symbol: enif_'; then + echo "::error::the NIF has unresolved non-BEAM symbols; see above" + exit 1 + fi + inference: name: Inference smoke runs-on: ubuntu-22.04 diff --git a/.github/workflows/precompile.yml b/.github/workflows/precompile.yml index cdec0d9..54f8cb7 100644 --- a/.github/workflows/precompile.yml +++ b/.github/workflows/precompile.yml @@ -47,11 +47,18 @@ jobs: fail-fast: false matrix: include: + # `toolkit` and `variant` are spelled out on every leg, empty where + # there is no CUDA. An absent matrix key is null, and `null != ''` in a + # GitHub expression resolves through numeric coercion rather than + # string comparison; being explicit keeps the `if:` guards below from + # depending on that. - os: macos-14 target: aarch64-apple-darwin otp: '27.0' elixir: '1.18' backend: metal + toolkit: '' + variant: '' # OTP 25 reports NIF 2.16, OTP 26/27/28 all report 2.17, and OTP 29 is # the first release to report 2.18 (verified against # erts/emulator/beam/erl_nif.h in the OTP source). So OTP 27 and OTP 29 @@ -62,16 +69,55 @@ jobs: otp: '29.0' elixir: '1.20' backend: metal + toolkit: '' + variant: '' - os: ubuntu-22.04 target: x86_64-linux-gnu otp: '27.0' elixir: '1.18' backend: cpu + toolkit: '' + variant: '' - os: ubuntu-22.04 target: x86_64-linux-gnu otp: '29.0' elixir: '1.20' backend: cpu + toolkit: '' + variant: '' + # CUDA artifacts are per major version because the NIF links + # libcudart/libcublas/libcublasLt dynamically and those sonames are + # major-versioned: one Linux CUDA build cannot serve both. 22.04 is + # kept for the same reason as the CPU legs -- it is the oldest glibc + # these artifacts have to load against. + - os: ubuntu-22.04 + target: x86_64-linux-gnu-cu12 + otp: '27.0' + elixir: '1.18' + backend: cuda + toolkit: '12-9' + variant: cu12 + - os: ubuntu-22.04 + target: x86_64-linux-gnu-cu12 + otp: '29.0' + elixir: '1.20' + backend: cuda + toolkit: '12-9' + variant: cu12 + - os: ubuntu-22.04 + target: x86_64-linux-gnu-cu13 + otp: '27.0' + elixir: '1.18' + backend: cuda + toolkit: '13-0' + variant: cu13 + - os: ubuntu-22.04 + target: x86_64-linux-gnu-cu13 + otp: '29.0' + elixir: '1.20' + backend: cuda + toolkit: '13-0' + variant: cu13 runs-on: ${{ matrix.os }} permissions: @@ -80,6 +126,12 @@ jobs: - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 with: submodules: recursive + # Before setup-beam, which installs into the tool cache this would remove. + - name: Free disk space for the CUDA toolkit + if: matrix.toolkit != '' + run: | + sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /usr/local/share/boost + df -h / - uses: erlef/setup-beam@54075bcc5e249e4758d363f27d099f55d843f124 # v1 with: otp-version: ${{ matrix.otp }} @@ -87,6 +139,17 @@ jobs: - name: Install cmake (Linux) if: runner.os == 'Linux' run: sudo apt-get update && sudo apt-get install -y cmake + - name: Install the CUDA toolkit + if: matrix.toolkit != '' + run: | + curl -fsSLO https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64/cuda-keyring_1.1-1_all.deb + sudo dpkg -i cuda-keyring_1.1-1_all.deb + sudo apt-get update + sudo apt-get install -y --no-install-recommends cuda-toolkit-${{ matrix.toolkit }} + # Not added to PATH on purpose: the Makefile resolves the toolkit + # itself, and leaving nvcc off PATH keeps this leg honest about the + # discovery path most real machines take. + ls -d /usr/local/cuda-* - name: Set version from tag run: | TAG_VERSION="${GITHUB_REF#refs/tags/v}" @@ -125,6 +188,24 @@ jobs: # artifact to whatever CPU the runner happened to have and hand users a # SIGILL on older hardware. LLAMA_PORTABLE: '1' + # Names the artifact. Detection cannot be trusted here: it requires a + # driver before it will claim a CUDA target, and a release runner has + # a toolkit and no driver, so an unset variant would silently publish + # a CUDA build under the CPU name. Empty on the CPU legs, which pins + # them to the CPU name even on a runner that happens to have CUDA. + LLAMA_CUDA_VARIANT: ${{ matrix.variant }} + - name: The artifact must be named for the target this leg builds + run: | + set -eu + # A mismatch here means the matrix and the precompiler disagree about + # what this leg produced. Left unchecked it surfaces as the `checksum` + # job failing on a missing artifact, long after the build that could + # have explained it. + ls cache/ + # Anchored on the version, so it cannot pass on a neighbouring name: + # `*-x86_64-linux-gnu-*` alone would happily match the cu12 artifact. + ls cache/*-${{ matrix.target }}-"${TAG_VERSION}".tar.gz >/dev/null \ + || { echo "::error::this leg produced no artifact named for ${{ matrix.target }}"; exit 1; } - name: Upload artifacts to the draft release env: GH_TOKEN: ${{ github.token }} diff --git a/CHANGELOG.md b/CHANGELOG.md index 7ed0aa8..b68210f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,102 @@ # Changelog +## Unreleased + +CUDA is a first-class target: the NIF now links correctly against it, and the +release publishes prebuilt CUDA artifacts for CUDA 12 and CUDA 13. + +Verified on 2x NVIDIA DGX Spark (GB10, `sm_121a`, aarch64, CUDA 13.0.2) against +this base (llama.cpp b10280) — a source build with `LLAMA_BACKEND=cuda` loads, +reports `backend: "CUDA"` from `LlamaCppEx.devices()`, offloads 31/31 layers to +the GPU, and passes the smoke suite: **528 tests, 0 failures**. + +### Fixed + +- **MTP hybrid rollback corrupted the KV cache after a partial accept.** This is + a different bug from the `load_mtp` one fixed in v0.8.42, and the two are + complementary: that one stopped the MTP layers being read off disk at all, + this one silently misplaces context once they are working. + + The verification batch is `[sampled, drafts...]` at positions starting at + `n_past`, so `sampled` occupies batch element 0. When only some drafts are + accepted the target's KV is rolled back to `n_past` and re-decoded — but the + re-decode started at the first *accepted* token rather than at `sampled`. + Every token therefore landed one position early, and the last accepted token + was written into the context even though it becomes the next iteration's + `sampled` and is decoded again there, duplicating it. Reading the slice from + `prompt[size - n_accepted_total - 1]` restores `sampled, accepted[0..k-2]` + across `[n_past, n_past + k)`, which is exactly the span `n_past` then + advances over. + + Only reachable on a partial accept, which is why a working MTP setup can still + post plausible acceptance rates while quietly drifting. +- **The CUDA NIF could not be loaded** — `ggml-cuda.a` leaves the CUDA runtime, + cuBLAS/cuBLASLt and the CUDA driver API unresolved, but the Linux link line + only ever added `-lstdc++ -lm -lpthread`. The resulting `.so` linked and then + died at load with `undefined symbol: cuMemCreate`, a driver-API symbol + ggml-cuda's VMM pool calls. +- **The toolkit was only ever looked for on `PATH`** — and that is the one place + it frequently is not. DGX OS installs `nvcc` via `/etc/profile.d/nv_paths.sh`, + which `ssh host mix compile`, systemd units and most CI shells never source; + environment modules behave the same way. The two consequences were both + silent: `LLAMA_BACKEND=auto` produced a CPU-only build on a machine with a + complete toolkit, and `LLAMA_BACKEND=cuda` produced a link line with no `-L` + at all. Discovery now tries `CUDA_HOME`, `CUDA_PATH`, `nvcc` on `PATH`, + `/usr/local/cuda`, `/opt/cuda`, then the newest `/usr/local/cuda-*`, is shared + between backend auto-detection and the link line so they cannot disagree, and + passes `-DCMAKE_CUDA_COMPILER` so cmake's own `find_package(CUDAToolkit)` + does not repeat the mistake. Selecting CUDA with no toolkit present is now an + error naming `CUDA_HOME`, not `cannot find -lcudart`. +- **`undefined symbol: ncclAllReduce`** — ggml's `GGML_CUDA_NCCL` defaults to ON + and links libnccl through cmake whenever the build host happens to have NCCL, + which every DGX and most multi-GPU boxes do. The Makefile assembles its link + line by hand from ggml's static archives, so cmake's `target_link_libraries` + is invisible to it and the symbols went unresolved. The flag is now always + stated explicitly rather than inherited, defaulting to OFF; `LLAMA_CUDA_NCCL=1` + turns it on and adds the matching `-lnccl`. +- **Library path assumed `lib64`** — Debian's packaged `nvidia-cuda-toolkit` + only has `lib`, and the stubs directory is now probed rather than assumed. + +### Added + +- **`LlamaCppEx.MTP.init/2` now says when a checkpoint has no MTP head**, via a + new `LlamaCppEx.NIF.model_n_layer_nextn/1` wrapping upstream's + `llama_model_n_layer_nextn`. Previously this surfaced as + `{:error, "failed to create context"}`, with the real reason — llama.cpp's + `context type MTP requested but model doesn't contain MTP layers` — buried in + engine output the caller may not be showing. + + This is not the `load_mtp` case and no flag recovers it: most GGUF conversions + of an MTP-capable model simply drop the head. Unsloth's + `Qwen3.6-35B-A3B-UD-Q4_K_XL` reports zero nextn layers; their separate + `Qwen3.6-35B-A3B-MTP-GGUF` build of the same model carries them. The message + now says so and points at the `-MTP` build. + +- **Precompiled CUDA artifacts** — `x86_64-linux-gnu-cu12` and + `x86_64-linux-gnu-cu13` join the existing `aarch64-apple-darwin` (Metal) and + `x86_64-linux-gnu` (CPU) targets, at NIF 2.17 and 2.18. `mix compile` on an + x86_64 Linux box with a driver and a CUDA runtime now downloads a GPU build + instead of silently installing a CPU one. + + They are separate artifacts because the NIF links `libcudart`/`libcublas`/ + `libcublasLt` dynamically and those sonames are major-versioned; one "Linux + CUDA" binary cannot serve both. Selection requires **both** a CUDA runtime and + a driver (`libcuda.so.1`) — a CUDA build links `-lcuda` and cannot be + `dlopen`ed at all without one, so a toolkit-only machine keeps the CPU + artifact rather than getting a NIF that fails to load. `LLAMA_CUDA_VARIANT` + (`cu12`, `cu13`, `none`) overrides the probe. +- **`LLAMA_CUDA_NCCL`** build variable, off by default. See above. +- **A CUDA link gate in CI** — nothing in CI had ever selected the CUDA backend, + which is how a NIF that could not resolve `cuMemCreate` reached a release. The + new `cuda-link` job builds against both CUDA 12 and CUDA 13 on every pull + request with the toolkit deliberately off `PATH`, asserts the `.so` declares + `libcudart`, `libcublas` and `libcuda.so.1`, and then resolves every symbol + with `ldd -r` — pointing the loader at the toolkit's driver stub, which + carries the `libcuda.so.1` soname, so a GPU-less runner can still perform a + real resolution. `enif_*` is excluded, being supplied by the BEAM at load. +- **Precompiler unit tests** — `test/precompiler_test.exs` pins the artifact + selection rules, including the case that motivates the driver check. + ## v0.8.42 llama.cpp bump to b10280, on top of b10217 from v0.8.41. Unlike the last two diff --git a/Makefile b/Makefile index aa4ae3b..e6a7cc1 100644 --- a/Makefile +++ b/Makefile @@ -54,6 +54,52 @@ LDFLAGS = -shared # Platform detection UNAME_S := $(shell uname -s) +# --- CUDA toolkit discovery -------------------------------------------------- +# Resolved once, up here, because two separate decisions depend on it: `auto` +# uses it to decide whether CUDA is available at all, and the Linux link line +# below needs the library directory. +# +# Probing only `which nvcc` was wrong in both places. nvcc is routinely absent +# from a *non-login* PATH -- DGX OS installs it via /etc/profile.d/nv_paths.sh, +# which `ssh host make`, systemd units and most CI shells never source -- so on +# a machine with a complete toolkit `auto` silently produced a CPU-only build, +# and an explicit LLAMA_BACKEND=cuda produced no -L flags and failed the link on +# libcudart. Honour CUDA_HOME and CUDA_PATH first (both are conventional and +# either may be exported by a module system), then nvcc on PATH, then the +# standard install locations. +ifeq ($(strip $(CUDA_HOME)),) + CUDA_HOME := $(strip $(CUDA_PATH)) +endif +ifeq ($(strip $(CUDA_HOME)),) + CUDA_HOME := $(patsubst %/bin/nvcc,%,$(shell command -v nvcc 2>/dev/null)) +endif +ifeq ($(strip $(CUDA_HOME)),) + CUDA_HOME := $(firstword $(wildcard /usr/local/cuda /opt/cuda) \ + $(shell ls -d /usr/local/cuda-* 2>/dev/null | sort -V | tail -1)) +endif + +# Presence of the compiler, not of the directory: /usr/local/cuda survives a +# partial uninstall, and a runtime-only install has libraries but cannot build. +NVCC := $(wildcard $(CUDA_HOME)/bin/nvcc) + +# x86_64 and sbsa toolkits both expose lib64 (a symlink to targets//lib +# on sbsa); Debian's packaged nvidia-cuda-toolkit only has lib. +CUDA_LIBDIR := $(firstword $(wildcard $(CUDA_HOME)/lib64 $(CUDA_HOME)/lib)) + +# ggml's own GGML_CUDA_NCCL defaults to ON and quietly links libnccl through +# cmake whenever NCCL happens to be installed on the build host. cmake's +# target_link_libraries is invisible to this file -- the link line below is +# assembled by hand from the static archives ggml leaves behind -- so on a host +# with NCCL (every DGX, most multi-GPU boxes) ggml-cuda.a came out carrying +# undefined nccl* symbols and the resulting NIF failed to load with +# `undefined symbol: ncclAllReduce`. +# +# So whether NCCL is used is declared here rather than discovered, and it is off +# by default. It only accelerates collectives across multiple GPUs, while +# linking it makes libnccl.so.2 a hard load-time requirement of the artifact -- +# the same trade as -march=native, and the same answer. +LLAMA_CUDA_NCCL ?= 0 + # Backend selection (auto, metal, cuda, vulkan, cpu) LLAMA_BACKEND ?= auto @@ -73,7 +119,7 @@ ifeq ($(LLAMA_BACKEND),auto) ifeq ($(UNAME_S),Darwin) CMAKE_FLAGS += -DGGML_METAL=ON -DGGML_METAL_EMBED_LIBRARY=ON else - ifneq ($(shell which nvcc 2>/dev/null),) + ifneq ($(NVCC),) CMAKE_FLAGS += -DGGML_CUDA=ON endif endif @@ -87,6 +133,22 @@ else ifeq ($(LLAMA_BACKEND),cpu) CMAKE_FLAGS += -DGGML_METAL=OFF -DGGML_CUDA=OFF -DGGML_VULKAN=OFF endif +# cmake locates the toolkit by searching PATH for nvcc, so it has the same blind +# spot the discovery block above exists to cover: without this, a build that +# correctly selected CUDA still fails in `find_package(CUDAToolkit)`. +ifneq (,$(filter -DGGML_CUDA=ON,$(CMAKE_FLAGS))) + ifneq ($(NVCC),) + CMAKE_FLAGS += -DCMAKE_CUDA_COMPILER=$(NVCC) + endif + # Always stated, never left to ggml's default, so the archives cmake produces + # and the link line assembled below cannot disagree about NCCL. + ifneq ($(filter 1 true yes,$(LLAMA_CUDA_NCCL)),) + CMAKE_FLAGS += -DGGML_CUDA_NCCL=ON + else + CMAKE_FLAGS += -DGGML_CUDA_NCCL=OFF + endif +endif + # Portable builds, for artifacts that leave this machine. ggml defaults # GGML_NATIVE to ON unless cross-compiling (vendor/llama.cpp/ggml/CMakeLists.txt), # which adds -march=native (ggml/src/ggml-cpu/CMakeLists.txt) and ties the binary @@ -125,6 +187,31 @@ else ifneq ($(shell $(CXX) -fopenmp -E - < /dev/null 2>/dev/null && echo yes),) LDFLAGS += -lgomp endif + # ggml-cuda.a leaves the CUDA runtime, cuBLAS/cuBLASLt and the CUDA driver API + # unresolved, but this line only ever added -lstdc++ -lm -lpthread. The .so + # then linked and failed at load with `undefined symbol: cuMemCreate`, a + # driver-API symbol ggml-cuda's VMM pool calls. + ifneq (,$(filter -DGGML_CUDA=ON,$(CMAKE_FLAGS))) + ifeq ($(strip $(CUDA_LIBDIR)),) + $(error CUDA backend selected but no CUDA toolkit libraries were found. \ + Set CUDA_HOME to the toolkit root, the directory holding bin/nvcc.) + endif + LDFLAGS += -L$(CUDA_LIBDIR) + # -lcuda is the driver API, which is shipped by the driver and not by the + # toolkit, so it is missing on every GPU-less build host including the + # release runners. The stub carries SONAME libcuda.so.1, so linking against + # it resolves the symbols at build time and still loads the real driver + # library at run time. + ifneq ($(wildcard $(CUDA_LIBDIR)/stubs),) + LDFLAGS += -L$(CUDA_LIBDIR)/stubs + endif + LDFLAGS += -lcudart -lcublas -lcublasLt -lcuda + # Matches the -DGGML_CUDA_NCCL above. Opting in without this pairing is the + # `undefined symbol: ncclAllReduce` load failure. + ifneq ($(filter 1 true yes,$(LLAMA_CUDA_NCCL)),) + LDFLAGS += -lnccl + endif + endif endif # CPU count for parallel builds diff --git a/README.md b/README.md index e3f00cc..2a43eec 100644 --- a/README.md +++ b/README.md @@ -47,14 +47,33 @@ end Elixir `~> 1.18`, enforced by `mix.exs`. The Erlang/OTP floor depends on how the NIF is obtained: -Precompiled NIFs are published for `aarch64-apple-darwin` (Metal) and -`x86_64-linux-gnu` (CPU) at NIF versions 2.17 and 2.18 — that is **Erlang/OTP 26 -or newer** (OTP 26, 27 and 28 report NIF 2.17; OTP 29 reports 2.18). On those -platforms `mix deps.get` and `mix compile` download a binary and none of the -build tooling below is needed. +Precompiled NIFs are published at NIF versions 2.17 and 2.18 — that is +**Erlang/OTP 26 or newer** (OTP 26, 27 and 28 report NIF 2.17; OTP 29 reports +2.18) — for: -Everything else builds from source: OTP 25 (NIF 2.16), other architectures, -musl, Windows, and every GPU backend except Metal. A source build needs +| Target | Backend | Selected when | +|---|---|---| +| `aarch64-apple-darwin` | Metal | Apple Silicon | +| `x86_64-linux-gnu` | CPU | no usable CUDA install found | +| `x86_64-linux-gnu-cu12` | CUDA | driver plus a CUDA 12 runtime | +| `x86_64-linux-gnu-cu13` | CUDA | driver plus a CUDA 13 runtime | + +On those platforms `mix deps.get` and `mix compile` download a binary and none +of the build tooling below is needed. + +The CUDA variants are separate artifacts rather than one because the NIF links +`libcudart`/`libcublas`/`libcublasLt` dynamically and those sonames are +major-versioned: a cu13 build cannot load against a CUDA 12 install. Selection +is automatic and deliberately conservative — it requires **both** a CUDA runtime +and a driver (`libcuda.so.1`), because a CUDA artifact on a machine with no +driver cannot be loaded at all, and falls back to the CPU artifact otherwise. +Override with `LLAMA_CUDA_VARIANT=cu12|cu13|none` if the probe gets it wrong. + +No CUDA toolkit is needed to *run* these; only the runtime libraries they link. + +Everything else builds from source: OTP 25 (NIF 2.16), other architectures +(including aarch64 Linux — DGX Spark and friends), musl, Windows, Vulkan, and +any CUDA major version without a published artifact. A source build needs - a C++17 compiler (GCC, Clang, or MSVC), - CMake 3.14+, @@ -63,10 +82,10 @@ musl, Windows, and every GPU backend except Metal. A source build needs ### Backend Selection -What the published artifacts actually contain is Metal on Apple Silicon and -plain CPU on Linux. **There is no CUDA or Vulkan artifact**, and a downloaded -artifact never runs the Makefile, so nothing is auto-detected at install time. -GPU acceleration beyond Metal always means an explicit source build: +A downloaded artifact never runs the Makefile, so nothing is auto-detected at +install time — the backend is whatever that artifact was built with. Vulkan, and +CUDA on any platform without a published variant, always mean an explicit source +build: ```bash mix compile # Precompiled artifact when one matches this @@ -78,9 +97,14 @@ LLAMA_BACKEND=cpu mix compile # CPU only ``` Setting `LLAMA_BACKEND` to anything forces a source build and bypasses the -precompiled artifact — that is how a CUDA or Vulkan build is obtained. When a -source build runs with `LLAMA_BACKEND` unset it picks Metal on macOS, CUDA if -`nvcc` is on `PATH`, and CPU otherwise. +precompiled artifact. When a source build runs with `LLAMA_BACKEND` unset it +picks Metal on macOS, CUDA if a toolkit is found, and CPU otherwise. + +CUDA is located by `CUDA_HOME`, then `CUDA_PATH`, then `nvcc` on `PATH`, then +`/usr/local/cuda`, `/opt/cuda` and the versioned `/usr/local/cuda-*` directories. +`PATH` alone is not enough to rely on: DGX OS, environment modules and most CI +shells leave `nvcc` off a non-login `PATH`, which used to mean a silent CPU-only +build on a machine with a perfectly good toolkit. Power users can pass arbitrary CMake flags: @@ -88,12 +112,18 @@ Power users can pass arbitrary CMake flags: LLAMA_CMAKE_ARGS="-DGGML_CUDA_FORCE_CUBLAS=ON" mix compile ``` -Two more build variables: +Three more build variables: - `LLAMA_PORTABLE=1` drops `-march=native`. ggml turns it on by default, which tunes the binary to the exact CPU it was built on; the release workflow sets this so published artifacts run on every machine of that architecture. Leave it unset locally, where the native flags are free performance. +- `LLAMA_CUDA_NCCL=1` builds and links ggml's NCCL collectives, which speed up + multi-GPU work. Off by default: ggml would otherwise enable NCCL silently + whenever the build host happens to have it, and since the Makefile assembles + the link line by hand rather than through cmake, that produced a NIF that + failed to load with `undefined symbol: ncclAllReduce`. Turning it on also + makes `libnccl.so.2` a load-time requirement. - `LLAMA_COMMIT=` overrides the pinned llama.cpp commit used when `vendor/llama.cpp` has to be cloned. @@ -525,7 +555,41 @@ Multi-Token Prediction speculative decoding (upstream PR [#22673](https://github > - Qwen 3.6 35B-A3B-MTP (hybrid MoE): plain 39.5 → MTP **44.0 tok/s (1.11×)** > - Qwen 3.6 27B (dense): plain 10.7 → MTP **10.6 tok/s (~1.0×, neutral)** > -> Larger `n_draft` hurts on Metal because verify cost grows faster than acceptance benefit. On NVIDIA, `n_draft: 3` is the right default — that's what the upstream 2× number assumes. +> Larger `n_draft` hurts on Metal because verify cost grows faster than acceptance benefit. + +> **Performance note: NVIDIA GB10 (DGX Spark).** MTP does pay here, but nothing +> like the upstream 2×, and the best `n_draft` is not 3. Qwen3.6-35B-A3B +> UD-Q4_K_XL from the `-MTP` build, 128-token greedy generations, plain and MTP +> interleaved in one process so drift hits both arms equally (n=11 each): +> +> | | median tok/s | range | vs plain | +> |---|---|---|---| +> | plain | 61.4 | 61.2–62.4 | — | +> | MTP `n_draft: 2` | **71.2** | 62.5–75.5 | **+16%** | +> +> The ranges do not overlap — MTP's slowest run beat plain's fastest — so the +> gain is real despite MTP being the noisier arm by an order of magnitude. +> +> Sweeping `n_draft` on the same model, though, puts the optimum at 2 rather +> than 3, and the engine's own counters say why: +> +> | `n_draft` | acceptance | tokens/iteration | tok/s | +> |---|---|---|---| +> | 2 | 68.5% | 2.38 | 63.0 | +> | 3 | 57.2% | 2.73 | 52.8 | +> +> Going from 2 to 3 buys 15% more tokens per iteration and pays 31% more drafting +> plus 10% more verify for them, because the third draft position is the one +> least likely to be accepted. Marginal acceptance decays faster than marginal +> cost, so the extra draft loses money. In a five-run sweep `n_draft: 3` came out +> 2% *below* plain and `n_draft: 4` 14% below. +> +> So the shape matches Apple Silicon even though the cause differs: on Metal the +> wide verify is expensive, while GB10 is a unified-memory part whose MoE decode +> is memory-bandwidth bound, and a wider verify reads more expert weights per +> step. Both end up wanting a narrower draft than a datacenter GPU does. Treat +> `n_draft: 3` as the datacenter default the upstream 2× assumes, not as a value +> that transfers. ### Other speculative types (EAGLE-3, DFlash, n-gram) @@ -663,7 +727,10 @@ end) `LlamaCppEx.MTP.init/2`: - * `:n_draft` — draft tokens proposed per iteration (default `3`). On NVIDIA, 2–4 is the sweet spot. On Apple Silicon, set this to `1` — see the Apple Silicon performance note above. + * `:n_draft` — draft tokens proposed per iteration (default `3`). The optimum + is hardware-specific and worth measuring rather than assuming: `1` on Apple + Silicon, `2` on GB10 (where `3` measured *slower* than no speculation at + all), `2–4` on datacenter NVIDIA. See the two performance notes above. * `:n_ctx`, `:n_threads`, `:flash_attn`, `:type_k`/`:type_v`, `:offload_kqv`, … — any `LlamaCppEx.Context` option; applied to both target and draft contexts. `LlamaCppEx.MTP.stream/3`: @@ -680,10 +747,14 @@ See [`examples/mtp_speculative.exs`](examples/mtp_speculative.exs) for a runnabl ## Benchmarks -Measured on Apple M4 Max (64 GB), Metal backend (`n_gpu_layers: -1`). +Each subsection names its own hardware and backend — the numbers below span +Apple Silicon (Metal) and NVIDIA (CUDA) and are not comparable across sections +unless they say so. Unless noted otherwise, `n_gpu_layers: -1`. ### Single-model generation speed +Apple M4 Max (64 GB), Metal backend. + | Model | Quantization | Tokens/sec | |-------|-------------|------------| | Llama 3.2 3B Instruct | Q4_K_XL | 125.6 | @@ -703,6 +774,48 @@ New `qwen35moe` architecture with Gated Delta Net (hybrid linear/full attention) 128-token generation, `temp: 0.0`, 3-run average (43.3 / 44.1 / 44.0 t/s). +### CUDA: NVIDIA DGX Spark (GB10) + +Same model and quantization as the M1 Max row above, so the two are directly +comparable. GB10 (`sm_121a`, aarch64, 128 GB unified), CUDA 13.0.2, driver +580.173.02, llama.cpp b10280, source build with `LLAMA_BACKEND=cuda`. + +| Model | Quantization | Tokens/sec (GB10) | Tokens/sec (M1 Max) | +|-------|-------------|-------------------|---------------------| +| Qwen3.6-35B-A3B | UD-Q4_K_XL | **62.1** | 43.8 | + +128-token generation, `temp: 0.0`, `n_gpu_layers: -1`. Median of 5 runs after a +discarded warm-up: 61.7 / 62.0 / 62.1 / 62.2 / 62.2 t/s — a 0.9% spread, so the +1.42x over M1 Max is well outside the noise. All 41 layers offload; the model +takes 20 799 MiB of device memory. + +Two notes on method, both learned the hard way: + +- **Each run uses a distinct prompt.** Repeating one prompt hits the context + reuse path and reports a throughput the engine never achieved. +- **Tokens are counted by re-encoding the output**, not by counting stream + chunks — a chunk is not a token, and under speculative decoding it can carry + several. + +The first call after load is discarded: it pays CUDA graph capture and the +allocator's first-touch layout, and is not representative of steady state. + +With MTP speculative decoding, using the separate `-MTP` build of the same model +(the plain UD-Q4_K_XL carries no MTP head — `MTP.init/2` now says so rather than +failing with a bare context error): + +| | median tok/s | vs plain | +|---|---|---| +| plain | 61.4 | — | +| `n_draft: 2` | **71.2** | +16% | +| `n_draft: 3` | 61.1 | −2% | +| `n_draft: 4` | 53.8 | −14% | + +`n_draft: 2` is the optimum on this hardware, not the documented default of 3. +See the GB10 performance note under [Speculative decoding +(MTP)](#speculative-decoding-mtp) for the acceptance and timing counters behind +that. + ### Single-sequence generation (Qwen3-4B Q4_K_M) | Prompt | 32 tokens | 128 tokens | diff --git a/c_src/llama_cpp_ex/llama_nif.cpp b/c_src/llama_cpp_ex/llama_nif.cpp index ac60bfe..766b10c 100644 --- a/c_src/llama_cpp_ex/llama_nif.cpp +++ b/c_src/llama_cpp_ex/llama_nif.cpp @@ -379,6 +379,17 @@ int64_t model_n_embd(ErlNifEnv* env, fine::ResourcePtr model) { } FINE_NIF(model_n_embd, 0); +// Number of MTP / "next-N" prediction layers the checkpoint carries. Zero means +// the GGUF has no MTP head at all, which is a different situation from a model +// loaded with load_mtp: false: no flag can recover it, only a different file. +// Without this, asking for an MTP context on such a model surfaces as a bare +// "failed to create context" while the real reason is one line above it in +// llama.cpp's own log. +int64_t model_n_layer_nextn(ErlNifEnv* env, fine::ResourcePtr model) { + return llama_model_n_layer_nextn(model->model); +} +FINE_NIF(model_n_layer_nextn, 0); + std::string model_desc(ErlNifEnv* env, fine::ResourcePtr model) { char buf[256]; llama_model_desc(model->model, buf, sizeof(buf)); @@ -2178,12 +2189,20 @@ fine::Ok<> generate_mtp_tokens( // Re-decode the accepted tokens on the target so the next // iteration's draft starts from a consistent state. + // + // The KV must be rebuilt with the token that was `sampled` at + // the top of this iteration (batch element 0, at pos n_past), + // followed by all but the LAST emitted token. The last emitted + // token becomes the next iteration's `sampled` and is decoded + // then — including it here would duplicate it in the context. + // `sampled` sits at prompt[size - n_accepted_total - 1], since + // the accept loop pushed n_accepted_total tokens after it. if (n_accepted_total > 0) { llama_batch redo = llama_batch_init(n_accepted_total, 0, 1); BatchFreeGuard redo_guard(redo); for (int i = 0; i < n_accepted_total; i++) { llama_token tok = - prompt[prompt.size() - n_accepted_total + i]; + prompt[prompt.size() - n_accepted_total - 1 + i]; common_batch_add(redo, tok, n_past + static_cast(i), { seq_id }, diff --git a/docs/cross-platform-builds.md b/docs/cross-platform-builds.md index 854c4cd..b2327a9 100644 --- a/docs/cross-platform-builds.md +++ b/docs/cross-platform-builds.md @@ -11,12 +11,24 @@ with. |---|---|---|---| | macOS (Apple Silicon) | Yes — Metal | Metal | Tested | | macOS (Intel) | No | CPU | Supported, source build | -| Linux (x86_64, glibc) | Yes — **CPU only** | CUDA if `nvcc` found, else CPU | Supported | -| Linux (x86_64) + NVIDIA | No | CUDA if `nvcc` found | Supported via `LLAMA_BACKEND=cuda` | +| Linux (x86_64, glibc) | Yes — CPU | CUDA if a toolkit is found, else CPU | Supported | +| Linux (x86_64) + NVIDIA, CUDA 12 | Yes — **CUDA** (`-cu12`) | CUDA | Supported | +| Linux (x86_64) + NVIDIA, CUDA 13 | Yes — **CUDA** (`-cu13`) | CUDA | Supported | | Linux (x86_64) + AMD | No | CPU | Supported via `LLAMA_BACKEND=vulkan` | +| Linux (aarch64) + NVIDIA | No | CUDA if a toolkit is found | Tested on DGX Spark (GB10), source build | | Linux (aarch64), musl | No | as above | Supported, source build | | Windows (WSL2) | No | Same as Linux | Supported, source build | +### Why CUDA is split by major version + +The NIF links `libcudart`, `libcublas` and `libcublasLt` dynamically, and those +sonames are major-versioned — `libcudart.so.12` and `libcudart.so.13` are +different files with no compatibility shim in either direction. One "Linux CUDA" +artifact therefore cannot exist; each CUDA major gets its own target name. + +Running one needs the CUDA **runtime** libraries, not the toolkit: no `nvcc` +required. + Artifacts are published for NIF 2.17 and 2.18, which means Erlang/OTP 26 or newer. Anything else — including OTP 25, which reports NIF 2.16 — falls back to a source build. The NIF-version-to-OTP mapping is verified against @@ -32,19 +44,44 @@ entry is declared even though OTP 25 works. mix compile ``` -If a precompiled artifact matches this OS, architecture and NIF version, -`mix compile` downloads it and stops. **The Makefile does not run, so nothing is -auto-detected**, and the artifact's backend is whatever it was built with: Metal -on Apple Silicon, CPU on `x86_64-linux-gnu`. A Linux user with a working CUDA -toolkit still gets a CPU-only binary from this path. +If a precompiled artifact matches this OS, architecture, NIF version and CUDA +variant, `mix compile` downloads it and stops. **The Makefile does not run, so +nothing is auto-detected** — the artifact's backend is whatever it was built +with. + +On `x86_64-linux-gnu` the choice between the CPU artifact and a CUDA one is made +by `LlamaCppEx.Precompiler` in `mix.exs`, which looks for two things and needs +both: + +1. a driver, `libcuda.so.1`, and +2. a CUDA runtime, `libcudart.so.13` then `libcudart.so.12`, newest first. + +The driver half is not belt-and-braces. A CUDA build links `-lcuda`, so on a +machine with a toolkit and no driver it cannot be `dlopen`ed at all — handing +that machine a CUDA artifact would turn a working CPU install into a NIF that +fails to load. Absent either half, the CPU artifact is chosen. + +Both are looked up through `ldconfig -p` and, failing that, on disk under +`/usr/local/cuda*/lib64`, `/usr/local/cuda-*/targets/*/lib`, +`/usr/lib/x86_64-linux-gnu` and `/usr/lib64`. + +`LLAMA_CUDA_VARIANT` overrides the result: `cu12`, `cu13`, or `none` to force the +CPU artifact. Only when no artifact matches does the Makefile run, and only then is a backend detected: 1. **macOS** → Metal -2. **Linux with `nvcc` in PATH** → CUDA +2. **Linux with a CUDA toolkit** → CUDA 3. **Otherwise** → CPU +"With a CUDA toolkit" means the Makefile found one, which is a wider test than +`nvcc` being on `PATH`: `CUDA_HOME`, then `CUDA_PATH`, then `nvcc` on `PATH`, +then `/usr/local/cuda`, `/opt/cuda`, then the newest `/usr/local/cuda-*`. DGX OS +puts `nvcc` on the login `PATH` only, via `/etc/profile.d/nv_paths.sh`, which +`ssh host mix compile`, systemd units and most CI shells never source — probing +`PATH` alone silently produced CPU-only builds on machines with a full toolkit. + ### Explicit Backend Setting `LLAMA_BACKEND` forces a source build (`make_force_build` in `mix.exs`), @@ -142,28 +179,45 @@ mix compile ### Linux (NVIDIA CUDA) -**A plain `mix compile` will not give you CUDA.** The published -`x86_64-linux-gnu` artifact is a CPU build, and downloading it skips the Makefile -entirely, so `nvcc` is never looked for. CUDA requires an explicit source build: +On x86_64 a plain `mix compile` now does give you CUDA, provided the machine has +a driver and a CUDA 12 or CUDA 13 runtime. No toolkit and no build tools are +needed for that path — the `-cu12`/`-cu13` artifact is downloaded like any other: + +```bash +mix deps.get +mix compile +``` + +Confirm what you got: + +```elixir +LlamaCppEx.devices() # backend "CUDA" on a CUDA artifact +``` + +A source build is still required for aarch64 Linux, for a CUDA major with no +published artifact, and any time you want flags of your own: ```bash # Prerequisites sudo apt-get install build-essential cmake git # Install CUDA toolkit: https://developer.nvidia.com/cuda-downloads -nvcc --version mix deps.get LLAMA_BACKEND=cuda mix compile ``` `LLAMA_BACKEND=cuda` forces the source build and selects CUDA explicitly. The -Makefile's `nvcc` auto-detection only ever applies to a source build that ran for -some other reason, so do not rely on it. +toolkit does not have to be on `PATH`; see the discovery order above, or set +`CUDA_HOME` to the directory holding `bin/nvcc`. If nothing is found the build +fails with that message rather than quietly linking against nothing. **CUDA version compatibility:** -- CUDA 11.7+ recommended -- CUDA 12.x preferred for latest GPU architectures -- The build uses static CUDA libraries by default +- CUDA 12 and CUDA 13 have published artifacts. Older majors build from source. +- Architectures come from ggml's portable default list under `LLAMA_PORTABLE=1`, + which covers Turing through Blackwell and includes `sm_121a` for GB10; a local + build without it compiles for the GPU actually present. +- NCCL is off unless `LLAMA_CUDA_NCCL=1`. See the README's build variables for + why the default is not ggml's. ### Linux (Vulkan) @@ -257,27 +311,58 @@ cmake --version ### CUDA Not Detected -First check whether a source build ran at all. If `mix compile` downloaded a -precompiled artifact then the Makefile never executed and `nvcc` was never -consulted — the binary is CPU-only by construction. Force a source build: +Find out which binary you are running before changing anything: + +```elixir +LlamaCppEx.devices() # backend "CUDA", or "CPU" if this is a CPU build +``` + +If it says CPU on a CUDA machine, the artifact probe declined. It requires both +a driver and a CUDA runtime: + +```bash +ldconfig -p | grep -E 'libcuda\.so\.1|libcudart\.so\.(12|13)' +``` + +A missing `libcuda.so.1` means no driver, and a CUDA build could not have been +loaded anyway. A missing `libcudart.so.N` means no CUDA runtime for a published +major. Name the variant directly if your layout defeats the probe: + +```bash +LLAMA_CUDA_VARIANT=cu13 mix deps.compile llama_cpp_ex --force +``` + +To build from source instead: ```bash LLAMA_BACKEND=cuda mix compile ``` -Then verify `nvcc` is in your PATH: +That fails loudly when no toolkit is found. `nvcc` does **not** need to be on +`PATH` — `CUDA_HOME`, `CUDA_PATH`, `/usr/local/cuda`, `/opt/cuda` and +`/usr/local/cuda-*` are all checked — but if the toolkit lives somewhere else: ```bash -which nvcc -nvcc --version +CUDA_HOME=/opt/nvidia/cuda-13.0 LLAMA_BACKEND=cuda mix compile ``` -If using a non-standard CUDA installation path: +### `undefined symbol` when the NIF loads + +A CUDA build that compiles and links but dies at load is a missing library on +the link line, not a broken toolkit. Two have bitten this project: +`cuMemCreate` (the driver API, fixed by linking `-lcuda`) and `ncclAllReduce` +(ggml enabling NCCL behind cmake's back, fixed by stating `-DGGML_CUDA_NCCL` +explicitly). Reproduce the diagnosis with: ```bash -LLAMA_CMAKE_ARGS="-DCMAKE_CUDA_COMPILER=/usr/local/cuda-12/bin/nvcc" mix compile +ldd -r _build/dev/lib/llama_cpp_ex/priv/llama_cpp_ex_nif.so | grep undefined ``` +Ignore `enif_*`: those are the NIF API and the BEAM resolves them when it loads +the library. Anything else unresolved is a real missing dependency. The +`cuda-link` job in `.github/workflows/ci.yml` runs exactly this check on every +pull request for both CUDA majors. + ### Metal Errors on macOS Ensure Xcode Command Line Tools are installed: diff --git a/lib/llama_cpp_ex/mtp.ex b/lib/llama_cpp_ex/mtp.ex index 8d3a941..fdbf2ad 100644 --- a/lib/llama_cpp_ex/mtp.ex +++ b/lib/llama_cpp_ex/mtp.ex @@ -110,6 +110,19 @@ defmodule LlamaCppEx.MTP do "model was loaded without load_mtp: true, so its MTP head layers are " <> "absent; reload it with LlamaCppEx.load_model(path, load_mtp: true)"} + LlamaCppEx.NIF.model_n_layer_nextn(model.ref) == 0 -> + # Distinct from the branch above and not fixable by any flag: the + # checkpoint simply has no MTP head. llama.cpp logs "context type MTP + # requested but model doesn't contain MTP layers" and returns null, which + # reaches the caller as a bare "failed to create context" with the real + # reason buried in engine output the caller may not even be showing. + # Most GGUF conversions of an MTP-capable model drop the head; the + # publisher usually ships it as a separate `-MTP` repository. + {:error, + "this GGUF contains no MTP head (0 nextn layers), so MTP speculative " <> + "decoding is unavailable for it; use an MTP-preserving conversion of " <> + "the model, which publishers typically ship as a separate -MTP build"} + true -> do_init(model, opts, n_draft) end diff --git a/lib/llama_cpp_ex/nif.ex b/lib/llama_cpp_ex/nif.ex index 946c9cf..bcfa3f0 100644 --- a/lib/llama_cpp_ex/nif.ex +++ b/lib/llama_cpp_ex/nif.ex @@ -42,6 +42,7 @@ defmodule LlamaCppEx.NIF do def model_n_ctx_train(_model), do: :erlang.nif_error(:not_loaded) def model_n_embd(_model), do: :erlang.nif_error(:not_loaded) + def model_n_layer_nextn(_model), do: :erlang.nif_error(:not_loaded) def model_desc(_model), do: :erlang.nif_error(:not_loaded) def model_size(_model), do: :erlang.nif_error(:not_loaded) def model_n_params(_model), do: :erlang.nif_error(:not_loaded) diff --git a/mix.exs b/mix.exs index 7f8ee00..75c9088 100644 --- a/mix.exs +++ b/mix.exs @@ -1,7 +1,39 @@ defmodule LlamaCppEx.Precompiler do @moduledoc false - @all_targets ["aarch64-apple-darwin", "x86_64-linux-gnu"] + # Linux CUDA artifacts are published per CUDA major version because the NIF + # links libcudart, libcublas and libcublasLt dynamically and their sonames are + # major-versioned: libcudart.so.12 against a CUDA 13 install does not resolve, + # and there is no compatibility shim in either direction. So the variant has + # to be part of the target name -- one "linux CUDA" artifact cannot exist. + # + # Newest first: a host with both toolkits installed should get the newer one. + @cuda_majors ["13", "12"] + + # Only x86_64 Linux gets CUDA variants today. aarch64 Linux (DGX Spark and + # friends) still resolves to no artifact and source-builds, which the + # Makefile's toolkit discovery now handles; adding it here is a matrix entry + # in .github/workflows/precompile.yml plus this list. + @cuda_targets for major <- @cuda_majors, do: "x86_64-linux-gnu-cu#{major}" + + @all_targets ["aarch64-apple-darwin", "x86_64-linux-gnu"] ++ @cuda_targets + + # Set by each CUDA leg of the precompile workflow. Detection below deliberately + # refuses to name a CUDA target on a machine with no driver, which is exactly + # what a release runner is, so the build has to state its own variant. + # Also the escape hatch for a host whose layout defeats the probe: "cu12", + # "cu13", or "none" to force the CPU artifact. + @variant_env "LLAMA_CUDA_VARIANT" + + # Where a CUDA runtime shows up when ldconfig has nothing to say -- a container + # with no ldconfig cache, or an install that was never registered. + @cuda_lib_globs [ + "/usr/local/cuda/lib64", + "/usr/local/cuda-*/lib64", + "/usr/local/cuda-*/targets/*/lib", + "/usr/lib/x86_64-linux-gnu", + "/usr/lib64" + ] def all_supported_targets(:fetch), do: @all_targets @@ -17,11 +49,75 @@ defmodule LlamaCppEx.Precompiler do cond do system_arch =~ ~r/aarch64.*apple.*darwin/ -> {:ok, "aarch64-apple-darwin"} - system_arch =~ ~r/x86_64.*linux.*gnu/ -> {:ok, "x86_64-linux-gnu"} + system_arch =~ ~r/x86_64.*linux.*gnu/ -> {:ok, "x86_64-linux-gnu" <> cuda_suffix()} true -> {:error, "unsupported target: #{system_arch}"} end end + @doc false + # Exposed for tests: the probe is pure over the two facts it looks up, so the + # interesting cases can be exercised without a CUDA install. + def cuda_suffix(env \\ &System.get_env/1, present? \\ &library_present?/1) do + case env.(@variant_env) do + nil -> detect_cuda_suffix(present?) + "" -> "" + "none" -> "" + "cu" <> major when major in @cuda_majors -> "-cu#{major}" + other -> raise ArgumentError, bad_variant_message(other) + end + end + + defp bad_variant_message(value) do + allowed = Enum.map_join(@cuda_majors, ", ", &"cu#{&1}") + "#{@variant_env}=#{inspect(value)} is not a known CUDA variant (#{allowed}, none)" + end + + # Two conditions, both required. The runtime libraries are what the artifact + # links against, and the driver is what -lcuda resolves to at load time: a + # machine with the toolkit but no driver cannot dlopen a CUDA build at all, so + # handing it one would turn a working CPU install into a NIF that fails to + # load. nvcc is deliberately not consulted -- running a CUDA build needs no + # compiler, and plenty of GPU hosts have no toolkit installed. + defp detect_cuda_suffix(present?) do + if present?.("libcuda.so.1") do + Enum.find_value(@cuda_majors, "", fn major -> + if present?.("libcudart.so.#{major}"), do: "-cu#{major}" + end) + else + "" + end + end + + defp library_present?(soname) do + ldconfig_lists?(soname) or on_disk?(soname) + end + + defp ldconfig_lists?(soname) do + # ldconfig lives in /sbin, which is routinely off a non-root PATH. + case Enum.find( + ["ldconfig", "/sbin/ldconfig", "/usr/sbin/ldconfig"], + &System.find_executable/1 + ) do + nil -> + false + + ldconfig -> + case System.cmd(ldconfig, ["-p"], stderr_to_stdout: true) do + {output, 0} -> String.contains?(output, soname) + _ -> false + end + end + catch + # An ldconfig that is present but unusable is a "no", never a build failure. + _, _ -> false + end + + defp on_disk?(soname) do + Enum.any?(@cuda_lib_globs, fn glob -> + glob |> Path.join(soname) |> Path.wildcard() |> Enum.any?() + end) + end + def build_native(args), do: ElixirMake.Precompiler.mix_compile(args) def precompile(args, _target) do @@ -184,7 +280,14 @@ defmodule LlamaCppEx.MixProject do # LLAMA_BACKEND auto | metal | cuda | vulkan | cpu # LLAMA_CMAKE_ARGS extra flags appended to the llama.cpp cmake invocation # LLAMA_PORTABLE 1 to drop -march=native, set by the precompile workflow - @make_env_passthrough ["LLAMA_BACKEND", "LLAMA_CMAKE_ARGS", "LLAMA_PORTABLE"] + # LLAMA_CUDA_NCCL 1 to build and link ggml's NCCL multi-GPU collectives, + # which also makes libnccl.so.2 a load-time requirement + @make_env_passthrough [ + "LLAMA_BACKEND", + "LLAMA_CMAKE_ARGS", + "LLAMA_PORTABLE", + "LLAMA_CUDA_NCCL" + ] defp make_env do base = %{"FINE_INCLUDE_DIR" => Fine.include_dir()} diff --git a/test/mtp_test.exs b/test/mtp_test.exs index 19a2abd..b55f8d6 100644 --- a/test/mtp_test.exs +++ b/test/mtp_test.exs @@ -58,6 +58,36 @@ defmodule LlamaCppEx.MTPTest do end end + # A checkpoint with no MTP head is a different failure from a model loaded + # without the flag, and no flag recovers it. Most GGUF conversions of an + # MTP-capable model drop the head — unsloth's Qwen3.6-35B-A3B-UD-Q4_K_XL has + # zero nextn layers while their separate -MTP build of the same model has + # them — so this is the case a user actually lands on first. + # + # One gate tag (`:smoke`), never `:mtp` as well: the generation model is an + # ordinary checkpoint, which is exactly what makes it the right fixture here. + describe "init/2 on a checkpoint with no MTP head" do + @describetag :smoke + + setup do + path = LlamaCppEx.TestModels.path!(:gen) + {:ok, model} = LlamaCppEx.load_model(path, n_gpu_layers: 0, load_mtp: true) + %{model: model} + end + + test "reports zero nextn layers rather than guessing", %{model: model} do + assert LlamaCppEx.NIF.model_n_layer_nextn(model.ref) == 0 + end + + test "refuses with the reason, not 'failed to create context'", %{model: model} do + assert {:error, message} = MTP.init(model, n_draft: 3) + assert message =~ "no MTP head" + assert message =~ "-MTP" + # The bare context error is what this guard exists to replace. + refute message =~ "failed to create context" + end + end + describe "the %MTP{} struct" do test "enforces every field, because each one is a live NIF resource" do # A partially built MTP would hand a nil reference to the NIF. diff --git a/test/precompiler_test.exs b/test/precompiler_test.exs new file mode 100644 index 0000000..8c04a92 --- /dev/null +++ b/test/precompiler_test.exs @@ -0,0 +1,121 @@ +defmodule LlamaCppEx.PrecompilerTest do + use ExUnit.Case, async: true + + alias LlamaCppEx.Precompiler + + # The precompiler decides which release artifact a user downloads. Getting it + # wrong is not a build failure, it is a NIF that either silently runs on the + # CPU or refuses to dlopen, so the selection rules are pinned here. + + defp env(map), do: fn key -> Map.get(map, key) end + defp libs(list), do: fn soname -> soname in list end + + @driver "libcuda.so.1" + @cuda12 "libcudart.so.12" + @cuda13 "libcudart.so.13" + + describe "all_supported_targets/1" do + test "declares a CPU and a CUDA artifact per supported Linux CUDA major" do + targets = Precompiler.all_supported_targets(:fetch) + + assert "x86_64-linux-gnu" in targets + assert "x86_64-linux-gnu-cu12" in targets + assert "x86_64-linux-gnu-cu13" in targets + assert "aarch64-apple-darwin" in targets + end + + test "every declared target is unique" do + targets = Precompiler.all_supported_targets(:fetch) + assert targets == Enum.uniq(targets) + end + + test "compile mode offers only the target this machine actually is" do + case Precompiler.current_target() do + {:ok, target} -> assert Precompiler.all_supported_targets(:compile) == [target] + {:error, _} -> assert Precompiler.all_supported_targets(:compile) == [] + end + end + end + + describe "cuda_suffix/2 detection" do + test "no CUDA runtime and no driver selects the CPU artifact" do + assert Precompiler.cuda_suffix(env(%{}), libs([])) == "" + end + + test "driver plus CUDA 13 runtime selects cu13" do + assert Precompiler.cuda_suffix(env(%{}), libs([@driver, @cuda13])) == "-cu13" + end + + test "driver plus CUDA 12 runtime selects cu12" do + assert Precompiler.cuda_suffix(env(%{}), libs([@driver, @cuda12])) == "-cu12" + end + + test "both runtimes installed prefers the newer major" do + assert Precompiler.cuda_suffix(env(%{}), libs([@driver, @cuda12, @cuda13])) == "-cu13" + end + + # The regression this ordering exists to prevent: a CUDA artifact links + # -lcuda, so with no driver present it cannot be dlopen'd at all. Handing a + # toolkit-only machine a CUDA build turns a working CPU install into a NIF + # that fails to load, which is strictly worse than running on the CPU. + test "toolkit without a driver stays on the CPU artifact" do + assert Precompiler.cuda_suffix(env(%{}), libs([@cuda12, @cuda13])) == "" + end + + test "driver without any CUDA runtime stays on the CPU artifact" do + assert Precompiler.cuda_suffix(env(%{}), libs([@driver])) == "" + end + + test "an unpublished CUDA major is not selected" do + assert Precompiler.cuda_suffix(env(%{}), libs([@driver, "libcudart.so.11"])) == "" + end + end + + describe "cuda_suffix/2 explicit variant" do + # Release runners have the toolkit but no driver, so detection would name + # them CPU. Each CUDA leg states its own variant instead. + test "the variant override wins over detection" do + assert Precompiler.cuda_suffix(env(%{"LLAMA_CUDA_VARIANT" => "cu12"}), libs([])) == "-cu12" + assert Precompiler.cuda_suffix(env(%{"LLAMA_CUDA_VARIANT" => "cu13"}), libs([])) == "-cu13" + end + + test "none forces the CPU artifact even on a working CUDA host" do + full = libs([@driver, @cuda13]) + assert Precompiler.cuda_suffix(env(%{"LLAMA_CUDA_VARIANT" => "none"}), full) == "" + assert Precompiler.cuda_suffix(env(%{"LLAMA_CUDA_VARIANT" => ""}), full) == "" + end + + test "an unknown variant fails loudly rather than publishing a wrong name" do + assert_raise ArgumentError, ~r/not a known CUDA variant/, fn -> + Precompiler.cuda_suffix(env(%{"LLAMA_CUDA_VARIANT" => "cu11"}), libs([])) + end + + assert_raise ArgumentError, ~r/not a known CUDA variant/, fn -> + Precompiler.cuda_suffix(env(%{"LLAMA_CUDA_VARIANT" => "yes"}), libs([])) + end + end + + test "every variant the override accepts is a target that gets published" do + targets = Precompiler.all_supported_targets(:fetch) + + for variant <- ["cu12", "cu13"] do + suffix = Precompiler.cuda_suffix(env(%{"LLAMA_CUDA_VARIANT" => variant}), libs([])) + assert ("x86_64-linux-gnu" <> suffix) in targets + end + end + end + + describe "current_target/0" do + test "names a target this build could actually download" do + case Precompiler.current_target() do + {:ok, target} -> + # macOS and x86_64 Linux are published; anything else must report an + # error so elixir_make falls back to a source build. + assert target in Precompiler.all_supported_targets(:fetch) + + {:error, message} -> + assert message =~ "unsupported target" + end + end + end +end