diff --git a/.github/actions/install-wasi-sdk-wabt/action.yml b/.github/actions/install-wasi-sdk-wabt/action.yml index 31d22c18d5..16949aae82 100644 --- a/.github/actions/install-wasi-sdk-wabt/action.yml +++ b/.github/actions/install-wasi-sdk-wabt/action.yml @@ -15,6 +15,13 @@ inputs: os: description: "Operating system to install on (ubuntu, macos)" required: true + # Default 29 because addr2line.py (samples/debug-tools*) requires + # llvm-symbolizer / llvm-addr2line, which only ship in wasi-sdk 29+. + # Callers can pin a different version when they have a specific need. + wasi_sdk_version: + description: "wasi-sdk major version to install (e.g. '25', '29', '33')" + required: false + default: "29" runs: using: "composite" @@ -29,13 +36,17 @@ runs: - name: Set up wasi-sdk and wabt on Ubuntu if: ${{ startsWith(inputs.os, 'ubuntu') }} shell: bash + env: + SDK_VER: ${{ inputs.wasi_sdk_version }} run: | + SDK_DIR="wasi-sdk-${SDK_VER}.0-x86_64-linux" echo "Downloading wasi-sdk for Ubuntu..." - sudo wget -O wasi-sdk.tar.gz --progress=dot:giga https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-25/wasi-sdk-25.0-x86_64-linux.tar.gz + sudo wget -O wasi-sdk.tar.gz --progress=dot:giga \ + "https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-${SDK_VER}/${SDK_DIR}.tar.gz" echo "Extracting wasi-sdk..." sudo tar -xf wasi-sdk.tar.gz - sudo ln -sf wasi-sdk-25.0-x86_64-linux/ wasi-sdk + sudo ln -sf "${SDK_DIR}/" wasi-sdk echo "Downloading wabt for Ubuntu..." sudo wget -O wabt.tar.gz --progress=dot:giga https://github.com/WebAssembly/wabt/releases/download/1.0.37/wabt-1.0.37-ubuntu-20.04.tar.gz @@ -47,19 +58,23 @@ runs: /opt/wasi-sdk/bin/clang --version /opt/wabt/bin/wasm-interp --version - echo "::notice::wasi-sdk-25 and wabt-1.0.37 installed on ubuntu" + echo "::notice::wasi-sdk-${SDK_VER} and wabt-1.0.37 installed on ubuntu" working-directory: /opt - name: Set up wasi-sdk and wabt on macOS on Intel if: ${{ inputs.os == 'macos-15-intel' }} shell: bash + env: + SDK_VER: ${{ inputs.wasi_sdk_version }} run: | + SDK_DIR="wasi-sdk-${SDK_VER}.0-x86_64-macos" echo "Downloading wasi-sdk for macOS on Intel..." - sudo wget -O wasi-sdk.tar.gz --progress=dot:giga https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-25/wasi-sdk-25.0-x86_64-macos.tar.gz + sudo wget -O wasi-sdk.tar.gz --progress=dot:giga \ + "https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-${SDK_VER}/${SDK_DIR}.tar.gz" echo "Extracting wasi-sdk..." sudo tar -xf wasi-sdk.tar.gz - sudo ln -sf wasi-sdk-25.0-x86_64-macos wasi-sdk + sudo ln -sf "${SDK_DIR}" wasi-sdk echo "Downloading wabt for macOS on Intel..." sudo wget -O wabt.tar.gz --progress=dot:giga https://github.com/WebAssembly/wabt/releases/download/1.0.36/wabt-1.0.36-macos-12.tar.gz @@ -71,19 +86,23 @@ runs: /opt/wasi-sdk/bin/clang --version /opt/wabt/bin/wasm-interp --version - echo "::notice::wasi-sdk-25 and wabt-1.0.36 installed on ${{ inputs.os }}" + echo "::notice::wasi-sdk-${SDK_VER} and wabt-1.0.36 installed on ${{ inputs.os }}" working-directory: /opt - name: Set up wasi-sdk and wabt on macOS on ARM if: ${{ inputs.os == 'macos-15' }} shell: bash + env: + SDK_VER: ${{ inputs.wasi_sdk_version }} run: | + SDK_DIR="wasi-sdk-${SDK_VER}.0-arm64-macos" echo "Downloading wasi-sdk for macOS on ARM..." - sudo wget -O wasi-sdk.tar.gz --progress=dot:giga https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-25/wasi-sdk-25.0-arm64-macos.tar.gz + sudo wget -O wasi-sdk.tar.gz --progress=dot:giga \ + "https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-${SDK_VER}/${SDK_DIR}.tar.gz" echo "Extracting wasi-sdk..." sudo tar -xf wasi-sdk.tar.gz - sudo ln -sf wasi-sdk-25.0-arm64-macos wasi-sdk + sudo ln -sf "${SDK_DIR}" wasi-sdk echo "Downloading wabt for macOS on ARM..." sudo wget -O wabt.tar.gz --progress=dot:giga https://github.com/WebAssembly/wabt/releases/download/1.0.37/wabt-1.0.37-macos-14.tar.gz @@ -95,12 +114,14 @@ runs: /opt/wasi-sdk/bin/clang --version /opt/wabt/bin/wasm-interp --version - echo "::notice::wasi-sdk-25 and wabt-1.0.37 installed on ${{ inputs.os }}" + echo "::notice::wasi-sdk-${SDK_VER} and wabt-1.0.37 installed on ${{ inputs.os }}" working-directory: /opt - name: Set up wasi-sdk and wabt on Windows if: ${{ startsWith(inputs.os, 'windows') }} shell: bash + env: + SDK_VER: ${{ inputs.wasi_sdk_version }} run: | choco install -y wget @@ -108,7 +129,8 @@ runs: mkdir -p /opt/wabt echo "Downloading wasi-sdk for Windows..." - wget -O wasi-sdk.tar.gz --progress=dot:giga https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-25/wasi-sdk-25.0-x86_64-windows.tar.gz + wget -O wasi-sdk.tar.gz --progress=dot:giga \ + "https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-${SDK_VER}/wasi-sdk-${SDK_VER}.0-x86_64-windows.tar.gz" echo "Extracting wasi-sdk..." tar --strip-components=1 -xf wasi-sdk.tar.gz -C /opt/wasi-sdk @@ -122,4 +144,4 @@ runs: /opt/wasi-sdk/bin/clang --version /opt/wabt/bin/wasm-interp --version - echo "::notice::wasi-sdk-25 and wabt-1.0.37 installed on Windows" + echo "::notice::wasi-sdk-${SDK_VER} and wabt-1.0.37 installed on Windows" diff --git a/.github/workflows/compilation_on_android_ubuntu.yml b/.github/workflows/compilation_on_android_ubuntu.yml index ee1cdab36d..76edbb886c 100644 --- a/.github/workflows/compilation_on_android_ubuntu.yml +++ b/.github/workflows/compilation_on_android_ubuntu.yml @@ -602,9 +602,29 @@ jobs: mkdir build && cd build cmake .. cmake --build . --config Debug --parallel 4 - ./iwasm wasm-apps/trap.wasm | grep "#" > call_stack.txt - ./iwasm wasm-apps/trap.aot | grep "#" > call_stack_aot.txt - bash -x ../symbolicate.sh + cd .. + ./verify.sh + + - name: Install binaryen (for debug-tools-optimized) + run: | + sudo mkdir -p /opt/binaryen + curl -L https://github.com/WebAssembly/binaryen/releases/download/version_117/binaryen-version_117-x86_64-linux.tar.gz \ + | sudo tar xz -C /opt/binaryen --strip-components=1 + /opt/binaryen/bin/wasm-opt --version + + - name: Build Sample [debug-tools-optimized] + run: | + cd samples/debug-tools-optimized + for INTERP_FLAG in "" "-DUSE_FAST_INTERP=ON"; do + rm -rf build && mkdir build && cd build + cmake .. $INTERP_FLAG + make -j$(nproc) + cd .. + ./verify.sh oob wasm + ./verify.sh oob aot + ./verify.sh stackoverflow wasm + ./verify.sh stackoverflow aot + done - name: Build Sample [native-stack-overflow] run: | diff --git a/.github/workflows/compilation_on_macos.yml b/.github/workflows/compilation_on_macos.yml index ce5619fa5d..e3afadb58d 100644 --- a/.github/workflows/compilation_on_macos.yml +++ b/.github/workflows/compilation_on_macos.yml @@ -413,9 +413,8 @@ jobs: mkdir build && cd build cmake .. cmake --build . --config Debug --parallel 4 - ./iwasm wasm-apps/trap.wasm | grep "#" > call_stack.txt - ./iwasm wasm-apps/trap.aot | grep "#" > call_stack_aot.txt - bash -x ../symbolicate.sh + cd .. + ./verify.sh - name: Build Sample [native-stack-overflow] run: | diff --git a/.github/workflows/nightly_run.yml b/.github/workflows/nightly_run.yml index c5b8237c8d..9ec2a9cb95 100644 --- a/.github/workflows/nightly_run.yml +++ b/.github/workflows/nightly_run.yml @@ -667,6 +667,43 @@ jobs: ./build.sh --aot ./run.sh --aot + addr2line_tests_multi_sdk: + runs-on: ubuntu-22.04 + steps: + - name: checkout + uses: actions/checkout@v6.0.2 + + - name: Install wasi-sdk 29.0 + run: | + sudo mkdir -p /opt/wasi-sdk-29.0-x86_64-linux + curl -L https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-29/wasi-sdk-29.0-x86_64-linux.tar.gz \ + | sudo tar xz -C /opt/wasi-sdk-29.0-x86_64-linux --strip-components=1 + + - name: Install wasi-sdk 33.0 + run: | + sudo mkdir -p /opt/wasi-sdk-33.0-x86_64-linux + curl -L https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-33/wasi-sdk-33.0-x86_64-linux.tar.gz \ + | sudo tar xz -C /opt/wasi-sdk-33.0-x86_64-linux --strip-components=1 + + - name: Install wabt + run: | + sudo mkdir -p /opt/wabt + curl -L https://github.com/WebAssembly/wabt/releases/download/1.0.34/wabt-1.0.34-ubuntu.tar.gz \ + | sudo tar xz -C /opt/wabt --strip-components=1 + + - name: Install binaryen + run: | + sudo mkdir -p /opt/binaryen + curl -L https://github.com/WebAssembly/binaryen/releases/download/version_117/binaryen-version_117-x86_64-linux.tar.gz \ + | sudo tar xz -C /opt/binaryen --strip-components=1 + + - name: Install pytest + run: pip install pytest + + - name: Run addr2line.py tests across both SDKs + working-directory: test-tools/addr2line/tests + run: ./run_tests.sh --multi-sdk -v + test: needs: [build_iwasm, build_llvm_libraries_on_ubuntu, build_wamrc] runs-on: ${{ matrix.os }} diff --git a/samples/debug-tools-optimized/CMakeLists.txt b/samples/debug-tools-optimized/CMakeLists.txt new file mode 100644 index 0000000000..622b96f981 --- /dev/null +++ b/samples/debug-tools-optimized/CMakeLists.txt @@ -0,0 +1,89 @@ +# Copyright (C) 2026 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +cmake_minimum_required(VERSION 3.14) + +include(CheckPIESupported) + +project(debug_tools_optimized_sample) + +list(APPEND CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/../cmake) + +# WAMR features switch +string(TOLOWER ${CMAKE_HOST_SYSTEM_NAME} WAMR_BUILD_PLATFORM) +if (APPLE) + add_definitions(-DBH_PLATFORM_DARWIN) +endif () + +set(CMAKE_SHARED_LIBRARY_LINK_C_FLAGS "") +set(CMAKE_SHARED_LIBRARY_LINK_CXX_FLAGS "") + +if (NOT DEFINED WAMR_BUILD_TARGET) + if (CMAKE_SYSTEM_PROCESSOR MATCHES "^(arm64|aarch64)") + set (WAMR_BUILD_TARGET "AARCH64") + elseif (CMAKE_SYSTEM_PROCESSOR STREQUAL "riscv64") + set (WAMR_BUILD_TARGET "RISCV64") + elseif (CMAKE_SIZEOF_VOID_P EQUAL 8) + set (WAMR_BUILD_TARGET "X86_64") + elseif (CMAKE_SIZEOF_VOID_P EQUAL 4) + set (WAMR_BUILD_TARGET "X86_32") + else () + message(SEND_ERROR "Unsupported build target platform!") + endif () +endif () + +if (NOT CMAKE_BUILD_TYPE) + set (CMAKE_BUILD_TYPE Release) +endif () + +option(USE_FAST_INTERP "Build iwasm with fast interpreter instead of classic interpreter" OFF) + +set(WAMR_BUILD_INTERP 1) +set(WAMR_BUILD_LIBC_WASI 1) +if (USE_FAST_INTERP) + set(WAMR_BUILD_FAST_INTERP 1) + message(STATUS "Building iwasm with fast interpreter (USE_FAST_INTERP=ON)") +else () + set(WAMR_BUILD_FAST_INTERP 0) + message(STATUS "Building iwasm with classic interpreter (default; pass -DUSE_FAST_INTERP=ON for fast-interp)") +endif () +set(WAMR_BUILD_AOT 1) +set(WAMR_BUILD_DUMP_CALL_STACK 1) +# Disable hardware bound checks so OOB memory traps go through the +# interpreter's exception path (which calls SYNC_ALL_TO_FRAME and captures +# frame_ip), instead of the SIGSEGV signal-handler-and-longjmp path +# (which doesn't update frame->ip). Capturing frame_ip is what lets +# addr2line.py recover inline frames for the OOB sample. +set(WAMR_DISABLE_HW_BOUND_CHECK 1) + +if (NOT (CMAKE_C_COMPILER MATCHES ".*clang.*" OR CMAKE_C_COMPILER_ID MATCHES ".*Clang")) + set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,--gc-sections") +endif () + +set(WAMR_ROOT_DIR ${CMAKE_CURRENT_LIST_DIR}/../..) +include (${WAMR_ROOT_DIR}/build-scripts/runtime_lib.cmake) + +add_library(vmlib ${WAMR_RUNTIME_LIB_SOURCE}) + +################ wasm application ################ +include(ExternalProject) + +ExternalProject_Add(wasm-apps + SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/wasm-apps" + CONFIGURE_COMMAND ${CMAKE_COMMAND} -S ${CMAKE_CURRENT_SOURCE_DIR}/wasm-apps -B build + BUILD_COMMAND ${CMAKE_COMMAND} --build build + INSTALL_COMMAND ${CMAKE_COMMAND} --install build --prefix ${CMAKE_CURRENT_BINARY_DIR} +) + +################ wamr runtime ################ +include (${SHARED_DIR}/utils/uncommon/shared_uncommon.cmake) + +set (RUNTIME_SOURCE_ALL + ${CMAKE_CURRENT_LIST_DIR}/../../product-mini/platforms/linux/main.c + ${UNCOMMON_SHARED_SOURCE} +) +add_executable (iwasm ${RUNTIME_SOURCE_ALL}) +check_pie_supported() +set_target_properties (iwasm PROPERTIES POSITION_INDEPENDENT_CODE ON) +target_link_libraries(iwasm vmlib -lm -ldl) +add_dependencies(iwasm wasm-apps) diff --git a/samples/debug-tools-optimized/README.md b/samples/debug-tools-optimized/README.md new file mode 100644 index 0000000000..ac9c2814e9 --- /dev/null +++ b/samples/debug-tools-optimized/README.md @@ -0,0 +1,283 @@ +# debug-tools-optimized — Debugging Production-Optimized WASM + +This sample demonstrates symbolication of crashes in **production-optimized** WASM +binaries using the merged `addr2line.py`. The wasm apps are compiled with +`-Oz -g -flto` and post-processed with `wasm-opt -Oz -g`, then stripped to produce +minimal production binaries. A "debug companion" binary built in parallel retains +DWARF inline info, enabling source-level call stack recovery offline. + +## Why this exists alongside `samples/debug-tools/` + +The existing `samples/debug-tools/` sample uses `-O0 -g`: each function is preserved, +no inlining happens. This sample uses `-Oz -g -flto`, which: + +- Aggressively inlines functions across translation units (cross-TU inlining via LTO) +- Strips the production binary to minimum size +- Tests `addr2line.py`'s inline expansion (`DW_TAG_inlined_subroutine` resolution) + +If you only need to debug development builds, `samples/debug-tools/` is sufficient. +If you need to debug **shipped** binaries, this sample shows how. + +## Build pipeline + +``` +clang -Oz -g -flto source1.c source2.c → .wasm (intermediate) + └─ wasm-opt -Oz -g → .debug.wasm (companion: code + DWARF + names) + └─ llvm-strip --strip-all → .prod.wasm (production: code only) +``` + +## Why production is derived from the debug companion + +`wasm-opt -Oz` (without `-g`) and `wasm-opt -Oz -g` produce **structurally different +binaries**: `-g` inhibits some inlining passes to preserve DWARF integrity. If we ran +them as separate pipelines, the production binary's code offsets would not match the +companion's DWARF address space — offline decode would silently break. + +Instead, we run `wasm-opt -Oz -g` once and derive production by stripping the +companion (`llvm-strip --strip-all`). Custom sections (DWARF, names) live *after* the +code section in the WASM binary format, so stripping them doesn't shift code offsets. +This **guarantees byte-identical code** between production and companion. + +## Why `-flto` + +Without LTO, functions in separate `.c` files remain separate WASM functions even +under `-Oz`. With LTO, the compiler sees all sources as one unit and inlines +aggressively across files. This is what makes the `do_bad_access → trigger_oob → +app_main` chain collapse into a single WASM function with multiple inlined +subroutines. + +## Why `recurse()` is non-tail-recursive + +`stackoverflow_recurse.c` uses +`int r = recurse(depth + 1); return r + buf[0];` instead of +`return recurse(depth + 1);`. Tail calls (`return f(...)`) get converted to loops at +`-Oz -flto`, eliminating the recursion that we want to test. The non-tail form forces +a real `call` instruction so each iteration pushes a new frame. + +## Prerequisites + +- wasi-sdk at `WASI_SDK_PATH` or `/opt/wasi-sdk` +- binaryen at `BINARYEN_PATH` or `/opt/binaryen` +- wabt at `WABT_PATH` or `/opt/wabt` +- Python 3 + +## Quick start + +```bash +mkdir -p build && cd build +cmake .. && make -j$(nproc) +cd .. + +# wasm (interpreter) +./symbolicate.sh oob +./symbolicate.sh stackoverflow + +# aot +./symbolicate.sh oob aot +./symbolicate.sh stackoverflow aot +``` + +To build with the fast interpreter instead of the classic interpreter: + +```bash +mkdir -p build && cd build +cmake .. -DUSE_FAST_INTERP=ON && make -j$(nproc) +``` + +The same `symbolicate.sh` invocations work for both build modes — it auto-detects +which interpreter the iwasm binary uses (by inspecting it for the `wasm_interp_fast.c` +symbol) and passes the right `--mode` to `addr2line.py`. + +### `verify.sh` — assertion-based smoke test + +`verify.sh ` runs `symbolicate.sh` and asserts the +symbolicated output matches the expected shape: for the OOB sample, +all three inline frames (`do_bad_access (inlined into trigger_oob)`, +`trigger_oob (inlined into app_main)`, `app_main`) plus their source +files; for the stackoverflow sample, `recurse` and `app_main` resolved +across the recursive chain. It auto-detects classic vs fast-interp +builds and relaxes the OOB assertion for fast-interp + wasm (where the +runtime offset doesn't map to source, so addr2line.py only emits the +outermost function name). Used by CI; useful locally to check a build: + +```bash +./verify.sh oob wasm # PASS or fails with the captured output +./verify.sh stackoverflow aot +``` + +The build pipeline produces three artifacts per app: + +| Artifact | Purpose | +|----------|---------| +| `.debug.wasm` | Debug companion — DWARF + name section, used for symbolication | +| `.prod.wasm` | Production wasm — fully stripped, runs in interpreter mode | +| `.prod.aot` | Production AOT — wamrc-compiled with `--enable-dump-call-stack` | + +Both `.prod.wasm` and `.prod.aot` use the same `.debug.wasm` companion for offline +symbolication. AOT and classic-interp produce nearly identical call stack offsets +(within 1-2 bytes), and `addr2line.py` resolves both correctly. + +## Build-side tools (and why each is needed) + +This sample chains four compile-time and post-build tools, each filling a +specific gap; dropping any of them would break the inline-DWARF round trip +the sample is meant to demonstrate: + +| Tool | From | What it does here | Why we need it | +|------|------|-------------------|----------------| +| `clang -Oz -g -flto` | wasi-sdk | Compiles `.c` → `.wasm` with size-optimized code, DWARF debug info, and link-time optimization | `-flto` enables cross-translation-unit inlining. Without LTO, `oob_main.c` and `oob_access.c` would stay as separate WASM functions; with LTO they collapse into a single function with `DW_TAG_inlined_subroutine` entries — exactly what we need to test inline-aware symbolication. | +| `wasm-opt -Oz -g` | binaryen | Post-optimization pass on the wasm; preserves DWARF integrity | Further size shrink (LEB compaction, dead-code elimination) on top of clang. The `-g` flag inhibits transformations that would corrupt DWARF, producing a "debug companion" with the same code layout as production. | +| `llvm-strip --strip-all` | wasi-sdk | Strips DWARF and name sections from the companion → production binary | Custom sections (DWARF, name) live AFTER the code section in the wasm format, so stripping them doesn't shift code offsets. This guarantees the production binary's runtime offsets map 1:1 to the companion's DWARF — without this property, offline decode would silently break. | +| `wamrc --enable-dump-call-stack --bounds-checks=1` | wamr-compiler | Compiles `.wasm` → `.aot` for ahead-of-time execution | AOT is the realistic embedded deployment target. `--bounds-checks=1` forces software memory bounds checks instead of hardware traps, so OOB exceptions go through the runtime exception path (which captures `frame_ip`) instead of a SIGSEGV that bypasses ip capture — without this, the OOB sample's AOT path would crash with no captured call stack. | + +For the **decode-side** tools (`llvm-symbolizer`, `llvm-dwarfdump`, +`wasm-objdump`, `llvm-cxxfilt`) and the interval-table overlay that +addr2line.py applies to correct the LLVM symbolizer's outermost +function-name lookup on wasm, see +[`test-tools/addr2line/README.md`](../../test-tools/addr2line/README.md). + +## Three execution modes, three offset spaces + +`addr2line.py` supports a `--mode={interp,aot,fast-interp}` flag because each +mode reports offsets differently: + +| Mode | Offset space | Adjustment | Source resolution | +|------|--------------|-----------|-------------------| +| `interp` (default) | File-absolute, post-advance | `offset - code_start - 1` | Full file:line, inline expansion | +| `aot` | File-absolute, instruction-start | `offset - code_start` | Full file:line, inline expansion | +| `fast-interp` | Function-relative, transformed bytecode | (not mappable) | Function-name only | + +**Why fast-interp can't show source lines**: at load time, fast-interp rewrites the +WASM bytecode in memory (replaces opcodes with handler indices, expands LEBs to +fixed widths, etc.). The runtime ip then points into this *transformed* buffer, not +the original WASM bytes — so there's no way to map the offset back to a source line. +Function-name lookup still works via `wasm-objdump` + `llvm-dwarfdump --name=...`. + +`symbolicate.sh` handles all three modes transparently. For manual invocation: + +```bash +# Classic interp (default) +python3 ../../test-tools/addr2line/addr2line.py --wasm-file ... callstack.txt + +# AOT +python3 ../../test-tools/addr2line/addr2line.py --mode=aot --wasm-file ... callstack.txt + +# Fast interp +python3 ../../test-tools/addr2line/addr2line.py --mode=fast-interp --wasm-file ... callstack.txt +``` + +## Why `iwasm -f app_main` (and not just `iwasm `) + +The `symbolicate.sh` script invokes `iwasm -f app_main` instead of letting iwasm run +the default wasi `_start` entry. This matters for two reasons: + +1. **Compiler folding**: Under `-Oz -flto`, when control reaches the OOB write through + `_start → __wasi_main_void → main → app_main → ...`, LLVM observes the entire chain + leading to undefined behavior and may rewrite it as `unreachable`. Calling + `app_main` directly preserves the explicit OOB instruction and produces the + expected `out of bounds memory access` exception. + +2. **Cleaner trap point**: With `-f app_main`, the WAMR call stack starts at our app's + entry, not deep inside wasi-libc startup, making the symbolication output more + focused on user code. + +## Expected output + +### oob app + +``` +=== Running iwasm on oob.prod.wasm (expect crash) === + +#00: 0x1dd8 - app_main + +Exception: out of bounds memory access + +=== Captured call stack === +#00: 0x1dd8 - app_main + +=== Symbolicated call stack (using debug companion) === +0: do_bad_access (inlined into trigger_oob) + at .../wasm-apps/oob_access.c:11:15 + trigger_oob (inlined into app_main) + at .../wasm-apps/oob_main.c:17:5 + app_main + at .../wasm-apps/oob_main.c:23:5 +``` + +Although `do_bad_access` and `trigger_oob` were both inlined into `app_main` +under `-Oz -flto` (the runtime sees only a single WASM function), the debug +companion's DWARF retains `DW_TAG_inlined_subroutine` entries that describe +the inline chain. `addr2line.py` walks them and reports the trap site at all +three source levels. + +This works because the iwasm in this sample is built with +`WAMR_DISABLE_HW_BOUND_CHECK=1`: OOB memory access goes through the +interpreter's exception path (which captures `frame_ip` via +`SYNC_ALL_TO_FRAME`), not through a SIGSEGV signal handler that +longjmps out without updating ip. The AOT build uses `--bounds-checks=1` +for the same reason. + +### stackoverflow app + +``` +=== Running iwasm on stackoverflow.prod.wasm (expect crash) === + +#00: 0x1e15 - $f12 +#01: 0x1e15 - $f12 + ... (~20 identical frames as the wasm operand stack overflows) ... +#21: 0x1e15 - $f12 +#22: 0x1dd0 - app_main + +Exception: wasm operand stack overflow + +=== Captured call stack === +... same as above ... + +=== Symbolicated call stack (using debug companion) === +0: recurse + at .../wasm-apps/stackoverflow_recurse.c:20:13 + ... (one resolved frame per captured frame) ... +22: app_main + at .../wasm-apps/stackoverflow_main.c:14:5 +``` + +Stack overflow produces non-zero offsets at every frame (the runtime +captures the ip of the call instruction in each caller), so `addr2line.py` +resolves source file, line number, and function name correctly across the +recursive chain. `llvm-symbolizer`'s outermost frame's function name is +unreliable on wasm for two independent reasons: LLVM unconditionally +overrides that frame's name from an object-file symbol-table lookup +(which can return a wrong function on some wasm layouts), and +`wasm-opt -Oz -g` leaves dead-code-eliminated `DW_TAG_subprogram` DIEs +behind with `low_pc = 0` which then violate DWARF's DIE-range map +invariants. `addr2line.py` always applies an interval-table overlay +to the outermost frame's name, so it renders as `recurse` / `app_main` +in both cases. + +## Manual decode + +If you have a captured stack from another iwasm run (e.g., from a remote board or +saved log), you can symbolicate it directly: + +```bash +python3 ../../test-tools/addr2line/addr2line.py \ + --wasi-sdk /opt/wasi-sdk \ + --wabt /opt/wabt \ + --wasm-file build/wasm-apps/oob.debug.wasm \ + /path/to/saved/call_stack.txt +``` + +## Environment variables + +| Variable | Default | Used by | +|----------|---------|---------| +| `WASI_SDK_PATH` | `/opt/wasi-sdk` | Build (clang, llvm-strip) and decode (llvm-symbolizer, llvm-dwarfdump) | +| `BINARYEN_PATH` | `/opt/binaryen` | Build (wasm-opt) | +| `WABT_PATH` | `/opt/wabt` | Decode (wasm-objdump) | + +## References + +- [addr2line.py](../../test-tools/addr2line/addr2line.py) +- [WAMR Dump Call Stack Feature](../../doc/build_wamr.md#dump-call-stack-feature) +- [Zephyr coredump-debug sample](../../product-mini/platforms/zephyr/coredump-debug/) — same workflow on embedded +- [debug-tools sample](../debug-tools/) — non-optimized debug build for comparison diff --git a/samples/debug-tools-optimized/symbolicate.sh b/samples/debug-tools-optimized/symbolicate.sh new file mode 100755 index 0000000000..1b77fc897b --- /dev/null +++ b/samples/debug-tools-optimized/symbolicate.sh @@ -0,0 +1,109 @@ +#!/bin/bash +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +set -euo pipefail + +# Run a wasm or aot app, capture the WAMR call stack, and symbolicate it +# using addr2line.py with the debug companion. +# +# Usage: ./symbolicate.sh [oob|stackoverflow] [wasm|aot] + +SCRIPT_DIR=$(cd "$(dirname "$0")" && pwd) +APP="${1:-oob}" +MODE="${2:-wasm}" + +if [ "$APP" != "oob" ] && [ "$APP" != "stackoverflow" ]; then + echo "Usage: $0 [oob|stackoverflow] [wasm|aot]" >&2 + exit 1 +fi +if [ "$MODE" != "wasm" ] && [ "$MODE" != "aot" ]; then + echo "Usage: $0 [oob|stackoverflow] [wasm|aot]" >&2 + exit 1 +fi + +WASI_SDK_PATH="${WASI_SDK_PATH:-/opt/wasi-sdk}" +WABT_PATH="${WABT_PATH:-/opt/wabt}" +WAMR_ROOT="${SCRIPT_DIR}/../.." + +BUILD_DIR="${SCRIPT_DIR}/build" +if [ "$MODE" = "wasm" ]; then + PROD_FILE="${BUILD_DIR}/wasm-apps/${APP}.prod.wasm" +else + PROD_FILE="${BUILD_DIR}/wasm-apps/${APP}.prod.aot" +fi +DEBUG_WASM="${BUILD_DIR}/wasm-apps/${APP}.debug.wasm" +IWASM="${BUILD_DIR}/iwasm" + +if [ ! -x "${IWASM}" ]; then + echo "iwasm not found at ${IWASM}" >&2 + echo "Run: mkdir -p build && cd build && cmake .. && make" >&2 + exit 1 +fi +if [ ! -f "${PROD_FILE}" ]; then + echo "Production binary not found at ${PROD_FILE}" >&2 + exit 1 +fi +if [ ! -f "${DEBUG_WASM}" ]; then + echo "Debug companion not found at ${DEBUG_WASM}" >&2 + exit 1 +fi + +CALL_STACK_FILE=$(mktemp) +LOG_FILE=$(mktemp) +trap 'rm -f "${CALL_STACK_FILE}" "${LOG_FILE}"' EXIT + +echo "=== Running iwasm on ${APP}.prod.${MODE} (expect crash) ===" +# -f app_main calls the exported app_main directly, bypassing wasi _start. +# This preserves the OOB / stack-overflow trap behavior — running _start +# under -Oz -flto would lower the OOB pattern to `unreachable` and produce +# misleading call-stack info. +# +# stackoverflow uses --stack-size=4096 so the WASM operand stack overflows +# at a reasonable depth (~20 frames). With the default stack size the +# recursion would still trap, just much later (~400 frames) and with a +# much larger captured call-stack to symbolicate. +IWASM_ARGS=() +if [ "$APP" = "stackoverflow" ]; then + IWASM_ARGS+=(--stack-size=4096) +fi +"${IWASM}" "${IWASM_ARGS[@]}" -f app_main "${PROD_FILE}" 2>&1 | tee "${LOG_FILE}" || true + +echo "" +echo "=== Captured call stack ===" +grep -E "^#[0-9]+:" "${LOG_FILE}" > "${CALL_STACK_FILE}" || true +cat "${CALL_STACK_FILE}" + +if [ ! -s "${CALL_STACK_FILE}" ]; then + echo "(no call stack captured)" + exit 1 +fi + +echo "" +echo "=== Symbolicated call stack (using debug companion) ===" +# Pick the right --mode for addr2line.py: +# - aot: offsets are file-absolute, no adjustment (wamrc commits ip +# at instruction start). Always use --mode=aot regardless of +# how iwasm itself was built. +# - wasm + classic interp: offsets are file-absolute, post-advance → --mode=interp +# - wasm + fast interp: offsets are function-relative in transformed +# bytecode → --mode=fast-interp (function-name lookup only) +# +# Detect fast-interp by inspecting CMakeCache.txt for WAMR_BUILD_FAST_INTERP=1. +if [ "$MODE" = "aot" ]; then + A2L_MODE=aot +else + A2L_MODE=interp + # Detect fast-interp by inspecting iwasm for the wasm_interp_fast.c symbol. + if strings "${IWASM}" 2>/dev/null | grep -q "wasm_interp_fast.c"; then + A2L_MODE=fast-interp + fi +fi + +# DWARF only lives in the .debug.wasm — addr2line.py uses it for all modes. +python3 "${WAMR_ROOT}/test-tools/addr2line/addr2line.py" \ + --wasi-sdk "${WASI_SDK_PATH}" \ + --wabt "${WABT_PATH}" \ + --wasm-file "${DEBUG_WASM}" \ + --mode "${A2L_MODE}" \ + "${CALL_STACK_FILE}" diff --git a/samples/debug-tools-optimized/verify.sh b/samples/debug-tools-optimized/verify.sh new file mode 100755 index 0000000000..e77b14cede --- /dev/null +++ b/samples/debug-tools-optimized/verify.sh @@ -0,0 +1,108 @@ +#!/bin/bash +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +# +# Verify symbolicated output for one (app, mode) combination. +# +# Usage: ./verify.sh +# +# Runs ./symbolicate.sh and asserts the output contains the expected source +# file references. Auto-detects whether iwasm was built with classic or fast +# interpreter (via the symbolicate.sh script itself) and relaxes the +# fast-interp + oob + wasm assertion since fast-interp can't resolve +# offset=0 (trap-at-entry) to a source line. + +set -euo pipefail + +SCRIPT_DIR=$(cd "$(dirname "$0")" && pwd) +APP="${1:-}" +MODE="${2:-}" + +if [ "$APP" != "oob" ] && [ "$APP" != "stackoverflow" ]; then + echo "Usage: $0 " >&2 + exit 2 +fi +if [ "$MODE" != "wasm" ] && [ "$MODE" != "aot" ]; then + echo "Usage: $0 " >&2 + exit 2 +fi + +# Detect interpreter mode by checking the iwasm binary (same logic as symbolicate.sh) +IWASM="${SCRIPT_DIR}/build/iwasm" +if [ ! -x "$IWASM" ]; then + echo "iwasm not found at $IWASM; build the sample first" >&2 + exit 1 +fi +INTERP="classic" +if strings "$IWASM" 2>/dev/null | grep -q "wasm_interp_fast.c"; then + INTERP="fast" +fi + +OUT=$(mktemp) +trap 'rm -f "$OUT"' EXIT + +"${SCRIPT_DIR}/symbolicate.sh" "$APP" "$MODE" 2>&1 | tee "$OUT" > /dev/null + +assert() { + local pattern="$1" + if ! grep -q "$pattern" "$OUT"; then + echo "FAIL [$INTERP/$APP/$MODE]: pattern '$pattern' not found in output" >&2 + cat "$OUT" >&2 + exit 1 + fi +} + +# assert_re: assert a POSIX extended regex matches somewhere in the captured output. +# Used for compound expectations like "function name X on its own line". +assert_re() { + local pattern="$1" + if ! grep -Eq "$pattern" "$OUT"; then + echo "FAIL [$INTERP/$APP/$MODE]: regex '$pattern' did not match output" >&2 + cat "$OUT" >&2 + exit 1 + fi +} + +case "$APP" in + oob) + # Runtime always reports the OOB exception type. + assert "out of bounds memory access" + # do_bad_access -> trigger_oob -> app_main are all inlined into a + # single WASM function under -Oz -flto. On classic-interp + wasm + # and aot builds, DWARF retains the inline chain, so addr2line.py + # emits all three names with "(inlined into )" annotations. + # On fast-interp + wasm the runtime offset doesn't map to source + # (transformed bytecode), so addr2line.py falls back to + # function-name lookup and emits only the outermost frame. + if [ "$INTERP" = "fast" ] && [ "$MODE" = "wasm" ]; then + assert_re '^0: app_main$' + else + assert_re '^0: do_bad_access \(inlined into trigger_oob\)$' + assert_re '^[[:space:]]+trigger_oob \(inlined into app_main\)$' + assert_re '^[[:space:]]+app_main$' + assert "oob_access.c" + assert "oob_main.c" + fi + ;; + stackoverflow) + # stackoverflow_recurse.c uses non-tail recursion that reliably + # exhausts the WASM operand stack (with --stack-size=4096 in + # symbolicate.sh, depth lands around 20 frames). + assert "wasm operand stack overflow" + # The captured call stack is recurse repeated, ending in app_main. + # addr2line.py must resolve both function names exactly. + assert_re '^[0-9]+: recurse$' + assert "stackoverflow_recurse.c" + if [ "$INTERP" = "fast" ] && [ "$MODE" = "wasm" ]; then + # fast-interp falls back to function-name lookup; the + # outermost app_main frame may not surface stackoverflow_main.c + # but the function name itself does. + assert_re '^[0-9]+: app_main$' + else + assert_re '^[0-9]+: app_main$' + assert "stackoverflow_main.c" + fi + ;; +esac + +echo "PASS [$INTERP/$APP/$MODE]" diff --git a/samples/debug-tools-optimized/wasm-apps/CMakeLists.txt b/samples/debug-tools-optimized/wasm-apps/CMakeLists.txt new file mode 100644 index 0000000000..b2a56b0f6d --- /dev/null +++ b/samples/debug-tools-optimized/wasm-apps/CMakeLists.txt @@ -0,0 +1,119 @@ +# Copyright (C) 2026 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +cmake_minimum_required(VERSION 3.14) +project(debug_tools_optimized_wasm_apps LANGUAGES NONE) + +set(WAMR_ROOT_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../../..") + +# --- wasi-sdk tool discovery --- +if(NOT DEFINED WASI_SDK_PATH) + if(DEFINED ENV{WASI_SDK_PATH}) + set(WASI_SDK_PATH "$ENV{WASI_SDK_PATH}") + else() + set(WASI_SDK_PATH "/opt/wasi-sdk") + endif() +endif() + +set(WASI_SDK_CLANG "${WASI_SDK_PATH}/bin/clang") +set(WASI_SDK_STRIP "${WASI_SDK_PATH}/bin/llvm-strip") + +if(NOT EXISTS "${WASI_SDK_CLANG}") + message(FATAL_ERROR "wasi-sdk clang not found at ${WASI_SDK_CLANG}.") +endif() +if(NOT EXISTS "${WASI_SDK_STRIP}") + message(FATAL_ERROR "llvm-strip not found at ${WASI_SDK_STRIP}.") +endif() + +# --- binaryen (wasm-opt) discovery --- +if(NOT DEFINED BINARYEN_PATH) + if(DEFINED ENV{BINARYEN_PATH}) + set(BINARYEN_PATH "$ENV{BINARYEN_PATH}") + else() + set(BINARYEN_PATH "/opt/binaryen") + endif() +endif() + +set(WASM_OPT "${BINARYEN_PATH}/bin/wasm-opt") +if(NOT EXISTS "${WASM_OPT}") + message(FATAL_ERROR "wasm-opt not found at ${WASM_OPT}. Set BINARYEN_PATH.") +endif() + +# --- wamrc discovery (for AOT) --- +list(APPEND CMAKE_MODULE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/../../cmake) +find_package(WAMRC REQUIRED) + +set(WASM_COMPILE_FLAGS + --target=wasm32-wasi + -Oz -g -flto + -Wl,--export=app_main +) + +# Build a wasm app from one or more source files. +# Usage: build_wasm_app( [ ...]) +function(build_wasm_app APP_NAME) + set(SOURCES ${ARGN}) + + set(SOURCE_PATHS "") + foreach(SRC ${SOURCES}) + list(APPEND SOURCE_PATHS "${CMAKE_CURRENT_SOURCE_DIR}/${SRC}") + endforeach() + + set(WASM_INTERMEDIATE "${CMAKE_CURRENT_BINARY_DIR}/${APP_NAME}.wasm") + set(WASM_DEBUG "${CMAKE_CURRENT_BINARY_DIR}/${APP_NAME}.debug.wasm") + set(WASM_PROD "${CMAKE_CURRENT_BINARY_DIR}/${APP_NAME}.prod.wasm") + set(AOT_PROD "${CMAKE_CURRENT_BINARY_DIR}/${APP_NAME}.prod.aot") + + # 1. Compile all sources together with -Oz -g -flto + add_custom_command( + OUTPUT "${WASM_INTERMEDIATE}" + COMMAND ${WASI_SDK_CLANG} ${WASM_COMPILE_FLAGS} + -o "${WASM_INTERMEDIATE}" + ${SOURCE_PATHS} + DEPENDS ${SOURCE_PATHS} + COMMENT "[${APP_NAME}] Compiling -> ${APP_NAME}.wasm (intermediate, -Oz -g -flto)" + ) + + # 2. Debug companion (preserves DWARF + name section) + add_custom_command( + OUTPUT "${WASM_DEBUG}" + COMMAND ${WASM_OPT} -Oz -g -o "${WASM_DEBUG}" "${WASM_INTERMEDIATE}" + DEPENDS "${WASM_INTERMEDIATE}" + COMMENT "[${APP_NAME}] wasm-opt -Oz -g -> ${APP_NAME}.debug.wasm (companion)" + ) + + # 3. Production (strip debug companion to guarantee identical code) + add_custom_command( + OUTPUT "${WASM_PROD}" + COMMAND ${WASI_SDK_STRIP} --strip-all -o "${WASM_PROD}" "${WASM_DEBUG}" + DEPENDS "${WASM_DEBUG}" + COMMENT "[${APP_NAME}] strip -> ${APP_NAME}.prod.wasm (production)" + ) + + # 4. AOT (compiled from production wasm with --enable-dump-call-stack) + # The .debug.wasm companion is still used for symbolication — DWARF only + # lives in the wasm; AOT carries the call-stack metadata only. + # --bounds-checks=1 forces explicit memory bounds checks instead of + # relying on hardware traps; this matches the iwasm build (which + # disables HW bound checks so OOB traps go through the interpreter + # exception path and capture frame_ip — without it, the AOT path + # would SIGSEGV with no captured call stack and inline DWARF info + # would be unrecoverable on the OOB sample. + add_custom_command( + OUTPUT "${AOT_PROD}" + DEPENDS ${WAMRC_BIN} "${WASM_PROD}" + COMMAND ${WAMRC_BIN} --size-level=0 --enable-dump-call-stack + --bounds-checks=1 + -o "${AOT_PROD}" "${WASM_PROD}" + COMMENT "[${APP_NAME}] wamrc -> ${APP_NAME}.prod.aot (AOT)" + ) + + add_custom_target(${APP_NAME}_wasm ALL + DEPENDS "${WASM_INTERMEDIATE}" "${WASM_DEBUG}" "${WASM_PROD}" "${AOT_PROD}" + ) + + install(FILES "${WASM_DEBUG}" "${WASM_PROD}" "${AOT_PROD}" DESTINATION wasm-apps) +endfunction() + +build_wasm_app(oob oob_main.c oob_access.c) +build_wasm_app(stackoverflow stackoverflow_main.c stackoverflow_recurse.c) diff --git a/samples/debug-tools-optimized/wasm-apps/oob_access.c b/samples/debug-tools-optimized/wasm-apps/oob_access.c new file mode 100644 index 0000000000..ccb08a2943 --- /dev/null +++ b/samples/debug-tools-optimized/wasm-apps/oob_access.c @@ -0,0 +1,12 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +void +do_bad_access(int offset) +{ + volatile int *p = (volatile int *)0; + /* Write to an address way beyond linear memory to trigger OOB trap */ + p[offset] = 0xDEAD; +} diff --git a/samples/debug-tools-optimized/wasm-apps/oob_main.c b/samples/debug-tools-optimized/wasm-apps/oob_main.c new file mode 100644 index 0000000000..30a8b5e948 --- /dev/null +++ b/samples/debug-tools-optimized/wasm-apps/oob_main.c @@ -0,0 +1,26 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +/* Deliberate out-of-bounds memory access for coredump debug demo. + Multi-file: app_main -> trigger_oob (this file) -> do_bad_access + (oob_access.c) Under -Oz, these get inlined into app_main. The debug + companion retains DWARF inline info so the full chain can be recovered + offline. */ + +void +do_bad_access(int offset); + +void +trigger_oob(void) +{ + /* 0x7FFFFFFF is well beyond any WASM linear memory */ + do_bad_access(0x7FFFFFFF); +} + +void +app_main(void) +{ + trigger_oob(); +} diff --git a/samples/debug-tools-optimized/wasm-apps/stackoverflow_main.c b/samples/debug-tools-optimized/wasm-apps/stackoverflow_main.c new file mode 100644 index 0000000000..0e57a2e640 --- /dev/null +++ b/samples/debug-tools-optimized/wasm-apps/stackoverflow_main.c @@ -0,0 +1,16 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +/* Recursive stack overflow for coredump debug demo. + Multi-file: app_main (this file) -> recurse (stackoverflow_recurse.c) */ + +int +recurse(int depth); + +void +app_main(void) +{ + recurse(0); +} diff --git a/samples/debug-tools-optimized/wasm-apps/stackoverflow_recurse.c b/samples/debug-tools-optimized/wasm-apps/stackoverflow_recurse.c new file mode 100644 index 0000000000..0cbf7a37bf --- /dev/null +++ b/samples/debug-tools-optimized/wasm-apps/stackoverflow_recurse.c @@ -0,0 +1,25 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +static volatile int sink; + +int +recurse(int depth) +{ + /* Allocate stack space each frame to accelerate overflow */ + volatile char buf[128]; + buf[0] = (char)depth; + buf[127] = (char)(depth >> 8); + sink = buf[0] + buf[127]; + if (depth == 10000) { + __builtin_trap(); + } + /* TODO: Use return value after the call to prevent tail-call optimization, + * it will stack overflow */ + int r = recurse(depth + 1); + return r + buf[0]; + /* TODO: will optimize to loop, and it will trap when reach certain level */ + // return recurse(depth + 1); +} diff --git a/samples/debug-tools/README.md b/samples/debug-tools/README.md index b0358b9e4f..7ed69b0882 100644 --- a/samples/debug-tools/README.md +++ b/samples/debug-tools/README.md @@ -52,25 +52,104 @@ $ python3 ../../../test-tools/addr2line/addr2line.py \ call_stack.txt ``` +This sample also ships `symbolicate.sh` (runs the three modes shown below in +sequence) and `verify.sh` (asserts inline expansion appears in both the +classic-interp and AOT outputs — used by CI, useful locally as a smoke test). +Both expect to be run from the sample root after building: + +```bash +$ ./symbolicate.sh +$ ./verify.sh +``` + The output should be something like: ```text -0: c - at wasm-micro-runtime/samples/debug-tools/wasm-apps/trap.c:5:1 +0: trap_helper (inlined into c) + at /path/to/wasm-micro-runtime/samples/debug-tools/wasm-apps/trap.c:9:5 + c + at /path/to/wasm-micro-runtime/samples/debug-tools/wasm-apps/trap.c:16:12 1: b - at wasm-micro-runtime/samples/debug-tools/wasm-apps/trap.c:11:12 + at /path/to/wasm-micro-runtime/samples/debug-tools/wasm-apps/trap.c:23:12 2: a - at wasm-micro-runtime/samples/debug-tools/wasm-apps/trap.c:17:12 + at /path/to/wasm-micro-runtime/samples/debug-tools/wasm-apps/trap.c:29:12 3: main - at wasm-micro-runtime/samples/debug-tools/wasm-apps/trap.c:24:5 + at /path/to/wasm-micro-runtime/samples/debug-tools/wasm-apps/trap.c:36:5 4: __main_void - at unknown:?:? + at ??:0 5: _start + at wasisdk://v25.0/build/sysroot/wasi-libc-wasm32-wasi/libc-bottom-half/crt/crt1-command.c:43:13 ``` -If WAMR is run in fast interpreter mode (`WAMR_BUILD_FAST_INTERP=1`), addresses in the stack trace cannot be tracked back to location info. -If WAMR <= `1.3.2` is used, the stack trace does not contain addresses. -In those two cases, run the script with `--no-addr`: the line info returned refers to the start of the function +Frame `0` shows **inline expansion** in action — `trap_helper` (the actual trap site) +and `c` (its caller) appear together under index `0` because `trap_helper` was inlined +into `c` by `__attribute__((always_inline))`. The `(inlined into c)` suffix on +`trap_helper` makes the relationship explicit: the runtime saw one WASM frame for `c`, +but addr2line.py reconstructs the full source-level call chain from DWARF +`DW_TAG_inlined_subroutine` entries. Each frame in the chain except the outermost +gets the `(inlined into )` annotation. + +### Inline expansion + +addr2line.py automatically expands inline call chains when the binary's DWARF contains +`DW_TAG_inlined_subroutine` entries. This happens when functions are inlined either by +optimization (e.g. `-O2`, `-Oz`) or by `__attribute__((always_inline))`. + +The included `trap.c` marks `trap_helper` as always_inline and places `__builtin_trap()` +inside it. The trap address therefore falls within the inlined region, producing the +multi-line frame `0` shown above. Each inlined frame's source location is reported +independently, so you can see exactly which inlined call chain led to the trap. + +Inline expansion requires no flags — it's automatic whenever the DWARF carries the +relevant inline metadata. + +For details on the tools `addr2line.py` orchestrates internally and why each +is needed, see [`test-tools/addr2line/README.md`](../../test-tools/addr2line/README.md). +The longer-form discussion of the interval-table overlay — applied to correct +the LLVM symbolizer's outermost function-name lookup on wasm — also lives there. + +### Execution modes — `--mode={interp,aot,fast-interp}` + +Different WAMR execution modes report call-stack offsets in different address spaces. +Pass the right `--mode` to addr2line.py so offsets are interpreted correctly: + +| Mode | When | Adjustment | Source resolution | +|------|------|-----------|-------------------| +| `interp` (default) | Classic interpreter (`WAMR_BUILD_FAST_INTERP=0`) | `offset - code_start - 1` (post-advance) | Full file:line, inline expansion | +| `aot` | AOT-compiled .aot files | `offset - code_start` (wamrc commits at instruction start) | Full file:line, inline expansion | +| `fast-interp` | Fast interpreter (`WAMR_BUILD_FAST_INTERP=1`) | (not mappable) | Function-name only | + +For AOT call stacks: + +```bash +$ python3 ../../../test-tools/addr2line/addr2line.py \ + --wasi-sdk /opt/wasi-sdk \ + --wabt /opt/wabt \ + --wasm-file wasm-apps/trap.wasm \ + --mode aot \ + call_stack_aot.txt +``` + +For fast-interp call stacks: + +```bash +$ python3 ../../../test-tools/addr2line/addr2line.py \ + --wasi-sdk /opt/wasi-sdk \ + --wabt /opt/wabt \ + --wasm-file wasm-apps/trap.wasm \ + --mode fast-interp \ + call_stack.txt +``` + +Fast-interp transforms the WASM bytecode in-memory at load time (replaces opcodes +with handler indices, expands LEBs to fixed widths). The runtime ip therefore +points into the *transformed* buffer — there's no way to map an offset back to a +source line. `--mode=fast-interp` falls back to function-name lookup (equivalent +to `--no-addr`). + +If WAMR <= `1.3.2` is used, the stack trace does not contain addresses at all. +In that case, run the script with `--no-addr`: the line info returned refers to +the start of the function. ```bash $ python3 ../../../test-tools/addr2line/addr2line.py \ diff --git a/samples/debug-tools/symbolicate.sh b/samples/debug-tools/symbolicate.sh old mode 100644 new mode 100755 index 709622f03d..e5a43e4c7e --- a/samples/debug-tools/symbolicate.sh +++ b/samples/debug-tools/symbolicate.sh @@ -1,23 +1,52 @@ #!/bin/bash -set -euox pipefail - -# Symbolicate .wasm -python3 ../../../test-tools/addr2line/addr2line.py \ - --wasi-sdk /opt/wasi-sdk \ - --wabt /opt/wabt \ - --wasm-file wasm-apps/trap.wasm \ - call_stack.txt - -# Symbolicate .wasm with `--no-addr` -python3 ../../../test-tools/addr2line/addr2line.py \ - --wasi-sdk /opt/wasi-sdk \ - --wabt /opt/wabt \ - --wasm-file wasm-apps/trap.wasm \ - call_stack.txt --no-addr - -# Symbolicate .aot -python3 ../../../test-tools/addr2line/addr2line.py \ - --wasi-sdk /opt/wasi-sdk \ - --wabt /opt/wabt \ - --wasm-file wasm-apps/trap.wasm \ - call_stack_aot.txt \ No newline at end of file +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +set -euo pipefail + +# Symbolicate the captured call stack files (call_stack.txt for .wasm, +# call_stack_aot.txt for .aot) located in this sample's build/ directory. +# Self-locating so invocation works from any cwd. + +SCRIPT_DIR=$(cd "$(dirname "$0")" && pwd) +BUILD_DIR="${SCRIPT_DIR}/build" +ADDR2LINE="${SCRIPT_DIR}/../../test-tools/addr2line/addr2line.py" +WASM_FILE="${BUILD_DIR}/wasm-apps/trap.wasm" +CALL_STACK="${BUILD_DIR}/call_stack.txt" +CALL_STACK_AOT="${BUILD_DIR}/call_stack_aot.txt" + +WASI_SDK_PATH="${WASI_SDK_PATH:-/opt/wasi-sdk}" +WABT_PATH="${WABT_PATH:-/opt/wabt}" + +banner() { + echo + echo "===== $* =====" +} + +# .wasm + classic-interp: addr2line.py defaults to --mode=interp. +banner "wasm / --mode=interp (default; classic interpreter, post-advance offsets)" +python3 "$ADDR2LINE" \ + --wasi-sdk "$WASI_SDK_PATH" \ + --wabt "$WABT_PATH" \ + --wasm-file "$WASM_FILE" \ + "$CALL_STACK" + +# Same call stack with --no-addr: function-level resolution only — use +# this for fast-interp call stacks (transformed bytecode has no source +# mapping) and for very old iwasm dumps that don't carry offsets. +banner "wasm / --no-addr (function-name lookup only; fast-interp & WAMR<=1.3.2 fallback)" +python3 "$ADDR2LINE" \ + --wasi-sdk "$WASI_SDK_PATH" \ + --wabt "$WABT_PATH" \ + --wasm-file "$WASM_FILE" \ + --no-addr \ + "$CALL_STACK" + +# .aot: --mode=aot — wamrc commits ip at instruction-start (not post-advance). +banner "aot / --mode=aot (instruction-start offsets)" +python3 "$ADDR2LINE" \ + --wasi-sdk "$WASI_SDK_PATH" \ + --wabt "$WABT_PATH" \ + --wasm-file "$WASM_FILE" \ + --mode aot \ + "$CALL_STACK_AOT" diff --git a/samples/debug-tools/verify.sh b/samples/debug-tools/verify.sh new file mode 100755 index 0000000000..59a148f70e --- /dev/null +++ b/samples/debug-tools/verify.sh @@ -0,0 +1,83 @@ +#!/bin/bash +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +# +# Verify symbolicated output for samples/debug-tools. +# +# Usage: ./verify.sh +# +# Captures call stacks for both .wasm and .aot, runs symbolicate.sh, and +# asserts that the inline expansion of trap_helper (always_inline) shows +# up under both --mode=interp (the default for the .wasm path) and +# --mode=aot (the third invocation in symbolicate.sh). + +set -euo pipefail + +SCRIPT_DIR=$(cd "$(dirname "$0")" && pwd) +BUILD_DIR="${SCRIPT_DIR}/build" + +if [ ! -x "${BUILD_DIR}/iwasm" ]; then + echo "iwasm not found at ${BUILD_DIR}/iwasm; build the sample first" >&2 + exit 1 +fi + +cd "${BUILD_DIR}" +./iwasm wasm-apps/trap.wasm 2>&1 | grep "^#" > call_stack.txt || true +./iwasm wasm-apps/trap.aot 2>&1 | grep "^#" > call_stack_aot.txt || true + +OUT=$(mktemp) +trap 'rm -f "$OUT"' EXIT +"${SCRIPT_DIR}/symbolicate.sh" 2>&1 | tee "$OUT" > /dev/null + +assert() { + local pattern="$1" + local where="$2" + if ! grep -q "$pattern" "$OUT"; then + echo "FAIL: pattern '$pattern' ($where) not found in symbolicate.sh output" >&2 + cat "$OUT" >&2 + exit 1 + fi +} + +# assert_re: like assert but with POSIX extended regex (anchored line match). +assert_re() { + local pattern="$1" + local where="$2" + if ! grep -Eq "$pattern" "$OUT"; then + echo "FAIL: regex '$pattern' ($where) did not match symbolicate.sh output" >&2 + cat "$OUT" >&2 + exit 1 + fi +} + +# trap.c's runtime call chain is _start -> ... -> main -> a -> b -> c +# (with c calling __builtin_trap inside trap_helper, an always_inline +# helper). Symbolicate.sh runs three invocations against the same call +# stack. We assert the precise frames each one produces. +# +# All invocations should symbolicate the user-code frames `c`, `b`, `a`, +# `main` (each on its own line, prefixed by the frame index). +assert_re '^[0-9]+: c$' "user frame: c" +assert_re '^[0-9]+: b$' "user frame: b" +assert_re '^[0-9]+: a$' "user frame: a" +assert_re '^[0-9]+: main$' "user frame: main" +assert "trap.c" "source file" + +# Inline expansion of trap_helper (always_inline) only surfaces in the +# address-based runs (default --mode=interp and --mode=aot). It MUST +# appear at least once in the captured output, with the +# "(inlined into c)" annotation that print_frames adds for inlined +# frames. +assert_re '^[[:space:]]*0:[[:space:]]+trap_helper[[:space:]]+\(inlined into c\)$' \ + "inline expansion: trap_helper (inlined into c)" + +# The AOT block is the third (last) invocation in symbolicate.sh. +# Confirm --mode=aot also produces inline expansion (the outermost +# frame at index 0 should still be trap_helper inlined into c). +if ! tail -25 "$OUT" | grep -Eq '^[[:space:]]*0:[[:space:]]+trap_helper[[:space:]]+\(inlined into c\)$'; then + echo "FAIL: AOT block (last 25 lines) missing 'trap_helper (inlined into c)' — --mode=aot may be broken" >&2 + cat "$OUT" >&2 + exit 1 +fi + +echo "PASS [debug-tools symbolication: precise frames + inline expansion verified for both wasm and aot]" diff --git a/samples/debug-tools/wasm-apps/CMakeLists.txt b/samples/debug-tools/wasm-apps/CMakeLists.txt index 9858b98dab..82ea845c1b 100644 --- a/samples/debug-tools/wasm-apps/CMakeLists.txt +++ b/samples/debug-tools/wasm-apps/CMakeLists.txt @@ -34,6 +34,9 @@ function (compile_sample SOURCE_FILE) COMMAND ${WAMRC_BIN} --size-level=0 --enable-dump-call-stack -o ${AOT_FILE} ${WASM_FILE} WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} ) + # Force the .wasm producer to run before the AOT step under `make -j`; + # the DEPENDS line above only names the file, not the CMake target. + add_dependencies(${FILE_NAME}_aot ${FILE_NAME}) ## wasm + sourcemap if (DEFINED EMSCRIPTEN) diff --git a/samples/debug-tools/wasm-apps/trap.c b/samples/debug-tools/wasm-apps/trap.c index 364c430d19..27a5d0245b 100644 --- a/samples/debug-tools/wasm-apps/trap.c +++ b/samples/debug-tools/wasm-apps/trap.c @@ -1,7 +1,18 @@ +__attribute__((always_inline)) static inline int +trap_helper(int n) +{ + /* Forced-inline helper to demonstrate inline call stack expansion. + The trap is inside this helper so the runtime offset falls within + the inlined region — addr2line.py will then show both trap_helper + and c() in the call chain (innermost first). */ + __builtin_trap(); + return n + 100; +} + int c(int n) { - __builtin_trap(); + return trap_helper(n); } int diff --git a/test-tools/addr2line/README.md b/test-tools/addr2line/README.md new file mode 100644 index 0000000000..0254912810 --- /dev/null +++ b/test-tools/addr2line/README.md @@ -0,0 +1,168 @@ +# `addr2line.py` — symbolicating WAMR call stacks + +`addr2line.py` turns a WAMR call-stack dump (`#00: 0x0040 - $f0` lines) back +into source-level information (function names, files, line numbers, inline +chains). It is a thin orchestrator over four LLVM/wabt tools, with workarounds +for two LLVM symbolization issues on wasm targets (one a design choice, one an +invariant violation from `wasm-opt -g`). + +The two `samples/debug-tools*/` samples are end-to-end demos of the workflow. +This README focuses on the tool itself: how it works, why each subprocess is +necessary, and why the interval-table overlay is required. + +## Quick reference + +```bash +python3 addr2line.py \ + --wasi-sdk /opt/wasi-sdk \ + --wabt /opt/wabt \ + --wasm-file /path/to/app.debug.wasm \ + call_stack.txt +``` + +| Flag | Default | Purpose | +|------|---------|---------| +| `--mode={interp,aot,fast-interp}` | `interp` | How to interpret runtime offsets — see *Three offset spaces* below. | +| `--no-addr` | off | Resolve by function name only (for very old iwasm dumps that lack offsets, or for fast-interp). | + +## What the script does, step by step + +1. **Parse each captured frame** (`#NN: 0xADDR - SYMBOL`). +2. **Convert the runtime offset to a DWARF address.** WAMR reports + file-absolute byte offsets; DWARF addresses are relative to the start of + the WASM Code section, so the script subtracts `code_section_start` (read + from `wasm-objdump -h`). +3. **Apply a mode-dependent adjustment.** See *Three offset spaces* below. +4. **Resolve the address.** `llvm-symbolizer -f -i` for line info and inline + chain, with the outermost frame's function name overlaid from a startup- + built interval table (see "Why the interval-table overlay exists" below). +5. **Demangle and render.** C++ symbols pass through `llvm-cxxfilt`; inline + frames are annotated `(inlined into )` so the chain reads top-down. + +## Three offset spaces (`--mode`) + +WAMR captures `frame_ip` differently in each execution mode, so the offset in +a captured frame means different things: + +| Mode | Offset space | Adjustment | What addr2line does | +|------|--------------|-----------|----------------------| +| `interp` (default) | File-absolute, *post*-advance — interpreter has already incremented `frame_ip` past the trapping opcode (and past LEB reads inside the handler) before the trap fires. | `offset - code_start - 1` | Full symbolication: file, line, column, inline chain. | +| `aot` | File-absolute, *instruction-start* — `wamrc` emits a `commit_ip` before every WASM operation. | `offset - code_start` | Full symbolication. | +| `fast-interp` | Function-relative, into the *transformed* in-memory bytecode (opcodes replaced with handler indices, LEBs expanded to fixed widths). The mapping back to source is destroyed by the transform. | (none) | Function-name lookup only — equivalent to `--no-addr`. One frame per input; no inline expansion (the `DW_TAG_inlined_subroutine` overlay is address-keyed and unavailable here). | + +Picking the wrong `--mode` lands the resolver inside an adjacent function and +silently produces wrong file/line. + +## Why the interval-table overlay exists + +`llvm-symbolizer -e -f -i ` is the natural way to map an address +to a function + source location, with `-i` expanding inline frames. It's +correct for line/column info and for inline frame names — but its +**outermost frame's function name** goes through two paths that can both +produce wrong answers on wasm: + +1. **Symbol-table override on the outermost frame.** `symbolizeInlinedCode` + unconditionally rewrites the outermost frame's `FunctionName` from + `getNameFromSymbolTable` whenever `FunctionNameKind::LinkageName` and + `UseSymbolTable` are both set — which is the default. Inline frames are + provably NOT rewritten (the override targets only + `getMutableFrame(getNumberOfFrames() - 1)`). On some wasi-sdk / wasm + layouts the symbol table's lookup returns the wrong function for an + address that DWARF places correctly. This is an intentional LLVM design + choice, not a bug — it exists because `-gline-tables-only` builds have + only line tables in DWARF and the symbol table gives better linkage + names in that case. The tradeoff bites on full-DWARF wasm builds. +2. **`AddrDieMap` invariant violation from `wasm-opt` DCE ghosts.** + `wasm-opt -Oz -g` leaves DIEs for dead-code-eliminated compiler-rt / + libm helpers behind with `low_pc = 0` and small `high_pc` (typically + `[0, 0)` or `[0, 6)`, sometimes up to `[0, 0x80)`). This is a + consequence of binaryen's DWARF preservation model: its address- + rewrite pass in `src/wasm/wasm-debug.cpp`'s + `LocationUpdater::getNewFuncStart` returns `0` as a tombstone when + an old address doesn't map to any surviving IR node, and `updateDIE` + writes that `0` into the `DW_AT_low_pc` — but leaves the + `DW_TAG_subprogram` in place, and sometimes leaves the original + `DW_AT_high_pc`. Binaryen's README calls this out: DWARF support is + optional, tracks address relocation rather than DIE-tree + restructuring, and is "not suitable for a fully optimized release + build." LLVM's `updateAddressDieMap` in `DWARFUnit.cpp` assumes + children are inserted after parents and each child range ⊆ + parent range; the ghost DIEs violate that invariant and the map's + `upper_bound − 1` lookup returns an insertion-order-dependent DIE. + +Empirical truth table (probing an inlined address of `trigger_oob` in a +small test module across all four toolchain combinations): + +| wasi-sdk | wasm-opt -Oz -g | symbolizer outer | mechanism | +|---|---|---|---| +| 29 (clang 21) | no | `free` ❌ | Symbol-table override | +| 29 (clang 21) | yes | `free` ❌ | Symbol-table override | +| 33 (clang 22) | no | `app_main` ✅ | Neither mechanism triggers | +| 33 (clang 22) | yes | `fmaf` ❌ | AddrDieMap invariant violated by a `[low_pc=0, 0x72)` `fmaf` ghost | + +The overlay fixes all four cases. Inner (inlined) frames are not affected +because they come from a different LLVM code path +(`getInliningInfoForAddress`), which walks `getParent()` links from the +leaf DIE and does NOT go through the symbol-table override applied to +the outermost concrete frame. + +### How the overlay works + +At startup, addr2line.py runs one `llvm-dwarfdump --debug-info` pass and +builds an in-memory list `[(low_pc, high_pc, name), ...]` covering every +real callable `DW_TAG_subprogram`. Three skip guards keep the list clean: + +- `DW_AT_declaration=true` → forward declarations (e.g. WASI imports like + `__imported_wasi_snapshot_preview1_*`) have no code, so they're excluded. +- `low_pc == 0` → structurally impossible for real code, since the wasm + binary spec guarantees the Code section begins after at least a Type + + Function section header. Any `low_pc = 0` DIE is a wasm-opt DCE ghost + or a broken producer's output. +- `high_pc <= low_pc` → empty or degenerate range. Defensive; never + observed on wasi-sdk 29 or 33. + +For each captured call-stack address, `lookup_subprogram_name` returns +the innermost (smallest range) surviving DW_TAG_subprogram covering the +address. That result overwrites the outermost frame's function name in +the symbolizer output; inner frame names are left alone. + +## Tools used by addr2line.py + +| Tool | From | Why this tool, and not another | +|------|------|--------------------------------| +| `llvm-symbolizer` | wasi-sdk | The actual address → `(function, file:line:column)` resolver, with inline expansion via `-i`. Ships in wasi-sdk 29+ (the project's minimum supported version). | +| `llvm-dwarfdump` | wasi-sdk | Single `--debug-info` pass at startup builds a `(low_pc, high_pc, name)` interval table for every real callable `DW_TAG_subprogram`. Used to overlay the outermost frame's function name in every resolution. The overlay is necessary because `llvm-symbolizer -f` unconditionally rewrites that frame's `FunctionName` from an object-file symbol-table lookup (LLVM design, not a bug) and — separately — `wasm-opt -Oz -g` output can violate the `AddrDieMap` invariant so DWARF's own DIE lookup returns the wrong entry. Reading DWARF ranges directly bypasses both mechanisms. No equivalent CLI exposes this lookup as structured output, so we parse the text. | +| `wasm-objdump` | wabt | Reports the Code-section start offset (needed to convert file-absolute runtime offsets to DWARF addresses) and the function-index → name table (a last-resort fallback for declaration-only DW_TAG_subprograms like `__main_void`). | +| `llvm-cxxfilt` | wasi-sdk | C++ symbol demangling, applied as a final pass on resolved names. | + +The script also detects an emscripten-style sourceMappingURL section and +falls back to `emsymbolizer` for that case. + +## Behavior matrix + +| Input | Resolver path | Output | +|-------|---------------|--------| +| `--mode=interp` or `--mode=aot`, address in DWARF range | Unified | `llvm-symbolizer -f -i` for line info + inline chain; outermost frame's function name overlaid from a startup-built interval table (innermost match wins) | +| `--mode=fast-interp` | Function-name | Single frame per input, no inline expansion, no source mapping | +| `--no-addr` | Function-name | Resolves the *function* — file:line refers to its declaration, not the trap site | +| Offset = 0 | Function-name fallback (with `(offset=0 — function entry)` note) | Used when WAMR couldn't capture `frame_ip` (e.g., trap at function entry, or top frame of an `iwasm -f` invocation) | +| Address outside any DW_TAG_subprogram | wasm-objdump function-index table | Single name; useful for declaration-only entries | +| Source has emscripten `sourceMappingURL` custom section | `emsymbolizer` source-map path | Single frame per address; no inline expansion (source maps don't carry it) | + +## Tests + +Run the pytest suite under `test-tools/addr2line/tests/`. It builds tiny +purpose-built C/C++ apps, captures their trap sites, and asserts the resolver +behaves correctly under a number of scenarios (single trap, always_inline, +deep inline chain, cross-TU LTO inlining, mid-function trap, multi-frame +call stack, C++ demangling, AOT mode, fast-interp mode, `--no-addr`, +`offset=0`, empty input). Multi-SDK runs (`pytest --multi-sdk`) exercise +the overlay across multiple toolchain versions and require multiple wasi-sdk +installations under `/opt`. + +## See also + +- `test-tools/addr2line/addr2line.py` — the script +- `test-tools/addr2line/tests/` — pytest harness +- `samples/debug-tools/` — minimal end-to-end sample +- `samples/debug-tools-optimized/` — production-realistic workflow (LTO, wasm-opt, strip, AOT, debug companion) diff --git a/test-tools/addr2line/addr2line.py b/test-tools/addr2line/addr2line.py index 800ba37504..810609096c 100644 --- a/test-tools/addr2line/addr2line.py +++ b/test-tools/addr2line/addr2line.py @@ -14,7 +14,7 @@ """ This is a tool to convert addresses, which are from a call-stack dump generated by iwasm, into line info for a wasm file. -When a wasm file is compiled with debug info, it is possible to transfer the address to line info. +When a wasm file is compiled with debug info (-g), it is possible to transfer the address to line info. For example, there is a call-stack dump: @@ -26,27 +26,47 @@ ``` - store the call-stack dump into a file, e.g. call_stack.txt -- run the following command to convert the address into line info: +- run the following command: ``` - $ cd test-tools/addr2line - $ python3 addr2line.py --wasi-sdk --wabt --wasm-file call_stack.txt + $ python3 addr2line.py --wasi-sdk --wabt --wasm-file call_stack.txt ``` - The script will use *wasm-objdump* in wabt to transform address, then use *llvm-dwarfdump* to lookup the line info for each address - in the call-stack dump. -- if addresses are not available in the stack trace (i.e. iwasm <= 1.3.2) or iwasm is used in fast interpreter mode, - run the following command to convert the function index into line info (passing the `--no-addr` option): - ``` - $ python3 addr2line.py --wasi-sdk --wabt --wasm-file call_stack.txt --no-addr - ``` - The script will use *wasm-objdump* in wabt to get the function names corresponding to function indexes, then use *llvm-dwarfdump* to lookup the line info for each - function index in the call-stack dump. + +The script handles two workflows: +- Same-binary: pass the wasm file that ran (must have DWARF debug info) +- Companion: pass a sibling debug companion (.debug.wasm produced by `wasm-opt -Oz -g`). + The production binary that ran can be stripped — the companion has identical code + layout but retains DWARF, so the runtime offsets map correctly. + +Inline expansion (DW_TAG_inlined_subroutine) is automatic — no flag needed. + +By default, offsets are interpreted as coming from classic interpreter mode +(frame_ip has advanced past the trapping opcode). For AOT call stacks, pass +`--mode=aot` so offsets are used verbatim — wamrc commits ip at the start of +each WASM operation, not after. + +For fast-interp call stacks, pass `--mode=fast-interp`. Fast-interp transforms +the bytecode in-memory at load time, so its offsets are meaningless for source +mapping; the script falls back to function-name lookup (same as `--no-addr`). + +If addresses are not available (iwasm <= 1.3.2), pass `--no-addr` to look up +by function index/name only. + +- addr2line.py always applies an interval-table overlay to the outermost + frame's function name. `llvm-symbolizer` unconditionally rewrites that + frame's name from an object-file symbol-table lookup (see + `symbolizeInlinedCode` in + `llvm/lib/DebugInfo/Symbolize/SymbolizableObjectFile.cpp`), and the + symbol-table can return wrong names on wasm. Separately, + `wasm-opt -Oz -g` leaves dead-code-eliminated `DW_TAG_subprogram` DIEs + with `low_pc = 0` behind, violating LLVM's `AddrDieMap` parents-before- + children invariant. The overlay bypasses both by reading DWARF ranges + directly. Inner inlined frames come from `getInliningInfoForAddress` + (a different code path) and are not affected by either mechanism. """ def locate_sourceMappingURL_section(wasm_objdump: Path, wasm_file: Path) -> bool: - """ - Figure out if the wasm file has a sourceMappingURL section. - """ + """Figure out if the wasm file has a sourceMappingURL section.""" cmd = f"{wasm_objdump} -h {wasm_file}" p = subprocess.run( shlex.split(cmd), @@ -55,24 +75,18 @@ def locate_sourceMappingURL_section(wasm_objdump: Path, wasm_file: Path) -> bool text=True, universal_newlines=True, ) - outputs = p.stdout.split(os.linesep) - - for line in outputs: - line = line.strip() - if "sourceMappingURL" in line: + for line in p.stdout.splitlines(): + if "sourceMappingURL" in line.strip(): return True - return False def get_code_section_start(wasm_objdump: Path, wasm_file: Path) -> int: """ - Find the start offset of Code section in a wasm file. + Find the start file offset of the Code section in a wasm file. - if the code section header likes: + Code section header looks like: Code start=0x0000017c end=0x00004382 (size=0x00004206) count: 47 - - the start offset is 0x0000017c """ cmd = f"{wasm_objdump} -h {wasm_file}" p = subprocess.run( @@ -82,155 +96,228 @@ def get_code_section_start(wasm_objdump: Path, wasm_file: Path) -> int: text=True, universal_newlines=True, ) - outputs = p.stdout.split(os.linesep) - - for line in outputs: + for line in p.stdout.splitlines(): line = line.strip() - if "Code" in line: - return int(line.split()[1].split("=")[1], 16) - + if "Code" in line and "start=" in line: + m = re.search(r"start=(0x[0-9a-fA-F]+)", line) + if m: + return int(m.group(1), 16) return -1 -def get_line_info_from_function_addr_dwarf( - dwarf_dump: Path, wasm_file: Path, offset: int -) -> tuple[str, str, str, str]: +def build_subprogram_intervals(dwarf_dump: Path, wasm_file: Path) -> list: """ - Find the location info of a given offset in a wasm file. + Return a list of (low_pc, high_pc, name) for every real callable + DW_TAG_subprogram in the wasm's DWARF. + + Filtered out during parse: + - DIEs with DW_AT_declaration=true (forward declarations; no code). + - DIEs with low_pc == 0. DWARF low_pc is Code-section-relative, and + the section's first byte is the function-count LEB, not any real + function body — so any low_pc=0 DIE is either a declaration or a + dead-code ghost left by wasm-opt (see the "binaryen tombstone" + note in the design doc). + - DIEs with high_pc <= low_pc (empty or degenerate ranges). + + The resulting list is used to overlay the outermost frame's function + name — llvm-symbolizer picks the wrong DW_TAG_subprogram at addresses + where multiple DIEs claim to cover the same PC. See the design doc at + docs/superpowers/specs/2026-07-06-addr2line-collapse-modern-legacy-design.md + for the two independent bug sources. + + Parsed once per run; lookups against the result are O(N) in the + subprogram count, which is small. """ - cmd = f"{dwarf_dump} --lookup={offset} {wasm_file}" - p = subprocess.run( + cmd = f"{dwarf_dump} --debug-info {wasm_file}" + raw = subprocess.run( shlex.split(cmd), check=False, capture_output=True, text=True, universal_newlines=True, - ) - outputs = p.stdout.split(os.linesep) - - function_name, function_file = "", "unknown" - function_line, function_column = "?", "?" - - for line in outputs: - line = line.strip() - - if "DW_AT_name" in line: - function_name = get_dwarf_tag_value("DW_AT_name", line) - - if "DW_AT_decl_file" in line: - function_file = get_dwarf_tag_value("DW_AT_decl_file", line) - - if "Line info" in line: - _, function_line, function_column = parse_line_info(line) - - return (function_name, function_file, function_line, function_column) + ).stdout + + intervals = [] + sp_indent = -1 + low = high = name = None + is_declaration = False + + def _flush(): + # Skip guards: declaration-only, low_pc=0 (wasm-opt DCE ghost or + # invalid), or empty/degenerate range. + if is_declaration: + return + if low is None or low == 0: + return + if high is None or high <= low: + return + if name is None: + return + intervals.append((low, high, name)) + + for raw_line in raw.splitlines(): + # llvm-dwarfdump prefixes DIE-tag lines with their offset + # ("0xNNN:") whose width varies; replace with a fixed sentinel so + # the tag-line indent matches attribute-line indent. + body = re.sub(r"^0x[0-9a-fA-F]+:", " " * 11, raw_line) + stripped = body.lstrip() + if not stripped: + continue + indent = len(body) - len(stripped) + if stripped.startswith("DW_TAG_subprogram"): + _flush() + sp_indent = indent + low = high = name = None + is_declaration = False + continue + if sp_indent < 0: + continue + # A DIE at sp_indent or shallower closes the current subprogram. + if stripped.startswith("DW_TAG_") and indent <= sp_indent: + _flush() + sp_indent = -1 + low = high = name = None + is_declaration = False + continue + # Only attributes one level deeper than the tag belong to the + # subprogram itself; deeper indents are nested DIEs. + if indent != sp_indent + 2: + continue -def get_dwarf_tag_value(tag: str, line: str) -> str: - # Try extracting value as string - STR_PATTERN = rf"{tag}\s+\(\"(.*)\"\)" - m = re.match(STR_PATTERN, line) - if m: - return m.groups()[0] + m = re.match(r"DW_AT_low_pc\s+\(0x([0-9a-fA-F]+)\)", stripped) + if m: + low = int(m.group(1), 16) + continue + m = re.match(r"DW_AT_high_pc\s+\(0x([0-9a-fA-F]+)\)", stripped) + if m: + v = int(m.group(1), 16) + # high_pc is rendered as either an absolute address or — for + # DW_FORM_data* — as size-from-low_pc; pick the larger of the + # two interpretations. + high = v if low is None or v >= low else low + v + continue + m = re.match(r'DW_AT_name\s+\("(.+)"\)', stripped) + if m and name is None: + name = m.group(1) + continue + # DW_AT_declaration(true) → forward declaration; never callable. + if stripped.startswith("DW_AT_declaration") and "true" in stripped: + is_declaration = True - # Try extracting value as integer - INT_PATTERN = rf"{tag}\s+\((\d+)\)" - m = re.match(INT_PATTERN, line) - return m.groups()[0] + _flush() + return intervals -def get_line_info_from_function_name_dwarf( - dwarf_dump: Path, wasm_file: Path, function_name: str -) -> tuple[str, str, str]: - """ - Find the location info of a given function in a wasm file. +def lookup_subprogram_name(intervals: list, dwarf_addr: int) -> str: """ - cmd = f"{dwarf_dump} --name={function_name} {wasm_file}" - p = subprocess.run( - shlex.split(cmd), - check=False, - capture_output=True, - text=True, - universal_newlines=True, - ) - outputs = p.stdout.split(os.linesep) - - function_name, function_file = "", "unknown" - function_line = "?" - - for line in outputs: - line = line.strip() - - if "DW_AT_name" in line: - function_name = get_dwarf_tag_value("DW_AT_name", line) + Return the name of the innermost (smallest-range) DW_TAG_subprogram + whose [low_pc, high_pc) covers `dwarf_addr`, or None. - if "DW_AT_decl_file" in line: - function_file = get_dwarf_tag_value("DW_AT_decl_file", line) - - if "DW_AT_decl_line" in line: - function_line = get_dwarf_tag_value("DW_AT_decl_line", line) - - return (function_name, function_file, function_line) - - -def get_line_info_from_function_addr_sourcemapping( - emsymbolizer: Path, wasm_file: Path, offset: int -) -> tuple[str, str, str, str]: + "Innermost" matters because wasi-libc declarations (low_pc=0, + high_pc spanning much of the binary) overlap real functions; we want + the tightly-bounded one. """ - Find the location info of a given offset in a wasm file which is compiled with emcc. - - {emsymbolizer} {wasm_file} {offset of file} - - there usually are two lines: - ?? - relative path to source file:line:column + best = None + for low, high, name in intervals: + if low <= dwarf_addr < high: + if best is None or (high - low) < (best[1] - best[0]): + best = (low, high, name) + return best[2] if best else None + + +def resolve_address( + symbolizer: Path, wasm_file: Path, dwarf_addr: int, + subprogram_intervals: list, +) -> list: """ - debug_info_source = wasm_file.with_name(f"{wasm_file.name}.map") - cmd = f"{emsymbolizer} -t code -f {debug_info_source} {wasm_file} {offset}" + Resolve one DWARF address to an inline-frame chain, with an + outermost-name overlay from the interval table. + + Runs: -e -f -i 0x + + The symbolizer produces pairs of lines per frame (function name, + then file:line[:column]). With -i, multiple pairs are returned — + innermost inline frame first, outermost real function last. + + The symbolizer's outermost function-name lookup is unreliable on + wasm targets for two independent reasons: + 1. Symbol-table override (LLVM design, not a bug). + `symbolizeInlinedCode` unconditionally rewrites the outermost + frame's FunctionName from `getNameFromSymbolTable` whenever + `FunctionNameKind::LinkageName` and `UseSymbolTable` are both + set — which is the default. Inline frames are provably NOT + rewritten (the override targets only + `getMutableFrame(getNumberOfFrames() - 1)`), so an inline + chain like `do_bad_access → trigger_oob → app_main` from + DWARF gets its LAST entry overwritten by whatever the symbol + table's `getNameFromSymbolTable` returns for that address. + On some wasi-sdk / wasm layouts that returns a wrong function. + 2. `wasm-opt -Oz -g` leaves DIEs for dead-code-eliminated + compiler-rt / libm helpers behind with low_pc=0 and small + high_pc. Those ghost DIEs violate LLVM `AddrDieMap`'s + parents-before-children invariant (child range ⊆ parent + range), so the map's `upper_bound − 1` lookup returns a + ghost DIE for real addresses. This is separate from + mechanism 1 and applies on any clang version. + In both cases only the OUTERMOST frame's function name is wrong; + inline frames and line/column info are correct. See the + design doc at + docs/superpowers/specs/2026-07-06-addr2line-collapse-modern-legacy-design.md + for source-verified references into + llvm/lib/DebugInfo/Symbolize/SymbolizableObjectFile.cpp and + llvm/lib/DebugInfo/DWARF/DWARFUnit.cpp. + + This function unconditionally overlays the outermost frame's + function name with the result of an interval-table lookup that + picks the innermost DW_TAG_subprogram covering `dwarf_addr`. When + the interval table has no covering entry, the symbolizer's + outermost name is left as-is. + + Returns a list of (function_name, file, line, column) tuples. + Each field is a string; line and column may be "?" if unknown. + """ + addr_str = f"0x{dwarf_addr:x}" + cmd = f"{symbolizer} -e {wasm_file} -f -i {addr_str}" p = subprocess.run( shlex.split(cmd), check=False, capture_output=True, text=True, universal_newlines=True, - cwd=Path.cwd(), ) - outputs = p.stdout.split(os.linesep) - - function_name, function_file = "", "unknown" - function_line, function_column = "?", "?" + output_lines = [line for line in p.stdout.splitlines() if line.strip()] - for line in outputs: - line = line.strip() - - if not line: - continue + frames = [] + for i in range(0, len(output_lines) - 1, 2): + func_name = output_lines[i].strip() + location = output_lines[i + 1].strip() - m = re.match(r"(.*):(\d+):(\d+)", line) + # Parse "file:line:column" or "file:line" or "??:0" + m = re.match(r"^(.*):(\d+):(\d+)$", location) if m: - function_file, function_line, function_column = m.groups() - continue + file, line, column = m.group(1), m.group(2), m.group(3) else: - # it's always ??, not sure about that - if "??" != line: - function_name = line + m = re.match(r"^(.*):(\d+)$", location) + if m: + file, line, column = m.group(1), m.group(2), "?" + else: + file, line, column = location, "?", "?" - return (function_name, function_file, function_line, function_column) + frames.append((func_name, file, line, column)) + # Overlay the outermost frame's name from the interval table when + # available. Inner frames are left alone. + if frames: + name = lookup_subprogram_name(subprogram_intervals, dwarf_addr) + if name: + _, file, line, column = frames[-1] + frames[-1] = (name, file, line, column) -def parse_line_info(line_info: str) -> tuple[str, str, str]: - """ - line_info -> [file, line, column] - """ - PATTERN = r"Line info: file \'(.+)\', line ([0-9]+), column ([0-9]+)" - m = re.search(PATTERN, line_info) - assert m is not None + return frames - file, line, column = m.groups() - return (file, int(line), int(column)) - -def parse_call_stack_line(line: str) -> tuple[str, str, str]: +def parse_call_stack_line(line: str) -> tuple: """ New format (WAMR > 1.3.2): #00: 0x0a04 - $f18 => (00, 0x0a04, $f18) @@ -241,7 +328,6 @@ def parse_call_stack_line(line: str) -> tuple[str, str, str]: _start (always): #05: 0x011f - _start => (05, 0x011f, _start) """ - # New format and Text format and _start PATTERN = r"#([0-9]+): 0x([0-9a-f]+) - (\S+)" m = re.match(PATTERN, line) @@ -257,7 +343,8 @@ def parse_call_stack_line(line: str) -> tuple[str, str, str]: return None -def parse_module_functions(wasm_objdump: Path, wasm_file: Path) -> dict[str, str]: +def parse_module_functions(wasm_objdump: Path, wasm_file: Path) -> dict: + """Map function index to name from wasm-objdump output.""" function_index_to_name = {} cmd = f"{wasm_objdump} -x {wasm_file} --section=function" @@ -268,16 +355,13 @@ def parse_module_functions(wasm_objdump: Path, wasm_file: Path) -> dict[str, str text=True, universal_newlines=True, ) - outputs = p.stdout.split(os.linesep) - - for line in outputs: - if not f"func[" in line: + for line in p.stdout.splitlines(): + if "func[" not in line: continue - PATTERN = r".*func\[([0-9]+)\].*<(.*)>" m = re.match(PATTERN, line) - assert m is not None - + if m is None: + continue index = m.groups()[0] name = m.groups()[1] function_index_to_name[index] = name @@ -285,6 +369,121 @@ def parse_module_functions(wasm_objdump: Path, wasm_file: Path) -> dict[str, str return function_index_to_name +def get_line_info_from_function_name_dwarf( + dwarf_dump: Path, wasm_file: Path, function_name: str +) -> tuple: + """ + Used by --no-addr mode (fast-interp call stacks, or WAMR <= 1.3.2 + output where addresses are absent). + + Resolves a function name to its declaration's (file, decl_line) by + parsing `llvm-dwarfdump --name=` output: collect every + matching DW_TAG_subprogram block, then prefer a non-sysroot match + over a wasi-libc declaration of the same name (short names like + `b`, `c` frequently collide with wasi-libc helpers). + + TODO: limitation — `llvm-symbolizer` is address-driven and has no + name-to-source mode, so this helper manually parses llvm-dwarfdump + text output. The resolved location is the function's declaration + line, not the call site. A robust replacement would need a + programmatic DWARF reader (e.g. pyelftools) or a future LLVM CLI + exposing name-keyed lookup. + + Returns (function_name, file, line). + """ + cmd = f"{dwarf_dump} --name={function_name} {wasm_file}" + p = subprocess.run( + shlex.split(cmd), + check=False, + capture_output=True, + text=True, + universal_newlines=True, + ) + + # Collect every (name, file, line) triple from the output. Each + # DW_TAG_subprogram block is separated by a blank line. + candidates = [] + cur = {} + for line in p.stdout.splitlines() + [""]: + line = line.strip() + if not line: + if cur: + candidates.append(( + cur.get("name", ""), + cur.get("file", "unknown"), + cur.get("line", "?"), + )) + cur = {} + continue + m = re.match(r'DW_AT_name\s+\("(.+)"\)', line) + if m: + cur["name"] = m.group(1) + continue + m = re.match(r'DW_AT_decl_file\s+\("(.+)"\)', line) + if m: + cur["file"] = m.group(1) + continue + m = re.match(r"DW_AT_decl_line\s+\((\d+)\)", line) + if m: + cur["line"] = m.group(1) + + # Prefer a candidate whose source file isn't from wasi-sysroot / + # wasi-libc / wasi-sdk — short names like `b`, `c` collide with + # internal libc helpers and llvm-dwarfdump's name lookup returns + # every match across the binary. + def _is_sysroot(p): + return "wasi-sysroot" in p or "wasi-libc" in p or p.startswith("wasisdk://") + + for name, file, line in candidates: + if name == function_name and not _is_sysroot(file): + return (name, file, line) + # Fall back to the last candidate (preserves prior behavior when + # all matches are sysroot-ish or the name doesn't match exactly). + if candidates: + return candidates[-1] + return ("", "unknown", "?") + + +def get_line_info_from_function_addr_sourcemapping( + emsymbolizer: Path, wasm_file: Path, offset: int +) -> tuple: + """ + Find the location info of a given offset in a wasm file compiled with emcc. + + {emsymbolizer} {wasm_file} {offset} + + Output is two lines: + ?? + relative path to source:line:column + """ + debug_info_source = wasm_file.with_name(f"{wasm_file.name}.map") + cmd = f"{emsymbolizer} -t code -f {debug_info_source} {wasm_file} {offset}" + p = subprocess.run( + shlex.split(cmd), + check=False, + capture_output=True, + text=True, + universal_newlines=True, + cwd=Path.cwd(), + ) + + function_name, function_file = "", "unknown" + function_line, function_column = "?", "?" + + for line in p.stdout.splitlines(): + line = line.strip() + if not line: + continue + m = re.match(r"(.*):(\d+):(\d+)", line) + if m: + function_file, function_line, function_column = m.groups() + continue + if "??" != line: + function_name = line + + return (function_name, function_file, function_line, function_column) + + def demangle(cxxfilt: Path, function_name: str) -> str: cmd = f"{cxxfilt} -n {function_name}" p = subprocess.run( @@ -297,45 +496,133 @@ def demangle(cxxfilt: Path, function_name: str) -> str: return p.stdout.strip() +def print_frames(index_str: str, frames: list, cxxfilt: Path) -> None: + """ + Print resolved frames with consistent indentation. Inlined functions + (frames before the outermost) get an "(inlined into )" suffix + so the chain relationship is unambiguous. + + Single frame (no inlines): + 0: c + at trap.c:4:5 + + Multiple frames (inline chain — innermost first): + 0: do_bad_access (inlined into trigger_oob) + at oob_access.c:11:5 + trigger_oob (inlined into app_main) + at oob_main.c:17:5 + app_main + at oob_main.c:23:5 + + The first frame uses ":" prefix; subsequent frames use the same + width of leading spaces for vertical alignment. + """ + prefix = f"{index_str}:" + indent = " " * len(prefix) + + demangled_names = [ + demangle(cxxfilt, fn) if fn != "??" else "" + for fn, _, _, _ in frames + ] + + for i, (func_name, file, line, column) in enumerate(frames): + leading = prefix if i == 0 else indent + # All frames except the outermost are inlined into the next one in the chain. + if i < len(frames) - 1: + suffix = f" (inlined into {demangled_names[i + 1]})" + else: + suffix = "" + print(f"{leading} {demangled_names[i]}{suffix}") + # Don't print "??:0:0" — collapse to "??:0". + if file == "??" and line in ("0", "?"): + print("\tat ??:0") + elif column != "?": + print(f"\tat {file}:{line}:{column}") + else: + print(f"\tat {file}:{line}") + + def main(): parser = argparse.ArgumentParser(description="addr2line for wasm") - parser.add_argument("--wasi-sdk", type=Path, help="path to wasi-sdk") - parser.add_argument("--wabt", type=Path, help="path to wabt") - parser.add_argument("--wasm-file", type=Path, help="path to wasm file") + parser.add_argument("--wasi-sdk", type=Path, required=True, help="path to wasi-sdk") + parser.add_argument("--wabt", type=Path, required=True, help="path to wabt") + parser.add_argument("--wasm-file", type=Path, required=True, help="path to wasm file (must have DWARF)") parser.add_argument("call_stack_file", type=Path, help="path to a call stack file") parser.add_argument( "--no-addr", action="store_true", help="use call stack without addresses or from fast interpreter mode", ) + parser.add_argument( + "--mode", + choices=["interp", "aot", "fast-interp"], + default="interp", + help=( + "WAMR execution mode that produced the call stack. " + "interp: classic interpreter, frame_ip is post-advance, subtract 1 " + "from offsets to land on the trapping opcode. " + "aot: wamrc commits ip at the start of each WASM operation, use " + "offsets verbatim. " + "fast-interp: offsets are relative to a transformed in-memory " + "bytecode and cannot be mapped to source line — falls back to " + "function-name lookup (same as --no-addr). " + "Default: interp." + ), + ) parser.add_argument("--emsdk", type=Path, help="path to emsdk") args = parser.parse_args() + # fast-interp offsets aren't mappable; treat as --no-addr + if args.mode == "fast-interp": + args.no_addr = True + wasm_objdump = args.wabt.joinpath("bin/wasm-objdump") - assert wasm_objdump.exists() + assert wasm_objdump.exists(), f"wasm-objdump not found at {wasm_objdump}" + + # llvm-symbolizer handles address->source resolution and inline-frame + # expansion via `-f -i`. Ships in wasi-sdk 29+, which is the project's + # minimum supported wasi-sdk for the debug-tools samples. + llvm_symbolizer = args.wasi_sdk.joinpath("bin/llvm-symbolizer") + assert llvm_symbolizer.exists(), ( + f"llvm-symbolizer not found at {llvm_symbolizer}; " + f"need wasi-sdk 29 or newer" + ) llvm_dwarf_dump = args.wasi_sdk.joinpath("bin/llvm-dwarfdump") - assert llvm_dwarf_dump.exists() + assert llvm_dwarf_dump.exists(), f"llvm-dwarfdump not found at {llvm_dwarf_dump}" llvm_cxxfilt = args.wasi_sdk.joinpath("bin/llvm-cxxfilt") - assert llvm_cxxfilt.exists() + assert llvm_cxxfilt.exists(), f"llvm-cxxfilt not found at {llvm_cxxfilt}" emcc_production = locate_sourceMappingURL_section(wasm_objdump, args.wasm_file) if emcc_production: if args.emsdk is None: print("Please provide the path to emsdk via --emsdk") return -1 - emsymbolizer = args.emsdk.joinpath("upstream/emscripten/emsymbolizer") assert emsymbolizer.exists() code_section_start = get_code_section_start(wasm_objdump, args.wasm_file) if code_section_start == -1: + print("Could not find Code section in wasm file", file=sys.stderr) return -1 function_index_to_name = parse_module_functions(wasm_objdump, args.wasm_file) - assert args.call_stack_file.exists() + # Build the DW_TAG_subprogram interval table once at startup. The + # per-frame resolve_address() call overlays the outermost function + # name from this table to fix LLVM's mis-reports on wasm: the + # symbolizer applies an unconditional symbol-table override to the + # outermost frame (see SymbolizableObjectFile.cpp), and wasm-opt + # DCE ghosts violate AddrDieMap's invariant (see DWARFUnit.cpp). + # Empty when we're on the emscripten sourceMappingURL path, which + # doesn't use DWARF. + subprogram_intervals = ( + [] if emcc_production + else build_subprogram_intervals(llvm_dwarf_dump, args.wasm_file) + ) + + assert args.call_stack_file.exists(), f"call stack file not found: {args.call_stack_file}" with open(args.call_stack_file, "rt", encoding="ascii") as f: for i, line in enumerate(f): line = line.strip() @@ -348,63 +635,121 @@ def main(): continue _, offset, index = splitted + index_str = str(i) + if args.no_addr: # FIXME: w/ emcc production if not index.startswith("$f"): # E.g. _start or Text format - print(f"{i}: {index}") + print(f"{index_str}: {index}") continue - index = index[2:] + func_idx = index[2:] - if index not in function_index_to_name: - print(f"{i}: {line}") + if func_idx not in function_index_to_name: + print(f"{index_str}: {line}") continue if not emcc_production: _, function_file, function_line = ( get_line_info_from_function_name_dwarf( - llvm_dwarf_dump, - args.wasm_file, - function_index_to_name[index], + llvm_dwarf_dump, args.wasm_file, + function_index_to_name[func_idx], ) ) else: - _, function_file, function_line = _, "unknown", "?" + function_file, function_line = "unknown", "?" - function_name = demangle(llvm_cxxfilt, function_index_to_name[index]) - print(f"{i}: {function_name}") + function_name = demangle(llvm_cxxfilt, function_index_to_name[func_idx]) + print(f"{index_str}: {function_name}") print(f"\tat {function_file}:{function_line}") - else: - offset = int(offset, 16) - # match the algorithm in wasm_interp_create_call_stack() - # either a *offset* to *code* section start - # or a *offset* in a file - assert offset > code_section_start - offset = offset - code_section_start - - if emcc_production: - function_name, function_file, function_line, function_column = ( - get_line_info_from_function_addr_sourcemapping( - emsymbolizer, args.wasm_file, offset + continue + + # Address-based lookup + offset_int = int(offset, 16) + + # WAMR reports offset 0 when frame_ip wasn't captured (e.g., trap + # at function entry, or top frame of an iwasm -f invocation where + # ip wasn't sync'd before the trap). Fall back to function-name + # lookup for the function header instead of asserting. + if offset_int == 0 or offset_int <= code_section_start: + if not emcc_production: + # Resolve function name from index ($fN) or use the name directly + if index.startswith("$f"): + func_idx = index[2:] + func_name = function_index_to_name.get(func_idx, index) + else: + func_name = index + _, function_file, function_line = ( + get_line_info_from_function_name_dwarf( + llvm_dwarf_dump, args.wasm_file, func_name ) ) - else: - function_name, function_file, function_line, function_column = ( - get_line_info_from_function_addr_dwarf( - llvm_dwarf_dump, args.wasm_file, offset - ) + print(f"{index_str}: {demangle(llvm_cxxfilt, func_name)}") + print(f"\tat {function_file}:{function_line} " + f"(offset=0 — function entry, no inline info)") + continue + # emcc path: just print raw + print(f"{index_str}: {index}") + print(f"\tat unknown:?:? (offset=0 — frame_ip not captured)") + continue + + # Mode-dependent offset adjustment: + # - interp: subtract 1 because frame_ip has advanced past the trapping + # opcode before the trap handler runs (FETCH_OPCODE_AND_DISPATCH + # increments ip before dispatching, plus LEB reads inside handlers). + # - aot: use offset verbatim. wamrc commits ip at the start of each + # WASM operation, before any LEB reads — the offset is already at + # the opcode boundary. + # (fast-interp is handled above by forcing --no-addr; we never reach + # this branch for it.) + adjustment = 1 if args.mode == "interp" else 0 + dwarf_addr = offset_int - code_section_start - adjustment + + if emcc_production: + # Source-map path doesn't support inline expansion; emit single frame. + func_name, file, line, column = ( + get_line_info_from_function_addr_sourcemapping( + emsymbolizer, args.wasm_file, dwarf_addr ) + ) + # Fall back to function index name if unresolved + if func_name == "" and index.startswith("$f"): + func_name = function_index_to_name.get(index[2:], index) + func_name = demangle(llvm_cxxfilt, func_name) + print(f"{index_str}: {func_name}") + print(f"\tat {file}:{line}:{column}") + continue - # if can't parse function_name, use name section or - if function_name == "": - if index.startswith("$f"): - function_name = function_index_to_name.get(index[2:], index) - else: - function_name = index + frames = resolve_address( + llvm_symbolizer, args.wasm_file, dwarf_addr, + subprogram_intervals, + ) - function_name = demangle(llvm_cxxfilt, function_name) + if not frames: + # Fall back to function index name + if index.startswith("$f"): + func_name = function_index_to_name.get(index[2:], index) + else: + func_name = index + print(f"{index_str}: {demangle(llvm_cxxfilt, func_name)}") + print(f"\tat unknown:?:?") + continue - print(f"{i}: {function_name}") - print(f"\tat {function_file}:{function_line}:{function_column}") + # Last-resort fallback for the outermost frame: when neither + # llvm-symbolizer nor the dwarfdump subprogram-name overlay + # produced a clean name (e.g. the address is in a function + # whose DWARF has no PC range — declaration-only + # DW_TAG_subprogram), substitute the function index -> name + # from wasm-objdump. + if index.startswith("$f"): + expected = function_index_to_name.get(index[2:]) + actual = frames[-1][0] + if expected and ( + actual == "??" + or (actual != expected and frames[-1][1] in ("??", "unknown")) + ): + frames[-1] = (expected,) + frames[-1][1:] + + print_frames(index_str, frames, llvm_cxxfilt) return 0 diff --git a/test-tools/addr2line/tests/README.md b/test-tools/addr2line/tests/README.md new file mode 100644 index 0000000000..74a35b7532 --- /dev/null +++ b/test-tools/addr2line/tests/README.md @@ -0,0 +1,140 @@ +# addr2line.py tests + +Pytest suite for `test-tools/addr2line/addr2line.py`. Replaces the older +diagnostic harness that lived under `samples/debug-tools/llvm-bug-experiment/`. + +## What's covered + +Each test case exercises one specific aspect of `addr2line.py`: + +| Test | Source | Purpose | +|------|--------|---------| +| `test_build_subprogram_intervals_filters_invalid_dies` | `apps/simple.c` (+ wasm-opt post-process) | Builds a wasi-libc-linked wasm and post-processes with `wasm-opt -Oz -g` to trigger the DCE-ghost pattern. Asserts the resulting interval table contains no `low_pc = 0` entries and no `high_pc <= low_pc` entries. Guards the skip-guard invariant in `build_subprogram_intervals`. | +| `test_simple_resolution` | `apps/simple.c` | Baseline. One function, one trap. Asserts the resolved frame names the function and points at its source file — proves the basic address → `(func, file:line)` plumbing works end to end. | +| `test_inline_chain_basic` | `apps/always_inline.c` | A trap inside an `__attribute__((always_inline))` helper produces a 2-frame inline chain (the helper plus its caller). Asserts addr2line emits BOTH names and the `(inlined into )` annotation that ties them together — guards against future regressions in `print_frames`. | +| `test_inline_chain_deep` | `apps/deep_inline_chain.c` | Four nested always-inline functions all collapse into one WASM function under `-O0 -g`. Asserts all four names render in the inline chain AND that the outermost frame name is `app_main` (not a wasi-libc symbol like `__multi3` that overlaps app_main's PC range). This is the test that catches the "(inlined into free)" / wasi-libc-shadow class of legacy-resolver regressions. | +| `test_cross_tu_inline` | `apps/multi_file_recur_*.c` | The canonical trigger for both LLVM outermost-name failure modes: two TUs, `-Oz -flto`, then `wasm-opt -Oz -g` (which produces DCE ghosts). Both the symbol-table override and the AddrDieMap invariant violation apply here, so the test only pins the limited contract that the *line table* stays inside our source files (or returns `??:0`) — it must NEVER reach into wasi-libc / wasi-sysroot. | +| `test_trap_mid_function` | `apps/trap_in_loop.c` | Trap is inside a loop body, several instructions past function entry. Asserts the resolved line is > 10 (i.e. NOT the function declaration line). Catches future regressions where addr2line might collapse mid-function addresses back to function-entry. | +| `test_multi_frame_callstack` | `apps/multi_frame.c` | Four distinct WASM functions in a call chain. Asserts every frame renders with the correct function name and a distinct frame index — catches frame-numbering or per-frame-resolution bugs that a single-frame test misses. | +| `test_cxx_demangling` | `apps/cxx_mangled.cpp` | A templated, namespaced C++ function exercises the cxxfilt pass. Asserts mangled `_Z...` never leaks to the user and that the line table resolves to our `.cpp` (not wasi-libc). Doesn't pin the symbolizer's function-name choice — both clang < 22 and clang 22+ pick the wrong subprogram on some C++ template addresses on wasm. | +| `test_aot_mode` | `apps/simple.c` (`-Oz -g`) | `--mode=aot` uses no `-1` adjustment (wamrc commits ip at instruction-start, not post-advance). The test picks an address at `low_pc+delta` and asserts the function still resolves under `--mode=aot`. Skips when `crash()` gets inlined away under `-Oz`. | +| `test_fast_interp_mode` | `fixtures/fast_interp_stack.txt` + `apps/simple.c` | The fixture has `$f0`/`$f1` indices; `--mode=fast-interp` forces function-name lookup (offsets aren't mappable). Asserts both indices resolve to the real wasm name-section entries (`crash`/`app_main`). | +| `test_no_addr_mode` | `fixtures/fast_interp_stack.txt` + `apps/simple.c` | Same fixture; `--no-addr` is the explicit form of function-name lookup. Additionally asserts the DWARF decl-file (`simple.c`) is surfaced (function-name lookup falls back to declaration-line, not call-site). | +| `test_offset_zero_fallback` | `apps/simple.c` (inline cs) | WAMR reports `offset=0` when `frame_ip` wasn't captured (trap at function entry, top frame of `iwasm -f`, etc.). The test asserts addr2line falls back to function-name resolution gracefully and prints the `app_main` name — does NOT crash or assert. | +| `test_empty_input` | `fixtures/empty_stack.txt` + `apps/simple.c` | A zero-line call stack should exit cleanly (rc=0) and produce no output. Defends against an `IndexError`-style crash when the loop body never runs. | + +Sources under `apps/` are written for the test suite, not copied from +`samples/`. They're small (~10–25 lines each) and target specific edge +cases that aren't covered by the sample-CI integration tests. + +## Running + +```bash +# Default: against $WASI_SDK_PATH or /opt/wasi-sdk +./run_tests.sh +# or +python3 -m pytest -v + +# Fixture-only (fast, no builds) +./run_tests.sh -m "not slow" + +# Multi-SDK: parametrize build-based tests over every detected +# /opt/wasi-sdk-*-x86_64-linux installation. Run each build-based +# test once per detected wasi-sdk. +./run_tests.sh --multi-sdk + +# Verbose: print each test's addr2line input + stdout/stderr so you +# can see what got resolved. Use -v -s to surface it live; -v alone +# captures it and only shows it on failure. +./run_tests.sh -v -s +``` + +## Background: why the interval-table overlay exists + +`addr2line.py` exists primarily to convert WAMR call-stack dumps into +source `file:line:column`. On most well-behaved DWARF inputs this is +a thin wrapper around `llvm-symbolizer -f -i`. But on wasm targets, +the symbolizer's OUTERMOST frame's function name can be wrong for +two independent reasons — both source-verified against upstream +LLVM (`SymbolizableObjectFile.cpp` and `DWARFUnit.cpp`): + +1. **Symbol-table override on the outermost frame** (LLVM design + choice). `symbolizeInlinedCode` unconditionally rewrites the + outermost frame's `FunctionName` from `getNameFromSymbolTable` + whenever `FunctionNameKind::LinkageName` and `UseSymbolTable` are + both set. Inline frames are provably NOT rewritten — the override + is applied only to `getMutableFrame(getNumberOfFrames() - 1)`. On + some wasi-sdk builds the symbol-table lookup returns the wrong + function for an address that DWARF places correctly. +2. **`AddrDieMap` invariant violation** (root cause in binaryen's + DWARF preservation model). `wasm-opt -Oz -g` deletes functions + from the wasm module but its DWARF updater does not remove the + corresponding `DW_TAG_subprogram` DIEs — instead, when an old + address can no longer be mapped to a surviving IR node, + `LocationUpdater::getNewFuncStart` in `src/wasm/wasm-debug.cpp` + returns `0` as a tombstone value and `updateDIE` writes it into + `DW_AT_low_pc`. The `DW_TAG_subprogram` remains in place; sometimes + its `DW_AT_high_pc` retains its original value. Result: stale + "ghost" DIEs like `fmaf [0, 0x72)`. LLVM's `updateAddressDieMap` + in `DWARFUnit.cpp` assumes children are inserted after parents + and each child range ⊆ parent range; these ghosts violate that + invariant, so the map's `upper_bound − 1` lookup returns an + insertion-order-dependent DIE for real addresses. + +The symptom is the same in both cases — only the OUTERMOST frame's +function name is wrong. Line/column info and inner inlined frame +names are correct because they come from a different LLVM code path +(`getInliningInfoForAddress`, which walks `getParent()` links from +the leaf DIE and does NOT go through the symbol-table override). + +`addr2line.py` always runs `llvm-dwarfdump --debug-info` at startup, +builds a `(low_pc, high_pc, name)` interval table for every real +callable `DW_TAG_subprogram` (three skip guards described below), and +overlays the outermost frame's name from an innermost-wins lookup on +that table. + +### Skip guards applied to the interval table + +- `DW_AT_declaration=true` → forward declaration only, no code. +- `low_pc == 0` → structurally impossible for real code (wasm binary + format guarantees the Code section starts after Type + Function + section headers). Any `low_pc = 0` DIE is a wasm-opt DCE ghost or + a broken producer's output. +- `high_pc <= low_pc` → empty or degenerate range. Never observed + in practice; kept as a defense-in-depth guard against future + toolchain changes. + +Multi-SDK CI (`./run_tests.sh --multi-sdk`) exercises the parser +against DWARF produced by wasi-sdk 29 (clang 21) and wasi-sdk 33 +(clang 22) to catch DIE-encoding differences between clang versions. + +## Why we left samples/debug-tools/llvm-bug-experiment behind + +The earlier experiment was useful for diagnosing the bug but wasn't a +proper test suite — it was a one-shot script that printed status to +stdout. This pytest suite covers more cases, runs in CI per-PR (default +SDK) and nightly (multi-SDK), and is decoupled from sample evolution. + +## Layout + +``` +test-tools/addr2line/tests/ +├── README.md # this file +├── conftest.py # fixtures (wasi_sdk, build_wasm, run_addr2line, ...) +├── test_addr2line.py # the tests +├── pytest.ini # marker definitions +├── run_tests.sh # convenience wrapper +├── apps/ # purpose-built test sources +│ ├── simple.c +│ ├── always_inline.c +│ ├── deep_inline_chain.c +│ ├── multi_file_recur_main.c +│ ├── multi_file_recur.c +│ ├── trap_in_loop.c +│ ├── multi_frame.c +│ └── cxx_mangled.cpp +└── fixtures/ # plaintext call-stack inputs + ├── fast_interp_stack.txt + ├── aot_stack.txt + └── empty_stack.txt +``` diff --git a/test-tools/addr2line/tests/apps/always_inline.c b/test-tools/addr2line/tests/apps/always_inline.c new file mode 100644 index 0000000000..4be1c6eaed --- /dev/null +++ b/test-tools/addr2line/tests/apps/always_inline.c @@ -0,0 +1,27 @@ +/* + * Copyright (C) 2026 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +/* Forced-inline helper containing the trap. The trap address falls + inside the inlined region, so addr2line.py must report both + `inner` and `outer` under a single runtime frame, with `inner` + carrying the `(inlined into outer)` suffix. */ + +__attribute__((always_inline)) static inline void +inner(void) +{ + __builtin_trap(); +} + +void +outer(void) +{ + inner(); +} + +void +app_main(void) +{ + outer(); +} diff --git a/test-tools/addr2line/tests/apps/cxx_mangled.cpp b/test-tools/addr2line/tests/apps/cxx_mangled.cpp new file mode 100644 index 0000000000..29fafedf57 --- /dev/null +++ b/test-tools/addr2line/tests/apps/cxx_mangled.cpp @@ -0,0 +1,34 @@ +/* + * Copyright (C) 2026 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +/* C++ source with a templated and namespaced function. addr2line.py + should pass the mangled symbol through llvm-cxxfilt and produce a + readable demangled name in the output. */ + +namespace app { +namespace ns { + +template +__attribute__((noinline)) static void +crash_with(T) +{ + __builtin_trap(); +} + +class Worker +{ + public: + __attribute__((noinline)) void run() { crash_with(42); } +}; + +} // namespace ns +} // namespace app + +extern "C" void +app_main(void) +{ + app::ns::Worker w; + w.run(); +} diff --git a/test-tools/addr2line/tests/apps/deep_inline_chain.c b/test-tools/addr2line/tests/apps/deep_inline_chain.c new file mode 100644 index 0000000000..b392a13bc5 --- /dev/null +++ b/test-tools/addr2line/tests/apps/deep_inline_chain.c @@ -0,0 +1,35 @@ +/* + * Copyright (C) 2026 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +/* Four nested always_inline helpers. The trap inside level4 must + produce a 4-deep inline chain in the output, with consistent + indentation across all four frames. */ + +__attribute__((always_inline)) static inline void +level4(void) +{ + __builtin_trap(); +} +__attribute__((always_inline)) static inline void +level3(void) +{ + level4(); +} +__attribute__((always_inline)) static inline void +level2(void) +{ + level3(); +} +__attribute__((always_inline)) static inline void +level1(void) +{ + level2(); +} + +void +app_main(void) +{ + level1(); +} diff --git a/test-tools/addr2line/tests/apps/multi_file_recur.c b/test-tools/addr2line/tests/apps/multi_file_recur.c new file mode 100644 index 0000000000..c63ca8f25e --- /dev/null +++ b/test-tools/addr2line/tests/apps/multi_file_recur.c @@ -0,0 +1,24 @@ +/* + * Copyright (C) 2026 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +static volatile int sink; + +int +recur(int depth) +{ + /* Allocate stack space and use it so the call isn't trivially + optimized away. Trap inside the loop body. */ + volatile char buf[64]; + buf[0] = (char)depth; + buf[63] = (char)(depth >> 8); + sink = buf[0] + buf[63]; + if (depth == 100) { + __builtin_trap(); + } + /* Use the return value after the call to prevent tail-call + conversion to a loop under -Oz -flto. */ + int r = recur(depth + 1); + return r + buf[0]; +} diff --git a/test-tools/addr2line/tests/apps/multi_file_recur_main.c b/test-tools/addr2line/tests/apps/multi_file_recur_main.c new file mode 100644 index 0000000000..b24b355ce7 --- /dev/null +++ b/test-tools/addr2line/tests/apps/multi_file_recur_main.c @@ -0,0 +1,18 @@ +/* + * Copyright (C) 2026 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +/* Multi-file recursion: app_main calls into recur() defined in another + TU. Compiled with -Oz -g -flto, the linker can inline cross-TU and the + trap address resolves to a chain that crosses files. This is the + canonical case that triggers the LLVM symbolizer wasm bug on clang < 22. */ + +int +recur(int depth); + +void +app_main(void) +{ + recur(0); +} diff --git a/test-tools/addr2line/tests/apps/multi_frame.c b/test-tools/addr2line/tests/apps/multi_frame.c new file mode 100644 index 0000000000..144fa40664 --- /dev/null +++ b/test-tools/addr2line/tests/apps/multi_frame.c @@ -0,0 +1,32 @@ +/* + * Copyright (C) 2026 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +/* Three noinline functions producing a 3-frame WASM call stack at the + trap (plus app_main). addr2line.py must report all four frames with + distinct indices. */ + +__attribute__((noinline)) void +bot(void) +{ + __builtin_trap(); +} + +__attribute__((noinline)) void +mid(void) +{ + bot(); +} + +__attribute__((noinline)) void +top(void) +{ + mid(); +} + +void +app_main(void) +{ + top(); +} diff --git a/test-tools/addr2line/tests/apps/simple.c b/test-tools/addr2line/tests/apps/simple.c new file mode 100644 index 0000000000..5a2d0eeb6e --- /dev/null +++ b/test-tools/addr2line/tests/apps/simple.c @@ -0,0 +1,19 @@ +/* + * Copyright (C) 2026 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +/* Baseline test: single function, single trap. The address of + __builtin_trap() must symbolicate back to file=simple.c, function=crash. */ + +void +crash(void) +{ + __builtin_trap(); +} + +void +app_main(void) +{ + crash(); +} diff --git a/test-tools/addr2line/tests/apps/trap_in_loop.c b/test-tools/addr2line/tests/apps/trap_in_loop.c new file mode 100644 index 0000000000..15546da174 --- /dev/null +++ b/test-tools/addr2line/tests/apps/trap_in_loop.c @@ -0,0 +1,27 @@ +/* + * Copyright (C) 2026 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +/* Trap inside a for-loop body. The trap address is neither at function + entry nor at function exit; it's mid-function. addr2line.py must + resolve it to the trap line, not the function declaration line. */ + +static volatile int sink; + +void +crash(void) +{ + for (int i = 0; i < 10; i++) { + sink = i; + if (i == 7) { + __builtin_trap(); + } + } +} + +void +app_main(void) +{ + crash(); +} diff --git a/test-tools/addr2line/tests/conftest.py b/test-tools/addr2line/tests/conftest.py new file mode 100644 index 0000000000..242e321181 --- /dev/null +++ b/test-tools/addr2line/tests/conftest.py @@ -0,0 +1,277 @@ +# Copyright (C) 2026 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +""" +pytest fixtures for the addr2line.py test suite. + +Default mode: a single wasi-sdk (from $WASI_SDK_PATH or /opt/wasi-sdk). +With --multi-sdk: discover all /opt/wasi-sdk-*-x86_64-linux installations +and parametrize the wasi_sdk fixture so build-based tests run once per SDK. +""" + +import os +import shlex +import subprocess +import sys +from pathlib import Path + +import pytest + + +# --- CLI options ---------------------------------------------------------- + +def pytest_addoption(parser): + parser.addoption( + "--multi-sdk", + action="store_true", + default=False, + help="Run build-based tests against every detected " + "/opt/wasi-sdk-*-x86_64-linux installation (default: only " + "the SDK from WASI_SDK_PATH or /opt/wasi-sdk).", + ) + + +# --- Tool-discovery helpers ---------------------------------------------- + +def _detect_sdks_under_opt(): + """Return a sorted list of (version_str, Path) tuples for every + wasi-sdk under /opt that has a working clang AND ships + llvm-symbolizer. addr2line.py requires wasi-sdk 29+; pre-29 + bundles omit llvm-symbolizer and are filtered out here. Newest + version last.""" + found = [] + for d in sorted(Path("/opt").glob("wasi-sdk-*-x86_64-linux")): + if not (d / "bin" / "clang").exists(): + continue + if not (d / "bin" / "llvm-symbolizer").exists(): + continue + ver = d.name.replace("wasi-sdk-", "").replace("-x86_64-linux", "") + found.append((ver, d)) + return found + + +def _default_sdk(): + """Return the single SDK to use when --multi-sdk is not given.""" + env = os.environ.get("WASI_SDK_PATH") + if env and (Path(env) / "bin" / "clang").exists(): + return Path(env) + default = Path("/opt/wasi-sdk") + if (default / "bin" / "clang").exists(): + return default + return None + + +# --- Parametrization ------------------------------------------------------ + +def pytest_generate_tests(metafunc): + """Parametrize the wasi_sdk fixture based on --multi-sdk.""" + if "wasi_sdk" not in metafunc.fixturenames: + return + + multi = metafunc.config.getoption("--multi-sdk") + if multi: + sdks = _detect_sdks_under_opt() + if not sdks: + pytest.skip("--multi-sdk: no /opt/wasi-sdk-*-x86_64-linux found") + ids = [ver for ver, _ in sdks] + paths = [path for _, path in sdks] + metafunc.parametrize("wasi_sdk", paths, ids=ids, indirect=False) + else: + sdk = _default_sdk() + if sdk is None: + pytest.skip("no wasi-sdk found (set WASI_SDK_PATH or install /opt/wasi-sdk)") + metafunc.parametrize("wasi_sdk", [sdk], ids=[sdk.name], indirect=False) + + +# --- Session-scoped path fixtures ---------------------------------------- + +@pytest.fixture(scope="session") +def wabt(): + p = Path(os.environ.get("WABT_PATH", "/opt/wabt")) + if not (p / "bin" / "wasm-objdump").exists(): + pytest.skip(f"wabt not found at {p}") + return p + + +@pytest.fixture(scope="session") +def binaryen(): + p = Path(os.environ.get("BINARYEN_PATH", "/opt/binaryen")) + if not (p / "bin" / "wasm-opt").exists(): + pytest.skip(f"binaryen not found at {p}") + return p + + +@pytest.fixture(scope="session") +def addr2line_script(): + """Return the absolute path to addr2line.py.""" + here = Path(__file__).resolve().parent + return here.parent / "addr2line.py" + + +@pytest.fixture(scope="session") +def apps_dir(): + return Path(__file__).resolve().parent / "apps" + + +@pytest.fixture(scope="session") +def fixtures_dir(): + return Path(__file__).resolve().parent / "fixtures" + + +# --- Build / invoke fixtures --------------------------------------------- + +@pytest.fixture +def build_wasm(wasi_sdk, apps_dir, tmp_path_factory): + """Build wasm from sources under apps/ with the given flags. + + Cached per (sources tuple, flags tuple, language) within this test + session and SDK. + """ + cache = {} + + def _build(sources, flags, language="c"): + key = (tuple(sources), tuple(flags), language, str(wasi_sdk)) + if key in cache: + return cache[key] + + compiler = wasi_sdk / "bin" / ("clang++" if language == "cxx" else "clang") + outdir = tmp_path_factory.mktemp("build", numbered=True) + out_wasm = outdir / "out.wasm" + + cmd = [str(compiler)] + list(flags) + [ + "--target=wasm32-wasi", + "-Wl,--export=app_main", + "-Wl,--no-entry", + "-nostartfiles", + "-o", str(out_wasm), + ] + [str(apps_dir / s) for s in sources] + + p = subprocess.run(cmd, capture_output=True, text=True) + if p.returncode != 0: + raise AssertionError( + f"build failed:\n cmd: {' '.join(cmd)}\n" + f" stdout: {p.stdout}\n stderr: {p.stderr}" + ) + cache[key] = out_wasm + return out_wasm + + return _build + + +@pytest.fixture +def wasm_opt_pass(binaryen, tmp_path): + """Run wasm-opt on a wasm file and return the new path.""" + def _pass(input_wasm, args): + out = tmp_path / (input_wasm.stem + ".opt.wasm") + cmd = [str(binaryen / "bin" / "wasm-opt")] + list(args) + [ + "-o", str(out), str(input_wasm), + ] + p = subprocess.run(cmd, capture_output=True, text=True) + if p.returncode != 0: + raise AssertionError( + f"wasm-opt failed:\n cmd: {' '.join(cmd)}\n" + f" stdout: {p.stdout}\n stderr: {p.stderr}" + ) + return out + return _pass + + +@pytest.fixture +def run_addr2line(addr2line_script, wabt, tmp_path, wasi_sdk, request): + """Invoke addr2line.py and capture stdout/stderr/exitcode. + + `call_stack` may be either a list of strings (one per frame line) or + a Path to an existing call-stack file. `extra_args` is appended to + the command line (e.g. ['--mode', 'aot']). + + Under `pytest -v`/`-vv` (verbosity >= 1) the invocation and resolved + output are printed; combine with `-s` to see them live, otherwise + pytest captures them and only surfaces them on failure. + """ + verbose = request.config.getoption("verbose") >= 1 + + def _run(wasm_file, call_stack, sdk_override=None, extra_args=()): + sdk = sdk_override if sdk_override is not None else wasi_sdk + if isinstance(call_stack, (list, tuple)): + cs_file = tmp_path / "callstack.txt" + cs_file.write_text("\n".join(call_stack) + ("\n" if call_stack else "")) + else: + cs_file = Path(call_stack) + + cmd = [ + sys.executable, str(addr2line_script), + "--wasi-sdk", str(sdk), + "--wabt", str(wabt), + "--wasm-file", str(wasm_file), + *list(extra_args), + str(cs_file), + ] + p = subprocess.run(cmd, capture_output=True, text=True) + + if verbose: + test_id = request.node.name + print(f"\n--- [{test_id}] addr2line input ---") + if isinstance(call_stack, (list, tuple)): + for line in call_stack: + print(f" {line}") + else: + print(f" (from {cs_file})") + extra = " ".join(extra_args) if extra_args else "(none)" + print(f"--- [{test_id}] extra args: {extra} ---") + print(f"--- [{test_id}] addr2line stdout (rc={p.returncode}) ---") + print(p.stdout.rstrip()) + if p.stderr.strip(): + print(f"--- [{test_id}] addr2line stderr ---") + print(p.stderr.rstrip()) + print(f"--- [{test_id}] end ---") + + return p.stdout, p.stderr, p.returncode + + return _run + + +# --- Helpers ------------------------------------------------------------- + +def find_dwarf_low_pc(wasi_sdk: Path, wasm: Path, func_name: str) -> int | None: + """Find the DW_AT_low_pc of the named DW_TAG_subprogram in `wasm`. + Returns int or None if not found.""" + p = subprocess.run( + [str(wasi_sdk / "bin" / "llvm-dwarfdump"), str(wasm)], + capture_output=True, text=True, + ) + in_subp = False + low = None + for line in p.stdout.splitlines(): + s = line.strip() + if "DW_TAG_subprogram" in s: + in_subp = True + low = None + continue + if in_subp: + import re + m = re.match(r'DW_AT_low_pc\s+\((0x[0-9a-fA-F]+)\)', s) + if m: + low = int(m.group(1), 16) + continue + m = re.match(r'DW_AT_name\s+\("' + re.escape(func_name) + r'"\)', s) + if m and low is not None: + return low + if s.startswith("DW_TAG_") or s == "": + in_subp = False + low = None + return None + + +def code_section_start(wabt: Path, wasm: Path) -> int: + """Read the Code section start offset from wasm-objdump -h.""" + p = subprocess.run( + [str(wabt / "bin" / "wasm-objdump"), "-h", str(wasm)], + capture_output=True, text=True, check=True, + ) + import re + for line in p.stdout.splitlines(): + s = line.strip() + if "Code" in s and "start=" in s: + m = re.search(r"start=(0x[0-9a-fA-F]+)", s) + if m: + return int(m.group(1), 16) + raise AssertionError("Code section not found in " + str(wasm)) diff --git a/test-tools/addr2line/tests/fixtures/aot_stack.txt b/test-tools/addr2line/tests/fixtures/aot_stack.txt new file mode 100644 index 0000000000..0d3426f793 --- /dev/null +++ b/test-tools/addr2line/tests/fixtures/aot_stack.txt @@ -0,0 +1,2 @@ +#00: 0x1e0a - $f12 +#01: 0x1dcc - app_main diff --git a/test-tools/addr2line/tests/fixtures/empty_stack.txt b/test-tools/addr2line/tests/fixtures/empty_stack.txt new file mode 100644 index 0000000000..e69de29bb2 diff --git a/test-tools/addr2line/tests/fixtures/fast_interp_stack.txt b/test-tools/addr2line/tests/fixtures/fast_interp_stack.txt new file mode 100644 index 0000000000..51783a5b72 --- /dev/null +++ b/test-tools/addr2line/tests/fixtures/fast_interp_stack.txt @@ -0,0 +1,2 @@ +#00: 0x0001 - $f0 +#01: 0x0001 - $f1 diff --git a/test-tools/addr2line/tests/pytest.ini b/test-tools/addr2line/tests/pytest.ini new file mode 100644 index 0000000000..cca0386e57 --- /dev/null +++ b/test-tools/addr2line/tests/pytest.ini @@ -0,0 +1,5 @@ +[pytest] +markers = + multi_sdk: parametrize over all detected wasi-sdk-*-x86_64-linux installations under /opt + slow: long-running test (typically a build + run); excluded by default if running -m "not slow" +testpaths = . diff --git a/test-tools/addr2line/tests/run_tests.sh b/test-tools/addr2line/tests/run_tests.sh new file mode 100755 index 0000000000..7143e6b2fe --- /dev/null +++ b/test-tools/addr2line/tests/run_tests.sh @@ -0,0 +1,19 @@ +#!/bin/bash +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +# +# Thin wrapper around pytest for the addr2line.py test suite. +# +# Usage: +# ./run_tests.sh # default: single SDK from $WASI_SDK_PATH +# ./run_tests.sh -m "not slow" # only fixture-based tests +# ./run_tests.sh --multi-sdk # parametrize over /opt/wasi-sdk-* +# +# Env vars consumed: +# WASI_SDK_PATH (default /opt/wasi-sdk) +# WABT_PATH (default /opt/wabt) +# BINARYEN_PATH (default /opt/binaryen) + +set -euo pipefail +cd "$(dirname "$0")" +exec python3 -m pytest "$@" diff --git a/test-tools/addr2line/tests/test_addr2line.py b/test-tools/addr2line/tests/test_addr2line.py new file mode 100644 index 0000000000..4e322a58ca --- /dev/null +++ b/test-tools/addr2line/tests/test_addr2line.py @@ -0,0 +1,298 @@ +# Copyright (C) 2026 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +""" +Test suite for test-tools/addr2line/addr2line.py. + +Most tests build a wasm from a single apps/*.c source, capture the trap +address from DWARF, and check that addr2line.py resolves it correctly. + +Multi-SDK behavior is opt-in via `pytest --multi-sdk` — build-based +tests re-run once per detected SDK to catch DWARF-encoding changes +across clang versions. +""" + +import re +import subprocess +from pathlib import Path + +import pytest + +from conftest import find_dwarf_low_pc, code_section_start + + +# Format a wasm file offset that points at a real instruction inside +# the function body, given its DWARF low_pc and the wasm Code section +# start. Offset is `code_section_start + low_pc + delta + 1` because +# addr2line.py applies a -1 adjustment for classic-interp mode. +def _stack_line(wabt, wasm, wasi_sdk, func_name, delta=1, frame_idx=0, + func_label=None): + cs = code_section_start(wabt, wasm) + low = find_dwarf_low_pc(wasi_sdk, wasm, func_name) + if low is None: + pytest.skip(f"DW_TAG_subprogram for {func_name} not found in {wasm}") + # +1 to compensate for addr2line.py's -1 in interp mode + file_off = cs + low + delta + 1 + label = func_label or "$f0" + return f"#{frame_idx:02d}: 0x{file_off:04x} - {label}" + + +# --- Build-based tests --------------------------------------------------- + +@pytest.mark.slow +def test_simple_resolution(build_wasm, wabt, run_addr2line, wasi_sdk): + """Single-function trap resolves to function=crash, file=simple.c.""" + wasm = build_wasm(["simple.c"], ["-O0", "-g"]) + line = _stack_line(wabt, wasm, wasi_sdk, "crash", delta=1, + func_label="$f0") + out, err, rc = run_addr2line(wasm, [line]) + assert rc == 0 + assert "crash" in out + assert "simple.c" in out + + +@pytest.mark.slow +def test_inline_chain_basic(build_wasm, wabt, run_addr2line, wasi_sdk): + """always_inline helper at trap site produces 2-frame inline chain.""" + wasm = build_wasm(["always_inline.c"], ["-O0", "-g"]) + # Trap is inside `inner` (always_inline'd into outer) + line = _stack_line(wabt, wasm, wasi_sdk, "outer", delta=2, + func_label="$f0") + out, err, rc = run_addr2line(wasm, [line]) + assert rc == 0 + assert "inner" in out + assert "outer" in out + # Inline annotation must appear + assert "(inlined into" in out + + +@pytest.mark.slow +def test_inline_chain_deep(build_wasm, wabt, run_addr2line, wasi_sdk): + """4-level inline chain produces all 4 frames in order.""" + wasm = build_wasm(["deep_inline_chain.c"], ["-O0", "-g"]) + # All 4 levels inlined into app_main; pick low_pc(app_main)+small + line = _stack_line(wabt, wasm, wasi_sdk, "app_main", delta=2, + func_label="$f0") + out, err, rc = run_addr2line(wasm, [line]) + assert rc == 0 + # All four levels should appear somewhere in the output + for fn in ("level1", "level2", "level3", "level4"): + assert fn in out, f"missing {fn} in output:\n{out}" + assert "(inlined into" in out + # The OUTERMOST frame must be app_main (not a wasi-libc symbol like + # `free` / `__multi3` that overlaps app_main's PC range — that's the + # exact regression the interval-table overlay in resolve_address + # exists to prevent). + assert re.search(r"^\s+app_main\b", out, re.MULTILINE), ( + f"outermost frame is not app_main:\n{out}" + ) + + +@pytest.mark.slow +def test_cross_tu_inline(build_wasm, wasm_opt_pass, wabt, run_addr2line, + wasi_sdk): + """Multi-file recursion under -Oz -g -flto + wasm-opt -Oz -g. + + `wasm-opt -Oz -g` reorders DWARF in ways that confuse both legacy + and modern symbolizer paths on the function-name lookup; this test + pins the more limited contract that the line table itself stays + inside our sources (or returns `??:0`), never a wasi-libc path. + Outermost-name correctness for non-mangled DWARF is covered by + test_inline_chain_deep. + """ + raw = build_wasm( + ["multi_file_recur_main.c", "multi_file_recur.c"], + ["-Oz", "-g", "-flto"], + ) + debug = wasm_opt_pass(raw, ["-Oz", "-g"]) + line = _stack_line(wabt, debug, wasi_sdk, "recur", delta=0x40, + func_label="$f0") + out, err, rc = run_addr2line(debug, [line]) + assert rc == 0 + assert ( + "multi_file_recur" in out + or "??:0" in out + ), f"line table did not resolve to multi_file_recur* source:\n{out}" + + +@pytest.mark.slow +def test_trap_mid_function(build_wasm, wabt, run_addr2line, wasi_sdk): + """Trap inside a loop body resolves to the trap line, not entry line.""" + wasm = build_wasm(["trap_in_loop.c"], ["-O0", "-g"]) + # Probe well past entry so the line table catches the trap line + line = _stack_line(wabt, wasm, wasi_sdk, "crash", delta=0x20, + func_label="$f0") + out, err, rc = run_addr2line(wasm, [line]) + assert rc == 0 + assert "crash" in out + assert "trap_in_loop.c" in out + # The trap is on its own line — output should reference a line >= 14 + # (function declaration is on line ~10, trap is below it) + m = re.search(r"trap_in_loop\.c:(\d+)", out) + assert m is not None, f"no file:line in output:\n{out}" + line_num = int(m.group(1)) + assert line_num > 10, ( + f"resolved to line {line_num}, expected > 10 (post-declaration)" + ) + + +@pytest.mark.slow +def test_multi_frame_callstack(build_wasm, wabt, run_addr2line, wasi_sdk): + """Multi-frame call stack renders all frames with distinct indices.""" + wasm = build_wasm(["multi_frame.c"], ["-O0", "-g"]) + lines = [] + for idx, fn in enumerate(["bot", "mid", "top", "app_main"]): + lines.append( + _stack_line(wabt, wasm, wasi_sdk, fn, delta=2, + frame_idx=idx, func_label="$f0") + ) + out, err, rc = run_addr2line(wasm, lines) + assert rc == 0 + for fn in ("bot", "mid", "top", "app_main"): + assert fn in out, f"missing {fn} in output:\n{out}" + + +@pytest.mark.slow +def test_cxx_demangling(build_wasm, wabt, run_addr2line, wasi_sdk): + """C++ symbols come through the toolchain in a readable form. + + The DWARF that wasi-sdk's clang emits stores the demangled name in + DW_AT_name and the mangled `_Z...` form in DW_AT_linkage_name; both + llvm-symbolizer and the wasm name section emit the demangled form, + so addr2line.py's llvm-cxxfilt pass is usually a no-op. This test + asserts the end-to-end output is human-readable: source file from + our sample, line numbers present, no mangled `_Z...` leaks. + + Symbolizer-reported function names for templated/namespaced symbols + are not asserted — both clang < 22 and clang 22+ pick the wrong + DW_TAG_subprogram for some C++ template addresses on wasm targets + (e.g. reporting `app_main` for an address that DWARF clearly puts + inside `crash_with`). The line table is correct in both cases. + """ + wasm = build_wasm(["cxx_mangled.cpp"], ["-O0", "-g"], language="cxx") + # Look up by the DWARF DW_AT_name (already demangled, with template args). + line = _stack_line(wabt, wasm, wasi_sdk, "crash_with", + delta=2, func_label="$f0") + out, err, rc = run_addr2line(wasm, [line]) + assert rc == 0 + # No mangled `_Z...` should ever leak to the user. + assert "_ZN" not in out, f"mangled prefix leaked:\n{out}" + # Source file should resolve to our sample (line table is reliable). + assert "cxx_mangled.cpp" in out, ( + f"line table did not resolve to cxx_mangled.cpp:\n{out}" + ) + + +@pytest.mark.slow +def test_aot_mode(build_wasm, wasm_opt_pass, wabt, run_addr2line, wasi_sdk): + """--mode=aot uses no -1 adjustment and resolves correctly. + + For an address that's exactly at low_pc+delta (no -1), --mode=aot must + still resolve to the right function (whereas --mode=interp would + apply -1 and possibly land outside the function range). + """ + raw = build_wasm(["simple.c"], ["-Oz", "-g"]) + debug = wasm_opt_pass(raw, ["-Oz", "-g"]) + cs = code_section_start(wabt, debug) + low = find_dwarf_low_pc(wasi_sdk, debug, "crash") + if low is None: + pytest.skip("crash() inlined away under -Oz; skipping AOT mode test") + aot_offset = cs + low + 4 # AOT path uses offset verbatim + cs_line = f"#00: 0x{aot_offset:04x} - $f0" + out, err, rc = run_addr2line(debug, [cs_line], extra_args=["--mode", "aot"]) + assert rc == 0 + assert "crash" in out + + +# --- Fixture-based tests (no build) -------------------------------------- + +def test_fast_interp_mode(addr2line_script, wabt, wasi_sdk, fixtures_dir, + run_addr2line, build_wasm): + """--mode=fast-interp falls back to function-name lookup (no addr math). + + The fixture's $f0 and $f1 must resolve to simple.c's two functions + (`crash` and `app_main`) via the wasm name section. + """ + wasm = build_wasm(["simple.c"], ["-O0", "-g"]) + out, err, rc = run_addr2line( + wasm, fixtures_dir / "fast_interp_stack.txt", + extra_args=["--mode", "fast-interp"], + ) + assert rc == 0 + assert "crash" in out, f"expected $f0 -> crash:\n{out}" + assert "app_main" in out, f"expected $f1 -> app_main:\n{out}" + + +def test_no_addr_mode(build_wasm, run_addr2line, fixtures_dir): + """--no-addr resolves by function name, not address. + + Same fixture as fast-interp: $f0 -> crash, $f1 -> app_main. Under + --no-addr addr2line should additionally surface the decl-line from + DWARF (not just the function name). + """ + wasm = build_wasm(["simple.c"], ["-O0", "-g"]) + out, err, rc = run_addr2line( + wasm, fixtures_dir / "fast_interp_stack.txt", + extra_args=["--no-addr"], + ) + assert rc == 0 + assert "crash" in out, f"expected $f0 -> crash:\n{out}" + assert "app_main" in out, f"expected $f1 -> app_main:\n{out}" + assert "simple.c" in out, f"expected DWARF source file:\n{out}" + + +def test_offset_zero_fallback(build_wasm, run_addr2line): + """Runtime offset=0 falls back gracefully (no assertion crash).""" + wasm = build_wasm(["simple.c"], ["-O0", "-g"]) + out, err, rc = run_addr2line(wasm, ["#00: 0x0000 - app_main"]) + assert rc == 0 + # Output should mention app_main even with no useful offset + assert "app_main" in out + + +def test_empty_input(build_wasm, run_addr2line, fixtures_dir): + """Empty call stack file produces clean exit, minimal output.""" + wasm = build_wasm(["simple.c"], ["-O0", "-g"]) + out, err, rc = run_addr2line(wasm, fixtures_dir / "empty_stack.txt") + assert rc == 0 + + +# --- Cross-cutting ------------------------------------------------------- + + + +@pytest.mark.slow +def test_build_subprogram_intervals_filters_invalid_dies( + build_wasm, wasm_opt_pass, wasi_sdk, addr2line_script +): + """Guard against wasm-opt DCE ghosts and DW_AT_declaration DIEs + leaking into the interval table. + + Empirically, `wasm-opt -Oz -g` leaves DIEs for dead-code-eliminated + compiler-rt / libm helpers behind with low_pc=0 and tiny/empty + high_pc. Those must NOT appear as entries in the interval table. + """ + import sys + sys.path.insert(0, str(addr2line_script.parent)) + from addr2line import build_subprogram_intervals + + # Build a wasi-libc-linked wasm and post-process with wasm-opt -Oz -g + # to trigger the DCE-ghost pattern. + raw = build_wasm(["simple.c"], ["-Oz", "-g", "-flto"]) + debug = wasm_opt_pass(raw, ["-Oz", "-g"]) + dwarf_dump = wasi_sdk / "bin" / "llvm-dwarfdump" + + intervals = build_subprogram_intervals(dwarf_dump, debug) + assert intervals, "expected at least one subprogram in the interval table" + + # No entry may have low_pc == 0 (structurally impossible for real code: + # the wasm Code section never starts at file offset 0). + zero_lowpc = [(lo, hi, nm) for (lo, hi, nm) in intervals if lo == 0] + assert not zero_lowpc, ( + f"interval table contains low_pc=0 entries (wasm-opt DCE ghosts): " + f"{zero_lowpc[:5]}" + ) + + # No entry may have high_pc <= low_pc. + empty = [(lo, hi, nm) for (lo, hi, nm) in intervals if hi <= lo] + assert not empty, f"interval table contains empty ranges: {empty[:5]}" +