From cc18e64e1049200c3bae6444926224cbfd4854ac Mon Sep 17 00:00:00 2001 From: Cemberk Date: Wed, 12 Aug 2026 16:56:44 -0500 Subject: [PATCH] test(profiling): verify the TraceLens and dynolog tools without a GPU Neither tool could be exercised in CI: TraceLens pins protobuf and xprof and reports only on recorded GPU traces, and dynolog ships as a GitHub release asset and attaches to a running PyTorch workload. Stand-ins for both now let the real scripts run against fabricated trace artifacts, covering the half of the integration madengine owns -- which report generator each trace kind is routed to, the flags it receives, the retry loop that waits for the workload to register, and the guarantee that a failed analysis never fails a model run. Two defects the new tests found: The analyzer's module fallback ran main() and discarded its return value, so whenever TraceLens' console scripts were absent every failed report was recorded as a success. It now exits with that value, the way pip's console scripts do. Stopping dynolog burned its full 20s grace period and then claimed the daemon "did not stop gracefully". Both processes are reparented to PID 1 when the pre-script exits, and a container's PID 1 is the model command rather than an init that reaps children, so a terminated process lingers as a zombie that kill -0 still reports as alive. Co-authored-by: Cursor --- .../common/post_scripts/dynolog_stop.sh | 24 +- .../scripts/common/tools/tracelens_analyze.py | 19 +- tests/e2e/test_dynolog_dummy_pipeline.py | 358 ++++++++++ tests/e2e/test_tracelens_dummy_pipeline.py | 654 ++++++++++++++++++ .../TraceLens/Reporting/__init__.py | 4 + .../TraceLens/Reporting/_dummy.py | 222 ++++++ .../Reporting/compare_perf_reports_pytorch.py | 8 + ...te_multi_rank_collective_report_pytorch.py | 8 + ...nerate_perf_report_pftrace_hip_activity.py | 8 + .../generate_perf_report_pftrace_hip_api.py | 8 + ...enerate_perf_report_pftrace_memory_copy.py | 8 + .../Reporting/generate_perf_report_pytorch.py | 8 + .../Reporting/generate_perf_report_rocprof.py | 8 + .../dummy_tracelens/TraceLens/__init__.py | 12 + 14 files changed, 1339 insertions(+), 10 deletions(-) create mode 100644 tests/e2e/test_dynolog_dummy_pipeline.py create mode 100644 tests/e2e/test_tracelens_dummy_pipeline.py create mode 100644 tests/fixtures/dummy_tracelens/TraceLens/Reporting/__init__.py create mode 100644 tests/fixtures/dummy_tracelens/TraceLens/Reporting/_dummy.py create mode 100644 tests/fixtures/dummy_tracelens/TraceLens/Reporting/compare_perf_reports_pytorch.py create mode 100644 tests/fixtures/dummy_tracelens/TraceLens/Reporting/generate_multi_rank_collective_report_pytorch.py create mode 100644 tests/fixtures/dummy_tracelens/TraceLens/Reporting/generate_perf_report_pftrace_hip_activity.py create mode 100644 tests/fixtures/dummy_tracelens/TraceLens/Reporting/generate_perf_report_pftrace_hip_api.py create mode 100644 tests/fixtures/dummy_tracelens/TraceLens/Reporting/generate_perf_report_pftrace_memory_copy.py create mode 100644 tests/fixtures/dummy_tracelens/TraceLens/Reporting/generate_perf_report_pytorch.py create mode 100644 tests/fixtures/dummy_tracelens/TraceLens/Reporting/generate_perf_report_rocprof.py create mode 100644 tests/fixtures/dummy_tracelens/TraceLens/__init__.py diff --git a/src/madengine/scripts/common/post_scripts/dynolog_stop.sh b/src/madengine/scripts/common/post_scripts/dynolog_stop.sh index 7fbc9472..7f9ebb22 100644 --- a/src/madengine/scripts/common/post_scripts/dynolog_stop.sh +++ b/src/madengine/scripts/common/post_scripts/dynolog_stop.sh @@ -22,6 +22,24 @@ if [ ! -f "$DYNOLOG_START_FILE" ]; then exit 0 fi +# Both processes are started in the background by the pre-script, so once it exits +# they are reparented to PID 1, which in a container is the model command rather +# than an init that reaps children. A terminated process therefore lingers as a +# zombie and `kill -0` keeps succeeding, so the process state has to be checked as +# well, or every stop would burn the full grace period below. +is_running() { + local pid=$1 + kill -0 "$pid" 2>/dev/null || return 1 + if [ -r "/proc/$pid/stat" ]; then + # State is the field after the command name, which itself may contain + # spaces and is always parenthesised. + local state + state=$(sed 's/.*) //' "/proc/$pid/stat" 2>/dev/null | cut -d' ' -f1) + [ "$state" = "Z" ] && return 1 + fi + return 0 +} + stop_pid() { local name=$1 local pid_file=$2 @@ -31,14 +49,14 @@ stop_pid() { fi local pid pid=$(cat "$pid_file") - if kill -0 "$pid" 2>/dev/null; then + if is_running "$pid"; then kill -TERM "$pid" 2>/dev/null || true local waited=0 - while kill -0 "$pid" 2>/dev/null && [ $waited -lt 20 ]; do + while is_running "$pid" && [ $waited -lt 20 ]; do sleep 0.5 waited=$((waited + 1)) done - if kill -0 "$pid" 2>/dev/null; then + if is_running "$pid"; then echo "⚠️ $name did not stop gracefully, force killing..." kill -9 "$pid" 2>/dev/null || true fi diff --git a/src/madengine/scripts/common/tools/tracelens_analyze.py b/src/madengine/scripts/common/tools/tracelens_analyze.py index bfe5606e..63e30587 100644 --- a/src/madengine/scripts/common/tools/tracelens_analyze.py +++ b/src/madengine/scripts/common/tools/tracelens_analyze.py @@ -207,6 +207,8 @@ def _build_command(python: str, script_name: str, args: Sequence[str]) -> List[s Prefers the installed console script (clearer logs, honours the package's own entry-point wiring) and falls back to importing the module's ``main``. + The fallback exits with ``main()``'s return value, the same way the console + scripts pip generates do, so a report failure is not silently swallowed. """ bindir = os.path.dirname(os.path.abspath(python)) candidate = os.path.join(bindir, script_name) @@ -216,7 +218,12 @@ def _build_command(python: str, script_name: str, args: Sequence[str]) -> List[s if on_path: return [on_path, *args] module = _ENTRY_POINTS[script_name] - return [python, "-c", f"from {module} import main; main()", *args] + return [ + python, + "-c", + f"import sys; from {module} import main; sys.exit(main())", + *args, + ] def _run(command: Sequence[str], cwd: Optional[str] = None) -> Tuple[int, str]: @@ -461,9 +468,9 @@ def analyze( code, output = _run(_build_command(interpreter, tool, args)) results.append( { - "trace_file": os.path.relpath(trace, root) - if os.path.exists(trace) - else trace, + "trace_file": ( + os.path.relpath(trace, root) if os.path.exists(trace) else trace + ), "kind": kind, "tracelens_tool": tool, "status": "SUCCESS" if code == 0 else "FAILURE", @@ -605,9 +612,7 @@ def main(argv: Optional[Sequence[str]] = None) -> int: ) failed = summary["status"] != "SUCCESS" elif args.discover_only: - traces, unsupported = discover_traces( - args.root, exclude_dirs=[args.output_dir] - ) + traces, unsupported = discover_traces(args.root, exclude_dirs=[args.output_dir]) for kind, paths in sorted(traces.items()): for path in paths: print(f"{kind}\t{path}") diff --git a/tests/e2e/test_dynolog_dummy_pipeline.py b/tests/e2e/test_dynolog_dummy_pipeline.py new file mode 100644 index 00000000..5084a087 --- /dev/null +++ b/tests/e2e/test_dynolog_dummy_pipeline.py @@ -0,0 +1,358 @@ +"""Whole-pipeline `torch_profiler_dynolog` tests that run wherever CI runs. + +On-demand PyTorch tracing needs three things madengine cannot provide in CI: the +dynolog daemon (published only as a GitHub release asset), a GPU, and a PyTorch +workload for the daemon to attach to. These tests substitute a dummy ``dynolog`` +and ``dyno`` on ``PATH`` and then run the tool's real scripts against them, which +covers what madengine owns: the daemon lifecycle, the retry loop that waits for +the workload to register, the flags the trace request carries, and the +diagnostics printed when nothing was captured. + +The stand-in for ``dyno`` also writes the trace file, which in a real run is +written by the workload's own Kineto instance. + +Copyright (c) Advanced Micro Devices, Inc. All rights reserved. +""" + +# built-in modules +import contextlib +import os +import shutil +import signal +import subprocess +import time +from pathlib import Path + +# third-party modules +import pytest + +# project modules +from madengine.utils.path_utils import get_madengine_root + +pytestmark = pytest.mark.skipif( + os.name != "posix", reason="drives the Linux in-container profiling scripts" +) + +COMMON_SCRIPTS = get_madengine_root() / "scripts" / "common" +START_SCRIPT = COMMON_SCRIPTS / "pre_scripts" / "dynolog_start.sh" +TRIGGER_SCRIPT = COMMON_SCRIPTS / "tools" / "dynolog_trigger.sh" +STOP_SCRIPT = COMMON_SCRIPTS / "post_scripts" / "dynolog_stop.sh" + +# The scripts hand off to each other through fixed paths, because the pre-script, +# the trigger, and the post-script are three separate processes in a container. +DYNOLOG_PID_FILE = Path("/tmp/madengine_dynolog.pid") +TRIGGER_PID_FILE = Path("/tmp/madengine_dynolog_trigger.pid") +STARTED_FILE = Path("/tmp/madengine_dynolog.started") +RESULT_FILE = Path("/tmp/madengine_dynolog_trigger.result") +HANDOFF_FILES = ( + DYNOLOG_PID_FILE, + TRIGGER_PID_FILE, + STARTED_FILE, + RESULT_FILE, + Path("/tmp/madengine_dynolog.log"), + Path("/tmp/madengine_dynolog_trigger.log"), +) + +# A daemon that stays up until it is signalled, so the stop script has something +# real to terminate. +DYNOLOG_STUB = """#!/bin/sh +echo "dummy dynolog listening: $*" +while : ; do sleep 0.2; done +""" + +# Records each trace request, rejects the first $DUMMY_DYNO_REJECTS of them the +# way `--fail-on-no-process` does before the workload has registered, then +# accepts and writes the trace Kineto would have written. +DYNO_STUB = """#!/bin/sh +printf '%s\\n' "$*" >> "$DUMMY_DYNO_LOG" +requests=$(wc -l < "$DUMMY_DYNO_LOG") +if [ "$requests" -le "${DUMMY_DYNO_REJECTS:-0}" ]; then + echo "No processes were matched, exiting" >&2 + exit 1 +fi +log_file="" +previous="" +for arg in "$@"; do + if [ "$previous" = "--log-file" ]; then log_file=$arg; fi + previous=$arg +done +if [ "${DUMMY_DYNO_WRITE_TRACE:-0}" = "1" ] && [ -n "$log_file" ]; then + # Kineto appends the process id to the requested filename. + printf '{"traceEvents": [], "schemaVersion": 1}' > "${log_file%.json}_4242.json" +fi +echo "response length: 1" +exit 0 +""" + + +class DummyDynolog: + """Dummy ``dynolog`` and ``dyno`` binaries, installed the way the .deb is. + + ``pre_scripts/trace.sh dynolog`` ends with both binaries on ``PATH``, which + is all the rest of the tool depends on. + """ + + def __init__(self, bin_dir: Path) -> None: + bin_dir.mkdir(parents=True) + self.bin_dir = bin_dir + self.log = bin_dir / "dyno_requests.log" + self.log.touch() + for name, body in (("dynolog", DYNOLOG_STUB), ("dyno", DYNO_STUB)): + script = bin_dir / name + script.write_text(body, encoding="utf-8") + script.chmod(0o755) + + def environ(self, on_path: bool = True, **overrides: str) -> dict: + env = dict(os.environ, DUMMY_DYNO_LOG=str(self.log)) + if on_path: + env["PATH"] = f"{self.bin_dir}{os.pathsep}{env.get('PATH', '')}" + env.update(overrides) + return env + + def requests(self) -> list: + """Return the ``dyno gputrace`` requests made so far, one per line.""" + return [ + line for line in self.log.read_text(encoding="utf-8").splitlines() if line + ] + + def cleanup(self) -> None: + """Kill anything left running and clear the handoff files.""" + for pid_file in (TRIGGER_PID_FILE, DYNOLOG_PID_FILE): + if pid_file.is_file(): + with contextlib.suppress(ValueError, OSError): + os.kill(int(pid_file.read_text().strip()), signal.SIGKILL) + for path in HANDOFF_FILES: + with contextlib.suppress(OSError): + path.unlink() + + +@pytest.fixture +def dummy_dynolog(tmp_path): + dummy = DummyDynolog(tmp_path / "dynolog-bin") + dummy.cleanup() + yield dummy + dummy.cleanup() + + +@pytest.fixture +def workdir(tmp_path): + """A working directory with scripts/common staged, as a run has.""" + work = tmp_path / "workdir" + (work / "scripts").mkdir(parents=True) + shutil.copytree(COMMON_SCRIPTS, work / "scripts" / "common") + return work + + +def run_script(script: Path, work: Path, env: dict, timeout: int = 120): + return subprocess.run( + ["bash", str(script)], + cwd=work, + env=env, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + encoding="utf-8", + errors="replace", + timeout=timeout, + ) + + +def alive(pid_file: Path) -> bool: + if not pid_file.is_file(): + return False + try: + os.kill(int(pid_file.read_text().strip()), 0) + except (ValueError, OSError): + return False + return True + + +def wait_until(predicate, timeout: float = 10.0) -> bool: + deadline = time.time() + timeout + while time.time() < deadline: + if predicate(): + return True + time.sleep(0.2) + return predicate() + + +class TestDaemonLifecycle: + """Starting and stopping the daemon around a model run.""" + + def test_start_arms_the_daemon_and_the_trigger(self, workdir, dummy_dynolog): + """The pre-script leaves a daemon running and a trigger waiting.""" + result = run_script( + START_SCRIPT, + workdir, + dummy_dynolog.environ(TORCH_PROFILE_WARMUP_S="30"), + ) + + assert result.returncode == 0, result.stdout + assert "dynolog daemon started" in result.stdout + assert "trace trigger armed" in result.stdout + assert alive(DYNOLOG_PID_FILE), result.stdout + assert alive(TRIGGER_PID_FILE), result.stdout + # The post-script uses this marker to tell "not started" from "failed". + assert STARTED_FILE.is_file() + # Kineto writes into this directory itself, so it must exist up front. + assert (workdir / "torch_profiler_output").is_dir() + + def test_start_without_the_daemon_installed_fails(self, workdir, dummy_dynolog): + """Without the pre-script's install step there is nothing to start.""" + result = run_script(START_SCRIPT, workdir, dummy_dynolog.environ(on_path=False)) + + assert result.returncode != 0 + assert "pre-script must run first" in result.stdout + assert not STARTED_FILE.is_file() + + def test_stop_leaves_nothing_running(self, workdir, dummy_dynolog): + """The post-script has to reap both processes; they outlive the workload.""" + run_script( + START_SCRIPT, workdir, dummy_dynolog.environ(TORCH_PROFILE_WARMUP_S="30") + ) + + result = run_script(STOP_SCRIPT, workdir, dummy_dynolog.environ()) + + assert result.returncode == 0, result.stdout + assert "dynolog cleanup complete" in result.stdout + assert wait_until(lambda: not alive(DYNOLOG_PID_FILE)), result.stdout + assert wait_until(lambda: not alive(TRIGGER_PID_FILE)), result.stdout + assert not STARTED_FILE.is_file() + # Both processes answer SIGTERM. Reaching the force-kill path instead + # would cost the full grace period at the end of every profiled run. + assert "did not stop gracefully" not in result.stdout + + def test_stop_without_start_is_a_no_op(self, workdir, dummy_dynolog): + """Stacking the tool on a failed run must not turn into a second failure.""" + result = run_script(STOP_SCRIPT, workdir, dummy_dynolog.environ()) + + assert result.returncode == 0, result.stdout + assert "dynolog was not started" in result.stdout + + +class TestTraceRequest: + """The trigger, which is what actually asks for a trace.""" + + @staticmethod + def run_trigger(work: Path, dummy: DummyDynolog, **env: str): + settings = dict( + TORCH_PROFILE_WARMUP_S="0", + TORCH_PROFILE_RETRY_INTERVAL_S="0", + TORCH_PROFILE_MAX_ATTEMPTS="5", + ) + settings.update(env) + return run_script(TRIGGER_SCRIPT, work, dummy.environ(**settings)) + + def test_trigger_retries_until_the_workload_registers(self, workdir, dummy_dynolog): + """A trace cannot be requested until PyTorch has registered, so we poll.""" + result = self.run_trigger(workdir, dummy_dynolog, DUMMY_DYNO_REJECTS="2") + + assert result.returncode == 0, result.stdout + assert len(dummy_dynolog.requests()) == 3 + assert "accepted on attempt 3" in result.stdout + assert RESULT_FILE.read_text().strip() == "accepted" + + def test_request_carries_the_data_tracelens_needs(self, workdir, dummy_dynolog): + """Shapes, stacks, and modules are what make the TraceLens reports useful.""" + self.run_trigger(workdir, dummy_dynolog) + + request = dummy_dynolog.requests()[0] + for flag in ( + "--record-shapes", + "--with-stacks", + "--with-modules", + "--iterations 5", + "--process-limit 64", + "--fail-on-no-process", + ): + assert flag in request, request + # An absolute path, because the workload's working directory is its own. + assert ( + f"--log-file {workdir}/torch_profiler_output/libkineto_trace.json" + in request + ) + + def test_disabling_iterations_switches_to_a_timed_capture( + self, workdir, dummy_dynolog + ): + """Iteration counting needs an optimizer step, which not every model has.""" + self.run_trigger( + workdir, + dummy_dynolog, + TORCH_PROFILE_ITERATIONS="0", + TORCH_PROFILE_DURATION_MS="750", + ) + + request = dummy_dynolog.requests()[0] + assert "--duration-ms 750" in request + assert "--iterations" not in request + + def test_optional_capture_flags_follow_their_env_vars(self, workdir, dummy_dynolog): + """The expensive captures are opt-in, and the default ones opt-out.""" + self.run_trigger( + workdir, + dummy_dynolog, + TORCH_PROFILE_RECORD_SHAPES="0", + TORCH_PROFILE_WITH_STACKS="0", + TORCH_PROFILE_WITH_MODULES="0", + TORCH_PROFILE_WITH_FLOPS="1", + TORCH_PROFILE_PROFILE_MEMORY="1", + ) + + request = dummy_dynolog.requests()[0] + assert "--record-shapes" not in request + assert "--with-stacks" not in request + assert "--with-modules" not in request + assert "--with-flops" in request + assert "--profile-memory" in request + + def test_giving_up_says_what_to_check(self, workdir, dummy_dynolog): + """Nothing registering is the common failure, so it must be self-explaining.""" + result = self.run_trigger( + workdir, + dummy_dynolog, + DUMMY_DYNO_REJECTS="99", + TORCH_PROFILE_MAX_ATTEMPTS="2", + ) + + assert result.returncode != 0 + assert len(dummy_dynolog.requests()) == 2 + assert "gave up after 2 attempts" in result.stdout + assert "KINETO_USE_DAEMON=1" in result.stdout + assert "TORCH_PROFILE_WARMUP_S" in result.stdout + assert RESULT_FILE.read_text().strip() == "no_process" + + +class TestCaptureReporting: + """What the post-script tells the user about the traces it found.""" + + def test_captured_traces_are_reported(self, workdir, dummy_dynolog): + """A successful capture is confirmed with a count, per rank.""" + run_script( + START_SCRIPT, workdir, dummy_dynolog.environ(TORCH_PROFILE_WARMUP_S="30") + ) + TestTraceRequest.run_trigger(workdir, dummy_dynolog, DUMMY_DYNO_WRITE_TRACE="1") + + result = run_script(STOP_SCRIPT, workdir, dummy_dynolog.environ()) + + assert result.returncode == 0, result.stdout + assert "Captured 1 torch.profiler trace(s)" in result.stdout + traces = list((workdir / "torch_profiler_output").glob("*.json")) + assert [p.name for p in traces] == ["libkineto_trace_4242.json"] + + def test_missing_traces_are_explained(self, workdir, dummy_dynolog): + """An empty output directory needs a reason, not just a warning.""" + run_script( + START_SCRIPT, workdir, dummy_dynolog.environ(TORCH_PROFILE_WARMUP_S="30") + ) + TestTraceRequest.run_trigger( + workdir, + dummy_dynolog, + DUMMY_DYNO_REJECTS="99", + TORCH_PROFILE_MAX_ATTEMPTS="1", + ) + + result = run_script(STOP_SCRIPT, workdir, dummy_dynolog.environ()) + + assert result.returncode == 0, result.stdout + assert "No torch.profiler traces were captured" in result.stdout + assert "never matched a PyTorch process" in result.stdout diff --git a/tests/e2e/test_tracelens_dummy_pipeline.py b/tests/e2e/test_tracelens_dummy_pipeline.py new file mode 100644 index 00000000..444a5f2a --- /dev/null +++ b/tests/e2e/test_tracelens_dummy_pipeline.py @@ -0,0 +1,654 @@ +"""Whole-pipeline TraceLens tests that run wherever CI runs. + +The real TraceLens pins ``protobuf`` and ``xprof`` and only produces reports from +recorded GPU traces, so CI can neither install it nor feed it real input. These +tests put the dummy TraceLens in ``tests/fixtures/dummy_tracelens`` on +``PYTHONPATH`` in its place and then drive the integration for real: fabricated +trace artifacts, the packaged analyzer, the in-container ``tracelens`` +post-script, and ``madengine report tracelens``. No GPU, Docker, or network. + +This covers the half of the integration madengine owns — which report generator +each trace kind is routed to, the flags it receives, where reports land, what the +summary records, and the guarantee that a failed analysis never fails a model +run. It cannot confirm that upstream TraceLens still accepts those flags. + +Substituting TraceLens is not just a convenience: the real one reports only on +kernels it can link back to the runtime calls that launched them, so no +fabricated trace will produce a report, however well shaped. The trace artifacts +below are therefore structural stand-ins, and the GPU-gated tests in +test_tracelens_workflows.py are what exercise real analysis. + +Copyright (c) Advanced Micro Devices, Inc. All rights reserved. +""" + +# built-in modules +import csv +import gzip +import importlib.util +import json +import os +import shutil +import subprocess +import sys +from pathlib import Path + +# third-party modules +import pytest + +# project modules +from madengine.utils.path_utils import get_madengine_root + +# The in-container half of the integration is shell, and the venv layout the +# analyzer looks into is POSIX-only. +pytestmark = pytest.mark.skipif( + os.name != "posix", reason="drives Linux container scripts and a POSIX venv layout" +) + +DUMMY_TRACELENS = Path(__file__).resolve().parents[1] / "fixtures" / "dummy_tracelens" +COMMON_SCRIPTS = get_madengine_root() / "scripts" / "common" +ANALYZER = COMMON_SCRIPTS / "tools" / "tracelens_analyze.py" +POST_SCRIPT = COMMON_SCRIPTS / "post_scripts" / "tracelens.sh" + +PYTORCH_REPORT = "TraceLens_generate_perf_report_pytorch" +ROCPROF_REPORT = "TraceLens_generate_perf_report_rocprof" +PFTRACE_REPORTS = ( + "TraceLens_generate_perf_report_pftrace_hip_activity", + "TraceLens_generate_perf_report_pftrace_hip_api", + "TraceLens_generate_perf_report_pftrace_memory_copy", +) +COLLECTIVE_REPORT = "TraceLens_generate_multi_rank_collective_report_pytorch" + + +def _entry_points() -> dict: + """Return the analyzer's entry point -> module map. + + Read from the analyzer itself so the dummy console scripts are always named + after what madengine actually looks for. + """ + spec = importlib.util.spec_from_file_location("tracelens_analyze", ANALYZER) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module._ENTRY_POINTS + + +ENTRY_POINTS = _entry_points() + + +class DummyTraceLens: + """The dummy package, installed the way the pre-script installs the real one. + + ``pre_scripts/trace.sh tracelens`` builds an isolated venv whose ``bin/`` + holds a python and TraceLens' report console scripts, and the analyzer + prefers those console scripts over importing the modules. Reproducing that + layout means these tests exercise the same lookup a real run does. + """ + + def __init__(self, root: Path, console_scripts: bool = True) -> None: + self.root = root + self.log = root / "invocations.jsonl" + bin_dir = root / "bin" + bin_dir.mkdir(parents=True) + + self.python = bin_dir / "python3" + self._write_executable( + self.python, f'#!/bin/sh\nexec "{sys.executable}" "$@"\n' + ) + if console_scripts: + for name, module in ENTRY_POINTS.items(): + self._write_executable( + bin_dir / name, + f"#!{sys.executable}\n" + "import sys\n" + f"from {module} import main\n" + "sys.exit(main())\n", + ) + self._vars = { + "DUMMY_TRACELENS_LOG": str(self.log), + "TRACELENS_VENV": str(root), + } + + @staticmethod + def _write_executable(path: Path, text: str) -> None: + path.write_text(text, encoding="utf-8") + path.chmod(0o755) + + def fail(self, *entry_points: str) -> None: + """Make the named report generators fail. ``"all"`` fails every one.""" + self._vars["DUMMY_TRACELENS_FAIL"] = ",".join(entry_points) + + def environ(self, **overrides: str) -> dict: + """Return an environment in which the dummy shadows any real TraceLens.""" + existing = os.environ.get("PYTHONPATH", "") + python_path = os.pathsep.join(p for p in (str(DUMMY_TRACELENS), existing) if p) + env = dict(os.environ, PYTHONPATH=python_path, **self._vars) + env.update(overrides) + return env + + def invocations(self, entry_point: str = "") -> list: + """Return the report generator calls recorded so far.""" + if not self.log.is_file(): + return [] + records = [ + json.loads(line) + for line in self.log.read_text(encoding="utf-8").splitlines() + if line.strip() + ] + if entry_point: + records = [r for r in records if r["entry_point"] == entry_point] + return records + + def reports_run(self) -> list: + """Return the entry point names that ran, in order.""" + return [record["entry_point"] for record in self.invocations()] + + +@pytest.fixture +def dummy_tracelens(tmp_path): + return DummyTraceLens(tmp_path / "tracelens-venv") + + +def write_chrome_trace(path: Path) -> Path: + """Write a minimal Chrome Trace Event document, as Kineto does. + + Gzips the payload when ``path`` ends in ``.gz``, which is how + ``tensorboard_trace_handler`` writes traces. + """ + payload = { + "schemaVersion": 1, + "distributedInfo": {"backend": "nccl", "rank": 0, "world_size": 1}, + "traceEvents": [ + { + "ph": "X", + "cat": "kernel", + "name": "void gemm_kernel(float*, float*)", + "pid": 1, + "tid": 7, + "ts": 100, + "dur": 42, + "args": {"stream": 7, "grid": [8, 1, 1], "block": [256, 1, 1]}, + }, + { + "ph": "X", + "cat": "gpu_memcpy", + "name": "Memcpy DtoH", + "pid": 1, + "tid": 7, + "ts": 200, + "dur": 8, + "args": {"bytes": 4096}, + }, + ], + } + path.parent.mkdir(parents=True, exist_ok=True) + if path.name.endswith(".gz"): + with gzip.open(path, "wt", encoding="utf-8") as handle: + json.dump(payload, handle) + else: + path.write_text(json.dumps(payload), encoding="utf-8") + return path + + +def write_rocprof_json(path: Path) -> Path: + """Write a rocprofv3 JSON result document, as ``rocprofv3_lightweight`` does. + + Shaped like the real output down to the ``buffer_records`` TraceLens reads + kernel dispatches from, but structural only: see the module docstring. + """ + payload = { + "rocprofiler-sdk-tool": [ + { + "metadata": {"pid": 1234}, + "agents": [{"id": {"handle": 0}, "type": 2, "name": "gfx942"}], + "kernel_symbols": [ + {"id": 1, "formatted_kernel_name": "gemm_kernel(float*, float*)"} + ], + "buffer_records": { + "kernel_dispatch": [ + { + "correlation_id": {"internal": 1}, + "start_timestamp": 1000, + "end_timestamp": 43000, + "dispatch_info": { + "kernel_id": 1, + "agent_id": {"handle": 0}, + }, + } + ], + "memory_copy": [], + }, + } + ] + } + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(payload), encoding="utf-8") + return path + + +def write_pftrace(path: Path) -> Path: + """Write a placeholder Perfetto trace, as ``rocprofv3_perfetto`` does.""" + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(b"\x0a\x0fdummy-perfetto-trace") + return path + + +def write_rocprof_db(path: Path) -> Path: + """Write rocprofv3's default SQLite output, which TraceLens cannot read.""" + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(b"SQLite format 3\x00") + return path + + +@pytest.fixture +def profiled_run(tmp_path): + """A directory shaped like the one a profiled madengine run leaves behind. + + Holds one artifact of every kind madengine's profiling tools can produce, + including the SQLite output TraceLens cannot read. + """ + work = tmp_path / "workdir" + write_chrome_trace(work / "torch_profiler_output" / "libkineto_trace_1234.json") + write_rocprof_json(work / "rocprof_output" / "1234_results.json") + write_pftrace(work / "rocprof_output" / "model.pftrace") + write_rocprof_db(work / "rocprof_output" / "1234_results.db") + return work + + +def run_analyzer( + work: Path, dummy: DummyTraceLens, *extra: str, output_dir: str = "tracelens_output" +) -> subprocess.CompletedProcess: + """Run the packaged analyzer from ``work``, as the post-script does.""" + return subprocess.run( + [ + sys.executable, + str(ANALYZER), + "--root", + ".", + "--output-dir", + output_dir, + "--python", + str(dummy.python), + "--json-summary", + f"{output_dir}/tracelens_summary.json", + *extra, + ], + cwd=work, + env=dummy.environ(), + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + encoding="utf-8", + errors="replace", + timeout=300, + ) + + +def summary_rows(work: Path, output_dir: str = "tracelens_output") -> list: + """Read the analyzer's summary CSV into a list of dict rows.""" + path = work / output_dir / "tracelens_summary.csv" + assert path.is_file(), f"analyzer wrote no summary at {path}" + with open(path, newline="", encoding="utf-8") as handle: + return list(csv.DictReader(handle)) + + +def only(records: list, entry_point: str) -> dict: + """Return the single recorded invocation of ``entry_point``.""" + matches = [r for r in records if r["entry_point"] == entry_point] + assert len(matches) == 1, f"expected one {entry_point} call, got {len(matches)}" + return matches[0] + + +class TestAnalyzerWithDummyTraceLens: + """Every trace kind a run can produce, analyzed by the real analyzer.""" + + def test_each_trace_kind_is_routed_to_its_report_generator( + self, profiled_run, dummy_tracelens + ): + """A Kineto trace, a rocprofv3 JSON, and a pftrace pick different reports.""" + result = run_analyzer(profiled_run, dummy_tracelens) + + assert result.returncode == 0, result.stdout + assert sorted(dummy_tracelens.reports_run()) == sorted( + [PYTORCH_REPORT, ROCPROF_REPORT, *PFTRACE_REPORTS] + ) + records = dummy_tracelens.invocations() + assert only(records, PYTORCH_REPORT)["args"]["profile_json_path"].endswith( + "libkineto_trace_1234.json" + ) + assert only(records, ROCPROF_REPORT)["args"]["profile_json_path"].endswith( + "1234_results.json" + ) + for report in PFTRACE_REPORTS: + assert only(records, report)["args"]["trace_path"].endswith(".pftrace") + + def test_reports_are_written_into_the_output_directory( + self, profiled_run, dummy_tracelens + ): + """Each report generator's workbook and per-sheet CSVs land together.""" + run_analyzer(profiled_run, dummy_tracelens) + + output_dir = profiled_run / "tracelens_output" + written = sorted(p.name for p in output_dir.iterdir()) + assert "tracelens_summary.csv" in written + assert "tracelens_summary.json" in written + # One workbook plus one CSV directory per PyTorch trace analyzed. + pytorch_stem = "torch_profiler_output_libkineto_trace_1234" + assert (output_dir / f"{pytorch_stem}.xlsx").is_file(), written + assert (output_dir / f"{pytorch_stem}_csv" / "kernel_summary.csv").is_file() + # The pftrace activity report is markdown rather than a workbook. + assert list(output_dir.glob("*_activity.md")), written + + def test_summary_records_each_report_and_the_unreadable_db( + self, profiled_run, dummy_tracelens + ): + """The summary is the run's record: five reports, one skipped artifact.""" + run_analyzer(profiled_run, dummy_tracelens) + + rows = summary_rows(profiled_run) + assert [r["status"] for r in rows].count("SUCCESS") == 5 + skipped = [r for r in rows if r["status"] == "SKIPPED"] + assert len(skipped) == 1 + assert skipped[0]["trace_file"].endswith("1234_results.db") + # The skip has to say which preset the user should have run instead. + assert "rocprofv3_lightweight" in skipped[0]["detail"] + + def test_gzipped_kineto_trace_is_analyzed(self, tmp_path, dummy_tracelens): + """tensorboard_trace_handler's gzipped traces are picked up too.""" + work = tmp_path / "gz" + write_chrome_trace(work / "traces" / "worker0.pt.trace.json.gz") + + result = run_analyzer(work, dummy_tracelens) + + assert result.returncode == 0, result.stdout + record = only(dummy_tracelens.invocations(), PYTORCH_REPORT) + assert record["args"]["profile_json_path"].endswith(".pt.trace.json.gz") + + def test_per_rank_traces_add_a_collective_report(self, tmp_path, dummy_tracelens): + """Several ranks' traces also produce the multi-rank collective report.""" + work = tmp_path / "distributed" + for rank in (0, 1): + write_chrome_trace( + work / "torch_profiler_output" / f"libkineto_trace_rank{rank}_99.json" + ) + + result = run_analyzer(work, dummy_tracelens) + + assert result.returncode == 0, result.stdout + assert dummy_tracelens.reports_run().count(PYTORCH_REPORT) == 2 + collective = only(dummy_tracelens.invocations(), COLLECTIVE_REPORT) + # The dummy fails unless the glob really matched both per-rank traces. + assert collective["args"]["world_size"] == "2" + assert collective["args"]["use_multiprocessing"] is True + + def test_single_rank_run_skips_the_collective_report( + self, profiled_run, dummy_tracelens + ): + """One rank cannot have collectives to compare, so that report is skipped.""" + run_analyzer(profiled_run, dummy_tracelens) + + assert COLLECTIVE_REPORT not in dummy_tracelens.reports_run() + + def test_gpu_arch_enables_roofline_classification( + self, profiled_run, dummy_tracelens + ): + """--gpu-arch reaches the PyTorch report as TraceLens' arch flag.""" + run_analyzer(profiled_run, dummy_tracelens, "--gpu-arch", "MI300X") + + record = only(dummy_tracelens.invocations(), PYTORCH_REPORT) + assert record["args"]["gpu_arch_platform"] == "MI300X" + + def test_mode_restricts_analysis_to_one_trace_kind( + self, profiled_run, dummy_tracelens + ): + """--mode rocprof leaves the Kineto trace and the pftrace alone.""" + run_analyzer(profiled_run, dummy_tracelens, "--mode", "rocprof") + + assert dummy_tracelens.reports_run() == [ROCPROF_REPORT] + + def test_extra_flags_are_forwarded_to_every_report( + self, profiled_run, dummy_tracelens + ): + """Flags after ``--`` reach TraceLens untouched, for options we do not wrap.""" + run_analyzer(profiled_run, dummy_tracelens, "--", "--top_k_kernels", "5") + + for record in dummy_tracelens.invocations(): + assert record["extra"] == ["--top_k_kernels", "5"], record["entry_point"] + + def test_console_script_from_the_venv_is_preferred( + self, profiled_run, dummy_tracelens + ): + """With TraceLens' console scripts installed, they are what we run.""" + run_analyzer(profiled_run, dummy_tracelens) + + record = only(dummy_tracelens.invocations(), PYTORCH_REPORT) + assert record["argv0"] == str(dummy_tracelens.root / "bin" / PYTORCH_REPORT) + + def test_module_fallback_still_reports_failures(self, tmp_path, profiled_run): + """Without console scripts we import the module, and still see its exit code. + + The fallback runs ``main()`` in a fresh interpreter; if its return value + were dropped, every failed report would be recorded as a success. + """ + dummy = DummyTraceLens(tmp_path / "no-console-scripts", console_scripts=False) + dummy.fail(PYTORCH_REPORT) + + result = run_analyzer(profiled_run, dummy) + + assert result.returncode != 0, result.stdout + assert only(dummy.invocations(), PYTORCH_REPORT)["argv0"] == "-c" + pytorch_rows = [r for r in summary_rows(profiled_run) if r["kind"] == "pytorch"] + assert [r["status"] for r in pytorch_rows] == ["FAILURE"] + assert "forced failure" in pytorch_rows[0]["detail"] + + def test_one_failed_report_does_not_hide_the_others( + self, profiled_run, dummy_tracelens + ): + """A failing report is recorded as such; the rest still run and succeed.""" + dummy_tracelens.fail(ROCPROF_REPORT) + + result = run_analyzer(profiled_run, dummy_tracelens) + + assert result.returncode != 0, result.stdout + statuses = { + r["tracelens_tool"]: r["status"] for r in summary_rows(profiled_run) + } + assert statuses[ROCPROF_REPORT] == "FAILURE" + assert statuses[PYTORCH_REPORT] == "SUCCESS" + assert all(statuses[report] == "SUCCESS" for report in PFTRACE_REPORTS) + + def test_run_without_traces_says_which_tool_to_stack( + self, tmp_path, dummy_tracelens + ): + """No traces is a no-op with guidance, not a failure.""" + work = tmp_path / "empty" + work.mkdir() + + result = run_analyzer(work, dummy_tracelens) + + assert result.returncode == 0, result.stdout + assert "No supported trace artifacts found" in result.stdout + assert "torch_profiler_dynolog" in result.stdout + assert dummy_tracelens.reports_run() == [] + assert summary_rows(work) == [] + + +class TestInContainerPostScript: + """The `tracelens` tool's post-script, run the way a container runs it.""" + + @staticmethod + def stage_scripts(work: Path) -> None: + """Copy the analyzer to where ContainerRunner puts it during a run.""" + tools_dir = work / "scripts" / "common" / "tools" + tools_dir.mkdir(parents=True, exist_ok=True) + shutil.copy2(ANALYZER, tools_dir / ANALYZER.name) + + @classmethod + def run_post_script( + cls, work: Path, dummy: DummyTraceLens, **env: str + ) -> subprocess.CompletedProcess: + cls.stage_scripts(work) + return subprocess.run( + ["bash", str(POST_SCRIPT)], + cwd=work, + env=dummy.environ(**env), + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + encoding="utf-8", + errors="replace", + timeout=300, + ) + + def test_post_script_analyzes_the_traces_the_run_produced( + self, profiled_run, dummy_tracelens + ): + """The tool needs no configuration: it finds the traces and reports on them.""" + result = self.run_post_script(profiled_run, dummy_tracelens) + + assert result.returncode == 0, result.stdout + assert sorted(dummy_tracelens.reports_run()) == sorted( + [PYTORCH_REPORT, ROCPROF_REPORT, *PFTRACE_REPORTS] + ) + # tracelens_output/ is the directory the post-script hands to collection. + assert (profiled_run / "tracelens_output" / "tracelens_summary.csv").is_file() + assert (profiled_run / "tracelens_output" / "tracelens_summary.json").is_file() + + def test_failed_analysis_does_not_fail_the_model_run( + self, profiled_run, dummy_tracelens + ): + """Reporting is not the workload: TraceLens failing must not fail the run.""" + dummy_tracelens.fail("all") + + result = self.run_post_script(profiled_run, dummy_tracelens) + + assert result.returncode == 0, result.stdout + assert "WARNING: TraceLens analysis reported failures" in result.stdout + statuses = {r["status"] for r in summary_rows(profiled_run)} + assert "FAILURE" in statuses + + def test_missing_venv_fails_loudly(self, profiled_run, dummy_tracelens, tmp_path): + """A missing venv means the pre-script never ran, which is worth failing on.""" + result = self.run_post_script( + profiled_run, dummy_tracelens, TRACELENS_VENV=str(tmp_path / "absent") + ) + + assert result.returncode != 0 + assert "pre-script must run first" in result.stdout + + def test_mode_and_output_dir_are_configurable_by_env_var( + self, profiled_run, dummy_tracelens + ): + """The tool's env vars are how a user narrows analysis in a models.json.""" + result = self.run_post_script( + profiled_run, + dummy_tracelens, + TRACELENS_MODE="pytorch", + TRACELENS_OUTPUT_DIR="custom_tracelens", + ) + + assert result.returncode == 0, result.stdout + assert dummy_tracelens.reports_run() == [PYTORCH_REPORT] + rows = summary_rows(profiled_run, output_dir="custom_tracelens") + assert [r["kind"] for r in rows] == ["pytorch", "unsupported"] + + def test_gpu_arch_and_world_size_reach_tracelens(self, tmp_path, dummy_tracelens): + """Distributed runs pass rank count and arch through the tool's env vars.""" + work = tmp_path / "distributed" + for rank in (0, 1): + write_chrome_trace( + work / "torch_profiler_output" / f"libkineto_trace_rank{rank}_99.json" + ) + + result = self.run_post_script( + work, + dummy_tracelens, + TRACELENS_GPU_ARCH="MI300X", + TRACELENS_WORLD_SIZE="2", + ) + + assert result.returncode == 0, result.stdout + records = dummy_tracelens.invocations() + assert only(records, COLLECTIVE_REPORT)["args"]["world_size"] == "2" + assert all( + r["args"]["gpu_arch_platform"] == "MI300X" + for r in records + if r["entry_point"] == PYTORCH_REPORT + ) + + +class TestHostSideReportCommand: + """`madengine report tracelens` over artifacts already collected to the host.""" + + @staticmethod + def run_cli(dummy: DummyTraceLens, *args: str) -> subprocess.CompletedProcess: + """Run the CLI in a wide, colourless console so assertions survive wrapping.""" + env = dummy.environ( + COLUMNS="300", NO_COLOR="1", TERM="dumb", PYTHONIOENCODING="utf-8" + ) + return subprocess.run( + [sys.executable, "-m", "madengine.cli.app", "report", *args], + env=env, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + encoding="utf-8", + errors="replace", + timeout=300, + ) + + def test_report_tracelens_analyzes_collected_artifacts( + self, profiled_run, dummy_tracelens + ): + """The host path produces the same reports without touching the container.""" + output_dir = profiled_run / "host_reports" + + result = self.run_cli( + dummy_tracelens, + "tracelens", + "--root", + str(profiled_run), + "--output-dir", + str(output_dir), + ) + + assert result.returncode == 0, result.stdout + assert sorted(dummy_tracelens.reports_run()) == sorted( + [PYTORCH_REPORT, ROCPROF_REPORT, *PFTRACE_REPORTS] + ) + assert (output_dir / "tracelens_summary.csv").is_file() + # The rendered table is how a user sees what was analyzed. + assert "SUCCESS" in result.stdout + assert "SKIPPED" in result.stdout + + def test_report_tracelens_compare_diffs_two_reports( + self, profiled_run, dummy_tracelens + ): + """tracelens-compare turns two runs' reports into one comparison workbook.""" + output_dir = profiled_run / "host_reports" + self.run_cli( + dummy_tracelens, + "tracelens", + "--root", + str(profiled_run), + "--output-dir", + str(output_dir), + ) + reports = sorted(output_dir.glob("*.xlsx")) + assert len(reports) >= 2, [p.name for p in output_dir.iterdir()] + comparison = profiled_run / "comparison.xlsx" + + result = self.run_cli( + dummy_tracelens, + "tracelens-compare", + str(reports[0]), + str(reports[1]), + "--output", + str(comparison), + "--names", + "baseline", + "--names", + "candidate", + ) + + assert result.returncode == 0, result.stdout + assert comparison.is_file() + record = only( + dummy_tracelens.invocations(), "TraceLens_compare_perf_reports_pytorch" + ) + assert record["args"]["names"] == ["baseline", "candidate"] diff --git a/tests/fixtures/dummy_tracelens/TraceLens/Reporting/__init__.py b/tests/fixtures/dummy_tracelens/TraceLens/Reporting/__init__.py new file mode 100644 index 00000000..d2b6e8f7 --- /dev/null +++ b/tests/fixtures/dummy_tracelens/TraceLens/Reporting/__init__.py @@ -0,0 +1,4 @@ +"""Dummy TraceLens report generators mirroring the real entry-point modules. + +Copyright (c) Advanced Micro Devices, Inc. All rights reserved. +""" diff --git a/tests/fixtures/dummy_tracelens/TraceLens/Reporting/_dummy.py b/tests/fixtures/dummy_tracelens/TraceLens/Reporting/_dummy.py new file mode 100644 index 00000000..026cbdf7 --- /dev/null +++ b/tests/fixtures/dummy_tracelens/TraceLens/Reporting/_dummy.py @@ -0,0 +1,222 @@ +"""Shared implementation behind every dummy TraceLens report generator. + +Each entry point declares the flags madengine's analyzer is expected to pass. A +flag that madengine drops or renames therefore fails the test suite rather than +surfacing as an obscure TraceLens usage error at profiling time. Invocations are +appended to ``$DUMMY_TRACELENS_LOG`` as JSON lines so tests can assert which +report generator ran for which trace. + +This encodes madengine's side of the contract; it cannot verify that upstream +TraceLens still accepts these flags. Bumping the pinned TraceLens revision needs +a real run against real traces. + +Set ``DUMMY_TRACELENS_FAIL`` to a comma-separated list of entry-point names (or +``all``) to make those reports fail, which is how tests cover failure handling. + +Copyright (c) Advanced Micro Devices, Inc. All rights reserved. +""" + +import argparse +import glob +import json +import os +import sys +from typing import Callable, Dict, List, Optional, Sequence + +# Per entry point: value-taking flags madengine always passes, bare switches it +# always passes, value-taking flags it passes conditionally, and which flag +# carries the input trace so the dummy can check it was given a real file. +_SPECS: Dict[str, Dict[str, object]] = { + "TraceLens_generate_perf_report_pytorch": { + "required": ("--profile_json_path", "--output_xlsx_path", "--output_csvs_dir"), + "switches": ("--enable_kernel_summary", "--short_kernel_study"), + "optional": ("--gpu_arch_platform",), + "input_file": "profile_json_path", + }, + "TraceLens_generate_perf_report_rocprof": { + "required": ("--profile_json_path", "--output_xlsx_path", "--output_csvs_dir"), + "switches": ("--kernel_details", "--short_kernel_study"), + "input_file": "profile_json_path", + }, + "TraceLens_generate_perf_report_pftrace_hip_activity": { + "required": ("--trace_path", "--output_csvs_dir", "--output_md_path"), + "input_file": "trace_path", + }, + "TraceLens_generate_perf_report_pftrace_hip_api": { + "required": ("--trace_path", "--output_xlsx_path", "--output_csvs_dir"), + "input_file": "trace_path", + }, + "TraceLens_generate_perf_report_pftrace_memory_copy": { + "required": ("--trace_path", "--output_xlsx_path", "--output_csvs_dir"), + "input_file": "trace_path", + }, + "TraceLens_generate_multi_rank_collective_report_pytorch": { + "required": ( + "--trace_glob", + "--rank_regex", + "--world_size", + "--output_xlsx_path", + "--output_csvs_dir", + ), + "switches": ("--use_multiprocessing",), + "input_glob": "trace_glob", + }, +} + +_FORCED_FAILURE_EXIT_CODE = 3 + + +def _parser(entry_point: str) -> argparse.ArgumentParser: + spec = _SPECS[entry_point] + parser = argparse.ArgumentParser(prog=entry_point) + for flag in spec["required"]: # type: ignore[union-attr] + parser.add_argument(flag, required=True) + for flag in spec.get("optional", ()): # type: ignore[union-attr] + parser.add_argument(flag) + for flag in spec.get("switches", ()): # type: ignore[union-attr] + parser.add_argument(flag, action="store_true", required=True) + return parser + + +def _fail(message: str) -> int: + print(f"dummy TraceLens: {message}", file=sys.stderr) + return 1 + + +def _forced_failure(entry_point: str) -> bool: + requested = os.environ.get("DUMMY_TRACELENS_FAIL", "") + wanted = {name.strip() for name in requested.split(",") if name.strip()} + return "all" in wanted or entry_point in wanted + + +def _check_input_file(path: str) -> Optional[int]: + if not os.path.isfile(path): + return _fail(f"input trace {path} does not exist") + if os.path.getsize(path) == 0: + return _fail(f"input trace {path} is empty") + return None + + +def _check_input_glob(pattern: str, world_size: str) -> Optional[int]: + matches = [p for p in glob.glob(pattern, recursive=True) if os.path.isfile(p)] + if len(matches) < 2: + return _fail( + f"a collective report needs at least two per-rank traces; " + f"{pattern} matched {len(matches)}" + ) + if int(world_size) < 2: + return _fail(f"--world_size must be at least 2, got {world_size}") + return None + + +def _write_reports(entry_point: str, parsed: argparse.Namespace) -> List[str]: + """Write the report files the flags promise, returning their paths.""" + written: List[str] = [] + for dest, value in sorted(vars(parsed).items()): + if not isinstance(value, str): + continue + if dest.endswith("_xlsx_path"): + _write_text(value, f"dummy TraceLens workbook from {entry_point}\n") + written.append(value) + elif dest.endswith("_md_path"): + _write_text(value, f"# dummy TraceLens report from {entry_point}\n") + written.append(value) + elif dest.endswith("_csvs_dir"): + os.makedirs(value, exist_ok=True) + summary = os.path.join(value, "kernel_summary.csv") + _write_text(summary, "kernel,duration_us\ndummy_gemm_kernel,42\n") + written.append(summary) + return written + + +def _write_text(path: str, text: str) -> None: + parent = os.path.dirname(os.path.abspath(path)) + os.makedirs(parent, exist_ok=True) + with open(path, "w", encoding="utf-8") as handle: + handle.write(text) + + +def _log(entry_point: str, parsed: argparse.Namespace, extra: Sequence[str]) -> None: + log_path = os.environ.get("DUMMY_TRACELENS_LOG", "") + if not log_path: + return + record = { + "entry_point": entry_point, + # "-c" when madengine fell back to importing the module, the console + # script path when it found the installed entry point. + "argv0": sys.argv[0], + "args": vars(parsed), + "extra": list(extra), + "cwd": os.getcwd(), + } + with open(log_path, "a", encoding="utf-8") as handle: + handle.write(json.dumps(record, sort_keys=True) + "\n") + + +def _report(entry_point: str, argv: Optional[Sequence[str]]) -> int: + # Unrecognised flags are recorded rather than rejected: madengine forwards + # user-supplied TraceLens flags verbatim, and tests assert on that. + parsed, extra = _parser(entry_point).parse_known_args(argv) + _log(entry_point, parsed, extra) + + spec = _SPECS[entry_point] + if "input_file" in spec: + failure = _check_input_file(getattr(parsed, str(spec["input_file"]))) + if failure is not None: + return failure + if "input_glob" in spec: + failure = _check_input_glob( + getattr(parsed, str(spec["input_glob"])), parsed.world_size + ) + if failure is not None: + return failure + + if _forced_failure(entry_point): + print(f"dummy TraceLens: forced failure for {entry_point}", file=sys.stderr) + return _FORCED_FAILURE_EXIT_CODE + + for path in _write_reports(entry_point, parsed): + print(f"dummy TraceLens: wrote {path}") + return 0 + + +def main_for(entry_point: str) -> Callable[[Optional[Sequence[str]]], int]: + """Return the ``main`` for one dummy report generator. + + Like the console scripts pip generates for the real package, ``main`` + returns an exit code rather than raising ``SystemExit``. + """ + + def main(argv: Optional[Sequence[str]] = None) -> int: + return _report(entry_point, argv) + + main.__name__ = "main" + main.__doc__ = f"Dummy implementation of {entry_point}." + return main + + +def compare_main(argv: Optional[Sequence[str]] = None) -> int: + """Dummy implementation of ``TraceLens_compare_perf_reports_pytorch``.""" + entry_point = "TraceLens_compare_perf_reports_pytorch" + parser = argparse.ArgumentParser(prog=entry_point) + parser.add_argument("reports", nargs="+") + parser.add_argument("-o", "--output", required=True) + parser.add_argument("--names", nargs="+", default=[]) + parsed, extra = parser.parse_known_args(argv) + _log(entry_point, parsed, extra) + + if len(parsed.reports) < 2: + return _fail("comparing reports needs at least two inputs") + for report in parsed.reports: + if not os.path.exists(report): + return _fail(f"report {report} does not exist") + if parsed.names and len(parsed.names) != len(parsed.reports): + return _fail(f"got {len(parsed.names)} names for {len(parsed.reports)} reports") + + if _forced_failure(entry_point): + print(f"dummy TraceLens: forced failure for {entry_point}", file=sys.stderr) + return _FORCED_FAILURE_EXIT_CODE + + _write_text(parsed.output, f"dummy TraceLens comparison of {parsed.reports}\n") + print(f"dummy TraceLens: wrote {parsed.output}") + return 0 diff --git a/tests/fixtures/dummy_tracelens/TraceLens/Reporting/compare_perf_reports_pytorch.py b/tests/fixtures/dummy_tracelens/TraceLens/Reporting/compare_perf_reports_pytorch.py new file mode 100644 index 00000000..09b5a671 --- /dev/null +++ b/tests/fixtures/dummy_tracelens/TraceLens/Reporting/compare_perf_reports_pytorch.py @@ -0,0 +1,8 @@ +"""Dummy stand-in for the TraceLens report comparison entry point. + +Copyright (c) Advanced Micro Devices, Inc. All rights reserved. +""" + +from TraceLens.Reporting._dummy import compare_main + +main = compare_main diff --git a/tests/fixtures/dummy_tracelens/TraceLens/Reporting/generate_multi_rank_collective_report_pytorch.py b/tests/fixtures/dummy_tracelens/TraceLens/Reporting/generate_multi_rank_collective_report_pytorch.py new file mode 100644 index 00000000..f28172e9 --- /dev/null +++ b/tests/fixtures/dummy_tracelens/TraceLens/Reporting/generate_multi_rank_collective_report_pytorch.py @@ -0,0 +1,8 @@ +"""Dummy stand-in for the TraceLens multi-rank collective report generator. + +Copyright (c) Advanced Micro Devices, Inc. All rights reserved. +""" + +from TraceLens.Reporting._dummy import main_for + +main = main_for("TraceLens_generate_multi_rank_collective_report_pytorch") diff --git a/tests/fixtures/dummy_tracelens/TraceLens/Reporting/generate_perf_report_pftrace_hip_activity.py b/tests/fixtures/dummy_tracelens/TraceLens/Reporting/generate_perf_report_pftrace_hip_activity.py new file mode 100644 index 00000000..ac4efd01 --- /dev/null +++ b/tests/fixtures/dummy_tracelens/TraceLens/Reporting/generate_perf_report_pftrace_hip_activity.py @@ -0,0 +1,8 @@ +"""Dummy stand-in for the TraceLens pftrace HIP activity report generator. + +Copyright (c) Advanced Micro Devices, Inc. All rights reserved. +""" + +from TraceLens.Reporting._dummy import main_for + +main = main_for("TraceLens_generate_perf_report_pftrace_hip_activity") diff --git a/tests/fixtures/dummy_tracelens/TraceLens/Reporting/generate_perf_report_pftrace_hip_api.py b/tests/fixtures/dummy_tracelens/TraceLens/Reporting/generate_perf_report_pftrace_hip_api.py new file mode 100644 index 00000000..05d8fb10 --- /dev/null +++ b/tests/fixtures/dummy_tracelens/TraceLens/Reporting/generate_perf_report_pftrace_hip_api.py @@ -0,0 +1,8 @@ +"""Dummy stand-in for the TraceLens pftrace HIP API report generator. + +Copyright (c) Advanced Micro Devices, Inc. All rights reserved. +""" + +from TraceLens.Reporting._dummy import main_for + +main = main_for("TraceLens_generate_perf_report_pftrace_hip_api") diff --git a/tests/fixtures/dummy_tracelens/TraceLens/Reporting/generate_perf_report_pftrace_memory_copy.py b/tests/fixtures/dummy_tracelens/TraceLens/Reporting/generate_perf_report_pftrace_memory_copy.py new file mode 100644 index 00000000..f39cb371 --- /dev/null +++ b/tests/fixtures/dummy_tracelens/TraceLens/Reporting/generate_perf_report_pftrace_memory_copy.py @@ -0,0 +1,8 @@ +"""Dummy stand-in for the TraceLens pftrace memory copy report generator. + +Copyright (c) Advanced Micro Devices, Inc. All rights reserved. +""" + +from TraceLens.Reporting._dummy import main_for + +main = main_for("TraceLens_generate_perf_report_pftrace_memory_copy") diff --git a/tests/fixtures/dummy_tracelens/TraceLens/Reporting/generate_perf_report_pytorch.py b/tests/fixtures/dummy_tracelens/TraceLens/Reporting/generate_perf_report_pytorch.py new file mode 100644 index 00000000..cff1cedf --- /dev/null +++ b/tests/fixtures/dummy_tracelens/TraceLens/Reporting/generate_perf_report_pytorch.py @@ -0,0 +1,8 @@ +"""Dummy stand-in for the TraceLens PyTorch trace report generator. + +Copyright (c) Advanced Micro Devices, Inc. All rights reserved. +""" + +from TraceLens.Reporting._dummy import main_for + +main = main_for("TraceLens_generate_perf_report_pytorch") diff --git a/tests/fixtures/dummy_tracelens/TraceLens/Reporting/generate_perf_report_rocprof.py b/tests/fixtures/dummy_tracelens/TraceLens/Reporting/generate_perf_report_rocprof.py new file mode 100644 index 00000000..61982762 --- /dev/null +++ b/tests/fixtures/dummy_tracelens/TraceLens/Reporting/generate_perf_report_rocprof.py @@ -0,0 +1,8 @@ +"""Dummy stand-in for the TraceLens rocprofv3 JSON report generator. + +Copyright (c) Advanced Micro Devices, Inc. All rights reserved. +""" + +from TraceLens.Reporting._dummy import main_for + +main = main_for("TraceLens_generate_perf_report_rocprof") diff --git a/tests/fixtures/dummy_tracelens/TraceLens/__init__.py b/tests/fixtures/dummy_tracelens/TraceLens/__init__.py new file mode 100644 index 00000000..86e5155e --- /dev/null +++ b/tests/fixtures/dummy_tracelens/TraceLens/__init__.py @@ -0,0 +1,12 @@ +"""A stand-in for the real TraceLens package, for use in tests. + +The real package pins ``protobuf`` and ``xprof`` and only produces reports from +recorded GPU traces, so CI can neither install it nor feed it real input. This +stand-in is placed on ``PYTHONPATH`` instead, which lets the tests drive the +whole madengine analysis pipeline — the packaged analyzer, the in-container +post-script, and ``madengine report tracelens`` — with no GPU and no network. + +Copyright (c) Advanced Micro Devices, Inc. All rights reserved. +""" + +__version__ = "0.0.0.dummy"