diff --git a/src/madengine/scripts/common/post_scripts/dynolog_stop.sh b/src/madengine/scripts/common/post_scripts/dynolog_stop.sh index 7fbc9472..7f9ebb22 100644 --- a/src/madengine/scripts/common/post_scripts/dynolog_stop.sh +++ b/src/madengine/scripts/common/post_scripts/dynolog_stop.sh @@ -22,6 +22,24 @@ if [ ! -f "$DYNOLOG_START_FILE" ]; then exit 0 fi +# Both processes are started in the background by the pre-script, so once it exits +# they are reparented to PID 1, which in a container is the model command rather +# than an init that reaps children. A terminated process therefore lingers as a +# zombie and `kill -0` keeps succeeding, so the process state has to be checked as +# well, or every stop would burn the full grace period below. +is_running() { + local pid=$1 + kill -0 "$pid" 2>/dev/null || return 1 + if [ -r "/proc/$pid/stat" ]; then + # State is the field after the command name, which itself may contain + # spaces and is always parenthesised. + local state + state=$(sed 's/.*) //' "/proc/$pid/stat" 2>/dev/null | cut -d' ' -f1) + [ "$state" = "Z" ] && return 1 + fi + return 0 +} + stop_pid() { local name=$1 local pid_file=$2 @@ -31,14 +49,14 @@ stop_pid() { fi local pid pid=$(cat "$pid_file") - if kill -0 "$pid" 2>/dev/null; then + if is_running "$pid"; then kill -TERM "$pid" 2>/dev/null || true local waited=0 - while kill -0 "$pid" 2>/dev/null && [ $waited -lt 20 ]; do + while is_running "$pid" && [ $waited -lt 20 ]; do sleep 0.5 waited=$((waited + 1)) done - if kill -0 "$pid" 2>/dev/null; then + if is_running "$pid"; then echo "⚠️ $name did not stop gracefully, force killing..." kill -9 "$pid" 2>/dev/null || true fi diff --git a/src/madengine/scripts/common/tools/tracelens_analyze.py b/src/madengine/scripts/common/tools/tracelens_analyze.py index bfe5606e..63e30587 100644 --- a/src/madengine/scripts/common/tools/tracelens_analyze.py +++ b/src/madengine/scripts/common/tools/tracelens_analyze.py @@ -207,6 +207,8 @@ def _build_command(python: str, script_name: str, args: Sequence[str]) -> List[s Prefers the installed console script (clearer logs, honours the package's own entry-point wiring) and falls back to importing the module's ``main``. + The fallback exits with ``main()``'s return value, the same way the console + scripts pip generates do, so a report failure is not silently swallowed. """ bindir = os.path.dirname(os.path.abspath(python)) candidate = os.path.join(bindir, script_name) @@ -216,7 +218,12 @@ def _build_command(python: str, script_name: str, args: Sequence[str]) -> List[s if on_path: return [on_path, *args] module = _ENTRY_POINTS[script_name] - return [python, "-c", f"from {module} import main; main()", *args] + return [ + python, + "-c", + f"import sys; from {module} import main; sys.exit(main())", + *args, + ] def _run(command: Sequence[str], cwd: Optional[str] = None) -> Tuple[int, str]: @@ -461,9 +468,9 @@ def analyze( code, output = _run(_build_command(interpreter, tool, args)) results.append( { - "trace_file": os.path.relpath(trace, root) - if os.path.exists(trace) - else trace, + "trace_file": ( + os.path.relpath(trace, root) if os.path.exists(trace) else trace + ), "kind": kind, "tracelens_tool": tool, "status": "SUCCESS" if code == 0 else "FAILURE", @@ -605,9 +612,7 @@ def main(argv: Optional[Sequence[str]] = None) -> int: ) failed = summary["status"] != "SUCCESS" elif args.discover_only: - traces, unsupported = discover_traces( - args.root, exclude_dirs=[args.output_dir] - ) + traces, unsupported = discover_traces(args.root, exclude_dirs=[args.output_dir]) for kind, paths in sorted(traces.items()): for path in paths: print(f"{kind}\t{path}") diff --git a/tests/e2e/test_dynolog_dummy_pipeline.py b/tests/e2e/test_dynolog_dummy_pipeline.py new file mode 100644 index 00000000..5084a087 --- /dev/null +++ b/tests/e2e/test_dynolog_dummy_pipeline.py @@ -0,0 +1,358 @@ +"""Whole-pipeline `torch_profiler_dynolog` tests that run wherever CI runs. + +On-demand PyTorch tracing needs three things madengine cannot provide in CI: the +dynolog daemon (published only as a GitHub release asset), a GPU, and a PyTorch +workload for the daemon to attach to. These tests substitute a dummy ``dynolog`` +and ``dyno`` on ``PATH`` and then run the tool's real scripts against them, which +covers what madengine owns: the daemon lifecycle, the retry loop that waits for +the workload to register, the flags the trace request carries, and the +diagnostics printed when nothing was captured. + +The stand-in for ``dyno`` also writes the trace file, which in a real run is +written by the workload's own Kineto instance. + +Copyright (c) Advanced Micro Devices, Inc. All rights reserved. +""" + +# built-in modules +import contextlib +import os +import shutil +import signal +import subprocess +import time +from pathlib import Path + +# third-party modules +import pytest + +# project modules +from madengine.utils.path_utils import get_madengine_root + +pytestmark = pytest.mark.skipif( + os.name != "posix", reason="drives the Linux in-container profiling scripts" +) + +COMMON_SCRIPTS = get_madengine_root() / "scripts" / "common" +START_SCRIPT = COMMON_SCRIPTS / "pre_scripts" / "dynolog_start.sh" +TRIGGER_SCRIPT = COMMON_SCRIPTS / "tools" / "dynolog_trigger.sh" +STOP_SCRIPT = COMMON_SCRIPTS / "post_scripts" / "dynolog_stop.sh" + +# The scripts hand off to each other through fixed paths, because the pre-script, +# the trigger, and the post-script are three separate processes in a container. +DYNOLOG_PID_FILE = Path("/tmp/madengine_dynolog.pid") +TRIGGER_PID_FILE = Path("/tmp/madengine_dynolog_trigger.pid") +STARTED_FILE = Path("/tmp/madengine_dynolog.started") +RESULT_FILE = Path("/tmp/madengine_dynolog_trigger.result") +HANDOFF_FILES = ( + DYNOLOG_PID_FILE, + TRIGGER_PID_FILE, + STARTED_FILE, + RESULT_FILE, + Path("/tmp/madengine_dynolog.log"), + Path("/tmp/madengine_dynolog_trigger.log"), +) + +# A daemon that stays up until it is signalled, so the stop script has something +# real to terminate. +DYNOLOG_STUB = """#!/bin/sh +echo "dummy dynolog listening: $*" +while : ; do sleep 0.2; done +""" + +# Records each trace request, rejects the first $DUMMY_DYNO_REJECTS of them the +# way `--fail-on-no-process` does before the workload has registered, then +# accepts and writes the trace Kineto would have written. +DYNO_STUB = """#!/bin/sh +printf '%s\\n' "$*" >> "$DUMMY_DYNO_LOG" +requests=$(wc -l < "$DUMMY_DYNO_LOG") +if [ "$requests" -le "${DUMMY_DYNO_REJECTS:-0}" ]; then + echo "No processes were matched, exiting" >&2 + exit 1 +fi +log_file="" +previous="" +for arg in "$@"; do + if [ "$previous" = "--log-file" ]; then log_file=$arg; fi + previous=$arg +done +if [ "${DUMMY_DYNO_WRITE_TRACE:-0}" = "1" ] && [ -n "$log_file" ]; then + # Kineto appends the process id to the requested filename. + printf '{"traceEvents": [], "schemaVersion": 1}' > "${log_file%.json}_4242.json" +fi +echo "response length: 1" +exit 0 +""" + + +class DummyDynolog: + """Dummy ``dynolog`` and ``dyno`` binaries, installed the way the .deb is. + + ``pre_scripts/trace.sh dynolog`` ends with both binaries on ``PATH``, which + is all the rest of the tool depends on. + """ + + def __init__(self, bin_dir: Path) -> None: + bin_dir.mkdir(parents=True) + self.bin_dir = bin_dir + self.log = bin_dir / "dyno_requests.log" + self.log.touch() + for name, body in (("dynolog", DYNOLOG_STUB), ("dyno", DYNO_STUB)): + script = bin_dir / name + script.write_text(body, encoding="utf-8") + script.chmod(0o755) + + def environ(self, on_path: bool = True, **overrides: str) -> dict: + env = dict(os.environ, DUMMY_DYNO_LOG=str(self.log)) + if on_path: + env["PATH"] = f"{self.bin_dir}{os.pathsep}{env.get('PATH', '')}" + env.update(overrides) + return env + + def requests(self) -> list: + """Return the ``dyno gputrace`` requests made so far, one per line.""" + return [ + line for line in self.log.read_text(encoding="utf-8").splitlines() if line + ] + + def cleanup(self) -> None: + """Kill anything left running and clear the handoff files.""" + for pid_file in (TRIGGER_PID_FILE, DYNOLOG_PID_FILE): + if pid_file.is_file(): + with contextlib.suppress(ValueError, OSError): + os.kill(int(pid_file.read_text().strip()), signal.SIGKILL) + for path in HANDOFF_FILES: + with contextlib.suppress(OSError): + path.unlink() + + +@pytest.fixture +def dummy_dynolog(tmp_path): + dummy = DummyDynolog(tmp_path / "dynolog-bin") + dummy.cleanup() + yield dummy + dummy.cleanup() + + +@pytest.fixture +def workdir(tmp_path): + """A working directory with scripts/common staged, as a run has.""" + work = tmp_path / "workdir" + (work / "scripts").mkdir(parents=True) + shutil.copytree(COMMON_SCRIPTS, work / "scripts" / "common") + return work + + +def run_script(script: Path, work: Path, env: dict, timeout: int = 120): + return subprocess.run( + ["bash", str(script)], + cwd=work, + env=env, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + encoding="utf-8", + errors="replace", + timeout=timeout, + ) + + +def alive(pid_file: Path) -> bool: + if not pid_file.is_file(): + return False + try: + os.kill(int(pid_file.read_text().strip()), 0) + except (ValueError, OSError): + return False + return True + + +def wait_until(predicate, timeout: float = 10.0) -> bool: + deadline = time.time() + timeout + while time.time() < deadline: + if predicate(): + return True + time.sleep(0.2) + return predicate() + + +class TestDaemonLifecycle: + """Starting and stopping the daemon around a model run.""" + + def test_start_arms_the_daemon_and_the_trigger(self, workdir, dummy_dynolog): + """The pre-script leaves a daemon running and a trigger waiting.""" + result = run_script( + START_SCRIPT, + workdir, + dummy_dynolog.environ(TORCH_PROFILE_WARMUP_S="30"), + ) + + assert result.returncode == 0, result.stdout + assert "dynolog daemon started" in result.stdout + assert "trace trigger armed" in result.stdout + assert alive(DYNOLOG_PID_FILE), result.stdout + assert alive(TRIGGER_PID_FILE), result.stdout + # The post-script uses this marker to tell "not started" from "failed". + assert STARTED_FILE.is_file() + # Kineto writes into this directory itself, so it must exist up front. + assert (workdir / "torch_profiler_output").is_dir() + + def test_start_without_the_daemon_installed_fails(self, workdir, dummy_dynolog): + """Without the pre-script's install step there is nothing to start.""" + result = run_script(START_SCRIPT, workdir, dummy_dynolog.environ(on_path=False)) + + assert result.returncode != 0 + assert "pre-script must run first" in result.stdout + assert not STARTED_FILE.is_file() + + def test_stop_leaves_nothing_running(self, workdir, dummy_dynolog): + """The post-script has to reap both processes; they outlive the workload.""" + run_script( + START_SCRIPT, workdir, dummy_dynolog.environ(TORCH_PROFILE_WARMUP_S="30") + ) + + result = run_script(STOP_SCRIPT, workdir, dummy_dynolog.environ()) + + assert result.returncode == 0, result.stdout + assert "dynolog cleanup complete" in result.stdout + assert wait_until(lambda: not alive(DYNOLOG_PID_FILE)), result.stdout + assert wait_until(lambda: not alive(TRIGGER_PID_FILE)), result.stdout + assert not STARTED_FILE.is_file() + # Both processes answer SIGTERM. Reaching the force-kill path instead + # would cost the full grace period at the end of every profiled run. + assert "did not stop gracefully" not in result.stdout + + def test_stop_without_start_is_a_no_op(self, workdir, dummy_dynolog): + """Stacking the tool on a failed run must not turn into a second failure.""" + result = run_script(STOP_SCRIPT, workdir, dummy_dynolog.environ()) + + assert result.returncode == 0, result.stdout + assert "dynolog was not started" in result.stdout + + +class TestTraceRequest: + """The trigger, which is what actually asks for a trace.""" + + @staticmethod + def run_trigger(work: Path, dummy: DummyDynolog, **env: str): + settings = dict( + TORCH_PROFILE_WARMUP_S="0", + TORCH_PROFILE_RETRY_INTERVAL_S="0", + TORCH_PROFILE_MAX_ATTEMPTS="5", + ) + settings.update(env) + return run_script(TRIGGER_SCRIPT, work, dummy.environ(**settings)) + + def test_trigger_retries_until_the_workload_registers(self, workdir, dummy_dynolog): + """A trace cannot be requested until PyTorch has registered, so we poll.""" + result = self.run_trigger(workdir, dummy_dynolog, DUMMY_DYNO_REJECTS="2") + + assert result.returncode == 0, result.stdout + assert len(dummy_dynolog.requests()) == 3 + assert "accepted on attempt 3" in result.stdout + assert RESULT_FILE.read_text().strip() == "accepted" + + def test_request_carries_the_data_tracelens_needs(self, workdir, dummy_dynolog): + """Shapes, stacks, and modules are what make the TraceLens reports useful.""" + self.run_trigger(workdir, dummy_dynolog) + + request = dummy_dynolog.requests()[0] + for flag in ( + "--record-shapes", + "--with-stacks", + "--with-modules", + "--iterations 5", + "--process-limit 64", + "--fail-on-no-process", + ): + assert flag in request, request + # An absolute path, because the workload's working directory is its own. + assert ( + f"--log-file {workdir}/torch_profiler_output/libkineto_trace.json" + in request + ) + + def test_disabling_iterations_switches_to_a_timed_capture( + self, workdir, dummy_dynolog + ): + """Iteration counting needs an optimizer step, which not every model has.""" + self.run_trigger( + workdir, + dummy_dynolog, + TORCH_PROFILE_ITERATIONS="0", + TORCH_PROFILE_DURATION_MS="750", + ) + + request = dummy_dynolog.requests()[0] + assert "--duration-ms 750" in request + assert "--iterations" not in request + + def test_optional_capture_flags_follow_their_env_vars(self, workdir, dummy_dynolog): + """The expensive captures are opt-in, and the default ones opt-out.""" + self.run_trigger( + workdir, + dummy_dynolog, + TORCH_PROFILE_RECORD_SHAPES="0", + TORCH_PROFILE_WITH_STACKS="0", + TORCH_PROFILE_WITH_MODULES="0", + TORCH_PROFILE_WITH_FLOPS="1", + TORCH_PROFILE_PROFILE_MEMORY="1", + ) + + request = dummy_dynolog.requests()[0] + assert "--record-shapes" not in request + assert "--with-stacks" not in request + assert "--with-modules" not in request + assert "--with-flops" in request + assert "--profile-memory" in request + + def test_giving_up_says_what_to_check(self, workdir, dummy_dynolog): + """Nothing registering is the common failure, so it must be self-explaining.""" + result = self.run_trigger( + workdir, + dummy_dynolog, + DUMMY_DYNO_REJECTS="99", + TORCH_PROFILE_MAX_ATTEMPTS="2", + ) + + assert result.returncode != 0 + assert len(dummy_dynolog.requests()) == 2 + assert "gave up after 2 attempts" in result.stdout + assert "KINETO_USE_DAEMON=1" in result.stdout + assert "TORCH_PROFILE_WARMUP_S" in result.stdout + assert RESULT_FILE.read_text().strip() == "no_process" + + +class TestCaptureReporting: + """What the post-script tells the user about the traces it found.""" + + def test_captured_traces_are_reported(self, workdir, dummy_dynolog): + """A successful capture is confirmed with a count, per rank.""" + run_script( + START_SCRIPT, workdir, dummy_dynolog.environ(TORCH_PROFILE_WARMUP_S="30") + ) + TestTraceRequest.run_trigger(workdir, dummy_dynolog, DUMMY_DYNO_WRITE_TRACE="1") + + result = run_script(STOP_SCRIPT, workdir, dummy_dynolog.environ()) + + assert result.returncode == 0, result.stdout + assert "Captured 1 torch.profiler trace(s)" in result.stdout + traces = list((workdir / "torch_profiler_output").glob("*.json")) + assert [p.name for p in traces] == ["libkineto_trace_4242.json"] + + def test_missing_traces_are_explained(self, workdir, dummy_dynolog): + """An empty output directory needs a reason, not just a warning.""" + run_script( + START_SCRIPT, workdir, dummy_dynolog.environ(TORCH_PROFILE_WARMUP_S="30") + ) + TestTraceRequest.run_trigger( + workdir, + dummy_dynolog, + DUMMY_DYNO_REJECTS="99", + TORCH_PROFILE_MAX_ATTEMPTS="1", + ) + + result = run_script(STOP_SCRIPT, workdir, dummy_dynolog.environ()) + + assert result.returncode == 0, result.stdout + assert "No torch.profiler traces were captured" in result.stdout + assert "never matched a PyTorch process" in result.stdout diff --git a/tests/e2e/test_tracelens_dummy_pipeline.py b/tests/e2e/test_tracelens_dummy_pipeline.py new file mode 100644 index 00000000..444a5f2a --- /dev/null +++ b/tests/e2e/test_tracelens_dummy_pipeline.py @@ -0,0 +1,654 @@ +"""Whole-pipeline TraceLens tests that run wherever CI runs. + +The real TraceLens pins ``protobuf`` and ``xprof`` and only produces reports from +recorded GPU traces, so CI can neither install it nor feed it real input. These +tests put the dummy TraceLens in ``tests/fixtures/dummy_tracelens`` on +``PYTHONPATH`` in its place and then drive the integration for real: fabricated +trace artifacts, the packaged analyzer, the in-container ``tracelens`` +post-script, and ``madengine report tracelens``. No GPU, Docker, or network. + +This covers the half of the integration madengine owns — which report generator +each trace kind is routed to, the flags it receives, where reports land, what the +summary records, and the guarantee that a failed analysis never fails a model +run. It cannot confirm that upstream TraceLens still accepts those flags. + +Substituting TraceLens is not just a convenience: the real one reports only on +kernels it can link back to the runtime calls that launched them, so no +fabricated trace will produce a report, however well shaped. The trace artifacts +below are therefore structural stand-ins, and the GPU-gated tests in +test_tracelens_workflows.py are what exercise real analysis. + +Copyright (c) Advanced Micro Devices, Inc. All rights reserved. +""" + +# built-in modules +import csv +import gzip +import importlib.util +import json +import os +import shutil +import subprocess +import sys +from pathlib import Path + +# third-party modules +import pytest + +# project modules +from madengine.utils.path_utils import get_madengine_root + +# The in-container half of the integration is shell, and the venv layout the +# analyzer looks into is POSIX-only. +pytestmark = pytest.mark.skipif( + os.name != "posix", reason="drives Linux container scripts and a POSIX venv layout" +) + +DUMMY_TRACELENS = Path(__file__).resolve().parents[1] / "fixtures" / "dummy_tracelens" +COMMON_SCRIPTS = get_madengine_root() / "scripts" / "common" +ANALYZER = COMMON_SCRIPTS / "tools" / "tracelens_analyze.py" +POST_SCRIPT = COMMON_SCRIPTS / "post_scripts" / "tracelens.sh" + +PYTORCH_REPORT = "TraceLens_generate_perf_report_pytorch" +ROCPROF_REPORT = "TraceLens_generate_perf_report_rocprof" +PFTRACE_REPORTS = ( + "TraceLens_generate_perf_report_pftrace_hip_activity", + "TraceLens_generate_perf_report_pftrace_hip_api", + "TraceLens_generate_perf_report_pftrace_memory_copy", +) +COLLECTIVE_REPORT = "TraceLens_generate_multi_rank_collective_report_pytorch" + + +def _entry_points() -> dict: + """Return the analyzer's entry point -> module map. + + Read from the analyzer itself so the dummy console scripts are always named + after what madengine actually looks for. + """ + spec = importlib.util.spec_from_file_location("tracelens_analyze", ANALYZER) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module._ENTRY_POINTS + + +ENTRY_POINTS = _entry_points() + + +class DummyTraceLens: + """The dummy package, installed the way the pre-script installs the real one. + + ``pre_scripts/trace.sh tracelens`` builds an isolated venv whose ``bin/`` + holds a python and TraceLens' report console scripts, and the analyzer + prefers those console scripts over importing the modules. Reproducing that + layout means these tests exercise the same lookup a real run does. + """ + + def __init__(self, root: Path, console_scripts: bool = True) -> None: + self.root = root + self.log = root / "invocations.jsonl" + bin_dir = root / "bin" + bin_dir.mkdir(parents=True) + + self.python = bin_dir / "python3" + self._write_executable( + self.python, f'#!/bin/sh\nexec "{sys.executable}" "$@"\n' + ) + if console_scripts: + for name, module in ENTRY_POINTS.items(): + self._write_executable( + bin_dir / name, + f"#!{sys.executable}\n" + "import sys\n" + f"from {module} import main\n" + "sys.exit(main())\n", + ) + self._vars = { + "DUMMY_TRACELENS_LOG": str(self.log), + "TRACELENS_VENV": str(root), + } + + @staticmethod + def _write_executable(path: Path, text: str) -> None: + path.write_text(text, encoding="utf-8") + path.chmod(0o755) + + def fail(self, *entry_points: str) -> None: + """Make the named report generators fail. ``"all"`` fails every one.""" + self._vars["DUMMY_TRACELENS_FAIL"] = ",".join(entry_points) + + def environ(self, **overrides: str) -> dict: + """Return an environment in which the dummy shadows any real TraceLens.""" + existing = os.environ.get("PYTHONPATH", "") + python_path = os.pathsep.join(p for p in (str(DUMMY_TRACELENS), existing) if p) + env = dict(os.environ, PYTHONPATH=python_path, **self._vars) + env.update(overrides) + return env + + def invocations(self, entry_point: str = "") -> list: + """Return the report generator calls recorded so far.""" + if not self.log.is_file(): + return [] + records = [ + json.loads(line) + for line in self.log.read_text(encoding="utf-8").splitlines() + if line.strip() + ] + if entry_point: + records = [r for r in records if r["entry_point"] == entry_point] + return records + + def reports_run(self) -> list: + """Return the entry point names that ran, in order.""" + return [record["entry_point"] for record in self.invocations()] + + +@pytest.fixture +def dummy_tracelens(tmp_path): + return DummyTraceLens(tmp_path / "tracelens-venv") + + +def write_chrome_trace(path: Path) -> Path: + """Write a minimal Chrome Trace Event document, as Kineto does. + + Gzips the payload when ``path`` ends in ``.gz``, which is how + ``tensorboard_trace_handler`` writes traces. + """ + payload = { + "schemaVersion": 1, + "distributedInfo": {"backend": "nccl", "rank": 0, "world_size": 1}, + "traceEvents": [ + { + "ph": "X", + "cat": "kernel", + "name": "void gemm_kernel(float*, float*)", + "pid": 1, + "tid": 7, + "ts": 100, + "dur": 42, + "args": {"stream": 7, "grid": [8, 1, 1], "block": [256, 1, 1]}, + }, + { + "ph": "X", + "cat": "gpu_memcpy", + "name": "Memcpy DtoH", + "pid": 1, + "tid": 7, + "ts": 200, + "dur": 8, + "args": {"bytes": 4096}, + }, + ], + } + path.parent.mkdir(parents=True, exist_ok=True) + if path.name.endswith(".gz"): + with gzip.open(path, "wt", encoding="utf-8") as handle: + json.dump(payload, handle) + else: + path.write_text(json.dumps(payload), encoding="utf-8") + return path + + +def write_rocprof_json(path: Path) -> Path: + """Write a rocprofv3 JSON result document, as ``rocprofv3_lightweight`` does. + + Shaped like the real output down to the ``buffer_records`` TraceLens reads + kernel dispatches from, but structural only: see the module docstring. + """ + payload = { + "rocprofiler-sdk-tool": [ + { + "metadata": {"pid": 1234}, + "agents": [{"id": {"handle": 0}, "type": 2, "name": "gfx942"}], + "kernel_symbols": [ + {"id": 1, "formatted_kernel_name": "gemm_kernel(float*, float*)"} + ], + "buffer_records": { + "kernel_dispatch": [ + { + "correlation_id": {"internal": 1}, + "start_timestamp": 1000, + "end_timestamp": 43000, + "dispatch_info": { + "kernel_id": 1, + "agent_id": {"handle": 0}, + }, + } + ], + "memory_copy": [], + }, + } + ] + } + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(payload), encoding="utf-8") + return path + + +def write_pftrace(path: Path) -> Path: + """Write a placeholder Perfetto trace, as ``rocprofv3_perfetto`` does.""" + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(b"\x0a\x0fdummy-perfetto-trace") + return path + + +def write_rocprof_db(path: Path) -> Path: + """Write rocprofv3's default SQLite output, which TraceLens cannot read.""" + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(b"SQLite format 3\x00") + return path + + +@pytest.fixture +def profiled_run(tmp_path): + """A directory shaped like the one a profiled madengine run leaves behind. + + Holds one artifact of every kind madengine's profiling tools can produce, + including the SQLite output TraceLens cannot read. + """ + work = tmp_path / "workdir" + write_chrome_trace(work / "torch_profiler_output" / "libkineto_trace_1234.json") + write_rocprof_json(work / "rocprof_output" / "1234_results.json") + write_pftrace(work / "rocprof_output" / "model.pftrace") + write_rocprof_db(work / "rocprof_output" / "1234_results.db") + return work + + +def run_analyzer( + work: Path, dummy: DummyTraceLens, *extra: str, output_dir: str = "tracelens_output" +) -> subprocess.CompletedProcess: + """Run the packaged analyzer from ``work``, as the post-script does.""" + return subprocess.run( + [ + sys.executable, + str(ANALYZER), + "--root", + ".", + "--output-dir", + output_dir, + "--python", + str(dummy.python), + "--json-summary", + f"{output_dir}/tracelens_summary.json", + *extra, + ], + cwd=work, + env=dummy.environ(), + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + encoding="utf-8", + errors="replace", + timeout=300, + ) + + +def summary_rows(work: Path, output_dir: str = "tracelens_output") -> list: + """Read the analyzer's summary CSV into a list of dict rows.""" + path = work / output_dir / "tracelens_summary.csv" + assert path.is_file(), f"analyzer wrote no summary at {path}" + with open(path, newline="", encoding="utf-8") as handle: + return list(csv.DictReader(handle)) + + +def only(records: list, entry_point: str) -> dict: + """Return the single recorded invocation of ``entry_point``.""" + matches = [r for r in records if r["entry_point"] == entry_point] + assert len(matches) == 1, f"expected one {entry_point} call, got {len(matches)}" + return matches[0] + + +class TestAnalyzerWithDummyTraceLens: + """Every trace kind a run can produce, analyzed by the real analyzer.""" + + def test_each_trace_kind_is_routed_to_its_report_generator( + self, profiled_run, dummy_tracelens + ): + """A Kineto trace, a rocprofv3 JSON, and a pftrace pick different reports.""" + result = run_analyzer(profiled_run, dummy_tracelens) + + assert result.returncode == 0, result.stdout + assert sorted(dummy_tracelens.reports_run()) == sorted( + [PYTORCH_REPORT, ROCPROF_REPORT, *PFTRACE_REPORTS] + ) + records = dummy_tracelens.invocations() + assert only(records, PYTORCH_REPORT)["args"]["profile_json_path"].endswith( + "libkineto_trace_1234.json" + ) + assert only(records, ROCPROF_REPORT)["args"]["profile_json_path"].endswith( + "1234_results.json" + ) + for report in PFTRACE_REPORTS: + assert only(records, report)["args"]["trace_path"].endswith(".pftrace") + + def test_reports_are_written_into_the_output_directory( + self, profiled_run, dummy_tracelens + ): + """Each report generator's workbook and per-sheet CSVs land together.""" + run_analyzer(profiled_run, dummy_tracelens) + + output_dir = profiled_run / "tracelens_output" + written = sorted(p.name for p in output_dir.iterdir()) + assert "tracelens_summary.csv" in written + assert "tracelens_summary.json" in written + # One workbook plus one CSV directory per PyTorch trace analyzed. + pytorch_stem = "torch_profiler_output_libkineto_trace_1234" + assert (output_dir / f"{pytorch_stem}.xlsx").is_file(), written + assert (output_dir / f"{pytorch_stem}_csv" / "kernel_summary.csv").is_file() + # The pftrace activity report is markdown rather than a workbook. + assert list(output_dir.glob("*_activity.md")), written + + def test_summary_records_each_report_and_the_unreadable_db( + self, profiled_run, dummy_tracelens + ): + """The summary is the run's record: five reports, one skipped artifact.""" + run_analyzer(profiled_run, dummy_tracelens) + + rows = summary_rows(profiled_run) + assert [r["status"] for r in rows].count("SUCCESS") == 5 + skipped = [r for r in rows if r["status"] == "SKIPPED"] + assert len(skipped) == 1 + assert skipped[0]["trace_file"].endswith("1234_results.db") + # The skip has to say which preset the user should have run instead. + assert "rocprofv3_lightweight" in skipped[0]["detail"] + + def test_gzipped_kineto_trace_is_analyzed(self, tmp_path, dummy_tracelens): + """tensorboard_trace_handler's gzipped traces are picked up too.""" + work = tmp_path / "gz" + write_chrome_trace(work / "traces" / "worker0.pt.trace.json.gz") + + result = run_analyzer(work, dummy_tracelens) + + assert result.returncode == 0, result.stdout + record = only(dummy_tracelens.invocations(), PYTORCH_REPORT) + assert record["args"]["profile_json_path"].endswith(".pt.trace.json.gz") + + def test_per_rank_traces_add_a_collective_report(self, tmp_path, dummy_tracelens): + """Several ranks' traces also produce the multi-rank collective report.""" + work = tmp_path / "distributed" + for rank in (0, 1): + write_chrome_trace( + work / "torch_profiler_output" / f"libkineto_trace_rank{rank}_99.json" + ) + + result = run_analyzer(work, dummy_tracelens) + + assert result.returncode == 0, result.stdout + assert dummy_tracelens.reports_run().count(PYTORCH_REPORT) == 2 + collective = only(dummy_tracelens.invocations(), COLLECTIVE_REPORT) + # The dummy fails unless the glob really matched both per-rank traces. + assert collective["args"]["world_size"] == "2" + assert collective["args"]["use_multiprocessing"] is True + + def test_single_rank_run_skips_the_collective_report( + self, profiled_run, dummy_tracelens + ): + """One rank cannot have collectives to compare, so that report is skipped.""" + run_analyzer(profiled_run, dummy_tracelens) + + assert COLLECTIVE_REPORT not in dummy_tracelens.reports_run() + + def test_gpu_arch_enables_roofline_classification( + self, profiled_run, dummy_tracelens + ): + """--gpu-arch reaches the PyTorch report as TraceLens' arch flag.""" + run_analyzer(profiled_run, dummy_tracelens, "--gpu-arch", "MI300X") + + record = only(dummy_tracelens.invocations(), PYTORCH_REPORT) + assert record["args"]["gpu_arch_platform"] == "MI300X" + + def test_mode_restricts_analysis_to_one_trace_kind( + self, profiled_run, dummy_tracelens + ): + """--mode rocprof leaves the Kineto trace and the pftrace alone.""" + run_analyzer(profiled_run, dummy_tracelens, "--mode", "rocprof") + + assert dummy_tracelens.reports_run() == [ROCPROF_REPORT] + + def test_extra_flags_are_forwarded_to_every_report( + self, profiled_run, dummy_tracelens + ): + """Flags after ``--`` reach TraceLens untouched, for options we do not wrap.""" + run_analyzer(profiled_run, dummy_tracelens, "--", "--top_k_kernels", "5") + + for record in dummy_tracelens.invocations(): + assert record["extra"] == ["--top_k_kernels", "5"], record["entry_point"] + + def test_console_script_from_the_venv_is_preferred( + self, profiled_run, dummy_tracelens + ): + """With TraceLens' console scripts installed, they are what we run.""" + run_analyzer(profiled_run, dummy_tracelens) + + record = only(dummy_tracelens.invocations(), PYTORCH_REPORT) + assert record["argv0"] == str(dummy_tracelens.root / "bin" / PYTORCH_REPORT) + + def test_module_fallback_still_reports_failures(self, tmp_path, profiled_run): + """Without console scripts we import the module, and still see its exit code. + + The fallback runs ``main()`` in a fresh interpreter; if its return value + were dropped, every failed report would be recorded as a success. + """ + dummy = DummyTraceLens(tmp_path / "no-console-scripts", console_scripts=False) + dummy.fail(PYTORCH_REPORT) + + result = run_analyzer(profiled_run, dummy) + + assert result.returncode != 0, result.stdout + assert only(dummy.invocations(), PYTORCH_REPORT)["argv0"] == "-c" + pytorch_rows = [r for r in summary_rows(profiled_run) if r["kind"] == "pytorch"] + assert [r["status"] for r in pytorch_rows] == ["FAILURE"] + assert "forced failure" in pytorch_rows[0]["detail"] + + def test_one_failed_report_does_not_hide_the_others( + self, profiled_run, dummy_tracelens + ): + """A failing report is recorded as such; the rest still run and succeed.""" + dummy_tracelens.fail(ROCPROF_REPORT) + + result = run_analyzer(profiled_run, dummy_tracelens) + + assert result.returncode != 0, result.stdout + statuses = { + r["tracelens_tool"]: r["status"] for r in summary_rows(profiled_run) + } + assert statuses[ROCPROF_REPORT] == "FAILURE" + assert statuses[PYTORCH_REPORT] == "SUCCESS" + assert all(statuses[report] == "SUCCESS" for report in PFTRACE_REPORTS) + + def test_run_without_traces_says_which_tool_to_stack( + self, tmp_path, dummy_tracelens + ): + """No traces is a no-op with guidance, not a failure.""" + work = tmp_path / "empty" + work.mkdir() + + result = run_analyzer(work, dummy_tracelens) + + assert result.returncode == 0, result.stdout + assert "No supported trace artifacts found" in result.stdout + assert "torch_profiler_dynolog" in result.stdout + assert dummy_tracelens.reports_run() == [] + assert summary_rows(work) == [] + + +class TestInContainerPostScript: + """The `tracelens` tool's post-script, run the way a container runs it.""" + + @staticmethod + def stage_scripts(work: Path) -> None: + """Copy the analyzer to where ContainerRunner puts it during a run.""" + tools_dir = work / "scripts" / "common" / "tools" + tools_dir.mkdir(parents=True, exist_ok=True) + shutil.copy2(ANALYZER, tools_dir / ANALYZER.name) + + @classmethod + def run_post_script( + cls, work: Path, dummy: DummyTraceLens, **env: str + ) -> subprocess.CompletedProcess: + cls.stage_scripts(work) + return subprocess.run( + ["bash", str(POST_SCRIPT)], + cwd=work, + env=dummy.environ(**env), + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + encoding="utf-8", + errors="replace", + timeout=300, + ) + + def test_post_script_analyzes_the_traces_the_run_produced( + self, profiled_run, dummy_tracelens + ): + """The tool needs no configuration: it finds the traces and reports on them.""" + result = self.run_post_script(profiled_run, dummy_tracelens) + + assert result.returncode == 0, result.stdout + assert sorted(dummy_tracelens.reports_run()) == sorted( + [PYTORCH_REPORT, ROCPROF_REPORT, *PFTRACE_REPORTS] + ) + # tracelens_output/ is the directory the post-script hands to collection. + assert (profiled_run / "tracelens_output" / "tracelens_summary.csv").is_file() + assert (profiled_run / "tracelens_output" / "tracelens_summary.json").is_file() + + def test_failed_analysis_does_not_fail_the_model_run( + self, profiled_run, dummy_tracelens + ): + """Reporting is not the workload: TraceLens failing must not fail the run.""" + dummy_tracelens.fail("all") + + result = self.run_post_script(profiled_run, dummy_tracelens) + + assert result.returncode == 0, result.stdout + assert "WARNING: TraceLens analysis reported failures" in result.stdout + statuses = {r["status"] for r in summary_rows(profiled_run)} + assert "FAILURE" in statuses + + def test_missing_venv_fails_loudly(self, profiled_run, dummy_tracelens, tmp_path): + """A missing venv means the pre-script never ran, which is worth failing on.""" + result = self.run_post_script( + profiled_run, dummy_tracelens, TRACELENS_VENV=str(tmp_path / "absent") + ) + + assert result.returncode != 0 + assert "pre-script must run first" in result.stdout + + def test_mode_and_output_dir_are_configurable_by_env_var( + self, profiled_run, dummy_tracelens + ): + """The tool's env vars are how a user narrows analysis in a models.json.""" + result = self.run_post_script( + profiled_run, + dummy_tracelens, + TRACELENS_MODE="pytorch", + TRACELENS_OUTPUT_DIR="custom_tracelens", + ) + + assert result.returncode == 0, result.stdout + assert dummy_tracelens.reports_run() == [PYTORCH_REPORT] + rows = summary_rows(profiled_run, output_dir="custom_tracelens") + assert [r["kind"] for r in rows] == ["pytorch", "unsupported"] + + def test_gpu_arch_and_world_size_reach_tracelens(self, tmp_path, dummy_tracelens): + """Distributed runs pass rank count and arch through the tool's env vars.""" + work = tmp_path / "distributed" + for rank in (0, 1): + write_chrome_trace( + work / "torch_profiler_output" / f"libkineto_trace_rank{rank}_99.json" + ) + + result = self.run_post_script( + work, + dummy_tracelens, + TRACELENS_GPU_ARCH="MI300X", + TRACELENS_WORLD_SIZE="2", + ) + + assert result.returncode == 0, result.stdout + records = dummy_tracelens.invocations() + assert only(records, COLLECTIVE_REPORT)["args"]["world_size"] == "2" + assert all( + r["args"]["gpu_arch_platform"] == "MI300X" + for r in records + if r["entry_point"] == PYTORCH_REPORT + ) + + +class TestHostSideReportCommand: + """`madengine report tracelens` over artifacts already collected to the host.""" + + @staticmethod + def run_cli(dummy: DummyTraceLens, *args: str) -> subprocess.CompletedProcess: + """Run the CLI in a wide, colourless console so assertions survive wrapping.""" + env = dummy.environ( + COLUMNS="300", NO_COLOR="1", TERM="dumb", PYTHONIOENCODING="utf-8" + ) + return subprocess.run( + [sys.executable, "-m", "madengine.cli.app", "report", *args], + env=env, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + encoding="utf-8", + errors="replace", + timeout=300, + ) + + def test_report_tracelens_analyzes_collected_artifacts( + self, profiled_run, dummy_tracelens + ): + """The host path produces the same reports without touching the container.""" + output_dir = profiled_run / "host_reports" + + result = self.run_cli( + dummy_tracelens, + "tracelens", + "--root", + str(profiled_run), + "--output-dir", + str(output_dir), + ) + + assert result.returncode == 0, result.stdout + assert sorted(dummy_tracelens.reports_run()) == sorted( + [PYTORCH_REPORT, ROCPROF_REPORT, *PFTRACE_REPORTS] + ) + assert (output_dir / "tracelens_summary.csv").is_file() + # The rendered table is how a user sees what was analyzed. + assert "SUCCESS" in result.stdout + assert "SKIPPED" in result.stdout + + def test_report_tracelens_compare_diffs_two_reports( + self, profiled_run, dummy_tracelens + ): + """tracelens-compare turns two runs' reports into one comparison workbook.""" + output_dir = profiled_run / "host_reports" + self.run_cli( + dummy_tracelens, + "tracelens", + "--root", + str(profiled_run), + "--output-dir", + str(output_dir), + ) + reports = sorted(output_dir.glob("*.xlsx")) + assert len(reports) >= 2, [p.name for p in output_dir.iterdir()] + comparison = profiled_run / "comparison.xlsx" + + result = self.run_cli( + dummy_tracelens, + "tracelens-compare", + str(reports[0]), + str(reports[1]), + "--output", + str(comparison), + "--names", + "baseline", + "--names", + "candidate", + ) + + assert result.returncode == 0, result.stdout + assert comparison.is_file() + record = only( + dummy_tracelens.invocations(), "TraceLens_compare_perf_reports_pytorch" + ) + assert record["args"]["names"] == ["baseline", "candidate"] diff --git a/tests/fixtures/dummy_tracelens/TraceLens/Reporting/__init__.py b/tests/fixtures/dummy_tracelens/TraceLens/Reporting/__init__.py new file mode 100644 index 00000000..d2b6e8f7 --- /dev/null +++ b/tests/fixtures/dummy_tracelens/TraceLens/Reporting/__init__.py @@ -0,0 +1,4 @@ +"""Dummy TraceLens report generators mirroring the real entry-point modules. + +Copyright (c) Advanced Micro Devices, Inc. All rights reserved. +""" diff --git a/tests/fixtures/dummy_tracelens/TraceLens/Reporting/_dummy.py b/tests/fixtures/dummy_tracelens/TraceLens/Reporting/_dummy.py new file mode 100644 index 00000000..026cbdf7 --- /dev/null +++ b/tests/fixtures/dummy_tracelens/TraceLens/Reporting/_dummy.py @@ -0,0 +1,222 @@ +"""Shared implementation behind every dummy TraceLens report generator. + +Each entry point declares the flags madengine's analyzer is expected to pass. A +flag that madengine drops or renames therefore fails the test suite rather than +surfacing as an obscure TraceLens usage error at profiling time. Invocations are +appended to ``$DUMMY_TRACELENS_LOG`` as JSON lines so tests can assert which +report generator ran for which trace. + +This encodes madengine's side of the contract; it cannot verify that upstream +TraceLens still accepts these flags. Bumping the pinned TraceLens revision needs +a real run against real traces. + +Set ``DUMMY_TRACELENS_FAIL`` to a comma-separated list of entry-point names (or +``all``) to make those reports fail, which is how tests cover failure handling. + +Copyright (c) Advanced Micro Devices, Inc. All rights reserved. +""" + +import argparse +import glob +import json +import os +import sys +from typing import Callable, Dict, List, Optional, Sequence + +# Per entry point: value-taking flags madengine always passes, bare switches it +# always passes, value-taking flags it passes conditionally, and which flag +# carries the input trace so the dummy can check it was given a real file. +_SPECS: Dict[str, Dict[str, object]] = { + "TraceLens_generate_perf_report_pytorch": { + "required": ("--profile_json_path", "--output_xlsx_path", "--output_csvs_dir"), + "switches": ("--enable_kernel_summary", "--short_kernel_study"), + "optional": ("--gpu_arch_platform",), + "input_file": "profile_json_path", + }, + "TraceLens_generate_perf_report_rocprof": { + "required": ("--profile_json_path", "--output_xlsx_path", "--output_csvs_dir"), + "switches": ("--kernel_details", "--short_kernel_study"), + "input_file": "profile_json_path", + }, + "TraceLens_generate_perf_report_pftrace_hip_activity": { + "required": ("--trace_path", "--output_csvs_dir", "--output_md_path"), + "input_file": "trace_path", + }, + "TraceLens_generate_perf_report_pftrace_hip_api": { + "required": ("--trace_path", "--output_xlsx_path", "--output_csvs_dir"), + "input_file": "trace_path", + }, + "TraceLens_generate_perf_report_pftrace_memory_copy": { + "required": ("--trace_path", "--output_xlsx_path", "--output_csvs_dir"), + "input_file": "trace_path", + }, + "TraceLens_generate_multi_rank_collective_report_pytorch": { + "required": ( + "--trace_glob", + "--rank_regex", + "--world_size", + "--output_xlsx_path", + "--output_csvs_dir", + ), + "switches": ("--use_multiprocessing",), + "input_glob": "trace_glob", + }, +} + +_FORCED_FAILURE_EXIT_CODE = 3 + + +def _parser(entry_point: str) -> argparse.ArgumentParser: + spec = _SPECS[entry_point] + parser = argparse.ArgumentParser(prog=entry_point) + for flag in spec["required"]: # type: ignore[union-attr] + parser.add_argument(flag, required=True) + for flag in spec.get("optional", ()): # type: ignore[union-attr] + parser.add_argument(flag) + for flag in spec.get("switches", ()): # type: ignore[union-attr] + parser.add_argument(flag, action="store_true", required=True) + return parser + + +def _fail(message: str) -> int: + print(f"dummy TraceLens: {message}", file=sys.stderr) + return 1 + + +def _forced_failure(entry_point: str) -> bool: + requested = os.environ.get("DUMMY_TRACELENS_FAIL", "") + wanted = {name.strip() for name in requested.split(",") if name.strip()} + return "all" in wanted or entry_point in wanted + + +def _check_input_file(path: str) -> Optional[int]: + if not os.path.isfile(path): + return _fail(f"input trace {path} does not exist") + if os.path.getsize(path) == 0: + return _fail(f"input trace {path} is empty") + return None + + +def _check_input_glob(pattern: str, world_size: str) -> Optional[int]: + matches = [p for p in glob.glob(pattern, recursive=True) if os.path.isfile(p)] + if len(matches) < 2: + return _fail( + f"a collective report needs at least two per-rank traces; " + f"{pattern} matched {len(matches)}" + ) + if int(world_size) < 2: + return _fail(f"--world_size must be at least 2, got {world_size}") + return None + + +def _write_reports(entry_point: str, parsed: argparse.Namespace) -> List[str]: + """Write the report files the flags promise, returning their paths.""" + written: List[str] = [] + for dest, value in sorted(vars(parsed).items()): + if not isinstance(value, str): + continue + if dest.endswith("_xlsx_path"): + _write_text(value, f"dummy TraceLens workbook from {entry_point}\n") + written.append(value) + elif dest.endswith("_md_path"): + _write_text(value, f"# dummy TraceLens report from {entry_point}\n") + written.append(value) + elif dest.endswith("_csvs_dir"): + os.makedirs(value, exist_ok=True) + summary = os.path.join(value, "kernel_summary.csv") + _write_text(summary, "kernel,duration_us\ndummy_gemm_kernel,42\n") + written.append(summary) + return written + + +def _write_text(path: str, text: str) -> None: + parent = os.path.dirname(os.path.abspath(path)) + os.makedirs(parent, exist_ok=True) + with open(path, "w", encoding="utf-8") as handle: + handle.write(text) + + +def _log(entry_point: str, parsed: argparse.Namespace, extra: Sequence[str]) -> None: + log_path = os.environ.get("DUMMY_TRACELENS_LOG", "") + if not log_path: + return + record = { + "entry_point": entry_point, + # "-c" when madengine fell back to importing the module, the console + # script path when it found the installed entry point. + "argv0": sys.argv[0], + "args": vars(parsed), + "extra": list(extra), + "cwd": os.getcwd(), + } + with open(log_path, "a", encoding="utf-8") as handle: + handle.write(json.dumps(record, sort_keys=True) + "\n") + + +def _report(entry_point: str, argv: Optional[Sequence[str]]) -> int: + # Unrecognised flags are recorded rather than rejected: madengine forwards + # user-supplied TraceLens flags verbatim, and tests assert on that. + parsed, extra = _parser(entry_point).parse_known_args(argv) + _log(entry_point, parsed, extra) + + spec = _SPECS[entry_point] + if "input_file" in spec: + failure = _check_input_file(getattr(parsed, str(spec["input_file"]))) + if failure is not None: + return failure + if "input_glob" in spec: + failure = _check_input_glob( + getattr(parsed, str(spec["input_glob"])), parsed.world_size + ) + if failure is not None: + return failure + + if _forced_failure(entry_point): + print(f"dummy TraceLens: forced failure for {entry_point}", file=sys.stderr) + return _FORCED_FAILURE_EXIT_CODE + + for path in _write_reports(entry_point, parsed): + print(f"dummy TraceLens: wrote {path}") + return 0 + + +def main_for(entry_point: str) -> Callable[[Optional[Sequence[str]]], int]: + """Return the ``main`` for one dummy report generator. + + Like the console scripts pip generates for the real package, ``main`` + returns an exit code rather than raising ``SystemExit``. + """ + + def main(argv: Optional[Sequence[str]] = None) -> int: + return _report(entry_point, argv) + + main.__name__ = "main" + main.__doc__ = f"Dummy implementation of {entry_point}." + return main + + +def compare_main(argv: Optional[Sequence[str]] = None) -> int: + """Dummy implementation of ``TraceLens_compare_perf_reports_pytorch``.""" + entry_point = "TraceLens_compare_perf_reports_pytorch" + parser = argparse.ArgumentParser(prog=entry_point) + parser.add_argument("reports", nargs="+") + parser.add_argument("-o", "--output", required=True) + parser.add_argument("--names", nargs="+", default=[]) + parsed, extra = parser.parse_known_args(argv) + _log(entry_point, parsed, extra) + + if len(parsed.reports) < 2: + return _fail("comparing reports needs at least two inputs") + for report in parsed.reports: + if not os.path.exists(report): + return _fail(f"report {report} does not exist") + if parsed.names and len(parsed.names) != len(parsed.reports): + return _fail(f"got {len(parsed.names)} names for {len(parsed.reports)} reports") + + if _forced_failure(entry_point): + print(f"dummy TraceLens: forced failure for {entry_point}", file=sys.stderr) + return _FORCED_FAILURE_EXIT_CODE + + _write_text(parsed.output, f"dummy TraceLens comparison of {parsed.reports}\n") + print(f"dummy TraceLens: wrote {parsed.output}") + return 0 diff --git a/tests/fixtures/dummy_tracelens/TraceLens/Reporting/compare_perf_reports_pytorch.py b/tests/fixtures/dummy_tracelens/TraceLens/Reporting/compare_perf_reports_pytorch.py new file mode 100644 index 00000000..09b5a671 --- /dev/null +++ b/tests/fixtures/dummy_tracelens/TraceLens/Reporting/compare_perf_reports_pytorch.py @@ -0,0 +1,8 @@ +"""Dummy stand-in for the TraceLens report comparison entry point. + +Copyright (c) Advanced Micro Devices, Inc. All rights reserved. +""" + +from TraceLens.Reporting._dummy import compare_main + +main = compare_main diff --git a/tests/fixtures/dummy_tracelens/TraceLens/Reporting/generate_multi_rank_collective_report_pytorch.py b/tests/fixtures/dummy_tracelens/TraceLens/Reporting/generate_multi_rank_collective_report_pytorch.py new file mode 100644 index 00000000..f28172e9 --- /dev/null +++ b/tests/fixtures/dummy_tracelens/TraceLens/Reporting/generate_multi_rank_collective_report_pytorch.py @@ -0,0 +1,8 @@ +"""Dummy stand-in for the TraceLens multi-rank collective report generator. + +Copyright (c) Advanced Micro Devices, Inc. All rights reserved. +""" + +from TraceLens.Reporting._dummy import main_for + +main = main_for("TraceLens_generate_multi_rank_collective_report_pytorch") diff --git a/tests/fixtures/dummy_tracelens/TraceLens/Reporting/generate_perf_report_pftrace_hip_activity.py b/tests/fixtures/dummy_tracelens/TraceLens/Reporting/generate_perf_report_pftrace_hip_activity.py new file mode 100644 index 00000000..ac4efd01 --- /dev/null +++ b/tests/fixtures/dummy_tracelens/TraceLens/Reporting/generate_perf_report_pftrace_hip_activity.py @@ -0,0 +1,8 @@ +"""Dummy stand-in for the TraceLens pftrace HIP activity report generator. + +Copyright (c) Advanced Micro Devices, Inc. All rights reserved. +""" + +from TraceLens.Reporting._dummy import main_for + +main = main_for("TraceLens_generate_perf_report_pftrace_hip_activity") diff --git a/tests/fixtures/dummy_tracelens/TraceLens/Reporting/generate_perf_report_pftrace_hip_api.py b/tests/fixtures/dummy_tracelens/TraceLens/Reporting/generate_perf_report_pftrace_hip_api.py new file mode 100644 index 00000000..05d8fb10 --- /dev/null +++ b/tests/fixtures/dummy_tracelens/TraceLens/Reporting/generate_perf_report_pftrace_hip_api.py @@ -0,0 +1,8 @@ +"""Dummy stand-in for the TraceLens pftrace HIP API report generator. + +Copyright (c) Advanced Micro Devices, Inc. All rights reserved. +""" + +from TraceLens.Reporting._dummy import main_for + +main = main_for("TraceLens_generate_perf_report_pftrace_hip_api") diff --git a/tests/fixtures/dummy_tracelens/TraceLens/Reporting/generate_perf_report_pftrace_memory_copy.py b/tests/fixtures/dummy_tracelens/TraceLens/Reporting/generate_perf_report_pftrace_memory_copy.py new file mode 100644 index 00000000..f39cb371 --- /dev/null +++ b/tests/fixtures/dummy_tracelens/TraceLens/Reporting/generate_perf_report_pftrace_memory_copy.py @@ -0,0 +1,8 @@ +"""Dummy stand-in for the TraceLens pftrace memory copy report generator. + +Copyright (c) Advanced Micro Devices, Inc. All rights reserved. +""" + +from TraceLens.Reporting._dummy import main_for + +main = main_for("TraceLens_generate_perf_report_pftrace_memory_copy") diff --git a/tests/fixtures/dummy_tracelens/TraceLens/Reporting/generate_perf_report_pytorch.py b/tests/fixtures/dummy_tracelens/TraceLens/Reporting/generate_perf_report_pytorch.py new file mode 100644 index 00000000..cff1cedf --- /dev/null +++ b/tests/fixtures/dummy_tracelens/TraceLens/Reporting/generate_perf_report_pytorch.py @@ -0,0 +1,8 @@ +"""Dummy stand-in for the TraceLens PyTorch trace report generator. + +Copyright (c) Advanced Micro Devices, Inc. All rights reserved. +""" + +from TraceLens.Reporting._dummy import main_for + +main = main_for("TraceLens_generate_perf_report_pytorch") diff --git a/tests/fixtures/dummy_tracelens/TraceLens/Reporting/generate_perf_report_rocprof.py b/tests/fixtures/dummy_tracelens/TraceLens/Reporting/generate_perf_report_rocprof.py new file mode 100644 index 00000000..61982762 --- /dev/null +++ b/tests/fixtures/dummy_tracelens/TraceLens/Reporting/generate_perf_report_rocprof.py @@ -0,0 +1,8 @@ +"""Dummy stand-in for the TraceLens rocprofv3 JSON report generator. + +Copyright (c) Advanced Micro Devices, Inc. All rights reserved. +""" + +from TraceLens.Reporting._dummy import main_for + +main = main_for("TraceLens_generate_perf_report_rocprof") diff --git a/tests/fixtures/dummy_tracelens/TraceLens/__init__.py b/tests/fixtures/dummy_tracelens/TraceLens/__init__.py new file mode 100644 index 00000000..86e5155e --- /dev/null +++ b/tests/fixtures/dummy_tracelens/TraceLens/__init__.py @@ -0,0 +1,12 @@ +"""A stand-in for the real TraceLens package, for use in tests. + +The real package pins ``protobuf`` and ``xprof`` and only produces reports from +recorded GPU traces, so CI can neither install it nor feed it real input. This +stand-in is placed on ``PYTHONPATH`` instead, which lets the tests drive the +whole madengine analysis pipeline — the packaged analyzer, the in-container +post-script, and ``madengine report tracelens`` — with no GPU and no network. + +Copyright (c) Advanced Micro Devices, Inc. All rights reserved. +""" + +__version__ = "0.0.0.dummy"