diff --git a/docs/profiling.md b/docs/profiling.md index 39352804..76c688b4 100644 --- a/docs/profiling.md +++ b/docs/profiling.md @@ -394,7 +394,11 @@ For a short-lived workload, shorten the warmup so the request lands while the mo } ``` -**No trace produced?** The teardown script reports how many traces it found. `no PyTorch process registered` means `dyno gputrace` never matched the workload: confirm the model is PyTorch, and raise `TORCH_PROFILE_WARMUP_S` and `TORCH_PROFILE_MAX_ATTEMPTS` for slow-starting jobs. +**No trace produced?** The teardown script reports how many traces it found. `no PyTorch process registered` means `dyno gputrace` never matched the workload: confirm the model is PyTorch, and raise `TORCH_PROFILE_WARMUP_S` and `TORCH_PROFILE_MAX_ATTEMPTS` for slow-starting jobs. `dyno rejected the trace request` means the request itself was refused, which points at a dynolog version mismatch rather than at the workload. + +**Trace produced but empty?** Iteration-based capture waits for the workload's next `optimizer.step()`, so the request has to land after training has actually started. A request that arrives while the model is still being built (or while MIOpen is autotuning the first convolution) yields a trace with no GPU activity. The warmup must cover startup, not just process launch. + +Every process that registers with dynolog is traced, including the `torchrun` launcher, which supervises its children and runs no kernels itself. An `N`-rank job therefore produces `N + 1` traces, and the launcher's holds nothing to report. ### tracelens - TraceLens Trace Analysis @@ -411,6 +415,8 @@ Each trace format is routed to the matching TraceLens report: **Unreadable formats:** TraceLens cannot read rocprofv3's default SQLite (`*_results.db`) or RPD (`.rpd`) databases. Those are listed in the summary as `SKIPPED` with a pointer at a preset that works — use `rocprofv3_lightweight` for JSON or `rocprofv3_perfetto` for `.pftrace`. For `rpd`, point TraceLens at the `trace.json` its post-script writes alongside the database. +**Traces with nothing to analyze:** a trace that holds no GPU activity is also reported as `SKIPPED` rather than as a failure. The usual source is the `torchrun` launcher process, which dynolog traces along with the ranks that do the work. + #### Analyzing on the Host (Recommended) Running analysis on the host keeps TraceLens' pinned `protobuf` and `xprof` out of your workload image: @@ -467,6 +473,8 @@ Use a mode-specific variant to restrict analysis to one trace kind: | `tracelens_pftrace` | Perfetto traces only | | `tracelens_collective` | Multi-rank collective report only | +Kineto traces yield both an `.xlsx` workbook and a CSV directory per trace. rocprofv3 traces yield the CSV directory only: TraceLens' rocprof report writes either format but not both, and madengine asks for the CSVs. + **Full PyTorch pipeline** — capture and analyze in one run: ```bash diff --git a/src/madengine/scripts/common/tools/tracelens_analyze.py b/src/madengine/scripts/common/tools/tracelens_analyze.py index cd23dc9b..75c74458 100644 --- a/src/madengine/scripts/common/tools/tracelens_analyze.py +++ b/src/madengine/scripts/common/tools/tracelens_analyze.py @@ -97,6 +97,14 @@ # Read size used when checking a trace for undecodable bytes and rewriting it. _SANITIZE_CHUNK_BYTES = 1 << 20 +# What TraceLens says when a trace holds no GPU activity for it to report on. +_NO_GPU_EVENTS_ERROR = "No GPU events found in the trace" +_NO_GPU_EVENTS_REASON = ( + "the trace holds no GPU activity, so there is nothing to report. dynolog " + "configures every process that registered with it, which for a torchrun job " + "includes the launcher: it only supervises its children and runs no kernels." +) + SUMMARY_CSV_FIELDS = ( "trace_file", "kind", @@ -329,6 +337,11 @@ def _failure_detail(returncode: int, output: str) -> str: return lines[-1] if lines else f"exit code {returncode}" +def _has_no_gpu_events(output: str) -> bool: + """Return True when TraceLens refused a trace for carrying no GPU activity.""" + return _NO_GPU_EVENTS_ERROR in output + + def _report_stem(path: str, root: str) -> str: """Return a filesystem-safe, collision-resistant name for a trace's reports.""" relative = os.path.relpath(path, root) @@ -342,6 +355,13 @@ def _report_stem(path: str, root: str) -> str: def _pytorch_args( trace: str, out_base: str, gpu_arch: Optional[str], extra: Sequence[str] ) -> List[str]: + # --short_kernel_study is deliberately not requested. On some real traces its + # sheets have MultiIndex columns, and TraceLens writes every sheet with + # `index=False`, which pandas refuses: "Writing to Excel with MultiIndex + # columns and no index ('index'=False) is not yet implemented". That loses the + # whole workbook after the CSVs are already written. Pass it back through the + # analyzer's trailing extra args if you want those sheets. + # https://github.com/AMD-AGI/TraceLens/issues/938 args = [ "--profile_json_path", trace, @@ -350,7 +370,6 @@ def _pytorch_args( "--output_csvs_dir", f"{out_base}_csv", "--enable_kernel_summary", - "--short_kernel_study", ] if gpu_arch: args += ["--gpu_arch_platform", gpu_arch] @@ -358,6 +377,10 @@ def _pytorch_args( def _rocprof_args(trace: str, out_base: str, extra: Sequence[str]) -> List[str]: + # --short_kernel_study is safe to keep here, unlike for the PyTorch report: + # this generator writes either the CSVs or the workbook, never both, so with + # a CSV directory requested it never reaches the Excel writer that the flag's + # MultiIndex sheets break. That also means --output_xlsx_path is ignored. return [ "--profile_json_path", trace, @@ -567,6 +590,16 @@ def analyze( for trace, kind, tool, args in jobs: print(f"[tracelens] {tool}: {trace}", flush=True) code, output = _run(_build_command(interpreter, tool, args)) + if code == 0: + status, detail, produced = "SUCCESS", "", os.path.relpath(output_dir, root) + elif _has_no_gpu_events(output): + # Nothing was wrong with the analysis, and nothing was produced. + print(f"[tracelens] skipping {trace}: {_NO_GPU_EVENTS_REASON}", flush=True) + status, detail, produced = "SKIPPED", _NO_GPU_EVENTS_REASON, "" + else: + status = "FAILURE" + detail = _failure_detail(code, output) + produced = os.path.relpath(output_dir, root) results.append( { "trace_file": ( @@ -574,9 +607,9 @@ def analyze( ), "kind": kind, "tracelens_tool": tool, - "status": "SUCCESS" if code == 0 else "FAILURE", - "output": os.path.relpath(output_dir, root), - "detail": "" if code == 0 else _failure_detail(code, output), + "status": status, + "output": produced, + "detail": detail, } ) diff --git a/tests/e2e/test_tracelens_dummy_pipeline.py b/tests/e2e/test_tracelens_dummy_pipeline.py index b3da5679..27024b93 100644 --- a/tests/e2e/test_tracelens_dummy_pipeline.py +++ b/tests/e2e/test_tracelens_dummy_pipeline.py @@ -188,6 +188,32 @@ def write_chrome_trace(path: Path) -> Path: return path +def write_chrome_trace_without_gpu_events(path: Path) -> Path: + """Write the kind of trace dynolog collects from a torchrun launcher. + + The launcher registers with dynolog like any other PyTorch process, so it is + configured and traced alongside the ranks, but it only supervises children: + its trace carries Python frames and no GPU work at all. + """ + payload = { + "schemaVersion": 1, + "traceEvents": [ + { + "ph": "X", + "cat": "python_function", + "name": "torch/distributed/run.py(892): main", + "pid": 724, + "tid": 724, + "ts": 100, + "dur": 9000, + } + ], + } + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(payload), encoding="utf-8") + return path + + def write_rocprof_json(path: Path) -> Path: """Write a rocprofv3 JSON result document, as ``rocprofv3_lightweight`` does. @@ -391,6 +417,32 @@ def test_a_valid_trace_is_analyzed_where_it_lies(self, profiled_run, dummy_trace assert analyzed == str(profiled_run / "rocprof_output" / "1234_results.json") assert "sanitized copy" not in result.stdout + def test_a_launcher_trace_is_skipped_rather_than_failed( + self, tmp_path, dummy_tracelens + ): + """A torchrun job hands madengine one trace with no GPU work every run. + + dynolog configures every process that registered with it, so the ranks' + traces arrive next to the launcher's. Failing on the launcher would mean + every distributed run ends with a failure row beside its real report. + """ + work = tmp_path / "torchrun" + write_chrome_trace(work / "torch_profiler_output" / "libkineto_trace_892.json") + write_chrome_trace_without_gpu_events( + work / "torch_profiler_output" / "libkineto_trace_724.json" + ) + + result = run_analyzer(work, dummy_tracelens) + + assert result.returncode == 0, result.stdout + rows = {Path(row["trace_file"]).name: row for row in summary_rows(work)} + assert rows["libkineto_trace_892.json"]["status"] == "SUCCESS" + launcher = rows["libkineto_trace_724.json"] + assert launcher["status"] == "SKIPPED", launcher + assert "no GPU activity" in launcher["detail"] + # No report was written for it, so naming an output would mislead. + assert launcher["output"] == "" + def test_gzipped_kineto_trace_is_analyzed(self, tmp_path, dummy_tracelens): """tensorboard_trace_handler's gzipped traces are picked up too.""" work = tmp_path / "gz" diff --git a/tests/fixtures/dummy_tracelens/TraceLens/Reporting/_dummy.py b/tests/fixtures/dummy_tracelens/TraceLens/Reporting/_dummy.py index 0f81bb00..feb423a7 100644 --- a/tests/fixtures/dummy_tracelens/TraceLens/Reporting/_dummy.py +++ b/tests/fixtures/dummy_tracelens/TraceLens/Reporting/_dummy.py @@ -18,6 +18,7 @@ import argparse import glob +import gzip import json import os import sys @@ -29,9 +30,10 @@ _SPECS: Dict[str, Dict[str, object]] = { "TraceLens_generate_perf_report_pytorch": { "required": ("--profile_json_path", "--output_xlsx_path", "--output_csvs_dir"), - "switches": ("--enable_kernel_summary", "--short_kernel_study"), + "switches": ("--enable_kernel_summary",), "optional": ("--gpu_arch_platform",), "input_file": "profile_json_path", + "needs_gpu_events": True, }, "TraceLens_generate_perf_report_rocprof": { "required": ("--profile_json_path", "--output_xlsx_path", "--output_csvs_dir"), @@ -65,6 +67,14 @@ _FORCED_FAILURE_EXIT_CODE = 3 +# Kineto categories that carry GPU work. A trace without any of them gives +# TraceLens nothing to report on, and it says so rather than writing an empty +# report. dynolog traces every process that registered with it, so a torchrun +# job hands madengine one such trace per run: the launcher's. +_GPU_ACTIVITY_CATEGORIES = frozenset( + {"kernel", "gpu_memcpy", "gpu_memset", "gpu_user_annotation"} +) + def _parser(entry_point: str) -> argparse.ArgumentParser: spec = _SPECS[entry_point] @@ -108,6 +118,21 @@ def _check_input_file(path: str) -> Optional[int]: return None +def _check_gpu_events(path: str) -> Optional[int]: + opener = gzip.open if path.endswith(".gz") else open + try: + with opener(path, "rt", encoding="utf-8") as handle: + payload = json.load(handle) + except (OSError, ValueError): + # Unreadable input is what the other checks are for. + return None + events = payload.get("traceEvents", []) if isinstance(payload, dict) else [] + for event in events: + if isinstance(event, dict) and event.get("cat") in _GPU_ACTIVITY_CATEGORIES: + return None + return _fail("ValueError: No GPU events found in the trace") + + def _check_input_glob(pattern: str, world_size: str) -> Optional[int]: matches = [p for p in glob.glob(pattern, recursive=True) if os.path.isfile(p)] if len(matches) < 2: @@ -172,9 +197,14 @@ def _report(entry_point: str, argv: Optional[Sequence[str]]) -> int: spec = _SPECS[entry_point] if "input_file" in spec: - failure = _check_input_file(getattr(parsed, str(spec["input_file"]))) + trace = getattr(parsed, str(spec["input_file"])) + failure = _check_input_file(trace) if failure is not None: return failure + if spec.get("needs_gpu_events"): + failure = _check_gpu_events(trace) + if failure is not None: + return failure if "input_glob" in spec: failure = _check_input_glob( getattr(parsed, str(spec["input_glob"])), parsed.world_size diff --git a/tests/unit/test_tracelens_analyze.py b/tests/unit/test_tracelens_analyze.py index 6a6312ab..9e3e1354 100644 --- a/tests/unit/test_tracelens_analyze.py +++ b/tests/unit/test_tracelens_analyze.py @@ -153,6 +153,18 @@ def test_pytorch_args_omit_roofline_without_arch(self, analyzer): args = analyzer._pytorch_args("t.json", "/out/t", None, []) assert "--gpu_arch_platform" not in args + def test_pytorch_args_do_not_request_the_short_kernel_study(self, analyzer): + # TraceLens writes that sheet with index=False, and pandas refuses when + # its columns are a MultiIndex, which loses the whole workbook. + args = analyzer._pytorch_args("t.json", "/out/t", "MI300X", []) + assert "--short_kernel_study" not in args + + def test_the_short_kernel_study_can_be_asked_for_explicitly(self, analyzer): + args = analyzer._pytorch_args( + "t.json", "/out/t", "MI300X", ["--short_kernel_study"] + ) + assert "--short_kernel_study" in args + def test_pftrace_produces_three_complementary_reports(self, analyzer): jobs = analyzer._pftrace_jobs("t.pftrace", "/out/t", []) assert [tool for tool, _ in jobs] == [ @@ -227,6 +239,36 @@ def fake_run(command, cwd=None): assert len(rows) == 11 assert set(rows[0]) == set(analyzer.SUMMARY_CSV_FIELDS) + def test_a_trace_without_gpu_activity_is_skipped_not_failed( + self, analyzer, tmp_path, monkeypatch + ): + """dynolog traces the torchrun launcher too, and it runs no kernels. + + Reporting that as a failure means every multi-process run profiled + through dynolog ends with a failure row next to its real report. + """ + _write(tmp_path, "torch_profiler_output/libkineto_trace_724.json", CHROME_TRACE) + monkeypatch.setattr( + analyzer, + "_run", + lambda command, cwd=None: ( + 1, + "Traceback (most recent call last):\n" + "ValueError: No GPU events found in the trace", + ), + ) + + summary = analyzer.analyze( + root=str(tmp_path), output_dir=str(tmp_path / "out"), python=sys.executable + ) + assert summary["failed"] == 0 + assert summary["skipped"] == 1 + row = summary["results"][0] + assert row["status"] == "SKIPPED" + assert "no GPU activity" in row["detail"] + # Nothing was written, so pointing at a report directory would mislead. + assert row["output"] == "" + def test_records_failure_detail(self, analyzer, tmp_path, monkeypatch): _write(tmp_path, "rocprof_output/1_results.json", b"{}") monkeypatch.setattr(