diff --git a/CHANGELOG.md b/CHANGELOG.md index d15ac5bafd..f1206820b8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,6 +23,7 @@ This file contains the changelog for the Deeploy project. The changelog is divid - Fix GAP9 L3 Board Tests: readfs Flash Ordering and Duplicate Input Data [#196](https://github.com/pulp-platform/Deeploy/pull/196) - Add SoCDAML Part III: hands-on lab for adding a new int8 operator [#194](https://github.com/pulp-platform/Deeploy/pull/194) +- Report Test Cycles in PyTest and the CI Job Summary [#207](https://github.com/pulp-platform/Deeploy/pull/207) ### Added - tests for Regular and DW Conv2D with 3x3 kernel - Neureka's engine-aware DW lowering pass `NeurekaNCHWtoNHWCDwConvPass` @@ -44,6 +45,7 @@ This file contains the changelog for the Deeploy project. The changelog is divid - SoCDAML Part III lab: add an int8 `iLeakyReLU` to Deeploy and optimise it on Siracusa from scalar to tiled multi-core XPULP SIMD, with student skeletons and a TA reference under `Tutorials/` - Document that `--profileTiling` crashes GVSoC on the larger microLlama graphs (invalid access) +- Per-test cycle counts in the PyTest output and, under GitHub Actions, a `Performance Summary` table on the job summary page ### Changed - Refactor the topology optimization pass `NeurekaReshapePointwiseConvolutionPass` and Neureka's Tile constraints - `aie.dialects` API: move `link_with` from `aie_d.core()` to `aie_d.external_func()` (mlir-aie v1.3.2) diff --git a/DeeployTest/conftest.py b/DeeployTest/conftest.py index e37ddcf99b..b31e7fd106 100644 --- a/DeeployTest/conftest.py +++ b/DeeployTest/conftest.py @@ -3,10 +3,13 @@ # SPDX-License-Identifier: Apache-2.0 import os +import re from pathlib import Path +from typing import Any, Dict, List import coloredlogs import pytest +from testUtils.pytestRunner import PERF_MARKER, get_worker_id from Deeploy.Logging import DEFAULT_FMT from Deeploy.Logging import DEFAULT_LOGGER as log @@ -156,3 +159,72 @@ def toolchain(request): def cmake_args(request): """Return additional CMake arguments.""" return request.config.getoption("--cmake-args") + + +# --------------------------------------------------------------------------- +# Performance summary +# +# Every test funnels through run_and_assert_test, which prints one PERF_MARKER +# line per simulation. pytest_runtest_logreport runs on the xdist master for +# every worker's report, so scraping the captured stdout there is what makes the +# numbers from all four workers land in one process. Under GitHub Actions the +# same table is appended to the job summary, so the cycles show up on the check +# page without opening the log. +# --------------------------------------------------------------------------- + +_PERF_RESULTS: List[Dict[str, Any]] = [] + +_CYCLES_RE = re.compile(re.escape(PERF_MARKER) + r"\s+runtime_cycles=(\d+)") + + +def pytest_runtest_logreport(report: pytest.TestReport) -> None: + """Collect the runtime cycles each test reported.""" + if report.when != "call" or report.outcome not in ("passed", "failed"): + return + + blob = "\n".join(filter(None, [getattr(report, "capstdout", None), getattr(report, "capstderr", None)])) + matches = _CYCLES_RE.findall(blob) + if not matches: + return + + # A test that runs several simulations reports the last one, matching what + # its assertions were made against. + _PERF_RESULTS.append({ + "nodeid": report.nodeid, + "outcome": report.outcome, + "cycles": int(matches[-1]), + }) + + +def pytest_terminal_summary(terminalreporter, exitstatus, config) -> None: # noqa: ARG001 + """Print the cycle table and append it to the GitHub Actions job summary.""" + # This hook also fires in every xdist worker, each holding only the tests it + # ran. The master's logreport hook sees all of them, so it is the only one + # with a complete table -- and the only one that may append to the job + # summary, which is a shared file. + if get_worker_id() != "master" or not _PERF_RESULTS: + return + + results = sorted(_PERF_RESULTS, key = lambda r: r["nodeid"]) + + terminalreporter.write_sep("=", "Performance Summary") + width = max(len(r["nodeid"]) for r in results) + for r in results: + mark = "PASS" if r["outcome"] == "passed" else "FAIL" + terminalreporter.write_line(f" [{mark}] {r['nodeid']:<{width}} {r['cycles']:>12,} cycles") + + summaryPath = os.environ.get("GITHUB_STEP_SUMMARY") + if not summaryPath: + return + + lines = ["## Performance Summary", "", "| Test | Status | Runtime (cycles) |", "|---|:---:|---:|"] + for r in results: + status = ":white_check_mark:" if r["outcome"] == "passed" else ":x:" + lines.append(f"| `{r['nodeid']}` | {status} | {r['cycles']:,} |") + lines.append("") + + try: + with open(summaryPath, "a") as f: + f.write("\n".join(lines) + "\n") + except OSError as e: + terminalreporter.write_line(f"[perf-summary] could not write GITHUB_STEP_SUMMARY: {e}") diff --git a/DeeployTest/testUtils/pytestRunner.py b/DeeployTest/testUtils/pytestRunner.py index c0a597e587..953b2dafd9 100644 --- a/DeeployTest/testUtils/pytestRunner.py +++ b/DeeployTest/testUtils/pytestRunner.py @@ -8,7 +8,11 @@ from .core import DeeployTestConfig, build_binary, configure_cmake, get_test_paths, run_complete_test, run_simulation +#: Marker prefix for the per-test cycle line scraped by conftest's perf summary. +PERF_MARKER = "[deeploy-perf]" + __all__ = [ + 'PERF_MARKER', 'get_worker_id', 'create_test_config', 'run_and_assert_test', @@ -123,6 +127,12 @@ def run_and_assert_test(test_name: str, config: DeeployTestConfig, skipgen: bool """ result = run_complete_test(config, skipgen = skipgen, skipsim = skipsim) + # Printed before the assertions so a failing test still reports its cycles. + # stdout is the only channel that reaches the xdist master, where the + # end-of-session summary is assembled. + if result.runtime_cycles is not None: + print(f"{PERF_MARKER} runtime_cycles={result.runtime_cycles}") + assert result.success, (f"Test {test_name} failed with {result.error_count} errors out of {result.total_count}\n" f"Output:\n{result.stdout}")