From bdefd58f50c897222542d523b1bdc61fa677b3ba Mon Sep 17 00:00:00 2001 From: runwangdl Date: Thu, 20 Aug 2026 09:01:24 +0000 Subject: [PATCH] Report per-test cycle counts in PyTest and the CI job summary run_and_assert_test already had the parsed cycle count in its TestResult and dropped it. Print it on a canonical marker line -- before the assertions, so a failing test still reports -- and collect those in conftest. The collection runs in pytest_runtest_logreport, which fires on the xdist master for every worker's report; a module-level list filled inside the tests would stay in the worker process and never reach the summary. The terminal hook does run in the workers too, each holding only its own tests, so writing the job summary is guarded to the master -- otherwise every worker appends its own partial table to the same shared file. --- CHANGELOG.md | 2 + DeeployTest/conftest.py | 72 +++++++++++++++++++++++++++ DeeployTest/testUtils/pytestRunner.py | 10 ++++ 3 files changed, 84 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d15ac5bafd..f1206820b8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,6 +23,7 @@ This file contains the changelog for the Deeploy project. The changelog is divid - Fix GAP9 L3 Board Tests: readfs Flash Ordering and Duplicate Input Data [#196](https://github.com/pulp-platform/Deeploy/pull/196) - Add SoCDAML Part III: hands-on lab for adding a new int8 operator [#194](https://github.com/pulp-platform/Deeploy/pull/194) +- Report Test Cycles in PyTest and the CI Job Summary [#207](https://github.com/pulp-platform/Deeploy/pull/207) ### Added - tests for Regular and DW Conv2D with 3x3 kernel - Neureka's engine-aware DW lowering pass `NeurekaNCHWtoNHWCDwConvPass` @@ -44,6 +45,7 @@ This file contains the changelog for the Deeploy project. The changelog is divid - SoCDAML Part III lab: add an int8 `iLeakyReLU` to Deeploy and optimise it on Siracusa from scalar to tiled multi-core XPULP SIMD, with student skeletons and a TA reference under `Tutorials/` - Document that `--profileTiling` crashes GVSoC on the larger microLlama graphs (invalid access) +- Per-test cycle counts in the PyTest output and, under GitHub Actions, a `Performance Summary` table on the job summary page ### Changed - Refactor the topology optimization pass `NeurekaReshapePointwiseConvolutionPass` and Neureka's Tile constraints - `aie.dialects` API: move `link_with` from `aie_d.core()` to `aie_d.external_func()` (mlir-aie v1.3.2) diff --git a/DeeployTest/conftest.py b/DeeployTest/conftest.py index e37ddcf99b..b31e7fd106 100644 --- a/DeeployTest/conftest.py +++ b/DeeployTest/conftest.py @@ -3,10 +3,13 @@ # SPDX-License-Identifier: Apache-2.0 import os +import re from pathlib import Path +from typing import Any, Dict, List import coloredlogs import pytest +from testUtils.pytestRunner import PERF_MARKER, get_worker_id from Deeploy.Logging import DEFAULT_FMT from Deeploy.Logging import DEFAULT_LOGGER as log @@ -156,3 +159,72 @@ def toolchain(request): def cmake_args(request): """Return additional CMake arguments.""" return request.config.getoption("--cmake-args") + + +# --------------------------------------------------------------------------- +# Performance summary +# +# Every test funnels through run_and_assert_test, which prints one PERF_MARKER +# line per simulation. pytest_runtest_logreport runs on the xdist master for +# every worker's report, so scraping the captured stdout there is what makes the +# numbers from all four workers land in one process. Under GitHub Actions the +# same table is appended to the job summary, so the cycles show up on the check +# page without opening the log. +# --------------------------------------------------------------------------- + +_PERF_RESULTS: List[Dict[str, Any]] = [] + +_CYCLES_RE = re.compile(re.escape(PERF_MARKER) + r"\s+runtime_cycles=(\d+)") + + +def pytest_runtest_logreport(report: pytest.TestReport) -> None: + """Collect the runtime cycles each test reported.""" + if report.when != "call" or report.outcome not in ("passed", "failed"): + return + + blob = "\n".join(filter(None, [getattr(report, "capstdout", None), getattr(report, "capstderr", None)])) + matches = _CYCLES_RE.findall(blob) + if not matches: + return + + # A test that runs several simulations reports the last one, matching what + # its assertions were made against. + _PERF_RESULTS.append({ + "nodeid": report.nodeid, + "outcome": report.outcome, + "cycles": int(matches[-1]), + }) + + +def pytest_terminal_summary(terminalreporter, exitstatus, config) -> None: # noqa: ARG001 + """Print the cycle table and append it to the GitHub Actions job summary.""" + # This hook also fires in every xdist worker, each holding only the tests it + # ran. The master's logreport hook sees all of them, so it is the only one + # with a complete table -- and the only one that may append to the job + # summary, which is a shared file. + if get_worker_id() != "master" or not _PERF_RESULTS: + return + + results = sorted(_PERF_RESULTS, key = lambda r: r["nodeid"]) + + terminalreporter.write_sep("=", "Performance Summary") + width = max(len(r["nodeid"]) for r in results) + for r in results: + mark = "PASS" if r["outcome"] == "passed" else "FAIL" + terminalreporter.write_line(f" [{mark}] {r['nodeid']:<{width}} {r['cycles']:>12,} cycles") + + summaryPath = os.environ.get("GITHUB_STEP_SUMMARY") + if not summaryPath: + return + + lines = ["## Performance Summary", "", "| Test | Status | Runtime (cycles) |", "|---|:---:|---:|"] + for r in results: + status = ":white_check_mark:" if r["outcome"] == "passed" else ":x:" + lines.append(f"| `{r['nodeid']}` | {status} | {r['cycles']:,} |") + lines.append("") + + try: + with open(summaryPath, "a") as f: + f.write("\n".join(lines) + "\n") + except OSError as e: + terminalreporter.write_line(f"[perf-summary] could not write GITHUB_STEP_SUMMARY: {e}") diff --git a/DeeployTest/testUtils/pytestRunner.py b/DeeployTest/testUtils/pytestRunner.py index c0a597e587..953b2dafd9 100644 --- a/DeeployTest/testUtils/pytestRunner.py +++ b/DeeployTest/testUtils/pytestRunner.py @@ -8,7 +8,11 @@ from .core import DeeployTestConfig, build_binary, configure_cmake, get_test_paths, run_complete_test, run_simulation +#: Marker prefix for the per-test cycle line scraped by conftest's perf summary. +PERF_MARKER = "[deeploy-perf]" + __all__ = [ + 'PERF_MARKER', 'get_worker_id', 'create_test_config', 'run_and_assert_test', @@ -123,6 +127,12 @@ def run_and_assert_test(test_name: str, config: DeeployTestConfig, skipgen: bool """ result = run_complete_test(config, skipgen = skipgen, skipsim = skipsim) + # Printed before the assertions so a failing test still reports its cycles. + # stdout is the only channel that reaches the xdist master, where the + # end-of-session summary is assembled. + if result.runtime_cycles is not None: + print(f"{PERF_MARKER} runtime_cycles={result.runtime_cycles}") + assert result.success, (f"Test {test_name} failed with {result.error_count} errors out of {result.total_count}\n" f"Output:\n{result.stdout}")