Report Test Cycles in PyTest and the CI Job Summary

CHANGELOG.md

@@ -23,6 +23,7 @@ This file contains the changelog for the Deeploy project. The changelog is divid
 - Fix GAP9 L3 Board Tests: readfs Flash Ordering and Duplicate Input Data [#196](https://github.com/pulp-platform/Deeploy/pull/196)
 - Add SoCDAML Part III: hands-on lab for adding a new int8 operator [#194](https://github.com/pulp-platform/Deeploy/pull/194)
 
+- Report Test Cycles in PyTest and the CI Job Summary [#207](https://github.com/pulp-platform/Deeploy/pull/207)
 ### Added
 - tests for Regular and DW Conv2D with 3x3 kernel
 - Neureka's engine-aware DW lowering pass `NeurekaNCHWtoNHWCDwConvPass`
@@ -44,6 +45,7 @@ This file contains the changelog for the Deeploy project. The changelog is divid
 - SoCDAML Part III lab: add an int8 `iLeakyReLU` to Deeploy and optimise it on Siracusa from scalar to tiled multi-core XPULP SIMD, with student skeletons and a TA reference under `Tutorials/`
 - Document that `--profileTiling` crashes GVSoC on the larger microLlama graphs (invalid access)
 
+- Per-test cycle counts in the PyTest output and, under GitHub Actions, a `Performance Summary` table on the job summary page
 ### Changed
 - Refactor the topology optimization pass `NeurekaReshapePointwiseConvolutionPass` and Neureka's Tile constraints
 - `aie.dialects` API: move `link_with` from `aie_d.core()` to `aie_d.external_func()` (mlir-aie v1.3.2)

DeeployTest/conftest.py

@@ -3,10 +3,13 @@
 # SPDX-License-Identifier: Apache-2.0
 
 import os
+import re
 from pathlib import Path
+from typing import Any, Dict, List
 
 import coloredlogs
 import pytest
+from testUtils.pytestRunner import PERF_MARKER, get_worker_id
 
 from Deeploy.Logging import DEFAULT_FMT
 from Deeploy.Logging import DEFAULT_LOGGER as log
@@ -156,3 +159,72 @@ def toolchain(request):
 def cmake_args(request):
     """Return additional CMake arguments."""
     return request.config.getoption("--cmake-args")
+
+
+# ---------------------------------------------------------------------------
+# Performance summary
+#
+# Every test funnels through run_and_assert_test, which prints one PERF_MARKER
+# line per simulation. pytest_runtest_logreport runs on the xdist master for
+# every worker's report, so scraping the captured stdout there is what makes the
+# numbers from all four workers land in one process. Under GitHub Actions the
+# same table is appended to the job summary, so the cycles show up on the check
+# page without opening the log.
+# ---------------------------------------------------------------------------
+
+_PERF_RESULTS: List[Dict[str, Any]] = []
+
+_CYCLES_RE = re.compile(re.escape(PERF_MARKER) + r"\s+runtime_cycles=(\d+)")
+
+
+def pytest_runtest_logreport(report: pytest.TestReport) -> None:
+    """Collect the runtime cycles each test reported."""
+    if report.when != "call" or report.outcome not in ("passed", "failed"):
+        return
+
+    blob = "\n".join(filter(None, [getattr(report, "capstdout", None), getattr(report, "capstderr", None)]))
+    matches = _CYCLES_RE.findall(blob)
+    if not matches:
+        return
+
+    # A test that runs several simulations reports the last one, matching what
+    # its assertions were made against.
+    _PERF_RESULTS.append({
+        "nodeid": report.nodeid,
+        "outcome": report.outcome,
+        "cycles": int(matches[-1]),
+    })
+
+
+def pytest_terminal_summary(terminalreporter, exitstatus, config) -> None:  # noqa: ARG001
+    """Print the cycle table and append it to the GitHub Actions job summary."""
+    # This hook also fires in every xdist worker, each holding only the tests it
+    # ran. The master's logreport hook sees all of them, so it is the only one
+    # with a complete table -- and the only one that may append to the job
+    # summary, which is a shared file.
+    if get_worker_id() != "master" or not _PERF_RESULTS:
+        return
+
+    results = sorted(_PERF_RESULTS, key = lambda r: r["nodeid"])
+
+    terminalreporter.write_sep("=", "Performance Summary")
+    width = max(len(r["nodeid"]) for r in results)
+    for r in results:
+        mark = "PASS" if r["outcome"] == "passed" else "FAIL"
+        terminalreporter.write_line(f"  [{mark}] {r['nodeid']:<{width}}  {r['cycles']:>12,} cycles")
+
+    summaryPath = os.environ.get("GITHUB_STEP_SUMMARY")
+    if not summaryPath:
+        return
+
+    lines = ["## Performance Summary", "", "| Test | Status | Runtime (cycles) |", "|---|:---:|---:|"]
+    for r in results:
+        status = ":white_check_mark:" if r["outcome"] == "passed" else ":x:"
+        lines.append(f"| `{r['nodeid']}` | {status} | {r['cycles']:,} |")
+    lines.append("")
+
+    try:
+        with open(summaryPath, "a") as f:
+            f.write("\n".join(lines) + "\n")
+    except OSError as e:
+        terminalreporter.write_line(f"[perf-summary] could not write GITHUB_STEP_SUMMARY: {e}")

DeeployTest/testUtils/pytestRunner.py

@@ -8,7 +8,11 @@
 
 from .core import DeeployTestConfig, build_binary, configure_cmake, get_test_paths, run_complete_test, run_simulation
 
+#: Marker prefix for the per-test cycle line scraped by conftest's perf summary.
+PERF_MARKER = "[deeploy-perf]"
+
 __all__ = [
+    'PERF_MARKER',
     'get_worker_id',
     'create_test_config',
     'run_and_assert_test',
@@ -123,6 +127,12 @@ def run_and_assert_test(test_name: str, config: DeeployTestConfig, skipgen: bool
     """
     result = run_complete_test(config, skipgen = skipgen, skipsim = skipsim)
 
+    # Printed before the assertions so a failing test still reports its cycles.
+    # stdout is the only channel that reaches the xdist master, where the
+    # end-of-session summary is assembled.
+    if result.runtime_cycles is not None:
+        print(f"{PERF_MARKER} runtime_cycles={result.runtime_cycles}")
+
     assert result.success, (f"Test {test_name} failed with {result.error_count} errors out of {result.total_count}\n"
                             f"Output:\n{result.stdout}")