From 3b3ab5d394e2b7a3ece62a0e05836232a2f19545 Mon Sep 17 00:00:00 2001 From: Ivan Podkidyshev Date: Wed, 29 Jul 2026 11:21:44 +0200 Subject: [PATCH] better comparison reports --- .../report_generator/comparison_report.py | 426 +++++++++++++++++- src/cloudai/report_generator/groups.py | 12 + src/cloudai/util/comparison-report-v2.jinja2 | 306 +++++++++++++ .../workloads/common/llm_serving_report.py | 128 +----- .../nccl_test/nccl_comparison_report.py | 38 +- .../nixl_bench/nixl_summary_report.py | 35 +- .../nixl_ep/nixl_ep_comparison_report.py | 102 +---- .../osu_bench/osu_comparison_report.py | 73 +-- .../test_comparison_report.py | 118 ++++- .../test_report_groups.py | 42 +- .../ai_dynamo/test_comparison_report.py | 1 + .../common/test_llm_serving_report.py | 2 + .../nixl_ep/test_comparison_report.py | 1 + 13 files changed, 957 insertions(+), 327 deletions(-) create mode 100644 src/cloudai/util/comparison-report-v2.jinja2 diff --git a/src/cloudai/report_generator/comparison_report.py b/src/cloudai/report_generator/comparison_report.py index 6f3ccd960..7a9ec9634 100644 --- a/src/cloudai/report_generator/comparison_report.py +++ b/src/cloudai/report_generator/comparison_report.py @@ -18,9 +18,12 @@ import logging from abc import ABC, abstractmethod +from collections import Counter, defaultdict +from collections.abc import Mapping +from dataclasses import dataclass from itertools import cycle from pathlib import Path -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Any, ClassVar, Literal import jinja2 from pydantic import Field @@ -43,6 +46,21 @@ import pandas as pd +@dataclass +class ComparisonSection: + """Normalized comparison data consumed by both report renderers.""" + + group: GroupedTestRuns + title: str + dfs: list[pd.DataFrame] + info_columns: list[str] + data_columns: list[str] + y_axis_label: str + chart_type: Literal["line", "bar"] = "line" + legacy_chart_title: str | None = None + legacy_category_prefix: str = "" + + class ComparisonReportConfig(ReportConfig): """Configuration for a comparison report.""" @@ -53,12 +71,41 @@ class ComparisonReportConfig(ReportConfig): class ComparisonReport(Reporter, ABC): """Base class for comparison reports that generate both charts and tables.""" + CHART_COLORS: ClassVar[tuple[str, ...]] = ( + "#5e9c00", + "#39424e", + "#2563eb", + "#b45309", + "#7c3aed", + "#0891b2", + "#be123c", + ) + CHART_POINT_STYLES: ClassVar[tuple[str, ...]] = ( + "circle", + "rectRot", + "triangle", + "rect", + "star", + "crossRot", + "dash", + ) + CHART_DASH_STYLES: ClassVar[tuple[tuple[int, ...], ...]] = ( + (), + (8, 4), + (2, 3), + (10, 3, 2, 3), + (5, 5), + (12, 4), + (3, 2), + ) + def __init__( self, system: System, test_scenario: TestScenario, results_root: Path, config: ComparisonReportConfig ) -> None: super().__init__(system, test_scenario, results_root, config) self.template_path = Path(__file__).parent.parent / "util" self.template_name = "nixl_report_template.jinja2" + self.v2_template_name = "comparison-report-v2.jinja2" self.report_file_name: str = "comparison_report.html" self.group_by: list[str] = config.group_by @@ -66,10 +113,8 @@ def __init__( def extract_data_as_df(self, tr: TestRun) -> pd.DataFrame: ... @abstractmethod - def create_tables(self, cmp_groups: list[GroupedTestRuns]) -> list[Table]: ... - - @abstractmethod - def create_charts(self, cmp_groups: list[GroupedTestRuns]) -> list[bk.figure]: ... + def build_sections(self, cmp_groups: list[GroupedTestRuns]) -> list[ComparisonSection]: + """Return normalized sections without performing any rendering.""" def comparison_values(self, tr: TestRun) -> dict[str, object]: """Return TestRun values used to label differences between compared runs.""" @@ -96,15 +141,61 @@ def group_name(self, trs: list[TestRun]) -> str: def create_group(self, trs: list[TestRun], group_idx: str = "0") -> GroupedTestRuns: """Create a comparison group using report-specific comparison values.""" diff = diff_comparison_values([self.comparison_values(tr) for tr in trs]) + compact_names = [self._compact_case_name(tr) for tr in trs] + duplicate_names = Counter(compact_names) + duplicate_indexes: defaultdict[str, int] = defaultdict(int) items: list[TRGroupItem] = [] for idx, tr in enumerate(trs): name = f"{group_idx}.{idx}" if diff: item_name_parts = [f"{field}={vals[idx]}" for field, vals in diff.items()] name = " ".join(item_name_parts).replace("extra_env_vars.", "") - items.append(TRGroupItem(name=name, tr=tr)) + + compact_name = compact_names[idx] + if duplicate_names[compact_name] > 1: + duplicate_indexes[compact_name] += 1 + compact_name = f"{compact_name} · run={duplicate_indexes[compact_name]}" + + diff_label = self._format_diff_label(diff, idx) + full_name = f"{compact_name} — {diff_label}" if diff_label else compact_name + items.append( + TRGroupItem( + name=name, + tr=tr, + compact_name=compact_name, + full_name=full_name, + ) + ) return GroupedTestRuns(name=self.group_name(trs), items=items) + @staticmethod + def _compact_case_name(tr: TestRun) -> str: + parts = [tr.name] + if tr.step > 0: + parts.append(f"step={tr.step}") + if tr.iterations > 1: + parts.append(f"iter={tr.current_iteration}") + return " · ".join(parts) + + @classmethod + def _flatten_diff_value(cls, field: str, value: object) -> list[str]: + if isinstance(value, Mapping): + flattened: list[str] = [] + for key, nested_value in value.items(): + nested_field = f"{field}.{key}" if field else str(key) + flattened.extend(cls._flatten_diff_value(nested_field, nested_value)) + return flattened + + value_text = ",".join(str(item) for item in value) if isinstance(value, (list, tuple)) else str(value) + return [f"{field.replace('extra_env_vars.', '')}={value_text}"] + + @classmethod + def _format_diff_label(cls, diff: dict[str, list[object]], item_idx: int) -> str: + parts: list[str] = [] + for field, values in diff.items(): + parts.extend(cls._flatten_diff_value(field, values[item_idx])) + return " · ".join(parts) + def group_test_runs(self) -> list[GroupedTestRuns]: """Group loaded TestRuns for this comparison report.""" if not self.group_by: @@ -125,9 +216,56 @@ def group_test_runs(self) -> list[GroupedTestRuns]: return [self.create_group(group, group_idx=str(group_idx)) for group_idx, group in enumerate(groups)] - def get_bokeh_html(self) -> tuple[str, str]: - cmp_groups = self.group_test_runs() - charts: list[bk.figure] = self.create_charts(cmp_groups) + def create_tables(self, cmp_groups: list[GroupedTestRuns]) -> list[Table]: + """Render legacy Rich tables from normalized sections.""" + return [ + self.create_table( + section.group, + section.dfs, + section.title, + section.info_columns, + section.data_columns, + ) + for section in self.build_sections(cmp_groups) + ] + + def create_charts(self, cmp_groups: list[GroupedTestRuns]) -> list[bk.figure]: + """Render legacy Bokeh charts from normalized sections.""" + charts: list[bk.figure] = [] + for section in self.build_sections(cmp_groups): + if section.chart_type == "bar": + charts.append(self.create_bar_chart(section)) + else: + charts.append( + self.create_chart( + section.group, + section.dfs, + section.legacy_chart_title or section.title, + section.info_columns, + section.data_columns, + section.y_axis_label, + ) + ) + return charts + + def get_bokeh_html(self, sections: list[ComparisonSection] | None = None) -> tuple[str, str]: + if sections is None: + sections = self.build_sections(self.group_test_runs()) + charts: list[bk.figure] = [] + for section in sections: + if section.chart_type == "bar": + charts.append(self.create_bar_chart(section)) + else: + charts.append( + self.create_chart( + section.group, + section.dfs, + section.legacy_chart_title or section.title, + section.info_columns, + section.data_columns, + section.y_axis_label, + ) + ) # layout with 2 charts per row rows = [] @@ -141,7 +279,7 @@ def get_bokeh_html(self) -> tuple[str, str]: bokeh_script, bokeh_div = lazy.bokeh_embed.components(layout) return bokeh_script, bokeh_div - def generate(self): + def generate(self) -> None: self.load_test_runs() if not self.trs: logging.debug(f"Skipping {self.__class__.__name__} report generation, no results found.") @@ -149,17 +287,26 @@ def generate(self): console = Console(record=True) cmp_groups = self.group_test_runs() - - tables = self.create_tables(cmp_groups) - for table in tables: + sections = self.build_sections(cmp_groups) + + for section in sections: + table = self.create_table( + section.group, + section.dfs, + section.title, + section.info_columns, + section.data_columns, + ) console.print(table) console.print() - bokeh_script, bokeh_div = self.get_bokeh_html() + bokeh_script, bokeh_div = self.get_bokeh_html(sections) - template = jinja2.Environment(loader=jinja2.FileSystemLoader(self.template_path)).get_template( - self.template_name + env = jinja2.Environment( + loader=jinja2.FileSystemLoader(self.template_path), + autoescape=jinja2.select_autoescape(), ) + template = env.get_template(self.template_name) html_content = template.render( title=f"{self.test_scenario.name} Comparison Report", bokeh_script=bokeh_script, @@ -173,6 +320,22 @@ def generate(self): logging.info(f"Comparison report created: {html_file}") + v2_template = env.get_template(self.v2_template_name) + v2_content = v2_template.render( + name=f"{self.test_scenario.name} Comparison Report", + sections=self._build_v2_sections(sections), + ) + v2_file = self.results_root / self.v2_report_file_name + with v2_file.open("w") as f: + f.write(v2_content) + + logging.info(f"Comparison report v2 created: {v2_file}") + + @property + def v2_report_file_name(self) -> str: + report_path = Path(self.report_file_name) + return f"{report_path.stem}_v2{report_path.suffix}" + @staticmethod def _extract_cmp_values(data: list) -> tuple[float | None, float | None]: val1, val2 = None, None @@ -191,6 +354,175 @@ def _format_diff_cell(val1: float | None, val2: float | None) -> str: diff_percent = (diff / val2) * 100 return f"{diff:+.2f} ({diff_percent:+.2f}%)" + @staticmethod + def _display_value(value: Any) -> str: + if value is None: + return "n/a" + try: + if lazy.pd.isna(value): + return "n/a" + except (TypeError, ValueError): + pass + return str(value) + + @staticmethod + def _numeric_value(value: Any) -> float | None: + try: + numeric = float(value) + except (TypeError, ValueError): + return None + if not lazy.np.isfinite(numeric): + return None + return numeric + + @staticmethod + def _chart_label(item: TRGroupItem, data_column: str, include_metric: bool, *, full: bool) -> str: + item_label = item.v2_full_name if full else item.v2_compact_name + return f"{item_label} · {data_column}" if include_metric else item_label + + def _build_v2_bar_datasets(self, section: ComparisonSection) -> tuple[list[str], list[dict[str, Any]]]: + widest_df = max(section.dfs, key=len) + labels = [self._display_value(value) for value in widest_df[section.info_columns[0]].tolist()] + datasets: list[dict[str, Any]] = [] + include_metric = len(section.data_columns) > 1 + for data_column in section.data_columns: + for item_idx, (item, df) in enumerate(zip(section.group.items, section.dfs, strict=True)): + color = self.CHART_COLORS[len(datasets) % len(self.CHART_COLORS)] + datasets.append( + { + "label": self._chart_label(item, data_column, include_metric, full=False), + "compactLabel": self._chart_label(item, data_column, include_metric, full=False), + "fullLabel": self._chart_label(item, data_column, include_metric, full=True), + "comparisonMetric": data_column, + "data": [ + self._numeric_value(df[data_column].get(row_idx, None)) for row_idx in range(len(labels)) + ], + "backgroundColor": f"{color}cc", + "borderColor": color, + "borderWidth": 1, + "pointStyle": self.CHART_POINT_STYLES[item_idx % len(self.CHART_POINT_STYLES)], + } + ) + return labels, datasets + + def _build_v2_line_datasets(self, section: ComparisonSection) -> list[dict[str, Any]]: + datasets: list[dict[str, Any]] = [] + include_metric = len(section.data_columns) > 1 + for data_column in section.data_columns: + for item_idx, (item, df) in enumerate(zip(section.group.items, section.dfs, strict=True)): + points: list[dict[str, float]] = [] + if not df.empty and data_column in df: + pairs = zip( + df[section.info_columns[0]].tolist(), + df[data_column].tolist(), + strict=True, + ) + for x_value, y_value in pairs: + x = self._numeric_value(x_value) + y = self._numeric_value(y_value) + if x is not None and y is not None: + points.append({"x": x, "y": y}) + color = self.CHART_COLORS[len(datasets) % len(self.CHART_COLORS)] + datasets.append( + { + "label": self._chart_label(item, data_column, include_metric, full=False), + "compactLabel": self._chart_label(item, data_column, include_metric, full=False), + "fullLabel": self._chart_label(item, data_column, include_metric, full=True), + "comparisonMetric": data_column, + "data": points, + "borderColor": color, + "backgroundColor": color, + "borderDash": self.CHART_DASH_STYLES[item_idx % len(self.CHART_DASH_STYLES)], + "pointStyle": self.CHART_POINT_STYLES[item_idx % len(self.CHART_POINT_STYLES)], + "pointRadius": 4, + "pointHoverRadius": 6, + "borderWidth": 2, + "tension": 0.12, + } + ) + return datasets + + @staticmethod + def _find_v2_overlaps(datasets: list[dict[str, Any]]) -> list[str]: + overlaps: list[str] = [] + for left_idx, left in enumerate(datasets): + if not left["data"]: + continue + for right in datasets[left_idx + 1 :]: + if left["comparisonMetric"] != right["comparisonMetric"]: + continue + if left["data"] == right["data"]: + overlaps.append(f"{left['compactLabel']} and {right['compactLabel']} overlap exactly.") + return overlaps + + def _build_v2_chart(self, section: ComparisonSection, chart_idx: int) -> dict[str, Any]: + if section.chart_type == "bar": + chart_labels, datasets = self._build_v2_bar_datasets(section) + else: + chart_labels = None + datasets = self._build_v2_line_datasets(section) + + return { + "id": f"comparison-chart-{chart_idx}", + "type": section.chart_type, + "labels": chart_labels, + "datasets": datasets, + "x_axis_label": section.info_columns[0], + "y_axis_label": section.y_axis_label, + "overlaps": self._find_v2_overlaps(datasets), + } + + def _build_v2_table(self, section: ComparisonSection) -> dict[str, Any]: + widest_df = max(section.dfs, key=len) + show_diff = len(section.group.items) == 2 + data_headers: list[dict[str, str]] = [] + for data_column in section.data_columns: + for item in section.group.items: + data_headers.append( + { + "compact_name": item.v2_compact_name, + "full_name": item.v2_full_name, + "metric": data_column, + } + ) + if show_diff: + data_headers.append( + { + "compact_name": "Difference", + "full_name": "Difference", + "metric": data_column, + } + ) + + rows: list[dict[str, list[str]]] = [] + for row_idx in range(len(widest_df)): + info_cells = [self._display_value(widest_df[column].get(row_idx, None)) for column in section.info_columns] + data_cells: list[str] = [] + for data_column in section.data_columns: + raw_values = [df[data_column].get(row_idx, None) for df in section.dfs] + data_cells.extend(self._display_value(value) for value in raw_values) + if show_diff: + val1, val2 = self._extract_cmp_values(raw_values) + data_cells.append(self._format_diff_cell(val1, val2)) + rows.append({"info_cells": info_cells, "data_cells": data_cells}) + + return { + "info_headers": section.info_columns, + "data_headers": data_headers, + "rows": rows, + } + + def _build_v2_sections(self, sections: list[ComparisonSection]) -> list[dict[str, Any]]: + return [ + { + "title": section.title, + "group_name": "All cases" if section.group.name == "all-in-one" else section.group.name, + "chart": self._build_v2_chart(section, idx), + "table": self._build_v2_table(section), + } + for idx, section in enumerate(sections) + ] + def create_table( self, group: GroupedTestRuns, @@ -303,3 +635,65 @@ def create_chart( p.xaxis.major_label_orientation = lazy.np.pi / 4 return p + + def create_bar_chart(self, section: ComparisonSection) -> bk.figure: + """Render the legacy grouped bar chart used by categorical sections.""" + factors: list[tuple[str, str]] = [] + values: list[float] = [] + categories: list[str] = [] + runs: list[str] = [] + colors: list[str] = [] + color_cycle = cycle(["#1f77b4", "#17becf", "#2ca02c", "#bcbd22", "#ff7f0e"]) + color_by_run = {item.name: next(color_cycle) for item in section.group.items} + info_column = section.info_columns[0] + + for df, item in zip(section.dfs, section.group.items, strict=True): + for _, row in df.iterrows(): + category = f"{section.legacy_category_prefix}{row[info_column]}" + for data_column in section.data_columns: + value = self._numeric_value(row[data_column]) + if value is None: + continue + factor_category = category if len(section.data_columns) == 1 else f"{category} {data_column}" + factors.append((factor_category, item.name)) + values.append(value) + categories.append(factor_category) + runs.append(item.name) + colors.append(color_by_run[item.name]) + + x_range = lazy.bokeh_models.FactorRange(*factors) + x_range.range_padding = 0.1 + plot = lazy.bokeh_plotting.figure( + title=f"{section.title}: {section.group.name}", + x_range=x_range, + y_axis_label=section.y_axis_label, + width=800, + height=500, + tools="save,reset", + ) + hover = lazy.bokeh_models.HoverTool( + tooltips=[ + (info_column.title(), "@category"), + ("Run", "@run"), + ("Value", "@value{0.0000}"), + ] + ) + plot.add_tools(hover) + + if not values: + return plot + + source = lazy.bokeh_models.ColumnDataSource( + data={ + "x": factors, + "category": categories, + "run": runs, + "value": values, + "color": colors, + } + ) + plot.vbar(x="x", top="value", width=0.8, fill_color="color", line_color="color", source=source) + plot.xaxis.major_label_orientation = 0.8 + y_max = max(values) + plot.y_range = lazy.bokeh_models.Range1d(start=0, end=y_max * 1.1 if y_max > 0 else 1) + return plot diff --git a/src/cloudai/report_generator/groups.py b/src/cloudai/report_generator/groups.py index 887e5cdd0..97a5afe3b 100644 --- a/src/cloudai/report_generator/groups.py +++ b/src/cloudai/report_generator/groups.py @@ -25,6 +25,18 @@ class TRGroupItem: name: str tr: TestRun + compact_name: str | None = None + full_name: str | None = None + + @property + def v2_compact_name(self) -> str: + """Return the concise label used by v2 reports.""" + return self.compact_name or self.name + + @property + def v2_full_name(self) -> str: + """Return the detailed label used by v2 reports.""" + return self.full_name or self.v2_compact_name @dataclass diff --git a/src/cloudai/util/comparison-report-v2.jinja2 b/src/cloudai/util/comparison-report-v2.jinja2 new file mode 100644 index 000000000..eb97b20ff --- /dev/null +++ b/src/cloudai/util/comparison-report-v2.jinja2 @@ -0,0 +1,306 @@ +{% extends "base-report.jinja2" %} + +{% block extra_head %} + + + + +{% endblock %} + +{% block content %} +{% if sections %} +
+

Compact case labels are shown by default. Full labels expose only parameters that differ.

+ +
+ +
+ {% for section in sections %} +
+

{{ section.title }}

+

{{ section.group_name }}

+ +
+ +
+

Interactive chart unavailable. The complete numeric data remains available below.

+ + + + {% for overlap in section.chart.overlaps %} +

{{ overlap }}

+ {% endfor %} + +
+ + + + {% for header in section.table.info_headers %} + + {% endfor %} + {% for header in section.table.data_headers %} + + {% endfor %} + + + + {% for row in section.table.rows %} + + {% for value in row.info_cells %} + + {% endfor %} + {% for value in row.data_cells %} + + {% endfor %} + + {% endfor %} + +
{{ header }} + {{ header.compact_name }} + {{ header.metric }} +
{{ value }}{{ value }}
+
+
+ {% endfor %} +
+{% else %} +

No comparison data found.

+{% endif %} +{% endblock %} diff --git a/src/cloudai/workloads/common/llm_serving_report.py b/src/cloudai/workloads/common/llm_serving_report.py index e0dd6666b..ef9e46d24 100644 --- a/src/cloudai/workloads/common/llm_serving_report.py +++ b/src/cloudai/workloads/common/llm_serving_report.py @@ -17,12 +17,9 @@ from __future__ import annotations import abc -import itertools import math import pathlib -from typing import TYPE_CHECKING, Any, cast - -import rich.table +from typing import TYPE_CHECKING, Any import cloudai.core import cloudai.models.workload @@ -31,7 +28,6 @@ from cloudai.util.lazy_imports import lazy if TYPE_CHECKING: - import bokeh.plotting as bk import pandas as pd from cloudai.workloads.common import llm_serving @@ -190,130 +186,50 @@ def _group_df(df: pd.DataFrame, metric_group: str) -> pd.DataFrame: .reset_index(drop=True) ) - def create_tables( + def build_sections( self, cmp_groups: list[cloudai.report_generator.groups.GroupedTestRuns] - ) -> list[rich.table.Table]: - tables: list[rich.table.Table] = [] + ) -> list[cloudai.report_generator.comparison_report.ComparisonSection]: + sections: list[cloudai.report_generator.comparison_report.ComparisonSection] = [] for group in cmp_groups: extracted_dfs = [self._extract_data_as_df_cached(item.tr) for item in group.items] - tables.extend( + sections.extend( [ - self.create_table( - group, + cloudai.report_generator.comparison_report.ComparisonSection( + group=group, dfs=[self._group_df(df, "latency") for df in extracted_dfs], title="Latency", info_columns=["metric"], data_columns=["value"], + y_axis_label="Latency (ms)", + chart_type="bar", ), - self.create_table( - group, + cloudai.report_generator.comparison_report.ComparisonSection( + group=group, dfs=[self._group_df(df, "success") for df in extracted_dfs], title="Successful Prompts", info_columns=["metric"], data_columns=["value"], + y_axis_label="Prompts / %", + chart_type="bar", ), - self.create_table( - group, + cloudai.report_generator.comparison_report.ComparisonSection( + group=group, dfs=[self._group_df(df, "throughput") for df in extracted_dfs], title="Throughput", info_columns=["metric"], data_columns=["value"], + y_axis_label="Throughput", + chart_type="bar", ), - self.create_table( - group, + cloudai.report_generator.comparison_report.ComparisonSection( + group=group, dfs=[self._group_df(df, "quality") for df in extracted_dfs], title="Quality", info_columns=["metric"], data_columns=["value"], + y_axis_label="Score", + chart_type="bar", ), ] ) - return tables - - def _create_metric_bar_chart( - self, - group: cloudai.report_generator.groups.GroupedTestRuns, - dfs: list[pd.DataFrame], - title: str, - y_axis_label: str, - ) -> bk.figure: - factors: list[tuple[str, str]] = [] - values: list[float] = [] - colors: list[str] = [] - color_cycler = itertools.cycle(["#1f77b4", "#17becf", "#2ca02c", "#bcbd22", "#ff7f0e"]) - color_by_run = {item.name: next(color_cycler) for item in group.items} - - for df, run_name in zip(dfs, [item.name for item in group.items], strict=True): - for _, row in df.iterrows(): - value = row["value"] - if not isinstance(value, (float, int)): - continue - factors.append((row["metric"], run_name)) - values.append(float(value)) - colors.append(color_by_run[run_name]) - - x_range = lazy.bokeh_models.FactorRange(*factors) - cast(Any, x_range).range_padding = 0.1 - p = lazy.bokeh_plotting.figure( - title=f"{title}: {group.name}", - x_range=x_range, - y_axis_label=y_axis_label, - width=800, - height=500, - tools="save,reset", - ) - hover = lazy.bokeh_models.HoverTool( - tooltips=[("Metric", "@metric"), ("Run", "@run"), ("Value", "@value{0.0000}")] - ) - p.add_tools(hover) - - if not values: - return p - - source = lazy.bokeh_models.ColumnDataSource( - data={ - "x": factors, - "metric": [metric for metric, _ in factors], - "run": [run for _, run in factors], - "value": values, - "color": colors, - } - ) - p.vbar(x="x", top="value", width=0.8, fill_color="color", line_color="color", source=source) - p.xaxis.major_label_orientation = 0.8 - p.y_range = lazy.bokeh_models.Range1d(start=0, end=max(values) * 1.1) - return p - - def create_charts(self, cmp_groups: list[cloudai.report_generator.groups.GroupedTestRuns]) -> list[bk.figure]: - charts: list[bk.figure] = [] - for group in cmp_groups: - extracted_dfs = [self._extract_data_as_df_cached(item.tr) for item in group.items] - charts.extend( - [ - self._create_metric_bar_chart( - group, - [self._group_df(df, "latency") for df in extracted_dfs], - "Latency", - "Latency (ms)", - ), - self._create_metric_bar_chart( - group, - [self._group_df(df, "success") for df in extracted_dfs], - "Successful Prompts", - "Prompts / %", - ), - self._create_metric_bar_chart( - group, - [self._group_df(df, "throughput") for df in extracted_dfs], - "Throughput", - "Throughput", - ), - self._create_metric_bar_chart( - group, - [self._group_df(df, "quality") for df in extracted_dfs], - "Quality", - "Score", - ), - ] - ) - return charts + return sections diff --git a/src/cloudai/workloads/nccl_test/nccl_comparison_report.py b/src/cloudai/workloads/nccl_test/nccl_comparison_report.py index d86d6945d..cb36a67ef 100644 --- a/src/cloudai/workloads/nccl_test/nccl_comparison_report.py +++ b/src/cloudai/workloads/nccl_test/nccl_comparison_report.py @@ -19,10 +19,8 @@ from pathlib import Path from typing import TYPE_CHECKING -from rich.table import Table - from cloudai.core import System, TestRun, TestScenario -from cloudai.report_generator.comparison_report import ComparisonReport, ComparisonReportConfig +from cloudai.report_generator.comparison_report import ComparisonReport, ComparisonReportConfig, ComparisonSection from cloudai.report_generator.groups import GroupedTestRuns from cloudai.report_generator.util import ( add_human_readable_sizes, @@ -33,7 +31,6 @@ from .performance_report_generation_strategy import extract_nccl_data if TYPE_CHECKING: - import bokeh.plotting as bk import pandas as pd @@ -54,29 +51,32 @@ def load_test_runs(self): super().load_test_runs() self.trs = [tr for tr in self.trs if isinstance(tr.test, NCCLTestDefinition)] - def create_tables(self, cmp_groups: list[GroupedTestRuns]) -> list[Table]: - tables: list[Table] = [] + def build_sections(self, cmp_groups: list[GroupedTestRuns]) -> list[ComparisonSection]: + sections: list[ComparisonSection] = [] for group in cmp_groups: dfs = [self.extract_data_as_df(item.tr) for item in group.items] - tables.extend( + sections.extend( [ - self.create_table( - group, + ComparisonSection( + group=group, dfs=dfs, title="Latency", info_columns=list(self.INFO_COLUMNS), data_columns=list(self.LATENCY_DATA_COLUMNS), + y_axis_label="Time (us)", + legacy_chart_title="Latecy", ), - self.create_table( - group, + ComparisonSection( + group=group, dfs=dfs, title="Bandwidth", info_columns=list(self.INFO_COLUMNS), data_columns=list(self.BANDWIDTH_DATA_COLUMNS), + y_axis_label="Busbw (GB/s)", ), ] ) - return tables + return sections def extract_data_as_df(self, tr: TestRun) -> pd.DataFrame: parsed_data_rows, gpu_type, num_devices_per_node, num_ranks = extract_nccl_data(tr.output_path / "stdout.txt") @@ -117,17 +117,3 @@ def extract_data_as_df(self, tr: TestRun) -> pd.DataFrame: df = add_human_readable_sizes(df, "Size (B)", "Size Human-readable") return df - - def create_charts(self, cmp_groups: list[GroupedTestRuns]) -> list[bk.figure]: - charts: list[bk.figure] = [] - for group in cmp_groups: - dfs = [self.extract_data_as_df(item.tr) for item in group.items] - if chart := self.create_chart( - group, dfs, "Latecy", list(self.INFO_COLUMNS), list(self.LATENCY_DATA_COLUMNS), "Time (us)" - ): - charts.append(chart) - if chart := self.create_chart( - group, dfs, "Bandwidth", list(self.INFO_COLUMNS), list(self.BANDWIDTH_DATA_COLUMNS), "Busbw (GB/s)" - ): - charts.append(chart) - return charts diff --git a/src/cloudai/workloads/nixl_bench/nixl_summary_report.py b/src/cloudai/workloads/nixl_bench/nixl_summary_report.py index 001421492..c46b62d08 100644 --- a/src/cloudai/workloads/nixl_bench/nixl_summary_report.py +++ b/src/cloudai/workloads/nixl_bench/nixl_summary_report.py @@ -19,17 +19,14 @@ from pathlib import Path from typing import TYPE_CHECKING -from rich.table import Table - from cloudai.core import System, TestRun, TestScenario -from cloudai.report_generator.comparison_report import ComparisonReport, ComparisonReportConfig +from cloudai.report_generator.comparison_report import ComparisonReport, ComparisonReportConfig, ComparisonSection from cloudai.report_generator.groups import GroupedTestRuns from cloudai.util.lazy_imports import lazy from .nixl_bench import NIXLBenchTestDefinition if TYPE_CHECKING: - import bokeh.plotting as bk import pandas as pd @@ -48,41 +45,31 @@ def load_test_runs(self): super().load_test_runs() self.trs = [tr for tr in self.trs if isinstance(tr.test, NIXLBenchTestDefinition)] - def create_tables(self, cmp_groups: list[GroupedTestRuns]) -> list[Table]: - tables: list[Table] = [] + def build_sections(self, cmp_groups: list[GroupedTestRuns]) -> list[ComparisonSection]: + sections: list[ComparisonSection] = [] for group in cmp_groups: dfs = [self.extract_data_as_df(item.tr) for item in group.items] - tables.extend( + sections.extend( [ - self.create_table( - group, + ComparisonSection( + group=group, dfs=dfs, title="Latency", info_columns=list(self.INFO_COLUMNS), data_columns=["avg_lat"], + y_axis_label="Time (us)", ), - self.create_table( - group, + ComparisonSection( + group=group, dfs=dfs, title="Bandwidth", info_columns=list(self.INFO_COLUMNS), data_columns=["bw_gb_sec"], + y_axis_label="Busbw (GB/s)", ), ] ) - return tables - - def create_charts(self, cmp_groups: list[GroupedTestRuns]) -> list[bk.figure]: - charts: list[bk.figure] = [] - for group in cmp_groups: - dfs = [self.extract_data_as_df(item.tr) for item in group.items] - charts.extend( - [ - self.create_chart(group, dfs, "Latency", list(self.INFO_COLUMNS), ["avg_lat"], "Time (us)"), - self.create_chart(group, dfs, "Bandwidth", list(self.INFO_COLUMNS), ["bw_gb_sec"], "Busbw (GB/s)"), - ] - ) - return charts + return sections def extract_data_as_df(self, tr: TestRun) -> pd.DataFrame: if (tr.output_path / "nixlbench.csv").exists(): diff --git a/src/cloudai/workloads/nixl_ep/nixl_ep_comparison_report.py b/src/cloudai/workloads/nixl_ep/nixl_ep_comparison_report.py index ddc614ffe..533141c05 100644 --- a/src/cloudai/workloads/nixl_ep/nixl_ep_comparison_report.py +++ b/src/cloudai/workloads/nixl_ep/nixl_ep_comparison_report.py @@ -16,11 +16,8 @@ from __future__ import annotations -import itertools import pathlib -from typing import TYPE_CHECKING, Any, ClassVar, cast - -import rich.table +from typing import TYPE_CHECKING, ClassVar import cloudai.core import cloudai.report_generator.comparison_report @@ -30,7 +27,6 @@ from cloudai.util.lazy_imports import lazy if TYPE_CHECKING: - import bokeh.plotting as bk import pandas as pd @@ -123,104 +119,40 @@ def _has_metric(dfs: list[pd.DataFrame], column: str) -> bool: def _available_columns(self, dfs: list[pd.DataFrame], columns: tuple[str, ...]) -> list[str]: return [column for column in columns if self._has_metric(dfs, column)] - def create_tables( + def build_sections( self, cmp_groups: list[cloudai.report_generator.groups.GroupedTestRuns] - ) -> list[rich.table.Table]: - tables: list[rich.table.Table] = [] + ) -> list[cloudai.report_generator.comparison_report.ComparisonSection]: + sections: list[cloudai.report_generator.comparison_report.ComparisonSection] = [] for group in cmp_groups: dfs = [self.extract_data_as_df(item.tr) for item in group.items] bandwidth_columns = self._available_columns(dfs, self.BANDWIDTH_COLUMNS) time_columns = self._available_columns(dfs, self.TIME_COLUMNS) for bandwidth_column in bandwidth_columns: - tables.append( - self.create_table( - group, + sections.append( + cloudai.report_generator.comparison_report.ComparisonSection( + group=group, dfs=dfs, title=bandwidth_column, info_columns=[self.NODE_COLUMN], data_columns=[bandwidth_column], + y_axis_label="Bandwidth (GB/s)", + chart_type="bar", + legacy_category_prefix="Node ", ) ) for time_column in time_columns: - tables.append( - self.create_table( - group, + sections.append( + cloudai.report_generator.comparison_report.ComparisonSection( + group=group, dfs=dfs, title=time_column, info_columns=[self.NODE_COLUMN], data_columns=[time_column], + y_axis_label="Time (us)", + chart_type="bar", + legacy_category_prefix="Node ", ) ) - return tables - - def _create_metric_bar_chart( - self, - group: cloudai.report_generator.groups.GroupedTestRuns, - dfs: list[pd.DataFrame], - metric_column: str, - y_axis_label: str, - ) -> bk.figure: - factors: list[tuple[str, str]] = [] - values: list[float] = [] - nodes: list[str] = [] - runs: list[str] = [] - colors: list[str] = [] - color_cycler = itertools.cycle(["#1f77b4", "#17becf", "#2ca02c", "#bcbd22", "#ff7f0e"]) - color_by_run = {item.name: next(color_cycler) for item in group.items} - - for df, item in zip(dfs, group.items, strict=True): - if metric_column not in df.columns: - continue - for _, row in df.iterrows(): - value = row[metric_column] - if not isinstance(value, (int, float)): - continue - node = f"Node {row[self.NODE_COLUMN]}" - factors.append((node, item.name)) - values.append(float(value)) - nodes.append(node) - runs.append(item.name) - colors.append(color_by_run[item.name]) - - x_range = lazy.bokeh_models.FactorRange(*factors) - cast(Any, x_range).range_padding = 0.1 - plot = lazy.bokeh_plotting.figure( - title=f"{metric_column}: {group.name}", - x_range=x_range, - y_axis_label=y_axis_label, - width=800, - height=500, - tools="save,reset", - ) - hover = lazy.bokeh_models.HoverTool(tooltips=[("Node", "@node"), ("Run", "@run"), ("Value", "@value{0.0000}")]) - plot.add_tools(hover) - - if not values: - return plot - - source = lazy.bokeh_models.ColumnDataSource( - data={ - "x": factors, - "node": nodes, - "run": runs, - "value": values, - "color": colors, - } - ) - plot.vbar(x="x", top="value", width=0.8, fill_color="color", line_color="color", source=source) - plot.xaxis.major_label_orientation = 0.8 - y_max = max(values) - plot.y_range = lazy.bokeh_models.Range1d(start=0, end=y_max * 1.1 if y_max > 0 else 1) - return plot - - def create_charts(self, cmp_groups: list[cloudai.report_generator.groups.GroupedTestRuns]) -> list[bk.figure]: - charts: list[bk.figure] = [] - for group in cmp_groups: - dfs = [self.extract_data_as_df(item.tr) for item in group.items] - for column in self._available_columns(dfs, self.BANDWIDTH_COLUMNS): - charts.append(self._create_metric_bar_chart(group, dfs, column, "Bandwidth (GB/s)")) - for column in self._available_columns(dfs, self.TIME_COLUMNS): - charts.append(self._create_metric_bar_chart(group, dfs, column, "Time (us)")) - return charts + return sections diff --git a/src/cloudai/workloads/osu_bench/osu_comparison_report.py b/src/cloudai/workloads/osu_bench/osu_comparison_report.py index f348a4b3e..aa051d77e 100644 --- a/src/cloudai/workloads/osu_bench/osu_comparison_report.py +++ b/src/cloudai/workloads/osu_bench/osu_comparison_report.py @@ -20,17 +20,14 @@ from pathlib import Path from typing import TYPE_CHECKING -from rich.table import Table - from cloudai.core import System, TestRun, TestScenario -from cloudai.report_generator.comparison_report import ComparisonReport, ComparisonReportConfig +from cloudai.report_generator.comparison_report import ComparisonReport, ComparisonReportConfig, ComparisonSection from cloudai.report_generator.groups import GroupedTestRuns from cloudai.util.lazy_imports import lazy from .osu_bench import OSUBenchTestDefinition if TYPE_CHECKING: - import bokeh.plotting as bk import pandas as pd @@ -68,81 +65,43 @@ def _has_metric(dfs: list["pd.DataFrame"], col: str) -> bool: """Only include a metric if all compared DataFrames have it.""" return bool(dfs) and all((col in df.columns) and df[col].notna().any() for df in dfs) - def create_tables(self, cmp_groups: list[GroupedTestRuns]) -> list[Table]: - tables: list[Table] = [] + def build_sections(self, cmp_groups: list[GroupedTestRuns]) -> list[ComparisonSection]: + sections: list[ComparisonSection] = [] for group in cmp_groups: dfs = [self.extract_data_as_df(item.tr) for item in group.items] if self._has_metric(dfs, "avg_lat"): - tables.append( - self.create_table( - group, + sections.append( + ComparisonSection( + group=group, dfs=dfs, title="Latency", info_columns=list(self.INFO_COLUMNS), data_columns=["avg_lat"], + y_axis_label="Time (us)", ) ) if self._has_metric(dfs, "mb_sec"): - tables.append( - self.create_table( - group, + sections.append( + ComparisonSection( + group=group, dfs=dfs, title="Bandwidth", info_columns=list(self.INFO_COLUMNS), data_columns=["mb_sec"], + y_axis_label="Bandwidth (MB/s)", ) ) if self._has_metric(dfs, "messages_sec"): - tables.append( - self.create_table( - group, + sections.append( + ComparisonSection( + group=group, dfs=dfs, title="Message Rate", info_columns=list(self.INFO_COLUMNS), data_columns=["messages_sec"], + y_axis_label="Messages/s", ) ) - return tables - - def create_charts(self, cmp_groups: list[GroupedTestRuns]) -> list[bk.figure]: - charts: list[bk.figure] = [] - for group in cmp_groups: - dfs = [self.extract_data_as_df(item.tr) for item in group.items] - - if self._has_metric(dfs, "avg_lat"): - charts.append( - self.create_chart( - group, - dfs, - "Latency", - list(self.INFO_COLUMNS), - ["avg_lat"], - "Time (us)", - ) - ) - if self._has_metric(dfs, "mb_sec"): - charts.append( - self.create_chart( - group, - dfs, - "Bandwidth", - list(self.INFO_COLUMNS), - ["mb_sec"], - "Bandwidth (MB/s)", - ) - ) - if self._has_metric(dfs, "messages_sec"): - charts.append( - self.create_chart( - group, - dfs, - "Message Rate", - list(self.INFO_COLUMNS), - ["messages_sec"], - "Messages/s", - ) - ) - - return charts + return sections diff --git a/tests/report_generation_strategy/test_comparison_report.py b/tests/report_generation_strategy/test_comparison_report.py index b76150915..9fa1b8fc6 100644 --- a/tests/report_generation_strategy/test_comparison_report.py +++ b/tests/report_generation_strategy/test_comparison_report.py @@ -14,31 +14,52 @@ # See the License for the specific language governing permissions and # limitations under the License. +import copy from pathlib import Path -import bokeh.plotting as bk import pandas as pd import pytest import toml from packaging.requirements import Requirement from packaging.version import Version -from rich.table import Table from cloudai.core import TestRun, TestScenario -from cloudai.report_generator.comparison_report import ComparisonReport, ComparisonReportConfig +from cloudai.report_generator.comparison_report import ComparisonReport, ComparisonReportConfig, ComparisonSection from cloudai.report_generator.groups import GroupedTestRuns, TRGroupItem from cloudai.systems.slurm import SlurmSystem +from cloudai.workloads.ai_dynamo import AIDynamoComparisonReport +from cloudai.workloads.nccl_test import NcclComparisonReport +from cloudai.workloads.nixl_bench.nixl_summary_report import NIXLBenchComparisonReport +from cloudai.workloads.nixl_ep import NixlEPComparisonReport +from cloudai.workloads.osu_bench import OSUBenchComparisonReport +from cloudai.workloads.sglang import SGLangComparisonReport +from cloudai.workloads.vllm import VLLMComparisonReport class MyComparisonReport(ComparisonReport): def extract_data_as_df(self, tr: TestRun) -> pd.DataFrame: return pd.DataFrame() - def create_tables(self, cmp_groups: list[GroupedTestRuns]) -> list[Table]: + def build_sections(self, cmp_groups: list[GroupedTestRuns]) -> list[ComparisonSection]: return [] - def create_charts(self, cmp_groups: list[GroupedTestRuns]) -> list[bk.figure]: - return [] + +class RenderableComparisonReport(MyComparisonReport): + def load_test_runs(self) -> None: + """Keep test-provided runs instead of loading result directories.""" + + def build_sections(self, cmp_groups: list[GroupedTestRuns]) -> list[ComparisonSection]: + return [ + ComparisonSection( + group=group, + dfs=[pd.DataFrame({"size": [1, 2, 4], "value": [10, 20, 40]}) for _ in group.items], + title="Throughput", + info_columns=["size"], + data_columns=["value"], + y_axis_label="Requests/s", + ) + for group in cmp_groups + ] @pytest.fixture @@ -55,6 +76,91 @@ def test_jinja_template_path(cmp_report: MyComparisonReport) -> None: assert full_path.is_file() +def test_v2_jinja_template_path(cmp_report: MyComparisonReport) -> None: + full_path = cmp_report.template_path / cmp_report.v2_template_name + assert full_path.exists() + assert full_path.is_file() + + +def test_generate_writes_legacy_and_v2_reports(slurm_system: SlurmSystem, nccl_tr: TestRun) -> None: + slurm_system.output_path.mkdir(parents=True) + report = RenderableComparisonReport( + slurm_system, + TestScenario(name="comparison", test_runs=[]), + slurm_system.output_path, + ComparisonReportConfig(enable=True), + ) + report.trs = [nccl_tr, copy.deepcopy(nccl_tr)] + + report.generate() + + legacy_path = slurm_system.output_path / "comparison_report.html" + v2_path = slurm_system.output_path / "comparison_report_v2.html" + assert legacy_path.exists() + assert v2_path.exists() + + v2_content = v2_path.read_text() + assert "Show full labels" in v2_content + assert "chart.js@4.4.3" in v2_content + assert "overlap exactly" in v2_content + assert nccl_tr.name in v2_content + + +def test_v2_payload_defaults_to_compact_labels(cmp_report: MyComparisonReport, nccl_tr: TestRun) -> None: + long_image = "nvcr.io/example/" + ("very-long-image-name-" * 10) + ":latest" + item = TRGroupItem( + name=f"docker_image_url={long_image}", + tr=nccl_tr, + compact_name="case-a", + full_name=f"case-a — docker_image_url={long_image}", + ) + section = ComparisonSection( + group=GroupedTestRuns(name="all-in-one", items=[item]), + dfs=[pd.DataFrame({"size": [1], "value": [10]})], + title="Throughput", + info_columns=["size"], + data_columns=["value"], + y_axis_label="Requests/s", + ) + + chart = cmp_report._build_v2_chart(section, 0) + table = cmp_report._build_v2_table(section) + + assert chart["datasets"][0]["label"] == "case-a" + assert chart["datasets"][0]["fullLabel"] == f"case-a — docker_image_url={long_image}" + assert table["data_headers"][0]["compact_name"] == "case-a" + assert table["data_headers"][0]["full_name"] == f"case-a — docker_image_url={long_image}" + + +@pytest.mark.parametrize( + ("report_cls", "legacy_name", "v2_name"), + [ + (NIXLBenchComparisonReport, "nixl_comparison.html", "nixl_comparison_v2.html"), + (NixlEPComparisonReport, "nixl_ep_comparison.html", "nixl_ep_comparison_v2.html"), + (NcclComparisonReport, "nccl_comparison.html", "nccl_comparison_v2.html"), + (OSUBenchComparisonReport, "osu_bench_comparison.html", "osu_bench_comparison_v2.html"), + (VLLMComparisonReport, "vllm_comparison.html", "vllm_comparison_v2.html"), + (SGLangComparisonReport, "sglang_comparison.html", "sglang_comparison_v2.html"), + (AIDynamoComparisonReport, "ai_dynamo_comparison.html", "ai_dynamo_comparison_v2.html"), + ], +) +def test_v2_report_file_names( + slurm_system: SlurmSystem, + report_cls: type[ComparisonReport], + legacy_name: str, + v2_name: str, +) -> None: + report = report_cls( + slurm_system, + TestScenario(name="comparison", test_runs=[]), + slurm_system.output_path, + ComparisonReportConfig(enable=True), + ) + + assert report.report_file_name == legacy_name + assert report.v2_report_file_name == v2_name + + class TestCreateTable: def test_single_data_point(self, cmp_report: MyComparisonReport, nccl_tr: TestRun) -> None: table = cmp_report.create_table( diff --git a/tests/report_generation_strategy/test_report_groups.py b/tests/report_generation_strategy/test_report_groups.py index 7b78e31e3..ebb325f2c 100644 --- a/tests/report_generation_strategy/test_report_groups.py +++ b/tests/report_generation_strategy/test_report_groups.py @@ -16,13 +16,11 @@ import copy -import bokeh.plotting as bk import pandas as pd -from rich.table import Table from cloudai import TestRun from cloudai.core import TestScenario -from cloudai.report_generator.comparison_report import ComparisonReport, ComparisonReportConfig +from cloudai.report_generator.comparison_report import ComparisonReport, ComparisonReportConfig, ComparisonSection from cloudai.report_generator.groups import GroupedTestRuns from cloudai.report_generator.util import diff_comparison_values from cloudai.systems.slurm import SlurmSystem @@ -32,10 +30,7 @@ class GroupingComparisonReport(ComparisonReport): def extract_data_as_df(self, tr: TestRun) -> pd.DataFrame: return pd.DataFrame() - def create_tables(self, cmp_groups: list[GroupedTestRuns]) -> list[Table]: - return [] - - def create_charts(self, cmp_groups: list[GroupedTestRuns]) -> list[bk.figure]: + def build_sections(self, cmp_groups: list[GroupedTestRuns]) -> list[ComparisonSection]: return [] @@ -126,6 +121,39 @@ def test_multiple_trs_in_a_group(self, slurm_system: SlurmSystem, nccl_tr: TestR assert groups[1].items[0].name == "NCCL_IB_SPLIT_DATA_ON_QPS=0" assert groups[1].items[1].name == "NCCL_IB_SPLIT_DATA_ON_QPS=1" + def test_v2_compact_name_includes_step_and_multi_iteration( + self, slurm_system: SlurmSystem, nccl_tr: TestRun + ) -> None: + nccl_tr.name = "case-a" + nccl_tr.step = 3 + nccl_tr.iterations = 2 + nccl_tr.current_iteration = 0 + + item = _comparison_report(slurm_system, [nccl_tr], group_by=[]).group_test_runs()[0].items[0] + + assert item.v2_compact_name == "case-a · step=3 · iter=0" + + def test_v2_duplicate_compact_names_are_disambiguated(self, slurm_system: SlurmSystem, nccl_tr: TestRun) -> None: + items = ( + _comparison_report(slurm_system, [nccl_tr, copy.deepcopy(nccl_tr)], group_by=[]).group_test_runs()[0].items + ) + + assert [item.v2_compact_name for item in items] == [ + f"{nccl_tr.name} · run=1", + f"{nccl_tr.name} · run=2", + ] + + def test_v2_full_label_flattens_nested_differences(self) -> None: + diff: dict[str, list[object]] = { + "prefill": [ + {"gpu_ids": ["0", "1"], "tensor_parallel_size": 2}, + {"gpu_ids": ["2", "3"], "tensor_parallel_size": 4}, + ] + } + + assert ComparisonReport._format_diff_label(diff, 0) == ("prefill.gpu_ids=0,1 · prefill.tensor_parallel_size=2") + assert ComparisonReport._format_diff_label(diff, 1) == ("prefill.gpu_ids=2,3 · prefill.tensor_parallel_size=4") + class TestComparisonValues: def test_diff_comparison_values_normalizes_numeric_equivalents(self) -> None: diff --git a/tests/workloads/ai_dynamo/test_comparison_report.py b/tests/workloads/ai_dynamo/test_comparison_report.py index b5cbdb5c3..200b659b5 100644 --- a/tests/workloads/ai_dynamo/test_comparison_report.py +++ b/tests/workloads/ai_dynamo/test_comparison_report.py @@ -88,3 +88,4 @@ def test_ai_dynamo_comparison_report_generates_html(slurm_system: SlurmSystem) - report.generate() assert (slurm_system.output_path / "ai_dynamo_comparison.html").exists() + assert (slurm_system.output_path / "ai_dynamo_comparison_v2.html").exists() diff --git a/tests/workloads/common/test_llm_serving_report.py b/tests/workloads/common/test_llm_serving_report.py index dced14748..6d020460e 100644 --- a/tests/workloads/common/test_llm_serving_report.py +++ b/tests/workloads/common/test_llm_serving_report.py @@ -144,6 +144,7 @@ def test_llm_comparison_report_generates_html(slurm_system: cloudai.systems.slur report.generate() assert (slurm_system.output_path / "vllm_comparison.html").exists() + assert (slurm_system.output_path / "vllm_comparison_v2.html").exists() def test_sglang_comparison_report_generates_html(slurm_system: cloudai.systems.slurm.SlurmSystem) -> None: @@ -246,3 +247,4 @@ def test_sglang_comparison_report_generates_html(slurm_system: cloudai.systems.s report.generate() assert (slurm_system.output_path / "sglang_comparison.html").exists() + assert (slurm_system.output_path / "sglang_comparison_v2.html").exists() diff --git a/tests/workloads/nixl_ep/test_comparison_report.py b/tests/workloads/nixl_ep/test_comparison_report.py index ddd482439..2daafcd3b 100644 --- a/tests/workloads/nixl_ep/test_comparison_report.py +++ b/tests/workloads/nixl_ep/test_comparison_report.py @@ -92,3 +92,4 @@ def test_nixl_ep_comparison_report_generates_html(slurm_system: cloudai.systems. report.generate() assert (slurm_system.output_path / "nixl_ep_comparison.html").exists() + assert (slurm_system.output_path / "nixl_ep_comparison_v2.html").exists()