From 7eaa762b5690ed32d2da6c1517f68e9b8561fff5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Filip=20V=C3=A1gner?= Date: Mon, 27 Jul 2026 21:20:57 +0200 Subject: [PATCH] Use fmf for result filtering --- tmt-recipe-tool/README.md | 108 ++++++++++-------- tmt-recipe-tool/pyproject.toml | 1 + tmt-recipe-tool/tmt_recipe_tool/cli.py | 78 +++++-------- tmt-recipe-tool/tmt_recipe_tool/filtering.py | 38 ++++++ tmt-recipe-tool/tmt_recipe_tool/models.py | 11 +- tmt-recipe-tool/tmt_recipe_tool/recipe.py | 29 +++-- .../tmt_recipe_tool/reportportal.py | 24 ++-- 7 files changed, 164 insertions(+), 125 deletions(-) create mode 100644 tmt-recipe-tool/tmt_recipe_tool/filtering.py diff --git a/tmt-recipe-tool/README.md b/tmt-recipe-tool/README.md index e8c2acd..6ed1b08 100644 --- a/tmt-recipe-tool/README.md +++ b/tmt-recipe-tool/README.md @@ -1,8 +1,8 @@ # tmt-recipe-tool -A command-line tool for filtering and rerunning [tmt](https://tmt.readthedocs.io/) tests based on result outcomes. +A command-line tool for filtering and rerunning [tmt](https://tmt.readthedocs.io/) tests based on result attributes. -Given a [tmt recipe](https://tmt.readthedocs.io/en/stable/spec/recipe.html) and its associated test results, `tmt-recipe-tool` can produce a new recipe containing only the tests that matched specific outcomes (e.g. failed or errored tests), and optionally rerun them immediately. +Given a [tmt recipe](https://tmt.readthedocs.io/en/stable/spec/recipe.html) and its associated test results, `tmt-recipe-tool` can produce a new recipe containing only the tests that match a filter expression, and optionally rerun them immediately. Results can be sourced either from a local tmt [results file](https://tmt.readthedocs.io/en/stable/spec/results.html) or from a [ReportPortal](https://tmt.readthedocs.io/en/stable/plugins/report.html#reportportal) instance when the recipe's report phase is configured with `how: reportportal`. @@ -25,91 +25,105 @@ uv sync --group dev ## Usage ``` -tmt-recipe-tool [OPTIONS] COMMAND [ARGS]... +tmt-recipe-tool [OPTIONS] ``` -### Global options +### Options -| Option | Description | -|--------|-------------| -| `-i, --input PATH` | Path to the input recipe file | -| `-o, --output PATH` | Path to save the modified recipe (if omitted, the recipe is not saved) | -| `--run` | Rerun the modified recipe with tmt after processing | -| `--feeling-safe` | Pass `--feeling-safe` to tmt, allowing potentially unsafe operations | -| `--run-workdir PATH` | Path to the tmt run workdir, used as the base directory for resolving relative results paths | -| `--version` | Show version and exit | +| Option | Default | Description | +|--------|---------|-------------| +| `-i, --input PATH` | | Path to the input recipe file (required) | +| `-o, --output PATH` | | Path to save the modified recipe (if omitted, the recipe is not saved) | +| `-f, --filter EXPRESSION` | `result: fail, error, failed` | Keep tests matching this filter expression | +| `--use-reportportal` | `false` | Fetch test results from ReportPortal instead of a local results file | +| `--run` | | Rerun the modified recipe with tmt after processing | +| `--feeling-safe` | | Pass `--feeling-safe` to tmt, allowing potentially unsafe operations | +| `--run-workdir PATH` | | Path to the tmt run workdir, used as the base directory for resolving relative results paths | +| `--version` | | Show version and exit | -### Commands +### Filter expression -#### `filter-tests` +Filtering uses [fmf](https://fmf.readthedocs.io/en/stable/modules.html#fmf.filter) expression syntax internally. Matching is case-insensitive; values are regular expressions matched against the whole field. -Filter recipe tests by their result outcome, keeping only those that match. +| Operator | Meaning | +|----------|---------| +| `&` | AND | +| `\|` | OR | +| `key: -value` | NOT | +| `key: a, b` | OR within the same key (`key: a \| key: b`) | -``` -tmt-recipe-tool -i RECIPE filter-tests [--use-reportportal] [--result RESULT]... -``` +Parentheses are not supported. Use disjunctive normal form instead (`A & C \| B & C` for `(A \| B) & C`). Precedence is negation, then `&`, then `|`. -| Option | Default | Description | -|--------|---------|-------------| -| `--result RESULT` | `fail`, `error`, `warn` | Keep tests with this outcome (repeatable) | -| `--use-reportportal` | `false` | Fetch test results from ReportPortal instead of a local results file | +Supported fields: -When `--use-reportportal` is used, results are fetched from any report phase in the recipe that has `how: reportportal`. The phase must include `launch-uuid` and `test-uuids` (populated automatically by the tmt [ReportPortal](https://tmt.readthedocs.io/en/stable/plugins/report/reportportal.html) plugin after a run finishes). The `launch-uuid` and `test-uuids` fields are stripped from the output recipe so that a subsequent run creates a new ReportPortal launch. Plans that do not have a `reportportal` report phase always fall back to their local `results.yaml` file, even when `--use-reportportal` is passed. +| Field | Description | +|-------|-------------| +| `name` | Test name | +| `result` | Result status (see below) | +| `defect` | ReportPortal defect type (`product_bug`, `automation_bug`, `system_issue`, `no_defect`, `to_investigate`). Set to `none` for local results and for ReportPortal items with no defects. | -Because ReportPortal only has three result statuses (`PASSED`, `FAILED`, `SKIPPED`), tmt outcomes are mapped before filtering: +### Result statuses -| tmt outcome | ReportPortal status | -|-------------|---------------------| -| `pass` | `PASSED` | -| `fail` | `FAILED` | -| `warn` | `FAILED` | -| `error` | `FAILED` | -| `info` | `SKIPPED` | -| `skip` | `SKIPPED` | -| `pending` | `SKIPPED` | +Statuses are **not** mapped between sources. Use the values from the source you are filtering: -As a result, `fail`, `warn`, and `error` are indistinguishable when filtering via ReportPortal, and all three will select tests with status `FAILED`. Similarly, `info`, `skip`, and `pending` will all select tests with status `SKIPPED`. +| Source | `result` values | +|--------|-----------------| +| tmt results | `pass`, `fail`, `warn`, `error`, `info`, `skip`, `pending` | +| ReportPortal results | `passed`, `failed`, `skipped` | + +The default filter includes both `fail`/`error` (tmt) and `failed` (ReportPortal) so it works with either source. + +### ReportPortal + +When `--use-reportportal` is used, results are fetched from any report phase in the recipe that has `how: reportportal`. The phase must include `launch-uuid` and `test-uuids` (populated automatically by the tmt [ReportPortal](https://tmt.readthedocs.io/en/stable/plugins/report/reportportal.html) plugin after a run finishes). The `launch-uuid` and `test-uuids` fields are stripped from the output recipe so that a subsequent run creates a new ReportPortal launch. Plans that do not have a `reportportal` report phase always fall back to their local `results.yaml` file, even when `--use-reportportal` is passed. ### Examples -Filter a recipe to keep only failed and errored tests, saving the result: +Filter a recipe with the default expression (failed and errored tests), saving the result: ```bash -tmt-recipe-tool -i recipe.yaml -o filtered.yaml filter-tests +tmt-recipe-tool -i recipe.yaml -o filtered.yaml ``` Keep only tests that passed: ```bash -tmt-recipe-tool -i recipe.yaml -o passed.yaml filter-tests --result pass +tmt-recipe-tool -i recipe.yaml -o passed.yaml -f 'result: pass' +``` + +Keep failed tests with a specific defect type (ReportPortal): + +```bash +tmt-recipe-tool -i recipe.yaml -o bugs.yaml --use-reportportal \ + -f 'result: failed & defect: product_bug' ``` -Filter and immediately rerun the failing tests: +Keep failed tests, or tests whose name matches a pattern: ```bash -tmt-recipe-tool -i recipe.yaml --run filter-tests +tmt-recipe-tool -i recipe.yaml -o subset.yaml -f 'result: fail | name: .*/smoke.*' ``` -Rerun with `--feeling-safe` to allow potentially unsafe tmt operations: +Filter and immediately rerun: ```bash -tmt-recipe-tool -i recipe.yaml --run --feeling-safe filter-tests +tmt-recipe-tool -i recipe.yaml --run ``` -Combine multiple result filters: +Rerun with `--feeling-safe`: ```bash -tmt-recipe-tool -i recipe.yaml -o subset.yaml filter-tests --result fail --result error +tmt-recipe-tool -i recipe.yaml --run --feeling-safe ``` -Filter using ReportPortal results (requires the recipe to have a `reportportal` report phase with `launch-uuid` and `test-uuids`): +Filter using ReportPortal results: ```bash -tmt-recipe-tool -i recipe.yaml -o filtered.yaml filter-tests --use-reportportal +tmt-recipe-tool -i recipe.yaml -o filtered.yaml --use-reportportal ``` -Fetch failures from ReportPortal and rerun them immediately: +Fetch failures from ReportPortal and rerun them: ```bash -tmt-recipe-tool -i recipe.yaml --run filter-tests --use-reportportal --result fail +tmt-recipe-tool -i recipe.yaml --run --use-reportportal -f 'result: failed' ``` diff --git a/tmt-recipe-tool/pyproject.toml b/tmt-recipe-tool/pyproject.toml index 313c3f2..5f83d98 100644 --- a/tmt-recipe-tool/pyproject.toml +++ b/tmt-recipe-tool/pyproject.toml @@ -11,6 +11,7 @@ requires-python = ">=3.9" license = "MIT" dependencies = [ "click>=8.0.3", + "fmf>=1.7.0", "pydantic>=2.0", "requests>=2.25.1", "ruamel.yaml>=0.16.6", diff --git a/tmt-recipe-tool/tmt_recipe_tool/cli.py b/tmt-recipe-tool/tmt_recipe_tool/cli.py index ba4e416..958b2e9 100644 --- a/tmt-recipe-tool/tmt_recipe_tool/cli.py +++ b/tmt-recipe-tool/tmt_recipe_tool/cli.py @@ -4,13 +4,12 @@ import click +from tmt_recipe_tool.filtering import DEFAULT_FILTER from tmt_recipe_tool.recipe import _save_recipe, filter_recipe from tmt_recipe_tool.utils import run_tmt_recipe -DEFAULT_TEST_FILTER_RESULTS = ["fail", "error", "warn"] - -@click.group(invoke_without_command=False, no_args_is_help=True) +@click.command(no_args_is_help=True) @click.version_option(package_name="tmt-recipe-tool") @click.option( "-i", @@ -30,6 +29,21 @@ "Path to the output recipe file. If not specified, the modified recipe will not be saved." ), ) +@click.option( + "-f", + "--filter", + metavar="EXPRESSION", + type=str, + default=DEFAULT_FILTER, + show_default=True, + help="Keep tests matching this fmf filter expression. Available keys: name, result, defect.", +) +@click.option( + "--use-reportportal", + is_flag=True, + default=False, + help="Fetch test results from ReportPortal instead of a local results file.", +) @click.option( "--run", is_flag=True, @@ -54,62 +68,24 @@ "relative results paths." ), ) -@click.pass_context def main( - ctx: click.Context, input: Path, # noqa: A002 output: Optional[Path], + filter: str, # noqa: A002 + use_reportportal: bool, run: bool, feeling_safe: bool, run_workdir: Optional[Path], **kwargs: Any, ) -> None: - """tmt-recipe-tool - Filter and rerun tmt recipes based on test result outcomes.""" - ctx.ensure_object(dict) - - -@main.command() -@click.option( - "--result", - "results", - metavar="RESULT", - multiple=True, - show_default=True, - default=DEFAULT_TEST_FILTER_RESULTS, - help=( - "Keep only the tests that have the specified result outcome. " - "Can be specified multiple times." - ), -) -@click.option( - "--use-reportportal", - is_flag=True, - default=False, - help="Fetch test results from ReportPortal instead of a local results file.", -) -@click.pass_context -def filter_tests( - ctx: click.Context, results: list[str], use_reportportal: bool, **kwargs: Any -) -> None: - """Filter recipe tests by their result outcome.""" - assert ctx.parent is not None - - ctx.obj["recipe"] = filter_recipe( - ctx.parent.params["input"], - results, - run_workdir=ctx.parent.params.get("run_workdir"), + """tmt-recipe-tool - Filter and optionally rerun a tmt recipe using a filter expression.""" + recipe = filter_recipe( + input, + filter, + run_workdir=run_workdir, use_reportportal=use_reportportal, ) - -@main.result_callback() -@click.pass_context -def post_action(ctx, *args, **kwargs): - """Save and optionally run the modified recipe after a subcommand completes.""" - recipe = ctx.obj.get("recipe", None) - if not recipe: - return - empty_plans = [p.name for p in recipe.plans if not p.discover.tests] if empty_plans: click.echo( @@ -117,15 +93,13 @@ def post_action(ctx, *args, **kwargs): err=True, ) - output = ctx.params.get("output", None) if output: _save_recipe(recipe, output) print(f"Modified recipe saved to: '{output}'") else: print("No output path provided, the modified recipe will not be saved.") - if ctx.params.get("run", False): - feeling_safe = ctx.params.get("feeling_safe", False) + if run: if not output: with tempfile.TemporaryDirectory(prefix="tmt_recipe_tool_") as tmp: output = Path(tmp) / "recipe.yaml" @@ -138,4 +112,4 @@ def post_action(ctx, *args, **kwargs): "Warning: None of the plans have any tests after filtering.", err=True, ) - ctx.exit(3) + raise SystemExit(3) diff --git a/tmt-recipe-tool/tmt_recipe_tool/filtering.py b/tmt-recipe-tool/tmt_recipe_tool/filtering.py new file mode 100644 index 0000000..e6933cd --- /dev/null +++ b/tmt-recipe-tool/tmt_recipe_tool/filtering.py @@ -0,0 +1,38 @@ +from __future__ import annotations + +from typing import Any, Optional + +import fmf # type: ignore[import-untyped] +from fmf.utils import FilterError # type: ignore[import-untyped] + +DEFAULT_FILTER = "result: fail, error, failed" + + +class FilterExpressionError(Exception): + """Raised when an fmf filter expression is invalid or cannot be evaluated.""" + + +def build_filter_data( + *, + name: Optional[str] = None, + result: Optional[str] = None, + defects: Optional[list[str]] = None, +) -> dict[str, Any]: + return { + "name": name, + "result": result, + "defect": defects or None, + } + + +def matches_filter(filter: str, data: dict[str, Any]) -> bool: # noqa: A002 + """ + Return True if ``data`` matches the fmf filter expression. + + Matching is case-insensitive and values are treated + as regular expressions (full match). + """ + try: + return fmf.filter(filter, data, sensitive=False, regexp=True, name=None) + except FilterError as exc: + raise FilterExpressionError(f"Invalid filter expression {filter!r}.") from exc diff --git a/tmt-recipe-tool/tmt_recipe_tool/models.py b/tmt-recipe-tool/tmt_recipe_tool/models.py index 2a48f91..732e87d 100644 --- a/tmt-recipe-tool/tmt_recipe_tool/models.py +++ b/tmt-recipe-tool/tmt_recipe_tool/models.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import Literal, Optional +from typing import Any, Literal, Optional from pydantic import BaseModel, Field @@ -20,10 +20,17 @@ class ReportPortalPhase(BaseModel, extra="allow"): launch_uuid: Optional[str] = Field(None, alias="launch-uuid") launch_url: Optional[str] = Field(None, alias="launch-url") ssl_verify: bool = Field(True, alias="ssl-verify") - test_uuids: dict[int, dict[Optional[str], str]] = Field(default_factory=dict, alias="test-uuids") + test_uuids: dict[int, dict[Optional[str], str]] = Field( + default_factory=dict, alias="test-uuids" + ) api_version: str = Field(alias="api-version") +class RPResultStatistics(BaseModel, extra="allow"): + defects: dict[str, Any] + + class ReportPortalResult(BaseModel, extra="allow"): uuid: str status: str + statistics: RPResultStatistics diff --git a/tmt-recipe-tool/tmt_recipe_tool/recipe.py b/tmt-recipe-tool/tmt_recipe_tool/recipe.py index d686f5c..7d7adf6 100644 --- a/tmt-recipe-tool/tmt_recipe_tool/recipe.py +++ b/tmt-recipe-tool/tmt_recipe_tool/recipe.py @@ -7,6 +7,7 @@ import tmt.utils from pydantic import ValidationError +from tmt_recipe_tool.filtering import build_filter_data, matches_filter from tmt_recipe_tool.models import Result from tmt_recipe_tool.reportportal import edit_rp_phases, filter_tests_from_rp, get_rp_phases from tmt_recipe_tool.utils import create_tmt_logger, load_yaml @@ -70,27 +71,33 @@ def _load_results(results_path: Path, plan_name: str) -> list[Result]: def _filter_tests( tests: list[tmt.recipe._RecipeTest], results: list[Result], - filter_results: list[str], + filter: str, # noqa: A002 ) -> Iterable[tmt.recipe._RecipeTest]: - """Return only the tests whose result outcome matches the filter.""" + """ + Yield tests whose local result matches the fmf filter expression. + """ for test in tests: for result in results: - if ( - test.name == result.name - and test.serial_number == result.serial_number - and result.result in filter_results - ): + if test.name != result.name or test.serial_number != result.serial_number: + continue + data = build_filter_data(name=test.name, result=result.result) + if matches_filter(filter, data): yield test break def filter_recipe( input_path: Path, - filter_results: list[str], + filter: str, # noqa: A002 run_workdir: Optional[Path] = None, use_reportportal: bool = False, ) -> tmt.recipe.Recipe: - """Load a recipe and keep only tests matching the specified result outcomes.""" + """ + Load a recipe and keep only tests matching the fmf filter expression. + + Results are taken from ReportPortal when ``use_reportportal`` is set and the + plan has a reportportal phase, otherwise from the plan's local results file. + """ recipe = _load_recipe(input_path) filtered_plans = [] @@ -101,14 +108,14 @@ def filter_recipe( rp_phases = get_rp_phases(plan) if rp_phases and use_reportportal: plan.discover.tests = list( - filter_tests_from_rp(plan.discover.tests, rp_phases, filter_results) + filter_tests_from_rp(plan.discover.tests, rp_phases, filter) ) plan.report.phases = edit_rp_phases(plan.report.phases) else: results = _load_results( _resolve_results_path(plan, input_path, run_workdir), plan.name ) - plan.discover.tests = list(_filter_tests(plan.discover.tests, results, filter_results)) + plan.discover.tests = list(_filter_tests(plan.discover.tests, results, filter)) filtered_plans.append(plan) recipe.plans = filtered_plans diff --git a/tmt-recipe-tool/tmt_recipe_tool/reportportal.py b/tmt-recipe-tool/tmt_recipe_tool/reportportal.py index 6e70808..b080f35 100644 --- a/tmt-recipe-tool/tmt_recipe_tool/reportportal.py +++ b/tmt-recipe-tool/tmt_recipe_tool/reportportal.py @@ -6,9 +6,9 @@ import requests # type: ignore[import-untyped] from tmt.recipe import _RecipePlan, _RecipeTest from tmt.steps import _RawStepData -from tmt.steps.report.reportportal import ReportReportPortal from tmt.utils import retry_session +from tmt_recipe_tool.filtering import build_filter_data, matches_filter from tmt_recipe_tool.models import ReportPortalPhase, ReportPortalResult from tmt_recipe_tool.utils import create_tmt_logger @@ -20,11 +20,6 @@ 504, # Gateway Timeout ) -TMT_TO_RP_RESULT_STATUS = { - outcome.value: rp_status - for outcome, rp_status in ReportReportPortal.TMT_TO_RP_RESULT_STATUS.items() -} - class ReportPortalError(Exception): """Raised when fetching results from ReportPortal fails.""" @@ -120,13 +115,11 @@ def _fetch_rp_results( def filter_tests_from_rp( tests: list[_RecipeTest], rp_phases: list[ReportPortalPhase], - filter_results: list[str], + filter: str, # noqa: A002 ) -> Iterable[_RecipeTest]: - """Filter the tests from the ReportPortal results""" - filter_results = list( - {TMT_TO_RP_RESULT_STATUS.get(result, result) for result in filter_results} - ) - + """ + Yield tests whose ReportPortal result matches the fmf filter expression. + """ filtered_tests: dict[int, _RecipeTest] = {} for phase in rp_phases: @@ -160,7 +153,12 @@ def filter_tests_from_rp( result = rp_results.get(uuid, None) if not result: continue - if result.status in filter_results: + data = build_filter_data( + name=test.name, + result=result.status, + defects=list(result.statistics.defects.keys()), + ) + if matches_filter(filter, data): filtered_tests[test.serial_number] = test break