Share one Kit app across test files instead of booting it per file

.github/actions/run-package-tests/action.yml

@@ -79,6 +79,13 @@ inputs:
     description: 'Additional pytest options'
     default: ''
     required: false
+  test-path:
+    description: >-
+      Path handed to pytest. Defaults to "tools", which loads tools/conftest.py and runs each
+      test file in its own subprocess. Point it at a test directory instead to run those files
+      together in a single pytest process, bypassing the per-file orchestrator.
+    default: 'tools'
+    required: false
   extra-pip-packages:
     description: 'Space-separated pip packages to install inside the Docker container before pytest starts'
     default: ''
@@ -291,7 +298,7 @@ runs:
     - name: Run Tests
       uses: ./.github/actions/run-tests
       with:
-        test-path: "tools"
+        test-path: ${{ inputs.test-path }}
         result-file: "${{ inputs.result-file != '' && inputs.result-file || format('{0}-report.xml', github.job) }}"
         container-name: "${{ inputs.container-name }}-${{ github.run_id }}-${{ github.run_attempt }}"
         image-tag: ${{ inputs.image-tag }}

.github/workflows/build.yaml

@@ -892,6 +892,116 @@ jobs:
         omni-github-test-type: warp-cache-warm
   #endregion
 
+  #region kit-reuse timing probe
+  # TEMPORARY. Measures what sharing one Kit app across test files is worth, so the decision to
+  # migrate the remaining ~125 files is based on a number rather than an estimate. Both jobs run
+  # the same 24 `kit` files from source/isaaclab/test/sim; the only difference is how many Kit
+  # apps get booted. Compare the two job durations in the Actions UI, then delete this region.
+  #
+  # The four `kit_cameras` files in that directory are excluded from both sides. The first
+  # camera-enabled boot in a fresh container compiles shaders for ~600 s, which is an order of
+  # magnitude larger than the Kit startup being measured and would swamp the comparison.
+  #
+  # The file lists are spelled out rather than selected with `-m kit` because pytest's marker
+  # filtering deselects tests but still imports every collected module, so `-m` alone cannot keep
+  # a kit_cameras module from calling launch_kit(cameras=True). Files in TESTS_TO_SKIP are left
+  # out of both sides so the two jobs cover exactly the same tests.
+  test-kit-reuse-probe-per-file:
+    name: "kit-reuse-probe-per-file"
+    runs-on: [self-hosted, gpu]
+    timeout-minutes: 120
+    continue-on-error: true
+    needs: [build, config]
+    if: needs.build.result == 'success'
+    steps:
+    - uses: actions/checkout@v6
+      with:
+        fetch-depth: 1
+        lfs: true
+    # Baseline: the default test-path of "tools" runs tools/conftest.py, which gives each file
+    # its own subprocess, so Kit boots 30 times.
+    - uses: ./.github/actions/run-package-tests
+      with:
+        image-tag: ${{ needs.config.outputs.ci_image_tag }}
+        isaacsim-base-image: ${{ needs.config.outputs.isaacsim_image_name }}
+        isaacsim-version: ${{ needs.config.outputs.isaacsim_image_tag }}
+        filter-pattern: "isaaclab/test/sim"
+        include-files: >-
+          test_articulation_fragments.py,
+          test_build_simulation_context_headless.py,
+          test_cloner.py,
+          test_collision_fragments.py,
+          test_joint_drive_fragments.py,
+          test_mass_fragments.py,
+          test_material_fragments.py,
+          test_mesh_collision_fragments.py,
+          test_mesh_converter.py,
+          test_schema_fragments.py,
+          test_schema_writer_nested_targets.py,
+          test_schemas.py,
+          test_simulation_context.py,
+          test_spawn_from_files.py,
+          test_spawn_lights.py,
+          test_spawn_materials.py,
+          test_spawn_meshes.py,
+          test_spawn_sensors.py,
+          test_spawn_shapes.py,
+          test_spawn_wrappers.py,
+          test_tendon_fragments.py,
+          test_utils_stage.py,
+          test_utils_transforms.py,
+          test_views_xform_prim.py
+        container-name: isaac-lab-kit-reuse-probe-per-file
+        omni-github-test-type: kit-reuse-probe-per-file
+
+  test-kit-reuse-probe-batched:
+    name: "kit-reuse-probe-batched"
+    runs-on: [self-hosted, gpu]
+    timeout-minutes: 120
+    continue-on-error: true
+    needs: [build, config]
+    if: needs.build.result == 'success'
+    steps:
+    - uses: actions/checkout@v6
+      with:
+        fetch-depth: 1
+        lfs: true
+    # Batched: pointing test-path at the files themselves bypasses tools/conftest.py, so all 24
+    # run in one pytest process and launch_kit() boots Kit once.
+    - uses: ./.github/actions/run-package-tests
+      with:
+        image-tag: ${{ needs.config.outputs.ci_image_tag }}
+        isaacsim-base-image: ${{ needs.config.outputs.isaacsim_image_name }}
+        isaacsim-version: ${{ needs.config.outputs.isaacsim_image_tag }}
+        test-path: >-
+          source/isaaclab/test/sim/test_articulation_fragments.py
+          source/isaaclab/test/sim/test_build_simulation_context_headless.py
+          source/isaaclab/test/sim/test_cloner.py
+          source/isaaclab/test/sim/test_collision_fragments.py
+          source/isaaclab/test/sim/test_joint_drive_fragments.py
+          source/isaaclab/test/sim/test_mass_fragments.py
+          source/isaaclab/test/sim/test_material_fragments.py
+          source/isaaclab/test/sim/test_mesh_collision_fragments.py
+          source/isaaclab/test/sim/test_mesh_converter.py
+          source/isaaclab/test/sim/test_schema_fragments.py
+          source/isaaclab/test/sim/test_schema_writer_nested_targets.py
+          source/isaaclab/test/sim/test_schemas.py
+          source/isaaclab/test/sim/test_simulation_context.py
+          source/isaaclab/test/sim/test_spawn_from_files.py
+          source/isaaclab/test/sim/test_spawn_lights.py
+          source/isaaclab/test/sim/test_spawn_materials.py
+          source/isaaclab/test/sim/test_spawn_meshes.py
+          source/isaaclab/test/sim/test_spawn_sensors.py
+          source/isaaclab/test/sim/test_spawn_shapes.py
+          source/isaaclab/test/sim/test_spawn_wrappers.py
+          source/isaaclab/test/sim/test_tendon_fragments.py
+          source/isaaclab/test/sim/test_utils_stage.py
+          source/isaaclab/test/sim/test_utils_transforms.py
+          source/isaaclab/test/sim/test_views_xform_prim.py
+        container-name: isaac-lab-kit-reuse-probe-batched
+        omni-github-test-type: kit-reuse-probe-batched
+  #endregion
+
 #region disabled quarantined tests
 # test-quarantined:
 #   name: "Quarantined Tests"

pyproject.toml

@@ -338,6 +338,11 @@ markers = [
     "benchmark: test covers the Isaac Lab benchmark framework and infrastructure",
     "rendering: test exercises the rendering / camera / visualizer pipeline",
     "smoke: tests for core installation, task, and RL functionality",
+    "kit: test file needs a booted headless Kit app; it calls isaaclab.test.launch.launch_kit() at module scope rather than constructing AppLauncher",
+    "kit_cameras: like `kit`, but the app is booted with cameras enabled via launch_kit(cameras=True)",
+    "kitless: test file runs without Kit; no AppLauncher and no module-scope import of omni/carb/isaacsim",
+    "kit_solo: keep this file in its own process; it is never grouped with other files",
+    "newton_ci: mark test to run in the Newton CI lane",
 ]
 
 # Add pypi.nvidia.com so that `uv pip install isaaclab[isaacsim]` works without --extra-index-url.

source/isaaclab/changelog.d/mataylor-kit-test-markers.rst

@@ -0,0 +1,15 @@
+Added
+^^^^^
+
+* Added :func:`~isaaclab.test.launch.launch_kit` so test modules can share one Kit app per
+  pytest process instead of each launching their own. It is idempotent: the first module to
+  call it boots Kit and later modules receive the running app.
+* Added the ``kit``, ``kit_cameras``, ``kitless``, and ``kit_solo`` pytest markers so a test
+  file can declare its Kit launch configuration, plus a test that checks each file's markers
+  against what it actually does at module scope.
+
+Fixed
+^^^^^
+
+* Fixed ``test_operational_space.py`` assigning ``pytestmark`` twice, which silently dropped
+  its ``arm_ci`` marker and kept the file out of the ARM CI lane.

source/isaaclab/isaaclab/test/launch.py

@@ -0,0 +1,84 @@
+# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md).
+# All rights reserved.
+#
+# SPDX-License-Identifier: BSD-3-Clause
+
+"""Shared Kit launch helper for Isaac Lab tests.
+
+Test modules that need Isaac Sim call :func:`launch_kit` at module scope in place of
+constructing :class:`~isaaclab.app.AppLauncher` directly::
+
+    from isaaclab.test.launch import launch_kit
+
+    launch_kit()  # or launch_kit(cameras=True)
+
+The call must stay at module scope: a test module's own imports (``pxr``, ``omni``,
+``isaaclab_physx``, ...) run during pytest collection, before any fixture executes, so Kit
+must already be running by then.
+
+:func:`launch_kit` is idempotent within a process. The first test module to call it boots
+Kit; every later module gets the running app back. A pytest process covering several test
+files therefore pays Kit startup once rather than once per file.
+
+Declare the matching marker on the module so the test runner can group files that share a
+launch configuration into one process::
+
+    pytestmark = pytest.mark.kit  # launch_kit()
+    pytestmark = pytest.mark.kit_cameras  # launch_kit(cameras=True)
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+_app: Any = None
+"""The Kit application booted by :func:`launch_kit`, or None before the first call."""
+
+_cameras: bool = False
+"""Whether :attr:`_app` was booted with camera and render extensions enabled."""
+
+
+def launch_kit(*, cameras: bool = False) -> Any:
+    """Boot the shared Kit app for this process, or return the one already running.
+
+    Args:
+        cameras: Whether the app must be booted with camera and render extensions enabled.
+            Passed through to :paramref:`~isaaclab.app.AppLauncher.enable_cameras`.
+
+    Returns:
+        The running ``SimulationApp``.
+
+    Raises:
+        RuntimeError: If a camera-enabled app is requested but Kit is already running in
+            this process without cameras, or if Kit was started by something other than
+            this function. Both mean the test files sharing this process do not share a
+            launch configuration and must be split across processes.
+    """
+    global _app, _cameras
+
+    if _app is not None:
+        if cameras and not _cameras:
+            raise RuntimeError(
+                "launch_kit(cameras=True) was called, but Kit is already running in this process"
+                " without cameras. Camera extensions cannot be enabled after startup. Mark this"
+                " file `pytest.mark.kit_cameras` so it is grouped with other camera tests instead"
+                " of with plain `pytest.mark.kit` files."
+            )
+        return _app
+
+    from isaaclab.utils import has_kit
+
+    if has_kit():
+        raise RuntimeError(
+            "Kit is already running but was not started by launch_kit(), so its launch"
+            " configuration is unknown. Another test file in this process still constructs"
+            " AppLauncher directly; run that file in its own process."
+        )
+
+    from isaaclab.app import AppLauncher
+
+    from .utils import resolve_test_sim_device
+
+    _app = AppLauncher(headless=True, enable_cameras=cameras, device=resolve_test_sim_device()).app
+    _cameras = cameras
+    return _app

source/isaaclab/test/controllers/test_operational_space.py

@@ -16,8 +16,6 @@
 import torch
 from flaky import flaky
 
-pytestmark = pytest.mark.arm_ci
-
 import isaaclab.envs.mdp as mdp
 import isaaclab.sim as sim_utils
 from isaaclab import cloner
@@ -51,7 +49,7 @@
 
 from isaaclab_assets import FRANKA_PANDA_CFG, G1_29DOF_CFG  # isort:skip
 
-pytestmark = pytest.mark.integration
+pytestmark = [pytest.mark.arm_ci, pytest.mark.integration]
 
 
 @pytest.fixture

source/isaaclab/test/sim/test_articulation_fragments.py

@@ -3,14 +3,9 @@
 #
 # SPDX-License-Identifier: BSD-3-Clause
 
-"""Launch Isaac Sim Simulator first."""
+from isaaclab.test.launch import launch_kit
 
-from isaaclab.app import AppLauncher
-
-# launch omniverse app
-simulation_app = AppLauncher(headless=True).app
-
-"""Rest everything follows."""
+launch_kit()
 
 import os
 
@@ -21,6 +16,8 @@
 import isaaclab.sim as sim_utils
 from isaaclab.sim import SimulationCfg, SimulationContext
 
+pytestmark = pytest.mark.kit
+
 
 def _make_xform(stage, path="/World/Art"):
     UsdGeom.Xform.Define(stage, path)

source/isaaclab/test/sim/test_build_simulation_context_headless.py

@@ -13,21 +13,16 @@
 ``test_build_simulation_context_nonheadless.py``.
 """
 
-"""Launch Isaac Sim Simulator first."""
+from isaaclab.test.launch import launch_kit
 
-from isaaclab.app import AppLauncher
-
-# launch omniverse app
-simulation_app = AppLauncher(headless=True).app
-
-"""Rest everything follows."""
+launch_kit()
 
 import pytest
 
 from isaaclab.sim.simulation_cfg import SimulationCfg
 from isaaclab.sim.simulation_context import build_simulation_context
 
-pytestmark = pytest.mark.integration
+pytestmark = [pytest.mark.kit, pytest.mark.integration]
 
 
 @pytest.mark.parametrize("gravity_enabled", [True, False])

source/isaaclab/test/sim/test_build_simulation_context_nonheadless.py

@@ -12,21 +12,16 @@
 ``test_build_simulation_context_headless.py``.
 """
 
-"""Launch Isaac Sim Simulator first."""
+from isaaclab.test.launch import launch_kit
 
-from isaaclab.app import AppLauncher
-
-# launch omniverse app
-simulation_app = AppLauncher(headless=True).app
-
-"""Rest everything follows."""
+launch_kit()
 
 import pytest
 
 from isaaclab.sim.simulation_cfg import SimulationCfg
 from isaaclab.sim.simulation_context import build_simulation_context
 
-pytestmark = pytest.mark.integration
+pytestmark = [pytest.mark.kit, pytest.mark.integration]
 
 
 @pytest.mark.parametrize("gravity_enabled", [True, False])

source/isaaclab/test/sim/test_cloner.py

@@ -5,14 +5,9 @@
 
 """Tests for USD cloner utilities (no PhysX dependency)."""
 
-"""Launch Isaac Sim Simulator first."""
+from isaaclab.test.launch import launch_kit
 
-from isaaclab.app import AppLauncher
-
-# launch omniverse app
-simulation_app = AppLauncher(headless=True).app
-
-"""Rest everything follows."""
+launch_kit()
 
 from types import SimpleNamespace
 from unittest.mock import MagicMock
@@ -37,7 +32,7 @@
 )
 from isaaclab.sim import build_simulation_context
 
-pytestmark = [pytest.mark.integration, pytest.mark.isaacsim_ci]
+pytestmark = [pytest.mark.kit, pytest.mark.integration, pytest.mark.isaacsim_ci]
 
 
 @pytest.fixture(params=["cpu", "cuda"])

source/isaaclab/test/sim/test_collision_fragments.py

@@ -3,14 +3,9 @@
 #
 # SPDX-License-Identifier: BSD-3-Clause
 
-"""Launch Isaac Sim Simulator first."""
+from isaaclab.test.launch import launch_kit
 
-from isaaclab.app import AppLauncher
-
-# launch omniverse app
-simulation_app = AppLauncher(headless=True).app
-
-"""Rest everything follows."""
+launch_kit()
 
 import pytest
 
@@ -19,7 +14,7 @@
 import isaaclab.sim as sim_utils
 from isaaclab.sim import SimulationCfg, SimulationContext
 
-pytestmark = pytest.mark.integration
+pytestmark = [pytest.mark.kit, pytest.mark.integration]
 
 
 def _make_xform(stage, path="/World/Body"):