Run libev unit tests in CI and stop silently skipping tests

.github/workflows/integration-tests.yml

@@ -98,4 +98,4 @@ jobs:
         if [[ "${{ matrix.python-version }}" =~ t$ ]]; then
           export PYTHON_GIL=0
         fi
-        uv run pytest tests/integration/standard/ tests/integration/cqlengine/
+        uv run pytest -v tests/integration/standard/ tests/integration/cqlengine/

cassandra/cluster.py

@@ -3438,7 +3438,7 @@ def pool_finished_setting_keyspace(pool, host_errors):
                 errors[pool.host] = host_errors
 
             if not remaining_callbacks:
-                callback(host_errors)
+                callback(errors)
 
         for pool in tuple(self._pools.values()):
             pool._set_keyspace_for_all_conns(keyspace, pool_finished_setting_keyspace)

pyproject.toml

@@ -49,6 +49,7 @@ dev = [
     "gevent",
     "eventlet>=0.33.3",
     "cython>=3.2",
+    "setuptools",
     "packaging>=25.0",
     "futurist",
     "pyyaml",
@@ -157,22 +158,71 @@ enable = ["pypy"]
 [tool.cibuildwheel.linux]
 
 before-build = "rm -rf ~/.pyxbld && rpm --import https://repo.almalinux.org/almalinux/RPM-GPG-KEY-AlmaLinux && yum install -y libffi-devel libev libev-devel openssl openssl-devel"
+# Install the optional lz4 compression dependency so the lz4 segment tests run
+# (and fail loudly under CASS_DRIVER_NO_SKIP) instead of skipping silently.
+test-extras = ["compress-lz4"]
+# Extensions are mandatory on Linux (CASS_DRIVER_BUILD_EXTENSIONS_ARE_MUST=yes),
+# so skipping is disabled (CASS_DRIVER_NO_SKIP=1): a missing dependency such as
+# libev fails loudly instead of being silently skipped. Tests that cannot run in
+# the default configuration are listed explicitly:
+#   * event-loop reactor tests are run separately with the matching
+#     EVENT_LOOP_MANAGER (gevent/eventlet/asyncio);
+#   * asyncore is deprecated and unavailable on modern Python, so it is ignored;
+#   * column_encryption is disabled upstream (scylladb/python-driver#365);
+#   * test_deserialize_date_range_month is disabled upstream (PYTHON-912).
+# PyPy uses the pp* override below. All Linux CPython reactor commands run with
+# CASS_DRIVER_NO_SKIP=1 so unexpected skips fail loudly.
 test-command = [
-    "pytest {package}/tests/unit",
-    "EVENT_LOOP_MANAGER=gevent pytest {package}/tests/unit/io/test_geventreactor.py",
+    "CASS_DRIVER_NO_SKIP=1 pytest --import-mode=append {package}/tests/unit -v --ignore={package}/tests/unit/column_encryption --ignore={package}/tests/unit/io/test_geventreactor.py --ignore={package}/tests/unit/io/test_eventletreactor.py --ignore={package}/tests/unit/io/test_asyncioreactor.py --ignore={package}/tests/unit/io/test_asyncorereactor.py -k 'not test_deserialize_date_range_month'",
+    "EVENT_LOOP_MANAGER=gevent CASS_DRIVER_NO_SKIP=1 pytest --import-mode=append {package}/tests/unit/io/test_geventreactor.py -v",
+    "EVENT_LOOP_MANAGER=asyncio CASS_DRIVER_NO_SKIP=1 pytest --import-mode=append {package}/tests/unit/io/test_asyncioreactor.py -v",
+    "EVENT_LOOP_MANAGER=eventlet CASS_DRIVER_NO_SKIP=1 pytest --import-mode=append {package}/tests/unit/io/test_eventletreactor.py -v",
 ]
 
 [tool.cibuildwheel.macos]
 build-frontend = "build"
+# Install lz4 so the lz4 segment tests run instead of skipping (see Linux note).
+test-extras = ["compress-lz4"]
+# Same policy as Linux (extensions are mandatory here too, libev comes from
+# Homebrew). The extra -k exclusions are timing-sensitive tests that are flaky
+# on macOS runners. The gevent/eventlet/asyncio reactor test files only contain
+# those timing-sensitive timer tests, so they are not run separately here.
 test-command = [
-    "pytest {project}/tests/unit -k 'not (test_multi_timer_validation or test_empty_connections or test_timer_cancellation)'",
+    "CASS_DRIVER_NO_SKIP=1 pytest --import-mode=append {project}/tests/unit -v --ignore={project}/tests/unit/column_encryption --ignore={project}/tests/unit/io/test_geventreactor.py --ignore={project}/tests/unit/io/test_eventletreactor.py --ignore={project}/tests/unit/io/test_asyncioreactor.py --ignore={project}/tests/unit/io/test_asyncorereactor.py -k 'not (test_multi_timer_validation or test_empty_connections or test_timer_cancellation or test_deserialize_date_range_month)'",
 ]
 
 [tool.cibuildwheel.windows]
 build-frontend = "build"
+# On Windows the C extensions are optional (CASS_DRIVER_BUILD_EXTENSIONS_ARE_MUST
+# is overridden to "no" below), so extension-dependent tests (e.g. libev) are
+# legitimately skipped here. CASS_DRIVER_NO_SKIP is therefore NOT enabled on
+# Windows; we only add -v so skips are visible in the log.
 test-command = [
-    "pytest {project}/tests/unit  -k \"not (test_deserialize_date_range_year or test_datetype or test_libevreactor)\"",
+    "pytest --import-mode=append {project}/tests/unit -v -k \"not (test_deserialize_date_range_year or test_datetype or test_libevreactor)\"",
 ]
 
 # TODO: set CASS_DRIVER_BUILD_EXTENSIONS_ARE_MUST to yes when https://github.com/scylladb/python-driver/issues/429 is fixed
 environment = { CASS_DRIVER_BUILD_CONCURRENCY = "2", CASS_DRIVER_BUILD_EXTENSIONS_ARE_MUST = "no" }
+
+# PyPy never builds the libev/cmurmur3/Cython C extensions (setup.py forces
+# is_pypy to skip them even when CASS_DRIVER_BUILD_EXTENSIONS_ARE_MUST=yes), so
+# the tests that depend on those extensions legitimately skip. Enforcing
+# CASS_DRIVER_NO_SKIP would turn those expected skips into failures, so it is
+# NOT enabled for PyPy (same reasoning as Windows). The reactor tests are not
+# run separately here because eventlet is unsupported on PyPy (@notpypy) and the
+# extension-backed reactors are unavailable; with no-skip off they simply skip.
+# test-extras is cleared (no compress-lz4): PyPy has no prebuilt lz4 wheel, so
+# pip would try to compile it from source and fail. The lz4 tests just skip here.
+# test_deserialize_date_range_year and test_datetype are excluded because they
+# fail on Windows (the C runtime's gmtime rejects the far-future timestamps they
+# use); the CPython Windows command excludes them for the same reason. The
+# timer tests (test_multi_timer_validation, test_empty_connections,
+# test_timer_cancellation) are timing-sensitive and flaky on macOS, matching the
+# CPython macOS exclusions. The override matches PyPy on all OSes, so these are
+# deselected everywhere here (they are still covered by the CPython runs).
+[[tool.cibuildwheel.overrides]]
+select = "pp*"
+test-extras = []
+test-command = [
+    "pytest --import-mode=append {package}/tests/unit -v --ignore={package}/tests/unit/column_encryption -k \"not (test_deserialize_date_range_month or test_deserialize_date_range_year or test_datetype or test_multi_timer_validation or test_empty_connections or test_timer_cancellation)\"",
+]

tests/conftest.py

@@ -16,9 +16,45 @@
 import os
 import warnings
 
+import pytest
+
 # Directory containing the Cython-compiled driver modules.
 _CASSANDRA_DIR = os.path.join(os.path.dirname(__file__), os.pardir, "cassandra")
 
+# When set (e.g. in CI) a skipped test is turned into a failure. Tests skip
+# themselves when their requirements are missing (a library is not installed,
+# the wrong event loop is selected, ...). That is convenient locally, but in CI
+# it is a footgun: a test may be silently skipped because we forgot to install
+# something. Enabling this forces every skip to be explicit on the command line
+# (via -k / --ignore / --deselect) instead of being hidden in the output.
+_NO_SKIP = bool(os.environ.get("CASS_DRIVER_NO_SKIP"))
+
+
+@pytest.hookimpl(hookwrapper=True)
+def pytest_runtest_makereport(item, call):
+    """Turn skips into failures when CASS_DRIVER_NO_SKIP is set.
+
+    xfailed tests (which are reported as skipped) are left untouched so that
+    ``xfail_strict`` keeps working as configured.
+    """
+    outcome = yield
+    if not _NO_SKIP:
+        return
+    report = outcome.get_result()
+    if report.skipped and not hasattr(report, "wasxfail"):
+        reason = ""
+        if isinstance(report.longrepr, tuple) and len(report.longrepr) == 3:
+            reason = report.longrepr[2]
+        elif report.longrepr:
+            reason = str(report.longrepr)
+        report.outcome = "failed"
+        report.longrepr = (
+            "Test was skipped but skipping is disabled in this environment "
+            "(CASS_DRIVER_NO_SKIP is set). Run it in a suitable configuration "
+            "or deselect it explicitly on the command line. "
+            "Original skip reason: {!r}".format(reason)
+        )
+
 
 def pytest_configure(config):
     """Warn when a compiled Cython extension is older than its .py source.

tests/unit/io/test_libevreactor.py

@@ -69,24 +69,33 @@ def test_watchers_are_finished(self):
         @test_category connection
         """
         from cassandra.io.libevreactor import _global_loop
-        with patch.object(_global_loop, "_thread"),\
-             patch.object(_global_loop, "notify"):
-
-            self.make_connection()
-
-            # We have to make a copy because the connections shouldn't
-            # be alive when we verify them
-            live_connections = set(_global_loop._live_conns)
-
-            # This simulates the process ending without cluster.shutdown()
-            # being called, then with atexit _cleanup for libevreactor would
-            # be called
-            libev__cleanup(_global_loop)
-            for conn in live_connections:
-                assert conn._write_watcher.stop.mock_calls
-                assert conn._read_watcher.stop.mock_calls
-
-        _global_loop._shutdown = False
+        reactor_needs_restore = False
+        try:
+            with patch.object(_global_loop, "_thread"),\
+                 patch.object(_global_loop, "notify"):
+
+                self.make_connection()
+
+                # We have to make a copy because the connections shouldn't
+                # be alive when we verify them
+                live_connections = set(_global_loop._live_conns)
+
+                # This simulates the process ending without cluster.shutdown()
+                # being called, then with atexit _cleanup for libevreactor would
+                # be called
+                reactor_needs_restore = True
+                libev__cleanup(_global_loop)
+                for conn in live_connections:
+                    assert conn._write_watcher.stop.mock_calls
+                    assert conn._read_watcher.stop.mock_calls
+
+        finally:
+            if reactor_needs_restore:
+                _global_loop._shutdown = False
+                # _cleanup stopped the prepare watcher; restart it so the shared
+                # singleton loop is left in a working state for subsequent tests
+                # (otherwise timers would never be scheduled and tests would hang).
+                _global_loop._preparer.start()
 
 
 class LibevTimerPatcher(unittest.TestCase):

tests/unit/io/test_libevreactor_shutdown.py

@@ -117,8 +117,11 @@ def test_shutdown_cleanup_works_with_fix(self):
 import sys
 import os
 
-# Add the driver path
-sys.path.insert(0, {driver_path!r})
+# Add the driver path as a fallback only. Append (not insert at 0) so that an
+# installed build of the driver (e.g. the compiled wheel under cibuildwheel)
+# takes precedence over the in-tree pure-Python source, which lacks the libev
+# C extension and would make the import fail.
+sys.path.append({driver_path!r})
 
 # Import and setup
 from cassandra.io import libevreactor
@@ -162,9 +165,18 @@ def test_shutdown_cleanup_works_with_fix(self):
             )
 
             output = result.stdout
+            error_output = result.stderr
             print("\n=== Subprocess Output ===")
             print(output)
             print("=== End Output ===\n")
+            print("\n=== Subprocess Error Output ===")
+            print(error_output)
+            print("=== End Error Output ===\n")
+
+            self.assertEqual(
+                result.returncode, 0,
+                "Subprocess failed\nstdout:\n{}\nstderr:\n{}".format(output, error_output)
+            )
 
             # Verify the output shows the fix is working
             self.assertIn("Global loop initialized: True", output)