diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index 5e76d6bbb4..acebb1d617 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -98,4 +98,4 @@ jobs: if [[ "${{ matrix.python-version }}" =~ t$ ]]; then export PYTHON_GIL=0 fi - uv run pytest tests/integration/standard/ tests/integration/cqlengine/ + uv run pytest -v tests/integration/standard/ tests/integration/cqlengine/ diff --git a/cassandra/cluster.py b/cassandra/cluster.py index 57a8ef10aa..12ade2018f 100644 --- a/cassandra/cluster.py +++ b/cassandra/cluster.py @@ -3438,7 +3438,7 @@ def pool_finished_setting_keyspace(pool, host_errors): errors[pool.host] = host_errors if not remaining_callbacks: - callback(host_errors) + callback(errors) for pool in tuple(self._pools.values()): pool._set_keyspace_for_all_conns(keyspace, pool_finished_setting_keyspace) diff --git a/pyproject.toml b/pyproject.toml index c5ff52a426..698ff4c37b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -49,6 +49,7 @@ dev = [ "gevent", "eventlet>=0.33.3", "cython>=3.2", + "setuptools", "packaging>=25.0", "futurist", "pyyaml", @@ -157,22 +158,71 @@ enable = ["pypy"] [tool.cibuildwheel.linux] before-build = "rm -rf ~/.pyxbld && rpm --import https://repo.almalinux.org/almalinux/RPM-GPG-KEY-AlmaLinux && yum install -y libffi-devel libev libev-devel openssl openssl-devel" +# Install the optional lz4 compression dependency so the lz4 segment tests run +# (and fail loudly under CASS_DRIVER_NO_SKIP) instead of skipping silently. +test-extras = ["compress-lz4"] +# Extensions are mandatory on Linux (CASS_DRIVER_BUILD_EXTENSIONS_ARE_MUST=yes), +# so skipping is disabled (CASS_DRIVER_NO_SKIP=1): a missing dependency such as +# libev fails loudly instead of being silently skipped. Tests that cannot run in +# the default configuration are listed explicitly: +# * event-loop reactor tests are run separately with the matching +# EVENT_LOOP_MANAGER (gevent/eventlet/asyncio); +# * asyncore is deprecated and unavailable on modern Python, so it is ignored; +# * column_encryption is disabled upstream (scylladb/python-driver#365); +# * test_deserialize_date_range_month is disabled upstream (PYTHON-912). +# PyPy uses the pp* override below. All Linux CPython reactor commands run with +# CASS_DRIVER_NO_SKIP=1 so unexpected skips fail loudly. test-command = [ - "pytest {package}/tests/unit", - "EVENT_LOOP_MANAGER=gevent pytest {package}/tests/unit/io/test_geventreactor.py", + "CASS_DRIVER_NO_SKIP=1 pytest --import-mode=append {package}/tests/unit -v --ignore={package}/tests/unit/column_encryption --ignore={package}/tests/unit/io/test_geventreactor.py --ignore={package}/tests/unit/io/test_eventletreactor.py --ignore={package}/tests/unit/io/test_asyncioreactor.py --ignore={package}/tests/unit/io/test_asyncorereactor.py -k 'not test_deserialize_date_range_month'", + "EVENT_LOOP_MANAGER=gevent CASS_DRIVER_NO_SKIP=1 pytest --import-mode=append {package}/tests/unit/io/test_geventreactor.py -v", + "EVENT_LOOP_MANAGER=asyncio CASS_DRIVER_NO_SKIP=1 pytest --import-mode=append {package}/tests/unit/io/test_asyncioreactor.py -v", + "EVENT_LOOP_MANAGER=eventlet CASS_DRIVER_NO_SKIP=1 pytest --import-mode=append {package}/tests/unit/io/test_eventletreactor.py -v", ] [tool.cibuildwheel.macos] build-frontend = "build" +# Install lz4 so the lz4 segment tests run instead of skipping (see Linux note). +test-extras = ["compress-lz4"] +# Same policy as Linux (extensions are mandatory here too, libev comes from +# Homebrew). The extra -k exclusions are timing-sensitive tests that are flaky +# on macOS runners. The gevent/eventlet/asyncio reactor test files only contain +# those timing-sensitive timer tests, so they are not run separately here. test-command = [ - "pytest {project}/tests/unit -k 'not (test_multi_timer_validation or test_empty_connections or test_timer_cancellation)'", + "CASS_DRIVER_NO_SKIP=1 pytest --import-mode=append {project}/tests/unit -v --ignore={project}/tests/unit/column_encryption --ignore={project}/tests/unit/io/test_geventreactor.py --ignore={project}/tests/unit/io/test_eventletreactor.py --ignore={project}/tests/unit/io/test_asyncioreactor.py --ignore={project}/tests/unit/io/test_asyncorereactor.py -k 'not (test_multi_timer_validation or test_empty_connections or test_timer_cancellation or test_deserialize_date_range_month)'", ] [tool.cibuildwheel.windows] build-frontend = "build" +# On Windows the C extensions are optional (CASS_DRIVER_BUILD_EXTENSIONS_ARE_MUST +# is overridden to "no" below), so extension-dependent tests (e.g. libev) are +# legitimately skipped here. CASS_DRIVER_NO_SKIP is therefore NOT enabled on +# Windows; we only add -v so skips are visible in the log. test-command = [ - "pytest {project}/tests/unit -k \"not (test_deserialize_date_range_year or test_datetype or test_libevreactor)\"", + "pytest --import-mode=append {project}/tests/unit -v -k \"not (test_deserialize_date_range_year or test_datetype or test_libevreactor)\"", ] # TODO: set CASS_DRIVER_BUILD_EXTENSIONS_ARE_MUST to yes when https://github.com/scylladb/python-driver/issues/429 is fixed environment = { CASS_DRIVER_BUILD_CONCURRENCY = "2", CASS_DRIVER_BUILD_EXTENSIONS_ARE_MUST = "no" } + +# PyPy never builds the libev/cmurmur3/Cython C extensions (setup.py forces +# is_pypy to skip them even when CASS_DRIVER_BUILD_EXTENSIONS_ARE_MUST=yes), so +# the tests that depend on those extensions legitimately skip. Enforcing +# CASS_DRIVER_NO_SKIP would turn those expected skips into failures, so it is +# NOT enabled for PyPy (same reasoning as Windows). The reactor tests are not +# run separately here because eventlet is unsupported on PyPy (@notpypy) and the +# extension-backed reactors are unavailable; with no-skip off they simply skip. +# test-extras is cleared (no compress-lz4): PyPy has no prebuilt lz4 wheel, so +# pip would try to compile it from source and fail. The lz4 tests just skip here. +# test_deserialize_date_range_year and test_datetype are excluded because they +# fail on Windows (the C runtime's gmtime rejects the far-future timestamps they +# use); the CPython Windows command excludes them for the same reason. The +# timer tests (test_multi_timer_validation, test_empty_connections, +# test_timer_cancellation) are timing-sensitive and flaky on macOS, matching the +# CPython macOS exclusions. The override matches PyPy on all OSes, so these are +# deselected everywhere here (they are still covered by the CPython runs). +[[tool.cibuildwheel.overrides]] +select = "pp*" +test-extras = [] +test-command = [ + "pytest --import-mode=append {package}/tests/unit -v --ignore={package}/tests/unit/column_encryption -k \"not (test_deserialize_date_range_month or test_deserialize_date_range_year or test_datetype or test_multi_timer_validation or test_empty_connections or test_timer_cancellation)\"", +] diff --git a/tests/conftest.py b/tests/conftest.py index 8fd2fc923b..8eed388549 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -16,9 +16,45 @@ import os import warnings +import pytest + # Directory containing the Cython-compiled driver modules. _CASSANDRA_DIR = os.path.join(os.path.dirname(__file__), os.pardir, "cassandra") +# When set (e.g. in CI) a skipped test is turned into a failure. Tests skip +# themselves when their requirements are missing (a library is not installed, +# the wrong event loop is selected, ...). That is convenient locally, but in CI +# it is a footgun: a test may be silently skipped because we forgot to install +# something. Enabling this forces every skip to be explicit on the command line +# (via -k / --ignore / --deselect) instead of being hidden in the output. +_NO_SKIP = bool(os.environ.get("CASS_DRIVER_NO_SKIP")) + + +@pytest.hookimpl(hookwrapper=True) +def pytest_runtest_makereport(item, call): + """Turn skips into failures when CASS_DRIVER_NO_SKIP is set. + + xfailed tests (which are reported as skipped) are left untouched so that + ``xfail_strict`` keeps working as configured. + """ + outcome = yield + if not _NO_SKIP: + return + report = outcome.get_result() + if report.skipped and not hasattr(report, "wasxfail"): + reason = "" + if isinstance(report.longrepr, tuple) and len(report.longrepr) == 3: + reason = report.longrepr[2] + elif report.longrepr: + reason = str(report.longrepr) + report.outcome = "failed" + report.longrepr = ( + "Test was skipped but skipping is disabled in this environment " + "(CASS_DRIVER_NO_SKIP is set). Run it in a suitable configuration " + "or deselect it explicitly on the command line. " + "Original skip reason: {!r}".format(reason) + ) + def pytest_configure(config): """Warn when a compiled Cython extension is older than its .py source. diff --git a/tests/unit/io/test_libevreactor.py b/tests/unit/io/test_libevreactor.py index cf7e7caf77..a228a71de8 100644 --- a/tests/unit/io/test_libevreactor.py +++ b/tests/unit/io/test_libevreactor.py @@ -69,24 +69,33 @@ def test_watchers_are_finished(self): @test_category connection """ from cassandra.io.libevreactor import _global_loop - with patch.object(_global_loop, "_thread"),\ - patch.object(_global_loop, "notify"): - - self.make_connection() - - # We have to make a copy because the connections shouldn't - # be alive when we verify them - live_connections = set(_global_loop._live_conns) - - # This simulates the process ending without cluster.shutdown() - # being called, then with atexit _cleanup for libevreactor would - # be called - libev__cleanup(_global_loop) - for conn in live_connections: - assert conn._write_watcher.stop.mock_calls - assert conn._read_watcher.stop.mock_calls - - _global_loop._shutdown = False + reactor_needs_restore = False + try: + with patch.object(_global_loop, "_thread"),\ + patch.object(_global_loop, "notify"): + + self.make_connection() + + # We have to make a copy because the connections shouldn't + # be alive when we verify them + live_connections = set(_global_loop._live_conns) + + # This simulates the process ending without cluster.shutdown() + # being called, then with atexit _cleanup for libevreactor would + # be called + reactor_needs_restore = True + libev__cleanup(_global_loop) + for conn in live_connections: + assert conn._write_watcher.stop.mock_calls + assert conn._read_watcher.stop.mock_calls + + finally: + if reactor_needs_restore: + _global_loop._shutdown = False + # _cleanup stopped the prepare watcher; restart it so the shared + # singleton loop is left in a working state for subsequent tests + # (otherwise timers would never be scheduled and tests would hang). + _global_loop._preparer.start() class LibevTimerPatcher(unittest.TestCase): diff --git a/tests/unit/io/test_libevreactor_shutdown.py b/tests/unit/io/test_libevreactor_shutdown.py index 9578d22df1..e2f76f8a3e 100644 --- a/tests/unit/io/test_libevreactor_shutdown.py +++ b/tests/unit/io/test_libevreactor_shutdown.py @@ -117,8 +117,11 @@ def test_shutdown_cleanup_works_with_fix(self): import sys import os -# Add the driver path -sys.path.insert(0, {driver_path!r}) +# Add the driver path as a fallback only. Append (not insert at 0) so that an +# installed build of the driver (e.g. the compiled wheel under cibuildwheel) +# takes precedence over the in-tree pure-Python source, which lacks the libev +# C extension and would make the import fail. +sys.path.append({driver_path!r}) # Import and setup from cassandra.io import libevreactor @@ -162,9 +165,18 @@ def test_shutdown_cleanup_works_with_fix(self): ) output = result.stdout + error_output = result.stderr print("\n=== Subprocess Output ===") print(output) print("=== End Output ===\n") + print("\n=== Subprocess Error Output ===") + print(error_output) + print("=== End Error Output ===\n") + + self.assertEqual( + result.returncode, 0, + "Subprocess failed\nstdout:\n{}\nstderr:\n{}".format(output, error_output) + ) # Verify the output shows the fix is working self.assertIn("Global loop initialized: True", output)