diff --git a/changes/4101.changed.md b/changes/4101.changed.md new file mode 100644 index 0000000000..8edb3d3964 --- /dev/null +++ b/changes/4101.changed.md @@ -0,0 +1,30 @@ +Gave `zarr.config` a statically-typed, dataclass-backed representation. +`zarr.config` now provides precise static types for attribute access +(`zarr.config.array.order`) and for the dotted-string API +(`zarr.config.get("array.order")`), and validates configuration *keys* at +runtime with suggestions for typos. (Value validation is not yet performed; an +out-of-range value such as `config.set({"array.order": "Q"})` is accepted and +surfaces at its use site.) `donfig` is retained as the reader for environment +variables (`ZARR_FOO__BAR`) and YAML config files, so their locations and +precedence are unchanged. The string API, `config.set` (permanent and as a +context manager, including cross-thread visibility of permanent overrides), +`config.reset`, `config.enable_gpu`, and the `deprecations` mechanism are all +preserved. The former `Config` class name remains importable from +`zarr.core.config` as an alias of the new manager. + +Note: `zarr.config.defaults` now returns a nested `dict` directly; donfig +previously returned a one-element `list[dict]`, so callers that used +`config.defaults[0]` must be updated to use `config.defaults`. Subtree reads +such as `config.get("array")` now return a typed settings object rather than a +plain `dict`; it supports attribute access (`config.get("array").order`), item +access (`config.get("array")["order"]`), membership (`"order" in ...`), +iteration, `.keys()`, and `dict(...)`, but is immutable. The `codecs` mapping is +likewise exposed read-only through `zarr.config.codecs`; change codec selections +via `config.set` rather than by mutating the mapping in place. + +A few donfig `Config` methods are not reimplemented: `config.clear()` and +`config.merge()` are gone (use `config.reset()` to restore defaults), and +`config.pprint()` no longer accepts `stream`/`width` arguments. Assigning a whole +structured subtree to a mapping (for example `config.set({"array": {"order": +"F"}})`) is now rejected with an error; set leaf keys such as `array.order` +instead. diff --git a/changes/README.md b/changes/README.md index 889a52baa4..c815c457c3 100644 --- a/changes/README.md +++ b/changes/README.md @@ -6,6 +6,7 @@ Please put a new file in this directory named `xxxx..md`, where - `xxxx` is the pull request number associated with this entry - `` is one of: - feature + - changed - bugfix - doc - removal diff --git a/ci/check_changelog_entries.py b/ci/check_changelog_entries.py index 42d7cc1708..f548401fc0 100644 --- a/ci/check_changelog_entries.py +++ b/ci/check_changelog_entries.py @@ -10,7 +10,7 @@ import sys from pathlib import Path -VALID_CHANGELOG_TYPES = ["feature", "bugfix", "doc", "removal", "misc"] +VALID_CHANGELOG_TYPES = ["feature", "changed", "bugfix", "doc", "removal", "misc"] REPO_ROOT = Path(__file__).parent.parent.resolve() DEFAULT_DIRECTORY = REPO_ROOT / "changes" diff --git a/docs/user-guide/config.md b/docs/user-guide/config.md index 71c021b070..0f7942fa9a 100644 --- a/docs/user-guide/config.md +++ b/docs/user-guide/config.md @@ -1,7 +1,8 @@ # Runtime configuration -[`zarr.config`][] is responsible for managing the configuration of zarr and -is based on the [donfig](https://github.com/pytroll/donfig) Python library. +[`zarr.config`][] is a `ZarrConfigManager` instance that manages all runtime +settings for zarr. It provides both typed attribute access and a dotted-string +key API. Configuration values can be set using code like the following: @@ -18,12 +19,27 @@ zarr.config.set({'array.order': 'F'}) print(zarr.config.get('array.order')) ``` -Alternatively, configuration values can be set using environment variables, e.g. +Alternatively, configuration values can be set using environment variables. +The variable name uses a `ZARR_` prefix, with `__` to denote nesting, e.g. `ZARR_ARRAY__ORDER=F`. -The configuration can also be read from a YAML file in standard locations. -For more information, see the -[donfig documentation](https://donfig.readthedocs.io/en/latest/). +The configuration can also be read from YAML files. Environment variables and +YAML files are read by [`donfig`](https://donfig.readthedocs.io/), so zarr uses +donfig's [standard search +locations](https://donfig.readthedocs.io/en/latest/configuration.html#yaml-files), +in increasing order of precedence: + +- `/etc/zarr/` (override the `/etc` prefix with the `ZARR_ROOT_CONFIG` + environment variable), +- `/etc/zarr/` and each entry in Python's `site.PREFIXES` (e.g. + inside a virtual environment), +- `~/.config/zarr/`, +- the path in the `ZARR_CONFIG` environment variable, which may point at a + single file or a directory and takes precedence over all of the above. + +Place a `zarr.yaml` in any of these directories, or point `ZARR_CONFIG` at a +specific file. Values read from these files are validated against zarr's typed +configuration schema; unrecognized keys are ignored with a warning. Configuration options include the following: @@ -40,14 +56,11 @@ Configuration options include the following: For selecting custom implementations of codecs, pipelines, buffers and ndbuffers, first register the implementations in the registry and then select them in the config. For example, an implementation of the bytes codec in a class `'custompackage.NewBytesCodec'`, -requires the value of `codecs.bytes.name` to be `'custompackage.NewBytesCodec'`. +requires the value of `codecs.bytes` to be `'custompackage.NewBytesCodec'`. This is the current default configuration: ```python exec="true" session="config" source="above" result="ansi" from pprint import pprint -import io -output = io.StringIO() -zarr.config.pprint(stream=output, width=60) -print(output.getvalue()) +pprint(zarr.config.to_dict()) ``` diff --git a/docs/user-guide/installation.md b/docs/user-guide/installation.md index c902acf171..a710f417bc 100644 --- a/docs/user-guide/installation.md +++ b/docs/user-guide/installation.md @@ -10,7 +10,7 @@ Required dependencies include: - [numcodecs](https://numcodecs.readthedocs.io) (0.14 or later) - [google-crc32c](https://github.com/googleapis/python-crc32c) (1.5 or later) - [typing_extensions](https://typing-extensions.readthedocs.io) (4.9 or later) -- [donfig](https://donfig.readthedocs.io) (0.8 or later) +- [pyyaml](https://pyyaml.org) (6 or later) ## pip diff --git a/pyproject.toml b/pyproject.toml index 6a7238ff8f..c0162e2dce 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -496,6 +496,39 @@ title_format = "## {version} ({project_date})" issue_format = "[#{issue}](https://github.com/zarr-developers/zarr-python/issues/{issue})" start_string = "\n" +# Fragment categories. These mirror towncrier's built-in defaults, with an +# added ``changed`` category for behavior changes that are neither new features +# nor bugfixes (e.g. an intentional change to existing, documented behavior). +[[tool.towncrier.type]] +directory = "feature" +name = "Features" +showcontent = true + +[[tool.towncrier.type]] +directory = "changed" +name = "Changed" +showcontent = true + +[[tool.towncrier.type]] +directory = "bugfix" +name = "Bugfixes" +showcontent = true + +[[tool.towncrier.type]] +directory = "doc" +name = "Improved Documentation" +showcontent = true + +[[tool.towncrier.type]] +directory = "removal" +name = "Deprecations and Removals" +showcontent = true + +[[tool.towncrier.type]] +directory = "misc" +name = "Misc" +showcontent = false + [tool.codespell] ignore-words-list = "astroid" diff --git a/src/zarr/core/config.py b/src/zarr/core/config.py index 08d2a50ace..18138bb050 100644 --- a/src/zarr/core/config.py +++ b/src/zarr/core/config.py @@ -1,76 +1,782 @@ """ -The config module is responsible for managing the configuration of zarr and is based on the Donfig python library. -For selecting custom implementations of codecs, pipelines, buffers and ndbuffers, first register the implementations -in the registry and then select them in the config. +Typed configuration for zarr. -Example: - An implementation of the bytes codec in a class ``your.module.NewBytesCodec`` requires the value of ``codecs.bytes`` - to be ``your.module.NewBytesCodec``. Donfig can be configured programmatically, by environment variables, or from - YAML files in standard locations. +The module exposes a single `config` object (a `ZarrConfigManager` instance) that +holds all runtime settings. Values can be read, overridden, and restored through a +simple string-key API: - ```python - from your.module import NewBytesCodec - from zarr.core.config import register_codec, config +- `config.get(key)` — read a dotted-key value (e.g. `config.get("async.concurrency")`). +- `config.set({key: value})` — permanent override; also usable as a context manager to + restore the previous state on exit. +- `config.reset()` — rebuild from defaults + environment. +- `config.refresh()` — alias for `reset`; called by the registry after env changes. +- `config.defaults` — nested dict of built-in default values. +- `config.enable_gpu()` — switch buffer/ndbuffer to GPU implementations. - register_codec("bytes", NewBytesCodec) - config.set({"codecs.bytes": "your.module.NewBytesCodec"}) - ``` +Environment variables use the `ZARR_` prefix and `__` for nesting: - Instead of setting the value programmatically with ``config.set``, you can also set the value with an environment - variable. The environment variable ``ZARR_CODECS__BYTES`` can be set to ``your.module.NewBytesCodec``. The double - underscore ``__`` is used to indicate nested access. +```bash +export ZARR_CODECS__BYTES="your.module.NewBytesCodec" +``` - ```bash - export ZARR_CODECS__BYTES="your.module.NewBytesCodec" - ``` +Programmatic override: -For more information, see the Donfig documentation at https://github.com/pytroll/donfig. +```python +from your.module import NewBytesCodec +from zarr.core.config import config + +config.set({"codecs.bytes": "your.module.NewBytesCodec"}) +``` + +For selecting custom implementations of codecs, pipelines, buffers, and ndbuffers, +register the implementation in the registry first, then set the path via `config.set`. """ from __future__ import annotations -from typing import TYPE_CHECKING, Any, Literal, cast +import difflib +import threading +import warnings +from collections.abc import Iterator, Mapping +from dataclasses import dataclass, field, fields, is_dataclass, replace +from types import MappingProxyType +from typing import Any, Literal, Self, cast, overload -from donfig import Config as DConfig +from zarr.errors import ZarrDeprecationWarning, ZarrUserWarning -if TYPE_CHECKING: - from donfig.config_obj import ConfigSet +DEFAULT_CODECS: dict[str, str] = { + "blosc": "zarr.codecs.blosc.BloscCodec", + "gzip": "zarr.codecs.gzip.GzipCodec", + "zstd": "zarr.codecs.zstd.ZstdCodec", + "bytes": "zarr.codecs.bytes.BytesCodec", + "endian": "zarr.codecs.bytes.BytesCodec", + "crc32c": "zarr.codecs.crc32c_.Crc32cCodec", + "sharding_indexed": "zarr.codecs.sharding.ShardingCodec", + "transpose": "zarr.codecs.transpose.TransposeCodec", + "vlen-utf8": "zarr.codecs.vlen_utf8.VLenUTF8Codec", + "vlen-bytes": "zarr.codecs.vlen_utf8.VLenBytesCodec", + "numcodecs.bz2": "zarr.codecs.numcodecs.BZ2", + "numcodecs.crc32": "zarr.codecs.numcodecs.CRC32", + "numcodecs.crc32c": "zarr.codecs.numcodecs.CRC32C", + "numcodecs.lz4": "zarr.codecs.numcodecs.LZ4", + "numcodecs.lzma": "zarr.codecs.numcodecs.LZMA", + "numcodecs.zfpy": "zarr.codecs.numcodecs.ZFPY", + "numcodecs.adler32": "zarr.codecs.numcodecs.Adler32", + "numcodecs.astype": "zarr.codecs.numcodecs.AsType", + "numcodecs.bitround": "zarr.codecs.numcodecs.BitRound", + "numcodecs.blosc": "zarr.codecs.numcodecs.Blosc", + "numcodecs.delta": "zarr.codecs.numcodecs.Delta", + "numcodecs.fixedscaleoffset": "zarr.codecs.numcodecs.FixedScaleOffset", + "numcodecs.fletcher32": "zarr.codecs.numcodecs.Fletcher32", + "numcodecs.gzip": "zarr.codecs.numcodecs.GZip", + "numcodecs.jenkins_lookup3": "zarr.codecs.numcodecs.JenkinsLookup3", + "numcodecs.pcodec": "zarr.codecs.numcodecs.PCodec", + "numcodecs.packbits": "zarr.codecs.numcodecs.PackBits", + "numcodecs.shuffle": "zarr.codecs.numcodecs.Shuffle", + "numcodecs.quantize": "zarr.codecs.numcodecs.Quantize", + "numcodecs.zlib": "zarr.codecs.numcodecs.Zlib", + "numcodecs.zstd": "zarr.codecs.numcodecs.Zstd", +} +# Map serialized dotted-key segments to Python field names where they differ +# (Python keywords cannot be used as identifiers). +_FIELD_ALIASES: dict[str, str] = {"async": "async_"} +_SERIALIZED_NAMES: dict[str, str] = {v: k for k, v in _FIELD_ALIASES.items()} -class BadConfigError(ValueError): - _msg = "bad Config: %r" + +class _ConfigNode: + """Mixin giving the frozen config dataclasses donfig-style item access. + + donfig returned configuration subtrees as plain `dict`s, so callers could + write `config.get("array")["order"]`. The typed config returns dataclass + instances instead; this mixin restores subscripting (`node["order"]`, and + dotted `node["a.b"]`) alongside the typed attribute access (`node.order`), + raising `KeyError` for unknown keys just like the old dicts did. + + `__slots__ = ()` keeps the subclasses fully slotted (no `__dict__`). + """ + + __slots__ = () + + def __getitem__(self, key: str) -> object: + # `self` is always a config node (a frozen dataclass); `get_path` walks + # such nodes structurally, so the cast is sound. + return get_path(cast("ZarrConfig", self), key) + + def __contains__(self, key: object) -> bool: + # Mirror donfig's `"array" in config.get(...)` support. Only string keys + # can resolve; a non-string (or unknown key) is simply not contained. + if not isinstance(key, str): + return False + try: + get_path(cast("ZarrConfig", self), key) + except KeyError: + return False + return True + + def __iter__(self) -> Iterator[str]: + # Yield immediate child key names (serialized), so iteration, `keys()`, + # and `dict(node)` behave like the plain dicts donfig returned. Note this + # deliberately does NOT make the node a `collections.abc.Mapping`: the + # internal `isinstance(obj, Mapping)` checks must stay false for config + # nodes so they are distinguished from the open `codecs` mapping. + return iter(_children(self)) + + def keys(self) -> list[str]: + return _children(self) + + def __len__(self) -> int: + return len(_children(self)) + + +@dataclass(frozen=True, slots=True) +class ArraySettings(_ConfigNode): + order: Literal["C", "F"] = "C" + write_empty_chunks: bool = False + read_missing_chunks: bool = True + target_shard_size_bytes: int | None = None + rectilinear_chunks: bool = False + sharding_coalesce_max_gap_bytes: int = 1 << 20 + sharding_coalesce_max_bytes: int = 16 << 20 -class Config(DConfig): # type: ignore[misc] - """The Config will collect configuration from config files and environment variables +@dataclass(frozen=True, slots=True) +class AsyncSettings(_ConfigNode): + concurrency: int = 10 + timeout: float | None = None - Example environment variables: - Grabs environment variables of the form "ZARR_FOO__BAR_BAZ=123" and - turns these into config variables of the form ``{"foo": {"bar-baz": 123}}`` - It transforms the key and value in the following way: - - Lower-cases the key text - - Treats ``__`` (double-underscore) as nested access - - Calls ``ast.literal_eval`` on the value +@dataclass(frozen=True, slots=True) +class ThreadingSettings(_ConfigNode): + max_workers: int | None = None + +@dataclass(frozen=True, slots=True) +class CodecPipelineSettings(_ConfigNode): + path: str = "zarr.core.codec_pipeline.BatchedCodecPipeline" + batch_size: int = 1 + + +@dataclass(frozen=True, slots=True) +class ZarrConfig(_ConfigNode): + default_zarr_format: Literal[2, 3] = 3 + array: ArraySettings = field(default_factory=ArraySettings) + async_: AsyncSettings = field(default_factory=AsyncSettings) + threading: ThreadingSettings = field(default_factory=ThreadingSettings) + json_indent: int = 2 + codec_pipeline: CodecPipelineSettings = field(default_factory=CodecPipelineSettings) + # A plain dict (not MappingProxyType) so snapshots stay picklable / + # deep-copyable; public read access goes through the manager's `codecs` + # property, which wraps this in a read-only view. + codecs: Mapping[str, str] = field(default_factory=lambda: dict(DEFAULT_CODECS)) + buffer: str = "zarr.buffer.cpu.Buffer" + ndbuffer: str = "zarr.buffer.cpu.NDBuffer" + + +def make_default_config() -> ZarrConfig: + """Return a fresh `ZarrConfig` populated with the built-in defaults.""" + return ZarrConfig() + + +def _resolve_field(obj: object, segment: str) -> str: + """Translate a serialized key segment to the dataclass field name.""" + return _FIELD_ALIASES.get(segment, segment) + + +def get_path(cfg: ZarrConfig, key: str) -> object: + """Read a dotted-string key from a `ZarrConfig` snapshot. + + Raises + ------ + KeyError + If the key does not resolve to a value. """ + obj: object = cfg + segments = key.split(".") + for i, segment in enumerate(segments): + if isinstance(obj, Mapping): + # remaining segments index into an open mapping (e.g. codecs.*) + remainder = ".".join(segments[i:]) + try: + return obj[remainder] + except KeyError: + raise KeyError(key) from None + # A prior segment resolved to a scalar leaf, but the key has more + # segments — descend no further. Without this guard, `hasattr` would + # match ordinary Python attributes/methods (e.g. `array.order.upper` + # returning `str.upper`, or `default_zarr_format.numerator` returning + # the int's numerator) instead of raising for the invalid key. + if not is_dataclass(obj): + raise KeyError(key) + field_name = _resolve_field(obj, segment) + if field_name not in {f.name for f in fields(obj)}: + raise KeyError(key) + obj = getattr(obj, field_name) + return obj - def reset(self) -> None: - self.clear() - self.refresh() - def enable_gpu(self) -> ConfigSet: - """ - Configure Zarr to use GPUs where possible. +def replace_path(cfg: ZarrConfig, key: str, value: object) -> ZarrConfig: + """Return a new `ZarrConfig` with the dotted-string key set to ``value``.""" + segments = key.split(".") + return cast(ZarrConfig, _replace_recursive(cfg, segments, value, key)) + + +# `obj: Any` is load-bearing here: the function dispatches dynamically between a +# `Mapping` (codecs subtree) and a dataclass instance, and `dataclasses.replace` +# requires a dataclass-typed argument that `object` would reject. +def _replace_recursive(obj: Any, segments: list[str], value: object, key: str) -> object: + segment = segments[0] + if isinstance(obj, Mapping): + remainder = ".".join(segments) + # Plain dict (see the `codecs` field note); the manager property wraps + # it read-only for public access. + return {**obj, remainder: value} + if not is_dataclass(obj): + # `key` tries to descend past a scalar leaf (e.g. `array.order.upper`). + raise KeyError(key) + field_name = _resolve_field(obj, segment) + if field_name not in {f.name for f in fields(obj)}: + raise KeyError(key) + # `is_dataclass` narrows `obj` to `... | type[...]`, which `replace` rejects; + # at runtime `obj` is always a dataclass *instance* here, so re-widen to Any. + node: Any = obj + child = getattr(node, field_name) + if len(segments) == 1: + # Refuse to overwrite a structured subtree (a nested dataclass) wholesale + # — doing so would drop its sibling fields and break typed attribute + # access. Set leaf keys instead. The open `codecs` mapping is not a + # dataclass, so wholesale replacement there is still allowed. + if is_dataclass(child): + raise TypeError( + f"Cannot assign to the structured config subtree {key!r} directly; " + f"set leaf keys instead, e.g. config.set({{'{key}.': ...}})." + ) + return replace(node, **{field_name: value}) + new_child = _replace_recursive(child, segments[1:], value, key) + return replace(node, **{field_name: new_child}) + + +def delete_path(cfg: ZarrConfig, key: str) -> ZarrConfig: + """Return a new `ZarrConfig` with ``key`` removed from an open mapping. + + Only keys inside an open mapping (``codecs.*``) can be deleted; structured + fields always exist and cannot be removed. Used to undo a scoped ``set`` that + *added* a new codec key when its ``with`` block exits. + """ + segments = key.split(".") + return cast("ZarrConfig", _delete_recursive(cfg, segments, key)) + + +def _delete_recursive(obj: Any, segments: list[str], key: str) -> object: + if isinstance(obj, Mapping): + remainder = ".".join(segments) + return {k: v for k, v in obj.items() if k != remainder} + if not is_dataclass(obj): + raise KeyError(key) + field_name = _resolve_field(obj, segments[0]) + if field_name not in {f.name for f in fields(obj)}: + raise KeyError(key) + node: Any = obj + child = getattr(node, field_name) + if len(segments) == 1: + # a structured field can't be deleted; only open-mapping leaves can + raise KeyError(key) + return replace(node, **{field_name: _delete_recursive(child, segments[1:], key)}) + + +_ROSTER_LIMIT = 10 + + +def _children(obj: object) -> list[str]: + """Return the immediate child key names of a config node (else an empty list).""" + if isinstance(obj, Mapping): + return list(obj) + if is_dataclass(obj): + return [_SERIALIZED_NAMES.get(f.name, f.name) for f in fields(obj)] + return [] + + +def _resolve_for_suggestion(cfg: ZarrConfig, key: str) -> tuple[str, list[str], str]: + """Walk ``key`` as far as it resolves. + + Returns the deepest resolvable dotted prefix, that node's child key names, + and the first segment that failed to resolve (the remainder is treated as a + single key once an open mapping like ``codecs`` is reached). For + ``"array.bogus"`` this is ``("array", [], "bogus")``; + for an unknown top-level key, ``("", [], )``. + """ + obj: object = cfg + prefix = "" + segments = key.split(".") + for i, segment in enumerate(segments): + if isinstance(obj, Mapping): + # the remainder indexes into an open mapping as a single key + return prefix, _children(obj), ".".join(segments[i:]) + if not is_dataclass(obj): + # a prior segment resolved to a scalar leaf; nothing deeper is valid + return prefix, _children(obj), segment + field_name = _resolve_field(obj, segment) + if field_name not in {f.name for f in fields(obj)}: + return prefix, _children(obj), segment + obj = getattr(obj, field_name) + prefix = f"{prefix}.{segment}" if prefix else segment + return prefix, _children(obj), "" + + +def _unknown_key_error(key: str, cfg: ZarrConfig) -> KeyError: + """Build a `KeyError` for an unknown config key. + + Resolves ``key`` to the deepest valid level, then suggests the closest child + key there if one is similar enough; otherwise lists the available keys at + that level (capped at `_ROSTER_LIMIT`). + """ + msg = f"{key!r} is not a valid configuration key." + prefix, children, failed = _resolve_for_suggestion(cfg, key) + matches = difflib.get_close_matches(failed, children, n=1) if failed != "" else [] + if len(matches) > 0: + suggestion = f"{prefix}.{matches[0]}" if prefix != "" else matches[0] + return KeyError(f"{msg} Did you mean {suggestion!r}?") + if len(children) > 0: + shown = sorted(children) + roster = ", ".join(shown[:_ROSTER_LIMIT]) + if len(shown) > _ROSTER_LIMIT: + roster += f", ... ({len(shown) - _ROSTER_LIMIT} more)" + where = f" under {prefix!r}" if prefix != "" else "" + msg = f"{msg} Valid keys{where}: {roster}." + return KeyError(msg) + + +def to_nested_dict(cfg: ZarrConfig) -> dict[str, Any]: + """Convert a `ZarrConfig` to a donfig-style nested dict (serialized keys). + + Returns a heterogeneous, JSON-like tree (nested dicts and scalars) that + callers navigate by key, so `Any` values are appropriate here. + """ + + # `obj: Any` is also load-bearing: `dataclasses.fields` requires a + # dataclass-typed argument that `object` would reject. + def convert(obj: Any) -> Any: + if isinstance(obj, Mapping): + return dict(obj) + if hasattr(type(obj), "__dataclass_fields__"): + out: dict[str, Any] = {} + for f in fields(obj): + serialized = _SERIALIZED_NAMES.get(f.name, f.name) + out[serialized] = convert(getattr(obj, f.name)) + return out + return obj + + return convert(cfg) # type: ignore[no-any-return] + + +def _flatten_mapping(data: Mapping[str, object], prefix: str = "") -> dict[str, object]: + out: dict[str, object] = {} + for k, v in data.items(): + key = f"{prefix}{k}" if not prefix else f"{prefix}.{k}" + if isinstance(v, Mapping): + out.update(_flatten_mapping(v, key)) + else: + out[key] = v + return out + + +def apply_overrides(cfg: ZarrConfig, overrides: Mapping[str, object]) -> ZarrConfig: + """Apply a flat dotted-key override map to a snapshot. + + Used exclusively by `build_config` for env/YAML ingest. Unknown keys are + skipped with a warning rather than raising, so a stray environment variable + or extra YAML key never prevents `import zarr` from succeeding. + """ + for key, value in overrides.items(): + try: + cfg = replace_path(cfg, key, value) + except KeyError: + warnings.warn( + f"Unrecognized zarr config key {key!r} from environment or YAML — ignoring.", + ZarrUserWarning, + stacklevel=2, + ) + return cfg + + +# donfig's env collection also surfaces the `ZARR_CONFIG` / `ZARR_ROOT_CONFIG` +# path directives as if they were config values (keys `config` / `root_config`); +# drop them so they don't trip `apply_overrides`'s unknown-key warning. +_DONFIG_META_KEYS: frozenset[str] = frozenset({"config", "root_config"}) + + +def _canonicalize_override_keys(overrides: Mapping[str, object]) -> dict[str, object]: + """Map underscore codec names onto their hyphenated built-in defaults. + + Environment variables cannot contain hyphens, so ``ZARR_CODECS__VLEN_UTF8`` + flattens to the key ``codecs.vlen_utf8``. The built-in codec is registered + under the hyphenated name ``vlen-utf8`` (likewise ``vlen-bytes``), so without + this remapping the override would land under a dead ``vlen_utf8`` key and be + silently ignored while the registry keeps reading the untouched default. + When a ``codecs.`` key does not match a default but its hyphenated + variant does, rewrite it to the hyphenated form. New (non-default) codec + names and underscore-named defaults (e.g. ``sharding_indexed``) are untouched. + """ + out: dict[str, object] = {} + for key, value in overrides.items(): + if key.startswith("codecs."): + name = key[len("codecs.") :] + if name not in DEFAULT_CODECS and name.replace("_", "-") in DEFAULT_CODECS: + key = f"codecs.{name.replace('_', '-')}" + out[key] = value + return out + + +def build_config() -> ZarrConfig: + """Build the base snapshot: typed defaults overlaid with donfig's ingest. + + `donfig` reads `ZARR_*` environment variables and YAML config files from its + standard locations + (https://donfig.readthedocs.io/en/latest/configuration.html#yaml-files) and + merges them into a nested override mapping. That mapping is flattened to + dotted keys and applied on top of the typed defaults. `donfig` owns discovery, + parsing, and precedence; this module owns the typed representation. Unknown + keys are warned about and skipped by `apply_overrides`, so a stray variable or + a version-skewed config file never blocks `import zarr`. + """ + import donfig + + overrides = _flatten_mapping(donfig.Config("zarr").config) + overrides = { + key: value + for key, value in overrides.items() + if key.split(".", 1)[0] not in _DONFIG_META_KEYS + } + overrides = _canonicalize_override_keys(overrides) + return apply_overrides(make_default_config(), overrides) + + +_MISSING = object() + + +# A per-key record of how to undo a scoped `set`: (resolved_key, existed_before, +# previous_value). If the key did not exist before (a newly added `codecs.*` +# entry), it is removed on exit instead of restored. +_RestoreEntry = tuple[str, bool, object] + + +class _ConfigSet: + """Context manager returned by ``ZarrConfigManager.set``. + + ``set`` applies its overrides immediately to the process-global config, so a + bare ``config.set(...)`` is permanent and visible from every thread (donfig's + last-writer-wins semantics). Used as a ``with`` block, the overrides are + reverted on ``__exit__``: each key it set is restored to its previous value + (or removed, if it was newly added), matching ``donfig``. Reverting only the + keys it touched leaves concurrent changes to *other* keys intact. + + Note: like ``donfig``, a ``with config.set(...)`` is **not** isolated to the + calling thread / async task — during the block the change is globally visible. + Context-local isolation is a possible future enhancement (see gh-4101). + """ + + def __init__(self, manager: ZarrConfigManager, restore: list[_RestoreEntry]) -> None: + self._manager = manager + self._restore = restore + + def __enter__(self) -> Self: + # The overrides were already applied globally by `set`; nothing to do. + return self + + def __exit__(self, *exc: object) -> None: + self._manager._revert(self._restore) + + +class ZarrConfigManager: + """Typed, donfig-compatible configuration object.""" + + def __init__(self) -> None: + self._base: ZarrConfig = build_config() + # Serializes read-modify-write of the process-global `_base` so + # concurrent `set`s / reverts to different keys don't lose updates + # (each rebuilds a whole immutable snapshot from `_base`). + self._lock = threading.Lock() + + # --- state resolution ------------------------------------------------- + def _current(self) -> ZarrConfig: + # A single process-global snapshot: a `set` (bare or scoped) is globally + # visible, and a `with config.set(...)` reverts its keys on exit. This + # mirrors donfig; there is no per-thread/async overlay. + return self._base + + def _revert(self, restore: list[_RestoreEntry]) -> None: + # Undo a scoped `set` on `with`-block exit: restore each key it touched to + # its previous value (or remove a key it newly added), leaving concurrent + # changes to other keys intact. + with self._lock: + cfg = self._base + for resolved, existed, old in reversed(restore): + cfg = replace_path(cfg, resolved, old) if existed else delete_path(cfg, resolved) + self._base = cfg + + # --- typed attribute access ------------------------------------------ + @property + def default_zarr_format(self) -> Literal[2, 3]: + return self._current().default_zarr_format + + @property + def array(self) -> ArraySettings: + return self._current().array + + @property + def async_(self) -> AsyncSettings: + return self._current().async_ + + @property + def threading(self) -> ThreadingSettings: + return self._current().threading + + @property + def codec_pipeline(self) -> CodecPipelineSettings: + return self._current().codec_pipeline + + @property + def json_indent(self) -> int: + return self._current().json_indent + + @property + def codecs(self) -> Mapping[str, str]: + # Read-only view: the underlying snapshot field is a plain dict (kept + # picklable), but callers must not mutate a live snapshot in place. + return MappingProxyType(self._current().codecs) + + @property + def buffer(self) -> str: + return self._current().buffer + + @property + def ndbuffer(self) -> str: + return self._current().ndbuffer + + # --- string API: get -------------------------------------------------- + @overload + def get(self, key: Literal["default_zarr_format"]) -> Literal[2, 3]: ... + @overload + def get(self, key: Literal["array.order"]) -> Literal["C", "F"]: ... + @overload + def get(self, key: Literal["array.write_empty_chunks"]) -> bool: ... + @overload + def get(self, key: Literal["array.read_missing_chunks"]) -> bool: ... + @overload + def get(self, key: Literal["array.target_shard_size_bytes"]) -> int | None: ... + @overload + def get(self, key: Literal["array.rectilinear_chunks"]) -> bool: ... + @overload + def get(self, key: Literal["array.sharding_coalesce_max_gap_bytes"]) -> int: ... + @overload + def get(self, key: Literal["array.sharding_coalesce_max_bytes"]) -> int: ... + @overload + def get(self, key: Literal["async.concurrency"]) -> int: ... + @overload + def get(self, key: Literal["async.timeout"]) -> float | None: ... + @overload + def get(self, key: Literal["threading.max_workers"]) -> int | None: ... + @overload + def get(self, key: Literal["json_indent"]) -> int: ... + @overload + def get(self, key: Literal["codec_pipeline.path"]) -> str: ... + @overload + def get(self, key: Literal["codec_pipeline.batch_size"]) -> int: ... + @overload + def get(self, key: Literal["buffer"]) -> str: ... + @overload + def get(self, key: Literal["ndbuffer"]) -> str: ... + @overload + # The fallback `-> Any` is deliberate: it lets `config.get("codecs", {})` be + # used as a mapping (e.g. `.get(name)` in the registry) and supports unknown + # keys. `object` here would force every such call site to narrow first. + def get(self, key: str, default: object = ...) -> Any: ... + + def get(self, key: str, default: object = _MISSING) -> Any: + resolved = self._apply_deprecation(key, raise_on_removed=False) + if resolved is None: + # Key was removed; treat as absent — honour the caller's default. + if default is _MISSING: + raise KeyError(key) + return default + current = self._current() + try: + return get_path(current, resolved) + except KeyError: + if default is _MISSING: + raise _unknown_key_error(key, current) from None + return default + + # --- string API: set -------------------------------------------------- + # + # NOTE: `set` accepts `Mapping[str, Any]`, so — unlike `get`, which is fully + # typed via per-key overloads — it does NOT statically validate values: + # `config.set({"array.order": "Q"})` is not a type error; it is caught at + # runtime instead. This is a deliberate, documented limitation. + # + # Static value typing would require an *open* TypedDict — declared structured + # keys validated by type, PLUS arbitrary `codecs.` string keys allowed + # (PEP 728 `extra_items`/`closed`). mypy (2.x) supports PEP 728 in no syntax + # and offers no feature flag for it. A *closed* TypedDict would instead reject + # the open codec-selection idiom + # `config.set({"codecs.bytes": "your.module.NewBytesCodec"})` and any + # dynamically built `dict[str, Any]` — a backwards-compatibility regression + # (the `codecs` namespace maps a codec name to a class path and is extended at + # runtime by users/plugins, so its keys cannot be enumerated statically). + # So `set` is intentionally permissive and validated at runtime: unknown + # structured keys raise (see `replace_path`), while `codecs.*` stays writable. + # + # REVISIT when mypy ships PEP 728 open-TypedDict support, or if zarr adopts a + # type checker that supports it (e.g. pyright's open/closed TypedDicts). At + # that point `set` can take an open TypedDict for static value validation + # while keeping `codecs.*` open. + def set(self, updates: Mapping[str, object] | None = None, **kwargs: object) -> _ConfigSet: + """Apply one or more config overrides. + + Accepts either a mapping of dotted keys to values, keyword arguments + (for top-level keys), or both:: + + config.set({"array.order": "F"}) + config.set(default_zarr_format=2) + + `set` validates *keys* — an unknown key raises with a suggestion — but + does **not** validate *values*: `config.set({"array.order": "Q"})` is + accepted, and the invalid value surfaces later at its use site rather + than here. Static value typing is prevented by the open `codecs.*` + namespace (see the implementation comment above); runtime value + validation is planned via the unified `parse_json` checker (gh-3285). """ + all_updates: dict[str, object] = {} + if updates: + all_updates.update(updates) + all_updates.update(kwargs) + # Apply immediately and globally (donfig semantics). Hold the lock across + # the read-modify-write of `_base` so concurrent `set`s to different keys + # don't clobber each other (each rebuilds a full snapshot). Record how to + # undo each key so a `with config.set(...)` can revert on exit. + restore: list[_RestoreEntry] = [] + with self._lock: + cfg = self._base + for key, value in all_updates.items(): + resolved = self._apply_deprecation(key, raise_on_removed=True) + try: + old = get_path(cfg, resolved) + existed = True + except KeyError: + old = None + existed = False + try: + cfg = replace_path(cfg, resolved, value) + except KeyError: + raise _unknown_key_error(key, cfg) from None + restore.append((resolved, existed, old)) + self._base = cfg + return _ConfigSet(self, restore) + + # --- lifecycle -------------------------------------------------------- + def reset(self) -> None: + # Rebuild the global base from defaults + env/YAML. Build outside the lock + # (it reads env/YAML) and swap atomically under it. + new_base = build_config() + with self._lock: + self._base = new_base + + def refresh(self) -> None: + self.reset() + + def enable_gpu(self) -> _ConfigSet: return self.set( {"buffer": "zarr.buffer.gpu.Buffer", "ndbuffer": "zarr.buffer.gpu.NDBuffer"} ) + # --- compat / introspection ------------------------------------------ + def __getitem__(self, key: str) -> Any: + # donfig-style item access: `config["array.order"]` mirrors `get`. + return self.get(key) + + def __contains__(self, key: object) -> bool: + # donfig-style membership: `"array.order" in config`. + if not isinstance(key, str): + return False + try: + self.get(key) + except KeyError: + return False + return True + + def __iter__(self) -> Iterator[str]: + # Defining `__getitem__` above would otherwise make Python fall back to + # the legacy integer-index iteration protocol (`config[0]` -> confusing + # `'int' object has no attribute 'split'`). The manager is a keyed config, + # not a sequence, so fail clearly instead. + raise TypeError( + "ZarrConfigManager is not iterable; use config.to_dict() to iterate " + "its contents, or config.get() for a single value." + ) + + @property + def defaults(self) -> dict[str, Any]: + return to_nested_dict(make_default_config()) + + def to_dict(self) -> dict[str, Any]: + return to_nested_dict(self._current()) + + def update(self, updates: Mapping[str, object]) -> None: + self.set(updates) + + def pprint(self) -> None: + import pprint as _pp + + _pp.pprint(self.to_dict()) + + # --- deprecations ----------------------------------------------------- + @overload + def _apply_deprecation(self, key: str, *, raise_on_removed: Literal[True]) -> str: ... + @overload + def _apply_deprecation(self, key: str, *, raise_on_removed: Literal[False]) -> str | None: ... + + def _apply_deprecation(self, key: str, *, raise_on_removed: bool) -> str | None: + """Resolve a possibly-deprecated config key. + + Parameters + ---------- + key : str + The dotted config key supplied by the caller. + raise_on_removed : bool + When `True` (used by `set`), raise `BadConfigError` if the key has been + removed. When `False` (used by `get`), return `None` instead so the + caller can treat the key as absent and honour the caller's default. + + Returns + ------- + str or None + The canonical (possibly redirected) key, or `None` when the key was + removed and `raise_on_removed` is `False`. + """ + if key not in deprecations: + return key + new_key = deprecations[key] + if new_key is None: + if raise_on_removed: + raise BadConfigError( + f"Configuration key {key!r} has been removed and no longer has any effect." + ) + return None + warnings.warn( + f"Configuration key {key!r} has been renamed to {new_key!r}.", + ZarrDeprecationWarning, + stacklevel=3, + ) + return new_key + + +class BadConfigError(ValueError): + _msg = "bad Config: %r" + # these keys were removed from the config as part of the 3.1.0 release. -# these deprecations should be removed in 3.1.1 or thereabouts. -deprecations = { +# These deprecations should be removed in 3.1.1 or thereabouts. +deprecations: dict[str, str | None] = { "array.v2_default_compressor.numeric": None, "array.v2_default_compressor.string": None, "array.v2_default_compressor.bytes": None, @@ -87,71 +793,17 @@ def enable_gpu(self) -> ConfigSet: "array.v3_default_compressors": None, } -# The default configuration for zarr -config = Config( - "zarr", - defaults=[ - { - "default_zarr_format": 3, - "array": { - "order": "C", - "write_empty_chunks": False, - "read_missing_chunks": True, - "target_shard_size_bytes": None, - "rectilinear_chunks": False, - "sharding_coalesce_max_gap_bytes": 1 << 20, # 1 MiB - "sharding_coalesce_max_bytes": 16 << 20, # 16 MiB - }, - "async": {"concurrency": 10, "timeout": None}, - "threading": {"max_workers": None}, - "json_indent": 2, - "codec_pipeline": { - "path": "zarr.core.codec_pipeline.BatchedCodecPipeline", - "batch_size": 1, - }, - "codecs": { - "blosc": "zarr.codecs.blosc.BloscCodec", - "gzip": "zarr.codecs.gzip.GzipCodec", - "zstd": "zarr.codecs.zstd.ZstdCodec", - "bytes": "zarr.codecs.bytes.BytesCodec", - "endian": "zarr.codecs.bytes.BytesCodec", # compatibility with earlier versions of ZEP1 - "crc32c": "zarr.codecs.crc32c_.Crc32cCodec", - "sharding_indexed": "zarr.codecs.sharding.ShardingCodec", - "transpose": "zarr.codecs.transpose.TransposeCodec", - "vlen-utf8": "zarr.codecs.vlen_utf8.VLenUTF8Codec", - "vlen-bytes": "zarr.codecs.vlen_utf8.VLenBytesCodec", - "numcodecs.bz2": "zarr.codecs.numcodecs.BZ2", - "numcodecs.crc32": "zarr.codecs.numcodecs.CRC32", - "numcodecs.crc32c": "zarr.codecs.numcodecs.CRC32C", - "numcodecs.lz4": "zarr.codecs.numcodecs.LZ4", - "numcodecs.lzma": "zarr.codecs.numcodecs.LZMA", - "numcodecs.zfpy": "zarr.codecs.numcodecs.ZFPY", - "numcodecs.adler32": "zarr.codecs.numcodecs.Adler32", - "numcodecs.astype": "zarr.codecs.numcodecs.AsType", - "numcodecs.bitround": "zarr.codecs.numcodecs.BitRound", - "numcodecs.blosc": "zarr.codecs.numcodecs.Blosc", - "numcodecs.delta": "zarr.codecs.numcodecs.Delta", - "numcodecs.fixedscaleoffset": "zarr.codecs.numcodecs.FixedScaleOffset", - "numcodecs.fletcher32": "zarr.codecs.numcodecs.Fletcher32", - "numcodecs.gzip": "zarr.codecs.numcodecs.GZip", - "numcodecs.jenkins_lookup3": "zarr.codecs.numcodecs.JenkinsLookup3", - "numcodecs.pcodec": "zarr.codecs.numcodecs.PCodec", - "numcodecs.packbits": "zarr.codecs.numcodecs.PackBits", - "numcodecs.shuffle": "zarr.codecs.numcodecs.Shuffle", - "numcodecs.quantize": "zarr.codecs.numcodecs.Quantize", - "numcodecs.zlib": "zarr.codecs.numcodecs.Zlib", - "numcodecs.zstd": "zarr.codecs.numcodecs.Zstd", - }, - "buffer": "zarr.buffer.cpu.Buffer", - "ndbuffer": "zarr.buffer.cpu.NDBuffer", - } - ], - deprecations=deprecations, -) - - -def parse_indexing_order(data: Any) -> Literal["C", "F"]: +# Backwards-compatible alias: before the typed rewrite, this module exposed the +# donfig subclass as `Config`. Keep the name so `from zarr.core.config import +# Config` and `isinstance(x, Config)` continue to work for downstream code. +Config = ZarrConfigManager + +config = ZarrConfigManager() + + +def parse_indexing_order(data: object) -> Literal["C", "F"]: if data in ("C", "F"): - return cast("Literal['C', 'F']", data) + # the membership check narrows `data` to Literal["C", "F"] + return data msg = f"Expected one of ('C', 'F'), got {data} instead." raise ValueError(msg) diff --git a/tests/test_config.py b/tests/test_config.py index a758378dc7..1eac0a1253 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -22,7 +22,7 @@ from zarr.core.buffer import NDBuffer from zarr.core.buffer.core import Buffer from zarr.core.codec_pipeline import BatchedCodecPipeline -from zarr.core.config import BadConfigError, config +from zarr.core.config import DEFAULT_CODECS, BadConfigError, config from zarr.core.indexing import SelectorTuple from zarr.errors import ChunkNotFoundError, ZarrUserWarning from zarr.registry import ( @@ -45,66 +45,28 @@ def test_config_defaults_set() -> None: - # regression test for available defaults - assert ( - config.defaults - == [ - { - "default_zarr_format": 3, - "array": { - "order": "C", - "write_empty_chunks": False, - "read_missing_chunks": True, - "target_shard_size_bytes": None, - "rectilinear_chunks": False, - "sharding_coalesce_max_gap_bytes": 1 << 20, - "sharding_coalesce_max_bytes": 16 << 20, - }, - "async": {"concurrency": 10, "timeout": None}, - "threading": {"max_workers": None}, - "json_indent": 2, - "codec_pipeline": { - "path": "zarr.core.codec_pipeline.BatchedCodecPipeline", - "batch_size": 1, - }, - "codecs": { - "blosc": "zarr.codecs.blosc.BloscCodec", - "gzip": "zarr.codecs.gzip.GzipCodec", - "zstd": "zarr.codecs.zstd.ZstdCodec", - "bytes": "zarr.codecs.bytes.BytesCodec", - "endian": "zarr.codecs.bytes.BytesCodec", # compatibility with earlier versions of ZEP1 - "crc32c": "zarr.codecs.crc32c_.Crc32cCodec", - "sharding_indexed": "zarr.codecs.sharding.ShardingCodec", - "transpose": "zarr.codecs.transpose.TransposeCodec", - "vlen-utf8": "zarr.codecs.vlen_utf8.VLenUTF8Codec", - "vlen-bytes": "zarr.codecs.vlen_utf8.VLenBytesCodec", - "numcodecs.bz2": "zarr.codecs.numcodecs.BZ2", - "numcodecs.crc32": "zarr.codecs.numcodecs.CRC32", - "numcodecs.crc32c": "zarr.codecs.numcodecs.CRC32C", - "numcodecs.lz4": "zarr.codecs.numcodecs.LZ4", - "numcodecs.lzma": "zarr.codecs.numcodecs.LZMA", - "numcodecs.zfpy": "zarr.codecs.numcodecs.ZFPY", - "numcodecs.adler32": "zarr.codecs.numcodecs.Adler32", - "numcodecs.astype": "zarr.codecs.numcodecs.AsType", - "numcodecs.bitround": "zarr.codecs.numcodecs.BitRound", - "numcodecs.blosc": "zarr.codecs.numcodecs.Blosc", - "numcodecs.delta": "zarr.codecs.numcodecs.Delta", - "numcodecs.fixedscaleoffset": "zarr.codecs.numcodecs.FixedScaleOffset", - "numcodecs.fletcher32": "zarr.codecs.numcodecs.Fletcher32", - "numcodecs.gzip": "zarr.codecs.numcodecs.GZip", - "numcodecs.jenkins_lookup3": "zarr.codecs.numcodecs.JenkinsLookup3", - "numcodecs.pcodec": "zarr.codecs.numcodecs.PCodec", - "numcodecs.packbits": "zarr.codecs.numcodecs.PackBits", - "numcodecs.shuffle": "zarr.codecs.numcodecs.Shuffle", - "numcodecs.quantize": "zarr.codecs.numcodecs.Quantize", - "numcodecs.zlib": "zarr.codecs.numcodecs.Zlib", - "numcodecs.zstd": "zarr.codecs.numcodecs.Zstd", - }, - "buffer": "zarr.buffer.cpu.Buffer", - "ndbuffer": "zarr.buffer.cpu.NDBuffer", - } - ] - ) + assert config.defaults == { + "default_zarr_format": 3, + "array": { + "order": "C", + "write_empty_chunks": False, + "read_missing_chunks": True, + "target_shard_size_bytes": None, + "rectilinear_chunks": False, + "sharding_coalesce_max_gap_bytes": 1 << 20, + "sharding_coalesce_max_bytes": 16 << 20, + }, + "async": {"concurrency": 10, "timeout": None}, + "threading": {"max_workers": None}, + "json_indent": 2, + "codec_pipeline": { + "path": "zarr.core.codec_pipeline.BatchedCodecPipeline", + "batch_size": 1, + }, + "codecs": dict(DEFAULT_CODECS), + "buffer": "zarr.buffer.cpu.Buffer", + "ndbuffer": "zarr.buffer.cpu.NDBuffer", + } assert config.get("array.order") == "C" assert config.get("async.concurrency") == 10 assert config.get("async.timeout") is None @@ -156,9 +118,6 @@ def test_config_codec_pipeline_class(store: Store) -> None: # has default value assert get_pipeline_class().__name__ != "" - config.set({"codec_pipeline.name": "zarr.core.codec_pipeline.BatchedCodecPipeline"}) - assert get_pipeline_class() == zarr.core.codec_pipeline.BatchedCodecPipeline - _mock = Mock() class MockCodecPipeline(BatchedCodecPipeline): @@ -206,7 +165,7 @@ class MockEnvCodecPipeline(CodecPipeline): @pytest.mark.parametrize("store", ["local", "memory"], indirect=["store"]) def test_config_codec_implementation(store: Store) -> None: # has default value - assert fully_qualified_name(get_codec_class("blosc")) == config.defaults[0]["codecs"]["blosc"] + assert fully_qualified_name(get_codec_class("blosc")) == config.defaults["codecs"]["blosc"] _mock = Mock() @@ -259,7 +218,7 @@ def test_config_ndbuffer_implementation(store: Store) -> None: def test_config_buffer_implementation() -> None: # has default value - assert config.defaults[0]["buffer"] == "zarr.buffer.cpu.Buffer" + assert config.defaults["buffer"] == "zarr.buffer.cpu.Buffer" arr = zeros(shape=(100,), store=StoreExpectingTestBuffer()) diff --git a/tests/test_config_typed.py b/tests/test_config_typed.py new file mode 100644 index 0000000000..7bc79c566e --- /dev/null +++ b/tests/test_config_typed.py @@ -0,0 +1,920 @@ +from __future__ import annotations + +import copy +import dataclasses +import os +import pickle +import threading +import typing +from concurrent.futures import ThreadPoolExecutor + +import pytest + +from tests.conftest import Expect, ExpectFail +from zarr.core.config import ( + _SERIALIZED_NAMES, + DEFAULT_CODECS, + BadConfigError, + ZarrConfig, + ZarrConfigManager, + apply_overrides, + build_config, + get_path, + make_default_config, + replace_path, + to_nested_dict, +) + +if typing.TYPE_CHECKING: + import pathlib + +# --------------------------------------------------------------------------- +# Module-level constants used in parametrize lists (evaluated at collection time) +# --------------------------------------------------------------------------- + +_REMOVED_KEY = "array.v2_default_compressor.numeric" +_DEFAULT = make_default_config() + + +def _build_config_with_env(monkeypatch: pytest.MonkeyPatch, env: dict[str, str]) -> ZarrConfig: + """Run `build_config` under a controlled ``ZARR_*`` environment. + + `build_config` delegates env/YAML reading to donfig, which reads + ``os.environ``. This clears any ambient ``ZARR_*`` variables for determinism, + sets the requested ones, then builds. + """ + for name in list(os.environ): + if name.startswith("ZARR_"): + monkeypatch.delenv(name, raising=False) + for name, value in env.items(): + monkeypatch.setenv(name, value) + return build_config() + + +# --------------------------------------------------------------------------- +# 1. get_path — success cases +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + "case", + [ + Expect(input="array.order", output="C", id="array-order"), + Expect(input="async.concurrency", output=10, id="async-concurrency-alias"), + Expect(input="json_indent", output=2, id="json-indent"), + Expect(input="codecs", output=DEFAULT_CODECS, id="codecs-dict"), + Expect(input="codecs.blosc", output="zarr.codecs.blosc.BloscCodec", id="codecs-blosc"), + ], + ids=lambda c: c.id, +) +def test_get_path(case: Expect[str, object]) -> None: + assert get_path(make_default_config(), case.input) == case.output + + +@pytest.mark.parametrize( + "case", + [ + ExpectFail(input="array.nonexistent", exception=KeyError, id="nonexistent-key"), + ], + ids=lambda c: c.id, +) +def test_get_path_raises(case: ExpectFail[str]) -> None: + with case.raises(): + get_path(make_default_config(), case.input) + + +# --------------------------------------------------------------------------- +# 2. replace_path +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + "case", + [ + Expect(input=("array.order", "F"), output="F", id="array-order"), + Expect(input=("async.concurrency", 99), output=99, id="async-concurrency-alias"), + Expect( + input=("codecs.my_codec", "my.module.MyCodec"), + output="my.module.MyCodec", + id="codec-new-key", + ), + ], + ids=lambda c: c.id, +) +def test_replace_path(case: Expect[tuple[str, object], object]) -> None: + key, value = case.input + result = replace_path(make_default_config(), key, value) + assert get_path(result, key) == case.output + + +def test_replace_path_is_immutable() -> None: + """Original config is unchanged after replace_path (frozen dataclass).""" + cfg = make_default_config() + _ = replace_path(cfg, "array.order", "F") + assert cfg.array.order == "C" + # the open `codecs` dict must not be mutated in place either: a frozen + # dataclass forbids attribute re-assignment but not `dict.__setitem__`. + _ = replace_path(cfg, "codecs.my_codec", "my.module.MyCodec") + assert "my_codec" not in cfg.codecs + + +# --------------------------------------------------------------------------- +# 3. build_config — env ingest (donfig reads ZARR_* from the environment) +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + "case", + [ + Expect(input={}, output=_DEFAULT, id="empty-environ"), + Expect( + input={"ZARR_CONFIG": "/nonexistent/path.yaml"}, + output=_DEFAULT, + id="zarr-config-nonexistent", + ), + Expect( + input={"ZARR_JSON_INDENT": "4"}, + output=replace_path(_DEFAULT, "json_indent", 4), + id="json-indent-env", + ), + Expect( + input={ + "ZARR_ARRAY__ORDER": "F", + "ZARR_ASYNC__CONCURRENCY": "32", + "ZARR_CODECS__MY_CODEC": "my.module.MyCodec", + }, + output=replace_path( + replace_path( + replace_path(_DEFAULT, "array.order", "F"), + "async.concurrency", + 32, + ), + "codecs.my_codec", + "my.module.MyCodec", + ), + id="nested-literal-and-codecs", + ), + ], + ids=lambda c: c.id, +) +def test_build_config_env( + case: Expect[dict[str, str], ZarrConfig], monkeypatch: pytest.MonkeyPatch +) -> None: + """`ZARR_*` environment variables are read (via donfig) and applied onto the + typed defaults: dotted nesting (`ZARR_ARRAY__ORDER`), literal parsing + (`ZARR_ASYNC__CONCURRENCY=32` -> int), and the open `codecs` namespace all + work; a `ZARR_CONFIG` pointing nowhere is a no-op.""" + assert _build_config_with_env(monkeypatch, case.input) == case.output + + +# --------------------------------------------------------------------------- +# 5. apply_overrides +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + "case", + [ + Expect( + input={"array.order": "F", "codecs.x": "pkg.X"}, + output=replace_path(replace_path(_DEFAULT, "array.order", "F"), "codecs.x", "pkg.X"), + id="array-order-and-codec", + ), + ], + ids=lambda c: c.id, +) +def test_apply_overrides(case: Expect[dict[str, object], ZarrConfig]) -> None: + assert apply_overrides(make_default_config(), case.input) == case.output + + +# --------------------------------------------------------------------------- +# 6. to_nested_dict +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + "case", + [ + Expect( + input=make_default_config(), + output=("C", 10, "zarr.codecs.blosc.BloscCodec"), + id="default-serialized-keys", + ), + ], + ids=lambda c: c.id, +) +def test_to_nested_dict(case: Expect[ZarrConfig, tuple[str, int, str]]) -> None: + nested = to_nested_dict(case.input) + order, concurrency, blosc = case.output + assert nested["array"]["order"] == order + assert nested["async"]["concurrency"] == concurrency + assert "async_" not in nested # serialized key, not the Python attribute name + assert nested["codecs"]["blosc"] == blosc + + +# --------------------------------------------------------------------------- +# 7. ZarrConfigManager.get — proxy string access +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + "case", + [ + Expect(input="array.order", output="C", id="array-order"), + Expect(input="async.concurrency", output=10, id="async-concurrency-alias"), + Expect(input="codecs", output=DEFAULT_CODECS, id="codecs-dict"), + ], + ids=lambda c: c.id, +) +def test_proxy_get(case: Expect[str, object]) -> None: + assert ZarrConfigManager().get(case.input) == case.output + + +@pytest.mark.parametrize( + "case", + [ + Expect(input=("does.not.exist", "fallback"), output="fallback", id="default-fallback"), + ], + ids=lambda c: c.id, +) +def test_proxy_get_with_default(case: Expect[tuple[str, object], object]) -> None: + key, default = case.input + assert ZarrConfigManager().get(key, default) == case.output + + +# --------------------------------------------------------------------------- +# 8. Removed-deprecated-key behavior +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + "case", + [ + Expect(input="fallback", output="fallback", id="get-with-default"), + ], + ids=lambda c: c.id, +) +def test_removed_deprecated_key_get_default(case: Expect[str, str]) -> None: + """get() with a removed deprecated key and a default returns the default silently.""" + assert ZarrConfigManager().get(_REMOVED_KEY, case.input) == case.output + + +@pytest.mark.parametrize( + "case", + [ + ExpectFail(input=_REMOVED_KEY, exception=KeyError, id="get-no-default"), + ], + ids=lambda c: c.id, +) +def test_removed_deprecated_key_get_raises(case: ExpectFail[str]) -> None: + """get() with a removed deprecated key and no default raises KeyError.""" + with case.raises(): + ZarrConfigManager().get(case.input) + + +# --------------------------------------------------------------------------- +# 9. set() must raise for both removed-deprecated keys and totally unknown keys +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + "case", + [ + ExpectFail( + input={_REMOVED_KEY: "some_value"}, + exception=BadConfigError, + id="set-removed-deprecated", + ), + ExpectFail( + input={"totally.bogus.key": 1}, + exception=KeyError, + id="set-unknown-key", + ), + ], + ids=lambda c: c.id, +) +def test_set_invalid_key_raises(case: ExpectFail[dict[str, object]]) -> None: + """set() raises for both removed deprecated keys and totally unknown structured keys.""" + with case.raises(): + ZarrConfigManager().set(case.input) + + +# --------------------------------------------------------------------------- +# 10. Unknown keys produce a helpful "did you mean" message (get and set) +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + "case", + [ + # close match at the deepest resolvable level -> "Did you mean ...?" + ExpectFail( + input="arr4y", exception=KeyError, msg=r"Did you mean .array.", id="suggest-top" + ), + ExpectFail( + input="array.0rder", + exception=KeyError, + msg=r"Did you mean .array\.order.", + id="suggest-nested", + ), + ExpectFail( + input="codecs.bl0sc", + exception=KeyError, + msg=r"Did you mean .codecs\.blosc.", + id="suggest-codec", + ), + # no close match -> roster of available keys at the last resolvable level + ExpectFail(input="foo", exception=KeyError, msg=r"Valid keys: .*array", id="roster-top"), + ExpectFail( + input="array.foo", + exception=KeyError, + msg=r"Valid keys under .array.: .*order", + id="roster-nested", + ), + ExpectFail( + input="codecs.zzzzzzzz", + exception=KeyError, + msg=r"under .codecs.: .*more\)", + id="roster-truncated", + ), + ], + ids=lambda c: c.id, +) +def test_get_unknown_key_message(case: ExpectFail[str]) -> None: + """get() on an unknown key suggests the closest key or lists what's available.""" + with case.raises(): + ZarrConfigManager().get(case.input) + + +@pytest.mark.parametrize( + "case", + [ + ExpectFail( + input={"array.0rder": "F"}, + exception=KeyError, + msg=r"Did you mean .array\.order.", + id="set-suggest", + ), + ExpectFail( + input={"array.foo": "F"}, + exception=KeyError, + msg=r"Valid keys under .array.: .*order", + id="set-roster", + ), + ], + ids=lambda c: c.id, +) +def test_set_unknown_key_message(case: ExpectFail[dict[str, object]]) -> None: + """set() shares the same helpful unknown-key error as get().""" + with case.raises(): + ZarrConfigManager().set(case.input) + + +# --------------------------------------------------------------------------- +# Default config values (dedicated — direct attribute assertions are clearest here) +# --------------------------------------------------------------------------- + + +def test_default_config_values() -> None: + cfg = make_default_config() + assert cfg.default_zarr_format == 3 + assert cfg.array.order == "C" + assert cfg.array.sharding_coalesce_max_bytes == 16 << 20 + assert cfg.async_.concurrency == 10 + assert cfg.async_.timeout is None + assert cfg.threading.max_workers is None + assert cfg.json_indent == 2 + assert cfg.codec_pipeline.path == "zarr.core.codec_pipeline.BatchedCodecPipeline" + assert cfg.codecs["blosc"] == "zarr.codecs.blosc.BloscCodec" + assert cfg.codecs == DEFAULT_CODECS + # proxy attribute access via ZarrConfigManager + mgr = ZarrConfigManager() + assert mgr.array.order == "C" + + +# --------------------------------------------------------------------------- +# Stateful / behavioral tests (kept as dedicated functions) +# --------------------------------------------------------------------------- + + +def test_set_permanent_and_context() -> None: + cfg = ZarrConfigManager() + cfg.set({"array.order": "F"}) + assert cfg.get("array.order") == "F" # permanent + with cfg.set({"array.order": "C"}): + assert cfg.get("array.order") == "C" + assert cfg.get("array.order") == "F" # restored to permanent value + cfg.reset() + assert cfg.get("array.order") == "C" + + +def test_permanent_set_visible_in_worker_thread() -> None: + cfg = ZarrConfigManager() + cfg.set({"async.concurrency": 77}) + try: + with ThreadPoolExecutor(max_workers=1) as ex: + seen = ex.submit(lambda: cfg.get("async.concurrency")).result() + assert seen == 77 # a permanent set is on the shared global base + finally: + cfg.reset() + + +def test_permanent_set_cross_thread_last_writer_wins() -> None: + """A permanent `set` from any thread updates the shared global base, so a later + permanent `set` in another thread is visible everywhere — even after the first + thread already did its own permanent `set`. (Regression: the first set used to + pin the setting thread's view, so it kept seeing its own stale value.)""" + cfg = ZarrConfigManager() + cfg.set({"async.concurrency": 1}) + worker = threading.Thread(target=lambda: cfg.set({"async.concurrency": 999})) + worker.start() + worker.join() + assert cfg.get("async.concurrency") == 999 + + +def test_with_block_is_global_and_reverts_only_its_keys() -> None: + """A `with config.set(...)` applies globally (donfig semantics: it is NOT + isolated to the calling thread) and, on exit, reverts only the keys it set — + leaving a concurrent permanent `set` to a *different* key intact.""" + cfg = ZarrConfigManager() + cfg.set({"async.concurrency": 5}) + with cfg.set({"async.concurrency": 999}): + assert cfg.get("async.concurrency") == 999 # visible in this context + with ThreadPoolExecutor(max_workers=1) as ex: + seen = ex.submit(lambda: cfg.get("async.concurrency")).result() + assert seen == 999 # globally visible: the worker sees the override too + # a concurrent permanent set to a DIFFERENT key must survive block exit + cfg.set({"array.order": "F"}) + assert cfg.get("async.concurrency") == 5 # the block's key reverted + assert cfg.get("array.order") == "F" # the other key was not clobbered + + +def test_permanent_set_inside_with_block_persists() -> None: + """On `with config.set(...)` exit, only the block's own keys are reverted, so a + permanent `set` to a different key made inside the block persists.""" + cfg = ZarrConfigManager() + with cfg.set({"array.order": "F"}): + cfg.set({"async.concurrency": 5}) # permanent set to a different key + assert cfg.get("array.order") == "C" # the block's key reverted + assert cfg.get("async.concurrency") == 5 # the other key persisted + + +def test_with_block_removes_newly_added_codec_key_on_exit() -> None: + """A scoped `set` that *adds* a new codec key removes it again on block exit, + while a pre-existing key it overrode is restored to its prior value.""" + cfg = ZarrConfigManager() + assert "brand_new_codec" not in cfg.codecs + with cfg.set({"codecs.brand_new_codec": "pkg.New", "codecs.blosc": "pkg.OverrideBlosc"}): + assert cfg.codecs["brand_new_codec"] == "pkg.New" + assert cfg.codecs["blosc"] == "pkg.OverrideBlosc" + assert "brand_new_codec" not in cfg.codecs # newly added key removed + assert cfg.codecs["blosc"] == DEFAULT_CODECS["blosc"] # existing key restored + + +# --------------------------------------------------------------------------- +# Subtree item access (donfig back-compat for `config.get("array")["order"]`) +# --------------------------------------------------------------------------- + + +def test_subtree_get_item_access_matches_attribute() -> None: + """A subtree `get` returns a typed dataclass that also supports donfig-style + item access: `["order"]`, `.order`, and `get("array.order")` all agree, and + an unknown key raises `KeyError` like the old dicts did.""" + cfg = ZarrConfigManager() + array = cfg.get("array") + assert array["order"] == array.order == cfg.get("array.order") + assert array["sharding_coalesce_max_bytes"] == array.sharding_coalesce_max_bytes + with pytest.raises(KeyError): + array["does_not_exist"] + + +def test_config_node_dotted_and_alias_item_access() -> None: + """Item access on a config node resolves dotted keys, the `async` alias, and + the open `codecs` mapping, mirroring `get_path`.""" + cfg = make_default_config() + assert cfg["array.order"] == "C" + assert cfg["async.concurrency"] == 10 # the `async` alias resolves + assert cfg.async_["concurrency"] == cfg.async_.concurrency # node item access + assert cfg["codecs.bytes"] == DEFAULT_CODECS["bytes"] + + +def test_defaults_and_enable_gpu() -> None: + cfg = ZarrConfigManager() + assert cfg.defaults["array"]["order"] == "C" + with cfg.set({"buffer": "x"}): + pass + cfg.enable_gpu() + try: + assert cfg.get("buffer") == "zarr.buffer.gpu.Buffer" + assert cfg.get("ndbuffer") == "zarr.buffer.gpu.NDBuffer" + finally: + cfg.reset() + + +def test_refresh_not_shadowed_by_prior_scope(monkeypatch: pytest.MonkeyPatch) -> None: + """refresh() must be visible in the calling context even after a prior set()/reset().""" + mgr = ZarrConfigManager() + # apply a permanent override in this context, then rebuild over a changed env + mgr.set({"array.order": "F"}) + assert mgr.get("array.order") == "F" + # change the environment so a rebuild differs, then refresh + monkeypatch.setenv("ZARR_JSON_INDENT", "7") + mgr.refresh() + # refresh rebuilds the global base and must be visible in THIS context + assert mgr.get("json_indent") == 7 + assert mgr.get("array.order") == "C" # the prior permanent set is gone after rebuild + + +# --------------------------------------------------------------------------- +# Tolerant ingest: unknown env/YAML keys must warn and be skipped, not crash +# --------------------------------------------------------------------------- + + +def test_build_config_unknown_env_key_warns_and_skips(monkeypatch: pytest.MonkeyPatch) -> None: + """build_config with an unrecognized env var warns and skips it; known keys still apply.""" + with pytest.warns(UserWarning, match="future.key"): + cfg = _build_config_with_env( + monkeypatch, {"ZARR_FUTURE__KEY": "1", "ZARR_ARRAY__ORDER": "F"} + ) + # Known key was applied + assert cfg.array.order == "F" + # All other fields are still at default + default = make_default_config() + from dataclasses import fields as dc_fields + + for f in dc_fields(default): + if f.name != "array": + assert getattr(cfg, f.name) == getattr(default, f.name) + + +def test_apply_overrides_unknown_key_warns_and_returns_default() -> None: + """apply_overrides with a totally unknown key warns and returns an otherwise-default config.""" + default = make_default_config() + with pytest.warns(UserWarning, match="totally.bogus.key"): + result = apply_overrides(default, {"totally.bogus.key": 123}) + assert result == default + + +# --------------------------------------------------------------------------- +# donfig is used as the env/YAML reader +# --------------------------------------------------------------------------- + + +def test_donfig_is_used_for_ingest() -> None: + """donfig backs env/YAML ingest, so building a config imports it and its + reader produces a mapping we can consume.""" + import donfig + + assert isinstance(donfig.Config("zarr").config, dict) + # build_config runs donfig's reader and returns the typed representation + assert isinstance(build_config(), ZarrConfig) + + +# --------------------------------------------------------------------------- +# YAML config files (read by donfig, applied onto the typed defaults) +# --------------------------------------------------------------------------- + + +def test_yaml_codecs_block_merges_not_replaces( + tmp_path: pathlib.Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """A YAML file with a codecs: block must MERGE into the defaults, not replace them.""" + yaml_file = tmp_path / "zarr.yaml" + yaml_file.write_text("codecs:\n bytes: my.custom.BytesCodec\n mycodec: my.Mod.MyCodec\n") + cfg = _build_config_with_env(monkeypatch, {"ZARR_CONFIG": str(yaml_file)}) + # overrides applied + assert cfg.codecs["bytes"] == "my.custom.BytesCodec" + assert cfg.codecs["mycodec"] == "my.Mod.MyCodec" + # defaults PRESERVED + assert cfg.codecs["blosc"] == "zarr.codecs.blosc.BloscCodec" + assert cfg.codecs["zstd"] == "zarr.codecs.zstd.ZstdCodec" + # exactly one net-new key added ("bytes" overwrites existing; "mycodec" is new) + assert len(cfg.codecs) == len(DEFAULT_CODECS) + 1 + + +def test_yaml_dotted_codec_name_merges( + tmp_path: pathlib.Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """Dotted codec keys like numcodecs.bz2 in YAML must merge, not replace the whole dict.""" + yaml_file = tmp_path / "zarr.yaml" + yaml_file.write_text("codecs:\n numcodecs.bz2: my.Override\n") + cfg = _build_config_with_env(monkeypatch, {"ZARR_CONFIG": str(yaml_file)}) + # dotted key correctly round-tripped + assert cfg.codecs["numcodecs.bz2"] == "my.Override" + # all other defaults preserved + assert cfg.codecs["blosc"] == "zarr.codecs.blosc.BloscCodec" + assert len(cfg.codecs) == len(DEFAULT_CODECS) # bz2 was already there; just overwritten + + +def test_yaml_file_via_zarr_config_is_read( + tmp_path: pathlib.Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """A YAML file pointed to by ZARR_CONFIG is read; a non-existent path is a no-op.""" + yaml_file = tmp_path / "zarr.yaml" + yaml_file.write_text("json_indent: 9\n") + cfg = _build_config_with_env(monkeypatch, {"ZARR_CONFIG": str(yaml_file)}) + assert cfg.json_indent == 9 + # Non-existent path must still not raise, and leaves defaults intact + cfg2 = _build_config_with_env(monkeypatch, {"ZARR_CONFIG": "/nonexistent/path.yaml"}) + assert cfg2.json_indent == make_default_config().json_indent + + +def test_yaml_config_directory_is_scanned( + tmp_path: pathlib.Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """ZARR_CONFIG may point at a directory; a zarr.yaml inside it is loaded.""" + (tmp_path / "zarr.yaml").write_text("array:\n order: F\njson_indent: 8\n") + cfg = _build_config_with_env(monkeypatch, {"ZARR_CONFIG": str(tmp_path)}) + assert cfg.array.order == "F" + assert cfg.json_indent == 8 + + +def test_env_var_overrides_yaml(tmp_path: pathlib.Path, monkeypatch: pytest.MonkeyPatch) -> None: + """Precedence is defaults < YAML < environment: an env var wins over a YAML file.""" + yaml_file = tmp_path / "zarr.yaml" + yaml_file.write_text("json_indent: 3\n") + cfg = _build_config_with_env( + monkeypatch, {"ZARR_CONFIG": str(yaml_file), "ZARR_JSON_INDENT": "9"} + ) + assert cfg.json_indent == 9 + + +def test_yaml_unknown_key_warns_and_skips( + tmp_path: pathlib.Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """An unrecognized key in a YAML file warns and is skipped; known keys still apply + (so a version-skewed config file can't prevent `import zarr`).""" + yaml_file = tmp_path / "zarr.yaml" + yaml_file.write_text("future_key: 1\narray:\n order: F\n") + with pytest.warns(UserWarning, match="future_key"): + cfg = _build_config_with_env(monkeypatch, {"ZARR_CONFIG": str(yaml_file)}) + assert cfg.array.order == "F" + + +def test_discovers_home_config_dir(tmp_path: pathlib.Path, monkeypatch: pytest.MonkeyPatch) -> None: + """A `zarr.yaml` in `~/.config/zarr` is discovered (donfig's primary location), + and `ZARR_CONFIG` takes precedence over it while home-only keys still apply. + + Regression guard against silently dropping donfig's config-file locations. + """ + home = tmp_path / "home" + config_dir = home / ".config" / "zarr" + config_dir.mkdir(parents=True) + (config_dir / "zarr.yaml").write_text("json_indent: 1\narray:\n order: F\n") + # Redirect `~` to the temporary home so the real user config is not consulted; + # donfig computes its `~/.config/zarr` path via os.path.expanduser. + monkeypatch.setattr(os.path, "expanduser", lambda p: str(home) if p == "~" else p) + + # Without ZARR_CONFIG, the home config directory is picked up. + cfg = _build_config_with_env(monkeypatch, {}) + assert cfg.json_indent == 1 + assert cfg.array.order == "F" + + # ZARR_CONFIG (highest precedence) overrides the home file's overlapping keys. + override = tmp_path / "override.yaml" + override.write_text("json_indent: 2\n") + cfg2 = _build_config_with_env(monkeypatch, {"ZARR_CONFIG": str(override)}) + assert cfg2.json_indent == 2 # ZARR_CONFIG wins + assert cfg2.array.order == "F" # home-only key still applies + + +# --------------------------------------------------------------------------- +# Drift-protection: every structured leaf key must have a get() overload +# --------------------------------------------------------------------------- + + +def _structured_leaf_specs(cfg_cls: type, prefix: str = "") -> dict[str, object]: + """Walk a settings dataclass recursively and return ``{dotted_key: resolved_type}``. + + Uses ``typing.get_type_hints`` instead of ``f.type`` so that the + ``from __future__ import annotations`` string-annotation form is resolved + to real types before ``dataclasses.is_dataclass`` is called. The open + ``codecs`` mapping is intentionally excluded. + """ + specs: dict[str, object] = {} + resolved_hints = typing.get_type_hints(cfg_cls) + for f in dataclasses.fields(cfg_cls): + serialized = _SERIALIZED_NAMES.get(f.name, f.name) + key = f"{prefix}.{serialized}" if prefix else serialized + resolved_type = resolved_hints[f.name] + if dataclasses.is_dataclass(resolved_type): + specs.update(_structured_leaf_specs(typing.cast(type, resolved_type), key)) + elif f.name == "codecs": + # open mapping — intentionally not enumerated + continue + else: + specs[key] = resolved_type + return specs + + +def _structured_leaf_keys(cfg_cls: type, prefix: str = "") -> list[str]: + """Return every dotted leaf key for a settings dataclass (derived from specs).""" + return list(_structured_leaf_specs(cfg_cls, prefix)) + + +def test_every_structured_key_has_a_get_overload() -> None: + """Enumerate every typed leaf key in ZarrConfig and assert a matching get() overload exists.""" + overloads = typing.get_overloads(ZarrConfigManager.get) + literal_keys: set[str] = set() + for ov in overloads: + hints = typing.get_type_hints(ov) + key_hint = hints.get("key") + if typing.get_origin(key_hint) is typing.Literal: + literal_keys.update(typing.get_args(key_hint)) + leaf_keys = _structured_leaf_keys(ZarrConfig) + missing = set(leaf_keys) - literal_keys + assert not missing, f"get() overloads missing for: {sorted(missing)}" + + +def test_get_overload_return_types_match_fields() -> None: + """Assert that each get() overload's return type matches the dataclass field type. + + Builds two maps using ``typing.get_type_hints`` — one from the dataclass + field annotations, one from the overload return hints — then compares them + key by key. A mismatch (e.g. ``-> str`` instead of ``-> Literal["C","F"]``) + is reported as a clear failure rather than a missing-overload failure. + """ + # Build map: key -> return type from overloads + overloads = typing.get_overloads(ZarrConfigManager.get) + overload_return: dict[str, object] = {} + for ov in overloads: + hints = typing.get_type_hints(ov) + key_hint = hints.get("key") + if typing.get_origin(key_hint) is typing.Literal: + (literal_val,) = typing.get_args(key_hint) + overload_return[literal_val] = hints["return"] + + # Build map: key -> field type from the dataclass schema + field_specs = _structured_leaf_specs(ZarrConfig) + + missing: list[str] = [] + mismatched: list[str] = [] + for key, expected_type in field_specs.items(): + if key not in overload_return: + missing.append(f" {key!r}: missing overload") + elif overload_return[key] != expected_type: + mismatched.append( + f" {key!r}: overload returns {overload_return[key]!r}," + f" field type is {expected_type!r}" + ) + + errors: list[str] = [] + if missing: + errors.append("get() overloads missing for keys:\n" + "\n".join(missing)) + if mismatched: + errors.append( + "get() overload return types do not match field types:\n" + "\n".join(mismatched) + ) + assert not errors, "\n\n".join(errors) + + +# --------------------------------------------------------------------------- +# Regression tests for the #4101 review +# --------------------------------------------------------------------------- + + +def test_env_codec_override_canonicalizes_hyphenated_name( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """`ZARR_CODECS__VLEN_UTF8` must override the hyphenated default `vlen-utf8`. + + Environment variables can't contain hyphens, so the flattened key is + `codecs.vlen_utf8`; without canonicalization it lands under a dead key while + the registry keeps reading the untouched `vlen-utf8` default. + """ + cfg = _build_config_with_env(monkeypatch, {"ZARR_CODECS__VLEN_UTF8": "my.Override"}) + assert cfg.codecs["vlen-utf8"] == "my.Override" + assert "vlen_utf8" not in cfg.codecs # no dead underscore key + # underscore-named defaults (e.g. sharding_indexed) are untouched + cfg2 = _build_config_with_env(monkeypatch, {"ZARR_CODECS__SHARDING_INDEXED": "my.Shard"}) + assert cfg2.codecs["sharding_indexed"] == "my.Shard" + # a brand-new codec name with underscores stays as written + cfg3 = _build_config_with_env(monkeypatch, {"ZARR_CODECS__MY_NEW": "my.New"}) + assert cfg3.codecs["my_new"] == "my.New" + + +@pytest.mark.parametrize("key", ["array.order.upper", "default_zarr_format.numerator"]) +def test_get_does_not_descend_into_scalar_attributes(key: str) -> None: + """A dotted key that walks past a scalar leaf must raise, not resolve a stray + Python attribute (e.g. `str.upper` / `int.numerator`).""" + with pytest.raises(KeyError): + ZarrConfigManager().get(key) + + +def test_set_rejects_descending_into_scalar() -> None: + with pytest.raises(KeyError): + ZarrConfigManager().set({"array.order.upper": "X"}) + + +def test_codecs_public_mapping_is_read_only() -> None: + """`config.codecs` exposes a read-only view, so a live snapshot can't be + mutated in place; a mutable copy is still available via `dict(...)`.""" + cfg = ZarrConfigManager() + with pytest.raises(TypeError): + cfg.codecs["blosc"] = "x" # type: ignore[index] + assert dict(cfg.codecs)["blosc"] == cfg.codecs["blosc"] + + +def test_config_snapshot_is_picklable_and_deepcopyable() -> None: + """A `ZarrConfig` snapshot must stay picklable / deep-copyable: the `codecs` + field is a plain dict, and only the public view is a read-only proxy.""" + cfg = replace_path(make_default_config(), "codecs.x", "pkg.X") + assert pickle.loads(pickle.dumps(cfg)) == cfg + assert copy.deepcopy(cfg) == cfg + + +def test_set_structured_subtree_dict_is_rejected() -> None: + """Assigning a dict to a whole structured subtree is rejected (it would drop + sibling fields and break attribute access); leaf keys must be used instead.""" + cfg = ZarrConfigManager() + with pytest.raises(TypeError): + cfg.set({"array": {"order": "F"}}) + # the leaf-key form works and preserves siblings + cfg.set({"array.order": "F"}) + assert cfg.get("array.order") == "F" + assert cfg.get("array.write_empty_chunks") is False + + +def test_manager_is_not_iterable() -> None: + """Iterating the manager raises a clear TypeError rather than falling into the + legacy integer-index protocol (which gave a confusing error).""" + with pytest.raises(TypeError): + list(ZarrConfigManager()) + + +def test_subtree_node_supports_mapping_style_reads() -> None: + """A subtree returned by `get` supports donfig-style `in`, iteration, `keys`, + and `dict()` alongside attribute and item access.""" + array = ZarrConfigManager().get("array") + assert "order" in array + assert "bogus" not in array + assert "order" in list(array) + assert "order" in set(array.keys()) + assert dict(array)["order"] == array.order == "C" + assert len(array) == len(dataclasses.fields(array)) + + +def test_manager_item_and_membership_access() -> None: + """donfig-style `config["k"]` and `"k" in config` mirror `get`.""" + cfg = ZarrConfigManager() + assert cfg["array.order"] == cfg.get("array.order") + assert "array.order" in cfg + assert "bogus.key" not in cfg + assert 123 not in cfg # non-string key is absent, not an error + + +def test_config_alias_preserved() -> None: + """`Config` remains importable as an alias of `ZarrConfigManager` (pre-typed + imports and `isinstance` checks keep working).""" + from zarr.core.config import Config + + assert Config is ZarrConfigManager + assert isinstance(ZarrConfigManager(), Config) + + +def test_concurrent_permanent_sets_to_distinct_keys_all_survive() -> None: + """A permanent `set` rebuilds the whole snapshot from `_base`; the manager + locks that read-modify-write so concurrent sets to distinct keys don't lose + updates.""" + cfg = ZarrConfigManager() + n = 64 + barrier = threading.Barrier(n) + + def worker(i: int) -> None: + barrier.wait() + cfg.set({f"codecs.k{i}": f"v{i}"}) + + with ThreadPoolExecutor(max_workers=n) as ex: + list(ex.map(worker, range(n))) + + codecs = cfg.get("codecs") + assert all(codecs.get(f"k{i}") == f"v{i}" for i in range(n)) + + +# --------------------------------------------------------------------------- +# Static-typing smoke test (only checked by mypy, not executed at runtime) +# --------------------------------------------------------------------------- + +if typing.TYPE_CHECKING: + + def _typing_smoke(cfg: ZarrConfigManager) -> None: + # --- positive assertions: each distinct return shape --- + typing.assert_type(cfg.get("array.order"), typing.Literal["C", "F"]) + typing.assert_type(cfg.get("async.concurrency"), int) + typing.assert_type(cfg.get("array.write_empty_chunks"), bool) + typing.assert_type(cfg.get("async.timeout"), float | None) + typing.assert_type(cfg.get("threading.max_workers"), int | None) + typing.assert_type(cfg.get("default_zarr_format"), typing.Literal[2, 3]) + typing.assert_type(cfg.get("buffer"), str) + typing.assert_type(cfg.array.order, typing.Literal["C", "F"]) + + # --- negative: precision-from-above guards --- + # The return type is Literal["C","F"], which is narrower than str. + # If the overload were widened to -> str, assert_type would pass and + # the ignore below would become unused, causing warn_unused_ignores to + # fail CI. + typing.assert_type(cfg.get("array.order"), str) # type: ignore[assert-type] + typing.assert_type(cfg.get("default_zarr_format"), int) # type: ignore[assert-type] + + # --- negative: bad key type must be rejected by all overloads --- + cfg.get(123) # type: ignore[call-overload]