diff --git a/.github/workflows/zarr-metadata.yml b/.github/workflows/zarr-metadata.yml index 95e8251227..e660483220 100644 --- a/.github/workflows/zarr-metadata.yml +++ b/.github/workflows/zarr-metadata.yml @@ -82,7 +82,10 @@ jobs: - name: Sync test dependency group run: uv sync --group test --python 3.11 - name: Run pyright - run: uv run --group test --with pyright pyright src + # Pinned to the last version that types PEP 661 sentinels in class + # attributes correctly; 1.1.405+ regressed (microsoft/pyright#11115). + # Unpin when the fix lands. + run: uv run --group test --with 'pyright==1.1.404' pyright src zarr-metadata-complete: name: zarr-metadata complete diff --git a/packages/zarr-metadata/changes/210.feature.md b/packages/zarr-metadata/changes/210.feature.md new file mode 100644 index 0000000000..f711d46af5 --- /dev/null +++ b/packages/zarr-metadata/changes/210.feature.md @@ -0,0 +1,69 @@ +Added `zarr_metadata.model`: frozen-dataclass models (`ArrayMetadataModelV2`, +`ArrayMetadataModelV3`, `GroupMetadataModelV2`, `GroupMetadataModelV3`, +`ConsolidatedMetadataModelV2`, `ConsolidatedMetadataModelV3`, `NamedConfigModelV3`) +that are canonical, lossless representations of Zarr metadata documents, plus +structural validators (`validate_*` / `is_*` / `parse_*`). Every v3 extension +point (data type, chunk grid, chunk key encoding, codecs, storage transformers) +is held as a name + configuration pair; nothing is interpreted. Model fields +are annotated with the role alias `MetadataFieldModelV3` (today exactly +`NamedConfigModelV3`), so the annotations convey the logical meaning of the +field and stay put if the spec ever adds a new field form. + +Validation is strict about what the types declare: v2 `dtype` / `order` / +`compressor` / `filters` / `dimension_separator` shapes and the fixed +`zarr_format` / `node_type` literals are all enforced. Every +`ValidationProblem` carries a machine-readable `kind` +(`missing_key` / `invalid_type` / `invalid_value` / `invalid_json`) so +consumers can dispatch on the failure mode without matching message strings, +and every ingestion failure — including missing store keys and undecodable +bytes in `from_key_value` — surfaces as `MetadataValidationError`. An +adversarial review added further structural checks: JSON booleans are not +accepted as dimension lengths, dimensions are non-negative, +`dimension_names` must have one entry per dimension of `shape`, `attributes` +and `configuration` values are JSON-checked recursively (like `fill_value`), +and the inline consolidated-metadata envelope and entries are deep-validated +so the group validator's verdict always agrees with the model constructor. + +The v3 models expose `must_understand_fields`: the subset of `extra_fields` +not explicitly waived with `must_understand: false` (fields are implicitly +must-understand per the spec). Readers discharge the spec's fail-to-open +duty by subtracting the extension names they recognize; the model only +partitions by obligation, since recognition is reader-specific. + +Optional pydantic integration ships as `zarr_metadata.pydantic` (importing it +requires pydantic v2; the core package does not depend on it): one `Annotated` +field type per model, validating raw documents through `from_json`, passing +core-model instances through unchanged, and serializing via `to_json`. The +instances are the core model classes, so values interoperate freely with +non-pydantic code. + +`create_default` keeps its output self-consistent: overriding `shape` without +a chunk grid derives one regular chunk covering the array (v3 +`chunk_shape == shape`; v2 `chunks == shape`) instead of silently keeping the +scalar default's 0-d grid. + +A v2 `.zarray` that omits `dimension_separator` is interpreted with the v2 +convention's default `"."` (the model previously normalized absence to `"/"`, +which would misaddress the chunks of real-world default-separator arrays). +The value is never null: absent, `"."`, or `"/"` are the only spellings. + +Optional document keys use `UNSET` — a PEP 661 sentinel +(`typing_extensions.Sentinel`), usable directly in type expressions — never +`None`: in a model, `None` always corresponds to a JSON `null` in the +document (a v2 `compressor`, an unnamed dimension inside `dimension_names`), +and `UNSET` always means the key is absent. Checker note: ty types the +sentinel exactly; pyright needs `<= 1.1.404` until microsoft/pyright#11115 +is fixed (this package's CI pins it); mypy users need a `cast` or +`type: ignore` at narrowing sites until python/mypy#21647 merges. This keeps semantically distinct spellings +distinct — an absent `dimension_names` ("there are no dimension names") and +an explicit `[null, null]` ("every dimension has a name, which is null") are +different documents and round-trip as such. The `consolidated_metadata: null` +written by a historical zarr-python bug is the one deliberate exception to +faithful round-tripping: those stores remain readable, but the bug spelling +is repaired to absence on read and never written back. + +The v2 models treat the `.zattrs` file's presence as part of the store: +`attributes` is `UNSET` when no `.zattrs` file exists (and `to_key_value` +emits none), while an explicit empty `.zattrs` is `{}` and round-trips as a +file. Previously `to_key_value` always emitted `.zattrs`, silently adding a +file to stores that never had one. diff --git a/packages/zarr-metadata/pyproject.toml b/packages/zarr-metadata/pyproject.toml index 05667d59e3..87f35db1db 100644 --- a/packages/zarr-metadata/pyproject.toml +++ b/packages/zarr-metadata/pyproject.toml @@ -32,7 +32,7 @@ classifiers = [ ] keywords = ["zarr"] dependencies = [ - "typing_extensions>=4.13", + "typing_extensions>=4.14", ] [project.urls] @@ -82,6 +82,9 @@ checks = [ "PR06", ] +# CI pins pyright==1.1.404: later versions regress PEP 661 sentinel typing in +# class attributes (microsoft/pyright#11115), which zarr_metadata.model._sentinel +# relies on. Use the same pin locally; unpin when the fix lands. [tool.pyright] include = ["src"] enableExperimentalFeatures = true diff --git a/packages/zarr-metadata/src/zarr_metadata/__init__.py b/packages/zarr-metadata/src/zarr_metadata/__init__.py index 46949570a2..4c1328bfe1 100644 --- a/packages/zarr-metadata/src/zarr_metadata/__init__.py +++ b/packages/zarr-metadata/src/zarr_metadata/__init__.py @@ -1,6 +1,24 @@ from importlib.metadata import version from zarr_metadata._common import JSONValue, NamedConfigV3 +from zarr_metadata.model import ( + UNSET, + ArrayMetadataModelV2, + ArrayMetadataModelV2Partial, + ArrayMetadataModelV3, + ArrayMetadataModelV3Partial, + ConsolidatedMetadataModelV2, + ConsolidatedMetadataModelV3, + GroupMetadataModelV2, + GroupMetadataModelV2Partial, + GroupMetadataModelV3, + GroupMetadataModelV3Partial, + MetadataFieldModelV3, + MetadataValidationError, + NamedConfigModelV3, + ProblemKind, + ValidationProblem, +) from zarr_metadata.v2.array import ( ARRAY_DIMENSION_SEPARATOR_V2, ARRAY_ORDER_V2, @@ -231,10 +249,15 @@ "UINT16_DATA_TYPE_NAME", "UINT32_DATA_TYPE_NAME", "UINT64_DATA_TYPE_NAME", + "UNSET", "V2_CHUNK_KEY_ENCODING_NAME", "V2_CHUNK_KEY_ENCODING_SEPARATOR", "ZSTD_CODEC_NAME", "ArrayDimensionSeparatorV2", + "ArrayMetadataModelV2", + "ArrayMetadataModelV2Partial", + "ArrayMetadataModelV3", + "ArrayMetadataModelV3Partial", "ArrayMetadataV2", "ArrayMetadataV2Partial", "ArrayMetadataV3", @@ -259,6 +282,8 @@ "Complex64FillValue", "Complex128DataTypeName", "Complex128FillValue", + "ConsolidatedMetadataModelV2", + "ConsolidatedMetadataModelV3", "ConsolidatedMetadataV2", "ConsolidatedMetadataV3", "Crc32cCodecMetadata", @@ -275,6 +300,10 @@ "Float32FillValue", "Float64DataTypeName", "Float64FillValue", + "GroupMetadataModelV2", + "GroupMetadataModelV2Partial", + "GroupMetadataModelV3", + "GroupMetadataModelV3Partial", "GroupMetadataV2", "GroupMetadataV2Partial", "GroupMetadataV3", @@ -290,13 +319,17 @@ "Int64DataTypeName", "Int64FillValue", "JSONValue", + "MetadataFieldModelV3", "MetadataV3", + "MetadataValidationError", + "NamedConfigModelV3", "NamedConfigV3", "NumpyDatetime64DataTypeName", "NumpyDatetime64FillValue", "NumpyTimeUnit", "NumpyTimedelta64DataTypeName", "NumpyTimedelta64FillValue", + "ProblemKind", "RawBytesDataTypeName", "RawBytesFillValue", "RectilinearChunkGridMetadata", @@ -325,6 +358,7 @@ "V2ChunkKeyEncodingMetadata", "V2ChunkKeyEncodingName", "V2ChunkKeyEncodingSeparator", + "ValidationProblem", "ZArrayMetadata", "ZAttrsMetadata", "ZGroupMetadata", diff --git a/packages/zarr-metadata/src/zarr_metadata/_common.py b/packages/zarr-metadata/src/zarr_metadata/_common.py index 598a12e80c..b335cd0bd6 100644 --- a/packages/zarr-metadata/src/zarr_metadata/_common.py +++ b/packages/zarr-metadata/src/zarr_metadata/_common.py @@ -13,7 +13,7 @@ JSONValue = TypeAliasType( "JSONValue", - "int | float | bool | None | str | list[JSONValue] | tuple[JSONValue, ...] | Mapping[str, JSONValue]", # type: ignore[reportInvalidTypeForm] + "int | float | bool | None | str | list[JSONValue] | tuple[JSONValue, ...] | Mapping[str, JSONValue]", ) """A recursive type alias for JSON-encodable values. diff --git a/packages/zarr-metadata/src/zarr_metadata/model/__init__.py b/packages/zarr-metadata/src/zarr_metadata/model/__init__.py new file mode 100644 index 0000000000..1257109465 --- /dev/null +++ b/packages/zarr-metadata/src/zarr_metadata/model/__init__.py @@ -0,0 +1,120 @@ +"""In-memory models for Zarr metadata documents. + +Models are frozen dataclasses that hold a canonical, lossless representation +of the JSON documents; they never interpret extension points (codecs, chunk +grids, data types). Validators check JSON structure, not domain validity. +Each document concept gets a `validate_*` function returning every problem +found (a `list[ValidationProblem]`, each with a machine-readable `kind`), an +`is_*` type guard, and a `parse_*` function that narrows or raises +`MetadataValidationError`. Model `from_json` / `from_key_value` constructors +raise `MetadataValidationError` for every ingestion failure, including +missing store keys and undecodable bytes. +""" + +from zarr_metadata.model._array import ( + ARRAY_METADATA_STORE_KEY_V2, + ARRAY_METADATA_STORE_KEY_V3, + ATTRIBUTES_STORE_KEY_V2, + ArrayMetadataModelV2, + ArrayMetadataModelV2Partial, + ArrayMetadataModelV3, + ArrayMetadataModelV3Partial, + MetadataFieldModelV3, + NamedConfigModelV3, +) +from zarr_metadata.model._group import ( + CONSOLIDATED_METADATA_KEY_V3, + CONSOLIDATED_METADATA_STORE_KEY_V2, + GROUP_METADATA_STORE_KEY_V2, + GROUP_METADATA_STORE_KEY_V3, + ConsolidatedMetadataModelV2, + ConsolidatedMetadataModelV3, + GroupMetadataModelV2, + GroupMetadataModelV2Partial, + GroupMetadataModelV3, + GroupMetadataModelV3Partial, +) +from zarr_metadata.model._sentinel import UNSET +from zarr_metadata.model._validation import ( + ARRAY_METADATA_OPTIONAL_KEYS_V3, + ARRAY_METADATA_REQUIRED_KEYS_V2, + ARRAY_METADATA_REQUIRED_KEYS_V3, + ARRAY_METADATA_STANDARD_KEYS_V3, + GROUP_METADATA_OPTIONAL_KEYS_V3, + GROUP_METADATA_REQUIRED_KEYS_V2, + GROUP_METADATA_REQUIRED_KEYS_V3, + GROUP_METADATA_STANDARD_KEYS_V3, + MetadataValidationError, + ProblemKind, + ValidationProblem, + is_array_metadata_v2, + is_array_metadata_v3, + is_group_metadata_v2, + is_group_metadata_v3, + is_json, + is_metadata_field_v3, + parse_array_metadata_v2, + parse_array_metadata_v3, + parse_group_metadata_v2, + parse_group_metadata_v3, + parse_json, + parse_metadata_field_v3, + validate_array_metadata_v2, + validate_array_metadata_v3, + validate_group_metadata_v2, + validate_group_metadata_v3, + validate_json, + validate_metadata_field_v3, +) + +__all__ = [ + "ARRAY_METADATA_OPTIONAL_KEYS_V3", + "ARRAY_METADATA_REQUIRED_KEYS_V2", + "ARRAY_METADATA_REQUIRED_KEYS_V3", + "ARRAY_METADATA_STANDARD_KEYS_V3", + "ARRAY_METADATA_STORE_KEY_V2", + "ARRAY_METADATA_STORE_KEY_V3", + "ATTRIBUTES_STORE_KEY_V2", + "CONSOLIDATED_METADATA_KEY_V3", + "CONSOLIDATED_METADATA_STORE_KEY_V2", + "GROUP_METADATA_OPTIONAL_KEYS_V3", + "GROUP_METADATA_REQUIRED_KEYS_V2", + "GROUP_METADATA_REQUIRED_KEYS_V3", + "GROUP_METADATA_STANDARD_KEYS_V3", + "GROUP_METADATA_STORE_KEY_V2", + "GROUP_METADATA_STORE_KEY_V3", + "UNSET", + "ArrayMetadataModelV2", + "ArrayMetadataModelV2Partial", + "ArrayMetadataModelV3", + "ArrayMetadataModelV3Partial", + "ConsolidatedMetadataModelV2", + "ConsolidatedMetadataModelV3", + "GroupMetadataModelV2", + "GroupMetadataModelV2Partial", + "GroupMetadataModelV3", + "GroupMetadataModelV3Partial", + "MetadataFieldModelV3", + "MetadataValidationError", + "NamedConfigModelV3", + "ProblemKind", + "ValidationProblem", + "is_array_metadata_v2", + "is_array_metadata_v3", + "is_group_metadata_v2", + "is_group_metadata_v3", + "is_json", + "is_metadata_field_v3", + "parse_array_metadata_v2", + "parse_array_metadata_v3", + "parse_group_metadata_v2", + "parse_group_metadata_v3", + "parse_json", + "parse_metadata_field_v3", + "validate_array_metadata_v2", + "validate_array_metadata_v3", + "validate_group_metadata_v2", + "validate_group_metadata_v3", + "validate_json", + "validate_metadata_field_v3", +] diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_array.py b/packages/zarr-metadata/src/zarr_metadata/model/_array.py new file mode 100644 index 0000000000..f68c1b0f5e --- /dev/null +++ b/packages/zarr-metadata/src/zarr_metadata/model/_array.py @@ -0,0 +1,466 @@ +"""In-memory models for Zarr array metadata documents.""" + +from __future__ import annotations + +import dataclasses +import json +from collections.abc import Mapping +from dataclasses import dataclass, field +from typing import TYPE_CHECKING, Final, Literal, TypeAlias, cast + +from typing_extensions import TypedDict, Unpack + +from zarr_metadata.model._sentinel import UNSET +from zarr_metadata.model._validation import ( + ARRAY_METADATA_STANDARD_KEYS_V3, + MetadataValidationError, + ValidationProblem, + arrays_to_tuples, + load_store_json, + parse_array_metadata_v2, + parse_array_metadata_v3, + parse_metadata_field_v3, +) + +if TYPE_CHECKING: + from zarr_metadata._common import JSONValue + from zarr_metadata.v2.array import ( + ArrayDimensionSeparatorV2, + ArrayMetadataV2, + ArrayOrderV2, + DataTypeMetadataV2, + ) + from zarr_metadata.v2.codec import CodecMetadataV2 + from zarr_metadata.v3._common import MetadataV3 + from zarr_metadata.v3.array import ArrayMetadataV3, ExtensionFieldV3 + +ArrayMetadataStoreKeyV3 = Literal["zarr.json"] +ARRAY_METADATA_STORE_KEY_V3: Final[ArrayMetadataStoreKeyV3] = "zarr.json" + +ArrayMetadataStoreKeyV2 = Literal[".zarray"] +ARRAY_METADATA_STORE_KEY_V2: Final[ArrayMetadataStoreKeyV2] = ".zarray" + +AttributesStoreKeyV2 = Literal[".zattrs"] +ATTRIBUTES_STORE_KEY_V2: Final[AttributesStoreKeyV2] = ".zattrs" + + +@dataclass(frozen=True, slots=True, kw_only=True) +class NamedConfigModelV3: + """A v3 metadata field in normalized form: a name plus a configuration. + + This is the in-memory model of `MetadataV3` (a bare name string or a + `{name, configuration}` mapping): the bare-name and missing-configuration + forms normalize to an empty configuration. + """ + + name: str + configuration: dict[str, JSONValue] + + def to_json(self) -> MetadataV3: + return {"name": self.name, "configuration": self.configuration} + + @classmethod + def from_json(cls, data: object) -> NamedConfigModelV3: + field = parse_metadata_field_v3(data) + if isinstance(field, str): + return cls(name=field, configuration={}) + # Sound cast: parse_metadata_field_v3 checked the configuration is a + # string-keyed mapping of JSON values; arrays_to_tuples only converts + # lists to tuples within that shape. + configuration = cast( + "dict[str, JSONValue]", arrays_to_tuples(dict(field.get("configuration", {}))) + ) + return cls(name=field["name"], configuration=configuration) + + +MetadataFieldModelV3: TypeAlias = NamedConfigModelV3 +"""The in-memory model of one field of a v3 metadata document. + +This is the role-named alias for annotation positions: model fields and +consumer signatures should say `MetadataFieldModelV3` (the logical meaning) +rather than `NamedConfigModelV3` (the serialized form the field currently +takes). Today every metadata field normalizes to a named configuration, so +the alias is exactly `NamedConfigModelV3`; if a future spec revision adds a +field form that cannot be normalized to name + configuration, this alias +widens to a union and annotation sites do not change. Mirrors the raw-layer +split between `NamedConfigV3` (shape) and `MetadataV3` (field union). +""" + + +def must_understand_subset( + extra_fields: Mapping[str, ExtensionFieldV3], +) -> dict[str, ExtensionFieldV3]: + """The subset of `extra_fields` the reader is obligated to understand. + + Per the v3 spec, an extension field is implicitly `must_understand: True` + unless it explicitly says otherwise, and an implementation MUST fail to + open a group or array carrying fields it does not recognize that are not + explicitly `must_understand: false`. A non-mapping field value cannot + carry the explicit waiver, so it always requires understanding (the + runtime isinstance check defends against values looser than the declared + `ExtensionFieldV3`). + """ + fields = cast("Mapping[str, object]", extra_fields) + return cast( + "dict[str, ExtensionFieldV3]", + { + name: value + for name, value in fields.items() + if not ( + isinstance(value, Mapping) + and cast("Mapping[str, object]", value).get("must_understand") is False + ) + }, + ) + + +class ArrayMetadataModelV3Partial(TypedDict, total=False): + """ + Partial form of the constructor-settable fields of `ArrayMetadataModelV3`. + + Every key is optional and typed with the model's own (not serialized) + value types, so it describes valid keyword arguments to + `ArrayMetadataModelV3.update`. The `init=False` fields `zarr_format` and + `node_type` are intentionally excluded, since they cannot be passed to + `dataclasses.replace`. + + Drift between this type and the model's settable fields is prevented by + `tests/model/test_array.py::test_partial_keys_match_settable_model_fields`. + """ + + shape: tuple[int, ...] + fill_value: JSONValue + data_type: MetadataFieldModelV3 + chunk_grid: MetadataFieldModelV3 + codecs: tuple[MetadataFieldModelV3, ...] + chunk_key_encoding: MetadataFieldModelV3 + dimension_names: tuple[str | None, ...] | UNSET + attributes: dict[str, JSONValue] + storage_transformers: tuple[MetadataFieldModelV3, ...] + extra_fields: dict[str, ExtensionFieldV3] + + +@dataclass(frozen=True, slots=True, kw_only=True) +class ArrayMetadataModelV3: + """In-memory model of a v3 array metadata document. + + A canonical, lossless representation of the `zarr.json` content for an + array. Extension points (`data_type`, `chunk_grid`, `chunk_key_encoding`, + `codecs`, `storage_transformers`) are held as `MetadataFieldModelV3` + values (currently always `NamedConfigModelV3` name + configuration pairs) + and are never interpreted; `fill_value` is held + verbatim in its JSON form. + """ + + zarr_format: Literal[3] = field(default=3, init=False) + node_type: Literal["array"] = field(default="array", init=False) + shape: tuple[int, ...] + fill_value: JSONValue + data_type: MetadataFieldModelV3 + chunk_grid: MetadataFieldModelV3 + codecs: tuple[MetadataFieldModelV3, ...] + chunk_key_encoding: MetadataFieldModelV3 + dimension_names: tuple[str | None, ...] | UNSET + attributes: dict[str, JSONValue] + storage_transformers: tuple[MetadataFieldModelV3, ...] + extra_fields: dict[str, ExtensionFieldV3] + + @classmethod + def create_default( + cls, **overrides: Unpack[ArrayMetadataModelV3Partial] + ) -> ArrayMetadataModelV3: + """ + Create a default (empty) v3 array metadata model, with optional overrides. + + The default is a structurally-valid scalar `uint8` array — the array + analog of `list()` returning `[]`. Any field can be overridden by keyword + (the same fields accepted by `update`). Overriding `shape` without + `chunk_grid` derives a consistent default grid: one regular chunk + covering the array (`chunk_shape` equal to `shape`). + + The derivation is deliberately one-way. A user-supplied `chunk_grid` + is an extension point and is taken verbatim — deriving `shape` from + it would require interpreting the grid's configuration, which this + layer never does (and cannot do for unrecognized grid names). So + overriding `chunk_grid` without `shape` keeps the scalar default + `shape=()`, and consistency between the two is the caller's + responsibility. + """ + if "shape" in overrides and "chunk_grid" not in overrides: + overrides["chunk_grid"] = NamedConfigModelV3( + name="regular", configuration={"chunk_shape": tuple(overrides["shape"])} + ) + default = cls( + shape=(), + fill_value=0, + data_type=NamedConfigModelV3(name="uint8", configuration={}), + chunk_grid=NamedConfigModelV3(name="regular", configuration={"chunk_shape": ()}), + codecs=(NamedConfigModelV3(name="bytes", configuration={}),), + chunk_key_encoding=NamedConfigModelV3(name="default", configuration={}), + dimension_names=UNSET, + attributes={}, + storage_transformers=(), + extra_fields={}, + ) + return default.update(**overrides) + + def update(self, **kwargs: Unpack[ArrayMetadataModelV3Partial]) -> ArrayMetadataModelV3: + """ + Return a new `ArrayMetadataModelV3` with the given fields updated. + + Only the constructor-settable fields listed in + `ArrayMetadataModelV3Partial` can be updated; any attempt to update + other fields (including the fixed `zarr_format` / `node_type`) is + rejected at the type level. Each given field fully replaces its + previous value, including `extra_fields`. + + This is useful for test fixtures that want to override a few fields of a + base template without having to re-specify the entire document. + + No re-validation is performed (`update` is `dataclasses.replace`), so + a repair or edit can produce an invalid document; validity is checked + on `from_json`, not on field replacement. + """ + return dataclasses.replace(self, **kwargs) + + def __post_init__(self) -> None: + overlap = set(self.extra_fields.keys()).intersection(ARRAY_METADATA_STANDARD_KEYS_V3) + if overlap: + raise MetadataValidationError( + [ + ValidationProblem( + ("extra_fields",), + "Extra fields cannot overlap with standard ArrayMetadataV3 fields", + "invalid_value", + ) + ] + ) + + def to_json(self) -> ArrayMetadataV3: + out: ArrayMetadataV3 = { + "zarr_format": self.zarr_format, + "node_type": self.node_type, + "shape": self.shape, + "fill_value": self.fill_value, + "data_type": self.data_type.to_json(), + "chunk_grid": self.chunk_grid.to_json(), + "codecs": tuple(codec.to_json() for codec in self.codecs), + "chunk_key_encoding": self.chunk_key_encoding.to_json(), + } + if self.dimension_names is not UNSET: + out["dimension_names"] = self.dimension_names + if len(self.attributes) > 0: + out["attributes"] = self.attributes + if len(self.storage_transformers) > 0: + out["storage_transformers"] = tuple( + transformer.to_json() for transformer in self.storage_transformers + ) + # Extra fields are the TypedDict's `extra_items` (PEP 728). Assign them + # by key rather than `out.update(**...)`: type checkers understand the + # indexed-write path against `extra_items`, but not the `update(**...)` + # overload. + for key, value in self.extra_fields.items(): + out[key] = value + return out + + @classmethod + def from_json(cls, data: object) -> ArrayMetadataModelV3: + parsed = parse_array_metadata_v3(arrays_to_tuples(data)) + # Sound cast: the TypedDict types all non-standard keys as its + # `extra_items` (`ExtensionFieldV3`); the comprehension's inferred value + # type is the union over ALL keys because the key filter cannot narrow it. + extra_fields = cast( + "dict[str, ExtensionFieldV3]", + {k: v for k, v in parsed.items() if k not in ARRAY_METADATA_STANDARD_KEYS_V3}, + ) + return cls( + shape=parsed["shape"], + fill_value=parsed["fill_value"], + data_type=NamedConfigModelV3.from_json(parsed["data_type"]), + chunk_grid=NamedConfigModelV3.from_json(parsed["chunk_grid"]), + codecs=tuple(NamedConfigModelV3.from_json(c) for c in parsed["codecs"]), + chunk_key_encoding=NamedConfigModelV3.from_json(parsed["chunk_key_encoding"]), + dimension_names=parsed.get("dimension_names", UNSET), + attributes=dict(parsed.get("attributes", {})), + storage_transformers=tuple( + NamedConfigModelV3.from_json(t) for t in parsed.get("storage_transformers", ()) + ), + extra_fields=extra_fields, + ) + + @property + def must_understand_fields(self) -> dict[str, ExtensionFieldV3]: + """Extra fields the reader is obligated to understand. + + Everything in `extra_fields` not explicitly waived with + `must_understand: false` (the spec's implicit-true rule). A compliant + reader MUST fail to open the array if this contains any field it does + not recognize; the model layer only partitions by obligation, since + recognition is reader-specific. + """ + return must_understand_subset(self.extra_fields) + + @classmethod + def from_key_value(cls, mapping: Mapping[str, bytes]) -> ArrayMetadataModelV3: + return cls.from_json(load_store_json(mapping, ARRAY_METADATA_STORE_KEY_V3)) + + def to_key_value(self, *, indent: int | str | None = None) -> Mapping[str, bytes]: + return { + ARRAY_METADATA_STORE_KEY_V3: json.dumps(self.to_json(), indent=indent).encode("utf-8") + } + + +class ArrayMetadataModelV2Partial(TypedDict, total=False): + """ + Partial form of the constructor-settable fields of `ArrayMetadataModelV2`. + + Every key is optional and typed with the model's own value types, so it + describes valid keyword arguments to `ArrayMetadataModelV2.update` and + `create_default`. The `init=False` field `zarr_format` is intentionally + excluded, since it cannot be passed to `dataclasses.replace`. + + Drift between this type and the model's settable fields is prevented by + `tests/model/test_array.py::test_v2_partial_keys_match_settable_model_fields`. + """ + + shape: tuple[int, ...] + dtype: DataTypeMetadataV2 + chunks: tuple[int, ...] + fill_value: JSONValue + order: ArrayOrderV2 + compressor: CodecMetadataV2 | None + filters: tuple[CodecMetadataV2, ...] | None + dimension_separator: ArrayDimensionSeparatorV2 + attributes: dict[str, JSONValue] | UNSET + + +@dataclass(frozen=True, slots=True, kw_only=True) +class ArrayMetadataModelV2: + """In-memory model of a v2 array metadata document. + + A canonical, lossless representation of the `.zarray` content plus the + sibling `.zattrs` attributes. `dtype`, `compressor`, and `filters` are + held in their raw JSON forms and are never interpreted; `fill_value` is + held verbatim in its JSON form. `attributes` is `UNSET` when no + `.zattrs` file (or merged `attributes` key) exists — distinct from an + explicit empty `.zattrs`, which is `{}` and round-trips as a file. One + spelling normalization: a `.zarray` that omits `dimension_separator` + means `"."` by the v2 convention, and the model holds and re-emits that + value explicitly. + """ + + zarr_format: Literal[2] = field(default=2, init=False) + shape: tuple[int, ...] + dtype: DataTypeMetadataV2 + chunks: tuple[int, ...] + fill_value: JSONValue + order: ArrayOrderV2 + compressor: CodecMetadataV2 | None + filters: tuple[CodecMetadataV2, ...] | None + # "." is the v2 convention's default for an ABSENT dimension_separator key; + # from_json normalizes absence to it (a semantics-preserving spelling + # normalization, like the v3 bare-string metadata-field form). The value + # is never None: the document grammar has no null spelling for this field. + dimension_separator: ArrayDimensionSeparatorV2 = field(default=".") + attributes: dict[str, JSONValue] | UNSET + + def update(self, **kwargs: Unpack[ArrayMetadataModelV2Partial]) -> ArrayMetadataModelV2: + """ + Return a new `ArrayMetadataModelV2` with the given fields updated. + + Only the constructor-settable fields listed in + `ArrayMetadataModelV2Partial` can be updated; the fixed `zarr_format` is + rejected at the type level. Each given field fully replaces its previous + value. + """ + return dataclasses.replace(self, **kwargs) + + @classmethod + def create_default( + cls, **overrides: Unpack[ArrayMetadataModelV2Partial] + ) -> ArrayMetadataModelV2: + """ + Create a default (empty) v2 array metadata model, with optional overrides. + + The default is a structurally-valid scalar `uint8` (`"|u1"`) array — the + array analog of `list()` returning `[]`. Any field can be overridden by + keyword (the same fields accepted by `update`). Overriding `shape` + without `chunks` derives `chunks` equal to `shape` (one chunk covering + the array). + + The derivation is deliberately one-way, matching the v3 model: + overriding `chunks` without `shape` keeps the scalar default + `shape=()`, and consistency between the two is the caller's + responsibility. + """ + if "shape" in overrides and "chunks" not in overrides: + overrides["chunks"] = tuple(overrides["shape"]) + default = cls( + shape=(), + dtype="|u1", + chunks=(), + fill_value=0, + order="C", + compressor=None, + filters=None, + attributes=UNSET, + ) + return default.update(**overrides) + + def to_json(self) -> ArrayMetadataV2: + """Return the merged in-memory document form. + + `attributes` is included when set (even empty). This is not the + on-disk `.zarray` content: a conforming `.zarray` must exclude + `attributes` (they live in the sibling `.zattrs` file). Use + `to_key_value` to produce the spec-conforming split for storage. + """ + out: ArrayMetadataV2 = { + "zarr_format": self.zarr_format, + "shape": self.shape, + "dtype": self.dtype, + "order": self.order, + "chunks": self.chunks, + "fill_value": self.fill_value, + "dimension_separator": self.dimension_separator, + "compressor": self.compressor, + "filters": self.filters, + } + if self.attributes is not UNSET: + out["attributes"] = self.attributes + return out + + @classmethod + def from_json(cls, data: object) -> ArrayMetadataModelV2: + parsed = parse_array_metadata_v2(arrays_to_tuples(data)) + return cls( + shape=parsed["shape"], + dtype=parsed["dtype"], + chunks=parsed["chunks"], + fill_value=parsed["fill_value"], + order=parsed["order"], + compressor=parsed["compressor"], + filters=parsed["filters"], + dimension_separator=parsed.get("dimension_separator", "."), + attributes=(dict(parsed["attributes"]) if "attributes" in parsed else UNSET), + ) + + @classmethod + def from_key_value(cls, mapping: Mapping[str, bytes]) -> ArrayMetadataModelV2: + zarray = load_store_json(mapping, ARRAY_METADATA_STORE_KEY_V2) + if ATTRIBUTES_STORE_KEY_V2 in mapping: + zattrs = load_store_json(mapping, ATTRIBUTES_STORE_KEY_V2) + return cls.from_json({**zarray, "attributes": zattrs}) + return cls.from_json(dict(zarray)) + + def to_key_value(self, *, indent: int | str | None = None) -> Mapping[str, bytes]: + # Attributes live only in the sibling `.zattrs` file; the `.zarray` + # document must exclude them. The `.zattrs` key is present exactly + # when attributes are set (even empty) — UNSET emits no file. + zarray = {k: v for k, v in self.to_json().items() if k != "attributes"} + out = {ARRAY_METADATA_STORE_KEY_V2: json.dumps(zarray, indent=indent).encode("utf-8")} + if self.attributes is not UNSET: + out[ATTRIBUTES_STORE_KEY_V2] = json.dumps(self.attributes, indent=indent).encode( + "utf-8" + ) + return out diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_group.py b/packages/zarr-metadata/src/zarr_metadata/model/_group.py new file mode 100644 index 0000000000..222b0b3bf9 --- /dev/null +++ b/packages/zarr-metadata/src/zarr_metadata/model/_group.py @@ -0,0 +1,404 @@ +"""In-memory models for Zarr group and consolidated metadata documents.""" + +from __future__ import annotations + +import dataclasses +import json +from collections.abc import Mapping +from dataclasses import dataclass, field +from typing import TYPE_CHECKING, Final, Literal, cast + +from typing_extensions import TypedDict, Unpack + +from zarr_metadata.model._array import ( + ATTRIBUTES_STORE_KEY_V2, + ArrayMetadataModelV3, + must_understand_subset, +) +from zarr_metadata.model._sentinel import UNSET +from zarr_metadata.model._validation import ( + GROUP_METADATA_STANDARD_KEYS_V3, + MetadataValidationError, + ValidationProblem, + arrays_to_tuples, + load_store_json, + parse_group_metadata_v2, + parse_group_metadata_v3, + validate_consolidated_metadata_v3, +) + +if TYPE_CHECKING: + from zarr_metadata._common import JSONValue + from zarr_metadata.v2.group import GroupMetadataV2 + from zarr_metadata.v3.array import ExtensionFieldV3 + from zarr_metadata.v3.consolidated import ConsolidatedMetadataV3 + from zarr_metadata.v3.group import GroupMetadataV3 + +GroupMetadataStoreKeyV3 = Literal["zarr.json"] +GROUP_METADATA_STORE_KEY_V3: Final[GroupMetadataStoreKeyV3] = "zarr.json" + +GroupMetadataStoreKeyV2 = Literal[".zgroup"] +GROUP_METADATA_STORE_KEY_V2: Final[GroupMetadataStoreKeyV2] = ".zgroup" + +ConsolidatedMetadataStoreKeyV2 = Literal[".zmetadata"] +CONSOLIDATED_METADATA_STORE_KEY_V2: Final[ConsolidatedMetadataStoreKeyV2] = ".zmetadata" + +# The key under which consolidated metadata is embedded in a v3 group document. +# This is a reference-implementation convention (not a spec artifact), stored +# as an extension field on the group's `zarr.json`. +CONSOLIDATED_METADATA_KEY_V3: Final = "consolidated_metadata" + + +class GroupMetadataModelV3Partial(TypedDict, total=False): + """ + Partial form of the constructor-settable fields of `GroupMetadataModelV3`. + + Every key is optional and typed with the model's own value types, so it + describes valid keyword arguments to `GroupMetadataModelV3.update` and + `create_default`. The `init=False` fields `zarr_format` and `node_type` + are intentionally excluded, since they cannot be passed to + `dataclasses.replace`. + + Drift between this type and the model's settable fields is prevented by + `tests/model/test_group.py::test_group_partial_keys_match_settable_model_fields`. + """ + + attributes: dict[str, JSONValue] + consolidated_metadata: ConsolidatedMetadataModelV3 | UNSET + extra_fields: dict[str, ExtensionFieldV3] + + +@dataclass(frozen=True, slots=True, kw_only=True) +class GroupMetadataModelV3: + """In-memory model of a v3 group metadata document. + + A canonical, lossless representation of the `zarr.json` content for a + group. The `consolidated_metadata` reference-implementation convention is + modeled as a typed field holding thin child models; every other unknown + top-level key lands in `extra_fields` verbatim. + """ + + zarr_format: Literal[3] = field(default=3, init=False) + node_type: Literal["group"] = field(default="group", init=False) + attributes: dict[str, JSONValue] + consolidated_metadata: ConsolidatedMetadataModelV3 | UNSET + extra_fields: dict[str, ExtensionFieldV3] + + def __post_init__(self) -> None: + reserved = GROUP_METADATA_STANDARD_KEYS_V3 | {CONSOLIDATED_METADATA_KEY_V3} + if set(self.extra_fields.keys()).intersection(reserved): + raise MetadataValidationError( + [ + ValidationProblem( + ("extra_fields",), + "Extra fields cannot overlap with standard GroupMetadataV3 fields", + "invalid_value", + ) + ] + ) + + @classmethod + def create_default( + cls, **overrides: Unpack[GroupMetadataModelV3Partial] + ) -> GroupMetadataModelV3: + """ + Create a default (empty) v3 group metadata model, with optional overrides. + + The default is a structurally-valid group with no attributes — the group + analog of `list()` returning `[]`. Any field can be overridden by keyword + (the same fields accepted by `update`). + """ + default = cls(attributes={}, consolidated_metadata=UNSET, extra_fields={}) + return default.update(**overrides) + + def update(self, **kwargs: Unpack[GroupMetadataModelV3Partial]) -> GroupMetadataModelV3: + """ + Return a new `GroupMetadataModelV3` with the given fields updated. + + Only the constructor-settable fields listed in + `GroupMetadataModelV3Partial` can be updated; the fixed `zarr_format` / + `node_type` are rejected at the type level. Each given field fully + replaces its previous value, including `extra_fields`. + """ + return dataclasses.replace(self, **kwargs) + + def to_json(self) -> GroupMetadataV3: + out: GroupMetadataV3 = { + "zarr_format": self.zarr_format, + "node_type": self.node_type, + } + if len(self.attributes) > 0: + out["attributes"] = self.attributes + if self.consolidated_metadata is not UNSET: + # The consolidated-metadata shape ({kind, must_understand, metadata}, + # no `name`) predates the strict v3.1 extension-field rules, so it is + # not assignable to `ExtensionFieldV3`; see the discussion on + # `zarr_metadata.v3.consolidated`. + out[CONSOLIDATED_METADATA_KEY_V3] = cast( + "ExtensionFieldV3", self.consolidated_metadata.to_json() + ) + for key, value in self.extra_fields.items(): + out[key] = value + return out + + @classmethod + def from_json(cls, data: object) -> GroupMetadataModelV3: + parsed = parse_group_metadata_v3(arrays_to_tuples(data)) + # Cast to object: the TypedDict's extra_items type does not admit null, + # but wild documents (historical zarr-python) contain it. + consolidated_raw = cast("object", parsed.get(CONSOLIDATED_METADATA_KEY_V3, UNSET)) + consolidated: ConsolidatedMetadataModelV3 | UNSET + if consolidated_raw is UNSET or consolidated_raw is None: + # consolidated_metadata: null was written by a historical + # zarr-python bug; it gets no model representation. It is read as + # absence and never written back — repaired, not preserved. + consolidated = UNSET + else: + consolidated = ConsolidatedMetadataModelV3.from_json(consolidated_raw) + # Sound cast: the TypedDict types all non-standard keys as its + # `extra_items` (`ExtensionFieldV3`); the comprehension's inferred value + # type is the union over ALL keys because the key filter cannot narrow it. + extra_fields = cast( + "dict[str, ExtensionFieldV3]", + { + k: v + for k, v in parsed.items() + if k not in GROUP_METADATA_STANDARD_KEYS_V3 and k != CONSOLIDATED_METADATA_KEY_V3 + }, + ) + return cls( + attributes=dict(parsed.get("attributes", {})), + consolidated_metadata=consolidated, + extra_fields=extra_fields, + ) + + @property + def must_understand_fields(self) -> dict[str, ExtensionFieldV3]: + """Extra fields the reader is obligated to understand. + + Everything in `extra_fields` not explicitly waived with + `must_understand: false` (the spec's implicit-true rule). A compliant + reader MUST fail to open the group if this contains any field it does + not recognize; the model layer only partitions by obligation, since + recognition is reader-specific. + """ + return must_understand_subset(self.extra_fields) + + @classmethod + def from_key_value(cls, mapping: Mapping[str, bytes]) -> GroupMetadataModelV3: + return cls.from_json(load_store_json(mapping, GROUP_METADATA_STORE_KEY_V3)) + + def to_key_value(self, *, indent: int | str | None = None) -> Mapping[str, bytes]: + return { + GROUP_METADATA_STORE_KEY_V3: json.dumps(self.to_json(), indent=indent).encode("utf-8") + } + + +@dataclass(frozen=True, slots=True, kw_only=True) +class ConsolidatedMetadataModelV3: + """In-memory model of v3 inline consolidated metadata. + + Models the reference-implementation convention where consolidated metadata + is embedded as an extension field on a group's `zarr.json`. Each entry in + `metadata` is a complete child document, held as a thin array or group + model. `must_understand` is typed permissively as `bool` to mirror the + document shape, but only `False` is valid; this is enforced at runtime. + """ + + kind: Literal["inline"] = field(default="inline", init=False) + must_understand: bool = False + metadata: dict[str, ArrayMetadataModelV3 | GroupMetadataModelV3] + + def __post_init__(self) -> None: + if self.must_understand is not False: + raise MetadataValidationError( + [ + ValidationProblem( + ("must_understand",), + f"Invalid value for 'must_understand'. Expected False. " + f"Got {self.must_understand!r}.", + "invalid_value", + ) + ] + ) + + def to_json(self) -> ConsolidatedMetadataV3: + # `must_understand` is emitted as the literal False: the field is typed + # permissively as `bool`, but `__post_init__` guarantees the value. + return { + "kind": self.kind, + "must_understand": False, + "metadata": {key: node.to_json() for key, node in self.metadata.items()}, + } + + @classmethod + def from_json(cls, data: object) -> ConsolidatedMetadataModelV3: + problems = validate_consolidated_metadata_v3(data) + if problems: + raise MetadataValidationError(problems) + env = cast("Mapping[str, object]", data) + entries: dict[str, ArrayMetadataModelV3 | GroupMetadataModelV3] = {} + for key, entry in cast("Mapping[str, object]", env["metadata"]).items(): + node_type = cast("Mapping[str, object]", entry).get("node_type") + if node_type == "array": + entries[key] = ArrayMetadataModelV3.from_json(entry) + else: + entries[key] = GroupMetadataModelV3.from_json(entry) + return cls(metadata=entries) + + +class GroupMetadataModelV2Partial(TypedDict, total=False): + """ + Partial form of the constructor-settable fields of `GroupMetadataModelV2`. + + Every key is optional and typed with the model's own value types, so it + describes valid keyword arguments to `GroupMetadataModelV2.update` and + `create_default`. The `init=False` field `zarr_format` is intentionally + excluded, since it cannot be passed to `dataclasses.replace`. + + Drift between this type and the model's settable fields is prevented by + `tests/model/test_group.py::test_group_partial_keys_match_settable_model_fields`. + """ + + attributes: dict[str, JSONValue] | UNSET + + +@dataclass(frozen=True, slots=True, kw_only=True) +class GroupMetadataModelV2: + """In-memory model of a v2 group metadata document. + + A canonical, lossless representation of the `.zgroup` content plus the + sibling `.zattrs` attributes, folded into a single in-memory value + (mirroring the merged `GroupMetadataV2` document form). `attributes` is + `UNSET` when no `.zattrs` file (or merged `attributes` key) exists — + distinct from an explicit empty `.zattrs`, which is `{}` and round-trips + as a file. + """ + + zarr_format: Literal[2] = field(default=2, init=False) + attributes: dict[str, JSONValue] | UNSET + + @classmethod + def create_default( + cls, **overrides: Unpack[GroupMetadataModelV2Partial] + ) -> GroupMetadataModelV2: + """ + Create a default (empty) v2 group metadata model, with optional overrides. + + The default is a structurally-valid group with no attributes — the group + analog of `list()` returning `[]`. Any field can be overridden by keyword + (the same fields accepted by `update`). + """ + default = cls(attributes=UNSET) + return default.update(**overrides) + + def update(self, **kwargs: Unpack[GroupMetadataModelV2Partial]) -> GroupMetadataModelV2: + """ + Return a new `GroupMetadataModelV2` with the given fields updated. + + Only the constructor-settable fields listed in + `GroupMetadataModelV2Partial` can be updated; the fixed `zarr_format` + is rejected at the type level. Each given field fully replaces its + previous value. + """ + return dataclasses.replace(self, **kwargs) + + def to_json(self) -> GroupMetadataV2: + """Return the merged in-memory document form. + + `attributes` is included when set (even empty). This is not the + on-disk `.zgroup` content: a conforming `.zgroup` must exclude + `attributes` (they live in the sibling `.zattrs` file). Use + `to_key_value` to produce the spec-conforming split for storage. + """ + out: GroupMetadataV2 = {"zarr_format": self.zarr_format} + if self.attributes is not UNSET: + out["attributes"] = self.attributes + return out + + @classmethod + def from_json(cls, data: object) -> GroupMetadataModelV2: + parsed = parse_group_metadata_v2(arrays_to_tuples(data)) + return cls(attributes=(dict(parsed["attributes"]) if "attributes" in parsed else UNSET)) + + @classmethod + def from_key_value(cls, mapping: Mapping[str, bytes]) -> GroupMetadataModelV2: + zgroup = load_store_json(mapping, GROUP_METADATA_STORE_KEY_V2) + if ATTRIBUTES_STORE_KEY_V2 in mapping: + zattrs = load_store_json(mapping, ATTRIBUTES_STORE_KEY_V2) + return cls.from_json({**zgroup, "attributes": zattrs}) + return cls.from_json(dict(zgroup)) + + def to_key_value(self, *, indent: int | str | None = None) -> Mapping[str, bytes]: + # Attributes live only in the sibling `.zattrs` file; the `.zgroup` + # document must exclude them. The `.zattrs` key is present exactly + # when attributes are set (even empty) — UNSET emits no file. + zgroup = {k: v for k, v in self.to_json().items() if k != "attributes"} + out = {GROUP_METADATA_STORE_KEY_V2: json.dumps(zgroup, indent=indent).encode("utf-8")} + if self.attributes is not UNSET: + out[ATTRIBUTES_STORE_KEY_V2] = json.dumps(self.attributes, indent=indent).encode( + "utf-8" + ) + return out + + +@dataclass(frozen=True, slots=True, kw_only=True) +class ConsolidatedMetadataModelV2: + """In-memory model of a v2 `.zmetadata` document. + + The `metadata` map holds the flat file-keyed entries (`"path/.zarray"`, + `"path/.zattrs"`, ...) verbatim, preserving byte-faithful round-tripping. + Entries are deliberately NOT merged into per-node models: which nodes had + a `.zattrs` file at all is information the canonical representation must + keep. Interpreting entries into node models is consumer work. + """ + + zarr_consolidated_format: Literal[1] = field(default=1, init=False) + metadata: dict[str, JSONValue] + + def to_json(self) -> dict[str, JSONValue]: + return { + "zarr_consolidated_format": self.zarr_consolidated_format, + "metadata": self.metadata, + } + + @classmethod + def from_json(cls, data: object) -> ConsolidatedMetadataModelV2: + if not isinstance(data, Mapping): + raise MetadataValidationError( + [ValidationProblem((), "expected a mapping", "invalid_type")] + ) + doc = cast("Mapping[str, object]", data) + problems: list[ValidationProblem] = [ + ValidationProblem((key,), "missing required key", "missing_key") + for key in ("zarr_consolidated_format", "metadata") + if key not in doc + ] + if "metadata" in doc: + entries = doc["metadata"] + if not isinstance(entries, Mapping) or not all( + isinstance(k, str) for k in cast("Mapping[object, object]", entries) + ): + problems.append( + ValidationProblem( + ("metadata",), "expected a mapping with string keys", "invalid_type" + ) + ) + if problems: + raise MetadataValidationError(problems) + entries_tupled = cast( + "dict[str, JSONValue]", + arrays_to_tuples(dict(cast("Mapping[str, object]", doc["metadata"]))), + ) + return cls(metadata=entries_tupled) + + @classmethod + def from_key_value(cls, mapping: Mapping[str, bytes]) -> ConsolidatedMetadataModelV2: + return cls.from_json(load_store_json(mapping, CONSOLIDATED_METADATA_STORE_KEY_V2)) + + def to_key_value(self, *, indent: int | str | None = None) -> Mapping[str, bytes]: + return { + CONSOLIDATED_METADATA_STORE_KEY_V2: json.dumps(self.to_json(), indent=indent).encode( + "utf-8" + ) + } diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_sentinel.py b/packages/zarr-metadata/src/zarr_metadata/model/_sentinel.py new file mode 100644 index 0000000000..eeae0da97a --- /dev/null +++ b/packages/zarr-metadata/src/zarr_metadata/model/_sentinel.py @@ -0,0 +1,29 @@ +"""The absence sentinel for optional metadata-document keys. + +The models observe one invariant: `None` in a model always corresponds to a +JSON `null` in the document (a v2 `compressor`/`filters` value, an unnamed +dimension inside `dimension_names`), and `UNSET` always means the document +key is absent. The two are never interchangeable, so a model value can never +leak into a document as a spelling the writer did not intend. + +Check with identity: `if model.dimension_names is UNSET: ...`. + +Checker support (PEP 661 is Final; stdlib `sentinel` arrives in Python +3.15): ty types this spelling exactly, including `is`/`is not` narrowing. +Pyright supports it but a regression (1.1.405+, tracked as +https://github.com/microsoft/pyright/issues/11115) degrades class-attribute +reads to `Unknown`, so this package pins pyright to the last good version +until the fix lands. Mypy support is in review +(https://github.com/python/mypy/pull/21647); until it merges, mypy-checked +consumers of these fields need a `cast` or `type: ignore` at narrowing +sites. This is a deliberate short-term cost: the sentinel is the standard, +and the checkers are converging on it. +""" + +from __future__ import annotations + +from typing_extensions import Sentinel + +UNSET = Sentinel("UNSET") +"""Marks a metadata-document key as absent (PEP 661 sentinel; usable directly +in type expressions, e.g. `tuple[str, ...] | UNSET`). Test with `is UNSET`.""" diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_validation.py b/packages/zarr-metadata/src/zarr_metadata/model/_validation.py new file mode 100644 index 0000000000..fae09187a4 --- /dev/null +++ b/packages/zarr-metadata/src/zarr_metadata/model/_validation.py @@ -0,0 +1,604 @@ +"""Structural validation for Zarr metadata documents. + +Validators check JSON structure (key presence, value shapes, and fixed +literals like `zarr_format`), not domain validity. Each concept gets a +`validate_*` function returning every problem found, an `is_*` type guard, +and a `parse_*` function that narrows or raises `MetadataValidationError`. + +Every `ValidationProblem` carries a machine-readable `kind` alongside its +human-readable `message`, so consumers can dispatch on the failure mode +(`missing_key`, `invalid_type`, `invalid_value`, `invalid_json`) without +string-matching messages. +""" + +from __future__ import annotations + +import json +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from typing import Any, Final, Literal, cast + +from typing_extensions import TypeIs + +from zarr_metadata._common import JSONValue +from zarr_metadata.v2.array import ArrayMetadataV2 +from zarr_metadata.v2.group import GroupMetadataV2 +from zarr_metadata.v3._common import MetadataV3 +from zarr_metadata.v3.array import ArrayMetadataV3 +from zarr_metadata.v3.group import GroupMetadataV3 + +ProblemKind = Literal["missing_key", "invalid_type", "invalid_value", "invalid_json"] +"""Machine-readable classification of a `ValidationProblem`. + +- `missing_key`: a required key (document key or store key) is absent. +- `invalid_type`: a value has the wrong structural type (e.g. a string where + a mapping is required, a non-JSON-serializable object). +- `invalid_value`: a value has an acceptable type but an invalid content + (e.g. `zarr_format: 2` in a v3 document, `order: "Q"`). +- `invalid_json`: bytes that do not decode as JSON. +""" + + +@dataclass(frozen=True, slots=True) +class ValidationProblem: + """A single structural problem found while validating a metadata document. + + `loc` is the path from the document root to the offending value, e.g. + `("codecs", 0, "name")`. An empty `loc` refers to the document as a whole. + `kind` classifies the failure mode for programmatic dispatch; `message` + is the human-readable description. + """ + + loc: tuple[str | int, ...] + message: str + kind: ProblemKind + + def __str__(self) -> str: + location = ".".join(str(part) for part in self.loc) if self.loc else "" + return f"{location}: {self.message}" + + +class MetadataValidationError(ValueError): + """Raised when a value fails structural metadata validation. + + Carries every problem found (not just the first) in `.problems`. + """ + + def __init__(self, problems: list[ValidationProblem]) -> None: + self.problems = problems + super().__init__("\n".join(str(problem) for problem in problems)) + + +def _prefix(loc_head: str | int, problems: list[ValidationProblem]) -> list[ValidationProblem]: + """Prepend `loc_head` to the `loc` of every problem (for nested validators).""" + return [ValidationProblem((loc_head, *p.loc), p.message, p.kind) for p in problems] + + +def validate_json(value: object) -> list[ValidationProblem]: + """Return every reason `value` is not JSON-serializable (recursively).""" + if isinstance(value, (str, int, float, bool)) or value is None: + return [] + problems: list[ValidationProblem] = [] + if isinstance(value, Mapping): + for key, item in cast("Mapping[object, object]", value).items(): + if not isinstance(key, str): + problems.append( + ValidationProblem((), f"non-string key {key!r} in JSON object", "invalid_type") + ) + continue + problems.extend(_prefix(key, validate_json(item))) + return problems + if isinstance(value, Sequence) and not isinstance(value, (bytes, bytearray)): + for index, item in enumerate(cast("Sequence[object]", value)): + problems.extend(_prefix(index, validate_json(item))) + return problems + return [ValidationProblem((), f"not a JSON-serializable value: {value!r}", "invalid_type")] + + +def is_json(value: object) -> TypeIs[JSONValue]: + """Whether `value` is a JSON-serializable structure (recursively).""" + return not validate_json(value) + + +def parse_json(value: object) -> JSONValue: + """Return `value` narrowed to `JSONValue`, or raise `MetadataValidationError`.""" + problems = validate_json(value) + if problems: + raise MetadataValidationError(problems) + return cast(JSONValue, value) + + +# The standard top-level keys of a v3 array metadata document. Anything outside +# this set is an extension field. Built from the TypedDict's required/optional +# key sets (which resolve inherited keys, unlike `__annotations__`). +ARRAY_METADATA_REQUIRED_KEYS_V3: Final[frozenset[str]] = frozenset( + ArrayMetadataV3.__required_keys__ +) +ARRAY_METADATA_OPTIONAL_KEYS_V3: Final[frozenset[str]] = frozenset( + ArrayMetadataV3.__optional_keys__ +) +ARRAY_METADATA_STANDARD_KEYS_V3: Final[frozenset[str]] = ( + ARRAY_METADATA_REQUIRED_KEYS_V3 | ARRAY_METADATA_OPTIONAL_KEYS_V3 +) + +ARRAY_METADATA_REQUIRED_KEYS_V2: Final[frozenset[str]] = frozenset( + ArrayMetadataV2.__required_keys__ +) + +# The standard top-level keys of a v3 group metadata document. Anything outside +# this set is an extension field. +GROUP_METADATA_REQUIRED_KEYS_V3: Final[frozenset[str]] = frozenset( + GroupMetadataV3.__required_keys__ +) +GROUP_METADATA_OPTIONAL_KEYS_V3: Final[frozenset[str]] = frozenset( + GroupMetadataV3.__optional_keys__ +) +GROUP_METADATA_STANDARD_KEYS_V3: Final[frozenset[str]] = ( + GROUP_METADATA_REQUIRED_KEYS_V3 | GROUP_METADATA_OPTIONAL_KEYS_V3 +) + +GROUP_METADATA_REQUIRED_KEYS_V2: Final[frozenset[str]] = frozenset( + GroupMetadataV2.__required_keys__ +) + + +def _missing_keys(required: frozenset[str], doc: Mapping[str, object]) -> list[ValidationProblem]: + """One `missing_key` problem per required key absent from `doc`.""" + return [ + ValidationProblem((key,), "missing required key", "missing_key") + for key in sorted(required - doc.keys()) + ] + + +def _check_literal( + doc: Mapping[str, object], key: str, expected: object +) -> list[ValidationProblem]: + """One `invalid_value` problem if `doc[key]` is present but not `expected`.""" + if key in doc and doc[key] != expected: + return [ + ValidationProblem((key,), f"expected {expected!r}, got {doc[key]!r}", "invalid_value") + ] + return [] + + +def validate_metadata_field_v3(value: object) -> list[ValidationProblem]: + """Return every reason `value` is not a v3 metadata field. + + A metadata field is a bare name string or a `{name, configuration}` mapping. + """ + if isinstance(value, str): + return [] + if not isinstance(value, Mapping): + return [ + ValidationProblem( + (), + "expected a metadata field (string or {name, configuration})", + "invalid_type", + ) + ] + field = cast("Mapping[object, object]", value) + problems: list[ValidationProblem] = [] + if not isinstance(field.get("name"), str): + problems.append(ValidationProblem(("name",), "expected a string name", "invalid_type")) + if "configuration" in field: + configuration = field["configuration"] + if not isinstance(configuration, Mapping): + problems.append( + ValidationProblem(("configuration",), "expected a mapping", "invalid_type") + ) + elif not all(isinstance(k, str) for k in cast("Mapping[object, object]", configuration)): + problems.append( + ValidationProblem(("configuration",), "expected string keys", "invalid_type") + ) + else: + for key, item in cast("Mapping[str, object]", configuration).items(): + problems.extend(_prefix("configuration", _prefix(key, validate_json(item)))) + return problems + + +def is_metadata_field_v3(value: object) -> TypeIs[MetadataV3]: + """Whether `value` is a v3 metadata field: a bare name or a named config.""" + return not validate_metadata_field_v3(value) + + +def parse_metadata_field_v3(value: object) -> MetadataV3: + """Return `value` narrowed to `MetadataV3`, or raise `MetadataValidationError`.""" + problems = validate_metadata_field_v3(value) + if problems: + raise MetadataValidationError(problems) + return cast(MetadataV3, value) + + +def _is_int_sequence(value: object) -> bool: + """Whether `value` is a non-string sequence of integers. + + JSON booleans decode to `bool`, which is an `int` subclass in Python but + is not an integer in a metadata document, so booleans are excluded. + """ + return ( + not isinstance(value, str) + and isinstance(value, Sequence) + and all( + isinstance(item, int) and not isinstance(item, bool) + for item in cast("Sequence[object]", value) + ) + ) + + +def _validate_dim_sequence(doc: Mapping[str, object], key: str) -> list[ValidationProblem]: + """Validate a dimension sequence (`shape` / `chunks`) if present in `doc`. + + Dimension lengths are non-negative integers. + """ + if key not in doc: + return [] + value = doc[key] + if not _is_int_sequence(value): + return [ValidationProblem((key,), "expected a sequence of int", "invalid_type")] + if any(item < 0 for item in cast("Sequence[int]", value)): + return [ValidationProblem((key,), "expected non-negative integers", "invalid_value")] + return [] + + +def _is_dtype_v2(value: object) -> bool: + """Whether `value` is shaped like a v2 dtype: a string or field records. + + A field record is a `(name, dtype)` or `(name, dtype, shape)` sequence, + where `dtype` is itself a string or nested field records and `shape` is a + sequence of int. The string content is NOT interpreted — whether the + string names a real dtype is domain validity, not structure. + """ + if isinstance(value, str): + return True + if not isinstance(value, Sequence): + return False + for record in cast("Sequence[object]", value): + if isinstance(record, str) or not isinstance(record, Sequence): + return False + fields = cast("Sequence[object]", record) + if len(fields) not in (2, 3): + return False + if not isinstance(fields[0], str): + return False + if not _is_dtype_v2(fields[1]): + return False + if len(fields) == 3 and not _is_int_sequence(fields[2]): + return False + return True + + +def _is_codec_v2(value: object) -> bool: + """Whether `value` is shaped like a v2 codec config: a mapping with a string `id`.""" + return isinstance(value, Mapping) and isinstance( + cast("Mapping[object, object]", value).get("id"), str + ) + + +def _validate_attributes(value: object) -> list[ValidationProblem]: + """Validate an `attributes` value: a mapping with string keys. + + Returns a problem at `("attributes",)` if it is not, else `[]`. Shared by the + v2 and v3 validators. Unlike the other `validate_*` functions (which + return value-relative locs for the caller to `_prefix`), this emits the + already-parent-relative `("attributes",)` loc, since it is only ever called + with a document's `attributes` value. + """ + if not isinstance(value, Mapping) or not all( + isinstance(k, str) for k in cast("Mapping[object, object]", value) + ): + return [ + ValidationProblem( + ("attributes",), "expected a mapping with string keys", "invalid_type" + ) + ] + problems: list[ValidationProblem] = [] + for key, item in cast("Mapping[str, object]", value).items(): + problems.extend(_prefix("attributes", _prefix(key, validate_json(item)))) + return problems + + +def validate_array_metadata_v3(value: object) -> list[ValidationProblem]: + """Return every reason `value` is not a structurally-valid v3 array doc. + + Checks structure, not domain validity. Unknown top-level keys are allowed + (they map to `extra_fields`). + """ + if not isinstance(value, Mapping): + return [ValidationProblem((), "expected a mapping", "invalid_type")] + doc = cast("Mapping[str, object]", value) + problems: list[ValidationProblem] = _missing_keys(ARRAY_METADATA_REQUIRED_KEYS_V3, doc) + problems.extend(_check_literal(doc, "zarr_format", 3)) + problems.extend(_check_literal(doc, "node_type", "array")) + problems.extend(_validate_dim_sequence(doc, "shape")) + if "fill_value" in doc: + problems.extend(_prefix("fill_value", validate_json(doc["fill_value"]))) + for key in ("data_type", "chunk_grid", "chunk_key_encoding"): + if key in doc: + problems.extend(_prefix(key, validate_metadata_field_v3(doc[key]))) + for key in ("codecs", "storage_transformers"): + if key in doc: + entries = doc[key] + if isinstance(entries, str) or not isinstance(entries, Sequence): + problems.append(ValidationProblem((key,), "expected a sequence", "invalid_type")) + else: + for index, entry in enumerate(cast("Sequence[object]", entries)): + problems.extend(_prefix(key, _prefix(index, validate_metadata_field_v3(entry)))) + if "attributes" in doc: + problems.extend(_validate_attributes(doc["attributes"])) + if "dimension_names" in doc: + # Simple typed sequences (dimension_names, shape, chunks) report a single + # field-level loc, not per-bad-item locs; per-index locs are reserved for + # the metadata-field lists (codecs, storage_transformers). + names = doc["dimension_names"] + if isinstance(names, str) or not isinstance(names, Sequence): + problems.append( + ValidationProblem(("dimension_names",), "expected a sequence", "invalid_type") + ) + elif not all( + item is None or isinstance(item, str) for item in cast("Sequence[object]", names) + ): + problems.append( + ValidationProblem( + ("dimension_names",), "expected items of str or None", "invalid_type" + ) + ) + elif _is_int_sequence(doc.get("shape")) and len(cast("Sequence[object]", names)) != len( + cast("Sequence[int]", doc["shape"]) + ): + problems.append( + ValidationProblem( + ("dimension_names",), + "expected one name per dimension of shape", + "invalid_value", + ) + ) + return problems + + +def is_array_metadata_v3(value: object) -> TypeIs[ArrayMetadataV3]: + """Whether `value` is a structurally-valid v3 array metadata document.""" + return not validate_array_metadata_v3(value) + + +def parse_array_metadata_v3(value: object) -> ArrayMetadataV3: + """Return `value` narrowed to `ArrayMetadataV3`, or raise `MetadataValidationError`.""" + problems = validate_array_metadata_v3(value) + if problems: + raise MetadataValidationError(problems) + return cast(ArrayMetadataV3, value) + + +def validate_array_metadata_v2(value: object) -> list[ValidationProblem]: + """Return every reason `value` is not a structurally-valid v2 array doc. + + Checks structure, not domain validity: `dtype` must be a string or field + records, but the string content is not interpreted; `compressor` and + `filters` are required keys that may be `None`, and otherwise must be + codec configurations (mappings with a string `id`). + """ + if not isinstance(value, Mapping): + return [ValidationProblem((), "expected a mapping", "invalid_type")] + doc = cast("Mapping[str, object]", value) + problems: list[ValidationProblem] = _missing_keys(ARRAY_METADATA_REQUIRED_KEYS_V2, doc) + problems.extend(_check_literal(doc, "zarr_format", 2)) + problems.extend(_validate_dim_sequence(doc, "shape")) + problems.extend(_validate_dim_sequence(doc, "chunks")) + if "dtype" in doc and not _is_dtype_v2(doc["dtype"]): + problems.append( + ValidationProblem( + ("dtype",), + "expected a v2 dtype string or a sequence of field records", + "invalid_type", + ) + ) + if "order" in doc and doc["order"] not in ("C", "F"): + problems.append( + ValidationProblem( + ("order",), f"expected 'C' or 'F', got {doc['order']!r}", "invalid_value" + ) + ) + if "compressor" in doc: + compressor = doc["compressor"] + if compressor is not None and not _is_codec_v2(compressor): + problems.append( + ValidationProblem( + ("compressor",), + "expected null or a codec configuration with a string 'id'", + "invalid_type", + ) + ) + if "filters" in doc: + filters = doc["filters"] + if filters is not None and ( + isinstance(filters, str) + or not isinstance(filters, Sequence) + or not all(_is_codec_v2(item) for item in cast("Sequence[object]", filters)) + ): + problems.append( + ValidationProblem( + ("filters",), + "expected null or a sequence of codec configurations with string 'id's", + "invalid_type", + ) + ) + if "dimension_separator" in doc and doc["dimension_separator"] not in (".", "/"): + problems.append( + ValidationProblem( + ("dimension_separator",), + f"expected '.' or '/', got {doc['dimension_separator']!r}", + "invalid_value", + ) + ) + if "fill_value" in doc: + problems.extend(_prefix("fill_value", validate_json(doc["fill_value"]))) + if "attributes" in doc: + problems.extend(_validate_attributes(doc["attributes"])) + return problems + + +def is_array_metadata_v2(value: object) -> TypeIs[ArrayMetadataV2]: + """Whether `value` is a structurally-valid v2 array metadata document.""" + return not validate_array_metadata_v2(value) + + +def parse_array_metadata_v2(value: object) -> ArrayMetadataV2: + """Return `value` narrowed to `ArrayMetadataV2`, or raise `MetadataValidationError`.""" + problems = validate_array_metadata_v2(value) + if problems: + raise MetadataValidationError(problems) + return cast(ArrayMetadataV2, value) + + +def validate_consolidated_metadata_v3(value: object) -> list[ValidationProblem]: + """Return every reason `value` is not a valid inline consolidated envelope. + + Locs are value-relative (the caller prefixes with `consolidated_metadata` + where appropriate). Entries recurse into the array and group document + validators, so a validator verdict always agrees with what + `ConsolidatedMetadataModelV3.from_json` accepts. + """ + if not isinstance(value, Mapping): + return [ValidationProblem((), "expected a mapping", "invalid_type")] + env = cast("Mapping[str, object]", value) + problems: list[ValidationProblem] = [ + ValidationProblem((key,), "missing required key", "missing_key") + for key in ("kind", "must_understand", "metadata") + if key not in env + ] + problems.extend(_check_literal(env, "kind", "inline")) + if "must_understand" in env and env["must_understand"] is not False: + problems.append(ValidationProblem(("must_understand",), "expected False", "invalid_value")) + if "metadata" in env: + entries = env["metadata"] + if not isinstance(entries, Mapping): + problems.append(ValidationProblem(("metadata",), "expected a mapping", "invalid_type")) + else: + for key, entry in cast("Mapping[object, object]", entries).items(): + if not isinstance(key, str): + problems.append( + ValidationProblem(("metadata",), f"non-string key {key!r}", "invalid_type") + ) + continue + entry_obj: object = entry + node_type: object = None + if isinstance(entry, Mapping): + node_type = cast("Mapping[str, object]", entry).get("node_type") + if node_type == "array": + problems.extend( + _prefix("metadata", _prefix(key, validate_array_metadata_v3(entry_obj))) + ) + elif node_type == "group": + problems.extend( + _prefix("metadata", _prefix(key, validate_group_metadata_v3(entry_obj))) + ) + else: + problems.append( + ValidationProblem( + ("metadata", key, "node_type"), + "expected 'array' or 'group'", + "invalid_value", + ) + ) + return problems + + +def validate_group_metadata_v3(value: object) -> list[ValidationProblem]: + """Return every reason `value` is not a structurally-valid v3 group doc. + + Checks structure, not domain validity. Unknown top-level keys are allowed + (they map to `extra_fields`); a `consolidated_metadata` key, if present, + is deep-validated (envelope and entries) via + `validate_consolidated_metadata_v3`. + """ + if not isinstance(value, Mapping): + return [ValidationProblem((), "expected a mapping", "invalid_type")] + doc = cast("Mapping[str, object]", value) + problems: list[ValidationProblem] = _missing_keys(GROUP_METADATA_REQUIRED_KEYS_V3, doc) + problems.extend(_check_literal(doc, "zarr_format", 3)) + problems.extend(_check_literal(doc, "node_type", "group")) + if "attributes" in doc: + problems.extend(_validate_attributes(doc["attributes"])) + if "consolidated_metadata" in doc and doc["consolidated_metadata"] is not None: + # consolidated_metadata: null (a historical zarr-python bug) is + # structurally accepted so those stores remain readable, but the model + # repairs it to absence on read and never writes it back. + problems.extend( + _prefix( + "consolidated_metadata", + validate_consolidated_metadata_v3(doc["consolidated_metadata"]), + ) + ) + return problems + + +def is_group_metadata_v3(value: object) -> TypeIs[GroupMetadataV3]: + """Whether `value` is a structurally-valid v3 group metadata document.""" + return not validate_group_metadata_v3(value) + + +def parse_group_metadata_v3(value: object) -> GroupMetadataV3: + """Return `value` narrowed to `GroupMetadataV3`, or raise `MetadataValidationError`.""" + problems = validate_group_metadata_v3(value) + if problems: + raise MetadataValidationError(problems) + return cast(GroupMetadataV3, value) + + +def validate_group_metadata_v2(value: object) -> list[ValidationProblem]: + """Return every reason `value` is not a structurally-valid v2 group doc. + + Validates the in-memory merged form: the `.zgroup` fields plus an + optional `attributes` mapping folded in from `.zattrs`. + """ + if not isinstance(value, Mapping): + return [ValidationProblem((), "expected a mapping", "invalid_type")] + doc = cast("Mapping[str, object]", value) + problems: list[ValidationProblem] = _missing_keys(GROUP_METADATA_REQUIRED_KEYS_V2, doc) + problems.extend(_check_literal(doc, "zarr_format", 2)) + if "attributes" in doc: + problems.extend(_validate_attributes(doc["attributes"])) + return problems + + +def is_group_metadata_v2(value: object) -> TypeIs[GroupMetadataV2]: + """Whether `value` is a structurally-valid v2 group metadata document.""" + return not validate_group_metadata_v2(value) + + +def parse_group_metadata_v2(value: object) -> GroupMetadataV2: + """Return `value` narrowed to `GroupMetadataV2`, or raise `MetadataValidationError`.""" + problems = validate_group_metadata_v2(value) + if problems: + raise MetadataValidationError(problems) + return cast(GroupMetadataV2, value) + + +def load_store_json(mapping: Mapping[str, bytes], key: str) -> Any: + """Decode the JSON document stored at `key` in `mapping`. + + Every ingestion failure surfaces as `MetadataValidationError`: a missing + store key is a `missing_key` problem and undecodable bytes are an + `invalid_json` problem, rather than leaking `KeyError` / + `json.JSONDecodeError` to callers. + """ + if key not in mapping: + raise MetadataValidationError( + [ValidationProblem((key,), "missing store key", "missing_key")] + ) + try: + return json.loads(mapping[key]) + except json.JSONDecodeError as exc: + raise MetadataValidationError( + [ValidationProblem((key,), f"invalid JSON: {exc}", "invalid_json")] + ) from exc + + +def arrays_to_tuples(obj: object) -> object: + """Recursively convert every list in a JSON-decoded structure to a tuple.""" + if isinstance(obj, list): + return tuple(arrays_to_tuples(item) for item in cast("list[object]", obj)) + if isinstance(obj, dict): + return { + key: arrays_to_tuples(value) for key, value in cast("dict[object, object]", obj).items() + } + return obj diff --git a/packages/zarr-metadata/src/zarr_metadata/pydantic.py b/packages/zarr-metadata/src/zarr_metadata/pydantic.py new file mode 100644 index 0000000000..3eada76316 --- /dev/null +++ b/packages/zarr-metadata/src/zarr_metadata/pydantic.py @@ -0,0 +1,131 @@ +"""Optional pydantic (v2) integration: field types over the core models. + +Importing this module requires pydantic; the core package deliberately does +not depend on it, so this module is never imported by `zarr_metadata` itself. + +Each exported name is an `Annotated` field type over the corresponding core +model class — the instances ARE the core classes, so values interoperate +freely with non-pydantic code (equality, isinstance, nesting). Validation +delegates to the library: a raw document routes through `from_json` (the +single source of truth for structural validation and normalization, so +pydantic's field-level coercion can never bypass it), an existing model +instance passes through unchanged, and serialization emits the canonical +document via `to_json`. `MetadataValidationError` subclasses `ValueError`, +so a failed parse surfaces as a pydantic `ValidationError` carrying the +loc-annotated problem messages. + +Usage: + + import zarr_metadata.pydantic as zmp + + class ArrayManifest(BaseModel): + path: str + metadata: zmp.ArrayMetadataV3 + +Static type checkers see each field type as its core model class, so +`manifest.metadata` is an `ArrayMetadataModelV3`. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Annotated, TypeVar + +from pydantic import BeforeValidator, InstanceOf, PlainSerializer, WithJsonSchema + +from zarr_metadata.model import ( + ArrayMetadataModelV2, + ArrayMetadataModelV3, + ConsolidatedMetadataModelV2, + ConsolidatedMetadataModelV3, + GroupMetadataModelV2, + GroupMetadataModelV3, + NamedConfigModelV3, +) + +if TYPE_CHECKING: + from collections.abc import Callable + +_M = TypeVar("_M") + + +def _coerce_to(cls: type[_M], parse: Callable[[object], _M]) -> Callable[[object], _M]: + """A validator that passes instances of `cls` through and parses anything else.""" + + def coerce(value: object) -> _M: + if isinstance(value, cls): + return value + return parse(value) + + return coerce + + +# JSON schemas describe the DOCUMENT form each field accepts (the validation +# input), not the in-memory model shape. +_DOCUMENT_SCHEMA = {"type": "object"} +_FIELD_SCHEMA = {"anyOf": [{"type": "string"}, {"type": "object"}]} + +ArrayMetadataV3 = Annotated[ + InstanceOf[ArrayMetadataModelV3], + BeforeValidator(_coerce_to(ArrayMetadataModelV3, ArrayMetadataModelV3.from_json)), + PlainSerializer(ArrayMetadataModelV3.to_json, return_type=dict), + WithJsonSchema(_DOCUMENT_SCHEMA | {"title": "ArrayMetadataV3"}), +] +"""Field type for a v3 array metadata document (`zarr.json` content).""" + +ArrayMetadataV2 = Annotated[ + InstanceOf[ArrayMetadataModelV2], + BeforeValidator(_coerce_to(ArrayMetadataModelV2, ArrayMetadataModelV2.from_json)), + PlainSerializer(ArrayMetadataModelV2.to_json, return_type=dict), + WithJsonSchema(_DOCUMENT_SCHEMA | {"title": "ArrayMetadataV2"}), +] +"""Field type for a v2 array metadata document (merged `.zarray` + `.zattrs` form).""" + +GroupMetadataV3 = Annotated[ + InstanceOf[GroupMetadataModelV3], + BeforeValidator(_coerce_to(GroupMetadataModelV3, GroupMetadataModelV3.from_json)), + PlainSerializer(GroupMetadataModelV3.to_json, return_type=dict), + WithJsonSchema(_DOCUMENT_SCHEMA | {"title": "GroupMetadataV3"}), +] +"""Field type for a v3 group metadata document (`zarr.json` content).""" + +GroupMetadataV2 = Annotated[ + InstanceOf[GroupMetadataModelV2], + BeforeValidator(_coerce_to(GroupMetadataModelV2, GroupMetadataModelV2.from_json)), + PlainSerializer(GroupMetadataModelV2.to_json, return_type=dict), + WithJsonSchema(_DOCUMENT_SCHEMA | {"title": "GroupMetadataV2"}), +] +"""Field type for a v2 group metadata document (merged `.zgroup` + `.zattrs` form).""" + +ConsolidatedMetadataV3 = Annotated[ + InstanceOf[ConsolidatedMetadataModelV3], + BeforeValidator(_coerce_to(ConsolidatedMetadataModelV3, ConsolidatedMetadataModelV3.from_json)), + PlainSerializer(ConsolidatedMetadataModelV3.to_json, return_type=dict), + WithJsonSchema(_DOCUMENT_SCHEMA | {"title": "ConsolidatedMetadataV3"}), +] +"""Field type for v3 inline consolidated metadata.""" + +ConsolidatedMetadataV2 = Annotated[ + InstanceOf[ConsolidatedMetadataModelV2], + BeforeValidator(_coerce_to(ConsolidatedMetadataModelV2, ConsolidatedMetadataModelV2.from_json)), + PlainSerializer(ConsolidatedMetadataModelV2.to_json, return_type=dict), + WithJsonSchema(_DOCUMENT_SCHEMA | {"title": "ConsolidatedMetadataV2"}), +] +"""Field type for a v2 `.zmetadata` document.""" + +MetadataFieldV3 = Annotated[ + InstanceOf[NamedConfigModelV3], + BeforeValidator(_coerce_to(NamedConfigModelV3, NamedConfigModelV3.from_json)), + PlainSerializer(NamedConfigModelV3.to_json, return_type=dict), + WithJsonSchema(_FIELD_SCHEMA | {"title": "MetadataFieldV3"}), +] +"""Field type for one v3 metadata field (bare name string or name + configuration).""" + +__all__ = [ + "ArrayMetadataV2", + "ArrayMetadataV3", + "ConsolidatedMetadataV2", + "ConsolidatedMetadataV3", + "GroupMetadataV2", + "GroupMetadataV3", + "MetadataFieldV3", +] diff --git a/packages/zarr-metadata/src/zarr_metadata/v2/codec.py b/packages/zarr-metadata/src/zarr_metadata/v2/codec.py index 6d194b7e29..8f7f2b3f88 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v2/codec.py +++ b/packages/zarr-metadata/src/zarr_metadata/v2/codec.py @@ -10,7 +10,7 @@ from zarr_metadata._common import JSONValue -class CodecMetadataV2(TypedDict, extra_items=JSONValue): # type: ignore[call-arg] +class CodecMetadataV2(TypedDict, extra_items=JSONValue): """ A numcodecs configuration dict, used as a v2 compressor or filter. diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/array.py b/packages/zarr-metadata/src/zarr_metadata/v3/array.py index a8b0fa3358..a3fec24a6c 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/array.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/array.py @@ -9,7 +9,7 @@ from zarr_metadata.v3._common import MetadataV3 -class ExtensionFieldV3(TypedDict, extra_items=JSONValue): # type: ignore[call-arg] +class ExtensionFieldV3(TypedDict, extra_items=JSONValue): """ Required shape of any extension field on a v3 metadata document. @@ -41,7 +41,7 @@ class ExtensionFieldV3(TypedDict, extra_items=JSONValue): # type: ignore[call-a must_understand: bool -class ArrayMetadataV3(TypedDict, extra_items=ExtensionFieldV3): # type: ignore[call-arg] +class ArrayMetadataV3(TypedDict, extra_items=ExtensionFieldV3): """ Zarr v3 array metadata document (the `zarr.json` content for an array). @@ -63,7 +63,7 @@ class ArrayMetadataV3(TypedDict, extra_items=ExtensionFieldV3): # type: ignore[ dimension_names: NotRequired[tuple[str | None, ...]] -class ArrayMetadataV3Partial(TypedDict, total=False, extra_items=ExtensionFieldV3): # type: ignore[call-arg] +class ArrayMetadataV3Partial(TypedDict, total=False, extra_items=ExtensionFieldV3): """ Partial form of `ArrayMetadataV3`: every field is `NotRequired`. diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/crc32c.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/crc32c.py index ea35ae5f1d..6b9b46c43d 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/crc32c.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/crc32c.py @@ -18,7 +18,7 @@ """Literal type of the `name` field of the `crc32c` codec.""" -class Empty(TypedDict, closed=True): # type: ignore[call-arg] +class Empty(TypedDict, closed=True): """An empty mapping""" diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/group.py b/packages/zarr-metadata/src/zarr_metadata/v3/group.py index 27186b6059..be990b1ae7 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/group.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/group.py @@ -12,7 +12,7 @@ from zarr_metadata.v3.array import ExtensionFieldV3 -class GroupMetadataV3(TypedDict, extra_items=ExtensionFieldV3): # type: ignore[call-arg] +class GroupMetadataV3(TypedDict, extra_items=ExtensionFieldV3): """ Zarr v3 group metadata document (the `zarr.json` content for a group). @@ -26,7 +26,7 @@ class GroupMetadataV3(TypedDict, extra_items=ExtensionFieldV3): # type: ignore[ attributes: NotRequired[Mapping[str, JSONValue]] -class GroupMetadataV3Partial(TypedDict, total=False, extra_items=ExtensionFieldV3): # type: ignore[call-arg] +class GroupMetadataV3Partial(TypedDict, total=False, extra_items=ExtensionFieldV3): """ Partial form of `GroupMetadataV3`: every field is `NotRequired`. diff --git a/packages/zarr-metadata/tests/model/__init__.py b/packages/zarr-metadata/tests/model/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/packages/zarr-metadata/tests/model/_cases.py b/packages/zarr-metadata/tests/model/_cases.py new file mode 100644 index 0000000000..f925c69dd6 --- /dev/null +++ b/packages/zarr-metadata/tests/model/_cases.py @@ -0,0 +1,45 @@ +from __future__ import annotations + +import re +from dataclasses import dataclass +from typing import TYPE_CHECKING, Generic, TypeVar + +import pytest + +if TYPE_CHECKING: + from contextlib import AbstractContextManager + +TIn = TypeVar("TIn") +TOut = TypeVar("TOut") + + +@dataclass(frozen=True) +class Expect(Generic[TIn, TOut]): + """A test case with explicit input, expected output, and a human-readable id.""" + + input: TIn + output: TOut + id: str + + +@dataclass(frozen=True) +class ExpectFail(Generic[TIn]): + """A test case that should raise an exception. + + `msg` is a regex matched against the exception text (pytest's native + `match=` semantics). Leave it `None` to assert only the exception type. Set + `escape=True` when `msg` is a literal that contains regex metacharacters + such as `(`, `[`, or `.`; `escape` has no effect when `msg` is `None`. + """ + + input: TIn + exception: type[Exception] + id: str + msg: str | None = None + escape: bool = False + + def raises(self) -> AbstractContextManager[pytest.ExceptionInfo[Exception]]: + if self.msg is None: + return pytest.raises(self.exception) + pattern = re.escape(self.msg) if self.escape else self.msg + return pytest.raises(self.exception, match=pattern) diff --git a/packages/zarr-metadata/tests/model/test_array.py b/packages/zarr-metadata/tests/model/test_array.py new file mode 100644 index 0000000000..ff27923635 --- /dev/null +++ b/packages/zarr-metadata/tests/model/test_array.py @@ -0,0 +1,1390 @@ +"""Tests for the metadata models in ``zarr_metadata.model``.""" + +import dataclasses +import json +from collections.abc import Callable +from typing import TYPE_CHECKING + +import pytest +from typing_extensions import Unpack + +from tests.model._cases import Expect, ExpectFail +from zarr_metadata.model import ( + ARRAY_METADATA_OPTIONAL_KEYS_V3, + ARRAY_METADATA_REQUIRED_KEYS_V3, + ARRAY_METADATA_STANDARD_KEYS_V3, + UNSET, + ArrayMetadataModelV2, + ArrayMetadataModelV2Partial, + ArrayMetadataModelV3, + ArrayMetadataModelV3Partial, + MetadataFieldModelV3, + MetadataValidationError, + NamedConfigModelV3, + ValidationProblem, + is_array_metadata_v2, + is_array_metadata_v3, + is_json, + is_metadata_field_v3, + parse_array_metadata_v2, + parse_array_metadata_v3, + parse_json, + parse_metadata_field_v3, + validate_array_metadata_v2, + validate_array_metadata_v3, + validate_json, + validate_metadata_field_v3, +) +from zarr_metadata.model._validation import _prefix, arrays_to_tuples + +if TYPE_CHECKING: + from zarr_metadata._common import JSONValue + from zarr_metadata.v2 import CodecMetadataV2 + +# --- public exports -------------------------------------------------------- + + +def test_guards_exported_from_package() -> None: + """The wire-type guard/parser functions are exported from the package.""" + import zarr_metadata.model + + for name in ( + "is_json", + "parse_json", + "is_metadata_field_v3", + "parse_metadata_field_v3", + "is_array_metadata_v3", + "parse_array_metadata_v3", + "is_array_metadata_v2", + "parse_array_metadata_v2", + ): + assert name in zarr_metadata.model.__all__ + assert hasattr(zarr_metadata.model, name) + + +def test_validation_diagnostics_exported_from_package() -> None: + """The validation-diagnostic types and validators are exported from the package.""" + import zarr_metadata.model + + for name in ( + "ValidationProblem", + "MetadataValidationError", + "validate_json", + "validate_metadata_field_v3", + "validate_array_metadata_v3", + "validate_array_metadata_v2", + ): + assert name in zarr_metadata.model.__all__ + assert hasattr(zarr_metadata.model, name) + + +def test_expect_expectfail_smoke() -> None: + """The Expect/ExpectFail test-case dataclasses behave as expected.""" + e = Expect(input=1, output=2, id="x") + assert (e.input, e.output, e.id) == (1, 2, "x") + f = ExpectFail(input=1, exception=ValueError, id="y", msg="boom") + with f.raises(): + raise ValueError("boom") + + +def test_v3_from_json_error_lists_all_problems() -> None: + """A malformed v3 document surfaces every problem via MetadataValidationError.problems.""" + doc: dict[str, object] = dict(ArrayMetadataModelV3.create_default().to_json()) + del doc["shape"] + doc["data_type"] = 5 + with pytest.raises(MetadataValidationError) as exc_info: + ArrayMetadataModelV3.from_json(doc) + locs = {p.loc for p in exc_info.value.problems} + assert ("shape",) in locs + assert ("data_type",) in locs + + +# --- JSON type / fill_value contract --------------------------------------- + + +def test_json_value_type_accepts_json_shapes() -> None: + # JSONValue is the package's public JSON type alias; assigning JSON-shaped + # values to it is valid. + """The JSONValue type alias accepts JSON-shaped values.""" + value: JSONValue = {"a": [1, 2.0, "x", True, None]} + assert value == {"a": [1, 2.0, "x", True, None]} + + +def test_string_nan_fill_value_roundtrips() -> None: + # Non-finite floats are represented as the spec strings ("NaN", "Infinity", + # "-Infinity") by the caller — the metadata layer does not interpret dtypes. + # The string form round-trips cleanly under default dataclass equality, + # unlike a raw float('nan') (which is an invalid fill_value the caller must + # not pass). + """A string 'NaN' fill_value round-trips cleanly (non-finite floats are the caller's responsibility).""" + m = ArrayMetadataModelV3.create_default(fill_value="NaN") + assert ArrayMetadataModelV3.from_json(m.to_json()) == m + assert ArrayMetadataModelV3.from_json(m.to_json()).fill_value == "NaN" + + +# --- NamedConfigModelV3.to_json ------------------------------------------------ + +ZARR_TO_JSON_CASES = [ + Expect( + NamedConfigModelV3(name="regular", configuration={"chunk_shape": [1]}), + {"name": "regular", "configuration": {"chunk_shape": [1]}}, + id="with-configuration", + ), + Expect( + NamedConfigModelV3(name="bytes", configuration={}), + {"name": "bytes", "configuration": {}}, + id="without-configuration", + ), +] + + +@pytest.mark.parametrize("case", ZARR_TO_JSON_CASES, ids=lambda c: c.id) +def test_zarr_metadata_v3_to_json(case: Expect[NamedConfigModelV3, dict[str, object]]) -> None: + """NamedConfigModelV3.to_json emits the canonical object form.""" + assert case.input.to_json() == case.output + + +# --- NamedConfigModelV3.from_json ----------------------------------------------- + +ZARR_FROM_JSON_CASES = [ + Expect("bytes", NamedConfigModelV3(name="bytes", configuration={}), id="bare-string"), + Expect( + {"name": "regular", "configuration": {"chunk_shape": [1]}}, + NamedConfigModelV3(name="regular", configuration={"chunk_shape": (1,)}), + id="object-with-config", + ), + Expect( + {"name": "bytes"}, + NamedConfigModelV3(name="bytes", configuration={}), + id="object-without-config", + ), +] + + +@pytest.mark.parametrize("case", ZARR_FROM_JSON_CASES, ids=lambda c: c.id) +def test_zarr_metadata_v3_from_json(case: Expect[object, NamedConfigModelV3]) -> None: + """NamedConfigModelV3.from_json parses both the bare-string and object forms.""" + assert NamedConfigModelV3.from_json(case.input) == case.output + + +# --- V3 baseline ----------------------------------------------------------- + + +def test_v3_to_json_emits_canonical_document() -> None: + """V3 to_json emits exactly the expected document (which covers every + spec-required key by construction).""" + out = ArrayMetadataModelV3.create_default( + shape=(10,), data_type=NamedConfigModelV3(name="int32", configuration={}) + ).to_json() + assert out == { + "zarr_format": 3, + "node_type": "array", + "shape": (10,), + "fill_value": 0, + "data_type": {"name": "int32", "configuration": {}}, + "chunk_grid": {"name": "regular", "configuration": {"chunk_shape": (10,)}}, + "codecs": ({"name": "bytes", "configuration": {}},), + "chunk_key_encoding": {"name": "default", "configuration": {}}, + } + + +def test_v3_dimension_names_included_when_present() -> None: + """V3 to_json includes dimension_names when they are set.""" + out: dict[str, object] = dict( + ArrayMetadataModelV3.create_default(dimension_names=("x",)).to_json() + ) + assert out["dimension_names"] == ("x",) + + +def test_v3_dimension_names_omitted_when_none() -> None: + """V3 to_json omits dimension_names when they are UNSET.""" + out = ArrayMetadataModelV3.create_default(dimension_names=UNSET).to_json() + assert "dimension_names" not in out + + +# --- BUG 1: attributes gated on dimension_names ---------------------------- + + +def test_v3_attributes_included_when_dimension_names_is_none() -> None: + """Attributes must be emitted regardless of dimension_names. + + Regression: attributes were gated on ``dimension_names is not None``, + so non-empty attributes were silently dropped when there were no + dimension names. + """ + out: dict[str, object] = dict( + ArrayMetadataModelV3.create_default( + dimension_names=UNSET, attributes={"foo": "bar"} + ).to_json() + ) + assert out["attributes"] == {"foo": "bar"} + + +# --- BUG 2: single storage transformer dropped ----------------------------- + + +def test_v3_single_storage_transformer_included() -> None: + """A single storage transformer must be emitted. + + Regression: the guard used ``> 1`` instead of ``> 0``, dropping a + lone storage transformer. + """ + st = NamedConfigModelV3(name="some_transformer", configuration={}) + out: dict[str, object] = dict( + ArrayMetadataModelV3.create_default(storage_transformers=(st,)).to_json() + ) + assert out["storage_transformers"] == ({"name": "some_transformer", "configuration": {}},) + + +def test_v3_no_storage_transformers_omitted() -> None: + """V3 to_json omits storage_transformers when empty.""" + out = ArrayMetadataModelV3.create_default(storage_transformers=()).to_json() + assert "storage_transformers" not in out + + +# --- V3 extra fields ------------------------------------------------------- + + +def test_v3_extra_fields_merged() -> None: + """V3 to_json merges extra_fields into the top-level document.""" + out = ArrayMetadataModelV3.create_default( + extra_fields={"my_ext": {"must_understand": False}} + ).to_json() + assert out["my_ext"] == {"must_understand": False} + + +def test_v3_extra_fields_overlapping_standard_field_rejected() -> None: + """Constructing a V3 model with an extra field that collides with a standard key is rejected.""" + with pytest.raises(ValueError): + ArrayMetadataModelV3.create_default(extra_fields={"shape": {"must_understand": False}}) + + +# --- V3 key/value ---------------------------------------------------------- + + +def test_v3_to_key_value_is_valid_json_under_zarr_json() -> None: + """V3 to_key_value produces valid JSON bytes under the zarr.json key.""" + kv = ArrayMetadataModelV3.create_default(attributes={"a": 1}).to_key_value() + assert set(kv) == {"zarr.json"} + parsed = json.loads(kv["zarr.json"].decode("utf-8")) + assert parsed["zarr_format"] == 3 + assert parsed["attributes"] == {"a": 1} + + +# --- V3 standard-key sets -------------------------------------------------- + + +def test_standard_keys_is_union_of_required_and_optional() -> None: + """The standard-key set is the union of the required and optional key sets.""" + assert ( + ARRAY_METADATA_STANDARD_KEYS_V3 + == ARRAY_METADATA_REQUIRED_KEYS_V3 | ARRAY_METADATA_OPTIONAL_KEYS_V3 + ) + + +def test_standard_keys_contains_known_fields_and_excludes_extensions() -> None: + """The standard-key set contains known fields and excludes extension keys.""" + assert { + "zarr_format", + "node_type", + "shape", + "codecs", + } <= ARRAY_METADATA_STANDARD_KEYS_V3 + assert "my_ext" not in ARRAY_METADATA_STANDARD_KEYS_V3 + + +# --- create_default -------------------------------------------------------- + + +def test_v3_create_default_is_valid_empty_array() -> None: + """V3 create_default builds a structurally valid empty array that round-trips.""" + m = ArrayMetadataModelV3.create_default() + assert m.shape == () + assert m.data_type == NamedConfigModelV3(name="uint8", configuration={}) + assert m.fill_value == 0 + assert m.attributes == {} + assert m.extra_fields == {} + # the default document is structurally valid and round-trips + assert validate_array_metadata_v3(m.to_json()) == [] + assert ArrayMetadataModelV3.from_json(m.to_json()) == m + + +def test_v3_create_default_applies_overrides() -> None: + """V3 create_default applies keyword overrides over the defaults.""" + m = ArrayMetadataModelV3.create_default(shape=(4, 4), attributes={"a": 1}) + assert m.shape == (4, 4) + assert m.attributes == {"a": 1} + # un-overridden fields keep their defaults + assert m.data_type == NamedConfigModelV3(name="uint8", configuration={}) + + +def test_v2_create_default_is_valid_empty_array() -> None: + """V2 create_default builds a structurally valid empty array that round-trips.""" + m = ArrayMetadataModelV2.create_default() + assert m.shape == () + assert m.chunks == () + assert m.fill_value == 0 + assert m.compressor is None + assert m.filters is None + assert m.attributes is UNSET + assert validate_array_metadata_v2(m.to_json()) == [] + assert ArrayMetadataModelV2.from_json(m.to_json()) == m + + +def test_v2_create_default_applies_overrides() -> None: + """V2 create_default applies keyword overrides over the defaults.""" + m = ArrayMetadataModelV2.create_default(shape=(8,), attributes={"k": "v"}) + assert m.shape == (8,) + assert m.attributes == {"k": "v"} + assert m.dtype == "|u1" # default dtype unchanged + + +# --- V3 update ------------------------------------------------------------- + +# Cluster 3: update same-shape pairs across versions — parametrized + +UPDATE_NEW_INSTANCE_PARAMS = [ + pytest.param(ArrayMetadataModelV3, id="v3"), + pytest.param(ArrayMetadataModelV2, id="v2"), +] + + +@pytest.mark.parametrize("model_cls", UPDATE_NEW_INSTANCE_PARAMS) +def test_update_returns_new_instance( + model_cls: type[ArrayMetadataModelV3 | ArrayMetadataModelV2], +) -> None: + """update returns a new instance with the field replaced, leaving the original unchanged.""" + base = model_cls.create_default(shape=(10,)) + updated = base.update(shape=(20,)) + assert updated.shape == (20,) + assert base.shape == (10,) # original unchanged + assert isinstance(updated, model_cls) + + +UPDATE_NO_ARGS_PARAMS = [ + pytest.param(ArrayMetadataModelV3, id="v3"), + pytest.param(ArrayMetadataModelV2, id="v2"), +] + + +@pytest.mark.parametrize("model_cls", UPDATE_NO_ARGS_PARAMS) +def test_update_no_args_returns_equal_model( + model_cls: type[ArrayMetadataModelV3 | ArrayMetadataModelV2], +) -> None: + """update with no arguments returns a model equal to the original.""" + base = model_cls.create_default() + assert base.update() == base + + +# V3-only update tests — kept direct (extra_fields is v3-specific) + + +def test_update_can_replace_extra_fields() -> None: + """update can replace the extra_fields mapping.""" + base = ArrayMetadataModelV3.create_default(extra_fields={}) + updated = base.update(extra_fields={"my_ext": {"must_understand": False}}) + assert updated.extra_fields == {"my_ext": {"must_understand": False}} + + +def test_update_replaces_extra_fields_rather_than_merging() -> None: + """update replaces extra_fields wholesale rather than merging.""" + base = ArrayMetadataModelV3.create_default(extra_fields={"a": {"must_understand": False}}) + updated = base.update(extra_fields={"b": {"must_understand": True}}) + assert updated.extra_fields == {"b": {"must_understand": True}} + + +def test_partial_keys_match_settable_model_fields() -> None: + """The partial TypedDict must list exactly the constructor-settable fields. + + Guards against drift: adding/removing a settable field on the model + without updating ``ArrayMetadataModelV3Partial`` fails here. + """ + settable = {f.name for f in dataclasses.fields(ArrayMetadataModelV3) if f.init} + assert set(ArrayMetadataModelV3Partial.__annotations__) == settable + + +# --- V2 model -------------------------------------------------------------- + + +def test_v2_partial_keys_match_settable_model_fields() -> None: + """The v2 partial TypedDict must list exactly the settable fields.""" + settable = {f.name for f in dataclasses.fields(ArrayMetadataModelV2) if f.init} + assert set(ArrayMetadataModelV2Partial.__annotations__) == settable + + +def test_v2_to_key_value_splits_zarray_and_zattrs() -> None: + """V2 to_key_value splits the document into .zarray and .zattrs.""" + kv = ArrayMetadataModelV2.create_default(attributes={"a": 1}).to_key_value() + assert set(kv) == {".zarray", ".zattrs"} + zarray = json.loads(kv[".zarray"].decode("utf-8")) + zattrs = json.loads(kv[".zattrs"].decode("utf-8")) + assert zarray["zarr_format"] == 2 + assert zattrs == {"a": 1} + + +def test_v2_zarray_excludes_attributes() -> None: + """The on-disk ``.zarray`` document must not contain user attributes. + + In v2, attributes live only in the sibling ``.zattrs`` file. The bundled + ``ArrayMetadataV2`` / ``to_json()`` carry attributes for convenience, but + ``to_key_value()`` must split them out. + """ + kv = ArrayMetadataModelV2.create_default(attributes={"a": 1}).to_key_value() + zarray = json.loads(kv[".zarray"].decode("utf-8")) + assert "attributes" not in zarray + + +def test_v2_to_json_still_includes_attributes() -> None: + """``to_json()`` is the bundled in-memory form and keeps attributes.""" + out: dict[str, object] = dict( + ArrayMetadataModelV2.create_default(attributes={"a": 1}).to_json() + ) + assert out["attributes"] == {"a": 1} + + +# --- arrays_to_tuples helper ---------------------------------------------- + +ARRAYS_TO_TUPLES_CASES = [ + Expect([1, 2, 3], (1, 2, 3), id="top-level-list"), + Expect({"a": [1, [2, 3]], "b": "x"}, {"a": (1, (2, 3)), "b": "x"}, id="nested-in-dict"), + Expect(5, 5, id="scalar-int"), + Expect("s", "s", id="scalar-str"), + Expect(None, None, id="scalar-none"), + Expect( + {"name": "bytes", "configuration": {"nums": [1, 2]}}, + {"name": "bytes", "configuration": {"nums": (1, 2)}}, + id="dict-keys-preserved", + ), +] + + +@pytest.mark.parametrize("case", ARRAYS_TO_TUPLES_CASES, ids=lambda c: c.id) +def test_arrays_to_tuples(case: Expect[object, object]) -> None: + """arrays_to_tuples recursively converts JSON arrays to tuples.""" + assert arrays_to_tuples(case.input) == case.output + + +# --- ArrayMetadataModelV3.from_json ---------------------------------------- + + +def test_v3_from_json_reconstructs_required_fields() -> None: + """V3 from_json reconstructs the required fields from a document.""" + doc = ArrayMetadataModelV3.create_default( + shape=(7,), + attributes={"a": 1}, + data_type=NamedConfigModelV3(name="int32", configuration={}), + ).to_json() + model = ArrayMetadataModelV3.from_json(doc) + assert model.shape == (7,) + assert model.data_type == NamedConfigModelV3(name="int32", configuration={}) + assert model.attributes == {"a": 1} + + +def test_v3_from_json_defaults_for_omitted_optionals() -> None: + """V3 from_json supplies defaults for omitted optional fields.""" + doc = ArrayMetadataModelV3.create_default( + attributes={}, storage_transformers=(), dimension_names=UNSET + ).to_json() + # to_json omits these entirely; from_json must restore defaults + model = ArrayMetadataModelV3.from_json(doc) + assert model.attributes == {} + assert model.storage_transformers == () + assert model.dimension_names is UNSET + + +def test_v3_from_json_routes_unknown_keys_to_extra_fields() -> None: + """V3 from_json routes unknown top-level keys into extra_fields.""" + doc = ArrayMetadataModelV3.create_default( + extra_fields={"my_ext": {"must_understand": False}} + ).to_json() + model = ArrayMetadataModelV3.from_json(doc) + assert model.extra_fields == {"my_ext": {"must_understand": False}} + + +def test_v3_from_json_standard_keys_not_in_extra_fields() -> None: + """V3 from_json keeps standard keys out of extra_fields.""" + doc = ArrayMetadataModelV3.create_default( + shape=(10,), attributes={"a": 1}, dimension_names=("x",) + ).to_json() + model = ArrayMetadataModelV3.from_json(doc) + assert model.extra_fields == {} + + +def test_v3_from_json_nested_arrays_in_attributes_become_tuples() -> None: + """V3 from_json converts nested arrays in attributes into tuples.""" + doc = ArrayMetadataModelV3.create_default(attributes={"scale": [[1, 2], [3, 4]]}).to_json() + model = ArrayMetadataModelV3.from_json(doc) + assert model.attributes == {"scale": ((1, 2), (3, 4))} + + +# --- ArrayMetadataModelV3.from_key_value ---------------------------------- + + +def test_v3_from_key_value_parses_zarr_json() -> None: + """V3 from_key_value parses the zarr.json entry into a model.""" + kv = ArrayMetadataModelV3.create_default(shape=(3,)).to_key_value() + model = ArrayMetadataModelV3.from_key_value(kv) + assert model.shape == (3,) + + +# --- Cluster 2: from_key_value missing-key raises (parametrized) ----------- + +FROM_KEY_VALUE_MISSING_PARAMS = [ + pytest.param( + ArrayMetadataModelV3, + ExpectFail({}, MetadataValidationError, id="v3-missing-zarr-json", msg="missing store key"), + id="v3-missing-zarr-json", + ), + pytest.param( + ArrayMetadataModelV2, + ExpectFail({}, MetadataValidationError, id="v2-missing-zarray", msg="missing store key"), + id="v2-missing-zarray", + ), +] + + +@pytest.mark.parametrize(("model_cls", "case"), FROM_KEY_VALUE_MISSING_PARAMS) +def test_from_key_value_missing_key_raises( + model_cls: type[ArrayMetadataModelV3 | ArrayMetadataModelV2], + case: ExpectFail[dict[str, bytes]], +) -> None: + """from_key_value raises MetadataValidationError when the required store key is absent.""" + with case.raises(): + model_cls.from_key_value(case.input) + + +# --- Cluster 1: round-trips (model → json → model, parametrized) ----------- + +ROUNDTRIP_MODEL_JSON_PARAMS = [ + pytest.param( + ArrayMetadataModelV3, + ArrayMetadataModelV3.create_default( + shape=(10,), + attributes={"a": 1}, + dimension_names=("x",), + storage_transformers=(NamedConfigModelV3(name="t", configuration={}),), + extra_fields={"ext": {"must_understand": False}}, + ), + id="v3-full", + ), + pytest.param( + ArrayMetadataModelV3, + ArrayMetadataModelV3.create_default( + attributes={}, + dimension_names=UNSET, + storage_transformers=(), + extra_fields={}, + ), + id="v3-empty-optionals", + ), + pytest.param( + ArrayMetadataModelV2, + ArrayMetadataModelV2.create_default(attributes={"a": 1}, filters=None, compressor=None), + id="v2-basic", + ), +] + + +@pytest.mark.parametrize(("model_cls", "model"), ROUNDTRIP_MODEL_JSON_PARAMS) +def test_roundtrip_model_json_model( + model_cls: type[ArrayMetadataModelV3 | ArrayMetadataModelV2], + model: ArrayMetadataModelV3 | ArrayMetadataModelV2, +) -> None: + """A model round-trips through to_json/from_json back to an equal model.""" + assert model_cls.from_json(model.to_json()) == model + + +# --- Round-trips (model → key_value → model, parametrized) ----------------- + +ROUNDTRIP_KEY_VALUE_PARAMS = [ + pytest.param( + ArrayMetadataModelV3, + ArrayMetadataModelV3.create_default(attributes={"a": 1}), + id="v3", + ), + pytest.param( + ArrayMetadataModelV2, + ArrayMetadataModelV2.create_default(attributes={"a": 1}), + id="v2", + ), +] + + +@pytest.mark.parametrize(("model_cls", "model"), ROUNDTRIP_KEY_VALUE_PARAMS) +def test_roundtrip_via_key_value( + model_cls: type[ArrayMetadataModelV3 | ArrayMetadataModelV2], + model: ArrayMetadataModelV3 | ArrayMetadataModelV2, +) -> None: + """A model round-trips through to_key_value/from_key_value back to an equal model.""" + assert model_cls.from_key_value(model.to_key_value()) == model + + +# --- Round-trips (json → model → json, direction distinct — kept direct) --- + + +def test_v3_roundtrip_json_model_json() -> None: + """A v3 document round-trips through from_json/to_json back to an equal document.""" + doc = ArrayMetadataModelV3.create_default( + shape=(10,), attributes={"a": 1}, dimension_names=("x",) + ).to_json() + assert ArrayMetadataModelV3.from_json(doc).to_json() == doc + + +def test_v2_roundtrip_json_model_json() -> None: + """A v2 document round-trips through from_json/to_json back to an equal document.""" + doc = ArrayMetadataModelV2.create_default(attributes={"a": 1}).to_json() + assert ArrayMetadataModelV2.from_json(doc).to_json() == doc + + +def test_v3_parser_accepts_bare_string_data_type() -> None: + """V3 from_json accepts a bare-string data_type and re-serializes it canonically.""" + doc = ArrayMetadataModelV3.create_default().to_json() + doc["data_type"] = "int32" # bare-string form, not canonical object form + model = ArrayMetadataModelV3.from_json(doc) + # parses correctly, re-serializes to canonical object form + assert model.data_type == NamedConfigModelV3(name="int32", configuration={}) + assert model.to_json()["data_type"] == {"name": "int32", "configuration": {}} + + +def test_v2_roundtrip_with_compressor_and_filters() -> None: + # Non-None compressor/filters must round-trip; extra assertion on .compressor. + """A v2 model with non-None compressor and filters round-trips.""" + compressor: CodecMetadataV2 = {"id": "blosc", "clevel": 5} + filters: tuple[CodecMetadataV2, ...] = ({"id": "delta"},) + m = ArrayMetadataModelV2.create_default(compressor=compressor, filters=filters) + restored = ArrayMetadataModelV2.from_json(m.to_json()) + assert restored == m + assert restored.compressor == {"id": "blosc", "clevel": 5} + + +# --- ArrayMetadataModelV2.from_json ---------------------------------------- + + +def test_v2_from_json_reconstructs_fields() -> None: + """V2 from_json reconstructs the fields from a document.""" + doc = ArrayMetadataModelV2.create_default( + shape=(4,), attributes={"a": 1}, dtype=" None: + """V2 from_json reads an absent attributes key as UNSET, distinct from an + explicit empty mapping.""" + absent = ArrayMetadataModelV2.from_json(ArrayMetadataModelV2.create_default().to_json()) + explicit = ArrayMetadataModelV2.from_json( + ArrayMetadataModelV2.create_default(attributes={}).to_json() + ) + assert absent.attributes is UNSET + assert explicit.attributes == {} + assert absent != explicit + + +# --- ArrayMetadataModelV2.from_key_value -------------------------------- + + +def test_v2_from_key_value_remerges_zattrs() -> None: + """V2 from_key_value re-merges .zattrs back into attributes.""" + kv = ArrayMetadataModelV2.create_default(attributes={"a": 1}, shape=(10,)).to_key_value() + model = ArrayMetadataModelV2.from_key_value(kv) + assert model.attributes == {"a": 1} + assert model.shape == (10,) + + +def test_v2_zattrs_presence_round_trips() -> None: + """The .zattrs file's presence is part of the store: an absent file reads + as UNSET and emits no .zattrs; an explicit empty file reads as {} and + emits .zattrs — the two stores stay distinct through a round-trip.""" + explicit_kv = dict(ArrayMetadataModelV2.create_default(attributes={}).to_key_value()) + assert ".zattrs" in explicit_kv + absent_kv = dict(explicit_kv) + del absent_kv[".zattrs"] + + absent = ArrayMetadataModelV2.from_key_value(absent_kv) + explicit = ArrayMetadataModelV2.from_key_value(explicit_kv) + assert absent.attributes is UNSET + assert explicit.attributes == {} + assert ".zattrs" not in absent.to_key_value() + assert ".zattrs" in explicit.to_key_value() + + +def test_v2_from_json_nested_arrays_in_attributes_become_tuples() -> None: + """V2 from_json converts nested arrays in attributes into tuples.""" + doc = ArrayMetadataModelV2.create_default(attributes={"axes": [[0, 1], [2, 3]]}).to_json() + model = ArrayMetadataModelV2.from_json(doc) + assert model.attributes == {"axes": ((0, 1), (2, 3))} + + +# --- scalar wire-type guards (is_/validate_/parse_) ------------------------ +# +# Each value is modelled once as Expect[object, frozenset[tuple[str | int, ...]]] +# where `output` is the set of expected problem locs validate_* must report — +# frozenset() means VALID. Valid iff output == frozenset(). + +JSON_VALIDATE_CASES: list[Expect[object, frozenset[tuple[str | int, ...]]]] = [ + Expect("s", frozenset(), id="str"), + Expect(1, frozenset(), id="int"), + Expect(1.5, frozenset(), id="float"), + Expect(True, frozenset(), id="bool"), + Expect(None, frozenset(), id="none"), + Expect({"a": [1, {"b": None}], "c": "x"}, frozenset(), id="nested-containers"), + Expect((1, 2, 3), frozenset(), id="tuple-array"), + Expect(float("nan"), frozenset(), id="nan"), + Expect(float("inf"), frozenset(), id="inf"), + Expect(object(), frozenset({()}), id="object"), + Expect(b"abc", frozenset({()}), id="bytes"), + Expect(bytearray(b"abc"), frozenset({()}), id="bytearray"), + Expect({1: "x"}, frozenset({()}), id="non-str-key"), + Expect([1, object()], frozenset({(1,)}), id="non-json-list-item"), + Expect({"ok": object()}, frozenset({("ok",)}), id="non-json-value"), +] + + +@pytest.mark.parametrize("case", JSON_VALIDATE_CASES, ids=lambda c: c.id) +def test_is_json(case: Expect[object, frozenset[tuple[str | int, ...]]]) -> None: + """is_json reports whether a value is JSON-serializable.""" + assert is_json(case.input) is (case.output == frozenset()) + + +@pytest.mark.parametrize("case", JSON_VALIDATE_CASES, ids=lambda c: c.id) +def test_validate_json(case: Expect[object, frozenset[tuple[str | int, ...]]]) -> None: + """validate_json reports the problems (and their locs) for a value.""" + problems = validate_json(case.input) + assert (problems == []) is (case.output == frozenset()) + assert {p.loc for p in problems} >= case.output + + +@pytest.mark.parametrize("case", JSON_VALIDATE_CASES, ids=lambda c: c.id) +def test_parse_json(case: Expect[object, frozenset[tuple[str | int, ...]]]) -> None: + """parse_json returns valid JSON values and raises on invalid ones.""" + if case.output == frozenset(): + assert parse_json(case.input) is case.input + else: + with pytest.raises(MetadataValidationError): + parse_json(case.input) + + +def test_validate_json_reports_json_in_message() -> None: + """validate_json's message for a non-JSON value mentions JSON.""" + problems = validate_json(object()) + assert problems[0].loc == () + assert "JSON" in problems[0].message + + +METADATA_FIELD_VALIDATE_CASES: list[Expect[object, frozenset[tuple[str | int, ...]]]] = [ + Expect("bytes", frozenset(), id="bare-string"), + Expect({"name": "x", "configuration": {"a": 1}}, frozenset(), id="named-config"), + Expect({"name": "bytes"}, frozenset(), id="name-only"), + Expect(5, frozenset({()}), id="not-str-or-mapping"), + Expect({"configuration": {}}, frozenset({("name",)}), id="missing-name"), + Expect({"name": 3}, frozenset({("name",)}), id="non-str-name"), + Expect( + {"name": "x", "configuration": [1]}, + frozenset({("configuration",)}), + id="config-not-mapping", + ), + Expect( + {"name": "x", "configuration": {1: "y"}}, + frozenset({("configuration",)}), + id="config-non-str-key", + ), +] + + +@pytest.mark.parametrize("case", METADATA_FIELD_VALIDATE_CASES, ids=lambda c: c.id) +def test_is_metadata_field_v3(case: Expect[object, frozenset[tuple[str | int, ...]]]) -> None: + """is_metadata_field_v3 reports whether a value is a v3 metadata field.""" + assert is_metadata_field_v3(case.input) is (case.output == frozenset()) + + +@pytest.mark.parametrize("case", METADATA_FIELD_VALIDATE_CASES, ids=lambda c: c.id) +def test_validate_metadata_field_v3( + case: Expect[object, frozenset[tuple[str | int, ...]]], +) -> None: + """validate_metadata_field_v3 reports the problems for a metadata-field value.""" + problems = validate_metadata_field_v3(case.input) + assert (problems == []) is (case.output == frozenset()) + assert {p.loc for p in problems} >= case.output + + +@pytest.mark.parametrize("case", METADATA_FIELD_VALIDATE_CASES, ids=lambda c: c.id) +def test_parse_metadata_field_v3( + case: Expect[object, frozenset[tuple[str | int, ...]]], +) -> None: + """parse_metadata_field_v3 returns valid fields and raises on invalid ones.""" + if case.output == frozenset(): + assert parse_metadata_field_v3(case.input) is case.input + else: + with pytest.raises(MetadataValidationError): + parse_metadata_field_v3(case.input) + + +# --- array-document wire-type guards (is_/validate_/parse_) ---------------- +# +# Each case starts from a valid document (built by `make`) and applies a +# mutation. `expected_locs` are loc paths `validate_*` must report for the +# invalid cases (a subset check, so accumulation of OTHER problems is allowed). + + +def _build_v3(**overrides: Unpack[ArrayMetadataModelV3Partial]) -> dict[str, object]: + return dict(ArrayMetadataModelV3.create_default(**overrides).to_json()) + + +def _build_v2(**overrides: Unpack[ArrayMetadataModelV2Partial]) -> dict[str, object]: + return dict(ArrayMetadataModelV2.create_default(**overrides).to_json()) + + +def _mutate(build: Callable[[], dict], mutate: Callable[[dict], object]) -> Callable[[], dict]: + def _factory() -> dict: + doc = build() + mutate(doc) + return doc + + return _factory + + +def _del(key: str) -> Callable[[dict], object]: + return lambda doc: doc.pop(key) + + +def _set(key: str, value: object) -> Callable[[dict], object]: + return lambda doc: doc.__setitem__(key, value) + + +V3_DOC_CASES: list[Expect[Callable[[], object], frozenset[tuple[str | int, ...]]]] = [ + Expect(_build_v3, frozenset(), id="valid"), + Expect( + lambda: _build_v3(shape=(10,), attributes={"a": 1}, dimension_names=("x",)), + frozenset(), + id="valid-with-attributes-and-dim-names", + ), + Expect( + lambda: _build_v3(extra_fields={"my_ext": {"must_understand": False}}), + frozenset(), + id="valid-with-extra-fields", + ), + Expect(_mutate(_build_v3, _del("shape")), frozenset({("shape",)}), id="missing-shape"), + Expect( + _mutate(_build_v3, _set("data_type", 5)), + frozenset({("data_type",)}), + id="bad-data-type", + ), + Expect( + _mutate(_build_v3, _set("shape", "not-a-shape")), + frozenset({("shape",)}), + id="shape-not-sequence", + ), + Expect( + _mutate(_build_v3, _set("shape", [1, "x"])), + frozenset({("shape",)}), + id="shape-non-int-item", + ), + Expect( + _mutate(_build_v3, _set("codecs", (5,))), + frozenset({("codecs", 0)}), + id="bad-codec-entry", + ), + Expect(lambda: [1, 2, 3], frozenset({()}), id="non-mapping-list"), + Expect(lambda: "nope", frozenset({()}), id="non-mapping-str"), + Expect( + _mutate(_mutate(_build_v3, _del("shape")), _set("data_type", 5)), + frozenset({("shape",), ("data_type",)}), + id="missing-shape-and-bad-data-type", + ), +] + +V2_DOC_CASES: list[Expect[Callable[[], object], frozenset[tuple[str | int, ...]]]] = [ + Expect(_build_v2, frozenset(), id="valid"), + Expect(lambda: _build_v2(attributes={"a": 1}), frozenset(), id="valid-with-attributes"), + Expect( + lambda: _build_v2(compressor=None, filters=None), + frozenset(), + id="valid-none-compressor-filters", + ), + Expect( + _mutate(_build_v2, _del("chunks")), + frozenset({("chunks",)}), + id="missing-chunks", + ), + Expect( + _mutate(_build_v2, _set("shape", [1, "x"])), + frozenset({("shape",)}), + id="bad-shape", + ), + Expect( + _mutate(_mutate(_build_v2, _del("chunks")), _set("shape", [1, "x"])), + frozenset({("chunks",), ("shape",)}), + id="missing-chunks-and-bad-shape", + ), +] + +ALL_DOC_CASES = [ + *( + pytest.param( + is_array_metadata_v3, + validate_array_metadata_v3, + parse_array_metadata_v3, + c, + id=f"v3-{c.id}", + ) + for c in V3_DOC_CASES + ), + *( + pytest.param( + is_array_metadata_v2, + validate_array_metadata_v2, + parse_array_metadata_v2, + c, + id=f"v2-{c.id}", + ) + for c in V2_DOC_CASES + ), +] + + +@pytest.mark.parametrize(("is_fn", "validate_fn", "parse_fn", "case"), ALL_DOC_CASES) +def test_array_metadata_guards( + is_fn: Callable[[object], bool], + validate_fn: Callable[[object], list[ValidationProblem]], + parse_fn: Callable[[object], object], + case: Expect[Callable[[], object], frozenset[tuple[str | int, ...]]], +) -> None: + """is_/validate_/parse_ array-metadata guards agree on validity and locs for each case.""" + doc = case.input() + valid = case.output == frozenset() + assert is_fn(doc) is valid + problems = validate_fn(doc) + assert (problems == []) is valid + assert {p.loc for p in problems} >= case.output + if valid: + assert parse_fn(doc) is doc + else: + with pytest.raises(MetadataValidationError): + parse_fn(doc) + + +# --- strict from_json validation ------------------------------------------- + + +FROM_JSON_REJECT_PARAMS = [ + pytest.param( + ArrayMetadataModelV3, + ExpectFail(lambda: {"zarr_format": 3}, MetadataValidationError, id="x"), + id="v3-missing-required", + ), + pytest.param( + ArrayMetadataModelV3, + ExpectFail(_mutate(_build_v3, _set("data_type", 5)), MetadataValidationError, id="x"), + id="v3-bad-field-type", + ), + pytest.param( + ArrayMetadataModelV2, + ExpectFail(lambda: {"zarr_format": 2}, MetadataValidationError, id="x"), + id="v2-missing-required", + ), + pytest.param( + NamedConfigModelV3, + ExpectFail(lambda: 5, MetadataValidationError, id="x"), + id="zarr-metadata-bad-input", + ), +] + + +@pytest.mark.parametrize(("model", "case"), FROM_JSON_REJECT_PARAMS) +def test_from_json_rejects_malformed( + model: type[ArrayMetadataModelV3 | ArrayMetadataModelV2 | NamedConfigModelV3], + case: ExpectFail[Callable[[], object]], +) -> None: + """from_json raises MetadataValidationError on a malformed document.""" + with case.raises(): + model.from_json(case.input()) + + +# --- ValidationProblem / MetadataValidationError / _prefix ----------------- +# Small structural tests — not "parametrize over inputs" shaped, kept direct. + + +def test_validation_problem_str_with_loc() -> None: + """ValidationProblem.__str__ renders a non-empty loc as a dotted path.""" + p = ValidationProblem(loc=("codecs", 0, "name"), message="expected str", kind="invalid_type") + assert str(p) == "codecs.0.name: expected str" + + +def test_validation_problem_str_empty_loc() -> None: + """ValidationProblem.__str__ renders an empty loc as .""" + p = ValidationProblem(loc=(), message="not a mapping", kind="invalid_type") + assert str(p) == ": not a mapping" + + +def test_validation_problem_is_frozen() -> None: + """ValidationProblem is immutable (frozen dataclass).""" + p = ValidationProblem(loc=("shape",), message="x", kind="invalid_type") + with pytest.raises(dataclasses.FrozenInstanceError): + # setattr: assigning to a frozen field is an intentional runtime error, + # spelled dynamically so it is not also a static type error. + setattr(p, "message", "y") # noqa: B010 + + +def test_metadata_validation_error_holds_problems() -> None: + """MetadataValidationError carries its problem list and renders them in its message.""" + problems = [ + ValidationProblem(loc=("shape",), message="missing required key", kind="missing_key"), + ValidationProblem( + loc=("data_type",), message="expected a metadata field", kind="invalid_type" + ), + ] + err = MetadataValidationError(problems) + assert err.problems == problems + assert "shape: missing required key" in str(err) + assert "data_type: expected a metadata field" in str(err) + + +def test_prefix_prepends_loc_head() -> None: + """_prefix prepends a loc head to each problem's loc.""" + problems = [ValidationProblem(loc=("name",), message="expected str", kind="invalid_type")] + prefixed = _prefix(0, problems) + assert prefixed == [ + ValidationProblem(loc=(0, "name"), message="expected str", kind="invalid_type") + ] + + +# --- Stricter v2/v3 field validation and error kinds ------------------------- + + +def test_v2_dtype_must_be_string_or_records() -> None: + """A non-string, non-records v2 dtype is rejected with an invalid_type problem.""" + doc = dict(ArrayMetadataModelV2.create_default().to_json()) | {"dtype": 42} + problems = validate_array_metadata_v2(doc) + assert [(p.loc, p.kind) for p in problems] == [(("dtype",), "invalid_type")] + + +def test_v2_structured_dtype_records_accepted() -> None: + """A structured v2 dtype (field records, optionally nested/shaped) validates.""" + dtype = (("a", " None: + """A field record with the wrong arity is rejected.""" + doc = dict(ArrayMetadataModelV2.create_default().to_json()) | {"dtype": (("a",),)} + problems = validate_array_metadata_v2(doc) + assert [p.loc for p in problems] == [("dtype",)] + + +def test_v2_order_literal_enforced() -> None: + """An order other than 'C' or 'F' is rejected with an invalid_value problem.""" + doc = dict(ArrayMetadataModelV2.create_default().to_json()) | {"order": "Q"} + problems = validate_array_metadata_v2(doc) + assert [(p.loc, p.kind) for p in problems] == [(("order",), "invalid_value")] + + +def test_v2_compressor_must_be_codec_or_none() -> None: + """A compressor that is not null or a codec config mapping is rejected.""" + doc = dict(ArrayMetadataModelV2.create_default().to_json()) | {"compressor": "zlib"} + problems = validate_array_metadata_v2(doc) + assert [(p.loc, p.kind) for p in problems] == [(("compressor",), "invalid_type")] + + +def test_v2_compressor_requires_string_id() -> None: + """A compressor mapping without a string id is rejected.""" + doc = dict(ArrayMetadataModelV2.create_default().to_json()) | {"compressor": {"level": 3}} + problems = validate_array_metadata_v2(doc) + assert [p.loc for p in problems] == [("compressor",)] + + +def test_v2_filters_must_be_codec_sequence_or_none() -> None: + """Filters that are not null or a sequence of codec configs are rejected.""" + for bad in (7, (5,), "gzip"): + doc = dict(ArrayMetadataModelV2.create_default().to_json()) | {"filters": bad} + problems = validate_array_metadata_v2(doc) + assert [(p.loc, p.kind) for p in problems] == [(("filters",), "invalid_type")], bad + + +def test_v2_dimension_separator_literal_enforced() -> None: + """A dimension_separator other than '.' or '/' is rejected.""" + doc = dict(ArrayMetadataModelV2.create_default().to_json()) | {"dimension_separator": "-"} + problems = validate_array_metadata_v2(doc) + assert [(p.loc, p.kind) for p in problems] == [(("dimension_separator",), "invalid_value")] + + +def test_v2_zarr_format_literal_enforced() -> None: + """A v2 document claiming zarr_format 3 is rejected with an invalid_value problem.""" + doc = dict(ArrayMetadataModelV2.create_default().to_json()) | {"zarr_format": 3} + problems = validate_array_metadata_v2(doc) + assert [(p.loc, p.kind) for p in problems] == [(("zarr_format",), "invalid_value")] + + +def test_v3_zarr_format_literal_enforced() -> None: + """A v3 document claiming zarr_format 2 is rejected with an invalid_value problem.""" + doc = dict(ArrayMetadataModelV3.create_default().to_json()) | {"zarr_format": 2} + problems = validate_array_metadata_v3(doc) + assert [(p.loc, p.kind) for p in problems] == [(("zarr_format",), "invalid_value")] + + +def test_v3_node_type_literal_enforced() -> None: + """A v3 array document claiming node_type 'group' is rejected.""" + doc = dict(ArrayMetadataModelV3.create_default().to_json()) | {"node_type": "group"} + problems = validate_array_metadata_v3(doc) + assert [(p.loc, p.kind) for p in problems] == [(("node_type",), "invalid_value")] + + +def test_missing_key_kind_is_machine_readable() -> None: + """A missing required key is distinguishable by kind, without message matching.""" + doc = dict(ArrayMetadataModelV3.create_default().to_json()) + del doc["chunk_key_encoding"] + problems = validate_array_metadata_v3(doc) + assert problems == [ + ValidationProblem(("chunk_key_encoding",), "missing required key", "missing_key") + ] + + +# --- Unified error channels --------------------------------------------------- + + +def test_from_key_value_invalid_json_raises_metadata_error() -> None: + """Undecodable store bytes raise MetadataValidationError (kind invalid_json), not JSONDecodeError.""" + with pytest.raises(MetadataValidationError) as exc_info: + ArrayMetadataModelV3.from_key_value({"zarr.json": b"{not json"}) + assert [p.kind for p in exc_info.value.problems] == ["invalid_json"] + + +def test_from_key_value_missing_key_kind() -> None: + """A missing store key surfaces as a missing_key problem at the store-key loc.""" + with pytest.raises(MetadataValidationError) as exc_info: + ArrayMetadataModelV2.from_key_value({}) + assert exc_info.value.problems == [ + ValidationProblem((".zarray",), "missing store key", "missing_key") + ] + + +def test_extra_fields_overlap_raises_metadata_error() -> None: + """The extra-fields overlap invariant raises MetadataValidationError (a ValueError).""" + with pytest.raises(MetadataValidationError, match="Extra fields") as exc_info: + ArrayMetadataModelV3.create_default(extra_fields={"shape": {"must_understand": False}}) + assert [p.kind for p in exc_info.value.problems] == ["invalid_value"] + + +def test_extension_point_fields_annotated_with_role_alias() -> None: + """Extension-point fields are annotated with MetadataFieldModelV3 (the + logical role), not NamedConfigModelV3 (the current serialized form), so a + future widening of the field union does not move annotation sites.""" + assert MetadataFieldModelV3 is NamedConfigModelV3 + annotations = ArrayMetadataModelV3.__annotations__ + for field_name in ("data_type", "chunk_grid", "chunk_key_encoding"): + assert annotations[field_name] == "MetadataFieldModelV3" + for field_name in ("codecs", "storage_transformers"): + assert annotations[field_name] == "tuple[MetadataFieldModelV3, ...]" + + +# --- Adversarial-probe fixes: documents that used to pass validation --------- + + +def test_shape_rejects_json_booleans() -> None: + """JSON booleans are not integers: shape/chunks containing true/false are + rejected (bool is an int subclass in Python, so isinstance alone passes).""" + v3 = dict(ArrayMetadataModelV3.create_default().to_json()) | {"shape": (True, True)} + assert [p.loc for p in validate_array_metadata_v3(v3)] == [("shape",)] + v2 = dict(ArrayMetadataModelV2.create_default().to_json()) | {"chunks": (True,)} + assert [p.loc for p in validate_array_metadata_v2(v2)] == [("chunks",)] + + +def test_shape_rejects_negative_dimensions() -> None: + """Dimension lengths must be non-negative; a negative entry is invalid_value.""" + v3 = dict(ArrayMetadataModelV3.create_default().to_json()) | {"shape": (-1,)} + assert [(p.loc, p.kind) for p in validate_array_metadata_v3(v3)] == [ + (("shape",), "invalid_value") + ] + v2 = dict(ArrayMetadataModelV2.create_default().to_json()) | {"chunks": (-5,)} + assert [(p.loc, p.kind) for p in validate_array_metadata_v2(v2)] == [ + (("chunks",), "invalid_value") + ] + + +def test_dimension_names_length_must_match_shape() -> None: + """dimension_names must have one entry per dimension of shape.""" + doc = dict(ArrayMetadataModelV3.create_default(shape=(10,)).to_json()) | { + "dimension_names": ("x", "y", "z") + } + assert [(p.loc, p.kind) for p in validate_array_metadata_v3(doc)] == [ + (("dimension_names",), "invalid_value") + ] + + +def test_attributes_values_must_be_json() -> None: + """Attribute values are JSON-checked recursively (like fill_value), so a + non-serializable value is a validation problem, not a later TypeError.""" + doc = dict(ArrayMetadataModelV3.create_default().to_json()) | {"attributes": {"a": {1, 2}}} + problems = validate_array_metadata_v3(doc) + assert [(p.loc, p.kind) for p in problems] == [(("attributes", "a"), "invalid_type")] + + +def test_configuration_values_must_be_json() -> None: + """Configuration values are JSON-checked recursively, so an int-keyed dict + cannot pass validation and be silently rewritten by json.dumps.""" + doc = dict(ArrayMetadataModelV3.create_default().to_json()) | { + "chunk_grid": {"name": "regular", "configuration": {"chunk_shape": {1: 2}}} + } + problems = validate_array_metadata_v3(doc) + assert [(p.loc, p.kind) for p in problems] == [ + (("chunk_grid", "configuration", "chunk_shape"), "invalid_type") + ] + + +# --- must_understand partition (spec: MUST fail to open unrecognized fields) -- + + +def test_must_understand_fields_partition() -> None: + """must_understand_fields contains every extra field not explicitly waived + with must_understand: false, including implicitly-true and non-mapping + fields, so a reader can discharge the spec's fail-to-open duty by + subtracting the extensions it recognizes.""" + model = ArrayMetadataModelV3.create_default( + extra_fields={ + "ext_a": {"name": "a", "must_understand": False}, + "ext_b": {"name": "b"}, + "ext_c": {"name": "c", "must_understand": True}, + "ext_d": 123, + } + ) + assert set(model.must_understand_fields) == {"ext_b", "ext_c", "ext_d"} + recognized = {"ext_b"} + assert model.must_understand_fields.keys() - recognized == {"ext_c", "ext_d"} + + +def test_must_understand_fields_empty_when_all_waived() -> None: + """must_understand_fields is empty when every extra field is explicitly waived.""" + model = ArrayMetadataModelV3.create_default( + extra_fields={"ext_a": {"name": "a", "must_understand": False}} + ) + assert model.must_understand_fields == {} + + +def test_dimension_names_null_field_rejected() -> None: + """A dimension_names field whose VALUE is null is invalid: the spec permits + null as an element (an unnamed dimension), never as the field value — "not + specified" is spelled by omitting the key. Consumers bridging from an + in-memory None sentinel must drop the key, not write null.""" + doc = dict(ArrayMetadataModelV3.create_default().to_json()) | {"dimension_names": None} + problems = validate_array_metadata_v3(doc) + assert [(p.loc, p.kind) for p in problems] == [(("dimension_names",), "invalid_type")] + # and the model's own None spelling correctly maps to key absence + assert ( + "dimension_names" + not in ArrayMetadataModelV3.create_default(dimension_names=UNSET).to_json() + ) + + +# --- create_default derives the chunk grid from shape ------------------------ + + +def test_v3_create_default_chunk_grid_follows_shape() -> None: + """Overriding shape without chunk_grid derives a consistent default grid: + one chunk covering the array (chunk_shape == shape), instead of silently + keeping the scalar default's 0-d grid.""" + model = ArrayMetadataModelV3.create_default(shape=(100, 100)) + assert model.chunk_grid == NamedConfigModelV3( + name="regular", configuration={"chunk_shape": (100, 100)} + ) + + +def test_v3_create_default_explicit_chunk_grid_respected() -> None: + """An explicit chunk_grid override wins over the shape-derived default.""" + grid = NamedConfigModelV3(name="regular", configuration={"chunk_shape": (10, 10)}) + model = ArrayMetadataModelV3.create_default(shape=(100, 100), chunk_grid=grid) + assert model.chunk_grid == grid + + +def test_v2_create_default_chunks_follow_shape() -> None: + """Overriding shape without chunks derives chunks == shape.""" + model = ArrayMetadataModelV2.create_default(shape=(100, 100)) + assert model.chunks == (100, 100) + + +def test_v2_create_default_explicit_chunks_respected() -> None: + """An explicit chunks override wins over the shape-derived default.""" + model = ArrayMetadataModelV2.create_default(shape=(100, 100), chunks=(10, 10)) + assert model.chunks == (10, 10) + + +def test_v3_create_default_zero_length_dimensions() -> None: + """chunk_shape == shape is spec-sound even with zero-length dimensions: + 'The chunk shape elements are non-zero when the corresponding dimensions + of the arrays have non-zero length' — the constraint is conditional, so a + zero chunk length is permitted exactly where the dimension is empty.""" + model = ArrayMetadataModelV3.create_default(shape=(0, 3)) + assert model.chunk_grid.configuration["chunk_shape"] == (0, 3) + + +def test_create_default_derivation_is_one_way() -> None: + """Overriding the chunk grid (v3) or chunks (v2) without shape leaves the + scalar default shape=() untouched: a user-supplied chunk_grid is an + extension point taken verbatim, and deriving shape from it would require + interpreting grid configurations, which the model layer never does.""" + grid = NamedConfigModelV3(name="regular", configuration={"chunk_shape": (10, 10)}) + v3 = ArrayMetadataModelV3.create_default(chunk_grid=grid) + assert v3.shape == () + assert v3.chunk_grid == grid + v2 = ArrayMetadataModelV2.create_default(chunks=(10, 10)) + assert v2.shape == () + assert v2.chunks == (10, 10) + + +# --- v2 dimension_separator default (roborev job 426) ------------------------- + + +def test_v2_absent_dimension_separator_means_dot() -> None: + """A .zarray that omits dimension_separator uses the v2 convention default + '.', not '/': chunk keys of real-world default-separator v2 arrays look + like '0.0'. The model normalizes the absent key to an explicit '.' — a + semantics-preserving spelling normalization.""" + doc = dict(ArrayMetadataModelV2.create_default().to_json()) + del doc["dimension_separator"] + model = ArrayMetadataModelV2.from_json(doc) + assert model.dimension_separator == "." + assert model.to_json()["dimension_separator"] == "." + + +def test_v2_from_key_value_without_separator_means_dot() -> None: + """The .zarray store-file path applies the same '.' default for an absent + dimension_separator key.""" + doc = { + k: v + for k, v in ArrayMetadataModelV2.create_default().to_json().items() + if k not in ("dimension_separator", "attributes") + } + import json as _json + + model = ArrayMetadataModelV2.from_key_value({".zarray": _json.dumps(doc).encode()}) + assert model.dimension_separator == "." + + +def test_v2_null_dimension_separator_rejected() -> None: + """dimension_separator may be absent, '.', or '/' — never null: the + document grammar has no null spelling for this field.""" + doc = dict(ArrayMetadataModelV2.create_default().to_json()) | {"dimension_separator": None} + problems = validate_array_metadata_v2(doc) + assert [(p.loc, p.kind) for p in problems] == [(("dimension_separator",), "invalid_value")] + + +def test_dimension_names_absent_and_all_null_are_distinct() -> None: + """An absent dimension_names field and an explicit all-null one are + semantically different documents: the explicit form says every dimension + has a name, which is null; absence says there are no dimension names. + The model preserves the distinction (UNSET vs a tuple of Nones), and both + spellings round-trip faithfully.""" + absent_doc = dict(ArrayMetadataModelV3.create_default(shape=(2, 3)).to_json()) + explicit_doc = absent_doc | {"dimension_names": (None, None)} + + absent = ArrayMetadataModelV3.from_json(absent_doc) + explicit = ArrayMetadataModelV3.from_json(explicit_doc) + + assert absent.dimension_names is UNSET + assert explicit.dimension_names == (None, None) + assert absent != explicit + assert "dimension_names" not in absent.to_json() + assert absent.to_json() == absent_doc + assert explicit.to_json() == explicit_doc diff --git a/packages/zarr-metadata/tests/model/test_group.py b/packages/zarr-metadata/tests/model/test_group.py new file mode 100644 index 0000000000..58b92f5d71 --- /dev/null +++ b/packages/zarr-metadata/tests/model/test_group.py @@ -0,0 +1,375 @@ +"""Tests for the group and consolidated metadata models in `zarr_metadata.model`.""" + +import dataclasses +import json + +import pytest + +from zarr_metadata.model import UNSET +from zarr_metadata.model._array import ArrayMetadataModelV3 +from zarr_metadata.model._group import ( + ConsolidatedMetadataModelV2, + ConsolidatedMetadataModelV3, + GroupMetadataModelV2, + GroupMetadataModelV2Partial, + GroupMetadataModelV3, + GroupMetadataModelV3Partial, +) +from zarr_metadata.model._validation import ( + MetadataValidationError, + parse_group_metadata_v2, + parse_group_metadata_v3, + validate_group_metadata_v2, + validate_group_metadata_v3, +) + +# --- GroupMetadataModelV3 --------------------------------------------------- + + +def test_group_v3_roundtrip() -> None: + """A v3 group document round-trips through the model unchanged.""" + doc = {"zarr_format": 3, "node_type": "group", "attributes": {"a": (1, 2)}} + model = GroupMetadataModelV3.from_json(doc) + assert model.to_json() == doc + + +def test_group_v3_omits_empty_attributes() -> None: + """to_json omits the attributes key when attributes is empty.""" + model = GroupMetadataModelV3.create_default() + assert "attributes" not in model.to_json() + + +def test_group_v3_lists_become_tuples() -> None: + """from_json converts JSON arrays in attributes to tuples.""" + doc = {"zarr_format": 3, "node_type": "group", "attributes": {"a": [1, 2]}} + model = GroupMetadataModelV3.from_json(doc) + assert model.attributes == {"a": (1, 2)} + + +def test_group_v3_extra_fields_roundtrip() -> None: + """Unknown top-level keys land in extra_fields and reappear in to_json.""" + doc = { + "zarr_format": 3, + "node_type": "group", + "my_extension": {"name": "thing", "must_understand": False}, + } + model = GroupMetadataModelV3.from_json(doc) + assert model.extra_fields == {"my_extension": {"name": "thing", "must_understand": False}} + assert model.to_json() == doc + + +def test_group_v3_extra_fields_overlap_rejected() -> None: + """Constructing a v3 group model with extra_fields shadowing a standard key raises.""" + with pytest.raises(ValueError, match="Extra fields"): + GroupMetadataModelV3( + attributes={}, + consolidated_metadata=UNSET, + extra_fields={"node_type": {"name": "x", "must_understand": False}}, + ) + + +def test_group_v3_consolidated_extra_field_rejected() -> None: + """extra_fields may not shadow the consolidated_metadata convention key.""" + with pytest.raises(ValueError, match="Extra fields"): + GroupMetadataModelV3( + attributes={}, + consolidated_metadata=UNSET, + extra_fields={"consolidated_metadata": {"name": "x", "must_understand": False}}, + ) + + +def test_group_v3_missing_required_key() -> None: + """parse_group_metadata_v3 reports each missing required key.""" + with pytest.raises(MetadataValidationError, match="node_type"): + parse_group_metadata_v3({"zarr_format": 3}) + + +def test_group_v3_bad_attributes() -> None: + """parse_group_metadata_v3 rejects a non-mapping attributes value.""" + with pytest.raises(MetadataValidationError, match="attributes"): + parse_group_metadata_v3({"zarr_format": 3, "node_type": "group", "attributes": 5}) + + +def test_group_v3_key_value_roundtrip() -> None: + """from_key_value(to_key_value()) is the identity for v3 groups.""" + model = GroupMetadataModelV3.create_default(attributes={"a": 1}) + assert GroupMetadataModelV3.from_key_value(model.to_key_value()) == model + + +def test_group_v3_update() -> None: + """update replaces the given fields and returns a new instance.""" + base = GroupMetadataModelV3.create_default() + updated = base.update(attributes={"a": 1}) + assert updated.attributes == {"a": 1} + assert base.attributes == {} + + +# --- GroupMetadataModelV2 --------------------------------------------------- + + +def test_group_v2_key_value_split() -> None: + """v2 to_key_value writes .zgroup and .zattrs; from_key_value merges them.""" + model = GroupMetadataModelV2.create_default(attributes={"a": 1}) + kv = model.to_key_value() + assert set(kv) == {".zgroup", ".zattrs"} + assert json.loads(kv[".zgroup"]) == {"zarr_format": 2} + assert GroupMetadataModelV2.from_key_value(kv) == model + + +def test_group_v2_zattrs_presence_round_trips() -> None: + """A v2 group with no .zattrs file parses with UNSET attributes and emits + no .zattrs; an explicit empty .zattrs stays a file — the stores remain + distinct through a round-trip.""" + absent = GroupMetadataModelV2.from_key_value({".zgroup": b'{"zarr_format": 2}'}) + assert absent.attributes is UNSET + assert ".zattrs" not in absent.to_key_value() + explicit = GroupMetadataModelV2.from_key_value( + {".zgroup": b'{"zarr_format": 2}', ".zattrs": b"{}"} + ) + assert explicit.attributes == {} + assert ".zattrs" in explicit.to_key_value() + assert absent != explicit + + +def test_group_v2_json_roundtrip() -> None: + """A merged-form v2 group document round-trips through the model unchanged.""" + doc = {"zarr_format": 2, "attributes": {"a": 1}} + model = GroupMetadataModelV2.from_json(doc) + assert model.to_json() == doc + + +def test_group_v2_omits_empty_attributes() -> None: + """to_json omits the attributes key when attributes is empty.""" + assert "attributes" not in GroupMetadataModelV2.create_default().to_json() + + +def test_group_v2_not_a_mapping() -> None: + """parse_group_metadata_v2 rejects a non-mapping document.""" + with pytest.raises(MetadataValidationError, match="expected a mapping"): + parse_group_metadata_v2([1, 2, 3]) + + +def test_group_v2_missing_required_key() -> None: + """parse_group_metadata_v2 reports a missing zarr_format key.""" + with pytest.raises(MetadataValidationError, match="zarr_format"): + parse_group_metadata_v2({}) + + +# --- Partial TypedDict drift guards ----------------------------------------- + + +def test_group_partial_keys_match_settable_model_fields() -> None: + """Each group partial TypedDict must list exactly the settable model fields. + + Guards against drift: adding/removing a settable field on a group model + without updating its `*Partial` TypedDict fails here. + """ + for model_cls, partial_cls in ( + (GroupMetadataModelV3, GroupMetadataModelV3Partial), + (GroupMetadataModelV2, GroupMetadataModelV2Partial), + ): + settable = {f.name for f in dataclasses.fields(model_cls) if f.init} + assert set(partial_cls.__annotations__) == settable + + +# --- ConsolidatedMetadataModelV3 -------------------------------------------- + + +def test_consolidated_v3_roundtrip() -> None: + """A v3 group with inline consolidated metadata round-trips, with child + entries parsed into array/group models.""" + child = ArrayMetadataModelV3.create_default(shape=(2,)).to_json() + doc = { + "zarr_format": 3, + "node_type": "group", + "consolidated_metadata": { + "kind": "inline", + "must_understand": False, + "metadata": {"a": child, "g": {"zarr_format": 3, "node_type": "group"}}, + }, + } + model = GroupMetadataModelV3.from_json(doc) + assert isinstance(model.consolidated_metadata, ConsolidatedMetadataModelV3) + assert isinstance(model.consolidated_metadata.metadata["a"], ArrayMetadataModelV3) + assert isinstance(model.consolidated_metadata.metadata["g"], GroupMetadataModelV3) + assert model.to_json() == doc + + +def test_consolidated_v3_must_understand_true_rejected() -> None: + """ConsolidatedMetadataModelV3 enforces must_understand=False at runtime.""" + with pytest.raises(ValueError, match="must_understand"): + ConsolidatedMetadataModelV3(must_understand=True, metadata={}) + + +def test_consolidated_v3_from_json_must_understand_true_rejected() -> None: + """from_json rejects a consolidated document carrying must_understand=true.""" + with pytest.raises(MetadataValidationError, match="must_understand"): + ConsolidatedMetadataModelV3.from_json( + {"kind": "inline", "must_understand": True, "metadata": {}} + ) + + +def test_consolidated_v3_entry_without_node_type_rejected() -> None: + """from_json rejects a consolidated entry lacking a recognizable node_type.""" + with pytest.raises(MetadataValidationError, match="node_type"): + ConsolidatedMetadataModelV3.from_json( + {"kind": "inline", "must_understand": False, "metadata": {"a": {"zarr_format": 3}}} + ) + + +def test_consolidated_v3_not_a_mapping() -> None: + """from_json rejects a non-mapping consolidated document.""" + with pytest.raises(MetadataValidationError, match="expected a mapping"): + ConsolidatedMetadataModelV3.from_json(5) + + +# --- ConsolidatedMetadataModelV2 -------------------------------------------- + + +def test_consolidated_v2_verbatim_roundtrip() -> None: + """The v2 .zmetadata model holds the flat file-keyed map verbatim, + including nodes that have no .zattrs entry.""" + doc = { + "zarr_consolidated_format": 1, + "metadata": { + ".zgroup": {"zarr_format": 2}, + "a/.zarray": { + "zarr_format": 2, + "shape": (2,), + "chunks": (2,), + "dtype": "|u1", + "fill_value": 0, + "order": "C", + "compressor": None, + "filters": None, + }, + }, + } + model = ConsolidatedMetadataModelV2.from_json(doc) + assert model.to_json() == doc + + +def test_consolidated_v2_key_value_roundtrip() -> None: + """from_key_value(to_key_value()) is the identity for .zmetadata documents.""" + model = ConsolidatedMetadataModelV2.from_json( + {"zarr_consolidated_format": 1, "metadata": {".zgroup": {"zarr_format": 2}}} + ) + assert ConsolidatedMetadataModelV2.from_key_value(model.to_key_value()) == model + + +def test_consolidated_v2_lists_become_tuples() -> None: + """from_json converts JSON arrays inside entries to tuples.""" + doc = { + "zarr_consolidated_format": 1, + "metadata": {"a/.zarray": {"shape": [2, 3]}}, + } + model = ConsolidatedMetadataModelV2.from_json(doc) + assert model.metadata == {"a/.zarray": {"shape": (2, 3)}} + + +def test_consolidated_v2_envelope_validation() -> None: + """from_json rejects a .zmetadata document missing the metadata key.""" + with pytest.raises(MetadataValidationError, match="metadata"): + ConsolidatedMetadataModelV2.from_json({"zarr_consolidated_format": 1}) + + +def test_consolidated_v2_not_a_mapping() -> None: + """from_json rejects a non-mapping .zmetadata document.""" + with pytest.raises(MetadataValidationError, match="expected a mapping"): + ConsolidatedMetadataModelV2.from_json([1]) + + +# --- Literal-value enforcement ----------------------------------------------- + + +def test_group_v3_literals_enforced() -> None: + """A v3 group doc with wrong zarr_format or node_type is rejected with invalid_value.""" + base = GroupMetadataModelV3.create_default().to_json() + for key, bad in (("zarr_format", 2), ("node_type", "array")): + problems = validate_group_metadata_v3(dict(base) | {key: bad}) + assert [(p.loc, p.kind) for p in problems] == [((key,), "invalid_value")], key + + +def test_group_v2_zarr_format_literal_enforced() -> None: + """A v2 group doc claiming zarr_format 3 is rejected with invalid_value.""" + problems = validate_group_metadata_v2({"zarr_format": 3}) + assert [(p.loc, p.kind) for p in problems] == [(("zarr_format",), "invalid_value")] + + +# --- Consolidated envelope validated by the group validator ------------------ + + +def test_group_v3_validator_agrees_with_from_json_on_consolidated() -> None: + """The group validator validates the consolidated envelope and entries, so + is_group_metadata_v3 never vouches for a document from_json would reject.""" + bad_docs = ( + # empty envelope: missing kind/must_understand/metadata + {"zarr_format": 3, "node_type": "group", "consolidated_metadata": {}}, + # entry without a recognizable node_type + { + "zarr_format": 3, + "node_type": "group", + "consolidated_metadata": { + "kind": "inline", + "must_understand": False, + "metadata": {"a": {"zarr_format": 3}}, + }, + }, + # must_understand: true + { + "zarr_format": 3, + "node_type": "group", + "consolidated_metadata": { + "kind": "inline", + "must_understand": True, + "metadata": {}, + }, + }, + ) + for doc in bad_docs: + assert validate_group_metadata_v3(doc) != [], doc + with pytest.raises(MetadataValidationError): + GroupMetadataModelV3.from_json(doc) + + +def test_group_v3_valid_consolidated_passes_validator() -> None: + """A well-formed consolidated group validates cleanly (control case).""" + child = ArrayMetadataModelV3.create_default(shape=(2,)).to_json() + doc = { + "zarr_format": 3, + "node_type": "group", + "consolidated_metadata": { + "kind": "inline", + "must_understand": False, + "metadata": {"a": child, "g": {"zarr_format": 3, "node_type": "group"}}, + }, + } + assert validate_group_metadata_v3(doc) == [] + + +# --- must_understand partition ------------------------------------------------ + + +def test_group_must_understand_fields_partition() -> None: + """The group model partitions extra fields by the spec's implicit-true rule, + like the array model.""" + model = GroupMetadataModelV3.create_default( + extra_fields={ + "waived": {"name": "w", "must_understand": False}, + "implicit": {"name": "i"}, + } + ) + assert set(model.must_understand_fields) == {"implicit"} + + +def test_group_v3_null_consolidated_metadata_repaired_to_absence() -> None: + """consolidated_metadata: null was written by a historical zarr-python bug. + Those stores must remain readable, but the bug spelling is not honored: + it is read as absence (UNSET) and never written back — the round-trip + deliberately repairs the document rather than preserving the bug.""" + null_doc = {"zarr_format": 3, "node_type": "group", "consolidated_metadata": None} + assert validate_group_metadata_v3(null_doc) == [] + model = GroupMetadataModelV3.from_json(null_doc) + assert model.consolidated_metadata is UNSET + assert "consolidated_metadata" not in model.to_json() + assert model == GroupMetadataModelV3.from_json({"zarr_format": 3, "node_type": "group"}) diff --git a/packages/zarr-metadata/tests/model/test_pydantic.py b/packages/zarr-metadata/tests/model/test_pydantic.py new file mode 100644 index 0000000000..7e80d52ee8 --- /dev/null +++ b/packages/zarr-metadata/tests/model/test_pydantic.py @@ -0,0 +1,294 @@ +"""Executable example: integrating the metadata models with pydantic (v2). + +Pydantic's native dataclass introspection CAN be made to work (see +`test_native_dataclass_introspection_is_possible_but_diverges`): the models +keep their annotation-only imports behind `TYPE_CHECKING`, so a bare +`TypeAdapter(ArrayMetadataModelV3)` raises `class-not-fully-defined`, but +`rebuild(_types_namespace=...)` with the names supplied resolves the schema. +It is still the wrong tool: it validates the MODEL SHAPE, not the DOCUMENT — +no `from_json` normalization (a bare-string `data_type` is rejected), and +pydantic's lax coercion silently re-opens holes the library's validators +close (`shape=[True, -5]` coerces to `(1, -5)`; a wrong `dimension_names` +count passes). The recommended integration delegates wholesale — treat the +model as an opaque value: + +- `InstanceOf` makes pydantic's core schema an is-instance check (no field + introspection), +- validation goes through `from_json` (the single source of truth for what + a well-formed document is, including normalization: bare-string metadata + fields, arrays-to-tuples), +- serialization goes through `to_json` (the canonical document form). + +`MetadataValidationError` subclasses `ValueError`, so pydantic converts a +failed parse into its own `ValidationError` with the loc-annotated problem +messages intact. +""" + +from collections.abc import Mapping +from typing import Annotated, Generic, TypeVar + +import pytest +from pydantic import ( + BaseModel, + BeforeValidator, + ConfigDict, + InstanceOf, + PlainSerializer, + PydanticSchemaGenerationError, + PydanticUserError, + TypeAdapter, + ValidationError, + model_validator, +) + +from zarr_metadata import JSONValue +from zarr_metadata.model import ArrayMetadataModelV3, NamedConfigModelV3 + +# --- the integration (this is the example) ----------------------------------- + + +def _as_array_metadata_v3(value: object) -> ArrayMetadataModelV3: + """Accept an existing model instance or a raw metadata document.""" + if isinstance(value, ArrayMetadataModelV3): + return value + return ArrayMetadataModelV3.from_json(value) + + +# return_type is explicit because to_json's own annotation (`ArrayMetadataV3`) +# is a TYPE_CHECKING-only name pydantic cannot resolve at runtime. +ArrayMetadataV3Field = Annotated[ + InstanceOf[ArrayMetadataModelV3], + BeforeValidator(_as_array_metadata_v3), + PlainSerializer(ArrayMetadataModelV3.to_json, return_type=dict), +] +"""A pydantic-ready field type for v3 array metadata. + +Validates raw documents via `from_json`, passes model instances through, +and serializes to the canonical document form via `to_json`. +""" + + +class ArrayManifest(BaseModel): + """Example consumer model: a named array with its metadata document.""" + + path: str + metadata: ArrayMetadataV3Field + + +# --- tests pinning the example ------------------------------------------------ + +VALID_DOC = { + "zarr_format": 3, + "node_type": "array", + "shape": [10], + "data_type": "uint8", + "fill_value": 0, + "chunk_grid": {"name": "regular", "configuration": {"chunk_shape": [5]}}, + "chunk_key_encoding": {"name": "default"}, + "codecs": [{"name": "bytes"}], +} + + +def test_raw_document_is_validated_into_a_model() -> None: + """A raw metadata document on a pydantic field is parsed by from_json, + with the library's normalization applied (tuples, canonical field form).""" + manifest = ArrayManifest.model_validate({"path": "a/b", "metadata": VALID_DOC}) + assert isinstance(manifest.metadata, ArrayMetadataModelV3) + assert manifest.metadata.shape == (10,) + assert manifest.metadata.data_type.name == "uint8" + + +def test_model_instance_passes_through() -> None: + """An already-constructed model instance is accepted unchanged.""" + model = ArrayMetadataModelV3.from_json(VALID_DOC) + manifest = ArrayManifest(path="a/b", metadata=model) + assert manifest.metadata is model + + +def test_invalid_document_surfaces_problems_in_validation_error() -> None: + """A structurally-invalid document fails pydantic validation, carrying the + loc-annotated problem messages from MetadataValidationError.""" + doc = dict(VALID_DOC) + del doc["chunk_key_encoding"] + with pytest.raises(ValidationError) as exc_info: + ArrayManifest.model_validate({"path": "a/b", "metadata": doc}) + assert "chunk_key_encoding: missing required key" in str(exc_info.value) + + +def test_dump_emits_canonical_document() -> None: + """model_dump serializes the field via to_json — the canonical document, + not pydantic's field-by-field view of the dataclass.""" + manifest = ArrayManifest.model_validate({"path": "a/b", "metadata": VALID_DOC}) + dumped = manifest.model_dump() + assert dumped["metadata"] == manifest.metadata.to_json() + # the bare-string data_type was normalized to the canonical object form + assert dumped["metadata"]["data_type"] == {"name": "uint8", "configuration": {}} + + +def test_json_roundtrip_through_pydantic() -> None: + """model_dump_json output re-validates to an equal manifest (JSON emits + tuples as arrays; from_json converts them back).""" + manifest = ArrayManifest.model_validate({"path": "a/b", "metadata": VALID_DOC}) + revived = ArrayManifest.model_validate_json(manifest.model_dump_json()) + assert revived == manifest + + +def test_type_adapter_standalone() -> None: + """The annotated alias also works without a BaseModel, via TypeAdapter.""" + adapter = TypeAdapter(ArrayMetadataV3Field) + model = adapter.validate_python(VALID_DOC) + assert isinstance(model, ArrayMetadataModelV3) + assert adapter.dump_python(model) == model.to_json() + + +# --- the road not taken: native dataclass introspection ---------------------- + + +def test_native_dataclass_introspection_is_not_supported() -> None: + """Pydantic cannot field-introspect the model dataclasses: the UNSET + sentinel (PEP 661, typing_extensions.Sentinel) in the optional-field + annotations has no pydantic schema (as of pydantic 2.13), so even the + rebuild-with-namespace recipe fails. Introspection was already the wrong + tool before the sentinel existed — it validated the model shape rather + than the document, and its lax coercion re-opened validator holes (e.g. + shape=[True, -5] coerced to (1, -5)) — so the delegation patterns above + are the only supported integrations. If this test ever fails because + pydantic learned to handle sentinels, revisit whether the introspection + path needs its divergences documented again.""" + from zarr_metadata._common import JSONValue + from zarr_metadata.model import UNSET + from zarr_metadata.v3._common import MetadataV3 + from zarr_metadata.v3.array import ArrayMetadataV3, ExtensionFieldV3 + + def build_and_use() -> None: + adapter = TypeAdapter(ArrayMetadataModelV3) + adapter.rebuild( + force=True, + _types_namespace={ + "JSONValue": JSONValue, + "ExtensionFieldV3": ExtensionFieldV3, + "MetadataV3": MetadataV3, + "ArrayMetadataV3": ArrayMetadataV3, + "NamedConfigModelV3": NamedConfigModelV3, + "MetadataFieldModelV3": NamedConfigModelV3, + "UNSET": UNSET, + }, + ) + adapter.validate_python({}) + + with pytest.raises((AttributeError, PydanticSchemaGenerationError, PydanticUserError)): + build_and_use() + + +# --- a first-class pydantic model, engine-backed (the pydantic-zarr pattern) -- +# +# When a consumer wants a real BaseModel — JSON schema generation, and +# generics for typed attributes, as in pydantic-zarr's ArraySpec — the model +# fields are pydantic-native, but validation and serialization still route +# through the library: a mode="before" validator canonicalizes every input +# document with from_json(...).to_json(), so the structural validators and +# normalization run BEFORE pydantic parses fields (no coercion divergence), +# and the document form is the bridge in both directions. + +AttrsT = TypeVar("AttrsT") + + +class NamedConfig(BaseModel): + """Pydantic mirror of a normalized metadata field (name + configuration).""" + + name: str + configuration: dict[str, JSONValue] = {} + + +class ArrayMetadataV3Spec(BaseModel, Generic[AttrsT]): + """A pydantic-native, attribute-typed view of a v3 array metadata document. + + The library is the engine: every input is canonicalized and structurally + validated by `ArrayMetadataModelV3.from_json` before pydantic sees the + fields, and `to_document` / `to_metadata_model` emit through the library. + """ + + model_config = ConfigDict(frozen=True) + + zarr_format: int = 3 + node_type: str = "array" + shape: tuple[int, ...] + data_type: NamedConfig + chunk_grid: NamedConfig + chunk_key_encoding: NamedConfig + fill_value: JSONValue + codecs: tuple[NamedConfig, ...] + attributes: AttrsT + dimension_names: tuple[str | None, ...] | None = None + storage_transformers: tuple[NamedConfig, ...] = () + + @model_validator(mode="before") + @classmethod + def _canonicalize(cls, data: object) -> object: + """Route every input document through the library's validation and + normalization; pydantic then parses only canonical documents.""" + if isinstance(data, Mapping): + doc = dict(ArrayMetadataModelV3.from_json(data).to_json()) + doc.setdefault("attributes", {}) + return doc + return data + + def to_metadata_model(self) -> ArrayMetadataModelV3: + """Bridge back to the canonical model, via the document form. + + In the document, "no dimension names" is key-absence, not null; the + pydantic-side None translates to dropping the key. + """ + doc = self.model_dump() + if doc["dimension_names"] is None: + del doc["dimension_names"] + return ArrayMetadataModelV3.from_json(doc) + + def to_document(self) -> dict[str, object]: + """The canonical document (omit-empty conventions applied).""" + return dict(self.to_metadata_model().to_json()) + + +class MicroscopyAttrs(BaseModel): + """Example of consumer-typed attributes, pydantic-zarr style.""" + + resolution_um: float + + +def test_spec_typed_attributes() -> None: + """The generic parameter types the attributes, so consumers get validated, + attribute-level access — the pydantic-zarr ArraySpec pattern.""" + doc = dict(VALID_DOC) | {"attributes": {"resolution_um": 0.5}} + spec = ArrayMetadataV3Spec[MicroscopyAttrs].model_validate(doc) + assert spec.attributes.resolution_um == 0.5 + assert spec.data_type == NamedConfig(name="uint8") + + +def test_spec_engine_validates_before_pydantic() -> None: + """The library's structural validation runs before pydantic's parsing, so + coercion cannot re-open validator holes (contrast with the native + introspection test above, where [True, -5] coerced to (1, -5)).""" + with pytest.raises(ValidationError, match="shape"): + ArrayMetadataV3Spec[MicroscopyAttrs].model_validate( + dict(VALID_DOC) | {"shape": [True, -5], "attributes": {"resolution_um": 0.5}} + ) + + +def test_spec_bridges_to_canonical_model_and_document() -> None: + """to_metadata_model / to_document round-trip through the document form, + and the emitted document matches what the library itself would emit.""" + doc = dict(VALID_DOC) | {"attributes": {"resolution_um": 0.5}} + spec = ArrayMetadataV3Spec[MicroscopyAttrs].model_validate(doc) + model = spec.to_metadata_model() + assert isinstance(model, ArrayMetadataModelV3) + assert spec.to_document() == dict(model.to_json()) + # and back: the document revalidates to an equal spec + assert ArrayMetadataV3Spec[MicroscopyAttrs].model_validate(spec.to_document()) == spec + + +def test_spec_json_schema_generation() -> None: + """A real BaseModel means model_json_schema works — the capability the + opaque InstanceOf pattern cannot provide.""" + schema = ArrayMetadataV3Spec[MicroscopyAttrs].model_json_schema() + assert schema["properties"]["shape"]["type"] == "array" + assert "MicroscopyAttrs" in schema["$defs"] diff --git a/packages/zarr-metadata/tests/model/test_pydantic_module.py b/packages/zarr-metadata/tests/model/test_pydantic_module.py new file mode 100644 index 0000000000..81b9b962af --- /dev/null +++ b/packages/zarr-metadata/tests/model/test_pydantic_module.py @@ -0,0 +1,125 @@ +"""Tests for `zarr_metadata.pydantic`, the optional pydantic field-type module. + +The hand-rolled recipes in `test_pydantic.py` document how the integration +works; this module ships it. Instances are the CORE model classes (no +parallel hierarchy), so values interoperate freely with non-pydantic code. +""" + +import pytest +from pydantic import BaseModel, TypeAdapter, ValidationError + +import zarr_metadata.pydantic as zmp +from zarr_metadata.model import ( + ArrayMetadataModelV2, + ArrayMetadataModelV3, + ConsolidatedMetadataModelV2, + ConsolidatedMetadataModelV3, + GroupMetadataModelV2, + GroupMetadataModelV3, + NamedConfigModelV3, +) + +V3_ARRAY_DOC = dict(ArrayMetadataModelV3.create_default(shape=(4,)).to_json()) +V2_ARRAY_DOC = dict(ArrayMetadataModelV2.create_default(shape=(4,), chunks=(2,)).to_json()) +V3_GROUP_DOC = {"zarr_format": 3, "node_type": "group", "attributes": {"a": 1}} +V2_GROUP_DOC = {"zarr_format": 2, "attributes": {"a": 1}} +V3_CONSOLIDATED_DOC = { + "kind": "inline", + "must_understand": False, + "metadata": {"a": dict(V3_ARRAY_DOC)}, +} +V2_CONSOLIDATED_DOC = { + "zarr_consolidated_format": 1, + "metadata": {".zgroup": {"zarr_format": 2}}, +} + +FIELD_CASES = [ + pytest.param(zmp.ArrayMetadataV3, ArrayMetadataModelV3, V3_ARRAY_DOC, id="array-v3"), + pytest.param(zmp.ArrayMetadataV2, ArrayMetadataModelV2, V2_ARRAY_DOC, id="array-v2"), + pytest.param(zmp.GroupMetadataV3, GroupMetadataModelV3, V3_GROUP_DOC, id="group-v3"), + pytest.param(zmp.GroupMetadataV2, GroupMetadataModelV2, V2_GROUP_DOC, id="group-v2"), + pytest.param( + zmp.ConsolidatedMetadataV3, + ConsolidatedMetadataModelV3, + V3_CONSOLIDATED_DOC, + id="consolidated-v3", + ), + pytest.param( + zmp.ConsolidatedMetadataV2, + ConsolidatedMetadataModelV2, + V2_CONSOLIDATED_DOC, + id="consolidated-v2", + ), + pytest.param(zmp.MetadataFieldV3, NamedConfigModelV3, {"name": "bytes"}, id="field-v3"), +] + + +@pytest.mark.parametrize(("field_type", "model_cls", "doc"), FIELD_CASES) +def test_field_type_validates_and_dumps_canonically( + field_type: object, model_cls: type, doc: dict[str, object] +) -> None: + """Each field type parses its raw document into the CORE model class, + passes existing instances through unchanged, and dumps the canonical + document via to_json.""" + adapter = TypeAdapter(field_type) + model = adapter.validate_python(doc) + assert type(model) is model_cls + assert adapter.validate_python(model) is model + assert adapter.dump_python(model) == dict(model.to_json()) + + +def test_core_instances_interoperate() -> None: + """A core model instance (e.g. handed out by zarr-python) drops straight + into a pydantic field — the reason the module ships Annotated aliases over + the core classes rather than pydantic-aware subclasses.""" + + class Manifest(BaseModel): + metadata: zmp.ArrayMetadataV3 + + core = ArrayMetadataModelV3.from_json(V3_ARRAY_DOC) + manifest = Manifest(metadata=core) + assert manifest.metadata is core + + +def test_validation_error_carries_problems() -> None: + """A defective document fails with the library's loc-annotated messages.""" + + class Manifest(BaseModel): + metadata: zmp.ArrayMetadataV3 + + doc = dict(V3_ARRAY_DOC) + del doc["chunk_key_encoding"] + with pytest.raises(ValidationError, match="chunk_key_encoding: missing required key"): + Manifest.model_validate({"metadata": doc}) + + +def test_json_schema_generation() -> None: + """model_json_schema works, describing the document form each field accepts.""" + + class Manifest(BaseModel): + metadata: zmp.ArrayMetadataV3 + codec: zmp.MetadataFieldV3 + + schema = Manifest.model_json_schema() + assert schema["properties"]["metadata"] == {"type": "object", "title": "ArrayMetadataV3"} + assert schema["properties"]["codec"]["anyOf"] == [{"type": "string"}, {"type": "object"}] + + +def test_json_roundtrip() -> None: + """model_dump_json output re-validates to an equal pydantic model.""" + + class Manifest(BaseModel): + metadata: zmp.ArrayMetadataV3 + + manifest = Manifest.model_validate({"metadata": V3_ARRAY_DOC}) + assert Manifest.model_validate_json(manifest.model_dump_json()) == manifest + + +def test_core_package_does_not_import_pydantic() -> None: + """Importing zarr_metadata (in a fresh interpreter) must not import + pydantic: the integration is opt-in via zarr_metadata.pydantic.""" + import subprocess + import sys + + code = "import sys, zarr_metadata; assert 'pydantic' not in sys.modules, 'leaked'" + subprocess.run([sys.executable, "-c", code], check=True) diff --git a/packages/zarr-metadata/tests/test_public_api.py b/packages/zarr-metadata/tests/test_public_api.py index d3270579c3..ee26e80c5a 100644 --- a/packages/zarr-metadata/tests/test_public_api.py +++ b/packages/zarr-metadata/tests/test_public_api.py @@ -39,6 +39,23 @@ def _group_rank(s: str) -> int: "NamedConfigV3", "MetadataV3", "JSONValue", + # Category A' — metadata models (in-memory dataclasses over the documents) + "ArrayMetadataModelV2", + "ArrayMetadataModelV2Partial", + "ArrayMetadataModelV3", + "ArrayMetadataModelV3Partial", + "GroupMetadataModelV2", + "GroupMetadataModelV2Partial", + "GroupMetadataModelV3", + "GroupMetadataModelV3Partial", + "ConsolidatedMetadataModelV2", + "ConsolidatedMetadataModelV3", + "NamedConfigModelV3", + "MetadataFieldModelV3", + "ValidationProblem", + "MetadataValidationError", + "ProblemKind", + "UNSET", # v2 data-type encoding union "DataTypeMetadataV2", # Category B — codec canonical unions