From 61f6f991aef855f55415aab32746bd9d4876422c Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 5 Aug 2026 23:12:49 +0000 Subject: [PATCH] Ship type information (PEP 561) faust-cchardet is a drop-in replacement for chardet, and chardet ships py.typed with a fully annotated detect(). So switching to this package currently *downgrades* a user's type coverage: mypy reports error: Skipping analyzing "cchardet": module is installed, but missing library stubs or py.typed marker [import-untyped] and every call becomes Any. All of these pass silently today: cchardet.detect(raw)["encoding"].upper() # None-deref at runtime cchardet.detect(raw)["confidence"] # float used as int cchardet.detect(raw)["encodng"] # typo -> KeyError cchardet.detect("a str, not bytes") # TypeError at runtime With this change mypy catches all four, the typo with a "Did you mean encoding?" suggestion. Adds a py.typed marker, a DetectionResult TypedDict describing what detect() and UniversalDetector.result actually return, annotations across the public API and the CLI, and _cchardet.pyi -- without a stub the compiled extension is opaque and the package degrades to Any even with py.typed present. Both files are installed via py.install_sources, and src/tests/test_typing.py guards that, since dropping them from that list fails silently. Deliberately NOT adding a mypy CI gate. The shipped Python surface is 148 lines; mypy finds zero real bugs in it and --strict finds only missing-annotation boilerplate. All the substantive logic is Cython and C++, which mypy cannot see. The value here is entirely outward-facing. Two documentation bugs fixed in passing, both surfaced by writing the types down: - detect()'s docstring claimed `msg: str`, but passing a str raises TypeError: expected bytes. It takes bytes. - detect() guarded `isinstance(msg, (bytes, bytearray))` when picking the BOM prefix, implying bytearray support. The extension signature is `bytes msg`, so bytearray and memoryview both raise TypeError before that line is reached -- the fallback branch was unreachable. __enter__ uses a bound TypeVar rather than typing.Self, which is 3.11+; this package supports 3.10. Verified on 3.10: mypy clean in default and --strict mode, 134 passed, and the built wheel contains both py.typed and _cchardet.pyi. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01TWqKLkfwd8fPxhU4KjXUVB --- src/cchardet/__init__.py | 75 ++++++++++++++++++++++----------- src/cchardet/_cchardet.pyi | 24 +++++++++++ src/cchardet/cli/cchardetect.py | 6 ++- src/cchardet/meson.build | 6 +++ src/cchardet/py.typed | 0 src/tests/test_typing.py | 56 ++++++++++++++++++++++++ 6 files changed, 141 insertions(+), 26 deletions(-) create mode 100644 src/cchardet/_cchardet.pyi create mode 100644 src/cchardet/py.typed create mode 100644 src/tests/test_typing.py diff --git a/src/cchardet/__init__.py b/src/cchardet/__init__.py index a694ea6..53701c1 100644 --- a/src/cchardet/__init__.py +++ b/src/cchardet/__init__.py @@ -1,6 +1,27 @@ +from types import TracebackType +from typing import Literal, TypedDict, TypeVar + from cchardet import _cchardet from .version import __version__ +__all__ = ["DetectionResult", "UniversalDetector", "__version__", "detect"] + +# typing.Self is 3.11+, and this package supports 3.10, so __enter__ is typed +# with a bound TypeVar instead -- subclasses still get their own type back. +_SelfT = TypeVar("_SelfT", bound="UniversalDetector") + + +class DetectionResult(TypedDict): + """What :func:`detect` and :attr:`UniversalDetector.result` return. + + Both values are ``None`` when nothing could be detected, so callers have to + handle that before using the encoding -- which is exactly what a type + checker enforces once this package ships its ``py.typed`` marker. + """ + + encoding: str | None + confidence: float | None + # Upstream freedesktop uchardet emits a few Mac charset labels with a hyphen # (e.g. "MAC-CENTRALEUROPE", "MAC-CYRILLIC") that Python's codec registry cannot @@ -19,7 +40,7 @@ _UTF8_BOM = b"\xef\xbb\xbf" -def _normalize_encoding(encoding, leading_bytes): +def _normalize_encoding(encoding: str | None, leading_bytes: bytes) -> str | None: """Normalize freedesktop uchardet labels to match the previous uchardet (and Python's ``chardet``) so results stay usable with open(encoding=...) / bytes.decode(). @@ -42,43 +63,50 @@ def _normalize_encoding(encoding, leading_bytes): return encoding -def detect(msg): - """ +def detect(msg: bytes) -> DetectionResult: + """Detect the character encoding of ``msg``. + Args: - msg: str + msg: the raw bytes to inspect. Must be ``bytes`` -- the extension + rejects ``str``, ``bytearray`` and ``memoryview``. Returns: - { - "encoding": str, - "confidence": float - } + A :class:`DetectionResult`. Both members are ``None`` when nothing + could be detected. """ - encoding, confidence = _cchardet.detect_with_confidence(msg) - if isinstance(encoding, bytes): - encoding = encoding.decode() + raw_encoding, confidence = _cchardet.detect_with_confidence(msg) + encoding = raw_encoding.decode() if raw_encoding is not None else None - leading = msg[:3] if isinstance(msg, (bytes, bytearray)) else b"" - encoding = _normalize_encoding(encoding, leading) + # detect_with_confidence above rejects anything that is not bytes, so the + # old isinstance() guard on msg here could never take its fallback branch. + encoding = _normalize_encoding(encoding, msg[:3]) return {"encoding": encoding, "confidence": confidence} class UniversalDetector(object): - def __init__(self): + def __init__(self) -> None: self._detector = _cchardet.UniversalDetector() self._leading = b"" - def __enter__(self): + def __enter__(self: _SelfT) -> _SelfT: return self - def __exit__(self, exception_type, exception_value, traceback): + def __exit__( + self, + exception_type: type[BaseException] | None, + exception_value: BaseException | None, + traceback: TracebackType | None, + ) -> Literal[False]: + # Literal[False] rather than bool: it tells the checker this context + # manager never swallows an exception. self.close() return False - def reset(self): + def reset(self) -> None: self._detector.reset() self._leading = b"" - def feed(self, data): + def feed(self, data: bytes) -> None: # Remember the first bytes so result can spot a UTF-8 BOM (uchardet only # exposes the label, not the raw bytes). The BOM is 3 bytes and is # virtually always delivered in the first chunk. @@ -86,18 +114,17 @@ def feed(self, data): self._leading = bytes(data[:3]) self._detector.feed(data) - def close(self): + def close(self) -> None: self._detector.close() @property - def done(self): + def done(self) -> bool: return self._detector.done @property - def result(self): - encoding, confidence = self._detector.result - if isinstance(encoding, bytes): - encoding = encoding.decode() + def result(self) -> DetectionResult: + raw_encoding, confidence = self._detector.result + encoding = raw_encoding.decode() if raw_encoding is not None else None if encoding is not None: encoding = _normalize_encoding(encoding, self._leading) return {"encoding": encoding, "confidence": confidence} diff --git a/src/cchardet/_cchardet.pyi b/src/cchardet/_cchardet.pyi new file mode 100644 index 0000000..961e730 --- /dev/null +++ b/src/cchardet/_cchardet.pyi @@ -0,0 +1,24 @@ +"""Type stubs for the Cython extension module. + +Type checkers cannot see inside a compiled extension, so without this stub the +whole package degrades to ``Any`` even with a ``py.typed`` marker. Kept in sync +with ``_cchardet.pyx`` by hand -- it is a small, stable surface. + +Note the ``bytes`` argument types: the Cython signatures are ``bytes msg``, so +``bytearray`` and ``memoryview`` raise ``TypeError`` at runtime rather than +being accepted as buffers. +""" + +class UniversalDetector: + def __init__(self) -> None: ... + def reset(self) -> None: ... + def feed(self, msg: bytes) -> None: ... + def close(self) -> None: ... + @property + def done(self) -> bool: ... + @property + def result(self) -> tuple[bytes, float] | tuple[None, None]: ... + +def detect_with_confidence( + msg: bytes, +) -> tuple[bytes, float] | tuple[None, None]: ... diff --git a/src/cchardet/cli/cchardetect.py b/src/cchardet/cli/cchardetect.py index 485174c..4f563e6 100644 --- a/src/cchardet/cli/cchardetect.py +++ b/src/cchardet/cli/cchardetect.py @@ -1,17 +1,19 @@ import argparse import sys +from collections.abc import Iterator +from typing import IO from .. import UniversalDetector, __version__ -def read_chunks(f, chunk_size): +def read_chunks(f: IO[bytes], chunk_size: int) -> Iterator[bytes]: chunk = f.read(chunk_size) while chunk: yield chunk chunk = f.read(chunk_size) -def main(): +def main() -> None: parser = argparse.ArgumentParser() parser.add_argument( "files", diff --git a/src/cchardet/meson.build b/src/cchardet/meson.build index c5bb494..d81e536 100644 --- a/src/cchardet/meson.build +++ b/src/cchardet/meson.build @@ -125,6 +125,12 @@ py.install_sources( '__init__.py', '__main__.py', 'version.py', + # PEP 561: py.typed marks the package as typed, and the .pyi describes the + # compiled extension that type checkers cannot introspect. Both have to be + # installed alongside the modules or downstream type checking silently falls + # back to Any -- src/tests/test_typing.py guards that. + 'py.typed', + '_cchardet.pyi', subdir: 'cchardet', ) diff --git a/src/cchardet/py.typed b/src/cchardet/py.typed new file mode 100644 index 0000000..e69de29 diff --git a/src/tests/test_typing.py b/src/tests/test_typing.py new file mode 100644 index 0000000..f3e345f --- /dev/null +++ b/src/tests/test_typing.py @@ -0,0 +1,56 @@ +"""PEP 561 packaging: the type information has to actually ship. + +Annotating the source does nothing for downstream users on its own. A type +checker only looks at an installed package if it carries a ``py.typed`` marker, +and it cannot see into the compiled extension without ``_cchardet.pyi``. Both +are installed by ``py.install_sources`` in src/cchardet/meson.build, which is +easy to drop when that list is edited -- and dropping it fails silently, +degrading every downstream ``cchardet`` annotation to ``Any`` with no error +anywhere. Hence these tests. +""" + +import pathlib + +import cchardet + +_PACKAGE_DIR = pathlib.Path(cchardet.__file__).parent + + +def test_py_typed_marker_is_installed(): + assert (_PACKAGE_DIR / "py.typed").is_file(), ( + "py.typed is missing from the installed package, so type checkers will " + "ignore cchardet's annotations entirely (PEP 561)" + ) + + +def test_extension_stub_is_installed(): + assert (_PACKAGE_DIR / "_cchardet.pyi").is_file(), ( + "_cchardet.pyi is missing from the installed package, so the compiled " + "extension is untyped and the public API degrades to Any" + ) + + +def test_detection_result_is_exported(): + """The TypedDict is part of the public API -- downstream code annotates + against it, so removing or renaming it is a breaking change.""" + assert cchardet.DetectionResult in (cchardet.DetectionResult,) + assert set(cchardet.DetectionResult.__annotations__) == {"encoding", "confidence"} + assert "DetectionResult" in cchardet.__all__ + + +def test_result_shape_matches_the_declared_type(): + """Guard against the annotations drifting from what is actually returned.""" + detected = cchardet.detect("こんにちは".encode("shift_jis")) + assert set(detected) == {"encoding", "confidence"} + assert isinstance(detected["encoding"], str) + assert isinstance(detected["confidence"], float) + + with cchardet.UniversalDetector() as detector: + detector.feed("こんにちは".encode("shift_jis")) + streamed = detector.result + assert set(streamed) == {"encoding", "confidence"} + + # The None case the annotation forces callers to handle. + empty = cchardet.detect(b"") + assert empty["encoding"] is None + assert empty["confidence"] is None