From 9ab8a330a6e05dfa45d8d60bd3ca4dead14ef0e4 Mon Sep 17 00:00:00 2001 From: m0g3r <87276771+m0g3r@users.noreply.github.com> Date: Tue, 18 Aug 2026 09:00:35 +0200 Subject: [PATCH] fix(avro): read doubles at full precision in the Cython decoder `CythonBinaryDecoder.read_double` was declared `cpdef float`, which in Cython is the C single-precision type, so every Avro double decoded by the fast decoder was silently rounded to 32-bit precision. Values outside the single-precision range collapse entirely: 1e308 becomes inf and 5e-324 becomes 0.0. `new_decoder` returns the Cython decoder whenever the extension is built, so this is the default read path. It affects any double read from a manifest, most visibly identity partition values on a float/double column: writing a partition value of 429496729622.314 and reading the manifest back returns 429496729600.0. The pure-Python `StreamingBinaryDecoder` was always correct, and `read_float` is unaffected because a value decoded from four bytes is already representable as a C float. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01T2GWEoizz8ZGQbjatT8aZy --- pyiceberg/avro/decoder_fast.pyx | 2 +- tests/avro/test_decoder.py | 17 +++++++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/pyiceberg/avro/decoder_fast.pyx b/pyiceberg/avro/decoder_fast.pyx index 52caec3308..ffd23dd977 100644 --- a/pyiceberg/avro/decoder_fast.pyx +++ b/pyiceberg/avro/decoder_fast.pyx @@ -138,7 +138,7 @@ cdef class CythonBinaryDecoder: """ return float(STRUCT_FLOAT.unpack(self.read(4))[0]) - cpdef float read_double(self): + cpdef double read_double(self): """Reads a value from the stream as a double. A double is written as 8 bytes. diff --git a/tests/avro/test_decoder.py b/tests/avro/test_decoder.py index 163ad8405e..1cf8346347 100644 --- a/tests/avro/test_decoder.py +++ b/tests/avro/test_decoder.py @@ -160,6 +160,23 @@ def test_read_double(decoder_class: Callable[[bytes], BinaryDecoder]) -> None: assert decoder.read_double() == 19.25 +@pytest.mark.parametrize("decoder_class", AVAILABLE_DECODERS) +@pytest.mark.parametrize( + "value", + [ + 3.141592653589793, + 429496729622.314, + 0.1, + 1.0000000000000002, # smallest double above 1.0 + 1e308, # overflows to inf in single precision + 5e-324, # underflows to 0.0 in single precision + ], +) +def test_read_double_keeps_full_precision(decoder_class: Callable[[bytes], BinaryDecoder], value: float) -> None: + decoder = decoder_class(struct.pack(" None: decoder = decoder_class(b"\x00\x00\x00\x00\x00\x40\x33\x40")