From 59e63dbe75d7339e09ddb5e042dbaa39c1911119 Mon Sep 17 00:00:00 2001 From: Trenton Holmes <797416+stumpylog@users.noreply.github.com> Date: Mon, 22 Jun 2026 10:31:18 -0700 Subject: [PATCH] Clamp DiskANN int8 quantization to avoid float->int8 UB diskann_quantize_vector's INT8 path cast the quantized float directly to i8. When the input vector is not normalized to [-1, 1] the intermediate value falls outside [-128, 127], and converting an out-of-range float to a signed integer type is undefined behavior in C (in practice it wraps to garbage). Saturate the value to [-128, 127] before the cast so out-of-range inputs quantize to the nearest representable int8. Surfaced by an ASan/UBSan run of the loadable test suite (test_diskann_insert_int8_quantizer_knn). Co-Authored-By: Claude Opus 4.8 --- sqlite-vec-diskann.c | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/sqlite-vec-diskann.c b/sqlite-vec-diskann.c index 5bd298bc..0d74a608 100644 --- a/sqlite-vec-diskann.c +++ b/sqlite-vec-diskann.c @@ -157,7 +157,13 @@ int diskann_quantize_vector( case VEC0_DISKANN_QUANTIZER_INT8: { f32 step = (1.0f - (-1.0f)) / 255.0f; for (size_t i = 0; i < dimensions; i++) { - ((i8 *)out)[i] = (i8)(((src[i] - (-1.0f)) / step) - 128.0f); + // Saturate before the cast: converting a float outside [-128, 127] to a + // signed integer type is undefined behavior in C (and silently wraps to + // garbage). Inputs are expected to be normalized to [-1, 1]; clamp so + // out-of-range values quantize to the nearest representable int8. + const f32 raw = ((src[i] - (-1.0f)) / step) - 128.0f; + const f32 q = raw < -128.0f ? -128.0f : (raw > 127.0f ? 127.0f : raw); + ((i8 *)out)[i] = (i8)q; } return SQLITE_OK; }