From 246dd131171fa6eee72e63b0949cc3e8b4f7b0d1 Mon Sep 17 00:00:00 2001 From: Dor Forer Date: Wed, 19 Aug 2026 14:14:27 +0300 Subject: [PATCH 1/4] Give ARM kernel instantiations tier-local linkage Two tier translation units that include the same kernel header emitted the same weak (COMDAT) symbols, compiled under different -march flags. The linker then kept one body and discarded the other, chosen by link order, with nothing in the source deciding which. For each of the eight ARM tier TUs (NEON, NEON_DOTPROD, NEON_HP, NEON_FHM, NEON_BF16, SVE, SVE2, SVE_BF16), wrap the kernel-header includes in an anonymous namespace so the kernel instantiations get internal linkage, unique to each translation unit. Only the Choose_* entry points, still declared in the tier header, keep external linkage. The shared dependencies (space_includes.h, spaces.h, the type headers, and the ARM intrinsics headers) are hoisted above the anonymous namespace: wrapping them along with the kernel includes pulls the standard library into the anonymous namespace and fails to compile. Measured on arm-r8g.xlarge (gcc 12) with nm -g --defined-only before this change: NEON and NEON_DOTPROD shared 93 externally-defined symbols, SVE and SVE2 shared 166. This change makes the whole class of collision structurally impossible rather than fixing only those two known pairs. x86 was measured at 0 shared symbols across all 15 tier objects, every pair, because x86 kernel names embed the ISA and no x86 tier TU reuses another tier's kernel headers. x86 tier TUs are therefore untouched. Verification of the after state (all 28 ARM pairs sharing zero symbols, and every object still exporting its own Choose_* entry points) requires building on real ARM hardware and was not run as part of this change. --- src/VecSim/spaces/functions/NEON.cpp | 16 ++++++++++++++++ src/VecSim/spaces/functions/NEON_BF16.cpp | 15 +++++++++++++++ src/VecSim/spaces/functions/NEON_DOTPROD.cpp | 16 ++++++++++++++++ src/VecSim/spaces/functions/NEON_FHM.cpp | 15 +++++++++++++++ src/VecSim/spaces/functions/NEON_HP.cpp | 15 +++++++++++++++ src/VecSim/spaces/functions/SVE.cpp | 16 ++++++++++++++++ src/VecSim/spaces/functions/SVE2.cpp | 15 +++++++++++++++ src/VecSim/spaces/functions/SVE_BF16.cpp | 15 +++++++++++++++ 8 files changed, 123 insertions(+) diff --git a/src/VecSim/spaces/functions/NEON.cpp b/src/VecSim/spaces/functions/NEON.cpp index 0c9a286e3..8d5c286be 100644 --- a/src/VecSim/spaces/functions/NEON.cpp +++ b/src/VecSim/spaces/functions/NEON.cpp @@ -7,6 +7,21 @@ * GNU Affero General Public License v3 (AGPLv3). */ #include "NEON.h" + +// Hoisted above the anonymous namespace below so that the standard library and the shared +// type headers keep external linkage. Wrapping them would pull and friends into +// the anonymous namespace and fail to compile. +#include "VecSim/spaces/space_includes.h" +#include "VecSim/spaces/spaces.h" +#include "VecSim/types/bfloat16.h" +#include "VecSim/types/float16.h" +#include "VecSim/types/sq8.h" +#include + +// Kernel instantiations get internal linkage, unique to this translation unit, so two tiers +// that share a kernel header cannot emit the same weak symbol and let link order pick the +// body. Only this tier's Choose_* entry points stay external. +namespace { #include "VecSim/spaces/L2/L2_NEON_FP32.h" #include "VecSim/spaces/IP/IP_NEON_FP32.h" #include "VecSim/spaces/L2/L2_NEON_INT8.h" @@ -19,6 +34,7 @@ #include "VecSim/spaces/IP/IP_NEON_SQ8_FP32.h" #include "VecSim/spaces/IP/IP_NEON_SQ8_SQ8.h" #include "VecSim/spaces/L2/L2_NEON_SQ8_SQ8.h" +} // namespace namespace spaces { diff --git a/src/VecSim/spaces/functions/NEON_BF16.cpp b/src/VecSim/spaces/functions/NEON_BF16.cpp index 4de205bb8..1a515943e 100644 --- a/src/VecSim/spaces/functions/NEON_BF16.cpp +++ b/src/VecSim/spaces/functions/NEON_BF16.cpp @@ -8,8 +8,23 @@ */ #include "NEON_BF16.h" +// Hoisted above the anonymous namespace below so that the standard library and the shared +// type headers keep external linkage. Wrapping them would pull and friends into +// the anonymous namespace and fail to compile. +#include "VecSim/spaces/space_includes.h" +#include "VecSim/spaces/spaces.h" +#include "VecSim/types/bfloat16.h" +#include "VecSim/types/float16.h" +#include "VecSim/types/sq8.h" +#include + +// Kernel instantiations get internal linkage, unique to this translation unit, so two tiers +// that share a kernel header cannot emit the same weak symbol and let link order pick the +// body. Only this tier's Choose_* entry points stay external. +namespace { #include "VecSim/spaces/L2/L2_NEON_BF16.h" #include "VecSim/spaces/IP/IP_NEON_BF16.h" +} // namespace namespace spaces { diff --git a/src/VecSim/spaces/functions/NEON_DOTPROD.cpp b/src/VecSim/spaces/functions/NEON_DOTPROD.cpp index 12f762093..ade17bb31 100644 --- a/src/VecSim/spaces/functions/NEON_DOTPROD.cpp +++ b/src/VecSim/spaces/functions/NEON_DOTPROD.cpp @@ -7,12 +7,28 @@ * GNU Affero General Public License v3 (AGPLv3). */ #include "NEON.h" + +// Hoisted above the anonymous namespace below so that the standard library and the shared +// type headers keep external linkage. Wrapping them would pull and friends into +// the anonymous namespace and fail to compile. +#include "VecSim/spaces/space_includes.h" +#include "VecSim/spaces/spaces.h" +#include "VecSim/types/bfloat16.h" +#include "VecSim/types/float16.h" +#include "VecSim/types/sq8.h" +#include + +// Kernel instantiations get internal linkage, unique to this translation unit, so two tiers +// that share a kernel header cannot emit the same weak symbol and let link order pick the +// body. Only this tier's Choose_* entry points stay external. +namespace { #include "VecSim/spaces/IP/IP_NEON_DOTPROD_INT8.h" #include "VecSim/spaces/IP/IP_NEON_DOTPROD_UINT8.h" #include "VecSim/spaces/IP/IP_NEON_DOTPROD_SQ8_SQ8.h" #include "VecSim/spaces/L2/L2_NEON_DOTPROD_INT8.h" #include "VecSim/spaces/L2/L2_NEON_DOTPROD_UINT8.h" #include "VecSim/spaces/L2/L2_NEON_DOTPROD_SQ8_SQ8.h" +} // namespace namespace spaces { diff --git a/src/VecSim/spaces/functions/NEON_FHM.cpp b/src/VecSim/spaces/functions/NEON_FHM.cpp index 2beceed0c..ed92bec47 100644 --- a/src/VecSim/spaces/functions/NEON_FHM.cpp +++ b/src/VecSim/spaces/functions/NEON_FHM.cpp @@ -8,8 +8,23 @@ */ #include "NEON_FHM.h" +// Hoisted above the anonymous namespace below so that the standard library and the shared +// type headers keep external linkage. Wrapping them would pull and friends into +// the anonymous namespace and fail to compile. +#include "VecSim/spaces/space_includes.h" +#include "VecSim/spaces/spaces.h" +#include "VecSim/types/bfloat16.h" +#include "VecSim/types/float16.h" +#include "VecSim/types/sq8.h" +#include + +// Kernel instantiations get internal linkage, unique to this translation unit, so two tiers +// that share a kernel header cannot emit the same weak symbol and let link order pick the +// body. Only this tier's Choose_* entry points stay external. +namespace { #include "VecSim/spaces/IP/IP_NEON_SQ8_FP16.h" #include "VecSim/spaces/L2/L2_NEON_SQ8_FP16.h" +} // namespace namespace spaces { diff --git a/src/VecSim/spaces/functions/NEON_HP.cpp b/src/VecSim/spaces/functions/NEON_HP.cpp index 20d93a517..412963066 100644 --- a/src/VecSim/spaces/functions/NEON_HP.cpp +++ b/src/VecSim/spaces/functions/NEON_HP.cpp @@ -8,10 +8,25 @@ */ #include "NEON_HP.h" +// Hoisted above the anonymous namespace below so that the standard library and the shared +// type headers keep external linkage. Wrapping them would pull and friends into +// the anonymous namespace and fail to compile. +#include "VecSim/spaces/space_includes.h" +#include "VecSim/spaces/spaces.h" +#include "VecSim/types/bfloat16.h" +#include "VecSim/types/float16.h" +#include "VecSim/types/sq8.h" +#include + +// Kernel instantiations get internal linkage, unique to this translation unit, so two tiers +// that share a kernel header cannot emit the same weak symbol and let link order pick the +// body. Only this tier's Choose_* entry points stay external. +namespace { #include "VecSim/spaces/L2/L2_NEON_FP16.h" #include "VecSim/spaces/IP/IP_NEON_FP16.h" #include "VecSim/spaces/IP/IP_NEON_SQ8_FP16.h" #include "VecSim/spaces/L2/L2_NEON_SQ8_FP16.h" +} // namespace namespace spaces { diff --git a/src/VecSim/spaces/functions/SVE.cpp b/src/VecSim/spaces/functions/SVE.cpp index bd197c84c..e306accbb 100644 --- a/src/VecSim/spaces/functions/SVE.cpp +++ b/src/VecSim/spaces/functions/SVE.cpp @@ -8,6 +8,21 @@ */ #include "SVE.h" +// Hoisted above the anonymous namespace below so that the standard library and the shared +// type headers keep external linkage. Wrapping them would pull and friends into +// the anonymous namespace and fail to compile. +#include "VecSim/spaces/space_includes.h" +#include "VecSim/spaces/spaces.h" +#include "VecSim/types/bfloat16.h" +#include "VecSim/types/float16.h" +#include "VecSim/types/sq8.h" +#include +#include + +// Kernel instantiations get internal linkage, unique to this translation unit, so two tiers +// that share a kernel header cannot emit the same weak symbol and let link order pick the +// body. Only this tier's Choose_* entry points stay external. +namespace { #include "VecSim/spaces/L2/L2_SVE_FP32.h" #include "VecSim/spaces/IP/IP_SVE_FP32.h" @@ -30,6 +45,7 @@ #include "VecSim/spaces/IP/IP_SVE_SQ8_SQ8.h" #include "VecSim/spaces/L2/L2_SVE_SQ8_SQ8.h" +} // namespace namespace spaces { diff --git a/src/VecSim/spaces/functions/SVE2.cpp b/src/VecSim/spaces/functions/SVE2.cpp index 9eea81523..d15780d73 100644 --- a/src/VecSim/spaces/functions/SVE2.cpp +++ b/src/VecSim/spaces/functions/SVE2.cpp @@ -8,6 +8,20 @@ */ #include "SVE2.h" +// Hoisted above the anonymous namespace below so that the standard library and the shared +// type headers keep external linkage. Wrapping them would pull and friends into +// the anonymous namespace and fail to compile. +#include "VecSim/spaces/space_includes.h" +#include "VecSim/spaces/spaces.h" +#include "VecSim/types/bfloat16.h" +#include "VecSim/types/float16.h" +#include "VecSim/types/sq8.h" +#include + +// Kernel instantiations get internal linkage, unique to this translation unit, so two tiers +// that share a kernel header cannot emit the same weak symbol and let link order pick the +// body. Only this tier's Choose_* entry points stay external. +namespace { #include "VecSim/spaces/L2/L2_SVE_FP32.h" #include "VecSim/spaces/IP/IP_SVE_FP32.h" @@ -26,6 +40,7 @@ #include "VecSim/spaces/L2/L2_SVE2_SQ8_FP16.h" // SVE2 fast path: FMLALB/FMLALT widening #include "VecSim/spaces/IP/IP_SVE_SQ8_SQ8.h" // SVE2 implementation is identical to SVE #include "VecSim/spaces/L2/L2_SVE_SQ8_SQ8.h" // SVE2 implementation is identical to SVE +} // namespace namespace spaces { diff --git a/src/VecSim/spaces/functions/SVE_BF16.cpp b/src/VecSim/spaces/functions/SVE_BF16.cpp index b457cdb7f..d7b9a6994 100644 --- a/src/VecSim/spaces/functions/SVE_BF16.cpp +++ b/src/VecSim/spaces/functions/SVE_BF16.cpp @@ -8,8 +8,23 @@ */ #include "SVE_BF16.h" +// Hoisted above the anonymous namespace below so that the standard library and the shared +// type headers keep external linkage. Wrapping them would pull and friends into +// the anonymous namespace and fail to compile. +#include "VecSim/spaces/space_includes.h" +#include "VecSim/spaces/spaces.h" +#include "VecSim/types/bfloat16.h" +#include "VecSim/types/float16.h" +#include "VecSim/types/sq8.h" +#include + +// Kernel instantiations get internal linkage, unique to this translation unit, so two tiers +// that share a kernel header cannot emit the same weak symbol and let link order pick the +// body. Only this tier's Choose_* entry points stay external. +namespace { #include "VecSim/spaces/IP/IP_SVE_BF16.h" #include "VecSim/spaces/L2/L2_SVE_BF16.h" +} // namespace namespace spaces { From 0d42fb4f9c6140956eddba55a3598db88e2b54b7 Mon Sep 17 00:00:00 2001 From: Dor Forer Date: Wed, 19 Aug 2026 18:00:34 +0300 Subject: [PATCH 2/4] Mirror the NEON_FHM dispatch predicate in its unit-test guards The three SQ8_FP16 optimization tests gated their FHM branch on optimization.asimdfhm alone, while the dispatcher now requires features.asimdhp && features.asimdfhm. The benchmark registrations already match the dispatcher; these three did not. Harmless in practice, since no core reports asimdfhm without asimdhp, but a test whose guard is looser than the code it tests will not catch the case it looks like it covers. --- tests/unit/test_spaces.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/unit/test_spaces.cpp b/tests/unit/test_spaces.cpp index 18ef7a625..96456a9c5 100644 --- a/tests/unit/test_spaces.cpp +++ b/tests/unit/test_spaces.cpp @@ -3372,7 +3372,7 @@ TEST_P(SQ8_FP16_SpacesOptimizationTest, SQ8_FP16_L2SqrTest) { } #endif #ifdef OPT_NEON_FHM - if (optimization.asimdfhm) { + if (optimization.asimdhp && optimization.asimdfhm) { unsigned char alignment = 0; arch_opt_func = L2_SQ8_FP16_GetDistFunc(dim, &alignment, &optimization); ASSERT_EQ(arch_opt_func, Choose_SQ8_FP16_L2_implementation_NEON_FHM(dim)) @@ -3498,7 +3498,7 @@ TEST_P(SQ8_FP16_SpacesOptimizationTest, SQ8_FP16_InnerProductTest) { } #endif #ifdef OPT_NEON_FHM - if (optimization.asimdfhm) { + if (optimization.asimdhp && optimization.asimdfhm) { unsigned char alignment = 0; arch_opt_func = IP_SQ8_FP16_GetDistFunc(dim, &alignment, &optimization); ASSERT_EQ(arch_opt_func, Choose_SQ8_FP16_IP_implementation_NEON_FHM(dim)) @@ -3624,7 +3624,7 @@ TEST_P(SQ8_FP16_SpacesOptimizationTest, SQ8_FP16_CosineTest) { } #endif #ifdef OPT_NEON_FHM - if (optimization.asimdfhm) { + if (optimization.asimdhp && optimization.asimdfhm) { unsigned char alignment = 0; arch_opt_func = Cosine_SQ8_FP16_GetDistFunc(dim, &alignment, &optimization); ASSERT_EQ(arch_opt_func, Choose_SQ8_FP16_Cosine_implementation_NEON_FHM(dim)) From 748d33d98419ef70d73bb0e29c414c2b719198ad Mon Sep 17 00:00:00 2001 From: Dor Forer Date: Wed, 19 Aug 2026 13:49:26 +0300 Subject: [PATCH 3/4] Gate the AVX512 VNNI tier on avx512vl at every call site The AVX512F_BW_VL_VNNI tier is compiled with -mavx512vl, but 6 of 12 runtime predicate sites omitted the avx512vl check, while 6 included it. A CPU with avx512f, avx512bw, and avx512vnni but without avx512vl would therefore be handed a function pointer into a TU the compiler was licensed to emit VL-encoded instructions in. Add avx512vl to all 12 sites, making them consistent. --- src/VecSim/spaces/IP_space.cpp | 10 ++++++---- src/VecSim/spaces/L2_space.cpp | 5 +++-- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/src/VecSim/spaces/IP_space.cpp b/src/VecSim/spaces/IP_space.cpp index b7f963d2c..d13cc1fc2 100644 --- a/src/VecSim/spaces/IP_space.cpp +++ b/src/VecSim/spaces/IP_space.cpp @@ -77,7 +77,7 @@ dist_func_t IP_SQ8_FP32_GetDistFunc(size_t dim, unsigned char *alignment, } // Alignment hints below refer to the SQ8 (first) operand per the GetDistFunc contract. #ifdef OPT_AVX512_F_BW_VL_VNNI - if (features.avx512f && features.avx512bw && features.avx512vnni) { + if (features.avx512f && features.avx512bw && features.avx512vl && features.avx512vnni) { if (dim % 16 == 0) // SQ8 chunk = 16 bytes *alignment = 16 * sizeof(uint8_t); return Choose_SQ8_FP32_IP_implementation_AVX512F_BW_VL_VNNI(dim); @@ -146,7 +146,7 @@ dist_func_t Cosine_SQ8_FP32_GetDistFunc(size_t dim, unsigned char *alignm } // Alignment hints below refer to the SQ8 (first) operand per the GetDistFunc contract. #ifdef OPT_AVX512_F_BW_VL_VNNI - if (features.avx512f && features.avx512bw && features.avx512vnni) { + if (features.avx512f && features.avx512bw && features.avx512vl && features.avx512vnni) { if (dim % 16 == 0) // SQ8 chunk = 16 bytes *alignment = 16 * sizeof(uint8_t); return Choose_SQ8_FP32_Cosine_implementation_AVX512F_BW_VL_VNNI(dim); @@ -375,7 +375,8 @@ dist_func_t IP_SQ8_SQ8_GetDistFunc(size_t dim, unsigned char *alignment, #ifdef CPU_FEATURES_ARCH_X86_64 #ifdef OPT_AVX512_F_BW_VL_VNNI // AVX512 VNNI SQ8_SQ8 uses 64-element chunks; residual handling is in 32-byte sub-chunks. - if (dim >= 64 && features.avx512f && features.avx512bw && features.avx512vnni) { + if (dim >= 64 && features.avx512f && features.avx512bw && features.avx512vl && + features.avx512vnni) { if (dim % 32 == 0) // align to 256 bits when there is no offsetting residual *alignment = 32 * sizeof(uint8_t); return Choose_SQ8_SQ8_IP_implementation_AVX512F_BW_VL_VNNI(dim); @@ -427,7 +428,8 @@ dist_func_t Cosine_SQ8_SQ8_GetDistFunc(size_t dim, unsigned char *alignme #ifdef CPU_FEATURES_ARCH_X86_64 #ifdef OPT_AVX512_F_BW_VL_VNNI // AVX512 VNNI SQ8_SQ8 uses 64-element chunks; residual handling is in 32-byte sub-chunks. - if (dim >= 64 && features.avx512f && features.avx512bw && features.avx512vnni) { + if (dim >= 64 && features.avx512f && features.avx512bw && features.avx512vl && + features.avx512vnni) { if (dim % 32 == 0) // align to 256 bits when there is no offsetting residual *alignment = 32 * sizeof(uint8_t); return Choose_SQ8_SQ8_Cosine_implementation_AVX512F_BW_VL_VNNI(dim); diff --git a/src/VecSim/spaces/L2_space.cpp b/src/VecSim/spaces/L2_space.cpp index 2c9223b67..bca916ab3 100644 --- a/src/VecSim/spaces/L2_space.cpp +++ b/src/VecSim/spaces/L2_space.cpp @@ -76,7 +76,7 @@ dist_func_t L2_SQ8_FP32_GetDistFunc(size_t dim, unsigned char *alignment, } // Alignment hints below refer to the SQ8 (first) operand per the GetDistFunc contract. #ifdef OPT_AVX512_F_BW_VL_VNNI - if (features.avx512f && features.avx512bw && features.avx512vnni) { + if (features.avx512f && features.avx512bw && features.avx512vl && features.avx512vnni) { if (dim % 16 == 0) // SQ8 chunk = 16 bytes; no point in aligning if there's a residual *alignment = 16 * sizeof(uint8_t); return Choose_SQ8_FP32_L2_implementation_AVX512F_BW_VL_VNNI(dim); @@ -563,7 +563,8 @@ dist_func_t L2_SQ8_SQ8_GetDistFunc(size_t dim, unsigned char *alignment, #ifdef CPU_FEATURES_ARCH_X86_64 #ifdef OPT_AVX512_F_BW_VL_VNNI // AVX512 VNNI SQ8_SQ8 uses 64-element chunks; residual handling is in 32-byte sub-chunks. - if (dim >= 64 && features.avx512f && features.avx512bw && features.avx512vnni) { + if (dim >= 64 && features.avx512f && features.avx512bw && features.avx512vl && + features.avx512vnni) { if (dim % 32 == 0) // align to 256 bits when there is no offsetting residual *alignment = 32 * sizeof(uint8_t); return Choose_SQ8_SQ8_L2_implementation_AVX512F_BW_VL_VNNI(dim); From ee83c9417e0766355f20fe87a79c6ce6b82d0ab6 Mon Sep 17 00:00:00 2001 From: Dor Forer Date: Sun, 23 Aug 2026 10:37:26 +0300 Subject: [PATCH 4/4] Mirror the AVX512 VNNI VL predicate in unit tests --- tests/unit/test_spaces.cpp | 70 +++++++++++++++++++++++++------------- 1 file changed, 46 insertions(+), 24 deletions(-) diff --git a/tests/unit/test_spaces.cpp b/tests/unit/test_spaces.cpp index 96456a9c5..f9ffa2c3e 100644 --- a/tests/unit/test_spaces.cpp +++ b/tests/unit/test_spaces.cpp @@ -2356,7 +2356,8 @@ TEST_P(SQ8_FP32_SpacesOptimizationTest, SQ8_FP32_L2SqrTest) { float baseline = SQ8_FP32_L2Sqr(v2_compressed.data(), v1_orig.data(), dim); // Test different optimizations based on CPU features #ifdef OPT_AVX512_F_BW_VL_VNNI - if (optimization.avx512f && optimization.avx512bw && optimization.avx512vnni) { + if (optimization.avx512f && optimization.avx512bw && optimization.avx512vl && + optimization.avx512vnni) { unsigned char alignment = 0; arch_opt_func = L2_SQ8_FP32_GetDistFunc(dim, &alignment, &optimization); ASSERT_EQ(arch_opt_func, Choose_SQ8_FP32_L2_implementation_AVX512F_BW_VL_VNNI(dim)) @@ -2482,7 +2483,8 @@ TEST_P(SQ8_FP32_SpacesOptimizationTest, SQ8_FP32_InnerProductTest) { // Test different optimizations based on CPU features #ifdef OPT_AVX512_F_BW_VL_VNNI - if (optimization.avx512f && optimization.avx512bw && optimization.avx512vnni) { + if (optimization.avx512f && optimization.avx512bw && optimization.avx512vl && + optimization.avx512vnni) { unsigned char alignment = 0; arch_opt_func = IP_SQ8_FP32_GetDistFunc(dim, &alignment, &optimization); ASSERT_EQ(arch_opt_func, Choose_SQ8_FP32_IP_implementation_AVX512F_BW_VL_VNNI(dim)) @@ -2633,7 +2635,8 @@ TEST_P(SQ8_FP32_SpacesOptimizationTest, SQ8_FP32_CosineTest) { // Test different optimizations based on CPU features #ifdef OPT_AVX512_F_BW_VL_VNNI - if (optimization.avx512f && optimization.avx512bw && optimization.avx512vnni) { + if (optimization.avx512f && optimization.avx512bw && optimization.avx512vl && + optimization.avx512vnni) { unsigned char alignment = 0; arch_opt_func = Cosine_SQ8_FP32_GetDistFunc(dim, &alignment, &optimization); ASSERT_EQ(arch_opt_func, Choose_SQ8_FP32_Cosine_implementation_AVX512F_BW_VL_VNNI(dim)) @@ -2745,7 +2748,8 @@ TEST(SQ8_FP32_EdgeCases, SelfDistanceCosine) { } #endif #ifdef OPT_AVX512_F_BW_VL_VNNI - if (optimization.avx512f && optimization.avx512bw && optimization.avx512vnni) { + if (optimization.avx512f && optimization.avx512bw && optimization.avx512vl && + optimization.avx512vnni) { unsigned char alignment = 0; auto arch_opt_func = Cosine_SQ8_FP32_GetDistFunc(dim, &alignment, &optimization); float result = arch_opt_func(v_quantized.data(), v_orig.data(), dim); @@ -2846,7 +2850,8 @@ TEST(SQ8_FP32_EdgeCases, SelfDistanceL2) { } #endif #ifdef OPT_AVX512_F_BW_VL_VNNI - if (optimization.avx512f && optimization.avx512bw && optimization.avx512vnni) { + if (optimization.avx512f && optimization.avx512bw && optimization.avx512vl && + optimization.avx512vnni) { unsigned char alignment = 0; auto arch_opt_func = L2_SQ8_FP32_GetDistFunc(dim, &alignment, &optimization); float result = arch_opt_func(v_quantized.data(), v_orig.data(), dim); @@ -2955,7 +2960,8 @@ TEST(SQ8_FP32_EdgeCases, CosineSymmetryTest) { } #endif #ifdef OPT_AVX512_F_BW_VL_VNNI - if (optimization.avx512f && optimization.avx512bw && optimization.avx512vnni) { + if (optimization.avx512f && optimization.avx512bw && optimization.avx512vl && + optimization.avx512vnni) { unsigned char alignment = 0; auto arch_opt_func = Cosine_SQ8_FP32_GetDistFunc(dim, &alignment, &optimization); float cos_12 = arch_opt_func(v2_quantized.data(), v1_fp32.data(), dim); @@ -3022,7 +3028,8 @@ TEST(SQ8_FP32_EdgeCases, CosineZeroVectorTest) { } #endif #ifdef OPT_AVX512_F_BW_VL_VNNI - if (optimization.avx512f && optimization.avx512bw && optimization.avx512vnni) { + if (optimization.avx512f && optimization.avx512bw && optimization.avx512vl && + optimization.avx512vnni) { unsigned char alignment = 0; auto arch_opt_func = Cosine_SQ8_FP32_GetDistFunc(dim, &alignment, &optimization); float result = arch_opt_func(v_nonzero_quantized.data(), v_zero.data(), dim); @@ -3102,7 +3109,8 @@ TEST(SQ8_FP32_EdgeCases, CosineConstantVectorTest) { } #endif #ifdef OPT_AVX512_F_BW_VL_VNNI - if (optimization.avx512f && optimization.avx512bw && optimization.avx512vnni) { + if (optimization.avx512f && optimization.avx512bw && optimization.avx512vl && + optimization.avx512vnni) { unsigned char alignment = 0; auto arch_opt_func = Cosine_SQ8_FP32_GetDistFunc(dim, &alignment, &optimization); float result = arch_opt_func(v_const_quantized.data(), v_query.data(), dim); @@ -3182,7 +3190,8 @@ TEST(SQ8_FP32_EdgeCases, CosineExtremeValuesTest) { } #endif #ifdef OPT_AVX512_F_BW_VL_VNNI - if (optimization.avx512f && optimization.avx512bw && optimization.avx512vnni) { + if (optimization.avx512f && optimization.avx512bw && optimization.avx512vl && + optimization.avx512vnni) { unsigned char alignment = 0; auto arch_opt_func = Cosine_SQ8_FP32_GetDistFunc(dim, &alignment, &optimization); float result = arch_opt_func(v2_quantized.data(), v1.data(), dim); @@ -3925,7 +3934,8 @@ TEST_P(SQ8_SQ8_SpacesOptimizationTest, SQ8_SQ8_InnerProductTest) { #endif #ifdef OPT_AVX512_F_BW_VL_VNNI - if (optimization.avx512f && optimization.avx512bw && optimization.avx512vnni) { + if (optimization.avx512f && optimization.avx512bw && optimization.avx512vl && + optimization.avx512vnni) { unsigned char alignment = 0; arch_opt_func = IP_SQ8_SQ8_GetDistFunc(dim, &alignment, &optimization); ASSERT_EQ(arch_opt_func, Choose_SQ8_SQ8_IP_implementation_AVX512F_BW_VL_VNNI(dim)) @@ -4008,7 +4018,8 @@ TEST_P(SQ8_SQ8_SpacesOptimizationTest, SQ8_SQ8_CosineTest) { #endif #ifdef OPT_AVX512_F_BW_VL_VNNI - if (optimization.avx512f && optimization.avx512bw && optimization.avx512vnni) { + if (optimization.avx512f && optimization.avx512bw && optimization.avx512vl && + optimization.avx512vnni) { unsigned char alignment = 0; arch_opt_func = Cosine_SQ8_SQ8_GetDistFunc(dim, &alignment, &optimization); ASSERT_EQ(arch_opt_func, Choose_SQ8_SQ8_Cosine_implementation_AVX512F_BW_VL_VNNI(dim)) @@ -4091,7 +4102,8 @@ TEST_P(SQ8_SQ8_SpacesOptimizationTest, SQ8_SQ8_L2SqrTest) { #endif #ifdef OPT_AVX512_F_BW_VL_VNNI - if (optimization.avx512f && optimization.avx512bw && optimization.avx512vnni) { + if (optimization.avx512f && optimization.avx512bw && optimization.avx512vl && + optimization.avx512vnni) { unsigned char alignment = 0; arch_opt_func = L2_SQ8_SQ8_GetDistFunc(dim, &alignment, &optimization); ASSERT_EQ(arch_opt_func, Choose_SQ8_SQ8_L2_implementation_AVX512F_BW_VL_VNNI(dim)) @@ -4170,7 +4182,8 @@ TEST(SQ8_SQ8_EdgeCases, SelfDistanceCosine) { } #endif #ifdef OPT_AVX512_F_BW_VL_VNNI - if (optimization.avx512f && optimization.avx512bw && optimization.avx512vnni) { + if (optimization.avx512f && optimization.avx512bw && optimization.avx512vl && + optimization.avx512vnni) { unsigned char alignment = 0; auto arch_opt_func = Cosine_SQ8_SQ8_GetDistFunc(dim, &alignment, &optimization); float result = arch_opt_func(v_quantized.data(), v_quantized.data(), dim); @@ -4242,7 +4255,8 @@ TEST(SQ8_SQ8_EdgeCases, CosineSymmetryTest) { } #endif #ifdef OPT_AVX512_F_BW_VL_VNNI - if (optimization.avx512f && optimization.avx512bw && optimization.avx512vnni) { + if (optimization.avx512f && optimization.avx512bw && optimization.avx512vl && + optimization.avx512vnni) { unsigned char alignment = 0; auto arch_opt_func = Cosine_SQ8_SQ8_GetDistFunc(dim, &alignment, &optimization); float cos_12 = arch_opt_func(v1_quantized.data(), v2_quantized.data(), dim); @@ -4310,7 +4324,8 @@ TEST(SQ8_SQ8_EdgeCases, CosineZeroVectorTest) { } #endif #ifdef OPT_AVX512_F_BW_VL_VNNI - if (optimization.avx512f && optimization.avx512bw && optimization.avx512vnni) { + if (optimization.avx512f && optimization.avx512bw && optimization.avx512vl && + optimization.avx512vnni) { unsigned char alignment = 0; auto arch_opt_func = Cosine_SQ8_SQ8_GetDistFunc(dim, &alignment, &optimization); float result = arch_opt_func(v_zero_quantized.data(), v_nonzero_quantized.data(), dim); @@ -4382,7 +4397,8 @@ TEST(SQ8_SQ8_EdgeCases, CosineConstantVectorTest) { } #endif #ifdef OPT_AVX512_F_BW_VL_VNNI - if (optimization.avx512f && optimization.avx512bw && optimization.avx512vnni) { + if (optimization.avx512f && optimization.avx512bw && optimization.avx512vl && + optimization.avx512vnni) { unsigned char alignment = 0; auto arch_opt_func = Cosine_SQ8_SQ8_GetDistFunc(dim, &alignment, &optimization); float result = arch_opt_func(v_const_quantized.data(), v_random_quantized.data(), dim); @@ -4463,7 +4479,8 @@ TEST(SQ8_SQ8_EdgeCases, CosineExtremeValuesTest) { } #endif #ifdef OPT_AVX512_F_BW_VL_VNNI - if (optimization.avx512f && optimization.avx512bw && optimization.avx512vnni) { + if (optimization.avx512f && optimization.avx512bw && optimization.avx512vl && + optimization.avx512vnni) { unsigned char alignment = 0; auto arch_opt_func = Cosine_SQ8_SQ8_GetDistFunc(dim, &alignment, &optimization); float result = arch_opt_func(v1_quantized.data(), v2_quantized.data(), dim); @@ -4532,7 +4549,8 @@ TEST(SQ8_SQ8_EdgeCases, SelfDistanceL2) { } #endif #ifdef OPT_AVX512_F_BW_VL_VNNI - if (optimization.avx512f && optimization.avx512bw && optimization.avx512vnni) { + if (optimization.avx512f && optimization.avx512bw && optimization.avx512vl && + optimization.avx512vnni) { unsigned char alignment = 0; auto arch_opt_func = L2_SQ8_SQ8_GetDistFunc(dim, &alignment, &optimization); float result = arch_opt_func(v_quantized.data(), v_quantized.data(), dim); @@ -4604,7 +4622,8 @@ TEST(SQ8_SQ8_EdgeCases, L2SymmetryTest) { } #endif #ifdef OPT_AVX512_F_BW_VL_VNNI - if (optimization.avx512f && optimization.avx512bw && optimization.avx512vnni) { + if (optimization.avx512f && optimization.avx512bw && optimization.avx512vl && + optimization.avx512vnni) { unsigned char alignment = 0; auto arch_opt_func = L2_SQ8_SQ8_GetDistFunc(dim, &alignment, &optimization); float l2_12 = arch_opt_func(v1_quantized.data(), v2_quantized.data(), dim); @@ -4672,7 +4691,8 @@ TEST(SQ8_SQ8_EdgeCases, L2ZeroVectorTest) { } #endif #ifdef OPT_AVX512_F_BW_VL_VNNI - if (optimization.avx512f && optimization.avx512bw && optimization.avx512vnni) { + if (optimization.avx512f && optimization.avx512bw && optimization.avx512vl && + optimization.avx512vnni) { unsigned char alignment = 0; auto arch_opt_func = L2_SQ8_SQ8_GetDistFunc(dim, &alignment, &optimization); float result = arch_opt_func(v_zero_quantized.data(), v_nonzero_quantized.data(), dim); @@ -4743,7 +4763,8 @@ TEST(SQ8_SQ8_EdgeCases, L2ConstantVectorTest) { } #endif #ifdef OPT_AVX512_F_BW_VL_VNNI - if (optimization.avx512f && optimization.avx512bw && optimization.avx512vnni) { + if (optimization.avx512f && optimization.avx512bw && optimization.avx512vl && + optimization.avx512vnni) { unsigned char alignment = 0; auto arch_opt_func = L2_SQ8_SQ8_GetDistFunc(dim, &alignment, &optimization); float result = arch_opt_func(v_const_quantized.data(), v_random_quantized.data(), dim); @@ -4817,7 +4838,8 @@ TEST(SQ8_SQ8_EdgeCases, L2ExtremeValuesTest) { } #endif #ifdef OPT_AVX512_F_BW_VL_VNNI - if (optimization.avx512f && optimization.avx512bw && optimization.avx512vnni) { + if (optimization.avx512f && optimization.avx512bw && optimization.avx512vl && + optimization.avx512vnni) { unsigned char alignment = 0; auto arch_opt_func = L2_SQ8_SQ8_GetDistFunc(dim, &alignment, &optimization); float result = arch_opt_func(v1_quantized.data(), v2_quantized.data(), dim); @@ -4846,7 +4868,7 @@ TEST_F(SpacesTest, SQ8_FP32_DispatcherAlignmentHints) { spaces::dist_func_t (*get)(size_t, unsigned char *, const void *)) { auto opt = features; #ifdef OPT_AVX512_F_BW_VL_VNNI - if (opt.avx512f && opt.avx512bw && opt.avx512vnni) { + if (opt.avx512f && opt.avx512bw && opt.avx512vl && opt.avx512vnni) { unsigned char alignment = 0; (void)get(dim, &alignment, &opt); ASSERT_EQ(alignment, 16u) << kind << ": AVX512 SQ8_FP32 hint should be 16"; @@ -4897,7 +4919,7 @@ TEST_F(SpacesTest, SQ8_SQ8_DispatcherAlignmentHints) { spaces::dist_func_t (*get)(size_t, unsigned char *, const void *)) { auto opt = features; #ifdef OPT_AVX512_F_BW_VL_VNNI - if (opt.avx512f && opt.avx512bw && opt.avx512vnni) { + if (opt.avx512f && opt.avx512bw && opt.avx512vl && opt.avx512vnni) { unsigned char alignment = 0; (void)get(dim, &alignment, &opt); ASSERT_EQ(alignment, 32u) << kind << ": AVX512 SQ8_SQ8 hint should be 32";