diff --git a/src/VecSim/spaces/IP_space.cpp b/src/VecSim/spaces/IP_space.cpp index b7f963d2c..d13cc1fc2 100644 --- a/src/VecSim/spaces/IP_space.cpp +++ b/src/VecSim/spaces/IP_space.cpp @@ -77,7 +77,7 @@ dist_func_t IP_SQ8_FP32_GetDistFunc(size_t dim, unsigned char *alignment, } // Alignment hints below refer to the SQ8 (first) operand per the GetDistFunc contract. #ifdef OPT_AVX512_F_BW_VL_VNNI - if (features.avx512f && features.avx512bw && features.avx512vnni) { + if (features.avx512f && features.avx512bw && features.avx512vl && features.avx512vnni) { if (dim % 16 == 0) // SQ8 chunk = 16 bytes *alignment = 16 * sizeof(uint8_t); return Choose_SQ8_FP32_IP_implementation_AVX512F_BW_VL_VNNI(dim); @@ -146,7 +146,7 @@ dist_func_t Cosine_SQ8_FP32_GetDistFunc(size_t dim, unsigned char *alignm } // Alignment hints below refer to the SQ8 (first) operand per the GetDistFunc contract. #ifdef OPT_AVX512_F_BW_VL_VNNI - if (features.avx512f && features.avx512bw && features.avx512vnni) { + if (features.avx512f && features.avx512bw && features.avx512vl && features.avx512vnni) { if (dim % 16 == 0) // SQ8 chunk = 16 bytes *alignment = 16 * sizeof(uint8_t); return Choose_SQ8_FP32_Cosine_implementation_AVX512F_BW_VL_VNNI(dim); @@ -375,7 +375,8 @@ dist_func_t IP_SQ8_SQ8_GetDistFunc(size_t dim, unsigned char *alignment, #ifdef CPU_FEATURES_ARCH_X86_64 #ifdef OPT_AVX512_F_BW_VL_VNNI // AVX512 VNNI SQ8_SQ8 uses 64-element chunks; residual handling is in 32-byte sub-chunks. - if (dim >= 64 && features.avx512f && features.avx512bw && features.avx512vnni) { + if (dim >= 64 && features.avx512f && features.avx512bw && features.avx512vl && + features.avx512vnni) { if (dim % 32 == 0) // align to 256 bits when there is no offsetting residual *alignment = 32 * sizeof(uint8_t); return Choose_SQ8_SQ8_IP_implementation_AVX512F_BW_VL_VNNI(dim); @@ -427,7 +428,8 @@ dist_func_t Cosine_SQ8_SQ8_GetDistFunc(size_t dim, unsigned char *alignme #ifdef CPU_FEATURES_ARCH_X86_64 #ifdef OPT_AVX512_F_BW_VL_VNNI // AVX512 VNNI SQ8_SQ8 uses 64-element chunks; residual handling is in 32-byte sub-chunks. - if (dim >= 64 && features.avx512f && features.avx512bw && features.avx512vnni) { + if (dim >= 64 && features.avx512f && features.avx512bw && features.avx512vl && + features.avx512vnni) { if (dim % 32 == 0) // align to 256 bits when there is no offsetting residual *alignment = 32 * sizeof(uint8_t); return Choose_SQ8_SQ8_Cosine_implementation_AVX512F_BW_VL_VNNI(dim); diff --git a/src/VecSim/spaces/L2_space.cpp b/src/VecSim/spaces/L2_space.cpp index 2c9223b67..bca916ab3 100644 --- a/src/VecSim/spaces/L2_space.cpp +++ b/src/VecSim/spaces/L2_space.cpp @@ -76,7 +76,7 @@ dist_func_t L2_SQ8_FP32_GetDistFunc(size_t dim, unsigned char *alignment, } // Alignment hints below refer to the SQ8 (first) operand per the GetDistFunc contract. #ifdef OPT_AVX512_F_BW_VL_VNNI - if (features.avx512f && features.avx512bw && features.avx512vnni) { + if (features.avx512f && features.avx512bw && features.avx512vl && features.avx512vnni) { if (dim % 16 == 0) // SQ8 chunk = 16 bytes; no point in aligning if there's a residual *alignment = 16 * sizeof(uint8_t); return Choose_SQ8_FP32_L2_implementation_AVX512F_BW_VL_VNNI(dim); @@ -563,7 +563,8 @@ dist_func_t L2_SQ8_SQ8_GetDistFunc(size_t dim, unsigned char *alignment, #ifdef CPU_FEATURES_ARCH_X86_64 #ifdef OPT_AVX512_F_BW_VL_VNNI // AVX512 VNNI SQ8_SQ8 uses 64-element chunks; residual handling is in 32-byte sub-chunks. - if (dim >= 64 && features.avx512f && features.avx512bw && features.avx512vnni) { + if (dim >= 64 && features.avx512f && features.avx512bw && features.avx512vl && + features.avx512vnni) { if (dim % 32 == 0) // align to 256 bits when there is no offsetting residual *alignment = 32 * sizeof(uint8_t); return Choose_SQ8_SQ8_L2_implementation_AVX512F_BW_VL_VNNI(dim); diff --git a/src/VecSim/spaces/functions/NEON.cpp b/src/VecSim/spaces/functions/NEON.cpp index 0c9a286e3..8d5c286be 100644 --- a/src/VecSim/spaces/functions/NEON.cpp +++ b/src/VecSim/spaces/functions/NEON.cpp @@ -7,6 +7,21 @@ * GNU Affero General Public License v3 (AGPLv3). */ #include "NEON.h" + +// Hoisted above the anonymous namespace below so that the standard library and the shared +// type headers keep external linkage. Wrapping them would pull and friends into +// the anonymous namespace and fail to compile. +#include "VecSim/spaces/space_includes.h" +#include "VecSim/spaces/spaces.h" +#include "VecSim/types/bfloat16.h" +#include "VecSim/types/float16.h" +#include "VecSim/types/sq8.h" +#include + +// Kernel instantiations get internal linkage, unique to this translation unit, so two tiers +// that share a kernel header cannot emit the same weak symbol and let link order pick the +// body. Only this tier's Choose_* entry points stay external. +namespace { #include "VecSim/spaces/L2/L2_NEON_FP32.h" #include "VecSim/spaces/IP/IP_NEON_FP32.h" #include "VecSim/spaces/L2/L2_NEON_INT8.h" @@ -19,6 +34,7 @@ #include "VecSim/spaces/IP/IP_NEON_SQ8_FP32.h" #include "VecSim/spaces/IP/IP_NEON_SQ8_SQ8.h" #include "VecSim/spaces/L2/L2_NEON_SQ8_SQ8.h" +} // namespace namespace spaces { diff --git a/src/VecSim/spaces/functions/NEON_BF16.cpp b/src/VecSim/spaces/functions/NEON_BF16.cpp index 4de205bb8..1a515943e 100644 --- a/src/VecSim/spaces/functions/NEON_BF16.cpp +++ b/src/VecSim/spaces/functions/NEON_BF16.cpp @@ -8,8 +8,23 @@ */ #include "NEON_BF16.h" +// Hoisted above the anonymous namespace below so that the standard library and the shared +// type headers keep external linkage. Wrapping them would pull and friends into +// the anonymous namespace and fail to compile. +#include "VecSim/spaces/space_includes.h" +#include "VecSim/spaces/spaces.h" +#include "VecSim/types/bfloat16.h" +#include "VecSim/types/float16.h" +#include "VecSim/types/sq8.h" +#include + +// Kernel instantiations get internal linkage, unique to this translation unit, so two tiers +// that share a kernel header cannot emit the same weak symbol and let link order pick the +// body. Only this tier's Choose_* entry points stay external. +namespace { #include "VecSim/spaces/L2/L2_NEON_BF16.h" #include "VecSim/spaces/IP/IP_NEON_BF16.h" +} // namespace namespace spaces { diff --git a/src/VecSim/spaces/functions/NEON_DOTPROD.cpp b/src/VecSim/spaces/functions/NEON_DOTPROD.cpp index 12f762093..ade17bb31 100644 --- a/src/VecSim/spaces/functions/NEON_DOTPROD.cpp +++ b/src/VecSim/spaces/functions/NEON_DOTPROD.cpp @@ -7,12 +7,28 @@ * GNU Affero General Public License v3 (AGPLv3). */ #include "NEON.h" + +// Hoisted above the anonymous namespace below so that the standard library and the shared +// type headers keep external linkage. Wrapping them would pull and friends into +// the anonymous namespace and fail to compile. +#include "VecSim/spaces/space_includes.h" +#include "VecSim/spaces/spaces.h" +#include "VecSim/types/bfloat16.h" +#include "VecSim/types/float16.h" +#include "VecSim/types/sq8.h" +#include + +// Kernel instantiations get internal linkage, unique to this translation unit, so two tiers +// that share a kernel header cannot emit the same weak symbol and let link order pick the +// body. Only this tier's Choose_* entry points stay external. +namespace { #include "VecSim/spaces/IP/IP_NEON_DOTPROD_INT8.h" #include "VecSim/spaces/IP/IP_NEON_DOTPROD_UINT8.h" #include "VecSim/spaces/IP/IP_NEON_DOTPROD_SQ8_SQ8.h" #include "VecSim/spaces/L2/L2_NEON_DOTPROD_INT8.h" #include "VecSim/spaces/L2/L2_NEON_DOTPROD_UINT8.h" #include "VecSim/spaces/L2/L2_NEON_DOTPROD_SQ8_SQ8.h" +} // namespace namespace spaces { diff --git a/src/VecSim/spaces/functions/NEON_FHM.cpp b/src/VecSim/spaces/functions/NEON_FHM.cpp index 2beceed0c..ed92bec47 100644 --- a/src/VecSim/spaces/functions/NEON_FHM.cpp +++ b/src/VecSim/spaces/functions/NEON_FHM.cpp @@ -8,8 +8,23 @@ */ #include "NEON_FHM.h" +// Hoisted above the anonymous namespace below so that the standard library and the shared +// type headers keep external linkage. Wrapping them would pull and friends into +// the anonymous namespace and fail to compile. +#include "VecSim/spaces/space_includes.h" +#include "VecSim/spaces/spaces.h" +#include "VecSim/types/bfloat16.h" +#include "VecSim/types/float16.h" +#include "VecSim/types/sq8.h" +#include + +// Kernel instantiations get internal linkage, unique to this translation unit, so two tiers +// that share a kernel header cannot emit the same weak symbol and let link order pick the +// body. Only this tier's Choose_* entry points stay external. +namespace { #include "VecSim/spaces/IP/IP_NEON_SQ8_FP16.h" #include "VecSim/spaces/L2/L2_NEON_SQ8_FP16.h" +} // namespace namespace spaces { diff --git a/src/VecSim/spaces/functions/NEON_HP.cpp b/src/VecSim/spaces/functions/NEON_HP.cpp index 20d93a517..412963066 100644 --- a/src/VecSim/spaces/functions/NEON_HP.cpp +++ b/src/VecSim/spaces/functions/NEON_HP.cpp @@ -8,10 +8,25 @@ */ #include "NEON_HP.h" +// Hoisted above the anonymous namespace below so that the standard library and the shared +// type headers keep external linkage. Wrapping them would pull and friends into +// the anonymous namespace and fail to compile. +#include "VecSim/spaces/space_includes.h" +#include "VecSim/spaces/spaces.h" +#include "VecSim/types/bfloat16.h" +#include "VecSim/types/float16.h" +#include "VecSim/types/sq8.h" +#include + +// Kernel instantiations get internal linkage, unique to this translation unit, so two tiers +// that share a kernel header cannot emit the same weak symbol and let link order pick the +// body. Only this tier's Choose_* entry points stay external. +namespace { #include "VecSim/spaces/L2/L2_NEON_FP16.h" #include "VecSim/spaces/IP/IP_NEON_FP16.h" #include "VecSim/spaces/IP/IP_NEON_SQ8_FP16.h" #include "VecSim/spaces/L2/L2_NEON_SQ8_FP16.h" +} // namespace namespace spaces { diff --git a/src/VecSim/spaces/functions/SVE.cpp b/src/VecSim/spaces/functions/SVE.cpp index bd197c84c..e306accbb 100644 --- a/src/VecSim/spaces/functions/SVE.cpp +++ b/src/VecSim/spaces/functions/SVE.cpp @@ -8,6 +8,21 @@ */ #include "SVE.h" +// Hoisted above the anonymous namespace below so that the standard library and the shared +// type headers keep external linkage. Wrapping them would pull and friends into +// the anonymous namespace and fail to compile. +#include "VecSim/spaces/space_includes.h" +#include "VecSim/spaces/spaces.h" +#include "VecSim/types/bfloat16.h" +#include "VecSim/types/float16.h" +#include "VecSim/types/sq8.h" +#include +#include + +// Kernel instantiations get internal linkage, unique to this translation unit, so two tiers +// that share a kernel header cannot emit the same weak symbol and let link order pick the +// body. Only this tier's Choose_* entry points stay external. +namespace { #include "VecSim/spaces/L2/L2_SVE_FP32.h" #include "VecSim/spaces/IP/IP_SVE_FP32.h" @@ -30,6 +45,7 @@ #include "VecSim/spaces/IP/IP_SVE_SQ8_SQ8.h" #include "VecSim/spaces/L2/L2_SVE_SQ8_SQ8.h" +} // namespace namespace spaces { diff --git a/src/VecSim/spaces/functions/SVE2.cpp b/src/VecSim/spaces/functions/SVE2.cpp index 9eea81523..d15780d73 100644 --- a/src/VecSim/spaces/functions/SVE2.cpp +++ b/src/VecSim/spaces/functions/SVE2.cpp @@ -8,6 +8,20 @@ */ #include "SVE2.h" +// Hoisted above the anonymous namespace below so that the standard library and the shared +// type headers keep external linkage. Wrapping them would pull and friends into +// the anonymous namespace and fail to compile. +#include "VecSim/spaces/space_includes.h" +#include "VecSim/spaces/spaces.h" +#include "VecSim/types/bfloat16.h" +#include "VecSim/types/float16.h" +#include "VecSim/types/sq8.h" +#include + +// Kernel instantiations get internal linkage, unique to this translation unit, so two tiers +// that share a kernel header cannot emit the same weak symbol and let link order pick the +// body. Only this tier's Choose_* entry points stay external. +namespace { #include "VecSim/spaces/L2/L2_SVE_FP32.h" #include "VecSim/spaces/IP/IP_SVE_FP32.h" @@ -26,6 +40,7 @@ #include "VecSim/spaces/L2/L2_SVE2_SQ8_FP16.h" // SVE2 fast path: FMLALB/FMLALT widening #include "VecSim/spaces/IP/IP_SVE_SQ8_SQ8.h" // SVE2 implementation is identical to SVE #include "VecSim/spaces/L2/L2_SVE_SQ8_SQ8.h" // SVE2 implementation is identical to SVE +} // namespace namespace spaces { diff --git a/src/VecSim/spaces/functions/SVE_BF16.cpp b/src/VecSim/spaces/functions/SVE_BF16.cpp index b457cdb7f..d7b9a6994 100644 --- a/src/VecSim/spaces/functions/SVE_BF16.cpp +++ b/src/VecSim/spaces/functions/SVE_BF16.cpp @@ -8,8 +8,23 @@ */ #include "SVE_BF16.h" +// Hoisted above the anonymous namespace below so that the standard library and the shared +// type headers keep external linkage. Wrapping them would pull and friends into +// the anonymous namespace and fail to compile. +#include "VecSim/spaces/space_includes.h" +#include "VecSim/spaces/spaces.h" +#include "VecSim/types/bfloat16.h" +#include "VecSim/types/float16.h" +#include "VecSim/types/sq8.h" +#include + +// Kernel instantiations get internal linkage, unique to this translation unit, so two tiers +// that share a kernel header cannot emit the same weak symbol and let link order pick the +// body. Only this tier's Choose_* entry points stay external. +namespace { #include "VecSim/spaces/IP/IP_SVE_BF16.h" #include "VecSim/spaces/L2/L2_SVE_BF16.h" +} // namespace namespace spaces { diff --git a/tests/unit/test_spaces.cpp b/tests/unit/test_spaces.cpp index 18ef7a625..f9ffa2c3e 100644 --- a/tests/unit/test_spaces.cpp +++ b/tests/unit/test_spaces.cpp @@ -2356,7 +2356,8 @@ TEST_P(SQ8_FP32_SpacesOptimizationTest, SQ8_FP32_L2SqrTest) { float baseline = SQ8_FP32_L2Sqr(v2_compressed.data(), v1_orig.data(), dim); // Test different optimizations based on CPU features #ifdef OPT_AVX512_F_BW_VL_VNNI - if (optimization.avx512f && optimization.avx512bw && optimization.avx512vnni) { + if (optimization.avx512f && optimization.avx512bw && optimization.avx512vl && + optimization.avx512vnni) { unsigned char alignment = 0; arch_opt_func = L2_SQ8_FP32_GetDistFunc(dim, &alignment, &optimization); ASSERT_EQ(arch_opt_func, Choose_SQ8_FP32_L2_implementation_AVX512F_BW_VL_VNNI(dim)) @@ -2482,7 +2483,8 @@ TEST_P(SQ8_FP32_SpacesOptimizationTest, SQ8_FP32_InnerProductTest) { // Test different optimizations based on CPU features #ifdef OPT_AVX512_F_BW_VL_VNNI - if (optimization.avx512f && optimization.avx512bw && optimization.avx512vnni) { + if (optimization.avx512f && optimization.avx512bw && optimization.avx512vl && + optimization.avx512vnni) { unsigned char alignment = 0; arch_opt_func = IP_SQ8_FP32_GetDistFunc(dim, &alignment, &optimization); ASSERT_EQ(arch_opt_func, Choose_SQ8_FP32_IP_implementation_AVX512F_BW_VL_VNNI(dim)) @@ -2633,7 +2635,8 @@ TEST_P(SQ8_FP32_SpacesOptimizationTest, SQ8_FP32_CosineTest) { // Test different optimizations based on CPU features #ifdef OPT_AVX512_F_BW_VL_VNNI - if (optimization.avx512f && optimization.avx512bw && optimization.avx512vnni) { + if (optimization.avx512f && optimization.avx512bw && optimization.avx512vl && + optimization.avx512vnni) { unsigned char alignment = 0; arch_opt_func = Cosine_SQ8_FP32_GetDistFunc(dim, &alignment, &optimization); ASSERT_EQ(arch_opt_func, Choose_SQ8_FP32_Cosine_implementation_AVX512F_BW_VL_VNNI(dim)) @@ -2745,7 +2748,8 @@ TEST(SQ8_FP32_EdgeCases, SelfDistanceCosine) { } #endif #ifdef OPT_AVX512_F_BW_VL_VNNI - if (optimization.avx512f && optimization.avx512bw && optimization.avx512vnni) { + if (optimization.avx512f && optimization.avx512bw && optimization.avx512vl && + optimization.avx512vnni) { unsigned char alignment = 0; auto arch_opt_func = Cosine_SQ8_FP32_GetDistFunc(dim, &alignment, &optimization); float result = arch_opt_func(v_quantized.data(), v_orig.data(), dim); @@ -2846,7 +2850,8 @@ TEST(SQ8_FP32_EdgeCases, SelfDistanceL2) { } #endif #ifdef OPT_AVX512_F_BW_VL_VNNI - if (optimization.avx512f && optimization.avx512bw && optimization.avx512vnni) { + if (optimization.avx512f && optimization.avx512bw && optimization.avx512vl && + optimization.avx512vnni) { unsigned char alignment = 0; auto arch_opt_func = L2_SQ8_FP32_GetDistFunc(dim, &alignment, &optimization); float result = arch_opt_func(v_quantized.data(), v_orig.data(), dim); @@ -2955,7 +2960,8 @@ TEST(SQ8_FP32_EdgeCases, CosineSymmetryTest) { } #endif #ifdef OPT_AVX512_F_BW_VL_VNNI - if (optimization.avx512f && optimization.avx512bw && optimization.avx512vnni) { + if (optimization.avx512f && optimization.avx512bw && optimization.avx512vl && + optimization.avx512vnni) { unsigned char alignment = 0; auto arch_opt_func = Cosine_SQ8_FP32_GetDistFunc(dim, &alignment, &optimization); float cos_12 = arch_opt_func(v2_quantized.data(), v1_fp32.data(), dim); @@ -3022,7 +3028,8 @@ TEST(SQ8_FP32_EdgeCases, CosineZeroVectorTest) { } #endif #ifdef OPT_AVX512_F_BW_VL_VNNI - if (optimization.avx512f && optimization.avx512bw && optimization.avx512vnni) { + if (optimization.avx512f && optimization.avx512bw && optimization.avx512vl && + optimization.avx512vnni) { unsigned char alignment = 0; auto arch_opt_func = Cosine_SQ8_FP32_GetDistFunc(dim, &alignment, &optimization); float result = arch_opt_func(v_nonzero_quantized.data(), v_zero.data(), dim); @@ -3102,7 +3109,8 @@ TEST(SQ8_FP32_EdgeCases, CosineConstantVectorTest) { } #endif #ifdef OPT_AVX512_F_BW_VL_VNNI - if (optimization.avx512f && optimization.avx512bw && optimization.avx512vnni) { + if (optimization.avx512f && optimization.avx512bw && optimization.avx512vl && + optimization.avx512vnni) { unsigned char alignment = 0; auto arch_opt_func = Cosine_SQ8_FP32_GetDistFunc(dim, &alignment, &optimization); float result = arch_opt_func(v_const_quantized.data(), v_query.data(), dim); @@ -3182,7 +3190,8 @@ TEST(SQ8_FP32_EdgeCases, CosineExtremeValuesTest) { } #endif #ifdef OPT_AVX512_F_BW_VL_VNNI - if (optimization.avx512f && optimization.avx512bw && optimization.avx512vnni) { + if (optimization.avx512f && optimization.avx512bw && optimization.avx512vl && + optimization.avx512vnni) { unsigned char alignment = 0; auto arch_opt_func = Cosine_SQ8_FP32_GetDistFunc(dim, &alignment, &optimization); float result = arch_opt_func(v2_quantized.data(), v1.data(), dim); @@ -3372,7 +3381,7 @@ TEST_P(SQ8_FP16_SpacesOptimizationTest, SQ8_FP16_L2SqrTest) { } #endif #ifdef OPT_NEON_FHM - if (optimization.asimdfhm) { + if (optimization.asimdhp && optimization.asimdfhm) { unsigned char alignment = 0; arch_opt_func = L2_SQ8_FP16_GetDistFunc(dim, &alignment, &optimization); ASSERT_EQ(arch_opt_func, Choose_SQ8_FP16_L2_implementation_NEON_FHM(dim)) @@ -3498,7 +3507,7 @@ TEST_P(SQ8_FP16_SpacesOptimizationTest, SQ8_FP16_InnerProductTest) { } #endif #ifdef OPT_NEON_FHM - if (optimization.asimdfhm) { + if (optimization.asimdhp && optimization.asimdfhm) { unsigned char alignment = 0; arch_opt_func = IP_SQ8_FP16_GetDistFunc(dim, &alignment, &optimization); ASSERT_EQ(arch_opt_func, Choose_SQ8_FP16_IP_implementation_NEON_FHM(dim)) @@ -3624,7 +3633,7 @@ TEST_P(SQ8_FP16_SpacesOptimizationTest, SQ8_FP16_CosineTest) { } #endif #ifdef OPT_NEON_FHM - if (optimization.asimdfhm) { + if (optimization.asimdhp && optimization.asimdfhm) { unsigned char alignment = 0; arch_opt_func = Cosine_SQ8_FP16_GetDistFunc(dim, &alignment, &optimization); ASSERT_EQ(arch_opt_func, Choose_SQ8_FP16_Cosine_implementation_NEON_FHM(dim)) @@ -3925,7 +3934,8 @@ TEST_P(SQ8_SQ8_SpacesOptimizationTest, SQ8_SQ8_InnerProductTest) { #endif #ifdef OPT_AVX512_F_BW_VL_VNNI - if (optimization.avx512f && optimization.avx512bw && optimization.avx512vnni) { + if (optimization.avx512f && optimization.avx512bw && optimization.avx512vl && + optimization.avx512vnni) { unsigned char alignment = 0; arch_opt_func = IP_SQ8_SQ8_GetDistFunc(dim, &alignment, &optimization); ASSERT_EQ(arch_opt_func, Choose_SQ8_SQ8_IP_implementation_AVX512F_BW_VL_VNNI(dim)) @@ -4008,7 +4018,8 @@ TEST_P(SQ8_SQ8_SpacesOptimizationTest, SQ8_SQ8_CosineTest) { #endif #ifdef OPT_AVX512_F_BW_VL_VNNI - if (optimization.avx512f && optimization.avx512bw && optimization.avx512vnni) { + if (optimization.avx512f && optimization.avx512bw && optimization.avx512vl && + optimization.avx512vnni) { unsigned char alignment = 0; arch_opt_func = Cosine_SQ8_SQ8_GetDistFunc(dim, &alignment, &optimization); ASSERT_EQ(arch_opt_func, Choose_SQ8_SQ8_Cosine_implementation_AVX512F_BW_VL_VNNI(dim)) @@ -4091,7 +4102,8 @@ TEST_P(SQ8_SQ8_SpacesOptimizationTest, SQ8_SQ8_L2SqrTest) { #endif #ifdef OPT_AVX512_F_BW_VL_VNNI - if (optimization.avx512f && optimization.avx512bw && optimization.avx512vnni) { + if (optimization.avx512f && optimization.avx512bw && optimization.avx512vl && + optimization.avx512vnni) { unsigned char alignment = 0; arch_opt_func = L2_SQ8_SQ8_GetDistFunc(dim, &alignment, &optimization); ASSERT_EQ(arch_opt_func, Choose_SQ8_SQ8_L2_implementation_AVX512F_BW_VL_VNNI(dim)) @@ -4170,7 +4182,8 @@ TEST(SQ8_SQ8_EdgeCases, SelfDistanceCosine) { } #endif #ifdef OPT_AVX512_F_BW_VL_VNNI - if (optimization.avx512f && optimization.avx512bw && optimization.avx512vnni) { + if (optimization.avx512f && optimization.avx512bw && optimization.avx512vl && + optimization.avx512vnni) { unsigned char alignment = 0; auto arch_opt_func = Cosine_SQ8_SQ8_GetDistFunc(dim, &alignment, &optimization); float result = arch_opt_func(v_quantized.data(), v_quantized.data(), dim); @@ -4242,7 +4255,8 @@ TEST(SQ8_SQ8_EdgeCases, CosineSymmetryTest) { } #endif #ifdef OPT_AVX512_F_BW_VL_VNNI - if (optimization.avx512f && optimization.avx512bw && optimization.avx512vnni) { + if (optimization.avx512f && optimization.avx512bw && optimization.avx512vl && + optimization.avx512vnni) { unsigned char alignment = 0; auto arch_opt_func = Cosine_SQ8_SQ8_GetDistFunc(dim, &alignment, &optimization); float cos_12 = arch_opt_func(v1_quantized.data(), v2_quantized.data(), dim); @@ -4310,7 +4324,8 @@ TEST(SQ8_SQ8_EdgeCases, CosineZeroVectorTest) { } #endif #ifdef OPT_AVX512_F_BW_VL_VNNI - if (optimization.avx512f && optimization.avx512bw && optimization.avx512vnni) { + if (optimization.avx512f && optimization.avx512bw && optimization.avx512vl && + optimization.avx512vnni) { unsigned char alignment = 0; auto arch_opt_func = Cosine_SQ8_SQ8_GetDistFunc(dim, &alignment, &optimization); float result = arch_opt_func(v_zero_quantized.data(), v_nonzero_quantized.data(), dim); @@ -4382,7 +4397,8 @@ TEST(SQ8_SQ8_EdgeCases, CosineConstantVectorTest) { } #endif #ifdef OPT_AVX512_F_BW_VL_VNNI - if (optimization.avx512f && optimization.avx512bw && optimization.avx512vnni) { + if (optimization.avx512f && optimization.avx512bw && optimization.avx512vl && + optimization.avx512vnni) { unsigned char alignment = 0; auto arch_opt_func = Cosine_SQ8_SQ8_GetDistFunc(dim, &alignment, &optimization); float result = arch_opt_func(v_const_quantized.data(), v_random_quantized.data(), dim); @@ -4463,7 +4479,8 @@ TEST(SQ8_SQ8_EdgeCases, CosineExtremeValuesTest) { } #endif #ifdef OPT_AVX512_F_BW_VL_VNNI - if (optimization.avx512f && optimization.avx512bw && optimization.avx512vnni) { + if (optimization.avx512f && optimization.avx512bw && optimization.avx512vl && + optimization.avx512vnni) { unsigned char alignment = 0; auto arch_opt_func = Cosine_SQ8_SQ8_GetDistFunc(dim, &alignment, &optimization); float result = arch_opt_func(v1_quantized.data(), v2_quantized.data(), dim); @@ -4532,7 +4549,8 @@ TEST(SQ8_SQ8_EdgeCases, SelfDistanceL2) { } #endif #ifdef OPT_AVX512_F_BW_VL_VNNI - if (optimization.avx512f && optimization.avx512bw && optimization.avx512vnni) { + if (optimization.avx512f && optimization.avx512bw && optimization.avx512vl && + optimization.avx512vnni) { unsigned char alignment = 0; auto arch_opt_func = L2_SQ8_SQ8_GetDistFunc(dim, &alignment, &optimization); float result = arch_opt_func(v_quantized.data(), v_quantized.data(), dim); @@ -4604,7 +4622,8 @@ TEST(SQ8_SQ8_EdgeCases, L2SymmetryTest) { } #endif #ifdef OPT_AVX512_F_BW_VL_VNNI - if (optimization.avx512f && optimization.avx512bw && optimization.avx512vnni) { + if (optimization.avx512f && optimization.avx512bw && optimization.avx512vl && + optimization.avx512vnni) { unsigned char alignment = 0; auto arch_opt_func = L2_SQ8_SQ8_GetDistFunc(dim, &alignment, &optimization); float l2_12 = arch_opt_func(v1_quantized.data(), v2_quantized.data(), dim); @@ -4672,7 +4691,8 @@ TEST(SQ8_SQ8_EdgeCases, L2ZeroVectorTest) { } #endif #ifdef OPT_AVX512_F_BW_VL_VNNI - if (optimization.avx512f && optimization.avx512bw && optimization.avx512vnni) { + if (optimization.avx512f && optimization.avx512bw && optimization.avx512vl && + optimization.avx512vnni) { unsigned char alignment = 0; auto arch_opt_func = L2_SQ8_SQ8_GetDistFunc(dim, &alignment, &optimization); float result = arch_opt_func(v_zero_quantized.data(), v_nonzero_quantized.data(), dim); @@ -4743,7 +4763,8 @@ TEST(SQ8_SQ8_EdgeCases, L2ConstantVectorTest) { } #endif #ifdef OPT_AVX512_F_BW_VL_VNNI - if (optimization.avx512f && optimization.avx512bw && optimization.avx512vnni) { + if (optimization.avx512f && optimization.avx512bw && optimization.avx512vl && + optimization.avx512vnni) { unsigned char alignment = 0; auto arch_opt_func = L2_SQ8_SQ8_GetDistFunc(dim, &alignment, &optimization); float result = arch_opt_func(v_const_quantized.data(), v_random_quantized.data(), dim); @@ -4817,7 +4838,8 @@ TEST(SQ8_SQ8_EdgeCases, L2ExtremeValuesTest) { } #endif #ifdef OPT_AVX512_F_BW_VL_VNNI - if (optimization.avx512f && optimization.avx512bw && optimization.avx512vnni) { + if (optimization.avx512f && optimization.avx512bw && optimization.avx512vl && + optimization.avx512vnni) { unsigned char alignment = 0; auto arch_opt_func = L2_SQ8_SQ8_GetDistFunc(dim, &alignment, &optimization); float result = arch_opt_func(v1_quantized.data(), v2_quantized.data(), dim); @@ -4846,7 +4868,7 @@ TEST_F(SpacesTest, SQ8_FP32_DispatcherAlignmentHints) { spaces::dist_func_t (*get)(size_t, unsigned char *, const void *)) { auto opt = features; #ifdef OPT_AVX512_F_BW_VL_VNNI - if (opt.avx512f && opt.avx512bw && opt.avx512vnni) { + if (opt.avx512f && opt.avx512bw && opt.avx512vl && opt.avx512vnni) { unsigned char alignment = 0; (void)get(dim, &alignment, &opt); ASSERT_EQ(alignment, 16u) << kind << ": AVX512 SQ8_FP32 hint should be 16"; @@ -4897,7 +4919,7 @@ TEST_F(SpacesTest, SQ8_SQ8_DispatcherAlignmentHints) { spaces::dist_func_t (*get)(size_t, unsigned char *, const void *)) { auto opt = features; #ifdef OPT_AVX512_F_BW_VL_VNNI - if (opt.avx512f && opt.avx512bw && opt.avx512vnni) { + if (opt.avx512f && opt.avx512bw && opt.avx512vl && opt.avx512vnni) { unsigned char alignment = 0; (void)get(dim, &alignment, &opt); ASSERT_EQ(alignment, 32u) << kind << ": AVX512 SQ8_SQ8 hint should be 32";