[MOD-17845] Gate AVX512 VNNI dispatch on AVX512VL

src/VecSim/spaces/IP_space.cpp

@@ -77,7 +77,7 @@ dist_func_t<float> IP_SQ8_FP32_GetDistFunc(size_t dim, unsigned char *alignment,
     }
     // Alignment hints below refer to the SQ8 (first) operand per the GetDistFunc contract.
 #ifdef OPT_AVX512_F_BW_VL_VNNI
-    if (features.avx512f && features.avx512bw && features.avx512vnni) {
+    if (features.avx512f && features.avx512bw && features.avx512vl && features.avx512vnni) {
         if (dim % 16 == 0) // SQ8 chunk = 16 bytes
             *alignment = 16 * sizeof(uint8_t);
         return Choose_SQ8_FP32_IP_implementation_AVX512F_BW_VL_VNNI(dim);
@@ -146,7 +146,7 @@ dist_func_t<float> Cosine_SQ8_FP32_GetDistFunc(size_t dim, unsigned char *alignm
     }
     // Alignment hints below refer to the SQ8 (first) operand per the GetDistFunc contract.
 #ifdef OPT_AVX512_F_BW_VL_VNNI
-    if (features.avx512f && features.avx512bw && features.avx512vnni) {
+    if (features.avx512f && features.avx512bw && features.avx512vl && features.avx512vnni) {
         if (dim % 16 == 0) // SQ8 chunk = 16 bytes
             *alignment = 16 * sizeof(uint8_t);
         return Choose_SQ8_FP32_Cosine_implementation_AVX512F_BW_VL_VNNI(dim);
@@ -375,7 +375,8 @@ dist_func_t<float> IP_SQ8_SQ8_GetDistFunc(size_t dim, unsigned char *alignment,
 #ifdef CPU_FEATURES_ARCH_X86_64
 #ifdef OPT_AVX512_F_BW_VL_VNNI
     // AVX512 VNNI SQ8_SQ8 uses 64-element chunks; residual handling is in 32-byte sub-chunks.
-    if (dim >= 64 && features.avx512f && features.avx512bw && features.avx512vnni) {
+    if (dim >= 64 && features.avx512f && features.avx512bw && features.avx512vl &&
+        features.avx512vnni) {
         if (dim % 32 == 0) // align to 256 bits when there is no offsetting residual
             *alignment = 32 * sizeof(uint8_t);
         return Choose_SQ8_SQ8_IP_implementation_AVX512F_BW_VL_VNNI(dim);
@@ -427,7 +428,8 @@ dist_func_t<float> Cosine_SQ8_SQ8_GetDistFunc(size_t dim, unsigned char *alignme
 #ifdef CPU_FEATURES_ARCH_X86_64
 #ifdef OPT_AVX512_F_BW_VL_VNNI
     // AVX512 VNNI SQ8_SQ8 uses 64-element chunks; residual handling is in 32-byte sub-chunks.
-    if (dim >= 64 && features.avx512f && features.avx512bw && features.avx512vnni) {
+    if (dim >= 64 && features.avx512f && features.avx512bw && features.avx512vl &&
+        features.avx512vnni) {
         if (dim % 32 == 0) // align to 256 bits when there is no offsetting residual
             *alignment = 32 * sizeof(uint8_t);
         return Choose_SQ8_SQ8_Cosine_implementation_AVX512F_BW_VL_VNNI(dim);

src/VecSim/spaces/L2_space.cpp

@@ -76,7 +76,7 @@ dist_func_t<float> L2_SQ8_FP32_GetDistFunc(size_t dim, unsigned char *alignment,
     }
     // Alignment hints below refer to the SQ8 (first) operand per the GetDistFunc contract.
 #ifdef OPT_AVX512_F_BW_VL_VNNI
-    if (features.avx512f && features.avx512bw && features.avx512vnni) {
+    if (features.avx512f && features.avx512bw && features.avx512vl && features.avx512vnni) {
         if (dim % 16 == 0) // SQ8 chunk = 16 bytes; no point in aligning if there's a residual
             *alignment = 16 * sizeof(uint8_t);
         return Choose_SQ8_FP32_L2_implementation_AVX512F_BW_VL_VNNI(dim);
@@ -563,7 +563,8 @@ dist_func_t<float> L2_SQ8_SQ8_GetDistFunc(size_t dim, unsigned char *alignment,
 #ifdef CPU_FEATURES_ARCH_X86_64
 #ifdef OPT_AVX512_F_BW_VL_VNNI
     // AVX512 VNNI SQ8_SQ8 uses 64-element chunks; residual handling is in 32-byte sub-chunks.
-    if (dim >= 64 && features.avx512f && features.avx512bw && features.avx512vnni) {
+    if (dim >= 64 && features.avx512f && features.avx512bw && features.avx512vl &&
+        features.avx512vnni) {
         if (dim % 32 == 0) // align to 256 bits when there is no offsetting residual
             *alignment = 32 * sizeof(uint8_t);
         return Choose_SQ8_SQ8_L2_implementation_AVX512F_BW_VL_VNNI(dim);

src/VecSim/spaces/functions/NEON.cpp

@@ -7,6 +7,21 @@
  * GNU Affero General Public License v3 (AGPLv3).
  */
 #include "NEON.h"
+
+// Hoisted above the anonymous namespace below so that the standard library and the shared
+// type headers keep external linkage. Wrapping them would pull <cstring> and friends into
+// the anonymous namespace and fail to compile.
+#include "VecSim/spaces/space_includes.h"
+#include "VecSim/spaces/spaces.h"
+#include "VecSim/types/bfloat16.h"
+#include "VecSim/types/float16.h"
+#include "VecSim/types/sq8.h"
+#include <arm_neon.h>
+
+// Kernel instantiations get internal linkage, unique to this translation unit, so two tiers
+// that share a kernel header cannot emit the same weak symbol and let link order pick the
+// body. Only this tier's Choose_* entry points stay external.
+namespace {
 #include "VecSim/spaces/L2/L2_NEON_FP32.h"
 #include "VecSim/spaces/IP/IP_NEON_FP32.h"
 #include "VecSim/spaces/L2/L2_NEON_INT8.h"
@@ -19,6 +34,7 @@
 #include "VecSim/spaces/IP/IP_NEON_SQ8_FP32.h"
 #include "VecSim/spaces/IP/IP_NEON_SQ8_SQ8.h"
 #include "VecSim/spaces/L2/L2_NEON_SQ8_SQ8.h"
+} // namespace
 
 namespace spaces {
 

src/VecSim/spaces/functions/NEON_BF16.cpp

@@ -8,8 +8,23 @@
  */
 #include "NEON_BF16.h"
 
+// Hoisted above the anonymous namespace below so that the standard library and the shared
+// type headers keep external linkage. Wrapping them would pull <cstring> and friends into
+// the anonymous namespace and fail to compile.
+#include "VecSim/spaces/space_includes.h"
+#include "VecSim/spaces/spaces.h"
+#include "VecSim/types/bfloat16.h"
+#include "VecSim/types/float16.h"
+#include "VecSim/types/sq8.h"
+#include <arm_neon.h>
+
+// Kernel instantiations get internal linkage, unique to this translation unit, so two tiers
+// that share a kernel header cannot emit the same weak symbol and let link order pick the
+// body. Only this tier's Choose_* entry points stay external.
+namespace {
 #include "VecSim/spaces/L2/L2_NEON_BF16.h"
 #include "VecSim/spaces/IP/IP_NEON_BF16.h"
+} // namespace
 
 namespace spaces {
 

src/VecSim/spaces/functions/NEON_DOTPROD.cpp

@@ -7,12 +7,28 @@
  * GNU Affero General Public License v3 (AGPLv3).
  */
 #include "NEON.h"
+
+// Hoisted above the anonymous namespace below so that the standard library and the shared
+// type headers keep external linkage. Wrapping them would pull <cstring> and friends into
+// the anonymous namespace and fail to compile.
+#include "VecSim/spaces/space_includes.h"
+#include "VecSim/spaces/spaces.h"
+#include "VecSim/types/bfloat16.h"
+#include "VecSim/types/float16.h"
+#include "VecSim/types/sq8.h"
+#include <arm_neon.h>
+
+// Kernel instantiations get internal linkage, unique to this translation unit, so two tiers
+// that share a kernel header cannot emit the same weak symbol and let link order pick the
+// body. Only this tier's Choose_* entry points stay external.
+namespace {
 #include "VecSim/spaces/IP/IP_NEON_DOTPROD_INT8.h"
 #include "VecSim/spaces/IP/IP_NEON_DOTPROD_UINT8.h"
 #include "VecSim/spaces/IP/IP_NEON_DOTPROD_SQ8_SQ8.h"
 #include "VecSim/spaces/L2/L2_NEON_DOTPROD_INT8.h"
 #include "VecSim/spaces/L2/L2_NEON_DOTPROD_UINT8.h"
 #include "VecSim/spaces/L2/L2_NEON_DOTPROD_SQ8_SQ8.h"
+} // namespace
 
 namespace spaces {
 

src/VecSim/spaces/functions/NEON_FHM.cpp

@@ -8,8 +8,23 @@
  */
 #include "NEON_FHM.h"
 
+// Hoisted above the anonymous namespace below so that the standard library and the shared
+// type headers keep external linkage. Wrapping them would pull <cstring> and friends into
+// the anonymous namespace and fail to compile.
+#include "VecSim/spaces/space_includes.h"
+#include "VecSim/spaces/spaces.h"
+#include "VecSim/types/bfloat16.h"
+#include "VecSim/types/float16.h"
+#include "VecSim/types/sq8.h"
+#include <arm_neon.h>
+
+// Kernel instantiations get internal linkage, unique to this translation unit, so two tiers
+// that share a kernel header cannot emit the same weak symbol and let link order pick the
+// body. Only this tier's Choose_* entry points stay external.
+namespace {
 #include "VecSim/spaces/IP/IP_NEON_SQ8_FP16.h"
 #include "VecSim/spaces/L2/L2_NEON_SQ8_FP16.h"
+} // namespace
 
 namespace spaces {
 

src/VecSim/spaces/functions/NEON_HP.cpp

@@ -8,10 +8,25 @@
  */
 #include "NEON_HP.h"
 
+// Hoisted above the anonymous namespace below so that the standard library and the shared
+// type headers keep external linkage. Wrapping them would pull <cstring> and friends into
+// the anonymous namespace and fail to compile.
+#include "VecSim/spaces/space_includes.h"
+#include "VecSim/spaces/spaces.h"
+#include "VecSim/types/bfloat16.h"
+#include "VecSim/types/float16.h"
+#include "VecSim/types/sq8.h"
+#include <arm_neon.h>
+
+// Kernel instantiations get internal linkage, unique to this translation unit, so two tiers
+// that share a kernel header cannot emit the same weak symbol and let link order pick the
+// body. Only this tier's Choose_* entry points stay external.
+namespace {
 #include "VecSim/spaces/L2/L2_NEON_FP16.h"
 #include "VecSim/spaces/IP/IP_NEON_FP16.h"
 #include "VecSim/spaces/IP/IP_NEON_SQ8_FP16.h"
 #include "VecSim/spaces/L2/L2_NEON_SQ8_FP16.h"
+} // namespace
 
 namespace spaces {
 

src/VecSim/spaces/functions/SVE.cpp

@@ -8,6 +8,21 @@
  */
 #include "SVE.h"
 
+// Hoisted above the anonymous namespace below so that the standard library and the shared
+// type headers keep external linkage. Wrapping them would pull <cstring> and friends into
+// the anonymous namespace and fail to compile.
+#include "VecSim/spaces/space_includes.h"
+#include "VecSim/spaces/spaces.h"
+#include "VecSim/types/bfloat16.h"
+#include "VecSim/types/float16.h"
+#include "VecSim/types/sq8.h"
+#include <arm_sve.h>
+#include <arm_neon.h>
+
+// Kernel instantiations get internal linkage, unique to this translation unit, so two tiers
+// that share a kernel header cannot emit the same weak symbol and let link order pick the
+// body. Only this tier's Choose_* entry points stay external.
+namespace {
 #include "VecSim/spaces/L2/L2_SVE_FP32.h"
 #include "VecSim/spaces/IP/IP_SVE_FP32.h"
 
@@ -30,6 +45,7 @@
 
 #include "VecSim/spaces/IP/IP_SVE_SQ8_SQ8.h"
 #include "VecSim/spaces/L2/L2_SVE_SQ8_SQ8.h"
+} // namespace
 
 namespace spaces {
 

src/VecSim/spaces/functions/SVE2.cpp

@@ -8,6 +8,20 @@
  */
 #include "SVE2.h"
 
+// Hoisted above the anonymous namespace below so that the standard library and the shared
+// type headers keep external linkage. Wrapping them would pull <cstring> and friends into
+// the anonymous namespace and fail to compile.
+#include "VecSim/spaces/space_includes.h"
+#include "VecSim/spaces/spaces.h"
+#include "VecSim/types/bfloat16.h"
+#include "VecSim/types/float16.h"
+#include "VecSim/types/sq8.h"
+#include <arm_sve.h>
+
+// Kernel instantiations get internal linkage, unique to this translation unit, so two tiers
+// that share a kernel header cannot emit the same weak symbol and let link order pick the
+// body. Only this tier's Choose_* entry points stay external.
+namespace {
 #include "VecSim/spaces/L2/L2_SVE_FP32.h"
 #include "VecSim/spaces/IP/IP_SVE_FP32.h"
 
@@ -26,6 +40,7 @@
 #include "VecSim/spaces/L2/L2_SVE2_SQ8_FP16.h" // SVE2 fast path: FMLALB/FMLALT widening
 #include "VecSim/spaces/IP/IP_SVE_SQ8_SQ8.h"   // SVE2 implementation is identical to SVE
 #include "VecSim/spaces/L2/L2_SVE_SQ8_SQ8.h"   // SVE2 implementation is identical to SVE
+} // namespace
 
 namespace spaces {
 

src/VecSim/spaces/functions/SVE_BF16.cpp

@@ -8,8 +8,23 @@
  */
 #include "SVE_BF16.h"
 
+// Hoisted above the anonymous namespace below so that the standard library and the shared
+// type headers keep external linkage. Wrapping them would pull <cstring> and friends into
+// the anonymous namespace and fail to compile.
+#include "VecSim/spaces/space_includes.h"
+#include "VecSim/spaces/spaces.h"
+#include "VecSim/types/bfloat16.h"
+#include "VecSim/types/float16.h"
+#include "VecSim/types/sq8.h"
+#include <arm_sve.h>
+
+// Kernel instantiations get internal linkage, unique to this translation unit, so two tiers
+// that share a kernel header cannot emit the same weak symbol and let link order pick the
+// body. Only this tier's Choose_* entry points stay external.
+namespace {
 #include "VecSim/spaces/IP/IP_SVE_BF16.h"
 #include "VecSim/spaces/L2/L2_SVE_BF16.h"
+} // namespace
 
 namespace spaces {