[MOD-17844] Stop ARM SIMD tiers executing instructions the running CPU lacks

cmake/aarch64InstructionFlags.cmake

@@ -9,7 +9,8 @@ CHECK_CXX_COMPILER_FLAG("-march=armv8-a" CXX_ARMV8A)
 CHECK_CXX_COMPILER_FLAG("-march=armv8.2-a+dotprod" CXX_NEON_DOTPROD)
 CHECK_CXX_COMPILER_FLAG("-march=armv8-a+sve" CXX_SVE)
 CHECK_CXX_COMPILER_FLAG("-march=armv9-a+sve2" CXX_SVE2)
-CHECK_CXX_COMPILER_FLAG("-march=armv8.2-a+fp16fml" CXX_NEON_HP)
+CHECK_CXX_COMPILER_FLAG("-march=armv8.2-a+fp16" CXX_NEON_HP)
+CHECK_CXX_COMPILER_FLAG("-march=armv8.2-a+fp16fml" CXX_NEON_FHM)
 CHECK_CXX_COMPILER_FLAG("-march=armv8.2-a+bf16" CXX_NEON_BF16)
 CHECK_CXX_COMPILER_FLAG("-march=armv8.2-a+sve+bf16" CXX_SVE_BF16)
 
@@ -29,6 +30,10 @@ if (CXX_NEON_HP)
   message(STATUS "Using ARMv8.2-a with NEON half-percision extension")
   add_compile_definitions(OPT_NEON_HP)
 endif()
+if (CXX_NEON_FHM)
+  message(STATUS "Using ARMv8.2-a with NEON FHM extension")
+  add_compile_definitions(OPT_NEON_FHM)
+endif()
 if (CXX_NEON_BF16)
   add_compile_definitions(OPT_NEON_BF16)
 endif()

src/VecSim/spaces/CMakeLists.txt

@@ -141,10 +141,17 @@ if (CMAKE_SYSTEM_PROCESSOR MATCHES "(aarch64)|(arm64)|(ARM64)|(armv.*)")
 	# NEON half-precision support
 	if (CXX_NEON_HP AND CXX_ARMV8A)
 		message("Building with NEON+HP")
-		set_source_files_properties(functions/NEON_HP.cpp PROPERTIES COMPILE_FLAGS "-march=armv8.2-a+fp16fml")
+		set_source_files_properties(functions/NEON_HP.cpp PROPERTIES COMPILE_FLAGS "-march=armv8.2-a+fp16")
 		list(APPEND OPTIMIZATIONS functions/NEON_HP.cpp)
   endif()
 
+	# NEON FHM (FEAT_FHM / asimdfhm) support
+	if (CXX_NEON_FHM AND CXX_ARMV8A)
+		message("Building with NEON+FHM")
+		set_source_files_properties(functions/NEON_FHM.cpp PROPERTIES COMPILE_FLAGS "-march=armv8.2-a+fp16fml")
+		list(APPEND OPTIMIZATIONS functions/NEON_FHM.cpp)
+	endif()
+
 	# NEON bfloat16 support
 	if (CXX_NEON_BF16)
 		message("Building with NEON + BF16")

src/VecSim/spaces/IP_space.cpp

@@ -29,6 +29,7 @@
 #include "VecSim/spaces/functions/NEON.h"
 #include "VecSim/spaces/functions/NEON_DOTPROD.h"
 #include "VecSim/spaces/functions/NEON_HP.h"
+#include "VecSim/spaces/functions/NEON_FHM.h"
 #include "VecSim/spaces/functions/NEON_BF16.h"
 #include "VecSim/spaces/functions/SVE.h"
 #include "VecSim/spaces/functions/SVE_BF16.h"
@@ -241,10 +242,12 @@ dist_func_t<float> IP_SQ8_FP16_GetDistFunc(size_t dim, unsigned char *alignment,
         return Choose_SQ8_FP16_IP_implementation_SVE(dim);
     }
 #endif
-#ifdef OPT_NEON_HP
-    if (features.asimdfhm) {
+#ifdef OPT_NEON_FHM
+    if (features.asimdhp && features.asimdfhm) {
         return Choose_SQ8_FP16_IP_implementation_NEON_FHM(dim);
     }
+#endif
+#ifdef OPT_NEON_HP
     if (features.asimdhp) {
         return Choose_SQ8_FP16_IP_implementation_NEON_HP(dim);
     }
@@ -313,10 +316,12 @@ dist_func_t<float> Cosine_SQ8_FP16_GetDistFunc(size_t dim, unsigned char *alignm
         return Choose_SQ8_FP16_Cosine_implementation_SVE(dim);
     }
 #endif
-#ifdef OPT_NEON_HP
-    if (features.asimdfhm) {
+#ifdef OPT_NEON_FHM
+    if (features.asimdhp && features.asimdfhm) {
         return Choose_SQ8_FP16_Cosine_implementation_NEON_FHM(dim);
     }
+#endif
+#ifdef OPT_NEON_HP
     if (features.asimdhp) {
         return Choose_SQ8_FP16_Cosine_implementation_NEON_HP(dim);
     }

src/VecSim/spaces/L2_space.cpp

@@ -28,6 +28,7 @@
 #include "VecSim/spaces/functions/NEON.h"
 #include "VecSim/spaces/functions/NEON_DOTPROD.h"
 #include "VecSim/spaces/functions/NEON_HP.h"
+#include "VecSim/spaces/functions/NEON_FHM.h"
 #include "VecSim/spaces/functions/NEON_BF16.h"
 #include "VecSim/spaces/functions/SVE.h"
 #include "VecSim/spaces/functions/SVE_BF16.h"
@@ -170,10 +171,12 @@ dist_func_t<float> L2_SQ8_FP16_GetDistFunc(size_t dim, unsigned char *alignment,
         return Choose_SQ8_FP16_L2_implementation_SVE(dim);
     }
 #endif
-#ifdef OPT_NEON_HP
-    if (features.asimdfhm) {
+#ifdef OPT_NEON_FHM
+    if (features.asimdhp && features.asimdfhm) {
         return Choose_SQ8_FP16_L2_implementation_NEON_FHM(dim);
     }
+#endif
+#ifdef OPT_NEON_HP
     if (features.asimdhp) {
         return Choose_SQ8_FP16_L2_implementation_NEON_HP(dim);
     }

src/VecSim/spaces/functions/NEON.cpp

@@ -7,6 +7,21 @@
  * GNU Affero General Public License v3 (AGPLv3).
  */
 #include "NEON.h"
+
+// Hoisted above the anonymous namespace below so that the standard library and the shared
+// type headers keep external linkage. Wrapping them would pull <cstring> and friends into
+// the anonymous namespace and fail to compile.
+#include "VecSim/spaces/space_includes.h"
+#include "VecSim/spaces/spaces.h"
+#include "VecSim/types/bfloat16.h"
+#include "VecSim/types/float16.h"
+#include "VecSim/types/sq8.h"
+#include <arm_neon.h>
+
+// Kernel instantiations get internal linkage, unique to this translation unit, so two tiers
+// that share a kernel header cannot emit the same weak symbol and let link order pick the
+// body. Only this tier's Choose_* entry points stay external.
+namespace {
 #include "VecSim/spaces/L2/L2_NEON_FP32.h"
 #include "VecSim/spaces/IP/IP_NEON_FP32.h"
 #include "VecSim/spaces/L2/L2_NEON_INT8.h"
@@ -19,6 +34,7 @@
 #include "VecSim/spaces/IP/IP_NEON_SQ8_FP32.h"
 #include "VecSim/spaces/IP/IP_NEON_SQ8_SQ8.h"
 #include "VecSim/spaces/L2/L2_NEON_SQ8_SQ8.h"
+} // namespace
 
 namespace spaces {
 

src/VecSim/spaces/functions/NEON_BF16.cpp

@@ -8,8 +8,23 @@
  */
 #include "NEON_BF16.h"
 
+// Hoisted above the anonymous namespace below so that the standard library and the shared
+// type headers keep external linkage. Wrapping them would pull <cstring> and friends into
+// the anonymous namespace and fail to compile.
+#include "VecSim/spaces/space_includes.h"
+#include "VecSim/spaces/spaces.h"
+#include "VecSim/types/bfloat16.h"
+#include "VecSim/types/float16.h"
+#include "VecSim/types/sq8.h"
+#include <arm_neon.h>
+
+// Kernel instantiations get internal linkage, unique to this translation unit, so two tiers
+// that share a kernel header cannot emit the same weak symbol and let link order pick the
+// body. Only this tier's Choose_* entry points stay external.
+namespace {
 #include "VecSim/spaces/L2/L2_NEON_BF16.h"
 #include "VecSim/spaces/IP/IP_NEON_BF16.h"
+} // namespace
 
 namespace spaces {
 

src/VecSim/spaces/functions/NEON_DOTPROD.cpp

@@ -7,12 +7,28 @@
  * GNU Affero General Public License v3 (AGPLv3).
  */
 #include "NEON.h"
+
+// Hoisted above the anonymous namespace below so that the standard library and the shared
+// type headers keep external linkage. Wrapping them would pull <cstring> and friends into
+// the anonymous namespace and fail to compile.
+#include "VecSim/spaces/space_includes.h"
+#include "VecSim/spaces/spaces.h"
+#include "VecSim/types/bfloat16.h"
+#include "VecSim/types/float16.h"
+#include "VecSim/types/sq8.h"
+#include <arm_neon.h>
+
+// Kernel instantiations get internal linkage, unique to this translation unit, so two tiers
+// that share a kernel header cannot emit the same weak symbol and let link order pick the
+// body. Only this tier's Choose_* entry points stay external.
+namespace {
 #include "VecSim/spaces/IP/IP_NEON_DOTPROD_INT8.h"
 #include "VecSim/spaces/IP/IP_NEON_DOTPROD_UINT8.h"
 #include "VecSim/spaces/IP/IP_NEON_DOTPROD_SQ8_SQ8.h"
 #include "VecSim/spaces/L2/L2_NEON_DOTPROD_INT8.h"
 #include "VecSim/spaces/L2/L2_NEON_DOTPROD_UINT8.h"
 #include "VecSim/spaces/L2/L2_NEON_DOTPROD_SQ8_SQ8.h"
+} // namespace
 
 namespace spaces {
 

src/VecSim/spaces/functions/NEON_FHM.cpp

@@ -0,0 +1,53 @@
+/*
+ * Copyright (c) 2006-Present, Redis Ltd.
+ * All rights reserved.
+ *
+ * Licensed under your choice of the Redis Source Available License 2.0
+ * (RSALv2); or (b) the Server Side Public License v1 (SSPLv1); or (c) the
+ * GNU Affero General Public License v3 (AGPLv3).
+ */
+#include "NEON_FHM.h"
+
+// Hoisted above the anonymous namespace below so that the standard library and the shared
+// type headers keep external linkage. Wrapping them would pull <cstring> and friends into
+// the anonymous namespace and fail to compile.
+#include "VecSim/spaces/space_includes.h"
+#include "VecSim/spaces/spaces.h"
+#include "VecSim/types/bfloat16.h"
+#include "VecSim/types/float16.h"
+#include "VecSim/types/sq8.h"
+#include <arm_neon.h>
+
+// Kernel instantiations get internal linkage, unique to this translation unit, so two tiers
+// that share a kernel header cannot emit the same weak symbol and let link order pick the
+// body. Only this tier's Choose_* entry points stay external.
+namespace {
+#include "VecSim/spaces/IP/IP_NEON_SQ8_FP16.h"
+#include "VecSim/spaces/L2/L2_NEON_SQ8_FP16.h"
+} // namespace
+
+namespace spaces {
+
+#include "implementation_chooser.h"
+
+dist_func_t<float> Choose_SQ8_FP16_IP_implementation_NEON_FHM(size_t dim) {
+    dist_func_t<float> ret_dist_func;
+    CHOOSE_IMPLEMENTATION(ret_dist_func, dim, 16, SQ8_FP16_InnerProductSIMD16_NEON_FHM);
+    return ret_dist_func;
+}
+
+dist_func_t<float> Choose_SQ8_FP16_L2_implementation_NEON_FHM(size_t dim) {
+    dist_func_t<float> ret_dist_func;
+    CHOOSE_IMPLEMENTATION(ret_dist_func, dim, 16, SQ8_FP16_L2SqrSIMD16_NEON_FHM);
+    return ret_dist_func;
+}
+
+dist_func_t<float> Choose_SQ8_FP16_Cosine_implementation_NEON_FHM(size_t dim) {
+    dist_func_t<float> ret_dist_func;
+    CHOOSE_IMPLEMENTATION(ret_dist_func, dim, 16, SQ8_FP16_CosineSIMD16_NEON_FHM);
+    return ret_dist_func;
+}
+
+#include "implementation_chooser_cleanup.h"
+
+} // namespace spaces

src/VecSim/spaces/functions/NEON_FHM.h

@@ -0,0 +1,19 @@
+/*
+ * Copyright (c) 2006-Present, Redis Ltd.
+ * All rights reserved.
+ *
+ * Licensed under your choice of the Redis Source Available License 2.0
+ * (RSALv2); or (b) the Server Side Public License v1 (SSPLv1); or (c) the
+ * GNU Affero General Public License v3 (AGPLv3).
+ */
+#pragma once
+
+#include "VecSim/spaces/spaces.h"
+
+namespace spaces {
+
+dist_func_t<float> Choose_SQ8_FP16_IP_implementation_NEON_FHM(size_t dim);
+dist_func_t<float> Choose_SQ8_FP16_L2_implementation_NEON_FHM(size_t dim);
+dist_func_t<float> Choose_SQ8_FP16_Cosine_implementation_NEON_FHM(size_t dim);
+
+} // namespace spaces

src/VecSim/spaces/functions/NEON_HP.cpp

@@ -8,10 +8,25 @@
  */
 #include "NEON_HP.h"
 
+// Hoisted above the anonymous namespace below so that the standard library and the shared
+// type headers keep external linkage. Wrapping them would pull <cstring> and friends into
+// the anonymous namespace and fail to compile.
+#include "VecSim/spaces/space_includes.h"
+#include "VecSim/spaces/spaces.h"
+#include "VecSim/types/bfloat16.h"
+#include "VecSim/types/float16.h"
+#include "VecSim/types/sq8.h"
+#include <arm_neon.h>
+
+// Kernel instantiations get internal linkage, unique to this translation unit, so two tiers
+// that share a kernel header cannot emit the same weak symbol and let link order pick the
+// body. Only this tier's Choose_* entry points stay external.
+namespace {
 #include "VecSim/spaces/L2/L2_NEON_FP16.h"
 #include "VecSim/spaces/IP/IP_NEON_FP16.h"
 #include "VecSim/spaces/IP/IP_NEON_SQ8_FP16.h"
 #include "VecSim/spaces/L2/L2_NEON_SQ8_FP16.h"
+} // namespace
 
 namespace spaces {
 
@@ -47,25 +62,6 @@ dist_func_t<float> Choose_SQ8_FP16_Cosine_implementation_NEON_HP(size_t dim) {
     return ret_dist_func;
 }
 
-// FMLAL (FEAT_FHM / asimdfhm) variants.
-dist_func_t<float> Choose_SQ8_FP16_IP_implementation_NEON_FHM(size_t dim) {
-    dist_func_t<float> ret_dist_func;
-    CHOOSE_IMPLEMENTATION(ret_dist_func, dim, 16, SQ8_FP16_InnerProductSIMD16_NEON_FHM);
-    return ret_dist_func;
-}
-
-dist_func_t<float> Choose_SQ8_FP16_L2_implementation_NEON_FHM(size_t dim) {
-    dist_func_t<float> ret_dist_func;
-    CHOOSE_IMPLEMENTATION(ret_dist_func, dim, 16, SQ8_FP16_L2SqrSIMD16_NEON_FHM);
-    return ret_dist_func;
-}
-
-dist_func_t<float> Choose_SQ8_FP16_Cosine_implementation_NEON_FHM(size_t dim) {
-    dist_func_t<float> ret_dist_func;
-    CHOOSE_IMPLEMENTATION(ret_dist_func, dim, 16, SQ8_FP16_CosineSIMD16_NEON_FHM);
-    return ret_dist_func;
-}
-
 #include "implementation_chooser_cleanup.h"
 
 } // namespace spaces

src/VecSim/spaces/functions/NEON_HP.h

@@ -20,8 +20,4 @@ dist_func_t<float> Choose_SQ8_FP16_IP_implementation_NEON_HP(size_t dim);
 dist_func_t<float> Choose_SQ8_FP16_L2_implementation_NEON_HP(size_t dim);
 dist_func_t<float> Choose_SQ8_FP16_Cosine_implementation_NEON_HP(size_t dim);
 
-dist_func_t<float> Choose_SQ8_FP16_IP_implementation_NEON_FHM(size_t dim);
-dist_func_t<float> Choose_SQ8_FP16_L2_implementation_NEON_FHM(size_t dim);
-dist_func_t<float> Choose_SQ8_FP16_Cosine_implementation_NEON_FHM(size_t dim);
-
 } // namespace spaces

src/VecSim/spaces/functions/SVE.cpp

@@ -8,6 +8,21 @@
  */
 #include "SVE.h"
 
+// Hoisted above the anonymous namespace below so that the standard library and the shared
+// type headers keep external linkage. Wrapping them would pull <cstring> and friends into
+// the anonymous namespace and fail to compile.
+#include "VecSim/spaces/space_includes.h"
+#include "VecSim/spaces/spaces.h"
+#include "VecSim/types/bfloat16.h"
+#include "VecSim/types/float16.h"
+#include "VecSim/types/sq8.h"
+#include <arm_sve.h>
+#include <arm_neon.h>
+
+// Kernel instantiations get internal linkage, unique to this translation unit, so two tiers
+// that share a kernel header cannot emit the same weak symbol and let link order pick the
+// body. Only this tier's Choose_* entry points stay external.
+namespace {
 #include "VecSim/spaces/L2/L2_SVE_FP32.h"
 #include "VecSim/spaces/IP/IP_SVE_FP32.h"
 
@@ -30,6 +45,7 @@
 
 #include "VecSim/spaces/IP/IP_SVE_SQ8_SQ8.h"
 #include "VecSim/spaces/L2/L2_SVE_SQ8_SQ8.h"
+} // namespace
 
 namespace spaces {
 

src/VecSim/spaces/functions/SVE2.cpp

@@ -8,6 +8,20 @@
  */
 #include "SVE2.h"
 
+// Hoisted above the anonymous namespace below so that the standard library and the shared
+// type headers keep external linkage. Wrapping them would pull <cstring> and friends into
+// the anonymous namespace and fail to compile.
+#include "VecSim/spaces/space_includes.h"
+#include "VecSim/spaces/spaces.h"
+#include "VecSim/types/bfloat16.h"
+#include "VecSim/types/float16.h"
+#include "VecSim/types/sq8.h"
+#include <arm_sve.h>
+
+// Kernel instantiations get internal linkage, unique to this translation unit, so two tiers
+// that share a kernel header cannot emit the same weak symbol and let link order pick the
+// body. Only this tier's Choose_* entry points stay external.
+namespace {
 #include "VecSim/spaces/L2/L2_SVE_FP32.h"
 #include "VecSim/spaces/IP/IP_SVE_FP32.h"
 
@@ -26,6 +40,7 @@
 #include "VecSim/spaces/L2/L2_SVE2_SQ8_FP16.h" // SVE2 fast path: FMLALB/FMLALT widening
 #include "VecSim/spaces/IP/IP_SVE_SQ8_SQ8.h"   // SVE2 implementation is identical to SVE
 #include "VecSim/spaces/L2/L2_SVE_SQ8_SQ8.h"   // SVE2 implementation is identical to SVE
+} // namespace
 
 namespace spaces {