From 0ebdd4653ab207439bbf20cc598ca9b0975d352b Mon Sep 17 00:00:00 2001 From: CodSpeed Bot Date: Wed, 29 Jul 2026 14:47:06 +0000 Subject: [PATCH] ggml-cpu: keep the generic SIMD vec kernel accumulators in registers The generic GGML_SIMD kernels keep their vector operands and accumulators in small arrays indexed by the inner loop counter. Those arrays only stay in registers if the loop is unrolled, which GCC does at -O3 but not at -O2, so RelWithDebInfo builds pay a stack load/store round trip for every vector operation (and re-zero the accumulator array on every call to ggml_vec_dot_f32). Add a GGML_UNROLL(n) helper and apply it to the constant-trip inner loops of the generic kernels, making their codegen independent of the optimization level. No arithmetic is changed and results are bit-identical; -O3 output is unchanged. --- ggml/src/ggml-cpu/simd-mappings.h | 17 +++++++++++++++++ ggml/src/ggml-cpu/vec.cpp | 2 ++ ggml/src/ggml-cpu/vec.h | 9 +++++++++ 3 files changed, 28 insertions(+) diff --git a/ggml/src/ggml-cpu/simd-mappings.h b/ggml/src/ggml-cpu/simd-mappings.h index fca5119e1a13..7d8486edbf6a 100644 --- a/ggml/src/ggml-cpu/simd-mappings.h +++ b/ggml/src/ggml-cpu/simd-mappings.h @@ -26,6 +26,23 @@ extern "C" { // simd mappings // +// Force full unrolling of a loop with a compile-time constant trip count. +// +// The generic SIMD kernels below hold their vector operands and accumulators in +// small arrays indexed by the inner loop counter (`for (j = 0; j < GGML_F32_ARR; j++)`). +// Those arrays only stay in registers if the loop is unrolled: GCC does that at +// -O3, but not at -O2 (the level used by RelWithDebInfo builds), where the +// arrays are left in memory and every vector operation pays a load/store round +// trip to the stack. Unrolling explicitly makes the codegen independent of the +// optimization level. +#if (defined(__clang__) && __clang_major__ >= 9) || (!defined(__clang__) && defined(__GNUC__) && __GNUC__ >= 8) + #define GGML_DO_PRAGMA_(x) _Pragma(#x) + #define GGML_DO_PRAGMA(x) GGML_DO_PRAGMA_(x) + #define GGML_UNROLL(n) GGML_DO_PRAGMA(GCC unroll n) +#else + #define GGML_UNROLL(n) +#endif + // FP16 to FP32 conversion // 16-bit float diff --git a/ggml/src/ggml-cpu/vec.cpp b/ggml/src/ggml-cpu/vec.cpp index ff2b636df86c..8a49645bc223 100644 --- a/ggml/src/ggml-cpu/vec.cpp +++ b/ggml/src/ggml-cpu/vec.cpp @@ -109,6 +109,7 @@ void ggml_vec_dot_f32(int n, float * GGML_RESTRICT s, size_t bs, const float * G GGML_F32_VEC ay[GGML_F32_ARR]; for (int i = 0; i < np; i += GGML_F32_STEP) { + GGML_UNROLL(GGML_F32_ARR) for (int j = 0; j < GGML_F32_ARR; j++) { ax[j] = GGML_F32_VEC_LOAD(x + i + j*GGML_F32_EPR); ay[j] = GGML_F32_VEC_LOAD(y + i + j*GGML_F32_EPR); @@ -350,6 +351,7 @@ void ggml_vec_dot_f16(int n, float * GGML_RESTRICT s, size_t bs, ggml_fp16_t * G GGML_F16_VEC ay[GGML_F16_ARR]; for (int i = 0; i < np; i += GGML_F16_STEP) { + GGML_UNROLL(GGML_F16_ARR) for (int j = 0; j < GGML_F16_ARR; j++) { ax[j] = GGML_F16_VEC_LOAD(x + i + j*GGML_F16_EPR, j); ay[j] = GGML_F16_VEC_LOAD(y + i + j*GGML_F16_EPR, j); diff --git a/ggml/src/ggml-cpu/vec.h b/ggml/src/ggml-cpu/vec.h index 5de9cb5b7e09..824d522a701b 100644 --- a/ggml/src/ggml-cpu/vec.h +++ b/ggml/src/ggml-cpu/vec.h @@ -284,9 +284,11 @@ inline static void ggml_vec_dot_f16_unroll(const int n, const int xs, float * GG GGML_F16_VEC ay[GGML_F16_ARR]; for (int i = 0; i < np; i += GGML_F16_STEP) { + GGML_UNROLL(GGML_F16_ARR) for (int j = 0; j < GGML_F16_ARR; j++) { ay[j] = GGML_F16_VEC_LOAD(y + i + j*GGML_F16_EPR, j); + GGML_UNROLL(GGML_VEC_DOT_UNROLL) for (int k = 0; k < GGML_VEC_DOT_UNROLL; ++k) { ax[j] = GGML_F16_VEC_LOAD(x[k] + i + j*GGML_F16_EPR, j); @@ -414,6 +416,7 @@ inline static void ggml_vec_mad_f32(const int n, float * GGML_RESTRICT y, const GGML_F32_VEC ay[GGML_F32_ARR]; for (int i = 0; i < np; i += GGML_F32_STEP) { + GGML_UNROLL(GGML_F32_ARR) for (int j = 0; j < GGML_F32_ARR; j++) { ax[j] = GGML_F32_VEC_LOAD(x + i + j*GGML_F32_EPR); ay[j] = GGML_F32_VEC_LOAD(y + i + j*GGML_F32_EPR); @@ -562,6 +565,7 @@ inline static void ggml_vec_mad_f16(const int n, ggml_fp16_t * GGML_RESTRICT y, GGML_F16_VEC ay[GGML_F16_ARR]; for (int i = 0; i < np; i += GGML_F16_STEP) { + GGML_UNROLL(GGML_F16_ARR) for (int j = 0; j < GGML_F16_ARR; j++) { ax[j] = GGML_F16_VEC_LOAD(x + i + j*GGML_F16_EPR, j); ay[j] = GGML_F16_VEC_LOAD(y + i + j*GGML_F16_EPR, j); @@ -623,9 +627,11 @@ inline static void ggml_vec_mad_f32_unroll(const int n, const int xs, const int GGML_F32_VEC ay[GGML_F32_ARR]; for (int i = 0; i < np; i += GGML_F32_STEP) { + GGML_UNROLL(GGML_F32_ARR) for (int j = 0; j < GGML_F32_ARR; j++) { ay[j] = GGML_F32_VEC_LOAD(y + i + j*GGML_F32_EPR); + GGML_UNROLL(GGML_VEC_MAD_UNROLL) for (int k = 0; k < GGML_VEC_MAD_UNROLL; ++k) { ax[k][j] = GGML_F32_VEC_LOAD(x[k] + i + j*GGML_F32_EPR); ay[j] = GGML_F32_VEC_FMA(ay[j], ax[k][j], vx[k]); @@ -678,6 +684,7 @@ inline static void ggml_vec_mad1_f32(const int n, float * y, const float * x, co GGML_F32_VEC ay[GGML_F32_ARR]; for (int i = 0; i < np; i += GGML_F32_STEP) { + GGML_UNROLL(GGML_F32_ARR) for (int j = 0; j < GGML_F32_ARR; j++) { ay[j] = GGML_F32_VEC_LOAD(x + i + j*GGML_F32_EPR); ay[j] = GGML_F32_VEC_FMA(vb, ay[j], vs); @@ -745,6 +752,7 @@ inline static void ggml_vec_scale_f32(const int n, float * y, const float v) { GGML_F32_VEC ay[GGML_F32_ARR]; for (int i = 0; i < np; i += GGML_F32_STEP) { + GGML_UNROLL(GGML_F32_ARR) for (int j = 0; j < GGML_F32_ARR; j++) { ay[j] = GGML_F32_VEC_LOAD(y + i + j*GGML_F32_EPR); ay[j] = GGML_F32_VEC_MUL(ay[j], vx); @@ -838,6 +846,7 @@ inline static void ggml_vec_scale_f16(const int n, ggml_fp16_t * y, const float GGML_F16_VEC ay[GGML_F16_ARR]; for (int i = 0; i < np; i += GGML_F16_STEP) { + GGML_UNROLL(GGML_F16_ARR) for (int j = 0; j < GGML_F16_ARR; j++) { ay[j] = GGML_F16_VEC_LOAD(y + i + j*GGML_F16_EPR, j); ay[j] = GGML_F16_VEC_MUL(ay[j], vx);