From 5bf0bc3e328ed2a7dcc8a4d69916f05952398656 Mon Sep 17 00:00:00 2001
From: CodSpeed Bot <no-reply@codspeed.io>
Date: Wed, 29 Jul 2026 10:13:07 +0000
Subject: [PATCH] ggml-cpu: cache-aware tiling in ggml_compute_forward_mul_mat

Keep all of dim1 in a single chunk when the src0 rows already provide
enough chunks for every thread, so the weights are streamed once per
matmul instead of once per dim1 chunk.

Size the row block so the src0 rows of a block stay resident in L1
across the column loop instead of being re-read from L2 for every
column.

Both changes only affect tiling, the arithmetic is untouched and the
results are bit-identical.
---
 ggml/src/ggml-cpu/ggml-cpu.c | 23 ++++++++++++++++++++++-
 1 file changed, 22 insertions(+), 1 deletion(-)

diff --git a/ggml/src/ggml-cpu/ggml-cpu.c b/ggml/src/ggml-cpu/ggml-cpu.c
index 491316f74912..09506b4636b8 100644
--- a/ggml/src/ggml-cpu/ggml-cpu.c
+++ b/ggml/src/ggml-cpu/ggml-cpu.c
@@ -1199,7 +1199,20 @@ static void ggml_compute_forward_mul_mat_one_chunk(
     assert(ne13 % ne03 == 0);
 
     // block-tiling attempt
-    const int64_t blck_0 = 16;
+    //
+    // The src0 rows of a block are re-read for every column of the block, so they
+    // should stay in L1 across the column loop. With large src0 rows (F32/F16
+    // weights) a 16-row block is several times the size of L1 and every column
+    // re-reads all of its rows from L2, so shrink the block until its rows fit.
+    // When even a single row exceeds the budget there is nothing to reuse either
+    // way and the largest block is kept, keeping the loop overhead amortized.
+    const size_t blck_0_budget = 24*1024; // bytes of src0 rows to keep resident in L1
+    int64_t blck_0 = 16;
+    if (nb01 <= blck_0_budget) {
+        while (blck_0 > 1 && (size_t) blck_0*nb01 > blck_0_budget) {
+            blck_0 /= 2;
+        }
+    }
     const int64_t blck_1 = 16;
 
     const size_t src1_col_stride = src1_cont || src1->type != vec_dot_type ? row_size : nb11;
@@ -1407,6 +1420,14 @@ UseGgmlGemm2:;
     int64_t nchunk0 = (nr0 + chunk_size - 1) / chunk_size;
     int64_t nchunk1 = (nr1 + chunk_size - 1) / chunk_size;
 
+    // Every chunk streams all the src0 rows it covers, so splitting dim1 makes the
+    // src0 matrix (the weights, by far the largest operand) be read from memory once
+    // per dim1 chunk. When the src0 rows alone already provide enough chunks to keep
+    // every thread busy, keep all of dim1 in a single chunk and read src0 only once.
+    if (nchunk1 > 1 && nchunk0 >= nth*4) {
+        nchunk1 = 1;
+    }
+
     // If the chunking is poor for the number of threads on this setup, scrap the whole plan.  Re-chunk it by thread.
     //   Also, chunking by thread was measured to have perform better on NUMA systems.  See https://github.com/ggml-org/llama.cpp/pull/6915
     //   In theory, chunking should be just as useful on NUMA and non NUMA systems, but testing disagreed with that.