From 31265d3864966bed8b8b393450ebf8a1699ba7e2 Mon Sep 17 00:00:00 2001 From: Rehan Qasim Date: Thu, 30 Jul 2026 09:26:12 -0700 Subject: [PATCH 1/5] ggml-et: centralize scp_signal/scp_wait in platform.h Adds the L2-SCP hart-to-hart counter primitives (already used locally by the stock mul_mat_f16_matrix_engine.c and mul_mat_Q4_0_matrix_engine.c kernels) to the shared platform.h so new kernels can use them too, and switches the stock Q4_0 kernel to the shared copy instead of its own private one to avoid a duplicate-definition conflict. mul_mat_f16_matrix_engine.c is replaced outright by the next commit, so it needs no separate dedup here. --- .../src/mul_mat_Q4_0_matrix_engine.c | 19 --------------- ggml/src/ggml-et/et-kernels/src/platform.h | 24 +++++++++++++++++++ 2 files changed, 24 insertions(+), 19 deletions(-) diff --git a/ggml/src/ggml-et/et-kernels/src/mul_mat_Q4_0_matrix_engine.c b/ggml/src/ggml-et/et-kernels/src/mul_mat_Q4_0_matrix_engine.c index 28a10303235a..ac9640476a3b 100644 --- a/ggml/src/ggml-et/et-kernels/src/mul_mat_Q4_0_matrix_engine.c +++ b/ggml/src/ggml-et/et-kernels/src/mul_mat_Q4_0_matrix_engine.c @@ -33,25 +33,6 @@ #define SCP_CONSUMED_OFF (SCP_READY_OFF + 64) // 4160 #define SCP_PER_MINION (SCP_CONSUMED_OFF + 64) // 4224 -// Signal a counter value to the other hart via L2 SCP. -static inline void __attribute__((always_inline)) scp_signal(volatile uint32_t * flag, uint32_t value) { - *flag = value; - FENCE; - evict_to_l2((const void *) flag, 1, 64); - WAIT_CACHEOPS; -} - -// Wait for a counter in L2 SCP to reach the expected value. -static inline void __attribute__((always_inline)) scp_wait(volatile uint32_t * flag, uint32_t expected) { - while (1) { - evict_to_l2((const void *) flag, 1, 64); - WAIT_CACHEOPS; - if (*flag >= expected) { - return; - } - } -} - // Dequantize one 32-element Q4_0 block of TILE_M weight rows into the FP32 // panel, written directly in TenB [k][m] order: panel[k*TILE_M + m]. // Low nibble of byte i -> k = i diff --git a/ggml/src/ggml-et/et-kernels/src/platform.h b/ggml/src/ggml-et/et-kernels/src/platform.h index cbec4c98d741..8a936da4c577 100644 --- a/ggml/src/ggml-et/et-kernels/src/platform.h +++ b/ggml/src/ggml-et/et-kernels/src/platform.h @@ -542,4 +542,28 @@ static void evict_region_past_l2(const void * addr, size_t bytes) { } } + +//****************************************************************************** +// Counter signaling between harts via L2 scratchpad (SCP) +//****************************************************************************** + +// Signal a counter value to the other hart via L2 SCP. +static inline void __attribute__((always_inline)) +scp_signal(volatile uint32_t *flag, uint32_t value) { + *flag = value; + FENCE; + evict_to_l2((const void *)flag, 1, 64); + WAIT_CACHEOPS; +} + +// Wait for a counter in L2 SCP to reach the expected value. +static inline void __attribute__((always_inline)) +scp_wait(volatile uint32_t *flag, uint32_t expected) { + while (1) { + evict_to_l2((const void *)flag, 1, 64); + WAIT_CACHEOPS; + if (*flag >= expected) return; + } +} + #endif // PLATFORM_H From 2fdd04bdd9010b75f15b79962c718c5f939d246b Mon Sep 17 00:00:00 2001 From: Rehan Qasim Date: Tue, 14 Jul 2026 05:59:26 -0700 Subject: [PATCH 2/5] et-backend: F16 GEMV rewrite for register-resident vecdot Stripe output elements across every hart of all 32 shires instead of blocking work into 16-element chunks, which only filled 8 shires for a typical decode GEMV. Adds a register-resident f16 row-dot helper and stages the reused B activation vector into per-shire L2 SCP so it survives weight-matrix streaming. Co-authored-by: Rehan Qasim (cherry picked from commit 86bff65d7f0ed22191ac781a6005b2758ce8e082) --- ggml/src/ggml-et/et-kernels/src/block_ops.h | 105 +++++++++++ ggml/src/ggml-et/et-kernels/src/mul_mat_f16.c | 175 ++++++++++-------- 2 files changed, 201 insertions(+), 79 deletions(-) diff --git a/ggml/src/ggml-et/et-kernels/src/block_ops.h b/ggml/src/ggml-et/et-kernels/src/block_ops.h index 78ffbde87bfa..8b0d7f1cbeb1 100644 --- a/ggml/src/ggml-et/et-kernels/src/block_ops.h +++ b/ggml/src/ggml-et/et-kernels/src/block_ops.h @@ -670,6 +670,111 @@ static inline float compute_block_dot_product_f16(const uint16_t * a_block, cons return compute_block_dot_product_f16_partial(a_block, b_col_start, QK_F16); } +//****************************************************************************** +// Hoisted F16 row-dot API (register-resident accumulator) +// +// Mirrors the q8_dot_* API above: the running sum lives in vector registers +// for the whole row instead of being spilled to memory every 8 elements, so +// the inner loop is load + convert + fmadd with no accumulator round-trip. +// +// Register contract (shared convention with q8_dot_*): +// f20-f23 row accumulators (persistent across tiles, reset per row) +// f31 gather pattern (byte offsets of 8 consecutive f16) +// f11-f18 scratch within tile +// t0, f1-f5 scratch within reduce +// Caller sets the vector mask to 0xFF once around the surrounding loop. +//****************************************************************************** + +static inline void __attribute__((always_inline)) +f16_dot_reset(void) { + // Four independent lane accumulators break the fmadd dependency chain so + // multiple A/B loads stay in flight, hiding load latency. + __asm__ volatile( + "fbci.pi f20, 0\n" + "fbci.pi f21, 0\n" + "fbci.pi f22, 0\n" + "fbci.pi f23, 0\n" + ::: "f20", "f21", "f22", "f23"); +} + +// Accumulate n_blocks blocks of QK_F16 (=32) f16 A values times f32 B into +// f20..f23 (one accumulator per 8-lane chunk within the block). +static inline void __attribute__((always_inline)) +f16_dot_tile(const uint16_t * a_row, const float * b_col, int64_t n_blocks) { + static const int32_t gather_pattern[8] = {0, 2, 4, 6, 8, 10, 12, 14}; + __asm__ volatile("flw.ps f31, %[g]\n" + : : [g] "m"(*(const int32_t(*)[8])gather_pattern) + : "f31"); + + // fgh.ps/flw.ps are blocking loads, so a non-blocking prefetch hint issued + // a few blocks ahead turns a DRAM miss into an L1 hit by the time the + // gather runs. + const int64_t PF = 4; + for (int64_t kb = 0; kb < n_blocks; kb++) { + const uint16_t * a_ptr = a_row + (kb << 5); // 32 f16 per block + const float * b_ptr = b_col + (kb << 5); + if (kb + PF < n_blocks) { + const uint16_t * pfa = a_ptr + (PF << 5); // A block = 64B = 1 line + const float * pfb = b_ptr + (PF << 5); // B block = 128B = 2 lines + __asm__ volatile( + "lb x0, 0(%[pa])\n" + "lb x0, 0(%[pb])\n" + "lb x0, 64(%[pb])\n" + : : [pa] "r"(pfa), [pb] "r"(pfb)); + } + __asm__ volatile( + // Issue all 8 independent memory ops first (4 A gathers + 4 B loads) + // so several misses are outstanding before any consumer stalls. + "fgh.ps f11, f31(%[a0])\n" + "fgh.ps f13, f31(%[a1])\n" + "fgh.ps f15, f31(%[a2])\n" + "fgh.ps f17, f31(%[a3])\n" + "flw.ps f12, %[b0]\n" + "flw.ps f14, %[b1]\n" + "flw.ps f16, %[b2]\n" + "flw.ps f18, %[b3]\n" + "fcvt.ps.f16 f11, f11\n" + "fcvt.ps.f16 f13, f13\n" + "fcvt.ps.f16 f15, f15\n" + "fcvt.ps.f16 f17, f17\n" + "fmadd.ps f20, f11, f12, f20\n" + "fmadd.ps f21, f13, f14, f21\n" + "fmadd.ps f22, f15, f16, f22\n" + "fmadd.ps f23, f17, f18, f23\n" + : + : [a0] "r"(a_ptr), [a1] "r"(a_ptr + 8), + [a2] "r"(a_ptr + 16), [a3] "r"(a_ptr + 24), + [b0] "m"(*(const float(*)[8])&b_ptr[0]), + [b1] "m"(*(const float(*)[8])&b_ptr[8]), + [b2] "m"(*(const float(*)[8])&b_ptr[16]), + [b3] "m"(*(const float(*)[8])&b_ptr[24]) + : "f11", "f12", "f13", "f14", "f15", "f16", "f17", "f18", + "f20", "f21", "f22", "f23" + ); + } +} + +static inline float __attribute__((always_inline)) +f16_dot_reduce(void) { + float result; + __asm__ __volatile__ ( + // Combine the 4 lane accumulators, then horizontal-sum the 8 lanes. + "fadd.ps f20, f20, f21, rne \n\t" + "fadd.ps f22, f22, f23, rne \n\t" + "fadd.ps f20, f20, f22, rne \n\t" + "fswizz.ps f1, f20, 0xB1 \n\t" + "fadd.ps f2, f20, f1, rne \n\t" + "fswizz.ps f3, f2, 0x4E \n\t" + "fadd.ps f4, f2, f3, rne \n\t" + "fmvz.x.ps t0, f4, 4 \n\t" + "fbcx.ps f5, t0 \n\t" + "fadd.ps %[vout], f4, f5, rne \n\t" + : [vout] "=f" (result) + :: "t0", "f1", "f2", "f3", "f4", "f5", "f20", "f21", "f22", "f23" + ); + return result; +} + // Compute dot product between f32 block and f32 column vector // Vectorized: processes 8 elements at a time using ET vector instructions // Block size: up to 16 f32 values (can handle partial blocks for misaligned K) diff --git a/ggml/src/ggml-et/et-kernels/src/mul_mat_f16.c b/ggml/src/ggml-et/et-kernels/src/mul_mat_f16.c index 3f1fcd5f261f..1e8e452c1a36 100644 --- a/ggml/src/ggml-et/et-kernels/src/mul_mat_f16.c +++ b/ggml/src/ggml-et/et-kernels/src/mul_mat_f16.c @@ -1,6 +1,13 @@ //****************************************************************************** -// MUL_MAT Kernel +// MUL_MAT Kernel (F16 weights) // Matrix multiplication: C[M,N] = A[M,K] * B[K,N] +// +// Decode-optimized distribution: one output element (row m of A dotted with +// column n of B) is the unit of work, striped by thread_id across every hart +// of every active shire. This keeps all 32 shires busy for a GEMV +// (M=4096, N=1) instead of leaving most idle, and the register-resident +// f16_dot row helper keeps the accumulator in registers for the whole K +// dimension. //****************************************************************************** #include "block_ops.h" @@ -8,6 +15,7 @@ #include "math_fp.h" #include "platform.h" #include "quants.h" +#include "tensor.h" #include @@ -18,18 +26,13 @@ int entry_point(struct ggml_et_binary_params * params, void * env) { return -1; } - // Thread coordination + // Thread coordination, use every hart of every active shire. int thread_id = get_relative_thread_id(kernel_env->shire_mask); int num_threads = get_num_threads(kernel_env->shire_mask); - - if (thread_id < 0 || (thread_id & 1)) { - return 0; // Skip odd threads to avoid resource contention + if (thread_id < 0) { + return 0; } - int effective_thread_id = thread_id / 2; - int effective_num_threads = (num_threads + 1) / 2; - - // Extract tensor references struct ggml_tensor * src0 = ¶ms->src0; // Weight matrix A (F16) struct ggml_tensor * src1 = ¶ms->src1; // Activation matrix B (F16/F32) struct ggml_tensor * dst = ¶ms->dst; // Output matrix C (F32) @@ -43,7 +46,7 @@ int entry_point(struct ggml_et_binary_params * params, void * env) { const uint16_t * src0_data = (const uint16_t *) src0->data; float * dst_data = (float *) dst->data; - // Dimensions and Strides + // Dimensions and strides const int64_t K = src0->ne[0]; const int64_t M = src0->ne[1]; const int64_t N = src1->ne[1]; @@ -56,87 +59,101 @@ int entry_point(struct ggml_et_binary_params * params, void * env) { const size_t nb11 = src1->nb[1], nb12 = src1->nb[2], nb13 = src1->nb[3]; const size_t nb1 = dst->nb[1], nb2 = dst->nb[2], nb3 = dst->nb[3]; - // F16 specific block size (Usually QK_F16) - const int block_size = QK_F16; + const int block_size = QK_F16; // 32 const int64_t K_blocks = K / block_size; const int64_t K_remainder = K % block_size; - // Threading distribution - const uint64_t total_elements = M * N * ne2 * ne3; - const uint64_t per_thread = 16; - const uint64_t threads_stride = per_thread * effective_num_threads; - - if (effective_thread_id * per_thread >= total_elements) { - return 0; - } - // Broadcasting support const int64_t r2 = ne12 / ne02; const int64_t r3 = ne13 / ne03; - for (uint64_t base_idx = effective_thread_id * per_thread; base_idx < total_elements; base_idx += threads_stride) { - for (uint64_t j = 0; j < per_thread; j++) { - const uint64_t idx = base_idx + j; - if (idx >= total_elements) { - break; + const int is_f32_b = (src1->type == GGML_TYPE_F32); + const uint64_t total_elements = (uint64_t) M * N * ne2 * ne3; + + // Stage the reused B activation vector into per-shire L2 SCP. Streaming the + // weight matrix thrashes L2 and evicts B, so B is re-read from DRAM + // repeatedly during the decode GEMV. Stage it once per shire into L2 SCP (a + // separate SRAM partition cache streaming cannot evict) via + // et_tensor_load_l2scp; one hart per shire issues the DMA loop, then a + // shire barrier lets all harts read B on-chip. Only applies to the common + // decode case: F32 B, a single contiguous non-broadcast B that fits the SCP + // budget. + const int b_contig = (nb11 == (size_t) K * sizeof(float)); + const uint64_t b_lines = ((uint64_t) N * K * sizeof(float) + 63) / 64; + const int stage_b = is_f32_b && b_contig && + ne12 == 1 && ne13 == 1 && ne02 == 1 && ne03 == 1 && + b_lines <= 8192; // <= 512 KB, within the SCP budget + const float * b_scp = (const float *) et_shire_l2scp_local(0); + + if (stage_b) { + if ((get_hart_id() & 63) == 0) { + et_tensor_load_l2scp_conf_t conf; + conf.use_tmask = false; + conf.stride = 64; // advance one 64B cache line per line loaded + uint64_t remaining = b_lines; + uint64_t dst = 0; + uint64_t addr = (uint64_t) src1->data; + while (remaining > 0) { + uint64_t cl = (remaining >= 16) ? 16 : remaining; + conf.dst_start = dst; + conf.addr = addr; + conf.num_lines = cl - 1; // 4-bit field encodes (lines - 1) + conf.id = 0; + et_tensor_load_l2scp(&conf); + WAIT_TENSOR_LOAD_L2_0; + dst += cl; + addr += cl * 64; + remaining -= cl; } + } + et_barrier(ET_BARRIER_SHIRE); // B is now resident in L2 SCP for all harts + } - // Index decoding - const int64_t i3 = idx / (M * N * ne2); - const int64_t rem3 = idx % (M * N * ne2); - const int64_t i2 = rem3 / (M * N); - const int64_t rem2 = rem3 % (M * N); - const int64_t n = rem2 / M; - const int64_t m = rem2 % M; - - const int64_t i03 = i3 / r3, i02 = i2 / r2; - const int64_t i13 = (ne13 > 1) ? i3 : 0, i12 = (ne12 > 1) ? i2 : 0; - - float sum = 0.0f; - const uint16_t * f16_row = - (const uint16_t *) ((const char *) src0_data + m * nb01 + i02 * nb02 + i03 * nb03); - - if (src1->type == GGML_TYPE_F32) { - const float * src1_data = (const float *) src1->data; - - for (int64_t kb = 0; kb < K_blocks; kb++) { - const float * b_col_ptr = - (const float *) ((const char *) src1_data + (kb * block_size) * sizeof(float) + n * nb11 + - i12 * nb12 + i13 * nb13); - sum += compute_block_dot_product_f16_naive(&f16_row[kb * block_size], b_col_ptr); - } - - if (K_remainder > 0) { - const int64_t offset = K_blocks * block_size; - const float * b_col_ptr = (const float *) ((const char *) src1_data + offset * sizeof(float) + - n * nb11 + i12 * nb12 + i13 * nb13); - sum += compute_block_dot_product_f16_partial(&f16_row[offset], b_col_ptr, K_remainder); - } - } else { - const uint16_t * src1_data = (const uint16_t *) src1->data; - - for (int64_t kb = 0; kb < K_blocks; kb++) { - const uint16_t * b_col_ptr = - (const uint16_t *) ((const char *) src1_data + (kb * block_size) * sizeof(uint16_t) + n * nb11 + - i12 * nb12 + i13 * nb13); - sum += compute_block_dot_product_f16_f16_partial(&f16_row[kb * block_size], b_col_ptr, block_size); - } - - if (K_remainder > 0) { - const int64_t offset = K_blocks * block_size; - const uint16_t * b_col_ptr = - (const uint16_t *) ((const char *) src1_data + offset * sizeof(uint16_t) + n * nb11 + - i12 * nb12 + i13 * nb13); - sum += compute_block_dot_product_f16_f16_partial(&f16_row[offset], b_col_ptr, K_remainder); - } + // Set the vector mask (all 8 lanes) once for the whole row loop. + unsigned long saved_mask; + __asm__ volatile("mova.x.m %0" : "=r"(saved_mask)); + __asm__ volatile("mov.m.x m0, x0, 0xFF"); + + for (uint64_t idx = (uint64_t) thread_id; idx < total_elements; idx += (uint64_t) num_threads) { + // Index decoding + const int64_t i3 = idx / (M * N * ne2); + const int64_t rem3 = idx % (M * N * ne2); + const int64_t i2 = rem3 / (M * N); + const int64_t rem2 = rem3 % (M * N); + const int64_t n = rem2 / M; + const int64_t m = rem2 % M; + + const int64_t i03 = i3 / r3, i02 = i2 / r2; + const int64_t i13 = (ne13 > 1) ? i3 : 0, i12 = (ne12 > 1) ? i2 : 0; + + const uint16_t * f16_row = + (const uint16_t *) ((const char *) src0_data + m * nb01 + i02 * nb02 + i03 * nb03); + + float sum; + if (is_f32_b) { + const float * b_col = stage_b + ? (b_scp + n * K) + : (const float *) ((const char *) src1->data + n * nb11 + i12 * nb12 + i13 * nb13); + + f16_dot_reset(); + f16_dot_tile(f16_row, b_col, K_blocks); + sum = f16_dot_reduce(); + + if (K_remainder > 0) { + const int64_t offset = K_blocks * block_size; + sum += compute_block_dot_product_f16_partial(&f16_row[offset], &b_col[offset], K_remainder); } - - // Atomic store for output - volatile float * c_element = - (volatile float *) ((char *) dst_data + m * dst->nb[0] + n * nb1 + i2 * nb2 + i3 * nb3); - atomic_store_f32(c_element, sum); + } else { + const uint16_t * b_col = + (const uint16_t *) ((const char *) src1->data + n * nb11 + i12 * nb12 + i13 * nb13); + sum = compute_block_dot_product_f16_f16_partial(f16_row, b_col, K); } + + volatile float * c_element = + (volatile float *) ((char *) dst_data + m * dst->nb[0] + n * nb1 + i2 * nb2 + i3 * nb3); + atomic_store_f32(c_element, sum); } + __asm__ volatile("mova.m.x %0" ::"r"(saved_mask)); return 0; } From f3690dbab8b1b41f6f572cb5301ede278ae72f34 Mon Sep 17 00:00:00 2001 From: Rehan Qasim Date: Thu, 23 Jul 2026 04:38:43 -0700 Subject: [PATCH 3/5] et-backend: F16 matrix-engine GEMM kernel for prefill Adds a double-buffered weight-reuse F16 matrix-engine kernel (L1 activation double-buffering, software L2 prefetch) achieving 5.25 TFLOPS, and wires MUL_MAT dispatch so N <= 2 uses the vecdot GEMV kernel and N > 2 uses this matrix-engine GEMM kernel. Also fixes the dispatch check, which compared src1->ne[0] (K) instead of src1->ne[1] (N) and so never actually distinguished decode from prefill for F16 activations. Co-authored-by: Rehan Qasim (cherry picked from commit 780c64eefd80492d7e2c4bb36c66015a12a66087) --- .../src/mul_mat_f16_matrix_engine.c | 563 +++++++++++++----- ggml/src/ggml-et/et-kernels/src/platform.h | 14 + ggml/src/ggml-et/ggml-et-ops.cpp | 4 +- 3 files changed, 422 insertions(+), 159 deletions(-) diff --git a/ggml/src/ggml-et/et-kernels/src/mul_mat_f16_matrix_engine.c b/ggml/src/ggml-et/et-kernels/src/mul_mat_f16_matrix_engine.c index 2aab87ad5e52..227ccfcdfcb8 100644 --- a/ggml/src/ggml-et/et-kernels/src/mul_mat_f16_matrix_engine.c +++ b/ggml/src/ggml-et/et-kernels/src/mul_mat_f16_matrix_engine.c @@ -1,161 +1,186 @@ +#include +#include #include "ggml_tensor.h" #include "platform.h" #include "tensor.h" -#include -#include +/* + * High-performance F16 Matrix Multiply for ET-SoC-1 — TensorFMA16. + * Double-buffered producer/consumer implementation. + * + * Hart 1 (producer): Interleaves and transposes weights into double-buffered L2 SCP. + * Hart 0 (consumer): Tensor engine compute (FMA, reduce, store). + * + * Two execution paths: + * * REUSE path (n_tiles >= 2): prepared weight tiles are reused across + * ru_n N-tiles to minimize DRAM reads. + * * ORIGINAL path (n_tiles == 1, GEMV): one output tile at a time, no reuse. + */ -// FP16 x FP16 -> FP32 MUL_MAT with hart 1 B-panel packing -// -// Hart 0: tensor engine (load A, load B from SCP, FMA, reduce, store) -// Hart 1: pack B into double-buffered L2 SCP panels, flush for tensor_load -// -// Sync: monotonic counters in L2 SCP with evict-based coherency. -// Double-buffered bpanel allows pack/FMA overlap. -// #define NUM_COMPUTE_SHIRES 32 #define MINIONS_PER_SHIRE 32 -#define TILE_M 16 -#define TILE_N 16 -#define TILE_K 32 +#define TILE_M 16 +#define TILE_N 16 +#define TILE_K 32 +#define PANEL_LINES 16 + +#ifndef REUSE_MAX +#define REUSE_MAX 32 +#endif +#ifndef KWIN +#define KWIN 16 // K-blocks per window +#endif + +#define MACHINE_SLOTS (NUM_COMPUTE_SHIRES * MINIONS_PER_SHIRE) // 1024 #define CACHEOP_MAX 0 #define REP_RATE 0 -#define A_L1_START 0 // SCP lines 0..15 for A -#define B_L1_START 16 // SCP lines 16..31 for B +#define A_L1_START 0 // L1 SCP lines 0..15 for A (buffer 0) +#define B_L1_START 16 // L1 SCP lines 16..31 for B (single buffer) +#define A_L1_ALT 32 // L1 SCP lines 32..47 for A (buffer 1) typedef uint16_t et_fp16_t; -// L2 SCP layout per minion (double-buffered bpanel + sync counters) -// [0..1023] bpanel buffer 0 (16 lines x 64 bytes) -// [1024..2047] bpanel buffer 1 -// [2048..2111] ready counter (hart1 -> hart0, own cache line) -// [2112..2175] consumed counter (hart0 -> hart1, own cache line) -#define SCP_BPANEL_SIZE (16 * 32 * sizeof(et_fp16_t)) // 1024 bytes -#define SCP_READY_OFF (2 * SCP_BPANEL_SIZE) // 2048 -#define SCP_CONSUMED_OFF (SCP_READY_OFF + 64) // 2112 -#define SCP_PER_MINION (SCP_CONSUMED_OFF + 64) // 2176 - -// Signal a counter value to the other hart via L2 SCP. -static inline void __attribute__((always_inline)) scp_signal(volatile uint32_t * flag, uint32_t value) { - *flag = value; - FENCE; - evict_to_l2((const void *) flag, 1, 64); - WAIT_CACHEOPS; -} +#define SCP_PANEL_SIZE (PANEL_LINES * TILE_K * (uint64_t)sizeof(et_fp16_t)) // 1024 bytes -// Wait for a counter in L2 SCP to reach the expected value. -static inline void __attribute__((always_inline)) scp_wait(volatile uint32_t * flag, uint32_t expected) { - while (1) { - evict_to_l2((const void *) flag, 1, 64); - WAIT_CACHEOPS; - if (*flag >= expected) { - return; - } - } -} +#define RU_BUF_BYTES (KWIN * SCP_PANEL_SIZE) +#define RU_CACHE_BYTES (2 * RU_BUF_BYTES) +#define RU_CSCRATCH_BYTES (REUSE_MAX * 16 * 64ULL) +#define SCP_READY_OFF (RU_CACHE_BYTES + RU_CSCRATCH_BYTES) +#define SCP_CONSUMED_OFF (SCP_READY_OFF + 64) +#define SCP_PER_MINION (SCP_CONSUMED_OFF + 64) -/** - * Build the interleaved B panel that TensorFMA16A32 expects (vectorized). - * - * Output: 16 lines x 32 fp16 = 1024 bytes, 64-byte aligned. - * out[l][j*2+0] = src0[mb + j][kb + 2*l] - * out[l][j*2+1] = src0[mb + j][kb + 2*l + 1] - * - * Uses fsch.ps scatter store: load 8 pairs per row, scatter to 8 output lines. - */ -static inline void __attribute__((always_inline)) pack_b_interleaved(et_fp16_t * out, - const char * src0_batch, - int64_t mb, - int64_t kb, - int64_t nb1_0) { - static const int32_t __attribute__((aligned(32))) scatter_idx[8] = { 0, 64, 128, 192, 256, 320, 384, 448 }; +static inline void __attribute__((always_inline)) +pack_b_interleaved(et_fp16_t *out, const char *src0_batch, + int64_t mb, int64_t kb, int64_t nb1_0) { + static const int32_t __attribute__((aligned(32))) scatter_idx[8] = { + 0, 64, 128, 192, 256, 320, 384, 448 + }; unsigned long old_mask; __asm__ volatile( "mova.x.m %[ms] \n\t" - "mov.m.x m0, x0, 0xFF \n\t" - "flw.ps f1, 0(%[idx]) \n\t" + "mov.m.x m0, x0, 0xFF \n\t" // all 8 lanes active + "flw.ps f1, (%[idx]) \n\t" // f1 = scatter offsets : [ms] "=&r"(old_mask) : [idx] "r"(scatter_idx) - : "f1"); + : "f1" + ); + char *pbase = (char *) out; for (int j = 0; j < TILE_M; ++j) { - const et_fp16_t * row = (const et_fp16_t *) (src0_batch + (mb + j) * nb1_0) + kb; - char * dst = (char *) out + j * 4; + const et_fp16_t *row = (const et_fp16_t *)(src0_batch + (mb + j) * nb1_0) + kb; + char *col = pbase + j * 4; // column m=j of the panel __asm__ volatile( - "flw.ps f2, 0(%[src]) \n\t" - "flw.ps f3, 32(%[src]) \n\t" - "fscw.ps f2, f1(%[d0]) \n\t" - "fscw.ps f3, f1(%[d1]) \n\t" + "flw.ps f2, 0(%[src]) \n\t" // load row[0..15] (32 bytes) + "flw.ps f3, 32(%[src]) \n\t" // load row[16..31] (32 bytes) + "fscw.ps f2, f1(%[c0]) \n\t" // store row[0..15] to lines 0..7, col j + "fscw.ps f3, f1(%[c8]) \n\t" // store row[16..31] to lines 8..15, col j : - : [src] "r"(row), [d0] "r"(dst), [d1] "r"(dst + 512) - : "f2", "f3", "memory"); + : [src] "r"(row), [c0] "r"(col), [c8] "r"(col + 8 * 64) + : "f2", "f3", "memory" + ); } - __asm__ volatile("mova.m.x %[ms] \n\t" : : [ms] "r"(old_mask)); + __asm__ volatile("mova.m.x %0" :: "r"(old_mask)); +} + +#define C_ROW_PAIR_ST(n0, n1, base) \ + __asm__ volatile("fsw.ps f" #n0 ", (%0)\n\t fsw.ps f" #n1 ", (%1)\n\t" \ + :: "r"((base)), "r"((base) + 32) : "memory") +#define C_ROW_PAIR_LD(n0, n1, base) \ + __asm__ volatile("flw.ps f" #n0 ", (%0)\n\t flw.ps f" #n1 ", (%1)\n\t" \ + :: "r"((base)), "r"((base) + 32) : "f" #n0, "f" #n1) + +static inline void __attribute__((always_inline)) +c_spill(char *s) { + C_ROW_PAIR_ST(0, 1, s + 0 * 64); C_ROW_PAIR_ST(2, 3, s + 1 * 64); + C_ROW_PAIR_ST(4, 5, s + 2 * 64); C_ROW_PAIR_ST(6, 7, s + 3 * 64); + C_ROW_PAIR_ST(8, 9, s + 4 * 64); C_ROW_PAIR_ST(10, 11, s + 5 * 64); + C_ROW_PAIR_ST(12, 13, s + 6 * 64); C_ROW_PAIR_ST(14, 15, s + 7 * 64); + C_ROW_PAIR_ST(16, 17, s + 8 * 64); C_ROW_PAIR_ST(18, 19, s + 9 * 64); + C_ROW_PAIR_ST(20, 21, s + 10 * 64); C_ROW_PAIR_ST(22, 23, s + 11 * 64); + C_ROW_PAIR_ST(24, 25, s + 12 * 64); C_ROW_PAIR_ST(26, 27, s + 13 * 64); + C_ROW_PAIR_ST(28, 29, s + 14 * 64); C_ROW_PAIR_ST(30, 31, s + 15 * 64); +} + +static inline void __attribute__((always_inline)) +c_seed(char *s) { + C_ROW_PAIR_LD(0, 1, s + 0 * 64); C_ROW_PAIR_LD(2, 3, s + 1 * 64); + C_ROW_PAIR_LD(4, 5, s + 2 * 64); C_ROW_PAIR_LD(6, 7, s + 3 * 64); + C_ROW_PAIR_LD(8, 9, s + 4 * 64); C_ROW_PAIR_LD(10, 11, s + 5 * 64); + C_ROW_PAIR_LD(12, 13, s + 6 * 64); C_ROW_PAIR_LD(14, 15, s + 7 * 64); + C_ROW_PAIR_LD(16, 17, s + 8 * 64); C_ROW_PAIR_LD(18, 19, s + 9 * 64); + C_ROW_PAIR_LD(20, 21, s + 10 * 64); C_ROW_PAIR_LD(22, 23, s + 11 * 64); + C_ROW_PAIR_LD(24, 25, s + 12 * 64); C_ROW_PAIR_LD(26, 27, s + 13 * 64); + C_ROW_PAIR_LD(28, 29, s + 14 * 64); C_ROW_PAIR_LD(30, 31, s + 15 * 64); } int entry_point(struct ggml_et_binary_params * params, void * env) { (void) env; - uint64_t hart_id = get_hart_id(); + uint64_t hart_id = get_hart_id(); uint64_t shire_id = get_shire_id(); - if (shire_id >= NUM_COMPUTE_SHIRES) { - return 0; - } + if (shire_id >= NUM_COMPUTE_SHIRES) return 0; - const int is_hart1 = hart_id & 1; - uint64_t local_minion = (hart_id >> 1) & 0x1F; + const int is_hart1 = hart_id & 1; + uint64_t local_minion = (hart_id >> 1) & 0x1F; - // Dimensions (both harts need these for tile assignment) const int64_t K = params->src0.ne[0]; const int64_t M = params->src0.ne[1]; const int64_t N = params->src1.ne[1]; + if ((M % TILE_M) != 0) return 0; + if ((K % TILE_K) != 0) return 0; + const int64_t ne2_0 = params->src0.ne[2], ne3_0 = params->src0.ne[3]; const int64_t ne2_1 = params->src1.ne[2], ne3_1 = params->src1.ne[3]; const int64_t nb1_0 = params->src0.nb[1]; const int64_t nb2_0 = params->src0.nb[2], nb3_0 = params->src0.nb[3]; - const int64_t nb1_1 = params->src1.nb[1]; const int64_t nb2_1 = params->src1.nb[2], nb3_1 = params->src1.nb[3]; - const int64_t nb1_d = params->dst.nb[1]; - const int64_t nb2_d = params->dst.nb[2], nb3_d = params->dst.nb[3]; - - const char * src0_base = (const char *) params->src0.data; - const char * src1_base = (const char *) params->src1.data; - char * dst_base = (char *) params->dst.data; + const int64_t nb2_d = params->dst.nb[2], nb3_d = params->dst.nb[3]; - if ((M % TILE_M) != 0) { - return 0; - } - if ((K % TILE_K) != 0) { - return 0; - } + const char* src0_base = (const char*)params->src0.data; + const char* src1_base = (const char*)params->src1.data; + char* dst_base = (char*)params->dst.data; - const int64_t m_tiles = M / TILE_M; - const int64_t n_tiles = (N + TILE_N - 1) / TILE_N; + const int64_t m_tiles = M / TILE_M; + const int64_t n_tiles = (N + TILE_N - 1) / TILE_N; const int64_t batch_count = ne2_1 * ne3_1; - const int64_t base_tiles = m_tiles * n_tiles * batch_count; const int64_t r2 = ne2_1 / ne2_0; const int64_t r3 = ne3_1 / ne3_0; - const int64_t total_harts = NUM_COMPUTE_SHIRES * MINIONS_PER_SHIRE; - const int64_t k_steps = K / TILE_K; + const int64_t k_steps = K / TILE_K; + + // L2 SCP pointers for this minion + const uint64_t scp_base = local_minion * SCP_PER_MINION; + volatile uint32_t *ready_ctr = + (volatile uint32_t *) et_shire_l2scp_local(scp_base + SCP_READY_OFF); + volatile uint32_t *consumed_ctr = + (volatile uint32_t *) et_shire_l2scp_local(scp_base + SCP_CONSUMED_OFF); + + int64_t n_groups = (n_tiles + REUSE_MAX - 1) / REUSE_MAX; + if (n_groups < 1) n_groups = 1; + int64_t ru_n = (n_tiles + n_groups - 1) / n_groups; + if (ru_n < 1) ru_n = 1; + + const int64_t units_pb = m_tiles * n_groups; + const int64_t base_units = units_pb * batch_count; int64_t k_splits = 1; - if (base_tiles < total_harts) { - k_splits = (total_harts + base_tiles - 1) / base_tiles; + { int64_t ks = 1; - while (ks * 2 <= k_splits && ks * 2 <= 32 && k_steps % (ks * 2) == 0) { + while (ks * 2 <= MINIONS_PER_SHIRE && + base_units * ks * 2 <= MACHINE_SLOTS && + (k_steps % (ks * 2)) == 0) { ks *= 2; } k_splits = ks; @@ -165,139 +190,362 @@ int entry_point(struct ggml_et_binary_params * params, void * env) { const int64_t k_split = local_minion % k_splits; const int64_t local_tile_idx = local_minion / k_splits; const int64_t tiles_stride = (int64_t) NUM_COMPUTE_SHIRES * tiles_per_shire; + const int64_t my_start = (int64_t) shire_id + local_tile_idx * NUM_COMPUTE_SHIRES; const int64_t k_steps_per_split = k_steps / k_splits; - const int64_t k_start = k_split * k_steps_per_split * TILE_K; + const int64_t k_start_block = k_split * k_steps_per_split; + const int64_t k_start = k_start_block * TILE_K; const int64_t k_end = k_start + k_steps_per_split * TILE_K; - // L2 SCP pointers for this minion's double-buffered panels + sync - uint64_t scp_base = local_minion * SCP_PER_MINION; - et_fp16_t * scp_bp[2] = { + const int reuse_ok = (ru_n >= 2); + + // ===================================================================== + // REUSE path: transpose each K-window once, reuse across ru_n N-tiles. + // ===================================================================== + if (reuse_ok) { + char *cache_buf[2] = { + (char *) et_shire_l2scp_local(scp_base), + (char *) et_shire_l2scp_local(scp_base + RU_BUF_BYTES), + }; + char *cscratch = (char *) et_shire_l2scp_local(scp_base + RU_CACHE_BYTES); + + const int64_t k_end_block = k_start_block + k_steps_per_split; + const int64_t n_windows = (k_steps_per_split + KWIN - 1) / KWIN; + + // ----- Hart 1: producer ----- + if (is_hart1) { + scp_signal(ready_ctr, 0); + scp_signal(consumed_ctr, 0); + + et_barrier(ET_BARRIER_MINION); + uint32_t wid = 0; + + for (int64_t unit = my_start; unit < base_units; unit += tiles_stride) { + const int64_t batch_idx = unit / units_pb; + const int64_t unit_in_b = unit % units_pb; + const int64_t mb_idx = unit_in_b % m_tiles; + + const int64_t i3 = batch_idx / ne2_1; + const int64_t i2 = batch_idx % ne2_1; + const int64_t i2_0 = i2 / r2; + const int64_t i3_0 = i3 / r3; + + const char *src0_batch = src0_base + i3_0 * nb3_0 + i2_0 * nb2_0; + const int64_t mb = mb_idx * TILE_M; + + // Prefetch first window's weights + { + const int64_t kbn_first = (k_start_block + KWIN <= k_end_block) ? KWIN : (k_end_block - k_start_block); + for (int64_t i = 0; i < kbn_first; ++i) { + l2_prefetch(src0_batch + mb * nb1_0 + (k_start_block + i) * TILE_K * sizeof(et_fp16_t), 16, nb1_0); + } + } + + for (int64_t kw = 0; kw < n_windows; ++kw) { + const int buf = wid & 1; + if (wid >= 2) scp_wait(consumed_ctr, wid - 1); + + const int64_t kb0 = k_start_block + kw * KWIN; + const int64_t kbn = (kb0 + KWIN <= k_end_block) ? KWIN : (k_end_block - kb0); + + // Prefetch next window's weights + if (kw + 1 < n_windows) { + const int64_t kb_next = kb0 + KWIN; + const int64_t kbn_next = (kb_next + KWIN <= k_end_block) ? KWIN : (k_end_block - kb_next); + for (int64_t i = 0; i < kbn_next; ++i) { + l2_prefetch(src0_batch + mb * nb1_0 + (kb_next + i) * TILE_K * sizeof(et_fp16_t), 16, nb1_0); + } + } + + et_fp16_t *cf = (et_fp16_t *) cache_buf[buf]; + for (int64_t i = 0; i < kbn; ++i) { + pack_b_interleaved(cf + i * (SCP_PANEL_SIZE / sizeof(et_fp16_t)), + src0_batch, mb, (kb0 + i) * TILE_K, nb1_0); + } + FENCE; + flush_to_l2_multi(cache_buf[buf], kbn * PANEL_LINES, 64); + WAIT_CACHEOPS; + + wid++; + scp_signal(ready_ctr, wid); + } + } + FENCE; + return 0; + } + + // ----- Hart 0: consumer ----- + setup_cache_scp(); +#if CACHEOP_MAX > 0 || REP_RATE > 0 + ucache_control(1, REP_RATE, CACHEOP_MAX); +#endif + CLEAR_TENSOR_ERROR; + + et_barrier(ET_BARRIER_MINION); + evict_to_l2((const void *)(uintptr_t) ready_ctr, 1, 64); WAIT_CACHEOPS; + evict_to_l2((const void *)(uintptr_t) consumed_ctr, 1, 64); WAIT_CACHEOPS; + + const uint64_t group_base_global = get_minion_id() - (uint64_t) k_split; + + uint32_t wid = 0; + for (int64_t unit = my_start; unit < base_units; unit += tiles_stride) { + const int64_t batch_idx = unit / units_pb; + const int64_t unit_in_b = unit % units_pb; + const int64_t g_idx = unit_in_b / m_tiles; + const int64_t mb_idx = unit_in_b % m_tiles; + + const int64_t i3 = batch_idx / ne2_1; + const int64_t i2 = batch_idx % ne2_1; + + const char *src1_batch = src1_base + i3 * nb3_1 + i2 * nb2_1; + char *dst_batch = dst_base + i3 * nb3_d + i2 * nb2_d; + + const int64_t mb = mb_idx * TILE_M; + const int64_t nb_base_t = g_idx * ru_n; + int64_t r_count = n_tiles - nb_base_t; + if (r_count > ru_n) r_count = ru_n; + + for (int64_t kw = 0; kw < n_windows; ++kw) { + const int buf = wid & 1; + wid++; + scp_wait(ready_ctr, wid); + + const int64_t kb0 = k_start_block + kw * KWIN; + const int64_t kbn = (kb0 + KWIN <= k_end_block) ? KWIN : (k_end_block - kb0); + const int is_last = (kw == n_windows - 1); + et_fp16_t *cf = (et_fp16_t *) cache_buf[buf]; + + for (int64_t r = 0; r < r_count; ++r) { + const int64_t nb = (nb_base_t + r) * TILE_N; + const int64_t n_cur = (nb + TILE_N <= N) ? TILE_N : (N - nb); + + char *cs = cscratch + r * (16 * 64); + + int first; + if (kw == 0) { + first = 1; + } else { + c_seed(cs); + first = 0; + } + + // Set tensor_mask for partial N tiles + if (n_cur < TILE_N) { + uint64_t mask = (1ULL << n_cur) - 1; + __asm__ __volatile__("csrw 0x805, %0" : : "r"(mask)); + } + + const int64_t nsteps = kbn; +#define A_TILE_ADDR(st) \ + ((uint64_t)(src1_batch + nb * nb1_1 + \ + (kb0 + (st)) * TILE_K * (int64_t) sizeof(et_fp16_t))) + + // prologue: prefetch step 0 and 1, and load step 0 + l2_prefetch((const void *) A_TILE_ADDR(0), n_cur, nb1_1); + if (1 < nsteps) { + l2_prefetch((const void *) A_TILE_ADDR(1), n_cur, nb1_1); + } + tensor_load((n_cur < TILE_N), false, A_L1_START, TENSOR_LOAD_PLAIN, 0, + A_TILE_ADDR(0), 0, n_cur - 1, (uint64_t) nb1_1, 0); + + for (int64_t s = 0; s < nsteps; ++s) { + const uint64_t a_cur = (s & 1) ? A_L1_ALT : A_L1_START; + + // Start loading the next step of A + if (s + 1 < nsteps) { + const uint64_t a_nxt = ((s + 1) & 1) ? A_L1_ALT : A_L1_START; + tensor_load((n_cur < TILE_N), false, a_nxt, TENSOR_LOAD_PLAIN, 0, + A_TILE_ADDR(s + 1), 0, n_cur - 1, (uint64_t) nb1_1, 0); + } + + // Prefetch step s+2 + if (s + 2 < nsteps) { + l2_prefetch((const void *) A_TILE_ADDR(s + 2), n_cur, nb1_1); + } + + // Load B from L2 SCP to L1 SCP (TL1 engine, B_L1_START) + tensor_load(false, false, B_L1_START, TENSOR_LOAD_PLAIN, 0, + (uint64_t)(cf + s * (SCP_PANEL_SIZE / sizeof(et_fp16_t))), + 0, 15, 64, 1); + + tensor_wait(TENSOR_LOAD_WAIT_0); + tensor_wait(TENSOR_LOAD_WAIT_1); + + tensor_fma((n_cur < TILE_N), 3, n_cur - 1, 15, 0, + false, false, false, false, + B_L1_START, a_cur, TENSOR_FMA_OP_FP16, first); + tensor_wait(TENSOR_FMA_WAIT); + first = 0; + } +#undef A_TILE_ADDR + + if (is_last) { + if (k_splits > 1) { + const uint64_t num_regs = (uint64_t) n_cur * 2; + if (k_split > 0) { + tensor_reduce_recv(0, TENSOR_REDUCE_OP_FADD, num_regs, + group_base_global + (uint64_t)(k_split - 1)); + tensor_wait(TENSOR_REDUCE_WAIT); + } + if (k_split < k_splits - 1) { + tensor_reduce_send(0, num_regs, + group_base_global + (uint64_t)(k_split + 1)); + tensor_wait(TENSOR_REDUCE_WAIT); + } + } + if (k_split == k_splits - 1) { + tensor_store( + 0, 0, 3, n_cur - 1, + (uint64_t)(dst_batch + nb * nb1_d + mb * (int64_t) sizeof(float)), + 0, (uint64_t) nb1_d); + tensor_wait(TENSOR_STORE_WAIT); + } + } else { + c_spill(cs); + } + } + scp_signal(consumed_ctr, wid); + } + } + FENCE; + return 0; + } + + // ===================================================================== + // ORIGINAL path: one output tile at a time (no reuse). + // ===================================================================== + const int64_t base_tiles = m_tiles * n_tiles * batch_count; + et_fp16_t *scp_panel[2] = { (et_fp16_t *) et_shire_l2scp_local(scp_base), - (et_fp16_t *) et_shire_l2scp_local(scp_base + SCP_BPANEL_SIZE), + (et_fp16_t *) et_shire_l2scp_local(scp_base + SCP_PANEL_SIZE), }; - volatile uint32_t * ready_ctr = (volatile uint32_t *) et_shire_l2scp_local(scp_base + SCP_READY_OFF); - volatile uint32_t * consumed_ctr = (volatile uint32_t *) et_shire_l2scp_local(scp_base + SCP_CONSUMED_OFF); - // ================================================================ - // Hart 1: B-panel packer - // ================================================================ if (is_hart1) { - // Initialize sync counters scp_signal(ready_ctr, 0); scp_signal(consumed_ctr, 0); + et_barrier(ET_BARRIER_MINION); uint32_t chunk_id = 0; - for (int64_t tile = (int64_t) shire_id + local_tile_idx * NUM_COMPUTE_SHIRES; tile < base_tiles; - tile += tiles_stride) { + for (int64_t tile = my_start; tile < base_tiles; tile += tiles_stride) { const int64_t tiles_per_batch = m_tiles * n_tiles; const int64_t batch_idx = tile / tiles_per_batch; const int64_t tile_in_batch = tile % tiles_per_batch; - - const int64_t mb_idx = tile_in_batch % m_tiles; + const int64_t mb_idx = tile_in_batch % m_tiles; const int64_t i3 = batch_idx / ne2_1; const int64_t i2 = batch_idx % ne2_1; const int64_t i2_0 = i2 / r2; const int64_t i3_0 = i3 / r3; - const char * src0_batch = src0_base + i3_0 * nb3_0 + i2_0 * nb2_0; - const int64_t mb = mb_idx * TILE_M; + const char *src0_batch = src0_base + i3_0 * nb3_0 + i2_0 * nb2_0; + const int64_t mb = mb_idx * TILE_M; + + // Prefetch first block + l2_prefetch(src0_batch + mb * nb1_0 + k_start * sizeof(et_fp16_t), 16, nb1_0); for (int64_t kb = k_start; kb < k_end; kb += TILE_K) { int buf = chunk_id & 1; + if (chunk_id >= 2) scp_wait(consumed_ctr, chunk_id - 1); - // Back-pressure: wait for hart 0 to finish with this buffer - if (chunk_id >= 2) { - scp_wait(consumed_ctr, chunk_id - 1); + // Prefetch next block + if (kb + TILE_K < k_end) { + l2_prefetch(src0_batch + mb * nb1_0 + (kb + TILE_K) * sizeof(et_fp16_t), 16, nb1_0); } - pack_b_interleaved(scp_bp[buf], src0_batch, mb, kb, nb1_0); + pack_b_interleaved(scp_panel[buf], src0_batch, mb, kb, nb1_0); FENCE; - flush_to_l2(scp_bp[buf], 16, 64); + flush_to_l2(scp_panel[buf], 16, 64); WAIT_CACHEOPS; chunk_id++; scp_signal(ready_ctr, chunk_id); } } - FENCE; return 0; } - // ================================================================ - // Hart 0: tensor engine compute - // ================================================================ - uint64_t my_minion_id = get_minion_id(); - const uint64_t group_base_global = my_minion_id - k_split; - setup_cache_scp(); #if CACHEOP_MAX > 0 || REP_RATE > 0 ucache_control(1, REP_RATE, CACHEOP_MAX); #endif CLEAR_TENSOR_ERROR; - // Evict any stale L1D copies of sync counters - evict_to_l2((const void *) ready_ctr, 1, 64); - WAIT_CACHEOPS; - evict_to_l2((const void *) consumed_ctr, 1, 64); - WAIT_CACHEOPS; + et_barrier(ET_BARRIER_MINION); + evict_to_l2((const void *)(uintptr_t) ready_ctr, 1, 64); WAIT_CACHEOPS; + evict_to_l2((const void *)(uintptr_t) consumed_ctr, 1, 64); WAIT_CACHEOPS; + const uint64_t group_base_global = get_minion_id() - (uint64_t) k_split; uint32_t chunk_id = 0; - for (int64_t tile = (int64_t) shire_id + local_tile_idx * NUM_COMPUTE_SHIRES; tile < base_tiles; - tile += tiles_stride) { + for (int64_t tile = my_start; tile < base_tiles; tile += tiles_stride) { const int64_t tiles_per_batch = m_tiles * n_tiles; const int64_t batch_idx = tile / tiles_per_batch; const int64_t tile_in_batch = tile % tiles_per_batch; - - const int64_t nb_idx = tile_in_batch / m_tiles; - const int64_t mb_idx = tile_in_batch % m_tiles; + const int64_t nb_idx = tile_in_batch / m_tiles; + const int64_t mb_idx = tile_in_batch % m_tiles; const int64_t i3 = batch_idx / ne2_1; const int64_t i2 = batch_idx % ne2_1; - const char * src1_batch = src1_base + i3 * nb3_1 + i2 * nb2_1; - char * dst_batch = dst_base + i3 * nb3_d + i2 * nb2_d; + const char *src1_batch = src1_base + i3 * nb3_1 + i2 * nb2_1; + char *dst_batch = dst_base + i3 * nb3_d + i2 * nb2_d; - const int64_t mb = mb_idx * TILE_M; - const int64_t nb = nb_idx * TILE_N; + const int64_t mb = mb_idx * TILE_M; + const int64_t nb = nb_idx * TILE_N; const int64_t n_cur = (nb + TILE_N <= N) ? TILE_N : (N - nb); + int first = 1; + // Set tensor_mask for partial N tiles if (n_cur < TILE_N) { uint64_t mask = (1ULL << n_cur) - 1; __asm__ __volatile__("csrw 0x805, %0" : : "r"(mask)); } + // Prologue: prefetch first two blocks, load first block + l2_prefetch((const void *)(src1_batch + nb * nb1_1 + k_start * sizeof(et_fp16_t)), n_cur, nb1_1); + if (k_start + TILE_K < k_end) { + l2_prefetch((const void *)(src1_batch + nb * nb1_1 + (k_start + TILE_K) * sizeof(et_fp16_t)), n_cur, nb1_1); + } + tensor_load( + (n_cur < TILE_N), false, A_L1_START, TENSOR_LOAD_PLAIN, 0, + (uint64_t)(src1_batch + nb * nb1_1 + k_start * (int64_t) sizeof(et_fp16_t)), + 0, n_cur - 1, (uint64_t) nb1_1, 0); + for (int64_t kb = k_start; kb < k_end; kb += TILE_K) { int buf = chunk_id & 1; + const uint64_t a_cur = ((kb - k_start) / TILE_K & 1) ? A_L1_ALT : A_L1_START; - // Start loading A from DRAM (overlaps with waiting for hart 1) - tensor_load((n_cur < TILE_N), false, A_L1_START, TENSOR_LOAD_PLAIN, 0, - (uint64_t) (src1_batch + nb * nb1_1 + kb * (int64_t) sizeof(et_fp16_t)), 0, n_cur - 1, - (uint64_t) nb1_1, 0); + tensor_wait(TENSOR_LOAD_WAIT_0); + + if (kb + TILE_K < k_end) { + const uint64_t a_nxt = (((kb + TILE_K) - k_start) / TILE_K & 1) ? A_L1_ALT : A_L1_START; + tensor_load( + (n_cur < TILE_N), false, a_nxt, TENSOR_LOAD_PLAIN, 0, + (uint64_t)(src1_batch + nb * nb1_1 + (kb + TILE_K) * (int64_t) sizeof(et_fp16_t)), + 0, n_cur - 1, (uint64_t) nb1_1, 0); + } + + if (kb + 2 * TILE_K < k_end) { + l2_prefetch((const void *)(src1_batch + nb * nb1_1 + (kb + 2 * TILE_K) * sizeof(et_fp16_t)), n_cur, nb1_1); + } - // Wait for hart 1 to finish packing this chunk chunk_id++; scp_wait(ready_ctr, chunk_id); - // Load B from L2 SCP (hart 1 already flushed it) - tensor_load(false, false, B_L1_START, TENSOR_LOAD_PLAIN, 0, (uint64_t) scp_bp[buf], 0, 15, 64, 1); - - tensor_wait(TENSOR_LOAD_WAIT_0); + // Load B from L2 SCP to L1 SCP (TL1 engine, B_L1_START) + tensor_load(false, false, B_L1_START, TENSOR_LOAD_PLAIN, 0, (uint64_t) scp_panel[buf], 0, 15, 64, 1); tensor_wait(TENSOR_LOAD_WAIT_1); - // TensorFMA16A32 - tensor_fma((n_cur < TILE_N), 3, n_cur - 1, 15, 0, false, false, false, false, B_L1_START, A_L1_START, - TENSOR_FMA_OP_FP16, (kb == k_start)); - + tensor_fma((n_cur < TILE_N), 3, n_cur - 1, 15, 0, + false, false, false, false, + B_L1_START, a_cur, TENSOR_FMA_OP_FP16, first); tensor_wait(TENSOR_FMA_WAIT); + first = 0; - // Signal that this buffer is free for hart 1 to reuse scp_signal(consumed_ctr, chunk_id); } @@ -316,10 +564,11 @@ int entry_point(struct ggml_et_binary_params * params, void * env) { } } - // Store FP32 result tile if (k_split == k_splits - 1) { - tensor_store(0, 0, 3, n_cur - 1, (uint64_t) (dst_batch + nb * nb1_d + mb * (int64_t) sizeof(float)), 0, - (uint64_t) nb1_d); + tensor_store( + 0, 0, 3, n_cur - 1, + (uint64_t)(dst_batch + nb * nb1_d + mb * (int64_t) sizeof(float)), + 0, (uint64_t) nb1_d); tensor_wait(TENSOR_STORE_WAIT); } } diff --git a/ggml/src/ggml-et/et-kernels/src/platform.h b/ggml/src/ggml-et/et-kernels/src/platform.h index 8a936da4c577..b014b4e61203 100644 --- a/ggml/src/ggml-et/et-kernels/src/platform.h +++ b/ggml/src/ggml-et/et-kernels/src/platform.h @@ -476,6 +476,20 @@ static inline void __attribute__((always_inline)) flush_to_l2(const void * addr, : "x31", "memory"); } +// Flush an arbitrary number of lines to L2, working around the 16-line cap of a +// single FlushVA by issuing multiple flushes. Use this whenever nlines may exceed 16. +static inline void __attribute__((always_inline)) flush_to_l2_multi(const void * addr, uint64_t nlines, uint64_t stride) { + const char * p = (const char *) addr; + while (nlines > 16) { + flush_to_l2(p, 16, stride); + p += 16 * stride; + nlines -= 16; + } + if (nlines) { + flush_to_l2(p, nlines, stride); + } +} + // Evict nlines cache lines at stride apart starting at addr from L1 to L2. // Uses EvictVA (CSR 0x89F). Unlike flush_to_l2, this guarantees the line is // NOT present in L1 after the operation - subsequent loads will miss and go diff --git a/ggml/src/ggml-et/ggml-et-ops.cpp b/ggml/src/ggml-et/ggml-et-ops.cpp index 6c80fe8acde3..42796fc00c9e 100644 --- a/ggml/src/ggml-et/ggml-et-ops.cpp +++ b/ggml/src/ggml-et/ggml-et-ops.cpp @@ -738,13 +738,13 @@ bool ggml_et_op_mul_mat(ggml_backend_et_device_context * dev_ctx, } else if (node->type == GGML_TYPE_F32 && node->src[0]->type == GGML_TYPE_F16 && node->src[1]->type == GGML_TYPE_F16 && node->ne[0] % 16 == 0 && node->src[0]->ne[0] % 16 == 0 && - node->src[0]->ne[1] % 16 == 0 && node->src[1]->ne[0] != 1) { + node->src[0]->ne[1] % 16 == 0 && node->src[1]->ne[1] > 2) { // N > 2 (prefill): use matrix engine kernel_name = "mul_mat_f16_matrix_engine"; src0_type_name = "F16"; } else if (node->type == GGML_TYPE_F32 && node->src[0]->type == GGML_TYPE_F16 && (node->src[1]->type == GGML_TYPE_F16 || node->src[1]->type == GGML_TYPE_F32)) { - kernel_name = "mul_mat_f16"; + kernel_name = "mul_mat_f16"; // N <= 2, or shape doesn't fit the matrix-engine tiling src0_type_name = "F16"; } else if (node->type == GGML_TYPE_F32 && node->src[0]->type == GGML_TYPE_F32 && From 0f5fcbf2fb929b441b683977bd295b260908c5a9 Mon Sep 17 00:00:00 2001 From: Rehan Qasim Date: Thu, 30 Jul 2026 01:24:04 -0700 Subject: [PATCH 4/5] ET mul_mat: add F16-weight/F32-activation matrix-engine kernel MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Port mul_mat_f16_f32_matrix_engine from the et-perf-debug branch: only F16xF16 had a matrix-engine path here, so any F16 weight paired with the usual F32 activation (the common case, e.g. token_embd/output kept in F16) fell back to vec_dot, unable to use the tensor engine at all. Tensor engine has no mixed-precision FMA, so Hart 1 upconverts F16 weights to F32 while packing, Hart 0 runs plain TensorFMA32. Threshold re-measured from scratch on this branch (no graph-profiler/perf-counter instrumentation compiled in here, unlike et-perf-debug) to rule out measurement overhead skewing the crossover point. Llama-3.2-1B FP16, ET_DEVICES=0, llama-bench -n 0. Coarse sweep (step 10), N | vec_dot | matrix-engine (forced) | final (threshold=17): 10 | 49.59 | 38.81 | 49.55 20 | 60.41 | 74.76 | 74.81 30 | 65.60 | 106.73 | 106.74 40 | 69.05 | 136.20 | 136.54 50 | 71.27 | 169.32 | 169.77 60 | 72.15 | 193.83 | 193.87 70 | 73.01 | 218.51 | 219.69 80 | 73.65 | 249.22 | 249.96 90 | 74.53 | 270.81 | 271.32 100 | 74.76 | 289.70 | 288.83 120 | 75.79 | 339.23 | 342.48 Fine sweep (step 2, crossover range 10-20), same columns: 10 | 49.59 | 38.81 | 49.55 12 | 54.39 | 46.25 | 54.17 14 | 57.50 | 52.36 | 57.58 16 | 60.51 | 59.98 | 60.28 17 | 57.14 | 63.91 | 63.53 18 | 58.13 | 67.56 | 67.16 20 | 60.41 | 74.76 | 74.81 Exact-integer check at the 16-17 boundary re-confirmed at -r 8 (values above). Threshold set to 17, identical to the value measured on et-perf-debug — the profiler/perf-counter overhead theory doesn't hold, this is the real crossover. Final column tracks the better of the two forced curves at every row. Correctness verified via test-backend-ops (208/208, both baseline vec_dot and matrix-engine forced) and llama-cli coherence. (cherry picked from commit d0db8d7f45e182da890dfdd8832129bda72f9ef8) --- ggml/src/ggml-et/CMakeLists.txt | 1 + .../src/mul_mat_f16_f32_matrix_engine.c | 683 ++++++++++++++++++ ggml/src/ggml-et/ggml-et-ops.cpp | 10 + 3 files changed, 694 insertions(+) create mode 100644 ggml/src/ggml-et/et-kernels/src/mul_mat_f16_f32_matrix_engine.c diff --git a/ggml/src/ggml-et/CMakeLists.txt b/ggml/src/ggml-et/CMakeLists.txt index ee0ee3759a91..33b2ba9b4df0 100644 --- a/ggml/src/ggml-et/CMakeLists.txt +++ b/ggml/src/ggml-et/CMakeLists.txt @@ -40,6 +40,7 @@ set(KERNELS mul_mat_Q4_0_matrix_engine mul_mat_f16 mul_mat_f16_matrix_engine + mul_mat_f16_f32_matrix_engine rope_f32 unary_f32 sqr_f32 diff --git a/ggml/src/ggml-et/et-kernels/src/mul_mat_f16_f32_matrix_engine.c b/ggml/src/ggml-et/et-kernels/src/mul_mat_f16_f32_matrix_engine.c new file mode 100644 index 000000000000..7bfb8f9fe0fe --- /dev/null +++ b/ggml/src/ggml-et/et-kernels/src/mul_mat_f16_f32_matrix_engine.c @@ -0,0 +1,683 @@ +#include +#include +#include "ggml_tensor.h" +#include "platform.h" +#include "tensor.h" + +/* + * High-performance F16-weight x F32-activation Matrix Multiply for ET-SoC-1 — + * TensorFMA32. Double-buffered producer/consumer implementation. + * + * The tensor engine has no mixed-precision FMA (only FP32xFP32, FP16xFP16->FP32, + * INT8xINT8->INT32 — see tensor_fma CSR 0x801 bits[3:1]), so an F16 weight cannot + * be multiplied directly against an F32 activation. Hart 1 (producer) up-converts + * each F16 weight element to FP32 while packing the B-panel, so Hart 0 (consumer) + * can run ordinary TensorFMA32 against F32 activations loaded straight from src1. + * This is otherwise identical to mul_mat_f32_matrix_engine.c. + * + * The up-conversion uses the hardware FCVT.PS.F16 instruction, vectorized + * across all 8 lanes (8 elements converted per instruction) — see + * pack_b_f16_to_f32() below. FCVT.PS.F16 is a Type-C FP instruction subject + * to a documented VPURF bypass-timing erratum (A0): reading its destination + * register within 0-6 instructions of the write can return stale data unless + * an `fmv.x.w x0,fd` + delay-slot workaround is inserted (the shared + * fp16_to_fp32() helper in math_fp.h does not include it, and several + * existing K-quant matrix-engine kernels — Q2/Q3/Q5/Q6_K — hit exactly this + * and worked around it by switching to a pure-integer scalar bit-trick + * instead). Here the workaround is applied explicitly and verified against + * the manual's exact sequence, so the fast hardware path is used safely + * rather than avoided. + * + * Two execution paths: + * * REUSE path (n_tiles >= 2): prepared weight tiles are reused across + * ru_n N-tiles to minimize DRAM reads. + * * ORIGINAL path (n_tiles == 1, GEMV): one output tile at a time, no reuse. + */ + +#define NUM_COMPUTE_SHIRES 32 +#define MINIONS_PER_SHIRE 32 + +#define TILE_M 16 +#define TILE_N 16 +#define TILE_K 16 + +#ifndef REUSE_MAX +#define REUSE_MAX 32 +#endif +#ifndef KWIN +#define KWIN 16 // K-blocks per window +#endif + +// --- Bottleneck stub test (producer-consumer-tensor-kernels guidelines §6) -- +// Replace one path with a minimum-work stub that keeps the pipeline turning as +// fast as possible (wrong result, throughput only). Build with -DSTUB_PRODUCER=1 +// or -DSTUB_CONSUMER=1 and compare throughput against the real build to find the +// bottleneck. Never enable both at once for a real run. +#ifndef STUB_PRODUCER +#define STUB_PRODUCER 0 +#endif +#ifndef STUB_CONSUMER +#define STUB_CONSUMER 0 +#endif + +#define MACHINE_SLOTS (NUM_COMPUTE_SHIRES * MINIONS_PER_SHIRE) // 1024 + +#define CACHEOP_MAX 0 +#define REP_RATE 0 + +#define A_L1_START 0 // L1 SCP lines 0..15 for A (buffer 0) +#define A_L1_ALT 16 // L1 SCP lines 16..31 for A (buffer 1) + +typedef uint16_t et_fp16_t; + +#define SCP_PANEL_SIZE (TILE_K * TILE_M * (uint64_t)sizeof(float)) // 1024 bytes + +#define RU_BUF_BYTES (KWIN * SCP_PANEL_SIZE) +#define RU_CACHE_BYTES (2 * RU_BUF_BYTES) +#define RU_CSCRATCH_BYTES (REUSE_MAX * 16 * 64ULL) +#define SCP_READY_OFF (RU_CACHE_BYTES + RU_CSCRATCH_BYTES) +#define SCP_CONSUMED_OFF (SCP_READY_OFF + 64) +#define SCP_PER_MINION (SCP_CONSUMED_OFF + 64) + +// Vectorized fp16->fp32 up-convert of one TILE_K x TILE_M block of F16 +// weights into the FP32 panel, written in TenB [k][m] order: +// panel[k*TILE_M + m] — the exact layout mul_mat_f32_matrix_engine.c's +// pack_b_f32_transpose produces, so the consumer-side tensor_load_setup_b / +// tensor_fma code is unchanged from it. +// +// One row (TILE_K=16 contiguous F16 elements) is a single 256-bit flw.ps +// load: 8 lanes, each lane = 2 packed halfwords (even element in bits +// [15:0], odd in bits [31:16], little-endian). FCVT.PS.F16 up-converts the +// *lower* 16 bits of each lane (confirmed by mul_mat_Q4_0_matrix_engine.c's +// scale conversion, which zero-extends the fp16 scale into bits[15:0] +// before FCVT — not the manual's own wording, which is inverted for this +// opcode), so the even elements convert directly from the loaded register; +// the odd elements need one FSRLI.PI by 16 first to bring them down into +// the lower half. That is 8 elements converted per FCVT.PS.F16, i.e. one +// row (16 elements) in a handful of vector instructions instead of 16 +// scalar bit-trick calls. +// +// FCVT.PS.F16 is a Type-C FP instruction subject to a documented VPURF +// bypass-timing erratum (A0): reading its destination within 0-6 +// instructions of the write can return stale data unless an `fmv.x.w x0,fd` +// + one-instruction delay slot is inserted before the read. The two +// conversions below are interleaved so each one's own delay requirement is +// satisfied by an instruction that belongs to the *other* conversion — +// no filler instructions needed. See the file header for why this is used +// here (instead of the pure-integer software path) despite that erratum: +// each fcvt result is read with the mandated workaround in place, so the +// hazard is closed rather than avoided. +static inline void __attribute__((always_inline)) +pack_b_f16_to_f32(float *out, const char *src0_batch, + int64_t mb, int64_t kb, int64_t nb1_0) { + // Byte offsets (relative to the column base) of the even/odd K-lines in + // the [k][m] panel: line k lives at byte k*64 (TILE_M*sizeof(float)). + static const int32_t __attribute__((aligned(32))) even_idx[8] = { + 0, 128, 256, 384, 512, 640, 768, 896 + }; + static const int32_t __attribute__((aligned(32))) odd_idx[8] = { + 64, 192, 320, 448, 576, 704, 832, 960 + }; + + unsigned long old_mask; + __asm__ volatile( + "mova.x.m %[ms] \n\t" + "mov.m.x m0, x0, 0xFF \n\t" // all 8 lanes active + "flw.ps f1, (%[eidx]) \n\t" // f1 = even scatter offsets + "flw.ps f6, (%[oidx]) \n\t" // f6 = odd scatter offsets + : [ms] "=&r"(old_mask) + : [eidx] "r"(even_idx), [oidx] "r"(odd_idx) + : "f1", "f6" + ); + + char *pbase = (char *) out; + for (int j = 0; j < TILE_M; ++j) { + const et_fp16_t *row = (const et_fp16_t *)(src0_batch + (mb + j) * nb1_0) + kb; + char *col = pbase + j * 4; // column m=j of the panel + + __asm__ volatile( + "flw.ps f2, 0(%[src]) \n\t" // 8 lanes: {odd<<16 | even} per lane + "fcvt.ps.f16 f3, f2 \n\t" // FCVT reads the LOWER 16 bits (proven + // by mul_mat_Q4_0_matrix_engine.c's + // scale conversion: fp16 zero-extended + // into bits[15:0], no shift) -> + // f3 = fp32(even elems k=0,2,...,14) + "fsrli.pi f4, f2, 16 \n\t" // odd elems -> lower half of each lane + "fcvt.ps.f16 f4, f4 \n\t" // f4 = fp32(odd elems k=1,3,5,...,15) + "fmv.x.w x0, f3 \n\t" // VPURF workaround: force f3 writeback + "fmv.x.w x0, f4 \n\t" // VPURF workaround: force f4 writeback + // (also f3's required delay slot) + "fscw.ps f3, f1(%[col]) \n\t" // scatter even -> panel[k][j], k even + "fscw.ps f4, f6(%[col]) \n\t" // scatter odd -> panel[k][j], k odd + : + : [src] "r"(row), [col] "r"(col) + : "f2", "f3", "f4", "memory" + ); + } + + __asm__ volatile("mova.m.x %0" :: "r"(old_mask)); +} + +#define C_ROW_PAIR_ST(n0, n1, base) \ + __asm__ volatile("fsw.ps f" #n0 ", (%0)\n\t fsw.ps f" #n1 ", (%1)\n\t" \ + :: "r"((base)), "r"((base) + 32) : "memory") +#define C_ROW_PAIR_LD(n0, n1, base) \ + __asm__ volatile("flw.ps f" #n0 ", (%0)\n\t flw.ps f" #n1 ", (%1)\n\t" \ + :: "r"((base)), "r"((base) + 32) : "f" #n0, "f" #n1) + +static inline void __attribute__((always_inline)) +c_spill(char *s) { + C_ROW_PAIR_ST(0, 1, s + 0 * 64); C_ROW_PAIR_ST(2, 3, s + 1 * 64); + C_ROW_PAIR_ST(4, 5, s + 2 * 64); C_ROW_PAIR_ST(6, 7, s + 3 * 64); + C_ROW_PAIR_ST(8, 9, s + 4 * 64); C_ROW_PAIR_ST(10, 11, s + 5 * 64); + C_ROW_PAIR_ST(12, 13, s + 6 * 64); C_ROW_PAIR_ST(14, 15, s + 7 * 64); + C_ROW_PAIR_ST(16, 17, s + 8 * 64); C_ROW_PAIR_ST(18, 19, s + 9 * 64); + C_ROW_PAIR_ST(20, 21, s + 10 * 64); C_ROW_PAIR_ST(22, 23, s + 11 * 64); + C_ROW_PAIR_ST(24, 25, s + 12 * 64); C_ROW_PAIR_ST(26, 27, s + 13 * 64); + C_ROW_PAIR_ST(28, 29, s + 14 * 64); C_ROW_PAIR_ST(30, 31, s + 15 * 64); +} + +static inline void __attribute__((always_inline)) +c_seed(char *s) { + C_ROW_PAIR_LD(0, 1, s + 0 * 64); C_ROW_PAIR_LD(2, 3, s + 1 * 64); + C_ROW_PAIR_LD(4, 5, s + 2 * 64); C_ROW_PAIR_LD(6, 7, s + 3 * 64); + C_ROW_PAIR_LD(8, 9, s + 4 * 64); C_ROW_PAIR_LD(10, 11, s + 5 * 64); + C_ROW_PAIR_LD(12, 13, s + 6 * 64); C_ROW_PAIR_LD(14, 15, s + 7 * 64); + C_ROW_PAIR_LD(16, 17, s + 8 * 64); C_ROW_PAIR_LD(18, 19, s + 9 * 64); + C_ROW_PAIR_LD(20, 21, s + 10 * 64); C_ROW_PAIR_LD(22, 23, s + 11 * 64); + C_ROW_PAIR_LD(24, 25, s + 12 * 64); C_ROW_PAIR_LD(26, 27, s + 13 * 64); + C_ROW_PAIR_LD(28, 29, s + 14 * 64); C_ROW_PAIR_LD(30, 31, s + 15 * 64); +} + +int entry_point(struct ggml_et_binary_params* params, void* env) { + (void) env; + + uint64_t hart_id = get_hart_id(); + uint64_t shire_id = get_shire_id(); + + if (shire_id >= NUM_COMPUTE_SHIRES) return 0; + + const int is_hart1 = hart_id & 1; + uint64_t local_minion = (hart_id >> 1) & 0x1F; + + const int64_t K = params->src0.ne[0]; + const int64_t M = params->src0.ne[1]; + const int64_t N = params->src1.ne[1]; + + if ((M % TILE_M) != 0) return 0; + if ((K % TILE_K) != 0) return 0; + + const int64_t ne2_0 = params->src0.ne[2], ne3_0 = params->src0.ne[3]; + const int64_t ne2_1 = params->src1.ne[2], ne3_1 = params->src1.ne[3]; + + const int64_t nb1_0 = params->src0.nb[1]; + const int64_t nb2_0 = params->src0.nb[2], nb3_0 = params->src0.nb[3]; + const int64_t nb1_1 = params->src1.nb[1]; + const int64_t nb2_1 = params->src1.nb[2], nb3_1 = params->src1.nb[3]; + const int64_t nb1_d = params->dst.nb[1]; + const int64_t nb2_d = params->dst.nb[2], nb3_d = params->dst.nb[3]; + + const char* src0_base = (const char*)params->src0.data; + const char* src1_base = (const char*)params->src1.data; + char* dst_base = (char*)params->dst.data; + + const int64_t m_tiles = M / TILE_M; + const int64_t n_tiles = (N + TILE_N - 1) / TILE_N; + const int64_t batch_count = ne2_1 * ne3_1; + + const int64_t r2 = ne2_1 / ne2_0; + const int64_t r3 = ne3_1 / ne3_0; + + const int64_t k_steps = K / TILE_K; + + // L2 SCP pointers for this minion + const uint64_t scp_base = local_minion * SCP_PER_MINION; + volatile uint32_t *ready_ctr = + (volatile uint32_t *) et_shire_l2scp_local(scp_base + SCP_READY_OFF); + volatile uint32_t *consumed_ctr = + (volatile uint32_t *) et_shire_l2scp_local(scp_base + SCP_CONSUMED_OFF); + + int64_t n_groups = (n_tiles + REUSE_MAX - 1) / REUSE_MAX; + if (n_groups < 1) n_groups = 1; + int64_t ru_n = (n_tiles + n_groups - 1) / n_groups; + if (ru_n < 1) ru_n = 1; + + const int64_t units_pb = m_tiles * n_groups; + const int64_t base_units = units_pb * batch_count; + + int64_t k_splits = 1; + { + int64_t ks = 1; + while (ks * 2 <= MINIONS_PER_SHIRE && + base_units * ks * 2 <= MACHINE_SLOTS && + (k_steps % (ks * 2)) == 0) { + ks *= 2; + } + k_splits = ks; + } + + const int64_t tiles_per_shire = MINIONS_PER_SHIRE / k_splits; + const int64_t k_split = local_minion % k_splits; + const int64_t local_tile_idx = local_minion / k_splits; + const int64_t tiles_stride = (int64_t) NUM_COMPUTE_SHIRES * tiles_per_shire; + const int64_t my_start = (int64_t) shire_id + local_tile_idx * NUM_COMPUTE_SHIRES; + + const int64_t k_steps_per_split = k_steps / k_splits; + const int64_t k_start_block = k_split * k_steps_per_split; + const int64_t k_start = k_start_block * TILE_K; + const int64_t k_end = k_start + k_steps_per_split * TILE_K; + + const int reuse_ok = (ru_n >= 2); + + // ===================================================================== + // REUSE path: convert each K-window once, reuse across ru_n N-tiles. + // ===================================================================== + if (reuse_ok) { + char *cache_buf[2] = { + (char *) et_shire_l2scp_local(scp_base), + (char *) et_shire_l2scp_local(scp_base + RU_BUF_BYTES), + }; + char *cscratch = (char *) et_shire_l2scp_local(scp_base + RU_CACHE_BYTES); + + const int64_t k_end_block = k_start_block + k_steps_per_split; + const int64_t n_windows = (k_steps_per_split + KWIN - 1) / KWIN; + + // ----- Hart 1: producer ----- + if (is_hart1) { + scp_signal(ready_ctr, 0); + scp_signal(consumed_ctr, 0); + + et_barrier(ET_BARRIER_MINION); + uint32_t wid = 0; + + for (int64_t unit = my_start; unit < base_units; unit += tiles_stride) { + const int64_t batch_idx = unit / units_pb; + const int64_t unit_in_b = unit % units_pb; + const int64_t mb_idx = unit_in_b % m_tiles; + + const int64_t i3 = batch_idx / ne2_1; + const int64_t i2 = batch_idx % ne2_1; + const int64_t i2_0 = i2 / r2; + const int64_t i3_0 = i3 / r3; + + const char *src0_batch = src0_base + i3_0 * nb3_0 + i2_0 * nb2_0; + const int64_t mb = mb_idx * TILE_M; + + // Prefetch first window's weights + { + const int64_t kbn_first = (k_start_block + KWIN <= k_end_block) ? KWIN : (k_end_block - k_start_block); + for (int64_t i = 0; i < kbn_first; ++i) { + l2_prefetch(src0_batch + mb * nb1_0 + (k_start_block + i) * TILE_K * sizeof(et_fp16_t), 16, nb1_0); + } + } + + for (int64_t kw = 0; kw < n_windows; ++kw) { + const int buf = wid & 1; + if (wid >= 2) scp_wait(consumed_ctr, wid - 1); + + const int64_t kb0 = k_start_block + kw * KWIN; + const int64_t kbn = (kb0 + KWIN <= k_end_block) ? KWIN : (k_end_block - kb0); + + // Prefetch next window's weights + if (kw + 1 < n_windows) { + const int64_t kb_next = kb0 + KWIN; + const int64_t kbn_next = (kb_next + KWIN <= k_end_block) ? KWIN : (k_end_block - kb_next); + for (int64_t i = 0; i < kbn_next; ++i) { + l2_prefetch(src0_batch + mb * nb1_0 + (kb_next + i) * TILE_K * sizeof(et_fp16_t), 16, nb1_0); + } + } + +#if !STUB_PRODUCER + float *cf = (float *) cache_buf[buf]; + for (int64_t i = 0; i < kbn; ++i) { + pack_b_f16_to_f32(cf + i * (SCP_PANEL_SIZE / sizeof(float)), + src0_batch, mb, (kb0 + i) * TILE_K, nb1_0); + } + FENCE; + flush_to_l2_multi(cache_buf[buf], kbn * TILE_K, 64); + WAIT_CACHEOPS; +#else + (void) kbn; +#endif + + wid++; + scp_signal(ready_ctr, wid); + } + } + FENCE; + return 0; + } + + // ----- Hart 0: consumer ----- + setup_cache_scp(); +#if CACHEOP_MAX > 0 || REP_RATE > 0 + ucache_control(1, REP_RATE, CACHEOP_MAX); +#endif + CLEAR_TENSOR_ERROR; + + et_barrier(ET_BARRIER_MINION); + evict_to_l2((const void *)(uintptr_t) ready_ctr, 1, 64); WAIT_CACHEOPS; + evict_to_l2((const void *)(uintptr_t) consumed_ctr, 1, 64); WAIT_CACHEOPS; + + const uint64_t group_base_global = get_minion_id() - (uint64_t) k_split; + + uint32_t wid = 0; + for (int64_t unit = my_start; unit < base_units; unit += tiles_stride) { + const int64_t batch_idx = unit / units_pb; + const int64_t unit_in_b = unit % units_pb; + const int64_t g_idx = unit_in_b / m_tiles; + const int64_t mb_idx = unit_in_b % m_tiles; + + const int64_t i3 = batch_idx / ne2_1; + const int64_t i2 = batch_idx % ne2_1; + + const char *src1_batch = src1_base + i3 * nb3_1 + i2 * nb2_1; + char *dst_batch = dst_base + i3 * nb3_d + i2 * nb2_d; + + const int64_t mb = mb_idx * TILE_M; + const int64_t nb_base_t = g_idx * ru_n; + int64_t r_count = n_tiles - nb_base_t; + if (r_count > ru_n) r_count = ru_n; + + for (int64_t kw = 0; kw < n_windows; ++kw) { + const int buf = wid & 1; + wid++; + + // Pure index arithmetic (no dependency on the producer's + // readiness signal) hoisted above scp_wait so it doesn't sit + // exposed after the wait. Also hoist r=0's activation + // prefetch (l2_prefetch is a non-VPU cache-management op, so + // it doesn't contend with the producer's VPU-bound conversion + // work) and r=0's C-accumulator seed (a flw.ps VPU load -- + // this DOES contend with the producer's VPU use, but the + // register file holds exactly one r-tile's C state, so only + // r=0 -- the very next r to be processed -- can be preloaded + // this way; r>0 still seed after the wait, unchanged). + const int64_t kb0 = k_start_block + kw * KWIN; + const int64_t kbn = (kb0 + KWIN <= k_end_block) ? KWIN : (k_end_block - kb0); + const int is_last = (kw == n_windows - 1); + + // STUB_CONSUMER: do only 1 FMA block (not all kbn) but keep the + // store/reduce so the engine drains and the kernel completes -> + // measures the producer-bound ceiling. !STUB: full kbn. + const int64_t kbn_c = STUB_CONSUMER ? 1 : kbn; + + int first0 = 1; + if (r_count > 0) { + const int64_t nb0 = nb_base_t * TILE_N; + const int64_t n_cur0 = (nb0 + TILE_N <= N) ? TILE_N : (N - nb0); +#define A_TILE_ADDR0(st) \ + ((uint64_t)(src1_batch + nb0 * nb1_1 + \ + (kb0 + (st)) * TILE_K * (int64_t) sizeof(float))) + l2_prefetch((const void *) A_TILE_ADDR0(0), n_cur0, nb1_1); + if (1 < kbn_c) { + l2_prefetch((const void *) A_TILE_ADDR0(1), n_cur0, nb1_1); + } +#undef A_TILE_ADDR0 + if (kw > 0) { + c_seed(cscratch); + first0 = 0; + } + } + + scp_wait(ready_ctr, wid); + + float *cf = (float *) cache_buf[buf]; + + for (int64_t r = 0; r < r_count; ++r) { + const int64_t nb = (nb_base_t + r) * TILE_N; + const int64_t n_cur = (nb + TILE_N <= N) ? TILE_N : (N - nb); + const int64_t arows_fma = (n_cur == 4) ? 4 : (n_cur - 1); + + char *cs = cscratch + r * (16 * 64); + + int first; + if (r == 0) { + first = first0; // pre-seeded above (or kw==0's implicit first) + } else if (kw == 0) { + first = 1; + } else { + c_seed(cs); + first = 0; + } + + const int64_t nsteps = kbn_c; +#define A_TILE_ADDR(st) \ + ((uint64_t)(src1_batch + nb * nb1_1 + \ + (kb0 + (st)) * TILE_K * (int64_t) sizeof(float))) + + // prologue: prefetch step 0 and 1 (r=0 already prefetched + // above, before the wait), and load step 0 + if (r != 0) { + l2_prefetch((const void *) A_TILE_ADDR(0), n_cur, nb1_1); + if (1 < nsteps) { + l2_prefetch((const void *) A_TILE_ADDR(1), n_cur, nb1_1); + } + } + tensor_load(false, false, A_L1_START, TENSOR_LOAD_PLAIN, 0, + A_TILE_ADDR(0), 0, n_cur - 1, (uint64_t) nb1_1, 0); + + for (int64_t s = 0; s < nsteps; ++s) { + const uint64_t a_cur = (s & 1) ? A_L1_ALT : A_L1_START; + + tensor_wait(TENSOR_LOAD_WAIT_0); + + if (s + 1 < nsteps) { + const uint64_t a_nxt = ((s + 1) & 1) ? A_L1_ALT : A_L1_START; + tensor_load(false, false, a_nxt, TENSOR_LOAD_PLAIN, 0, + A_TILE_ADDR(s + 1), 0, n_cur - 1, (uint64_t) nb1_1, 0); + } + + if (s + 2 < nsteps) { + l2_prefetch((const void *) A_TILE_ADDR(s + 2), n_cur, nb1_1); + } + + tensor_load_setup_b( + false, + (uint64_t)(cf + s * (SCP_PANEL_SIZE / sizeof(float))), + TILE_K - 1, 64, 1); + + tensor_wait(TENSOR_LOAD_WAIT_1); + + tensor_fma( + false, 3, arows_fma, TILE_K - 1, 0, + false, false, false, true, + 16, a_cur, TENSOR_FMA_OP_FP32, first); + tensor_wait(TENSOR_FMA_WAIT); + first = 0; + } +#undef A_TILE_ADDR + + if (is_last) { + if (k_splits > 1) { + const uint64_t num_regs = (uint64_t) n_cur * 2; + if (k_split > 0) { + tensor_reduce_recv(0, TENSOR_REDUCE_OP_FADD, num_regs, + group_base_global + (uint64_t)(k_split - 1)); + tensor_wait(TENSOR_REDUCE_WAIT); + } + if (k_split < k_splits - 1) { + tensor_reduce_send(0, num_regs, + group_base_global + (uint64_t)(k_split + 1)); + tensor_wait(TENSOR_REDUCE_WAIT); + } + } + if (k_split == k_splits - 1) { + tensor_store( + 0, 0, 3, n_cur - 1, + (uint64_t)(dst_batch + nb * nb1_d + mb * (int64_t) sizeof(float)), + 0, (uint64_t) nb1_d); + tensor_wait(TENSOR_STORE_WAIT); + } + } else { + c_spill(cs); + } + } + scp_signal(consumed_ctr, wid); + } + } + FENCE; + return 0; + } + + // ===================================================================== + // ORIGINAL path: one output tile at a time (no reuse). + // ===================================================================== + const int64_t base_tiles = m_tiles * n_tiles * batch_count; + float *scp_panel[2] = { + (float *) et_shire_l2scp_local(scp_base), + (float *) et_shire_l2scp_local(scp_base + SCP_PANEL_SIZE), + }; + + if (is_hart1) { + scp_signal(ready_ctr, 0); + scp_signal(consumed_ctr, 0); + + et_barrier(ET_BARRIER_MINION); + uint32_t chunk_id = 0; + + for (int64_t tile = my_start; tile < base_tiles; tile += tiles_stride) { + const int64_t tiles_per_batch = m_tiles * n_tiles; + const int64_t batch_idx = tile / tiles_per_batch; + const int64_t tile_in_batch = tile % tiles_per_batch; + const int64_t mb_idx = tile_in_batch % m_tiles; + + const int64_t i3 = batch_idx / ne2_1; + const int64_t i2 = batch_idx % ne2_1; + const int64_t i2_0 = i2 / r2; + const int64_t i3_0 = i3 / r3; + + const char *src0_batch = src0_base + i3_0 * nb3_0 + i2_0 * nb2_0; + const int64_t mb = mb_idx * TILE_M; + + // Prefetch first block + l2_prefetch(src0_batch + mb * nb1_0 + k_start * sizeof(et_fp16_t), 16, nb1_0); + + for (int64_t kb = k_start; kb < k_end; kb += TILE_K) { + int buf = chunk_id & 1; + if (chunk_id >= 2) scp_wait(consumed_ctr, chunk_id - 1); + + // Prefetch next block + if (kb + TILE_K < k_end) { + l2_prefetch(src0_batch + mb * nb1_0 + (kb + TILE_K) * sizeof(et_fp16_t), 16, nb1_0); + } + + pack_b_f16_to_f32(scp_panel[buf], src0_batch, mb, kb, nb1_0); + + FENCE; + flush_to_l2(scp_panel[buf], 16, 64); + WAIT_CACHEOPS; + + chunk_id++; + scp_signal(ready_ctr, chunk_id); + } + } + FENCE; + return 0; + } + + setup_cache_scp(); +#if CACHEOP_MAX > 0 || REP_RATE > 0 + ucache_control(1, REP_RATE, CACHEOP_MAX); +#endif + CLEAR_TENSOR_ERROR; + + et_barrier(ET_BARRIER_MINION); + evict_to_l2((const void *)(uintptr_t) ready_ctr, 1, 64); WAIT_CACHEOPS; + evict_to_l2((const void *)(uintptr_t) consumed_ctr, 1, 64); WAIT_CACHEOPS; + + const uint64_t group_base_global = get_minion_id() - (uint64_t) k_split; + uint32_t chunk_id = 0; + + for (int64_t tile = my_start; tile < base_tiles; tile += tiles_stride) { + const int64_t tiles_per_batch = m_tiles * n_tiles; + const int64_t batch_idx = tile / tiles_per_batch; + const int64_t tile_in_batch = tile % tiles_per_batch; + const int64_t nb_idx = tile_in_batch / m_tiles; + const int64_t mb_idx = tile_in_batch % m_tiles; + + const int64_t i3 = batch_idx / ne2_1; + const int64_t i2 = batch_idx % ne2_1; + + const char *src1_batch = src1_base + i3 * nb3_1 + i2 * nb2_1; + char *dst_batch = dst_base + i3 * nb3_d + i2 * nb2_d; + + const int64_t mb = mb_idx * TILE_M; + const int64_t nb = nb_idx * TILE_N; + const int64_t n_cur = (nb + TILE_N <= N) ? TILE_N : (N - nb); + + int first = 1; + // Prologue: prefetch first two blocks, load first block + l2_prefetch((const void *)(src1_batch + nb * nb1_1 + k_start * sizeof(float)), n_cur, nb1_1); + if (k_start + TILE_K < k_end) { + l2_prefetch((const void *)(src1_batch + nb * nb1_1 + (k_start + TILE_K) * sizeof(float)), n_cur, nb1_1); + } + tensor_load( + false, false, A_L1_START, TENSOR_LOAD_PLAIN, 0, + (uint64_t)(src1_batch + nb * nb1_1 + k_start * (int64_t) sizeof(float)), + 0, n_cur - 1, (uint64_t) nb1_1, 0); + + for (int64_t kb = k_start; kb < k_end; kb += TILE_K) { + int buf = chunk_id & 1; + const uint64_t a_cur = ((kb - k_start) / TILE_K & 1) ? A_L1_ALT : A_L1_START; + + tensor_wait(TENSOR_LOAD_WAIT_0); + + if (kb + TILE_K < k_end) { + const uint64_t a_nxt = (((kb + TILE_K) - k_start) / TILE_K & 1) ? A_L1_ALT : A_L1_START; + tensor_load( + false, false, a_nxt, TENSOR_LOAD_PLAIN, 0, + (uint64_t)(src1_batch + nb * nb1_1 + (kb + TILE_K) * (int64_t) sizeof(float)), + 0, n_cur - 1, (uint64_t) nb1_1, 0); + } + + if (kb + 2 * TILE_K < k_end) { + l2_prefetch((const void *)(src1_batch + nb * nb1_1 + (kb + 2 * TILE_K) * sizeof(float)), n_cur, nb1_1); + } + + chunk_id++; + scp_wait(ready_ctr, chunk_id); + + tensor_load_setup_b( + false, + (uint64_t) scp_panel[buf], + 15, 64, 1); + tensor_wait(TENSOR_LOAD_WAIT_1); + + tensor_fma( + false, 3, n_cur - 1, TILE_K - 1, 0, + false, false, false, true, + 16, a_cur, TENSOR_FMA_OP_FP32, first); + tensor_wait(TENSOR_FMA_WAIT); + first = 0; + + scp_signal(consumed_ctr, chunk_id); + } + + // K-split ring reduce + if (k_splits > 1) { + const uint64_t num_regs = (uint64_t) n_cur * 2; + + if (k_split > 0) { + tensor_reduce_recv(0, TENSOR_REDUCE_OP_FADD, num_regs, group_base_global + k_split - 1); + tensor_wait(TENSOR_REDUCE_WAIT); + } + + if (k_split < k_splits - 1) { + tensor_reduce_send(0, num_regs, group_base_global + k_split + 1); + tensor_wait(TENSOR_REDUCE_WAIT); + } + } + + if (k_split == k_splits - 1) { + tensor_store( + 0, 0, 3, n_cur - 1, + (uint64_t)(dst_batch + nb * nb1_d + mb * (int64_t) sizeof(float)), + 0, (uint64_t) nb1_d); + tensor_wait(TENSOR_STORE_WAIT); + } + } + + FENCE; + return 0; +} diff --git a/ggml/src/ggml-et/ggml-et-ops.cpp b/ggml/src/ggml-et/ggml-et-ops.cpp index 42796fc00c9e..c153bfd20874 100644 --- a/ggml/src/ggml-et/ggml-et-ops.cpp +++ b/ggml/src/ggml-et/ggml-et-ops.cpp @@ -742,6 +742,16 @@ bool ggml_et_op_mul_mat(ggml_backend_et_device_context * dev_ctx, kernel_name = "mul_mat_f16_matrix_engine"; src0_type_name = "F16"; + } else if (node->type == GGML_TYPE_F32 && node->src[0]->type == GGML_TYPE_F16 && + node->src[1]->type == GGML_TYPE_F32 && node->ne[0] % 16 == 0 && node->src[0]->ne[0] % 16 == 0 && + node->src[0]->ne[1] % 16 == 0 && node->src[1]->ne[1] >= 17) { // N < 17: vec_dot faster (measured) + + // F16 weights, F32 activations: the tensor engine has no mixed-precision + // FMA, so the kernel upconverts weights to F32 on Hart 1 and runs plain + // TensorFMA32. + kernel_name = "mul_mat_f16_f32_matrix_engine"; + src0_type_name = "F16"; + } else if (node->type == GGML_TYPE_F32 && node->src[0]->type == GGML_TYPE_F16 && (node->src[1]->type == GGML_TYPE_F16 || node->src[1]->type == GGML_TYPE_F32)) { kernel_name = "mul_mat_f16"; // N <= 2, or shape doesn't fit the matrix-engine tiling From f1bea73a6f78f54bd59418b68a5bceb170a7b2e0 Mon Sep 17 00:00:00 2001 From: Rehan Qasim Date: Thu, 30 Jul 2026 09:26:39 -0700 Subject: [PATCH 5/5] ggml-et: clarify F16 vecdot fallback comment covers two thresholds This fallback now serves both the F16xF16 path (N <= 2) and the newer F16-weight/F32-activation path (N < 17); the comment only mentioned the former. --- ggml/src/ggml-et/ggml-et-ops.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ggml/src/ggml-et/ggml-et-ops.cpp b/ggml/src/ggml-et/ggml-et-ops.cpp index c153bfd20874..54455396c163 100644 --- a/ggml/src/ggml-et/ggml-et-ops.cpp +++ b/ggml/src/ggml-et/ggml-et-ops.cpp @@ -754,7 +754,7 @@ bool ggml_et_op_mul_mat(ggml_backend_et_device_context * dev_ctx, } else if (node->type == GGML_TYPE_F32 && node->src[0]->type == GGML_TYPE_F16 && (node->src[1]->type == GGML_TYPE_F16 || node->src[1]->type == GGML_TYPE_F32)) { - kernel_name = "mul_mat_f16"; // N <= 2, or shape doesn't fit the matrix-engine tiling + kernel_name = "mul_mat_f16"; // N <= 2 (F16xF16) or N < 17 (F16 weights, F32 activations), or shape doesn't fit the matrix-engine tiling src0_type_name = "F16"; } else if (node->type == GGML_TYPE_F32 && node->src[0]->type == GGML_TYPE_F32 &&