diff --git a/ggml/src/ggml-et/et-kernels/src/block_ops.h b/ggml/src/ggml-et/et-kernels/src/block_ops.h index 78ffbde87bf..e5f1bea8213 100644 --- a/ggml/src/ggml-et/et-kernels/src/block_ops.h +++ b/ggml/src/ggml-et/et-kernels/src/block_ops.h @@ -7,6 +7,7 @@ # define BLOCK_OPS_H # include "math_fp.h" +# include "platform.h" # include "quants.h" # include @@ -735,6 +736,98 @@ static inline float compute_block_dot_product_f32_f16_partial(const float * a return sum; } +//****************************************************************************** +// F32 x F32 register-resident row dot (decode GEMV). The running sum lives in +// vector registers f20..f23 for the whole row instead of being spilled to +// memory every few elements, so the inner loop is just paired flw.ps loads and +// fmadd, no gather or convert needed since both operands are already f32. +// Four independent lane accumulators break the fmadd dependency chain so +// several A/B loads stay in flight, hiding load latency. +//****************************************************************************** +static inline void __attribute__((always_inline)) +f32_dot_reset(void) { + __asm__ volatile( + "fbci.pi f20, 0\n" + "fbci.pi f21, 0\n" + "fbci.pi f22, 0\n" + "fbci.pi f23, 0\n" + ::: "f20", "f21", "f22", "f23"); +} + +// Accumulate n_blocks blocks of 32 f32 A values times 32 f32 B into f20..f23. +static inline void __attribute__((always_inline)) +f32_dot_tile(const float * a_row, const float * b_col, int64_t n_blocks) { + // Prefetch A ahead into L2, not L1. The weight row is streamed once and + // never reused, but L1 is shared by both harts of a minion, so a simple + // L1 prefetch gets evicted before the load consumes it and the line is + // re-fetched from DRAM. l2_prefetch stages the line into L2 instead, which + // is large enough to hold the in-flight window, so a re-touch hits L2 + // rather than causing a second DRAM read. B lives in on-chip L2 SCP + // already, so it needs no prefetch. + // + // Each block is 128B = 2 lines. Prefetch 16 lines (8 blocks) at a time, + // staying PF_AHEAD blocks in front of the consuming load. + const int64_t PF_AHEAD = 16; + for (int64_t kb = 0; kb < n_blocks; kb++) { + const float * a_ptr = a_row + (kb << 5); // 32 f32 per block = 128B + const float * b_ptr = b_col + (kb << 5); + if ((kb & 7) == 0) { + const int64_t pf_block = kb + PF_AHEAD; + if (pf_block + 8 <= n_blocks) { + l2_prefetch(a_row + (pf_block << 5), 16, 64); // 16 lines = 8 blocks + } + } + __asm__ volatile( + // Issue all 8 independent loads first (4 A + 4 B) so several misses + // are outstanding before any consumer stalls in-order issue. + "flw.ps f11, %[a0]\n" + "flw.ps f13, %[a1]\n" + "flw.ps f15, %[a2]\n" + "flw.ps f17, %[a3]\n" + "flw.ps f12, %[b0]\n" + "flw.ps f14, %[b1]\n" + "flw.ps f16, %[b2]\n" + "flw.ps f18, %[b3]\n" + "fmadd.ps f20, f11, f12, f20\n" + "fmadd.ps f21, f13, f14, f21\n" + "fmadd.ps f22, f15, f16, f22\n" + "fmadd.ps f23, f17, f18, f23\n" + : + : [a0] "m"(*(const float(*)[8])&a_ptr[0]), + [a1] "m"(*(const float(*)[8])&a_ptr[8]), + [a2] "m"(*(const float(*)[8])&a_ptr[16]), + [a3] "m"(*(const float(*)[8])&a_ptr[24]), + [b0] "m"(*(const float(*)[8])&b_ptr[0]), + [b1] "m"(*(const float(*)[8])&b_ptr[8]), + [b2] "m"(*(const float(*)[8])&b_ptr[16]), + [b3] "m"(*(const float(*)[8])&b_ptr[24]) + : "f11", "f12", "f13", "f14", "f15", "f16", "f17", "f18", + "f20", "f21", "f22", "f23" + ); + } +} + +static inline float __attribute__((always_inline)) +f32_dot_reduce(void) { + float result; + __asm__ __volatile__ ( + // Combine the 4 lane accumulators, then horizontal-sum the 8 lanes. + "fadd.ps f20, f20, f21, rne \n\t" + "fadd.ps f22, f22, f23, rne \n\t" + "fadd.ps f20, f20, f22, rne \n\t" + "fswizz.ps f1, f20, 0xB1 \n\t" + "fadd.ps f2, f20, f1, rne \n\t" + "fswizz.ps f3, f2, 0x4E \n\t" + "fadd.ps f4, f2, f3, rne \n\t" + "fmvz.x.ps t0, f4, 4 \n\t" + "fbcx.ps f5, t0 \n\t" + "fadd.ps %[vout], f4, f5, rne \n\t" + : [vout] "=f" (result) + :: "t0", "f1", "f2", "f3", "f4", "f5", "f20", "f21", "f22", "f23" + ); + return result; +} + // Compute dot product between f32 block and f32 column vector // Vectorized: processes 8 elements at a time using ET vector instructions // Block size: 16 f32 values (64 bytes = 1 cache line) diff --git a/ggml/src/ggml-et/et-kernels/src/mul_mat_Q4_0_matrix_engine.c b/ggml/src/ggml-et/et-kernels/src/mul_mat_Q4_0_matrix_engine.c index 28a10303235..ac9640476a3 100644 --- a/ggml/src/ggml-et/et-kernels/src/mul_mat_Q4_0_matrix_engine.c +++ b/ggml/src/ggml-et/et-kernels/src/mul_mat_Q4_0_matrix_engine.c @@ -33,25 +33,6 @@ #define SCP_CONSUMED_OFF (SCP_READY_OFF + 64) // 4160 #define SCP_PER_MINION (SCP_CONSUMED_OFF + 64) // 4224 -// Signal a counter value to the other hart via L2 SCP. -static inline void __attribute__((always_inline)) scp_signal(volatile uint32_t * flag, uint32_t value) { - *flag = value; - FENCE; - evict_to_l2((const void *) flag, 1, 64); - WAIT_CACHEOPS; -} - -// Wait for a counter in L2 SCP to reach the expected value. -static inline void __attribute__((always_inline)) scp_wait(volatile uint32_t * flag, uint32_t expected) { - while (1) { - evict_to_l2((const void *) flag, 1, 64); - WAIT_CACHEOPS; - if (*flag >= expected) { - return; - } - } -} - // Dequantize one 32-element Q4_0 block of TILE_M weight rows into the FP32 // panel, written directly in TenB [k][m] order: panel[k*TILE_M + m]. // Low nibble of byte i -> k = i diff --git a/ggml/src/ggml-et/et-kernels/src/mul_mat_f16_matrix_engine.c b/ggml/src/ggml-et/et-kernels/src/mul_mat_f16_matrix_engine.c index 2aab87ad5e5..7c20db76c7e 100644 --- a/ggml/src/ggml-et/et-kernels/src/mul_mat_f16_matrix_engine.c +++ b/ggml/src/ggml-et/et-kernels/src/mul_mat_f16_matrix_engine.c @@ -38,25 +38,6 @@ typedef uint16_t et_fp16_t; #define SCP_CONSUMED_OFF (SCP_READY_OFF + 64) // 2112 #define SCP_PER_MINION (SCP_CONSUMED_OFF + 64) // 2176 -// Signal a counter value to the other hart via L2 SCP. -static inline void __attribute__((always_inline)) scp_signal(volatile uint32_t * flag, uint32_t value) { - *flag = value; - FENCE; - evict_to_l2((const void *) flag, 1, 64); - WAIT_CACHEOPS; -} - -// Wait for a counter in L2 SCP to reach the expected value. -static inline void __attribute__((always_inline)) scp_wait(volatile uint32_t * flag, uint32_t expected) { - while (1) { - evict_to_l2((const void *) flag, 1, 64); - WAIT_CACHEOPS; - if (*flag >= expected) { - return; - } - } -} - /** * Build the interleaved B panel that TensorFMA16A32 expects (vectorized). * diff --git a/ggml/src/ggml-et/et-kernels/src/mul_mat_f32.c b/ggml/src/ggml-et/et-kernels/src/mul_mat_f32.c index 107bc509301..7f6880e0cff 100644 --- a/ggml/src/ggml-et/et-kernels/src/mul_mat_f32.c +++ b/ggml/src/ggml-et/et-kernels/src/mul_mat_f32.c @@ -1,11 +1,23 @@ +//****************************************************************************** +// MUL_MAT Kernel (F32 weights) +// Matrix multiplication: C[M,N] = A[M,K] * B[K,N] +// +// Decode-optimized distribution: one output element (row m of A dotted with +// column n of B) is the unit of work, striped by thread_id across every hart +// of every active shire. This keeps all 32 shires busy for a GEMV +// (M=4096, N=1) instead of leaving most idle. The reused B vector is staged +// into per-shire L2 SCP and the accumulator stays register-resident across K +// via f32_dot_*. +//****************************************************************************** + #include "block_ops.h" #include "ggml_tensor.h" +#include "math_fp.h" #include "platform.h" #include "quants.h" +#include "tensor.h" -#include #include -#include int entry_point(struct ggml_et_binary_params * params, void * env) { kernel_environment_t * kernel_env = (kernel_environment_t *) env; @@ -14,18 +26,13 @@ int entry_point(struct ggml_et_binary_params * params, void * env) { return -1; } - // Thread coordination + // Thread coordination, use every hart of every active shire. int thread_id = get_relative_thread_id(kernel_env->shire_mask); int num_threads = get_num_threads(kernel_env->shire_mask); - - if (thread_id < 0 || (thread_id & 1)) { - return 0; // Skip odd threads to avoid resource contention + if (thread_id < 0) { + return 0; } - int effective_thread_id = thread_id / 2; - int effective_num_threads = (num_threads + 1) / 2; - - // Extract tensor references struct ggml_tensor * src0 = ¶ms->src0; // Weight matrix A (F32) struct ggml_tensor * src1 = ¶ms->src1; // Activation matrix B (F16/F32) struct ggml_tensor * dst = ¶ms->dst; // Output matrix C (F32) @@ -39,7 +46,7 @@ int entry_point(struct ggml_et_binary_params * params, void * env) { const float * src0_data = (const float *) src0->data; float * dst_data = (float *) dst->data; - // Dimensions and Strides + // Dimensions and strides const int64_t K = src0->ne[0]; const int64_t M = src0->ne[1]; const int64_t N = src1->ne[1]; @@ -52,86 +59,101 @@ int entry_point(struct ggml_et_binary_params * params, void * env) { const size_t nb11 = src1->nb[1], nb12 = src1->nb[2], nb13 = src1->nb[3]; const size_t nb1 = dst->nb[1], nb2 = dst->nb[2], nb3 = dst->nb[3]; - // F32 specific block size and counts - const int block_size = QK_F32; + const int block_size = 32; // 32 f32 per row-dot tile (128B) const int64_t K_blocks = K / block_size; const int64_t K_remainder = K % block_size; - // Threading distribution - const uint64_t total_elements = M * N * ne2 * ne3; - const uint64_t per_thread = 16; - const uint64_t threads_stride = per_thread * effective_num_threads; - - if (effective_thread_id * per_thread >= total_elements) { - return 0; - } - // Broadcasting support const int64_t r2 = ne12 / ne02; const int64_t r3 = ne13 / ne03; - for (uint64_t base_idx = effective_thread_id * per_thread; base_idx < total_elements; base_idx += threads_stride) { - for (uint64_t j = 0; j < per_thread; j++) { - const uint64_t idx = base_idx + j; - if (idx >= total_elements) { - break; + const int is_f32_b = (src1->type == GGML_TYPE_F32); + const uint64_t total_elements = (uint64_t) M * N * ne2 * ne3; + + // Stage the reused B activation vector into per-shire L2 SCP. Streaming the + // weight matrix thrashes L2 and evicts B, so B is re-read from DRAM + // repeatedly during the decode GEMV. Stage it once per shire into L2 SCP (a + // separate SRAM partition cache streaming cannot evict) via + // et_tensor_load_l2scp; one hart per shire issues the DMA loop, then a + // shire barrier lets all harts read B on-chip. Only applies to the common + // decode case: F32 B, a single contiguous non-broadcast B that fits the SCP + // budget. + const int b_contig = (nb11 == (size_t) K * sizeof(float)); + const uint64_t b_lines = ((uint64_t) N * K * sizeof(float) + 63) / 64; + const int stage_b = is_f32_b && b_contig && + ne12 == 1 && ne13 == 1 && ne02 == 1 && ne03 == 1 && + b_lines <= 8192; // <= 512 KB, within the SCP budget + const float * b_scp = (const float *) et_shire_l2scp_local(0); + + if (stage_b) { + if ((get_hart_id() & 63) == 0) { + et_tensor_load_l2scp_conf_t conf; + conf.use_tmask = false; + conf.stride = 64; // advance one 64B cache line per line loaded + uint64_t remaining = b_lines; + uint64_t dst_ln = 0; + uint64_t addr = (uint64_t) src1->data; + while (remaining > 0) { + uint64_t cl = (remaining >= 16) ? 16 : remaining; + conf.dst_start = dst_ln; + conf.addr = addr; + conf.num_lines = cl - 1; // 4-bit field encodes (lines - 1) + conf.id = 0; + et_tensor_load_l2scp(&conf); + WAIT_TENSOR_LOAD_L2_0; + dst_ln += cl; + addr += cl * 64; + remaining -= cl; } + } + et_barrier(ET_BARRIER_SHIRE); // B is now resident in L2 SCP for all harts + } - // Index decoding - const int64_t i3 = idx / (M * N * ne2); - const int64_t rem3 = idx % (M * N * ne2); - const int64_t i2 = rem3 / (M * N); - const int64_t rem2 = rem3 % (M * N); - const int64_t n = rem2 / M; - const int64_t m = rem2 % M; - - const int64_t i03 = i3 / r3, i02 = i2 / r2; - const int64_t i13 = (ne13 > 1) ? i3 : 0, i12 = (ne12 > 1) ? i2 : 0; - - float sum = 0.0f; - const float * f32_row = (const float *) ((const char *) src0_data + m * nb01 + i02 * nb02 + i03 * nb03); - - if (src1->type == GGML_TYPE_F32) { - const float * src1_data = (const float *) src1->data; - - for (int64_t kb = 0; kb < K_blocks; kb++) { - const float * b_col_ptr = - (const float *) ((const char *) src1_data + (kb * block_size) * sizeof(float) + n * nb11 + - i12 * nb12 + i13 * nb13); - sum += compute_block_dot_product_f32(&f32_row[kb * block_size], b_col_ptr); - } - - if (K_remainder > 0) { - const int64_t offset = K_blocks * block_size; - const float * b_col_ptr = (const float *) ((const char *) src1_data + offset * sizeof(float) + - n * nb11 + i12 * nb12 + i13 * nb13); - sum += compute_block_dot_product_f32_partial(&f32_row[offset], b_col_ptr, K_remainder); - } - } else { - const uint16_t * src1_data = (const uint16_t *) src1->data; - - for (int64_t kb = 0; kb < K_blocks; kb++) { - const uint16_t * b_col_ptr = - (const uint16_t *) ((const char *) src1_data + (kb * block_size) * sizeof(uint16_t) + n * nb11 + - i12 * nb12 + i13 * nb13); - sum += compute_block_dot_product_f32_f16_partial(&f32_row[kb * block_size], b_col_ptr, block_size); - } - - if (K_remainder > 0) { - const int64_t offset = K_blocks * block_size; - const uint16_t * b_col_ptr = - (const uint16_t *) ((const char *) src1_data + offset * sizeof(uint16_t) + n * nb11 + - i12 * nb12 + i13 * nb13); - sum += compute_block_dot_product_f32_f16_partial(&f32_row[offset], b_col_ptr, K_remainder); - } + // Set the vector mask (all 8 lanes) once for the whole row loop. + unsigned long saved_mask; + __asm__ volatile("mova.x.m %0" : "=r"(saved_mask)); + __asm__ volatile("mov.m.x m0, x0, 0xFF"); + + for (uint64_t idx = (uint64_t) thread_id; idx < total_elements; idx += (uint64_t) num_threads) { + // Index decoding + const int64_t i3 = idx / (M * N * ne2); + const int64_t rem3 = idx % (M * N * ne2); + const int64_t i2 = rem3 / (M * N); + const int64_t rem2 = rem3 % (M * N); + const int64_t n = rem2 / M; + const int64_t m = rem2 % M; + + const int64_t i03 = i3 / r3, i02 = i2 / r2; + const int64_t i13 = (ne13 > 1) ? i3 : 0, i12 = (ne12 > 1) ? i2 : 0; + + const float * f32_row = + (const float *) ((const char *) src0_data + m * nb01 + i02 * nb02 + i03 * nb03); + + float sum; + if (is_f32_b) { + const float * b_col = stage_b + ? (b_scp + n * K) + : (const float *) ((const char *) src1->data + n * nb11 + i12 * nb12 + i13 * nb13); + + f32_dot_reset(); + f32_dot_tile(f32_row, b_col, K_blocks); + sum = f32_dot_reduce(); + + if (K_remainder > 0) { + const int64_t offset = K_blocks * block_size; + sum += compute_block_dot_product_f32_partial(&f32_row[offset], &b_col[offset], K_remainder); } - - // Atomic store for output - volatile float * c_element = - (volatile float *) ((char *) dst_data + m * dst->nb[0] + n * nb1 + i2 * nb2 + i3 * nb3); - atomic_store_f32(c_element, sum); + } else { + const uint16_t * b_col = + (const uint16_t *) ((const char *) src1->data + n * nb11 + i12 * nb12 + i13 * nb13); + sum = compute_block_dot_product_f32_f16_partial(f32_row, b_col, K); } + + volatile float * c_element = + (volatile float *) ((char *) dst_data + m * dst->nb[0] + n * nb1 + i2 * nb2 + i3 * nb3); + atomic_store_f32(c_element, sum); } + __asm__ volatile("mova.m.x %0" ::"r"(saved_mask)); return 0; } diff --git a/ggml/src/ggml-et/et-kernels/src/mul_mat_f32_matrix_engine.c b/ggml/src/ggml-et/et-kernels/src/mul_mat_f32_matrix_engine.c index b2b61d51967..c22501d455f 100644 --- a/ggml/src/ggml-et/et-kernels/src/mul_mat_f32_matrix_engine.c +++ b/ggml/src/ggml-et/et-kernels/src/mul_mat_f32_matrix_engine.c @@ -1,48 +1,138 @@ +#include +#include #include "ggml_tensor.h" #include "platform.h" #include "tensor.h" -#include -#include - /* - * F32 Matrix Multiply for ET-SoC-1 — TensorFMA32. + * High-performance F32 Matrix Multiply for ET-SoC-1 — TensorFMA32. + * Double-buffered producer/consumer implementation. * - * K-parallel + interleaved tiles + ring reduce. - * No batched-K yet (needs investigation on hang). - * This is the last known working version. + * Hart 1 (producer): Transposes weights into double-buffered L2 SCP. + * Hart 0 (consumer): Tensor engine compute (FMA, reduce, store). + * + * Two execution paths: + * * REUSE path (n_tiles >= 2): prepared weight tiles are reused across + * ru_n N-tiles to minimize DRAM reads. + * * ORIGINAL path (n_tiles == 1, GEMV): one output tile at a time, no reuse. */ #define NUM_COMPUTE_SHIRES 32 #define MINIONS_PER_SHIRE 32 -#define TILE_K 16 -#define TILE_M 16 -/* ── Tuning knobs ───────────────────────────────────────────────────── */ -#define TILE_N 16 +#define TILE_M 16 +#define TILE_N 16 +#define TILE_K 16 + +#ifndef REUSE_MAX +#define REUSE_MAX 32 +#endif +#ifndef KWIN +#define KWIN 16 // K-blocks per window +#endif + +#define MACHINE_SLOTS (NUM_COMPUTE_SHIRES * MINIONS_PER_SHIRE) // 1024 + #define CACHEOP_MAX 0 #define REP_RATE 0 -/* ─────────────────────────────────────────────────────────────────── */ +#define A_L1_START 0 // L1 SCP lines 0..15 for A (buffer 0) +#define A_L1_ALT 16 // L1 SCP lines 16..31 for A (buffer 1) + +#define SCP_PANEL_SIZE (TILE_K * TILE_M * (uint64_t)sizeof(float)) // 1024 bytes + +#define RU_BUF_BYTES (KWIN * SCP_PANEL_SIZE) +#define RU_CACHE_BYTES (2 * RU_BUF_BYTES) +#define RU_CSCRATCH_BYTES (REUSE_MAX * 16 * 64ULL) +#define SCP_READY_OFF (RU_CACHE_BYTES + RU_CSCRATCH_BYTES) +#define SCP_CONSUMED_OFF (SCP_READY_OFF + 64) +#define SCP_PER_MINION (SCP_CONSUMED_OFF + 64) + +static inline void __attribute__((always_inline)) +pack_b_f32_transpose(float *out, const char *src0_batch, + int64_t mb, int64_t kb, int64_t nb1_0) { + static const int32_t __attribute__((aligned(32))) scatter_idx[8] = { + 0, 64, 128, 192, 256, 320, 384, 448 + }; + + unsigned long old_mask; + __asm__ volatile( + "mova.x.m %[ms] \n\t" + "mov.m.x m0, x0, 0xFF \n\t" // all 8 lanes active + "flw.ps f1, (%[idx]) \n\t" // f1 = scatter offsets + : [ms] "=&r"(old_mask) + : [idx] "r"(scatter_idx) + : "f1" + ); + + char *pbase = (char *) out; + for (int j = 0; j < TILE_M; ++j) { + const float *row = (const float *)(src0_batch + (mb + j) * nb1_0) + kb; + char *col = pbase + j * 4; // column m=j of the panel + + __asm__ volatile( + "flw.ps f2, 0(%[src]) \n\t" // load row[0..7] (32 bytes) + "flw.ps f3, 32(%[src]) \n\t" // load row[8..15] (32 bytes) + "fscw.ps f2, f1(%[c0]) \n\t" // store row[0..7] to lines 0..7, col j + "fscw.ps f3, f1(%[c8]) \n\t" // store row[8..15] to lines 8..15, col j + : + : [src] "r"(row), [c0] "r"(col), [c8] "r"(col + 8 * 64) + : "f2", "f3", "memory" + ); + } + + __asm__ volatile("mova.m.x %0" :: "r"(old_mask)); +} + +#define C_ROW_PAIR_ST(n0, n1, base) \ + __asm__ volatile("fsw.ps f" #n0 ", (%0)\n\t fsw.ps f" #n1 ", (%1)\n\t" \ + :: "r"((base)), "r"((base) + 32) : "memory") +#define C_ROW_PAIR_LD(n0, n1, base) \ + __asm__ volatile("flw.ps f" #n0 ", (%0)\n\t flw.ps f" #n1 ", (%1)\n\t" \ + :: "r"((base)), "r"((base) + 32) : "f" #n0, "f" #n1) + +static inline void __attribute__((always_inline)) +c_spill(char *s) { + C_ROW_PAIR_ST(0, 1, s + 0 * 64); C_ROW_PAIR_ST(2, 3, s + 1 * 64); + C_ROW_PAIR_ST(4, 5, s + 2 * 64); C_ROW_PAIR_ST(6, 7, s + 3 * 64); + C_ROW_PAIR_ST(8, 9, s + 4 * 64); C_ROW_PAIR_ST(10, 11, s + 5 * 64); + C_ROW_PAIR_ST(12, 13, s + 6 * 64); C_ROW_PAIR_ST(14, 15, s + 7 * 64); + C_ROW_PAIR_ST(16, 17, s + 8 * 64); C_ROW_PAIR_ST(18, 19, s + 9 * 64); + C_ROW_PAIR_ST(20, 21, s + 10 * 64); C_ROW_PAIR_ST(22, 23, s + 11 * 64); + C_ROW_PAIR_ST(24, 25, s + 12 * 64); C_ROW_PAIR_ST(26, 27, s + 13 * 64); + C_ROW_PAIR_ST(28, 29, s + 14 * 64); C_ROW_PAIR_ST(30, 31, s + 15 * 64); +} + +static inline void __attribute__((always_inline)) +c_seed(char *s) { + C_ROW_PAIR_LD(0, 1, s + 0 * 64); C_ROW_PAIR_LD(2, 3, s + 1 * 64); + C_ROW_PAIR_LD(4, 5, s + 2 * 64); C_ROW_PAIR_LD(6, 7, s + 3 * 64); + C_ROW_PAIR_LD(8, 9, s + 4 * 64); C_ROW_PAIR_LD(10, 11, s + 5 * 64); + C_ROW_PAIR_LD(12, 13, s + 6 * 64); C_ROW_PAIR_LD(14, 15, s + 7 * 64); + C_ROW_PAIR_LD(16, 17, s + 8 * 64); C_ROW_PAIR_LD(18, 19, s + 9 * 64); + C_ROW_PAIR_LD(20, 21, s + 10 * 64); C_ROW_PAIR_LD(22, 23, s + 11 * 64); + C_ROW_PAIR_LD(24, 25, s + 12 * 64); C_ROW_PAIR_LD(26, 27, s + 13 * 64); + C_ROW_PAIR_LD(28, 29, s + 14 * 64); C_ROW_PAIR_LD(30, 31, s + 15 * 64); +} -int entry_point(struct ggml_et_binary_params * params, void * env) { - uint64_t hart_id = get_hart_id(); +int entry_point(struct ggml_et_binary_params* params, void* env) { + (void) env; + + uint64_t hart_id = get_hart_id(); uint64_t shire_id = get_shire_id(); - if (shire_id >= NUM_COMPUTE_SHIRES) { - return 0; - } - if (hart_id & 1) { - return 0; - } + if (shire_id >= NUM_COMPUTE_SHIRES) return 0; + const int is_hart1 = hart_id & 1; uint64_t local_minion = (hart_id >> 1) & 0x1F; - uint64_t my_minion_id = get_minion_id(); const int64_t K = params->src0.ne[0]; const int64_t M = params->src0.ne[1]; const int64_t N = params->src1.ne[1]; + if ((M % TILE_M) != 0) return 0; + if ((K % TILE_K) != 0) return 0; + const int64_t ne2_0 = params->src0.ne[2], ne3_0 = params->src0.ne[3]; const int64_t ne2_1 = params->src1.ne[2], ne3_1 = params->src1.ne[3]; @@ -51,33 +141,42 @@ int entry_point(struct ggml_et_binary_params * params, void * env) { const int64_t nb1_1 = params->src1.nb[1]; const int64_t nb2_1 = params->src1.nb[2], nb3_1 = params->src1.nb[3]; const int64_t nb1_d = params->dst.nb[1]; - const int64_t nb2_d = params->dst.nb[2], nb3_d = params->dst.nb[3]; - - const char * src0_base = (const char *) params->src0.data; - const char * src1_base = (const char *) params->src1.data; - char * dst_base = (char *) params->dst.data; + const int64_t nb2_d = params->dst.nb[2], nb3_d = params->dst.nb[3]; - setup_cache_scp(); -#if CACHEOP_MAX > 0 || REP_RATE > 0 - ucache_control(1, REP_RATE, CACHEOP_MAX); -#endif - CLEAR_TENSOR_ERROR; + const char* src0_base = (const char*)params->src0.data; + const char* src1_base = (const char*)params->src1.data; + char* dst_base = (char*)params->dst.data; - const int64_t m_tiles = M / TILE_M; - const int64_t n_tiles = (N + TILE_N - 1) / TILE_N; + const int64_t m_tiles = M / TILE_M; + const int64_t n_tiles = (N + TILE_N - 1) / TILE_N; const int64_t batch_count = ne2_1 * ne3_1; - const int64_t base_tiles = m_tiles * n_tiles * batch_count; const int64_t r2 = ne2_1 / ne2_0; const int64_t r3 = ne3_1 / ne3_0; - const int64_t total_harts = NUM_COMPUTE_SHIRES * MINIONS_PER_SHIRE; - const int64_t k_steps = K / TILE_K; - int64_t k_splits = 1; - if (base_tiles < total_harts) { - k_splits = (total_harts + base_tiles - 1) / base_tiles; + const int64_t k_steps = K / TILE_K; + + // L2 SCP pointers for this minion + const uint64_t scp_base = local_minion * SCP_PER_MINION; + volatile uint32_t *ready_ctr = + (volatile uint32_t *) et_shire_l2scp_local(scp_base + SCP_READY_OFF); + volatile uint32_t *consumed_ctr = + (volatile uint32_t *) et_shire_l2scp_local(scp_base + SCP_CONSUMED_OFF); + + int64_t n_groups = (n_tiles + REUSE_MAX - 1) / REUSE_MAX; + if (n_groups < 1) n_groups = 1; + int64_t ru_n = (n_tiles + n_groups - 1) / n_groups; + if (ru_n < 1) ru_n = 1; + + const int64_t units_pb = m_tiles * n_groups; + const int64_t base_units = units_pb * batch_count; + + int64_t k_splits = 1; + { int64_t ks = 1; - while (ks * 2 <= k_splits && ks * 2 <= 32 && k_steps % (ks * 2) == 0) { + while (ks * 2 <= MINIONS_PER_SHIRE && + base_units * ks * 2 <= MACHINE_SLOTS && + (k_steps % (ks * 2)) == 0) { ks *= 2; } k_splits = ks; @@ -87,49 +186,357 @@ int entry_point(struct ggml_et_binary_params * params, void * env) { const int64_t k_split = local_minion % k_splits; const int64_t local_tile_idx = local_minion / k_splits; const int64_t tiles_stride = (int64_t) NUM_COMPUTE_SHIRES * tiles_per_shire; + const int64_t my_start = (int64_t) shire_id + local_tile_idx * NUM_COMPUTE_SHIRES; const int64_t k_steps_per_split = k_steps / k_splits; - const int64_t k_start = k_split * k_steps_per_split * TILE_K; + const int64_t k_start_block = k_split * k_steps_per_split; + const int64_t k_start = k_start_block * TILE_K; const int64_t k_end = k_start + k_steps_per_split * TILE_K; - const uint64_t group_base_global = my_minion_id - k_split; + const int reuse_ok = (ru_n >= 2); + + // ===================================================================== + // REUSE path: transpose each K-window once, reuse across ru_n N-tiles. + // ===================================================================== + if (reuse_ok) { + char *cache_buf[2] = { + (char *) et_shire_l2scp_local(scp_base), + (char *) et_shire_l2scp_local(scp_base + RU_BUF_BYTES), + }; + char *cscratch = (char *) et_shire_l2scp_local(scp_base + RU_CACHE_BYTES); + + const int64_t k_end_block = k_start_block + k_steps_per_split; + const int64_t n_windows = (k_steps_per_split + KWIN - 1) / KWIN; + + // ----- Hart 1: producer ----- + if (is_hart1) { + scp_signal(ready_ctr, 0); + scp_signal(consumed_ctr, 0); + + et_barrier(ET_BARRIER_MINION); + uint32_t wid = 0; + + for (int64_t unit = my_start; unit < base_units; unit += tiles_stride) { + const int64_t batch_idx = unit / units_pb; + const int64_t unit_in_b = unit % units_pb; + const int64_t mb_idx = unit_in_b % m_tiles; + + const int64_t i3 = batch_idx / ne2_1; + const int64_t i2 = batch_idx % ne2_1; + const int64_t i2_0 = i2 / r2; + const int64_t i3_0 = i3 / r3; + + const char *src0_batch = src0_base + i3_0 * nb3_0 + i2_0 * nb2_0; + const int64_t mb = mb_idx * TILE_M; + + // Prefetch first window's weights + { + const int64_t kbn_first = (k_start_block + KWIN <= k_end_block) ? KWIN : (k_end_block - k_start_block); + for (int64_t i = 0; i < kbn_first; ++i) { + l2_prefetch(src0_batch + mb * nb1_0 + (k_start_block + i) * TILE_K * sizeof(float), 16, nb1_0); + } + } + + for (int64_t kw = 0; kw < n_windows; ++kw) { + const int buf = wid & 1; + if (wid >= 2) scp_wait(consumed_ctr, wid - 1); + + const int64_t kb0 = k_start_block + kw * KWIN; + const int64_t kbn = (kb0 + KWIN <= k_end_block) ? KWIN : (k_end_block - kb0); + + // Prefetch next window's weights + if (kw + 1 < n_windows) { + const int64_t kb_next = kb0 + KWIN; + const int64_t kbn_next = (kb_next + KWIN <= k_end_block) ? KWIN : (k_end_block - kb_next); + for (int64_t i = 0; i < kbn_next; ++i) { + l2_prefetch(src0_batch + mb * nb1_0 + (kb_next + i) * TILE_K * sizeof(float), 16, nb1_0); + } + } + + float *cf = (float *) cache_buf[buf]; + for (int64_t i = 0; i < kbn; ++i) { + pack_b_f32_transpose(cf + i * (SCP_PANEL_SIZE / sizeof(float)), + src0_batch, mb, (kb0 + i) * TILE_K, nb1_0); + } + FENCE; + flush_to_l2_multi(cache_buf[buf], kbn * TILE_K, 64); + WAIT_CACHEOPS; + + wid++; + scp_signal(ready_ctr, wid); + } + } + FENCE; + return 0; + } - for (int64_t tile = (int64_t) shire_id + local_tile_idx * NUM_COMPUTE_SHIRES; tile < base_tiles; - tile += tiles_stride) { + // ----- Hart 0: consumer ----- + setup_cache_scp(); +#if CACHEOP_MAX > 0 || REP_RATE > 0 + ucache_control(1, REP_RATE, CACHEOP_MAX); +#endif + CLEAR_TENSOR_ERROR; + + et_barrier(ET_BARRIER_MINION); + evict_to_l2((const void *)(uintptr_t) ready_ctr, 1, 64); WAIT_CACHEOPS; + evict_to_l2((const void *)(uintptr_t) consumed_ctr, 1, 64); WAIT_CACHEOPS; + + const uint64_t group_base_global = get_minion_id() - (uint64_t) k_split; + + uint32_t wid = 0; + for (int64_t unit = my_start; unit < base_units; unit += tiles_stride) { + const int64_t batch_idx = unit / units_pb; + const int64_t unit_in_b = unit % units_pb; + const int64_t g_idx = unit_in_b / m_tiles; + const int64_t mb_idx = unit_in_b % m_tiles; + + const int64_t i3 = batch_idx / ne2_1; + const int64_t i2 = batch_idx % ne2_1; + + const char *src1_batch = src1_base + i3 * nb3_1 + i2 * nb2_1; + char *dst_batch = dst_base + i3 * nb3_d + i2 * nb2_d; + + const int64_t mb = mb_idx * TILE_M; + const int64_t nb_base_t = g_idx * ru_n; + int64_t r_count = n_tiles - nb_base_t; + if (r_count > ru_n) r_count = ru_n; + + for (int64_t kw = 0; kw < n_windows; ++kw) { + const int buf = wid & 1; + wid++; + scp_wait(ready_ctr, wid); + + const int64_t kb0 = k_start_block + kw * KWIN; + const int64_t kbn = (kb0 + KWIN <= k_end_block) ? KWIN : (k_end_block - kb0); + const int is_last = (kw == n_windows - 1); + float *cf = (float *) cache_buf[buf]; + + for (int64_t r = 0; r < r_count; ++r) { + const int64_t nb = (nb_base_t + r) * TILE_N; + const int64_t n_cur = (nb + TILE_N <= N) ? TILE_N : (N - nb); + const int64_t arows_fma = (n_cur == 4) ? 4 : (n_cur - 1); + + char *cs = cscratch + r * (16 * 64); + + int first; + if (kw == 0) { + first = 1; + } else { + c_seed(cs); + first = 0; + } + + const int64_t nsteps = kbn; +#define A_TILE_ADDR(st) \ + ((uint64_t)(src1_batch + nb * nb1_1 + \ + (kb0 + (st)) * TILE_K * (int64_t) sizeof(float))) + + // prologue: prefetch step 0 and 1, and load step 0 + l2_prefetch((const void *) A_TILE_ADDR(0), n_cur, nb1_1); + if (1 < nsteps) { + l2_prefetch((const void *) A_TILE_ADDR(1), n_cur, nb1_1); + } + tensor_load(false, false, A_L1_START, TENSOR_LOAD_PLAIN, 0, + A_TILE_ADDR(0), 0, n_cur - 1, (uint64_t) nb1_1, 0); + + for (int64_t s = 0; s < nsteps; ++s) { + const uint64_t a_cur = (s & 1) ? A_L1_ALT : A_L1_START; + + tensor_wait(TENSOR_LOAD_WAIT_0); + + if (s + 1 < nsteps) { + const uint64_t a_nxt = ((s + 1) & 1) ? A_L1_ALT : A_L1_START; + tensor_load(false, false, a_nxt, TENSOR_LOAD_PLAIN, 0, + A_TILE_ADDR(s + 1), 0, n_cur - 1, (uint64_t) nb1_1, 0); + } + + if (s + 2 < nsteps) { + l2_prefetch((const void *) A_TILE_ADDR(s + 2), n_cur, nb1_1); + } + + tensor_load_setup_b( + false, + (uint64_t)(cf + s * (SCP_PANEL_SIZE / sizeof(float))), + TILE_K - 1, 64, 1); + + tensor_wait(TENSOR_LOAD_WAIT_1); + + tensor_fma( + false, 3, arows_fma, TILE_K - 1, 0, + false, false, false, true, + 16, a_cur, TENSOR_FMA_OP_FP32, first); + tensor_wait(TENSOR_FMA_WAIT); + first = 0; + } +#undef A_TILE_ADDR + + if (is_last) { + if (k_splits > 1) { + const uint64_t num_regs = (uint64_t) n_cur * 2; + if (k_split > 0) { + tensor_reduce_recv(0, TENSOR_REDUCE_OP_FADD, num_regs, + group_base_global + (uint64_t)(k_split - 1)); + tensor_wait(TENSOR_REDUCE_WAIT); + } + if (k_split < k_splits - 1) { + tensor_reduce_send(0, num_regs, + group_base_global + (uint64_t)(k_split + 1)); + tensor_wait(TENSOR_REDUCE_WAIT); + } + } + if (k_split == k_splits - 1) { + tensor_store( + 0, 0, 3, n_cur - 1, + (uint64_t)(dst_batch + nb * nb1_d + mb * (int64_t) sizeof(float)), + 0, (uint64_t) nb1_d); + tensor_wait(TENSOR_STORE_WAIT); + } + } else { + c_spill(cs); + } + } + scp_signal(consumed_ctr, wid); + } + } + FENCE; + return 0; + } + + // ===================================================================== + // ORIGINAL path: one output tile at a time (no reuse). + // ===================================================================== + const int64_t base_tiles = m_tiles * n_tiles * batch_count; + float *scp_panel[2] = { + (float *) et_shire_l2scp_local(scp_base), + (float *) et_shire_l2scp_local(scp_base + SCP_PANEL_SIZE), + }; + + if (is_hart1) { + scp_signal(ready_ctr, 0); + scp_signal(consumed_ctr, 0); + + et_barrier(ET_BARRIER_MINION); + uint32_t chunk_id = 0; + + for (int64_t tile = my_start; tile < base_tiles; tile += tiles_stride) { + const int64_t tiles_per_batch = m_tiles * n_tiles; + const int64_t batch_idx = tile / tiles_per_batch; + const int64_t tile_in_batch = tile % tiles_per_batch; + const int64_t mb_idx = tile_in_batch % m_tiles; + + const int64_t i3 = batch_idx / ne2_1; + const int64_t i2 = batch_idx % ne2_1; + const int64_t i2_0 = i2 / r2; + const int64_t i3_0 = i3 / r3; + + const char *src0_batch = src0_base + i3_0 * nb3_0 + i2_0 * nb2_0; + const int64_t mb = mb_idx * TILE_M; + + // Prefetch first block + l2_prefetch(src0_batch + mb * nb1_0 + k_start * sizeof(float), 16, nb1_0); + + for (int64_t kb = k_start; kb < k_end; kb += TILE_K) { + int buf = chunk_id & 1; + if (chunk_id >= 2) scp_wait(consumed_ctr, chunk_id - 1); + + // Prefetch next block + if (kb + TILE_K < k_end) { + l2_prefetch(src0_batch + mb * nb1_0 + (kb + TILE_K) * sizeof(float), 16, nb1_0); + } + + pack_b_f32_transpose(scp_panel[buf], src0_batch, mb, kb, nb1_0); + + FENCE; + flush_to_l2(scp_panel[buf], 16, 64); + WAIT_CACHEOPS; + + chunk_id++; + scp_signal(ready_ctr, chunk_id); + } + } + FENCE; + return 0; + } + + setup_cache_scp(); +#if CACHEOP_MAX > 0 || REP_RATE > 0 + ucache_control(1, REP_RATE, CACHEOP_MAX); +#endif + CLEAR_TENSOR_ERROR; + + et_barrier(ET_BARRIER_MINION); + evict_to_l2((const void *)(uintptr_t) ready_ctr, 1, 64); WAIT_CACHEOPS; + evict_to_l2((const void *)(uintptr_t) consumed_ctr, 1, 64); WAIT_CACHEOPS; + + const uint64_t group_base_global = get_minion_id() - (uint64_t) k_split; + uint32_t chunk_id = 0; + + for (int64_t tile = my_start; tile < base_tiles; tile += tiles_stride) { const int64_t tiles_per_batch = m_tiles * n_tiles; const int64_t batch_idx = tile / tiles_per_batch; const int64_t tile_in_batch = tile % tiles_per_batch; const int64_t nb_idx = tile_in_batch / m_tiles; const int64_t mb_idx = tile_in_batch % m_tiles; - const int64_t i3 = batch_idx / ne2_1; - const int64_t i2 = batch_idx % ne2_1; - const int64_t i2_0 = i2 / r2; - const int64_t i3_0 = i3 / r3; + const int64_t i3 = batch_idx / ne2_1; + const int64_t i2 = batch_idx % ne2_1; - const char * src0_batch = src0_base + i3_0 * nb3_0 + i2_0 * nb2_0; - const char * src1_batch = src1_base + i3 * nb3_1 + i2 * nb2_1; - char * dst_batch = dst_base + i3 * nb3_d + i2 * nb2_d; + const char *src1_batch = src1_base + i3 * nb3_1 + i2 * nb2_1; + char *dst_batch = dst_base + i3 * nb3_d + i2 * nb2_d; - const int64_t mb = mb_idx * TILE_M; - const int64_t nb = nb_idx * TILE_N; + const int64_t mb = mb_idx * TILE_M; + const int64_t nb = nb_idx * TILE_N; const int64_t n_cur = (nb + TILE_N <= N) ? TILE_N : (N - nb); - for (int64_t kb = k_start; kb < k_end; kb += TILE_K) { - tensor_load(false, false, 0, 0, 0, (uint64_t) (src1_batch + nb * nb1_1 + kb * sizeof(float)), 0, n_cur - 1, - (uint64_t) nb1_1, 0); + int first = 1; + // Prologue: prefetch first two blocks, load first block + l2_prefetch((const void *)(src1_batch + nb * nb1_1 + k_start * sizeof(float)), n_cur, nb1_1); + if (k_start + TILE_K < k_end) { + l2_prefetch((const void *)(src1_batch + nb * nb1_1 + (k_start + TILE_K) * sizeof(float)), n_cur, nb1_1); + } + tensor_load( + false, false, A_L1_START, TENSOR_LOAD_PLAIN, 0, + (uint64_t)(src1_batch + nb * nb1_1 + k_start * (int64_t) sizeof(float)), + 0, n_cur - 1, (uint64_t) nb1_1, 0); - tensor_load(false, false, TILE_K, 7, 0, (uint64_t) (src0_batch + mb * nb1_0 + kb * sizeof(float)), 0, - TILE_K - 1, (uint64_t) nb1_0, 1); + for (int64_t kb = k_start; kb < k_end; kb += TILE_K) { + int buf = chunk_id & 1; + const uint64_t a_cur = ((kb - k_start) / TILE_K & 1) ? A_L1_ALT : A_L1_START; tensor_wait(TENSOR_LOAD_WAIT_0); - tensor_wait(TENSOR_LOAD_WAIT_1); - tensor_fma(false, 3, n_cur - 1, TILE_K - 1, 0, false, false, false, false, TILE_K, 0, 0, (kb == k_start)); + if (kb + TILE_K < k_end) { + const uint64_t a_nxt = (((kb + TILE_K) - k_start) / TILE_K & 1) ? A_L1_ALT : A_L1_START; + tensor_load( + false, false, a_nxt, TENSOR_LOAD_PLAIN, 0, + (uint64_t)(src1_batch + nb * nb1_1 + (kb + TILE_K) * (int64_t) sizeof(float)), + 0, n_cur - 1, (uint64_t) nb1_1, 0); + } + + if (kb + 2 * TILE_K < k_end) { + l2_prefetch((const void *)(src1_batch + nb * nb1_1 + (kb + 2 * TILE_K) * sizeof(float)), n_cur, nb1_1); + } + + chunk_id++; + scp_wait(ready_ctr, chunk_id); + tensor_load_setup_b( + false, + (uint64_t) scp_panel[buf], + 15, 64, 1); + tensor_wait(TENSOR_LOAD_WAIT_1); + + tensor_fma( + false, 3, n_cur - 1, TILE_K - 1, 0, + false, false, false, true, + 16, a_cur, TENSOR_FMA_OP_FP32, first); tensor_wait(TENSOR_FMA_WAIT); + first = 0; + + scp_signal(consumed_ctr, chunk_id); } + // K-split ring reduce if (k_splits > 1) { const uint64_t num_regs = (uint64_t) n_cur * 2; @@ -137,6 +544,7 @@ int entry_point(struct ggml_et_binary_params * params, void * env) { tensor_reduce_recv(0, TENSOR_REDUCE_OP_FADD, num_regs, group_base_global + k_split - 1); tensor_wait(TENSOR_REDUCE_WAIT); } + if (k_split < k_splits - 1) { tensor_reduce_send(0, num_regs, group_base_global + k_split + 1); tensor_wait(TENSOR_REDUCE_WAIT); @@ -144,8 +552,10 @@ int entry_point(struct ggml_et_binary_params * params, void * env) { } if (k_split == k_splits - 1) { - tensor_store(0, 0, 3, n_cur - 1, (uint64_t) (dst_batch + nb * nb1_d + mb * sizeof(float)), 0, - (uint64_t) nb1_d); + tensor_store( + 0, 0, 3, n_cur - 1, + (uint64_t)(dst_batch + nb * nb1_d + mb * (int64_t) sizeof(float)), + 0, (uint64_t) nb1_d); tensor_wait(TENSOR_STORE_WAIT); } } diff --git a/ggml/src/ggml-et/et-kernels/src/platform.h b/ggml/src/ggml-et/et-kernels/src/platform.h index cbec4c98d74..b014b4e6120 100644 --- a/ggml/src/ggml-et/et-kernels/src/platform.h +++ b/ggml/src/ggml-et/et-kernels/src/platform.h @@ -476,6 +476,20 @@ static inline void __attribute__((always_inline)) flush_to_l2(const void * addr, : "x31", "memory"); } +// Flush an arbitrary number of lines to L2, working around the 16-line cap of a +// single FlushVA by issuing multiple flushes. Use this whenever nlines may exceed 16. +static inline void __attribute__((always_inline)) flush_to_l2_multi(const void * addr, uint64_t nlines, uint64_t stride) { + const char * p = (const char *) addr; + while (nlines > 16) { + flush_to_l2(p, 16, stride); + p += 16 * stride; + nlines -= 16; + } + if (nlines) { + flush_to_l2(p, nlines, stride); + } +} + // Evict nlines cache lines at stride apart starting at addr from L1 to L2. // Uses EvictVA (CSR 0x89F). Unlike flush_to_l2, this guarantees the line is // NOT present in L1 after the operation - subsequent loads will miss and go @@ -542,4 +556,28 @@ static void evict_region_past_l2(const void * addr, size_t bytes) { } } + +//****************************************************************************** +// Counter signaling between harts via L2 scratchpad (SCP) +//****************************************************************************** + +// Signal a counter value to the other hart via L2 SCP. +static inline void __attribute__((always_inline)) +scp_signal(volatile uint32_t *flag, uint32_t value) { + *flag = value; + FENCE; + evict_to_l2((const void *)flag, 1, 64); + WAIT_CACHEOPS; +} + +// Wait for a counter in L2 SCP to reach the expected value. +static inline void __attribute__((always_inline)) +scp_wait(volatile uint32_t *flag, uint32_t expected) { + while (1) { + evict_to_l2((const void *)flag, 1, 64); + WAIT_CACHEOPS; + if (*flag >= expected) return; + } +} + #endif // PLATFORM_H diff --git a/ggml/src/ggml-et/ggml-et-ops.cpp b/ggml/src/ggml-et/ggml-et-ops.cpp index 6c80fe8acde..901dde7a12a 100644 --- a/ggml/src/ggml-et/ggml-et-ops.cpp +++ b/ggml/src/ggml-et/ggml-et-ops.cpp @@ -749,13 +749,13 @@ bool ggml_et_op_mul_mat(ggml_backend_et_device_context * dev_ctx, } else if (node->type == GGML_TYPE_F32 && node->src[0]->type == GGML_TYPE_F32 && node->src[1]->type == GGML_TYPE_F32 && node->ne[0] % 16 == 0 && node->src[0]->ne[0] % 16 == 0 && - node->src[0]->ne[1] % 16 == 0 && node->src[1]->ne[0] != 1) { // GEMV is faster with the generic path + node->src[0]->ne[1] % 16 == 0 && node->src[1]->ne[1] >= 7) { // N < 7: vec_dot faster (measured) kernel_name = "mul_mat_f32_matrix_engine"; src0_type_name = "F32"; } else if (node->type == GGML_TYPE_F32 && node->src[0]->type == GGML_TYPE_F32 && (node->src[1]->type == GGML_TYPE_F16 || node->src[1]->type == GGML_TYPE_F32)) { - kernel_name = "mul_mat_f32"; + kernel_name = "mul_mat_f32"; // N < 7, or shape doesn't fit the matrix-engine tiling src0_type_name = "F32"; } else { GGML_LOG_ERROR("ET: MUL_MAT operation with unsupported types: dst=%s src0=%s src1=%s\n",