Skip to content

et-backend: F32 vecdot GEMV + matrix-engine GEMM #29

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
RehanQasim-dev wants to merge 5 commits into
base: et
Choose a base branch
from
Draft

et-backend: F32 vecdot GEMV + matrix-engine GEMM #29

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
b651214
ggml-et: centralize scp_signal/scp_wait in platform.h
RehanQasim-dev Jul 30, 2026
4f94811
et-backend: F32 GEMV rewrite for register-resident vecdot
RehanQasim-dev Jul 14, 2026
21c34f9
et-backend: F32 matrix-engine GEMM kernel for prefill
RehanQasim-dev Jul 23, 2026
2a6aec8
ET mul_mat: tune F32xF32 matrix-engine threshold (N>2 -> N>=7)
RehanQasim-dev Jul 30, 2026
e84d0da
ggml-et: fix stale threshold comment on F32 vecdot fallback
RehanQasim-dev Jul 30, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
93 changes: 93 additions & 0 deletions ggml/src/ggml-et/et-kernels/src/block_ops.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
# define BLOCK_OPS_H

# include "math_fp.h"
# include "platform.h"
# include "quants.h"

# include <stdint.h>
Expand Down Expand Up @@ -735,6 +736,98 @@ static inline float compute_block_dot_product_f32_f16_partial(const float * a
return sum;
}

//******************************************************************************
// F32 x F32 register-resident row dot (decode GEMV). The running sum lives in
// vector registers f20..f23 for the whole row instead of being spilled to
// memory every few elements, so the inner loop is just paired flw.ps loads and
// fmadd, no gather or convert needed since both operands are already f32.
// Four independent lane accumulators break the fmadd dependency chain so
// several A/B loads stay in flight, hiding load latency.
//******************************************************************************
static inline void __attribute__((always_inline))
f32_dot_reset(void) {
__asm__ volatile(
"fbci.pi f20, 0\n"
"fbci.pi f21, 0\n"
"fbci.pi f22, 0\n"
"fbci.pi f23, 0\n"
::: "f20", "f21", "f22", "f23");
}

// Accumulate n_blocks blocks of 32 f32 A values times 32 f32 B into f20..f23.
static inline void __attribute__((always_inline))
f32_dot_tile(const float * a_row, const float * b_col, int64_t n_blocks) {
// Prefetch A ahead into L2, not L1. The weight row is streamed once and
// never reused, but L1 is shared by both harts of a minion, so a simple
// L1 prefetch gets evicted before the load consumes it and the line is
// re-fetched from DRAM. l2_prefetch stages the line into L2 instead, which
// is large enough to hold the in-flight window, so a re-touch hits L2
// rather than causing a second DRAM read. B lives in on-chip L2 SCP
// already, so it needs no prefetch.
//
// Each block is 128B = 2 lines. Prefetch 16 lines (8 blocks) at a time,
// staying PF_AHEAD blocks in front of the consuming load.
const int64_t PF_AHEAD = 16;
for (int64_t kb = 0; kb < n_blocks; kb++) {
const float * a_ptr = a_row + (kb << 5); // 32 f32 per block = 128B
const float * b_ptr = b_col + (kb << 5);
if ((kb & 7) == 0) {
const int64_t pf_block = kb + PF_AHEAD;
if (pf_block + 8 <= n_blocks) {
l2_prefetch(a_row + (pf_block << 5), 16, 64); // 16 lines = 8 blocks
}
}
__asm__ volatile(
// Issue all 8 independent loads first (4 A + 4 B) so several misses
// are outstanding before any consumer stalls in-order issue.
"flw.ps f11, %[a0]\n"
"flw.ps f13, %[a1]\n"
"flw.ps f15, %[a2]\n"
"flw.ps f17, %[a3]\n"
"flw.ps f12, %[b0]\n"
"flw.ps f14, %[b1]\n"
"flw.ps f16, %[b2]\n"
"flw.ps f18, %[b3]\n"
"fmadd.ps f20, f11, f12, f20\n"
"fmadd.ps f21, f13, f14, f21\n"
"fmadd.ps f22, f15, f16, f22\n"
"fmadd.ps f23, f17, f18, f23\n"
:
: [a0] "m"(*(const float(*)[8])&a_ptr[0]),
[a1] "m"(*(const float(*)[8])&a_ptr[8]),
[a2] "m"(*(const float(*)[8])&a_ptr[16]),
[a3] "m"(*(const float(*)[8])&a_ptr[24]),
[b0] "m"(*(const float(*)[8])&b_ptr[0]),
[b1] "m"(*(const float(*)[8])&b_ptr[8]),
[b2] "m"(*(const float(*)[8])&b_ptr[16]),
[b3] "m"(*(const float(*)[8])&b_ptr[24])
: "f11", "f12", "f13", "f14", "f15", "f16", "f17", "f18",
"f20", "f21", "f22", "f23"
);
}
}

static inline float __attribute__((always_inline))
f32_dot_reduce(void) {
float result;
__asm__ __volatile__ (
// Combine the 4 lane accumulators, then horizontal-sum the 8 lanes.
"fadd.ps f20, f20, f21, rne \n\t"
"fadd.ps f22, f22, f23, rne \n\t"
"fadd.ps f20, f20, f22, rne \n\t"
"fswizz.ps f1, f20, 0xB1 \n\t"
"fadd.ps f2, f20, f1, rne \n\t"
"fswizz.ps f3, f2, 0x4E \n\t"
"fadd.ps f4, f2, f3, rne \n\t"
"fmvz.x.ps t0, f4, 4 \n\t"
"fbcx.ps f5, t0 \n\t"
"fadd.ps %[vout], f4, f5, rne \n\t"
: [vout] "=f" (result)
:: "t0", "f1", "f2", "f3", "f4", "f5", "f20", "f21", "f22", "f23"
);
return result;
}

// Compute dot product between f32 block and f32 column vector
// Vectorized: processes 8 elements at a time using ET vector instructions
// Block size: 16 f32 values (64 bytes = 1 cache line)
Expand Down
19 changes: 0 additions & 19 deletions ggml/src/ggml-et/et-kernels/src/mul_mat_Q4_0_matrix_engine.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,25 +33,6 @@
#define SCP_CONSUMED_OFF (SCP_READY_OFF + 64) // 4160
#define SCP_PER_MINION (SCP_CONSUMED_OFF + 64) // 4224

// Signal a counter value to the other hart via L2 SCP.
static inline void __attribute__((always_inline)) scp_signal(volatile uint32_t * flag, uint32_t value) {
*flag = value;
FENCE;
evict_to_l2((const void *) flag, 1, 64);
WAIT_CACHEOPS;
}

// Wait for a counter in L2 SCP to reach the expected value.
static inline void __attribute__((always_inline)) scp_wait(volatile uint32_t * flag, uint32_t expected) {
while (1) {
evict_to_l2((const void *) flag, 1, 64);
WAIT_CACHEOPS;
if (*flag >= expected) {
return;
}
}
}

// Dequantize one 32-element Q4_0 block of TILE_M weight rows into the FP32
// panel, written directly in TenB [k][m] order: panel[k*TILE_M + m].
// Low nibble of byte i -> k = i
Expand Down
19 changes: 0 additions & 19 deletions ggml/src/ggml-et/et-kernels/src/mul_mat_f16_matrix_engine.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,25 +38,6 @@ typedef uint16_t et_fp16_t;
#define SCP_CONSUMED_OFF (SCP_READY_OFF + 64) // 2112
#define SCP_PER_MINION (SCP_CONSUMED_OFF + 64) // 2176

// Signal a counter value to the other hart via L2 SCP.
static inline void __attribute__((always_inline)) scp_signal(volatile uint32_t * flag, uint32_t value) {
*flag = value;
FENCE;
evict_to_l2((const void *) flag, 1, 64);
WAIT_CACHEOPS;
}

// Wait for a counter in L2 SCP to reach the expected value.
static inline void __attribute__((always_inline)) scp_wait(volatile uint32_t * flag, uint32_t expected) {
while (1) {
evict_to_l2((const void *) flag, 1, 64);
WAIT_CACHEOPS;
if (*flag >= expected) {
return;
}
}
}

/**
* Build the interleaved B panel that TensorFMA16A32 expects (vectorized).
*
Expand Down
180 changes: 101 additions & 79 deletions ggml/src/ggml-et/et-kernels/src/mul_mat_f32.c
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,11 +1,23 @@
//******************************************************************************
// MUL_MAT Kernel (F32 weights)
// Matrix multiplication: C[M,N] = A[M,K] * B[K,N]
//
// Decode-optimized distribution: one output element (row m of A dotted with
// column n of B) is the unit of work, striped by thread_id across every hart
// of every active shire. This keeps all 32 shires busy for a GEMV
// (M=4096, N=1) instead of leaving most idle. The reused B vector is staged
// into per-shire L2 SCP and the accumulator stays register-resident across K
// via f32_dot_*.
//******************************************************************************

#include "block_ops.h"
#include "ggml_tensor.h"
#include "math_fp.h"
#include "platform.h"
#include "quants.h"
#include "tensor.h"

#include <etsoc/common/utils.h>
#include <stdint.h>
#include <stdio.h>

int entry_point(struct ggml_et_binary_params * params, void * env) {
kernel_environment_t * kernel_env = (kernel_environment_t *) env;
Expand All @@ -14,18 +26,13 @@ int entry_point(struct ggml_et_binary_params * params, void * env) {
return -1;
}

// Thread coordination
// Thread coordination, use every hart of every active shire.
int thread_id = get_relative_thread_id(kernel_env->shire_mask);
int num_threads = get_num_threads(kernel_env->shire_mask);

if (thread_id < 0 || (thread_id & 1)) {
return 0; // Skip odd threads to avoid resource contention
if (thread_id < 0) {
return 0;
}

int effective_thread_id = thread_id / 2;
int effective_num_threads = (num_threads + 1) / 2;

// Extract tensor references
struct ggml_tensor * src0 = &params->src0; // Weight matrix A (F32)
struct ggml_tensor * src1 = &params->src1; // Activation matrix B (F16/F32)
struct ggml_tensor * dst = &params->dst; // Output matrix C (F32)
Expand All @@ -39,7 +46,7 @@ int entry_point(struct ggml_et_binary_params * params, void * env) {
const float * src0_data = (const float *) src0->data;
float * dst_data = (float *) dst->data;

// Dimensions and Strides
// Dimensions and strides
const int64_t K = src0->ne[0];
const int64_t M = src0->ne[1];
const int64_t N = src1->ne[1];
Expand All @@ -52,86 +59,101 @@ int entry_point(struct ggml_et_binary_params * params, void * env) {
const size_t nb11 = src1->nb[1], nb12 = src1->nb[2], nb13 = src1->nb[3];
const size_t nb1 = dst->nb[1], nb2 = dst->nb[2], nb3 = dst->nb[3];

// F32 specific block size and counts
const int block_size = QK_F32;
const int block_size = 32; // 32 f32 per row-dot tile (128B)
const int64_t K_blocks = K / block_size;
const int64_t K_remainder = K % block_size;

// Threading distribution
const uint64_t total_elements = M * N * ne2 * ne3;
const uint64_t per_thread = 16;
const uint64_t threads_stride = per_thread * effective_num_threads;

if (effective_thread_id * per_thread >= total_elements) {
return 0;
}

// Broadcasting support
const int64_t r2 = ne12 / ne02;
const int64_t r3 = ne13 / ne03;

for (uint64_t base_idx = effective_thread_id * per_thread; base_idx < total_elements; base_idx += threads_stride) {
for (uint64_t j = 0; j < per_thread; j++) {
const uint64_t idx = base_idx + j;
if (idx >= total_elements) {
break;
const int is_f32_b = (src1->type == GGML_TYPE_F32);
const uint64_t total_elements = (uint64_t) M * N * ne2 * ne3;

// Stage the reused B activation vector into per-shire L2 SCP. Streaming the
// weight matrix thrashes L2 and evicts B, so B is re-read from DRAM
// repeatedly during the decode GEMV. Stage it once per shire into L2 SCP (a
// separate SRAM partition cache streaming cannot evict) via
// et_tensor_load_l2scp; one hart per shire issues the DMA loop, then a
// shire barrier lets all harts read B on-chip. Only applies to the common
// decode case: F32 B, a single contiguous non-broadcast B that fits the SCP
// budget.
const int b_contig = (nb11 == (size_t) K * sizeof(float));
const uint64_t b_lines = ((uint64_t) N * K * sizeof(float) + 63) / 64;
const int stage_b = is_f32_b && b_contig &&
ne12 == 1 && ne13 == 1 && ne02 == 1 && ne03 == 1 &&
b_lines <= 8192; // <= 512 KB, within the SCP budget
const float * b_scp = (const float *) et_shire_l2scp_local(0);

if (stage_b) {
if ((get_hart_id() & 63) == 0) {
et_tensor_load_l2scp_conf_t conf;
conf.use_tmask = false;
conf.stride = 64; // advance one 64B cache line per line loaded
uint64_t remaining = b_lines;
uint64_t dst_ln = 0;
uint64_t addr = (uint64_t) src1->data;
while (remaining > 0) {
uint64_t cl = (remaining >= 16) ? 16 : remaining;
conf.dst_start = dst_ln;
conf.addr = addr;
conf.num_lines = cl - 1; // 4-bit field encodes (lines - 1)
conf.id = 0;
et_tensor_load_l2scp(&conf);
WAIT_TENSOR_LOAD_L2_0;
dst_ln += cl;
addr += cl * 64;
remaining -= cl;
}
}
et_barrier(ET_BARRIER_SHIRE); // B is now resident in L2 SCP for all harts
}

// Index decoding
const int64_t i3 = idx / (M * N * ne2);
const int64_t rem3 = idx % (M * N * ne2);
const int64_t i2 = rem3 / (M * N);
const int64_t rem2 = rem3 % (M * N);
const int64_t n = rem2 / M;
const int64_t m = rem2 % M;

const int64_t i03 = i3 / r3, i02 = i2 / r2;
const int64_t i13 = (ne13 > 1) ? i3 : 0, i12 = (ne12 > 1) ? i2 : 0;

float sum = 0.0f;
const float * f32_row = (const float *) ((const char *) src0_data + m * nb01 + i02 * nb02 + i03 * nb03);

if (src1->type == GGML_TYPE_F32) {
const float * src1_data = (const float *) src1->data;

for (int64_t kb = 0; kb < K_blocks; kb++) {
const float * b_col_ptr =
(const float *) ((const char *) src1_data + (kb * block_size) * sizeof(float) + n * nb11 +
i12 * nb12 + i13 * nb13);
sum += compute_block_dot_product_f32(&f32_row[kb * block_size], b_col_ptr);
}

if (K_remainder > 0) {
const int64_t offset = K_blocks * block_size;
const float * b_col_ptr = (const float *) ((const char *) src1_data + offset * sizeof(float) +
n * nb11 + i12 * nb12 + i13 * nb13);
sum += compute_block_dot_product_f32_partial(&f32_row[offset], b_col_ptr, K_remainder);
}
} else {
const uint16_t * src1_data = (const uint16_t *) src1->data;

for (int64_t kb = 0; kb < K_blocks; kb++) {
const uint16_t * b_col_ptr =
(const uint16_t *) ((const char *) src1_data + (kb * block_size) * sizeof(uint16_t) + n * nb11 +
i12 * nb12 + i13 * nb13);
sum += compute_block_dot_product_f32_f16_partial(&f32_row[kb * block_size], b_col_ptr, block_size);
}

if (K_remainder > 0) {
const int64_t offset = K_blocks * block_size;
const uint16_t * b_col_ptr =
(const uint16_t *) ((const char *) src1_data + offset * sizeof(uint16_t) + n * nb11 +
i12 * nb12 + i13 * nb13);
sum += compute_block_dot_product_f32_f16_partial(&f32_row[offset], b_col_ptr, K_remainder);
}
// Set the vector mask (all 8 lanes) once for the whole row loop.
unsigned long saved_mask;
__asm__ volatile("mova.x.m %0" : "=r"(saved_mask));
__asm__ volatile("mov.m.x m0, x0, 0xFF");

for (uint64_t idx = (uint64_t) thread_id; idx < total_elements; idx += (uint64_t) num_threads) {
// Index decoding
const int64_t i3 = idx / (M * N * ne2);
const int64_t rem3 = idx % (M * N * ne2);
const int64_t i2 = rem3 / (M * N);
const int64_t rem2 = rem3 % (M * N);
const int64_t n = rem2 / M;
const int64_t m = rem2 % M;

const int64_t i03 = i3 / r3, i02 = i2 / r2;
const int64_t i13 = (ne13 > 1) ? i3 : 0, i12 = (ne12 > 1) ? i2 : 0;

const float * f32_row =
(const float *) ((const char *) src0_data + m * nb01 + i02 * nb02 + i03 * nb03);

float sum;
if (is_f32_b) {
const float * b_col = stage_b
? (b_scp + n * K)
: (const float *) ((const char *) src1->data + n * nb11 + i12 * nb12 + i13 * nb13);

f32_dot_reset();
f32_dot_tile(f32_row, b_col, K_blocks);
sum = f32_dot_reduce();

if (K_remainder > 0) {
const int64_t offset = K_blocks * block_size;
sum += compute_block_dot_product_f32_partial(&f32_row[offset], &b_col[offset], K_remainder);
}

// Atomic store for output
volatile float * c_element =
(volatile float *) ((char *) dst_data + m * dst->nb[0] + n * nb1 + i2 * nb2 + i3 * nb3);
atomic_store_f32(c_element, sum);
} else {
const uint16_t * b_col =
(const uint16_t *) ((const char *) src1->data + n * nb11 + i12 * nb12 + i13 * nb13);
sum = compute_block_dot_product_f32_f16_partial(f32_row, b_col, K);
}

volatile float * c_element =
(volatile float *) ((char *) dst_data + m * dst->nb[0] + n * nb1 + i2 * nb2 + i3 * nb3);
atomic_store_f32(c_element, sum);
}

__asm__ volatile("mova.m.x %0" ::"r"(saved_mask));
return 0;
}
Loading