diff --git a/linalg/src/x86_64_fma.rs b/linalg/src/x86_64_fma.rs
index 21acfe8853..b490eb8b08 100644
--- a/linalg/src/x86_64_fma.rs
+++ b/linalg/src/x86_64_fma.rs
@@ -61,6 +61,7 @@ pub mod panel_extract;
 pub mod rms_norm;
 pub mod softmax;
 
+const AVX: fn() -> bool = || is_x86_feature_detected!("avx");
 const AVX2: fn() -> bool = || is_x86_feature_detected!("avx2");
 const FMA: fn() -> bool = || is_x86_feature_detected!("fma");
 const AVX512F: fn() -> bool = || is_x86_feature_detected!("avx512f");
@@ -70,6 +71,11 @@ const AVX512VNNI: fn() -> bool = || is_x86_feature_detected!("avx512vnni");
 tanh_impl!(f32, fma_tanh_f32, 8, 8, is_x86_feature_detected!("fma"));
 sigmoid_impl!(f32, fma_sigmoid_f32, 8, 8, is_x86_feature_detected!("fma"));
 
+// AVX-without-FMA ports of the fma kernels above (each vfmadd132ps expanded
+// to an in-place vmulps+vaddps pair) for CPUs outside the fma tier.
+tanh_impl!(f32, avx_tanh_f32, 8, 8, is_x86_feature_detected!("avx"));
+sigmoid_impl!(f32, avx_sigmoid_f32, 8, 8, is_x86_feature_detected!("avx"));
+
 // AVX-512 (zmm, 16-wide) variants. The assembly lives in x86_64/avx512/; the
 // main loop handles 64 lanes (4 zmm) per iteration with a 16-lane tail, so
 // nr()=16 (any multiple of 16 is safe).
@@ -78,6 +84,21 @@ sigmoid_impl!(f32, avx512_sigmoid_f32, 16, 16, is_x86_feature_detected!("avx512f
 
 fn plug_avx2(_ops: &mut Ops) {}
 
+/// Element-wise kernels for AVX-capable CPUs outside the fma tier: the
+/// mul_by_scalar / max / min asm is plain AVX, and sigmoid / tanh have
+/// dedicated mul+add ports. softmax uses fma asm and keeps its generic
+/// fallback on this tier.
+fn plug_avx(ops: &mut Ops) {
+    ops.sigmoid_f32 = Box::new(|| avx_sigmoid_f32::ew());
+    ops.tanh_f32 = Box::new(|| avx_tanh_f32::ew());
+
+    ops.mul_by_scalar_f32 = Box::new(|| by_scalar::x86_64_avx_f32_mul_by_scalar_32n::ew());
+    ops.max_f32 = Box::new(|| max::x86_64_fma_max_f32_32n::red());
+    ops.min_f32 = Box::new(|| min::x86_64_fma_min_f32_32n::red());
+
+    log::info!("sigmoid_f32, tanh_f32, mul_by_scalar_f32, max_f32, min_f32: x86_64/avx activated");
+}
+
 fn plug_fma(ops: &mut Ops) {
     panel_extract::plug(ops);
 
@@ -146,6 +167,11 @@ fn plug_avx512f(ops: &mut Ops) {
 
 pub fn plug(ops: &mut Ops) {
     mmm::plug(ops);
+    if is_x86_feature_detected!("avx")
+        && !(is_x86_feature_detected!("avx2") && is_x86_feature_detected!("fma"))
+    {
+        plug_avx(ops);
+    }
     if is_x86_feature_detected!("avx2") {
         plug_avx2(ops);
         if is_x86_feature_detected!("fma") {
diff --git a/linalg/src/x86_64_fma/by_scalar.rs b/linalg/src/x86_64_fma/by_scalar.rs
index dacef2425f..970a99bf61 100644
--- a/linalg/src/x86_64_fma/by_scalar.rs
+++ b/linalg/src/x86_64_fma/by_scalar.rs
@@ -17,7 +17,9 @@ unsafe fn x86_64_avx_f32_mul_by_scalar_32n_run(buf: &mut [f32], scalar: f32) {
         let len = buf.len();
         let ptr = buf.as_ptr();
         std::arch::asm!("
-            vbroadcastss ymm0, xmm0
+            // reg-source vbroadcastss needs avx2; this kernel must stay avx-safe
+            vpermilps xmm0, xmm0, 0
+            vinsertf128 ymm0, ymm0, xmm0, 1
             2:
                 vmovaps ymm4, [{ptr}]
                 vmovaps ymm5, [{ptr} + 32]
@@ -48,7 +50,7 @@ unsafe fn x86_64_avx_f32_mul_by_scalar_32n_run(buf: &mut [f32], scalar: f32) {
 pub mod test_x86_64_avx_f32_mul_by_scalar_32n {
     use super::*;
     by_scalar_frame_tests!(
-        is_x86_feature_detected!("avx2"),
+        is_x86_feature_detected!("avx"),
         f32,
         x86_64_avx_f32_mul_by_scalar_32n,
         |a, b| a * b
diff --git a/linalg/src/x86_64_fma/max.rs b/linalg/src/x86_64_fma/max.rs
index 6bbc99077e..66dd4cf628 100644
--- a/linalg/src/x86_64_fma/max.rs
+++ b/linalg/src/x86_64_fma/max.rs
@@ -24,7 +24,9 @@ unsafe fn x86_64_fma_max_f32_32n_run(buf: &[f32]) -> f32 {
         let ptr = buf.as_ptr();
         let mut acc = f32::MIN;
         std::arch::asm!("
-            vbroadcastss ymm0, xmm0
+            // reg-source vbroadcastss needs avx2; this kernel must stay avx-safe
+            vpermilps xmm0, xmm0, 0
+            vinsertf128 ymm0, ymm0, xmm0, 1
             vmovaps ymm1, ymm0
             vmovaps ymm2, ymm0
             vmovaps ymm3, ymm0
@@ -63,7 +65,7 @@ unsafe fn x86_64_fma_max_f32_32n_run(buf: &[f32]) -> f32 {
 #[cfg(test)]
 mod test_x86_64_fma_max_f32_32n {
     use super::*;
-    crate::max_frame_tests!(is_x86_feature_detected!("avx2"), f32, x86_64_fma_max_f32_32n);
+    crate::max_frame_tests!(is_x86_feature_detected!("avx"), f32, x86_64_fma_max_f32_32n);
 }
 
 // AVX-512 version: processes 64 f32 per loop iteration (4 zmm registers of 16
diff --git a/linalg/src/x86_64_fma/min.rs b/linalg/src/x86_64_fma/min.rs
index 09f967a788..6137726bb9 100644
--- a/linalg/src/x86_64_fma/min.rs
+++ b/linalg/src/x86_64_fma/min.rs
@@ -24,7 +24,9 @@ unsafe fn x86_64_fma_min_f32_32n_run(buf: &[f32]) -> f32 {
         let ptr = buf.as_ptr();
         let mut acc = f32::MAX;
         std::arch::asm!("
-            vbroadcastss ymm0, xmm0
+            // reg-source vbroadcastss needs avx2; this kernel must stay avx-safe
+            vpermilps xmm0, xmm0, 0
+            vinsertf128 ymm0, ymm0, xmm0, 1
             vmovaps ymm1, ymm0
             vmovaps ymm2, ymm0
             vmovaps ymm3, ymm0
@@ -63,5 +65,5 @@ unsafe fn x86_64_fma_min_f32_32n_run(buf: &[f32]) -> f32 {
 #[cfg(test)]
 mod test_x86_64_fma_min_f32_32n {
     use super::*;
-    crate::min_frame_tests!(is_x86_feature_detected!("avx2"), f32, x86_64_fma_min_f32_32n);
+    crate::min_frame_tests!(is_x86_feature_detected!("avx"), f32, x86_64_fma_min_f32_32n);
 }
diff --git a/linalg/src/x86_64_fma/mmm.rs b/linalg/src/x86_64_fma/mmm.rs
index 3cd40ad9d0..89cf5d7fb2 100644
--- a/linalg/src/x86_64_fma/mmm.rs
+++ b/linalg/src/x86_64_fma/mmm.rs
@@ -65,6 +65,17 @@ fn pick_mmm(candidates: &[KernelChoice], m: Option<usize>, n: Option<usize>) ->
     (best.ctor)()
 }
 
+// AVX-without-FMA f32 tier for pre-Haswell CPUs (Sandy Bridge / Ivy Bridge):
+// same tile geometries as their fma_ siblings but the inner loops use
+// vmulps+vaddps, and add_unicast avoids the avx2-only vgatherdps.
+MMMExternKernel!(avx_mmm_f32_8x8 <f32>(8, 8)@(256,4) where(AVX) quality(ManuallyOptimized));
+MMMExternKernel!(avx_mmm_f32_16x5<f32>(16,5)@(256,4) where(AVX) quality(ManuallyOptimized));
+MMMExternKernel!(avx_mmm_f32_16x6<f32>(16,6)@(256,4) where(AVX) quality(ManuallyOptimized));
+MMMExternKernel!(avx_mmm_f32_24x4<f32>(24,4)@(256,4) where(AVX) quality(ManuallyOptimized));
+MMMExternKernel!(avx_mmm_f32_32x3<f32>(32,3)@(256,4) where(AVX) quality(ManuallyOptimized));
+MMMExternKernel!(avx_mmm_f32_40x2<f32>(40,2)@(256,4) where(AVX) quality(ManuallyOptimized));
+MMMExternKernel!(avx_mmm_f32_64x1<f32>(64,1)@(256,4) where(AVX) quality(ManuallyOptimized));
+
 MMMExternKernel!(fma_mmm_f32_8x8 <f32>(8, 8)@(256,4) where(FMA) quality(ManuallyOptimized));
 MMMExternKernel!(fma_mmm_f32_16x6<f32>(16,6)@(256,4) where(FMA) quality(ManuallyOptimized));
 MMMExternKernel!(fma_mmm_f32_16x5<f32>(16,5)@(256,4) where(FMA) quality(ManuallyOptimized));
@@ -105,6 +116,15 @@ MMMExternKernel!(avx512_mmm_f32_48x4 <f32>( 48, 4)@(512,4) where (AVX512F) quali
 MMMExternKernel!(avx512_mmm_f32_64x3 <f32>( 64, 3)@(512,4) where (AVX512F) quality(ManuallyOptimized));
 MMMExternKernel!(avx512_mmm_f32_80x2 <f32>( 80, 2)@(512,4) where (AVX512F) quality(ManuallyOptimized));
 
+// 128-bit VEX i32 sibling of avx2_mmm_i32_8x8 for the avx-without-avx2 tier:
+// same i8i8 widening scheme (i8 products computed in i16 lanes) and the same
+// quantization epilogue semantics, on 8x4 xmm column pairs.
+MMMExternKernel! { avx_mmm_i32_8x4<i32>(8,4)@(256,4) where(AVX)
+    packing[1] = i8i8 => |k| k.with_packing(PackedFormat::new(DatumType::I8, 8, 256), PackedFormat::new(DatumType::I8, 4, 4));
+    quality(ManuallyOptimized)
+    store(i8)
+}
+
 MMMExternKernel! { avx2_mmm_i32_8x8<i32>(8,8)@(256,4) where(AVX2)
     packing[1] = i8i8 => |k| k.with_packing(PackedFormat::new(DatumType::I8, 8, 256), PackedFormat::new(DatumType::I8, 8, 4));
     quality(ManuallyOptimized)
@@ -222,6 +242,14 @@ MMMExternKernel! { avx512amx_mmm_f32_16x16<f32>(16,16)@(64,4) where(AVX512AMX_BF
 }
 
 pub fn plug(ops: &mut Ops) {
+    // The fma f32 tier below needs avx2 (vgatherdps) on top of fma; whenever it
+    // can't plug, cover every avx-capable CPU (Sandy/Ivy Bridge without fma,
+    // AMD Bulldozer-family with fma but no avx2) with the mul+add tier.
+    if is_x86_feature_detected!("avx")
+        && !(is_x86_feature_detected!("avx2") && is_x86_feature_detected!("fma"))
+    {
+        plug_avx(ops);
+    }
     if is_x86_feature_detected!("avx2") {
         plug_avx2(ops);
         // AVX-VNNI runs on AVX2-only Atom-class cores (Alder Lake-E, Sierra
@@ -381,6 +409,50 @@ pub fn plug_avx2(ops: &mut Ops) {
     log::info!("qmmm_i32: x86_64/avx2 activated");
 }
 
+/// f32 and i32 kernels for AVX-capable CPUs that can't run the fma tier
+/// (Sandy/Ivy Bridge without fma; AMD Bulldozer-family with fma but no avx2).
+/// Never active alongside plug_fma: these kernels replace the generic
+/// fallback, not the fma_ ones. On avx2-without-fma CPUs plug_avx2 still runs
+/// afterwards and upgrades qmmm_i32 to the wider avx2 kernel.
+pub fn plug_avx(ops: &mut Ops) {
+    ops.mmm_impls.push(avx_mmm_i32_8x4.mmm());
+    ops.qmmm_i32 = Box::new(|_, _, _| avx_mmm_i32_8x4.mmm());
+
+    ops.mmm_impls.extend([
+        avx_mmm_f32_8x8.mmm(),
+        avx_mmm_f32_16x5.mmm(),
+        avx_mmm_f32_16x6.mmm(),
+        avx_mmm_f32_24x4.mmm(),
+        avx_mmm_f32_32x3.mmm(),
+        avx_mmm_f32_40x2.mmm(),
+        avx_mmm_f32_64x1.mmm(), // mmv candidate (nr==1; excluded from n>=2 picks)
+    ]);
+
+    ops.mmv_f32 = Box::new(|_, _| avx_mmm_f32_64x1.mmm());
+
+    const AVX_CHOICES: &[KernelChoice] = &[
+        KernelChoice { mr: 16, nr: 6, scale: 1.0, ctor: || avx_mmm_f32_16x6.mmm() },
+        KernelChoice { mr: 16, nr: 5, scale: 0.98, ctor: || avx_mmm_f32_16x5.mmm() },
+        KernelChoice { mr: 24, nr: 4, scale: 0.95, ctor: || avx_mmm_f32_24x4.mmm() },
+        KernelChoice { mr: 32, nr: 3, scale: 0.93, ctor: || avx_mmm_f32_32x3.mmm() },
+        KernelChoice { mr: 40, nr: 2, scale: 0.90, ctor: || avx_mmm_f32_40x2.mmm() },
+        KernelChoice { mr: 8, nr: 8, scale: 0.80, ctor: || avx_mmm_f32_8x8.mmm() },
+    ];
+    ops.mmm_f32 = Box::new(|m, _, n| match n {
+        None => avx_mmm_f32_16x6.mmm(),
+        Some(1) => avx_mmm_f32_64x1.mmm(),
+        Some(2) => avx_mmm_f32_40x2.mmm(),
+        Some(3) => avx_mmm_f32_32x3.mmm(),
+        Some(4) => avx_mmm_f32_24x4.mmm(),
+        Some(5) => avx_mmm_f32_16x5.mmm(),
+        Some(6) => avx_mmm_f32_16x6.mmm(),
+        Some(8) => avx_mmm_f32_8x8.mmm(),
+        Some(_) => pick_mmm(AVX_CHOICES, m, n),
+    });
+
+    log::info!("mmm_f32, mmv_f32, qmmm_i32: x86_64/avx (no fma) activated");
+}
+
 pub fn plug_fma(ops: &mut Ops) {
     ops.mmm_impls.extend([
         fma_mmm_f32_8x8.mmm(),
diff --git a/linalg/x86_64/fma/2x5/packed_packed_loop1/mul-add.S.raw b/linalg/x86_64/fma/2x5/packed_packed_loop1/mul-add.S.raw
new file mode 100644
index 0000000000..9b63caf308
--- /dev/null
+++ b/linalg/x86_64/fma/2x5/packed_packed_loop1/mul-add.S.raw
@@ -0,0 +1,37 @@
+	// Tile size: 2x5, AVX without FMA (vmulps+vaddps)
+	// Accumulators: ymm0-9
+	// Row regs: ymm10-11
+	// Col reg: ymm14; mul scratch: ymm15
+
+	vmovaps			ymm10,	[rax]
+	vmovaps			ymm11,	[rax + 32]
+
+	vbroadcastss	ymm14,	dword ptr [rcx]
+	vmulps			ymm15,	ymm10,	ymm14
+	vaddps			ymm0,	ymm0,	ymm15
+	vmulps			ymm15,	ymm11,	ymm14
+	vaddps			ymm1,	ymm1,	ymm15
+
+	vbroadcastss	ymm14,	dword ptr [rcx + 4]
+	vmulps			ymm15,	ymm10,	ymm14
+	vaddps			ymm2,	ymm2,	ymm15
+	vmulps			ymm15,	ymm11,	ymm14
+	vaddps			ymm3,	ymm3,	ymm15
+
+	vbroadcastss	ymm14,	dword ptr [rcx + 8]
+	vmulps			ymm15,	ymm10,	ymm14
+	vaddps			ymm4,	ymm4,	ymm15
+	vmulps			ymm15,	ymm11,	ymm14
+	vaddps			ymm5,	ymm5,	ymm15
+
+	vbroadcastss	ymm14,	dword ptr [rcx + 12]
+	vmulps			ymm15,	ymm10,	ymm14
+	vaddps			ymm6,	ymm6,	ymm15
+	vmulps			ymm15,	ymm11,	ymm14
+	vaddps			ymm7,	ymm7,	ymm15
+
+	vbroadcastss	ymm14,	dword ptr [rcx + 16]
+	vmulps			ymm15,	ymm10,	ymm14
+	vaddps			ymm8,	ymm8,	ymm15
+	vmulps			ymm15,	ymm11,	ymm14
+	vaddps			ymm9,	ymm9,	ymm15
diff --git a/linalg/x86_64/fma/2x6/packed_packed_loop1/mul-add.S.raw b/linalg/x86_64/fma/2x6/packed_packed_loop1/mul-add.S.raw
new file mode 100644
index 0000000000..924ecd9861
--- /dev/null
+++ b/linalg/x86_64/fma/2x6/packed_packed_loop1/mul-add.S.raw
@@ -0,0 +1,46 @@
+	// Tile size: 2x6, AVX without FMA (vmulps+vaddps)
+	// Accumulators: ymm0-11
+	// Row regs: ymm12-13
+	// Col reg: ymm14; mul scratch: ymm15
+
+	vmovaps			ymm12,	[rax]
+	vmovaps			ymm13,	[rax + 32]
+
+	vbroadcastss	ymm14,	dword ptr [rcx]
+	vmulps			ymm15,	ymm12,	ymm14
+	vaddps			ymm0,	ymm0,	ymm15
+	vmulps			ymm15,	ymm13,	ymm14
+	vaddps			ymm1,	ymm1,	ymm15
+
+	vbroadcastss	ymm14,	dword ptr [rcx + 4]
+	vmulps			ymm15,	ymm12,	ymm14
+	vaddps			ymm2,	ymm2,	ymm15
+	vmulps			ymm15,	ymm13,	ymm14
+	vaddps			ymm3,	ymm3,	ymm15
+
+	vbroadcastss	ymm14,	dword ptr [rcx + 8]
+	vmulps			ymm15,	ymm12,	ymm14
+	vaddps			ymm4,	ymm4,	ymm15
+	vmulps			ymm15,	ymm13,	ymm14
+	vaddps			ymm5,	ymm5,	ymm15
+
+	vbroadcastss	ymm14,	dword ptr [rcx + 12]
+	vmulps			ymm15,	ymm12,	ymm14
+	vaddps			ymm6,	ymm6,	ymm15
+	vmulps			ymm15,	ymm13,	ymm14
+	vaddps			ymm7,	ymm7,	ymm15
+
+	vbroadcastss	ymm14,	dword ptr [rcx + 16]
+	vmulps			ymm15,	ymm12,	ymm14
+	vaddps			ymm8,	ymm8,	ymm15
+	vmulps			ymm15,	ymm13,	ymm14
+	vaddps			ymm9,	ymm9,	ymm15
+
+	vbroadcastss	ymm14,	dword ptr [rcx + 20]
+	vmulps			ymm15,	ymm12,	ymm14
+	vaddps			ymm10,	ymm10,	ymm15
+	vmulps			ymm15,	ymm13,	ymm14
+	vaddps			ymm11,	ymm11,	ymm15
+
+	add rax, 64
+	add rcx, 24
diff --git a/linalg/x86_64/fma/3x4/packed_packed_loop1/mul-add.S.raw b/linalg/x86_64/fma/3x4/packed_packed_loop1/mul-add.S.raw
new file mode 100644
index 0000000000..1f0e63a863
--- /dev/null
+++ b/linalg/x86_64/fma/3x4/packed_packed_loop1/mul-add.S.raw
@@ -0,0 +1,48 @@
+	// Tile size: 3x4, AVX without FMA (vmulps+vaddps)
+	// Accumulators: ymm0-11
+	// Row regs: ymm12-14
+	// Col broadcast doubles as mul scratch: ymm15 (re-broadcast per row)
+
+	vmovaps			ymm12,	[rax]
+	vmovaps			ymm13,	[rax + 32]
+	vmovaps			ymm14,	[rax + 64]
+
+	vbroadcastss	ymm15,	dword ptr [rcx + 0]
+	vmulps			ymm15,	ymm15,	ymm12
+	vaddps			ymm0,	ymm0,	ymm15
+	vbroadcastss	ymm15,	dword ptr [rcx + 0]
+	vmulps			ymm15,	ymm15,	ymm13
+	vaddps			ymm1,	ymm1,	ymm15
+	vbroadcastss	ymm15,	dword ptr [rcx + 0]
+	vmulps			ymm15,	ymm15,	ymm14
+	vaddps			ymm2,	ymm2,	ymm15
+
+	vbroadcastss	ymm15,	dword ptr [rcx + 4]
+	vmulps			ymm15,	ymm15,	ymm12
+	vaddps			ymm3,	ymm3,	ymm15
+	vbroadcastss	ymm15,	dword ptr [rcx + 4]
+	vmulps			ymm15,	ymm15,	ymm13
+	vaddps			ymm4,	ymm4,	ymm15
+	vbroadcastss	ymm15,	dword ptr [rcx + 4]
+	vmulps			ymm15,	ymm15,	ymm14
+	vaddps			ymm5,	ymm5,	ymm15
+
+	vbroadcastss	ymm15,	dword ptr [rcx + 8]
+	vmulps			ymm15,	ymm15,	ymm12
+	vaddps			ymm6,	ymm6,	ymm15
+	vbroadcastss	ymm15,	dword ptr [rcx + 8]
+	vmulps			ymm15,	ymm15,	ymm13
+	vaddps			ymm7,	ymm7,	ymm15
+	vbroadcastss	ymm15,	dword ptr [rcx + 8]
+	vmulps			ymm15,	ymm15,	ymm14
+	vaddps			ymm8,	ymm8,	ymm15
+
+	vbroadcastss	ymm15,	dword ptr [rcx + 12]
+	vmulps			ymm15,	ymm15,	ymm12
+	vaddps			ymm9,	ymm9,	ymm15
+	vbroadcastss	ymm15,	dword ptr [rcx + 12]
+	vmulps			ymm15,	ymm15,	ymm13
+	vaddps			ymm10,	ymm10,	ymm15
+	vbroadcastss	ymm15,	dword ptr [rcx + 12]
+	vmulps			ymm15,	ymm15,	ymm14
+	vaddps			ymm11,	ymm11,	ymm15
diff --git a/linalg/x86_64/fma/4x3/packed_packed_loop1/mul-add.S.raw b/linalg/x86_64/fma/4x3/packed_packed_loop1/mul-add.S.raw
new file mode 100644
index 0000000000..faad38110b
--- /dev/null
+++ b/linalg/x86_64/fma/4x3/packed_packed_loop1/mul-add.S.raw
@@ -0,0 +1,55 @@
+	// Tile size: 4x3, AVX without FMA (vmulps+vaddps)
+	// Accumulators: ymm0-11
+	// Row reg: ymm12
+	// Col broadcasts double as mul scratch: ymm13-15 (re-broadcast per row)
+
+	vmovaps			ymm12,	[rax]
+
+	vbroadcastss	ymm13,	dword ptr [rcx + 0]
+	vmulps			ymm13,	ymm13,	ymm12
+	vaddps			ymm0,	ymm0,	ymm13
+	vbroadcastss	ymm14,	dword ptr [rcx + 4]
+	vmulps			ymm14,	ymm14,	ymm12
+	vaddps			ymm4,	ymm4,	ymm14
+	vbroadcastss	ymm15,	dword ptr [rcx + 8]
+	vmulps			ymm15,	ymm15,	ymm12
+	vaddps			ymm8,	ymm8,	ymm15
+
+	vmovaps			ymm12,	[rax + 32]
+
+	vbroadcastss	ymm13,	dword ptr [rcx + 0]
+	vmulps			ymm13,	ymm13,	ymm12
+	vaddps			ymm1,	ymm1,	ymm13
+	vbroadcastss	ymm14,	dword ptr [rcx + 4]
+	vmulps			ymm14,	ymm14,	ymm12
+	vaddps			ymm5,	ymm5,	ymm14
+	vbroadcastss	ymm15,	dword ptr [rcx + 8]
+	vmulps			ymm15,	ymm15,	ymm12
+	vaddps			ymm9,	ymm9,	ymm15
+
+	vmovaps			ymm12,	[rax + 64]
+
+	vbroadcastss	ymm13,	dword ptr [rcx + 0]
+	vmulps			ymm13,	ymm13,	ymm12
+	vaddps			ymm2,	ymm2,	ymm13
+	vbroadcastss	ymm14,	dword ptr [rcx + 4]
+	vmulps			ymm14,	ymm14,	ymm12
+	vaddps			ymm6,	ymm6,	ymm14
+	vbroadcastss	ymm15,	dword ptr [rcx + 8]
+	vmulps			ymm15,	ymm15,	ymm12
+	vaddps			ymm10,	ymm10,	ymm15
+
+	vmovaps			ymm12,	[rax + 96]
+
+	vbroadcastss	ymm13,	dword ptr [rcx + 0]
+	vmulps			ymm13,	ymm13,	ymm12
+	vaddps			ymm3,	ymm3,	ymm13
+	vbroadcastss	ymm14,	dword ptr [rcx + 4]
+	vmulps			ymm14,	ymm14,	ymm12
+	vaddps			ymm7,	ymm7,	ymm14
+	vbroadcastss	ymm15,	dword ptr [rcx + 8]
+	vmulps			ymm15,	ymm15,	ymm12
+	vaddps			ymm11,	ymm11,	ymm15
+
+	add rcx, 12
+	add rax, 128
diff --git a/linalg/x86_64/fma/5x2/packed_packed_loop1/mul-add.S.raw b/linalg/x86_64/fma/5x2/packed_packed_loop1/mul-add.S.raw
new file mode 100644
index 0000000000..8231f8cfeb
--- /dev/null
+++ b/linalg/x86_64/fma/5x2/packed_packed_loop1/mul-add.S.raw
@@ -0,0 +1,45 @@
+	// Tile size: 5x2, AVX without FMA (vmulps+vaddps)
+	// Accumulators: ymm0-9
+	// Row regs: ymm10-14
+	// Col broadcast doubles as mul scratch: ymm15 (re-broadcast per row)
+
+	vmovaps			ymm10,	[rax]
+	vmovaps			ymm11,	[rax + 32]
+	vmovaps			ymm12,	[rax + 64]
+	vmovaps			ymm13,	[rax + 96]
+	vmovaps			ymm14,	[rax + 128]
+
+	vbroadcastss	ymm15,	dword ptr [rcx + 0]
+	vmulps			ymm15,	ymm15,	ymm10
+	vaddps			ymm0,	ymm0,	ymm15
+	vbroadcastss	ymm15,	dword ptr [rcx + 0]
+	vmulps			ymm15,	ymm15,	ymm11
+	vaddps			ymm1,	ymm1,	ymm15
+	vbroadcastss	ymm15,	dword ptr [rcx + 0]
+	vmulps			ymm15,	ymm15,	ymm12
+	vaddps			ymm2,	ymm2,	ymm15
+	vbroadcastss	ymm15,	dword ptr [rcx + 0]
+	vmulps			ymm15,	ymm15,	ymm13
+	vaddps			ymm3,	ymm3,	ymm15
+	vbroadcastss	ymm15,	dword ptr [rcx + 0]
+	vmulps			ymm15,	ymm15,	ymm14
+	vaddps			ymm4,	ymm4,	ymm15
+
+	vbroadcastss	ymm15,	dword ptr [rcx + 4]
+	vmulps			ymm15,	ymm15,	ymm10
+	vaddps			ymm5,	ymm5,	ymm15
+	vbroadcastss	ymm15,	dword ptr [rcx + 4]
+	vmulps			ymm15,	ymm15,	ymm11
+	vaddps			ymm6,	ymm6,	ymm15
+	vbroadcastss	ymm15,	dword ptr [rcx + 4]
+	vmulps			ymm15,	ymm15,	ymm12
+	vaddps			ymm7,	ymm7,	ymm15
+	vbroadcastss	ymm15,	dword ptr [rcx + 4]
+	vmulps			ymm15,	ymm15,	ymm13
+	vaddps			ymm8,	ymm8,	ymm15
+	vbroadcastss	ymm15,	dword ptr [rcx + 4]
+	vmulps			ymm15,	ymm15,	ymm14
+	vaddps			ymm9,	ymm9,	ymm15
+
+	add rax, 160
+	add rcx, 8
diff --git a/linalg/x86_64/fma/8x1/packed_packed_loop1/mul-add.S.raw b/linalg/x86_64/fma/8x1/packed_packed_loop1/mul-add.S.raw
new file mode 100644
index 0000000000..e3d066dfb2
--- /dev/null
+++ b/linalg/x86_64/fma/8x1/packed_packed_loop1/mul-add.S.raw
@@ -0,0 +1,40 @@
+	// Tile size: 8x1, AVX without FMA (vmulps+vaddps)
+	// Accumulators: ymm0-7
+	// Col reg: ymm15
+	// Row regs double as mul scratch: ymm8-14 (+ ymm8 reuse)
+
+	vbroadcastss    ymm15,  dword ptr [rcx]
+
+	vmovaps         ymm8,  [rax + 0]
+	vmovaps         ymm9,  [rax + 32]
+	vmovaps         ymm10, [rax + 64]
+	vmovaps         ymm11, [rax + 96]
+
+	vmulps          ymm8,  ymm8,  ymm15
+	vaddps          ymm0,  ymm0,  ymm8
+	vmulps          ymm9,  ymm9,  ymm15
+	vaddps          ymm1,  ymm1,  ymm9
+
+	vmovaps         ymm12, [rax + 128]
+	vmovaps         ymm13, [rax + 160]
+
+	vmulps          ymm10, ymm10, ymm15
+	vaddps          ymm2,  ymm2,  ymm10
+	vmulps          ymm11, ymm11, ymm15
+	vaddps          ymm3,  ymm3,  ymm11
+
+	vmovaps         ymm14, [rax + 192]
+	vmovaps         ymm8,  [rax + 224]
+
+	vmulps          ymm12, ymm12, ymm15
+	vaddps          ymm4,  ymm4,  ymm12
+	vmulps          ymm13, ymm13, ymm15
+	vaddps          ymm5,  ymm5,  ymm13
+
+	vmulps          ymm14, ymm14, ymm15
+	vaddps          ymm6,  ymm6,  ymm14
+	vmulps          ymm8,  ymm8,  ymm15
+	vaddps          ymm7,  ymm7,  ymm8
+
+	add rcx, 4
+	add rax, 256
diff --git a/linalg/x86_64/fma/avx_mmm_f32_16x5.S.j2 b/linalg/x86_64/fma/avx_mmm_f32_16x5.S.j2
new file mode 100644
index 0000000000..97314bab0e
--- /dev/null
+++ b/linalg/x86_64/fma/avx_mmm_f32_16x5.S.j2
@@ -0,0 +1,171 @@
+{#
+// vim: set syntax=asm :
+/* mmm 16 x 5, AVX without FMA (pre-Haswell: Sandy Bridge / Ivy Bridge)
+
+    ymm0 ymm2 ymm4 ymm6 ymm8
+    ymm1 ymm3 ymm5 ymm7 ymm9
+
+System V ABI:
+    args: rdi, rsi, rdx, rcx, r8, r9
+    preserve: rbx, rsp, rbp, r12, r13, r14, r15
+    scratch: rax, rdi, rsi, rdx, rcx, r8, r9, r10, r11
+    return: rax (+rdx)
+
+Windows ABI:
+    args: RCX, RDX, R8, R9
+    preserve: RBX, RBP, RDI, RSI, RSP, R12, R13, R14, R15, and XMM6-15
+    scratch: RAX, RCX, RDX, R8, R9, R10, R11, XMM0-5, and the upper portions of YMM0-15 and ZMM0-15
+    return: rax (+rdx)
+*/
+#}
+
+{% set prefix = "avx" %}{% set type = "f32" %}{% set size = "16x5" %}{% set suffix = suffix %}{% set G = G %}{% include "preamble.j2" %}
+
+{{L}}clear:
+    vzeroall
+    jmp     {{L}}non_linear_loop
+
+{{L}}add_mat_mul:
+    mov     rcx,    [rdi + 24]   // B
+    mov     rax,    [rdi + 16]   // A
+
+    mov     rbx,    [rdi + 8]    // k
+    test    rbx,    rbx
+    jz      {{L}}non_linear_loop
+
+{{L}}main_loop_packed_packed:
+    {% include "2x5/packed_packed_loop1/mul-add.S.raw" %}
+
+    add             rcx,    20
+    add             rax,    64
+    dec             rbx
+    jnz             {{L}}main_loop_packed_packed
+
+    jmp             {{L}}non_linear_loop
+
+// NON LINEAR / ADDC
+
+{% set from = 0 %}{% set to = 9 %}{% set type = "f32" %}{% include "fma_mmm_f32_scalars.j2" %}
+{% set mr = 16 %}{% set from = 0 %}{% set to = 9 %}{% set type = "f32" %}{% include "fma_mmm_f32_per_rows.j2" %}
+{% set mr = 16 %}{% set from = 0 %}{% set to = 9 %}{% set type = "f32" %}{% include "fma_mmm_f32_per_cols.j2" %}
+{% set from = 0 %}{% set to = 9 %}{% include "fma_mmm_load_tile.j2" %}
+
+{{L}}add_unicast:
+
+    mov     r10,    [rdi + 8]           // c ptr
+    mov     rsi,    [rdi + 16]          // row stride
+    mov     rbx,    [rdi + 24]          // col stride
+
+    cmp     rsi,    4
+    jne     {{L}}add_unicast_generic
+
+{% for i in range(0, 5) %}
+    vmovups         ymm12,  [r10]
+    vmovups         ymm13,  [r10 + 32]
+    vaddps          ymm{{ i * 2 }},     ymm{{ i * 2 }},     ymm12
+    vaddps          ymm{{ i * 2 + 1 }}, ymm{{ i * 2 + 1 }}, ymm13
+    add             r10,    rbx
+{% endfor %}
+
+    jmp    {{L}}non_linear_loop
+
+// no vgatherdps before avx2: build each 8-row column half with scalar
+// inserts (all VEX-encoded to avoid sse/avx transition stalls)
+{{L}}add_unicast_generic:
+{% for i in range(0, 5) %}
+    mov         r9,     r10
+    vmovss      xmm12, dword ptr [r9]
+    add         r9, rsi
+    {% for lane in range(1, 4) %}
+    vinsertps   xmm12, xmm12, dword ptr [r9], {{ lane * 16 }}
+    add         r9, rsi
+    {% endfor %}
+    vmovss      xmm14, dword ptr [r9]
+    add         r9, rsi
+    {% for lane in range(1, 4) %}
+    vinsertps   xmm14, xmm14, dword ptr [r9], {{ lane * 16 }}
+    add         r9, rsi
+    {% endfor %}
+    vinsertf128 ymm12, ymm12, xmm14, 1
+
+    vmovss      xmm13, dword ptr [r9]
+    add         r9, rsi
+    {% for lane in range(1, 4) %}
+    vinsertps   xmm13, xmm13, dword ptr [r9], {{ lane * 16 }}
+    add         r9, rsi
+    {% endfor %}
+    vmovss      xmm14, dword ptr [r9]
+    add         r9, rsi
+    {% for lane in range(1, 4) %}
+    vinsertps   xmm14, xmm14, dword ptr [r9], {{ lane * 16 }}
+    add         r9, rsi
+    {% endfor %}
+    vinsertf128 ymm13, ymm13, xmm14, 1
+
+    vaddps          ymm{{ i * 2 }},     ymm{{ i * 2 }},     ymm12
+    vaddps          ymm{{ i * 2 + 1 }}, ymm{{ i * 2 + 1 }}, ymm13
+    add             r10,    rbx
+{% endfor %}
+
+    jmp    {{L}}non_linear_loop
+
+{{L}}add_row_col_products:
+    mov             rax, [ rdi + 8 ]
+    mov             rbx, [ rdi + 16 ]
+
+    vmovups         ymm12,  [rax]
+    vmovups         ymm13,  [rax + 32]
+
+{% for i in range(0, 5) %}
+    vbroadcastss    ymm14, dword ptr [rbx + {{ i * 4 }} ]
+    vmulps          ymm15,  ymm12, ymm14
+    vaddps          ymm{{ i * 2 }},   ymm{{ i * 2 }},   ymm15
+    vmulps          ymm14,  ymm13, ymm14
+    vaddps          ymm{{ i * 2 + 1 }}, ymm{{ i * 2 + 1 }}, ymm14
+{% endfor %}
+    jmp    {{L}}non_linear_loop
+
+{{L}}store:
+    mov     r8,     [rdi + 8]           // c ptr
+    mov     rsi,    [rdi + 16]          // row stride
+    mov     rbx,    [rdi + 24]          // col stride
+
+    lea     r9,     [ r8 + rbx ]
+    lea     r10,    [ r8 + 2 * rbx ]
+    lea     r12,    [ r8 + 4 * rbx ]
+    lea     r11,    [ r10 + rbx ]
+    cmp     rbx,    64
+    jne     {{L}}store_strides_generic
+
+    {% for row in range(0, 2) %}
+        {% for col in range(0, 5) %}
+            vmovups ymmword ptr [r{{ col + 8 }}], ymm{{ col * 2 + row }}
+            add r{{ col + 8 }}, 32
+       {% endfor %}
+    {% endfor %}
+
+    jmp     {{L}}non_linear_loop
+
+{{L}}store_strides_generic:
+    // tops of cols
+
+    {% for quarter in range(0, 4) %}
+        {% if quarter != 0 %}
+            // move next four rows at top (xmm0,2,..10)
+            vperm2f128  ymm0,   ymm0,   ymm1,  {{quarter}}
+            vperm2f128  ymm2,   ymm2,   ymm3,  {{quarter}}
+            vperm2f128  ymm4,   ymm4,   ymm5,  {{quarter}}
+            vperm2f128  ymm6,   ymm6,   ymm7,  {{quarter}}
+            vperm2f128  ymm8,   ymm8,   ymm9,  {{quarter}}
+        {% endif %}
+        {% for row in range(0, 4) %}
+            {% for i in range(0, 5) %}
+                vextractps  dword ptr [r{{ i + 8 }}], xmm{{ i * 2 }}, {{row}}
+                add         r{{ i + 8 }}, rsi
+            {% endfor %}
+        {% endfor %}
+    {% endfor %}
+
+    jmp     {{L}}non_linear_loop
+
+{% set prefix = "avx" %}{% set type = "f32" %}{% set size = "16x5" %}{% set suffix = suffix %}{% set G = G %}{% set L = L %}{% include "postamble.j2" %}
diff --git a/linalg/x86_64/fma/avx_mmm_f32_16x6.S.j2 b/linalg/x86_64/fma/avx_mmm_f32_16x6.S.j2
new file mode 100644
index 0000000000..a5796047f3
--- /dev/null
+++ b/linalg/x86_64/fma/avx_mmm_f32_16x6.S.j2
@@ -0,0 +1,159 @@
+{#
+// vim: set syntax=asm :
+
+/* mmm 16 x 6, AVX without FMA (pre-Haswell: Sandy Bridge / Ivy Bridge)
+
+    ymm0 ymm2 ymm4 ymm6 ymm8 ymm10
+    ymm1 ymm3 ymm5 ymm7 ymm9 ymm11
+
+System V ABI:
+    args: rdi, rsi, rdx, rcx, r8, r9
+    preserve: rbx, rsp, rbp, r12, r13, r14, r15
+    scratch: rax, rdi, rsi, rdx, rcx, r8, r9, r10, r11
+    return: rax (+rdx)
+
+Windows ABI:
+    args: RCX, RDX, R8, R9
+    preserve: RBX, RBP, RDI, RSI, RSP, R12, R13, R14, R15, and XMM6-15
+    scratch: RAX, RCX, RDX, R8, R9, R10, R11, XMM0-5, and the upper portions of YMM0-15 and ZMM0-15
+    return: rax (+rdx)
+*/
+#}
+
+{% set prefix = "avx" %}{% set type = "f32" %}{% set size = "16x6" %}{% set suffix = suffix %}{% set G = G %}{% include "preamble.j2" %}
+
+{{L}}clear:
+    vzeroall
+    jmp     {{L}}non_linear_loop
+
+{{L}}add_mat_mul:
+    mov     rcx,    [rdi + 24]   // B
+    mov     rax,    [rdi + 16]   // A
+
+    mov     rbx,    [rdi + 8]    // k
+    test    rbx,    rbx
+    jz      {{L}}non_linear_loop
+
+{{L}}main_loop_packed_packed:
+	{% include "2x6/packed_packed_loop1/mul-add.S.raw" %}
+
+    dec             rbx
+    jnz             {{L}}main_loop_packed_packed
+
+    jmp             {{L}}non_linear_loop
+
+// NON LINEAR / ADDC
+
+{% set from = 0 %}{% set to = 11 %}{% set type = "f32" %}{% include "fma_mmm_f32_scalars.j2" %}
+{% set mr = 16 %}{% set from = 0 %}{% set to = 11 %}{% set type = "f32" %}{% include "fma_mmm_f32_per_rows.j2" %}
+{% set mr = 16 %}{% set from = 0 %}{% set to = 11 %}{% set type = "f32" %}{% include "fma_mmm_f32_per_cols.j2" %}
+{% set from = 0 %}{% set to = 11 %}{% include "fma_mmm_load_tile.j2" %}
+
+{{L}}add_unicast:
+
+    mov     r10,    [rdi + 8]           // c ptr
+    mov     rsi,    [rdi + 16]          // row stride
+    mov     rbx,    [rdi + 24]          // col stride
+
+    cmp     rsi,    4
+    jne     {{L}}add_unicast_generic
+
+{% for i in range(0, 6) %}
+    vmovups         ymm12,  [r10]
+    vmovups         ymm13,  [r10 + 32]
+    vaddps          ymm{{ i * 2 }},     ymm{{ i * 2 }},     ymm12
+    vaddps          ymm{{ i * 2 + 1 }}, ymm{{ i * 2 + 1 }}, ymm13
+    add             r10,    rbx
+{% endfor %}
+
+    jmp    {{L}}non_linear_loop
+
+// no vgatherdps before avx2: build each 8-row column half with scalar
+// inserts (all VEX-encoded to avoid sse/avx transition stalls)
+{{L}}add_unicast_generic:
+{% for i in range(0, 6) %}
+    mov         r9,     r10
+    vmovss      xmm12, dword ptr [r9]
+    add         r9, rsi
+    {% for lane in range(1, 4) %}
+    vinsertps   xmm12, xmm12, dword ptr [r9], {{ lane * 16 }}
+    add         r9, rsi
+    {% endfor %}
+    vmovss      xmm14, dword ptr [r9]
+    add         r9, rsi
+    {% for lane in range(1, 4) %}
+    vinsertps   xmm14, xmm14, dword ptr [r9], {{ lane * 16 }}
+    add         r9, rsi
+    {% endfor %}
+    vinsertf128 ymm12, ymm12, xmm14, 1
+
+    vmovss      xmm13, dword ptr [r9]
+    add         r9, rsi
+    {% for lane in range(1, 4) %}
+    vinsertps   xmm13, xmm13, dword ptr [r9], {{ lane * 16 }}
+    add         r9, rsi
+    {% endfor %}
+    vmovss      xmm14, dword ptr [r9]
+    add         r9, rsi
+    {% for lane in range(1, 4) %}
+    vinsertps   xmm14, xmm14, dword ptr [r9], {{ lane * 16 }}
+    add         r9, rsi
+    {% endfor %}
+    vinsertf128 ymm13, ymm13, xmm14, 1
+
+    vaddps          ymm{{ i * 2 }},     ymm{{ i * 2 }},     ymm12
+    vaddps          ymm{{ i * 2 + 1 }}, ymm{{ i * 2 + 1 }}, ymm13
+    add             r10,    rbx
+{% endfor %}
+
+    jmp    {{L}}non_linear_loop
+
+{{L}}add_row_col_products:
+    mov             rax, [ rdi + 8 ]
+    mov             rbx, [ rdi + 16 ]
+
+    vmovups         ymm12,  [rax]
+    vmovups         ymm13,  [rax + 32]
+
+{% for i in range(0, 6) %}
+    vbroadcastss    ymm14, dword ptr [rbx + {{ i * 4 }} ]
+    vmulps          ymm15,  ymm12, ymm14
+    vaddps          ymm{{ i * 2 }},   ymm{{ i * 2 }},   ymm15
+    vmulps          ymm14,  ymm13, ymm14
+    vaddps          ymm{{ i * 2 + 1 }}, ymm{{ i * 2 + 1 }}, ymm14
+{% endfor %}
+    jmp    {{L}}non_linear_loop
+
+{{L}}store:
+    mov     r8,     [rdi + 8]           // c ptr
+    mov     rsi,    [rdi + 16]          // row stride
+    mov     rbx,    [rdi + 24]          // col stride
+
+    // tops of cols
+    lea     r9,     [ r8 + rbx ]
+    lea     r10,    [ r8 + 2 * rbx ]
+    lea     r12,    [ r8 + 4 * rbx ]
+    lea     r11,    [ r10 + rbx ]
+    lea     r13,    [ r12 + rbx ]
+
+    {% for quarter in range(0, 4) %}
+        {% if quarter != 0 %}
+            // move next four rows at top (xmm0,2,..10)
+            vperm2f128  ymm0,   ymm0,   ymm1,  {{quarter}}
+            vperm2f128  ymm2,   ymm2,   ymm3,  {{quarter}}
+            vperm2f128  ymm4,   ymm4,   ymm5,  {{quarter}}
+            vperm2f128  ymm6,   ymm6,   ymm7,  {{quarter}}
+            vperm2f128  ymm8,   ymm8,   ymm9,  {{quarter}}
+            vperm2f128  ymm10,  ymm10,  ymm11, {{quarter}}
+        {% endif %}
+        {% for row in range(0, 4) %}
+            {% for i in range(0, 6) %}
+                vextractps  dword ptr [r{{ i + 8 }}], xmm{{ i * 2 }}, {{row}}
+                add         r{{ i + 8 }}, rsi
+            {% endfor %}
+        {% endfor %}
+    {% endfor %}
+
+    jmp     {{L}}non_linear_loop
+
+{% set prefix = "avx" %}{% set type = "f32" %}{% set size = "16x6" %}{% set suffix = suffix %}{% set G = G %}{% set L = L %}{% include "postamble.j2" %}
diff --git a/linalg/x86_64/fma/avx_mmm_f32_24x4.S.j2 b/linalg/x86_64/fma/avx_mmm_f32_24x4.S.j2
new file mode 100644
index 0000000000..6ca7f75f72
--- /dev/null
+++ b/linalg/x86_64/fma/avx_mmm_f32_24x4.S.j2
@@ -0,0 +1,163 @@
+{#
+// vim: set syntax=asm :
+/* mmm 24 x 4, AVX without FMA (pre-Haswell: Sandy Bridge / Ivy Bridge)
+
+    ymm0 ymm3 ymm6 ymm9
+    ymm1 ymm4 ymm7 ymm10
+    ymm2 ymm5 ymm8 ymm11
+
+System V ABI:
+    args: rdi, rsi, rdx, rcx, r8, r9
+    preserve: rbx, rsp, rbp, r12, r13, r14, r15
+    scratch: rax, rdi, rsi, rdx, rcx, r8, r9, r10, r11
+    return: rax (+rdx)
+
+Windows ABI:
+    args: RCX, RDX, R8, R9
+    preserve: RBX, RBP, RDI, RSI, RSP, R12, R13, R14, R15, and XMM6-15
+    scratch: RAX, RCX, RDX, R8, R9, R10, R11, XMM0-5, and the upper portions of YMM0-15 and ZMM0-15
+    return: rax (+rdx)
+*/
+#}
+
+{% set prefix = "avx" %}{% set type = "f32" %}{% set size = "24x4" %}{% set suffix = suffix %}{% set G = G %}{% include "preamble.j2" %}
+
+{{L}}clear:
+    vzeroall
+    jmp     {{L}}non_linear_loop
+
+{{L}}add_mat_mul:
+    mov     rcx,    [rdi + 24]   // B
+    mov     rax,    [rdi + 16]   // A
+
+    mov     rbx,    [rdi + 8]    // k
+    test    rbx,    rbx
+    jz      {{L}}non_linear_loop
+
+{{L}}main_loop_packed_packed:
+    {% include "3x4/packed_packed_loop1/mul-add.S.raw" %}
+
+    add             rcx,    16
+    add             rax,    96
+    dec             rbx
+    jnz             {{L}}main_loop_packed_packed
+
+    jmp             {{L}}non_linear_loop
+
+// NON LINEAR / ADDC
+
+{% set from = 0 %}{% set to = 11 %}{% set type = "f32" %}{% include "fma_mmm_f32_scalars.j2" %}
+{% set mr = 24 %}{% set from = 0 %}{% set to = 11 %}{% set type = "f32" %}{% include "fma_mmm_f32_per_rows.j2" %}
+{% set mr = 24 %}{% set from = 0 %}{% set to = 11 %}{% set type = "f32" %}{% include "fma_mmm_f32_per_cols.j2" %}
+{% set from = 0 %}{% set to = 11 %}{% include "fma_mmm_load_tile.j2" %}
+
+{{L}}add_unicast:
+
+    mov     r8,    [rdi + 8]           // c ptr
+    mov     rsi,    [rdi + 16]          // row stride
+    mov     rbx,    [rdi + 24]          // col stride
+
+    cmp rsi, 4
+    jne {{L}}unicast_generic
+
+    lea             r9,  [ r8 + rbx ]
+    lea             r10, [ r9 + rbx]
+    lea             r11, [ r10 + rbx ]
+
+{% for col in range(0, 4) %}
+    {% for row in range(0, 3) %}
+        vmovups ymm12,  [ r{{ col + 8 }} ]
+        add r{{ col + 8 }}, 32
+        vaddps ymm{{ col * 3 + row }}, ymm{{ col * 3 + row }}, ymm12
+    {% endfor %}
+{% endfor %}
+    jmp    {{L}}non_linear_loop
+
+// no vgatherdps before avx2: build each 8-row block with scalar inserts
+// (all VEX-encoded to avoid sse/avx transition stalls)
+{{L}}unicast_generic:
+    lea             r9, [ r8 + rsi * 8 ]
+    lea             r10, [ r9 + rsi * 8 ]
+
+{% for col in range(0, 4) %}
+   {% for row in range(0, 3) %}
+    mov         r11, r{{ row + 8 }}
+    vmovss      xmm12, dword ptr [r11]
+    add         r11, rsi
+    {% for lane in range(1, 4) %}
+    vinsertps   xmm12, xmm12, dword ptr [r11], {{ lane * 16 }}
+    add         r11, rsi
+    {% endfor %}
+    vmovss      xmm13, dword ptr [r11]
+    add         r11, rsi
+    {% for lane in range(1, 4) %}
+    vinsertps   xmm13, xmm13, dword ptr [r11], {{ lane * 16 }}
+    add         r11, rsi
+    {% endfor %}
+    vinsertf128 ymm12, ymm12, xmm13, 1
+    add         r{{ row + 8 }}, rbx
+    vaddps      ymm{{ col * 3 + row }}, ymm{{ col * 3 + row }}, ymm12
+   {% endfor %}
+{% endfor %}
+
+    jmp    {{L}}non_linear_loop
+
+{{L}}add_row_col_products:
+    mov             rax, [ rdi + 8 ]
+    mov             rbx, [ rdi + 16 ]
+
+    vmovups         ymm12,  [rax]
+    vmovups         ymm13,  [rax + 32]
+    vmovups         ymm15,  [rax + 64]
+{% for i in range(0, 4) %}
+    vbroadcastss    ymm14, dword ptr [rbx + {{ i * 4 }} ]
+    vmulps          ymm14,  ymm14, ymm12
+    vaddps          ymm{{ i * 3 }},   ymm{{ i * 3 }},   ymm14
+    vbroadcastss    ymm14, dword ptr [rbx + {{ i * 4 }} ]
+    vmulps          ymm14,  ymm14, ymm13
+    vaddps          ymm{{ i * 3 + 1 }}, ymm{{ i * 3 + 1 }}, ymm14
+    vbroadcastss    ymm14, dword ptr [rbx + {{ i * 4 }} ]
+    vmulps          ymm14,  ymm14, ymm15
+    vaddps          ymm{{ i * 3 + 2 }}, ymm{{ i * 3 + 2 }}, ymm14
+{% endfor %}
+
+    jmp    {{L}}non_linear_loop
+
+{{L}}store:
+    mov     r8,     [rdi + 8]           // c ptr
+    mov     rsi,    [rdi + 16]          // row stride
+    mov     rbx,    [rdi + 24]          // col stride
+
+    lea     r9,     [ r8  +     rbx ]
+    lea     r10,    [ r8  + 2 * rbx ]
+    lea     r11,    [ r10 +     rbx ]
+
+    cmp         rsi, 4
+    jne         {{L}}store_strides_generic
+
+    {% for col in range(0, 4) %}
+       {% for row in range(0, 3) %}
+            vmovups ymmword ptr [r{{ col + 8 }}], ymm{{ col * 3 + row }}
+            add r{{ col + 8 }}, 32
+       {% endfor %}
+    {% endfor %}
+
+    jmp     {{L}}non_linear_loop
+
+{{L}}store_strides_generic:
+    {% for col in range(0, 4) %}
+       {% for row in range(0, 3) %}
+           {% for i in range(0, 4) %}
+                vextractps  dword ptr [r{{ col + 8 }}], xmm{{ col * 3 + row }}, {{i}}
+                add         r{{ col + 8 }}, rsi
+           {% endfor %}
+           vperm2f128  ymm{{ col * 3 + row }}, ymm{{ col * 3 + row }}, ymm{{ col * 3 + row }}, 1
+           {% for i in range(0, 4) %}
+                vextractps  dword ptr [r{{ col + 8 }}], xmm{{ col * 3 + row }}, {{i}}
+                add         r{{ col + 8 }}, rsi
+           {% endfor %}
+       {% endfor %}
+    {% endfor %}
+    jmp     {{L}}non_linear_loop
+
+{% set prefix = "avx" %}{% set type = "f32" %}{% set size = "24x4" %}{% set suffix = suffix %}{% set G = G %}{% set L = L %}{% include "postamble.j2" %}
diff --git a/linalg/x86_64/fma/avx_mmm_f32_32x3.S.j2 b/linalg/x86_64/fma/avx_mmm_f32_32x3.S.j2
new file mode 100644
index 0000000000..eaab63927b
--- /dev/null
+++ b/linalg/x86_64/fma/avx_mmm_f32_32x3.S.j2
@@ -0,0 +1,161 @@
+{#
+// vim: set syntax=asm :
+/* mmm 32 x 3, AVX without FMA (pre-Haswell: Sandy Bridge / Ivy Bridge)
+
+    ymm0 ymm4 ymm8
+    ymm1 ymm5 ymm9
+    ymm2 ymm6 ymm10
+    ymm3 ymm7 ymm11
+
+System V ABI:
+    args: rdi, rsi, rdx, rcx, r8, r9
+    preserve: rbx, rsp, rbp, r12, r13, r14, r15
+    scratch: rax, rdi, rsi, rdx, rcx, r8, r9, r10, r11
+    return: rax (+rdx)
+
+Windows ABI:
+    args: RCX, RDX, R8, R9
+    preserve: RBX, RBP, RDI, RSI, RSP, R12, R13, R14, R15, and XMM6-15
+    scratch: RAX, RCX, RDX, R8, R9, R10, R11, XMM0-5, and the upper portions of YMM0-15 and ZMM0-15
+    return: rax (+rdx)
+*/
+#}
+
+{% set prefix = "avx" %}{% set type = "f32" %}{% set size = "32x3" %}{% set suffix = suffix %}{% set G = G %}{% include "preamble.j2" %}
+
+{{L}}clear:
+    vzeroall
+    jmp     {{L}}non_linear_loop
+
+{{L}}add_mat_mul:
+    mov     rbx,    [rdi + 8]    // k
+    mov     rcx,    [rdi + 24]   // B
+    mov     rax,    [rdi + 16]   // A
+
+    test    rbx,    rbx
+    jz      {{L}}non_linear_loop
+
+{{L}}main_loop_packed_packed:
+    {% include "4x3/packed_packed_loop1/mul-add.S.raw" %}
+
+    dec             rbx
+    jnz             {{L}}main_loop_packed_packed
+
+    jmp             {{L}}non_linear_loop
+
+// NON LINEAR / ADDC
+
+{% set from = 0 %}{% set to = 11 %}{% set type = "f32" %}{% include "fma_mmm_f32_scalars.j2" %}
+{% set mr = 32 %}{% set from = 0 %}{% set to = 11 %}{% set type = "f32" %}{% include "fma_mmm_f32_per_rows.j2" %}
+{% set mr = 32 %}{% set from = 0 %}{% set to = 11 %}{% set type = "f32" %}{% include "fma_mmm_f32_per_cols.j2" %}
+{% set from = 0 %}{% set to = 11 %}{% include "fma_mmm_load_tile.j2" %}
+
+{{L}}add_unicast:
+    mov     r8,    [rdi + 8]           // c ptr
+    mov     rsi,    [rdi + 16]          // row stride
+    mov     rbx,    [rdi + 24]          // col stride
+
+    cmp     rsi, 4
+    jne     {{L}}unicast_generic
+
+    lea             r9,  [ r8 + rbx ]
+    lea             r10, [ r9 + rbx]
+
+{% for col in range(0, 3) %}
+    {% for row in range(0, 4) %}
+        vmovups     ymm12, [ r{{ col + 8 }} ]
+        add         r{{ col + 8 }}, 32
+        vaddps      ymm{{ col * 4 + row }}, ymm{{ col * 4 + row }}, ymm12
+    {% endfor %}
+{% endfor %}
+
+    jmp    {{L}}non_linear_loop
+
+// no vgatherdps before avx2: build each 8-row block with scalar inserts
+// (all VEX-encoded to avoid sse/avx transition stalls)
+{{L}}unicast_generic:
+    lea             r9,  [ r8 + rsi * 8 ]
+    lea             r10, [ r9 + rsi * 8 ]
+    lea             r11, [ r10 + rsi * 8 ]
+
+{% for col in range(0, 3) %}
+   {% for row in range(0, 4) %}
+    mov         rax, r{{ row + 8 }}
+    vmovss      xmm12, dword ptr [rax]
+    add         rax, rsi
+    {% for lane in range(1, 4) %}
+    vinsertps   xmm12, xmm12, dword ptr [rax], {{ lane * 16 }}
+    add         rax, rsi
+    {% endfor %}
+    vmovss      xmm13, dword ptr [rax]
+    add         rax, rsi
+    {% for lane in range(1, 4) %}
+    vinsertps   xmm13, xmm13, dword ptr [rax], {{ lane * 16 }}
+    add         rax, rsi
+    {% endfor %}
+    vinsertf128 ymm12, ymm12, xmm13, 1
+    add         r{{ row + 8 }}, rbx
+    vaddps      ymm{{ col * 4 + row }}, ymm{{ col * 4 + row }}, ymm12
+   {% endfor %}
+{% endfor %}
+
+    jmp    {{L}}non_linear_loop
+
+{{L}}add_row_col_products:
+    mov             rax, [ rdi + 8 ]
+    mov             rbx, [ rdi + 16 ]
+
+    vbroadcastss    ymm13, dword ptr [rbx]
+    vbroadcastss    ymm14, dword ptr [rbx + 4]
+    vbroadcastss    ymm15, dword ptr [rbx + 8]
+{% for i in range(0, 4) %}
+    vmovups         ymm12,  [rax + {{ i * 32 }}]
+    vmulps          ymm12,  ymm12, ymm13
+    vaddps          ymm{{ 0 + i }}, ymm{{ 0 + i }}, ymm12
+    vmovups         ymm12,  [rax + {{ i * 32 }}]
+    vmulps          ymm12,  ymm12, ymm14
+    vaddps          ymm{{ 4 + i }}, ymm{{ 4 + i }}, ymm12
+    vmovups         ymm12,  [rax + {{ i * 32 }}]
+    vmulps          ymm12,  ymm12, ymm15
+    vaddps          ymm{{ 8 + i }}, ymm{{ 8 + i }}, ymm12
+{% endfor %}
+    jmp    {{L}}non_linear_loop
+
+{{L}}store:
+    mov     r8,     [rdi + 8]           // c ptr
+    mov     rsi,    [rdi + 16]          // row stride
+    mov     rbx,    [rdi + 24]          // col stride
+
+    lea     r9,     [ r8 + rbx ]
+    lea     r10,    [ r8 + 2 * rbx ]
+
+    cmp         rsi, 4
+    jne         {{L}}store_strides_generic
+
+    {% for col in range(0, 3) %}
+        {% for row in range(0, 4) %}
+            vmovups ymmword ptr [r{{ col + 8 }}], ymm{{ col * 4 + row }}
+            add     r{{ col + 8 }}, 32
+       {% endfor %}
+    {% endfor %}
+
+    jmp     {{L}}non_linear_loop
+
+{{L}}store_strides_generic:
+
+    {% for col in range(0, 3) %}
+       {% for row in range(0, 4) %}
+           {% for i in range(0, 4) %}
+                vextractps  dword ptr [r{{ col + 8 }}], xmm{{ col * 4 + row }}, {{i}}
+                add         r{{ col + 8 }}, rsi
+           {% endfor %}
+           vperm2f128  ymm{{ col * 4 + row }}, ymm{{ col * 4 + row }}, ymm{{ col * 4 + row }}, 1
+           {% for i in range(0, 4) %}
+                vextractps  dword ptr [r{{ col + 8 }}], xmm{{ col * 4 + row }}, {{i}}
+                add         r{{ col + 8 }}, rsi
+           {% endfor %}
+       {% endfor %}
+    {% endfor %}
+    jmp     {{L}}non_linear_loop
+
+{% set prefix = "avx" %}{% set type = "f32" %}{% set size = "32x3" %}{% set suffix = suffix %}{% set G = G %}{% set L = L %}{% include "postamble.j2" %}
diff --git a/linalg/x86_64/fma/avx_mmm_f32_40x2.S.j2 b/linalg/x86_64/fma/avx_mmm_f32_40x2.S.j2
new file mode 100644
index 0000000000..93da2287eb
--- /dev/null
+++ b/linalg/x86_64/fma/avx_mmm_f32_40x2.S.j2
@@ -0,0 +1,155 @@
+{#
+// vim: set syntax=asm :
+/* mmm 40 x 2, AVX without FMA (pre-Haswell: Sandy Bridge / Ivy Bridge)
+
+    ymm0 ymm5
+    ymm1 ymm6
+    ymm2 ymm7
+    ymm3 ymm8
+    ymm4 ymm9
+
+System V ABI:
+    args: rdi, rsi, rdx, rcx, r8, r9
+    preserve: rbx, rsp, rbp, r12, r13, r14, r15
+    scratch: rax, rdi, rsi, rdx, rcx, r8, r9, r10, r11
+    return: rax (+rdx)
+
+Windows ABI:
+    args: RCX, RDX, R8, R9
+    preserve: RBX, RBP, RDI, RSI, RSP, R12, R13, R14, R15, and XMM6-15
+    scratch: RAX, RCX, RDX, R8, R9, R10, R11, XMM0-5, and the upper portions of YMM0-15 and ZMM0-15
+    return: rax (+rdx)
+*/
+#}
+
+{% set prefix = "avx" %}{% set type = "f32" %}{% set size = "40x2" %}{% set suffix = suffix %}{% set G = G %}{% include "preamble.j2" %}
+
+{{L}}clear:
+    vzeroall
+    jmp     {{L}}non_linear_loop
+
+{{L}}add_mat_mul:
+    mov     rcx,    [rdi + 24]   // B
+    mov     rax,    [rdi + 16]   // A
+
+    mov     rbx,    [rdi + 8]    // k
+    test    rbx,    rbx
+    jz      {{L}}non_linear_loop
+
+{{L}}main_loop_packed_packed:
+    {% include "5x2/packed_packed_loop1/mul-add.S.raw" %}
+
+    dec             rbx
+    jnz             {{L}}main_loop_packed_packed
+
+    jmp             {{L}}non_linear_loop
+
+// NON LINEAR / ADDC
+
+{% set from = 0 %}{% set to = 9 %}{% set type = "f32" %}{% include "fma_mmm_f32_scalars.j2" %}
+{% set mr = 40 %}{% set from = 0 %}{% set to = 9 %}{% set type = "f32" %}{% include "fma_mmm_f32_per_rows.j2" %}
+{% set mr = 40 %}{% set from = 0 %}{% set to = 9 %}{% set type = "f32" %}{% include "fma_mmm_f32_per_cols.j2" %}
+{% set from = 0 %}{% set to = 9 %}{% include "fma_mmm_load_tile.j2" %}
+
+{{L}}add_unicast:
+    mov     r8,    [rdi + 8]           // c ptr
+    mov     rsi,    [rdi + 16]          // row stride
+    mov     rbx,    [rdi + 24]          // col stride
+
+    cmp rsi, 4
+    jne {{L}}unicast_generic
+
+    lea             r9,  [ r8 + rbx ]
+
+{% for col in range(0, 2) %}
+    {% for row in range(0, 5) %}
+        vmovups ymm12,  [ r{{ col + 8 }} ]
+        add		r{{ col + 8 }}, 32
+        vaddps 	ymm{{ col * 5 + row }}, ymm{{ col * 5 + row }}, ymm12
+    {% endfor %}
+{% endfor %}
+    jmp    {{L}}non_linear_loop
+
+// no vgatherdps before avx2: build each 8-row block with scalar inserts
+// (all VEX-encoded to avoid sse/avx transition stalls)
+{{L}}unicast_generic:
+    lea             r9,  [ r8 + rsi * 8]
+    lea             r10, [ r9 + rsi * 8]
+    lea             r11, [ r10 + rsi * 8]
+    lea             r12, [ r11 + rsi * 8]
+
+{% for col in range(0, 2) %}
+   {% for row in range(0, 5) %}
+    mov         rax, r{{ row + 8 }}
+    vmovss      xmm12, dword ptr [rax]
+    add         rax, rsi
+    {% for lane in range(1, 4) %}
+    vinsertps   xmm12, xmm12, dword ptr [rax], {{ lane * 16 }}
+    add         rax, rsi
+    {% endfor %}
+    vmovss      xmm13, dword ptr [rax]
+    add         rax, rsi
+    {% for lane in range(1, 4) %}
+    vinsertps   xmm13, xmm13, dword ptr [rax], {{ lane * 16 }}
+    add         rax, rsi
+    {% endfor %}
+    vinsertf128 ymm12, ymm12, xmm13, 1
+    add         r{{ row + 8 }}, rbx
+    vaddps      ymm{{ col * 5 + row }}, ymm{{ col * 5 + row }}, ymm12
+   {% endfor %}
+{% endfor %}
+
+    jmp    {{L}}non_linear_loop
+
+{{L}}add_row_col_products:
+    mov             rax, [ rdi + 8 ]
+    mov             rbx, [ rdi + 16 ]
+
+    vbroadcastss    ymm10, dword ptr [rbx]
+    vbroadcastss    ymm11, dword ptr [rbx + 4]
+{% for i in range(0, 5) %}
+    vmovups         ymm12,  [rax + {{ i * 32 }}]
+    vmulps          ymm13,  ymm12, ymm10
+    vaddps          ymm{{ 0 + i }}, ymm{{ 0 + i }}, ymm13
+    vmulps          ymm13,  ymm12, ymm11
+    vaddps          ymm{{ 5 + i }}, ymm{{ 5 + i }}, ymm13
+{% endfor %}
+    jmp    {{L}}non_linear_loop
+
+
+{{L}}store:
+    mov     r8,     [rdi + 8]           // c ptr
+    mov     rsi,    [rdi + 16]          // row stride
+    mov     rbx,    [rdi + 24]          // col stride
+
+    lea     r9,     [ r8  +     rbx ]
+
+    cmp         rsi, 4
+    jne         {{L}}store_strides_generic
+
+    {% for col in range(0, 2) %}
+       {% for row in range(0, 5) %}
+            vmovups ymmword ptr [r{{ col + 8 }}], ymm{{ col * 5 + row }}
+            add 	r{{ col + 8 }}, 32
+       {% endfor %}
+    {% endfor %}
+
+    jmp     {{L}}non_linear_loop
+
+{{L}}store_strides_generic:
+    {% for col in range(0, 2) %}
+       {% for row in range(0, 5) %}
+           {% for i in range(0, 4) %}
+                vextractps  dword ptr [r{{ col + 8 }}], xmm{{ col * 5 + row }}, {{i}}
+                add         r{{ col + 8 }}, rsi
+           {% endfor %}
+           vperm2f128  ymm{{ col * 5 + row }}, ymm{{ col * 5 + row }}, ymm{{ col * 5 + row }}, 1
+           {% for i in range(0, 4) %}
+                vextractps  dword ptr [r{{ col + 8 }}], xmm{{ col * 5 + row }}, {{i}}
+                add         r{{ col + 8 }}, rsi
+           {% endfor %}
+       {% endfor %}
+    {% endfor %}
+    jmp     {{L}}non_linear_loop
+
+{% set prefix = "avx" %}{% set type = "f32" %}{% set size = "40x2" %}{% set suffix = suffix %}{% set G = G %}{% set L = L %}{% include "postamble.j2" %}
diff --git a/linalg/x86_64/fma/avx_mmm_f32_64x1.S.j2 b/linalg/x86_64/fma/avx_mmm_f32_64x1.S.j2
new file mode 100644
index 0000000000..59f672e413
--- /dev/null
+++ b/linalg/x86_64/fma/avx_mmm_f32_64x1.S.j2
@@ -0,0 +1,133 @@
+{#
+// vim: set syntax=asm :
+
+/* mmm 64 x 1, AVX without FMA (pre-Haswell: Sandy Bridge / Ivy Bridge)
+
+    ymm0
+    ymm1
+    ...
+    ymm7
+
+System V ABI:
+    args: rdi, rsi, rdx, rcx, r8, r9
+    preserve: rbx, rsp, rbp, r12, r13, r14, r15
+    scratch: rax, rdi, rsi, rdx, rcx, r8, r9, r10, r11
+    return: rax (+rdx)
+
+Windows ABI:
+    args: RCX, RDX, R8, R9
+    preserve: RBX, RBP, RDI, RSI, RSP, R12, R13, R14, R15, and XMM6-15
+    scratch: RAX, RCX, RDX, R8, R9, R10, R11, XMM0-5, and the upper portions of YMM0-15 and ZMM0-15
+    return: rax (+rdx)
+*/
+#}
+
+{% set prefix = "avx" %}{% set type = "f32" %}{% set size = "64x1" %}{% set suffix = suffix %}{% set G = G %}{% include "preamble.j2" %}
+
+{{L}}clear:
+    vzeroall
+    jmp     {{L}}non_linear_loop
+
+{{L}}add_mat_mul:
+    mov     rcx,    [rdi + 24]   // B
+    mov     rax,    [rdi + 16]   // A
+
+    mov     rbx,    [rdi + 8]    // k
+    test    rbx,    rbx
+    jz      {{L}}non_linear_loop
+
+{{align}} 16
+{{L}}main_loop_packed_packed:
+	{% include "8x1/packed_packed_loop1/mul-add.S.raw" %}
+
+    dec             rbx
+    jnz             {{L}}main_loop_packed_packed
+
+    jmp             {{L}}non_linear_loop
+
+{% set from = 0 %}{% set to = 7 %}{% set type = "f32" %}{% include "fma_mmm_f32_scalars.j2" %}
+{% set mr = 64 %}{% set from = 0 %}{% set to = 7 %}{% set type = "f32" %}{% include "fma_mmm_f32_per_rows.j2" %}
+{% set mr = 64 %}{% set from = 0 %}{% set to = 7 %}{% set type = "f32" %}{% include "fma_mmm_f32_per_cols.j2" %}
+{% set from = 0 %}{% set to = 7 %}{% include "fma_mmm_load_tile.j2" %}
+
+{{L}}add_unicast:
+    mov     r10,    [rdi + 8]           // c ptr
+    mov     rsi,    [rdi + 16]          // row stride
+
+	cmp rsi, 4
+	jne {{L}}add_unicast_generic
+
+    {% for row in range(0, 8) %}
+        vaddps ymm{{row}}, ymm{{row}}, [ r10 + {{ row * 32 }} ]
+    {% endfor %}
+    jmp    {{L}}non_linear_loop
+
+// no vgatherdps before avx2: build each 8-row block with scalar inserts
+// (all VEX-encoded to avoid sse/avx transition stalls)
+{{L}}add_unicast_generic:
+{% for i in range(0, 8) %}
+    vmovss      xmm12, dword ptr [r10]
+    add         r10, rsi
+    {% for lane in range(1, 4) %}
+    vinsertps   xmm12, xmm12, dword ptr [r10], {{ lane * 16 }}
+    add         r10, rsi
+    {% endfor %}
+    vmovss      xmm13, dword ptr [r10]
+    add         r10, rsi
+    {% for lane in range(1, 4) %}
+    vinsertps   xmm13, xmm13, dword ptr [r10], {{ lane * 16 }}
+    add         r10, rsi
+    {% endfor %}
+    vinsertf128 ymm12, ymm12, xmm13, 1
+
+    vaddps          ymm{{i}},   ymm{{i}},   ymm12
+{% endfor %}
+
+    jmp    {{L}}non_linear_loop
+
+{{L}}add_row_col_products:
+    mov             rax, [ rdi + 8 ]
+    mov             rbx, [ rdi + 16 ]
+
+    vbroadcastss    ymm14, dword ptr [rbx]
+
+{% for i in range(0, 8) %}
+    vmovups         ymm12,  [rax + {{ i * 32 }}]
+    vmulps          ymm12,  ymm12, ymm14
+    vaddps          ymm{{i}}, ymm{{i}}, ymm12
+{% endfor %}
+    jmp    {{L}}non_linear_loop
+
+{{L}}store:
+    mov     r8,     [rdi + 8]           // c ptr
+    mov     rsi,    [rdi + 16]          // row stride
+
+	cmp rsi, 4
+	jne {{L}}store_generic
+
+	{% for row in range(0, 8) %}
+        vmovups [r8 + {{ row * 32 }}], ymm{{row}}
+    {% endfor %}
+
+    jmp     {{L}}non_linear_loop
+
+{{L}}store_generic:
+
+    {% for vec in range(0, 8) %}
+        {% for half in range(0, 2) %}
+            {% if half == 0 %}
+                vmovaps xmm9, xmm{{vec}}
+            {% else %}
+                vperm2f128 ymm9, ymm{{vec}}, ymm{{vec}}, 1
+            {% endif %}
+            {% for row in range(0, 4) %}
+                vextractps  dword ptr [r8], xmm9, {{row}}
+                add         r8, rsi
+            {% endfor %}
+        {% endfor %}
+    {% endfor %}
+
+    jmp    {{L}}non_linear_loop
+
+
+{% set prefix = "avx" %}{% set type = "f32" %}{% set size = "64x1" %}{% set suffix = suffix %}{% set G = G %}{% set L = L %}{% include "postamble.j2" %}
diff --git a/linalg/x86_64/fma/avx_mmm_f32_8x8.S.j2 b/linalg/x86_64/fma/avx_mmm_f32_8x8.S.j2
new file mode 100644
index 0000000000..ea737da5a5
--- /dev/null
+++ b/linalg/x86_64/fma/avx_mmm_f32_8x8.S.j2
@@ -0,0 +1,145 @@
+{#
+// vim: set syntax=asm :
+
+/* mmm 8 x 8, AVX without FMA (pre-Haswell: Sandy Bridge / Ivy Bridge)
+
+    ymm0 ymm1 ymm2 ymm3 ymm4 ymm5 ymm6 ymm7
+
+System V ABI:
+    args: rdi, rsi, rdx, rcx, r8, r9
+    preserve: rbx, rsp, rbp, r12, r13, r14, r15
+    scratch: rax, rdi, rsi, rdx, rcx, r8, r9, r10, r11
+    return: rax (+rdx)
+
+Windows ABI:
+    args: RCX, RDX, R8, R9
+    preserve: RBX, RBP, RDI, RSI, RSP, R12, R13, R14, R15, and XMM6-15
+    scratch: RAX, RCX, RDX, R8, R9, R10, R11, XMM0-5, and the upper portions of YMM0-15 and ZMM0-15
+    return: rax (+rdx)
+*/
+#}
+
+{% set prefix = "avx" %}{% set type = "f32" %}{% set size = "8x8" %}{% set suffix = suffix %}{% set G = G %}{% include "preamble.j2" %}
+
+{{L}}clear:
+    vzeroall
+    jmp             {{L}}non_linear_loop
+
+{{L}}add_mat_mul:
+    mov     rbx,    [rdi + 24]   // B
+    mov     rax,    [rdi + 16]   // A
+
+    mov     rcx,    [rdi + 8]    // k
+    test    rcx,    rcx
+    jz      {{L}}non_linear_loop
+
+{{L}}main_loop_packed_packed:
+    vmovaps         ymm12,  [rax]
+
+    {% for i in range(0, 8) %}
+        vbroadcastss    ymm{{ 13 + i % 3 }}, dword ptr [rbx + {{i}} * 4]
+        vmulps          ymm{{ 13 + i % 3 }}, ymm{{ 13 + i % 3 }}, ymm12
+        vaddps          ymm{{i}}, ymm{{i}}, ymm{{ 13 + i % 3 }}
+    {% endfor %}
+
+    add             rax,    32
+    add             rbx,    32
+    dec             rcx
+    jnz             {{L}}main_loop_packed_packed
+    jmp             {{L}}non_linear_loop
+
+// NON LINEAR / ADDC
+
+{% set from = 0 %}{% set to = 7 %}{% set type = "f32" %}{% include "fma_mmm_f32_scalars.j2" %}
+{% set mr = 8 %}{% set from = 0 %}{% set to = 7 %}{% set type = "f32" %}{% include "fma_mmm_f32_per_rows.j2" %}
+{% set mr = 8 %}{% set from = 0 %}{% set to = 7 %}{% set type = "f32" %}{% include "fma_mmm_f32_per_cols.j2" %}
+{% set from = 0 %}{% set to = 7 %}{% include "fma_mmm_load_tile.j2" %}
+
+{{L}}add_unicast:
+
+    mov     r10,    [rdi + 8]           // c ptr
+    mov     rsi,    [rdi + 16]          // row stride
+    mov     rbx,    [rdi + 24]          // col stride
+
+    cmp     rsi,    4
+    jne     {{L}}add_unicast_generic
+
+{% for i in range(0, 8) %}
+    vmovups         ymm12,  [r10]
+    vaddps          ymm{{i}},   ymm{{i}},   ymm12
+    add             r10,    rbx
+{% endfor %}
+
+    jmp    {{L}}non_linear_loop
+
+// no vgatherdps before avx2: build each 8-row column with scalar inserts
+// (all VEX-encoded to avoid sse/avx transition stalls)
+{{L}}add_unicast_generic:
+{% for i in range(0, 8) %}
+    mov         r9,     r10
+    vmovss      xmm12, dword ptr [r9]
+    add         r9, rsi
+    {% for lane in range(1, 4) %}
+    vinsertps   xmm12, xmm12, dword ptr [r9], {{ lane * 16 }}
+    add         r9, rsi
+    {% endfor %}
+    vmovss      xmm13, dword ptr [r9]
+    add         r9, rsi
+    {% for lane in range(1, 4) %}
+    vinsertps   xmm13, xmm13, dword ptr [r9], {{ lane * 16 }}
+    add         r9, rsi
+    {% endfor %}
+    vinsertf128 ymm12, ymm12, xmm13, 1
+
+    vaddps          ymm{{i}},   ymm{{i}},   ymm12
+    add             r10,    rbx
+{% endfor %}
+
+    jmp    {{L}}non_linear_loop
+
+{{L}}add_row_col_products:
+    mov             rax, [ rdi + 8 ]
+    mov             rbx, [ rdi + 16 ]
+
+    vmovups         ymm12,  [rax]
+
+{% for i in range(0, 8) %}
+    vbroadcastss    ymm14, dword ptr [rbx + {{ i * 4 }} ]
+    vmulps          ymm14,  ymm14, ymm12
+    vaddps          ymm{{i}},   ymm{{i}},   ymm14
+{% endfor %}
+    jmp    {{L}}non_linear_loop
+
+{{L}}store:
+    mov     r8,     [rdi + 8]           // c ptr
+    mov     rsi,    [rdi + 16]          // row stride
+    mov     rbx,    [rdi + 24]          // col stride
+
+    // tops of cols
+    lea     r9,     [ r8 + rbx ]
+    lea     r10,    [ r8 + 2 * rbx ]
+    lea     r12,    [ r8 + 4 * rbx ]
+    lea     r11,    [ r10 + rbx ]
+    lea     r13,    [ r12 + rbx ]
+    lea     r14,    [ r12 + 2 * rbx ]
+    lea     r15,    [ r13 + 2 * rbx ]
+
+    {% for quarter in range(0, 2) %}
+        {% if quarter != 0 %}
+            // move next four rows at top (xmm0,2,..10)
+            {% for r in range(0, 8) %}
+                vperm2f128  ymm{{r}},   ymm{{r}},   ymm{{r}},  {{quarter}}
+            {% endfor %}
+        {% endif %}
+        {% for row in range(0, 4) %}
+            {% for i in range(0, 8) %}
+                vextractps  dword ptr [r{{ i + 8 }}], xmm{{i}}, {{row}}
+                add         r{{ i + 8 }}, rsi
+            {% endfor %}
+        {% endfor %}
+    {% endfor %}
+
+    jmp     {{L}}non_linear_loop
+
+
+{% set prefix = "avx" %}{% set type = "f32" %}{% set size = "8x8" %}{% set suffix = suffix %}{% set G = G %}{% set L = L %}{% include "postamble.j2" %}
diff --git a/linalg/x86_64/fma/avx_mmm_i32_8x4.S.j2 b/linalg/x86_64/fma/avx_mmm_i32_8x4.S.j2
new file mode 100644
index 0000000000..38aceecdb3
--- /dev/null
+++ b/linalg/x86_64/fma/avx_mmm_i32_8x4.S.j2
@@ -0,0 +1,571 @@
+{#
+// vim: set syntax=asm :
+
+/* mmm 8x4 i32, 128-bit VEX (AVX without AVX2) for pre-Haswell CPUs.
+
+    Col-major accumulator pairs: col j held in xmm{2j} (rows 0-3) and
+    xmm{2j+1} (rows 4-7), j in 0..4 -> xmm0-7.
+
+System V ABI:
+    args: rdi, rsi, rdx, rcx, r8, r9
+    preserve: rbx, rsp, rbp, r12, r13, r14, r15
+    scratch: rax, rdi, rsi, rdx, rcx, r8, r9, r10, r11
+    return: rax (+rdx)
+
+Windows ABI:
+    args: RCX, RDX, R8, R9
+    preserve: RBX, RBP, RDI, RSI, RSP, R12, R13, R14, R15, and XMM6-15
+    scratch: RAX, RCX, RDX, R8, R9, R10, R11, XMM0-5, and the upper portions of YMM0-15 and ZMM0-15
+    return: rax (+rdx)
+*/
+#}
+
+{% set prefix = "avx" %}{% set type = "i32" %}{% set size = "8x4" %}{% set suffix = suffix %}{% set G = G %}{% include "preamble.j2" %}
+
+{{L}}clear:
+    vzeroall
+    jmp     {{L}}non_linear_loop
+
+{{L}}add_mat_mul:
+    mov     r12,    [rdi + 32]   // packing
+    mov     rbx,    [rdi + 24]   // B
+    mov     rax,    [rdi + 16]   // A
+
+    mov     rcx,    [rdi + 8]    // k
+    test    rcx,    rcx
+    jz      {{L}}non_linear_loop
+
+    cmp     r12, 1
+    je      {{L}}main_loop_packed_packed_i8i8
+
+{{L}}main_loop_packed_packed:
+    vmovaps         xmm12,  [rax]
+    vmovaps         xmm13,  [rax + 16]
+
+    {% for j in range(0, 4) %}
+        vbroadcastss    xmm14, dword ptr [rbx + {{j}} * 4]
+        vpmulld         xmm15, xmm12, xmm14
+        vpaddd          xmm{{ j * 2 }}, xmm{{ j * 2 }}, xmm15
+        vpmulld         xmm15, xmm13, xmm14
+        vpaddd          xmm{{ j * 2 + 1 }}, xmm{{ j * 2 + 1 }}, xmm15
+    {% endfor %}
+
+    add             rax,    32
+    add             rbx,    16
+    dec             rcx
+    jnz             {{L}}main_loop_packed_packed
+
+    jmp             {{L}}non_linear_loop
+
+// i8 x i8 products fit i16 exactly, so the row column is multiplied in
+// 16-bit lanes then widened to the i32 accumulators.
+{{L}}main_loop_packed_packed_i8i8:
+    vmovq           xmm8, qword ptr [rax]          // 8 A bytes
+    vpmovsxbw       xmm8, xmm8                     // A rows as i16x8
+
+    vmovd           xmm9, dword ptr [rbx]          // 4 B bytes
+    vpmovsxbw       xmm9, xmm9                     // b0..b3 as i16 in words 0-3
+
+    {% for j in range(0, 4) %}
+        vpshuflw        xmm10, xmm9, {{ j * 85 }}
+        vpshufd         xmm10, xmm10, 0            // b{{j}} as i16x8
+        vpmullw         xmm10, xmm10, xmm8
+        vpmovsxwd       xmm11, xmm10               // rows 0-3 -> i32
+        vpaddd          xmm{{ j * 2 }}, xmm{{ j * 2 }}, xmm11
+        vpunpckhqdq     xmm10, xmm10, xmm10        // rows 4-7 to low words
+        vpmovsxwd       xmm10, xmm10
+        vpaddd          xmm{{ j * 2 + 1 }}, xmm{{ j * 2 + 1 }}, xmm10
+    {% endfor %}
+
+    add             rax,    8
+    add             rbx,    4
+    dec             rcx
+    jnz             {{L}}main_loop_packed_packed_i8i8
+
+    jmp             {{L}}non_linear_loop
+
+{% for label, op in [("scalar_min", "vpminsd"), ("scalar_max", "vpmaxsd"), ("scalar_mul", "vpmulld"), ("scalar_add", "vpaddd"), ("scalar_sub", "vpsubd")] %}
+{{L}}{{label}}:
+    vbroadcastss    xmm12, dword ptr [rdi + 8]
+    {% for reg in range(0, 8) %}
+    {{op}}          xmm{{reg}}, xmm12, xmm{{reg}}
+    {% endfor %}
+    jmp    {{L}}non_linear_loop
+{% endfor %}
+
+{{L}}scalar_sub_flipped:
+    vbroadcastss    xmm12, dword ptr [rdi + 8]
+    {% for reg in range(0, 8) %}
+    vpsubd          xmm{{reg}}, xmm{{reg}}, xmm12
+    {% endfor %}
+    jmp    {{L}}non_linear_loop
+
+{{L}}leaky_relu:
+    vbroadcastss    xmm15, dword ptr [rdi + 8]
+    vpxor           xmm14, xmm14, xmm14
+
+    {% for reg in range(0, 8) %}
+    vpmulld         xmm12, xmm{{reg}}, xmm15
+    vpcmpgtd        xmm13, xmm14, xmm{{reg}}
+    vblendvps       xmm{{reg}}, xmm{{reg}}, xmm12, xmm13
+    {% endfor %}
+
+    jmp    {{L}}non_linear_loop
+
+{% for label, op in [("per_row_min", "vpminsd"), ("per_row_max", "vpmaxsd"), ("per_row_add", "vpaddd"), ("per_row_mul", "vpmulld"), ("per_row_sub", "vpsubd")] %}
+{{L}}{{label}}:
+    mov             rax, [ rdi + 8 ]
+    vmovups         xmm12,  [rax]
+    vmovups         xmm13,  [rax + 16]
+    {% for j in range(0, 4) %}
+    {{op}}          xmm{{ j * 2 }}, xmm12, xmm{{ j * 2 }}
+    {{op}}          xmm{{ j * 2 + 1 }}, xmm13, xmm{{ j * 2 + 1 }}
+    {% endfor %}
+    jmp {{L}}non_linear_loop
+{% endfor %}
+
+{{L}}per_row_sub_flipped:
+    mov             rax, [ rdi + 8 ]
+    vmovups         xmm12,  [rax]
+    vmovups         xmm13,  [rax + 16]
+    {% for j in range(0, 4) %}
+    vpsubd          xmm{{ j * 2 }}, xmm{{ j * 2 }}, xmm12
+    vpsubd          xmm{{ j * 2 + 1 }}, xmm{{ j * 2 + 1 }}, xmm13
+    {% endfor %}
+    jmp {{L}}non_linear_loop
+
+{% for label, op in [("per_col_min", "vpminsd"), ("per_col_max", "vpmaxsd"), ("per_col_add", "vpaddd"), ("per_col_mul", "vpmulld"), ("per_col_sub", "vpsubd")] %}
+{{L}}{{label}}:
+    mov             rax, [ rdi + 8 ]
+    {% for j in range(0, 4) %}
+    vbroadcastss    xmm14, dword ptr [ rax + {{ j * 4 }} ]
+    {{op}}          xmm{{ j * 2 }}, xmm14, xmm{{ j * 2 }}
+    {{op}}          xmm{{ j * 2 + 1 }}, xmm14, xmm{{ j * 2 + 1 }}
+    {% endfor %}
+    jmp {{L}}non_linear_loop
+{% endfor %}
+
+{{L}}per_col_sub_flipped:
+    mov             rax, [ rdi + 8 ]
+    {% for j in range(0, 4) %}
+    vbroadcastss    xmm14, dword ptr [ rax + {{ j * 4 }} ]
+    vpsubd          xmm{{ j * 2 }}, xmm{{ j * 2 }}, xmm14
+    vpsubd          xmm{{ j * 2 + 1 }}, xmm{{ j * 2 + 1 }}, xmm14
+    {% endfor %}
+    jmp {{L}}non_linear_loop
+
+{{L}}load_tile:
+    mov          r8, [rdi + 8]
+    {% for reg in range(0, 8) %}
+    vmovups         xmm{{reg}}, xmmword ptr [r8 + {{ reg * 16 }}]
+    {% endfor %}
+    jmp    {{L}}non_linear_loop
+
+{{L}}add_unicast:
+    mov     r10,    [rdi + 8]           // c ptr
+    mov     rsi,    [rdi + 16]          // row stride
+    mov     rbx,    [rdi + 24]          // col stride
+    mov     r8,     [rdi + 32]          // item size
+
+    cmp     r8,    4
+    je      {{L}}non_linear_addc_i32
+
+    {% for j in range(0, 4) %}
+    mov     r9,     r10
+    {% for lane in range(0, 4) %}
+    mov     al, [ r9 ]
+    add     r9, rsi
+    movsx   eax, al
+    vpinsrd xmm12, xmm12, eax, {{lane}}
+    {% endfor %}
+    {% for lane in range(0, 4) %}
+    mov     al, [ r9 ]
+    add     r9, rsi
+    movsx   eax, al
+    vpinsrd xmm13, xmm13, eax, {{lane}}
+    {% endfor %}
+    vpaddd  xmm{{ j * 2 }}, xmm{{ j * 2 }}, xmm12
+    vpaddd  xmm{{ j * 2 + 1 }}, xmm{{ j * 2 + 1 }}, xmm13
+    add     r10, rbx
+    {% endfor %}
+
+    jmp    {{L}}non_linear_loop
+
+{{L}}non_linear_addc_i32:
+    {% for j in range(0, 4) %}
+    mov     r9,     r10
+    {% for lane in range(0, 4) %}
+    vpinsrd xmm12, xmm12, dword ptr [ r9 ], {{lane}}
+    add     r9, rsi
+    {% endfor %}
+    {% for lane in range(0, 4) %}
+    vpinsrd xmm13, xmm13, dword ptr [ r9 ], {{lane}}
+    add     r9, rsi
+    {% endfor %}
+    vpaddd  xmm{{ j * 2 }}, xmm{{ j * 2 }}, xmm12
+    vpaddd  xmm{{ j * 2 + 1 }}, xmm{{ j * 2 + 1 }}, xmm13
+    add     r10, rbx
+    {% endfor %}
+
+    jmp    {{L}}non_linear_loop
+
+{{L}}add_row_col_products:
+    mov             rax, [ rdi + 8 ]
+    mov             rbx, [ rdi + 16 ]
+
+    vmovups         xmm12,  [rax]
+    vmovups         xmm13,  [rax + 16]
+
+{% for j in range(0, 4) %}
+    vbroadcastss    xmm14, dword ptr [rbx + {{ j * 4 }} ]
+    vpmulld         xmm15, xmm12, xmm14
+    vpaddd          xmm{{ j * 2 }}, xmm{{ j * 2 }}, xmm15
+    vpmulld         xmm15, xmm13, xmm14
+    vpaddd          xmm{{ j * 2 + 1 }}, xmm{{ j * 2 + 1 }}, xmm15
+{% endfor %}
+    jmp    {{L}}non_linear_loop
+
+{{L}}q_scale:
+    mov             r8, [ rdi + 16 ]        // policy
+    vbroadcastss    xmm8, dword ptr [rdi + 24] // multi
+
+    mov             rax, 1
+    vmovq           xmm9, rax
+    vpunpcklqdq     xmm9, xmm9, xmm9        // xmm9 <- 1 as 2 qwords
+
+    mov             rax, [ rdi + 8 ]        // xmm10 <- shift + 31
+    add             rax, 31
+    vmovq           xmm10, rax
+    vpunpcklqdq     xmm10, xmm10, xmm10
+
+    vpsubq          xmm12, xmm10, xmm9      // shift+31 - 1
+    vpsllq          xmm11, xmm9, xmm12      // xmm11 <- 1 << (shift + 31 - 1)
+
+    cmp     r8, 1
+    je      {{L}}q_scale_rounding_zero
+    cmp     r8, 2
+    je      {{L}}q_scale_rounding_away
+    cmp     r8, 3
+    je      {{L}}q_scale_rounding_minus_inf
+    cmp     r8, 4
+    je      {{L}}q_scale_rounding_plus_inf
+    cmp     r8, 5
+    je      {{L}}q_scale_rounding_even
+    cmp     r8, 6
+    je      {{L}}q_scale_rounding_odd
+
+    jmp    {{L}}unsupported
+
+// even/odd i64 lane split as in the avx2 kernel: vpblendw 51 = dword-blend 0x5
+{{L}}q_scale_rounding_zero:           // signum * ( (abs + nudge) >> shift )
+{% for i in range(0, 8) %}
+    vpabsd      xmm14, xmm{{i}}
+    vpsrldq     xmm15, xmm14, 4
+    vpmuldq     xmm14, xmm14, xmm8
+    vpmuldq     xmm15, xmm15, xmm8
+
+    vpaddq      xmm14, xmm14, xmm11
+    vpaddq      xmm15, xmm15, xmm11
+
+    vpsubq      xmm14, xmm14, xmm9
+    vpsubq      xmm15, xmm15, xmm9
+
+    vpsrlq      xmm14, xmm14, xmm10
+    vpsrlq      xmm15, xmm15, xmm10
+
+    vpslldq     xmm15, xmm15, 4
+    vpblendw    xmm14, xmm15, xmm14, 51
+    vpsignd     xmm{{i}}, xmm14, xmm{{i}}
+{% endfor %}
+
+    jmp    {{L}}non_linear_loop
+
+{{L}}q_scale_rounding_away:           // signum * ( (abs + nudge) >> shift )
+{% for i in range(0, 8) %}
+    vpabsd      xmm14, xmm{{i}}
+    vpsrldq     xmm15, xmm14, 4
+    vpmuldq     xmm14, xmm14, xmm8
+    vpmuldq     xmm15, xmm15, xmm8
+
+    vpaddq      xmm14, xmm14, xmm11
+    vpaddq      xmm15, xmm15, xmm11
+
+    vpsrlq      xmm14, xmm14, xmm10
+    vpsrlq      xmm15, xmm15, xmm10
+
+    vpslldq     xmm15, xmm15, 4
+    vpblendw    xmm14, xmm15, xmm14, 51
+    vpsignd     xmm{{i}}, xmm14, xmm{{i}}
+{% endfor %}
+
+    jmp    {{L}}non_linear_loop
+
+{{L}}q_scale_rounding_minus_inf:
+{% for i in range(0, 8) %}
+    vpabsd      xmm14, xmm{{i}}
+    vpxor       xmm13, xmm13, xmm13
+    vpcmpgtd    xmm13, xmm{{i}}, xmm13
+    vpsrld      xmm13, xmm13, 31
+
+    vpsrldq     xmm15, xmm14, 4
+    vpmuldq     xmm14, xmm14, xmm8
+    vpmuldq     xmm15, xmm15, xmm8
+
+    vpaddq      xmm14, xmm14, xmm11
+    vpaddq      xmm15, xmm15, xmm11
+
+    vpxor       xmm12, xmm12, xmm12
+    vpblendw    xmm12, xmm12, xmm13, 51
+    vpsubq      xmm14, xmm14, xmm12
+
+    vpsrldq     xmm13, xmm13, 4
+    vpxor       xmm12, xmm12, xmm12
+    vpblendw    xmm12, xmm12, xmm13, 51
+    vpsubq      xmm15, xmm15, xmm12
+
+    vpsrlq      xmm14, xmm14, xmm10
+    vpsrlq      xmm15, xmm15, xmm10
+
+    vpslldq     xmm15, xmm15, 4
+    vpblendw    xmm14, xmm15, xmm14, 51
+    vpsignd     xmm{{i}}, xmm14, xmm{{i}}
+{% endfor %}
+
+    jmp    {{L}}non_linear_loop
+
+{{L}}q_scale_rounding_plus_inf:
+
+    vpshufd     xmm9, xmm9, 0               // xmm9 <- 1 as 4 dwords
+
+{% for i in range(0, 8) %}
+    vpabsd      xmm14, xmm{{i}}
+    vpxor       xmm13, xmm13, xmm13
+
+    vpcmpgtd    xmm13, xmm{{i}}, xmm13
+    vpaddd      xmm13, xmm13, xmm9          // if val >= 0 { 0i32 } else { 1i32 }
+
+    vpsrldq     xmm15, xmm14, 4
+    vpmuldq     xmm14, xmm14, xmm8
+    vpmuldq     xmm15, xmm15, xmm8
+
+    vpaddq      xmm14, xmm14, xmm11
+    vpaddq      xmm15, xmm15, xmm11
+
+    vpxor       xmm12, xmm12, xmm12
+    vpblendw    xmm12, xmm12, xmm13, 51
+    vpsubq      xmm14, xmm14, xmm12
+
+    vpsrldq     xmm13, xmm13, 4
+    vpxor       xmm12, xmm12, xmm12
+    vpblendw    xmm12, xmm12, xmm13, 51
+    vpsubq      xmm15, xmm15, xmm12
+
+    vpsrlq      xmm14, xmm14, xmm10
+    vpsrlq      xmm15, xmm15, xmm10
+
+    vpslldq     xmm15, xmm15, 4
+    vpblendw    xmm14, xmm15, xmm14, 51
+    vpsignd     xmm{{i}}, xmm14, xmm{{i}}
+{% endfor %}
+
+    jmp    {{L}}non_linear_loop
+
+{{L}}q_scale_rounding_even:
+{% for i in range(0, 8) %}
+    vpabsd      xmm14, xmm{{i}}
+    vpsrldq     xmm15, xmm14, 4
+    vpmuldq     xmm14, xmm14, xmm8
+    vpmuldq     xmm15, xmm15, xmm8
+
+    vpsrlq      xmm12, xmm14, xmm10
+    vpand       xmm12, xmm12, xmm9
+    vpaddq      xmm14, xmm14, xmm12
+    vpsubq      xmm14, xmm14, xmm9
+
+    vpsrlq      xmm12, xmm15, xmm10
+    vpand       xmm12, xmm12, xmm9
+    vpaddq      xmm15, xmm15, xmm12
+    vpsubq      xmm15, xmm15, xmm9
+
+    vpaddq      xmm14, xmm14, xmm11
+    vpaddq      xmm15, xmm15, xmm11
+
+    vpsrlq      xmm14, xmm14, xmm10
+    vpsrlq      xmm15, xmm15, xmm10
+
+    vpslldq     xmm15, xmm15, 4
+    vpblendw    xmm14, xmm15, xmm14, 51
+    vpsignd     xmm{{i}}, xmm14, xmm{{i}}
+{% endfor %}
+    jmp    {{L}}non_linear_loop
+
+{{L}}q_scale_rounding_odd:
+{% for i in range(0, 8) %}
+    vpabsd      xmm14, xmm{{i}}
+    vpsrldq     xmm15, xmm14, 4
+    vpmuldq     xmm14, xmm14, xmm8
+    vpmuldq     xmm15, xmm15, xmm8
+
+    vpsrlq      xmm12, xmm14, xmm10
+    vpand       xmm12, xmm12, xmm9
+    vpsubq      xmm14, xmm14, xmm12
+
+    vpsrlq      xmm12, xmm15, xmm10
+    vpand       xmm12, xmm12, xmm9
+    vpsubq      xmm15, xmm15, xmm12
+
+    vpaddq      xmm14, xmm14, xmm11
+    vpaddq      xmm15, xmm15, xmm11
+
+    vpsrlq      xmm14, xmm14, xmm10
+    vpsrlq      xmm15, xmm15, xmm10
+
+    vpslldq     xmm15, xmm15, 4
+    vpblendw    xmm14, xmm15, xmm14, 51
+    vpsignd     xmm{{i}}, xmm14, xmm{{i}}
+{% endfor %}
+
+    jmp    {{L}}non_linear_loop
+
+{{L}}q_shl:
+    mov             eax, [ rdi + 8 ]
+    vmovd           xmm10, eax
+
+{% for i in range(0, 8) %}
+    vpslld      xmm{{i}}, xmm{{i}}, xmm10
+{% endfor %}
+    jmp     {{L}}non_linear_loop
+
+{{L}}q_shr:
+    mov             r8, [ rdi + 16 ]        // policy
+
+    mov             eax, 1
+    vmovd           xmm9, eax
+    vpshufd         xmm9, xmm9, 0           // xmm9 <- 1u32 (4 times)
+
+    mov             eax, [ rdi + 8 ]
+    vmovd           xmm10, eax              // xmm10 <- shift (as count)
+
+    mov             ebx, 1
+    mov             cl, al
+    sub             cl, 1                   // cl <- shift - 1
+    sal             ebx, cl                 // ebx <- (1 << (shift - 1))
+    vmovd           xmm11, ebx
+    vpshufd         xmm11, xmm11, 0         // xmm11 <- "half"
+
+    vpxor           xmm12, xmm12, xmm12     // xmm12 <- zeroes
+
+    cmp     r8, 1
+    je      {{L}}q_shr_rounding_zero
+    cmp     r8, 2
+    je      {{L}}q_shr_rounding_away
+    cmp     r8, 3
+    je      {{L}}q_shr_rounding_minus_inf
+    cmp     r8, 4
+    je      {{L}}q_shr_rounding_plus_inf
+    cmp     r8, 5
+    je      {{L}}q_shr_rounding_even
+    cmp     r8, 6
+    je      {{L}}q_shr_rounding_odd
+
+    jmp    {{L}}unsupported
+
+{{L}}q_shr_rounding_zero:
+{% for i in range(0, 8) %}
+    vpabsd      xmm14, xmm{{i}}
+    vpsubd      xmm14, xmm14, xmm9
+    vpaddd      xmm14, xmm14, xmm11
+    vpsrad      xmm14, xmm14, xmm10
+    vpsignd     xmm{{i}}, xmm14, xmm{{i}}
+{% endfor %}
+    jmp     {{L}}non_linear_loop
+
+{{L}}q_shr_rounding_away:
+{% for i in range(0, 8) %}
+    vpabsd      xmm14, xmm{{i}}
+    vpaddd      xmm14, xmm14, xmm11
+    vpsrad      xmm14, xmm14, xmm10
+    vpsignd     xmm{{i}}, xmm14, xmm{{i}}
+{% endfor %}
+    jmp     {{L}}non_linear_loop
+
+{{L}}q_shr_rounding_minus_inf:
+{% for i in range(0, 8) %}
+    vpsubd  xmm{{i}}, xmm{{i}}, xmm9
+    vpaddd  xmm{{i}}, xmm{{i}}, xmm11
+    vpsrad  xmm{{i}}, xmm{{i}}, xmm10
+{% endfor %}
+    jmp     {{L}}non_linear_loop
+
+{{L}}q_shr_rounding_plus_inf:
+{% for i in range(0, 8) %}
+    vpaddd  xmm{{i}}, xmm{{i}}, xmm11
+    vpsrad  xmm{{i}}, xmm{{i}}, xmm10
+{% endfor %}
+    jmp     {{L}}non_linear_loop
+
+{{L}}q_shr_rounding_even:
+{% for i in range(0, 8) %}
+    vpabsd  xmm14, xmm{{i}}
+    vpsrad  xmm13, xmm14, xmm10
+    vpand   xmm13, xmm13, xmm9
+    vpsubd  xmm13, xmm13, xmm9          // nudge = ((abs >>l shift) & 0x01) - 1
+    vpaddd  xmm14, xmm14, xmm13
+    vpaddd  xmm14, xmm14, xmm11
+    vpsrad  xmm14, xmm14, xmm10
+    vpsignd xmm{{i}}, xmm14, xmm{{i}}
+{% endfor %}
+    jmp     {{L}}non_linear_loop
+
+{{L}}q_shr_rounding_odd:
+{% for i in range(0, 8) %}
+    vpabsd  xmm14, xmm{{i}}
+    vpsrad  xmm13, xmm14, xmm10
+    vpand   xmm13, xmm13, xmm9
+    vpsubd  xmm13, xmm12, xmm13         // nudge = - ((abs >>l shift) & 0x01)
+    vpaddd  xmm14, xmm14, xmm13
+    vpaddd  xmm14, xmm14, xmm11
+    vpsrad  xmm14, xmm14, xmm10
+    vpsignd xmm{{i}}, xmm14, xmm{{i}}
+{% endfor %}
+    jmp     {{L}}non_linear_loop
+
+{{L}}store:
+    mov     r8,     [rdi + 8]           // c ptr
+    mov     rsi,    [rdi + 16]          // row stride
+    mov     rdx,    [rdi + 24]          // col stride
+    mov     rcx,    [rdi + 32]          // item size
+
+    cmp     rcx,    4
+    je      {{L}}store_strides_i32
+
+    {% for col in range(0, 4) %}
+        mov r10, r8
+        {% for half in range(0, 2) %}
+            {% for row in range(0, 4) %}
+            vextractps  ebx, xmm{{ col * 2 + half }}, {{row}}
+            mov         byte ptr [r10], bl
+            add         r10, rsi
+            {% endfor %}
+        {% endfor %}
+        add r8, rdx
+    {% endfor %}
+
+    jmp     {{L}}non_linear_loop
+
+{{L}}store_strides_i32:
+    {% for col in range(0, 4) %}
+        mov r10, r8
+        {% for half in range(0, 2) %}
+            {% for row in range(0, 4) %}
+            vextractps  ebx, xmm{{ col * 2 + half }}, {{row}}
+            mov         dword ptr [r10], ebx
+            add         r10, rsi
+            {% endfor %}
+        {% endfor %}
+        add r8, rdx
+    {% endfor %}
+
+    jmp     {{L}}non_linear_loop
+
+{% set prefix = "avx" %}{% set type = "i32" %}{% set size = "8x4" %}{% set suffix = suffix %}{% set G = G %}{% set L = L %}{% include "postamble.j2" %}
diff --git a/linalg/x86_64/fma/avx_sigmoid_f32.S.j2 b/linalg/x86_64/fma/avx_sigmoid_f32.S.j2
new file mode 100644
index 0000000000..dbaa4c5a4a
--- /dev/null
+++ b/linalg/x86_64/fma/avx_sigmoid_f32.S.j2
@@ -0,0 +1,364 @@
+{#
+// vim: set syntax=asm :
+
+System V ABI:
+    args: rdi, rsi, rdx, rcx, r8, r9
+    preserve: rbx, rsp, rbp, r12, r13, r14, r15
+    scratch: rax, rdi, rsi, rdx, rcx, r8, r9, r10, r11
+    return: rax (+rdx)
+
+Windows ABI:
+    args: RCX, RDX, R8, R9
+    preserve: RBX, RBP, RDI, RSI, RSP, R12, R13, R14, R15, and XMM6-15
+    scratch: RAX, RCX, RDX, R8, R9, R10, R11, XMM0-5, and the upper portions of YMM0-15 and ZMM0-15
+    return: rax (+rdx)
+
+#}
+
+{% if msvc %}
+
+_text segment
+avx_sigmoid_f32_{{suffix}} proc
+
+{% else %}
+
+.intel_syntax noprefix
+.text
+.p2align 5
+.globl {{G}}avx_sigmoid_f32_{{suffix}}
+{{G}}avx_sigmoid_f32_{{suffix}}:
+.cfi_startproc
+{% endif %}
+
+    push        rbp
+    mov         rbp, rsp
+
+
+{% if family == "windows" %}
+// https://www.agner.org/optimize/calling_conventions.pdf xmm6-15 are not scratch
+// https://stackoverflow.com/questions/43358429/save-value-of-xmm-registers
+    and rsp,-16
+    lea rsp,[rsp-160]
+    vmovaps [rsp], xmm6
+    vmovaps [rsp+16*1],xmm7
+    vmovaps [rsp+16*2],xmm8
+    vmovaps [rsp+16*3],xmm9
+    vmovaps [rsp+16*4],xmm10
+    vmovaps [rsp+16*5],xmm11
+    vmovaps [rsp+16*6],xmm12
+    vmovaps [rsp+16*7],xmm13
+    vmovaps [rsp+16*8],xmm14
+    vmovaps [rsp+16*9],xmm15
+
+    // move around arguments to mimick SysV rdi,rsi passing
+    push        rdi
+    push        rsi
+    mov         rdi, rcx
+    mov         rsi, rdx
+
+{% endif %}
+
+    push        rbx
+    push        r12
+    push        r13
+    push        r14
+    push        r15
+
+    sub         rsp, 8
+
+{% if family == "unix" %}
+// FIXME
+// .cfi_def_cfa_offset 64 
+{% endif %}
+
+    stmxcsr     [rsp + 4]
+{% if msvc %}
+    mov         rax, 1FC0h
+{% else %}
+    mov         rax, 0x1FC0
+{% endif %}
+    mov         [rsp], eax
+    ldmxcsr     [rsp]
+// ----------------------------------------------------------------------
+
+    cmp     rsi, 0
+    je      {{L}}done
+
+    cmp     rsi, 32
+    jl      {{L}}loop_1
+
+{{L}}loop_4:
+
+    vmovaps         ymm4, [rdi]
+    vmovaps         ymm5, [rdi + 32]
+    vmovaps         ymm6, [rdi + 64]
+    vmovaps         ymm7, [rdi + 96]
+
+    vbroadcastss    ymm0, dword ptr [{{offset}} {{L}}coeffs_num_low]
+    vbroadcastss    ymm1, dword ptr [{{offset}} {{L}}coeffs_num_high]
+    vbroadcastss    ymm2, dword ptr [{{offset}} {{L}}coeffs_num_alpha_13]
+    vbroadcastss    ymm3, dword ptr [{{offset}} {{L}}coeffs_num_alpha_11]
+
+    vmaxps          ymm4, ymm4, ymm0
+    vmaxps          ymm5, ymm5, ymm0
+    vmaxps          ymm6, ymm6, ymm0
+    vmaxps          ymm7, ymm7, ymm0
+    vbroadcastss    ymm0, dword ptr [{{offset}} {{L}}coeffs_num_alpha_9]
+
+    vminps          ymm4, ymm4, ymm1
+    vminps          ymm5, ymm5, ymm1
+    vminps          ymm6, ymm6, ymm1
+    vminps          ymm7, ymm7, ymm1        // ymm4..7 <- x
+    vbroadcastss    ymm1, dword ptr [{{offset}} {{L}}coeffs_num_alpha_7]
+
+    vmulps          ymm8, ymm4, ymm4
+    vmulps          ymm9, ymm5, ymm5
+    vmulps          ymm10, ymm6, ymm6
+    vmulps          ymm11, ymm7, ymm7        // ymm8..11 <- x^2
+
+    vmovaps         ymm12, ymm2
+    vmovaps         ymm13, ymm2
+    vmovaps         ymm14, ymm2
+    vmovaps         ymm15, ymm2
+    vbroadcastss    ymm2, dword ptr [{{offset}} {{L}}coeffs_num_alpha_5]
+    vmulps          ymm12, ymm12, ymm8
+    vaddps          ymm12, ymm12, ymm3
+    vmulps          ymm13, ymm13, ymm9
+    vaddps          ymm13, ymm13, ymm3
+    vmulps          ymm14, ymm14, ymm10
+    vaddps          ymm14, ymm14, ymm3
+    vmulps          ymm15, ymm15, ymm11
+    vaddps          ymm15, ymm15, ymm3
+    vbroadcastss    ymm3, dword ptr [{{offset}} {{L}}coeffs_num_alpha_3]
+    vmulps          ymm12, ymm12, ymm8
+    vaddps          ymm12, ymm12, ymm0
+    vmulps          ymm13, ymm13, ymm9
+    vaddps          ymm13, ymm13, ymm0
+    vmulps          ymm14, ymm14, ymm10
+    vaddps          ymm14, ymm14, ymm0
+    vmulps          ymm15, ymm15, ymm11
+    vaddps          ymm15, ymm15, ymm0
+    vbroadcastss    ymm0, dword ptr [{{offset}} {{L}}coeffs_num_alpha_1]
+    vmulps          ymm12, ymm12, ymm8
+    vaddps          ymm12, ymm12, ymm1
+    vmulps          ymm13, ymm13, ymm9
+    vaddps          ymm13, ymm13, ymm1
+    vmulps          ymm14, ymm14, ymm10
+    vaddps          ymm14, ymm14, ymm1
+    vmulps          ymm15, ymm15, ymm11
+    vaddps          ymm15, ymm15, ymm1
+    vbroadcastss    ymm1, dword ptr [{{offset}} {{L}}coeffs_num_beta_6]
+    vmulps          ymm12, ymm12, ymm8
+    vaddps          ymm12, ymm12, ymm2
+    vmulps          ymm13, ymm13, ymm9
+    vaddps          ymm13, ymm13, ymm2
+    vmulps          ymm14, ymm14, ymm10
+    vaddps          ymm14, ymm14, ymm2
+    vmulps          ymm15, ymm15, ymm11
+    vaddps          ymm15, ymm15, ymm2
+    vbroadcastss    ymm2, dword ptr [{{offset}} {{L}}coeffs_num_beta_4]
+    vmulps          ymm12, ymm12, ymm8
+    vaddps          ymm12, ymm12, ymm3
+    vmulps          ymm13, ymm13, ymm9
+    vaddps          ymm13, ymm13, ymm3
+    vmulps          ymm14, ymm14, ymm10
+    vaddps          ymm14, ymm14, ymm3
+    vmulps          ymm15, ymm15, ymm11
+    vaddps          ymm15, ymm15, ymm3
+    vbroadcastss    ymm3, dword ptr [{{offset}} {{L}}coeffs_num_beta_2]
+    vmulps          ymm12, ymm12, ymm8
+    vaddps          ymm12, ymm12, ymm0
+    vmulps          ymm13, ymm13, ymm9
+    vaddps          ymm13, ymm13, ymm0
+    vmulps          ymm14, ymm14, ymm10
+    vaddps          ymm14, ymm14, ymm0
+    vmulps          ymm15, ymm15, ymm11
+    vaddps          ymm15, ymm15, ymm0
+    vbroadcastss    ymm0, dword ptr [{{offset}} {{L}}coeffs_num_beta_0]
+    vmulps          ymm4, ymm4, ymm12
+    vmulps          ymm5, ymm5, ymm13
+    vmulps          ymm6, ymm6, ymm14
+    vmulps          ymm7, ymm7, ymm15   // ymm4..7 <- num
+
+    vmovaps         ymm12, ymm1
+    vmovaps         ymm13, ymm1
+    vmovaps         ymm14, ymm1
+    vmovaps         ymm15, ymm1
+
+    vbroadcastss    ymm1, dword ptr [{{offset}} {{L}}coeffs_num_half]
+    vmulps          ymm12, ymm12, ymm8
+    vaddps          ymm12, ymm12, ymm2
+    vmulps          ymm13, ymm13, ymm9
+    vaddps          ymm13, ymm13, ymm2
+    vmulps          ymm14, ymm14, ymm10
+    vaddps          ymm14, ymm14, ymm2
+    vmulps          ymm15, ymm15, ymm11
+    vaddps          ymm15, ymm15, ymm2
+    vmulps          ymm12, ymm12, ymm8
+    vaddps          ymm12, ymm12, ymm3
+    vmulps          ymm13, ymm13, ymm9
+    vaddps          ymm13, ymm13, ymm3
+    vmulps          ymm14, ymm14, ymm10
+    vaddps          ymm14, ymm14, ymm3
+    vmulps          ymm15, ymm15, ymm11
+    vaddps          ymm15, ymm15, ymm3
+    vmulps          ymm12, ymm12, ymm8
+    vaddps          ymm12, ymm12, ymm0
+    vmulps          ymm13, ymm13, ymm9
+    vaddps          ymm13, ymm13, ymm0
+    vmulps          ymm14, ymm14, ymm10
+    vaddps          ymm14, ymm14, ymm0
+    vmulps          ymm15, ymm15, ymm11
+    vaddps          ymm15, ymm15, ymm0  // ymm12..15 <- denum
+
+    vdivps          ymm4, ymm4, ymm12
+    vdivps          ymm5, ymm5, ymm13
+    vdivps          ymm6, ymm6, ymm14
+    vdivps          ymm7, ymm7, ymm15
+    vaddps          ymm4, ymm4, ymm1
+    vaddps          ymm5, ymm5, ymm1
+    vaddps          ymm6, ymm6, ymm1
+    vaddps          ymm7, ymm7, ymm1
+
+    vmovaps [rdi], ymm4
+    vmovaps [rdi + 32], ymm5
+    vmovaps [rdi + 64], ymm6
+    vmovaps [rdi + 96], ymm7
+
+    add     rdi, 128
+    sub     rsi, 32
+    cmp     rsi, 32
+    jg      {{L}}loop_4
+
+    cmp     rsi, 0
+    je      {{L}}done
+
+{{L}}loop_1:
+    vmovaps         ymm4, [rdi]
+
+    vbroadcastss    ymm0, dword ptr [{{offset}} {{L}}coeffs_num_low]
+    vbroadcastss    ymm1, dword ptr [{{offset}} {{L}}coeffs_num_high]
+    vbroadcastss    ymm2, dword ptr [{{offset}} {{L}}coeffs_num_alpha_13]
+    vbroadcastss    ymm3, dword ptr [{{offset}} {{L}}coeffs_num_alpha_11]
+
+    vmaxps          ymm4, ymm4, ymm0
+    vbroadcastss    ymm0, dword ptr [{{offset}} {{L}}coeffs_num_alpha_9]
+
+    vminps          ymm4, ymm4, ymm1        // ymm4 <- x
+    vbroadcastss    ymm1, dword ptr [{{offset}} {{L}}coeffs_num_alpha_7]
+
+    vmulps          ymm8, ymm4, ymm4        // ymm8 <- x^2
+
+    vmovaps         ymm12, ymm2
+    vbroadcastss    ymm2, dword ptr [{{offset}} {{L}}coeffs_num_alpha_5]
+    vmulps          ymm12, ymm12, ymm8
+    vaddps          ymm12, ymm12, ymm3
+    vbroadcastss    ymm3, dword ptr [{{offset}} {{L}}coeffs_num_alpha_3]
+    vmulps          ymm12, ymm12, ymm8
+    vaddps          ymm12, ymm12, ymm0
+    vbroadcastss    ymm0, dword ptr [{{offset}} {{L}}coeffs_num_alpha_1]
+    vmulps          ymm12, ymm12, ymm8
+    vaddps          ymm12, ymm12, ymm1
+    vbroadcastss    ymm1, dword ptr [{{offset}} {{L}}coeffs_num_beta_6]
+    vmulps          ymm12, ymm12, ymm8
+    vaddps          ymm12, ymm12, ymm2
+    vbroadcastss    ymm2, dword ptr [{{offset}} {{L}}coeffs_num_beta_4]
+    vmulps          ymm12, ymm12, ymm8
+    vaddps          ymm12, ymm12, ymm3
+    vbroadcastss    ymm3, dword ptr [{{offset}} {{L}}coeffs_num_beta_2]
+    vmulps          ymm12, ymm12, ymm8
+    vaddps          ymm12, ymm12, ymm0
+    vbroadcastss    ymm0, dword ptr [{{offset}} {{L}}coeffs_num_beta_0]
+    vmulps          ymm4, ymm4, ymm12
+
+    vmovaps         ymm12, ymm1
+    vbroadcastss    ymm1, dword ptr [{{offset}} {{L}}coeffs_num_half]
+    vmulps          ymm12, ymm12, ymm8
+    vaddps          ymm12, ymm12, ymm2
+    vmulps          ymm12, ymm12, ymm8
+    vaddps          ymm12, ymm12, ymm3
+    vmulps          ymm12, ymm12, ymm8
+    vaddps          ymm12, ymm12, ymm0
+
+    vdivps          ymm4, ymm4, ymm12
+    vaddps          ymm4, ymm4, ymm1
+
+    vmovaps [rdi], ymm4
+    add     rdi, 32
+    sub     rsi, 8
+    jnz     {{L}}loop_1
+{{L}}done:
+
+// ----------------------------------------------------------------------
+
+    ldmxcsr     [rsp + 4]
+
+    add         rsp, 8
+
+    pop r15
+    pop r14
+    pop r13
+    pop r12
+    pop rbx
+
+{% if family == "windows" %}
+    pop rsi
+    pop rdi
+
+    vmovaps xmm15, [rsp+16*9]
+    vmovaps xmm14, [rsp+16*8]
+    vmovaps xmm13, [rsp+16*7]
+    vmovaps xmm12, [rsp+16*6]
+    vmovaps xmm11, [rsp+16*5]
+    vmovaps xmm10, [rsp+16*4]
+    vmovaps xmm9, [rsp+16*3]
+    vmovaps xmm8, [rsp+16*2]
+    vmovaps xmm7, [rsp+16*1]
+    vmovaps xmm6, [rsp]
+{% endif %}
+
+    mov rsp, rbp
+    pop rbp
+    ret
+
+{% set float %}{% if msvc %} real4 {%else%} .float {%endif%}{% endset %}
+
+{{L}}coeffs_num_low:
+    {{float}} -18.6                   // low
+{{L}}coeffs_num_high:
+    {{float}} 18.6                     // high         
+
+{{L}}coeffs_num_alpha_13:
+    {{float}} -4.433153405e-18
+{{L}}coeffs_num_alpha_11:
+    {{float}} 1.169974371e-14
+{{L}}coeffs_num_alpha_9:
+    {{float}} -1.875289645e-11
+{{L}}coeffs_num_alpha_7:
+    {{float}} 4.257889523e-8
+{{L}}coeffs_num_alpha_5:
+    {{float}} 0.00004811817576
+{{L}}coeffs_num_alpha_3:
+    {{float}} 0.008163842030
+{{L}}coeffs_num_alpha_1:
+    {{float}} 0.2499999971
+
+{{L}}coeffs_num_beta_6:
+    {{float}} 3.922935744e-6
+{{L}}coeffs_num_beta_4:
+    {{float}} 0.001524872358
+{{L}}coeffs_num_beta_2:
+    {{float}} 0.1159886749
+{{L}}coeffs_num_beta_0:
+    {{float}} 1.0;
+
+{{L}}coeffs_num_half:
+    {{float}} 0.5
+
+{% if msvc %}
+avx_sigmoid_f32_{{suffix}} endp
+_text ends
+end
+{% else %}
+.cfi_endproc
+{% endif %}
diff --git a/linalg/x86_64/fma/avx_tanh_f32.S.j2 b/linalg/x86_64/fma/avx_tanh_f32.S.j2
new file mode 100644
index 0000000000..7039b34691
--- /dev/null
+++ b/linalg/x86_64/fma/avx_tanh_f32.S.j2
@@ -0,0 +1,358 @@
+{#
+// vim: set syntax=asm :
+
+System V ABI:
+    args: rdi, rsi, rdx, rcx, r8, r9
+    preserve: rbx, rsp, rbp, r12, r13, r14, r15
+    scratch: rax, rdi, rsi, rdx, rcx, r8, r9, r10, r11
+    return: rax (+rdx)
+
+Windows ABI:
+    args: RCX, RDX, R8, R9
+    preserve: RBX, RBP, RDI, RSI, RSP, R12, R13, R14, R15, and XMM6-15
+    scratch: RAX, RCX, RDX, R8, R9, R10, R11, XMM0-5, and the upper portions of YMM0-15 and ZMM0-15
+    return: rax (+rdx)
+
+#}
+
+{% if msvc %}
+
+_text segment
+avx_tanh_f32_{{suffix}} proc
+
+{% else %}
+
+.intel_syntax noprefix
+.text
+.p2align 5
+.globl {{G}}avx_tanh_f32_{{suffix}}
+{{G}}avx_tanh_f32_{{suffix}}:
+.cfi_startproc
+{% endif %}
+
+    push        rbp
+    mov         rbp, rsp
+
+
+{% if family == "windows" %}
+// https://www.agner.org/optimize/calling_conventions.pdf xmm6-15 are not scratch
+// https://stackoverflow.com/questions/43358429/save-value-of-xmm-registers
+    and rsp,-16
+    lea rsp,[rsp-160]
+    vmovaps [rsp], xmm6
+    vmovaps [rsp+16*1],xmm7
+    vmovaps [rsp+16*2],xmm8
+    vmovaps [rsp+16*3],xmm9
+    vmovaps [rsp+16*4],xmm10
+    vmovaps [rsp+16*5],xmm11
+    vmovaps [rsp+16*6],xmm12
+    vmovaps [rsp+16*7],xmm13
+    vmovaps [rsp+16*8],xmm14
+    vmovaps [rsp+16*9],xmm15
+
+    // move around arguments to mimick SysV rdi,rsi passing
+    push        rdi
+    push        rsi
+    mov         rdi, rcx
+    mov         rsi, rdx
+
+{% endif %}
+
+    push        rbx
+    push        r12
+    push        r13
+    push        r14
+    push        r15
+
+    sub         rsp, 8
+
+{% if family == "unix" %}
+// FIXME
+// .cfi_def_cfa_offset 64 
+{% endif %}
+
+    stmxcsr     [rsp + 4]
+{% if msvc %}
+    mov         rax, 1FC0h
+{% else %}
+    mov         rax, 0x1FC0
+{% endif %}
+    mov         [rsp], eax
+    ldmxcsr     [rsp]
+// ----------------------------------------------------------------------
+
+{% set offset %}{% if msvc %} offset {%else%} rip + {%endif%} {% endset %}
+
+    cmp     rsi, 0
+    je      {{L}}done
+
+    cmp     rsi, 32
+    jl      {{L}}loop_1
+
+{{L}}loop_4:
+
+    vmovaps         ymm4, [rdi]
+    vmovaps         ymm5, [rdi + 32]
+    vmovaps         ymm6, [rdi + 64]
+    vmovaps         ymm7, [rdi + 96]
+
+    vbroadcastss    ymm0, dword ptr [{{offset}} {{L}}coeffs_num_low]
+    vbroadcastss    ymm1, dword ptr [{{offset}} {{L}}coeffs_num_high]
+    vbroadcastss    ymm2, dword ptr [{{offset}} {{L}}coeffs_num_alpha_13]
+    vbroadcastss    ymm3, dword ptr [{{offset}} {{L}}coeffs_num_alpha_11]
+
+    vmaxps          ymm4, ymm4, ymm0
+    vmaxps          ymm5, ymm5, ymm0
+    vmaxps          ymm6, ymm6, ymm0
+    vmaxps          ymm7, ymm7, ymm0
+    vbroadcastss    ymm0, dword ptr [{{offset}} {{L}}coeffs_num_alpha_9]
+
+    vminps          ymm4, ymm4, ymm1
+    vminps          ymm5, ymm5, ymm1
+    vminps          ymm6, ymm6, ymm1
+    vminps          ymm7, ymm7, ymm1        // ymm4..7 <- x
+    vbroadcastss    ymm1, dword ptr [{{offset}} {{L}}coeffs_num_alpha_7]
+
+    vmulps          ymm8, ymm4, ymm4
+    vmulps          ymm9, ymm5, ymm5
+    vmulps          ymm10, ymm6, ymm6
+    vmulps          ymm11, ymm7, ymm7        // ymm8..11 <- x^2
+
+    vmovaps         ymm12, ymm2
+    vmovaps         ymm13, ymm2
+    vmovaps         ymm14, ymm2
+    vmovaps         ymm15, ymm2
+    vbroadcastss    ymm2, dword ptr [{{offset}} {{L}}coeffs_num_alpha_5]
+    vmulps          ymm12, ymm12, ymm8
+    vaddps          ymm12, ymm12, ymm3
+    vmulps          ymm13, ymm13, ymm9
+    vaddps          ymm13, ymm13, ymm3
+    vmulps          ymm14, ymm14, ymm10
+    vaddps          ymm14, ymm14, ymm3
+    vmulps          ymm15, ymm15, ymm11
+    vaddps          ymm15, ymm15, ymm3
+    vbroadcastss    ymm3, dword ptr [{{offset}} {{L}}coeffs_num_alpha_3]
+    vmulps          ymm12, ymm12, ymm8
+    vaddps          ymm12, ymm12, ymm0
+    vmulps          ymm13, ymm13, ymm9
+    vaddps          ymm13, ymm13, ymm0
+    vmulps          ymm14, ymm14, ymm10
+    vaddps          ymm14, ymm14, ymm0
+    vmulps          ymm15, ymm15, ymm11
+    vaddps          ymm15, ymm15, ymm0
+    vbroadcastss    ymm0, dword ptr [{{offset}} {{L}}coeffs_num_alpha_1]
+    vmulps          ymm12, ymm12, ymm8
+    vaddps          ymm12, ymm12, ymm1
+    vmulps          ymm13, ymm13, ymm9
+    vaddps          ymm13, ymm13, ymm1
+    vmulps          ymm14, ymm14, ymm10
+    vaddps          ymm14, ymm14, ymm1
+    vmulps          ymm15, ymm15, ymm11
+    vaddps          ymm15, ymm15, ymm1
+    vbroadcastss    ymm1, dword ptr [{{offset}} {{L}}coeffs_num_beta_6]
+    vmulps          ymm12, ymm12, ymm8
+    vaddps          ymm12, ymm12, ymm2
+    vmulps          ymm13, ymm13, ymm9
+    vaddps          ymm13, ymm13, ymm2
+    vmulps          ymm14, ymm14, ymm10
+    vaddps          ymm14, ymm14, ymm2
+    vmulps          ymm15, ymm15, ymm11
+    vaddps          ymm15, ymm15, ymm2
+    vbroadcastss    ymm2, dword ptr [{{offset}} {{L}}coeffs_num_beta_4]
+    vmulps          ymm12, ymm12, ymm8
+    vaddps          ymm12, ymm12, ymm3
+    vmulps          ymm13, ymm13, ymm9
+    vaddps          ymm13, ymm13, ymm3
+    vmulps          ymm14, ymm14, ymm10
+    vaddps          ymm14, ymm14, ymm3
+    vmulps          ymm15, ymm15, ymm11
+    vaddps          ymm15, ymm15, ymm3
+    vbroadcastss    ymm3, dword ptr [{{offset}} {{L}}coeffs_num_beta_2]
+    vmulps          ymm12, ymm12, ymm8
+    vaddps          ymm12, ymm12, ymm0
+    vmulps          ymm13, ymm13, ymm9
+    vaddps          ymm13, ymm13, ymm0
+    vmulps          ymm14, ymm14, ymm10
+    vaddps          ymm14, ymm14, ymm0
+    vmulps          ymm15, ymm15, ymm11
+    vaddps          ymm15, ymm15, ymm0
+    vbroadcastss    ymm0, dword ptr [{{offset}} {{L}}coeffs_num_beta_0]
+    vmulps          ymm4, ymm4, ymm12
+    vmulps          ymm5, ymm5, ymm13
+    vmulps          ymm6, ymm6, ymm14
+    vmulps          ymm7, ymm7, ymm15   // ymm4..7 <- num
+
+    vmovaps         ymm12, ymm1
+    vmovaps         ymm13, ymm1
+    vmovaps         ymm14, ymm1
+    vmovaps         ymm15, ymm1
+    vmulps          ymm12, ymm12, ymm8
+    vaddps          ymm12, ymm12, ymm2
+    vmulps          ymm13, ymm13, ymm9
+    vaddps          ymm13, ymm13, ymm2
+    vmulps          ymm14, ymm14, ymm10
+    vaddps          ymm14, ymm14, ymm2
+    vmulps          ymm15, ymm15, ymm11
+    vaddps          ymm15, ymm15, ymm2
+    vmulps          ymm12, ymm12, ymm8
+    vaddps          ymm12, ymm12, ymm3
+    vmulps          ymm13, ymm13, ymm9
+    vaddps          ymm13, ymm13, ymm3
+    vmulps          ymm14, ymm14, ymm10
+    vaddps          ymm14, ymm14, ymm3
+    vmulps          ymm15, ymm15, ymm11
+    vaddps          ymm15, ymm15, ymm3
+    vmulps          ymm12, ymm12, ymm8
+    vaddps          ymm12, ymm12, ymm0
+    vmulps          ymm13, ymm13, ymm9
+    vaddps          ymm13, ymm13, ymm0
+    vmulps          ymm14, ymm14, ymm10
+    vaddps          ymm14, ymm14, ymm0
+    vmulps          ymm15, ymm15, ymm11
+    vaddps          ymm15, ymm15, ymm0  // ymm12..15 <- denum
+
+    vdivps          ymm4, ymm4, ymm12
+    vdivps          ymm5, ymm5, ymm13
+    vdivps          ymm6, ymm6, ymm14
+    vdivps          ymm7, ymm7, ymm15
+
+    vmovaps [rdi], ymm4
+    vmovaps [rdi + 32], ymm5
+    vmovaps [rdi + 64], ymm6
+    vmovaps [rdi + 96], ymm7
+
+    add     rdi, 128
+    sub     rsi, 32
+    cmp     rsi, 32
+    jg      {{L}}loop_4
+
+    cmp     rsi, 0
+    je      {{L}}done
+
+{{L}}loop_1:
+    vmovaps         ymm4, [rdi]
+
+    vbroadcastss    ymm0, dword ptr [{{offset}} {{L}}coeffs_num_low]
+    vbroadcastss    ymm1, dword ptr [{{offset}} {{L}}coeffs_num_high]
+    vbroadcastss    ymm2, dword ptr [{{offset}} {{L}}coeffs_num_alpha_13]
+    vbroadcastss    ymm3, dword ptr [{{offset}} {{L}}coeffs_num_alpha_11]
+
+    vmaxps          ymm4, ymm4, ymm0
+    vbroadcastss    ymm0, dword ptr [{{offset}} {{L}}coeffs_num_alpha_9]
+
+    vminps          ymm4, ymm4, ymm1        // ymm4 <- x
+    vbroadcastss    ymm1, dword ptr [{{offset}} {{L}}coeffs_num_alpha_7]
+
+    vmulps          ymm8, ymm4, ymm4        // ymm8 <- x^2
+
+    vmovaps         ymm12, ymm2
+    vbroadcastss    ymm2, dword ptr [{{offset}} {{L}}coeffs_num_alpha_5]
+    vmulps          ymm12, ymm12, ymm8
+    vaddps          ymm12, ymm12, ymm3
+    vbroadcastss    ymm3, dword ptr [{{offset}} {{L}}coeffs_num_alpha_3]
+    vmulps          ymm12, ymm12, ymm8
+    vaddps          ymm12, ymm12, ymm0
+    vbroadcastss    ymm0, dword ptr [{{offset}} {{L}}coeffs_num_alpha_1]
+    vmulps          ymm12, ymm12, ymm8
+    vaddps          ymm12, ymm12, ymm1
+    vbroadcastss    ymm1, dword ptr [{{offset}} {{L}}coeffs_num_beta_6]
+    vmulps          ymm12, ymm12, ymm8
+    vaddps          ymm12, ymm12, ymm2
+    vbroadcastss    ymm2, dword ptr [{{offset}} {{L}}coeffs_num_beta_4]
+    vmulps          ymm12, ymm12, ymm8
+    vaddps          ymm12, ymm12, ymm3
+    vbroadcastss    ymm3, dword ptr [{{offset}} {{L}}coeffs_num_beta_2]
+    vmulps          ymm12, ymm12, ymm8
+    vaddps          ymm12, ymm12, ymm0
+    vbroadcastss    ymm0, dword ptr [{{offset}} {{L}}coeffs_num_beta_0]
+    vmulps          ymm4, ymm4, ymm12
+
+    vmovaps         ymm12, ymm1
+    vmulps          ymm12, ymm12, ymm8
+    vaddps          ymm12, ymm12, ymm2
+    vmulps          ymm12, ymm12, ymm8
+    vaddps          ymm12, ymm12, ymm3
+    vmulps          ymm12, ymm12, ymm8
+    vaddps          ymm12, ymm12, ymm0
+
+    vdivps          ymm4, ymm4, ymm12
+
+    vmovaps [rdi], ymm4
+    add     rdi, 32
+    sub     rsi, 8
+    jnz     {{L}}loop_1
+
+{{L}}done:
+
+// ----------------------------------------------------------------------
+
+    ldmxcsr     [rsp + 4]
+
+    add         rsp, 8
+
+    pop r15
+    pop r14
+    pop r13
+    pop r12
+    pop rbx
+
+{% if family == "windows" %}
+    pop rsi
+    pop rdi
+
+    vmovaps xmm15, [rsp+16*9]
+    vmovaps xmm14, [rsp+16*8]
+    vmovaps xmm13, [rsp+16*7]
+    vmovaps xmm12, [rsp+16*6]
+    vmovaps xmm11, [rsp+16*5]
+    vmovaps xmm10, [rsp+16*4]
+    vmovaps xmm9, [rsp+16*3]
+    vmovaps xmm8, [rsp+16*2]
+    vmovaps xmm7, [rsp+16*1]
+    vmovaps xmm6, [rsp]
+{% endif %}
+
+    mov rsp, rbp
+    pop rbp
+    ret
+
+{% set float %}{% if msvc %} real4 {%else%} .float {%endif%}{% endset %}
+
+{{L}}coeffs_num_low:
+    {{float}} -8.9
+{{L}}coeffs_num_high:
+    {{float}} 8.9
+
+{{L}}coeffs_num_alpha_13:
+    {{float}} -8.488492677e-14
+{{L}}coeffs_num_alpha_11:
+    {{float}} 5.277853000e-11
+{{L}}coeffs_num_alpha_9:
+    {{float}} -2.022500419e-8
+{{L}}coeffs_num_alpha_7:
+    {{float}} 0.00001115424833
+{{L}}coeffs_num_alpha_5:
+    {{float}} 0.003103950131
+{{L}}coeffs_num_alpha_3:
+    {{float}} 0.1308400453
+{{L}}coeffs_num_alpha_1:
+    {{float}} 0.9999999934
+
+{{L}}coeffs_num_beta_6:
+    {{float}} 0.0002546136580
+{{L}}coeffs_num_beta_4:
+    {{float}} 0.02449515379
+{{L}}coeffs_num_beta_2:
+    {{float}} 0.4641733162
+{{L}}coeffs_num_beta_0:
+    {{float}} 1.0
+
+
+
+{% if msvc %}
+avx_tanh_f32_{{suffix}} endp
+_text ends
+end
+{% else %}
+.cfi_endproc
+{% endif %}
diff --git a/linalg/x86_64/fma/fma_mmm_f32_scalars.j2 b/linalg/x86_64/fma/fma_mmm_f32_scalars.j2
index f01a690911..e76d2a9cb6 100644
--- a/linalg/x86_64/fma/fma_mmm_f32_scalars.j2
+++ b/linalg/x86_64/fma/fma_mmm_f32_scalars.j2
@@ -19,8 +19,8 @@
         vbroadcastss    ymm15, xmm15
     {% endif %}
 
-    // ymm14 <- all zero
-    vpxor           ymm14, ymm14, ymm14
+    // ymm14 <- all zero (vxorps: 256-bit vpxor needs avx2, this must stay avx-safe)
+    vxorps          ymm14, ymm14, ymm14
 
     {% for reg in range(from, to + 1) %}
         // ymm12 <- alpha * x
diff --git a/linalg/x86_64/fma/postamble.j2 b/linalg/x86_64/fma/postamble.j2
index 616a98975f..53c224e5d5 100644
--- a/linalg/x86_64/fma/postamble.j2
+++ b/linalg/x86_64/fma/postamble.j2
@@ -29,7 +29,7 @@
     ret
 
 {% if msvc %}
-fma_mmm_{{type}}_{{size}}_{{suffix}} endp
+{{ prefix | default("fma") }}_mmm_{{type}}_{{size}}_{{suffix}} endp
 _text ends
 end
 
diff --git a/linalg/x86_64/fma/preamble.j2 b/linalg/x86_64/fma/preamble.j2
index 9d98c1f1ed..6c22860e04 100644
--- a/linalg/x86_64/fma/preamble.j2
+++ b/linalg/x86_64/fma/preamble.j2
@@ -2,15 +2,15 @@
 {% if msvc %}
 
 _text segment
-fma_mmm_{{type}}_{{size}}_{{suffix}} proc
+{{ prefix | default("fma") }}_mmm_{{type}}_{{size}}_{{suffix}} proc
 
 {% else %}
 
 .intel_syntax noprefix
 .text
 .p2align 5
-.globl {{G}}fma_mmm_{{type}}_{{size}}_{{suffix}}
-{{G}}fma_mmm_{{type}}_{{size}}_{{suffix}}:
+.globl {{G}}{{ prefix | default("fma") }}_mmm_{{type}}_{{size}}_{{suffix}}
+{{G}}{{ prefix | default("fma") }}_mmm_{{type}}_{{size}}_{{suffix}}:
 .cfi_startproc
 
 {% endif %}