From db663bb6047407e6ab26408691248eeafac6ab39 Mon Sep 17 00:00:00 2001 From: Colm Donelan Date: Wed, 8 Jul 2026 17:37:25 +0100 Subject: [PATCH 1/3] KFI-543 Integrate SME2 kernels matmul_clamp_f16_qai8dxp_qsi8cxp Added: * kai_matmul_clamp_f16_qai8dxp1vlx4_qsi8cxp4vlx4_1vlx4vl_sme2_mopa * kai_matmul_clamp_f16_qai8dxp1x4_qsi8cxp4vlx4_1x4vl_sme2_dot Signed-off-by: Colm Donelan --- cmake/gen/neon_aarch64_microkernels.cmake | 1 + cmake/gen/neonsme2_microkernels.cmake | 2 + gen/neon_aarch64_microkernels.bzl | 1 + gen/neonsme2_microkernels.bzl | 2 + src/configs/gemm-config.c | 51 ++++ src/configs/pack-lh-config.c | 28 +++ src/operator-run.c | 6 +- src/operators/fingerprint_id.h.inc | 1 + src/operators/fully-connected-nc.c | 69 ++++++ ...p8-f16-qc8w-gemm-minmax-16x64c4-neonsme2.c | 47 ++++ ...qp8-f16-qc8w-gemm-minmax-1x64c4-neonsme2.c | 46 ++++ src/subgraph.c | 20 +- src/subgraph/fully-connected.c | 22 ++ .../x8-packq-aarch64-neon-f16qp8-u2.c | 27 +++ src/xnnpack/config.h | 2 + src/xnnpack/fingerprint_check.c | 1 + src/xnnpack/gemm.h | 9 + src/xnnpack/internal.h | 26 ++ src/xnnpack/operator-type-defs.inc | 2 + src/xnnpack/packq.h | 5 + src/xnnpack/simd/f16-scalar.h | 11 +- test/subgraph/fully-connected.cc | 227 ++++++++++++++++++ third_party/kleidiai/BUILD.gn | 6 + 23 files changed, 600 insertions(+), 12 deletions(-) create mode 100644 src/qp8-f16-qc8w-gemm/qp8-f16-qc8w-gemm-minmax-16x64c4-neonsme2.c create mode 100644 src/qp8-f16-qc8w-gemm/qp8-f16-qc8w-gemm-minmax-1x64c4-neonsme2.c create mode 100644 src/x8-packq/x8-packq-aarch64-neon-f16qp8-u2.c diff --git a/cmake/gen/neon_aarch64_microkernels.cmake b/cmake/gen/neon_aarch64_microkernels.cmake index a544fdcab3c..fecce4e6db1 100644 --- a/cmake/gen/neon_aarch64_microkernels.cmake +++ b/cmake/gen/neon_aarch64_microkernels.cmake @@ -16,6 +16,7 @@ SET(PROD_NEON_AARCH64_MICROKERNEL_SRCS src/f32-vsqrt/gen/f32-vsqrt-aarch64-neon-sqrt.c src/x8-lut/gen/x8-lut-aarch64-neon-tbx128x4-u64.c src/x8-packq/x8-packq-aarch64-neon-f32qp8-u2.c + src/x8-packq/x8-packq-aarch64-neon-f16qp8-u2.c src/x24-transposec/x24-transposec-4x4-aarch64-neon-tbl128.c src/x32-transposec/x32-transposec-4x4-aarch64-neon-tbl128.c) diff --git a/cmake/gen/neonsme2_microkernels.cmake b/cmake/gen/neonsme2_microkernels.cmake index c349181e175..f4e6a8ecc3e 100644 --- a/cmake/gen/neonsme2_microkernels.cmake +++ b/cmake/gen/neonsme2_microkernels.cmake @@ -23,6 +23,8 @@ SET(PROD_NEONSME2_MICROKERNEL_SRCS src/qp8-f32-qc4w-gemm/qp8-f32-qc4w-gemm-minmax-16x64c4-neonsme2.c src/qp8-f32-qc8w-gemm/qp8-f32-qc8w-gemm-minmax-1x64c4-neonsme2.c src/qp8-f32-qc8w-gemm/qp8-f32-qc8w-gemm-minmax-16x64c4-neonsme2.c + src/qp8-f16-qc8w-gemm/qp8-f16-qc8w-gemm-minmax-1x64c4-neonsme2.c + src/qp8-f16-qc8w-gemm/qp8-f16-qc8w-gemm-minmax-16x64c4-neonsme2.c src/x8-pack-lh/x8-packlh-igemm-neonsme2.c src/x8-pack-lh/x8-packlh-neonsme2.c src/x16-pack-lh/x16-packlh-igemm-neonsme2.c diff --git a/gen/neon_aarch64_microkernels.bzl b/gen/neon_aarch64_microkernels.bzl index fd218f89eb3..1f9f88053d5 100644 --- a/gen/neon_aarch64_microkernels.bzl +++ b/gen/neon_aarch64_microkernels.bzl @@ -12,6 +12,7 @@ PROD_NEON_AARCH64_MICROKERNEL_SRCS = [ "src/f32-vsqrt/gen/f32-vsqrt-aarch64-neon-sqrt.c", "src/x8-lut/gen/x8-lut-aarch64-neon-tbx128x4-u64.c", "src/x8-packq/x8-packq-aarch64-neon-f32qp8-u2.c", + "src/x8-packq/x8-packq-aarch64-neon-f16qp8-u2.c", "src/x24-transposec/x24-transposec-4x4-aarch64-neon-tbl128.c", "src/x32-transposec/x32-transposec-4x4-aarch64-neon-tbl128.c", ] diff --git a/gen/neonsme2_microkernels.bzl b/gen/neonsme2_microkernels.bzl index cf22def575b..a2a886c5e7d 100644 --- a/gen/neonsme2_microkernels.bzl +++ b/gen/neonsme2_microkernels.bzl @@ -19,6 +19,8 @@ PROD_NEONSME2_MICROKERNEL_SRCS = [ "src/qp8-f32-qc4w-gemm/qp8-f32-qc4w-gemm-minmax-16x64c4-neonsme2.c", "src/qp8-f32-qc8w-gemm/qp8-f32-qc8w-gemm-minmax-1x64c4-neonsme2.c", "src/qp8-f32-qc8w-gemm/qp8-f32-qc8w-gemm-minmax-16x64c4-neonsme2.c", + "src/qp8-f16-qc8w-gemm/qp8-f16-qc8w-gemm-minmax-1x64c4-neonsme2.c", + "src/qp8-f16-qc8w-gemm/qp8-f16-qc8w-gemm-minmax-16x64c4-neonsme2.c", "src/x8-pack-lh/x8-packlh-igemm-neonsme2.c", "src/x8-pack-lh/x8-packlh-neonsme2.c", "src/x16-pack-lh/x16-packlh-igemm-neonsme2.c", diff --git a/src/configs/gemm-config.c b/src/configs/gemm-config.c index fe1ae8d4a3f..d732090ed77 100644 --- a/src/configs/gemm-config.c +++ b/src/configs/gemm-config.c @@ -58,6 +58,7 @@ static struct xnn_gemm_config qdu8_f32_qc2w_gemm_config = {0}; static struct xnn_gemm_config qd8_f32_qc8w_gemm_config = {0}; static struct xnn_gemm_config qp8_f32_qc4w_gemm_config = {0}; static struct xnn_gemm_config qp8_f32_qc8w_gemm_config = {0}; +static struct xnn_gemm_config qp8_f16_qc8w_gemm_config = {0}; static struct xnn_gemm_config qp8_f32_qb4w_gemm_config = {0}; static struct xnn_gemm_config qdu8_f32_qc4w_gemm_config = {0}; static struct xnn_gemm_config qdu8_f16_qc8w_gemm_config = {0}; @@ -94,6 +95,7 @@ XNN_INIT_ONCE_GUARD(qdu8_f32_qc2w_gemm); XNN_INIT_ONCE_GUARD(qd8_f32_qc8w_gemm); XNN_INIT_ONCE_GUARD(qp8_f32_qc4w_gemm); XNN_INIT_ONCE_GUARD(qp8_f32_qc8w_gemm); +XNN_INIT_ONCE_GUARD(qp8_f16_qc8w_gemm); XNN_INIT_ONCE_GUARD(qp8_f32_qb4w_gemm); XNN_INIT_ONCE_GUARD(qdu8_f32_qc4w_gemm); XNN_INIT_ONCE_GUARD(qdu8_f16_qc8w_gemm); @@ -3007,6 +3009,44 @@ static void init_qp8_f32_qc8w_gemm_config(void) { #endif // XNN_ARCH_ARM64 && XNN_ENABLE_KLEIDIAI } +static void init_qp8_f16_qc8w_gemm_config(void) { + qp8_f16_qc8w_gemm_config.log2_input_element_size = XNN_LOG2_SIZEOF_INT8_T; + qp8_f16_qc8w_gemm_config.log2_filter_element_size = XNN_LOG2_SIZEOF_UINT8_T; + qp8_f16_qc8w_gemm_config.log2_filter_element_bit_size = + XNN_LOG2_SIZEOF_UINT8_T + 3; + qp8_f16_qc8w_gemm_config.bias_element_size = sizeof(float); + +#if XNN_ARCH_ARM64 && XNN_ENABLE_KLEIDIAI + const struct xnn_hardware_config* hardware_config = + xnn_init_hardware_config(); + assert(hardware_config != NULL); + if (XNN_ENABLE_ARM_SME2 && (hardware_config->arch_flags & xnn_arch_arm_sme2)) { +#if XNN_ENABLE_ARM_SME2 + const size_t mr = + xnn_qp8_f16_qc8w_gemm_minmax_ukernel_16x64c4__neonsme2_get_mr(); + const size_t nr = + xnn_qp8_f16_qc8w_gemm_minmax_ukernel_16x64c4__neonsme2_get_nr(); + qp8_f16_qc8w_gemm_config.minmax.qp8gemm[XNN_MR_TO_INDEX(1)] = + XNN_INIT_HMP_QP8GEMM_UKERNEL( + xnn_qp8_f16_qc8w_gemm_minmax_ukernel_1x64c4__neonsme2); + qp8_f16_qc8w_gemm_config.minmax.qp8gemm[XNN_MR_TO_INDEX(mr)] = + XNN_INIT_HMP_QP8GEMM_UKERNEL( + xnn_qp8_f16_qc8w_gemm_minmax_ukernel_16x64c4__neonsme2); + qp8_f16_qc8w_gemm_config.init.f16 = xnn_init_f16_minmax_scalar_params; + qp8_f16_qc8w_gemm_config.pack_weights_and_biases = + xnn_pack_kai_qs8_weights_and_biases; + qp8_f16_qc8w_gemm_config.packed_stride_weights_and_biases = + xnn_packed_stride_kai_qs8_weights_and_biases; + qp8_f16_qc8w_gemm_config.mr = mr; + qp8_f16_qc8w_gemm_config.nr = nr; + qp8_f16_qc8w_gemm_config.log2_kr = 2; + qp8_f16_qc8w_gemm_config.mr_packed = mr; +#endif // XNN_ENABLE_ARM_SME2 + } + assert(qp8_f16_qc8w_gemm_config.mr <= XNN_MAX_MR); +#endif // XNN_ARCH_ARM64 && XNN_ENABLE_KLEIDIAI +} + static void init_qp8_f32_qb4w_gemm_config(void) { // Common parameters. qp8_f32_qb4w_gemm_config.log2_input_element_size = XNN_LOG2_SIZEOF_INT8_T; @@ -6595,6 +6635,17 @@ const struct xnn_gemm_config* xnn_init_qp8_f32_qc8w_gemm_config() { return NULL; } +const struct xnn_gemm_config* xnn_init_qp8_f16_qc8w_gemm_config() { + if (xnn_init_hardware_config() == NULL) { + return NULL; + } + XNN_INIT_ONCE(qp8_f16_qc8w_gemm); + if (qp8_f16_qc8w_gemm_config.minmax.qp8gemm[0].function[0] != NULL) { + return &qp8_f16_qc8w_gemm_config; + } + return NULL; +} + const struct xnn_gemm_config* xnn_init_qp8_f32_qb4w_gemm_config() { const struct xnn_hardware_config* hardware_config = xnn_init_hardware_config(); diff --git a/src/configs/pack-lh-config.c b/src/configs/pack-lh-config.c index 099d22c5a4d..ed32abb78b7 100644 --- a/src/configs/pack-lh-config.c +++ b/src/configs/pack-lh-config.c @@ -16,6 +16,7 @@ #include "src/xnnpack/packq.h" static struct xnn_pack_lh_config qp8_pack_lh_config = {0}; +static struct xnn_pack_lh_config qp8_f16_pack_lh_config = {0}; static struct xnn_pack_lh_config x8_pack_lh_config = {0}; static struct xnn_pack_lh_config x16_pack_lh_config = {0}; static struct xnn_pack_lh_config x32_pack_lh_config = {0}; @@ -24,6 +25,7 @@ static struct xnn_pack_lh_config x32_igemm_pack_lh_config = {0}; static struct xnn_pack_lh_config x16_igemm_pack_lh_config = {0}; XNN_INIT_ONCE_GUARD(qp8_pack_lh); +XNN_INIT_ONCE_GUARD(qp8_f16_pack_lh); XNN_INIT_ONCE_GUARD(x8_pack_lh); XNN_INIT_ONCE_GUARD(x16_pack_lh); XNN_INIT_ONCE_GUARD(x32_pack_lh); @@ -57,6 +59,32 @@ const struct xnn_pack_lh_config* xnn_init_qp8_pack_lh_config() { return &qp8_pack_lh_config; } +static void init_qp8_f16_pack_lh_config(void) { +#if XNN_ARCH_ARM64 && XNN_ENABLE_KLEIDIAI + qp8_f16_pack_lh_config.pack_lh_fn = + (xnn_pack_lh_ukernel_fn)xnn_x8_packq_f16qp8_ukernel__aarch64_neon_u2; + qp8_f16_pack_lh_config.size_fn = + (xnn_pack_lh_size_fn)xnn_x8_packq_f32qp8_packed_size; + qp8_f16_pack_lh_config.offset_fn = + (xnn_pack_lh_offset_fn)xnn_x8_packq_f32qp8_packed_offset; + qp8_f16_pack_lh_config.log2_input_element_size = XNN_LOG2_SIZEOF_FLOAT16; + qp8_f16_pack_lh_config.log2_packed_element_size = 0; +#endif // XNN_ARCH_ARM64 && XNN_ENABLE_KLEIDIAI +} + +const struct xnn_pack_lh_config* xnn_init_qp8_f16_pack_lh_config() { + const struct xnn_hardware_config* hardware_config = + xnn_init_hardware_config(); + if (hardware_config == NULL) { + return NULL; + } + XNN_INIT_ONCE(qp8_f16_pack_lh); + if (qp8_f16_pack_lh_config.pack_lh_fn == NULL) { + return NULL; + } + return &qp8_f16_pack_lh_config; +} + static void init_x32_pack_lh_config(void) { #if XNN_ARCH_ARM64 && XNN_ENABLE_KLEIDIAI const struct xnn_hardware_config* hardware_config = diff --git a/src/operator-run.c b/src/operator-run.c index 7ba272b5e98..63285d688b4 100644 --- a/src/operator-run.c +++ b/src/operator-run.c @@ -527,7 +527,8 @@ XNN_NO_SANITIZE_FUNCTION void xnn_compute_hmp_grouped_qp8gemm(struct gemm_contex context->qp8_ukernel.function[uarch_index]( mr_step, nr_block_size, k_scaled, (const void*)(a + a_offset), packed_w, (void*)(c + mr_block_start * cm_stride), cm_stride, - /*dst_stride_col=*/sizeof(float), context->fused_params); + // QP8 kernels can produce f16 or f32 outputs, so use the configured output element size. + /*dst_stride_col=*/1 << context->log2_csize, context->fused_params); } mr_block_size -= mr_step; mr_block_start += mr_step; @@ -597,7 +598,8 @@ XNN_INLINE static XNN_NO_SANITIZE_FUNCTION void compute_hmp_qp8gemm( mr_step, nr_block_size, k_scaled, (const void*)((uintptr_t)a + a_offset), packed_w, (void*)((uintptr_t)c + mr_block_start * cm_stride), cm_stride, - /*dst_stride_col=*/sizeof(float), context->fused_params); + // QP8 kernels can produce f16 or f32 outputs, so use the configured output element size. + /*dst_stride_col=*/1 << context->log2_csize, context->fused_params); } mr_block_size -= mr_step; diff --git a/src/operators/fingerprint_id.h.inc b/src/operators/fingerprint_id.h.inc index 15df7ccd500..f3e03be4dbb 100644 --- a/src/operators/fingerprint_id.h.inc +++ b/src/operators/fingerprint_id.h.inc @@ -34,6 +34,7 @@ XNN_FINGERPRINT_ID(fully_connected_nc, qd8, f32, qc4w) XNN_FINGERPRINT_ID(fully_connected_nc, qdu8, f32, qc4w) XNN_FINGERPRINT_ID(fully_connected_nc, qp8, f32, qc4w) XNN_FINGERPRINT_ID(fully_connected_nc, qp8, f32, qc8w) +XNN_FINGERPRINT_ID(fully_connected_nc, qp8, f16, qc8w) XNN_FINGERPRINT_ID(fully_connected_nc, qp8, f32, qb4w) XNN_FINGERPRINT_ID(fully_connected_nc, qd8, f32, qb4w) XNN_FINGERPRINT_ID(fully_connected_nc, qdu8, f32, qb4w) diff --git a/src/operators/fully-connected-nc.c b/src/operators/fully-connected-nc.c index 7c46e42bbbc..c04090d586c 100644 --- a/src/operators/fully-connected-nc.c +++ b/src/operators/fully-connected-nc.c @@ -67,6 +67,7 @@ static enum xnn_operator_type get_operator_type( XNNPACK_FINGERPRINT_TO_OP_TYPE(qdu8, f32, qc4w); XNNPACK_FINGERPRINT_TO_OP_TYPE(qp8, f32, qc4w); XNNPACK_FINGERPRINT_TO_OP_TYPE(qp8, f32, qc8w); + XNNPACK_FINGERPRINT_TO_OP_TYPE(qp8, f16, qc8w); XNNPACK_FINGERPRINT_TO_OP_TYPE(qp8, f32, qb4w); XNNPACK_FINGERPRINT_TO_OP_TYPE(qd8, f32, qb4w); XNNPACK_FINGERPRINT_TO_OP_TYPE(qdu8, f32, qb4w); @@ -1157,6 +1158,21 @@ static const struct fc_variant qp8_f32_qc8w_variant = { .kernel_scale_element_size = sizeof(float), }; +static const struct fc_variant qp8_f16_qc8w_variant = { + .check_output_bounds = check_output_bounds_f32, + .check_kernel_zero_point = UNUSED, + .check_block_size = UNUSED, + .check_flags = UNUSED, + .setup_gemm_ukernels = setup_gemm_ukernels, + .setup_params = setup_params_f16, + .setup_packing_params = setup_packing_params_qs8_qc8w, + .setup_packing_functions = setup_packing_functions_from_gemm_config, + .setup_scale_params = setup_scale_params_qs8_qc8w, + .fingerprint_constraints = {}, + .extra_weights_bytes = sizeof(float) * 2, + .kernel_scale_element_size = sizeof(float), +}; + static const struct fc_variant qd8_f16_qb4w_variant = { .check_output_bounds = check_output_bounds_f32, .check_kernel_zero_point = check_kernel_zero_point_is_0_or_8_qu8, @@ -1482,6 +1498,11 @@ static enum xnn_status setup_variant_and_gemm_config( context->gemm_config = xnn_init_qp8_f32_qc8w_gemm_config(); context->fingerprint_id = xnn_fingerprint_id_fully_connected_nc_qp8_f32_qc8w; break; + case xnn_operator_type_fully_connected_nc_qp8_f16_qc8w: + *variant = &qp8_f16_qc8w_variant; + context->gemm_config = xnn_init_qp8_f16_qc8w_gemm_config(); + context->fingerprint_id = xnn_fingerprint_id_fully_connected_nc_qp8_f16_qc8w; + break; case xnn_operator_type_fully_connected_nc_qp8_f32_qb4w: *variant = &qp8_f32_qb4w_variant; context->gemm_config = xnn_init_qp8_f32_qb4w_gemm_config(); @@ -2143,6 +2164,30 @@ enum xnn_status xnn_create_fully_connected_nc_qp8_f32_qc8w( return create_fully_connected_nc_helper(&context); } +enum xnn_status xnn_create_fully_connected_nc_qp8_f16_qc8w( + size_t input_channels, size_t output_channels, size_t input_stride, + size_t output_stride, const float* kernel_scale, const void* kernel, + const float* bias, float output_min, float output_max, uint32_t flags, + xnn_weights_cache_t weights_cache, xnn_operator_t* fully_connected_op_out) { + struct fc_context context = { + .input_channels = input_channels, + .output_channels = output_channels, + .input_stride = input_stride, + .output_stride = output_stride, + .kernel_scale.f32 = kernel_scale, + .kernel = kernel, + .bias = bias, + .output_min = output_min, + .output_max = output_max, + .flags = flags, + .weights_cache = weights_cache, + .operator_type = xnn_operator_type_fully_connected_nc_qp8_f16_qc8w, + .fully_connected_op_out = fully_connected_op_out, + .should_fingerprint = true, + }; + return create_fully_connected_nc_helper(&context); +} + enum xnn_status xnn_create_fully_connected_nc_qp8_f32_qb4w( size_t input_channels, size_t output_channels, size_t input_stride, size_t output_stride, size_t block_size, uint8_t kernel_zero_point, @@ -2909,6 +2954,9 @@ static XNN_NO_SANITIZE_FUNCTION enum xnn_status reshape_fully_connected_nc( case xnn_operator_type_fully_connected_nc_qp8_f32_qc8w: packed_lh_config = xnn_init_qp8_pack_lh_config(); break; + case xnn_operator_type_fully_connected_nc_qp8_f16_qc8w: + packed_lh_config = xnn_init_qp8_f16_pack_lh_config(); + break; case xnn_operator_type_fully_connected_nc_pf16: packed_lh_config = xnn_init_x16_pack_lh_config(); break; @@ -3463,6 +3511,19 @@ enum xnn_status xnn_reshape_fully_connected_nc_qp8_f32_qc8w( threadpool); } +enum xnn_status xnn_reshape_fully_connected_nc_qp8_f16_qc8w( + xnn_operator_t fully_connected_op, size_t batch_size, + size_t* workspace_size, pthreadpool_t threadpool) { + return reshape_fully_connected_nc( + fully_connected_op, xnn_operator_type_fully_connected_nc_qp8_f16_qc8w, + batch_size, + /*dynamic_quantization=*/false, + /*log2_output_element_size=*/XNN_LOG2_SIZEOF_FLOAT16, + &fully_connected_op->params.f16_minmax, + sizeof(fully_connected_op->params.f16_minmax), workspace_size, + threadpool); +} + enum xnn_status xnn_reshape_fully_connected_nc_qp8_f32_qb4w( xnn_operator_t fully_connected_op, size_t batch_size, size_t* workspace_size, pthreadpool_t threadpool) { @@ -3812,6 +3873,14 @@ enum xnn_status xnn_setup_fully_connected_nc_qp8_f32_qc8w( input, output, workspace, /*row_sum=*/NULL, /*quantization_params=*/NULL); } +enum xnn_status xnn_setup_fully_connected_nc_qp8_f16_qc8w( + xnn_operator_t fully_connected_op, const int8_t* input, float* output, + void* workspace) { + return setup_fully_connected_nc( + fully_connected_op, xnn_operator_type_fully_connected_nc_qp8_f16_qc8w, + input, output, workspace, /*row_sum=*/NULL, /*quantization_params=*/NULL); +} + enum xnn_status xnn_setup_fully_connected_nc_qp8_f32_qb4w( xnn_operator_t fully_connected_op, const int8_t* input, float* output, void* workspace) { diff --git a/src/qp8-f16-qc8w-gemm/qp8-f16-qc8w-gemm-minmax-16x64c4-neonsme2.c b/src/qp8-f16-qc8w-gemm/qp8-f16-qc8w-gemm-minmax-16x64c4-neonsme2.c new file mode 100644 index 00000000000..c1dbf97095b --- /dev/null +++ b/src/qp8-f16-qc8w-gemm/qp8-f16-qc8w-gemm-minmax-16x64c4-neonsme2.c @@ -0,0 +1,47 @@ +// Copyright 2026 Google LLC +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include +#include +#include + +#include "src/xnnpack/math.h" +#include "src/xnnpack/microparams.h" + +#if XNN_ENABLE_KLEIDIAI +#include "kai/ukernels/matmul/matmul_clamp_f16_qai8dxp_qsi8cxp/kai_matmul_clamp_f16_qai8dxp1vlx4_qsi8cxp4vlx4_1vlx4vl_sme2_mopa.h" + +size_t xnn_qp8_f16_qc8w_gemm_minmax_ukernel_16x64c4__neonsme2_get_mr() { + return + kai_get_mr_matmul_clamp_f16_qai8dxp1vlx4_qsi8cxp4vlx4_1vlx4vl_sme2_mopa(); +} + +size_t xnn_qp8_f16_qc8w_gemm_minmax_ukernel_16x64c4__neonsme2_get_nr() { + return + kai_get_nr_matmul_clamp_f16_qai8dxp1vlx4_qsi8cxp4vlx4_1vlx4vl_sme2_mopa(); +} +#endif // XNN_ENABLE_KLEIDIAI + +// Wraps the +// `kai_matmul_clamp_f16_qai8dxp1vlx4_qsi8cxp4vlx4_1vlx4vl_sme2_mopa` GEMM +// microkernel with a name that is compatible with our tooling. +void xnn_qp8_f16_qc8w_gemm_minmax_ukernel_16x64c4__neonsme2( + size_t m, size_t n, size_t k, const void* lhs_packed, + const void* rhs_packed, float* dst, size_t dst_stride_row, + size_t dst_stride_col, struct xnn_f32_minmax_params* minmax_params) { +#if XNN_ENABLE_KLEIDIAI + const struct xnn_f16_minmax_params* f16_minmax_params = + (const struct xnn_f16_minmax_params*) minmax_params; + kai_run_matmul_clamp_f16_qai8dxp1vlx4_qsi8cxp4vlx4_1vlx4vl_sme2_mopa( + m, n, k, lhs_packed, rhs_packed, dst, dst_stride_row, dst_stride_col, + xnn_float16_to_float(f16_minmax_params->scalar.min), + xnn_float16_to_float(f16_minmax_params->scalar.max)); +#else + assert( + "Calling KleidiAI microkernel wrapper, but XNNPACK was compiled without " + "`XNN_ENABLE_KLEIDIAI`." && + 0); +#endif // XNN_ENABLE_KLEIDIAI +} diff --git a/src/qp8-f16-qc8w-gemm/qp8-f16-qc8w-gemm-minmax-1x64c4-neonsme2.c b/src/qp8-f16-qc8w-gemm/qp8-f16-qc8w-gemm-minmax-1x64c4-neonsme2.c new file mode 100644 index 00000000000..f4315994cb9 --- /dev/null +++ b/src/qp8-f16-qc8w-gemm/qp8-f16-qc8w-gemm-minmax-1x64c4-neonsme2.c @@ -0,0 +1,46 @@ +// Copyright 2026 Google LLC +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include +#include +#include + +#include "src/xnnpack/math.h" +#include "src/xnnpack/microparams.h" + +#if XNN_ENABLE_KLEIDIAI +#include "kai/ukernels/matmul/matmul_clamp_f16_qai8dxp_qsi8cxp/kai_matmul_clamp_f16_qai8dxp1x4_qsi8cxp4vlx4_1x4vl_sme2_dot.h" + +size_t xnn_qp8_f16_qc8w_gemm_minmax_ukernel_1x64c4__neonsme2_get_mr() { + return + kai_get_mr_matmul_clamp_f16_qai8dxp1x4_qsi8cxp4vlx4_1x4vl_sme2_dot(); +} + +size_t xnn_qp8_f16_qc8w_gemm_minmax_ukernel_1x64c4__neonsme2_get_nr() { + return + kai_get_nr_matmul_clamp_f16_qai8dxp1x4_qsi8cxp4vlx4_1x4vl_sme2_dot(); +} +#endif // XNN_ENABLE_KLEIDIAI + +// Wraps the `kai_matmul_clamp_f16_qai8dxp1x4_qsi8cxp4vlx4_1x4vl_sme2_dot` +// GEMM microkernel with a name that is compatible with our tooling. +void xnn_qp8_f16_qc8w_gemm_minmax_ukernel_1x64c4__neonsme2( + size_t m, size_t n, size_t k, const void* lhs_packed, + const void* rhs_packed, float* dst, size_t dst_stride_row, + size_t dst_stride_col, struct xnn_f32_minmax_params* minmax_params) { +#if XNN_ENABLE_KLEIDIAI + const struct xnn_f16_minmax_params* f16_minmax_params = + (const struct xnn_f16_minmax_params*) minmax_params; + kai_run_matmul_clamp_f16_qai8dxp1x4_qsi8cxp4vlx4_1x4vl_sme2_dot( + m, n, k, lhs_packed, rhs_packed, dst, dst_stride_row, dst_stride_col, + xnn_float16_to_float(f16_minmax_params->scalar.min), + xnn_float16_to_float(f16_minmax_params->scalar.max)); +#else + assert( + "Calling KleidiAI microkernel wrapper, but XNNPACK was compiled without " + "`XNN_ENABLE_KLEIDIAI`." && + 0); +#endif // XNN_ENABLE_KLEIDIAI +} diff --git a/src/subgraph.c b/src/subgraph.c index ce16e06b7e0..8a80f76b7b8 100644 --- a/src/subgraph.c +++ b/src/subgraph.c @@ -3988,7 +3988,25 @@ enum xnn_status xnn_subgraph_optimize_packed_lhs(xnn_subgraph_t subgraph, // We may inline the `qdint8` packing regardless of whether we have // a specialized `qpint8` kernel or not. assumed_datatype = xnn_datatype_qdint8; - if (output_datatype == xnn_datatype_fp32) { + if (output_datatype == xnn_datatype_fp16) { + // The QP8/F16 packer consumes the convert's original FP16 input. + if (input_value->producer != XNN_INVALID_NODE_ID) { + const struct xnn_node* producer = + &subgraph->nodes[input_value->producer]; + const bool converts_from_fp16 = + producer->type == xnn_node_type_convert && + subgraph->values[producer->inputs[0]].datatype == + xnn_datatype_fp16; + const bool has_qc8w_weights = + kernel_datatype == xnn_datatype_qcint8 && + (node->flags & XNN_FLAG_TRANSPOSE_WEIGHTS) == 0; + if (converts_from_fp16 && has_qc8w_weights && + (gemm_config = xnn_init_qp8_f16_qc8w_gemm_config()) != + NULL) { + assumed_datatype = xnn_datatype_qpint8; + } + } + } else if (output_datatype == xnn_datatype_fp32) { switch (kernel_datatype) { case xnn_datatype_qbint4: // The qp8_f32_qb4w kernels only support unsigned 4-bit diff --git a/src/subgraph/fully-connected.c b/src/subgraph/fully-connected.c index e28ecbda084..87fd8bc4bba 100644 --- a/src/subgraph/fully-connected.c +++ b/src/subgraph/fully-connected.c @@ -57,6 +57,7 @@ enum fully_connected_op_type { fc_type_qdu8_f32_qb4w, fc_type_qdu8_f16_qc4w, fc_type_qp8_f32_qc8w, + fc_type_qp8_f16_qc8w, fc_type_pf16_f16_f16, fc_type_pqs8_qs8_qc8w, fc_type_bf16_bf16_f32, @@ -137,6 +138,8 @@ enum fully_connected_op_type get_fully_connected_op_type( return fc_type_qd8_f16_qc8w; case xnn_datatype_qduint8: return fc_type_qdu8_f16_qc8w; + case xnn_datatype_qpint8: + return fc_type_qp8_f16_qc8w; default: XNN_UNREACHABLE; } @@ -715,6 +718,15 @@ static enum xnn_status create_fully_connected_operator( node->activation.output_min, node->activation.output_max, node->flags, weights_cache, fully_connected_op_ptr); break; + case fc_type_qp8_f16_qc8w: + status = xnn_create_fully_connected_nc_qp8_f16_qc8w( + input_channels, output_channels, + /*input_stride=*/input_channels, + /*output_stride=*/output_channels, + filter_value->quantization.channelwise_scale, kernel_data, bias_data, + node->activation.output_min, node->activation.output_max, node->flags, + weights_cache, fully_connected_op_ptr); + break; case fc_type_f32_f32_qc8w: status = xnn_create_fully_connected_nc_f32_qc8w( input_channels, output_channels, @@ -1103,6 +1115,10 @@ static enum xnn_status reshape_fully_connected_operator( status = xnn_reshape_fully_connected_nc_qp8_f32_qc8w( fully_connected_op, batch_size, &opdata->workspace_size, threadpool); break; + case xnn_operator_type_fully_connected_nc_qp8_f16_qc8w: + status = xnn_reshape_fully_connected_nc_qp8_f16_qc8w( + fully_connected_op, batch_size, &opdata->workspace_size, threadpool); + break; case xnn_operator_type_fully_connected_nc_qp8_f32_qb4w: status = xnn_reshape_fully_connected_nc_qp8_f32_qb4w( fully_connected_op, batch_size, &opdata->workspace_size, threadpool); @@ -1430,6 +1446,12 @@ static enum xnn_status setup_fully_connected_operator( return xnn_setup_fully_connected_nc_qp8_f32_qc8w( fully_connected_op, input_data, output_data, opdata->workspace); } + case xnn_operator_type_fully_connected_nc_qp8_f16_qc8w: { + assert(kernel_data == NULL); + assert(bias_data == NULL); + return xnn_setup_fully_connected_nc_qp8_f16_qc8w( + fully_connected_op, input_data, output_data, opdata->workspace); + } case xnn_operator_type_fully_connected_nc_qp8_f32_qb4w: { assert(kernel_data == NULL); assert(bias_data == NULL); diff --git a/src/x8-packq/x8-packq-aarch64-neon-f16qp8-u2.c b/src/x8-packq/x8-packq-aarch64-neon-f16qp8-u2.c new file mode 100644 index 00000000000..b20f9481e82 --- /dev/null +++ b/src/x8-packq/x8-packq-aarch64-neon-f16qp8-u2.c @@ -0,0 +1,27 @@ +// Copyright 2026 Google LLC +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include +#include +#include + +#include "src/xnnpack/common.h" + +#if XNN_ENABLE_KLEIDIAI +#include "kai/ukernels/matmul/pack/kai_lhs_quant_pack_qai8dxp_f16_neon.h" +#endif // XNN_ENABLE_KLEIDIAI + +// Wraps KleidiAI's FP16 `qai8dxp` LHS quantize-and-pack microkernel. +void xnn_x8_packq_f16qp8_ukernel__aarch64_neon_u2( + size_t m, size_t k, size_t mr, size_t kr, size_t sr, size_t m_idx_start, + const void* XNN_RESTRICT lhs, size_t lhs_stride, + void* XNN_RESTRICT lhs_packed) { +#if XNN_ENABLE_KLEIDIAI + kai_run_lhs_quant_pack_qai8dxp_f16_neon(m, k, mr, kr, sr, m_idx_start, lhs, + lhs_stride, lhs_packed); +#else + assert("Not compiled with XNN_ENABLE_KLEIDIAI" && 0); +#endif // XNN_ENABLE_KLEIDIAI +} diff --git a/src/xnnpack/config.h b/src/xnnpack/config.h index 7ef21baf6c2..0ec00f970c6 100644 --- a/src/xnnpack/config.h +++ b/src/xnnpack/config.h @@ -26,6 +26,7 @@ XNN_INTERNAL const struct xnn_cmul_config* xnn_init_f16_cmul_config(); XNN_INTERNAL const struct xnn_cmul_config* xnn_init_f32_cmul_config(); XNN_INTERNAL const struct xnn_pack_lh_config* xnn_init_qp8_pack_lh_config(); +XNN_INTERNAL const struct xnn_pack_lh_config* xnn_init_qp8_f16_pack_lh_config(); XNN_INTERNAL const struct xnn_pack_lh_config* xnn_init_x8_pack_lh_config(); XNN_INTERNAL const struct xnn_pack_lh_config* xnn_init_x16_pack_lh_config(); XNN_INTERNAL const struct xnn_pack_lh_config* xnn_init_x32_pack_lh_config(); @@ -403,6 +404,7 @@ XNN_INTERNAL const struct xnn_gemm_config* xnn_init_qd8_f32_qc4w_gemm_config(); XNN_INTERNAL const struct xnn_gemm_config* xnn_init_qd8_f32_qc8w_gemm_config(); XNN_INTERNAL const struct xnn_gemm_config* xnn_init_qp8_f32_qc4w_gemm_config(); XNN_INTERNAL const struct xnn_gemm_config* xnn_init_qp8_f32_qc8w_gemm_config(); +XNN_INTERNAL const struct xnn_gemm_config* xnn_init_qp8_f16_qc8w_gemm_config(); XNN_INTERNAL const struct xnn_gemm_config* xnn_init_qp8_f32_qb4w_gemm_config(); XNN_INTERNAL const struct xnn_gemm_config* xnn_init_qdu8_f32_qc4w_gemm_config(); XNN_INTERNAL const struct xnn_gemm_config* xnn_init_qdu8_f16_qc8w_gemm_config(); diff --git a/src/xnnpack/fingerprint_check.c b/src/xnnpack/fingerprint_check.c index 27e78dedba3..d7a4cbf8fd2 100644 --- a/src/xnnpack/fingerprint_check.c +++ b/src/xnnpack/fingerprint_check.c @@ -37,6 +37,7 @@ static enum xnn_status compute_fingerprint( case xnn_fingerprint_id_fully_connected_nc_qdu8_f32_qc4w: case xnn_fingerprint_id_fully_connected_nc_qp8_f32_qc4w: case xnn_fingerprint_id_fully_connected_nc_qp8_f32_qc8w: + case xnn_fingerprint_id_fully_connected_nc_qp8_f16_qc8w: case xnn_fingerprint_id_fully_connected_nc_qp8_f32_qb4w: case xnn_fingerprint_id_fully_connected_nc_qd8_f32_qb4w: case xnn_fingerprint_id_fully_connected_nc_qdu8_f32_qb4w: diff --git a/src/xnnpack/gemm.h b/src/xnnpack/gemm.h index 7b9fc6d3e6e..e08087c69df 100644 --- a/src/xnnpack/gemm.h +++ b/src/xnnpack/gemm.h @@ -4078,6 +4078,10 @@ size_t xnn_qp8_f32_qc8w_gemm_minmax_ukernel_1x64c4__neonsme_get_mr(); size_t xnn_qp8_f32_qc8w_gemm_minmax_ukernel_1x64c4__neonsme_get_nr(); size_t xnn_qp8_f32_qc8w_gemm_minmax_ukernel_16x64c4__neonsme_get_mr(); size_t xnn_qp8_f32_qc8w_gemm_minmax_ukernel_16x64c4__neonsme_get_nr(); +size_t xnn_qp8_f16_qc8w_gemm_minmax_ukernel_1x64c4__neonsme2_get_mr(); +size_t xnn_qp8_f16_qc8w_gemm_minmax_ukernel_1x64c4__neonsme2_get_nr(); +size_t xnn_qp8_f16_qc8w_gemm_minmax_ukernel_16x64c4__neonsme2_get_mr(); +size_t xnn_qp8_f16_qc8w_gemm_minmax_ukernel_16x64c4__neonsme2_get_nr(); #define DECLARE_QP8_F32_QC8W_GEMM_MINMAX_UKERNEL_FUNCTION(fn_name) \ XNN_INTERNAL void fn_name( \ @@ -4102,6 +4106,11 @@ DECLARE_QP8_F32_QC8W_GEMM_MINMAX_UKERNEL_FUNCTION( DECLARE_QP8_F32_QC8W_GEMM_MINMAX_UKERNEL_FUNCTION( xnn_qp8_f32_qc8w_gemm_minmax_ukernel_16x64c4__neonsme) +DECLARE_QP8_F32_QC8W_GEMM_MINMAX_UKERNEL_FUNCTION( + xnn_qp8_f16_qc8w_gemm_minmax_ukernel_1x64c4__neonsme2) +DECLARE_QP8_F32_QC8W_GEMM_MINMAX_UKERNEL_FUNCTION( + xnn_qp8_f16_qc8w_gemm_minmax_ukernel_16x64c4__neonsme2) + #define DECLARE_QP8_F32_QB4W_GEMM_MINMAX_UKERNEL_FUNCTION(fn_name) \ XNN_INTERNAL void fn_name( \ size_t m, size_t n, size_t k, const void* lhs_packed, \ diff --git a/src/xnnpack/internal.h b/src/xnnpack/internal.h index ee93855e532..a4c83dd14be 100644 --- a/src/xnnpack/internal.h +++ b/src/xnnpack/internal.h @@ -101,6 +101,20 @@ enum xnn_status xnn_create_fully_connected_nc_qp8_f32_qc8w( xnn_weights_cache_t weights_cache, // xnn_operator_t* fully_connected_op_out); +enum xnn_status xnn_create_fully_connected_nc_qp8_f16_qc8w( + size_t input_channels, // + size_t output_channels, // + size_t input_stride, // + size_t output_stride, // + const float* kernel_scale, // + const void* kernel, // + const float* bias, // + float output_min, // + float output_max, // + uint32_t flags, // + xnn_weights_cache_t weights_cache, // + xnn_operator_t* fully_connected_op_out); + enum xnn_status xnn_setup_fully_connected_nc_qp8_f32_qc4w( xnn_operator_t fully_connected_op, // const int8_t* input, // @@ -113,6 +127,12 @@ enum xnn_status xnn_setup_fully_connected_nc_qp8_f32_qc8w( float* output, // void* workspace); +enum xnn_status xnn_setup_fully_connected_nc_qp8_f16_qc8w( + xnn_operator_t fully_connected_op, // + const int8_t* input, // + float* output, // + void* workspace); + enum xnn_status xnn_reshape_fully_connected_nc_qp8_f32_qc4w( xnn_operator_t fully_connected_op, // size_t batch_size, // @@ -127,6 +147,12 @@ enum xnn_status xnn_reshape_fully_connected_nc_qp8_f32_qc8w( // pthreadpool_t threadpool); +enum xnn_status xnn_reshape_fully_connected_nc_qp8_f16_qc8w( + xnn_operator_t fully_connected_op, // + size_t batch_size, // + size_t* workspace_size, // + pthreadpool_t threadpool); + enum xnn_status xnn_create_batch_matrix_multiply_nc_qp8_f32_qc8w( uint32_t flags, xnn_operator_t* batch_matrix_multiply_op_out); diff --git a/src/xnnpack/operator-type-defs.inc b/src/xnnpack/operator-type-defs.inc index 439714cdbeb..63a0bf2a9ed 100644 --- a/src/xnnpack/operator-type-defs.inc +++ b/src/xnnpack/operator-type-defs.inc @@ -155,6 +155,8 @@ XNN_ENUM_ITEM(xnn_operator_type_fully_connected_nc_qp8_f32_qc4w, "Fully Connected (NC, QP8, F32, QC4W)") XNN_ENUM_ITEM(xnn_operator_type_fully_connected_nc_qp8_f32_qc8w, "Fully Connected (NC, QP8, F32, QC8W)") +XNN_ENUM_ITEM(xnn_operator_type_fully_connected_nc_qp8_f16_qc8w, + "Fully Connected (NC, QP8, F16, QC8W)") XNN_ENUM_ITEM(xnn_operator_type_fully_connected_nc_qp8_f32_qb4w, "Fully Connected (NC, QP8, F32, QB4W)") XNN_ENUM_ITEM(xnn_operator_type_fully_connected_nc_qs8, diff --git a/src/xnnpack/packq.h b/src/xnnpack/packq.h index 2d34a10c87b..a87107418ec 100644 --- a/src/xnnpack/packq.h +++ b/src/xnnpack/packq.h @@ -148,6 +148,11 @@ XNN_INLINE static float xnn_x8_packq_f32qp8_get_dequantized( #undef XNN_UKERNEL +XNN_INTERNAL void xnn_x8_packq_f16qp8_ukernel__aarch64_neon_u2( + size_t m, size_t k, size_t mr_packed, size_t kr, size_t sr, + size_t m_idx_start, const void* XNN_RESTRICT lhs, size_t lhs_stride, + void* XNN_RESTRICT lhs_packed); + #ifdef __cplusplus } // extern "C" #endif diff --git a/src/xnnpack/simd/f16-scalar.h b/src/xnnpack/simd/f16-scalar.h index e0f979fb044..43bf9d63271 100644 --- a/src/xnnpack/simd/f16-scalar.h +++ b/src/xnnpack/simd/f16-scalar.h @@ -53,18 +53,11 @@ static XNN_INLINE xnn_simd_f16_t xnn_mul_f16(xnn_simd_f16_t a, #endif // XNN_HAVE_FLOAT16 } -// If we're computing the fused ops in `float`, act as if we're going to -// round like native FMA. +// Plain scalar f16 expressions do not explicitly use fused operations. #if XNN_HAVE_FLOAT16 -#if ((XNN_ARCH_X86 || XNN_ARCH_X86_64) && defined(__FMA__)) || \ - (XNN_ARCH_ARM64 && __ARM_FEATURE_FMA && defined(__ARM_FEATURE_FP16_FML)) -#define XNN_SIMD_HAS_NATIVE_FMA 1 -#else #define XNN_SIMD_HAS_NATIVE_FMA 0 -#endif // ((XNN_ARCH_X86 || XNN_ARCH_X86_64) && defined(__FMA__)) || - // (XNN_ARCH_ARM64 && __ARM_FEATURE_FMA && - // defined(__ARM_FEATURE_FP16_FML)) #else +// Computing f16 fused ops via float rounds once at the final f16 conversion. #define XNN_SIMD_HAS_NATIVE_FMA 1 #endif // XNN_HAVE_FLOAT16 diff --git a/test/subgraph/fully-connected.cc b/test/subgraph/fully-connected.cc index da5fb441e3b..fcf61e5d31b 100644 --- a/test/subgraph/fully-connected.cc +++ b/test/subgraph/fully-connected.cc @@ -766,6 +766,233 @@ TEST(FullyConnectedF32, dynamic_b) { TestDynamicB(); } +TEST(FullyConnectedQP8F16QC8W, optimize_packed_lhs_inline) { + ASSERT_EQ(xnn_status_success, xnn_initialize(nullptr)); + + xnn_subgraph_t subgraph = nullptr; + ASSERT_EQ(xnn_status_success, xnn_create_subgraph(2, 0, &subgraph)); + + const size_t input_dims[] = {2, 4}; + uint32_t input_id = XNN_INVALID_VALUE_ID; + ASSERT_EQ(xnn_status_success, + xnn_define_tensor_value( + subgraph, xnn_datatype_fp16, 2, input_dims, nullptr, + /*external_id=*/0, XNN_VALUE_FLAG_EXTERNAL_INPUT, &input_id)); + + uint32_t qd_input_id = XNN_INVALID_VALUE_ID; + ASSERT_EQ(xnn_status_success, + xnn_define_dynamically_quantized_tensor_value( + subgraph, xnn_datatype_qdint8, 2, /*num_nonbatch_dims=*/1, + input_dims, XNN_INVALID_VALUE_ID, /*flags=*/0, &qd_input_id)); + + const size_t kernel_dims[] = {3, 4}; + const float kernel_scale[] = {0.5f, 0.75f, 1.0f}; + const int8_t kernel[12] = {}; + uint32_t kernel_id = XNN_INVALID_VALUE_ID; + ASSERT_EQ(xnn_status_success, + xnn_define_channelwise_quantized_tensor_value( + subgraph, xnn_datatype_qcint8, kernel_scale, 2, + /*channel_dim=*/0, kernel_dims, kernel, XNN_INVALID_VALUE_ID, + /*flags=*/0, &kernel_id)); + + const size_t bias_dims[] = {3}; + const float bias[3] = {}; + uint32_t bias_id = XNN_INVALID_VALUE_ID; + ASSERT_EQ(xnn_status_success, + xnn_define_tensor_value(subgraph, xnn_datatype_fp32, 1, bias_dims, + bias, XNN_INVALID_VALUE_ID, /*flags=*/0, + &bias_id)); + + const size_t output_dims[] = {2, 3}; + uint32_t output_id = XNN_INVALID_VALUE_ID; + ASSERT_EQ(xnn_status_success, + xnn_define_tensor_value( + subgraph, xnn_datatype_fp16, 2, output_dims, nullptr, + /*external_id=*/1, XNN_VALUE_FLAG_EXTERNAL_OUTPUT, &output_id)); + + ASSERT_EQ(xnn_status_success, + xnn_define_unary(subgraph, xnn_unary_convert, /*params=*/nullptr, + input_id, qd_input_id, /*flags=*/0)); + ASSERT_EQ(xnn_status_success, + xnn_define_fully_connected(subgraph, -1e30f, 1e30f, qd_input_id, + kernel_id, bias_id, output_id, + /*flags=*/0)); + + ASSERT_EQ(xnn_status_success, xnn_subgraph_optimize(subgraph, /*flags=*/0)); + + const xnn_node* fc_node = nullptr; + for (size_t i = 0; i < subgraph->num_nodes; i++) { + if (subgraph->nodes[i].type == xnn_node_type_fully_connected) { + fc_node = &subgraph->nodes[i]; + break; + } + } + ASSERT_NE(fc_node, nullptr); + if (fc_node->packed_input_datatype != xnn_datatype_qpint8) { + xnn_delete_subgraph(subgraph); + GTEST_SKIP() << "packed-LHS QP8 F16 QC8W rewrite is unavailable"; + } + EXPECT_NE(fc_node->inputs[0], qd_input_id); + EXPECT_EQ(fc_node->packed_input_datatype, xnn_datatype_qpint8); + EXPECT_EQ(subgraph->values[fc_node->inputs[0]].datatype, xnn_datatype_fp16); + + xnn_delete_subgraph(subgraph); +} + +TEST(FullyConnectedQP8F16QC8W, optimize_packed_lhs_rejects_fp32_source) { + ASSERT_EQ(xnn_status_success, xnn_initialize(nullptr)); + + xnn_subgraph_t subgraph = nullptr; + ASSERT_EQ(xnn_status_success, xnn_create_subgraph(2, 0, &subgraph)); + + const size_t input_dims[] = {2, 4}; + uint32_t input_id = XNN_INVALID_VALUE_ID; + ASSERT_EQ(xnn_status_success, + xnn_define_tensor_value( + subgraph, xnn_datatype_fp32, 2, input_dims, nullptr, + /*external_id=*/0, XNN_VALUE_FLAG_EXTERNAL_INPUT, &input_id)); + + uint32_t qd_input_id = XNN_INVALID_VALUE_ID; + ASSERT_EQ(xnn_status_success, + xnn_define_dynamically_quantized_tensor_value( + subgraph, xnn_datatype_qdint8, 2, /*num_nonbatch_dims=*/1, + input_dims, XNN_INVALID_VALUE_ID, /*flags=*/0, &qd_input_id)); + + const size_t kernel_dims[] = {3, 4}; + const float kernel_scale[] = {0.5f, 0.75f, 1.0f}; + const int8_t kernel[12] = {}; + uint32_t kernel_id = XNN_INVALID_VALUE_ID; + ASSERT_EQ(xnn_status_success, + xnn_define_channelwise_quantized_tensor_value( + subgraph, xnn_datatype_qcint8, kernel_scale, 2, + /*channel_dim=*/0, kernel_dims, kernel, XNN_INVALID_VALUE_ID, + /*flags=*/0, &kernel_id)); + + const size_t bias_dims[] = {3}; + const float bias[3] = {}; + uint32_t bias_id = XNN_INVALID_VALUE_ID; + ASSERT_EQ(xnn_status_success, + xnn_define_tensor_value(subgraph, xnn_datatype_fp32, 1, bias_dims, + bias, XNN_INVALID_VALUE_ID, /*flags=*/0, + &bias_id)); + + const size_t output_dims[] = {2, 3}; + uint32_t output_id = XNN_INVALID_VALUE_ID; + ASSERT_EQ(xnn_status_success, + xnn_define_tensor_value( + subgraph, xnn_datatype_fp16, 2, output_dims, nullptr, + /*external_id=*/1, XNN_VALUE_FLAG_EXTERNAL_OUTPUT, &output_id)); + + ASSERT_EQ(xnn_status_success, + xnn_define_unary(subgraph, xnn_unary_convert, /*params=*/nullptr, + input_id, qd_input_id, /*flags=*/0)); + ASSERT_EQ(xnn_status_success, + xnn_define_fully_connected(subgraph, -1e30f, 1e30f, qd_input_id, + kernel_id, bias_id, output_id, + /*flags=*/0)); + + const xnn_status status = xnn_subgraph_optimize(subgraph, /*flags=*/0); + if (status == xnn_status_unsupported_hardware) { + xnn_delete_subgraph(subgraph); + GTEST_SKIP() << "packed-LHS QP8 F16 QC8W path is unavailable"; + } + ASSERT_EQ(xnn_status_success, status); + + const xnn_node* fc_node = nullptr; + for (size_t i = 0; i < subgraph->num_nodes; i++) { + if (subgraph->nodes[i].type == xnn_node_type_fully_connected) { + fc_node = &subgraph->nodes[i]; + break; + } + } + ASSERT_NE(fc_node, nullptr); + EXPECT_NE(fc_node->packed_input_datatype, xnn_datatype_qpint8); + + xnn_delete_subgraph(subgraph); +} + +TEST(FullyConnectedQP8F16QC8W, optimize_packed_lhs_no_inline) { + ASSERT_EQ(xnn_status_success, xnn_initialize(nullptr)); + + xnn_subgraph_t subgraph = nullptr; + ASSERT_EQ(xnn_status_success, xnn_create_subgraph(2, 0, &subgraph)); + + const size_t input_dims[] = {2, 4}; + uint32_t input_id = XNN_INVALID_VALUE_ID; + ASSERT_EQ(xnn_status_success, + xnn_define_tensor_value( + subgraph, xnn_datatype_fp16, 2, input_dims, nullptr, + /*external_id=*/0, XNN_VALUE_FLAG_EXTERNAL_INPUT, &input_id)); + + uint32_t qd_input_id = XNN_INVALID_VALUE_ID; + ASSERT_EQ(xnn_status_success, + xnn_define_dynamically_quantized_tensor_value( + subgraph, xnn_datatype_qdint8, 2, /*num_nonbatch_dims=*/1, + input_dims, XNN_INVALID_VALUE_ID, /*flags=*/0, &qd_input_id)); + + const size_t kernel_dims[] = {3, 4}; + const float kernel_scale[] = {0.5f, 0.75f, 1.0f}; + const int8_t kernel[12] = {}; + uint32_t kernel_id = XNN_INVALID_VALUE_ID; + ASSERT_EQ(xnn_status_success, + xnn_define_channelwise_quantized_tensor_value( + subgraph, xnn_datatype_qcint8, kernel_scale, 2, + /*channel_dim=*/0, kernel_dims, kernel, XNN_INVALID_VALUE_ID, + /*flags=*/0, &kernel_id)); + + const size_t bias_dims[] = {3}; + const float bias[3] = {}; + uint32_t bias_id = XNN_INVALID_VALUE_ID; + ASSERT_EQ(xnn_status_success, + xnn_define_tensor_value(subgraph, xnn_datatype_fp32, 1, bias_dims, + bias, XNN_INVALID_VALUE_ID, /*flags=*/0, + &bias_id)); + + const size_t output_dims[] = {2, 3}; + uint32_t output_id = XNN_INVALID_VALUE_ID; + ASSERT_EQ(xnn_status_success, + xnn_define_tensor_value( + subgraph, xnn_datatype_fp16, 2, output_dims, nullptr, + /*external_id=*/1, XNN_VALUE_FLAG_EXTERNAL_OUTPUT, &output_id)); + + ASSERT_EQ(xnn_status_success, + xnn_define_unary(subgraph, xnn_unary_convert, /*params=*/nullptr, + input_id, qd_input_id, /*flags=*/0)); + ASSERT_EQ(xnn_status_success, + xnn_define_fully_connected(subgraph, -1e30f, 1e30f, qd_input_id, + kernel_id, bias_id, output_id, + /*flags=*/0)); + + const xnn_status status = + xnn_subgraph_optimize(subgraph, XNN_FLAG_NO_INLINED_LHS_PACKING); + if (status == xnn_status_unsupported_hardware) { + xnn_delete_subgraph(subgraph); + GTEST_SKIP() << "packed-LHS QP8 F16 QC8W path is unavailable"; + } + ASSERT_EQ(xnn_status_success, status); + + const xnn_node* fc_node = nullptr; + for (size_t i = 0; i < subgraph->num_nodes; i++) { + if (subgraph->nodes[i].type == xnn_node_type_fully_connected) { + fc_node = &subgraph->nodes[i]; + break; + } + } + ASSERT_NE(fc_node, nullptr); + if (fc_node->inputs[0] == qd_input_id) { + xnn_delete_subgraph(subgraph); + GTEST_SKIP() << "packed-LHS QP8 F16 QC8W rewrite is unavailable"; + } + if (subgraph->values[fc_node->inputs[0]].datatype != xnn_datatype_qpint8) { + xnn_delete_subgraph(subgraph); + GTEST_SKIP() << "packed-LHS QP8 F16 QC8W rewrite is unavailable"; + } + EXPECT_EQ(subgraph->values[fc_node->inputs[0]].datatype, xnn_datatype_qpint8); + EXPECT_NE(subgraph->values[fc_node->inputs[0]].gemm_config, nullptr); + + xnn_delete_subgraph(subgraph); +} + #ifndef XNNPACK_USE_YNNPACK TEST(FullyConnectedQS8, filter_zero_point_must_be_zero) { ASSERT_EQ(xnn_status_success, xnn_initialize(nullptr)); diff --git a/third_party/kleidiai/BUILD.gn b/third_party/kleidiai/BUILD.gn index c32d0a4a017..2bad56c1ec0 100644 --- a/third_party/kleidiai/BUILD.gn +++ b/third_party/kleidiai/BUILD.gn @@ -111,6 +111,12 @@ source_set("matmul") { "src/kai/ukernels/matmul/matmul_clamp_f16_qai8dxp_qsi8cxp/kai_matmul_clamp_f16_qai8dxp4x8_qsi8cxp4x8_16x4_neon_i8mm.c", "src/kai/ukernels/matmul/matmul_clamp_f16_qai8dxp_qsi8cxp/kai_matmul_clamp_f16_qai8dxp4x8_qsi8cxp4x8_16x4_neon_i8mm.h", "src/kai/ukernels/matmul/matmul_clamp_f16_qai8dxp_qsi8cxp/kai_matmul_clamp_f16_qai8dxp4x8_qsi8cxp4x8_16x4_neon_i8mm_asm.S", + "src/kai/ukernels/matmul/matmul_clamp_f16_qai8dxp_qsi8cxp/kai_matmul_clamp_f16_qai8dxp1vlx4_qsi8cxp4vlx4_1vlx4vl_sme2_mopa.c", + "src/kai/ukernels/matmul/matmul_clamp_f16_qai8dxp_qsi8cxp/kai_matmul_clamp_f16_qai8dxp1vlx4_qsi8cxp4vlx4_1vlx4vl_sme2_mopa.h", + "src/kai/ukernels/matmul/matmul_clamp_f16_qai8dxp_qsi8cxp/kai_matmul_clamp_f16_qai8dxp1vlx4_qsi8cxp4vlx4_1vlx4vl_sme2_mopa_asm.S", + "src/kai/ukernels/matmul/matmul_clamp_f16_qai8dxp_qsi8cxp/kai_matmul_clamp_f16_qai8dxp1x4_qsi8cxp4vlx4_1x4vl_sme2_dot.c", + "src/kai/ukernels/matmul/matmul_clamp_f16_qai8dxp_qsi8cxp/kai_matmul_clamp_f16_qai8dxp1x4_qsi8cxp4vlx4_1x4vl_sme2_dot.h", + "src/kai/ukernels/matmul/matmul_clamp_f16_qai8dxp_qsi8cxp/kai_matmul_clamp_f16_qai8dxp1x4_qsi8cxp4vlx4_1x4vl_sme2_dot_asm.S", "src/kai/ukernels/matmul/matmul_clamp_f16_qai8dxp_qsi8cxp/kai_matmul_clamp_f16_qai8dxp_qsi8cxp_interface.h", "src/kai/ukernels/matmul/matmul_clamp_f16_qsi8d32p_qai4c32p/kai_matmul_clamp_f16_qsi8d32p1vlx4_qai4c32p4vlx4_1vlx4vl_sme2_mopa.c", "src/kai/ukernels/matmul/matmul_clamp_f16_qsi8d32p_qai4c32p/kai_matmul_clamp_f16_qsi8d32p1vlx4_qai4c32p4vlx4_1vlx4vl_sme2_mopa.h", From 0101ef7b34514c3b04a4ef62b5a6c286e0f417fd Mon Sep 17 00:00:00 2001 From: Colm Donelan Date: Mon, 13 Jul 2026 12:53:22 +0100 Subject: [PATCH 2/3] Fixing minor comments around the location and format of asserts Signed-off-by: Colm Donelan --- src/configs/gemm-config.c | 2 +- .../qp8-f16-qc8w-gemm-minmax-1x64c4-neonsme2.c | 6 ++---- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/src/configs/gemm-config.c b/src/configs/gemm-config.c index d732090ed77..fca9bccee17 100644 --- a/src/configs/gemm-config.c +++ b/src/configs/gemm-config.c @@ -3041,9 +3041,9 @@ static void init_qp8_f16_qc8w_gemm_config(void) { qp8_f16_qc8w_gemm_config.nr = nr; qp8_f16_qc8w_gemm_config.log2_kr = 2; qp8_f16_qc8w_gemm_config.mr_packed = mr; + assert(qp8_f16_qc8w_gemm_config.mr <= XNN_MAX_MR); #endif // XNN_ENABLE_ARM_SME2 } - assert(qp8_f16_qc8w_gemm_config.mr <= XNN_MAX_MR); #endif // XNN_ARCH_ARM64 && XNN_ENABLE_KLEIDIAI } diff --git a/src/qp8-f16-qc8w-gemm/qp8-f16-qc8w-gemm-minmax-1x64c4-neonsme2.c b/src/qp8-f16-qc8w-gemm/qp8-f16-qc8w-gemm-minmax-1x64c4-neonsme2.c index f4315994cb9..9771e289b14 100644 --- a/src/qp8-f16-qc8w-gemm/qp8-f16-qc8w-gemm-minmax-1x64c4-neonsme2.c +++ b/src/qp8-f16-qc8w-gemm/qp8-f16-qc8w-gemm-minmax-1x64c4-neonsme2.c @@ -38,9 +38,7 @@ void xnn_qp8_f16_qc8w_gemm_minmax_ukernel_1x64c4__neonsme2( xnn_float16_to_float(f16_minmax_params->scalar.min), xnn_float16_to_float(f16_minmax_params->scalar.max)); #else - assert( - "Calling KleidiAI microkernel wrapper, but XNNPACK was compiled without " - "`XNN_ENABLE_KLEIDIAI`." && - 0); + assert("Calling KleidiAI microkernel wrapper, but XNNPACK was compiled without " + "`XNN_ENABLE_KLEIDIAI`." && 0); #endif // XNN_ENABLE_KLEIDIAI } From be4f956194857f1c85d81aa95a81c59a7403084d Mon Sep 17 00:00:00 2001 From: Colm Donelan Date: Thu, 16 Jul 2026 10:15:14 +0100 Subject: [PATCH 3/3] Backing out change to XNN_SIMD_HAS_NATIVE_FMA in simd/f16-scalar.h * Reverting an earlier change to src/xnnpack/simd/f16-scalar.h as it appears to be as a result of CI artifact. * Fixing an assert format in qp8-f16-qc8w-gemm-minmax-16x64c4-neonsme2.c Signed-off-by: Colm Donelan --- .../qp8-f16-qc8w-gemm-minmax-16x64c4-neonsme2.c | 6 ++---- src/xnnpack/simd/f16-scalar.h | 11 +++++++++-- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/src/qp8-f16-qc8w-gemm/qp8-f16-qc8w-gemm-minmax-16x64c4-neonsme2.c b/src/qp8-f16-qc8w-gemm/qp8-f16-qc8w-gemm-minmax-16x64c4-neonsme2.c index c1dbf97095b..b1839f3b5a4 100644 --- a/src/qp8-f16-qc8w-gemm/qp8-f16-qc8w-gemm-minmax-16x64c4-neonsme2.c +++ b/src/qp8-f16-qc8w-gemm/qp8-f16-qc8w-gemm-minmax-16x64c4-neonsme2.c @@ -39,9 +39,7 @@ void xnn_qp8_f16_qc8w_gemm_minmax_ukernel_16x64c4__neonsme2( xnn_float16_to_float(f16_minmax_params->scalar.min), xnn_float16_to_float(f16_minmax_params->scalar.max)); #else - assert( - "Calling KleidiAI microkernel wrapper, but XNNPACK was compiled without " - "`XNN_ENABLE_KLEIDIAI`." && - 0); + assert("Calling KleidiAI microkernel wrapper, but XNNPACK was compiled without " + "`XNN_ENABLE_KLEIDIAI`." && 0); #endif // XNN_ENABLE_KLEIDIAI } diff --git a/src/xnnpack/simd/f16-scalar.h b/src/xnnpack/simd/f16-scalar.h index 43bf9d63271..e0f979fb044 100644 --- a/src/xnnpack/simd/f16-scalar.h +++ b/src/xnnpack/simd/f16-scalar.h @@ -53,11 +53,18 @@ static XNN_INLINE xnn_simd_f16_t xnn_mul_f16(xnn_simd_f16_t a, #endif // XNN_HAVE_FLOAT16 } -// Plain scalar f16 expressions do not explicitly use fused operations. +// If we're computing the fused ops in `float`, act as if we're going to +// round like native FMA. #if XNN_HAVE_FLOAT16 +#if ((XNN_ARCH_X86 || XNN_ARCH_X86_64) && defined(__FMA__)) || \ + (XNN_ARCH_ARM64 && __ARM_FEATURE_FMA && defined(__ARM_FEATURE_FP16_FML)) +#define XNN_SIMD_HAS_NATIVE_FMA 1 +#else #define XNN_SIMD_HAS_NATIVE_FMA 0 +#endif // ((XNN_ARCH_X86 || XNN_ARCH_X86_64) && defined(__FMA__)) || + // (XNN_ARCH_ARM64 && __ARM_FEATURE_FMA && + // defined(__ARM_FEATURE_FP16_FML)) #else -// Computing f16 fused ops via float rounds once at the final f16 conversion. #define XNN_SIMD_HAS_NATIVE_FMA 1 #endif // XNN_HAVE_FLOAT16