diff --git a/cmake/gen/avx256vnni_microkernels.cmake b/cmake/gen/avx256vnni_microkernels.cmake index fd798d67d74..6808b9eb59e 100644 --- a/cmake/gen/avx256vnni_microkernels.cmake +++ b/cmake/gen/avx256vnni_microkernels.cmake @@ -21,6 +21,7 @@ SET(PROD_AVX256VNNI_MICROKERNEL_SRCS src/qs8-packw/gen/qs8-packw-x64c4-gemm-goi-avx256vnni-prfm.c src/qs8-qc4w-gemm/gen/qs8-qc4w-gemm-1x8c8-minmax-fp32-avx256vnni-prfm.c src/qs8-qc4w-gemm/gen/qs8-qc4w-gemm-7x8c8-minmax-fp32-avx256vnni-prfm.c + src/qs8-qc4w-packw/gen/qs8-qc4w-packw-x8c8-gemm-goi-avx256vnni.c src/qs8-qc4w-packw/gen/qs8-qc4w-packw-x16c8-gemm-goi-avx256vnni.c src/qs8-qc4w-packw/gen/qs8-to-qu8-qc4w-packw-x8c8-gemm-goi-avx256vnni.c src/qs8-qc4w-packw/gen/qs8-to-qu8-qc4w-packw-x16c8-gemm-goi-avx256vnni.c @@ -135,7 +136,6 @@ SET(NON_PROD_AVX256VNNI_MICROKERNEL_SRCS src/qs8-qc4w-gemm/gen/qs8-qc4w-gemm-8x8c8-minmax-fp32-avx256vnni-prfm.c src/qs8-qc4w-gemm/gen/qs8-qc4w-gemm-8x8c8-minmax-fp32-avx256vnni.c src/qs8-qc4w-packw/gen/qs8-qc4w-packw-x8c8-gemm-goi-avx256vnni-prfm.c - src/qs8-qc4w-packw/gen/qs8-qc4w-packw-x8c8-gemm-goi-avx256vnni.c src/qs8-qc4w-packw/gen/qs8-qc4w-packw-x16c8-gemm-goi-avx256vnni-prfm.c src/qs8-qc4w-packw/gen/qs8-to-qu8-qc4w-packw-x8c8-gemm-goi-avx256vnni-prfm.c src/qs8-qc4w-packw/gen/qs8-to-qu8-qc4w-packw-x16c8-gemm-goi-avx256vnni-prfm.c diff --git a/cmake/gen/scalar_microkernels.cmake b/cmake/gen/scalar_microkernels.cmake index e013182b389..17849c82a87 100644 --- a/cmake/gen/scalar_microkernels.cmake +++ b/cmake/gen/scalar_microkernels.cmake @@ -169,6 +169,8 @@ SET(PROD_SCALAR_MICROKERNEL_SRCS src/qs8-qc2w-gemm/gen/qs8-qc2w-gemm-4x4-minmax-fp32-scalar-fmagic.c src/qs8-qc4w-gemm/gen/qs8-qc4w-gemm-1x4-minmax-fp32-scalar-fmagic.c src/qs8-qc4w-gemm/gen/qs8-qc4w-gemm-3x4-minmax-fp32-scalar-fmagic.c + src/qs8-qc4w-packw/gen/qs8-qc4uw-packw-x16c8-gemm-goi-scalar.c + src/qs8-qc4w-packw/gen/qs8-to-qu8-qc4uw-packw-x16c8-gemm-goi-scalar.c src/qs8-qc8w-dwconv/gen/qs8-qc8w-dwconv-3p1c-minmax-fp32-scalar-fmagic.c src/qs8-qc8w-dwconv/gen/qs8-qc8w-dwconv-3p2c-minmax-fp32-scalar-lrintf.c src/qs8-qc8w-dwconv/gen/qs8-qc8w-dwconv-9p1c-minmax-fp32-scalar-fmagic.c @@ -640,14 +642,12 @@ SET(NON_PROD_SCALAR_MICROKERNEL_SRCS src/qs8-qc4w-gemm/gen/qs8-qc4w-gemm-4x4-minmax-fp32-scalar-fmagic.c src/qs8-qc4w-packw/gen/qs8-qc4uw-packw-x4c8-gemm-goi-scalar.c src/qs8-qc4w-packw/gen/qs8-qc4uw-packw-x8c8-gemm-goi-scalar.c - src/qs8-qc4w-packw/gen/qs8-qc4uw-packw-x16c8-gemm-goi-scalar.c src/qs8-qc4w-packw/gen/qs8-qc4w-packw-x4c8-gemm-goi-scalar.c src/qs8-qc4w-packw/gen/qs8-qc4w-packw-x8c8-gemm-goi-scalar.c src/qs8-qc4w-packw/gen/qs8-qc4w-packw-x16c8-gemm-goi-scalar.c src/qs8-qc4w-packw/gen/qs8-qc4w-packw-x32c8-gemm-goi-scalar.c src/qs8-qc4w-packw/gen/qs8-to-qu8-qc4uw-packw-x4c8-gemm-goi-scalar.c src/qs8-qc4w-packw/gen/qs8-to-qu8-qc4uw-packw-x8c8-gemm-goi-scalar.c - src/qs8-qc4w-packw/gen/qs8-to-qu8-qc4uw-packw-x16c8-gemm-goi-scalar.c src/qs8-qc4w-packw/gen/qs8-to-qu8-qc4w-packw-x4c8-gemm-goi-scalar.c src/qs8-qc4w-packw/gen/qs8-to-qu8-qc4w-packw-x8c8-gemm-goi-scalar.c src/qs8-qc4w-packw/gen/qs8-to-qu8-qc4w-packw-x16c8-gemm-goi-scalar.c diff --git a/cmake/gen/sse2_microkernels.cmake b/cmake/gen/sse2_microkernels.cmake index 06b9298dcbc..671e71d065b 100644 --- a/cmake/gen/sse2_microkernels.cmake +++ b/cmake/gen/sse2_microkernels.cmake @@ -66,7 +66,9 @@ SET(PROD_SSE2_MICROKERNEL_SRCS src/qs8-dwconv/gen/qs8-dwconv-25p8c-minmax-fp32-sse2-mul16-add16.c src/qs8-f32-vcvt/gen/qs8-f32-vcvt-sse2-u32.c src/qs8-qc4w-packw/gen/qs8-qc4uw-packw-x4c8-gemm-goi-sse2.c + src/qs8-qc4w-packw/gen/qs8-qc4uw-packw-x8c8-gemm-goi-sse2.c src/qs8-qc4w-packw/gen/qs8-to-qu8-qc4uw-packw-x4c8-gemm-goi-sse2.c + src/qs8-qc4w-packw/gen/qs8-to-qu8-qc4uw-packw-x8c8-gemm-goi-sse2.c src/qs8-qc8w-dwconv/gen/qs8-qc8w-dwconv-3p8c-minmax-fp32-sse2-mul16.c src/qs8-qc8w-dwconv/gen/qs8-qc8w-dwconv-9p8c-minmax-fp32-sse2-mul16.c src/qs8-qc8w-dwconv/gen/qs8-qc8w-dwconv-25p8c-minmax-fp32-sse2-mul16.c @@ -198,10 +200,8 @@ SET(NON_PROD_SSE2_MICROKERNEL_SRCS src/qs8-f32-vcvt/gen/qs8-f32-vcvt-sse2-u8.c src/qs8-f32-vcvt/gen/qs8-f32-vcvt-sse2-u16.c src/qs8-f32-vcvt/gen/qs8-f32-vcvt-sse2-u24.c - src/qs8-qc4w-packw/gen/qs8-qc4uw-packw-x8c8-gemm-goi-sse2.c src/qs8-qc4w-packw/gen/qs8-qc4w-packw-x4c8-gemm-goi-sse2.c src/qs8-qc4w-packw/gen/qs8-qc4w-packw-x8c8-gemm-goi-sse2.c - src/qs8-qc4w-packw/gen/qs8-to-qu8-qc4uw-packw-x8c8-gemm-goi-sse2.c src/qs8-qc4w-packw/gen/qs8-to-qu8-qc4w-packw-x4c8-gemm-goi-sse2.c src/qs8-qc4w-packw/gen/qs8-to-qu8-qc4w-packw-x8c8-gemm-goi-sse2.c src/qs8-qc8w-dwconv/gen/qs8-qc8w-dwconv-9p8c-minmax-fp32-sse2-mul16-add16.c diff --git a/gen/avx256vnni_microkernels.bzl b/gen/avx256vnni_microkernels.bzl index 972f65d7042..8ccc80d72a4 100644 --- a/gen/avx256vnni_microkernels.bzl +++ b/gen/avx256vnni_microkernels.bzl @@ -17,6 +17,7 @@ PROD_AVX256VNNI_MICROKERNEL_SRCS = [ "src/qs8-packw/gen/qs8-packw-x64c4-gemm-goi-avx256vnni-prfm.c", "src/qs8-qc4w-gemm/gen/qs8-qc4w-gemm-1x8c8-minmax-fp32-avx256vnni-prfm.c", "src/qs8-qc4w-gemm/gen/qs8-qc4w-gemm-7x8c8-minmax-fp32-avx256vnni-prfm.c", + "src/qs8-qc4w-packw/gen/qs8-qc4w-packw-x8c8-gemm-goi-avx256vnni.c", "src/qs8-qc4w-packw/gen/qs8-qc4w-packw-x16c8-gemm-goi-avx256vnni.c", "src/qs8-qc4w-packw/gen/qs8-to-qu8-qc4w-packw-x8c8-gemm-goi-avx256vnni.c", "src/qs8-qc4w-packw/gen/qs8-to-qu8-qc4w-packw-x16c8-gemm-goi-avx256vnni.c", @@ -132,7 +133,6 @@ NON_PROD_AVX256VNNI_MICROKERNEL_SRCS = [ "src/qs8-qc4w-gemm/gen/qs8-qc4w-gemm-8x8c8-minmax-fp32-avx256vnni-prfm.c", "src/qs8-qc4w-gemm/gen/qs8-qc4w-gemm-8x8c8-minmax-fp32-avx256vnni.c", "src/qs8-qc4w-packw/gen/qs8-qc4w-packw-x8c8-gemm-goi-avx256vnni-prfm.c", - "src/qs8-qc4w-packw/gen/qs8-qc4w-packw-x8c8-gemm-goi-avx256vnni.c", "src/qs8-qc4w-packw/gen/qs8-qc4w-packw-x16c8-gemm-goi-avx256vnni-prfm.c", "src/qs8-qc4w-packw/gen/qs8-to-qu8-qc4w-packw-x8c8-gemm-goi-avx256vnni-prfm.c", "src/qs8-qc4w-packw/gen/qs8-to-qu8-qc4w-packw-x16c8-gemm-goi-avx256vnni-prfm.c", diff --git a/gen/scalar_microkernels.bzl b/gen/scalar_microkernels.bzl index 945a05fca02..9ed6130e5d3 100644 --- a/gen/scalar_microkernels.bzl +++ b/gen/scalar_microkernels.bzl @@ -165,6 +165,8 @@ PROD_SCALAR_MICROKERNEL_SRCS = [ "src/qs8-qc2w-gemm/gen/qs8-qc2w-gemm-4x4-minmax-fp32-scalar-fmagic.c", "src/qs8-qc4w-gemm/gen/qs8-qc4w-gemm-1x4-minmax-fp32-scalar-fmagic.c", "src/qs8-qc4w-gemm/gen/qs8-qc4w-gemm-3x4-minmax-fp32-scalar-fmagic.c", + "src/qs8-qc4w-packw/gen/qs8-qc4uw-packw-x16c8-gemm-goi-scalar.c", + "src/qs8-qc4w-packw/gen/qs8-to-qu8-qc4uw-packw-x16c8-gemm-goi-scalar.c", "src/qs8-qc8w-dwconv/gen/qs8-qc8w-dwconv-3p1c-minmax-fp32-scalar-fmagic.c", "src/qs8-qc8w-dwconv/gen/qs8-qc8w-dwconv-3p2c-minmax-fp32-scalar-lrintf.c", "src/qs8-qc8w-dwconv/gen/qs8-qc8w-dwconv-9p1c-minmax-fp32-scalar-fmagic.c", @@ -637,14 +639,12 @@ NON_PROD_SCALAR_MICROKERNEL_SRCS = [ "src/qs8-qc4w-gemm/gen/qs8-qc4w-gemm-4x4-minmax-fp32-scalar-fmagic.c", "src/qs8-qc4w-packw/gen/qs8-qc4uw-packw-x4c8-gemm-goi-scalar.c", "src/qs8-qc4w-packw/gen/qs8-qc4uw-packw-x8c8-gemm-goi-scalar.c", - "src/qs8-qc4w-packw/gen/qs8-qc4uw-packw-x16c8-gemm-goi-scalar.c", "src/qs8-qc4w-packw/gen/qs8-qc4w-packw-x4c8-gemm-goi-scalar.c", "src/qs8-qc4w-packw/gen/qs8-qc4w-packw-x8c8-gemm-goi-scalar.c", "src/qs8-qc4w-packw/gen/qs8-qc4w-packw-x16c8-gemm-goi-scalar.c", "src/qs8-qc4w-packw/gen/qs8-qc4w-packw-x32c8-gemm-goi-scalar.c", "src/qs8-qc4w-packw/gen/qs8-to-qu8-qc4uw-packw-x4c8-gemm-goi-scalar.c", "src/qs8-qc4w-packw/gen/qs8-to-qu8-qc4uw-packw-x8c8-gemm-goi-scalar.c", - "src/qs8-qc4w-packw/gen/qs8-to-qu8-qc4uw-packw-x16c8-gemm-goi-scalar.c", "src/qs8-qc4w-packw/gen/qs8-to-qu8-qc4w-packw-x4c8-gemm-goi-scalar.c", "src/qs8-qc4w-packw/gen/qs8-to-qu8-qc4w-packw-x8c8-gemm-goi-scalar.c", "src/qs8-qc4w-packw/gen/qs8-to-qu8-qc4w-packw-x16c8-gemm-goi-scalar.c", diff --git a/gen/sse2_microkernels.bzl b/gen/sse2_microkernels.bzl index 93be562ec31..9feccae3965 100644 --- a/gen/sse2_microkernels.bzl +++ b/gen/sse2_microkernels.bzl @@ -62,7 +62,9 @@ PROD_SSE2_MICROKERNEL_SRCS = [ "src/qs8-dwconv/gen/qs8-dwconv-25p8c-minmax-fp32-sse2-mul16-add16.c", "src/qs8-f32-vcvt/gen/qs8-f32-vcvt-sse2-u32.c", "src/qs8-qc4w-packw/gen/qs8-qc4uw-packw-x4c8-gemm-goi-sse2.c", + "src/qs8-qc4w-packw/gen/qs8-qc4uw-packw-x8c8-gemm-goi-sse2.c", "src/qs8-qc4w-packw/gen/qs8-to-qu8-qc4uw-packw-x4c8-gemm-goi-sse2.c", + "src/qs8-qc4w-packw/gen/qs8-to-qu8-qc4uw-packw-x8c8-gemm-goi-sse2.c", "src/qs8-qc8w-dwconv/gen/qs8-qc8w-dwconv-3p8c-minmax-fp32-sse2-mul16.c", "src/qs8-qc8w-dwconv/gen/qs8-qc8w-dwconv-9p8c-minmax-fp32-sse2-mul16.c", "src/qs8-qc8w-dwconv/gen/qs8-qc8w-dwconv-25p8c-minmax-fp32-sse2-mul16.c", @@ -195,10 +197,8 @@ NON_PROD_SSE2_MICROKERNEL_SRCS = [ "src/qs8-f32-vcvt/gen/qs8-f32-vcvt-sse2-u8.c", "src/qs8-f32-vcvt/gen/qs8-f32-vcvt-sse2-u16.c", "src/qs8-f32-vcvt/gen/qs8-f32-vcvt-sse2-u24.c", - "src/qs8-qc4w-packw/gen/qs8-qc4uw-packw-x8c8-gemm-goi-sse2.c", "src/qs8-qc4w-packw/gen/qs8-qc4w-packw-x4c8-gemm-goi-sse2.c", "src/qs8-qc4w-packw/gen/qs8-qc4w-packw-x8c8-gemm-goi-sse2.c", - "src/qs8-qc4w-packw/gen/qs8-to-qu8-qc4uw-packw-x8c8-gemm-goi-sse2.c", "src/qs8-qc4w-packw/gen/qs8-to-qu8-qc4w-packw-x4c8-gemm-goi-sse2.c", "src/qs8-qc4w-packw/gen/qs8-to-qu8-qc4w-packw-x8c8-gemm-goi-sse2.c", "src/qs8-qc8w-dwconv/gen/qs8-qc8w-dwconv-9p8c-minmax-fp32-sse2-mul16-add16.c", diff --git a/src/configs/gemm-config.c b/src/configs/gemm-config.c index ef21e78e1ed..f2336380c21 100644 --- a/src/configs/gemm-config.c +++ b/src/configs/gemm-config.c @@ -2189,6 +2189,9 @@ static void init_qdu8_f16_qc4w_gemm_config(void) { qdu8_f16_qc4w_gemm_config.arch = xnn_arch_x86_avx256vnni; qdu8_f16_qc4w_gemm_config.minmax.dqgemm[XNN_MR_TO_INDEX(1)] = XNN_INIT_HMP_DQGEMM_UKERNEL(xnn_qd8_f16_qc4w_gemm_minmax_ukernel_1x8c8__avx256vnni); qdu8_f16_qc4w_gemm_config.minmax.dqgemm[XNN_MR_TO_INDEX(8)] = XNN_INIT_HMP_DQGEMM_UKERNEL(xnn_qd8_f16_qc4w_gemm_minmax_ukernel_8x8c8__avx256vnni); + qdu8_f16_qc4w_gemm_config.pack_weights_and_biases = NULL; // Override the default packing function. + qdu8_f16_qc4w_gemm_config.packed_stride_weights_and_biases = NULL; // Override the default packing function. + qdu8_f16_qc4w_gemm_config.pack_gemm_goi = (xnn_packw_gemm_goi_ukernel_fn) xnn_qs8_qc4w_packw_gemm_goi_ukernel_x8c8__avx256vnni; qdu8_f16_qc4w_gemm_config.init.f16_qc4w = xnn_init_f16_qc4w_minmax_scalar_params; qdu8_f16_qc4w_gemm_config.mr = 8; qdu8_f16_qc4w_gemm_config.nr = 8; @@ -2201,6 +2204,9 @@ static void init_qdu8_f16_qc4w_gemm_config(void) { qdu8_f16_qc4w_gemm_config.arch = xnn_arch_x86_avxvnni; qdu8_f16_qc4w_gemm_config.minmax.dqgemm[XNN_MR_TO_INDEX(1)] = XNN_INIT_HMP_DQGEMM_UKERNEL(xnn_qd8_f16_qc4w_gemm_minmax_ukernel_1x8c8__avxvnni_prfm); qdu8_f16_qc4w_gemm_config.minmax.dqgemm[XNN_MR_TO_INDEX(5)] = XNN_INIT_HMP_DQGEMM_UKERNEL(xnn_qd8_f16_qc4w_gemm_minmax_ukernel_5x8c8__avxvnni_prfm); + qdu8_f16_qc4w_gemm_config.pack_weights_and_biases = NULL; // Override the default packing function. + qdu8_f16_qc4w_gemm_config.packed_stride_weights_and_biases = NULL; // Override the default packing function. + qdu8_f16_qc4w_gemm_config.pack_gemm_goi = (xnn_packw_gemm_goi_ukernel_fn) xnn_qs8_qc4w_packw_gemm_goi_ukernel_x8c8__avxvnni; qdu8_f16_qc4w_gemm_config.init.f16_qc4w = xnn_init_f16_qc4w_minmax_scalar_params; qdu8_f16_qc4w_gemm_config.mr = 5; qdu8_f16_qc4w_gemm_config.nr = 8; @@ -2208,7 +2214,7 @@ static void init_qdu8_f16_qc4w_gemm_config(void) { qdu8_f16_qc4w_gemm_config.planes = 2; } else #endif - #if XNN_ENABLE_AVX256SKX + #if XNN_ENABLE_AVX256SKX && XNN_ENABLE_SSE2 if (hardware_config->arch_flags & xnn_arch_x86_avx256skx) { qdu8_f16_qc4w_gemm_config.arch = xnn_arch_x86_avx256skx; qdu8_f16_qc4w_gemm_config.minmax.dqgemm[XNN_MR_TO_INDEX(1)] = XNN_INIT_HMP_DQGEMM_UKERNEL(xnn_qd8_f16_qc4w_gemm_minmax_ukernel_1x8c8__avx256skx_madd_prfm); @@ -2216,7 +2222,8 @@ static void init_qdu8_f16_qc4w_gemm_config(void) { qdu8_f16_qc4w_gemm_config.pack_weights_and_biases = NULL; // Override the default packing function. qdu8_f16_qc4w_gemm_config.packed_stride_weights_and_biases = NULL; // Override the default packing function. qdu8_f16_qc4w_gemm_config.pack_gemm_gio = (xnn_packw_gemm_gio_ukernel_fn) xnn_pack_qs8_qc4uw_gemm_gio_w; - qdu8_f16_qc4w_gemm_config.pack_gemm_goi = (xnn_packw_gemm_goi_ukernel_fn) xnn_pack_qs8_qc4uw_gemm_goi_w; + // TODO(fbarchard): generate avx2 packw instead of sse2 in future + qdu8_f16_qc4w_gemm_config.pack_gemm_goi = (xnn_packw_gemm_goi_ukernel_fn) xnn_qs8_qc4uw_packw_gemm_goi_ukernel_x8c8__sse2; qdu8_f16_qc4w_gemm_config.init.f16_qc4w = xnn_init_f16_qc4w_minmax_scalar_params; qdu8_f16_qc4w_gemm_config.mr = 8; qdu8_f16_qc4w_gemm_config.nr = 8; @@ -2224,20 +2231,23 @@ static void init_qdu8_f16_qc4w_gemm_config(void) { qdu8_f16_qc4w_gemm_config.planes = 2; } else #endif - if (hardware_config->arch_flags & xnn_arch_x86_avx2) { - qdu8_f16_qc4w_gemm_config.arch = xnn_arch_x86_avx2; - qdu8_f16_qc4w_gemm_config.minmax.dqgemm[XNN_MR_TO_INDEX(1)] = XNN_INIT_HMP_DQGEMM_UKERNEL(xnn_qd8_f16_qc4w_gemm_minmax_ukernel_1x8c8__avx2_madd_prfm); - qdu8_f16_qc4w_gemm_config.minmax.dqgemm[XNN_MR_TO_INDEX(4)] = XNN_INIT_HMP_DQGEMM_UKERNEL(xnn_qd8_f16_qc4w_gemm_minmax_ukernel_4x8c8__avx2_madd_prfm); - qdu8_f16_qc4w_gemm_config.pack_weights_and_biases = NULL; // Override the default packing function. - qdu8_f16_qc4w_gemm_config.packed_stride_weights_and_biases = NULL; // Override the default packing function. - qdu8_f16_qc4w_gemm_config.pack_gemm_gio = (xnn_packw_gemm_gio_ukernel_fn) xnn_pack_qs8_qc4uw_gemm_gio_w; - qdu8_f16_qc4w_gemm_config.pack_gemm_goi = (xnn_packw_gemm_goi_ukernel_fn) xnn_pack_qs8_qc4uw_gemm_goi_w; - qdu8_f16_qc4w_gemm_config.init.f16_qc4w = xnn_init_f16_qc4w_minmax_scalar_params; - qdu8_f16_qc4w_gemm_config.mr = 4; - qdu8_f16_qc4w_gemm_config.nr = 8; - qdu8_f16_qc4w_gemm_config.log2_kr = 3; - qdu8_f16_qc4w_gemm_config.planes = 2; - } + #if XNN_ENABLE_AVX2 && XNN_ENABLE_SSE2 + if (hardware_config->arch_flags & xnn_arch_x86_avx2) { + qdu8_f16_qc4w_gemm_config.arch = xnn_arch_x86_avx2; + qdu8_f16_qc4w_gemm_config.minmax.dqgemm[XNN_MR_TO_INDEX(1)] = XNN_INIT_HMP_DQGEMM_UKERNEL(xnn_qd8_f16_qc4w_gemm_minmax_ukernel_1x8c8__avx2_madd_prfm); + qdu8_f16_qc4w_gemm_config.minmax.dqgemm[XNN_MR_TO_INDEX(4)] = XNN_INIT_HMP_DQGEMM_UKERNEL(xnn_qd8_f16_qc4w_gemm_minmax_ukernel_4x8c8__avx2_madd_prfm); + qdu8_f16_qc4w_gemm_config.pack_weights_and_biases = NULL; // Override the default packing function. + qdu8_f16_qc4w_gemm_config.packed_stride_weights_and_biases = NULL; // Override the default packing function. + qdu8_f16_qc4w_gemm_config.pack_gemm_gio = (xnn_packw_gemm_gio_ukernel_fn) xnn_pack_qs8_qc4uw_gemm_gio_w; + // TODO(fbarchard): generate avx2 packw instead of sse2 in future + qdu8_f16_qc4w_gemm_config.pack_gemm_goi = (xnn_packw_gemm_goi_ukernel_fn) xnn_qs8_qc4uw_packw_gemm_goi_ukernel_x8c8__sse2; + qdu8_f16_qc4w_gemm_config.init.f16_qc4w = xnn_init_f16_qc4w_minmax_scalar_params; + qdu8_f16_qc4w_gemm_config.mr = 4; + qdu8_f16_qc4w_gemm_config.nr = 8; + qdu8_f16_qc4w_gemm_config.log2_kr = 3; + qdu8_f16_qc4w_gemm_config.planes = 2; + } else + #endif #endif assert(qdu8_f16_qc4w_gemm_config.mr <= XNN_MAX_MR); assert(qdu8_f16_qc4w_gemm_config.mr <= (XNN_EXTRA_QUANTIZATION_PARAMS + 1)); @@ -4109,27 +4119,31 @@ static void init_qdu8_f32_qc4w_gemm_config(void) { const struct xnn_hardware_config* hardware_config = xnn_init_hardware_config(); assert(hardware_config != NULL); (void) hardware_config; // May be unused. - #if XNN_ENABLE_AVX512VNNIGFNI + #if XNN_ENABLE_AVX512VNNIGFNI && XNN_ENABLE_AVX256VNNI // AMD Zen4 and Zen5 have gfni but it is slow. if ((hardware_config->arch_flags & xnn_arch_x86_avx512vnnigfni) && hardware_config->uarch[XNN_UARCH_INDEX] != xnn_uarch_zen4 && hardware_config->uarch[XNN_UARCH_INDEX] != xnn_uarch_zen5) { qdu8_f32_qc4w_gemm_config.arch = xnn_arch_x86_avx512vnnigfni; qdu8_f32_qc4w_gemm_config.minmax.dqgemm[XNN_MR_TO_INDEX(1)] = XNN_INIT_HMP_DQGEMM_UKERNEL(xnn_qd8_f32_qc4w_gemm_minmax_ukernel_1x16c8__avx512vnnigfni_prfm); qdu8_f32_qc4w_gemm_config.minmax.dqgemm[XNN_MR_TO_INDEX(14)] = XNN_INIT_HMP_DQGEMM_UKERNEL(xnn_qd8_f32_qc4w_gemm_minmax_ukernel_14x16c8__avx512vnnigfni_prfm); qdu8_f32_qc4w_gemm_config.init.f32_qc4w = xnn_init_f32_qc4w_minmax_scalar_params; - + qdu8_f32_qc4w_gemm_config.pack_gemm_goi = (xnn_packw_gemm_goi_ukernel_fn) xnn_qs8_qc4w_packw_gemm_goi_ukernel_x16c8__avx256vnni; + qdu8_f32_qc4w_gemm_config.pack_weights_and_biases = NULL; // Override the default packing function. + qdu8_f32_qc4w_gemm_config.packed_stride_weights_and_biases = NULL; // Override the default packing function. qdu8_f32_qc4w_gemm_config.mr = 14; qdu8_f32_qc4w_gemm_config.nr = 16; qdu8_f32_qc4w_gemm_config.log2_kr = 3; qdu8_f32_qc4w_gemm_config.planes = 2; } else #endif // XNN_ENABLE_AVX512VNNIGFNI - #if XNN_ENABLE_AVX512VNNI + #if XNN_ENABLE_AVX512VNNI && XNN_ENABLE_AVX256VNNI if (hardware_config->arch_flags & xnn_arch_x86_avx512vnni) { qdu8_f32_qc4w_gemm_config.arch = xnn_arch_x86_avx512vnni; qdu8_f32_qc4w_gemm_config.minmax.dqgemm[XNN_MR_TO_INDEX(1)] = XNN_INIT_HMP_DQGEMM_UKERNEL(xnn_qd8_f32_qc4w_gemm_minmax_ukernel_1x16c8__avx512vnni_prfm); qdu8_f32_qc4w_gemm_config.minmax.dqgemm[XNN_MR_TO_INDEX(8)] = XNN_INIT_HMP_DQGEMM_UKERNEL(xnn_qd8_f32_qc4w_gemm_minmax_ukernel_8x16c8__avx512vnni_prfm); qdu8_f32_qc4w_gemm_config.init.f32_qc4w = xnn_init_f32_qc4w_minmax_scalar_params; - + qdu8_f32_qc4w_gemm_config.pack_gemm_goi = (xnn_packw_gemm_goi_ukernel_fn) xnn_qs8_qc4w_packw_gemm_goi_ukernel_x16c8__avx256vnni; + qdu8_f32_qc4w_gemm_config.pack_weights_and_biases = NULL; // Override the default packing function. + qdu8_f32_qc4w_gemm_config.packed_stride_weights_and_biases = NULL; // Override the default packing function. qdu8_f32_qc4w_gemm_config.mr = 8; qdu8_f32_qc4w_gemm_config.nr = 16; qdu8_f32_qc4w_gemm_config.log2_kr = 3; @@ -4142,7 +4156,9 @@ static void init_qdu8_f32_qc4w_gemm_config(void) { qdu8_f32_qc4w_gemm_config.minmax.dqgemm[XNN_MR_TO_INDEX(1)] = XNN_INIT_HMP_DQGEMM_UKERNEL(xnn_qd8_f32_qc4w_gemm_minmax_ukernel_1x8c8__avxvnni_prfm); qdu8_f32_qc4w_gemm_config.minmax.dqgemm[XNN_MR_TO_INDEX(5)] = XNN_INIT_HMP_DQGEMM_UKERNEL(xnn_qd8_f32_qc4w_gemm_minmax_ukernel_5x8c8__avxvnni_prfm); qdu8_f32_qc4w_gemm_config.init.f32_qc4w = xnn_init_f32_qc4w_minmax_scalar_params; - + qdu8_f32_qc4w_gemm_config.pack_gemm_goi = (xnn_packw_gemm_goi_ukernel_fn) xnn_qs8_qc4w_packw_gemm_goi_ukernel_x8c8__avxvnni; + qdu8_f32_qc4w_gemm_config.pack_weights_and_biases = NULL; // Override the default packing function. + qdu8_f32_qc4w_gemm_config.packed_stride_weights_and_biases = NULL; // Override the default packing function. qdu8_f32_qc4w_gemm_config.mr = 5; qdu8_f32_qc4w_gemm_config.nr = 8; qdu8_f32_qc4w_gemm_config.log2_kr = 3; @@ -4157,7 +4173,8 @@ static void init_qdu8_f32_qc4w_gemm_config(void) { qdu8_f32_qc4w_gemm_config.pack_weights_and_biases = NULL; // Override the default packing function. qdu8_f32_qc4w_gemm_config.packed_stride_weights_and_biases = NULL; // Override the default packing function. qdu8_f32_qc4w_gemm_config.pack_gemm_gio = (xnn_packw_gemm_gio_ukernel_fn) xnn_pack_qs8_qc4uw_gemm_gio_w; - qdu8_f32_qc4w_gemm_config.pack_gemm_goi = (xnn_packw_gemm_goi_ukernel_fn) xnn_pack_qs8_qc4uw_gemm_goi_w; + // TODO(fbarchard): generate avx512/avx2 packw instead of scalar in future + qdu8_f32_qc4w_gemm_config.pack_gemm_goi = (xnn_packw_gemm_goi_ukernel_fn) xnn_qs8_qc4uw_packw_gemm_goi_ukernel_x16c8__scalar; qdu8_f32_qc4w_gemm_config.init.f32_qc4w = xnn_init_f32_qc4w_minmax_scalar_params; qdu8_f32_qc4w_gemm_config.mr = 8; qdu8_f32_qc4w_gemm_config.nr = 16; @@ -4165,7 +4182,7 @@ static void init_qdu8_f32_qc4w_gemm_config(void) { qdu8_f32_qc4w_gemm_config.planes = 2; } else #endif - #if XNN_ENABLE_AVX256SKX + #if XNN_ENABLE_AVX256SKX && XNN_ENABLE_SSE2 if (hardware_config->arch_flags & xnn_arch_x86_avx256skx) { qdu8_f32_qc4w_gemm_config.arch = xnn_arch_x86_avx256skx; qdu8_f32_qc4w_gemm_config.minmax.dqgemm[XNN_MR_TO_INDEX(1)] = XNN_INIT_HMP_DQGEMM_UKERNEL(xnn_qd8_f32_qc4w_gemm_minmax_ukernel_1x8c8__avx256skx_madd_prfm); @@ -4173,7 +4190,8 @@ static void init_qdu8_f32_qc4w_gemm_config(void) { qdu8_f32_qc4w_gemm_config.pack_weights_and_biases = NULL; // Override the default packing function. qdu8_f32_qc4w_gemm_config.packed_stride_weights_and_biases = NULL; // Override the default packing function. qdu8_f32_qc4w_gemm_config.pack_gemm_gio = (xnn_packw_gemm_gio_ukernel_fn) xnn_pack_qs8_qc4uw_gemm_gio_w; - qdu8_f32_qc4w_gemm_config.pack_gemm_goi = (xnn_packw_gemm_goi_ukernel_fn) xnn_pack_qs8_qc4uw_gemm_goi_w; + // TODO(fbarchard): generate avx2 packw instead of sse2 in future + qdu8_f32_qc4w_gemm_config.pack_gemm_goi = (xnn_packw_gemm_goi_ukernel_fn) xnn_qs8_qc4uw_packw_gemm_goi_ukernel_x8c8__sse2; qdu8_f32_qc4w_gemm_config.init.f32_qc4w = xnn_init_f32_qc4w_minmax_scalar_params; qdu8_f32_qc4w_gemm_config.mr = 8; qdu8_f32_qc4w_gemm_config.nr = 8; @@ -4181,7 +4199,7 @@ static void init_qdu8_f32_qc4w_gemm_config(void) { qdu8_f32_qc4w_gemm_config.planes = 2; } else #endif - #if XNN_ENABLE_AVX2 + #if XNN_ENABLE_AVX2 && XNN_ENABLE_SSE2 if (hardware_config->arch_flags & xnn_arch_x86_avx2) { qdu8_f32_qc4w_gemm_config.arch = xnn_arch_x86_avx2; qdu8_f32_qc4w_gemm_config.minmax.dqgemm[XNN_MR_TO_INDEX(1)] = XNN_INIT_HMP_DQGEMM_UKERNEL(xnn_qd8_f32_qc4w_gemm_minmax_ukernel_1x8c8__avx2_madd_prfm); @@ -4189,7 +4207,8 @@ static void init_qdu8_f32_qc4w_gemm_config(void) { qdu8_f32_qc4w_gemm_config.pack_weights_and_biases = NULL; // Override the default packing function. qdu8_f32_qc4w_gemm_config.packed_stride_weights_and_biases = NULL; // Override the default packing function. qdu8_f32_qc4w_gemm_config.pack_gemm_gio = (xnn_packw_gemm_gio_ukernel_fn) xnn_pack_qs8_qc4uw_gemm_gio_w; - qdu8_f32_qc4w_gemm_config.pack_gemm_goi = (xnn_packw_gemm_goi_ukernel_fn) xnn_pack_qs8_qc4uw_gemm_goi_w; + // TODO(fbarchard): generate avx2 packw instead of sse2 in future + qdu8_f32_qc4w_gemm_config.pack_gemm_goi = (xnn_packw_gemm_goi_ukernel_fn) xnn_qs8_qc4uw_packw_gemm_goi_ukernel_x8c8__sse2; qdu8_f32_qc4w_gemm_config.init.f32_qc4w = xnn_init_f32_qc4w_minmax_scalar_params; qdu8_f32_qc4w_gemm_config.mr = 4; qdu8_f32_qc4w_gemm_config.nr = 8; @@ -4197,20 +4216,22 @@ static void init_qdu8_f32_qc4w_gemm_config(void) { qdu8_f32_qc4w_gemm_config.planes = 2; } else #endif - if (hardware_config->arch_flags & xnn_arch_x86_ssse3) { - qdu8_f32_qc4w_gemm_config.arch = xnn_arch_x86_ssse3; - qdu8_f32_qc4w_gemm_config.minmax.dqgemm[XNN_MR_TO_INDEX(1)] = XNN_INIT_HMP_DQGEMM_UKERNEL(xnn_qd8_f32_qc4w_gemm_minmax_ukernel_1x4c8__ssse3_madd_prfm); - qdu8_f32_qc4w_gemm_config.minmax.dqgemm[XNN_MR_TO_INDEX(5)] = XNN_INIT_HMP_DQGEMM_UKERNEL(xnn_qd8_f32_qc4w_gemm_minmax_ukernel_5x4c8__ssse3_madd_prfm); - qdu8_f32_qc4w_gemm_config.pack_weights_and_biases = NULL; // Override the default packing function. - qdu8_f32_qc4w_gemm_config.packed_stride_weights_and_biases = NULL; // Override the default packing function. - qdu8_f32_qc4w_gemm_config.pack_gemm_gio = (xnn_packw_gemm_gio_ukernel_fn) xnn_pack_qs8_qc4uw_gemm_gio_w; - qdu8_f32_qc4w_gemm_config.pack_gemm_goi = (xnn_packw_gemm_goi_ukernel_fn) xnn_pack_qs8_qc4uw_gemm_goi_w; - qdu8_f32_qc4w_gemm_config.init.f32_qc4w = xnn_init_f32_qc4w_minmax_scalar_params; - qdu8_f32_qc4w_gemm_config.mr = 5; - qdu8_f32_qc4w_gemm_config.nr = 4; - qdu8_f32_qc4w_gemm_config.log2_kr = 3; - qdu8_f32_qc4w_gemm_config.planes = 2; - } + #if XNN_ENABLE_SSSE3 && XNN_ENABLE_SSE2 + if (hardware_config->arch_flags & xnn_arch_x86_ssse3) { + qdu8_f32_qc4w_gemm_config.arch = xnn_arch_x86_ssse3; + qdu8_f32_qc4w_gemm_config.minmax.dqgemm[XNN_MR_TO_INDEX(1)] = XNN_INIT_HMP_DQGEMM_UKERNEL(xnn_qd8_f32_qc4w_gemm_minmax_ukernel_1x4c8__ssse3_madd_prfm); + qdu8_f32_qc4w_gemm_config.minmax.dqgemm[XNN_MR_TO_INDEX(5)] = XNN_INIT_HMP_DQGEMM_UKERNEL(xnn_qd8_f32_qc4w_gemm_minmax_ukernel_5x4c8__ssse3_madd_prfm); + qdu8_f32_qc4w_gemm_config.pack_weights_and_biases = NULL; // Override the default packing function. + qdu8_f32_qc4w_gemm_config.packed_stride_weights_and_biases = NULL; // Override the default packing function. + qdu8_f32_qc4w_gemm_config.pack_gemm_gio = (xnn_packw_gemm_gio_ukernel_fn) xnn_pack_qs8_qc4uw_gemm_gio_w; + qdu8_f32_qc4w_gemm_config.pack_gemm_goi = (xnn_packw_gemm_goi_ukernel_fn) xnn_qs8_qc4uw_packw_gemm_goi_ukernel_x4c8__sse2; + qdu8_f32_qc4w_gemm_config.init.f32_qc4w = xnn_init_f32_qc4w_minmax_scalar_params; + qdu8_f32_qc4w_gemm_config.mr = 5; + qdu8_f32_qc4w_gemm_config.nr = 4; + qdu8_f32_qc4w_gemm_config.log2_kr = 3; + qdu8_f32_qc4w_gemm_config.planes = 2; + } + #endif assert(qdu8_f32_qc4w_gemm_config.mr <= XNN_MAX_MR); assert(qdu8_f32_qc4w_gemm_config.mr <= (XNN_EXTRA_QUANTIZATION_PARAMS + 1)); #endif //XNN_ARCH_X86 || XNN_ARCH_X86_64 @@ -4959,13 +4980,13 @@ static void init_qs8_qc4w_gemm_config(void) { const struct xnn_hardware_config* hardware_config = xnn_init_hardware_config(); assert(hardware_config != NULL); (void) hardware_config; // May be unused. - #if XNN_ENABLE_AVX512VNNIGFNI + #if XNN_ENABLE_AVX512VNNIGFNI && XNN_ENABLE_AVX256VNNI if (hardware_config->arch_flags & xnn_arch_x86_avx512vnnigfni) { qs8_qc4w_gemm_config.arch = xnn_arch_x86_avx512vnnigfni; qs8_qc4w_gemm_config.minmax.dqgemm[XNN_MR_TO_INDEX(1)] = XNN_INIT_HMP_DQGEMM_UKERNEL(xnn_qs8_qc4w_gemm_minmax_fp32_ukernel_1x16c8__avx512vnnigfni_prfm); qs8_qc4w_gemm_config.minmax.dqgemm[XNN_MR_TO_INDEX(7)] = XNN_INIT_HMP_DQGEMM_UKERNEL(xnn_qs8_qc4w_gemm_minmax_fp32_ukernel_7x16c8__avx512vnnigfni_prfm); qs8_qc4w_gemm_config.init.qs8_qc8w = xnn_init_qs8_qc8w_conv_minmax_fp32_scalar_params; - qs8_qc4w_gemm_config.pack_gemm_goi = (xnn_packw_gemm_goi_ukernel_fn) xnn_pack_qs8_to_qu8_qc4w_gemm_goi_w; + qs8_qc4w_gemm_config.pack_gemm_goi = (xnn_packw_gemm_goi_ukernel_fn) xnn_qs8_to_qu8_qc4w_packw_gemm_goi_ukernel_x16c8__avx256vnni; qs8_qc4w_gemm_config.planes = 2; qs8_qc4w_gemm_config.mr = 7; qs8_qc4w_gemm_config.nr = 16; @@ -4973,7 +4994,7 @@ static void init_qs8_qc4w_gemm_config(void) { } else #endif - #if XNN_ENABLE_AVX512VNNI && XNN_ARCH_X86_64 && !XNN_PLATFORM_WINDOWS && XNN_ENABLE_ASSEMBLY + #if XNN_ENABLE_AVX512VNNI && XNN_ENABLE_AVX256VNNI && XNN_ARCH_X86_64 && !XNN_PLATFORM_WINDOWS && XNN_ENABLE_ASSEMBLY if (hardware_config->arch_flags & xnn_arch_x86_avx512vnni) { qs8_qc4w_gemm_config.arch = xnn_arch_x86_avx512vnni; qs8_qc4w_gemm_config.minmax.dqgemm[XNN_MR_TO_INDEX(1)] = XNN_INIT_HMP_DQGEMM_UKERNEL(xnn_qs8_qc4w_gemm_minmax_fp32_ukernel_1x16c8__asm_amd64_avx512vnni); @@ -4992,7 +5013,7 @@ static void init_qs8_qc4w_gemm_config(void) { qs8_qc4w_gemm_config.minmax.gemm[XNN_MR_TO_INDEX(1)] = XNN_INIT_HMP_GEMM_UKERNEL(xnn_qs8_qc4w_gemm_minmax_fp32_ukernel_1x8c8__avx256vnni_prfm); qs8_qc4w_gemm_config.minmax.gemm[XNN_MR_TO_INDEX(7)] = XNN_INIT_HMP_GEMM_UKERNEL(xnn_qs8_qc4w_gemm_minmax_fp32_ukernel_7x8c8__avx256vnni_prfm); qs8_qc4w_gemm_config.init.qs8_qc8w = xnn_init_qs8_qc8w_conv_minmax_fp32_scalar_params; - qs8_qc4w_gemm_config.pack_gemm_goi = (xnn_packw_gemm_goi_ukernel_fn) xnn_pack_qs8_to_qu8_qc4w_gemm_goi_w; + qs8_qc4w_gemm_config.pack_gemm_goi = (xnn_packw_gemm_goi_ukernel_fn) xnn_qs8_to_qu8_qc4w_packw_gemm_goi_ukernel_x8c8__avx256vnni; qs8_qc4w_gemm_config.planes = 2; qs8_qc4w_gemm_config.mr = 7; qs8_qc4w_gemm_config.nr = 8; @@ -5006,7 +5027,7 @@ static void init_qs8_qc4w_gemm_config(void) { qs8_qc4w_gemm_config.minmax.gemm[XNN_MR_TO_INDEX(1)] = XNN_INIT_HMP_GEMM_UKERNEL(xnn_qs8_qc4w_gemm_minmax_fp32_ukernel_1x8c8__avxvnni_prfm); qs8_qc4w_gemm_config.minmax.gemm[XNN_MR_TO_INDEX(5)] = XNN_INIT_HMP_GEMM_UKERNEL(xnn_qs8_qc4w_gemm_minmax_fp32_ukernel_5x8c8__avxvnni_prfm); qs8_qc4w_gemm_config.init.qs8_qc8w = xnn_init_qs8_qc8w_conv_minmax_fp32_scalar_params; - qs8_qc4w_gemm_config.pack_gemm_goi = (xnn_packw_gemm_goi_ukernel_fn) xnn_pack_qs8_to_qu8_qc4w_gemm_goi_w; + qs8_qc4w_gemm_config.pack_gemm_goi = (xnn_packw_gemm_goi_ukernel_fn) xnn_qs8_to_qu8_qc4w_packw_gemm_goi_ukernel_x8c8__avxvnni; qs8_qc4w_gemm_config.planes = 2; qs8_qc4w_gemm_config.mr = 5; qs8_qc4w_gemm_config.nr = 8; @@ -5019,70 +5040,87 @@ static void init_qs8_qc4w_gemm_config(void) { qs8_qc4w_gemm_config.minmax.gemm[XNN_MR_TO_INDEX(1)] = XNN_INIT_HMP_GEMM_UKERNEL(xnn_qs8_qc4w_gemm_minmax_fp32_ukernel_1x16c8__avx512skx_madd_prfm); qs8_qc4w_gemm_config.minmax.gemm[XNN_MR_TO_INDEX(7)] = XNN_INIT_HMP_GEMM_UKERNEL(xnn_qs8_qc4w_gemm_minmax_fp32_ukernel_7x16c8__avx512skx_madd_prfm); qs8_qc4w_gemm_config.init.qs8_qc8w = xnn_init_qs8_qc8w_conv_minmax_fp32_scalar_params; - qs8_qc4w_gemm_config.pack_gemm_goi = (xnn_packw_gemm_goi_ukernel_fn) xnn_pack_qs8_to_qu8_qc4uw_gemm_goi_w; + // TODO(fbarchard): generate avx512/avx2 packw instead of scalar in future + qs8_qc4w_gemm_config.pack_gemm_goi = (xnn_packw_gemm_goi_ukernel_fn) xnn_qs8_to_qu8_qc4uw_packw_gemm_goi_ukernel_x16c8__scalar; + // TODO(fbarchard): add xnn_pack_qs8_to_qu8_qc4uw_gemm_gio_w in future + qs8_qc4w_gemm_config.pack_weights_and_biases = NULL; + qs8_qc4w_gemm_config.packed_stride_weights_and_biases = NULL; qs8_qc4w_gemm_config.planes = 2; qs8_qc4w_gemm_config.mr = 7; qs8_qc4w_gemm_config.nr = 16; qs8_qc4w_gemm_config.log2_kr = 3; } else #endif - #if XNN_ENABLE_AVX256SKX + #if XNN_ENABLE_AVX256SKX && XNN_ENABLE_SSE2 if (hardware_config->arch_flags & xnn_arch_x86_avx256skx) { qs8_qc4w_gemm_config.arch = xnn_arch_x86_avx256skx; qs8_qc4w_gemm_config.minmax.gemm[XNN_MR_TO_INDEX(1)] = XNN_INIT_HMP_GEMM_UKERNEL(xnn_qs8_qc4w_gemm_minmax_fp32_ukernel_1x8c8__avx256skx_madd_prfm); qs8_qc4w_gemm_config.minmax.gemm[XNN_MR_TO_INDEX(7)] = XNN_INIT_HMP_GEMM_UKERNEL(xnn_qs8_qc4w_gemm_minmax_fp32_ukernel_7x8c8__avx256skx_madd_prfm); qs8_qc4w_gemm_config.init.qs8_qc8w = xnn_init_qs8_qc8w_conv_minmax_fp32_scalar_params; - qs8_qc4w_gemm_config.pack_gemm_goi = (xnn_packw_gemm_goi_ukernel_fn) xnn_pack_qs8_to_qu8_qc4uw_gemm_goi_w; + // TODO(fbarchard): generate avx2 packw instead of sse2 in future + qs8_qc4w_gemm_config.pack_gemm_goi = (xnn_packw_gemm_goi_ukernel_fn) xnn_qs8_to_qu8_qc4uw_packw_gemm_goi_ukernel_x8c8__sse2; + // TODO(fbarchard): add xnn_pack_qs8_to_qu8_qc4uw_gemm_gio_w in future + qs8_qc4w_gemm_config.pack_weights_and_biases = NULL; + qs8_qc4w_gemm_config.packed_stride_weights_and_biases = NULL; qs8_qc4w_gemm_config.planes = 2; qs8_qc4w_gemm_config.mr = 7; qs8_qc4w_gemm_config.nr = 8; qs8_qc4w_gemm_config.log2_kr = 3; } else #endif - #if XNN_ENABLE_AVX2 + #if XNN_ENABLE_AVX2 && XNN_ENABLE_SSE2 if (hardware_config->arch_flags & xnn_arch_x86_avx2) { qs8_qc4w_gemm_config.arch = xnn_arch_x86_avx2; qs8_qc4w_gemm_config.minmax.dqgemm[XNN_MR_TO_INDEX(1)] = XNN_INIT_HMP_DQGEMM_UKERNEL(xnn_qs8_qc4w_gemm_minmax_fp32_ukernel_1x8c8__avx2_madd_prfm); qs8_qc4w_gemm_config.minmax.dqgemm[XNN_MR_TO_INDEX(7)] = XNN_INIT_HMP_DQGEMM_UKERNEL(xnn_qs8_qc4w_gemm_minmax_fp32_ukernel_7x8c8__avx2_madd_prfm); qs8_qc4w_gemm_config.init.qs8_qc8w = xnn_init_qs8_qc8w_conv_minmax_fp32_scalar_params; - qs8_qc4w_gemm_config.pack_gemm_goi = (xnn_packw_gemm_goi_ukernel_fn) xnn_pack_qs8_to_qu8_qc4uw_gemm_goi_w; + // TODO(fbarchard): generate avx2 packw instead of sse2 in future + qs8_qc4w_gemm_config.pack_gemm_goi = (xnn_packw_gemm_goi_ukernel_fn) xnn_qs8_to_qu8_qc4uw_packw_gemm_goi_ukernel_x8c8__sse2; + // TODO(fbarchard): add xnn_pack_qs8_to_qu8_qc4uw_gemm_gio_w in future + qs8_qc4w_gemm_config.pack_weights_and_biases = NULL; + qs8_qc4w_gemm_config.packed_stride_weights_and_biases = NULL; qs8_qc4w_gemm_config.planes = 2; qs8_qc4w_gemm_config.mr = 7; qs8_qc4w_gemm_config.nr = 8; qs8_qc4w_gemm_config.log2_kr = 3; } else #endif - #if XNN_ENABLE_AVX + #if XNN_ENABLE_AVX && XNN_ENABLE_SSE2 if (hardware_config->arch_flags & xnn_arch_x86_avx) { qs8_qc4w_gemm_config.arch = xnn_arch_x86_avx; qs8_qc4w_gemm_config.minmax.dqgemm[XNN_MR_TO_INDEX(1)] = XNN_INIT_HMP_DQGEMM_UKERNEL(xnn_qs8_qc4w_gemm_minmax_fp32_ukernel_1x4c8__avx_madd_prfm); qs8_qc4w_gemm_config.minmax.dqgemm[XNN_MR_TO_INDEX(5)] = XNN_INIT_HMP_DQGEMM_UKERNEL(xnn_qs8_qc4w_gemm_minmax_fp32_ukernel_5x4c8__avx_madd_prfm); qs8_qc4w_gemm_config.init.qs8_qc8w = xnn_init_qs8_qc8w_conv_minmax_fp32_scalar_params; - qs8_qc4w_gemm_config.pack_gemm_goi = (xnn_packw_gemm_goi_ukernel_fn) xnn_pack_qs8_to_qu8_qc4uw_gemm_goi_w; + qs8_qc4w_gemm_config.pack_gemm_goi = (xnn_packw_gemm_goi_ukernel_fn) xnn_qs8_to_qu8_qc4uw_packw_gemm_goi_ukernel_x4c8__sse2; + qs8_qc4w_gemm_config.planes = 2; + qs8_qc4w_gemm_config.mr = 5; + qs8_qc4w_gemm_config.nr = 4; + qs8_qc4w_gemm_config.log2_kr = 3; + } else + #endif + #if XNN_ENABLE_SSSE3 && XNN_ENABLE_SSE2 + if (hardware_config->arch_flags & xnn_arch_x86_ssse3) { + qs8_qc4w_gemm_config.arch = xnn_arch_x86_ssse3; + qs8_qc4w_gemm_config.minmax.dqgemm[XNN_MR_TO_INDEX(1)] = XNN_INIT_HMP_DQGEMM_UKERNEL(xnn_qs8_qc4w_gemm_minmax_fp32_ukernel_1x4c8__ssse3_madd_prfm); + qs8_qc4w_gemm_config.minmax.dqgemm[XNN_MR_TO_INDEX(5)] = XNN_INIT_HMP_DQGEMM_UKERNEL(xnn_qs8_qc4w_gemm_minmax_fp32_ukernel_5x4c8__ssse3_madd_prfm); + qs8_qc4w_gemm_config.init.qs8_qc8w = xnn_init_qs8_qc8w_conv_minmax_fp32_scalar_params; + qs8_qc4w_gemm_config.pack_gemm_goi = (xnn_packw_gemm_goi_ukernel_fn) xnn_qs8_to_qu8_qc4uw_packw_gemm_goi_ukernel_x4c8__sse2; qs8_qc4w_gemm_config.planes = 2; qs8_qc4w_gemm_config.mr = 5; qs8_qc4w_gemm_config.nr = 4; qs8_qc4w_gemm_config.log2_kr = 3; } else #endif - if (hardware_config->arch_flags & xnn_arch_x86_ssse3) { - qs8_qc4w_gemm_config.arch = xnn_arch_x86_ssse3; - qs8_qc4w_gemm_config.minmax.dqgemm[XNN_MR_TO_INDEX(1)] = XNN_INIT_HMP_DQGEMM_UKERNEL(xnn_qs8_qc4w_gemm_minmax_fp32_ukernel_1x4c8__ssse3_madd_prfm); - qs8_qc4w_gemm_config.minmax.dqgemm[XNN_MR_TO_INDEX(5)] = XNN_INIT_HMP_DQGEMM_UKERNEL(xnn_qs8_qc4w_gemm_minmax_fp32_ukernel_5x4c8__ssse3_madd_prfm); - qs8_qc4w_gemm_config.init.qs8_qc8w = xnn_init_qs8_qc8w_conv_minmax_fp32_scalar_params; - qs8_qc4w_gemm_config.pack_gemm_goi = (xnn_packw_gemm_goi_ukernel_fn) xnn_pack_qs8_to_qu8_qc4uw_gemm_goi_w; - qs8_qc4w_gemm_config.planes = 2; - qs8_qc4w_gemm_config.mr = 5; - qs8_qc4w_gemm_config.nr = 4; - qs8_qc4w_gemm_config.log2_kr = 3; - } else #endif //XNN_ARCH_X86 || XNN_ARCH_X86_64 { qs8_qc4w_gemm_config.init.qs8_qc8w = xnn_init_qs8_qc8w_conv_minmax_fp32_scalar_params; qs8_qc4w_gemm_config.minmax.gemm[XNN_MR_TO_INDEX(1)] = XNN_INIT_HMP_GEMM_UKERNEL(xnn_qs8_qc4w_gemm_minmax_fp32_ukernel_1x4__scalar_fmagic); qs8_qc4w_gemm_config.minmax.gemm[XNN_MR_TO_INDEX(3)] = XNN_INIT_HMP_GEMM_UKERNEL(xnn_qs8_qc4w_gemm_minmax_fp32_ukernel_3x4__scalar_fmagic); qs8_qc4w_gemm_config.pack_gemm_goi = (xnn_packw_gemm_goi_ukernel_fn) xnn_pack_qs8_qc4w_gemm_goi_w; - qs8_qc4w_gemm_config.planes = 2; + qs8_qc4w_gemm_config.pack_gemm_gio = (xnn_packw_gemm_gio_ukernel_fn) xnn_pack_qs8_qc4w_gemm_gio_w; + qs8_qc4w_gemm_config.pack_weights_and_biases = NULL; + qs8_qc4w_gemm_config.packed_stride_weights_and_biases = NULL; + qs8_qc4w_gemm_config.planes = 1; qs8_qc4w_gemm_config.mr = 3; qs8_qc4w_gemm_config.nr = 4; }