diff --git a/src/configs/gemm-config.c b/src/configs/gemm-config.c index f2336380c21..8c1f146a98e 100644 --- a/src/configs/gemm-config.c +++ b/src/configs/gemm-config.c @@ -5042,7 +5042,7 @@ static void init_qs8_qc4w_gemm_config(void) { qs8_qc4w_gemm_config.init.qs8_qc8w = xnn_init_qs8_qc8w_conv_minmax_fp32_scalar_params; // TODO(fbarchard): generate avx512/avx2 packw instead of scalar in future qs8_qc4w_gemm_config.pack_gemm_goi = (xnn_packw_gemm_goi_ukernel_fn) xnn_qs8_to_qu8_qc4uw_packw_gemm_goi_ukernel_x16c8__scalar; - // TODO(fbarchard): add xnn_pack_qs8_to_qu8_qc4uw_gemm_gio_w in future + qs8_qc4w_gemm_config.pack_gemm_gio = (xnn_packw_gemm_gio_ukernel_fn) xnn_pack_qs8_to_qu8_qc4uw_gemm_gio_w; qs8_qc4w_gemm_config.pack_weights_and_biases = NULL; qs8_qc4w_gemm_config.packed_stride_weights_and_biases = NULL; qs8_qc4w_gemm_config.planes = 2; @@ -5059,7 +5059,7 @@ static void init_qs8_qc4w_gemm_config(void) { qs8_qc4w_gemm_config.init.qs8_qc8w = xnn_init_qs8_qc8w_conv_minmax_fp32_scalar_params; // TODO(fbarchard): generate avx2 packw instead of sse2 in future qs8_qc4w_gemm_config.pack_gemm_goi = (xnn_packw_gemm_goi_ukernel_fn) xnn_qs8_to_qu8_qc4uw_packw_gemm_goi_ukernel_x8c8__sse2; - // TODO(fbarchard): add xnn_pack_qs8_to_qu8_qc4uw_gemm_gio_w in future + qs8_qc4w_gemm_config.pack_gemm_gio = (xnn_packw_gemm_gio_ukernel_fn) xnn_pack_qs8_to_qu8_qc4uw_gemm_gio_w; qs8_qc4w_gemm_config.pack_weights_and_biases = NULL; qs8_qc4w_gemm_config.packed_stride_weights_and_biases = NULL; qs8_qc4w_gemm_config.planes = 2; @@ -5076,7 +5076,7 @@ static void init_qs8_qc4w_gemm_config(void) { qs8_qc4w_gemm_config.init.qs8_qc8w = xnn_init_qs8_qc8w_conv_minmax_fp32_scalar_params; // TODO(fbarchard): generate avx2 packw instead of sse2 in future qs8_qc4w_gemm_config.pack_gemm_goi = (xnn_packw_gemm_goi_ukernel_fn) xnn_qs8_to_qu8_qc4uw_packw_gemm_goi_ukernel_x8c8__sse2; - // TODO(fbarchard): add xnn_pack_qs8_to_qu8_qc4uw_gemm_gio_w in future + qs8_qc4w_gemm_config.pack_gemm_gio = (xnn_packw_gemm_gio_ukernel_fn) xnn_pack_qs8_to_qu8_qc4uw_gemm_gio_w; qs8_qc4w_gemm_config.pack_weights_and_biases = NULL; qs8_qc4w_gemm_config.packed_stride_weights_and_biases = NULL; qs8_qc4w_gemm_config.planes = 2; diff --git a/src/reference/packing.cc b/src/reference/packing.cc index a7d019685b1..fa98686bfc3 100644 --- a/src/reference/packing.cc +++ b/src/reference/packing.cc @@ -1622,11 +1622,12 @@ void xnn_pack_qs8_qc4w_gemm_gio_w( // Same as qc4w but unsigned 4 bit output // Applies kv ^ 0x88 to convert int4 to uint4 // Does not multiply bias by 16 -void xnn_pack_qs8_qc4uw_gemm_gio_w( - size_t g, size_t nc, size_t kc, size_t nr, size_t kr, size_t sr, - size_t k_stride, const uint8_t* k, const int32_t* b, const float* scale, - void* packed_weights, size_t extra_bytes, - const struct xnn_qs8_qc4w_packing_params* params) { +static void pack_qs8_qc4uw_gemm_gio_w(size_t g, size_t nc, size_t kc, size_t nr, + size_t kr, size_t sr, size_t k_stride, + const uint8_t* k, const int32_t* b, + const float* scale, void* packed_weights, + size_t extra_bytes, uint32_t izp, + uint32_t kernel_zero_point) { assert(g != 0); assert(nc != 0); assert(kc != 0); @@ -1635,12 +1636,9 @@ void xnn_pack_qs8_qc4uw_gemm_gio_w( assert(sr >= 1 && sr <= 16); assert(k != nullptr); assert(packed_weights != nullptr); - assert(params != nullptr); - assert(params->kernel_zero_point == 8 || params->kernel_zero_point == 0); + assert(kernel_zero_point == 8 || kernel_zero_point == 0); const size_t skr = sr * kr; - const uint32_t izp = (uint32_t)params->input_zero_point; - const uint32_t kernel_zero_point = (uint32_t)params->kernel_zero_point; do { size_t nr_block_start = 0; do { @@ -1717,6 +1715,27 @@ void xnn_pack_qs8_qc4uw_gemm_gio_w( } while (--g != 0); } +void xnn_pack_qs8_qc4uw_gemm_gio_w( + size_t g, size_t nc, size_t kc, size_t nr, size_t kr, size_t sr, + size_t k_stride, const uint8_t* k, const int32_t* b, const float* scale, + void* packed_weights, size_t extra_bytes, + const struct xnn_qs8_qc4w_packing_params* params) { + assert(params != nullptr); + pack_qs8_qc4uw_gemm_gio_w( + g, nc, kc, nr, kr, sr, k_stride, k, b, scale, packed_weights, extra_bytes, + params->input_zero_point, params->kernel_zero_point); +} +// For qs8_qc4w madd +void xnn_pack_qs8_to_qu8_qc4uw_gemm_gio_w( + size_t g, size_t nc, size_t kc, size_t nr, size_t kr, size_t sr, + size_t k_stride, const uint8_t* k, const int32_t* b, const float* scale, + void* packed_weights, size_t extra_bytes, + const struct xnn_qs8_qc4w_packing_params* params) { + assert(params != nullptr); + pack_qs8_qc4uw_gemm_gio_w( + g, nc, kc, nr, kr, sr, k_stride, k, b, scale, packed_weights, extra_bytes, + params->input_zero_point + 0x80, params->kernel_zero_point); +} void xnn_pack_f32_qs8w_gemm_goi_w(size_t g, size_t nc, size_t kc, size_t nr, size_t kr, size_t sr, const int8_t* k, const float* bias, const float* scale, diff --git a/src/xnnpack/pack.h b/src/xnnpack/pack.h index 4ad5a19373d..8e6bf1994ed 100644 --- a/src/xnnpack/pack.h +++ b/src/xnnpack/pack.h @@ -634,6 +634,11 @@ XNN_INTERNAL void xnn_pack_qs8_qc4uw_gemm_gio_w( const float* scale, void* packed_weights, size_t extra_bytes, const struct xnn_qs8_qc4w_packing_params* params); +XNN_INTERNAL void xnn_pack_qs8_to_qu8_qc4uw_gemm_gio_w( + size_t g, size_t nc, size_t kc, size_t nr, size_t kr, size_t sr, + size_t k_stride, const uint8_t* kernel, const int32_t* bias, + const float* scale, void* packed_weights, size_t extra_bytes, + const struct xnn_qs8_qc4w_packing_params* params); XNN_INTERNAL void xnn_pack_qs8_qb4w_gemm_gio_w( size_t g, size_t nc, size_t kc, size_t nr, size_t kr, size_t sr, size_t k_stride, size_t bl, const uint8_t* kernel, const float* bias,