From 8dc71ce8cd3f4b3128d5181d3624d9924a3bdafe Mon Sep 17 00:00:00 2001 From: Kotomi-Du Date: Tue, 27 Oct 2026 17:02:39 -0700 Subject: [PATCH 1/4] make sure it is marked as grouped case for 3D weights even though ngroup=1 --- .../impls/onednn/fully_connected_onednn.cpp | 10 ++- .../test_cases/fully_connected_gpu_test.cpp | 79 +++++++++++++++++++ 2 files changed, 85 insertions(+), 4 deletions(-) diff --git a/src/plugins/intel_gpu/src/graph/impls/onednn/fully_connected_onednn.cpp b/src/plugins/intel_gpu/src/graph/impls/onednn/fully_connected_onednn.cpp index cf180dae74b4..c75865b1331e 100644 --- a/src/plugins/intel_gpu/src/graph/impls/onednn/fully_connected_onednn.cpp +++ b/src/plugins/intel_gpu/src/graph/impls/onednn/fully_connected_onednn.cpp @@ -284,6 +284,7 @@ struct fully_connected_onednn : typed_primitive_onednn_impl { auto weight_rank = std::count_if(weight_shape.begin(), weight_shape.end(), [](ov::Dimension d) { return d.get_length() > 1; }); weight_rank = std::max(static_cast(2), weight_rank); + const bool has_batch_dim = weight_rank > 2; auto shift_size = std::max(prim->input_size - 2, 0); const auto& arg = impl_params->get_program().get_node(impl_params->desc->id).as(); @@ -301,7 +302,7 @@ struct fully_connected_onednn : typed_primitive_onednn_impl { const auto ngroups = scale_layout.get_dim(get_ifm_dim_idx(scale_layout, transpose_b_scale, weight_rank)); if (scale_layout.count() == 1) { _attrs->set_scales(DNNL_ARG_WEIGHTS, COMMON, dnnl::memory::dims{}, _ds_data_type); - } else if (ngroups == 1) { + } else if (ngroups == 1 && !has_batch_dim) { _attrs->set_scales(DNNL_ARG_WEIGHTS, per_oc, dnnl::memory::dims{}, _ds_data_type); } else { _attrs->set_scales(DNNL_ARG_WEIGHTS, grouped, {_ds_group_size, 1}, _ds_data_type); @@ -319,7 +320,7 @@ struct fully_connected_onednn : typed_primitive_onednn_impl { _attrs->set_zero_points(DNNL_ARG_WEIGHTS, COMMON, dnnl::memory::dims{}, _dzp_data_type); } else { auto ngroups = dzp_layout.get_dim(get_ifm_dim_idx(dzp_layout, transpose_b_zp, weight_rank)); - if (ngroups == 1) { + if (ngroups == 1 && !has_batch_dim) { _attrs->set_zero_points(DNNL_ARG_WEIGHTS, per_oc, dnnl::memory::dims{}, _dzp_data_type); } else { _attrs->set_zero_points(DNNL_ARG_WEIGHTS, grouped, {_ds_group_size, 1}, _dzp_data_type); @@ -391,6 +392,7 @@ struct fully_connected_onednn : typed_primitive_onednn_impl { auto weight_rank = std::count_if(weight_shape.begin(), weight_shape.end(), [](ov::Dimension d) { return d.get_length() > 1; }); weight_rank = std::max(static_cast(2), weight_rank); OPENVINO_ASSERT(weight_rank <= 3, "Currently only weights with equal to or less than 3D is supported"); + const bool has_batch_dim = weight_rank > 2; auto shift_size = std::max(prim->input_size - 2, 0); int per_oc = PER_OC << shift_size; int grouped = (1 << prim->input_size) - 1; @@ -409,7 +411,7 @@ struct fully_connected_onednn : typed_primitive_onednn_impl { "[GPU] group_size should be aligned to 16 if it is not a single scale group or the group_size is not one."); if (scale_layout.count() == 1) { attr->set_scales(DNNL_ARG_WEIGHTS, COMMON, dnnl::memory::dims{}, ds_data_type); - } else if (ngroups == 1 && weight_rank <= 2) { + } else if (ngroups == 1 && !has_batch_dim) { attr->set_scales(DNNL_ARG_WEIGHTS, per_oc, dnnl::memory::dims{}, ds_data_type); } else { // should use {K, 1} for the group size + per tensor mask for 3d @@ -433,7 +435,7 @@ struct fully_connected_onednn : typed_primitive_onednn_impl { dzp_rank = std::max(static_cast(2), dzp_rank); auto ngroups = dzp_layout.get_dim(get_ifm_dim_idx(dzp_layout, prim->transpose_b_zp, weight_rank)); - if (ngroups == 1 && dzp_rank <= 2) { + if (ngroups == 1 && !has_batch_dim) { attr->set_zero_points(DNNL_ARG_WEIGHTS, per_oc, dnnl::memory::dims{}, dzp_data_type); } else { // should use {K, 1} for the group size + per tensor mask for 3d diff --git a/src/plugins/intel_gpu/tests/unit/test_cases/fully_connected_gpu_test.cpp b/src/plugins/intel_gpu/tests/unit/test_cases/fully_connected_gpu_test.cpp index 18831e38c183..2464cd27edbc 100644 --- a/src/plugins/intel_gpu/tests/unit/test_cases/fully_connected_gpu_test.cpp +++ b/src/plugins/intel_gpu/tests/unit/test_cases/fully_connected_gpu_test.cpp @@ -5251,6 +5251,85 @@ TEST(fully_connected_3d_onednn_gpu, no_biases_int8) { } } +class fully_connected_onednn_expert_zero_points_test + : public testing::TestWithParam> {}; + +TEST_P(fully_connected_onednn_expert_zero_points_test, per_expert_per_channel) { + auto& engine = get_test_engine(); + if (!engine.get_device_info().supports_immad) + GTEST_SKIP() << "oneDNN compressed FC requires IMMAD support."; + + const auto [experts, transpose_b_zp, is_caching_test, scale_group_size] = GetParam(); + constexpr int64_t tokens = 2; + constexpr int64_t ofm = 16; + constexpr int64_t ifm = 64; + const size_t weights_rank = experts == 1 ? 2 : 3; + const auto weights_shape = experts == 1 ? ov::PartialShape{ofm, ifm} + : ov::PartialShape{experts, ofm, ifm}; + const auto zp_shape = experts == 1 + ? (transpose_b_zp ? ov::PartialShape{ofm, 1} : ov::PartialShape{1, ofm}) + : (transpose_b_zp ? ov::PartialShape{experts, ofm, 1} + : ov::PartialShape{experts, 1, ofm}); + const auto scale_groups = ifm / scale_group_size; + const auto scale_shape = experts == 1 + ? (transpose_b_zp ? ov::PartialShape{ofm, scale_groups} + : ov::PartialShape{scale_groups, ofm}) + : (transpose_b_zp ? ov::PartialShape{experts, ofm, scale_groups} + : ov::PartialShape{experts, scale_groups, ofm}); + + auto input = engine.allocate_memory({ov::PartialShape{experts, tokens, ifm}, data_types::f16, format::bfyx}); + auto weights = engine.allocate_memory({weights_shape, data_types::u8, format::bfyx}); + auto scale = engine.allocate_memory({scale_shape, data_types::f16, format::bfyx}); + auto zero_points = engine.allocate_memory({zp_shape, data_types::u8, format::bfyx}); + set_values(input, std::vector(experts * tokens * ifm, ov::float16(1.0f))); + set_values(weights, std::vector(experts * ofm * ifm, 9)); + set_values(scale, std::vector(experts * ofm * scale_groups, ov::float16(1.0f))); + std::vector zp_values(experts * ofm); + for (int64_t expert = 0; expert < experts; ++expert) { + for (int64_t oc = 0; oc < ofm; ++oc) { + zp_values[expert * ofm + oc] = static_cast((expert + oc) % 8); + } + } + set_values(zero_points, zp_values); + + topology topology( + input_layout("input", input->get_layout()), + data("weights", weights), + data("scale", scale), + data("zero_points", zero_points), + fully_connected("fc", input_info("input"), "weights", "", "scale", "zero_points", + data_types::f16, 3, weights_rank, true, transpose_b_zp, transpose_b_zp)); + auto config = get_test_default_config(engine); + config.set_property(ov::intel_gpu::allow_new_shape_infer(true)); + config.set_property(ov::intel_gpu::optimize_data(true)); + const ov::intel_gpu::ImplementationDesc impl = {format::bfyx, "", impl_types::onednn}; + config.set_property(ov::intel_gpu::force_implementations(ov::intel_gpu::ImplForcingMap{{"fc", impl}})); + auto network = get_network(engine, topology, config, get_test_stream_ptr(), is_caching_test); + ASSERT_TRUE(network->get_primitive("fc")->get_impl()->is_onednn()); + network->set_input_data("input", input); + auto output = network->execute().at("fc").get_memory(); + ASSERT_EQ(output->get_layout().get_partial_shape(), (ov::PartialShape{experts, tokens, ofm})); + mem_lock result(output, get_test_stream()); + ASSERT_EQ(result.size(), static_cast(experts * tokens * ofm)); + for (int64_t expert = 0; expert < experts; ++expert) { + for (int64_t token = 0; token < tokens; ++token) { + for (int64_t oc = 0; oc < ofm; ++oc) { + const auto idx = (expert * tokens + token) * ofm + oc; + const auto expected = static_cast(ifm * (9 - zp_values[expert * ofm + oc])); + EXPECT_FLOAT_EQ(static_cast(result[idx]), expected) + << "expert=" << expert << ", token=" << token << ", oc=" << oc; + } + } + } +} + +INSTANTIATE_TEST_SUITE_P(smoke, + fully_connected_onednn_expert_zero_points_test, + testing::Combine(testing::Values(int64_t{1}, int64_t{64}), + testing::Bool(), + testing::Bool(), + testing::Values(int64_t{32}, int64_t{64}))); + TEST(fully_connected_3d_onednn_gpu, compressed_int4_scale_static) { tests::random_generator rg(GET_SUITE_NAME); From e0294c672f416b051449b31bad44b2e2be47c3fb Mon Sep 17 00:00:00 2001 From: Kotomi-Du Date: Wed, 28 Oct 2026 19:55:58 -0700 Subject: [PATCH 2/4] update unit test --- .../impls/onednn/fully_connected_onednn.cpp | 2 +- .../test_cases/fully_connected_gpu_test.cpp | 41 ++++++++++++------- 2 files changed, 27 insertions(+), 16 deletions(-) diff --git a/src/plugins/intel_gpu/src/graph/impls/onednn/fully_connected_onednn.cpp b/src/plugins/intel_gpu/src/graph/impls/onednn/fully_connected_onednn.cpp index be52ea82e426..c88e0f6fb61b 100644 --- a/src/plugins/intel_gpu/src/graph/impls/onednn/fully_connected_onednn.cpp +++ b/src/plugins/intel_gpu/src/graph/impls/onednn/fully_connected_onednn.cpp @@ -440,7 +440,7 @@ struct fully_connected_onednn : typed_primitive_onednn_impl { dzp_rank = std::max(static_cast(2), dzp_rank); auto ngroups = dzp_layout.get_dim(get_ifm_dim_idx(dzp_layout, prim->transpose_b_zp, weight_rank)); - if (ngroups == 1 && !has_batch_dim) { + if (ngroups == 1 && dzp_rank <= 2 && !has_batch_dim) { attr->set_zero_points(DNNL_ARG_WEIGHTS, per_oc, dnnl::memory::dims{}, dzp_data_type); } else { // should use {K, 1} for the group size + per tensor mask for 3d diff --git a/src/plugins/intel_gpu/tests/unit/test_cases/fully_connected_gpu_test.cpp b/src/plugins/intel_gpu/tests/unit/test_cases/fully_connected_gpu_test.cpp index 781269e4eb61..688491866494 100644 --- a/src/plugins/intel_gpu/tests/unit/test_cases/fully_connected_gpu_test.cpp +++ b/src/plugins/intel_gpu/tests/unit/test_cases/fully_connected_gpu_test.cpp @@ -5259,23 +5259,25 @@ TEST_P(fully_connected_onednn_expert_zero_points_test, per_expert_per_channel) { if (!engine.get_device_info().supports_immad) GTEST_SKIP() << "oneDNN compressed FC requires IMMAD support."; - const auto [experts, transpose_b_zp, is_caching_test, scale_group_size] = GetParam(); + const auto [experts, transpose_b_param, is_caching_test, group_per_channel] = GetParam(); constexpr int64_t tokens = 2; constexpr int64_t ofm = 16; constexpr int64_t ifm = 64; + const auto scale_groups = group_per_channel; + const auto scale_group_size = ifm / scale_groups; const size_t weights_rank = experts == 1 ? 2 : 3; const auto weights_shape = experts == 1 ? ov::PartialShape{ofm, ifm} : ov::PartialShape{experts, ofm, ifm}; const auto zp_shape = experts == 1 - ? (transpose_b_zp ? ov::PartialShape{ofm, 1} : ov::PartialShape{1, ofm}) - : (transpose_b_zp ? ov::PartialShape{experts, ofm, 1} - : ov::PartialShape{experts, 1, ofm}); - const auto scale_groups = ifm / scale_group_size; + ? (transpose_b_param ? ov::PartialShape{ofm, scale_groups} + : ov::PartialShape{scale_groups, ofm}) + : (transpose_b_param ? ov::PartialShape{experts, ofm, scale_groups} + : ov::PartialShape{experts, scale_groups, ofm}); const auto scale_shape = experts == 1 - ? (transpose_b_zp ? ov::PartialShape{ofm, scale_groups} - : ov::PartialShape{scale_groups, ofm}) - : (transpose_b_zp ? ov::PartialShape{experts, ofm, scale_groups} - : ov::PartialShape{experts, scale_groups, ofm}); + ? (transpose_b_param ? ov::PartialShape{ofm, scale_groups} + : ov::PartialShape{scale_groups, ofm}) + : (transpose_b_param ? ov::PartialShape{experts, ofm, scale_groups} + : ov::PartialShape{experts, scale_groups, ofm}); auto input = engine.allocate_memory({ov::PartialShape{experts, tokens, ifm}, data_types::f16, format::bfyx}); auto weights = engine.allocate_memory({weights_shape, data_types::u8, format::bfyx}); @@ -5284,10 +5286,16 @@ TEST_P(fully_connected_onednn_expert_zero_points_test, per_expert_per_channel) { set_values(input, std::vector(experts * tokens * ifm, ov::float16(1.0f))); set_values(weights, std::vector(experts * ofm * ifm, 9)); set_values(scale, std::vector(experts * ofm * scale_groups, ov::float16(1.0f))); - std::vector zp_values(experts * ofm); + std::vector zp_values(experts * ofm * scale_groups); + const auto zp_idx = [=](int64_t expert, int64_t group, int64_t oc) { + return transpose_b_param ? static_cast(expert * ofm * scale_groups + oc * scale_groups + group) + : static_cast(expert * scale_groups * ofm + group * ofm + oc); + }; for (int64_t expert = 0; expert < experts; ++expert) { - for (int64_t oc = 0; oc < ofm; ++oc) { - zp_values[expert * ofm + oc] = static_cast((expert + oc) % 8); + for (int64_t group = 0; group < scale_groups; ++group) { + for (int64_t oc = 0; oc < ofm; ++oc) { + zp_values[zp_idx(expert, group, oc)] = static_cast((expert + group + oc) % 8); + } } } set_values(zero_points, zp_values); @@ -5298,7 +5306,7 @@ TEST_P(fully_connected_onednn_expert_zero_points_test, per_expert_per_channel) { data("scale", scale), data("zero_points", zero_points), fully_connected("fc", input_info("input"), "weights", "", "scale", "zero_points", - data_types::f16, 3, weights_rank, true, transpose_b_zp, transpose_b_zp)); + data_types::f16, 3, weights_rank, true, transpose_b_param, transpose_b_param)); auto config = get_test_default_config(engine); config.set_property(ov::intel_gpu::allow_new_shape_infer(true)); config.set_property(ov::intel_gpu::optimize_data(true)); @@ -5315,7 +5323,10 @@ TEST_P(fully_connected_onednn_expert_zero_points_test, per_expert_per_channel) { for (int64_t token = 0; token < tokens; ++token) { for (int64_t oc = 0; oc < ofm; ++oc) { const auto idx = (expert * tokens + token) * ofm + oc; - const auto expected = static_cast(ifm * (9 - zp_values[expert * ofm + oc])); + float expected = 0.0f; + for (int64_t group = 0; group < scale_groups; ++group) { + expected += static_cast(scale_group_size * (9 - zp_values[zp_idx(expert, group, oc)])); + } EXPECT_FLOAT_EQ(static_cast(result[idx]), expected) << "expert=" << expert << ", token=" << token << ", oc=" << oc; } @@ -5328,7 +5339,7 @@ INSTANTIATE_TEST_SUITE_P(smoke, testing::Combine(testing::Values(int64_t{1}, int64_t{64}), testing::Bool(), testing::Bool(), - testing::Values(int64_t{32}, int64_t{64}))); + testing::Values(int64_t{1}, int64_t{2}))); TEST(fully_connected_3d_onednn_gpu, compressed_int4_scale_static) { tests::random_generator rg(GET_SUITE_NAME); From 15f587a69bbd80b3ab19a7d61726b154c94f1679 Mon Sep 17 00:00:00 2001 From: Kotomi-Du Date: Thu, 29 Oct 2026 16:44:37 -0700 Subject: [PATCH 3/4] fix build issue --- .../tests/unit/test_cases/fully_connected_gpu_test.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/plugins/intel_gpu/tests/unit/test_cases/fully_connected_gpu_test.cpp b/src/plugins/intel_gpu/tests/unit/test_cases/fully_connected_gpu_test.cpp index 688491866494..e88b9006b942 100644 --- a/src/plugins/intel_gpu/tests/unit/test_cases/fully_connected_gpu_test.cpp +++ b/src/plugins/intel_gpu/tests/unit/test_cases/fully_connected_gpu_test.cpp @@ -5287,9 +5287,10 @@ TEST_P(fully_connected_onednn_expert_zero_points_test, per_expert_per_channel) { set_values(weights, std::vector(experts * ofm * ifm, 9)); set_values(scale, std::vector(experts * ofm * scale_groups, ov::float16(1.0f))); std::vector zp_values(experts * ofm * scale_groups); + const bool transpose_zp_layout = transpose_b_param; const auto zp_idx = [=](int64_t expert, int64_t group, int64_t oc) { - return transpose_b_param ? static_cast(expert * ofm * scale_groups + oc * scale_groups + group) - : static_cast(expert * scale_groups * ofm + group * ofm + oc); + return transpose_zp_layout ? static_cast(expert * ofm * scale_groups + oc * scale_groups + group) + : static_cast(expert * scale_groups * ofm + group * ofm + oc); }; for (int64_t expert = 0; expert < experts; ++expert) { for (int64_t group = 0; group < scale_groups; ++group) { From b4896ce087afa664533bfa1c77f95baf03e862ca Mon Sep 17 00:00:00 2001 From: Kotomi-Du Date: Wed, 4 Nov 2026 09:42:41 -0800 Subject: [PATCH 4/4] remove space --- .../intel_gpu/src/graph/impls/onednn/fully_connected_onednn.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/plugins/intel_gpu/src/graph/impls/onednn/fully_connected_onednn.cpp b/src/plugins/intel_gpu/src/graph/impls/onednn/fully_connected_onednn.cpp index 1c607fa0bfb4..892e6726f3b0 100644 --- a/src/plugins/intel_gpu/src/graph/impls/onednn/fully_connected_onednn.cpp +++ b/src/plugins/intel_gpu/src/graph/impls/onednn/fully_connected_onednn.cpp @@ -415,7 +415,7 @@ struct fully_connected_onednn : typed_primitive_onednn_impl { dzp_rank = std::max(static_cast(2), dzp_rank); auto ngroups = dzp_layout.get_dim(get_ifm_dim_idx(dzp_layout, prim->transpose_b_zp, weight_rank)); - if (ngroups == 1 && dzp_rank <= 2 && !has_batch_dim) { + if (ngroups == 1 && dzp_rank <= 2 && !has_batch_dim) { attr->set_zero_points(DNNL_ARG_WEIGHTS, per_oc, dnnl::memory::dims{}, dzp_data_type); } else { // should use {K, 1} for the group size + per tensor mask for 3d