diff --git a/src/plugins/intel_gpu/src/graph/impls/onednn/fully_connected_onednn.hpp b/src/plugins/intel_gpu/src/graph/impls/onednn/fully_connected_onednn.hpp index 9960d13efba310..915c8312c2100e 100644 --- a/src/plugins/intel_gpu/src/graph/impls/onednn/fully_connected_onednn.hpp +++ b/src/plugins/intel_gpu/src/graph/impls/onednn/fully_connected_onednn.hpp @@ -5,6 +5,7 @@ #include "fully_connected_inst.h" #include "impls/onednn/utils.hpp" #include "intel_gpu/primitives/fully_connected.hpp" +#include "intel_gpu/primitives/input_layout.hpp" #include "intel_gpu/runtime/utils.hpp" #include "registry/implementation_manager.hpp" #include "intel_gpu/runtime/debug_configuration.hpp" @@ -140,14 +141,29 @@ struct FullyConnectedImplementationManager : public ImplementationManager { in_out_fmts_t query_formats(const program_node& node) const override { assert(node.is_type()); + const auto& fc_node = node.as(); + const auto& fc_prim = fc_node.get_primitive(); std::vector in_fmts(node.get_dependencies().size(), format::any); std::vector out_fmts(node.get_outputs_count(), format::any); size_t out_rank = node.get_output_layout().get_rank(); - for (size_t idx = 0 ; idx < node.get_dependencies().size() ; idx++) { + for (size_t idx = 0; idx < node.get_dependencies().size(); idx++) { if (node.get_dependency(idx).is_constant()) { continue; } + const size_t scale_idx = fc_prim->bias.is_valid() ? 3 : 2; + const size_t zp_idx = scale_idx + 1; + const bool is_decompression_parameter = + ((fc_prim->decompression_scale.is_valid() && idx == scale_idx) || (fc_prim->decompression_zero_point.is_valid() && idx == zp_idx)); + if (is_decompression_parameter) { + const auto& in_layout = node.get_input_layout(idx); + // oneDNN binds scale/zp as a flat format_tag::a buffer, so only plain layouts can be passed through. + // Keep plain permutations (e.g. fbyx from prepare_quantization); reorder blocked layouts to plain. + in_fmts[idx] = format::is_blocked(in_layout.format) ? format::get_default_format(in_layout.get_rank()) : in_layout.format.value; + GPU_DEBUG_LOG << node.id() << " : " << (format::is_blocked(in_layout.format) ? "Reorder blocked" : "Keep plain") << " scale/zp format at index " + << idx << std::endl; + continue; + } auto target_format = format::get_default_format(out_rank); diff --git a/src/plugins/intel_gpu/src/plugin/transformations/compressed_weights_pattern.hpp b/src/plugins/intel_gpu/src/plugin/transformations/compressed_weights_pattern.hpp index 3cda16a1c43eac..35dc1f365fa9aa 100644 --- a/src/plugins/intel_gpu/src/plugin/transformations/compressed_weights_pattern.hpp +++ b/src/plugins/intel_gpu/src/plugin/transformations/compressed_weights_pattern.hpp @@ -29,13 +29,31 @@ using ov::pass::operator|; auto weights_convert_reshape_m = wrap_type({weights_param_convert_m, any_input()});\ auto decompressed_weights_m = convert_m | weights_convert_reshape_m;\ \ - auto sub_const_m = wrap_type();\ + auto sub_value_const_m = wrap_type();\ + auto sub_value_param_m = wrap_type(ov::pass::pattern::has_static_shape());\ + auto sub_value_param_reshape_m =\ + wrap_type({sub_value_param_m, any_input()}, ov::pass::pattern::has_static_shape());\ + auto sub_value_param_convert_m = wrap_type({sub_value_param_m});\ + auto sub_value_param_convert_reshape_m =\ + wrap_type({sub_value_param_convert_m, any_input()}, ov::pass::pattern::has_static_shape());\ + auto sub_const_m = sub_value_const_m | sub_value_param_m | sub_value_param_reshape_m |\ + sub_value_param_convert_reshape_m;\ auto sub_convert_const_m = wrap_type({sub_const_m});\ auto sub_with_convert_m = wrap_type({decompressed_weights_m, sub_convert_const_m});\ auto sub_no_convert_m = wrap_type({decompressed_weights_m, sub_const_m});\ auto subtract_m = sub_with_convert_m | sub_no_convert_m;\ \ - auto mul_const_m = wrap_type();\ + auto mul_value_const_m = wrap_type();\ + auto mul_value_param_m = wrap_type(ov::pass::pattern::has_static_shape());\ + auto mul_value_param_reshape_m =\ + wrap_type({mul_value_param_m, any_input()}, ov::pass::pattern::has_static_shape());\ + auto mul_value_param_convert_m = wrap_type({mul_value_param_m});\ + auto mul_value_param_convert_reshape_m =\ + wrap_type({mul_value_param_convert_m, any_input()}, ov::pass::pattern::has_static_shape());\ + auto mul_value_param_reshape_convert_m =\ + wrap_type({mul_value_param_reshape_m});\ + auto mul_const_m = mul_value_const_m | mul_value_param_m | mul_value_param_reshape_m |\ + mul_value_param_convert_reshape_m | mul_value_param_reshape_convert_m;\ auto mul_with_sub_m = wrap_type({subtract_m, mul_const_m});\ auto mul_const_convert_m = ov::pass::pattern::optional(mul_const_m);\ auto mul_no_sub_m = wrap_type({decompressed_weights_m, mul_const_convert_m});\ diff --git a/src/plugins/intel_gpu/src/plugin/transformations/convert_fc_to_compressed.cpp b/src/plugins/intel_gpu/src/plugin/transformations/convert_fc_to_compressed.cpp index 6ebb1728f9d197..06acdcdc4bc695 100644 --- a/src/plugins/intel_gpu/src/plugin/transformations/convert_fc_to_compressed.cpp +++ b/src/plugins/intel_gpu/src/plugin/transformations/convert_fc_to_compressed.cpp @@ -48,7 +48,9 @@ ConvertFullyConnectedToFullyConnectedCompressed::ConvertFullyConnectedToFullyCon } const bool has_transpose_before_reshape = pattern_map.count(transpose_before_reshape_m) != 0u; const bool has_transpose = pattern_map.count(transpose_m) != 0u; - auto scale_shape = pattern_map.at(mul_const_m).get_shape(); + const auto scale_output = pattern_map.at(mul_const_m); + auto scale_shape = scale_output.get_shape(); + const bool with_zero_point = pattern_map.count(subtract_m) > 0; bool sub_with_convert = pattern_map.count(sub_convert_const_m) > 0; if (!fc->get_input_partial_shape(1).is_static()) { @@ -86,18 +88,19 @@ ConvertFullyConnectedToFullyConnectedCompressed::ConvertFullyConnectedToFullyCon weight_u8 = true; } - auto reshape_const = [has_transpose, grouped, is_weight_3d](std::shared_ptr node) { - auto constant = ov::as_type_ptr(node); - OPENVINO_ASSERT(constant != nullptr); - ov::Shape current_shape = constant->get_shape(); + std::vector> result_nodes = {}; + auto reshape_input = [has_transpose, grouped, is_weight_3d, &result_nodes](std::shared_ptr node) { + const auto& partial_shape = node->get_output_partial_shape(0); + OPENVINO_ASSERT(partial_shape.is_static()); + const ov::Shape current_shape = partial_shape.to_shape(); if (current_shape.size() <= 2) { - return constant; + return node; } ov::Shape new_shape; if (current_shape.size() == 3) { if (is_weight_3d) { - return constant; + return node; } new_shape = (has_transpose || !grouped) ? ov::Shape{current_shape[0] * current_shape[1], current_shape[2]} : ov::Shape{current_shape[0], current_shape[1] * current_shape[2]}; @@ -110,52 +113,66 @@ ConvertFullyConnectedToFullyConnectedCompressed::ConvertFullyConnectedToFullyCon } else { OPENVINO_THROW("Unexpected constant shape rank ", current_shape.size(), " with is_weight_3d=", is_weight_3d); } - auto new_constant = std::make_shared(*constant, new_shape); - - ov::copy_weightless_cache_attr(constant, new_constant); - return new_constant; + if (auto constant = ov::as_type_ptr(node)) { + auto new_constant = std::make_shared(*constant, new_shape); + ov::copy_weightless_cache_attr(constant, new_constant); + return std::static_pointer_cast(new_constant); + } + auto shape_const = std::make_shared( + ov::element::i32, + ov::Shape{new_shape.size()}, + std::vector(new_shape.begin(), new_shape.end())); + auto reshape = std::make_shared(node, shape_const, false); + reshape->set_friendly_name(node->get_friendly_name() + "_reshape"); + ov::copy_runtime_info(node, {shape_const, reshape}); + result_nodes.push_back(shape_const); + result_nodes.push_back(reshape); + return std::static_pointer_cast(reshape); }; auto convert_zp_const = [&](std::shared_ptr node) { auto constant = ov::as_type_ptr(node); - std::shared_ptr result = nullptr; + const auto zp_type = node->get_element_type(); + std::shared_ptr result = node; // Convert ZP to u8 (or i8 for a scalar i4 ZP) - if (constant->get_element_type() == ov::element::u8) { - result = constant; - } else if (constant->get_element_type() == ov::element::u4 || constant->get_element_type() == ov::element::u3 || - constant->get_element_type() == ov::element::u2) { + if (zp_type == ov::element::u8) { + result = node; + } else if (zp_type == ov::element::u4 || zp_type == ov::element::u3 || zp_type == ov::element::u2) { result = std::make_shared(node, ov::element::u8); // Only unsigned ZP types can be converted to u8. - } else if (weight_u8 && sub_with_convert && !constant->get_element_type().is_signed()) { + } else if (weight_u8 && sub_with_convert && !zp_type.is_signed()) { result = std::make_shared(node, ov::element::u8); - } else if (constant->get_element_type() == ov::element::i4 && ov::shape_size(constant->get_shape()) == 1) { + } else if (zp_type == ov::element::i4 && node->get_output_partial_shape(0).is_static() && + ov::shape_size(node->get_output_shape(0)) == 1) { // oneDNN doesn't support sub-byte per-tensor (common) zero points, so a scalar i4 ZP is converted to i8. result = std::make_shared(node, ov::element::i8); - } else { - result = constant; } - ov::copy_weightless_cache_attr(node, result); + if (result != node) { + ov::copy_runtime_info(node, result); + if (constant) { + ov::copy_weightless_cache_attr(constant, result); + } + } return result; }; const ov::Output& fc_input_a = fc->input(0).get_source_output(); const auto& scale = has_transpose_before_reshape ? pattern_map.at(mul_const_m).get_node_shared_ptr() - : reshape_const(pattern_map.at(mul_const_m).get_node_shared_ptr()); + : reshape_input(pattern_map.at(mul_const_m).get_node_shared_ptr()); std::shared_ptr optional_zero_point = nullptr; - const bool with_zero_point = pattern_map.count(subtract_m) > 0; if (with_zero_point) { const auto zero_point = has_transpose_before_reshape ? pattern_map.at(sub_const_m).get_node_shared_ptr() - : reshape_const(pattern_map.at(sub_const_m).get_node_shared_ptr()); + : reshape_input(pattern_map.at(sub_const_m).get_node_shared_ptr()); optional_zero_point = convert_zp_const(zero_point); } std::shared_ptr fc_input_b = weight_ptr; if (weights_are_constant && !has_transpose_before_reshape) { - fc_input_b = reshape_const(weight_ptr); + fc_input_b = reshape_input(weight_ptr); } else if (pattern_map.count(weights_reshape_m)) { fc_input_b = pattern_map.at(weights_reshape_m).get_node_shared_ptr(); } @@ -165,7 +182,6 @@ ConvertFullyConnectedToFullyConnectedCompressed::ConvertFullyConnectedToFullyCon bool transpose_b_scale = fc->get_transpose_b(); bool transpose_b_zp = fc->get_transpose_b(); std::shared_ptr fc_input_bias = pattern_map.at(bias_m).get_node_shared_ptr(); - std::vector> result_nodes = {}; if (with_zero_point && ov::is_type(optional_zero_point)) { result_nodes.push_back(optional_zero_point); } @@ -230,6 +246,7 @@ ConvertFullyConnectedToFullyConnectedCompressed::ConvertFullyConnectedToFullyCon ov::Shape{output_shape.size()}, output_shape); auto reshaped = std::make_shared(node, shape_const, false); + ov::copy_runtime_info(node, {shape_const, reshaped}); result_nodes.push_back(shape_const); result_nodes.push_back(reshaped); return std::static_pointer_cast(reshaped); @@ -244,6 +261,8 @@ ConvertFullyConnectedToFullyConnectedCompressed::ConvertFullyConnectedToFullyCon if (pattern_map.count(mul2_m)) { auto mul2_op_const = ov::as_type_ptr(pattern_map.at(mul2_const_m).get_node_shared_ptr()); fc_input_scale = ov::op::util::make_try_fold(fc_input_scale, mul2_op_const); + // With a Parameter scale this can't fold and stays an executable Multiply, so it needs runtime info. + result_nodes.push_back(fc_input_scale); } std::shared_ptr new_fc = nullptr; diff --git a/src/plugins/intel_gpu/tests/unit/test_cases/fully_connected_gpu_test.cpp b/src/plugins/intel_gpu/tests/unit/test_cases/fully_connected_gpu_test.cpp index 9211c81c6e7700..eab011d897085f 100644 --- a/src/plugins/intel_gpu/tests/unit/test_cases/fully_connected_gpu_test.cpp +++ b/src/plugins/intel_gpu/tests/unit/test_cases/fully_connected_gpu_test.cpp @@ -3275,6 +3275,102 @@ void test_compressed_int4_scale_dynamic_batch_gemv(bool is_caching_test, } } + void test_compressed_int8_parameter_scale_zp(bool is_per_group, bool transpose_scale_zp) { + auto& engine = get_test_engine(); + + constexpr int64_t ofm = 4; + constexpr int64_t ifm = 32; + const int64_t groups = is_per_group ? 2 : 1; + const int64_t group_size = ifm / groups; + + // Non-transposed scale/zp are [groups, N]; transposed scale/zp are [N, groups]. + const ov::PartialShape param_shape = transpose_scale_zp ? ov::PartialShape{ofm, groups} : ov::PartialShape{groups, ofm}; + auto param_offset = [&](int64_t group, int64_t output) { + return static_cast(transpose_scale_zp ? output * groups + group : group * ofm + output); + }; + + auto input_mem = engine.allocate_memory({ {1, 2, ifm}, data_types::f16, format::bfyx }); + auto weights_mem = engine.allocate_memory({ {ofm, ifm}, data_types::u8, format::bfyx }); + auto scale_mem = engine.allocate_memory({ param_shape, data_types::f16, format::bfyx }); + auto zp_mem = engine.allocate_memory({ param_shape, data_types::u8, format::bfyx }); + + std::vector input_data(2 * ifm); + for (size_t i = 0; i < input_data.size(); ++i) { + input_data[i] = static_cast(static_cast(i % 3) - 1); + } + std::vector weights_data(ofm * ifm); + for (int64_t output = 0; output < ofm; ++output) { + for (int64_t k = 0; k < ifm; ++k) { + weights_data[output * ifm + k] = static_cast((output * 3 + k) % 8); + } + } + std::vector scale_data(ofm * groups); + std::vector zp_data(ofm * groups); + for (int64_t group = 0; group < groups; ++group) { + for (int64_t output = 0; output < ofm; ++output) { + scale_data[param_offset(group, output)] = 0.5f * static_cast((group + output) % 2 + 1); + zp_data[param_offset(group, output)] = static_cast((group + output) % 3); + } + } + + std::vector input_f16(input_data.begin(), input_data.end()); + std::vector scale_f16(scale_data.begin(), scale_data.end()); + set_values(input_mem, input_f16); + set_values(weights_mem, weights_data); + set_values(scale_mem, scale_f16); + set_values(zp_mem, zp_data); + + topology topology( + input_layout("input", input_mem->get_layout()), + data("weights", weights_mem), + input_layout("scale", scale_mem->get_layout()), + input_layout("zp", zp_mem->get_layout()), + fully_connected("fc_prim", input_info("input"), "weights", "", "scale", "zp", data_types::f16, 3, 2, true, + transpose_scale_zp, transpose_scale_zp)); + + auto config = get_test_default_config(engine); + config.set_property(ov::intel_gpu::allow_new_shape_infer(true)); + config.set_property(ov::intel_gpu::optimize_data(true)); + + network network(engine, topology, config); + network.set_input_data("input", input_mem); + network.set_input_data("scale", scale_mem); + network.set_input_data("zp", zp_mem); + + // Grouped transposed scale/zp [N, groups] must be reordered from bfyx to fbyx to make N innermost. + // Per-channel [N, 1] already has N innermost, so it stays as the bfyx parameter. + const bool expect_reorder = transpose_scale_zp && is_per_group; + auto fc_inst = network.get_primitive("fc_prim"); + for (size_t idx : {size_t{2}, size_t{3}}) { + const auto& dep = fc_inst->get_node().get_dependency(idx); + if (expect_reorder) { + EXPECT_TRUE(dep.is_type()) << "input " << idx; + EXPECT_EQ(fc_inst->get_input_layout(idx).format, format::fbyx) << "input " << idx; + } else { + EXPECT_TRUE(dep.is_type()) << "input " << idx; + EXPECT_EQ(fc_inst->get_input_layout(idx).format, format::bfyx) << "input " << idx; + } + } + + auto outputs = network.execute(); + auto output_mem = outputs.at("fc_prim").get_memory(); + cldnn::mem_lock output_ptr(output_mem, get_test_stream()); + ASSERT_EQ((ov::PartialShape{1, 2, ofm}), output_mem->get_layout().get_partial_shape()); + + for (int64_t token = 0; token < 2; ++token) { + for (int64_t output = 0; output < ofm; ++output) { + float expected = 0.0f; + for (int64_t k = 0; k < ifm; ++k) { + const auto offset = param_offset(k / group_size, output); + const float weight = (static_cast(weights_data[output * ifm + k]) - zp_data[offset]) * scale_data[offset]; + expected += input_data[token * ifm + k] * weight; + } + EXPECT_NEAR(static_cast(output_ptr[token * ofm + output]), expected, 1e-2f) + << "token = " << token << ", output = " << output; + } + } + } + void test_dynamic(bool is_caching_test) { auto& engine = get_test_engine(); @@ -6115,6 +6211,22 @@ TEST_F(fully_connected_gpu_tests, compressed_int8_transpose_scale_zp) { this->test_compressed_int8_transpose_scale_zp(true); } +TEST_F(fully_connected_gpu_tests, compressed_int8_parameter_scale_zp_per_channel) { + this->test_compressed_int8_parameter_scale_zp(false, false); +} + +TEST_F(fully_connected_gpu_tests, compressed_int8_parameter_scale_zp_per_channel_transposed) { + this->test_compressed_int8_parameter_scale_zp(false, true); +} + +TEST_F(fully_connected_gpu_tests, compressed_int8_parameter_scale_zp_per_group) { + this->test_compressed_int8_parameter_scale_zp(true, false); +} + +TEST_F(fully_connected_gpu_tests, compressed_int8_parameter_scale_zp_per_group_transposed) { + this->test_compressed_int8_parameter_scale_zp(true, true); +} + TEST_F(fully_connected_gpu_tests, compressed_int8_scale_b1) { this->test_compressed_int8_scale(false, true, 1, false, false); } diff --git a/src/plugins/intel_gpu/tests/unit/transformations/convert_fc_to_compressed_test.cpp b/src/plugins/intel_gpu/tests/unit/transformations/convert_fc_to_compressed_test.cpp index d6c7c61c1dde33..daf61a790b97f8 100644 --- a/src/plugins/intel_gpu/tests/unit/transformations/convert_fc_to_compressed_test.cpp +++ b/src/plugins/intel_gpu/tests/unit/transformations/convert_fc_to_compressed_test.cpp @@ -83,6 +83,155 @@ TEST_F(TransformationTestsF, ConvertFCToCompressed2) { } } +enum class ScaleZpTopology { ReshapeOfConvert, ConvertOfReshape }; + +struct ParameterScaleZpParams { + ScaleZpTopology topology; + bool with_zp; +}; + +class ConvertFCToCompressedParameterScaleZpTest : public testing::WithParamInterface, + public TransformationTestsF { +public: + ConvertFCToCompressedParameterScaleZpTest() : TransformationTestsF() { + comparator.enable(FunctionsComparator::ATTRIBUTES); + } + + static std::string getTestCaseName(const testing::TestParamInfo& info) { + const auto& [topology, with_zp] = info.param; + return std::string(topology == ScaleZpTopology::ReshapeOfConvert ? "ReshapeOfConvert" : "ConvertOfReshape") + + (with_zp ? "_WithZp" : "_NoZp"); + } +}; + +// Scale and zero point are model Parameters [32, 1, 1] reshaped to [32, 1], as Reshape(Convert(param)) or +// Convert(Reshape(param)). The pass must create FullyConnectedCompressed that takes the Parameter-based scale/zp. +TEST_P(ConvertFCToCompressedParameterScaleZpTest, ConvertsParameterScaleAndZeroPoint) { + const auto& [topology, with_zp] = GetParam(); + const bool reshape_of_convert = topology == ScaleZpTopology::ReshapeOfConvert; + + auto to_f16 = [](const ov::Output& node) -> std::shared_ptr { + return std::make_shared(node, ov::element::f16); + }; + auto reshape_to_2d = [](const ov::Output& node) -> std::shared_ptr { + auto shape = ov::op::v0::Constant::create(ov::element::i32, ov::Shape{2}, {32, 1}); + return std::make_shared(node, shape, false); + }; + auto build = [&](const ov::Output& param) { + return reshape_of_convert ? reshape_to_2d(to_f16(param)) : to_f16(reshape_to_2d(param)); + }; + + { + auto input = std::make_shared(ov::element::f16, ov::PartialShape{-1, 16}); + auto weights = ov::op::v0::Constant::create(ov::element::u8, ov::Shape{32, 16}, {1}); + auto scale = std::make_shared(ov::element::f32, ov::Shape{32, 1, 1}); + ov::ParameterVector params{input, scale}; + std::shared_ptr dequantized = to_f16(weights); + if (with_zp) { + auto zp = std::make_shared(ov::element::u8, ov::Shape{32, 1, 1}); + params.push_back(zp); + dequantized = std::make_shared(dequantized, build(zp)); + } + dequantized = std::make_shared(dequantized, build(scale)); + auto no_bias = std::make_shared(); + auto fc = std::make_shared(input, dequantized, no_bias, ov::element::f16, true); + + model = std::make_shared(ov::OutputVector{fc}, params); + manager.register_pass(); + } + { + auto input = std::make_shared(ov::element::f16, ov::PartialShape{-1, 16}); + auto weights = ov::op::v0::Constant::create(ov::element::u8, ov::Shape{32, 16}, {1}); + auto scale = std::make_shared(ov::element::f32, ov::Shape{32, 1, 1}); + ov::ParameterVector params{input, scale}; + // Without a zero point, Multiply's optional Convert absorbs a trailing scale Convert, + // so Convert(Reshape(scale)) is passed as Reshape(scale) in its original precision. + auto expected_scale = (!with_zp && !reshape_of_convert) ? reshape_to_2d(scale) : build(scale); + auto no_bias = std::make_shared(); + std::shared_ptr fc_compressed; + if (with_zp) { + auto zp = std::make_shared(ov::element::u8, ov::Shape{32, 1, 1}); + params.push_back(zp); + // A u8 zp is passed without its Convert; an f16 Reshape(Convert(zp)) is passed as is. + auto expected_zp = reshape_of_convert ? build(zp) : reshape_to_2d(zp); + fc_compressed = std::make_shared(input, + weights, + no_bias, + expected_scale, + expected_zp, + ov::element::f16, + true, + true, + true); + } else { + fc_compressed = std::make_shared(input, + weights, + no_bias, + expected_scale, + ov::element::f16, + true, + true, + true); + } + + model_ref = std::make_shared(ov::OutputVector{fc_compressed}, params); + } +} + +INSTANTIATE_TEST_SUITE_P(TransformationTests, + ConvertFCToCompressedParameterScaleZpTest, + testing::Values(ParameterScaleZpParams{ScaleZpTopology::ReshapeOfConvert, true}, + ParameterScaleZpParams{ScaleZpTopology::ReshapeOfConvert, false}, + ParameterScaleZpParams{ScaleZpTopology::ConvertOfReshape, true}, + ParameterScaleZpParams{ScaleZpTopology::ConvertOfReshape, false}), + ConvertFCToCompressedParameterScaleZpTest::getTestCaseName); + +// Grouped weights with a Parameter scale followed by the extra post-reshape Multiply (mul2_m tail). +// w: Parameter u8 [8,4,16] → Convert f16 ─┐ +// scale: Parameter f16 [8,4,1] ──────────┴→ Multiply → Reshape [8,64] → Multiply(·, Constant [8,1]) → FullyConnected +TEST_F(TransformationTestsF, ConvertFCToCompressedParameterWeightsAndScaleWithExtraMultiply) { + comparator.enable(FunctionsComparator::ATTRIBUTES); + { + auto input = std::make_shared(ov::element::f16, ov::PartialShape{-1, 64}); + auto weights = std::make_shared(ov::element::u8, ov::Shape{8, 4, 16}); + auto convert = std::make_shared(weights, ov::element::f16); + auto scale = std::make_shared(ov::element::f16, ov::Shape{8, 4, 1}); + auto mul = std::make_shared(convert, scale); + auto reshape_const = ov::op::v0::Constant::create(ov::element::i32, ov::Shape{2}, {8, 64}); + auto reshape = std::make_shared(mul, reshape_const, false); + auto extra_mul_const = ov::op::v0::Constant::create(ov::element::f16, ov::Shape{8, 1}, {2}); + auto extra_mul = std::make_shared(reshape, extra_mul_const); + auto no_bias = std::make_shared(); + auto fc = std::make_shared(input, extra_mul, no_bias, ov::element::f16, true); + + model = std::make_shared(ov::OutputVector{fc}, ov::ParameterVector{input, weights, scale}); + manager.register_pass(); + } + { + auto input = std::make_shared(ov::element::f16, ov::PartialShape{-1, 64}); + auto weights = std::make_shared(ov::element::u8, ov::Shape{8, 4, 16}); + // Non-constant weights can't be folded; the [8,4,16] Parameter is reshaped to the FC weights shape [8,64]. + auto weights_shape = ov::op::v0::Constant::create(ov::element::i32, ov::Shape{2}, {8, 64}); + auto weights_reshape = std::make_shared(weights, weights_shape, false); + auto scale = std::make_shared(ov::element::f16, ov::Shape{8, 4, 1}); + auto scale_shape = ov::op::v0::Constant::create(ov::element::i32, ov::Shape{2}, {8, 4}); + auto scale_reshape = std::make_shared(scale, scale_shape, false); + auto extra_mul_const = ov::op::v0::Constant::create(ov::element::f16, ov::Shape{8, 1}, {2}); + auto scaled = std::make_shared(scale_reshape, extra_mul_const); + auto no_bias = std::make_shared(); + auto fc_compressed = std::make_shared(input, + weights_reshape, + no_bias, + scaled, + ov::element::f16, + true, + true, + true); + + model_ref = std::make_shared(ov::OutputVector{fc_compressed}, ov::ParameterVector{input, weights, scale}); + } +} + TEST_F(TransformationTestsF, ConvertFCToCompressed3) { { auto input1 = std::make_shared(ov::element::f32, ov::PartialShape{ -1, 16 }); diff --git a/src/plugins/intel_gpu/tests/unit/transformations/convert_matmul_to_fc_test.cpp b/src/plugins/intel_gpu/tests/unit/transformations/convert_matmul_to_fc_test.cpp index 76c6cba66ae788..27a9edcefece63 100644 --- a/src/plugins/intel_gpu/tests/unit/transformations/convert_matmul_to_fc_test.cpp +++ b/src/plugins/intel_gpu/tests/unit/transformations/convert_matmul_to_fc_test.cpp @@ -636,6 +636,132 @@ TEST_F(TransformationTestsF, ConvertMatMulToFullyConnectedTest_compressed_u4_wei } } +TEST_F(TransformationTestsF, ConvertMatMulToFullyConnectedTest_compressed_u8_weights_static_scale_zp_parameters) { + { + auto data = std::make_shared(ov::element::f16, ov::PartialShape{-1, 16}); + auto weights = ov::opset1::Constant::create(ov::element::u8, ov::Shape{32, 16}, {1}); + auto convert = std::make_shared(weights, ov::element::f16); + auto zp = std::make_shared(ov::element::u8, ov::Shape{32, 1}); + auto zp_convert = std::make_shared(zp, ov::element::f16); + auto sub = std::make_shared(convert, zp_convert); + auto scale = std::make_shared(ov::element::f16, ov::Shape{32, 1}); + auto mul = std::make_shared(sub, scale); + auto matmul = std::make_shared(data, mul, false, true); + + model = std::make_shared(ov::OutputVector{matmul}, ov::ParameterVector{data, zp, scale}); + manager.register_pass(true); + manager.register_pass(); + } + { + auto data = std::make_shared(ov::element::f16, ov::PartialShape{-1, 16}); + auto weights = ov::opset1::Constant::create(ov::element::u8, ov::Shape{32, 16}, {1}); + auto zp = std::make_shared(ov::element::u8, ov::Shape{32, 1}); + auto scale = std::make_shared(ov::element::f16, ov::Shape{32, 1}); + auto no_bias = std::make_shared(); + auto fc_compressed = std::make_shared(data, + weights, + no_bias, + scale, + zp, + ov::element::f16, + true, + true, + true); + + model_ref = std::make_shared(ov::OutputVector{fc_compressed}, ov::ParameterVector{data, zp, scale}); + } + comparator.enable(FunctionsComparator::ATTRIBUTES); +} + +enum class DynamicScaleZpInput { Scale, ZeroPoint }; +enum class DynamicScaleZpSource { Parameter, ReshapeOfConvert, ConvertOfReshape }; + +struct DynamicScaleZpParams { + DynamicScaleZpInput input; + DynamicScaleZpSource source; +}; + +class ConvertMatMulToFullyConnectedDynamicScaleZpTest : public testing::WithParamInterface, + public TransformationTestsF { +public: + ConvertMatMulToFullyConnectedDynamicScaleZpTest() : TransformationTestsF() { + comparator.enable(FunctionsComparator::ATTRIBUTES); + } + + static std::string getTestCaseName(const testing::TestParamInfo& info) { + const auto& [input, source] = info.param; + std::string name = input == DynamicScaleZpInput::Scale ? "Scale" : "ZeroPoint"; + switch (source) { + case DynamicScaleZpSource::Parameter: + return name + "_DynamicParameter"; + case DynamicScaleZpSource::ReshapeOfConvert: + return name + "_ReshapeOfConvert_DynamicTarget"; + case DynamicScaleZpSource::ConvertOfReshape: + return name + "_ConvertOfReshape_DynamicTarget"; + } + return name; + } +}; + +// The scale or zero point has a dynamic shape, either as a dynamic Parameter [?, 1] or as a static Parameter reshaped to a +// non-constant target shape. It broadcasts the decompressed weights to a dynamic shape, which FullyConnected can't take, +// so the MatMul must be left unchanged instead of failing on shape_b.is_static() in ConvertMatMulToFullyConnected. +TEST_P(ConvertMatMulToFullyConnectedDynamicScaleZpTest, KeepsMatMul) { + const auto& [input_kind, source] = GetParam(); + const bool is_scale = input_kind == DynamicScaleZpInput::Scale; + + auto data = std::make_shared(ov::element::f16, ov::PartialShape{-1, 16}); + auto weights = ov::opset1::Constant::create(ov::element::u8, ov::Shape{1, 16}, {1}); + std::shared_ptr dequantized = std::make_shared(weights, ov::element::f16); + ov::ParameterVector params{data}; + + const auto param_type = is_scale ? ov::element::f32 : ov::element::u8; + std::shared_ptr value; + if (source == DynamicScaleZpSource::Parameter) { + auto param = std::make_shared(param_type, ov::PartialShape{-1, 1}); + params.push_back(param); + value = std::make_shared(param, ov::element::f16); + } else { + auto param = std::make_shared(param_type, ov::Shape{1, 1, 1}); + auto target_shape = std::make_shared(ov::element::i32, ov::Shape{2}); + params.push_back(param); + params.push_back(target_shape); + if (source == DynamicScaleZpSource::ReshapeOfConvert) { + auto convert = std::make_shared(param, ov::element::f16); + value = std::make_shared(convert, target_shape, false); + } else { + auto reshape = std::make_shared(param, target_shape, false); + value = std::make_shared(reshape, ov::element::f16); + } + } + + if (is_scale) { + dequantized = std::make_shared(dequantized, value); + } else { + dequantized = std::make_shared(dequantized, value); + auto scale = ov::opset1::Constant::create(ov::element::f16, ov::Shape{1, 1}, {1}); + dequantized = std::make_shared(dequantized, scale); + } + ASSERT_TRUE(dequantized->get_output_partial_shape(0).is_dynamic()); + auto matmul = std::make_shared(data, dequantized, false, true); + + // No model_ref: the model is expected to be unchanged. + model = std::make_shared(ov::OutputVector{matmul}, params); + manager.register_pass(true); + manager.register_pass(); +} + +INSTANTIATE_TEST_SUITE_P( + TransformationTests, + ConvertMatMulToFullyConnectedDynamicScaleZpTest, + testing::Values(DynamicScaleZpParams{DynamicScaleZpInput::Scale, DynamicScaleZpSource::Parameter}, + DynamicScaleZpParams{DynamicScaleZpInput::Scale, DynamicScaleZpSource::ReshapeOfConvert}, + DynamicScaleZpParams{DynamicScaleZpInput::Scale, DynamicScaleZpSource::ConvertOfReshape}, + DynamicScaleZpParams{DynamicScaleZpInput::ZeroPoint, DynamicScaleZpSource::Parameter}, + DynamicScaleZpParams{DynamicScaleZpInput::ZeroPoint, DynamicScaleZpSource::ReshapeOfConvert}, + DynamicScaleZpParams{DynamicScaleZpInput::ZeroPoint, DynamicScaleZpSource::ConvertOfReshape}), + ConvertMatMulToFullyConnectedDynamicScaleZpTest::getTestCaseName); + // Checked blocked cases TEST(TransformationTests, ConvertMatMulToFullyConnectedExceptionTest_sibling_matmul_no_convert) { auto CreateMatMul = [&](bool mat1_transpose_a, bool mat1_transpose_b,