From 31d9a5c3ceb9dc67adfccbdb175badfb7f70e07c Mon Sep 17 00:00:00 2001 From: Kotomi-Du Date: Thu, 5 Nov 2026 20:24:17 -0800 Subject: [PATCH 01/10] initial change --- ...rt_weight_compressed_conv1x1_to_matmul.cpp | 70 +++++++++++-------- ...ight_compressed_conv1x1_to_matmul_test.cpp | 43 ++++++++++++ .../impls/onednn/fully_connected_onednn.hpp | 12 ++++ .../compressed_weights_pattern.hpp | 18 ++++- .../convert_fc_to_compressed.cpp | 69 +++++++++++------- .../convert_fc_to_compressed_test.cpp | 41 +++++++++++ 6 files changed, 197 insertions(+), 56 deletions(-) diff --git a/src/common/transformations/src/transformations/op_conversions/convert_weight_compressed_conv1x1_to_matmul.cpp b/src/common/transformations/src/transformations/op_conversions/convert_weight_compressed_conv1x1_to_matmul.cpp index bcde1252d385c0..eae5b2b427439e 100644 --- a/src/common/transformations/src/transformations/op_conversions/convert_weight_compressed_conv1x1_to_matmul.cpp +++ b/src/common/transformations/src/transformations/op_conversions/convert_weight_compressed_conv1x1_to_matmul.cpp @@ -134,6 +134,10 @@ ov::pass::ConvertWeightCompressedConv1x1ToMatmul::ConvertWeightCompressedConv1x1 auto weight = pattern_map.at(weights_m).get_node_shared_ptr(); auto scale = pattern_map.at(weights_scales_m).get_node_shared_ptr(); auto zp = (pattern_map.count(weights_zp_m) > 0) ? pattern_map.at(weights_zp_m).get_node_shared_ptr() : nullptr; + if (!scale->get_output_partial_shape(0).is_static() || + (zp && !zp->get_output_partial_shape(0).is_static())) { + return false; + } // Determine activation source based on which pattern branch matched std::shared_ptr activation; @@ -169,46 +173,44 @@ ov::pass::ConvertWeightCompressedConv1x1ToMatmul::ConvertWeightCompressedConv1x1 } } - auto reshape_const_to_2d = [](std::shared_ptr node) { - auto constant = ov::as_type_ptr(node); - OPENVINO_ASSERT(constant != nullptr); - ov::Shape current_shape = constant->get_shape(); + auto reshape_input_to_2d = [](std::shared_ptr node) -> std::shared_ptr { + const auto& partial_shape = node->get_output_partial_shape(0); + if (!partial_shape.is_static()) { + return nullptr; + } + const ov::Shape current_shape = partial_shape.to_shape(); if (current_shape.size() == 2) - return constant; + return node; + ov::Shape new_shape; if (current_shape.size() <= 1) { - auto new_shape = ov::Shape{(current_shape.size() == 1) ? current_shape[0] : 1, 1}; - - auto new_constant = std::make_shared(*constant, new_shape); - - ov::copy_weightless_cache_attr(constant, new_constant); - return new_constant; + new_shape = ov::Shape{(current_shape.size() == 1) ? current_shape[0] : 1, 1}; } else if (current_shape.size() == 4) { OPENVINO_ASSERT(current_shape[2] == 1 && current_shape[3] == 1); - - auto new_shape = ov::Shape{current_shape[0], current_shape[1]}; - - auto new_constant = std::make_shared(*constant, new_shape); - - ov::copy_weightless_cache_attr(constant, new_constant); - return new_constant; + new_shape = ov::Shape{current_shape[0], current_shape[1]}; } else { OPENVINO_ASSERT(current_shape.size() == 5); OPENVINO_ASSERT(current_shape[3] == 1 && current_shape[4] == 1); + new_shape = ov::Shape{current_shape[0], current_shape[1], current_shape[2]}; + } - auto new_shape = ov::Shape{current_shape[0], current_shape[1], current_shape[2]}; - + if (auto constant = ov::as_type_ptr(node)) { auto new_constant = std::make_shared(*constant, new_shape); - ov::copy_weightless_cache_attr(constant, new_constant); return new_constant; } + + auto shape_const = ov::op::v0::Constant::create(ov::element::i32, ov::Shape{new_shape.size()}, new_shape); + auto reshape = std::make_shared(node, shape_const, false); + reshape->set_friendly_name(node->get_friendly_name() + "_Reshape"); + ov::copy_runtime_info(node, {shape_const, reshape}); + return reshape; }; // add reshape after weight std::shared_ptr weight_squeezed_convert; if (ov::as_type_ptr(weight)) { - auto Reshape_weight = reshape_const_to_2d(weight); + auto Reshape_weight = reshape_input_to_2d(weight); MatcherPass::register_new_node(Reshape_weight); Reshape_weight->set_friendly_name(weight->get_friendly_name() + "_Reshape_weight"); weight_squeezed_convert = @@ -235,17 +237,27 @@ ov::pass::ConvertWeightCompressedConv1x1ToMatmul::ConvertWeightCompressedConv1x1 ov::disable_constant_folding(weight_squeezed_convert); // add reshape after scales - auto Reshape_scale = reshape_const_to_2d(scale); - MatcherPass::register_new_node(Reshape_scale); - Reshape_scale->set_friendly_name(scale->get_friendly_name() + "_Reshape_scale"); - ov::copy_runtime_info(scale, Reshape_scale); + auto Reshape_scale = reshape_input_to_2d(scale); + if (!Reshape_scale) { + return false; + } + if (Reshape_scale != scale) { + MatcherPass::register_new_node(Reshape_scale); + Reshape_scale->set_friendly_name(scale->get_friendly_name() + "_Reshape_scale"); + ov::copy_runtime_info(scale, Reshape_scale); + } auto scaled_weight = weight_mult->clone_with_new_inputs({weight_squeezed_convert, Reshape_scale}); if (zp) { // add reshape after zero points - auto Reshape_zp = reshape_const_to_2d(zp); - MatcherPass::register_new_node(Reshape_zp); - Reshape_zp->set_friendly_name(zp->get_friendly_name() + "_Reshape_zp"); + auto Reshape_zp = reshape_input_to_2d(zp); + if (!Reshape_zp) { + return false; + } + if (Reshape_zp != zp) { + MatcherPass::register_new_node(Reshape_zp); + Reshape_zp->set_friendly_name(zp->get_friendly_name() + "_Reshape_zp"); + } auto weights_zp_convert = ov::as_type_ptr(pattern_map.at(weights_zp_convert_m).get_node_shared_ptr()); auto zp_squeezed_convert = weights_zp_convert->clone_with_new_inputs({Reshape_zp}); diff --git a/src/common/transformations/tests/op_conversions/convert_weight_compressed_conv1x1_to_matmul_test.cpp b/src/common/transformations/tests/op_conversions/convert_weight_compressed_conv1x1_to_matmul_test.cpp index 773d8adacbe559..17740f3c14e7af 100644 --- a/src/common/transformations/tests/op_conversions/convert_weight_compressed_conv1x1_to_matmul_test.cpp +++ b/src/common/transformations/tests/op_conversions/convert_weight_compressed_conv1x1_to_matmul_test.cpp @@ -296,6 +296,49 @@ INSTANTIATE_TEST_SUITE_P(TransformationTests, "None")), ConvertWeightCompressedConv1x1ToMatmulTest::get_test_case_name); +TEST_F(TransformationTestsF, ConvertWeightCompressedConv1x1ToMatmulWithScaleAndZeroPointParameters) { + manager.register_pass(); + { + auto input = std::make_shared(ov::element::f16, ov::Shape{1, 1, 10, 10}); + auto input_order = ov::op::v0::Constant::create(ov::element::i32, ov::Shape{4}, {0, 3, 1, 2}); + auto activation = std::make_shared(input, input_order); + auto weights = ov::op::v0::Constant::create(ov::element::i4, ov::Shape{15, 10, 1, 1}, {1}); + auto weights_convert = std::make_shared(weights, ov::element::f16); + auto zero_point = std::make_shared(ov::element::i4, ov::Shape{15, 1, 1, 1}); + auto zero_point_convert = std::make_shared(zero_point, ov::element::f16); + auto zero_adjusted_weights = std::make_shared(weights_convert, zero_point_convert); + auto scale = std::make_shared(ov::element::f16, ov::Shape{15, 1, 1, 1}); + auto dequantized_weights = std::make_shared(zero_adjusted_weights, scale); + auto convolution = std::make_shared(activation, + dequantized_weights, + ov::Strides{1, 1}, + ov::CoordinateDiff{0, 0}, + ov::CoordinateDiff{0, 0}, + ov::Strides{1, 1}, + ov::op::PadType::EXPLICIT); + auto output_order = ov::op::v0::Constant::create(ov::element::i32, ov::Shape{4}, {0, 2, 3, 1}); + auto output = std::make_shared(convolution, output_order); + model = std::make_shared(ov::OutputVector{output}, ov::ParameterVector{input, scale, zero_point}); + } + { + auto input = std::make_shared(ov::element::f16, ov::Shape{1, 1, 10, 10}); + auto weights = ov::op::v0::Constant::create(ov::element::i4, ov::Shape{15, 10}, {1}); + auto weights_convert = std::make_shared(weights, ov::element::f16); + auto zero_point = std::make_shared(ov::element::i4, ov::Shape{15, 1, 1, 1}); + auto zero_point_shape = ov::op::v0::Constant::create(ov::element::i32, ov::Shape{2}, {15, 1}); + auto zero_point_reshape = std::make_shared(zero_point, zero_point_shape, false); + auto zero_point_convert = std::make_shared(zero_point_reshape, ov::element::f16); + auto zero_adjusted_weights = std::make_shared(weights_convert, zero_point_convert); + auto scale = std::make_shared(ov::element::f16, ov::Shape{15, 1, 1, 1}); + auto scale_shape = ov::op::v0::Constant::create(ov::element::i32, ov::Shape{2}, {15, 1}); + auto scale_reshape = std::make_shared(scale, scale_shape, false); + auto dequantized_weights = std::make_shared(zero_adjusted_weights, scale_reshape); + auto matmul = std::make_shared(input, dequantized_weights, false, true); + model_ref = std::make_shared(ov::OutputVector{matmul}, + ov::ParameterVector{input, scale, zero_point}); + } +} + // Checked blocked cases class ConvertWeightCompressedConv1x1ToMatmulTest_Blocked : public TransformationTestsF { public: diff --git a/src/plugins/intel_gpu/src/graph/impls/onednn/fully_connected_onednn.hpp b/src/plugins/intel_gpu/src/graph/impls/onednn/fully_connected_onednn.hpp index 9960d13efba310..e56b209e59ba18 100644 --- a/src/plugins/intel_gpu/src/graph/impls/onednn/fully_connected_onednn.hpp +++ b/src/plugins/intel_gpu/src/graph/impls/onednn/fully_connected_onednn.hpp @@ -5,6 +5,7 @@ #include "fully_connected_inst.h" #include "impls/onednn/utils.hpp" #include "intel_gpu/primitives/fully_connected.hpp" +#include "intel_gpu/primitives/input_layout.hpp" #include "intel_gpu/runtime/utils.hpp" #include "registry/implementation_manager.hpp" #include "intel_gpu/runtime/debug_configuration.hpp" @@ -140,6 +141,8 @@ struct FullyConnectedImplementationManager : public ImplementationManager { in_out_fmts_t query_formats(const program_node& node) const override { assert(node.is_type()); + const auto& fc_node = node.as(); + const auto& fc_prim = fc_node.get_primitive(); std::vector in_fmts(node.get_dependencies().size(), format::any); std::vector out_fmts(node.get_outputs_count(), format::any); @@ -148,6 +151,15 @@ struct FullyConnectedImplementationManager : public ImplementationManager { if (node.get_dependency(idx).is_constant()) { continue; } + const size_t scale_idx = fc_prim->bias.is_valid() ? 3 : 2; + const size_t zp_idx = scale_idx + 1; + const bool is_decompression_parameter = + ((fc_prim->decompression_scale.is_valid() && idx == scale_idx) || + (fc_prim->decompression_zero_point.is_valid() && idx == zp_idx)) && + node.get_dependency(idx).is_type(); + if (is_decompression_parameter) { + continue; + } auto target_format = format::get_default_format(out_rank); diff --git a/src/plugins/intel_gpu/src/plugin/transformations/compressed_weights_pattern.hpp b/src/plugins/intel_gpu/src/plugin/transformations/compressed_weights_pattern.hpp index 3cda16a1c43eac..e0a780316f9e53 100644 --- a/src/plugins/intel_gpu/src/plugin/transformations/compressed_weights_pattern.hpp +++ b/src/plugins/intel_gpu/src/plugin/transformations/compressed_weights_pattern.hpp @@ -29,13 +29,27 @@ using ov::pass::operator|; auto weights_convert_reshape_m = wrap_type({weights_param_convert_m, any_input()});\ auto decompressed_weights_m = convert_m | weights_convert_reshape_m;\ \ - auto sub_const_m = wrap_type();\ + auto sub_value_const_m = wrap_type();\ + auto sub_value_param_m = wrap_type();\ + auto sub_value_param_reshape_m = wrap_type({sub_value_param_m, any_input()});\ + auto sub_value_param_convert_m = wrap_type({sub_value_param_m});\ + auto sub_value_param_convert_reshape_m =\ + wrap_type({sub_value_param_convert_m, any_input()});\ + auto sub_const_m = sub_value_const_m | sub_value_param_m | sub_value_param_reshape_m |\ + sub_value_param_convert_reshape_m;\ auto sub_convert_const_m = wrap_type({sub_const_m});\ auto sub_with_convert_m = wrap_type({decompressed_weights_m, sub_convert_const_m});\ auto sub_no_convert_m = wrap_type({decompressed_weights_m, sub_const_m});\ auto subtract_m = sub_with_convert_m | sub_no_convert_m;\ \ - auto mul_const_m = wrap_type();\ + auto mul_value_const_m = wrap_type();\ + auto mul_value_param_m = wrap_type();\ + auto mul_value_param_reshape_m = wrap_type({mul_value_param_m, any_input()});\ + auto mul_value_param_convert_m = wrap_type({mul_value_param_m});\ + auto mul_value_param_convert_reshape_m =\ + wrap_type({mul_value_param_convert_m, any_input()});\ + auto mul_const_m = mul_value_const_m | mul_value_param_m | mul_value_param_reshape_m |\ + mul_value_param_convert_reshape_m;\ auto mul_with_sub_m = wrap_type({subtract_m, mul_const_m});\ auto mul_const_convert_m = ov::pass::pattern::optional(mul_const_m);\ auto mul_no_sub_m = wrap_type({decompressed_weights_m, mul_const_convert_m});\ diff --git a/src/plugins/intel_gpu/src/plugin/transformations/convert_fc_to_compressed.cpp b/src/plugins/intel_gpu/src/plugin/transformations/convert_fc_to_compressed.cpp index 6ebb1728f9d197..5d871a28b49737 100644 --- a/src/plugins/intel_gpu/src/plugin/transformations/convert_fc_to_compressed.cpp +++ b/src/plugins/intel_gpu/src/plugin/transformations/convert_fc_to_compressed.cpp @@ -48,7 +48,15 @@ ConvertFullyConnectedToFullyConnectedCompressed::ConvertFullyConnectedToFullyCon } const bool has_transpose_before_reshape = pattern_map.count(transpose_before_reshape_m) != 0u; const bool has_transpose = pattern_map.count(transpose_m) != 0u; - auto scale_shape = pattern_map.at(mul_const_m).get_shape(); + const auto scale_output = pattern_map.at(mul_const_m); + if (!scale_output.get_partial_shape().is_static()) { + return false; + } + auto scale_shape = scale_output.get_shape(); + const bool with_zero_point = pattern_map.count(subtract_m) > 0; + if (with_zero_point && !pattern_map.at(sub_const_m).get_partial_shape().is_static()) { + return false; + } bool sub_with_convert = pattern_map.count(sub_convert_const_m) > 0; if (!fc->get_input_partial_shape(1).is_static()) { @@ -86,18 +94,18 @@ ConvertFullyConnectedToFullyConnectedCompressed::ConvertFullyConnectedToFullyCon weight_u8 = true; } - auto reshape_const = [has_transpose, grouped, is_weight_3d](std::shared_ptr node) { - auto constant = ov::as_type_ptr(node); - OPENVINO_ASSERT(constant != nullptr); - ov::Shape current_shape = constant->get_shape(); + auto reshape_input = [has_transpose, grouped, is_weight_3d](std::shared_ptr node) { + const auto& partial_shape = node->get_output_partial_shape(0); + OPENVINO_ASSERT(partial_shape.is_static()); + const ov::Shape current_shape = partial_shape.to_shape(); if (current_shape.size() <= 2) { - return constant; + return node; } ov::Shape new_shape; if (current_shape.size() == 3) { if (is_weight_3d) { - return constant; + return node; } new_shape = (has_transpose || !grouped) ? ov::Shape{current_shape[0] * current_shape[1], current_shape[2]} : ov::Shape{current_shape[0], current_shape[1] * current_shape[2]}; @@ -110,52 +118,62 @@ ConvertFullyConnectedToFullyConnectedCompressed::ConvertFullyConnectedToFullyCon } else { OPENVINO_THROW("Unexpected constant shape rank ", current_shape.size(), " with is_weight_3d=", is_weight_3d); } - auto new_constant = std::make_shared(*constant, new_shape); - - ov::copy_weightless_cache_attr(constant, new_constant); - return new_constant; + if (auto constant = ov::as_type_ptr(node)) { + auto new_constant = std::make_shared(*constant, new_shape); + ov::copy_weightless_cache_attr(constant, new_constant); + return std::static_pointer_cast(new_constant); + } + auto shape_const = std::make_shared( + ov::element::i32, + ov::Shape{new_shape.size()}, + std::vector(new_shape.begin(), new_shape.end())); + auto reshape = std::make_shared(node, shape_const, false); + reshape->set_friendly_name(node->get_friendly_name() + "_reshape"); + ov::copy_runtime_info(node, {shape_const, reshape}); + return std::static_pointer_cast(reshape); }; auto convert_zp_const = [&](std::shared_ptr node) { auto constant = ov::as_type_ptr(node); - std::shared_ptr result = nullptr; + const auto zp_type = node->get_element_type(); + std::shared_ptr result = node; // Convert ZP to u8 (or i8 for a scalar i4 ZP) - if (constant->get_element_type() == ov::element::u8) { - result = constant; - } else if (constant->get_element_type() == ov::element::u4 || constant->get_element_type() == ov::element::u3 || - constant->get_element_type() == ov::element::u2) { + if (zp_type == ov::element::u4 || zp_type == ov::element::u3 || zp_type == ov::element::u2) { result = std::make_shared(node, ov::element::u8); // Only unsigned ZP types can be converted to u8. - } else if (weight_u8 && sub_with_convert && !constant->get_element_type().is_signed()) { + } else if (weight_u8 && sub_with_convert && !zp_type.is_signed()) { result = std::make_shared(node, ov::element::u8); - } else if (constant->get_element_type() == ov::element::i4 && ov::shape_size(constant->get_shape()) == 1) { + } else if (zp_type == ov::element::i4 && node->get_output_partial_shape(0).is_static() && + ov::shape_size(node->get_output_shape(0)) == 1) { // oneDNN doesn't support sub-byte per-tensor (common) zero points, so a scalar i4 ZP is converted to i8. result = std::make_shared(node, ov::element::i8); - } else { - result = constant; } - ov::copy_weightless_cache_attr(node, result); + if (result != node) { + ov::copy_runtime_info(node, result); + if (constant) { + ov::copy_weightless_cache_attr(constant, result); + } + } return result; }; const ov::Output& fc_input_a = fc->input(0).get_source_output(); const auto& scale = has_transpose_before_reshape ? pattern_map.at(mul_const_m).get_node_shared_ptr() - : reshape_const(pattern_map.at(mul_const_m).get_node_shared_ptr()); + : reshape_input(pattern_map.at(mul_const_m).get_node_shared_ptr()); std::shared_ptr optional_zero_point = nullptr; - const bool with_zero_point = pattern_map.count(subtract_m) > 0; if (with_zero_point) { const auto zero_point = has_transpose_before_reshape ? pattern_map.at(sub_const_m).get_node_shared_ptr() - : reshape_const(pattern_map.at(sub_const_m).get_node_shared_ptr()); + : reshape_input(pattern_map.at(sub_const_m).get_node_shared_ptr()); optional_zero_point = convert_zp_const(zero_point); } std::shared_ptr fc_input_b = weight_ptr; if (weights_are_constant && !has_transpose_before_reshape) { - fc_input_b = reshape_const(weight_ptr); + fc_input_b = reshape_input(weight_ptr); } else if (pattern_map.count(weights_reshape_m)) { fc_input_b = pattern_map.at(weights_reshape_m).get_node_shared_ptr(); } @@ -230,6 +248,7 @@ ConvertFullyConnectedToFullyConnectedCompressed::ConvertFullyConnectedToFullyCon ov::Shape{output_shape.size()}, output_shape); auto reshaped = std::make_shared(node, shape_const, false); + ov::copy_runtime_info(node, {shape_const, reshaped}); result_nodes.push_back(shape_const); result_nodes.push_back(reshaped); return std::static_pointer_cast(reshaped); diff --git a/src/plugins/intel_gpu/tests/unit/transformations/convert_fc_to_compressed_test.cpp b/src/plugins/intel_gpu/tests/unit/transformations/convert_fc_to_compressed_test.cpp index d6c7c61c1dde33..04c30a23d2589e 100644 --- a/src/plugins/intel_gpu/tests/unit/transformations/convert_fc_to_compressed_test.cpp +++ b/src/plugins/intel_gpu/tests/unit/transformations/convert_fc_to_compressed_test.cpp @@ -83,6 +83,47 @@ TEST_F(TransformationTestsF, ConvertFCToCompressed2) { } } +TEST_F(TransformationTestsF, ConvertFCToCompressedWithScaleAndZeroPointParameters) { + { + auto input = std::make_shared(ov::element::f16, ov::PartialShape{-1, 16}); + auto weights = ov::op::v0::Constant::create(ov::element::u8, ov::Shape{32, 16}, {1}); + auto weights_convert = std::make_shared(weights, ov::element::f16); + auto scale = std::make_shared(ov::element::f32, ov::Shape{32, 1, 1}); + auto scale_convert = std::make_shared(scale, ov::element::f16); + auto scale_shape = ov::op::v0::Constant::create(ov::element::i32, ov::Shape{2}, {32, 1}); + auto scale_reshape = std::make_shared(scale_convert, scale_shape, false); + auto zero_point = std::make_shared(ov::element::u8, ov::Shape{32, 1, 1}); + auto zero_point_shape = ov::op::v0::Constant::create(ov::element::i32, ov::Shape{2}, {32, 1}); + auto zero_point_reshape = std::make_shared(zero_point, zero_point_shape, false); + auto zero_point_convert = std::make_shared(zero_point_reshape, ov::element::f16); + auto subtract = std::make_shared(weights_convert, zero_point_convert); + auto dequantized_weights = std::make_shared(subtract, scale_reshape); + auto no_bias = std::make_shared(); + auto fc = std::make_shared(input, dequantized_weights, no_bias); + + model = std::make_shared(ov::OutputVector{fc}, ov::ParameterVector{input, scale, zero_point}); + manager.register_pass(); + } + { + auto input = std::make_shared(ov::element::f16, ov::PartialShape{-1, 16}); + auto weights = ov::op::v0::Constant::create(ov::element::u8, ov::Shape{32, 16}, {1}); + auto scale = std::make_shared(ov::element::f32, ov::Shape{32, 1, 1}); + auto scale_convert = std::make_shared(scale, ov::element::f16); + auto scale_shape = ov::op::v0::Constant::create(ov::element::i32, ov::Shape{2}, {32, 1}); + auto scale_reshape = std::make_shared(scale_convert, scale_shape, false); + auto zero_point = std::make_shared(ov::element::u8, ov::Shape{32, 1, 1}); + auto zero_point_shape = ov::op::v0::Constant::create(ov::element::i32, ov::Shape{2}, {32, 1}); + auto zero_point_reshape = std::make_shared(zero_point, zero_point_shape, false); + auto no_bias = std::make_shared(); + auto fc_compressed = + std::make_shared( + input, weights, no_bias, scale_reshape, zero_point_reshape); + + model_ref = std::make_shared(ov::OutputVector{fc_compressed}, + ov::ParameterVector{input, scale, zero_point}); + } +} + TEST_F(TransformationTestsF, ConvertFCToCompressed3) { { auto input1 = std::make_shared(ov::element::f32, ov::PartialShape{ -1, 16 }); From e9ab3454ee1fae01ff8bd28cd82874950e68313b Mon Sep 17 00:00:00 2001 From: Kotomi-Du Date: Thu, 5 Nov 2026 21:25:00 -0800 Subject: [PATCH 02/10] revert changes related to conv1x1 --- ...rt_weight_compressed_conv1x1_to_matmul.cpp | 70 ++++++++----------- ...ight_compressed_conv1x1_to_matmul_test.cpp | 43 ------------ 2 files changed, 29 insertions(+), 84 deletions(-) diff --git a/src/common/transformations/src/transformations/op_conversions/convert_weight_compressed_conv1x1_to_matmul.cpp b/src/common/transformations/src/transformations/op_conversions/convert_weight_compressed_conv1x1_to_matmul.cpp index eae5b2b427439e..bcde1252d385c0 100644 --- a/src/common/transformations/src/transformations/op_conversions/convert_weight_compressed_conv1x1_to_matmul.cpp +++ b/src/common/transformations/src/transformations/op_conversions/convert_weight_compressed_conv1x1_to_matmul.cpp @@ -134,10 +134,6 @@ ov::pass::ConvertWeightCompressedConv1x1ToMatmul::ConvertWeightCompressedConv1x1 auto weight = pattern_map.at(weights_m).get_node_shared_ptr(); auto scale = pattern_map.at(weights_scales_m).get_node_shared_ptr(); auto zp = (pattern_map.count(weights_zp_m) > 0) ? pattern_map.at(weights_zp_m).get_node_shared_ptr() : nullptr; - if (!scale->get_output_partial_shape(0).is_static() || - (zp && !zp->get_output_partial_shape(0).is_static())) { - return false; - } // Determine activation source based on which pattern branch matched std::shared_ptr activation; @@ -173,44 +169,46 @@ ov::pass::ConvertWeightCompressedConv1x1ToMatmul::ConvertWeightCompressedConv1x1 } } - auto reshape_input_to_2d = [](std::shared_ptr node) -> std::shared_ptr { - const auto& partial_shape = node->get_output_partial_shape(0); - if (!partial_shape.is_static()) { - return nullptr; - } - const ov::Shape current_shape = partial_shape.to_shape(); + auto reshape_const_to_2d = [](std::shared_ptr node) { + auto constant = ov::as_type_ptr(node); + OPENVINO_ASSERT(constant != nullptr); + ov::Shape current_shape = constant->get_shape(); if (current_shape.size() == 2) - return node; + return constant; - ov::Shape new_shape; if (current_shape.size() <= 1) { - new_shape = ov::Shape{(current_shape.size() == 1) ? current_shape[0] : 1, 1}; + auto new_shape = ov::Shape{(current_shape.size() == 1) ? current_shape[0] : 1, 1}; + + auto new_constant = std::make_shared(*constant, new_shape); + + ov::copy_weightless_cache_attr(constant, new_constant); + return new_constant; } else if (current_shape.size() == 4) { OPENVINO_ASSERT(current_shape[2] == 1 && current_shape[3] == 1); - new_shape = ov::Shape{current_shape[0], current_shape[1]}; + + auto new_shape = ov::Shape{current_shape[0], current_shape[1]}; + + auto new_constant = std::make_shared(*constant, new_shape); + + ov::copy_weightless_cache_attr(constant, new_constant); + return new_constant; } else { OPENVINO_ASSERT(current_shape.size() == 5); OPENVINO_ASSERT(current_shape[3] == 1 && current_shape[4] == 1); - new_shape = ov::Shape{current_shape[0], current_shape[1], current_shape[2]}; - } - if (auto constant = ov::as_type_ptr(node)) { + auto new_shape = ov::Shape{current_shape[0], current_shape[1], current_shape[2]}; + auto new_constant = std::make_shared(*constant, new_shape); + ov::copy_weightless_cache_attr(constant, new_constant); return new_constant; } - - auto shape_const = ov::op::v0::Constant::create(ov::element::i32, ov::Shape{new_shape.size()}, new_shape); - auto reshape = std::make_shared(node, shape_const, false); - reshape->set_friendly_name(node->get_friendly_name() + "_Reshape"); - ov::copy_runtime_info(node, {shape_const, reshape}); - return reshape; }; // add reshape after weight std::shared_ptr weight_squeezed_convert; if (ov::as_type_ptr(weight)) { - auto Reshape_weight = reshape_input_to_2d(weight); + auto Reshape_weight = reshape_const_to_2d(weight); MatcherPass::register_new_node(Reshape_weight); Reshape_weight->set_friendly_name(weight->get_friendly_name() + "_Reshape_weight"); weight_squeezed_convert = @@ -237,27 +235,17 @@ ov::pass::ConvertWeightCompressedConv1x1ToMatmul::ConvertWeightCompressedConv1x1 ov::disable_constant_folding(weight_squeezed_convert); // add reshape after scales - auto Reshape_scale = reshape_input_to_2d(scale); - if (!Reshape_scale) { - return false; - } - if (Reshape_scale != scale) { - MatcherPass::register_new_node(Reshape_scale); - Reshape_scale->set_friendly_name(scale->get_friendly_name() + "_Reshape_scale"); - ov::copy_runtime_info(scale, Reshape_scale); - } + auto Reshape_scale = reshape_const_to_2d(scale); + MatcherPass::register_new_node(Reshape_scale); + Reshape_scale->set_friendly_name(scale->get_friendly_name() + "_Reshape_scale"); + ov::copy_runtime_info(scale, Reshape_scale); auto scaled_weight = weight_mult->clone_with_new_inputs({weight_squeezed_convert, Reshape_scale}); if (zp) { // add reshape after zero points - auto Reshape_zp = reshape_input_to_2d(zp); - if (!Reshape_zp) { - return false; - } - if (Reshape_zp != zp) { - MatcherPass::register_new_node(Reshape_zp); - Reshape_zp->set_friendly_name(zp->get_friendly_name() + "_Reshape_zp"); - } + auto Reshape_zp = reshape_const_to_2d(zp); + MatcherPass::register_new_node(Reshape_zp); + Reshape_zp->set_friendly_name(zp->get_friendly_name() + "_Reshape_zp"); auto weights_zp_convert = ov::as_type_ptr(pattern_map.at(weights_zp_convert_m).get_node_shared_ptr()); auto zp_squeezed_convert = weights_zp_convert->clone_with_new_inputs({Reshape_zp}); diff --git a/src/common/transformations/tests/op_conversions/convert_weight_compressed_conv1x1_to_matmul_test.cpp b/src/common/transformations/tests/op_conversions/convert_weight_compressed_conv1x1_to_matmul_test.cpp index 17740f3c14e7af..773d8adacbe559 100644 --- a/src/common/transformations/tests/op_conversions/convert_weight_compressed_conv1x1_to_matmul_test.cpp +++ b/src/common/transformations/tests/op_conversions/convert_weight_compressed_conv1x1_to_matmul_test.cpp @@ -296,49 +296,6 @@ INSTANTIATE_TEST_SUITE_P(TransformationTests, "None")), ConvertWeightCompressedConv1x1ToMatmulTest::get_test_case_name); -TEST_F(TransformationTestsF, ConvertWeightCompressedConv1x1ToMatmulWithScaleAndZeroPointParameters) { - manager.register_pass(); - { - auto input = std::make_shared(ov::element::f16, ov::Shape{1, 1, 10, 10}); - auto input_order = ov::op::v0::Constant::create(ov::element::i32, ov::Shape{4}, {0, 3, 1, 2}); - auto activation = std::make_shared(input, input_order); - auto weights = ov::op::v0::Constant::create(ov::element::i4, ov::Shape{15, 10, 1, 1}, {1}); - auto weights_convert = std::make_shared(weights, ov::element::f16); - auto zero_point = std::make_shared(ov::element::i4, ov::Shape{15, 1, 1, 1}); - auto zero_point_convert = std::make_shared(zero_point, ov::element::f16); - auto zero_adjusted_weights = std::make_shared(weights_convert, zero_point_convert); - auto scale = std::make_shared(ov::element::f16, ov::Shape{15, 1, 1, 1}); - auto dequantized_weights = std::make_shared(zero_adjusted_weights, scale); - auto convolution = std::make_shared(activation, - dequantized_weights, - ov::Strides{1, 1}, - ov::CoordinateDiff{0, 0}, - ov::CoordinateDiff{0, 0}, - ov::Strides{1, 1}, - ov::op::PadType::EXPLICIT); - auto output_order = ov::op::v0::Constant::create(ov::element::i32, ov::Shape{4}, {0, 2, 3, 1}); - auto output = std::make_shared(convolution, output_order); - model = std::make_shared(ov::OutputVector{output}, ov::ParameterVector{input, scale, zero_point}); - } - { - auto input = std::make_shared(ov::element::f16, ov::Shape{1, 1, 10, 10}); - auto weights = ov::op::v0::Constant::create(ov::element::i4, ov::Shape{15, 10}, {1}); - auto weights_convert = std::make_shared(weights, ov::element::f16); - auto zero_point = std::make_shared(ov::element::i4, ov::Shape{15, 1, 1, 1}); - auto zero_point_shape = ov::op::v0::Constant::create(ov::element::i32, ov::Shape{2}, {15, 1}); - auto zero_point_reshape = std::make_shared(zero_point, zero_point_shape, false); - auto zero_point_convert = std::make_shared(zero_point_reshape, ov::element::f16); - auto zero_adjusted_weights = std::make_shared(weights_convert, zero_point_convert); - auto scale = std::make_shared(ov::element::f16, ov::Shape{15, 1, 1, 1}); - auto scale_shape = ov::op::v0::Constant::create(ov::element::i32, ov::Shape{2}, {15, 1}); - auto scale_reshape = std::make_shared(scale, scale_shape, false); - auto dequantized_weights = std::make_shared(zero_adjusted_weights, scale_reshape); - auto matmul = std::make_shared(input, dequantized_weights, false, true); - model_ref = std::make_shared(ov::OutputVector{matmul}, - ov::ParameterVector{input, scale, zero_point}); - } -} - // Checked blocked cases class ConvertWeightCompressedConv1x1ToMatmulTest_Blocked : public TransformationTestsF { public: From 167732b1dc02f2da84f623cba004bbd0a61df4cb Mon Sep 17 00:00:00 2001 From: Kotomi-Du Date: Thu, 5 Nov 2026 21:41:06 -0800 Subject: [PATCH 03/10] update logic This reverts commit a2a07c0d1fd67a1ee49eb2f9c24ffa3cd3e02e67. --- .../src/graph/impls/onednn/fully_connected_onednn.hpp | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/plugins/intel_gpu/src/graph/impls/onednn/fully_connected_onednn.hpp b/src/plugins/intel_gpu/src/graph/impls/onednn/fully_connected_onednn.hpp index e56b209e59ba18..03b8679e691271 100644 --- a/src/plugins/intel_gpu/src/graph/impls/onednn/fully_connected_onednn.hpp +++ b/src/plugins/intel_gpu/src/graph/impls/onednn/fully_connected_onednn.hpp @@ -154,11 +154,13 @@ struct FullyConnectedImplementationManager : public ImplementationManager { const size_t scale_idx = fc_prim->bias.is_valid() ? 3 : 2; const size_t zp_idx = scale_idx + 1; const bool is_decompression_parameter = - ((fc_prim->decompression_scale.is_valid() && idx == scale_idx) || - (fc_prim->decompression_zero_point.is_valid() && idx == zp_idx)) && - node.get_dependency(idx).is_type(); + ((fc_prim->decompression_scale.is_valid() && idx == scale_idx) || + (fc_prim->decompression_zero_point.is_valid() && idx == zp_idx)); if (is_decompression_parameter) { - continue; + in_fmts[idx] = node.get_input_layout(idx).format; + GPU_DEBUG_LOG << node.id() << " : Keep scale/zp parameter format " + << "at index " << idx << std::endl; + continue; } auto target_format = format::get_default_format(out_rank); From 547571223038648d033bb08d0a94902c18a5d304 Mon Sep 17 00:00:00 2001 From: Kotomi-Du Date: Thu, 5 Nov 2026 23:14:57 -0800 Subject: [PATCH 04/10] update unit tests --- .../test_cases/fully_connected_gpu_test.cpp | 112 ++++++++++++++++++ 1 file changed, 112 insertions(+) diff --git a/src/plugins/intel_gpu/tests/unit/test_cases/fully_connected_gpu_test.cpp b/src/plugins/intel_gpu/tests/unit/test_cases/fully_connected_gpu_test.cpp index 9211c81c6e7700..eab011d897085f 100644 --- a/src/plugins/intel_gpu/tests/unit/test_cases/fully_connected_gpu_test.cpp +++ b/src/plugins/intel_gpu/tests/unit/test_cases/fully_connected_gpu_test.cpp @@ -3275,6 +3275,102 @@ void test_compressed_int4_scale_dynamic_batch_gemv(bool is_caching_test, } } + void test_compressed_int8_parameter_scale_zp(bool is_per_group, bool transpose_scale_zp) { + auto& engine = get_test_engine(); + + constexpr int64_t ofm = 4; + constexpr int64_t ifm = 32; + const int64_t groups = is_per_group ? 2 : 1; + const int64_t group_size = ifm / groups; + + // Non-transposed scale/zp are [groups, N]; transposed scale/zp are [N, groups]. + const ov::PartialShape param_shape = transpose_scale_zp ? ov::PartialShape{ofm, groups} : ov::PartialShape{groups, ofm}; + auto param_offset = [&](int64_t group, int64_t output) { + return static_cast(transpose_scale_zp ? output * groups + group : group * ofm + output); + }; + + auto input_mem = engine.allocate_memory({ {1, 2, ifm}, data_types::f16, format::bfyx }); + auto weights_mem = engine.allocate_memory({ {ofm, ifm}, data_types::u8, format::bfyx }); + auto scale_mem = engine.allocate_memory({ param_shape, data_types::f16, format::bfyx }); + auto zp_mem = engine.allocate_memory({ param_shape, data_types::u8, format::bfyx }); + + std::vector input_data(2 * ifm); + for (size_t i = 0; i < input_data.size(); ++i) { + input_data[i] = static_cast(static_cast(i % 3) - 1); + } + std::vector weights_data(ofm * ifm); + for (int64_t output = 0; output < ofm; ++output) { + for (int64_t k = 0; k < ifm; ++k) { + weights_data[output * ifm + k] = static_cast((output * 3 + k) % 8); + } + } + std::vector scale_data(ofm * groups); + std::vector zp_data(ofm * groups); + for (int64_t group = 0; group < groups; ++group) { + for (int64_t output = 0; output < ofm; ++output) { + scale_data[param_offset(group, output)] = 0.5f * static_cast((group + output) % 2 + 1); + zp_data[param_offset(group, output)] = static_cast((group + output) % 3); + } + } + + std::vector input_f16(input_data.begin(), input_data.end()); + std::vector scale_f16(scale_data.begin(), scale_data.end()); + set_values(input_mem, input_f16); + set_values(weights_mem, weights_data); + set_values(scale_mem, scale_f16); + set_values(zp_mem, zp_data); + + topology topology( + input_layout("input", input_mem->get_layout()), + data("weights", weights_mem), + input_layout("scale", scale_mem->get_layout()), + input_layout("zp", zp_mem->get_layout()), + fully_connected("fc_prim", input_info("input"), "weights", "", "scale", "zp", data_types::f16, 3, 2, true, + transpose_scale_zp, transpose_scale_zp)); + + auto config = get_test_default_config(engine); + config.set_property(ov::intel_gpu::allow_new_shape_infer(true)); + config.set_property(ov::intel_gpu::optimize_data(true)); + + network network(engine, topology, config); + network.set_input_data("input", input_mem); + network.set_input_data("scale", scale_mem); + network.set_input_data("zp", zp_mem); + + // Grouped transposed scale/zp [N, groups] must be reordered from bfyx to fbyx to make N innermost. + // Per-channel [N, 1] already has N innermost, so it stays as the bfyx parameter. + const bool expect_reorder = transpose_scale_zp && is_per_group; + auto fc_inst = network.get_primitive("fc_prim"); + for (size_t idx : {size_t{2}, size_t{3}}) { + const auto& dep = fc_inst->get_node().get_dependency(idx); + if (expect_reorder) { + EXPECT_TRUE(dep.is_type()) << "input " << idx; + EXPECT_EQ(fc_inst->get_input_layout(idx).format, format::fbyx) << "input " << idx; + } else { + EXPECT_TRUE(dep.is_type()) << "input " << idx; + EXPECT_EQ(fc_inst->get_input_layout(idx).format, format::bfyx) << "input " << idx; + } + } + + auto outputs = network.execute(); + auto output_mem = outputs.at("fc_prim").get_memory(); + cldnn::mem_lock output_ptr(output_mem, get_test_stream()); + ASSERT_EQ((ov::PartialShape{1, 2, ofm}), output_mem->get_layout().get_partial_shape()); + + for (int64_t token = 0; token < 2; ++token) { + for (int64_t output = 0; output < ofm; ++output) { + float expected = 0.0f; + for (int64_t k = 0; k < ifm; ++k) { + const auto offset = param_offset(k / group_size, output); + const float weight = (static_cast(weights_data[output * ifm + k]) - zp_data[offset]) * scale_data[offset]; + expected += input_data[token * ifm + k] * weight; + } + EXPECT_NEAR(static_cast(output_ptr[token * ofm + output]), expected, 1e-2f) + << "token = " << token << ", output = " << output; + } + } + } + void test_dynamic(bool is_caching_test) { auto& engine = get_test_engine(); @@ -6115,6 +6211,22 @@ TEST_F(fully_connected_gpu_tests, compressed_int8_transpose_scale_zp) { this->test_compressed_int8_transpose_scale_zp(true); } +TEST_F(fully_connected_gpu_tests, compressed_int8_parameter_scale_zp_per_channel) { + this->test_compressed_int8_parameter_scale_zp(false, false); +} + +TEST_F(fully_connected_gpu_tests, compressed_int8_parameter_scale_zp_per_channel_transposed) { + this->test_compressed_int8_parameter_scale_zp(false, true); +} + +TEST_F(fully_connected_gpu_tests, compressed_int8_parameter_scale_zp_per_group) { + this->test_compressed_int8_parameter_scale_zp(true, false); +} + +TEST_F(fully_connected_gpu_tests, compressed_int8_parameter_scale_zp_per_group_transposed) { + this->test_compressed_int8_parameter_scale_zp(true, true); +} + TEST_F(fully_connected_gpu_tests, compressed_int8_scale_b1) { this->test_compressed_int8_scale(false, true, 1, false, false); } From 85feee875ada70d227a945a7808a0cf0026c961d Mon Sep 17 00:00:00 2001 From: Kotomi-Du Date: Thu, 5 Nov 2026 23:17:53 -0800 Subject: [PATCH 05/10] recover missing path --- .../src/plugin/transformations/convert_fc_to_compressed.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/plugins/intel_gpu/src/plugin/transformations/convert_fc_to_compressed.cpp b/src/plugins/intel_gpu/src/plugin/transformations/convert_fc_to_compressed.cpp index 5d871a28b49737..d156e604ac121f 100644 --- a/src/plugins/intel_gpu/src/plugin/transformations/convert_fc_to_compressed.cpp +++ b/src/plugins/intel_gpu/src/plugin/transformations/convert_fc_to_compressed.cpp @@ -138,7 +138,9 @@ ConvertFullyConnectedToFullyConnectedCompressed::ConvertFullyConnectedToFullyCon const auto zp_type = node->get_element_type(); std::shared_ptr result = node; // Convert ZP to u8 (or i8 for a scalar i4 ZP) - if (zp_type == ov::element::u4 || zp_type == ov::element::u3 || zp_type == ov::element::u2) { + if (zp_type == ov::element::u8) { + result = node; + } else if (zp_type == ov::element::u4 || zp_type == ov::element::u3 || zp_type == ov::element::u2) { result = std::make_shared(node, ov::element::u8); // Only unsigned ZP types can be converted to u8. } else if (weight_u8 && sub_with_convert && !zp_type.is_signed()) { From fe4ee237afec343e27dde7ec8ef3f81bc2463906 Mon Sep 17 00:00:00 2001 From: Kotomi-Du Date: Fri, 6 Nov 2026 16:55:16 -0800 Subject: [PATCH 06/10] gate for static scale/zp only; add extra pattern match Convert(Reshape(Parameter) --- .../compressed_weights_pattern.hpp | 8 +- .../convert_fc_to_compressed_test.cpp | 100 +++++++++++++----- .../convert_matmul_to_fc_test.cpp | 64 +++++++++++ 3 files changed, 145 insertions(+), 27 deletions(-) diff --git a/src/plugins/intel_gpu/src/plugin/transformations/compressed_weights_pattern.hpp b/src/plugins/intel_gpu/src/plugin/transformations/compressed_weights_pattern.hpp index e0a780316f9e53..b383d8432a308b 100644 --- a/src/plugins/intel_gpu/src/plugin/transformations/compressed_weights_pattern.hpp +++ b/src/plugins/intel_gpu/src/plugin/transformations/compressed_weights_pattern.hpp @@ -30,7 +30,7 @@ using ov::pass::operator|; auto decompressed_weights_m = convert_m | weights_convert_reshape_m;\ \ auto sub_value_const_m = wrap_type();\ - auto sub_value_param_m = wrap_type();\ + auto sub_value_param_m = wrap_type(ov::pass::pattern::has_static_shape());\ auto sub_value_param_reshape_m = wrap_type({sub_value_param_m, any_input()});\ auto sub_value_param_convert_m = wrap_type({sub_value_param_m});\ auto sub_value_param_convert_reshape_m =\ @@ -43,13 +43,15 @@ using ov::pass::operator|; auto subtract_m = sub_with_convert_m | sub_no_convert_m;\ \ auto mul_value_const_m = wrap_type();\ - auto mul_value_param_m = wrap_type();\ + auto mul_value_param_m = wrap_type(ov::pass::pattern::has_static_shape());\ auto mul_value_param_reshape_m = wrap_type({mul_value_param_m, any_input()});\ auto mul_value_param_convert_m = wrap_type({mul_value_param_m});\ auto mul_value_param_convert_reshape_m =\ wrap_type({mul_value_param_convert_m, any_input()});\ + auto mul_value_param_reshape_convert_m =\ + wrap_type({mul_value_param_reshape_m});\ auto mul_const_m = mul_value_const_m | mul_value_param_m | mul_value_param_reshape_m |\ - mul_value_param_convert_reshape_m;\ + mul_value_param_convert_reshape_m | mul_value_param_reshape_convert_m;\ auto mul_with_sub_m = wrap_type({subtract_m, mul_const_m});\ auto mul_const_convert_m = ov::pass::pattern::optional(mul_const_m);\ auto mul_no_sub_m = wrap_type({decompressed_weights_m, mul_const_convert_m});\ diff --git a/src/plugins/intel_gpu/tests/unit/transformations/convert_fc_to_compressed_test.cpp b/src/plugins/intel_gpu/tests/unit/transformations/convert_fc_to_compressed_test.cpp index 04c30a23d2589e..30451fbcaa0e68 100644 --- a/src/plugins/intel_gpu/tests/unit/transformations/convert_fc_to_compressed_test.cpp +++ b/src/plugins/intel_gpu/tests/unit/transformations/convert_fc_to_compressed_test.cpp @@ -83,47 +83,99 @@ TEST_F(TransformationTestsF, ConvertFCToCompressed2) { } } -TEST_F(TransformationTestsF, ConvertFCToCompressedWithScaleAndZeroPointParameters) { +enum class ParameterScaleTopology { ConvertThenReshape, ReshapeThenConvert }; + +struct ParameterScaleZpParams { + ParameterScaleTopology scale_topology; + bool with_zp; +}; + +class ConvertFCToCompressedParameterScaleZpTest : public testing::WithParamInterface, + public TransformationTestsF { +public: + static std::string getTestCaseName(const testing::TestParamInfo& info) { + std::ostringstream result; + result << (info.param.scale_topology == ParameterScaleTopology::ConvertThenReshape ? "ReshapeOfConvert" + : "ConvertOfReshape") + << (info.param.with_zp ? "_WithZp" : "_NoZp"); + return result.str(); + } +}; + +// Scale (and zero point) come from model Parameters [32, 1, 1] reshaped to [32, 1]: +// - ConvertThenReshape: Reshape(Convert(scale)) +// - ReshapeThenConvert: Convert(Reshape(scale)) +// The zero point is Convert(Reshape(zp)) and is passed to FullyConnectedCompressed without the Convert. +TEST_P(ConvertFCToCompressedParameterScaleZpTest, ConvertsParameterScaleAndZeroPoint) { + const auto& [scale_topology, with_zp] = GetParam(); + const bool convert_then_reshape = scale_topology == ParameterScaleTopology::ConvertThenReshape; + + auto reshape_to_2d = [](const ov::Output& node) { + auto shape = ov::op::v0::Constant::create(ov::element::i32, ov::Shape{2}, {32, 1}); + return std::make_shared(node, shape, false); + }; + auto to_f16 = [](const ov::Output& node) { + return std::make_shared(node, ov::element::f16); + }; + auto build_scale = [&](const std::shared_ptr& scale) -> std::shared_ptr { + return convert_then_reshape ? std::shared_ptr(reshape_to_2d(to_f16(scale))) + : std::shared_ptr(to_f16(reshape_to_2d(scale))); + }; + { auto input = std::make_shared(ov::element::f16, ov::PartialShape{-1, 16}); auto weights = ov::op::v0::Constant::create(ov::element::u8, ov::Shape{32, 16}, {1}); - auto weights_convert = std::make_shared(weights, ov::element::f16); + std::shared_ptr dequantized = to_f16(weights); auto scale = std::make_shared(ov::element::f32, ov::Shape{32, 1, 1}); - auto scale_convert = std::make_shared(scale, ov::element::f16); - auto scale_shape = ov::op::v0::Constant::create(ov::element::i32, ov::Shape{2}, {32, 1}); - auto scale_reshape = std::make_shared(scale_convert, scale_shape, false); - auto zero_point = std::make_shared(ov::element::u8, ov::Shape{32, 1, 1}); - auto zero_point_shape = ov::op::v0::Constant::create(ov::element::i32, ov::Shape{2}, {32, 1}); - auto zero_point_reshape = std::make_shared(zero_point, zero_point_shape, false); - auto zero_point_convert = std::make_shared(zero_point_reshape, ov::element::f16); - auto subtract = std::make_shared(weights_convert, zero_point_convert); - auto dequantized_weights = std::make_shared(subtract, scale_reshape); + ov::ParameterVector params{input, scale}; + if (with_zp) { + auto zero_point = std::make_shared(ov::element::u8, ov::Shape{32, 1, 1}); + dequantized = std::make_shared(dequantized, to_f16(reshape_to_2d(zero_point))); + params.push_back(zero_point); + } + dequantized = std::make_shared(dequantized, build_scale(scale)); auto no_bias = std::make_shared(); - auto fc = std::make_shared(input, dequantized_weights, no_bias); + auto fc = std::make_shared(input, dequantized, no_bias); - model = std::make_shared(ov::OutputVector{fc}, ov::ParameterVector{input, scale, zero_point}); + model = std::make_shared(ov::OutputVector{fc}, params); manager.register_pass(); } { auto input = std::make_shared(ov::element::f16, ov::PartialShape{-1, 16}); auto weights = ov::op::v0::Constant::create(ov::element::u8, ov::Shape{32, 16}, {1}); auto scale = std::make_shared(ov::element::f32, ov::Shape{32, 1, 1}); - auto scale_convert = std::make_shared(scale, ov::element::f16); - auto scale_shape = ov::op::v0::Constant::create(ov::element::i32, ov::Shape{2}, {32, 1}); - auto scale_reshape = std::make_shared(scale_convert, scale_shape, false); - auto zero_point = std::make_shared(ov::element::u8, ov::Shape{32, 1, 1}); - auto zero_point_shape = ov::op::v0::Constant::create(ov::element::i32, ov::Shape{2}, {32, 1}); - auto zero_point_reshape = std::make_shared(zero_point, zero_point_shape, false); + // Without a zero point, Multiply's optional Convert absorbs a trailing scale Convert, + // so Convert(Reshape(scale)) is passed as Reshape(scale) in its original precision. + std::shared_ptr expected_scale = + (!with_zp && !convert_then_reshape) ? std::shared_ptr(reshape_to_2d(scale)) : build_scale(scale); auto no_bias = std::make_shared(); - auto fc_compressed = - std::make_shared( - input, weights, no_bias, scale_reshape, zero_point_reshape); + ov::ParameterVector params{input, scale}; + std::shared_ptr fc_compressed; + if (with_zp) { + auto zero_point = std::make_shared(ov::element::u8, ov::Shape{32, 1, 1}); + fc_compressed = std::make_shared(input, + weights, + no_bias, + expected_scale, + reshape_to_2d(zero_point)); + params.push_back(zero_point); + } else { + fc_compressed = + std::make_shared(input, weights, no_bias, expected_scale); + } - model_ref = std::make_shared(ov::OutputVector{fc_compressed}, - ov::ParameterVector{input, scale, zero_point}); + model_ref = std::make_shared(ov::OutputVector{fc_compressed}, params); } } +INSTANTIATE_TEST_SUITE_P(TransformationTests, + ConvertFCToCompressedParameterScaleZpTest, + testing::Values(ParameterScaleZpParams{ParameterScaleTopology::ConvertThenReshape, true}, + ParameterScaleZpParams{ParameterScaleTopology::ConvertThenReshape, false}, + ParameterScaleZpParams{ParameterScaleTopology::ReshapeThenConvert, true}, + ParameterScaleZpParams{ParameterScaleTopology::ReshapeThenConvert, false}), + ConvertFCToCompressedParameterScaleZpTest::getTestCaseName); + TEST_F(TransformationTestsF, ConvertFCToCompressed3) { { auto input1 = std::make_shared(ov::element::f32, ov::PartialShape{ -1, 16 }); diff --git a/src/plugins/intel_gpu/tests/unit/transformations/convert_matmul_to_fc_test.cpp b/src/plugins/intel_gpu/tests/unit/transformations/convert_matmul_to_fc_test.cpp index 76c6cba66ae788..0b9a214a192dcd 100644 --- a/src/plugins/intel_gpu/tests/unit/transformations/convert_matmul_to_fc_test.cpp +++ b/src/plugins/intel_gpu/tests/unit/transformations/convert_matmul_to_fc_test.cpp @@ -636,6 +636,70 @@ TEST_F(TransformationTestsF, ConvertMatMulToFullyConnectedTest_compressed_u4_wei } } +TEST_F(TransformationTestsF, ConvertMatMulToFullyConnectedTest_compressed_u8_weights_static_scale_zp_parameters) { + { + auto data = std::make_shared(ov::element::f16, ov::PartialShape{-1, 16}); + auto weights = ov::opset1::Constant::create(ov::element::u8, ov::Shape{32, 16}, {1}); + auto convert = std::make_shared(weights, ov::element::f16); + auto zp = std::make_shared(ov::element::u8, ov::Shape{32, 1}); + auto zp_convert = std::make_shared(zp, ov::element::f16); + auto sub = std::make_shared(convert, zp_convert); + auto scale = std::make_shared(ov::element::f16, ov::Shape{32, 1}); + auto mul = std::make_shared(sub, scale); + auto matmul = std::make_shared(data, mul, false, true); + + model = std::make_shared(ov::OutputVector{matmul}, ov::ParameterVector{data, zp, scale}); + manager.register_pass(true); + manager.register_pass(); + } + { + auto data = std::make_shared(ov::element::f16, ov::PartialShape{-1, 16}); + auto weights = ov::opset1::Constant::create(ov::element::u8, ov::Shape{32, 16}, {1}); + auto zp = std::make_shared(ov::element::u8, ov::Shape{32, 1}); + auto scale = std::make_shared(ov::element::f16, ov::Shape{32, 1}); + auto no_bias = std::make_shared(); + auto fc_compressed = + std::make_shared(data, weights, no_bias, scale, zp); + + model_ref = std::make_shared(ov::OutputVector{fc_compressed}, ov::ParameterVector{data, zp, scale}); + } +} + +// Dynamic scale/zp Parameters broadcast the decompressed weights to a dynamic shape, which FullyConnected can't take, +// so the MatMul must be left unchanged instead of failing. +TEST_F(TransformationTestsF, ConvertMatMulToFullyConnectedTest_compressed_u8_weights_dynamic_scale_parameter) { + { + auto data = std::make_shared(ov::element::f16, ov::PartialShape{-1, 16}); + auto weights = ov::opset1::Constant::create(ov::element::u8, ov::Shape{1, 16}, {1}); + auto convert = std::make_shared(weights, ov::element::f16); + auto scale = std::make_shared(ov::element::f16, ov::PartialShape{-1, 1}); + auto mul = std::make_shared(convert, scale); + auto matmul = std::make_shared(data, mul, false, true); + + model = std::make_shared(ov::OutputVector{matmul}, ov::ParameterVector{data, scale}); + manager.register_pass(true); + manager.register_pass(); + } +} + +TEST_F(TransformationTestsF, ConvertMatMulToFullyConnectedTest_compressed_u8_weights_dynamic_zp_parameter) { + { + auto data = std::make_shared(ov::element::f16, ov::PartialShape{-1, 16}); + auto weights = ov::opset1::Constant::create(ov::element::u8, ov::Shape{1, 16}, {1}); + auto convert = std::make_shared(weights, ov::element::f16); + auto zp = std::make_shared(ov::element::u8, ov::PartialShape{-1, 1}); + auto zp_convert = std::make_shared(zp, ov::element::f16); + auto sub = std::make_shared(convert, zp_convert); + auto scale = ov::opset1::Constant::create(ov::element::f16, ov::Shape{1, 1}, {1}); + auto mul = std::make_shared(sub, scale); + auto matmul = std::make_shared(data, mul, false, true); + + model = std::make_shared(ov::OutputVector{matmul}, ov::ParameterVector{data, zp}); + manager.register_pass(true); + manager.register_pass(); + } +} + // Checked blocked cases TEST(TransformationTests, ConvertMatMulToFullyConnectedExceptionTest_sibling_matmul_no_convert) { auto CreateMatMul = [&](bool mat1_transpose_a, bool mat1_transpose_b, From 33a5041db9b8670cf1643cb9e5694aff93a0d3a1 Mon Sep 17 00:00:00 2001 From: Kotomi-Du Date: Fri, 6 Nov 2026 18:16:46 -0800 Subject: [PATCH 07/10] add static gate for reshape --- .../compressed_weights_pattern.hpp | 10 +- .../convert_fc_to_compressed_test.cpp | 86 ++++++------ .../convert_matmul_to_fc_test.cpp | 124 +++++++++++++----- 3 files changed, 147 insertions(+), 73 deletions(-) diff --git a/src/plugins/intel_gpu/src/plugin/transformations/compressed_weights_pattern.hpp b/src/plugins/intel_gpu/src/plugin/transformations/compressed_weights_pattern.hpp index b383d8432a308b..35dc1f365fa9aa 100644 --- a/src/plugins/intel_gpu/src/plugin/transformations/compressed_weights_pattern.hpp +++ b/src/plugins/intel_gpu/src/plugin/transformations/compressed_weights_pattern.hpp @@ -31,10 +31,11 @@ using ov::pass::operator|; \ auto sub_value_const_m = wrap_type();\ auto sub_value_param_m = wrap_type(ov::pass::pattern::has_static_shape());\ - auto sub_value_param_reshape_m = wrap_type({sub_value_param_m, any_input()});\ + auto sub_value_param_reshape_m =\ + wrap_type({sub_value_param_m, any_input()}, ov::pass::pattern::has_static_shape());\ auto sub_value_param_convert_m = wrap_type({sub_value_param_m});\ auto sub_value_param_convert_reshape_m =\ - wrap_type({sub_value_param_convert_m, any_input()});\ + wrap_type({sub_value_param_convert_m, any_input()}, ov::pass::pattern::has_static_shape());\ auto sub_const_m = sub_value_const_m | sub_value_param_m | sub_value_param_reshape_m |\ sub_value_param_convert_reshape_m;\ auto sub_convert_const_m = wrap_type({sub_const_m});\ @@ -44,10 +45,11 @@ using ov::pass::operator|; \ auto mul_value_const_m = wrap_type();\ auto mul_value_param_m = wrap_type(ov::pass::pattern::has_static_shape());\ - auto mul_value_param_reshape_m = wrap_type({mul_value_param_m, any_input()});\ + auto mul_value_param_reshape_m =\ + wrap_type({mul_value_param_m, any_input()}, ov::pass::pattern::has_static_shape());\ auto mul_value_param_convert_m = wrap_type({mul_value_param_m});\ auto mul_value_param_convert_reshape_m =\ - wrap_type({mul_value_param_convert_m, any_input()});\ + wrap_type({mul_value_param_convert_m, any_input()}, ov::pass::pattern::has_static_shape());\ auto mul_value_param_reshape_convert_m =\ wrap_type({mul_value_param_reshape_m});\ auto mul_const_m = mul_value_const_m | mul_value_param_m | mul_value_param_reshape_m |\ diff --git a/src/plugins/intel_gpu/tests/unit/transformations/convert_fc_to_compressed_test.cpp b/src/plugins/intel_gpu/tests/unit/transformations/convert_fc_to_compressed_test.cpp index 30451fbcaa0e68..8652012afbc4e3 100644 --- a/src/plugins/intel_gpu/tests/unit/transformations/convert_fc_to_compressed_test.cpp +++ b/src/plugins/intel_gpu/tests/unit/transformations/convert_fc_to_compressed_test.cpp @@ -83,59 +83,58 @@ TEST_F(TransformationTestsF, ConvertFCToCompressed2) { } } -enum class ParameterScaleTopology { ConvertThenReshape, ReshapeThenConvert }; +enum class ScaleZpTopology { ReshapeOfConvert, ConvertOfReshape }; struct ParameterScaleZpParams { - ParameterScaleTopology scale_topology; + ScaleZpTopology topology; bool with_zp; }; class ConvertFCToCompressedParameterScaleZpTest : public testing::WithParamInterface, public TransformationTestsF { public: + ConvertFCToCompressedParameterScaleZpTest() : TransformationTestsF() { + comparator.enable(FunctionsComparator::ATTRIBUTES); + } + static std::string getTestCaseName(const testing::TestParamInfo& info) { - std::ostringstream result; - result << (info.param.scale_topology == ParameterScaleTopology::ConvertThenReshape ? "ReshapeOfConvert" - : "ConvertOfReshape") - << (info.param.with_zp ? "_WithZp" : "_NoZp"); - return result.str(); + const auto& [topology, with_zp] = info.param; + return std::string(topology == ScaleZpTopology::ReshapeOfConvert ? "ReshapeOfConvert" : "ConvertOfReshape") + + (with_zp ? "_WithZp" : "_NoZp"); } }; -// Scale (and zero point) come from model Parameters [32, 1, 1] reshaped to [32, 1]: -// - ConvertThenReshape: Reshape(Convert(scale)) -// - ReshapeThenConvert: Convert(Reshape(scale)) -// The zero point is Convert(Reshape(zp)) and is passed to FullyConnectedCompressed without the Convert. +// Scale and zero point are model Parameters [32, 1, 1] reshaped to [32, 1], as Reshape(Convert(param)) or +// Convert(Reshape(param)). The pass must create FullyConnectedCompressed that takes the Parameter-based scale/zp. TEST_P(ConvertFCToCompressedParameterScaleZpTest, ConvertsParameterScaleAndZeroPoint) { - const auto& [scale_topology, with_zp] = GetParam(); - const bool convert_then_reshape = scale_topology == ParameterScaleTopology::ConvertThenReshape; + const auto& [topology, with_zp] = GetParam(); + const bool reshape_of_convert = topology == ScaleZpTopology::ReshapeOfConvert; - auto reshape_to_2d = [](const ov::Output& node) { + auto to_f16 = [](const ov::Output& node) -> std::shared_ptr { + return std::make_shared(node, ov::element::f16); + }; + auto reshape_to_2d = [](const ov::Output& node) -> std::shared_ptr { auto shape = ov::op::v0::Constant::create(ov::element::i32, ov::Shape{2}, {32, 1}); return std::make_shared(node, shape, false); }; - auto to_f16 = [](const ov::Output& node) { - return std::make_shared(node, ov::element::f16); - }; - auto build_scale = [&](const std::shared_ptr& scale) -> std::shared_ptr { - return convert_then_reshape ? std::shared_ptr(reshape_to_2d(to_f16(scale))) - : std::shared_ptr(to_f16(reshape_to_2d(scale))); + auto build = [&](const ov::Output& param) { + return reshape_of_convert ? reshape_to_2d(to_f16(param)) : to_f16(reshape_to_2d(param)); }; { auto input = std::make_shared(ov::element::f16, ov::PartialShape{-1, 16}); auto weights = ov::op::v0::Constant::create(ov::element::u8, ov::Shape{32, 16}, {1}); - std::shared_ptr dequantized = to_f16(weights); auto scale = std::make_shared(ov::element::f32, ov::Shape{32, 1, 1}); ov::ParameterVector params{input, scale}; + std::shared_ptr dequantized = to_f16(weights); if (with_zp) { - auto zero_point = std::make_shared(ov::element::u8, ov::Shape{32, 1, 1}); - dequantized = std::make_shared(dequantized, to_f16(reshape_to_2d(zero_point))); - params.push_back(zero_point); + auto zp = std::make_shared(ov::element::u8, ov::Shape{32, 1, 1}); + params.push_back(zp); + dequantized = std::make_shared(dequantized, build(zp)); } - dequantized = std::make_shared(dequantized, build_scale(scale)); + dequantized = std::make_shared(dequantized, build(scale)); auto no_bias = std::make_shared(); - auto fc = std::make_shared(input, dequantized, no_bias); + auto fc = std::make_shared(input, dequantized, no_bias, ov::element::f16, true); model = std::make_shared(ov::OutputVector{fc}, params); manager.register_pass(); @@ -144,24 +143,35 @@ TEST_P(ConvertFCToCompressedParameterScaleZpTest, ConvertsParameterScaleAndZeroP auto input = std::make_shared(ov::element::f16, ov::PartialShape{-1, 16}); auto weights = ov::op::v0::Constant::create(ov::element::u8, ov::Shape{32, 16}, {1}); auto scale = std::make_shared(ov::element::f32, ov::Shape{32, 1, 1}); + ov::ParameterVector params{input, scale}; // Without a zero point, Multiply's optional Convert absorbs a trailing scale Convert, // so Convert(Reshape(scale)) is passed as Reshape(scale) in its original precision. - std::shared_ptr expected_scale = - (!with_zp && !convert_then_reshape) ? std::shared_ptr(reshape_to_2d(scale)) : build_scale(scale); + auto expected_scale = (!with_zp && !reshape_of_convert) ? reshape_to_2d(scale) : build(scale); auto no_bias = std::make_shared(); - ov::ParameterVector params{input, scale}; std::shared_ptr fc_compressed; if (with_zp) { - auto zero_point = std::make_shared(ov::element::u8, ov::Shape{32, 1, 1}); + auto zp = std::make_shared(ov::element::u8, ov::Shape{32, 1, 1}); + params.push_back(zp); + // A u8 zp is passed without its Convert; an f16 Reshape(Convert(zp)) is passed as is. + auto expected_zp = reshape_of_convert ? build(zp) : reshape_to_2d(zp); fc_compressed = std::make_shared(input, weights, no_bias, expected_scale, - reshape_to_2d(zero_point)); - params.push_back(zero_point); + expected_zp, + ov::element::f16, + true, + true, + true); } else { - fc_compressed = - std::make_shared(input, weights, no_bias, expected_scale); + fc_compressed = std::make_shared(input, + weights, + no_bias, + expected_scale, + ov::element::f16, + true, + true, + true); } model_ref = std::make_shared(ov::OutputVector{fc_compressed}, params); @@ -170,10 +180,10 @@ TEST_P(ConvertFCToCompressedParameterScaleZpTest, ConvertsParameterScaleAndZeroP INSTANTIATE_TEST_SUITE_P(TransformationTests, ConvertFCToCompressedParameterScaleZpTest, - testing::Values(ParameterScaleZpParams{ParameterScaleTopology::ConvertThenReshape, true}, - ParameterScaleZpParams{ParameterScaleTopology::ConvertThenReshape, false}, - ParameterScaleZpParams{ParameterScaleTopology::ReshapeThenConvert, true}, - ParameterScaleZpParams{ParameterScaleTopology::ReshapeThenConvert, false}), + testing::Values(ParameterScaleZpParams{ScaleZpTopology::ReshapeOfConvert, true}, + ParameterScaleZpParams{ScaleZpTopology::ReshapeOfConvert, false}, + ParameterScaleZpParams{ScaleZpTopology::ConvertOfReshape, true}, + ParameterScaleZpParams{ScaleZpTopology::ConvertOfReshape, false}), ConvertFCToCompressedParameterScaleZpTest::getTestCaseName); TEST_F(TransformationTestsF, ConvertFCToCompressed3) { diff --git a/src/plugins/intel_gpu/tests/unit/transformations/convert_matmul_to_fc_test.cpp b/src/plugins/intel_gpu/tests/unit/transformations/convert_matmul_to_fc_test.cpp index 0b9a214a192dcd..27a9edcefece63 100644 --- a/src/plugins/intel_gpu/tests/unit/transformations/convert_matmul_to_fc_test.cpp +++ b/src/plugins/intel_gpu/tests/unit/transformations/convert_matmul_to_fc_test.cpp @@ -658,48 +658,110 @@ TEST_F(TransformationTestsF, ConvertMatMulToFullyConnectedTest_compressed_u8_wei auto zp = std::make_shared(ov::element::u8, ov::Shape{32, 1}); auto scale = std::make_shared(ov::element::f16, ov::Shape{32, 1}); auto no_bias = std::make_shared(); - auto fc_compressed = - std::make_shared(data, weights, no_bias, scale, zp); + auto fc_compressed = std::make_shared(data, + weights, + no_bias, + scale, + zp, + ov::element::f16, + true, + true, + true); model_ref = std::make_shared(ov::OutputVector{fc_compressed}, ov::ParameterVector{data, zp, scale}); } + comparator.enable(FunctionsComparator::ATTRIBUTES); } -// Dynamic scale/zp Parameters broadcast the decompressed weights to a dynamic shape, which FullyConnected can't take, -// so the MatMul must be left unchanged instead of failing. -TEST_F(TransformationTestsF, ConvertMatMulToFullyConnectedTest_compressed_u8_weights_dynamic_scale_parameter) { - { - auto data = std::make_shared(ov::element::f16, ov::PartialShape{-1, 16}); - auto weights = ov::opset1::Constant::create(ov::element::u8, ov::Shape{1, 16}, {1}); - auto convert = std::make_shared(weights, ov::element::f16); - auto scale = std::make_shared(ov::element::f16, ov::PartialShape{-1, 1}); - auto mul = std::make_shared(convert, scale); - auto matmul = std::make_shared(data, mul, false, true); - - model = std::make_shared(ov::OutputVector{matmul}, ov::ParameterVector{data, scale}); - manager.register_pass(true); - manager.register_pass(); +enum class DynamicScaleZpInput { Scale, ZeroPoint }; +enum class DynamicScaleZpSource { Parameter, ReshapeOfConvert, ConvertOfReshape }; + +struct DynamicScaleZpParams { + DynamicScaleZpInput input; + DynamicScaleZpSource source; +}; + +class ConvertMatMulToFullyConnectedDynamicScaleZpTest : public testing::WithParamInterface, + public TransformationTestsF { +public: + ConvertMatMulToFullyConnectedDynamicScaleZpTest() : TransformationTestsF() { + comparator.enable(FunctionsComparator::ATTRIBUTES); + } + + static std::string getTestCaseName(const testing::TestParamInfo& info) { + const auto& [input, source] = info.param; + std::string name = input == DynamicScaleZpInput::Scale ? "Scale" : "ZeroPoint"; + switch (source) { + case DynamicScaleZpSource::Parameter: + return name + "_DynamicParameter"; + case DynamicScaleZpSource::ReshapeOfConvert: + return name + "_ReshapeOfConvert_DynamicTarget"; + case DynamicScaleZpSource::ConvertOfReshape: + return name + "_ConvertOfReshape_DynamicTarget"; + } + return name; + } +}; + +// The scale or zero point has a dynamic shape, either as a dynamic Parameter [?, 1] or as a static Parameter reshaped to a +// non-constant target shape. It broadcasts the decompressed weights to a dynamic shape, which FullyConnected can't take, +// so the MatMul must be left unchanged instead of failing on shape_b.is_static() in ConvertMatMulToFullyConnected. +TEST_P(ConvertMatMulToFullyConnectedDynamicScaleZpTest, KeepsMatMul) { + const auto& [input_kind, source] = GetParam(); + const bool is_scale = input_kind == DynamicScaleZpInput::Scale; + + auto data = std::make_shared(ov::element::f16, ov::PartialShape{-1, 16}); + auto weights = ov::opset1::Constant::create(ov::element::u8, ov::Shape{1, 16}, {1}); + std::shared_ptr dequantized = std::make_shared(weights, ov::element::f16); + ov::ParameterVector params{data}; + + const auto param_type = is_scale ? ov::element::f32 : ov::element::u8; + std::shared_ptr value; + if (source == DynamicScaleZpSource::Parameter) { + auto param = std::make_shared(param_type, ov::PartialShape{-1, 1}); + params.push_back(param); + value = std::make_shared(param, ov::element::f16); + } else { + auto param = std::make_shared(param_type, ov::Shape{1, 1, 1}); + auto target_shape = std::make_shared(ov::element::i32, ov::Shape{2}); + params.push_back(param); + params.push_back(target_shape); + if (source == DynamicScaleZpSource::ReshapeOfConvert) { + auto convert = std::make_shared(param, ov::element::f16); + value = std::make_shared(convert, target_shape, false); + } else { + auto reshape = std::make_shared(param, target_shape, false); + value = std::make_shared(reshape, ov::element::f16); + } } -} -TEST_F(TransformationTestsF, ConvertMatMulToFullyConnectedTest_compressed_u8_weights_dynamic_zp_parameter) { - { - auto data = std::make_shared(ov::element::f16, ov::PartialShape{-1, 16}); - auto weights = ov::opset1::Constant::create(ov::element::u8, ov::Shape{1, 16}, {1}); - auto convert = std::make_shared(weights, ov::element::f16); - auto zp = std::make_shared(ov::element::u8, ov::PartialShape{-1, 1}); - auto zp_convert = std::make_shared(zp, ov::element::f16); - auto sub = std::make_shared(convert, zp_convert); + if (is_scale) { + dequantized = std::make_shared(dequantized, value); + } else { + dequantized = std::make_shared(dequantized, value); auto scale = ov::opset1::Constant::create(ov::element::f16, ov::Shape{1, 1}, {1}); - auto mul = std::make_shared(sub, scale); - auto matmul = std::make_shared(data, mul, false, true); - - model = std::make_shared(ov::OutputVector{matmul}, ov::ParameterVector{data, zp}); - manager.register_pass(true); - manager.register_pass(); + dequantized = std::make_shared(dequantized, scale); } + ASSERT_TRUE(dequantized->get_output_partial_shape(0).is_dynamic()); + auto matmul = std::make_shared(data, dequantized, false, true); + + // No model_ref: the model is expected to be unchanged. + model = std::make_shared(ov::OutputVector{matmul}, params); + manager.register_pass(true); + manager.register_pass(); } +INSTANTIATE_TEST_SUITE_P( + TransformationTests, + ConvertMatMulToFullyConnectedDynamicScaleZpTest, + testing::Values(DynamicScaleZpParams{DynamicScaleZpInput::Scale, DynamicScaleZpSource::Parameter}, + DynamicScaleZpParams{DynamicScaleZpInput::Scale, DynamicScaleZpSource::ReshapeOfConvert}, + DynamicScaleZpParams{DynamicScaleZpInput::Scale, DynamicScaleZpSource::ConvertOfReshape}, + DynamicScaleZpParams{DynamicScaleZpInput::ZeroPoint, DynamicScaleZpSource::Parameter}, + DynamicScaleZpParams{DynamicScaleZpInput::ZeroPoint, DynamicScaleZpSource::ReshapeOfConvert}, + DynamicScaleZpParams{DynamicScaleZpInput::ZeroPoint, DynamicScaleZpSource::ConvertOfReshape}), + ConvertMatMulToFullyConnectedDynamicScaleZpTest::getTestCaseName); + // Checked blocked cases TEST(TransformationTests, ConvertMatMulToFullyConnectedExceptionTest_sibling_matmul_no_convert) { auto CreateMatMul = [&](bool mat1_transpose_a, bool mat1_transpose_b, From 7633546f71d1601d2d523e7184398e14511e269c Mon Sep 17 00:00:00 2001 From: Kotomi-Du Date: Fri, 6 Nov 2026 18:25:10 -0800 Subject: [PATCH 08/10] make sure runtime info is captured for scale parameter --- .../convert_fc_to_compressed.cpp | 8 +++- .../convert_fc_to_compressed_test.cpp | 43 +++++++++++++++++++ 2 files changed, 49 insertions(+), 2 deletions(-) diff --git a/src/plugins/intel_gpu/src/plugin/transformations/convert_fc_to_compressed.cpp b/src/plugins/intel_gpu/src/plugin/transformations/convert_fc_to_compressed.cpp index d156e604ac121f..c363b2443d5f07 100644 --- a/src/plugins/intel_gpu/src/plugin/transformations/convert_fc_to_compressed.cpp +++ b/src/plugins/intel_gpu/src/plugin/transformations/convert_fc_to_compressed.cpp @@ -94,7 +94,8 @@ ConvertFullyConnectedToFullyConnectedCompressed::ConvertFullyConnectedToFullyCon weight_u8 = true; } - auto reshape_input = [has_transpose, grouped, is_weight_3d](std::shared_ptr node) { + std::vector> result_nodes = {}; + auto reshape_input = [has_transpose, grouped, is_weight_3d, &result_nodes](std::shared_ptr node) { const auto& partial_shape = node->get_output_partial_shape(0); OPENVINO_ASSERT(partial_shape.is_static()); const ov::Shape current_shape = partial_shape.to_shape(); @@ -130,6 +131,8 @@ ConvertFullyConnectedToFullyConnectedCompressed::ConvertFullyConnectedToFullyCon auto reshape = std::make_shared(node, shape_const, false); reshape->set_friendly_name(node->get_friendly_name() + "_reshape"); ov::copy_runtime_info(node, {shape_const, reshape}); + result_nodes.push_back(shape_const); + result_nodes.push_back(reshape); return std::static_pointer_cast(reshape); }; @@ -185,7 +188,6 @@ ConvertFullyConnectedToFullyConnectedCompressed::ConvertFullyConnectedToFullyCon bool transpose_b_scale = fc->get_transpose_b(); bool transpose_b_zp = fc->get_transpose_b(); std::shared_ptr fc_input_bias = pattern_map.at(bias_m).get_node_shared_ptr(); - std::vector> result_nodes = {}; if (with_zero_point && ov::is_type(optional_zero_point)) { result_nodes.push_back(optional_zero_point); } @@ -265,6 +267,8 @@ ConvertFullyConnectedToFullyConnectedCompressed::ConvertFullyConnectedToFullyCon if (pattern_map.count(mul2_m)) { auto mul2_op_const = ov::as_type_ptr(pattern_map.at(mul2_const_m).get_node_shared_ptr()); fc_input_scale = ov::op::util::make_try_fold(fc_input_scale, mul2_op_const); + // With a Parameter scale this can't fold and stays an executable Multiply, so it needs runtime info. + result_nodes.push_back(fc_input_scale); } std::shared_ptr new_fc = nullptr; diff --git a/src/plugins/intel_gpu/tests/unit/transformations/convert_fc_to_compressed_test.cpp b/src/plugins/intel_gpu/tests/unit/transformations/convert_fc_to_compressed_test.cpp index 8652012afbc4e3..8ddfc892274ac6 100644 --- a/src/plugins/intel_gpu/tests/unit/transformations/convert_fc_to_compressed_test.cpp +++ b/src/plugins/intel_gpu/tests/unit/transformations/convert_fc_to_compressed_test.cpp @@ -186,6 +186,49 @@ INSTANTIATE_TEST_SUITE_P(TransformationTests, ParameterScaleZpParams{ScaleZpTopology::ConvertOfReshape, false}), ConvertFCToCompressedParameterScaleZpTest::getTestCaseName); +// Grouped weights with a Parameter scale followed by the extra post-reshape Multiply (mul2_m tail). +// w: Constant u8 [8,4,16] → Convert f16 ─┐ +// scale: Parameter f16 [8,4,1] ──────────┴→ Multiply → Reshape [8,64] → Multiply(·, Constant [8,1]) → FullyConnected +TEST_F(TransformationTestsF, ConvertFCToCompressedParameterScaleWithExtraMultiply) { + comparator.enable(FunctionsComparator::ATTRIBUTES); + { + auto input = std::make_shared(ov::element::f16, ov::PartialShape{-1, 64}); + auto weights = ov::op::v0::Constant::create(ov::element::u8, ov::Shape{8, 4, 16}, {1}); + auto convert = std::make_shared(weights, ov::element::f16); + auto scale = std::make_shared(ov::element::f16, ov::Shape{8, 4, 1}); + auto mul = std::make_shared(convert, scale); + auto reshape_const = ov::op::v0::Constant::create(ov::element::i32, ov::Shape{2}, {8, 64}); + auto reshape = std::make_shared(mul, reshape_const, false); + auto extra_mul_const = ov::op::v0::Constant::create(ov::element::f16, ov::Shape{8, 1}, {2}); + auto extra_mul = std::make_shared(reshape, extra_mul_const); + auto no_bias = std::make_shared(); + auto fc = std::make_shared(input, extra_mul, no_bias, ov::element::f16, true); + + model = std::make_shared(ov::OutputVector{fc}, ov::ParameterVector{input, scale}); + manager.register_pass(); + } + { + auto input = std::make_shared(ov::element::f16, ov::PartialShape{-1, 64}); + auto weights = ov::op::v0::Constant::create(ov::element::u8, ov::Shape{8, 64}, {1}); + auto scale = std::make_shared(ov::element::f16, ov::Shape{8, 4, 1}); + auto scale_shape = ov::op::v0::Constant::create(ov::element::i32, ov::Shape{2}, {8, 4}); + auto scale_reshape = std::make_shared(scale, scale_shape, false); + auto extra_mul_const = ov::op::v0::Constant::create(ov::element::f16, ov::Shape{8, 1}, {2}); + auto scaled = std::make_shared(scale_reshape, extra_mul_const); + auto no_bias = std::make_shared(); + auto fc_compressed = std::make_shared(input, + weights, + no_bias, + scaled, + ov::element::f16, + true, + true, + true); + + model_ref = std::make_shared(ov::OutputVector{fc_compressed}, ov::ParameterVector{input, scale}); + } +} + TEST_F(TransformationTestsF, ConvertFCToCompressed3) { { auto input1 = std::make_shared(ov::element::f32, ov::PartialShape{ -1, 16 }); From 5c63d403aab864d8c51c642c473bfed01e59a56f Mon Sep 17 00:00:00 2001 From: Kotomi-Du Date: Fri, 6 Nov 2026 19:03:07 -0800 Subject: [PATCH 09/10] refine code --- .../impls/onednn/fully_connected_onednn.hpp | 12 ++++++++---- .../convert_fc_to_compressed.cpp | 6 ------ .../convert_fc_to_compressed_test.cpp | 17 ++++++++++------- 3 files changed, 18 insertions(+), 17 deletions(-) diff --git a/src/plugins/intel_gpu/src/graph/impls/onednn/fully_connected_onednn.hpp b/src/plugins/intel_gpu/src/graph/impls/onednn/fully_connected_onednn.hpp index 03b8679e691271..067150106ae04d 100644 --- a/src/plugins/intel_gpu/src/graph/impls/onednn/fully_connected_onednn.hpp +++ b/src/plugins/intel_gpu/src/graph/impls/onednn/fully_connected_onednn.hpp @@ -157,10 +157,14 @@ struct FullyConnectedImplementationManager : public ImplementationManager { ((fc_prim->decompression_scale.is_valid() && idx == scale_idx) || (fc_prim->decompression_zero_point.is_valid() && idx == zp_idx)); if (is_decompression_parameter) { - in_fmts[idx] = node.get_input_layout(idx).format; - GPU_DEBUG_LOG << node.id() << " : Keep scale/zp parameter format " - << "at index " << idx << std::endl; - continue; + const auto& in_layout = node.get_input_layout(idx); + // oneDNN binds scale/zp as a flat format_tag::a buffer, so only plain layouts can be passed through. + // Keep plain permutations (e.g. fbyx from prepare_quantization); reorder blocked layouts to plain. + in_fmts[idx] = format::is_blocked(in_layout.format) ? format::get_default_format(in_layout.get_rank()) + : in_layout.format.value; + GPU_DEBUG_LOG << node.id() << " : " << (format::is_blocked(in_layout.format) ? "Reorder blocked" : "Keep plain") + << " scale/zp format at index " << idx << std::endl; + continue; } auto target_format = format::get_default_format(out_rank); diff --git a/src/plugins/intel_gpu/src/plugin/transformations/convert_fc_to_compressed.cpp b/src/plugins/intel_gpu/src/plugin/transformations/convert_fc_to_compressed.cpp index c363b2443d5f07..06acdcdc4bc695 100644 --- a/src/plugins/intel_gpu/src/plugin/transformations/convert_fc_to_compressed.cpp +++ b/src/plugins/intel_gpu/src/plugin/transformations/convert_fc_to_compressed.cpp @@ -49,14 +49,8 @@ ConvertFullyConnectedToFullyConnectedCompressed::ConvertFullyConnectedToFullyCon const bool has_transpose_before_reshape = pattern_map.count(transpose_before_reshape_m) != 0u; const bool has_transpose = pattern_map.count(transpose_m) != 0u; const auto scale_output = pattern_map.at(mul_const_m); - if (!scale_output.get_partial_shape().is_static()) { - return false; - } auto scale_shape = scale_output.get_shape(); const bool with_zero_point = pattern_map.count(subtract_m) > 0; - if (with_zero_point && !pattern_map.at(sub_const_m).get_partial_shape().is_static()) { - return false; - } bool sub_with_convert = pattern_map.count(sub_convert_const_m) > 0; if (!fc->get_input_partial_shape(1).is_static()) { diff --git a/src/plugins/intel_gpu/tests/unit/transformations/convert_fc_to_compressed_test.cpp b/src/plugins/intel_gpu/tests/unit/transformations/convert_fc_to_compressed_test.cpp index 8ddfc892274ac6..daf61a790b97f8 100644 --- a/src/plugins/intel_gpu/tests/unit/transformations/convert_fc_to_compressed_test.cpp +++ b/src/plugins/intel_gpu/tests/unit/transformations/convert_fc_to_compressed_test.cpp @@ -187,13 +187,13 @@ INSTANTIATE_TEST_SUITE_P(TransformationTests, ConvertFCToCompressedParameterScaleZpTest::getTestCaseName); // Grouped weights with a Parameter scale followed by the extra post-reshape Multiply (mul2_m tail). -// w: Constant u8 [8,4,16] → Convert f16 ─┐ +// w: Parameter u8 [8,4,16] → Convert f16 ─┐ // scale: Parameter f16 [8,4,1] ──────────┴→ Multiply → Reshape [8,64] → Multiply(·, Constant [8,1]) → FullyConnected -TEST_F(TransformationTestsF, ConvertFCToCompressedParameterScaleWithExtraMultiply) { +TEST_F(TransformationTestsF, ConvertFCToCompressedParameterWeightsAndScaleWithExtraMultiply) { comparator.enable(FunctionsComparator::ATTRIBUTES); { auto input = std::make_shared(ov::element::f16, ov::PartialShape{-1, 64}); - auto weights = ov::op::v0::Constant::create(ov::element::u8, ov::Shape{8, 4, 16}, {1}); + auto weights = std::make_shared(ov::element::u8, ov::Shape{8, 4, 16}); auto convert = std::make_shared(weights, ov::element::f16); auto scale = std::make_shared(ov::element::f16, ov::Shape{8, 4, 1}); auto mul = std::make_shared(convert, scale); @@ -204,12 +204,15 @@ TEST_F(TransformationTestsF, ConvertFCToCompressedParameterScaleWithExtraMultipl auto no_bias = std::make_shared(); auto fc = std::make_shared(input, extra_mul, no_bias, ov::element::f16, true); - model = std::make_shared(ov::OutputVector{fc}, ov::ParameterVector{input, scale}); + model = std::make_shared(ov::OutputVector{fc}, ov::ParameterVector{input, weights, scale}); manager.register_pass(); } { auto input = std::make_shared(ov::element::f16, ov::PartialShape{-1, 64}); - auto weights = ov::op::v0::Constant::create(ov::element::u8, ov::Shape{8, 64}, {1}); + auto weights = std::make_shared(ov::element::u8, ov::Shape{8, 4, 16}); + // Non-constant weights can't be folded; the [8,4,16] Parameter is reshaped to the FC weights shape [8,64]. + auto weights_shape = ov::op::v0::Constant::create(ov::element::i32, ov::Shape{2}, {8, 64}); + auto weights_reshape = std::make_shared(weights, weights_shape, false); auto scale = std::make_shared(ov::element::f16, ov::Shape{8, 4, 1}); auto scale_shape = ov::op::v0::Constant::create(ov::element::i32, ov::Shape{2}, {8, 4}); auto scale_reshape = std::make_shared(scale, scale_shape, false); @@ -217,7 +220,7 @@ TEST_F(TransformationTestsF, ConvertFCToCompressedParameterScaleWithExtraMultipl auto scaled = std::make_shared(scale_reshape, extra_mul_const); auto no_bias = std::make_shared(); auto fc_compressed = std::make_shared(input, - weights, + weights_reshape, no_bias, scaled, ov::element::f16, @@ -225,7 +228,7 @@ TEST_F(TransformationTestsF, ConvertFCToCompressedParameterScaleWithExtraMultipl true, true); - model_ref = std::make_shared(ov::OutputVector{fc_compressed}, ov::ParameterVector{input, scale}); + model_ref = std::make_shared(ov::OutputVector{fc_compressed}, ov::ParameterVector{input, weights, scale}); } } From 08cd784e524fafd733810ae3f64d9130fa4a0a4e Mon Sep 17 00:00:00 2001 From: Kotomi-Du Date: Sat, 7 Nov 2026 17:36:44 -0800 Subject: [PATCH 10/10] fix clang issue --- .../graph/impls/onednn/fully_connected_onednn.hpp | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/src/plugins/intel_gpu/src/graph/impls/onednn/fully_connected_onednn.hpp b/src/plugins/intel_gpu/src/graph/impls/onednn/fully_connected_onednn.hpp index 067150106ae04d..915c8312c2100e 100644 --- a/src/plugins/intel_gpu/src/graph/impls/onednn/fully_connected_onednn.hpp +++ b/src/plugins/intel_gpu/src/graph/impls/onednn/fully_connected_onednn.hpp @@ -147,23 +147,21 @@ struct FullyConnectedImplementationManager : public ImplementationManager { std::vector out_fmts(node.get_outputs_count(), format::any); size_t out_rank = node.get_output_layout().get_rank(); - for (size_t idx = 0 ; idx < node.get_dependencies().size() ; idx++) { + for (size_t idx = 0; idx < node.get_dependencies().size(); idx++) { if (node.get_dependency(idx).is_constant()) { continue; } const size_t scale_idx = fc_prim->bias.is_valid() ? 3 : 2; const size_t zp_idx = scale_idx + 1; const bool is_decompression_parameter = - ((fc_prim->decompression_scale.is_valid() && idx == scale_idx) || - (fc_prim->decompression_zero_point.is_valid() && idx == zp_idx)); + ((fc_prim->decompression_scale.is_valid() && idx == scale_idx) || (fc_prim->decompression_zero_point.is_valid() && idx == zp_idx)); if (is_decompression_parameter) { const auto& in_layout = node.get_input_layout(idx); // oneDNN binds scale/zp as a flat format_tag::a buffer, so only plain layouts can be passed through. // Keep plain permutations (e.g. fbyx from prepare_quantization); reorder blocked layouts to plain. - in_fmts[idx] = format::is_blocked(in_layout.format) ? format::get_default_format(in_layout.get_rank()) - : in_layout.format.value; - GPU_DEBUG_LOG << node.id() << " : " << (format::is_blocked(in_layout.format) ? "Reorder blocked" : "Keep plain") - << " scale/zp format at index " << idx << std::endl; + in_fmts[idx] = format::is_blocked(in_layout.format) ? format::get_default_format(in_layout.get_rank()) : in_layout.format.value; + GPU_DEBUG_LOG << node.id() << " : " << (format::is_blocked(in_layout.format) ? "Reorder blocked" : "Keep plain") << " scale/zp format at index " + << idx << std::endl; continue; }