[GPU] support scale/zp as input

src/plugins/intel_gpu/src/graph/impls/onednn/fully_connected_onednn.hpp

@@ -5,6 +5,7 @@
 #include "fully_connected_inst.h"
 #include "impls/onednn/utils.hpp"
 #include "intel_gpu/primitives/fully_connected.hpp"
+#include "intel_gpu/primitives/input_layout.hpp"
 #include "intel_gpu/runtime/utils.hpp"
 #include "registry/implementation_manager.hpp"
 #include "intel_gpu/runtime/debug_configuration.hpp"
@@ -140,6 +141,8 @@ struct FullyConnectedImplementationManager : public ImplementationManager {
 
     in_out_fmts_t query_formats(const program_node& node) const override {
         assert(node.is_type<fully_connected>());
+        const auto& fc_node = node.as<fully_connected>();
+        const auto& fc_prim = fc_node.get_primitive();
         std::vector<format::type> in_fmts(node.get_dependencies().size(), format::any);
         std::vector<format::type> out_fmts(node.get_outputs_count(), format::any);
 
@@ -148,6 +151,21 @@ struct FullyConnectedImplementationManager : public ImplementationManager {
             if (node.get_dependency(idx).is_constant()) {
                 continue;
             }
+            const size_t scale_idx = fc_prim->bias.is_valid() ? 3 : 2;
+            const size_t zp_idx = scale_idx + 1;
+            const bool is_decompression_parameter =
+                 ((fc_prim->decompression_scale.is_valid() && idx == scale_idx) ||
+                  (fc_prim->decompression_zero_point.is_valid() && idx == zp_idx));
+            if (is_decompression_parameter) {
+                const auto& in_layout = node.get_input_layout(idx);
+                // oneDNN binds scale/zp as a flat format_tag::a buffer, so only plain layouts can be passed through.
+                // Keep plain permutations (e.g. fbyx from prepare_quantization); reorder blocked layouts to plain.
+                in_fmts[idx] = format::is_blocked(in_layout.format) ? format::get_default_format(in_layout.get_rank())
+                                                                    : in_layout.format.value;
+                GPU_DEBUG_LOG << node.id() << " : " << (format::is_blocked(in_layout.format) ? "Reorder blocked" : "Keep plain")
+                              << " scale/zp format at index " << idx << std::endl;
+                continue;
+            }
 
             auto target_format = format::get_default_format(out_rank);
 

src/plugins/intel_gpu/src/plugin/transformations/compressed_weights_pattern.hpp

@@ -29,13 +29,31 @@ using ov::pass::operator|;
         auto weights_convert_reshape_m = wrap_type<ov::op::v1::Reshape>({weights_param_convert_m, any_input()});\
         auto decompressed_weights_m = convert_m | weights_convert_reshape_m;\
 \
-        auto sub_const_m = wrap_type<ov::op::v0::Constant>();\
+        auto sub_value_const_m = wrap_type<ov::op::v0::Constant>();\
+        auto sub_value_param_m = wrap_type<ov::op::v0::Parameter>(ov::pass::pattern::has_static_shape());\
+        auto sub_value_param_reshape_m =\
+            wrap_type<ov::op::v1::Reshape>({sub_value_param_m, any_input()}, ov::pass::pattern::has_static_shape());\
+        auto sub_value_param_convert_m = wrap_type<ov::op::v0::Convert>({sub_value_param_m});\
+        auto sub_value_param_convert_reshape_m =\
+            wrap_type<ov::op::v1::Reshape>({sub_value_param_convert_m, any_input()}, ov::pass::pattern::has_static_shape());\
+        auto sub_const_m = sub_value_const_m | sub_value_param_m | sub_value_param_reshape_m |\
+                           sub_value_param_convert_reshape_m;\
         auto sub_convert_const_m = wrap_type<ov::op::v0::Convert>({sub_const_m});\
         auto sub_with_convert_m = wrap_type<ov::op::v1::Subtract>({decompressed_weights_m, sub_convert_const_m});\
         auto sub_no_convert_m = wrap_type<ov::op::v1::Subtract>({decompressed_weights_m, sub_const_m});\
         auto subtract_m = sub_with_convert_m | sub_no_convert_m;\
 \
-        auto mul_const_m = wrap_type<ov::op::v0::Constant>();\
+        auto mul_value_const_m = wrap_type<ov::op::v0::Constant>();\
+        auto mul_value_param_m = wrap_type<ov::op::v0::Parameter>(ov::pass::pattern::has_static_shape());\
+        auto mul_value_param_reshape_m =\
+            wrap_type<ov::op::v1::Reshape>({mul_value_param_m, any_input()}, ov::pass::pattern::has_static_shape());\
+        auto mul_value_param_convert_m = wrap_type<ov::op::v0::Convert>({mul_value_param_m});\
+        auto mul_value_param_convert_reshape_m =\
+            wrap_type<ov::op::v1::Reshape>({mul_value_param_convert_m, any_input()}, ov::pass::pattern::has_static_shape());\
+        auto mul_value_param_reshape_convert_m =\
+             wrap_type<ov::op::v0::Convert>({mul_value_param_reshape_m});\
+        auto mul_const_m = mul_value_const_m | mul_value_param_m | mul_value_param_reshape_m |\
+                           mul_value_param_convert_reshape_m | mul_value_param_reshape_convert_m;\
         auto mul_with_sub_m = wrap_type<ov::op::v1::Multiply>({subtract_m, mul_const_m});\
         auto mul_const_convert_m = ov::pass::pattern::optional<ov::op::v0::Convert>(mul_const_m);\
         auto mul_no_sub_m = wrap_type<ov::op::v1::Multiply>({decompressed_weights_m, mul_const_convert_m});\

src/plugins/intel_gpu/src/plugin/transformations/convert_fc_to_compressed.cpp

@@ -48,7 +48,9 @@ ConvertFullyConnectedToFullyConnectedCompressed::ConvertFullyConnectedToFullyCon
         }
         const bool has_transpose_before_reshape = pattern_map.count(transpose_before_reshape_m) != 0u;
         const bool has_transpose = pattern_map.count(transpose_m) != 0u;
-        auto scale_shape = pattern_map.at(mul_const_m).get_shape();
+        const auto scale_output = pattern_map.at(mul_const_m);
+        auto scale_shape = scale_output.get_shape();
+        const bool with_zero_point = pattern_map.count(subtract_m) > 0;
         bool sub_with_convert = pattern_map.count(sub_convert_const_m) > 0;
 
         if (!fc->get_input_partial_shape(1).is_static()) {
@@ -86,18 +88,19 @@ ConvertFullyConnectedToFullyConnectedCompressed::ConvertFullyConnectedToFullyCon
             weight_u8 = true;
         }
 
-        auto reshape_const = [has_transpose, grouped, is_weight_3d](std::shared_ptr<ov::Node> node) {
-            auto constant = ov::as_type_ptr<ov::op::v0::Constant>(node);
-            OPENVINO_ASSERT(constant != nullptr);
-            ov::Shape current_shape = constant->get_shape();
+        std::vector<std::shared_ptr<ov::Node>> result_nodes = {};
+        auto reshape_input = [has_transpose, grouped, is_weight_3d, &result_nodes](std::shared_ptr<ov::Node> node) {
+            const auto& partial_shape = node->get_output_partial_shape(0);
+            OPENVINO_ASSERT(partial_shape.is_static());
+            const ov::Shape current_shape = partial_shape.to_shape();
             if (current_shape.size() <= 2) {
-                return constant;
+                return node;
             }
 
             ov::Shape new_shape;
             if (current_shape.size() == 3) {
                 if (is_weight_3d) {
-                    return constant;
+                    return node;
                 }
                 new_shape = (has_transpose || !grouped) ? ov::Shape{current_shape[0] * current_shape[1], current_shape[2]}
                                                         : ov::Shape{current_shape[0], current_shape[1] * current_shape[2]};
@@ -110,52 +113,66 @@ ConvertFullyConnectedToFullyConnectedCompressed::ConvertFullyConnectedToFullyCon
             } else {
                 OPENVINO_THROW("Unexpected constant shape rank ", current_shape.size(), " with is_weight_3d=", is_weight_3d);
             }
-            auto new_constant = std::make_shared<ov::op::v0::Constant>(*constant, new_shape);
-
-            ov::copy_weightless_cache_attr(constant, new_constant);
-            return new_constant;
+            if (auto constant = ov::as_type_ptr<ov::op::v0::Constant>(node)) {
+                auto new_constant = std::make_shared<ov::op::v0::Constant>(*constant, new_shape);
+                ov::copy_weightless_cache_attr(constant, new_constant);
+                return std::static_pointer_cast<ov::Node>(new_constant);
+            }
+            auto shape_const = std::make_shared<ov::op::v0::Constant>(
+                ov::element::i32,
+                ov::Shape{new_shape.size()},
+                std::vector<int64_t>(new_shape.begin(), new_shape.end()));
+            auto reshape = std::make_shared<ov::op::v1::Reshape>(node, shape_const, false);
+            reshape->set_friendly_name(node->get_friendly_name() + "_reshape");
+            ov::copy_runtime_info(node, {shape_const, reshape});
+            result_nodes.push_back(shape_const);
+            result_nodes.push_back(reshape);
+            return std::static_pointer_cast<ov::Node>(reshape);
         };
 
         auto convert_zp_const = [&](std::shared_ptr<ov::Node> node) {
             auto constant = ov::as_type_ptr<ov::op::v0::Constant>(node);
-            std::shared_ptr<ov::Node> result = nullptr;
+            const auto zp_type = node->get_element_type();
+            std::shared_ptr<ov::Node> result = node;
             // Convert ZP to u8 (or i8 for a scalar i4 ZP)
-            if (constant->get_element_type() == ov::element::u8) {
-                result = constant;
-            } else if (constant->get_element_type() == ov::element::u4 || constant->get_element_type() == ov::element::u3 ||
-                       constant->get_element_type() == ov::element::u2) {
+            if (zp_type == ov::element::u8) {
+                result = node;
+            } else if (zp_type == ov::element::u4 || zp_type == ov::element::u3 || zp_type == ov::element::u2) {
                 result = std::make_shared<ov::op::v0::Convert>(node, ov::element::u8);
                 // Only unsigned ZP types can be converted to u8.
-            } else if (weight_u8 && sub_with_convert && !constant->get_element_type().is_signed()) {
+            } else if (weight_u8 && sub_with_convert && !zp_type.is_signed()) {
                 result = std::make_shared<ov::op::v0::Convert>(node, ov::element::u8);
-            } else if (constant->get_element_type() == ov::element::i4 && ov::shape_size(constant->get_shape()) == 1) {
+            } else if (zp_type == ov::element::i4 && node->get_output_partial_shape(0).is_static() &&
+                       ov::shape_size(node->get_output_shape(0)) == 1) {
                 // oneDNN doesn't support sub-byte per-tensor (common) zero points, so a scalar i4 ZP is converted to i8.
                 result = std::make_shared<ov::op::v0::Convert>(node, ov::element::i8);
-            } else {
-                result = constant;
             }
 
-            ov::copy_weightless_cache_attr(node, result);
+            if (result != node) {
+                ov::copy_runtime_info(node, result);
+                if (constant) {
+                    ov::copy_weightless_cache_attr(constant, result);
+                }
+            }
             return result;
         };
 
         const ov::Output<Node>& fc_input_a = fc->input(0).get_source_output();
         const auto& scale = has_transpose_before_reshape
                                 ? pattern_map.at(mul_const_m).get_node_shared_ptr()
-                                : reshape_const(pattern_map.at(mul_const_m).get_node_shared_ptr());
+                                : reshape_input(pattern_map.at(mul_const_m).get_node_shared_ptr());
         std::shared_ptr<ov::Node> optional_zero_point = nullptr;
 
-        const bool with_zero_point = pattern_map.count(subtract_m) > 0;
         if (with_zero_point) {
             const auto zero_point = has_transpose_before_reshape
                                         ? pattern_map.at(sub_const_m).get_node_shared_ptr()
-                                        : reshape_const(pattern_map.at(sub_const_m).get_node_shared_ptr());
+                                        : reshape_input(pattern_map.at(sub_const_m).get_node_shared_ptr());
             optional_zero_point = convert_zp_const(zero_point);
         }
 
         std::shared_ptr<ov::Node> fc_input_b = weight_ptr;
         if (weights_are_constant && !has_transpose_before_reshape) {
-            fc_input_b = reshape_const(weight_ptr);
+            fc_input_b = reshape_input(weight_ptr);
         } else if (pattern_map.count(weights_reshape_m)) {
             fc_input_b = pattern_map.at(weights_reshape_m).get_node_shared_ptr();
         }
@@ -165,7 +182,6 @@ ConvertFullyConnectedToFullyConnectedCompressed::ConvertFullyConnectedToFullyCon
         bool transpose_b_scale = fc->get_transpose_b();
         bool transpose_b_zp = fc->get_transpose_b();
         std::shared_ptr<ov::Node> fc_input_bias = pattern_map.at(bias_m).get_node_shared_ptr();
-        std::vector<std::shared_ptr<ov::Node>> result_nodes = {};
         if (with_zero_point && ov::is_type<ov::op::v0::Convert>(optional_zero_point)) {
             result_nodes.push_back(optional_zero_point);
         }
@@ -230,6 +246,7 @@ ConvertFullyConnectedToFullyConnectedCompressed::ConvertFullyConnectedToFullyCon
                                                                 ov::Shape{output_shape.size()},
                                                                 output_shape);
                 auto reshaped = std::make_shared<ov::op::v1::Reshape>(node, shape_const, false);
+                ov::copy_runtime_info(node, {shape_const, reshaped});
                 result_nodes.push_back(shape_const);
                 result_nodes.push_back(reshaped);
                 return std::static_pointer_cast<ov::Node>(reshaped);
@@ -244,6 +261,8 @@ ConvertFullyConnectedToFullyConnectedCompressed::ConvertFullyConnectedToFullyCon
         if (pattern_map.count(mul2_m)) {
             auto mul2_op_const = ov::as_type_ptr<ov::op::v0::Constant>(pattern_map.at(mul2_const_m).get_node_shared_ptr());
             fc_input_scale = ov::op::util::make_try_fold<ov::op::v1::Multiply>(fc_input_scale, mul2_op_const);
+            // With a Parameter scale this can't fold and stays an executable Multiply, so it needs runtime info.
+            result_nodes.push_back(fc_input_scale);
         }
 
         std::shared_ptr<ov::Node> new_fc = nullptr;

src/plugins/intel_gpu/tests/unit/test_cases/fully_connected_gpu_test.cpp

@@ -3275,6 +3275,102 @@ void test_compressed_int4_scale_dynamic_batch_gemv(bool is_caching_test,
         }
     }
 
+    void test_compressed_int8_parameter_scale_zp(bool is_per_group, bool transpose_scale_zp) {
+        auto& engine = get_test_engine();
+
+        constexpr int64_t ofm = 4;
+        constexpr int64_t ifm = 32;
+        const int64_t groups = is_per_group ? 2 : 1;
+        const int64_t group_size = ifm / groups;
+
+        // Non-transposed scale/zp are [groups, N]; transposed scale/zp are [N, groups].
+        const ov::PartialShape param_shape = transpose_scale_zp ? ov::PartialShape{ofm, groups} : ov::PartialShape{groups, ofm};
+        auto param_offset = [&](int64_t group, int64_t output) {
+            return static_cast<size_t>(transpose_scale_zp ? output * groups + group : group * ofm + output);
+        };
+
+        auto input_mem = engine.allocate_memory({ {1, 2, ifm}, data_types::f16, format::bfyx });
+        auto weights_mem = engine.allocate_memory({ {ofm, ifm}, data_types::u8, format::bfyx });
+        auto scale_mem = engine.allocate_memory({ param_shape, data_types::f16, format::bfyx });
+        auto zp_mem = engine.allocate_memory({ param_shape, data_types::u8, format::bfyx });
+
+        std::vector<float> input_data(2 * ifm);
+        for (size_t i = 0; i < input_data.size(); ++i) {
+            input_data[i] = static_cast<float>(static_cast<int64_t>(i % 3) - 1);
+        }
+        std::vector<uint8_t> weights_data(ofm * ifm);
+        for (int64_t output = 0; output < ofm; ++output) {
+            for (int64_t k = 0; k < ifm; ++k) {
+                weights_data[output * ifm + k] = static_cast<uint8_t>((output * 3 + k) % 8);
+            }
+        }
+        std::vector<float> scale_data(ofm * groups);
+        std::vector<uint8_t> zp_data(ofm * groups);
+        for (int64_t group = 0; group < groups; ++group) {
+            for (int64_t output = 0; output < ofm; ++output) {
+                scale_data[param_offset(group, output)] = 0.5f * static_cast<float>((group + output) % 2 + 1);
+                zp_data[param_offset(group, output)] = static_cast<uint8_t>((group + output) % 3);
+            }
+        }
+
+        std::vector<ov::float16> input_f16(input_data.begin(), input_data.end());
+        std::vector<ov::float16> scale_f16(scale_data.begin(), scale_data.end());
+        set_values(input_mem, input_f16);
+        set_values(weights_mem, weights_data);
+        set_values(scale_mem, scale_f16);
+        set_values(zp_mem, zp_data);
+
+        topology topology(
+            input_layout("input", input_mem->get_layout()),
+            data("weights", weights_mem),
+            input_layout("scale", scale_mem->get_layout()),
+            input_layout("zp", zp_mem->get_layout()),
+            fully_connected("fc_prim", input_info("input"), "weights", "", "scale", "zp", data_types::f16, 3, 2, true,
+                            transpose_scale_zp, transpose_scale_zp));
+
+        auto config = get_test_default_config(engine);
+        config.set_property(ov::intel_gpu::allow_new_shape_infer(true));
+        config.set_property(ov::intel_gpu::optimize_data(true));
+
+        network network(engine, topology, config);
+        network.set_input_data("input", input_mem);
+        network.set_input_data("scale", scale_mem);
+        network.set_input_data("zp", zp_mem);
+
+        // Grouped transposed scale/zp [N, groups] must be reordered from bfyx to fbyx to make N innermost.
+        // Per-channel [N, 1] already has N innermost, so it stays as the bfyx parameter.
+        const bool expect_reorder = transpose_scale_zp && is_per_group;
+        auto fc_inst = network.get_primitive("fc_prim");
+        for (size_t idx : {size_t{2}, size_t{3}}) {
+            const auto& dep = fc_inst->get_node().get_dependency(idx);
+            if (expect_reorder) {
+                EXPECT_TRUE(dep.is_type<reorder>()) << "input " << idx;
+                EXPECT_EQ(fc_inst->get_input_layout(idx).format, format::fbyx) << "input " << idx;
+            } else {
+                EXPECT_TRUE(dep.is_type<input_layout>()) << "input " << idx;
+                EXPECT_EQ(fc_inst->get_input_layout(idx).format, format::bfyx) << "input " << idx;
+            }
+        }
+
+        auto outputs = network.execute();
+        auto output_mem = outputs.at("fc_prim").get_memory();
+        cldnn::mem_lock<ov::float16, mem_lock_type::read> output_ptr(output_mem, get_test_stream());
+        ASSERT_EQ((ov::PartialShape{1, 2, ofm}), output_mem->get_layout().get_partial_shape());
+
+        for (int64_t token = 0; token < 2; ++token) {
+            for (int64_t output = 0; output < ofm; ++output) {
+                float expected = 0.0f;
+                for (int64_t k = 0; k < ifm; ++k) {
+                    const auto offset = param_offset(k / group_size, output);
+                    const float weight = (static_cast<float>(weights_data[output * ifm + k]) - zp_data[offset]) * scale_data[offset];
+                    expected += input_data[token * ifm + k] * weight;
+                }
+                EXPECT_NEAR(static_cast<float>(output_ptr[token * ofm + output]), expected, 1e-2f)
+                    << "token = " << token << ", output = " << output;
+            }
+        }
+    }
+
     void test_dynamic(bool is_caching_test) {
         auto& engine = get_test_engine();
 
@@ -6115,6 +6211,22 @@ TEST_F(fully_connected_gpu_tests, compressed_int8_transpose_scale_zp) {
     this->test_compressed_int8_transpose_scale_zp(true);
 }
 
+TEST_F(fully_connected_gpu_tests, compressed_int8_parameter_scale_zp_per_channel) {
+    this->test_compressed_int8_parameter_scale_zp(false, false);
+}
+
+TEST_F(fully_connected_gpu_tests, compressed_int8_parameter_scale_zp_per_channel_transposed) {
+    this->test_compressed_int8_parameter_scale_zp(false, true);
+}
+
+TEST_F(fully_connected_gpu_tests, compressed_int8_parameter_scale_zp_per_group) {
+    this->test_compressed_int8_parameter_scale_zp(true, false);
+}
+
+TEST_F(fully_connected_gpu_tests, compressed_int8_parameter_scale_zp_per_group_transposed) {
+    this->test_compressed_int8_parameter_scale_zp(true, true);
+}
+
 TEST_F(fully_connected_gpu_tests, compressed_int8_scale_b1) {
     this->test_compressed_int8_scale(false, true, 1, false, false);
 }