Skip to content

[GPU] Fix OC-broadcast FC decompression masks #39106

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
e-ddykim wants to merge 1 commit into
base: master
Choose a base branch
from
Open

[GPU] Fix OC-broadcast FC decompression masks #39106

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
36 changes: 32 additions & 4 deletions src/plugins/intel_gpu/src/graph/impls/onednn/fully_connected_onednn.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,28 @@ struct fully_connected_onednn : typed_primitive_onednn_impl<fully_connected> {
return std::max<int64_t>(0, base + (transpose_b_param ? logical_rank - 1 : logical_rank - 2));
}

// Match the logical weights dimensions used by get_matmul_primitive_descriptor().
static int64_t get_weights_oc(const layout& weights_layout, bool weights_transposed, size_t weights_rank) {
const auto& pshape = weights_layout.get_partial_shape();
if (pshape.rank().is_dynamic() || weights_rank < 2 || weights_rank > pshape.size())
return -1;
const auto oc_idx = weights_transposed ? weights_rank - 2 : weights_rank - 1;
const auto& oc = pshape[oc_idx];
return oc.is_static() ? oc.get_length() : -1;
}

static int get_grouped_mask(const layout& param_layout, bool transpose_b_param, size_t weights_rank, int grouped, int per_oc, int64_t weights_oc) {
const auto rank = static_cast<int64_t>(param_layout.get_partial_shape().size());
// A single weights OC is not broadcast; retain the GEMM-compatible grouped mask.
if (rank <= 1 || weights_oc == 1)
return grouped;
const auto ifm_idx = get_ifm_dim_idx(param_layout, transpose_b_param, weights_rank);
const auto oc_idx = transpose_b_param ? ifm_idx - 1 : ifm_idx + 1;
if (oc_idx < 0 || oc_idx >= rank)
return grouped;
return param_layout.get_dim(oc_idx) == 1 ? (grouped & ~per_oc) : grouped;
}

DECLARE_OBJECT_TYPE_SERIALIZATION(cldnn::onednn::fully_connected_onednn)

private:
Expand Down Expand Up @@ -290,6 +312,7 @@ struct fully_connected_onednn : typed_primitive_onednn_impl<fully_connected> {
int idx = !arg.bias_term() ? 1 : 2;
int per_oc = PER_OC << shift_size;
int grouped = (1 << prim->input_size) - 1;
const auto weights_oc = get_weights_oc(weights_layout, prim->weights_transposed, prim->weights_rank);

bool has_decompression_scale = prim->decompression_scale.is_valid();
if (has_decompression_scale) {
Expand All @@ -304,7 +327,8 @@ struct fully_connected_onednn : typed_primitive_onednn_impl<fully_connected> {
} else if (ngroups == 1) {
_attrs->set_scales(DNNL_ARG_WEIGHTS, per_oc, dnnl::memory::dims{}, _ds_data_type);
} else {
_attrs->set_scales(DNNL_ARG_WEIGHTS, grouped, {_ds_group_size, 1}, _ds_data_type);
const auto mask = get_grouped_mask(scale_layout, transpose_b_scale, weight_rank, grouped, per_oc, weights_oc);
_attrs->set_scales(DNNL_ARG_WEIGHTS, mask, {_ds_group_size, 1}, _ds_data_type);
}
}

Expand All @@ -322,7 +346,8 @@ struct fully_connected_onednn : typed_primitive_onednn_impl<fully_connected> {
if (ngroups == 1) {
_attrs->set_zero_points(DNNL_ARG_WEIGHTS, per_oc, dnnl::memory::dims{}, _dzp_data_type);
} else {
_attrs->set_zero_points(DNNL_ARG_WEIGHTS, grouped, {_ds_group_size, 1}, _dzp_data_type);
const auto mask = get_grouped_mask(dzp_layout, transpose_b_zp, weight_rank, grouped, per_oc, weights_oc);
_attrs->set_zero_points(DNNL_ARG_WEIGHTS, mask, {_ds_group_size, 1}, _dzp_data_type);
}
}
}
Expand Down Expand Up @@ -394,6 +419,7 @@ struct fully_connected_onednn : typed_primitive_onednn_impl<fully_connected> {
auto shift_size = std::max<size_t>(prim->input_size - 2, 0);
int per_oc = PER_OC << shift_size;
int grouped = (1 << prim->input_size) - 1;
const auto weights_oc = get_weights_oc(weights_layout, prim->weights_transposed, prim->weights_rank);

if (prim->decompression_scale.is_valid()) {
auto decompression_scale_idx = ++idx;
Expand All @@ -416,7 +442,8 @@ struct fully_connected_onednn : typed_primitive_onednn_impl<fully_connected> {
// Example:
// input[32, 6, 2088], W_t[32, 5760, 2088], scale[32, 1, 5760]
// set scale group as [32, 2088, 1]
attr->set_scales(DNNL_ARG_WEIGHTS, grouped, {group_size, 1}, ds_data_type);
const auto mask = get_grouped_mask(scale_layout, prim->transpose_b_scale, weight_rank, grouped, per_oc, weights_oc);
attr->set_scales(DNNL_ARG_WEIGHTS, mask, {group_size, 1}, ds_data_type);
}
}

Expand All @@ -437,7 +464,8 @@ struct fully_connected_onednn : typed_primitive_onednn_impl<fully_connected> {
attr->set_zero_points(DNNL_ARG_WEIGHTS, per_oc, dnnl::memory::dims{}, dzp_data_type);
} else {
// should use {K, 1} for the group size + per tensor mask for 3d
attr->set_zero_points(DNNL_ARG_WEIGHTS, grouped, {group_size, 1}, dzp_data_type);
const auto mask = get_grouped_mask(dzp_layout, prim->transpose_b_zp, weight_rank, grouped, per_oc, weights_oc);
attr->set_zero_points(DNNL_ARG_WEIGHTS, mask, {group_size, 1}, dzp_data_type);
}
}
}
Expand Down
170 changes: 170 additions & 0 deletions src/plugins/intel_gpu/tests/unit/test_cases/fully_connected_gpu_test.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3274,6 +3274,120 @@ void test_compressed_int4_scale_dynamic_batch_gemv(bool is_caching_test,
}
}

void test_compressed_int8_broadcast_scale_zp(bool is_caching_test,
bool transpose_parameters,
size_t input_rank = 3,
bool with_zp = true,
bool broadcast = true) {
auto& engine = get_test_engine();
if (!engine.get_device_info().supports_fp16)
GTEST_SKIP() << "The test requires FP16 support.";

constexpr int64_t ofm = 3, ifm = 32, batch = 2, groups = 2;
const auto input_shape = input_rank == 2 ? ov::PartialShape{batch, ifm} : ov::PartialShape{1, batch, ifm};
const int64_t param_oc = broadcast ? 1 : ofm;
const auto param_shape = transpose_parameters ? ov::PartialShape{param_oc, groups} : ov::PartialShape{groups, param_oc};
auto input_mem = engine.allocate_memory({input_shape, data_types::f16, format::bfyx});
auto weights_mem = engine.allocate_memory({ov::PartialShape{ofm, ifm}, data_types::u8, format::bfyx});
auto scale_mem = engine.allocate_memory({param_shape, data_types::f16, format::bfyx});
auto zp_mem = engine.allocate_memory({param_shape, data_types::u8, format::bfyx});
set_values<ov::float16>(input_mem, std::vector<ov::float16>(batch * ifm, ov::float16(1.0f)));
set_values<uint8_t>(weights_mem, std::vector<uint8_t>(ofm * ifm, 3));
std::vector<ov::float16> scales;
std::vector<uint8_t> zps;
for (int64_t i = 0; i < param_oc * groups; ++i) {
const auto group = transpose_parameters ? i % groups : i / param_oc;
scales.emplace_back(float(group + 2));
zps.emplace_back(static_cast<uint8_t>(group + 1));
}
set_values<ov::float16>(scale_mem, scales);
set_values<uint8_t>(zp_mem, zps);

auto fc_prim = fully_connected("fc_prim",
input_info("input"),
"weights",
"",
"scale",
with_zp ? "zp" : "",
data_types::f16,
input_rank,
2,
true,
transpose_parameters,
transpose_parameters);
topology topology(input_layout("input", input_mem->get_layout()), data("weights", weights_mem), data("scale", scale_mem), fc_prim);
if (with_zp)
topology.add(data("zp", zp_mem));
auto config = get_test_default_config(engine);
config.set_property(ov::intel_gpu::allow_new_shape_infer(true));
config.set_property(ov::intel_gpu::optimize_data(true));
config.set_user_property(ov::hint::dynamic_quantization_group_size(0));

auto network = get_network(engine, topology, config, get_test_stream_ptr(), is_caching_test);
network->set_input_data("input", input_mem);
const auto outputs = network->execute();
if (engine.get_device_info().supports_immad) {
ASSERT_TRUE(network->get_primitive("fc_prim")->get_impl()->is_onednn());
}
ASSERT_EQ(outputs.size(), size_t{1});
const auto output_mem = outputs.at("fc_prim").get_memory();
const auto output_shape = input_rank == 2 ? ov::PartialShape{batch, ofm} : ov::PartialShape{1, batch, ofm};
ASSERT_EQ(output_mem->get_layout().get_partial_shape(), output_shape);
mem_lock<ov::float16, mem_lock_type::read> output_ptr(output_mem, get_test_stream());
ASSERT_EQ(output_ptr.size(), size_t{batch * ofm});
// Both K groups share their decompression parameters across the output channels.
const auto expected = ov::float16(with_zp ? 112.0f : 240.0f);
for (size_t i = 0; i < output_ptr.size(); ++i)
EXPECT_EQ(output_ptr[i], expected) << "i = " << i;
}

void test_compressed_int4_single_oc_grouped(bool is_caching_test, bool with_zp) {
SCOPED_TRACE(::testing::Message() << "cached=" << is_caching_test << " zp=" << with_zp);
auto& engine = get_test_engine();
if (!engine.get_device_info().supports_immad || !engine.get_device_info().supports_fp16)
GTEST_SKIP() << "The test requires oneDNN FC and FP16 support.";
constexpr int64_t ifm = 2048, batch = 2, groups = 16;
auto input_mem = engine.allocate_memory({ov::PartialShape{batch, ifm}, data_types::f16, format::bfyx});
auto weights_mem = engine.allocate_memory({ov::PartialShape{1, ifm}, data_types::u4, format::bfyx});
auto scale_mem = engine.allocate_memory({ov::PartialShape{1, groups}, data_types::f16, format::bfyx});
auto zp_mem = engine.allocate_memory({ov::PartialShape{1, groups}, data_types::u8, format::bfyx});
set_values<ov::float16>(input_mem, std::vector<ov::float16>(batch * ifm, ov::float16(1.0f)));
set_values<uint8_t>(weights_mem, std::vector<uint8_t>(ifm / 2, 0x33));
std::vector<ov::float16> scales;
std::vector<uint8_t> zps;
for (int64_t group = 0; group < groups; ++group) {
scales.emplace_back(float(group % 2 + 2));
zps.emplace_back(static_cast<uint8_t>(group % 2 + 1));
}
set_values<ov::float16>(scale_mem, scales);
set_values<uint8_t>(zp_mem, zps);

auto fc_prim = fully_connected("fc_prim", input_info("input"), "weights", "", "scale", with_zp ? "zp" : "", data_types::f16, 2, 2, true, true, true);
topology topology(input_layout("input", {ov::PartialShape{-1, ifm}, data_types::f16, format::bfyx}),
data("weights", weights_mem),
data("scale", scale_mem),
fc_prim);
if (with_zp)
topology.add(data("zp", zp_mem));
auto config = get_test_default_config(engine);
config.set_property(ov::intel_gpu::allow_new_shape_infer(true));
config.set_property(ov::intel_gpu::optimize_data(true));
config.set_user_property(ov::hint::dynamic_quantization_group_size(0));
auto network = get_network(engine, topology, config, get_test_stream_ptr(), is_caching_test);
network->set_input_data("input", input_mem);
const auto output_mem = network->execute().at("fc_prim").get_memory();
const auto* fc_impl = network->get_primitive("fc_prim")->get_impl();
ASSERT_TRUE(fc_impl->is_onednn());
if (with_zp) {
// N=1 must retain the grouped mask, not fall back to oneDNN's reference matmul.
EXPECT_EQ(fc_impl->get_kernel_name().find(":ref:"), std::string::npos) << fc_impl->get_kernel_name();
}
mem_lock<ov::float16, mem_lock_type::read> output_ptr(output_mem, get_test_stream());
ASSERT_EQ(output_ptr.size(), size_t{batch});
for (size_t i = 0; i < output_ptr.size(); ++i)
EXPECT_EQ(output_ptr[i], ov::float16(with_zp ? 7168.0f : 15360.0f)) << "i = " << i;
}

void test_dynamic(bool is_caching_test) {
auto& engine = get_test_engine();

Expand Down Expand Up @@ -5773,6 +5887,62 @@ TEST_F(fully_connected_gpu_tests, compressed_int8_transpose_scale_zp) {
this->test_compressed_int8_transpose_scale_zp(true);
}

TEST_F(fully_connected_gpu_tests, compressed_int8_broadcast_scale_zp) {
this->test_compressed_int8_broadcast_scale_zp(false, false);
}

TEST_F(fully_connected_gpu_tests, compressed_int8_broadcast_scale_zp_cached) {
this->test_compressed_int8_broadcast_scale_zp(true, false);
}

using fc_grouped_broadcast_params = std::tuple<bool, bool, std::pair<size_t, bool>>;
class fc_grouped_oc_broadcast_onednn : public fully_connected_gpu_tests, public ::testing::WithParamInterface<fc_grouped_broadcast_params> {
public:
static std::string getTestCaseName(const ::testing::TestParamInfo<fc_grouped_broadcast_params>& info) {
const auto& [cached, transposed, shape_and_zp] = info.param;
return std::string(cached ? "Cached" : "Fresh") + (transposed ? "Transposed" : "Plain") + "Rank" + std::to_string(shape_and_zp.first) +
(shape_and_zp.second ? "WithZP" : "ScaleOnly");
}
};

TEST_P(fc_grouped_oc_broadcast_onednn, grouped_scale_zp) {
auto& engine = get_test_engine();
if (!engine.get_device_info().supports_immad)
GTEST_SKIP() << "The test requires oneDNN FC support.";
const auto& [cached, transposed, shape_and_zp] = GetParam();
const auto& [rank, with_zp] = shape_and_zp;
this->test_compressed_int8_broadcast_scale_zp(cached, transposed, rank, with_zp);
}

INSTANTIATE_TEST_SUITE_P(
fully_connected_gpu,
fc_grouped_oc_broadcast_onednn,
::testing::Combine(::testing::Bool(),
::testing::Bool(),
::testing::Values(std::pair<size_t, bool>{2, true}, std::pair<size_t, bool>{3, true}, std::pair<size_t, bool>{3, false})),
fc_grouped_oc_broadcast_onednn::getTestCaseName);

TEST_F(fully_connected_gpu_tests, compressed_int8_grouped_per_oc_control) {
if (!get_test_engine().get_device_info().supports_immad)
GTEST_SKIP() << "The test requires oneDNN FC support.";
for (bool cached : {false, true})
for (bool transposed : {false, true})
for (bool with_zp : {false, true}) {
SCOPED_TRACE(::testing::Message() << "cached=" << cached << " transposed=" << transposed << " zp=" << with_zp);
this->test_compressed_int8_broadcast_scale_zp(cached, transposed, 2, with_zp, false);
}
}

TEST_F(fully_connected_gpu_tests, compressed_int4_single_oc_grouped_dynamic) {
for (bool with_zp : {false, true})
this->test_compressed_int4_single_oc_grouped(false, with_zp);
}

TEST_F(fully_connected_gpu_tests, compressed_int4_single_oc_grouped_dynamic_cached) {
for (bool with_zp : {false, true})
this->test_compressed_int4_single_oc_grouped(true, with_zp);
}

TEST_F(fully_connected_gpu_tests, compressed_int8_scale_b1) {
this->test_compressed_int8_scale(false, true, 1, false, false);
}
Expand Down
59 changes: 59 additions & 0 deletions src/plugins/intel_gpu/tests/unit/transformations/convert_fc_to_compressed_test.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
#include "openvino/op/transpose.hpp"
#include "openvino/op/convert.hpp"
#include "openvino/op/multiply.hpp"
#include "openvino/op/matmul.hpp"
#include "openvino/op/parameter.hpp"
#include "openvino/op/constant.hpp"
#include "openvino/op/reshape.hpp"
Expand All @@ -20,6 +21,7 @@
#include "intel_gpu/op/placeholder.hpp"

#include "plugin/transformations/convert_fc_to_compressed.hpp"
#include "plugin/transformations/convert_matmul_to_fc.hpp"

#include <memory>

Expand Down Expand Up @@ -892,6 +894,63 @@ TEST_F(TransformationTestsF, ConvertFCToCompressed24_mxfp4e2m1) {
}
}

class ConvertFCToCompressedBroadcastParameterWeights : public TransformationTestsF, public ::testing::WithParamInterface<bool> {
protected:
void SetUp() override {
TransformationTestsF::SetUp();
comparator.enable(FunctionsComparator::CONST_VALUES);
comparator.enable(FunctionsComparator::ATTRIBUTES);
manager.register_pass<ConvertMatMulToFullyConnected>(true);
manager.register_pass<ConvertFullyConnectedToFullyConnectedCompressed>();
}
};

TEST_P(ConvertFCToCompressedBroadcastParameterWeights, PreservesOCBroadcastScaleAndZeroPoint) {
constexpr size_t oc = 3, ifm = 32, groups = 2;
const bool transposed = GetParam();
const auto weights_shape = transposed ? ov::Shape{oc, groups, ifm / groups} : ov::Shape{groups, ifm / groups, oc};
const auto param_shape = transposed ? ov::Shape{1, groups, 1} : ov::Shape{groups, 1, 1};
const auto flat_weights_shape = transposed ? std::vector<int32_t>{oc, ifm} : std::vector<int32_t>{ifm, oc};
const auto flat_param_shape = transposed ? ov::Shape{1, groups} : ov::Shape{groups, 1};
{
auto input = std::make_shared<ov::op::v0::Parameter>(ov::element::f16, ov::PartialShape{-1, ifm});
auto weights = std::make_shared<ov::op::v0::Parameter>(ov::element::u8, weights_shape);
auto convert = std::make_shared<ov::op::v0::Convert>(weights, ov::element::f16);
auto zp = ov::op::v0::Constant::create(ov::element::u8, param_shape, {1, 2});
auto zp_convert = std::make_shared<ov::op::v0::Convert>(zp, ov::element::f16);
auto subtract = std::make_shared<ov::op::v1::Subtract>(convert, zp_convert);
auto scale = ov::op::v0::Constant::create(ov::element::f16, param_shape, {2, 3});
auto multiply = std::make_shared<ov::op::v1::Multiply>(subtract, scale);
auto shape = ov::op::v0::Constant::create(ov::element::i32, ov::Shape{2}, flat_weights_shape);
auto reshape = std::make_shared<ov::op::v1::Reshape>(multiply, shape, false);
auto matmul = std::make_shared<ov::op::v0::MatMul>(input, reshape, false, transposed);
model = std::make_shared<ov::Model>(ov::OutputVector{matmul}, ov::ParameterVector{input, weights});
}
{
auto input = std::make_shared<ov::op::v0::Parameter>(ov::element::f16, ov::PartialShape{-1, ifm});
auto weights = std::make_shared<ov::op::v0::Parameter>(ov::element::u8, weights_shape);
auto shape = ov::op::v0::Constant::create(ov::element::i32, ov::Shape{2}, flat_weights_shape);
std::shared_ptr<ov::Node> reshape = std::make_shared<ov::op::v1::Reshape>(weights, shape, false);
if (!transposed) {
auto order = ov::op::v0::Constant::create(ov::element::i32, ov::Shape{2}, {1, 0});
reshape = std::make_shared<ov::op::v1::Transpose>(reshape, order);
}
auto scale = ov::op::v0::Constant::create(ov::element::f16, flat_param_shape, {2, 3});
auto zp = ov::op::v0::Constant::create(ov::element::u8, flat_param_shape, {1, 2});
auto bias = std::make_shared<ov::intel_gpu::op::Placeholder>();
auto fc =
std::make_shared<ov::intel_gpu::op::FullyConnectedCompressed>(input, reshape, bias, scale, zp, ov::element::f16, true, transposed, transposed);
model_ref = std::make_shared<ov::Model>(ov::OutputVector{fc}, ov::ParameterVector{input, weights});
}
}

INSTANTIATE_TEST_SUITE_P(ConvertFCToCompressed,
ConvertFCToCompressedBroadcastParameterWeights,
::testing::Bool(),
[](const ::testing::TestParamInfo<bool>& info) {
return info.param ? "TransposedWeights" : "NonTransposedWeights";
});

} // namespace intel_gpu
} // namespace test
} // namespace ov
Loading