[GPU] Transpose grouped FC decompression scales at compile time to reduce peak GPU memory

src/plugins/intel_gpu/include/intel_gpu/op/fully_connected_compressed.hpp

@@ -49,6 +49,8 @@ class FullyConnectedCompressed : public FullyConnected {
     bool visit_attributes(ov::AttributeVisitor& visitor) override;
     bool get_transpose_b_scale() const { return m_transpose_b_scale; }
     bool get_transpose_b_zp() const { return m_transpose_b_zp; }
+    void set_transpose_b_scale(bool transpose_b_scale) { m_transpose_b_scale = transpose_b_scale; }
+    void set_transpose_b_zp(bool transpose_b_zp) { m_transpose_b_zp = transpose_b_zp; }
 
     std::shared_ptr<Node> clone_with_new_inputs(const ov::OutputVector& new_args) const override;
 

src/plugins/intel_gpu/include/intel_gpu/primitives/data.hpp

@@ -15,6 +15,7 @@
 #include "openvino/op/constant.hpp"
 #include "openvino/op/convert.hpp"
 #include "openvino/op/tensor_iterator.hpp"
+#include "openvino/op/transpose.hpp"
 #include "openvino/op/util/op_types.hpp"
 #include "openvino/pass/constant_folding.hpp"
 #include "openvino/pass/manager.hpp"
@@ -142,6 +143,12 @@ struct weightless_cache_manager {
         reorder_rep = {input_layout, reorder};
     }
 
+    // The constant holds its weights-file data transposed with this order (applied after the precision conversion)
+    void apply_transpose(std::vector<size_t> order) {
+        OPENVINO_ASSERT(order.size() == shape.size(), "[GPU] Weightless cache: transpose order rank ", order.size(), " != constant rank ", shape.size());
+        transpose_order = std::move(order);
+    }
+
     bool save(BinaryOutputBuffer& ob, size_t data_size) const {
         if (!do_weightless_caching) {
             ob << false;
@@ -151,7 +158,8 @@ struct weightless_cache_manager {
         ob << true;
         ob << bin_offset;
         ob << do_precision_conversion;
-        if (do_precision_conversion) {
+        ob << transpose_order;
+        if (do_precision_conversion || should_run_transpose()) {
             ob << original_size;
             ob << make_data(&original_dtype, sizeof(ov::element::Type));
             ob << make_data(&curr_dtype, sizeof(ov::element::Type));
@@ -180,7 +188,8 @@ struct weightless_cache_manager {
 
         ib >> bin_offset;
         ib >> do_precision_conversion;
-        if (do_precision_conversion) {
+        ib >> transpose_order;
+        if (do_precision_conversion || should_run_transpose()) {
             ib >> original_size;
             ib >> make_data(&original_dtype, sizeof(ov::element::Type));
             ib >> make_data(&curr_dtype, sizeof(ov::element::Type));
@@ -222,6 +231,7 @@ struct weightless_cache_manager {
 private:
     bool do_weightless_caching = false;
     bool do_precision_conversion = false;
+    std::vector<size_t> transpose_order;
     reorder_replication reorder_rep{};
 
     size_t bin_offset = SIZE_MAX;
@@ -234,8 +244,22 @@ struct weightless_cache_manager {
         return reorder_rep.reorder != nullptr;
     }
 
+    bool should_run_transpose() const {
+        return !transpose_order.empty();
+    }
+
     bool should_run_transformations() {
-        return do_precision_conversion || should_run_reorder();
+        return do_precision_conversion || should_run_transpose() || should_run_reorder();
+    }
+
+    // Shape of the data in the weights file: `shape` before the transpose
+    ov::Shape get_original_shape() const {
+        if (!should_run_transpose())
+            return shape;
+        ov::Shape original_shape(shape.size());
+        for (size_t i = 0; i < transpose_order.size(); ++i)
+            original_shape[transpose_order[i]] = shape[i];
+        return original_shape;
     }
 
     void run_transformations(engine& engine,
@@ -266,23 +290,41 @@ struct weightless_cache_manager {
             return original_size;
         };
 
-        if (do_precision_conversion) {
+        if (do_precision_conversion || should_run_transpose()) {
             std::shared_ptr<ov::op::v0::Constant> orig_constant = nullptr;
             if (std::holds_alternative<std::shared_ptr<ov::op::v0::Constant>>(constant_ptr)) {
                 orig_constant = std::get<std::shared_ptr<ov::op::v0::Constant>>(constant_ptr);
+                // The model constant may have another rank (e.g. [N, G, 1] before the FC compression reshape)
+                if (should_run_transpose() && orig_constant->get_shape() != get_original_shape()) {
+                    OPENVINO_ASSERT(ov::shape_size(orig_constant->get_shape()) == ov::shape_size(get_original_shape()),
+                                    "[GPU] Weightless cache: model constant ",
+                                    orig_constant->get_shape(),
+                                    " does not match ",
+                                    get_original_shape());
+                    orig_constant = std::make_shared<ov::op::v0::Constant>(orig_constant->get_element_type(),
+                                                                           get_original_shape(),
+                                                                           orig_constant->get_data_ptr(),
+                                                                           orig_constant);
+                }
             } else {
                 auto shared_buf = std::get<shared_mapped_memory_ptr>(constant_ptr);
-                orig_constant =
-                    std::make_shared<ov::op::v0::Constant>(original_dtype, shape, get_intermediate_data(), shared_buf);
+                orig_constant = std::make_shared<ov::op::v0::Constant>(original_dtype, get_original_shape(), get_intermediate_data(), shared_buf);
             }
 
             ov::ParameterVector inputParams;
             ov::ResultVector results;
             ov::pass::Manager manager("Plugin:GPU:weightless_cache_transformations");
             std::shared_ptr<ov::Model> model = nullptr;
 
-            auto convert_op = std::make_shared<ov::op::v0::Convert>(orig_constant, curr_dtype);
-            results.push_back(std::make_shared<ov::op::v0::Result>(convert_op->output(0)));
+            ov::Output<ov::Node> transformed = orig_constant;
+            if (do_precision_conversion) {
+                transformed = std::make_shared<ov::op::v0::Convert>(transformed, curr_dtype);
+            }
+            if (should_run_transpose()) {
+                auto order = std::make_shared<ov::op::v0::Constant>(ov::element::i64, ov::Shape{transpose_order.size()}, transpose_order);
+                transformed = std::make_shared<ov::op::v1::Transpose>(transformed, order);
+            }
+            results.push_back(std::make_shared<ov::op::v0::Result>(transformed));
             model = std::make_shared<ov::Model>(results, inputParams, "aux");
             manager.register_pass<ov::pass::ConstantFolding>();
 

src/plugins/intel_gpu/sources.cmake

@@ -205,6 +205,7 @@ set(GPU_PLUGIN_TRANSFORMATIONS_SOURCES
     ${CMAKE_CURRENT_LIST_DIR}/src/plugin/transformations/stateless_kv_fusion.cpp
     ${CMAKE_CURRENT_LIST_DIR}/src/plugin/transformations/swoosh_fusion.cpp
     ${CMAKE_CURRENT_LIST_DIR}/src/plugin/transformations/swiglu_fusion_with_clamp.cpp
+    ${CMAKE_CURRENT_LIST_DIR}/src/plugin/transformations/transpose_fc_decompression_params.cpp
     ${CMAKE_CURRENT_LIST_DIR}/src/plugin/transformations/transpose_fusion.cpp
     ${CMAKE_CURRENT_LIST_DIR}/src/plugin/transformations/unsqueeze_broadcast_reshape_matmul_fusion.cpp
 )

src/plugins/intel_gpu/src/graph/graph_optimizer/prepare_quantization.cpp

@@ -554,6 +554,8 @@ static void optimize_weights_decompression_parameters(fully_connected_node& fc_n
     auto reorder_bfyx = [&](size_t dep_id, cldnn::format format) {
         auto& dep = fc_node.get_dependency(dep_id);
         auto target_layout = dep.get_output_layout();
+        if (target_layout.format == format)
+            return;
         target_layout.format = format;
         auto reorder_prim = std::make_shared<reorder>(dep.id() + "_reorder_" + fc_node.id(), dep.id(), target_layout);
         p.add_intermediate(reorder_prim, fc_node, dep_id, true);

src/plugins/intel_gpu/src/plugin/program_builder.cpp

@@ -331,6 +331,10 @@ void ProgramBuilder::add_primitive(const ov::Node& op, std::shared_ptr<cldnn::pr
                                                          attr.original_dtype,
                                                          op.get_output_element_type(0),
                                                          op.get_output_shape(0));
+                auto transpose_order = rt_info.find("gpu_weightless_transpose_order");
+                if (transpose_order != rt_info.end()) {
+                    data_prim->cache_info->apply_transpose(transpose_order->second.as<std::vector<size_t>>());
+                }
             }
         }
     }

src/plugins/intel_gpu/src/plugin/transformations/transpose_fc_decompression_params.cpp

@@ -0,0 +1,69 @@
+// Copyright (C) 2018-2026 Intel Corporation
+// SPDX-License-Identifier: Apache-2.0
+//
+
+#include "transpose_fc_decompression_params.hpp"
+
+#include "intel_gpu/op/fully_connected_compressed.hpp"
+#include "openvino/core/rt_info.hpp"
+#include "openvino/core/rt_info/weightless_caching_attributes.hpp"
+#include "openvino/core/validation_util.hpp"
+#include "openvino/op/constant.hpp"
+#include "openvino/op/transpose.hpp"
+#include "openvino/pass/pattern/op/wrap_type.hpp"
+
+namespace ov::intel_gpu {
+
+TransposeFCDecompressionParams::TransposeFCDecompressionParams() {
+    using namespace ov::pass::pattern;
+
+    // The input count depends on the zero point and dynamic quantization inputs, so the parameters are checked in the callback
+    auto fc_predicate = [](const ov::Output<ov::Node>& output) -> bool {
+        const auto fc = ov::as_type_ptr<op::FullyConnectedCompressed>(output.get_node_shared_ptr());
+        return fc->get_transpose_b() && fc->get_input_partial_shape(1).size() == 2;
+    };
+    auto fc_m = wrap_type<op::FullyConnectedCompressed>(fc_predicate);
+
+    ov::matcher_pass_callback callback = [=](Matcher& m) {
+        const auto fc = ov::as_type_ptr<op::FullyConnectedCompressed>(m.get_match_root());
+        OPENVINO_ASSERT(fc, "TransposeFCDecompressionParams: matched node is expected to be FullyConnectedCompressed");
+        // Weights are a Constant, so N is static
+        const auto n = fc->get_input_shape(1)[0];
+
+        // [N, G] -> [G, N]; false when the parameter keeps its layout
+        auto transpose_param = [&](size_t idx, bool transposed, const std::string& name_suffix) {
+            if (!transposed || idx >= fc->get_input_size())
+                return false;
+            const auto param = ov::as_type_ptr<ov::op::v0::Constant>(fc->get_input_node_shared_ptr(idx));
+            if (!param)
+                return false;
+            const auto& shape = param->get_shape();
+            if (shape.size() != 2 || shape[0] != n || shape[1] == 1 || param->get_element_type().bitwidth() < 8)
+                return false;
+            const auto order = ov::op::v0::Constant::create(ov::element::i32, ov::Shape{2}, {1, 0});
+            auto new_param = ov::util::get_constant_from_source(std::make_shared<ov::op::v1::Transpose>(param, order));
+            OPENVINO_ASSERT(new_param, "TransposeFCDecompressionParams: failed to fold the transpose of ", param->get_friendly_name());
+            new_param->set_friendly_name(fc->get_friendly_name() + name_suffix);
+            ov::copy_runtime_info(param, new_param);
+            if (param->get_rt_info().count(ov::WeightlessCacheAttribute::get_type_info_static()) != 0u) {
+                ov::copy_weightless_cache_attr(param, new_param);
+                new_param->get_rt_info()["gpu_weightless_transpose_order"] = std::vector<size_t>{1, 0};
+            }
+            fc->input(idx).replace_source_output(new_param);
+            return true;
+        };
+
+        const bool scale_transposed = transpose_param(3, fc->get_transpose_b_scale(), "_scale");
+        if (scale_transposed)
+            fc->set_transpose_b_scale(false);
+        const bool zp_transposed = transpose_param(4, fc->get_transpose_b_zp(), "_zp");
+        if (zp_transposed)
+            fc->set_transpose_b_zp(false);
+        return scale_transposed || zp_transposed;
+    };
+
+    auto m = std::make_shared<ov::pass::pattern::Matcher>(fc_m, "TransposeFCDecompressionParams");
+    this->register_matcher(m, callback);
+}
+
+}  // namespace ov::intel_gpu

src/plugins/intel_gpu/src/plugin/transformations/transpose_fc_decompression_params.hpp

@@ -0,0 +1,41 @@
+// Copyright (C) 2018-2026 Intel Corporation
+// SPDX-License-Identifier: Apache-2.0
+//
+
+#pragma once
+
+#include "openvino/pass/matcher_pass.hpp"
+
+namespace ov::intel_gpu {
+
+/**
+ * @brief Stores grouped FullyConnectedCompressed decompression scale / zero point constants in the [groups, N] layout.
+ *
+ * The OCL and oneDNN FC kernels read grouped decompression parameters with N innermost. With transposed weights
+ * [N, K] the parameters are [N, groups], so prepare_quantization adds a reorder to fbyx, which keeps a second,
+ * reordered copy of each parameter in device memory. This pass transposes such constants at compile time and
+ * clears transpose_b_scale / transpose_b_zp, so the parameter is used as is.
+ *
+ * Before:                                     After:
+ *
+ *   Data  Weights [N, K]  Scale [N, G]          Data  Weights [N, K]  Scale' [G, N]
+ *     │        │             │                    │        │             │
+ *  ┌──┴────────┴─────────────┴──┐              ┌──┴────────┴─────────────┴──┐
+ *  │ FullyConnectedCompressed   │              │ FullyConnectedCompressed   │
+ *  │ transpose_b_scale = true   │              │ transpose_b_scale = false  │
+ *  └────────────────────────────┘              └────────────────────────────┘
+ *
+ * The same applies to the zero point (transpose_b_zp). A parameter is transposed only when:
+ *  - the weights are 2D [N, K] with transpose_b;
+ *  - it is a Constant [N, G] with G > 1 (a single group has the same memory order in both layouts);
+ *  - its element type is at least 8 bits wide (sub-byte zero points keep the existing layout).
+ * The new constant keeps the WeightlessCacheAttribute of the original one and records the transpose order in the
+ * "gpu_weightless_transpose_order" rt_info entry, so a weightless cache load rebuilds its data from the weights file.
+ */
+class TransposeFCDecompressionParams : public ov::pass::MatcherPass {
+public:
+    OPENVINO_MATCHER_PASS_RTTI("TransposeFCDecompressionParams");
+    TransposeFCDecompressionParams();
+};
+
+}  // namespace ov::intel_gpu

src/plugins/intel_gpu/src/plugin/transformations_pipeline.cpp

@@ -132,6 +132,7 @@
 #include "plugin/transformations/reduce_fc_dimensions.hpp"
 #include "plugin/transformations/remove_fq_before_dw_conv.hpp"
 #include "plugin/transformations/sink_reshape.hpp"
+#include "plugin/transformations/transpose_fc_decompression_params.hpp"
 #include "plugin/transformations/transpose_fusion.hpp"
 #include "plugin/transformations/sdpa_transpose_fusion.hpp"
 #include "plugin/transformations/unsqueeze_broadcast_reshape_matmul_fusion.hpp"
@@ -2136,6 +2137,9 @@ void TransformationsPipeline::apply(std::shared_ptr<ov::Model> func) {
             }
         }
 
+        // After all FullyConnected passes, which read the scale / zp layout flags
+        manager.register_pass<ov::intel_gpu::TransposeFCDecompressionParams>();
+
         // Remove Pad in front of MaxPool if both the pads_begin and pads_end are zero.
         manager.register_pass<ov::pass::EliminatePad>();
 

src/plugins/intel_gpu/tests/functional/behavior/model_cache.cpp

@@ -377,8 +377,9 @@ TEST_F(CheckModelCacheOnednnConvAZP, ActivationZeroPointRestoredFromCache) {
     ov::test::utils::compare(orig_req.get_output_tensor(), new_req.get_output_tensor(), compiled_model.output().get_element_type());
 }
 
-TEST_P(CheckWeightlessCacheAccuracyLowPrecision, MatmulWeightsDecompression) {
-    ov::test::MatMulDecompressionShapeParams shape_params{{{}, {{1, 4, 16}}}, {1, 16, 32}};
+std::shared_ptr<ov::Model> make_matmul_decompression_model(const ov::test::MatMulDecompressionShapeParams& shape_params,
+                                                           const ov::element::Type& weights_dtype,
+                                                           const std::string& name) {
     auto dynShape = shape_params.data_shape.first;
     if (dynShape.rank() == 0) {
         dynShape = shape_params.data_shape.second.front();
@@ -387,7 +388,7 @@ TEST_P(CheckWeightlessCacheAccuracyLowPrecision, MatmulWeightsDecompression) {
     const auto weights_subgraph = ov::test::utils::initMatMulDecompressionSubgraph(shape_params.weights_shape,
                                                                                    shape_params.decompression_group_size,
                                                                                    ov::element::f32,
-                                                                                   model_dtype,
+                                                                                   weights_dtype,
                                                                                    ov::element::f32,
                                                                                    ov::element::dynamic,
                                                                                    true,
@@ -400,7 +401,18 @@ TEST_P(CheckWeightlessCacheAccuracyLowPrecision, MatmulWeightsDecompression) {
     for (const auto& output : matmul->outputs()) {
         results.push_back(std::make_shared<ov::op::v0::Result>(output));
     }
-    model = std::make_shared<ov::Model>(results, params, "MatmulWeightsDecompression");
+    return std::make_shared<ov::Model>(results, params, name);
+}
+
+TEST_P(CheckWeightlessCacheAccuracyLowPrecision, MatmulWeightsDecompression) {
+    model = make_matmul_decompression_model({{{}, {{1, 4, 16}}}, {1, 16, 32}}, model_dtype, "MatmulWeightsDecompression");
+    OV_ASSERT_NO_THROW(run());
+}
+
+// Grouped scale / zero point [N, groups] with transposed weights: the plugin changes their layout for
+// FullyConnectedCompressed, which a weightless cache load must reproduce
+TEST_P(CheckWeightlessCacheAccuracyLowPrecision, MatmulGroupedWeightsDecompression) {
+    model = make_matmul_decompression_model({{{}, {{1, 4, 128}}}, {128, 64}, 32}, model_dtype, "MatmulGroupedWeightsDecompression");
     OV_ASSERT_NO_THROW(run());
 }
 

src/plugins/intel_gpu/tests/unit/sources.cmake

@@ -362,6 +362,7 @@ set(GPU_UNIT_TESTS_SRCS
     ${CMAKE_CURRENT_LIST_DIR}/transformations/stateless_kv_fusion_test.cpp
     ${CMAKE_CURRENT_LIST_DIR}/transformations/swoosh_fusion_test.cpp
     ${CMAKE_CURRENT_LIST_DIR}/transformations/swish_with_clamp_fusion_test.cpp
+    ${CMAKE_CURRENT_LIST_DIR}/transformations/transpose_fc_decompression_params_test.cpp
     ${CMAKE_CURRENT_LIST_DIR}/transformations/transpose_matmul_fusion_test.cpp
     ${CMAKE_CURRENT_LIST_DIR}/transformations/transpose_sdpa_fusion_test.cpp
     ${CMAKE_CURRENT_LIST_DIR}/transformations/transpose_split_fusion_test.cpp
@@ -489,6 +490,8 @@ set(GPU_UNIT_TESTS_TRANSFORMATIONS_SRCS
     ${CMAKE_HOME_DIRECTORY}/src/plugins/intel_gpu/src/plugin/transformations/swoosh_fusion.hpp
     ${CMAKE_HOME_DIRECTORY}/src/plugins/intel_gpu/src/plugin/transformations/swiglu_fusion_with_clamp.cpp
     ${CMAKE_HOME_DIRECTORY}/src/plugins/intel_gpu/src/plugin/transformations/swiglu_fusion_with_clamp.hpp
+    ${CMAKE_HOME_DIRECTORY}/src/plugins/intel_gpu/src/plugin/transformations/transpose_fc_decompression_params.cpp
+    ${CMAKE_HOME_DIRECTORY}/src/plugins/intel_gpu/src/plugin/transformations/transpose_fc_decompression_params.hpp
     ${CMAKE_HOME_DIRECTORY}/src/plugins/intel_gpu/src/plugin/transformations/transpose_fusion.cpp
     ${CMAKE_HOME_DIRECTORY}/src/plugins/intel_gpu/src/plugin/transformations/transpose_fusion.hpp
     ${CMAKE_HOME_DIRECTORY}/src/plugins/intel_gpu/src/plugin/transformations/unsqueeze_broadcast_reshape_matmul_fusion.cpp