[GPU] FullyConnectedGGUF: MatMul on GGUF Q5_K / Q6_K weights without conversion

src/core/dev_api/dev_headers.cmake

@@ -22,6 +22,7 @@ set(DEV_HEADERS
     ${CMAKE_CURRENT_LIST_DIR}/openvino/core/type/float_util.hpp
     ${CMAKE_CURRENT_LIST_DIR}/openvino/core/type/nf4.hpp
     ${CMAKE_CURRENT_LIST_DIR}/openvino/op/gated_delta_net.hpp
+    ${CMAKE_CURRENT_LIST_DIR}/openvino/op/gguf_dequantize.hpp
     ${CMAKE_CURRENT_LIST_DIR}/openvino/op/group_query_attention.hpp
     ${CMAKE_CURRENT_LIST_DIR}/openvino/op/moe.hpp
     ${CMAKE_CURRENT_LIST_DIR}/openvino/op/ops_decl.hpp

src/core/dev_api/openvino/op/gguf_dequantize.hpp

@@ -0,0 +1,81 @@
+// Copyright (C) 2018-2026 Intel Corporation
+// SPDX-License-Identifier: Apache-2.0
+//
+
+#pragma once
+
+#include "openvino/op/op.hpp"
+
+namespace ov::op::internal {
+/// \note GGUFDequantize op class is under development and subject to change
+///
+/// \brief Decodes GGUF k-quant weight blocks, exactly as stored in a GGUF file, into a floating-point tensor.
+///
+/// GGUF stores every row of a quantized weight as a sequence of blocks of 256 values. Each block carries its own
+/// scales, so the bytes cannot be expressed as an OpenVINO low-precision element type plus separate scale and
+/// zero-point tensors without converting them. This op keeps the file bytes: a plugin with kernels for the format
+/// can read them directly, any other consumer gets the decoded values through constant folding or evaluate().
+///
+/// Input 0: u8 tensor [..., N, B], B = K / 256 * block_bytes(format): N rows of K values each.
+/// Output 0: [..., N, K] of output_type (f32, f16 or bf16).
+///
+/// Formats (ggml block layouts, little-endian):
+///   Q4_K: 144 bytes per block: f16 d, f16 dmin, 12 bytes of 6-bit scales and mins (8 sub-blocks of 32), 128 bytes of
+///         4-bit quants. w = d * sc * q - dmin * m.
+///   Q5_K: 176 bytes per block: as Q4_K plus 32 bytes holding the 5th bit of every quant.
+///   Q6_K: 210 bytes per block: 128 bytes low 4 bits, 64 bytes high 2 bits, 16 int8 scales (one per 16 values),
+///         f16 d. w = d * sc * (q - 32).
+class OPENVINO_API GGUFDequantize : public ov::op::Op {
+public:
+    OPENVINO_OP("GGUFDequantize");
+
+    enum class Format { Q4_K, Q5_K, Q6_K };
+
+    /// \brief Values per block, the same for every k-quant format.
+    static constexpr size_t block_size = 256;
+
+    GGUFDequantize() = default;
+    /// \brief Constructs a GGUFDequantize operation.
+    ///
+    /// \param data        u8 tensor with the GGUF blocks, [..., N, K / 256 * block_bytes(format)]
+    /// \param format      GGUF block format of the data
+    /// \param output_type Element type of the decoded values (f32, f16 or bf16)
+    GGUFDequantize(const Output<Node>& data, Format format, const ov::element::Type& output_type = ov::element::f32);
+
+    bool visit_attributes(AttributeVisitor& visitor) override;
+    void validate_and_infer_types() override;
+    std::shared_ptr<Node> clone_with_new_inputs(const OutputVector& new_args) const override;
+    bool evaluate(TensorVector& outputs, const TensorVector& inputs) const override;
+    bool has_evaluate() const override;
+
+    Format get_format() const {
+        return m_format;
+    }
+    const ov::element::Type& get_destination_type() const {
+        return m_output_type;
+    }
+
+    /// \brief Bytes of one 256-value block of the format.
+    static size_t block_bytes(Format format);
+
+private:
+    Format m_format = Format::Q4_K;
+    ov::element::Type m_output_type = ov::element::f32;
+};
+
+}  // namespace ov::op::internal
+
+namespace ov {
+OPENVINO_API std::ostream& operator<<(std::ostream& s, const ov::op::internal::GGUFDequantize::Format& format);
+
+template <>
+class AttributeAdapter<ov::op::internal::GGUFDequantize::Format>
+    : public EnumAttributeAdapterBase<ov::op::internal::GGUFDequantize::Format> {
+public:
+    AttributeAdapter(ov::op::internal::GGUFDequantize::Format& value)
+        : EnumAttributeAdapterBase<ov::op::internal::GGUFDequantize::Format>(value) {}
+
+    OPENVINO_RTTI("AttributeAdapter<ov::op::internal::GGUFDequantize::Format>");
+    ~AttributeAdapter() override = default;
+};
+}  // namespace ov

src/core/reference/include/openvino/reference/gguf_dequantize.hpp

@@ -0,0 +1,128 @@
+// Copyright (C) 2018-2026 Intel Corporation
+// SPDX-License-Identifier: Apache-2.0
+//
+
+#pragma once
+
+#include <cstddef>
+#include <cstdint>
+#include <cstring>
+
+#include "openvino/core/except.hpp"
+#include "openvino/core/type/float16.hpp"
+
+namespace ov::reference {
+namespace gguf {
+
+/// \brief GGUF k-quant block formats (ggml block layouts, little-endian).
+enum class Format { Q4_K, Q5_K, Q6_K };
+
+constexpr size_t block_size = 256;
+
+/// \brief Bytes of one 256-value block: 144 (Q4_K), 176 (Q5_K), 210 (Q6_K).
+constexpr size_t block_bytes(Format format) {
+    return format == Format::Q4_K ? 144 : format == Format::Q5_K ? 176 : 210;
+}
+
+inline float load_f16(const uint8_t* p) {
+    uint16_t bits;
+    std::memcpy(&bits, p, sizeof(bits));
+    return static_cast<float>(ov::float16::from_bits(bits));
+}
+
+// 6-bit scale and min of sub-block j (0..7) from the 12 packed bytes of a Q4_K / Q5_K block (ggml get_scale_min_k4).
+inline void scale_min_k4(size_t j, const uint8_t* q, uint8_t& sc, uint8_t& m) {
+    if (j < 4) {
+        sc = q[j] & 63;
+        m = q[j + 4] & 63;
+    } else {
+        sc = (q[j + 4] & 0xF) | ((q[j - 4] >> 6) << 4);
+        m = (q[j + 4] >> 4) | ((q[j] >> 6) << 4);
+    }
+}
+
+// Decodes one Q4_K or Q5_K block of 256 values; q5: the block has the Q5_K plane of 5th bits.
+template <typename T>
+void dequantize_block_k4(const uint8_t* blk, bool q5, T* y) {
+    const float d = load_f16(blk);
+    const float dmin = load_f16(blk + 2);
+    const uint8_t* scales = blk + 4;
+    const uint8_t* qh = q5 ? blk + 16 : nullptr;
+    const uint8_t* ql = blk + (q5 ? 48 : 16);
+    for (size_t j = 0; j < block_size / 64; ++j) {
+        uint8_t sc, m;
+        scale_min_k4(2 * j, scales, sc, m);
+        const float d1 = d * sc, m1 = dmin * m;
+        scale_min_k4(2 * j + 1, scales, sc, m);
+        const float d2 = d * sc, m2 = dmin * m;
+        for (size_t l = 0; l < 32; ++l) {
+            int lo = ql[l] & 0xF;
+            int hi = ql[l] >> 4;
+            if (qh) {
+                lo += (qh[l] >> (2 * j)) & 1 ? 16 : 0;
+                hi += (qh[l] >> (2 * j + 1)) & 1 ? 16 : 0;
+            }
+            y[l] = static_cast<T>(d1 * static_cast<float>(lo) - m1);
+            y[l + 32] = static_cast<T>(d2 * static_cast<float>(hi) - m2);
+        }
+        ql += 32;
+        y += 64;
+    }
+}
+
+template <typename T>
+void dequantize_block_q6_k(const uint8_t* blk, T* y) {
+    const uint8_t* ql = blk;
+    const uint8_t* qh = blk + 128;
+    const auto* sc = reinterpret_cast<const int8_t*>(blk + 192);
+    const float d = load_f16(blk + 208);
+    for (size_t n = 0; n < block_size / 128; ++n) {
+        for (size_t l = 0; l < 32; ++l) {
+            const size_t is = l / 16;
+            const int q1 = static_cast<int>((ql[l] & 0xF) | (((qh[l] >> 0) & 3) << 4)) - 32;
+            const int q2 = static_cast<int>((ql[l + 32] & 0xF) | (((qh[l] >> 2) & 3) << 4)) - 32;
+            const int q3 = static_cast<int>((ql[l] >> 4) | (((qh[l] >> 4) & 3) << 4)) - 32;
+            const int q4 = static_cast<int>((ql[l + 32] >> 4) | (((qh[l] >> 6) & 3) << 4)) - 32;
+            y[l] = static_cast<T>(d * sc[is + 0] * q1);
+            y[l + 32] = static_cast<T>(d * sc[is + 2] * q2);
+            y[l + 64] = static_cast<T>(d * sc[is + 4] * q3);
+            y[l + 96] = static_cast<T>(d * sc[is + 6] * q4);
+        }
+        y += 128;
+        ql += 64;
+        qh += 32;
+        sc += 8;
+    }
+}
+
+}  // namespace gguf
+
+/// \brief Decodes GGUF Q4_K / Q5_K / Q6_K blocks as stored in a GGUF file.
+///
+/// \param data     n_blocks consecutive blocks of the format
+/// \param out      n_blocks * 256 decoded values
+/// \param n_blocks Number of blocks
+/// \param format   Block format of data
+template <typename T>
+void gguf_dequantize(const uint8_t* data, T* out, size_t n_blocks, gguf::Format format) {
+    const size_t bytes = gguf::block_bytes(format);
+    for (size_t b = 0; b < n_blocks; ++b) {
+        const uint8_t* blk = data + b * bytes;
+        T* y = out + b * gguf::block_size;
+        switch (format) {
+        case gguf::Format::Q4_K:
+            gguf::dequantize_block_k4(blk, false, y);
+            break;
+        case gguf::Format::Q5_K:
+            gguf::dequantize_block_k4(blk, true, y);
+            break;
+        case gguf::Format::Q6_K:
+            gguf::dequantize_block_q6_k(blk, y);
+            break;
+        default:
+            OPENVINO_THROW("Unknown GGUF block format");
+        }
+    }
+}
+
+}  // namespace ov::reference

src/core/reference/sources.cmake

@@ -133,6 +133,7 @@ set(REFERENCE_HEADERS
     ${CMAKE_CURRENT_LIST_DIR}/include/openvino/reference/gather_nd.hpp
     ${CMAKE_CURRENT_LIST_DIR}/include/openvino/reference/gather_tree.hpp
     ${CMAKE_CURRENT_LIST_DIR}/include/openvino/reference/gelu.hpp
+    ${CMAKE_CURRENT_LIST_DIR}/include/openvino/reference/gguf_dequantize.hpp
     ${CMAKE_CURRENT_LIST_DIR}/include/openvino/reference/generate_proposal.hpp
     ${CMAKE_CURRENT_LIST_DIR}/include/openvino/reference/greater.hpp
     ${CMAKE_CURRENT_LIST_DIR}/include/openvino/reference/greater_eq.hpp

src/core/shape_inference/include/gguf_dequantize_shape_inference.hpp

@@ -0,0 +1,52 @@
+// Copyright (C) 2018-2026 Intel Corporation
+// SPDX-License-Identifier: Apache-2.0
+//
+
+#pragma once
+
+#include "openvino/op/gguf_dequantize.hpp"
+#include "utils.hpp"
+
+namespace ov {
+namespace op {
+namespace internal {
+template <class TShape, class TRShape = result_shape_t<TShape>>
+std::vector<TRShape> shape_infer(const GGUFDequantize* op,
+                                 const std::vector<TShape>& input_shapes,
+                                 const ITensorAccessor& = make_tensor_accessor()) {
+    NODE_SHAPE_INFER_CHECK(op, input_shapes, input_shapes.size() == 1);
+
+    // [..., N, K / 256 * block_bytes] -> [..., N, K]
+    auto output_shape = TRShape(input_shapes[0]);
+    if (output_shape.rank().is_dynamic()) {
+        return {std::move(output_shape)};
+    }
+    NODE_SHAPE_INFER_CHECK(op,
+                           input_shapes,
+                           output_shape.size() >= 2,
+                           "The GGUF block data must have rank 2 or more ([..., N, bytes per row]). Got: ",
+                           input_shapes[0]);
+
+    auto& row = output_shape[output_shape.size() - 1];
+    if (row.is_static()) {
+        const auto bytes = static_cast<size_t>(row.get_length());
+        const auto block_bytes = GGUFDequantize::block_bytes(op->get_format());
+        NODE_SHAPE_INFER_CHECK(op,
+                               input_shapes,
+                               bytes % block_bytes == 0,
+                               "Bytes per row (",
+                               bytes,
+                               ") must be a whole number of ",
+                               op->get_format(),
+                               " blocks of ",
+                               block_bytes,
+                               " bytes");
+        row = static_cast<typename TRShape::value_type::value_type>(bytes / block_bytes * GGUFDequantize::block_size);
+    } else {
+        row = typename TRShape::value_type{};  // dynamic (an interval of bytes is not a whole number of blocks)
+    }
+    return {std::move(output_shape)};
+}
+}  // namespace internal
+}  // namespace op
+}  // namespace ov