From 07edf07df3ca23f4080b5e88a3c23258122b7cf6 Mon Sep 17 00:00:00 2001 From: Cyberpunk1210 Date: Tue, 13 Oct 2026 09:31:03 +0800 Subject: [PATCH 01/12] [Transformations] Support u2 expert weights in MoE op fusion Prepares MoeOpFusion for INT2 (u2) MoE models, and stops it fusing layers whose expert GEMMs mix dtypes, which the fused kernel decodes wrongly today. u2 itself stays inert until the GPU kernels land in a follow-up PR: the only caller is the GPU pipeline, and it does not yet offer u2 to ConvertTiledMoeBlockToGatherMatmuls / ConvertGatherMatmulToGatherMatmulCompressed, so no u2 GatherMatmulCompressed reaches this pass. Changes to the compressed branch of the 3-GEMM callback: - Weight dtype gate. Previously any combination fused, but moe_3gemm decodes all three GEMMs with the gate GEMM's dtype, so a layer with e.g. u4 gate/up and u8 down (NNCF mixed precision keeps sensitive experts at INT8) produced wrong output with no error. A layer now fuses only when its three GEMMs share one dtype in {u2, u4, i4, u8, i8}, or mix only u2 and u8 -- the one mix the follow-up kernels decode per GEMM. Other mixes stay in the GatherMatmul form, which is correct. - Scalar zero point. A rank-0 zp constant is a folded per-tensor zero point. An all-zero one is normalized to "no zp" (symmetric); a non-zero one is kept only for u2 GEMMs, and op::MOE validation now skips the num_experts check for rank-0 inputs. Mixing symmetric and asymmetric GEMMs in one layer bails out. - Group size. MOECompressed needs one group size for all three GEMMs, so a per-channel scale (and its u8 zp) is broadcast to the group-wise layout of the others; genuinely different group sizes bail out. Sub-byte zps are not expanded -- they are bit-packed and Type::size() rounds up to a byte -- so that case declines to fuse. - The 2-GEMM matcher applies the same scalar-zp normalization and keeps u2 in GatherMatmul form, since moe_gemm's only implementation (oneDNN) has no u2. Tests: compressed GatherMatmulCompressed variants of the MoE block -- until now every model in this file used plain f32 GatherMatmul, so the whole compressed half of the callback was untested -- covering the dtype admission table, the scalar-zp normalization, group-size unification and the per-channel scale/zp broadcast. Co-Authored-By: Claude Opus 5.5 (1M context) Signed-off-by: Cyberpunk1210 --- .../common_optimizations/moe_op_fusion.cpp | 382 +++++++++++++++--- ...ert_gather_matmuls_moe_block_to_moe_op.cpp | 280 +++++++++++++ src/core/src/op/moe.cpp | 6 +- 3 files changed, 621 insertions(+), 47 deletions(-) diff --git a/src/common/transformations/src/transformations/common_optimizations/moe_op_fusion.cpp b/src/common/transformations/src/transformations/common_optimizations/moe_op_fusion.cpp index d74ecd665392bb..1d447f4dd4cdb9 100644 --- a/src/common/transformations/src/transformations/common_optimizations/moe_op_fusion.cpp +++ b/src/common/transformations/src/transformations/common_optimizations/moe_op_fusion.cpp @@ -4,7 +4,10 @@ #include "transformations/common_optimizations/moe_op_fusion.hpp" +#include #include +#include +#include #include "itt.hpp" #include "openvino/core/graph_util.hpp" @@ -157,22 +160,81 @@ Convert3GatherMatmulMoeBlockToMoeOp::Convert3GatherMatmulMoeBlockToMoeOp() { const bool is_compressed = is_gate_compressed; if (is_compressed) { - // Build MOECompressed with 12 inputs: hidden, routing, topk, - // gate_w, gate_scale, gate_zp, up_w, up_scale, up_zp, down_w, down_scale, down_zp - ov::OutputVector moe_inputs = { - hidden_states, - routing, - topk_indices, - gate_w, - pm.at(gate_scale_m), - pm.at(gate_zp_m), - up_w, - pm.at(up_scale_m), - pm.at(up_zp_m), - down_w, - pm.at(down_scale_m), - pm.at(down_zp_m), + // A layer that mixes dtypes across the three expert GEMMs is only decoded correctly by + // moe_3gemm for the {u2, u8} combinations its per-GEMM dispatch covers. Anything else + // keeps the per-GEMM-correct GatherMatmul form rather than being mis-decoded. + { + const auto gate_dt = pm.at(gate_w_m).get_element_type(); + const auto up_dt = pm.at(up_w_m).get_element_type(); + const auto down_dt = pm.at(down_w_m).get_element_type(); + const auto is_u2_or_u8 = [](ov::element::Type t) { + return t == ov::element::u2 || t == ov::element::u8; + }; + const bool uniform_ok = + (up_dt == gate_dt && down_dt == gate_dt) && + (gate_dt == ov::element::u2 || gate_dt == ov::element::u4 || gate_dt == ov::element::i4 || + gate_dt == ov::element::u8 || gate_dt == ov::element::i8); + const bool mixed_ok = is_u2_or_u8(gate_dt) && is_u2_or_u8(up_dt) && is_u2_or_u8(down_dt); + if (!(uniform_ok || mixed_ok)) { + return false; + } + } + auto gate_zp = pm.at(gate_zp_m); + auto up_zp = pm.at(up_zp_m); + auto down_zp = pm.at(down_zp_m); + + // Non-zero scalar zp broadcast is only implemented for the u2 batched GEMV + // kernels, so it is retained per-GEMM only when that GEMM's weights are u2. + + // A scalar (rank-0) zp constant is a folded per-tensor zero point. + // An all-zero scalar zp is equivalent to symmetric quantization: normalize + // it to the "absent zp" convention (dynamic-typed empty constant). + // A non-zero scalar zp is passed through unchanged (u2 weights only): + // MOE validation explicitly exempts rank-0 inputs from the num_experts + // check, and the batched GEMV kernels broadcast the single element over + // all experts/groups/channels. + // Anything else (non-constant, packed dtype, or mixed sym/asym forms) + // bails out and leaves the graph in the GatherMatmul form. + auto normalize_scalar_zp = [&](ov::Output& zp, ov::element::Type wt) -> bool { + const auto& zp_shape = zp.get_partial_shape(); + if (!zp_shape.is_static() || zp_shape.rank().get_length() != 0) { + return true; // not a scalar zp: nothing to do + } + const auto zp_const = ov::as_type_ptr(zp.get_node_shared_ptr()); + if (!zp_const) { + return false; + } + // Restrict to byte/wide integer types: cast_vector on packed (u2/u4/i4) + // constants is not meaningful here and may throw. + const auto zp_et = zp_const->get_element_type(); + if (zp_et != ov::element::i8 && zp_et != ov::element::u8 && zp_et != ov::element::i32 && + zp_et != ov::element::u32 && zp_et != ov::element::i64) { + return false; + } + for (const auto v : zp_const->cast_vector()) { + if (v != 0) { + if (wt != ov::element::u2) { + return false; // non-zero scalar zp only supported for u2 weights + } + // Keep the rank-0 scalar constant as-is: MOE validation exempts rank-0 + // inputs from the num_experts check, and the u2 kernels broadcast it. + return true; + } + } + zp = std::make_shared(ov::element::dynamic, ov::Shape{0}); + return true; }; + if (!normalize_scalar_zp(gate_zp, pm.at(gate_w_m).get_element_type()) || + !normalize_scalar_zp(up_zp, pm.at(up_w_m).get_element_type()) || + !normalize_scalar_zp(down_zp, pm.at(down_w_m).get_element_type())) { + return false; + } + const size_t num_dynamic_zp = static_cast(gate_zp.get_element_type() == ov::element::dynamic) + + static_cast(up_zp.get_element_type() == ov::element::dynamic) + + static_cast(down_zp.get_element_type() == ov::element::dynamic); + if (num_dynamic_zp != 0 && num_dynamic_zp != 3) { + return false; // mixed symmetric/asymmetric GEMMs are not supported + } // Populate compressed config from weight shapes auto wei_partial_shape = gate_w.get_partial_shape(); @@ -182,15 +244,118 @@ Convert3GatherMatmulMoeBlockToMoeOp::Convert3GatherMatmulMoeBlockToMoeOp() { auto topk_shape = topk_indices.get_partial_shape(); OPENVINO_ASSERT(topk_shape[1].is_static(), "K dimension in moe topk input should be static."); - // group_size derived from down-scale; weight_logical_K handles rank-3/4. + auto gate_scale = pm.at(gate_scale_m); + auto up_scale = pm.at(up_scale_m); + auto down_scale = pm.at(down_scale_m); + + // group_size derived from scales; weight_logical_K handles rank-3/4. const auto gate_K = weight_logical_K(weight_shape); + const auto up_K = weight_logical_K(pm.at(up_w_m).get_partial_shape().to_shape()); const auto down_K = weight_logical_K(pm.at(down_w_m).get_partial_shape().to_shape()); - const auto down_scale_shape = pm.at(down_scale_m).get_partial_shape().to_shape(); - const size_t down_num_groups = (down_scale_shape.size() >= 3) ? down_scale_shape[2] : 1; - const size_t group_size = - (down_num_groups <= 1) ? std::numeric_limits::max() : (down_K / down_num_groups); + + auto scale_num_groups = [](const ov::Output& scale) -> size_t { + const auto& ps = scale.get_partial_shape(); + if (!ps.is_static()) + return SIZE_MAX; + const auto s = ps.to_shape(); + return (s.size() >= 3) ? s[2] : 1; + }; + const size_t gate_G = scale_num_groups(gate_scale); + const size_t up_G = scale_num_groups(up_scale); + const size_t down_G = scale_num_groups(down_scale); + + // MOECompressed requires one group_size shared by all three GEMMs. + // A per-channel scale ([E, N, 1], group_size == K) is mathematically + // identical to a group-wise scale with the same value repeated across + // groups, so broadcast per-channel scale constants to the group-wise + // num_groups used by the other GEMMs. Genuinely inconsistent group + // sizes (two different group-wise gs) or non-constant scales bail out. + size_t group_size = std::numeric_limits::max(); + for (const auto& kg : {std::make_pair(gate_K, gate_G), {up_K, up_G}, {down_K, down_G}}) { + const size_t K = kg.first; + const size_t G = kg.second; + if (G == SIZE_MAX) + return false; // dynamic scale shape + if (G > 1) { + if (K % G != 0) + return false; + const size_t gs = K / G; + if (group_size == std::numeric_limits::max()) { + group_size = gs; + } else if (group_size != gs) { + return false; // different group-wise group sizes + } + } + } + + // Broadcast a per-channel scale constant [E, N, 1] (or [E, N]) to + // [E, N, K/group_size] by repeating each channel value across groups. + auto broadcast_per_channel_scale = [&](ov::Output& scale, size_t K) -> bool { + const auto c = ov::as_type_ptr(scale.get_node_shared_ptr()); + if (!c) + return false; + const auto s = c->get_shape(); + if (s.size() < 2 || K % group_size != 0) + return false; + const size_t E = s[0]; + const size_t N = s[1]; + const size_t target_G = K / group_size; + if (target_G == 1) + return true; // nothing to expand + const auto et = c->get_element_type(); + // Sub-byte types are bit-packed and Type::size() rounds up to a whole byte, so the + // per-element memcpy below would read past the constant and interleave neighbouring + // channels. Fall back to the unfused form rather than materialise corrupt values. + if (et.bitwidth() < 8) + return false; + const size_t esz = et.size(); + const auto* src = static_cast(c->get_data_ptr()); + std::vector buf(E * N * target_G * esz); + const size_t total = E * N * target_G; + for (size_t i = 0; i < total; ++i) { + std::memcpy(buf.data() + i * esz, src + (i / target_G) * esz, esz); + } + scale = std::make_shared(et, ov::Shape{E, N, target_G}, buf.data()); + return true; + }; + if (group_size != std::numeric_limits::max()) { + if ((gate_G == 1 && !broadcast_per_channel_scale(gate_scale, gate_K)) || + (up_G == 1 && !broadcast_per_channel_scale(up_scale, up_K)) || + (down_G == 1 && !broadcast_per_channel_scale(down_scale, down_K))) { + return false; + } + // A3: broadcast a per-channel (u8) zp [E, N, 1] to [E, N, K/group_size], the + // same expansion as its scale, so zshape == sshape and the byfx reorder plus + // in-kernel (group, channel) zp indexing line up. Scalar (u2) zps are rank-0 + // and were normalized above; they are left untouched here. + if (num_dynamic_zp == 0) { + if ((gate_G == 1 && !broadcast_per_channel_scale(gate_zp, gate_K)) || + (up_G == 1 && !broadcast_per_channel_scale(up_zp, up_K)) || + (down_G == 1 && !broadcast_per_channel_scale(down_zp, down_K))) { + return false; + } + } + } + // dynamic-typed zp = symmetric placeholder. - const bool has_zp = pm.at(gate_zp_m).get_element_type() != ov::element::dynamic; + const bool has_zp = num_dynamic_zp == 0; + + // Build MOECompressed with 12 inputs: hidden, routing, topk, + // gate_w, gate_scale, gate_zp, up_w, up_scale, up_zp, down_w, down_scale, down_zp + ov::OutputVector moe_inputs = { + hidden_states, + routing, + topk_indices, + gate_w, + gate_scale, + gate_zp, + up_w, + up_scale, + up_zp, + down_w, + down_scale, + down_zp, + }; MOECompressed::Config compressed_config{ {ov::op::internal::MOE::Expert_type::GEMM3_SWIGLU, 0.0f, expert_beta, 0, activation_type}, @@ -369,6 +534,153 @@ Convert2GatherMatmulMoeBlockToMoeOp::Convert2GatherMatmulMoeBlockToMoeOp() { const bool is_compressed = is_gate_up_compressed; if (is_compressed) { + // The GEMM2 fused path lowers to cldnn::moe_gemm, whose only registered + // impl (oneDNN) does not accept u2 weights. Keep u2 in GatherMatmul form + // until moe_gemm gains u2 support, otherwise compilation fails. + if (pm.at(gate_up_w_m).get_element_type() == ov::element::u2 || + pm.at(down_w_m).get_element_type() == ov::element::u2) { + return false; + } + auto gate_up_zp = pm.at(gate_up_zp_m); + auto down_zp = pm.at(down_zp_m); + + // Non-zero scalar zp broadcast is only implemented for the u2 fused + // kernels, so it is retained per-GEMM only when that GEMM's weights are u2. + + // Same scalar-zp normalization as the 3-GEMM matcher above. + auto normalize_scalar_zp = [&](ov::Output& zp, ov::element::Type wt) -> bool { + const auto& zp_shape = zp.get_partial_shape(); + if (!zp_shape.is_static() || zp_shape.rank().get_length() != 0) { + return true; // not a scalar zp: nothing to do + } + const auto zp_const = ov::as_type_ptr(zp.get_node_shared_ptr()); + if (!zp_const) { + return false; + } + // Restrict to byte/wide integer types: cast_vector on packed (u2/u4/i4) + // constants is not meaningful here and may throw. + const auto zp_et = zp_const->get_element_type(); + if (zp_et != ov::element::i8 && zp_et != ov::element::u8 && zp_et != ov::element::i32 && + zp_et != ov::element::u32 && zp_et != ov::element::i64) { + return false; + } + for (const auto v : zp_const->cast_vector()) { + if (v != 0) { + if (wt != ov::element::u2) { + return false; // non-zero scalar zp only supported for u2 weights + } + // Keep the rank-0 scalar constant as-is: the fused backend + // broadcasts the single element over all experts/groups/channels. + return true; + } + } + zp = std::make_shared(ov::element::dynamic, ov::Shape{0}); + return true; + }; + if (!normalize_scalar_zp(gate_up_zp, pm.at(gate_up_w_m).get_element_type()) || + !normalize_scalar_zp(down_zp, pm.at(down_w_m).get_element_type())) { + return false; + } + const size_t num_dynamic_zp = static_cast(gate_up_zp.get_element_type() == ov::element::dynamic) + + static_cast(down_zp.get_element_type() == ov::element::dynamic); + if (num_dynamic_zp != 0 && num_dynamic_zp != 2) { + return false; // mixed symmetric/asymmetric GEMMs are not supported + } + + // Populate compressed config from weight shapes + auto weight_shape = gate_up_w.get_shape(); + const size_t hidden = weight_logical_K(weight_shape); + + auto topk_indices_shape = topk_indices.get_partial_shape(); + auto topk_rank = topk_indices_shape.rank().get_length(); + OPENVINO_ASSERT(topk_indices_shape[topk_rank - 1].is_static(), + "K dimension in moe topk_indices input should be static."); + + auto gate_up_scale = pm.at(gate_up_scale_m); + auto down_scale = pm.at(down_scale_m); + + // group_size derived from scales; weight_logical_K handles rank-3/4. + const auto down_K = weight_logical_K(pm.at(down_w_m).get_partial_shape().to_shape()); + + auto scale_num_groups = [](const ov::Output& scale) -> size_t { + const auto& ps = scale.get_partial_shape(); + if (!ps.is_static()) + return SIZE_MAX; + const auto s = ps.to_shape(); + return (s.size() >= 3) ? s[2] : 1; + }; + const size_t gate_up_G = scale_num_groups(gate_up_scale); + const size_t down_G = scale_num_groups(down_scale); + + // Same group_size unification as the 3-GEMM matcher above, over two GEMMs. + size_t group_size = std::numeric_limits::max(); + for (const auto& kg : {std::make_pair(hidden, gate_up_G), {down_K, down_G}}) { + const size_t K = kg.first; + const size_t G = kg.second; + if (G == SIZE_MAX) + return false; // dynamic scale shape + if (G > 1) { + if (K % G != 0) + return false; + const size_t gs = K / G; + if (group_size == std::numeric_limits::max()) { + group_size = gs; + } else if (group_size != gs) { + return false; // different group-wise group sizes + } + } + } + + // Broadcast a per-channel scale constant [E, N, 1] (or [E, N]) to + // [E, N, K/group_size] by repeating each channel value across groups. + auto broadcast_per_channel_scale = [&](ov::Output& scale, size_t K) -> bool { + const auto c = ov::as_type_ptr(scale.get_node_shared_ptr()); + if (!c) + return false; + const auto s = c->get_shape(); + if (s.size() < 2 || K % group_size != 0) + return false; + const size_t E = s[0]; + const size_t N = s[1]; + const size_t target_G = K / group_size; + if (target_G == 1) + return true; // nothing to expand + const auto et = c->get_element_type(); + // Sub-byte types are bit-packed and Type::size() rounds up to a whole byte, so the + // per-element memcpy below would read past the constant and interleave neighbouring + // channels. Fall back to the unfused form rather than materialise corrupt values. + if (et.bitwidth() < 8) + return false; + const size_t esz = et.size(); + const auto* src = static_cast(c->get_data_ptr()); + std::vector buf(E * N * target_G * esz); + const size_t total = E * N * target_G; + for (size_t i = 0; i < total; ++i) { + std::memcpy(buf.data() + i * esz, src + (i / target_G) * esz, esz); + } + scale = std::make_shared(et, ov::Shape{E, N, target_G}, buf.data()); + return true; + }; + if (group_size != std::numeric_limits::max()) { + if ((gate_up_G == 1 && !broadcast_per_channel_scale(gate_up_scale, hidden)) || + (down_G == 1 && !broadcast_per_channel_scale(down_scale, down_K))) { + return false; + } + // Broadcast a per-channel (u8) zp [E, N, 1] to [E, N, K/group_size], the + // same expansion as its scale, so zshape == sshape and the (group, channel) + // zp indexing line up. Scalar (u2) zps are rank-0 and were normalized + // above; they are left untouched here. + if (num_dynamic_zp == 0) { + if ((gate_up_G == 1 && !broadcast_per_channel_scale(gate_up_zp, hidden)) || + (down_G == 1 && !broadcast_per_channel_scale(down_zp, down_K))) { + return false; + } + } + } + + // dynamic-typed zp = symmetric placeholder. + const bool has_zp = num_dynamic_zp == 0; + // Build MOECompressed inputs // GEMM2 compressed layout: hidden, routing, topk, // gate_up_w, gate_up_scale, [gate_up_zp,] gate_up_bias, @@ -378,42 +690,22 @@ Convert2GatherMatmulMoeBlockToMoeOp::Convert2GatherMatmulMoeBlockToMoeOp() { moe_inputs.push_back(routing); moe_inputs.push_back(topk_indices); - // Absent zp is represented as Constant(element::dynamic, Shape{0}) - bool has_zp = pm.at(gate_up_zp_m).get_element_type() != ov::element::dynamic; - // gate_up params moe_inputs.push_back(gate_up_w); - moe_inputs.push_back(pm.at(gate_up_scale_m)); + moe_inputs.push_back(gate_up_scale); if (has_zp) { - moe_inputs.push_back(pm.at(gate_up_zp_m)); + moe_inputs.push_back(gate_up_zp); } moe_inputs.push_back(gate_up_bias); // down params moe_inputs.push_back(down_w); - moe_inputs.push_back(pm.at(down_scale_m)); + moe_inputs.push_back(down_scale); if (has_zp) { - moe_inputs.push_back(pm.at(down_zp_m)); + moe_inputs.push_back(down_zp); } moe_inputs.push_back(down_bias); - // Populate compressed config from weight shapes - auto weight_shape = gate_up_w.get_shape(); - auto scale_shape = pm.at(gate_up_scale_m).get_shape(); - const size_t hidden = weight_logical_K(weight_shape); - - auto topk_indices_shape = topk_indices.get_partial_shape(); - auto topk_rank = topk_indices_shape.rank().get_length(); - OPENVINO_ASSERT(topk_indices_shape[topk_rank - 1].is_static(), - "K dimension in moe topk_indices input should be static."); - - // group_size derived from down-scale; weight_logical_K handles rank-3/4. - const auto down_K = weight_logical_K(pm.at(down_w_m).get_partial_shape().to_shape()); - const auto down_scale_shape = pm.at(down_scale_m).get_partial_shape().to_shape(); - const size_t down_num_groups = (down_scale_shape.size() >= 3) ? down_scale_shape[2] : 1; - const size_t group_size = - (down_num_groups <= 1) ? std::numeric_limits::max() : (down_K / down_num_groups); - MOECompressed::Config compressed_config{ {ov::op::internal::MOE::Expert_type::GEMM2_BIAS_SWIGLU_CLAMP, expert_alpha, expert_beta, gate_idx}, hidden, diff --git a/src/common/transformations/tests/common_optimizations/convert_gather_matmuls_moe_block_to_moe_op.cpp b/src/common/transformations/tests/common_optimizations/convert_gather_matmuls_moe_block_to_moe_op.cpp index 115a0cbb28fe0b..e1069b75ff6a81 100644 --- a/src/common/transformations/tests/common_optimizations/convert_gather_matmuls_moe_block_to_moe_op.cpp +++ b/src/common/transformations/tests/common_optimizations/convert_gather_matmuls_moe_block_to_moe_op.cpp @@ -4,6 +4,7 @@ #include +#include #include #include #include @@ -35,10 +36,14 @@ #include "openvino/op/transpose.hpp" #include "openvino/op/unsqueeze.hpp" #include "ov_ops/gather_matmul.hpp" +#include "ov_ops/gather_matmul_compressed.hpp" +#include "ov_ops/moe_compressed.hpp" #include "transformations/common_optimizations/convert_tiled_moe_block_to_gather_matmuls.hpp" #include "transformations/common_optimizations/moe_op_fusion.hpp" using GatherMatmul = ov::op::internal::GatherMatmul; +using GatherMatmulCompressed = ov::op::internal::GatherMatmulCompressed; +using MOECompressed = ov::op::internal::MOECompressed; // ============================================================================ // IR model builders (original MOE pattern before any transformation) @@ -837,3 +842,278 @@ TEST_F(TransformationTestsF, Convert2GatherMatmulMoeBlockToMoeOp_restores_output manager.register_pass(); model_ref = build_2gemm_bgm_to_moe_reference_model(true, true); } + +// Compressed (GatherMatmulCompressed) variants. Every builder above produces PLAIN GatherMatmul +// with f32 weights, so the entire compressed half of the callback -- the weight-dtype admission +// gate, the scalar-zp normalization, the group_size unification and the per-channel scale/zp +// broadcast -- is unreachable from the tests in this file. The models below exist to reach it. + +namespace { + +// The compressed branch only inspects weight dtypes and scale/zp shapes, so the smallest shapes +// that still express the interesting cases are enough. gate/up carry 4 scale groups over K=16 and +// down carries 2 over K=8, i.e. both sides must unify to the same group_size of 4 -- a single G +// everywhere would not exercise that. +constexpr size_t kE = 2; +constexpr size_t kHidden = 16; +constexpr size_t kInter = 8; +constexpr size_t kTopk = 2; +constexpr size_t kGroupSize = 4; + +// An absent optional input. GatherMatmul itself spells a missing bias this way; note it has to be +// the 2-argument Constant ctor, since Constant::create cannot produce a dynamic element type. +std::shared_ptr absent_input() { + return std::make_shared(ov::element::dynamic, ov::Shape{0}); +} + +// Quantized values are never dequantized by these tests, but they must still be distinct per +// element so that a wrong broadcast/expansion is visible rather than masked by a uniform fill. +// Constant::create range-checks every element, and u2 holds only 0..3, so the pattern is clamped +// to whatever the requested type can actually represent. +std::shared_ptr int_const(ov::element::Type dt, const ov::Shape& shape, int32_t base = 0) { + const int32_t hi = + dt.bitwidth() < 8 ? static_cast((1u << (dt.bitwidth() - (dt.is_signed() ? 1 : 0))) - 1) : 127; + const size_t n = ov::shape_size(shape); + std::vector v(n); + for (size_t i = 0; i < n; ++i) { + v[i] = std::min(base + static_cast(i % 4), hi); + } + return ov::op::v0::Constant::create(dt, shape, v); +} + +std::shared_ptr scale_const(const ov::Shape& shape) { + const size_t n = ov::shape_size(shape); + std::vector v(n); + for (size_t i = 0; i < n; ++i) { + v[i] = 0.5f + 0.25f * static_cast(i); + } + return ov::op::v0::Constant::create(ov::element::f16, shape, v); +} + +struct CompressedMoeSpec { + ov::element::Type gate_dt = ov::element::u4; + ov::element::Type up_dt = ov::element::u4; + ov::element::Type down_dt = ov::element::u4; + // Scale (and zp) group counts. G == 1 means per-channel, which is what arms the + // broadcast_per_channel_scale path; G > 1 is group-wise and sets group_size. + size_t gate_G = kHidden / kGroupSize; + size_t up_G = kHidden / kGroupSize; + size_t down_G = kInter / kGroupSize; + // element::dynamic == no zp at all (symmetric). Anything else builds real zps whose shape + // mirrors the corresponding scale. + ov::element::Type zp_dt = ov::element::dynamic; +}; + +// Same topology as build_3gemm_bgm_model, with the three GatherMatmul replaced by +// GatherMatmulCompressed. The Parameter stays f32 on purpose: the router MatMul below multiplies +// by an f32 Constant and v0::MatMul requires both operands to share an element type, so making +// the Parameter f16 throws inside this builder before the pass ever runs. +std::shared_ptr build_3gemm_bgm_compressed_model(const CompressedMoeSpec& spec) { + using namespace ov; + + auto input = std::make_shared(element::f32, PartialShape{2, Dimension::dynamic(), kHidden}); + auto experts_reshape = std::make_shared( + input, + op::v0::Constant::create(element::i64, Shape{2}, std::vector{-1, static_cast(kHidden)}), + false); + auto unsqueeze = + std::make_shared(experts_reshape, op::v0::Constant::create(element::i32, Shape{}, {0})); + + auto router_matmul = + std::make_shared(experts_reshape, + op::v0::Constant::create(element::f32, Shape{kE, kHidden}, {1.0f}), + false, + true); + auto router_topk = std::make_shared(router_matmul, + op::v0::Constant::create(element::i64, Shape{}, {kTopk}), + -1, + op::v11::TopK::Mode::MAX, + op::v11::TopK::SortType::SORT_VALUES, + element::i64); + auto topk_indices = router_topk->output(1); + auto chosen_experts = router_topk->output(0); + + auto gate_w = int_const(spec.gate_dt, Shape{kE, kInter, kHidden}); + auto up_w = int_const(spec.up_dt, Shape{kE, kInter, kHidden}, 1); + auto down_w = int_const(spec.down_dt, Shape{kE, kHidden, kInter}, 2); + + auto gate_scale = scale_const(Shape{kE, kInter, spec.gate_G}); + auto up_scale = scale_const(Shape{kE, kInter, spec.up_G}); + auto down_scale = scale_const(Shape{kE, kHidden, spec.down_G}); + + const bool has_zp = spec.zp_dt != element::dynamic; + // 10/11/12 rather than 0..3: a zp of 0 is indistinguishable from the symmetric case, and T6 + // asserts on the expanded values. + auto gate_zp = has_zp ? int_const(spec.zp_dt, Shape{kE, kInter, spec.gate_G}, 10) : absent_input(); + auto up_zp = has_zp ? int_const(spec.zp_dt, Shape{kE, kInter, spec.up_G}, 11) : absent_input(); + auto down_zp = has_zp ? int_const(spec.zp_dt, Shape{kE, kHidden, spec.down_G}, 12) : absent_input(); + + auto bgm_gate = + std::make_shared(unsqueeze, gate_w, topk_indices, absent_input(), gate_scale, gate_zp); + auto gate_act = std::make_shared(bgm_gate); + auto bgm_up = + std::make_shared(unsqueeze, up_w, topk_indices, absent_input(), up_scale, up_zp); + auto swiglu = std::make_shared(gate_act, bgm_up); + auto bgm_down = + std::make_shared(swiglu, down_w, topk_indices, absent_input(), down_scale, down_zp); + + auto router_transpose = std::make_shared( + chosen_experts, + op::v0::Constant::create(element::i64, Shape{2}, std::vector{1, 0})); + auto router_unsqueeze = + std::make_shared(router_transpose, op::v0::Constant::create(element::i32, Shape{}, {-1})); + + auto final_mul = std::make_shared(bgm_down, router_unsqueeze); + auto reduce_sum = + std::make_shared(final_mul, + op::v0::Constant::create(element::i64, Shape{1}, std::vector{0}), + false); + auto end_reshape = std::make_shared( + reduce_sum, + op::v0::Constant::create(element::i64, Shape{3}, std::vector{2, -1, static_cast(kHidden)}), + true); + + return std::make_shared(ov::OutputVector{end_reshape}, ov::ParameterVector{input}); +} + +// Runs the 3-GEMM pass and returns the fused node, or nullptr if the pass declined. +std::shared_ptr run_3gemm_compressed(const std::shared_ptr& model) { + ov::pass::Manager manager; + manager.register_pass(); + manager.run_passes(model); + for (const auto& op : model->get_ops()) { + if (auto moe = ov::as_type_ptr(op)) { + return moe; + } + } + return nullptr; +} + +} // namespace + +// C6. Uniform i8 is the one and only dtype the admission gate regressed: upstream has no gate in +// the compressed branch at all, so an i8 MoE layer fused before this PR and stopped fusing on it. +// After the fix the accepted uniform set is exactly the set the producing pass can emit +// (supported_compressed_weights_types_with_u2 = {u4, i4, i8, u8, u2}). +TEST(Convert3GatherMatmulMoeBlockToMoeOpCompressed, uniform_i8_fuses) { + CompressedMoeSpec spec; + spec.gate_dt = spec.up_dt = spec.down_dt = ov::element::i8; + auto model = build_3gemm_bgm_compressed_model(spec); + + ASSERT_EQ(count_ops_of_type(model), 3); + auto moe = run_3gemm_compressed(model); + + ASSERT_NE(moe, nullptr) << "uniform i8 must fuse into MOECompressed"; + EXPECT_EQ(count_ops_of_type(model), 0); + const auto& cfg = moe->get_config(); + EXPECT_EQ(cfg.group_size, kGroupSize); + EXPECT_EQ(cfg.hidden_size, kHidden); + EXPECT_EQ(cfg.inter_size, kInter); + EXPECT_EQ(cfg.num_expert, kE); + EXPECT_EQ(cfg.top_k, kTopk); + EXPECT_FALSE(cfg.has_zp); +} + +// C6. The full admission table. Uniform is allowed for every dtype the producer can emit; mixing +// is allowed only within {u2, u8}, which is what moe_3gemm's per-GEMM dispatch actually decodes. +struct DtypeGateParams { + ov::element::Type gate_dt; + ov::element::Type up_dt; + ov::element::Type down_dt; + bool should_fuse; +}; + +class MoeOpFusionDtypeGate : public ::testing::TestWithParam {}; + +TEST_P(MoeOpFusionDtypeGate, admits_uniform_and_u2_u8_mixes_only) { + const auto& p = GetParam(); + CompressedMoeSpec spec; + spec.gate_dt = p.gate_dt; + spec.up_dt = p.up_dt; + spec.down_dt = p.down_dt; + + auto model = build_3gemm_bgm_compressed_model(spec); + auto moe = run_3gemm_compressed(model); + + if (p.should_fuse) { + EXPECT_NE(moe, nullptr) << p.gate_dt << "/" << p.up_dt << "/" << p.down_dt << " should fuse"; + } else { + EXPECT_EQ(moe, nullptr) << p.gate_dt << "/" << p.up_dt << "/" << p.down_dt << " should NOT fuse"; + // Declining must leave the per-GEMM-correct form in place, not a half-rewritten graph. + EXPECT_EQ(count_ops_of_type(model), 3); + } +} + +INSTANTIATE_TEST_SUITE_P(smoke, + MoeOpFusionDtypeGate, + ::testing::Values( + // Uniform: every dtype the producing pass can emit. + DtypeGateParams{ov::element::u2, ov::element::u2, ov::element::u2, true}, + DtypeGateParams{ov::element::u4, ov::element::u4, ov::element::u4, true}, + DtypeGateParams{ov::element::i4, ov::element::i4, ov::element::i4, true}, + DtypeGateParams{ov::element::u8, ov::element::u8, ov::element::u8, true}, + DtypeGateParams{ov::element::i8, ov::element::i8, ov::element::i8, true}, + // Mixed within {u2, u8}: the combinations the real NNCF mixed-precision models produce. + DtypeGateParams{ov::element::u2, ov::element::u8, ov::element::u2, true}, + DtypeGateParams{ov::element::u8, ov::element::u2, ov::element::u8, true}, + DtypeGateParams{ov::element::u2, ov::element::u2, ov::element::u8, true}, + // Mixed involving anything else: no per-GEMM decode exists, so it must stay unfused. + DtypeGateParams{ov::element::u4, ov::element::u8, ov::element::u8, false}, + DtypeGateParams{ov::element::i8, ov::element::u8, ov::element::i8, false}, + DtypeGateParams{ov::element::u2, ov::element::u4, ov::element::u2, false}, + DtypeGateParams{ov::element::u4, ov::element::i4, ov::element::u4, false}, + // Not producible by the pipeline today; documents that the gate is a whitelist. + DtypeGateParams{ov::element::f16, ov::element::f16, ov::element::f16, false})); + +// C7. A per-channel zp coexisting with a group-wise GEMM sends the zp through +// broadcast_per_channel_scale, whose per-element memcpy uses Type::size() -- which rounds a +// sub-byte type up to a whole byte. Without the guard the expansion reads past the constant and +// interleaves neighbouring channels, and the result still passes MOECompressed validation, so the +// layer fuses with silently wrong zero points. The fix declines instead. +TEST(Convert3GatherMatmulMoeBlockToMoeOpCompressed, subbyte_per_channel_zp_declines) { + CompressedMoeSpec spec; + spec.gate_dt = spec.up_dt = spec.down_dt = ov::element::u4; + spec.zp_dt = ov::element::u4; + // gate/up per-channel (G == 1) against a group-wise down: group_size comes out of down as 4, + // so gate/up must be expanded from 1 group to 4 -- that is the call the guard now rejects. + spec.gate_G = 1; + spec.up_G = 1; + + auto model = build_3gemm_bgm_compressed_model(spec); + auto moe = run_3gemm_compressed(model); + + EXPECT_EQ(moe, nullptr) << "a sub-byte per-channel zp must not be expanded"; + EXPECT_EQ(count_ops_of_type(model), 3); +} + +// C7 negative control: the same shape with byte-wide zps is legitimately expandable, and this is +// the only test that asserts the expansion arithmetic rather than just that it was skipped. +// Green on both sides of the fix. +TEST(Convert3GatherMatmulMoeBlockToMoeOpCompressed, u8_per_channel_zp_expands) { + CompressedMoeSpec spec; + spec.gate_dt = spec.up_dt = spec.down_dt = ov::element::u8; + spec.zp_dt = ov::element::u8; + spec.gate_G = 1; + spec.up_G = 1; + + auto model = build_3gemm_bgm_compressed_model(spec); + auto moe = run_3gemm_compressed(model); + + ASSERT_NE(moe, nullptr); + EXPECT_TRUE(moe->get_config().has_zp); + + const size_t target_G = kHidden / kGroupSize; + // input 5 is w0_zp; it started as [E, inter, 1] and must come out as [E, inter, target_G] + // with each channel value repeated across the groups. + auto gate_zp = ov::as_type_ptr(moe->input_value(5).get_node_shared_ptr()); + ASSERT_NE(gate_zp, nullptr); + EXPECT_EQ(gate_zp->get_shape(), (ov::Shape{kE, kInter, target_G})); + const auto expanded = gate_zp->cast_vector(); + ASSERT_EQ(expanded.size(), kE * kInter * target_G); + for (size_t ch = 0; ch < kE * kInter; ++ch) { + const int32_t expected = 10 + static_cast(ch % 4); + for (size_t g = 0; g < target_G; ++g) { + EXPECT_EQ(expanded[ch * target_G + g], expected) << "channel " << ch << " group " << g; + } + } +} diff --git a/src/core/src/op/moe.cpp b/src/core/src/op/moe.cpp index 47e5ccafc3c2c3..b1cbc8827047d5 100644 --- a/src/core/src/op/moe.cpp +++ b/src/core/src/op/moe.cpp @@ -54,8 +54,10 @@ void MOE::validate_and_infer_types() { for (size_t i = 4; i < base_inputs_count; i++) { const auto& ps = get_input_partial_shape(i); NODE_VALIDATION_CHECK(this, ps.is_static(), "Weights must have static shape."); - // Note: dynamic element type means empty zero point input - if (get_input_element_type(i) != ov::element::dynamic) { + // Note: dynamic element type means empty zero point input; + // a scalar (rank-0) input is a folded per-tensor zero point which + // has no num_experts dimension to check. + if (get_input_element_type(i) != ov::element::dynamic && ps.size() > 0) { NODE_VALIDATION_CHECK(this, num_experts == get_input_shape(i)[0], "All weight inputs must have the same first dimension (num_experts)."); From efd2d1acecb1c689266de2c2c436fdd2f05aec14 Mon Sep 17 00:00:00 2001 From: Cyberpunk1210 Date: Tue, 20 Oct 2026 19:13:51 +0800 Subject: [PATCH 02/12] [GPU] Decode each GEMM of the fused MoE 3-GEMM with its own weight dtype The moe_3gemm decode kernels took one weight dtype from the gate GEMM (WEIGHT_COMPRESSEION_DT) and derived from it the GEMV variant, the per-expert weight stride and the per-expert zp stride for all three GEMMs. A layer whose GEMMs differ -- NNCF mixed precision produces e.g. gate/up u8 with down u4 -- therefore decoded at least one GEMM with the wrong dtype and the wrong stride: wrong output, no error. Emit GATE/UP/DOWN_WEIGHT_DT and use them for the GEMV dispatch, the expert weight stride and the zp stride of each GEMM. The activation interleave stays keyed on one dtype per kernel: gate and up share one SLM copy whose layout is picked from the gate dtype, so they must still agree on whether they are u4; down stages its own copy and is free. The prefill paths already took each GEMM's dtype. Also fix the host-side GEMV check: FAKE_GROUP_SIZE in the kernel is the minimum of the gate/up and the down group size, but the assert only looked at gate/up, so a smaller down group slipped through to an ELEMS_PER_LANE with no kernel branch. Tests: moe_3gemm_compressed_gpu_mixed_dtype, gate/up u8 + down u4 and gate/up u4 + down u8, for single-token decode, batched GEMV and just above the batched-GEMV threshold, against a reference built from the exact dequantized weights. Co-Authored-By: Claude Opus 5.5 (1M context) Signed-off-by: Cyberpunk1210 --- .../impls/ocl_v2/moe/moe_3gemm_swiglu_opt.cpp | 22 +- .../impls/ocl_v2/moe_3gemm_swiglu_mlp.cl | 70 ++++--- .../unit/test_cases/moe_3gemm_gpu_test.cpp | 189 ++++++++++++++++++ 3 files changed, 246 insertions(+), 35 deletions(-) diff --git a/src/plugins/intel_gpu/src/graph/impls/ocl_v2/moe/moe_3gemm_swiglu_opt.cpp b/src/plugins/intel_gpu/src/graph/impls/ocl_v2/moe/moe_3gemm_swiglu_opt.cpp index 05a746e44411da..086627c57d9523 100644 --- a/src/plugins/intel_gpu/src/graph/impls/ocl_v2/moe/moe_3gemm_swiglu_opt.cpp +++ b/src/plugins/intel_gpu/src/graph/impls/ocl_v2/moe/moe_3gemm_swiglu_opt.cpp @@ -429,10 +429,12 @@ static void add_common_consts(const RuntimeParams& params, JitConstants& jit) { // Smaller values have no kernel branch and would silently produce wrong results. { const size_t sg = (info.arch >= gpu_arch::xe2) ? 32u : 16u; - const size_t fake_gs = std::min(gate_up_group_size, size_t{128}); + // Mirrors FAKE_GROUP_SIZE in moe_3gemm_swiglu_mlp.cl, which also takes the down group size; + // checking gate/up alone let a smaller down group through to a branch that does not exist. + const size_t fake_gs = std::min({gate_up_group_size, down_group_size, size_t{128}}); OPENVINO_ASSERT(fake_gs >= 2 * sg, "MoE GEMV kernel does not support group_size=", - gate_up_group_size, + fake_gs, " on this hardware (SUBGROUP_SIZE=", sg, "). Minimum supported group_size is ", @@ -474,6 +476,22 @@ static void add_common_consts(const RuntimeParams& params, JitConstants& jit) { jit.make("MOE_SCALE_DT", "half"); // not use jit.make("MOE_ZP_DT", "half"); // not use } + + // Per-GEMM weight dtype codes: gate, up and down may differ under NNCF mixed precision, and + // each is decoded with its own. Codes mirror WEIGHT_COMPRESSEION_DT: 0=u4/i4, 1=u8/i8, 2=f16. + const auto moe_weight_code = [](ov::element::Type dt) -> int { + if (dt == ov::element::u4 || dt == ov::element::i4) + return 0; + if (dt == ov::element::f16) + return 2; + return 1; // u8/i8 (byte) path + }; + const auto gate_wdt = params.get_input_layout(static_cast(MOE3GemmInputIndex::WEIGHT_0)).data_type; + const auto up_wdt = params.get_input_layout(static_cast(MOE3GemmInputIndex::WEIGHT_1)).data_type; + const auto down_wdt = params.get_input_layout(static_cast(MOE3GemmInputIndex::WEIGHT_2)).data_type; + jit.make("GATE_WEIGHT_DT", moe_weight_code(gate_wdt)); + jit.make("UP_WEIGHT_DT", moe_weight_code(up_wdt)); + jit.make("DOWN_WEIGHT_DT", moe_weight_code(down_wdt)); } class MoE3GemmSwigluMLPGateUp : public KernelGenerator { diff --git a/src/plugins/intel_gpu/src/graph/impls/ocl_v2/moe_3gemm_swiglu_mlp.cl b/src/plugins/intel_gpu/src/graph/impls/ocl_v2/moe_3gemm_swiglu_mlp.cl index e65d26b2faec09..269c960f1bd721 100644 --- a/src/plugins/intel_gpu/src/graph/impls/ocl_v2/moe_3gemm_swiglu_mlp.cl +++ b/src/plugins/intel_gpu/src/graph/impls/ocl_v2/moe_3gemm_swiglu_mlp.cl @@ -86,6 +86,13 @@ inline float moe_mlp_fast_erf(float x) { # define DEQUANT_8BIT(v) convert_half(v) #endif +// Per-GEMM helpers (mixed-dtype MoE): per-expert byte strides given a compression code DT +// (0=u4/i4, 1=u8/i8, 2=f16) and group size GS. N*K equals INTERMEDIATE_SIZE*HIDDEN_SIZE for all +// three GEMMs, so one product serves gate/up/down. +#define MOE_WEI_PROD (INTERMEDIATE_SIZE * HIDDEN_SIZE) +#define MOE_EXPERT_WEI_BYTES(DT) ((DT) == 0 ? (MOE_WEI_PROD) / 2 : (MOE_WEI_PROD)) +#define MOE_EXPERT_ZP_BYTES(DT, GS) ((DT) == 0 ? (MOE_WEI_PROD) / 2 / (GS) : (MOE_WEI_PROD) / (GS)) + #if GATE_UP_ENABLE inline void gate_up_gemv_n2x_u4(const __global uchar* weight, __global half* scales, @@ -471,15 +478,12 @@ __attribute__((intel_reqd_sub_group_size(SUBGROUP_SIZE))) KERNEL(mlp_gate_up)( bool is_shared = false; # endif -# if WEIGHT_COMPRESSEION_DT == 0 - const int expert_wei_size = INTERMEDIATE_SIZE * HIDDEN_SIZE / 2; - const int expert_scale_size = INTERMEDIATE_SIZE * HIDDEN_SIZE / GATE_UP_GROUP_SIZE; - const int expert_zp_size = INTERMEDIATE_SIZE * HIDDEN_SIZE / 2 / GATE_UP_GROUP_SIZE; -# else - const int expert_wei_size = INTERMEDIATE_SIZE * HIDDEN_SIZE; - const int expert_scale_size = INTERMEDIATE_SIZE * HIDDEN_SIZE / GATE_UP_GROUP_SIZE; - const int expert_zp_size = INTERMEDIATE_SIZE * HIDDEN_SIZE / GATE_UP_GROUP_SIZE; -# endif + // Per-GEMM strides (gate and up may differ in dtype). + const int gate_expert_wei_size = MOE_EXPERT_WEI_BYTES(GATE_WEIGHT_DT); + const int up_expert_wei_size = MOE_EXPERT_WEI_BYTES(UP_WEIGHT_DT); + const int expert_scale_size = INTERMEDIATE_SIZE * HIDDEN_SIZE / GATE_UP_GROUP_SIZE; // f16 scale, dtype-independent + const int gate_zp_stride = MOE_EXPERT_ZP_BYTES(GATE_WEIGHT_DT, GATE_UP_GROUP_SIZE); + const int up_zp_stride = MOE_EXPERT_ZP_BYTES(UP_WEIGHT_DT, GATE_UP_GROUP_SIZE); int expert_id = 0; // gate, [HIDDEN_SIZE, INTERMEDIATE_SIZE] @@ -494,14 +498,14 @@ __attribute__((intel_reqd_sub_group_size(SUBGROUP_SIZE))) KERNEL(mlp_gate_up)( if (!is_shared) { expert_id = expert_list[token_idx * MAX_TOPK + expert_slot]; // gate, [HIDDEN_SIZE, INTERMEDIATE_SIZE] - gate_weight = (__global MOE_WEI_DT*)(gate_weight_addr + expert_id * expert_wei_size); + gate_weight = (__global MOE_WEI_DT*)(gate_weight_addr + expert_id * gate_expert_wei_size); gate_scale = (__global MOE_SCALE_DT*)(gate_scale_addr + expert_id * expert_scale_size); - gate_zp = (__global MOE_ZP_DT*)(gate_zp_addr + expert_id * expert_zp_size); + gate_zp = (__global MOE_ZP_DT*)(gate_zp_addr + expert_id * gate_zp_stride); // up, [HIDDEN_SIZE, INTERMEDIATE_SIZE] - up_weight = (__global MOE_WEI_DT*)(up_weight_addr + expert_id * expert_wei_size); + up_weight = (__global MOE_WEI_DT*)(up_weight_addr + expert_id * up_expert_wei_size); up_scale = (__global MOE_SCALE_DT*)(up_scale_addr + expert_id * expert_scale_size); - up_zp = (__global MOE_ZP_DT*)(up_zp_addr + expert_id * expert_zp_size); + up_zp = (__global MOE_ZP_DT*)(up_zp_addr + expert_id * up_zp_stride); } # if SHARED_EXPERT_ENABLE else { @@ -533,7 +537,7 @@ __attribute__((intel_reqd_sub_group_size(SUBGROUP_SIZE))) KERNEL(mlp_gate_up)( __local float shared_gate_partial[SUBGROUP_NUM]; // one slot per subgroup for scalar gate reduction # endif -# if WEIGHT_COMPRESSEION_DT == 0 +# if GATE_WEIGHT_DT == 0 //# interleaving x into x2 int id_sg = get_sub_group_id(); int num_sg = get_num_sub_groups(); @@ -625,14 +629,19 @@ __attribute__((intel_reqd_sub_group_size(SUBGROUP_SIZE))) KERNEL(mlp_gate_up)( // shared_gate_out[token_idx] is consumed by the next kernel (mlp_down) — no barrier needed here. # endif -# if WEIGHT_COMPRESSEION_DT == 0 + // Dispatch up (silu=false) then gate (silu=true) by each projection's own dtype. +# if UP_WEIGHT_DT == 0 gate_up_gemv_n2x_u4(up_weight, up_scale, up_zp, y, INTERMEDIATE_SIZE, HIDDEN_SIZE, x2, xg_sum, false); - gate_up_gemv_n2x_u4(gate_weight, gate_scale, gate_zp, y, INTERMEDIATE_SIZE, HIDDEN_SIZE, x2, xg_sum, true); -# elif WEIGHT_COMPRESSEION_DT == 1 +# elif UP_WEIGHT_DT == 1 gate_up_gemv_n2x_u8(up_weight, up_scale, up_zp, y, INTERMEDIATE_SIZE, HIDDEN_SIZE, x2, xg_sum, false); - gate_up_gemv_n2x_u8(gate_weight, gate_scale, gate_zp, y, INTERMEDIATE_SIZE, HIDDEN_SIZE, x2, xg_sum, true); -# elif WEIGHT_COMPRESSEION_DT == 2 +# elif UP_WEIGHT_DT == 2 gate_up_gemv_n2x_f16(up_weight, up_zp, y, INTERMEDIATE_SIZE, HIDDEN_SIZE, x2, false); +# endif +# if GATE_WEIGHT_DT == 0 + gate_up_gemv_n2x_u4(gate_weight, gate_scale, gate_zp, y, INTERMEDIATE_SIZE, HIDDEN_SIZE, x2, xg_sum, true); +# elif GATE_WEIGHT_DT == 1 + gate_up_gemv_n2x_u8(gate_weight, gate_scale, gate_zp, y, INTERMEDIATE_SIZE, HIDDEN_SIZE, x2, xg_sum, true); +# elif GATE_WEIGHT_DT == 2 gate_up_gemv_n2x_f16(gate_weight, gate_zp, y, INTERMEDIATE_SIZE, HIDDEN_SIZE, x2, true); # endif } @@ -996,15 +1005,10 @@ __attribute__((intel_reqd_sub_group_size(SUBGROUP_SIZE))) KERNEL(mlp_down)(const bool is_shared = false; # endif -# if WEIGHT_COMPRESSEION_DT == 0 - const int expert_wei_size = INTERMEDIATE_SIZE * HIDDEN_SIZE / 2; - const int expert_scale_size = INTERMEDIATE_SIZE * HIDDEN_SIZE / DOWN_GROUP_SIZE; - const int expert_zp_size = INTERMEDIATE_SIZE * HIDDEN_SIZE / 2 / DOWN_GROUP_SIZE; -# else - const int expert_wei_size = INTERMEDIATE_SIZE * HIDDEN_SIZE; - const int expert_scale_size = INTERMEDIATE_SIZE * HIDDEN_SIZE / DOWN_GROUP_SIZE; - const int expert_zp_size = INTERMEDIATE_SIZE * HIDDEN_SIZE / DOWN_GROUP_SIZE; -# endif + // Per-GEMM strides for the down projection. + const int expert_wei_size = MOE_EXPERT_WEI_BYTES(DOWN_WEIGHT_DT); + const int expert_scale_size = INTERMEDIATE_SIZE * HIDDEN_SIZE / DOWN_GROUP_SIZE; // f16 scale, dtype-independent + const int down_zp_stride = MOE_EXPERT_ZP_BYTES(DOWN_WEIGHT_DT, DOWN_GROUP_SIZE); int expert_id = 0; // down, [INTERMEDIATE_SIZE, HIDDEN_SIZE] @@ -1017,7 +1021,7 @@ __attribute__((intel_reqd_sub_group_size(SUBGROUP_SIZE))) KERNEL(mlp_down)(const // down, [INTERMEDIATE_SIZE, HIDDEN_SIZE] weight = (__global MOE_WEI_DT*)(down_weight_addr + expert_id * expert_wei_size); scales = (__global MOE_SCALE_DT*)(down_scale_addr + expert_id * expert_scale_size); - zps = (__global MOE_ZP_DT*)(down_zp_addr + expert_id * expert_zp_size); + zps = (__global MOE_ZP_DT*)(down_zp_addr + expert_id * down_zp_stride); } # if SHARED_EXPERT_ENABLE else { @@ -1044,7 +1048,7 @@ __attribute__((intel_reqd_sub_group_size(SUBGROUP_SIZE))) KERNEL(mlp_down)(const __local float xg_sum[1]; // unused placeholder for function signature #endif -# if WEIGHT_COMPRESSEION_DT == 0 +# if DOWN_WEIGHT_DT == 0 //# interleaving x into x2 int id_sg = get_sub_group_id(); int num_sg = get_num_sub_groups(); @@ -1108,11 +1112,11 @@ __attribute__((intel_reqd_sub_group_size(SUBGROUP_SIZE))) KERNEL(mlp_down)(const MOE_DTYPE routing_weight_val = routing_weights[token_idx * MAX_TOPK + expert_slot]; # endif -# if WEIGHT_COMPRESSEION_DT == 0 +# if DOWN_WEIGHT_DT == 0 down_gemv_n2x_u4(weight, scales, zps, routing_weight_val, y, N, K, x2, xg_sum); -# elif WEIGHT_COMPRESSEION_DT == 1 +# elif DOWN_WEIGHT_DT == 1 down_gemv_n2x_u8(weight, scales, zps, routing_weight_val, y, N, K, x2, xg_sum); -# elif WEIGHT_COMPRESSEION_DT == 2 +# elif DOWN_WEIGHT_DT == 2 down_gemv_n2x_f16(weight, routing_weight_val, y, N, K, x2); # endif } diff --git a/src/plugins/intel_gpu/tests/unit/test_cases/moe_3gemm_gpu_test.cpp b/src/plugins/intel_gpu/tests/unit/test_cases/moe_3gemm_gpu_test.cpp index e796ee05c7b3a3..c7ff0313f8ce6d 100644 --- a/src/plugins/intel_gpu/tests/unit/test_cases/moe_3gemm_gpu_test.cpp +++ b/src/plugins/intel_gpu/tests/unit/test_cases/moe_3gemm_gpu_test.cpp @@ -1605,3 +1605,192 @@ TEST(moe_3gemm_expert_mask, shape_predictor_overflow_regression) { total += static_cast(result.batch[e].size()); ASSERT_EQ(total, actual_tokens * top_k); } + +// ============================================================================ +// Mixed weight dtypes across the three expert GEMMs. +// +// The decode kernels took one weight dtype, expert stride and zp stride from the gate GEMM and +// applied them to all three, so a layer with gate/up at u8 and down at u4 decoded down as u8: +// wrong output, no error. Each GEMM is now decoded with its own dtype. gate and up still have to +// agree on whether they are u4, because they read one SLM copy of the activations whose layout the +// gate dtype picks; down stages its own copy and is free. +// +// The reference uses the exact dequantized weights, so the tolerance only has to absorb f16 +// accumulation, and a GEMM decoded with the other dtype's stride is far outside it. +// ============================================================================ + +struct MoeMixedDtypeParams { + size_t seq_len; + bool gate_up_u4; + bool down_u4; +}; + +class moe_3gemm_compressed_gpu_mixed_dtype : public ::testing::TestWithParam { +public: + static std::string GetTestCaseName(const testing::TestParamInfo& obj) { + const auto& p = obj.param; + return std::string(p.gate_up_u4 ? "gate_up_u4" : "gate_up_u8") + (p.down_u4 ? "_down_u4" : "_down_u8") + "_seq" + + std::to_string(p.seq_len); + } +}; + +TEST_P(moe_3gemm_compressed_gpu_mixed_dtype, per_gemm_decode_accuracy) { + const auto& param = GetParam(); + auto& engine = get_test_engine(); + if (!engine.get_device_info().supports_immad) { + GTEST_SKIP() << "No immad support"; + } + + tests::random_generator rg(GET_SUITE_NAME); + Moe3GemmConfig config; + config.batch_size = 1; + config.seq_len = param.seq_len; + config.hidden_size = 128; + config.inter_size = 256; + config.num_experts = 4; + config.top_k = 2; + config.group_size = 128; + // quantize(), pack() and dequantize() all key on config.is_u4, so each dtype gets its own reference. + config.is_u4 = param.gate_up_u4; + Moe3GemmReference gate_up_ref(config, rg); + config.is_u4 = param.down_u4; + Moe3GemmReference down_ref(config, rg); + + auto hidden_states = rg.generate_random_1d(config.batch_size * config.seq_len * config.hidden_size, -1.0f, 1.0f, 1000); + auto routing_weights = rg.generate_random_1d(config.batch_size * config.seq_len * config.num_experts, 0.0f, 1.0f, 1000); + + auto w0_data = rg.generate_random_1d(config.num_experts * config.hidden_size * config.inter_size, -1.0f, 0.0f, 1000); + auto w1_data = rg.generate_random_1d(config.num_experts * config.hidden_size * config.inter_size, -1.0f, 0.0f, 1000); + auto w2_data = rg.generate_random_1d(config.num_experts * config.inter_size * config.hidden_size, -1.0f, 0.0f, 1000); + for (size_t i = 0; i < config.num_experts * config.hidden_size * config.inter_size; ++i) { + w0_data[i] /= 7.0f; + w1_data[i] /= 11.0f; + w2_data[i] /= 7.0f; + } + auto [w0_q, w0_scale, w0_zp] = gate_up_ref.quantize(w0_data, config.num_experts, config.hidden_size, config.inter_size, config.group_size); + auto [w1_q, w1_scale, w1_zp] = gate_up_ref.quantize(w1_data, config.num_experts, config.hidden_size, config.inter_size, config.group_size); + auto [w2_q, w2_scale, w2_zp] = down_ref.quantize(w2_data, config.num_experts, config.inter_size, config.hidden_size, config.group_size); + auto w0_qdq = gate_up_ref.dequantize(w0_q, w0_scale, w0_zp, config.num_experts, config.hidden_size, config.inter_size, config.group_size); + auto w1_qdq = gate_up_ref.dequantize(w1_q, w1_scale, w1_zp, config.num_experts, config.hidden_size, config.inter_size, config.group_size); + auto w2_qdq = down_ref.dequantize(w2_q, w2_scale, w2_zp, config.num_experts, config.inter_size, config.hidden_size, config.group_size); + + const int64_t E = static_cast(config.num_experts); + const int64_t group_num = static_cast(config.hidden_size / config.group_size); + const int64_t group_num2 = static_cast(config.inter_size / config.group_size); + + auto make_weight = [&](Moe3GemmReference& r, const std::vector& q, int64_t ofm, int64_t G) { + auto dt = r.config.is_u4 ? data_types::u4 : data_types::u8; + auto mem = engine.allocate_memory({dt, format::bfyx, {E, ofm, static_cast(config.group_size), G}}); + set_values(mem, r.pack(q)); + get_test_stream().finish(); + return mem; + }; + // Logical [E, ofm, G]; physical [E, G, ofm] via byfx for G>1, plain bfyx for per-channel. + auto make_scale = [&](const std::vector& v, int64_t ofm, int64_t G) { + auto fmt = G > 1 ? format::byfx : format::bfyx; + auto shape = G > 1 ? ov::PartialShape{E, ofm, G, 1} : ov::PartialShape{E, ofm, 1}; + auto mem = engine.allocate_memory(layout{shape, data_types::f16, fmt}); + set_values(mem, v); + get_test_stream().finish(); + return mem; + }; + auto make_zp = [&](Moe3GemmReference& r, const std::vector& v, int64_t ofm, int64_t G) { + auto dt = r.config.is_u4 ? data_types::u4 : data_types::u8; + auto fmt = G > 1 ? format::byfx : format::bfyx; + auto shape = G > 1 ? ov::PartialShape{E, ofm, G, 1} : ov::PartialShape{E, ofm, 1}; + auto mem = engine.allocate_memory(layout{shape, dt, fmt}); + set_values(mem, r.pack(v)); + get_test_stream().finish(); + return mem; + }; + auto create_f16_tensor_3d = [&](const std::vector& values, int64_t d0, int64_t d1, int64_t d2) { + auto mem = engine.allocate_memory(layout{ov::PartialShape{d0, d1, d2}, data_types::f16, format::bfyx}); + set_values(mem, values); + get_test_stream().finish(); + return mem; + }; + + auto hidden_states_mem = create_f16_tensor_3d(hidden_states, config.batch_size, config.seq_len, config.hidden_size); + auto routing_weights_mem = create_f16_tensor_3d(routing_weights, config.batch_size, config.seq_len, config.num_experts); + + const int64_t inter = static_cast(config.inter_size); + const int64_t hidden = static_cast(config.hidden_size); + + topology topology; + topology.add(input_layout("hidden_states", hidden_states_mem->get_layout())); + topology.add(input_layout("routing_weights", routing_weights_mem->get_layout())); + topology.add(data("w0_weight", make_weight(gate_up_ref, w0_q, inter, group_num))); + topology.add(data("w0_scale", make_scale(w0_scale, inter, group_num))); + topology.add(data("w0_zp", make_zp(gate_up_ref, w0_zp, inter, group_num))); + topology.add(data("w1_weight", make_weight(gate_up_ref, w1_q, inter, group_num))); + topology.add(data("w1_scale", make_scale(w1_scale, inter, group_num))); + topology.add(data("w1_zp", make_zp(gate_up_ref, w1_zp, inter, group_num))); + topology.add(data("w2_weight", make_weight(down_ref, w2_q, hidden, group_num2))); + topology.add(data("w2_scale", make_scale(w2_scale, hidden, group_num2))); + topology.add(data("w2_zp", make_zp(down_ref, w2_zp, hidden, group_num2))); + + MoERouterFused::Config router_config; + router_config.num_expert = config.num_experts; + router_config.top_k = config.top_k; + router_config.routing_type = MoERouterFused::RoutingType::SOFTMAX; + topology.add(moe_router_fused("router", std::vector{input_info("routing_weights")}, router_config)); + + cldnn::MOECompressed::Config moe_config; + moe_config.hidden_size = config.hidden_size; + moe_config.inter_size = config.inter_size; + moe_config.num_expert = config.num_experts; + moe_config.top_k = config.top_k; + moe_config.group_size = config.group_size; + moe_config.out_type = data_types::f16; + moe_config.has_zp = true; + + std::vector moe_inputs{input_info("hidden_states"), + input_info("router", 0), + input_info("router", 1), + input_info("w0_weight"), + input_info("w0_scale"), + input_info("w0_zp"), + input_info("w1_weight"), + input_info("w1_scale"), + input_info("w1_zp"), + input_info("w2_weight"), + input_info("w2_scale"), + input_info("w2_zp")}; + topology.add(moe_3gemm_fused_compressed("moe", moe_inputs, moe_config)); + + auto net_config = get_test_default_config(engine); + net_config.set_property(ov::intel_gpu::allow_new_shape_infer(true)); + network network(engine, topology, net_config); + network.set_input_data("hidden_states", hidden_states_mem); + network.set_input_data("routing_weights", routing_weights_mem); + + auto outputs = network.execute(); + auto output_prim = outputs.begin()->second.get_memory(); + get_test_stream().flush(); + cldnn::mem_lock output_ptr(output_prim, get_test_stream()); + + auto ref_output = gate_up_ref.run_reference_softmax(hidden_states, routing_weights, w0_qdq, w1_qdq, w2_qdq); + const float tolerance = 0.15f; + bool all_zero = true; + for (size_t i = 0; i < ref_output.size(); ++i) { + if (static_cast(output_ptr[i]) != 0.0f) + all_zero = false; + ASSERT_NEAR(static_cast(output_ptr[i]), static_cast(ref_output[i]), tolerance) << "i = " << i; + } + ASSERT_FALSE(all_zero) << "output is all zeros - the MoE stage was silently skipped"; +} + +INSTANTIATE_TEST_SUITE_P(smoke, + moe_3gemm_compressed_gpu_mixed_dtype, + ::testing::Values( + // Decode GEMV, single token. + MoeMixedDtypeParams{1, false, true}, + MoeMixedDtypeParams{1, true, false}, + // Batched GEMV (MTP-style token counts). + MoeMixedDtypeParams{4, false, true}, + MoeMixedDtypeParams{16, true, false}, + // Just above the 32-token batched-GEMV threshold: the prefill GEMM path, which + // already took each GEMM's dtype; kept so a regression there is caught too. + MoeMixedDtypeParams{33, false, true}, + MoeMixedDtypeParams{33, true, false}), + moe_3gemm_compressed_gpu_mixed_dtype::GetTestCaseName); From 3e4b53e09b2e9c510f88a802ff13e23d0f7ffd8e Mon Sep 17 00:00:00 2001 From: Cyberpunk1210 Date: Tue, 13 Oct 2026 09:33:46 +0800 Subject: [PATCH 03/12] [GPU] Support u2 in the fused MoE 3-GEMM path Brings the fused moe_3gemm_swiglu path to u2 (INT2) expert weights and enables u2 for the GPU MoE passes. Depends on "[Transformations] Support u2 expert weights in MoE op fusion" and on "[GPU] Decode each GEMM of the fused MoE 3-GEMM with its own weight dtype". Prefill runs u2 through the existing oneDNN grouped GEMM by unpacking it to u4; a native u2 prefill GEMM follows separately. Decode (GEMV and batched GEMV). u2 GEMV variants of gate_up and down, dispatched per GEMM as dtype code 3. They read activations from SLM so that lane L's activations line up with the contiguous K values packed in its weight byte (w[4L..4L+3]); the u4 path's strided read targets an interleaved layout and is left unchanged. Per-group u2 zero points are decoded in-kernel; a scalar zero point (INT2_SYM) is broadcast, and whether a GEMM's zp is scalar is answered per GEMM (GATE/UP/DOWN_ZP_SCALAR), since a mixed INT2_SYM/INT2_ASYM layer gives gate and up different forms. Only the u2 variants have an ELEMS_PER_LANE == 1 branch, so the smaller group size is admitted only when all three GEMMs are u2. Prefill. oneDNN and the micro-GEMM have no u2 type, so the u2 expert weights (and zp) are unpacked once into u4 scratch and the u4 grouped GEMM consumes them. The unpack runs once per bound weight buffer rather than per prefill -- the weights are constant, since expert offload is rejected for u2 -- which removed ~430 ms of TTFT per prefill on Qwen3.6-35B-A3B. Shared experts. A u2 shared projection is reported to oneDNN as u4, so it is unpacked the same way; the unpack is per projection because NNCF mixed precision makes any subset of the projections u2. Mixed precision. MoE op fusion now also admits u4 in a mixed layer, as long as gate and up agree on whether they are u4 (they share one SLM activation copy whose layout the gate dtype picks). This admits the NNCF recipe that keeps gate/up at u2 and gives down u4. It lands here rather than with the fusion change because only now does every admitted combination decode per GEMM. Pipeline: u2 is offered to ConvertTiledMoeBlockToGatherMatmuls and ConvertGatherMatmulToGatherMatmulCompressed only when MoE fusion is enabled -- GatherMatmulCompressed has no u2 kernel, and those passes now also run under OV_GPU_MOE_DISABLE_FUSION -- and never to ConvertGroupedMatMulToGroupedMatMulCompressed (oneDNN-only). FuseGatedMLP and KeepMOE3GemmConstPrecision accept u2. Tests: moe_3gemm u2 cases for decode GEMV, batched GEMV and prefill, scalar and per-group zero points, mixed u2/u8 and u2/u4 layers, and the u2 shared-expert unpack; the MoE op fusion dtype table for the u4 mixes. Co-Authored-By: Claude Opus 5.5 (1M context) Signed-off-by: Cyberpunk1210 --- .../common_optimizations/moe_op_fusion.cpp | 18 +- ...ert_gather_matmuls_moe_block_to_moe_op.cpp | 22 +- .../impls/ocl_v2/moe/moe_3gemm_swiglu_opt.cpp | 560 +++++++++++++- .../impls/ocl_v2/moe/moe_3gemm_swiglu_opt.hpp | 6 +- .../impls/ocl_v2/moe/moe_gemm_gen_opt.hpp | 2 +- .../impls/ocl_v2/moe_3gemm_swiglu_mlp.cl | 392 +++++++++- .../src/graph/impls/ocl_v2/utils/jitter.cpp | 5 + .../plugin/transformations/fuse_gated_mlp.cpp | 6 +- .../keep_moe_3gemm_const_precision.cpp | 24 +- .../src/plugin/transformations_pipeline.cpp | 11 +- .../unit/test_cases/moe_3gemm_gpu_test.cpp | 703 ++++++++++++++++++ 11 files changed, 1684 insertions(+), 65 deletions(-) diff --git a/src/common/transformations/src/transformations/common_optimizations/moe_op_fusion.cpp b/src/common/transformations/src/transformations/common_optimizations/moe_op_fusion.cpp index 1d447f4dd4cdb9..75c9c5c8a35c5f 100644 --- a/src/common/transformations/src/transformations/common_optimizations/moe_op_fusion.cpp +++ b/src/common/transformations/src/transformations/common_optimizations/moe_op_fusion.cpp @@ -160,21 +160,27 @@ Convert3GatherMatmulMoeBlockToMoeOp::Convert3GatherMatmulMoeBlockToMoeOp() { const bool is_compressed = is_gate_compressed; if (is_compressed) { - // A layer that mixes dtypes across the three expert GEMMs is only decoded correctly by - // moe_3gemm for the {u2, u8} combinations its per-GEMM dispatch covers. Anything else - // keeps the per-GEMM-correct GatherMatmul form rather than being mis-decoded. + // A layer that mixes dtypes across the three expert GEMMs is decoded per GEMM by + // moe_3gemm (GATE/UP/DOWN_WEIGHT_DT) for u2, u4 and u8. One constraint is shared state + // rather than dispatch: gate and up read ONE SLM copy of the activations, and its layout + // is chosen from the gate dtype alone -- interleaved (even/odd halves) for u4, plain for + // u2/u8. So gate and up must agree on whether they are u4; down stages its own copy and + // is free. This is what admits the NNCF recipe that keeps gate/up at u2 and gives down + // u4. Signed types stay uniform-only: WEIGHT_IS_SIGNED is set once, from the gate. + // Anything else keeps the per-GEMM-correct GatherMatmul form rather than being mis-decoded. { const auto gate_dt = pm.at(gate_w_m).get_element_type(); const auto up_dt = pm.at(up_w_m).get_element_type(); const auto down_dt = pm.at(down_w_m).get_element_type(); - const auto is_u2_or_u8 = [](ov::element::Type t) { - return t == ov::element::u2 || t == ov::element::u8; + const auto is_mixable = [](ov::element::Type t) { + return t == ov::element::u2 || t == ov::element::u4 || t == ov::element::u8; }; const bool uniform_ok = (up_dt == gate_dt && down_dt == gate_dt) && (gate_dt == ov::element::u2 || gate_dt == ov::element::u4 || gate_dt == ov::element::i4 || gate_dt == ov::element::u8 || gate_dt == ov::element::i8); - const bool mixed_ok = is_u2_or_u8(gate_dt) && is_u2_or_u8(up_dt) && is_u2_or_u8(down_dt); + const bool mixed_ok = is_mixable(gate_dt) && is_mixable(up_dt) && is_mixable(down_dt) && + ((gate_dt == ov::element::u4) == (up_dt == ov::element::u4)); if (!(uniform_ok || mixed_ok)) { return false; } diff --git a/src/common/transformations/tests/common_optimizations/convert_gather_matmuls_moe_block_to_moe_op.cpp b/src/common/transformations/tests/common_optimizations/convert_gather_matmuls_moe_block_to_moe_op.cpp index e1069b75ff6a81..4c023623f12e94 100644 --- a/src/common/transformations/tests/common_optimizations/convert_gather_matmuls_moe_block_to_moe_op.cpp +++ b/src/common/transformations/tests/common_optimizations/convert_gather_matmuls_moe_block_to_moe_op.cpp @@ -1015,7 +1015,9 @@ TEST(Convert3GatherMatmulMoeBlockToMoeOpCompressed, uniform_i8_fuses) { } // C6. The full admission table. Uniform is allowed for every dtype the producer can emit; mixing -// is allowed only within {u2, u8}, which is what moe_3gemm's per-GEMM dispatch actually decodes. +// is allowed within {u2, u4, u8}, which is what moe_3gemm's per-GEMM dispatch actually decodes, +// provided gate and up agree on whether they are u4 -- they share one SLM activation copy whose +// layout (interleaved for u4, plain otherwise) is picked from the gate dtype. struct DtypeGateParams { ov::element::Type gate_dt; ov::element::Type up_dt; @@ -1025,7 +1027,7 @@ struct DtypeGateParams { class MoeOpFusionDtypeGate : public ::testing::TestWithParam {}; -TEST_P(MoeOpFusionDtypeGate, admits_uniform_and_u2_u8_mixes_only) { +TEST_P(MoeOpFusionDtypeGate, admits_uniform_and_unsigned_mixes_only) { const auto& p = GetParam(); CompressedMoeSpec spec; spec.gate_dt = p.gate_dt; @@ -1057,11 +1059,23 @@ INSTANTIATE_TEST_SUITE_P(smoke, DtypeGateParams{ov::element::u2, ov::element::u8, ov::element::u2, true}, DtypeGateParams{ov::element::u8, ov::element::u2, ov::element::u8, true}, DtypeGateParams{ov::element::u2, ov::element::u2, ov::element::u8, true}, - // Mixed involving anything else: no per-GEMM decode exists, so it must stay unfused. + // u4 on down only: down stages its own activation copy, so it may differ from gate/up. + // The first row is the recipe this exists for -- gate/up u2, down u4. + DtypeGateParams{ov::element::u2, ov::element::u2, ov::element::u4, true}, + DtypeGateParams{ov::element::u8, ov::element::u8, ov::element::u4, true}, + DtypeGateParams{ov::element::u2, ov::element::u8, ov::element::u4, true}, + // u4 on both gate and up: they agree, so their shared activation copy is interleaved + // for both. + DtypeGateParams{ov::element::u4, ov::element::u4, ov::element::u2, true}, + DtypeGateParams{ov::element::u4, ov::element::u4, ov::element::u8, true}, + // gate and up disagree on u4: one of them would read the other's activation layout. DtypeGateParams{ov::element::u4, ov::element::u8, ov::element::u8, false}, - DtypeGateParams{ov::element::i8, ov::element::u8, ov::element::i8, false}, DtypeGateParams{ov::element::u2, ov::element::u4, ov::element::u2, false}, + DtypeGateParams{ov::element::u4, ov::element::u2, ov::element::u4, false}, + // Signed types mix with nothing: WEIGHT_IS_SIGNED is layer-wide, set from the gate. + DtypeGateParams{ov::element::i8, ov::element::u8, ov::element::i8, false}, DtypeGateParams{ov::element::u4, ov::element::i4, ov::element::u4, false}, + DtypeGateParams{ov::element::u2, ov::element::u2, ov::element::i4, false}, // Not producible by the pipeline today; documents that the gate is a whitelist. DtypeGateParams{ov::element::f16, ov::element::f16, ov::element::f16, false})); diff --git a/src/plugins/intel_gpu/src/graph/impls/ocl_v2/moe/moe_3gemm_swiglu_opt.cpp b/src/plugins/intel_gpu/src/graph/impls/ocl_v2/moe/moe_3gemm_swiglu_opt.cpp index 086627c57d9523..12418aaf401379 100644 --- a/src/plugins/intel_gpu/src/graph/impls/ocl_v2/moe/moe_3gemm_swiglu_opt.cpp +++ b/src/plugins/intel_gpu/src/graph/impls/ocl_v2/moe/moe_3gemm_swiglu_opt.cpp @@ -425,20 +425,27 @@ static void add_common_consts(const RuntimeParams& params, JitConstants& jit) { GPU_DEBUG_TRACE_DETAIL << "[DEBUG] moe_3gemm_swiglu_opt: group_size=" << desc->_config.group_size << ", gate_up_group_size=" << gate_up_group_size << ", down_group_size=" << down_group_size << std::endl; - // Validate GEMV kernel compatibility: ELEMS_PER_LANE = FAKE_GROUP_SIZE / SUBGROUP_SIZE must be >= 2. - // Smaller values have no kernel branch and would silently produce wrong results. + // ELEMS_PER_LANE = FAKE_GROUP_SIZE / SUBGROUP_SIZE selects the GEMV inner-loop variant, and only + // the u2 variants have an ELEMS_PER_LANE==1 branch. The kernel dispatches per GEMM, so relaxing + // the bound is safe only when all three are u2: a u4/u8 GEMM at ELEMS_PER_LANE 1 has no matching + // branch and silently falls into the 8-element #else path, reading past its quant group. + const ov::element::Type weight_dt = params.get_input_layout(static_cast(MOE3GemmInputIndex::WEIGHT_0)).data_type; { + const auto up_weight_dt = params.get_input_layout(static_cast(MOE3GemmInputIndex::WEIGHT_1)).data_type; + const auto down_weight_dt = params.get_input_layout(static_cast(MOE3GemmInputIndex::WEIGHT_2)).data_type; + const bool all_u2 = weight_dt == ov::element::u2 && up_weight_dt == ov::element::u2 && down_weight_dt == ov::element::u2; const size_t sg = (info.arch >= gpu_arch::xe2) ? 32u : 16u; // Mirrors FAKE_GROUP_SIZE in moe_3gemm_swiglu_mlp.cl, which also takes the down group size; // checking gate/up alone let a smaller down group through to a branch that does not exist. const size_t fake_gs = std::min({gate_up_group_size, down_group_size, size_t{128}}); - OPENVINO_ASSERT(fake_gs >= 2 * sg, + const size_t min_fake_gs = all_u2 ? sg : 2 * sg; + OPENVINO_ASSERT(fake_gs >= min_fake_gs, "MoE GEMV kernel does not support group_size=", fake_gs, " on this hardware (SUBGROUP_SIZE=", sg, "). Minimum supported group_size is ", - 2 * sg, + min_fake_gs, ". Use a larger quantization group size."); } @@ -454,8 +461,29 @@ static void add_common_consts(const RuntimeParams& params, JitConstants& jit) { jit.make("MOE_DTYPE", params.get_input_layout(0).data_type == ov::element::f16 ? "half" : "float"); jit.make("MOE_DTYPE_SIZE", params.get_input_layout(0).data_type == ov::element::f16 ? 2 : 4); jit.make("HAS_ZP", desc->_config.has_zp ? 1 : 0); + if (desc->_config.has_zp) { + // Only the element type of a broadcast scalar zp is shared; *which* GEMMs use one is + // answered per GEMM by GATE/UP/DOWN_ZP_SCALAR below. A layer-wide "is it scalar" flag is + // deliberately not emitted: the three GEMMs can disagree, and a single answer silently + // decodes one GEMM's zp with another's rule. + const char* scalar_zp_dt = "uchar"; + for (const auto zp_idx : {MOE3GemmInputIndex::ZP_0, MOE3GemmInputIndex::ZP_1, MOE3GemmInputIndex::ZP_2}) { + const auto& zp_layout = params.get_input_layout(static_cast(zp_idx)); + if (zp_layout.count() == 1) { + OPENVINO_ASSERT(zp_layout.data_type == ov::element::i8 || zp_layout.data_type == ov::element::u8, + "Scalar MoE zp must be i8/u8, got ", + zp_layout.data_type); + // A u2 zero point is 0..3, which reads identically as char or uchar, so one type + // serves all three GEMMs even when only some of them are scalar. + if (zp_layout.data_type == ov::element::i8) { + scalar_zp_dt = "char"; + } + break; + } + } + jit.make("MOE_ZP_SCALAR_DT", scalar_zp_dt); + } - ov::element::Type weight_dt = params.get_input_layout(static_cast(MOE3GemmInputIndex::WEIGHT_0)).data_type; bool is_signed_weight = (weight_dt == ov::element::i4 || weight_dt == ov::element::i8); jit.make("WEIGHT_IS_SIGNED", is_signed_weight ? 1 : 0); // auto scale_dt = params.get_input_layout(static_cast(MOE3GemmInputIndex::SCALE_0)).data_type; @@ -465,6 +493,18 @@ static void add_common_consts(const RuntimeParams& params, JitConstants& jit) { jit.make("MOE_WEI_DT", "uchar"); jit.make("MOE_SCALE_DT", "half"); jit.make("MOE_ZP_DT", "uchar"); + } else if (weight_dt == ov::element::u2) { + // u2: 4 values per byte, LSB-first. Served by the batched GEMV kernels for all + // token counts (micro-gemm and oneDNN have no u2 support). + OPENVINO_ASSERT(desc->_config.hidden_size % 4 == 0 && desc->_config.inter_size % 4 == 0, + "MoE u2 weights require hidden_size and inter_size to be multiples of 4, got hidden_size=", + desc->_config.hidden_size, + ", inter_size=", + desc->_config.inter_size); + jit.make("WEIGHT_COMPRESSEION_DT", 3); + jit.make("MOE_WEI_DT", "uchar"); + jit.make("MOE_SCALE_DT", "half"); + jit.make("MOE_ZP_DT", "uchar"); } else if (weight_dt == ov::element::u8 || weight_dt == ov::element::i8) { jit.make("WEIGHT_COMPRESSEION_DT", 1); jit.make("MOE_WEI_DT", "uchar"); @@ -478,12 +518,14 @@ static void add_common_consts(const RuntimeParams& params, JitConstants& jit) { } // Per-GEMM weight dtype codes: gate, up and down may differ under NNCF mixed precision, and - // each is decoded with its own. Codes mirror WEIGHT_COMPRESSEION_DT: 0=u4/i4, 1=u8/i8, 2=f16. + // each is decoded with its own. Codes mirror WEIGHT_COMPRESSEION_DT: 0=u4/i4, 1=u8/i8, 2=f16, 3=u2. const auto moe_weight_code = [](ov::element::Type dt) -> int { if (dt == ov::element::u4 || dt == ov::element::i4) return 0; if (dt == ov::element::f16) return 2; + if (dt == ov::element::u2) + return 3; return 1; // u8/i8 (byte) path }; const auto gate_wdt = params.get_input_layout(static_cast(MOE3GemmInputIndex::WEIGHT_0)).data_type; @@ -492,6 +534,17 @@ static void add_common_consts(const RuntimeParams& params, JitConstants& jit) { jit.make("GATE_WEIGHT_DT", moe_weight_code(gate_wdt)); jit.make("UP_WEIGHT_DT", moe_weight_code(up_wdt)); jit.make("DOWN_WEIGHT_DT", moe_weight_code(down_wdt)); + // Per-GEMM scalar-zp flags: a single-element zp is broadcast in-kernel (u2 GEMMs), + // vs per-channel/group zp indexed by (group, channel) (u8 GEMMs). + if (desc->_config.has_zp) { + jit.make("GATE_ZP_SCALAR", params.get_input_layout(static_cast(MOE3GemmInputIndex::ZP_0)).count() == 1 ? 1 : 0); + jit.make("UP_ZP_SCALAR", params.get_input_layout(static_cast(MOE3GemmInputIndex::ZP_1)).count() == 1 ? 1 : 0); + jit.make("DOWN_ZP_SCALAR", params.get_input_layout(static_cast(MOE3GemmInputIndex::ZP_2)).count() == 1 ? 1 : 0); + } else { + jit.make("GATE_ZP_SCALAR", 0); + jit.make("UP_ZP_SCALAR", 0); + jit.make("DOWN_ZP_SCALAR", 0); + } } class MoE3GemmSwigluMLPGateUp : public KernelGenerator { @@ -601,6 +654,31 @@ class MoE3GemmSwigluMLPReduce : public KernelGenerator { bool _disable_shared_experts; }; +// u2 -> u4 weight unpack kernel (prefill only): micro-gemm and oneDNN have no u2 +// dtype, so u2 expert weights are unpacked into u4 scratch buffers and the regular +// u4 grouped GEMM prefill path consumes them. +class MoE3GemmSwigluMLPU2Unpack : public KernelGenerator { +public: + MoE3GemmSwigluMLPU2Unpack() : KernelGenerator("moe_3gemm_swiglu_mlp", "u2_unpack") {} + +protected: + [[nodiscard]] JitConstants get_jit_constants(const RuntimeParams& params) const override { + auto jit = KernelGenerator::get_jit_constants(params); + add_common_consts(params, jit); + jit.make("U2_UNPACK_ENABLE", 1); + return jit; + } + + [[nodiscard]] Arguments get_arguments_desc(const RuntimeParams& params) const override { + Arguments args; + return args; + } + + [[nodiscard]] DispatchDataFunc get_dispatch_data_func() const override { + return DispatchDataFunc{[](const RuntimeParams& params, KernelData& kd, ImplRuntimeParams* rt_params) {}}; + } +}; + dnnl::memory convert2dnnl(const memory::ptr& ptr, const std::vector& dim, dnnl::memory::format_tag tag, int64_t offset = 0) { OV_ITT_SCOPED_TASK(ov::intel_gpu::itt::domains::intel_gpu_plugin, openvino::itt::handle("convert2dnnl")); return ptr->get_onednn_memory(dnnl::memory::desc(dnnl::memory::dims(dim), convert_data_type(ptr->get_layout().data_type), tag), offset); @@ -640,6 +718,9 @@ class moe_3gemm_swiglu_opt_impl : public PrimitiveImplOCL { Stage::Ptr grouped_gemm_prefill_swiglu = make_stage(/*use_grouped_gemm=*/true); Stage::Ptr prefill_scatter_reduce_row_lut = make_stage(); + // u2 prefill: unpack u2 weights into u4 scratch for the grouped GEMM path + Stage::Ptr u2_unpack = make_stage(); + struct dnnl_weights { dnnl::memory weight; dnnl::memory scale; @@ -1010,6 +1091,47 @@ class moe_3gemm_swiglu_opt_impl : public PrimitiveImplOCL { bool use_micro_gemm_prefill = false; bool use_gpu_mask_gen_prefill = false; bool use_grouped_gemm_prefill = false; + bool _weights_u2 = false; // any u2 GEMM: batched GEMV for decode, u4-unpack + grouped GEMM for prefill + bool _gemm_weights_u2[3] = {false, false, false}; // per-GEMM (gate/up/down) u2 flags + bool _shared_weights_u2[4] = {false, false, false, false}; // per-projection (gate/up/down/scalar-gate) shared-expert u2 flags + // u2 prefill scratch: unpacked u4 copies of this layer's expert weights (and zp). + // Allocated lazily on first prefill and reused across calls (unpack_u2_weights_for_prefill). + memory::ptr _u2_unpack_weight[3]; + memory::ptr _u2_unpack_zp[3]; + // Same for the shared expert's weights/zp; consumed by the oneDNN shared-expert + // primitives built in init_shared_primitives. + memory::ptr _u2_unpack_shared_weight[4]; + memory::ptr _u2_unpack_shared_zp[4]; + + // Identifies the source buffer a u4 unpack scratch was last filled from, so the unpack + // kernel runs once instead of on every prefill. The unpack is a pure function of a + // *constant* weight/zp buffer: OTD (expert offload) is rejected for u2 in the ctor, so + // nothing rebinds these mid-run. Keyed on buffer identity + byte size so a rebind still + // re-runs the unpack; a freshly (re)allocated destination also forces a re-run. + struct unpack_src_key { + const void* src = nullptr; + size_t bytes = 0; + + bool matches(const memory::ptr& src_mem) const { + return src != nullptr && src == static_cast(src_mem.get()) && bytes == src_mem->size(); + } + void set(const memory::ptr& src_mem) { + src = static_cast(src_mem.get()); + bytes = src_mem->size(); + } + void reset() { + src = nullptr; + bytes = 0; + } + }; + unpack_src_key _u2_unpack_weight_key[3]; + unpack_src_key _u2_unpack_zp_key[3]; + unpack_src_key _u2_unpack_shared_weight_key[4]; + unpack_src_key _u2_unpack_shared_zp_key[4]; + + bool has_u2_shared_weights() const { + return _shared_weights_u2[0] || _shared_weights_u2[1] || _shared_weights_u2[2] || _shared_weights_u2[3]; + } size_t batched_gemv_threshold = 32; // token_num <= threshold uses batched GEMV path moe_3gemm_swiglu_opt_impl() : PrimitiveImplOCL(moe_3gemm_swiglu_opt::get_type_info_static()) {} @@ -1040,9 +1162,35 @@ class moe_3gemm_swiglu_opt_impl : public PrimitiveImplOCL { use_micro_gemm_prefill = false; } + // u2 (2-bit) weights: micro-gemm (gemmstone) and oneDNN have no u2 dtype, so the + // u4-only prefill paths cannot consume u2 data directly. Decode (small token + // counts) is served by the hand-written batched GEMV kernels; prefill unpacks + // the u2 weights into u4 scratch buffers and runs the grouped GEMM path. + // In NNCF mixed-precision layers ANY u2 GEMM (gate/up/down) selects this split. + _gemm_weights_u2[0] = (weight_dt == data_types::u2); + _gemm_weights_u2[1] = (params.get_input_layout(static_cast(MOE3GemmInputIndex::WEIGHT_1)).data_type == data_types::u2); + _gemm_weights_u2[2] = (params.get_input_layout(static_cast(MOE3GemmInputIndex::WEIGHT_2)).data_type == data_types::u2); + _weights_u2 = _gemm_weights_u2[0] || _gemm_weights_u2[1] || _gemm_weights_u2[2]; + + // Shared expert per-projection u2 flags (gate/up/down/scalar-gate). Under NNCF + // mixed precision any subset of them may be u2; u2 shared weights are unpacked + // into u4 scratch for the oneDNN shared-expert primitives (init_shared_primitives). + if (params.input_layouts.size() > static_cast(MOE3GemmInputIndex::SHARED_GATE_WEIGHT)) { + _shared_weights_u2[0] = (params.get_input_layout(static_cast(MOE3GemmInputIndex::SHARED_GATE_WEIGHT)).data_type == data_types::u2); + _shared_weights_u2[1] = (params.get_input_layout(static_cast(MOE3GemmInputIndex::SHARED_UP_WEIGHT)).data_type == data_types::u2); + _shared_weights_u2[2] = (params.get_input_layout(static_cast(MOE3GemmInputIndex::SHARED_DOWN_WEIGHT)).data_type == data_types::u2); + _shared_weights_u2[3] = (params.get_input_layout(static_cast(MOE3GemmInputIndex::SHARED_GATE_GATE_WEIGHT)).data_type == data_types::u2); + } + // grouped_gemm path: single OneDNN grouped matmul per GEMM layer (all experts at once). // micro_gemm takes priority; grouped_gemm falls back to onednn loop by default. use_grouped_gemm_prefill = config.get_moe_use_grouped_gemm_prefill(); + if (_weights_u2) { + // gemmstone has no u2 support at all; the grouped GEMM prefill consumes the + // u2 weights via the u4-unpacked scratch (see unpack_u2_weights_for_prefill). + use_micro_gemm_prefill = false; + use_grouped_gemm_prefill = true; + } // grouped_gemm supersedes micro_gemm if (use_grouped_gemm_prefill) { use_micro_gemm_prefill = false; @@ -1058,6 +1206,7 @@ class moe_3gemm_swiglu_opt_impl : public PrimitiveImplOCL { // Weight provider: select based on OTD configuration. auto cur_moe = node.as().get_primitive(); + OPENVINO_ASSERT(!_weights_u2 || cur_moe->_otd.lru_expert_num == 0, "moe_3gemm: u2 weights are not supported with OTD (expert offload) yet"); if (cur_moe->_otd.lru_expert_num > 0) { _weight_provider = std::make_shared(cur_moe->_otd.lru_expert_num, cur_moe->_config, @@ -1087,6 +1236,9 @@ class moe_3gemm_swiglu_opt_impl : public PrimitiveImplOCL { add_stage(grouped_gemm_prefill_swiglu, params); add_stage(prefill_scatter_reduce_row_lut, params); } + if (_weights_u2 || has_u2_shared_weights()) { + add_stage(u2_unpack, params); + } } void init(const std::shared_ptr& cur_moe) { @@ -1202,6 +1354,9 @@ class moe_3gemm_swiglu_opt_impl : public PrimitiveImplOCL { OPENVINO_ASSERT(addr.shared_weight[0], "MoE shared expert enabled (num_shared_expert > 0) but shared weight buffers are not bound"); + // layout.count() is the logical element count even for sub-byte dtypes (u2/u4 + // packing only enters through bytes_count()), so this stays correct when the + // shared weights are u2 (e.g. [512, 32, 64] u2 -> 512 with hidden 2048). _shared_intermediate_size = static_cast(addr.shared_weight[0]->get_layout().count() / _hidden_size); auto eng = engine.get_onednn_engine(); using t = onednn_matmul::type; @@ -1211,14 +1366,36 @@ class moe_3gemm_swiglu_opt_impl : public PrimitiveImplOCL { return m && m->get_layout().data_type != data_types::dynamic; }; + // u2 shared weights: consume the u4-unpacked scratch (filled by + // unpack_u2_weights_for_prefill) with u4 descriptors, mirroring the + // routed-expert grouped GEMM substitution. Non-u2 projections keep the + // original buffers/dtypes (mixed precision). + auto shared_weight_dt = [&](int i) { + return _shared_weights_u2[i] ? dnnl::memory::data_type::u4 : convert_data_type(addr.shared_weight[i]->get_layout().data_type); + }; + auto shared_weight_mem = [&](int i, const std::vector& dims, dnnl::memory::format_tag tag) { + if (_shared_weights_u2[i]) { + OPENVINO_ASSERT(_u2_unpack_shared_weight[i], "moe_3gemm shared weight ", i, " is u2 but its u4 unpack scratch is not allocated"); + return _u2_unpack_shared_weight[i]->get_onednn_memory(dnnl::memory::desc(dnnl::memory::dims(dims), dnnl::memory::data_type::u4, tag)); + } + return convert2dnnl(addr.shared_weight[i], dims, tag); + }; + auto shared_zp_mem = [&](int i, const std::vector& dims, dnnl::memory::format_tag tag) { + if (_shared_weights_u2[i]) { + OPENVINO_ASSERT(_u2_unpack_shared_zp[i], "moe_3gemm shared weight ", i, " is u2 but its u4 zp unpack scratch is not allocated"); + return _u2_unpack_shared_zp[i]->get_onednn_memory(dnnl::memory::desc(dnnl::memory::dims(dims), dnnl::memory::data_type::u4, tag)); + } + return convert2dnnl(addr.shared_zp[i], dims, tag); + }; + // 1. Up (Standard Linear) - auto up_w_dt = convert_data_type(addr.shared_weight[1]->get_layout().data_type); - auto up_w = convert2dnnl(addr.shared_weight[1], {_hidden_size, _shared_intermediate_size}, dnnl::memory::format_tag::ba); + auto up_w_dt = shared_weight_dt(1); + auto up_w = shared_weight_mem(1, {_hidden_size, _shared_intermediate_size}, dnnl::memory::format_tag::ba); auto up_s = addr.shared_scale[1] ? convert2dnnl(addr.shared_scale[1], {_hidden_size / _gate_up_group_size, _shared_intermediate_size}, dnnl::memory::format_tag::ab) : dnnl::memory(); auto up_z = is_valid_zp(addr.shared_zp[1]) - ? convert2dnnl(addr.shared_zp[1], {_hidden_size / _gate_up_group_size, _shared_intermediate_size}, dnnl::memory::format_tag::ab) + ? shared_zp_mem(1, {_hidden_size / _gate_up_group_size, _shared_intermediate_size}, dnnl::memory::format_tag::ab) : dnnl::memory(); _shared_up_proj = std::make_shared(onednn_linear::create(eng, dnnl::memory::data_type::f16, @@ -1233,13 +1410,13 @@ class moe_3gemm_swiglu_opt_impl : public PrimitiveImplOCL { up_z)); // 2. Gate (SiLU + BinMul) - auto gate_w_dt = convert_data_type(addr.shared_weight[0]->get_layout().data_type); - auto gate_w = convert2dnnl(addr.shared_weight[0], {_hidden_size, _shared_intermediate_size}, dnnl::memory::format_tag::ba); + auto gate_w_dt = shared_weight_dt(0); + auto gate_w = shared_weight_mem(0, {_hidden_size, _shared_intermediate_size}, dnnl::memory::format_tag::ba); auto gate_s = addr.shared_scale[0] ? convert2dnnl(addr.shared_scale[0], {_hidden_size / _gate_up_group_size, _shared_intermediate_size}, dnnl::memory::format_tag::ab) : dnnl::memory(); auto gate_z = is_valid_zp(addr.shared_zp[0]) - ? convert2dnnl(addr.shared_zp[0], {_hidden_size / _gate_up_group_size, _shared_intermediate_size}, dnnl::memory::format_tag::ab) + ? shared_zp_mem(0, {_hidden_size / _gate_up_group_size, _shared_intermediate_size}, dnnl::memory::format_tag::ab) : dnnl::memory(); _shared_gate_proj = std::make_shared(onednn_linear::create(eng, dnnl::memory::data_type::f16, @@ -1256,12 +1433,13 @@ class moe_3gemm_swiglu_opt_impl : public PrimitiveImplOCL { // 3. Scalar Gate (Sigmoid) // It is very small weight with shape of [Hidden, 1], and not need to keep compressed, so KeepMOE3GemmConstPrecision will not keep its precision and - // ConvertPrecision will convert to f16. So Scalar gate is [Hidden, 1], f16 weights, no scale/zp for now + // ConvertPrecision will convert to f16. So Scalar gate is [Hidden, 1], f16 weights, no scale/zp for now. + // If it stayed u2 (int2 exports), it is consumed via the u4-unpacked scratch like the other shared projections. dnnl::memory sg_w; dnnl::memory::data_type sg_w_dt = dnnl::memory::data_type::f16; if (addr.shared_weight[3]) { - sg_w_dt = convert_data_type(addr.shared_weight[3]->get_layout().data_type); - sg_w = convert2dnnl(addr.shared_weight[3], {_hidden_size, 1}, dnnl::memory::format_tag::ba); + sg_w_dt = shared_weight_dt(3); + sg_w = shared_weight_mem(3, {_hidden_size, 1}, dnnl::memory::format_tag::ba); } if (sg_w) { @@ -1271,13 +1449,13 @@ class moe_3gemm_swiglu_opt_impl : public PrimitiveImplOCL { } // 4. Down (BinMul + Sum) - auto down_w_dt = convert_data_type(addr.shared_weight[2]->get_layout().data_type); - auto down_w = convert2dnnl(addr.shared_weight[2], {_shared_intermediate_size, _hidden_size}, dnnl::memory::format_tag::ba); + auto down_w_dt = shared_weight_dt(2); + auto down_w = shared_weight_mem(2, {_shared_intermediate_size, _hidden_size}, dnnl::memory::format_tag::ba); auto down_s = addr.shared_scale[2] ? convert2dnnl(addr.shared_scale[2], {_shared_intermediate_size / _down_group_size, _hidden_size}, dnnl::memory::format_tag::ab) : dnnl::memory(); auto down_z = is_valid_zp(addr.shared_zp[2]) - ? convert2dnnl(addr.shared_zp[2], {_shared_intermediate_size / _down_group_size, _hidden_size}, dnnl::memory::format_tag::ab) + ? shared_zp_mem(2, {_shared_intermediate_size / _down_group_size, _hidden_size}, dnnl::memory::format_tag::ab) : dnnl::memory(); _shared_down_proj = std::make_shared(onednn_linear::create(eng, dnnl::memory::data_type::f16, @@ -1337,6 +1515,18 @@ class moe_3gemm_swiglu_opt_impl : public PrimitiveImplOCL { ib >> use_grouped_gemm_prefill; const kernel_impl_params* impl_params = reinterpret_cast(ib.getKernelImplParams()); auto cur_moe = impl_params->typed_desc(); + // The u2 flags gate what init() builds (the u2 unpack stage), so they have to + // be restored before it runs. + _gemm_weights_u2[0] = impl_params->get_input_layout(static_cast(MOE3GemmInputIndex::WEIGHT_0)).data_type == data_types::u2; + _gemm_weights_u2[1] = impl_params->get_input_layout(static_cast(MOE3GemmInputIndex::WEIGHT_1)).data_type == data_types::u2; + _gemm_weights_u2[2] = impl_params->get_input_layout(static_cast(MOE3GemmInputIndex::WEIGHT_2)).data_type == data_types::u2; + _weights_u2 = _gemm_weights_u2[0] || _gemm_weights_u2[1] || _gemm_weights_u2[2]; + if (impl_params->input_layouts.size() > static_cast(MOE3GemmInputIndex::SHARED_GATE_WEIGHT)) { + _shared_weights_u2[0] = impl_params->get_input_layout(static_cast(MOE3GemmInputIndex::SHARED_GATE_WEIGHT)).data_type == data_types::u2; + _shared_weights_u2[1] = impl_params->get_input_layout(static_cast(MOE3GemmInputIndex::SHARED_UP_WEIGHT)).data_type == data_types::u2; + _shared_weights_u2[2] = impl_params->get_input_layout(static_cast(MOE3GemmInputIndex::SHARED_DOWN_WEIGHT)).data_type == data_types::u2; + _shared_weights_u2[3] = impl_params->get_input_layout(static_cast(MOE3GemmInputIndex::SHARED_GATE_GATE_WEIGHT)).data_type == data_types::u2; + } init(cur_moe); if (cur_moe->_otd.lru_expert_num > 0) { _weight_provider = std::make_shared(cur_moe->_otd.lru_expert_num, @@ -1359,6 +1549,39 @@ class moe_3gemm_swiglu_opt_impl : public PrimitiveImplOCL { cur_moe->use_micro_gemm_prefill = use_micro_gemm_prefill; cur_moe->use_gpu_mask_gen_prefill = use_gpu_mask_gen_prefill; cur_moe->use_grouped_gemm_prefill = use_grouped_gemm_prefill; + cur_moe->_weights_u2 = _weights_u2; + cur_moe->_gemm_weights_u2[0] = _gemm_weights_u2[0]; + cur_moe->_gemm_weights_u2[1] = _gemm_weights_u2[1]; + cur_moe->_gemm_weights_u2[2] = _gemm_weights_u2[2]; + // Don't share the u4 unpack scratch with the clone; the clone fills its own on first + // prefill. make_deep_copy() above copied the cache keys too, so clear them as well — + // otherwise the clone would see a "already unpacked" key next to a null buffer. + cur_moe->_u2_unpack_weight[0] = nullptr; + cur_moe->_u2_unpack_weight[1] = nullptr; + cur_moe->_u2_unpack_weight[2] = nullptr; + cur_moe->_u2_unpack_zp[0] = nullptr; + cur_moe->_u2_unpack_zp[1] = nullptr; + cur_moe->_u2_unpack_zp[2] = nullptr; + for (int i = 0; i < 3; i++) { + cur_moe->_u2_unpack_weight_key[i].reset(); + cur_moe->_u2_unpack_zp_key[i].reset(); + } + for (int i = 0; i < 4; i++) { + cur_moe->_u2_unpack_shared_weight_key[i].reset(); + cur_moe->_u2_unpack_shared_zp_key[i].reset(); + } + cur_moe->_shared_weights_u2[0] = _shared_weights_u2[0]; + cur_moe->_shared_weights_u2[1] = _shared_weights_u2[1]; + cur_moe->_shared_weights_u2[2] = _shared_weights_u2[2]; + cur_moe->_shared_weights_u2[3] = _shared_weights_u2[3]; + cur_moe->_u2_unpack_shared_weight[0] = nullptr; + cur_moe->_u2_unpack_shared_weight[1] = nullptr; + cur_moe->_u2_unpack_shared_weight[2] = nullptr; + cur_moe->_u2_unpack_shared_weight[3] = nullptr; + cur_moe->_u2_unpack_shared_zp[0] = nullptr; + cur_moe->_u2_unpack_shared_zp[1] = nullptr; + cur_moe->_u2_unpack_shared_zp[2] = nullptr; + cur_moe->_u2_unpack_shared_zp[3] = nullptr; cur_moe->batched_gemv_threshold = batched_gemv_threshold; cur_moe->_activation_type = _activation_type; return cur_moe; @@ -2145,9 +2368,15 @@ class moe_3gemm_swiglu_opt_impl : public PrimitiveImplOCL { // In OTD mode, weight buffer holds only resident_slot_count() slots, not full num_expert. int num_experts = get_num_grouped_experts(static_cast(config.num_expert)); auto a_dt = convert_data_type(instance.input_memory_ptr(static_cast(MOE3GemmInputIndex::HIDDEN_STATES))->get_layout().data_type); - auto gw_dt = convert_data_type(instance.input_memory_ptr(static_cast(MOE3GemmInputIndex::WEIGHT_0))->get_layout().data_type); - auto uw_dt = convert_data_type(instance.input_memory_ptr(static_cast(MOE3GemmInputIndex::WEIGHT_1))->get_layout().data_type); - auto dw_dt = convert_data_type(instance.input_memory_ptr(static_cast(MOE3GemmInputIndex::WEIGHT_2))->get_layout().data_type); + // u2 GEMMs are unpacked into u4 scratch buffers before the grouped GEMM runs, + // so their weight/ZP descriptors are created with the u4 dtype. + auto grouped_weight_dt = [&](MOE3GemmInputIndex idx) { + auto dt = instance.input_memory_ptr(static_cast(idx))->get_layout().data_type; + return dt == data_types::u2 ? dnnl::memory::data_type::u4 : convert_data_type(dt); + }; + auto gw_dt = grouped_weight_dt(MOE3GemmInputIndex::WEIGHT_0); + auto uw_dt = grouped_weight_dt(MOE3GemmInputIndex::WEIGHT_1); + auto dw_dt = grouped_weight_dt(MOE3GemmInputIndex::WEIGHT_2); // Use the model config to determine ZP presence (symmetric vs asymmetric quantization) bool has_zp = config.has_zp; @@ -2339,6 +2568,241 @@ class moe_3gemm_swiglu_opt_impl : public PrimitiveImplOCL { return result_event; } + // Returns the weight/ZP memory consumed by the grouped GEMM for one projection: + // the unpacked u4 scratch for u2 GEMMs, the original buffer otherwise. + const memory::ptr& grouped_weight_mem(int gemm_idx, const scratch_buffers& scratch) const { + return _gemm_weights_u2[gemm_idx] ? _u2_unpack_weight[gemm_idx] : scratch.moe_fusion_wei_addr.weight[gemm_idx]; + } + const memory::ptr& grouped_zp_mem(int gemm_idx, const scratch_buffers& scratch) const { + return _gemm_weights_u2[gemm_idx] ? _u2_unpack_zp[gemm_idx] : scratch.moe_fusion_wei_addr.zp[gemm_idx]; + } + + // u2 prefill: unpack the u2 expert weights (and ZP) into u4-packed scratch buffers so + // the grouped GEMM path (which has no u2 dtype) can serve prefill. The scratch buffers are + // allocated on first use and the unpack runs ONCE per bound weight buffer, not per prefill: + // the weights are constant (OTD is rejected for u2 in the ctor), so re-unpacking identical + // data every prefill was pure overhead — a full streaming pass over every expert of every + // layer, which measured ~430ms of TTFT per prefill on Qwen3.6-35B-A3B and made a multi-turn + // benchmark ~1.6x slower end-to-end than the same model at int4. See the *_key members. + cldnn::event::ptr unpack_u2_weights_for_prefill(typed_primitive_inst& instance, + scratch_buffers& scratch, + const std::vector& events) { + auto cur_moe = instance.get_typed_desc(); + const auto& config = cur_moe->_config; + auto& engine = instance.get_network().get_engine(); + cldnn::event::ptr ret = events.empty() ? nullptr : events[0]; + + const size_t lws = 256; + const auto round_up = [lws](size_t v) { + return (v + lws - 1) / lws * lws; + }; + for (int i = 0; i < 3; i++) { + if (!_gemm_weights_u2[i]) + continue; + + // Weights: byte-wise u2 -> u4 unpack, doubles the buffer size. The u2 and u4 + // expert weight buffers share the same logical element order ([expert, oc, ic], + // ic innermost, LSB-first packing), so a straight byte-wise unpack reproduces + // exactly the layout the u4 prefill paths expect. + const auto& src_mem = scratch.moe_fusion_wei_addr.weight[i]; + const size_t src_bytes = src_mem->size(); + OPENVINO_ASSERT(src_bytes % 4 == 0, "moe_3gemm u2 weight buffer byte size ", src_bytes, " is not a multiple of 4"); + const bool wei_realloc = !_u2_unpack_weight[i] || _u2_unpack_weight[i]->size() < src_bytes * 2; + if (wei_realloc) { + auto dst_layout = cldnn::layout({1, 1, 1, static_cast(src_bytes * 2)}, ov::element::u8, cldnn::format::bfyx); + _u2_unpack_weight[i] = engine.allocate_memory(dst_layout, allocation_type::usm_device, false); + } + // A fresh allocation is uninitialised, so it always forces the unpack to run. + if (wei_realloc || !_u2_unpack_weight_key[i].matches(src_mem)) { + const int src_uints = static_cast(src_bytes / 4); + ret = execute_stage({ret}, + instance, + *u2_unpack, + {src_mem}, + {_u2_unpack_weight[i]}, + {round_up(src_bytes / 4), 1, 1}, + {lws, 1, 1}, + false, + {src_uints, 0}); + _u2_unpack_weight_key[i].set(src_mem); + } + + // ZP: u2-packed ZP is unpacked like the weights; a scalar (per-tensor) ZP is + // broadcast into a full u4 ZP tensor [num_expert, num_groups, oc]. + if (!config.has_zp) + continue; + const auto& zp_src = scratch.moe_fusion_wei_addr.zp[i]; + OPENVINO_ASSERT(zp_src, "moe_3gemm u2 GEMM ", i, " has_zp but no ZP buffer bound"); + size_t zp_dst_bytes = 0; + size_t work_items = 0; + int zp_mode = 0; + int zp_count = 0; + if (zp_src->get_layout().count() == 1) { + // Size matches make_quant_md() in get_grouped_kernel: [num_expert, K / group_size, oc] u4. + const size_t k = (i == 2) ? static_cast(_intermediate_size) : static_cast(_hidden_size); + const size_t oc = (i == 2) ? static_cast(_hidden_size) : static_cast(_intermediate_size); + const size_t gs = (i == 2) ? static_cast(_down_group_size) : static_cast(_gate_up_group_size); + const size_t num_groups = (gs == 0 || gs >= k) ? 1 : (k / gs); + zp_dst_bytes = static_cast(config.num_expert) * num_groups * oc / 2; + work_items = zp_dst_bytes; // one output byte per work item + zp_mode = 1; + zp_count = static_cast(zp_dst_bytes); + } else if (zp_src->get_layout().data_type == data_types::u2) { + // u2-packed zp: same byte-wise unpack as the weights. + const size_t zp_src_bytes = zp_src->size(); + OPENVINO_ASSERT(zp_src_bytes % 4 == 0, "moe_3gemm u2 zp buffer byte size ", zp_src_bytes, " is not a multiple of 4"); + zp_dst_bytes = zp_src_bytes * 2; + work_items = zp_src_bytes / 4; // one input uint per work item + zp_mode = 0; + zp_count = static_cast(zp_src_bytes / 4); + } else { + // Byte-wide zp (u8/i8, e.g. per-channel or per-group): one VALUE per + // byte, NOT u2-packed data. Pack two adjacent values into one u4 + // byte (mode 2); a byte-wise u2 unpack would double the values into + // garbage and silently corrupt prefill dequant. + const size_t zp_vals = zp_src->get_layout().count(); + const size_t k = (i == 2) ? static_cast(_intermediate_size) : static_cast(_hidden_size); + const size_t oc = (i == 2) ? static_cast(_hidden_size) : static_cast(_intermediate_size); + const size_t gs = (i == 2) ? static_cast(_down_group_size) : static_cast(_gate_up_group_size); + const size_t num_groups = (gs == 0 || gs >= k) ? 1 : (k / gs); + const size_t expected = static_cast(config.num_expert) * num_groups * oc; + OPENVINO_ASSERT(zp_vals == expected, + "moe_3gemm u2 zp element count ", + zp_vals, + " does not match grouped GEMM zp descriptor [", + config.num_expert, + ", ", + num_groups, + ", ", + oc, + "]"); + zp_dst_bytes = (zp_vals + 1) / 2; + work_items = zp_dst_bytes; // one output byte per work item + zp_mode = 2; + zp_count = static_cast(zp_vals); + } + const bool zp_realloc = !_u2_unpack_zp[i] || _u2_unpack_zp[i]->size() < zp_dst_bytes; + if (zp_realloc) { + auto dst_layout = cldnn::layout({1, 1, 1, static_cast(zp_dst_bytes)}, ov::element::u8, cldnn::format::bfyx); + _u2_unpack_zp[i] = engine.allocate_memory(dst_layout, allocation_type::usm_device, false); + } + if (zp_realloc || !_u2_unpack_zp_key[i].matches(zp_src)) { + ret = execute_stage({ret}, + instance, + *u2_unpack, + {zp_src}, + {_u2_unpack_zp[i]}, + {round_up(work_items), 1, 1}, + {lws, 1, 1}, + false, + {zp_count, zp_mode}); + _u2_unpack_zp_key[i].set(zp_src); + } + } + + // Shared expert: same u2 -> u4 unpack per projection (gate/up/down/scalar-gate), + // consumed by the oneDNN shared-expert primitives built in init_shared_primitives. + // The u2 and u4 buffers share the logical element order ([oc, ic], ic innermost), + // so a straight byte-wise unpack matches the {ic, oc}/ba u4 descriptors. + for (int i = 0; i < 4; i++) { + if (!_shared_weights_u2[i]) + continue; + + const auto& src_mem = scratch.moe_fusion_wei_addr.shared_weight[i]; + OPENVINO_ASSERT(src_mem, "moe_3gemm shared weight ", i, " is u2 but its buffer is not bound"); + const size_t src_bytes = src_mem->size(); + OPENVINO_ASSERT(src_bytes % 4 == 0, "moe_3gemm u2 shared weight buffer byte size ", src_bytes, " is not a multiple of 4"); + const bool wei_realloc = !_u2_unpack_shared_weight[i] || _u2_unpack_shared_weight[i]->size() < src_bytes * 2; + if (wei_realloc) { + auto dst_layout = cldnn::layout({1, 1, 1, static_cast(src_bytes * 2)}, ov::element::u8, cldnn::format::bfyx); + _u2_unpack_shared_weight[i] = engine.allocate_memory(dst_layout, allocation_type::usm_device, false); + } + if (wei_realloc || !_u2_unpack_shared_weight_key[i].matches(src_mem)) { + const int src_uints = static_cast(src_bytes / 4); + ret = execute_stage({ret}, + instance, + *u2_unpack, + {src_mem}, + {_u2_unpack_shared_weight[i]}, + {round_up(src_bytes / 4), 1, 1}, + {lws, 1, 1}, + false, + {src_uints, 0}); + _u2_unpack_shared_weight_key[i].set(src_mem); + } + + // The scalar gate (i == 3) has no scale/zp (onednn_linear with ic_group_size = -1). + if (i == 3) + continue; + const auto& zp_src = scratch.moe_fusion_wei_addr.shared_zp[i]; + if (!zp_src || zp_src->get_layout().data_type == data_types::dynamic) + continue; // symmetric placeholder: no zp to unpack + // ZP sizes match the descriptors built in init_shared_primitives: + // gate/up: {_hidden_size / _gate_up_group_size, _shared_intermediate_size} u4, + // down: {_shared_intermediate_size / _down_group_size, _hidden_size} u4. + size_t zp_dst_bytes = 0; + size_t work_items = 0; + int zp_mode = 0; + int zp_count = 0; + if (zp_src->get_layout().count() == 1) { + // Scalar (per-tensor) zp: broadcast into the full u4 zp tensor. + const size_t k = (i == 2) ? static_cast(_shared_intermediate_size) : static_cast(_hidden_size); + const size_t oc = (i == 2) ? static_cast(_hidden_size) : static_cast(_shared_intermediate_size); + const size_t gs = (i == 2) ? static_cast(_down_group_size) : static_cast(_gate_up_group_size); + const size_t num_groups = (gs == 0 || gs >= k) ? 1 : (k / gs); + zp_dst_bytes = num_groups * oc / 2; + work_items = zp_dst_bytes; // one output byte per work item + zp_mode = 1; + zp_count = static_cast(zp_dst_bytes); + } else if (zp_src->get_layout().data_type == data_types::u2) { + const size_t zp_src_bytes = zp_src->size(); + OPENVINO_ASSERT(zp_src_bytes % 4 == 0, "moe_3gemm u2 shared zp buffer byte size ", zp_src_bytes, " is not a multiple of 4"); + zp_dst_bytes = zp_src_bytes * 2; + work_items = zp_src_bytes / 4; // one input uint per work item + zp_mode = 0; + zp_count = static_cast(zp_src_bytes / 4); + } else { + // Byte-wide zp (u8/i8): pack two values per u4 byte (mode 2), same as + // the routed-expert path; a byte-wise u2 unpack would corrupt it. + const size_t zp_vals = zp_src->get_layout().count(); + const size_t k = (i == 2) ? static_cast(_shared_intermediate_size) : static_cast(_hidden_size); + const size_t oc = (i == 2) ? static_cast(_hidden_size) : static_cast(_shared_intermediate_size); + const size_t gs = (i == 2) ? static_cast(_down_group_size) : static_cast(_gate_up_group_size); + const size_t num_groups = (gs == 0 || gs >= k) ? 1 : (k / gs); + OPENVINO_ASSERT(zp_vals == num_groups * oc, + "moe_3gemm u2 shared zp element count ", + zp_vals, + " does not match shared-expert zp descriptor [", + num_groups, + ", ", + oc, + "]"); + zp_dst_bytes = (zp_vals + 1) / 2; + work_items = zp_dst_bytes; + zp_mode = 2; + zp_count = static_cast(zp_vals); + } + const bool zp_realloc = !_u2_unpack_shared_zp[i] || _u2_unpack_shared_zp[i]->size() < zp_dst_bytes; + if (zp_realloc) { + auto dst_layout = cldnn::layout({1, 1, 1, static_cast(zp_dst_bytes)}, ov::element::u8, cldnn::format::bfyx); + _u2_unpack_shared_zp[i] = engine.allocate_memory(dst_layout, allocation_type::usm_device, false); + } + if (zp_realloc || !_u2_unpack_shared_zp_key[i].matches(zp_src)) { + ret = execute_stage({ret}, + instance, + *u2_unpack, + {zp_src}, + {_u2_unpack_shared_zp[i]}, + {round_up(work_items), 1, 1}, + {lws, 1, 1}, + false, + {zp_count, zp_mode}); + _u2_unpack_shared_zp_key[i].set(zp_src); + } + } + return ret; + } + // Third prefill path: OneDNN grouped GEMM (one matmul call per GEMM layer, all experts together). // This avoids the per-expert loop of exec_prefill_onednn while keeping full weight-format // compatibility (quantized or fp16 weights). @@ -2487,6 +2951,14 @@ class moe_3gemm_swiglu_opt_impl : public PrimitiveImplOCL { // ---------------------------------------------------------------- // Steps 3-5: OneDNN grouped GEMM – gate, up, SiLU, down // ---------------------------------------------------------------- + // A shorter buffer list than get_internal_buffer_descs() reserved means the allocation and + // the execution path selected here disagree, which would otherwise read out of bounds. + OPENVINO_ASSERT(intermediates_memories.size() > MOE_INTERNAL_BUFFER_GROUPED_OFFSETS, + "[GPU] moe_3gemm grouped GEMM prefill needs internal buffer ", + MOE_INTERNAL_BUFFER_GROUPED_OFFSETS, + " (GROUPED_OFFSETS), but only ", + intermediates_memories.size(), + " were allocated"); auto& gk = get_grouped_kernel(total_gathered_tokens, instance); auto row_offsets = intermediates_memories[MOE_INTERNAL_BUFFER_GROUPED_OFFSETS]; @@ -2501,7 +2973,7 @@ class moe_3gemm_swiglu_opt_impl : public PrimitiveImplOCL { { auto src_mem = scratch.x->get_onednn_grouped_memory(gk.gate_pd.src_desc(), *row_offsets); auto dst_mem = scratch.gate->get_onednn_grouped_memory(gk.gate_pd.dst_desc(), *row_offsets); - auto w_mem = scratch.moe_fusion_wei_addr.weight[0]->get_onednn_memory(gk.gate_pd.weights_desc()); + auto w_mem = grouped_weight_mem(0, scratch)->get_onednn_memory(gk.gate_pd.weights_desc()); auto scale_mem = scratch.moe_fusion_wei_addr.scale[0]->get_onednn_memory(gk.gate_scale_md); std::unordered_map args{{DNNL_ARG_SRC, src_mem}, @@ -2510,7 +2982,7 @@ class moe_3gemm_swiglu_opt_impl : public PrimitiveImplOCL { {DNNL_ARG_ATTR_SCALES | DNNL_ARG_WEIGHTS, scale_mem}, {DNNL_ARG_HINT_MAX_GROUP_SIZE, hint_mem}}; if (gk.has_zp) { - args.insert({DNNL_ARG_ATTR_ZERO_POINTS | DNNL_ARG_WEIGHTS, scratch.moe_fusion_wei_addr.zp[0]->get_onednn_memory(gk.gate_zp_md)}); + args.insert({DNNL_ARG_ATTR_ZERO_POINTS | DNNL_ARG_WEIGHTS, grouped_zp_mem(0, scratch)->get_onednn_memory(gk.gate_zp_md)}); } gk.gate_prim.execute(dnn_stream, args); } @@ -2519,7 +2991,7 @@ class moe_3gemm_swiglu_opt_impl : public PrimitiveImplOCL { { auto src_mem = scratch.x->get_onednn_grouped_memory(gk.up_pd.src_desc(), *row_offsets); auto dst_mem = scratch.up->get_onednn_grouped_memory(gk.up_pd.dst_desc(), *row_offsets); - auto w_mem = scratch.moe_fusion_wei_addr.weight[1]->get_onednn_memory(gk.up_pd.weights_desc()); + auto w_mem = grouped_weight_mem(1, scratch)->get_onednn_memory(gk.up_pd.weights_desc()); auto scale_mem = scratch.moe_fusion_wei_addr.scale[1]->get_onednn_memory(gk.up_scale_md); std::unordered_map args{{DNNL_ARG_SRC, src_mem}, @@ -2528,7 +3000,7 @@ class moe_3gemm_swiglu_opt_impl : public PrimitiveImplOCL { {DNNL_ARG_ATTR_SCALES | DNNL_ARG_WEIGHTS, scale_mem}, {DNNL_ARG_HINT_MAX_GROUP_SIZE, hint_mem}}; if (gk.has_zp) { - args.insert({DNNL_ARG_ATTR_ZERO_POINTS | DNNL_ARG_WEIGHTS, scratch.moe_fusion_wei_addr.zp[1]->get_onednn_memory(gk.up_zp_md)}); + args.insert({DNNL_ARG_ATTR_ZERO_POINTS | DNNL_ARG_WEIGHTS, grouped_zp_mem(1, scratch)->get_onednn_memory(gk.up_zp_md)}); } gk.up_prim.execute(dnn_stream, args); } @@ -2552,7 +3024,7 @@ class moe_3gemm_swiglu_opt_impl : public PrimitiveImplOCL { { auto src_mem = scratch.gate->get_onednn_grouped_memory(gk.down_pd.src_desc(), *row_offsets); auto dst_mem = scratch.y->get_onednn_grouped_memory(gk.down_pd.dst_desc(), *row_offsets); - auto w_mem = scratch.moe_fusion_wei_addr.weight[2]->get_onednn_memory(gk.down_pd.weights_desc()); + auto w_mem = grouped_weight_mem(2, scratch)->get_onednn_memory(gk.down_pd.weights_desc()); auto scale_mem = scratch.moe_fusion_wei_addr.scale[2]->get_onednn_memory(gk.down_scale_md); std::unordered_map args{{DNNL_ARG_SRC, src_mem}, @@ -2561,7 +3033,7 @@ class moe_3gemm_swiglu_opt_impl : public PrimitiveImplOCL { {DNNL_ARG_ATTR_SCALES | DNNL_ARG_WEIGHTS, scale_mem}, {DNNL_ARG_HINT_MAX_GROUP_SIZE, hint_mem}}; if (gk.has_zp) { - args.insert({DNNL_ARG_ATTR_ZERO_POINTS | DNNL_ARG_WEIGHTS, scratch.moe_fusion_wei_addr.zp[2]->get_onednn_memory(gk.down_zp_md)}); + args.insert({DNNL_ARG_ATTR_ZERO_POINTS | DNNL_ARG_WEIGHTS, grouped_zp_mem(2, scratch)->get_onednn_memory(gk.down_zp_md)}); } gk.down_prim.execute(dnn_stream, args); } @@ -2624,6 +3096,9 @@ class moe_3gemm_swiglu_opt_impl : public PrimitiveImplOCL { // Batched GEMV: for small token counts (including single token, MTP/speculative decoding), // use optimized GEMV kernels with batch dimension. Avoids gather/scatter overhead. + // u2 weights: decode takes this path; prefill (token_num > threshold) unpacks the + // u2 weights (routed and shared expert) into u4 scratch buffers and runs the grouped + // GEMM path below. if (token_num <= batched_gemv_threshold) { return exec_batched_gemv(events, instance, scratch, token_num); } @@ -2654,7 +3129,36 @@ class moe_3gemm_swiglu_opt_impl : public PrimitiveImplOCL { << ", use_micro_gemm_prefill=" << use_micro_gemm_prefill << ", use_grouped_gemm_prefill=" << use_grouped_gemm_prefill << std::endl; update_rt_params(instance); - if (use_micro_gemm_prefill) { + // u2 prefill: unpack the u2 weights (and zp) into u4 scratch buffers. Routed-expert + // u2 GEMMs are consumed by the grouped GEMM path below; shared-expert u2 projections + // are consumed by the oneDNN shared-expert primitives (init_shared_primitives), which + // run on every prefill path, so the unpack is not tied to _weights_u2 alone. + cldnn::event::ptr unpack_event = nullptr; + if (_weights_u2 || has_u2_shared_weights()) { + try { + unpack_event = unpack_u2_weights_for_prefill(instance, scratch, events); + } catch (const std::exception& e) { + // The batched GEMV fuses the shared expert in-kernel, so returning it + // directly also skips the oneDNN shared-expert block below. + GPU_DEBUG_TRACE_DETAIL << "u2 weight unpack for prefill failed (" << e.what() << "), falling back to batched GEMV" << std::endl; + instance.output_memory_ptr(0)->fill(stream, false); + return exec_batched_gemv(events, instance, scratch, token_num); + } + } + if (_weights_u2) { + // Run the grouped GEMM path on the unpacked u4 weights as regular u4 weights. If + // the grouped GEMM cannot be created/executed, fall back to the batched GEMV + // (the pre-u2-prefill behavior). + try { + ret_env = exec_prefill_grouped_gemm({unpack_event}, stream, instance, scratch); + } catch (const std::exception& e) { + GPU_DEBUG_TRACE_DETAIL << "u2 grouped GEMM prefill failed (" << e.what() << "), falling back to batched GEMV" << std::endl; + instance.output_memory_ptr(0)->fill(stream, false); + // Return directly: the batched GEMV fuses the shared expert in-kernel, so it + // must skip the oneDNN shared-expert block below (it would apply it twice). + return exec_batched_gemv(events, instance, scratch, token_num); + } + } else if (use_micro_gemm_prefill) { ret_env = exec_prefill_micro_gemm(events, instance, scratch, use_gpu_mask_gen); } else if (use_grouped_gemm_prefill) { ret_env = exec_prefill_grouped_gemm(events, stream, instance, scratch); diff --git a/src/plugins/intel_gpu/src/graph/impls/ocl_v2/moe/moe_3gemm_swiglu_opt.hpp b/src/plugins/intel_gpu/src/graph/impls/ocl_v2/moe/moe_3gemm_swiglu_opt.hpp index e527bdb4855b63..e545d3c5040c08 100644 --- a/src/plugins/intel_gpu/src/graph/impls/ocl_v2/moe/moe_3gemm_swiglu_opt.hpp +++ b/src/plugins/intel_gpu/src/graph/impls/ocl_v2/moe/moe_3gemm_swiglu_opt.hpp @@ -41,10 +41,11 @@ struct moe_3gemm_swiglu_opt : public ImplementationManager { return false; } - // Only support weight: u4, i4, u8, i8 + // Only support weight: u4, i4, u2, u8, i8 static constexpr std::array supported_wei_type = { ov::element::u4, ov::element::i4, + ov::element::u2, ov::element::u8, ov::element::i8, }; @@ -62,12 +63,13 @@ struct moe_3gemm_swiglu_opt : public ImplementationManager { return false; } - // Only support zp: u4, i4, u8, i8 (skip check for symmetric quantization where ZP is element::dynamic placeholder) + // Only support zp: u4, i4, u2, u8, i8 (skip check for symmetric quantization where ZP is element::dynamic placeholder) const auto& config = std::static_pointer_cast(node.get_primitive())->_config; if (config.has_zp) { static constexpr std::array supported_zp_type = { ov::element::u4, // asym-quant type ov::element::i4, // sym-quant type + ov::element::u2, // asym-quant type ov::element::u8, // asym-quant type ov::element::i8, // sym-quant type }; diff --git a/src/plugins/intel_gpu/src/graph/impls/ocl_v2/moe/moe_gemm_gen_opt.hpp b/src/plugins/intel_gpu/src/graph/impls/ocl_v2/moe/moe_gemm_gen_opt.hpp index 25ccb0b8075a98..11db3d8bffdc00 100644 --- a/src/plugins/intel_gpu/src/graph/impls/ocl_v2/moe/moe_gemm_gen_opt.hpp +++ b/src/plugins/intel_gpu/src/graph/impls/ocl_v2/moe/moe_gemm_gen_opt.hpp @@ -35,7 +35,7 @@ class MoEGemmOptGeneratorBase : public MoEGemmBase { static moe_config get_moe_cfg(const kernel_impl_params& params) { moe_config moe_cfg; auto desc = params.typed_desc(); - std::vector quantized_types = {data_types::u4, data_types::i4, data_types::u8, data_types::i8}; + std::vector quantized_types = {data_types::u4, data_types::i4, data_types::u2, data_types::u8, data_types::i8}; moe_cfg.has_bias = desc->has_bias; if (std::any_of(quantized_types.begin(), quantized_types.end(), [=](const cldnn::data_types& t) -> bool { return t == params.input_layouts[1].data_type; diff --git a/src/plugins/intel_gpu/src/graph/impls/ocl_v2/moe_3gemm_swiglu_mlp.cl b/src/plugins/intel_gpu/src/graph/impls/ocl_v2/moe_3gemm_swiglu_mlp.cl index 269c960f1bd721..7919720c622ede 100644 --- a/src/plugins/intel_gpu/src/graph/impls/ocl_v2/moe_3gemm_swiglu_mlp.cl +++ b/src/plugins/intel_gpu/src/graph/impls/ocl_v2/moe_3gemm_swiglu_mlp.cl @@ -29,7 +29,8 @@ // Number of K-elements each work-item handles per gk-iteration via the // intel_sub_group_block_read tile. Drives the inner-loop variant selection. -// Supported values: 2 (FAKE_GROUP_SIZE = 1 * SUBGROUP_SIZE * 2, e.g. SG=16 + 32), +// Supported values: 1 (u2 kernels only: FAKE_GROUP_SIZE = SUBGROUP_SIZE, e.g. SG=32 + FAKE=32), +// 2 (FAKE_GROUP_SIZE = 1 * SUBGROUP_SIZE * 2, e.g. SG=16 + 32), // 4 (e.g. SG=32 + FAKE=128 or SG=16 + FAKE=64), and 8 (SG=16 + FAKE=128). #define ELEMS_PER_LANE (FAKE_GROUP_SIZE / SUBGROUP_SIZE) @@ -86,14 +87,60 @@ inline float moe_mlp_fast_erf(float x) { # define DEQUANT_8BIT(v) convert_half(v) #endif +// 2-bit (u2) dequant: 4 values per byte, LSB-first; u2 weights are always unsigned. +#define DEQUANT_2BIT(v, s) convert_half(((v) >> (s)) & 0x3) + // Per-GEMM helpers (mixed-dtype MoE): per-expert byte strides given a compression code DT -// (0=u4/i4, 1=u8/i8, 2=f16) and group size GS. N*K equals INTERMEDIATE_SIZE*HIDDEN_SIZE for all +// (0=u4/i4, 1=u8/i8, 2=f16, 3=u2) and group size GS. N*K equals INTERMEDIATE_SIZE*HIDDEN_SIZE for all // three GEMMs, so one product serves gate/up/down. #define MOE_WEI_PROD (INTERMEDIATE_SIZE * HIDDEN_SIZE) -#define MOE_EXPERT_WEI_BYTES(DT) ((DT) == 0 ? (MOE_WEI_PROD) / 2 : (MOE_WEI_PROD)) -#define MOE_EXPERT_ZP_BYTES(DT, GS) ((DT) == 0 ? (MOE_WEI_PROD) / 2 / (GS) : (MOE_WEI_PROD) / (GS)) +#define MOE_EXPERT_WEI_BYTES(DT) ((DT) == 0 ? (MOE_WEI_PROD) / 2 : (DT) == 3 ? (MOE_WEI_PROD) / 4 : (MOE_WEI_PROD)) +#define MOE_EXPERT_ZP_BYTES(DT, GS) \ + ((DT) == 0 ? (MOE_WEI_PROD) / 2 / (GS) : (DT) == 3 ? (MOE_WEI_PROD) / 4 / (GS) : (MOE_WEI_PROD) / (GS)) + +#if defined(U2_UNPACK_ENABLE) + +// Unpacks u2-packed data (4 values per byte, LSB-first) into u4-packed data +// (2 values per byte, LSB-first), doubling the byte size while preserving the +// logical element order. Used to feed u2 MoE expert weights/zp to the u4-only +// prefill GEMM paths (micro-gemm/oneDNN have no u2 dtype). +inline uint moe_unpack_u2_byte(uint b) { + // byte (v3 v2 v1 v0) -> uint16 ((v3 | v2 << 4) << 8) | (v0 | v1 << 4) + return (b & 0x3u) | ((b & 0xCu) << 2) | ((b & 0x30u) << 4) | ((b & 0xC0u) << 6); +} + +// broadcast_zp == 0: src holds src_count uints of u2-packed data; each work item +// unpacks one uint (4 bytes) into 2 uints (8 bytes) of u4 data. +// broadcast_zp == 1: src holds a single scalar zp element (byte); each work item +// replicates it into both nibbles of one output byte, so a +// per-tensor zp is materialized as a full u4 zp tensor and +// src_count is the output byte count. +// broadcast_zp == 2: src holds src_count single-byte zp VALUES (u8/i8, e.g. a +// per-channel zp); each work item packs two adjacent values +// into one u4 output byte, so dst holds ceil(src_count/2) bytes. +KERNEL(moe_unpack_u2_to_u4)(const __global uchar* src, __global uchar* dst, int src_count, int broadcast_zp) { + const size_t i = get_global_id(0); + if (broadcast_zp == 2) { + if (i >= ((size_t)src_count + 1) / 2) + return; + const uint v0 = src[2 * i]; + const uint v1 = (2 * i + 1 < (size_t)src_count) ? src[2 * i + 1] : 0; + dst[i] = (uchar)((v0 & 0xFu) | ((v1 & 0xFu) << 4)); + return; + } + if (i >= (size_t)src_count) + return; + if (broadcast_zp != 0) { + const uint v = src[0]; + dst[i] = (uchar)(v | (v << 4)); + return; + } + const uint v = ((const __global uint*)src)[i]; + ((__global uint*)dst)[2 * i] = moe_unpack_u2_byte(v & 0xFFu) | (moe_unpack_u2_byte((v >> 8) & 0xFFu) << 16); + ((__global uint*)dst)[2 * i + 1] = moe_unpack_u2_byte((v >> 16) & 0xFFu) | (moe_unpack_u2_byte(v >> 24) << 16); +} -#if GATE_UP_ENABLE +#elif GATE_UP_ENABLE inline void gate_up_gemv_n2x_u4(const __global uchar* weight, __global half* scales, __global uchar* zps, @@ -223,6 +270,170 @@ inline void gate_up_gemv_n2x_u4(const __global uchar* weight, } } +// zp_scalar must come from the caller, not from a preprocessor branch: one compiled body serves +// both the gate and the up projection, and a mixed INT2_SYM/INT2_ASYM layer gives them different +// zero-point forms. Both callers pass a jit constant, so the selects below fold away. +inline void gate_up_gemv_n2x_u2(const __global uchar* weight, + __global half* scales, + __global uchar* zps, + __global half* y, + int N, + int K, + half* x2, + float* xg_sum, + const bool silu, + const bool zp_scalar) { + int id_local = get_sub_group_local_id(); + + int n_start = get_global_id(2) * N_BLOCK; + int n_end = n_start + N_BLOCK; + unroll_for(int n = n_start; n < n_end; n += 2) { + const __global uchar* B = weight + n * K / 4; + float sum_all0 = 0; + float sum_all1 = 0; + __global half* S = scales + n; +#if HAS_ZP + __global uchar* Z = zps + n / 4; + // n is even, so channels n and n+1 always share one packed zp byte. + const int zshift = (n & 3) * 2; +#endif + unroll_for(int gk = 0; gk < K / FAKE_GROUP_SIZE; gk++) { + int scale_offset = (gk * FAKE_GROUP_SIZE / GATE_UP_GROUP_SIZE) * N; + half s0 = S[scale_offset]; + half s1 = S[scale_offset + 1]; +#if HAS_ZP + const half z_scalar = convert_half(((__global MOE_ZP_SCALAR_DT*)zps)[0]); + const uchar z = zp_scalar ? (uchar)0 : Z[(gk * FAKE_GROUP_SIZE / GATE_UP_GROUP_SIZE) * N / 4]; + const half z_hf0 = zp_scalar ? z_scalar : convert_half((z >> zshift) & 0x3); + const half z_hf1 = zp_scalar ? z_scalar : convert_half((z >> (zshift + 2)) & 0x3); +#endif + +# if ELEMS_PER_LANE == 4 + // Each lane reads 4 K-elements (1 byte = 4 x 2-bit values). + half2 sum0; + half2 sum1; + half4 a = vload4(id_local, x2 + gk * FAKE_GROUP_SIZE); + uchar b = intel_sub_group_block_read_uc((const __global uchar*)B + gk * FAKE_GROUP_SIZE / 4); + uchar b2 = intel_sub_group_block_read_uc((const __global uchar*)(B + (K / 4) + gk * FAKE_GROUP_SIZE / 4)); + + sum0.s0 = fma(a.s0, (DEQUANT_2BIT(b, 0)), 0); + sum0.s1 = fma(a.s1, (DEQUANT_2BIT(b, 2)), 0); + sum0.s0 = fma(a.s2, (DEQUANT_2BIT(b, 4)), sum0.s0); + sum0.s1 = fma(a.s3, (DEQUANT_2BIT(b, 6)), sum0.s1); + + sum1.s0 = fma(a.s0, (DEQUANT_2BIT(b2, 0)), 0); + sum1.s1 = fma(a.s1, (DEQUANT_2BIT(b2, 2)), 0); + sum1.s0 = fma(a.s2, (DEQUANT_2BIT(b2, 4)), sum1.s0); + sum1.s1 = fma(a.s3, (DEQUANT_2BIT(b2, 6)), sum1.s1); + +# if HAS_ZP + sum_all0 += (sum0[0] + sum0[1] - xg_sum[gk] * z_hf0) * s0; + sum_all1 += (sum1[0] + sum1[1] - xg_sum[gk] * z_hf1) * s1; +# else + sum_all0 += (sum0[0] + sum0[1]) * s0; + sum_all1 += (sum1[0] + sum1[1]) * s1; +# endif +# elif ELEMS_PER_LANE == 2 + // Each lane handles 2 K-elements; two lanes share one packed byte. + // NOTE: keep per-lane scalar loads here. A subgroup block read would make + // lanes >= FAKE_GROUP_SIZE/4 read past the tile, and at the end of the + // mmap'd weights buffer that overhang faults (CL_OUT_OF_RESOURCES). + half sum0; + half sum1; + half2 a = vload2(id_local, x2 + gk * FAKE_GROUP_SIZE); + uchar b = B[gk * FAKE_GROUP_SIZE / 4 + (id_local >> 1)]; + uchar b2 = B[(K / 4) + gk * FAKE_GROUP_SIZE / 4 + (id_local >> 1)]; + const int wshift = (id_local & 1) * 4; + + sum0 = fma(a.s0, (DEQUANT_2BIT(b, wshift)), (half)0); + sum0 = fma(a.s1, (DEQUANT_2BIT(b, wshift + 2)), sum0); + + sum1 = fma(a.s0, (DEQUANT_2BIT(b2, wshift)), (half)0); + sum1 = fma(a.s1, (DEQUANT_2BIT(b2, wshift + 2)), sum1); + +# if HAS_ZP + sum_all0 += (sum0 - xg_sum[gk] * z_hf0) * s0; + sum_all1 += (sum1 - xg_sum[gk] * z_hf1) * s1; +# else + sum_all0 += sum0 * s0; + sum_all1 += sum1 * s1; +# endif +# elif ELEMS_PER_LANE == 1 + // Each lane handles 1 K-element; four lanes share one packed byte. + // Same per-lane scalar loads as the ==2 branch: subgroup block reads + // would overhang the weights buffer and fault (CL_OUT_OF_RESOURCES). + half sum0; + half sum1; + half a = x2[gk * FAKE_GROUP_SIZE + id_local]; + uchar b = B[gk * FAKE_GROUP_SIZE / 4 + (id_local >> 2)]; + uchar b2 = B[(K / 4) + gk * FAKE_GROUP_SIZE / 4 + (id_local >> 2)]; + const int wshift = (id_local & 3) * 2; + + sum0 = fma(a, (DEQUANT_2BIT(b, wshift)), (half)0); + sum1 = fma(a, (DEQUANT_2BIT(b2, wshift)), (half)0); + +# if HAS_ZP + sum_all0 += (sum0 - xg_sum[gk] * z_hf0) * s0; + sum_all1 += (sum1 - xg_sum[gk] * z_hf1) * s1; +# else + sum_all0 += sum0 * s0; + sum_all1 += sum1 * s1; +# endif +# else + half4 sum0; + half4 sum1; + // The block read is strided: lane L gets byte L and byte L+SUBGROUP_SIZE, which cover + // K elements 4L.. and 4L+FAKE_GROUP_SIZE/2.. respectively. The activations must be + // loaded in two matching contiguous quads, not as one vload8 of 8L... + half4 alo = vload4(id_local, x2 + gk * FAKE_GROUP_SIZE); + half4 ahi = vload4(id_local, x2 + gk * FAKE_GROUP_SIZE + FAKE_GROUP_SIZE / 2); + uchar2 b = intel_sub_group_block_read_uc2((const __global uchar*)B + gk * FAKE_GROUP_SIZE / 4); + uchar2 b2 = intel_sub_group_block_read_uc2((const __global uchar*)(B + (K / 4) + gk * FAKE_GROUP_SIZE / 4)); + + sum0.s0 = fma(alo.s0, (DEQUANT_2BIT(b.s0, 0)), 0); + sum0.s1 = fma(alo.s1, (DEQUANT_2BIT(b.s0, 2)), 0); + sum0.s2 = fma(alo.s2, (DEQUANT_2BIT(b.s0, 4)), 0); + sum0.s3 = fma(alo.s3, (DEQUANT_2BIT(b.s0, 6)), 0); + + sum0.s0 = fma(ahi.s0, (DEQUANT_2BIT(b.s1, 0)), sum0.s0); + sum0.s1 = fma(ahi.s1, (DEQUANT_2BIT(b.s1, 2)), sum0.s1); + sum0.s2 = fma(ahi.s2, (DEQUANT_2BIT(b.s1, 4)), sum0.s2); + sum0.s3 = fma(ahi.s3, (DEQUANT_2BIT(b.s1, 6)), sum0.s3); + + sum1.s0 = fma(alo.s0, (DEQUANT_2BIT(b2.s0, 0)), 0); + sum1.s1 = fma(alo.s1, (DEQUANT_2BIT(b2.s0, 2)), 0); + sum1.s2 = fma(alo.s2, (DEQUANT_2BIT(b2.s0, 4)), 0); + sum1.s3 = fma(alo.s3, (DEQUANT_2BIT(b2.s0, 6)), 0); + + sum1.s0 = fma(ahi.s0, (DEQUANT_2BIT(b2.s1, 0)), sum1.s0); + sum1.s1 = fma(ahi.s1, (DEQUANT_2BIT(b2.s1, 2)), sum1.s1); + sum1.s2 = fma(ahi.s2, (DEQUANT_2BIT(b2.s1, 4)), sum1.s2); + sum1.s3 = fma(ahi.s3, (DEQUANT_2BIT(b2.s1, 6)), sum1.s3); + +# if HAS_ZP + sum_all0 += (sum0[0] + sum0[1] + sum0[2] + sum0[3] - xg_sum[gk] * z_hf0) * s0; + sum_all1 += (sum1[0] + sum1[1] + sum1[2] + sum1[3] - xg_sum[gk] * z_hf1) * s1; +# else + sum_all0 += (sum0[0] + sum0[1] + sum0[2] + sum0[3]) * s0; + sum_all1 += (sum1[0] + sum1[1] + sum1[2] + sum1[3]) * s1; +# endif +# endif + } + + sum_all0 = sub_group_reduce_add(sum_all0); + sum_all1 = sub_group_reduce_add(sum_all1); + if (id_local == 0) { + if (silu) { + y[n] *= MOE_GATE_ACT(sum_all0); + y[n + 1] *= MOE_GATE_ACT(sum_all1); + } else { + y[n] = sum_all0; + y[n + 1] = sum_all1; + } + } + } +} + inline void gate_up_gemv_n2x_u8(const __global uchar* weight, __global half* scales, __global uchar* zps, @@ -482,8 +693,8 @@ __attribute__((intel_reqd_sub_group_size(SUBGROUP_SIZE))) KERNEL(mlp_gate_up)( const int gate_expert_wei_size = MOE_EXPERT_WEI_BYTES(GATE_WEIGHT_DT); const int up_expert_wei_size = MOE_EXPERT_WEI_BYTES(UP_WEIGHT_DT); const int expert_scale_size = INTERMEDIATE_SIZE * HIDDEN_SIZE / GATE_UP_GROUP_SIZE; // f16 scale, dtype-independent - const int gate_zp_stride = MOE_EXPERT_ZP_BYTES(GATE_WEIGHT_DT, GATE_UP_GROUP_SIZE); - const int up_zp_stride = MOE_EXPERT_ZP_BYTES(UP_WEIGHT_DT, GATE_UP_GROUP_SIZE); + const int gate_zp_stride = GATE_ZP_SCALAR ? 0 : MOE_EXPERT_ZP_BYTES(GATE_WEIGHT_DT, GATE_UP_GROUP_SIZE); + const int up_zp_stride = UP_ZP_SCALAR ? 0 : MOE_EXPERT_ZP_BYTES(UP_WEIGHT_DT, GATE_UP_GROUP_SIZE); int expert_id = 0; // gate, [HIDDEN_SIZE, INTERMEDIATE_SIZE] @@ -636,6 +847,8 @@ __attribute__((intel_reqd_sub_group_size(SUBGROUP_SIZE))) KERNEL(mlp_gate_up)( gate_up_gemv_n2x_u8(up_weight, up_scale, up_zp, y, INTERMEDIATE_SIZE, HIDDEN_SIZE, x2, xg_sum, false); # elif UP_WEIGHT_DT == 2 gate_up_gemv_n2x_f16(up_weight, up_zp, y, INTERMEDIATE_SIZE, HIDDEN_SIZE, x2, false); +# elif UP_WEIGHT_DT == 3 + gate_up_gemv_n2x_u2(up_weight, up_scale, up_zp, y, INTERMEDIATE_SIZE, HIDDEN_SIZE, x2, xg_sum, false, UP_ZP_SCALAR); # endif # if GATE_WEIGHT_DT == 0 gate_up_gemv_n2x_u4(gate_weight, gate_scale, gate_zp, y, INTERMEDIATE_SIZE, HIDDEN_SIZE, x2, xg_sum, true); @@ -643,6 +856,8 @@ __attribute__((intel_reqd_sub_group_size(SUBGROUP_SIZE))) KERNEL(mlp_gate_up)( gate_up_gemv_n2x_u8(gate_weight, gate_scale, gate_zp, y, INTERMEDIATE_SIZE, HIDDEN_SIZE, x2, xg_sum, true); # elif GATE_WEIGHT_DT == 2 gate_up_gemv_n2x_f16(gate_weight, gate_zp, y, INTERMEDIATE_SIZE, HIDDEN_SIZE, x2, true); +# elif GATE_WEIGHT_DT == 3 + gate_up_gemv_n2x_u2(gate_weight, gate_scale, gate_zp, y, INTERMEDIATE_SIZE, HIDDEN_SIZE, x2, xg_sum, true, GATE_ZP_SCALAR); # endif } @@ -770,6 +985,164 @@ inline void down_gemv_n2x_u4(const __global uchar* weight, } } +inline void down_gemv_n2x_u2(const __global uchar* weight, + __global half* scales, + __global uchar* zps, + MOE_DTYPE routing_weight_val, + __global half* y, + int N, + int K, + half* x2, + float* xg_sum) { + int id_local = get_sub_group_local_id(); + int n_start = get_global_id(2) * N_BLOCK; + int n_end = n_start + N_BLOCK; + + unroll_for(int n = n_start; n < n_end; n += 2) { + const __global uchar* B = weight + n * K / 4; + __global half* S = scales + n; +#if HAS_ZP + __global uchar* Z = zps + n / 4; + // n is even, so channels n and n+1 always share one packed zp byte. + const int zshift = (n & 3) * 2; +#endif + float sum_all0 = 0; + float sum_all1 = 0; + unroll_for(int gk = 0; gk < K / FAKE_GROUP_SIZE; gk++) { + int scale_offset = (gk * FAKE_GROUP_SIZE / DOWN_GROUP_SIZE) * N; + half s0 = S[scale_offset]; + half s1 = S[scale_offset + 1]; +#if HAS_ZP +# if DOWN_ZP_SCALAR + const half z_hf0 = convert_half(((__global MOE_ZP_SCALAR_DT*)zps)[0]); + const half z_hf1 = z_hf0; +# else + const uchar z = Z[(gk * FAKE_GROUP_SIZE / DOWN_GROUP_SIZE) * N / 4]; + const half z_hf0 = convert_half((z >> zshift) & 0x3); + const half z_hf1 = convert_half((z >> (zshift + 2)) & 0x3); +# endif +#endif + +# if ELEMS_PER_LANE == 4 + // Each lane reads 4 K-elements (1 byte = 4 x 2-bit values). + half2 sum0; + half2 sum1; + half4 a = vload4(id_local, x2 + gk * FAKE_GROUP_SIZE); + uchar b = intel_sub_group_block_read_uc((const __global uchar*)B + gk * FAKE_GROUP_SIZE / 4); + uchar b2 = intel_sub_group_block_read_uc((const __global uchar*)(B + (K / 4) + gk * FAKE_GROUP_SIZE / 4)); + + sum0.s0 = fma(a.s0, (DEQUANT_2BIT(b, 0)), 0); + sum0.s1 = fma(a.s1, (DEQUANT_2BIT(b, 2)), 0); + sum0.s0 = fma(a.s2, (DEQUANT_2BIT(b, 4)), sum0.s0); + sum0.s1 = fma(a.s3, (DEQUANT_2BIT(b, 6)), sum0.s1); + + sum1.s0 = fma(a.s0, (DEQUANT_2BIT(b2, 0)), 0); + sum1.s1 = fma(a.s1, (DEQUANT_2BIT(b2, 2)), 0); + sum1.s0 = fma(a.s2, (DEQUANT_2BIT(b2, 4)), sum1.s0); + sum1.s1 = fma(a.s3, (DEQUANT_2BIT(b2, 6)), sum1.s1); + +# if HAS_ZP + sum_all0 += (sum0[0] + sum0[1] - xg_sum[gk] * z_hf0) * s0; + sum_all1 += (sum1[0] + sum1[1] - xg_sum[gk] * z_hf1) * s1; +# else + sum_all0 += (sum0[0] + sum0[1]) * s0; + sum_all1 += (sum1[0] + sum1[1]) * s1; +# endif +# elif ELEMS_PER_LANE == 2 + // Each lane handles 2 K-elements; two lanes share one packed byte. + // NOTE: keep per-lane scalar loads here. A subgroup block read would make + // lanes >= FAKE_GROUP_SIZE/4 read past the tile, and at the end of the + // mmap'd weights buffer that overhang faults (CL_OUT_OF_RESOURCES). + half sum0; + half sum1; + half2 a = vload2(id_local, x2 + gk * FAKE_GROUP_SIZE); + uchar b = B[gk * FAKE_GROUP_SIZE / 4 + (id_local >> 1)]; + uchar b2 = B[(K / 4) + gk * FAKE_GROUP_SIZE / 4 + (id_local >> 1)]; + const int wshift = (id_local & 1) * 4; + + sum0 = fma(a.s0, (DEQUANT_2BIT(b, wshift)), (half)0); + sum0 = fma(a.s1, (DEQUANT_2BIT(b, wshift + 2)), sum0); + + sum1 = fma(a.s0, (DEQUANT_2BIT(b2, wshift)), (half)0); + sum1 = fma(a.s1, (DEQUANT_2BIT(b2, wshift + 2)), sum1); + +# if HAS_ZP + sum_all0 += (sum0 - xg_sum[gk] * z_hf0) * s0; + sum_all1 += (sum1 - xg_sum[gk] * z_hf1) * s1; +# else + sum_all0 += sum0 * s0; + sum_all1 += sum1 * s1; +# endif +# elif ELEMS_PER_LANE == 1 + // Each lane handles 1 K-element; four lanes share one packed byte. + // Same per-lane scalar loads as the ==2 branch: subgroup block reads + // would overhang the weights buffer and fault (CL_OUT_OF_RESOURCES). + half sum0; + half sum1; + half a = x2[gk * FAKE_GROUP_SIZE + id_local]; + uchar b = B[gk * FAKE_GROUP_SIZE / 4 + (id_local >> 2)]; + uchar b2 = B[(K / 4) + gk * FAKE_GROUP_SIZE / 4 + (id_local >> 2)]; + const int wshift = (id_local & 3) * 2; + + sum0 = fma(a, (DEQUANT_2BIT(b, wshift)), (half)0); + sum1 = fma(a, (DEQUANT_2BIT(b2, wshift)), (half)0); + +# if HAS_ZP + sum_all0 += (sum0 - xg_sum[gk] * z_hf0) * s0; + sum_all1 += (sum1 - xg_sum[gk] * z_hf1) * s1; +# else + sum_all0 += sum0 * s0; + sum_all1 += sum1 * s1; +# endif +# else + half4 sum0; + half4 sum1; + // The block read is strided: lane L gets byte L and byte L+SUBGROUP_SIZE, which cover + // K elements 4L.. and 4L+FAKE_GROUP_SIZE/2.. respectively. The activations must be + // loaded in two matching contiguous quads, not as one vload8 of 8L... + half4 alo = vload4(id_local, x2 + gk * FAKE_GROUP_SIZE); + half4 ahi = vload4(id_local, x2 + gk * FAKE_GROUP_SIZE + FAKE_GROUP_SIZE / 2); + uchar2 b = intel_sub_group_block_read_uc2((const __global uchar*)B + gk * FAKE_GROUP_SIZE / 4); + uchar2 b2 = intel_sub_group_block_read_uc2((const __global uchar*)(B + (K / 4) + gk * FAKE_GROUP_SIZE / 4)); + + sum0.s0 = fma(alo.s0, (DEQUANT_2BIT(b.s0, 0)), 0); + sum0.s1 = fma(alo.s1, (DEQUANT_2BIT(b.s0, 2)), 0); + sum0.s2 = fma(alo.s2, (DEQUANT_2BIT(b.s0, 4)), 0); + sum0.s3 = fma(alo.s3, (DEQUANT_2BIT(b.s0, 6)), 0); + + sum0.s0 = fma(ahi.s0, (DEQUANT_2BIT(b.s1, 0)), sum0.s0); + sum0.s1 = fma(ahi.s1, (DEQUANT_2BIT(b.s1, 2)), sum0.s1); + sum0.s2 = fma(ahi.s2, (DEQUANT_2BIT(b.s1, 4)), sum0.s2); + sum0.s3 = fma(ahi.s3, (DEQUANT_2BIT(b.s1, 6)), sum0.s3); + + sum1.s0 = fma(alo.s0, (DEQUANT_2BIT(b2.s0, 0)), 0); + sum1.s1 = fma(alo.s1, (DEQUANT_2BIT(b2.s0, 2)), 0); + sum1.s2 = fma(alo.s2, (DEQUANT_2BIT(b2.s0, 4)), 0); + sum1.s3 = fma(alo.s3, (DEQUANT_2BIT(b2.s0, 6)), 0); + + sum1.s0 = fma(ahi.s0, (DEQUANT_2BIT(b2.s1, 0)), sum1.s0); + sum1.s1 = fma(ahi.s1, (DEQUANT_2BIT(b2.s1, 2)), sum1.s1); + sum1.s2 = fma(ahi.s2, (DEQUANT_2BIT(b2.s1, 4)), sum1.s2); + sum1.s3 = fma(ahi.s3, (DEQUANT_2BIT(b2.s1, 6)), sum1.s3); + +# if HAS_ZP + sum_all0 += (sum0[0] + sum0[1] + sum0[2] + sum0[3] - xg_sum[gk] * z_hf0) * s0; + sum_all1 += (sum1[0] + sum1[1] + sum1[2] + sum1[3] - xg_sum[gk] * z_hf1) * s1; +# else + sum_all0 += (sum0[0] + sum0[1] + sum0[2] + sum0[3]) * s0; + sum_all1 += (sum1[0] + sum1[1] + sum1[2] + sum1[3]) * s1; +# endif +# endif + } + sum_all0 = sub_group_reduce_add(sum_all0); + sum_all1 = sub_group_reduce_add(sum_all1); + if (id_local == 0) { + y[n] = sum_all0 * routing_weight_val; + y[n + 1] = sum_all1 * routing_weight_val; + } + } +} + inline void down_gemv_n2x_u8(const __global uchar* weight, __global half* scales, __global uchar* zps, @@ -1008,7 +1381,7 @@ __attribute__((intel_reqd_sub_group_size(SUBGROUP_SIZE))) KERNEL(mlp_down)(const // Per-GEMM strides for the down projection. const int expert_wei_size = MOE_EXPERT_WEI_BYTES(DOWN_WEIGHT_DT); const int expert_scale_size = INTERMEDIATE_SIZE * HIDDEN_SIZE / DOWN_GROUP_SIZE; // f16 scale, dtype-independent - const int down_zp_stride = MOE_EXPERT_ZP_BYTES(DOWN_WEIGHT_DT, DOWN_GROUP_SIZE); + const int down_zp_stride = DOWN_ZP_SCALAR ? 0 : MOE_EXPERT_ZP_BYTES(DOWN_WEIGHT_DT, DOWN_GROUP_SIZE); int expert_id = 0; // down, [INTERMEDIATE_SIZE, HIDDEN_SIZE] @@ -1118,6 +1491,8 @@ __attribute__((intel_reqd_sub_group_size(SUBGROUP_SIZE))) KERNEL(mlp_down)(const down_gemv_n2x_u8(weight, scales, zps, routing_weight_val, y, N, K, x2, xg_sum); # elif DOWN_WEIGHT_DT == 2 down_gemv_n2x_f16(weight, routing_weight_val, y, N, K, x2); +# elif DOWN_WEIGHT_DT == 3 + down_gemv_n2x_u2(weight, scales, zps, routing_weight_val, y, N, K, x2, xg_sum); # endif } @@ -1141,4 +1516,5 @@ __attribute__((intel_reqd_sub_group_size(SUBGROUP_SIZE))) KERNEL(mlp_reduce)(con } intel_sub_group_block_write_us((__global ushort*)(y + token_idx * HIDDEN_SIZE + n), as_ushort(sum[0])); } + #endif diff --git a/src/plugins/intel_gpu/src/graph/impls/ocl_v2/utils/jitter.cpp b/src/plugins/intel_gpu/src/graph/impls/ocl_v2/utils/jitter.cpp index 369c88a2b92f2e..267631e70a8b13 100644 --- a/src/plugins/intel_gpu/src/graph/impls/ocl_v2/utils/jitter.cpp +++ b/src/plugins/intel_gpu/src/graph/impls/ocl_v2/utils/jitter.cpp @@ -301,6 +301,11 @@ JitConstants make_type_jit_constants(const std::string& name, const ov::element: type_size = "0.5f"; is_fp = false; break; + case ov::element::u2: + type = "uchar"; + type_size = "0.25f"; + is_fp = false; + break; case ov::element::bf16: type = "ushort"; max_val = "FLT_MAX"; diff --git a/src/plugins/intel_gpu/src/plugin/transformations/fuse_gated_mlp.cpp b/src/plugins/intel_gpu/src/plugin/transformations/fuse_gated_mlp.cpp index 075360e7ccd3ea..6227e1ba1ba470 100644 --- a/src/plugins/intel_gpu/src/plugin/transformations/fuse_gated_mlp.cpp +++ b/src/plugins/intel_gpu/src/plugin/transformations/fuse_gated_mlp.cpp @@ -116,7 +116,8 @@ FuseGatedMLP::FuseGatedMLP() { auto is_compressed_const = [](const ov::Output& output) { const auto et = output.get_element_type(); - return et == ov::element::u8 || et == ov::element::i8 || et == ov::element::u4 || et == ov::element::i4; + return et == ov::element::u8 || et == ov::element::i8 || et == ov::element::u4 || et == ov::element::i4 || + et == ov::element::u2; }; auto parse_decompression = [&](const ov::Output& maybe_decompressed_weight) -> decompression_info { @@ -207,7 +208,8 @@ FuseGatedMLP::FuseGatedMLP() { const auto weight_et = compressed_weights.get_element_type(); const auto zp_et = zp_output.get_element_type(); - if ((weight_et == ov::element::u8 || weight_et == ov::element::u4) && zp_et == ov::element::u4) { + if ((weight_et == ov::element::u8 || weight_et == ov::element::u4 || weight_et == ov::element::u2) && + (zp_et == ov::element::u4 || zp_et == ov::element::u2)) { auto zp_convert = std::make_shared(zp_output, ov::element::u8); new_ops.push_back(zp_convert); MatcherPass::register_new_node(zp_convert); diff --git a/src/plugins/intel_gpu/src/plugin/transformations/keep_moe_3gemm_const_precision.cpp b/src/plugins/intel_gpu/src/plugin/transformations/keep_moe_3gemm_const_precision.cpp index 1ae62e4e698a1b..3c1d7e080abdc4 100644 --- a/src/plugins/intel_gpu/src/plugin/transformations/keep_moe_3gemm_const_precision.cpp +++ b/src/plugins/intel_gpu/src/plugin/transformations/keep_moe_3gemm_const_precision.cpp @@ -17,20 +17,20 @@ namespace ov::intel_gpu { using namespace ov::pass::pattern; KeepMOE3GemmConstPrecision::KeepMOE3GemmConstPrecision() { - auto wei_0_m = wrap_type(type_matches(ov::element::u4)); - auto wei_1_m = wrap_type(type_matches(ov::element::u4)); - auto wei_2_m = wrap_type(type_matches(ov::element::u4)); - auto zp_0_m = wrap_type(type_matches(ov::element::u4)); - auto zp_1_m = wrap_type(type_matches(ov::element::u4)); - auto zp_2_m = wrap_type(type_matches(ov::element::u4)); + auto wei_0_m = wrap_type(type_matches_any({ov::element::u4, ov::element::u2})); + auto wei_1_m = wrap_type(type_matches_any({ov::element::u4, ov::element::u2})); + auto wei_2_m = wrap_type(type_matches_any({ov::element::u4, ov::element::u2})); + auto zp_0_m = wrap_type(type_matches_any({ov::element::u4, ov::element::u2})); + auto zp_1_m = wrap_type(type_matches_any({ov::element::u4, ov::element::u2})); + auto zp_2_m = wrap_type(type_matches_any({ov::element::u4, ov::element::u2})); // Shared expert weights - auto sh_gate_wei_m = wrap_type(type_matches(ov::element::u4)); - auto sh_gate_zp_m = wrap_type(type_matches(ov::element::u4)); - auto sh_up_wei_m = wrap_type(type_matches(ov::element::u4)); - auto sh_up_zp_m = wrap_type(type_matches(ov::element::u4)); - auto sh_down_wei_m = wrap_type(type_matches(ov::element::u4)); - auto sh_down_zp_m = wrap_type(type_matches(ov::element::u4)); + auto sh_gate_wei_m = wrap_type(type_matches_any({ov::element::u4, ov::element::u2})); + auto sh_gate_zp_m = wrap_type(type_matches_any({ov::element::u4, ov::element::u2})); + auto sh_up_wei_m = wrap_type(type_matches_any({ov::element::u4, ov::element::u2})); + auto sh_up_zp_m = wrap_type(type_matches_any({ov::element::u4, ov::element::u2})); + auto sh_down_wei_m = wrap_type(type_matches_any({ov::element::u4, ov::element::u2})); + auto sh_down_zp_m = wrap_type(type_matches_any({ov::element::u4, ov::element::u2})); // Without shared expert: 12 inputs [hs, routing, topk, w0, s0, zp0, w1, s1, zp1, w2, s2, zp2] auto moe_no_shared_m = wrap_type( diff --git a/src/plugins/intel_gpu/src/plugin/transformations_pipeline.cpp b/src/plugins/intel_gpu/src/plugin/transformations_pipeline.cpp index b3bad016ccb35b..449579431e4362 100644 --- a/src/plugins/intel_gpu/src/plugin/transformations_pipeline.cpp +++ b/src/plugins/intel_gpu/src/plugin/transformations_pipeline.cpp @@ -699,6 +699,13 @@ void TransformationsPipeline::apply(std::shared_ptr func) { ov::element::i4, ov::element::i8, ov::element::u8}; + // u2 is offered only to the TiledMoeBlock -> GatherMatmul passes, and only when the fused MoE + // path runs. GroupedMatMulCompressed executes on oneDNN, which has no u2, and GatherMatmulCompressed + // has no u2 kernel of its own: with fusion disabled a u2 GatherMatmul would have nothing to run on. + std::vector gather_matmul_weights_types = supported_compressed_weights_types; + if (!config.get_moe_disable_fusion()) { + gather_matmul_weights_types.push_back(ov::element::u2); + } manager.register_pass( supported_compressed_weights_types); @@ -708,10 +715,10 @@ void TransformationsPipeline::apply(std::shared_ptr func) { if (infer_precision == ov::element::f16) { supported_activation_types.insert(supported_activation_types.end(), {ov::element::f32, ov::element::bf16}); } - manager.register_pass(supported_compressed_weights_types, + manager.register_pass(gather_matmul_weights_types, supported_activation_types); manager.register_pass(supported_activation_types, - supported_compressed_weights_types); + gather_matmul_weights_types); manager.register_pass(supported_activation_types); if (!config.get_moe_disable_fusion()) { diff --git a/src/plugins/intel_gpu/tests/unit/test_cases/moe_3gemm_gpu_test.cpp b/src/plugins/intel_gpu/tests/unit/test_cases/moe_3gemm_gpu_test.cpp index c7ff0313f8ce6d..7b4346205baccc 100644 --- a/src/plugins/intel_gpu/tests/unit/test_cases/moe_3gemm_gpu_test.cpp +++ b/src/plugins/intel_gpu/tests/unit/test_cases/moe_3gemm_gpu_test.cpp @@ -44,7 +44,11 @@ struct Moe3GemmReference { std::vector pack(const std::vector& unpacked) { if (!config.is_u4) return unpacked; + return pack_u4(unpacked); + } + // u4 is 2 values per byte, low nibble first. + static std::vector pack_u4(const std::vector& unpacked) { std::vector packed; packed.reserve((unpacked.size() + 1) / 2); for (size_t i = 0; i < unpacked.size(); i += 2) { @@ -55,6 +59,103 @@ struct Moe3GemmReference { return packed; } + // u2 is 4 values per byte, LSB-first -- the same order the GEMV kernels' shift/mask assumes. + static std::vector pack_u2(const std::vector& unpacked) { + std::vector packed((unpacked.size() + 3) / 4, 0); + for (size_t i = 0; i < unpacked.size(); ++i) { + packed[i / 4] |= static_cast((unpacked[i] & 0x3) << ((i % 4) * 2)); + } + return packed; + } + + // u2 quantization with 4 levels. Unlike quantize() above this also returns the EXACT + // dequantized weights: at 4 levels the quantization error swamps any kernel bug, so the + // reference has to be computed from what the kernel will actually decode, not from the + // original floats. `fixed_zp >= 0` forces one broadcast zero point for every group, which is + // what a scalar (per-tensor) zp input means. + // + // Layouts match quantize(): q is [E, cols, rows] (cols-major), scale/zp are + // [E, num_groups_per_row, cols], and dequant is [E, rows, cols] row-major so it can be fed + // straight to run_reference_softmax in place of the original weights. + // + // `bits` = 4 gives the same exact-dequant quantizer on the u4 grid (levels 0..15), for the + // mixed-precision layers that pair u2 GEMMs with a u4 one. The values are unpacked either way; + // pack with pack_u2 or pack_u4 to match. + struct QuantU2 { + std::vector q; + std::vector scale; + std::vector zp; + std::vector dequant; + }; + + static QuantU2 quantize_u2(const std::vector& data, + size_t expert_num, + size_t rows, + size_t cols, + size_t group_size, + int fixed_zp, + int bits = 2) { + const float max_range = static_cast((1 << bits) - 1); + const size_t num_groups_per_row = rows / group_size; + const size_t num_groups = cols * num_groups_per_row; + + QuantU2 out; + out.q.resize(expert_num * rows * cols); + out.scale.resize(expert_num * num_groups); + out.zp.resize(expert_num * num_groups); + out.dequant.resize(expert_num * rows * cols); + + for (size_t e = 0; e < expert_num; ++e) { + const size_t offset = e * rows * cols; + for (size_t c = 0; c < cols; ++c) { + for (size_t g = 0; g < num_groups_per_row; ++g) { + float min_val = std::numeric_limits::max(); + float max_val = std::numeric_limits::lowest(); + const size_t group_start = g * group_size; + for (size_t i = 0; i < group_size; ++i) { + const float val = data[offset + (group_start + i) * cols + c]; + min_val = std::min(min_val, val); + max_val = std::max(max_val, val); + } + + uint8_t zp_val; + float scale; + if (fixed_zp >= 0) { + // The zp is not ours to choose, so pick the scale that keeps the group + // inside 0..3 around it. + zp_val = static_cast(fixed_zp); + const float lo = -static_cast(zp_val); + const float hi = max_range - static_cast(zp_val); + scale = std::max(max_val / std::max(hi, 1e-5f), min_val / std::min(lo, -1e-5f)); + scale = std::max(scale, 1e-5f); + } else { + scale = std::max((max_val - min_val) / max_range, 1e-5f); + zp_val = static_cast( + std::round(std::max(0.0f, std::min(max_range, -min_val / scale)))); + } + + // Round-trip the scale through f16 before dequantizing: that is the value the + // kernel reads, and a f32 scale here would put a systematic bias in the + // reference that the tight tolerance below would flag. + const float scale_f16 = static_cast(static_cast(scale)); + out.scale[e * num_groups + g * cols + c] = static_cast(scale); + out.zp[e * num_groups + g * cols + c] = zp_val; + + for (size_t i = 0; i < group_size; ++i) { + const size_t r = group_start + i; + const float val = data[offset + r * cols + c]; + const float q_f = val / scale_f16 + static_cast(zp_val); + const uint8_t q = static_cast(std::round(std::max(0.0f, std::min(max_range, q_f)))); + out.q[offset + c * rows + r] = q; + out.dequant[offset + r * cols + c] = + (static_cast(q) - static_cast(zp_val)) * scale_f16; + } + } + } + } + return out; + } + std::tuple, std::vector, std::vector> quantize(const std::vector& data, size_t expert_num, size_t rows, @@ -1794,3 +1895,605 @@ INSTANTIATE_TEST_SUITE_P(smoke, MoeMixedDtypeParams{33, false, true}, MoeMixedDtypeParams{33, true, false}), moe_3gemm_compressed_gpu_mixed_dtype::GetTestCaseName); +// ============================================================================ +// u2 (UINT2) MoE with per-GEMM zero-point forms. +// +// The gate and the up projection are both served by ONE compiled body of gate_up_gemv_n2x_u2, but a +// mixed-precision layer can pair an INT2_SYM GEMM (a single broadcast scalar zp) with an INT2_ASYM +// one (a per-group zp). A layer-wide "is the zp scalar" jit constant therefore answers for both and +// decodes whichever GEMM disagrees with it using the wrong rule. +// +// The reference is computed from the EXACT dequantized weights rather than the original floats: u2 +// has four levels, so quantization error alone would need a tolerance far too loose to see a +// mis-read zero point. +// +// `down_u4` makes the down projection u4 with a per-group zp while gate/up stay u2: the NNCF recipe +// that keeps gate/up at INT2_SYM and gives down INT4_ASYM. Gate/up and down each stage their own +// SLM copy of the activations, and the u4 GEMV wants that copy interleaved where u2 wants it plain, +// so this is the case where a layout chosen from the gate dtype would mis-feed down. +// ============================================================================ + +struct MoeU2ZpParams { + size_t seq_len; + size_t hidden_size; + size_t inter_size; + size_t group_size; + bool gate_zp_scalar; + bool up_zp_scalar; + bool down_zp_scalar; + bool down_u4; + const char* name; +}; + +class moe_3gemm_compressed_gpu_u2 : public ::testing::TestWithParam { +public: + static std::string GetTestCaseName(const testing::TestParamInfo& obj) { + const auto& p = obj.param; + return std::string(p.name) + "_seq" + std::to_string(p.seq_len) + "_gs" + std::to_string(p.group_size) + "_h" + + std::to_string(p.hidden_size) + "_i" + std::to_string(p.inter_size); + } +}; + +TEST_P(moe_3gemm_compressed_gpu_u2, mixed_zp_form_accuracy) { + const auto& param = GetParam(); + auto& engine = get_test_engine(); + if (!engine.get_device_info().supports_immad) { + GTEST_SKIP() << "No immad support"; + } + // A non-zero scalar zp exists only for u2 weights (the fusion pass declines it for anything else). + ASSERT_FALSE(param.down_u4 && param.down_zp_scalar) << "a u4 down projection takes a per-group zp"; + + tests::random_generator rg(GET_SUITE_NAME); + Moe3GemmConfig config; + config.batch_size = 1; + config.seq_len = param.seq_len; + config.hidden_size = param.hidden_size; + config.inter_size = param.inter_size; + config.num_experts = 4; + config.top_k = 2; + // ELEMS_PER_LANE is group_size / SUBGROUP_SIZE, so on xe2 (SUBGROUP_SIZE 32) the three group + // sizes below select the kernel's three unpack arms: 32 -> 1, 64 -> 2, 128 -> 4. There is no + // arm for 8, which is what group_size 128 would ask for below xe2. + config.group_size = param.group_size; + config.is_u4 = false; + + Moe3GemmReference ref(config, rg); + + auto hidden_states = + rg.generate_random_1d(config.batch_size * config.seq_len * config.hidden_size, -1.0f, 1.0f, 1000); + auto routing_weights = + rg.generate_random_1d(config.batch_size * config.seq_len * config.num_experts, 0.0f, 1.0f, 1000); + + auto w0_data = + rg.generate_random_1d(config.num_experts * config.hidden_size * config.inter_size, -1.0f, 1.0f, 1000); + auto w1_data = + rg.generate_random_1d(config.num_experts * config.hidden_size * config.inter_size, -1.0f, 1.0f, 1001); + auto w2_data = + rg.generate_random_1d(config.num_experts * config.inter_size * config.hidden_size, -1.0f, 1.0f, 1002); + for (auto& v : w0_data) + v /= 7.0f; + for (auto& v : w1_data) + v /= 11.0f; + for (auto& v : w2_data) + v /= 7.0f; + + // A scalar zp has to be non-zero: zp 0 decodes identically under both rules and would make the + // test blind to the bug it exists to catch. + constexpr int kScalarZp = 2; + auto q0 = Moe3GemmReference::quantize_u2(w0_data, + config.num_experts, + config.hidden_size, + config.inter_size, + config.group_size, + param.gate_zp_scalar ? kScalarZp : -1); + auto q1 = Moe3GemmReference::quantize_u2(w1_data, + config.num_experts, + config.hidden_size, + config.inter_size, + config.group_size, + param.up_zp_scalar ? kScalarZp : -1); + auto q2 = Moe3GemmReference::quantize_u2(w2_data, + config.num_experts, + config.inter_size, + config.hidden_size, + config.group_size, + param.down_zp_scalar ? kScalarZp : -1, + param.down_u4 ? 4 : 2); + + const int64_t E = static_cast(config.num_experts); + const int64_t group_num = static_cast(config.hidden_size / config.group_size); + const int64_t group_num2 = static_cast(config.inter_size / config.group_size); + + auto make_u2_weight = [&](const std::vector& q, int64_t ofm, int64_t gnum) { + auto mem = engine.allocate_memory( + {data_types::u2, format::bfyx, {E, ofm, static_cast(config.group_size), gnum}}); + set_values(mem, Moe3GemmReference::pack_u2(q)); + get_test_stream().finish(); + return mem; + }; + auto make_scale = [&](const std::vector& v, int64_t ofm, int64_t G) { + auto mem = engine.allocate_memory( + layout{ov::PartialShape{E, ofm, G, 1}, data_types::f16, format::byfx}); + set_values(mem, v); + get_test_stream().finish(); + return mem; + }; + // A per-group zp for a u2 GEMM must itself be u2, packed 4 channels per byte along the + // innermost (channel) dimension of the byfx [E, G, N] physical layout. + auto make_u2_zp = [&](const std::vector& v, int64_t ofm, int64_t G) { + auto mem = + engine.allocate_memory(layout{ov::PartialShape{E, ofm, G, 1}, data_types::u2, format::byfx}); + set_values(mem, Moe3GemmReference::pack_u2(v)); + get_test_stream().finish(); + return mem; + }; + auto make_scalar_zp = [&]() { + auto mem = engine.allocate_memory(cldnn::layout(ov::PartialShape{1}, data_types::u8, format::bfyx), false); + set_values(mem, {static_cast(kScalarZp)}); + get_test_stream().finish(); + return mem; + }; + // u4 counterparts for the down_u4 case: same shapes, 2 values per byte. + auto make_u4_weight = [&](const std::vector& q, int64_t ofm, int64_t gnum) { + auto mem = engine.allocate_memory( + {data_types::u4, format::bfyx, {E, ofm, static_cast(config.group_size), gnum}}); + set_values(mem, Moe3GemmReference::pack_u4(q)); + get_test_stream().finish(); + return mem; + }; + auto make_u4_zp = [&](const std::vector& v, int64_t ofm, int64_t G) { + auto mem = + engine.allocate_memory(layout{ov::PartialShape{E, ofm, G, 1}, data_types::u4, format::byfx}); + set_values(mem, Moe3GemmReference::pack_u4(v)); + get_test_stream().finish(); + return mem; + }; + auto create_f16_tensor_3d = [&](const std::vector& values, int64_t d0, int64_t d1, int64_t d2) { + auto mem = engine.allocate_memory(layout{ov::PartialShape{d0, d1, d2}, data_types::f16, format::bfyx}); + set_values(mem, values); + get_test_stream().finish(); + return mem; + }; + + auto hidden_states_mem = create_f16_tensor_3d(hidden_states, config.batch_size, config.seq_len, config.hidden_size); + auto routing_weights_mem = + create_f16_tensor_3d(routing_weights, config.batch_size, config.seq_len, config.num_experts); + + const int64_t inter = static_cast(config.inter_size); + const int64_t hidden = static_cast(config.hidden_size); + + topology topology; + topology.add(input_layout("hidden_states", hidden_states_mem->get_layout())); + topology.add(input_layout("routing_weights", routing_weights_mem->get_layout())); + topology.add(data("w0_weight", make_u2_weight(q0.q, inter, group_num))); + topology.add(data("w0_scale", make_scale(q0.scale, inter, group_num))); + topology.add(data("w0_zp", param.gate_zp_scalar ? make_scalar_zp() : make_u2_zp(q0.zp, inter, group_num))); + topology.add(data("w1_weight", make_u2_weight(q1.q, inter, group_num))); + topology.add(data("w1_scale", make_scale(q1.scale, inter, group_num))); + topology.add(data("w1_zp", param.up_zp_scalar ? make_scalar_zp() : make_u2_zp(q1.zp, inter, group_num))); + topology.add(data("w2_weight", + param.down_u4 ? make_u4_weight(q2.q, hidden, group_num2) : make_u2_weight(q2.q, hidden, group_num2))); + topology.add(data("w2_scale", make_scale(q2.scale, hidden, group_num2))); + topology.add(data("w2_zp", + param.down_zp_scalar ? make_scalar_zp() + : param.down_u4 ? make_u4_zp(q2.zp, hidden, group_num2) + : make_u2_zp(q2.zp, hidden, group_num2))); + + MoERouterFused::Config router_config; + router_config.num_expert = config.num_experts; + router_config.top_k = config.top_k; + router_config.routing_type = MoERouterFused::RoutingType::SOFTMAX; + topology.add(moe_router_fused("router", std::vector{input_info("routing_weights")}, router_config)); + + cldnn::MOECompressed::Config moe_config; + moe_config.hidden_size = config.hidden_size; + moe_config.inter_size = config.inter_size; + moe_config.num_expert = config.num_experts; + moe_config.top_k = config.top_k; + moe_config.group_size = config.group_size; + moe_config.out_type = data_types::f16; + moe_config.has_zp = true; + + std::vector moe_inputs{input_info("hidden_states"), + input_info("router", 0), + input_info("router", 1), + input_info("w0_weight"), + input_info("w0_scale"), + input_info("w0_zp"), + input_info("w1_weight"), + input_info("w1_scale"), + input_info("w1_zp"), + input_info("w2_weight"), + input_info("w2_scale"), + input_info("w2_zp")}; + topology.add(moe_3gemm_fused_compressed("moe", moe_inputs, moe_config)); + + auto net_config = get_test_default_config(engine); + net_config.set_property(ov::intel_gpu::allow_new_shape_infer(true)); + network network(engine, topology, net_config); + network.set_input_data("hidden_states", hidden_states_mem); + network.set_input_data("routing_weights", routing_weights_mem); + + auto outputs = network.execute(); + auto output_prim = outputs.begin()->second.get_memory(); + get_test_stream().flush(); + cldnn::mem_lock output_ptr(output_prim, get_test_stream()); + + auto ref_output = ref.run_reference_softmax(hidden_states, routing_weights, q0.dequant, q1.dequant, q2.dequant); + + // Tight because the reference decodes exactly what the kernel decodes; what is left is f16 + // accumulation over hidden_size and then inter_size terms. + const float tolerance = 0.05f; + bool all_zero = true; + for (size_t i = 0; i < ref_output.size(); ++i) { + if (static_cast(output_ptr[i]) != 0.0f) + all_zero = false; + ASSERT_NEAR(static_cast(output_ptr[i]), static_cast(ref_output[i]), tolerance) << "i = " << i; + } + // A u2 prefill that fails at runtime falls back silently, and an all-zero output would + // otherwise satisfy every tolerance check above. + ASSERT_FALSE(all_zero) << "output is all zeros - the MoE stage was silently skipped"; +} + +INSTANTIATE_TEST_SUITE_P( + smoke, + moe_3gemm_compressed_gpu_u2, + ::testing::Values( + // Controls: all three GEMMs agree, so a layer-wide flag happens to be right. + MoeU2ZpParams{1, 128, 256, 64, true, true, true, false, "all_scalar"}, + MoeU2ZpParams{1, 128, 256, 64, false, false, false, false, "all_per_group"}, + // gate and up disagree, and they share one compiled gate_up_gemv_n2x_u2 body. + MoeU2ZpParams{1, 128, 256, 64, true, false, false, false, "gate_scalar_up_per_group"}, + MoeU2ZpParams{1, 128, 256, 64, false, true, false, false, "gate_per_group_up_scalar"}, + // down disagrees with gate/up, so a layer-wide flag set from the gate mis-decodes + // down_gemv_n2x_u2's own zp. + MoeU2ZpParams{1, 128, 256, 64, true, true, false, false, "down_per_group_rest_scalar"}, + MoeU2ZpParams{1, 128, 256, 64, false, false, true, false, "down_scalar_rest_per_group"}, + // Batched GEMV (MTP-style token counts). + MoeU2ZpParams{4, 128, 256, 64, true, false, false, false, "gate_scalar_up_per_group"}, + MoeU2ZpParams{16, 128, 256, 64, false, false, false, false, "all_per_group"}, + // The other two unpack arms: group_size 32 is ELEMS_PER_LANE 1 on xe2, 128 is 4. + MoeU2ZpParams{1, 128, 256, 32, false, false, true, false, "down_scalar_rest_per_group"}, + MoeU2ZpParams{1, 128, 256, 128, false, false, true, false, "down_scalar_rest_per_group"}, + MoeU2ZpParams{16, 128, 256, 32, false, false, false, false, "all_per_group"}, + // Above the 32-token batched-GEMV threshold, i.e. the prefill path (u2 unpacked to u4, + // then the oneDNN grouped GEMM) rather than the GEMV kernels. + MoeU2ZpParams{64, 128, 256, 64, false, false, false, false, "prefill_all_per_group"}, + MoeU2ZpParams{64, 128, 256, 64, true, true, true, false, "prefill_all_scalar"}, + // u4 down with u2 gate/up. The first row is the shipped recipe: INT2_SYM gate/up (scalar + // zp), INT4_ASYM down (per-group zp). No group_size 32 here: a layer that is not all-u2 + // needs FAKE_GROUP_SIZE >= 2 * SUBGROUP_SIZE, which 32 does not meet on xe2. + MoeU2ZpParams{1, 128, 256, 64, true, true, false, true, "down_u4_rest_scalar"}, + MoeU2ZpParams{1, 128, 256, 64, false, false, false, true, "down_u4_rest_per_group"}, + MoeU2ZpParams{1, 128, 256, 128, true, true, false, true, "down_u4_rest_scalar"}, + // Batched GEMV (MTP-style token counts). + MoeU2ZpParams{4, 128, 256, 64, true, true, false, true, "down_u4_rest_scalar"}, + MoeU2ZpParams{16, 128, 256, 64, false, false, false, true, "down_u4_rest_per_group"}, + // Prefill: the u2 GEMMs are unpacked to u4 scratch next to the native u4 down, after which + // all three reach the grouped GEMM as u4. + MoeU2ZpParams{64, 128, 256, 64, true, true, false, true, "prefill_down_u4_rest_scalar"}, + MoeU2ZpParams{64, 256, 256, 64, false, false, false, true, "prefill_down_u4_rest_per_group"}), + moe_3gemm_compressed_gpu_u2::GetTestCaseName); + +// ============================================================================ +// u2 shared expert. +// +// A u2 shared-expert projection cannot be handed to the oneDNN primitives: +// get_onednn_memory() reports it as u4, so it is unpacked into a u4 scratch buffer first. Under +// NNCF mixed precision any SUBSET of the three projections is u2 and the unpack is driven by a +// per-projection flag, so a subset is what has to be covered -- a layer-wide flag would either +// unpack a u8 projection (reading it at a quarter of its true stride) or leave a u2 one packed. +// ============================================================================ + +struct MoeU2SharedParams { + bool gate_u2; + bool up_u2; + bool down_u2; + size_t seq_len; + size_t group_size; + const char* name; +}; + +class moe_3gemm_compressed_gpu_u2_shared : public ::testing::TestWithParam { +public: + static std::string GetTestCaseName(const testing::TestParamInfo& obj) { + const auto& p = obj.param; + return std::string(p.name) + "_seq" + std::to_string(p.seq_len) + "_gs" + std::to_string(p.group_size); + } +}; + +TEST_P(moe_3gemm_compressed_gpu_u2_shared, mixed_u2_shared_projections) { + const auto& param = GetParam(); + auto& engine = get_test_engine(); + if (!engine.get_device_info().supports_immad) { + GTEST_SKIP() << "No immad support"; + } + + tests::random_generator rg(GET_SUITE_NAME); + Moe3GemmConfig config; + config.batch_size = 1; + config.seq_len = param.seq_len; + config.hidden_size = 128; + config.inter_size = 256; + config.num_experts = 4; + config.top_k = 2; + config.group_size = param.group_size; + config.is_u4 = false; // the projections this case does not make u2 are u8 + config.has_shared_expert = true; + + Moe3GemmReference ref(config, rg); + + auto hidden_states = + rg.generate_random_1d(config.batch_size * config.seq_len * config.hidden_size, -1.0f, 1.0f, 1000); + auto routing_weights = + rg.generate_random_1d(config.batch_size * config.seq_len * config.num_experts, 0.0f, 1.0f, 1000); + + auto w0_data = + rg.generate_random_1d(config.num_experts * config.hidden_size * config.inter_size, -1.0f, 1.0f, 1000); + auto w1_data = + rg.generate_random_1d(config.num_experts * config.hidden_size * config.inter_size, -1.0f, 1.0f, 1001); + auto w2_data = + rg.generate_random_1d(config.num_experts * config.inter_size * config.hidden_size, -1.0f, 1.0f, 1002); + auto s_gate_data = rg.generate_random_1d(config.hidden_size * config.inter_size, -1.0f, 1.0f, 1003); + auto s_up_data = rg.generate_random_1d(config.hidden_size * config.inter_size, -1.0f, 1.0f, 1004); + auto s_down_data = rg.generate_random_1d(config.inter_size * config.hidden_size, -1.0f, 1.0f, 1005); + auto s_gate_scalar_data = rg.generate_random_1d(config.hidden_size, -1.0f, 0.0f, 1006); + for (auto& v : w0_data) + v /= 7.0f; + for (auto& v : w1_data) + v /= 11.0f; + for (auto& v : w2_data) + v /= 7.0f; + for (auto& v : s_gate_data) + v /= 7.0f; + for (auto& v : s_up_data) + v /= 11.0f; + for (auto& v : s_down_data) + v /= 7.0f; + + const int64_t E = static_cast(config.num_experts); + const int64_t group_num = static_cast(config.hidden_size / config.group_size); + const int64_t group_num2 = static_cast(config.inter_size / config.group_size); + const int64_t inter = static_cast(config.inter_size); + const int64_t hidden = static_cast(config.hidden_size); + + auto q0 = Moe3GemmReference::quantize_u2(w0_data, + config.num_experts, + config.hidden_size, + config.inter_size, + config.group_size, + -1); + auto q1 = Moe3GemmReference::quantize_u2(w1_data, + config.num_experts, + config.hidden_size, + config.inter_size, + config.group_size, + -1); + auto q2 = Moe3GemmReference::quantize_u2(w2_data, + config.num_experts, + config.inter_size, + config.hidden_size, + config.group_size, + -1); + + auto make_weight = [&](const std::vector& q, bool u2, int64_t e, int64_t ofm, int64_t gnum) { + auto mem = engine.allocate_memory({u2 ? data_types::u2 : data_types::u8, + format::bfyx, + {e, ofm, static_cast(config.group_size), gnum}}); + set_values(mem, u2 ? Moe3GemmReference::pack_u2(q) : q); + get_test_stream().finish(); + return mem; + }; + auto make_scale = [&](const std::vector& v, int64_t e, int64_t ofm, int64_t G) { + auto mem = engine.allocate_memory(layout{ov::PartialShape{e, ofm, G, 1}, data_types::f16, format::byfx}); + set_values(mem, v); + get_test_stream().finish(); + return mem; + }; + auto make_zp = [&](const std::vector& v, bool u2, int64_t e, int64_t ofm, int64_t G) { + auto mem = engine.allocate_memory( + layout{ov::PartialShape{e, ofm, G, 1}, u2 ? data_types::u2 : data_types::u8, format::byfx}); + set_values(mem, u2 ? Moe3GemmReference::pack_u2(v) : v); + get_test_stream().finish(); + return mem; + }; + auto create_f16_tensor_3d = [&](const std::vector& values, int64_t d0, int64_t d1, int64_t d2) { + auto mem = engine.allocate_memory(layout{ov::PartialShape{d0, d1, d2}, data_types::f16, format::bfyx}); + set_values(mem, values); + get_test_stream().finish(); + return mem; + }; + auto make_scalar_gate = [&](const std::vector& v) { + std::vector h; + h.reserve(v.size()); + for (float f : v) + h.push_back(static_cast(f)); + auto mem = engine.allocate_memory({data_types::f16, format::bfyx, {1, 1, 1, static_cast(v.size())}}); + set_values(mem, h); + get_test_stream().finish(); + return mem; + }; + + // quantize() does not hand back the exact dequantized weights the way quantize_u2 does, so + // reconstruct them. Feeding run_reference_softmax the original floats instead leaves the u8 + // quantization error in the reference, and on the decode path -- where the batched GEMV fuses + // the shared expert in-kernel -- that error measured ~0.37, several times the tolerance a u2 + // test wants. Layouts follow quantize(): q is [cols, rows] cols-major, scale/zp are + // [groups_per_row, cols], and the result is [rows, cols] row-major. expert_num is 1 here. + auto dequantize_u8 = [&](const std::vector& q, + const std::vector& scale, + const std::vector& zp, + size_t rows, + size_t cols) { + const size_t groups_per_row = rows / config.group_size; + std::vector out(rows * cols); + for (size_t c = 0; c < cols; ++c) { + for (size_t g = 0; g < groups_per_row; ++g) { + // The kernel reads an f16 scale, so the reference has to dequantize with the + // f16-rounded value or it carries a systematic bias of its own. + const float s = static_cast(scale[g * cols + c]); + const float z = static_cast(zp[g * cols + c]); + for (size_t i = 0; i < config.group_size; ++i) { + const size_t r = g * config.group_size + i; + out[r * cols + c] = (static_cast(q[c * rows + r]) - z) * s; + } + } + } + return out; + }; + + // One shared projection, quantized either as u2 or as u8. Either way the reference is computed + // from the EXACT dequantized weights -- at four levels the u2 quantization error alone would + // need a tolerance far too loose to see a mis-decoded weight. + struct SharedProj { + cldnn::memory::ptr weight; + cldnn::memory::ptr scale; + cldnn::memory::ptr zp; + std::vector ref; + }; + auto make_shared = [&](const std::vector& data, bool u2, size_t rows, size_t cols, int64_t gnum) { + SharedProj p; + const int64_t ofm = static_cast(cols); + if (u2) { + auto q = Moe3GemmReference::quantize_u2(data, 1, rows, cols, config.group_size, -1); + p.weight = make_weight(q.q, true, 1, ofm, gnum); + p.scale = make_scale(q.scale, 1, ofm, gnum); + p.zp = make_zp(q.zp, true, 1, ofm, gnum); + p.ref = q.dequant; + } else { + auto [q, scale, zp] = ref.quantize(data, 1, rows, cols, config.group_size); + p.weight = make_weight(q, false, 1, ofm, gnum); + p.scale = make_scale(scale, 1, ofm, gnum); + p.zp = make_zp(zp, false, 1, ofm, gnum); + p.ref = dequantize_u8(q, scale, zp, rows, cols); + } + return p; + }; + + auto sg = make_shared(s_gate_data, param.gate_u2, config.hidden_size, config.inter_size, group_num); + auto su = make_shared(s_up_data, param.up_u2, config.hidden_size, config.inter_size, group_num); + auto sd = make_shared(s_down_data, param.down_u2, config.inter_size, config.hidden_size, group_num2); + + auto hidden_states_mem = create_f16_tensor_3d(hidden_states, config.batch_size, config.seq_len, config.hidden_size); + auto routing_weights_mem = + create_f16_tensor_3d(routing_weights, config.batch_size, config.seq_len, config.num_experts); + + topology topology; + topology.add(input_layout("hidden_states", hidden_states_mem->get_layout())); + topology.add(input_layout("routing_weights", routing_weights_mem->get_layout())); + topology.add(data("w0_weight", make_weight(q0.q, true, E, inter, group_num))); + topology.add(data("w0_scale", make_scale(q0.scale, E, inter, group_num))); + topology.add(data("w0_zp", make_zp(q0.zp, true, E, inter, group_num))); + topology.add(data("w1_weight", make_weight(q1.q, true, E, inter, group_num))); + topology.add(data("w1_scale", make_scale(q1.scale, E, inter, group_num))); + topology.add(data("w1_zp", make_zp(q1.zp, true, E, inter, group_num))); + topology.add(data("w2_weight", make_weight(q2.q, true, E, hidden, group_num2))); + topology.add(data("w2_scale", make_scale(q2.scale, E, hidden, group_num2))); + topology.add(data("w2_zp", make_zp(q2.zp, true, E, hidden, group_num2))); + topology.add(data("s_gate_weight", sg.weight)); + topology.add(data("s_gate_scale", sg.scale)); + topology.add(data("s_gate_zp", sg.zp)); + topology.add(data("s_up_weight", su.weight)); + topology.add(data("s_up_scale", su.scale)); + topology.add(data("s_up_zp", su.zp)); + topology.add(data("s_down_weight", sd.weight)); + topology.add(data("s_down_scale", sd.scale)); + topology.add(data("s_down_zp", sd.zp)); + topology.add(data("s_gate_scalar", make_scalar_gate(s_gate_scalar_data))); + + MoERouterFused::Config router_config; + router_config.num_expert = config.num_experts; + router_config.top_k = config.top_k; + router_config.routing_type = MoERouterFused::RoutingType::SOFTMAX; + topology.add(moe_router_fused("router", std::vector{input_info("routing_weights")}, router_config)); + + cldnn::MOECompressed::Config moe_config; + moe_config.hidden_size = config.hidden_size; + moe_config.inter_size = config.inter_size; + moe_config.num_expert = config.num_experts; + moe_config.top_k = config.top_k; + moe_config.group_size = config.group_size; + moe_config.out_type = data_types::f16; + moe_config.num_shared_expert = 1; + moe_config.has_zp = true; + + topology.add(moe_3gemm_fused_compressed("moe", + {input_info("hidden_states"), + input_info("router", 0), + input_info("router", 1), + input_info("w0_weight"), + input_info("w0_scale"), + input_info("w0_zp"), + input_info("w1_weight"), + input_info("w1_scale"), + input_info("w1_zp"), + input_info("w2_weight"), + input_info("w2_scale"), + input_info("w2_zp"), + input_info("s_gate_weight"), + input_info("s_gate_scale"), + input_info("s_gate_zp"), + input_info("s_up_weight"), + input_info("s_up_scale"), + input_info("s_up_zp"), + input_info("s_down_weight"), + input_info("s_down_scale"), + input_info("s_down_zp"), + input_info("s_gate_scalar")}, + moe_config)); + + auto net_config = get_test_default_config(engine); + net_config.set_property(ov::intel_gpu::allow_new_shape_infer(true)); + network network(engine, topology, net_config); + network.set_input_data("hidden_states", hidden_states_mem); + network.set_input_data("routing_weights", routing_weights_mem); + + auto outputs = network.execute(); + auto output_prim = outputs.begin()->second.get_memory(); + get_test_stream().flush(); + cldnn::mem_lock output_ptr(output_prim, get_test_stream()); + + auto ref_output = ref.run_reference_softmax(hidden_states, + routing_weights, + q0.dequant, + q1.dequant, + q2.dequant, + sg.ref, + su.ref, + sd.ref, + s_gate_scalar_data); + + const float tolerance = 0.1f; + bool all_zero = true; + for (size_t i = 0; i < ref_output.size(); ++i) { + if (static_cast(output_ptr[i]) != 0.0f) + all_zero = false; + ASSERT_NEAR(static_cast(output_ptr[i]), static_cast(ref_output[i]), tolerance) << "i = " << i; + } + // A u2 unpack that fails at runtime falls back silently, and an all-zero output would satisfy + // every tolerance check above. + ASSERT_FALSE(all_zero) << "output is all zeros - the MoE stage was silently skipped"; +} + +INSTANTIATE_TEST_SUITE_P( + smoke, + moe_3gemm_compressed_gpu_u2_shared, + ::testing::Values( + // Decode, i.e. the batched GEMV, which fuses the shared expert in-kernel. Only the + // all-u2 case is listed: the GEMV picks ONE decode rule per compiled body, from the routed + // experts' dtype (GATE_WEIGHT_DT/UP_WEIGHT_DT), and the shared expert aliases those same + // pointers -- so a shared projection whose dtype differs from the routed one is decoded by + // the wrong rule. Measured: a u8 shared projection under u2 routed experts is off by 0.37. + // Fixing it means a per-slot decode inside one kernel body, so the mixed cases below are + // prefill-only until that happens. + MoeU2SharedParams{true, true, true, 1, 64, "all_u2"}, + // Above the 32-token batched-GEMV threshold the shared expert goes through the oneDNN + // primitives, which do resolve its dtype per projection. down has the transposed shape of + // gate/up, so it is covered on its own. + MoeU2SharedParams{true, true, true, 64, 64, "all_u2_prefill"}, + MoeU2SharedParams{false, false, false, 64, 64, "none_u2_prefill"}, + MoeU2SharedParams{true, false, false, 64, 64, "gate_u2_prefill"}, + MoeU2SharedParams{false, false, true, 64, 64, "down_u2_prefill"}, + MoeU2SharedParams{true, false, true, 64, 64, "gate_down_u2_prefill"}), + moe_3gemm_compressed_gpu_u2_shared::GetTestCaseName); + From a02ce935ebafefc91036742cee5fcfe9598d3772 Mon Sep 17 00:00:00 2001 From: Cyberpunk1210 Date: Wed, 21 Oct 2026 16:19:10 +0800 Subject: [PATCH 04/12] [GPU] Fail u2 MoE prefill errors instead of falling back to batched GEMV A failed u2 weight unpack or grouped GEMM used to be caught and silently replaced by the batched GEMV. No other GPU primitive degrades that way at execution time, so let the error propagate. Also drop measurement figures from comments. Co-Authored-By: Claude Opus 5.5 (1M context) Signed-off-by: Cyberpunk1210 --- .../impls/ocl_v2/moe/moe_3gemm_swiglu_opt.cpp | 27 +++---------------- .../unit/test_cases/moe_3gemm_gpu_test.cpp | 8 +++--- 2 files changed, 8 insertions(+), 27 deletions(-) diff --git a/src/plugins/intel_gpu/src/graph/impls/ocl_v2/moe/moe_3gemm_swiglu_opt.cpp b/src/plugins/intel_gpu/src/graph/impls/ocl_v2/moe/moe_3gemm_swiglu_opt.cpp index 12418aaf401379..c8244cd19a79ae 100644 --- a/src/plugins/intel_gpu/src/graph/impls/ocl_v2/moe/moe_3gemm_swiglu_opt.cpp +++ b/src/plugins/intel_gpu/src/graph/impls/ocl_v2/moe/moe_3gemm_swiglu_opt.cpp @@ -2582,8 +2582,7 @@ class moe_3gemm_swiglu_opt_impl : public PrimitiveImplOCL { // allocated on first use and the unpack runs ONCE per bound weight buffer, not per prefill: // the weights are constant (OTD is rejected for u2 in the ctor), so re-unpacking identical // data every prefill was pure overhead — a full streaming pass over every expert of every - // layer, which measured ~430ms of TTFT per prefill on Qwen3.6-35B-A3B and made a multi-turn - // benchmark ~1.6x slower end-to-end than the same model at int4. See the *_key members. + // layer on every prefill. See the *_key members. cldnn::event::ptr unpack_u2_weights_for_prefill(typed_primitive_inst& instance, scratch_buffers& scratch, const std::vector& events) { @@ -3135,29 +3134,11 @@ class moe_3gemm_swiglu_opt_impl : public PrimitiveImplOCL { // run on every prefill path, so the unpack is not tied to _weights_u2 alone. cldnn::event::ptr unpack_event = nullptr; if (_weights_u2 || has_u2_shared_weights()) { - try { - unpack_event = unpack_u2_weights_for_prefill(instance, scratch, events); - } catch (const std::exception& e) { - // The batched GEMV fuses the shared expert in-kernel, so returning it - // directly also skips the oneDNN shared-expert block below. - GPU_DEBUG_TRACE_DETAIL << "u2 weight unpack for prefill failed (" << e.what() << "), falling back to batched GEMV" << std::endl; - instance.output_memory_ptr(0)->fill(stream, false); - return exec_batched_gemv(events, instance, scratch, token_num); - } + unpack_event = unpack_u2_weights_for_prefill(instance, scratch, events); } if (_weights_u2) { - // Run the grouped GEMM path on the unpacked u4 weights as regular u4 weights. If - // the grouped GEMM cannot be created/executed, fall back to the batched GEMV - // (the pre-u2-prefill behavior). - try { - ret_env = exec_prefill_grouped_gemm({unpack_event}, stream, instance, scratch); - } catch (const std::exception& e) { - GPU_DEBUG_TRACE_DETAIL << "u2 grouped GEMM prefill failed (" << e.what() << "), falling back to batched GEMV" << std::endl; - instance.output_memory_ptr(0)->fill(stream, false); - // Return directly: the batched GEMV fuses the shared expert in-kernel, so it - // must skip the oneDNN shared-expert block below (it would apply it twice). - return exec_batched_gemv(events, instance, scratch, token_num); - } + // The grouped GEMM path serves the unpacked u4 copies as regular u4 weights. + ret_env = exec_prefill_grouped_gemm({unpack_event}, stream, instance, scratch); } else if (use_micro_gemm_prefill) { ret_env = exec_prefill_micro_gemm(events, instance, scratch, use_gpu_mask_gen); } else if (use_grouped_gemm_prefill) { diff --git a/src/plugins/intel_gpu/tests/unit/test_cases/moe_3gemm_gpu_test.cpp b/src/plugins/intel_gpu/tests/unit/test_cases/moe_3gemm_gpu_test.cpp index 7b4346205baccc..8878a670ef42ce 100644 --- a/src/plugins/intel_gpu/tests/unit/test_cases/moe_3gemm_gpu_test.cpp +++ b/src/plugins/intel_gpu/tests/unit/test_cases/moe_3gemm_gpu_test.cpp @@ -2130,7 +2130,7 @@ TEST_P(moe_3gemm_compressed_gpu_u2, mixed_zp_form_accuracy) { all_zero = false; ASSERT_NEAR(static_cast(output_ptr[i]), static_cast(ref_output[i]), tolerance) << "i = " << i; } - // A u2 prefill that fails at runtime falls back silently, and an all-zero output would + // An all-zero output (e.g. a skipped MoE stage) would // otherwise satisfy every tolerance check above. ASSERT_FALSE(all_zero) << "output is all zeros - the MoE stage was silently skipped"; } @@ -2317,7 +2317,7 @@ TEST_P(moe_3gemm_compressed_gpu_u2_shared, mixed_u2_shared_projections) { // quantize() does not hand back the exact dequantized weights the way quantize_u2 does, so // reconstruct them. Feeding run_reference_softmax the original floats instead leaves the u8 // quantization error in the reference, and on the decode path -- where the batched GEMV fuses - // the shared expert in-kernel -- that error measured ~0.37, several times the tolerance a u2 + // the shared expert in-kernel -- that error is several times the tolerance a u2 // test wants. Layouts follow quantize(): q is [cols, rows] cols-major, scale/zp are // [groups_per_row, cols], and the result is [rows, cols] row-major. expert_num is 1 here. auto dequantize_u8 = [&](const std::vector& q, @@ -2470,7 +2470,7 @@ TEST_P(moe_3gemm_compressed_gpu_u2_shared, mixed_u2_shared_projections) { all_zero = false; ASSERT_NEAR(static_cast(output_ptr[i]), static_cast(ref_output[i]), tolerance) << "i = " << i; } - // A u2 unpack that fails at runtime falls back silently, and an all-zero output would satisfy + // An all-zero output (e.g. a skipped MoE stage) would satisfy // every tolerance check above. ASSERT_FALSE(all_zero) << "output is all zeros - the MoE stage was silently skipped"; } @@ -2483,7 +2483,7 @@ INSTANTIATE_TEST_SUITE_P( // all-u2 case is listed: the GEMV picks ONE decode rule per compiled body, from the routed // experts' dtype (GATE_WEIGHT_DT/UP_WEIGHT_DT), and the shared expert aliases those same // pointers -- so a shared projection whose dtype differs from the routed one is decoded by - // the wrong rule. Measured: a u8 shared projection under u2 routed experts is off by 0.37. + // the wrong rule: a u8 shared projection under u2 routed experts is far outside tolerance. // Fixing it means a per-slot decode inside one kernel body, so the mixed cases below are // prefill-only until that happens. MoeU2SharedParams{true, true, true, 1, 64, "all_u2"}, From 8b085b8a9008248cd21c2939c52c3e71b96ebe69 Mon Sep 17 00:00:00 2001 From: Cyberpunk1210 Date: Thu, 22 Oct 2026 11:47:03 +0800 Subject: [PATCH 05/12] [GPU] Trim u2 fused MoE 3-GEMM comments Comment-only cleanup of the u2 fused MoE 3-GEMM changes and tests: keep the why, drop restated code and history. Co-Authored-By: Claude Opus 5.5 (1M context) Signed-off-by: Cyberpunk1210 --- .../common_optimizations/moe_op_fusion.cpp | 12 +-- ...ert_gather_matmuls_moe_block_to_moe_op.cpp | 1 - .../impls/ocl_v2/moe/moe_3gemm_swiglu_opt.cpp | 96 ++++++------------- .../impls/ocl_v2/moe_3gemm_swiglu_mlp.cl | 26 ++--- .../unit/test_cases/moe_3gemm_gpu_test.cpp | 81 +++++----------- 5 files changed, 63 insertions(+), 153 deletions(-) diff --git a/src/common/transformations/src/transformations/common_optimizations/moe_op_fusion.cpp b/src/common/transformations/src/transformations/common_optimizations/moe_op_fusion.cpp index 75c9c5c8a35c5f..e02755a4b0774a 100644 --- a/src/common/transformations/src/transformations/common_optimizations/moe_op_fusion.cpp +++ b/src/common/transformations/src/transformations/common_optimizations/moe_op_fusion.cpp @@ -160,14 +160,10 @@ Convert3GatherMatmulMoeBlockToMoeOp::Convert3GatherMatmulMoeBlockToMoeOp() { const bool is_compressed = is_gate_compressed; if (is_compressed) { - // A layer that mixes dtypes across the three expert GEMMs is decoded per GEMM by - // moe_3gemm (GATE/UP/DOWN_WEIGHT_DT) for u2, u4 and u8. One constraint is shared state - // rather than dispatch: gate and up read ONE SLM copy of the activations, and its layout - // is chosen from the gate dtype alone -- interleaved (even/odd halves) for u4, plain for - // u2/u8. So gate and up must agree on whether they are u4; down stages its own copy and - // is free. This is what admits the NNCF recipe that keeps gate/up at u2 and gives down - // u4. Signed types stay uniform-only: WEIGHT_IS_SIGNED is set once, from the gate. - // Anything else keeps the per-GEMM-correct GatherMatmul form rather than being mis-decoded. + // moe_3gemm decodes each GEMM with its own dtype, except that gate and up share one SLM copy of the + // activations whose layout follows the gate dtype (interleaved for u4), so gate and up must agree on + // being u4. Signed types stay uniform-only (WEIGHT_IS_SIGNED comes from the gate). Anything else + // stays as GatherMatmul. { const auto gate_dt = pm.at(gate_w_m).get_element_type(); const auto up_dt = pm.at(up_w_m).get_element_type(); diff --git a/src/common/transformations/tests/common_optimizations/convert_gather_matmuls_moe_block_to_moe_op.cpp b/src/common/transformations/tests/common_optimizations/convert_gather_matmuls_moe_block_to_moe_op.cpp index 4c023623f12e94..717d5e651e0f8e 100644 --- a/src/common/transformations/tests/common_optimizations/convert_gather_matmuls_moe_block_to_moe_op.cpp +++ b/src/common/transformations/tests/common_optimizations/convert_gather_matmuls_moe_block_to_moe_op.cpp @@ -1060,7 +1060,6 @@ INSTANTIATE_TEST_SUITE_P(smoke, DtypeGateParams{ov::element::u8, ov::element::u2, ov::element::u8, true}, DtypeGateParams{ov::element::u2, ov::element::u2, ov::element::u8, true}, // u4 on down only: down stages its own activation copy, so it may differ from gate/up. - // The first row is the recipe this exists for -- gate/up u2, down u4. DtypeGateParams{ov::element::u2, ov::element::u2, ov::element::u4, true}, DtypeGateParams{ov::element::u8, ov::element::u8, ov::element::u4, true}, DtypeGateParams{ov::element::u2, ov::element::u8, ov::element::u4, true}, diff --git a/src/plugins/intel_gpu/src/graph/impls/ocl_v2/moe/moe_3gemm_swiglu_opt.cpp b/src/plugins/intel_gpu/src/graph/impls/ocl_v2/moe/moe_3gemm_swiglu_opt.cpp index c8244cd19a79ae..0c8c1cdb31747c 100644 --- a/src/plugins/intel_gpu/src/graph/impls/ocl_v2/moe/moe_3gemm_swiglu_opt.cpp +++ b/src/plugins/intel_gpu/src/graph/impls/ocl_v2/moe/moe_3gemm_swiglu_opt.cpp @@ -425,10 +425,8 @@ static void add_common_consts(const RuntimeParams& params, JitConstants& jit) { GPU_DEBUG_TRACE_DETAIL << "[DEBUG] moe_3gemm_swiglu_opt: group_size=" << desc->_config.group_size << ", gate_up_group_size=" << gate_up_group_size << ", down_group_size=" << down_group_size << std::endl; - // ELEMS_PER_LANE = FAKE_GROUP_SIZE / SUBGROUP_SIZE selects the GEMV inner-loop variant, and only - // the u2 variants have an ELEMS_PER_LANE==1 branch. The kernel dispatches per GEMM, so relaxing - // the bound is safe only when all three are u2: a u4/u8 GEMM at ELEMS_PER_LANE 1 has no matching - // branch and silently falls into the 8-element #else path, reading past its quant group. + // Only the u2 GEMV variants have an ELEMS_PER_LANE == 1 branch; a u4/u8 GEMM there would silently take the + // 8-element path and read past its quant group, so the bound is relaxed only when all three GEMMs are u2. const ov::element::Type weight_dt = params.get_input_layout(static_cast(MOE3GemmInputIndex::WEIGHT_0)).data_type; { const auto up_weight_dt = params.get_input_layout(static_cast(MOE3GemmInputIndex::WEIGHT_1)).data_type; @@ -462,10 +460,7 @@ static void add_common_consts(const RuntimeParams& params, JitConstants& jit) { jit.make("MOE_DTYPE_SIZE", params.get_input_layout(0).data_type == ov::element::f16 ? 2 : 4); jit.make("HAS_ZP", desc->_config.has_zp ? 1 : 0); if (desc->_config.has_zp) { - // Only the element type of a broadcast scalar zp is shared; *which* GEMMs use one is - // answered per GEMM by GATE/UP/DOWN_ZP_SCALAR below. A layer-wide "is it scalar" flag is - // deliberately not emitted: the three GEMMs can disagree, and a single answer silently - // decodes one GEMM's zp with another's rule. + // Whether a zp is scalar is emitted per GEMM (GATE/UP/DOWN_ZP_SCALAR below): the three GEMMs can disagree. const char* scalar_zp_dt = "uchar"; for (const auto zp_idx : {MOE3GemmInputIndex::ZP_0, MOE3GemmInputIndex::ZP_1, MOE3GemmInputIndex::ZP_2}) { const auto& zp_layout = params.get_input_layout(static_cast(zp_idx)); @@ -494,8 +489,7 @@ static void add_common_consts(const RuntimeParams& params, JitConstants& jit) { jit.make("MOE_SCALE_DT", "half"); jit.make("MOE_ZP_DT", "uchar"); } else if (weight_dt == ov::element::u2) { - // u2: 4 values per byte, LSB-first. Served by the batched GEMV kernels for all - // token counts (micro-gemm and oneDNN have no u2 support). + // u2: 4 values per byte, LSB-first. OPENVINO_ASSERT(desc->_config.hidden_size % 4 == 0 && desc->_config.inter_size % 4 == 0, "MoE u2 weights require hidden_size and inter_size to be multiples of 4, got hidden_size=", desc->_config.hidden_size, @@ -1094,20 +1088,15 @@ class moe_3gemm_swiglu_opt_impl : public PrimitiveImplOCL { bool _weights_u2 = false; // any u2 GEMM: batched GEMV for decode, u4-unpack + grouped GEMM for prefill bool _gemm_weights_u2[3] = {false, false, false}; // per-GEMM (gate/up/down) u2 flags bool _shared_weights_u2[4] = {false, false, false, false}; // per-projection (gate/up/down/scalar-gate) shared-expert u2 flags - // u2 prefill scratch: unpacked u4 copies of this layer's expert weights (and zp). - // Allocated lazily on first prefill and reused across calls (unpack_u2_weights_for_prefill). + // u2 prefill scratch: u4 copies of this layer's expert weights (and zp), allocated on first prefill. memory::ptr _u2_unpack_weight[3]; memory::ptr _u2_unpack_zp[3]; - // Same for the shared expert's weights/zp; consumed by the oneDNN shared-expert - // primitives built in init_shared_primitives. + // Same for the shared expert's weights/zp (oneDNN shared-expert primitives). memory::ptr _u2_unpack_shared_weight[4]; memory::ptr _u2_unpack_shared_zp[4]; - // Identifies the source buffer a u4 unpack scratch was last filled from, so the unpack - // kernel runs once instead of on every prefill. The unpack is a pure function of a - // *constant* weight/zp buffer: OTD (expert offload) is rejected for u2 in the ctor, so - // nothing rebinds these mid-run. Keyed on buffer identity + byte size so a rebind still - // re-runs the unpack; a freshly (re)allocated destination also forces a re-run. + // Source buffer (identity + byte size) each scratch was last unpacked from, so the unpack runs once: the + // weights are constant (OTD is rejected for u2), and a rebind or a fresh destination still re-runs it. struct unpack_src_key { const void* src = nullptr; size_t bytes = 0; @@ -1162,19 +1151,14 @@ class moe_3gemm_swiglu_opt_impl : public PrimitiveImplOCL { use_micro_gemm_prefill = false; } - // u2 (2-bit) weights: micro-gemm (gemmstone) and oneDNN have no u2 dtype, so the - // u4-only prefill paths cannot consume u2 data directly. Decode (small token - // counts) is served by the hand-written batched GEMV kernels; prefill unpacks - // the u2 weights into u4 scratch buffers and runs the grouped GEMM path. - // In NNCF mixed-precision layers ANY u2 GEMM (gate/up/down) selects this split. + // micro-gemm and oneDNN have no u2 dtype: decode uses the batched GEMV kernels, prefill unpacks u2 into u4 + // scratch for the grouped GEMM path. Any u2 GEMM in the layer selects this split. _gemm_weights_u2[0] = (weight_dt == data_types::u2); _gemm_weights_u2[1] = (params.get_input_layout(static_cast(MOE3GemmInputIndex::WEIGHT_1)).data_type == data_types::u2); _gemm_weights_u2[2] = (params.get_input_layout(static_cast(MOE3GemmInputIndex::WEIGHT_2)).data_type == data_types::u2); _weights_u2 = _gemm_weights_u2[0] || _gemm_weights_u2[1] || _gemm_weights_u2[2]; - // Shared expert per-projection u2 flags (gate/up/down/scalar-gate). Under NNCF - // mixed precision any subset of them may be u2; u2 shared weights are unpacked - // into u4 scratch for the oneDNN shared-expert primitives (init_shared_primitives). + // Any subset of the shared-expert projections may be u2; those are unpacked into u4 scratch for oneDNN. if (params.input_layouts.size() > static_cast(MOE3GemmInputIndex::SHARED_GATE_WEIGHT)) { _shared_weights_u2[0] = (params.get_input_layout(static_cast(MOE3GemmInputIndex::SHARED_GATE_WEIGHT)).data_type == data_types::u2); _shared_weights_u2[1] = (params.get_input_layout(static_cast(MOE3GemmInputIndex::SHARED_UP_WEIGHT)).data_type == data_types::u2); @@ -1186,8 +1170,7 @@ class moe_3gemm_swiglu_opt_impl : public PrimitiveImplOCL { // micro_gemm takes priority; grouped_gemm falls back to onednn loop by default. use_grouped_gemm_prefill = config.get_moe_use_grouped_gemm_prefill(); if (_weights_u2) { - // gemmstone has no u2 support at all; the grouped GEMM prefill consumes the - // u2 weights via the u4-unpacked scratch (see unpack_u2_weights_for_prefill). + // gemmstone has no u2; prefill consumes the u4-unpacked scratch. use_micro_gemm_prefill = false; use_grouped_gemm_prefill = true; } @@ -1354,9 +1337,7 @@ class moe_3gemm_swiglu_opt_impl : public PrimitiveImplOCL { OPENVINO_ASSERT(addr.shared_weight[0], "MoE shared expert enabled (num_shared_expert > 0) but shared weight buffers are not bound"); - // layout.count() is the logical element count even for sub-byte dtypes (u2/u4 - // packing only enters through bytes_count()), so this stays correct when the - // shared weights are u2 (e.g. [512, 32, 64] u2 -> 512 with hidden 2048). + // layout.count() is the logical element count even for sub-byte dtypes, so this holds for u2 shared weights. _shared_intermediate_size = static_cast(addr.shared_weight[0]->get_layout().count() / _hidden_size); auto eng = engine.get_onednn_engine(); using t = onednn_matmul::type; @@ -1366,10 +1347,7 @@ class moe_3gemm_swiglu_opt_impl : public PrimitiveImplOCL { return m && m->get_layout().data_type != data_types::dynamic; }; - // u2 shared weights: consume the u4-unpacked scratch (filled by - // unpack_u2_weights_for_prefill) with u4 descriptors, mirroring the - // routed-expert grouped GEMM substitution. Non-u2 projections keep the - // original buffers/dtypes (mixed precision). + // u2 shared projections use the u4-unpacked scratch with u4 descriptors; the others keep their buffers. auto shared_weight_dt = [&](int i) { return _shared_weights_u2[i] ? dnnl::memory::data_type::u4 : convert_data_type(addr.shared_weight[i]->get_layout().data_type); }; @@ -1434,7 +1412,7 @@ class moe_3gemm_swiglu_opt_impl : public PrimitiveImplOCL { // 3. Scalar Gate (Sigmoid) // It is very small weight with shape of [Hidden, 1], and not need to keep compressed, so KeepMOE3GemmConstPrecision will not keep its precision and // ConvertPrecision will convert to f16. So Scalar gate is [Hidden, 1], f16 weights, no scale/zp for now. - // If it stayed u2 (int2 exports), it is consumed via the u4-unpacked scratch like the other shared projections. + // A u2 scalar gate is consumed via the u4-unpacked scratch like the other shared projections. dnnl::memory sg_w; dnnl::memory::data_type sg_w_dt = dnnl::memory::data_type::f16; if (addr.shared_weight[3]) { @@ -1553,9 +1531,8 @@ class moe_3gemm_swiglu_opt_impl : public PrimitiveImplOCL { cur_moe->_gemm_weights_u2[0] = _gemm_weights_u2[0]; cur_moe->_gemm_weights_u2[1] = _gemm_weights_u2[1]; cur_moe->_gemm_weights_u2[2] = _gemm_weights_u2[2]; - // Don't share the u4 unpack scratch with the clone; the clone fills its own on first - // prefill. make_deep_copy() above copied the cache keys too, so clear them as well — - // otherwise the clone would see a "already unpacked" key next to a null buffer. + // The clone fills its own unpack scratch on first prefill; clear the copied keys too, or it would see an + // "already unpacked" key next to a null buffer. cur_moe->_u2_unpack_weight[0] = nullptr; cur_moe->_u2_unpack_weight[1] = nullptr; cur_moe->_u2_unpack_weight[2] = nullptr; @@ -2577,12 +2554,8 @@ class moe_3gemm_swiglu_opt_impl : public PrimitiveImplOCL { return _gemm_weights_u2[gemm_idx] ? _u2_unpack_zp[gemm_idx] : scratch.moe_fusion_wei_addr.zp[gemm_idx]; } - // u2 prefill: unpack the u2 expert weights (and ZP) into u4-packed scratch buffers so - // the grouped GEMM path (which has no u2 dtype) can serve prefill. The scratch buffers are - // allocated on first use and the unpack runs ONCE per bound weight buffer, not per prefill: - // the weights are constant (OTD is rejected for u2 in the ctor), so re-unpacking identical - // data every prefill was pure overhead — a full streaming pass over every expert of every - // layer on every prefill. See the *_key members. + // u2 prefill: unpack the u2 expert weights (and ZP) into u4 scratch for the grouped GEMM path, which has no u2 + // dtype. Runs once per bound weight buffer, not per prefill (see the *_key members). cldnn::event::ptr unpack_u2_weights_for_prefill(typed_primitive_inst& instance, scratch_buffers& scratch, const std::vector& events) { @@ -2599,10 +2572,8 @@ class moe_3gemm_swiglu_opt_impl : public PrimitiveImplOCL { if (!_gemm_weights_u2[i]) continue; - // Weights: byte-wise u2 -> u4 unpack, doubles the buffer size. The u2 and u4 - // expert weight buffers share the same logical element order ([expert, oc, ic], - // ic innermost, LSB-first packing), so a straight byte-wise unpack reproduces - // exactly the layout the u4 prefill paths expect. + // Byte-wise u2 -> u4 unpack: both share the logical element order ([expert, oc, ic], LSB-first), so the + // result is exactly the u4 layout the prefill path expects. const auto& src_mem = scratch.moe_fusion_wei_addr.weight[i]; const size_t src_bytes = src_mem->size(); OPENVINO_ASSERT(src_bytes % 4 == 0, "moe_3gemm u2 weight buffer byte size ", src_bytes, " is not a multiple of 4"); @@ -2647,7 +2618,6 @@ class moe_3gemm_swiglu_opt_impl : public PrimitiveImplOCL { zp_mode = 1; zp_count = static_cast(zp_dst_bytes); } else if (zp_src->get_layout().data_type == data_types::u2) { - // u2-packed zp: same byte-wise unpack as the weights. const size_t zp_src_bytes = zp_src->size(); OPENVINO_ASSERT(zp_src_bytes % 4 == 0, "moe_3gemm u2 zp buffer byte size ", zp_src_bytes, " is not a multiple of 4"); zp_dst_bytes = zp_src_bytes * 2; @@ -2655,10 +2625,7 @@ class moe_3gemm_swiglu_opt_impl : public PrimitiveImplOCL { zp_mode = 0; zp_count = static_cast(zp_src_bytes / 4); } else { - // Byte-wide zp (u8/i8, e.g. per-channel or per-group): one VALUE per - // byte, NOT u2-packed data. Pack two adjacent values into one u4 - // byte (mode 2); a byte-wise u2 unpack would double the values into - // garbage and silently corrupt prefill dequant. + // Byte-wide zp (u8/i8) holds one value per byte, not u2-packed data: pack two per u4 byte (mode 2). const size_t zp_vals = zp_src->get_layout().count(); const size_t k = (i == 2) ? static_cast(_intermediate_size) : static_cast(_hidden_size); const size_t oc = (i == 2) ? static_cast(_hidden_size) : static_cast(_intermediate_size); @@ -2699,10 +2666,8 @@ class moe_3gemm_swiglu_opt_impl : public PrimitiveImplOCL { } } - // Shared expert: same u2 -> u4 unpack per projection (gate/up/down/scalar-gate), - // consumed by the oneDNN shared-expert primitives built in init_shared_primitives. - // The u2 and u4 buffers share the logical element order ([oc, ic], ic innermost), - // so a straight byte-wise unpack matches the {ic, oc}/ba u4 descriptors. + // Shared expert: same byte-wise unpack per projection ([oc, ic], ic innermost), matching the u4 + // descriptors of init_shared_primitives. for (int i = 0; i < 4; i++) { if (!_shared_weights_u2[i]) continue; @@ -2761,8 +2726,7 @@ class moe_3gemm_swiglu_opt_impl : public PrimitiveImplOCL { zp_mode = 0; zp_count = static_cast(zp_src_bytes / 4); } else { - // Byte-wide zp (u8/i8): pack two values per u4 byte (mode 2), same as - // the routed-expert path; a byte-wise u2 unpack would corrupt it. + // Byte-wide zp (u8/i8): pack two values per u4 byte (mode 2), as for the routed experts. const size_t zp_vals = zp_src->get_layout().count(); const size_t k = (i == 2) ? static_cast(_shared_intermediate_size) : static_cast(_hidden_size); const size_t oc = (i == 2) ? static_cast(_hidden_size) : static_cast(_shared_intermediate_size); @@ -3095,9 +3059,7 @@ class moe_3gemm_swiglu_opt_impl : public PrimitiveImplOCL { // Batched GEMV: for small token counts (including single token, MTP/speculative decoding), // use optimized GEMV kernels with batch dimension. Avoids gather/scatter overhead. - // u2 weights: decode takes this path; prefill (token_num > threshold) unpacks the - // u2 weights (routed and shared expert) into u4 scratch buffers and runs the grouped - // GEMM path below. + // u2: decode takes this path; prefill unpacks into u4 scratch and runs the grouped GEMM path below. if (token_num <= batched_gemv_threshold) { return exec_batched_gemv(events, instance, scratch, token_num); } @@ -3128,10 +3090,8 @@ class moe_3gemm_swiglu_opt_impl : public PrimitiveImplOCL { << ", use_micro_gemm_prefill=" << use_micro_gemm_prefill << ", use_grouped_gemm_prefill=" << use_grouped_gemm_prefill << std::endl; update_rt_params(instance); - // u2 prefill: unpack the u2 weights (and zp) into u4 scratch buffers. Routed-expert - // u2 GEMMs are consumed by the grouped GEMM path below; shared-expert u2 projections - // are consumed by the oneDNN shared-expert primitives (init_shared_primitives), which - // run on every prefill path, so the unpack is not tied to _weights_u2 alone. + // Not tied to _weights_u2 alone: the shared-expert oneDNN primitives consume their u4 scratch on every + // prefill path. cldnn::event::ptr unpack_event = nullptr; if (_weights_u2 || has_u2_shared_weights()) { unpack_event = unpack_u2_weights_for_prefill(instance, scratch, events); diff --git a/src/plugins/intel_gpu/src/graph/impls/ocl_v2/moe_3gemm_swiglu_mlp.cl b/src/plugins/intel_gpu/src/graph/impls/ocl_v2/moe_3gemm_swiglu_mlp.cl index 7919720c622ede..c67835814a00c5 100644 --- a/src/plugins/intel_gpu/src/graph/impls/ocl_v2/moe_3gemm_swiglu_mlp.cl +++ b/src/plugins/intel_gpu/src/graph/impls/ocl_v2/moe_3gemm_swiglu_mlp.cl @@ -100,10 +100,8 @@ inline float moe_mlp_fast_erf(float x) { #if defined(U2_UNPACK_ENABLE) -// Unpacks u2-packed data (4 values per byte, LSB-first) into u4-packed data -// (2 values per byte, LSB-first), doubling the byte size while preserving the -// logical element order. Used to feed u2 MoE expert weights/zp to the u4-only -// prefill GEMM paths (micro-gemm/oneDNN have no u2 dtype). +// Unpacks u2-packed data (4 values per byte) into u4-packed data (2 per byte), keeping the logical element +// order, for the u4-only prefill GEMM paths. inline uint moe_unpack_u2_byte(uint b) { // byte (v3 v2 v1 v0) -> uint16 ((v3 | v2 << 4) << 8) | (v0 | v1 << 4) return (b & 0x3u) | ((b & 0xCu) << 2) | ((b & 0x30u) << 4) | ((b & 0xC0u) << 6); @@ -334,10 +332,8 @@ inline void gate_up_gemv_n2x_u2(const __global uchar* weight, sum_all1 += (sum1[0] + sum1[1]) * s1; # endif # elif ELEMS_PER_LANE == 2 - // Each lane handles 2 K-elements; two lanes share one packed byte. - // NOTE: keep per-lane scalar loads here. A subgroup block read would make - // lanes >= FAKE_GROUP_SIZE/4 read past the tile, and at the end of the - // mmap'd weights buffer that overhang faults (CL_OUT_OF_RESOURCES). + // Each lane handles 2 K-elements; two lanes share one packed byte. Per-lane scalar loads: a subgroup + // block read would overhang the tile and fault at the end of the weights buffer. half sum0; half sum1; half2 a = vload2(id_local, x2 + gk * FAKE_GROUP_SIZE); @@ -359,9 +355,7 @@ inline void gate_up_gemv_n2x_u2(const __global uchar* weight, sum_all1 += sum1 * s1; # endif # elif ELEMS_PER_LANE == 1 - // Each lane handles 1 K-element; four lanes share one packed byte. - // Same per-lane scalar loads as the ==2 branch: subgroup block reads - // would overhang the weights buffer and fault (CL_OUT_OF_RESOURCES). + // Each lane handles 1 K-element; four lanes share one packed byte. Scalar loads as in the ==2 branch. half sum0; half sum1; half a = x2[gk * FAKE_GROUP_SIZE + id_local]; @@ -1049,10 +1043,8 @@ inline void down_gemv_n2x_u2(const __global uchar* weight, sum_all1 += (sum1[0] + sum1[1]) * s1; # endif # elif ELEMS_PER_LANE == 2 - // Each lane handles 2 K-elements; two lanes share one packed byte. - // NOTE: keep per-lane scalar loads here. A subgroup block read would make - // lanes >= FAKE_GROUP_SIZE/4 read past the tile, and at the end of the - // mmap'd weights buffer that overhang faults (CL_OUT_OF_RESOURCES). + // Each lane handles 2 K-elements; two lanes share one packed byte. Per-lane scalar loads: a subgroup + // block read would overhang the tile and fault at the end of the weights buffer. half sum0; half sum1; half2 a = vload2(id_local, x2 + gk * FAKE_GROUP_SIZE); @@ -1074,9 +1066,7 @@ inline void down_gemv_n2x_u2(const __global uchar* weight, sum_all1 += sum1 * s1; # endif # elif ELEMS_PER_LANE == 1 - // Each lane handles 1 K-element; four lanes share one packed byte. - // Same per-lane scalar loads as the ==2 branch: subgroup block reads - // would overhang the weights buffer and fault (CL_OUT_OF_RESOURCES). + // Each lane handles 1 K-element; four lanes share one packed byte. Scalar loads as in the ==2 branch. half sum0; half sum1; half a = x2[gk * FAKE_GROUP_SIZE + id_local]; diff --git a/src/plugins/intel_gpu/tests/unit/test_cases/moe_3gemm_gpu_test.cpp b/src/plugins/intel_gpu/tests/unit/test_cases/moe_3gemm_gpu_test.cpp index 8878a670ef42ce..fcab29eedd51ec 100644 --- a/src/plugins/intel_gpu/tests/unit/test_cases/moe_3gemm_gpu_test.cpp +++ b/src/plugins/intel_gpu/tests/unit/test_cases/moe_3gemm_gpu_test.cpp @@ -59,7 +59,7 @@ struct Moe3GemmReference { return packed; } - // u2 is 4 values per byte, LSB-first -- the same order the GEMV kernels' shift/mask assumes. + // u2 is 4 values per byte, LSB-first. static std::vector pack_u2(const std::vector& unpacked) { std::vector packed((unpacked.size() + 3) / 4, 0); for (size_t i = 0; i < unpacked.size(); ++i) { @@ -68,19 +68,9 @@ struct Moe3GemmReference { return packed; } - // u2 quantization with 4 levels. Unlike quantize() above this also returns the EXACT - // dequantized weights: at 4 levels the quantization error swamps any kernel bug, so the - // reference has to be computed from what the kernel will actually decode, not from the - // original floats. `fixed_zp >= 0` forces one broadcast zero point for every group, which is - // what a scalar (per-tensor) zp input means. - // - // Layouts match quantize(): q is [E, cols, rows] (cols-major), scale/zp are - // [E, num_groups_per_row, cols], and dequant is [E, rows, cols] row-major so it can be fed - // straight to run_reference_softmax in place of the original weights. - // - // `bits` = 4 gives the same exact-dequant quantizer on the u4 grid (levels 0..15), for the - // mixed-precision layers that pair u2 GEMMs with a u4 one. The values are unpacked either way; - // pack with pack_u2 or pack_u4 to match. + // u2 quantization that also returns the exact dequantized weights: at 4 levels the quantization error would + // swamp a kernel bug. `fixed_zp >= 0` forces one broadcast (scalar) zp. Layouts match quantize(); dequant is + // [E, rows, cols] row-major for run_reference_softmax. `bits` = 4 does the same on the u4 grid. struct QuantU2 { std::vector q; std::vector scale; @@ -134,9 +124,7 @@ struct Moe3GemmReference { std::round(std::max(0.0f, std::min(max_range, -min_val / scale)))); } - // Round-trip the scale through f16 before dequantizing: that is the value the - // kernel reads, and a f32 scale here would put a systematic bias in the - // reference that the tight tolerance below would flag. + // Round-trip the scale through f16: that is the value the kernel reads. const float scale_f16 = static_cast(static_cast(scale)); out.scale[e * num_groups + g * cols + c] = static_cast(scale); out.zp[e * num_groups + g * cols + c] = zp_val; @@ -1898,19 +1886,11 @@ INSTANTIATE_TEST_SUITE_P(smoke, // ============================================================================ // u2 (UINT2) MoE with per-GEMM zero-point forms. // -// The gate and the up projection are both served by ONE compiled body of gate_up_gemv_n2x_u2, but a -// mixed-precision layer can pair an INT2_SYM GEMM (a single broadcast scalar zp) with an INT2_ASYM -// one (a per-group zp). A layer-wide "is the zp scalar" jit constant therefore answers for both and -// decodes whichever GEMM disagrees with it using the wrong rule. -// -// The reference is computed from the EXACT dequantized weights rather than the original floats: u2 -// has four levels, so quantization error alone would need a tolerance far too loose to see a -// mis-read zero point. -// -// `down_u4` makes the down projection u4 with a per-group zp while gate/up stay u2: the NNCF recipe -// that keeps gate/up at INT2_SYM and gives down INT4_ASYM. Gate/up and down each stage their own -// SLM copy of the activations, and the u4 GEMV wants that copy interleaved where u2 wants it plain, -// so this is the case where a layout chosen from the gate dtype would mis-feed down. +// gate and up share one compiled gate_up_gemv_n2x_u2 body, but a mixed layer can pair a scalar zp +// (INT2_SYM) with a per-group one (INT2_ASYM), so the zp form must be resolved per GEMM. The reference +// uses the exact dequantized weights: u2 quantization error alone would hide a mis-read zero point. +// `down_u4` makes down u4 with a per-group zp while gate/up stay u2; down stages its own activation +// copy, which u4 wants interleaved and u2 plain. // ============================================================================ struct MoeU2ZpParams { @@ -2149,7 +2129,7 @@ INSTANTIATE_TEST_SUITE_P( // down_gemv_n2x_u2's own zp. MoeU2ZpParams{1, 128, 256, 64, true, true, false, false, "down_per_group_rest_scalar"}, MoeU2ZpParams{1, 128, 256, 64, false, false, true, false, "down_scalar_rest_per_group"}, - // Batched GEMV (MTP-style token counts). + // Batched GEMV (small token counts). MoeU2ZpParams{4, 128, 256, 64, true, false, false, false, "gate_scalar_up_per_group"}, MoeU2ZpParams{16, 128, 256, 64, false, false, false, false, "all_per_group"}, // The other two unpack arms: group_size 32 is ELEMS_PER_LANE 1 on xe2, 128 is 4. @@ -2160,13 +2140,12 @@ INSTANTIATE_TEST_SUITE_P( // then the oneDNN grouped GEMM) rather than the GEMV kernels. MoeU2ZpParams{64, 128, 256, 64, false, false, false, false, "prefill_all_per_group"}, MoeU2ZpParams{64, 128, 256, 64, true, true, true, false, "prefill_all_scalar"}, - // u4 down with u2 gate/up. The first row is the shipped recipe: INT2_SYM gate/up (scalar - // zp), INT4_ASYM down (per-group zp). No group_size 32 here: a layer that is not all-u2 - // needs FAKE_GROUP_SIZE >= 2 * SUBGROUP_SIZE, which 32 does not meet on xe2. + // u4 down with u2 gate/up. No group_size 32 here: a layer that is not all-u2 needs + // FAKE_GROUP_SIZE >= 2 * SUBGROUP_SIZE, which 32 does not meet on xe2. MoeU2ZpParams{1, 128, 256, 64, true, true, false, true, "down_u4_rest_scalar"}, MoeU2ZpParams{1, 128, 256, 64, false, false, false, true, "down_u4_rest_per_group"}, MoeU2ZpParams{1, 128, 256, 128, true, true, false, true, "down_u4_rest_scalar"}, - // Batched GEMV (MTP-style token counts). + // Batched GEMV (small token counts). MoeU2ZpParams{4, 128, 256, 64, true, true, false, true, "down_u4_rest_scalar"}, MoeU2ZpParams{16, 128, 256, 64, false, false, false, true, "down_u4_rest_per_group"}, // Prefill: the u2 GEMMs are unpacked to u4 scratch next to the native u4 down, after which @@ -2178,11 +2157,8 @@ INSTANTIATE_TEST_SUITE_P( // ============================================================================ // u2 shared expert. // -// A u2 shared-expert projection cannot be handed to the oneDNN primitives: -// get_onednn_memory() reports it as u4, so it is unpacked into a u4 scratch buffer first. Under -// NNCF mixed precision any SUBSET of the three projections is u2 and the unpack is driven by a -// per-projection flag, so a subset is what has to be covered -- a layer-wide flag would either -// unpack a u8 projection (reading it at a quarter of its true stride) or leave a u2 one packed. +// A u2 shared-expert projection is unpacked into u4 scratch for the oneDNN primitives, per projection: +// any subset of the three can be u2. // ============================================================================ struct MoeU2SharedParams { @@ -2314,12 +2290,8 @@ TEST_P(moe_3gemm_compressed_gpu_u2_shared, mixed_u2_shared_projections) { return mem; }; - // quantize() does not hand back the exact dequantized weights the way quantize_u2 does, so - // reconstruct them. Feeding run_reference_softmax the original floats instead leaves the u8 - // quantization error in the reference, and on the decode path -- where the batched GEMV fuses - // the shared expert in-kernel -- that error is several times the tolerance a u2 - // test wants. Layouts follow quantize(): q is [cols, rows] cols-major, scale/zp are - // [groups_per_row, cols], and the result is [rows, cols] row-major. expert_num is 1 here. + // Reconstruct the exact dequantized weights (quantize() does not return them): the u8 quantization error + // alone would exceed the u2 tolerance. Layouts follow quantize(); expert_num is 1 here. auto dequantize_u8 = [&](const std::vector& q, const std::vector& scale, const std::vector& zp, @@ -2329,8 +2301,7 @@ TEST_P(moe_3gemm_compressed_gpu_u2_shared, mixed_u2_shared_projections) { std::vector out(rows * cols); for (size_t c = 0; c < cols; ++c) { for (size_t g = 0; g < groups_per_row; ++g) { - // The kernel reads an f16 scale, so the reference has to dequantize with the - // f16-rounded value or it carries a systematic bias of its own. + // The kernel reads an f16 scale, so dequantize with the f16-rounded value. const float s = static_cast(scale[g * cols + c]); const float z = static_cast(zp[g * cols + c]); for (size_t i = 0; i < config.group_size; ++i) { @@ -2342,9 +2313,7 @@ TEST_P(moe_3gemm_compressed_gpu_u2_shared, mixed_u2_shared_projections) { return out; }; - // One shared projection, quantized either as u2 or as u8. Either way the reference is computed - // from the EXACT dequantized weights -- at four levels the u2 quantization error alone would - // need a tolerance far too loose to see a mis-decoded weight. + // One shared projection, quantized as u2 or u8; the reference uses the exact dequantized weights. struct SharedProj { cldnn::memory::ptr weight; cldnn::memory::ptr scale; @@ -2479,13 +2448,9 @@ INSTANTIATE_TEST_SUITE_P( smoke, moe_3gemm_compressed_gpu_u2_shared, ::testing::Values( - // Decode, i.e. the batched GEMV, which fuses the shared expert in-kernel. Only the - // all-u2 case is listed: the GEMV picks ONE decode rule per compiled body, from the routed - // experts' dtype (GATE_WEIGHT_DT/UP_WEIGHT_DT), and the shared expert aliases those same - // pointers -- so a shared projection whose dtype differs from the routed one is decoded by - // the wrong rule: a u8 shared projection under u2 routed experts is far outside tolerance. - // Fixing it means a per-slot decode inside one kernel body, so the mixed cases below are - // prefill-only until that happens. + // Decode (batched GEMV, shared expert fused in-kernel): only all-u2 is listed. The GEMV picks one decode + // rule per compiled body from the routed experts' dtype, so a shared projection with another dtype would + // be mis-decoded; the mixed cases below are prefill-only. MoeU2SharedParams{true, true, true, 1, 64, "all_u2"}, // Above the 32-token batched-GEMV threshold the shared expert goes through the oneDNN // primitives, which do resolve its dtype per projection. down has the transposed shape of From ddaa10a5d48f46869bd7119d2612d5092e77486a Mon Sep 17 00:00:00 2001 From: Cyberpunk1210 Date: Thu, 22 Oct 2026 11:45:20 +0800 Subject: [PATCH 06/12] [Transformations] Trim u2 MoE fusion comments Comment-only cleanup: keep the why, drop restated code, history and internal labels. Co-Authored-By: Claude Opus 5.5 (1M context) Signed-off-by: Cyberpunk1210 --- .../common_optimizations/moe_op_fusion.cpp | 78 ++++++------------- ...ert_gather_matmuls_moe_block_to_moe_op.cpp | 55 ++++--------- src/core/src/op/moe.cpp | 5 +- 3 files changed, 40 insertions(+), 98 deletions(-) diff --git a/src/common/transformations/src/transformations/common_optimizations/moe_op_fusion.cpp b/src/common/transformations/src/transformations/common_optimizations/moe_op_fusion.cpp index e02755a4b0774a..df22bc7c25af72 100644 --- a/src/common/transformations/src/transformations/common_optimizations/moe_op_fusion.cpp +++ b/src/common/transformations/src/transformations/common_optimizations/moe_op_fusion.cpp @@ -185,29 +185,19 @@ Convert3GatherMatmulMoeBlockToMoeOp::Convert3GatherMatmulMoeBlockToMoeOp() { auto up_zp = pm.at(up_zp_m); auto down_zp = pm.at(down_zp_m); - // Non-zero scalar zp broadcast is only implemented for the u2 batched GEMV - // kernels, so it is retained per-GEMM only when that GEMM's weights are u2. - - // A scalar (rank-0) zp constant is a folded per-tensor zero point. - // An all-zero scalar zp is equivalent to symmetric quantization: normalize - // it to the "absent zp" convention (dynamic-typed empty constant). - // A non-zero scalar zp is passed through unchanged (u2 weights only): - // MOE validation explicitly exempts rank-0 inputs from the num_experts - // check, and the batched GEMV kernels broadcast the single element over - // all experts/groups/channels. - // Anything else (non-constant, packed dtype, or mixed sym/asym forms) - // bails out and leaves the graph in the GatherMatmul form. + // A rank-0 zp is a folded per-tensor zero point. All-zero is symmetric and becomes the absent-zp constant. + // Non-zero is kept as is for u2 weights only: MOE validation exempts rank-0 inputs and only the u2 kernels + // broadcast it. auto normalize_scalar_zp = [&](ov::Output& zp, ov::element::Type wt) -> bool { const auto& zp_shape = zp.get_partial_shape(); if (!zp_shape.is_static() || zp_shape.rank().get_length() != 0) { - return true; // not a scalar zp: nothing to do + return true; } const auto zp_const = ov::as_type_ptr(zp.get_node_shared_ptr()); if (!zp_const) { return false; } - // Restrict to byte/wide integer types: cast_vector on packed (u2/u4/i4) - // constants is not meaningful here and may throw. + // cast_vector is not meaningful on packed (u2/u4/i4) constants. const auto zp_et = zp_const->get_element_type(); if (zp_et != ov::element::i8 && zp_et != ov::element::u8 && zp_et != ov::element::i32 && zp_et != ov::element::u32 && zp_et != ov::element::i64) { @@ -216,10 +206,8 @@ Convert3GatherMatmulMoeBlockToMoeOp::Convert3GatherMatmulMoeBlockToMoeOp() { for (const auto v : zp_const->cast_vector()) { if (v != 0) { if (wt != ov::element::u2) { - return false; // non-zero scalar zp only supported for u2 weights + return false; } - // Keep the rank-0 scalar constant as-is: MOE validation exempts rank-0 - // inputs from the num_experts check, and the u2 kernels broadcast it. return true; } } @@ -266,12 +254,8 @@ Convert3GatherMatmulMoeBlockToMoeOp::Convert3GatherMatmulMoeBlockToMoeOp() { const size_t up_G = scale_num_groups(up_scale); const size_t down_G = scale_num_groups(down_scale); - // MOECompressed requires one group_size shared by all three GEMMs. - // A per-channel scale ([E, N, 1], group_size == K) is mathematically - // identical to a group-wise scale with the same value repeated across - // groups, so broadcast per-channel scale constants to the group-wise - // num_groups used by the other GEMMs. Genuinely inconsistent group - // sizes (two different group-wise gs) or non-constant scales bail out. + // MOECompressed needs one group_size for all GEMMs. A per-channel scale equals a group-wise one with the + // value repeated, so per-channel scales are broadcast to the common group count; other mismatches bail out. size_t group_size = std::numeric_limits::max(); for (const auto& kg : {std::make_pair(gate_K, gate_G), {up_K, up_G}, {down_K, down_G}}) { const size_t K = kg.first; @@ -290,8 +274,7 @@ Convert3GatherMatmulMoeBlockToMoeOp::Convert3GatherMatmulMoeBlockToMoeOp() { } } - // Broadcast a per-channel scale constant [E, N, 1] (or [E, N]) to - // [E, N, K/group_size] by repeating each channel value across groups. + // [E, N, 1] (or [E, N]) -> [E, N, K/group_size], each channel value repeated across groups. auto broadcast_per_channel_scale = [&](ov::Output& scale, size_t K) -> bool { const auto c = ov::as_type_ptr(scale.get_node_shared_ptr()); if (!c) @@ -303,11 +286,10 @@ Convert3GatherMatmulMoeBlockToMoeOp::Convert3GatherMatmulMoeBlockToMoeOp() { const size_t N = s[1]; const size_t target_G = K / group_size; if (target_G == 1) - return true; // nothing to expand + return true; const auto et = c->get_element_type(); - // Sub-byte types are bit-packed and Type::size() rounds up to a whole byte, so the - // per-element memcpy below would read past the constant and interleave neighbouring - // channels. Fall back to the unfused form rather than materialise corrupt values. + // Type::size() rounds sub-byte types up to a byte, so the per-element copy below would corrupt + // packed data. if (et.bitwidth() < 8) return false; const size_t esz = et.size(); @@ -326,10 +308,7 @@ Convert3GatherMatmulMoeBlockToMoeOp::Convert3GatherMatmulMoeBlockToMoeOp() { (down_G == 1 && !broadcast_per_channel_scale(down_scale, down_K))) { return false; } - // A3: broadcast a per-channel (u8) zp [E, N, 1] to [E, N, K/group_size], the - // same expansion as its scale, so zshape == sshape and the byfx reorder plus - // in-kernel (group, channel) zp indexing line up. Scalar (u2) zps are rank-0 - // and were normalized above; they are left untouched here. + // Per-channel zps get the same expansion as their scales so the shapes match; rank-0 zps are kept. if (num_dynamic_zp == 0) { if ((gate_G == 1 && !broadcast_per_channel_scale(gate_zp, gate_K)) || (up_G == 1 && !broadcast_per_channel_scale(up_zp, up_K)) || @@ -536,9 +515,7 @@ Convert2GatherMatmulMoeBlockToMoeOp::Convert2GatherMatmulMoeBlockToMoeOp() { const bool is_compressed = is_gate_up_compressed; if (is_compressed) { - // The GEMM2 fused path lowers to cldnn::moe_gemm, whose only registered - // impl (oneDNN) does not accept u2 weights. Keep u2 in GatherMatmul form - // until moe_gemm gains u2 support, otherwise compilation fails. + // GEMM2 lowers to cldnn::moe_gemm, whose only impl (oneDNN) does not take u2 weights. if (pm.at(gate_up_w_m).get_element_type() == ov::element::u2 || pm.at(down_w_m).get_element_type() == ov::element::u2) { return false; @@ -546,21 +523,17 @@ Convert2GatherMatmulMoeBlockToMoeOp::Convert2GatherMatmulMoeBlockToMoeOp() { auto gate_up_zp = pm.at(gate_up_zp_m); auto down_zp = pm.at(down_zp_m); - // Non-zero scalar zp broadcast is only implemented for the u2 fused - // kernels, so it is retained per-GEMM only when that GEMM's weights are u2. - // Same scalar-zp normalization as the 3-GEMM matcher above. auto normalize_scalar_zp = [&](ov::Output& zp, ov::element::Type wt) -> bool { const auto& zp_shape = zp.get_partial_shape(); if (!zp_shape.is_static() || zp_shape.rank().get_length() != 0) { - return true; // not a scalar zp: nothing to do + return true; } const auto zp_const = ov::as_type_ptr(zp.get_node_shared_ptr()); if (!zp_const) { return false; } - // Restrict to byte/wide integer types: cast_vector on packed (u2/u4/i4) - // constants is not meaningful here and may throw. + // cast_vector is not meaningful on packed (u2/u4/i4) constants. const auto zp_et = zp_const->get_element_type(); if (zp_et != ov::element::i8 && zp_et != ov::element::u8 && zp_et != ov::element::i32 && zp_et != ov::element::u32 && zp_et != ov::element::i64) { @@ -569,10 +542,8 @@ Convert2GatherMatmulMoeBlockToMoeOp::Convert2GatherMatmulMoeBlockToMoeOp() { for (const auto v : zp_const->cast_vector()) { if (v != 0) { if (wt != ov::element::u2) { - return false; // non-zero scalar zp only supported for u2 weights + return false; } - // Keep the rank-0 scalar constant as-is: the fused backend - // broadcasts the single element over all experts/groups/channels. return true; } } @@ -633,8 +604,7 @@ Convert2GatherMatmulMoeBlockToMoeOp::Convert2GatherMatmulMoeBlockToMoeOp() { } } - // Broadcast a per-channel scale constant [E, N, 1] (or [E, N]) to - // [E, N, K/group_size] by repeating each channel value across groups. + // [E, N, 1] (or [E, N]) -> [E, N, K/group_size], each channel value repeated across groups. auto broadcast_per_channel_scale = [&](ov::Output& scale, size_t K) -> bool { const auto c = ov::as_type_ptr(scale.get_node_shared_ptr()); if (!c) @@ -646,11 +616,10 @@ Convert2GatherMatmulMoeBlockToMoeOp::Convert2GatherMatmulMoeBlockToMoeOp() { const size_t N = s[1]; const size_t target_G = K / group_size; if (target_G == 1) - return true; // nothing to expand + return true; const auto et = c->get_element_type(); - // Sub-byte types are bit-packed and Type::size() rounds up to a whole byte, so the - // per-element memcpy below would read past the constant and interleave neighbouring - // channels. Fall back to the unfused form rather than materialise corrupt values. + // Type::size() rounds sub-byte types up to a byte, so the per-element copy below would corrupt + // packed data. if (et.bitwidth() < 8) return false; const size_t esz = et.size(); @@ -668,10 +637,7 @@ Convert2GatherMatmulMoeBlockToMoeOp::Convert2GatherMatmulMoeBlockToMoeOp() { (down_G == 1 && !broadcast_per_channel_scale(down_scale, down_K))) { return false; } - // Broadcast a per-channel (u8) zp [E, N, 1] to [E, N, K/group_size], the - // same expansion as its scale, so zshape == sshape and the (group, channel) - // zp indexing line up. Scalar (u2) zps are rank-0 and were normalized - // above; they are left untouched here. + // Per-channel zps get the same expansion as their scales so the shapes match; rank-0 zps are kept. if (num_dynamic_zp == 0) { if ((gate_up_G == 1 && !broadcast_per_channel_scale(gate_up_zp, hidden)) || (down_G == 1 && !broadcast_per_channel_scale(down_zp, down_K))) { diff --git a/src/common/transformations/tests/common_optimizations/convert_gather_matmuls_moe_block_to_moe_op.cpp b/src/common/transformations/tests/common_optimizations/convert_gather_matmuls_moe_block_to_moe_op.cpp index 717d5e651e0f8e..26b637c067f268 100644 --- a/src/common/transformations/tests/common_optimizations/convert_gather_matmuls_moe_block_to_moe_op.cpp +++ b/src/common/transformations/tests/common_optimizations/convert_gather_matmuls_moe_block_to_moe_op.cpp @@ -843,33 +843,24 @@ TEST_F(TransformationTestsF, Convert2GatherMatmulMoeBlockToMoeOp_restores_output model_ref = build_2gemm_bgm_to_moe_reference_model(true, true); } -// Compressed (GatherMatmulCompressed) variants. Every builder above produces PLAIN GatherMatmul -// with f32 weights, so the entire compressed half of the callback -- the weight-dtype admission -// gate, the scalar-zp normalization, the group_size unification and the per-channel scale/zp -// broadcast -- is unreachable from the tests in this file. The models below exist to reach it. +// Compressed (GatherMatmulCompressed) variants: the builders above produce plain GatherMatmul only, which never +// reaches the compressed branch of the callback. namespace { -// The compressed branch only inspects weight dtypes and scale/zp shapes, so the smallest shapes -// that still express the interesting cases are enough. gate/up carry 4 scale groups over K=16 and -// down carries 2 over K=8, i.e. both sides must unify to the same group_size of 4 -- a single G -// everywhere would not exercise that. +// gate/up carry 4 scale groups over K=16 and down 2 over K=8, so both must unify to group_size 4. constexpr size_t kE = 2; constexpr size_t kHidden = 16; constexpr size_t kInter = 8; constexpr size_t kTopk = 2; constexpr size_t kGroupSize = 4; -// An absent optional input. GatherMatmul itself spells a missing bias this way; note it has to be -// the 2-argument Constant ctor, since Constant::create cannot produce a dynamic element type. +// An absent optional input. Constant::create cannot produce a dynamic element type, hence the 2-argument ctor. std::shared_ptr absent_input() { return std::make_shared(ov::element::dynamic, ov::Shape{0}); } -// Quantized values are never dequantized by these tests, but they must still be distinct per -// element so that a wrong broadcast/expansion is visible rather than masked by a uniform fill. -// Constant::create range-checks every element, and u2 holds only 0..3, so the pattern is clamped -// to whatever the requested type can actually represent. +// Distinct per-element values so a wrong expansion is visible, clamped to the type's range (u2 holds 0..3). std::shared_ptr int_const(ov::element::Type dt, const ov::Shape& shape, int32_t base = 0) { const int32_t hi = dt.bitwidth() < 8 ? static_cast((1u << (dt.bitwidth() - (dt.is_signed() ? 1 : 0))) - 1) : 127; @@ -904,10 +895,8 @@ struct CompressedMoeSpec { ov::element::Type zp_dt = ov::element::dynamic; }; -// Same topology as build_3gemm_bgm_model, with the three GatherMatmul replaced by -// GatherMatmulCompressed. The Parameter stays f32 on purpose: the router MatMul below multiplies -// by an f32 Constant and v0::MatMul requires both operands to share an element type, so making -// the Parameter f16 throws inside this builder before the pass ever runs. +// Same topology as build_3gemm_bgm_model with GatherMatmulCompressed. The Parameter stays f32: the router +// MatMul multiplies it by an f32 Constant and v0::MatMul needs matching element types. std::shared_ptr build_3gemm_bgm_compressed_model(const CompressedMoeSpec& spec) { using namespace ov; @@ -942,8 +931,7 @@ std::shared_ptr build_3gemm_bgm_compressed_model(const CompressedMoeS auto down_scale = scale_const(Shape{kE, kHidden, spec.down_G}); const bool has_zp = spec.zp_dt != element::dynamic; - // 10/11/12 rather than 0..3: a zp of 0 is indistinguishable from the symmetric case, and T6 - // asserts on the expanded values. + // Non-zero zps: a zp of 0 is indistinguishable from the symmetric case. auto gate_zp = has_zp ? int_const(spec.zp_dt, Shape{kE, kInter, spec.gate_G}, 10) : absent_input(); auto up_zp = has_zp ? int_const(spec.zp_dt, Shape{kE, kInter, spec.up_G}, 11) : absent_input(); auto down_zp = has_zp ? int_const(spec.zp_dt, Shape{kE, kHidden, spec.down_G}, 12) : absent_input(); @@ -991,10 +979,7 @@ std::shared_ptr run_3gemm_compressed(const std::shared_ptr(model), 3); } -// C7 negative control: the same shape with byte-wide zps is legitimately expandable, and this is -// the only test that asserts the expansion arithmetic rather than just that it was skipped. -// Green on both sides of the fix. +// Control: the same shape with byte-wide zps is expandable; this checks the expansion arithmetic. TEST(Convert3GatherMatmulMoeBlockToMoeOpCompressed, u8_per_channel_zp_expands) { CompressedMoeSpec spec; spec.gate_dt = spec.up_dt = spec.down_dt = ov::element::u8; diff --git a/src/core/src/op/moe.cpp b/src/core/src/op/moe.cpp index b1cbc8827047d5..e307d0f5828332 100644 --- a/src/core/src/op/moe.cpp +++ b/src/core/src/op/moe.cpp @@ -54,9 +54,8 @@ void MOE::validate_and_infer_types() { for (size_t i = 4; i < base_inputs_count; i++) { const auto& ps = get_input_partial_shape(i); NODE_VALIDATION_CHECK(this, ps.is_static(), "Weights must have static shape."); - // Note: dynamic element type means empty zero point input; - // a scalar (rank-0) input is a folded per-tensor zero point which - // has no num_experts dimension to check. + // Note: dynamic element type means empty zero point input; a rank-0 input is a per-tensor + // zero point with no num_experts dimension. if (get_input_element_type(i) != ov::element::dynamic && ps.size() > 0) { NODE_VALIDATION_CHECK(this, num_experts == get_input_shape(i)[0], From ac3cd0a738915475f3055fb84d2daee1ffeb13a9 Mon Sep 17 00:00:00 2001 From: Cyberpunk1210 Date: Thu, 22 Oct 2026 11:42:42 +0800 Subject: [PATCH 07/12] [GPU] Trim per-GEMM MoE dtype comments Comment-only cleanup: keep the why, drop restated code and history. Co-Authored-By: Claude Opus 5.5 (1M context) Signed-off-by: Cyberpunk1210 --- .../graph/impls/ocl_v2/moe/moe_3gemm_swiglu_opt.cpp | 3 +-- .../src/graph/impls/ocl_v2/moe_3gemm_swiglu_mlp.cl | 8 ++------ .../tests/unit/test_cases/moe_3gemm_gpu_test.cpp | 11 ++++------- 3 files changed, 7 insertions(+), 15 deletions(-) diff --git a/src/plugins/intel_gpu/src/graph/impls/ocl_v2/moe/moe_3gemm_swiglu_opt.cpp b/src/plugins/intel_gpu/src/graph/impls/ocl_v2/moe/moe_3gemm_swiglu_opt.cpp index 0c8c1cdb31747c..b6a7b775b364c1 100644 --- a/src/plugins/intel_gpu/src/graph/impls/ocl_v2/moe/moe_3gemm_swiglu_opt.cpp +++ b/src/plugins/intel_gpu/src/graph/impls/ocl_v2/moe/moe_3gemm_swiglu_opt.cpp @@ -433,8 +433,7 @@ static void add_common_consts(const RuntimeParams& params, JitConstants& jit) { const auto down_weight_dt = params.get_input_layout(static_cast(MOE3GemmInputIndex::WEIGHT_2)).data_type; const bool all_u2 = weight_dt == ov::element::u2 && up_weight_dt == ov::element::u2 && down_weight_dt == ov::element::u2; const size_t sg = (info.arch >= gpu_arch::xe2) ? 32u : 16u; - // Mirrors FAKE_GROUP_SIZE in moe_3gemm_swiglu_mlp.cl, which also takes the down group size; - // checking gate/up alone let a smaller down group through to a branch that does not exist. + // Mirrors FAKE_GROUP_SIZE in moe_3gemm_swiglu_mlp.cl, which also depends on the down group size. const size_t fake_gs = std::min({gate_up_group_size, down_group_size, size_t{128}}); const size_t min_fake_gs = all_u2 ? sg : 2 * sg; OPENVINO_ASSERT(fake_gs >= min_fake_gs, diff --git a/src/plugins/intel_gpu/src/graph/impls/ocl_v2/moe_3gemm_swiglu_mlp.cl b/src/plugins/intel_gpu/src/graph/impls/ocl_v2/moe_3gemm_swiglu_mlp.cl index c67835814a00c5..d91a2029e84313 100644 --- a/src/plugins/intel_gpu/src/graph/impls/ocl_v2/moe_3gemm_swiglu_mlp.cl +++ b/src/plugins/intel_gpu/src/graph/impls/ocl_v2/moe_3gemm_swiglu_mlp.cl @@ -90,9 +90,8 @@ inline float moe_mlp_fast_erf(float x) { // 2-bit (u2) dequant: 4 values per byte, LSB-first; u2 weights are always unsigned. #define DEQUANT_2BIT(v, s) convert_half(((v) >> (s)) & 0x3) -// Per-GEMM helpers (mixed-dtype MoE): per-expert byte strides given a compression code DT -// (0=u4/i4, 1=u8/i8, 2=f16, 3=u2) and group size GS. N*K equals INTERMEDIATE_SIZE*HIDDEN_SIZE for all -// three GEMMs, so one product serves gate/up/down. +// Per-expert byte strides for compression code DT (0=u4/i4, 1=u8/i8, 2=f16, 3=u2) and group size GS. +// N*K is INTERMEDIATE_SIZE*HIDDEN_SIZE for all three GEMMs. #define MOE_WEI_PROD (INTERMEDIATE_SIZE * HIDDEN_SIZE) #define MOE_EXPERT_WEI_BYTES(DT) ((DT) == 0 ? (MOE_WEI_PROD) / 2 : (DT) == 3 ? (MOE_WEI_PROD) / 4 : (MOE_WEI_PROD)) #define MOE_EXPERT_ZP_BYTES(DT, GS) \ @@ -683,7 +682,6 @@ __attribute__((intel_reqd_sub_group_size(SUBGROUP_SIZE))) KERNEL(mlp_gate_up)( bool is_shared = false; # endif - // Per-GEMM strides (gate and up may differ in dtype). const int gate_expert_wei_size = MOE_EXPERT_WEI_BYTES(GATE_WEIGHT_DT); const int up_expert_wei_size = MOE_EXPERT_WEI_BYTES(UP_WEIGHT_DT); const int expert_scale_size = INTERMEDIATE_SIZE * HIDDEN_SIZE / GATE_UP_GROUP_SIZE; // f16 scale, dtype-independent @@ -834,7 +832,6 @@ __attribute__((intel_reqd_sub_group_size(SUBGROUP_SIZE))) KERNEL(mlp_gate_up)( // shared_gate_out[token_idx] is consumed by the next kernel (mlp_down) — no barrier needed here. # endif - // Dispatch up (silu=false) then gate (silu=true) by each projection's own dtype. # if UP_WEIGHT_DT == 0 gate_up_gemv_n2x_u4(up_weight, up_scale, up_zp, y, INTERMEDIATE_SIZE, HIDDEN_SIZE, x2, xg_sum, false); # elif UP_WEIGHT_DT == 1 @@ -1368,7 +1365,6 @@ __attribute__((intel_reqd_sub_group_size(SUBGROUP_SIZE))) KERNEL(mlp_down)(const bool is_shared = false; # endif - // Per-GEMM strides for the down projection. const int expert_wei_size = MOE_EXPERT_WEI_BYTES(DOWN_WEIGHT_DT); const int expert_scale_size = INTERMEDIATE_SIZE * HIDDEN_SIZE / DOWN_GROUP_SIZE; // f16 scale, dtype-independent const int down_zp_stride = DOWN_ZP_SCALAR ? 0 : MOE_EXPERT_ZP_BYTES(DOWN_WEIGHT_DT, DOWN_GROUP_SIZE); diff --git a/src/plugins/intel_gpu/tests/unit/test_cases/moe_3gemm_gpu_test.cpp b/src/plugins/intel_gpu/tests/unit/test_cases/moe_3gemm_gpu_test.cpp index fcab29eedd51ec..2f581281a2c2e0 100644 --- a/src/plugins/intel_gpu/tests/unit/test_cases/moe_3gemm_gpu_test.cpp +++ b/src/plugins/intel_gpu/tests/unit/test_cases/moe_3gemm_gpu_test.cpp @@ -1698,11 +1698,9 @@ TEST(moe_3gemm_expert_mask, shape_predictor_overflow_regression) { // ============================================================================ // Mixed weight dtypes across the three expert GEMMs. // -// The decode kernels took one weight dtype, expert stride and zp stride from the gate GEMM and -// applied them to all three, so a layer with gate/up at u8 and down at u4 decoded down as u8: -// wrong output, no error. Each GEMM is now decoded with its own dtype. gate and up still have to -// agree on whether they are u4, because they read one SLM copy of the activations whose layout the -// gate dtype picks; down stages its own copy and is free. +// Each GEMM must be decoded with its own dtype. gate and up still have to agree on whether they are +// u4, because they read one SLM copy of the activations whose layout the gate dtype picks; down +// stages its own copy and is free. // // The reference uses the exact dequantized weights, so the tolerance only has to absorb f16 // accumulation, and a GEMM decoded with the other dtype's stride is far outside it. @@ -1878,8 +1876,7 @@ INSTANTIATE_TEST_SUITE_P(smoke, // Batched GEMV (MTP-style token counts). MoeMixedDtypeParams{4, false, true}, MoeMixedDtypeParams{16, true, false}, - // Just above the 32-token batched-GEMV threshold: the prefill GEMM path, which - // already took each GEMM's dtype; kept so a regression there is caught too. + // Just above the 32-token batched-GEMV threshold: the prefill GEMM path. MoeMixedDtypeParams{33, false, true}, MoeMixedDtypeParams{33, true, false}), moe_3gemm_compressed_gpu_mixed_dtype::GetTestCaseName); From 86e52535dce47b02412629b802957183284875c5 Mon Sep 17 00:00:00 2001 From: Cyberpunk1210 Date: Thu, 22 Oct 2026 14:14:33 +0800 Subject: [PATCH 08/12] [GPU] Make the u2 unpack work-group size constexpr Drops an unneeded lambda capture (clang -Wunused-lambda-capture). No behaviour change. Co-Authored-By: Claude Opus 5.5 (1M context) Signed-off-by: Cyberpunk1210 --- .../src/graph/impls/ocl_v2/moe/moe_3gemm_swiglu_opt.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/plugins/intel_gpu/src/graph/impls/ocl_v2/moe/moe_3gemm_swiglu_opt.cpp b/src/plugins/intel_gpu/src/graph/impls/ocl_v2/moe/moe_3gemm_swiglu_opt.cpp index b6a7b775b364c1..3c5f66d0534c49 100644 --- a/src/plugins/intel_gpu/src/graph/impls/ocl_v2/moe/moe_3gemm_swiglu_opt.cpp +++ b/src/plugins/intel_gpu/src/graph/impls/ocl_v2/moe/moe_3gemm_swiglu_opt.cpp @@ -2563,8 +2563,8 @@ class moe_3gemm_swiglu_opt_impl : public PrimitiveImplOCL { auto& engine = instance.get_network().get_engine(); cldnn::event::ptr ret = events.empty() ? nullptr : events[0]; - const size_t lws = 256; - const auto round_up = [lws](size_t v) { + static constexpr size_t lws = 256; + const auto round_up = [](size_t v) { return (v + lws - 1) / lws * lws; }; for (int i = 0; i < 3; i++) { From 745feceaf2089018226087641fde487131ffe370 Mon Sep 17 00:00:00 2001 From: Cyberpunk1210 Date: Fri, 23 Oct 2026 18:11:22 +0800 Subject: [PATCH 09/12] [Transformations] Check MoE weight dtypes in the pattern, broadcast with folded ops - Each compressed expert weight's dtype is now a pattern predicate (type_matches_any in the 3-GEMM matcher, "not u2" in the 2-GEMM matcher); only the cross-GEMM rule (uniform, or all mixable with gate/up agreeing on u4) stays in the callback. - Per-channel scale/zp expansion uses Reshape + Broadcast with make_try_fold instead of a raw byte copy. - The scalar-zp normalization is shared by both matchers and takes whether a non-zero zp may pass; the 2-GEMM matcher never admits one, which the old comment misstated. No change in what fuses. Co-Authored-By: Claude Opus 5.5 (1M context) Signed-off-by: Cyberpunk1210 --- .../common_optimizations/moe_op_fusion.cpp | 256 ++++++++---------- 1 file changed, 108 insertions(+), 148 deletions(-) diff --git a/src/common/transformations/src/transformations/common_optimizations/moe_op_fusion.cpp b/src/common/transformations/src/transformations/common_optimizations/moe_op_fusion.cpp index df22bc7c25af72..f92886c5673aa3 100644 --- a/src/common/transformations/src/transformations/common_optimizations/moe_op_fusion.cpp +++ b/src/common/transformations/src/transformations/common_optimizations/moe_op_fusion.cpp @@ -4,7 +4,6 @@ #include "transformations/common_optimizations/moe_op_fusion.hpp" -#include #include #include #include @@ -14,6 +13,7 @@ #include "openvino/core/rt_info.hpp" #include "openvino/core/type/element_type.hpp" #include "openvino/op/add.hpp" +#include "openvino/op/broadcast.hpp" #include "openvino/op/clamp.hpp" #include "openvino/op/constant.hpp" #include "openvino/op/convert.hpp" @@ -34,6 +34,7 @@ #include "ov_ops/gather_matmul.hpp" #include "ov_ops/gather_matmul_compressed.hpp" #include "ov_ops/moe_compressed.hpp" +#include "transformations/utils/utils.hpp" namespace ov::pass { @@ -43,6 +44,7 @@ using ov::op::internal::MOECompressed; namespace v0 = ov::op::v0; namespace v1 = ov::op::v1; +namespace v3 = ov::op::v3; namespace v4 = ov::op::v4; namespace v7 = ov::op::v7; namespace v8 = ov::op::v8; @@ -53,6 +55,65 @@ static size_t weight_logical_K(const ov::Shape& shape) { return shape.size() == 4 ? shape[2] * shape[3] : shape[2]; } +// A rank-0 zp is a folded per-tensor zero point. All-zero is symmetric and becomes the absent-zp constant. A non-zero +// one is kept only if allow_nonzero: MOE validation exempts rank-0 inputs, and only the u2 kernels broadcast it. +static bool normalize_scalar_zp(ov::Output& zp, bool allow_nonzero) { + const auto& zp_shape = zp.get_partial_shape(); + if (!zp_shape.is_static() || zp_shape.rank().get_length() != 0) { + return true; + } + const auto zp_const = ov::as_type_ptr(zp.get_node_shared_ptr()); + if (!zp_const) { + return false; + } + // cast_vector is not meaningful on packed (u2/u4/i4) constants. + const auto zp_et = zp_const->get_element_type(); + if (zp_et != ov::element::i8 && zp_et != ov::element::u8 && zp_et != ov::element::i32 && + zp_et != ov::element::u32 && zp_et != ov::element::i64) { + return false; + } + for (const auto v : zp_const->cast_vector()) { + if (v != 0) { + return allow_nonzero; + } + } + zp = std::make_shared(ov::element::dynamic, ov::Shape{0}); + return true; +} + +// [E, N, 1] (or [E, N]) -> [E, N, K/group_size], each channel value repeated across groups. +static bool broadcast_per_channel(ov::Output& scale, size_t K, size_t group_size) { + const auto c = ov::as_type_ptr(scale.get_node_shared_ptr()); + if (!c) + return false; + const auto s = c->get_shape(); + if (s.size() < 2 || K % group_size != 0) + return false; + const size_t target_G = K / group_size; + if (target_G == 1) + return true; + // Packed sub-byte constants are left unfused. + if (c->get_element_type().bitwidth() < 8) + return false; + const auto channel_shape = v0::Constant::create(ov::element::i64, {3}, {s[0], s[1], size_t{1}}); + const auto target_shape = v0::Constant::create(ov::element::i64, {3}, {s[0], s[1], target_G}); + const auto expanded = + ov::op::util::make_try_fold(ov::op::util::make_try_fold(c, channel_shape, false), + target_shape); + const auto expanded_const = ov::as_type_ptr(expanded); + if (!expanded_const) + return false; + scale = expanded_const; + return true; +} + +// Compressed weight dtypes the fused 3-GEMM MoE decodes. +static const std::vector moe_3gemm_weight_types = {ov::element::u2, + ov::element::u4, + ov::element::i4, + ov::element::u8, + ov::element::i8}; + Convert3GatherMatmulMoeBlockToMoeOp::Convert3GatherMatmulMoeBlockToMoeOp() { MATCHER_SCOPE(Convert3GatherMatmulMoeBlockToMoeOp); @@ -68,8 +129,9 @@ Convert3GatherMatmulMoeBlockToMoeOp::Convert3GatherMatmulMoeBlockToMoeOp() { // Compressed BGM (6 inputs: A, B, indices, bias, scale, zp) auto gate_scale_m = pattern::any_input(); auto gate_zp_m = pattern::any_input(); + auto gate_wc_m = pattern::any_input(pattern::type_matches_any(moe_3gemm_weight_types)); auto bgm_gate_6_m = pattern::wrap_type( - {unsqueeze_m, gate_w_m, topk_indices_m, pattern::any_input(), gate_scale_m, gate_zp_m}); + {unsqueeze_m, gate_wc_m, topk_indices_m, pattern::any_input(), gate_scale_m, gate_zp_m}); // Or-pattern auto bgm_gate_m = bgm_gate_4_m | bgm_gate_6_m; @@ -80,8 +142,9 @@ Convert3GatherMatmulMoeBlockToMoeOp::Convert3GatherMatmulMoeBlockToMoeOp() { auto bgm_up_4_m = pattern::wrap_type({unsqueeze_m, up_w_m, topk_indices_m, pattern::any_input()}); auto up_scale_m = pattern::any_input(); auto up_zp_m = pattern::any_input(); + auto up_wc_m = pattern::any_input(pattern::type_matches_any(moe_3gemm_weight_types)); auto bgm_up_6_m = pattern::wrap_type( - {unsqueeze_m, up_w_m, topk_indices_m, pattern::any_input(), up_scale_m, up_zp_m}); + {unsqueeze_m, up_wc_m, topk_indices_m, pattern::any_input(), up_scale_m, up_zp_m}); auto bgm_up_m = bgm_up_4_m | bgm_up_6_m; auto swiglu_m = pattern::wrap_type({swish_m, bgm_up_m}); @@ -90,8 +153,9 @@ Convert3GatherMatmulMoeBlockToMoeOp::Convert3GatherMatmulMoeBlockToMoeOp() { auto bgm_down_4_m = pattern::wrap_type({swiglu_m, down_w_m, topk_indices_m, pattern::any_input()}); auto down_scale_m = pattern::any_input(); auto down_zp_m = pattern::any_input(); + auto down_wc_m = pattern::any_input(pattern::type_matches_any(moe_3gemm_weight_types)); auto bgm_down_6_m = pattern::wrap_type( - {swiglu_m, down_w_m, topk_indices_m, pattern::any_input(), down_scale_m, down_zp_m}); + {swiglu_m, down_wc_m, topk_indices_m, pattern::any_input(), down_scale_m, down_zp_m}); auto bgm_down_m = bgm_down_4_m | bgm_down_6_m; auto routing_m = pattern::any_input(); @@ -119,9 +183,13 @@ Convert3GatherMatmulMoeBlockToMoeOp::Convert3GatherMatmulMoeBlockToMoeOp() { auto routing = pm.at(routing_m); auto topk_indices = pm.at(topk_indices_m); - auto gate_w = pm.at(gate_w_m); - auto up_w = pm.at(up_w_m); - auto down_w = pm.at(down_w_m); + const auto weight = [&](const std::shared_ptr& plain_m, + const std::shared_ptr& compressed_m) { + return pm.count(compressed_m) ? pm.at(compressed_m) : pm.at(plain_m); + }; + auto gate_w = weight(gate_w_m, gate_wc_m); + auto up_w = weight(up_w_m, up_wc_m); + auto down_w = weight(down_w_m, down_wc_m); // Extract expert_beta and detect activation type from the matched gate activation node. float expert_beta = 1.0f; @@ -160,24 +228,21 @@ Convert3GatherMatmulMoeBlockToMoeOp::Convert3GatherMatmulMoeBlockToMoeOp() { const bool is_compressed = is_gate_compressed; if (is_compressed) { - // moe_3gemm decodes each GEMM with its own dtype, except that gate and up share one SLM copy of the - // activations whose layout follows the gate dtype (interleaved for u4), so gate and up must agree on - // being u4. Signed types stay uniform-only (WEIGHT_IS_SIGNED comes from the gate). Anything else - // stays as GatherMatmul. + // Each weight's dtype is checked by the pattern. Across the three: moe_3gemm decodes each GEMM with its own + // dtype, except that gate and up share one SLM copy of the activations whose layout follows the gate + // dtype (interleaved for u4), so gate and up must agree on being u4. Signed types stay uniform-only + // (WEIGHT_IS_SIGNED comes from the gate). { - const auto gate_dt = pm.at(gate_w_m).get_element_type(); - const auto up_dt = pm.at(up_w_m).get_element_type(); - const auto down_dt = pm.at(down_w_m).get_element_type(); + const auto gate_dt = gate_w.get_element_type(); + const auto up_dt = up_w.get_element_type(); + const auto down_dt = down_w.get_element_type(); const auto is_mixable = [](ov::element::Type t) { return t == ov::element::u2 || t == ov::element::u4 || t == ov::element::u8; }; - const bool uniform_ok = - (up_dt == gate_dt && down_dt == gate_dt) && - (gate_dt == ov::element::u2 || gate_dt == ov::element::u4 || gate_dt == ov::element::i4 || - gate_dt == ov::element::u8 || gate_dt == ov::element::i8); - const bool mixed_ok = is_mixable(gate_dt) && is_mixable(up_dt) && is_mixable(down_dt) && - ((gate_dt == ov::element::u4) == (up_dt == ov::element::u4)); - if (!(uniform_ok || mixed_ok)) { + const bool uniform = up_dt == gate_dt && down_dt == gate_dt; + const bool mixable = is_mixable(gate_dt) && is_mixable(up_dt) && is_mixable(down_dt) && + ((gate_dt == ov::element::u4) == (up_dt == ov::element::u4)); + if (!(uniform || mixable)) { return false; } } @@ -185,38 +250,9 @@ Convert3GatherMatmulMoeBlockToMoeOp::Convert3GatherMatmulMoeBlockToMoeOp() { auto up_zp = pm.at(up_zp_m); auto down_zp = pm.at(down_zp_m); - // A rank-0 zp is a folded per-tensor zero point. All-zero is symmetric and becomes the absent-zp constant. - // Non-zero is kept as is for u2 weights only: MOE validation exempts rank-0 inputs and only the u2 kernels - // broadcast it. - auto normalize_scalar_zp = [&](ov::Output& zp, ov::element::Type wt) -> bool { - const auto& zp_shape = zp.get_partial_shape(); - if (!zp_shape.is_static() || zp_shape.rank().get_length() != 0) { - return true; - } - const auto zp_const = ov::as_type_ptr(zp.get_node_shared_ptr()); - if (!zp_const) { - return false; - } - // cast_vector is not meaningful on packed (u2/u4/i4) constants. - const auto zp_et = zp_const->get_element_type(); - if (zp_et != ov::element::i8 && zp_et != ov::element::u8 && zp_et != ov::element::i32 && - zp_et != ov::element::u32 && zp_et != ov::element::i64) { - return false; - } - for (const auto v : zp_const->cast_vector()) { - if (v != 0) { - if (wt != ov::element::u2) { - return false; - } - return true; - } - } - zp = std::make_shared(ov::element::dynamic, ov::Shape{0}); - return true; - }; - if (!normalize_scalar_zp(gate_zp, pm.at(gate_w_m).get_element_type()) || - !normalize_scalar_zp(up_zp, pm.at(up_w_m).get_element_type()) || - !normalize_scalar_zp(down_zp, pm.at(down_w_m).get_element_type())) { + if (!normalize_scalar_zp(gate_zp, gate_w.get_element_type() == ov::element::u2) || + !normalize_scalar_zp(up_zp, up_w.get_element_type() == ov::element::u2) || + !normalize_scalar_zp(down_zp, down_w.get_element_type() == ov::element::u2)) { return false; } const size_t num_dynamic_zp = static_cast(gate_zp.get_element_type() == ov::element::dynamic) + @@ -240,8 +276,8 @@ Convert3GatherMatmulMoeBlockToMoeOp::Convert3GatherMatmulMoeBlockToMoeOp() { // group_size derived from scales; weight_logical_K handles rank-3/4. const auto gate_K = weight_logical_K(weight_shape); - const auto up_K = weight_logical_K(pm.at(up_w_m).get_partial_shape().to_shape()); - const auto down_K = weight_logical_K(pm.at(down_w_m).get_partial_shape().to_shape()); + const auto up_K = weight_logical_K(up_w.get_partial_shape().to_shape()); + const auto down_K = weight_logical_K(down_w.get_partial_shape().to_shape()); auto scale_num_groups = [](const ov::Output& scale) -> size_t { const auto& ps = scale.get_partial_shape(); @@ -274,33 +310,8 @@ Convert3GatherMatmulMoeBlockToMoeOp::Convert3GatherMatmulMoeBlockToMoeOp() { } } - // [E, N, 1] (or [E, N]) -> [E, N, K/group_size], each channel value repeated across groups. - auto broadcast_per_channel_scale = [&](ov::Output& scale, size_t K) -> bool { - const auto c = ov::as_type_ptr(scale.get_node_shared_ptr()); - if (!c) - return false; - const auto s = c->get_shape(); - if (s.size() < 2 || K % group_size != 0) - return false; - const size_t E = s[0]; - const size_t N = s[1]; - const size_t target_G = K / group_size; - if (target_G == 1) - return true; - const auto et = c->get_element_type(); - // Type::size() rounds sub-byte types up to a byte, so the per-element copy below would corrupt - // packed data. - if (et.bitwidth() < 8) - return false; - const size_t esz = et.size(); - const auto* src = static_cast(c->get_data_ptr()); - std::vector buf(E * N * target_G * esz); - const size_t total = E * N * target_G; - for (size_t i = 0; i < total; ++i) { - std::memcpy(buf.data() + i * esz, src + (i / target_G) * esz, esz); - } - scale = std::make_shared(et, ov::Shape{E, N, target_G}, buf.data()); - return true; + const auto broadcast_per_channel_scale = [&](ov::Output& scale, size_t K) { + return broadcast_per_channel(scale, K, group_size); }; if (group_size != std::numeric_limits::max()) { if ((gate_G == 1 && !broadcast_per_channel_scale(gate_scale, gate_K)) || @@ -410,8 +421,13 @@ Convert2GatherMatmulMoeBlockToMoeOp::Convert2GatherMatmulMoeBlockToMoeOp() { // Compressed BGM (6 inputs) auto gate_up_scale_m = pattern::any_input(); auto gate_up_zp_m = pattern::any_input(); + // GEMM2 lowers to cldnn::moe_gemm, whose only impl (oneDNN) does not take u2 weights. + const auto not_u2 = [](const ov::Output& output) { + return output.get_element_type() != ov::element::u2; + }; + auto gate_up_wc_m = pattern::any_input(not_u2); auto bgm_gate_up_6_m = pattern::wrap_type( - {unsqueeze_m, gate_up_w_m, topk_indices_m, gate_up_bias_m, gate_up_scale_m, gate_up_zp_m}); + {unsqueeze_m, gate_up_wc_m, topk_indices_m, gate_up_bias_m, gate_up_scale_m, gate_up_zp_m}); auto bgm_gate_up_m = bgm_gate_up_4_m | bgm_gate_up_6_m; // Activation subgraph between gate_up and down BGMs @@ -435,8 +451,9 @@ Convert2GatherMatmulMoeBlockToMoeOp::Convert2GatherMatmulMoeBlockToMoeOp() { // Compressed BGM (6 inputs) auto down_scale_m = pattern::any_input(); auto down_zp_m = pattern::any_input(); + auto down_wc_m = pattern::any_input(not_u2); auto bgm_down_6_m = pattern::wrap_type( - {multiply2_m, down_w_m, topk_indices_m, down_bias_m, down_scale_m, down_zp_m}); + {multiply2_m, down_wc_m, topk_indices_m, down_bias_m, down_scale_m, down_zp_m}); auto bgm_down_m = bgm_down_4_m | bgm_down_6_m; // No-op Reshape between Transpose and Unsqueeze remains in the graph because @@ -468,9 +485,9 @@ Convert2GatherMatmulMoeBlockToMoeOp::Convert2GatherMatmulMoeBlockToMoeOp() { auto routing_transpose_node = pm.at(routing_transpose_m).get_node_shared_ptr(); ov::Output routing = routing_transpose_node->input_value(0); auto topk_indices = pm.at(topk_indices_m); - auto gate_up_w = pm.at(gate_up_w_m); + auto gate_up_w = pm.count(gate_up_wc_m) ? pm.at(gate_up_wc_m) : pm.at(gate_up_w_m); auto gate_up_bias = pm.at(gate_up_bias_m); - auto down_w = pm.at(down_w_m); + auto down_w = pm.count(down_wc_m) ? pm.at(down_wc_m) : pm.at(down_w_m); auto down_bias = pm.at(down_bias_m); // Extract expert_beta from Swish beta @@ -515,43 +532,11 @@ Convert2GatherMatmulMoeBlockToMoeOp::Convert2GatherMatmulMoeBlockToMoeOp() { const bool is_compressed = is_gate_up_compressed; if (is_compressed) { - // GEMM2 lowers to cldnn::moe_gemm, whose only impl (oneDNN) does not take u2 weights. - if (pm.at(gate_up_w_m).get_element_type() == ov::element::u2 || - pm.at(down_w_m).get_element_type() == ov::element::u2) { - return false; - } auto gate_up_zp = pm.at(gate_up_zp_m); auto down_zp = pm.at(down_zp_m); - // Same scalar-zp normalization as the 3-GEMM matcher above. - auto normalize_scalar_zp = [&](ov::Output& zp, ov::element::Type wt) -> bool { - const auto& zp_shape = zp.get_partial_shape(); - if (!zp_shape.is_static() || zp_shape.rank().get_length() != 0) { - return true; - } - const auto zp_const = ov::as_type_ptr(zp.get_node_shared_ptr()); - if (!zp_const) { - return false; - } - // cast_vector is not meaningful on packed (u2/u4/i4) constants. - const auto zp_et = zp_const->get_element_type(); - if (zp_et != ov::element::i8 && zp_et != ov::element::u8 && zp_et != ov::element::i32 && - zp_et != ov::element::u32 && zp_et != ov::element::i64) { - return false; - } - for (const auto v : zp_const->cast_vector()) { - if (v != 0) { - if (wt != ov::element::u2) { - return false; - } - return true; - } - } - zp = std::make_shared(ov::element::dynamic, ov::Shape{0}); - return true; - }; - if (!normalize_scalar_zp(gate_up_zp, pm.at(gate_up_w_m).get_element_type()) || - !normalize_scalar_zp(down_zp, pm.at(down_w_m).get_element_type())) { + // u2 never reaches this matcher, so a non-zero scalar zp always stays unfused. + if (!normalize_scalar_zp(gate_up_zp, false) || !normalize_scalar_zp(down_zp, false)) { return false; } const size_t num_dynamic_zp = static_cast(gate_up_zp.get_element_type() == ov::element::dynamic) + @@ -573,7 +558,7 @@ Convert2GatherMatmulMoeBlockToMoeOp::Convert2GatherMatmulMoeBlockToMoeOp() { auto down_scale = pm.at(down_scale_m); // group_size derived from scales; weight_logical_K handles rank-3/4. - const auto down_K = weight_logical_K(pm.at(down_w_m).get_partial_shape().to_shape()); + const auto down_K = weight_logical_K(down_w.get_partial_shape().to_shape()); auto scale_num_groups = [](const ov::Output& scale) -> size_t { const auto& ps = scale.get_partial_shape(); @@ -604,33 +589,8 @@ Convert2GatherMatmulMoeBlockToMoeOp::Convert2GatherMatmulMoeBlockToMoeOp() { } } - // [E, N, 1] (or [E, N]) -> [E, N, K/group_size], each channel value repeated across groups. - auto broadcast_per_channel_scale = [&](ov::Output& scale, size_t K) -> bool { - const auto c = ov::as_type_ptr(scale.get_node_shared_ptr()); - if (!c) - return false; - const auto s = c->get_shape(); - if (s.size() < 2 || K % group_size != 0) - return false; - const size_t E = s[0]; - const size_t N = s[1]; - const size_t target_G = K / group_size; - if (target_G == 1) - return true; - const auto et = c->get_element_type(); - // Type::size() rounds sub-byte types up to a byte, so the per-element copy below would corrupt - // packed data. - if (et.bitwidth() < 8) - return false; - const size_t esz = et.size(); - const auto* src = static_cast(c->get_data_ptr()); - std::vector buf(E * N * target_G * esz); - const size_t total = E * N * target_G; - for (size_t i = 0; i < total; ++i) { - std::memcpy(buf.data() + i * esz, src + (i / target_G) * esz, esz); - } - scale = std::make_shared(et, ov::Shape{E, N, target_G}, buf.data()); - return true; + const auto broadcast_per_channel_scale = [&](ov::Output& scale, size_t K) { + return broadcast_per_channel(scale, K, group_size); }; if (group_size != std::numeric_limits::max()) { if ((gate_up_G == 1 && !broadcast_per_channel_scale(gate_up_scale, hidden)) || From 237f067518726dcc9393fc4f716dbdd18f9d3cbe Mon Sep 17 00:00:00 2001 From: Cyberpunk1210 Date: Tue, 27 Oct 2026 15:44:13 +0800 Subject: [PATCH 10/12] [Transformations] Check for Constant first in normalize_scalar_zp, drop the broadcast wrapper lambda Same behaviour: non-Constant zp still passes unless rank-0; the wrapper lambda only forwarded group_size. Co-Authored-By: Claude Opus 5.5 (1M context) Signed-off-by: Cyberpunk1210 --- .../common_optimizations/moe_op_fusion.cpp | 36 ++++++++----------- 1 file changed, 15 insertions(+), 21 deletions(-) diff --git a/src/common/transformations/src/transformations/common_optimizations/moe_op_fusion.cpp b/src/common/transformations/src/transformations/common_optimizations/moe_op_fusion.cpp index f92886c5673aa3..d45bfd0f65e3a9 100644 --- a/src/common/transformations/src/transformations/common_optimizations/moe_op_fusion.cpp +++ b/src/common/transformations/src/transformations/common_optimizations/moe_op_fusion.cpp @@ -58,13 +58,13 @@ static size_t weight_logical_K(const ov::Shape& shape) { // A rank-0 zp is a folded per-tensor zero point. All-zero is symmetric and becomes the absent-zp constant. A non-zero // one is kept only if allow_nonzero: MOE validation exempts rank-0 inputs, and only the u2 kernels broadcast it. static bool normalize_scalar_zp(ov::Output& zp, bool allow_nonzero) { - const auto& zp_shape = zp.get_partial_shape(); - if (!zp_shape.is_static() || zp_shape.rank().get_length() != 0) { - return true; - } const auto zp_const = ov::as_type_ptr(zp.get_node_shared_ptr()); if (!zp_const) { - return false; + const auto rank = zp.get_partial_shape().rank(); + return rank.is_dynamic() || rank.get_length() != 0; + } + if (!zp_const->get_shape().empty()) { + return true; } // cast_vector is not meaningful on packed (u2/u4/i4) constants. const auto zp_et = zp_const->get_element_type(); @@ -310,20 +310,17 @@ Convert3GatherMatmulMoeBlockToMoeOp::Convert3GatherMatmulMoeBlockToMoeOp() { } } - const auto broadcast_per_channel_scale = [&](ov::Output& scale, size_t K) { - return broadcast_per_channel(scale, K, group_size); - }; if (group_size != std::numeric_limits::max()) { - if ((gate_G == 1 && !broadcast_per_channel_scale(gate_scale, gate_K)) || - (up_G == 1 && !broadcast_per_channel_scale(up_scale, up_K)) || - (down_G == 1 && !broadcast_per_channel_scale(down_scale, down_K))) { + if ((gate_G == 1 && !broadcast_per_channel(gate_scale, gate_K, group_size)) || + (up_G == 1 && !broadcast_per_channel(up_scale, up_K, group_size)) || + (down_G == 1 && !broadcast_per_channel(down_scale, down_K, group_size))) { return false; } // Per-channel zps get the same expansion as their scales so the shapes match; rank-0 zps are kept. if (num_dynamic_zp == 0) { - if ((gate_G == 1 && !broadcast_per_channel_scale(gate_zp, gate_K)) || - (up_G == 1 && !broadcast_per_channel_scale(up_zp, up_K)) || - (down_G == 1 && !broadcast_per_channel_scale(down_zp, down_K))) { + if ((gate_G == 1 && !broadcast_per_channel(gate_zp, gate_K, group_size)) || + (up_G == 1 && !broadcast_per_channel(up_zp, up_K, group_size)) || + (down_G == 1 && !broadcast_per_channel(down_zp, down_K, group_size))) { return false; } } @@ -589,18 +586,15 @@ Convert2GatherMatmulMoeBlockToMoeOp::Convert2GatherMatmulMoeBlockToMoeOp() { } } - const auto broadcast_per_channel_scale = [&](ov::Output& scale, size_t K) { - return broadcast_per_channel(scale, K, group_size); - }; if (group_size != std::numeric_limits::max()) { - if ((gate_up_G == 1 && !broadcast_per_channel_scale(gate_up_scale, hidden)) || - (down_G == 1 && !broadcast_per_channel_scale(down_scale, down_K))) { + if ((gate_up_G == 1 && !broadcast_per_channel(gate_up_scale, hidden, group_size)) || + (down_G == 1 && !broadcast_per_channel(down_scale, down_K, group_size))) { return false; } // Per-channel zps get the same expansion as their scales so the shapes match; rank-0 zps are kept. if (num_dynamic_zp == 0) { - if ((gate_up_G == 1 && !broadcast_per_channel_scale(gate_up_zp, hidden)) || - (down_G == 1 && !broadcast_per_channel_scale(down_zp, down_K))) { + if ((gate_up_G == 1 && !broadcast_per_channel(gate_up_zp, hidden, group_size)) || + (down_G == 1 && !broadcast_per_channel(down_zp, down_K, group_size))) { return false; } } From 5fb76f4dbfd0ba8e1241c5ff8d424cc29de9e554 Mon Sep 17 00:00:00 2001 From: Cyberpunk1210 Date: Thu, 29 Oct 2026 13:41:34 +0800 Subject: [PATCH 11/12] [GPU] MoE 3-GEMM: validate all three weight inputs, drop the duplicate WEIGHT_COMPRESSEION_DT validate_impl now checks WEIGHT/SCALE/ZP of gate, up and down, plus the cross-input constraints the kernel already relies on. Co-Authored-By: Claude Opus 5.5 (1M context) Signed-off-by: Cyberpunk1210 --- .../impls/ocl_v2/moe/moe_3gemm_swiglu_opt.cpp | 6 +- .../impls/ocl_v2/moe/moe_3gemm_swiglu_opt.hpp | 67 +++++++++++++------ 2 files changed, 48 insertions(+), 25 deletions(-) diff --git a/src/plugins/intel_gpu/src/graph/impls/ocl_v2/moe/moe_3gemm_swiglu_opt.cpp b/src/plugins/intel_gpu/src/graph/impls/ocl_v2/moe/moe_3gemm_swiglu_opt.cpp index 3c5f66d0534c49..45c1a88fb0988a 100644 --- a/src/plugins/intel_gpu/src/graph/impls/ocl_v2/moe/moe_3gemm_swiglu_opt.cpp +++ b/src/plugins/intel_gpu/src/graph/impls/ocl_v2/moe/moe_3gemm_swiglu_opt.cpp @@ -483,7 +483,6 @@ static void add_common_consts(const RuntimeParams& params, JitConstants& jit) { // auto scale_dt = params.get_input_layout(static_cast(MOE3GemmInputIndex::SCALE_0)).data_type; // auto zp_dt = params.get_input_layout(static_cast(MOE3GemmInputIndex::ZP_0)).data_type; if (weight_dt == ov::element::u4 || weight_dt == ov::element::i4) { - jit.make("WEIGHT_COMPRESSEION_DT", 0); jit.make("MOE_WEI_DT", "uchar"); jit.make("MOE_SCALE_DT", "half"); jit.make("MOE_ZP_DT", "uchar"); @@ -494,24 +493,21 @@ static void add_common_consts(const RuntimeParams& params, JitConstants& jit) { desc->_config.hidden_size, ", inter_size=", desc->_config.inter_size); - jit.make("WEIGHT_COMPRESSEION_DT", 3); jit.make("MOE_WEI_DT", "uchar"); jit.make("MOE_SCALE_DT", "half"); jit.make("MOE_ZP_DT", "uchar"); } else if (weight_dt == ov::element::u8 || weight_dt == ov::element::i8) { - jit.make("WEIGHT_COMPRESSEION_DT", 1); jit.make("MOE_WEI_DT", "uchar"); jit.make("MOE_SCALE_DT", "half"); jit.make("MOE_ZP_DT", "uchar"); } else if (weight_dt == ov::element::f16) { - jit.make("WEIGHT_COMPRESSEION_DT", 2); jit.make("MOE_WEI_DT", "half"); jit.make("MOE_SCALE_DT", "half"); // not use jit.make("MOE_ZP_DT", "half"); // not use } // Per-GEMM weight dtype codes: gate, up and down may differ under NNCF mixed precision, and - // each is decoded with its own. Codes mirror WEIGHT_COMPRESSEION_DT: 0=u4/i4, 1=u8/i8, 2=f16, 3=u2. + // each is decoded with its own. Codes: 0=u4/i4, 1=u8/i8, 2=f16, 3=u2. const auto moe_weight_code = [](ov::element::Type dt) -> int { if (dt == ov::element::u4 || dt == ov::element::i4) return 0; diff --git a/src/plugins/intel_gpu/src/graph/impls/ocl_v2/moe/moe_3gemm_swiglu_opt.hpp b/src/plugins/intel_gpu/src/graph/impls/ocl_v2/moe/moe_3gemm_swiglu_opt.hpp index e545d3c5040c08..6325ad41ced141 100644 --- a/src/plugins/intel_gpu/src/graph/impls/ocl_v2/moe/moe_3gemm_swiglu_opt.hpp +++ b/src/plugins/intel_gpu/src/graph/impls/ocl_v2/moe/moe_3gemm_swiglu_opt.hpp @@ -41,6 +41,14 @@ struct moe_3gemm_swiglu_opt : public ImplementationManager { return false; } + using Idx = MOE3GemmInputIndex; + const auto dt_of = [&](Idx idx) { + return ov::element::Type(node.get_input_layout(static_cast(idx)).data_type); + }; + const auto is_signed = [](ov::element::Type dt) { + return dt == ov::element::i4 || dt == ov::element::i8; + }; + // Only support weight: u4, i4, u2, u8, i8 static constexpr std::array supported_wei_type = { ov::element::u4, @@ -49,34 +57,53 @@ struct moe_3gemm_swiglu_opt : public ImplementationManager { ov::element::u8, ov::element::i8, }; - const auto& wei_layout = node.get_input_layout(static_cast(MOE3GemmInputIndex::WEIGHT_0)); - if (!one_of(wei_layout.data_type, supported_wei_type)) { - return false; - } - // Only support scale: f16 static constexpr std::array supported_scale_type = { ov::element::f16, }; - const auto& scale_layout = node.get_input_layout(static_cast(MOE3GemmInputIndex::SCALE_0)); - if (!one_of(scale_layout.data_type, supported_scale_type)) { - return false; - } - // Only support zp: u4, i4, u2, u8, i8 (skip check for symmetric quantization where ZP is element::dynamic placeholder) + static constexpr std::array supported_zp_type = { + ov::element::u4, // asym-quant type + ov::element::i4, // sym-quant type + ov::element::u2, // asym-quant type + ov::element::u8, // asym-quant type + ov::element::i8, // sym-quant type + }; const auto& config = std::static_pointer_cast(node.get_primitive())->_config; - if (config.has_zp) { - static constexpr std::array supported_zp_type = { - ov::element::u4, // asym-quant type - ov::element::i4, // sym-quant type - ov::element::u2, // asym-quant type - ov::element::u8, // asym-quant type - ov::element::i8, // sym-quant type - }; - const auto& zp_layout = node.get_input_layout(static_cast(MOE3GemmInputIndex::ZP_0)); - if (!one_of(zp_layout.data_type, supported_zp_type)) { + + static constexpr std::array weights = {Idx::WEIGHT_0, Idx::WEIGHT_1, Idx::WEIGHT_2}; + static constexpr std::array scales = {Idx::SCALE_0, Idx::SCALE_1, Idx::SCALE_2}; + static constexpr std::array zps = {Idx::ZP_0, Idx::ZP_1, Idx::ZP_2}; + for (size_t i = 0; i < weights.size(); ++i) { + const auto wei_dt = dt_of(weights[i]); + if (!one_of(wei_dt, supported_wei_type) || !one_of(dt_of(scales[i]), supported_scale_type)) { + return false; + } + // WEIGHT_IS_SIGNED is derived from WEIGHT_0 and applied to all three GEMMs. + if (is_signed(wei_dt) != is_signed(dt_of(weights[0]))) { return false; } + if (config.has_zp) { + const auto zp_dt = dt_of(zps[i]); + // A single-element zp is broadcast in-kernel (GATE/UP/DOWN_ZP_SCALAR, zp stride 0), so only its type is + // checked. Otherwise the kernel strides zp by the weight's bit width (MOE_EXPERT_ZP_BYTES): they must match. + const auto& zp_layout = node.get_input_layout(static_cast(zps[i])); + const bool scalar_zp = zp_layout.is_static() && zp_layout.count() == 1; + if (!one_of(zp_dt, supported_zp_type) || (!scalar_zp && zp_dt.bitwidth() != wei_dt.bitwidth())) { + return false; + } + } + } + + // Shared-expert GEMMs are decoded with the same GATE/UP/DOWN_WEIGHT_DT and MOE_*_DT as the routed ones. + if (node.get_dependencies().size() > static_cast(Idx::SHARED_GATE_WEIGHT)) { + static constexpr std::array shared_weights = {Idx::SHARED_GATE_WEIGHT, Idx::SHARED_UP_WEIGHT, Idx::SHARED_DOWN_WEIGHT}; + static constexpr std::array shared_scales = {Idx::SHARED_GATE_SCALE, Idx::SHARED_UP_SCALE, Idx::SHARED_DOWN_SCALE}; + for (size_t i = 0; i < shared_weights.size(); ++i) { + if (dt_of(shared_weights[i]) != dt_of(weights[i]) || dt_of(shared_scales[i]) != dt_of(scales[i])) { + return false; + } + } } return true; From aa04719d92463de3d2357d242991db138ed75c29 Mon Sep 17 00:00:00 2001 From: Cyberpunk1210 Date: Thu, 29 Oct 2026 14:15:26 +0800 Subject: [PATCH 12/12] [GPU] MoE 3-GEMM: do not tie shared-expert dtypes to the routed GEMMs Prefill runs the shared expert through oneDNN with per-projection dtypes; validate_impl cannot tell decode from prefill, so the check rejected valid models. Co-Authored-By: Claude Opus 5.5 (1M context) Signed-off-by: Cyberpunk1210 --- .../graph/impls/ocl_v2/moe/moe_3gemm_swiglu_opt.hpp | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/src/plugins/intel_gpu/src/graph/impls/ocl_v2/moe/moe_3gemm_swiglu_opt.hpp b/src/plugins/intel_gpu/src/graph/impls/ocl_v2/moe/moe_3gemm_swiglu_opt.hpp index 6325ad41ced141..b504e0c53bc7e3 100644 --- a/src/plugins/intel_gpu/src/graph/impls/ocl_v2/moe/moe_3gemm_swiglu_opt.hpp +++ b/src/plugins/intel_gpu/src/graph/impls/ocl_v2/moe/moe_3gemm_swiglu_opt.hpp @@ -95,17 +95,6 @@ struct moe_3gemm_swiglu_opt : public ImplementationManager { } } - // Shared-expert GEMMs are decoded with the same GATE/UP/DOWN_WEIGHT_DT and MOE_*_DT as the routed ones. - if (node.get_dependencies().size() > static_cast(Idx::SHARED_GATE_WEIGHT)) { - static constexpr std::array shared_weights = {Idx::SHARED_GATE_WEIGHT, Idx::SHARED_UP_WEIGHT, Idx::SHARED_DOWN_WEIGHT}; - static constexpr std::array shared_scales = {Idx::SHARED_GATE_SCALE, Idx::SHARED_UP_SCALE, Idx::SHARED_DOWN_SCALE}; - for (size_t i = 0; i < shared_weights.size(); ++i) { - if (dt_of(shared_weights[i]) != dt_of(weights[i]) || dt_of(shared_scales[i]) != dt_of(scales[i])) { - return false; - } - } - } - return true; } };