Skip to content

[GPU] Support u2 weights in the optimized bf_tiled FC kernel #39058

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
Cyberpunk1210 wants to merge 11 commits into
base: master
Choose a base branch
from
+539 −59
Open

[GPU] Support u2 weights in the optimized bf_tiled FC kernel #39058

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
11 commits
Select commit Hold shift + click to select a range
4b942d3
[GPU] Support u2 weights in the optimized bf_tiled FC kernel
Cyberpunk1210 Oct 13, 2026
14906c3
Merge remote-tracking branch 'upstream/master' into int2-fc
Cyberpunk1210 Oct 21, 2026
c5922a4
[GPU] Trim u2 FC comments and order UINT2 before UINT4 in toCLType
Cyberpunk1210 Oct 22, 2026
a4691a8
[GPU] Simplify u2 tiling checks in bf_tiled VerifyTuneParams
Cyberpunk1210 Oct 22, 2026
8d492de
[GPU] Use COMPRESSED_WEIGHTS_SUB_BYTE for the packed paths in bf_tiled
Cyberpunk1210 Oct 23, 2026
ac8888c
[GPU] Skip u2 dynamic quantization in the pipeline callback, split re…
Cyberpunk1210 Oct 27, 2026
82ca58e
[GPU] Split the remaining packed-u2 shift expression in reorder_weigh…
Cyberpunk1210 Oct 27, 2026
0fb1480
[GPU] Add is_2bit, name the u2 osv64 line size, add u2 bf_tiled SLM t…
Cyberpunk1210 Oct 28, 2026
97f092e
Merge remote-tracking branch 'upstream/master' into int2-fc
Cyberpunk1210 Oct 28, 2026
5d5e2fd
[GPU] Rename reorder_weights_int4 to reorder_weights_compressed, keep…
Cyberpunk1210 Oct 29, 2026
1aa3b36
[GPU] Word the u2 dyn-quant trace for any compressed activation
Cyberpunk1210 Oct 30, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion src/plugins/intel_gpu/src/graph/fully_connected.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -334,10 +334,11 @@ kernel_impl_params fully_connected_inst::get_fake_aligned_params(kernel_impl_par
size_t fake_align_base = 8;
if (orig_impl_param.dev_type == cldnn::device_type::integrated_gpu) {
auto weights_layout_dt = orig_impl_param.weights_layout.value().data_type;
auto is_2bit = weights_layout_dt == data_types::u2;
auto is_4bit = weights_layout_dt == data_types::i4 || weights_layout_dt == data_types::u4;
auto is_8bit = weights_layout_dt == data_types::i8 || weights_layout_dt == data_types::u8;
auto is_extra_alignment_needed = batch_size >= 256;
fake_align_base = (is_4bit || is_8bit) && is_extra_alignment_needed ? 64 : 16;
fake_align_base = (is_2bit || is_4bit || is_8bit) && is_extra_alignment_needed ? 64 : 16;
}

std::fill(input_shape.begin(), input_shape.end() - 1, 1);
Expand Down
2 changes: 1 addition & 1 deletion src/plugins/intel_gpu/src/graph/primitive_inst.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1540,7 +1540,7 @@ bool primitive_inst::use_async_compilation() {
auto batch_size = std::accumulate(input_shape.begin(), input_shape.end() - 1, size_t{1}, std::multiplies<size_t>());

// Disable async compilation for all int4 FC, except in the case of batch_size == 1
if (one_of(weights_dt, {data_types::i4, data_types::u4}) && batch_size != 1) {
if (one_of(weights_dt, {data_types::i4, data_types::u4, data_types::u2}) && batch_size != 1) {
compile_fc_impls = false;
}
} else {
Expand Down
55 changes: 34 additions & 21 deletions src/plugins/intel_gpu/src/kernel_selector/cl_kernels/fully_connected_gpu_bf_tiled.cl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,9 @@
#include "include/batch_headers/sub_group_block_write.cl"
#include "include/batch_headers/sub_group_shuffle.cl"

// os_is_yx_osv64_isv2 u2: one k-pair line is 64 OFM x 2 K x 2 bit.
#define U2_OSV64_ISV2_LINE_SIZE_BYTES 32

// JIT Parameters:
// SIMD - sub-group size/simd width, one of {8, 16};
// TILE_B - number of batches processed by each work-item;
Expand Down Expand Up @@ -86,9 +89,9 @@ KERNEL(quantize_input)(
# error "fully_connected_gpu_bf_tiled.cl - TILE_K_OFM must be equal to TILE_K * TILE_OFM and at most 8"
#endif

#if COMPRESSED_WEIGHTS_INT4
# if TILE_K_OFM != TILE_K_OFM_PACKED * 2
# error "fully_connected_gpu_bf_tiled.cl - TILE_K_OFM must be divisible by 2 for 4-bit compressed case"
#if COMPRESSED_WEIGHTS_SUB_BYTE
# if TILE_K_OFM != TILE_K_OFM_PACKED * FILTER_ELEMENTS_PER_BYTE
# error "fully_connected_gpu_bf_tiled.cl - TILE_K_OFM must be divisible by the number of values packed per byte for compressed case"
# endif
# if FILTER_LAYOUT_OS_IS_YX_OSV32_ISV2 && TILE_K != 4 && TILE_K != 2 && TILE_K != 1
# error "fully_connected_gpu_bf_tiled.cl - TILE_K must be one of {1, 2, 4}"
Expand All @@ -100,7 +103,7 @@ KERNEL(quantize_input)(
# error "fully_connected_gpu_bf_tiled.cl - outer_ofm should be 2 when swiglu is fused"
# endif
#endif
#if TILE_K == 4 && COMPRESSED_WEIGHTS_INT4 && FILTER_LAYOUT_OS_IS_YX_OSV32_ISV2
#if TILE_K == 4 && COMPRESSED_WEIGHTS_SUB_BYTE && FILTER_LAYOUT_OS_IS_YX_OSV32_ISV2
// Data stored in memory : f0k0k1|f16k0k1|f0k2k3|f16k2k3
// => unpack as f0k0k1|f0k2k3|f16k0k1|f16k2k3 so that the weight access order is preserved
#define UNPACK_INT4 UNPACK_INT4x2_OSV32_ISV2
Expand Down Expand Up @@ -146,7 +149,7 @@ KERNEL(quantize_input)(

#define INPUT_ELEMENTS_COUNT IFM_SIZE

#if IS_DYNAMIC && COMPRESSED_WEIGHTS_INT4
#if IS_DYNAMIC && COMPRESSED_WEIGHTS_SUB_BYTE
#pragma disable_includes_optimization
#define FORCED_TILE_B 1
#include "include/fully_connected_gpu_bf_tiled_common.cl"
Expand Down Expand Up @@ -242,7 +245,7 @@ inline void FUNC(fc_bf_tiled_kernel_default)(
ACCUMULATOR_VEC_TYPE acc[TILE_B] = { };
INPUT_VEC_TYPE in_0[TILE_B] = { };

#if !USE_SLM || !COMPRESSED_WEIGHTS_INT4
#if !USE_SLM || !COMPRESSED_WEIGHTS_SUB_BYTE
FILTER_VEC_TYPE wei = 0;
#endif

Expand All @@ -254,8 +257,8 @@ inline void FUNC(fc_bf_tiled_kernel_default)(
uint input_offset = out_b * TILE_IN_B_PITCH + INPUT0_OFFSET;
#endif

#if COMPRESSED_WEIGHTS_INT4
#if TILE_OFM == 1 && FILTER_LAYOUT_OS_IS_YX_OSV32_ISV2
#if COMPRESSED_WEIGHTS_SUB_BYTE

@michal-miotk michal-miotk Oct 28, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

looks like this branch is not touched by u2 - please express that in #if

@Cyberpunk1210 Cyberpunk1210 Oct 29, 2026

Copy link
Copy Markdown
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Done, added && COMPRESSED_WEIGHTS_INT4.

#if TILE_OFM == 1 && FILTER_LAYOUT_OS_IS_YX_OSV32_ISV2 && COMPRESSED_WEIGHTS_INT4
const int power_of_two_for_simd = 4;
const int power_of_two_for_osv = 5;
const uint osv32_weight_base = (( (int) (out_f >> power_of_two_for_osv) ) << power_of_two_for_osv);
Expand All @@ -270,15 +273,16 @@ inline void FUNC(fc_bf_tiled_kernel_default)(
const int power_of_two_for_simd = 5;
const int power_of_two_for_osv = 6;
const uint osv64_weight_base = (( (int) (out_f >> power_of_two_for_osv) ) << power_of_two_for_osv);
const uint osv_weight_stride = (INPUT_ELEMENTS_COUNT >> 1);
const uint out_f_offset = (int)((out_f >> power_of_two_for_simd) & 0x1) << power_of_two_for_simd;
const uint osv_weight_stride = (INPUT_ELEMENTS_COUNT / FILTER_ELEMENTS_PER_BYTE);
// A u2 byte packs two OFMs, so the upper 32-OFM half starts 16 bytes (not 32) into the line.
const uint out_f_offset = (int)((out_f >> power_of_two_for_simd) & 0x1) * ((1u << power_of_two_for_osv) / FILTER_ELEMENTS_PER_BYTE);
// out_f(32) : 0 * osv_weight_stride + 32;
// out_f(64) : 64 * osv_weight_stride + 0;
// out_f(128) : 64 * osv_weight_stride + 32;
// ...
uint weights_offset = osv64_weight_base * osv_weight_stride + out_f_offset;
#else
uint weights_offset = out_f * (INPUT_ELEMENTS_COUNT / 2);
uint weights_offset = out_f * (INPUT_ELEMENTS_COUNT / FILTER_ELEMENTS_PER_BYTE);
#endif
#else
uint weights_offset = out_f * INPUT_ELEMENTS_COUNT;
Expand Down Expand Up @@ -384,7 +388,7 @@ inline void FUNC(fc_bf_tiled_kernel_default)(
// It doesn't also show noticable performance improvement on tested configurations.
ACCUMULATOR_VEC_TYPE acc_tmp[TILE_B] = { };

#if USE_SLM && COMPRESSED_WEIGHTS_INT4
#if USE_SLM && COMPRESSED_WEIGHTS_SUB_BYTE
#if TILE_OFM != 2
#error "FC bf_tiled kernel: can't use SLM optimization with TILE_OFM != 2"
#endif
Expand All @@ -400,21 +404,30 @@ inline void FUNC(fc_bf_tiled_kernel_default)(
__local SLM_FILTER_VEC* slm_wei_vec = (__local SLM_FILTER_VEC*)wei_local_mem;

#if FILTER_LAYOUT_OS_IS_YX_OSV64_ISV2
#if COMPRESSED_WEIGHTS_UINT2
// u2 k-pair lines are 32 bytes, not 64.
uint weights_idx = weights_offset + local_id * SIMD * FILTER_LOAD_ITERS * FILTER_LOAD_BLOCK_SIZE;
#else
uint weights_idx = weights_offset + local_id * SIMD * FILTER_LOAD_ITERS * FILTER_LOAD_BLOCK_SIZE * 2;
#endif
#else
uint weights_idx = weights_offset + local_id * SIMD * FILTER_LOAD_ITERS * FILTER_ACTUAL_LOAD_BLOCK_SIZE;
#endif
uint wei_local_idx = local_id * SIMD * FILTER_LOAD_ITERS * FILTER_LOAD_BLOCK_SIZE + sglid;
unroll_for(uint load_iter = 0; load_iter < FILTER_LOAD_ITERS; ++load_iter) {
#if FILTER_LAYOUT_OS_IYX_OSV16
SLM_FILTER_PACKED_VEC wei_packed0 = BLOCK_READN(FILTER_TYPE, FILTER_ACTUAL_LOAD_BLOCK_SIZE, weights, weights_idx);
SLM_FILTER_PACKED_VEC wei_packed1 = BLOCK_READN(FILTER_TYPE, FILTER_ACTUAL_LOAD_BLOCK_SIZE, weights, (weights_idx + ((IFM_SIZE / 2) * 16)));
SLM_FILTER_PACKED_VEC wei_packed1 = BLOCK_READN(FILTER_TYPE, FILTER_ACTUAL_LOAD_BLOCK_SIZE, weights, (weights_idx + ((IFM_SIZE / FILTER_ELEMENTS_PER_BYTE) * 16)));
SLM_FILTER_UNPACKED_VEC wei_unpacked;
wei_unpacked.s0123 = UNPACK_INT4(ACCUMULATOR_TYPE, *((INT4_PACKED_TYPE_PRELOAD*)&wei_packed0));
wei_unpacked.s4567 = UNPACK_INT4(ACCUMULATOR_TYPE, *((INT4_PACKED_TYPE_PRELOAD*)&wei_packed1));
#elif FILTER_LAYOUT_OS_IS_YX_OSV64_ISV2
SLM_FILTER_PACKED_VEC wei_packed0 = BLOCK_READN(FILTER_TYPE, FILTER_ACTUAL_LOAD_BLOCK_SIZE, weights, weights_idx);
#if COMPRESSED_WEIGHTS_UINT2
SLM_FILTER_PACKED_VEC wei_packed1 = BLOCK_READN(FILTER_TYPE, FILTER_ACTUAL_LOAD_BLOCK_SIZE, weights, (weights_idx + U2_OSV64_ISV2_LINE_SIZE_BYTES));
#else
SLM_FILTER_PACKED_VEC wei_packed1 = BLOCK_READN(FILTER_TYPE, FILTER_ACTUAL_LOAD_BLOCK_SIZE, weights, (weights_idx + (FILTER_LOAD_BLOCK_SIZE * SIMD)));
#endif
SLM_FILTER_UNPACKED_VEC wei_unpacked;
SLM_FILTER_UNPACKED_VEC wei_unpacked_tmp;
wei_unpacked_tmp.s0123 = UNPACK_INT4(ACCUMULATOR_TYPE, *((INT4_PACKED_TYPE_PRELOAD*)&wei_packed0));
Expand Down Expand Up @@ -501,7 +514,7 @@ inline void FUNC(fc_bf_tiled_kernel_default)(
barrier(CLK_LOCAL_MEM_FENCE);
#endif
unroll_for(uint ki = 0; ki < (TILE_IFM * SIMD) / TILE_K; ++ki) {
#if COMPRESSED_WEIGHTS_INT4
#if COMPRESSED_WEIGHTS_SUB_BYTE
#if USE_SLM
FILTER_VEC_TYPE wei = 0;
#define LOAD_FROM_SLM(vec2) vec2 = slm_wei_vec[wei_local_idx]; wei_local_idx += SIMD;
Expand Down Expand Up @@ -692,7 +705,7 @@ inline void FUNC(fc_bf_tiled_kernel_default)(
FILTER_VEC_TYPE wei = 0;
#endif

#if COMPRESSED_WEIGHTS_INT4
#if COMPRESSED_WEIGHTS_SUB_BYTE
FILTER_PACKED_VEC_TYPE wei_packed = FILTER_BLOCK_READ(weights, weights_offset);
wei = UNPACK_INT4(ACCUMULATOR_TYPE, *((INT4_PACKED_TYPE*)&wei_packed));
#else
Expand Down Expand Up @@ -870,7 +883,7 @@ inline void FUNC(fc_bf_tiled_kernel_default)(
// Dyc Quantize
#if USE_SLM && DYNAMIC_QUANTIZE

#if COMPRESSED_WEIGHTS_INT4
#if COMPRESSED_WEIGHTS_SUB_BYTE
#define SLM_WEIGHT_TYPE DQ_TYPE
#else
#define SLM_WEIGHT_TYPE FILTER_TYPE
Expand Down Expand Up @@ -948,7 +961,7 @@ inline void FUNC(fc_bf_tiled_kernel_dyn_quan)(
uint input_offset = out_b * TILE_IN_B_PITCH + INPUT0_OFFSET;
#endif

#if COMPRESSED_WEIGHTS_INT4
#if COMPRESSED_WEIGHTS_SUB_BYTE
#if FILTER_LAYOUT_OS_IS_YX_OSV64_ISV2
const int power_of_two_for_simd = 5;
const int power_of_two_for_osv = 6;
Expand Down Expand Up @@ -1117,7 +1130,7 @@ inline void FUNC(fc_bf_tiled_kernel_dyn_quan)(

__local uint* char_slm_weight = (__local uint*)wei_local_mem;

#if COMPRESSED_WEIGHTS_INT4
#if COMPRESSED_WEIGHTS_SUB_BYTE
#if FILTER_LAYOUT_OS_IS_YX_OSV64_ISV2
uint weights_idx = weights_offset + local_id * SIMD * FILTER_LOAD_ITERS * FILTER_LOAD_BLOCK_SIZE * 2;
#else
Expand All @@ -1130,10 +1143,10 @@ inline void FUNC(fc_bf_tiled_kernel_dyn_quan)(

// DQ_DECOMPRESSION_SCALE_POST_OP SHOULD be enabled for dynamic quantize FC : scale is ACCUMULATOR_VAL_ONE
unroll_for(uint load_iter = 0; load_iter < FILTER_LOAD_ITERS; ++load_iter) {
#if COMPRESSED_WEIGHTS_INT4
#if COMPRESSED_WEIGHTS_SUB_BYTE
#if FILTER_LAYOUT_OS_IYX_OSV16
SLM_FILTER_PACKED_VEC wei_packed0 = BLOCK_READN(FILTER_TYPE, FILTER_ACTUAL_LOAD_BLOCK_SIZE, weights, weights_idx);
SLM_FILTER_PACKED_VEC wei_packed1 = BLOCK_READN(FILTER_TYPE, FILTER_ACTUAL_LOAD_BLOCK_SIZE, weights, (weights_idx + ((IFM_SIZE / 2) * 16)));
SLM_FILTER_PACKED_VEC wei_packed1 = BLOCK_READN(FILTER_TYPE, FILTER_ACTUAL_LOAD_BLOCK_SIZE, weights, (weights_idx + ((IFM_SIZE / FILTER_ELEMENTS_PER_BYTE) * 16)));
SLM_WEIGHT_UNPACKED_VEC dq_wei_unpacked;
// loaded weights 'wei_packed' of os_iyx_osv16 format have continuous values along TILE_K. So no need to transpose while unpacking
dq_wei_unpacked.s0123 = (UNPACK_INT4(DQ_TYPE, *((INT4_PACKED_TYPE_PRELOAD*)&wei_packed0)));
Expand Down Expand Up @@ -1491,7 +1504,7 @@ KERNEL(fc)(
__local ACCUMULATOR_TYPE wei_local_mem[TILE_IFM * SIMD * TILE_OFM * SIMD];
#endif
#endif
#if IS_DYNAMIC && COMPRESSED_WEIGHTS_INT4
#if IS_DYNAMIC && COMPRESSED_WEIGHTS_SUB_BYTE
const int batch_size = BATCH_SIZE;
if (batch_size == 1) {
FUNC_CALL(fc_bf_tiled_kernel_forced_tile_b1)(
Expand Down
21 changes: 21 additions & 0 deletions src/plugins/intel_gpu/src/kernel_selector/cl_kernels/include/batch_headers/fetch_weights.cl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -415,6 +415,16 @@ inline uint get_os_zyxi_osv16_index(uint o, uint i, uint z, uint y, uint x, uint
((o) / (sub_group_size))*(CAT(prefix, _OFM_PITCH)/2) \
)

#define GET_FILTER_OS_IYX_OSV_INDEX_INT2_PACKED(prefix, o, i, y, x, sub_group_size) \
CAT(prefix, _OFFSET) + \
((o) % (sub_group_size)) + \
(sub_group_size)*( \
(x)*CAT(prefix, _X_PITCH) + \
(y)*CAT(prefix, _Y_PITCH) + \
(i)*CAT(prefix, _IFM_PITCH) + \
((o) / (sub_group_size))*(CAT(prefix, _OFM_PITCH)/4) \
)

#define GET_FILTER_OS_IS_YX_OSV_ISV_INDEX_INT4_PACKED(prefix, o, i, y, x, sub_group_size) \
CAT(prefix, _OFFSET) + \
((o) % (sub_group_size)) + \
Expand All @@ -425,6 +435,17 @@ inline uint get_os_zyxi_osv16_index(uint o, uint i, uint z, uint y, uint x, uint
((o) / (sub_group_size))*(CAT(prefix, _OFM_PITCH)/2) \
)

// u2 os_is_yx_osv64_isv2: a 64-OFM block is a sequence of 32-byte k-pair lines (sub_group_size == 32).
#define GET_FILTER_OS_IS_YX_OSV_ISV_INDEX_INT2_PACKED(prefix, o, i, y, x, sub_group_size) \
CAT(prefix, _OFFSET) + \
((o) % (sub_group_size)) + \
(sub_group_size)*( \
(x)*CAT(prefix, _X_PITCH) + \
(y)*CAT(prefix, _Y_PITCH) + \
(i)*CAT(prefix, _IFM_PITCH) + \
((o) / (sub_group_size))*(CAT(prefix, _OFM_PITCH)/2) \
)

#define GET_FILTER_OS_IYX_OSV_ROTATE_180_INDEX(prefix, o, i, y, x, sub_group_size) \
CAT(prefix, _OFFSET) + \
((o) % (sub_group_size)) + \
Expand Down
8 changes: 4 additions & 4 deletions ...s/intel_gpu/src/kernel_selector/cl_kernels/include/fully_connected_gpu_bf_tiled_common.cl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,8 +59,8 @@ inline void (FUNC_NAME)(
uint input_offset = out_b * TILE_IN_B_PITCH + INPUT0_OFFSET;
#endif

#if COMPRESSED_WEIGHTS_INT4
uint weights_offset = out_f * (INPUT_ELEMENTS_COUNT / 2);
#if COMPRESSED_WEIGHTS_SUB_BYTE
uint weights_offset = out_f * (INPUT_ELEMENTS_COUNT / FILTER_ELEMENTS_PER_BYTE);
#else
uint weights_offset = out_f * INPUT_ELEMENTS_COUNT;
#endif
Expand Down Expand Up @@ -150,7 +150,7 @@ inline void (FUNC_NAME)(
ACCUMULATOR_VEC_TYPE acc_tmp[FORCED_TILE_B] = { };

unroll_for(uint ki = 0; ki < (TILE_IFM * SIMD) / TILE_K; ++ki) {
#if COMPRESSED_WEIGHTS_INT4
#if COMPRESSED_WEIGHTS_SUB_BYTE
FILTER_PACKED_VEC_TYPE wei_packed = FILTER_BLOCK_READ(weights, weights_offset);
wei = UNPACK_INT4(ACCUMULATOR_TYPE, *((INT4_PACKED_TYPE*)&wei_packed));
#else
Expand Down Expand Up @@ -290,7 +290,7 @@ inline void (FUNC_NAME)(
#undef LOAD_IN_0
input_offset += TILE_IFM * SIMD - TILE_IN_B_PITCH * FORCED_TILE_B;
unroll_for(uint ki = 0; ki < CEIL_DIV(LEFTOVER_IFM, TILE_K); ++ki) {
#if COMPRESSED_WEIGHTS_INT4
#if COMPRESSED_WEIGHTS_SUB_BYTE
FILTER_PACKED_VEC_TYPE wei_packed = FILTER_BLOCK_READ(weights, weights_offset);
wei = UNPACK_INT4(ACCUMULATOR_TYPE, *((INT4_PACKED_TYPE*)&wei_packed));
#else
Expand Down
Loading
Loading