Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 12 additions & 0 deletions docs/contrib_ops/cuda/dynamic_sparse_attention.md
Original file line number Diff line number Diff line change
Expand Up @@ -82,6 +82,18 @@ and 8 otherwise.
| `ORT_DSA_DECODE_INT8_TC_TILES_PER_SPLIT` | `0` | `ORT_DSA_DECODE_INT8_TC=1` and the conditions above | `0` | Automatic: the smallest power-of-two number of 64-candidate tiles per split for which all `ceil(max_selected / 64) * rows * kv_heads` tiles fit one wave of resident CTAs (`rows = batch_size * sequence_length`; resident CTAs = SM count times the CTAs per SM that the kernel's shared-memory and register footprint allow, queried with `cudaOccupancyMaxActiveBlocksPerMultiprocessor`: 396 on an H200). The split count is `ceil(tiles / tiles_per_split)`, at most 32 and never more than the grouped decode workspace was sized for. |
| | | | `N > 0` | A requested `N` tiles per split instead of the automatic choice. The 32-split cap, the workspace size and the even distribution of the tiles over the splits still apply, so the number of tiles of a split can exceed `N` (8192 candidates are 128 tiles: `N = 1` asks for 128 splits and runs 32 splits of 4 tiles). Used by the tests to force many or few splits. |

The decode and prefill switches also cover an FP16 main cache with BF16 queries.
`ORT_DSA_PREFILL_KERNEL=1` runs the prefill with `DynamicSparseAttentionFp16TcKernel<false, 64>`
([dynamic_sparse_attention_prefill.cuh](../../../onnxruntime/contrib_ops/cuda/bert/dynamic_sparse_attention_prefill.cuh))
and `ORT_DSA_DECODE_INT8_TC=1` runs the grouped decode with `DynamicSparseAttentionFp16TcKernel<true, kCandTile>`; the
switch name keeps the INT8 spelling. The kernel gathers the FP16 K/V rows with `cp.async` into a two-buffer pipeline and
uses FP16 `mma` with FP32 accumulation. The decode kernel is split over the candidate tiles with the partial layout of
the INT8 kernel and the same merge kernel, and splits each probability tile into an FP16 high part plus an FP16 residual.
`kCandTile` is 64, or 32 on GPUs with about 100 KB of shared memory per SM (SM 8.6, 8.9, 12.x), where two CTAs then stay
resident. The row sum is kept per lane and reduced at the end, so the rescaling of the output accumulator after a new
maximum depends on the running maximum only. With `ORT_DSA_DECODE_INT8_TC=0` the decode uses the CUDA-core grouped
kernel and with `ORT_DSA_PREFILL_KERNEL=0` the generic prefill kernels.

The decode time is not flat in the tiles per split: with too few splits the CTAs are long and SMs stay idle, with too
many the grid spills into a second, partly filled wave. On an H200, 8 rows and 2048 candidates (16 pairs of row and KV
head with 32 tiles each), kernel plus merge,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -525,8 +525,8 @@ static const char* const kOrtSessionOptionsCudaDsaPrefillInt8Tc = "ep.cuda.dsa_p
/// "0" routes the BF16 prefill through the generic kernels. Default "1". Overrides ORT_DSA_PREFILL_KERNEL.
static const char* const kOrtSessionOptionsCudaDsaPrefillKernel = "ep.cuda.dsa_prefill_kernel";

/// "1" makes the INT8-cache BF16 decode (up to 8 query rows) use the split-K tensor-core kernel instead of the
/// CUDA-core grouped kernel (the tensor-core kernel accumulates in a different order and does not reproduce the
/// "1" makes the INT8-cache and FP16-cache BF16 decode (up to 8 query rows) use the split-K tensor-core kernel instead of
/// the CUDA-core grouped kernel (the tensor-core kernel accumulates in a different order and does not reproduce the
/// CUDA-core results bit for bit; "0" restores the CUDA-core kernel). Default "1". Overrides ORT_DSA_DECODE_INT8_TC.
static const char* const kOrtSessionOptionsCudaDsaDecodeInt8Tc = "ep.cuda.dsa_decode_int8_tc";

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -313,8 +313,7 @@ Status DynamicSparseAttention<T>::ComputeInternal(OpKernelContext* context) cons
const bool initialize_key_cache = data.past_key != data.present_key;
const bool initialize_value_cache = data.past_value != data.present_value;
return LaunchDynamicSparseAttention<CudaT>(
stream, parameters, data, initialize_key_cache, initialize_value_cache,
GetDeviceProp().maxThreadsPerBlock, GetDeviceProp().sharedMemPerBlock);
stream, parameters, data, initialize_key_cache, initialize_value_cache, GetDeviceProp());
}

template class DynamicSparseAttention<float>;
Expand Down
170 changes: 127 additions & 43 deletions onnxruntime/contrib_ops/cuda/bert/dynamic_sparse_attention_impl.cu

Large diffs are not rendered by default.

Original file line number Diff line number Diff line change
Expand Up @@ -96,8 +96,7 @@ Status LaunchDynamicSparseAttention(
const DynamicSparseAttentionData<T>& data,
bool initialize_key_cache,
bool initialize_value_cache,
int max_threads_per_block,
size_t max_shared_memory_per_block);
const cudaDeviceProp& device_prop);

} // namespace cuda
} // namespace contrib
Expand Down
479 changes: 479 additions & 0 deletions onnxruntime/contrib_ops/cuda/bert/dynamic_sparse_attention_prefill.cuh

Large diffs are not rendered by default.

140 changes: 98 additions & 42 deletions onnxruntime/contrib_ops/cuda/bert/gated_delta_net_impl.cu
Original file line number Diff line number Diff line change
Expand Up @@ -313,52 +313,108 @@ __device__ __forceinline__ void RunChunkedStep(
lane, tid);

if constexpr (kExactFp32StateAndU) {
SmemGemmFp32State<BT, kDVB, DK, false>(s.t_f, L::kVf, s.k_h, L::kKh, s.s_f, L::kVf, s.s_h,
L::kVh, warp_id, lane, tid);
__syncthreads();
for (int idx = tid; idx < BT * kDVB; idx += kChunkedThreads) {
const int t = idx / kDVB, c = idx % kDVB;
const float vv = ElemToFloat(s.v_h[t * L::kVh + c]);
const float r = needs_retrieval ? vv - __expf(s.gc[t]) * s.t_f[t * L::kVf + c] : vv;
s.t_f[t * L::kVf + c] = s.beta[t] * r;
}
__syncthreads();
float* u_f = reinterpret_cast<float*>(s.v_h);
SmemGemmFp32State<BT, kDVB, BT, false>(u_f, L::kVf, s.ti_h, L::kMh, s.t_f, L::kVf, s.u_h,
L::kVh, warp_id, lane, tid);
__syncthreads();
if constexpr (std::is_same_v<MT, __nv_bfloat16>) {
SmemGemm<BT, BT, DK, false, true, false>(s.t_f, L::kVf, s.q_h, L::kKh, s.k_h, L::kKh, warp_id,
lane);
__syncthreads();
for (int idx = tid; idx < BT * BT; idx += kChunkedThreads) {
const int t = idx / BT, sc = idx % BT;
s.m_h[t * L::kMh + sc] = FloatToElem<MT>(
(sc <= t && t < len) ? s.t_f[t * L::kVf + sc] * __expf(s.gc[t] - s.gc[sc]) : 0.0f);
}
for (int pair_idx = tid; pair_idx < (BT * DK) / 2; pair_idx += kChunkedThreads) {
const int t = pair_idx / (DK / 2), d = (pair_idx % (DK / 2)) * 2;
const float sq = __expf(s.gc[t]);
const uint32_t qh2 = *reinterpret_cast<const uint32_t*>(s.q_h + t * L::kKh + d);
const MT* qh_e = reinterpret_cast<const MT*>(&qh2);
*reinterpret_cast<uint32_t*>(s.q_h + t * L::kKh + d) =
PackHalf2(ScaleElem(qh_e[0], sq), ScaleElem(qh_e[1], sq));
}
__syncthreads();

SmemGemm<BT, BT, DK, false, true, false>(s.t_f, L::kVf, s.q_h, L::kKh, s.k_h, L::kKh, warp_id,
lane);
__syncthreads();
for (int idx = tid; idx < BT * BT; idx += kChunkedThreads) {
const int t = idx / BT, sc = idx % BT;
s.m_h[t * L::kMh + sc] = FloatToElem<MT>(
(sc <= t && t < len) ? s.t_f[t * L::kVf + sc] * __expf(s.gc[t] - s.gc[sc]) : 0.0f);
}
for (int idx = tid; idx < BT * DK; idx += kChunkedThreads) {
const int t = idx / DK, d = idx % DK;
s.k_h[t * L::kKh + d] = ScaleElem(s.k_h[t * L::kKh + d], __expf(g_total - s.gc[t]));
s.q_h[t * L::kKh + d] = ScaleElem(s.q_h[t * L::kKh + d], __expf(s.gc[t]));
}
__syncthreads();
float* r_f = reinterpret_cast<float*>(s.v_h);
SmemGemmFp32StateDualS0Bf16<BT, kDVB, DK>(
r_f, L::kVf, s.t_f, L::kVf, s.k_h, L::kKh, s.q_h, L::kKh,
s.s_f, L::kVf, s.s_h, L::kVh, s.v_h, L::kVh, s.gc, s.beta,
needs_retrieval, warp_id, lane, tid);
__syncthreads();

SmemGemmFp32State<BT, kDVB, BT, false>(s.t_f, L::kVf, s.m_h, L::kMh, u_f, L::kVf, s.u_h,
L::kVh, warp_id, lane, tid);
__syncthreads();
SmemGemmFp32State<BT, kDVB, DK, true>(s.t_f, L::kVf, s.q_h, L::kKh, s.s_f, L::kVf, s.s_h,
L::kVh, warp_id, lane, tid);
__syncthreads();
StoreChunkOutput<T, DK, DV, BT>(pack, p, s.t_f, tok_base, chunk0, len, hv, v0, v_stride, tid);
const float decay_all = __expf(g_total);
for (int pair_idx = tid; pair_idx < (BT * DK) / 2; pair_idx += kChunkedThreads) {
const int t = pair_idx / (DK / 2), d = (pair_idx % (DK / 2)) * 2;
const float sk = __expf(g_total - s.gc[t]);
const uint32_t kh2 = *reinterpret_cast<const uint32_t*>(s.k_h + t * L::kKh + d);
const MT* kh_e = reinterpret_cast<const MT*>(&kh2);
*reinterpret_cast<uint32_t*>(s.k_h + t * L::kKh + d) =
PackHalf2(ScaleElem(kh_e[0], sk), ScaleElem(kh_e[1], sk));
}
for (int pair_idx = tid; pair_idx < (DK * kDVB) / 2; pair_idx += kChunkedThreads) {
const int r = pair_idx / (kDVB / 2), c = (pair_idx % (kDVB / 2)) * 2;
const float2 sf2 = *reinterpret_cast<const float2*>(s.s_f + r * L::kVf + c);
*reinterpret_cast<float2*>(s.s_f + r * L::kVf + c) =
make_float2(sf2.x * decay_all, sf2.y * decay_all);
}
__syncthreads();

const float decay_all = __expf(g_total);
for (int idx = tid; idx < DK * kDVB; idx += kChunkedThreads) {
s.s_f[(idx / kDVB) * L::kVf + idx % kDVB] *= decay_all;
float* u_f = reinterpret_cast<float*>(s.s_h);
SmemGemmFp32State<BT, kDVB, BT, false>(u_f, L::kVf, s.ti_h, L::kMh, r_f, L::kVf, s.u_h,
L::kVh, warp_id, lane, tid);
__syncthreads();

SmemGemmFp32StateDualUBf16<BT, DK, kDVB>(
s.t_f, L::kVf, s.s_f, L::kVf, s.m_h, L::kMh, s.k_h, L::kKh,
u_f, L::kVf, s.u_h, L::kVh, warp_id, lane, tid);
__syncthreads();
StoreChunkOutput<T, DK, DV, BT>(pack, p, s.t_f, tok_base, chunk0, len, hv, v0, v_stride, tid);
__syncthreads();
} else {
SmemGemmFp32State<BT, kDVB, DK, false>(s.t_f, L::kVf, s.k_h, L::kKh, s.s_f, L::kVf, s.s_h,
L::kVh, warp_id, lane, tid);
__syncthreads();
for (int idx = tid; idx < BT * kDVB; idx += kChunkedThreads) {
const int t = idx / kDVB, c = idx % kDVB;
const float vv = ElemToFloat(s.v_h[t * L::kVh + c]);
const float r = needs_retrieval ? vv - __expf(s.gc[t]) * s.t_f[t * L::kVf + c] : vv;
s.t_f[t * L::kVf + c] = s.beta[t] * r;
}
__syncthreads();
float* u_f = reinterpret_cast<float*>(s.v_h);
SmemGemmFp32State<BT, kDVB, BT, false>(u_f, L::kVf, s.ti_h, L::kMh, s.t_f, L::kVf, s.u_h,
L::kVh, warp_id, lane, tid);
__syncthreads();

SmemGemm<BT, BT, DK, false, true, false>(s.t_f, L::kVf, s.q_h, L::kKh, s.k_h, L::kKh, warp_id,
lane);
__syncthreads();
for (int idx = tid; idx < BT * BT; idx += kChunkedThreads) {
const int t = idx / BT, sc = idx % BT;
s.m_h[t * L::kMh + sc] = FloatToElem<MT>(
(sc <= t && t < len) ? s.t_f[t * L::kVf + sc] * __expf(s.gc[t] - s.gc[sc]) : 0.0f);
}
for (int idx = tid; idx < BT * DK; idx += kChunkedThreads) {
const int t = idx / DK, d = idx % DK;
s.k_h[t * L::kKh + d] = ScaleElem(s.k_h[t * L::kKh + d], __expf(g_total - s.gc[t]));
s.q_h[t * L::kKh + d] = ScaleElem(s.q_h[t * L::kKh + d], __expf(s.gc[t]));
}
__syncthreads();

SmemGemmFp32State<BT, kDVB, BT, false>(s.t_f, L::kVf, s.m_h, L::kMh, u_f, L::kVf, s.u_h,
L::kVh, warp_id, lane, tid);
__syncthreads();
SmemGemmFp32State<BT, kDVB, DK, true>(s.t_f, L::kVf, s.q_h, L::kKh, s.s_f, L::kVf, s.s_h,
L::kVh, warp_id, lane, tid);
__syncthreads();
StoreChunkOutput<T, DK, DV, BT>(pack, p, s.t_f, tok_base, chunk0, len, hv, v0, v_stride, tid);

const float decay_all = __expf(g_total);
for (int idx = tid; idx < DK * kDVB; idx += kChunkedThreads) {
s.s_f[(idx / kDVB) * L::kVf + idx % kDVB] *= decay_all;
}
__syncthreads();
SmemGemmFp32State<DK, kDVB, BT, true, kChunkedWarps, MT, /*TA=*/true>(
s.s_f, L::kVf, s.k_h, L::kKh, u_f, L::kVf, s.u_h, L::kVh, warp_id, lane, tid);
__syncthreads();
}
__syncthreads();
SmemGemmFp32State<DK, kDVB, BT, true, kChunkedWarps, MT, /*TA=*/true>(
s.s_f, L::kVf, s.k_h, L::kKh, u_f, L::kVf, s.u_h, L::kVh, warp_id, lane, tid);
__syncthreads();
} else {
SmemGemm<BT, kDVB, DK, false, false, false>(s.t_f, L::kVf, s.k_h, L::kKh, s.s_h, L::kVh,
warp_id, lane);
Expand Down
Loading
Loading