Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 21 additions & 3 deletions src/models/qwen38_graph.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -787,6 +787,19 @@ bool Qwen38DflashSkipsInputSanitizer() {
return skip;
}

void SetQwen38DflashSanitizerPolicy(Config& drafter_config, bool skip_sanitizer) {
constexpr std::string_view kPolicy = "session.paged_attention.skip_input_sanitizer";
auto& entries = drafter_config.model.decoder.session_options.config_entries;
entries.erase(std::remove_if(entries.begin(), entries.end(), [&](const auto& entry) { return entry.first == kPolicy; }),
entries.end());
entries.emplace_back(std::string{kPolicy}, skip_sanitizer ? "1" : "0");
}

bool Qwen38DflashPadsDecodeSteps(int max_batch, int sliding_window, bool skip_sanitizer,
uint32_t runtime_api_version) noexcept {
return max_batch == 1 && sliding_window > 0 && skip_sanitizer && runtime_api_version >= 32;
}

void ValidateQwen38DflashFeatureLayout(size_t aux_width, ONNXTensorElementDataType aux_type) {
if (aux_width == kQwen38DflashFeatureWidth && aux_type == ONNX_TENSOR_ELEMENT_DATA_TYPE_BFLOAT16) return;
throw std::runtime_error("qwen38_graph: the DFlash drafter reads auxiliary hidden states of " + std::to_string(aux_width) +
Expand Down Expand Up @@ -930,6 +943,7 @@ void Qwen38GraphEngine::Initialize(bool is_session_recreation) {
else if (mtp_enabled_ && !fold_)
BuildHead();
if (q.prefill.enabled) BuildFastPrefill();
bool dflash_pads_decode_steps = false;
if (!model_.config_->model.dflash2.filename.empty()) {
if (model_.config_->model.dflash2.is_dspark)
throw std::runtime_error("qwen38_flash_graph does not support model.dspark");
Expand All @@ -939,13 +953,13 @@ void Qwen38GraphEngine::Initialize(bool is_session_recreation) {
throw std::runtime_error("DFlash experiment requires the target graph to expose aux_hidden_states");
if (model_.config_->model.dflash2.num_draft_tokens != verify_rows_ - 1)
throw std::runtime_error("DFlash draft count must match target prefix-state width");
const bool skip_sanitizer = Qwen38DflashSkipsInputSanitizer();
auto draft_config = CreateDflash2Config(*model_.config_);
draft_config->model.embedding.filename.clear();
SetQwen38DflashSanitizerPolicy(*draft_config, skip_sanitizer);
dflash_model_ = std::make_shared<Dflash2Model>(
std::move(draft_config), GetOrtEnv(), nullptr,
[this](OrtSessionOptions& opts) {
opts.AddConfigEntry("session.paged_attention.skip_input_sanitizer",
Qwen38DflashSkipsInputSanitizer() ? "1" : "0");
if (model_.pool_) model_.pool_->AddTo(opts, "dflash2");
if (!model_.embed_values_.empty()) {
opts.AddInitializer("model.embed_tokens.weight", *model_.embed_values_.front());
Expand All @@ -957,10 +971,12 @@ void Qwen38GraphEngine::Initialize(bool is_session_recreation) {
is_session_recreation);
const size_t page_size = Qwen38DflashCachePageSize(dflash_model_->session_info_,
dflash_model_->config_->model.dflash2.inputs.past_key_names);
dflash_pads_decode_steps = Qwen38DflashPadsDecodeSteps(max_batch_, model_.config_->model.dflash2.sliding_window,
skip_sanitizer, Ort::runtime_api_version);
dflash_ = std::make_unique<Dflash2Drafter>(
dflash_model_, page_size,
Dflash2Drafter::PoolBlocks(*model_.config_, page_size, static_cast<size_t>(max_batch_)),
static_cast<size_t>(max_batch_));
static_cast<size_t>(max_batch_), dflash_pads_decode_steps);
dflash_->SetPinnedBlockTableWidth(true);
ValidateQwen38DflashFeatureLayout(dflash_->AuxHiddenSize(), dflash_->AuxType());
// The slots are drafted one after the other without draining the stream in between.
Expand Down Expand Up @@ -1031,6 +1047,8 @@ void Qwen38GraphEngine::Initialize(bool is_session_recreation) {
Draft(d_dflash_prompt_.data(), r, pos, 1000, true);
pos += r;
}
// A drafter that pads its decode steps replays the one graph the first row width captured for every width.
if (dflash_pads_decode_steps) break;
}
SyncStream();
Reset();
Expand Down
16 changes: 15 additions & 1 deletion src/models/qwen38_graph.h
Original file line number Diff line number Diff line change
Expand Up @@ -286,7 +286,8 @@ inline constexpr int Qwen38WarmupRuns(bool session_captures_graph) noexcept {

// What the DFlash engine runs while its model is created. model.qwen38_graph.warmup=false opts out of all of it, as for the
// other paths: no forward, hence no CUDA graph captured and no startup cost, for the drafter or for the target. With
// warmup=true the drafter ingests a warm-up prompt and drafts every row width (`drafter`) and `target_tokens` dummy prompt
// warmup=true the drafter ingests a warm-up prompt and drafts every row width (`drafter`; a drafter that pads its decode steps,
// see Qwen38DflashPadsDecodeSteps, replays one graph for all of them and drafts the first only) and `target_tokens` dummy prompt
// tokens step through the S=verify_rows target graphs, enough runs of every (rows, ring bank) graph to capture it unless
// the target offloads experts (its warmup stays eager; see CreateQwen38OffloadWarmupRunOptions).
struct Qwen38DflashWarmupPlan {
Expand All @@ -301,6 +302,19 @@ Qwen38DflashWarmupPlan PlanQwen38DflashWarmup(bool warmup, int verify_rows, int
// models/env_utils.h.
bool Qwen38DflashSkipsInputSanitizer();

// Records `skip_sanitizer` as session.paged_attention.skip_input_sanitizer in the decoder session options of the DFlash
// drafter's projected config, replacing any value the package set: the drafter session is created from these entries, and
// Dflash2Drafter reads the policy back from them to decide whether it may pad a decode step to its fixed shape.
void SetQwen38DflashSanitizerPolicy(Config& drafter_config, bool skip_sanitizer);

// Whether the DFlash drafter of an engine with `max_batch` slots runs Dflash2Drafter's fixed single-request mode, which pads
// every decode step of 1..block_size committed rows into one captured graph. Padding needs a single slot, a windowed drafter
// (`sliding_window` > 0) and an ONNX Runtime that keeps the drafter's logical lengths: API 32 or later with the paged attention
// input sanitizer skipped. Otherwise a fixed drafter would run every step short of a full block eagerly, so the drafter keeps
// one captured graph per row count instead.
bool Qwen38DflashPadsDecodeSteps(int max_batch, int sliding_window, bool skip_sanitizer,
uint32_t runtime_api_version) noexcept;

// The rows of target features the engine hands the DFlash drafter (the auxiliary hidden states of the target, five layers of
// 2560 values) are this many bfloat16 values wide.
inline constexpr size_t kQwen38DflashFeatureWidth = 12800;
Expand Down
17 changes: 3 additions & 14 deletions src/qwen38_plane/qwen38_plane.cu
Original file line number Diff line number Diff line change
Expand Up @@ -693,6 +693,8 @@ static inline void host_gather_ple_rows_batch(StagedPleWorkspace& ws, int rows,

if (num_misses > 0) {
#ifdef _WIN32
// The gathered rows stay in the working set: they are clean file-backed pages that the memory manager trims under memory
// pressure, so aging them out with a VirtualUnlock call per row is not worth its cost on every step.
if (num_misses > 1) {
(void)PrefetchVirtualMemory(GetCurrentProcess(), (ULONG_PTR)num_misses, ranges, 0);
}
Expand All @@ -710,13 +712,6 @@ static inline void host_gather_ple_rows_batch(StagedPleWorkspace& ws, int rows,
ws.ngram_cache_keys[cache_slot][2] = d_c1s[r];
ws.ngram_cache_valid[cache_slot] = 1;
}
#ifdef _WIN32
if (ple_is_staged(HP.pad_)) {
for (int k = 0; k < num_misses; ++k) {
(void)VirtualUnlock(ranges[k].VirtualAddress, ranges[k].NumberOfBytes);
}
}
#endif
}
}

Expand Down Expand Up @@ -822,20 +817,14 @@ struct PleGatherPool {
}

#ifdef _WIN32
// The rows stay in the working set, as in host_gather_ple_rows_batch.
if (!ranges.empty()) {
(void)PrefetchVirtualMemory(GetCurrentProcess(), (ULONG_PTR)ranges.size(), ranges.data(), 0);
}
#endif
for (size_t k = 0; k < miss_srcs.size(); ++k) {
memcpy(miss_dsts[k], miss_srcs[k], (size_t)rb);
}
#ifdef _WIN32
if (ple_is_staged(HP.pad_)) {
for (const auto& r : ranges) {
(void)VirtualUnlock(r.VirtualAddress, r.NumberOfBytes);
}
}
#endif
}
}

Expand Down
28 changes: 28 additions & 0 deletions test/cpp/engine/qwen38_graph_model_tests.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -2307,6 +2307,34 @@ TEST(Qwen38DflashSanitizerTest, AValueThatIsNotABooleanIsRefusedNamingTheSwitch)
}
}

// Dflash2Drafter decides whether it may pad from the projected config, not from the session options, so the policy the
// session runs with has to be recorded there, replacing what the package set.
TEST(Qwen38DflashSanitizerTest, ThePolicyIsRecordedInTheDrafterConfig) {
constexpr const char* kPolicy = "session.paged_attention.skip_input_sanitizer";
for (const bool skip : {true, false}) {
Config config;
auto& entries = config.model.decoder.session_options.config_entries;
entries.emplace_back(kPolicy, skip ? "0" : "1");
entries.emplace_back("session.other", "x");
SetQwen38DflashSanitizerPolicy(config, skip);
const std::vector<Config::NamedString> expected{{"session.other", "x"}, {kPolicy, skip ? "1" : "0"}};
EXPECT_EQ(entries, expected) << skip;
}
Config config;
SetQwen38DflashSanitizerPolicy(config, true);
EXPECT_EQ(config.model.decoder.session_options.config_entries, (std::vector<Config::NamedString>{{kPolicy, "1"}}));
}

TEST(Qwen38DflashSanitizerTest, DecodeStepsArePaddedOnlyWhenTheRuntimeKeepsTheLogicalLengths) {
Comment thread
tianleiwu marked this conversation as resolved.
Resolved
EXPECT_TRUE(Qwen38DflashPadsDecodeSteps(1, 2048, true, 32));
EXPECT_TRUE(Qwen38DflashPadsDecodeSteps(1, 2048, true, 33));
// Otherwise every step short of a full block would run eagerly: the drafter keeps one graph per row count.
EXPECT_FALSE(Qwen38DflashPadsDecodeSteps(1, 2048, true, 31)); // the sanitizer extends the lengths to the padded rows
EXPECT_FALSE(Qwen38DflashPadsDecodeSteps(1, 2048, false, 33)); // ORT_PAGED_ATTENTION_SKIP_SANITIZER=0 keeps it
EXPECT_FALSE(Qwen38DflashPadsDecodeSteps(2, 2048, true, 33)); // fixed proposals serve a single request
EXPECT_FALSE(Qwen38DflashPadsDecodeSteps(1, 0, true, 33)); // fixed proposals need a windowed drafter
}

TEST(Qwen38PromptTokenTest, AcceptsEveryIdOfTheVocabulary) {
const std::vector<int32_t> tokens = {0, 1, 1000, 248319};
EXPECT_NO_THROW(ValidateQwen38PromptTokens(tokens, 248320));
Expand Down
66 changes: 66 additions & 0 deletions test/python/builder/_qwen38_tools_test_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -152,3 +152,69 @@ def prefill_chunk_problem(section):
if enabled:
return None if chunk <= section.get("kv_capacity", 0) else f"prefill.chunk {chunk} exceeds kv_capacity"
return None if chunk % 2 == 0 else f"prefill.chunk {chunk} is odd while prefill.enabled is false"


_NO_MEMORY_BOUND = 2**64 - 1


def _eligibilities_intersect(a, b):
"""``EligibilitiesIntersect`` of ``src/config.cpp``: whether one device could satisfy both eligibility objects."""

def lower(key):
return max(a.get(key, 0), b.get(key, 0))

def upper(key):
return min(a.get(key, _NO_MEMORY_BOUND), b.get(key, _NO_MEMORY_BOUND))

minimum_free = lower("minimum_free_device_memory_bytes")
maximum_total = upper("maximum_total_device_memory_bytes")
maximum_host = upper("maximum_effective_host_memory_bytes")
integrated = (a.get("is_integrated"), b.get("is_integrated"))
return (
lower("minimum_total_device_memory_bytes") <= maximum_total
and minimum_free <= upper("maximum_free_device_memory_bytes")
and minimum_free <= maximum_total
and lower("minimum_effective_host_memory_bytes") <= maximum_host
and lower("minimum_available_host_memory_bytes") <= maximum_host
and (None in integrated or integrated[0] == integrated[1])
)


def runtime_profiles_problem(profiles):
"""Returns why the engine's ``ValidateRuntimeProfiles`` (``src/config.cpp``) rejects ``runtime_profiles``, or None.

Every profile needs a unique id and ``minimum_total_device_memory_bytes``, no maximum may be below its minimum,
and no device may satisfy the eligibility of two profiles.
"""
ids = set()
for profile in profiles:
profile_id = profile.get("id")
if not profile_id:
return "runtime profile id must not be empty"
if profile_id in ids:
return f"duplicate runtime profile id: {profile_id}"
ids.add(profile_id)
eligibility = profile.get("eligibility", {})
if "minimum_total_device_memory_bytes" not in eligibility:
return f"runtime profile '{profile_id}' is missing minimum_total_device_memory_bytes"
for kind in ("total_device", "free_device", "effective_host"):
if eligibility.get(f"maximum_{kind}_memory_bytes", _NO_MEMORY_BOUND) < eligibility.get(
f"minimum_{kind}_memory_bytes", 0
):
return f"runtime profile '{profile_id}' has maximum_{kind}_memory_bytes below its minimum"
if eligibility.get("minimum_free_device_memory_bytes", 0) > eligibility.get(
"maximum_total_device_memory_bytes", _NO_MEMORY_BOUND
):
return f"runtime profile '{profile_id}' requires more free device memory than its maximum total"
if eligibility.get("minimum_available_host_memory_bytes", 0) > eligibility.get(
"maximum_effective_host_memory_bytes", _NO_MEMORY_BOUND
):
return (
f"runtime profile '{profile_id}' requires more available host memory "
"than its maximum effective host memory"
)
for index, first in enumerate(profiles):
for second in profiles[index + 1 :]:
if _eligibilities_intersect(first["eligibility"], second["eligibility"]):
return f"runtime profile eligibility ranges overlap: '{first['id']}' and '{second['id']}'"
return None
29 changes: 24 additions & 5 deletions test/python/builder/test_qwen38_package_builders.py
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@
from _qwen38_tools_test_utils import (
REPO_ROOT,
prefill_chunk_problem,
runtime_profiles_problem,
unknown_qwen38_graph_keys,
)
from onnx import TensorProto, helper, numpy_helper
Expand Down Expand Up @@ -483,15 +484,19 @@ def test_config_migration_and_rc5_session_options(builders):
assert cfg_int3["model"]["context_length"] == 262144
assert cfg_int3["search"]["max_length"] == 262144
assert cfg_int3["engine"] == {"dynamic_batching": {"max_batch_size": 1, "max_scheduled_tokens": 8192}}
assert len(cfg_int3["runtime_profiles"]) == 7
p_128k = cfg_int3["runtime_profiles"][-2]
p_256k = cfg_int3["runtime_profiles"][-1]
assert p_128k["id"] == "gpu-128k"
assert len(cfg_int3["runtime_profiles"]) == 9
assert runtime_profiles_problem(cfg_int3["runtime_profiles"]) is None
by_id = {p["id"]: p for p in cfg_int3["runtime_profiles"]}
p_128k = by_id["gpu-128k"]
p_256k = by_id["gpu-256k"]
assert p_128k["overlay"]["model"]["decoder"]["filename"] == "model.onnx"
assert p_128k["overlay"]["engine"]["dynamic_batching"]["num_blocks"] == 525
assert p_256k["id"] == "gpu-256k"
assert p_256k["overlay"]["model"]["decoder"]["filename"] == "model.onnx"
assert p_256k["overlay"]["engine"]["dynamic_batching"]["num_blocks"] == 1037
assert by_id["integrated-gpu-128k"]["eligibility"]["is_integrated"] is True
assert by_id["integrated-gpu-128k"]["overlay"]["engine"]["dynamic_batching"]["num_blocks"] == 525
assert by_id["integrated-gpu-256k"]["eligibility"]["is_integrated"] is True
assert by_id["integrated-gpu-256k"]["overlay"]["engine"]["dynamic_batching"]["num_blocks"] == 1037

# Per-model VRAM thresholds scale with QMoE size (2b375 < top08 < top16 < all_int4)
v_2b375 = pkg.estimate_peak_vram_and_thresholds(2, 14, 32)
Expand Down Expand Up @@ -522,6 +527,20 @@ def test_config_migration_and_rc5_session_options(builders):
assert cfg_int4_only["model"]["eos_token_id"] == [248319]


@pytest.mark.parametrize("x_int4, y_int3, z_int2", [(2, 14, 32), (8, 40, 0), (16, 32, 0), (48, 0, 0)])
def test_default_runtime_profiles_pass_engine_validation(builders, x_int4, y_int3, z_int2):
profiles = builders.pkg.default_runtime_profiles("model.onnx", x_int4=x_int4, y_int3=y_int3, z_int2=z_int2)
assert runtime_profiles_problem(profiles) is None
by_id = {p["id"]: p for p in profiles}
for tier in ("128k", "256k"):
discrete, integrated = by_id[f"gpu-{tier}"], by_id[f"integrated-gpu-{tier}"]
# A unified-memory tier is the discrete tier gated on free instead of total device memory.
assert integrated["overlay"] == discrete["overlay"]
minimum = discrete["eligibility"]["minimum_total_device_memory_bytes"]
assert integrated["eligibility"]["minimum_free_device_memory_bytes"] == minimum
assert integrated["eligibility"]["minimum_total_device_memory_bytes"] == minimum


def test_procedural_graph_builders_emit_complete_onnx_models_without_templates(builders):
gb = builders.gb

Expand Down
50 changes: 50 additions & 0 deletions test/python/builder/test_qwen38_qualification_tools.py
Original file line number Diff line number Diff line change
Expand Up @@ -1128,6 +1128,56 @@ def test_windows_loading_resolves_relative_dll_directories(tools, tmp_path, monk
assert registrations == [str(provider_path)]


def test_windows_loading_clears_posix_only_plane_so_and_ignores_duplicate_ep_registration(tools, tmp_path, monkeypatch):
util = tools.util
package = tmp_path / "package"
(package / "provider").mkdir(parents=True)
(package / "provider/libqwen38_plane.so").write_bytes(b"\x7fELF")
provider_path = package / "provider/onnxruntime_providers_cuda.dll"
provider_path.touch()
(package / "genai_config.json").write_text(
json.dumps(
{
"model": {
"qwen38_graph": {
"fold": {"enabled": False},
"mtp": {"enabled": False},
"plane_library": "provider/libqwen38_plane.so",
}
}
}
)
)
overlays = []
call_count = 0

def register_ep(name, path):
nonlocal call_count
call_count += 1
if call_count > 1:
raise RuntimeError("EP library is already registered under CUDAExecutionProvider")

monkeypatch.setattr(util, "os", types.SimpleNamespace(name="nt", environ={}, add_dll_directory=lambda path: path))
monkeypatch.setattr(util.og, "register_execution_provider_library", register_ep, raising=False)
monkeypatch.setattr(
util.og,
"Config",
lambda path: types.SimpleNamespace(overlay=lambda raw: overlays.append(json.loads(raw))),
raising=False,
)
monkeypatch.setattr(util.og, "Model", lambda config: config, raising=False)

util.load_model(package)
assert overlays == [{"model": {"qwen38_graph": {"plane_library": ""}}}]

# A companion Windows .dll suppresses the built-in fallback, and duplicate EP registration is tolerated.
overlays.clear()
(package / "provider/qwen38_plane.dll").write_bytes(b"MZ")
util.load_model(package)
assert call_count == 2
assert overlays == []


def test_host_budget_overrides_package_and_overlay_runtime_profiles(tools, tmp_path, monkeypatch):
util = tools.util
entry = util.HOST_EXPERT_BYTES_ENTRY
Expand Down
Loading
Loading