From fc4c4576988a2135106269b2071010cb66e35080 Mon Sep 17 00:00:00 2001 From: Tianlei Wu Date: Fri, 13 Nov 2026 09:43:50 -0800 Subject: [PATCH 1/2] [Qwen38] Enable DFlash2 fixed_single_request fast path, drop staged PLE VirtualUnlock, and add integrated GPU runtime profiles - Pass fixed_single_request = (max_batch_ == 1) when constructing Dflash2Drafter in Qwen38GraphEngine::Initialize and break after r == 1 in single-slot decode warmup, consolidating 8 per-row-count decode CUDA graphs into one fixed 8-row graph (-37.1% load time, -47% per-step DFlash2 input prep + launch gap). - Remove per-miss VirtualUnlock syscall loops in host_gather_ple_rows_batch and PleGatherPool::GatherSpans on Windows staged file-mapped PLE shards, eliminating up to 98,304 VirtualUnlock calls per 8K prefill and 96 per decode step (-36.9% decode PLE host gather gap). - Add integrated-GPU (`is_integrated: true`) runtime_profiles gated on `minimum_free_device_memory_bytes` in `build_package.py` and `build_package_gguf.py` so unified-memory GPUs (e.g. NVIDIA RTX Spark / DGX Spark) automatically select 32K/64K/128K/256K (`num_blocks = 1037`) tiers. - Clear POSIX-only `.so` `plane_library` entries automatically on Windows in `qwen38_engine_util.py:load_model` when no companion `.dll` exists and tolerate duplicate EP registration in the same Python process. - Add unit tests in `test_build_package_gguf.py`, `test_qwen38_package_builders.py`, and `test_qwen38_qualification_tools.py`. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/models/qwen38_graph.cpp | 4 +- src/qwen38_plane/qwen38_plane.cu | 14 ------ .../builder/test_qwen38_package_builders.py | 13 +++-- .../test_qwen38_qualification_tools.py | 47 ++++++++++++++++++ test/python/test_build_package_gguf.py | 43 +++++++++++++--- .../python/qwen38_flash_next/build_package.py | 49 +++++++++++++++++++ tools/python/qwen38_flash_next/package_io.py | 36 ++++++++++++++ .../qwen38_flash_next/qwen38_engine_util.py | 36 +++++++++++++- 8 files changed, 213 insertions(+), 29 deletions(-) diff --git a/src/models/qwen38_graph.cpp b/src/models/qwen38_graph.cpp index 31fbd21b95..084107d3c0 100644 --- a/src/models/qwen38_graph.cpp +++ b/src/models/qwen38_graph.cpp @@ -957,10 +957,11 @@ void Qwen38GraphEngine::Initialize(bool is_session_recreation) { is_session_recreation); const size_t page_size = Qwen38DflashCachePageSize(dflash_model_->session_info_, dflash_model_->config_->model.dflash2.inputs.past_key_names); + const bool fixed_single_request = (max_batch_ == 1); dflash_ = std::make_unique( dflash_model_, page_size, Dflash2Drafter::PoolBlocks(*model_.config_, page_size, static_cast(max_batch_)), - static_cast(max_batch_)); + static_cast(max_batch_), fixed_single_request); dflash_->SetPinnedBlockTableWidth(true); ValidateQwen38DflashFeatureLayout(dflash_->AuxHiddenSize(), dflash_->AuxType()); // The slots are drafted one after the other without draining the stream in between. @@ -1031,6 +1032,7 @@ void Qwen38GraphEngine::Initialize(bool is_session_recreation) { Draft(d_dflash_prompt_.data(), r, pos, 1000, true); pos += r; } + if (max_batch_ == 1) break; // fixed_single_request_ pads all 1..verify_rows_ decode steps into one fixed 8-row graph } SyncStream(); Reset(); diff --git a/src/qwen38_plane/qwen38_plane.cu b/src/qwen38_plane/qwen38_plane.cu index 756353e1f0..d2ed61c939 100644 --- a/src/qwen38_plane/qwen38_plane.cu +++ b/src/qwen38_plane/qwen38_plane.cu @@ -710,13 +710,6 @@ static inline void host_gather_ple_rows_batch(StagedPleWorkspace& ws, int rows, ws.ngram_cache_keys[cache_slot][2] = d_c1s[r]; ws.ngram_cache_valid[cache_slot] = 1; } -#ifdef _WIN32 - if (ple_is_staged(HP.pad_)) { - for (int k = 0; k < num_misses; ++k) { - (void)VirtualUnlock(ranges[k].VirtualAddress, ranges[k].NumberOfBytes); - } - } -#endif } } @@ -829,13 +822,6 @@ struct PleGatherPool { for (size_t k = 0; k < miss_srcs.size(); ++k) { memcpy(miss_dsts[k], miss_srcs[k], (size_t)rb); } -#ifdef _WIN32 - if (ple_is_staged(HP.pad_)) { - for (const auto& r : ranges) { - (void)VirtualUnlock(r.VirtualAddress, r.NumberOfBytes); - } - } -#endif } } diff --git a/test/python/builder/test_qwen38_package_builders.py b/test/python/builder/test_qwen38_package_builders.py index 73157471f1..01236e39c1 100644 --- a/test/python/builder/test_qwen38_package_builders.py +++ b/test/python/builder/test_qwen38_package_builders.py @@ -483,15 +483,18 @@ def test_config_migration_and_rc5_session_options(builders): assert cfg_int3["model"]["context_length"] == 262144 assert cfg_int3["search"]["max_length"] == 262144 assert cfg_int3["engine"] == {"dynamic_batching": {"max_batch_size": 1, "max_scheduled_tokens": 8192}} - assert len(cfg_int3["runtime_profiles"]) == 7 - p_128k = cfg_int3["runtime_profiles"][-2] - p_256k = cfg_int3["runtime_profiles"][-1] - assert p_128k["id"] == "gpu-128k" + assert len(cfg_int3["runtime_profiles"]) == 9 + by_id = {p["id"]: p for p in cfg_int3["runtime_profiles"]} + p_128k = by_id["gpu-128k"] + p_256k = by_id["gpu-256k"] assert p_128k["overlay"]["model"]["decoder"]["filename"] == "model.onnx" assert p_128k["overlay"]["engine"]["dynamic_batching"]["num_blocks"] == 525 - assert p_256k["id"] == "gpu-256k" assert p_256k["overlay"]["model"]["decoder"]["filename"] == "model.onnx" assert p_256k["overlay"]["engine"]["dynamic_batching"]["num_blocks"] == 1037 + assert by_id["integrated-gpu-128k"]["eligibility"]["is_integrated"] is True + assert by_id["integrated-gpu-128k"]["overlay"]["engine"]["dynamic_batching"]["num_blocks"] == 525 + assert by_id["integrated-gpu-256k"]["eligibility"]["is_integrated"] is True + assert by_id["integrated-gpu-256k"]["overlay"]["engine"]["dynamic_batching"]["num_blocks"] == 1037 # Per-model VRAM thresholds scale with QMoE size (2b375 < top08 < top16 < all_int4) v_2b375 = pkg.estimate_peak_vram_and_thresholds(2, 14, 32) diff --git a/test/python/builder/test_qwen38_qualification_tools.py b/test/python/builder/test_qwen38_qualification_tools.py index 64685b2ad2..88a2767ba1 100644 --- a/test/python/builder/test_qwen38_qualification_tools.py +++ b/test/python/builder/test_qwen38_qualification_tools.py @@ -1128,6 +1128,53 @@ def test_windows_loading_resolves_relative_dll_directories(tools, tmp_path, monk assert registrations == [str(provider_path)] +def test_windows_loading_clears_posix_only_plane_so_and_ignores_duplicate_ep_registration(tools, tmp_path, monkeypatch): + util = tools.util + package = tmp_path / "package" + (package / "provider").mkdir(parents=True) + (package / "provider/libqwen38_plane.so").write_bytes(b"\x7fELF") + provider_path = package / "provider/onnxruntime_providers_cuda.dll" + provider_path.touch() + (package / "genai_config.json").write_text( + json.dumps( + { + "model": { + "qwen38_graph": { + "fold": {"enabled": False}, + "mtp": {"enabled": False}, + "plane_library": "provider/libqwen38_plane.so", + } + } + } + ) + ) + overlays = [] + call_count = 0 + + def register_ep(name, path): + nonlocal call_count + call_count += 1 + if call_count > 1: + raise RuntimeError("EP library is already registered under CUDAExecutionProvider") + + monkeypatch.setattr(util, "os", types.SimpleNamespace(name="nt", environ={}, add_dll_directory=lambda path: path)) + monkeypatch.setattr(util.og, "register_execution_provider_library", register_ep, raising=False) + monkeypatch.setattr( + util.og, "Config", lambda path: types.SimpleNamespace(overlay=lambda raw: overlays.append(json.loads(raw))), raising=False + ) + monkeypatch.setattr(util.og, "Model", lambda config: config, raising=False) + + util.load_model(package) + assert overlays == [{"model": {"qwen38_graph": {"plane_library": ""}}}] + + # A companion Windows .dll suppresses the built-in fallback, and duplicate EP registration is tolerated. + overlays.clear() + (package / "provider/qwen38_plane.dll").write_bytes(b"MZ") + util.load_model(package) + assert call_count == 2 + assert overlays == [] + + def test_host_budget_overrides_package_and_overlay_runtime_profiles(tools, tmp_path, monkeypatch): util = tools.util entry = util.HOST_EXPERT_BYTES_ENTRY diff --git a/test/python/test_build_package_gguf.py b/test/python/test_build_package_gguf.py index 52ea14395d..1338984fc3 100644 --- a/test/python/test_build_package_gguf.py +++ b/test/python/test_build_package_gguf.py @@ -120,12 +120,30 @@ def test_gguf_runtime_profiles_and_block_counts(package_builder, weight_bytes, e cfg = package_builder.default_base_genai_config() package_builder.configure_gguf_genai_config(cfg, total_weight_bytes=weight_bytes) profiles = {p["id"]: p for p in cfg["runtime_profiles"]} - assert list(profiles) == ["gpu-32k", "gpu-64k", "gpu-128k", "gpu-256k"] + assert list(profiles) == [ + "gpu-32k", + "gpu-64k", + "gpu-128k", + "gpu-256k", + "integrated-gpu-32k", + "integrated-gpu-64k", + "integrated-gpu-128k", + "integrated-gpu-256k", + ] assert profiles["gpu-32k"]["overlay"]["engine"]["dynamic_batching"]["num_blocks"] == 142 assert profiles["gpu-32k"]["overlay"]["search"]["chunk_size"] == 4096 assert profiles["gpu-64k"]["overlay"]["engine"]["dynamic_batching"]["num_blocks"] == 270 assert profiles["gpu-128k"]["overlay"]["engine"]["dynamic_batching"]["num_blocks"] == 525 assert profiles["gpu-256k"]["overlay"]["engine"]["dynamic_batching"]["num_blocks"] == 1037 + assert profiles["integrated-gpu-32k"]["eligibility"]["is_integrated"] is True + assert profiles["integrated-gpu-256k"]["eligibility"]["is_integrated"] is True + assert profiles["integrated-gpu-256k"]["overlay"]["engine"]["dynamic_batching"]["num_blocks"] == 1037 + assert ( + profiles["integrated-gpu-32k"]["eligibility"]["minimum_free_device_memory_bytes"], + profiles["integrated-gpu-64k"]["eligibility"]["minimum_free_device_memory_bytes"], + profiles["integrated-gpu-128k"]["eligibility"]["minimum_free_device_memory_bytes"], + profiles["integrated-gpu-256k"]["eligibility"]["minimum_free_device_memory_bytes"], + ) == expected_mins assert ( profiles["gpu-32k"]["eligibility"]["minimum_total_device_memory_bytes"], profiles["gpu-64k"]["eligibility"]["minimum_total_device_memory_bytes"], @@ -1297,13 +1315,24 @@ def _assert_valid_exported_package(out_dir: Path, *, expected_ple_format: str = assert prefill_chunk_problem(qwen_sec) is None assert qwen_sec["plane_library"] == "" profiles = cfg["runtime_profiles"] - assert [p["id"] for p in profiles] == ["gpu-32k", "gpu-64k", "gpu-128k", "gpu-256k"] + assert [p["id"] for p in profiles] == [ + "gpu-32k", + "gpu-64k", + "gpu-128k", + "gpu-256k", + "integrated-gpu-32k", + "integrated-gpu-64k", + "integrated-gpu-128k", + "integrated-gpu-256k", + ] assert profiles[0]["overlay"]["search"]["chunk_size"] == 4096 - for prev_p, next_p in pairwise(profiles): - assert ( - prev_p["eligibility"]["maximum_total_device_memory_bytes"] + 1 - == next_p["eligibility"]["minimum_total_device_memory_bytes"] - ) + for tiers, memory_kind, integrated in ((profiles[:4], "total", False), (profiles[4:], "free", True)): + assert all(p["eligibility"]["is_integrated"] is integrated for p in tiers) + for prev_p, next_p in pairwise(tiers): + assert ( + prev_p["eligibility"][f"maximum_{memory_kind}_device_memory_bytes"] + 1 + == next_p["eligibility"][f"minimum_{memory_kind}_device_memory_bytes"] + ) return models diff --git a/tools/python/qwen38_flash_next/build_package.py b/tools/python/qwen38_flash_next/build_package.py index efc57cbd18..f323706634 100644 --- a/tools/python/qwen38_flash_next/build_package.py +++ b/tools/python/qwen38_flash_next/build_package.py @@ -1908,6 +1908,55 @@ def default_runtime_profiles( }, }, }, + { + "id": "integrated-gpu-128k", + "eligibility": { + "minimum_free_device_memory_bytes": min_128k, + "maximum_free_device_memory_bytes": min_256k - 1, + "is_integrated": True, + }, + "overlay": { + "model": { + "decoder": { + "filename": decoder_filename, + "session_options": { + "session.moe.offload.mode": "disabled", + }, + } + }, + "engine": { + "dynamic_batching": { + "num_blocks": NUM_BLOCKS_128K, + "max_batch_size": 1, + "max_scheduled_tokens": 8192 if min_256k <= 68_000_000_000 else 4096, + } + }, + }, + }, + { + "id": "integrated-gpu-256k", + "eligibility": { + "minimum_free_device_memory_bytes": min_256k, + "is_integrated": True, + }, + "overlay": { + "model": { + "decoder": { + "filename": decoder_filename, + "session_options": { + "session.moe.offload.mode": "disabled", + }, + } + }, + "engine": { + "dynamic_batching": { + "num_blocks": NUM_BLOCKS_256K, + "max_batch_size": 1, + "max_scheduled_tokens": 8192 if min_256k <= 68_000_000_000 else 4096, + } + }, + }, + }, ] diff --git a/tools/python/qwen38_flash_next/package_io.py b/tools/python/qwen38_flash_next/package_io.py index 1ec398db60..6140919105 100644 --- a/tools/python/qwen38_flash_next/package_io.py +++ b/tools/python/qwen38_flash_next/package_io.py @@ -927,4 +927,40 @@ def make_overlay(num_blocks: int, max_scheduled: int, chunk_size: int | None = N }, "overlay": make_overlay(GGUF_NUM_BLOCKS_256K, prefill_chunk_long), }, + # Unified-memory GPUs report host RAM as total device memory, so their tiers are gated on free device memory. + { + "id": "integrated-gpu-32k", + "eligibility": { + "minimum_free_device_memory_bytes": min_32k, + "maximum_free_device_memory_bytes": min_64k - 1, + "is_integrated": True, + }, + "overlay": make_overlay(GGUF_NUM_BLOCKS_32K, prefill_chunk_32k, chunk_size=prefill_chunk_32k), + }, + { + "id": "integrated-gpu-64k", + "eligibility": { + "minimum_free_device_memory_bytes": min_64k, + "maximum_free_device_memory_bytes": min_128k - 1, + "is_integrated": True, + }, + "overlay": make_overlay(GGUF_NUM_BLOCKS_64K, prefill_chunk_long), + }, + { + "id": "integrated-gpu-128k", + "eligibility": { + "minimum_free_device_memory_bytes": min_128k, + "maximum_free_device_memory_bytes": min_256k - 1, + "is_integrated": True, + }, + "overlay": make_overlay(GGUF_NUM_BLOCKS_128K, prefill_chunk_long), + }, + { + "id": "integrated-gpu-256k", + "eligibility": { + "minimum_free_device_memory_bytes": min_256k, + "is_integrated": True, + }, + "overlay": make_overlay(GGUF_NUM_BLOCKS_256K, prefill_chunk_long), + }, ] diff --git a/tools/python/qwen38_flash_next/qwen38_engine_util.py b/tools/python/qwen38_flash_next/qwen38_engine_util.py index 71b32018dc..b99e745025 100644 --- a/tools/python/qwen38_flash_next/qwen38_engine_util.py +++ b/tools/python/qwen38_flash_next/qwen38_engine_util.py @@ -205,6 +205,29 @@ def _link_package_asset(link_path: Path, target: Path) -> None: _winapi.CreateJunction(str(target), str(link_path)) else: os.link(target, link_path) +def _needs_windows_builtin_plane_overlay( + package: Path, package_config: dict, *overlays: dict | None, is_windows: bool | None = None +) -> bool: + """Return True on Windows when `plane_library` names an existing `.so` with no `.dll` alias and no overlay overrides it.""" + if not (os.name == "nt" if is_windows is None else is_windows): + return False + for doc in overlays: + if isinstance(doc, dict) and "plane_library" in doc.get("model", {}).get("qwen38_graph", {}): + return False + configured = package_config.get("model", {}).get("qwen38_graph", {}).get("plane_library", "") + if not configured or not configured.endswith(".so"): + return False + so_path = package / configured + if not so_path.exists(): + return False + if so_path.with_suffix(".dll").exists(): + return False + stem = so_path.stem + if stem.startswith("lib") and len(stem) > 3 and (so_path.parent / f"{stem[3:]}.dll").exists(): + return False + return True + + def load_model(package, provider=None, overlay=None): """Registers the plugin CUDA EP (`provider` path, default $QWEN_PROVIDER or /provider/libonnxruntime_providers_cuda.so) and loads the package.""" package = Path(package).resolve() @@ -231,7 +254,16 @@ def load_model(package, provider=None, overlay=None): prov_dir = Path(provider).parent if prov_dir.is_dir(): _WIN_DLL_HANDLES.append(os.add_dll_directory(str(prov_dir))) - og.register_execution_provider_library("CUDAExecutionProvider", provider) + try: + og.register_execution_provider_library("CUDAExecutionProvider", provider) + except RuntimeError as exc: + if "already registered" not in str(exc): + raise + win_plane_overlay = ( + {"model": {"qwen38_graph": {"plane_library": ""}}} + if _needs_windows_builtin_plane_overlay(package, package_config, env_overlay, overlay) + else None + ) directory = None try: config_package = package @@ -246,7 +278,7 @@ def load_model(package, provider=None, overlay=None): json.dumps(with_host_expert_budget(package_config, budget)), encoding="utf-8" ) config = og.Config(str(config_package)) - for document in (env_overlay, overlay): + for document in (win_plane_overlay, env_overlay, overlay): if document: effective = with_host_expert_budget(document, budget) if budget is not None else document config.overlay(json.dumps(effective)) From cf935e99371008d20ea00f29f7bf21f685a44501 Mon Sep 17 00:00:00 2001 From: Tianlei Wu Date: Fri, 13 Nov 2026 23:58:12 +0000 Subject: [PATCH 2/2] [Qwen38] Fix integrated runtime profile schema and gate the padded DFlash2 decode graph on padding support - Add minimum_total_device_memory_bytes to every integrated-gpu-* runtime profile in build_package.py and package_io.py. ValidateRuntimeProfiles rejects a profile without it, so every package built with the new tiers failed to load on all devices. Use max_scheduled_tokens 8192 as in the discrete tiers; the 4096 branch had no effect on single-slot packages. - Record the DFlash drafter's session.paged_attention.skip_input_sanitizer policy in its projected config, which Dflash2Drafter reads to enable padding. Enable fixed_single_request and the one-width decode warmup only when padding is supported (single slot, windowed drafter, ORT API >= 32, sanitizer skipped); otherwise keep one captured graph per decode row count instead of running 1-7-row steps eagerly. - Add runtime_profiles_problem, a Python mirror of ValidateRuntimeProfiles, to the qwen38 tool tests and assert it on the QMoE and GGUF profiles; add C++ tests for SetQwen38DflashSanitizerPolicy and Qwen38DflashPadsDecodeSteps. - Document the integrated tiers in the README, note why the staged PLE gather no longer ages rows out of the working set, and fix ruff formatting in qwen38_engine_util.py and test_qwen38_qualification_tools.py. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/models/qwen38_graph.cpp | 26 ++++++-- src/models/qwen38_graph.h | 16 ++++- src/qwen38_plane/qwen38_plane.cu | 3 + test/cpp/engine/qwen38_graph_model_tests.cpp | 28 ++++++++ .../builder/_qwen38_tools_test_utils.py | 66 +++++++++++++++++++ .../builder/test_qwen38_package_builders.py | 16 +++++ .../test_qwen38_qualification_tools.py | 5 +- test/python/test_build_package_gguf.py | 17 ++++- tools/python/qwen38_flash_next/README.md | 3 +- .../python/qwen38_flash_next/build_package.py | 8 ++- tools/python/qwen38_flash_next/package_io.py | 5 ++ .../qwen38_flash_next/qwen38_engine_util.py | 15 +++-- 12 files changed, 192 insertions(+), 16 deletions(-) diff --git a/src/models/qwen38_graph.cpp b/src/models/qwen38_graph.cpp index 084107d3c0..a756b494b0 100644 --- a/src/models/qwen38_graph.cpp +++ b/src/models/qwen38_graph.cpp @@ -787,6 +787,19 @@ bool Qwen38DflashSkipsInputSanitizer() { return skip; } +void SetQwen38DflashSanitizerPolicy(Config& drafter_config, bool skip_sanitizer) { + constexpr std::string_view kPolicy = "session.paged_attention.skip_input_sanitizer"; + auto& entries = drafter_config.model.decoder.session_options.config_entries; + entries.erase(std::remove_if(entries.begin(), entries.end(), [&](const auto& entry) { return entry.first == kPolicy; }), + entries.end()); + entries.emplace_back(std::string{kPolicy}, skip_sanitizer ? "1" : "0"); +} + +bool Qwen38DflashPadsDecodeSteps(int max_batch, int sliding_window, bool skip_sanitizer, + uint32_t runtime_api_version) noexcept { + return max_batch == 1 && sliding_window > 0 && skip_sanitizer && runtime_api_version >= 32; +} + void ValidateQwen38DflashFeatureLayout(size_t aux_width, ONNXTensorElementDataType aux_type) { if (aux_width == kQwen38DflashFeatureWidth && aux_type == ONNX_TENSOR_ELEMENT_DATA_TYPE_BFLOAT16) return; throw std::runtime_error("qwen38_graph: the DFlash drafter reads auxiliary hidden states of " + std::to_string(aux_width) + @@ -930,6 +943,7 @@ void Qwen38GraphEngine::Initialize(bool is_session_recreation) { else if (mtp_enabled_ && !fold_) BuildHead(); if (q.prefill.enabled) BuildFastPrefill(); + bool dflash_pads_decode_steps = false; if (!model_.config_->model.dflash2.filename.empty()) { if (model_.config_->model.dflash2.is_dspark) throw std::runtime_error("qwen38_flash_graph does not support model.dspark"); @@ -939,13 +953,13 @@ void Qwen38GraphEngine::Initialize(bool is_session_recreation) { throw std::runtime_error("DFlash experiment requires the target graph to expose aux_hidden_states"); if (model_.config_->model.dflash2.num_draft_tokens != verify_rows_ - 1) throw std::runtime_error("DFlash draft count must match target prefix-state width"); + const bool skip_sanitizer = Qwen38DflashSkipsInputSanitizer(); auto draft_config = CreateDflash2Config(*model_.config_); draft_config->model.embedding.filename.clear(); + SetQwen38DflashSanitizerPolicy(*draft_config, skip_sanitizer); dflash_model_ = std::make_shared( std::move(draft_config), GetOrtEnv(), nullptr, [this](OrtSessionOptions& opts) { - opts.AddConfigEntry("session.paged_attention.skip_input_sanitizer", - Qwen38DflashSkipsInputSanitizer() ? "1" : "0"); if (model_.pool_) model_.pool_->AddTo(opts, "dflash2"); if (!model_.embed_values_.empty()) { opts.AddInitializer("model.embed_tokens.weight", *model_.embed_values_.front()); @@ -957,11 +971,12 @@ void Qwen38GraphEngine::Initialize(bool is_session_recreation) { is_session_recreation); const size_t page_size = Qwen38DflashCachePageSize(dflash_model_->session_info_, dflash_model_->config_->model.dflash2.inputs.past_key_names); - const bool fixed_single_request = (max_batch_ == 1); + dflash_pads_decode_steps = Qwen38DflashPadsDecodeSteps(max_batch_, model_.config_->model.dflash2.sliding_window, + skip_sanitizer, Ort::runtime_api_version); dflash_ = std::make_unique( dflash_model_, page_size, Dflash2Drafter::PoolBlocks(*model_.config_, page_size, static_cast(max_batch_)), - static_cast(max_batch_), fixed_single_request); + static_cast(max_batch_), dflash_pads_decode_steps); dflash_->SetPinnedBlockTableWidth(true); ValidateQwen38DflashFeatureLayout(dflash_->AuxHiddenSize(), dflash_->AuxType()); // The slots are drafted one after the other without draining the stream in between. @@ -1032,7 +1047,8 @@ void Qwen38GraphEngine::Initialize(bool is_session_recreation) { Draft(d_dflash_prompt_.data(), r, pos, 1000, true); pos += r; } - if (max_batch_ == 1) break; // fixed_single_request_ pads all 1..verify_rows_ decode steps into one fixed 8-row graph + // A drafter that pads its decode steps replays the one graph the first row width captured for every width. + if (dflash_pads_decode_steps) break; } SyncStream(); Reset(); diff --git a/src/models/qwen38_graph.h b/src/models/qwen38_graph.h index fbd2878e74..7badf36203 100644 --- a/src/models/qwen38_graph.h +++ b/src/models/qwen38_graph.h @@ -286,7 +286,8 @@ inline constexpr int Qwen38WarmupRuns(bool session_captures_graph) noexcept { // What the DFlash engine runs while its model is created. model.qwen38_graph.warmup=false opts out of all of it, as for the // other paths: no forward, hence no CUDA graph captured and no startup cost, for the drafter or for the target. With -// warmup=true the drafter ingests a warm-up prompt and drafts every row width (`drafter`) and `target_tokens` dummy prompt +// warmup=true the drafter ingests a warm-up prompt and drafts every row width (`drafter`; a drafter that pads its decode steps, +// see Qwen38DflashPadsDecodeSteps, replays one graph for all of them and drafts the first only) and `target_tokens` dummy prompt // tokens step through the S=verify_rows target graphs, enough runs of every (rows, ring bank) graph to capture it unless // the target offloads experts (its warmup stays eager; see CreateQwen38OffloadWarmupRunOptions). struct Qwen38DflashWarmupPlan { @@ -301,6 +302,19 @@ Qwen38DflashWarmupPlan PlanQwen38DflashWarmup(bool warmup, int verify_rows, int // models/env_utils.h. bool Qwen38DflashSkipsInputSanitizer(); +// Records `skip_sanitizer` as session.paged_attention.skip_input_sanitizer in the decoder session options of the DFlash +// drafter's projected config, replacing any value the package set: the drafter session is created from these entries, and +// Dflash2Drafter reads the policy back from them to decide whether it may pad a decode step to its fixed shape. +void SetQwen38DflashSanitizerPolicy(Config& drafter_config, bool skip_sanitizer); + +// Whether the DFlash drafter of an engine with `max_batch` slots runs Dflash2Drafter's fixed single-request mode, which pads +// every decode step of 1..block_size committed rows into one captured graph. Padding needs a single slot, a windowed drafter +// (`sliding_window` > 0) and an ONNX Runtime that keeps the drafter's logical lengths: API 32 or later with the paged attention +// input sanitizer skipped. Otherwise a fixed drafter would run every step short of a full block eagerly, so the drafter keeps +// one captured graph per row count instead. +bool Qwen38DflashPadsDecodeSteps(int max_batch, int sliding_window, bool skip_sanitizer, + uint32_t runtime_api_version) noexcept; + // The rows of target features the engine hands the DFlash drafter (the auxiliary hidden states of the target, five layers of // 2560 values) are this many bfloat16 values wide. inline constexpr size_t kQwen38DflashFeatureWidth = 12800; diff --git a/src/qwen38_plane/qwen38_plane.cu b/src/qwen38_plane/qwen38_plane.cu index d2ed61c939..112212ebd7 100644 --- a/src/qwen38_plane/qwen38_plane.cu +++ b/src/qwen38_plane/qwen38_plane.cu @@ -693,6 +693,8 @@ static inline void host_gather_ple_rows_batch(StagedPleWorkspace& ws, int rows, if (num_misses > 0) { #ifdef _WIN32 + // The gathered rows stay in the working set: they are clean file-backed pages that the memory manager trims under memory + // pressure, so aging them out with a VirtualUnlock call per row is not worth its cost on every step. if (num_misses > 1) { (void)PrefetchVirtualMemory(GetCurrentProcess(), (ULONG_PTR)num_misses, ranges, 0); } @@ -815,6 +817,7 @@ struct PleGatherPool { } #ifdef _WIN32 + // The rows stay in the working set, as in host_gather_ple_rows_batch. if (!ranges.empty()) { (void)PrefetchVirtualMemory(GetCurrentProcess(), (ULONG_PTR)ranges.size(), ranges.data(), 0); } diff --git a/test/cpp/engine/qwen38_graph_model_tests.cpp b/test/cpp/engine/qwen38_graph_model_tests.cpp index 7ce563037d..a03bae3941 100644 --- a/test/cpp/engine/qwen38_graph_model_tests.cpp +++ b/test/cpp/engine/qwen38_graph_model_tests.cpp @@ -2307,6 +2307,34 @@ TEST(Qwen38DflashSanitizerTest, AValueThatIsNotABooleanIsRefusedNamingTheSwitch) } } +// Dflash2Drafter decides whether it may pad from the projected config, not from the session options, so the policy the +// session runs with has to be recorded there, replacing what the package set. +TEST(Qwen38DflashSanitizerTest, ThePolicyIsRecordedInTheDrafterConfig) { + constexpr const char* kPolicy = "session.paged_attention.skip_input_sanitizer"; + for (const bool skip : {true, false}) { + Config config; + auto& entries = config.model.decoder.session_options.config_entries; + entries.emplace_back(kPolicy, skip ? "0" : "1"); + entries.emplace_back("session.other", "x"); + SetQwen38DflashSanitizerPolicy(config, skip); + const std::vector expected{{"session.other", "x"}, {kPolicy, skip ? "1" : "0"}}; + EXPECT_EQ(entries, expected) << skip; + } + Config config; + SetQwen38DflashSanitizerPolicy(config, true); + EXPECT_EQ(config.model.decoder.session_options.config_entries, (std::vector{{kPolicy, "1"}})); +} + +TEST(Qwen38DflashSanitizerTest, DecodeStepsArePaddedOnlyWhenTheRuntimeKeepsTheLogicalLengths) { + EXPECT_TRUE(Qwen38DflashPadsDecodeSteps(1, 2048, true, 32)); + EXPECT_TRUE(Qwen38DflashPadsDecodeSteps(1, 2048, true, 33)); + // Otherwise every step short of a full block would run eagerly: the drafter keeps one graph per row count. + EXPECT_FALSE(Qwen38DflashPadsDecodeSteps(1, 2048, true, 31)); // the sanitizer extends the lengths to the padded rows + EXPECT_FALSE(Qwen38DflashPadsDecodeSteps(1, 2048, false, 33)); // ORT_PAGED_ATTENTION_SKIP_SANITIZER=0 keeps it + EXPECT_FALSE(Qwen38DflashPadsDecodeSteps(2, 2048, true, 33)); // fixed proposals serve a single request + EXPECT_FALSE(Qwen38DflashPadsDecodeSteps(1, 0, true, 33)); // fixed proposals need a windowed drafter +} + TEST(Qwen38PromptTokenTest, AcceptsEveryIdOfTheVocabulary) { const std::vector tokens = {0, 1, 1000, 248319}; EXPECT_NO_THROW(ValidateQwen38PromptTokens(tokens, 248320)); diff --git a/test/python/builder/_qwen38_tools_test_utils.py b/test/python/builder/_qwen38_tools_test_utils.py index 8d8e9916c6..980f3526c7 100644 --- a/test/python/builder/_qwen38_tools_test_utils.py +++ b/test/python/builder/_qwen38_tools_test_utils.py @@ -152,3 +152,69 @@ def prefill_chunk_problem(section): if enabled: return None if chunk <= section.get("kv_capacity", 0) else f"prefill.chunk {chunk} exceeds kv_capacity" return None if chunk % 2 == 0 else f"prefill.chunk {chunk} is odd while prefill.enabled is false" + + +_NO_MEMORY_BOUND = 2**64 - 1 + + +def _eligibilities_intersect(a, b): + """``EligibilitiesIntersect`` of ``src/config.cpp``: whether one device could satisfy both eligibility objects.""" + + def lower(key): + return max(a.get(key, 0), b.get(key, 0)) + + def upper(key): + return min(a.get(key, _NO_MEMORY_BOUND), b.get(key, _NO_MEMORY_BOUND)) + + minimum_free = lower("minimum_free_device_memory_bytes") + maximum_total = upper("maximum_total_device_memory_bytes") + maximum_host = upper("maximum_effective_host_memory_bytes") + integrated = (a.get("is_integrated"), b.get("is_integrated")) + return ( + lower("minimum_total_device_memory_bytes") <= maximum_total + and minimum_free <= upper("maximum_free_device_memory_bytes") + and minimum_free <= maximum_total + and lower("minimum_effective_host_memory_bytes") <= maximum_host + and lower("minimum_available_host_memory_bytes") <= maximum_host + and (None in integrated or integrated[0] == integrated[1]) + ) + + +def runtime_profiles_problem(profiles): + """Returns why the engine's ``ValidateRuntimeProfiles`` (``src/config.cpp``) rejects ``runtime_profiles``, or None. + + Every profile needs a unique id and ``minimum_total_device_memory_bytes``, no maximum may be below its minimum, + and no device may satisfy the eligibility of two profiles. + """ + ids = set() + for profile in profiles: + profile_id = profile.get("id") + if not profile_id: + return "runtime profile id must not be empty" + if profile_id in ids: + return f"duplicate runtime profile id: {profile_id}" + ids.add(profile_id) + eligibility = profile.get("eligibility", {}) + if "minimum_total_device_memory_bytes" not in eligibility: + return f"runtime profile '{profile_id}' is missing minimum_total_device_memory_bytes" + for kind in ("total_device", "free_device", "effective_host"): + if eligibility.get(f"maximum_{kind}_memory_bytes", _NO_MEMORY_BOUND) < eligibility.get( + f"minimum_{kind}_memory_bytes", 0 + ): + return f"runtime profile '{profile_id}' has maximum_{kind}_memory_bytes below its minimum" + if eligibility.get("minimum_free_device_memory_bytes", 0) > eligibility.get( + "maximum_total_device_memory_bytes", _NO_MEMORY_BOUND + ): + return f"runtime profile '{profile_id}' requires more free device memory than its maximum total" + if eligibility.get("minimum_available_host_memory_bytes", 0) > eligibility.get( + "maximum_effective_host_memory_bytes", _NO_MEMORY_BOUND + ): + return ( + f"runtime profile '{profile_id}' requires more available host memory " + "than its maximum effective host memory" + ) + for index, first in enumerate(profiles): + for second in profiles[index + 1 :]: + if _eligibilities_intersect(first["eligibility"], second["eligibility"]): + return f"runtime profile eligibility ranges overlap: '{first['id']}' and '{second['id']}'" + return None diff --git a/test/python/builder/test_qwen38_package_builders.py b/test/python/builder/test_qwen38_package_builders.py index 01236e39c1..c0c0c5b2a5 100644 --- a/test/python/builder/test_qwen38_package_builders.py +++ b/test/python/builder/test_qwen38_package_builders.py @@ -41,6 +41,7 @@ from _qwen38_tools_test_utils import ( REPO_ROOT, prefill_chunk_problem, + runtime_profiles_problem, unknown_qwen38_graph_keys, ) from onnx import TensorProto, helper, numpy_helper @@ -484,6 +485,7 @@ def test_config_migration_and_rc5_session_options(builders): assert cfg_int3["search"]["max_length"] == 262144 assert cfg_int3["engine"] == {"dynamic_batching": {"max_batch_size": 1, "max_scheduled_tokens": 8192}} assert len(cfg_int3["runtime_profiles"]) == 9 + assert runtime_profiles_problem(cfg_int3["runtime_profiles"]) is None by_id = {p["id"]: p for p in cfg_int3["runtime_profiles"]} p_128k = by_id["gpu-128k"] p_256k = by_id["gpu-256k"] @@ -525,6 +527,20 @@ def test_config_migration_and_rc5_session_options(builders): assert cfg_int4_only["model"]["eos_token_id"] == [248319] +@pytest.mark.parametrize("x_int4, y_int3, z_int2", [(2, 14, 32), (8, 40, 0), (16, 32, 0), (48, 0, 0)]) +def test_default_runtime_profiles_pass_engine_validation(builders, x_int4, y_int3, z_int2): + profiles = builders.pkg.default_runtime_profiles("model.onnx", x_int4=x_int4, y_int3=y_int3, z_int2=z_int2) + assert runtime_profiles_problem(profiles) is None + by_id = {p["id"]: p for p in profiles} + for tier in ("128k", "256k"): + discrete, integrated = by_id[f"gpu-{tier}"], by_id[f"integrated-gpu-{tier}"] + # A unified-memory tier is the discrete tier gated on free instead of total device memory. + assert integrated["overlay"] == discrete["overlay"] + minimum = discrete["eligibility"]["minimum_total_device_memory_bytes"] + assert integrated["eligibility"]["minimum_free_device_memory_bytes"] == minimum + assert integrated["eligibility"]["minimum_total_device_memory_bytes"] == minimum + + def test_procedural_graph_builders_emit_complete_onnx_models_without_templates(builders): gb = builders.gb diff --git a/test/python/builder/test_qwen38_qualification_tools.py b/test/python/builder/test_qwen38_qualification_tools.py index 88a2767ba1..da649cb32a 100644 --- a/test/python/builder/test_qwen38_qualification_tools.py +++ b/test/python/builder/test_qwen38_qualification_tools.py @@ -1160,7 +1160,10 @@ def register_ep(name, path): monkeypatch.setattr(util, "os", types.SimpleNamespace(name="nt", environ={}, add_dll_directory=lambda path: path)) monkeypatch.setattr(util.og, "register_execution_provider_library", register_ep, raising=False) monkeypatch.setattr( - util.og, "Config", lambda path: types.SimpleNamespace(overlay=lambda raw: overlays.append(json.loads(raw))), raising=False + util.og, + "Config", + lambda path: types.SimpleNamespace(overlay=lambda raw: overlays.append(json.loads(raw))), + raising=False, ) monkeypatch.setattr(util.og, "Model", lambda config: config, raising=False) diff --git a/test/python/test_build_package_gguf.py b/test/python/test_build_package_gguf.py index 1338984fc3..363844fcbf 100644 --- a/test/python/test_build_package_gguf.py +++ b/test/python/test_build_package_gguf.py @@ -4,6 +4,7 @@ from __future__ import annotations import argparse +import copy import errno import hashlib import importlib.util @@ -26,7 +27,11 @@ if str(BUILDER_TEST_DIR) not in sys.path: sys.path.insert(0, str(BUILDER_TEST_DIR)) -from _qwen38_tools_test_utils import prefill_chunk_problem, unknown_qwen38_graph_keys # noqa: E402 +from _qwen38_tools_test_utils import ( # noqa: E402 + prefill_chunk_problem, + runtime_profiles_problem, + unknown_qwen38_graph_keys, +) @pytest.fixture @@ -119,6 +124,7 @@ def test_gguf_runtime_profiles_and_block_counts(package_builder, weight_bytes, e cfg = package_builder.default_base_genai_config() package_builder.configure_gguf_genai_config(cfg, total_weight_bytes=weight_bytes) + assert runtime_profiles_problem(cfg["runtime_profiles"]) is None profiles = {p["id"]: p for p in cfg["runtime_profiles"]} assert list(profiles) == [ "gpu-32k", @@ -150,6 +156,14 @@ def test_gguf_runtime_profiles_and_block_counts(package_builder, weight_bytes, e profiles["gpu-128k"]["eligibility"]["minimum_total_device_memory_bytes"], profiles["gpu-256k"]["eligibility"]["minimum_total_device_memory_bytes"], ) == expected_mins + for tier in ("32k", "64k", "128k", "256k"): + assert profiles[f"integrated-gpu-{tier}"]["overlay"] == profiles[f"gpu-{tier}"]["overlay"] + # The engine refuses the whole config when a profile lacks minimum_total_device_memory_bytes. + unbounded = copy.deepcopy(cfg["runtime_profiles"]) + del unbounded[-1]["eligibility"]["minimum_total_device_memory_bytes"] + assert runtime_profiles_problem(unbounded) == ( + "runtime profile 'integrated-gpu-256k' is missing minimum_total_device_memory_bytes" + ) assert cfg["model"]["qwen38_graph"]["plane_library"] == "" assert "ep.cuda.qmoe_int_dequant_max_scratch_bytes" not in cfg["model"]["decoder"]["session_options"] assert cfg["model"]["bot_token_id"] == 248058 @@ -1315,6 +1329,7 @@ def _assert_valid_exported_package(out_dir: Path, *, expected_ple_format: str = assert prefill_chunk_problem(qwen_sec) is None assert qwen_sec["plane_library"] == "" profiles = cfg["runtime_profiles"] + assert runtime_profiles_problem(profiles) is None assert [p["id"] for p in profiles] == [ "gpu-32k", "gpu-64k", diff --git a/tools/python/qwen38_flash_next/README.md b/tools/python/qwen38_flash_next/README.md index fdf15d4762..941c0138c7 100644 --- a/tools/python/qwen38_flash_next/README.md +++ b/tools/python/qwen38_flash_next/README.md @@ -192,6 +192,7 @@ an instantly answering device. The default standard-RoPE export with unified caches supports: - **Single unified `model.onnx` across all context lengths (`32K` / `128K` / `256K`)**: cache capacity dimensions (`kv_capacity` and `state_capacity`) are symbolic and `contrib_const_dsa_total_seq` is set to `[262152]`, so a single `model.onnx` serves `32K` (`num_blocks: 142`), `128K` (`num_blocks: 525`), and `256K` (`num_blocks: 1037`) without duplicate `model_128k.onnx` / `model_256k.onnx` files. - **Pure GPU execution** up to **256K (`262,144`) context length** on H200 / high-VRAM GPUs (`gpu-128k`, `gpu-256k` -> `model.onnx`). +- **Unified-memory GPUs** (`is_integrated: true`, e.g. NVIDIA DGX Spark): they report host RAM as total device memory, so `integrated-gpu-128k` / `integrated-gpu-256k` apply the `gpu-128k` / `gpu-256k` overlays when the same thresholds hold for free device memory (`minimum_free_device_memory_bytes`). - **QMoE CPU/GPU hybrid offload** on 20–36.5 GB consumer GPUs (`consumer-gpu-24g`, `consumer-gpu-24g-host-64g`, `consumer-gpu-30g-ram-48g-32k`, `consumer-gpu-32g` -> `model.onnx` + `expert-profile.bin`). - **Authoritative protocol boundary token IDs** (`bot_token_id: 248058`, `eot_token_id: 248059`, `bor_token_id: 248068`, `eor_token_id: 248069`) and **Engine dynamic batching** (`max_batch_size: 1`, `max_scheduled_tokens: 8192`). @@ -494,7 +495,7 @@ Runtime distribution binaries (`libonnxruntime*`, `libonnxruntime_providers_cuda | **`UD-IQ2_XS`** | `44.46 GiB` (`47.74 GB`) | `50.14 GiB` (`53.84 GB`) | `50.49 GiB` (`54.21 GB`) | `51.59 GiB` (`55.39 GB`) | `54.28 GiB` (`58.28 GB`) | `56 GB` / `57 GB` / `58 GB` / `61 GB` | | **`UD-IQ3_XXS`** | `51.73 GiB` (`55.55 GB`) | `57.44 GiB` (`61.68 GB`) | `57.79 GiB` (`62.05 GB`) | `58.89 GiB` (`63.24 GB`) | `61.59 GiB` (`66.13 GB`) | `64 GB` / `65 GB` / `66 GB` / `69 GB` | -`gpu-32k` sets `overlay.search.chunk_size = 4096` to match its 4096-token prefill memory calibration. `build_package_gguf.py` also supports the component precision options of section 3 (including `v_rc5_fp16`, `kv_fp16`, and `fp16` rewrites on packed QSA projections, `--ple-engram {default,iq4_nl,q8_0}` resolved under `/engram/` or `--engram-gguf`, and `--consolidate-data-files`; RC5-only INT8 overrides `--ple-engram int8`, `--indexer-dtype int8`, and `--gdn-gate-dtype int8` are rejected), and automatically recalculates `vram_and_blocks` and `runtime_profiles` VRAM thresholds after precision overrides: +`gpu-32k` sets `overlay.search.chunk_size = 4096` to match its 4096-token prefill memory calibration. Unified-memory GPUs (`is_integrated: true`) report host RAM as total device memory, so `integrated-gpu-32k` / `integrated-gpu-64k` / `integrated-gpu-128k` / `integrated-gpu-256k` apply the same overlays when these thresholds hold for free device memory. `build_package_gguf.py` also supports the component precision options of section 3 (including `v_rc5_fp16`, `kv_fp16`, and `fp16` rewrites on packed QSA projections, `--ple-engram {default,iq4_nl,q8_0}` resolved under `/engram/` or `--engram-gguf`, and `--consolidate-data-files`; RC5-only INT8 overrides `--ple-engram int8`, `--indexer-dtype int8`, and `--gdn-gate-dtype int8` are rejected), and automatically recalculates `vram_and_blocks` and `runtime_profiles` VRAM thresholds after precision overrides: ```bash # Build GGUF UD-IQ3_XXS and UD-IQ2_XS with Q8_0 engram, FP32 long-context path, and FP16 KV cache: diff --git a/tools/python/qwen38_flash_next/build_package.py b/tools/python/qwen38_flash_next/build_package.py index f323706634..368a12fac4 100644 --- a/tools/python/qwen38_flash_next/build_package.py +++ b/tools/python/qwen38_flash_next/build_package.py @@ -1908,9 +1908,12 @@ def default_runtime_profiles( }, }, }, + # Unified-memory GPUs report host RAM as total device memory, so their tiers are gated on free device memory. + # The engine requires minimum_total_device_memory_bytes on every profile; free memory never exceeds total memory. { "id": "integrated-gpu-128k", "eligibility": { + "minimum_total_device_memory_bytes": min_128k, "minimum_free_device_memory_bytes": min_128k, "maximum_free_device_memory_bytes": min_256k - 1, "is_integrated": True, @@ -1928,7 +1931,7 @@ def default_runtime_profiles( "dynamic_batching": { "num_blocks": NUM_BLOCKS_128K, "max_batch_size": 1, - "max_scheduled_tokens": 8192 if min_256k <= 68_000_000_000 else 4096, + "max_scheduled_tokens": 8192, } }, }, @@ -1936,6 +1939,7 @@ def default_runtime_profiles( { "id": "integrated-gpu-256k", "eligibility": { + "minimum_total_device_memory_bytes": min_256k, "minimum_free_device_memory_bytes": min_256k, "is_integrated": True, }, @@ -1952,7 +1956,7 @@ def default_runtime_profiles( "dynamic_batching": { "num_blocks": NUM_BLOCKS_256K, "max_batch_size": 1, - "max_scheduled_tokens": 8192 if min_256k <= 68_000_000_000 else 4096, + "max_scheduled_tokens": 8192, } }, }, diff --git a/tools/python/qwen38_flash_next/package_io.py b/tools/python/qwen38_flash_next/package_io.py index 6140919105..dd69ef81e8 100644 --- a/tools/python/qwen38_flash_next/package_io.py +++ b/tools/python/qwen38_flash_next/package_io.py @@ -928,9 +928,11 @@ def make_overlay(num_blocks: int, max_scheduled: int, chunk_size: int | None = N "overlay": make_overlay(GGUF_NUM_BLOCKS_256K, prefill_chunk_long), }, # Unified-memory GPUs report host RAM as total device memory, so their tiers are gated on free device memory. + # The engine requires minimum_total_device_memory_bytes on every profile; free memory never exceeds total memory. { "id": "integrated-gpu-32k", "eligibility": { + "minimum_total_device_memory_bytes": min_32k, "minimum_free_device_memory_bytes": min_32k, "maximum_free_device_memory_bytes": min_64k - 1, "is_integrated": True, @@ -940,6 +942,7 @@ def make_overlay(num_blocks: int, max_scheduled: int, chunk_size: int | None = N { "id": "integrated-gpu-64k", "eligibility": { + "minimum_total_device_memory_bytes": min_64k, "minimum_free_device_memory_bytes": min_64k, "maximum_free_device_memory_bytes": min_128k - 1, "is_integrated": True, @@ -949,6 +952,7 @@ def make_overlay(num_blocks: int, max_scheduled: int, chunk_size: int | None = N { "id": "integrated-gpu-128k", "eligibility": { + "minimum_total_device_memory_bytes": min_128k, "minimum_free_device_memory_bytes": min_128k, "maximum_free_device_memory_bytes": min_256k - 1, "is_integrated": True, @@ -958,6 +962,7 @@ def make_overlay(num_blocks: int, max_scheduled: int, chunk_size: int | None = N { "id": "integrated-gpu-256k", "eligibility": { + "minimum_total_device_memory_bytes": min_256k, "minimum_free_device_memory_bytes": min_256k, "is_integrated": True, }, diff --git a/tools/python/qwen38_flash_next/qwen38_engine_util.py b/tools/python/qwen38_flash_next/qwen38_engine_util.py index b99e745025..3276f603e1 100644 --- a/tools/python/qwen38_flash_next/qwen38_engine_util.py +++ b/tools/python/qwen38_flash_next/qwen38_engine_util.py @@ -205,11 +205,16 @@ def _link_package_asset(link_path: Path, target: Path) -> None: _winapi.CreateJunction(str(target), str(link_path)) else: os.link(target, link_path) -def _needs_windows_builtin_plane_overlay( - package: Path, package_config: dict, *overlays: dict | None, is_windows: bool | None = None -) -> bool: - """Return True on Windows when `plane_library` names an existing `.so` with no `.dll` alias and no overlay overrides it.""" - if not (os.name == "nt" if is_windows is None else is_windows): + + +def _needs_windows_builtin_plane_overlay(package: Path, package_config: dict, *overlays: dict | None) -> bool: + """Return True on Windows when `plane_library` names an existing `.so` with no `.dll` alias and no overlay sets it. + + The engine loads such a `.so` as configured, which fails on Windows (see ResolveWindowsSharedLibraryPath in + src/models/qwen38_graph_plane.cpp, whose `.dll` aliases this mirrors); an empty `plane_library` selects the + built-in plane instead. + """ + if os.name != "nt": return False for doc in overlays: if isinstance(doc, dict) and "plane_library" in doc.get("model", {}).get("qwen38_graph", {}):