Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 26 additions & 0 deletions .github/skills/ort-test/SKILL.md
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,32 @@ You can also run all tests via the build script (assumes a prior successful buil
.\build.bat --config Release --test # Windows
```

### Experimental Linux provider-test sharding

`--cmake_extra_defines onnxruntime_ENABLE_PROVIDER_TEST_SHARDING=ON` registers four
`onnxruntime_provider_test_shard_[0-3]` CTest entries instead of the single provider entry.
The option defaults to OFF. Only Linux CPU x64 Debug ASan and arm64 Debug CI opt in;
other workflows and direct execution of the binary are unchanged.

Run the entries concurrently from the build output directory:

```bash
ctest -C Debug --output-on-failure --parallel 16 -R '^onnxruntime_provider_test_shard_'
```

Each entry reserves four CTest processor slots and sets default intra-op/OpenMP threads to three
and inter-op threads to two. Keeping inter-op threads above one avoids falling back from parallel
to sequential execution. Tests that explicitly select their own thread count retain it.
This bounds the usual pools, not total memory or every explicitly created thread: notably,
`Gather_overflow_check` needs multiple GiB. Measure peak memory and contention before broader rollout.

The runner gives each process private working/temp directories; existing testdata, model inputs
and shared libraries remain shared via symlinks. It preserves inherited GoogleTest filters/disabled-test policy,
requires GoogleTest's shard-status acknowledgment and rejects an empty shard. XML reports are always
written for this experiment as `<binary>.<config>.shard_<index>.results.xml`, beside the executable,
including when ordinary report generation is off. A narrow filter can leave a shard empty and
intentionally fail CTest; use the standalone binary for individual-case debugging.

### Locating the build output directory

The default path follows the pattern `build/<Platform>/<Config>/` where Platform is `Linux`, `MacOS`, or `Windows`. With Visual Studio multi-config generators on Windows, the config may appear twice (e.g., `build/Windows/Release/Release/`). The path can also be customized via `--build_dir`.
Expand Down
4 changes: 2 additions & 2 deletions .github/workflows/linux_ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@ jobs:
architecture: x64
dockerfile_path: tools/ci_build/github/linux/docker/inference/x86_64/default/cpu/Dockerfile
docker_image_repo: onnxruntimecpubuildcix64
extra_build_flags: '--enable_address_sanitizer'
extra_build_flags: '--enable_address_sanitizer --cmake_extra_defines onnxruntime_ENABLE_PROVIDER_TEST_SHARDING=ON'
job_identifier: build-linux-x64-debug
# python_path_prefix: '' # Default empty string is fine, no prefix needed
secrets:
Expand Down Expand Up @@ -115,7 +115,7 @@ jobs:
dockerfile_path: tools/ci_build/github/linux/docker/inference/aarch64/default/cpu/Dockerfile
docker_image_repo: onnxruntimecpubuildciaarch64
# ASan disabled due to excessive runtime (>4hr). Includes wheel build for basic checks.
extra_build_flags: '--use_binskim_compliant_compile_flags --build_shared_lib --enable_arm_neon_nchwc'
extra_build_flags: '--use_binskim_compliant_compile_flags --build_shared_lib --enable_arm_neon_nchwc --cmake_extra_defines onnxruntime_ENABLE_PROVIDER_TEST_SHARDING=ON'
job_identifier: build-linux-arm64-debug
secrets:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
Expand Down
1 change: 1 addition & 0 deletions cmake/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -96,6 +96,7 @@ option(onnxruntime_USE_ARM_NEON_NCHWC "Build with ARM Neon NCHWc kernels in MLAS
option(onnxruntime_USE_KLEIDIAI "Build with KleidiAI integration in MLAS" OFF)
option(onnxruntime_USE_QMX_KLEIDIAI_COEXIST "Build with QMX and Arm KLEIDIAI libraries" OFF)
option(onnxruntime_BUILD_UNIT_TESTS "Build ONNXRuntime unit tests" ON)
option(onnxruntime_ENABLE_PROVIDER_TEST_SHARDING "Run native Linux provider tests in four GoogleTest shards" OFF)
option(onnxruntime_RUN_MLAS_TESTS "Run a targeted subset of MLAS unit tests with CTest" OFF)
# Declare both prerequisites before evaluating this dependent option on a fresh configure.
# Enable ONNX Runtime CUDA EP's internal unit tests that directly access the EP's internal functions instead of through
Expand Down
26 changes: 26 additions & 0 deletions cmake/onnxruntime_provider_test_shards.cmake
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
# Copyright (c) Microsoft Corporation. All rights reserved.
# Licensed under the MIT License.

function(onnxruntime_add_provider_test_shards target)
set(test_names)
foreach(shard_index RANGE 0 3)
set(test_name "onnxruntime_provider_test_shard_${shard_index}")
add_test(NAME ${test_name}
COMMAND "${Python_EXECUTABLE}" "${REPO_ROOT}/tools/ci_build/run_provider_test_shard.py"
--binary "$<TARGET_FILE:${target}>"
--shard-index ${shard_index}
--report "$<TARGET_FILE:${target}>.$<CONFIG>.shard_${shard_index}.results.xml"
-- ${ARGN}
WORKING_DIRECTORY "$<TARGET_FILE_DIR:${target}>"
)
# Inter-op needs at least two threads to retain parallel-executor coverage.
set_tests_properties(${test_name} PROPERTIES
TIMEOUT 10800
PROCESSORS 4
ENVIRONMENT_MODIFICATION
"ORT_INTRA_OP_NUM_THREADS=set:3;ORT_INTER_OP_NUM_THREADS=set:2;OMP_NUM_THREADS=set:3"
)
list(APPEND test_names ${test_name})
endforeach()
set(onnxruntime_provider_test_names ${test_names} PARENT_SCOPE)
endfunction()
36 changes: 27 additions & 9 deletions cmake/onnxruntime_unittests.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,12 @@ if (onnxruntime_ENABLE_TRAINING)
endif()

include(onnxruntime_telemetry_tests.cmake)
include("${CMAKE_CURRENT_LIST_DIR}/onnxruntime_provider_test_shards.cmake")

if(onnxruntime_ENABLE_PROVIDER_TEST_SHARDING AND
(NOT CMAKE_SYSTEM_NAME STREQUAL "Linux" OR CMAKE_CROSSCOMPILING))
message(FATAL_ERROR "Provider test sharding requires a native Linux build.")
endif()

# Exclude files based on CMake options.
function(filter_test_srcs test_srcs_var)
Expand Down Expand Up @@ -187,7 +193,11 @@ function(AddTest)
endif()

set(TEST_ARGS ${_UT_TEST_ARGS})
if (onnxruntime_GENERATE_TEST_REPORTS AND NOT _UT_NO_GTEST_REPORT)
set(shard_provider_test OFF)
if(onnxruntime_ENABLE_PROVIDER_TEST_SHARDING AND _UT_TEST_NAME STREQUAL "onnxruntime_provider_test")
set(shard_provider_test ON)
endif()
if (onnxruntime_GENERATE_TEST_REPORTS AND NOT _UT_NO_GTEST_REPORT AND NOT shard_provider_test)
# generate a report file next to the test program
if (CMAKE_SYSTEM_NAME STREQUAL "Emscripten")
# WebAssembly use a memory file system, so we do not use full path
Expand All @@ -197,7 +207,7 @@ function(AddTest)
list(APPEND TEST_ARGS
"--gtest_output=xml:$<SHELL_PATH:$<TARGET_FILE:${_UT_TARGET}>.$<CONFIG>.results.xml>")
endif()
endif(onnxruntime_GENERATE_TEST_REPORTS)
endif()

if (IOS)
# target_sources(${_UT_TARGET} PRIVATE ${TEST_SRC_DIR}/xctest/orttestmain.m)
Expand Down Expand Up @@ -304,12 +314,17 @@ function(AddTest)
# Set test timeout to 3 hours.
set_tests_properties(${_UT_TEST_NAME} PROPERTIES TIMEOUT 10800)
else()
add_test(NAME ${_UT_TEST_NAME}
COMMAND ${_UT_TARGET} ${TEST_ARGS}
WORKING_DIRECTORY $<TARGET_FILE_DIR:${_UT_TARGET}>
)
# Set test timeout to 3 hours.
set_tests_properties(${_UT_TEST_NAME} PROPERTIES TIMEOUT 10800)
if(shard_provider_test)
onnxruntime_add_provider_test_shards(${_UT_TARGET} ${TEST_ARGS})
set(onnxruntime_provider_test_names ${onnxruntime_provider_test_names} PARENT_SCOPE)
else()
add_test(NAME ${_UT_TEST_NAME}
COMMAND ${_UT_TARGET} ${TEST_ARGS}
WORKING_DIRECTORY $<TARGET_FILE_DIR:${_UT_TARGET}>
)
# Set test timeout to 3 hours.
set_tests_properties(${_UT_TEST_NAME} PROPERTIES TIMEOUT 10800)
endif()
endif()
endif()
endfunction(AddTest)
Expand Down Expand Up @@ -1663,6 +1678,7 @@ block()
list(APPEND onnxruntime_provider_test_deps ${onnxruntime_test_providers_runtime_dependencies})
endif()

set(onnxruntime_provider_test_names onnxruntime_provider_test)
AddTest(
TARGET ${onnxruntime_provider_test_target}
TEST_NAME onnxruntime_provider_test
Expand Down Expand Up @@ -1692,7 +1708,9 @@ block()
endif()

onnxruntime_apply_test_target_workarounds(${onnxruntime_provider_test_target})
onnxruntime_set_plugin_ep_test_environment(onnxruntime_provider_test)
foreach(provider_test_name IN LISTS onnxruntime_provider_test_names)
onnxruntime_set_plugin_ep_test_environment(${provider_test_name})
endforeach()

if (onnxruntime_USE_CUDA AND onnxruntime_ENABLE_CUDA_EP_INTERNAL_TESTS AND
NOT onnxruntime_DISABLE_CONTRIB_OPS)
Expand Down
68 changes: 68 additions & 0 deletions tools/ci_build/run_provider_test_shard.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,68 @@
# Copyright (c) Microsoft Corporation. All rights reserved.
# Licensed under the MIT License.

"""Run one of four provider-test shards with isolated writable state."""

import argparse
import os
import subprocess
import tempfile
import xml.etree.ElementTree as ET
from pathlib import Path


def run_shard(binary, shard_index, report, test_args):
binary = binary.resolve(strict=True)
report = report.resolve()
report.unlink(missing_ok=True)
with tempfile.TemporaryDirectory(prefix=f"provider-shard-{shard_index}-", dir=binary.parent) as directory:
root = Path(directory)
work = root / "work"
work.mkdir()
temporary = root / "tmp"
temporary.mkdir()
# Keep ../models and testdata lookups equivalent to execution beside the binary.
(work / "testdata").symlink_to(binary.parent / "testdata", target_is_directory=True)
(root / "models").symlink_to(binary.parent.parent / "models", target_is_directory=True)
for library in binary.parent.glob("*.so*"):
(work / library.name).symlink_to(library)
status = root / "shard-status"
env = os.environ.copy()
env.update(
GTEST_TOTAL_SHARDS="4",
GTEST_SHARD_INDEX=str(shard_index),
GTEST_SHARD_STATUS_FILE=str(status),
TMPDIR=str(temporary),
TMP=str(temporary),
TEMP=str(temporary),
)
result = subprocess.run(
[str(binary), *test_args, f"--gtest_output=xml:{report}"],
cwd=work,
env=env,
check=False,
)
if result.returncode:
return result.returncode
if not status.is_file():
raise RuntimeError("The provider test binary did not acknowledge GoogleTest sharding")
executed = [case for case in ET.parse(report).iter("testcase") if case.get("status") == "run"]
if not executed:
raise RuntimeError(f"Provider test shard {shard_index} executed zero tests")
print(f"Provider shard {shard_index}: {len(executed)} executed tests; report: {report}", flush=True)
return 0


def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--binary", required=True, type=Path)
parser.add_argument("--shard-index", required=True, type=int, choices=range(4))
parser.add_argument("--report", required=True, type=Path)
parser.add_argument("test_args", nargs=argparse.REMAINDER)
args = parser.parse_args()
test_args = args.test_args[1:] if args.test_args[:1] == ["--"] else args.test_args
return run_shard(args.binary, args.shard_index, args.report, test_args)


if __name__ == "__main__":
raise SystemExit(main())
153 changes: 153 additions & 0 deletions tools/ci_build/test_run_provider_test_shard.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,153 @@
# Copyright (c) Microsoft Corporation. All rights reserved.
# Licensed under the MIT License.

import json
import os
import subprocess
import sys
import tempfile
import unittest
from pathlib import Path
from unittest.mock import patch

sys.path.insert(0, str(Path(__file__).resolve().parent))

from run_provider_test_shard import run_shard


@unittest.skipUnless(sys.platform.startswith("linux"), "Provider sharding requires native Linux")
class TestProviderTestShard(unittest.TestCase):
def setUp(self):
self.directory = tempfile.TemporaryDirectory(prefix="ort shard test ")
self.addCleanup(self.directory.cleanup)
self.root = Path(self.directory.name)
self.binary = self.root / "provider_test"
self.binary.touch()
self.report = self.root / "provider.shard_2.results.xml"

def launch(self, command, *, cwd, env, check):
self.assertFalse(check)
self.assertEqual(env["GTEST_TOTAL_SHARDS"], "4")
self.assertEqual(env["GTEST_SHARD_INDEX"], "2")
self.assertEqual(command[1:], ["--gtest_filter=Example.*", f"--gtest_output=xml:{self.report}"])
self.assertNotEqual(cwd, self.root)
self.assertEqual((cwd / "testdata").resolve(), self.root / "testdata")
self.assertEqual((cwd.parent / "models").resolve(), self.root.parent / "models")
self.assertEqual((cwd / "libprovider.so").resolve(), self.root / "libprovider.so")
self.assertTrue(Path(env["TMPDIR"]).is_dir())
self.assertEqual(env["TMPDIR"], env["TMP"])
self.assertEqual(env["TMPDIR"], env["TEMP"])
self.assertFalse(self.report.exists(), "A stale report must not satisfy the nonempty check")
Path(env["GTEST_SHARD_STATUS_FILE"]).touch()
self.report.write_text(
'<testsuites><testsuite><testcase name="selected" status="run"/>'
'<testcase name="disabled" status="notrun"/></testsuite></testsuites>',
encoding="utf-8",
)
return subprocess.CompletedProcess(command, 0)

def run_with(self, launch):
(self.root / "libprovider.so").touch()
with patch("run_provider_test_shard.subprocess.run", side_effect=launch):
return run_shard(self.binary, 2, self.report, ["--gtest_filter=Example.*"])

def test_isolation_filter_forwarding_and_stale_report_removal(self):
self.report.write_text("stale report", encoding="utf-8")
with patch.dict(os.environ, {"GTEST_TOTAL_SHARDS": "99", "GTEST_SHARD_INDEX": "98"}):
self.assertEqual(self.run_with(self.launch), 0)
self.assertEqual(list(self.root.glob("provider-shard-*")), [])

def test_nonzero_exit_is_propagated(self):
self.assertEqual(self.run_with(lambda *args, **kwargs: subprocess.CompletedProcess(args[0], 7)), 7)

def test_missing_shard_acknowledgment_is_rejected(self):
def launch(command, **kwargs):
result = self.launch(command, **kwargs)
Path(kwargs["env"]["GTEST_SHARD_STATUS_FILE"]).unlink()
return result

with self.assertRaisesRegex(RuntimeError, "did not acknowledge"):
self.run_with(launch)

def test_zero_selected_tests_are_rejected(self):
def launch(command, **kwargs):
result = self.launch(command, **kwargs)
self.report.write_text(
'<testsuites><testsuite><testcase status="notrun"/></testsuite></testsuites>',
encoding="utf-8",
)
return result

with self.assertRaisesRegex(RuntimeError, "executed zero tests"):
self.run_with(launch)

def test_missing_report_is_rejected(self):
def launch(command, **kwargs):
result = self.launch(command, **kwargs)
self.report.unlink()
return result

with self.assertRaises(FileNotFoundError):
self.run_with(launch)


class TestProviderShardRegistration(unittest.TestCase):
def test_four_entries_share_binary_and_preserve_environment(self):
repo_root = Path(__file__).resolve().parents[2]
with tempfile.TemporaryDirectory(prefix="ort shard cmake ") as directory:
source = Path(directory)
build = source / "build"
(source / "CMakeLists.txt").write_text(
f"""
cmake_minimum_required(VERSION 3.28)
project(ProviderShards LANGUAGES NONE)
enable_testing()
set(REPO_ROOT "{repo_root.as_posix()}")
set(Python_EXECUTABLE "{Path(sys.executable).as_posix()}")
include("${{REPO_ROOT}}/cmake/onnxruntime_provider_test_shards.cmake")
add_executable(provider IMPORTED)
set_target_properties(provider PROPERTIES IMPORTED_LOCATION "{Path(sys.executable).as_posix()}")
onnxruntime_add_provider_test_shards(provider "--gtest_filter=Example.*")
foreach(test_name IN LISTS onnxruntime_provider_test_names)
set_tests_properties(${{test_name}} PROPERTIES ENVIRONMENT "PLUGIN_CONFIG=retained" LABELS "provider")
endforeach()
""",
encoding="utf-8",
)
subprocess.run(
["cmake", "-S", str(source), "-B", str(build), "-DCMAKE_BUILD_TYPE=Debug"],
check=True,
timeout=60,
)
result = subprocess.run(
["ctest", "--test-dir", str(build), "-C", "Debug", "--show-only=json-v1"],
check=True,
capture_output=True,
text=True,
timeout=30,
)
tests = json.loads(result.stdout)["tests"]
self.assertEqual(len(tests), 4)
reports = set()
for index, test in enumerate(tests):
self.assertEqual(test["name"], f"onnxruntime_provider_test_shard_{index}")
command = test["command"]
self.assertEqual(command[command.index("--shard-index") + 1], str(index))
self.assertEqual(command[command.index("--binary") + 1], str(Path(sys.executable)).replace("\\", "/"))
self.assertEqual(command[-2:], ["--", "--gtest_filter=Example.*"])
reports.add(command[command.index("--report") + 1])
properties = {prop["name"]: prop["value"] for prop in test["properties"]}
self.assertEqual(properties["TIMEOUT"], 10800)
self.assertEqual(properties["PROCESSORS"], 4)
self.assertEqual(properties["ENVIRONMENT"], ["PLUGIN_CONFIG=retained"])
self.assertEqual(properties["LABELS"], ["provider"])
self.assertEqual(
properties["ENVIRONMENT_MODIFICATION"],
["ORT_INTRA_OP_NUM_THREADS=set:3", "ORT_INTER_OP_NUM_THREADS=set:2", "OMP_NUM_THREADS=set:3"],
)
self.assertEqual(len(reports), 4)
self.assertTrue(all(report.endswith(".results.xml") and ".Debug.shard_" in report for report in reports))


if __name__ == "__main__":
unittest.main()
Loading