KTOR-9796 Add flaky-test quarantine infrastructure and transient-only retry

build-logic/build.gradle.kts

@@ -22,6 +22,12 @@ dependencies {
     implementation(files(libs.javaClass.superclass.protectionDomain.codeSource.location))
 }
 
+// We want to share test constants logic between tests and build-logic.
+// Adding that module as a project dependency would be circular as it uses plugins from build-logic.
+sourceSets.main {
+    kotlin.srcDir("../ktor-shared/ktor-test-constants/common/src")
+}
+
 // Should be synced with gradle/gradle-daemon-jvm.properties
 kotlin {
     jvmToolchain(21)

build-logic/src/main/kotlin/ktorbuild.kmp.gradle.kts

@@ -44,6 +44,7 @@ kotlin {
 val targets = ktorBuild.targets
 
 configureCommon()
+configureFlakyTests()
 if (targets.hasJvm) configureJvm()
 if (targets.hasJs) configureJs()
 if (targets.hasWasmJs) configureWasmJs()

build-logic/src/main/kotlin/ktorbuild/targets/FlakyTestsConfig.kt

@@ -0,0 +1,49 @@
+/*
+ * Copyright 2014-2026 JetBrains s.r.o and contributors. Use of this source code is governed by the Apache 2.0 license.
+ */
+
+package ktorbuild.targets
+
+import io.ktor.test.constants.FLAKY_MODE_PROPERTY
+import io.ktor.test.constants.FLAKY_NAME_TOKEN
+import io.ktor.test.constants.FlakyTestsMode
+import org.gradle.api.Project
+import org.gradle.api.tasks.testing.AbstractTestTask
+import org.gradle.kotlin.dsl.withType
+
+/** Matches the token anywhere in the test name, which also covers the `[target]` name suffixes. */
+private const val FLAKY_TEST_PATTERN = "*$FLAKY_NAME_TOKEN*"
+
+internal const val FLAKY_TEST_TASK = "flakyTest"
+
+internal fun Project.flakyTestsMode(): FlakyTestsMode =
+    FlakyTestsMode.of(providers.gradleProperty(FLAKY_MODE_PROPERTY).orNull)
+
+/** Test tasks that select `@Flaky` by annotation and must therefore *not* also get the name filter. */
+internal val ANNOTATION_DRIVEN_TEST_TASKS = setOf("jvmTest", "stressTest", FLAKY_TEST_TASK)
+
+/**
+ * Applies the [FLAKY_MODE_PROPERTY] mode to every test task that cannot select `@Flaky` by
+ * annotation — every target without a JUnit Platform, plus JVM tasks outside
+ * [ANNOTATION_DRIVEN_TEST_TASKS]. The name token exists only for those.
+ */
+internal fun Project.configureFlakyTests() {
+    val mode = flakyTestsMode()
+
+    tasks.withType<AbstractTestTask>().configureEach {
+        if (name !in ANNOTATION_DRIVEN_TEST_TASKS) selectFlakyTests(mode)
+    }
+}
+
+private fun AbstractTestTask.selectFlakyTests(mode: FlakyTestsMode) {
+    when (mode) {
+        FlakyTestsMode.EXCLUDE -> filter.excludeTestsMatching(FLAKY_TEST_PATTERN)
+
+        FlakyTestsMode.ONLY -> {
+            filter.includeTestsMatching(FLAKY_TEST_PATTERN)
+            filter.isFailOnNoMatchingTests = false
+        }
+
+        FlakyTestsMode.ALL -> Unit
+    }
+}

build-logic/src/main/kotlin/ktorbuild/targets/JvmConfig.kt

@@ -4,6 +4,8 @@
 
 package ktorbuild.targets
 
+import io.ktor.test.constants.FLAKY_MODE_PROPERTY
+import io.ktor.test.constants.FlakyTestsMode
 import ktorbuild.ProjectTag
 import ktorbuild.addProjectTag
 import ktorbuild.internal.java
@@ -12,13 +14,16 @@ import ktorbuild.internal.ktorBuild
 import ktorbuild.internal.libs
 import org.gradle.api.Project
 import org.gradle.api.provider.Provider
+import org.gradle.api.tasks.TaskProvider
 import org.gradle.api.tasks.testing.Test
 import org.gradle.jvm.tasks.Jar
 import org.gradle.jvm.toolchain.JavaLanguageVersion
 import org.gradle.jvm.toolchain.JavaToolchainService
 import org.gradle.kotlin.dsl.*
 import org.jetbrains.kotlin.gradle.targets.jvm.tasks.KotlinJvmTest
 
+private const val TEST_BASE_MODULE = "ktor-test-base"
+
 internal fun Project.configureJvm() {
     addProjectTag(ProjectTag.Jvm)
 
@@ -41,25 +46,60 @@ internal fun Project.configureJvm() {
 }
 
 private fun Project.configureTests() {
-    val jvmTest = tasks.named<KotlinJvmTest>("jvmTest") {
+    val flakyTestsMode = flakyTestsMode()
+
+    tasks.withType<Test>().matching { it.name in ANNOTATION_DRIVEN_TEST_TASKS }.configureEach {
         maxHeapSize = "2g"
-        exclude("**/*StressTest*")
+        systemProperty("junit.jupiter.extensions.autodetection.enabled", "true")
+        systemProperty(FLAKY_MODE_PROPERTY, flakyTestsMode.propertyValue)
         useJUnitPlatform()
         configureJavaToolchain(java.toolchain.languageVersion, ktorBuild.jvmTestToolchain)
     }
 
-    tasks.register<Test>("stressTest") {
-        classpath = files(jvmTest.map { it.classpath })
-        testClassesDirs = files(jvmTest.map { it.testClassesDirs })
+    val jvmTest = tasks.named<KotlinJvmTest>("jvmTest") {
+        exclude("**/*StressTest*")
+    }
 
-        maxHeapSize = "2g"
+    tasks.register<Test>("stressTest") {
+        inheritTestFilesFrom(jvmTest)
         jvmArgs("-XX:+HeapDumpOnOutOfMemoryError")
         setForkEvery(1)
         systemProperty("enable.stress.tests", "true")
         include("**/*StressTest*")
-        useJUnitPlatform()
-        configureJavaToolchain(java.toolchain.languageVersion, ktorBuild.jvmTestToolchain)
     }
+
+    tasks.register<Test>(FLAKY_TEST_TASK) {
+        inheritTestFilesFrom(jvmTest)
+        inheritExecutionSettingsFrom(jvmTest)
+
+        val classpath = classpath
+
+        onlyIf("FlakyTestCondition is on the test runtime classpath") {
+            classpath.any { it.invariantSeparatorsPath.contains(TEST_BASE_MODULE) }
+        }
+
+        ignoreFailures = true
+        outputs.upToDateWhen { false }
+        systemProperty(FLAKY_MODE_PROPERTY, FlakyTestsMode.ONLY.propertyValue)
+        exclude("**/*StressTest*")
+    }
+}
+
+/** Copies the test classpath and classes directories of [source] onto this task. */
+private fun Test.inheritTestFilesFrom(source: TaskProvider<KotlinJvmTest>) {
+    val jvmTest = source.get()
+    classpath = jvmTest.classpath
+    testClassesDirs = jvmTest.testClassesDirs
+}
+
+/**
+ * Copies the execution environment of [source] onto this task: the system properties and JVM
+ * arguments a module set on its own `jvmTest`, which decide how that module's tests behave.
+ */
+private fun Test.inheritExecutionSettingsFrom(source: TaskProvider<KotlinJvmTest>) {
+    val jvmTest = source.get()
+    systemProperties(jvmTest.systemProperties)
+    jvmArgs(jvmTest.jvmArgs.orEmpty())
 }
 
 private fun Project.configureJarManifest() {

gradle/libs.versions.toml

@@ -42,6 +42,7 @@ jackson3 = "3.2.2"
 jackson-annotations = "2.22"
 
 junit = "5.14.4"
+konsist = "0.17.3"
 slf4j = "2.0.19"
 logback = "1.6.3"
 
@@ -211,6 +212,7 @@ slf4j-api = { module = "org.slf4j:slf4j-api", version.ref = "slf4j" }
 slf4j-simple = { module = "org.slf4j:slf4j-simple", version.ref = "slf4j" }
 logback-classic = { module = "ch.qos.logback:logback-classic", version.ref = "logback" }
 junit = { module = "org.junit.jupiter:junit-jupiter", version.ref = "junit" }
+konsist = { module = "com.lemonappdev:konsist", version.ref = "konsist" }
 
 apache-httpasyncclient = { module = "org.apache.httpcomponents:httpasyncclient", version.ref = "apache" }
 apache-client5 = { module = "org.apache.httpcomponents.client5:httpclient5", version.ref = "apache5" }

ktor-server/ktor-server-test-base/common/src/io/ktor/server/test/base/BaseTest.kt

@@ -5,6 +5,7 @@
 package io.ktor.server.test.base
 
 import io.ktor.test.*
+import io.ktor.utils.io.InternalAPI
 import kotlinx.coroutines.CoroutineScope
 import kotlinx.coroutines.test.TestResult
 import kotlin.time.Duration
@@ -23,3 +24,36 @@ expect abstract class BaseTest() {
         block: suspend CoroutineScope.() -> Unit
     ): TestResult
 }
+
+/**
+ * The retry, guard and lifecycle scaffolding shared by the [BaseTest.runTest] actuals.
+ *
+ * Only the coroutine context differs between targets, so that part stays in the actual and arrives
+ * here as [runAttempt] — the platform's `runTestWithRealTime` call. Everything else is identical:
+ * an assertion failure thrown by [block] is recorded and ends the retry loop, so a deterministic
+ * failure is reported by the attempt that produced it instead of being masked as flakiness.
+ * `runTest`'s own timeout error is an `AssertionError` too, but stays retryable because it is thrown
+ * around [block] rather than by it, so the guard never sees it.
+ */
+@OptIn(InternalAPI::class)
+internal fun BaseTest.runTestAttempts(
+    retries: Int,
+    runAttempt: (block: suspend CoroutineScope.() -> Unit) -> TestResult,
+    block: suspend CoroutineScope.() -> Unit,
+): TestResult {
+    val guard = DeterministicFailureGuard()
+    return retryTest(retries, shouldRetry = { !guard.hasFailure }) { retry ->
+        runAttempt {
+            if (retry > 0) println("[Retry $retry/$retries]")
+            beforeTest()
+            try {
+                block()
+            } catch (cause: Throwable) {
+                guard.record(cause)
+                throw cause
+            } finally {
+                afterTest()
+            }
+        }
+    }
+}

ktor-server/ktor-server-test-base/common/test/io/ktor/server/test/base/BaseTestTest.kt

@@ -15,7 +15,7 @@ class BaseTestTest : BaseTest() {
     fun `runTest - retry test by default on non-JVM platform`(): TestResult {
         var retryCount = 0
         return runTest {
-            if (!PlatformUtils.IS_JVM && retryCount++ < 1) fail("This test should be retried")
+            if (!PlatformUtils.IS_JVM && retryCount++ < 1) transientFailure("This test should be retried")
         }
     }
 
@@ -31,7 +31,7 @@ class BaseTestTest : BaseTest() {
     fun `runTest - more than one retry`(): TestResult {
         var retryCount = 0
         return runTest(retries = 3) {
-            if (retryCount++ < 3) fail("This test should be retried")
+            if (retryCount++ < 3) transientFailure("This test should be retried")
         }
     }
 
@@ -43,3 +43,6 @@ class BaseTestTest : BaseTest() {
         }
     }
 }
+
+/** Simulates a transient (retryable) failure, as opposed to a deterministic assertion failure. */
+private fun transientFailure(message: String): Nothing = throw IllegalStateException(message)

ktor-server/ktor-server-test-base/jvm/src/io/ktor/server/test/base/BaseTestJvm.kt

@@ -7,6 +7,7 @@ package io.ktor.server.test.base
 import io.ktor.test.*
 import io.ktor.test.dispatcher.*
 import io.ktor.test.junit.*
+import io.ktor.utils.io.InternalAPI
 import kotlinx.coroutines.CoroutineName
 import kotlinx.coroutines.CoroutineScope
 import kotlinx.coroutines.debug.junit5.CoroutinesTimeout
@@ -45,19 +46,14 @@ actual abstract class BaseTest actual constructor() {
         errorCollector += error
     }
 
+    @OptIn(InternalAPI::class)
     actual fun runTest(
         timeout: Duration,
         retries: Int,
         block: suspend CoroutineScope.() -> Unit
-    ): TestResult = retryTest(retries) { retry ->
-        runTestWithRealTime(CoroutineName("test-$testName"), timeout) {
-            if (retry > 0) println("[Retry $retry/$retries]")
-            beforeTest()
-            try {
-                block()
-            } finally {
-                afterTest()
-            }
-        }
-    }
+    ): TestResult = runTestAttempts(
+        retries,
+        runAttempt = { attempt -> runTestWithRealTime(CoroutineName("test-$testName"), timeout, attempt) },
+        block = block,
+    )
 }

ktor-server/ktor-server-test-base/jvm/test/io/ktor/server/test/base/BaseTestRetryJvmTest.kt

@@ -0,0 +1,47 @@
+/*
+ * Copyright 2014-2026 JetBrains s.r.o and contributors. Use of this source code is governed by the Apache 2.0 license.
+ */
+
+package io.ktor.server.test.base
+
+import kotlin.test.Test
+import kotlin.test.assertEquals
+import kotlin.test.assertFailsWith
+import kotlin.test.fail
+
+/**
+ * Verifies how [BaseTest.runTest] classifies failures for retries.
+ *
+ * These assertions live in the JVM source set because there `TestResult` is `Unit`, so a failing
+ * test propagates its failure synchronously and can be asserted on. On Web the same call returns
+ * a `Promise`, which would reject asynchronously instead.
+ */
+class BaseTestRetryJvmTest {
+
+    private class Fixture : BaseTest()
+
+    @Test
+    fun `assertion failure is not retried`() {
+        var attempts = 0
+
+        assertFailsWith<AssertionError> {
+            Fixture().runTest(retries = 3) {
+                attempts++
+                fail("Deterministic failure")
+            }
+        }
+
+        assertEquals(1, attempts, "An assertion failure is deterministic and must not be retried")
+    }
+
+    @Test
+    fun `transient failure is retried`() {
+        var attempts = 0
+
+        Fixture().runTest(retries = 3) {
+            if (attempts++ < 3) throw IllegalStateException("Transient failure")
+        }
+
+        assertEquals(4, attempts, "Transient failures should be retried until they pass")
+    }
+}

ktor-server/ktor-server-test-base/nonJvm/src/io/ktor/server/test/base/BaseTest.nonJvm.kt

@@ -50,17 +50,11 @@ actual abstract class BaseTest actual constructor() {
         timeout: Duration,
         retries: Int,
         block: suspend CoroutineScope.() -> Unit
-    ): TestResult = retryTest(retries) { retry ->
-        runTestWithRealTime(timeout = timeout) {
-            if (retry > 0) println("[Retry $retry/$retries]")
-            beforeTest()
-            try {
-                block()
-            } finally {
-                afterTest()
-            }
-        }
-    }
+    ): TestResult = runTestAttempts(
+        retries,
+        runAttempt = { attempt -> runTestWithRealTime(timeout = timeout, testBody = attempt) },
+        block = block,
+    )
 }
 
 private class UnhandledErrorsException(override val message: String) : Exception()

ktor-shared/ktor-test-base/build.gradle.kts

@@ -14,12 +14,17 @@ kotlin {
             api(libs.kotlin.test)
             api(projects.ktorUtils)
             api(projects.ktorTestDispatcher)
+            api(projects.ktorTestConstants)
         }
 
         jvmMain.dependencies {
             api(libs.kotlin.test.junit5)
             api(libs.junit)
             api(libs.kotlinx.coroutines.debug)
         }
+
+        jvmTest.dependencies {
+            implementation(libs.konsist)
+        }
     }
 }