diff --git a/.github/scripts/print-log-tails.sh b/.github/scripts/print-log-tails.sh new file mode 100755 index 0000000..168d49e --- /dev/null +++ b/.github/scripts/print-log-tails.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +set -u +limit=32768 +for file in "$@"; do + echo "::group::last ${limit} bytes of ${file}" + if [ -f "$file" ]; then + tail -c "$limit" "$file" + echo + else + echo "(no such file)" + fi + echo "::endgroup::" +done diff --git a/.github/scripts/run-shard.sh b/.github/scripts/run-shard.sh new file mode 100755 index 0000000..f405e05 --- /dev/null +++ b/.github/scripts/run-shard.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash +set -uo pipefail + +plan=$(realpath "$1") +shard=$2 +projects=$3 +root=$(dirname "$plan") +stryker="$root/node_modules/.bin/stryker" +tails="$(realpath "$(dirname "$0")")/print-log-tails.sh" + +abort() { + echo "::error::stryker shard: $1" + "$tails" "$2/stderr.log" "$2/stdout.log" + exit "$3" +} + +while IFS=$'\t' read -r index project; do + out="$root/reports/shards/$index/$project" + mkdir -p "$out" || abort "$project: cannot create $out" "$out" 3 + cached="$root/$project/reports/stryker-incremental.json" + if [ -f "$cached" ] && [ ! -f "$out/stryker-incremental.json" ]; then + cp "$cached" "$out/stryker-incremental.json" || abort "$project: cannot seed its incremental file" "$out" 3 + fi + (cd "$root/$project" && "$stryker" run \ + --plan "$plan" --shard "$shard" --project "$project" \ + --progressStreamFile "$out/mutation-stream.jsonl" \ + --incremental --incrementalFile "$out/stryker-incremental.json" \ + < /dev/null > "$out/stdout.log" 2> "$out/stderr.log") + code=$? + case $code in + 0) ;; + 1) echo "stryker shard: $project scored below thresholds.break over this shard's mutants; the merged report carries the project verdict" ;; + *) abort "$project exited $code" "$out" "$code" ;; + esac + [ -f "$out/mutation-stream.jsonl" ] || + abort "$project completed without leaving its progress stream at $out/mutation-stream.jsonl" "$out" 3 +done < "$projects" diff --git a/.github/workflows/release-gate.yml b/.github/workflows/release-gate.yml index 5fff451..2a94a8d 100644 --- a/.github/workflows/release-gate.yml +++ b/.github/workflows/release-gate.yml @@ -3,7 +3,6 @@ name: Release gate on: push: branches: [main] - pull_request: workflow_dispatch: permissions: @@ -11,7 +10,7 @@ permissions: concurrency: group: release-gate-${{ github.ref }} - cancel-in-progress: ${{ github.event_name == 'pull_request' }} + cancel-in-progress: false jobs: plan: @@ -21,16 +20,11 @@ jobs: outputs: matrix: ${{ steps.publish.outputs.matrix }} has-shards: ${{ steps.publish.outputs.has-shards }} - scope: ${{ steps.projects.outputs.scope }} steps: - uses: actions/checkout@v7 with: - fetch-depth: 2 persist-credentials: false - uses: ./.github/actions/dev-shell - - name: List the files the pull request changes - if: github.event_name == 'pull_request' - run: mkdir -p .cache && git diff --name-only HEAD^1 HEAD > .cache/changed-files.txt - name: Discover mutation projects uses: ./.github/actions/sandbox with: @@ -38,27 +32,21 @@ jobs: jsr.io registry.npmjs.org command: >- - deno run --config=scripts/deno.json --allow-read --allow-write --allow-env=MUTATION_SCOPE - scripts/stryker-plan-gate.ts projects --changed .cache/changed-files.txt - --out .cache/mutation-projects.txt --scope-out .cache/mutation-scope.txt + deno run --config=scripts/deno.json --allow-read --allow-write + scripts/stryker-plan-gate.ts projects --out .cache/mutation-projects.txt - id: projects - name: Publish the project list and the mutation scope - run: | - echo "projects=$(cat .cache/mutation-projects.txt)" >> "$GITHUB_OUTPUT" - echo "scope=$(cat .cache/mutation-scope.txt)" >> "$GITHUB_OUTPUT" + name: Publish the project list + run: echo "projects=$(cat .cache/mutation-projects.txt)" >> "$GITHUB_OUTPUT" - name: Plan mutation shards if: steps.projects.outputs.projects != '' uses: ./.github/actions/sandbox env: MUTATION_PROJECTS: ${{ steps.projects.outputs.projects }} - MUTATION_SCOPE: ${{ steps.projects.outputs.scope }} with: hosts: | jsr.io registry.npmjs.org - pass-env: | - MUTATION_PROJECTS - MUTATION_SCOPE + pass-env: MUTATION_PROJECTS command: ./node_modules/.bin/stryker plan --target-seconds 900 --projects "$MUTATION_PROJECTS" --out stryker-plan.json - name: Gate the plan uses: ./.github/actions/sandbox @@ -67,9 +55,8 @@ jobs: jsr.io registry.npmjs.org command: >- - deno run --config=scripts/deno.json --allow-read --allow-write --allow-env=MUTATION_SCOPE - scripts/stryker-plan-gate.ts gate --plan stryker-plan.json --changed .cache/changed-files.txt - --out .cache/mutation-plan.out + deno run --config=scripts/deno.json --allow-read --allow-write + scripts/stryker-plan-gate.ts gate --plan stryker-plan.json --out .cache/mutation-plan.out - id: publish name: Publish the plan run: cat .cache/mutation-plan.out >> "$GITHUB_OUTPUT" @@ -91,7 +78,6 @@ jobs: matrix: ${{ fromJSON(needs.plan.outputs.matrix) }} env: CI: "true" - MUTATION_SCOPE: ${{ needs.plan.outputs.scope }} steps: - uses: actions/checkout@v7 with: @@ -103,7 +89,6 @@ jobs: name: stryker-plan path: . - name: Stryker incremental cache - if: github.event_name != 'pull_request' uses: actions/cache@v6 with: path: | @@ -111,6 +96,21 @@ jobs: reports/shards/**/stryker-incremental.json key: stryker-shard-${{ matrix.shard }}-${{ github.sha }} restore-keys: stryker-shard-${{ matrix.shard }}- + - name: Select the shard's projects + uses: ./.github/actions/sandbox + env: + MUTATION_SHARD: ${{ matrix.shard }} + with: + hosts: | + jsr.io + registry.npmjs.org + pass-env: | + MUTATION_SHARD + command: >- + mkdir -p .cache && + deno run --config=scripts/deno.json --allow-read --allow-write + scripts/stryker-plan-gate.ts shard --plan stryker-plan.json --shard "$MUTATION_SHARD" + --out .cache/shard-projects.tsv - name: Mutation at break 100 uses: ./.github/actions/sandbox env: @@ -119,8 +119,7 @@ jobs: pass-env: | GITHUB_ACTIONS MUTATION_SHARD - MUTATION_SCOPE - command: ./node_modules/.bin/stryker run --plan stryker-plan.json --shard "$MUTATION_SHARD" + command: .github/scripts/run-shard.sh stryker-plan.json "$MUTATION_SHARD" .cache/shard-projects.tsv - if: always() uses: actions/upload-artifact@v6 with: @@ -133,41 +132,6 @@ jobs: name: mutation-report-shard-${{ strategy.job-index }} path: packages/*/reports/mutation-report.* if-no-files-found: ignore - - name: Re-run a failed shard's project runs with their output kept - if: failure() - timeout-minutes: 20 - uses: ./.github/actions/sandbox - env: - MUTATION_SHARD: ${{ matrix.shard }} - with: - pass-env: | - GITHUB_ACTIONS - MUTATION_SHARD - MUTATION_SCOPE - command: | - set -uo pipefail - root="$PWD" - jq -r --arg shard "$MUTATION_SHARD" \ - '.shards[] | select("\(.index)/\(.count)" == $shard) | .projects[].project' \ - stryker-plan.json > .cache/shard-projects.txt - while read -r project; do - out="$root/reports/diagnostics/$project" - mkdir -p "$out" - (cd "$project" && "$root/node_modules/.bin/stryker" run --plan "$root/stryker-plan.json" \ - --shard "$MUTATION_SHARD" --project "$project" \ - --progressStreamFile "$out/mutation-stream.jsonl" \ - --incremental --incrementalFile "$out/stryker-incremental.json" \ - > "$out/stdout.log" 2> "$out/stderr.log") - echo "$project exited $?" | tee "$out/exit.txt" - done < .cache/shard-projects.txt - - if: failure() - uses: actions/upload-artifact@v6 - with: - name: mutation-diagnostics-shard-${{ strategy.job-index }} - path: | - reports/diagnostics/ - packages/*/stryker.log - if-no-files-found: ignore verdict: name: verdict ยท merged report @@ -183,7 +147,7 @@ jobs: HAS_SHARDS: ${{ needs.plan.outputs.has-shards }} run: | if [ "$PLAN" != success ]; then echo "::error::the plan job ended $PLAN"; exit 1; fi - if [ "$HAS_SHARDS" != true ]; then echo "no mutated file is in this change's scope"; exit 0; fi + if [ "$HAS_SHARDS" != true ]; then echo "the plan scheduled no shards"; exit 0; fi if [ "$MUTATION" != success ]; then echo "::error::a mutation shard ended $MUTATION"; exit 1; fi - if: needs.plan.outputs.has-shards == 'true' uses: actions/checkout@v7 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 590edcf..9d9f3bf 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -51,7 +51,7 @@ pnpm check:ci nix build .#workspace-tarballs ``` -Mutation testing is not part of `pnpm check:ci`. The release gate (`.github/workflows/release-gate.yml`) runs `stryker plan` once over every workspace package that declares a `mutation` script, then `stryker run` for each planned shard at a break threshold of 100 on every push to `main`. The same workflow runs on every pull request, scoped to what the pull request changes. It mutates the declared files the pull request touches. It mutates a package's whole declared set when the pull request touches any other file in that package: a test, a fixture, a config, or a source file outside the set. It mutates every package's whole set when the pull request changes the gate itself: the workflow, `scripts/stryker-plan-gate.ts`, `stryker.shared.ts`, the lockfile or the Nix toolchain. A pull request that touches no mutated package plans nothing, and its verdict job passes. Pull-request runs skip the incremental cache, so every scoped mutant runs fresh. +Mutation testing is not part of `pnpm check:ci`, and pull requests do not run it. The release gate (`.github/workflows/release-gate.yml`) runs on every push to `main`: `stryker plan` once over every workspace package that declares a `mutation` script, then, in each planned shard, every project's `stryker run --shard --project` child at a break threshold of 100, with its stdout and stderr kept beside its progress stream (`.github/scripts/run-shard.sh`). A survivor on `main` gets a behaviour test in a follow-up pull request, and the next gate run proves it. ## Pull Requests & Commits diff --git a/README.md b/README.md index bcda25f..b6f344e 100644 --- a/README.md +++ b/README.md @@ -23,16 +23,16 @@ Each package is also its own flake attribute, named after the last segment of it ## Toolchain -| Tool | Role | -| ----------------------- | ------------------------------------------------------------------------------------------------------------------- | -| **Nix** | Pins Node.js 24, pnpm 12.9.0, Deno and dprint, and builds the package tarballs | -| **Sandbox** | pnpm-release-management's deny-by-default launcher; all dependency code runs in it | -| **pnpm + Turbo** | Workspace catalog with exact pins, cached task graph | -| **TypeScript 7 (tsgo)** | Typechecking through `@effect/tsgo` | -| **oxlint** | The `@systemfsoftware/oxlint-config-recommended` preset, at error severity | -| **Vitest** | Unit and integration tests | -| **Stryker** | Mutation testing at a break threshold of 100, on `main` and scoped to each pull request's change (the release gate) | -| **dprint** | Formatting for code and Markdown | +| Tool | Role | +| ----------------------- | ---------------------------------------------------------------------------------- | +| **Nix** | Pins Node.js 24, pnpm 12.9.0, Deno and dprint, and builds the package tarballs | +| **Sandbox** | pnpm-release-management's deny-by-default launcher; all dependency code runs in it | +| **pnpm + Turbo** | Workspace catalog with exact pins, cached task graph | +| **TypeScript 7 (tsgo)** | Typechecking through `@effect/tsgo` | +| **oxlint** | The `@systemfsoftware/oxlint-config-recommended` preset, at error severity | +| **Vitest** | Unit and integration tests | +| **Stryker** | Mutation testing at a break threshold of 100, on `main` only (the release gate) | +| **dprint** | Formatting for code and Markdown | ## Contributing diff --git a/sandbox-proofs/release-gate.test.ts b/sandbox-proofs/release-gate.test.ts index 6726dc1..729ceb9 100644 --- a/sandbox-proofs/release-gate.test.ts +++ b/sandbox-proofs/release-gate.test.ts @@ -115,6 +115,8 @@ const UPLOAD_ARTIFACT = 'actions/upload-artifact@v6' const DOWNLOAD_ARTIFACT = 'actions/download-artifact@v6' const GLOB_SEGMENT = /[*?[]/ const JOB_INDEX = '${{ strategy.job-index }}' +const SHARD_RUNNER = '.github/scripts/run-shard.sh' +const SHARD_SELECTOR = 'stryker-plan-gate.ts shard' interface Plan { readonly version: number @@ -134,6 +136,15 @@ const flagOf = (command: string, flag: string): string => { return value } +const shardRunnerPlanOf = (command: string): string => { + const words = command.split(/\s+/) + const plan = words[words.indexOf(SHARD_RUNNER) + 1] + if (!words.includes(SHARD_RUNNER) || plan === undefined) { + throw new Error(`${JSON.stringify(command)} passes no plan to ${SHARD_RUNNER}`) + } + return plan +} + type StepInputs = NonNullable const onlyArtifactStep = async ( @@ -197,7 +208,7 @@ const downloadedPlanOf = async (job: JobName): Promise => { } const shardArtifactTree = async (plan: Plan): Promise> => { - const shardPlan = flagOf((await onlySandboxStep('mutation', 'stryker run')).command, '--plan') + const shardPlan = shardRunnerPlanOf((await onlySandboxStep('mutation', SHARD_RUNNER)).command) const downloadedPlan = await downloadedPlanOf('mutation') if (downloadedPlan !== join(shardPlan)) { throw new Error(`a shard job downloads the plan to ${downloadedPlan} but runs ${shardPlan}`) @@ -239,6 +250,10 @@ const writeFixture = async (): Promise => { for (const file of ['deno.json', 'deno.lock', 'stryker-plan-gate.ts']) { await Deno.copyFile(`${repoRoot}scripts/${file}`, `${root}/scripts/${file}`) } + await Deno.mkdir(`${root}/.github/scripts`, { recursive: true }) + for (const file of ['run-shard.sh', 'print-log-tails.sh']) { + await Deno.copyFile(`${repoRoot}.github/scripts/${file}`, `${root}/.github/scripts/${file}`) + } await Deno.writeTextFile(`${root}/pnpm-workspace.yaml`, 'packages:\n - packages/*\n') await Deno.writeTextFile( `${root}/debt-ledger.yaml`, @@ -282,9 +297,10 @@ Deno.test('each release-gate planner step runs in the sandbox on only its declar } }) -Deno.test('the release-gate mutation step passes stryker every variable its main-CI guard reads, and never the local override', async () => { +Deno.test('the release-gate shard steps pass stryker every variable its main-CI guard reads, never the local override, and leave each project stream where the shard artifact uploads it', async () => { const required = [...await guardEnvReads()].filter((name) => name !== GUARD_OVERRIDE) - const step = await onlySandboxStep('mutation', 'stryker run') + const select = await onlySandboxStep('mutation', SHARD_SELECTOR) + const step = await onlySandboxStep('mutation', SHARD_RUNNER) const missing = required.filter((name) => !step.passEnv.includes(name)) if (missing.length > 0 || step.passEnv.includes(GUARD_OVERRIDE)) { throw new Error( @@ -293,10 +309,23 @@ Deno.test('the release-gate mutation step passes stryker every variable its main } const root = await writeFixture() try { - await writeStrykerStub(root, 'env > .cache/stryker-env.txt\n') + await writeStrykerStub( + root, + [ + 'while [ $# -gt 0 ]; do', + ' case $1 in --plan) plan=$2; shift ;; --progressStreamFile) stream=$2; shift ;; esac', + ' shift', + 'done', + 'env > "$(dirname "$plan")/.cache/stryker-env.txt"', + ': > "$stream"', + '', + ].join('\n'), + ) const runner = Object.fromEntries([...required, GUARD_OVERRIDE].map((name) => [name, `runner-${name}`])) - const outcome = await inSandbox(step, root, { ...runner, MUTATION_SHARD: '1/1' }) - if (outcome.code !== 0) throw new Error(`"${step.name}" exited ${outcome.code}:\n${outcome.out}`) + for (const shardStep of [select, step]) { + const outcome = await inSandbox(shardStep, root, { ...runner, MUTATION_SHARD: '1/1' }) + if (outcome.code !== 0) throw new Error(`"${shardStep.name}" exited ${outcome.code}:\n${outcome.out}`) + } const seen = new Map( (await Deno.readTextFile(`${root}/.cache/stryker-env.txt`)).split('\n') .filter((line) => line.includes('=')) @@ -306,6 +335,11 @@ Deno.test('the release-gate mutation step passes stryker every variable its main if (lost.length > 0 || seen.has(GUARD_OVERRIDE)) { throw new Error(`stryker lost [${lost}]${seen.has(GUARD_OVERRIDE) ? ` and saw ${GUARD_OVERRIDE}` : ''}`) } + const marker = await strykerConstant('SHARD_OUT_MARKER') + const stream = join(root, dirname(shardRunnerPlanOf(step.command)), marker, '1', PROJECT, STREAM_FILE) + await Deno.stat(stream).catch(() => { + throw new Error(`the shard runner left no progress stream at ${stream}`) + }) } finally { await Deno.remove(root, { recursive: true }) } diff --git a/scripts/stryker-plan-gate.test.ts b/scripts/stryker-plan-gate.test.ts index cc8f9e6..1f61c5a 100644 --- a/scripts/stryker-plan-gate.test.ts +++ b/scripts/stryker-plan-gate.test.ts @@ -1,12 +1,11 @@ import { assertEquals } from '@std/assert' import { join } from '@std/path' -import { gatePlan, type Refusal, scopeOf, type ShardPlan } from './stryker-plan-gate.ts' +import { gatePlan, type Refusal, selectShardProjects, type ShardPlan } from './stryker-plan-gate.ts' interface FixturePackage { readonly dir: string readonly name: string readonly mutates: boolean - readonly strykerConfig?: string } interface FixtureOptions { @@ -28,7 +27,6 @@ const writeFixture = async (options: FixtureOptions): Promise<{ root: string; pl ...(pkg.mutates ? { scripts: { mutation: 'stryker run' } } : {}), } await Deno.writeTextFile(join(dir, 'package.json'), `${JSON.stringify(manifest, null, 2)}\n`) - if (pkg.strykerConfig !== undefined) await Deno.writeTextFile(join(dir, 'stryker.config.ts'), pkg.strykerConfig) } if (options.plan !== undefined) { await Deno.writeTextFile(join(root, 'plan.json'), `${JSON.stringify(options.plan)}\n`) @@ -70,11 +68,49 @@ Deno.test('a valid plan yields the plan matrix and has-shards', async () => { matrix: { include: [{ shard: '1/2', predictedSeconds: 5 }, { shard: '2/2', predictedSeconds: 3 }] }, hasShards: true, unmutated: [], - outOfScope: [], }, }) }) +Deno.test('a shard label selects that shard index and its projects in plan order', async () => { + const plan: ShardPlan = { + ...TWO_PROJECT_PLAN, + shards: [ + TWO_PROJECT_PLAN.shards[0]!, + { + index: 2, + count: 2, + predictedSeconds: 3, + projects: [ + { project: 'packages/site', mutants: ['fedcba9876543210'] }, + { project: 'packages/core', mutants: ['00112233445566ff'] }, + ], + }, + ], + } + const { root, planFile } = await writeFixture({ plan }) + assertEquals(await selectShardProjects({ root, planFile, shard: '2/2' }), { + ok: true, + selected: { index: 2, projects: ['packages/site', 'packages/core'] }, + }) +}) + +Deno.test('a shard label the plan does not carry is refused with the labels it does carry', async () => { + const { root, planFile } = await writeFixture({ plan: TWO_PROJECT_PLAN }) + assertEquals(await selectShardProjects({ root, planFile, shard: '2/3' }), { + ok: false, + refusal: { _tag: 'ShardUnknown', shard: '2/3', labels: ['1/2', '2/2'] }, + }) +}) + +Deno.test('shard selection without a plan file is refused', async () => { + const { root, planFile } = await writeFixture({}) + assertEquals(await selectShardProjects({ root, planFile, shard: '1/1' }), { + ok: false, + refusal: { _tag: 'PlanMissing', file: join(root, planFile) }, + }) +}) + Deno.test('a planned project that resolves outside the repository is refused (C13-13)', async () => { const { root, planFile } = await writeFixture({ packages: [{ dir: 'packages/core', name: '@fixture/core', mutates: true }], @@ -153,7 +189,6 @@ Deno.test('a mutation package with no scheduled mutants is allowed by a ledger e result: { matrix: { include: [{ shard: '1/1', predictedSeconds: 1 }] }, hasShards: true, - outOfScope: [], unmutated: [{ package: '@fixture/site', dir: 'packages/site', @@ -180,7 +215,6 @@ Deno.test('an empty plan is allowed by a ledger exemption', async () => { result: { matrix: { include: [] }, hasShards: false, - outOfScope: [], unmutated: [{ package: '@fixture/core', dir: 'packages/core', @@ -215,74 +249,3 @@ Deno.test('a mutation package without a plan is refused when a plan is required' assertEquals(outcome.ok, false) assertEquals(outcome.ok === false ? tagsOf(outcome.refusals) : [], ['PlanMissing']) }) - -const CORE = { dir: 'packages/core', mutate: ['src/a.ts', 'src/b.ts'] } -const SITE = { dir: 'packages/site', mutate: ['src/page.ts'] } - -Deno.test('a change scopes mutation to the declared files it touches and leaves untouched members out', () => { - assertEquals(scopeOf([CORE, SITE], ['packages/core/src/b.ts', 'README.md']), [ - { _tag: 'ChangedFiles', project: 'packages/core', files: ['src/b.ts'] }, - ]) -}) - -Deno.test('a change to a member file outside its declared set mutates that member whole', () => { - for (const file of ['tests/a.test.ts', 'src/helper.ts', 'src/a.test.ts', 'stryker.config.ts', 'package.json']) { - assertEquals(scopeOf([CORE, SITE], ['packages/core/src/a.ts', `packages/core/${file}`]), [ - { _tag: 'WholeSet', project: 'packages/core' }, - ], file) - } -}) - -Deno.test('a change to the gate mutates every member whole, touched or not', () => { - for ( - const file of ['.github/workflows/release-gate.yml', '.github/actions/sandbox/action.yml', 'stryker.shared.ts'] - ) { - assertEquals(scopeOf([CORE, SITE], [file]), [ - { _tag: 'WholeSet', project: 'packages/core' }, - { _tag: 'WholeSet', project: 'packages/site' }, - ], file) - } -}) - -Deno.test('a member directory that only prefixes another is not touched by it', () => { - assertEquals(scopeOf([CORE], ['packages/core-extra/src/a.ts']), []) -}) - -const CORE_CONFIG = "export default { mutate: ['src/a.ts'] }\n" - -Deno.test('a change that touches no mutated member passes the gate with no plan and names the members it left out', async () => { - const { root, planFile } = await writeFixture({ - packages: [{ dir: 'packages/core', name: '@fixture/core', mutates: true, strykerConfig: CORE_CONFIG }], - }) - assertEquals(await gatePlan({ root, planFile, changed: ['docs/notes.md'] }), { - ok: true, - result: { - matrix: { include: [] }, - hasShards: false, - unmutated: [], - outOfScope: [{ package: '@fixture/core', dir: 'packages/core' }], - }, - }) -}) - -Deno.test('a change still refuses a touched mutated member the plan scheduled nothing for', async () => { - const { root, planFile } = await writeFixture({ - packages: [ - { dir: 'packages/core', name: '@fixture/core', mutates: true, strykerConfig: CORE_CONFIG }, - { dir: 'packages/site', name: '@fixture/site', mutates: true, strykerConfig: CORE_CONFIG }, - ], - plan: { ...TWO_PROJECT_PLAN, shards: [TWO_PROJECT_PLAN.shards[1]!] }, - }) - const outcome = await gatePlan({ root, planFile, changed: ['packages/core/src/a.ts'] }) - assertEquals(outcome.ok ? outcome : outcome.refusals, [ - { _tag: 'MutationPackageWithoutMutants', package: '@fixture/core' }, - ]) -}) - -Deno.test('a change touching a member whose stryker config declares no mutate list is refused', async () => { - const { root, planFile } = await writeFixture({ - packages: [{ dir: 'packages/core', name: '@fixture/core', mutates: true, strykerConfig: 'export default {}\n' }], - }) - const outcome = await gatePlan({ root, planFile, changed: ['packages/core/src/a.ts'] }) - assertEquals(outcome.ok ? [] : tagsOf(outcome.refusals), ['MalformedStrykerConfig']) -}) diff --git a/scripts/stryker-plan-gate.ts b/scripts/stryker-plan-gate.ts index de7c2d4..df98615 100755 --- a/scripts/stryker-plan-gate.ts +++ b/scripts/stryker-plan-gate.ts @@ -1,7 +1,7 @@ -#!/usr/bin/env -S deno run --config=scripts/deno.json --allow-read --allow-write --allow-env=MUTATION_SCOPE +#!/usr/bin/env -S deno run --config=scripts/deno.json --allow-read --allow-write import { parseArgs } from '@std/cli/parse-args' -import { dirname, join, relative, resolve, toFileUrl } from '@std/path' +import { dirname, join, relative, resolve } from '@std/path' import { parse as parseYaml } from '@std/yaml' import { Schema as S } from 'effect' import * as Result from 'effect/Result' @@ -55,13 +55,10 @@ const LedgerEntry = S.Struct({ const Ledger = S.Struct({ entries: S.Array(LedgerEntry) }) -const StrykerConfigModule = S.Struct({ default: S.Struct({ mutate: S.Array(S.String) }) }) - const MUTATION_EXEMPTION_RULE = 'XS1' export type Refusal = | { readonly _tag: 'MalformedWorkspace'; readonly file: string } - | { readonly _tag: 'MalformedStrykerConfig'; readonly file: string; readonly reason: string } | { readonly _tag: 'MalformedManifest'; readonly file: string } | { readonly _tag: 'MalformedLedger'; readonly file: string } | { readonly _tag: 'PlanMissing'; readonly file: string } @@ -69,13 +66,12 @@ export type Refusal = | { readonly _tag: 'PlannedProjectEscapesRoot'; readonly project: string } | { readonly _tag: 'MutationPackageWithoutMutants'; readonly package: string } | { readonly _tag: 'VacuousPlan' } + | { readonly _tag: 'ShardUnknown'; readonly shard: string; readonly labels: readonly string[] } export const renderRefusal = (refusal: Refusal): string => { switch (refusal._tag) { case 'MalformedWorkspace': return `malformed pnpm-workspace.yaml (${refusal.file})` - case 'MalformedStrykerConfig': - return `stryker config ${refusal.file} has no default export with a \`mutate\` file list: ${refusal.reason}` case 'MalformedManifest': return `malformed package manifest (${refusal.file})` case 'MalformedLedger': @@ -90,6 +86,8 @@ export const renderRefusal = (refusal: Refusal): string => { return `${refusal.package}: declares a mutation script but stryker plan scheduled no mutants for it, and no ${MUTATION_EXEMPTION_RULE} debt-ledger entry exempts it` case 'VacuousPlan': return 'stryker plan scheduled no mutants and no debt-ledger entry exempts anything; the release gate refuses a vacuous set' + case 'ShardUnknown': + return `unknown shard ${refusal.shard}; plan has ${refusal.labels.join(', ')}` } } @@ -189,92 +187,6 @@ export const discoverMembers = async (root: string): Promise => { return { ok: true, members } } -export const GATE_FILES: readonly string[] = [ - '.github/workflows/release-gate.yml', - 'scripts/stryker-plan-gate.ts', - 'scripts/deno.json', - 'scripts/deno.lock', - 'stryker.shared.ts', - 'package.json', - 'pnpm-lock.yaml', - 'pnpm-workspace.yaml', - 'flake.nix', - 'flake.lock', -] - -const GATE_DIRECTORIES: readonly string[] = ['.github/actions/'] - -const changesGate = (file: string): boolean => - GATE_FILES.includes(file) || GATE_DIRECTORIES.some((directory) => file.startsWith(directory)) - -export interface ScopedMember { - readonly dir: string - readonly mutate: readonly string[] -} - -export type ProjectScope = - | { readonly _tag: 'WholeSet'; readonly project: string } - | { readonly _tag: 'ChangedFiles'; readonly project: string; readonly files: readonly string[] } - -export const scopeOf = (members: readonly ScopedMember[], changed: readonly string[]): readonly ProjectScope[] => - members.flatMap((member): ProjectScope[] => { - const whole: ProjectScope = { _tag: 'WholeSet', project: member.dir } - if (changed.some(changesGate)) return [whole] - const prefix = `${member.dir}/` - const touched = changed.filter((file) => file.startsWith(prefix)).map((file) => file.slice(prefix.length)) - if (touched.length === 0) return [] - return touched.every((file) => member.mutate.includes(file)) - ? [{ _tag: 'ChangedFiles', project: member.dir, files: touched }] - : [whole] - }) - -export const renderScope = (scope: readonly ProjectScope[]): string => - scope.flatMap((entry) => - entry._tag === 'WholeSet' ? [entry.project] : entry.files.map((file) => `${entry.project}/${file}`) - ).join(',') - -const declaredMutate = async ( - root: string, - member: Member, -): Promise> => { - const file = join(root, member.dir, 'stryker.config.ts') - try { - const decoded = S.decodeUnknownResult(StrykerConfigModule)(await import(toFileUrl(resolve(file)).href)) - return Result.isFailure(decoded) - ? Result.fail({ _tag: 'MalformedStrykerConfig', file, reason: decoded.failure.message }) - : Result.succeed({ dir: member.dir, mutate: decoded.success.default.mutate }) - } catch (error) { - return Result.fail({ _tag: 'MalformedStrykerConfig', file, reason: String(error) }) - } -} - -export type ScopeOutcome = - | { readonly ok: true; readonly scope: readonly ProjectScope[] } - | { readonly ok: false; readonly refusal: Refusal } - -export const resolveScope = async ( - root: string, - members: readonly Member[], - changed: readonly string[] | undefined, -): Promise => { - const mutating = members.filter((member) => member.mutates) - if (changed === undefined) { - return { ok: true, scope: mutating.map((member) => ({ _tag: 'WholeSet', project: member.dir })) } - } - const scoped: ScopedMember[] = [] - for (const member of mutating) { - const declared = await declaredMutate(root, member) - if (Result.isFailure(declared)) return { ok: false, refusal: declared.failure } - scoped.push(declared.success) - } - return { ok: true, scope: scopeOf(scoped, changed) } -} - -export const readChanged = async (file: string | undefined): Promise => { - const text = file === undefined ? undefined : await readText(file) - return text?.split('\n').map((line) => line.trim()).filter((line) => line !== '') -} - type LedgerRead = | { readonly ok: true; readonly entries: ReadonlyArray } | { readonly ok: false; readonly refusal: Refusal } @@ -317,7 +229,6 @@ const decodePlan = (file: string, text: string): PlanRead => { export interface GateInput { readonly root: string readonly planFile: string - readonly changed?: readonly string[] | undefined } export interface Unmutated { @@ -326,16 +237,10 @@ export interface Unmutated { readonly exemption: typeof LedgerEntry.Type | undefined } -export interface OutOfScope { - readonly package: string - readonly dir: string -} - export interface GateResult { readonly matrix: ShardPlan['matrix'] readonly hasShards: boolean readonly unmutated: readonly Unmutated[] - readonly outOfScope: readonly OutOfScope[] } export const renderUnmutated = (member: Unmutated): string => @@ -343,28 +248,19 @@ export const renderUnmutated = (member: Unmutated): string => ? `0 mutants for ${member.package} (${member.dir}): no mutation script and no ${MUTATION_EXEMPTION_RULE} debt-ledger entry` : `0 mutants for ${member.package} (${member.dir}): accepted by ${MUTATION_EXEMPTION_RULE} debt-ledger entry "${member.exemption.reason}", removed by ${member.exemption.removedBy}` -export const renderOutOfScope = (member: OutOfScope): string => - `0 mutants for ${member.package} (${member.dir}): the change touches none of its files` - export type GateOutcome = | { readonly ok: true; readonly result: GateResult } | { readonly ok: false; readonly refusals: readonly Refusal[] } -export const gatePlan = async ({ root, planFile, changed }: GateInput): Promise => { +export const gatePlan = async ({ root, planFile }: GateInput): Promise => { const discovery = await discoverMembers(root) if (!discovery.ok) return { ok: false, refusals: [discovery.refusal] } const ledger = await readLedger(root) if (!ledger.ok) return { ok: false, refusals: [ledger.refusal] } - const resolved = await resolveScope(root, discovery.members, changed) - if (!resolved.ok) return { ok: false, refusals: [resolved.refusal] } - const inScope = new Set(resolved.scope.map((entry) => entry.project)) const resolvedPlanFile = resolve(root, planFile) const planText = await readText(resolvedPlanFile) - const mutationMembers = discovery.members.filter((member) => member.mutates && inScope.has(member.dir)) - const outOfScope = discovery.members - .filter((member) => member.mutates && !inScope.has(member.dir)) - .map((member) => ({ package: member.name, dir: member.dir })) + const mutationMembers = discovery.members.filter((member) => member.mutates) if (planText === undefined && mutationMembers.length > 0) { return { ok: false, refusals: [{ _tag: 'PlanMissing', file: resolvedPlanFile }] } } @@ -396,23 +292,41 @@ export const gatePlan = async ({ root, planFile, changed }: GateInput): Promise< } const scheduledTotal = [...scheduled.values()].reduce((total, count) => total + count, 0) - const scopeIsEmpty = changed !== undefined && inScope.size === 0 - if (scheduledTotal === 0 && ledger.entries.length === 0 && !scopeIsEmpty) refusals.push({ _tag: 'VacuousPlan' }) + if (scheduledTotal === 0 && ledger.entries.length === 0) refusals.push({ _tag: 'VacuousPlan' }) if (refusals.length > 0) return { ok: false, refusals } const unmutated = discovery.members .filter((member) => (scheduled.get(member.dir) ?? 0) === 0) - .filter((member) => !outOfScope.some((out) => out.dir === member.dir)) .map((member) => ({ package: member.name, dir: member.dir, exemption: exemption(member.name) })) - return { - ok: true, - result: { matrix: plan.matrix, hasShards: plan.matrix.include.length > 0, unmutated, outOfScope }, - } + return { ok: true, result: { matrix: plan.matrix, hasShards: plan.matrix.include.length > 0, unmutated } } +} + +export interface ShardProjects { + readonly index: number + readonly projects: readonly string[] +} + +export type ShardSelection = + | { readonly ok: true; readonly selected: ShardProjects } + | { readonly ok: false; readonly refusal: Refusal } + +export const selectShardProjects = async ( + { root, planFile, shard }: { readonly root: string; readonly planFile: string; readonly shard: string }, +): Promise => { + const resolvedPlanFile = resolve(root, planFile) + const planText = await readText(resolvedPlanFile) + if (planText === undefined) return { ok: false, refusal: { _tag: 'PlanMissing', file: resolvedPlanFile } } + const read = decodePlan(resolvedPlanFile, planText) + if (!read.ok) return read + const found = read.plan.shards.find((candidate) => labelOf(candidate) === shard) + return found === undefined + ? { ok: false, refusal: { _tag: 'ShardUnknown', shard, labels: read.plan.shards.map(labelOf) } } + : { ok: true, selected: { index: found.index, projects: found.projects.map((entry) => entry.project) } } } if (import.meta.main) { const args = parseArgs(Deno.args, { - string: ['root', 'plan', 'out', 'changed', 'scope-out'], + string: ['root', 'plan', 'out', 'shard'], default: { root: '.' }, }) const mode = args._[0] @@ -420,7 +334,6 @@ if (import.meta.main) { if (args.out === undefined) console.log(text.trimEnd()) else await Deno.writeTextFile(args.out, text) } - const changed = await readChanged(args.changed) if (mode === 'projects') { const discovery = await discoverMembers(args.root) @@ -428,14 +341,23 @@ if (import.meta.main) { console.error(`stryker-plan-gate: ${renderRefusal(discovery.refusal)}`) Deno.exit(1) } - const resolved = await resolveScope(args.root, discovery.members, changed) - if (!resolved.ok) { - console.error(`stryker-plan-gate: ${renderRefusal(resolved.refusal)}`) - Deno.exit(1) - } - const projects = resolved.scope.map((entry) => entry.project).sort() + const projects = discovery.members.filter((member) => member.mutates).map((member) => member.dir).sort() await emit(`${projects.join(',')}\n`) - if (args['scope-out'] !== undefined) await Deno.writeTextFile(args['scope-out'], `${renderScope(resolved.scope)}\n`) + Deno.exit(0) + } + + if (mode === 'shard') { + if (args.plan === undefined || args.shard === undefined) { + console.error('stryker-plan-gate: shard needs --plan --shard ') + Deno.exit(2) + } + const selection = await selectShardProjects({ root: args.root, planFile: args.plan, shard: args.shard }) + if (!selection.ok) { + console.error(`stryker-plan-gate: ${renderRefusal(selection.refusal)}`) + Deno.exit(2) + } + const { index, projects } = selection.selected + await emit(projects.map((project) => `${index}\t${project}\n`).join('')) Deno.exit(0) } @@ -443,13 +365,12 @@ if (import.meta.main) { console.error('stryker-plan-gate: gate needs --plan ') Deno.exit(2) } - const outcome = await gatePlan({ root: args.root, planFile: args.plan, changed }) + const outcome = await gatePlan({ root: args.root, planFile: args.plan }) if (!outcome.ok) { for (const refusal of outcome.refusals) console.error(`stryker-plan-gate: ${renderRefusal(refusal)}`) Deno.exit(1) } for (const member of outcome.result.unmutated) console.error(`stryker-plan-gate: ${renderUnmutated(member)}`) - for (const member of outcome.result.outOfScope) console.error(`stryker-plan-gate: ${renderOutOfScope(member)}`) console.error(`stryker-plan-gate: ${outcome.result.matrix.include.length} shard(s)`) await emit(`matrix=${JSON.stringify(outcome.result.matrix)}\nhas-shards=${outcome.result.hasShards}\n`) Deno.exit(0) diff --git a/stryker.shared.ts b/stryker.shared.ts index c051663..8f51b7c 100644 --- a/stryker.shared.ts +++ b/stryker.shared.ts @@ -1,37 +1,29 @@ import type { StrykerConfig } from '@systemfsoftware/stryker-js/config' -import { relative } from 'node:path' -import { fileURLToPath } from 'node:url' const slackForAContendedFullSuiteRunMs = 45_000 - -const repoRoot = fileURLToPath(new URL('.', import.meta.url)) - -const scopedMutate = (mutate: ReadonlyArray, scope: string | undefined): string[] => { - if (scope === undefined) return [...mutate] - const entries = scope.split(',') - const project = relative(repoRoot, process.cwd()).replaceAll('\\', '/') - return entries.includes(project) ? [...mutate] : mutate.filter((file) => entries.includes(`${project}/${file}`)) -} +const heapPerTestRunnerIsolateMb = 2048 export const packageStrykerConfig = (mutate: ReadonlyArray): StrykerConfig => ({ checkers: [{ plugin: '@systemfsoftware/stryker-js-typescript-checker' }], + // At the default of 4 (2 checkers, each driving tsgo), shards on the 16 GB hosted runner exited 3 or lost the VM. + concurrency: 2, coverageAnalysis: 'perTest', disableBail: true, htmlReporter: { fileName: 'reports/mutation-report.html' }, - fileLogLevel: 'info', ignorePatterns: ['reports', 'coverage', 'dist'], incremental: true, incrementalFile: 'reports/stryker-incremental.json', ignorers: ['@systemfsoftware/stryker-ignorer-effect-schema-declarations'], jsonReporter: { fileName: 'reports/mutation-report.json' }, - mutate: scopedMutate(mutate, process.env['MUTATION_SCOPE']), + mutate: [...mutate], packageManager: 'pnpm', reporters: ['progress', 'html', 'json', 'progress-stream'], testRunner: { plugin: '@systemfsoftware/stryker-js-vitest-runner', options: { configFile: 'vitest.config.ts', dir: '.', related: true }, }, + testRunnerNodeArgs: [`--max-old-space-size=${heapPerTestRunnerIsolateMb}`], timeoutMS: slackForAContendedFullSuiteRunMs, thresholds: { break: 100, high: 100, low: 100 }, }) satisfies StrykerConfig