run-evals.js

#!/usr/bin/env node
// @ts-check
/**
 * Pack the locally-built `next` package and run agent evals against it.
 *
 *   pnpm eval <eval-name>             run one eval and its configured variants
 *   pnpm eval <eval-name> --dry       preview without executing
 *   pnpm eval --all                   run every eval (slow — normally only CI does this)
 *   NEXT_SKIP_PACK=1 pnpm eval ...    reuse tarball from last run
 *
 * Mirrors run-tests.js: pack once, hand paths to child via env, forward args.
 *
 * We only pack `next`, not the whole workspace. The sandbox is remote Linux:
 *   - @next/swc: local darwin binary wouldn't run there; the sandbox downloads
 *     the right one at runtime (packages/next/src/build/swc/index.ts).
 *   - @next/env etc: resolved from npm at the pinned canary version.
 *
 * The experiments/ dir is generated fresh on every run and gitignored. This
 * keeps the variants in one place instead of maintaining N committed
 * experiment files that only differ by setup.
 */
const path = require('path')
const fs = require('fs')
const { execFileSync, spawnSync } = require('child_process')

const ROOT = __dirname

const EVALS_DIR = path.join(ROOT, 'evals')
const FIXTURES_DIR = path.join(EVALS_DIR, 'evals')
const EVAL_CONFIG_PATH = path.join(EVALS_DIR, 'eval.config.json')
const EXPERIMENTS_DIR = path.join(EVALS_DIR, 'experiments')
const TARBALL_DIR = path.join(EVALS_DIR, '.tarballs')
const TARBALL = path.join(TARBALL_DIR, 'next.tgz')

/** @typedef {{ skills?: string[], timeout?: number }} EvalConfig */
/** @type {Record<string, EvalConfig>} */
const EVAL_CONFIG = JSON.parse(fs.readFileSync(EVAL_CONFIG_PATH, 'utf-8'))

// The two variants we always compare. Order matters for output readability:
// baseline first so a contributor sees "does the agent fail without docs?"
// before "does it pass with docs?".
const BASE_VARIANTS = [
  {
    suffix: 'baseline',
    imports: `import { installNextJs } from '../lib/setup.js'`,
    setup: `await installNextJs(sandbox)`,
  },
  {
    suffix: 'agents-md',
    imports: `import { installNextJs, writeAgentsMd } from '../lib/setup.js'`,
    setup: `await installNextJs(sandbox)\n    await writeAgentsMd(sandbox)`,
  },
]

function pack() {
  fs.mkdirSync(TARBALL_DIR, { recursive: true })
  const out = execFileSync(
    'pnpm',
    ['pack', '--pack-destination', TARBALL_DIR],
    { cwd: path.join(ROOT, 'packages/next'), encoding: 'utf8' }
  )
  const produced = out.trim().split('\n').pop()
  const src = path.isAbsolute(produced)
    ? produced
    : path.join(TARBALL_DIR, produced)
  fs.renameSync(src, TARBALL)
}

/** @param {string | null} evalName  null means all evals */
function writeExperiments(evalName, variants, timeout) {
  fs.rmSync(EXPERIMENTS_DIR, { recursive: true, force: true })
  fs.mkdirSync(EXPERIMENTS_DIR, { recursive: true })

  for (const v of variants) {
    const selectedEvals = v.evals ?? (evalName ? [evalName] : null)
    const evalsField = selectedEvals
      ? `\n  evals: ${JSON.stringify(selectedEvals.length === 1 ? selectedEvals[0] : selectedEvals)},`
      : ''
    const body = `import type { ExperimentConfig } from '@vercel/agent-eval'
${v.imports}

const config: ExperimentConfig = {
  // Via the Vercel AI Gateway, so the OIDC token from \`vc env pull\` is the only
  // credential needed (it auths the sandbox, the codegen model, and the judge).
  agent: 'vercel-ai-gateway/claude-code',
  model: 'claude-opus-4-8',${evalsField}
  // Cheap fixed grader for the agentic judge clauses in EVAL.ts files — every
  // run is graded by the same model regardless of the model under test.
  judge: { model: 'claude-haiku-4-5' },
  scripts: ['build'],
  runs: 1,
  earlyExit: true,
  timeout: ${timeout},
  sandbox: 'auto',
  setup: async (sandbox) => {
    ${v.setup}
  },
}

export default config
`
    fs.writeFileSync(path.join(EXPERIMENTS_DIR, `${v.suffix}.ts`), body)
  }
}

function listEvals() {
  return fs
    .readdirSync(FIXTURES_DIR, { withFileTypes: true })
    .filter((d) => d.isDirectory())
    .map((d) => d.name)
}

function readFixtureConfig(evalName) {
  const config = EVAL_CONFIG[evalName] ?? {}
  const skillNames = config.skills ?? []
  if (
    !Array.isArray(skillNames) ||
    skillNames.some((name) => typeof name !== 'string')
  ) {
    throw new Error(
      `${EVAL_CONFIG_PATH}: ${evalName}.skills must be an array of skill names`
    )
  }
  if (
    config.timeout !== undefined &&
    (typeof config.timeout !== 'number' || config.timeout <= 0)
  ) {
    throw new Error(
      `${EVAL_CONFIG_PATH}: ${evalName}.timeout must be a positive number`
    )
  }
  return { skills: skillNames, timeout: config.timeout ?? 720 }
}

function getExperimentSettings(evalName) {
  const skillEvals = (evalName ? [evalName] : listEvals()).map((name) => ({
    name,
    ...readFixtureConfig(name),
  }))
  const timeout = Math.max(...skillEvals.map((config) => config.timeout))
  const configuredSkillEvals = skillEvals.filter(
    ({ skills }) => skills.length > 0
  )

  if (configuredSkillEvals.length === 0) {
    return { variants: BASE_VARIANTS, timeout }
  }

  /** @type {Map<string, { skills: string[], evals: string[] }>} */
  const skillGroups = new Map()
  for (const { name, skills } of configuredSkillEvals) {
    const skillNames = [...new Set(skills)].sort()
    const key = skillNames.join(',')
    const group = skillGroups.get(key) ?? { skills: skillNames, evals: [] }
    group.evals.push(name)
    skillGroups.set(key, group)
  }

  const multipleSkillGroups = skillGroups.size > 1
  const skillVariants = [...skillGroups.values()].map(({ skills, evals }) => ({
    suffix: multipleSkillGroups ? `skills-${skills.join('-')}` : 'skills',
    imports: `import { installLocalSkills, installNextJs } from '../lib/setup.js'`,
    setup: `await installNextJs(sandbox)\n    await installLocalSkills(sandbox, ${JSON.stringify(skills)})`,
    evals,
  }))

  return {
    timeout,
    variants: [...BASE_VARIANTS, ...skillVariants],
  }
}

function main() {
  const argv = require('yargs/yargs')(process.argv.slice(2))
    .command(
      '$0 [eval-name]',
      'Run an eval (baseline + agents-md variants)',
      (y) =>
        y.positional('eval-name', {
          type: 'string',
          describe: 'Fixture directory name',
        })
    )
    .boolean('all')
    .describe('all', 'Run every eval (slow — normally only CI does this)')
    .boolean('dry')
    .describe('dry', 'Preview without executing')
    .conflicts('all', 'eval-name')
    .check((argv) => {
      if (!argv.all && !argv.evalName) {
        throw new Error(
          `Missing <eval-name>.\n\nAvailable evals:\n${listEvals()
            .map((n) => `  ${n}`)
            .join('\n')}`
        )
      }
      if (
        argv.evalName &&
        !fs.existsSync(path.join(FIXTURES_DIR, argv.evalName))
      ) {
        throw new Error(
          `Unknown eval: ${argv.evalName}\n(looked in ${FIXTURES_DIR})`
        )
      }
      return true
    })
    .strict()
    .help().argv

  /** @type {string | null} */
  const evalName = argv.all ? null : /** @type {string} */ (argv.evalName)
  const { variants, timeout } = getExperimentSettings(evalName)
  // agent-eval 1.3 dropped run-all/--dry: `run` takes explicit experiment names,
  // and `status` is the read-only preview.
  const agentEvalArgs = argv.dry
    ? ['status']
    : ['run', ...variants.map((v) => v.suffix), '--force']

  if (!fs.existsSync(path.join(ROOT, 'packages/next/dist'))) {
    console.error(
      'packages/next/dist not found. Run `pnpm --filter=next build` first.'
    )
    process.exit(1)
  }

  if (process.env.NEXT_SKIP_PACK && fs.existsSync(TARBALL)) {
    console.log('> Reusing existing tarball (NEXT_SKIP_PACK=1)')
  } else {
    console.log('> Packing next...')
    pack()
    const mb = (fs.statSync(TARBALL).size / 1024 / 1024).toFixed(1)
    console.log(`  ${TARBALL} (${mb} MB)`)
  }

  // agent-eval loads .env / .env.local from its own cwd (evals/). `vc env pull`
  // writes to the repo root, so symlink them into evals/ for agent-eval to find.
  for (const envFile of ['.env', '.env.local']) {
    const src = path.join(ROOT, envFile)
    const dest = path.join(EVALS_DIR, envFile)
    try {
      // Remove stale symlink or file before creating a fresh one.
      fs.rmSync(dest, { force: true })
      if (fs.existsSync(src)) {
        fs.symlinkSync(src, dest)
      }
    } catch {}
  }

  writeExperiments(evalName, variants, timeout)
  console.log(
    evalName
      ? `> Running ${evalName} (${variants.map((v) => v.suffix).join(' + ')})`
      : `> Running all evals (${variants.map((v) => v.suffix).join(' + ')})`
  )

  // Same handoff pattern as run-tests.js with NEXT_TEST_PKG_PATHS. We invoke
  // the bin directly rather than via `pnpm exec` because pnpm resets cwd to
  // the workspace root, but agent-eval resolves experiments/ from process.cwd().
  const bin = path.join(ROOT, 'node_modules/.bin/agent-eval')
  const result = spawnSync(bin, agentEvalArgs, {
    cwd: EVALS_DIR,
    stdio: 'inherit',
    env: { ...process.env, NEXT_EVAL_TARBALL: TARBALL },
  })
  if (result.error) {
    // ENOENT (missing bin), EACCES, etc. — spawnSync returns status: null
    // without printing anything, so surface it.
    console.error(`Failed to run ${bin}: ${result.error.message}`)
    if (/** @type {NodeJS.ErrnoException} */ (result.error).code === 'ENOENT') {
      console.error('Did you run `pnpm install`?')
    }
    process.exit(1)
  }
  process.exit(result.status ?? 1)
}

main()