|
| 1 | +"""The regression gate: a pull request's benchmarks held to those of its base. |
| 2 | +
|
| 3 | +``python -m benchmarks.gate BASE.json HEAD.json [--summary FILE] [--fail-timing]`` |
| 4 | +
|
| 5 | +Both files are written by one job on one machine, from the base of the pull request and |
| 6 | +from its head, with ``--blocks overhead,memory``. Two kinds of number are held: |
| 7 | +
|
| 8 | +- **Memory, exactly.** Every Python-allocation cell of the memory block (tracemalloc, |
| 9 | + the same on every run) equals the base's within :data:`MEMORY_SLACK`: a new copy in |
| 10 | + Python fails. The peak resident rise of a batch stays within :data:`BATCH_FACTOR` |
| 11 | + times libphash's bound for its threads. |
| 12 | +- **Time, against C.** A runner's speed varies from job to job and from minute to |
| 13 | + minute, so no absolute time is compared: each cell of an overhead table is divided by |
| 14 | + the C cell of its table (its baseline row, or its ``C`` column), and that ratio is |
| 15 | + compared with the base's. A ratio that grows by more than :data:`WARN_CHEAP` (a call |
| 16 | + whose C cell takes under :data:`CHEAP` seconds) or :data:`WARN_WORK` (a load or a |
| 17 | + hash of a whole image) is reported as a warning; with ``--fail-timing``, growth |
| 18 | + beyond :data:`FAIL_CHEAP` or :data:`FAIL_WORK` fails. Rows that time C itself, and |
| 19 | + tables without a C cell, are published, not gated. |
| 20 | +
|
| 21 | +A cell that only one side has (a case added or removed) is skipped. The exit status is |
| 22 | +1 when a check fails and 0 otherwise; GitHub annotations go to standard output, and a |
| 23 | +Markdown summary to ``--summary``. |
| 24 | +""" |
| 25 | + |
| 26 | +from __future__ import annotations |
| 27 | + |
| 28 | +import argparse |
| 29 | +import json |
| 30 | +import sys |
| 31 | +from dataclasses import dataclass |
| 32 | +from pathlib import Path |
| 33 | +from typing import TYPE_CHECKING, Any, cast |
| 34 | + |
| 35 | +if TYPE_CHECKING: |
| 36 | + from collections.abc import Iterator, Sequence |
| 37 | + |
| 38 | +#: How far a Python-allocation cell may move, in bytes: allocator rounding, not a copy. |
| 39 | +MEMORY_SLACK = 1024 |
| 40 | +#: The bound on a batch's peak resident rise, as a multiple of libphash's estimate. |
| 41 | +BATCH_FACTOR = 1.1 |
| 42 | +#: A C cell faster than this, in seconds, is a cheap call, where cffi's share is large. |
| 43 | +CHEAP = 10e-6 |
| 44 | +WARN_CHEAP = 0.15 |
| 45 | +WARN_WORK = 0.05 |
| 46 | +FAIL_CHEAP = 0.25 |
| 47 | +FAIL_WORK = 0.10 |
| 48 | + |
| 49 | +Table = dict[str, Any] |
| 50 | + |
| 51 | + |
| 52 | +@dataclass(frozen=True) |
| 53 | +class Ratio: |
| 54 | + """One timed cell, against its C cell, at the base and at the head.""" |
| 55 | + |
| 56 | + table: str |
| 57 | + row: str |
| 58 | + column: str |
| 59 | + base: float |
| 60 | + head: float |
| 61 | + cheap: bool |
| 62 | + |
| 63 | + @property |
| 64 | + def growth(self) -> float: |
| 65 | + """Return how much the ratio grew, 0.1 for 10 %.""" |
| 66 | + return self.head / self.base - 1 |
| 67 | + |
| 68 | + def exceeds(self, *, fail: bool) -> bool: |
| 69 | + """Say whether the growth passes the warning or the failure threshold.""" |
| 70 | + if fail: |
| 71 | + return self.growth > (FAIL_CHEAP if self.cheap else FAIL_WORK) |
| 72 | + return self.growth > (WARN_CHEAP if self.cheap else WARN_WORK) |
| 73 | + |
| 74 | + |
| 75 | +def tables(result: dict[str, Any], block: str) -> dict[str, Table]: |
| 76 | + """Return the tables of one block of a result, by key.""" |
| 77 | + for b in result["blocks"]: |
| 78 | + if b["key"] == block: |
| 79 | + return {t["key"]: t for t in b["tables"]} |
| 80 | + return {} |
| 81 | + |
| 82 | + |
| 83 | +def value(table: Table, row: str, column: str) -> float | None: |
| 84 | + """Return a cell's value, or None where the table has no such cell.""" |
| 85 | + cell = table["rows"].get(row, {}).get(column) |
| 86 | + return None if cell is None else float(cell["value"]) |
| 87 | + |
| 88 | + |
| 89 | +def memory_failures(base: dict[str, Any], head: dict[str, Any]) -> Iterator[str]: |
| 90 | + """Yield a line for each memory cell that breaks its bound.""" |
| 91 | + before = tables(base, "memory") |
| 92 | + for key, table in tables(head, "memory").items(): |
| 93 | + for column in table["columns"]: |
| 94 | + if "Python" not in column: |
| 95 | + continue |
| 96 | + for row in table["rows"]: |
| 97 | + b = value(before.get(key, {"rows": {}}), row, column) |
| 98 | + h = value(table, row, column) |
| 99 | + if b is not None and h is not None and abs(h - b) > MEMORY_SLACK: |
| 100 | + yield ( |
| 101 | + f"{table['title']} / {row} / {column}: " |
| 102 | + f"{h:,.0f} B, the base {b:,.0f} B" |
| 103 | + ) |
| 104 | + if key == "batch_memory": |
| 105 | + for column in table["columns"]: |
| 106 | + rise = value(table, "peak resident rise", column) |
| 107 | + bound = value(table, "libphash's bound", column) |
| 108 | + if rise is None or bound is None or rise <= BATCH_FACTOR * bound: |
| 109 | + continue |
| 110 | + yield ( |
| 111 | + f"{table['title']} / {column}: peak resident rise {rise:,.0f} B, " |
| 112 | + f"over {BATCH_FACTOR} times libphash's bound of {bound:,.0f} B" |
| 113 | + ) |
| 114 | + |
| 115 | + |
| 116 | +def ratios(base: dict[str, Any], head: dict[str, Any]) -> Iterator[Ratio]: |
| 117 | + """Yield each timed cell of the overhead block that both sides have.""" |
| 118 | + before = tables(base, "overhead") |
| 119 | + for key, table in tables(head, "overhead").items(): |
| 120 | + if table["unit"] != "s" or key not in before: |
| 121 | + continue |
| 122 | + for row in table["rows"]: |
| 123 | + if row == "C" or row.startswith("C:"): |
| 124 | + continue |
| 125 | + for column in table["columns"]: |
| 126 | + if column == "C": |
| 127 | + continue |
| 128 | + b, h = (_ratio(t, row, column) for t in (before[key], table)) |
| 129 | + if b is None or h is None: |
| 130 | + continue |
| 131 | + yield Ratio(table["title"], row, column, b[0], h[0], b[1] < CHEAP) |
| 132 | + |
| 133 | + |
| 134 | +def _ratio(table: Table, row: str, column: str) -> tuple[float, float] | None: |
| 135 | + """Return a cell over its C cell, and the C cell; None without either.""" |
| 136 | + if table["baseline_row"]: |
| 137 | + if row == table["baseline_row"]: |
| 138 | + return None |
| 139 | + reference = value(table, table["baseline_row"], column) |
| 140 | + elif "C" in table["columns"]: |
| 141 | + reference = value(table, row, "C") |
| 142 | + else: |
| 143 | + return None |
| 144 | + cell = value(table, row, column) |
| 145 | + if cell is None or not reference: |
| 146 | + return None |
| 147 | + return cell / reference, reference |
| 148 | + |
| 149 | + |
| 150 | +def main(argv: Sequence[str] | None = None) -> int: |
| 151 | + """Compare the two results; return 1 when a check fails.""" |
| 152 | + parser = argparse.ArgumentParser( |
| 153 | + prog="python -m benchmarks.gate", |
| 154 | + description="Hold the benchmarks of a pull request to those of its base.", |
| 155 | + ) |
| 156 | + parser.add_argument("base", type=Path, help="the base's results (JSON)") |
| 157 | + parser.add_argument("head", type=Path, help="the head's results (JSON)") |
| 158 | + parser.add_argument("--summary", type=Path, help="write a Markdown summary here") |
| 159 | + parser.add_argument( |
| 160 | + "--fail-timing", action="store_true", help="fail on timing, not only warn" |
| 161 | + ) |
| 162 | + args = parser.parse_args(argv) |
| 163 | + base = cast("dict[str, Any]", json.loads(args.base.read_text(encoding="utf-8"))) |
| 164 | + head = cast("dict[str, Any]", json.loads(args.head.read_text(encoding="utf-8"))) |
| 165 | + |
| 166 | + memory = list(memory_failures(base, head)) |
| 167 | + timed = list(ratios(base, head)) |
| 168 | + failing = [r for r in timed if args.fail_timing and r.exceeds(fail=True)] |
| 169 | + slower = [r for r in timed if r.exceeds(fail=False)] |
| 170 | + |
| 171 | + for line in memory: |
| 172 | + print(f"::error title=Memory::{line}") # noqa: T201 -- an annotation |
| 173 | + for r in slower: |
| 174 | + level = "error" if r in failing else "warning" |
| 175 | + print( # noqa: T201 -- an annotation |
| 176 | + f"::{level} title=Time::{r.table} / {r.row} / {r.column}: " |
| 177 | + f"{r.growth:+.0%} against C" |
| 178 | + ) |
| 179 | + if args.summary: |
| 180 | + args.summary.write_text(_summary(memory, timed, slower), encoding="utf-8") |
| 181 | + return 1 if memory or failing else 0 |
| 182 | + |
| 183 | + |
| 184 | +def _summary(memory: list[str], timed: list[Ratio], slower: list[Ratio]) -> str: |
| 185 | + lines = ["## Benchmarks against the base", "", "### Memory", ""] |
| 186 | + lines += [f"- {m}" for m in memory] or ["Every Python allocation as at the base."] |
| 187 | + lines += ["", "### Time against C", ""] |
| 188 | + if slower: |
| 189 | + lines += ["| Table | Row | Column | Growth |", "|---|---|---|---|"] |
| 190 | + lines += [ |
| 191 | + f"| {r.table} | {r.row} | {r.column} | {r.growth:+.1%} |" for r in slower |
| 192 | + ] |
| 193 | + else: |
| 194 | + lines.append(f"No ratio of the {len(timed)} grew past its threshold.") |
| 195 | + return "\n".join(lines) + "\n" |
| 196 | + |
| 197 | + |
| 198 | +if __name__ == "__main__": |
| 199 | + sys.exit(main()) |
0 commit comments