feat(grafana-alertcheck): add stop subcommand to reap a detached recorder

grafana-alertcheck/.changeset/v0.1.2.md

@@ -0,0 +1,2 @@
+- Add the `stop` subcommand: reap a detached recorder after a failed work step. It is idempotent, so it is safe as an `if: always()` step.
+- `check` now exits early (fail-fast) on a condition that cannot become a pass — a post-`from` bad onset or an inability — instead of always waiting for `to + transitionGrace + drainTimeout`. An early exit is never a pass; pass `--no-fail-fast` to always wait for the full window and its coverage proof.

grafana-alertcheck/.changeset/v0.1.3.md

@@ -0,0 +1,2 @@
+- The `check` output now speaks plain words. Outcome values are renamed: `clean` → `healthy`, `newly_bad` → `new_failure`, `persistently_bad` → `still_failing`, `flapping` → `unstable`, `skipped` → `paused`, `unobservable` → `not_verified`, and the early-exit `terminated_early.kind` follows the same rename. A `--min-observed` deficit that no rule explains is now reported as `not_counted` instead of being blamed on a paused rule. This changes the `--output json` vocabulary and anything downstream of it, including the action's `outcomes` output.
+- The human table dropped its internal column names. `RESULTS` is now `ALERT`/`VERDICT`/`BROKEN FOR`/`CHECKED EVERY`/`WINDOW COVERED`/`DETAILS`; `VIOLATIONS` uses `GRAFANA STATE`/`GRAFANA HEALTH` and a single-word `INSTANCES` column (the previous `INSTANCE COUNT` header read as two columns, one of them empty); `THRESHOLDS` became `LIMITS USED`, with limits named in plain words and explained by a legend under the table. The footer now spells out the extra observation time, the evaluation wait and the clock difference from Grafana.

grafana-alertcheck/README.md

@@ -9,7 +9,7 @@ watch  → your work → check
 
 `watch` starts a background recorder that polls each named alert into a JSONL log. After the work emits a
 `from`/`to` pair, `check` proves continuous coverage of that window, classifies each alert's state
-timeline, and exits `0`, `1`, or `2`.
+timeline, and exits `0`, `1`, or `2`. If the work fails first, `stop` reaps the recorder.
 
 It **fails closed**: if it cannot get an answer, it stops the release — never a pass on an unproven window.
 

grafana-alertcheck/cmd/check.go

@@ -16,7 +16,7 @@ import (
 
 const checkUsage = "usage: grafana-alertcheck check [--in <file>] [--pidfile F] --from RFC3339 --to RFC3339 " +
 	"[--alerts ...] [--folder F] [--states ...] [--preexisting ...] [--min-observed N] [--allow-paused] " +
-	"[--nodata-is-unobservable] [--concurrency N] [--output json]"
+	"[--nodata-is-unobservable] [--no-fail-fast] [--concurrency N] [--output json]"
 
 // runCheck is the classify step's CLI surface: parse flags into a gate.Config,
 // run gate.Check, and translate its (Result, error) into output and an exit
@@ -38,6 +38,7 @@ func runCheck(args []string, stdin io.Reader, stdout, stderr io.Writer) int {
 	minObserved := fs.Int("min-observed", 0, "minimum rules that must be observed (default: every resolved rule)")
 	allowPaused := fs.Bool("allow-paused", false, "do not count a rule paused before the window against --min-observed")
 	nodataIsUnobservable := fs.Bool("nodata-is-unobservable", false, "treat a sustained health=nodata as unobservable rather than a note")
+	noFailFast := fs.Bool("no-fail-fast", false, "collect to to+transitionGrace even after a certain failure, for a full-window coverage proof instead of the fastest feedback")
 	output := fs.String("output", "", `"json" writes the machine-readable Result to stdout in addition to the table; default is the table alone`)
 
 	if err := fs.Parse(args); err != nil {
@@ -86,6 +87,7 @@ func runCheck(args []string, stdin io.Reader, stdout, stderr io.Writer) int {
 		MinObserved:          *minObserved,
 		AllowPaused:          *allowPaused,
 		NodataIsUnobservable: *nodataIsUnobservable,
+		NoFailFast:           *noFailFast,
 		Log:                  *in,
 		PidFile:              *pidfile,
 		Concurrency:          *common.concurrency,

grafana-alertcheck/cmd/main.go

@@ -12,7 +12,7 @@ func main() {
 	os.Exit(run(os.Args[1:], os.Stdout, os.Stderr))
 }
 
-const usage = "usage: grafana-alertcheck <list|watch|check>"
+const usage = "usage: grafana-alertcheck <list|watch|check|stop>"
 
 // run is the whole of main's testable surface: parse the subcommand, dispatch,
 // return the process exit code. Exit codes below 2 (pass/violations) belong to
@@ -37,6 +37,8 @@ func run(args []string, stdout, stderr io.Writer) int {
 		return runWatch(args[1:], os.Stdin, stdout, stderr)
 	case "check":
 		return runCheck(args[1:], os.Stdin, stdout, stderr)
+	case "stop":
+		return runStop(args[1:], os.Stdin, stdout, stderr)
 	case "-h", "-help", "--help":
 		fmt.Fprintln(stdout, usage)
 		return 0

grafana-alertcheck/cmd/stop.go

@@ -0,0 +1,63 @@
+package main
+
+import (
+	"context"
+	"errors"
+	"flag"
+	"fmt"
+	"io"
+	"os/signal"
+	"syscall"
+
+	"github.com/smartcontractkit/chainlink-testing-framework/grafana-alertcheck/internal/gate"
+)
+
+const stopUsage = "usage: grafana-alertcheck stop --out <file> [--pidfile F]"
+
+// runStop reaps a detached recorder without reading or classifying its log —
+// what an `if: always()` step calls when the work failed. It is idempotent, so
+// it is a no-op after check has already stopped the recorder.
+func runStop(args []string, _ io.Reader, _, stderr io.Writer) int {
+	fs := flag.NewFlagSet("stop", flag.ContinueOnError)
+	fs.SetOutput(stderr)
+	fs.Usage = func() { fmt.Fprintln(stderr, stopUsage) }
+
+	out := fs.String("out", "", "JSONL log path whose recorder to stop")
+	pidfile := fs.String("pidfile", "", "pidfile path (default <out>.pid)")
+
+	if err := fs.Parse(args); err != nil {
+		if errors.Is(err, flag.ErrHelp) {
+			return 0
+		}
+		return 2
+	}
+	if fs.NArg() != 0 {
+		fmt.Fprintf(stderr, "stop: unexpected arguments %v\n", fs.Args())
+		return 2
+	}
+	if *out == "" {
+		fmt.Fprintln(stderr, "stop: --out is required")
+		return 2
+	}
+
+	// SIGINT/SIGTERM cancel the wait cleanly; an interrupted cleanup is a
+	// could-not-complete, never a silent success.
+	ctx, cancel := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
+	defer cancel()
+
+	held, err := gate.StopRecorder(ctx, gate.StopConfig{
+		Log:     *out,
+		PidFile: *pidfile,
+		Clock:   gate.SystemClock{},
+		Notes:   newNoteStyler(stderr),
+		Cleanup: true,
+	})
+	if held != nil {
+		_ = held.Close()
+	}
+	if err != nil {
+		fmt.Fprintln(stderr, err)
+		return 2
+	}
+	return 0
+}

grafana-alertcheck/cmd/stop_test.go

@@ -0,0 +1,28 @@
+package main
+
+import (
+	"bytes"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestRunStop_RequiresOut(t *testing.T) {
+	var stdout, stderr bytes.Buffer
+	code := run([]string{"stop"}, &stdout, &stderr)
+	require.Equal(t, 2, code)
+	require.Contains(t, stderr.String(), "--out is required")
+}
+
+// Nothing recorded: an `if: always()` stop still exits 0.
+func TestRunStop_NoPidfileIsANoOp(t *testing.T) {
+	out := filepath.Join(t.TempDir(), "log.jsonl")
+	require.NoError(t, os.WriteFile(out, nil, 0o644)) // nolint:gosec // test-only temp file
+
+	var stdout, stderr bytes.Buffer
+	code := run([]string{"stop", "--out", out}, &stdout, &stderr)
+	require.Equal(t, 0, code)
+	require.Contains(t, stderr.String(), "nothing to stop")
+}

grafana-alertcheck/cmd/table.go

@@ -11,22 +11,29 @@ import (
 	"github.com/smartcontractkit/chainlink-testing-framework/grafana-alertcheck/internal/gate"
 )
 
+// limitsLegend explains each LIMITS USED column in one plain sentence, so the
+// table needs no documentation lookup.
+const limitsLegend = `  max gap without check — the longest gap between two checks we accept before we say the alert was not watched.
+  query failing for — how long Grafana may keep failing to run the alert's query before we stop trusting its state.
+  no evaluation for — how long Grafana may go without evaluating the alert before we stop trusting its state.`
+
 // renderTable is the human table. It always writes to the writer it is given,
 // which the caller (runCheck) always points at stderr — stdout is reserved for
 // the machine-readable --output json.
 //
-// Three titled tables, in order (the name column is RULE in all of them — one
-// row is one resolved alert rule, never a firing instance):
+// Three titled tables, in order (the ALERT column is one resolved alert rule,
+// never a firing instance):
+//
+//  1. RESULTS, one line per rule: verdict, time broken, check cadence,
+//     whether the window was observed, and any notes;
+//  2. VIOLATIONS, one line per distinct violation, with the raw Grafana state
+//     and health and the number of instances it stands for;
+//  3. LIMITS USED, the coverage thresholds that answer "why" on exit 2. Each
+//     column is named in plain words and explained by the legend below it, so
+//     the table needs no documentation lookup.
 //
-//  1. RESULTS, one line per rule: outcome, BadFor, pollEvery, proved-or-not
-//     with the largest gap;
-//  2. VIOLATIONS, one line per distinct violation.
-//  3. THRESHOLDS, the numbers that answer "why" on exit 2: each non-skipped
-//     rule's maxGap/healthGrace/evalStaleAfter, followed by the global
-//     transitionGrace and drainTimeout, and the largest measured clock skew
-//     alongside its own error bound (RTT/2) — SkewHardLimit is a separate,
-//     fixed input threshold and is reported next to it, never as if it were
-//     that bound.
+// The global footer then reports the extra observation time, the drain limit
+// and the largest measured clock difference, also in plain words.
 func renderTable(w io.Writer, res gate.Result) error {
 	alertOf := make(map[string]string, len(res.Verdicts))
 	for _, v := range res.Verdicts {
@@ -40,9 +47,19 @@ func renderTable(w io.Writer, res gate.Result) error {
 	// wall of progress text.
 	fmt.Fprintln(w)
 
+	if te := res.TerminatedEarly; te != nil {
+		detail := string(te.Reason)
+		if detail == "" {
+			detail = string(te.Outcome)
+		}
+		fmt.Fprintf(w, "EARLY EXIT: %s %q at %s (%s); the window [%s, %s] was not fully observed\n\n",
+			te.Kind, te.Alert, te.At.Format(time.RFC3339), detail,
+			res.From.Format(time.RFC3339), res.To.Format(time.RFC3339))
+	}
+
 	fmt.Fprintln(w, "RESULTS")
 	tw := tabwriter.NewWriter(w, 0, 4, 2, ' ', 0)
-	fmt.Fprintln(tw, "RULE\tOUTCOME\tBADFOR\tPOLLEVERY\tPROVED\tNOTE")
+	fmt.Fprintln(tw, "ALERT\tVERDICT\tBROKEN FOR\tCHECKED EVERY\tWINDOW COVERED\tDETAILS")
 	for _, v := range sortedVerdicts(res.Verdicts) {
 		fmt.Fprintf(tw, "%s\t%s\t%s\t%s\t%s\t%s\n",
 			v.Alert, v.Outcome, v.BadFor.Round(time.Second), v.PollEvery.Round(time.Second),
@@ -55,7 +72,7 @@ func renderTable(w io.Writer, res gate.Result) error {
 	if len(res.Violations) > 0 {
 		fmt.Fprintln(w, "\nVIOLATIONS")
 		vtw := tabwriter.NewWriter(w, 0, 4, 2, ' ', 0)
-		fmt.Fprintln(vtw, "RULE\tOUTCOME\tSTATE\tHEALTH\tINSTANCE COUNT\tNOTE")
+		fmt.Fprintln(vtw, "ALERT\tVERDICT\tGRAFANA STATE\tGRAFANA HEALTH\tINSTANCES\tDETAILS")
 		for _, g := range groupedViolations(res.Violations) {
 			fmt.Fprintf(vtw, "%s\t%s\t%s\t%s\t%s\t%s\n", alertLabel(g.v, alertOf), g.v.Outcome, g.v.State, g.v.Health, instanceCount(g), g.v.Note)
 		}
@@ -64,14 +81,13 @@ func renderTable(w io.Writer, res gate.Result) error {
 		}
 	}
 
-	// The per-rule thresholds answer "why" on exit 2: a table, not the prose
-	// "rule NAME: maxGap=... healthGrace=... evalStaleAfter=..." that repeated
-	// the rule name a fourth time. It is separated from the result above by a
-	// blank line.
+	// The per-rule limits answer "why" on exit 2. The columns are spelled out
+	// and explained by limitsLegend right below, so an operator does not have
+	// to look anything up.
 	fmt.Fprintln(w)
-	fmt.Fprintln(w, "THRESHOLDS")
+	fmt.Fprintln(w, "LIMITS USED")
 	ttw := tabwriter.NewWriter(w, 0, 4, 2, ' ', 0)
-	fmt.Fprintln(ttw, "RULE\tMAXGAP\tHEALTHGRACE\tEVALSTALEAFTER")
+	fmt.Fprintln(ttw, "ALERT\tMAX GAP WITHOUT CHECK\tQUERY FAILING FOR\tNO EVALUATION FOR")
 	for _, uid := range sortedThresholdUIDs(res.Thresholds, alertOf) {
 		t := res.Thresholds[uid]
 		fmt.Fprintf(ttw, "%s\t%s\t%s\t%s\n",
@@ -80,11 +96,17 @@ func renderTable(w io.Writer, res gate.Result) error {
 	if err := ttw.Flush(); err != nil {
 		return fmt.Errorf("render table: %w", err)
 	}
+	fmt.Fprintln(w, limitsLegend)
 
 	fmt.Fprintln(w)
-	fmt.Fprintf(w, "global: transitionGrace=%s (source: %s) drainTimeout=%s\n",
-		res.Global.TransitionGrace, res.Global.GraceSource, res.Global.DrainTimeout)
-	fmt.Fprintf(w, "largest measured clock skew: %s (bound ±%s, hard limit %s), grafana %s\n",
+	if res.Global.TransitionGrace > 0 {
+		fmt.Fprintf(w, "extra watching after your window: +%s — so an alert that only starts firing at the end is still caught (slowest: %s)\n",
+			res.Global.TransitionGrace, res.Global.GraceSource)
+	} else {
+		fmt.Fprintln(w, "extra watching after your window: none")
+	}
+	fmt.Fprintf(w, "max wait for all alerts to finish evaluating: %s\n", res.Global.DrainTimeout)
+	fmt.Fprintf(w, "clock difference from Grafana: %s, accurate to ±%s (checks fail above %s); Grafana %s\n",
 		res.ClockSkew.Round(time.Millisecond), res.ClockSkewBound.Round(time.Millisecond),
 		gate.SkewHardLimit, res.GrafanaVersion)
 	// The verdict — the single number a terminal operator reads last — sits on
@@ -94,8 +116,8 @@ func renderTable(w io.Writer, res gate.Result) error {
 	return nil
 }
 
-// violationsLabel colours the "violations: N" prefix of the footer: green for a
-// clean run, red otherwise. The rest of the line is written uncoloured.
+// violationsLabel colours the "violations: N" prefix of the footer: green when
+// there are none, red otherwise. The rest of the line is written uncoloured.
 func violationsLabel(n int, enabled bool) string {
 	s := fmt.Sprintf("violations: %d", n)
 	if !enabled {
@@ -107,10 +129,10 @@ func violationsLabel(n int, enabled bool) string {
 	return ansiRed + s + ansiReset
 }
 
-// provedLabel is the table's PROVED column: "yes" for a clean coverage
-// proof, "no" with the reason and largest gap for an unobservable rule, and
-// "-" for a rule decide never asked proveCoverage about at all (skipped —
-// paused before the window opened).
+// provedLabel is the table's WINDOW COVERED column: "yes" for a fully
+// observed window, "no" with the reason and largest gap for a not-verified
+// rule, and "-" for a rule decide never asked proveCoverage about at all
+// (paused before the window opened).
 func provedLabel(cov gate.CoverageResult) string {
 	if cov.Reason == "" && !cov.Unobservable && !cov.Proved {
 		return "-"
@@ -182,8 +204,11 @@ func sameRendered(a, b gate.Violation) bool {
 	return violationSignature(a) == violationSignature(b)
 }
 
+// instanceCount is the INSTANCES column: how many alert instances one grouped
+// violation row stands for. Paused and not-counted rows stand for no instance
+// at all, so they render "-".
 func instanceCount(g violationGroup) string {
-	if g.v.Outcome == gate.OutcomeSkipped {
+	if g.v.Outcome == gate.OutcomePaused || g.v.Outcome == gate.OutcomeNotCounted {
 		return "-"
 	}
 	return strconv.Itoa(g.n)