From 0a041bee0cba2b98302662ae48ca351ca74bb8fe Mon Sep 17 00:00:00 2001 From: Paula Ruiz Rodriguez <50167687+Paururo@users.noreply.github.com> Date: Wed, 23 Sep 2026 14:15:47 +0200 Subject: [PATCH 01/14] Find deletions by comparing each sample's unread stretches with the cohort Each sample's all-positions VCF is reduced by a new per-sample step, DEPTH_PROFILE, to its depth per window, the stretches no read covers and a per-gene table, published under stats/. The report compares the stretches across the samples of each reference: one a sample lacks while the others read it is a deletion, private or shared; one nobody reads is a repeat or a part of the reference none of these genomes has, and is listed apart rather than counted as anyone's. Samples under --report_depth_min are not assessed, since stretches without reads turn up there by chance. A Deletions panel lists the regions with the genes they remove and zooms the landscape onto each, and _deletions.tsv keeps them with every carrier's own coordinates. The landscape gains a Deletions track and a SNPs / kb track that divides each bin's SNPs by the positions deep enough to call, so a bin half of which was not read no longer looks half as variable. --- CHANGELOG.md | 12 + bin/depth_profile.py | 251 +++++++++++++++++++++ bin/qc_report.py | 28 ++- bin/qcreport/coverage.py | 285 ++++++++++++++++++++++++ bin/qcreport/render.py | 15 +- bin/report_assets/js/03_state.js | 2 +- bin/report_assets/js/09_genome_genes.js | 23 +- bin/report_assets/js/09b_coverage.js | 80 +++++++ bin/report_assets/js/14_boot.js | 11 +- bin/report_assets/shell.html | 7 +- docs/configuration.md | 2 + docs/outputs.md | 28 +++ docs/qc-report.md | 2 +- modules/report.nf | 45 +++- nextflow.config | 6 + subworkflows/cohort_report.nf | 10 +- tests/js/coverage.test.mjs | 43 ++++ tests/unit/test_depth_profile.py | 196 ++++++++++++++++ tests/unit/test_qc_report_coverage.py | 209 +++++++++++++++++ 19 files changed, 1229 insertions(+), 26 deletions(-) create mode 100644 bin/depth_profile.py create mode 100644 bin/qcreport/coverage.py create mode 100644 bin/report_assets/js/09b_coverage.js create mode 100644 tests/js/coverage.test.mjs create mode 100644 tests/unit/test_depth_profile.py create mode 100644 tests/unit/test_qc_report_coverage.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 5af0e0b..67da6a8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,18 @@ based on [Keep a Changelog](https://keepachangelog.com/), and the project follow ### Added +- **Deletions, and SNPs per callable kb along the genome.** Each sample's all-positions VCF is + reduced to its depth per window, the stretches no read covers and a per-gene table + (`DEPTH_PROFILE`, published under `stats/`). The report compares those stretches across the + samples of each reference: one a sample lacks while the others read it is a deletion, private + or shared; one nobody reads is a repeat or a part of the reference none of these genomes has, + and is listed apart instead of being counted as anyone's. A *Deletions* panel lists them with + the genes they remove and zooms the landscape onto each, and `_deletions.tsv` + keeps them with every carrier's own coordinates. The landscape gains a *Deletions* track and a + *SNPs / kb* track that divides each bin's SNPs by the positions deep enough to call, so a bin + half of which was not read no longer looks half as variable. Samples under + `--report_depth_min` are not assessed: stretches without reads turn up there by chance. + - **Two guards against a sample mapped to a genome it does not belong to.** In a 185-sample cohort, 22 samples annotated as one lineage were really another, were routed to that lineage's reference, carried about 2,000 SNPs where their line mates carried 5, and produced 507 of the diff --git a/bin/depth_profile.py b/bin/depth_profile.py new file mode 100644 index 0000000..c4fd42a --- /dev/null +++ b/bin/depth_profile.py @@ -0,0 +1,251 @@ +#!/usr/bin/env python3 +"""What one sample's reads cover, from its all-positions VCF. + +The all-positions VCF has a record for every reference position, so it holds the depth of the +whole genome. This reduces it to three small tables for the cohort report: + + .depth_windows.tsv one row per window: mean depth, the share of positions no read + covers, and the share deep enough for the consensus to call + (depth above --min-dp) + .zero_depth.tsv every stretch of at least --min-run positions no read covers + .gene_depth.tsv per gene of the GFF: the share of its positions read at all, the + share the consensus can call, its mean depth, and that depth + against the sample's genome-wide median (with --gff) + +A stretch no read covers is a deletion only if the rest of the cohort reads it; otherwise it is +a part of the reference none of these genomes has, or one no read can be placed on. That +comparison needs the cohort, so it is the report's to make, not this script's. + +Each table opens with '#' lines naming the sample, the reference and the sample's genome-wide +depth, so a table can be read without its file name. +""" +from __future__ import annotations + +import argparse +import gzip +import os +import sys + +import numpy as np + +from qcreport.parsers import _gff_attr + + +def _open(p): + return gzip.open(p, 'rt', encoding='utf-8', errors='replace') if str(p).endswith('.gz') \ + else open(p, encoding='utf-8', errors='replace') + + +def read_depths(path): + """(sample, {contig: depth array}, {contig: read-at-all mask}) from an all-positions VCF. + + Arrays are sized from the ##contig lengths and grown when a record lies beyond them. A + position without a record is not a position of depth 0, so the mask keeps the two apart. + """ + lengths, order, depth, known = {}, [], {}, {} + sample = None + dp_index = {} # FORMAT string -> index of DP in it (-1 when it has none) + + def arrays(contig, need): + if contig not in depth: + n = max(lengths.get(contig, 0), need) + depth[contig] = np.zeros(n, dtype=np.int32) + known[contig] = np.zeros(n, dtype=bool) + order.append(contig) + elif need > len(depth[contig]): + grow = max(need, 2 * len(depth[contig])) + depth[contig] = np.concatenate([depth[contig], np.zeros(grow - len(depth[contig]), np.int32)]) + known[contig] = np.concatenate([known[contig], np.zeros(grow - len(known[contig]), bool)]) + return depth[contig], known[contig] + + last = (None, None, None) # (contig, depth array, mask): most lines stay on one contig + with _open(path) as fh: + for line in fh: + if line[0] == '#': + if line.startswith('##contig='): + body = line.strip()[len('##contig=<'):-1] + fields = dict(f.split('=', 1) for f in body.split(',') if '=' in f) + try: + lengths[fields['ID']] = int(fields.get('length', 0)) + except (KeyError, ValueError): + pass + elif line.startswith('#CHROM'): + cols = line.rstrip('\n').split('\t') + sample = cols[9] if len(cols) > 9 else None + continue + c = line.rstrip('\n').split('\t') + if len(c) < 10: + continue + k = dp_index.get(c[8]) + if k is None: + keys = c[8].split(':') + k = dp_index[c[8]] = keys.index('DP') if 'DP' in keys else -1 + try: + pos = int(c[1]) + except ValueError: + continue + vals = c[9].split(':') + raw = vals[k] if 0 <= k < len(vals) else '.' + try: + dp = int(raw) + except ValueError: + continue # a depth the record does not state is not a depth of 0 + contig = c[0] + if last[0] != contig or pos > len(last[1]): + d, m = arrays(contig, pos) + last = (contig, d, m) + last[1][pos - 1] = dp + last[2][pos - 1] = True + # Trim what growing over-allocated past the last record of a contig with no stated length. + for contig in order: + if not lengths.get(contig): + idx = np.flatnonzero(known[contig]) + n = int(idx[-1]) + 1 if idx.size else 0 + depth[contig], known[contig] = depth[contig][:n], known[contig][:n] + return sample, {c: depth[c] for c in order}, {c: known[c] for c in order} + + +def windows(depth, known, size, min_dp): + """[(contig, start, end, mean depth, zero fraction, callable fraction)], 1-based inclusive. + + Fractions are of the positions with a record; a window with none has no values. + """ + out = [] + for contig, d in depth.items(): + m = known[contig] + for s in range(0, len(d), size): + e = min(len(d), s + size) + dm, km = d[s:e], m[s:e] + n = int(km.sum()) + if n == 0: + out.append((contig, s + 1, e, None, None, None)) + continue + dk = dm[km] + out.append((contig, s + 1, e, float(dk.mean()), float((dk == 0).mean()), + float((dk > min_dp).mean()))) + return out + + +def zero_runs(depth, known, min_run): + """[(contig, start, end)] of the stretches of at least `min_run` read positions at depth 0.""" + out = [] + for contig, d in depth.items(): + z = ((d == 0) & known[contig]).astype(np.int8) + edges = np.diff(np.concatenate([[0], z, [0]])) + starts, ends = np.flatnonzero(edges == 1), np.flatnonzero(edges == -1) + for s, e in zip(starts, ends): + if e - s >= min_run: + out.append((contig, int(s) + 1, int(e))) + return out + + +def read_genes(path): + """[(contig, start, end, name, locus_tag)] of the gene features of a GFF3, CDS where there + are no genes (a GFF from an annotator that writes only CDS). Empty without a readable file.""" + if not path or not os.path.exists(path) or os.path.basename(path).startswith('NO_FILE'): + return [] + genes, cds = [], [] + try: + with _open(path) as fh: + for line in fh: + if line.startswith('#') or '\t' not in line: + continue + c = line.rstrip('\n').split('\t') + if len(c) < 9 or c[2] not in ('gene', 'CDS'): + continue + try: + start, end = int(c[3]), int(c[4]) + except ValueError: + continue + name = None + for key in ('Name', 'gene', 'locus_tag', 'ID'): + name = _gff_attr(c[8], key) + if name: + break + row = (c[0], start, end, name or f'{start}-{end}', _gff_attr(c[8], 'locus_tag') or '') + (genes if c[2] == 'gene' else cds).append(row) + except OSError: + return [] + return sorted(genes or cds, key=lambda g: (g[0], g[1])) + + +def gene_depths(depth, known, genes, min_dp, median): + """[(contig, start, end, name, locus_tag, length, breadth, callable, mean depth, relative)]. + + breadth is the share of the gene's read positions at depth 1 or more, callable the share + above `min_dp`, relative the mean depth against the genome-wide median (None at median 0). + """ + out = [] + for contig, start, end, name, locus in genes: + d = depth.get(contig) + if d is None: + continue + s, e = max(0, start - 1), min(len(d), end) + if e <= s: + continue + km = known[contig][s:e] + dk = d[s:e][km] + if dk.size == 0: + out.append((contig, start, end, name, locus, end - start + 1, None, None, None, None)) + continue + mean = float(dk.mean()) + out.append((contig, start, end, name, locus, end - start + 1, float((dk > 0).mean()), + float((dk > min_dp).mean()), mean, (mean / median) if median else None)) + return out + + +def _fmt(v, nd=4): + return '' if v is None else (f'{v:.{nd}f}' if isinstance(v, float) else str(v)) + + +def main(argv=None): + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument('--vcf', required=True, help="the sample's all-positions VCF") + ap.add_argument('--gff', default=None, help="the reference's GFF3, for the per-gene table") + ap.add_argument('--reference', default='', help="the reference id, written into each table") + ap.add_argument('--window', type=int, default=1000, help="window size in bp") + ap.add_argument('--min-dp', type=int, default=7, + help="depth at or below which the consensus calls nothing (--consensus_min_dp)") + ap.add_argument('--min-run', type=int, default=50, + help="shortest stretch without reads that is written out") + ap.add_argument('--out-prefix', required=True) + a = ap.parse_args(argv) + + sample, depth, known = read_depths(a.vcf) + if sample is None: + sys.stderr.write(f"[depth_profile] {a.vcf} names no sample\n") + return 1 + read = np.concatenate([depth[c][known[c]] for c in depth]) if depth else np.zeros(0, np.int32) + median = float(np.median(read)) if read.size else 0.0 + mean = float(read.mean()) if read.size else 0.0 + head = [f"# sample={sample}", f"# reference={a.reference}", f"# genome_len={int(read.size)}", + f"# median_dp={median:g}", f"# mean_dp={mean:.2f}", + f"# zero_frac={float((read == 0).mean()) if read.size else 0:.4f}", + f"# callable_frac={float((read > a.min_dp).mean()) if read.size else 0:.4f}", + f"# min_dp={a.min_dp}", f"# window={a.window}", f"# min_run={a.min_run}", + "# contigs=" + ",".join(f"{c}:{len(depth[c])}" for c in depth)] + + with open(f"{a.out_prefix}.depth_windows.tsv", 'w') as fh: + fh.write("\n".join(head) + "\ncontig\tstart\tend\tmean_dp\tzero_frac\tcallable_frac\n") + for row in windows(depth, known, a.window, a.min_dp): + fh.write("\t".join(_fmt(v, 2 if i == 3 else 4) for i, v in enumerate(row)) + "\n") + runs = zero_runs(depth, known, a.min_run) + with open(f"{a.out_prefix}.zero_depth.tsv", 'w') as fh: + fh.write("\n".join(head) + "\ncontig\tstart\tend\tlength\n") + for contig, s, e in runs: + fh.write(f"{contig}\t{s}\t{e}\t{e - s + 1}\n") + genes = read_genes(a.gff) + if genes: + with open(f"{a.out_prefix}.gene_depth.tsv", 'w') as fh: + fh.write("\n".join(head) + "\ncontig\tstart\tend\tgene\tlocus_tag\tlength\tbreadth\t" + "callable\tmean_dp\trel_depth\n") + for row in gene_depths(depth, known, genes, a.min_dp, median): + fh.write("\t".join(_fmt(v, 2 if i == 8 else 4) for i, v in enumerate(row)) + "\n") + sys.stderr.write(f"[depth_profile] {sample}: {int(read.size)} positions, median depth {median:g}, " + f"{len(runs)} stretch(es) of {a.min_run}+ bp without reads" + + (f", {len(genes)} gene(s)" if genes else "") + "\n") + return 0 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/bin/qc_report.py b/bin/qc_report.py index b637a85..7dd5a1f 100644 --- a/bin/qc_report.py +++ b/bin/qc_report.py @@ -26,6 +26,7 @@ import sys from datetime import datetime, timezone +from qcreport.coverage import build_coverage, parse_depth_profiles, snp_per_kb, write_deletions from qcreport.metrics import (ANC_DEF, DEF, DEFS, DIST, METRICS, build_gene_map, discover_extra_metrics, flag_sample, het_frac, is_ancient, lineage_counts_parsed, lineage_fracs, robust) from qcreport.panels import build_dynamics, build_epistasis, build_snp_matrix @@ -87,6 +88,16 @@ def build_parser(): help="TSV 'mapping_poscanonical_pos' (e.g. pathotypr_liftover.py apply --out-map) -> the " "reference-of-interest COORDINATE per variant, alignment-free. Fills pos_h37rv for the SNP " "tables; an alternative to --vcfs-h37rv when the references do not share coordinates.") + ap.add_argument("--depth-profiles", nargs="*", default=[], + help="Per-sample depth tables from depth_profile.py (windows, stretches without reads, " + "genes) -> deletions and SNPs per callable kb along the genome (optional).") + ap.add_argument("--deletion-min-len", type=int, default=200, + help="Shortest stretch without reads reported as a deletion, in bp.") + ap.add_argument("--deletion-min-depth", type=float, default=10.0, + help="Median depth a sample needs before its stretches without reads are assessed: " + "in a thinly read sample they turn up by chance.") + ap.add_argument("--out-deletions", default=None, + help="Write the deletion regions (with --depth-profiles) as a TSV.") ap.add_argument("--gate", action="store_true") return ap @@ -236,11 +247,18 @@ def build_payload(args, thr, anc_thr): by_ref.setdefault(ref, []).append(lin.split(";")[0]) ref_major = {r: max(set(v), key=v.count) for r, v in by_ref.items() if len(v) >= 3} dates = parse_collection_dates(args.metadata) + # What each sample's reads cover, compared across the cohort: the stretches no read covers that + # other samples do read (deletions), and how much of each bin could be called at all. + profiles = parse_depth_profiles(args.depth_profiles) + coverage, del_tracks = (build_coverage(profiles, min_len=args.deletion_min_len, + min_depth=args.deletion_min_depth)[:2] + if profiles else (None, {})) jsamples, counts = [], {"PASS": 0, "WARN": 0, "FAIL": 0} for sid, m in summ.items(): anc = is_ancient(m) dmg = dmg_by_sample.get(sid) + snp_prof = parse_profile(m.get("snp_profile")) verdict, flags = flag_sample(m, thr, snp_med, snp_sig, ancient=anc, anc_thr=anc_thr, dmg=dmg, ref_lineage=ref_major.get(ref_of.get(sid))) counts[verdict] += 1 @@ -260,9 +278,12 @@ def build_payload(args, thr, anc_thr): "ref": ref_of.get(sid), "ref_lin": ref_major.get(ref_of.get(sid)), "miss": miss_by_sample.get(sid), - "trk": ({k: v for k, v in (("snp", parse_profile(m.get("snp_profile"))), + "trk": ({k: v for k, v in (("snp", snp_prof), + ("snpkb", snp_per_kb(snp_prof, profiles[sid]) + if sid in profiles else None), ("het", parse_profile(m.get("het_profile"))), - ("indel", parse_profile(m.get("indel_profile")))) if v is not None} + ("indel", parse_profile(m.get("indel_profile"))), + ("del", del_tracks.get(sid))) if v is not None} or None), "ann_db_error": (clean_str(m.get("ann_db_error")) == "yes"), "m": dict({k: to_float(m.get(k)) for k in metric_keys + extra_keys + EFF_KEYS}, @@ -303,6 +324,7 @@ def build_payload(args, thr, anc_thr): "dynamics": _dynamics, "epistasis": build_epistasis(_dynamics), "snp_matrix": build_snp_matrix(_variants, provenance.get('reference', '')), + "coverage": coverage, "sample_meta": _sample_meta, "metrics": [{"key": k, "label": l, "kind": kind, "dir": d} for k, l, kind, d in METRICS], "extra": extra_metrics, @@ -357,6 +379,8 @@ def main(): with open(args.out_html, "w", encoding="utf-8") as fh: fh.write(build_html(args.title, payload)) write_flags(args.out_flags, jsamples) + if args.out_deletions and payload.get("coverage"): + write_deletions(args.out_deletions, payload["coverage"]["regions"]) sys.stderr.write(f"[qc_report] {len(jsamples)} samples -> {counts['PASS']} PASS, {counts['WARN']} WARN, " f"{counts['FAIL']} FAIL. Wrote {args.out_html} + {args.out_flags}\n") diff --git a/bin/qcreport/coverage.py b/bin/qcreport/coverage.py new file mode 100644 index 0000000..44afb30 --- /dev/null +++ b/bin/qcreport/coverage.py @@ -0,0 +1,285 @@ +"""Deletions and callable depth along the genome, from the per-sample depth profiles. + +bin/depth_profile.py reduces each sample's all-positions VCF to windows, the stretches no read +covers, and a per-gene table. What those stretches mean needs the cohort: + +- a stretch no sample reads is a part of the reference none of these genomes has, or one no + read can be placed on (a repeat). It is not a deletion of any sample; +- a stretch some samples read and others do not is a deletion in the others, shared by a + lineage when several carry it and private to one sample when only it does; +- in a sample read too thinly, stretches without reads turn up by chance, so its stretches are + not assessed at all rather than reported as deletions. + +The windows also give, per bin of the genome landscape, how much of each sample could be called, +which is what turns a count of SNPs per bin into a density: a bin half of which was not read +holds half the SNPs whatever the genome looks like there. +""" +from __future__ import annotations + +import math +import os + +from .parsers import NBINS, to_float + + +def _kind(header): + """Which of depth_profile.py's tables a column header belongs to. The gene table also has a + mean_dp column, so it is recognised first.""" + if "gene" in header: + return "genes" + if "zero_frac" in header: + return "windows" + if "length" in header: + return "zero" + return None + + +def parse_depth_profiles(paths): + """{sample: profile} from depth_profile.py's tables, any of the three per sample. + + The table is recognised by its columns and the sample by its '# sample=' line, so neither + depends on a file name. A profile holds the header values (reference, median_dp, contigs...) + and the rows of whichever tables were given. + """ + out = {} + for path in paths or []: + if not path or not os.path.exists(path) or os.path.basename(path).startswith("NO_FILE"): + continue + meta, header, rows = {}, None, [] + try: + with open(path, encoding="utf-8", errors="replace") as fh: + for line in fh: + line = line.rstrip("\n") + if line.startswith("#"): + k, sep, v = line[1:].strip().partition("=") + if sep: + meta[k.strip()] = v.strip() + continue + if header is None: + header = line.split("\t") + continue + if line: + rows.append(dict(zip(header, line.split("\t")))) + except OSError: + continue + sample = meta.get("sample") + kind = _kind(header or []) + if not sample or not kind: + continue + p = out.setdefault(sample, {"windows": [], "zero": [], "genes": []}) + for k in ("reference", "median_dp", "mean_dp", "genome_len", "zero_frac", "callable_frac", + "min_dp", "window", "min_run", "contigs"): + if k in meta and k not in p: + p[k] = meta[k] + if kind == "windows": + p["windows"] = [(r["contig"], int(r["start"]), int(r["end"]), to_float(r.get("mean_dp")), + to_float(r.get("zero_frac")), to_float(r.get("callable_frac"))) + for r in rows if r.get("start", "").isdigit() and r.get("end", "").isdigit()] + elif kind == "zero": + p["zero"] = [(r["contig"], int(r["start"]), int(r["end"])) + for r in rows if r.get("start", "").isdigit() and r.get("end", "").isdigit()] + else: + p["genes"] = [{"contig": r["contig"], "start": int(r["start"]), "end": int(r["end"]), + "gene": r.get("gene", ""), "locus": r.get("locus_tag", ""), + "breadth": to_float(r.get("breadth")), "rel": to_float(r.get("rel_depth"))} + for r in rows if r.get("start", "").isdigit() and r.get("end", "").isdigit()] + for p in out.values(): + p["median_dp"] = to_float(p.get("median_dp")) or 0.0 + p["contigs"] = _contigs(p.get("contigs")) + return out + + +def _contigs(text): + """[(name, length)] from '# contigs=a:100,b:20', in the order of the reference.""" + out = [] + for item in (text or "").split(","): + name, sep, n = item.rpartition(":") + if sep and n.isdigit(): + out.append((name, int(n))) + return out + + +def _offsets(contigs): + """({contig: offset}, genome length): the contigs laid end to end, as the landscape does.""" + offs, total = {}, 0 + for name, n in contigs: + offs[name] = total + total += n + return offs, total + + +def _spread(intervals, contigs, nbins=NBINS): + """Per-bin amounts from [(contig, start, end, amount per position)], 1-based inclusive. + + Uses the landscape's own binning, bin = global position * nbins // genome length (see + stats_to_legacy.analyze_vcf), so a value lands in the bin the SNPs of those positions do. + """ + offs, total = _offsets(contigs) + out = [0.0] * nbins + if total <= 0: + return out + for contig, s, e, per in intervals: + if contig not in offs or not per: + continue + g0, g1 = offs[contig] + s, offs[contig] + e + b = min(nbins - 1, g0 * nbins // total) + while g0 <= g1: + nxt = min(g1, ((b + 1) * total - 1) // nbins) # last position of bin b + if nxt < g0: + nxt = g0 + out[b] += per * (nxt - g0 + 1) + g0, b = nxt + 1, min(nbins - 1, b + 1) + return out + + +def callable_bp(profile, nbins=NBINS): + """Per-bin number of positions deep enough for the consensus to call, or None.""" + rows = [(c, s, e, f) for c, s, e, _, _, f in profile.get("windows", []) if f is not None] + if not rows or not profile.get("contigs"): + return None + return _spread(rows, profile["contigs"], nbins) + + +def _bin_len(contigs, nbins=NBINS): + return _spread([(c, 1, n, 1.0) for c, n in contigs], contigs, nbins) + + +def snp_per_kb(snp_profile, profile, nbins=NBINS): + """SNPs per callable kb in each bin, or None without both profiles. + + A bin with fewer than a tenth of its positions callable has no value: a few called bases + make any count look dense or empty. + """ + cb = callable_bp(profile, nbins) + if not snp_profile or cb is None: + return None + full = _bin_len(profile["contigs"], nbins) + return [round(1000.0 * (snp_profile[b] or 0) / cb[b], 3) + if cb[b] > 0 and full[b] and cb[b] >= 0.1 * full[b] else None for b in range(nbins)] + + +def _overlap(a0, a1, b0, b1): + return max(0, min(a1, b1) - max(a0, b0) + 1) + + +def build_coverage(profiles, min_len=200, min_depth=10.0, cohort_frac=0.9, gene_breadth=0.5, + nbins=NBINS): + """(report section, {sample: per-bin deleted fraction}, rows for the deletions TSV). + + Stretches without reads of at least `min_len` bp, in samples whose genome-wide median depth + is at least `min_depth`, are merged across samples into regions and classed by how many of + the assessed samples on the same reference lack them: + + cohort at least `cohort_frac` of them, and at least two: not a deletion of anyone + shared two or more, fewer than that: a deletion several samples carry + private one sample, with at least one other assessed sample reading the region + alone the only assessed sample on its reference, so nothing to compare it with + + Genes are named for a region when a carrier reads less than `gene_breadth` of them. + """ + by_ref = {} + for sid, p in profiles.items(): + by_ref.setdefault(p.get("reference") or ",".join(c for c, _ in p.get("contigs", [])), []).append(sid) + + regions, tracks, not_assessed = [], {}, [] + for ref, samples in sorted(by_ref.items()): + assessed = [s for s in samples if profiles[s]["median_dp"] >= min_depth] + not_assessed += [s for s in samples if s not in assessed] + runs = sorted((c, s, e, sid) for sid in assessed for c, s, e in profiles[sid]["zero"] + if e - s + 1 >= min_len) + merged = [] + for c, s, e, sid in runs: + if merged and merged[-1]["contig"] == c and s <= merged[-1]["end"]: + r = merged[-1] + r["end"] = max(r["end"], e) + r["carriers"].setdefault(sid, []).append((s, e)) + else: + merged.append({"contig": c, "start": s, "end": e, "carriers": {sid: [(s, e)]}}) + n = len(assessed) + for r in merged: + k = len(r["carriers"]) + if n < 2: + cls = "alone" + elif k >= max(2, math.ceil(cohort_frac * n)): + cls = "cohort" + elif k >= 2: + cls = "shared" + else: + cls = "private" + genes = {} + for sid in r["carriers"]: + for g in profiles[sid]["genes"]: + if g["contig"] == r["contig"] and _overlap(g["start"], g["end"], r["start"], r["end"]) \ + and g["breadth"] is not None and g["breadth"] < gene_breadth: + genes.setdefault(g["gene"], g["locus"]) + regions.append({ + "ref": ref, "contig": r["contig"], "start": r["start"], "end": r["end"], + "len": r["end"] - r["start"] + 1, "cls": cls, "n": k, "of": n, + "genes": sorted(genes), + "samples": sorted([sid, min(a for a, _ in iv), max(b for _, b in iv)] + for sid, iv in r["carriers"].items()), + }) + + # Per-bin share of each sample's positions inside a deletion it carries (cohort-wide gaps + # excluded, since they are nobody's deletion). + per_sample = {} + for reg in regions: + if reg["cls"] == "cohort": + continue + for sid, s, e in reg["samples"]: + per_sample.setdefault(sid, []).append((reg["contig"], s, e, 1.0)) + for sid, ivs in per_sample.items(): + contigs = profiles[sid].get("contigs") or [] + full = _bin_len(contigs, nbins) + lost = _spread(ivs, contigs, nbins) + tracks[sid] = [round(min(1.0, lost[b] / full[b]), 3) if full[b] else 0.0 for b in range(nbins)] + for sid in profiles: + if sid not in tracks and profiles[sid]["median_dp"] >= min_depth and profiles[sid].get("contigs"): + tracks[sid] = [0.0] * nbins + + lost_genes = _lost_genes(profiles, by_ref, min_depth, gene_breadth) + regions.sort(key=lambda r: (r["cls"] == "cohort", -r["n"] if r["cls"] == "shared" else 0, + r["ref"], r["contig"], r["start"])) + for i, r in enumerate(regions, 1): + r["id"] = f"D{i}" + # Each reference's contigs in order, so the report can place a region on the landscape, which + # lays the contigs of a sample's genome end to end. + refs = {ref: [list(c) for c in profiles[samples[0]].get("contigs", [])] for ref, samples in by_ref.items()} + section = {"min_len": min_len, "min_depth": min_depth, "cohort_frac": cohort_frac, "refs": refs, + "regions": regions, "genes_lost": lost_genes, "not_assessed": sorted(not_assessed), + "n_assessed": sum(1 for p in profiles.values() if p["median_dp"] >= min_depth)} + return section, tracks, regions + + +def _lost_genes(profiles, by_ref, min_depth, gene_breadth, cohort_breadth=0.9): + """Genes a sample reads less than `gene_breadth` of while at least half of the assessed + samples on its reference read `cohort_breadth` of them: gene loss, rather than a gene + nobody here carries.""" + out = [] + for ref, samples in sorted(by_ref.items()): + assessed = [s for s in samples if profiles[s]["median_dp"] >= min_depth and profiles[s]["genes"]] + if len(assessed) < 2: + continue + table = {} + for sid in assessed: + for g in profiles[sid]["genes"]: + if g["breadth"] is not None: + table.setdefault((g["contig"], g["start"], g["end"], g["gene"], g["locus"]), {})[sid] = g["breadth"] + for (contig, s, e, gene, locus), vals in sorted(table.items()): + lost = sorted([sid, round(b, 3)] for sid, b in vals.items() if b < gene_breadth) + read = sum(1 for b in vals.values() if b >= cohort_breadth) + if lost and read >= 0.5 * len(vals): + out.append({"ref": ref, "contig": contig, "start": s, "end": e, "gene": gene, + "locus": locus, "samples": lost, "of": len(vals)}) + return out + + +def write_deletions(path, regions): + """The regions as a TSV: one row per region, its carriers with their own coordinates.""" + with open(path, "w", encoding="utf-8") as fh: + fh.write("region\treference\tcontig\tstart\tend\tlength\tclass\tn_samples\tn_assessed\t" + "samples\tgenes\n") + for r in regions: + fh.write("\t".join(str(v) for v in ( + r["id"], r["ref"], r["contig"], r["start"], r["end"], r["len"], r["cls"], r["n"], r["of"], + ",".join(f"{sid}:{s}-{e}" for sid, s, e in r["samples"]), ",".join(r["genes"]))) + "\n") diff --git a/bin/qcreport/render.py b/bin/qcreport/render.py index 2195f80..29e5501 100644 --- a/bin/qcreport/render.py +++ b/bin/qcreport/render.py @@ -47,7 +47,8 @@ def _asset(name): _JS_MODULES = [ "js/01_prelude.js", "js/02_qcspace.js", "js/03_state.js", "js/04_helpers.js", "js/05_insights_qc.js", "js/06_insights_genome.js", "js/07_render_core.js", "js/07b_summary.js", - "js/08_curation.js", "js/09_genome_genes.js", "js/10_dynamics.js", "js/11_epistasis.js", + "js/08_curation.js", "js/09_genome_genes.js", "js/09b_coverage.js", "js/10_dynamics.js", + "js/11_epistasis.js", "js/12_snpmatrix.js", "js/13_drug_kraken.js", "js/14_boot.js", ] JS = "".join(_asset(m) for m in _JS_MODULES) @@ -104,7 +105,17 @@ def build_html(title, payload): "(ambiguous) % and the longest gap. A phylogeny-oriented view of completeness, not just average depth.", "genome": "Per-position callability / variant density along the reference, binned into a heatmap per " "sample. Brush a region to list the genes under it. Reveals systematically low-callability " - "regions (repeats, deletions) shared across samples.", + "regions (repeats, deletions) shared across samples. 'Deletions' shows the share of each bin " + "in a stretch the sample has no reads for while other samples read it; 'SNPs / kb' divides " + "the SNPs of a bin by the positions deep enough to call, so a half-read bin does not look " + "half as variable.", + "deletions": "Stretches of the reference a sample has no reads for, compared with the other samples on " + "the same reference. Lacked by one sample it is a private deletion; by several, a shared " + "one (often a lineage's); by all, a repeat or a part of the reference these genomes do not " + "have, which is nobody's deletion and is listed apart. Only samples read deeply enough are " + "assessed, since in a thin sample stretches without reads turn up by chance. From the " + "all-positions VCF, so a deletion shorter than the minimum length, or one the caller " + "already reported as an indel, is not listed here.", "function": "The snpEff functional class of each sample's variants (HIGH/MODERATE/LOW/MODIFIER impact and " "effect types such as missense / synonymous). A per-sample mutational-impact profile.", "geneburden": "Genes carrying the most impactful (HIGH/MODERATE) variants across the cohort, with the " diff --git a/bin/report_assets/js/03_state.js b/bin/report_assets/js/03_state.js index 7c0306c..4f401c2 100644 --- a/bin/report_assets/js/03_state.js +++ b/bin/report_assets/js/03_state.js @@ -1,6 +1,6 @@ var extraSet={}; R.extra.forEach(function(e){extraSet[e.key]=1;R.metrics.push(e);R.defs[e.key]=R.defs[e.key]||['Auto-detected metric from the summary TSV (not a named QC metric).',''];}); var st={sortKey:'v',asc:false,q:'',onlyFlagged:false,hidden:{},hi:null,flagFilter:null,ptype:'beeswarm', // default view: worst QC first (a returning user's saved sort overrides this) - sx:'mean_depth',sy:'breadth_pct',excl:{},detail:null,colorBy:'qc',groupLin:false,ancOnly:null,linFilter:null,metaFilter:{},gtrack:'missing',maskOn:false,gsel:null,gzoom:null,gbq:'',hotq:'',pnpsq:'',vardoseq:'',colf:{},showColF:false}; + sx:'mean_depth',sy:'breadth_pct',excl:{},detail:null,colorBy:'qc',groupLin:false,ancOnly:null,linFilter:null,metaFilter:{},gtrack:'missing',maskOn:false,gsel:null,gzoom:null,gbq:'',hotq:'',delq:'',pnpsq:'',vardoseq:'',colf:{},showColF:false}; var SGEO=null, GGEO=null; // scatter + genome brush geometry caches (for inverse-mapping the rubber-band) var _thdb,_qdb,_mxdb,_cfdb; // debounce timers: keep live inputs snappy at cohort scale (defer heavy re-renders) // The table opens on the metrics a verdict is made of; the other ~35 are one click away under "columns". diff --git a/bin/report_assets/js/09_genome_genes.js b/bin/report_assets/js/09_genome_genes.js index 2513938..0c7a7e4 100644 --- a/bin/report_assets/js/09_genome_genes.js +++ b/bin/report_assets/js/09_genome_genes.js @@ -1,5 +1,8 @@ -var GTRACKS=[{k:'missing',lab:'Missing',base:[214,64,58]},{k:'snp',lab:'SNPs',base:[14,139,168]}, +var GTRACKS=[{k:'missing',lab:'Missing',base:[214,64,58]},{k:'del',lab:'Deletions',base:[150,40,110]}, + {k:'snp',lab:'SNPs',base:[14,139,168]},{k:'snpkb',lab:'SNPs / kb',base:[22,100,150]}, {k:'het',lab:'Het',base:[221,138,26]},{k:'indel',lab:'Indels',base:[124,92,191]}]; +// A track whose values are a share of the bin (0-1) rather than a count: drawn on a fixed 0-100% scale. +function gtrackFrac(k){return k=='missing'||k=='del';} function gtrackHas(k){return R.samples.some(function(s){return k=='missing'?!!s.miss:(s.trk&&s.trk[k]);});} function gtrackMax(k){var mx=0;R.samples.forEach(function(s){var p=k=='missing'?s.miss:(s.trk&&s.trk[k]);if(p)p.forEach(function(v){if(v!=null&&v>mx)mx=v;});});return mx;} function gcol(base,mv){if(mv==null)return TH.cellnull;if(mv>1)mv=1;var a=isDark()?[30,42,56]:[238,244,240]; @@ -55,7 +58,7 @@ function renderGenome(){ var rows0=samp.filter(function(s){return (tk=='missing')?s.miss:(s.trk&&s.trk[tk]);}); if(!rows0.length){host.innerHTML='no data for this track.';return;} var nb=(tk=='missing')?rows0[0].miss.length:rows0[0].trk[tk].length, gl=R.genome_len||nb; - var mx=(tk=='missing')?1:(gtrackMax(tk)||1); + var mx=gtrackFrac(tk)?1:(gtrackMax(tk)||1); var vis={}; visible().forEach(function(s){vis[s.s]=1;}); var rows=rows0.slice().sort(function(a,b){var la=(R.lineages||[]).indexOf(a.lineage),lb=(R.lineages||[]).indexOf(b.lineage); if(la<0)la=999; if(lb<0)lb=999; if(la!=lb)return la-lb; return a.s.localeCompare(b.s);}); @@ -73,18 +76,18 @@ function renderGenome(){ var pmax=Math.max.apply(null,agg.slice(z0,z1+1).map(function(v){return v||0;}).concat([1e-9])); var pts=[];for(var pi=z0;pi<=z1;pi++){var pv=agg[pi]||0;pts.push(x(pi).toFixed(1)+','+(profH-(pv/pmax)*(profH-10)).toFixed(1));} var bs='rgb('+base[0]+','+base[1]+','+base[2]+')',bf='rgba('+base[0]+','+base[1]+','+base[2]+',.15)'; - var TLAB={missing:'missing %',snp:'SNP density',het:'het density',indel:'indel density'}; + var TLAB={missing:'missing %',del:'deleted %',snp:'SNP density',snpkb:'SNPs per callable kb',het:'het density',indel:'indel density'}; var svg=''; svg+=''; svg+=''; svg+=''; svg+=''+TLAB[tk]+''; - svg+='peak '+(tk=='missing'?(pmax*100).toFixed(0)+'%':(Math.round(pmax*10)/10)+'/bin')+''; - for(var pj=z0;pj<=z1;pj++)svg+=''; + svg+='peak '+(gtrackFrac(tk)?(pmax*100).toFixed(0)+'%':(Math.round(pmax*10)/10)+(tk=='snpkb'?'/kb':'/bin'))+''; + for(var pj=z0;pj<=z1;pj++)svg+=''; rows.forEach(function(s,r){var dim=(st.q||st.onlyFlagged||st.flagFilter||st.ancOnly||st.linFilter)&&!vis[s.s],yy=y0+r*rowH; svg+=''; - for(var i=z0;i<=z1;i++){var vv=raw(s,i),mv=vv==null?null:(tk=='missing'?vv:vv/mx); - svg+='';} + for(var i=z0;i<=z1;i++){var vv=raw(s,i),mv=vv==null?null:(gtrackFrac(tk)?vv:vv/mx); + svg+='';} }); var yb=y0+rows.length*rowH; if(mb)for(var mk=z0;mk<=z1;mk++){var mf=mb[mk]||0;if(mf<=0)continue; // grey out the masked zones @@ -92,7 +95,7 @@ function renderGenome(){ [0,0.25,0.5,0.75,1].forEach(function(t){var px=gut+t*plotW,pos=Math.round((z0+t*winN)/nb*gl); svg+=''+ ''+fmtpos(pos)+'';}); - if(tk=='snp'&&st.geneMark&&st.geneMark.b1>=z0&&st.geneMark.b0<=z1){var gm=st.geneMark,mx0=Math.max(gut,x(gm.b0)),mx1=Math.min(gut+plotW,x(gm.b1+1)); + if((tk=='snp'||tk=='snpkb'||tk=='del')&&st.geneMark&&st.geneMark.b1>=z0&&st.geneMark.b0<=z1){var gm=st.geneMark,mx0=Math.max(gut,x(gm.b0)),mx1=Math.min(gut+plotW,x(gm.b1+1)); svg+=''+ ''+esc(gm.name||'')+'';} svg+=''; @@ -104,7 +107,9 @@ function renderGenome(){ var gz=el('genzoom'); if(gz){ if(genomeZoom<=0)gz.value=rowH; gz.oninput=function(){ genomeZoom=+this.value; renderGenome(); }; } // row-height zoom (re-renders; slider lives in the h2 so the drag survives) var lg=el('genome_legend'); // the scale of the track on screen (it used to be a fixed purple key over a red heatmap) if(lg)lg.innerHTML=''+ - (tk=='missing'?'callable → all missing':'none → '+(Math.round(mx*10)/10)+' '+TLAB[tk].replace(' density','s')+' per bin')+''+ + (tk=='missing'?'callable → all missing':tk=='del'?'read → the whole bin in a stretch without reads that other samples read': + tk=='snpkb'?'none → '+(Math.round(mx*10)/10)+' SNPs per callable kb (grey: under a tenth of the bin callable)': + 'none → '+(Math.round(mx*10)/10)+' '+TLAB[tk].replace(' density','s')+' per bin')+''+ (mb?'masked':'')+ 'top strip = cohort mean; hover a cell for the position and value'; } diff --git a/bin/report_assets/js/09b_coverage.js b/bin/report_assets/js/09b_coverage.js new file mode 100644 index 0000000..592b848 --- /dev/null +++ b/bin/report_assets/js/09b_coverage.js @@ -0,0 +1,80 @@ +// ---- Deletions: the stretches no read covers, read against the rest of the cohort ---- +// A stretch without reads is a deletion only when other samples on the same reference read it. +// When nobody does, it is a repeat or a part of the reference none of these genomes has, and +// those are listed apart rather than mixed in with what the samples lost. +var DEL_CLS={private:['private','only this sample lacks it; the other samples on its reference read it'], + shared:['shared','several samples lack it; the others on the reference read it'], + alone:['alone','the only sample on its reference read deeply enough, so nothing tells a deletion from a stretch the reference has and these genomes do not'], + cohort:['nobody reads it','no sample reads it: a repeat, or a part of the reference none of these genomes has']}; +var delView='del'; +function delBins(r){ // a region's landscape bins: the landscape lays each sample's contigs end to end + var C=R.coverage,nb=R.nbins||200,cs=(C.refs&&C.refs[r.ref])||[],off=0,tot=0,i; + for(i=0;i1; + return (multi&&x.ref!=x.contig?esc(x.ref)+' · ':'')+esc(x.contig)+':'+x.start.toLocaleString('en-US')+'–'+x.end.toLocaleString('en-US');} +function delMatch(r,q){ + if(!q)return true; + if((r.contig+':'+r.start+'-'+r.end).toLowerCase().indexOf(q)>=0)return true; + if(r.genes.some(function(g){return g.toLowerCase().indexOf(q)>=0;}))return true; + return r.samples.some(function(x){return String(x[0]).toLowerCase().indexOf(q)>=0;});} +function renderDeletions(){ + var host=el('del_body'); if(!host)return; + var C=R.coverage; + if(!(C&&C.regions)){host.innerHTML='';return;} + var q=(st.delq||'').toLowerCase(), all=C.regions; + var cnt={private:0,shared:0,alone:0,cohort:0}; all.forEach(function(r){cnt[r.cls]=(cnt[r.cls]||0)+1;}); + var nDel=cnt.private+cnt.shared+cnt.alone; + var regs=all.filter(function(r){return ((delView=='cohort')==(r.cls=='cohort'))&&delMatch(r,q);}); + var big=el('deletionsPanel')&&el('deletionsPanel').classList.contains('expanded'), cap=q?400:(big?300:40); + var lede=''+nDel+' '+_plural(nDel,'stretch','stretches')+' of '+C.min_len+' bp or more that some samples do not read and others do: '+ + ''+cnt.private+' private to one sample, '+cnt.shared+' shared'+(cnt.alone?(', '+cnt.alone+' in a sample alone on its reference'):'')+'. '+ + ''+cnt.cohort+' '+_plural(cnt.cohort,'stretch','stretches')+' no sample reads '+_plural(cnt.cohort,'is','are')+' set apart: a repeat or a part of the reference these genomes lack is nobody's deletion. '+ + 'Measured on the '+C.n_assessed+' samples read at a median depth of '+C.min_depth+'× or more'+ + (C.not_assessed&&C.not_assessed.length?('; '+C.not_assessed.length+' thinner '+_plural(C.not_assessed.length,'sample was','samples were')+' left out, since stretches without reads turn up there by chance'):'')+'.'; + var seg=''+ + ''; + var rows=regs.slice(0,cap).map(function(r){ + var bb=delBins(r), names=r.samples.map(function(x){return x[0];}); + var who=_list(names.map(function(s){return esc(s);}),4); + var tipS=r.samples.map(function(x){return x[0]+': '+x[1].toLocaleString('en-US')+'-'+x[2].toLocaleString('en-US');}).join('\n'); + var genes=r.genes.length?_list(r.genes.map(function(g){return esc(g)+geneRvTag(g);}),6):'none'; + return ''+ + ''+delWhere(r)+''+ + ''+r.len.toLocaleString('en-US')+''+ + ''+r.n+' of '+r.of+''+ + ''+who+''+ + ''+genes+''+ + ''+esc(DEL_CLS[r.cls][0])+'';}).join(''); + var head='StretchbpSamplesWhichGenes it removesClass'; + var empty=''+(q?('nothing matches "'+esc(q)+'".'):(delView=='cohort'?'every stretch without reads is read by some sample.':'no sample lacks a stretch the others read.'))+''; + var more=regs.length>cap?('
Showing '+cap+' of '+regs.length+'. Search, open the panel full, or download the table for the rest.
'):''; + var gl=C.genes_lost||[], gtab=''; + if(gl.length&&delView=='del'){ + gtab='
Genes some samples do not read (under half of the gene read, where most samples on the reference read nearly all of it)
'+ + '
'+ + gl.filter(function(g){return !q||g.gene.toLowerCase().indexOf(q)>=0||g.samples.some(function(x){return String(x[0]).toLowerCase().indexOf(q)>=0;});}).slice(0,cap).map(function(g){ + return ''+ + ''+ + '';}).join('')+ + '
GenePositionSamplesWhich (share of the gene read)
'+esc(g.gene)+geneRvTag(g.gene)+''+delWhere(g)+''+g.samples.length+' of '+g.of+''+_list(g.samples.map(function(x){return esc(x[0])+' ('+Math.round(x[1]*100)+'%)';}),6)+'
';} + host.innerHTML='
'+lede+'
'+ + '
'+seg+ + '
'+ + '
'+head+''+(rows||empty)+'
'+more+gtab; + Array.prototype.forEach.call(host.querySelectorAll('#delseg button'),function(b){b.onclick=function(){delView=b.getAttribute('data-v');renderDeletions();};}); + var dlb=el('deldl'); if(dlb)dlb.onclick=function(){ + var lines=['region\treference\tcontig\tstart\tend\tlength\tclass\tn_samples\tn_assessed\tsamples\tgenes']; + all.forEach(function(r){lines.push([r.id,r.ref,r.contig,r.start,r.end,r.len,r.cls,r.n,r.of, + r.samples.map(function(x){return x[0]+':'+x[1]+'-'+x[2];}).join(','),r.genes.join(',')].join('\t'));}); + dl(lines.join('\n')+'\n','deletions.tsv','text/tab-separated-values');}; + Array.prototype.forEach.call(host.querySelectorAll('#deltable tbody tr[data-b0]'),function(tr){tr.onclick=function(){ + var b0=+tr.getAttribute('data-b0'),b1=+tr.getAttribute('data-b1'),nb=R.nbins||200,pad=Math.max(3,Math.round((b1-b0)*0.6)+2); + if(gtrackHas('del'))st.gtrack='del'; + st.geneMark={b0:b0,b1:b1,name:tr.getAttribute('data-id')||''}; st.gzoom={b0:Math.max(0,b0-pad),b1:Math.min(nb-1,b1+pad)}; + Array.prototype.forEach.call(document.querySelectorAll('#gtrack button'),function(x){x.classList.toggle('on',x.getAttribute('data-gt')==st.gtrack);}); + renderGenome(); el('genome').scrollIntoView();};}); +} diff --git a/bin/report_assets/js/14_boot.js b/bin/report_assets/js/14_boot.js index 4c840d0..1d075ab 100644 --- a/bin/report_assets/js/14_boot.js +++ b/bin/report_assets/js/14_boot.js @@ -5,7 +5,7 @@ var PAGES=['summary','qc','genome','variants','drug','gconv','diag']; var PAGE_RENDER={ summary:function(){renderExec();}, qc:function(){renderOverview();renderFlags();renderCuration();renderTable();renderLineages();renderKraken();renderPlots();renderADNA();}, - genome:function(){renderStacks();renderGenome();renderFunction();renderGeneBurden();renderHotspots();renderPnps();}, + genome:function(){renderStacks();renderGenome();renderDeletions();renderFunction();renderGeneBurden();renderHotspots();renderPnps();}, variants:function(){renderDynamics();renderEpistasis();renderSnpMatrix();renderVarDose();}, drug:function(){renderDrug();}, gconv:function(){renderGconv();}, @@ -63,7 +63,7 @@ Array.prototype.forEach.call(document.querySelectorAll('#colmenu input'),functio if(cn)cn.onclick=function(e){e.preventDefault();Array.prototype.forEach.call(document.querySelectorAll('#colmenu input'),function(cb){cb.checked=false;st.hidden[cb.getAttribute('data-k')]=1;});renderTable();saveState();};})(); el('q').oninput=function(e){st.q=e.target.value.toLowerCase().trim();renderTable();clearTimeout(_qdb);_qdb=setTimeout(function(){renderPlots();renderScatter();renderCorr();renderQCspace();renderRefBias();renderStacks();renderGenome();renderFunction();renderTemporal();renderGconv();},160);}; // per-panel gene search (Functional gene burden / Variable genes / pN-pS): filter each gene table by gene name -[['gbq','gbq',renderGeneBurden],['hotq','hotq',renderHotspots],['pnpsq','pnpsq',renderPnps],['vardoseq','vardoseq',renderVarDose]].forEach(function(w){var inp=el(w[0]);if(inp)inp.oninput=function(e){st[w[1]]=e.target.value.trim();w[2]();};}); +[['gbq','gbq',renderGeneBurden],['hotq','hotq',renderHotspots],['delq','delq',renderDeletions],['pnpsq','pnpsq',renderPnps],['vardoseq','vardoseq',renderVarDose]].forEach(function(w){var inp=el(w[0]);if(inp)inp.oninput=function(e){st[w[1]]=e.target.value.trim();w[2]();};}); el('of').onchange=function(e){st.onlyFlagged=e.target.checked;renderAll();}; Array.prototype.forEach.call(document.querySelectorAll('#ptype button'),function(b){b.onclick=function(){st.ptype=b.getAttribute('data-t'); Array.prototype.forEach.call(document.querySelectorAll('#ptype button'),function(x){x.classList.toggle('on',x==b);});renderPlots();};}); @@ -158,11 +158,11 @@ wireHover(el('plots')); wireHover(el('scatter')); wireHover(el('qcpca_body')); w document.addEventListener('mousemove',move);document.addEventListener('mouseup',up);e.preventDefault();});})(); // genome landscape hover + click (function(){var g=el('genome_body'); if(!g)return; - var LAB={missing:'missing',snp:'homozygous SNPs',het:'het variants',indel:'indels'}; + var LAB={missing:'missing',del:'deleted',snp:'homozygous SNPs',snpkb:'SNPs per callable kb',het:'het variants',indel:'indels'}; g.addEventListener('mousemove',function(e){var t=e.target; if(t.tagName=='rect'&&t.hasAttribute('data-v')){var bin=+t.getAttribute('data-bin'),nb=R.nbins||200,gl=R.genome_len||nb,tk=st.gtrack||'missing'; var p0=Math.round(bin/nb*gl),p1=Math.round((bin+1)/nb*gl),v=t.getAttribute('data-v'); - var val=(v==='')?'NA':(tk=='missing'?v+'% missing':v+' '+LAB[tk]); + var val=(v==='')?'NA':(tk=='missing'?v+'% missing':tk=='del'?v+'% in a stretch without reads that other samples read':v+' '+LAB[tk]); tip(''+esc(t.getAttribute('data-s')||'cohort (mean)')+'
'+fmtpos(p0)+' - '+fmtpos(p1)+'
'+val,e.clientX,e.clientY);} else tip('');}); g.addEventListener('mouseleave',function(){tip('');}); @@ -206,6 +206,7 @@ function genomeResetZoom(){st.gsel=null;st.gzoom=null;st.geneMark=null;var gg=el var rd=el('gselreadout');if(rd)rd.innerHTML='gene '+esc(g.name)+' · '+fmtpos(g.start)+' - '+fmtpos(g.end)+' '; var cb=el('gselclear');if(cb)cb.onclick=function(ev){ev.stopPropagation();genomeResetZoom();};};})(); if(!R.samples.some(function(s){return s.miss||s.trk;})){var gs=el('genome');if(gs)gs.style.display='none';var ng=el('nav-genome');if(ng)ng.style.display='none';} +if(!(R.coverage&&R.coverage.regions)){var dlx=el('deletions');if(dlx)dlx.style.display='none';var ndlx=el('nav-del');if(ndlx)ndlx.style.display='none';} if(!R.samples.some(function(s){return s.trk&&s.trk.snp;})){var hsx=el('hotspots');if(hsx)hsx.style.display='none';var nhx=el('nav-hot');if(nhx)nhx.style.display='none';} // signature layer gates: hide a panel when its data type is absent cohort-wide (organism-agnostic; render fns also degrade per-view) if(!pca2(R.samples).ok){var qp=el('qcpca');if(qp)qp.style.display='none';var nqp=el('nav-pca');if(nqp)nqp.style.display='none';} @@ -336,7 +337,7 @@ Array.prototype.forEach.call(document.querySelectorAll('.exp-h'),function(b){b.o var willExpand=!panel.classList.contains('expanded'); collapseExpanded(); if(willExpand){panel.classList.add('expanded');document.body.classList.add('has-expanded');b.innerHTML=icon('minimize')+'close';} var rn=b.getAttribute('data-render'); - setTimeout(function(){if(rn=='genome')renderGenome();else if(rn=='plots')renderPlots();else if(rn=='scatter')renderScatter();else if(rn=='corr')renderCorr();else if(rn=='pca')renderQCspace();else if(rn=='divcomp')renderRefBias();else if(rn=='function')renderFunction();else if(rn=='geneburden')renderGeneBurden();else if(rn=='hotspots')renderHotspots();else if(rn=='pnps')renderPnps();else if(rn=='dosetx')renderDoseTx();else if(rn=='vardose')renderVarDose();else if(rn=='gconv')renderGconv();else if(rn=='table')renderTable();},20);};}); + setTimeout(function(){if(rn=='genome')renderGenome();else if(rn=='plots')renderPlots();else if(rn=='scatter')renderScatter();else if(rn=='corr')renderCorr();else if(rn=='pca')renderQCspace();else if(rn=='divcomp')renderRefBias();else if(rn=='function')renderFunction();else if(rn=='geneburden')renderGeneBurden();else if(rn=='hotspots')renderHotspots();else if(rn=='deletions')renderDeletions();else if(rn=='pnps')renderPnps();else if(rn=='dosetx')renderDoseTx();else if(rn=='vardose')renderVarDose();else if(rn=='gconv')renderGconv();else if(rn=='table')renderTable();},20);};}); if(el('expClose'))el('expClose').onclick=collapseExpanded; document.addEventListener('keydown',function(e){if(e.key=='Escape')collapseExpanded();}); // per-sample detail modal close (button, backdrop, Esc) diff --git a/bin/report_assets/shell.html b/bin/report_assets/shell.html index 58448ae..5c41f70 100644 --- a/bin/report_assets/shell.html +++ b/bin/report_assets/shell.html @@ -8,7 +8,7 @@
BAMpiro QC
1Summary
2Sample QC
Flagged samplesAll samplesLineagesContaminationDistributionsaDNA damage
-
3Genome & genes
ConsensusGenome landscapeFunctional impactGene burdenVariable genesSelection (dN/dS)
+
3Genome & genes
ConsensusGenome landscapeDeletionsFunctional impactGene burdenVariable genesSelection (dN/dS)
4Variants over time
SNP dynamicsCo-varying pairsSNP matrixVariant × dose
5Resistance
Resistance mutations
6Gene conversion
Candidate tracts
@@ -67,7 +67,7 @@
-
3 · Genome & genes

How much of the genome each sample recovers, and where its variants fall

Completeness of each consensus, where along the reference calls are missing or variants concentrate, and which genes carry them.

+
3 · Genome & genes

How much of the genome each sample recovers, and where its variants fall

Completeness of each consensus, where along the reference calls are missing, reads are absent or variants concentrate, and which genes carry them.

Consensus completeness callable / ambiguous (IUPAC) / missing share of each consensus, worst first

@@ -78,6 +78,9 @@
+

Deletions stretches a sample has no reads for while other samples on its reference do; click a row to show it in the landscape +

+

Functional impact snpEff impact classes per sample; the missense/silent ratio is a proxy, not a selection test

diff --git a/docs/configuration.md b/docs/configuration.md index 3607363..890b73c 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -167,6 +167,8 @@ Two documented, ready-to-use **organism** configs live in | | `--blindspot_liftover` | `true` | Lift the blind-spots onto the run reference (k-mer liftover) so the mask is correct on **any** reference. | | **QC Report** | `--make_qc_report` | `true` | Build the interactive HTML QC report + `qc_flags.tsv`. | | | `--make_snp_matrix` | `true` | Also emit the master SNP-matrix TSV. | +| | `--depth_window` | `1000` | Window (bp) of each sample's depth profile, behind the deletions and SNPs per callable kb. | +| | `--deletion_min_len` | `200` | Shortest stretch without reads reported as a deletion. | | | `--report_gate` | `false` | Fail the run if any sample is flagged **FAIL**. | | | `--report_depth_min` | `10` | Gate: min mean depth (all `report_*` cut-offs are editable live in the report). | | | `--report_breadth_min` | `90` | Gate: min breadth %. | diff --git a/docs/outputs.md b/docs/outputs.md index dcde5ab..5ab1767 100644 --- a/docs/outputs.md +++ b/docs/outputs.md @@ -14,6 +14,7 @@ results_bampiro/ ├── samples_legio_qc_flags.tsv # Per-sample PASS/WARN/FAIL verdicts ├── samples_legio_summary.tsv # Cohort metrics table (feeds the report) ├── samples_legio_snp_matrix.tsv # Master SNP matrix (site × sample AF & depth) +├── samples_legio_deletions.tsv # Stretches some samples have no reads for (see below) ├── samples_legio_gene_burden.tsv # Per-gene functional burden (cohort) ├── samples_legio_dr.tsv # 💊 Drug-resistance calls (pathotypr; only if --run_pathotypr) │ @@ -59,6 +60,9 @@ results_bampiro/ ├── MP00091__.dr_mutations.tsv # -> Pathotypr per-sample DR mutations (only if --run_pathotypr) ├── MP00091.LENS.gene_conversion.tsv # -> Conversion tracts (only if --find_gene_conversion) ├── MP00091.LENS.gene_conversion_loci.tsv # -> One row per paralog pair looked at, found or not + ├── MP00091.LENS.depth_windows.tsv # -> Depth per 1 kb window: mean, share without reads, share callable + ├── MP00091.LENS.zero_depth.tsv # -> Every stretch of 50+ bp no read covers + ├── MP00091.LENS.gene_depth.tsv # -> Per gene: share read, share callable, depth against the genome └── Locus_to_exclude_LENS.txt # -> List of repetitive regions excluded from calling ``` @@ -91,6 +95,30 @@ genotype's dosage passed off as a fraction. Rows are keyed on contig and position, so two references that share a contig name cannot be told apart; the matrix step warns when it sees one. +## Deletions + +`_deletions.tsv` lists the stretches of the reference that some samples have no reads +for. A stretch without reads means different things depending on the rest of the cohort, so each +one is compared with the other samples mapped to the same reference: + +| `class` | Meaning | +| :--- | :--- | +| `private` | One sample lacks it and the others read it: a deletion in that sample | +| `shared` | Several samples lack it and the others read it: often a deletion that defines a lineage | +| `cohort` | Nearly every sample lacks it: a repeat no read can be placed on, or a part of the reference none of these genomes has. Nobody's deletion | +| `alone` | The only sample on its reference read deeply enough, so there is nothing to compare it with | + +Only stretches of at least `--deletion_min_len` bp (200) count, and only in samples whose median +depth reaches `--report_depth_min`: in a thinly read sample, stretches without reads turn up by +chance. The `samples` column gives each carrier's own coordinates and `genes` the genes a carrier +reads less than half of. + +The per-sample tables behind it are in each sample's `stats/` folder and are useful on their own: +`gene_depth.tsv` says, for every gene of the GFF, how much of it the sample read and at what depth +against its genome-wide median, which is also where an amplification shows as a depth well above 1. +A deletion shorter than the minimum, or one the variant caller already reported as an indel, is +not in this file. + ## Gene conversion Off unless `--find_gene_conversion true`. Four files, and which one to open depends on the diff --git a/docs/qc-report.md b/docs/qc-report.md index 2244057..fd76197 100644 --- a/docs/qc-report.md +++ b/docs/qc-report.md @@ -28,7 +28,7 @@ on *Sample QC*, the samples with resistance mutations their lineage does not sha | :--- | :--- | | **1 · Summary** | What the run found, as sentences with their numbers: an *In short* paragraph, then one card per finding (sample QC, identity, resistance, variants over time, lineages, gene conversion, coverage). Every card links to the page that holds its evidence and says what it does not prove. | | **2 · Sample QC** | Which samples can be trusted: the verdicts, the live thresholds, the **flagged samples** with the value behind every flag, the **exclusion list** and its exports, the table of all samples, lineages, **contamination** (Kraken2), distributions and aDNA damage. | -| **3 · Genome & genes** | Consensus completeness, the genome landscape, functional impact (snpEff), gene burden, variable genes and dN/dS. | +| **3 · Genome & genes** | Consensus completeness, the genome landscape (missing calls, **deletions**, SNPs per bin and **per callable kb**, het, indels), the **deletions** found by comparing each sample's stretches without reads with the rest of the cohort, functional impact (snpEff), gene burden, variable genes and dN/dS. | | **4 · Variants over time** | SNP dynamics, co-varying pairs (epistasis), the SNP matrix and variant × dose. | | **5 · Resistance** | WHO-catalogue mutations grouped by mutation, the sample × drug matrix and every call. | | **6 · Gene conversion** | Candidate tracts and the evidence behind each verdict. | diff --git a/modules/report.nf b/modules/report.nf index 8802fe6..c5cc1ba 100644 --- a/modules/report.nf +++ b/modules/report.nf @@ -1,4 +1,4 @@ -include { asBool } from './utils' +include { asBool; getSampleDir; getSavePath } from './utils' nextflow.enable.dsl=2 /* ==================================================================== @@ -124,12 +124,14 @@ process QC_REPORT { path(dr_report) // run drug-resistance calls TSV (collect_dr) -> Drug resistance panel (may be NO_FILE) path(gene_conversion) // cohort gene-conversion tracts TSV (COLLECT_GENE_CONVERSION) -> Gene conversion panel (may be empty) path(kraken_reports) // per-sample Kraken2 .report files -> Taxonomic composition panel (may be empty) + path(depth_profiles, stageAs: 'depth/*') // per-sample DEPTH_PROFILE tables -> deletions + SNPs per callable kb (may be empty) val(provenance) // pre-quoted provenance tokens (container=..., reference=...) val(basename) output: path("${basename}_qc_report.html"), emit: html path("${basename}_qc_flags.tsv"), emit: flags + path("${basename}_deletions.tsv"), emit: deletions, optional: true script: def gate_arg = asBool(params.report_gate) ? "--gate" : "" @@ -149,12 +151,15 @@ process QC_REPORT { # A header-only TSV (the stage ran but found nothing) parses to no tracts, so the panel self-hides. GC_ARG=""; case "${gene_conversion}" in ""|NO_FILE*) ;; *) [ -s "${gene_conversion}" ] && GC_ARG="--gene-conversion ${gene_conversion}";; esac KRK_ARG=""; [ -n "${kraken_reports}" ] && KRK_ARG="--kraken ${kraken_reports}" + DEL_ARG=""; [ -n "${depth_profiles}" ] && DEL_ARG="--depth-profiles ${depth_profiles} --out-deletions ${basename}_deletions.tsv" python3 ${projectDir}/bin/qc_report.py \\ --summary ${summary} \\ ${cons_arg} \\ --gene-burden ${gene_burden} \\ --gff ${gff} \\ - \$MASK_ARG \$LC_ARG \$MD_ARG \$VCF_ARG \$VH_ARG \$PL_ARG \$DR_ARG \$GC_ARG \$KRK_ARG \\ + \$MASK_ARG \$LC_ARG \$MD_ARG \$VCF_ARG \$VH_ARG \$PL_ARG \$DR_ARG \$GC_ARG \$KRK_ARG \$DEL_ARG \\ + --deletion-min-len ${params.deletion_min_len} \\ + --deletion-min-depth ${params.report_depth_min} \\ --aa2-label "${params.canonical_label}" \\ --provenance ${provenance} \\ --version "${workflow.manifest.version}" \\ @@ -177,6 +182,42 @@ process QC_REPORT { """ } +process DEPTH_PROFILE { + // What one sample's reads cover, reduced from its all-positions VCF to windows, the stretches + // no read covers, and a per-gene table. The report reads them against the rest of the cohort, + // which is what tells a deletion from a part of the reference none of these genomes has. + tag "Depth profile: ${sampleId}" + publishDir path: { "${params.outdir}/${getSampleDir(sampleId, params)}" }, mode: params.publish_mode, saveAs: { filename -> getSavePath(filename, params) } + cpus 1 + memory { 2.GB * task.attempt } + + input: + tuple val(sampleId), val(refId), path(allpos_vcf), path(gff) + + output: + tuple val(sampleId), val(refId), path("${sampleId}.${refId}.depth_windows.tsv"), path("${sampleId}.${refId}.zero_depth.tsv"), emit: profile + // Absent when the GFF has neither gene nor CDS features. + tuple val(sampleId), val(refId), path("${sampleId}.${refId}.gene_depth.tsv"), optional: true, emit: genes + + script: + """ + set -euo pipefail + python3 ${projectDir}/bin/depth_profile.py \\ + --vcf ${allpos_vcf} \\ + --gff ${gff} \\ + --reference ${refId} \\ + --window ${params.depth_window} \\ + --min-dp ${params.consensus_min_dp} \\ + --out-prefix ${sampleId}.${refId} + """ + + stub: + """ + printf '# sample=${sampleId}\\ncontig\\tstart\\tend\\tmean_dp\\tzero_frac\\tcallable_frac\\n' > ${sampleId}.${refId}.depth_windows.tsv + printf '# sample=${sampleId}\\ncontig\\tstart\\tend\\tlength\\n' > ${sampleId}.${refId}.zero_depth.tsv + """ +} + process SNP_MATRIX { tag "SNP matrix" publishDir "${params.outdir}", mode: params.publish_mode diff --git a/nextflow.config b/nextflow.config index 22fd727..1e42c33 100644 --- a/nextflow.config +++ b/nextflow.config @@ -228,6 +228,12 @@ params { report_dup_max = 40 report_iupac_max = 2 report_mapping_min = 80 + // Deletions: every sample's depth is summarised per window of this many bp, and a stretch + // without reads of at least deletion_min_len bp that other samples on the reference DO read is + // reported as a deletion. Samples under report_depth_min are not assessed: stretches without + // reads turn up there by chance. + depth_window = 1000 + deletion_min_len = 200 // --- Consensus Settings --- consensus_wrap = 80 diff --git a/subworkflows/cohort_report.nf b/subworkflows/cohort_report.nf index b11aa5d..e1d4ad5 100644 --- a/subworkflows/cohort_report.nf +++ b/subworkflows/cohort_report.nf @@ -6,7 +6,7 @@ */ include { ANNOTATE_CANONICAL } from '../modules/annotation' -include { COLLECT_SUMMARY; COLLECT_DR; LIFT_VARIANTS; QC_REPORT; SNP_MATRIX } from '../modules/report' +include { COLLECT_SUMMARY; COLLECT_DR; DEPTH_PROFILE; LIFT_VARIANTS; QC_REPORT; SNP_MATRIX } from '../modules/report' include { asBool } from '../modules/utils' workflow COHORT_REPORT { @@ -96,9 +96,15 @@ workflow COHORT_REPORT { def pos_liftover = asBool(params.variant_liftover) ? LIFT_VARIANTS(report_vcfs, report_ref_fa, report_ref, tsv_name).map : file("${projectDir}/assets/NO_FILE_LIFTOVER") + // What each sample's reads cover, from its all-positions VCF and its reference's GFF: the + // report reads them against each other for deletions and SNPs per callable kb. + def depth = DEPTH_PROFILE(allpos_vcf.map { sId, rId, vcf, tbi -> tuple(sId, rId, vcf, file(refGffMap[rId])) }) + def depth_profiles = depth.profile.map { sId, rId, win, zero -> [win, zero] } + .mix(depth.genes.map { sId, rId, genes -> [genes] }) + .flatten().collect().ifEmpty([]) QC_REPORT(summ.summary, summ.gene_burden, cons_files, report_gff, report_mask, report_meta, report_vcfs, report_vcfs_h37rv, pos_liftover, dr_report, report_gconv, - report_kraken, provenance, tsv_name) + report_kraken, depth_profiles, provenance, tsv_name) } // 11b. Master SNP matrix: rows = SNP sites, columns = reference/annotation + per-sample AF & depth. diff --git a/tests/js/coverage.test.mjs b/tests/js/coverage.test.mjs new file mode 100644 index 0000000..b10a356 --- /dev/null +++ b/tests/js/coverage.test.mjs @@ -0,0 +1,43 @@ +// The deletions panel places a region on the genome landscape and searches it. Both read the +// payload alone, so they are checked on small hand-built cohorts. + +import { strict as assert } from "node:assert"; +import { describe, it } from "node:test"; +import { emptyReport, loadFunctions } from "./harness.mjs"; + +function report(coverage, more = {}) { + return { ...emptyReport(), samples: [], metrics: [], nbins: 10, genome_len: 1000, coverage, ...more }; +} + +describe("delBins, a region on the landscape", () => { + it("uses the landscape's own binning over one contig", () => { + const rep = report({ refs: { R: [["chr", 1000]] }, regions: [] }); + const fn = loadFunctions(["delBins"], rep); + assert.deepEqual(JSON.parse(JSON.stringify(fn.delBins({ ref: "R", contig: "chr", start: 250, end: 449 }))), [2, 4]); + }); + + it("lays the contigs of a reference end to end, as the landscape does", () => { + // 600 bp of chromosome then a 400 bp plasmid: plasmid position 1 is genome position 601. + const rep = report({ refs: { R: [["chr", 600], ["plasmid", 400]] }, regions: [] }); + const fn = loadFunctions(["delBins"], rep); + assert.deepEqual(JSON.parse(JSON.stringify(fn.delBins({ ref: "R", contig: "plasmid", start: 1, end: 400 }))), [6, 9]); + }); + + it("scales a reference of another length to its own genome", () => { + const rep = report({ refs: { A: [["a", 1000]], B: [["b", 2000]] }, regions: [] }); + const fn = loadFunctions(["delBins"], rep); + assert.deepEqual(JSON.parse(JSON.stringify(fn.delBins({ ref: "B", contig: "b", start: 1000, end: 1999 }))), [5, 9]); + }); +}); + +describe("delMatch, searching the deletions", () => { + const region = { ref: "R", contig: "chr", start: 1200, end: 3400, genes: ["katG"], samples: [["S0", 1200, 3000]] }; + const fn = loadFunctions(["delMatch"], report({ refs: {}, regions: [] })); + + it("finds a region by sample, gene or coordinates", () => { + assert.equal(fn.delMatch(region, "s0"), true); + assert.equal(fn.delMatch(region, "katg"), true); + assert.equal(fn.delMatch(region, "chr:1200"), true); + assert.equal(fn.delMatch(region, "rpob"), false); + }); +}); diff --git a/tests/unit/test_depth_profile.py b/tests/unit/test_depth_profile.py new file mode 100644 index 0000000..92a9594 --- /dev/null +++ b/tests/unit/test_depth_profile.py @@ -0,0 +1,196 @@ +"""Unit tests for bin/depth_profile.py: what one sample's reads cover, from its all-positions VCF.""" + +from __future__ import annotations + +import gzip + +import pytest + +from conftest import load_script + +dp = load_script("depth_profile") + + +def _backbone(contig, pos, depth): + gt = "0" if depth >= 30 else "./." + return "\t".join([contig, str(pos), ".", "A", ".", ".", ".", f"ADP={depth};WT=1;HET=0;HOM=0;NC=0", + "GT:DP:AD", f"{gt}:{depth}:{depth},0"]) + + +def _allpos(path, depths, sample="S1", contigs=None, extra=()): + """An all-positions VCF: `depths` maps contig -> list of depths from position 1 (None skips + the record), `contigs` maps contig -> the length its ##contig line states.""" + contigs = contigs if contigs is not None else {c: len(v) for c, v in depths.items()} + head = "##fileformat=VCFv4.2\n" + "".join( + f"##contig=\n" if n else f"##contig=\n" for c, n in contigs.items()) + head += f"#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\t{sample}\n" + body = [_backbone(c, i + 1, d) for c, v in depths.items() for i, d in enumerate(v) if d is not None] + text = head + "\n".join(list(body) + list(extra)) + "\n" + if str(path).endswith(".gz"): + with gzip.open(path, "wt") as fh: + fh.write(text) + else: + path.write_text(text) + return path + + +# ------------------------------------------------------------------------------------ reading + + +def test_read_depths_keeps_a_missing_record_apart_from_depth_zero(tmp_path): + p = _allpos(tmp_path / "a.vcf", {"chr": [5, 0, None, 7]}) + + sample, depth, known = dp.read_depths(str(p)) + + assert sample == "S1" + assert list(depth["chr"]) == [5, 0, 0, 7] + assert list(known["chr"]) == [True, True, False, True], "no record is not the same as no reads" + + +def test_read_depths_reads_dp_wherever_the_format_puts_it(tmp_path): + """A variant record keeps the caller's FORMAT, which need not put DP second.""" + variant = "\t".join(["chr", "2", ".", "C", "T", "50", ".", ".", "GT:AD:DP", "1:2,38:40"]) + p = _allpos(tmp_path / "a.vcf", {"chr": [5, None, 7]}, extra=[variant]) + + _, depth, _ = dp.read_depths(str(p)) + + assert list(depth["chr"]) == [5, 40, 7] + + +def test_read_depths_grows_a_contig_whose_length_is_not_stated(tmp_path): + p = _allpos(tmp_path / "a.vcf.gz", {"chr": [3] * 1500}, contigs={"chr": 0}) + + _, depth, known = dp.read_depths(str(p)) + + assert len(depth["chr"]) == 1500 + assert known["chr"].all() + + +def test_read_depths_does_not_take_an_unstated_depth_for_zero(tmp_path): + record = "\t".join(["chr", "2", ".", "C", ".", ".", ".", ".", "GT:DP:AD", "./.:.:0,0"]) + p = _allpos(tmp_path / "a.vcf", {"chr": [5, None, 7]}, extra=[record]) + + _, _, known = dp.read_depths(str(p)) + + assert not known["chr"][1] + + +# ------------------------------------------------------------------------------------ tables + + +def _arrays(values): + import numpy as np + d = np.array([0 if v is None else v for v in values], dtype=np.int32) + return {"chr": d}, {"chr": np.array([v is not None for v in values])} + + +def test_windows_measure_only_the_positions_with_a_record(): + depth, known = _arrays([10, 0, None, 20] + [8] * 4) + + rows = dp.windows(depth, known, 4, min_dp=7) + + assert rows[0][:3] == ("chr", 1, 4) + assert rows[0][3] == pytest.approx(10.0) # (10 + 0 + 20) / 3 + assert rows[0][4] == pytest.approx(1 / 3) + assert rows[0][5] == pytest.approx(2 / 3) # 10 and 20 are above 7; 0 is not + assert rows[1][5] == pytest.approx(1.0) + + +def test_a_window_without_records_has_no_values(): + depth, known = _arrays([None, None, 5, 5]) + + assert dp.windows(depth, known, 2, 7)[0][3:] == (None, None, None) + + +def test_zero_runs_keep_the_stretches_long_enough_with_one_based_ends(): + depth, known = _arrays([5, 0, 0, 0, 5, 0, 5, 0, 0, 0]) + + assert dp.zero_runs(depth, known, 3) == [("chr", 2, 4), ("chr", 8, 10)] + + +def test_a_missing_record_breaks_a_stretch_without_reads(): + depth, known = _arrays([0, 0, None, 0, 0]) + + assert dp.zero_runs(depth, known, 3) == [] + + +# ------------------------------------------------------------------------------------ genes + + +def _gff(path, rows): + path.write_text("##gff-version 3\n" + "".join("\t".join(r) + "\n" for r in rows)) + return str(path) + + +def test_read_genes_prefers_gene_features_and_keeps_the_contig(tmp_path): + p = _gff(tmp_path / "a.gff", [ + ["chr", ".", "gene", "1", "4", ".", "+", ".", "ID=g1;Name=dnaA;locus_tag=T_0001"], + ["chr", ".", "CDS", "1", "4", ".", "+", ".", "ID=c1;Parent=g1"], + ["plasmid", ".", "gene", "10", "20", ".", "+", ".", "ID=g2;locus_tag=T_0002"], + ]) + + assert dp.read_genes(p) == [("chr", 1, 4, "dnaA", "T_0001"), ("plasmid", 10, 20, "T_0002", "T_0002")] + + +def test_read_genes_falls_back_to_cds_when_there_are_no_genes(tmp_path): + p = _gff(tmp_path / "a.gff", [["chr", ".", "CDS", "5", "9", ".", "+", ".", "ID=cds1;Name=katG"]]) + + assert dp.read_genes(p) == [("chr", 5, 9, "katG", "")] + + +@pytest.mark.parametrize("name", ["NO_FILE", "NO_FILE_GFF", "absent.gff"]) +def test_read_genes_without_a_gff_is_empty(tmp_path, name): + p = tmp_path / name + if name.startswith("NO_FILE"): + p.write_text("") + assert dp.read_genes(str(p)) == [] + + +def test_gene_depths_measure_breadth_callable_depth_and_depth_against_the_median(): + depth, known = _arrays([0, 0, 10, 30, 20, 20]) + + row = dp.gene_depths(depth, known, [("chr", 1, 4, "g", "")], min_dp=7, median=20.0)[0] + + assert row[5] == 4 + assert row[6] == pytest.approx(0.5) # two of four positions read at all + assert row[7] == pytest.approx(0.5) + assert row[8] == pytest.approx(10.0) + assert row[9] == pytest.approx(0.5) + + +def test_gene_depths_of_a_gene_on_a_contig_the_sample_lacks_are_skipped(): + depth, known = _arrays([5, 5]) + + assert dp.gene_depths(depth, known, [("plasmid", 1, 2, "g", "")], 7, 5.0) == [] + + +# ------------------------------------------------------------------------------------ end to end + + +def test_end_to_end_writes_the_three_tables_with_their_provenance(tmp_path): + vcf = _allpos(tmp_path / "S1.all.pos.vcf.gz", {"chr": [20] * 100 + [0] * 60 + [20] * 40}) + gff = _gff(tmp_path / "ref.gff", [["chr", ".", "gene", "101", "160", ".", "+", ".", "Name=lost"]]) + + assert dp.main(["--vcf", str(vcf), "--gff", gff, "--reference", "REF", "--window", "50", + "--out-prefix", str(tmp_path / "S1")]) == 0 + + win = (tmp_path / "S1.depth_windows.tsv").read_text().splitlines() + assert "# sample=S1" in win and "# reference=REF" in win and "# median_dp=20" in win + assert "# contigs=chr:200" in win + rows = [r.split("\t") for r in win if not r.startswith("#")] + assert rows[0] == ["contig", "start", "end", "mean_dp", "zero_frac", "callable_frac"] + assert rows[3] == ["chr", "101", "150", "0.00", "1.0000", "0.0000"] + zero = [r for r in (tmp_path / "S1.zero_depth.tsv").read_text().splitlines() if not r.startswith("#")] + assert zero == ["contig\tstart\tend\tlength", "chr\t101\t160\t60"] + genes = [r.split("\t") for r in (tmp_path / "S1.gene_depth.tsv").read_text().splitlines() + if not r.startswith("#")] + assert genes[1][3] == "lost" and genes[1][6] == "0.0000" and genes[1][9] == "0.0000" + + +def test_end_to_end_without_a_gff_writes_no_gene_table(tmp_path): + vcf = _allpos(tmp_path / "S1.all.pos.vcf", {"chr": [20] * 10}) + + assert dp.main(["--vcf", str(vcf), "--out-prefix", str(tmp_path / "S1")]) == 0 + + assert not (tmp_path / "S1.gene_depth.tsv").exists() + assert (tmp_path / "S1.zero_depth.tsv").exists() diff --git a/tests/unit/test_qc_report_coverage.py b/tests/unit/test_qc_report_coverage.py new file mode 100644 index 0000000..050f9ac --- /dev/null +++ b/tests/unit/test_qc_report_coverage.py @@ -0,0 +1,209 @@ +"""Deletions and callable depth along the genome (bin/qcreport/coverage.py). + +A stretch no read covers means different things depending on the rest of the cohort: nobody reads +it (a part of the reference these genomes lack, or a repeat), several samples lack it (a shared +deletion), or one sample does (a private one). These tests pin that reading. +""" + +from __future__ import annotations + +import pytest + +from conftest import load_script +from qcreport import coverage as cov + +dp = load_script("depth_profile") + + +def profile(zero=(), median=40.0, ref="REF", length=10_000, genes=(), windows=None): + return {"reference": ref, "median_dp": median, "contigs": [("chr", length)], + "zero": [("chr", s, e) for s, e in zero], + "genes": [{"contig": "chr", "start": s, "end": e, "gene": g, "locus": "", "breadth": b, "rel": None} + for g, s, e, b in genes], + "windows": windows or []} + + +def cohort(n, **kw): + return {f"S{i}": profile(**kw) for i in range(n)} + + +# ------------------------------------------------------------------------------------ classes + + +def test_a_stretch_one_sample_lacks_is_its_private_deletion(): + profiles = cohort(5) + profiles["S0"] = profile(zero=[(1001, 3000)]) + + section, _, _ = cov.build_coverage(profiles, min_len=200) + + [r] = section["regions"] + assert (r["cls"], r["n"], r["of"], r["start"], r["end"]) == ("private", 1, 5, 1001, 3000) + assert r["samples"] == [["S0", 1001, 3000]] + + +def test_a_stretch_several_samples_lack_is_a_shared_deletion(): + profiles = cohort(6) + for s in ("S0", "S1"): + profiles[s] = profile(zero=[(1001, 3000)]) + + [r] = cov.build_coverage(profiles)[0]["regions"] + + assert (r["cls"], r["n"]) == ("shared", 2) + + +def test_a_stretch_nobody_reads_is_not_anybodys_deletion(): + """A repeat, or a part of the reference none of these genomes has.""" + profiles = cohort(10, zero=[(5001, 5800)]) + + section, tracks, _ = cov.build_coverage(profiles) + + [r] = section["regions"] + assert (r["cls"], r["n"], r["of"]) == ("cohort", 10, 10) + assert all(v == 0 for v in tracks["S0"]), "nobody's deletion lights up nobody's track" + + +def test_the_only_sample_on_a_reference_has_nothing_to_compare_with(): + profiles = cohort(4) + profiles["X"] = profile(zero=[(1001, 3000)], ref="OTHER") + + regions = cov.build_coverage(profiles)[0]["regions"] + + assert [(r["ref"], r["cls"]) for r in regions] == [("OTHER", "alone")] + + +def test_a_thinly_read_sample_is_not_assessed(): + """At 3x, stretches without reads turn up by chance: they would all read as deletions.""" + profiles = cohort(4) + profiles["thin"] = profile(zero=[(1001, 3000), (6001, 7000)], median=3.0) + + section, tracks, _ = cov.build_coverage(profiles, min_depth=10) + + assert section["regions"] == [] + assert section["not_assessed"] == ["thin"] + assert "thin" not in tracks + + +def test_a_stretch_shorter_than_the_minimum_is_not_a_deletion(): + profiles = cohort(4) + profiles["S0"] = profile(zero=[(1001, 1150)]) + + assert cov.build_coverage(profiles, min_len=200)[0]["regions"] == [] + + +def test_overlapping_stretches_of_different_samples_are_one_region(): + profiles = cohort(6) + profiles["S0"] = profile(zero=[(1001, 3000)]) + profiles["S1"] = profile(zero=[(1200, 3400)]) + + [r] = cov.build_coverage(profiles)[0]["regions"] + + assert (r["start"], r["end"], r["n"]) == (1001, 3400, 2) + assert r["samples"] == [["S0", 1001, 3000], ["S1", 1200, 3400]] + + +def test_a_region_names_the_genes_its_carriers_do_not_read(): + profiles = cohort(4, genes=[("katG", 1101, 2500, 1.0), ("rpoB", 6001, 7000, 1.0)]) + profiles["S0"] = profile(zero=[(1001, 3000)], genes=[("katG", 1101, 2500, 0.0), ("rpoB", 6001, 7000, 1.0)]) + + [r] = cov.build_coverage(profiles)[0]["regions"] + + assert r["genes"] == ["katG"] + + +# ------------------------------------------------------------------------------------ genes + + +def test_a_gene_most_samples_read_and_one_does_not_is_lost_in_that_one(): + profiles = cohort(4, genes=[("katG", 1101, 2500, 0.99)]) + profiles["S0"] = profile(genes=[("katG", 1101, 2500, 0.2)]) + + [g] = cov.build_coverage(profiles)[0]["genes_lost"] + + assert (g["gene"], g["samples"], g["of"]) == ("katG", [["S0", 0.2]], 4) + + +def test_a_gene_nobody_reads_is_not_lost_by_anyone(): + profiles = cohort(4, genes=[("absent", 1101, 2500, 0.0)]) + + assert cov.build_coverage(profiles)[0]["genes_lost"] == [] + + +# ------------------------------------------------------------------------------------ binning + + +def test_spread_uses_the_landscapes_own_binning(): + """stats_to_legacy puts a variant at global position g in bin g * nbins // genome length.""" + contigs = [("a", 60), ("b", 40)] + for contig, pos in (("a", 1), ("a", 9), ("a", 10), ("a", 60), ("b", 1), ("b", 40)): + g = dict(a=0, b=60)[contig] + pos + bins = cov._spread([(contig, pos, pos, 1.0)], contigs, nbins=10) + assert bins.index(1.0) == min(9, g * 10 // 100), (contig, pos) + + +def test_spread_splits_an_interval_across_bins(): + bins = cov._spread([("a", 5, 25, 1.0)], [("a", 100)], nbins=10) + + assert sum(bins) == 21 + assert bins[:3] == [5.0, 10.0, 6.0] + + +def test_snp_per_kb_divides_by_the_callable_positions_of_each_bin(): + windows = [("chr", 1, 500, 40.0, 0.0, 1.0), ("chr", 501, 1000, 40.0, 0.5, 0.5)] + p = profile(length=1000, windows=windows) + + dens = cov.snp_per_kb([2, 1], p, nbins=2) + + # The landscape's bins are 1-based (499 and 501 positions here), hence the approximation. + assert dens == [pytest.approx(4.0, rel=0.01), pytest.approx(4.0, rel=0.01)], \ + "1 SNP in 250 callable bp is 4 per kb" + + +def test_snp_per_kb_is_empty_where_almost_nothing_could_be_called(): + windows = [("chr", 1, 1000, 1.0, 0.95, 0.05)] + + assert cov.snp_per_kb([3], profile(length=1000, windows=windows), nbins=1) == [None] + + +def test_the_deletion_track_is_the_share_of_each_bin_a_sample_lacks(): + profiles = cohort(4, length=1000) + profiles["S0"] = profile(zero=[(1, 250)], length=1000) + + _, tracks, _ = cov.build_coverage(profiles, nbins=2) + + assert tracks["S0"] == [pytest.approx(0.5, abs=0.01), 0.0] + + +# ------------------------------------------------------------------------------------ files + + +def _write_allpos(path, sample, depths): + head = ("##fileformat=VCFv4.2\n##contig=\n#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\t" + "INFO\tFORMAT\t%s\n" % (len(depths), sample)) + body = "".join(f"chr\t{i + 1}\t.\tA\t.\t.\t.\t.\tGT:DP:AD\t0:{d}:{d},0\n" for i, d in enumerate(depths)) + path.write_text(head + body) + + +def test_the_tables_of_depth_profile_are_read_back_by_the_report(tmp_path): + gff = tmp_path / "ref.gff" + gff.write_text("chr\t.\tgene\t400\t600\t.\t+\t.\tName=lost\n") + paths = [] + for i in range(4): + depths = [30] * 1000 if i else [30] * 300 + [0] * 400 + [30] * 300 + _write_allpos(tmp_path / f"S{i}.vcf", f"S{i}", depths) + assert dp.main(["--vcf", str(tmp_path / f"S{i}.vcf"), "--gff", str(gff), "--reference", "R", + "--window", "100", "--out-prefix", str(tmp_path / f"S{i}")]) == 0 + paths += [str(tmp_path / f"S{i}.{k}.tsv") for k in ("depth_windows", "zero_depth", "gene_depth")] + + profiles = cov.parse_depth_profiles(paths) + + assert sorted(profiles) == ["S0", "S1", "S2", "S3"] + assert profiles["S0"]["median_dp"] == 30 and profiles["S0"]["contigs"] == [("chr", 1000)] + assert len(profiles["S0"]["windows"]) == 10 and profiles["S0"]["zero"] == [("chr", 301, 700)] + section, _, regions = cov.build_coverage(profiles) + assert [(r["cls"], r["genes"], r["samples"]) for r in regions] == [("private", ["lost"], [["S0", 301, 700]])] + assert section["genes_lost"][0]["gene"] == "lost" + + out = tmp_path / "deletions.tsv" + cov.write_deletions(out, regions) + assert out.read_text().splitlines()[1].split("\t") == [ + "D1", "R", "chr", "301", "700", "400", "private", "1", "4", "S0:301-700", "lost"] From 287d8e268e6b058c248e8761546fa58ebed915f7 Mon Sep 17 00:00:00 2001 From: Paula Ruiz Rodriguez <50167687+Paururo@users.noreply.github.com> Date: Wed, 23 Sep 2026 14:26:59 +0200 Subject: [PATCH 02/14] Measure how close the samples are, and flag the ones far from their group A new step, SNP_DISTANCES, counts the SNPs between every two consensus sequences over the positions both called, so a no-call, a masked position or a mixed site is never a difference: the distances a tree built on the same sequences sees. It writes the square matrix and one row per pair with how many of the reference's variable positions each comparison rests on. Samples of different references are not compared. The report gains a Relatedness page: a heatmap in the order that keeps every single-linkage cluster together, the clusters at a threshold that moves live (--snp_cluster_threshold, 12 by default, the usual TB cut), and the samples further than the threshold from every other member of their samplesheet group, in groups that are otherwise tight. Those are flagged GROUP_MISMATCH (WARN). On the cohort that motivated it, with distances approximated from its SNP matrix, 20 samples are flagged, 18 of them the samples typed as another lineage than their line; a rule that also asked for a close sample of another group flagged none. --- CHANGELOG.md | 12 ++ assets/NO_FILE.README.md | 1 + assets/NO_FILE_DISTANCES | 0 bin/qc_report.py | 22 ++- bin/qcreport/relatedness.py | 131 ++++++++++++++++++ bin/qcreport/render.py | 19 ++- bin/report_assets/css/05_pages.css | 12 ++ bin/report_assets/js/03_state.js | 4 +- bin/report_assets/js/07b_summary.js | 20 ++- bin/report_assets/js/09c_relatedness.js | 112 ++++++++++++++++ bin/report_assets/js/14_boot.js | 8 +- bin/report_assets/shell.html | 27 ++-- bin/snp_distances.py | 163 +++++++++++++++++++++++ conf/organism.config | 7 + docs/configuration.md | 2 + docs/outputs.md | 18 +++ docs/qc-report.md | 11 +- modules/report.nf | 40 +++++- nextflow.config | 5 + subworkflows/cohort_report.nf | 14 +- tests/js/relatedness.test.mjs | 53 ++++++++ tests/unit/test_qc_report_relatedness.py | 130 ++++++++++++++++++ tests/unit/test_snp_distances.py | 102 ++++++++++++++ 23 files changed, 890 insertions(+), 23 deletions(-) create mode 100644 assets/NO_FILE_DISTANCES create mode 100644 bin/qcreport/relatedness.py create mode 100644 bin/report_assets/js/09c_relatedness.js create mode 100644 bin/snp_distances.py create mode 100644 tests/js/relatedness.test.mjs create mode 100644 tests/unit/test_qc_report_relatedness.py create mode 100644 tests/unit/test_snp_distances.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 67da6a8..29532fa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,18 @@ based on [Keep a Changelog](https://keepachangelog.com/), and the project follow ### Added +- **How close the samples are to each other.** A new step, `SNP_DISTANCES`, counts the SNPs + between every two consensus sequences over the positions both called, so a no-call, a masked + position or a mixed site is never a difference: the distances a tree built on the same + sequences sees. They are written as `_snp_distances.tsv` and, one row per pair + with how many positions each comparison rests on, `_pairs.tsv`. The report gains a + *Relatedness* page: a heatmap in the order that keeps every single-linkage cluster together, + the clusters at a threshold that can be moved live (`--snp_cluster_threshold`, 12 by default, + the usual *M. tuberculosis* cut), and the samples far from the rest of their samplesheet group. + Those are flagged `GROUP_MISMATCH` (WARN) when their group is otherwise tight: on the cohort + that motivated it, the 20 samples it finds are the ones typed as another lineage than their + line, which sat 47 to 72 SNPs from each other and 2,000 from their line mates. + - **Deletions, and SNPs per callable kb along the genome.** Each sample's all-positions VCF is reduced to its depth per window, the stretches no read covers and a per-gene table (`DEPTH_PROFILE`, published under `stats/`). The report compares those stretches across the diff --git a/assets/NO_FILE.README.md b/assets/NO_FILE.README.md index 72bfc82..38ca64a 100644 --- a/assets/NO_FILE.README.md +++ b/assets/NO_FILE.README.md @@ -11,6 +11,7 @@ stages one of these empty files instead, and the process recognises it by the | `NO_FILE_DR` | drug-resistance calls (`QC_REPORT`) | `--run_pathotypr false` | | `NO_FILE_H37RV` | canonically annotated VCFs (`QC_REPORT`) | `--annotate_canonical false` | | `NO_FILE_LIFTOVER` | variant coordinate map (`QC_REPORT`) | `--variant_liftover false` | +| `NO_FILE_DISTANCES` | pairwise SNP distances (`QC_REPORT`) | `--make_snp_distances false`, or no consensus sequences | Two properties matter, and both are load-bearing: diff --git a/assets/NO_FILE_DISTANCES b/assets/NO_FILE_DISTANCES new file mode 100644 index 0000000..e69de29 diff --git a/bin/qc_report.py b/bin/qc_report.py index 7dd5a1f..e45eeea 100644 --- a/bin/qc_report.py +++ b/bin/qc_report.py @@ -35,6 +35,7 @@ parse_gene_conversion, parse_gff, parse_kraken, parse_lineage_colors, parse_metadata, parse_pnps, parse_profile, parse_sample_meta, parse_summary, parse_vcfs, to_float) +from qcreport.relatedness import build_relatedness, group_mismatches, parse_pairs from qcreport.render import REPO_URL, SECTION_INFO, build_html @@ -98,6 +99,12 @@ def build_parser(): "in a thinly read sample they turn up by chance.") ap.add_argument("--out-deletions", default=None, help="Write the deletion regions (with --depth-profiles) as a TSV.") + ap.add_argument("--snp-distances", default=None, + help="Pairwise SNP distances between consensus sequences (snp_distances.py --pairs) -> " + "the relatedness page and the GROUP_MISMATCH flag (optional).") + ap.add_argument("--cluster-snps", type=int, default=12, + help="SNPs within which two samples are drawn as one cluster, and beyond which a " + "sample is far from its group. The report can move it live.") ap.add_argument("--gate", action="store_true") return ap @@ -249,6 +256,12 @@ def build_payload(args, thr, anc_thr): dates = parse_collection_dates(args.metadata) # What each sample's reads cover, compared across the cohort: the stretches no read covers that # other samples do read (deletions), and how much of each bin could be called at all. + # How close the samples are to each other, and which of them sit far from their own group (a + # patient, a line) while close to a sample of another one. + series_meta = parse_metadata(args.metadata) + groups = {s: md.get("group") for s, md in series_meta.items()} + distances = parse_pairs(args.snp_distances) + outside = group_mismatches(distances, groups, args.cluster_snps) profiles = parse_depth_profiles(args.depth_profiles) coverage, del_tracks = (build_coverage(profiles, min_len=args.deletion_min_len, min_depth=args.deletion_min_depth)[:2] @@ -261,6 +274,9 @@ def build_payload(args, thr, anc_thr): snp_prof = parse_profile(m.get("snp_profile")) verdict, flags = flag_sample(m, thr, snp_med, snp_sig, ancient=anc, anc_thr=anc_thr, dmg=dmg, ref_lineage=ref_major.get(ref_of.get(sid))) + if sid in outside: + flags.append("GROUP_MISMATCH") + verdict = "FAIL" if verdict == "FAIL" else "WARN" counts[verdict] += 1 jsamples.append({"s": sid, "v": verdict, "f": flags, "lineage": clean_str(m.get("lineage")), @@ -278,6 +294,9 @@ def build_payload(args, thr, anc_thr): "ref": ref_of.get(sid), "ref_lin": ref_major.get(ref_of.get(sid)), "miss": miss_by_sample.get(sid), + # what a GROUP_MISMATCH compares: the nearest sample of its own group and the + # nearest of another, with their distances + "grpd": outside.get(sid), "trk": ({k: v for k, v in (("snp", snp_prof), ("snpkb", snp_per_kb(snp_prof, profiles[sid]) if sid in profiles else None), @@ -301,7 +320,7 @@ def build_payload(args, thr, anc_thr): _variants = load_variants(args) _sample_meta = parse_sample_meta(args.metadata) # shared by the dynamics filter and the SNP matrix header - _dynamics = build_dynamics(parse_metadata(args.metadata), _variants, _sample_meta) # feeds dynamics + epistasis + _dynamics = build_dynamics(series_meta, _variants, _sample_meta) # feeds dynamics + epistasis # front-load 'dose' so it survives the correlation matrix's top-N view dist_keys = (["dose"] + DIST) if dose_map else DIST now = datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M UTC") @@ -325,6 +344,7 @@ def build_payload(args, thr, anc_thr): "epistasis": build_epistasis(_dynamics), "snp_matrix": build_snp_matrix(_variants, provenance.get('reference', '')), "coverage": coverage, + "relatedness": build_relatedness(distances, groups, args.cluster_snps), "sample_meta": _sample_meta, "metrics": [{"key": k, "label": l, "kind": kind, "dir": d} for k, l, kind, d in METRICS], "extra": extra_metrics, diff --git a/bin/qcreport/relatedness.py b/bin/qcreport/relatedness.py new file mode 100644 index 0000000..9aea0df --- /dev/null +++ b/bin/qcreport/relatedness.py @@ -0,0 +1,131 @@ +"""How close the samples are to each other, from the pairwise SNP distances (bin/snp_distances.py). + +The distances are between consensus sequences, over the positions both samples called, so a gap +never reads as a difference. Two uses here: + +- the relatedness page: every pair of samples on a reference, which the report clusters at a + threshold the reader can move; +- GROUP_MISMATCH: a sample the samplesheet puts in a group (a patient, a line, an outbreak) but + that sits further than the threshold from every other member of it, in a group whose members + are normally within it of each other. That is a swapped or mislabelled sample, a contaminated + or mixed one, or a reinfection, and each of them changes how the sample should be read. The + sample nearest to it, and its group, go with the flag: a swap usually sits next to a sample of + another group, a contaminant next to nobody. + +The check only runs in groups that are genetically tight to begin with: a column like 'hospital' +also matches the samplesheet's group names, and its members need not be related at all. It needs +three members, since with two there is no telling which one is out of place. +""" +from __future__ import annotations + +import os + + +def parse_pairs(path): + """{"ref": {sample: reference}, "variable": {reference: n}, "pairs": {(a, b): (snps, compared)}}.""" + out = {"ref": {}, "variable": {}, "pairs": {}} + if not path or not os.path.exists(path) or os.path.basename(path).startswith("NO_FILE"): + return out + try: + with open(path, encoding="utf-8", errors="replace") as fh: + header = fh.readline().rstrip("\n").split("\t") + col = {c: i for i, c in enumerate(header)} + need = ("sample_a", "sample_b", "reference", "snps", "compared", "variable_sites") + if not all(c in col for c in need): + return out + for line in fh: + c = line.rstrip("\n").split("\t") + if len(c) < len(header): + continue + try: + snps, comp, var = int(c[col["snps"]]), int(c[col["compared"]]), int(c[col["variable_sites"]]) + except ValueError: + continue + a, b, ref = c[col["sample_a"]], c[col["sample_b"]], c[col["reference"]] + out["ref"][a] = out["ref"][b] = ref + out["variable"][ref] = var + out["pairs"][(a, b)] = (snps, comp) + except OSError: + return out + return out + + +def _dist(pairs, a, b): + return pairs.get((a, b)) or pairs.get((b, a)) + + +def build_relatedness(dist, groups=None, threshold=12, min_compared=0.5): + """The payload section for the relatedness page, or None without distances. + + Per reference: its samples, and the upper triangle of the SNP matrix and of the share of the + reference's variable positions both samples called (percent), row by row. + """ + if not dist["pairs"]: + return None + by_ref = {} + for s, ref in dist["ref"].items(): + by_ref.setdefault(ref, []).append(s) + refs = {} + for ref, samples in sorted(by_ref.items()): + samples.sort() + var = dist["variable"].get(ref) or 0 + snps, cmp = [], [] + for i, a in enumerate(samples): + for b in samples[i + 1:]: + d = _dist(dist["pairs"], a, b) + snps.append(d[0] if d else None) + cmp.append(round(100.0 * d[1] / var) if d and var else None) + refs[ref] = {"samples": samples, "variable": var, "snps": snps, "cmp": cmp} + return {"threshold": threshold, "min_compared": min_compared, "refs": refs, + "group": {s: g for s, g in (groups or {}).items() if g and s in dist["ref"]}} + + +def group_mismatches(dist, groups, threshold=12, min_compared=0.5): + """{sample: detail} of the samples far from the rest of their own group. + + A pair counts only when both samples called at least `min_compared` of their reference's + variable positions: a thinly called sample is close to everyone. A group is judged only when + it has three or more members with a distance and its members' median distance to their + nearest groupmate is within `threshold`. A member of it is then flagged when its nearest + groupmate is further than `threshold`. The detail names that groupmate, and the nearest + sample of any other group with its group (None when there is none on the reference). + """ + pairs, var = dist["pairs"], dist["variable"] + groups = {s: g for s, g in (groups or {}).items() if g and s in dist["ref"]} + if not pairs or not groups: + return {} + + def ok(a, b): + d = _dist(pairs, a, b) + v = var.get(dist["ref"].get(a)) or 0 + return d if d and v and d[1] >= min_compared * v else None + + near_in, near_out = {}, {} + samples = sorted(dist["ref"]) + for a in samples: + if a not in groups: + continue + for b in samples: + if b == a or dist["ref"].get(b) != dist["ref"].get(a): + continue + d = ok(a, b) + if not d: + continue + same = groups.get(b) == groups[a] + best = (near_in if same else near_out).get(a) + if best is None or d[0] < best[0]: + (near_in if same else near_out)[a] = (d[0], b) + + by_group = {} + for s, (d, _) in near_in.items(): + by_group.setdefault(groups[s], []).append(d) + tight = {g for g, ds in by_group.items() if len(ds) >= 3 and sorted(ds)[len(ds) // 2] <= threshold} + + out = {} + for s, (d_in, t_in) in near_in.items(): + if groups[s] not in tight or d_in <= threshold: + continue + d_out, t_out = near_out.get(s, (None, None)) + out[s] = {"g": groups[s], "din": d_in, "nin": t_in, "dout": d_out, "nout": t_out, + "gout": groups.get(t_out, "") if t_out else ""} + return out diff --git a/bin/qcreport/render.py b/bin/qcreport/render.py index 29e5501..a40a8b4 100644 --- a/bin/qcreport/render.py +++ b/bin/qcreport/render.py @@ -47,7 +47,8 @@ def _asset(name): _JS_MODULES = [ "js/01_prelude.js", "js/02_qcspace.js", "js/03_state.js", "js/04_helpers.js", "js/05_insights_qc.js", "js/06_insights_genome.js", "js/07_render_core.js", "js/07b_summary.js", - "js/08_curation.js", "js/09_genome_genes.js", "js/09b_coverage.js", "js/10_dynamics.js", + "js/08_curation.js", "js/09_genome_genes.js", "js/09b_coverage.js", "js/09c_relatedness.js", + "js/10_dynamics.js", "js/11_epistasis.js", "js/12_snpmatrix.js", "js/13_drug_kraken.js", "js/14_boot.js", ] @@ -109,6 +110,22 @@ def build_html(title, payload): "in a stretch the sample has no reads for while other samples read it; 'SNPs / kb' divides " "the SNPs of a bin by the positions deep enough to call, so a half-read bin does not look " "half as variable.", + "reldist": "Pairwise SNP distances between the consensus sequences, counted over the positions both " + "samples called: a gap, a masked position or a mixed site is never a difference, as in a " + "tree built on the same sequences. Samples are ordered so that every single-linkage " + "cluster sits together at any threshold. A pair that both called under half of the " + "reference's variable positions is blank: a thinly called sample looks close to everyone.", + "relclus": "Samples joined by a chain of pairs each within the threshold (single linkage). The " + "threshold is a convention of the organism and the question: about 12 SNPs is the usual " + "cut for recent Mycobacterium tuberculosis transmission, other organisms need their own. " + "A cluster spanning several samplesheet groups points at transmission between them, or at " + "a sample in the wrong group.", + "relgroup": "Samples the samplesheet puts in a group (patient, line, series) that sit further than the " + "threshold from every other member of it. Only groups whose members are normally within " + "the threshold of each other, and with three or more of them, are checked. Flagged " + "GROUP_MISMATCH (WARN): a swap, a mislabel, a contaminated or mixed culture, or a " + "reinfection. The nearest sample of another group helps tell them apart: a swap sits " + "next to one, a contaminant next to nobody.", "deletions": "Stretches of the reference a sample has no reads for, compared with the other samples on " "the same reference. Lacked by one sample it is a private deletion; by several, a shared " "one (often a lineage's); by all, a repeat or a part of the reference these genomes do not " diff --git a/bin/report_assets/css/05_pages.css b/bin/report_assets/css/05_pages.css index 3b51a3c..dcc086c 100644 --- a/bin/report_assets/css/05_pages.css +++ b/bin/report_assets/css/05_pages.css @@ -162,3 +162,15 @@ table.drmut td{text-align:left} table.drmut td.num,table.drmut th.num{text-align .page-h h1{font-size:22px} h2{font-size:17px} } +/* relatedness: distance heatmap, its controls and the distance histogram */ +.rel-ctl{display:flex;gap:12px;align-items:center;flex-wrap:wrap} +.rel-thr{display:inline-flex;gap:6px;align-items:center;font-size:12.5px;color:var(--txt2)} +.rel-thr input[type=number]{width:72px;padding:4px 6px;border:1px solid var(--line);border-radius:7px;background:var(--panel);color:var(--txt)} +.rel-wrap{overflow:auto;max-height:78vh;margin:6px 0} +.rel-wrap canvas{display:block;cursor:crosshair} +.rel-hist{display:flex;gap:6px;align-items:flex-end;margin:14px 0 4px;min-height:96px} +.rel-bar{display:flex;flex-direction:column;align-items:center;flex:1;min-width:34px} +.rel-bar i{display:block;width:70%;border-radius:3px 3px 0 0} +.rel-bv{font-size:10px;color:var(--txt2)} +.rel-bl{font-size:10px;color:var(--txt2);margin-top:3px;white-space:nowrap} +.rel-chip{display:inline-block;padding:1px 6px;margin:1px 0;border-radius:5px;background:var(--soft);font-size:11.5px} diff --git a/bin/report_assets/js/03_state.js b/bin/report_assets/js/03_state.js index 4f401c2..2260e3b 100644 --- a/bin/report_assets/js/03_state.js +++ b/bin/report_assets/js/03_state.js @@ -75,7 +75,7 @@ function pipelineFlags(s){return (s.f0||[]).filter(function(x){return !JS_FLAGS[ var FLAGLAB={LOW_DEPTH:'Low depth',LOW_BREADTH:'Low breadth',HIGH_MISSING:'Incomplete consensus',NO_DATA:'No output', MAPPING_LOW:'Few reads map',HIGH_DUP:'High duplication',HIGH_IUPAC:'Many ambiguous bases',TITV_LOW:'Low Ti/Tv', SNP_LOW:'Unusually few SNPs',SNP_HIGH:'Unusually many SNPs',HET_HIGH:'Mixed alleles',MIXED:'Mixed lineages', - LINEAGE_MISMATCH:'Wrong reference?',DAMAGE_LOW:'No aDNA damage',QC_OUTLIER:'Unusual QC profile', + LINEAGE_MISMATCH:'Wrong reference?',GROUP_MISMATCH:'Outside its group?',DAMAGE_LOW:'No aDNA damage',QC_OUTLIER:'Unusual QC profile', HIGH_IMPACT_EXCESS:'Excess high-impact variants',LOF_EXCESS:'Excess loss-of-function', PNPS_PROXY_HIGH:'High missense/silent',ANNOTATION_POOR:'Poor annotation'}; function flagLab(f){return FLAGLAB[f]||String(f).toLowerCase().replace(/_/g,' ');} @@ -127,6 +127,7 @@ function flagWhy(s,fl){var T=actv(s),M=MET;function u(k){return (M[k]&&M[k].kind case 'PNPS_PROXY_HIGH':return 'missense/silent ratio '+shortv(s.m.missense_silent,'float')+' above cohort (robust z > '+thr.snp_z+'); a spectrum proxy only, possible base-call error or contamination; not dN/dS, see the pN/pS panel'; case 'ANNOTATION_POOR':return 'annotated '+shortv(s.m.annotated_pct,'pct')+'% below floor '+ANNOT_FLOOR+' or below cohort; the GFF-to-snpEff database may be mismatched for this reference'; case 'LINEAGE_MISMATCH':return 'types as '+linMain(s.lineage)+(s.ref_lin?(' while the other samples on '+(s.ref?'reference '+s.ref:'its reference')+' type as '+linMain(s.ref_lin)):' unlike the other samples on its reference')+': probably mapped to a genome it does not belong to, so its SNPs measure that distance'; + case 'GROUP_MISMATCH':return s.grpd?(s.grpd.din+' SNPs from the nearest sample of its group '+s.grpd.g+' ('+s.grpd.nin+')'+(s.grpd.nout?', '+s.grpd.dout+' from '+s.grpd.nout+(s.grpd.gout?' of group '+s.grpd.gout:''):'')+': swapped, mislabelled, contaminated or reinfected'):'far from the rest of its group'; default:return flagLab(fl);}} // The top level of a lineage call ('A4;M_bovis;A4.7;A4.7.1' -> 'A4'), which is what a mismatch compares. function linMain(l){return l?String(l).split(';')[0]:'NA';} @@ -150,6 +151,7 @@ function flagReason(s,fl){var T=actv(s),m=s.m;function v(k){return nv(m[k]);} case 'HET_HIGH':return [flagLab(fl),v('het_frac')+'% of variant calls heterozygous, at most '+thr.het_max_frac+'%']; case 'MIXED':return [flagLab(fl),m.n_lineages+' lineages above '+thr.mixed_min_frac+'% of the markers']; case 'LINEAGE_MISMATCH':return [flagLab(fl),'types as '+esc(linMain(s.lineage))+(s.ref_lin?'; the rest of '+esc(s.ref||'its reference')+' types as '+esc(linMain(s.ref_lin)):'')]; + case 'GROUP_MISMATCH':return [flagLab(fl),s.grpd?(s.grpd.din.toLocaleString('en-US')+' SNPs from the rest of its group '+esc(s.grpd.g)+(s.grpd.nout?'; '+s.grpd.dout.toLocaleString('en-US')+' from '+esc(s.grpd.nout)+(s.grpd.gout?' ('+esc(s.grpd.gout)+')':''):'')):'far from the rest of its group']; case 'QC_OUTLIER':return [flagLab(fl),(s._mdrv&&s._mdrv.length?s._mdrv.slice(0,3).map(function(d){return esc(d.label)+' '+(d.z>=0?'+':'−')+Math.abs(d.z).toFixed(1)+'σ';}).join(', '):'d² '+v('qc_mahal'))+' against the cohort']; case 'HIGH_IMPACT_EXCESS':return [flagLab(fl),shortv(m.ann_high,'int')+' HIGH-impact variants; the cohort median is '+(ANNHI?shortv(ANNHI.med,'int'):'NA')]; case 'LOF_EXCESS':return [flagLab(fl),v('lof_pct')+'% of coding changes; the cohort median is '+(LOFHI?shortv(LOFHI.med,'float'):'NA')+'%']; diff --git a/bin/report_assets/js/07b_summary.js b/bin/report_assets/js/07b_summary.js index 2d01e2d..9cc3909 100644 --- a/bin/report_assets/js/07b_summary.js +++ b/bin/report_assets/js/07b_summary.js @@ -189,6 +189,23 @@ function findGconv(){ return {k:'gconv',eyebrow:'Gene conversion',tone:'',head:head,body:body.join(' '),link:{href:'#gconv',t:'Inspect the tracts'}, note:'Candidates to check in the reads, not confirmed events.'}; } +// Relatedness: the clusters at the pipeline's threshold, and the samples outside their own group. +function relSummary(){var S=R.relatedness;if(!(S&&S.refs))return null; + var thr=S.threshold,minC=100*(S.min_compared!=null?S.min_compared:0.5),nCl=0,big=0,inCl=0; + Object.keys(S.refs).forEach(function(k){relClusters(S.refs[k],thr,minC).forEach(function(c){if(c.length>1){nCl++;inCl+=c.length;if(c.length>big)big=c.length;}});}); + return {thr:thr,nCl:nCl,big:big,inCl:inCl,out:R.samples.filter(function(s){return s.grpd;})};} +function findRelatedness(){ + var g=relSummary(); if(!g)return null; + var head=''+g.nCl+' '+_plural(g.nCl,'cluster')+' of samples within '+g.thr+' SNPs of each other'; + var body=[]; + if(g.nCl)body.push(g.inCl+' samples sit in one, the largest holding '+g.big+'.'); + if(g.out.length)body.push(''+g.out.length+' '+_plural(g.out.length,'sample sits','samples sit')+' far from the rest of '+_plural(g.out.length,'its group','their groups')+' ('+ + _list(g.out.map(function(s){return esc(s.s);}),4)+'): swapped, mislabelled, contaminated or reinfected.'); + return {k:'rel',eyebrow:'Relatedness',tone:g.out.length?'warn':'',head:head,body:body.join(' '), + link:{href:'#p-related',t:'See the distances'}, + note:'SNPs between consensus sequences; the threshold is a convention of the organism.', + short:g.out.length?(g.out.length+' '+_plural(g.out.length,'sample sits','samples sit')+' outside '+_plural(g.out.length,'its group','their groups')+'.'):''}; +} function findCoverage(){ var S=R.samples; if(!S.length)return null; function med(k){return _median(S.map(function(s){return s.m[k];}));} @@ -198,7 +215,7 @@ function findCoverage(){ var body=(rd?'Depth ranges from '+fmt(rd[0],'float')+'× to '+fmt(rd[1],'float')+'×. ':'')+(c!=null?'The median consensus has '+c.toFixed(1)+'% of the genome as confident bases.':''); return {k:'cov',eyebrow:'Coverage',tone:'',head:head,body:body,link:{href:'#dist',t:'See the distributions'}}; } -function findings(){return [findQC(),findIdentity(),findResistance(),findDynamics(),findLineage(),findGconv(),findCoverage()].filter(function(f){return f;});} +function findings(){return [findQC(),findIdentity(),findRelatedness(),findResistance(),findDynamics(),findLineage(),findGconv(),findCoverage()].filter(function(f){return f;});} function findingCard(f){ var nums=(f.nums&&f.nums.length)?'
'+f.nums.map(function(x){return '
'+x.n+''+esc(x.l)+'
';}).join('')+'
':''; return '
'+esc(f.eyebrow)+'
'+ @@ -223,5 +240,6 @@ function renderNavBadges(){ set('bdg-qc',c.FAIL?String(c.FAIL):(c.WARN?String(c.WARN):''),c.FAIL?'bad':'warn',c.FAIL?c.FAIL+' samples to exclude':(c.WARN?c.WARN+' samples to review':'')); var d=drSummary(); set('bdg-drug',d&&d.nCarriers?String(d.nCarriers):'','bad',d&&d.nCarriers?d.nCarriers+' samples with resistance mutations beyond their lineage':''); var g=gconvSummary(); set('bdg-gconv',g&&g.nPassEvents?String(g.nPassEvents):'','neu',g?g.nPassEvents+' gene-conversion events in samples the QC does not fail':''); + var rl=relSummary(); set('bdg-related',rl&&rl.out.length?String(rl.out.length):'','warn',rl&&rl.out.length?rl.out.length+' samples far from the rest of their group':''); var y=dynSummary(); set('bdg-variants',y&&y.sweep.length?String(y.sweep.length):'','neu',y?y.sweep.length+' alleles sweeping toward fixation':''); } diff --git a/bin/report_assets/js/09c_relatedness.js b/bin/report_assets/js/09c_relatedness.js new file mode 100644 index 0000000..286e7c2 --- /dev/null +++ b/bin/report_assets/js/09c_relatedness.js @@ -0,0 +1,112 @@ +// ---- Relatedness: SNP distances between the consensus sequences, clustered at a threshold ---- +// The distances count only positions both samples called, so a gap is never a difference. A pair +// that compared too few of its reference's variable positions says little either way: it neither +// links a cluster nor orders the heatmap. +var REL={ref:null,thr:null}; +var RELPAL=['#4477aa','#ee6677','#228833','#ccbb44','#66ccee','#aa3377','#e69f00','#0072b2','#d55e00','#009e73','#cc79a7','#882255']; +function relSec(){return R.relatedness||null;} +function relRef(){var S=relSec();if(!S)return null;var ks=Object.keys(S.refs||{});if(!ks.length)return null; + if(!REL.ref||!S.refs[REL.ref])REL.ref=ks.slice().sort(function(a,b){return S.refs[b].samples.length-S.refs[a].samples.length;})[0]; + return S.refs[REL.ref];} +function relThr(){var S=relSec();if(REL.thr==null)REL.thr=(S&&S.threshold!=null)?S.threshold:12;return REL.thr;} +function relMinCmp(){var S=relSec();return 100*((S&&S.min_compared!=null)?S.min_compared:0.5);} +function relIdx(n,i,j){if(i>j){var t=i;i=j;j=t;}return i*n-i*(i+1)/2+(j-i-1);} // upper triangle, row by row +function relD(r,i,j){return i==j?0:r.snps[relIdx(r.samples.length,i,j)];} +function relC(r,i,j){return i==j?100:r.cmp[relIdx(r.samples.length,i,j)];} +function relOk(r,i,j,minCmp){var c=relC(r,i,j);return c!=null&&c>=minCmp;} +// Single-linkage clusters at the threshold (union-find), largest first. +function relClusters(r,thr,minCmp){var n=r.samples.length,p=[],i,j;for(i=0;i0?d/thr:0;return 'rgb('+Math.round(12+t*110)+','+Math.round(110+t*85)+','+Math.round(132+t*70)+')';} + var lt=Math.log(d-thr+1)/Math.log(Math.max(2,max-thr+1)),g=isDark()?Math.round(60+lt*110):Math.round(232-lt*140); + return 'rgb('+g+','+g+','+Math.min(255,g+8)+')';} +function relHistogram(r,thr,minCmp){ + var edges=[0,1,2,3,6,13,26,51,101,501,1001,Infinity],labels=['0','1','2','3-5','6-12','13-25','26-50','51-100','101-500','501-1,000','> 1,000']; + var cnt=zeros(labels.length),n=r.samples.length,tot=0,i,j,k; + for(i=0;i=edges[k]&&d'+labels.map(function(l,k){var within=edges[k+1]-1<=thr; + return '
'+(cnt[k]||'')+''+ + ''+l+'
';}).join('')+ + '
'+tot.toLocaleString('en-US')+' pairs, in SNPs; teal = within the threshold of '+thr+'.
';} +function renderRelatedness(){ + var host=el('rel_body'),S=relSec(); if(!host)return; + var r=relRef(); if(!(S&&r)){host.innerHTML='';return;} + var thr=relThr(),minCmp=relMinCmp(),n=r.samples.length,refs=Object.keys(S.refs); + var order=relOrder(r,minCmp),gcol=relGroupColor(),byName={};R.samples.forEach(function(s){byName[s.s]=s;}); + var cl=relClusters(r,thr,minCmp),multi=cl.filter(function(c){return c.length>1;}),alone=cl.filter(function(c){return c.length==1;}).length; + var max=0,i,j;for(i=0;imax)max=dd;} + var refseg=refs.length>1?(''+refs.map(function(k){return '';}).join('')+''):''; + host.innerHTML='
'+refseg+ + '
'+ + '
'+multi.length+' '+_plural(multi.length,'cluster')+' of two or more samples within '+thr+' SNPs'+ + (multi.length?(', the largest of '+multi[0].length):'')+'; '+alone+' '+_plural(alone,'sample stands','samples stand')+' alone. '+n+' samples on '+esc(REL.ref)+ + ', '+r.variable.toLocaleString('en-US')+' variable positions. Pairs that both called under '+minCmp+'% of them are left out of the clusters and shown blank.
'+ + '
'+ + '
identical'+thr+' SNPs'+ + ''+max.toLocaleString('en-US')+' SNPstoo few positions compared'+ + (Object.keys(S.group||{}).length?'left strip = group':'')+'samples in single-linkage order; hover a cell for the pair
'+ + relHistogram(r,thr,minCmp); + // A square that fits the screen both ways, so the last rows are never scrolled out of sight. + var cv=el('relcanvas'),W=Math.max(240,Math.min((host.clientWidth||900)-24,980)),H=Math.max(240,Math.min(760,Math.round((window.innerHeight||900)*0.72))), + strip=10,cs=Math.max(1,Math.floor(Math.min(W-strip-4,H)/Math.max(1,n))),side=cs*n; + var dpr=window.devicePixelRatio||1; cv.width=(side+strip+4)*dpr; cv.height=side*dpr; cv.style.width=(side+strip+4)+'px'; cv.style.height=side+'px'; + var ctx=cv.getContext&&cv.getContext('2d'); + if(ctx){ctx.scale(dpr,dpr); + for(var a=0;a=n||b>=n){tip('');return;} + var ia=order[a],ib=order[b],sa=r.samples[ia],sb=r.samples[ib],d=relD(r,ia,ib),c=relC(r,ia,ib),g=S.group||{}; + tip(''+esc(sa)+''+(g[sa]?' ('+esc(g[sa])+')':'')+' × '+esc(sb)+''+(g[sb]?' ('+esc(g[sb])+')':'')+'
'+ + (d==null?'no distance':(d.toLocaleString('en-US')+' SNPs'))+(ia==ib?'':' over '+(c==null?'NA':c+'%')+' of the variable positions'),e.clientX,e.clientY);}; + cv.onmouseleave=function(){tip('');}; + cv.onclick=function(e){var rc=cv.getBoundingClientRect(),a=Math.floor((e.clientY-rc.top)/cs);if(a>=0&&a1;}),g=S.group||{},gcol=relGroupColor(),hasG=Object.keys(g).length>0; + var rows=cl.map(function(c,k){var mx=0,a,b;for(a=0;amx)mx=d;} + var names=c.map(function(i){return r.samples[i];}),gs={};names.forEach(function(s){if(g[s])gs[g[s]]=1;});var ng=Object.keys(gs).length; + return ''+(k+1)+''+c.length+''+names.map(function(s){var col=gcol(s); + return ''+esc(s)+'';}).join(' ')+''+ + ''+mx.toLocaleString('en-US')+''+(hasG?'1?' style="color:var(--warn);font-weight:600" title="the cluster joins samples the samplesheet puts in different groups"':'')+'>'+ng+'':'')+'';}).join(''); + host.innerHTML='
Samples joined by a chain of pairs each within '+thr+' SNPs (single linkage). '+ + (hasG?'A cluster spanning several groups is where to look for transmission between them, or for a sample in the wrong group.':'Add a group column to the samplesheet (patient, line, outbreak) to see which clusters cross groups.')+'
'+ + '
'+(hasG?'':'')+''+ + (rows||'')+'
#SamplesMembersLargest distanceGroups
no two samples are within '+thr+' SNPs.
'; + var gh=el('relgroup_body'); + if(gh){var out=R.samples.filter(function(s){return s.grpd;}); + gh.innerHTML=out.length?('
'+out.length+' '+_plural(out.length,'sample sits','samples sit')+' further than '+(S.threshold)+' SNPs from every other member of '+_plural(out.length,'its group','their groups')+', in groups whose members are otherwise within that of each other. Each is flagged '+esc(flagLab('GROUP_MISMATCH'))+': a swapped or mislabelled sample, a contaminated or mixed one, or a reinfection. The nearest sample of another group helps tell them apart: a swap sits next to one, a contaminant next to nobody.
'+ + '
'+ + out.map(function(s){var d=s.grpd;return ''+ + '';}).join('')+ + '
SampleIts groupNearest of its groupNearest of another
'+esc(s.s)+''+esc(d.g)+''+esc(d.nin)+' · '+d.din.toLocaleString('en-US')+' SNPs'+(d.nout?(esc(d.nout)+(d.gout?' ('+esc(d.gout)+')':'')+' · '+d.dout.toLocaleString('en-US')+' SNPs'):'none on its reference')+'
'):'
'+(Object.keys(S.group||{}).length?'Every sample sits within '+S.threshold+' SNPs of another member of its group, where its group is tight enough to say.':'The samplesheet has no group column (patient, line, series...), so there is nothing to check a sample against.')+'
';} +} diff --git a/bin/report_assets/js/14_boot.js b/bin/report_assets/js/14_boot.js index 1d075ab..ab7ecb0 100644 --- a/bin/report_assets/js/14_boot.js +++ b/bin/report_assets/js/14_boot.js @@ -1,11 +1,12 @@ -// ---- pages: the report reads as seven pages, and the sidebar is both the page list and the contents of +// ---- pages: the report reads as eight pages, and the sidebar is both the page list and the contents of // the page on screen. Only that page is redrawn on a change (a new threshold, a highlighted sample, the // theme); the others are marked stale and redraw when opened, which is also when they know their width. -var PAGES=['summary','qc','genome','variants','drug','gconv','diag']; +var PAGES=['summary','qc','genome','related','variants','drug','gconv','diag']; var PAGE_RENDER={ summary:function(){renderExec();}, qc:function(){renderOverview();renderFlags();renderCuration();renderTable();renderLineages();renderKraken();renderPlots();renderADNA();}, genome:function(){renderStacks();renderGenome();renderDeletions();renderFunction();renderGeneBurden();renderHotspots();renderPnps();}, + related:function(){renderRelatedness();}, variants:function(){renderDynamics();renderEpistasis();renderSnpMatrix();renderVarDose();}, drug:function(){renderDrug();}, gconv:function(){renderGconv();}, @@ -206,6 +207,7 @@ function genomeResetZoom(){st.gsel=null;st.gzoom=null;st.geneMark=null;var gg=el var rd=el('gselreadout');if(rd)rd.innerHTML='gene '+esc(g.name)+' · '+fmtpos(g.start)+' - '+fmtpos(g.end)+' '; var cb=el('gselclear');if(cb)cb.onclick=function(ev){ev.stopPropagation();genomeResetZoom();};};})(); if(!R.samples.some(function(s){return s.miss||s.trk;})){var gs=el('genome');if(gs)gs.style.display='none';var ng=el('nav-genome');if(ng)ng.style.display='none';} +if(!relSec()){['reldist','relclus','relgroup','nav-reldist','nav-relclus','nav-relgroup'].forEach(function(id){var x=el(id);if(x)x.style.display='none';});} if(!(R.coverage&&R.coverage.regions)){var dlx=el('deletions');if(dlx)dlx.style.display='none';var ndlx=el('nav-del');if(ndlx)ndlx.style.display='none';} if(!R.samples.some(function(s){return s.trk&&s.trk.snp;})){var hsx=el('hotspots');if(hsx)hsx.style.display='none';var nhx=el('nav-hot');if(nhx)nhx.style.display='none';} // signature layer gates: hide a panel when its data type is absent cohort-wide (organism-agnostic; render fns also degrade per-view) @@ -234,7 +236,7 @@ if(!(R.kraken&&R.kraken.samples&&R.kraken.samples.length)){var kkx=el('kraken'); if(!R.mask_bins){mb.style.display='none';return;} mb.title=(R.mask_pct||0)+'% of the reference masked (PE/PPE, IS, DR, repeats); toggle to exclude these zones'; mb.onclick=function(){st.maskOn=!st.maskOn;mb.classList.toggle('on',st.maskOn);renderGenome();renderHotspots();};})(); -var rz;window.addEventListener('resize',function(){clearTimeout(rz);rz=setTimeout(function(){renderPlots();renderScatter();renderCorr();renderQCspace();renderRefBias();renderGenome();renderTemporal();renderGconv();},120);}); +var rz;window.addEventListener('resize',function(){clearTimeout(rz);rz=setTimeout(function(){renderPlots();renderScatter();renderCorr();renderQCspace();renderRefBias();renderGenome();renderTemporal();renderGconv();if(curPage=='related')renderRelatedness();},120);}); // metric help panel el('helpmenu').innerHTML=R.metrics.map(function(m){var d=R.defs[m.key]||['',''];return '
'+esc(m.label)+' '+esc(m.key)+'
'+esc(d[0]||'')+(d[1]?' ('+esc(d[1])+')':'')+'
';}).join(''); // ---- colour-by toggle (beeswarm + scatter); hides the whole lineage UI when there is no lineage data ---- diff --git a/bin/report_assets/shell.html b/bin/report_assets/shell.html index 5c41f70..4611311 100644 --- a/bin/report_assets/shell.html +++ b/bin/report_assets/shell.html @@ -9,10 +9,11 @@
1Summary
2Sample QC
Flagged samplesAll samplesLineagesContaminationDistributionsaDNA damage
3Genome & genes
ConsensusGenome landscapeDeletionsFunctional impactGene burdenVariable genesSelection (dN/dS)
-
4Variants over time
SNP dynamicsCo-varying pairsSNP matrixVariant × dose
-
5Resistance
Resistance mutations
-
6Gene conversion
Candidate tracts
-
7Diagnostics
Metric pairsCorrelation matrixQC spaceDivergenceDose × treatmentSampling dates
+
4Relatedness
DistancesClustersOutside their group
+
5Variants over time
SNP dynamicsCo-varying pairsSNP matrixVariant × dose
+
6Resistance
Resistance mutations
+
7Gene conversion
Candidate tracts
+
8Diagnostics
Metric pairsCorrelation matrixQC spaceDivergenceDose × treatmentSampling dates
@@ -101,8 +102,18 @@
+
+
4 · Relatedness

How close the samples are to each other

SNP differences between the consensus sequences, counted only where both samples called a base, so a gap or a masked position is never a difference. These are the distances a tree built on the consensus sequences sees. Samples mapped against different references are not compared.

+

Distances every pair of samples on a reference, in the order that keeps each cluster together; move the threshold to redraw the clusters

+
+

Clusters samples joined by a chain of pairs within the threshold

+
+

Outside their group samples far from the rest of their own patient, line or series

+
+
+
-
4 · Variants over time

How allele frequencies change along each series

Every variant followed through the time points of its series (a patient, a passage line), the pairs that move together, and the full site × sample matrix.

+
5 · Variants over time

How allele frequencies change along each series

Every variant followed through the time points of its series (a patient, a passage line), the pairs that move together, and the full site × sample matrix.

SNP dynamics allele-frequency trajectories per gene; the screen at the top ranks what is changing

Co-varying pairs variants whose trajectories move together (concordant) or against each other (discordant) within a series

@@ -116,13 +127,13 @@
-
5 · Resistance

Resistance-associated mutations

Mutations in the WHO catalogue called by pathotypr straight from the reads (H37Rv numbering). Those every sample of a lineage shares are separated from those only some samples carry. A genomic screen, not a drug-susceptibility result.

+
6 · Resistance

Resistance-associated mutations

Mutations in the WHO catalogue called by pathotypr straight from the reads (H37Rv numbering). Those every sample of a lineage shares are separated from those only some samples carry. A genomic screen, not a drug-susceptibility result.

Resistance mutations grade 1–2 = associated with resistance, 3 = uncertain, 4–5 = not associated

-
6 · Gene conversion

Candidate gene conversion between paralogs

Stretches where one copy of a gene family seems to have been overwritten by another, and the evidence that decides whether to believe each one: an ordinary substitution matching the paralog and reads that came from the donor produce the same bases, and the Bayes factor (BF) says how much better a conversion explains the reads.

+
7 · Gene conversion

Candidate gene conversion between paralogs

Stretches where one copy of a gene family seems to have been overwritten by another, and the evidence that decides whether to believe each one: an ordinary substitution matching the paralog and reads that came from the donor produce the same bases, and the Bayes factor (BF) says how much better a conversion explains the reads.

Candidate tracts one row per event; filter by verdict, search a sample, locus or donor

@@ -130,7 +141,7 @@
-
7 · Diagnostics

How the QC metrics relate

Views for digging into a problem once you have seen it elsewhere. None of them flags a sample on its own.

+
8 · Diagnostics

How the QC metrics relate

Views for digging into a problem once you have seen it elsewhere. None of them flags a sample on its own.

Metric pairs any two metrics against each other; drag a box to add the enclosed samples to the exclusion list vs diff --git a/bin/snp_distances.py b/bin/snp_distances.py new file mode 100644 index 0000000..cc554dd --- /dev/null +++ b/bin/snp_distances.py @@ -0,0 +1,163 @@ +#!/usr/bin/env python3 +"""Pairwise SNP distances between consensus sequences, as a phylogeny built on them would see. + +Two samples differ at a position when both called a base there (A, C, G or T) and the bases +differ. A position either one did not call, masked, or wrote as an ambiguity code (a mixed site) +says nothing about the pair and is left out of that comparison, so the distance never counts a +gap as a difference. The consensus already carries every masking decision of the pipeline, which +is why the distances are taken from it rather than from the VCFs. + +Only samples mapped against the same reference are compared: their consensus sequences share +the reference's coordinates, and sequences of different references do not. + +Two outputs: + -o square matrix, samples x samples, NA between samples of different references + --pairs one row per pair on a reference: the SNPs between them and how many of that reference's + variable positions both called, which says how much the distance rests on + + snp_distances.py --manifest consensus.tsv --dir cons -o snp_distances.tsv --pairs pairs.tsv + +The manifest is 'samplereferencefasta' (file name, looked up under --dir). Without one, +--consensus takes the files directly, names each sample after its first record (or the file name +up to the first dot) and groups them by sequence length. +""" +from __future__ import annotations + +import argparse +import gzip +import os +import sys + +import numpy as np + +ACGT = np.frombuffer(b"ACGT", dtype=np.uint8) + + +def read_fasta(path): + """(record names, the records' sequences concatenated as upper-case uint8).""" + op = gzip.open if str(path).endswith(".gz") else open + names, chunks = [], [] + with op(path, "rt", encoding="utf-8", errors="replace") as fh: + for line in fh: + if line.startswith(">"): + names.append(line[1:].split()[0] if line[1:].split() else "") + continue + chunks.append(line.strip()) + seq = "".join(chunks).upper().encode("ascii", "replace") + return names, np.frombuffer(seq, dtype=np.uint8) + + +def entries(args): + """[(sample, reference, path)] from --manifest or --consensus.""" + out = [] + if args.manifest: + with open(args.manifest, encoding="utf-8") as fh: + for line in fh: + c = line.rstrip("\n").split("\t") + if len(c) >= 3 and c[0] and not line.startswith("#"): + out.append((c[0], c[1], os.path.join(args.dir, c[2]) if args.dir else c[2])) + return out + for path in args.consensus or []: + names, seq = read_fasta(path) + # The pipeline names a one-record consensus after the sample and prefixes each record of + # a multi-contig one with it, so only a single record's name is the sample's. + sample = names[0] if len(names) == 1 and names[0] else os.path.basename(path).split(".")[0] + out.append((sample, f"L{len(seq)}", path)) + return out + + +def variable_positions(paths): + """(indices of the positions where two samples called different bases, length) over `paths`. + + A running 'first base called here' is kept instead of the reference, which this script is + never given: a later sample calling another base at a position marks it variable. + """ + first, variable, length = None, None, None + for path in paths: + _, seq = read_fasta(path) + if first is None: + length = len(seq) + first = np.zeros(length, dtype=np.uint8) + variable = np.zeros(length, dtype=bool) + if len(seq) != length: + raise ValueError(f"{path}: {len(seq)} bp where the other consensus sequences of its " + f"reference have {length}") + called = np.isin(seq, ACGT) + variable |= called & (first != 0) & (seq != first) + new = called & (first == 0) + first[new] = seq[new] + return (np.flatnonzero(variable) if variable is not None else np.zeros(0, dtype=np.int64)), length + + +def codes_at(paths, positions): + """samples x positions uint8: the base called (A, C, G or T as ASCII) or 0 where none was.""" + out = np.zeros((len(paths), len(positions)), dtype=np.uint8) + for i, path in enumerate(paths): + _, seq = read_fasta(path) + c = seq[positions] + out[i] = np.where(np.isin(c, ACGT), c, 0) + return out + + +def distances(codes): + """(snps, compared): samples x samples matrices. `compared` counts the positions both called.""" + n = codes.shape[0] + snps = np.zeros((n, n), dtype=np.int64) + compared = np.zeros((n, n), dtype=np.int64) + called = codes != 0 + for i in range(n): + both = called[i] & called[i:] + diff = both & (codes[i:] != codes[i]) + snps[i, i:] = snps[i:, i] = diff.sum(axis=1) + compared[i, i:] = compared[i:, i] = both.sum(axis=1) + return snps, compared + + +def main(argv=None): + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("--manifest", help="samplereferencefasta") + ap.add_argument("--dir", default="", help="folder the manifest's fasta names are under") + ap.add_argument("--consensus", nargs="*", default=[], help="consensus FASTAs, without a manifest") + ap.add_argument("-o", "--output", required=True, help="square matrix TSV") + ap.add_argument("--pairs", help="one row per pair TSV") + a = ap.parse_args(argv) + + items = entries(a) + groups = {} + for sample, ref, path in items: + if not os.path.exists(path): + sys.stderr.write(f"[snp_distances] WARN {sample}: {path} not found\n") + continue + groups.setdefault(ref, []).append((sample, path)) + samples = [s for ref in sorted(groups) for s, _ in groups[ref]] + index = {s: i for i, s in enumerate(samples)} + full = [["NA"] * len(samples) for _ in samples] + rows = [] + for ref in sorted(groups): + members = groups[ref] + paths = [p for _, p in members] + pos, length = variable_positions(paths) + codes = codes_at(paths, pos) + snps, compared = distances(codes) + for i, (si, _) in enumerate(members): + for j, (sj, _) in enumerate(members): + full[index[si]][index[sj]] = str(int(snps[i, j])) + if j > i: + rows.append((si, sj, ref, int(snps[i, j]), int(compared[i, j]), len(pos))) + sys.stderr.write(f"[snp_distances] {ref}: {len(members)} sample(s), {length} bp, " + f"{len(pos)} variable position(s)\n") + + with open(a.output, "w", encoding="utf-8") as fh: + fh.write("sample\t" + "\t".join(samples) + "\n") + for s in samples: + fh.write(s + "\t" + "\t".join(full[index[s]]) + "\n") + if a.pairs: + with open(a.pairs, "w", encoding="utf-8") as fh: + fh.write("sample_a\tsample_b\treference\tsnps\tcompared\tvariable_sites\n") + for r in rows: + fh.write("\t".join(map(str, r)) + "\n") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/conf/organism.config b/conf/organism.config index 851574e..863e438 100644 --- a/conf/organism.config +++ b/conf/organism.config @@ -65,4 +65,11 @@ params { // report_breadth_min = 90 // min genome breadth (%) // report_missing_max = 10 // max missing (%) // report_mapping_min = 80 // min mapped reads (%) + + // --- Relatedness --------------------------------------------------------- + // SNPs within which the report draws two samples into one cluster, and beyond which a + // sample is far from the rest of its samplesheet group. 12 is the usual M. tuberculosis + // transmission cut; a faster-evolving organism, or a question about long-term spread, + // needs its own. Also movable live in the report. + // snp_cluster_threshold = 12 } diff --git a/docs/configuration.md b/docs/configuration.md index 890b73c..42c8b94 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -169,6 +169,8 @@ Two documented, ready-to-use **organism** configs live in | | `--make_snp_matrix` | `true` | Also emit the master SNP-matrix TSV. | | | `--depth_window` | `1000` | Window (bp) of each sample's depth profile, behind the deletions and SNPs per callable kb. | | | `--deletion_min_len` | `200` | Shortest stretch without reads reported as a deletion. | +| | `--make_snp_distances` | `true` | Pairwise SNP distances between the consensus sequences, and the report's relatedness page. | +| | `--snp_cluster_threshold` | `12` | SNPs within which the report starts drawing clusters (movable there), and beyond which a sample is far from its group. 12 is the usual *M. tuberculosis* transmission cut; set your organism's own. | | | `--report_gate` | `false` | Fail the run if any sample is flagged **FAIL**. | | | `--report_depth_min` | `10` | Gate: min mean depth (all `report_*` cut-offs are editable live in the report). | | | `--report_breadth_min` | `90` | Gate: min breadth %. | diff --git a/docs/outputs.md b/docs/outputs.md index 5ab1767..22e7f43 100644 --- a/docs/outputs.md +++ b/docs/outputs.md @@ -15,6 +15,8 @@ results_bampiro/ ├── samples_legio_summary.tsv # Cohort metrics table (feeds the report) ├── samples_legio_snp_matrix.tsv # Master SNP matrix (site × sample AF & depth) ├── samples_legio_deletions.tsv # Stretches some samples have no reads for (see below) +├── samples_legio_snp_distances.tsv # SNPs between every two consensus sequences (square matrix) +├── samples_legio_snp_distances_pairs.tsv # The same, one row per pair, with how much each rests on ├── samples_legio_gene_burden.tsv # Per-gene functional burden (cohort) ├── samples_legio_dr.tsv # 💊 Drug-resistance calls (pathotypr; only if --run_pathotypr) │ @@ -95,6 +97,21 @@ genotype's dosage passed off as a fraction. Rows are keyed on contig and position, so two references that share a contig name cannot be told apart; the matrix step warns when it sees one. +## SNP distances + +`_snp_distances.tsv` is the square matrix of SNPs between every two consensus +sequences, `NA` between samples mapped against different references, which share no coordinates. +Two samples differ at a position only where both called a base (`A`, `C`, `G` or `T`) and the +bases differ: a no-call, a masked position or an ambiguity code at a mixed site is never a +difference, whichever sample it is in. These are the distances a tree built on the same +consensus sequences sees. + +`_snp_distances_pairs.tsv` has one row per pair with `compared`, the number of the +reference's variable positions both samples called. A thinly called sample is close to everyone +because it called little that could differ; the report leaves pairs that compared under half of +the variable positions out of its clusters. Off with `--make_snp_distances false`, and absent +without consensus sequences. + ## Deletions `_deletions.tsv` lists the stretches of the reference that some samples have no reads @@ -159,6 +176,7 @@ live-adjustable inside the HTML report. | `HIGH_IUPAC` | ambiguous/IUPAC % above | `--report_iupac_max` | :octicons-alert-fill-16:{ .amber } WARN | | `TITV_LOW` | Ti/Tv below | `--report_titv_min` | :octicons-alert-fill-16:{ .amber } WARN | | `LINEAGE_MISMATCH` | typed as a different lineage from the other samples mapped to its reference (it was probably mapped to the wrong genome) | - | :octicons-alert-fill-16:{ .amber } WARN | +| `GROUP_MISMATCH` | further than the cluster threshold from every other sample of its samplesheet group (patient, line, series), in a group whose members are otherwise within it of each other: swapped, mislabelled, contaminated or reinfected | `--snp_cluster_threshold` | :octicons-alert-fill-16:{ .amber } WARN | **FAIL** = a candidate for the report's exclusion list (FAIL samples start in it); **WARN** = review, usually keep. See [Downstream](downstream.md) for applying the list. diff --git a/docs/qc-report.md b/docs/qc-report.md index fd76197..aeda25d 100644 --- a/docs/qc-report.md +++ b/docs/qc-report.md @@ -18,7 +18,7 @@ back as a short report, plus a machine-readable `_qc_flags.tsv` of ## How it reads -The report is seven pages, listed in the sidebar in the order a reader needs them. A page +The report is eight pages, listed in the sidebar in the order a reader needs them. A page whose data the run did not produce disappears with its entry, and the rest are renumbered. The sidebar badge next to a page says whether it needs attention: the samples to exclude on *Sample QC*, the samples with resistance mutations their lineage does not share on @@ -29,10 +29,11 @@ on *Sample QC*, the samples with resistance mutations their lineage does not sha | **1 · Summary** | What the run found, as sentences with their numbers: an *In short* paragraph, then one card per finding (sample QC, identity, resistance, variants over time, lineages, gene conversion, coverage). Every card links to the page that holds its evidence and says what it does not prove. | | **2 · Sample QC** | Which samples can be trusted: the verdicts, the live thresholds, the **flagged samples** with the value behind every flag, the **exclusion list** and its exports, the table of all samples, lineages, **contamination** (Kraken2), distributions and aDNA damage. | | **3 · Genome & genes** | Consensus completeness, the genome landscape (missing calls, **deletions**, SNPs per bin and **per callable kb**, het, indels), the **deletions** found by comparing each sample's stretches without reads with the rest of the cohort, functional impact (snpEff), gene burden, variable genes and dN/dS. | -| **4 · Variants over time** | SNP dynamics, co-varying pairs (epistasis), the SNP matrix and variant × dose. | -| **5 · Resistance** | WHO-catalogue mutations grouped by mutation, the sample × drug matrix and every call. | -| **6 · Gene conversion** | Candidate tracts and the evidence behind each verdict. | -| **7 · Diagnostics** | Metric pairs, the correlation matrix, QC space, divergence vs completeness, dose × treatment and sampling dates: views for digging into a problem, none of which flags a sample. | +| **4 · Relatedness** | SNP distances between the consensus sequences, over the positions both samples called: a heatmap in the order that keeps each cluster together, the **clusters** at a threshold you can move, and the samples far from the rest of their own group (patient, line, series). | +| **5 · Variants over time** | SNP dynamics, co-varying pairs (epistasis), the SNP matrix and variant × dose. | +| **6 · Resistance** | WHO-catalogue mutations grouped by mutation, the sample × drug matrix and every call. | +| **7 · Gene conversion** | Candidate tracts and the evidence behind each verdict. | +| **8 · Diagnostics** | Metric pairs, the correlation matrix, QC space, divergence vs completeness, dose × treatment and sampling dates: views for digging into a problem, none of which flags a sample. | Printing (or saving as PDF) lays out every page one after another. diff --git a/modules/report.nf b/modules/report.nf index c5cc1ba..1603d5e 100644 --- a/modules/report.nf +++ b/modules/report.nf @@ -125,6 +125,7 @@ process QC_REPORT { path(gene_conversion) // cohort gene-conversion tracts TSV (COLLECT_GENE_CONVERSION) -> Gene conversion panel (may be empty) path(kraken_reports) // per-sample Kraken2 .report files -> Taxonomic composition panel (may be empty) path(depth_profiles, stageAs: 'depth/*') // per-sample DEPTH_PROFILE tables -> deletions + SNPs per callable kb (may be empty) + path(snp_distances) // SNP_DISTANCES pairs TSV -> relatedness page + GROUP_MISMATCH flag (may be NO_FILE) val(provenance) // pre-quoted provenance tokens (container=..., reference=...) val(basename) @@ -152,12 +153,14 @@ process QC_REPORT { GC_ARG=""; case "${gene_conversion}" in ""|NO_FILE*) ;; *) [ -s "${gene_conversion}" ] && GC_ARG="--gene-conversion ${gene_conversion}";; esac KRK_ARG=""; [ -n "${kraken_reports}" ] && KRK_ARG="--kraken ${kraken_reports}" DEL_ARG=""; [ -n "${depth_profiles}" ] && DEL_ARG="--depth-profiles ${depth_profiles} --out-deletions ${basename}_deletions.tsv" + DIST_ARG=""; case "${snp_distances}" in ""|NO_FILE*) ;; *) [ -s "${snp_distances}" ] && DIST_ARG="--snp-distances ${snp_distances}";; esac python3 ${projectDir}/bin/qc_report.py \\ --summary ${summary} \\ ${cons_arg} \\ --gene-burden ${gene_burden} \\ --gff ${gff} \\ - \$MASK_ARG \$LC_ARG \$MD_ARG \$VCF_ARG \$VH_ARG \$PL_ARG \$DR_ARG \$GC_ARG \$KRK_ARG \$DEL_ARG \\ + \$MASK_ARG \$LC_ARG \$MD_ARG \$VCF_ARG \$VH_ARG \$PL_ARG \$DR_ARG \$GC_ARG \$KRK_ARG \$DEL_ARG \$DIST_ARG \\ + --cluster-snps ${params.snp_cluster_threshold} \\ --deletion-min-len ${params.deletion_min_len} \\ --deletion-min-depth ${params.report_depth_min} \\ --aa2-label "${params.canonical_label}" \\ @@ -218,6 +221,41 @@ process DEPTH_PROFILE { """ } +process SNP_DISTANCES { + // Pairwise SNP distances between the consensus sequences: two samples differ where both called + // a base and the bases differ, so a gap, a masked position or a mixed site never counts. Taken + // from the consensus because it already carries every masking decision a tree built on it sees. + tag "SNP distances" + publishDir "${params.outdir}", mode: params.publish_mode + cpus 1 + memory { 4.GB * task.attempt } + + input: + path(manifest) // samplereferenceconsensus file name + path(fastas, stageAs: 'cons/*') // the masked consensus FASTAs + val(basename) + + output: + path("${basename}_snp_distances.tsv"), emit: square + path("${basename}_snp_distances_pairs.tsv"), emit: pairs + + script: + """ + set -euo pipefail + python3 ${projectDir}/bin/snp_distances.py \\ + --manifest ${manifest} \\ + --dir cons \\ + -o ${basename}_snp_distances.tsv \\ + --pairs ${basename}_snp_distances_pairs.tsv + """ + + stub: + """ + printf 'sample\\n' > ${basename}_snp_distances.tsv + printf 'sample_a\\tsample_b\\treference\\tsnps\\tcompared\\tvariable_sites\\n' > ${basename}_snp_distances_pairs.tsv + """ +} + process SNP_MATRIX { tag "SNP matrix" publishDir "${params.outdir}", mode: params.publish_mode diff --git a/nextflow.config b/nextflow.config index 1e42c33..e474f3a 100644 --- a/nextflow.config +++ b/nextflow.config @@ -234,6 +234,11 @@ params { // reads turn up there by chance. depth_window = 1000 deletion_min_len = 200 + // Pairwise SNP distances between the consensus sequences (_snp_distances.tsv) and the + // report's relatedness page. The threshold only sets where the report starts drawing clusters and + // can be moved live there; 12 SNPs is the usual TB transmission cut, so set your organism's own. + make_snp_distances = true + snp_cluster_threshold = 12 // --- Consensus Settings --- consensus_wrap = 80 diff --git a/subworkflows/cohort_report.nf b/subworkflows/cohort_report.nf index e1d4ad5..facfc81 100644 --- a/subworkflows/cohort_report.nf +++ b/subworkflows/cohort_report.nf @@ -6,7 +6,7 @@ */ include { ANNOTATE_CANONICAL } from '../modules/annotation' -include { COLLECT_SUMMARY; COLLECT_DR; DEPTH_PROFILE; LIFT_VARIANTS; QC_REPORT; SNP_MATRIX } from '../modules/report' +include { COLLECT_SUMMARY; COLLECT_DR; DEPTH_PROFILE; LIFT_VARIANTS; QC_REPORT; SNP_DISTANCES; SNP_MATRIX } from '../modules/report' include { asBool } from '../modules/utils' workflow COHORT_REPORT { @@ -102,9 +102,19 @@ workflow COHORT_REPORT { def depth_profiles = depth.profile.map { sId, rId, win, zero -> [win, zero] } .mix(depth.genes.map { sId, rId, genes -> [genes] }) .flatten().collect().ifEmpty([]) + // Pairwise SNP distances between the consensus sequences. The manifest names each file's sample + // and reference, so neither has to be read back out of a file name. + // Without consensus sequences SNP_DISTANCES never runs, and ifEmpty hands the report the + // placeholder instead of leaving it waiting for an input that never comes. + def snp_dist = asBool(params.make_snp_distances) + ? SNP_DISTANCES(masked_consensus.map { sId, rId, fa -> "${sId}\t${rId}\t${fa.name}" } + .collectFile(name: 'consensus_manifest.tsv', newLine: true), + cons_files, tsv_name).pairs + .ifEmpty(file("${projectDir}/assets/NO_FILE_DISTANCES")) + : file("${projectDir}/assets/NO_FILE_DISTANCES") QC_REPORT(summ.summary, summ.gene_burden, cons_files, report_gff, report_mask, report_meta, report_vcfs, report_vcfs_h37rv, pos_liftover, dr_report, report_gconv, - report_kraken, depth_profiles, provenance, tsv_name) + report_kraken, depth_profiles, snp_dist, provenance, tsv_name) } // 11b. Master SNP matrix: rows = SNP sites, columns = reference/annotation + per-sample AF & depth. diff --git a/tests/js/relatedness.test.mjs b/tests/js/relatedness.test.mjs new file mode 100644 index 0000000..cc66be6 --- /dev/null +++ b/tests/js/relatedness.test.mjs @@ -0,0 +1,53 @@ +// The relatedness page clusters the samples and orders its heatmap from the pairwise distances +// alone, so both are checked on hand-built references. + +import { strict as assert } from "node:assert"; +import { describe, it } from "node:test"; +import { emptyReport, loadFunctions } from "./harness.mjs"; + +const host = (x) => JSON.parse(JSON.stringify(x)); + +// Upper triangle, row by row, of a symmetric matrix given as {"a,b": snps}. +function ref(samples, d, cmp = 100) { + const snps = [], c = []; + for (let i = 0; i < samples.length; i++) + for (let j = i + 1; j < samples.length; j++) { + snps.push(d[`${samples[i]},${samples[j]}`] ?? d[`${samples[j]},${samples[i]}`] ?? 1000); + c.push(typeof cmp === "object" ? (cmp[`${samples[i]},${samples[j]}`] ?? 100) : cmp); + } + return { samples, variable: 100, snps, cmp: c }; +} + +const fns = loadFunctions(["relIdx", "relD", "relClusters", "relOrder"], { ...emptyReport(), samples: [], metrics: [] }); + +describe("relClusters, single linkage at a threshold", () => { + it("chains pairs within the threshold into one cluster", () => { + // a-b 3 and b-c 4 join a, b and c although a-c is 7; d is far from all of them. + const r = ref(["a", "b", "c", "d"], { "a,b": 3, "b,c": 4, "a,c": 7 }); + assert.deepEqual(host(fns.relClusters(r, 5, 50)), [[0, 1, 2], [3]]); + }); + + it("leaves out a pair that compared too few positions", () => { + const r = ref(["a", "b"], { "a,b": 0 }, { "a,b": 10 }); + assert.deepEqual(host(fns.relClusters(r, 5, 50)), [[0], [1]], "0 SNPs over 10% of the positions says nothing"); + }); +}); + +describe("relOrder, the heatmap's order", () => { + it("keeps every cluster contiguous", () => { + // Two tight pairs interleaved in the input: a,c close and b,d close. + const r = ref(["a", "b", "c", "d"], { "a,c": 1, "b,d": 2 }); + const order = host(fns.relOrder(r, 50)); + const pos = Object.fromEntries(order.map((x, i) => [x, i])); + assert.equal(Math.abs(pos[0] - pos[2]), 1, "a and c sit together"); + assert.equal(Math.abs(pos[1] - pos[3]), 1, "b and d sit together"); + assert.deepEqual([...order].sort(), [0, 1, 2, 3]); + }); + + it("indexes the upper triangle row by row", () => { + const r = ref(["a", "b", "c"], { "a,b": 1, "a,c": 2, "b,c": 3 }); + assert.equal(fns.relD(r, 0, 2), 2); + assert.equal(fns.relD(r, 2, 1), 3); + assert.equal(fns.relD(r, 1, 1), 0); + }); +}); diff --git a/tests/unit/test_qc_report_relatedness.py b/tests/unit/test_qc_report_relatedness.py new file mode 100644 index 0000000..9a1bdd5 --- /dev/null +++ b/tests/unit/test_qc_report_relatedness.py @@ -0,0 +1,130 @@ +"""Relatedness and GROUP_MISMATCH (bin/qcreport/relatedness.py). + +A sample the samplesheet puts in a group but that sits far from all of it, in a group whose +members are normally close, is swapped, mislabelled, contaminated or reinfected. These tests pin +when that is said, and when it is not: groups that are not genetic units, groups of two, and +pairs that rest on too few positions. +""" + +from __future__ import annotations + +from qcreport import relatedness as rel + + +def dist(pairs, variable=100, ref="R"): + """A parse_pairs() result from {(a, b): snps}, every pair comparing all variable positions.""" + out = {"ref": {}, "variable": {ref: variable}, "pairs": {}} + for (a, b), v in pairs.items(): + snps, comp = v if isinstance(v, tuple) else (v, variable) + out["pairs"][(a, b)] = (snps, comp) + out["ref"][a] = out["ref"][b] = ref + return out + + +def full(groups_of, within=2, between=2000, override=None): + """Every pair of the samples in `groups_of`: `within` SNPs inside a group, `between` across.""" + names = sorted(groups_of) + pairs = {} + for i, a in enumerate(names): + for b in names[i + 1:]: + pairs[(a, b)] = within if groups_of[a] == groups_of[b] else between + pairs.update(override or {}) + return dist(pairs) + + +LINES = {"a1": "A", "a2": "A", "a3": "A", "a4": "A", "b1": "B", "b2": "B", "b3": "B"} + + +def test_a_sample_far_from_its_group_and_close_to_another_is_flagged(): + """a4 is labelled line A but carries line B's genome: 2,000 SNPs from its line, 3 from b1.""" + over = {("a1", "a4"): 2000, ("a2", "a4"): 2000, ("a3", "a4"): 2000, ("a4", "b1"): 3, + ("a4", "b2"): 5, ("a4", "b3"): 4} + + out = rel.group_mismatches(full(LINES, override=over), LINES, threshold=12) + + assert list(out) == ["a4"] + assert out["a4"] == {"g": "A", "din": 2000, "nin": "a1", "dout": 3, "nout": "b1", "gout": "B"} + + +def test_a_sample_far_from_its_group_and_from_everyone_else_is_flagged_too(): + """A contaminant, or a culture of another strain: close to no sample at all. On a real + cohort the samples typed as another lineage sat 47 to 72 SNPs from each other and 2,000 from + their line mates, so requiring a close sample of another group missed every one of them.""" + over = {("a1", "a4"): 2000, ("a2", "a4"): 2000, ("a3", "a4"): 2000, + ("a4", "b1"): 60, ("a4", "b2"): 64, ("a4", "b3"): 70} + + out = rel.group_mismatches(full(LINES, override=over), LINES, threshold=12) + + assert out["a4"]["din"] == 2000 and (out["a4"]["dout"], out["a4"]["nout"]) == (60, "b1") + + +def test_a_consistent_cohort_flags_nobody(): + assert rel.group_mismatches(full(LINES), LINES) == {} + + +def test_a_group_that_is_not_a_genetic_unit_is_not_judged(): + """A column like 'hospital' also reads as a group; its members need not be related.""" + hospitals = {"s1": "H1", "s2": "H1", "s3": "H1", "s4": "H2", "s5": "H2", "s6": "H2"} + pairs = {(a, b): 500 for a in hospitals for b in hospitals if a < b} + pairs[("s1", "s4")] = 2 # a transmission across hospitals + + assert rel.group_mismatches(dist(pairs), hospitals) == {} + + +def test_a_group_of_two_cannot_say_which_member_is_out_of_place(): + groups = {"a1": "A", "a2": "A", "b1": "B", "b2": "B", "b3": "B"} + over = {("a1", "a2"): 2000, ("a2", "b1"): 3} + + assert rel.group_mismatches(full(groups, override=over), groups) == {} + + +def test_a_pair_resting_on_few_positions_does_not_count(): + """A distance over a handful of positions says nothing, near or far.""" + over = {("a1", "a4"): (2000, 10), ("a2", "a4"): (2000, 10), ("a3", "a4"): (2000, 10)} + + out = rel.group_mismatches(full(LINES, override=over), LINES, min_compared=0.5) + + assert out == {}, "a4's distances to its group compared 10 of 100 variable positions" + + +def test_samples_of_another_reference_are_never_compared(): + d = full(LINES, override={("a1", "a4"): 2000, ("a2", "a4"): 2000, ("a3", "a4"): 2000, + ("a4", "b1"): 900, ("a4", "b2"): 900, ("a4", "b3"): 900}) + d["ref"]["x"] = "OTHER" + d["pairs"][("a4", "x")] = (1, 100) + groups = dict(LINES, x="B") + + assert rel.group_mismatches(d, groups)["a4"]["nout"] == "b1", "x is on another reference" + + +def test_parse_pairs_reads_the_distances_tsv(tmp_path): + p = tmp_path / "pairs.tsv" + p.write_text("sample_a\tsample_b\treference\tsnps\tcompared\tvariable_sites\n" + "S1\tS2\tR\t4\t90\t100\nS1\tS3\tR\t7\t88\t100\n") + + d = rel.parse_pairs(str(p)) + + assert d["pairs"][("S1", "S2")] == (4, 90) + assert d["ref"] == {"S1": "R", "S2": "R", "S3": "R"} and d["variable"] == {"R": 100} + + +def test_parse_pairs_of_a_placeholder_is_empty(tmp_path): + p = tmp_path / "NO_FILE_DISTANCES" + p.write_text("") + + assert rel.parse_pairs(str(p)) == {"ref": {}, "variable": {}, "pairs": {}} + + +def test_build_relatedness_lays_out_each_references_upper_triangle(): + d = dist({("S1", "S2"): (4, 90), ("S1", "S3"): (7, 50), ("S2", "S3"): (9, 100)}) + + sec = rel.build_relatedness(d, {"S1": "P1", "S2": "P1"}, threshold=5) + + r = sec["refs"]["R"] + assert r["samples"] == ["S1", "S2", "S3"] + assert r["snps"] == [4, 7, 9] and r["cmp"] == [90, 50, 100] + assert sec["threshold"] == 5 and sec["group"] == {"S1": "P1", "S2": "P1"} + + +def test_build_relatedness_without_distances_is_none(): + assert rel.build_relatedness({"ref": {}, "variable": {}, "pairs": {}}) is None diff --git a/tests/unit/test_snp_distances.py b/tests/unit/test_snp_distances.py new file mode 100644 index 0000000..f4e9d33 --- /dev/null +++ b/tests/unit/test_snp_distances.py @@ -0,0 +1,102 @@ +"""Pairwise SNP distances between consensus sequences (bin/snp_distances.py). + +Two samples differ at a position only when both called a base there and the bases differ: a gap, +a masked position or an ambiguity code is not a difference, whichever sample it is in. +""" + +from __future__ import annotations + +from conftest import load_script + +sd = load_script("snp_distances") + + +def fasta(path, seq, name=None, wrap=5): + records = seq if isinstance(seq, list) else [(name or path.stem.split(".")[0], seq)] + path.write_text("".join(f">{n}\n" + "\n".join(s[i:i + wrap] for i in range(0, len(s), wrap)) + "\n" + for n, s in records)) + return str(path) + + +def run(tmp_path, seqs, refs=None): + """{(a, b): (snps, compared)} and the square matrix rows, from {sample: sequence}.""" + lines = [] + for s, seq in seqs.items(): + fasta(tmp_path / f"{s}.fa", seq, s) + lines.append(f"{s}\t{(refs or {}).get(s, 'REF')}\t{s}.fa") + (tmp_path / "manifest.tsv").write_text("\n".join(lines) + "\n") + assert sd.main(["--manifest", str(tmp_path / "manifest.tsv"), "--dir", str(tmp_path), + "-o", str(tmp_path / "square.tsv"), "--pairs", str(tmp_path / "pairs.tsv")]) == 0 + pairs = {} + for line in (tmp_path / "pairs.tsv").read_text().splitlines()[1:]: + a, b, _, snps, comp, _ = line.split("\t") + pairs[(a, b)] = (int(snps), int(comp)) + square = [r.split("\t") for r in (tmp_path / "square.tsv").read_text().splitlines()] + return pairs, square + + +def test_bases_that_differ_where_both_called_are_snps(tmp_path): + pairs, _ = run(tmp_path, {"A": "ACGTACGTAC", "B": "ACGTTCGTAA", "C": "ACGTACGTAC"}) + + assert pairs[("A", "B")][0] == 2 + assert pairs[("A", "C")][0] == 0 + + +def test_a_gap_mask_or_ambiguity_code_is_not_a_difference(tmp_path): + """The consensus writes '-' where depth was too low, 'X' where the pipeline masked, and an + IUPAC code at a mixed site. None of them says the samples differ.""" + pairs, _ = run(tmp_path, {"A": "ACGTACGTAC", "B": "-CXTRCGTAC", "C": "TCGTACGTAC"}) + + # Position 1 is the only variable one, and B did not call it: B differs from nobody there. + assert pairs[("A", "B")] == (0, 0) + assert pairs[("B", "C")] == (0, 0) + assert pairs[("A", "C")] == (1, 1) + + +def test_compared_counts_the_variable_positions_both_samples_called(tmp_path): + pairs, _ = run(tmp_path, {"A": "AAAAAAAAAA", "B": "CAAAAAAAAC", "C": "A-AAAAAAA-"}) + + assert pairs[("A", "B")] == (2, 2) + assert pairs[("A", "C")] == (0, 1), "C did not call position 10, so only position 1 compares" + + +def test_samples_of_different_references_are_not_compared(tmp_path): + pairs, square = run(tmp_path, {"A": "ACGTACGTAC", "B": "ACGTACGTAA", "X": "TTTTTTTTTTTT"}, + refs={"X": "OTHER"}) + + assert ("A", "X") not in pairs and ("B", "X") not in pairs + head = square[0] + row_a = dict(zip(head[1:], next(r for r in square[1:] if r[0] == "A")[1:])) + assert row_a == {"A": "0", "B": "1", "X": "NA"} + + +def test_a_multi_contig_consensus_is_compared_whole(tmp_path): + """A reference with a plasmid gives a consensus of two records, compared end to end.""" + fasta(tmp_path / "A.fa", [("A_chr", "ACGTA"), ("A_plasmid", "CCCC")]) + fasta(tmp_path / "B.fa", [("B_chr", "ACGTA"), ("B_plasmid", "CCCA")]) + + assert sd.main(["--consensus", str(tmp_path / "A.fa"), str(tmp_path / "B.fa"), + "-o", str(tmp_path / "sq.tsv"), "--pairs", str(tmp_path / "p.tsv")]) == 0 + + row = (tmp_path / "p.tsv").read_text().splitlines()[1].split("\t") + assert row[:2] == ["A", "B"], "a multi-record consensus is named after its file" + assert row[3:5] == ["1", "1"] + + +def test_a_single_record_names_its_sample(tmp_path): + fasta(tmp_path / "x.consensus.fasta", "ACGT", name="S1") + fasta(tmp_path / "y.consensus.fasta", "ACGA", name="S2") + + sd.main(["--consensus", str(tmp_path / "x.consensus.fasta"), str(tmp_path / "y.consensus.fasta"), + "-o", str(tmp_path / "sq.tsv")]) + + assert (tmp_path / "sq.tsv").read_text().splitlines()[0] == "sample\tS1\tS2" + + +def test_the_matrix_is_symmetric_with_a_zero_diagonal(tmp_path): + _, square = run(tmp_path, {"A": "ACGTACGTAC", "B": "TCGTACGTAA", "C": "ACCTACGTAC"}) + + head = square[0][1:] + m = {r[0]: dict(zip(head, r[1:])) for r in square[1:]} + assert all(m[s][s] == "0" for s in head) + assert all(m[a][b] == m[b][a] for a in head for b in head) From b9adeedc307b3ec38aa3fddff379cc7bce84e1d1 Mon Sep 17 00:00:00 2001 From: Paula Ruiz Rodriguez <50167687+Paururo@users.noreply.github.com> Date: Wed, 23 Sep 2026 14:34:06 +0200 Subject: [PATCH 03/14] Show what each series gained since its first time point For every samplesheet group followed over time (a patient, a passage line) the report lists the SNPs each later sample carries fixed that the group's first time point did not: new where that first time point was read at the site without the allele, risen where it held the allele as a minority, unknown where it was not read there, and lost for those fixed at the start and read without the allele later. The report now reads the SNP matrix for the depth, so a site the start never read is not counted as new; the matrix is built before the report for that reason. A chart follows each series through the median of each time point, coloured by treatment, and leaves out the samples the QC fails or places outside their series, which would otherwise flatten every real series: one mislabelled sample carries thousands of SNPs. On the Resistance page each series' grade 1-2 mutations are laid out per time point with those acquired since the start set apart. --- CHANGELOG.md | 12 +++ assets/NO_FILE.README.md | 1 + assets/NO_FILE_MATRIX | 0 bin/qc_report.py | 15 +++ bin/qcreport/render.py | 12 ++- bin/qcreport/series.py | 162 ++++++++++++++++++++++++++++ bin/report_assets/css/05_pages.css | 8 ++ bin/report_assets/js/07b_summary.js | 7 ++ bin/report_assets/js/10b_series.js | 93 ++++++++++++++++ bin/report_assets/js/14_boot.js | 6 +- bin/report_assets/shell.html | 8 +- docs/qc-report.md | 4 +- modules/report.nf | 5 +- subworkflows/cohort_report.nf | 24 +++-- tests/unit/test_qc_report_series.py | 122 +++++++++++++++++++++ 15 files changed, 460 insertions(+), 19 deletions(-) create mode 100644 assets/NO_FILE_MATRIX create mode 100644 bin/qcreport/series.py create mode 100644 bin/report_assets/js/10b_series.js create mode 100644 tests/unit/test_qc_report_series.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 29532fa..e278f49 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,18 @@ based on [Keep a Changelog](https://keepachangelog.com/), and the project follow ### Added +- **What each series gained since its first time point.** For every samplesheet group followed + over time (a patient, a passage line), the report lists the SNPs each later sample carries + fixed that the group's first time point did not: *new* where that first time point was read at + the site without the allele, *risen* where it held the allele as a minority, *unknown* where it + was not read there, and *lost* for those fixed at the start and read without the allele later. + The depth comes from the SNP matrix, which the report now reads, so a site the start never read + is not counted as new. A chart follows each series through the median of each time point, + coloured by treatment, and leaves out the samples the QC fails or places outside their series. + On the *Resistance* page, each series' grade 1-2 mutations are laid out per time point with + those acquired since the start set apart: on the cohort that motivated it, the bedaquiline + lines of one lineage acquire atpE E61D and I66M at passages 18 and 19. + - **How close the samples are to each other.** A new step, `SNP_DISTANCES`, counts the SNPs between every two consensus sequences over the positions both called, so a no-call, a masked position or a mixed site is never a difference: the distances a tree built on the same diff --git a/assets/NO_FILE.README.md b/assets/NO_FILE.README.md index 38ca64a..b54f532 100644 --- a/assets/NO_FILE.README.md +++ b/assets/NO_FILE.README.md @@ -12,6 +12,7 @@ stages one of these empty files instead, and the process recognises it by the | `NO_FILE_H37RV` | canonically annotated VCFs (`QC_REPORT`) | `--annotate_canonical false` | | `NO_FILE_LIFTOVER` | variant coordinate map (`QC_REPORT`) | `--variant_liftover false` | | `NO_FILE_DISTANCES` | pairwise SNP distances (`QC_REPORT`) | `--make_snp_distances false`, or no consensus sequences | +| `NO_FILE_MATRIX` | the master SNP matrix (`QC_REPORT`) | `--make_snp_matrix false` | Two properties matter, and both are load-bearing: diff --git a/assets/NO_FILE_MATRIX b/assets/NO_FILE_MATRIX new file mode 100644 index 0000000..e69de29 diff --git a/bin/qc_report.py b/bin/qc_report.py index e45eeea..a68189a 100644 --- a/bin/qc_report.py +++ b/bin/qc_report.py @@ -36,6 +36,7 @@ parse_kraken, parse_lineage_colors, parse_metadata, parse_pnps, parse_profile, parse_sample_meta, parse_summary, parse_vcfs, to_float) from qcreport.relatedness import build_relatedness, group_mismatches, parse_pairs +from qcreport.series import build_series, needed_cells, read_matrix_cells from qcreport.render import REPO_URL, SECTION_INFO, build_html @@ -105,6 +106,13 @@ def build_parser(): ap.add_argument("--cluster-snps", type=int, default=12, help="SNPs within which two samples are drawn as one cluster, and beyond which a " "sample is far from its group. The report can move it live.") + ap.add_argument("--snp-matrix", default=None, + help="The master SNP matrix TSV (build_snp_matrix.py with --depth-vcfs) -> what each " + "series gained since its first time point, telling a site absent there from one " + "not read (optional).") + ap.add_argument("--min-dp", type=int, default=7, + help="Depth a site needs before a sample without a call there counts as lacking the " + "allele (--consensus_min_dp).") ap.add_argument("--gate", action="store_true") return ap @@ -320,6 +328,12 @@ def build_payload(args, thr, anc_thr): _variants = load_variants(args) _sample_meta = parse_sample_meta(args.metadata) # shared by the dynamics filter and the SNP matrix header + # What each series gained since its first time point. The matrix is read only at the cells the + # comparison needs, and without it a site not called at the start cannot be told from one not read. + matrix_ok = bool(args.snp_matrix and os.path.exists(args.snp_matrix) + and not os.path.basename(args.snp_matrix).startswith("NO_FILE")) + _cells = read_matrix_cells(args.snp_matrix, needed_cells(series_meta, _variants)) if matrix_ok else {} + _series = build_series(series_meta, _variants, _cells, _sample_meta, args.min_dp, checked=matrix_ok) _dynamics = build_dynamics(series_meta, _variants, _sample_meta) # feeds dynamics + epistasis # front-load 'dose' so it survives the correlation matrix's top-N view dist_keys = (["dose"] + DIST) if dose_map else DIST @@ -345,6 +359,7 @@ def build_payload(args, thr, anc_thr): "snp_matrix": build_snp_matrix(_variants, provenance.get('reference', '')), "coverage": coverage, "relatedness": build_relatedness(distances, groups, args.cluster_snps), + "series": _series, "sample_meta": _sample_meta, "metrics": [{"key": k, "label": l, "kind": kind, "dir": d} for k, l, kind, d in METRICS], "extra": extra_metrics, diff --git a/bin/qcreport/render.py b/bin/qcreport/render.py index a40a8b4..eb18c27 100644 --- a/bin/qcreport/render.py +++ b/bin/qcreport/render.py @@ -48,7 +48,7 @@ def _asset(name): "js/01_prelude.js", "js/02_qcspace.js", "js/03_state.js", "js/04_helpers.js", "js/05_insights_qc.js", "js/06_insights_genome.js", "js/07_render_core.js", "js/07b_summary.js", "js/08_curation.js", "js/09_genome_genes.js", "js/09b_coverage.js", "js/09c_relatedness.js", - "js/10_dynamics.js", + "js/10_dynamics.js", "js/10b_series.js", "js/11_epistasis.js", "js/12_snpmatrix.js", "js/13_drug_kraken.js", "js/14_boot.js", ] @@ -110,6 +110,16 @@ def build_html(title, payload): "in a stretch the sample has no reads for while other samples read it; 'SNPs / kb' divides " "the SNPs of a bin by the positions deep enough to call, so a half-read bin does not look " "half as variable.", + "gains": "For every group of the samplesheet followed over time (a patient, a passage line), the SNPs " + "each later sample carries fixed (allele fraction 0.9 or more) that the group's first time " + "point did not. 'New' needs the first time point to have been read at the site without the " + "allele, which the SNP matrix's depth says; where it was not read the SNP is 'unknown' rather " + "than new. 'Lost' is the other way round: fixed at the start, read and absent later. SNPs " + "only, from the variant calls.", + "drseries": "The resistance mutations (WHO grade 1-2, from pathotypr) at every time point of every " + "series, with those acquired since the first time point set apart and those no longer " + "called crossed out. Answers when resistance appears along a series and whether it " + "persists. A genomic screen, not a drug-susceptibility result.", "reldist": "Pairwise SNP distances between the consensus sequences, counted over the positions both " "samples called: a gap, a masked position or a mixed site is never a difference, as in a " "tree built on the same sequences. Samples are ordered so that every single-linkage " diff --git a/bin/qcreport/series.py b/bin/qcreport/series.py new file mode 100644 index 0000000..cdebd56 --- /dev/null +++ b/bin/qcreport/series.py @@ -0,0 +1,162 @@ +"""What each series gained since its first time point. + +A series is the samples the samplesheet puts in one group (a patient, a passage line) at several +time points. For every later sample, the SNPs it carries fixed (allele fraction 0.9 or more) are +read against the series' first time point: + + new absent there, at a site read deeply enough to say so + risen present there already, below fixation (a minority that took over) + unknown the first time point was not read there, so nothing says whether it was absent + lost fixed at the first time point, read and absent in the later sample: a reversion, or a + sample that does not belong to the series + +Telling 'absent' from 'not read' needs the depth of the sites a sample has no call at, which is +what the SNP matrix holds. Without it every site not called at the start counts as new, and the +section says so rather than passing that off as checked. +""" +from __future__ import annotations + +import os + +FIXED = 0.9 + + +def _order(meta, samples): + return sorted(samples, key=lambda s: (meta[s].get("tnum") is None, + meta[s].get("tnum") if meta[s].get("tnum") is not None else 0, + str(meta[s].get("time") or ""))) + + +def series_groups(meta, variants): + """{group: (ordered samples, first-time-point samples)} for groups with two or more times.""" + groups = {} + for s, md in (meta or {}).items(): + if md.get("group") and s in variants: + groups.setdefault(md["group"], []).append(s) + out = {} + for g, samples in groups.items(): + samples = _order(meta, samples) + times = [meta[s].get("time") for s in samples] + if len(set(times)) < 2: + continue + out[g] = (samples, [s for s in samples if meta[s].get("time") == times[0]]) + return out + + +def needed_cells(meta, variants, fixed=FIXED): + """{site: {samples}} whose matrix cells the comparison reads: the first time point at every + site a later sample carries fixed, and every later sample at the sites fixed at the start.""" + need = {} + for samples, base in series_groups(meta, variants).values(): + later = [s for s in samples if s not in base] + for s in later: + for site, v in variants[s].items(): + if v.get("af") is not None and v["af"] >= fixed: + need.setdefault(site, set()).update(base) + for b in base: + for site, v in variants[b].items(): + if v.get("af") is not None and v["af"] >= fixed: + need.setdefault(site, set()).update(later) + return need + + +def read_matrix_cells(path, need): + """{(site, sample): (af, dp)} of the SNP matrix TSV (build_snp_matrix.py) for the cells in + `need`. af and dp are None where the cell is empty. Reads only the rows it needs.""" + out = {} + if not need or not path or not os.path.exists(path) or os.path.basename(path).startswith("NO_FILE"): + return out + try: + with open(path, encoding="utf-8", errors="replace") as fh: + header = fh.readline().rstrip("\n").split("\t") + col = {h[:-3]: i for i, h in enumerate(header) if h.endswith("|AF")} + for line in fh: + parts = line.split("\t", 3) + if len(parts) < 4: + continue + site = f"{parts[1]}:{parts[2]}" + wanted = need.get(site) + if not wanted: + continue + c = line.rstrip("\n").split("\t") + for s in wanted: + i = col.get(s) + if i is None or i + 1 >= len(c): + continue + af = float(c[i]) if c[i] not in ("", "NA") else None + dp = int(float(c[i + 1])) if c[i + 1] not in ("", "NA") else None + out[(site, s)] = (af, dp) + except (OSError, ValueError): + return {} + return out + + +def _state(site, sample, variants, cells, min_dp, fixed): + """'fixed', 'minority', 'absent', 'unread' or 'unchecked' for one sample at one site.""" + v = variants.get(sample, {}).get(site) + if v is not None and v.get("af") is not None: + return "fixed" if v["af"] >= fixed else ("minority" if v["af"] > 0 else "absent") + cell = cells.get((site, sample)) + if cell is None: + return "unchecked" + af, dp = cell + if af is not None and af > 0: + return "fixed" if af >= fixed else "minority" + if dp is None: + return "unchecked" + return "absent" if dp >= min_dp else "unread" + + +_RANK = ("fixed", "minority", "absent", "unread", "unchecked") + + +def build_series(meta, variants, cells, sample_meta=None, min_dp=7, fixed=FIXED, checked=True): + """The payload section, or None without a series of two or more time points.""" + groups = series_groups(meta, variants) + if not groups: + return None + tx = (sample_meta or {}).get("tx_field") + rows_meta = (sample_meta or {}).get("rows") or {} + out, sites = [], {} + for g, (samples, base) in sorted(groups.items()): + memo = {} + + def start(site): + if site not in memo: + memo[site] = min((_state(site, b, variants, cells, min_dp, fixed) for b in base), + key=_RANK.index) + return memo[site] + base_fixed = {site for b in base for site, v in variants[b].items() + if v.get("af") is not None and v["af"] >= fixed} + rows = [] + for s in samples: + if s in base: + continue + counts = {"new": [], "risen": [], "unknown": []} + for site, v in variants[s].items(): + if v.get("af") is None or v["af"] < fixed: + continue + st = start(site) + if st == "fixed": + continue + # Without the matrix, nothing tells a site absent at the start from one not read + # there: it counts as new, and the section carries checked=False to say so. + new = st == "absent" or (st == "unchecked" and not checked) + key = "new" if new else ("risen" if st == "minority" else "unknown") + counts[key].append(site) + lost = [site for site in base_fixed + if _state(site, s, variants, cells, min_dp, fixed) == "absent"] + for site in counts["new"] + counts["risen"] + lost: + v = next((variants[x][site] for x in samples + base if site in variants.get(x, {})), {}) + sites.setdefault(site, {"gene": v.get("gene", ""), "aa": v.get("aa", ""), + "eff": v.get("eff", ""), "alt": v.get("alt", "")}) + rows.append({"s": s, "time": meta[s].get("time"), "tnum": meta[s].get("tnum"), + "new": sorted(counts["new"]), "risen": sorted(counts["risen"]), + "unknown": len(counts["unknown"]), "lost": sorted(lost)}) + treat = [(rows_meta.get(s) or {}).get(tx) for s in samples] if tx else [] + treat = [t for t in treat if t] + out.append({"group": g, "tx": max(set(treat), key=treat.count) if treat else None, + "base": base, "t0": meta[base[0]].get("time"), "tnum0": meta[base[0]].get("tnum"), + "rows": rows}) + return {"groups": out, "sites": sites, "fixed": fixed, "min_dp": min_dp, "checked": checked, + "tx_field": tx} diff --git a/bin/report_assets/css/05_pages.css b/bin/report_assets/css/05_pages.css index dcc086c..b7eda26 100644 --- a/bin/report_assets/css/05_pages.css +++ b/bin/report_assets/css/05_pages.css @@ -174,3 +174,11 @@ table.drmut td{text-align:left} table.drmut td.num,table.drmut th.num{text-align .rel-bv{font-size:10px;color:var(--txt2)} .rel-bl{font-size:10px;color:var(--txt2);margin-top:3px;white-space:nowrap} .rel-chip{display:inline-block;padding:1px 6px;margin:1px 0;border-radius:5px;background:var(--soft);font-size:11.5px} +/* series: gains chart, and resistance mutations per time point */ +.ser-chart{overflow-x:auto} +.ser-dot{display:inline-block;width:9px;height:9px;border-radius:50%;margin-right:6px;vertical-align:middle} +.ser-tp{margin:2px 0} +.ser-mut{display:inline-block;padding:1px 6px;margin:1px 0;border-radius:5px;background:var(--soft);font-size:11.5px} +.ser-mut.acq{background:rgba(220,38,38,.12);font-weight:600} +.ser-mut.lost{text-decoration:line-through;opacity:.7} +.ser-tbl td{vertical-align:top} diff --git a/bin/report_assets/js/07b_summary.js b/bin/report_assets/js/07b_summary.js index 9cc3909..f5d56b1 100644 --- a/bin/report_assets/js/07b_summary.js +++ b/bin/report_assets/js/07b_summary.js @@ -165,12 +165,19 @@ function findResistance(){ nums:[{n:d.nCarriers,l:'samples',t:d.nCarriers?'bad':'good'},{n:d.acquired.length,l:'mutations',t:''},{n:d.drugs.length,l:_plural(d.drugs.length,'drug'),t:''}], link:{href:'#drug',t:'See every mutation'},note:'WHO catalogue grades 1–2, called from the reads. A genomic screen, not a drug-susceptibility result.',short:shortTxt}; } +// Per series, the fixed SNPs gained by its last time point: the median across series and the most. +function seriesGainSummary(){var S=R.series;if(!(S&&S.groups&&S.groups.length))return null; + var last=S.groups.map(function(g){var r=g.rows[g.rows.length-1];return {g:g.group,n:r?r['new'].length+r.risen.length:0};}); + var ns=last.map(function(x){return x.n;}).sort(function(a,b){return a-b;}),top=last.slice().sort(function(a,b){return b.n-a.n;})[0]; + return {median:ns[Math.floor(ns.length/2)],max:top.n,top:top.g,checked:S.checked};} function findDynamics(){ var d=dynSummary(); if(!d)return null; var head=''+d.sweep.length+' '+_plural(d.sweep.length,'allele','alleles')+' swept toward fixation and '+d.rise+' rose, across '+d.nSeries+' series'; var body=[]; if(d.parallel.length)body.push('Rising in two or more independent series: '+_list(d.parallel.map(function(x){return ''+esc(x.gene)+' ('+x.n+')';}),5)+'.'); body.push('Out of '+d.n.toLocaleString('en-US')+' trajectories; '+d.fall+' declined.'); + var sg=seriesGainSummary(); if(sg)body.push('By its last time point a series had gained a median of '+sg.median+' fixed '+_plural(sg.median,'SNP','SNPs')+ + ' since its first'+(sg.max>sg.median?(', up to '+sg.max+' in '+esc(sg.top)):'')+(sg.checked?'':' (the start not checked for depth)')+'.'); return {k:'dyn',eyebrow:'Variants over time',tone:'',head:head,body:body.join(' '), link:{href:'#dynamics',t:'Open the trajectories'},note:'A screen of allele-frequency trajectories, not a selection test: drift and linked passengers move too.', short:d.sweep.length?d.sweep.length+' '+_plural(d.sweep.length,'allele','alleles')+' swept toward fixation.':''}; diff --git a/bin/report_assets/js/10b_series.js b/bin/report_assets/js/10b_series.js new file mode 100644 index 0000000..f9a7a5c --- /dev/null +++ b/bin/report_assets/js/10b_series.js @@ -0,0 +1,93 @@ +// ---- Series: what each group gained since its first time point ---- +// A group of the samplesheet (a patient, a passage line) followed over time. Each later sample's +// fixed SNPs are read against the group's first time point, where the SNP matrix says whether a +// site was read without the allele (new) or not read at all (unknown). +var SERPAL=['#4477aa','#ee6677','#228833','#ccbb44','#66ccee','#aa3377','#e69f00','#0072b2','#d55e00','#009e73','#cc79a7','#882255']; +function serColorBy(){var S=R.series,m={},k=0;if(!S)return function(){return '#4f83c2';}; + var key=function(g){return S.tx_field?(g.tx||'NA'):g.group;}; + S.groups.map(key).sort().forEach(function(v){if(!(v in m)){m[v]=SERPAL[k%SERPAL.length];k++;}}); + var f=function(g){return m[key(g)];};f.map=m;return f;} +function serGained(r){return r['new'].length+r.risen.length;} +// A sample the QC fails or places outside its series (another lineage, far from its group) is not +// the series evolving: it is drawn hollow and left out of the line and of the scale, so one +// mislabelled sample carrying 2,000 SNPs does not flatten every real series to the floor. +var SER_OUT={GROUP_MISMATCH:1,LINEAGE_MISMATCH:1}; +function serOutside(sid){var s=null;R.samples.forEach(function(x){if(x.s===sid)s=x;}); + return !!s&&(s.v==='FAIL'||(s.f||[]).some(function(f){return SER_OUT[f];}));} +function serMutLabel(site){var x=(R.series.sites||{})[site]||{};return (x.gene?x.gene:site)+(x.aa?' '+x.aa.replace(/^p\./,''):'');} +function renderSeries(){ + var host=el('gains_body'),S=R.series; if(!host)return; if(!(S&&S.groups&&S.groups.length)){host.innerHTML='';return;} + var col=serColorBy(),numeric=S.groups.every(function(g){return g.tnum0!=null&&g.rows.every(function(r){return r.tnum!=null;});}); + var pts=[];S.groups.forEach(function(g){pts.push(numeric?g.tnum0:0);g.rows.forEach(function(r,i){pts.push(numeric?r.tnum:i+1);});}); + var x0=Math.min.apply(null,pts),x1=Math.max.apply(null,pts),ymax=1,nOut=0; + S.groups.forEach(function(g){g.rows.forEach(function(r){if(serOutside(r.s))nOut++;else ymax=Math.max(ymax,serGained(r));});}); + var W=Math.max(320,Math.min((host.clientWidth||900)-24,980)),H=260,pl=40,pr=12,pt=12,pb=28,iw=W-pl-pr,ih=H-pt-pb; + function X(v){return pl+(x1>x0?(v-x0)/(x1-x0):0.5)*iw;} function Y(v){return pt+ih-v/ymax*ih;} + var svg=''; + [0,0.5,1].forEach(function(t){var v=Math.round(t*ymax);svg+=''+ + ''+v+'';}); + var ticks={};pts.forEach(function(v){ticks[v]=1;}); + Object.keys(ticks).map(Number).sort(function(a,b){return a-b;}).forEach(function(v){ + svg+=''+(numeric?v:'')+'';}); + S.groups.forEach(function(g){var c=col(g),byT={},order=[]; + // the line runs through the median of the samples counted at each time point + g.rows.forEach(function(r,i){var x=numeric?r.tnum:i+1;if(serOutside(r.s))return;if(!(x in byT)){byT[x]=[];order.push(x);}byT[x].push(serGained(r));}); + var line=[[numeric?g.tnum0:0,0]].concat(order.sort(function(a,b){return a-b;}).map(function(x){var v=byT[x].slice().sort(function(a,b){return a-b;});return [x,v[Math.floor(v.length/2)]];})); + svg+=''; + svg+=''; + g.rows.forEach(function(r,i){var x=numeric?r.tnum:i+1,out=serOutside(r.s),v=serGained(r),y=out?Math.min(v,ymax):v; + var lab=esc(r.s)+' · '+esc(r.time)+': '+r['new'].length+' new, '+r.risen.length+' risen'+(r.unknown?', '+r.unknown+' unknown':'')+(r.lost.length?', '+r.lost.length+' lost':'')+(out?' · not counted: the QC fails it or places it outside its series':''); + svg+='';});}); + svg+='fixed SNPs gained since the first time point'; + var legend='
'+Object.keys(col.map).map(function(k){return ''+esc(k)+'';}).join('')+ + (nOut?''+nOut+' not counted (FAIL, or outside its series), capped at the top of the scale':'')+ + ''+(S.tx_field?'coloured by '+esc(S.tx_field):'coloured by group')+'; the line is the median per time point; hover a point for its sample
'; + var rows=[];S.groups.forEach(function(g){g.rows.forEach(function(r,i){ + var muts=r['new'].map(function(s){return ''+esc(serMutLabel(s))+'';}).concat(r.risen.map(function(s){return esc(serMutLabel(s))+' (risen)';})); + rows.push(''+(i===0?''+esc(g.group):'')+''+ + (S.tx_field?''+(i===0?esc(g.tx||''):'')+'':'')+ + ''+esc(r.s)+(serOutside(r.s)?' (not counted)':'')+''+esc(r.time)+''+r['new'].length+''+r.risen.length+''+ + ''+r.unknown+''+ + ''+r.lost.length+''+ + ''+(muts.length?_list(muts,8):'none')+'');});}); + host.innerHTML='
Fixed SNPs (allele fraction '+S.fixed+' or more) each sample carries that the first time point of its series did not: '+ + 'new where that first time point was read at the site without the allele, risen where it held the allele as a minority, unknown where it was not read there. '+ + 'Lost counts the SNPs fixed at the start and read without the allele later: a reversion, or a sample that does not belong to its series. '+ + (S.checked?'':'The SNP matrix was not available, so a site not called at the first time point could not be told from one not read there, and counts as new. ')+ + 'SNPs only; indels and deletions are not counted.
'+ + '
'+svg+'
'+legend+ + '
'+(S.tx_field?'':'')+ + ''+rows.join('')+'
Series'+esc(S.tx_field)+'SampleTimeNewRisenUnknownLostGained (new in bold)
'; + Array.prototype.forEach.call(host.querySelectorAll('circle[data-tip]'),function(c){ + c.onmousemove=function(e){tip(c.getAttribute('data-tip'),e.clientX,e.clientY);};c.onmouseleave=function(){tip('');};}); +} +// Resistance along each series: the WHO grade 1-2 mutations at every time point, those acquired +// since the first one set apart. Lineage markers are there from the start, so they read as inherited. +function renderDrSeries(){ + var host=el('drseries_body'),S=R.series,D=R.dr; if(!host)return; + if(!(S&&S.groups&&D&&D.calls)){host.innerHTML='';return;} + var by={};D.calls.forEach(function(c){if(c.gn==null||c.gn>2)return;(by[c.s]=by[c.s]||{})[c.gene+' '+c.mutation]=c.drug;}); + var has={};D.calls.forEach(function(c){has[c.s]=1;}); + var out=[]; + S.groups.forEach(function(g){ + var base={};g.base.forEach(function(b){var m=by[b]||{};Object.keys(m).forEach(function(k){base[k]=m[k];});}); + var any=Object.keys(base).length>0,cells=[]; + g.rows.forEach(function(r){var m=by[r.s]||{},keys=Object.keys(m),acq=keys.filter(function(k){return !(k in base);}); + var lost=has[r.s]?Object.keys(base).filter(function(k){return !(k in m);}):[]; + if(keys.length||lost.length)any=true; + cells.push({r:r,acq:acq,kept:keys.filter(function(k){return k in base;}),lost:lost,m:m});}); + if(any)out.push({g:g,base:base,cells:cells});}); + if(!out.length){host.innerHTML='
No series carries a grade 1–2 resistance mutation at any time point.
';return;} + function mut(k,drug,cls){return ''+esc(k)+(drug?' '+esc(drug)+'':'')+'';} + var nAcq=0;out.forEach(function(o){o.cells.forEach(function(c){nAcq+=c.acq.length?1:0;});}); + host.innerHTML='
Grade 1–2 mutations of the WHO catalogue at each time point of each series. Acquired ones were not there at the first time point; a crossed-out one was there and is no longer called. '+ + nAcq+' later '+_plural(nAcq,'sample carries','samples carry')+' at least one acquired mutation. A genomic screen, not a susceptibility result.
'+ + '
'+(S.tx_field?'':'')+''+ + out.map(function(o){var bk=Object.keys(o.base); + return ''+(S.tx_field?'':'')+ + ''+ + '';}).join('')+ + '
Series'+esc(S.tx_field)+'First time pointLater time points
'+esc(o.g.group)+''+esc(o.g.tx||'')+'
'+esc(o.g.t0)+'
'+(bk.length?bk.map(function(k){return mut(k,o.base[k],'');}).join(' '):'none')+'		'+o.cells.map(function(c){return '
'+esc(c.r.time)+' · '+esc(c.r.s)+' '+ + c.acq.map(function(k){return mut(k,c.m[k],'acq');}).concat(c.kept.map(function(k){return mut(k,c.m[k],'');})).concat(c.lost.map(function(k){return mut(k,o.base[k],'lost');})).join(' ')+ + (c.acq.length+c.kept.length+c.lost.length?'':''+(has[c.r.s]?'none':'no resistance calls for this sample')+'')+'
';}).join('')+'
'; +} diff --git a/bin/report_assets/js/14_boot.js b/bin/report_assets/js/14_boot.js index ab7ecb0..65a38d7 100644 --- a/bin/report_assets/js/14_boot.js +++ b/bin/report_assets/js/14_boot.js @@ -7,8 +7,8 @@ var PAGE_RENDER={ qc:function(){renderOverview();renderFlags();renderCuration();renderTable();renderLineages();renderKraken();renderPlots();renderADNA();}, genome:function(){renderStacks();renderGenome();renderDeletions();renderFunction();renderGeneBurden();renderHotspots();renderPnps();}, related:function(){renderRelatedness();}, - variants:function(){renderDynamics();renderEpistasis();renderSnpMatrix();renderVarDose();}, - drug:function(){renderDrug();}, + variants:function(){renderSeries();renderDynamics();renderEpistasis();renderSnpMatrix();renderVarDose();}, + drug:function(){renderDrug();renderDrSeries();}, gconv:function(){renderGconv();}, diag:function(){renderScatter();renderCorr();renderQCspace();renderRefBias();renderDoseTx();renderTemporal();}}; var curPage='summary', pageDirty={}; @@ -207,6 +207,8 @@ function genomeResetZoom(){st.gsel=null;st.gzoom=null;st.geneMark=null;var gg=el var rd=el('gselreadout');if(rd)rd.innerHTML='gene '+esc(g.name)+' · '+fmtpos(g.start)+' - '+fmtpos(g.end)+' '; var cb=el('gselclear');if(cb)cb.onclick=function(ev){ev.stopPropagation();genomeResetZoom();};};})(); if(!R.samples.some(function(s){return s.miss||s.trk;})){var gs=el('genome');if(gs)gs.style.display='none';var ng=el('nav-genome');if(ng)ng.style.display='none';} +if(!(R.series&&R.series.groups&&R.series.groups.length)){['gains','nav-gains','drseries','nav-drseries'].forEach(function(id){var x=el(id);if(x)x.style.display='none';});} +else if(!(R.dr&&R.dr.calls&&R.dr.calls.length)){['drseries','nav-drseries'].forEach(function(id){var x=el(id);if(x)x.style.display='none';});} if(!relSec()){['reldist','relclus','relgroup','nav-reldist','nav-relclus','nav-relgroup'].forEach(function(id){var x=el(id);if(x)x.style.display='none';});} if(!(R.coverage&&R.coverage.regions)){var dlx=el('deletions');if(dlx)dlx.style.display='none';var ndlx=el('nav-del');if(ndlx)ndlx.style.display='none';} if(!R.samples.some(function(s){return s.trk&&s.trk.snp;})){var hsx=el('hotspots');if(hsx)hsx.style.display='none';var nhx=el('nav-hot');if(nhx)nhx.style.display='none';} diff --git a/bin/report_assets/shell.html b/bin/report_assets/shell.html index 4611311..18a53fd 100644 --- a/bin/report_assets/shell.html +++ b/bin/report_assets/shell.html @@ -10,8 +10,8 @@
2Sample QC
Flagged samplesAll samplesLineagesContaminationDistributionsaDNA damage
3Genome & genes
ConsensusGenome landscapeDeletionsFunctional impactGene burdenVariable genesSelection (dN/dS)
4Relatedness
DistancesClustersOutside their group
-
5Variants over time
SNP dynamicsCo-varying pairsSNP matrixVariant × dose
-
6Resistance
Resistance mutations
+
5Variants over time
Gained per seriesSNP dynamicsCo-varying pairsSNP matrixVariant × dose
+
6Resistance
Resistance mutationsAlong each series
7Gene conversion
Candidate tracts
8Diagnostics
Metric pairsCorrelation matrixQC spaceDivergenceDose × treatmentSampling dates
@@ -114,6 +114,8 @@
5 · Variants over time

How allele frequencies change along each series

Every variant followed through the time points of its series (a patient, a passage line), the pairs that move together, and the full site × sample matrix.

+

Gained along each series fixed SNPs each sample carries that the first time point of its series did not

+

SNP dynamics allele-frequency trajectories per gene; the screen at the top ranks what is changing

Co-varying pairs variants whose trajectories move together (concordant) or against each other (discordant) within a series

@@ -130,6 +132,8 @@
6 · Resistance

Resistance-associated mutations

Mutations in the WHO catalogue called by pathotypr straight from the reads (H37Rv numbering). Those every sample of a lineage shares are separated from those only some samples carry. A genomic screen, not a drug-susceptibility result.

Resistance mutations grade 1–2 = associated with resistance, 3 = uncertain, 4–5 = not associated

+

Resistance along each series the resistance mutations at every time point, those acquired since the first one set apart

+
diff --git a/docs/qc-report.md b/docs/qc-report.md index aeda25d..0b434f8 100644 --- a/docs/qc-report.md +++ b/docs/qc-report.md @@ -30,8 +30,8 @@ on *Sample QC*, the samples with resistance mutations their lineage does not sha | **2 · Sample QC** | Which samples can be trusted: the verdicts, the live thresholds, the **flagged samples** with the value behind every flag, the **exclusion list** and its exports, the table of all samples, lineages, **contamination** (Kraken2), distributions and aDNA damage. | | **3 · Genome & genes** | Consensus completeness, the genome landscape (missing calls, **deletions**, SNPs per bin and **per callable kb**, het, indels), the **deletions** found by comparing each sample's stretches without reads with the rest of the cohort, functional impact (snpEff), gene burden, variable genes and dN/dS. | | **4 · Relatedness** | SNP distances between the consensus sequences, over the positions both samples called: a heatmap in the order that keeps each cluster together, the **clusters** at a threshold you can move, and the samples far from the rest of their own group (patient, line, series). | -| **5 · Variants over time** | SNP dynamics, co-varying pairs (epistasis), the SNP matrix and variant × dose. | -| **6 · Resistance** | WHO-catalogue mutations grouped by mutation, the sample × drug matrix and every call. | +| **5 · Variants over time** | What each series **gained since its first time point** (new, risen, unknown, lost, charted per treatment), SNP dynamics, co-varying pairs (epistasis), the SNP matrix and variant × dose. | +| **6 · Resistance** | WHO-catalogue mutations grouped by mutation, the sample × drug matrix, every call, and each series' mutations **per time point**, those acquired since the first one set apart. | | **7 · Gene conversion** | Candidate tracts and the evidence behind each verdict. | | **8 · Diagnostics** | Metric pairs, the correlation matrix, QC space, divergence vs completeness, dose × treatment and sampling dates: views for digging into a problem, none of which flags a sample. | diff --git a/modules/report.nf b/modules/report.nf index 1603d5e..eda57e6 100644 --- a/modules/report.nf +++ b/modules/report.nf @@ -126,6 +126,7 @@ process QC_REPORT { path(kraken_reports) // per-sample Kraken2 .report files -> Taxonomic composition panel (may be empty) path(depth_profiles, stageAs: 'depth/*') // per-sample DEPTH_PROFILE tables -> deletions + SNPs per callable kb (may be empty) path(snp_distances) // SNP_DISTANCES pairs TSV -> relatedness page + GROUP_MISMATCH flag (may be NO_FILE) + path(snp_matrix) // SNP_MATRIX TSV -> what each series gained since its first time point (may be NO_FILE) val(provenance) // pre-quoted provenance tokens (container=..., reference=...) val(basename) @@ -154,13 +155,15 @@ process QC_REPORT { KRK_ARG=""; [ -n "${kraken_reports}" ] && KRK_ARG="--kraken ${kraken_reports}" DEL_ARG=""; [ -n "${depth_profiles}" ] && DEL_ARG="--depth-profiles ${depth_profiles} --out-deletions ${basename}_deletions.tsv" DIST_ARG=""; case "${snp_distances}" in ""|NO_FILE*) ;; *) [ -s "${snp_distances}" ] && DIST_ARG="--snp-distances ${snp_distances}";; esac + MX_ARG=""; case "${snp_matrix}" in ""|NO_FILE*) ;; *) [ -s "${snp_matrix}" ] && MX_ARG="--snp-matrix ${snp_matrix}";; esac python3 ${projectDir}/bin/qc_report.py \\ --summary ${summary} \\ ${cons_arg} \\ --gene-burden ${gene_burden} \\ --gff ${gff} \\ - \$MASK_ARG \$LC_ARG \$MD_ARG \$VCF_ARG \$VH_ARG \$PL_ARG \$DR_ARG \$GC_ARG \$KRK_ARG \$DEL_ARG \$DIST_ARG \\ + \$MASK_ARG \$LC_ARG \$MD_ARG \$VCF_ARG \$VH_ARG \$PL_ARG \$DR_ARG \$GC_ARG \$KRK_ARG \$DEL_ARG \$DIST_ARG \$MX_ARG \\ --cluster-snps ${params.snp_cluster_threshold} \\ + --min-dp ${params.consensus_min_dp} \\ --deletion-min-len ${params.deletion_min_len} \\ --deletion-min-depth ${params.report_depth_min} \\ --aa2-label "${params.canonical_label}" \\ diff --git a/subworkflows/cohort_report.nf b/subworkflows/cohort_report.nf index facfc81..891cd3e 100644 --- a/subworkflows/cohort_report.nf +++ b/subworkflows/cohort_report.nf @@ -36,7 +36,15 @@ workflow COHORT_REPORT { .collect().ifEmpty([]) def ref_name = refMap.keySet().join(',') - // 11a. Consolidated QC report: aggregate every per-sample legacy log into one summary TSV, then + // 11a. Master SNP matrix: rows = SNP sites, columns = reference/annotation + per-sample AF & depth. + // The all-positions VCFs say what a sample's reads show at a site it has no call at. Built before + // the report, which reads it to tell a site absent at a series' first time point from one not read. + def snp_matrix = asBool(params.make_snp_matrix) + ? SNP_MATRIX(report_vcfs, allpos_vcf.map { sId, rId, vcf, tbi -> vcf }.collect().ifEmpty([]), + ref_name, tsv_name).matrix + : file("${projectDir}/assets/NO_FILE_MATRIX") + + // 11b. Consolidated QC report: aggregate every per-sample legacy log into one summary TSV, then // render the self-contained interactive HTML + PASS/WARN/FAIL flags. if (asBool(params.make_qc_report)) { def all_logs = legacy_log.collect() @@ -103,9 +111,9 @@ workflow COHORT_REPORT { .mix(depth.genes.map { sId, rId, genes -> [genes] }) .flatten().collect().ifEmpty([]) // Pairwise SNP distances between the consensus sequences. The manifest names each file's sample - // and reference, so neither has to be read back out of a file name. - // Without consensus sequences SNP_DISTANCES never runs, and ifEmpty hands the report the - // placeholder instead of leaving it waiting for an input that never comes. + // and reference, so neither has to be read back out of a file name. Without consensus sequences + // SNP_DISTANCES never runs, and ifEmpty hands the report the placeholder instead of leaving it + // waiting for an input that never comes. def snp_dist = asBool(params.make_snp_distances) ? SNP_DISTANCES(masked_consensus.map { sId, rId, fa -> "${sId}\t${rId}\t${fa.name}" } .collectFile(name: 'consensus_manifest.tsv', newLine: true), @@ -114,13 +122,7 @@ workflow COHORT_REPORT { : file("${projectDir}/assets/NO_FILE_DISTANCES") QC_REPORT(summ.summary, summ.gene_burden, cons_files, report_gff, report_mask, report_meta, report_vcfs, report_vcfs_h37rv, pos_liftover, dr_report, report_gconv, - report_kraken, depth_profiles, snp_dist, provenance, tsv_name) + report_kraken, depth_profiles, snp_dist, snp_matrix, provenance, tsv_name) } - // 11b. Master SNP matrix: rows = SNP sites, columns = reference/annotation + per-sample AF & depth. - // The all-positions VCFs say what a sample's reads show at a site it has no call at. - if (asBool(params.make_snp_matrix)) { - def depth_vcfs = allpos_vcf.map { sId, rId, vcf, tbi -> vcf }.collect().ifEmpty([]) - SNP_MATRIX(report_vcfs, depth_vcfs, ref_name, tsv_name) - } } diff --git a/tests/unit/test_qc_report_series.py b/tests/unit/test_qc_report_series.py new file mode 100644 index 0000000..9ecf96d --- /dev/null +++ b/tests/unit/test_qc_report_series.py @@ -0,0 +1,122 @@ +"""What each series gained since its first time point (bin/qcreport/series.py). + +A fixed SNP in a later sample is new only if the first time point was read at that site and did +not carry it. These tests pin that reading, the losses, and what happens without the depth the +SNP matrix provides. +""" + +from __future__ import annotations + +from qcreport import series as ser + + +def meta(**samples): + """{sample: (group, time)} -> the parse_metadata() shape.""" + return {s: {"group": g, "time": str(t), "tnum": float(t)} for s, (g, t) in samples.items()} + + +def var(af, gene="g", aa=""): + return {"af": af, "dp": 50, "gene": gene, "aa": aa, "eff": "missense_variant", "alt": "T"} + + +M = meta(p0=("line1", 0), p3=("line1", 3), p6=("line1", 6)) + + +def test_a_site_read_at_the_start_without_the_allele_is_new(): + variants = {"p0": {}, "p3": {"c:10": var(1.0, "atpE", "p.Ile66Met")}, "p6": {}} + cells = {("c:10", "p0"): (0.0, 40)} + + sec = ser.build_series(M, variants, cells) + + row = sec["groups"][0]["rows"][0] + assert (row["s"], row["new"], row["risen"], row["unknown"]) == ("p3", ["c:10"], [], 0) + assert sec["sites"]["c:10"]["gene"] == "atpE" and sec["checked"] is True + + +def test_a_site_the_start_did_not_read_is_unknown_not_new(): + variants = {"p0": {}, "p3": {"c:10": var(1.0)}, "p6": {}} + cells = {("c:10", "p0"): (None, 2)} + + row = ser.build_series(M, variants, cells, min_dp=7)["groups"][0]["rows"][0] + + assert (row["new"], row["unknown"]) == ([], 1) + + +def test_a_minority_at_the_start_that_became_fixed_has_risen(): + variants = {"p0": {"c:10": var(0.2)}, "p3": {"c:10": var(0.95)}, "p6": {}} + + row = ser.build_series(M, variants, {})["groups"][0]["rows"][0] + + assert (row["new"], row["risen"]) == ([], ["c:10"]) + + +def test_a_site_fixed_from_the_start_is_inherited_not_gained(): + variants = {"p0": {"c:10": var(1.0)}, "p3": {"c:10": var(1.0)}, "p6": {"c:10": var(1.0)}} + + rows = ser.build_series(M, variants, {})["groups"][0]["rows"] + + assert all(r["new"] == [] and r["risen"] == [] for r in rows) + + +def test_a_site_fixed_at_the_start_and_read_without_it_later_is_lost(): + variants = {"p0": {"c:10": var(1.0)}, "p3": {"c:10": var(1.0)}, "p6": {}} + cells = {("c:10", "p6"): (0.0, 55)} + + rows = {r["s"]: r for r in ser.build_series(M, variants, cells)["groups"][0]["rows"]} + + assert rows["p6"]["lost"] == ["c:10"] and rows["p3"]["lost"] == [] + + +def test_without_the_matrix_a_site_not_called_at_the_start_counts_as_new_and_says_so(): + variants = {"p0": {"c:10": var(1.0)}, "p3": {"c:20": var(1.0)}, "p6": {}} + + sec = ser.build_series(M, variants, {}, checked=False) + + rows = {r["s"]: r for r in sec["groups"][0]["rows"]} + assert rows["p3"]["new"] == ["c:20"] and sec["checked"] is False + assert rows["p6"]["lost"] == [], "a loss needs the depth to be told from a site not read" + + +def test_every_sample_of_the_first_time_point_is_the_start(): + m = meta(a=("L", 0), b=("L", 0), c=("L", 5)) + variants = {"a": {}, "b": {"c:10": var(1.0)}, "c": {"c:10": var(1.0)}} + + g = ser.build_series(m, variants, {})["groups"][0] + + assert g["base"] == ["a", "b"] and g["rows"][0]["new"] == [] + + +def test_a_group_with_a_single_time_point_is_not_a_series(): + m = meta(a=("L", 0), b=("L", 0)) + + assert ser.build_series(m, {"a": {}, "b": {}}, {}) is None + + +def test_the_series_carries_its_treatment(): + m = meta(a=("L", 0), b=("L", 3), c=("L", 6)) + sm = {"tx_field": "treatment", "rows": {"a": {"treatment": "CTRL"}, "b": {"treatment": "BDQ"}, + "c": {"treatment": "BDQ"}}} + + g = ser.build_series(m, {"a": {}, "b": {}, "c": {}}, {}, sample_meta=sm)["groups"][0] + + assert g["tx"] == "BDQ" + + +def test_needed_cells_asks_for_the_start_at_later_fixed_sites_and_back(): + variants = {"p0": {"c:5": var(1.0)}, "p3": {"c:10": var(1.0), "c:11": var(0.3)}, "p6": {}} + + need = ser.needed_cells(M, variants) + + assert need == {"c:10": {"p0"}, "c:5": {"p3", "p6"}} + + +def test_read_matrix_cells_reads_only_what_is_needed(tmp_path): + p = tmp_path / "m.tsv" + p.write_text("reference\tcontig\tpos\tref_allele\talt_allele\tgene\teffect\taa_change\t" + "p0|AF\tp0|DP\tp3|AF\tp3|DP\n" + "R\tc\t10\tA\tT\t\t\t\t0\t40\t1.0000\t38\n" + "R\tc\t20\tA\tT\t\t\t\t\t0\t1.0000\t30\n") + + cells = ser.read_matrix_cells(str(p), {"c:10": {"p0"}, "c:20": {"p0"}}) + + assert cells == {("c:10", "p0"): (0.0, 40), ("c:20", "p0"): (None, 0)} From 78798301a3ef5d6c15b4063ce002a02855560817 Mon Sep 17 00:00:00 2001 From: Paula Ruiz Rodriguez <50167687+Paururo@users.noreply.github.com> Date: Wed, 23 Sep 2026 14:38:33 +0200 Subject: [PATCH 04/14] Measure how low an allele frequency can be trusted A Minority variants panel counts each sample's calls below fixation, their spread of allele fractions and how many rest on three alternate reads or fewer, where a sequencing error and a real minority look the same. When the samplesheet names each sample's DNA extract (dna_id, extract, biosample...), libraries of the same DNA are compared: a real minority is in the DNA and another library calls it too, while an error is not reproduced. The share reproduced in each band of allele fraction says where the noise ends, against fixed calls as the ceiling. Pairs that share a FASTQ file are left out, since a merged sample shares the reads of the runs it was merged from and its agreement with them proves nothing, and so are libraries mapped against different references. A call counts as missed only where the other library was read, which the SNP matrix's depth says; without the matrix the comparison is not made rather than made biased. --- CHANGELOG.md | 9 ++ bin/qc_report.py | 14 ++- bin/qcreport/minority.py | 174 ++++++++++++++++++++++++++ bin/qcreport/render.py | 9 +- bin/report_assets/js/07b_summary.js | 18 ++- bin/report_assets/js/10c_minority.js | 42 +++++++ bin/report_assets/js/14_boot.js | 5 +- bin/report_assets/shell.html | 5 +- docs/qc-report.md | 5 +- tests/unit/test_qc_report_minority.py | 96 ++++++++++++++ 10 files changed, 369 insertions(+), 8 deletions(-) create mode 100644 bin/qcreport/minority.py create mode 100644 bin/report_assets/js/10c_minority.js create mode 100644 tests/unit/test_qc_report_minority.py diff --git a/CHANGELOG.md b/CHANGELOG.md index e278f49..8d358fe 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,15 @@ based on [Keep a Changelog](https://keepachangelog.com/), and the project follow ### Added +- **How low an allele frequency can be trusted.** A *Minority variants* panel counts each + sample's calls below fixation, their spread of allele fractions and how many rest on three + alternate reads or fewer, where an error and a real minority look the same. When the + samplesheet names each sample's DNA extract (a column such as `dna_id`, `extract` or + `biosample`), libraries of the same DNA are compared: a real minority is in the DNA and another + library calls it too, an error is not reproduced. Pairs that share a FASTQ file (a merged sample + and its runs) or were mapped against different references are left out, and a call only counts + as missed where the other library was read. + - **What each series gained since its first time point.** For every samplesheet group followed over time (a patient, a passage line), the report lists the SNPs each later sample carries fixed that the group's first time point did not: *new* where that first time point was read at diff --git a/bin/qc_report.py b/bin/qc_report.py index a68189a..17c46b5 100644 --- a/bin/qc_report.py +++ b/bin/qc_report.py @@ -36,6 +36,8 @@ parse_kraken, parse_lineage_colors, parse_metadata, parse_pnps, parse_profile, parse_sample_meta, parse_summary, parse_vcfs, to_float) from qcreport.relatedness import build_relatedness, group_mismatches, parse_pairs +from qcreport.minority import build_minority, replicate_pairs, replicate_sets +from qcreport.minority import needed_cells as replicate_cells from qcreport.series import build_series, needed_cells, read_matrix_cells from qcreport.render import REPO_URL, SECTION_INFO, build_html @@ -332,8 +334,17 @@ def build_payload(args, thr, anc_thr): # comparison needs, and without it a site not called at the start cannot be told from one not read. matrix_ok = bool(args.snp_matrix and os.path.exists(args.snp_matrix) and not os.path.basename(args.snp_matrix).startswith("NO_FILE")) - _cells = read_matrix_cells(args.snp_matrix, needed_cells(series_meta, _variants)) if matrix_ok else {} + # Libraries of the same DNA, for how far down a minority call reproduces. The matrix is read once + # for the cells both comparisons need. + rep_col, rep_val, rep_reads = replicate_sets(args.metadata) + rep_pairs, rep_shared = (replicate_pairs(rep_val, rep_reads, set(_variants), ref_of) + if rep_col else ([], 0)) + _need = needed_cells(series_meta, _variants) + for site, who in replicate_cells(rep_pairs, _variants).items(): + _need.setdefault(site, set()).update(who) + _cells = read_matrix_cells(args.snp_matrix, _need) if matrix_ok else {} _series = build_series(series_meta, _variants, _cells, _sample_meta, args.min_dp, checked=matrix_ok) + _minority = build_minority(_variants, rep_pairs, rep_shared, _cells, rep_col, args.min_dp, checked=matrix_ok) _dynamics = build_dynamics(series_meta, _variants, _sample_meta) # feeds dynamics + epistasis # front-load 'dose' so it survives the correlation matrix's top-N view dist_keys = (["dose"] + DIST) if dose_map else DIST @@ -360,6 +371,7 @@ def build_payload(args, thr, anc_thr): "coverage": coverage, "relatedness": build_relatedness(distances, groups, args.cluster_snps), "series": _series, + "minority": _minority, "sample_meta": _sample_meta, "metrics": [{"key": k, "label": l, "kind": kind, "dir": d} for k, l, kind, d in METRICS], "extra": extra_metrics, diff --git a/bin/qcreport/minority.py b/bin/qcreport/minority.py new file mode 100644 index 0000000..c16a745 --- /dev/null +++ b/bin/qcreport/minority.py @@ -0,0 +1,174 @@ +"""How low an allele frequency can be trusted: the variant calls below fixation. + +Every sample: how many calls sit below fixation (allele fraction under 0.9), their spread of +fractions, and how many of them rest on three alternate reads or fewer, where a sequencing error +and a real minority look the same. + +When the samplesheet says which samples are libraries of the same DNA (a column such as dna_id, +extract or biosample), the question has an empirical answer: a real minority variant is in the +DNA, so another library of it calls it too, while an error is not reproduced. The share of calls +one library makes that another library of the same DNA also makes, in each band of allele +fraction, is where the noise ends. Two samples that share a FASTQ file share reads, so their +agreement proves nothing and they are not compared; a merged sample and the runs it was merged +from are the usual case. A call is only counted as missing from the other library where that +library was read at the site (the SNP matrix's depth); a site it did not read says nothing. +""" +from __future__ import annotations + +import csv +import os +import re + +FIXED = 0.9 +EDGES = [0.0, 0.1, 0.2, 0.3, 0.5, 0.7, FIXED] +_REP_RE = re.compile(r"^(dna[_ ]?id|dna|extract(ion)?([_ ]?id)?|biosample([_ ]?id)?|specimen([_ ]?id)?|" + r"isolate[_ ]?id|library[_ ]?of|replicate[_ ]?of|tech(nical)?[_ ]?rep(licate)?([_ ]?of)?)$", re.I) +_SAMPLE_RE = re.compile(r"^(sample[_ ]?id|sample|id|name)$", re.I) + + +def _bin(af): + """The band of EDGES an allele fraction falls in, (lo, hi]; None at or above fixation.""" + for i in range(len(EDGES) - 1): + if EDGES[i] < af <= EDGES[i + 1]: + return i + return None + + +def replicate_sets(path): + """(column, {sample: value}, {sample: {fastq paths}}) from the samplesheet, or (None, {}, {}). + + The column is the first one named like a DNA or extract identifier. FASTQ paths come from + r1/r2 and are kept per sample, since a merged sample lists every run it was merged from. + """ + if not path or not os.path.exists(path): + return None, {}, {} + try: + with open(path, encoding="utf-8", errors="replace") as fh: + rows = list(csv.DictReader((line for line in fh if not line.startswith("#")), delimiter="\t")) + except OSError: + return None, {}, {} + if not rows: + return None, {}, {} + header = list(rows[0].keys()) + scol = next((h for h in header if h and _SAMPLE_RE.match(h.strip())), header[0]) + rcol = next((h for h in header if h and _REP_RE.match(h.strip())), None) + value, reads = {}, {} + for r in rows: + s = (r.get(scol) or "").strip() + if not s: + continue + for k in ("r1", "r2", "R1", "R2", "fastq_1", "fastq_2"): + v = (r.get(k) or "").strip() + if v: + reads.setdefault(s, set()).add(v) + if rcol and (r.get(rcol) or "").strip(): + value.setdefault(s, r[rcol].strip()) + return rcol, value, reads + + +def replicate_pairs(value, reads, samples, ref=None): + """([(a, b)], n left out for sharing reads): independent libraries of the same DNA. + + With `ref` ({sample: reference}), libraries of one DNA mapped against different references + are not paired either: their coordinates do not correspond, and the pairing is itself a + sign one of them went to the wrong genome. + """ + by = {} + for s, v in value.items(): + if s in samples: + by.setdefault(v, []).append(s) + pairs, shared = [], 0 + for members in by.values(): + members.sort() + for i, a in enumerate(members): + for b in members[i + 1:]: + if ref and ref.get(a) != ref.get(b): + continue + if reads.get(a, set()) & reads.get(b, set()): + shared += 1 + else: + pairs.append((a, b)) + return pairs, shared + + +def needed_cells(pairs, variants, fixed=FIXED): + """{site: {samples}}: each library at the sites the other of its pair called.""" + need = {} + for a, b in pairs: + for x, y in ((a, b), (b, a)): + for site, v in variants.get(x, {}).items(): + if v.get("af") and site not in variants.get(y, {}): + need.setdefault(site, set()).add(y) + return need + + +def build_minority(variants, pairs=(), shared=0, cells=None, column=None, min_dp=7, fixed=FIXED, + checked=True): + """The payload section, or None without variant calls. Without the SNP matrix (`checked` + False) nothing tells a call the other library missed from a site it did not read, so the + replicate comparison is left out rather than biased.""" + if not variants: + return None + samples, hist_all = {}, [0] * (len(EDGES) - 1) + low_all = total_all = 0 + for s, calls in variants.items(): + hist, alts, low = [0] * (len(EDGES) - 1), [], 0 + for v in calls.values(): + af = v.get("af") + if af is None or af <= 0 or af >= fixed: + continue + b = _bin(af) + if b is None: + continue + hist[b] += 1 + if v.get("dp"): + alt = round(af * v["dp"]) + alts.append(alt) + low += alt <= 3 + n = sum(hist) + for i, c in enumerate(hist): + hist_all[i] += c + low_all += low + total_all += len(alts) + alts.sort() + samples[s] = {"n": n, "hist": hist, "le3": low, "with_dp": len(alts), + "median_alt": alts[len(alts) // 2] if alts else None} + section = {"fixed": fixed, "edges": EDGES, "hist": hist_all, "samples": samples, + "le3": low_all, "with_dp": total_all, "replicates": None} + if column: + section["replicates"] = (_reproducibility(variants, pairs, shared, {} if cells is None else cells, + column, min_dp, fixed) + if checked else {"column": column, "pairs": len(pairs), "shared": shared, + "tested": None}) + return section + + +def _reproducibility(variants, pairs, shared, cells, column, min_dp, fixed): + """Per band of allele fraction, the calls of one library the other library of its pair + also makes, among those the other library was read at; and the same for fixed calls.""" + tested, repro = [0] * (len(EDGES) - 1), [0] * (len(EDGES) - 1) + fx_t = fx_r = 0 + for a, b in pairs: + for x, y in ((a, b), (b, a)): + vy = variants.get(y, {}) + for site, v in variants.get(x, {}).items(): + af = v.get("af") + if not af: + continue + if site in vy and (vy[site].get("af") or 0) > 0: + hit = True + else: + cell = cells.get((site, y)) + if cell is None or cell[1] is None or cell[1] < min_dp: + continue # the other library was not read there: nothing to learn + hit = bool(cell[0]) + if af >= fixed: + fx_t += 1 + fx_r += hit + continue + i = _bin(af) + if i is not None: + tested[i] += 1 + repro[i] += hit + return {"column": column, "pairs": len(pairs), "shared": shared, "tested": tested, + "reproduced": repro, "fixed_tested": fx_t, "fixed_reproduced": fx_r} diff --git a/bin/qcreport/render.py b/bin/qcreport/render.py index eb18c27..3406b6c 100644 --- a/bin/qcreport/render.py +++ b/bin/qcreport/render.py @@ -48,7 +48,7 @@ def _asset(name): "js/01_prelude.js", "js/02_qcspace.js", "js/03_state.js", "js/04_helpers.js", "js/05_insights_qc.js", "js/06_insights_genome.js", "js/07_render_core.js", "js/07b_summary.js", "js/08_curation.js", "js/09_genome_genes.js", "js/09b_coverage.js", "js/09c_relatedness.js", - "js/10_dynamics.js", "js/10b_series.js", + "js/10_dynamics.js", "js/10b_series.js", "js/10c_minority.js", "js/11_epistasis.js", "js/12_snpmatrix.js", "js/13_drug_kraken.js", "js/14_boot.js", ] @@ -116,6 +116,13 @@ def build_html(title, payload): "allele, which the SNP matrix's depth says; where it was not read the SNP is 'unknown' rather " "than new. 'Lost' is the other way round: fixed at the start, read and absent later. SNPs " "only, from the variant calls.", + "minority": "Variant calls below fixation (allele fraction under 0.9), the reads they rest on and " + "their spread of fractions. Three alternate reads or fewer is where a sequencing error " + "and a real minority look the same. When the samplesheet names libraries of the same DNA " + "(dna_id, extract, biosample...), each library's calls are looked up in the others: a real " + "minority is in the DNA and is reproduced, an error is not. Pairs that share a FASTQ file " + "share reads and are left out, and a call only counts as missed where the other library " + "was read (the SNP matrix's depth).", "drseries": "The resistance mutations (WHO grade 1-2, from pathotypr) at every time point of every " "series, with those acquired since the first time point set apart and those no longer " "called crossed out. Answers when resistance appears along a series and whether it " diff --git a/bin/report_assets/js/07b_summary.js b/bin/report_assets/js/07b_summary.js index f5d56b1..c02184c 100644 --- a/bin/report_assets/js/07b_summary.js +++ b/bin/report_assets/js/07b_summary.js @@ -213,6 +213,22 @@ function findRelatedness(){ note:'SNPs between consensus sequences; the threshold is a convention of the organism.', short:g.out.length?(g.out.length+' '+_plural(g.out.length,'sample sits','samples sit')+' outside '+_plural(g.out.length,'its group','their groups')+'.'):''}; } +// Minority variants: how many calls below fixation rest on a handful of reads, and how far down +// libraries of the same DNA reproduce them. +function findMinority(){ + var M=R.minority; if(!M)return null; + var tot=M.hist.reduce(function(a,b){return a+b;},0); if(!tot||!M.with_dp)return null; + var pct=Math.round(100*M.le3/M.with_dp),r=M.replicates,body=[]; + var head=''+pct+'% of the '+tot.toLocaleString('en-US')+' calls below fixation rest on three alternate reads or fewer'; + if(r&&r.tested){var low=r.tested[0]+r.tested[1],rl=r.reproduced[0]+r.reproduced[1],fl=minFloor(M); + body.push('Another library of the same DNA calls '+(low?Math.round(100*rl/low)+'%':'none')+' of those under allele fraction '+M.edges[2]+ + (r.fixed_tested?' and '+Math.round(100*r.fixed_reproduced/r.fixed_tested)+'% of the fixed ones':'')+'.'); + body.push(fl==null||fl<0?'No band below fixation reaches 80% reproduced.':(fl===0?'Every band is reproduced 80% or more.':'Calls reproduce 80% or more from allele fraction '+M.edges[fl]+' up.'));} + else if(r)body.push('Libraries of the same DNA are named ('+esc(r.column)+') but need the SNP matrix to be compared.'); + else body.push('Naming each sample's DNA extract in the samplesheet would measure where the noise ends.'); + return {k:'minor',eyebrow:'Minority variants',tone:'',head:head,body:body.join(' '),link:{href:'#minority',t:'See the calls'}, + note:'Below a few reads, an error and a real minority look the same.'}; +} function findCoverage(){ var S=R.samples; if(!S.length)return null; function med(k){return _median(S.map(function(s){return s.m[k];}));} @@ -222,7 +238,7 @@ function findCoverage(){ var body=(rd?'Depth ranges from '+fmt(rd[0],'float')+'× to '+fmt(rd[1],'float')+'×. ':'')+(c!=null?'The median consensus has '+c.toFixed(1)+'% of the genome as confident bases.':''); return {k:'cov',eyebrow:'Coverage',tone:'',head:head,body:body,link:{href:'#dist',t:'See the distributions'}}; } -function findings(){return [findQC(),findIdentity(),findRelatedness(),findResistance(),findDynamics(),findLineage(),findGconv(),findCoverage()].filter(function(f){return f;});} +function findings(){return [findQC(),findIdentity(),findRelatedness(),findResistance(),findDynamics(),findMinority(),findLineage(),findGconv(),findCoverage()].filter(function(f){return f;});} function findingCard(f){ var nums=(f.nums&&f.nums.length)?'
'+f.nums.map(function(x){return '
'+x.n+''+esc(x.l)+'
';}).join('')+'
':''; return '
'+esc(f.eyebrow)+'
'+ diff --git a/bin/report_assets/js/10c_minority.js b/bin/report_assets/js/10c_minority.js new file mode 100644 index 0000000..aed74ea --- /dev/null +++ b/bin/report_assets/js/10c_minority.js @@ -0,0 +1,42 @@ +// ---- Minority variants: how low an allele frequency can be trusted ---- +// Calls below fixation, the reads they rest on, and, where the samplesheet names libraries of the +// same DNA, how often another library of that DNA calls them too: a real minority is in the DNA, +// an error is not reproduced. +function minBandLabels(M){var e=M.edges;return e.slice(0,-1).map(function(lo,i){return i===0?'≤ '+e[1]:lo+'–'+e[i+1];});} +// The lowest band from which at least 80% of calls reproduce, over bands tested at least 20 times. +function minFloor(M){var r=M.replicates;if(!(r&&r.tested))return null; + for(var i=0;i=20&&r.reproduced[i]/r.tested[i]>=0.8)return i;}return -1;} +function minBars(vals,labels,fmt,color,title){var mx=Math.max.apply(null,vals.map(function(v){return v||0;}).concat([1e-9])); + return '
'+vals.map(function(v,i){return '
'+(v==null?'':fmt(v))+''+ + ''+labels[i]+'
';}).join('')+'
';} +function renderMinority(){ + var host=el('minority_body'),M=R.minority; if(!host)return; if(!M){host.innerHTML='';return;} + var labels=minBandLabels(M),tot=M.hist.reduce(function(a,b){return a+b;},0),n=Object.keys(M.samples).length; + var html='
'+tot.toLocaleString('en-US')+' variant calls below fixation (allele fraction under '+M.fixed+') across '+n+' samples; '+ + ''+(M.with_dp?Math.round(100*M.le3/M.with_dp):0)+'% of them rest on three alternate reads or fewer, where a sequencing error and a real minority look the same.
'+ + minBars(M.hist,labels,function(v){return v.toLocaleString('en-US');},'#4f83c2',function(i){return M.hist[i]+' calls at allele fraction '+labels[i].replace('≤','<=').replace('–','-');}); + var r=M.replicates; + if(r){ + if(r.tested==null){html+='
The samplesheet names libraries of the same DNA ('+esc(r.column)+', '+r.pairs+' independent '+_plural(r.pairs,'pair')+'), '+ + 'but without the SNP matrix a call another library missed cannot be told from a site it did not read, so they are not compared.
';} + else{ + var rates=r.tested.map(function(t,i){return t?r.reproduced[i]/t:null;}),fl=minFloor(M),fx=r.fixed_tested?r.fixed_reproduced/r.fixed_tested:null; + html+='
Reproduced by another library of the same DNA ('+ + esc(r.column)+'; '+r.pairs+' independent '+_plural(r.pairs,'pair')+(r.shared?(', '+r.shared+' left out for sharing reads'):'')+')
'+ + minBars(rates,labels,function(v){return Math.round(v*100)+'%';},function(i){return rates[i]!=null&&rates[i]>=0.8?'#22a06b':(rates[i]!=null&&rates[i]>=0.5?'#e6b25a':'#e0544f');}, + function(i){return r.reproduced[i]+' of '+r.tested[i]+' calls reproduced';})+ + '
'+(fl==null||fl<0?'No band of allele fraction reaches 80% reproduced'+(fx!=null?'; fixed calls reach '+Math.round(fx*100)+'%':'')+'.': + ((fl===0?'At every allele fraction, 80% or more of the calls are reproduced':'From allele fraction '+M.edges[fl]+' up, 80% or more of the calls are reproduced')+(fl>0?('; below it, '+Math.round(100*(rates.slice(0,fl).reduce(function(a,v,i){return a+(v||0)*r.tested[i];},0)/Math.max(1,r.tested.slice(0,fl).reduce(function(a,b){return a+b;},0))))+'% are'):'')+ + (fx!=null?'. Fixed calls: '+Math.round(fx*100)+'%, the ceiling':'')+'.'))+ + ' Only calls the other library was read at count; a site it did not read says nothing.
';}} + else html+='
Add a column naming each sample's DNA extract to the samplesheet (dna_id, extract, biosample...) and libraries of the same DNA are compared to measure where the noise ends.
'; + var rows=Object.keys(M.samples).map(function(s){var x=M.samples[s];return {s:s,x:x};}).sort(function(a,b){return b.x.n-a.x.n;}); + var big=el('minorityPanel')&&el('minorityPanel').classList.contains('expanded'),cap=big?rows.length:25; + html+='
'+ + labels.map(function(l){return '';}).join('')+''+ + rows.slice(0,cap).map(function(o){var x=o.x;return ''+ + ''+ + x.hist.map(function(c){return '';}).join('')+'';}).join('')+'
SampleCalls below fixationOn 3 reads or fewerMedian alternate reads'+l+'
'+esc(o.s)+''+x.n.toLocaleString('en-US')+''+(x.with_dp?Math.round(100*x.le3/x.with_dp)+'%':'NA')+''+(x.median_alt==null?'NA':x.median_alt)+''+(c||'')+'
'+ + (rows.length>cap?'
Showing '+cap+' of '+rows.length+' samples; open the panel full for the rest.
':''); + host.innerHTML=html; +} diff --git a/bin/report_assets/js/14_boot.js b/bin/report_assets/js/14_boot.js index 65a38d7..2d0a2e6 100644 --- a/bin/report_assets/js/14_boot.js +++ b/bin/report_assets/js/14_boot.js @@ -7,7 +7,7 @@ var PAGE_RENDER={ qc:function(){renderOverview();renderFlags();renderCuration();renderTable();renderLineages();renderKraken();renderPlots();renderADNA();}, genome:function(){renderStacks();renderGenome();renderDeletions();renderFunction();renderGeneBurden();renderHotspots();renderPnps();}, related:function(){renderRelatedness();}, - variants:function(){renderSeries();renderDynamics();renderEpistasis();renderSnpMatrix();renderVarDose();}, + variants:function(){renderSeries();renderMinority();renderDynamics();renderEpistasis();renderSnpMatrix();renderVarDose();}, drug:function(){renderDrug();renderDrSeries();}, gconv:function(){renderGconv();}, diag:function(){renderScatter();renderCorr();renderQCspace();renderRefBias();renderDoseTx();renderTemporal();}}; @@ -209,6 +209,7 @@ function genomeResetZoom(){st.gsel=null;st.gzoom=null;st.geneMark=null;var gg=el if(!R.samples.some(function(s){return s.miss||s.trk;})){var gs=el('genome');if(gs)gs.style.display='none';var ng=el('nav-genome');if(ng)ng.style.display='none';} if(!(R.series&&R.series.groups&&R.series.groups.length)){['gains','nav-gains','drseries','nav-drseries'].forEach(function(id){var x=el(id);if(x)x.style.display='none';});} else if(!(R.dr&&R.dr.calls&&R.dr.calls.length)){['drseries','nav-drseries'].forEach(function(id){var x=el(id);if(x)x.style.display='none';});} +if(!R.minority){['minority','nav-minority'].forEach(function(id){var x=el(id);if(x)x.style.display='none';});} if(!relSec()){['reldist','relclus','relgroup','nav-reldist','nav-relclus','nav-relgroup'].forEach(function(id){var x=el(id);if(x)x.style.display='none';});} if(!(R.coverage&&R.coverage.regions)){var dlx=el('deletions');if(dlx)dlx.style.display='none';var ndlx=el('nav-del');if(ndlx)ndlx.style.display='none';} if(!R.samples.some(function(s){return s.trk&&s.trk.snp;})){var hsx=el('hotspots');if(hsx)hsx.style.display='none';var nhx=el('nav-hot');if(nhx)nhx.style.display='none';} @@ -341,7 +342,7 @@ Array.prototype.forEach.call(document.querySelectorAll('.exp-h'),function(b){b.o var willExpand=!panel.classList.contains('expanded'); collapseExpanded(); if(willExpand){panel.classList.add('expanded');document.body.classList.add('has-expanded');b.innerHTML=icon('minimize')+'close';} var rn=b.getAttribute('data-render'); - setTimeout(function(){if(rn=='genome')renderGenome();else if(rn=='plots')renderPlots();else if(rn=='scatter')renderScatter();else if(rn=='corr')renderCorr();else if(rn=='pca')renderQCspace();else if(rn=='divcomp')renderRefBias();else if(rn=='function')renderFunction();else if(rn=='geneburden')renderGeneBurden();else if(rn=='hotspots')renderHotspots();else if(rn=='deletions')renderDeletions();else if(rn=='pnps')renderPnps();else if(rn=='dosetx')renderDoseTx();else if(rn=='vardose')renderVarDose();else if(rn=='gconv')renderGconv();else if(rn=='table')renderTable();},20);};}); + setTimeout(function(){if(rn=='genome')renderGenome();else if(rn=='plots')renderPlots();else if(rn=='scatter')renderScatter();else if(rn=='corr')renderCorr();else if(rn=='pca')renderQCspace();else if(rn=='divcomp')renderRefBias();else if(rn=='function')renderFunction();else if(rn=='geneburden')renderGeneBurden();else if(rn=='hotspots')renderHotspots();else if(rn=='deletions')renderDeletions();else if(rn=='minority')renderMinority();else if(rn=='pnps')renderPnps();else if(rn=='dosetx')renderDoseTx();else if(rn=='vardose')renderVarDose();else if(rn=='gconv')renderGconv();else if(rn=='table')renderTable();},20);};}); if(el('expClose'))el('expClose').onclick=collapseExpanded; document.addEventListener('keydown',function(e){if(e.key=='Escape')collapseExpanded();}); // per-sample detail modal close (button, backdrop, Esc) diff --git a/bin/report_assets/shell.html b/bin/report_assets/shell.html index 18a53fd..9689eb4 100644 --- a/bin/report_assets/shell.html +++ b/bin/report_assets/shell.html @@ -10,7 +10,7 @@
2Sample QC
Flagged samplesAll samplesLineagesContaminationDistributionsaDNA damage
3Genome & genes
ConsensusGenome landscapeDeletionsFunctional impactGene burdenVariable genesSelection (dN/dS)
4Relatedness
DistancesClustersOutside their group
-
5Variants over time
Gained per seriesSNP dynamicsCo-varying pairsSNP matrixVariant × dose
+
5Variants over time
Gained per seriesMinority variantsSNP dynamicsCo-varying pairsSNP matrixVariant × dose
6Resistance
Resistance mutationsAlong each series
7Gene conversion
Candidate tracts
8Diagnostics
Metric pairsCorrelation matrixQC spaceDivergenceDose × treatmentSampling dates
@@ -116,6 +116,9 @@
5 · Variants over time

How allele frequencies change along each series

Every variant followed through the time points of its series (a patient, a passage line), the pairs that move together, and the full site × sample matrix.

Gained along each series fixed SNPs each sample carries that the first time point of its series did not

+

Minority variants how low an allele frequency can be trusted: the calls below fixation and, with libraries of the same DNA, how often another library calls them too +

+

SNP dynamics allele-frequency trajectories per gene; the screen at the top ranks what is changing

Co-varying pairs variants whose trajectories move together (concordant) or against each other (discordant) within a series

diff --git a/docs/qc-report.md b/docs/qc-report.md index 0b434f8..791e1b4 100644 --- a/docs/qc-report.md +++ b/docs/qc-report.md @@ -30,7 +30,7 @@ on *Sample QC*, the samples with resistance mutations their lineage does not sha | **2 · Sample QC** | Which samples can be trusted: the verdicts, the live thresholds, the **flagged samples** with the value behind every flag, the **exclusion list** and its exports, the table of all samples, lineages, **contamination** (Kraken2), distributions and aDNA damage. | | **3 · Genome & genes** | Consensus completeness, the genome landscape (missing calls, **deletions**, SNPs per bin and **per callable kb**, het, indels), the **deletions** found by comparing each sample's stretches without reads with the rest of the cohort, functional impact (snpEff), gene burden, variable genes and dN/dS. | | **4 · Relatedness** | SNP distances between the consensus sequences, over the positions both samples called: a heatmap in the order that keeps each cluster together, the **clusters** at a threshold you can move, and the samples far from the rest of their own group (patient, line, series). | -| **5 · Variants over time** | What each series **gained since its first time point** (new, risen, unknown, lost, charted per treatment), SNP dynamics, co-varying pairs (epistasis), the SNP matrix and variant × dose. | +| **5 · Variants over time** | What each series **gained since its first time point** (new, risen, unknown, lost, charted per treatment), the **minority variants** (calls below fixation, the reads they rest on and, with libraries of the same DNA, how often another library reproduces them), SNP dynamics, co-varying pairs (epistasis), the SNP matrix and variant × dose. | | **6 · Resistance** | WHO-catalogue mutations grouped by mutation, the sample × drug matrix, every call, and each series' mutations **per time point**, those acquired since the first one set apart. | | **7 · Gene conversion** | Candidate tracts and the evidence behind each verdict. | | **8 · Diagnostics** | Metric pairs, the correlation matrix, QC space, divergence vs completeness, dose × treatment and sampling dates: views for digging into a problem, none of which flags a sample. | @@ -126,7 +126,8 @@ there is nothing to configure - add a column and the matching panel reacts. | :--- | :--- | :--- | | **sample id** | `sample`, `sample_id`, `name`, `gid`, `strain`, `isolate`… | Keys the metadata to each sample's VCF / stats (falls back to the first column). | | **time** | `timepoint`, `day`, `date`, `week`, `month`, `hour`, `passage`, `generation`, `visit`, `tp`, `t0`… | The x-axis of the **SNP dynamics** trajectories. | -| **group / series** | `group`, `patient`, `series`, `host`, `subject`, `cluster`, `experiment`, `donor`, `case`, `replicate`, `chain`, `samples`… | Connects samples into one longitudinal series (a trajectory set per group) for **SNP dynamics** and **epistasis**. | +| **group / series** | `group`, `patient`, `series`, `host`, `subject`, `cluster`, `experiment`, `donor`, `case`, `replicate`, `chain`, `samples`… | Connects samples into one longitudinal series (a trajectory set per group) for **SNP dynamics** and **epistasis**, and for what each series **gained since its first time point**. On *Relatedness*, a sample far from the rest of its group is flagged `GROUP_MISMATCH`. | +| **DNA extract** | `dna_id`, `dna`, `extract`, `extract_id`, `biosample`, `specimen_id`, `isolate_id`, `replicate_of`… | Names the libraries of one DNA: the **Minority variants** panel compares them to measure how low an allele frequency is reproduced. Samples sharing a FASTQ file are not compared. | | **any other column** | `treatment`, `site`, `region`, `ward`, `batch`… | A categorical annotation - becomes a **cohort filter** dropdown in the toolbar (restrict the whole QC view to one value) and a SNP-matrix header level. | | **collection date** | `collection_date`, `sampling_date`, `isolation_date`, `date`, `year`, `fecha` | The *Sampling dates* panel (Diagnostics) and the date in each sample's profile. The day the pipeline processed a sample is never read as one. | | **dose** | `dose`, `dosis` | A **numeric** column - becomes a first-class metric (selectable on the scatter axes + the correlation matrix, with a Spearman *r* + *p* read-out) and drives the **Dose × treatment** test. | diff --git a/tests/unit/test_qc_report_minority.py b/tests/unit/test_qc_report_minority.py new file mode 100644 index 0000000..e994fa4 --- /dev/null +++ b/tests/unit/test_qc_report_minority.py @@ -0,0 +1,96 @@ +"""How low an allele frequency can be trusted (bin/qcreport/minority.py). + +Calls below fixation are counted per sample with the reads they rest on, and, where the +samplesheet names libraries of the same DNA, measured against each other: a real minority is in +the DNA and another library calls it too; an error is not reproduced. +""" + +from __future__ import annotations + +from qcreport import minority as mn + + +def call(af, dp=100): + return {"af": af, "dp": dp} + + +def test_calls_below_fixation_are_counted_with_the_reads_they_rest_on(): + variants = {"S": {"c:1": call(0.05, 60), "c:2": call(0.15, 100), "c:3": call(0.5, 40), + "c:4": call(1.0, 50)}} + + sec = mn.build_minority(variants) + + s = sec["samples"]["S"] + assert s["n"] == 3, "the fixed call is not a minority" + assert s["hist"] == [1, 1, 0, 1, 0, 0] + assert s["le3"] == 1, "0.05 x 60 is 3 alternate reads" + assert s["median_alt"] == 15 + + +def _sheet(tmp_path, rows): + p = tmp_path / "samples.tsv" + p.write_text("sampleId\trunId\tr1\tr2\tdna_id\n" + "".join("\t".join(r) + "\n" for r in rows)) + return str(p) + + +def test_libraries_of_one_dna_are_found_and_those_sharing_reads_left_out(tmp_path): + """A merged sample lists the runs it was merged from: it shares their reads, so its + agreement with them proves nothing.""" + sheet = _sheet(tmp_path, [ + ["A", "r1", "a_1.fq", "a_2.fq", "DE1"], + ["B", "r1", "b_1.fq", "b_2.fq", "DE1"], + ["M", "r1", "a_1.fq", "a_2.fq", "DE1"], + ["M", "r2", "b_1.fq", "b_2.fq", "DE1"], + ["C", "r1", "c_1.fq", "c_2.fq", "DE2"], + ]) + + col, value, reads = mn.replicate_sets(sheet) + pairs, shared = mn.replicate_pairs(value, reads, {"A", "B", "M", "C"}) + + assert col == "dna_id" + assert reads["M"] == {"a_1.fq", "a_2.fq", "b_1.fq", "b_2.fq"} + assert pairs == [("A", "B")] and shared == 2 + + +def test_libraries_of_one_dna_on_different_references_are_not_paired(): + """Their coordinates do not correspond; one of them went to the wrong genome.""" + value = {"A": "DE1", "B": "DE1", "C": "DE1"} + reads = {"A": {"a.fq"}, "B": {"b.fq"}, "C": {"c.fq"}} + + pairs, _ = mn.replicate_pairs(value, reads, {"A", "B", "C"}, ref={"A": "R1", "B": "R1", "C": "R2"}) + + assert pairs == [("A", "B")] + + +def test_a_samplesheet_without_a_dna_column_has_no_replicates(tmp_path): + p = tmp_path / "s.tsv" + p.write_text("sampleId\tr1\nA\ta.fq\n") + + assert mn.replicate_sets(str(p))[0] is None + + +def test_reproducibility_counts_only_where_the_other_library_was_read(): + variants = {"A": {"c:1": call(0.05), "c:2": call(0.06), "c:3": call(0.4), "c:9": call(1.0)}, + "B": {"c:3": call(0.35), "c:9": call(1.0)}} + cells = {("c:1", "B"): (0.0, 80), # read, and not there: an error + ("c:2", "B"): (None, 2)} # not read: says nothing + + rep = mn.build_minority(variants, [("A", "B")], 0, cells, "dna_id", min_dp=7)["replicates"] + + assert rep["tested"][0] == 1 and rep["reproduced"][0] == 0 + assert rep["tested"][3] == 2 and rep["reproduced"][3] == 2, "c:3 in both directions" + assert (rep["fixed_tested"], rep["fixed_reproduced"]) == (2, 2) + + +def test_without_the_matrix_the_replicates_are_not_measured(): + variants = {"A": {"c:1": call(0.05)}, "B": {}} + + rep = mn.build_minority(variants, [("A", "B")], 0, {}, "dna_id", checked=False)["replicates"] + + assert rep["tested"] is None and rep["pairs"] == 1 + + +def test_needed_cells_asks_each_library_about_the_other_ones_calls(): + variants = {"A": {"c:1": call(0.05), "c:3": call(0.4)}, "B": {"c:3": call(0.4), "c:5": call(0.1)}} + + assert mn.needed_cells([("A", "B")], variants) == {"c:1": {"B"}, "c:5": {"A"}} From a497949ce6234668cdd3a274f752b3d6f1384ff6 Mon Sep 17 00:00:00 2001 From: Paula Ruiz Rodriguez <50167687+Paururo@users.noreply.github.com> Date: Wed, 23 Sep 2026 14:39:06 +0200 Subject: [PATCH 05/14] Check that --make_snp_distances false turns the distances off The stub run already checks that each optional cohort output stops when its flag is false; the distances step joins the list. --- tests/pipeline/test_stub_run.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/pipeline/test_stub_run.py b/tests/pipeline/test_stub_run.py index 462680a..3cf7d00 100644 --- a/tests/pipeline/test_stub_run.py +++ b/tests/pipeline/test_stub_run.py @@ -167,6 +167,7 @@ def test_gene_conversion_refuses_to_run_without_the_masking_it_reads_from( @pytest.mark.parametrize("flag,process", [ ("--make_qc_report", "QC_REPORT"), ("--make_snp_matrix", "SNP_MATRIX"), + ("--make_snp_distances", "SNP_DISTANCES"), ("--make_consensus", "CONSENSUS_FASTA"), ("--annotate_main_vcf", "ANNOTATE_MAIN_VCF"), ]) From 27d92b8d5a497e636e4a95ba78f9cd50c1e7f86f Mon Sep 17 00:00:00 2001 From: Paula Ruiz Rodriguez <50167687+Paururo@users.noreply.github.com> Date: Wed, 23 Sep 2026 14:40:10 +0200 Subject: [PATCH 06/14] Say on the summary how many deletions the run has The coverage card now counts the stretches some samples have no reads for while others do, private and shared, and those nobody reads. The relatedness card says how many samples belong to a cluster in plainer words. --- bin/report_assets/js/07b_summary.js | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/bin/report_assets/js/07b_summary.js b/bin/report_assets/js/07b_summary.js index c02184c..9ea4118 100644 --- a/bin/report_assets/js/07b_summary.js +++ b/bin/report_assets/js/07b_summary.js @@ -205,7 +205,7 @@ function findRelatedness(){ var g=relSummary(); if(!g)return null; var head=''+g.nCl+' '+_plural(g.nCl,'cluster')+' of samples within '+g.thr+' SNPs of each other'; var body=[]; - if(g.nCl)body.push(g.inCl+' samples sit in one, the largest holding '+g.big+'.'); + if(g.nCl)body.push(g.inCl+' samples belong to one; the largest holds '+g.big+'.'); if(g.out.length)body.push(''+g.out.length+' '+_plural(g.out.length,'sample sits','samples sit')+' far from the rest of '+_plural(g.out.length,'its group','their groups')+' ('+ _list(g.out.map(function(s){return esc(s.s);}),4)+'): swapped, mislabelled, contaminated or reinfected.'); return {k:'rel',eyebrow:'Relatedness',tone:g.out.length?'warn':'',head:head,body:body.join(' '), @@ -236,6 +236,10 @@ function findCoverage(){ var head='Median depth '+(d!=null?fmt(d,'float')+'×':'NA')+''+(b!=null?', breadth '+b.toFixed(1)+'%':''); var rd=_range(S.map(function(s){return s.m.mean_depth;})); var body=(rd?'Depth ranges from '+fmt(rd[0],'float')+'× to '+fmt(rd[1],'float')+'×. ':'')+(c!=null?'The median consensus has '+c.toFixed(1)+'% of the genome as confident bases.':''); + var C=R.coverage; + if(C&&C.regions){var k={private:0,shared:0,cohort:0};C.regions.forEach(function(r){if(r.cls in k)k[r.cls]++;}); + body+=' '+(k.private+k.shared)+' '+_plural(k.private+k.shared,'stretch','stretches')+' of '+C.min_len+' bp or more some samples have no reads for and others do ('+k.private+' private to one sample, '+k.shared+' shared); '+ + k.cohort+' no sample reads.';} return {k:'cov',eyebrow:'Coverage',tone:'',head:head,body:body,link:{href:'#dist',t:'See the distributions'}}; } function findings(){return [findQC(),findIdentity(),findRelatedness(),findResistance(),findDynamics(),findMinority(),findLineage(),findGconv(),findCoverage()].filter(function(f){return f;});} From 9c971c06cd8359310773bf2f506632bd353e0661 Mon Sep 17 00:00:00 2001 From: Paula Ruiz Rodriguez <50167687+Paururo@users.noreply.github.com> Date: Wed, 23 Sep 2026 15:39:34 +0200 Subject: [PATCH 07/14] Count a position as callable where the consensus can call a base The depth profile counted a position as callable above --consensus_min_dp (7), but the consensus writes a base only from --allpos_min_cov reads (30) and N in between, so SNPs per callable kb read low. Callable now means --allpos_min_cov reads or more, which the depth profile is given. --- bin/depth_profile.py | 33 ++++++++++++++++++-------------- bin/qcreport/render.py | 4 ++-- modules/report.nf | 2 +- tests/unit/test_depth_profile.py | 8 ++++---- 4 files changed, 26 insertions(+), 21 deletions(-) diff --git a/bin/depth_profile.py b/bin/depth_profile.py index c4fd42a..e99c276 100644 --- a/bin/depth_profile.py +++ b/bin/depth_profile.py @@ -5,8 +5,8 @@ whole genome. This reduces it to three small tables for the cohort report: .depth_windows.tsv one row per window: mean depth, the share of positions no read - covers, and the share deep enough for the consensus to call - (depth above --min-dp) + covers, and the share deep enough for the consensus to call a + base (depth of --callable-dp or more) .zero_depth.tsv every stretch of at least --min-run positions no read covers .gene_depth.tsv per gene of the GFF: the share of its positions read at all, the share the consensus can call, its mean depth, and that depth @@ -105,7 +105,7 @@ def arrays(contig, need): return sample, {c: depth[c] for c in order}, {c: known[c] for c in order} -def windows(depth, known, size, min_dp): +def windows(depth, known, size, callable_dp): """[(contig, start, end, mean depth, zero fraction, callable fraction)], 1-based inclusive. Fractions are of the positions with a record; a window with none has no values. @@ -122,7 +122,7 @@ def windows(depth, known, size, min_dp): continue dk = dm[km] out.append((contig, s + 1, e, float(dk.mean()), float((dk == 0).mean()), - float((dk > min_dp).mean()))) + float((dk >= callable_dp).mean()))) return out @@ -169,11 +169,12 @@ def read_genes(path): return sorted(genes or cds, key=lambda g: (g[0], g[1])) -def gene_depths(depth, known, genes, min_dp, median): +def gene_depths(depth, known, genes, callable_dp, median): """[(contig, start, end, name, locus_tag, length, breadth, callable, mean depth, relative)]. - breadth is the share of the gene's read positions at depth 1 or more, callable the share - above `min_dp`, relative the mean depth against the genome-wide median (None at median 0). + breadth is the share of the gene's read positions at depth 1 or more, callable the share at + `callable_dp` or more, relative the mean depth against the genome-wide median (None at + median 0). """ out = [] for contig, start, end, name, locus in genes: @@ -190,7 +191,7 @@ def gene_depths(depth, known, genes, min_dp, median): continue mean = float(dk.mean()) out.append((contig, start, end, name, locus, end - start + 1, float((dk > 0).mean()), - float((dk > min_dp).mean()), mean, (mean / median) if median else None)) + float((dk >= callable_dp).mean()), mean, (mean / median) if median else None)) return out @@ -204,8 +205,12 @@ def main(argv=None): ap.add_argument('--gff', default=None, help="the reference's GFF3, for the per-gene table") ap.add_argument('--reference', default='', help="the reference id, written into each table") ap.add_argument('--window', type=int, default=1000, help="window size in bp") - ap.add_argument('--min-dp', type=int, default=7, - help="depth at or below which the consensus calls nothing (--consensus_min_dp)") + # The consensus writes a reference base only where the backbone calls a confident reference, + # which it does from --allpos_min_cov reads (30); between --consensus_min_dp and that it + # writes N. A position read at 20x is not callable, and counting it so would make every + # SNP-per-callable-kb density read low. + ap.add_argument('--callable-dp', type=int, default=30, + help="depth from which the consensus can call a base (--allpos_min_cov)") ap.add_argument('--min-run', type=int, default=50, help="shortest stretch without reads that is written out") ap.add_argument('--out-prefix', required=True) @@ -221,13 +226,13 @@ def main(argv=None): head = [f"# sample={sample}", f"# reference={a.reference}", f"# genome_len={int(read.size)}", f"# median_dp={median:g}", f"# mean_dp={mean:.2f}", f"# zero_frac={float((read == 0).mean()) if read.size else 0:.4f}", - f"# callable_frac={float((read > a.min_dp).mean()) if read.size else 0:.4f}", - f"# min_dp={a.min_dp}", f"# window={a.window}", f"# min_run={a.min_run}", + f"# callable_frac={float((read >= a.callable_dp).mean()) if read.size else 0:.4f}", + f"# callable_dp={a.callable_dp}", f"# window={a.window}", f"# min_run={a.min_run}", "# contigs=" + ",".join(f"{c}:{len(depth[c])}" for c in depth)] with open(f"{a.out_prefix}.depth_windows.tsv", 'w') as fh: fh.write("\n".join(head) + "\ncontig\tstart\tend\tmean_dp\tzero_frac\tcallable_frac\n") - for row in windows(depth, known, a.window, a.min_dp): + for row in windows(depth, known, a.window, a.callable_dp): fh.write("\t".join(_fmt(v, 2 if i == 3 else 4) for i, v in enumerate(row)) + "\n") runs = zero_runs(depth, known, a.min_run) with open(f"{a.out_prefix}.zero_depth.tsv", 'w') as fh: @@ -239,7 +244,7 @@ def main(argv=None): with open(f"{a.out_prefix}.gene_depth.tsv", 'w') as fh: fh.write("\n".join(head) + "\ncontig\tstart\tend\tgene\tlocus_tag\tlength\tbreadth\t" "callable\tmean_dp\trel_depth\n") - for row in gene_depths(depth, known, genes, a.min_dp, median): + for row in gene_depths(depth, known, genes, a.callable_dp, median): fh.write("\t".join(_fmt(v, 2 if i == 8 else 4) for i, v in enumerate(row)) + "\n") sys.stderr.write(f"[depth_profile] {sample}: {int(read.size)} positions, median depth {median:g}, " f"{len(runs)} stretch(es) of {a.min_run}+ bp without reads" diff --git a/bin/qcreport/render.py b/bin/qcreport/render.py index 3406b6c..7f40093 100644 --- a/bin/qcreport/render.py +++ b/bin/qcreport/render.py @@ -108,8 +108,8 @@ def build_html(title, payload): "sample. Brush a region to list the genes under it. Reveals systematically low-callability " "regions (repeats, deletions) shared across samples. 'Deletions' shows the share of each bin " "in a stretch the sample has no reads for while other samples read it; 'SNPs / kb' divides " - "the SNPs of a bin by the positions deep enough to call, so a half-read bin does not look " - "half as variable.", + "the SNPs of a bin by the positions deep enough for the consensus to call a base " + "(--allpos_min_cov reads), so a half-read bin does not look half as variable.", "gains": "For every group of the samplesheet followed over time (a patient, a passage line), the SNPs " "each later sample carries fixed (allele fraction 0.9 or more) that the group's first time " "point did not. 'New' needs the first time point to have been read at the site without the " diff --git a/modules/report.nf b/modules/report.nf index eda57e6..2a9124d 100644 --- a/modules/report.nf +++ b/modules/report.nf @@ -213,7 +213,7 @@ process DEPTH_PROFILE { --gff ${gff} \\ --reference ${refId} \\ --window ${params.depth_window} \\ - --min-dp ${params.consensus_min_dp} \\ + --callable-dp ${params.allpos_min_cov} \\ --out-prefix ${sampleId}.${refId} """ diff --git a/tests/unit/test_depth_profile.py b/tests/unit/test_depth_profile.py index 92a9594..aec74cd 100644 --- a/tests/unit/test_depth_profile.py +++ b/tests/unit/test_depth_profile.py @@ -87,13 +87,13 @@ def _arrays(values): def test_windows_measure_only_the_positions_with_a_record(): depth, known = _arrays([10, 0, None, 20] + [8] * 4) - rows = dp.windows(depth, known, 4, min_dp=7) + rows = dp.windows(depth, known, 4, callable_dp=10) assert rows[0][:3] == ("chr", 1, 4) assert rows[0][3] == pytest.approx(10.0) # (10 + 0 + 20) / 3 assert rows[0][4] == pytest.approx(1 / 3) - assert rows[0][5] == pytest.approx(2 / 3) # 10 and 20 are above 7; 0 is not - assert rows[1][5] == pytest.approx(1.0) + assert rows[0][5] == pytest.approx(2 / 3) # 10 and 20 reach 10; 0 does not + assert rows[1][5] == pytest.approx(0.0), "8x is under the depth the consensus calls at" def test_a_window_without_records_has_no_values(): @@ -149,7 +149,7 @@ def test_read_genes_without_a_gff_is_empty(tmp_path, name): def test_gene_depths_measure_breadth_callable_depth_and_depth_against_the_median(): depth, known = _arrays([0, 0, 10, 30, 20, 20]) - row = dp.gene_depths(depth, known, [("chr", 1, 4, "g", "")], min_dp=7, median=20.0)[0] + row = dp.gene_depths(depth, known, [("chr", 1, 4, "g", "")], callable_dp=10, median=20.0)[0] assert row[5] == 4 assert row[6] == pytest.approx(0.5) # two of four positions read at all From bb5c2ff6ab16cea91f5385c4c7debbd9bc085b99 Mon Sep 17 00:00:00 2001 From: Paula Ruiz Rodriguez <50167687+Paururo@users.noreply.github.com> Date: Wed, 23 Sep 2026 15:40:16 +0200 Subject: [PATCH 08/14] Find deletions by reciprocal overlap, per sample and reference Stretches without reads were merged into a region whenever they overlapped at all, and the region was classed as a whole. A sample's 6 kb private deletion that overlapped a gap nine of ten samples share was swallowed into a "nobody reads it" region; ten neighbouring 700 bp deletions, none lacked by more than two samples, chained into one; and a carrier's two separate stretches were reported, and drawn, as one span across the region. Two samples' stretches are now one deletion when each covers at least half of the other, measured against the stretch that opened the region, and every carrier keeps its own. Profiles are keyed on the sample and its reference, so a sample mapped against two references is no longer a mix of both, and they are read into little memory: the windows reduced to the landscape's bins as they are read, gene breadths kept as one array per sample. A thousand samples used to take about 3 GB of the report's fixed 4 GB, which now also grows on retry. Naming a region's genes looks only at the genes each carrier barely reads. The depth tables and the SNP distances are produced whether or not a report is made. The class nearly every sample lacks is labelled as such rather than "no sample reads it", a sample alone on its reference no longer lights up the deletion track, clicking a stretch offers to reset the zoom and opens a nearly-unread one on the Missing track, and closing the full view redraws these panels. --- bin/qc_report.py | 20 +- bin/qcreport/coverage.py | 316 ++++++++++++++---------- bin/qcreport/render.py | 11 +- bin/report_assets/js/09_genome_genes.js | 2 +- bin/report_assets/js/09b_coverage.js | 14 +- bin/report_assets/js/14_boot.js | 2 +- docs/outputs.md | 14 +- modules/report.nf | 4 +- subworkflows/cohort_report.nf | 39 +-- tests/unit/test_qc_report_coverage.py | 187 ++++++++++---- 10 files changed, 390 insertions(+), 219 deletions(-) diff --git a/bin/qc_report.py b/bin/qc_report.py index 17c46b5..0b07556 100644 --- a/bin/qc_report.py +++ b/bin/qc_report.py @@ -272,16 +272,26 @@ def build_payload(args, thr, anc_thr): groups = {s: md.get("group") for s, md in series_meta.items()} distances = parse_pairs(args.snp_distances) outside = group_mismatches(distances, groups, args.cluster_snps) - profiles = parse_depth_profiles(args.depth_profiles) - coverage, del_tracks = (build_coverage(profiles, min_len=args.deletion_min_len, + profiles, gene_lists = parse_depth_profiles(args.depth_profiles, min_len=args.deletion_min_len) + coverage, del_tracks = (build_coverage(profiles, gene_lists, min_len=args.deletion_min_len, min_depth=args.deletion_min_depth)[:2] if profiles else (None, {})) + # A sample mapped against two references has a profile for each; its row in the report is the + # one of the reference the samplesheet gives it first. + profile_keys = {} + for key in profiles: + profile_keys.setdefault(key[0], []).append(key) + + def profile_of(sid): + keys = profile_keys.get(sid, []) + return (sid, ref_of.get(sid)) if (sid, ref_of.get(sid)) in profiles else (keys[0] if len(keys) == 1 else None) jsamples, counts = [], {"PASS": 0, "WARN": 0, "FAIL": 0} for sid, m in summ.items(): anc = is_ancient(m) dmg = dmg_by_sample.get(sid) snp_prof = parse_profile(m.get("snp_profile")) + pkey = profile_of(sid) verdict, flags = flag_sample(m, thr, snp_med, snp_sig, ancient=anc, anc_thr=anc_thr, dmg=dmg, ref_lineage=ref_major.get(ref_of.get(sid))) if sid in outside: @@ -308,11 +318,11 @@ def build_payload(args, thr, anc_thr): # nearest of another, with their distances "grpd": outside.get(sid), "trk": ({k: v for k, v in (("snp", snp_prof), - ("snpkb", snp_per_kb(snp_prof, profiles[sid]) - if sid in profiles else None), + ("snpkb", snp_per_kb(snp_prof, profiles[pkey]) + if pkey else None), ("het", parse_profile(m.get("het_profile"))), ("indel", parse_profile(m.get("indel_profile"))), - ("del", del_tracks.get(sid))) if v is not None} + ("del", del_tracks.get(pkey))) if v is not None} or None), "ann_db_error": (clean_str(m.get("ann_db_error")) == "yes"), "m": dict({k: to_float(m.get(k)) for k in metric_keys + extra_keys + EFF_KEYS}, diff --git a/bin/qcreport/coverage.py b/bin/qcreport/coverage.py index 44afb30..80b12ba 100644 --- a/bin/qcreport/coverage.py +++ b/bin/qcreport/coverage.py @@ -3,21 +3,30 @@ bin/depth_profile.py reduces each sample's all-positions VCF to windows, the stretches no read covers, and a per-gene table. What those stretches mean needs the cohort: -- a stretch no sample reads is a part of the reference none of these genomes has, or one no - read can be placed on (a repeat). It is not a deletion of any sample; +- a stretch almost no sample reads is a part of the reference none of these genomes has, or one + no read can be placed on (a repeat). It is not a deletion of any sample; - a stretch some samples read and others do not is a deletion in the others, shared by a lineage when several carry it and private to one sample when only it does; - in a sample read too thinly, stretches without reads turn up by chance, so its stretches are not assessed at all rather than reported as deletions. +Stretches of different samples are the same deletion when each covers at least half of the +other (reciprocal overlap), measured against the stretch that opened the region: overlap alone +would chain a run of neighbouring small deletions into one, and let one long private deletion +swallow a short gap every sample shares. + The windows also give, per bin of the genome landscape, how much of each sample could be called, which is what turns a count of SNPs per bin into a density: a bin half of which was not read holds half the SNPs whatever the genome looks like there. + +A sample mapped against two references has a profile for each, so profiles are keyed on the +sample and the reference together. """ from __future__ import annotations import math import os +from array import array from .parsers import NBINS, to_float @@ -34,59 +43,79 @@ def _kind(header): return None -def parse_depth_profiles(paths): - """{sample: profile} from depth_profile.py's tables, any of the three per sample. +def _read_table(path): + """(meta, header, rows as lists) of one depth_profile.py table, or None.""" + meta, header, rows = {}, None, [] + try: + with open(path, encoding="utf-8", errors="replace") as fh: + for line in fh: + line = line.rstrip("\n") + if line.startswith("#"): + k, sep, v = line[1:].strip().partition("=") + if sep: + meta[k.strip()] = v.strip() + continue + if header is None: + header = line.split("\t") + continue + if line: + rows.append(line.split("\t")) + except OSError: + return None + return meta, header or [], rows + + +def parse_depth_profiles(paths, min_len=0, nbins=NBINS): + """(profiles, gene lists) from depth_profile.py's tables, any of the three per sample. - The table is recognised by its columns and the sample by its '# sample=' line, so neither - depends on a file name. A profile holds the header values (reference, median_dp, contigs...) - and the rows of whichever tables were given. + profiles maps (sample, reference) to what the report needs and no more, so a cohort of a + thousand samples fits: the header values, callable positions per landscape bin (the windows + are not kept), the stretches without reads of at least `min_len` bp, and the breadth of each + gene as a compact array aligned with its reference's gene list (gene lists maps the reference + to [(contig, start, end, gene, locus_tag)], the same for every sample on it). """ - out = {} + profiles, gene_lists = {}, {} for path in paths or []: if not path or not os.path.exists(path) or os.path.basename(path).startswith("NO_FILE"): continue - meta, header, rows = {}, None, [] - try: - with open(path, encoding="utf-8", errors="replace") as fh: - for line in fh: - line = line.rstrip("\n") - if line.startswith("#"): - k, sep, v = line[1:].strip().partition("=") - if sep: - meta[k.strip()] = v.strip() - continue - if header is None: - header = line.split("\t") - continue - if line: - rows.append(dict(zip(header, line.split("\t")))) - except OSError: + table = _read_table(path) + if table is None: continue - sample = meta.get("sample") - kind = _kind(header or []) + meta, header, rows = table + sample, kind = meta.get("sample"), _kind(header) if not sample or not kind: continue - p = out.setdefault(sample, {"windows": [], "zero": [], "genes": []}) - for k in ("reference", "median_dp", "mean_dp", "genome_len", "zero_frac", "callable_frac", - "min_dp", "window", "min_run", "contigs"): - if k in meta and k not in p: - p[k] = meta[k] + ref = meta.get("reference") or "" + p = profiles.setdefault((sample, ref), {"sample": sample, "reference": ref, "median_dp": 0.0, + "contigs": [], "callable": None, "zero": [], "genes": None}) + if not p["contigs"]: + p["contigs"] = _contigs(meta.get("contigs")) + p["median_dp"] = to_float(meta.get("median_dp")) or 0.0 + col = {h: i for i, h in enumerate(header)} + + def cell(r, name): + i = col.get(name) + return r[i] if i is not None and i < len(r) else "" + + rows = [r for r in rows if cell(r, "start").isdigit() and cell(r, "end").isdigit()] if kind == "windows": - p["windows"] = [(r["contig"], int(r["start"]), int(r["end"]), to_float(r.get("mean_dp")), - to_float(r.get("zero_frac")), to_float(r.get("callable_frac"))) - for r in rows if r.get("start", "").isdigit() and r.get("end", "").isdigit()] + spans = [(cell(r, "contig"), int(cell(r, "start")), int(cell(r, "end")), + to_float(cell(r, "callable_frac"))) for r in rows] + spans = [s for s in spans if s[3] is not None] + p["callable"] = _spread(spans, p["contigs"], nbins) if spans and p["contigs"] else None elif kind == "zero": - p["zero"] = [(r["contig"], int(r["start"]), int(r["end"])) - for r in rows if r.get("start", "").isdigit() and r.get("end", "").isdigit()] + p["zero"] = [(cell(r, "contig"), int(cell(r, "start")), int(cell(r, "end"))) for r in rows + if int(cell(r, "end")) - int(cell(r, "start")) + 1 >= min_len] else: - p["genes"] = [{"contig": r["contig"], "start": int(r["start"]), "end": int(r["end"]), - "gene": r.get("gene", ""), "locus": r.get("locus_tag", ""), - "breadth": to_float(r.get("breadth")), "rel": to_float(r.get("rel_depth"))} - for r in rows if r.get("start", "").isdigit() and r.get("end", "").isdigit()] - for p in out.values(): - p["median_dp"] = to_float(p.get("median_dp")) or 0.0 - p["contigs"] = _contigs(p.get("contigs")) - return out + keys = [(cell(r, "contig"), int(cell(r, "start")), int(cell(r, "end")), cell(r, "gene"), + cell(r, "locus_tag")) for r in rows] + breadth = [to_float(cell(r, "breadth")) for r in rows] + known = gene_lists.setdefault(ref, keys) + if keys != known: # another GFF version for the same reference: align by key + at = {k: i for i, k in enumerate(keys)} + breadth = [breadth[at[k]] if k in at else None for k in known] + p["genes"] = array("f", [-1.0 if b is None else b for b in breadth]) + return profiles, gene_lists def _contigs(text): @@ -132,14 +161,6 @@ def _spread(intervals, contigs, nbins=NBINS): return out -def callable_bp(profile, nbins=NBINS): - """Per-bin number of positions deep enough for the consensus to call, or None.""" - rows = [(c, s, e, f) for c, s, e, _, _, f in profile.get("windows", []) if f is not None] - if not rows or not profile.get("contigs"): - return None - return _spread(rows, profile["contigs"], nbins) - - def _bin_len(contigs, nbins=NBINS): return _spread([(c, 1, n, 1.0) for c, n in contigs], contigs, nbins) @@ -150,25 +171,51 @@ def snp_per_kb(snp_profile, profile, nbins=NBINS): A bin with fewer than a tenth of its positions callable has no value: a few called bases make any count look dense or empty. """ - cb = callable_bp(profile, nbins) - if not snp_profile or cb is None: + cb = (profile or {}).get("callable") + if not snp_profile or not cb or not profile.get("contigs"): return None full = _bin_len(profile["contigs"], nbins) return [round(1000.0 * (snp_profile[b] or 0) / cb[b], 3) - if cb[b] > 0 and full[b] and cb[b] >= 0.1 * full[b] else None for b in range(nbins)] + if cb[b] > 0 and full[b] and cb[b] >= 0.1 * full[b] else None + for b in range(min(nbins, len(snp_profile)))] def _overlap(a0, a1, b0, b1): return max(0, min(a1, b1) - max(a0, b0) + 1) -def build_coverage(profiles, min_len=200, min_depth=10.0, cohort_frac=0.9, gene_breadth=0.5, - nbins=NBINS): - """(report section, {sample: per-bin deleted fraction}, rows for the deletions TSV). +def _cluster(stretches, frac=0.5): + """Regions of [(start, end, carrier)] sorted by start: a stretch joins the first open region + whose founding stretch and itself each overlap by at least `frac` of their length.""" + regions, open_ = [], [] + for s, e, who in stretches: + open_ = [r for r in open_ if r["e0"] >= s] + home = None + for r in open_: + ov = _overlap(r["s0"], r["e0"], s, e) + if ov >= frac * (r["e0"] - r["s0"] + 1) and ov >= frac * (e - s + 1): + home = r + break + if home is None: + home = {"s0": s, "e0": e, "members": {}} + regions.append(home) + open_.append(home) + home["members"].setdefault(who, []).append((s, e)) + return regions + + +def _median(v): + v = sorted(v) + return v[len(v) // 2] + + +def build_coverage(profiles, gene_lists=None, min_len=200, min_depth=10.0, cohort_frac=0.9, + gene_breadth=0.5, nbins=NBINS): + """(report section, {(sample, reference): per-bin deleted fraction}, rows for the TSV). Stretches without reads of at least `min_len` bp, in samples whose genome-wide median depth - is at least `min_depth`, are merged across samples into regions and classed by how many of - the assessed samples on the same reference lack them: + is at least `min_depth`, are grouped across samples into regions (see _cluster) and classed + by how many of the assessed samples on the same reference lack them: cohort at least `cohort_frac` of them, and at least two: not a deletion of anyone shared two or more, fewer than that: a deletion several samples carry @@ -177,97 +224,102 @@ def build_coverage(profiles, min_len=200, min_depth=10.0, cohort_frac=0.9, gene_ Genes are named for a region when a carrier reads less than `gene_breadth` of them. """ + gene_lists = gene_lists or {} by_ref = {} - for sid, p in profiles.items(): - by_ref.setdefault(p.get("reference") or ",".join(c for c, _ in p.get("contigs", [])), []).append(sid) - - regions, tracks, not_assessed = [], {}, [] - for ref, samples in sorted(by_ref.items()): - assessed = [s for s in samples if profiles[s]["median_dp"] >= min_depth] - not_assessed += [s for s in samples if s not in assessed] - runs = sorted((c, s, e, sid) for sid in assessed for c, s, e in profiles[sid]["zero"] - if e - s + 1 >= min_len) - merged = [] - for c, s, e, sid in runs: - if merged and merged[-1]["contig"] == c and s <= merged[-1]["end"]: - r = merged[-1] - r["end"] = max(r["end"], e) - r["carriers"].setdefault(sid, []).append((s, e)) - else: - merged.append({"contig": c, "start": s, "end": e, "carriers": {sid: [(s, e)]}}) + for key, p in profiles.items(): + by_ref.setdefault(p.get("reference") or ",".join(c for c, _ in p.get("contigs", [])), []).append(key) + + # The genes each profile barely reads, found once: naming a region's genes then looks at a + # handful of genes per carrier instead of every gene of the reference. + thin_genes = {} + for key, p in profiles.items(): + genes, b = gene_lists.get(p.get("reference") or "", []), p.get("genes") + thin_genes[key] = ([genes[i] for i in range(min(len(genes), len(b))) if 0 <= b[i] < gene_breadth] + if b is not None else []) + + regions, not_assessed = [], [] + for ref, keys in sorted(by_ref.items()): + assessed = [k for k in keys if profiles[k]["median_dp"] >= min_depth] + not_assessed += [k[0] for k in keys if k not in assessed] n = len(assessed) - for r in merged: - k = len(r["carriers"]) - if n < 2: - cls = "alone" - elif k >= max(2, math.ceil(cohort_frac * n)): - cls = "cohort" - elif k >= 2: - cls = "shared" - else: - cls = "private" - genes = {} - for sid in r["carriers"]: - for g in profiles[sid]["genes"]: - if g["contig"] == r["contig"] and _overlap(g["start"], g["end"], r["start"], r["end"]) \ - and g["breadth"] is not None and g["breadth"] < gene_breadth: - genes.setdefault(g["gene"], g["locus"]) - regions.append({ - "ref": ref, "contig": r["contig"], "start": r["start"], "end": r["end"], - "len": r["end"] - r["start"] + 1, "cls": cls, "n": k, "of": n, - "genes": sorted(genes), - "samples": sorted([sid, min(a for a, _ in iv), max(b for _, b in iv)] - for sid, iv in r["carriers"].items()), - }) - - # Per-bin share of each sample's positions inside a deletion it carries (cohort-wide gaps - # excluded, since they are nobody's deletion). - per_sample = {} + by_contig = {} + for k in assessed: + for c, s, e in profiles[k]["zero"]: + if e - s + 1 >= min_len: + by_contig.setdefault(c, []).append((s, e, k)) + for contig, stretches in sorted(by_contig.items()): + for r in _cluster(sorted(stretches)): + k = len(r["members"]) + if n < 2: + cls = "alone" + elif k >= max(2, math.ceil(cohort_frac * n)): + cls = "cohort" + elif k >= 2: + cls = "shared" + else: + cls = "private" + own = {who: (min(a for a, _ in iv), max(b for _, b in iv)) for who, iv in r["members"].items()} + start, end = _median([a for a, _ in own.values()]), _median([b for _, b in own.values()]) + names = {} + for who in r["members"]: + for gc, gs, ge, gene, locus in thin_genes.get(who, []): + if gc == contig and _overlap(gs, ge, start, end): + names.setdefault(gene, locus) + regions.append({ + "ref": ref, "contig": contig, "start": start, "end": end, "len": end - start + 1, + "cls": cls, "n": k, "of": n, "genes": sorted(names), + "samples": sorted([who[0], a, b] for who, (a, b) in own.items()), + "_members": r["members"], + }) + + # Per-bin share of each profile's positions inside its own stretches of the regions it + # carries. A stretch nearly nobody reads is nobody's deletion, and a sample alone on its + # reference has nothing to be compared with, so neither lights up the track. + per = {} for reg in regions: - if reg["cls"] == "cohort": + if reg["cls"] in ("cohort", "alone"): + continue + for who, iv in reg["_members"].items(): + per.setdefault(who, []).extend((reg["contig"], s, e, 1.0) for s, e in iv) + tracks = {} + for key, p in profiles.items(): + if p["median_dp"] < min_depth or not p.get("contigs"): continue - for sid, s, e in reg["samples"]: - per_sample.setdefault(sid, []).append((reg["contig"], s, e, 1.0)) - for sid, ivs in per_sample.items(): - contigs = profiles[sid].get("contigs") or [] - full = _bin_len(contigs, nbins) - lost = _spread(ivs, contigs, nbins) - tracks[sid] = [round(min(1.0, lost[b] / full[b]), 3) if full[b] else 0.0 for b in range(nbins)] - for sid in profiles: - if sid not in tracks and profiles[sid]["median_dp"] >= min_depth and profiles[sid].get("contigs"): - tracks[sid] = [0.0] * nbins - - lost_genes = _lost_genes(profiles, by_ref, min_depth, gene_breadth) + full = _bin_len(p["contigs"], nbins) + lost = _spread(per.get(key, []), p["contigs"], nbins) + tracks[key] = [round(min(1.0, lost[b] / full[b]), 3) if full[b] else 0.0 for b in range(nbins)] + + for r in regions: + del r["_members"] + lost_genes = _lost_genes(profiles, gene_lists, by_ref, min_depth, gene_breadth) regions.sort(key=lambda r: (r["cls"] == "cohort", -r["n"] if r["cls"] == "shared" else 0, r["ref"], r["contig"], r["start"])) for i, r in enumerate(regions, 1): r["id"] = f"D{i}" # Each reference's contigs in order, so the report can place a region on the landscape, which # lays the contigs of a sample's genome end to end. - refs = {ref: [list(c) for c in profiles[samples[0]].get("contigs", [])] for ref, samples in by_ref.items()} + refs = {ref: [list(c) for c in profiles[keys[0]].get("contigs", [])] for ref, keys in by_ref.items()} section = {"min_len": min_len, "min_depth": min_depth, "cohort_frac": cohort_frac, "refs": refs, - "regions": regions, "genes_lost": lost_genes, "not_assessed": sorted(not_assessed), + "regions": regions, "genes_lost": lost_genes, "not_assessed": sorted(set(not_assessed)), "n_assessed": sum(1 for p in profiles.values() if p["median_dp"] >= min_depth)} return section, tracks, regions -def _lost_genes(profiles, by_ref, min_depth, gene_breadth, cohort_breadth=0.9): +def _lost_genes(profiles, gene_lists, by_ref, min_depth, gene_breadth, cohort_breadth=0.9): """Genes a sample reads less than `gene_breadth` of while at least half of the assessed samples on its reference read `cohort_breadth` of them: gene loss, rather than a gene nobody here carries.""" out = [] - for ref, samples in sorted(by_ref.items()): - assessed = [s for s in samples if profiles[s]["median_dp"] >= min_depth and profiles[s]["genes"]] - if len(assessed) < 2: + for ref, keys in sorted(by_ref.items()): + genes = gene_lists.get(ref) or [] + assessed = [k for k in keys if profiles[k]["median_dp"] >= min_depth and profiles[k].get("genes")] + if len(assessed) < 2 or not genes: continue - table = {} - for sid in assessed: - for g in profiles[sid]["genes"]: - if g["breadth"] is not None: - table.setdefault((g["contig"], g["start"], g["end"], g["gene"], g["locus"]), {})[sid] = g["breadth"] - for (contig, s, e, gene, locus), vals in sorted(table.items()): - lost = sorted([sid, round(b, 3)] for sid, b in vals.items() if b < gene_breadth) - read = sum(1 for b in vals.values() if b >= cohort_breadth) + for i, (contig, s, e, gene, locus) in enumerate(genes): + vals = [(k[0], profiles[k]["genes"][i]) for k in assessed + if i < len(profiles[k]["genes"]) and profiles[k]["genes"][i] >= 0] + lost = sorted([sid, round(b, 3)] for sid, b in vals if b < gene_breadth) + read = sum(1 for _, b in vals if b >= cohort_breadth) if lost and read >= 0.5 * len(vals): out.append({"ref": ref, "contig": contig, "start": s, "end": e, "gene": gene, "locus": locus, "samples": lost, "of": len(vals)}) @@ -275,7 +327,7 @@ def _lost_genes(profiles, by_ref, min_depth, gene_breadth, cohort_breadth=0.9): def write_deletions(path, regions): - """The regions as a TSV: one row per region, its carriers with their own coordinates.""" + """The regions as a TSV: one row per region, each carrier with its own stretch.""" with open(path, "w", encoding="utf-8") as fh: fh.write("region\treference\tcontig\tstart\tend\tlength\tclass\tn_samples\tn_assessed\t" "samples\tgenes\n") diff --git a/bin/qcreport/render.py b/bin/qcreport/render.py index 7f40093..e5c6a96 100644 --- a/bin/qcreport/render.py +++ b/bin/qcreport/render.py @@ -145,11 +145,12 @@ def build_html(title, payload): "next to one, a contaminant next to nobody.", "deletions": "Stretches of the reference a sample has no reads for, compared with the other samples on " "the same reference. Lacked by one sample it is a private deletion; by several, a shared " - "one (often a lineage's); by all, a repeat or a part of the reference these genomes do not " - "have, which is nobody's deletion and is listed apart. Only samples read deeply enough are " - "assessed, since in a thin sample stretches without reads turn up by chance. From the " - "all-positions VCF, so a deletion shorter than the minimum length, or one the caller " - "already reported as an indel, is not listed here.", + "one (often a lineage's); by nearly all (90% or more), a repeat or a part of the reference " + "these genomes do not have, which is nobody's deletion and is listed apart. Two samples' " + "stretches are one deletion when each covers at least half of the other. Only samples " + "read deeply enough are assessed, since in a thin sample stretches without reads turn up " + "by chance. From the all-positions VCF, so a deletion shorter than the minimum length, or " + "one the caller already reported as an indel, is not listed here.", "function": "The snpEff functional class of each sample's variants (HIGH/MODERATE/LOW/MODIFIER impact and " "effect types such as missense / synonymous). A per-sample mutational-impact profile.", "geneburden": "Genes carrying the most impactful (HIGH/MODERATE) variants across the cohort, with the " diff --git a/bin/report_assets/js/09_genome_genes.js b/bin/report_assets/js/09_genome_genes.js index 0c7a7e4..da251ec 100644 --- a/bin/report_assets/js/09_genome_genes.js +++ b/bin/report_assets/js/09_genome_genes.js @@ -95,7 +95,7 @@ function renderGenome(){ [0,0.25,0.5,0.75,1].forEach(function(t){var px=gut+t*plotW,pos=Math.round((z0+t*winN)/nb*gl); svg+=''+ ''+fmtpos(pos)+'';}); - if((tk=='snp'||tk=='snpkb'||tk=='del')&&st.geneMark&&st.geneMark.b1>=z0&&st.geneMark.b0<=z1){var gm=st.geneMark,mx0=Math.max(gut,x(gm.b0)),mx1=Math.min(gut+plotW,x(gm.b1+1)); + if((tk=='snp'||tk=='snpkb'||tk=='del'||tk=='missing')&&st.geneMark&&st.geneMark.b1>=z0&&st.geneMark.b0<=z1){var gm=st.geneMark,mx0=Math.max(gut,x(gm.b0)),mx1=Math.min(gut+plotW,x(gm.b1+1)); svg+=''+ ''+esc(gm.name||'')+'';} svg+='
';}).join('')+ '

'+tot.toLocaleString('en-US')+' pairs, in SNPs; teal = within the threshold of '+thr+'.
';} +// The controls are built once: rebuilding them on every redraw replaced the slider under the +// pointer mid-drag and dropped the keyboard focus. The slider's range is fixed for the same reason. +function relControls(host,S){ + if(el('relthr'))return; + var refs=Object.keys(S.refs),smax=Math.max(50,Math.min(500,4*(S.threshold||12))); + host.innerHTML='
'+(refs.length>1?(''+refs.map(function(k){return '';}).join('')+''):'')+ + '
'; + function setThr(v,from){v=Math.max(0,Math.round(+v||0));REL.thr=v; + if(from!=='num')el('relthr').value=v; if(from!=='range')el('relthrr').value=Math.min(v,smax); + renderRelContent();renderRelClusters();} + el('relthr').onchange=function(){setThr(this.value,'num');}; + el('relthrr').oninput=function(){var v=this.value;clearTimeout(REL._db);REL._db=setTimeout(function(){setThr(v,'range');},120);}; + Array.prototype.forEach.call(host.querySelectorAll('#relrefs button'),function(b){b.onclick=function(){REL.ref=b.getAttribute('data-r'); + Array.prototype.forEach.call(host.querySelectorAll('#relrefs button'),function(x){x.classList.toggle('on',x==b);}); + renderRelContent();renderRelClusters();};}); +} function renderRelatedness(){ var host=el('rel_body'),S=relSec(); if(!host)return; - var r=relRef(); if(!(S&&r)){host.innerHTML='';return;} - var thr=relThr(),minCmp=relMinCmp(),n=r.samples.length,refs=Object.keys(S.refs); - var order=relOrder(r,minCmp),gcol=relGroupColor(),byName={};R.samples.forEach(function(s){byName[s.s]=s;}); + if(!(S&&relRef())){host.innerHTML='';return;} + relControls(host,S); renderRelContent(); renderRelClusters(); +} +function renderRelContent(){ + var box=el('rel_content'),S=relSec(),r=relRef(); if(!(box&&S&&r))return; + // Drawn only once it can be seen: the load-time pass over every page would otherwise paint a + // thousand-sample heatmap into a hidden page, and opening the page would paint it again. + if(box.offsetParent===null&&!document.body.classList.contains('print-all'))return; + var thr=relThr(),minCmp=relMinCmp(),n=r.samples.length,key=REL.ref+'|'+minCmp; + // The order and the largest distance do not depend on the threshold: worked out once per reference. + REL.cache=REL.cache||{}; + var C=REL.cache[key]||(REL.cache[key]={order:relOrder(r,minCmp),max:(function(){var m=0,i,j;for(i=0;im)m=d;}return m;})()}); + var order=C.order,max=C.max,gcol=relGroupColor(); var cl=relClusters(r,thr,minCmp),multi=cl.filter(function(c){return c.length>1;}),alone=cl.filter(function(c){return c.length==1;}).length; - var max=0,i,j;for(i=0;imax)max=dd;} - var refseg=refs.length>1?(''+refs.map(function(k){return '';}).join('')+''):''; - host.innerHTML='
'+refseg+ - '
'+ - '
'+multi.length+' '+_plural(multi.length,'cluster')+' of two or more samples within '+thr+' SNPs'+ + box.innerHTML='
'+multi.length+' '+_plural(multi.length,'cluster')+' of two or more samples within '+thr+' SNPs'+ (multi.length?(', the largest of '+multi[0].length):'')+'; '+alone+' '+_plural(alone,'sample stands','samples stand')+' alone. '+n+' samples on '+esc(REL.ref)+ ', '+r.variable.toLocaleString('en-US')+' variable positions. Pairs that both called under '+minCmp+'% of them are left out of the clusters and shown blank.
'+ '
'+ @@ -68,14 +94,15 @@ function renderRelatedness(){ (Object.keys(S.group||{}).length?'left strip = group':'')+'samples in single-linkage order; hover a cell for the pair
'+ relHistogram(r,thr,minCmp); // A square that fits the screen both ways, so the last rows are never scrolled out of sight. - var cv=el('relcanvas'),W=Math.max(240,Math.min((host.clientWidth||900)-24,980)),H=Math.max(240,Math.min(760,Math.round((window.innerHeight||900)*0.72))), + var cv=el('relcanvas'),W=Math.max(240,Math.min((box.clientWidth||900)-24,980)),H=Math.max(240,Math.min(760,Math.round((window.innerHeight||900)*0.72))), strip=10,cs=Math.max(1,Math.floor(Math.min(W-strip-4,H)/Math.max(1,n))),side=cs*n; var dpr=window.devicePixelRatio||1; cv.width=(side+strip+4)*dpr; cv.height=side*dpr; cv.style.width=(side+strip+4)+'px'; cv.style.height=side+'px'; var ctx=cv.getContext&&cv.getContext('2d'); if(ctx){ctx.scale(dpr,dpr); + var cc={}; // one colour string per distinct distance, not one per cell for(var a=0;a=n||b>=n){tip('');return;} var ia=order[a],ib=order[b],sa=r.samples[ia],sb=r.samples[ib],d=relD(r,ia,ib),c=relC(r,ia,ib),g=S.group||{}; @@ -83,20 +110,15 @@ function renderRelatedness(){ (d==null?'no distance':(d.toLocaleString('en-US')+' SNPs'))+(ia==ib?'':' over '+(c==null?'NA':c+'%')+' of the variable positions'),e.clientX,e.clientY);}; cv.onmouseleave=function(){tip('');}; cv.onclick=function(e){var rc=cv.getBoundingClientRect(),a=Math.floor((e.clientY-rc.top)/cs);if(a>=0&&a1;}),g=S.group||{},gcol=relGroupColor(),hasG=Object.keys(g).length>0; var rows=cl.map(function(c,k){var mx=0,a,b;for(a=0;amx)mx=d;} var names=c.map(function(i){return r.samples[i];}),gs={};names.forEach(function(s){if(g[s])gs[g[s]]=1;});var ng=Object.keys(gs).length; - return ''+(k+1)+''+c.length+''+names.map(function(s){var col=gcol(s); - return ''+esc(s)+'';}).join(' ')+''+ + // a large cluster wraps and names its first 40: a 93-sample one made the table 8,000 px wide + return ''+(k+1)+''+c.length+''+names.slice(0,40).map(function(s){var col=gcol(s); + return ''+esc(s)+'';}).join(' ')+(names.length>40?' and '+(names.length-40)+' more':'')+''+ ''+mx.toLocaleString('en-US')+''+(hasG?'1?' style="color:var(--warn);font-weight:600" title="the cluster joins samples the samplesheet puts in different groups"':'')+'>'+ng+'':'')+'';}).join(''); host.innerHTML='
Samples joined by a chain of pairs each within '+thr+' SNPs (single linkage). '+ (hasG?'A cluster spanning several groups is where to look for transmission between them, or for a sample in the wrong group.':'Add a group column to the samplesheet (patient, line, outbreak) to see which clusters cross groups.')+'
'+ diff --git a/bin/snp_distances.py b/bin/snp_distances.py index cc554dd..c7c48a3 100644 --- a/bin/snp_distances.py +++ b/bin/snp_distances.py @@ -8,7 +8,10 @@ is why the distances are taken from it rather than from the VCFs. Only samples mapped against the same reference are compared: their consensus sequences share -the reference's coordinates, and sequences of different references do not. +the reference's coordinates, and sequences of different references do not. A sample mapped +against two references appears once per reference, labelled sample@reference in the matrix. +A consensus of another length than most of its reference's is left out with a warning naming it +(a truncated file), rather than stopping every reference's distances. Two outputs: -o square matrix, samples x samples, NA between samples of different references @@ -27,6 +30,7 @@ import gzip import os import sys +from collections import Counter import numpy as np @@ -47,6 +51,17 @@ def read_fasta(path): return names, np.frombuffer(seq, dtype=np.uint8) +def seq_length(path): + """The number of sequence characters in a FASTA, without building the sequence.""" + op = gzip.open if str(path).endswith(".gz") else open + n = 0 + with op(path, "rt", encoding="utf-8", errors="replace") as fh: + for line in fh: + if not line.startswith(">"): + n += len(line.strip()) + return n + + def entries(args): """[(sample, reference, path)] from --manifest or --consensus.""" out = [] @@ -129,28 +144,41 @@ def main(argv=None): sys.stderr.write(f"[snp_distances] WARN {sample}: {path} not found\n") continue groups.setdefault(ref, []).append((sample, path)) - samples = [s for ref in sorted(groups) for s, _ in groups[ref]] - index = {s: i for i, s in enumerate(samples)} - full = [["NA"] * len(samples) for _ in samples] + for ref, members in list(groups.items()): + lengths = [seq_length(p) for _, p in members] + usual = Counter(lengths).most_common(1)[0][0] + for (s, p), n in zip(members, lengths): + if n != usual: + sys.stderr.write(f"[snp_distances] WARN {s}: {p} is {n} bp where the other consensus " + f"sequences of {ref} have {usual}; left out\n") + groups[ref] = [m for m, n in zip(members, lengths) if n == usual] + + # One matrix row per sample and reference; a sample on two references is named for each. + refs_of = Counter(s for members in groups.values() for s, _ in members) + labels = [(ref, s, f"{s}@{ref}" if refs_of[s] > 1 else s) for ref in sorted(groups) for s, _ in groups[ref]] + index = {(ref, s): i for i, (ref, s, _) in enumerate(labels)} + full = [["NA"] * len(labels) for _ in labels] rows = [] for ref in sorted(groups): members = groups[ref] + if not members: + continue paths = [p for _, p in members] pos, length = variable_positions(paths) codes = codes_at(paths, pos) snps, compared = distances(codes) for i, (si, _) in enumerate(members): for j, (sj, _) in enumerate(members): - full[index[si]][index[sj]] = str(int(snps[i, j])) + full[index[(ref, si)]][index[(ref, sj)]] = str(int(snps[i, j])) if j > i: rows.append((si, sj, ref, int(snps[i, j]), int(compared[i, j]), len(pos))) sys.stderr.write(f"[snp_distances] {ref}: {len(members)} sample(s), {length} bp, " f"{len(pos)} variable position(s)\n") with open(a.output, "w", encoding="utf-8") as fh: - fh.write("sample\t" + "\t".join(samples) + "\n") - for s in samples: - fh.write(s + "\t" + "\t".join(full[index[s]]) + "\n") + fh.write("sample\t" + "\t".join(lab for _, _, lab in labels) + "\n") + for i, (_, _, lab) in enumerate(labels): + fh.write(lab + "\t" + "\t".join(full[i]) + "\n") if a.pairs: with open(a.pairs, "w", encoding="utf-8") as fh: fh.write("sample_a\tsample_b\treference\tsnps\tcompared\tvariable_sites\n") diff --git a/docs/outputs.md b/docs/outputs.md index 1679db0..8db6398 100644 --- a/docs/outputs.md +++ b/docs/outputs.md @@ -101,6 +101,7 @@ apart; the matrix step warns when it sees one. `_snp_distances.tsv` is the square matrix of SNPs between every two consensus sequences, `NA` between samples mapped against different references, which share no coordinates. +A sample mapped against two references has a row and a column for each, named `sample@reference`. Two samples differ at a position only where both called a base (`A`, `C`, `G` or `T`) and the bases differ: a no-call, a masked position or an ambiguity code at a mixed site is never a difference, whichever sample it is in. These are the distances a tree built on the same diff --git a/tests/unit/test_qc_report_relatedness.py b/tests/unit/test_qc_report_relatedness.py index 9a1bdd5..5aea757 100644 --- a/tests/unit/test_qc_report_relatedness.py +++ b/tests/unit/test_qc_report_relatedness.py @@ -13,11 +13,11 @@ def dist(pairs, variable=100, ref="R"): """A parse_pairs() result from {(a, b): snps}, every pair comparing all variable positions.""" - out = {"ref": {}, "variable": {ref: variable}, "pairs": {}} + out = {"refs": {}, "variable": {ref: variable}, "pairs": {}} for (a, b), v in pairs.items(): snps, comp = v if isinstance(v, tuple) else (v, variable) - out["pairs"][(a, b)] = (snps, comp) - out["ref"][a] = out["ref"][b] = ref + out["pairs"][(ref, a, b)] = (snps, comp) + out["refs"].setdefault(ref, set()).update((a, b)) return out @@ -43,7 +43,7 @@ def test_a_sample_far_from_its_group_and_close_to_another_is_flagged(): out = rel.group_mismatches(full(LINES, override=over), LINES, threshold=12) assert list(out) == ["a4"] - assert out["a4"] == {"g": "A", "din": 2000, "nin": "a1", "dout": 3, "nout": "b1", "gout": "B"} + assert out["a4"] == {"g": "A", "ref": "R", "din": 2000, "nin": "a1", "dout": 3, "nout": "b1", "gout": "B"} def test_a_sample_far_from_its_group_and_from_everyone_else_is_flagged_too(): @@ -90,13 +90,43 @@ def test_a_pair_resting_on_few_positions_does_not_count(): def test_samples_of_another_reference_are_never_compared(): d = full(LINES, override={("a1", "a4"): 2000, ("a2", "a4"): 2000, ("a3", "a4"): 2000, ("a4", "b1"): 900, ("a4", "b2"): 900, ("a4", "b3"): 900}) - d["ref"]["x"] = "OTHER" - d["pairs"][("a4", "x")] = (1, 100) + d["refs"]["OTHER"] = {"a4", "x"} + d["variable"]["OTHER"] = 100 + d["pairs"][("OTHER", "a4", "x")] = (1, 100) groups = dict(LINES, x="B") assert rel.group_mismatches(d, groups)["a4"]["nout"] == "b1", "x is on another reference" +def test_a_sample_on_two_references_keeps_both_sets_of_distances(tmp_path): + """A is mapped against R1 and R2: its distance to B on each is a separate comparison.""" + p = tmp_path / "pairs.tsv" + p.write_text("sample_a\tsample_b\treference\tsnps\tcompared\tvariable_sites\n" + "A\tB\tR1\t4\t90\t100\nA\tB\tR2\t40\t90\t100\nA\tC\tR2\t7\t88\t100\n") + + sec = rel.build_relatedness(rel.parse_pairs(str(p))) + + assert sec["refs"]["R1"]["samples"] == ["A", "B"] and sec["refs"]["R1"]["snps"] == [4] + assert sec["refs"]["R2"]["samples"] == ["A", "B", "C"] and sec["refs"]["R2"]["snps"] == [40, 7, None] + + +def test_identical_samples_compared_everything_there_was(): + """With no variable position at all, snp_distances writes compared 0 of 0: that is every + sample identical wherever it was called, not a pair that compared nothing.""" + d = dist({("a", "b"): (0, 0)}, variable=0) + + sec = rel.build_relatedness(d) + + assert sec["refs"]["R"]["cmp"] == [100] + + +def test_the_share_compared_is_rounded_down_like_the_flag_counts_it(): + """495 of 1000 is under half: the page must not cluster a pair the flag leaves out.""" + d = dist({("a", "b"): (1, 495)}, variable=1000) + + assert rel.build_relatedness(d)["refs"]["R"]["cmp"] == [49] + + def test_parse_pairs_reads_the_distances_tsv(tmp_path): p = tmp_path / "pairs.tsv" p.write_text("sample_a\tsample_b\treference\tsnps\tcompared\tvariable_sites\n" @@ -104,15 +134,15 @@ def test_parse_pairs_reads_the_distances_tsv(tmp_path): d = rel.parse_pairs(str(p)) - assert d["pairs"][("S1", "S2")] == (4, 90) - assert d["ref"] == {"S1": "R", "S2": "R", "S3": "R"} and d["variable"] == {"R": 100} + assert d["pairs"][("R", "S1", "S2")] == (4, 90) + assert d["refs"] == {"R": {"S1", "S2", "S3"}} and d["variable"] == {"R": 100} def test_parse_pairs_of_a_placeholder_is_empty(tmp_path): p = tmp_path / "NO_FILE_DISTANCES" p.write_text("") - assert rel.parse_pairs(str(p)) == {"ref": {}, "variable": {}, "pairs": {}} + assert rel.parse_pairs(str(p)) == {"refs": {}, "variable": {}, "pairs": {}} def test_build_relatedness_lays_out_each_references_upper_triangle(): @@ -127,4 +157,4 @@ def test_build_relatedness_lays_out_each_references_upper_triangle(): def test_build_relatedness_without_distances_is_none(): - assert rel.build_relatedness({"ref": {}, "variable": {}, "pairs": {}}) is None + assert rel.build_relatedness({"refs": {}, "variable": {}, "pairs": {}}) is None diff --git a/tests/unit/test_snp_distances.py b/tests/unit/test_snp_distances.py index f4e9d33..3ec0999 100644 --- a/tests/unit/test_snp_distances.py +++ b/tests/unit/test_snp_distances.py @@ -100,3 +100,34 @@ def test_the_matrix_is_symmetric_with_a_zero_diagonal(tmp_path): m = {r[0]: dict(zip(head, r[1:])) for r in square[1:]} assert all(m[s][s] == "0" for s in head) assert all(m[a][b] == m[b][a] for a in head for b in head) + + +def test_a_sample_on_two_references_is_one_row_per_reference(tmp_path): + """Named once per reference, with each reference's distances in its own row and column.""" + fasta(tmp_path / "A1.fa", "ACGTACGTAC", "A") + fasta(tmp_path / "B.fa", "ACGTACGTAA", "B") + fasta(tmp_path / "A2.fa", "TTTTTTTT", "A") + fasta(tmp_path / "C.fa", "TTTTTTTA", "C") + (tmp_path / "m.tsv").write_text("A\tR1\tA1.fa\nB\tR1\tB.fa\nA\tR2\tA2.fa\nC\tR2\tC.fa\n") + + assert sd.main(["--manifest", str(tmp_path / "m.tsv"), "--dir", str(tmp_path), + "-o", str(tmp_path / "sq.tsv")]) == 0 + + rows = [r.split("\t") for r in (tmp_path / "sq.tsv").read_text().splitlines()] + assert rows[0] == ["sample", "A@R1", "B", "A@R2", "C"] + m = {r[0]: dict(zip(rows[0][1:], r[1:])) for r in rows[1:]} + assert m["A@R1"]["B"] == "1" and m["A@R2"]["C"] == "1" and m["A@R1"]["C"] == "NA" + + +def test_a_truncated_consensus_is_left_out_by_name_and_the_rest_compared(tmp_path, capsys): + fasta(tmp_path / "A.fa", "ACGTAC", "A") + fasta(tmp_path / "B.fa", "ACGTAA", "B") + fasta(tmp_path / "C.fa", "ACG", "C") + (tmp_path / "m.tsv").write_text("C\tR\tC.fa\nA\tR\tA.fa\nB\tR\tB.fa\n") + + assert sd.main(["--manifest", str(tmp_path / "m.tsv"), "--dir", str(tmp_path), + "-o", str(tmp_path / "sq.tsv"), "--pairs", str(tmp_path / "p.tsv")]) == 0 + + err = capsys.readouterr().err + assert "C: " in err and "left out" in err, "the short file, not the next one, is named" + assert (tmp_path / "p.tsv").read_text().splitlines()[1].split("\t")[:4] == ["A", "B", "R", "1"] From 716382a8999c2c2193a8e4a828e47460ff4bfa9d Mon Sep 17 00:00:00 2001 From: Paula Ruiz Rodriguez <50167687+Paururo@users.noreply.github.com> Date: Wed, 23 Sep 2026 15:48:06 +0200 Subject: [PATCH 10/14] Start each series at a sample it can be read against A series was read against its earliest sample whatever the QC made of it: a failed or swapped first time point made every later sample gain what that one lacked. The start is now the earliest time point with a sample the QC keeps and places in the series, and a sample whose time cannot be read has no place in it rather than being taken for the last. Times were read by their first number. A day-first date sorted by its day, so 03/06/2020 came before 15/01/2020; an ISO date by its year, so a year's samples were one time point; and "baseline", with no number, came last. Dates are now read as dates and a word for the start as time 0. A DNA-extract column such as replicate_of, which matches the group pattern, is no longer taken for the group. A site read at exactly --consensus_min_dp reads, which the consensus leaves uncalled, no longer counts as read. The chart placed times that are not numbers by row, one step per sample, so a time point's samples could not share a median; they are now categories, and every tick carries the time as the samplesheet writes it. Groups beyond the twelfth get hues of their own, and a sample name is no longer run as HTML by the tooltip. The summary's gain leaves out the samples the chart leaves out and takes the median at each series' last time point. --- bin/qc_report.py | 9 ++++-- bin/qcreport/parsers.py | 47 ++++++++++++++++++++++++++--- bin/qcreport/series.py | 37 +++++++++++++++-------- bin/report_assets/js/07b_summary.js | 14 ++++++--- bin/report_assets/js/10b_series.js | 39 +++++++++++++++--------- docs/qc-report.md | 4 +-- tests/js/summary.test.mjs | 16 ++++++++++ tests/unit/test_qc_report_logic.py | 25 ++++++++++++++- tests/unit/test_qc_report_series.py | 36 ++++++++++++++++++++++ 9 files changed, 186 insertions(+), 41 deletions(-) diff --git a/bin/qc_report.py b/bin/qc_report.py index 0b07556..5cc7f2a 100644 --- a/bin/qc_report.py +++ b/bin/qc_report.py @@ -349,11 +349,16 @@ def profile_of(sid): rep_col, rep_val, rep_reads = replicate_sets(args.metadata) rep_pairs, rep_shared = (replicate_pairs(rep_val, rep_reads, set(_variants), ref_of) if rep_col else ([], 0)) - _need = needed_cells(series_meta, _variants) + # A sample the QC fails or places outside its series (another lineage, far from its group) + # cannot be the first time point every later one is read against. + not_in_series = {s["s"] for s in jsamples + if s["v"] == "FAIL" or {"GROUP_MISMATCH", "LINEAGE_MISMATCH"} & set(s["f"])} + _need = needed_cells(series_meta, _variants, excluded=not_in_series) for site, who in replicate_cells(rep_pairs, _variants).items(): _need.setdefault(site, set()).update(who) _cells = read_matrix_cells(args.snp_matrix, _need) if matrix_ok else {} - _series = build_series(series_meta, _variants, _cells, _sample_meta, args.min_dp, checked=matrix_ok) + _series = build_series(series_meta, _variants, _cells, _sample_meta, args.min_dp, checked=matrix_ok, + excluded=not_in_series) _minority = build_minority(_variants, rep_pairs, rep_shared, _cells, rep_col, args.min_dp, checked=matrix_ok) _dynamics = build_dynamics(series_meta, _variants, _sample_meta) # feeds dynamics + epistasis # front-load 'dose' so it survives the correlation matrix's top-N view diff --git a/bin/qcreport/parsers.py b/bin/qcreport/parsers.py index 73026df..3d9f2af 100644 --- a/bin/qcreport/parsers.py +++ b/bin/qcreport/parsers.py @@ -13,6 +13,7 @@ """ from __future__ import annotations +import datetime import gzip import os import re @@ -748,6 +749,14 @@ def mapdamage_stats(sample_dir): _DYN_GROUP_RE = re.compile(r'(group|series|patient|host|subject|cluster|experiment|^line$|replicate|chain|pair|lineage_?id|donor|case|animal|^samples$)', re.I) _DOSE_RE = re.compile(r'^(dose|dosis)$', re.I) # a numeric annotation -> a metric, not a categorical level _TX_RE = re.compile(r'(treatment|tratamiento|regimen|therapy|^arm$)', re.I) # the categorical column the dose x treatment test groups by +# A column naming a sample's DNA extract, or the library it replicates. It ties technical +# replicates together, not a patient or a line, so it is never read as the group even where its +# name ('replicate_of') would match the group pattern. +_TECHREP_RE = re.compile(r'^(dna[_ ]?id|dna|extract(ion)?([_ ]?id)?|biosample([_ ]?id)?|specimen([_ ]?id)?|' + r'isolate[_ ]?id|library[_ ]?of|replicate[_ ]?of|tech(nical)?[_ ]?rep(licate)?([_ ]?of)?)$', re.I) +_BASELINE_RE = re.compile(r'^(baseline|base|start|pre|initial|ancestor|ancestral)$', re.I) +_ISO_DATE_RE = re.compile(r'^(\d{4})[-/.](\d{1,2})[-/.](\d{1,2})') +_DMY_DATE_RE = re.compile(r'^(\d{1,2})[-/.](\d{1,2})[-/.](\d{4})$') def _dyn_open(path): @@ -755,10 +764,37 @@ def _dyn_open(path): else open(path, 'r', encoding='utf-8', errors='replace') +def _day(y, mo, d): + try: + return float(datetime.date(y, mo, d).toordinal()) + except ValueError: + return None + + def _dyn_num(x): + """A time value as a number that sorts in time order, or None when it has none. + + A date becomes its day number: ISO year-month-day, or day/month/year, read as month/day/year + only when the day gives it away (03/15/2020). The first number of a date is the day, and + ordering by it put 03/06/2020 before 15/01/2020. A word for the start ('baseline', 'pre') + is time 0, which the first number of 'M3' or 'P6' comes after. Anything else is its first + number ('P3', 'day 14', 'week 2').""" if x is None: return None - m = re.search(r'-?\d+\.?\d*', str(x)) + s = str(x).strip() + if not s: + return None + if _BASELINE_RE.match(s): + return 0.0 + m = _ISO_DATE_RE.match(s) + if m: + return _day(*map(int, m.groups())) + m = _DMY_DATE_RE.match(s) + if m: + a, b, y = map(int, m.groups()) + d, mo = (b, a) if (b > 12 and a <= 12) else (a, b) + return _day(y, mo, d) + m = re.search(r'-?\d+\.?\d*', s) return float(m.group()) if m else None @@ -806,7 +842,8 @@ def parse_sample_meta(path): # tag the time / group columns (dynamics axes) and the treatment column (dose x treatment test) # so the report can route each correctly; they all stay as SNP-matrix header levels either way. time_field = next((h for _, h in fields if _DYN_TIME_RE.search(h.replace(' ', '_'))), None) - group_field = next((h for _, h in fields if _DYN_GROUP_RE.search(h.replace(' ', '_'))), None) + group_field = next((h for _, h in fields if _DYN_GROUP_RE.search(h.replace(' ', '_')) + and not _TECHREP_RE.match(h.strip())), None) tx_field = next((h for _, h in fields if _TX_RE.search(h.replace(' ', '_'))), None) return {'fields': [h for _, h in fields], 'rows': out, 'time_field': time_field, 'group_field': group_field, 'tx_field': tx_field} @@ -904,13 +941,13 @@ def parse_metadata(path): return {} if not header: return {} - def find(rx): + def find(rx, skip=None): for i, h in enumerate(header): - if rx.search(h.replace(' ', '_')): + if rx.search(h.replace(' ', '_')) and not (skip and skip.match(h.strip())): return i return None si = next((i for i, h in enumerate(header) if _DYN_SAMPLE_RE.match(h.strip())), 0) - ti, gi = find(_DYN_TIME_RE), find(_DYN_GROUP_RE) + ti, gi = find(_DYN_TIME_RE), find(_DYN_GROUP_RE, skip=_TECHREP_RE) out = {} for r in rows: if si >= len(r): diff --git a/bin/qcreport/series.py b/bin/qcreport/series.py index cdebd56..e9b0818 100644 --- a/bin/qcreport/series.py +++ b/bin/qcreport/series.py @@ -12,7 +12,13 @@ Telling 'absent' from 'not read' needs the depth of the sites a sample has no call at, which is what the SNP matrix holds. Without it every site not called at the start counts as new, and the -section says so rather than passing that off as checked. +section says so rather than passing that off as checked. A site counts as read above +--consensus_min_dp reads, where the consensus stops leaving it uncalled. + +The first time point is the earliest one with a sample the QC keeps and places in its series: +a failed or swapped first sample would make every later sample look as if it had gained +everything that sample lacks. A sample whose time cannot be read as a number has no place in +the series and is left out of it. """ from __future__ import annotations @@ -27,27 +33,32 @@ def _order(meta, samples): str(meta[s].get("time") or ""))) -def series_groups(meta, variants): - """{group: (ordered samples, first-time-point samples)} for groups with two or more times.""" +def series_groups(meta, variants, excluded=()): + """{group: (samples from the first time point on, in time order; first-time-point samples)} + for groups with two or more times among the samples not `excluded`.""" groups = {} for s, md in (meta or {}).items(): if md.get("group") and s in variants: groups.setdefault(md["group"], []).append(s) out = {} for g, samples in groups.items(): - samples = _order(meta, samples) - times = [meta[s].get("time") for s in samples] + timed = [s for s in samples if meta[s].get("tnum") is not None] + samples = _order(meta, timed if timed else samples) + kept = [s for s in samples if s not in excluded] + times = [meta[s].get("time") for s in kept] if len(set(times)) < 2: continue - out[g] = (samples, [s for s in samples if meta[s].get("time") == times[0]]) + base = [s for s in kept if meta[s].get("time") == times[0]] + start = samples.index(base[0]) + out[g] = ([s for s in samples[start:] if s in base or meta[s].get("time") != times[0]], base) return out -def needed_cells(meta, variants, fixed=FIXED): +def needed_cells(meta, variants, fixed=FIXED, excluded=()): """{site: {samples}} whose matrix cells the comparison reads: the first time point at every site a later sample carries fixed, and every later sample at the sites fixed at the start.""" need = {} - for samples, base in series_groups(meta, variants).values(): + for samples, base in series_groups(meta, variants, excluded).values(): later = [s for s in samples if s not in base] for s in later: for site, v in variants[s].items(): @@ -104,15 +115,17 @@ def _state(site, sample, variants, cells, min_dp, fixed): return "fixed" if af >= fixed else "minority" if dp is None: return "unchecked" - return "absent" if dp >= min_dp else "unread" + return "absent" if dp > min_dp else "unread" _RANK = ("fixed", "minority", "absent", "unread", "unchecked") -def build_series(meta, variants, cells, sample_meta=None, min_dp=7, fixed=FIXED, checked=True): - """The payload section, or None without a series of two or more time points.""" - groups = series_groups(meta, variants) +def build_series(meta, variants, cells, sample_meta=None, min_dp=7, fixed=FIXED, checked=True, + excluded=()): + """The payload section, or None without a series of two or more time points. `excluded` + are the samples that cannot be a series' first time point (failed, or placed outside it).""" + groups = series_groups(meta, variants, excluded) if not groups: return None tx = (sample_meta or {}).get("tx_field") diff --git a/bin/report_assets/js/07b_summary.js b/bin/report_assets/js/07b_summary.js index 9ea4118..a9e5184 100644 --- a/bin/report_assets/js/07b_summary.js +++ b/bin/report_assets/js/07b_summary.js @@ -166,18 +166,22 @@ function findResistance(){ link:{href:'#drug',t:'See every mutation'},note:'WHO catalogue grades 1–2, called from the reads. A genomic screen, not a drug-susceptibility result.',short:shortTxt}; } // Per series, the fixed SNPs gained by its last time point: the median across series and the most. +// Samples the QC fails or places outside their series are left out, as the chart leaves them out. function seriesGainSummary(){var S=R.series;if(!(S&&S.groups&&S.groups.length))return null; - var last=S.groups.map(function(g){var r=g.rows[g.rows.length-1];return {g:g.group,n:r?r['new'].length+r.risen.length:0};}); - var ns=last.map(function(x){return x.n;}).sort(function(a,b){return a-b;}),top=last.slice().sort(function(a,b){return b.n-a.n;})[0]; - return {median:ns[Math.floor(ns.length/2)],max:top.n,top:top.g,checked:S.checked};} + var last=[];S.groups.forEach(function(g){var kept=g.rows.filter(function(r){return !serOutside(r.s);}); + if(!kept.length)return;var t=kept[kept.length-1].time,at=kept.filter(function(r){return r.time===t;}); + last.push({g:g.group,n:_median(at.map(function(r){return r['new'].length+r.risen.length;}))});}); + if(!last.length)return null; + var top=last.slice().sort(function(a,b){return b.n-a.n;})[0]; + return {median:_median(last.map(function(x){return x.n;})),max:top.n,top:top.g,checked:S.checked};} function findDynamics(){ var d=dynSummary(); if(!d)return null; var head=''+d.sweep.length+' '+_plural(d.sweep.length,'allele','alleles')+' swept toward fixation and '+d.rise+' rose, across '+d.nSeries+' series'; var body=[]; if(d.parallel.length)body.push('Rising in two or more independent series: '+_list(d.parallel.map(function(x){return ''+esc(x.gene)+' ('+x.n+')';}),5)+'.'); body.push('Out of '+d.n.toLocaleString('en-US')+' trajectories; '+d.fall+' declined.'); - var sg=seriesGainSummary(); if(sg)body.push('By its last time point a series had gained a median of '+sg.median+' fixed '+_plural(sg.median,'SNP','SNPs')+ - ' since its first'+(sg.max>sg.median?(', up to '+sg.max+' in '+esc(sg.top)):'')+(sg.checked?'':' (the start not checked for depth)')+'.'); + var sg=seriesGainSummary(); if(sg)body.push('By its last time point a series had gained a median of '+sg.median.toLocaleString('en-US')+' fixed '+_plural(sg.median,'SNP','SNPs')+ + ' since its first'+(sg.max>sg.median?(', up to '+sg.max.toLocaleString('en-US')+' in '+esc(sg.top)):'')+(sg.checked?'':' (the start not checked for depth)')+'.'); return {k:'dyn',eyebrow:'Variants over time',tone:'',head:head,body:body.join(' '), link:{href:'#dynamics',t:'Open the trajectories'},note:'A screen of allele-frequency trajectories, not a selection test: drift and linked passengers move too.', short:d.sweep.length?d.sweep.length+' '+_plural(d.sweep.length,'allele','alleles')+' swept toward fixation.':''}; diff --git a/bin/report_assets/js/10b_series.js b/bin/report_assets/js/10b_series.js index f9a7a5c..fd497bd 100644 --- a/bin/report_assets/js/10b_series.js +++ b/bin/report_assets/js/10b_series.js @@ -5,7 +5,8 @@ var SERPAL=['#4477aa','#ee6677','#228833','#ccbb44','#66ccee','#aa3377','#e69f00','#0072b2','#d55e00','#009e73','#cc79a7','#882255']; function serColorBy(){var S=R.series,m={},k=0;if(!S)return function(){return '#4f83c2';}; var key=function(g){return S.tx_field?(g.tx||'NA'):g.group;}; - S.groups.map(key).sort().forEach(function(v){if(!(v in m)){m[v]=SERPAL[k%SERPAL.length];k++;}}); + // the palette first, then evenly spread hues, so a twentieth group does not repeat the eighth + S.groups.map(key).sort().forEach(function(v){if(!(v in m)){m[v]=kx0?(v-x0)/(x1-x0):0.5)*iw;} function Y(v){return pt+ih-v/ymax*ih;} + // Tooltip text lives here and the circles carry its index: text put in an attribute comes back + // decoded by getAttribute, and tip() writes it as HTML, so a sample named ran. + var TIPS=[];function tipIdx(h){TIPS.push(h);return TIPS.length-1;} var svg=''; [0,0.5,1].forEach(function(t){var v=Math.round(t*ymax);svg+=''+ ''+v+'';}); - var ticks={};pts.forEach(function(v){ticks[v]=1;}); - Object.keys(ticks).map(Number).sort(function(a,b){return a-b;}).forEach(function(v){ - svg+=''+(numeric?v:'')+'';}); + var tv=Object.keys(tl).map(Number).sort(function(a,b){return a-b;}),step=Math.max(1,Math.ceil(tv.length/14)); + tv.forEach(function(v,i){if(i%step)return; + svg+=''+esc(tl[v])+'';}); S.groups.forEach(function(g){var c=col(g),byT={},order=[]; // the line runs through the median of the samples counted at each time point - g.rows.forEach(function(r,i){var x=numeric?r.tnum:i+1;if(serOutside(r.s))return;if(!(x in byT)){byT[x]=[];order.push(x);}byT[x].push(serGained(r));}); - var line=[[numeric?g.tnum0:0,0]].concat(order.sort(function(a,b){return a-b;}).map(function(x){var v=byT[x].slice().sort(function(a,b){return a-b;});return [x,v[Math.floor(v.length/2)]];})); + g.rows.forEach(function(r){var x=tx(r.time,r.tnum);if(serOutside(r.s))return;if(!(x in byT)){byT[x]=[];order.push(x);}byT[x].push(serGained(r));}); + var line=[[tx(g.t0,g.tnum0),0]].concat(order.sort(function(a,b){return a-b;}).map(function(x){return [x,_median(byT[x])];})); svg+=''; - svg+=''; - g.rows.forEach(function(r,i){var x=numeric?r.tnum:i+1,out=serOutside(r.s),v=serGained(r),y=out?Math.min(v,ymax):v; + svg+=''; + g.rows.forEach(function(r){var x=tx(r.time,r.tnum),out=serOutside(r.s),v=serGained(r),y=out?Math.min(v,ymax):v; var lab=esc(r.s)+' · '+esc(r.time)+': '+r['new'].length+' new, '+r.risen.length+' risen'+(r.unknown?', '+r.unknown+' unknown':'')+(r.lost.length?', '+r.lost.length+' lost':'')+(out?' · not counted: the QC fails it or places it outside its series':''); - svg+='';});}); - svg+='fixed SNPs gained since the first time point'; + svg+='';});}); + svg+='fixed SNPs gained since the first time point'; var legend='
'+Object.keys(col.map).map(function(k){return ''+esc(k)+'';}).join('')+ (nOut?''+nOut+' not counted (FAIL, or outside its series), capped at the top of the scale':'')+ ''+(S.tx_field?'coloured by '+esc(S.tx_field):'coloured by group')+'; the line is the median per time point; hover a point for its sample
'; @@ -58,8 +69,8 @@ function renderSeries(){ '
'+svg+'
'+legend+ '
'+(S.tx_field?'':'')+ ''+rows.join('')+'
Series'+esc(S.tx_field)+'SampleTimeNewRisenUnknownLostGained (new in bold)
'; - Array.prototype.forEach.call(host.querySelectorAll('circle[data-tip]'),function(c){ - c.onmousemove=function(e){tip(c.getAttribute('data-tip'),e.clientX,e.clientY);};c.onmouseleave=function(){tip('');};}); + Array.prototype.forEach.call(host.querySelectorAll('circle[data-i]'),function(c){ + c.onmousemove=function(e){tip(TIPS[+c.getAttribute('data-i')],e.clientX,e.clientY);};c.onmouseleave=function(){tip('');};}); } // Resistance along each series: the WHO grade 1-2 mutations at every time point, those acquired // since the first one set apart. Lineage markers are there from the start, so they read as inherited. diff --git a/docs/qc-report.md b/docs/qc-report.md index 791e1b4..4a7aa00 100644 --- a/docs/qc-report.md +++ b/docs/qc-report.md @@ -125,8 +125,8 @@ there is nothing to configure - add a column and the matching panel reacts. | Column (matched by name) | Examples | What it drives | | :--- | :--- | :--- | | **sample id** | `sample`, `sample_id`, `name`, `gid`, `strain`, `isolate`… | Keys the metadata to each sample's VCF / stats (falls back to the first column). | -| **time** | `timepoint`, `day`, `date`, `week`, `month`, `hour`, `passage`, `generation`, `visit`, `tp`, `t0`… | The x-axis of the **SNP dynamics** trajectories. | -| **group / series** | `group`, `patient`, `series`, `host`, `subject`, `cluster`, `experiment`, `donor`, `case`, `replicate`, `chain`, `samples`… | Connects samples into one longitudinal series (a trajectory set per group) for **SNP dynamics** and **epistasis**, and for what each series **gained since its first time point**. On *Relatedness*, a sample far from the rest of its group is flagged `GROUP_MISMATCH`. | +| **time** | `timepoint`, `day`, `date`, `week`, `month`, `hour`, `passage`, `generation`, `visit`, `tp`, `t0`… | The x-axis of the **SNP dynamics** trajectories, and the order of each series. A date is read as a date (`2021-03-01`, or day first: `15/01/2020`, unless the day gives it away, as in `03/15/2020`), a word for the start (`baseline`, `pre`, `start`) comes first, and anything else by its first number (`P3`, `day 14`). | +| **group / series** | `group`, `patient`, `series`, `host`, `subject`, `cluster`, `experiment`, `donor`, `case`, `replicate`, `chain`, `samples`… | Connects samples into one longitudinal series (a trajectory set per group) for **SNP dynamics** and **epistasis**, and for what each series **gained since its first time point**: its earliest sample the QC keeps and places in the series. On *Relatedness*, a sample far from the rest of its group is flagged `GROUP_MISMATCH`. A DNA-extract column (below) is never taken for the group. | | **DNA extract** | `dna_id`, `dna`, `extract`, `extract_id`, `biosample`, `specimen_id`, `isolate_id`, `replicate_of`… | Names the libraries of one DNA: the **Minority variants** panel compares them to measure how low an allele frequency is reproduced. Samples sharing a FASTQ file are not compared. | | **any other column** | `treatment`, `site`, `region`, `ward`, `batch`… | A categorical annotation - becomes a **cohort filter** dropdown in the toolbar (restrict the whole QC view to one value) and a SNP-matrix header level. | | **collection date** | `collection_date`, `sampling_date`, `isolation_date`, `date`, `year`, `fecha` | The *Sampling dates* panel (Diagnostics) and the date in each sample's profile. The day the pipeline processed a sample is never read as one. | diff --git a/tests/js/summary.test.mjs b/tests/js/summary.test.mjs index 4267355..aa9e048 100644 --- a/tests/js/summary.test.mjs +++ b/tests/js/summary.test.mjs @@ -231,3 +231,19 @@ describe("findGconv", () => { assert.deepEqual(host(g.failSamples), ["C"], "B shares chr:1 with a kept sample"); }); }); + +describe("seriesGainSummary, what the series gained since their first time point", () => { + it("summarises series without the samples the chart leaves out, and with a true median", () => { + const row = (s, time, n) => ({ s, time, tnum: +time, new: Array(n).fill("c:1"), risen: [], unknown: 0, lost: [] }); + const rep = report([sample("a1", "L7"), sample("a2", "L7"), sample("bad", "L7", {}, { v: "WARN", f: ["LINEAGE_MISMATCH"] }), + sample("b1", "L7")], { + series: { checked: true, groups: [ + { group: "A", rows: [row("a1", "6", 0), row("a2", "6", 10), row("bad", "9", 2462)] }, + { group: "B", rows: [row("b1", "6", 3)] }, + ] }, + }); + const g = host(load(["seriesGainSummary", "serOutside"], rep).seriesGainSummary()); + assert.equal(g.max, 5, "A's last counted time point is 6, median of 0 and 10"); + assert.equal(g.median, 4, "the median of 5 and 3"); + }); +}); diff --git a/tests/unit/test_qc_report_logic.py b/tests/unit/test_qc_report_logic.py index 52cd20d..d4c709e 100644 --- a/tests/unit/test_qc_report_logic.py +++ b/tests/unit/test_qc_report_logic.py @@ -731,12 +731,35 @@ def test_dyn_opos_without_a_complete_original_coordinate_is_none(info): @pytest.mark.parametrize(("value", "expected"), [ - ("P3", 3.0), ("passage 12", 12.0), ("-2.5", -2.5), (7, 7.0), ("7.", 7.0), ("2025-01-15", 2025.0), + ("P3", 3.0), ("passage 12", 12.0), ("-2.5", -2.5), (7, 7.0), ("7.", 7.0), ("M6", 6.0), ]) def test_dyn_num_extracts_the_first_number_it_finds(value, expected): assert qc_parsers._dyn_num(value) == pytest.approx(expected) +def test_dyn_num_orders_dates_by_the_whole_date(): + """The first number of a day-first date is the day: 03/06/2020 came before 15/01/2020.""" + order = ["15/01/2020", "03/06/2020", "2020-07-01", "02/13/2021", "2021-03-01"] + + assert sorted(order, key=qc_parsers._dyn_num) == order + + +def test_dyn_num_puts_the_baseline_first(): + """'baseline', M3, M6: taken by their first number the baseline had none and came last.""" + assert qc_parsers._dyn_num("baseline") == 0.0 + assert sorted(["M6", "baseline", "M3"], key=qc_parsers._dyn_num) == ["baseline", "M3", "M6"] + + +def test_a_dna_extract_column_is_never_read_as_the_group(tmp_path): + """'replicate_of' matches the group pattern, but it ties libraries of one DNA together.""" + p = tmp_path / "meta.tsv" + p.write_text("sample\treplicate_of\tpatient\ttimepoint\nA\tA\tP1\t0\nB\tA\tP1\t3\n") + + meta = qc_parsers.parse_metadata(str(p)) + + assert meta["B"]["group"] == "P1" + + @pytest.mark.parametrize("value", [None, "", "abc", "NA"]) def test_dyn_num_without_a_number_is_none(value): assert qc_parsers._dyn_num(value) is None diff --git a/tests/unit/test_qc_report_series.py b/tests/unit/test_qc_report_series.py index 9ecf96d..2d48ec4 100644 --- a/tests/unit/test_qc_report_series.py +++ b/tests/unit/test_qc_report_series.py @@ -120,3 +120,39 @@ def test_read_matrix_cells_reads_only_what_is_needed(tmp_path): cells = ser.read_matrix_cells(str(p), {"c:10": {"p0"}, "c:20": {"p0"}}) assert cells == {("c:10", "p0"): (0.0, 40), ("c:20", "p0"): (None, 0)} + + +def test_a_failed_or_swapped_first_sample_is_not_the_start(): + """A swapped first time point lacks the line's SNPs and carries its own: read against it, + every later sample 'gains' the line and 'loses' the stranger.""" + m = meta(bad=("L", 0), p3=("L", 3), p6=("L", 6), p9=("L", 9)) + line = {f"c:{i}": var(1.0) for i in range(5)} + variants = {"bad": {"c:900": var(1.0)}, "p3": dict(line), "p6": dict(line, **{"c:50": var(1.0)}), + "p9": dict(line, **{"c:50": var(1.0)})} + + g = ser.build_series(m, variants, {("c:50", "p3"): (0.0, 40)}, excluded={"bad"})["groups"][0] + + assert g["base"] == ["p3"] + assert [r["s"] for r in g["rows"]] == ["p6", "p9"], "the excluded first sample is not a row either" + assert g["rows"][0]["new"] == ["c:50"] + + +def test_a_sample_whose_time_cannot_be_read_has_no_place_in_the_series(): + m = meta(a=("L", 0), b=("L", 3)) + m["x"] = {"group": "L", "time": "", "tnum": None} + variants = {"a": {}, "b": {}, "x": {"c:1": var(1.0)}} + + g = ser.build_series(m, variants, {})["groups"][0] + + assert [r["s"] for r in g["rows"]] == ["b"] + + +def test_a_site_read_at_exactly_the_consensus_minimum_is_not_read(): + """The consensus leaves a site uncalled at --consensus_min_dp reads or fewer.""" + variants = {"p0": {}, "p3": {"c:10": var(1.0)}, "p6": {}} + + at, above = (ser.build_series(M, variants, {("c:10", "p0"): (0.0, dp)}, min_dp=7)["groups"][0]["rows"][0] + for dp in (7, 8)) + + assert (at["new"], at["unknown"]) == ([], 1) + assert above["new"] == ["c:10"] From 871b0725dde29472259e0ff53cfcbb99b176724f Mon Sep 17 00:00:00 2001 From: Paula Ruiz Rodriguez <50167687+Paururo@users.noreply.github.com> Date: Wed, 23 Sep 2026 15:49:52 +0200 Subject: [PATCH 11/14] Count a missed minority only where the other library could have seen it A call counted as missed by another library of the same DNA wherever that library had --consensus_min_dp reads. At 8x a 5% minority is 0.4 alternate reads: its absence says nothing, and counting it made the low bands look like noise however real they were. A miss now counts only above --consensus_min_dp reads and where at least five alternate reads were expected at the call's fraction. The floor was the first band from the bottom to reach 80% reproduced, even when a band above it fell short again. It is now the lowest band from which every band up to fixation reaches it, and a failing top band, or too few comparisons to judge, is said as such. A DNA-extract column under which no two independent libraries share a DNA no longer reads as a floor never reached: the page says nothing was compared. --- bin/qcreport/minority.py | 13 +++++++++---- bin/qcreport/render.py | 3 ++- bin/report_assets/js/07b_summary.js | 8 +++++--- bin/report_assets/js/10c_minority.js | 11 ++++++++--- tests/js/summary.test.mjs | 24 ++++++++++++++++++++++++ tests/unit/test_qc_report_minority.py | 11 +++++++---- 6 files changed, 55 insertions(+), 15 deletions(-) diff --git a/bin/qcreport/minority.py b/bin/qcreport/minority.py index c16a745..2da1086 100644 --- a/bin/qcreport/minority.py +++ b/bin/qcreport/minority.py @@ -11,7 +11,9 @@ fraction, is where the noise ends. Two samples that share a FASTQ file share reads, so their agreement proves nothing and they are not compared; a merged sample and the runs it was merged from are the usual case. A call is only counted as missing from the other library where that -library was read at the site (the SNP matrix's depth); a site it did not read says nothing. +library was read at the site above --consensus_min_dp reads (the SNP matrix's depth) and deeply +enough to have called it: at least five alternate reads expected at the call's fraction, since +at 8x a 5% minority is not there to see, real or not. """ from __future__ import annotations @@ -19,10 +21,11 @@ import os import re +from .parsers import _TECHREP_RE as _REP_RE + FIXED = 0.9 EDGES = [0.0, 0.1, 0.2, 0.3, 0.5, 0.7, FIXED] -_REP_RE = re.compile(r"^(dna[_ ]?id|dna|extract(ion)?([_ ]?id)?|biosample([_ ]?id)?|specimen([_ ]?id)?|" - r"isolate[_ ]?id|library[_ ]?of|replicate[_ ]?of|tech(nical)?[_ ]?rep(licate)?([_ ]?of)?)$", re.I) +MIN_EXPECTED_ALT = 5 _SAMPLE_RE = re.compile(r"^(sample[_ ]?id|sample|id|name)$", re.I) @@ -159,8 +162,10 @@ def _reproducibility(variants, pairs, shared, cells, column, min_dp, fixed): hit = True else: cell = cells.get((site, y)) - if cell is None or cell[1] is None or cell[1] < min_dp: + if cell is None or cell[1] is None or cell[1] <= min_dp: continue # the other library was not read there: nothing to learn + if not cell[0] and af * cell[1] < MIN_EXPECTED_ALT: + continue # read, but too thinly to have called it at this fraction hit = bool(cell[0]) if af >= fixed: fx_t += 1 diff --git a/bin/qcreport/render.py b/bin/qcreport/render.py index e5c6a96..b85837b 100644 --- a/bin/qcreport/render.py +++ b/bin/qcreport/render.py @@ -122,7 +122,8 @@ def build_html(title, payload): "(dna_id, extract, biosample...), each library's calls are looked up in the others: a real " "minority is in the DNA and is reproduced, an error is not. Pairs that share a FASTQ file " "share reads and are left out, and a call only counts as missed where the other library " - "was read (the SNP matrix's depth).", + "was read deeply enough to have called it: above --consensus_min_dp reads, with at least " + "five alternate reads expected at the call's fraction (the SNP matrix's depth).", "drseries": "The resistance mutations (WHO grade 1-2, from pathotypr) at every time point of every " "series, with those acquired since the first time point set apart and those no longer " "called crossed out. Answers when resistance appears along a series and whether it " diff --git a/bin/report_assets/js/07b_summary.js b/bin/report_assets/js/07b_summary.js index a9e5184..fc0b68f 100644 --- a/bin/report_assets/js/07b_summary.js +++ b/bin/report_assets/js/07b_summary.js @@ -224,10 +224,12 @@ function findMinority(){ var tot=M.hist.reduce(function(a,b){return a+b;},0); if(!tot||!M.with_dp)return null; var pct=Math.round(100*M.le3/M.with_dp),r=M.replicates,body=[]; var head=''+pct+'% of the '+tot.toLocaleString('en-US')+' calls below fixation rest on three alternate reads or fewer'; - if(r&&r.tested){var low=r.tested[0]+r.tested[1],rl=r.reproduced[0]+r.reproduced[1],fl=minFloor(M); - body.push('Another library of the same DNA calls '+(low?Math.round(100*rl/low)+'%':'none')+' of those under allele fraction '+M.edges[2]+ + var tested=r&&r.tested?r.tested.reduce(function(a,b){return a+b;},0):0; + if(r&&r.tested&&tested){var low=r.tested[0]+r.tested[1],rl=r.reproduced[0]+r.reproduced[1],fl=minFloor(M); + if(low)body.push('Another library of the same DNA calls '+Math.round(100*rl/low)+'% of those under allele fraction '+M.edges[2]+ (r.fixed_tested?' and '+Math.round(100*r.fixed_reproduced/r.fixed_tested)+'% of the fixed ones':'')+'.'); - body.push(fl==null||fl<0?'No band below fixation reaches 80% reproduced.':(fl===0?'Every band is reproduced 80% or more.':'Calls reproduce 80% or more from allele fraction '+M.edges[fl]+' up.'));} + body.push(fl==null?'No band of allele fraction was compared 20 times, too few to say where the noise ends.':fl<0?'Even the highest band compared is not reproduced 80% of the time.':(fl===0?'Every band is reproduced 80% or more.':'Calls reproduce 80% or more from allele fraction '+M.edges[fl]+' up.'));} + else if(r&&r.tested)body.push('The samplesheet names DNA extracts ('+esc(r.column)+'), but no two independent libraries share one'+(r.shared?' (those that do share reads)':'')+', so nothing was compared.'); else if(r)body.push('Libraries of the same DNA are named ('+esc(r.column)+') but need the SNP matrix to be compared.'); else body.push('Naming each sample's DNA extract in the samplesheet would measure where the noise ends.'); return {k:'minor',eyebrow:'Minority variants',tone:'',head:head,body:body.join(' '),link:{href:'#minority',t:'See the calls'}, diff --git a/bin/report_assets/js/10c_minority.js b/bin/report_assets/js/10c_minority.js index aed74ea..eab65d5 100644 --- a/bin/report_assets/js/10c_minority.js +++ b/bin/report_assets/js/10c_minority.js @@ -3,9 +3,13 @@ // same DNA, how often another library of that DNA calls them too: a real minority is in the DNA, // an error is not reproduced. function minBandLabels(M){var e=M.edges;return e.slice(0,-1).map(function(lo,i){return i===0?'≤ '+e[1]:lo+'–'+e[i+1];});} -// The lowest band from which at least 80% of calls reproduce, over bands tested at least 20 times. +// The lowest band from which EVERY band up to fixation reproduces at 80% or more, bands tested +// under 20 times not counting either way: -1 when the top one fails, null with nothing tested. +// The first band that passes is not it, when a band above it falls short again. function minFloor(M){var r=M.replicates;if(!(r&&r.tested))return null; - for(var i=0;i=20&&r.reproduced[i]/r.tested[i]>=0.8)return i;}return -1;} + var fl=null;for(var i=r.tested.length-1;i>=0;i--){if(r.tested[i]<20)continue; + if(r.reproduced[i]/r.tested[i]>=0.8)fl=i;else break;} + return fl==null?(r.tested.some(function(t){return t>=20;})?-1:null):fl;} function minBars(vals,labels,fmt,color,title){var mx=Math.max.apply(null,vals.map(function(v){return v||0;}).concat([1e-9])); return '
'+vals.map(function(v,i){return '
'+(v==null?'':fmt(v))+''+ ''+labels[i]+'
';}).join('')+'
';} @@ -19,13 +23,14 @@ function renderMinority(){ if(r){ if(r.tested==null){html+='
The samplesheet names libraries of the same DNA ('+esc(r.column)+', '+r.pairs+' independent '+_plural(r.pairs,'pair')+'), '+ 'but without the SNP matrix a call another library missed cannot be told from a site it did not read, so they are not compared.
';} + else if(!r.tested.reduce(function(a,b){return a+b;},0)){html+='
The samplesheet names DNA extracts ('+esc(r.column)+'), but no two independent libraries share one'+(r.shared?' ('+r.shared+' pairs share reads and are left out)':'')+', so nothing was compared.
';} else{ var rates=r.tested.map(function(t,i){return t?r.reproduced[i]/t:null;}),fl=minFloor(M),fx=r.fixed_tested?r.fixed_reproduced/r.fixed_tested:null; html+='
Reproduced by another library of the same DNA ('+ esc(r.column)+'; '+r.pairs+' independent '+_plural(r.pairs,'pair')+(r.shared?(', '+r.shared+' left out for sharing reads'):'')+')
'+ minBars(rates,labels,function(v){return Math.round(v*100)+'%';},function(i){return rates[i]!=null&&rates[i]>=0.8?'#22a06b':(rates[i]!=null&&rates[i]>=0.5?'#e6b25a':'#e0544f');}, function(i){return r.reproduced[i]+' of '+r.tested[i]+' calls reproduced';})+ - '
'+(fl==null||fl<0?'No band of allele fraction reaches 80% reproduced'+(fx!=null?'; fixed calls reach '+Math.round(fx*100)+'%':'')+'.': + '
'+(fl==null||fl<0?(fl==null?'No band of allele fraction was compared 20 times, too few to say where the noise ends':'Even the highest band compared is not reproduced 80% of the time')+(fx!=null?'; fixed calls reach '+Math.round(fx*100)+'%':'')+'.': ((fl===0?'At every allele fraction, 80% or more of the calls are reproduced':'From allele fraction '+M.edges[fl]+' up, 80% or more of the calls are reproduced')+(fl>0?('; below it, '+Math.round(100*(rates.slice(0,fl).reduce(function(a,v,i){return a+(v||0)*r.tested[i];},0)/Math.max(1,r.tested.slice(0,fl).reduce(function(a,b){return a+b;},0))))+'% are'):'')+ (fx!=null?'. Fixed calls: '+Math.round(fx*100)+'%, the ceiling':'')+'.'))+ ' Only calls the other library was read at count; a site it did not read says nothing.
';}} diff --git a/tests/js/summary.test.mjs b/tests/js/summary.test.mjs index aa9e048..d8213cc 100644 --- a/tests/js/summary.test.mjs +++ b/tests/js/summary.test.mjs @@ -247,3 +247,27 @@ describe("seriesGainSummary, what the series gained since their first time point assert.equal(g.median, 4, "the median of 5 and 3"); }); }); + +describe("minFloor, the allele fraction from which calls are reproduced", () => { + it("reads the noise floor from the top band down", () => { + // 50/85/60/70/95/97%: the first passing band is 0.1-0.2, but 0.2-0.5 fall short again. + const tested = [100, 100, 100, 100, 100, 100]; + const rep = report([], { minority: { edges: [0, 0.1, 0.2, 0.3, 0.5, 0.7, 0.9], + replicates: { tested, reproduced: [50, 85, 60, 70, 95, 97] } } }); + assert.equal(load(["minFloor"], rep).minFloor(rep.minority), 4, "from 0.5 up"); + }); + + it("gives no floor when the band just below fixation already fails", () => { + const rep = report([], { minority: { edges: [0, 0.1, 0.2, 0.3, 0.5, 0.7, 0.9], + replicates: { tested: [100, 100, 100, 100, 100, 100], reproduced: [90, 90, 90, 90, 90, 50] } } }); + assert.equal(load(["minFloor"], rep).minFloor(rep.minority), -1); + }); + + it("gives none at all when no band was compared often enough to judge", () => { + const M = { edges: [0, 0.1, 0.2, 0.3, 0.5, 0.7, 0.9], hist: [4, 3, 2, 1, 1, 1], le3: 2, with_dp: 12, + replicates: { column: "dna_id", tested: [5, 4, 3, 2, 1, 1], reproduced: [1, 1, 1, 1, 1, 1] } }; + const fn = load(["minFloor", "findMinority"], report([], { minority: M })); + assert.equal(fn.minFloor(M), null); + assert.match(fn.findMinority().body, /too few to say where the noise ends/); + }); +}); diff --git a/tests/unit/test_qc_report_minority.py b/tests/unit/test_qc_report_minority.py index e994fa4..c3a8b83 100644 --- a/tests/unit/test_qc_report_minority.py +++ b/tests/unit/test_qc_report_minority.py @@ -69,11 +69,14 @@ def test_a_samplesheet_without_a_dna_column_has_no_replicates(tmp_path): assert mn.replicate_sets(str(p))[0] is None -def test_reproducibility_counts_only_where_the_other_library_was_read(): - variants = {"A": {"c:1": call(0.05), "c:2": call(0.06), "c:3": call(0.4), "c:9": call(1.0)}, +def test_reproducibility_counts_only_where_the_other_library_could_have_called_it(): + variants = {"A": {"c:1": call(0.05), "c:2": call(0.06), "c:3": call(0.4), "c:4": call(0.05), + "c:5": call(0.3), "c:9": call(1.0)}, "B": {"c:3": call(0.35), "c:9": call(1.0)}} - cells = {("c:1", "B"): (0.0, 80), # read, and not there: an error - ("c:2", "B"): (None, 2)} # not read: says nothing + cells = {("c:1", "B"): (0.0, 200), # read deeply, and not there: an error + ("c:2", "B"): (None, 2), # not read: says nothing + ("c:4", "B"): (0.0, 60), # read, but 5% of 60 is 3 reads: too thin to tell + ("c:5", "B"): (0.0, 7)} # at the consensus minimum: not read rep = mn.build_minority(variants, [("A", "B")], 0, cells, "dna_id", min_dp=7)["replicates"] From d9cd75a6c986093af2d81a14659bff62b85f541f Mon Sep 17 00:00:00 2001 From: Paula Ruiz Rodriguez <50167687+Paururo@users.noreply.github.com> Date: Wed, 23 Sep 2026 16:01:07 +0200 Subject: [PATCH 12/14] Read an NA matrix cell as a call, not as a site read without one The SNP matrix writes NA where a sample was called at a fraction its files do not keep. The report read it as an empty cell, so with the site's depth beside it, a series' first time point that carried the allele counted as read without it, which made the later call new, and a library that called it counted as having missed it. An NA cell is now a call of unknown fraction: unknown in a series, reproduced among the libraries of one DNA. --- bin/qcreport/minority.py | 2 +- bin/qcreport/series.py | 12 ++++++++++-- tests/unit/test_qc_report_minority.py | 11 +++++++++++ tests/unit/test_qc_report_series.py | 16 ++++++++++++++++ 4 files changed, 38 insertions(+), 3 deletions(-) diff --git a/bin/qcreport/minority.py b/bin/qcreport/minority.py index 2da1086..e78b462 100644 --- a/bin/qcreport/minority.py +++ b/bin/qcreport/minority.py @@ -166,7 +166,7 @@ def _reproducibility(variants, pairs, shared, cells, column, min_dp, fixed): continue # the other library was not read there: nothing to learn if not cell[0] and af * cell[1] < MIN_EXPECTED_ALT: continue # read, but too thinly to have called it at this fraction - hit = bool(cell[0]) + hit = bool(cell[0]) # an NA cell (CALLED) is a call too, of unknown fraction if af >= fixed: fx_t += 1 fx_r += hit diff --git a/bin/qcreport/series.py b/bin/qcreport/series.py index e9b0818..a1e7459 100644 --- a/bin/qcreport/series.py +++ b/bin/qcreport/series.py @@ -71,9 +71,15 @@ def needed_cells(meta, variants, fixed=FIXED, excluded=()): return need +# An NA cell of the SNP matrix: the sample was called there, at a fraction its files do not keep. +# It is neither absent nor a known fraction, so it is kept apart from an empty cell. +CALLED = "called" + + def read_matrix_cells(path, need): """{(site, sample): (af, dp)} of the SNP matrix TSV (build_snp_matrix.py) for the cells in - `need`. af and dp are None where the cell is empty. Reads only the rows it needs.""" + `need`. af and dp are None where the cell is empty, af is CALLED where it reads NA. Reads + only the rows it needs.""" out = {} if not need or not path or not os.path.exists(path) or os.path.basename(path).startswith("NO_FILE"): return out @@ -94,7 +100,7 @@ def read_matrix_cells(path, need): i = col.get(s) if i is None or i + 1 >= len(c): continue - af = float(c[i]) if c[i] not in ("", "NA") else None + af = CALLED if c[i] == "NA" else (float(c[i]) if c[i] else None) dp = int(float(c[i + 1])) if c[i + 1] not in ("", "NA") else None out[(site, s)] = (af, dp) except (OSError, ValueError): @@ -111,6 +117,8 @@ def _state(site, sample, variants, cells, min_dp, fixed): if cell is None: return "unchecked" af, dp = cell + if af == CALLED: + return "unchecked" # carried, at a fraction nobody knows: not absent, not fixed if af is not None and af > 0: return "fixed" if af >= fixed else "minority" if dp is None: diff --git a/tests/unit/test_qc_report_minority.py b/tests/unit/test_qc_report_minority.py index c3a8b83..c68fa5a 100644 --- a/tests/unit/test_qc_report_minority.py +++ b/tests/unit/test_qc_report_minority.py @@ -85,6 +85,17 @@ def test_reproducibility_counts_only_where_the_other_library_could_have_called_i assert (rep["fixed_tested"], rep["fixed_reproduced"]) == (2, 2) +def test_an_na_cell_of_the_other_library_is_a_call_it_made(): + """NA in the matrix is a call whose fraction the files do not keep, not a site read without it.""" + from qcreport.series import CALLED + variants = {"A": {"c:1": call(0.4)}, "B": {}} + + rep = mn.build_minority(variants, [("A", "B")], 0, {("c:1", "B"): (CALLED, 80)}, "dna_id", + min_dp=7)["replicates"] + + assert rep["tested"][3] == 1 and rep["reproduced"][3] == 1 + + def test_without_the_matrix_the_replicates_are_not_measured(): variants = {"A": {"c:1": call(0.05)}, "B": {}} diff --git a/tests/unit/test_qc_report_series.py b/tests/unit/test_qc_report_series.py index 2d48ec4..a543f79 100644 --- a/tests/unit/test_qc_report_series.py +++ b/tests/unit/test_qc_report_series.py @@ -122,6 +122,22 @@ def test_read_matrix_cells_reads_only_what_is_needed(tmp_path): assert cells == {("c:10", "p0"): (0.0, 40), ("c:20", "p0"): (None, 0)} +def test_an_na_cell_is_a_call_of_unknown_fraction_not_an_absence(tmp_path): + """The matrix writes NA for a call whose read counts its files do not keep: the start carried + the allele, so a later fixed call is not new, only not known to have risen.""" + p = tmp_path / "m.tsv" + p.write_text("reference\tcontig\tpos\tref_allele\talt_allele\tgene\teffect\taa_change\t" + "p0|AF\tp0|DP\n" + "R\tc\t10\tA\tT\t\t\t\tNA\t40\n") + cells = ser.read_matrix_cells(str(p), {"c:10": {"p0"}}) + variants = {"p0": {}, "p3": {"c:10": var(1.0)}, "p6": {}} + + row = ser.build_series(M, variants, cells)["groups"][0]["rows"][0] + + assert cells == {("c:10", "p0"): (ser.CALLED, 40)} + assert (row["new"], row["risen"], row["unknown"]) == ([], [], 1) + + def test_a_failed_or_swapped_first_sample_is_not_the_start(): """A swapped first time point lacks the line's SNPs and carries its own: read against it, every later sample 'gains' the line and 'loses' the stranger.""" From 131f3aab86b97f606b5e0b9dbf2c4ad294d0ba63 Mon Sep 17 00:00:00 2001 From: Paula Ruiz Rodriguez <50167687+Paururo@users.noreply.github.com> Date: Wed, 23 Sep 2026 16:01:46 +0200 Subject: [PATCH 13/14] Correct what the changelog says of the cohort analyses The relatedness entry credited GROUP_MISMATCH with finding exactly the samples typed as another lineage than their line, at distances that were only approximated; on those distances 18 of the 20 samples it flags are. The resistance example now says which lines acquire which atpE mutation. The entries also say what the fixes changed: a missed minority needs a library that could have called it, a series starts at a sample it can be read against and reads dates as dates, deletions are joined by reciprocal overlap, and callable means the depth from which the consensus calls a base. --- CHANGELOG.md | 44 +++++++++++++++++++++++++------------------- 1 file changed, 25 insertions(+), 19 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8d358fe..d7c690a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,19 +15,22 @@ based on [Keep a Changelog](https://keepachangelog.com/), and the project follow `biosample`), libraries of the same DNA are compared: a real minority is in the DNA and another library calls it too, an error is not reproduced. Pairs that share a FASTQ file (a merged sample and its runs) or were mapped against different references are left out, and a call only counts - as missed where the other library was read. + as missed where the other library could have called it: above `--consensus_min_dp` reads, with + at least five alternate reads expected at the call's fraction. - **What each series gained since its first time point.** For every samplesheet group followed - over time (a patient, a passage line), the report lists the SNPs each later sample carries - fixed that the group's first time point did not: *new* where that first time point was read at - the site without the allele, *risen* where it held the allele as a minority, *unknown* where it - was not read there, and *lost* for those fixed at the start and read without the allele later. - The depth comes from the SNP matrix, which the report now reads, so a site the start never read - is not counted as new. A chart follows each series through the median of each time point, + over time (a patient, a passage line), the report lists the SNPs each later sample carries fixed + that the group's first time point did not: *new* where that first time point was read at the + site without the allele, *risen* where it held the allele as a minority, *unknown* where it was + not read there, and *lost* for those fixed at the start and read without the allele later. The + depth comes from the SNP matrix, which the report now reads, so a site the start never read is + not counted as new. The first time point is the earliest with a sample the QC keeps and places + in the series, and a time that is a date is read as one (`2021-03-01`, `15/01/2020`), with + `baseline` or `pre` first. A chart follows each series through the median of each time point, coloured by treatment, and leaves out the samples the QC fails or places outside their series. On the *Resistance* page, each series' grade 1-2 mutations are laid out per time point with - those acquired since the start set apart: on the cohort that motivated it, the bedaquiline - lines of one lineage acquire atpE E61D and I66M at passages 18 and 19. + those acquired since the start set apart: on the cohort that motivated it, the three bedaquiline + lines of one lineage acquire atpE I66M at passages 18 and 19, two of them with E61D. - **How close the samples are to each other.** A new step, `SNP_DISTANCES`, counts the SNPs between every two consensus sequences over the positions both called, so a no-call, a masked @@ -38,20 +41,23 @@ based on [Keep a Changelog](https://keepachangelog.com/), and the project follow the clusters at a threshold that can be moved live (`--snp_cluster_threshold`, 12 by default, the usual *M. tuberculosis* cut), and the samples far from the rest of their samplesheet group. Those are flagged `GROUP_MISMATCH` (WARN) when their group is otherwise tight: on the cohort - that motivated it, the 20 samples it finds are the ones typed as another lineage than their - line, which sat 47 to 72 SNPs from each other and 2,000 from their line mates. + that motivated it, with distances approximated from its SNP matrix, 18 of the 20 samples it + flags are the ones typed as another lineage than their line. A sample mapped against two + references is compared on each, as `sample@reference`. - **Deletions, and SNPs per callable kb along the genome.** Each sample's all-positions VCF is reduced to its depth per window, the stretches no read covers and a per-gene table (`DEPTH_PROFILE`, published under `stats/`). The report compares those stretches across the - samples of each reference: one a sample lacks while the others read it is a deletion, private - or shared; one nobody reads is a repeat or a part of the reference none of these genomes has, - and is listed apart instead of being counted as anyone's. A *Deletions* panel lists them with - the genes they remove and zooms the landscape onto each, and `_deletions.tsv` - keeps them with every carrier's own coordinates. The landscape gains a *Deletions* track and a - *SNPs / kb* track that divides each bin's SNPs by the positions deep enough to call, so a bin - half of which was not read no longer looks half as variable. Samples under - `--report_depth_min` are not assessed: stretches without reads turn up there by chance. + samples of each reference: one a sample lacks while the others read it is a deletion, private or + shared; one nearly every sample lacks (90% or more) is a repeat or a part of the reference none + of these genomes has, and is listed apart instead of being counted as anyone's. Two samples' + stretches are one deletion when each covers at least half of the other. A *Deletions* panel + lists them with the genes they remove and zooms the landscape onto each, and + `_deletions.tsv` keeps them with every carrier's own coordinates. The landscape + gains a *Deletions* track and a *SNPs / kb* track that divides each bin's SNPs by the positions + deep enough for the consensus to call a base (`--allpos_min_cov` reads), so a bin half of which + was not read no longer looks half as variable. Samples under `--report_depth_min` are not + assessed: stretches without reads turn up there by chance. - **Two guards against a sample mapped to a genome it does not belong to.** In a 185-sample cohort, 22 samples annotated as one lineage were really another, were routed to that lineage's From 4652d6df48db5c9ab84307a2eb2a7db709810e1a Mon Sep 17 00:00:00 2001 From: Paula Ruiz Rodriguez <50167687+Paururo@users.noreply.github.com> Date: Wed, 23 Sep 2026 16:07:30 +0200 Subject: [PATCH 14/14] Judge whether a library could have made a call the same way either way A call counted as reproduced wherever the other library of its DNA made it, but as missed only where that library was deep enough to have made it. Whether a comparison counted depended on its outcome: a 5% call the other library made at 40 reads was reproduced, one it did not make at 40 reads was left out as too thin to tell, and the thin bands filled with the calls made by luck. The other library's depth, from its call where it made one and from the SNP matrix where it did not, now decides for both outcomes. --- CHANGELOG.md | 6 +++--- bin/qcreport/minority.py | 29 ++++++++++++++++----------- bin/qcreport/render.py | 6 +++--- bin/report_assets/js/10c_minority.js | 2 +- tests/unit/test_qc_report_minority.py | 14 +++++++++++++ 5 files changed, 38 insertions(+), 19 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d7c690a..2e966a2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,9 +14,9 @@ based on [Keep a Changelog](https://keepachangelog.com/), and the project follow samplesheet names each sample's DNA extract (a column such as `dna_id`, `extract` or `biosample`), libraries of the same DNA are compared: a real minority is in the DNA and another library calls it too, an error is not reproduced. Pairs that share a FASTQ file (a merged sample - and its runs) or were mapped against different references are left out, and a call only counts - as missed where the other library could have called it: above `--consensus_min_dp` reads, with - at least five alternate reads expected at the call's fraction. + and its runs) or were mapped against different references are left out, and a call only counts, + reproduced or not, where the other library could have called it: above `--consensus_min_dp` + reads, with at least five alternate reads expected at the call's fraction. - **What each series gained since its first time point.** For every samplesheet group followed over time (a patient, a passage line), the report lists the SNPs each later sample carries fixed diff --git a/bin/qcreport/minority.py b/bin/qcreport/minority.py index e78b462..e20dff8 100644 --- a/bin/qcreport/minority.py +++ b/bin/qcreport/minority.py @@ -10,10 +10,12 @@ one library makes that another library of the same DNA also makes, in each band of allele fraction, is where the noise ends. Two samples that share a FASTQ file share reads, so their agreement proves nothing and they are not compared; a merged sample and the runs it was merged -from are the usual case. A call is only counted as missing from the other library where that -library was read at the site above --consensus_min_dp reads (the SNP matrix's depth) and deeply -enough to have called it: at least five alternate reads expected at the call's fraction, since -at 8x a 5% minority is not there to see, real or not. +from are the usual case. A call is only compared where the other library was read deeply +enough to have called it: above --consensus_min_dp reads, with at least five alternate reads +expected at the call's fraction, since at 8x a 5% minority is not there to see, real or not. +The depth is the other library's, from its call where it made one and from the SNP matrix +where it did not, and the test is the same either way: counting its calls wherever they fall +but its misses only where it was deep enough would fill the thin bands with lucky calls. """ from __future__ import annotations @@ -148,7 +150,8 @@ def build_minority(variants, pairs=(), shared=0, cells=None, column=None, min_dp def _reproducibility(variants, pairs, shared, cells, column, min_dp, fixed): """Per band of allele fraction, the calls of one library the other library of its pair - also makes, among those the other library was read at; and the same for fixed calls.""" + also makes, among those the other library was read deeply enough to make; and the same for + fixed calls.""" tested, repro = [0] * (len(EDGES) - 1), [0] * (len(EDGES) - 1) fx_t = fx_r = 0 for a, b in pairs: @@ -158,15 +161,17 @@ def _reproducibility(variants, pairs, shared, cells, column, min_dp, fixed): af = v.get("af") if not af: continue - if site in vy and (vy[site].get("af") or 0) > 0: - hit = True + w = vy.get(site) + if w and (w.get("af") or 0) > 0: + hit, dp = True, w.get("dp") else: cell = cells.get((site, y)) - if cell is None or cell[1] is None or cell[1] <= min_dp: - continue # the other library was not read there: nothing to learn - if not cell[0] and af * cell[1] < MIN_EXPECTED_ALT: - continue # read, but too thinly to have called it at this fraction - hit = bool(cell[0]) # an NA cell (CALLED) is a call too, of unknown fraction + if cell is None: + continue + hit, dp = bool(cell[0]), cell[1] # an NA cell (CALLED) is a call too + # whether the other library could have made the call, judged on its depth alone + if dp is None or dp <= min_dp or af * dp < MIN_EXPECTED_ALT: + continue if af >= fixed: fx_t += 1 fx_r += hit diff --git a/bin/qcreport/render.py b/bin/qcreport/render.py index b85837b..8c5cb4c 100644 --- a/bin/qcreport/render.py +++ b/bin/qcreport/render.py @@ -121,9 +121,9 @@ def build_html(title, payload): "and a real minority look the same. When the samplesheet names libraries of the same DNA " "(dna_id, extract, biosample...), each library's calls are looked up in the others: a real " "minority is in the DNA and is reproduced, an error is not. Pairs that share a FASTQ file " - "share reads and are left out, and a call only counts as missed where the other library " - "was read deeply enough to have called it: above --consensus_min_dp reads, with at least " - "five alternate reads expected at the call's fraction (the SNP matrix's depth).", + "share reads and are left out, and a call only counts, reproduced or not, where the other " + "library was read deeply enough to have called it: above --consensus_min_dp reads, with at " + "least five alternate reads expected at the call's fraction.", "drseries": "The resistance mutations (WHO grade 1-2, from pathotypr) at every time point of every " "series, with those acquired since the first time point set apart and those no longer " "called crossed out. Answers when resistance appears along a series and whether it " diff --git a/bin/report_assets/js/10c_minority.js b/bin/report_assets/js/10c_minority.js index eab65d5..5a2f62e 100644 --- a/bin/report_assets/js/10c_minority.js +++ b/bin/report_assets/js/10c_minority.js @@ -33,7 +33,7 @@ function renderMinority(){ '
'+(fl==null||fl<0?(fl==null?'No band of allele fraction was compared 20 times, too few to say where the noise ends':'Even the highest band compared is not reproduced 80% of the time')+(fx!=null?'; fixed calls reach '+Math.round(fx*100)+'%':'')+'.': ((fl===0?'At every allele fraction, 80% or more of the calls are reproduced':'From allele fraction '+M.edges[fl]+' up, 80% or more of the calls are reproduced')+(fl>0?('; below it, '+Math.round(100*(rates.slice(0,fl).reduce(function(a,v,i){return a+(v||0)*r.tested[i];},0)/Math.max(1,r.tested.slice(0,fl).reduce(function(a,b){return a+b;},0))))+'% are'):'')+ (fx!=null?'. Fixed calls: '+Math.round(fx*100)+'%, the ceiling':'')+'.'))+ - ' Only calls the other library was read at count; a site it did not read says nothing.
';}} + ' A call only counts, reproduced or not, where the other library was read deeply enough to have made it.
';}} else html+='
Add a column naming each sample's DNA extract to the samplesheet (dna_id, extract, biosample...) and libraries of the same DNA are compared to measure where the noise ends.
'; var rows=Object.keys(M.samples).map(function(s){var x=M.samples[s];return {s:s,x:x};}).sort(function(a,b){return b.x.n-a.x.n;}); var big=el('minorityPanel')&&el('minorityPanel').classList.contains('expanded'),cap=big?rows.length:25; diff --git a/tests/unit/test_qc_report_minority.py b/tests/unit/test_qc_report_minority.py index c68fa5a..299b3ff 100644 --- a/tests/unit/test_qc_report_minority.py +++ b/tests/unit/test_qc_report_minority.py @@ -85,6 +85,20 @@ def test_reproducibility_counts_only_where_the_other_library_could_have_called_i assert (rep["fixed_tested"], rep["fixed_reproduced"]) == (2, 2) +def test_a_call_the_other_library_made_counts_only_where_it_could_have_made_it(): + """Its depth decides, not its call: counting its calls wherever they fall but its misses only + where it was deep enough fills the thin bands with lucky calls (100% reproduced at 5%).""" + variants = {"A": {"c:1": call(0.05, 40), "c:2": call(0.05, 200), "c:3": call(0.05, 40), + "c:4": call(0.05, 40)}, + "B": {"c:1": call(0.05, 40), "c:2": call(0.05, 200), "c:3": call(0.4, 6)}} + cells = {("c:4", "B"): (0.0, 40)} + + rep = mn.build_minority(variants, [("A", "B")], 0, cells, "dna_id", min_dp=7)["replicates"] + + assert (rep["tested"][0], rep["reproduced"][0]) == (2, 2), "c:2 both ways; 5% of 40 or of 6 is too thin" + assert (rep["tested"][3], rep["reproduced"][3]) == (1, 1), "B's 40% at c:3, which A's 40 reads see" + + def test_an_na_cell_of_the_other_library_is_a_call_it_made(): """NA in the matrix is a call whose fraction the files do not keep, not a site read without it.""" from qcreport.series import CALLED