audit-labs/audit-report
Turn audit-tools evidence packages into control-mapped, auditor-ready reports.
clone: git clone https://gitbay.org/audit-labs/audit-report.git
e1665106acf579481649d470be9d413338ad4875
verified · cmc
author: Christian Cleberg <hello@cleberg.net> · 2026-08-06T07:23:16Z
README.md | 25 +- audit_report/cli.py | 77 ++++- audit_report/diff.py | 381 +++++++++++++++++++++ audit_report/reporters/html.py | 4 +- .../aws_audit_acme_2025-12-01/account_security.csv | 2 + .../aws_audit_acme_2025-12-01/cloudtrail.csv | 2 + .../aws_audit_acme_2025-12-01/iam_users.csv | 3 + .../open_security_groups.csv | 2 + .../aws_audit_acme_2025-12-01/password_policy.csv | 2 + .../aws_audit_acme_2025-12-01/s3_public_access.csv | 0 tests/test_diff.py | 113 ++++++ 11 files changed, 593 insertions(+), 18 deletions(-) @@ -41,6 +41,28 @@ audit-report ./output/github_audit_acme_2026-07-29 --format md,html,json --out r audit-report ./output/aws_audit_prod_2026-07-29 --fail-on high --out report/ ``` +### Diff mode — compare two packages + +Pass `--baseline` to report how a package **drifted** from an earlier one. Both +are evaluated with the same ruleset; the report classifies each rule as +regressed, fixed, drifted (an ongoing failure whose evidence rows changed), +changed, or unchanged — and shows exactly which evidence rows appeared or +disappeared. + +```bash +# How did prod change between two audit runs? +audit-report ./output/aws_audit_prod_2026-07-29 \ + --baseline ./output/aws_audit_prod_2026-06-29 \ + --format md,html --out drift/ + +# Fail CI if this change regressed any high-severity control +audit-report ./output/aws_audit_prod_2026-07-29 \ + --baseline ./output/aws_audit_prod_2026-06-29 --fail-on high +``` + +In diff mode `--fail-on` gates on **regressions** at or above the given +severity, and output files are named `diff.*` instead of `report.*`. + You can also run it without installing: ```bash @@ -51,10 +73,11 @@ python -m audit_report ./output/aws_audit_default_2026-07-29 | Flag | Description | | --- | --- | +| `--baseline PATH` | Diff mode: report how `PACKAGE` drifted from this earlier package. | | `--ruleset PATH` | Use a specific ruleset instead of the bundled one for the detected platform. | | `--format md,html,json` | One or more output formats (default: `md`). | | `--out DIR` | Write `report.<ext>` files into `DIR`. Without it, the first format prints to stdout. | -| `--fail-on low\|medium\|high\|none` | Exit non-zero when a failing finding meets this severity (default: `none`). | +| `--fail-on low\|medium\|high\|none` | Exit non-zero when a failing finding — or, in diff mode, a regression — meets this severity (default: `none`). | ## What a report contains @@ -6,7 +6,7 @@ import argparse import sys from pathlib import Path -from . import __version__, reporters +from . import __version__, diff, reporters from .engine import FAIL, evaluate from .loader import load_package from .rules import load_ruleset @@ -34,6 +34,10 @@ def _parse_args(argv: list[str]) -> argparse.Namespace: description="Turn an audit-tools evidence package into a control-mapped report.", ) parser.add_argument("package", help="path to an audit-tools output package directory") + parser.add_argument( + "--baseline", + help="path to an earlier package; diff mode reports how PACKAGE drifted from it", + ) parser.add_argument( "--ruleset", help="path to a ruleset YAML (default: bundled ruleset for the detected platform)", @@ -51,7 +55,10 @@ def _parse_args(argv: list[str]) -> argparse.Namespace: "--fail-on", choices=["low", "medium", "high", "none"], default="none", - help="exit non-zero if any finding fails at or above this severity (default: none)", + help=( + "exit non-zero if any finding fails (or, in diff mode, regresses) at or " + "above this severity (default: none)" + ), ) parser.add_argument("--version", action="version", version=f"audit-report {__version__}") return parser.parse_args(argv) @@ -68,9 +75,58 @@ def _exit_code(findings, threshold: str) -> int: return 1 if breached else 0 +def _emit(render_one, formats: list[str], out: str | None, basename: str) -> None: + """Write one file per format into *out*, or print the first to stdout.""" + if out: + out_dir = Path(out) + out_dir.mkdir(parents=True, exist_ok=True) + for fmt in formats: + dest = out_dir / f"{basename}.{reporters.EXTENSIONS[fmt]}" + dest.write_text(render_one(fmt), encoding="utf-8") + print(f"wrote {dest}", file=sys.stderr) + else: + print(render_one(formats[0]), end="") + + +def _run_diff(args, package, ruleset, formats: list[str]) -> int: + try: + baseline = load_package(args.baseline) + except (FileNotFoundError, ValueError) as exc: + print(f"error: {exc}", file=sys.stderr) + return 2 + if baseline.platform != package.platform: + print( + f"error: cannot diff a {baseline.platform} package against a " + f"{package.platform} package", + file=sys.stderr, + ) + return 2 + + old = evaluate(baseline, ruleset) + new = evaluate(package, ruleset) + report = diff.build_diff(baseline, package, old, new) + + _emit(lambda fmt: diff.render(report, fmt), formats, args.out, "diff") + + counts = report.counts + print( + f"{package.platform}/{package.subject}: " + f"{counts[diff.REGRESSED]} regressed, {counts[diff.FIXED]} fixed, " + f"{counts[diff.DRIFTED]} drifted", + file=sys.stderr, + ) + return 1 if diff.has_regression(report, args.fail_on) else 0 + + def main(argv: list[str] | None = None) -> int: args = _parse_args(argv if argv is not None else sys.argv[1:]) + formats = [f.strip() for f in args.format.split(",") if f.strip()] + unknown = [f for f in formats if f not in reporters.EXTENSIONS] + if not formats or unknown: + print(f"error: unknown format(s): {', '.join(unknown) or '(none given)'}", file=sys.stderr) + return 2 + try: package = load_package(args.package) except (FileNotFoundError, ValueError) as exc: @@ -80,21 +136,12 @@ def main(argv: list[str] | None = None) -> int: ruleset_path = Path(args.ruleset) if args.ruleset else _default_ruleset(package.platform) ruleset = load_ruleset(ruleset_path) + if args.baseline: + return _run_diff(args, package, ruleset, formats) + findings = evaluate(package, ruleset) report = reporters.build_report(package, findings) - formats = [f.strip() for f in args.format.split(",") if f.strip()] - - if args.out: - out_dir = Path(args.out) - out_dir.mkdir(parents=True, exist_ok=True) - for fmt in formats: - content = reporters.render(report, fmt) - dest = out_dir / f"report.{reporters.EXTENSIONS[fmt]}" - dest.write_text(content, encoding="utf-8") - print(f"wrote {dest}", file=sys.stderr) - else: - # No --out: emit the first requested format to stdout. - print(reporters.render(report, formats[0]), end="") + _emit(lambda fmt: reporters.render(report, fmt), formats, args.out, "report") counts = report.counts print( new file mode 100644 @@ -0,0 +1,381 @@ +"""Diff mode — compare two evidence packages and report drift. + +Both packages are evaluated with the same ruleset; this module compares the two +sets of findings and classifies each rule's change: + +* **regressed** — a control that was supported (or not yet observed) now fails +* **fixed** — a control that failed now passes (or is no longer observed) +* **drifted** — an ongoing failure whose failing evidence rows changed +* **changed** — a non-failure status change (e.g. a table stopped being collected) +* **unchanged** — same status, same evidence + +Evidence rows are compared as whole rows, so drift shows exactly which items +appeared or disappeared (a new MFA-less user, a security group that was closed). +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from datetime import datetime, timezone +from html import escape + +from .engine import FAIL, Finding +from .loader import Package +from .reporters.html import CSS as _CSS + +ABSENT = "absent" # rule present in only one of the two packages + +REGRESSED = "regressed" +FIXED = "fixed" +DRIFTED = "drifted" +CHANGED = "changed" +UNCHANGED = "unchanged" + +# Order categories appear in a report and how they roll up in the summary. +CATEGORY_ORDER = [REGRESSED, FIXED, DRIFTED, CHANGED, UNCHANGED] +_STATUS_LABEL = {FAIL: "fail", "pass": "pass", "not_applicable": "n/a", ABSENT: "absent"} +_SEVERITY_ORDER = {"high": 0, "medium": 1, "low": 2} + + +def _row_key(row: dict[str, str]) -> tuple: + return tuple(sorted(row.items())) + + +@dataclass +class RuleDelta: + """How one rule's finding changed between the two packages.""" + + rule: object # audit_report.rules.Rule + old_status: str + new_status: str + category: str + new_reason: str = "" + evidence_added: list[dict[str, str]] = field(default_factory=list) + evidence_removed: list[dict[str, str]] = field(default_factory=list) + + @property + def controls(self) -> list[str]: + return self.rule.controls + + +def _classify(old_status, new_status, added, removed) -> str: + if old_status == ABSENT: + return REGRESSED if new_status == FAIL else CHANGED + if new_status == ABSENT: + return CHANGED # rule dropped from the current ruleset + if new_status == FAIL and old_status != FAIL: + return REGRESSED + if old_status == FAIL and new_status != FAIL: + return FIXED + if old_status == FAIL and new_status == FAIL: + return DRIFTED if (added or removed) else UNCHANGED + return CHANGED if old_status != new_status else UNCHANGED + + +def diff_findings(old: list[Finding], new: list[Finding]) -> list[RuleDelta]: + """Compare two finding lists (same ruleset) into a list of deltas.""" + old_by_id = {f.rule.id: f for f in old} + new_by_id = {f.rule.id: f for f in new} + + # New order first (ruleset order), then any rules only the baseline had. + ordered_ids = [f.rule.id for f in new] + ordered_ids += [f.rule.id for f in old if f.rule.id not in new_by_id] + + deltas: list[RuleDelta] = [] + for rid in ordered_ids: + of, nf = old_by_id.get(rid), new_by_id.get(rid) + rule = (nf or of).rule + old_status = of.status if of else ABSENT + new_status = nf.status if nf else ABSENT + + old_ev = {_row_key(r): r for r in (of.evidence if of else [])} + new_ev = {_row_key(r): r for r in (nf.evidence if nf else [])} + added = [r for k, r in new_ev.items() if k not in old_ev] + removed = [r for k, r in old_ev.items() if k not in new_ev] + + category = _classify(old_status, new_status, added, removed) + deltas.append( + RuleDelta( + rule=rule, + old_status=old_status, + new_status=new_status, + category=category, + new_reason=nf.reason if nf else "", + evidence_added=added, + evidence_removed=removed, + ) + ) + return deltas + + +@dataclass +class DiffReport: + """A computed comparison of two packages, ready to render.""" + + baseline: Package + current: Package + deltas: list[RuleDelta] + generated_at: str + + def by_category(self, category: str) -> list[RuleDelta]: + rows = [d for d in self.deltas if d.category == category] + return sorted(rows, key=lambda d: _SEVERITY_ORDER.get(d.rule.severity, 1)) + + @property + def counts(self) -> dict[str, int]: + return {cat: len(self.by_category(cat)) for cat in CATEGORY_ORDER} + + +def build_diff( + baseline: Package, + current: Package, + old_findings: list[Finding], + new_findings: list[Finding], +) -> DiffReport: + """Assemble a :class:`DiffReport` with a UTC timestamp.""" + stamp = datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M UTC") + return DiffReport( + baseline=baseline, + current=current, + deltas=diff_findings(old_findings, new_findings), + generated_at=stamp, + ) + + +def has_regression(diff: DiffReport, threshold: str) -> bool: + """True if any regressed rule meets the severity *threshold* ('none' = off).""" + if threshold == "none": + return False + floor = _SEVERITY_ORDER[threshold] + return any( + _SEVERITY_ORDER.get(d.rule.severity, 1) <= floor + for d in diff.by_category(REGRESSED) + ) + + +# --------------------------------------------------------------------------- # +# Rendering +# --------------------------------------------------------------------------- # + +_CATEGORY_HEADING = { + REGRESSED: "Regressions — a control is no longer supported", + FIXED: "Fixed — a previously failing control now passes", + DRIFTED: "Ongoing failures with changed evidence", + CHANGED: "Other status changes", + UNCHANGED: "Unchanged", +} + + +def _transition(delta: RuleDelta) -> str: + return f"{_STATUS_LABEL.get(delta.old_status, delta.old_status)} → {_STATUS_LABEL.get(delta.new_status, delta.new_status)}" + + +def _md_table(rows: list[dict[str, str]]) -> list[str]: + if not rows: + return [] + shown = rows[:10] + headers = list(shown[0].keys()) + out = [ + "| " + " | ".join(headers) + " |", + "| " + " | ".join("---" for _ in headers) + " |", + ] + out += ["| " + " | ".join(str(r.get(h, "")) for h in headers) + " |" for r in shown] + if len(rows) > len(shown): + out.append(f"\n_+{len(rows) - len(shown)} more row(s) omitted._") + return out + + +def _render_md(diff: DiffReport) -> str: + counts = diff.counts + out: list[str] = [] + out.append(f"# Evidence Drift — {diff.current.subject} ({diff.current.platform})") + out.append("") + out.append(f"- **Baseline:** `{diff.baseline.path.name}`") + out.append(f"- **Current:** `{diff.current.path.name}`") + out.append(f"- **Generated:** {diff.generated_at}") + out.append( + f"- **Drift:** {counts[REGRESSED]} regressed · {counts[FIXED]} fixed · " + f"{counts[DRIFTED]} drifted · {counts[CHANGED]} changed · " + f"{counts[UNCHANGED]} unchanged" + ) + out.append("") + out.append( + "> A *regression* means a setting moved into a state that no longer " + "supports a control since the baseline. As always this is evidence, not " + "a verdict." + ) + out.append("") + + for category in CATEGORY_ORDER: + rows = diff.by_category(category) + if not rows or category == UNCHANGED: + continue + out.append(f"## {_CATEGORY_HEADING[category]}") + out.append("") + for delta in rows: + rule = delta.rule + out.append(f"### {rule.title}") + out.append("") + out.append(f"- **Rule:** `{rule.id}` · **Severity:** {rule.severity}") + out.append(f"- **Controls:** {', '.join(rule.controls) or '—'}") + out.append(f"- **Change:** {_transition(delta)}") + if delta.new_reason: + out.append(f"- **Now:** {delta.new_reason}") + if category == REGRESSED and rule.remediation: + out.append(f"- **Remediation:** {rule.remediation.strip()}") + out.append("") + if delta.evidence_added: + out.append("**Newly failing rows:**") + out.append("") + out.extend(_md_table(delta.evidence_added)) + out.append("") + if delta.evidence_removed: + out.append("**No longer failing rows:**") + out.append("") + out.extend(_md_table(delta.evidence_removed)) + out.append("") + + unchanged = diff.counts[UNCHANGED] + if unchanged: + out.append(f"_{unchanged} rule(s) unchanged._") + return "\n".join(out).rstrip() + "\n" + + +def _html_table(rows: list[dict[str, str]], caption: str, cls: str) -> str: + shown = rows[:10] + headers = list(shown[0].keys()) + head = "".join(f"<th>{escape(h)}</th>" for h in headers) + body = "".join( + "<tr>" + "".join(f"<td>{escape(str(r.get(h, '')))}</td>" for h in headers) + "</tr>" + for r in shown + ) + return ( + f"<table class='{cls}'><caption>{escape(caption)}</caption>" + f"<thead><tr>{head}</tr></thead><tbody>{body}</tbody></table>" + ) + + +_DIFF_CSS = ( + _CSS + + """ +.delta { border: 1px solid #e5e5e5; border-radius: 6px; padding: 1rem 1.1rem; margin: .8rem 0; } +.delta.regressed { border-left: 4px solid #c1272d; } +.delta.fixed { border-left: 4px solid #1a7f37; } +.delta.drifted { border-left: 4px solid #d08700; } +.delta.changed { border-left: 4px solid #bbb; } +.delta h3 { margin: 0 0 .4rem; font-size: 1.05rem; } +.transition { font-weight: 700; } +table.added caption, table.removed caption { text-align: left; font-weight: 600; font-size: .85rem; padding: .2rem 0; } +table.added caption { color: #c1272d; } table.removed caption { color: #1a7f37; } +@media (prefers-color-scheme: dark) { + .delta { border-color: #2d2e33; } + table.added caption { color: #ff6b70; } table.removed caption { color: #4ac36a; } +} +""" +) + + +def _render_html(diff: DiffReport) -> str: + counts = diff.counts + parts: list[str] = [] + parts.append( + f"<h1>Evidence Drift — {escape(diff.current.subject)} " + f"({escape(diff.current.platform)})</h1>" + ) + parts.append( + f"<p class='meta'>Baseline <code>{escape(diff.baseline.path.name)}</code> → " + f"Current <code>{escape(diff.current.path.name)}</code> · " + f"Generated {escape(diff.generated_at)}</p>" + ) + parts.append( + "<p class='summary-pills'>" + f"<span>{counts[REGRESSED]} regressed</span>" + f"<span>{counts[FIXED]} fixed</span>" + f"<span>{counts[DRIFTED]} drifted</span>" + f"<span>{counts[CHANGED]} changed</span>" + f"<span>{counts[UNCHANGED]} unchanged</span></p>" + ) + parts.append( + "<p class='note'>A <strong>regression</strong> means a setting moved into " + "a state that no longer supports a control since the baseline. As always " + "this is evidence, not a verdict.</p>" + ) + + for category in CATEGORY_ORDER: + rows = diff.by_category(category) + if not rows or category == UNCHANGED: + continue + parts.append(f"<h2>{escape(_CATEGORY_HEADING[category])}</h2>") + for delta in rows: + rule = delta.rule + body = [ + f"<h3>{escape(rule.title)}</h3>", + ( + f"<dl><dt>Rule</dt><dd><code>{escape(rule.id)}</code> · " + f"{escape(rule.severity)}</dd>" + ), + f"<dt>Controls</dt><dd>{escape(', '.join(rule.controls) or '—')}</dd>", + f"<dt>Change</dt><dd class='transition'>{escape(_transition(delta))}</dd>", + ] + if delta.new_reason: + body.append(f"<dt>Now</dt><dd>{escape(delta.new_reason)}</dd>") + if category == REGRESSED and rule.remediation: + body.append(f"<dt>Remediation</dt><dd>{escape(rule.remediation.strip())}</dd>") + body.append("</dl>") + if delta.evidence_added: + body.append(_html_table(delta.evidence_added, "Newly failing rows", "added")) + if delta.evidence_removed: + body.append( + _html_table(delta.evidence_removed, "No longer failing rows", "removed") + ) + parts.append(f"<div class='delta {category}'>{''.join(body)}</div>") + + if counts[UNCHANGED]: + parts.append(f"<p class='meta'>{counts[UNCHANGED]} rule(s) unchanged.</p>") + parts.append("<footer>Generated by audit-report · Audit Labs · evidence, not a verdict.</footer>") + + return ( + "<!doctype html><html lang='en'><head><meta charset='utf-8'>" + "<meta name='viewport' content='width=device-width, initial-scale=1'>" + f"<title>Evidence Drift — {escape(diff.current.subject)}</title>" + f"<style>{_DIFF_CSS}</style></head><body><main>{''.join(parts)}</main></body></html>\n" + ) + + +def _render_json(diff: DiffReport) -> str: + import json as _json + + payload = { + "subject": diff.current.subject, + "platform": diff.current.platform, + "baseline_package": diff.baseline.path.name, + "current_package": diff.current.path.name, + "generated_at": diff.generated_at, + "summary": diff.counts, + "deltas": [ + { + "id": d.rule.id, + "title": d.rule.title, + "severity": d.rule.severity, + "controls": d.rule.controls, + "category": d.category, + "old_status": d.old_status, + "new_status": d.new_status, + "evidence_added": d.evidence_added, + "evidence_removed": d.evidence_removed, + } + for d in diff.deltas + ], + } + return _json.dumps(payload, indent=2) + "\n" + + +_RENDERERS = {"md": _render_md, "html": _render_html, "json": _render_json} + + +def render(diff: DiffReport, fmt: str) -> str: + """Render a diff in the named format ('md', 'html', or 'json').""" + try: + return _RENDERERS[fmt](diff) + except KeyError: + raise ValueError(f"unknown format: {fmt!r}") from None @@ -19,7 +19,7 @@ _STATUS_LABEL = {PASS: "PASS", FAIL: "FAIL", NOT_APPLICABLE: "N/A"} _STATUS_CLASS = {PASS: "pass", FAIL: "fail", NOT_APPLICABLE: "na"} _SEVERITY_ORDER = {"high": 0, "medium": 1, "low": 2} -_CSS = """ +CSS = """ :root { color-scheme: light dark; } * { box-sizing: border-box; } body { font-family: -apple-system, Segoe UI, Roboto, Helvetica, Arial, sans-serif; @@ -161,5 +161,5 @@ def render(report: Report) -> str: "<!doctype html><html lang='en'><head><meta charset='utf-8'>" "<meta name='viewport' content='width=device-width, initial-scale=1'>" f"<title>Evidence Report — {escape(pkg.subject)}</title>" - f"<style>{_CSS}</style></head><body><main>{body}</main></body></html>\n" + f"<style>{CSS}</style></head><body><main>{body}</main></body></html>\n" ) new file mode 100644 @@ -0,0 +1,2 @@ +root_mfa_enabled,root_access_keys_present,root_signing_certs_present,mfa_devices,users,groups,roles,policies +False,False,False,1,2,0,5,0 new file mode 100644 @@ -0,0 +1,2 @@ +name,home_region,multi_region,log_file_validation,is_logging,s3_bucket +main,us-east-1,True,True,True,acme-logs new file mode 100644 @@ -0,0 +1,3 @@ +user,mfa_enabled,access_keys,oldest_key_age_days,console_password,password_last_used,created +alice,True,1,30,True,2025-12-01T00:00:00+00:00,2025-01-01T00:00:00+00:00 +bob,False,1,400,True,2025-11-01T00:00:00+00:00,2025-02-01T00:00:00+00:00 new file mode 100644 @@ -0,0 +1,2 @@ +region,group_id,group_name,protocol,from_port,to_port,open_to +us-east-1,sg-old,legacy,tcp,22,22,0.0.0.0/0 new file mode 100644 @@ -0,0 +1,2 @@ +minimum_length,require_symbols,require_numbers,require_uppercase,require_lowercase,allow_users_to_change,max_age_days,reuse_prevention,hard_expiry +14,True,True,True,True,True,90,24,False new file mode 100644 new file mode 100644 @@ -0,0 +1,113 @@ +"""Tests for diff mode: status transitions, evidence drift, and the CLI.""" + +import json +from pathlib import Path + +from audit_report import diff +from audit_report.cli import main +from audit_report.engine import evaluate +from audit_report.loader import load_package +from audit_report.rules import load_ruleset + +FIXTURES = Path(__file__).parent / "fixtures" +RULESETS = Path("audit_report/rulesets") + +BASELINE = FIXTURES / "aws_audit_acme_2025-12-01" +CURRENT = FIXTURES / "aws_audit_acme_2026-01-01" + + +def _build(): + ruleset = load_ruleset(RULESETS / "aws.yaml") + old = load_package(BASELINE) + new = load_package(CURRENT) + return diff.build_diff( + old, new, evaluate(old, ruleset), evaluate(new, ruleset) + ) + + +def test_diff_categories(): + report = _build() + by_id = {d.rule.id: d for d in report.deltas} + + # Password policy was strong in the baseline, weak now -> regressed. + assert by_id["aws.iam.password-policy"].category == diff.REGRESSED + # Root MFA was off in the baseline, on now -> fixed. + assert by_id["aws.root.mfa"].category == diff.FIXED + # Open SSH fails in both, but on a different security group -> drifted. + ssh = by_id["aws.network.no-open-ssh"] + assert ssh.category == diff.DRIFTED + assert ssh.evidence_added[0]["group_name"] == "web" + assert ssh.evidence_removed[0]["group_name"] == "legacy" + # Bob still lacks MFA with the same evidence in both -> unchanged. + assert by_id["aws.iam.console-mfa"].category == diff.UNCHANGED + + +def test_diff_counts(): + counts = _build().counts + assert counts[diff.REGRESSED] == 1 + assert counts[diff.FIXED] == 1 + assert counts[diff.DRIFTED] == 1 + + +def test_has_regression_respects_threshold(): + report = _build() + # The regression (password policy) is medium severity. + assert diff.has_regression(report, "none") is False + assert diff.has_regression(report, "medium") is True + assert diff.has_regression(report, "high") is False # nothing high regressed + + +def test_diff_render_markdown(): + md = diff.render(_build(), "md") + assert "# Evidence Drift — acme" in md + assert "Regressions" in md + assert "Newly failing rows" in md + + +def test_diff_render_html_self_contained(): + html = diff.render(_build(), "html") + assert html.startswith("<!doctype html>") + assert "http://" not in html and "https://" not in html + assert "Evidence Drift" in html + + +def test_diff_render_json(): + data = json.loads(diff.render(_build(), "json")) + cats = {d["id"]: d["category"] for d in data["deltas"]} + assert cats["aws.root.mfa"] == "fixed" + assert data["baseline_package"] == "aws_audit_acme_2025-12-01" + + +def test_cli_diff_mode(tmp_path): + out = tmp_path / "out" + code = main( + [ + str(CURRENT), + "--baseline", + str(BASELINE), + "--format", + "md,html,json", + "--out", + str(out), + "--fail-on", + "medium", + ] + ) + assert (out / "diff.md").exists() + assert (out / "diff.html").exists() + assert (out / "diff.json").exists() + # A medium-severity regression is present -> non-zero exit. + assert code == 1 + + +def test_cli_diff_platform_mismatch_returns_2(): + code = main( + [ + str(FIXTURES / "github_audit_acme_2026-01-01"), + "--baseline", + str(BASELINE), + "--format", + "json", + ] + ) + assert code == 2