Commit `e1665106ac`

e1665106acf579481649d470be9d413338ad4875

parent: f14a20bc6c

Verified · cmc

cmc <hello@cleberg.net> · 2026-08-06 07:23 UTC

feat: add diff mode to compare two evidence packages

Passing --baseline evaluates both packages with the same ruleset and reports
drift: each rule is classified as regressed, fixed, drifted (an ongoing failure
whose evidence rows changed), changed, or unchanged, with the exact rows that
appeared or disappeared. Renders md/html/json as diff.*; --fail-on gates on
regressions for CI. Adds a baseline fixture and diff tests.

Layout: unified · split

README.md +24 −1

		@@ -41,6 +41,28 @@ audit-report ./output/github_audit_acme_2026-07-29 --format md,html,json --out r
41	41	audit-report ./output/aws_audit_prod_2026-07-29 --fail-on high --out report/
42	42	```
43	43
	44	### Diff mode — compare two packages
	45
	46	Pass `--baseline` to report how a package drifted from an earlier one. Both
	47	are evaluated with the same ruleset; the report classifies each rule as
	48	regressed, fixed, drifted (an ongoing failure whose evidence rows changed),
	49	changed, or unchanged — and shows exactly which evidence rows appeared or
	50	disappeared.
	51
	52	```bash
	53	# How did prod change between two audit runs?
	54	audit-report ./output/aws_audit_prod_2026-07-29 \
	55	--baseline ./output/aws_audit_prod_2026-06-29 \
	56	--format md,html --out drift/
	57
	58	# Fail CI if this change regressed any high-severity control
	59	audit-report ./output/aws_audit_prod_2026-07-29 \
	60	--baseline ./output/aws_audit_prod_2026-06-29 --fail-on high
	61	```
	62
	63	In diff mode `--fail-on` gates on regressions at or above the given
	64	severity, and output files are named `diff.` instead of `report.`.
	65
44	66	You can also run it without installing:
45	67
46	68	```bash
		@@ -51,10 +73,11 @@ python -m audit_report ./output/aws_audit_default_2026-07-29
51	73
52	74	\| Flag \| Description \|
53	75	\| --- \| --- \|
	76	\| `--baseline PATH` \| Diff mode: report how `PACKAGE` drifted from this earlier package. \|
54	77	\| `--ruleset PATH` \| Use a specific ruleset instead of the bundled one for the detected platform. \|
55	78	\| `--format md,html,json` \| One or more output formats (default: `md`). \|
56	79	\| `--out DIR` \| Write `report.<ext>` files into `DIR`. Without it, the first format prints to stdout. \|
57		\| `--fail-on low\\|medium\\|high\\|none` \| Exit non-zero when a failing finding meets this severity (default: `none`). \|
	80	\| `--fail-on low\\|medium\\|high\\|none` \| Exit non-zero when a failing finding — or, in diff mode, a regression — meets this severity (default: `none`). \|
58	81
59	82	## What a report contains
60	83

audit_report/cli.py +62 −15

		@@ -6,7 +6,7 @@ import argparse
6	6	import sys
7	7	from pathlib import Path
8	8
9		from . import __version__, reporters
	9	from . import __version__, diff, reporters
10	10	from .engine import FAIL, evaluate
11	11	from .loader import load_package
12	12	from .rules import load_ruleset
		@@ -34,6 +34,10 @@ def _parse_args(argv: list[str]) -> argparse.Namespace:
34	34	description="Turn an audit-tools evidence package into a control-mapped report.",
35	35	)
36	36	parser.add_argument("package", help="path to an audit-tools output package directory")
	37	parser.add_argument(
	38	"--baseline",
	39	help="path to an earlier package; diff mode reports how PACKAGE drifted from it",
	40	)
37	41	parser.add_argument(
38	42	"--ruleset",
39	43	help="path to a ruleset YAML (default: bundled ruleset for the detected platform)",
		@@ -51,7 +55,10 @@ def _parse_args(argv: list[str]) -> argparse.Namespace:
51	55	"--fail-on",
52	56	choices=["low", "medium", "high", "none"],
53	57	default="none",
54		help="exit non-zero if any finding fails at or above this severity (default: none)",
	58	help=(
	59	"exit non-zero if any finding fails (or, in diff mode, regresses) at or "
	60	"above this severity (default: none)"
	61	),
55	62	)
56	63	parser.add_argument("--version", action="version", version=f"audit-report {__version__}")
57	64	return parser.parse_args(argv)
		@@ -68,9 +75,58 @@ def _exit_code(findings, threshold: str) -> int:
68	75	return 1 if breached else 0
69	76
70	77
	78	def _emit(render_one, formats: list[str], out: str \| None, basename: str) -> None:
	79	"""Write one file per format into out, or print the first to stdout."""
	80	if out:
	81	out_dir = Path(out)
	82	out_dir.mkdir(parents=True, exist_ok=True)
	83	for fmt in formats:
	84	dest = out_dir / f"{basename}.{reporters.EXTENSIONS[fmt]}"
	85	dest.write_text(render_one(fmt), encoding="utf-8")
	86	print(f"wrote {dest}", file=sys.stderr)
	87	else:
	88	print(render_one(formats[0]), end="")
	89
	90
	91	def _run_diff(args, package, ruleset, formats: list[str]) -> int:
	92	try:
	93	baseline = load_package(args.baseline)
	94	except (FileNotFoundError, ValueError) as exc:
	95	print(f"error: {exc}", file=sys.stderr)
	96	return 2
	97	if baseline.platform != package.platform:
	98	print(
	99	f"error: cannot diff a {baseline.platform} package against a "
	100	f"{package.platform} package",
	101	file=sys.stderr,
	102	)
	103	return 2
	104
	105	old = evaluate(baseline, ruleset)
	106	new = evaluate(package, ruleset)
	107	report = diff.build_diff(baseline, package, old, new)
	108
	109	_emit(lambda fmt: diff.render(report, fmt), formats, args.out, "diff")
	110
	111	counts = report.counts
	112	print(
	113	f"{package.platform}/{package.subject}: "
	114	f"{counts[diff.REGRESSED]} regressed, {counts[diff.FIXED]} fixed, "
	115	f"{counts[diff.DRIFTED]} drifted",
	116	file=sys.stderr,
	117	)
	118	return 1 if diff.has_regression(report, args.fail_on) else 0
	119
	120
71	121	def main(argv: list[str] \| None = None) -> int:
72	122	args = _parse_args(argv if argv is not None else sys.argv[1:])
73	123
	124	formats = [f.strip() for f in args.format.split(",") if f.strip()]
	125	unknown = [f for f in formats if f not in reporters.EXTENSIONS]
	126	if not formats or unknown:
	127	print(f"error: unknown format(s): {', '.join(unknown) or '(none given)'}", file=sys.stderr)
	128	return 2
	129
74	130	try:
75	131	package = load_package(args.package)
76	132	except (FileNotFoundError, ValueError) as exc:
		@@ -80,21 +136,12 @@ def main(argv: list[str] \| None = None) -> int:
80	136	ruleset_path = Path(args.ruleset) if args.ruleset else _default_ruleset(package.platform)
81	137	ruleset = load_ruleset(ruleset_path)
82	138
	139	if args.baseline:
	140	return _run_diff(args, package, ruleset, formats)
	141
83	142	findings = evaluate(package, ruleset)
84	143	report = reporters.build_report(package, findings)
85		formats = [f.strip() for f in args.format.split(",") if f.strip()]
86
87		if args.out:
88		out_dir = Path(args.out)
89		out_dir.mkdir(parents=True, exist_ok=True)
90		for fmt in formats:
91		content = reporters.render(report, fmt)
92		dest = out_dir / f"report.{reporters.EXTENSIONS[fmt]}"
93		dest.write_text(content, encoding="utf-8")
94		print(f"wrote {dest}", file=sys.stderr)
95		else:
96		# No --out: emit the first requested format to stdout.
97		print(reporters.render(report, formats[0]), end="")
	144	_emit(lambda fmt: reporters.render(report, fmt), formats, args.out, "report")
98	145
99	146	counts = report.counts
100	147	print(

audit_report/diff.py added +381

		@@ -0,0 +1,381 @@
	1	"""Diff mode — compare two evidence packages and report drift.
	2
	3	Both packages are evaluated with the same ruleset; this module compares the two
	4	sets of findings and classifies each rule's change:
	5
	6	* regressed — a control that was supported (or not yet observed) now fails
	7	* fixed — a control that failed now passes (or is no longer observed)
	8	* drifted — an ongoing failure whose failing evidence rows changed
	9	* changed — a non-failure status change (e.g. a table stopped being collected)
	10	* unchanged — same status, same evidence
	11
	12	Evidence rows are compared as whole rows, so drift shows exactly which items
	13	appeared or disappeared (a new MFA-less user, a security group that was closed).
	14	"""
	15
	16	from __future__ import annotations
	17
	18	from dataclasses import dataclass, field
	19	from datetime import datetime, timezone
	20	from html import escape
	21
	22	from .engine import FAIL, Finding
	23	from .loader import Package
	24	from .reporters.html import CSS as _CSS
	25
	26	ABSENT = "absent" # rule present in only one of the two packages
	27
	28	REGRESSED = "regressed"
	29	FIXED = "fixed"
	30	DRIFTED = "drifted"
	31	CHANGED = "changed"
	32	UNCHANGED = "unchanged"
	33
	34	# Order categories appear in a report and how they roll up in the summary.
	35	CATEGORY_ORDER = [REGRESSED, FIXED, DRIFTED, CHANGED, UNCHANGED]
	36	_STATUS_LABEL = {FAIL: "fail", "pass": "pass", "not_applicable": "n/a", ABSENT: "absent"}
	37	_SEVERITY_ORDER = {"high": 0, "medium": 1, "low": 2}
	38
	39
	40	def _row_key(row: dict[str, str]) -> tuple:
	41	return tuple(sorted(row.items()))
	42
	43
	44	@dataclass
	45	class RuleDelta:
	46	"""How one rule's finding changed between the two packages."""
	47
	48	rule: object # audit_report.rules.Rule
	49	old_status: str
	50	new_status: str
	51	category: str
	52	new_reason: str = ""
	53	evidence_added: list[dict[str, str]] = field(default_factory=list)
	54	evidence_removed: list[dict[str, str]] = field(default_factory=list)
	55
	56	@property
	57	def controls(self) -> list[str]:
	58	return self.rule.controls
	59
	60
	61	def _classify(old_status, new_status, added, removed) -> str:
	62	if old_status == ABSENT:
	63	return REGRESSED if new_status == FAIL else CHANGED
	64	if new_status == ABSENT:
	65	return CHANGED # rule dropped from the current ruleset
	66	if new_status == FAIL and old_status != FAIL:
	67	return REGRESSED
	68	if old_status == FAIL and new_status != FAIL:
	69	return FIXED
	70	if old_status == FAIL and new_status == FAIL:
	71	return DRIFTED if (added or removed) else UNCHANGED
	72	return CHANGED if old_status != new_status else UNCHANGED
	73
	74
	75	def diff_findings(old: list[Finding], new: list[Finding]) -> list[RuleDelta]:
	76	"""Compare two finding lists (same ruleset) into a list of deltas."""
	77	old_by_id = {f.rule.id: f for f in old}
	78	new_by_id = {f.rule.id: f for f in new}
	79
	80	# New order first (ruleset order), then any rules only the baseline had.
	81	ordered_ids = [f.rule.id for f in new]
	82	ordered_ids += [f.rule.id for f in old if f.rule.id not in new_by_id]
	83
	84	deltas: list[RuleDelta] = []
	85	for rid in ordered_ids:
	86	of, nf = old_by_id.get(rid), new_by_id.get(rid)
	87	rule = (nf or of).rule
	88	old_status = of.status if of else ABSENT
	89	new_status = nf.status if nf else ABSENT
	90
	91	old_ev = {_row_key(r): r for r in (of.evidence if of else [])}
	92	new_ev = {_row_key(r): r for r in (nf.evidence if nf else [])}
	93	added = [r for k, r in new_ev.items() if k not in old_ev]
	94	removed = [r for k, r in old_ev.items() if k not in new_ev]
	95
	96	category = _classify(old_status, new_status, added, removed)
	97	deltas.append(
	98	RuleDelta(
	99	rule=rule,
	100	old_status=old_status,
	101	new_status=new_status,
	102	category=category,
	103	new_reason=nf.reason if nf else "",
	104	evidence_added=added,
	105	evidence_removed=removed,
	106	)
	107	)
	108	return deltas
	109
	110
	111	@dataclass
	112	class DiffReport:
	113	"""A computed comparison of two packages, ready to render."""
	114
	115	baseline: Package
	116	current: Package
	117	deltas: list[RuleDelta]
	118	generated_at: str
	119
	120	def by_category(self, category: str) -> list[RuleDelta]:
	121	rows = [d for d in self.deltas if d.category == category]
	122	return sorted(rows, key=lambda d: _SEVERITY_ORDER.get(d.rule.severity, 1))
	123
	124	@property
	125	def counts(self) -> dict[str, int]:
	126	return {cat: len(self.by_category(cat)) for cat in CATEGORY_ORDER}
	127
	128
	129	def build_diff(
	130	baseline: Package,
	131	current: Package,
	132	old_findings: list[Finding],
	133	new_findings: list[Finding],
	134	) -> DiffReport:
	135	"""Assemble a :class:`DiffReport` with a UTC timestamp."""
	136	stamp = datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M UTC")
	137	return DiffReport(
	138	baseline=baseline,
	139	current=current,
	140	deltas=diff_findings(old_findings, new_findings),
	141	generated_at=stamp,
	142	)
	143
	144
	145	def has_regression(diff: DiffReport, threshold: str) -> bool:
	146	"""True if any regressed rule meets the severity threshold ('none' = off)."""
	147	if threshold == "none":
	148	return False
	149	floor = _SEVERITY_ORDER[threshold]
	150	return any(
	151	_SEVERITY_ORDER.get(d.rule.severity, 1) <= floor
	152	for d in diff.by_category(REGRESSED)
	153	)
	154
	155
	156	# --------------------------------------------------------------------------- #
	157	# Rendering
	158	# --------------------------------------------------------------------------- #
	159
	160	_CATEGORY_HEADING = {
	161	REGRESSED: "Regressions — a control is no longer supported",
	162	FIXED: "Fixed — a previously failing control now passes",
	163	DRIFTED: "Ongoing failures with changed evidence",
	164	CHANGED: "Other status changes",
	165	UNCHANGED: "Unchanged",
	166	}
	167
	168
	169	def _transition(delta: RuleDelta) -> str:
	170	return f"{_STATUS_LABEL.get(delta.old_status, delta.old_status)} → {_STATUS_LABEL.get(delta.new_status, delta.new_status)}"
	171
	172
	173	def _md_table(rows: list[dict[str, str]]) -> list[str]:
	174	if not rows:
	175	return []
	176	shown = rows[:10]
	177	headers = list(shown[0].keys())
	178	out = [
	179	"\| " + " \| ".join(headers) + " \|",
	180	"\| " + " \| ".join("---" for _ in headers) + " \|",
	181	]
	182	out += ["\| " + " \| ".join(str(r.get(h, "")) for h in headers) + " \|" for r in shown]
	183	if len(rows) > len(shown):
	184	out.append(f"\n_+{len(rows) - len(shown)} more row(s) omitted._")
	185	return out
	186
	187
	188	def _render_md(diff: DiffReport) -> str:
	189	counts = diff.counts
	190	out: list[str] = []
	191	out.append(f"# Evidence Drift — {diff.current.subject} ({diff.current.platform})")
	192	out.append("")
	193	out.append(f"- Baseline: `{diff.baseline.path.name}`")
	194	out.append(f"- Current: `{diff.current.path.name}`")
	195	out.append(f"- Generated: {diff.generated_at}")
	196	out.append(
	197	f"- Drift: {counts[REGRESSED]} regressed · {counts[FIXED]} fixed · "
	198	f"{counts[DRIFTED]} drifted · {counts[CHANGED]} changed · "
	199	f"{counts[UNCHANGED]} unchanged"
	200	)
	201	out.append("")
	202	out.append(
	203	"> A regression means a setting moved into a state that no longer "
	204	"supports a control since the baseline. As always this is evidence, not "
	205	"a verdict."
	206	)
	207	out.append("")
	208
	209	for category in CATEGORY_ORDER:
	210	rows = diff.by_category(category)
	211	if not rows or category == UNCHANGED:
	212	continue
	213	out.append(f"## {_CATEGORY_HEADING[category]}")
	214	out.append("")
	215	for delta in rows:
	216	rule = delta.rule
	217	out.append(f"### {rule.title}")
	218	out.append("")
	219	out.append(f"- Rule: `{rule.id}` · Severity: {rule.severity}")
	220	out.append(f"- Controls: {', '.join(rule.controls) or '—'}")
	221	out.append(f"- Change: {_transition(delta)}")
	222	if delta.new_reason:
	223	out.append(f"- Now: {delta.new_reason}")
	224	if category == REGRESSED and rule.remediation:
	225	out.append(f"- Remediation: {rule.remediation.strip()}")
	226	out.append("")
	227	if delta.evidence_added:
	228	out.append("Newly failing rows:")
	229	out.append("")
	230	out.extend(_md_table(delta.evidence_added))
	231	out.append("")
	232	if delta.evidence_removed:
	233	out.append("No longer failing rows:")
	234	out.append("")
	235	out.extend(_md_table(delta.evidence_removed))
	236	out.append("")
	237
	238	unchanged = diff.counts[UNCHANGED]
	239	if unchanged:
	240	out.append(f"_{unchanged} rule(s) unchanged._")
	241	return "\n".join(out).rstrip() + "\n"
	242
	243
	244	def _html_table(rows: list[dict[str, str]], caption: str, cls: str) -> str:
	245	shown = rows[:10]
	246	headers = list(shown[0].keys())
	247	head = "".join(f"<th>{escape(h)}</th>" for h in headers)
	248	body = "".join(
	249	"<tr>" + "".join(f"<td>{escape(str(r.get(h, '')))}</td>" for h in headers) + "</tr>"
	250	for r in shown
	251	)
	252	return (
	253	f"<table class='{cls}'><caption>{escape(caption)}</caption>"
	254	f"<thead><tr>{head}</tr></thead><tbody>{body}</tbody></table>"
	255	)
	256
	257
	258	_DIFF_CSS = (
	259	_CSS
	260	+ """
	261	.delta { border: 1px solid #e5e5e5; border-radius: 6px; padding: 1rem 1.1rem; margin: .8rem 0; }
	262	.delta.regressed { border-left: 4px solid #c1272d; }
	263	.delta.fixed { border-left: 4px solid #1a7f37; }
	264	.delta.drifted { border-left: 4px solid #d08700; }
	265	.delta.changed { border-left: 4px solid #bbb; }
	266	.delta h3 { margin: 0 0 .4rem; font-size: 1.05rem; }
	267	.transition { font-weight: 700; }
	268	table.added caption, table.removed caption { text-align: left; font-weight: 600; font-size: .85rem; padding: .2rem 0; }
	269	table.added caption { color: #c1272d; } table.removed caption { color: #1a7f37; }
	270	@media (prefers-color-scheme: dark) {
	271	.delta { border-color: #2d2e33; }
	272	table.added caption { color: #ff6b70; } table.removed caption { color: #4ac36a; }
	273	}
	274	"""
	275	)
	276
	277
	278	def _render_html(diff: DiffReport) -> str:
	279	counts = diff.counts
	280	parts: list[str] = []
	281	parts.append(
	282	f"<h1>Evidence Drift — {escape(diff.current.subject)} "
	283	f"({escape(diff.current.platform)})</h1>"
	284	)
	285	parts.append(
	286	f"<p class='meta'>Baseline <code>{escape(diff.baseline.path.name)}</code> → "
	287	f"Current <code>{escape(diff.current.path.name)}</code> · "
	288	f"Generated {escape(diff.generated_at)}</p>"
	289	)
	290	parts.append(
	291	"<p class='summary-pills'>"
	292	f"<span>{counts[REGRESSED]} regressed</span>"
	293	f"<span>{counts[FIXED]} fixed</span>"
	294	f"<span>{counts[DRIFTED]} drifted</span>"
	295	f"<span>{counts[CHANGED]} changed</span>"
	296	f"<span>{counts[UNCHANGED]} unchanged</span></p>"
	297	)
	298	parts.append(
	299	"<p class='note'>A <strong>regression</strong> means a setting moved into "
	300	"a state that no longer supports a control since the baseline. As always "
	301	"this is evidence, not a verdict.</p>"
	302	)
	303
	304	for category in CATEGORY_ORDER:
	305	rows = diff.by_category(category)
	306	if not rows or category == UNCHANGED:
	307	continue
	308	parts.append(f"<h2>{escape(_CATEGORY_HEADING[category])}</h2>")
	309	for delta in rows:
	310	rule = delta.rule
	311	body = [
	312	f"<h3>{escape(rule.title)}</h3>",
	313	(
	314	f"<dl><dt>Rule</dt><dd><code>{escape(rule.id)}</code> · "
	315	f"{escape(rule.severity)}</dd>"
	316	),
	317	f"<dt>Controls</dt><dd>{escape(', '.join(rule.controls) or '—')}</dd>",
	318	f"<dt>Change</dt><dd class='transition'>{escape(_transition(delta))}</dd>",
	319	]
	320	if delta.new_reason:
	321	body.append(f"<dt>Now</dt><dd>{escape(delta.new_reason)}</dd>")
	322	if category == REGRESSED and rule.remediation:
	323	body.append(f"<dt>Remediation</dt><dd>{escape(rule.remediation.strip())}</dd>")
	324	body.append("</dl>")
	325	if delta.evidence_added:
	326	body.append(_html_table(delta.evidence_added, "Newly failing rows", "added"))
	327	if delta.evidence_removed:
	328	body.append(
	329	_html_table(delta.evidence_removed, "No longer failing rows", "removed")
	330	)
	331	parts.append(f"<div class='delta {category}'>{''.join(body)}</div>")
	332
	333	if counts[UNCHANGED]:
	334	parts.append(f"<p class='meta'>{counts[UNCHANGED]} rule(s) unchanged.</p>")
	335	parts.append("<footer>Generated by audit-report · Audit Labs · evidence, not a verdict.</footer>")
	336
	337	return (
	338	"<!doctype html><html lang='en'><head><meta charset='utf-8'>"
	339	"<meta name='viewport' content='width=device-width, initial-scale=1'>"
	340	f"<title>Evidence Drift — {escape(diff.current.subject)}</title>"
	341	f"<style>{_DIFF_CSS}</style></head><body><main>{''.join(parts)}</main></body></html>\n"
	342	)
	343
	344
	345	def _render_json(diff: DiffReport) -> str:
	346	import json as _json
	347
	348	payload = {
	349	"subject": diff.current.subject,
	350	"platform": diff.current.platform,
	351	"baseline_package": diff.baseline.path.name,
	352	"current_package": diff.current.path.name,
	353	"generated_at": diff.generated_at,
	354	"summary": diff.counts,
	355	"deltas": [
	356	{
	357	"id": d.rule.id,
	358	"title": d.rule.title,
	359	"severity": d.rule.severity,
	360	"controls": d.rule.controls,
	361	"category": d.category,
	362	"old_status": d.old_status,
	363	"new_status": d.new_status,
	364	"evidence_added": d.evidence_added,
	365	"evidence_removed": d.evidence_removed,
	366	}
	367	for d in diff.deltas
	368	],
	369	}
	370	return _json.dumps(payload, indent=2) + "\n"
	371
	372
	373	_RENDERERS = {"md": _render_md, "html": _render_html, "json": _render_json}
	374
	375
	376	def render(diff: DiffReport, fmt: str) -> str:
	377	"""Render a diff in the named format ('md', 'html', or 'json')."""
	378	try:
	379	return _RENDERERS[fmt](diff)
	380	except KeyError:
	381	raise ValueError(f"unknown format: {fmt!r}") from None

audit_report/reporters/html.py +2 −2

		@@ -19,7 +19,7 @@ _STATUS_LABEL = {PASS: "PASS", FAIL: "FAIL", NOT_APPLICABLE: "N/A"}
19	19	_STATUS_CLASS = {PASS: "pass", FAIL: "fail", NOT_APPLICABLE: "na"}
20	20	_SEVERITY_ORDER = {"high": 0, "medium": 1, "low": 2}
21	21
22		_CSS = """
	22	CSS = """
23	23	:root { color-scheme: light dark; }
24	24	* { box-sizing: border-box; }
25	25	body { font-family: -apple-system, Segoe UI, Roboto, Helvetica, Arial, sans-serif;
		@@ -161,5 +161,5 @@ def render(report: Report) -> str:
161	161	"<!doctype html><html lang='en'><head><meta charset='utf-8'>"
162	162	"<meta name='viewport' content='width=device-width, initial-scale=1'>"
163	163	f"<title>Evidence Report — {escape(pkg.subject)}</title>"
164		f"<style>{_CSS}</style></head><body><main>{body}</main></body></html>\n"
	164	f"<style>{CSS}</style></head><body><main>{body}</main></body></html>\n"
165	165	)

tests/fixtures/aws_audit_acme_2025-12-01/account_security.csv added +2

		@@ -0,0 +1,2 @@
	1	root_mfa_enabled,root_access_keys_present,root_signing_certs_present,mfa_devices,users,groups,roles,policies
	2	False,False,False,1,2,0,5,0

tests/fixtures/aws_audit_acme_2025-12-01/cloudtrail.csv added +2

		@@ -0,0 +1,2 @@
	1	name,home_region,multi_region,log_file_validation,is_logging,s3_bucket
	2	main,us-east-1,True,True,True,acme-logs

tests/fixtures/aws_audit_acme_2025-12-01/iam_users.csv added +3

		@@ -0,0 +1,3 @@
	1	user,mfa_enabled,access_keys,oldest_key_age_days,console_password,password_last_used,created
	2	alice,True,1,30,True,2025-12-01T00:00:00+00:00,2025-01-01T00:00:00+00:00
	3	bob,False,1,400,True,2025-11-01T00:00:00+00:00,2025-02-01T00:00:00+00:00

tests/fixtures/aws_audit_acme_2025-12-01/open_security_groups.csv added +2

		@@ -0,0 +1,2 @@
	1	region,group_id,group_name,protocol,from_port,to_port,open_to
	2	us-east-1,sg-old,legacy,tcp,22,22,0.0.0.0/0

tests/fixtures/aws_audit_acme_2025-12-01/password_policy.csv added +2

		@@ -0,0 +1,2 @@
	1	minimum_length,require_symbols,require_numbers,require_uppercase,require_lowercase,allow_users_to_change,max_age_days,reuse_prevention,hard_expiry
	2	14,True,True,True,True,True,90,24,False

tests/fixtures/aws_audit_acme_2025-12-01/s3_public_access.csv added

tests/test_diff.py added +113

		@@ -0,0 +1,113 @@
	1	"""Tests for diff mode: status transitions, evidence drift, and the CLI."""
	2
	3	import json
	4	from pathlib import Path
	5
	6	from audit_report import diff
	7	from audit_report.cli import main
	8	from audit_report.engine import evaluate
	9	from audit_report.loader import load_package
	10	from audit_report.rules import load_ruleset
	11
	12	FIXTURES = Path(__file__).parent / "fixtures"
	13	RULESETS = Path("audit_report/rulesets")
	14
	15	BASELINE = FIXTURES / "aws_audit_acme_2025-12-01"
	16	CURRENT = FIXTURES / "aws_audit_acme_2026-01-01"
	17
	18
	19	def _build():
	20	ruleset = load_ruleset(RULESETS / "aws.yaml")
	21	old = load_package(BASELINE)
	22	new = load_package(CURRENT)
	23	return diff.build_diff(
	24	old, new, evaluate(old, ruleset), evaluate(new, ruleset)
	25	)
	26
	27
	28	def test_diff_categories():
	29	report = _build()
	30	by_id = {d.rule.id: d for d in report.deltas}
	31
	32	# Password policy was strong in the baseline, weak now -> regressed.
	33	assert by_id["aws.iam.password-policy"].category == diff.REGRESSED
	34	# Root MFA was off in the baseline, on now -> fixed.
	35	assert by_id["aws.root.mfa"].category == diff.FIXED
	36	# Open SSH fails in both, but on a different security group -> drifted.
	37	ssh = by_id["aws.network.no-open-ssh"]
	38	assert ssh.category == diff.DRIFTED
	39	assert ssh.evidence_added[0]["group_name"] == "web"
	40	assert ssh.evidence_removed[0]["group_name"] == "legacy"
	41	# Bob still lacks MFA with the same evidence in both -> unchanged.
	42	assert by_id["aws.iam.console-mfa"].category == diff.UNCHANGED
	43
	44
	45	def test_diff_counts():
	46	counts = _build().counts
	47	assert counts[diff.REGRESSED] == 1
	48	assert counts[diff.FIXED] == 1
	49	assert counts[diff.DRIFTED] == 1
	50
	51
	52	def test_has_regression_respects_threshold():
	53	report = _build()
	54	# The regression (password policy) is medium severity.
	55	assert diff.has_regression(report, "none") is False
	56	assert diff.has_regression(report, "medium") is True
	57	assert diff.has_regression(report, "high") is False # nothing high regressed
	58
	59
	60	def test_diff_render_markdown():
	61	md = diff.render(_build(), "md")
	62	assert "# Evidence Drift — acme" in md
	63	assert "Regressions" in md
	64	assert "Newly failing rows" in md
	65
	66
	67	def test_diff_render_html_self_contained():
	68	html = diff.render(_build(), "html")
	69	assert html.startswith("<!doctype html>")
	70	assert "http://" not in html and "https://" not in html
	71	assert "Evidence Drift" in html
	72
	73
	74	def test_diff_render_json():
	75	data = json.loads(diff.render(_build(), "json"))
	76	cats = {d["id"]: d["category"] for d in data["deltas"]}
	77	assert cats["aws.root.mfa"] == "fixed"
	78	assert data["baseline_package"] == "aws_audit_acme_2025-12-01"
	79
	80
	81	def test_cli_diff_mode(tmp_path):
	82	out = tmp_path / "out"
	83	code = main(
	84	[
	85	str(CURRENT),
	86	"--baseline",
	87	str(BASELINE),
	88	"--format",
	89	"md,html,json",
	90	"--out",
	91	str(out),
	92	"--fail-on",
	93	"medium",
	94	]
	95	)
	96	assert (out / "diff.md").exists()
	97	assert (out / "diff.html").exists()
	98	assert (out / "diff.json").exists()
	99	# A medium-severity regression is present -> non-zero exit.
	100	assert code == 1
	101
	102
	103	def test_cli_diff_platform_mismatch_returns_2():
	104	code = main(
	105	[
	106	str(FIXTURES / "github_audit_acme_2026-01-01"),
	107	"--baseline",
	108	str(BASELINE),
	109	"--format",
	110	"json",
	111	]
	112	)
	113	assert code == 2

Commit e1665106ac

Commit `e1665106ac`