audit-labs/audit-report
Turn audit-tools evidence packages into control-mapped, auditor-ready reports.
clone: git clone https://gitbay.org/audit-labs/audit-report.git
v1.0.0: tests/test_diff.py · raw
1"""Tests for diff mode: status transitions, evidence drift, and the CLI."""
2
3import json
4from pathlib import Path
5
6from audit_report import diff
7from audit_report.cli import main
8from audit_report.engine import evaluate
9from audit_report.loader import load_package
10from audit_report.rules import load_ruleset
11
12FIXTURES = Path(__file__).parent / "fixtures"
13RULESETS = Path("audit_report/rulesets")
14
15BASELINE = FIXTURES / "aws_audit_acme_2025-12-01"
16CURRENT = FIXTURES / "aws_audit_acme_2026-01-01"
17
18
19def _build():
20 ruleset = load_ruleset(RULESETS / "aws.yaml")
21 old = load_package(BASELINE)
22 new = load_package(CURRENT)
23 return diff.build_diff(
24 old, new, evaluate(old, ruleset), evaluate(new, ruleset)
25 )
26
27
28def test_diff_categories():
29 report = _build()
30 by_id = {d.rule.id: d for d in report.deltas}
31
32 # Password policy was strong in the baseline, weak now -> regressed.
33 assert by_id["aws.iam.password-policy"].category == diff.REGRESSED
34 # Root MFA was off in the baseline, on now -> fixed.
35 assert by_id["aws.root.mfa"].category == diff.FIXED
36 # Open SSH fails in both, but on a different security group -> drifted.
37 ssh = by_id["aws.network.no-open-ssh"]
38 assert ssh.category == diff.DRIFTED
39 assert ssh.evidence_added[0]["group_name"] == "web"
40 assert ssh.evidence_removed[0]["group_name"] == "legacy"
41 # Bob still lacks MFA with the same evidence in both -> unchanged.
42 assert by_id["aws.iam.console-mfa"].category == diff.UNCHANGED
43
44
45def test_diff_counts():
46 counts = _build().counts
47 assert counts[diff.REGRESSED] == 1
48 assert counts[diff.FIXED] == 1
49 assert counts[diff.DRIFTED] == 1
50
51
52def test_has_regression_respects_threshold():
53 report = _build()
54 # The regression (password policy) is medium severity.
55 assert diff.has_regression(report, "none") is False
56 assert diff.has_regression(report, "medium") is True
57 assert diff.has_regression(report, "high") is False # nothing high regressed
58
59
60def test_diff_render_markdown():
61 md = diff.render(_build(), "md")
62 assert "# Evidence Drift — acme" in md
63 assert "Regressions" in md
64 assert "Newly failing rows" in md
65
66
67def test_diff_render_html_self_contained():
68 html = diff.render(_build(), "html")
69 assert html.startswith("<!doctype html>")
70 assert "http://" not in html and "https://" not in html
71 assert "Evidence Drift" in html
72
73
74def test_diff_render_json():
75 data = json.loads(diff.render(_build(), "json"))
76 cats = {d["id"]: d["category"] for d in data["deltas"]}
77 assert cats["aws.root.mfa"] == "fixed"
78 assert data["baseline_package"] == "aws_audit_acme_2025-12-01"
79
80
81def test_cli_diff_mode(tmp_path):
82 out = tmp_path / "out"
83 code = main(
84 [
85 str(CURRENT),
86 "--baseline",
87 str(BASELINE),
88 "--format",
89 "md,html,json",
90 "--out",
91 str(out),
92 "--fail-on",
93 "medium",
94 ]
95 )
96 assert (out / "diff.md").exists()
97 assert (out / "diff.html").exists()
98 assert (out / "diff.json").exists()
99 # A medium-severity regression is present -> non-zero exit.
100 assert code == 1
101
102
103def test_cli_diff_platform_mismatch_returns_2():
104 code = main(
105 [
106 str(FIXTURES / "github_audit_acme_2026-01-01"),
107 "--baseline",
108 str(BASELINE),
109 "--format",
110 "json",
111 ]
112 )
113 assert code == 2