audit-labs/audit-report
Turn audit-tools evidence packages into control-mapped, auditor-ready reports.
clone: git clone https://gitbay.org/audit-labs/audit-report.git
main: tests/test_diff.py · raw
1"""Tests for diff mode: status transitions, evidence drift, and the CLI."""
2
3import json
4from pathlib import Path
5
6from audit_report import diff
7from audit_report.cli import main
8from audit_report.engine import evaluate
9from audit_report.loader import load_package
10from audit_report.rules import load_ruleset
11
12FIXTURES = Path(__file__).parent / "fixtures"
13RULESETS = Path("audit_report/rulesets")
14
15BASELINE = FIXTURES / "aws_audit_acme_2025-12-01"
16CURRENT = FIXTURES / "aws_audit_acme_2026-01-01"
17
18
19def _build():
20 ruleset = load_ruleset(RULESETS / "aws.yaml")
21 old = load_package(BASELINE)
22 new = load_package(CURRENT)
23 return diff.build_diff(
24 old, new, evaluate(old, ruleset), evaluate(new, ruleset)
25 )
26
27
28def test_diff_categories():
29 report = _build()
30 by_id = {d.rule.id: d for d in report.deltas}
31
32 # Password policy was strong in the baseline, weak now -> regressed.
33 assert by_id["aws.iam.password-policy"].category == diff.REGRESSED
34 # Root MFA was off in the baseline, on now -> fixed.
35 assert by_id["aws.root.mfa"].category == diff.FIXED
36 # Open SSH fails in both, but on a different security group -> drifted.
37 ssh = by_id["aws.network.no-open-ssh"]
38 assert ssh.category == diff.DRIFTED
39 assert ssh.evidence_added[0]["group_name"] == "web"
40 assert ssh.evidence_removed[0]["group_name"] == "legacy"
41 # Bob still lacks MFA with the same evidence in both -> unchanged.
42 assert by_id["aws.iam.console-mfa"].category == diff.UNCHANGED
43
44
45def test_diff_counts():
46 counts = _build().counts
47 assert counts[diff.REGRESSED] == 1
48 assert counts[diff.FIXED] == 1
49 assert counts[diff.DRIFTED] == 1
50
51
52def test_has_regression_respects_threshold():
53 report = _build()
54 # The regression (password policy) is medium severity.
55 assert diff.has_regression(report, "none") is False
56 assert diff.has_regression(report, "medium") is True
57 assert diff.has_regression(report, "high") is False # nothing high regressed
58
59
60def test_diff_render_markdown():
61 md = diff.render(_build(), "md")
62 assert "# Evidence Drift — acme" in md
63 assert "Regressions" in md
64 assert "Newly failing rows" in md
65
66
67def test_diff_render_html_self_contained():
68 html = diff.render(_build(), "html")
69 assert html.startswith("<!doctype html>")
70 assert "http://" not in html
71 assert "https://" not in html
72 assert "Evidence Drift" in html
73
74
75def test_diff_render_json():
76 data = json.loads(diff.render(_build(), "json"))
77 cats = {d["id"]: d["category"] for d in data["deltas"]}
78 assert cats["aws.root.mfa"] == "fixed"
79 assert data["baseline_package"] == "aws_audit_acme_2025-12-01"
80
81
82def test_cli_diff_mode(tmp_path):
83 out = tmp_path / "out"
84 code = main(
85 [
86 str(CURRENT),
87 "--baseline",
88 str(BASELINE),
89 "--format",
90 "md,html,json",
91 "--out",
92 str(out),
93 "--fail-on",
94 "medium",
95 ]
96 )
97 assert (out / "diff.md").exists()
98 assert (out / "diff.html").exists()
99 assert (out / "diff.json").exists()
100 # A medium-severity regression is present -> non-zero exit.
101 assert code == 1
102
103
104def test_cli_diff_platform_mismatch_returns_2():
105 code = main(
106 [
107 str(FIXTURES / "github_audit_acme_2026-01-01"),
108 "--baseline",
109 str(BASELINE),
110 "--format",
111 "json",
112 ]
113 )
114 assert code == 2