audit-labs/control-coverage
Control coverage and blind-spot analysis for audit evidence.
clone: git clone https://gitbay.org/audit-labs/control-coverage.git
v0.1.0: control_coverage/trend.py · raw
1"""Coverage trend — how control coverage moved between two corpora.
2
3`audit-report` diffs two evidence *packages*; this diffs two whole *corpora* at
4the framework-coverage level. Evaluate an earlier corpus and a current one with
5the same catalogs and scope, then compare each control's assurance state to see
6what improved, what regressed, and how the coverage percentage moved.
7
8States are ranked ``supported > failing > asserted > unaddressed`` — going from
9"no data" to "failing data" still counts as more assurance, because you now have
10evidence. Two transitions are called out specially because they move the coverage
11numerator: **gained** (a blind spot became addressed) and **lost** (an addressed
12control became a blind spot).
13"""
14
15from __future__ import annotations
16
17from dataclasses import dataclass, field
18
19from .coverage import (
20 ASSERTED,
21 FAILING,
22 OUT_OF_SCOPE,
23 SUPPORTED,
24 UNADDRESSED,
25 CoverageReport,
26)
27
28# Assurance rank; higher is more assured. out_of_scope is handled separately.
29_RANK = {SUPPORTED: 3, FAILING: 2, ASSERTED: 1, UNADDRESSED: 0}
30_ADDRESSED = {SUPPORTED, FAILING, ASSERTED}
31
32REGRESSED = "regressed"
33IMPROVED = "improved"
34GAINED = "gained"
35LOST = "lost"
36RESCOPED = "rescoped"
37UNCHANGED = "unchanged"
38
39# Order categories appear in a report (most urgent first).
40CATEGORY_ORDER = [REGRESSED, LOST, GAINED, IMPROVED, RESCOPED, UNCHANGED]
41# Categories that count as a regression for the CI gate.
42_REGRESSION = {REGRESSED, LOST}
43
44
45def _categorize(old: str, new: str) -> str:
46 if old == new:
47 return UNCHANGED
48 if OUT_OF_SCOPE in (old, new):
49 return RESCOPED
50 if old == UNADDRESSED and new in _ADDRESSED:
51 return GAINED
52 if old in _ADDRESSED and new == UNADDRESSED:
53 return LOST
54 return IMPROVED if _RANK[new] > _RANK[old] else REGRESSED
55
56
57@dataclass
58class ControlDelta:
59 """How one control's assurance state changed between the two corpora."""
60
61 framework: str
62 id: str
63 title: str
64 old_state: str
65 new_state: str
66 category: str
67
68
69@dataclass
70class FrameworkTrend:
71 """Coverage movement for one framework."""
72
73 framework: str
74 name: str
75 deltas: list[ControlDelta]
76 old_coverage_pct: float
77 new_coverage_pct: float
78
79 def by_category(self, category: str) -> list[ControlDelta]:
80 return [d for d in self.deltas if d.category == category]
81
82 @property
83 def counts(self) -> dict[str, int]:
84 counts = {c: 0 for c in CATEGORY_ORDER}
85 for d in self.deltas:
86 counts[d.category] += 1
87 return counts
88
89 @property
90 def coverage_delta(self) -> float:
91 return round(self.new_coverage_pct - self.old_coverage_pct, 1)
92
93 @property
94 def regressions(self) -> int:
95 return sum(1 for d in self.deltas if d.category in _REGRESSION)
96
97
98@dataclass
99class TrendReport:
100 subject: str
101 generated_at: str
102 frameworks: list[FrameworkTrend] = field(default_factory=list)
103
104 @property
105 def total_regressions(self) -> int:
106 return sum(fc.regressions for fc in self.frameworks)
107
108
109def compare(baseline: CoverageReport, current: CoverageReport) -> TrendReport:
110 """Diff two coverage reports evaluated with the same catalogs and scope."""
111 old_by_fw = {fc.catalog.framework: fc for fc in baseline.frameworks}
112
113 frameworks: list[FrameworkTrend] = []
114 for cur in current.frameworks:
115 base = old_by_fw.get(cur.catalog.framework)
116 if base is None:
117 continue # framework only appears in the current run
118 old_state = {r.control.id: r.state for r in base.results}
119 deltas: list[ControlDelta] = []
120 for r in cur.results:
121 prev = old_state.get(r.control.id, UNADDRESSED)
122 deltas.append(
123 ControlDelta(
124 framework=cur.catalog.framework,
125 id=r.control.id,
126 title=r.control.title,
127 old_state=prev,
128 new_state=r.state,
129 category=_categorize(prev, r.state),
130 )
131 )
132 frameworks.append(
133 FrameworkTrend(
134 framework=cur.catalog.framework,
135 name=cur.catalog.name,
136 deltas=deltas,
137 old_coverage_pct=base.coverage_pct,
138 new_coverage_pct=cur.coverage_pct,
139 )
140 )
141
142 return TrendReport(
143 subject=current.subject,
144 generated_at=current.generated_at,
145 frameworks=frameworks,
146 )
147
148
149# --- renderers -------------------------------------------------------------
150
151_ARROW = {
152 REGRESSED: "▼",
153 LOST: "▼",
154 GAINED: "▲",
155 IMPROVED: "▲",
156 RESCOPED: "◆",
157 UNCHANGED: "·",
158}
159
160
161def to_dict(report: TrendReport) -> dict:
162 return {
163 "subject": report.subject,
164 "generated_at": report.generated_at,
165 "total_regressions": report.total_regressions,
166 "frameworks": [
167 {
168 "framework": fc.framework,
169 "name": fc.name,
170 "old_coverage_pct": fc.old_coverage_pct,
171 "new_coverage_pct": fc.new_coverage_pct,
172 "coverage_delta": fc.coverage_delta,
173 "counts": fc.counts,
174 "changes": [
175 {
176 "id": d.id,
177 "title": d.title,
178 "old_state": d.old_state,
179 "new_state": d.new_state,
180 "category": d.category,
181 }
182 for d in fc.deltas
183 if d.category != UNCHANGED
184 ],
185 }
186 for fc in report.frameworks
187 ],
188 }
189
190
191def render_json(report: TrendReport) -> str:
192 import json
193
194 return json.dumps(to_dict(report), indent=2, sort_keys=False) + "\n"
195
196
197def render_markdown(report: TrendReport) -> str:
198 out: list[str] = []
199 out.append(f"# Coverage Trend — {report.subject or 'Evidence corpus'}")
200 out.append("")
201 out.append(f"- **Generated:** {report.generated_at}")
202 out.append(f"- **Regressions:** {report.total_regressions}")
203 out.append("")
204
205 out.append("| Framework | Coverage (was → now) | Δ | Regressed | Lost | Gained | Improved |")
206 out.append("| --- | --- | ---: | ---: | ---: | ---: | ---: |")
207 for fc in report.frameworks:
208 c = fc.counts
209 sign = "+" if fc.coverage_delta >= 0 else ""
210 out.append(
211 f"| {fc.name} | {fc.old_coverage_pct}% → {fc.new_coverage_pct}% "
212 f"| {sign}{fc.coverage_delta} | {c[REGRESSED]} | {c[LOST]} | {c[GAINED]} | {c[IMPROVED]} |"
213 )
214 out.append("")
215
216 for fc in report.frameworks:
217 changes = [d for d in fc.deltas if d.category != UNCHANGED]
218 if not changes:
219 continue
220 out.append(f"## {fc.name}")
221 out.append("")
222 out.append("| Control | Change | Was → Now | Description |")
223 out.append("| --- | --- | --- | --- |")
224 ordered = sorted(changes, key=lambda d: CATEGORY_ORDER.index(d.category))
225 for d in ordered:
226 arrow = _ARROW[d.category]
227 out.append(
228 f"| **{d.id}** | {arrow} {d.category} | {d.old_state} → {d.new_state} | {d.title} |"
229 )
230 out.append("")
231
232 if all(all(x.category == UNCHANGED for x in fc.deltas) for fc in report.frameworks):
233 out.append("_No control changed state between the two corpora._")
234 out.append("")
235
236 return "\n".join(out).rstrip() + "\n"
237
238
239# Category badge colours, layered onto the shared coverage CSS.
240_TREND_CSS = """
241.badge.improved { background: #e5f6ea; color: #1a7f37; }
242.badge.gained { background: #e3eefb; color: #1667a8; }
243.badge.regressed { background: #fdeaea; color: #c1272d; }
244.badge.lost { background: #fbe7d8; color: #a8480a; }
245.badge.rescoped { background: #eee; color: #666; }
246.delta-up { color: #1a7f37; font-weight: 700; }
247.delta-down { color: #c1272d; font-weight: 700; }
248@media (prefers-color-scheme: dark) {
249 .badge.improved { background: #12321d; color: #4ac36a; }
250 .badge.gained { background: #12263a; color: #5aa6e6; }
251 .badge.regressed { background: #3a1416; color: #ff6b70; }
252 .badge.lost { background: #33220f; color: #e0913c; }
253 .badge.rescoped { background: #26272b; color: #999; }
254}
255"""
256
257
258def render_html(report: TrendReport) -> str:
259 from html import escape
260
261 from .reporters.html import CSS
262
263 def badge(category: str) -> str:
264 return f'<span class="badge {category}">{_ARROW[category]} {category}</span>'
265
266 def delta(value: float) -> str:
267 cls = "delta-up" if value >= 0 else "delta-down"
268 sign = "+" if value >= 0 else ""
269 return f'<span class="{cls}">{sign}{value}</span>'
270
271 title = report.subject or "Evidence corpus"
272 body = [
273 "<!doctype html><html lang='en'><head><meta charset='utf-8'>",
274 "<meta name='viewport' content='width=device-width, initial-scale=1'>",
275 f"<title>Coverage Trend — {escape(title)}</title>",
276 f"<style>{CSS}{_TREND_CSS}</style></head><body><main>",
277 f"<h1>Coverage Trend — {escape(title)}</h1>",
278 (
279 f'<p class="meta">Generated {escape(report.generated_at)} · '
280 f"{report.total_regressions} regression(s)</p>"
281 ),
282 "<h2>Summary</h2>",
283 (
284 "<table><thead><tr><th>Framework</th><th>Coverage (was → now)</th>"
285 "<th class='num'>Δ</th><th class='num'>Regressed</th><th class='num'>Lost</th>"
286 "<th class='num'>Gained</th><th class='num'>Improved</th></tr></thead><tbody>"
287 ),
288 ]
289 for fc in report.frameworks:
290 c = fc.counts
291 body.append(
292 "<tr>"
293 f"<td>{escape(fc.name)}</td>"
294 f"<td>{fc.old_coverage_pct}% → {fc.new_coverage_pct}%</td>"
295 f'<td class="num">{delta(fc.coverage_delta)}</td>'
296 f'<td class="num">{c[REGRESSED]}</td><td class="num">{c[LOST]}</td>'
297 f'<td class="num">{c[GAINED]}</td><td class="num">{c[IMPROVED]}</td>'
298 "</tr>"
299 )
300 body.append("</tbody></table>")
301
302 for fc in report.frameworks:
303 changes = sorted(
304 (d for d in fc.deltas if d.category != UNCHANGED),
305 key=lambda d: CATEGORY_ORDER.index(d.category),
306 )
307 if not changes:
308 continue
309 body.append(f"<h2>{escape(fc.name)}</h2>")
310 body.append(
311 "<table><thead><tr><th>Control</th><th>Change</th><th>Was → Now</th>"
312 "<th>Description</th></tr></thead><tbody>"
313 )
314 for d in changes:
315 body.append(
316 "<tr>"
317 f"<td><strong>{escape(d.id)}</strong></td>"
318 f"<td>{badge(d.category)}</td>"
319 f"<td>{d.old_state} → {d.new_state}</td>"
320 f"<td>{escape(d.title)}</td>"
321 "</tr>"
322 )
323 body.append("</tbody></table>")
324
325 if not any(any(x.category != UNCHANGED for x in fc.deltas) for fc in report.frameworks):
326 body.append("<p>No control changed state between the two corpora.</p>")
327
328 body.append(
329 "<footer>Generated by control-coverage · Audit Labs. Evidence, not a verdict.</footer>"
330 )
331 body.append("</main></body></html>")
332 return "".join(body)