audit-labs/control-coverage

  1"""Compute control coverage: join an evidence corpus onto framework catalogs.
  2
  3For every control in a catalog we assign one **assurance state**:
  4
  5* ``supported``   — in scope, at least one mapped finding passes and none fail.
  6* ``failing``     — in scope, at least one mapped finding fails.
  7* ``asserted``    — in scope, findings map here but their data was absent
  8                    (``not_applicable``): evidence was attempted, not obtained.
  9* ``unaddressed`` — in scope, *no* finding maps here at all. The blind spot.
 10* ``out_of_scope``— excluded by the scope file, with a recorded justification.
 11
 12When several findings touch one control the worst wins: a single failure makes the
 13control ``failing`` regardless of how many others pass. Coverage is then a headline
 14number the evidence-first tools cannot produce — of everything a framework
 15requires, how much the corpus even looks at, and how much it supports.
 16"""
 17
 18from __future__ import annotations
 19
 20from dataclasses import dataclass, field
 21
 22from .catalog import Catalog, Control
 23from .corpus import FAIL, PASS, Observation
 24
 25SUPPORTED = "supported"
 26FAILING = "failing"
 27ASSERTED = "asserted"
 28UNADDRESSED = "unaddressed"
 29OUT_OF_SCOPE = "out_of_scope"
 30
 31# Order states appear in reports and roll up in summaries (most urgent first).
 32STATE_ORDER = [FAILING, UNADDRESSED, ASSERTED, SUPPORTED, OUT_OF_SCOPE]
 33
 34# States that count as the control being "addressed" by the corpus at all.
 35_ADDRESSED = {SUPPORTED, FAILING, ASSERTED}
 36
 37
 38@dataclass
 39class ControlResult:
 40    """One control's assurance state and the evidence behind it."""
 41
 42    control: Control
 43    state: str
 44    observations: list[Observation] = field(default_factory=list)
 45    owner: str = ""
 46    exclusion_reason: str = ""
 47
 48    @property
 49    def addressed(self) -> bool:
 50        return self.state in _ADDRESSED
 51
 52
 53def _state_for(observations: list[Observation]) -> str:
 54    """Worst-wins resolution of a control's state from its observations."""
 55    if not observations:
 56        return UNADDRESSED
 57    statuses = {o.status for o in observations}
 58    if FAIL in statuses:
 59        return FAILING
 60    if PASS in statuses:
 61        return SUPPORTED
 62    return ASSERTED  # only not_applicable observations remain
 63
 64
 65@dataclass
 66class FrameworkCoverage:
 67    """Coverage of one framework catalog by the corpus."""
 68
 69    catalog: Catalog
 70    results: list[ControlResult]
 71
 72    def by_state(self, state: str) -> list[ControlResult]:
 73        return [r for r in self.results if r.state == state]
 74
 75    @property
 76    def counts(self) -> dict[str, int]:
 77        counts = dict.fromkeys(STATE_ORDER, 0)
 78        for r in self.results:
 79            counts[r.state] += 1
 80        return counts
 81
 82    @property
 83    def in_scope(self) -> int:
 84        return sum(1 for r in self.results if r.state != OUT_OF_SCOPE)
 85
 86    @property
 87    def addressed(self) -> int:
 88        return sum(1 for r in self.results if r.addressed)
 89
 90    @property
 91    def supported(self) -> int:
 92        return sum(1 for r in self.results if r.state == SUPPORTED)
 93
 94    @property
 95    def coverage_pct(self) -> float:
 96        """Share of in-scope controls the corpus touches at all (0–100)."""
 97        return round(100 * self.addressed / self.in_scope, 1) if self.in_scope else 0.0
 98
 99    @property
100    def assured_pct(self) -> float:
101        """Share of in-scope controls that are supported and not failing (0–100)."""
102        return round(100 * self.supported / self.in_scope, 1) if self.in_scope else 0.0
103
104    @property
105    def blind_spots(self) -> list[ControlResult]:
106        """In-scope controls no finding touches — the headline gap list."""
107        return self.by_state(UNADDRESSED)
108
109
110@dataclass
111class CoverageReport:
112    """Coverage across every requested framework, plus corpus-wide diagnostics."""
113
114    subject: str
115    generated_at: str
116    frameworks: list[FrameworkCoverage]
117    # Control codes cited by the corpus that no loaded catalog defines. These are
118    # typos, renamed controls, or controls outside the bundled catalogs — either
119    # way, evidence pointing at nothing is worth surfacing.
120    orphan_codes: list[str] = field(default_factory=list)
121    source_count: int = 0
122
123
124def _observations_by_control(observations: list[Observation]) -> dict[str, list[Observation]]:
125    grouped: dict[str, list[Observation]] = {}
126    for obs in observations:
127        grouped.setdefault(obs.control, []).append(obs)
128    return grouped
129
130
131def evaluate(
132    catalogs: list[Catalog],
133    observations: list[Observation],
134    scope=None,
135    subject: str = "",
136    generated_at: str = "",
137) -> CoverageReport:
138    """Produce a :class:`CoverageReport` from catalogs, a corpus, and a scope."""
139    grouped = _observations_by_control(observations)
140    catalog_codes: set[str] = set()
141    loaded_frameworks = {cat.framework for cat in catalogs}
142
143    frameworks: list[FrameworkCoverage] = []
144    for cat in catalogs:
145        catalog_codes |= cat.codes()
146        results: list[ControlResult] = []
147        for control in cat.controls:
148            code = control.code
149            obs = grouped.get(code, [])
150            owner = scope.owner(code) if scope else ""
151            if scope and scope.excluded(code):
152                results.append(
153                    ControlResult(control, OUT_OF_SCOPE, obs, owner, scope.reason(code))
154                )
155            elif scope and scope.family_excluded(cat.framework, control.family):
156                reason = scope.family_reason(cat.framework, control.family)
157                results.append(ControlResult(control, OUT_OF_SCOPE, obs, owner, reason))
158            else:
159                results.append(ControlResult(control, _state_for(obs), obs, owner))
160        frameworks.append(FrameworkCoverage(cat, results))
161
162    # A code is an orphan only when its framework *is* loaded but the catalog
163    # does not define it — a typo or a renamed control. Codes for frameworks we
164    # did not load this run are simply out of scope, not orphans.
165    cited = {o.control for o in observations}
166    orphans = sorted(
167        code
168        for code in cited
169        if code.split(":", 1)[0] in loaded_frameworks and code not in catalog_codes
170    )
171    sources = {o.source for o in observations}
172
173    return CoverageReport(
174        subject=subject,
175        generated_at=generated_at,
176        frameworks=frameworks,
177        orphan_codes=orphans,
178        source_count=len(sources),
179    )