audit-labs/control-coverage
Control coverage and blind-spot analysis for audit evidence.
clone: git clone https://gitbay.org/audit-labs/control-coverage.git
4187ceff35942b36fdbf3b3f520dfd92c99c003d
verified · cmc
author: Christian Cleberg <hello@cleberg.net> · 2026-08-07T01:07:59Z
.gitignore | 20 + CODEOWNERS | 1 + GUIDE.md | 198 +++++++ LICENSE | 674 +++++++++++++++++++++++ README.md | 184 +++++++ conftest.py | 11 + control_coverage/__init__.py | 3 + control_coverage/__main__.py | 6 + control_coverage/catalog.py | 116 ++++ control_coverage/catalogs/iso27001.yaml | 107 ++++ control_coverage/catalogs/nist80053.yaml | 210 +++++++ control_coverage/catalogs/soc2.yaml | 85 +++ control_coverage/cli.py | 263 +++++++++ control_coverage/corpus.py | 88 +++ control_coverage/coverage.py | 179 ++++++ control_coverage/crosswalk.py | 291 ++++++++++ control_coverage/reporters/__init__.py | 26 + control_coverage/reporters/html.py | 252 +++++++++ control_coverage/reporters/json.py | 55 ++ control_coverage/reporters/markdown.py | 153 +++++ control_coverage/reporters/soa.py | 68 +++ control_coverage/scope.py | 106 ++++ control_coverage/trend.py | 332 +++++++++++ examples/github-actions-coverage.yml | 43 ++ examples/soa.yaml | 27 + pyproject.toml | 31 ++ requirements-dev.txt | 3 + requirements.txt | 1 + ruff.toml | 14 + tests/fixtures/aws_audit_acme_2026-01-01.json | 46 ++ tests/fixtures/baseline_github.json | 46 ++ tests/fixtures/github_audit_acme_2026-01-01.json | 46 ++ tests/fixtures/scope.yaml | 11 + tests/test_catalog.py | 74 +++ tests/test_cli.py | 117 ++++ tests/test_corpus.py | 42 ++ tests/test_coverage.py | 91 +++ tests/test_crosswalk.py | 67 +++ tests/test_reporters.py | 55 ++ tests/test_scope.py | 65 +++ tests/test_trend.py | 75 +++ 41 files changed, 4282 insertions(+) new file mode 100644 @@ -0,0 +1,20 @@ +.venv +venv + +# Python +__pycache__/ +**/__pycache__/ +*.py[cod] +*.egg-info/ +.pytest_cache/ +.ruff_cache/ +build/ +dist/ + +# Generated output +/coverage-out/ +soa.md +soa.html +coverage.md +coverage.html +coverage.json new file mode 100644 @@ -0,0 +1 @@ +* @ccleberg @ekraai2 new file mode 100644 @@ -0,0 +1,198 @@ +# Using control-coverage + +A practical, task-oriented guide. For the conceptual overview see the +[README](README.md); this walks through actually running the tool. + +## The one thing to understand first + +Every other Audit Labs tool is **evidence-first**: it starts from what you collected +and tells you what it maps to. `control-coverage` is **control-first**: it starts +from the *complete* list of a framework's controls and tells you how much of it your +evidence addresses — and, more usefully, what it *doesn't*. + +So the input is your evidence, and the output is measured against a fixed yardstick +(the framework catalog) you didn't have to write. + +## 1. Get the input: audit-report JSON + +The corpus is one or more JSON reports from `audit-report`. Produce them with its +`--format json` flag, one per platform: + +```bash +audit-report ./output/aws_audit_prod_2026-02-01 --format json --out reports/ +audit-report ./output/github_audit_prod_2026-02-01 --format json --out reports/ +``` + +You now have `reports/*.json`. That directory *is* a corpus — coverage aggregates +every report in it into one per-framework picture, so AWS, GitHub, and GitLab +evidence all count toward the same SOC 2 number. + +> No audit-report packages yet? Any JSON with the same shape works — a list of +> `findings`, each with `controls: ["SOC2:CC6.1", ...]` and a `status` of `pass`, +> `fail`, or `not_applicable`. + +## 2. Install + +```bash +git clone https://github.com/audit-labs/control-coverage +cd control-coverage +python -m venv .venv && source .venv/bin/activate +pip install -e . +``` + +## 3. The four things you'll actually do + +### A. "How covered am I, and what am I missing?" + +```bash +control-coverage reports/ +``` + +Prints a Markdown report: a per-framework summary table, then the **blind spots** +(in-scope controls no finding touches), then the full matrix. Frameworks are inferred +from the codes your corpus cites. + +Want just the gap list, nothing else? + +```bash +control-coverage reports/ --framework SOC2 --blind-spots +``` + +Want files to hand off? Write all formats to a directory: + +```bash +control-coverage reports/ --format md,html,json --out coverage-out/ +``` + +- **md** — human-readable, good for a PR comment or a wiki paste. +- **html** — self-contained, printable, has coverage bars. Attach to a workpaper. +- **json** — for dashboards or further scripting. + +### B. "Produce a Statement of Applicability" + +First write a scope file. It selects frameworks and records exclusions — each with a +mandatory reason (an unjustified exclusion is rejected): + +```yaml +# soa.yaml +subject: Acme Production +frameworks: [SOC2, ISO] +exclusions: + - control: ISO:A.7.1 + reason: "Fully cloud-hosted; no physical premises are in scope for the ISMS." + - control: ISO:A.5.7 + reason: "No formal threat-intelligence program; risk accepted by the CISO for 2026." +exclude_families: + - {framework: SOC2, family: Privacy, reason: "Privacy category not in the SOC 2 audit scope."} +owners: + SOC2:CC6.1: platform-team +``` + +`exclusions` drops one control; `exclude_families` drops a whole category, ISO theme, +or NIST family at once (how audit scope is really decided). Both require a reason. + +Then generate coverage over the in-scope controls, plus the SoA itself: + +```bash +control-coverage reports/ --scope soa.yaml --format md,soa --out coverage-out/ +``` + +`coverage-out/soa.md` lists every control, whether it applies, its implementation +status (derived from your evidence, not asserted by hand), and the justification. +Excluded controls are recorded, not counted as gaps. + +### C. "What changed since last time?" + +Keep last month's reports around. Point `--baseline` at them: + +```bash +control-coverage reports/2026-02/ --baseline reports/2026-01/ --framework SOC2 +``` + +You get a movement report: + +- **improved** — a control got more assurance (e.g. failing → supported). +- **regressed** — a control lost assurance (e.g. supported → failing). +- **gained** — a blind spot became addressed (coverage went up). +- **lost** — an addressed control became a blind spot (coverage went down). + +`--baseline` accepts a single file or a directory. + +### D. "Which evidence is doing the most work?" + +```bash +control-coverage reports/ --framework SOC2,ISO,NIST --crosswalk +``` + +Two things come out: + +- **Evidence leverage** — each check and the controls it supports, across all three + frameworks. You'll see that one 2FA check earns SOC 2 CC6.1 + ISO A.5.17 + NIST IA-2. +- **Minimal evidence set** — the fewest checks that still cover every addressed + control. This is your walkthrough/sampling short-list: pull these and you've touched + everything the full corpus touches. + +## 4. Reading the numbers + +Every in-scope control is in exactly one state: + +| State | What it means | Counts toward… | +| --- | --- | --- | +| **supported** | Something passes here, nothing fails | coverage **and** assured | +| **failing** | Something fails here (worst wins) | coverage | +| **asserted** | Mapped, but the data was absent | coverage | +| **unaddressed** | Nothing maps here — a blind spot | neither | +| **out of scope** | Excluded in the scope file, with a reason | neither (removed from the denominator) | + +- **Coverage %** = supported + failing + asserted, over in-scope controls. *"How much + of the framework am I even looking at?"* +- **Assured %** = supported only, over in-scope controls. *"How much do I have good + evidence for?"* + +A low coverage number on a fresh corpus is expected — the framework is large and your +automated checks touch a slice of it. The value is knowing *exactly which* slice, and +watching coverage climb (via `--baseline`) as you add evidence. + +## 5. Wire it into CI + +Two independent gates, both exit non-zero to fail a build: + +```bash +# Fail if any framework's coverage drops below a floor +control-coverage reports/ --scope soa.yaml --fail-under 60 + +# Fail if anything regressed or lost coverage versus the last run +control-coverage reports/ --baseline last-run/ --fail-on-regression +``` + +See [`examples/github-actions-coverage.yml`](examples/github-actions-coverage.yml) for +a scheduled workflow that runs both and uploads the reports as artifacts. + +## 6. Frameworks and codes + +| Framework | Pass as | Catalog | +| --- | --- | --- | +| SOC 2 Trust Services Criteria | `SOC2` | All five categories, 61 controls | +| ISO/IEC 27001:2022 Annex A | `ISO` (or `iso27001`) | All 93 Annex A controls | +| NIST SP 800-53 Rev. 5 | `NIST` (or `800-53`) | Moderate baseline, 177 base controls | + +Control codes are `FRAMEWORK:ID` — `SOC2:CC6.1`, `ISO:A.5.17`, `NIST:IA-2` — the same +codes `audit-report` rulesets already emit, so the two tools line up with no +translation. If your corpus cites a code whose framework is loaded but the catalog +doesn't define it (a typo or a renamed control), it's reported under **Unmatched +control codes** rather than silently dropped. + +## Gotchas + +- **`--crosswalk` and `--baseline` can't be combined** — they're different analyses. +- **Trend and crosswalk emit `md`, `html`, and `json`** (not `soa`). Coverage emits all four. +- **SOC 2 defaults to all five categories (61 controls).** Most reports scope only some. + Drop the ones you're not audited on with `exclude_families` (see §2) so coverage + reflects your real perimeter — e.g. exclude `Privacy` and `Processing Integrity`. +- **NIST coverage looks low** because the moderate baseline is large (177 controls) and + most are organizational/physical/personnel controls no automated scanner evidences. + That's the point — those are your blind spots. Use a scope file to exclude the ones + handled by policy rather than tooling, so the number reflects your real perimeter. +- **An unaddressed control is a gap in *evidence*, not proof of a gap in *controls*.** + It may just mean the signal isn't collected yet. The tool produces evidence, never a + verdict. new file mode 100644 @@ -0,0 +1,674 @@ + GNU GENERAL PUBLIC LICENSE + Version 3, 29 June 2007 + + Copyright (C) 2007 Free Software Foundation, Inc. <https://fsf.org/> + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The GNU General Public License is a free, copyleft license for +software and other kinds of works. + + The licenses for most software and other practical works are designed +to take away your freedom to share and change the works. By contrast, +the GNU General Public License is intended to guarantee your freedom to +share and change all versions of a program--to make sure it remains free +software for all its users. We, the Free Software Foundation, use the +GNU General Public License for most of our software; it applies also to +any other work released this way by its authors. You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +them if you wish), that you receive source code or can get it if you +want it, that you can change the software or use pieces of it in new +free programs, and that you know you can do these things. + + To protect your rights, we need to prevent others from denying you +these rights or asking you to surrender the rights. Therefore, you have +certain responsibilities if you distribute copies of the software, or if +you modify it: responsibilities to respect the freedom of others. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must pass on to the recipients the same +freedoms that you received. You must make sure that they, too, receive +or can get the source code. And you must show them these terms so they +know their rights. + + Developers that use the GNU GPL protect your rights with two steps: +(1) assert copyright on the software, and (2) offer you this License +giving you legal permission to copy, distribute and/or modify it. + + For the developers' and authors' protection, the GPL clearly explains +that there is no warranty for this free software. For both users' and +authors' sake, the GPL requires that modified versions be marked as +changed, so that their problems will not be attributed erroneously to +authors of previous versions. + + Some devices are designed to deny users access to install or run +modified versions of the software inside them, although the manufacturer +can do so. This is fundamentally incompatible with the aim of +protecting users' freedom to change the software. The systematic +pattern of such abuse occurs in the area of products for individuals to +use, which is precisely where it is most unacceptable. Therefore, we +have designed this version of the GPL to prohibit the practice for those +products. If such problems arise substantially in other domains, we +stand ready to extend this provision to those domains in future versions +of the GPL, as needed to protect the freedom of users. + + Finally, every program is threatened constantly by software patents. +States should not allow patents to restrict development and use of +software on general-purpose computers, but in those that do, we wish to +avoid the special danger that patents applied to a free program could +make it effectively proprietary. To prevent this, the GPL assures that +patents cannot be used to render the program non-free. + + The precise terms and conditions for copying, distribution and +modification follow. + + TERMS AND CONDITIONS + + 0. Definitions. + + "This License" refers to version 3 of the GNU General Public License. + + "Copyright" also means copyright-like laws that apply to other kinds of +works, such as semiconductor masks. + + "The Program" refers to any copyrightable work licensed under this +License. Each licensee is addressed as "you". "Licensees" and +"recipients" may be individuals or organizations. + + To "modify" a work means to copy from or adapt all or part of the work +in a fashion requiring copyright permission, other than the making of an +exact copy. The resulting work is called a "modified version" of the +earlier work or a work "based on" the earlier work. + + A "covered work" means either the unmodified Program or a work based +on the Program. + + To "propagate" a work means to do anything with it that, without +permission, would make you directly or secondarily liable for +infringement under applicable copyright law, except executing it on a +computer or modifying a private copy. Propagation includes copying, +distribution (with or without modification), making available to the +public, and in some countries other activities as well. + + To "convey" a work means any kind of propagation that enables other +parties to make or receive copies. Mere interaction with a user through +a computer network, with no transfer of a copy, is not conveying. + + An interactive user interface displays "Appropriate Legal Notices" +to the extent that it includes a convenient and prominently visible +feature that (1) displays an appropriate copyright notice, and (2) +tells the user that there is no warranty for the work (except to the +extent that warranties are provided), that licensees may convey the +work under this License, and how to view a copy of this License. If +the interface presents a list of user commands or options, such as a +menu, a prominent item in the list meets this criterion. + + 1. Source Code. + + The "source code" for a work means the preferred form of the work +for making modifications to it. "Object code" means any non-source +form of a work. + + A "Standard Interface" means an interface that either is an official +standard defined by a recognized standards body, or, in the case of +interfaces specified for a particular programming language, one that +is widely used among developers working in that language. + + The "System Libraries" of an executable work include anything, other +than the work as a whole, that (a) is included in the normal form of +packaging a Major Component, but which is not part of that Major +Component, and (b) serves only to enable use of the work with that +Major Component, or to implement a Standard Interface for which an +implementation is available to the public in source code form. A +"Major Component", in this context, means a major essential component +(kernel, window system, and so on) of the specific operating system +(if any) on which the executable work runs, or a compiler used to +produce the work, or an object code interpreter used to run it. + + The "Corresponding Source" for a work in object code form means all +the source code needed to generate, install, and (for an executable +work) run the object code and to modify the work, including scripts to +control those activities. However, it does not include the work's +System Libraries, or general-purpose tools or generally available free +programs which are used unmodified in performing those activities but +which are not part of the work. For example, Corresponding Source +includes interface definition files associated with source files for +the work, and the source code for shared libraries and dynamically +linked subprograms that the work is specifically designed to require, +such as by intimate data communication or control flow between those +subprograms and other parts of the work. + + The Corresponding Source need not include anything that users +can regenerate automatically from other parts of the Corresponding +Source. + + The Corresponding Source for a work in source code form is that +same work. + + 2. Basic Permissions. + + All rights granted under this License are granted for the term of +copyright on the Program, and are irrevocable provided the stated +conditions are met. This License explicitly affirms your unlimited +permission to run the unmodified Program. The output from running a +covered work is covered by this License only if the output, given its +content, constitutes a covered work. This License acknowledges your +rights of fair use or other equivalent, as provided by copyright law. + + You may make, run and propagate covered works that you do not +convey, without conditions so long as your license otherwise remains +in force. You may convey covered works to others for the sole purpose +of having them make modifications exclusively for you, or provide you +with facilities for running those works, provided that you comply with +the terms of this License in conveying all material for which you do +not control copyright. Those thus making or running the covered works +for you must do so exclusively on your behalf, under your direction +and control, on terms that prohibit them from making any copies of +your copyrighted material outside their relationship with you. + + Conveying under any other circumstances is permitted solely under +the conditions stated below. Sublicensing is not allowed; section 10 +makes it unnecessary. + + 3. Protecting Users' Legal Rights From Anti-Circumvention Law. + + No covered work shall be deemed part of an effective technological +measure under any applicable law fulfilling obligations under article +11 of the WIPO copyright treaty adopted on 20 December 1996, or +similar laws prohibiting or restricting circumvention of such +measures. + + When you convey a covered work, you waive any legal power to forbid +circumvention of technological measures to the extent such circumvention +is effected by exercising rights under this License with respect to +the covered work, and you disclaim any intention to limit operation or +modification of the work as a means of enforcing, against the work's +users, your or third parties' legal rights to forbid circumvention of +technological measures. + + 4. Conveying Verbatim Copies. + + You may convey verbatim copies of the Program's source code as you +receive it, in any medium, provided that you conspicuously and +appropriately publish on each copy an appropriate copyright notice; +keep intact all notices stating that this License and any +non-permissive terms added in accord with section 7 apply to the code; +keep intact all notices of the absence of any warranty; and give all +recipients a copy of this License along with the Program. + + You may charge any price or no price for each copy that you convey, +and you may offer support or warranty protection for a fee. + + 5. Conveying Modified Source Versions. + + You may convey a work based on the Program, or the modifications to +produce it from the Program, in the form of source code under the +terms of section 4, provided that you also meet all of these conditions: + + a) The work must carry prominent notices stating that you modified + it, and giving a relevant date. + + b) The work must carry prominent notices stating that it is + released under this License and any conditions added under section + 7. This requirement modifies the requirement in section 4 to + "keep intact all notices". + + c) You must license the entire work, as a whole, under this + License to anyone who comes into possession of a copy. This + License will therefore apply, along with any applicable section 7 + additional terms, to the whole of the work, and all its parts, + regardless of how they are packaged. This License gives no + permission to license the work in any other way, but it does not + invalidate such permission if you have separately received it. + + d) If the work has interactive user interfaces, each must display + Appropriate Legal Notices; however, if the Program has interactive + interfaces that do not display Appropriate Legal Notices, your + work need not make them do so. + + A compilation of a covered work with other separate and independent +works, which are not by their nature extensions of the covered work, +and which are not combined with it such as to form a larger program, +in or on a volume of a storage or distribution medium, is called an +"aggregate" if the compilation and its resulting copyright are not +used to limit the access or legal rights of the compilation's users +beyond what the individual works permit. Inclusion of a covered work +in an aggregate does not cause this License to apply to the other +parts of the aggregate. + + 6. Conveying Non-Source Forms. + + You may convey a covered work in object code form under the terms +of sections 4 and 5, provided that you also convey the +machine-readable Corresponding Source under the terms of this License, +in one of these ways: + + a) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by the + Corresponding Source fixed on a durable physical medium + customarily used for software interchange. + + b) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by a + written offer, valid for at least three years and valid for as + long as you offer spare parts or customer support for that product + model, to give anyone who possesses the object code either (1) a + copy of the Corresponding Source for all the software in the + product that is covered by this License, on a durable physical + medium customarily used for software interchange, for a price no + more than your reasonable cost of physically performing this + conveying of source, or (2) access to copy the + Corresponding Source from a network server at no charge. + + c) Convey individual copies of the object code with a copy of the + written offer to provide the Corresponding Source. This + alternative is allowed only occasionally and noncommercially, and + only if you received the object code with such an offer, in accord + with subsection 6b. + + d) Convey the object code by offering access from a designated + place (gratis or for a charge), and offer equivalent access to the + Corresponding Source in the same way through the same place at no + further charge. You need not require recipients to copy the + Corresponding Source along with the object code. If the place to + copy the object code is a network server, the Corresponding Source + may be on a different server (operated by you or a third party) + that supports equivalent copying facilities, provided you maintain + clear directions next to the object code saying where to find the + Corresponding Source. Regardless of what server hosts the + Corresponding Source, you remain obligated to ensure that it is + available for as long as needed to satisfy these requirements. + + e) Convey the object code using peer-to-peer transmission, provided + you inform other peers where the object code and Corresponding + Source of the work are being offered to the general public at no + charge under subsection 6d. + + A separable portion of the object code, whose source code is excluded +from the Corresponding Source as a System Library, need not be +included in conveying the object code work. + + A "User Product" is either (1) a "consumer product", which means any +tangible personal property which is normally used for personal, family, +or household purposes, or (2) anything designed or sold for incorporation +into a dwelling. In determining whether a product is a consumer product, +doubtful cases shall be resolved in favor of coverage. For a particular +product received by a particular user, "normally used" refers to a +typical or common use of that class of product, regardless of the status +of the particular user or of the way in which the particular user +actually uses, or expects or is expected to use, the product. A product +is a consumer product regardless of whether the product has substantial +commercial, industrial or non-consumer uses, unless such uses represent +the only significant mode of use of the product. + + "Installation Information" for a User Product means any methods, +procedures, authorization keys, or other information required to install +and execute modified versions of a covered work in that User Product from +a modified version of its Corresponding Source. The information must +suffice to ensure that the continued functioning of the modified object +code is in no case prevented or interfered with solely because +modification has been made. + + If you convey an object code work under this section in, or with, or +specifically for use in, a User Product, and the conveying occurs as +part of a transaction in which the right of possession and use of the +User Product is transferred to the recipient in perpetuity or for a +fixed term (regardless of how the transaction is characterized), the +Corresponding Source conveyed under this section must be accompanied +by the Installation Information. But this requirement does not apply +if neither you nor any third party retains the ability to install +modified object code on the User Product (for example, the work has +been installed in ROM). + + The requirement to provide Installation Information does not include a +requirement to continue to provide support service, warranty, or updates +for a work that has been modified or installed by the recipient, or for +the User Product in which it has been modified or installed. Access to a +network may be denied when the modification itself materially and +adversely affects the operation of the network or violates the rules and +protocols for communication across the network. + + Corresponding Source conveyed, and Installation Information provided, +in accord with this section must be in a format that is publicly +documented (and with an implementation available to the public in +source code form), and must require no special password or key for +unpacking, reading or copying. + + 7. Additional Terms. + + "Additional permissions" are terms that supplement the terms of this +License by making exceptions from one or more of its conditions. +Additional permissions that are applicable to the entire Program shall +be treated as though they were included in this License, to the extent +that they are valid under applicable law. If additional permissions +apply only to part of the Program, that part may be used separately +under those permissions, but the entire Program remains governed by +this License without regard to the additional permissions. + + When you convey a copy of a covered work, you may at your option +remove any additional permissions from that copy, or from any part of +it. (Additional permissions may be written to require their own +removal in certain cases when you modify the work.) You may place +additional permissions on material, added by you to a covered work, +for which you have or can give appropriate copyright permission. + + Notwithstanding any other provision of this License, for material you +add to a covered work, you may (if authorized by the copyright holders of +that material) supplement the terms of this License with terms: + + a) Disclaiming warranty or limiting liability differently from the + terms of sections 15 and 16 of this License; or + + b) Requiring preservation of specified reasonable legal notices or + author attributions in that material or in the Appropriate Legal + Notices displayed by works containing it; or + + c) Prohibiting misrepresentation of the origin of that material, or + requiring that modified versions of such material be marked in + reasonable ways as different from the original version; or + + d) Limiting the use for publicity purposes of names of licensors or + authors of the material; or + + e) Declining to grant rights under trademark law for use of some + trade names, trademarks, or service marks; or + + f) Requiring indemnification of licensors and authors of that + material by anyone who conveys the material (or modified versions of + it) with contractual assumptions of liability to the recipient, for + any liability that these contractual assumptions directly impose on + those licensors and authors. + + All other non-permissive additional terms are considered "further +restrictions" within the meaning of section 10. If the Program as you +received it, or any part of it, contains a notice stating that it is +governed by this License along with a term that is a further +restriction, you may remove that term. If a license document contains +a further restriction but permits relicensing or conveying under this +License, you may add to a covered work material governed by the terms +of that license document, provided that the further restriction does +not survive such relicensing or conveying. + + If you add terms to a covered work in accord with this section, you +must place, in the relevant source files, a statement of the +additional terms that apply to those files, or a notice indicating +where to find the applicable terms. + + Additional terms, permissive or non-permissive, may be stated in the +form of a separately written license, or stated as exceptions; +the above requirements apply either way. + + 8. Termination. + + You may not propagate or modify a covered work except as expressly +provided under this License. Any attempt otherwise to propagate or +modify it is void, and will automatically terminate your rights under +this License (including any patent licenses granted under the third +paragraph of section 11). + + However, if you cease all violation of this License, then your +license from a particular copyright holder is reinstated (a) +provisionally, unless and until the copyright holder explicitly and +finally terminates your license, and (b) permanently, if the copyright +holder fails to notify you of the violation by some reasonable means +prior to 60 days after the cessation. + + Moreover, your license from a particular copyright holder is +reinstated permanently if the copyright holder notifies you of the +violation by some reasonable means, this is the first time you have +received notice of violation of this License (for any work) from that +copyright holder, and you cure the violation prior to 30 days after +your receipt of the notice. + + Termination of your rights under this section does not terminate the +licenses of parties who have received copies or rights from you under +this License. If your rights have been terminated and not permanently +reinstated, you do not qualify to receive new licenses for the same +material under section 10. + + 9. Acceptance Not Required for Having Copies. + + You are not required to accept this License in order to receive or +run a copy of the Program. Ancillary propagation of a covered work +occurring solely as a consequence of using peer-to-peer transmission +to receive a copy likewise does not require acceptance. However, +nothing other than this License grants you permission to propagate or +modify any covered work. These actions infringe copyright if you do +not accept this License. Therefore, by modifying or propagating a +covered work, you indicate your acceptance of this License to do so. + + 10. Automatic Licensing of Downstream Recipients. + + Each time you convey a covered work, the recipient automatically +receives a license from the original licensors, to run, modify and +propagate that work, subject to this License. You are not responsible +for enforcing compliance by third parties with this License. + + An "entity transaction" is a transaction transferring control of an +organization, or substantially all assets of one, or subdividing an +organization, or merging organizations. If propagation of a covered +work results from an entity transaction, each party to that +transaction who receives a copy of the work also receives whatever +licenses to the work the party's predecessor in interest had or could +give under the previous paragraph, plus a right to possession of the +Corresponding Source of the work from the predecessor in interest, if +the predecessor has it or can get it with reasonable efforts. + + You may not impose any further restrictions on the exercise of the +rights granted or affirmed under this License. For example, you may +not impose a license fee, royalty, or other charge for exercise of +rights granted under this License, and you may not initiate litigation +(including a cross-claim or counterclaim in a lawsuit) alleging that +any patent claim is infringed by making, using, selling, offering for +sale, or importing the Program or any portion of it. + + 11. Patents. + + A "contributor" is a copyright holder who authorizes use under this +License of the Program or a work on which the Program is based. The +work thus licensed is called the contributor's "contributor version". + + A contributor's "essential patent claims" are all patent claims +owned or controlled by the contributor, whether already acquired or +hereafter acquired, that would be infringed by some manner, permitted +by this License, of making, using, or selling its contributor version, +but do not include claims that would be infringed only as a +consequence of further modification of the contributor version. For +purposes of this definition, "control" includes the right to grant +patent sublicenses in a manner consistent with the requirements of +this License. + + Each contributor grants you a non-exclusive, worldwide, royalty-free +patent license under the contributor's essential patent claims, to +make, use, sell, offer for sale, import and otherwise run, modify and +propagate the contents of its contributor version. + + In the following three paragraphs, a "patent license" is any express +agreement or commitment, however denominated, not to enforce a patent +(such as an express permission to practice a patent or covenant not to +sue for patent infringement). To "grant" such a patent license to a +party means to make such an agreement or commitment not to enforce a +patent against the party. + + If you convey a covered work, knowingly relying on a patent license, +and the Corresponding Source of the work is not available for anyone +to copy, free of charge and under the terms of this License, through a +publicly available network server or other readily accessible means, +then you must either (1) cause the Corresponding Source to be so +available, or (2) arrange to deprive yourself of the benefit of the +patent license for this particular work, or (3) arrange, in a manner +consistent with the requirements of this License, to extend the patent +license to downstream recipients. "Knowingly relying" means you have +actual knowledge that, but for the patent license, your conveying the +covered work in a country, or your recipient's use of the covered work +in a country, would infringe one or more identifiable patents in that +country that you have reason to believe are valid. + + If, pursuant to or in connection with a single transaction or +arrangement, you convey, or propagate by procuring conveyance of, a +covered work, and grant a patent license to some of the parties +receiving the covered work authorizing them to use, propagate, modify +or convey a specific copy of the covered work, then the patent license +you grant is automatically extended to all recipients of the covered +work and works based on it. + + A patent license is "discriminatory" if it does not include within +the scope of its coverage, prohibits the exercise of, or is +conditioned on the non-exercise of one or more of the rights that are +specifically granted under this License. You may not convey a covered +work if you are a party to an arrangement with a third party that is +in the business of distributing software, under which you make payment +to the third party based on the extent of your activity of conveying +the work, and under which the third party grants, to any of the +parties who would receive the covered work from you, a discriminatory +patent license (a) in connection with copies of the covered work +conveyed by you (or copies made from those copies), or (b) primarily +for and in connection with specific products or compilations that +contain the covered work, unless you entered into that arrangement, +or that patent license was granted, prior to 28 March 2007. + + Nothing in this License shall be construed as excluding or limiting +any implied license or other defenses to infringement that may +otherwise be available to you under applicable patent law. + + 12. No Surrender of Others' Freedom. + + If conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot convey a +covered work so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you may +not convey it at all. For example, if you agree to terms that obligate you +to collect a royalty for further conveying from those to whom you convey +the Program, the only way you could satisfy both those terms and this +License would be to refrain entirely from conveying the Program. + + 13. Use with the GNU Affero General Public License. + + Notwithstanding any other provision of this License, you have +permission to link or combine any covered work with a work licensed +under version 3 of the GNU Affero General Public License into a single +combined work, and to convey the resulting work. The terms of this +License will continue to apply to the part which is the covered work, +but the special requirements of the GNU Affero General Public License, +section 13, concerning interaction through a network will apply to the +combination as such. + + 14. Revised Versions of this License. + + The Free Software Foundation may publish revised and/or new versions of +the GNU General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + + Each version is given a distinguishing version number. If the +Program specifies that a certain numbered version of the GNU General +Public License "or any later version" applies to it, you have the +option of following the terms and conditions either of that numbered +version or of any later version published by the Free Software +Foundation. If the Program does not specify a version number of the +GNU General Public License, you may choose any version ever published +by the Free Software Foundation. + + If the Program specifies that a proxy can decide which future +versions of the GNU General Public License can be used, that proxy's +public statement of acceptance of a version permanently authorizes you +to choose that version for the Program. + + Later license versions may give you additional or different +permissions. However, no additional obligations are imposed on any +author or copyright holder as a result of your choosing to follow a +later version. + + 15. Disclaimer of Warranty. + + THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY +APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT +HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY +OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM +IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF +ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. Limitation of Liability. + + IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS +THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY +GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE +USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF +DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD +PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), +EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF +SUCH DAMAGES. + + 17. Interpretation of Sections 15 and 16. + + If the disclaimer of warranty and limitation of liability provided +above cannot be given local legal effect according to their terms, +reviewing courts shall apply local law that most closely approximates +an absolute waiver of all civil liability in connection with the +Program, unless a warranty or assumption of liability accompanies a +copy of the Program in return for a fee. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +state the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + <one line to give the program's name and a brief idea of what it does.> + Copyright (C) <year> <name of author> + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see <https://www.gnu.org/licenses/>. + +Also add information on how to contact you by electronic and paper mail. + + If the program does terminal interaction, make it output a short +notice like this when it starts in an interactive mode: + + <program> Copyright (C) <year> <name of author> + This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, your program's commands +might be different; for a GUI interface, you would use an "about box". + + You should also get your employer (if you work as a programmer) or school, +if any, to sign a "copyright disclaimer" for the program, if necessary. +For more information on this, and how to apply and follow the GNU GPL, see +<https://www.gnu.org/licenses/>. + + The GNU General Public License does not permit incorporating your program +into proprietary programs. If your program is a subroutine library, you +may consider it more useful to permit linking proprietary applications with +the library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. But first, please read +<https://www.gnu.org/licenses/why-not-lgpl.html>. new file mode 100644 @@ -0,0 +1,184 @@ +# control-coverage + +[](LICENSE) +[]() + +Control-first coverage and blind-spot analysis over an evidence corpus. + +The rest of the Audit Labs toolchain is **evidence-first**: [audit-tools](https://github.com/audit-labs/audit-tools) +collects raw signals, [audit-report](https://github.com/audit-labs/audit-report) +maps each finding onto the controls it touches, and [evidence-seal](https://github.com/audit-labs/evidence-seal) +proves the package is authentic. That answers *"what did I collect, and what does it +map to?"* — but it can never tell you what you are **not** looking at, because it has +no list of everything a framework requires. + +`control-coverage` supplies that missing list — the **denominator**. It starts from +the *complete* catalog of a framework's controls and scores your evidence against it, +so it can report two numbers nothing else in the pipeline can: + +- **Coverage %** — of everything the framework requires, how much the evidence corpus + addresses at all. +- **Blind spots** — the in-scope controls that *no* finding touches. These are the + gaps an auditor finds for you if you don't find them first. + +It also produces a **Statement of Applicability** — the ISO 27001 artifact that lists +every Annex A control, whether it applies, and why — derived from your evidence +instead of hand-maintained. + +> Like every Audit Labs tool, this produces *evidence*, not a verdict. An unaddressed +> control is a gap in *evidence*, which may reflect a real gap in *controls* or simply +> a signal not yet collected. The final judgment belongs to the organization and its +> auditor. + +## Install + +```bash +git clone https://github.com/audit-labs/control-coverage +cd control-coverage +python -m venv .venv && source .venv/bin/activate +pip install -e . +``` + +Pure standard library plus PyYAML — no other dependencies. + +## Usage + +The input is one or more JSON reports from `audit-report` (its `--format json` +output). A corpus is typically one report per platform and date — AWS, GitHub, +GitLab — which `control-coverage` folds into a single per-framework picture. + +```bash +# Coverage across every framework the corpus cites, Markdown to stdout +control-coverage aws.json github.json + +# Just the blind spots — the controls nothing evidences yet +control-coverage aws.json github.json --framework SOC2 --blind-spots + +# A whole directory of reports, all formats into ./out/ +control-coverage ./reports/ --format md,html,json,soa --out out/ + +# Gate CI: exit non-zero if any framework's coverage is under 60% +control-coverage ./reports/ --fail-under 60 +``` + +### Trend — how coverage moved + +Point `--baseline` at an earlier corpus (a file or a directory) to see what changed: +controls that improved, regressed, and — the two that move the coverage number — +were *gained* (a blind spot became addressed) or *lost* (an addressed control became +a blind spot). + +```bash +control-coverage ./reports/2026-02/ --baseline ./reports/2026-01/ --framework SOC2 + +# Gate CI: fail the build if any control regressed or lost coverage +control-coverage ./reports/2026-02/ --baseline ./reports/2026-01/ --fail-on-regression +``` + +Trend mode outputs Markdown, HTML, or JSON (`--format md,html,json`). + +### Crosswalk — evidence leverage and the minimal set + +One check is rarely worth one control: enforced 2FA is evidence for SOC 2 CC6.1, +ISO A.5.17, and NIST IA-2 at once. `--crosswalk` shows that leverage per check and +computes the **minimal evidence set** — the fewest checks that still touch every +addressed control, which is what you want when scoping a walkthrough or a sample. + +```bash +control-coverage ./reports/ --framework SOC2,ISO,NIST --crosswalk +``` + +Crosswalk mode outputs Markdown, HTML, or JSON (`--format md,html,json`). + +### Scope and the Statement of Applicability + +Not every control applies to every organization. A **scope file** records which +controls are excluded and — required, never optional — *why*: + +```yaml +# soa.yaml +subject: Acme Production +frameworks: [SOC2, ISO] +exclusions: + - control: ISO:A.7.1 + reason: "Fully cloud-hosted; no physical premises are in scope for the ISMS." + - control: ISO:A.5.7 + reason: "No formal threat-intelligence program; risk accepted by the CISO for 2026." +owners: + SOC2:CC6.1: platform-team +``` + +```bash +# Coverage over in-scope controls, plus a ready-to-file SoA +control-coverage ./reports/ --scope soa.yaml --format md,soa --out out/ +``` + +Excluded controls are recorded with their justification rather than counted as gaps. +An exclusion with no reason is rejected — an unjustified exclusion is the single most +common SoA audit finding. + +## Assurance states + +Every in-scope control lands in exactly one state: + +| State | Meaning | +| --- | --- | +| **supported** | At least one mapped finding passes, and none fail. | +| **failing** | At least one mapped finding fails. The worst observation wins. | +| **asserted** | Findings map here, but their data was absent — evidence attempted, not obtained. | +| **unaddressed** | No finding maps here at all. **The blind spot.** | +| **out of scope** | Excluded by the scope file, with a recorded justification. | + +`coverage %` is the share of in-scope controls in any of the first three states; +`assured %` is the share that are `supported`. + +## Bundled catalogs + +| Framework | Code | Catalog | +| --- | --- | --- | +| SOC 2 (Trust Services Criteria) | `SOC2` | Complete — all five categories (Common Criteria + Availability, Confidentiality, Processing Integrity, Privacy), 61 controls | +| ISO/IEC 27001:2022 Annex A | `ISO` | Complete — all 93 controls | +| NIST SP 800-53 Rev. 5 | `NIST` | Moderate baseline — 177 base controls across 18 families | + +Most SOC 2 reports scope only some categories (Security is near-universal; Privacy and +Processing Integrity often are not). Use `exclude_families` in the scope file to drop a +whole category — or an ISO theme, or a NIST family — from the denominator in one line: + +```yaml +exclude_families: + - {framework: SOC2, family: Privacy, reason: "Privacy category not in the SOC 2 audit scope."} + - {framework: SOC2, family: Processing Integrity, reason: "PI category not in the SOC 2 audit scope."} +``` + +Control codes are written `FRAMEWORK:ID` (`SOC2:CC6.1`, `ISO:A.5.17`), matching the +codes `audit-report` rulesets already cite. A partial catalog is reported honestly as +coverage of the shipped subset, never as the whole standard. + +If the corpus cites a code whose framework is loaded but the catalog does not define +it — a typo or a renamed control — it is surfaced as an **unmatched control code** +rather than silently ignored. + +## How it fits the pipeline + +``` +audit-tools ──► CSV package ──► evidence-seal (seal + verify) + │ + ▼ + audit-report ──► per-package report (--format json) + │ + ▼ one or more reports = a corpus + control-coverage ──► coverage %, blind spots, SoA, + trend over time, evidence crosswalk +``` + +## Development + +```bash +pip install -e ".[dev]" +pytest +ruff check . +``` + +## License + +GPL-3.0-or-later. See [LICENSE](LICENSE). new file mode 100644 @@ -0,0 +1,11 @@ +"""Ensure the project root is importable and tests run from it. + +Tests reference the bundled catalogs by the relative path +``control_coverage/catalogs``, so pytest must be invoked from the project root. +This file's location pins that root for import resolution. +""" + +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).parent)) new file mode 100644 @@ -0,0 +1,3 @@ +"""control-coverage — control-first coverage and blind-spot analysis over an evidence corpus.""" + +__version__ = "0.1.0" new file mode 100644 @@ -0,0 +1,6 @@ +"""Enable ``python -m control_coverage``.""" + +from .cli import main + +if __name__ == "__main__": + raise SystemExit(main()) new file mode 100644 @@ -0,0 +1,116 @@ +"""Framework catalogs — the complete list of controls a framework defines. + +This is the piece the rest of the audit-labs ecosystem does not have. Tools like +audit-report are *evidence-first*: they start from what you collected and map each +finding onto whatever controls it touches. That can never tell you what you are +**not** looking at, because it has no list of everything a framework requires. + +A catalog is that list — the denominator. Loading ``soc2`` gives every Trust +Services Criterion; loading ``iso`` gives all 93 Annex A controls. Coverage is +then simply: of these, how many does the evidence corpus actually address? + +Control codes are written ``FRAMEWORK:ID`` (for example ``SOC2:CC6.1``, +``ISO:A.5.17``), matching the codes audit-report rulesets cite. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from pathlib import Path + +import yaml + +_CATALOG_DIR = Path(__file__).resolve().parent / "catalogs" + +# User-facing framework name -> catalog file stem. Aliases keep the CLI forgiving. +_ALIASES = { + "soc2": "soc2", + "soc 2": "soc2", + "iso": "iso27001", + "iso27001": "iso27001", + "iso 27001": "iso27001", + "nist": "nist80053", + "nist80053": "nist80053", + "800-53": "nist80053", +} + + +@dataclass(frozen=True) +class Control: + """One control in a framework catalog.""" + + framework: str + id: str + title: str + family: str = "" + + @property + def code(self) -> str: + """The full ``FRAMEWORK:ID`` code used to join against evidence.""" + return f"{self.framework}:{self.id}" + + +@dataclass +class Catalog: + """A framework's complete (or explicitly partial) set of controls.""" + + framework: str + name: str + version: str + coverage: str # "complete" or "partial" + source: str + controls: list[Control] + + @property + def complete(self) -> bool: + return self.coverage == "complete" + + def codes(self) -> set[str]: + return {c.code for c in self.controls} + + +def available() -> list[str]: + """Framework short codes with a bundled catalog (e.g. ``["ISO", "NIST", "SOC2"]``).""" + return sorted(load(p.stem).framework for p in _CATALOG_DIR.glob("*.yaml")) + + +def _resolve(name: str) -> Path: + stem = _ALIASES.get(name.strip().lower(), name.strip().lower()) + path = _CATALOG_DIR / f"{stem}.yaml" + if not path.exists(): + known = ", ".join(sorted(p.stem for p in _CATALOG_DIR.glob("*.yaml"))) + raise ValueError(f"unknown framework '{name}'. Bundled catalogs: {known}") + return path + + +def load(name: str) -> Catalog: + """Load a bundled catalog by framework name, short code, or alias.""" + path = _resolve(name) + raw = yaml.safe_load(path.read_text(encoding="utf-8")) + framework = raw["framework"] + controls = [ + Control( + framework=framework, + id=str(c["id"]), + title=str(c["title"]), + family=str(c.get("family", "")), + ) + for c in raw.get("controls", []) + ] + return Catalog( + framework=framework, + name=raw.get("name", framework), + version=str(raw.get("version", "")), + coverage=raw.get("coverage", "partial"), + source=raw.get("source", ""), + controls=controls, + ) + + +def load_frameworks(names: list[str]) -> list[Catalog]: + """Load several catalogs, de-duplicated by framework, in a stable order.""" + seen: dict[str, Catalog] = {} + for name in names: + cat = load(name) + seen[cat.framework] = cat + return [seen[k] for k in sorted(seen)] new file mode 100644 @@ -0,0 +1,107 @@ +# ISO/IEC 27001:2022 — Annex A (all 93 controls, four themes). +# +# This is exactly the list a Statement of Applicability enumerates. A control's +# full code is "ISO:<id>", matching the codes audit-report rulesets cite. +framework: ISO +name: ISO/IEC 27001:2022 Annex A +version: "2022" +coverage: complete +source: ISO/IEC 27001:2022 Annex A +controls: + # A.5 — Organizational controls + - {id: A.5.1, family: Organizational, title: "Policies for information security."} + - {id: A.5.2, family: Organizational, title: "Information security roles and responsibilities."} + - {id: A.5.3, family: Organizational, title: "Segregation of duties."} + - {id: A.5.4, family: Organizational, title: "Management responsibilities."} + - {id: A.5.5, family: Organizational, title: "Contact with authorities."} + - {id: A.5.6, family: Organizational, title: "Contact with special interest groups."} + - {id: A.5.7, family: Organizational, title: "Threat intelligence."} + - {id: A.5.8, family: Organizational, title: "Information security in project management."} + - {id: A.5.9, family: Organizational, title: "Inventory of information and other associated assets."} + - {id: A.5.10, family: Organizational, title: "Acceptable use of information and other associated assets."} + - {id: A.5.11, family: Organizational, title: "Return of assets."} + - {id: A.5.12, family: Organizational, title: "Classification of information."} + - {id: A.5.13, family: Organizational, title: "Labelling of information."} + - {id: A.5.14, family: Organizational, title: "Information transfer."} + - {id: A.5.15, family: Organizational, title: "Access control."} + - {id: A.5.16, family: Organizational, title: "Identity management."} + - {id: A.5.17, family: Organizational, title: "Authentication information."} + - {id: A.5.18, family: Organizational, title: "Access rights."} + - {id: A.5.19, family: Organizational, title: "Information security in supplier relationships."} + - {id: A.5.20, family: Organizational, title: "Addressing information security within supplier agreements."} + - {id: A.5.21, family: Organizational, title: "Managing information security in the ICT supply chain."} + - {id: A.5.22, family: Organizational, title: "Monitoring, review and change management of supplier services."} + - {id: A.5.23, family: Organizational, title: "Information security for use of cloud services."} + - {id: A.5.24, family: Organizational, title: "Information security incident management planning and preparation."} + - {id: A.5.25, family: Organizational, title: "Assessment and decision on information security events."} + - {id: A.5.26, family: Organizational, title: "Response to information security incidents."} + - {id: A.5.27, family: Organizational, title: "Learning from information security incidents."} + - {id: A.5.28, family: Organizational, title: "Collection of evidence."} + - {id: A.5.29, family: Organizational, title: "Information security during disruption."} + - {id: A.5.30, family: Organizational, title: "ICT readiness for business continuity."} + - {id: A.5.31, family: Organizational, title: "Legal, statutory, regulatory and contractual requirements."} + - {id: A.5.32, family: Organizational, title: "Intellectual property rights."} + - {id: A.5.33, family: Organizational, title: "Protection of records."} + - {id: A.5.34, family: Organizational, title: "Privacy and protection of personally identifiable information (PII)."} + - {id: A.5.35, family: Organizational, title: "Independent review of information security."} + - {id: A.5.36, family: Organizational, title: "Compliance with policies, rules and standards for information security."} + - {id: A.5.37, family: Organizational, title: "Documented operating procedures."} + # A.6 — People controls + - {id: A.6.1, family: People, title: "Screening."} + - {id: A.6.2, family: People, title: "Terms and conditions of employment."} + - {id: A.6.3, family: People, title: "Information security awareness, education and training."} + - {id: A.6.4, family: People, title: "Disciplinary process."} + - {id: A.6.5, family: People, title: "Responsibilities after termination or change of employment."} + - {id: A.6.6, family: People, title: "Confidentiality or non-disclosure agreements."} + - {id: A.6.7, family: People, title: "Remote working."} + - {id: A.6.8, family: People, title: "Information security event reporting."} + # A.7 — Physical controls + - {id: A.7.1, family: Physical, title: "Physical security perimeters."} + - {id: A.7.2, family: Physical, title: "Physical entry."} + - {id: A.7.3, family: Physical, title: "Securing offices, rooms and facilities."} + - {id: A.7.4, family: Physical, title: "Physical security monitoring."} + - {id: A.7.5, family: Physical, title: "Protecting against physical and environmental threats."} + - {id: A.7.6, family: Physical, title: "Working in secure areas."} + - {id: A.7.7, family: Physical, title: "Clear desk and clear screen."} + - {id: A.7.8, family: Physical, title: "Equipment siting and protection."} + - {id: A.7.9, family: Physical, title: "Security of assets off-premises."} + - {id: A.7.10, family: Physical, title: "Storage media."} + - {id: A.7.11, family: Physical, title: "Supporting utilities."} + - {id: A.7.12, family: Physical, title: "Cabling security."} + - {id: A.7.13, family: Physical, title: "Equipment maintenance."} + - {id: A.7.14, family: Physical, title: "Secure disposal or re-use of equipment."} + # A.8 — Technological controls + - {id: A.8.1, family: Technological, title: "User endpoint devices."} + - {id: A.8.2, family: Technological, title: "Privileged access rights."} + - {id: A.8.3, family: Technological, title: "Information access restriction."} + - {id: A.8.4, family: Technological, title: "Access to source code."} + - {id: A.8.5, family: Technological, title: "Secure authentication."} + - {id: A.8.6, family: Technological, title: "Capacity management."} + - {id: A.8.7, family: Technological, title: "Protection against malware."} + - {id: A.8.8, family: Technological, title: "Management of technical vulnerabilities."} + - {id: A.8.9, family: Technological, title: "Configuration management."} + - {id: A.8.10, family: Technological, title: "Information deletion."} + - {id: A.8.11, family: Technological, title: "Data masking."} + - {id: A.8.12, family: Technological, title: "Data leakage prevention."} + - {id: A.8.13, family: Technological, title: "Information backup."} + - {id: A.8.14, family: Technological, title: "Redundancy of information processing facilities."} + - {id: A.8.15, family: Technological, title: "Logging."} + - {id: A.8.16, family: Technological, title: "Monitoring activities."} + - {id: A.8.17, family: Technological, title: "Clock synchronization."} + - {id: A.8.18, family: Technological, title: "Use of privileged utility programs."} + - {id: A.8.19, family: Technological, title: "Installation of software on operational systems."} + - {id: A.8.20, family: Technological, title: "Networks security."} + - {id: A.8.21, family: Technological, title: "Security of network services."} + - {id: A.8.22, family: Technological, title: "Segregation of networks."} + - {id: A.8.23, family: Technological, title: "Web filtering."} + - {id: A.8.24, family: Technological, title: "Use of cryptography."} + - {id: A.8.25, family: Technological, title: "Secure development life cycle."} + - {id: A.8.26, family: Technological, title: "Application security requirements."} + - {id: A.8.27, family: Technological, title: "Secure system architecture and engineering principles."} + - {id: A.8.28, family: Technological, title: "Secure coding."} + - {id: A.8.29, family: Technological, title: "Security testing in development and acceptance."} + - {id: A.8.30, family: Technological, title: "Outsourced development."} + - {id: A.8.31, family: Technological, title: "Separation of development, test and production environments."} + - {id: A.8.32, family: Technological, title: "Change management."} + - {id: A.8.33, family: Technological, title: "Test information."} + - {id: A.8.34, family: Technological, title: "Protection of information systems during audit testing."} new file mode 100644 @@ -0,0 +1,210 @@ +# NIST SP 800-53 Rev. 5 — Moderate baseline (base controls). +# +# The base controls selected in the SP 800-53B moderate impact baseline, across +# all 18 baseline-applicable families. Control enhancements (e.g. AC-2(1)) are not +# enumerated — coverage is measured at the base-control level. The program +# management (PM) family is org-wide and not baseline-allocated; the privacy (PT) +# family is selected via the separate privacy baseline. A control's full code is +# "NIST:<id>". +framework: NIST +name: NIST SP 800-53 Rev. 5 (Moderate baseline) +version: "Rev. 5" +baseline: Moderate +coverage: complete +source: NIST SP 800-53B Moderate baseline (base controls) +controls: + # AC — Access Control + - {id: AC-1, family: "Access Control", title: "Policy and Procedures"} + - {id: AC-2, family: "Access Control", title: "Account Management"} + - {id: AC-3, family: "Access Control", title: "Access Enforcement"} + - {id: AC-4, family: "Access Control", title: "Information Flow Enforcement"} + - {id: AC-5, family: "Access Control", title: "Separation of Duties"} + - {id: AC-6, family: "Access Control", title: "Least Privilege"} + - {id: AC-7, family: "Access Control", title: "Unsuccessful Logon Attempts"} + - {id: AC-8, family: "Access Control", title: "System Use Notification"} + - {id: AC-11, family: "Access Control", title: "Device Lock"} + - {id: AC-12, family: "Access Control", title: "Session Termination"} + - {id: AC-14, family: "Access Control", title: "Permitted Actions Without Identification or Authentication"} + - {id: AC-17, family: "Access Control", title: "Remote Access"} + - {id: AC-18, family: "Access Control", title: "Wireless Access"} + - {id: AC-19, family: "Access Control", title: "Access Control for Mobile Devices"} + - {id: AC-20, family: "Access Control", title: "Use of External Systems"} + - {id: AC-21, family: "Access Control", title: "Information Sharing"} + - {id: AC-22, family: "Access Control", title: "Publicly Accessible Content"} + # AT — Awareness and Training + - {id: AT-1, family: "Awareness and Training", title: "Policy and Procedures"} + - {id: AT-2, family: "Awareness and Training", title: "Literacy Training and Awareness"} + - {id: AT-3, family: "Awareness and Training", title: "Role-Based Training"} + - {id: AT-4, family: "Awareness and Training", title: "Training Records"} + # AU — Audit and Accountability + - {id: AU-1, family: "Audit and Accountability", title: "Policy and Procedures"} + - {id: AU-2, family: "Audit and Accountability", title: "Event Logging"} + - {id: AU-3, family: "Audit and Accountability", title: "Content of Audit Records"} + - {id: AU-4, family: "Audit and Accountability", title: "Audit Log Storage Capacity"} + - {id: AU-5, family: "Audit and Accountability", title: "Response to Audit Logging Process Failures"} + - {id: AU-6, family: "Audit and Accountability", title: "Audit Record Review, Analysis, and Reporting"} + - {id: AU-7, family: "Audit and Accountability", title: "Audit Record Reduction and Report Generation"} + - {id: AU-8, family: "Audit and Accountability", title: "Time Stamps"} + - {id: AU-9, family: "Audit and Accountability", title: "Protection of Audit Information"} + - {id: AU-11, family: "Audit and Accountability", title: "Audit Record Retention"} + - {id: AU-12, family: "Audit and Accountability", title: "Audit Record Generation"} + # CA — Assessment, Authorization, and Monitoring + - {id: CA-1, family: "Assessment, Authorization, and Monitoring", title: "Policy and Procedures"} + - {id: CA-2, family: "Assessment, Authorization, and Monitoring", title: "Control Assessments"} + - {id: CA-3, family: "Assessment, Authorization, and Monitoring", title: "Information Exchange"} + - {id: CA-5, family: "Assessment, Authorization, and Monitoring", title: "Plan of Action and Milestones"} + - {id: CA-6, family: "Assessment, Authorization, and Monitoring", title: "Authorization"} + - {id: CA-7, family: "Assessment, Authorization, and Monitoring", title: "Continuous Monitoring"} + - {id: CA-9, family: "Assessment, Authorization, and Monitoring", title: "Internal System Connections"} + # CM — Configuration Management + - {id: CM-1, family: "Configuration Management", title: "Policy and Procedures"} + - {id: CM-2, family: "Configuration Management", title: "Baseline Configuration"} + - {id: CM-3, family: "Configuration Management", title: "Configuration Change Control"} + - {id: CM-4, family: "Configuration Management", title: "Impact Analyses"} + - {id: CM-5, family: "Configuration Management", title: "Access Restrictions for Change"} + - {id: CM-6, family: "Configuration Management", title: "Configuration Settings"} + - {id: CM-7, family: "Configuration Management", title: "Least Functionality"} + - {id: CM-8, family: "Configuration Management", title: "System Component Inventory"} + - {id: CM-9, family: "Configuration Management", title: "Configuration Management Plan"} + - {id: CM-10, family: "Configuration Management", title: "Software Usage Restrictions"} + - {id: CM-11, family: "Configuration Management", title: "User-Installed Software"} + - {id: CM-12, family: "Configuration Management", title: "Information Location"} + # CP — Contingency Planning + - {id: CP-1, family: "Contingency Planning", title: "Policy and Procedures"} + - {id: CP-2, family: "Contingency Planning", title: "Contingency Plan"} + - {id: CP-3, family: "Contingency Planning", title: "Contingency Training"} + - {id: CP-4, family: "Contingency Planning", title: "Contingency Plan Testing"} + - {id: CP-6, family: "Contingency Planning", title: "Alternate Storage Site"} + - {id: CP-7, family: "Contingency Planning", title: "Alternate Processing Site"} + - {id: CP-8, family: "Contingency Planning", title: "Telecommunications Services"} + - {id: CP-9, family: "Contingency Planning", title: "System Backup"} + - {id: CP-10, family: "Contingency Planning", title: "System Recovery and Reconstitution"} + # IA — Identification and Authentication + - {id: IA-1, family: "Identification and Authentication", title: "Policy and Procedures"} + - {id: IA-2, family: "Identification and Authentication", title: "Identification and Authentication (Organizational Users)"} + - {id: IA-3, family: "Identification and Authentication", title: "Device Identification and Authentication"} + - {id: IA-4, family: "Identification and Authentication", title: "Identifier Management"} + - {id: IA-5, family: "Identification and Authentication", title: "Authenticator Management"} + - {id: IA-6, family: "Identification and Authentication", title: "Authentication Feedback"} + - {id: IA-7, family: "Identification and Authentication", title: "Cryptographic Module Authentication"} + - {id: IA-8, family: "Identification and Authentication", title: "Identification and Authentication (Non-Organizational Users)"} + - {id: IA-11, family: "Identification and Authentication", title: "Re-authentication"} + - {id: IA-12, family: "Identification and Authentication", title: "Identity Proofing"} + # IR — Incident Response + - {id: IR-1, family: "Incident Response", title: "Policy and Procedures"} + - {id: IR-2, family: "Incident Response", title: "Incident Response Training"} + - {id: IR-3, family: "Incident Response", title: "Incident Response Testing"} + - {id: IR-4, family: "Incident Response", title: "Incident Handling"} + - {id: IR-5, family: "Incident Response", title: "Incident Monitoring"} + - {id: IR-6, family: "Incident Response", title: "Incident Reporting"} + - {id: IR-7, family: "Incident Response", title: "Incident Response Assistance"} + - {id: IR-8, family: "Incident Response", title: "Incident Response Plan"} + # MA — Maintenance + - {id: MA-1, family: "Maintenance", title: "Policy and Procedures"} + - {id: MA-2, family: "Maintenance", title: "Controlled Maintenance"} + - {id: MA-3, family: "Maintenance", title: "Maintenance Tools"} + - {id: MA-4, family: "Maintenance", title: "Nonlocal Maintenance"} + - {id: MA-5, family: "Maintenance", title: "Maintenance Personnel"} + - {id: MA-6, family: "Maintenance", title: "Timely Maintenance"} + # MP — Media Protection + - {id: MP-1, family: "Media Protection", title: "Policy and Procedures"} + - {id: MP-2, family: "Media Protection", title: "Media Access"} + - {id: MP-3, family: "Media Protection", title: "Media Marking"} + - {id: MP-4, family: "Media Protection", title: "Media Storage"} + - {id: MP-5, family: "Media Protection", title: "Media Transport"} + - {id: MP-6, family: "Media Protection", title: "Media Sanitization"} + - {id: MP-7, family: "Media Protection", title: "Media Use"} + # PE — Physical and Environmental Protection + - {id: PE-1, family: "Physical and Environmental Protection", title: "Policy and Procedures"} + - {id: PE-2, family: "Physical and Environmental Protection", title: "Physical Access Authorizations"} + - {id: PE-3, family: "Physical and Environmental Protection", title: "Physical Access Control"} + - {id: PE-4, family: "Physical and Environmental Protection", title: "Access Control for Transmission"} + - {id: PE-5, family: "Physical and Environmental Protection", title: "Access Control for Output Devices"} + - {id: PE-6, family: "Physical and Environmental Protection", title: "Monitoring Physical Access"} + - {id: PE-8, family: "Physical and Environmental Protection", title: "Visitor Access Records"} + - {id: PE-9, family: "Physical and Environmental Protection", title: "Power Equipment and Cabling"} + - {id: PE-10, family: "Physical and Environmental Protection", title: "Emergency Shutoff"} + - {id: PE-11, family: "Physical and Environmental Protection", title: "Emergency Power"} + - {id: PE-12, family: "Physical and Environmental Protection", title: "Emergency Lighting"} + - {id: PE-13, family: "Physical and Environmental Protection", title: "Fire Protection"} + - {id: PE-14, family: "Physical and Environmental Protection", title: "Environmental Controls"} + - {id: PE-15, family: "Physical and Environmental Protection", title: "Water Damage Protection"} + - {id: PE-16, family: "Physical and Environmental Protection", title: "Delivery and Removal"} + - {id: PE-17, family: "Physical and Environmental Protection", title: "Alternate Work Site"} + # PL — Planning + - {id: PL-1, family: "Planning", title: "Policy and Procedures"} + - {id: PL-2, family: "Planning", title: "System Security and Privacy Plans"} + - {id: PL-4, family: "Planning", title: "Rules of Behavior"} + - {id: PL-8, family: "Planning", title: "Security and Privacy Architectures"} + - {id: PL-10, family: "Planning", title: "Baseline Selection"} + - {id: PL-11, family: "Planning", title: "Baseline Tailoring"} + # PS — Personnel Security + - {id: PS-1, family: "Personnel Security", title: "Policy and Procedures"} + - {id: PS-2, family: "Personnel Security", title: "Position Risk Designation"} + - {id: PS-3, family: "Personnel Security", title: "Personnel Screening"} + - {id: PS-4, family: "Personnel Security", title: "Personnel Termination"} + - {id: PS-5, family: "Personnel Security", title: "Personnel Transfer"} + - {id: PS-6, family: "Personnel Security", title: "Access Agreements"} + - {id: PS-7, family: "Personnel Security", title: "External Personnel Security"} + - {id: PS-8, family: "Personnel Security", title: "Personnel Sanctions"} + - {id: PS-9, family: "Personnel Security", title: "Position Descriptions"} + # RA — Risk Assessment + - {id: RA-1, family: "Risk Assessment", title: "Policy and Procedures"} + - {id: RA-2, family: "Risk Assessment", title: "Security Categorization"} + - {id: RA-3, family: "Risk Assessment", title: "Risk Assessment"} + - {id: RA-5, family: "Risk Assessment", title: "Vulnerability Monitoring and Scanning"} + - {id: RA-7, family: "Risk Assessment", title: "Risk Response"} + # SA — System and Services Acquisition + - {id: SA-1, family: "System and Services Acquisition", title: "Policy and Procedures"} + - {id: SA-2, family: "System and Services Acquisition", title: "Allocation of Resources"} + - {id: SA-3, family: "System and Services Acquisition", title: "System Development Life Cycle"} + - {id: SA-4, family: "System and Services Acquisition", title: "Acquisition Process"} + - {id: SA-5, family: "System and Services Acquisition", title: "System Documentation"} + - {id: SA-8, family: "System and Services Acquisition", title: "Security and Privacy Engineering Principles"} + - {id: SA-9, family: "System and Services Acquisition", title: "External System Services"} + - {id: SA-10, family: "System and Services Acquisition", title: "Developer Configuration Management"} + - {id: SA-11, family: "System and Services Acquisition", title: "Developer Testing and Evaluation"} + - {id: SA-15, family: "System and Services Acquisition", title: "Development Process, Standards, and Tools"} + - {id: SA-22, family: "System and Services Acquisition", title: "Unsupported System Components"} + # SC — System and Communications Protection + - {id: SC-1, family: "System and Communications Protection", title: "Policy and Procedures"} + - {id: SC-2, family: "System and Communications Protection", title: "Separation of System and User Functionality"} + - {id: SC-4, family: "System and Communications Protection", title: "Information in Shared System Resources"} + - {id: SC-5, family: "System and Communications Protection", title: "Denial-of-Service Protection"} + - {id: SC-7, family: "System and Communications Protection", title: "Boundary Protection"} + - {id: SC-8, family: "System and Communications Protection", title: "Transmission Confidentiality and Integrity"} + - {id: SC-10, family: "System and Communications Protection", title: "Network Disconnect"} + - {id: SC-12, family: "System and Communications Protection", title: "Cryptographic Key Establishment and Management"} + - {id: SC-13, family: "System and Communications Protection", title: "Cryptographic Protection"} + - {id: SC-15, family: "System and Communications Protection", title: "Collaborative Computing Devices and Applications"} + - {id: SC-17, family: "System and Communications Protection", title: "Public Key Infrastructure Certificates"} + - {id: SC-18, family: "System and Communications Protection", title: "Mobile Code"} + - {id: SC-20, family: "System and Communications Protection", title: "Secure Name/Address Resolution Service (Authoritative Source)"} + - {id: SC-21, family: "System and Communications Protection", title: "Secure Name/Address Resolution Service (Recursive or Caching Resolver)"} + - {id: SC-22, family: "System and Communications Protection", title: "Architecture and Provisioning for Name/Address Resolution Service"} + - {id: SC-23, family: "System and Communications Protection", title: "Session Authenticity"} + - {id: SC-28, family: "System and Communications Protection", title: "Protection of Information at Rest"} + - {id: SC-39, family: "System and Communications Protection", title: "Process Isolation"} + # SI — System and Information Integrity + - {id: SI-1, family: "System and Information Integrity", title: "Policy and Procedures"} + - {id: SI-2, family: "System and Information Integrity", title: "Flaw Remediation"} + - {id: SI-3, family: "System and Information Integrity", title: "Malicious Code Protection"} + - {id: SI-4, family: "System and Information Integrity", title: "System Monitoring"} + - {id: SI-5, family: "System and Information Integrity", title: "Security Alerts, Advisories, and Directives"} + - {id: SI-7, family: "System and Information Integrity", title: "Software, Firmware, and Information Integrity"} + - {id: SI-8, family: "System and Information Integrity", title: "Spam Protection"} + - {id: SI-10, family: "System and Information Integrity", title: "Information Input Validation"} + - {id: SI-11, family: "System and Information Integrity", title: "Error Handling"} + - {id: SI-12, family: "System and Information Integrity", title: "Information Management and Retention"} + - {id: SI-16, family: "System and Information Integrity", title: "Memory Protection"} + # SR — Supply Chain Risk Management + - {id: SR-1, family: "Supply Chain Risk Management", title: "Policy and Procedures"} + - {id: SR-2, family: "Supply Chain Risk Management", title: "Supply Chain Risk Management Plan"} + - {id: SR-3, family: "Supply Chain Risk Management", title: "Supply Chain Controls and Processes"} + - {id: SR-5, family: "Supply Chain Risk Management", title: "Acquisition Strategies, Tools, and Methods"} + - {id: SR-6, family: "Supply Chain Risk Management", title: "Supplier Assessments and Reviews"} + - {id: SR-8, family: "Supply Chain Risk Management", title: "Notification Agreements"} + - {id: SR-9, family: "Supply Chain Risk Management", title: "Tamper Resistance and Detection"} + - {id: SR-10, family: "Supply Chain Risk Management", title: "Inspection of Systems or Components"} + - {id: SR-11, family: "Supply Chain Risk Management", title: "Component Authenticity"} + - {id: SR-12, family: "Supply Chain Risk Management", title: "Component Disposal"} new file mode 100644 @@ -0,0 +1,85 @@ +# SOC 2 — Trust Services Criteria (AICPA, 2017 with 2022 revised points of focus). +# +# The full Common Criteria (the "Security" category every SOC 2 report covers) +# plus the Availability category. A control's full code is "SOC2:<id>", matching +# the codes audit-report rulesets cite. +framework: SOC2 +name: SOC 2 (Trust Services Criteria) +version: "2017 (rev. 2022)" +coverage: complete +source: AICPA Trust Services Criteria +controls: + # CC1 — Control Environment + - {id: CC1.1, family: Control Environment, title: "The entity demonstrates a commitment to integrity and ethical values."} + - {id: CC1.2, family: Control Environment, title: "The board of directors demonstrates independence and exercises oversight of internal control."} + - {id: CC1.3, family: Control Environment, title: "Management establishes structures, reporting lines, and appropriate authorities and responsibilities."} + - {id: CC1.4, family: Control Environment, title: "The entity demonstrates a commitment to attract, develop, and retain competent individuals."} + - {id: CC1.5, family: Control Environment, title: "The entity holds individuals accountable for their internal control responsibilities."} + # CC2 — Communication and Information + - {id: CC2.1, family: Communication and Information, title: "The entity obtains or generates relevant, quality information to support internal control."} + - {id: CC2.2, family: Communication and Information, title: "The entity internally communicates information, including objectives and responsibilities for internal control."} + - {id: CC2.3, family: Communication and Information, title: "The entity communicates with external parties about matters affecting internal control."} + # CC3 — Risk Assessment + - {id: CC3.1, family: Risk Assessment, title: "The entity specifies objectives with sufficient clarity to enable identification of risks."} + - {id: CC3.2, family: Risk Assessment, title: "The entity identifies and analyzes risks to the achievement of its objectives."} + - {id: CC3.3, family: Risk Assessment, title: "The entity considers the potential for fraud in assessing risks."} + - {id: CC3.4, family: Risk Assessment, title: "The entity identifies and assesses changes that could significantly affect internal control."} + # CC4 — Monitoring Activities + - {id: CC4.1, family: Monitoring Activities, title: "The entity selects, develops, and performs ongoing and separate evaluations of internal control."} + - {id: CC4.2, family: Monitoring Activities, title: "The entity evaluates and communicates internal control deficiencies in a timely manner."} + # CC5 — Control Activities + - {id: CC5.1, family: Control Activities, title: "The entity selects and develops control activities that mitigate risks to acceptable levels."} + - {id: CC5.2, family: Control Activities, title: "The entity selects and develops general control activities over technology."} + - {id: CC5.3, family: Control Activities, title: "The entity deploys control activities through policies and procedures."} + # CC6 — Logical and Physical Access Controls + - {id: CC6.1, family: Logical and Physical Access Controls, title: "The entity implements logical access security software, infrastructure, and architectures over protected assets."} + - {id: CC6.2, family: Logical and Physical Access Controls, title: "The entity registers and authorizes new users before granting access, and removes access when appropriate."} + - {id: CC6.3, family: Logical and Physical Access Controls, title: "The entity authorizes, modifies, or removes access based on roles and least privilege."} + - {id: CC6.4, family: Logical and Physical Access Controls, title: "The entity restricts physical access to facilities and protected information assets."} + - {id: CC6.5, family: Logical and Physical Access Controls, title: "The entity discontinues logical and physical protections over assets only after the ability to read data has been removed."} + - {id: CC6.6, family: Logical and Physical Access Controls, title: "The entity implements logical access security measures against threats from outside its system boundaries."} + - {id: CC6.7, family: Logical and Physical Access Controls, title: "The entity restricts the transmission, movement, and removal of information to authorized users and processes."} + - {id: CC6.8, family: Logical and Physical Access Controls, title: "The entity implements controls to prevent or detect and act upon unauthorized or malicious software."} + # CC7 — System Operations + - {id: CC7.1, family: System Operations, title: "The entity uses detection and monitoring procedures to identify configuration changes and new vulnerabilities."} + - {id: CC7.2, family: System Operations, title: "The entity monitors system components for anomalies indicative of malicious acts or errors."} + - {id: CC7.3, family: System Operations, title: "The entity evaluates security events to determine whether they could or did result in a failure to meet objectives."} + - {id: CC7.4, family: System Operations, title: "The entity responds to identified security incidents through a defined program."} + - {id: CC7.5, family: System Operations, title: "The entity identifies, develops, and implements activities to recover from security incidents."} + # CC8 — Change Management + - {id: CC8.1, family: Change Management, title: "The entity authorizes, designs, develops, tests, approves, and implements changes to infrastructure, data, and software."} + # CC9 — Risk Mitigation + - {id: CC9.1, family: Risk Mitigation, title: "The entity identifies, selects, and develops risk mitigation activities for disruptions."} + - {id: CC9.2, family: Risk Mitigation, title: "The entity assesses and manages risks associated with vendors and business partners."} + # Availability category + - {id: A1.1, family: Availability, title: "The entity maintains, monitors, and evaluates current processing capacity to meet demand."} + - {id: A1.2, family: Availability, title: "The entity authorizes, designs, and implements environmental protections, backup, and recovery infrastructure."} + - {id: A1.3, family: Availability, title: "The entity tests recovery plan procedures supporting system recovery."} + # Confidentiality category + - {id: C1.1, family: Confidentiality, title: "The entity identifies and maintains confidential information to meet its objectives related to confidentiality."} + - {id: C1.2, family: Confidentiality, title: "The entity disposes of confidential information to meet its objectives related to confidentiality."} + # Processing Integrity category + - {id: PI1.1, family: Processing Integrity, title: "The entity obtains or generates, uses, and communicates relevant, quality information about processing objectives, including product and service specifications."} + - {id: PI1.2, family: Processing Integrity, title: "The entity implements policies and procedures over system inputs, including controls over completeness and accuracy, to meet its objectives."} + - {id: PI1.3, family: Processing Integrity, title: "The entity implements policies and procedures over system processing to result in products, services, and reporting that meet its objectives."} + - {id: PI1.4, family: Processing Integrity, title: "The entity implements policies and procedures to make available or deliver output completely, accurately, and in a timely manner to meet its objectives."} + - {id: PI1.5, family: Processing Integrity, title: "The entity implements policies and procedures to store inputs, items in processing, and outputs completely, accurately, and in a timely manner to meet its objectives."} + # Privacy category + - {id: P1.1, family: Privacy, title: "The entity provides notice to data subjects about its privacy practices to meet its objectives related to privacy."} + - {id: P2.1, family: Privacy, title: "The entity communicates choices about the collection, use, retention, disclosure, and disposal of personal information, and obtains consent, to meet its privacy objectives."} + - {id: P3.1, family: Privacy, title: "Personal information is collected consistent with the entity's objectives related to privacy."} + - {id: P3.2, family: Privacy, title: "For information requiring explicit consent, the entity communicates the need for and obtains consent prior to collection of personal information."} + - {id: P4.1, family: Privacy, title: "The entity limits the use of personal information to the purposes identified in its objectives related to privacy."} + - {id: P4.2, family: Privacy, title: "The entity retains personal information consistent with its objectives related to privacy."} + - {id: P4.3, family: Privacy, title: "The entity securely disposes of personal information to meet its objectives related to privacy."} + - {id: P5.1, family: Privacy, title: "The entity grants data subjects the ability to access their stored personal information for review and, upon request, provides copies, to meet its privacy objectives."} + - {id: P5.2, family: Privacy, title: "The entity corrects, amends, or appends personal information based on data subject input and communicates it to third parties, to meet its privacy objectives."} + - {id: P6.1, family: Privacy, title: "The entity discloses personal information to third parties only with the explicit consent of data subjects and consistent with its privacy objectives."} + - {id: P6.2, family: Privacy, title: "The entity creates and retains a complete, accurate, and timely record of authorized disclosures of personal information."} + - {id: P6.3, family: Privacy, title: "The entity creates and retains a complete, accurate, and timely record of detected or reported unauthorized disclosures of personal information."} + - {id: P6.4, family: Privacy, title: "The entity obtains privacy commitments from vendors and other third parties who have access to personal information, to meet its privacy objectives."} + - {id: P6.5, family: Privacy, title: "The entity obtains commitments from vendors and third parties to notify it of actual or suspected unauthorized disclosures of personal information."} + - {id: P6.6, family: Privacy, title: "The entity provides notification of breaches and incidents of unauthorized disclosure of personal information to affected data subjects, regulators, and others."} + - {id: P6.7, family: Privacy, title: "The entity provides data subjects with an accounting of the personal information held and disclosures made, upon request."} + - {id: P7.1, family: Privacy, title: "The entity collects and maintains accurate, up-to-date, complete, and relevant personal information to meet its privacy objectives."} + - {id: P8.1, family: Privacy, title: "The entity implements a process for receiving, addressing, resolving, and communicating the resolution of privacy inquiries, complaints, and disputes."} new file mode 100644 @@ -0,0 +1,263 @@ +"""Command-line entry point for control-coverage.""" + +from __future__ import annotations + +import argparse +import sys +from datetime import datetime, timezone +from pathlib import Path + +from . import __version__, catalog, corpus, reporters, scope +from .coverage import evaluate + + +def _parse_args(argv: list[str]) -> argparse.Namespace: + parser = argparse.ArgumentParser( + prog="control-coverage", + description=( + "Score an evidence corpus against complete framework catalogs: what " + "share of each framework does the evidence address, and which controls " + "are blind spots no finding touches?" + ), + ) + parser.add_argument( + "reports", + nargs="+", + help="audit-report JSON files, and/or directories containing them", + ) + parser.add_argument( + "--framework", + help=( + "comma-separated frameworks to evaluate (e.g. SOC2,ISO,NIST). " + "Default: every framework the corpus cites, or the scope file's list." + ), + ) + parser.add_argument( + "--scope", + help="path to a scope / Statement of Applicability YAML (marks exclusions)", + ) + parser.add_argument( + "--subject", + help="name for the subject of this corpus (overrides the scope file)", + ) + parser.add_argument( + "--format", + default="md", + help="comma-separated output formats: md, html, json, soa (default: md)", + ) + parser.add_argument( + "--out", + help="directory to write reports into (default: print the first format to stdout)", + ) + parser.add_argument( + "--blind-spots", + action="store_true", + help="print only the unaddressed in-scope controls, then exit", + ) + parser.add_argument( + "--baseline", + metavar="PATH", + help=( + "trend mode: an earlier corpus (file or directory) to compare against. " + "Reports how coverage moved — what improved, regressed, was gained or lost." + ), + ) + parser.add_argument( + "--crosswalk", + action="store_true", + help=( + "crosswalk mode: show which controls each piece of evidence supports " + "across frameworks, and the minimal evidence set that covers them all" + ), + ) + parser.add_argument( + "--fail-under", + type=float, + metavar="PCT", + help="exit non-zero if any framework's coverage %% is below PCT (CI gate)", + ) + parser.add_argument( + "--fail-on-regression", + action="store_true", + help="in trend mode, exit non-zero if any control regressed or lost coverage", + ) + parser.add_argument("--version", action="version", version=f"control-coverage {__version__}") + return parser.parse_args(argv) + + +def _select_frameworks(args, observations, scp) -> list[str]: + """Decide which framework catalogs to load, in priority order.""" + if args.framework: + return [f.strip() for f in args.framework.split(",") if f.strip()] + if scp.frameworks: + return scp.frameworks + # Infer from the corpus: every framework prefix the observations cite. + cited = sorted({o.control.split(":", 1)[0] for o in observations if ":" in o.control}) + if not cited: + raise SystemExit( + "error: could not infer frameworks from the corpus. Pass --framework." + ) + return cited + + +def _print_blind_spots(report) -> None: + total = 0 + for fc in report.frameworks: + spots = fc.blind_spots + if not spots: + continue + print(f"{fc.catalog.name} — {len(spots)} unaddressed:") + for r in spots: + print(f" {r.control.code} {r.control.title}") + total += len(spots) + print(f"\n{total} in-scope control(s) unaddressed across {len(report.frameworks)} framework(s).") + + +def _now() -> str: + return datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M:%S UTC") + + +def _build_report(paths, args, scp, names, subject): + """Load a corpus from *paths* and evaluate it into a CoverageReport.""" + try: + observations = corpus.load_corpus(paths) + except (ValueError, FileNotFoundError, OSError) as exc: + raise SystemExit(f"error: {exc}") from None + catalogs = catalog.load_frameworks(names) + return evaluate(catalogs, observations, scope=scp, subject=subject, generated_at=_now()) + + +def main(argv: list[str] | None = None) -> int: + args = _parse_args(sys.argv[1:] if argv is None else argv) + + if args.crosswalk and args.baseline: + raise SystemExit("error: --crosswalk and --baseline cannot be combined") + + try: + observations = corpus.load_corpus(args.reports) + except (ValueError, FileNotFoundError, OSError) as exc: + raise SystemExit(f"error: {exc}") from None + + scp = scope.load(args.scope) if args.scope else scope.empty() + + try: + names = _select_frameworks(args, observations, scp) + catalogs = catalog.load_frameworks(names) + except ValueError as exc: + raise SystemExit(f"error: {exc}") from None + + subject = args.subject or scp.subject + report = evaluate(catalogs, observations, scope=scp, subject=subject, generated_at=_now()) + + if args.baseline: + return _trend_mode(args, scp, names, subject, report) + + if args.crosswalk: + return _crosswalk_mode(args, report) + + if args.blind_spots: + _print_blind_spots(report) + return _exit_code(report, args.fail_under) + + formats = [f.strip() for f in args.format.split(",") if f.strip()] + if args.out: + out_dir = Path(args.out) + out_dir.mkdir(parents=True, exist_ok=True) + stem = _slug(subject) or "coverage" + for fmt in formats: + ext = reporters.EXTENSIONS.get(fmt, fmt) + name = "soa" if fmt == "soa" else "coverage" + path = out_dir / f"{name}.{ext}" if fmt == "soa" else out_dir / f"{stem}.{ext}" + path.write_text(reporters.render(report, fmt), encoding="utf-8") + print(f"wrote {path}") + else: + # Print the first requested format to stdout. + print(reporters.render(report, formats[0]), end="") + + return _exit_code(report, args.fail_under) + + +def _trend_mode(args, scp, names, subject, current) -> int: + from . import trend + + baseline = _build_report([args.baseline], args, scp, names, subject) + tr = trend.compare(baseline, current) + + formats = [f.strip() for f in args.format.split(",") if f.strip()] + renderers = { + "md": trend.render_markdown, + "html": trend.render_html, + "json": trend.render_json, + } + unknown = [f for f in formats if f not in renderers] + if unknown: + raise SystemExit(f"error: trend mode supports md, html, and json, not: {', '.join(unknown)}") + + if args.out: + out_dir = Path(args.out) + out_dir.mkdir(parents=True, exist_ok=True) + for fmt in formats: + path = out_dir / f"trend.{fmt}" + path.write_text(renderers[fmt](tr), encoding="utf-8") + print(f"wrote {path}") + else: + print(renderers[formats[0]](tr), end="") + + if args.fail_on_regression and tr.total_regressions: + print( + f"trend gate: {tr.total_regressions} control(s) regressed or lost coverage", + file=sys.stderr, + ) + return 1 + return 0 + + +def _crosswalk_mode(args, report) -> int: + from . import crosswalk + + xw = crosswalk.build(report) + formats = [f.strip() for f in args.format.split(",") if f.strip()] + renderers = { + "md": crosswalk.render_markdown, + "html": crosswalk.render_html, + "json": crosswalk.render_json, + } + unknown = [f for f in formats if f not in renderers] + if unknown: + raise SystemExit( + f"error: crosswalk mode supports md, html, and json, not: {', '.join(unknown)}" + ) + + if args.out: + out_dir = Path(args.out) + out_dir.mkdir(parents=True, exist_ok=True) + for fmt in formats: + path = out_dir / f"crosswalk.{fmt}" + path.write_text(renderers[fmt](xw), encoding="utf-8") + print(f"wrote {path}") + else: + print(renderers[formats[0]](xw), end="") + return 0 + + +def _exit_code(report, fail_under: float | None) -> int: + if fail_under is None: + return 0 + below = [fc for fc in report.frameworks if fc.coverage_pct < fail_under] + if below: + for fc in below: + print( + f"coverage gate: {fc.catalog.framework} at {fc.coverage_pct}% " + f"is below {fail_under}%", + file=sys.stderr, + ) + return 1 + return 0 + + +def _slug(text: str) -> str: + return "".join(c if c.isalnum() else "-" for c in text.lower()).strip("-") + + +if __name__ == "__main__": + raise SystemExit(main()) new file mode 100644 @@ -0,0 +1,88 @@ +"""Load an evidence corpus from audit-report JSON reports. + +audit-report emits one JSON document per evidence package. Each finding carries +the controls it maps to and a status:: + + {"findings": [ + {"id": "github.org.require-2fa", "status": "pass", "severity": "high", + "controls": ["SOC2:CC6.1", "ISO:A.5.17", "NIST:IA-2"], ...}, + ... + ]} + +A **corpus** is any number of these reports — typically one per platform and date +(AWS, GitHub, GitLab…) — flattened into a list of :class:`Observation`, one per +(finding, control) pair. Coverage is then computed by joining observations onto a +framework catalog. +""" + +from __future__ import annotations + +import json +from dataclasses import dataclass +from pathlib import Path + +# Statuses as emitted by audit-report's engine. +PASS = "pass" +FAIL = "fail" +NOT_APPLICABLE = "not_applicable" + + +@dataclass(frozen=True) +class Observation: + """One finding's bearing on one control, with provenance.""" + + control: str # "FRAMEWORK:ID" + status: str # pass | fail | not_applicable + rule_id: str + title: str + severity: str + source: str # report file / package name the finding came from + + +def _iter_report(doc: dict, source: str): + for f in doc.get("findings", []): + status = f.get("status", NOT_APPLICABLE) + rule_id = f.get("id", "") + title = f.get("title", "") + severity = f.get("severity", "medium") + for control in f.get("controls", []): + yield Observation( + control=control, + status=status, + rule_id=rule_id, + title=title, + severity=severity, + source=source, + ) + + +def load_report(path: str | Path) -> list[Observation]: + """Load observations from a single audit-report JSON file.""" + p = Path(path) + doc = json.loads(p.read_text(encoding="utf-8")) + # Prefer the package name audit-report records; fall back to the file name. + source = doc.get("source_package") or p.name + return list(_iter_report(doc, source)) + + +def _expand(paths: list[str | Path]) -> list[Path]: + """Resolve inputs: JSON files pass through; directories contribute their *.json.""" + resolved: list[Path] = [] + for raw in paths: + p = Path(raw) + if p.is_dir(): + resolved.extend(sorted(p.glob("*.json"))) + else: + resolved.append(p) + return resolved + + +def load_corpus(paths: list[str | Path]) -> list[Observation]: + """Load and flatten observations from files and/or directories of reports.""" + files = _expand(paths) + if not files: + raise ValueError("no audit-report JSON files found in the given paths") + observations: list[Observation] = [] + for f in files: + observations.extend(load_report(f)) + return observations new file mode 100644 @@ -0,0 +1,179 @@ +"""Compute control coverage: join an evidence corpus onto framework catalogs. + +For every control in a catalog we assign one **assurance state**: + +* ``supported`` — in scope, at least one mapped finding passes and none fail. +* ``failing`` — in scope, at least one mapped finding fails. +* ``asserted`` — in scope, findings map here but their data was absent + (``not_applicable``): evidence was attempted, not obtained. +* ``unaddressed`` — in scope, *no* finding maps here at all. The blind spot. +* ``out_of_scope``— excluded by the scope file, with a recorded justification. + +When several findings touch one control the worst wins: a single failure makes the +control ``failing`` regardless of how many others pass. Coverage is then a headline +number the evidence-first tools cannot produce — of everything a framework +requires, how much the corpus even looks at, and how much it supports. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field + +from .catalog import Catalog, Control +from .corpus import FAIL, PASS, Observation + +SUPPORTED = "supported" +FAILING = "failing" +ASSERTED = "asserted" +UNADDRESSED = "unaddressed" +OUT_OF_SCOPE = "out_of_scope" + +# Order states appear in reports and roll up in summaries (most urgent first). +STATE_ORDER = [FAILING, UNADDRESSED, ASSERTED, SUPPORTED, OUT_OF_SCOPE] + +# States that count as the control being "addressed" by the corpus at all. +_ADDRESSED = {SUPPORTED, FAILING, ASSERTED} + + +@dataclass +class ControlResult: + """One control's assurance state and the evidence behind it.""" + + control: Control + state: str + observations: list[Observation] = field(default_factory=list) + owner: str = "" + exclusion_reason: str = "" + + @property + def addressed(self) -> bool: + return self.state in _ADDRESSED + + +def _state_for(observations: list[Observation]) -> str: + """Worst-wins resolution of a control's state from its observations.""" + if not observations: + return UNADDRESSED + statuses = {o.status for o in observations} + if FAIL in statuses: + return FAILING + if PASS in statuses: + return SUPPORTED + return ASSERTED # only not_applicable observations remain + + +@dataclass +class FrameworkCoverage: + """Coverage of one framework catalog by the corpus.""" + + catalog: Catalog + results: list[ControlResult] + + def by_state(self, state: str) -> list[ControlResult]: + return [r for r in self.results if r.state == state] + + @property + def counts(self) -> dict[str, int]: + counts = {s: 0 for s in STATE_ORDER} + for r in self.results: + counts[r.state] += 1 + return counts + + @property + def in_scope(self) -> int: + return sum(1 for r in self.results if r.state != OUT_OF_SCOPE) + + @property + def addressed(self) -> int: + return sum(1 for r in self.results if r.addressed) + + @property + def supported(self) -> int: + return sum(1 for r in self.results if r.state == SUPPORTED) + + @property + def coverage_pct(self) -> float: + """Share of in-scope controls the corpus touches at all (0–100).""" + return round(100 * self.addressed / self.in_scope, 1) if self.in_scope else 0.0 + + @property + def assured_pct(self) -> float: + """Share of in-scope controls that are supported and not failing (0–100).""" + return round(100 * self.supported / self.in_scope, 1) if self.in_scope else 0.0 + + @property + def blind_spots(self) -> list[ControlResult]: + """In-scope controls no finding touches — the headline gap list.""" + return self.by_state(UNADDRESSED) + + +@dataclass +class CoverageReport: + """Coverage across every requested framework, plus corpus-wide diagnostics.""" + + subject: str + generated_at: str + frameworks: list[FrameworkCoverage] + # Control codes cited by the corpus that no loaded catalog defines. These are + # typos, renamed controls, or controls outside the bundled catalogs — either + # way, evidence pointing at nothing is worth surfacing. + orphan_codes: list[str] = field(default_factory=list) + source_count: int = 0 + + +def _observations_by_control(observations: list[Observation]) -> dict[str, list[Observation]]: + grouped: dict[str, list[Observation]] = {} + for obs in observations: + grouped.setdefault(obs.control, []).append(obs) + return grouped + + +def evaluate( + catalogs: list[Catalog], + observations: list[Observation], + scope=None, + subject: str = "", + generated_at: str = "", +) -> CoverageReport: + """Produce a :class:`CoverageReport` from catalogs, a corpus, and a scope.""" + grouped = _observations_by_control(observations) + catalog_codes: set[str] = set() + loaded_frameworks = {cat.framework for cat in catalogs} + + frameworks: list[FrameworkCoverage] = [] + for cat in catalogs: + catalog_codes |= cat.codes() + results: list[ControlResult] = [] + for control in cat.controls: + code = control.code + obs = grouped.get(code, []) + owner = scope.owner(code) if scope else "" + if scope and scope.excluded(code): + results.append( + ControlResult(control, OUT_OF_SCOPE, obs, owner, scope.reason(code)) + ) + elif scope and scope.family_excluded(cat.framework, control.family): + reason = scope.family_reason(cat.framework, control.family) + results.append(ControlResult(control, OUT_OF_SCOPE, obs, owner, reason)) + else: + results.append(ControlResult(control, _state_for(obs), obs, owner)) + frameworks.append(FrameworkCoverage(cat, results)) + + # A code is an orphan only when its framework *is* loaded but the catalog + # does not define it — a typo or a renamed control. Codes for frameworks we + # did not load this run are simply out of scope, not orphans. + cited = {o.control for o in observations} + orphans = sorted( + code + for code in cited + if code.split(":", 1)[0] in loaded_frameworks and code not in catalog_codes + ) + sources = {o.source for o in observations} + + return CoverageReport( + subject=subject, + generated_at=generated_at, + frameworks=frameworks, + orphan_codes=orphans, + source_count=len(sources), + ) new file mode 100644 @@ -0,0 +1,291 @@ +"""Crosswalk — which controls each piece of evidence supports, across frameworks. + +One check is rarely worth one control. Enforced 2FA is evidence for SOC 2 CC6.1, +ISO A.5.17, and NIST IA-2 at once. This module inverts the coverage result to show +that leverage: for every check (rule) in the corpus, the set of controls it +addresses and the frameworks it spans. + +It then answers a practical question auditors and evidence-owners both ask — *what +is the smallest set of checks that still covers everything?* — with a greedy +set-cover over the addressed controls. The result is an ordered "minimal evidence +set": collect these few checks and you have touched every control the full corpus +touches, which is what you want when scoping a walkthrough or a sample. + +"Addressed" here matches the coverage engine: a control any finding maps to, +whatever the finding's outcome. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field + +from .coverage import CoverageReport + + +@dataclass +class EvidenceItem: + """One check and the controls it supports across frameworks.""" + + rule_id: str + title: str + controls: list[str] # full FRAMEWORK:ID codes, sorted + frameworks: list[str] # framework short codes it spans, sorted + + @property + def count(self) -> int: + return len(self.controls) + + +@dataclass +class CoverStep: + """One pick in the greedy minimal-evidence set.""" + + rule_id: str + new_controls: int # controls this pick added that were not yet covered + cumulative: int + cumulative_pct: float + + +@dataclass +class Crosswalk: + subject: str + generated_at: str + items: list[EvidenceItem] = field(default_factory=list) + cover: list[CoverStep] = field(default_factory=list) + universe_size: int = 0 + + @property + def multi_framework(self) -> list[EvidenceItem]: + """Checks that earn coverage in more than one framework at once.""" + return [i for i in self.items if len(i.frameworks) > 1] + + +def build(report: CoverageReport) -> Crosswalk: + """Invert a coverage report into a crosswalk and a minimal evidence set.""" + rule_controls: dict[str, set[str]] = {} + rule_title: dict[str, str] = {} + rule_frameworks: dict[str, set[str]] = {} + universe: set[str] = set() + + for fc in report.frameworks: + for r in fc.results: + if not r.addressed: + continue + code = r.control.code + universe.add(code) + for obs in r.observations: + if not obs.rule_id: + continue + rule_controls.setdefault(obs.rule_id, set()).add(code) + rule_title.setdefault(obs.rule_id, obs.title) + rule_frameworks.setdefault(obs.rule_id, set()).add(fc.catalog.framework) + + items = [ + EvidenceItem( + rule_id=rid, + title=rule_title.get(rid, ""), + controls=sorted(codes), + frameworks=sorted(rule_frameworks.get(rid, set())), + ) + for rid, codes in rule_controls.items() + ] + # Most leverage first; rule id breaks ties for stable output. + items.sort(key=lambda i: (-i.count, i.rule_id)) + + cover = _greedy_cover(rule_controls, universe) + return Crosswalk( + subject=report.subject, + generated_at=report.generated_at, + items=items, + cover=cover, + universe_size=len(universe), + ) + + +def _greedy_cover(rule_controls: dict[str, set[str]], universe: set[str]) -> list[CoverStep]: + remaining = set(universe) + pool = {rid: set(codes) for rid, codes in rule_controls.items()} + total = len(universe) or 1 + steps: list[CoverStep] = [] + + while remaining: + best_rule, best_gain = None, 0 + for rid in sorted(pool): + gain = len(pool[rid] & remaining) + if gain > best_gain: + best_rule, best_gain = rid, gain + if not best_rule: # nothing left can cover the remainder + break + remaining -= pool[best_rule] + del pool[best_rule] + cumulative = len(universe) - len(remaining) + steps.append( + CoverStep( + rule_id=best_rule, + new_controls=best_gain, + cumulative=cumulative, + cumulative_pct=round(100 * cumulative / total, 1), + ) + ) + return steps + + +# --- renderers ------------------------------------------------------------- + + +def to_dict(xw: Crosswalk) -> dict: + return { + "subject": xw.subject, + "generated_at": xw.generated_at, + "universe_size": xw.universe_size, + "minimal_evidence_set": [ + { + "rule_id": s.rule_id, + "new_controls": s.new_controls, + "cumulative": s.cumulative, + "cumulative_pct": s.cumulative_pct, + } + for s in xw.cover + ], + "evidence": [ + { + "rule_id": i.rule_id, + "title": i.title, + "frameworks": i.frameworks, + "controls": i.controls, + "count": i.count, + } + for i in xw.items + ], + } + + +def render_json(xw: Crosswalk) -> str: + import json + + return json.dumps(to_dict(xw), indent=2, sort_keys=False) + "\n" + + +def render_markdown(xw: Crosswalk) -> str: + out: list[str] = [] + out.append(f"# Evidence Crosswalk — {xw.subject or 'Evidence corpus'}") + out.append("") + out.append(f"- **Generated:** {xw.generated_at}") + out.append(f"- **Addressed controls:** {xw.universe_size}") + out.append(f"- **Checks in corpus:** {len(xw.items)}") + out.append(f"- **Checks spanning multiple frameworks:** {len(xw.multi_framework)}") + out.append("") + + out.append("## Minimal evidence set") + out.append("") + if xw.cover: + out.append( + f"The {len(xw.cover)} check(s) below cover all {xw.universe_size} addressed " + "controls — the smallest set that touches everything the full corpus does." + ) + out.append("") + out.append("| # | Check | New controls | Cumulative | % of addressed |") + out.append("| ---: | --- | ---: | ---: | ---: |") + for n, s in enumerate(xw.cover, 1): + out.append( + f"| {n} | `{s.rule_id}` | +{s.new_controls} | {s.cumulative} | {s.cumulative_pct}% |" + ) + else: + out.append("_No addressed controls to cover._") + out.append("") + + out.append("## Evidence leverage") + out.append("") + out.append("Each check and the controls it supports, most leverage first.") + out.append("") + out.append("| Check | Frameworks | # | Controls |") + out.append("| --- | --- | ---: | --- |") + for i in xw.items: + codes = ", ".join(f"`{c}`" for c in i.controls) + fws = ", ".join(i.frameworks) + out.append(f"| `{i.rule_id}` | {fws} | {i.count} | {codes} |") + out.append("") + + return "\n".join(out).rstrip() + "\n" + + +_XW_CSS = """ +.fw { display: inline-block; font-size: .7rem; font-weight: 700; letter-spacing: .02em; + padding: .08rem .4rem; border-radius: 4px; margin-right: .25rem; background: #eef; color: #33488c; } +.track { position: relative; background: #eee; border-radius: 4px; height: 1rem; min-width: 5rem; } +.track > span { position: absolute; left: 0; top: 0; bottom: 0; background: #35b866; border-radius: 4px; } +.codes code { font-size: .78rem; } +@media (prefers-color-scheme: dark) { + .fw { background: #22243a; color: #9fb0f0; } + .track { background: #26272b; } +} +""" + + +def render_html(xw: Crosswalk) -> str: + from html import escape + + from .reporters.html import CSS + + title = xw.subject or "Evidence corpus" + body = [ + "<!doctype html><html lang='en'><head><meta charset='utf-8'>", + "<meta name='viewport' content='width=device-width, initial-scale=1'>", + f"<title>Evidence Crosswalk — {escape(title)}</title>", + f"<style>{CSS}{_XW_CSS}</style></head><body><main>", + f"<h1>Evidence Crosswalk — {escape(title)}</h1>", + ( + f'<p class="meta">Generated {escape(xw.generated_at)} · ' + f"{xw.universe_size} addressed control(s) · {len(xw.items)} check(s) · " + f"{len(xw.multi_framework)} spanning multiple frameworks</p>" + ), + "<h2>Minimal evidence set</h2>", + ] + if xw.cover: + body.append( + f"<p>The {len(xw.cover)} check(s) below cover all {xw.universe_size} addressed " + "controls — the smallest set that touches everything the full corpus does.</p>" + ) + body.append( + "<table><thead><tr><th class='num'>#</th><th>Check</th>" + "<th class='num'>New</th><th class='num'>Cumulative</th>" + "<th>% of addressed</th></tr></thead><tbody>" + ) + for n, s in enumerate(xw.cover, 1): + body.append( + "<tr>" + f'<td class="num">{n}</td>' + f"<td><code>{escape(s.rule_id)}</code></td>" + f'<td class="num">+{s.new_controls}</td>' + f'<td class="num">{s.cumulative}</td>' + f'<td><div class="track" title="{s.cumulative_pct}%">' + f'<span style="width:{s.cumulative_pct:.1f}%"></span></div> {s.cumulative_pct}%</td>' + "</tr>" + ) + body.append("</tbody></table>") + else: + body.append("<p>No addressed controls to cover.</p>") + + body.append("<h2>Evidence leverage</h2>") + body.append("<p>Each check and the controls it supports, most leverage first.</p>") + body.append( + "<table><thead><tr><th>Check</th><th>Frameworks</th><th class='num'>#</th>" + "<th>Controls</th></tr></thead><tbody>" + ) + for i in xw.items: + fws = "".join(f'<span class="fw">{escape(f)}</span>' for f in i.frameworks) + codes = ", ".join(f"<code>{escape(c)}</code>" for c in i.controls) + body.append( + "<tr>" + f"<td><code>{escape(i.rule_id)}</code></td>" + f"<td>{fws}</td>" + f'<td class="num">{i.count}</td>' + f'<td class="codes">{codes}</td>' + "</tr>" + ) + body.append("</tbody></table>") + body.append( + "<footer>Generated by control-coverage · Audit Labs. Evidence, not a verdict.</footer>" + ) + body.append("</main></body></html>") + return "".join(body) new file mode 100644 @@ -0,0 +1,26 @@ +"""Renderers for a coverage report: Markdown, HTML, JSON, and a Statement of Applicability.""" + +from __future__ import annotations + +from ..coverage import CoverageReport +from . import html, json, markdown, soa + +_RENDERERS = { + "md": markdown.render, + "markdown": markdown.render, + "html": html.render, + "json": json.render, + "soa": soa.render, +} + +# File extension per format (soa is Markdown by default). +EXTENSIONS = {"md": "md", "markdown": "md", "html": "html", "json": "json", "soa": "md"} + + +def render(report: CoverageReport, fmt: str) -> str: + try: + return _RENDERERS[fmt](report) + except KeyError: + raise ValueError( + f"unknown format '{fmt}'. Choose from: {', '.join(sorted(_RENDERERS))}" + ) from None new file mode 100644 @@ -0,0 +1,252 @@ +"""HTML renderer — a self-contained, printable coverage report. + +No external assets: all CSS is inlined so the file can be attached to an audit +workpaper and opened anywhere, including offline. +""" + +from __future__ import annotations + +from html import escape +from typing import TYPE_CHECKING + +from ..coverage import ( + ASSERTED, + FAILING, + OUT_OF_SCOPE, + SUPPORTED, + UNADDRESSED, +) + +if TYPE_CHECKING: + from ..coverage import CoverageReport, FrameworkCoverage + +_STATE_LABEL = { + SUPPORTED: "supported", + FAILING: "failing", + ASSERTED: "asserted", + UNADDRESSED: "unaddressed", + OUT_OF_SCOPE: "out of scope", +} +_STATE_CLASS = { + SUPPORTED: "supported", + FAILING: "failing", + ASSERTED: "asserted", + UNADDRESSED: "unaddressed", + OUT_OF_SCOPE: "oos", +} + +CSS = """ +:root { color-scheme: light dark; } +* { box-sizing: border-box; } +body { font-family: -apple-system, Segoe UI, Roboto, Helvetica, Arial, sans-serif; + margin: 0; padding: 2rem; line-height: 1.5; color: #1a1a1a; background: #fff; } +main { max-width: 64rem; margin: 0 auto; } +h1 { margin: 0 0 .25rem; font-size: 1.6rem; } +h2 { margin: 2rem 0 .75rem; font-size: 1.25rem; border-bottom: 2px solid #e5e5e5; padding-bottom: .25rem; } +h3 { margin: 1.4rem 0 .5rem; font-size: 1.02rem; } +.meta { color: #555; font-size: .9rem; margin: 0 0 1rem; } +.meta code { background: #f2f2f2; padding: .05rem .3rem; border-radius: 3px; } +.note { background: #f7f7f9; border-left: 3px solid #b9b9c6; padding: .6rem .9rem; + font-size: .9rem; color: #444; border-radius: 0 4px 4px 0; } +table { border-collapse: collapse; width: 100%; font-size: .85rem; margin: .5rem 0; } +th, td { border: 1px solid #e0e0e0; padding: .35rem .5rem; text-align: left; vertical-align: top; } +th { background: #f5f5f7; } +td.num, th.num { text-align: right; } +.badge { display: inline-block; font-weight: 700; font-size: .72rem; letter-spacing: .02em; + padding: .12rem .5rem; border-radius: 999px; white-space: nowrap; } +.badge.supported { background: #e5f6ea; color: #1a7f37; } +.badge.failing { background: #fdeaea; color: #c1272d; } +.badge.asserted { background: #fff4e0; color: #a8620a; } +.badge.unaddressed { background: #eceaf6; color: #5b4bb0; } +.badge.oos { background: #eee; color: #666; } +.bar { display: flex; height: 1.1rem; border-radius: 4px; overflow: hidden; margin: .4rem 0 .2rem; + border: 1px solid #ddd; } +.bar > span { display: block; } +.bar .supported { background: #35b866; } +.bar .failing { background: #e2565b; } +.bar .asserted { background: #eaa53c; } +.bar .unaddressed { background: #8877d8; } +.bar .oos { background: #cfcfcf; } +.headline { font-size: 1.5rem; font-weight: 700; } +.headline small { font-size: .85rem; font-weight: 500; color: #666; } +.legend { font-size: .78rem; color: #666; display: flex; flex-wrap: wrap; gap: .8rem; margin: .2rem 0 1rem; } +.legend i { display: inline-block; width: .8rem; height: .8rem; border-radius: 2px; vertical-align: -1px; margin-right: .25rem; } +footer { margin-top: 3rem; font-size: .8rem; color: #888; border-top: 1px solid #eee; padding-top: .75rem; } +@media (prefers-color-scheme: dark) { + body { color: #e6e6e6; background: #16171a; } + h2 { border-color: #333; } + .meta { color: #aaa; } .meta code { background: #26272b; } + .note { background: #1e1f24; border-color: #444; color: #bbb; } + th, td { border-color: #333; } th { background: #202126; } + .headline small, .legend { color: #999; } + .badge.supported { background: #12321d; color: #4ac36a; } + .badge.failing { background: #3a1416; color: #ff6b70; } + .badge.asserted { background: #33260f; color: #e6a94e; } + .badge.unaddressed { background: #211d3a; color: #9d8ef0; } + .badge.oos { background: #26272b; color: #999; } + .bar { border-color: #333; } + footer { border-color: #2a2b30; } +} +""" + +_LEGEND_COLORS = { + SUPPORTED: "#35b866", + FAILING: "#e2565b", + ASSERTED: "#eaa53c", + UNADDRESSED: "#8877d8", + OUT_OF_SCOPE: "#cfcfcf", +} + + +def _badge(state: str) -> str: + return f'<span class="badge {_STATE_CLASS[state]}">{_STATE_LABEL[state]}</span>' + + +def _bar(fc: FrameworkCoverage) -> str: + counts = fc.counts + total = sum(counts.values()) or 1 + segments = [] + for state in [SUPPORTED, FAILING, ASSERTED, UNADDRESSED, OUT_OF_SCOPE]: + n = counts[state] + if not n: + continue + pct = 100 * n / total + segments.append( + f'<span class="{_STATE_CLASS[state]}" style="width:{pct:.2f}%" ' + f'title="{n} {_STATE_LABEL[state]}"></span>' + ) + return '<div class="bar">' + "".join(segments) + "</div>" + + +def _legend() -> str: + items = [] + for state in [SUPPORTED, FAILING, ASSERTED, UNADDRESSED, OUT_OF_SCOPE]: + items.append( + f'<span><i style="background:{_LEGEND_COLORS[state]}"></i>{_STATE_LABEL[state]}</span>' + ) + return '<div class="legend">' + "".join(items) + "</div>" + + +def _checked_by(result) -> str: + if result.state == OUT_OF_SCOPE: + return f"<em>excluded: {escape(result.exclusion_reason)}</em>" + rules = sorted({o.rule_id for o in result.observations if o.rule_id}) + return ", ".join(f"<code>{escape(r)}</code>" for r in rules) + + +def _framework_section(fc: FrameworkCoverage) -> str: + cat = fc.catalog + partial = "" if cat.complete else ( + ' <small>(partial catalog — coverage is of the shipped subset)</small>' + ) + rows = [] + for r in fc.results: + rows.append( + "<tr>" + f"<td><strong>{escape(r.control.id)}</strong></td>" + f"<td>{_badge(r.state)}</td>" + f"<td>{escape(r.control.title)}</td>" + f"<td>{_checked_by(r)}</td>" + "</tr>" + ) + return ( + f"<h2>{escape(cat.name)}{partial}</h2>" + f'<p class="headline">{fc.coverage_pct}% <small>coverage · {fc.addressed}/{fc.in_scope} ' + f"in-scope controls addressed · {fc.assured_pct}% assured</small></p>" + f"{_bar(fc)}{_legend()}" + "<table><thead><tr><th>Control</th><th>Status</th><th>Description</th>" + "<th>Checked by</th></tr></thead><tbody>" + + "".join(rows) + + "</tbody></table>" + ) + + +def _summary_table(report: CoverageReport) -> str: + rows = [] + for fc in report.frameworks: + c = fc.counts + rows.append( + "<tr>" + f"<td>{escape(fc.catalog.name)}</td>" + f'<td class="num">{fc.in_scope}</td>' + f'<td class="num">{fc.addressed}</td>' + f'<td class="num">{fc.supported}</td>' + f'<td class="num">{c[FAILING]}</td>' + f'<td class="num">{len(fc.blind_spots)}</td>' + f'<td class="num">{fc.coverage_pct}%</td>' + f'<td class="num">{fc.assured_pct}%</td>' + "</tr>" + ) + return ( + "<table><thead><tr><th>Framework</th><th class='num'>In scope</th>" + "<th class='num'>Addressed</th><th class='num'>Supported</th>" + "<th class='num'>Failing</th><th class='num'>Blind spots</th>" + "<th class='num'>Coverage</th><th class='num'>Assured</th></tr></thead><tbody>" + + "".join(rows) + + "</tbody></table>" + ) + + +def _blind_spots(report: CoverageReport) -> str: + total = sum(len(fc.blind_spots) for fc in report.frameworks) + if total == 0: + return "<h2>Blind spots</h2><p>No in-scope control is left unaddressed by the corpus.</p>" + parts = [ + "<h2>Blind spots</h2>", + ( + f"<p>{total} in-scope control(s) are <strong>unaddressed</strong> — no finding " + "in the corpus maps to them.</p>" + ), + ] + for fc in report.frameworks: + spots = fc.blind_spots + if not spots: + continue + parts.append(f"<h3>{escape(fc.catalog.name)} ({len(spots)})</h3><ul>") + for r in spots: + fam = f" <em>· {escape(r.control.family)}</em>" if r.control.family else "" + parts.append( + f"<li><strong>{escape(r.control.id)}</strong> — {escape(r.control.title)}{fam}</li>" + ) + parts.append("</ul>") + return "".join(parts) + + +def render(report: CoverageReport) -> str: + title = report.subject or "Evidence corpus" + frameworks = ", ".join(fc.catalog.framework for fc in report.frameworks) + body = [ + "<!doctype html><html lang='en'><head><meta charset='utf-8'>", + "<meta name='viewport' content='width=device-width, initial-scale=1'>", + f"<title>Control Coverage — {escape(title)}</title>", + f"<style>{CSS}</style></head><body><main>", + f"<h1>Control Coverage — {escape(title)}</h1>", + ( + f'<p class="meta">Generated {escape(report.generated_at)} · ' + f"{report.source_count} evidence source(s) · frameworks: {escape(frameworks)}</p>" + ), + ( + '<p class="note">Coverage measures how much of a framework the evidence corpus ' + "addresses — not whether the organization is compliant. An unaddressed control is " + "a gap in <em>evidence</em>, which may reflect a real control gap or simply a signal " + "not yet collected. The final judgment belongs to the organization and its auditor.</p>" + ), + "<h2>Summary</h2>", + _summary_table(report), + _blind_spots(report), + ] + for fc in report.frameworks: + body.append(_framework_section(fc)) + + if report.orphan_codes: + codes = "".join(f"<li><code>{escape(c)}</code></li>" for c in report.orphan_codes) + body.append( + "<h2>Unmatched control codes</h2><p>The corpus cites these codes, but no loaded " + f"catalog defines them (typos, renamed, or out-of-catalog):</p><ul>{codes}</ul>" + ) + + body.append( + "<footer>Generated by control-coverage · Audit Labs. Evidence, not a verdict.</footer>" + ) + body.append("</main></body></html>") + return "".join(body) new file mode 100644 @@ -0,0 +1,55 @@ +"""JSON renderer — the coverage result as a machine-readable document. + +Stable key order so two runs diff cleanly. Suitable for dashboards, ticketing, +or gating a pipeline on the coverage percentage. +""" + +from __future__ import annotations + +import json as _json +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from ..coverage import CoverageReport + + +def to_dict(report: CoverageReport) -> dict: + return { + "subject": report.subject, + "generated_at": report.generated_at, + "source_count": report.source_count, + "frameworks": [ + { + "framework": fc.catalog.framework, + "name": fc.catalog.name, + "version": fc.catalog.version, + "catalog_coverage": fc.catalog.coverage, + "in_scope": fc.in_scope, + "addressed": fc.addressed, + "supported": fc.supported, + "coverage_pct": fc.coverage_pct, + "assured_pct": fc.assured_pct, + "counts": fc.counts, + "controls": [ + { + "id": r.control.id, + "code": r.control.code, + "title": r.control.title, + "family": r.control.family, + "state": r.state, + "owner": r.owner, + "exclusion_reason": r.exclusion_reason, + "checked_by": sorted({o.rule_id for o in r.observations if o.rule_id}), + "sources": sorted({o.source for o in r.observations}), + } + for r in fc.results + ], + } + for fc in report.frameworks + ], + "orphan_codes": report.orphan_codes, + } + + +def render(report: CoverageReport) -> str: + return _json.dumps(to_dict(report), indent=2, sort_keys=False) + "\n" new file mode 100644 @@ -0,0 +1,153 @@ +"""Markdown renderer — the coverage matrix and blind-spot list, auditor-facing.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from ..coverage import ( + ASSERTED, + FAILING, + OUT_OF_SCOPE, + STATE_ORDER, + SUPPORTED, + UNADDRESSED, +) + +if TYPE_CHECKING: + from ..coverage import CoverageReport, FrameworkCoverage + +_STATE_LABEL = { + SUPPORTED: "supported", + FAILING: "failing", + ASSERTED: "asserted", + UNADDRESSED: "unaddressed", + OUT_OF_SCOPE: "out of scope", +} +_STATE_MARK = { + SUPPORTED: "✓", + FAILING: "✗", + ASSERTED: "◐", + UNADDRESSED: "○", + OUT_OF_SCOPE: "—", +} + + +def _evidence_note(result) -> str: + """A short 'checked by' cell: rule ids or the exclusion reason.""" + if result.state == OUT_OF_SCOPE: + return f"_excluded: {result.exclusion_reason}_" + if not result.observations: + return "" + rules = sorted({o.rule_id for o in result.observations if o.rule_id}) + return ", ".join(f"`{r}`" for r in rules) + + +def _framework_section(fc: FrameworkCoverage) -> list[str]: + cat = fc.catalog + counts = fc.counts + out: list[str] = [] + out.append(f"## {cat.name}") + out.append("") + suffix = "" if cat.complete else " _(partial catalog — coverage is of the shipped subset)_" + out.append( + f"**Coverage {fc.coverage_pct}%** ({fc.addressed}/{fc.in_scope} in-scope " + f"controls addressed) · **assured {fc.assured_pct}%** " + f"({fc.supported} supported){suffix}" + ) + out.append("") + out.append( + "| " + " · ".join( + f"{_STATE_MARK[s]} {counts[s]} {_STATE_LABEL[s]}" + for s in STATE_ORDER + if counts[s] + ) + " |" + ) + out.append("|" + "---|") + out.append("") + out.append("| Control | Status | Description | Checked by |") + out.append("| --- | --- | --- | --- |") + for r in fc.results: + mark = _STATE_MARK[r.state] + label = _STATE_LABEL[r.state] + out.append( + f"| **{r.control.id}** | {mark} {label} | {r.control.title} | {_evidence_note(r)} |" + ) + out.append("") + return out + + +def _blind_spots(report: CoverageReport) -> list[str]: + out: list[str] = ["## Blind spots", ""] + total = sum(len(fc.blind_spots) for fc in report.frameworks) + if total == 0: + out.append("_No in-scope control is left unaddressed by the corpus._") + out.append("") + return out + out.append( + f"{total} in-scope control(s) are **unaddressed** — no finding in the corpus " + "maps to them. These are the framework requirements the evidence does not " + "look at yet." + ) + out.append("") + for fc in report.frameworks: + spots = fc.blind_spots + if not spots: + continue + out.append(f"### {fc.catalog.name} ({len(spots)})") + out.append("") + for r in spots: + fam = f" · _{r.control.family}_" if r.control.family else "" + out.append(f"- **{r.control.id}** — {r.control.title}{fam}") + out.append("") + return out + + +def render(report: CoverageReport) -> str: + out: list[str] = [] + title = report.subject or "Evidence corpus" + out.append(f"# Control Coverage — {title}") + out.append("") + out.append(f"- **Generated:** {report.generated_at}") + out.append(f"- **Corpus:** {report.source_count} evidence source(s)") + frameworks = ", ".join(fc.catalog.framework for fc in report.frameworks) + out.append(f"- **Frameworks:** {frameworks}") + out.append("") + out.append( + "> Coverage measures how much of a framework the evidence corpus addresses — " + "not whether the organization is compliant. An unaddressed control is a gap " + "in *evidence*, which may reflect a real gap in *controls* or simply a signal " + "not yet collected. The final judgment belongs to the organization and its auditor." + ) + out.append("") + + # Headline table across frameworks. + out.append("## Summary") + out.append("") + out.append("| Framework | In scope | Addressed | Supported | Failing | Blind spots | Coverage | Assured |") + out.append("| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: |") + for fc in report.frameworks: + c = fc.counts + out.append( + f"| {fc.catalog.name} | {fc.in_scope} | {fc.addressed} | {fc.supported} " + f"| {c[FAILING]} | {len(fc.blind_spots)} | {fc.coverage_pct}% | {fc.assured_pct}% |" + ) + out.append("") + + out.extend(_blind_spots(report)) + + for fc in report.frameworks: + out.extend(_framework_section(fc)) + + if report.orphan_codes: + out.append("## Unmatched control codes") + out.append("") + out.append( + "The corpus cites these control codes, but no loaded catalog defines them. " + "They are typos, renamed controls, or controls outside the bundled catalogs:" + ) + out.append("") + for code in report.orphan_codes: + out.append(f"- `{code}`") + out.append("") + + return "\n".join(out).rstrip() + "\n" new file mode 100644 @@ -0,0 +1,68 @@ +"""Statement of Applicability renderer. + +ISO 27001 requires a Statement of Applicability (SoA): for every Annex A control, +whether it applies, why, and its implementation status. This renders exactly that +from the coverage result — applicability comes from the scope file, and the +implementation status is derived from the evidence corpus rather than asserted by +hand, so the SoA stays honest to what the evidence actually shows. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from ..coverage import ASSERTED, FAILING, OUT_OF_SCOPE, SUPPORTED, UNADDRESSED + +if TYPE_CHECKING: + from ..coverage import CoverageReport + +# How each assurance state reads as an implementation status in an SoA. +_IMPL_STATUS = { + SUPPORTED: "Implemented — supporting evidence collected", + FAILING: "Deficient — evidence shows a non-supporting state", + ASSERTED: "Claimed — mapped, but evidence not yet obtained", + UNADDRESSED: "Not evidenced — no evidence collected yet", + OUT_OF_SCOPE: "Excluded", +} + + +def _justification(result) -> str: + if result.state == OUT_OF_SCOPE: + return result.exclusion_reason + rules = sorted({o.rule_id for o in result.observations if o.rule_id}) + sources = sorted({o.source for o in result.observations}) + if rules: + return f"Evidenced by {', '.join(rules)} in {', '.join(sources)}." + return "No control in the evidence corpus addresses this yet." + + +def render(report: CoverageReport) -> str: + out: list[str] = [] + subject = report.subject or "the organization" + out.append(f"# Statement of Applicability — {report.subject or 'Untitled'}") + out.append("") + out.append(f"- **Generated:** {report.generated_at}") + out.append(f"- **Derived from:** {report.source_count} evidence source(s)") + out.append("") + out.append( + f"This Statement of Applicability records, for each control in scope for " + f"{subject}, whether it applies and its implementation status. Applicability " + "decisions come from the documented scope; implementation status is derived " + "from collected evidence, not asserted." + ) + out.append("") + + for fc in report.frameworks: + out.append(f"## {fc.catalog.name}") + out.append("") + out.append("| Control | Description | Applicable | Status | Justification | Owner |") + out.append("| --- | --- | --- | --- | --- | --- |") + for r in fc.results: + applicable = "No" if r.state == OUT_OF_SCOPE else "Yes" + out.append( + f"| **{r.control.id}** | {r.control.title} | {applicable} " + f"| {_IMPL_STATUS[r.state]} | {_justification(r)} | {r.owner} |" + ) + out.append("") + + return "\n".join(out).rstrip() + "\n" new file mode 100644 @@ -0,0 +1,106 @@ +"""Scope — which controls are in play, and which are excluded with justification. + +Not every control applies to every organization. ISO 27001 formalizes this as the +**Statement of Applicability (SoA)**: for each Annex A control, a decision to apply +it or not, and the reason. This module reads a small YAML scope file expressing +exactly that, so coverage is computed over *in-scope* controls and exclusions are +recorded rather than silently counted as gaps:: + + subject: Acme Production + frameworks: [SOC2, ISO] + exclusions: + - {control: ISO:A.5.7, reason: "No formal threat-intel program; risk accepted 2026-Q1."} + - {control: ISO:A.7.1, reason: "Fully cloud-hosted; no physical premises in scope."} + exclude_families: + - {framework: SOC2, family: Privacy, reason: "Privacy category not in the SOC 2 audit scope."} + owners: + SOC2:CC6.1: platform-team + +``exclude_families`` removes a whole category at once — a SOC 2 Trust Services +category, an ISO Annex A theme, a NIST family — which is how audit scope is actually +decided (a SOC 2 report covers Security and maybe Availability, rarely Privacy). + +Every exclusion, per-control or per-family, must carry a reason — an exclusion without +justification is the single most common SoA audit finding, so we reject it rather than +accept it. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from pathlib import Path + +import yaml + + +@dataclass +class Scope: + """A parsed scope / Statement of Applicability.""" + + subject: str = "" + frameworks: list[str] = field(default_factory=list) + # control code -> justification for excluding it + exclusions: dict[str, str] = field(default_factory=dict) + # (framework, family) -> justification for excluding a whole family/category + family_exclusions: dict[tuple[str, str], str] = field(default_factory=dict) + # control code -> owning team/person (optional metadata) + owners: dict[str, str] = field(default_factory=dict) + + def excluded(self, code: str) -> bool: + return code in self.exclusions + + def reason(self, code: str) -> str: + return self.exclusions.get(code, "") + + def family_excluded(self, framework: str, family: str) -> bool: + return (framework, family) in self.family_exclusions + + def family_reason(self, framework: str, family: str) -> str: + return self.family_exclusions.get((framework, family), "") + + def owner(self, code: str) -> str: + return self.owners.get(code, "") + + +def empty() -> Scope: + """A scope that excludes nothing — every catalog control is in scope.""" + return Scope() + + +def load(path: str | Path) -> Scope: + """Load a scope file, validating that every exclusion carries a reason.""" + raw = yaml.safe_load(Path(path).read_text(encoding="utf-8")) or {} + + exclusions: dict[str, str] = {} + for i, item in enumerate(raw.get("exclusions", [])): + if not isinstance(item, dict) or "control" not in item: + raise ValueError(f"exclusion #{i + 1} must be a mapping with a 'control' key") + code = str(item["control"]).strip() + reason = str(item.get("reason", "")).strip() + if not reason: + raise ValueError(f"exclusion for '{code}' needs a non-empty 'reason'") + exclusions[code] = reason + + family_exclusions: dict[tuple[str, str], str] = {} + for i, item in enumerate(raw.get("exclude_families", [])): + if not isinstance(item, dict) or "framework" not in item or "family" not in item: + raise ValueError( + f"exclude_families #{i + 1} must be a mapping with 'framework' and 'family' keys" + ) + framework = str(item["framework"]).strip() + family = str(item["family"]).strip() + reason = str(item.get("reason", "")).strip() + if not reason: + raise ValueError(f"family exclusion for '{framework}:{family}' needs a non-empty 'reason'") + family_exclusions[(framework, family)] = reason + + owners = {str(k): str(v) for k, v in (raw.get("owners") or {}).items()} + frameworks = [str(f) for f in (raw.get("frameworks") or [])] + + return Scope( + subject=str(raw.get("subject", "")), + frameworks=frameworks, + exclusions=exclusions, + family_exclusions=family_exclusions, + owners=owners, + ) new file mode 100644 @@ -0,0 +1,332 @@ +"""Coverage trend — how control coverage moved between two corpora. + +`audit-report` diffs two evidence *packages*; this diffs two whole *corpora* at +the framework-coverage level. Evaluate an earlier corpus and a current one with +the same catalogs and scope, then compare each control's assurance state to see +what improved, what regressed, and how the coverage percentage moved. + +States are ranked ``supported > failing > asserted > unaddressed`` — going from +"no data" to "failing data" still counts as more assurance, because you now have +evidence. Two transitions are called out specially because they move the coverage +numerator: **gained** (a blind spot became addressed) and **lost** (an addressed +control became a blind spot). +""" + +from __future__ import annotations + +from dataclasses import dataclass, field + +from .coverage import ( + ASSERTED, + FAILING, + OUT_OF_SCOPE, + SUPPORTED, + UNADDRESSED, + CoverageReport, +) + +# Assurance rank; higher is more assured. out_of_scope is handled separately. +_RANK = {SUPPORTED: 3, FAILING: 2, ASSERTED: 1, UNADDRESSED: 0} +_ADDRESSED = {SUPPORTED, FAILING, ASSERTED} + +REGRESSED = "regressed" +IMPROVED = "improved" +GAINED = "gained" +LOST = "lost" +RESCOPED = "rescoped" +UNCHANGED = "unchanged" + +# Order categories appear in a report (most urgent first). +CATEGORY_ORDER = [REGRESSED, LOST, GAINED, IMPROVED, RESCOPED, UNCHANGED] +# Categories that count as a regression for the CI gate. +_REGRESSION = {REGRESSED, LOST} + + +def _categorize(old: str, new: str) -> str: + if old == new: + return UNCHANGED + if OUT_OF_SCOPE in (old, new): + return RESCOPED + if old == UNADDRESSED and new in _ADDRESSED: + return GAINED + if old in _ADDRESSED and new == UNADDRESSED: + return LOST + return IMPROVED if _RANK[new] > _RANK[old] else REGRESSED + + +@dataclass +class ControlDelta: + """How one control's assurance state changed between the two corpora.""" + + framework: str + id: str + title: str + old_state: str + new_state: str + category: str + + +@dataclass +class FrameworkTrend: + """Coverage movement for one framework.""" + + framework: str + name: str + deltas: list[ControlDelta] + old_coverage_pct: float + new_coverage_pct: float + + def by_category(self, category: str) -> list[ControlDelta]: + return [d for d in self.deltas if d.category == category] + + @property + def counts(self) -> dict[str, int]: + counts = {c: 0 for c in CATEGORY_ORDER} + for d in self.deltas: + counts[d.category] += 1 + return counts + + @property + def coverage_delta(self) -> float: + return round(self.new_coverage_pct - self.old_coverage_pct, 1) + + @property + def regressions(self) -> int: + return sum(1 for d in self.deltas if d.category in _REGRESSION) + + +@dataclass +class TrendReport: + subject: str + generated_at: str + frameworks: list[FrameworkTrend] = field(default_factory=list) + + @property + def total_regressions(self) -> int: + return sum(fc.regressions for fc in self.frameworks) + + +def compare(baseline: CoverageReport, current: CoverageReport) -> TrendReport: + """Diff two coverage reports evaluated with the same catalogs and scope.""" + old_by_fw = {fc.catalog.framework: fc for fc in baseline.frameworks} + + frameworks: list[FrameworkTrend] = [] + for cur in current.frameworks: + base = old_by_fw.get(cur.catalog.framework) + if base is None: + continue # framework only appears in the current run + old_state = {r.control.id: r.state for r in base.results} + deltas: list[ControlDelta] = [] + for r in cur.results: + prev = old_state.get(r.control.id, UNADDRESSED) + deltas.append( + ControlDelta( + framework=cur.catalog.framework, + id=r.control.id, + title=r.control.title, + old_state=prev, + new_state=r.state, + category=_categorize(prev, r.state), + ) + ) + frameworks.append( + FrameworkTrend( + framework=cur.catalog.framework, + name=cur.catalog.name, + deltas=deltas, + old_coverage_pct=base.coverage_pct, + new_coverage_pct=cur.coverage_pct, + ) + ) + + return TrendReport( + subject=current.subject, + generated_at=current.generated_at, + frameworks=frameworks, + ) + + +# --- renderers ------------------------------------------------------------- + +_ARROW = { + REGRESSED: "▼", + LOST: "▼", + GAINED: "▲", + IMPROVED: "▲", + RESCOPED: "◆", + UNCHANGED: "·", +} + + +def to_dict(report: TrendReport) -> dict: + return { + "subject": report.subject, + "generated_at": report.generated_at, + "total_regressions": report.total_regressions, + "frameworks": [ + { + "framework": fc.framework, + "name": fc.name, + "old_coverage_pct": fc.old_coverage_pct, + "new_coverage_pct": fc.new_coverage_pct, + "coverage_delta": fc.coverage_delta, + "counts": fc.counts, + "changes": [ + { + "id": d.id, + "title": d.title, + "old_state": d.old_state, + "new_state": d.new_state, + "category": d.category, + } + for d in fc.deltas + if d.category != UNCHANGED + ], + } + for fc in report.frameworks + ], + } + + +def render_json(report: TrendReport) -> str: + import json + + return json.dumps(to_dict(report), indent=2, sort_keys=False) + "\n" + + +def render_markdown(report: TrendReport) -> str: + out: list[str] = [] + out.append(f"# Coverage Trend — {report.subject or 'Evidence corpus'}") + out.append("") + out.append(f"- **Generated:** {report.generated_at}") + out.append(f"- **Regressions:** {report.total_regressions}") + out.append("") + + out.append("| Framework | Coverage (was → now) | Δ | Regressed | Lost | Gained | Improved |") + out.append("| --- | --- | ---: | ---: | ---: | ---: | ---: |") + for fc in report.frameworks: + c = fc.counts + sign = "+" if fc.coverage_delta >= 0 else "" + out.append( + f"| {fc.name} | {fc.old_coverage_pct}% → {fc.new_coverage_pct}% " + f"| {sign}{fc.coverage_delta} | {c[REGRESSED]} | {c[LOST]} | {c[GAINED]} | {c[IMPROVED]} |" + ) + out.append("") + + for fc in report.frameworks: + changes = [d for d in fc.deltas if d.category != UNCHANGED] + if not changes: + continue + out.append(f"## {fc.name}") + out.append("") + out.append("| Control | Change | Was → Now | Description |") + out.append("| --- | --- | --- | --- |") + ordered = sorted(changes, key=lambda d: CATEGORY_ORDER.index(d.category)) + for d in ordered: + arrow = _ARROW[d.category] + out.append( + f"| **{d.id}** | {arrow} {d.category} | {d.old_state} → {d.new_state} | {d.title} |" + ) + out.append("") + + if all(all(x.category == UNCHANGED for x in fc.deltas) for fc in report.frameworks): + out.append("_No control changed state between the two corpora._") + out.append("") + + return "\n".join(out).rstrip() + "\n" + + +# Category badge colours, layered onto the shared coverage CSS. +_TREND_CSS = """ +.badge.improved { background: #e5f6ea; color: #1a7f37; } +.badge.gained { background: #e3eefb; color: #1667a8; } +.badge.regressed { background: #fdeaea; color: #c1272d; } +.badge.lost { background: #fbe7d8; color: #a8480a; } +.badge.rescoped { background: #eee; color: #666; } +.delta-up { color: #1a7f37; font-weight: 700; } +.delta-down { color: #c1272d; font-weight: 700; } +@media (prefers-color-scheme: dark) { + .badge.improved { background: #12321d; color: #4ac36a; } + .badge.gained { background: #12263a; color: #5aa6e6; } + .badge.regressed { background: #3a1416; color: #ff6b70; } + .badge.lost { background: #33220f; color: #e0913c; } + .badge.rescoped { background: #26272b; color: #999; } +} +""" + + +def render_html(report: TrendReport) -> str: + from html import escape + + from .reporters.html import CSS + + def badge(category: str) -> str: + return f'<span class="badge {category}">{_ARROW[category]} {category}</span>' + + def delta(value: float) -> str: + cls = "delta-up" if value >= 0 else "delta-down" + sign = "+" if value >= 0 else "" + return f'<span class="{cls}">{sign}{value}</span>' + + title = report.subject or "Evidence corpus" + body = [ + "<!doctype html><html lang='en'><head><meta charset='utf-8'>", + "<meta name='viewport' content='width=device-width, initial-scale=1'>", + f"<title>Coverage Trend — {escape(title)}</title>", + f"<style>{CSS}{_TREND_CSS}</style></head><body><main>", + f"<h1>Coverage Trend — {escape(title)}</h1>", + ( + f'<p class="meta">Generated {escape(report.generated_at)} · ' + f"{report.total_regressions} regression(s)</p>" + ), + "<h2>Summary</h2>", + ( + "<table><thead><tr><th>Framework</th><th>Coverage (was → now)</th>" + "<th class='num'>Δ</th><th class='num'>Regressed</th><th class='num'>Lost</th>" + "<th class='num'>Gained</th><th class='num'>Improved</th></tr></thead><tbody>" + ), + ] + for fc in report.frameworks: + c = fc.counts + body.append( + "<tr>" + f"<td>{escape(fc.name)}</td>" + f"<td>{fc.old_coverage_pct}% → {fc.new_coverage_pct}%</td>" + f'<td class="num">{delta(fc.coverage_delta)}</td>' + f'<td class="num">{c[REGRESSED]}</td><td class="num">{c[LOST]}</td>' + f'<td class="num">{c[GAINED]}</td><td class="num">{c[IMPROVED]}</td>' + "</tr>" + ) + body.append("</tbody></table>") + + for fc in report.frameworks: + changes = sorted( + (d for d in fc.deltas if d.category != UNCHANGED), + key=lambda d: CATEGORY_ORDER.index(d.category), + ) + if not changes: + continue + body.append(f"<h2>{escape(fc.name)}</h2>") + body.append( + "<table><thead><tr><th>Control</th><th>Change</th><th>Was → Now</th>" + "<th>Description</th></tr></thead><tbody>" + ) + for d in changes: + body.append( + "<tr>" + f"<td><strong>{escape(d.id)}</strong></td>" + f"<td>{badge(d.category)}</td>" + f"<td>{d.old_state} → {d.new_state}</td>" + f"<td>{escape(d.title)}</td>" + "</tr>" + ) + body.append("</tbody></table>") + + if not any(any(x.category != UNCHANGED for x in fc.deltas) for fc in report.frameworks): + body.append("<p>No control changed state between the two corpora.</p>") + + body.append( + "<footer>Generated by control-coverage · Audit Labs. Evidence, not a verdict.</footer>" + ) + body.append("</main></body></html>") + return "".join(body) new file mode 100644 @@ -0,0 +1,43 @@ +# Gate a pipeline on framework coverage. +# +# Assumes an earlier job produced audit-report JSON packages under ./reports/ +# (one per platform). This job fails the build if SOC 2 or ISO coverage drops +# below the threshold, and publishes the coverage report + Statement of +# Applicability as build artifacts. +name: control-coverage + +on: + workflow_dispatch: + schedule: + - cron: "0 6 * * 1" # Mondays, 06:00 UTC + +jobs: + coverage: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Install control-coverage + run: pip install git+https://github.com/audit-labs/control-coverage + + # Your own step(s) should populate ./reports/*.json with audit-report output. + + - name: Coverage report + SoA + run: | + control-coverage ./reports/ \ + --scope examples/soa.yaml \ + --format md,html,json,soa \ + --out coverage-out/ + + - name: Fail if coverage regresses + run: control-coverage ./reports/ --scope examples/soa.yaml --fail-under 60 + + - uses: actions/upload-artifact@v4 + if: always() + with: + name: coverage + path: coverage-out/ new file mode 100644 @@ -0,0 +1,27 @@ +# Example scope / Statement of Applicability. +# +# `frameworks` selects which catalogs to evaluate. Each exclusion removes a +# control from the in-scope denominator and MUST carry a justification. `owners` +# is optional metadata that flows through to the SoA. +subject: Acme Production +frameworks: [SOC2, ISO] + +exclusions: + - control: ISO:A.7.1 + reason: "Fully cloud-hosted; no physical premises are in scope for the ISMS." + - control: ISO:A.7.2 + reason: "No physical premises — physical entry controls are not applicable." + - control: ISO:A.5.7 + reason: "No formal threat-intelligence program; risk accepted by the CISO for 2026." + +# Drop whole categories/families at once. A SOC 2 report here covers Security and +# Availability only, so the other three Trust Services categories are out of scope. +exclude_families: + - {framework: SOC2, family: Confidentiality, reason: "Confidentiality category not in the SOC 2 audit scope."} + - {framework: SOC2, family: Processing Integrity, reason: "Processing Integrity category not in the SOC 2 audit scope."} + - {framework: SOC2, family: Privacy, reason: "Privacy category not in the SOC 2 audit scope."} + +owners: + SOC2:CC6.1: platform-team + SOC2:CC7.2: security-ops + ISO:A.5.17: identity-team new file mode 100644 @@ -0,0 +1,31 @@ +[build-system] +requires = ["setuptools>=68"] +build-backend = "setuptools.build_meta" + +[project] +name = "control-coverage" +version = "0.1.0" +description = "Control-first coverage and blind-spot analysis over an evidence corpus, with a Statement of Applicability." +readme = "README.md" +requires-python = ">=3.10" +license = { text = "GPL-3.0-or-later" } +authors = [{ name = "Audit Labs" }] +dependencies = ["PyYAML>=6.0"] + +[project.optional-dependencies] +dev = ["pytest>=8.0", "ruff>=0.5"] + +[project.scripts] +control-coverage = "control_coverage.cli:main" + +[project.urls] +Homepage = "https://audit-labs.dev" +Repository = "https://github.com/audit-labs/control-coverage" + +[tool.setuptools] +packages = ["control_coverage", "control_coverage.reporters"] +include-package-data = true + +[tool.setuptools.package-data] +# Ship the bundled framework catalogs inside the wheel so the CLI works after install. +"control_coverage" = ["catalogs/*.yaml"] new file mode 100644 @@ -0,0 +1,3 @@ +PyYAML>=6.0 +pytest>=8.0 +ruff>=0.5 new file mode 100644 @@ -0,0 +1 @@ +PyYAML>=6.0 new file mode 100644 @@ -0,0 +1,14 @@ +# Ruff configuration for audit-tools. +# +# A few lint rules are disabled because they flag patterns this project uses +# deliberately: +# +# BLE001 - The audit collectors and their CLI wrappers intentionally catch +# broad exceptions so that one failing check never aborts a whole +# audit run. The error is reported and collection continues. +# DTZ011 - date.today() is used to build human-facing, date-stamped output +# folder names, where the local date is the intended value. +# S112 - try/except/continue is used to skip resources that are unavailable +# during collection (e.g. a repo without the requested branch). +[lint] +ignore = ["BLE001", "DTZ011", "S112"] new file mode 100644 @@ -0,0 +1,46 @@ +{ + "subject": "acme", + "platform": "aws", + "source_package": "aws_audit_acme_2026-01-01", + "generated_at": "2026-01-01 00:00:00 UTC", + "summary": {"pass": 2, "fail": 1, "not_applicable": 0}, + "coverage": ["SOC2:CC6.1", "SOC2:CC6.6", "SOC2:CC7.2", "ISO:A.8.15"], + "findings": [ + { + "id": "aws.iam.root-mfa", + "title": "Root account has MFA enabled", + "status": "pass", + "severity": "high", + "controls": ["SOC2:CC6.1", "ISO:A.5.17", "NIST:IA-2"], + "reason": "1 row asserted true", + "evidence": [] + }, + { + "id": "aws.ec2.no-open-sg", + "title": "No security group open to 0.0.0.0/0 on admin ports", + "status": "fail", + "severity": "high", + "controls": ["SOC2:CC6.6", "ISO:A.8.20", "NIST:SC-7"], + "reason": "2 rows failed the check", + "evidence": [{"group_id": "sg-1", "port": "22"}, {"group_id": "sg-2", "port": "3389"}] + }, + { + "id": "aws.cloudtrail.enabled", + "title": "CloudTrail logging is enabled in all regions", + "status": "pass", + "severity": "high", + "controls": ["SOC2:CC7.2", "ISO:A.8.15", "NIST:AU-2"], + "reason": "1 row asserted true", + "evidence": [] + }, + { + "id": "aws.legacy.old-code", + "title": "Legacy control citing an unknown code", + "status": "pass", + "severity": "low", + "controls": ["SOC2:CC6.99"], + "reason": "for orphan-code testing", + "evidence": [] + } + ] +} new file mode 100644 @@ -0,0 +1,46 @@ +{ + "subject": "acme", + "platform": "github", + "source_package": "github_audit_acme_2025-10-01", + "generated_at": "2025-10-01 00:00:00 UTC", + "summary": {"pass": 2, "fail": 2, "not_applicable": 0}, + "coverage": ["SOC2:CC6.1", "SOC2:CC6.3", "SOC2:CC7.1", "SOC2:CC9.2"], + "findings": [ + { + "id": "github.org.require-2fa", + "title": "Organization requires two-factor authentication", + "status": "fail", + "severity": "high", + "controls": ["SOC2:CC6.1", "ISO:A.5.17", "NIST:IA-2"], + "reason": "2fa not enforced at the time of this snapshot", + "evidence": [{"two_factor_required": "false"}] + }, + { + "id": "github.org.default-permission", + "title": "Base repository permission is read or less", + "status": "fail", + "severity": "medium", + "controls": ["SOC2:CC6.3", "ISO:A.5.15", "NIST:AC-6"], + "reason": "base permission is write", + "evidence": [{"default_repo_permission": "write"}] + }, + { + "id": "github.org.secret-scanning", + "title": "Secret scanning push protection is on for new repos", + "status": "pass", + "severity": "medium", + "controls": ["SOC2:CC7.1", "ISO:A.5.17", "NIST:CM-6"], + "reason": "1 row asserted true", + "evidence": [] + }, + { + "id": "github.org.vendor-review", + "title": "Third-party OAuth app access is restricted", + "status": "pass", + "severity": "medium", + "controls": ["SOC2:CC9.2"], + "reason": "1 row asserted true", + "evidence": [] + } + ] +} new file mode 100644 @@ -0,0 +1,46 @@ +{ + "subject": "acme", + "platform": "github", + "source_package": "github_audit_acme_2026-01-01", + "generated_at": "2026-01-01 00:00:00 UTC", + "summary": {"pass": 2, "fail": 1, "not_applicable": 1}, + "coverage": ["SOC2:CC6.1", "SOC2:CC6.3", "SOC2:CC7.1", "ISO:A.5.17"], + "findings": [ + { + "id": "github.org.require-2fa", + "title": "Organization requires two-factor authentication", + "status": "pass", + "severity": "high", + "controls": ["SOC2:CC6.1", "ISO:A.5.17", "NIST:IA-2"], + "reason": "1 row asserted true", + "evidence": [] + }, + { + "id": "github.org.default-permission", + "title": "Base repository permission is read or less", + "status": "fail", + "severity": "medium", + "controls": ["SOC2:CC6.3", "ISO:A.5.15", "NIST:AC-6"], + "reason": "base permission is write", + "evidence": [{"default_repo_permission": "write"}] + }, + { + "id": "github.org.secret-scanning", + "title": "Secret scanning push protection is on for new repos", + "status": "pass", + "severity": "medium", + "controls": ["SOC2:CC7.1", "ISO:A.5.17", "NIST:CM-6"], + "reason": "1 row asserted true", + "evidence": [] + }, + { + "id": "github.branch.require-reviews", + "title": "Default branch requires pull request reviews", + "status": "not_applicable", + "severity": "high", + "controls": ["SOC2:CC8.1", "ISO:A.8.32", "NIST:CM-3"], + "reason": "table 'branch_protections' not in package", + "evidence": [] + } + ] +} new file mode 100644 @@ -0,0 +1,11 @@ +# Example scope / Statement of Applicability for the test corpus. +subject: Acme Production +frameworks: [SOC2, ISO] +exclusions: + - control: ISO:A.7.1 + reason: "Fully cloud-hosted; no physical premises are in scope for the ISMS." + - control: ISO:A.5.7 + reason: "No formal threat-intelligence program; risk accepted by the CISO for 2026." +owners: + SOC2:CC6.1: platform-team + ISO:A.5.17: identity-team new file mode 100644 @@ -0,0 +1,74 @@ +"""Tests for loading framework catalogs.""" + +import pytest + +from control_coverage import catalog + + +def test_soc2_is_complete_common_criteria(): + cat = catalog.load("soc2") + assert cat.framework == "SOC2" + assert cat.complete + ids = {c.id for c in cat.controls} + # A representative spread across every Common Criteria group. + for cc in ["CC1.1", "CC5.3", "CC6.8", "CC7.5", "CC8.1", "CC9.2"]: + assert cc in ids + + +def test_soc2_has_all_five_tsc_categories(): + cat = catalog.load("soc2") + families = {c.family for c in cat.controls} + assert {"Availability", "Confidentiality", "Processing Integrity", "Privacy"} <= families + ids = {c.id for c in cat.controls} + for cid in ["C1.1", "PI1.5", "P6.7", "P8.1"]: + assert cid in ids + assert len(cat.controls) == 61 + + +def test_iso_has_all_93_annex_a_controls(): + cat = catalog.load("iso") + assert cat.framework == "ISO" + assert cat.complete + assert len(cat.controls) == 93 + + +def test_nist_is_the_moderate_baseline(): + cat = catalog.load("nist") + assert cat.complete # complete relative to the moderate baseline + assert "Moderate" in cat.name + assert len(cat.controls) > 150 + ids = {c.id for c in cat.controls} + # A spread across families the ecosystem's rulesets cite and beyond. + for cid in ["AC-6", "AU-2", "CM-6", "IA-2", "SC-7", "SI-2", "SR-3"]: + assert cid in ids + + +def test_control_code_joins_framework_and_id(): + cat = catalog.load("soc2") + ctrl = next(c for c in cat.controls if c.id == "CC6.1") + assert ctrl.code == "SOC2:CC6.1" + + +def test_titles_with_commas_survive_parsing(): + cat = catalog.load("soc2") + ctrl = next(c for c in cat.controls if c.id == "CC1.3") + assert "reporting lines" in ctrl.title # flow-scalar comma bug regression + + +def test_aliases_resolve(): + assert catalog.load("iso 27001").framework == "ISO" + assert catalog.load("800-53").framework == "NIST" + + +def test_unknown_framework_raises(): + with pytest.raises(ValueError, match="unknown framework"): + catalog.load("hipaa") + + +def test_load_frameworks_dedupes_and_sorts(): + cats = catalog.load_frameworks(["NIST", "SOC2", "soc 2"]) + assert [c.framework for c in cats] == ["NIST", "SOC2"] + + +def test_available_lists_bundled(): + assert set(catalog.available()) == {"SOC2", "ISO", "NIST"} new file mode 100644 @@ -0,0 +1,117 @@ +"""Tests for the command-line interface.""" + +from pathlib import Path + +import pytest + +from control_coverage import cli + +FIXTURES = Path(__file__).parent / "fixtures" +GITHUB = str(FIXTURES / "github_audit_acme_2026-01-01.json") +AWS = str(FIXTURES / "aws_audit_acme_2026-01-01.json") +SCOPE = str(FIXTURES / "scope.yaml") + + +def test_stdout_markdown_default(capsys): + rc = cli.main([GITHUB, AWS, "--framework", "SOC2"]) + out = capsys.readouterr().out + assert rc == 0 + assert "# Control Coverage" in out + assert "Coverage" in out + + +def test_frameworks_inferred_from_corpus(capsys): + cli.main([GITHUB, "--format", "json"]) + out = capsys.readouterr().out + # github fixture cites SOC2, ISO, NIST codes -> all three inferred. + for fw in ("SOC2", "ISO", "NIST"): + assert f'"framework": "{fw}"' in out + + +def test_scope_file_supplies_frameworks_and_subject(capsys): + cli.main([GITHUB, AWS, "--scope", SCOPE, "--format", "json"]) + out = capsys.readouterr().out + assert '"subject": "Acme Production"' in out + assert '"framework": "NIST"' not in out # scope lists only SOC2, ISO + + +def test_blind_spots_mode(capsys): + rc = cli.main([GITHUB, "--framework", "SOC2", "--blind-spots"]) + out = capsys.readouterr().out + assert rc == 0 + assert "unaddressed" in out + assert "SOC2:CC1.1" in out + + +def test_fail_under_gate_trips(capsys): + rc = cli.main([GITHUB, "--framework", "SOC2", "--fail-under", "90"]) + assert rc == 1 + err = capsys.readouterr().err + assert "coverage gate" in err + + +def test_fail_under_gate_passes(capsys): + rc = cli.main([GITHUB, "--framework", "SOC2", "--fail-under", "1"]) + assert rc == 0 + + +def test_out_dir_writes_files(tmp_path, capsys): + rc = cli.main( + [GITHUB, "--scope", SCOPE, "--format", "md,html,json,soa", "--out", str(tmp_path)] + ) + assert rc == 0 + written = {p.name for p in tmp_path.iterdir()} + assert "soa.md" in written + assert any(n.endswith(".html") for n in written) + assert any(n.endswith(".json") for n in written) + + +def test_missing_reports_errors(): + with pytest.raises(SystemExit): + cli.main([str(FIXTURES / "nope.json"), "--framework", "SOC2"]) + + +BASELINE = str(FIXTURES / "baseline_github.json") + + +def test_trend_mode_markdown(capsys): + rc = cli.main([GITHUB, AWS, "--framework", "SOC2", "--baseline", BASELINE]) + out = capsys.readouterr().out + assert rc == 0 + assert "# Coverage Trend" in out + + +def test_trend_html_output(tmp_path): + cli.main([GITHUB, AWS, "--framework", "SOC2", "--baseline", BASELINE, + "--format", "html,json", "--out", str(tmp_path)]) + names = {p.name for p in tmp_path.iterdir()} + assert "trend.html" in names and "trend.json" in names + + +def test_crosswalk_mode(capsys): + rc = cli.main([GITHUB, AWS, "--framework", "SOC2,ISO", "--crosswalk"]) + out = capsys.readouterr().out + assert rc == 0 + assert "Minimal evidence set" in out + + +def test_crosswalk_and_baseline_conflict(): + with pytest.raises(SystemExit, match="cannot be combined"): + cli.main([GITHUB, "--crosswalk", "--baseline", BASELINE]) + + +def test_trend_rejects_soa_format(): + with pytest.raises(SystemExit, match="trend mode supports"): + cli.main([GITHUB, "--framework", "SOC2", "--baseline", BASELINE, "--format", "soa"]) + + +def test_family_exclusion_via_cli(tmp_path, capsys): + scope_file = tmp_path / "scope.yaml" + scope_file.write_text( + "frameworks: [SOC2]\n" + "exclude_families:\n" + " - {framework: SOC2, family: Privacy, reason: 'Not in scope.'}\n" + ) + cli.main([GITHUB, "--scope", str(scope_file), "--format", "json"]) + out = capsys.readouterr().out + assert '"state": "out_of_scope"' in out new file mode 100644 @@ -0,0 +1,42 @@ +"""Tests for loading an evidence corpus from audit-report JSON.""" + +from pathlib import Path + +import pytest + +from control_coverage import corpus + +FIXTURES = Path(__file__).parent / "fixtures" +GITHUB = FIXTURES / "github_audit_acme_2026-01-01.json" +AWS = FIXTURES / "aws_audit_acme_2026-01-01.json" + + +def test_one_observation_per_finding_control_pair(): + obs = corpus.load_report(GITHUB) + # 4 findings with 3+3+3+3 controls = 12 observations. + assert len(obs) == 12 + + +def test_observation_carries_provenance(): + obs = corpus.load_report(GITHUB) + o = next(o for o in obs if o.control == "SOC2:CC6.3") + assert o.status == corpus.FAIL + assert o.rule_id == "github.org.default-permission" + assert o.source == "github_audit_acme_2026-01-01" + + +def test_load_corpus_flattens_multiple_reports(): + obs = corpus.load_corpus([GITHUB, AWS]) + sources = {o.source for o in obs} + assert sources == {"github_audit_acme_2026-01-01", "aws_audit_acme_2026-01-01"} + + +def test_directory_is_expanded_to_json_files(): + obs = corpus.load_corpus([FIXTURES]) + assert len(obs) > 0 + assert any(o.source.startswith("aws_") for o in obs) + + +def test_empty_directory_raises(tmp_path): + with pytest.raises(ValueError, match="no audit-report JSON"): + corpus.load_corpus([tmp_path]) new file mode 100644 @@ -0,0 +1,91 @@ +"""Tests for the coverage engine — the heart of the tool.""" + +from pathlib import Path + +from control_coverage import catalog, corpus, scope +from control_coverage.coverage import ( + ASSERTED, + FAILING, + OUT_OF_SCOPE, + SUPPORTED, + UNADDRESSED, + evaluate, +) + +FIXTURES = Path(__file__).parent / "fixtures" +GITHUB = FIXTURES / "github_audit_acme_2026-01-01.json" +AWS = FIXTURES / "aws_audit_acme_2026-01-01.json" + + +def _state(fc, control_id): + return next(r.state for r in fc.results if r.control.id == control_id) + + +def _report(frameworks, scp=None): + obs = corpus.load_corpus([GITHUB, AWS]) + cats = catalog.load_frameworks(frameworks) + return evaluate(cats, obs, scope=scp) + + +def test_worst_wins_and_blind_spots_for_soc2(): + fc = _report(["SOC2"]).frameworks[0] + assert _state(fc, "CC6.1") == SUPPORTED # two passes across github + aws + assert _state(fc, "CC6.3") == FAILING # one fail + assert _state(fc, "CC6.6") == FAILING + assert _state(fc, "CC8.1") == ASSERTED # only not_applicable observations + assert _state(fc, "CC1.1") == UNADDRESSED # nothing maps here + + +def test_soc2_rollup_numbers(): + fc = _report(["SOC2"]).frameworks[0] + assert fc.in_scope == 61 # full five-category Trust Services Criteria + assert fc.supported == 3 + assert fc.counts[FAILING] == 2 + assert fc.counts[ASSERTED] == 1 + assert fc.addressed == 6 + assert len(fc.blind_spots) == 55 + assert fc.coverage_pct == 9.8 # 6 / 61 + assert fc.assured_pct == 4.9 # 3 / 61 + + +def test_scope_marks_controls_out_of_scope_with_reason(): + scp = scope.load(FIXTURES / "scope.yaml") + fc = next(f for f in _report(["ISO"], scp).frameworks if f.catalog.framework == "ISO") + assert _state(fc, "A.7.1") == OUT_OF_SCOPE + assert _state(fc, "A.5.7") == OUT_OF_SCOPE + assert fc.in_scope == 91 # 93 Annex A controls minus 2 exclusions + excluded = next(r for r in fc.results if r.control.id == "A.7.1") + assert "cloud-hosted" in excluded.exclusion_reason + + +def test_orphan_only_flags_loaded_frameworks(): + # CC6.99 is a SOC2 typo; NIST codes are cited but NIST is not loaded here. + report = _report(["SOC2", "ISO"]) + assert "SOC2:CC6.99" in report.orphan_codes + assert not any(c.startswith("NIST:") for c in report.orphan_codes) + + +def test_family_exclusion_marks_whole_category_out_of_scope(): + from control_coverage.scope import Scope + + scp = Scope(family_exclusions={("SOC2", "Privacy"): "Not in the SOC 2 audit scope."}) + fc = _report(["SOC2"], scp).frameworks[0] + privacy = [r for r in fc.results if r.control.family == "Privacy"] + assert privacy # the catalog has Privacy controls + assert all(r.state == OUT_OF_SCOPE for r in privacy) + assert all("audit scope" in r.exclusion_reason for r in privacy) + # Security (Common Criteria) controls remain in scope. + cc61 = next(r for r in fc.results if r.control.id == "CC6.1") + assert cc61.state != OUT_OF_SCOPE + + +def test_owner_is_attached_from_scope(): + scp = scope.load(FIXTURES / "scope.yaml") + fc = _report(["SOC2"], scp).frameworks[0] + owner = next(r.owner for r in fc.results if r.control.id == "CC6.1") + assert owner == "platform-team" + + +def test_source_count_reflects_distinct_packages(): + report = _report(["SOC2"]) + assert report.source_count == 2 new file mode 100644 @@ -0,0 +1,67 @@ +"""Tests for the evidence crosswalk and minimal-evidence set.""" + +import json +from pathlib import Path + +from control_coverage import catalog, corpus, crosswalk +from control_coverage.coverage import evaluate + +FIXTURES = Path(__file__).parent / "fixtures" +GITHUB = FIXTURES / "github_audit_acme_2026-01-01.json" +AWS = FIXTURES / "aws_audit_acme_2026-01-01.json" + + +def _crosswalk(frameworks): + obs = corpus.load_corpus([GITHUB, AWS]) + cats = catalog.load_frameworks(frameworks) + return crosswalk.build(evaluate(cats, obs)) + + +def test_item_spans_multiple_frameworks(): + xw = _crosswalk(["SOC2", "ISO", "NIST"]) + twofa = next(i for i in xw.items if i.rule_id == "github.org.require-2fa") + # 2FA maps to SOC2:CC6.1, ISO:A.5.17, NIST:IA-2. + assert set(twofa.frameworks) == {"SOC2", "ISO", "NIST"} + assert "SOC2:CC6.1" in twofa.controls + + +def test_items_sorted_by_leverage(): + xw = _crosswalk(["SOC2", "ISO", "NIST"]) + counts = [i.count for i in xw.items] + assert counts == sorted(counts, reverse=True) + + +def test_minimal_cover_reaches_full_universe(): + xw = _crosswalk(["SOC2", "ISO", "NIST"]) + assert xw.cover # non-empty + assert xw.cover[-1].cumulative == xw.universe_size + assert xw.cover[-1].cumulative_pct == 100.0 + + +def test_cover_is_monotonic_and_no_wasted_picks(): + xw = _crosswalk(["SOC2"]) + cumulative = [s.cumulative for s in xw.cover] + assert cumulative == sorted(cumulative) + assert all(s.new_controls > 0 for s in xw.cover) # greedy never picks a no-op + + +def test_markdown_has_both_sections(): + md = crosswalk.render_markdown(_crosswalk(["SOC2", "ISO"])) + assert "## Minimal evidence set" in md + assert "## Evidence leverage" in md + assert "github.org.require-2fa" in md + + +def test_json_structure(): + doc = json.loads(crosswalk.render_json(_crosswalk(["SOC2", "ISO"]))) + assert doc["universe_size"] > 0 + assert "minimal_evidence_set" in doc + assert all("controls" in e for e in doc["evidence"]) + + +def test_html_is_self_contained(): + html = crosswalk.render_html(_crosswalk(["SOC2", "ISO", "NIST"])) + assert html.startswith("<!doctype html>") + assert "<style>" in html + assert "http://" not in html and "https://" not in html + assert "github.org.require-2fa" in html new file mode 100644 @@ -0,0 +1,55 @@ +"""Tests for the Markdown, HTML, JSON, and SoA renderers.""" + +import json as _json +from pathlib import Path + +from control_coverage import catalog, corpus, reporters, scope +from control_coverage.coverage import evaluate + +FIXTURES = Path(__file__).parent / "fixtures" + + +def _report(): + obs = corpus.load_corpus([FIXTURES / "github_audit_acme_2026-01-01.json"]) + cats = catalog.load_frameworks(["SOC2"]) + scp = scope.load(FIXTURES / "scope.yaml") + return evaluate(cats, obs, scope=scp, subject="Acme", generated_at="2026-01-01") + + +def test_markdown_has_summary_and_blind_spots(): + md = reporters.render(_report(), "md") + assert "# Control Coverage — Acme" in md + assert "## Summary" in md + assert "## Blind spots" in md + assert "CC1.1" in md # a blind spot is listed + + +def test_json_is_valid_and_structured(): + doc = _json.loads(reporters.render(_report(), "json")) + assert doc["subject"] == "Acme" + soc2 = doc["frameworks"][0] + assert soc2["framework"] == "SOC2" + assert soc2["coverage_pct"] >= 0 + states = {c["state"] for c in soc2["controls"]} + assert "unaddressed" in states + + +def test_html_is_self_contained(): + html = reporters.render(_report(), "html") + assert html.startswith("<!doctype html>") + assert "<style>" in html + assert "http://" not in html and "https://" not in html # no external assets + + +def test_soa_lists_applicability_and_status(): + soa = reporters.render(_report(), "soa") + assert "Statement of Applicability" in soa + assert "Applicable" in soa + assert "Implemented" in soa or "Not evidenced" in soa + + +def test_unknown_format_raises(): + import pytest + + with pytest.raises(ValueError, match="unknown format"): + reporters.render(_report(), "pdf") new file mode 100644 @@ -0,0 +1,65 @@ +"""Tests for scope / Statement of Applicability parsing.""" + +from pathlib import Path + +import pytest + +from control_coverage import scope + +FIXTURES = Path(__file__).parent / "fixtures" + + +def test_loads_exclusions_and_owners(): + scp = scope.load(FIXTURES / "scope.yaml") + assert scp.subject == "Acme Production" + assert scp.frameworks == ["SOC2", "ISO"] + assert scp.excluded("ISO:A.7.1") + assert "cloud-hosted" in scp.reason("ISO:A.7.1") + assert scp.owner("SOC2:CC6.1") == "platform-team" + + +def test_exclusion_without_reason_is_rejected(tmp_path): + bad = tmp_path / "bad.yaml" + bad.write_text("exclusions:\n - control: ISO:A.5.7\n") + with pytest.raises(ValueError, match="needs a non-empty 'reason'"): + scope.load(bad) + + +def test_exclusion_missing_control_key_is_rejected(tmp_path): + bad = tmp_path / "bad.yaml" + bad.write_text("exclusions:\n - reason: no control key\n") + with pytest.raises(ValueError, match="must be a mapping with a 'control' key"): + scope.load(bad) + + +def test_family_exclusion_parses(tmp_path): + f = tmp_path / "s.yaml" + f.write_text( + "exclude_families:\n" + " - {framework: SOC2, family: Privacy, reason: 'Not in the audit scope.'}\n" + ) + scp = scope.load(f) + assert scp.family_excluded("SOC2", "Privacy") + assert "audit scope" in scp.family_reason("SOC2", "Privacy") + assert not scp.family_excluded("SOC2", "Availability") + + +def test_family_exclusion_needs_reason(tmp_path): + f = tmp_path / "s.yaml" + f.write_text("exclude_families:\n - {framework: SOC2, family: Privacy}\n") + with pytest.raises(ValueError, match="needs a non-empty 'reason'"): + scope.load(f) + + +def test_family_exclusion_needs_framework_and_family(tmp_path): + f = tmp_path / "s.yaml" + f.write_text("exclude_families:\n - {family: Privacy, reason: x}\n") + with pytest.raises(ValueError, match="'framework' and 'family'"): + scope.load(f) + + +def test_empty_scope_excludes_nothing(): + scp = scope.empty() + assert not scp.excluded("ISO:A.7.1") + assert not scp.family_excluded("SOC2", "Privacy") + assert scp.frameworks == [] new file mode 100644 @@ -0,0 +1,75 @@ +"""Tests for coverage trend (diffing two corpora).""" + +from pathlib import Path + +from control_coverage import catalog, corpus, trend +from control_coverage.coverage import evaluate + +FIXTURES = Path(__file__).parent / "fixtures" +GITHUB = FIXTURES / "github_audit_acme_2026-01-01.json" +AWS = FIXTURES / "aws_audit_acme_2026-01-01.json" +BASELINE = FIXTURES / "baseline_github.json" + + +def _cov(paths): + obs = corpus.load_corpus(paths) + cats = catalog.load_frameworks(["SOC2"]) + return evaluate(cats, obs) + + +def _compare(): + return trend.compare(_cov([BASELINE]), _cov([GITHUB, AWS])) + + +def _delta(fc, control_id): + return next(d for d in fc.deltas if d.id == control_id) + + +def test_categories_reflect_state_movement(): + fc = _compare().frameworks[0] + assert _delta(fc, "CC6.1").category == trend.IMPROVED # failing -> supported + assert _delta(fc, "CC6.3").category == trend.UNCHANGED # failing -> failing + assert _delta(fc, "CC6.6").category == trend.GAINED # unaddressed -> failing + assert _delta(fc, "CC7.2").category == trend.GAINED # unaddressed -> supported + assert _delta(fc, "CC9.2").category == trend.LOST # supported -> unaddressed + + +def test_counts_and_coverage_delta(): + fc = _compare().frameworks[0] + c = fc.counts + assert c[trend.IMPROVED] == 1 + assert c[trend.GAINED] == 3 # CC6.6, CC7.2, CC8.1 + assert c[trend.LOST] == 1 + assert c[trend.REGRESSED] == 0 + assert fc.coverage_delta == round(fc.new_coverage_pct - fc.old_coverage_pct, 1) + assert fc.new_coverage_pct > fc.old_coverage_pct + + +def test_regressions_count_lost_and_regressed(): + report = _compare() + assert report.total_regressions == 1 # the single LOST control + + +def test_markdown_lists_changes_only(): + md = trend.render_markdown(_compare()) + assert "# Coverage Trend" in md + assert "CC9.2" in md # a lost control appears + assert "CC1.1" not in md # an unchanged blind spot does not + + +def test_json_omits_unchanged(): + import json + + doc = json.loads(trend.render_json(_compare())) + changes = doc["frameworks"][0]["changes"] + ids = {c["id"] for c in changes} + assert "CC9.2" in ids + assert "CC1.1" not in ids + + +def test_html_is_self_contained(): + html = trend.render_html(_compare()) + assert html.startswith("<!doctype html>") + assert "<style>" in html + assert "http://" not in html and "https://" not in html + assert "CC9.2" in html # a changed control shows up