audit-labs/audit-report
Turn audit-tools evidence packages into control-mapped, auditor-ready reports.
clone: git clone https://gitbay.org/audit-labs/audit-report.git
f3c12544e8bef2bf45f4aa68ce0dec2c44e531ca
verified · cmc
author: Christian Cleberg <hello@cleberg.net> · 2026-08-07T02:13:24Z
.github/workflows/ci.yml | 26 +++++++++++++++++++++++++ MAPPING.md | 40 ++++++++++++++++++++++++++++++++++++++ audit_report/cli.py | 2 +- audit_report/reporters/__init__.py | 25 ++++++++++++++++++++++-- audit_report/reporters/html.py | 13 ++++++++++++- audit_report/reporters/json.py | 2 ++ audit_report/reporters/markdown.py | 9 +++++++++ audit_report/rules.py | 22 ++++++++++++++++++--- audit_report/rulesets/aws.yaml | 5 +++++ audit_report/rulesets/github.yaml | 5 +++++ audit_report/rulesets/gitlab.yaml | 5 +++++ tests/test_reporters.py | 23 +++++++++++++++++++--- 12 files changed, 167 insertions(+), 10 deletions(-) new file mode 100644 @@ -0,0 +1,26 @@ +name: CI + +on: + push: + pull_request: + +jobs: + test: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ["3.10", "3.12"] + steps: + - uses: actions/checkout@v5 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v6 + with: + python-version: ${{ matrix.python-version }} + - name: Install + run: | + python -m pip install --upgrade pip + pip install -e ".[dev]" + - name: Ruff + run: ruff check . + - name: Tests + run: pytest -q new file mode 100644 @@ -0,0 +1,40 @@ +# Rulesets — provenance and control-mapping rationale + +An `audit-report` **ruleset** (`audit_report/rulesets/*.yaml`) turns collected +evidence into control-relevant findings: each rule names a table, a check, and the +control identifiers the signal is offered as evidence *for*. This document records +where those mappings come from and their limits. + +## What a mapping claims — and does not + +- A rule's `controls` list says: "this signal is relevant to these controls." +- A **fail** means a setting is in a state that does **not** support the control. + It is not a compliance verdict — the auditor still owns the conclusion. +- The control identifiers (`SOC2:CC6.1`, `ISO:A.5.17`, `NIST:IA-2`, …) are + reproduced; the frameworks' normative control text is not. See the catalog + provenance in [control-coverage `MAPPING.md`](https://github.com/audit-labs/control-coverage/blob/main/MAPPING.md). +- The mappings are the **maintainers' interpretation**, not reviewed or endorsed + by the AICPA, ISO/IEC, or NIST. + +## Framework revisions referenced + +- **SOC 2** — Trust Services Criteria 2017 (2022 revised points of focus). +- **ISO/IEC 27001** — 27001:2022 Annex A. +- **NIST SP 800-53** — Rev. 5. + +## Versioning and traceability + +- Each ruleset carries `name` and `version` fields. +- Every report stamps the tool name + version and the ruleset `name`, `version`, + and a **SHA-256 of the ruleset file** into its output (`tool` / `ruleset` in + JSON; the header line in Markdown/HTML). +- An auditor can therefore tie any finding back to the exact ruleset that produced + it, and re-perform against it. Bump `version` on any change to a rule's + controls, checks, or thresholds. + +## Authorship and review + +- **Author:** the audit-labs maintainer. +- **Review status:** maintainer self-review; no independent professional review. + Validate a ruleset against your own control set before relying on it. +- **Effective date:** 2026-08. @@ -181,7 +181,7 @@ def main(argv: list[str] | None = None) -> int: return _run_diff(args, package, ruleset, formats) findings = evaluate(package, ruleset) - report = reporters.build_report(package, findings) + report = reporters.build_report(package, findings, ruleset) _emit(lambda fmt: reporters.render(report, fmt), formats, args.out, "report") counts = report.counts @@ -5,8 +5,10 @@ from __future__ import annotations from dataclasses import dataclass from datetime import datetime, timezone +from .. import __version__ from ..engine import Finding, control_coverage, summarize from ..loader import Package +from ..rules import Ruleset from . import html as _html from . import json as _json from . import markdown as _markdown @@ -19,6 +21,7 @@ class Report: package: Package findings: list[Finding] generated_at: str + ruleset: Ruleset | None = None @property def counts(self) -> dict[str, int]: @@ -28,11 +31,29 @@ class Report: def coverage(self) -> dict[str, dict]: return control_coverage(self.findings) + @property + def provenance(self) -> dict[str, dict]: + """Tool + ruleset identity, so a report can be tied to what produced it.""" + rs = self.ruleset + return { + "tool": {"name": "audit-report", "version": __version__}, + "ruleset": { + "name": rs.name if rs else "", + "platform": rs.platform if rs else "", + "version": rs.version if rs else "", + "sha256": rs.sha256 if rs else "", + }, + } + -def build_report(package: Package, findings: list[Finding]) -> Report: +def build_report( + package: Package, findings: list[Finding], ruleset: Ruleset | None = None +) -> Report: """Assemble a :class:`Report` with a UTC generation timestamp.""" stamp = datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M UTC") - return Report(package=package, findings=findings, generated_at=stamp) + return Report( + package=package, findings=findings, generated_at=stamp, ruleset=ruleset + ) RENDERERS = { @@ -90,10 +90,21 @@ def render(report: Report) -> str: parts: list[str] = [] parts.append(f"<h1>Evidence Report — {escape(pkg.subject)}</h1>") + prov = report.provenance + tool, rs = prov["tool"], prov["ruleset"] + ruleset_meta = "" + if rs["sha256"]: + rs_ver = f" {escape(rs['version'])}" if rs["version"] else "" + ruleset_meta = ( + f" · Ruleset <code>{escape(rs['name'])}{rs_ver}</code> " + f"<code>sha256:{escape(rs['sha256'][:12])}</code>" + ) parts.append( f"<p class='meta'>Platform <code>{escape(pkg.platform)}</code> · " f"Source <code>{escape(pkg.path.name)}</code> · " - f"Generated {escape(report.generated_at)}</p>" + f"Generated {escape(report.generated_at)} · " + f"Tool <code>{escape(tool['name'])} {escape(tool['version'])}</code>" + f"{ruleset_meta}</p>" ) parts.append( "<p class='summary-pills'>" @@ -21,6 +21,8 @@ def to_dict(report: Report) -> dict: "platform": pkg.platform, "source_package": pkg.path.name, "generated_at": report.generated_at, + "tool": report.provenance["tool"], + "ruleset": report.provenance["ruleset"], "summary": report.counts, "coverage": report.coverage, "findings": [ @@ -42,6 +42,15 @@ def render(report: Report) -> str: out.append(f"- **Subject:** {pkg.subject}") out.append(f"- **Source package:** `{pkg.path.name}`") out.append(f"- **Generated:** {report.generated_at}") + prov = report.provenance + tool, rs = prov["tool"], prov["ruleset"] + out.append(f"- **Tool:** {tool['name']} {tool['version']}") + if rs["sha256"]: + rs_ver = f" {rs['version']}" if rs["version"] else "" + out.append( + f"- **Ruleset:** {rs['name']}{rs_ver} " + f"(`sha256:{rs['sha256'][:12]}`)" + ) out.append( f"- **Result:** {counts[FAIL]} failing · {counts[PASS]} passing · " f"{counts[NOT_APPLICABLE]} not applicable" @@ -17,6 +17,7 @@ Operators (``op``): ``equals``, ``not_equals``, ``is_true``, ``is_false``, from __future__ import annotations +import hashlib from dataclasses import dataclass, field from pathlib import Path @@ -49,10 +50,18 @@ class Rule: @dataclass class Ruleset: - """A named collection of rules for one platform.""" + """A named collection of rules for one platform. + + ``version`` and ``sha256`` identify *which* ruleset produced a report, so an + auditor can re-perform against the exact mapping used. ``sha256`` is the + digest of the ruleset file's bytes as loaded. + """ platform: str rules: list[Rule] + name: str = "" + version: str = "" + sha256: str = "" def _as_number(value: str) -> float | None: @@ -115,7 +124,8 @@ def match(condition: dict, row: dict[str, str]) -> bool: def load_ruleset(path: str | Path) -> Ruleset: """Parse a ruleset YAML file into a :class:`Ruleset`, validating each rule.""" - data = yaml.safe_load(Path(path).read_text(encoding="utf-8")) or {} + text = Path(path).read_text(encoding="utf-8") + data = yaml.safe_load(text) or {} platform = data.get("platform") if not platform: raise ValueError(f"{path}: ruleset is missing a 'platform'") @@ -137,4 +147,10 @@ def load_ruleset(path: str | Path) -> Ruleset: raise ValueError(f"{rule.id}: unknown check type {check_type!r}") rules.append(rule) - return Ruleset(platform=platform, rules=rules) + return Ruleset( + platform=platform, + rules=rules, + name=data.get("name", platform), + version=str(data.get("version", "")), + sha256=hashlib.sha256(text.encode("utf-8")).hexdigest(), + ) @@ -4,6 +4,11 @@ # (aws_audit_<profile>_<date>/). Each rule names a CSV table, a check, and the # controls the signal is offered as evidence for. A "fail" means a setting is in # a state that does NOT support the control — an auditor still owns the verdict. +# +# Provenance and control-mapping rationale: see MAPPING.md. Bump `version` on any +# change to a rule's controls, checks, or thresholds so reports stay traceable. +name: AWS ITGC ruleset +version: "2026.08.0" platform: aws rules: @@ -3,6 +3,11 @@ # Evaluated against an audit-tools GitHub evidence package # (github_audit_<org>_<date>/). Complements gh-attest: same spirit of mapping # GitHub signals to controls, applied offline to a captured CSV package. +# +# Provenance and control-mapping rationale: see MAPPING.md. Bump `version` on any +# change to a rule's controls, checks, or thresholds so reports stay traceable. +name: GitHub ITGC ruleset +version: "2026.08.0" platform: github rules: @@ -10,6 +10,11 @@ # * audit-tools omits a CSV entirely when a collector returns no rows, so an # absent table reports as "not applicable", not "pass". A rule can only # speak to data that was actually collected. +# +# Provenance and control-mapping rationale: see MAPPING.md. Bump `version` on any +# change to a rule's controls, checks, or thresholds so reports stay traceable. +name: GitLab ITGC ruleset +version: "2026.08.0" platform: gitlab rules: @@ -5,7 +5,7 @@ from pathlib import Path import pytest -from audit_report import reporters +from audit_report import __version__, reporters from audit_report.cli import main from audit_report.engine import evaluate from audit_report.loader import load_package @@ -17,8 +17,9 @@ RULESETS = Path("audit_report/rulesets") def _report(pkg_name="aws_audit_acme_2026-01-01", ruleset="aws.yaml"): pkg = load_package(FIXTURES / pkg_name) - findings = evaluate(pkg, load_ruleset(RULESETS / ruleset)) - return reporters.build_report(pkg, findings) + rs = load_ruleset(RULESETS / ruleset) + findings = evaluate(pkg, rs) + return reporters.build_report(pkg, findings, rs) def test_markdown_render_contains_sections(): @@ -46,6 +47,22 @@ def test_json_render_roundtrips(): assert ids["aws.iam.console-mfa"] == "fail" +def test_json_stamps_tool_and_ruleset_provenance(): + data = json.loads(reporters.render(_report(), "json")) + assert data["tool"] == {"name": "audit-report", "version": __version__} + rs = data["ruleset"] + assert rs["name"] == "AWS ITGC ruleset" + assert rs["platform"] == "aws" + assert rs["version"] == "2026.08.0" + assert len(rs["sha256"]) == 64 # full SHA-256 hex digest of the ruleset file + + +def test_markdown_shows_ruleset_provenance(): + md = reporters.render(_report(), "md") + assert "**Tool:** audit-report" in md + assert "sha256:" in md + + def test_html_escapes_evidence(tmp_path): pkg_dir = tmp_path / "aws_audit_x_2026-01-01" pkg_dir.mkdir()