audit-labs/audit-report
Turn audit-tools evidence packages into control-mapped, auditor-ready reports.
clone: git clone https://gitbay.org/audit-labs/audit-report.git
92ba96f5451fca75c3e2544c4b0c52a3c5abfba9
verified · cmc
author: Christian Cleberg <hello@cleberg.net> · 2026-08-06T07:29:57Z
README.md | 33 +++ audit_report/cli.py | 43 +++- audit_report/loader.py | 15 +- audit_report/trend.py | 254 +++++++++++++++++++++ docs/ci.md | 67 ++++++ examples/github-actions-audit.yml | 66 ++++++ examples/gitlab-ci-audit.yml | 33 +++ .../aws_audit_prod_2026-01-01/account_security.csv | 2 + .../aws_audit_prod_2026-01-01/cloudtrail.csv | 2 + .../series/aws_audit_prod_2026-01-01/iam_users.csv | 2 + .../open_security_groups.csv | 2 + .../aws_audit_prod_2026-01-01/password_policy.csv | 2 + .../aws_audit_prod_2026-01-01/s3_public_access.csv | 2 + .../aws_audit_prod_2026-02-01/account_security.csv | 2 + .../aws_audit_prod_2026-02-01/cloudtrail.csv | 2 + .../series/aws_audit_prod_2026-02-01/iam_users.csv | 2 + .../open_security_groups.csv | 2 + .../aws_audit_prod_2026-02-01/password_policy.csv | 2 + .../aws_audit_prod_2026-02-01/s3_public_access.csv | 1 + .../aws_audit_prod_2026-03-01/account_security.csv | 2 + .../aws_audit_prod_2026-03-01/cloudtrail.csv | 2 + .../series/aws_audit_prod_2026-03-01/iam_users.csv | 2 + .../open_security_groups.csv | 2 + .../aws_audit_prod_2026-03-01/password_policy.csv | 2 + .../aws_audit_prod_2026-03-01/s3_public_access.csv | 1 + tests/test_trend.py | 115 ++++++++++ 26 files changed, 658 insertions(+), 2 deletions(-) @@ -63,6 +63,25 @@ audit-report ./output/aws_audit_prod_2026-07-29 \ In diff mode `--fail-on` gates on **regressions** at or above the given severity, and output files are named `diff.*` instead of `report.*`. +### Trend mode — track controls over time + +Pass `--trend` and point at a **folder of dated packages** (for example +audit-tools' `output/`). Every package in the series is evaluated with the same +ruleset and laid out as a timeline — one row per rule, one column per date — so +you can watch a control drift in and out of compliance. + +```bash +# Heatmap of every control across all retained prod packages +audit-report ./output --trend --format html,json --out trend/ + +# Disambiguate when the folder holds more than one series +audit-report ./output --trend --subject prod --out trend/ +``` + +The series must share one platform and subject (use `--subject` to pick one) and +contain at least two packages. In trend mode `--fail-on` reflects the **latest** +package, so it can double as a snapshot gate. Output files are named `trend.*`. + You can also run it without installing: ```bash @@ -74,6 +93,8 @@ python -m audit_report ./output/aws_audit_default_2026-07-29 | Flag | Description | | --- | --- | | `--baseline PATH` | Diff mode: report how `PACKAGE` drifted from this earlier package. | +| `--trend` | Trend mode: treat `PACKAGE` as a folder of dated packages and chart each rule over time. | +| `--subject NAME` | In trend mode, pick one subject when the folder holds several series. | | `--ruleset PATH` | Use a specific ruleset instead of the bundled one for the detected platform. | | `--format md,html,json` | One or more output formats (default: `md`). | | `--out DIR` | Write `report.<ext>` files into `DIR`. Without it, the first format prints to stdout. | @@ -135,6 +156,18 @@ rules: `not_empty`. Control codes referenced by a rule must exist in [`audit_report/catalog.py`](audit_report/catalog.py). +## Continuous integration + +Run it on a schedule or in a pipeline to keep evidence current and gate on +regressions. See [docs/ci.md](docs/ci.md) and the ready-to-copy examples: + +- [`examples/github-actions-audit.yml`](examples/github-actions-audit.yml) +- [`examples/gitlab-ci-audit.yml`](examples/gitlab-ci-audit.yml) + +The `--fail-on` exit code (`1` = a finding/regression met the threshold, `2` = +usage error) lets a workflow separate "the audit found a problem" from "the job +is misconfigured". + ## Development ```bash @@ -6,7 +6,7 @@ import argparse import sys from pathlib import Path -from . import __version__, diff, reporters +from . import __version__, diff, reporters, trend from .engine import FAIL, evaluate from .loader import load_package from .rules import load_ruleset @@ -38,6 +38,15 @@ def _parse_args(argv: list[str]) -> argparse.Namespace: "--baseline", help="path to an earlier package; diff mode reports how PACKAGE drifted from it", ) + parser.add_argument( + "--trend", + action="store_true", + help="trend mode: treat PACKAGE as a folder of dated packages and chart each rule over time", + ) + parser.add_argument( + "--subject", + help="in trend mode, pick one subject when the folder holds several series", + ) parser.add_argument( "--ruleset", help="path to a ruleset YAML (default: bundled ruleset for the detected platform)", @@ -118,6 +127,31 @@ def _run_diff(args, package, ruleset, formats: list[str]) -> int: return 1 if diff.has_regression(report, args.fail_on) else 0 +def _run_trend(args, formats: list[str]) -> int: + try: + platform, subject, paths = trend.discover(args.package, args.subject) + except ValueError as exc: + print(f"error: {exc}", file=sys.stderr) + return 2 + + ruleset_path = Path(args.ruleset) if args.ruleset else _default_ruleset(platform) + ruleset = load_ruleset(ruleset_path) + + packages = [load_package(p) for p in paths] + findings_per = [evaluate(pkg, ruleset) for pkg in packages] + report = trend.build_trend(packages, findings_per) + + _emit(lambda fmt: trend.render(report, fmt), formats, args.out, "trend") + + fails = report.fails_per_date() + print( + f"{platform}/{subject}: {len(paths)} packages, " + f"failing {fails[0]} → {fails[-1]}", + file=sys.stderr, + ) + return _exit_code(trend.latest_findings(findings_per), args.fail_on) + + def main(argv: list[str] | None = None) -> int: args = _parse_args(argv if argv is not None else sys.argv[1:]) @@ -127,6 +161,13 @@ def main(argv: list[str] | None = None) -> int: print(f"error: unknown format(s): {', '.join(unknown) or '(none given)'}", file=sys.stderr) return 2 + if args.trend and args.baseline: + print("error: --trend and --baseline cannot be combined", file=sys.stderr) + return 2 + + if args.trend: + return _run_trend(args, formats) + try: package = load_package(args.package) except (FileNotFoundError, ValueError) as exc: @@ -29,6 +29,7 @@ class Package: path: Path platform: str subject: str # the org / profile the audit was run against + date: str = "" # trailing YYYY-MM-DD from the dir name, if present tables: dict[str, Table] = field(default_factory=dict) def table(self, name: str) -> Table: @@ -61,6 +62,12 @@ def _looks_like_date(token: str) -> bool: return len(bits) == 3 and all(b.isdigit() for b in bits) +def package_date(dir_name: str) -> str: + """Return the trailing ``YYYY-MM-DD`` in a package dir name, or ''.""" + tail = dir_name.rsplit("_", 1)[-1] + return tail if _looks_like_date(tail) else "" + + def load_package(path: str | Path) -> Package: """Load every ``*.csv`` in *path* into a :class:`Package`. @@ -80,4 +87,10 @@ def load_package(path: str | Path) -> Package: if not tables: raise ValueError(f"no CSV files found in {directory}") - return Package(path=directory, platform=platform, subject=subject, tables=tables) + return Package( + path=directory, + platform=platform, + subject=subject, + date=package_date(directory.name), + tables=tables, + ) new file mode 100644 @@ -0,0 +1,254 @@ +"""Trend mode — track each rule across a series of dated packages. + +Given a folder of audit-tools packages for the same platform and subject +(``aws_audit_prod_2026-01-01/``, ``…_2026-02-01/``, …), this evaluates every +package with the same ruleset and lays the results out as a timeline: one row +per rule, one column per package date, so you can see a control drift in and out +of compliance over time. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from datetime import datetime, timezone +from html import escape +from pathlib import Path + +from .engine import FAIL, NOT_APPLICABLE, PASS, Finding +from .loader import Package, detect_platform, package_date +from .reporters.html import CSS as _CSS + +_STATUS_SYMBOL = {PASS: "✓", FAIL: "✗", NOT_APPLICABLE: "·"} +_STATUS_CLASS = {PASS: "pass", FAIL: "fail", NOT_APPLICABLE: "na"} +_SEVERITY_ORDER = {"high": 0, "medium": 1, "low": 2} + + +def discover(parent: str | Path, subject: str | None = None) -> tuple[str, str, list[Path]]: + """Find a single series of packages under *parent*. + + Returns ``(platform, subject, [paths sorted by date])``. Raises ``ValueError`` + if no packages are found, if fewer than two share a platform/subject, or if + several distinct series are present and *subject* does not narrow it to one. + """ + directory = Path(parent) + if not directory.is_dir(): + raise ValueError(f"not a directory: {directory}") + + groups: dict[tuple[str, str], list[Path]] = {} + for child in sorted(directory.iterdir()): + if not child.is_dir() or not any(child.glob("*.csv")): + continue + platform, subj = detect_platform(child.name) + if platform == "unknown": + continue + if subject and subj != subject: + continue + groups.setdefault((platform, subj), []).append(child) + + if not groups: + raise ValueError( + f"no audit-tools packages found under {directory}" + + (f" for subject '{subject}'" if subject else "") + ) + if len(groups) > 1: + listed = ", ".join(f"{p}/{s}" for p, s in sorted(groups)) + raise ValueError( + f"multiple series found ({listed}); narrow with --subject and a " + "directory that holds one platform" + ) + + (platform, subj), paths = next(iter(groups.items())) + if len(paths) < 2: + raise ValueError("a trend needs at least two packages in the series") + + paths.sort(key=lambda p: (package_date(p.name), p.name)) + return platform, subj, paths + + +@dataclass +class TrendRow: + """One rule's status across the timeline.""" + + rule: object # audit_report.rules.Rule + statuses: list[str] + + @property + def transitions(self) -> int: + """How many times the status changed along the timeline.""" + return sum(1 for a, b in zip(self.statuses, self.statuses[1:]) if a != b) + + +@dataclass +class TrendReport: + """Rules-over-time view of a package series.""" + + platform: str + subject: str + dates: list[str] # column labels (package date or dir name) + rows: list[TrendRow] + generated_at: str + + def fails_per_date(self) -> list[int]: + return [ + sum(1 for row in self.rows if row.statuses[i] == FAIL) + for i in range(len(self.dates)) + ] + + +def build_trend( + packages: list[Package], findings_per_package: list[list[Finding]] +) -> TrendReport: + """Assemble a :class:`TrendReport` from aligned packages and findings.""" + dates = [pkg.date or pkg.path.name for pkg in packages] + + # Preserve rule order from the first package; align by rule id across dates. + order = [f.rule for f in findings_per_package[0]] + by_date = [{f.rule.id: f for f in findings} for findings in findings_per_package] + + rows = [ + TrendRow( + rule=rule, + statuses=[ + col.get(rule.id).status if col.get(rule.id) else NOT_APPLICABLE + for col in by_date + ], + ) + for rule in order + ] + stamp = datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M UTC") + return TrendReport( + platform=packages[0].platform, + subject=packages[0].subject, + dates=dates, + rows=rows, + generated_at=stamp, + ) + + +def latest_findings(findings_per_package: list[list[Finding]]) -> list[Finding]: + """The findings of the most recent package (for --fail-on gating).""" + return findings_per_package[-1] + + +# --------------------------------------------------------------------------- # +# Rendering +# --------------------------------------------------------------------------- # + + +def _render_md(trend: TrendReport) -> str: + out: list[str] = [] + out.append(f"# Evidence Trend — {trend.subject} ({trend.platform})") + out.append("") + out.append(f"- **Packages:** {len(trend.dates)}") + out.append(f"- **Timeline:** {trend.dates[0]} → {trend.dates[-1]}") + out.append(f"- **Generated:** {trend.generated_at}") + out.append("") + out.append("Legend: ✓ pass · ✗ fail · · not applicable") + out.append("") + + header = "| Rule | Sev | " + " | ".join(trend.dates) + " |" + sep = "| --- | --- | " + " | ".join("---" for _ in trend.dates) + " |" + out.append(header) + out.append(sep) + for row in sorted(trend.rows, key=lambda r: _SEVERITY_ORDER.get(r.rule.severity, 1)): + cells = " | ".join(_STATUS_SYMBOL.get(s, "?") for s in row.statuses) + out.append(f"| {row.rule.title} | {row.rule.severity} | {cells} |") + + fails = trend.fails_per_date() + out.append("| **Failing total** | | " + " | ".join(str(n) for n in fails) + " |") + return "\n".join(out).rstrip() + "\n" + + +_TREND_CSS = ( + _CSS + + """ +.trend { border-collapse: collapse; } +.trend th.date { font-size: .78rem; white-space: nowrap; } +.trend td.cell { text-align: center; font-weight: 700; width: 2.4rem; } +.trend td.cell.pass { background: #e5f6ea; color: #1a7f37; } +.trend td.cell.fail { background: #fdeaea; color: #c1272d; } +.trend td.cell.na { background: #f4f4f6; color: #999; } +.trend tr.totals td { font-weight: 700; background: #fafafa; } +.rulecol { max-width: 22rem; } +.legend { font-size: .85rem; color: #555; margin: .25rem 0 1rem; } +@media (prefers-color-scheme: dark) { + .trend td.cell.pass { background: #12321d; color: #4ac36a; } + .trend td.cell.fail { background: #3a1416; color: #ff6b70; } + .trend td.cell.na { background: #202126; color: #888; } + .trend tr.totals td { background: #1c1d21; } + .legend { color: #aaa; } +} +""" +) + + +def _render_html(trend: TrendReport) -> str: + date_heads = "".join(f"<th class='date'>{escape(d)}</th>" for d in trend.dates) + body_rows: list[str] = [] + for row in sorted(trend.rows, key=lambda r: _SEVERITY_ORDER.get(r.rule.severity, 1)): + cells = "".join( + f"<td class='cell {_STATUS_CLASS.get(s, 'na')}' title='{escape(s)}'>" + f"{_STATUS_SYMBOL.get(s, '?')}</td>" + for s in row.statuses + ) + body_rows.append( + f"<tr><td class='rulecol'>{escape(row.rule.title)}" + f"<br><code>{escape(row.rule.id)}</code></td>" + f"<td>{escape(row.rule.severity)}</td>{cells}</tr>" + ) + totals = "".join(f"<td class='cell'>{n}</td>" for n in trend.fails_per_date()) + + table = ( + "<table class='trend'><thead><tr><th class='rulecol'>Rule</th><th>Sev</th>" + f"{date_heads}</tr></thead><tbody>{''.join(body_rows)}" + f"<tr class='totals'><td>Failing total</td><td></td>{totals}</tr>" + "</tbody></table>" + ) + return ( + "<!doctype html><html lang='en'><head><meta charset='utf-8'>" + "<meta name='viewport' content='width=device-width, initial-scale=1'>" + f"<title>Evidence Trend — {escape(trend.subject)}</title>" + f"<style>{_TREND_CSS}</style></head><body><main>" + f"<h1>Evidence Trend — {escape(trend.subject)} ({escape(trend.platform)})</h1>" + f"<p class='meta'>{len(trend.dates)} packages · {escape(trend.dates[0])} → " + f"{escape(trend.dates[-1])} · Generated {escape(trend.generated_at)}</p>" + "<p class='legend'>✓ pass · ✗ fail · · not applicable — cell colour tracks " + "each control over time.</p>" + f"{table}" + "<footer>Generated by audit-report · Audit Labs · evidence, not a verdict.</footer>" + "</main></body></html>\n" + ) + + +def _render_json(trend: TrendReport) -> str: + import json as _json + + payload = { + "subject": trend.subject, + "platform": trend.platform, + "dates": trend.dates, + "generated_at": trend.generated_at, + "fails_per_date": trend.fails_per_date(), + "rules": [ + { + "id": row.rule.id, + "title": row.rule.title, + "severity": row.rule.severity, + "controls": row.rule.controls, + "statuses": row.statuses, + } + for row in trend.rows + ], + } + return _json.dumps(payload, indent=2) + "\n" + + +_RENDERERS = {"md": _render_md, "html": _render_html, "json": _render_json} + + +def render(trend: TrendReport, fmt: str) -> str: + """Render a trend in the named format ('md', 'html', or 'json').""" + try: + return _RENDERERS[fmt](trend) + except KeyError: + raise ValueError(f"unknown format: {fmt!r}") from None new file mode 100644 @@ -0,0 +1,67 @@ +# Running audit-report in CI + +The pattern is always the same three steps: + +1. **Collect** an evidence package with `audit-tools` (per-platform CLI). +2. **Report** on it with `audit-report`, writing `md`/`html`/`json` artifacts. +3. **Gate** the pipeline with `--fail-on` so a control regression can block a + merge or page a scheduled run. + +Ready-to-copy starting points: + +- [`examples/github-actions-audit.yml`](../examples/github-actions-audit.yml) +- [`examples/gitlab-ci-audit.yml`](../examples/gitlab-ci-audit.yml) + +> The collection step in the examples shows both a module entrypoint +> (`python -m audit_tools.github`) and a script fallback (`python audit.py`). +> Use whichever your installed `audit-tools` exposes; everything downstream only +> needs the `./output/<platform>_audit_<subject>_<date>/` directory it writes. + +## Gating strategies + +**Snapshot gate — the current state must be clean.** + +```bash +audit-report "$PKG" --fail-on high +``` + +Exits non-zero if any high-severity control is unsupported in the newest +package. Simple and strict; good for a scheduled run that should stay green. + +**Regression gate — this change must not make things worse.** + +Keep the previous package in the repo (or restore it from an artifact) and diff +against it. The build fails only when a control that used to pass now fails, +which avoids blocking on pre-existing debt. + +```bash +audit-report "$PKG" --baseline ./baseline/"$LAST_PKG" --fail-on high +``` + +**Trend artifact — show direction over time.** + +Point trend mode at a folder of retained packages to publish a heatmap of every +control across dates. Its `--fail-on` reflects the latest package, so it can +double as a snapshot gate while producing the timeline artifact. + +```bash +audit-report ./history --trend --format html,json --out ./trend +``` + +## Exit codes + +| Code | Meaning | +| --- | --- | +| `0` | Ran successfully; no `--fail-on` threshold was breached. | +| `1` | A finding (or, in diff mode, a regression) met the `--fail-on` severity. | +| `2` | Usage or input error (missing package, unknown format, mismatched platforms). | + +Distinguishing `1` from `2` lets a workflow tell "the audit found a problem" +(expected, actionable) from "the job is misconfigured" (fix the pipeline). + +## Retaining history + +`audit-report` never writes back to the package — it only reads. To build a +trend or a regression baseline, archive each run's package directory (a CI +artifact, a committed `history/` folder, or object storage) and feed the +collection back in on the next run. new file mode 100644 @@ -0,0 +1,66 @@ +# Example GitHub Actions workflow: collect evidence, then report on it. +# +# Copy into .github/workflows/audit.yml in the repository you want to audit and +# adjust the collection step to your platform. It runs on a schedule and on +# demand, produces a control-mapped report, and fails the run if a high-severity +# control regresses against the previous package committed to the repo. +# +# Requires two org/repo secrets for the GitHub collector: AUDIT_GITHUB_TOKEN +# (a read-only token for the org you audit) and the org name in AUDIT_ORG. + +name: compliance-evidence + +on: + schedule: + - cron: "0 6 * * 1" # every Monday 06:00 UTC + workflow_dispatch: {} + +permissions: + contents: read + +jobs: + audit: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Install tools + run: | + python -m pip install --upgrade pip + # The reporter: + pip install "audit-report @ git+https://github.com/audit-labs/audit-report" + # The collector (audit-tools ships CLIs per platform): + pip install "audit-tools @ git+https://github.com/audit-labs/audit-tools" + + - name: Collect evidence (GitHub example) + env: + GITHUB_TOKEN: ${{ secrets.AUDIT_GITHUB_TOKEN }} + GITHUB_ORG: ${{ secrets.AUDIT_ORG }} + run: | + # Produces ./output/github_audit_<org>_<date>/ + python -m audit_tools.github --out ./output || \ + python audit.py --out ./output # fall back to the script entrypoint + + - name: Locate the newest package + id: pkg + run: echo "dir=$(ls -d ./output/*_audit_* | sort | tail -n1)" >> "$GITHUB_OUTPUT" + + - name: Generate evidence report + run: | + audit-report "${{ steps.pkg.outputs.dir }}" \ + --format md,html,json --out ./report + + - name: Fail on any high-severity finding + run: audit-report "${{ steps.pkg.outputs.dir }}" --fail-on high + + - name: Publish the report as a build artifact + if: always() + uses: actions/upload-artifact@v4 + with: + name: evidence-report + path: report/ + retention-days: 90 new file mode 100644 @@ -0,0 +1,33 @@ +# Example GitLab CI configuration: collect evidence, then report on it. +# +# Copy into .gitlab-ci.yml (or include it) in the project you want to audit. +# It produces a control-mapped report as a job artifact and fails the pipeline +# if any high-severity control is not supported. +# +# Set CI/CD variables GITLAB_TOKEN (read-only) and GITLAB_GROUP for the group +# you audit. + +stages: [audit] + +compliance-evidence: + stage: audit + image: python:3.12-slim + rules: + - if: $CI_PIPELINE_SOURCE == "schedule" + - if: $CI_PIPELINE_SOURCE == "web" # manual "Run pipeline" + variables: + PIP_DISABLE_PIP_VERSION_CHECK: "1" + before_script: + - pip install "audit-report @ git+https://github.com/audit-labs/audit-report" + - pip install "audit-tools @ git+https://github.com/audit-labs/audit-tools" + script: + # Produces ./output/gitlab_audit_<group>_<date>/ + - python -m audit_tools.gitlab --out ./output || python audit.py --out ./output + - PKG=$(ls -d ./output/*_audit_* | sort | tail -n1) + - audit-report "$PKG" --format md,html,json --out ./report + - audit-report "$PKG" --fail-on high + artifacts: + when: always + paths: + - report/ + expire_in: 90 days new file mode 100644 @@ -0,0 +1,2 @@ +root_mfa_enabled,root_access_keys_present,root_signing_certs_present,mfa_devices,users,groups,roles,policies +False,True,False,0,1,0,3,0 new file mode 100644 @@ -0,0 +1,2 @@ +name,home_region,multi_region,log_file_validation,is_logging,s3_bucket +main,us-east-1,False,False,False, new file mode 100644 @@ -0,0 +1,2 @@ +user,mfa_enabled,access_keys,oldest_key_age_days,console_password,password_last_used,created +bob,False,1,400,True,2025-11-01T00:00:00+00:00,2025-02-01T00:00:00+00:00 new file mode 100644 @@ -0,0 +1,2 @@ +region,group_id,group_name,protocol,from_port,to_port,open_to +us-east-1,sg-web,web,tcp,22,22,0.0.0.0/0 new file mode 100644 @@ -0,0 +1,2 @@ +minimum_length,require_symbols,require_numbers,require_uppercase,require_lowercase,allow_users_to_change,max_age_days,reuse_prevention,hard_expiry +8,False,True,True,True,True,365,0,False new file mode 100644 @@ -0,0 +1,2 @@ +bucket,region,public_access_block,policy_public,acl_public +acme-assets,us-east-1,False,True,False new file mode 100644 @@ -0,0 +1,2 @@ +root_mfa_enabled,root_access_keys_present,root_signing_certs_present,mfa_devices,users,groups,roles,policies +True,True,False,1,1,0,3,0 new file mode 100644 @@ -0,0 +1,2 @@ +name,home_region,multi_region,log_file_validation,is_logging,s3_bucket +main,us-east-1,True,True,True,acme-logs new file mode 100644 @@ -0,0 +1,2 @@ +user,mfa_enabled,access_keys,oldest_key_age_days,console_password,password_last_used,created +bob,True,1,400,True,2026-01-15T00:00:00+00:00,2025-02-01T00:00:00+00:00 new file mode 100644 @@ -0,0 +1,2 @@ +region,group_id,group_name,protocol,from_port,to_port,open_to +us-east-1,sg-web,web,tcp,22,22,0.0.0.0/0 new file mode 100644 @@ -0,0 +1,2 @@ +minimum_length,require_symbols,require_numbers,require_uppercase,require_lowercase,allow_users_to_change,max_age_days,reuse_prevention,hard_expiry +14,True,True,True,True,True,90,24,False new file mode 100644 @@ -0,0 +1 @@ +bucket,region,public_access_block,policy_public,acl_public new file mode 100644 @@ -0,0 +1,2 @@ +root_mfa_enabled,root_access_keys_present,root_signing_certs_present,mfa_devices,users,groups,roles,policies +True,False,False,1,1,0,3,0 new file mode 100644 @@ -0,0 +1,2 @@ +name,home_region,multi_region,log_file_validation,is_logging,s3_bucket +main,us-east-1,True,True,True,acme-logs new file mode 100644 @@ -0,0 +1,2 @@ +user,mfa_enabled,access_keys,oldest_key_age_days,console_password,password_last_used,created +bob,True,1,30,True,2026-02-20T00:00:00+00:00,2025-02-01T00:00:00+00:00 new file mode 100644 @@ -0,0 +1,2 @@ +region,group_id,group_name,protocol,from_port,to_port,open_to +us-east-1,sg-db,db,tcp,443,443,10.0.0.0/8 new file mode 100644 @@ -0,0 +1,2 @@ +minimum_length,require_symbols,require_numbers,require_uppercase,require_lowercase,allow_users_to_change,max_age_days,reuse_prevention,hard_expiry +14,True,True,True,True,True,90,24,False new file mode 100644 @@ -0,0 +1 @@ +bucket,region,public_access_block,policy_public,acl_public new file mode 100644 @@ -0,0 +1,115 @@ +"""Tests for trend mode: discovery, timeline building, rendering, and the CLI.""" + +import json +from pathlib import Path + +import pytest + +from audit_report import trend +from audit_report.cli import main +from audit_report.engine import FAIL, PASS, evaluate +from audit_report.loader import load_package +from audit_report.rules import load_ruleset + +FIXTURES = Path(__file__).parent / "fixtures" +SERIES = FIXTURES / "series" +RULESETS = Path("audit_report/rulesets") + + +def _build(): + platform, _subject, paths = trend.discover(SERIES) + ruleset = load_ruleset(RULESETS / f"{platform}.yaml") + packages = [load_package(p) for p in paths] + findings = [evaluate(pkg, ruleset) for pkg in packages] + return trend.build_trend(packages, findings) + + +def test_discover_orders_by_date(): + platform, subject, paths = trend.discover(SERIES) + assert (platform, subject) == ("aws", "prod") + assert [p.name for p in paths] == [ + "aws_audit_prod_2026-01-01", + "aws_audit_prod_2026-02-01", + "aws_audit_prod_2026-03-01", + ] + + +def test_discover_rejects_mixed_series(): + # The fixtures root holds aws/github/gitlab packages for several subjects. + with pytest.raises(ValueError, match="multiple series"): + trend.discover(FIXTURES) + + +def test_discover_requires_two(tmp_path): + # A parent directory holding a single package is not a trend. + pkg = tmp_path / "aws_audit_solo_2026-01-01" + pkg.mkdir() + (pkg / "account_security.csv").write_text("root_mfa_enabled\nTrue\n", encoding="utf-8") + with pytest.raises(ValueError, match="at least two"): + trend.discover(tmp_path) + + +def test_discover_no_packages(tmp_path): + with pytest.raises(ValueError, match="no audit-tools packages"): + trend.discover(tmp_path) + + +def test_trend_timeline_and_totals(): + report = _build() + assert report.dates == ["2026-01-01", "2026-02-01", "2026-03-01"] + assert report.fails_per_date() == [8, 3, 0] + + by_id = {row.rule.id: row for row in report.rows} + # Root MFA: fail, then fixed and stays fixed. + assert by_id["aws.root.mfa"].statuses == [FAIL, PASS, PASS] + # Access-key rotation lags: fixed only in the final package. + assert by_id["aws.iam.key-rotation"].statuses == [FAIL, FAIL, PASS] + + +def test_trend_row_transitions(): + report = _build() + by_id = {row.rule.id: row for row in report.rows} + assert by_id["aws.root.mfa"].transitions == 1 # one fail->pass change + assert by_id["aws.s3.no-public-access"].transitions == 1 + + +def test_trend_render_markdown(): + md = trend.render(_build(), "md") + assert "# Evidence Trend — prod (aws)" in md + assert "Failing total" in md + assert "2026-03-01" in md + + +def test_trend_render_html_self_contained(): + html = trend.render(_build(), "html") + assert html.startswith("<!doctype html>") + assert "http://" not in html and "https://" not in html + assert "class='trend'" in html + + +def test_trend_render_json(): + data = json.loads(trend.render(_build(), "json")) + assert data["dates"] == ["2026-01-01", "2026-02-01", "2026-03-01"] + assert data["fails_per_date"] == [8, 3, 0] + rules = {r["id"]: r["statuses"] for r in data["rules"]} + assert rules["aws.root.mfa"] == ["fail", "pass", "pass"] + + +def test_cli_trend_mode(tmp_path): + out = tmp_path / "out" + code = main([str(SERIES), "--trend", "--format", "md,html,json", "--out", str(out)]) + assert (out / "trend.md").exists() + assert (out / "trend.html").exists() + assert (out / "trend.json").exists() + # The latest package is clean, so default --fail-on none exits 0. + assert code == 0 + + +def test_cli_trend_fail_on_uses_latest(tmp_path): + # Latest package (2026-03) has no failures, so even --fail-on low passes. + code = main([str(SERIES), "--trend", "--format", "json", "--out", str(tmp_path), "--fail-on", "low"]) + assert code == 0 + + +def test_cli_trend_and_baseline_conflict(): + assert main([str(SERIES), "--trend", "--baseline", str(SERIES), "--format", "json"]) == 2