92ba96f545 · audit-labs/audit-report

README.md +33

	@@ -63,6 +63,25 @@ audit-report ./output/aws_audit_prod_2026-07-29 \
63	In diff mode `--fail-on` gates on regressions at or above the given	63	In diff mode `--fail-on` gates on regressions at or above the given
64	severity, and output files are named `diff.` instead of `report.`.	64	severity, and output files are named `diff.` instead of `report.`.
65		65
		66	### Trend mode — track controls over time
		67
		68	Pass `--trend` and point at a folder of dated packages (for example
		69	audit-tools' `output/`). Every package in the series is evaluated with the same
		70	ruleset and laid out as a timeline — one row per rule, one column per date — so
		71	you can watch a control drift in and out of compliance.
		72
		73	```bash
		74	# Heatmap of every control across all retained prod packages
		75	audit-report ./output --trend --format html,json --out trend/
		76
		77	# Disambiguate when the folder holds more than one series
		78	audit-report ./output --trend --subject prod --out trend/
		79	```
		80
		81	The series must share one platform and subject (use `--subject` to pick one) and
		82	contain at least two packages. In trend mode `--fail-on` reflects the latest
		83	package, so it can double as a snapshot gate. Output files are named `trend.*`.
		84
66	You can also run it without installing:	85	You can also run it without installing:
67		86
68	```bash	87	```bash
	@@ -74,6 +93,8 @@ python -m audit_report ./output/aws_audit_default_2026-07-29
74	\| Flag \| Description \|	93	\| Flag \| Description \|
75	\| --- \| --- \|	94	\| --- \| --- \|
76	\| `--baseline PATH` \| Diff mode: report how `PACKAGE` drifted from this earlier package. \|	95	\| `--baseline PATH` \| Diff mode: report how `PACKAGE` drifted from this earlier package. \|
		96	\| `--trend` \| Trend mode: treat `PACKAGE` as a folder of dated packages and chart each rule over time. \|
		97	\| `--subject NAME` \| In trend mode, pick one subject when the folder holds several series. \|
77	\| `--ruleset PATH` \| Use a specific ruleset instead of the bundled one for the detected platform. \|	98	\| `--ruleset PATH` \| Use a specific ruleset instead of the bundled one for the detected platform. \|
78	\| `--format md,html,json` \| One or more output formats (default: `md`). \|	99	\| `--format md,html,json` \| One or more output formats (default: `md`). \|
79	\| `--out DIR` \| Write `report.<ext>` files into `DIR`. Without it, the first format prints to stdout. \|	100	\| `--out DIR` \| Write `report.<ext>` files into `DIR`. Without it, the first format prints to stdout. \|
	@@ -135,6 +156,18 @@ rules:
135	`not_empty`. Control codes referenced by a rule must exist in	156	`not_empty`. Control codes referenced by a rule must exist in
136	[`audit_report/catalog.py`](audit_report/catalog.py).	157	[`audit_report/catalog.py`](audit_report/catalog.py).
137		158
		159	## Continuous integration
		160
		161	Run it on a schedule or in a pipeline to keep evidence current and gate on
		162	regressions. See [docs/ci.md](docs/ci.md) and the ready-to-copy examples:
		163
		164	- [`examples/github-actions-audit.yml`](examples/github-actions-audit.yml)
		165	- [`examples/gitlab-ci-audit.yml`](examples/gitlab-ci-audit.yml)
		166
		167	The `--fail-on` exit code (`1` = a finding/regression met the threshold, `2` =
		168	usage error) lets a workflow separate "the audit found a problem" from "the job
		169	is misconfigured".
		170
138	## Development	171	## Development
139		172
140	```bash	173	```bash

audit_report/cli.py +42 −1

	@@ -6,7 +6,7 @@ import argparse
6	import sys	6	import sys
7	from pathlib import Path	7	from pathlib import Path
8		8
9	from . import __version__, diff, reporters	9	from . import __version__, diff, reporters, trend
10	from .engine import FAIL, evaluate	10	from .engine import FAIL, evaluate
11	from .loader import load_package	11	from .loader import load_package
12	from .rules import load_ruleset	12	from .rules import load_ruleset
	@@ -38,6 +38,15 @@ def _parse_args(argv: list[str]) -> argparse.Namespace:
38	"--baseline",	38	"--baseline",
39	help="path to an earlier package; diff mode reports how PACKAGE drifted from it",	39	help="path to an earlier package; diff mode reports how PACKAGE drifted from it",
40	)	40	)
		41	parser.add_argument(
		42	"--trend",
		43	action="store_true",
		44	help="trend mode: treat PACKAGE as a folder of dated packages and chart each rule over time",
		45	)
		46	parser.add_argument(
		47	"--subject",
		48	help="in trend mode, pick one subject when the folder holds several series",
		49	)
41	parser.add_argument(	50	parser.add_argument(
42	"--ruleset",	51	"--ruleset",
43	help="path to a ruleset YAML (default: bundled ruleset for the detected platform)",	52	help="path to a ruleset YAML (default: bundled ruleset for the detected platform)",
	@@ -118,6 +127,31 @@ def _run_diff(args, package, ruleset, formats: list[str]) -> int:
118	return 1 if diff.has_regression(report, args.fail_on) else 0	127	return 1 if diff.has_regression(report, args.fail_on) else 0
119		128
120		129
		130	def _run_trend(args, formats: list[str]) -> int:
		131	try:
		132	platform, subject, paths = trend.discover(args.package, args.subject)
		133	except ValueError as exc:
		134	print(f"error: {exc}", file=sys.stderr)
		135	return 2
		136
		137	ruleset_path = Path(args.ruleset) if args.ruleset else _default_ruleset(platform)
		138	ruleset = load_ruleset(ruleset_path)
		139
		140	packages = [load_package(p) for p in paths]
		141	findings_per = [evaluate(pkg, ruleset) for pkg in packages]
		142	report = trend.build_trend(packages, findings_per)
		143
		144	_emit(lambda fmt: trend.render(report, fmt), formats, args.out, "trend")
		145
		146	fails = report.fails_per_date()
		147	print(
		148	f"{platform}/{subject}: {len(paths)} packages, "
		149	f"failing {fails[0]} → {fails[-1]}",
		150	file=sys.stderr,
		151	)
		152	return _exit_code(trend.latest_findings(findings_per), args.fail_on)
		153
		154
121	def main(argv: list[str] \| None = None) -> int:	155	def main(argv: list[str] \| None = None) -> int:
122	args = _parse_args(argv if argv is not None else sys.argv[1:])	156	args = _parse_args(argv if argv is not None else sys.argv[1:])
123		157
	@@ -127,6 +161,13 @@ def main(argv: list[str] \| None = None) -> int:
127	print(f"error: unknown format(s): {', '.join(unknown) or '(none given)'}", file=sys.stderr)	161	print(f"error: unknown format(s): {', '.join(unknown) or '(none given)'}", file=sys.stderr)
128	return 2	162	return 2
129		163
		164	if args.trend and args.baseline:
		165	print("error: --trend and --baseline cannot be combined", file=sys.stderr)
		166	return 2
		167
		168	if args.trend:
		169	return _run_trend(args, formats)
		170
130	try:	171	try:
131	package = load_package(args.package)	172	package = load_package(args.package)
132	except (FileNotFoundError, ValueError) as exc:	173	except (FileNotFoundError, ValueError) as exc:

audit_report/loader.py +14 −1

     path: Path
     platform: str
     subject: str  # the org / profile the audit was run against
+    date: str = ""  # trailing YYYY-MM-DD from the dir name, if present
     tables: dict[str, Table] = field(default_factory=dict)
     def table(self, name: str) -> Table:
     return len(bits) == 3 and all(b.isdigit() for b in bits)
+def package_date(dir_name: str) -> str:
+    """Return the trailing ``YYYY-MM-DD`` in a package dir name, or ''."""
+    tail = dir_name.rsplit("_", 1)[-1]
+    return tail if _looks_like_date(tail) else ""
 def load_package(path: str | Path) -> Package:
     """Load every ``*.csv`` in *path* into a :class:`Package`.
     if not tables:
         raise ValueError(f"no CSV files found in {directory}")
-    return Package(path=directory, platform=platform, subject=subject, tables=tables)
+    return Package(
+        path=directory,
+        platform=platform,
+        subject=subject,
+        date=package_date(directory.name),
+        tables=tables,
+    )

audit_report/trend.py added +254

@@ -0,0 +1,254 @@
	1	"""Trend mode — track each rule across a series of dated packages.
	2
	3	Given a folder of audit-tools packages for the same platform and subject
	4	(``aws_audit_prod_2026-01-01/``, ``…_2026-02-01/``, …), this evaluates every
	5	package with the same ruleset and lays the results out as a timeline: one row
	6	per rule, one column per package date, so you can see a control drift in and out
	7	of compliance over time.
	8	"""
	9
	10	from __future__ import annotations
	11
	12	from dataclasses import dataclass
	13	from datetime import datetime, timezone
	14	from html import escape
	15	from pathlib import Path
	16
	17	from .engine import FAIL, NOT_APPLICABLE, PASS, Finding
	18	from .loader import Package, detect_platform, package_date
	19	from .reporters.html import CSS as _CSS
	20
	21	_STATUS_SYMBOL = {PASS: "✓", FAIL: "✗", NOT_APPLICABLE: "·"}
	22	_STATUS_CLASS = {PASS: "pass", FAIL: "fail", NOT_APPLICABLE: "na"}
	23	_SEVERITY_ORDER = {"high": 0, "medium": 1, "low": 2}
	24
	25
	26	def discover(parent: str \| Path, subject: str \| None = None) -> tuple[str, str, list[Path]]:
	27	"""Find a single series of packages under parent.
	28
	29	Returns ``(platform, subject, [paths sorted by date])``. Raises ``ValueError``
	30	if no packages are found, if fewer than two share a platform/subject, or if
	31	several distinct series are present and subject does not narrow it to one.
	32	"""
	33	directory = Path(parent)
	34	if not directory.is_dir():
	35	raise ValueError(f"not a directory: {directory}")
	36
	37	groups: dict[tuple[str, str], list[Path]] = {}
	38	for child in sorted(directory.iterdir()):
	39	if not child.is_dir() or not any(child.glob("*.csv")):
	40	continue
	41	platform, subj = detect_platform(child.name)
	42	if platform == "unknown":
	43	continue
	44	if subject and subj != subject:
	45	continue
	46	groups.setdefault((platform, subj), []).append(child)
	47
	48	if not groups:
	49	raise ValueError(
	50	f"no audit-tools packages found under {directory}"
	51	+ (f" for subject '{subject}'" if subject else "")
	52	)
	53	if len(groups) > 1:
	54	listed = ", ".join(f"{p}/{s}" for p, s in sorted(groups))
	55	raise ValueError(
	56	f"multiple series found ({listed}); narrow with --subject and a "
	57	"directory that holds one platform"
	58	)
	59
	60	(platform, subj), paths = next(iter(groups.items()))
	61	if len(paths) < 2:
	62	raise ValueError("a trend needs at least two packages in the series")
	63
	64	paths.sort(key=lambda p: (package_date(p.name), p.name))
	65	return platform, subj, paths
	66
	67
	68	@dataclass
	69	class TrendRow:
	70	"""One rule's status across the timeline."""
	71
	72	rule: object # audit_report.rules.Rule
	73	statuses: list[str]
	74
	75	@property
	76	def transitions(self) -> int:
	77	"""How many times the status changed along the timeline."""
	78	return sum(1 for a, b in zip(self.statuses, self.statuses[1:]) if a != b)
	79
	80
	81	@dataclass
	82	class TrendReport:
	83	"""Rules-over-time view of a package series."""
	84
	85	platform: str
	86	subject: str
	87	dates: list[str] # column labels (package date or dir name)
	88	rows: list[TrendRow]
	89	generated_at: str
	90
	91	def fails_per_date(self) -> list[int]:
	92	return [
	93	sum(1 for row in self.rows if row.statuses[i] == FAIL)
	94	for i in range(len(self.dates))
	95	]
	96
	97
	98	def build_trend(
	99	packages: list[Package], findings_per_package: list[list[Finding]]
	100	) -> TrendReport:
	101	"""Assemble a :class:`TrendReport` from aligned packages and findings."""
	102	dates = [pkg.date or pkg.path.name for pkg in packages]
	103
	104	# Preserve rule order from the first package; align by rule id across dates.
	105	order = [f.rule for f in findings_per_package[0]]
	106	by_date = [{f.rule.id: f for f in findings} for findings in findings_per_package]
	107
	108	rows = [
	109	TrendRow(
	110	rule=rule,
	111	statuses=[
	112	col.get(rule.id).status if col.get(rule.id) else NOT_APPLICABLE
	113	for col in by_date
	114	],
	115	)
	116	for rule in order
	117	]
	118	stamp = datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M UTC")
	119	return TrendReport(
	120	platform=packages[0].platform,
	121	subject=packages[0].subject,
	122	dates=dates,
	123	rows=rows,
	124	generated_at=stamp,
	125	)
	126
	127
	128	def latest_findings(findings_per_package: list[list[Finding]]) -> list[Finding]:
	129	"""The findings of the most recent package (for --fail-on gating)."""
	130	return findings_per_package[-1]
	131
	132
	133	# --------------------------------------------------------------------------- #
	134	# Rendering
	135	# --------------------------------------------------------------------------- #
	136
	137
	138	def _render_md(trend: TrendReport) -> str:
	139	out: list[str] = []
	140	out.append(f"# Evidence Trend — {trend.subject} ({trend.platform})")
	141	out.append("")
	142	out.append(f"- Packages: {len(trend.dates)}")
	143	out.append(f"- Timeline: {trend.dates[0]} → {trend.dates[-1]}")
	144	out.append(f"- Generated: {trend.generated_at}")
	145	out.append("")
	146	out.append("Legend: ✓ pass · ✗ fail · · not applicable")
	147	out.append("")
	148
	149	header = "\| Rule \| Sev \| " + " \| ".join(trend.dates) + " \|"
	150	sep = "\| --- \| --- \| " + " \| ".join("---" for _ in trend.dates) + " \|"
	151	out.append(header)
	152	out.append(sep)
	153	for row in sorted(trend.rows, key=lambda r: _SEVERITY_ORDER.get(r.rule.severity, 1)):
	154	cells = " \| ".join(_STATUS_SYMBOL.get(s, "?") for s in row.statuses)
	155	out.append(f"\| {row.rule.title} \| {row.rule.severity} \| {cells} \|")
	156
	157	fails = trend.fails_per_date()
	158	out.append("\| Failing total \| \| " + " \| ".join(str(n) for n in fails) + " \|")
	159	return "\n".join(out).rstrip() + "\n"
	160
	161
	162	_TREND_CSS = (
	163	_CSS
	164	+ """
	165	.trend { border-collapse: collapse; }
	166	.trend th.date { font-size: .78rem; white-space: nowrap; }
	167	.trend td.cell { text-align: center; font-weight: 700; width: 2.4rem; }
	168	.trend td.cell.pass { background: #e5f6ea; color: #1a7f37; }
	169	.trend td.cell.fail { background: #fdeaea; color: #c1272d; }
	170	.trend td.cell.na { background: #f4f4f6; color: #999; }
	171	.trend tr.totals td { font-weight: 700; background: #fafafa; }
	172	.rulecol { max-width: 22rem; }
	173	.legend { font-size: .85rem; color: #555; margin: .25rem 0 1rem; }
	174	@media (prefers-color-scheme: dark) {
	175	.trend td.cell.pass { background: #12321d; color: #4ac36a; }
	176	.trend td.cell.fail { background: #3a1416; color: #ff6b70; }
	177	.trend td.cell.na { background: #202126; color: #888; }
	178	.trend tr.totals td { background: #1c1d21; }
	179	.legend { color: #aaa; }
	180	}
	181	"""
	182	)
	183
	184
	185	def _render_html(trend: TrendReport) -> str:
	186	date_heads = "".join(f"<th class='date'>{escape(d)}</th>" for d in trend.dates)
	187	body_rows: list[str] = []
	188	for row in sorted(trend.rows, key=lambda r: _SEVERITY_ORDER.get(r.rule.severity, 1)):
	189	cells = "".join(
	190	f"<td class='cell {_STATUS_CLASS.get(s, 'na')}' title='{escape(s)}'>"
	191	f"{_STATUS_SYMBOL.get(s, '?')}</td>"
	192	for s in row.statuses
	193	)
	194	body_rows.append(
	195	f"<tr><td class='rulecol'>{escape(row.rule.title)}"
	196	f"<br><code>{escape(row.rule.id)}</code></td>"
	197	f"<td>{escape(row.rule.severity)}</td>{cells}</tr>"
	198	)
	199	totals = "".join(f"<td class='cell'>{n}</td>" for n in trend.fails_per_date())
	200
	201	table = (
	202	"<table class='trend'><thead><tr><th class='rulecol'>Rule</th><th>Sev</th>"
	203	f"{date_heads}</tr></thead><tbody>{''.join(body_rows)}"
	204	f"<tr class='totals'><td>Failing total</td><td></td>{totals}</tr>"
	205	"</tbody></table>"
	206	)
	207	return (
	208	"<!doctype html><html lang='en'><head><meta charset='utf-8'>"
	209	"<meta name='viewport' content='width=device-width, initial-scale=1'>"
	210	f"<title>Evidence Trend — {escape(trend.subject)}</title>"
	211	f"<style>{_TREND_CSS}</style></head><body><main>"
	212	f"<h1>Evidence Trend — {escape(trend.subject)} ({escape(trend.platform)})</h1>"
	213	f"<p class='meta'>{len(trend.dates)} packages · {escape(trend.dates[0])} → "
	214	f"{escape(trend.dates[-1])} · Generated {escape(trend.generated_at)}</p>"
	215	"<p class='legend'>✓ pass · ✗ fail · · not applicable — cell colour tracks "
	216	"each control over time.</p>"
	217	f"{table}"
	218	"<footer>Generated by audit-report · Audit Labs · evidence, not a verdict.</footer>"
	219	"</main></body></html>\n"
	220	)
	221
	222
	223	def _render_json(trend: TrendReport) -> str:
	224	import json as _json
	225
	226	payload = {
	227	"subject": trend.subject,
	228	"platform": trend.platform,
	229	"dates": trend.dates,
	230	"generated_at": trend.generated_at,
	231	"fails_per_date": trend.fails_per_date(),
	232	"rules": [
	233	{
	234	"id": row.rule.id,
	235	"title": row.rule.title,
	236	"severity": row.rule.severity,
	237	"controls": row.rule.controls,
	238	"statuses": row.statuses,
	239	}
	240	for row in trend.rows
	241	],
	242	}
	243	return _json.dumps(payload, indent=2) + "\n"
	244
	245
	246	_RENDERERS = {"md": _render_md, "html": _render_html, "json": _render_json}
	247
	248
	249	def render(trend: TrendReport, fmt: str) -> str:
	250	"""Render a trend in the named format ('md', 'html', or 'json')."""
	251	try:
	252	return _RENDERERS[fmt](trend)
	253	except KeyError:
	254	raise ValueError(f"unknown format: {fmt!r}") from None

docs/ci.md added +67

@@ -0,0 +1,67 @@
	1	# Running audit-report in CI
	2
	3	The pattern is always the same three steps:
	4
	5	1. Collect an evidence package with `audit-tools` (per-platform CLI).
	6	2. Report on it with `audit-report`, writing `md`/`html`/`json` artifacts.
	7	3. Gate the pipeline with `--fail-on` so a control regression can block a
	8	merge or page a scheduled run.
	9
	10	Ready-to-copy starting points:
	11
	12	- [`examples/github-actions-audit.yml`](../examples/github-actions-audit.yml)
	13	- [`examples/gitlab-ci-audit.yml`](../examples/gitlab-ci-audit.yml)
	14
	15	> The collection step in the examples shows both a module entrypoint
	16	> (`python -m audit_tools.github`) and a script fallback (`python audit.py`).
	17	> Use whichever your installed `audit-tools` exposes; everything downstream only
	18	> needs the `./output/<platform>_audit_<subject>_<date>/` directory it writes.
	19
	20	## Gating strategies
	21
	22	Snapshot gate — the current state must be clean.
	23
	24	```bash
	25	audit-report "$PKG" --fail-on high
	26	```
	27
	28	Exits non-zero if any high-severity control is unsupported in the newest
	29	package. Simple and strict; good for a scheduled run that should stay green.
	30
	31	Regression gate — this change must not make things worse.
	32
	33	Keep the previous package in the repo (or restore it from an artifact) and diff
	34	against it. The build fails only when a control that used to pass now fails,
	35	which avoids blocking on pre-existing debt.
	36
	37	```bash
	38	audit-report "$PKG" --baseline ./baseline/"$LAST_PKG" --fail-on high
	39	```
	40
	41	Trend artifact — show direction over time.
	42
	43	Point trend mode at a folder of retained packages to publish a heatmap of every
	44	control across dates. Its `--fail-on` reflects the latest package, so it can
	45	double as a snapshot gate while producing the timeline artifact.
	46
	47	```bash
	48	audit-report ./history --trend --format html,json --out ./trend
	49	```
	50
	51	## Exit codes
	52
	53	\| Code \| Meaning \|
	54	\| --- \| --- \|
	55	\| `0` \| Ran successfully; no `--fail-on` threshold was breached. \|
	56	\| `1` \| A finding (or, in diff mode, a regression) met the `--fail-on` severity. \|
	57	\| `2` \| Usage or input error (missing package, unknown format, mismatched platforms). \|
	58
	59	Distinguishing `1` from `2` lets a workflow tell "the audit found a problem"
	60	(expected, actionable) from "the job is misconfigured" (fix the pipeline).
	61
	62	## Retaining history
	63
	64	`audit-report` never writes back to the package — it only reads. To build a
	65	trend or a regression baseline, archive each run's package directory (a CI
	66	artifact, a committed `history/` folder, or object storage) and feed the
	67	collection back in on the next run.

examples/github-actions-audit.yml added +66

@@ -0,0 +1,66 @@
	1	# Example GitHub Actions workflow: collect evidence, then report on it.
	2	#
	3	# Copy into .github/workflows/audit.yml in the repository you want to audit and
	4	# adjust the collection step to your platform. It runs on a schedule and on
	5	# demand, produces a control-mapped report, and fails the run if a high-severity
	6	# control regresses against the previous package committed to the repo.
	7	#
	8	# Requires two org/repo secrets for the GitHub collector: AUDIT_GITHUB_TOKEN
	9	# (a read-only token for the org you audit) and the org name in AUDIT_ORG.
	10
	11	name: compliance-evidence
	12
	13	on:
	14	schedule:
	15	- cron: "0 6 * * 1" # every Monday 06:00 UTC
	16	workflow_dispatch: {}
	17
	18	permissions:
	19	contents: read
	20
	21	jobs:
	22	audit:
	23	runs-on: ubuntu-latest
	24	steps:
	25	- uses: actions/checkout@v4
	26
	27	- uses: actions/setup-python@v5
	28	with:
	29	python-version: "3.12"
	30
	31	- name: Install tools
	32	run: \|
	33	python -m pip install --upgrade pip
	34	# The reporter:
	35	pip install "audit-report @ git+https://github.com/audit-labs/audit-report"
	36	# The collector (audit-tools ships CLIs per platform):
	37	pip install "audit-tools @ git+https://github.com/audit-labs/audit-tools"
	38
	39	- name: Collect evidence (GitHub example)
	40	env:
	41	GITHUB_TOKEN: ${{ secrets.AUDIT_GITHUB_TOKEN }}
	42	GITHUB_ORG: ${{ secrets.AUDIT_ORG }}
	43	run: \|
	44	# Produces ./output/github_audit_<org>_<date>/
	45	python -m audit_tools.github --out ./output \|\| \
	46	python audit.py --out ./output # fall back to the script entrypoint
	47
	48	- name: Locate the newest package
	49	id: pkg
	50	run: echo "dir=$(ls -d ./output/_audit_ \| sort \| tail -n1)" >> "$GITHUB_OUTPUT"
	51
	52	- name: Generate evidence report
	53	run: \|
	54	audit-report "${{ steps.pkg.outputs.dir }}" \
	55	--format md,html,json --out ./report
	56
	57	- name: Fail on any high-severity finding
	58	run: audit-report "${{ steps.pkg.outputs.dir }}" --fail-on high
	59
	60	- name: Publish the report as a build artifact
	61	if: always()
	62	uses: actions/upload-artifact@v4
	63	with:
	64	name: evidence-report
	65	path: report/
	66	retention-days: 90

examples/gitlab-ci-audit.yml added +33

@@ -0,0 +1,33 @@
	1	# Example GitLab CI configuration: collect evidence, then report on it.
	2	#
	3	# Copy into .gitlab-ci.yml (or include it) in the project you want to audit.
	4	# It produces a control-mapped report as a job artifact and fails the pipeline
	5	# if any high-severity control is not supported.
	6	#
	7	# Set CI/CD variables GITLAB_TOKEN (read-only) and GITLAB_GROUP for the group
	8	# you audit.
	9
	10	stages: [audit]
	11
	12	compliance-evidence:
	13	stage: audit
	14	image: python:3.12-slim
	15	rules:
	16	- if: $CI_PIPELINE_SOURCE == "schedule"
	17	- if: $CI_PIPELINE_SOURCE == "web" # manual "Run pipeline"
	18	variables:
	19	PIP_DISABLE_PIP_VERSION_CHECK: "1"
	20	before_script:
	21	- pip install "audit-report @ git+https://github.com/audit-labs/audit-report"
	22	- pip install "audit-tools @ git+https://github.com/audit-labs/audit-tools"
	23	script:
	24	# Produces ./output/gitlab_audit_<group>_<date>/
	25	- python -m audit_tools.gitlab --out ./output \|\| python audit.py --out ./output
	26	- PKG=$(ls -d ./output/_audit_ \| sort \| tail -n1)
	27	- audit-report "$PKG" --format md,html,json --out ./report
	28	- audit-report "$PKG" --fail-on high
	29	artifacts:
	30	when: always
	31	paths:
	32	- report/
	33	expire_in: 90 days

tests/fixtures/series/aws_audit_prod_2026-01-01/account_security.csv added +2

@@ -0,0 +1,2 @@
	1	root_mfa_enabled,root_access_keys_present,root_signing_certs_present,mfa_devices,users,groups,roles,policies
	2	False,True,False,0,1,0,3,0

tests/fixtures/series/aws_audit_prod_2026-01-01/cloudtrail.csv added +2

@@ -0,0 +1,2 @@
	1	name,home_region,multi_region,log_file_validation,is_logging,s3_bucket
	2	main,us-east-1,False,False,False,

tests/fixtures/series/aws_audit_prod_2026-01-01/iam_users.csv added +2

@@ -0,0 +1,2 @@
	1	user,mfa_enabled,access_keys,oldest_key_age_days,console_password,password_last_used,created
	2	bob,False,1,400,True,2025-11-01T00:00:00+00:00,2025-02-01T00:00:00+00:00

tests/fixtures/series/aws_audit_prod_2026-01-01/open_security_groups.csv added +2

@@ -0,0 +1,2 @@
	1	region,group_id,group_name,protocol,from_port,to_port,open_to
	2	us-east-1,sg-web,web,tcp,22,22,0.0.0.0/0

tests/fixtures/series/aws_audit_prod_2026-01-01/password_policy.csv added +2

@@ -0,0 +1,2 @@
	1	minimum_length,require_symbols,require_numbers,require_uppercase,require_lowercase,allow_users_to_change,max_age_days,reuse_prevention,hard_expiry
	2	8,False,True,True,True,True,365,0,False

tests/fixtures/series/aws_audit_prod_2026-01-01/s3_public_access.csv added +2

@@ -0,0 +1,2 @@
	1	bucket,region,public_access_block,policy_public,acl_public
	2	acme-assets,us-east-1,False,True,False

tests/fixtures/series/aws_audit_prod_2026-02-01/account_security.csv added +2

@@ -0,0 +1,2 @@
	1	root_mfa_enabled,root_access_keys_present,root_signing_certs_present,mfa_devices,users,groups,roles,policies
	2	True,True,False,1,1,0,3,0

tests/fixtures/series/aws_audit_prod_2026-02-01/cloudtrail.csv added +2

@@ -0,0 +1,2 @@
	1	name,home_region,multi_region,log_file_validation,is_logging,s3_bucket
	2	main,us-east-1,True,True,True,acme-logs

tests/fixtures/series/aws_audit_prod_2026-02-01/iam_users.csv added +2

@@ -0,0 +1,2 @@
	1	user,mfa_enabled,access_keys,oldest_key_age_days,console_password,password_last_used,created
	2	bob,True,1,400,True,2026-01-15T00:00:00+00:00,2025-02-01T00:00:00+00:00

tests/fixtures/series/aws_audit_prod_2026-02-01/open_security_groups.csv added +2

@@ -0,0 +1,2 @@
	1	region,group_id,group_name,protocol,from_port,to_port,open_to
	2	us-east-1,sg-web,web,tcp,22,22,0.0.0.0/0

tests/fixtures/series/aws_audit_prod_2026-02-01/password_policy.csv added +2

@@ -0,0 +1,2 @@
	1	minimum_length,require_symbols,require_numbers,require_uppercase,require_lowercase,allow_users_to_change,max_age_days,reuse_prevention,hard_expiry
	2	14,True,True,True,True,True,90,24,False

tests/fixtures/series/aws_audit_prod_2026-02-01/s3_public_access.csv added +1

	@@ -0,0 +1 @@
		1	bucket,region,public_access_block,policy_public,acl_public

tests/fixtures/series/aws_audit_prod_2026-03-01/account_security.csv added +2

@@ -0,0 +1,2 @@
	1	root_mfa_enabled,root_access_keys_present,root_signing_certs_present,mfa_devices,users,groups,roles,policies
	2	True,False,False,1,1,0,3,0

tests/fixtures/series/aws_audit_prod_2026-03-01/cloudtrail.csv added +2

@@ -0,0 +1,2 @@
	1	name,home_region,multi_region,log_file_validation,is_logging,s3_bucket
	2	main,us-east-1,True,True,True,acme-logs

tests/fixtures/series/aws_audit_prod_2026-03-01/iam_users.csv added +2

@@ -0,0 +1,2 @@
	1	user,mfa_enabled,access_keys,oldest_key_age_days,console_password,password_last_used,created
	2	bob,True,1,30,True,2026-02-20T00:00:00+00:00,2025-02-01T00:00:00+00:00

tests/fixtures/series/aws_audit_prod_2026-03-01/open_security_groups.csv added +2

@@ -0,0 +1,2 @@
	1	region,group_id,group_name,protocol,from_port,to_port,open_to
	2	us-east-1,sg-db,db,tcp,443,443,10.0.0.0/8

tests/fixtures/series/aws_audit_prod_2026-03-01/password_policy.csv added +2

@@ -0,0 +1,2 @@
	1	minimum_length,require_symbols,require_numbers,require_uppercase,require_lowercase,allow_users_to_change,max_age_days,reuse_prevention,hard_expiry
	2	14,True,True,True,True,True,90,24,False

tests/fixtures/series/aws_audit_prod_2026-03-01/s3_public_access.csv added +1

	@@ -0,0 +1 @@
		1	bucket,region,public_access_block,policy_public,acl_public

tests/test_trend.py added +115

@@ -0,0 +1,115 @@
	1	"""Tests for trend mode: discovery, timeline building, rendering, and the CLI."""
	2
	3	import json
	4	from pathlib import Path
	5
	6	import pytest
	7
	8	from audit_report import trend
	9	from audit_report.cli import main
	10	from audit_report.engine import FAIL, PASS, evaluate
	11	from audit_report.loader import load_package
	12	from audit_report.rules import load_ruleset
	13
	14	FIXTURES = Path(__file__).parent / "fixtures"
	15	SERIES = FIXTURES / "series"
	16	RULESETS = Path("audit_report/rulesets")
	17
	18
	19	def _build():
	20	platform, _subject, paths = trend.discover(SERIES)
	21	ruleset = load_ruleset(RULESETS / f"{platform}.yaml")
	22	packages = [load_package(p) for p in paths]
	23	findings = [evaluate(pkg, ruleset) for pkg in packages]
	24	return trend.build_trend(packages, findings)
	25
	26
	27	def test_discover_orders_by_date():
	28	platform, subject, paths = trend.discover(SERIES)
	29	assert (platform, subject) == ("aws", "prod")
	30	assert [p.name for p in paths] == [
	31	"aws_audit_prod_2026-01-01",
	32	"aws_audit_prod_2026-02-01",
	33	"aws_audit_prod_2026-03-01",
	34	]
	35
	36
	37	def test_discover_rejects_mixed_series():
	38	# The fixtures root holds aws/github/gitlab packages for several subjects.
	39	with pytest.raises(ValueError, match="multiple series"):
	40	trend.discover(FIXTURES)
	41
	42
	43	def test_discover_requires_two(tmp_path):
	44	# A parent directory holding a single package is not a trend.
	45	pkg = tmp_path / "aws_audit_solo_2026-01-01"
	46	pkg.mkdir()
	47	(pkg / "account_security.csv").write_text("root_mfa_enabled\nTrue\n", encoding="utf-8")
	48	with pytest.raises(ValueError, match="at least two"):
	49	trend.discover(tmp_path)
	50
	51
	52	def test_discover_no_packages(tmp_path):
	53	with pytest.raises(ValueError, match="no audit-tools packages"):
	54	trend.discover(tmp_path)
	55
	56
	57	def test_trend_timeline_and_totals():
	58	report = _build()
	59	assert report.dates == ["2026-01-01", "2026-02-01", "2026-03-01"]
	60	assert report.fails_per_date() == [8, 3, 0]
	61
	62	by_id = {row.rule.id: row for row in report.rows}
	63	# Root MFA: fail, then fixed and stays fixed.
	64	assert by_id["aws.root.mfa"].statuses == [FAIL, PASS, PASS]
	65	# Access-key rotation lags: fixed only in the final package.
	66	assert by_id["aws.iam.key-rotation"].statuses == [FAIL, FAIL, PASS]
	67
	68
	69	def test_trend_row_transitions():
	70	report = _build()
	71	by_id = {row.rule.id: row for row in report.rows}
	72	assert by_id["aws.root.mfa"].transitions == 1 # one fail->pass change
	73	assert by_id["aws.s3.no-public-access"].transitions == 1
	74
	75
	76	def test_trend_render_markdown():
	77	md = trend.render(_build(), "md")
	78	assert "# Evidence Trend — prod (aws)" in md
	79	assert "Failing total" in md
	80	assert "2026-03-01" in md
	81
	82
	83	def test_trend_render_html_self_contained():
	84	html = trend.render(_build(), "html")
	85	assert html.startswith("<!doctype html>")
	86	assert "http://" not in html and "https://" not in html
	87	assert "class='trend'" in html
	88
	89
	90	def test_trend_render_json():
	91	data = json.loads(trend.render(_build(), "json"))
	92	assert data["dates"] == ["2026-01-01", "2026-02-01", "2026-03-01"]
	93	assert data["fails_per_date"] == [8, 3, 0]
	94	rules = {r["id"]: r["statuses"] for r in data["rules"]}
	95	assert rules["aws.root.mfa"] == ["fail", "pass", "pass"]
	96
	97
	98	def test_cli_trend_mode(tmp_path):
	99	out = tmp_path / "out"
	100	code = main([str(SERIES), "--trend", "--format", "md,html,json", "--out", str(out)])
	101	assert (out / "trend.md").exists()
	102	assert (out / "trend.html").exists()
	103	assert (out / "trend.json").exists()
	104	# The latest package is clean, so default --fail-on none exits 0.
	105	assert code == 0
	106
	107
	108	def test_cli_trend_fail_on_uses_latest(tmp_path):
	109	# Latest package (2026-03) has no failures, so even --fail-on low passes.
	110	code = main([str(SERIES), "--trend", "--format", "json", "--out", str(tmp_path), "--fail-on", "low"])
	111	assert code == 0
	112
	113
	114	def test_cli_trend_and_baseline_conflict():
	115	assert main([str(SERIES), "--trend", "--baseline", str(SERIES), "--format", "json"]) == 2

Commit 92ba96f545

Commit `92ba96f545`