audit-labs/audit-tools

A collection of scripts, queries, and other goodies you can use in an audit. audit automation compliance evidence scripts

Commit b37f594dee

b37f594dee47db2d794e05d44260cd880ea46b3a

parent: 8c281c79e7

Unsigned

cmc <hello@cleberg.net> · 2026-10-30 00:23 UTC

Qlty config and code cleanup (!1)

- Add Qlty config; split sampling validation and pass run facts
- Qlty: disable function_complexity

Layout: unified · split

.qlty/.gitignore added +7
@@ -0,0 +1,7 @@
1*
2!configs
3!configs/**
4!hooks
5!hooks/**
6!qlty.toml
7!.gitignore
.qlty/qlty.toml added +50
@@ -0,0 +1,50 @@
1config_version = "0"
2
3exclude_patterns = [
4 "**/vendor/**",
5 "**/node_modules/**",
6 "**/testdata/**",
7 "**/dist/**",
8 "**/.build/**",
9 "**/build/**",
10 "**/*.min.*",
11 "**/generated/**",
12 ".claude/**",
13 "**/Pods/**",
14 # Each application under applications/ runs standalone with its own
15 # reporters package, so the GitHub and GitLab CSV reporters are copies.
16 "applications/github/reporters/csv_reporter.py",
17 "applications/gitlab/reporters/csv_reporter.py",
18]
19
20test_patterns = [
21 "**/test/**",
22 "**/tests/**",
23 "**/Tests/**",
24 "**/*_test.*",
25 "**/test_*.*",
26 "**/*.test.*",
27 "**/*Tests.swift",
28 "e2e/**",
29]
30
31[[source]]
32name = "default"
33default = true
34
35[[plugin]]
36name = "osv-scanner"
37
38[[plugin]]
39name = "trufflehog"
40
41# Early returns are guard clauses, which keep the happy path unindented.
42[smells.return_statements]
43enabled = false
44
45# Function and file complexity are left to review rather than counted.
46[smells.function_complexity]
47enabled = false
48
49[smells.file_complexity]
50enabled = false
applications/github/collectors/audit_log.py +2 −7
@@ -162,13 +162,8 @@ def _dedupe_events(events):
162162
163163def _branch_or_pattern(event):
164164 """Find the most useful target label for branch and ruleset audit events."""
165 return (
166 event.get("name")
167 or event.get("branch")
168 or event.get("ruleset_name")
169 or event.get("ruleset_old_name")
170 or ""
171 )
165 keys = ("name", "branch", "ruleset_name", "ruleset_old_name")
166 return next((event[key] for key in keys if event.get(key)), "")
172167
173168
174169def _event_summary(event):
sampling/sampling_tool/cli.py +33 −54
@@ -4,12 +4,14 @@ from __future__ import annotations
44
55import sys
66from argparse import ArgumentParser, Namespace
7from dataclasses import dataclass
78from datetime import datetime, timezone
89from pathlib import Path
910from types import SimpleNamespace
1011
1112import pandas as pd
1213
14from .facts import RunFacts
1315from .filters import apply_filters, parse_filters
1416from .io import AuditSamplingError, load_population, sha256_file, write_csv
1517from .manifest import build_manifest, write_manifest
@@ -217,16 +219,8 @@ def run(options) -> Path:
217219 sampled, options, random_seed, timestamp, options.stratify_column
218220 )
219221
220 _write_outputs(
221 run_dir,
222 options,
223 validated,
224 excluded_rows,
225 duplicate_rows,
226 sample,
227 strata_rows,
228 output_files,
229 )
222 frames = RunFrames(validated, excluded_rows, duplicate_rows, sample, strata_rows)
223 _write_outputs(run_dir, options, frames, output_files)
230224 logger.log("final status: success")
231225 print("Audit sampling run completed.")
232226 except AuditSamplingError as exc:
@@ -243,48 +237,29 @@ def run(options) -> Path:
243237 logger.log("final status: failed")
244238 raise
245239 finally:
246 reconciliation = build_reconciliation(
247 source_rows=len(source),
248 blank_id_count=blank_id_count,
249 duplicate_id_count=duplicate_id_count or len(duplicate_rows),
250 excluded_rows=len(excluded_rows),
251 validated_rows=len(validated),
252 requested_sample_size=_requested_sample_size(options, strata_rows),
253 final_sample_size=len(sample),
254 )
255 write_csv(reconciliation, run_dir / "population_reconciliation.csv")
256 _track(output_files, "population_reconciliation.csv")
257 methodology = build_methodology(
240 facts = RunFacts(
258241 input_file=input_path,
259 input_sheet=options.sheet,
260242 input_sha256=input_hash,
261 source_row_count=len(source),
243 input_sheet=options.sheet,
262244 options=options,
263245 effective_id_column=effective_id_column,
264246 id_column_omitted=id_column_omitted,
247 source_row_count=len(source),
248 validated_population_count=len(validated),
249 excluded_row_count=len(excluded_rows),
265250 duplicate_id_count=duplicate_id_count or len(duplicate_rows),
266251 blank_id_count=blank_id_count,
252 requested_sample_size=_requested_sample_size(options, strata_rows),
267253 sample_size_actual=len(sample),
268254 random_seed=random_seed,
269 strata_summary=strata_rows,
270255 )
271 (run_dir / "methodology.txt").write_text(methodology)
256 write_csv(build_reconciliation(facts), run_dir / "population_reconciliation.csv")
257 _track(output_files, "population_reconciliation.csv")
258 (run_dir / "methodology.txt").write_text(build_methodology(facts, strata_rows))
272259 _track(output_files, "methodology.txt")
273260 manifest = build_manifest(
261 facts,
274262 run_timestamp_utc=timestamp,
275 input_file=input_path,
276 input_sha256=input_hash,
277 input_sheet=options.sheet,
278 options=options,
279 effective_id_column=effective_id_column,
280 id_column_omitted=id_column_omitted,
281 sample_size_actual=len(sample),
282 random_seed=random_seed,
283 source_row_count=len(source),
284 validated_population_count=len(validated),
285 excluded_row_count=len(excluded_rows),
286 duplicate_id_count=duplicate_id_count or len(duplicate_rows),
287 blank_id_count=blank_id_count,
288263 output_files=sorted(output_files + ["run.log", "manifest.json"]),
289264 )
290265 write_manifest(manifest, run_dir / "manifest.json")
@@ -310,29 +285,33 @@ def add_sample_metadata(
310285 return output
311286
312287
288@dataclass
289class RunFrames:
290 """The tables a successful run writes."""
291
292 validated: pd.DataFrame
293 excluded_rows: pd.DataFrame
294 duplicate_rows: pd.DataFrame
295 sample: pd.DataFrame
296 strata_rows: list[dict[str, object]]
297
298
313299def _write_outputs(
314 run_dir: Path,
315 options,
316 validated: pd.DataFrame,
317 excluded_rows: pd.DataFrame,
318 duplicate_rows: pd.DataFrame,
319 sample: pd.DataFrame,
320 strata_rows: list[dict[str, object]],
321 output_files: list[str],
300 run_dir: Path, options, frames: RunFrames, output_files: list[str]
322301) -> None:
323 write_csv(validated, run_dir / _POPULATION_VALIDATED_CSV)
302 write_csv(frames.validated, run_dir / _POPULATION_VALIDATED_CSV)
324303 _track(output_files, _POPULATION_VALIDATED_CSV)
325304 if options.method in {"random", "systematic", "stratified"}:
326 write_csv(sample, run_dir / "sample.csv")
305 write_csv(frames.sample, run_dir / "sample.csv")
327306 _track(output_files, "sample.csv")
328 if not excluded_rows.empty:
329 write_csv(excluded_rows, run_dir / _EXCLUDED_ROWS_CSV)
307 if not frames.excluded_rows.empty:
308 write_csv(frames.excluded_rows, run_dir / _EXCLUDED_ROWS_CSV)
330309 _track(output_files, _EXCLUDED_ROWS_CSV)
331 if not duplicate_rows.empty:
332 write_csv(duplicate_rows, run_dir / _DUPLICATE_IDS_CSV)
310 if not frames.duplicate_rows.empty:
311 write_csv(frames.duplicate_rows, run_dir / _DUPLICATE_IDS_CSV)
333312 _track(output_files, _DUPLICATE_IDS_CSV)
334313 if options.method == "stratified":
335 write_csv(build_strata_summary(strata_rows), run_dir / "strata_summary.csv")
314 write_csv(build_strata_summary(frames.strata_rows), run_dir / "strata_summary.csv")
336315 _track(output_files, "strata_summary.csv")
337316
338317
sampling/sampling_tool/facts.py added +25
@@ -0,0 +1,25 @@
1"""Facts about one sampling run, shared by its manifest, methodology, and
2reconciliation outputs."""
3
4from __future__ import annotations
5
6from dataclasses import dataclass
7from pathlib import Path
8
9
10@dataclass
11class RunFacts:
12 input_file: Path
13 input_sha256: str
14 input_sheet: str | None
15 options: object
16 effective_id_column: str | None
17 id_column_omitted: bool
18 source_row_count: int
19 validated_population_count: int
20 excluded_row_count: int
21 duplicate_id_count: int
22 blank_id_count: int
23 requested_sample_size: int | None
24 sample_size_actual: int
25 random_seed: int | None
sampling/sampling_tool/manifest.py +15 −28
@@ -6,52 +6,39 @@ import json
66from pathlib import Path
77
88from . import __version__, engine
9from .facts import RunFacts
910
1011
1112def build_manifest(
12 *,
13 run_timestamp_utc: str,
14 input_file: Path,
15 input_sha256: str,
16 input_sheet: str | None,
17 options,
18 effective_id_column: str | None,
19 id_column_omitted: bool,
20 sample_size_actual: int,
21 random_seed: int | None,
22 source_row_count: int,
23 validated_population_count: int,
24 excluded_row_count: int,
25 duplicate_id_count: int,
26 blank_id_count: int,
27 output_files: list[str],
13 facts: RunFacts, *, run_timestamp_utc: str, output_files: list[str]
2814) -> dict[str, object]:
15 options = facts.options
2916 return {
3017 "tool": "audit_sample",
3118 "version": __version__,
3219 "engine": engine.ENGINE,
3320 "engine_version": engine.ENGINE_VERSION,
3421 "run_timestamp_utc": run_timestamp_utc,
35 "input_file": str(input_file),
36 "input_sha256": input_sha256,
37 "input_sheet": input_sheet,
22 "input_file": str(facts.input_file),
23 "input_sha256": facts.input_sha256,
24 "input_sheet": facts.input_sheet,
3825 "method": options.method,
3926 "id_column": options.id_column,
40 "effective_id_column": effective_id_column,
41 "id_column_omitted": id_column_omitted,
27 "effective_id_column": facts.effective_id_column,
28 "id_column_omitted": facts.id_column_omitted,
4229 "stratify_column": options.stratify_column,
4330 "sample_size_requested": options.sample_size,
44 "sample_size_actual": sample_size_actual,
45 "random_seed": random_seed,
31 "sample_size_actual": facts.sample_size_actual,
32 "random_seed": facts.random_seed,
4633 "filters": options.filters,
4734 "dedupe_id": options.dedupe_id,
4835 "exclude_blank_id": options.exclude_blank_id,
4936 "allow_shortfall": options.allow_shortfall,
50 "source_row_count": source_row_count,
51 "validated_population_count": validated_population_count,
52 "excluded_row_count": excluded_row_count,
53 "duplicate_id_count": duplicate_id_count,
54 "blank_id_count": blank_id_count,
37 "source_row_count": facts.source_row_count,
38 "validated_population_count": facts.validated_population_count,
39 "excluded_row_count": facts.excluded_row_count,
40 "duplicate_id_count": facts.duplicate_id_count,
41 "blank_id_count": facts.blank_id_count,
5542 "output_files": output_files,
5643 }
5744
sampling/sampling_tool/reconciliation.py +11 −16
@@ -4,28 +4,23 @@ from __future__ import annotations
44
55import pandas as pd
66
7from .facts import RunFacts
78
8def build_reconciliation(
9 source_rows: int,
10 blank_id_count: int,
11 duplicate_id_count: int,
12 excluded_rows: int,
13 validated_rows: int,
14 requested_sample_size: int | None,
15 final_sample_size: int,
16) -> pd.DataFrame:
17 unsampled = validated_rows - final_sample_size
9
10def build_reconciliation(facts: RunFacts) -> pd.DataFrame:
11 unsampled = facts.validated_population_count - facts.sample_size_actual
12 requested_sample_size = facts.requested_sample_size
1813 rows = [
19 ("Source rows", source_rows),
20 ("Rows with blank ID", blank_id_count),
21 ("Duplicate IDs", duplicate_id_count),
22 ("Excluded rows", excluded_rows),
23 ("Validated population rows", validated_rows),
14 ("Source rows", facts.source_row_count),
15 ("Rows with blank ID", facts.blank_id_count),
16 ("Duplicate IDs", facts.duplicate_id_count),
17 ("Excluded rows", facts.excluded_row_count),
18 ("Validated population rows", facts.validated_population_count),
2419 (
2520 "Requested sample size",
2621 "" if requested_sample_size is None else requested_sample_size,
2722 ),
28 ("Final sample size", final_sample_size),
23 ("Final sample size", facts.sample_size_actual),
2924 ("Unsampled population rows", unsampled),
3025 ]
3126 return pd.DataFrame(rows, columns=["Metric", "Value"])
sampling/sampling_tool/reporting.py +14 −23
@@ -4,6 +4,8 @@ from __future__ import annotations
44
55from pathlib import Path
66
7from .facts import RunFacts
8
79
810class RunLogger:
911 def __init__(self) -> None:
@@ -23,29 +25,18 @@ class RunLogger:
2325
2426
2527def build_methodology(
26 *,
27 input_file: Path,
28 input_sheet: str | None,
29 input_sha256: str,
30 source_row_count: int,
31 options,
32 effective_id_column: str | None,
33 id_column_omitted: bool,
34 duplicate_id_count: int,
35 blank_id_count: int,
36 sample_size_actual: int,
37 random_seed: int | None,
38 strata_summary: list[dict[str, object]] | None,
28 facts: RunFacts, strata_summary: list[dict[str, object]] | None
3929) -> str:
30 options = facts.options
4031 lines = [
4132 "Audit Sampling Methodology",
4233 "",
43 f"Source file: {input_file}",
44 f"Source sheet: {input_sheet or ''}",
45 f"Input SHA-256: {input_sha256}",
46 f"Source row count: {source_row_count}",
34 f"Source file: {facts.input_file}",
35 f"Source sheet: {facts.input_sheet or ''}",
36 f"Input SHA-256: {facts.input_sha256}",
37 f"Source row count: {facts.source_row_count}",
4738 ]
48 if id_column_omitted:
39 if facts.id_column_omitted:
4940 lines.append(
5041 "ID column: omitted; _audit_row_id was generated from _source_row_number."
5142 )
@@ -53,15 +44,15 @@ def build_methodology(
5344 lines.append(f"ID column: {options.id_column}")
5445 lines.extend(
5546 [
56 f"Effective ID column: {effective_id_column or ''}",
57 f"Duplicate ID rows identified: {duplicate_id_count}",
58 f"Blank ID rows identified: {blank_id_count}",
47 f"Effective ID column: {facts.effective_id_column or ''}",
48 f"Duplicate ID rows identified: {facts.duplicate_id_count}",
49 f"Blank ID rows identified: {facts.blank_id_count}",
5950 f"Blank ID handling: {'excluded' if options.exclude_blank_id else 'retained'}",
6051 f"Filters applied: {options.filters or {}}",
6152 f"Sampling method: {options.method}",
6253 f"Sample size requested: {options.sample_size or ''}",
63 f"Sample size selected: {sample_size_actual}",
64 f"Random seed used: {random_seed or ''}",
54 f"Sample size selected: {facts.sample_size_actual}",
55 f"Random seed used: {facts.random_seed or ''}",
6556 ]
6657 )
6758 if options.method == "stratified":
sampling/sampling_tool/validation.py +109 −88
@@ -33,100 +33,20 @@ def validate_and_prepare(population: pd.DataFrame, options) -> ValidationResult:
3333 excluded_parts: list[pd.DataFrame] = []
3434 working = population.copy()
3535
36 if options.id_column:
37 if options.id_column not in working.columns:
38 raise AuditSamplingError(f"ID column not found: {options.id_column}")
39 effective_id_column = options.id_column
40 id_column_omitted = False
41 else:
42 effective_id_column = "_audit_row_id"
43 id_column_omitted = True
44 working[effective_id_column] = working["_source_row_number"]
45 warnings.append(
46 "--id-column omitted; using _source_row_number as generated _audit_row_id."
47 )
48
36 effective_id_column, id_column_omitted = _resolve_id_column(working, options, warnings)
4937 if options.stratify_column and options.stratify_column not in working.columns:
5038 raise AuditSamplingError(
5139 f"Stratification column not found: {options.stratify_column}"
5240 )
5341
54 id_blank_mask = blank_id_mask(working[effective_id_column])
55 blank_id_count = int(id_blank_mask.sum())
56 if blank_id_count:
57 if options.exclude_blank_id:
58 excluded = working.loc[id_blank_mask].copy()
59 excluded["_exclusion_reason"] = "Blank ID"
60 excluded_parts.append(excluded)
61 working = working.loc[~id_blank_mask].copy()
62 else:
63 warnings.append(
64 f"{blank_id_count} row(s) have blank IDs and were retained."
65 )
66
67 nonblank_ids = ~blank_id_mask(working[effective_id_column])
68 duplicate_mask = working.loc[nonblank_ids, effective_id_column].duplicated(
69 keep=False
42 working, blank_id_count = _handle_blank_ids(
43 working, effective_id_column, options, warnings, excluded_parts
7044 )
71 duplicate_rows = working.loc[nonblank_ids].loc[duplicate_mask].copy()
72 duplicate_id_count = len(duplicate_rows)
73 if duplicate_id_count:
74 if options.dedupe_id == "fail":
75 raise AuditSamplingError(
76 f"Duplicate IDs found in '{effective_id_column}'. "
77 "See duplicate_ids.csv.",
78 duplicate_rows=duplicate_rows,
79 )
80 keep = "first" if options.dedupe_id == "first" else "last"
81 drop_mask = working[effective_id_column].duplicated(keep=keep) & ~blank_id_mask(
82 working[effective_id_column]
83 )
84 excluded = working.loc[drop_mask].copy()
85 excluded["_exclusion_reason"] = f"Duplicate ID removed by dedupe={keep}"
86 excluded_parts.append(excluded)
87 working = working.loc[~drop_mask].copy()
88 warnings.append(
89 f"{len(excluded)} duplicate ID row(s) removed using dedupe={keep}."
90 )
91
92 if options.sample_size is not None and options.sample_size <= 0:
93 raise AuditSamplingError("--sample-size must be a positive integer.")
94 needs_sample_size = options.method in ("random", "systematic") or (
95 options.method == "stratified" and bool(options.strata_proportions)
45 working, duplicate_rows = _handle_duplicate_ids(
46 working, effective_id_column, options, warnings, excluded_parts
9647 )
97 if needs_sample_size:
98 if options.sample_size is None:
99 raise AuditSamplingError(f"--sample-size is required for {options.method}.")
100 if options.sample_size > len(working) and not (
101 options.method == "stratified" and options.allow_shortfall
102 ):
103 raise AuditSamplingError(
104 "--sample-size cannot exceed the validated population size."
105 )
106
107 strata_counts: dict[str, int] = {}
108 strata_proportions: dict[str, float] = {}
109 if options.method == "stratified":
110 if not options.stratify_column:
111 raise AuditSamplingError("--stratify-column is required for stratified.")
112 has_counts = bool(options.strata_counts)
113 has_proportions = bool(options.strata_proportions)
114 if has_counts == has_proportions:
115 raise AuditSamplingError(
116 "Use exactly one of --strata-counts or --strata-proportions."
117 )
118 strata_counts = parse_key_ints(options.strata_counts, "strata")
119 strata_proportions = parse_key_floats(options.strata_proportions, "strata")
120 requested_strata = set(strata_counts or strata_proportions)
121 actual_strata = set(working[options.stratify_column].dropna().astype(str))
122 missing = sorted(requested_strata - actual_strata)
123 if missing:
124 raise AuditSamplingError(
125 "Requested strata not found in population: " + ", ".join(missing)
126 )
127
128 if strata_counts:
129 _validate_stratum_counts_fit(working, options, strata_counts)
48 _check_sample_size(len(working), options)
49 strata_counts, strata_proportions = _validate_strata(working, options)
13050
13151 if excluded_parts:
13252 excluded_rows = pd.concat(excluded_parts, ignore_index=True)
@@ -140,13 +60,114 @@ def validate_and_prepare(population: pd.DataFrame, options) -> ValidationResult:
14060 effective_id_column=effective_id_column,
14161 id_column_omitted=id_column_omitted,
14262 blank_id_count=blank_id_count,
143 duplicate_id_count=duplicate_id_count,
63 duplicate_id_count=len(duplicate_rows),
14464 warnings=warnings,
14565 strata_counts=strata_counts,
14666 strata_proportions=strata_proportions,
14767 )
14868
14969
70def _resolve_id_column(working: pd.DataFrame, options, warnings: list[str]) -> tuple[str, bool]:
71 """The ID column to use, generating _audit_row_id when none was given."""
72 if options.id_column:
73 if options.id_column not in working.columns:
74 raise AuditSamplingError(f"ID column not found: {options.id_column}")
75 return options.id_column, False
76 working["_audit_row_id"] = working["_source_row_number"]
77 warnings.append(
78 "--id-column omitted; using _source_row_number as generated _audit_row_id."
79 )
80 return "_audit_row_id", True
81
82
83def _handle_blank_ids(working, id_column, options, warnings, excluded_parts):
84 """Exclude or warn about rows with blank IDs; returns (working, count)."""
85 id_blank_mask = blank_id_mask(working[id_column])
86 blank_id_count = int(id_blank_mask.sum())
87 if not blank_id_count:
88 return working, 0
89 if options.exclude_blank_id:
90 excluded = working.loc[id_blank_mask].copy()
91 excluded["_exclusion_reason"] = "Blank ID"
92 excluded_parts.append(excluded)
93 working = working.loc[~id_blank_mask].copy()
94 else:
95 warnings.append(
96 f"{blank_id_count} row(s) have blank IDs and were retained."
97 )
98 return working, blank_id_count
99
100
101def _handle_duplicate_ids(working, id_column, options, warnings, excluded_parts):
102 """Fail on, or dedupe, repeated non-blank IDs; returns (working, duplicates)."""
103 nonblank_ids = ~blank_id_mask(working[id_column])
104 duplicate_mask = working.loc[nonblank_ids, id_column].duplicated(keep=False)
105 duplicate_rows = working.loc[nonblank_ids].loc[duplicate_mask].copy()
106 if not len(duplicate_rows):
107 return working, duplicate_rows
108 if options.dedupe_id == "fail":
109 raise AuditSamplingError(
110 f"Duplicate IDs found in '{id_column}'. "
111 "See duplicate_ids.csv.",
112 duplicate_rows=duplicate_rows,
113 )
114 keep = "first" if options.dedupe_id == "first" else "last"
115 drop_mask = working[id_column].duplicated(keep=keep) & ~blank_id_mask(
116 working[id_column]
117 )
118 excluded = working.loc[drop_mask].copy()
119 excluded["_exclusion_reason"] = f"Duplicate ID removed by dedupe={keep}"
120 excluded_parts.append(excluded)
121 warnings.append(
122 f"{len(excluded)} duplicate ID row(s) removed using dedupe={keep}."
123 )
124 return working.loc[~drop_mask].copy(), duplicate_rows
125
126
127def _check_sample_size(population_size: int, options) -> None:
128 if options.sample_size is not None and options.sample_size <= 0:
129 raise AuditSamplingError("--sample-size must be a positive integer.")
130 needs_sample_size = options.method in ("random", "systematic") or (
131 options.method == "stratified" and bool(options.strata_proportions)
132 )
133 if not needs_sample_size:
134 return
135 if options.sample_size is None:
136 raise AuditSamplingError(f"--sample-size is required for {options.method}.")
137 if options.sample_size > population_size and not (
138 options.method == "stratified" and options.allow_shortfall
139 ):
140 raise AuditSamplingError(
141 "--sample-size cannot exceed the validated population size."
142 )
143
144
145def _validate_strata(working, options) -> tuple[dict[str, int], dict[str, float]]:
146 """Parse and check --strata-counts / --strata-proportions for stratified runs."""
147 if options.method != "stratified":
148 return {}, {}
149 if not options.stratify_column:
150 raise AuditSamplingError("--stratify-column is required for stratified.")
151 has_counts = bool(options.strata_counts)
152 has_proportions = bool(options.strata_proportions)
153 if has_counts == has_proportions:
154 raise AuditSamplingError(
155 "Use exactly one of --strata-counts or --strata-proportions."
156 )
157 strata_counts = parse_key_ints(options.strata_counts, "strata")
158 strata_proportions = parse_key_floats(options.strata_proportions, "strata")
159 requested_strata = set(strata_counts or strata_proportions)
160 actual_strata = set(working[options.stratify_column].dropna().astype(str))
161 missing = sorted(requested_strata - actual_strata)
162 if missing:
163 raise AuditSamplingError(
164 "Requested strata not found in population: " + ", ".join(missing)
165 )
166 if strata_counts:
167 _validate_stratum_counts_fit(working, options, strata_counts)
168 return strata_counts, strata_proportions
169
170
150171def _validate_stratum_counts_fit(population, options, counts: dict[str, int]) -> None:
151172 for stratum, requested in counts.items():
152173 available = int((population[options.stratify_column] == stratum).sum())
tui/aws_runner.py +12 −3
@@ -13,6 +13,7 @@ there is no per-item cache, so every check is called as ``fn(cfg)``.
1313import os
1414import sys
1515from collections.abc import Iterable
16from dataclasses import dataclass
1617from datetime import date
1718
1819from tui.common import Check, ProgressCallback, ProgressEvent
@@ -89,14 +90,21 @@ def default_output_dir(out: str, profile: str) -> str:
8990 return os.path.join(out, f"aws_audit_{subject}_{date.today().isoformat()}")
9091
9192
93@dataclass(frozen=True)
94class AwsTarget:
95 """The AWS profile, region, and account to audit."""
96
97 profile: str
98 region: str
99 account: str
100
101
92102# --- Runner -----------------------------------------------------------------
93103
94104
95105def run_audit(
106 target: AwsTarget,
96107 *,
97 profile: str,
98 region: str,
99 account: str,
100108 output_dir: str,
101109 selected_keys: Iterable[str],
102110 on_event: ProgressCallback,
@@ -109,6 +117,7 @@ def run_audit(
109117 can't be built (e.g. an unknown profile), that is reported and the run ends
110118 cleanly.
111119 """
120 profile, region, account = target.profile, target.region, target.account
112121 subject = profile or "default"
113122 selected = set(selected_keys)
114123 checks = [c for c in CHECKS if c.key in selected]
tui/github_runner.py +12 −3
@@ -12,6 +12,7 @@ This module reuses the collectors and CSV reporter under
1212import os
1313import sys
1414from collections.abc import Iterable
15from dataclasses import dataclass
1516from datetime import date
1617
1718from tui.common import Check, ProgressCallback, ProgressEvent
@@ -145,15 +146,22 @@ def default_output_dir(out: str, org: str) -> str:
145146 return os.path.join(out, f"github_audit_{org}_{date.today().isoformat()}")
146147
147148
149@dataclass(frozen=True)
150class GitHubTarget:
151 """An organization to audit and the token and default branch to audit it with."""
152
153 org: str
154 token: str
155 branch: str
156
157
148158# --- Runner -----------------------------------------------------------------
149159
150160
151161def run_audit(
162 target: GitHubTarget,
152163 *,
153 org: str,
154 token: str,
155164 output_dir: str,
156 branch: str,
157165 selected_keys: Iterable[str],
158166 on_event: ProgressCallback,
159167) -> list[tuple[str, int]]:
@@ -166,6 +174,7 @@ def run_audit(
166174 Returns the list of (label, row_count) sections that was written to the
167175 summary file.
168176 """
177 org, token, branch = target.org, target.token, target.branch
169178 cfg = build_cfg(token)
170179 selected = set(selected_keys)
171180 checks = [c for c in CHECKS if c.key in selected]
tui/gitlab_runner.py +12 −3
@@ -10,6 +10,7 @@ through a callback.
1010import os
1111import sys
1212from collections.abc import Iterable
13from dataclasses import dataclass
1314from datetime import date
1415
1516from tui.common import Check, ProgressCallback, ProgressEvent
@@ -109,14 +110,21 @@ def default_output_dir(out: str, group: str) -> str:
109110 return os.path.join(out, f"gitlab_audit_{safe_group}_{date.today().isoformat()}")
110111
111112
113@dataclass(frozen=True)
114class GitLabTarget:
115 """A group to audit and the token and instance URL to audit it with."""
116
117 group: str
118 token: str
119 base_url: str
120
121
112122# --- Runner -----------------------------------------------------------------
113123
114124
115125def run_audit(
126 target: GitLabTarget,
116127 *,
117 group: str,
118 token: str,
119 base_url: str,
120128 output_dir: str,
121129 selected_keys: Iterable[str],
122130 on_event: ProgressCallback,
@@ -127,6 +135,7 @@ def run_audit(
127135 A collector that raises is reported as an error and recorded with a count
128136 of 0, so one bad check never aborts the whole run.
129137 """
138 group, token, base_url = target.group, target.token, target.base_url
130139 cfg = build_cfg(token, base_url)
131140 selected = set(selected_keys)
132141 checks = [c for c in CHECKS if c.key in selected]
tui/platforms.py +3 −9
@@ -61,10 +61,8 @@ def _github_output_dir(s: dict) -> str:
6161
6262def _github_run(s: dict, output_dir, selected_keys, on_event):
6363 return github_runner.run_audit(
64 org=s["org"],
65 token=s["token"],
64 github_runner.GitHubTarget(org=s["org"], token=s["token"], branch=s["branch"]),
6665 output_dir=output_dir,
67 branch=s["branch"],
6866 selected_keys=selected_keys,
6967 on_event=on_event,
7068 )
@@ -76,9 +74,7 @@ def _gitlab_output_dir(s: dict) -> str:
7674
7775def _gitlab_run(s: dict, output_dir, selected_keys, on_event):
7876 return gitlab_runner.run_audit(
79 group=s["group"],
80 token=s["token"],
81 base_url=s["base_url"],
77 gitlab_runner.GitLabTarget(group=s["group"], token=s["token"], base_url=s["base_url"]),
8278 output_dir=output_dir,
8379 selected_keys=selected_keys,
8480 on_event=on_event,
@@ -91,9 +87,7 @@ def _aws_output_dir(s: dict) -> str:
9187
9288def _aws_run(s: dict, output_dir, selected_keys, on_event):
9389 return aws_runner.run_audit(
94 profile=s["profile"],
95 region=s["region"],
96 account=s["account"],
90 aws_runner.AwsTarget(profile=s["profile"], region=s["region"], account=s["account"]),
9791 output_dir=output_dir,
9892 selected_keys=selected_keys,
9993 on_event=on_event,
tui/tests/test_app.py +3 −5
@@ -18,7 +18,7 @@ def _run(coro):
1818
1919
2020def test_full_navigation(monkeypatch):
21 def fake_run_audit(*, org, token, output_dir, branch, selected_keys, on_event):
21 def fake_run_audit(target, *, output_dir, selected_keys, on_event):
2222 on_event(gh.ProgressEvent("start", "Member roster"))
2323 on_event(gh.ProgressEvent("done", "Member roster", count=3))
2424 on_event(gh.ProgressEvent("summary", output_dir, count=3))
@@ -91,7 +91,7 @@ def test_gitlab_navigation(monkeypatch):
9191 monkeypatch.delenv("GITLAB_GROUP", raising=False)
9292 monkeypatch.delenv("GITLAB_TOKEN", raising=False)
9393
94 def fake_run_audit(*, group, token, base_url, output_dir, selected_keys, on_event):
94 def fake_run_audit(target, *, output_dir, selected_keys, on_event):
9595 on_event(gh.ProgressEvent("done", "Group members", count=7))
9696 on_event(gh.ProgressEvent("summary", output_dir, count=7))
9797 return [("Group members", 7)]
@@ -131,9 +131,7 @@ def test_aws_navigation(monkeypatch):
131131 for var in ("AWS_PROFILE", "AWS_DEFAULT_REGION", "AWS_AUDIT_ACCOUNT"):
132132 monkeypatch.delenv(var, raising=False)
133133
134 def fake_run_audit(
135 *, profile, region, account, output_dir, selected_keys, on_event
136 ):
134 def fake_run_audit(target, *, output_dir, selected_keys, on_event):
137135 on_event(gh.ProgressEvent("done", "IAM users", count=5))
138136 on_event(gh.ProgressEvent("summary", output_dir, count=5))
139137 return [("IAM users", 5)]
tui/tests/test_aws_runner.py +1 −3
@@ -52,9 +52,7 @@ def fake_checks(monkeypatch):
5252def run(tmp_path, keys, profile="", region="us-east-1", account=""):
5353 events = []
5454 sections = r.run_audit(
55 profile=profile,
56 region=region,
57 account=account,
55 r.AwsTarget(profile=profile, region=region, account=account),
5856 output_dir=str(tmp_path),
5957 selected_keys=keys,
6058 on_event=events.append,
tui/tests/test_github_runner.py +1 −3
@@ -55,10 +55,8 @@ def fake_checks(monkeypatch):
5555def run(tmp_path, keys, fake_checks, branch="main"):
5656 events = []
5757 sections = r.run_audit(
58 org="acme",
59 token="tok",
58 r.GitHubTarget(org="acme", token="tok", branch=branch),
6059 output_dir=str(tmp_path),
61 branch=branch,
6260 selected_keys=keys,
6361 on_event=events.append,
6462 )
tui/tests/test_gitlab_runner.py +1 −3
@@ -49,9 +49,7 @@ def fake_checks(monkeypatch):
4949def run(tmp_path, keys, base_url="https://gitlab.com/api/v4"):
5050 events = []
5151 sections = r.run_audit(
52 group="grp",
53 token="tok",
54 base_url=base_url,
52 r.GitLabTarget(group="grp", token="tok", base_url=base_url),
5553 output_dir=str(tmp_path),
5654 selected_keys=keys,
5755 on_event=events.append,