audit-labs/audit-tools

A collection of scripts, queries, and other goodies you can use in an audit. audit automation compliance evidence scripts

Commit b37f594dee

b37f594dee47db2d794e05d44260cd880ea46b3a

parent: 8c281c79e7

Unsigned

cmc <hello@cleberg.net> · 2026-10-30 00:23 UTC

Qlty config and code cleanup (!1)

- Add Qlty config; split sampling validation and pass run facts
- Qlty: disable function_complexity

Layout: unified · split

.qlty/.gitignore added +7
@@ -0,0 +1,7 @@
1*
2!configs
3!configs/**
4!hooks
5!hooks/**
6!qlty.toml
7!.gitignore
.qlty/qlty.toml added +50
@@ -0,0 +1,50 @@
1config_version = "0"
2
3exclude_patterns = [
4 "**/vendor/**",
5 "**/node_modules/**",
6 "**/testdata/**",
7 "**/dist/**",
8 "**/.build/**",
9 "**/build/**",
10 "**/*.min.*",
11 "**/generated/**",
12 ".claude/**",
13 "**/Pods/**",
14 # Each application under applications/ runs standalone with its own
15 # reporters package, so the GitHub and GitLab CSV reporters are copies.
16 "applications/github/reporters/csv_reporter.py",
17 "applications/gitlab/reporters/csv_reporter.py",
18]
19
20test_patterns = [
21 "**/test/**",
22 "**/tests/**",
23 "**/Tests/**",
24 "**/*_test.*",
25 "**/test_*.*",
26 "**/*.test.*",
27 "**/*Tests.swift",
28 "e2e/**",
29]
30
31[[source]]
32name = "default"
33default = true
34
35[[plugin]]
36name = "osv-scanner"
37
38[[plugin]]
39name = "trufflehog"
40
41# Early returns are guard clauses, which keep the happy path unindented.
42[smells.return_statements]
43enabled = false
44
45# Function and file complexity are left to review rather than counted.
46[smells.function_complexity]
47enabled = false
48
49[smells.file_complexity]
50enabled = false
applications/github/collectors/audit_log.py +2 −7
@@ -162,13 +162,8 @@ def _dedupe_events(events):
162 162
163def _branch_or_pattern(event): 163def _branch_or_pattern(event):
164 """Find the most useful target label for branch and ruleset audit events.""" 164 """Find the most useful target label for branch and ruleset audit events."""
165 return ( 165 keys = ("name", "branch", "ruleset_name", "ruleset_old_name")
166 event.get("name") 166 return next((event[key] for key in keys if event.get(key)), "")
167 or event.get("branch")
168 or event.get("ruleset_name")
169 or event.get("ruleset_old_name")
170 or ""
171 )
172 167
173 168
174def _event_summary(event): 169def _event_summary(event):
sampling/sampling_tool/cli.py +33 −54
@@ -4,12 +4,14 @@ from __future__ import annotations
4 4
5import sys 5import sys
6from argparse import ArgumentParser, Namespace 6from argparse import ArgumentParser, Namespace
7from dataclasses import dataclass
7from datetime import datetime, timezone 8from datetime import datetime, timezone
8from pathlib import Path 9from pathlib import Path
9from types import SimpleNamespace 10from types import SimpleNamespace
10 11
11import pandas as pd 12import pandas as pd
12 13
14from .facts import RunFacts
13from .filters import apply_filters, parse_filters 15from .filters import apply_filters, parse_filters
14from .io import AuditSamplingError, load_population, sha256_file, write_csv 16from .io import AuditSamplingError, load_population, sha256_file, write_csv
15from .manifest import build_manifest, write_manifest 17from .manifest import build_manifest, write_manifest
@@ -217,16 +219,8 @@ def run(options) -> Path:
217 sampled, options, random_seed, timestamp, options.stratify_column 219 sampled, options, random_seed, timestamp, options.stratify_column
218 ) 220 )
219 221
220 _write_outputs( 222 frames = RunFrames(validated, excluded_rows, duplicate_rows, sample, strata_rows)
221 run_dir, 223 _write_outputs(run_dir, options, frames, output_files)
222 options,
223 validated,
224 excluded_rows,
225 duplicate_rows,
226 sample,
227 strata_rows,
228 output_files,
229 )
230 logger.log("final status: success") 224 logger.log("final status: success")
231 print("Audit sampling run completed.") 225 print("Audit sampling run completed.")
232 except AuditSamplingError as exc: 226 except AuditSamplingError as exc:
@@ -243,48 +237,29 @@ def run(options) -> Path:
243 logger.log("final status: failed") 237 logger.log("final status: failed")
244 raise 238 raise
245 finally: 239 finally:
246 reconciliation = build_reconciliation( 240 facts = RunFacts(
247 source_rows=len(source),
248 blank_id_count=blank_id_count,
249 duplicate_id_count=duplicate_id_count or len(duplicate_rows),
250 excluded_rows=len(excluded_rows),
251 validated_rows=len(validated),
252 requested_sample_size=_requested_sample_size(options, strata_rows),
253 final_sample_size=len(sample),
254 )
255 write_csv(reconciliation, run_dir / "population_reconciliation.csv")
256 _track(output_files, "population_reconciliation.csv")
257 methodology = build_methodology(
258 input_file=input_path, 241 input_file=input_path,
259 input_sheet=options.sheet,
260 input_sha256=input_hash, 242 input_sha256=input_hash,
261 source_row_count=len(source), 243 input_sheet=options.sheet,
262 options=options, 244 options=options,
263 effective_id_column=effective_id_column, 245 effective_id_column=effective_id_column,
264 id_column_omitted=id_column_omitted, 246 id_column_omitted=id_column_omitted,
247 source_row_count=len(source),
248 validated_population_count=len(validated),
249 excluded_row_count=len(excluded_rows),
265 duplicate_id_count=duplicate_id_count or len(duplicate_rows), 250 duplicate_id_count=duplicate_id_count or len(duplicate_rows),
266 blank_id_count=blank_id_count, 251 blank_id_count=blank_id_count,
252 requested_sample_size=_requested_sample_size(options, strata_rows),
267 sample_size_actual=len(sample), 253 sample_size_actual=len(sample),
268 random_seed=random_seed, 254 random_seed=random_seed,
269 strata_summary=strata_rows,
270 ) 255 )
271 (run_dir / "methodology.txt").write_text(methodology) 256 write_csv(build_reconciliation(facts), run_dir / "population_reconciliation.csv")
257 _track(output_files, "population_reconciliation.csv")
258 (run_dir / "methodology.txt").write_text(build_methodology(facts, strata_rows))
272 _track(output_files, "methodology.txt") 259 _track(output_files, "methodology.txt")
273 manifest = build_manifest( 260 manifest = build_manifest(
261 facts,
274 run_timestamp_utc=timestamp, 262 run_timestamp_utc=timestamp,
275 input_file=input_path,
276 input_sha256=input_hash,
277 input_sheet=options.sheet,
278 options=options,
279 effective_id_column=effective_id_column,
280 id_column_omitted=id_column_omitted,
281 sample_size_actual=len(sample),
282 random_seed=random_seed,
283 source_row_count=len(source),
284 validated_population_count=len(validated),
285 excluded_row_count=len(excluded_rows),
286 duplicate_id_count=duplicate_id_count or len(duplicate_rows),
287 blank_id_count=blank_id_count,
288 output_files=sorted(output_files + ["run.log", "manifest.json"]), 263 output_files=sorted(output_files + ["run.log", "manifest.json"]),
289 ) 264 )
290 write_manifest(manifest, run_dir / "manifest.json") 265 write_manifest(manifest, run_dir / "manifest.json")
@@ -310,29 +285,33 @@ def add_sample_metadata(
310 return output 285 return output
311 286
312 287
288@dataclass
289class RunFrames:
290 """The tables a successful run writes."""
291
292 validated: pd.DataFrame
293 excluded_rows: pd.DataFrame
294 duplicate_rows: pd.DataFrame
295 sample: pd.DataFrame
296 strata_rows: list[dict[str, object]]
297
298
313def _write_outputs( 299def _write_outputs(
314 run_dir: Path, 300 run_dir: Path, options, frames: RunFrames, output_files: list[str]
315 options,
316 validated: pd.DataFrame,
317 excluded_rows: pd.DataFrame,
318 duplicate_rows: pd.DataFrame,
319 sample: pd.DataFrame,
320 strata_rows: list[dict[str, object]],
321 output_files: list[str],
322) -> None: 301) -> None:
323 write_csv(validated, run_dir / _POPULATION_VALIDATED_CSV) 302 write_csv(frames.validated, run_dir / _POPULATION_VALIDATED_CSV)
324 _track(output_files, _POPULATION_VALIDATED_CSV) 303 _track(output_files, _POPULATION_VALIDATED_CSV)
325 if options.method in {"random", "systematic", "stratified"}: 304 if options.method in {"random", "systematic", "stratified"}:
326 write_csv(sample, run_dir / "sample.csv") 305 write_csv(frames.sample, run_dir / "sample.csv")
327 _track(output_files, "sample.csv") 306 _track(output_files, "sample.csv")
328 if not excluded_rows.empty: 307 if not frames.excluded_rows.empty:
329 write_csv(excluded_rows, run_dir / _EXCLUDED_ROWS_CSV) 308 write_csv(frames.excluded_rows, run_dir / _EXCLUDED_ROWS_CSV)
330 _track(output_files, _EXCLUDED_ROWS_CSV) 309 _track(output_files, _EXCLUDED_ROWS_CSV)
331 if not duplicate_rows.empty: 310 if not frames.duplicate_rows.empty:
332 write_csv(duplicate_rows, run_dir / _DUPLICATE_IDS_CSV) 311 write_csv(frames.duplicate_rows, run_dir / _DUPLICATE_IDS_CSV)
333 _track(output_files, _DUPLICATE_IDS_CSV) 312 _track(output_files, _DUPLICATE_IDS_CSV)
334 if options.method == "stratified": 313 if options.method == "stratified":
335 write_csv(build_strata_summary(strata_rows), run_dir / "strata_summary.csv") 314 write_csv(build_strata_summary(frames.strata_rows), run_dir / "strata_summary.csv")
336 _track(output_files, "strata_summary.csv") 315 _track(output_files, "strata_summary.csv")
337 316
338 317
sampling/sampling_tool/facts.py added +25
@@ -0,0 +1,25 @@
1"""Facts about one sampling run, shared by its manifest, methodology, and
2reconciliation outputs."""
3
4from __future__ import annotations
5
6from dataclasses import dataclass
7from pathlib import Path
8
9
10@dataclass
11class RunFacts:
12 input_file: Path
13 input_sha256: str
14 input_sheet: str | None
15 options: object
16 effective_id_column: str | None
17 id_column_omitted: bool
18 source_row_count: int
19 validated_population_count: int
20 excluded_row_count: int
21 duplicate_id_count: int
22 blank_id_count: int
23 requested_sample_size: int | None
24 sample_size_actual: int
25 random_seed: int | None
sampling/sampling_tool/manifest.py +15 −28
@@ -6,52 +6,39 @@ import json
6from pathlib import Path 6from pathlib import Path
7 7
8from . import __version__, engine 8from . import __version__, engine
9from .facts import RunFacts
9 10
10 11
11def build_manifest( 12def build_manifest(
12 *, 13 facts: RunFacts, *, run_timestamp_utc: str, output_files: list[str]
13 run_timestamp_utc: str,
14 input_file: Path,
15 input_sha256: str,
16 input_sheet: str | None,
17 options,
18 effective_id_column: str | None,
19 id_column_omitted: bool,
20 sample_size_actual: int,
21 random_seed: int | None,
22 source_row_count: int,
23 validated_population_count: int,
24 excluded_row_count: int,
25 duplicate_id_count: int,
26 blank_id_count: int,
27 output_files: list[str],
28) -> dict[str, object]: 14) -> dict[str, object]:
15 options = facts.options
29 return { 16 return {
30 "tool": "audit_sample", 17 "tool": "audit_sample",
31 "version": __version__, 18 "version": __version__,
32 "engine": engine.ENGINE, 19 "engine": engine.ENGINE,
33 "engine_version": engine.ENGINE_VERSION, 20 "engine_version": engine.ENGINE_VERSION,
34 "run_timestamp_utc": run_timestamp_utc, 21 "run_timestamp_utc": run_timestamp_utc,
35 "input_file": str(input_file), 22 "input_file": str(facts.input_file),
36 "input_sha256": input_sha256, 23 "input_sha256": facts.input_sha256,
37 "input_sheet": input_sheet, 24 "input_sheet": facts.input_sheet,
38 "method": options.method, 25 "method": options.method,
39 "id_column": options.id_column, 26 "id_column": options.id_column,
40 "effective_id_column": effective_id_column, 27 "effective_id_column": facts.effective_id_column,
41 "id_column_omitted": id_column_omitted, 28 "id_column_omitted": facts.id_column_omitted,
42 "stratify_column": options.stratify_column, 29 "stratify_column": options.stratify_column,
43 "sample_size_requested": options.sample_size, 30 "sample_size_requested": options.sample_size,
44 "sample_size_actual": sample_size_actual, 31 "sample_size_actual": facts.sample_size_actual,
45 "random_seed": random_seed, 32 "random_seed": facts.random_seed,
46 "filters": options.filters, 33 "filters": options.filters,
47 "dedupe_id": options.dedupe_id, 34 "dedupe_id": options.dedupe_id,
48 "exclude_blank_id": options.exclude_blank_id, 35 "exclude_blank_id": options.exclude_blank_id,
49 "allow_shortfall": options.allow_shortfall, 36 "allow_shortfall": options.allow_shortfall,
50 "source_row_count": source_row_count, 37 "source_row_count": facts.source_row_count,
51 "validated_population_count": validated_population_count, 38 "validated_population_count": facts.validated_population_count,
52 "excluded_row_count": excluded_row_count, 39 "excluded_row_count": facts.excluded_row_count,
53 "duplicate_id_count": duplicate_id_count, 40 "duplicate_id_count": facts.duplicate_id_count,
54 "blank_id_count": blank_id_count, 41 "blank_id_count": facts.blank_id_count,
55 "output_files": output_files, 42 "output_files": output_files,
56 } 43 }
57 44
sampling/sampling_tool/reconciliation.py +11 −16
@@ -4,28 +4,23 @@ from __future__ import annotations
4 4
5import pandas as pd 5import pandas as pd
6 6
7from .facts import RunFacts
7 8
8def build_reconciliation( 9
9 source_rows: int, 10def build_reconciliation(facts: RunFacts) -> pd.DataFrame:
10 blank_id_count: int, 11 unsampled = facts.validated_population_count - facts.sample_size_actual
11 duplicate_id_count: int, 12 requested_sample_size = facts.requested_sample_size
12 excluded_rows: int,
13 validated_rows: int,
14 requested_sample_size: int | None,
15 final_sample_size: int,
16) -> pd.DataFrame:
17 unsampled = validated_rows - final_sample_size
18 rows = [ 13 rows = [
19 ("Source rows", source_rows), 14 ("Source rows", facts.source_row_count),
20 ("Rows with blank ID", blank_id_count), 15 ("Rows with blank ID", facts.blank_id_count),
21 ("Duplicate IDs", duplicate_id_count), 16 ("Duplicate IDs", facts.duplicate_id_count),
22 ("Excluded rows", excluded_rows), 17 ("Excluded rows", facts.excluded_row_count),
23 ("Validated population rows", validated_rows), 18 ("Validated population rows", facts.validated_population_count),
24 ( 19 (
25 "Requested sample size", 20 "Requested sample size",
26 "" if requested_sample_size is None else requested_sample_size, 21 "" if requested_sample_size is None else requested_sample_size,
27 ), 22 ),
28 ("Final sample size", final_sample_size), 23 ("Final sample size", facts.sample_size_actual),
29 ("Unsampled population rows", unsampled), 24 ("Unsampled population rows", unsampled),
30 ] 25 ]
31 return pd.DataFrame(rows, columns=["Metric", "Value"]) 26 return pd.DataFrame(rows, columns=["Metric", "Value"])
sampling/sampling_tool/reporting.py +14 −23
@@ -4,6 +4,8 @@ from __future__ import annotations
4 4
5from pathlib import Path 5from pathlib import Path
6 6
7from .facts import RunFacts
8
7 9
8class RunLogger: 10class RunLogger:
9 def __init__(self) -> None: 11 def __init__(self) -> None:
@@ -23,29 +25,18 @@ class RunLogger:
23 25
24 26
25def build_methodology( 27def build_methodology(
26 *, 28 facts: RunFacts, strata_summary: list[dict[str, object]] | None
27 input_file: Path,
28 input_sheet: str | None,
29 input_sha256: str,
30 source_row_count: int,
31 options,
32 effective_id_column: str | None,
33 id_column_omitted: bool,
34 duplicate_id_count: int,
35 blank_id_count: int,
36 sample_size_actual: int,
37 random_seed: int | None,
38 strata_summary: list[dict[str, object]] | None,
39) -> str: 29) -> str:
30 options = facts.options
40 lines = [ 31 lines = [
41 "Audit Sampling Methodology", 32 "Audit Sampling Methodology",
42 "", 33 "",
43 f"Source file: {input_file}", 34 f"Source file: {facts.input_file}",
44 f"Source sheet: {input_sheet or ''}", 35 f"Source sheet: {facts.input_sheet or ''}",
45 f"Input SHA-256: {input_sha256}", 36 f"Input SHA-256: {facts.input_sha256}",
46 f"Source row count: {source_row_count}", 37 f"Source row count: {facts.source_row_count}",
47 ] 38 ]
48 if id_column_omitted: 39 if facts.id_column_omitted:
49 lines.append( 40 lines.append(
50 "ID column: omitted; _audit_row_id was generated from _source_row_number." 41 "ID column: omitted; _audit_row_id was generated from _source_row_number."
51 ) 42 )
@@ -53,15 +44,15 @@ def build_methodology(
53 lines.append(f"ID column: {options.id_column}") 44 lines.append(f"ID column: {options.id_column}")
54 lines.extend( 45 lines.extend(
55 [ 46 [
56 f"Effective ID column: {effective_id_column or ''}", 47 f"Effective ID column: {facts.effective_id_column or ''}",
57 f"Duplicate ID rows identified: {duplicate_id_count}", 48 f"Duplicate ID rows identified: {facts.duplicate_id_count}",
58 f"Blank ID rows identified: {blank_id_count}", 49 f"Blank ID rows identified: {facts.blank_id_count}",
59 f"Blank ID handling: {'excluded' if options.exclude_blank_id else 'retained'}", 50 f"Blank ID handling: {'excluded' if options.exclude_blank_id else 'retained'}",
60 f"Filters applied: {options.filters or {}}", 51 f"Filters applied: {options.filters or {}}",
61 f"Sampling method: {options.method}", 52 f"Sampling method: {options.method}",
62 f"Sample size requested: {options.sample_size or ''}", 53 f"Sample size requested: {options.sample_size or ''}",
63 f"Sample size selected: {sample_size_actual}", 54 f"Sample size selected: {facts.sample_size_actual}",
64 f"Random seed used: {random_seed or ''}", 55 f"Random seed used: {facts.random_seed or ''}",
65 ] 56 ]
66 ) 57 )
67 if options.method == "stratified": 58 if options.method == "stratified":
sampling/sampling_tool/validation.py +109 −88
@@ -33,100 +33,20 @@ def validate_and_prepare(population: pd.DataFrame, options) -> ValidationResult:
33 excluded_parts: list[pd.DataFrame] = [] 33 excluded_parts: list[pd.DataFrame] = []
34 working = population.copy() 34 working = population.copy()
35 35
36 if options.id_column: 36 effective_id_column, id_column_omitted = _resolve_id_column(working, options, warnings)
37 if options.id_column not in working.columns:
38 raise AuditSamplingError(f"ID column not found: {options.id_column}")
39 effective_id_column = options.id_column
40 id_column_omitted = False
41 else:
42 effective_id_column = "_audit_row_id"
43 id_column_omitted = True
44 working[effective_id_column] = working["_source_row_number"]
45 warnings.append(
46 "--id-column omitted; using _source_row_number as generated _audit_row_id."
47 )
48
49 if options.stratify_column and options.stratify_column not in working.columns: 37 if options.stratify_column and options.stratify_column not in working.columns:
50 raise AuditSamplingError( 38 raise AuditSamplingError(
51 f"Stratification column not found: {options.stratify_column}" 39 f"Stratification column not found: {options.stratify_column}"
52 ) 40 )
53 41
54 id_blank_mask = blank_id_mask(working[effective_id_column]) 42 working, blank_id_count = _handle_blank_ids(
55 blank_id_count = int(id_blank_mask.sum()) 43 working, effective_id_column, options, warnings, excluded_parts
56 if blank_id_count:
57 if options.exclude_blank_id:
58 excluded = working.loc[id_blank_mask].copy()
59 excluded["_exclusion_reason"] = "Blank ID"
60 excluded_parts.append(excluded)
61 working = working.loc[~id_blank_mask].copy()
62 else:
63 warnings.append(
64 f"{blank_id_count} row(s) have blank IDs and were retained."
65 )
66
67 nonblank_ids = ~blank_id_mask(working[effective_id_column])
68 duplicate_mask = working.loc[nonblank_ids, effective_id_column].duplicated(
69 keep=False
70 ) 44 )
71 duplicate_rows = working.loc[nonblank_ids].loc[duplicate_mask].copy() 45 working, duplicate_rows = _handle_duplicate_ids(
72 duplicate_id_count = len(duplicate_rows) 46 working, effective_id_column, options, warnings, excluded_parts
73 if duplicate_id_count:
74 if options.dedupe_id == "fail":
75 raise AuditSamplingError(
76 f"Duplicate IDs found in '{effective_id_column}'. "
77 "See duplicate_ids.csv.",
78 duplicate_rows=duplicate_rows,
79 )
80 keep = "first" if options.dedupe_id == "first" else "last"
81 drop_mask = working[effective_id_column].duplicated(keep=keep) & ~blank_id_mask(
82 working[effective_id_column]
83 )
84 excluded = working.loc[drop_mask].copy()
85 excluded["_exclusion_reason"] = f"Duplicate ID removed by dedupe={keep}"
86 excluded_parts.append(excluded)
87 working = working.loc[~drop_mask].copy()
88 warnings.append(
89 f"{len(excluded)} duplicate ID row(s) removed using dedupe={keep}."
90 )
91
92 if options.sample_size is not None and options.sample_size <= 0:
93 raise AuditSamplingError("--sample-size must be a positive integer.")
94 needs_sample_size = options.method in ("random", "systematic") or (
95 options.method == "stratified" and bool(options.strata_proportions)
96 ) 47 )
97 if needs_sample_size: 48 _check_sample_size(len(working), options)
98 if options.sample_size is None: 49 strata_counts, strata_proportions = _validate_strata(working, options)
99 raise AuditSamplingError(f"--sample-size is required for {options.method}.")
100 if options.sample_size > len(working) and not (
101 options.method == "stratified" and options.allow_shortfall
102 ):
103 raise AuditSamplingError(
104 "--sample-size cannot exceed the validated population size."
105 )
106
107 strata_counts: dict[str, int] = {}
108 strata_proportions: dict[str, float] = {}
109 if options.method == "stratified":
110 if not options.stratify_column:
111 raise AuditSamplingError("--stratify-column is required for stratified.")
112 has_counts = bool(options.strata_counts)
113 has_proportions = bool(options.strata_proportions)
114 if has_counts == has_proportions:
115 raise AuditSamplingError(
116 "Use exactly one of --strata-counts or --strata-proportions."
117 )
118 strata_counts = parse_key_ints(options.strata_counts, "strata")
119 strata_proportions = parse_key_floats(options.strata_proportions, "strata")
120 requested_strata = set(strata_counts or strata_proportions)
121 actual_strata = set(working[options.stratify_column].dropna().astype(str))
122 missing = sorted(requested_strata - actual_strata)
123 if missing:
124 raise AuditSamplingError(
125 "Requested strata not found in population: " + ", ".join(missing)
126 )
127
128 if strata_counts:
129 _validate_stratum_counts_fit(working, options, strata_counts)
130 50
131 if excluded_parts: 51 if excluded_parts:
132 excluded_rows = pd.concat(excluded_parts, ignore_index=True) 52 excluded_rows = pd.concat(excluded_parts, ignore_index=True)
@@ -140,13 +60,114 @@ def validate_and_prepare(population: pd.DataFrame, options) -> ValidationResult:
140 effective_id_column=effective_id_column, 60 effective_id_column=effective_id_column,
141 id_column_omitted=id_column_omitted, 61 id_column_omitted=id_column_omitted,
142 blank_id_count=blank_id_count, 62 blank_id_count=blank_id_count,
143 duplicate_id_count=duplicate_id_count, 63 duplicate_id_count=len(duplicate_rows),
144 warnings=warnings, 64 warnings=warnings,
145 strata_counts=strata_counts, 65 strata_counts=strata_counts,
146 strata_proportions=strata_proportions, 66 strata_proportions=strata_proportions,
147 ) 67 )
148 68
149 69
70def _resolve_id_column(working: pd.DataFrame, options, warnings: list[str]) -> tuple[str, bool]:
71 """The ID column to use, generating _audit_row_id when none was given."""
72 if options.id_column:
73 if options.id_column not in working.columns:
74 raise AuditSamplingError(f"ID column not found: {options.id_column}")
75 return options.id_column, False
76 working["_audit_row_id"] = working["_source_row_number"]
77 warnings.append(
78 "--id-column omitted; using _source_row_number as generated _audit_row_id."
79 )
80 return "_audit_row_id", True
81
82
83def _handle_blank_ids(working, id_column, options, warnings, excluded_parts):
84 """Exclude or warn about rows with blank IDs; returns (working, count)."""
85 id_blank_mask = blank_id_mask(working[id_column])
86 blank_id_count = int(id_blank_mask.sum())
87 if not blank_id_count:
88 return working, 0
89 if options.exclude_blank_id:
90 excluded = working.loc[id_blank_mask].copy()
91 excluded["_exclusion_reason"] = "Blank ID"
92 excluded_parts.append(excluded)
93 working = working.loc[~id_blank_mask].copy()
94 else:
95 warnings.append(
96 f"{blank_id_count} row(s) have blank IDs and were retained."
97 )
98 return working, blank_id_count
99
100
101def _handle_duplicate_ids(working, id_column, options, warnings, excluded_parts):
102 """Fail on, or dedupe, repeated non-blank IDs; returns (working, duplicates)."""
103 nonblank_ids = ~blank_id_mask(working[id_column])
104 duplicate_mask = working.loc[nonblank_ids, id_column].duplicated(keep=False)
105 duplicate_rows = working.loc[nonblank_ids].loc[duplicate_mask].copy()
106 if not len(duplicate_rows):
107 return working, duplicate_rows
108 if options.dedupe_id == "fail":
109 raise AuditSamplingError(
110 f"Duplicate IDs found in '{id_column}'. "
111 "See duplicate_ids.csv.",
112 duplicate_rows=duplicate_rows,
113 )
114 keep = "first" if options.dedupe_id == "first" else "last"
115 drop_mask = working[id_column].duplicated(keep=keep) & ~blank_id_mask(
116 working[id_column]
117 )
118 excluded = working.loc[drop_mask].copy()
119 excluded["_exclusion_reason"] = f"Duplicate ID removed by dedupe={keep}"
120 excluded_parts.append(excluded)
121 warnings.append(
122 f"{len(excluded)} duplicate ID row(s) removed using dedupe={keep}."
123 )
124 return working.loc[~drop_mask].copy(), duplicate_rows
125
126
127def _check_sample_size(population_size: int, options) -> None:
128 if options.sample_size is not None and options.sample_size <= 0:
129 raise AuditSamplingError("--sample-size must be a positive integer.")
130 needs_sample_size = options.method in ("random", "systematic") or (
131 options.method == "stratified" and bool(options.strata_proportions)
132 )
133 if not needs_sample_size:
134 return
135 if options.sample_size is None:
136 raise AuditSamplingError(f"--sample-size is required for {options.method}.")
137 if options.sample_size > population_size and not (
138 options.method == "stratified" and options.allow_shortfall
139 ):
140 raise AuditSamplingError(
141 "--sample-size cannot exceed the validated population size."
142 )
143
144
145def _validate_strata(working, options) -> tuple[dict[str, int], dict[str, float]]:
146 """Parse and check --strata-counts / --strata-proportions for stratified runs."""
147 if options.method != "stratified":
148 return {}, {}
149 if not options.stratify_column:
150 raise AuditSamplingError("--stratify-column is required for stratified.")
151 has_counts = bool(options.strata_counts)
152 has_proportions = bool(options.strata_proportions)
153 if has_counts == has_proportions:
154 raise AuditSamplingError(
155 "Use exactly one of --strata-counts or --strata-proportions."
156 )
157 strata_counts = parse_key_ints(options.strata_counts, "strata")
158 strata_proportions = parse_key_floats(options.strata_proportions, "strata")
159 requested_strata = set(strata_counts or strata_proportions)
160 actual_strata = set(working[options.stratify_column].dropna().astype(str))
161 missing = sorted(requested_strata - actual_strata)
162 if missing:
163 raise AuditSamplingError(
164 "Requested strata not found in population: " + ", ".join(missing)
165 )
166 if strata_counts:
167 _validate_stratum_counts_fit(working, options, strata_counts)
168 return strata_counts, strata_proportions
169
170
150def _validate_stratum_counts_fit(population, options, counts: dict[str, int]) -> None: 171def _validate_stratum_counts_fit(population, options, counts: dict[str, int]) -> None:
151 for stratum, requested in counts.items(): 172 for stratum, requested in counts.items():
152 available = int((population[options.stratify_column] == stratum).sum()) 173 available = int((population[options.stratify_column] == stratum).sum())
tui/aws_runner.py +12 −3
@@ -13,6 +13,7 @@ there is no per-item cache, so every check is called as ``fn(cfg)``.
13import os 13import os
14import sys 14import sys
15from collections.abc import Iterable 15from collections.abc import Iterable
16from dataclasses import dataclass
16from datetime import date 17from datetime import date
17 18
18from tui.common import Check, ProgressCallback, ProgressEvent 19from tui.common import Check, ProgressCallback, ProgressEvent
@@ -89,14 +90,21 @@ def default_output_dir(out: str, profile: str) -> str:
89 return os.path.join(out, f"aws_audit_{subject}_{date.today().isoformat()}") 90 return os.path.join(out, f"aws_audit_{subject}_{date.today().isoformat()}")
90 91
91 92
93@dataclass(frozen=True)
94class AwsTarget:
95 """The AWS profile, region, and account to audit."""
96
97 profile: str
98 region: str
99 account: str
100
101
92# --- Runner ----------------------------------------------------------------- 102# --- Runner -----------------------------------------------------------------
93 103
94 104
95def run_audit( 105def run_audit(
106 target: AwsTarget,
96 *, 107 *,
97 profile: str,
98 region: str,
99 account: str,
100 output_dir: str, 108 output_dir: str,
101 selected_keys: Iterable[str], 109 selected_keys: Iterable[str],
102 on_event: ProgressCallback, 110 on_event: ProgressCallback,
@@ -109,6 +117,7 @@ def run_audit(
109 can't be built (e.g. an unknown profile), that is reported and the run ends 117 can't be built (e.g. an unknown profile), that is reported and the run ends
110 cleanly. 118 cleanly.
111 """ 119 """
120 profile, region, account = target.profile, target.region, target.account
112 subject = profile or "default" 121 subject = profile or "default"
113 selected = set(selected_keys) 122 selected = set(selected_keys)
114 checks = [c for c in CHECKS if c.key in selected] 123 checks = [c for c in CHECKS if c.key in selected]
tui/github_runner.py +12 −3
@@ -12,6 +12,7 @@ This module reuses the collectors and CSV reporter under
12import os 12import os
13import sys 13import sys
14from collections.abc import Iterable 14from collections.abc import Iterable
15from dataclasses import dataclass
15from datetime import date 16from datetime import date
16 17
17from tui.common import Check, ProgressCallback, ProgressEvent 18from tui.common import Check, ProgressCallback, ProgressEvent
@@ -145,15 +146,22 @@ def default_output_dir(out: str, org: str) -> str:
145 return os.path.join(out, f"github_audit_{org}_{date.today().isoformat()}") 146 return os.path.join(out, f"github_audit_{org}_{date.today().isoformat()}")
146 147
147 148
149@dataclass(frozen=True)
150class GitHubTarget:
151 """An organization to audit and the token and default branch to audit it with."""
152
153 org: str
154 token: str
155 branch: str
156
157
148# --- Runner ----------------------------------------------------------------- 158# --- Runner -----------------------------------------------------------------
149 159
150 160
151def run_audit( 161def run_audit(
162 target: GitHubTarget,
152 *, 163 *,
153 org: str,
154 token: str,
155 output_dir: str, 164 output_dir: str,
156 branch: str,
157 selected_keys: Iterable[str], 165 selected_keys: Iterable[str],
158 on_event: ProgressCallback, 166 on_event: ProgressCallback,
159) -> list[tuple[str, int]]: 167) -> list[tuple[str, int]]:
@@ -166,6 +174,7 @@ def run_audit(
166 Returns the list of (label, row_count) sections that was written to the 174 Returns the list of (label, row_count) sections that was written to the
167 summary file. 175 summary file.
168 """ 176 """
177 org, token, branch = target.org, target.token, target.branch
169 cfg = build_cfg(token) 178 cfg = build_cfg(token)
170 selected = set(selected_keys) 179 selected = set(selected_keys)
171 checks = [c for c in CHECKS if c.key in selected] 180 checks = [c for c in CHECKS if c.key in selected]
tui/gitlab_runner.py +12 −3
@@ -10,6 +10,7 @@ through a callback.
10import os 10import os
11import sys 11import sys
12from collections.abc import Iterable 12from collections.abc import Iterable
13from dataclasses import dataclass
13from datetime import date 14from datetime import date
14 15
15from tui.common import Check, ProgressCallback, ProgressEvent 16from tui.common import Check, ProgressCallback, ProgressEvent
@@ -109,14 +110,21 @@ def default_output_dir(out: str, group: str) -> str:
109 return os.path.join(out, f"gitlab_audit_{safe_group}_{date.today().isoformat()}") 110 return os.path.join(out, f"gitlab_audit_{safe_group}_{date.today().isoformat()}")
110 111
111 112
113@dataclass(frozen=True)
114class GitLabTarget:
115 """A group to audit and the token and instance URL to audit it with."""
116
117 group: str
118 token: str
119 base_url: str
120
121
112# --- Runner ----------------------------------------------------------------- 122# --- Runner -----------------------------------------------------------------
113 123
114 124
115def run_audit( 125def run_audit(
126 target: GitLabTarget,
116 *, 127 *,
117 group: str,
118 token: str,
119 base_url: str,
120 output_dir: str, 128 output_dir: str,
121 selected_keys: Iterable[str], 129 selected_keys: Iterable[str],
122 on_event: ProgressCallback, 130 on_event: ProgressCallback,
@@ -127,6 +135,7 @@ def run_audit(
127 A collector that raises is reported as an error and recorded with a count 135 A collector that raises is reported as an error and recorded with a count
128 of 0, so one bad check never aborts the whole run. 136 of 0, so one bad check never aborts the whole run.
129 """ 137 """
138 group, token, base_url = target.group, target.token, target.base_url
130 cfg = build_cfg(token, base_url) 139 cfg = build_cfg(token, base_url)
131 selected = set(selected_keys) 140 selected = set(selected_keys)
132 checks = [c for c in CHECKS if c.key in selected] 141 checks = [c for c in CHECKS if c.key in selected]
tui/platforms.py +3 −9
@@ -61,10 +61,8 @@ def _github_output_dir(s: dict) -> str:
61 61
62def _github_run(s: dict, output_dir, selected_keys, on_event): 62def _github_run(s: dict, output_dir, selected_keys, on_event):
63 return github_runner.run_audit( 63 return github_runner.run_audit(
64 org=s["org"], 64 github_runner.GitHubTarget(org=s["org"], token=s["token"], branch=s["branch"]),
65 token=s["token"],
66 output_dir=output_dir, 65 output_dir=output_dir,
67 branch=s["branch"],
68 selected_keys=selected_keys, 66 selected_keys=selected_keys,
69 on_event=on_event, 67 on_event=on_event,
70 ) 68 )
@@ -76,9 +74,7 @@ def _gitlab_output_dir(s: dict) -> str:
76 74
77def _gitlab_run(s: dict, output_dir, selected_keys, on_event): 75def _gitlab_run(s: dict, output_dir, selected_keys, on_event):
78 return gitlab_runner.run_audit( 76 return gitlab_runner.run_audit(
79 group=s["group"], 77 gitlab_runner.GitLabTarget(group=s["group"], token=s["token"], base_url=s["base_url"]),
80 token=s["token"],
81 base_url=s["base_url"],
82 output_dir=output_dir, 78 output_dir=output_dir,
83 selected_keys=selected_keys, 79 selected_keys=selected_keys,
84 on_event=on_event, 80 on_event=on_event,
@@ -91,9 +87,7 @@ def _aws_output_dir(s: dict) -> str:
91 87
92def _aws_run(s: dict, output_dir, selected_keys, on_event): 88def _aws_run(s: dict, output_dir, selected_keys, on_event):
93 return aws_runner.run_audit( 89 return aws_runner.run_audit(
94 profile=s["profile"], 90 aws_runner.AwsTarget(profile=s["profile"], region=s["region"], account=s["account"]),
95 region=s["region"],
96 account=s["account"],
97 output_dir=output_dir, 91 output_dir=output_dir,
98 selected_keys=selected_keys, 92 selected_keys=selected_keys,
99 on_event=on_event, 93 on_event=on_event,
tui/tests/test_app.py +3 −5
@@ -18,7 +18,7 @@ def _run(coro):
18 18
19 19
20def test_full_navigation(monkeypatch): 20def test_full_navigation(monkeypatch):
21 def fake_run_audit(*, org, token, output_dir, branch, selected_keys, on_event): 21 def fake_run_audit(target, *, output_dir, selected_keys, on_event):
22 on_event(gh.ProgressEvent("start", "Member roster")) 22 on_event(gh.ProgressEvent("start", "Member roster"))
23 on_event(gh.ProgressEvent("done", "Member roster", count=3)) 23 on_event(gh.ProgressEvent("done", "Member roster", count=3))
24 on_event(gh.ProgressEvent("summary", output_dir, count=3)) 24 on_event(gh.ProgressEvent("summary", output_dir, count=3))
@@ -91,7 +91,7 @@ def test_gitlab_navigation(monkeypatch):
91 monkeypatch.delenv("GITLAB_GROUP", raising=False) 91 monkeypatch.delenv("GITLAB_GROUP", raising=False)
92 monkeypatch.delenv("GITLAB_TOKEN", raising=False) 92 monkeypatch.delenv("GITLAB_TOKEN", raising=False)
93 93
94 def fake_run_audit(*, group, token, base_url, output_dir, selected_keys, on_event): 94 def fake_run_audit(target, *, output_dir, selected_keys, on_event):
95 on_event(gh.ProgressEvent("done", "Group members", count=7)) 95 on_event(gh.ProgressEvent("done", "Group members", count=7))
96 on_event(gh.ProgressEvent("summary", output_dir, count=7)) 96 on_event(gh.ProgressEvent("summary", output_dir, count=7))
97 return [("Group members", 7)] 97 return [("Group members", 7)]
@@ -131,9 +131,7 @@ def test_aws_navigation(monkeypatch):
131 for var in ("AWS_PROFILE", "AWS_DEFAULT_REGION", "AWS_AUDIT_ACCOUNT"): 131 for var in ("AWS_PROFILE", "AWS_DEFAULT_REGION", "AWS_AUDIT_ACCOUNT"):
132 monkeypatch.delenv(var, raising=False) 132 monkeypatch.delenv(var, raising=False)
133 133
134 def fake_run_audit( 134 def fake_run_audit(target, *, output_dir, selected_keys, on_event):
135 *, profile, region, account, output_dir, selected_keys, on_event
136 ):
137 on_event(gh.ProgressEvent("done", "IAM users", count=5)) 135 on_event(gh.ProgressEvent("done", "IAM users", count=5))
138 on_event(gh.ProgressEvent("summary", output_dir, count=5)) 136 on_event(gh.ProgressEvent("summary", output_dir, count=5))
139 return [("IAM users", 5)] 137 return [("IAM users", 5)]
tui/tests/test_aws_runner.py +1 −3
@@ -52,9 +52,7 @@ def fake_checks(monkeypatch):
52def run(tmp_path, keys, profile="", region="us-east-1", account=""): 52def run(tmp_path, keys, profile="", region="us-east-1", account=""):
53 events = [] 53 events = []
54 sections = r.run_audit( 54 sections = r.run_audit(
55 profile=profile, 55 r.AwsTarget(profile=profile, region=region, account=account),
56 region=region,
57 account=account,
58 output_dir=str(tmp_path), 56 output_dir=str(tmp_path),
59 selected_keys=keys, 57 selected_keys=keys,
60 on_event=events.append, 58 on_event=events.append,
tui/tests/test_github_runner.py +1 −3
@@ -55,10 +55,8 @@ def fake_checks(monkeypatch):
55def run(tmp_path, keys, fake_checks, branch="main"): 55def run(tmp_path, keys, fake_checks, branch="main"):
56 events = [] 56 events = []
57 sections = r.run_audit( 57 sections = r.run_audit(
58 org="acme", 58 r.GitHubTarget(org="acme", token="tok", branch=branch),
59 token="tok",
60 output_dir=str(tmp_path), 59 output_dir=str(tmp_path),
61 branch=branch,
62 selected_keys=keys, 60 selected_keys=keys,
63 on_event=events.append, 61 on_event=events.append,
64 ) 62 )
tui/tests/test_gitlab_runner.py +1 −3
@@ -49,9 +49,7 @@ def fake_checks(monkeypatch):
49def run(tmp_path, keys, base_url="https://gitlab.com/api/v4"): 49def run(tmp_path, keys, base_url="https://gitlab.com/api/v4"):
50 events = [] 50 events = []
51 sections = r.run_audit( 51 sections = r.run_audit(
52 group="grp", 52 r.GitLabTarget(group="grp", token="tok", base_url=base_url),
53 token="tok",
54 base_url=base_url,
55 output_dir=str(tmp_path), 53 output_dir=str(tmp_path),
56 selected_keys=keys, 54 selected_keys=keys,
57 on_event=events.append, 55 on_event=events.append,