Commit `7cf1026a07`

7cf1026a078d97cf6efebefaf6f4784665492be4

parent: 64d53350b0

Unsigned

cmc <hello@cleberg.net> · 2026-07-07 22:16 UTC

ruff formatting

Layout: unified · split

COMMIT_MESSAGE.txt added +50

		@@ -0,0 +1,50 @@
	1	Add production-ready audit sampling CLI
	2
	3	Implement a reusable audit sampling tool for CSV and Excel populations.
	4
	5	This change adds sampling/audit_sample.py and a modular sampling_tool package
	6	that supports random sampling, stratified sampling, validate-only runs, exact
	7	match filters, source row tracking, duplicate-ID handling, blank-ID handling,
	8	reconciliation outputs, methodology documentation, manifests, and run logs.
	9
	10	Key behavior:
	11	- Supports .csv, .xlsx, .xls, and .xlsm population files using pandas.
	12	- Adds _source_row_number based on source data row numbers, with first data row
	13	recorded as row 2 to align with spreadsheet conventions.
	14	- Supports CLI options and YAML config files, with CLI arguments overriding
	15	config values.
	16	- Supports simple random samples without replacement using deterministic seeds.
	17	- Supports stratified sampling by explicit counts or proportional allocation
	18	using largest remainder rounding.
	19	- Generates documented output packages containing applicable sample,
	20	population, reconciliation, exclusion, duplicate-ID, strata summary,
	21	methodology, manifest, and log files.
	22	- Fails by default on duplicate IDs while still writing duplicate and
	23	reconciliation evidence.
	24	- Allows blank IDs to be retained with warnings or excluded with evidence.
	25	- Keeps existing sampling/stratified_sample.py in place for compatibility.
	26
	27	Documentation and examples:
	28	- Adds safe example populations and a stratified YAML config under
	29	sampling/examples.
	30	- Updates sampling/README.md with installation, examples, output descriptions,
	31	reproducibility notes, and limitations.
	32	- Fixes the top-level README clone URL typo.
	33	- Updates .gitignore so output and __pycache__ directories are ignored at any
	34	folder depth.
	35
	36	Tests:
	37	- Adds pytest coverage for random sampling, reproducibility, stratified counts,
	38	stratified proportions, validation behavior, duplicate-ID evidence,
	39	blank-ID exclusion, filters, validate-only behavior, and reconciliation math.
	40
	41	Validation:
	42	- Ran .venv/bin/python -m pytest sampling/tests.
	43	- Result: 10 passed.
	44	- Ran manual CLI validation for random, stratified-counts,
	45	stratified-proportions, validate-only, config-file, duplicate-ID failure, and
	46	blank-ID exclusion cases.
	47	- Programmatically validated 7 generated output packages for required files,
	48	manifest completeness, reconciliation tie-outs, sample schema, strata math,
	49	excluded-row evidence, duplicate-ID evidence, methodology content, and log
	50	status.

TESTING_PROOF_LOG.md added +307

		@@ -0,0 +1,307 @@
	1	# Testing Proof Log
	2
	3	Date: 2026-07-07
	4	Repository: `/Users/cmc/git/audit-labs/audit-tools`
	5
	6	## Unit Tests
	7
	8	Command:
	9
	10	```bash
	11	.venv/bin/python -m pytest sampling/tests
	12	```
	13
	14	Observed output:
	15
	16	```text
	17	============================= test session starts ==============================
	18	platform darwin -- Python 3.14.6, pytest-9.1.1, pluggy-1.6.0
	19	rootdir: /Users/cmc/git/audit-labs/audit-tools
	20	plugins: dash-4.4.0
	21	collected 10 items
	22
	23	sampling/tests/test_random_sample.py ... [ 30%]
	24	sampling/tests/test_reconciliation.py . [ 40%]
	25	sampling/tests/test_stratified_sample.py .. [ 60%]
	26	sampling/tests/test_validation.py .... [100%]
	27
	28	============================== 10 passed in 0.42s ==============================
	29	```
	30
	31	Result: PASS
	32
	33	## Manual CLI Runs
	34
	35	The following output packages were generated under `output/validation_suite`.
	36	The `output` directory is ignored by git.
	37
	38	### Random Sample
	39
	40	Command:
	41
	42	```bash
	43	.venv/bin/python sampling/audit_sample.py \
	44	--input sampling/examples/users_population.csv \
	45	--id-column "User ID" \
	46	--method random \
	47	--sample-size 5 \
	48	--seed 20260707 \
	49	--out ./output/validation_suite
	50	```
	51
	52	Output package:
	53
	54	```text
	55	output/validation_suite/sample_2026-07-07_220414
	56	```
	57
	58	Observed files:
	59
	60	```text
	61	manifest.json
	62	methodology.txt
	63	population_reconciliation.csv
	64	population_validated.csv
	65	run.log
	66	sample.csv
	67	```
	68
	69	Result: PASS
	70
	71	### Stratified Sample By Counts
	72
	73	Command:
	74
	75	```bash
	76	.venv/bin/python sampling/audit_sample.py \
	77	--input sampling/examples/changes_population.csv \
	78	--id-column "Change ID" \
	79	--method stratified \
	80	--stratify-column "Change Type" \
	81	--strata-counts "Normal=3,Emergency=2,Standard=2" \
	82	--seed 20260707 \
	83	--out ./output/validation_suite
	84	```
	85
	86	Output package:
	87
	88	```text
	89	output/validation_suite/sample_2026-07-07_220418
	90	```
	91
	92	Observed files:
	93
	94	```text
	95	manifest.json
	96	methodology.txt
	97	population_reconciliation.csv
	98	population_validated.csv
	99	run.log
	100	sample.csv
	101	strata_summary.csv
	102	```
	103
	104	Result: PASS
	105
	106	### Stratified Sample By Proportions
	107
	108	Command:
	109
	110	```bash
	111	.venv/bin/python sampling/audit_sample.py \
	112	--input sampling/examples/changes_population.csv \
	113	--id-column "Change ID" \
	114	--method stratified \
	115	--stratify-column "Change Type" \
	116	--strata-proportions "Normal=0.50,Emergency=0.25,Standard=0.25" \
	117	--sample-size 8 \
	118	--seed 20260707 \
	119	--out ./output/validation_suite
	120	```
	121
	122	Output package:
	123
	124	```text
	125	output/validation_suite/sample_2026-07-07_220425
	126	```
	127
	128	Observed files:
	129
	130	```text
	131	manifest.json
	132	methodology.txt
	133	population_reconciliation.csv
	134	population_validated.csv
	135	run.log
	136	sample.csv
	137	strata_summary.csv
	138	```
	139
	140	Result: PASS
	141
	142	### Validate-Only With Filter
	143
	144	Command:
	145
	146	```bash
	147	.venv/bin/python sampling/audit_sample.py \
	148	--input sampling/examples/changes_population.csv \
	149	--id-column "Change ID" \
	150	--method validate-only \
	151	--filter "Status=Closed" \
	152	--out ./output/validation_suite
	153	```
	154
	155	Output package:
	156
	157	```text
	158	output/validation_suite/sample_2026-07-07_220430
	159	```
	160
	161	Observed files:
	162
	163	```text
	164	excluded_rows.csv
	165	manifest.json
	166	methodology.txt
	167	population_reconciliation.csv
	168	population_validated.csv
	169	run.log
	170	```
	171
	172	Result: PASS. No `sample.csv` was produced, as expected for validate-only.
	173
	174	### YAML Config Run
	175
	176	Command:
	177
	178	```bash
	179	.venv/bin/python sampling/audit_sample.py \
	180	--config sampling/examples/stratified_config.yml \
	181	--out ./output/validation_suite
	182	```
	183
	184	Output package:
	185
	186	```text
	187	output/validation_suite/sample_2026-07-07_220436
	188	```
	189
	190	Observed files:
	191
	192	```text
	193	manifest.json
	194	methodology.txt
	195	population_reconciliation.csv
	196	population_validated.csv
	197	run.log
	198	sample.csv
	199	strata_summary.csv
	200	```
	201
	202	Result: PASS
	203
	204	### Duplicate-ID Failure Evidence
	205
	206	Command:
	207
	208	```bash
	209	.venv/bin/python sampling/audit_sample.py \
	210	--input /private/tmp/audit_sample_validation_inputs/duplicate_ids.csv \
	211	--id-column ID \
	212	--method validate-only \
	213	--out ./output/validation_suite
	214	```
	215
	216	Output package:
	217
	218	```text
	219	output/validation_suite/sample_2026-07-07_220453
	220	```
	221
	222	Observed files:
	223
	224	```text
	225	duplicate_ids.csv
	226	manifest.json
	227	methodology.txt
	228	population_reconciliation.csv
	229	population_validated.csv
	230	run.log
	231	```
	232
	233	Result: PASS. The command failed as intended because duplicate IDs are rejected
	234	by default, and `duplicate_ids.csv` was still written as evidence.
	235
	236	### Blank-ID Exclusion Evidence
	237
	238	Command:
	239
	240	```bash
	241	.venv/bin/python sampling/audit_sample.py \
	242	--input /private/tmp/audit_sample_validation_inputs/blank_ids.csv \
	243	--id-column ID \
	244	--method validate-only \
	245	--exclude-blank-id \
	246	--out ./output/validation_suite
	247	```
	248
	249	Output package:
	250
	251	```text
	252	output/validation_suite/sample_2026-07-07_220500
	253	```
	254
	255	Observed files:
	256
	257	```text
	258	excluded_rows.csv
	259	manifest.json
	260	methodology.txt
	261	population_reconciliation.csv
	262	population_validated.csv
	263	run.log
	264	```
	265
	266	Result: PASS. Blank-ID rows were excluded and written to `excluded_rows.csv`.
	267
	268	## Output Integrity Validation
	269
	270	Command:
	271
	272	```bash
	273	.venv/bin/python - <<'PY'
	274	# Programmatic validation over output/validation_suite:
	275	# - required files by method
	276	# - manifest required keys
	277	# - manifest output_files matches files on disk
	278	# - reconciliation tie-outs
	279	# - population_validated row counts and _source_row_number
	280	# - sample.csv metadata columns and counts
	281	# - strata_summary schema and counts
	282	# - excluded_rows.csv evidence
	283	# - duplicate_ids.csv evidence
	284	# - run.log final status
	285	PY
	286	```
	287
	288	Observed output:
	289
	290	```text
	291	Validated 7 output package(s).
	292	sample_2026-07-07_220414: method=random, status=success, file_count=6
	293	sample_2026-07-07_220418: method=stratified, status=success, file_count=7
	294	sample_2026-07-07_220425: method=stratified, status=success, file_count=7
	295	sample_2026-07-07_220430: method=validate-only, status=success, file_count=6
	296	sample_2026-07-07_220436: method=stratified, status=success, file_count=7
	297	sample_2026-07-07_220453: method=validate-only, status=failed, file_count=6
	298	sample_2026-07-07_220500: method=validate-only, status=success, file_count=6
	299	All output package integrity checks passed.
	300	```
	301
	302	Result: PASS
	303
	304	## Overall Result
	305
	306	All automated tests passed, all manual CLI paths completed with expected
	307	behavior, and all generated output packages passed integrity validation.

sampling/sampling_tool/cli.py +3 −1

		@@ -100,7 +100,9 @@ def merge_options(args: Namespace, config: dict[str, object]) -> SimpleNamespace
100	100	if merged["method"] is None:
101	101	raise AuditSamplingError("--method is required unless provided by --config.")
102	102	if merged["method"] not in {"random", "stratified", "validate-only"}:
103		raise AuditSamplingError("--method must be random, stratified, or validate-only.")
	103	raise AuditSamplingError(
	104	"--method must be random, stratified, or validate-only."
	105	)
104	106	if merged["sample_size"] is not None:
105	107	merged["sample_size"] = int(merged["sample_size"])
106	108	if merged["seed"] is not None:

sampling/sampling_tool/methods.py +6 −2

		@@ -86,7 +86,9 @@ def largest_remainder_allocation(
86	86	return allocation
87	87
88	88
89		def random_sample(population: pd.DataFrame, sample_size: int, seed: int) -> pd.DataFrame:
	89	def random_sample(
	90	population: pd.DataFrame, sample_size: int, seed: int
	91	) -> pd.DataFrame:
90	92	return population.sample(n=sample_size, random_state=seed)
91	93
92	94
		@@ -108,7 +110,9 @@ def stratified_sample(
108	110	f"requested {requested}. Use --allow-shortfall to continue."
109	111	)
110	112	if actual_count:
111		sampled = stratum_population.sample(n=actual_count, random_state=seed + index)
	113	sampled = stratum_population.sample(
	114	n=actual_count, random_state=seed + index
	115	)
112	116	samples.append(sampled)
113	117	summary.append(
114	118	{

sampling/sampling_tool/reconciliation.py +4 −1

		@@ -21,7 +21,10 @@ def build_reconciliation(
21	21	("Duplicate IDs", duplicate_id_count),
22	22	("Excluded rows", excluded_rows),
23	23	("Validated population rows", validated_rows),
24		("Requested sample size", "" if requested_sample_size is None else requested_sample_size),
	24	(
	25	"Requested sample size",
	26	"" if requested_sample_size is None else requested_sample_size,
	27	),
25	28	("Final sample size", final_sample_size),
26	29	("Unsampled population rows", unsampled),
27	30	]

Commit 7cf1026a07

Commit `7cf1026a07`