audit-labs/audit-tools
A collection of scripts, queries, and other goodies you can use in an audit.
clone: git clone https://gitbay.org/audit-labs/audit-tools.git
93e4c1e5cc359054150aa0cfb090b05b061110da
verified · cmc
author: Christian Cleberg <hello@cleberg.net> · 2026-07-07T22:16:25Z
COMMIT_MESSAGE.txt | 50 +++++ TESTING_PROOF_LOG.md | 307 +++++++++++++++++++++++++++++++ sampling/sampling_tool/cli.py | 4 +- sampling/sampling_tool/methods.py | 8 +- sampling/sampling_tool/reconciliation.py | 5 +- 5 files changed, 370 insertions(+), 4 deletions(-) new file mode 100644 @@ -0,0 +1,50 @@ +Add production-ready audit sampling CLI + +Implement a reusable audit sampling tool for CSV and Excel populations. + +This change adds sampling/audit_sample.py and a modular sampling_tool package +that supports random sampling, stratified sampling, validate-only runs, exact +match filters, source row tracking, duplicate-ID handling, blank-ID handling, +reconciliation outputs, methodology documentation, manifests, and run logs. + +Key behavior: +- Supports .csv, .xlsx, .xls, and .xlsm population files using pandas. +- Adds _source_row_number based on source data row numbers, with first data row + recorded as row 2 to align with spreadsheet conventions. +- Supports CLI options and YAML config files, with CLI arguments overriding + config values. +- Supports simple random samples without replacement using deterministic seeds. +- Supports stratified sampling by explicit counts or proportional allocation + using largest remainder rounding. +- Generates documented output packages containing applicable sample, + population, reconciliation, exclusion, duplicate-ID, strata summary, + methodology, manifest, and log files. +- Fails by default on duplicate IDs while still writing duplicate and + reconciliation evidence. +- Allows blank IDs to be retained with warnings or excluded with evidence. +- Keeps existing sampling/stratified_sample.py in place for compatibility. + +Documentation and examples: +- Adds safe example populations and a stratified YAML config under + sampling/examples. +- Updates sampling/README.md with installation, examples, output descriptions, + reproducibility notes, and limitations. +- Fixes the top-level README clone URL typo. +- Updates .gitignore so output and __pycache__ directories are ignored at any + folder depth. + +Tests: +- Adds pytest coverage for random sampling, reproducibility, stratified counts, + stratified proportions, validation behavior, duplicate-ID evidence, + blank-ID exclusion, filters, validate-only behavior, and reconciliation math. + +Validation: +- Ran .venv/bin/python -m pytest sampling/tests. +- Result: 10 passed. +- Ran manual CLI validation for random, stratified-counts, + stratified-proportions, validate-only, config-file, duplicate-ID failure, and + blank-ID exclusion cases. +- Programmatically validated 7 generated output packages for required files, + manifest completeness, reconciliation tie-outs, sample schema, strata math, + excluded-row evidence, duplicate-ID evidence, methodology content, and log + status. new file mode 100644 @@ -0,0 +1,307 @@ +# Testing Proof Log + +Date: 2026-07-07 +Repository: `/Users/cmc/git/audit-labs/audit-tools` + +## Unit Tests + +Command: + +```bash +.venv/bin/python -m pytest sampling/tests +``` + +Observed output: + +```text +============================= test session starts ============================== +platform darwin -- Python 3.14.6, pytest-9.1.1, pluggy-1.6.0 +rootdir: /Users/cmc/git/audit-labs/audit-tools +plugins: dash-4.4.0 +collected 10 items + +sampling/tests/test_random_sample.py ... [ 30%] +sampling/tests/test_reconciliation.py . [ 40%] +sampling/tests/test_stratified_sample.py .. [ 60%] +sampling/tests/test_validation.py .... [100%] + +============================== 10 passed in 0.42s ============================== +``` + +Result: PASS + +## Manual CLI Runs + +The following output packages were generated under `output/validation_suite`. +The `output` directory is ignored by git. + +### Random Sample + +Command: + +```bash +.venv/bin/python sampling/audit_sample.py \ + --input sampling/examples/users_population.csv \ + --id-column "User ID" \ + --method random \ + --sample-size 5 \ + --seed 20260707 \ + --out ./output/validation_suite +``` + +Output package: + +```text +output/validation_suite/sample_2026-07-07_220414 +``` + +Observed files: + +```text +manifest.json +methodology.txt +population_reconciliation.csv +population_validated.csv +run.log +sample.csv +``` + +Result: PASS + +### Stratified Sample By Counts + +Command: + +```bash +.venv/bin/python sampling/audit_sample.py \ + --input sampling/examples/changes_population.csv \ + --id-column "Change ID" \ + --method stratified \ + --stratify-column "Change Type" \ + --strata-counts "Normal=3,Emergency=2,Standard=2" \ + --seed 20260707 \ + --out ./output/validation_suite +``` + +Output package: + +```text +output/validation_suite/sample_2026-07-07_220418 +``` + +Observed files: + +```text +manifest.json +methodology.txt +population_reconciliation.csv +population_validated.csv +run.log +sample.csv +strata_summary.csv +``` + +Result: PASS + +### Stratified Sample By Proportions + +Command: + +```bash +.venv/bin/python sampling/audit_sample.py \ + --input sampling/examples/changes_population.csv \ + --id-column "Change ID" \ + --method stratified \ + --stratify-column "Change Type" \ + --strata-proportions "Normal=0.50,Emergency=0.25,Standard=0.25" \ + --sample-size 8 \ + --seed 20260707 \ + --out ./output/validation_suite +``` + +Output package: + +```text +output/validation_suite/sample_2026-07-07_220425 +``` + +Observed files: + +```text +manifest.json +methodology.txt +population_reconciliation.csv +population_validated.csv +run.log +sample.csv +strata_summary.csv +``` + +Result: PASS + +### Validate-Only With Filter + +Command: + +```bash +.venv/bin/python sampling/audit_sample.py \ + --input sampling/examples/changes_population.csv \ + --id-column "Change ID" \ + --method validate-only \ + --filter "Status=Closed" \ + --out ./output/validation_suite +``` + +Output package: + +```text +output/validation_suite/sample_2026-07-07_220430 +``` + +Observed files: + +```text +excluded_rows.csv +manifest.json +methodology.txt +population_reconciliation.csv +population_validated.csv +run.log +``` + +Result: PASS. No `sample.csv` was produced, as expected for validate-only. + +### YAML Config Run + +Command: + +```bash +.venv/bin/python sampling/audit_sample.py \ + --config sampling/examples/stratified_config.yml \ + --out ./output/validation_suite +``` + +Output package: + +```text +output/validation_suite/sample_2026-07-07_220436 +``` + +Observed files: + +```text +manifest.json +methodology.txt +population_reconciliation.csv +population_validated.csv +run.log +sample.csv +strata_summary.csv +``` + +Result: PASS + +### Duplicate-ID Failure Evidence + +Command: + +```bash +.venv/bin/python sampling/audit_sample.py \ + --input /private/tmp/audit_sample_validation_inputs/duplicate_ids.csv \ + --id-column ID \ + --method validate-only \ + --out ./output/validation_suite +``` + +Output package: + +```text +output/validation_suite/sample_2026-07-07_220453 +``` + +Observed files: + +```text +duplicate_ids.csv +manifest.json +methodology.txt +population_reconciliation.csv +population_validated.csv +run.log +``` + +Result: PASS. The command failed as intended because duplicate IDs are rejected +by default, and `duplicate_ids.csv` was still written as evidence. + +### Blank-ID Exclusion Evidence + +Command: + +```bash +.venv/bin/python sampling/audit_sample.py \ + --input /private/tmp/audit_sample_validation_inputs/blank_ids.csv \ + --id-column ID \ + --method validate-only \ + --exclude-blank-id \ + --out ./output/validation_suite +``` + +Output package: + +```text +output/validation_suite/sample_2026-07-07_220500 +``` + +Observed files: + +```text +excluded_rows.csv +manifest.json +methodology.txt +population_reconciliation.csv +population_validated.csv +run.log +``` + +Result: PASS. Blank-ID rows were excluded and written to `excluded_rows.csv`. + +## Output Integrity Validation + +Command: + +```bash +.venv/bin/python - <<'PY' +# Programmatic validation over output/validation_suite: +# - required files by method +# - manifest required keys +# - manifest output_files matches files on disk +# - reconciliation tie-outs +# - population_validated row counts and _source_row_number +# - sample.csv metadata columns and counts +# - strata_summary schema and counts +# - excluded_rows.csv evidence +# - duplicate_ids.csv evidence +# - run.log final status +PY +``` + +Observed output: + +```text +Validated 7 output package(s). +sample_2026-07-07_220414: method=random, status=success, file_count=6 +sample_2026-07-07_220418: method=stratified, status=success, file_count=7 +sample_2026-07-07_220425: method=stratified, status=success, file_count=7 +sample_2026-07-07_220430: method=validate-only, status=success, file_count=6 +sample_2026-07-07_220436: method=stratified, status=success, file_count=7 +sample_2026-07-07_220453: method=validate-only, status=failed, file_count=6 +sample_2026-07-07_220500: method=validate-only, status=success, file_count=6 +All output package integrity checks passed. +``` + +Result: PASS + +## Overall Result + +All automated tests passed, all manual CLI paths completed with expected +behavior, and all generated output packages passed integrity validation. @@ -100,7 +100,9 @@ def merge_options(args: Namespace, config: dict[str, object]) -> SimpleNamespace if merged["method"] is None: raise AuditSamplingError("--method is required unless provided by --config.") if merged["method"] not in {"random", "stratified", "validate-only"}: - raise AuditSamplingError("--method must be random, stratified, or validate-only.") + raise AuditSamplingError( + "--method must be random, stratified, or validate-only." + ) if merged["sample_size"] is not None: merged["sample_size"] = int(merged["sample_size"]) if merged["seed"] is not None: @@ -86,7 +86,9 @@ def largest_remainder_allocation( return allocation -def random_sample(population: pd.DataFrame, sample_size: int, seed: int) -> pd.DataFrame: +def random_sample( + population: pd.DataFrame, sample_size: int, seed: int +) -> pd.DataFrame: return population.sample(n=sample_size, random_state=seed) @@ -108,7 +110,9 @@ def stratified_sample( f"requested {requested}. Use --allow-shortfall to continue." ) if actual_count: - sampled = stratum_population.sample(n=actual_count, random_state=seed + index) + sampled = stratum_population.sample( + n=actual_count, random_state=seed + index + ) samples.append(sampled) summary.append( { @@ -21,7 +21,10 @@ def build_reconciliation( ("Duplicate IDs", duplicate_id_count), ("Excluded rows", excluded_rows), ("Validated population rows", validated_rows), - ("Requested sample size", "" if requested_sample_size is None else requested_sample_size), + ( + "Requested sample size", + "" if requested_sample_size is None else requested_sample_size, + ), ("Final sample size", final_sample_size), ("Unsampled population rows", unsampled), ]