audit-labs/audit-tools

A collection of scripts, queries, and other goodies you can use in an audit. audit automation compliance evidence scripts

Commit 8c281c79e7

8c281c79e70bda27151bc713a71baff11cab7484

parent: 707d120a78

Unsigned

cmc <hello@cleberg.net> · 2026-10-29 21:02 UTC

Standard-library sampling engine, sample sizes, and deviation limits (!3)

- `sampling/sampling_tool/engine.py` (standard library only): `select_random`, `select_systematic`, `select_stratified`, each seeded from `audit-sample:<seed>[:<stratum>]`, so a seed selects the same rows on every install, with or without pandas. `ENGINE_VERSION = "2"` is written to sample manifests.
- `attribute_sample_size(risk, tolerable, expected)` and `upper_deviation_limit(n, deviations, risk)` from the binomial distribution; limits are rounded up to one decimal. `evaluate_attribute_sample`; `FREQUENCY_PRESETS`.
- `random_sample` and `stratified_sample` (pandas) select through the engine; `audit-sample --method systematic`.
- Selections differ from version 1 for the same seed (they used `DataFrame.sample`); version 1 runs stay verifiable from their outputs.
- Tests: every cell of the 5% and 10% risk AICPA evaluation tables reproduced for n = 20, 25, 50, 100, 200, 500 (never below the table, at most 0.1 above where the table rounds to nearest; for example n = 200 with 6 deviations is 5.83%, shown as 5.8); 15 AICPA sample sizes; pinned selections. 64 passed (`tui/tests/test_app.py` needs `textual`).

Layout: unified · split

CHANGELOG.md +7
@@ -8,6 +8,13 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
88
99### Added
1010
11- `sampling_tool.engine`: one standard-library selection algorithm (random, systematic, stratified) seeded from strings, so a seed selects the same rows with or without pandas and on every install. Sample manifests record `engine` and `engine_version` (2). `audit-sample --method systematic`.
12- Attribute sample sizes and upper deviation limits from the binomial distribution, matching the AICPA Audit Sampling guide tables for large populations; default sample sizes by control frequency.
13
14### Changed
15
16- `random_sample` and `stratified_sample` select through the engine. The same seed now selects different rows than version 1 did; earlier runs remain verifiable from their saved outputs.
17
1118- Installable package: `pip install audit-tools` installs `tui`, `applications`,
1219 and `sampling` as packages, the `audit-sample` and `audit-tui` commands, and the
1320 SQL/OS procedure library and sampling examples under `<prefix>/share/audit-tools/`.
sampling/sampling_tool/cli.py +9 −5
@@ -18,6 +18,7 @@ from .methods import (
1818 largest_remainder_allocation,
1919 random_sample,
2020 stratified_sample,
21 systematic_sample,
2122)
2223from .reconciliation import build_reconciliation, build_strata_summary
2324from .reporting import RunLogger, build_methodology
@@ -34,7 +35,7 @@ def build_parser() -> ArgumentParser:
3435 parser.add_argument("--input")
3536 parser.add_argument("--sheet")
3637 parser.add_argument("--id-column")
37 parser.add_argument("--method", choices=["random", "stratified", "validate-only"])
38 parser.add_argument("--method", choices=["random", "systematic", "stratified", "validate-only"])
3839 parser.add_argument("--sample-size", type=int)
3940 parser.add_argument("--stratify-column")
4041 parser.add_argument("--strata-counts")
@@ -106,9 +107,9 @@ def merge_options(args: Namespace, config: dict[str, object]) -> SimpleNamespace
106107 raise AuditSamplingError("--input is required unless provided by --config.")
107108 if merged["method"] is None:
108109 raise AuditSamplingError("--method is required unless provided by --config.")
109 if merged["method"] not in {"random", "stratified", "validate-only"}:
110 if merged["method"] not in {"random", "systematic", "stratified", "validate-only"}:
110111 raise AuditSamplingError(
111 "--method must be random, stratified, or validate-only."
112 "--method must be random, systematic, stratified, or validate-only."
112113 )
113114 if merged["sample_size"] is not None:
114115 merged["sample_size"] = int(merged["sample_size"])
@@ -186,7 +187,7 @@ def run(options) -> Path:
186187 print(f"WARNING: {warning}")
187188 logger.warning(warning)
188189
189 if options.method in {"random", "stratified"}:
190 if options.method in {"random", "systematic", "stratified"}:
190191 random_seed, generated = ensure_seed(options.seed)
191192 if generated:
192193 warning = f"No seed provided; generated seed {random_seed}."
@@ -196,6 +197,9 @@ def run(options) -> Path:
196197 if options.method == "random":
197198 sample = random_sample(validated, options.sample_size, random_seed)
198199 sample = add_sample_metadata(sample, options, random_seed, timestamp, None)
200 elif options.method == "systematic":
201 sample = systematic_sample(validated, options.sample_size, random_seed)
202 sample = add_sample_metadata(sample, options, random_seed, timestamp, None)
199203 elif options.method == "stratified":
200204 counts = validation.strata_counts
201205 if validation.strata_proportions:
@@ -318,7 +322,7 @@ def _write_outputs(
318322) -> None:
319323 write_csv(validated, run_dir / _POPULATION_VALIDATED_CSV)
320324 _track(output_files, _POPULATION_VALIDATED_CSV)
321 if options.method in {"random", "stratified"}:
325 if options.method in {"random", "systematic", "stratified"}:
322326 write_csv(sample, run_dir / "sample.csv")
323327 _track(output_files, "sample.csv")
324328 if not excluded_rows.empty:
sampling/sampling_tool/engine.py added +124
@@ -0,0 +1,124 @@
1"""The sampling engine: one deterministic, standard-library selection algorithm.
2
3The same seed selects the same items on every install, with or without
4pandas, in the `audit-sample` command and in audit-workbench. Each stream is
5seeded from a string (`audit-sample:<seed>` and, per stratum,
6`audit-sample:<seed>:<stratum>`), which Python's `random` hashes the same way
7on every platform. Selections are lists of 0-based row positions.
8
9Also here: attribute sample sizes and upper deviation limits from the
10binomial distribution (large-population tables, as in the AICPA Audit
11Sampling guide), and default control-frequency sample sizes.
12"""
13
14from __future__ import annotations
15
16import math
17import random
18from decimal import ROUND_CEILING, Decimal
19
20ENGINE = "audit-sample-engine"
21ENGINE_VERSION = "2"
22
23# Sample sizes by control frequency, for controls tested without statistical sizing.
24# Firms can override these (audit-workbench reads `sampling_presets` from engagement.yaml).
25FREQUENCY_PRESETS = {
26 "annual": 1,
27 "quarterly": 2,
28 "monthly": 2,
29 "weekly": 5,
30 "daily": 20,
31 "multiple-times-daily": 25,
32}
33
34
35def _rng(seed: int, stratum: str | None = None) -> random.Random:
36 return random.Random(f"audit-sample:{seed}" + ("" if stratum is None else f":{stratum}"))
37
38
39def select_random(population_size: int, sample_size: int, seed: int) -> list[int]:
40 """Positions of a simple random sample without replacement, in selection order."""
41 if not 0 < sample_size <= population_size:
42 raise ValueError(f"sample size {sample_size} must be between 1 and the population size {population_size}")
43 return _rng(seed).sample(range(population_size), sample_size)
44
45
46def select_systematic(population_size: int, sample_size: int, seed: int) -> list[int]:
47 """Every k-th position (k = population / sample, fractional) from a random start."""
48 if not 0 < sample_size <= population_size:
49 raise ValueError(f"sample size {sample_size} must be between 1 and the population size {population_size}")
50 interval = population_size / sample_size
51 start = _rng(seed).random() * interval
52 return [int(start + i * interval) for i in range(sample_size)]
53
54
55def select_stratified(
56 strata: list[str], counts: dict[str, int], seed: int, allow_shortfall: bool = False
57) -> tuple[list[int], list[dict]]:
58 """Positions sampled per stratum (`strata[i]` is row i's stratum), and a summary per stratum."""
59 chosen: list[int] = []
60 summary: list[dict] = []
61 for stratum in sorted(counts):
62 requested = counts[stratum]
63 pool = [i for i, s in enumerate(strata) if s == stratum]
64 actual = min(requested, len(pool))
65 if actual < requested and not allow_shortfall:
66 raise ValueError(f"Stratum '{stratum}' has {len(pool)} rows; requested {requested}.")
67 chosen.extend(_rng(seed, stratum).sample(pool, actual) if actual else [])
68 summary.append(
69 {
70 "Stratum": stratum,
71 "Population Count": len(pool),
72 "Requested Sample Count": requested,
73 "Actual Sample Count": actual,
74 "Shortfall": requested - actual,
75 }
76 )
77 return chosen, summary
78
79
80def _binomial_cdf(k: int, n: int, p: float) -> float:
81 return sum(math.comb(n, i) * p**i * (1 - p) ** (n - i) for i in range(k + 1))
82
83
84def attribute_sample_size(risk: float, tolerable_rate: float, expected_rate: float = 0.0) -> dict:
85 """Smallest n whose upper deviation limit, with n × expected rate (rounded up) deviations, is within tolerance.
86
87 `risk` is the risk of overreliance (0.05 for 95% confidence). Large-population
88 (binomial) sizes, as in the AICPA tables.
89 """
90 if not (0 < risk < 1 and 0 < tolerable_rate < 1 and 0 <= expected_rate < tolerable_rate):
91 raise ValueError("need 0 < risk < 1, 0 < tolerable rate < 1, and 0 <= expected rate < tolerable rate")
92 for n in range(1, 5001):
93 k = math.ceil(n * expected_rate - 1e-12)
94 if _binomial_cdf(k, n, tolerable_rate) <= risk:
95 return {"sample_size": n, "expected_deviations": k}
96 raise ValueError("no sample size up to 5000 meets these parameters")
97
98
99def upper_deviation_limit(sample_size: int, deviations: int, risk: float) -> float:
100 """Upper deviation limit in percent, rounded up to one decimal (never understated)."""
101 if not (0 <= deviations <= sample_size and sample_size > 0 and 0 < risk < 1):
102 raise ValueError("need 0 <= deviations <= sample size and 0 < risk < 1")
103 if deviations == sample_size:
104 return 100.0
105 lo, hi = 0.0, 1.0
106 for _ in range(80):
107 mid = (lo + hi) / 2
108 if _binomial_cdf(deviations, sample_size, mid) > risk:
109 lo = mid
110 else:
111 hi = mid
112 return float(Decimal(hi * 100).quantize(Decimal("0.1"), rounding=ROUND_CEILING))
113
114
115def evaluate_attribute_sample(sample_size: int, deviations: int, risk: float, tolerable_rate: float) -> dict:
116 udl = upper_deviation_limit(sample_size, deviations, risk)
117 return {
118 "sample_size": sample_size,
119 "deviations": deviations,
120 "sample_rate_pct": round(100 * deviations / sample_size, 2),
121 "upper_deviation_limit_pct": udl,
122 "tolerable_rate_pct": round(100 * tolerable_rate, 2),
123 "supports_reliance": udl <= 100 * tolerable_rate,
124 }
sampling/sampling_tool/manifest.py +3 −1
@@ -5,7 +5,7 @@ from __future__ import annotations
55import json
66from pathlib import Path
77
8from . import __version__
8from . import __version__, engine
99
1010
1111def build_manifest(
@@ -29,6 +29,8 @@ def build_manifest(
2929 return {
3030 "tool": "audit_sample",
3131 "version": __version__,
32 "engine": engine.ENGINE,
33 "engine_version": engine.ENGINE_VERSION,
3234 "run_timestamp_utc": run_timestamp_utc,
3335 "input_file": str(input_file),
3436 "input_sha256": input_sha256,
sampling/sampling_tool/methods.py +14 −29
@@ -7,6 +7,7 @@ import random
77
88import pandas as pd
99
10from . import engine
1011from .io import AuditSamplingError
1112
1213
@@ -89,7 +90,13 @@ def largest_remainder_allocation(
8990def random_sample(
9091 population: pd.DataFrame, sample_size: int, seed: int
9192) -> pd.DataFrame:
92 return population.sample(n=sample_size, random_state=seed)
93 return population.iloc[engine.select_random(len(population), sample_size, seed)]
94
95
96def systematic_sample(
97 population: pd.DataFrame, sample_size: int, seed: int
98) -> pd.DataFrame:
99 return population.iloc[engine.select_systematic(len(population), sample_size, seed)]
93100
94101
95102def stratified_sample(
@@ -99,31 +106,9 @@ def stratified_sample(
99106 seed: int,
100107 allow_shortfall: bool,
101108) -> tuple[pd.DataFrame, list[dict[str, object]]]:
102 samples: list[pd.DataFrame] = []
103 summary: list[dict[str, object]] = []
104 for index, (stratum, requested) in enumerate(counts.items()):
105 stratum_population = population[population[stratify_column] == stratum]
106 actual_count = min(requested, len(stratum_population))
107 if actual_count < requested and not allow_shortfall:
108 raise AuditSamplingError(
109 f"Stratum '{stratum}' has {len(stratum_population)} rows; "
110 f"requested {requested}. Use --allow-shortfall to continue."
111 )
112 if actual_count:
113 sampled = stratum_population.sample(
114 n=actual_count, random_state=seed + index
115 )
116 samples.append(sampled)
117 summary.append(
118 {
119 "Stratum": stratum,
120 "Population Count": len(stratum_population),
121 "Requested Sample Count": requested,
122 "Actual Sample Count": actual_count,
123 "Shortfall": requested - actual_count,
124 }
125 )
126
127 if samples:
128 return pd.concat(samples), summary
129 return population.iloc[0:0].copy(), summary
109 strata = population[stratify_column].astype(str).tolist()
110 try:
111 positions, summary = engine.select_stratified(strata, counts, seed, allow_shortfall)
112 except ValueError as exc:
113 raise AuditSamplingError(f"{exc} Use --allow-shortfall to continue.") from exc
114 return population.iloc[positions], summary
sampling/sampling_tool/validation.py +1 −1
@@ -91,7 +91,7 @@ def validate_and_prepare(population: pd.DataFrame, options) -> ValidationResult:
9191
9292 if options.sample_size is not None and options.sample_size <= 0:
9393 raise AuditSamplingError("--sample-size must be a positive integer.")
94 needs_sample_size = options.method == "random" or (
94 needs_sample_size = options.method in ("random", "systematic") or (
9595 options.method == "stratified" and bool(options.strata_proportions)
9696 )
9797 if needs_sample_size:
sampling/tests/test_engine.py added +62
@@ -0,0 +1,62 @@
1"""The stdlib sampling engine: stable selections, AICPA sample sizes and upper deviation limits."""
2
3import pytest
4
5from sampling.sampling_tool import engine
6
7# AICPA Audit Sampling guide, sample evaluation tables (large population), upper deviation limit in %.
8UDL_5 = {
9 20: [14.0, 21.7, 28.3, 34.4, 40.2, 45.6, 50.8, 55.9, 60.7, 65.4, 69.9],
10 25: [11.3, 17.7, 23.2, 28.2, 33.0, 37.6, 42.0, 46.3, 50.4, 54.4, 58.4],
11 50: [5.9, 9.2, 12.1, 14.8, 17.4, 19.9, 22.4, 24.7, 27.1, 29.4, 31.6],
12 100: [3.0, 4.7, 6.2, 7.6, 9.0, 10.3, 11.5, 12.8, 14.0, 15.2, 16.4],
13 200: [1.5, 2.4, 3.2, 3.9, 4.6, 5.2, 5.8, 6.5, 7.2, 7.8, 8.4],
14 500: [0.6, 1.0, 1.3, 1.6, 1.9, 2.1, 2.4, 2.7, 2.9, 3.2, 3.4],
15}
16UDL_10 = {
17 20: [10.9, 18.1, 24.5, 30.5, 36.1, 41.5, 46.8, 51.9, 56.8, 61.6, 66.2],
18 50: [4.6, 7.6, 10.3, 12.9, 15.4, 17.8, 20.2, 22.5, 24.7, 27.0, 29.2],
19 100: [2.3, 3.9, 5.3, 6.6, 7.9, 9.1, 10.3, 11.5, 12.7, 13.9, 15.0],
20 500: [0.5, 0.8, 1.1, 1.4, 1.6, 1.9, 2.1, 2.4, 2.6, 2.9, 3.1],
21}
22
23
24@pytest.mark.parametrize("risk,table", [(0.05, UDL_5), (0.10, UDL_10)])
25def test_upper_deviation_limits_match_the_aicpa_tables(risk, table):
26 for n, row in table.items():
27 for d, published in enumerate(row):
28 got = engine.upper_deviation_limit(n, d, risk)
29 # Rounded up, so never below the table; the table rounds a few cells to nearest.
30 assert published <= got <= round(published + 0.1, 1), (n, d, published, got)
31 assert engine.upper_deviation_limit(50, 1, 0.05) == 9.2
32 assert engine.upper_deviation_limit(100, 2, 0.05) == 6.2
33
34
35@pytest.mark.parametrize(
36 "risk,tolerable,expected,size",
37 [
38 (0.05, 0.05, 0.0, 59), (0.05, 0.05, 0.0025, 93), (0.05, 0.05, 0.01, 93), (0.05, 0.05, 0.0125, 124),
39 (0.05, 0.05, 0.0175, 153), (0.05, 0.05, 0.02, 181), (0.05, 0.10, 0.0, 29), (0.05, 0.06, 0.0, 49),
40 (0.05, 0.07, 0.0, 42), (0.05, 0.08, 0.0, 36), (0.05, 0.09, 0.0, 32), (0.05, 0.15, 0.0, 19),
41 (0.05, 0.20, 0.0, 14), (0.10, 0.05, 0.0, 45), (0.10, 0.10, 0.0, 22),
42 ],
43)
44def test_sample_sizes_match_the_aicpa_tables(risk, tolerable, expected, size):
45 assert engine.attribute_sample_size(risk, tolerable, expected)["sample_size"] == size
46
47
48def test_selections_are_fixed_for_a_seed():
49 # Pinned so any change to the algorithm is visible; v2 selections must never drift.
50 assert engine.select_random(100, 5, 20260707) == [56, 23, 65, 64, 59]
51 assert sorted(engine.select_random(10, 10, 1)) == list(range(10))
52 assert engine.select_systematic(100, 10, 7) == [8, 18, 28, 38, 48, 58, 68, 78, 88, 98]
53 chosen, summary = engine.select_stratified(["a", "b", "a", "b", "a"], {"b": 1, "a": 2}, 3)
54 assert chosen == [2, 0, 1] and [s["Stratum"] for s in summary] == ["a", "b"]
55 with pytest.raises(ValueError, match="requested 3"):
56 engine.select_stratified(["a"], {"a": 3}, 1)
57
58
59def test_evaluation():
60 res = engine.evaluate_attribute_sample(60, 0, 0.05, 0.05)
61 assert res["upper_deviation_limit_pct"] == 4.9 and res["supports_reliance"] is True
62 assert engine.evaluate_attribute_sample(60, 2, 0.05, 0.05)["supports_reliance"] is False