Commit 8c281c79e7
Unsigned
Layout: unified · split
CHANGELOG.md +7
| @@ -8,6 +8,13 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). | ||
| 8 | 8 | |
| 9 | 9 | ### Added |
| 10 | 10 | |
| 11 | - `sampling_tool.engine`: one standard-library selection algorithm (random, systematic, stratified) seeded from strings, so a seed selects the same rows with or without pandas and on every install. Sample manifests record `engine` and `engine_version` (2). `audit-sample --method systematic`. | |
| 12 | - Attribute sample sizes and upper deviation limits from the binomial distribution, matching the AICPA Audit Sampling guide tables for large populations; default sample sizes by control frequency. | |
| 13 | ||
| 14 | ### Changed | |
| 15 | ||
| 16 | - `random_sample` and `stratified_sample` select through the engine. The same seed now selects different rows than version 1 did; earlier runs remain verifiable from their saved outputs. | |
| 17 | ||
| 11 | 18 | - Installable package: `pip install audit-tools` installs `tui`, `applications`, |
| 12 | 19 | and `sampling` as packages, the `audit-sample` and `audit-tui` commands, and the |
| 13 | 20 | SQL/OS procedure library and sampling examples under `<prefix>/share/audit-tools/`. |
sampling/sampling_tool/cli.py +9 −5
| @@ -18,6 +18,7 @@ from .methods import ( | ||
| 18 | 18 | largest_remainder_allocation, |
| 19 | 19 | random_sample, |
| 20 | 20 | stratified_sample, |
| 21 | systematic_sample, | |
| 21 | 22 | ) |
| 22 | 23 | from .reconciliation import build_reconciliation, build_strata_summary |
| 23 | 24 | from .reporting import RunLogger, build_methodology |
| @@ -34,7 +35,7 @@ def build_parser() -> ArgumentParser: | ||
| 34 | 35 | parser.add_argument("--input") |
| 35 | 36 | parser.add_argument("--sheet") |
| 36 | 37 | parser.add_argument("--id-column") |
| 37 | parser.add_argument("--method", choices=["random", "stratified", "validate-only"]) | |
| 38 | parser.add_argument("--method", choices=["random", "systematic", "stratified", "validate-only"]) | |
| 38 | 39 | parser.add_argument("--sample-size", type=int) |
| 39 | 40 | parser.add_argument("--stratify-column") |
| 40 | 41 | parser.add_argument("--strata-counts") |
| @@ -106,9 +107,9 @@ def merge_options(args: Namespace, config: dict[str, object]) -> SimpleNamespace | ||
| 106 | 107 | raise AuditSamplingError("--input is required unless provided by --config.") |
| 107 | 108 | if merged["method"] is None: |
| 108 | 109 | raise AuditSamplingError("--method is required unless provided by --config.") |
| 109 | if merged["method"] not in {"random", "stratified", "validate-only"}: | |
| 110 | if merged["method"] not in {"random", "systematic", "stratified", "validate-only"}: | |
| 110 | 111 | raise AuditSamplingError( |
| 111 | "--method must be random, stratified, or validate-only." | |
| 112 | "--method must be random, systematic, stratified, or validate-only." | |
| 112 | 113 | ) |
| 113 | 114 | if merged["sample_size"] is not None: |
| 114 | 115 | merged["sample_size"] = int(merged["sample_size"]) |
| @@ -186,7 +187,7 @@ def run(options) -> Path: | ||
| 186 | 187 | print(f"WARNING: {warning}") |
| 187 | 188 | logger.warning(warning) |
| 188 | 189 | |
| 189 | if options.method in {"random", "stratified"}: | |
| 190 | if options.method in {"random", "systematic", "stratified"}: | |
| 190 | 191 | random_seed, generated = ensure_seed(options.seed) |
| 191 | 192 | if generated: |
| 192 | 193 | warning = f"No seed provided; generated seed {random_seed}." |
| @@ -196,6 +197,9 @@ def run(options) -> Path: | ||
| 196 | 197 | if options.method == "random": |
| 197 | 198 | sample = random_sample(validated, options.sample_size, random_seed) |
| 198 | 199 | sample = add_sample_metadata(sample, options, random_seed, timestamp, None) |
| 200 | elif options.method == "systematic": | |
| 201 | sample = systematic_sample(validated, options.sample_size, random_seed) | |
| 202 | sample = add_sample_metadata(sample, options, random_seed, timestamp, None) | |
| 199 | 203 | elif options.method == "stratified": |
| 200 | 204 | counts = validation.strata_counts |
| 201 | 205 | if validation.strata_proportions: |
| @@ -318,7 +322,7 @@ def _write_outputs( | ||
| 318 | 322 | ) -> None: |
| 319 | 323 | write_csv(validated, run_dir / _POPULATION_VALIDATED_CSV) |
| 320 | 324 | _track(output_files, _POPULATION_VALIDATED_CSV) |
| 321 | if options.method in {"random", "stratified"}: | |
| 325 | if options.method in {"random", "systematic", "stratified"}: | |
| 322 | 326 | write_csv(sample, run_dir / "sample.csv") |
| 323 | 327 | _track(output_files, "sample.csv") |
| 324 | 328 | if not excluded_rows.empty: |
sampling/sampling_tool/engine.py added +124
| @@ -0,0 +1,124 @@ | ||
| 1 | """The sampling engine: one deterministic, standard-library selection algorithm. | |
| 2 | ||
| 3 | The same seed selects the same items on every install, with or without | |
| 4 | pandas, in the `audit-sample` command and in audit-workbench. Each stream is | |
| 5 | seeded from a string (`audit-sample:<seed>` and, per stratum, | |
| 6 | `audit-sample:<seed>:<stratum>`), which Python's `random` hashes the same way | |
| 7 | on every platform. Selections are lists of 0-based row positions. | |
| 8 | ||
| 9 | Also here: attribute sample sizes and upper deviation limits from the | |
| 10 | binomial distribution (large-population tables, as in the AICPA Audit | |
| 11 | Sampling guide), and default control-frequency sample sizes. | |
| 12 | """ | |
| 13 | ||
| 14 | from __future__ import annotations | |
| 15 | ||
| 16 | import math | |
| 17 | import random | |
| 18 | from decimal import ROUND_CEILING, Decimal | |
| 19 | ||
| 20 | ENGINE = "audit-sample-engine" | |
| 21 | ENGINE_VERSION = "2" | |
| 22 | ||
| 23 | # Sample sizes by control frequency, for controls tested without statistical sizing. | |
| 24 | # Firms can override these (audit-workbench reads `sampling_presets` from engagement.yaml). | |
| 25 | FREQUENCY_PRESETS = { | |
| 26 | "annual": 1, | |
| 27 | "quarterly": 2, | |
| 28 | "monthly": 2, | |
| 29 | "weekly": 5, | |
| 30 | "daily": 20, | |
| 31 | "multiple-times-daily": 25, | |
| 32 | } | |
| 33 | ||
| 34 | ||
| 35 | def _rng(seed: int, stratum: str | None = None) -> random.Random: | |
| 36 | return random.Random(f"audit-sample:{seed}" + ("" if stratum is None else f":{stratum}")) | |
| 37 | ||
| 38 | ||
| 39 | def select_random(population_size: int, sample_size: int, seed: int) -> list[int]: | |
| 40 | """Positions of a simple random sample without replacement, in selection order.""" | |
| 41 | if not 0 < sample_size <= population_size: | |
| 42 | raise ValueError(f"sample size {sample_size} must be between 1 and the population size {population_size}") | |
| 43 | return _rng(seed).sample(range(population_size), sample_size) | |
| 44 | ||
| 45 | ||
| 46 | def select_systematic(population_size: int, sample_size: int, seed: int) -> list[int]: | |
| 47 | """Every k-th position (k = population / sample, fractional) from a random start.""" | |
| 48 | if not 0 < sample_size <= population_size: | |
| 49 | raise ValueError(f"sample size {sample_size} must be between 1 and the population size {population_size}") | |
| 50 | interval = population_size / sample_size | |
| 51 | start = _rng(seed).random() * interval | |
| 52 | return [int(start + i * interval) for i in range(sample_size)] | |
| 53 | ||
| 54 | ||
| 55 | def select_stratified( | |
| 56 | strata: list[str], counts: dict[str, int], seed: int, allow_shortfall: bool = False | |
| 57 | ) -> tuple[list[int], list[dict]]: | |
| 58 | """Positions sampled per stratum (`strata[i]` is row i's stratum), and a summary per stratum.""" | |
| 59 | chosen: list[int] = [] | |
| 60 | summary: list[dict] = [] | |
| 61 | for stratum in sorted(counts): | |
| 62 | requested = counts[stratum] | |
| 63 | pool = [i for i, s in enumerate(strata) if s == stratum] | |
| 64 | actual = min(requested, len(pool)) | |
| 65 | if actual < requested and not allow_shortfall: | |
| 66 | raise ValueError(f"Stratum '{stratum}' has {len(pool)} rows; requested {requested}.") | |
| 67 | chosen.extend(_rng(seed, stratum).sample(pool, actual) if actual else []) | |
| 68 | summary.append( | |
| 69 | { | |
| 70 | "Stratum": stratum, | |
| 71 | "Population Count": len(pool), | |
| 72 | "Requested Sample Count": requested, | |
| 73 | "Actual Sample Count": actual, | |
| 74 | "Shortfall": requested - actual, | |
| 75 | } | |
| 76 | ) | |
| 77 | return chosen, summary | |
| 78 | ||
| 79 | ||
| 80 | def _binomial_cdf(k: int, n: int, p: float) -> float: | |
| 81 | return sum(math.comb(n, i) * p**i * (1 - p) ** (n - i) for i in range(k + 1)) | |
| 82 | ||
| 83 | ||
| 84 | def attribute_sample_size(risk: float, tolerable_rate: float, expected_rate: float = 0.0) -> dict: | |
| 85 | """Smallest n whose upper deviation limit, with n × expected rate (rounded up) deviations, is within tolerance. | |
| 86 | ||
| 87 | `risk` is the risk of overreliance (0.05 for 95% confidence). Large-population | |
| 88 | (binomial) sizes, as in the AICPA tables. | |
| 89 | """ | |
| 90 | if not (0 < risk < 1 and 0 < tolerable_rate < 1 and 0 <= expected_rate < tolerable_rate): | |
| 91 | raise ValueError("need 0 < risk < 1, 0 < tolerable rate < 1, and 0 <= expected rate < tolerable rate") | |
| 92 | for n in range(1, 5001): | |
| 93 | k = math.ceil(n * expected_rate - 1e-12) | |
| 94 | if _binomial_cdf(k, n, tolerable_rate) <= risk: | |
| 95 | return {"sample_size": n, "expected_deviations": k} | |
| 96 | raise ValueError("no sample size up to 5000 meets these parameters") | |
| 97 | ||
| 98 | ||
| 99 | def upper_deviation_limit(sample_size: int, deviations: int, risk: float) -> float: | |
| 100 | """Upper deviation limit in percent, rounded up to one decimal (never understated).""" | |
| 101 | if not (0 <= deviations <= sample_size and sample_size > 0 and 0 < risk < 1): | |
| 102 | raise ValueError("need 0 <= deviations <= sample size and 0 < risk < 1") | |
| 103 | if deviations == sample_size: | |
| 104 | return 100.0 | |
| 105 | lo, hi = 0.0, 1.0 | |
| 106 | for _ in range(80): | |
| 107 | mid = (lo + hi) / 2 | |
| 108 | if _binomial_cdf(deviations, sample_size, mid) > risk: | |
| 109 | lo = mid | |
| 110 | else: | |
| 111 | hi = mid | |
| 112 | return float(Decimal(hi * 100).quantize(Decimal("0.1"), rounding=ROUND_CEILING)) | |
| 113 | ||
| 114 | ||
| 115 | def evaluate_attribute_sample(sample_size: int, deviations: int, risk: float, tolerable_rate: float) -> dict: | |
| 116 | udl = upper_deviation_limit(sample_size, deviations, risk) | |
| 117 | return { | |
| 118 | "sample_size": sample_size, | |
| 119 | "deviations": deviations, | |
| 120 | "sample_rate_pct": round(100 * deviations / sample_size, 2), | |
| 121 | "upper_deviation_limit_pct": udl, | |
| 122 | "tolerable_rate_pct": round(100 * tolerable_rate, 2), | |
| 123 | "supports_reliance": udl <= 100 * tolerable_rate, | |
| 124 | } | |
sampling/sampling_tool/manifest.py +3 −1
| @@ -5,7 +5,7 @@ from __future__ import annotations | ||
| 5 | 5 | import json |
| 6 | 6 | from pathlib import Path |
| 7 | 7 | |
| 8 | from . import __version__ | |
| 8 | from . import __version__, engine | |
| 9 | 9 | |
| 10 | 10 | |
| 11 | 11 | def build_manifest( |
| @@ -29,6 +29,8 @@ def build_manifest( | ||
| 29 | 29 | return { |
| 30 | 30 | "tool": "audit_sample", |
| 31 | 31 | "version": __version__, |
| 32 | "engine": engine.ENGINE, | |
| 33 | "engine_version": engine.ENGINE_VERSION, | |
| 32 | 34 | "run_timestamp_utc": run_timestamp_utc, |
| 33 | 35 | "input_file": str(input_file), |
| 34 | 36 | "input_sha256": input_sha256, |
sampling/sampling_tool/methods.py +14 −29
| @@ -7,6 +7,7 @@ import random | ||
| 7 | 7 | |
| 8 | 8 | import pandas as pd |
| 9 | 9 | |
| 10 | from . import engine | |
| 10 | 11 | from .io import AuditSamplingError |
| 11 | 12 | |
| 12 | 13 | |
| @@ -89,7 +90,13 @@ def largest_remainder_allocation( | ||
| 89 | 90 | def random_sample( |
| 90 | 91 | population: pd.DataFrame, sample_size: int, seed: int |
| 91 | 92 | ) -> pd.DataFrame: |
| 92 | return population.sample(n=sample_size, random_state=seed) | |
| 93 | return population.iloc[engine.select_random(len(population), sample_size, seed)] | |
| 94 | ||
| 95 | ||
| 96 | def systematic_sample( | |
| 97 | population: pd.DataFrame, sample_size: int, seed: int | |
| 98 | ) -> pd.DataFrame: | |
| 99 | return population.iloc[engine.select_systematic(len(population), sample_size, seed)] | |
| 93 | 100 | |
| 94 | 101 | |
| 95 | 102 | def stratified_sample( |
| @@ -99,31 +106,9 @@ def stratified_sample( | ||
| 99 | 106 | seed: int, |
| 100 | 107 | allow_shortfall: bool, |
| 101 | 108 | ) -> tuple[pd.DataFrame, list[dict[str, object]]]: |
| 102 | samples: list[pd.DataFrame] = [] | |
| 103 | summary: list[dict[str, object]] = [] | |
| 104 | for index, (stratum, requested) in enumerate(counts.items()): | |
| 105 | stratum_population = population[population[stratify_column] == stratum] | |
| 106 | actual_count = min(requested, len(stratum_population)) | |
| 107 | if actual_count < requested and not allow_shortfall: | |
| 108 | raise AuditSamplingError( | |
| 109 | f"Stratum '{stratum}' has {len(stratum_population)} rows; " | |
| 110 | f"requested {requested}. Use --allow-shortfall to continue." | |
| 111 | ) | |
| 112 | if actual_count: | |
| 113 | sampled = stratum_population.sample( | |
| 114 | n=actual_count, random_state=seed + index | |
| 115 | ) | |
| 116 | samples.append(sampled) | |
| 117 | summary.append( | |
| 118 | { | |
| 119 | "Stratum": stratum, | |
| 120 | "Population Count": len(stratum_population), | |
| 121 | "Requested Sample Count": requested, | |
| 122 | "Actual Sample Count": actual_count, | |
| 123 | "Shortfall": requested - actual_count, | |
| 124 | } | |
| 125 | ) | |
| 126 | ||
| 127 | if samples: | |
| 128 | return pd.concat(samples), summary | |
| 129 | return population.iloc[0:0].copy(), summary | |
| 109 | strata = population[stratify_column].astype(str).tolist() | |
| 110 | try: | |
| 111 | positions, summary = engine.select_stratified(strata, counts, seed, allow_shortfall) | |
| 112 | except ValueError as exc: | |
| 113 | raise AuditSamplingError(f"{exc} Use --allow-shortfall to continue.") from exc | |
| 114 | return population.iloc[positions], summary | |
sampling/sampling_tool/validation.py +1 −1
| @@ -91,7 +91,7 @@ def validate_and_prepare(population: pd.DataFrame, options) -> ValidationResult: | ||
| 91 | 91 | |
| 92 | 92 | if options.sample_size is not None and options.sample_size <= 0: |
| 93 | 93 | raise AuditSamplingError("--sample-size must be a positive integer.") |
| 94 | needs_sample_size = options.method == "random" or ( | |
| 94 | needs_sample_size = options.method in ("random", "systematic") or ( | |
| 95 | 95 | options.method == "stratified" and bool(options.strata_proportions) |
| 96 | 96 | ) |
| 97 | 97 | if needs_sample_size: |
sampling/tests/test_engine.py added +62
| @@ -0,0 +1,62 @@ | ||
| 1 | """The stdlib sampling engine: stable selections, AICPA sample sizes and upper deviation limits.""" | |
| 2 | ||
| 3 | import pytest | |
| 4 | ||
| 5 | from sampling.sampling_tool import engine | |
| 6 | ||
| 7 | # AICPA Audit Sampling guide, sample evaluation tables (large population), upper deviation limit in %. | |
| 8 | UDL_5 = { | |
| 9 | 20: [14.0, 21.7, 28.3, 34.4, 40.2, 45.6, 50.8, 55.9, 60.7, 65.4, 69.9], | |
| 10 | 25: [11.3, 17.7, 23.2, 28.2, 33.0, 37.6, 42.0, 46.3, 50.4, 54.4, 58.4], | |
| 11 | 50: [5.9, 9.2, 12.1, 14.8, 17.4, 19.9, 22.4, 24.7, 27.1, 29.4, 31.6], | |
| 12 | 100: [3.0, 4.7, 6.2, 7.6, 9.0, 10.3, 11.5, 12.8, 14.0, 15.2, 16.4], | |
| 13 | 200: [1.5, 2.4, 3.2, 3.9, 4.6, 5.2, 5.8, 6.5, 7.2, 7.8, 8.4], | |
| 14 | 500: [0.6, 1.0, 1.3, 1.6, 1.9, 2.1, 2.4, 2.7, 2.9, 3.2, 3.4], | |
| 15 | } | |
| 16 | UDL_10 = { | |
| 17 | 20: [10.9, 18.1, 24.5, 30.5, 36.1, 41.5, 46.8, 51.9, 56.8, 61.6, 66.2], | |
| 18 | 50: [4.6, 7.6, 10.3, 12.9, 15.4, 17.8, 20.2, 22.5, 24.7, 27.0, 29.2], | |
| 19 | 100: [2.3, 3.9, 5.3, 6.6, 7.9, 9.1, 10.3, 11.5, 12.7, 13.9, 15.0], | |
| 20 | 500: [0.5, 0.8, 1.1, 1.4, 1.6, 1.9, 2.1, 2.4, 2.6, 2.9, 3.1], | |
| 21 | } | |
| 22 | ||
| 23 | ||
| 24 | @pytest.mark.parametrize("risk,table", [(0.05, UDL_5), (0.10, UDL_10)]) | |
| 25 | def test_upper_deviation_limits_match_the_aicpa_tables(risk, table): | |
| 26 | for n, row in table.items(): | |
| 27 | for d, published in enumerate(row): | |
| 28 | got = engine.upper_deviation_limit(n, d, risk) | |
| 29 | # Rounded up, so never below the table; the table rounds a few cells to nearest. | |
| 30 | assert published <= got <= round(published + 0.1, 1), (n, d, published, got) | |
| 31 | assert engine.upper_deviation_limit(50, 1, 0.05) == 9.2 | |
| 32 | assert engine.upper_deviation_limit(100, 2, 0.05) == 6.2 | |
| 33 | ||
| 34 | ||
| 35 | @pytest.mark.parametrize( | |
| 36 | "risk,tolerable,expected,size", | |
| 37 | [ | |
| 38 | (0.05, 0.05, 0.0, 59), (0.05, 0.05, 0.0025, 93), (0.05, 0.05, 0.01, 93), (0.05, 0.05, 0.0125, 124), | |
| 39 | (0.05, 0.05, 0.0175, 153), (0.05, 0.05, 0.02, 181), (0.05, 0.10, 0.0, 29), (0.05, 0.06, 0.0, 49), | |
| 40 | (0.05, 0.07, 0.0, 42), (0.05, 0.08, 0.0, 36), (0.05, 0.09, 0.0, 32), (0.05, 0.15, 0.0, 19), | |
| 41 | (0.05, 0.20, 0.0, 14), (0.10, 0.05, 0.0, 45), (0.10, 0.10, 0.0, 22), | |
| 42 | ], | |
| 43 | ) | |
| 44 | def test_sample_sizes_match_the_aicpa_tables(risk, tolerable, expected, size): | |
| 45 | assert engine.attribute_sample_size(risk, tolerable, expected)["sample_size"] == size | |
| 46 | ||
| 47 | ||
| 48 | def test_selections_are_fixed_for_a_seed(): | |
| 49 | # Pinned so any change to the algorithm is visible; v2 selections must never drift. | |
| 50 | assert engine.select_random(100, 5, 20260707) == [56, 23, 65, 64, 59] | |
| 51 | assert sorted(engine.select_random(10, 10, 1)) == list(range(10)) | |
| 52 | assert engine.select_systematic(100, 10, 7) == [8, 18, 28, 38, 48, 58, 68, 78, 88, 98] | |
| 53 | chosen, summary = engine.select_stratified(["a", "b", "a", "b", "a"], {"b": 1, "a": 2}, 3) | |
| 54 | assert chosen == [2, 0, 1] and [s["Stratum"] for s in summary] == ["a", "b"] | |
| 55 | with pytest.raises(ValueError, match="requested 3"): | |
| 56 | engine.select_stratified(["a"], {"a": 3}, 1) | |
| 57 | ||
| 58 | ||
| 59 | def test_evaluation(): | |
| 60 | res = engine.evaluate_attribute_sample(60, 0, 0.05, 0.05) | |
| 61 | assert res["upper_deviation_limit_pct"] == 4.9 and res["supports_reliance"] is True | |
| 62 | assert engine.evaluate_attribute_sample(60, 2, 0.05, 0.05)["supports_reliance"] is False | |