audit-labs/audit-report

  1"""Tests for rule evaluation against fixture packages, using bundled rulesets."""
  2
  3from pathlib import Path
  4
  5from audit_report.engine import (
  6    FAIL,
  7    NOT_APPLICABLE,
  8    PASS,
  9    control_coverage,
 10    evaluate,
 11    summarize,
 12)
 13from audit_report.loader import load_package
 14from audit_report.rules import load_ruleset
 15
 16FIXTURES = Path(__file__).parent / "fixtures"
 17RULESETS = Path("audit_report/rulesets")
 18
 19
 20def _run(pkg_name, ruleset_name):
 21    pkg = load_package(FIXTURES / pkg_name)
 22    ruleset = load_ruleset(RULESETS / ruleset_name)
 23    findings = evaluate(pkg, ruleset)
 24    return {f.rule.id: f for f in findings}
 25
 26
 27def test_aws_fixture_findings():
 28    findings = _run("aws_audit_acme_2026-01-01", "aws.yaml")
 29
 30    # bob has a console password and no MFA -> one failing row.
 31    mfa = findings["aws.iam.console-mfa"]
 32    assert mfa.status == FAIL
 33    assert len(mfa.evidence) == 1
 34    assert mfa.evidence[0]["user"] == "bob"
 35
 36    # bob's key is 400 days old.
 37    assert findings["aws.iam.key-rotation"].status == FAIL
 38
 39    # Root is healthy in the fixture.
 40    assert findings["aws.root.mfa"].status == PASS
 41    assert findings["aws.root.no-access-keys"].status == PASS
 42
 43    # min length 8 < 14 -> policy fails.
 44    assert findings["aws.iam.password-policy"].status == FAIL
 45
 46    # Only the port-22 group is open to the world.
 47    ssh = findings["aws.network.no-open-ssh"]
 48    assert ssh.status == FAIL
 49    assert len(ssh.evidence) == 1
 50    assert ssh.evidence[0]["group_name"] == "web"
 51
 52    # Empty s3_public_access table means no public buckets -> pass.
 53    assert findings["aws.s3.no-public-access"].status == PASS
 54
 55    # A healthy multi-region trail exists.
 56    assert findings["aws.cloudtrail.logging"].status == PASS
 57
 58
 59def test_github_fixture_findings():
 60    findings = _run("github_audit_acme_2026-01-01", "github.yaml")
 61    assert findings["github.org.require-2fa"].status == PASS
 62    assert findings["github.org.default-permission"].status == PASS
 63    assert findings["github.org.secret-scanning"].status == PASS
 64    # 'site' repo default branch is unprotected.
 65    branch = findings["github.branch.default-protected"]
 66    assert branch.status == FAIL
 67    assert branch.evidence[0]["repo"] == "site"
 68    # 'app' has 1 review, 'site' is unprotected -> require-reviews passes
 69    # (only counts protected branches with < 1 review).
 70    assert findings["github.branch.require-reviews"].status == PASS
 71
 72
 73def test_gitlab_fixture_findings():
 74    findings = _run("gitlab_audit_acme_2026-01-01", "gitlab.yaml")
 75
 76    # 'site' allows force push on a protected branch.
 77    force = findings["gitlab.branch.no-force-push"]
 78    assert force.status == FAIL
 79    assert force.evidence[0]["project"] == "acme/site"
 80
 81    # 'site' does not require code owner approval.
 82    assert findings["gitlab.branch.code-owner-approval"].status == FAIL
 83
 84    # The License-Check rule requires zero approvals.
 85    approvals = findings["gitlab.approvals.require-one"]
 86    assert approvals.status == FAIL
 87    assert approvals.evidence[0]["rule"] == "License-Check"
 88
 89    # 'site' is a public project.
 90    assert findings["gitlab.projects.no-public"].status == FAIL
 91
 92    # No password_policy.csv in the package (as on GitLab.com) -> not applicable.
 93    assert findings["gitlab.password-policy"].status == NOT_APPLICABLE
 94
 95    # An audit event with an action is present -> logging is evidenced.
 96    assert findings["gitlab.audit.logging-active"].status == PASS
 97
 98
 99def test_not_applicable_when_table_absent(tmp_path):
100    (tmp_path / "aws_audit_x_2026-01-01").mkdir()
101    pkg_dir = tmp_path / "aws_audit_x_2026-01-01"
102    (pkg_dir / "iam_users.csv").write_text(
103        "user,mfa_enabled,console_password,oldest_key_age_days\n", encoding="utf-8"
104    )
105    pkg = load_package(pkg_dir)
106    findings = {f.rule.id: f for f in evaluate(pkg, load_ruleset(RULESETS / "aws.yaml"))}
107    # Tables that were never collected are reported as not applicable.
108    assert findings["aws.root.mfa"].status == NOT_APPLICABLE
109
110
111def test_summary_and_coverage():
112    pkg = load_package(FIXTURES / "aws_audit_acme_2026-01-01")
113    findings = evaluate(pkg, load_ruleset(RULESETS / "aws.yaml"))
114    counts = summarize(findings)
115    assert counts[FAIL] + counts[PASS] + counts[NOT_APPLICABLE] == len(findings)
116
117    coverage = control_coverage(findings)
118    # SC-7 is cited by the open-SSH rule (fails), so the control rolls up to fail.
119    assert coverage["NIST:SC-7"]["status"] == FAIL
120    # A control cited only by passing rules (both root checks) rolls up to pass.
121    assert coverage["ISO:A.8.2"]["status"] == PASS