audit-labs/evidence-seal

  1#!/usr/bin/env bash
  2#
  3# End-to-end demonstration of every evidence-seal feature, run against the CLI.
  4#
  5# Exercises seal, verify, chain, keygen, sign, and RFC 3161 timestamping —
  6# including the adversarial cases that must fail — and prints a pass/fail tally.
  7# Exits non-zero if any check does not behave as expected.
  8#
  9# The timestamp tokens are minted by a local self-signed TSA (scripts/_local_tsa.py)
 10# so the full flow, including signature verification, runs offline. Needs the
 11# optional extras: pip install -e ".[sign,timestamp]"
 12#
 13# Usage: scripts/e2e.sh
 14set -u
 15
 16SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 17REPO="$(dirname "$SCRIPT_DIR")"
 18
 19# Prefer the project venv; fall back to whatever is on PATH.
 20if [[ -x "$REPO/.venv/bin/python" ]]; then
 21  PY="$REPO/.venv/bin/python"
 22else
 23  PY="$(command -v python3 || command -v python)"
 24fi
 25ES="$PY -m evidence_seal"
 26TSA="$PY $SCRIPT_DIR/_local_tsa.py"
 27
 28W="$(mktemp -d)"
 29trap 'rm -rf "$W"' EXIT
 30
 31pass=0; fail=0
 32step() { printf '\n\033[1m━━━ %s\033[0m\n' "$*"; }
 33run()  { printf '$ %s\n' "$*"; eval "$*"; }
 34# assert_exit <expected-code> <label> <command...>
 35assert_exit() {
 36  local exp="$1" label="$2"; shift 2
 37  printf '$ %s\n' "$*"
 38  eval "$*" >"$W/out" 2>&1; local rc=$?
 39  sed 's/^/   /' "$W/out"
 40  if [[ "$rc" == "$exp" ]]; then
 41    printf '   \033[32m✓ PASS\033[0m — %s (exit %s)\n' "$label" "$rc"; pass=$((pass+1))
 42  else
 43    printf '   \033[31m✗ FAIL\033[0m — %s (exit %s, expected %s)\n' "$label" "$rc" "$exp"; fail=$((fail+1))
 44  fi
 45}
 46
 47step "0. Version + command surface"
 48run "$ES --version"
 49
 50step "1. SEAL a realistic evidence package (with provenance metadata)"
 51mkdir -p "$W/pkg/logs"
 52printf 'user,mfa_enabled\nalice,true\nbob,false\n' > "$W/pkg/iam_users.csv"
 53printf 'repo,branch,protected\napp,main,true\n' > "$W/pkg/branch_protections.csv"
 54printf 'GitHub Audit — acme\n' > "$W/pkg/summary.txt"
 55printf 'scratch\n' > "$W/pkg/logs/debug.tmp"
 56assert_exit 0 "seal writes a manifest" \
 57  "$ES seal $W/pkg --out $W/m.json --ignore 'logs/*.tmp' --meta engagement=ACME-2026 --meta collector='Christian Cleberg'"
 58
 59step "2. VERIFY — intact package"
 60assert_exit 0 "verify clean" "$ES verify $W/pkg --manifest $W/m.json"
 61
 62step "3. TAMPER — modify a sealed file"
 63printf 'alice,true\nbob,flipped\n' >> "$W/pkg/iam_users.csv"
 64assert_exit 1 "verify catches modification" "$ES verify $W/pkg --manifest $W/m.json"
 65printf 'user,mfa_enabled\nalice,true\nbob,false\n' > "$W/pkg/iam_users.csv"
 66
 67step "4. TAMPER — added + removed files"
 68printf 'oops\n' > "$W/pkg/rogue.csv"; rm "$W/pkg/summary.txt"
 69assert_exit 1 "verify catches add+remove" "$ES verify $W/pkg --manifest $W/m.json"
 70rm "$W/pkg/rogue.csv"; printf 'GitHub Audit — acme\n' > "$W/pkg/summary.txt"
 71
 72step "5. IGNORE globs — excluded file never flags"
 73assert_exit 0 "ignored file does not break verify" "$ES verify $W/pkg --manifest $W/m.json"
 74
 75step "6. MANIFEST TAMPER — edit a recorded hash in the manifest itself"
 76"$PY" - "$W/m.json" "$W/m_bad.json" <<'PY'
 77import json, sys
 78m = json.load(open(sys.argv[1])); m["files"][0]["sha256"] = "0" * 64
 79json.dump(m, open(sys.argv[2], "w"), indent=2, sort_keys=True)
 80PY
 81assert_exit 1 "verify catches a doctored manifest" "$ES verify $W/pkg --manifest $W/m_bad.json"
 82
 83step "7. CHAIN OF CUSTODY — three sequential seals link"
 84mkdir -p "$W/chain"
 85printf 'v1\n' > "$W/pkg/state.csv"; $ES seal "$W/pkg" --out "$W/chain/jan.json" >/dev/null 2>&1
 86printf 'v2\n' > "$W/pkg/state.csv"; $ES seal "$W/pkg" --out "$W/chain/feb.json" --prev "$W/chain/jan.json" >/dev/null 2>&1
 87printf 'v3\n' > "$W/pkg/state.csv"; $ES seal "$W/pkg" --out "$W/chain/mar.json" --prev "$W/chain/feb.json" >/dev/null 2>&1
 88rm "$W/pkg/state.csv"
 89assert_exit 0 "chain in order is intact" "$ES chain $W/chain/jan.json $W/chain/feb.json $W/chain/mar.json"
 90
 91step "8. CHAIN — reordered links are detected"
 92assert_exit 1 "reordered chain is broken" "$ES chain $W/chain/mar.json $W/chain/jan.json $W/chain/feb.json"
 93
 94step "9. CHAIN — a spliced-out link is detected"
 95assert_exit 1 "missing middle seal is broken" "$ES chain $W/chain/jan.json $W/chain/mar.json"
 96
 97step "10. SIGNING — generate an ed25519 keypair"
 98assert_exit 0 "keygen" "$ES keygen --private $W/acme.key --public $W/acme.pub"
 99
100step "11. SIGN and VERIFY with the matching public key"
101assert_exit 0 "seal + sign" "$ES seal $W/pkg --out $W/signed.json --sign $W/acme.key"
102assert_exit 0 "verify requires correct signer" "$ES verify $W/pkg --manifest $W/signed.json --pubkey $W/acme.pub"
103
104step "12. SIGN — a wrong public key is rejected"
105$ES keygen --private "$W/other.key" --public "$W/other.pub" >/dev/null 2>&1
106assert_exit 1 "wrong signer rejected" "$ES verify $W/pkg --manifest $W/signed.json --pubkey $W/other.pub"
107
108step "13. TIMESTAMP — build an RFC 3161 request (.tsq)"
109assert_exit 0 "timestamp request" "$ES timestamp request $W/signed.json --out $W/req.tsq"
110
111step "14. TIMESTAMP — apply a TSA token and verify the binding"
112MID=$("$PY" -c "import json;print(json.load(open('$W/signed.json'))['id'])")
113$TSA "$MID" "$W/tsa_ok" >/dev/null 2>&1
114assert_exit 0 "apply token" "$ES timestamp apply $W/signed.json --token $W/tsa_ok/resp.tsr --out $W/stamped.json"
115assert_exit 0 "verify timestamp binding" "$ES timestamp verify $W/stamped.json"
116
117step "15. TIMESTAMP — full CMS signature verification with --tsa-cert"
118assert_exit 0 "full TSA signature verify" "$ES timestamp verify $W/stamped.json --tsa-cert $W/tsa_ok/tsa.pem"
119
120step "16. TIMESTAMP — wrong TSA certificate is rejected"
121$TSA "$MID" "$W/tsa_wrong" >/dev/null 2>&1
122assert_exit 1 "wrong TSA cert rejected" "$ES timestamp verify $W/stamped.json --tsa-cert $W/tsa_wrong/tsa.pem"
123
124step "17. TIMESTAMP — cert lacking the timeStamping EKU is rejected"
125$TSA "$MID" "$W/tsa_noeku" --no-eku >/dev/null 2>&1
126$ES timestamp apply "$W/signed.json" --token "$W/tsa_noeku/resp.tsr" --out "$W/stamped_noeku.json" >/dev/null 2>&1
127assert_exit 1 "cert without timeStamping EKU rejected" "$ES timestamp verify $W/stamped_noeku.json --tsa-cert $W/tsa_noeku/tsa.pem"
128
129step "18. COMPOSITION — sealed + signed + timestamped, verified together"
130$ES seal "$W/pkg" --out "$W/full.json" --sign "$W/acme.key" --meta engagement=ACME-2026 >/dev/null 2>&1
131FID=$("$PY" -c "import json;print(json.load(open('$W/full.json'))['id'])")
132$TSA "$FID" "$W/tsa_full" >/dev/null 2>&1
133$ES timestamp apply "$W/full.json" --token "$W/tsa_full/resp.tsr" >/dev/null 2>&1
134assert_exit 0 "integrity + signature + timestamp all verify at once" \
135  "$ES verify $W/pkg --manifest $W/full.json --pubkey $W/acme.pub --tsa-cert $W/tsa_full/tsa.pem"
136
137echo
138printf '\033[1m════════════════════════════════════════════\033[0m\n'
139printf '\033[1m  E2E RESULT: %s passed, %s failed\033[0m\n' "$pass" "$fail"
140printf '\033[1m════════════════════════════════════════════\033[0m\n'
141exit $fail