Commit `cf5fb77d29`

cf5fb77d2995c7f389b762c06e1a2564f48f52d6

parent: fbe8823be3

Verified · cmc

cmc <hello@cleberg.net> · 2026-08-06 23:26 UTC

test: add end-to-end feature script

scripts/e2e.sh exercises every CLI feature against the installed package —
seal, verify, chain, keygen, sign, and RFC 3161 timestamping, plus the
adversarial cases that must fail — and exits non-zero if anything misbehaves.
scripts/_local_tsa.py mints CMS-signed tokens locally so the timestamp flow,
including signature verification, runs offline. Runs from a self-cleaning temp
dir; 20/20 checks pass.

Layout: unified · split

README.md +1

		@@ -176,6 +176,7 @@ Two limits to be honest about:
176	176	pip install -e ".[dev]"
177	177	pytest
178	178	ruff check .
	179	./scripts/e2e.sh # every feature end-to-end through the CLI, offline
179	180	```
180	181
181	182	## License

scripts/_local_tsa.py added +39

		@@ -0,0 +1,39 @@
	1	"""A local RFC 3161 Time-Stamp Authority for the end-to-end demo.
	2
	3	Mints genuinely CMS-signed timestamp tokens offline so scripts/e2e.sh can
	4	exercise the full timestamp flow — including signature verification — without a
	5	network or a public TSA. It reuses the token issuer from the test suite.
	6
	7	Usage: python scripts/_local_tsa.py <manifest_id> <out_dir> [--no-eku] [--expired]
	8	Writes <out_dir>/resp.tsr (a TimeStampResp) and <out_dir>/tsa.pem (the cert).
	9	"""
	10
	11	import datetime
	12	import pathlib
	13	import sys
	14
	15	_HERE = pathlib.Path(__file__).resolve()
	16	_REPO = _HERE.parent.parent
	17	sys.path.insert(0, str(_REPO / "tests"))
	18	sys.path.insert(0, str(_REPO))
	19
	20	from test_timestamp import issue_signed_token # noqa: E402
	21
	22
	23	def main() -> None:
	24	manifest_id, out_dir = sys.argv[1], pathlib.Path(sys.argv[2])
	25	out_dir.mkdir(parents=True, exist_ok=True)
	26	eku = "--no-eku" not in sys.argv
	27	validity = None
	28	if "--expired" in sys.argv:
	29	validity = (
	30	datetime.datetime(2020, 1, 1, tzinfo=datetime.timezone.utc),
	31	datetime.datetime(2021, 1, 1, tzinfo=datetime.timezone.utc),
	32	)
	33	token, cert = issue_signed_token(manifest_id, timestamping_eku=eku, validity=validity)
	34	(out_dir / "resp.tsr").write_bytes(token)
	35	(out_dir / "tsa.pem").write_bytes(cert)
	36
	37
	38	if __name__ == "__main__":
	39	main()

scripts/e2e.sh added +141

		@@ -0,0 +1,141 @@
	1	#!/usr/bin/env bash
	2	#
	3	# End-to-end demonstration of every evidence-seal feature, run against the CLI.
	4	#
	5	# Exercises seal, verify, chain, keygen, sign, and RFC 3161 timestamping —
	6	# including the adversarial cases that must fail — and prints a pass/fail tally.
	7	# Exits non-zero if any check does not behave as expected.
	8	#
	9	# The timestamp tokens are minted by a local self-signed TSA (scripts/_local_tsa.py)
	10	# so the full flow, including signature verification, runs offline. Needs the
	11	# optional extras: pip install -e ".[sign,timestamp]"
	12	#
	13	# Usage: scripts/e2e.sh
	14	set -u
	15
	16	SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
	17	REPO="$(dirname "$SCRIPT_DIR")"
	18
	19	# Prefer the project venv; fall back to whatever is on PATH.
	20	if [ -x "$REPO/.venv/bin/python" ]; then
	21	PY="$REPO/.venv/bin/python"
	22	else
	23	PY="$(command -v python3 \|\| command -v python)"
	24	fi
	25	ES="$PY -m evidence_seal"
	26	TSA="$PY $SCRIPT_DIR/_local_tsa.py"
	27
	28	W="$(mktemp -d)"
	29	trap 'rm -rf "$W"' EXIT
	30
	31	pass=0; fail=0
	32	step() { printf '\n\033[1m━━━ %s\033[0m\n' "$*"; }
	33	run() { printf '$ %s\n' "$"; eval "$"; }
	34	# assert_exit <expected-code> <label> <command...>
	35	assert_exit() {
	36	local exp="$1" label="$2"; shift 2
	37	printf '$ %s\n' "$*"
	38	eval "$*" >"$W/out" 2>&1; local rc=$?
	39	sed 's/^/ /' "$W/out"
	40	if [ "$rc" = "$exp" ]; then
	41	printf ' \033[32m✓ PASS\033[0m — %s (exit %s)\n' "$label" "$rc"; pass=$((pass+1))
	42	else
	43	printf ' \033[31m✗ FAIL\033[0m — %s (exit %s, expected %s)\n' "$label" "$rc" "$exp"; fail=$((fail+1))
	44	fi
	45	}
	46
	47	step "0. Version + command surface"
	48	run "$ES --version"
	49
	50	step "1. SEAL a realistic evidence package (with provenance metadata)"
	51	mkdir -p "$W/pkg/logs"
	52	printf 'user,mfa_enabled\nalice,true\nbob,false\n' > "$W/pkg/iam_users.csv"
	53	printf 'repo,branch,protected\napp,main,true\n' > "$W/pkg/branch_protections.csv"
	54	printf 'GitHub Audit — acme\n' > "$W/pkg/summary.txt"
	55	printf 'scratch\n' > "$W/pkg/logs/debug.tmp"
	56	assert_exit 0 "seal writes a manifest" \
	57	"$ES seal $W/pkg --out $W/m.json --ignore 'logs/*.tmp' --meta engagement=ACME-2026 --meta collector='Christian Cleberg'"
	58
	59	step "2. VERIFY — intact package"
	60	assert_exit 0 "verify clean" "$ES verify $W/pkg --manifest $W/m.json"
	61
	62	step "3. TAMPER — modify a sealed file"
	63	printf 'alice,true\nbob,flipped\n' >> "$W/pkg/iam_users.csv"
	64	assert_exit 1 "verify catches modification" "$ES verify $W/pkg --manifest $W/m.json"
	65	printf 'user,mfa_enabled\nalice,true\nbob,false\n' > "$W/pkg/iam_users.csv"
	66
	67	step "4. TAMPER — added + removed files"
	68	printf 'oops\n' > "$W/pkg/rogue.csv"; rm "$W/pkg/summary.txt"
	69	assert_exit 1 "verify catches add+remove" "$ES verify $W/pkg --manifest $W/m.json"
	70	rm "$W/pkg/rogue.csv"; printf 'GitHub Audit — acme\n' > "$W/pkg/summary.txt"
	71
	72	step "5. IGNORE globs — excluded file never flags"
	73	assert_exit 0 "ignored file does not break verify" "$ES verify $W/pkg --manifest $W/m.json"
	74
	75	step "6. MANIFEST TAMPER — edit a recorded hash in the manifest itself"
	76	"$PY" - "$W/m.json" "$W/m_bad.json" <<'PY'
	77	import json, sys
	78	m = json.load(open(sys.argv[1])); m["files"][0]["sha256"] = "0" * 64
	79	json.dump(m, open(sys.argv[2], "w"), indent=2, sort_keys=True)
	80	PY
	81	assert_exit 1 "verify catches a doctored manifest" "$ES verify $W/pkg --manifest $W/m_bad.json"
	82
	83	step "7. CHAIN OF CUSTODY — three sequential seals link"
	84	mkdir -p "$W/chain"
	85	printf 'v1\n' > "$W/pkg/state.csv"; $ES seal "$W/pkg" --out "$W/chain/jan.json" >/dev/null 2>&1
	86	printf 'v2\n' > "$W/pkg/state.csv"; $ES seal "$W/pkg" --out "$W/chain/feb.json" --prev "$W/chain/jan.json" >/dev/null 2>&1
	87	printf 'v3\n' > "$W/pkg/state.csv"; $ES seal "$W/pkg" --out "$W/chain/mar.json" --prev "$W/chain/feb.json" >/dev/null 2>&1
	88	rm "$W/pkg/state.csv"
	89	assert_exit 0 "chain in order is intact" "$ES chain $W/chain/jan.json $W/chain/feb.json $W/chain/mar.json"
	90
	91	step "8. CHAIN — reordered links are detected"
	92	assert_exit 1 "reordered chain is broken" "$ES chain $W/chain/mar.json $W/chain/jan.json $W/chain/feb.json"
	93
	94	step "9. CHAIN — a spliced-out link is detected"
	95	assert_exit 1 "missing middle seal is broken" "$ES chain $W/chain/jan.json $W/chain/mar.json"
	96
	97	step "10. SIGNING — generate an ed25519 keypair"
	98	assert_exit 0 "keygen" "$ES keygen --private $W/acme.key --public $W/acme.pub"
	99
	100	step "11. SIGN and VERIFY with the matching public key"
	101	assert_exit 0 "seal + sign" "$ES seal $W/pkg --out $W/signed.json --sign $W/acme.key"
	102	assert_exit 0 "verify requires correct signer" "$ES verify $W/pkg --manifest $W/signed.json --pubkey $W/acme.pub"
	103
	104	step "12. SIGN — a wrong public key is rejected"
	105	$ES keygen --private "$W/other.key" --public "$W/other.pub" >/dev/null 2>&1
	106	assert_exit 1 "wrong signer rejected" "$ES verify $W/pkg --manifest $W/signed.json --pubkey $W/other.pub"
	107
	108	step "13. TIMESTAMP — build an RFC 3161 request (.tsq)"
	109	assert_exit 0 "timestamp request" "$ES timestamp request $W/signed.json --out $W/req.tsq"
	110
	111	step "14. TIMESTAMP — apply a TSA token and verify the binding"
	112	MID=$("$PY" -c "import json;print(json.load(open('$W/signed.json'))['id'])")
	113	$TSA "$MID" "$W/tsa_ok" >/dev/null 2>&1
	114	assert_exit 0 "apply token" "$ES timestamp apply $W/signed.json --token $W/tsa_ok/resp.tsr --out $W/stamped.json"
	115	assert_exit 0 "verify timestamp binding" "$ES timestamp verify $W/stamped.json"
	116
	117	step "15. TIMESTAMP — full CMS signature verification with --tsa-cert"
	118	assert_exit 0 "full TSA signature verify" "$ES timestamp verify $W/stamped.json --tsa-cert $W/tsa_ok/tsa.pem"
	119
	120	step "16. TIMESTAMP — wrong TSA certificate is rejected"
	121	$TSA "$MID" "$W/tsa_wrong" >/dev/null 2>&1
	122	assert_exit 1 "wrong TSA cert rejected" "$ES timestamp verify $W/stamped.json --tsa-cert $W/tsa_wrong/tsa.pem"
	123
	124	step "17. TIMESTAMP — cert lacking the timeStamping EKU is rejected"
	125	$TSA "$MID" "$W/tsa_noeku" --no-eku >/dev/null 2>&1
	126	$ES timestamp apply "$W/signed.json" --token "$W/tsa_noeku/resp.tsr" --out "$W/stamped_noeku.json" >/dev/null 2>&1
	127	assert_exit 1 "cert without timeStamping EKU rejected" "$ES timestamp verify $W/stamped_noeku.json --tsa-cert $W/tsa_noeku/tsa.pem"
	128
	129	step "18. COMPOSITION — sealed + signed + timestamped, verified together"
	130	$ES seal "$W/pkg" --out "$W/full.json" --sign "$W/acme.key" --meta engagement=ACME-2026 >/dev/null 2>&1
	131	FID=$("$PY" -c "import json;print(json.load(open('$W/full.json'))['id'])")
	132	$TSA "$FID" "$W/tsa_full" >/dev/null 2>&1
	133	$ES timestamp apply "$W/full.json" --token "$W/tsa_full/resp.tsr" >/dev/null 2>&1
	134	assert_exit 0 "integrity + signature + timestamp all verify at once" \
	135	"$ES verify $W/pkg --manifest $W/full.json --pubkey $W/acme.pub --tsa-cert $W/tsa_full/tsa.pem"
	136
	137	echo
	138	printf '\033[1m════════════════════════════════════════════\033[0m\n'
	139	printf '\033[1m E2E RESULT: %s passed, %s failed\033[0m\n' "$pass" "$fail"
	140	printf '\033[1m════════════════════════════════════════════\033[0m\n'
	141	exit $fail

Commit cf5fb77d29

Commit `cf5fb77d29`