Commit `ea57052e99`

ea57052e998007dc914257f84f383c63f945cf08

parent: 38723030eb

Unsigned

cmc <hello@cleberg.net> · 2026-07-20 22:38 UTC

fix(a11y): survive audit timeouts, and stop overclaiming CI floor coverage

Two problems the first CI run exposed.

Audit timeouts. Three tests failed with "Audit failed to complete in
time" (code -56) on the GitHub runner. That is the audit's own internal
deadline on a slower machine, not an app defect, and the harness had no
resilience to it. Audits now retry up to three times, and a screen that
still cannot be audited is reported via XCTSkip rather than passing.
Skips are distinct from passes in CI, so an unaudited screen stays
visible instead of being silently counted as clean. The Dynamic Type
sweep attempts every screen before skipping, so one slow screen cannot
drop the other four.

Overclaimed floor coverage. The two-simulator matrix was justified on
covering the oldest supported OS, but the macos-26 image ships only iOS
26.x runtimes, so "floor" resolved to 26.2 and "current" to 26.5 — the
run compared two 26.x images and never touched an 18.x one. The measured
non-nested coverage that motivated the matrix (18.6 vs 27.0) reproduces
locally but not on this runner. The workflow comment now states this
plainly, and the selection step emits a warning annotation when the
resolved floor sits a major version or more above the deployment target,
so the gap is visible in the CI UI rather than assumed away.

Installing an older runtime in CI is possible via xcodebuild
-downloadPlatform but costs several GB and minutes per job; left out
pending a call on whether that trade is worth it.

Layout: unified · split

.github/workflows/build.yml +30 −11

		@@ -20,18 +20,28 @@ name: Build
20	20	# TEST_RUNNER_-prefixed build setting reaches the UI test process.)
21	21	#
22	22	# The matrix runs two simulators because audit coverage is NOT nested — each
23		# runtime reports findings the other misses, in both directions. Measured on the
24		# Tracked Domains screen, iOS 18.6 reported 2 issues and iOS 27.0 reported 6
25		# (including contrast and element-detection issues 18.6 never raised); at
26		# accessibility text sizes the Dashboard produced a hit-region finding on 18.6
27		# that 27.0 did not. Testing only the newest image would leave the oldest
28		# supported OS unchecked; testing only the floor would miss newer audit checks.
	23	# runtime reports findings the other misses, in both directions. Measured
	24	# locally on the Tracked Domains screen, iOS 18.6 reported 2 issues and iOS 27.0
	25	# reported 6 (including contrast and element-detection issues 18.6 never
	26	# raised); at accessibility text sizes the Dashboard produced a hit-region
	27	# finding on 18.6 that 27.0 did not.
29	28	#
30		# Runtimes are resolved dynamically rather than pinned: the deployment target is
31		# 17.6, but no 17.6 simulator runtime ships, so "floor" means the oldest
32		# available runtime at or above the deployment target (18.6 at time of writing).
33		# The previous selector took the first iPhone from any runtime, which could pick
34		# a simulator BELOW the deployment target, where the app cannot install.
	29	# CAVEAT — this image cannot test the real support floor. The app's deployment
	30	# target is 17.6, but the macos-26 runner ships only iOS 26.x simulator
	31	# runtimes, so "floor" resolves to ~26.2 here rather than an 18.x image. CI
	32	# therefore compares two 26.x runtimes; genuine oldest-supported-OS coverage has
	33	# to come from a local run or a self-hosted runner with older runtimes
	34	# installed. The "Select simulator" step emits a warning annotation when the
	35	# resolved floor sits well above the deployment target, so this gap stays
	36	# visible instead of being silently assumed away.
	37	#
	38	# Installing an older runtime in CI (xcodebuild -downloadPlatform iOS
	39	# -buildVersion 18.6) is possible but costs several GB and minutes per job; not
	40	# done by default.
	41	#
	42	# Runtimes are resolved dynamically rather than pinned. The previous selector
	43	# took the first iPhone from any runtime, which could pick a simulator BELOW the
	44	# deployment target, where the app cannot install.
35	45	#
36	46	# pull_request only, plus manual dispatch. GitHub builds the merge result (PR
37	47	# merged into main), so a green PR validates exactly what will land on main.
		@@ -115,6 +125,15 @@ jobs:
115	125	echo "udid=$(echo "$selected" \| jq -r .udid)" >> "$GITHUB_OUTPUT"
116	126	echo "label=$label" >> "$GITHUB_OUTPUT"
117	127
	128	# Surface the floor-coverage gap rather than letting the matrix imply
	129	# coverage it does not have. One major version of slack is tolerated.
	130	if [ "${{ matrix.tier }}" = "floor" ]; then
	131	rank=$(echo "$selected" \| jq -r .rank)
	132	if [ "$rank" -ge $(( (DEPLOYMENT_TARGET_MAJOR + 1) * 1000 )) ]; then
	133	echo "::warning::Floor tier resolved to $label, well above the ${DEPLOYMENT_TARGET_MAJOR}.${DEPLOYMENT_TARGET_MINOR} deployment target. This image has no older runtime, so the oldest supported OS is NOT covered by this run."
	134	fi
	135	fi
	136
118	137	- name: Test on ${{ steps.sim.outputs.label }}
119	138	run: \|
120	139	set -o pipefail

DomainDigUITests/AccessibilityAuditHarness.swift +54 −7

		@@ -40,6 +40,9 @@ enum AccessibilityAuditHarness {
40	40	/// `.sufficientElementDescription`, `.trait`
41	41	static let enforcedAuditTypes: XCUIAccessibilityAuditType = []
42	42
	43	/// How many times to retry an audit that misses its internal deadline.
	44	private static let auditAttempts = 3
	45
43	46	/// Launches the app with feature gating lifted, optionally at a specific
44	47	/// content size category.
45	48	static func launch(contentSizeCategory: String? = nil) -> XCUIApplication {
		@@ -56,19 +59,53 @@ enum AccessibilityAuditHarness {
56	59	///
57	60	/// Findings are logged and attached to the result bundle so a CI run
58	61	/// produces the burndown list as an artifact rather than only a pass/fail.
	62	///
	63	/// Returns `false` if the audit could not complete, leaving the screen
	64	/// unaudited. Callers turn that into an `XCTSkip` — reporting a pass would
	65	/// claim coverage that did not happen.
	66	@discardableResult
59	67	static func audit(
60	68	_ app: XCUIApplication,
61	69	screen: String,
62	70	test: XCTestCase
63		) throws {
	71	) throws -> Bool {
64	72	var findings: [String] = []
	73	var timeout: Error?
	74
	75	// The audit traverses the whole element tree and has its own internal
	76	// deadline, which slower CI runners miss on the denser screens. That is a
	77	// tooling timeout, not an app defect, so retry before giving up.
	78	//
	79	// Only the timeout is retried. If a category is enforced and the audit
	80	// reports findings before timing out, those failures are already recorded
	81	// and a retry would duplicate them — accepted, because the alternative is
	82	// losing the run to an infrastructure hiccup.
	83	for attempt in 1...auditAttempts {
	84	findings.removeAll()
	85	timeout = nil
	86	do {
	87	try app.performAccessibilityAudit { issue in
	88	let isEnforced = !enforcedAuditTypes.intersection(issue.auditType).isEmpty
	89	let marker = isEnforced ? "FAIL" : "report"
	90	findings.append("[\(marker)][\(name(for: issue.auditType))] \(issue.compactDescription)")
	91	// true suppresses the finding, false reports it as a test failure.
	92	return !isEnforced
	93	}
	94	break
	95	} catch let error as NSError where error.isAccessibilityAuditTimeout {
	96	timeout = error
	97	print("\(screen): audit timed out (attempt \(attempt) of \(auditAttempts))")
	98	}
	99	}
65	100
66		try app.performAccessibilityAudit { issue in
67		let isEnforced = !enforcedAuditTypes.intersection(issue.auditType).isEmpty
68		let marker = isEnforced ? "FAIL" : "report"
69		findings.append("[\(marker)][\(name(for: issue.auditType))] \(issue.compactDescription)")
70		// true suppresses the finding, false reports it as a test failure.
71		return !isEnforced
	101	if timeout != nil {
	102	let message = "\(screen): audit did not complete in time after \(auditAttempts) attempts — screen NOT audited"
	103	print(message)
	104	let attachment = XCTAttachment(string: message)
	105	attachment.name = "a11y-audit-\(screen)-timeout"
	106	attachment.lifetime = .keepAlways
	107	test.add(attachment)
	108	return false
72	109	}
73	110
74	111	let summary = findings.isEmpty
		@@ -81,6 +118,8 @@ enum AccessibilityAuditHarness {
81	118	attachment.name = "a11y-audit-\(screen)"
82	119	attachment.lifetime = .keepAlways
83	120	test.add(attachment)
	121
	122	return true
84	123	}
85	124
86	125	/// `XCUIAccessibilityAuditType` is an option set whose description is just a
		@@ -102,6 +141,14 @@ enum AccessibilityAuditHarness {
102	141	}
103	142	}
104	143
	144	private extension NSError {
	145	/// `Audit failed to complete in time` — the audit's own deadline, raised by
	146	/// XCTest rather than by anything wrong with the app.
	147	var isAccessibilityAuditTimeout: Bool {
	148	domain == "com.apple.xcode.xctest.accessibilityAudit" && code == -56
	149	}
	150	}
	151
105	152	extension XCUIApplication {
106	153	/// Taps a root tab by its visible label.
107	154	///

DomainDigUITests/AccessibilityAuditTests.swift +30 −21

		@@ -17,33 +17,23 @@ final class AccessibilityAuditTests: XCTestCase {
17	17	// MARK: Per-screen audits
18	18
19	19	func testInspectScreen() throws {
20		let app = AccessibilityAuditHarness.launch()
21		app.selectRootTab("Inspect")
22		try AccessibilityAuditHarness.audit(app, screen: "inspect", test: self)
	20	try auditRootTab("Inspect")
23	21	}
24	22
25	23	func testDashboardScreen() throws {
26		let app = AccessibilityAuditHarness.launch()
27		app.selectRootTab("Dashboard")
28		try AccessibilityAuditHarness.audit(app, screen: "dashboard", test: self)
	24	try auditRootTab("Dashboard")
29	25	}
30	26
31	27	func testAuditScreen() throws {
32		let app = AccessibilityAuditHarness.launch()
33		app.selectRootTab("Audit")
34		try AccessibilityAuditHarness.audit(app, screen: "audit", test: self)
	28	try auditRootTab("Audit")
35	29	}
36	30
37	31	func testHistoryScreen() throws {
38		let app = AccessibilityAuditHarness.launch()
39		app.selectRootTab("History")
40		try AccessibilityAuditHarness.audit(app, screen: "history", test: self)
	32	try auditRootTab("History")
41	33	}
42	34
43	35	func testSettingsScreen() throws {
44		let app = AccessibilityAuditHarness.launch()
45		app.selectRootTab("Settings")
46		try AccessibilityAuditHarness.audit(app, screen: "settings", test: self)
	36	try auditRootTab("Settings")
47	37	}
48	38
49	39	func testTrackedDomainsScreen() throws {
		@@ -57,7 +47,8 @@ final class AccessibilityAuditTests: XCTestCase {
57	47	)
58	48	trackedDomains.tap()
59	49
60		try AccessibilityAuditHarness.audit(app, screen: "tracked-domains", test: self)
	50	let audited = try AccessibilityAuditHarness.audit(app, screen: "tracked-domains", test: self)
	51	try XCTSkipUnless(audited, "Tracked Domains audit did not complete in time")
61	52	}
62	53
63	54	// MARK: Dynamic Type
		@@ -67,18 +58,36 @@ final class AccessibilityAuditTests: XCTestCase {
67	58	/// This is where clipped text and fixed-height containers surface — the
68	59	/// `.accessibility5`-class failures that the fixed geometry in
69	60	/// `AppDensityMetrics` is expected to produce until phase 3 of #21 lands.
	61	///
	62	/// Every screen is attempted even if an earlier one times out, so one slow
	63	/// screen cannot silently drop the rest; the skip is reported at the end.
70	64	func testAllScreensAtLargestAccessibilitySize() throws {
71	65	let app = AccessibilityAuditHarness.launch(
72	66	contentSizeCategory: "UICTContentSizeCategoryAccessibilityXXXL"
73	67	)
74	68
	69	var unaudited: [String] = []
	70
75	71	for tab in ["Inspect", "Dashboard", "Audit", "History", "Settings"] {
76	72	app.selectRootTab(tab)
77		try AccessibilityAuditHarness.audit(
78		app,
79		screen: "\(tab.lowercased())-accessibilityXXXL",
80		test: self
81		)
	73	let screen = "\(tab.lowercased())-accessibilityXXXL"
	74	if try !AccessibilityAuditHarness.audit(app, screen: screen, test: self) {
	75	unaudited.append(tab)
	76	}
82	77	}
	78
	79	try XCTSkipUnless(
	80	unaudited.isEmpty,
	81	"Audit did not complete in time for: \(unaudited.joined(separator: ", "))"
	82	)
	83	}
	84
	85	// MARK: Helpers
	86
	87	private func auditRootTab(_ tab: String) throws {
	88	let app = AccessibilityAuditHarness.launch()
	89	app.selectRootTab(tab)
	90	let audited = try AccessibilityAuditHarness.audit(app, screen: tab.lowercased(), test: self)
	91	try XCTSkipUnless(audited, "Audit did not complete in time for \(tab)")
83	92	}
84	93	}

Commit ea57052e99

Commit `ea57052e99`