| @@ -28,17 +28,23 @@ enum AccessibilityAuditHarness { |
| 28 | 28 | /// reachable. `PurchaseService` honours this in `DEBUG` builds only. |
| 29 | 29 | private static let forceProPlusArgument = "DOMAIN_DIG_FORCE_PRO_PLUS" |
| 30 | 30 | |
| 31 | | /// Audit categories that fail the build. Everything else is reported only. |
| 31 | /// Audit categories that fail the build on the empty-state suite. A named |
| 32 | /// finding in any of these is a regression in the phase 1–5 work. |
| 32 | 33 | /// |
| 33 | | /// Empty until the accessibility pass starts landing. Suggested ratchet, |
| 34 | | /// following the phases in issue #21: |
| 35 | | /// |
| 36 | | /// - after phase 2 (semantic colors + light mode): `.contrast` |
| 37 | | /// - after phase 3 (Dynamic Type + reflow): `.textClipped`, `.dynamicType`, |
| 38 | | /// `.hitRegion` |
| 39 | | /// - after phase 4 (VoiceOver): `.elementDetection`, |
| 40 | | /// `.sufficientElementDescription`, `.trait` |
| 41 | | static let enforcedAuditTypes: XCUIAccessibilityAuditType = [] |
| 34 | /// `.contrast` is deliberately absent: the two long-standing Settings |
| 35 | /// findings come from rows scrolled under the translucent tab bar, and their |
| 36 | /// attribution flips between a row name and nil run-to-run, so there is no |
| 37 | /// suppression narrow enough to keep CI stable. Contrast stays report-only, |
| 38 | /// with the palette centralised in `Shared/Colors.xcassets` as the actual |
| 39 | /// guard. |
| 40 | static let enforcedAuditTypes: XCUIAccessibilityAuditType = [ |
| 41 | .textClipped, |
| 42 | .dynamicType, |
| 43 | .hitRegion, |
| 44 | .elementDetection, |
| 45 | .sufficientElementDescription, |
| 46 | .trait |
| 47 | ] |
| 42 | 48 | |
| 43 | 49 | /// How many times to retry an audit that misses its internal deadline. |
| 44 | 50 | private static let auditAttempts = 3 |
| @@ -72,11 +78,18 @@ enum AccessibilityAuditHarness { |
| 72 | 78 | /// Returns `false` if the audit could not complete, leaving the screen |
| 73 | 79 | /// unaudited. Callers turn that into an `XCTSkip` — reporting a pass would |
| 74 | 80 | /// claim coverage that did not happen. |
| 81 | /// `reportOnly` disables enforcement for this call. Used by the seeded |
| 82 | /// tests: bisection showed the audit degrades on `children: .ignore` |
| 83 | /// content — the correct VoiceOver treatment for dense rows — reporting |
| 84 | /// unattributed contrast/dynamicType failures on rows that measure 6–7:1 |
| 85 | /// and render correctly. Until that behaves, the seeded screens report |
| 86 | /// their burndown without gating CI. |
| 75 | 87 | @discardableResult |
| 76 | 88 | static func audit( |
| 77 | 89 | _ app: XCUIApplication, |
| 78 | 90 | screen: String, |
| 79 | | test: XCTestCase |
| 91 | test: XCTestCase, |
| 92 | reportOnly: Bool = false |
| 80 | 93 | ) throws -> Bool { |
| 81 | 94 | var findings: [String] = [] |
| 82 | 95 | var timeout: Error? |
| @@ -94,26 +107,22 @@ enum AccessibilityAuditHarness { |
| 94 | 107 | timeout = nil |
| 95 | 108 | do { |
| 96 | 109 | try app.performAccessibilityAudit { issue in |
| 97 | | // WCAG 1.4.3 exempts inactive components from contrast |
| 98 | | // requirements, but the audit flags them anyway. Inspect's |
| 99 | | // Run button is disabled until a domain is typed, so the |
| 100 | | // empty state reported a contrast failure that was never a |
| 101 | | // real defect. Suppressing on the rule beats driving the UI |
| 102 | | // to enable the control: typing raises the keyboard, which |
| 103 | | // then follows the audit onto later screens and flags the |
| 104 | | // system emoji picker's category buttons. |
| 105 | | if issue.auditType.contains(.contrast), issue.element?.isEnabled == false { |
| 106 | | return true |
| 107 | | } |
| 108 | | |
| 109 | | let isEnforced = !enforcedAuditTypes.intersection(issue.auditType).isEmpty |
| 110 | | let marker = isEnforced ? "FAIL" : "report" |
| 111 | 110 | // Include the element so the burndown says *what* to fix, not |
| 112 | 111 | // just that something is wrong. |
| 113 | 112 | let element = issue.element.map { el -> String in |
| 114 | 113 | let label = el.label.isEmpty ? el.identifier : el.label |
| 115 | 114 | return label.isEmpty ? "\(el.elementType)" : "\"\(label)\"" |
| 116 | 115 | } ?? "unknown element" |
| 116 | |
| 117 | // Characterised noise never fails, but is still logged with |
| 118 | // its reason — nothing disappears silently. |
| 119 | if let noise = noiseReason(for: issue) { |
| 120 | findings.append("[noise: \(noise)][\(name(for: issue.auditType))] \(issue.compactDescription) — \(element)") |
| 121 | return true |
| 122 | } |
| 123 | |
| 124 | let isEnforced = !reportOnly && !enforcedAuditTypes.intersection(issue.auditType).isEmpty |
| 125 | let marker = isEnforced ? "FAIL" : "report" |
| 117 | 126 | findings.append("[\(marker)][\(name(for: issue.auditType))] \(issue.compactDescription) — \(element)") |
| 118 | 127 | // true suppresses the finding, false reports it as a test failure. |
| 119 | 128 | return !isEnforced |
| @@ -149,6 +158,52 @@ enum AccessibilityAuditHarness { |
| 149 | 158 | return true |
| 150 | 159 | } |
| 151 | 160 | |
| 161 | /// Classifies findings that are measurement artifacts, not app defects. |
| 162 | /// Each rule exists because it was proven, not assumed; the evidence is |
| 163 | /// recorded inline. A classified finding is logged with its reason and |
| 164 | /// never fails the build. |
| 165 | private static func noiseReason(for issue: XCUIAccessibilityAuditIssue) -> String? { |
| 166 | // WCAG 1.4.3 exempts inactive components from contrast requirements, |
| 167 | // but the audit flags them anyway. Proven on Inspect's Run button, |
| 168 | // disabled until a domain is typed. (Driving the UI to enable it was |
| 169 | // worse: the raised keyboard followed the audit onto later screens and |
| 170 | // flagged the emoji picker.) |
| 171 | if issue.auditType.contains(.contrast), issue.element?.isEnabled == false { |
| 172 | return "disabled control, WCAG 1.4.3 exempt" |
| 173 | } |
| 174 | |
| 175 | // "Nearly passed" is the audit's near-miss band, not a failure. The |
| 176 | // only occurrences are iOS-rendered Settings section headers, whose |
| 177 | // styling is the system's. |
| 178 | if issue.compactDescription.localizedCaseInsensitiveContains("nearly passed") { |
| 179 | return "near-miss, not a failure" |
| 180 | } |
| 181 | |
| 182 | // Placeholder text in text/search fields is reported clipped at ANY |
| 183 | // length — shortening "Search portfolio" to "Search" changed nothing — |
| 184 | // and the search field's hit region at accessibility sizes is the |
| 185 | // system's own control. Reading `elementType` here is safe; reading |
| 186 | // `frame` is not (it kills element attribution for the whole audit). |
| 187 | if let type = issue.element?.elementType, type == .searchField || type == .textField { |
| 188 | if issue.auditType.contains(.textClipped) || issue.auditType.contains(.hitRegion) { |
| 189 | return "system field placeholder/hit region, length-independent" |
| 190 | } |
| 191 | } |
| 192 | |
| 193 | // Unattributed clipped-text/dynamic-type findings. Bisection showed the |
| 194 | // audit loses attribution inside NavigationLink rows and |
| 195 | // children-ignored elements and then reports failures on content that |
| 196 | // is visually verified correct (and, for the one long-standing |
| 197 | // empty-watchlist phantom, renders nothing clipped at all). Named |
| 198 | // findings in these categories still enforce. |
| 199 | if issue.element == nil, |
| 200 | issue.auditType.contains(.textClipped) || issue.auditType.contains(.dynamicType) { |
| 201 | return "unattributed, audit artifact on ignored/link content" |
| 202 | } |
| 203 | |
| 204 | return nil |
| 205 | } |
| 206 | |
| 152 | 207 | /// `XCUIAccessibilityAuditType` is an option set whose description is just a |
| 153 | 208 | /// raw bitmask, which makes the burndown list unreadable. Resolve it against |
| 154 | 209 | /// the named members rather than hard-coding bit positions, so this keeps |