krz/domain-dig

  1import XCTest
  2
  3/// Shared plumbing for the accessibility audit suite.
  4///
  5/// `performAccessibilityAudit` checks contrast, hit-region size, clipped text at
  6/// large Dynamic Type, element descriptions, and trait correctness — the same
  7/// categories the accessibility pass in issue #21 works through.
  8///
  9/// **The suite reports by default and fails only for enforced categories.** The
 10/// audit surfaces violations that exist today, so failing on everything would
 11/// block unrelated PRs until the whole pass lands. `enforcedAuditTypes` below is
 12/// the ratchet: widen it as each phase of #21 clears a category.
 13///
 14/// Two alternatives were tried and rejected:
 15///
 16/// - *A per-screen baseline count.* Audit coverage is not nested across OS
 17///   versions — the same screen legitimately yields different counts on the
 18///   floor simulator and the current one, so no single committed number is
 19///   correct for both.
 20/// - *An environment variable.* Neither a plain `xcodebuild` env var nor a
 21///   `TEST_RUNNER_`-prefixed build setting reaches this process, so the toggle
 22///   silently did nothing. A committed constant also makes "when did contrast
 23///   become enforced?" answerable with `git blame` instead of CI tribal
 24///   knowledge.
 25@MainActor
 26enum AccessibilityAuditHarness {
 27    /// Launch argument that lifts feature gating so Pro-only screens are
 28    /// reachable. `PurchaseService` honours this in `DEBUG` builds only.
 29    private static let forceProPlusArgument = "DOMAIN_DIG_FORCE_PRO_PLUS"
 30
 31    /// Audit categories that fail the build on the empty-state suite. A named
 32    /// finding in any of these is a regression in the phase 1–5 work.
 33    ///
 34    /// `.contrast` is deliberately absent: the two long-standing Settings
 35    /// findings come from rows scrolled under the translucent tab bar, and their
 36    /// attribution flips between a row name and nil run-to-run, so there is no
 37    /// suppression narrow enough to keep CI stable. Contrast stays report-only,
 38    /// with the palette centralised in `Shared/Colors.xcassets` as the actual
 39    /// guard.
 40    static let enforcedAuditTypes: XCUIAccessibilityAuditType = [
 41        .textClipped,
 42        .dynamicType,
 43        .hitRegion,
 44        .elementDetection,
 45        .sufficientElementDescription,
 46        .trait
 47    ]
 48
 49    /// How many times to retry an audit that misses its internal deadline.
 50    private static let auditAttempts = 3
 51
 52    /// Launch argument that seeds deterministic in-memory tracked domains and
 53    /// batch results (DEBUG builds only; never persisted). Without it the dense
 54    /// rows and portfolio sections render nothing, which is how five phases of
 55    /// row treatment went unmeasured.
 56    private static let seedFixturesArgument = "DOMAIN_DIG_SEED_FIXTURES"
 57
 58    /// Launches the app with feature gating lifted, optionally at a specific
 59    /// content size category and with the audit fixtures seeded.
 60    static func launch(contentSizeCategory: String? = nil, seeded: Bool = false) -> XCUIApplication {
 61        let app = XCUIApplication()
 62        app.launchArguments = [forceProPlusArgument]
 63        if seeded {
 64            app.launchArguments.append(seedFixturesArgument)
 65        }
 66        if let contentSizeCategory {
 67            app.launchArguments += ["-UIPreferredContentSizeCategoryName", contentSizeCategory]
 68        }
 69        app.launch()
 70        return app
 71    }
 72
 73    /// Runs a full audit and records every finding against the test.
 74    ///
 75    /// Findings are logged and attached to the result bundle so a CI run
 76    /// produces the burndown list as an artifact rather than only a pass/fail.
 77    ///
 78    /// Returns `false` if the audit could not complete, leaving the screen
 79    /// unaudited. Callers turn that into an `XCTSkip` — reporting a pass would
 80    /// claim coverage that did not happen.
 81    /// `reportOnly` disables enforcement for this call. Used by the seeded
 82    /// tests: bisection showed the audit degrades on `children: .ignore`
 83    /// content — the correct VoiceOver treatment for dense rows — reporting
 84    /// unattributed contrast/dynamicType failures on rows that measure 6–7:1
 85    /// and render correctly. Until that behaves, the seeded screens report
 86    /// their burndown without gating CI.
 87    @discardableResult
 88    static func audit(
 89        _ app: XCUIApplication,
 90        screen: String,
 91        test: XCTestCase,
 92        reportOnly: Bool = false
 93    ) throws -> Bool {
 94        var findings: [String] = []
 95        var timeout: Error?
 96
 97        // The audit traverses the whole element tree and has its own internal
 98        // deadline, which slower CI runners miss on the denser screens. That is a
 99        // tooling timeout, not an app defect, so retry before giving up.
100        //
101        // Only the timeout is retried. If a category is enforced and the audit
102        // reports findings before timing out, those failures are already recorded
103        // and a retry would duplicate them — accepted, because the alternative is
104        // losing the run to an infrastructure hiccup.
105        for attempt in 1...auditAttempts {
106            findings.removeAll()
107            timeout = nil
108            do {
109                try app.performAccessibilityAudit { issue in
110                    // Include the element so the burndown says *what* to fix, not
111                    // just that something is wrong.
112                    let element = issue.element.map { el -> String in
113                        let label = el.label.isEmpty ? el.identifier : el.label
114                        return label.isEmpty ? "\(el.elementType)" : "\"\(label)\""
115                    } ?? "unknown element"
116
117                    // Characterised noise never fails, but is still logged with
118                    // its reason — nothing disappears silently.
119                    if let noise = noiseReason(for: issue) {
120                        findings.append("[noise: \(noise)][\(name(for: issue.auditType))] \(issue.compactDescription) — \(element)")
121                        return true
122                    }
123
124                    let isEnforced = !reportOnly && !enforcedAuditTypes.intersection(issue.auditType).isEmpty
125                    let marker = isEnforced ? "FAIL" : "report"
126                    findings.append("[\(marker)][\(name(for: issue.auditType))] \(issue.compactDescription) — \(element)")
127                    // true suppresses the finding, false reports it as a test failure.
128                    return !isEnforced
129                }
130                break
131            } catch let error as NSError where error.isAccessibilityAuditTimeout {
132                timeout = error
133                print("\(screen): audit timed out (attempt \(attempt) of \(auditAttempts))")
134            }
135        }
136
137        if timeout != nil {
138            let message = "\(screen): audit did not complete in time after \(auditAttempts) attempts — screen NOT audited"
139            print(message)
140            let attachment = XCTAttachment(string: message)
141            attachment.name = "a11y-audit-\(screen)-timeout"
142            attachment.lifetime = .keepAlways
143            test.add(attachment)
144            return false
145        }
146
147        let summary = findings.isEmpty
148            ? "\(screen): no accessibility findings"
149            : "\(screen): \(findings.count) finding(s)\n" + findings.sorted().map { "  • \($0)" }.joined(separator: "\n")
150
151        print(summary)
152
153        let attachment = XCTAttachment(string: summary)
154        attachment.name = "a11y-audit-\(screen)"
155        attachment.lifetime = .keepAlways
156        test.add(attachment)
157
158        return true
159    }
160
161    /// Classifies findings that are measurement artifacts, not app defects.
162    /// Each rule exists because it was proven, not assumed; the evidence is
163    /// recorded inline. A classified finding is logged with its reason and
164    /// never fails the build.
165    private static func noiseReason(for issue: XCUIAccessibilityAuditIssue) -> String? {
166        // WCAG 1.4.3 exempts inactive components from contrast requirements,
167        // but the audit flags them anyway. Proven on Inspect's Run button,
168        // disabled until a domain is typed. (Driving the UI to enable it was
169        // worse: the raised keyboard followed the audit onto later screens and
170        // flagged the emoji picker.)
171        if issue.auditType.contains(.contrast), issue.element?.isEnabled == false {
172            return "disabled control, WCAG 1.4.3 exempt"
173        }
174
175        // "Nearly passed" is the audit's near-miss band, not a failure. The
176        // only occurrences are iOS-rendered Settings section headers, whose
177        // styling is the system's.
178        if issue.compactDescription.localizedCaseInsensitiveContains("nearly passed") {
179            return "near-miss, not a failure"
180        }
181
182        // Placeholder text in text/search fields is reported clipped at ANY
183        // length — shortening "Search portfolio" to "Search" changed nothing —
184        // and the search field's hit region at accessibility sizes is the
185        // system's own control. Reading `elementType` here is safe; reading
186        // `frame` is not (it kills element attribution for the whole audit).
187        if let type = issue.element?.elementType, type == .searchField || type == .textField {
188            if issue.auditType.contains(.textClipped) || issue.auditType.contains(.hitRegion) {
189                return "system field placeholder/hit region, length-independent"
190            }
191        }
192
193        // Unattributed clipped-text/dynamic-type findings. Bisection showed the
194        // audit loses attribution inside NavigationLink rows and
195        // children-ignored elements and then reports failures on content that
196        // is visually verified correct (and, for the one long-standing
197        // empty-watchlist phantom, renders nothing clipped at all). Named
198        // findings in these categories still enforce.
199        if issue.element == nil,
200           issue.auditType.contains(.textClipped) || issue.auditType.contains(.dynamicType) {
201            return "unattributed, audit artifact on ignored/link content"
202        }
203
204        return nil
205    }
206
207    /// `XCUIAccessibilityAuditType` is an option set whose description is just a
208    /// raw bitmask, which makes the burndown list unreadable. Resolve it against
209    /// the named members rather than hard-coding bit positions, so this keeps
210    /// working if Apple adds audit types.
211    private static func name(for type: XCUIAccessibilityAuditType) -> String {
212        let known: [(XCUIAccessibilityAuditType, String)] = [
213            (.contrast, "contrast"),
214            (.elementDetection, "elementDetection"),
215            (.hitRegion, "hitRegion"),
216            (.sufficientElementDescription, "sufficientElementDescription"),
217            (.dynamicType, "dynamicType"),
218            (.textClipped, "textClipped"),
219            (.trait, "trait")
220        ]
221        let matched = known.filter { type.contains($0.0) }.map(\.1)
222        return matched.isEmpty ? "unknown(\(type.rawValue))" : matched.joined(separator: "+")
223    }
224}
225
226private extension NSError {
227    /// `Audit failed to complete in time` — the audit's own deadline, raised by
228    /// XCTest rather than by anything wrong with the app.
229    var isAccessibilityAuditTimeout: Bool {
230        domain == "com.apple.xcode.xctest.accessibilityAudit" && code == -56
231    }
232}
233
234extension XCUIApplication {
235    /// Taps a root tab by its visible label.
236    ///
237    /// Falls back to a plain button query because the tab bar is only present in
238    /// the compact size class — in regular width `RootTabView` renders a
239    /// `NavigationSplitView` sidebar instead.
240    @MainActor
241    func selectRootTab(_ name: String) {
242        let tabButton = tabBars.buttons[name]
243        let element = tabButton.waitForExistence(timeout: 5) ? tabButton : buttons[name]
244        XCTAssertTrue(
245            element.waitForExistence(timeout: 5),
246            "Could not find a way to reach the \(name) screen"
247        )
248        element.tap()
249    }
250}