krz/domain-dig

  1import XCTest
  2
  3/// Shared plumbing for the accessibility audit suite.
  4///
  5/// `performAccessibilityAudit` checks contrast, hit-region size, clipped text at
  6/// large Dynamic Type, element descriptions, and trait correctness — the same
  7/// categories the accessibility pass in issue #21 works through.
  8///
  9/// **The suite reports by default and fails only for enforced categories.** The
 10/// audit surfaces violations that exist today, so failing on everything would
 11/// block unrelated PRs until the whole pass lands. `enforcedAuditTypes` below is
 12/// the ratchet: widen it as each phase of #21 clears a category.
 13///
 14/// Two alternatives were tried and rejected:
 15///
 16/// - *A per-screen baseline count.* Audit coverage is not nested across OS
 17///   versions — the same screen legitimately yields different counts on the
 18///   floor simulator and the current one, so no single committed number is
 19///   correct for both.
 20/// - *An environment variable.* Neither a plain `xcodebuild` env var nor a
 21///   `TEST_RUNNER_`-prefixed build setting reaches this process, so the toggle
 22///   silently did nothing. A committed constant also makes "when did contrast
 23///   become enforced?" answerable with `git blame` instead of CI tribal
 24///   knowledge.
 25@MainActor
 26enum AccessibilityAuditHarness {
 27    /// Launch argument that lifts feature gating so Pro-only screens are
 28    /// reachable. `PurchaseService` honours this in `DEBUG` builds only.
 29    private static let forceProPlusArgument = "DOMAIN_DIG_FORCE_PRO_PLUS"
 30
 31    /// Audit categories that fail the build on the empty-state suite. A named
 32    /// finding in any of these is a regression in the phase 1–5 work.
 33    ///
 34    /// `.contrast` is deliberately absent: the two long-standing Settings
 35    /// findings come from rows scrolled under the translucent tab bar, and their
 36    /// attribution flips between a row name and nil run-to-run, so there is no
 37    /// suppression narrow enough to keep CI stable. Contrast stays report-only,
 38    /// with the palette centralised in `Shared/Colors.xcassets` as the actual
 39    /// guard.
 40    static let enforcedAuditTypes: XCUIAccessibilityAuditType = [
 41        .textClipped,
 42        .dynamicType,
 43        .hitRegion,
 44        .elementDetection,
 45        .sufficientElementDescription,
 46        .trait
 47    ]
 48
 49    /// How many times to retry an audit that misses its internal deadline.
 50    private static let auditAttempts = 3
 51
 52    /// Launch argument that seeds deterministic in-memory tracked domains and
 53    /// batch results (DEBUG builds only; never persisted). Without it the dense
 54    /// rows and portfolio sections render nothing, which is how five phases of
 55    /// row treatment went unmeasured.
 56    private static let seedFixturesArgument = "DOMAIN_DIG_SEED_FIXTURES"
 57
 58    /// Launches the app with feature gating lifted, optionally at a specific
 59    /// content size category and with the audit fixtures seeded.
 60    static func launch(contentSizeCategory: String? = nil, seeded: Bool = false) -> XCUIApplication {
 61        let app = XCUIApplication()
 62        app.launchArguments = [forceProPlusArgument]
 63        if seeded {
 64            app.launchArguments.append(seedFixturesArgument)
 65        }
 66        if let contentSizeCategory {
 67            app.launchArguments += ["-UIPreferredContentSizeCategoryName", contentSizeCategory]
 68        }
 69        app.launch()
 70        return app
 71    }
 72
 73    /// Runs a full audit and records every finding against the test.
 74    ///
 75    /// Findings are logged and attached to the result bundle so a CI run
 76    /// produces the burndown list as an artifact rather than only a pass/fail.
 77    ///
 78    /// Returns `false` if the audit could not complete, leaving the screen
 79    /// unaudited. Callers turn that into an `XCTSkip` — reporting a pass would
 80    /// claim coverage that did not happen.
 81    /// `reportOnly` disables enforcement for this call. Used by the seeded
 82    /// tests: bisection showed the audit degrades on `children: .ignore`
 83    /// content — the correct VoiceOver treatment for dense rows — reporting
 84    /// unattributed contrast/dynamicType failures on rows that measure 6–7:1
 85    /// and render correctly. Until that behaves, the seeded screens report
 86    /// their burndown without gating CI.
 87    @discardableResult
 88    static func audit(
 89        _ app: XCUIApplication,
 90        screen: String,
 91        test: XCTestCase,
 92        reportOnly: Bool = false
 93    ) throws -> Bool {
 94        var findings: [String] = []
 95        var timeout: Error?
 96
 97        // The audit traverses the whole element tree and has its own internal
 98        // deadline, which slower CI runners miss on the denser screens. That is a
 99        // tooling timeout, not an app defect, so retry before giving up.
100        //
101        // Only the timeout is retried. If a category is enforced and the audit
102        // reports findings before timing out, those failures are already recorded
103        // and a retry would duplicate them — accepted, because the alternative is
104        // losing the run to an infrastructure hiccup.
105        for attempt in 1...auditAttempts {
106            findings.removeAll()
107            timeout = nil
108            do {
109                try app.performAccessibilityAudit { issue in
110                    // Include the element so the burndown says *what* to fix, not
111                    // just that something is wrong.
112                    let element = issue.element.map { el -> String in
113                        let label = el.label.isEmpty ? el.identifier : el.label
114                        return label.isEmpty ? "\(el.elementType)" : "\"\(label)\""
115                    } ?? "unknown element"
116
117                    // Characterised noise never fails, but is still logged with
118                    // its reason — nothing disappears silently.
119                    if let noise = noiseReason(for: issue) {
120                        findings.append("[noise: \(noise)][\(name(for: issue.auditType))] \(issue.compactDescription) — \(element)")
121                        return true
122                    }
123
124                    let isEnforced = !reportOnly && !enforcedAuditTypes.intersection(issue.auditType).isEmpty
125                    let marker = isEnforced ? "FAIL" : "report"
126                    findings.append("[\(marker)][\(name(for: issue.auditType))] \(issue.compactDescription) — \(element)")
127                    // true suppresses the finding, false reports it as a test failure.
128                    return !isEnforced
129                }
130                break
131            } catch let error as NSError where error.isAccessibilityAuditTimeout {
132                timeout = error
133                print("\(screen): audit timed out (attempt \(attempt) of \(auditAttempts))")
134            }
135        }
136
137        if timeout != nil {
138            let message = "\(screen): audit did not complete in time after \(auditAttempts) attempts — screen NOT audited"
139            print(message)
140            let attachment = XCTAttachment(string: message)
141            attachment.name = "a11y-audit-\(screen)-timeout"
142            attachment.lifetime = .keepAlways
143            test.add(attachment)
144            return false
145        }
146
147        let summary = findings.isEmpty
148            ? "\(screen): no accessibility findings"
149            : "\(screen): \(findings.count) finding(s)\n" + findings.sorted().map { "  • \($0)" }.joined(separator: "\n")
150
151        print(summary)
152
153        let attachment = XCTAttachment(string: summary)
154        attachment.name = "a11y-audit-\(screen)"
155        attachment.lifetime = .keepAlways
156        test.add(attachment)
157
158        return true
159    }
160
161    /// Classifies findings that are measurement artifacts, not app defects.
162    /// Each rule exists because it was proven, not assumed; the evidence is
163    /// recorded inline. A classified finding is logged with its reason and
164    /// never fails the build.
165    private static func noiseReason(for issue: XCUIAccessibilityAuditIssue) -> String? {
166        // WCAG 1.4.3 exempts inactive components from contrast requirements,
167        // but the audit flags them anyway. Proven on Inspect's Run button,
168        // disabled until a domain is typed. (Driving the UI to enable it was
169        // worse: the raised keyboard followed the audit onto later screens and
170        // flagged the emoji picker.)
171        if issue.auditType.contains(.contrast), issue.element?.isEnabled == false {
172            return "disabled control, WCAG 1.4.3 exempt"
173        }
174
175        // "Nearly passed" is the audit's near-miss band, not a failure. The
176        // only occurrences are iOS-rendered Settings section headers, whose
177        // styling is the system's.
178        if issue.compactDescription.localizedCaseInsensitiveContains("nearly passed") {
179            return "near-miss, not a failure"
180        }
181
182        // Placeholder text in text/search fields is reported clipped at ANY
183        // length — shortening "Search portfolio" to "Search" changed nothing —
184        // and the search field's hit region at accessibility sizes is the
185        // system's own control. Reading `elementType` here is safe; reading
186        // `frame` is not (it kills element attribution for the whole audit).
187        if let type = issue.element?.elementType,
188           type == .searchField || type == .textField,
189           issue.auditType.contains(.textClipped) || issue.auditType.contains(.hitRegion) {
190            return "system field placeholder/hit region, length-independent"
191        }
192
193        // Unattributed clipped-text/dynamic-type findings. Bisection showed the
194        // audit loses attribution inside NavigationLink rows and
195        // children-ignored elements and then reports failures on content that
196        // is visually verified correct (and, for the one long-standing
197        // empty-watchlist phantom, renders nothing clipped at all). Named
198        // findings in these categories still enforce.
199        if issue.element == nil,
200           issue.auditType.contains(.textClipped) || issue.auditType.contains(.dynamicType) {
201            return "unattributed, audit artifact on ignored/link content"
202        }
203
204        // iOS-27-only Settings `Section` header dynamicType finding. On iOS 27.0
205        // (and only there) the audit reports "font sizes partially unsupported"
206        // against a Settings section header — the same system-rendered headers
207        // already carved out for the contrast near-miss above. Proof it is a
208        // system-chrome artifact, not an app defect:
209        //   • Each is a plain `Section("Services")` etc. (ContentView.swift ~2768);
210        //     the app sets no font, so the scaling is UIKit's `.footnote` header.
211        //   • Version-specific: absent on the iOS 18.6 floor and on the 26.x
212        //     runtime CI runs (both audit clean); it surfaces only under 27.0.
213        //     ACCESSIBILITY.md's coverage table records the same asymmetry
214        //     ("dynamicType finding 18.6 missed").
215        //   • Attribution is unstable run-to-run across the header set
216        //     (Tier/Preferences/Services), exactly like the documented contrast
217        //     flip — so it lands on whichever header the traversal reaches first.
218        // Overriding every Section header with a custom scaling `Text` to chase
219        // this was rejected for the contrast case (ACCESSIBILITY.md) for trading
220        // platform convention for nothing; the same holds here. Scoped to
221        // dynamicType on the exact Settings header titles so a real regression
222        // on app-controlled text still enforces.
223        if issue.auditType.contains(.dynamicType),
224           let label = issue.element?.label,
225           settingsSectionHeaders.contains(label) {
226            return "iOS-rendered Settings section header, app sets no font (27.0-only)"
227        }
228
229        return nil
230    }
231
232    /// The Settings screen's `Section(_:)` header titles. UIKit renders these;
233    /// the app passes only a string literal. Used to scope the section-header
234    /// dynamicType carve-out narrowly (see `noiseReason`).
235    private static let settingsSectionHeaders: Set<String> = [
236        "Tier", "Preferences", "Services", "Data", "About"
237    ]
238
239    /// `XCUIAccessibilityAuditType` is an option set whose description is just a
240    /// raw bitmask, which makes the burndown list unreadable. Resolve it against
241    /// the named members rather than hard-coding bit positions, so this keeps
242    /// working if Apple adds audit types.
243    private static func name(for type: XCUIAccessibilityAuditType) -> String {
244        let known: [(XCUIAccessibilityAuditType, String)] = [
245            (.contrast, "contrast"),
246            (.elementDetection, "elementDetection"),
247            (.hitRegion, "hitRegion"),
248            (.sufficientElementDescription, "sufficientElementDescription"),
249            (.dynamicType, "dynamicType"),
250            (.textClipped, "textClipped"),
251            (.trait, "trait")
252        ]
253        let matched = known.filter { type.contains($0.0) }.map(\.1)
254        return matched.isEmpty ? "unknown(\(type.rawValue))" : matched.joined(separator: "+")
255    }
256}
257
258private extension NSError {
259    /// `Audit failed to complete in time` — the audit's own deadline, raised by
260    /// XCTest rather than by anything wrong with the app.
261    var isAccessibilityAuditTimeout: Bool {
262        domain == "com.apple.xcode.xctest.accessibilityAudit" && code == -56
263    }
264}
265
266extension XCUIApplication {
267    /// Taps a root tab by its visible label.
268    ///
269    /// Falls back to a plain button query because the tab bar is only present in
270    /// the compact size class — in regular width `RootTabView` renders a
271    /// `NavigationSplitView` sidebar instead.
272    @MainActor
273    func selectRootTab(_ name: String) {
274        let tabButton = tabBars.buttons[name]
275        let element = tabButton.waitForExistence(timeout: 5) ? tabButton : buttons[name]
276        XCTAssertTrue(
277            element.waitForExistence(timeout: 5),
278            "Could not find a way to reach the \(name) screen"
279        )
280        element.tap()
281    }
282}