krz/domain-dig

  1#!/usr/bin/env bash
  2#
  3# Run the accessibility audit suite against real simulator runtimes.
  4#
  5#   ./Scripts/audit-a11y.sh            # floor + current
  6#   ./Scripts/audit-a11y.sh floor      # oldest supported runtime only
  7#   ./Scripts/audit-a11y.sh current    # newest installed runtime only
  8#
  9# Why this exists rather than living entirely in CI: audit coverage is NOT
 10# nested across OS versions — each runtime reports findings the others miss, in
 11# both directions. Measured on Tracked Domains, iOS 18.6 reported 2 findings and
 12# iOS 27.0 reported 6; at accessibility text sizes the Dashboard produced a
 13# hit-region finding on 18.6 that 27.0 did not. The GitHub macos-26 image ships
 14# only iOS 26.x runtimes, so it structurally cannot cover the floor. This machine
 15# can.
 16#
 17# CI covers "current" on a clean checkout (which a local run cannot, since it
 18# would miss an uncommitted file). This script covers "floor" (which CI cannot).
 19# Together they cover both; neither duplicates the other.
 20#
 21# The deployment target is read from the project rather than hard-coded, so it
 22# cannot drift out of sync.
 23
 24set -euo pipefail
 25
 26cd "$(dirname "$0")/.."
 27
 28PROJECT="DomainDig.xcodeproj"
 29SCHEME="DomainDig"
 30TIER="${1:-both}"
 31
 32case "$TIER" in
 33  floor | current | both) ;;
 34  *)
 35    echo "usage: $0 [floor|current|both]" >&2
 36    exit 2
 37    ;;
 38esac
 39
 40echo "==> Reading deployment target from $PROJECT"
 41DEPLOYMENT_TARGET=$(
 42  xcodebuild -project "$PROJECT" -scheme "$SCHEME" -showBuildSettings 2>/dev/null \
 43    | awk -F' = ' '/ IPHONEOS_DEPLOYMENT_TARGET = /{print $2; exit}'
 44)
 45
 46if [ -z "${DEPLOYMENT_TARGET:-}" ]; then
 47  echo "error: could not read IPHONEOS_DEPLOYMENT_TARGET" >&2
 48  exit 1
 49fi
 50
 51DT_MAJOR="${DEPLOYMENT_TARGET%%.*}"
 52DT_MINOR="${DEPLOYMENT_TARGET##*.}"
 53[ "$DT_MINOR" = "$DEPLOYMENT_TARGET" ] && DT_MINOR=0
 54FLOOR_RANK=$(( DT_MAJOR * 1000 + DT_MINOR ))
 55
 56echo "    deployment target: $DEPLOYMENT_TARGET (rank $FLOOR_RANK)"
 57
 58# Pick an iPhone simulator at or above the deployment target. A runtime BELOW it
 59# is useless — the app cannot install there — which is why this filters rather
 60# than just taking the oldest installed runtime.
 61select_sim() {
 62  local which="$1"
 63  xcrun simctl list devices available --json \
 64    | jq -c --argjson floor "$FLOOR_RANK" --arg which "$which" '
 65        [ .devices | to_entries[]
 66          | (.key | capture("SimRuntime\\.iOS-(?<maj>[0-9]+)-(?<min>[0-9]+)$")) as $v
 67          | (($v.maj | tonumber) * 1000 + ($v.min | tonumber)) as $rank
 68          | select($rank >= $floor)
 69          | .value[]
 70          | select(.name | startswith("iPhone"))
 71          | { rank: $rank, udid: .udid, name: .name, os: "\($v.maj).\($v.min)" }
 72        ]
 73        | sort_by(.rank, .name)
 74        | if length == 0 then empty
 75          elif $which == "floor" then first
 76          else last end
 77      '
 78}
 79
 80run_tier() {
 81  local which="$1"
 82  local sim udid label
 83
 84  sim=$(select_sim "$which")
 85  if [ -z "$sim" ]; then
 86    echo "error: no iPhone simulator at or above iOS $DEPLOYMENT_TARGET installed" >&2
 87    echo "hint: install one with 'xcodebuild -downloadPlatform iOS'" >&2
 88    return 1
 89  fi
 90
 91  udid=$(echo "$sim" | jq -r .udid)
 92  label=$(echo "$sim" | jq -r '"\(.name) (iOS \(.os))"')
 93
 94  echo
 95  echo "==> $which: $label"
 96
 97  if [ "$which" = "floor" ] && [ "$(echo "$sim" | jq -r .rank)" -ge $(( (DT_MAJOR + 1) * 1000 )) ]; then
 98    echo "    NOTE: nearest installed runtime is a major version above the $DEPLOYMENT_TARGET"
 99    echo "          deployment target, so this is not true floor coverage."
100  fi
101
102  # Findings are printed by the suite itself; surface them plus the verdict.
103  set -o pipefail
104  xcodebuild test \
105    -project "$PROJECT" \
106    -scheme "$SCHEME" \
107    -destination "id=$udid" \
108    CODE_SIGNING_ALLOWED=NO 2>&1 \
109    | grep -E 'finding\(s\)|no accessibility findings|^  • |did not complete in time|Executed [0-9]+ test|\*\* TEST (SUCCEEDED|FAILED)' \
110    || true
111
112  return 0
113}
114
115status=0
116if [ "$TIER" = "both" ]; then
117  run_tier floor || status=1
118  run_tier current || status=1
119else
120  run_tier "$TIER" || status=1
121fi
122
123echo
124if [ "$status" -eq 0 ]; then
125  echo "==> Done. Findings above are the burndown list for issue #21."
126  echo "    They are reported, not enforced — widen"
127  echo "    AccessibilityAuditHarness.enforcedAuditTypes as each phase lands."
128fi
129exit "$status"