deploy/runner-auth-flood-test.sh

  1#!/bin/sh
  2# Scratch-repository test for #260: a build fails SSH logins while the
  3# runner works, and the runner must not be locked out with it.
  4#
  5# Run from a machine with an admin gitbay identity, after the Admin
  6# page's scratch procedure: the runner's key attached to the scratch
  7# repository and the runner scoped to it with -repos. The script
  8# replaces the repository's .gitbay/ci.yml.
  9#
 10#   deploy/runner-auth-flood-test.sh cmc/runner-scratch              # trusted: a push to main
 11#   deploy/runner-auth-flood-test.sh cmc/runner-scratch --untrusted  # a merge request from a fork
 12#
 13# The build's logins use a git-scoped key registered with --ttl 1s and
 14# expired by the time the build runs. With registration open an
 15# unknown key is admitted to run register and never counts against the
 16# SSH auth limiter; an expired key counts (internal/sshd/sshd.go, authenticate).
 17# The key is removed from the account when the script exits.
 18#
 19# The step waits a minute, so the operator can find pasta's cgroup
 20# (Admin page), probes what the build reaches, then makes 12 logins
 21# without pause, so the limiter (ssh_auth_rate, 10 a minute per address) locks
 22# the address those logins come from for most of the next minute. The
 23# runner reports the result right after the step; its report retries
 24# for half a minute.
 25#
 26# Before #260 a build reached the host's loopback through pasta, and its
 27# logins arrived from 127.0.0.1, where the runner polls: the report is
 28# refused ("too many authentication attempts" in the runner's journal),
 29# the build is failed by the server two minutes after its log stream
 30# ended, the runner's last-seen stops advancing for up to a minute, and
 31# the audit log has auth.throttled for 127.0.0.1.
 32#
 33# With the fix, trusted: GITBAY_SSH is git@169.254.1.2, the logins are
 34# denied and arrive from the host's public address, auth.throttled
 35# names that address, the build succeeds and the runner keeps polling.
 36# Untrusted: every login is refused by the builds table before it
 37# reaches sshd, and no auth.* entry comes from the build at all.
 38#
 39# The script also requires lines in the build log, so a missing ssh or
 40# bash in the image, or a table that matches nothing, fails rather than
 41# passes. Trusted: "logins 12 denied" (every login reached sshd) and
 42# 10.0.0.1:80 refused, not timed out. Untrusted: 169.254.1.2:22 and
 43# github.com:22 refused and "12 refused". The untrusted lines are the
 44# proof that pasta's sockets are in the build's cgroup: the uid table
 45# lets ci-runner reach both.
 46set -eu
 47
 48repo=${1:-}
 49[ -n "$repo" ] || { echo "usage: $0 <owner/name> [--untrusted]" >&2; exit 2; }
 50mode=trusted
 51[ "${2:-}" = --untrusted ] && mode=untrusted
 52host=${GITBAY_HOST:-gitbay.org}
 53account=${RUNNER_ACCOUNT:-ci}
 54job=flood260
 55
 56tmp=$(mktemp -d)
 57fp=
 58cleanup() {
 59    if [ -n "$fp" ]; then gitbay auth keys remove "$fp" >/dev/null || echo "remove key $fp by hand" >&2; fi
 60    fp=
 61    rm -rf "$tmp"
 62}
 63trap cleanup EXIT
 64trap 'cleanup; exit 130' INT TERM
 65
 66echo "==> an expired key"
 67ssh-keygen -q -t ed25519 -N '' -C auth-flood-260 -f "$tmp/key"
 68gitbay auth keys add --scope git --label auth-flood-260 --ttl 1s <"$tmp/key.pub" >/dev/null
 69fp=$(ssh-keygen -lf "$tmp/key.pub" | awk '{print $2}')
 70sleep 2
 71
 72if [ "$mode" = untrusted ]; then
 73    src="${repo%/*}/${repo#*/}-fork260"
 74    if ! gitbay repo show "$src" --json >/dev/null 2>&1; then
 75        echo "==> forking $repo to $src"
 76        gitbay repo fork "$repo" --name "${repo#*/}-fork260" >/dev/null
 77    fi
 78    branch="flood-260-$(date +%s)"
 79else
 80    src=$repo
 81    branch=main
 82fi
 83
 84echo "==> committing the $job job to $src ($branch)"
 85git clone -q "ssh://git@$host/$src.git" "$tmp/repo"
 86cd "$tmp/repo"
 87[ "$branch" = main ] || git checkout -q -b "$branch"
 88mkdir -p .gitbay
 89cp "$tmp/key" .gitbay/flood.key
 90cat >.gitbay/ci.yml <<EOF
 91jobs:
 92  $job:
 93    steps:
 94      - echo "GITBAY_SSH=\$GITBAY_SSH"
 95      - sh .gitbay/flood.sh
 96EOF
 97cat >.gitbay/flood.sh <<'EOF'
 98#!/bin/sh
 99# Written by deploy/runner-auth-flood-test.sh (#260).
100set -u
101sleep 60
102key=/tmp/flood.key
103cp .gitbay/flood.key "$key"
104chmod 600 "$key"
105dest=${GITBAY_SSH#*@}
106host=${dest%:*}
107port=${dest##*:}
108[ "$host" = "$dest" ] && port=22
109
110probe() {
111    timeout 5 bash -c "exec 3<>/dev/tcp/$1/$2" 2>/dev/null
112    case $? in
113    0) echo "open     $1:$2" ;;
114    124) echo "timeout  $1:$2" ;;
115    *) echo "refused  $1:$2" ;;
116    esac
117}
118getent hosts proxy.golang.org >/dev/null && echo "dns      ok" || echo "dns      failed"
119for t in "$host:22" "$host:80" "$host:443" "$host:2222" \
120    10.0.0.1:80 192.168.0.1:80 proxy.golang.org:443 github.com:22; do
121    probe "${t%:*}" "${t##*:}"
122done
123
124denied=0 refused=0 other=0 i=0
125while [ $i -lt 12 ]; do
126    i=$((i + 1))
127    out=$(ssh -F /dev/null -i "$key" -o IdentitiesOnly=yes -o BatchMode=yes \
128        -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ConnectTimeout=5 \
129        -p "$port" "git@$host" whoami 2>&1)
130    case $out in
131    *"Permission denied"*) denied=$((denied + 1)) ;;
132    *"Connection refused"*) refused=$((refused + 1)) ;;
133    *) other=$((other + 1)); echo "login $i: $out" ;;
134    esac
135done
136echo "logins   $denied denied, $refused refused, $other other"
137EOF
138git add .gitbay
139git commit -q -m "ci: auth flood test (#260)"
140git push -q origin "$branch"
141cd - >/dev/null
142
143if [ "$mode" = untrusted ]; then
144    gitbay mr create "$repo" --source "$src:$branch" --target main --title "#260 auth flood, untrusted" >/dev/null
145fi
146
147# One gitbay call per tick: the CLI shares one connection, and a burst of
148# logins is what the limiter is for.
149echo "==> waiting for the build"
150n=
151for _ in $(seq 1 12); do
152    sleep 5
153    n=$(gitbay build list "$repo" --job "$job" --limit 1 --json | jq -r '.data | (.items // .) | .[0].number // empty')
154    [ -n "$n" ] && break
155done
156[ -n "$n" ] || { echo "no $job build queued on $repo" >&2; exit 1; }
157echo "   build $n"
158status=
159for _ in $(seq 1 60); do
160    sleep 10
161    status=$(gitbay build show "$repo" "$n" --json | jq -r .data.status)
162    case $status in success | failure) break ;; esac
163done
164echo "   $status"
165
166echo "==> the runner after the build"
167seen() { gitbay admin runners --json | jq -r --arg a "$account" '[.data.runners[] | select(.username == $a) | .last_seen] | max // empty'; }
168first=$(seen)
169sleep 15
170second=$(seen)
171echo "   $account last seen $first, then $second"
172
173echo "==> build log"
174log=$(gitbay build log "$repo" "$n")
175printf '%s\n' "$log" | sed -n '/dns /,$p'
176
177echo "==> auth audit, last 15 minutes"
178gitbay audit --action auth. --since 15m --json |
179    jq -r '.data[] | "\(.action) \(.data | fromjson | .ip // "-")"' | sort | uniq -c
180
181echo
182fail=0
183[ "$status" = success ] || { echo "FAIL: build $n is $status; the runner could not report it"; fail=1; }
184[ -n "$second" ] && [ "$second" != "$first" ] || { echo "FAIL: the runner did not poll in 15 seconds after the build"; fail=1; }
185if gitbay audit --action auth.throttled --since 15m --json | jq -e '.data[] | select((.data | fromjson | .ip) == "127.0.0.1")' >/dev/null; then
186    echo "FAIL: 127.0.0.1, the runner's address, was throttled"
187    fail=1
188fi
189need() {
190    printf '%s\n' "$log" | grep -Eq "$1" || { echo "FAIL: the build log lacks \"$2\""; fail=1; }
191}
192if [ "$mode" = trusted ]; then
193    need 'logins +12 denied' "logins 12 denied"
194    need 'refused +10\.0\.0\.1:80( |$)' "refused 10.0.0.1:80"
195else
196    need 'refused +169\.254\.1\.2:22( |$)' "refused 169.254.1.2:22"
197    need 'refused +github\.com:22( |$)' "refused github.com:22"
198    need '12 refused' "12 refused"
199fi
200[ $fail = 0 ] && echo "PASS ($mode): the build's failed logins did not lock the runner out"
201exit $fail