Commit b5ea7bcf97
Verified · cmc
Layout: unified · split
deploy/runner-auth-flood-test.sh added +177
| @@ -0,0 +1,177 @@ | ||
| 1 | #!/bin/sh | |
| 2 | # Scratch-repository test for #260: a build fails SSH logins while the | |
| 3 | # runner works, and the runner must not be locked out with it. | |
| 4 | # | |
| 5 | # Run from a machine with an admin gitbay identity, after the Admin | |
| 6 | # page's scratch procedure: the runner's key attached to the scratch | |
| 7 | # repository and the runner scoped to it with -repos. The script | |
| 8 | # replaces the repository's .gitbay/ci.yml. | |
| 9 | # | |
| 10 | # deploy/runner-auth-flood-test.sh cmc/runner-scratch # trusted: a push to main | |
| 11 | # deploy/runner-auth-flood-test.sh cmc/runner-scratch --untrusted # a merge request from a fork | |
| 12 | # | |
| 13 | # The build's logins use a key registered with --ttl 1s and expired by | |
| 14 | # the time the build runs. With registration open an unknown key is | |
| 15 | # admitted to run register and never counts against the SSH auth | |
| 16 | # limiter; an expired key counts (internal/sshd/sshd.go, authenticate). | |
| 17 | # The key is removed from the account when the script exits. | |
| 18 | # | |
| 19 | # The step probes what the build reaches, then makes 12 logins without | |
| 20 | # pause, so the limiter (ssh_auth_rate, 10 a minute per address) locks | |
| 21 | # the address those logins come from for most of the next minute. The | |
| 22 | # runner reports the result right after the step; its report retries | |
| 23 | # for half a minute. | |
| 24 | # | |
| 25 | # Before #260 a build reached the host's loopback through pasta, and its | |
| 26 | # logins arrived from 127.0.0.1, where the runner polls: the report is | |
| 27 | # refused ("too many authentication attempts" in the runner's journal), | |
| 28 | # the build is failed by the server two minutes after its log stream | |
| 29 | # ended, the runner's last-seen stops advancing for up to a minute, and | |
| 30 | # the audit log has auth.throttled for 127.0.0.1. | |
| 31 | # | |
| 32 | # With the fix, trusted: GITBAY_SSH is git@169.254.1.2, the logins are | |
| 33 | # denied and arrive from the host's public address, auth.throttled | |
| 34 | # names that address, the build succeeds and the runner keeps polling. | |
| 35 | # Untrusted: every login is refused by the builds table before it | |
| 36 | # reaches sshd, and no auth.* entry comes from the build at all. | |
| 37 | set -eu | |
| 38 | ||
| 39 | repo=${1:-} | |
| 40 | [ -n "$repo" ] || { echo "usage: $0 <owner/name> [--untrusted]" >&2; exit 2; } | |
| 41 | mode=trusted | |
| 42 | [ "${2:-}" = --untrusted ] && mode=untrusted | |
| 43 | host=${GITBAY_HOST:-gitbay.org} | |
| 44 | account=${RUNNER_ACCOUNT:-ci} | |
| 45 | job=flood260 | |
| 46 | ||
| 47 | tmp=$(mktemp -d) | |
| 48 | fp= | |
| 49 | cleanup() { | |
| 50 | if [ -n "$fp" ]; then gitbay keys remove "$fp" >/dev/null || echo "remove key $fp by hand" >&2; fi | |
| 51 | rm -rf "$tmp" | |
| 52 | } | |
| 53 | trap cleanup EXIT | |
| 54 | ||
| 55 | echo "==> an expired key" | |
| 56 | ssh-keygen -q -t ed25519 -N '' -C auth-flood-260 -f "$tmp/key" | |
| 57 | gitbay keys add --label auth-flood-260 --ttl 1s <"$tmp/key.pub" >/dev/null | |
| 58 | fp=$(ssh-keygen -lf "$tmp/key.pub" | awk '{print $2}') | |
| 59 | sleep 2 | |
| 60 | ||
| 61 | if [ "$mode" = untrusted ]; then | |
| 62 | src="${repo%/*}/${repo#*/}-fork260" | |
| 63 | if ! gitbay repo show "$src" --json >/dev/null 2>&1; then | |
| 64 | echo "==> forking $repo to $src" | |
| 65 | gitbay repo fork "$repo" --name "${repo#*/}-fork260" >/dev/null | |
| 66 | fi | |
| 67 | branch="flood-260-$(date +%s)" | |
| 68 | else | |
| 69 | src=$repo | |
| 70 | branch=main | |
| 71 | fi | |
| 72 | ||
| 73 | echo "==> committing the $job job to $src ($branch)" | |
| 74 | git clone -q "ssh://git@$host/$src.git" "$tmp/repo" | |
| 75 | cd "$tmp/repo" | |
| 76 | [ "$branch" = main ] || git checkout -q -b "$branch" | |
| 77 | mkdir -p .gitbay | |
| 78 | cp "$tmp/key" .gitbay/flood.key | |
| 79 | cat >.gitbay/ci.yml <<EOF | |
| 80 | jobs: | |
| 81 | $job: | |
| 82 | steps: | |
| 83 | - echo "GITBAY_SSH=\$GITBAY_SSH" | |
| 84 | - sh .gitbay/flood.sh | |
| 85 | EOF | |
| 86 | cat >.gitbay/flood.sh <<'EOF' | |
| 87 | #!/bin/sh | |
| 88 | # Written by deploy/runner-auth-flood-test.sh (#260). | |
| 89 | set -u | |
| 90 | key=/tmp/flood.key | |
| 91 | cp .gitbay/flood.key "$key" | |
| 92 | chmod 600 "$key" | |
| 93 | dest=${GITBAY_SSH#*@} | |
| 94 | host=${dest%:*} | |
| 95 | port=${dest##*:} | |
| 96 | [ "$host" = "$dest" ] && port=22 | |
| 97 | ||
| 98 | probe() { | |
| 99 | timeout 5 bash -c "exec 3<>/dev/tcp/$1/$2" 2>/dev/null | |
| 100 | case $? in | |
| 101 | 0) echo "open $1:$2" ;; | |
| 102 | 124) echo "timeout $1:$2" ;; | |
| 103 | *) echo "refused $1:$2" ;; | |
| 104 | esac | |
| 105 | } | |
| 106 | getent hosts proxy.golang.org >/dev/null && echo "dns ok" || echo "dns failed" | |
| 107 | for t in 127.0.0.1:22 127.0.0.1:2222 "$host:22" "$host:80" "$host:443" "$host:2222" \ | |
| 108 | 10.0.0.1:80 192.168.0.1:80 proxy.golang.org:443 github.com:22; do | |
| 109 | probe "${t%:*}" "${t##*:}" | |
| 110 | done | |
| 111 | ||
| 112 | denied=0 refused=0 other=0 i=0 | |
| 113 | while [ $i -lt 12 ]; do | |
| 114 | i=$((i + 1)) | |
| 115 | out=$(ssh -F /dev/null -i "$key" -o IdentitiesOnly=yes -o BatchMode=yes \ | |
| 116 | -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ConnectTimeout=5 \ | |
| 117 | -p "$port" "git@$host" whoami 2>&1) | |
| 118 | case $out in | |
| 119 | *"Permission denied"*) denied=$((denied + 1)) ;; | |
| 120 | *"Connection refused"*) refused=$((refused + 1)) ;; | |
| 121 | *) other=$((other + 1)); echo "login $i: $out" ;; | |
| 122 | esac | |
| 123 | done | |
| 124 | echo "logins $denied denied, $refused refused, $other other" | |
| 125 | EOF | |
| 126 | git add .gitbay | |
| 127 | git commit -q -m "ci: auth flood test (#260)" | |
| 128 | git push -q origin "$branch" | |
| 129 | cd - >/dev/null | |
| 130 | ||
| 131 | if [ "$mode" = untrusted ]; then | |
| 132 | gitbay mr create "$repo" --source "$src:$branch" --target main --title "#260 auth flood, untrusted" >/dev/null | |
| 133 | fi | |
| 134 | ||
| 135 | # One gitbay call per tick: the CLI shares one connection, and a burst of | |
| 136 | # logins is what the limiter is for. | |
| 137 | echo "==> waiting for the build" | |
| 138 | n= | |
| 139 | for _ in $(seq 1 12); do | |
| 140 | sleep 5 | |
| 141 | n=$(gitbay build list "$repo" --job "$job" --limit 1 --json | jq -r '.data | (.items // .) | .[0].number // empty') | |
| 142 | [ -n "$n" ] && break | |
| 143 | done | |
| 144 | [ -n "$n" ] || { echo "no $job build queued on $repo" >&2; exit 1; } | |
| 145 | echo " build $n" | |
| 146 | status= | |
| 147 | for _ in $(seq 1 60); do | |
| 148 | sleep 10 | |
| 149 | status=$(gitbay build show "$repo" "$n" --json | jq -r .data.status) | |
| 150 | case $status in success | failure) break ;; esac | |
| 151 | done | |
| 152 | echo " $status" | |
| 153 | ||
| 154 | echo "==> the runner after the build" | |
| 155 | seen() { gitbay admin runners --json | jq -r --arg a "$account" '[.data.runners[] | select(.username == $a) | .last_seen] | max // empty'; } | |
| 156 | first=$(seen) | |
| 157 | sleep 15 | |
| 158 | second=$(seen) | |
| 159 | echo " $account last seen $first, then $second" | |
| 160 | ||
| 161 | echo "==> build log" | |
| 162 | gitbay build log "$repo" "$n" | sed -n '/^dns /,$p' | |
| 163 | ||
| 164 | echo "==> auth audit, last 15 minutes" | |
| 165 | gitbay audit --action auth. --since 15m --json | | |
| 166 | jq -r '.data[] | "\(.action) \(.data | fromjson | .ip // "-")"' | sort | uniq -c | |
| 167 | ||
| 168 | echo | |
| 169 | fail=0 | |
| 170 | [ "$status" = success ] || { echo "FAIL: build $n is $status; the runner could not report it"; fail=1; } | |
| 171 | [ -n "$second" ] && [ "$second" != "$first" ] || { echo "FAIL: the runner did not poll in 15 seconds after the build"; fail=1; } | |
| 172 | if gitbay audit --action auth.throttled --since 15m --json | jq -e '.data[] | select((.data | fromjson | .ip) == "127.0.0.1")' >/dev/null; then | |
| 173 | echo "FAIL: 127.0.0.1, the runner's address, was throttled" | |
| 174 | fail=1 | |
| 175 | fi | |
| 176 | [ $fail = 0 ] && echo "PASS ($mode): the build's failed logins did not lock the runner out" | |
| 177 | exit $fail | |