b5ea7bcf97 · krz/gitbay

1

#!/bin/sh

2

# Scratch-repository test for #260: a build fails SSH logins while the

3

# runner works, and the runner must not be locked out with it.

4

#

5

# Run from a machine with an admin gitbay identity, after the Admin

6

# page's scratch procedure: the runner's key attached to the scratch

7

# repository and the runner scoped to it with -repos. The script

8

# replaces the repository's .gitbay/ci.yml.

9

#

10

#   deploy/runner-auth-flood-test.sh cmc/runner-scratch              # trusted: a push to main

11

#   deploy/runner-auth-flood-test.sh cmc/runner-scratch --untrusted  # a merge request from a fork

12

#

13

# The build's logins use a key registered with --ttl 1s and expired by

14

# the time the build runs. With registration open an unknown key is

15

# admitted to run register and never counts against the SSH auth

16

# limiter; an expired key counts (internal/sshd/sshd.go, authenticate).

17

# The key is removed from the account when the script exits.

18

#

19

# The step probes what the build reaches, then makes 12 logins without

20

# pause, so the limiter (ssh_auth_rate, 10 a minute per address) locks

21

# the address those logins come from for most of the next minute. The

22

# runner reports the result right after the step; its report retries

23

# for half a minute.

24

#

25

# Before #260 a build reached the host's loopback through pasta, and its

26

# logins arrived from 127.0.0.1, where the runner polls: the report is

27

# refused ("too many authentication attempts" in the runner's journal),

28

# the build is failed by the server two minutes after its log stream

29

# ended, the runner's last-seen stops advancing for up to a minute, and

30

# the audit log has auth.throttled for 127.0.0.1.

31

#

32

# With the fix, trusted: GITBAY_SSH is git@169.254.1.2, the logins are

33

# denied and arrive from the host's public address, auth.throttled

34

# names that address, the build succeeds and the runner keeps polling.

35

# Untrusted: every login is refused by the builds table before it

36

# reaches sshd, and no auth.* entry comes from the build at all.

37

set -eu

38

39

repo=${1:-}

40

[ -n "$repo" ] || { echo "usage: $0 <owner/name> [--untrusted]" >&2; exit 2; }

41

mode=trusted

42

[ "${2:-}" = --untrusted ] && mode=untrusted

43

host=${GITBAY_HOST:-gitbay.org}

44

account=${RUNNER_ACCOUNT:-ci}

45

job=flood260

46

47

tmp=$(mktemp -d)

48

fp=

49

cleanup() {

50

    if [ -n "$fp" ]; then gitbay keys remove "$fp" >/dev/null || echo "remove key $fp by hand" >&2; fi

51

    rm -rf "$tmp"

52

}

53

trap cleanup EXIT

54

55

echo "==> an expired key"

56

ssh-keygen -q -t ed25519 -N '' -C auth-flood-260 -f "$tmp/key"

57

gitbay keys add --label auth-flood-260 --ttl 1s <"$tmp/key.pub" >/dev/null

58

fp=$(ssh-keygen -lf "$tmp/key.pub" | awk '{print $2}')

59

sleep 2

60

61

if [ "$mode" = untrusted ]; then

62

    src="${repo%/*}/${repo#*/}-fork260"

63

    if ! gitbay repo show "$src" --json >/dev/null 2>&1; then

64

        echo "==> forking $repo to $src"

65

        gitbay repo fork "$repo" --name "${repo#*/}-fork260" >/dev/null

66

fi

67

    branch="flood-260-$(date +%s)"

68

else

69

    src=$repo

70

branch=main

71

fi

72

73

echo "==> committing the $job job to $src ($branch)"

74

git clone -q "ssh://git@$host/$src.git" "$tmp/repo"

75

cd "$tmp/repo"

76

[ "$branch" = main ] || git checkout -q -b "$branch"

77

mkdir -p .gitbay

78

cp "$tmp/key" .gitbay/flood.key

79

cat >.gitbay/ci.yml <<EOF

80

jobs:

81

$job:

82

steps:

83

- echo "GITBAY_SSH=\$GITBAY_SSH"

84

- sh .gitbay/flood.sh

85

EOF

86

cat >.gitbay/flood.sh <<'EOF'

87

#!/bin/sh

88

# Written by deploy/runner-auth-flood-test.sh (#260).

89

set -u

90

key=/tmp/flood.key

91

cp .gitbay/flood.key "$key"

92

chmod 600 "$key"

93

dest=${GITBAY_SSH#*@}

94

host=${dest%:*}

95

port=${dest##*:}

96

[ "$host" = "$dest" ] && port=22

97

98

probe() {

99

timeout 5 bash -c "exec 3<>/dev/tcp/$1/$2" 2>/dev/null

100

case $? in

101

0) echo "open $1:$2" ;;

102

124) echo "timeout $1:$2" ;;

103

*) echo "refused $1:$2" ;;

104

esac

105

}

106

getent hosts proxy.golang.org >/dev/null && echo "dns      ok" || echo "dns      failed"

107

for t in 127.0.0.1:22 127.0.0.1:2222 "$host:22" "$host:80" "$host:443" "$host:2222" \

108

10.0.0.1:80 192.168.0.1:80 proxy.golang.org:443 github.com:22; do

109

probe "${t%:*}" "${t##*:}"

110

done

111

112

denied=0 refused=0 other=0 i=0

113

while [ $i -lt 12 ]; do

114

i=$((i + 1))

115

out=$(ssh -F /dev/null -i "$key" -o IdentitiesOnly=yes -o BatchMode=yes \

116

        -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ConnectTimeout=5 \

117

-p "$port" "git@$host" whoami 2>&1)

118

case $out in

119

*"Permission denied"*) denied=$((denied + 1)) ;;

120

*"Connection refused"*) refused=$((refused + 1)) ;;

121

*) other=$((other + 1)); echo "login $i: $out" ;;

122

esac

123

done

124

echo "logins $denied denied, $refused refused, $other other"

125

EOF

126

git add .gitbay

127

git commit -q -m "ci: auth flood test (#260)"

128

git push -q origin "$branch"

129

cd - >/dev/null

130

131

if [ "$mode" = untrusted ]; then

132

    gitbay mr create "$repo" --source "$src:$branch" --target main --title "#260 auth flood, untrusted" >/dev/null

133

fi

134

135

# One gitbay call per tick: the CLI shares one connection, and a burst of

136

# logins is what the limiter is for.

137

echo "==> waiting for the build"

138

n=

139

for _ in $(seq 1 12); do

140

sleep 5

141

    n=$(gitbay build list "$repo" --job "$job" --limit 1 --json | jq -r '.data | (.items // .) | .[0].number // empty')

142

    [ -n "$n" ] && break

143

done

144

[ -n "$n" ] || { echo "no $job build queued on $repo" >&2; exit 1; }

145

echo "   build $n"

146

status=

147

for _ in $(seq 1 60); do

148

sleep 10

149

    status=$(gitbay build show "$repo" "$n" --json | jq -r .data.status)

150

    case $status in success | failure) break ;; esac

151

done

152

echo "   $status"

153

154

echo "==> the runner after the build"

155

seen() { gitbay admin runners --json | jq -r --arg a "$account" '[.data.runners[] | select(.username == $a) | .last_seen] | max // empty'; }

156

first=$(seen)

157

sleep 15

158

second=$(seen)

159

echo "   $account last seen $first, then $second"

160

161

echo "==> build log"

162

gitbay build log "$repo" "$n" | sed -n '/^dns /,$p'

163

164

echo "==> auth audit, last 15 minutes"

165

gitbay audit --action auth. --since 15m --json |

166

    jq -r '.data[] | "\(.action) \(.data | fromjson | .ip // "-")"' | sort | uniq -c

167

168

echo

169

fail=0

170

[ "$status" = success ] || { echo "FAIL: build $n is $status; the runner could not report it"; fail=1; }

171

[ -n "$second" ] && [ "$second" != "$first" ] || { echo "FAIL: the runner did not poll in 15 seconds after the build"; fail=1; }

172

if gitbay audit --action auth.throttled --since 15m --json | jq -e '.data[] | select((.data | fromjson | .ip) == "127.0.0.1")' >/dev/null; then

173

echo "FAIL: 127.0.0.1, the runner's address, was throttled"

174

    fail=1

175

fi

176

[ $fail = 0 ] && echo "PASS ($mode): the build's failed logins did not lock the runner out"

177

exit $fail

@@ -0,0 +1,177 @@
	1	#!/bin/sh
	2	# Scratch-repository test for #260: a build fails SSH logins while the
	3	# runner works, and the runner must not be locked out with it.
	4	#
	5	# Run from a machine with an admin gitbay identity, after the Admin
	6	# page's scratch procedure: the runner's key attached to the scratch
	7	# repository and the runner scoped to it with -repos. The script
	8	# replaces the repository's .gitbay/ci.yml.
	9	#
	10	# deploy/runner-auth-flood-test.sh cmc/runner-scratch # trusted: a push to main
	11	# deploy/runner-auth-flood-test.sh cmc/runner-scratch --untrusted # a merge request from a fork
	12	#
	13	# The build's logins use a key registered with --ttl 1s and expired by
	14	# the time the build runs. With registration open an unknown key is
	15	# admitted to run register and never counts against the SSH auth
	16	# limiter; an expired key counts (internal/sshd/sshd.go, authenticate).
	17	# The key is removed from the account when the script exits.
	18	#
	19	# The step probes what the build reaches, then makes 12 logins without
	20	# pause, so the limiter (ssh_auth_rate, 10 a minute per address) locks
	21	# the address those logins come from for most of the next minute. The
	22	# runner reports the result right after the step; its report retries
	23	# for half a minute.
	24	#
	25	# Before #260 a build reached the host's loopback through pasta, and its
	26	# logins arrived from 127.0.0.1, where the runner polls: the report is
	27	# refused ("too many authentication attempts" in the runner's journal),
	28	# the build is failed by the server two minutes after its log stream
	29	# ended, the runner's last-seen stops advancing for up to a minute, and
	30	# the audit log has auth.throttled for 127.0.0.1.
	31	#
	32	# With the fix, trusted: GITBAY_SSH is git@169.254.1.2, the logins are
	33	# denied and arrive from the host's public address, auth.throttled
	34	# names that address, the build succeeds and the runner keeps polling.
	35	# Untrusted: every login is refused by the builds table before it
	36	# reaches sshd, and no auth.* entry comes from the build at all.
	37	set -eu
	38
	39	repo=${1:-}
	40	[ -n "$repo" ] \|\| { echo "usage: $0 <owner/name> [--untrusted]" >&2; exit 2; }
	41	mode=trusted
	42	[ "${2:-}" = --untrusted ] && mode=untrusted
	43	host=${GITBAY_HOST:-gitbay.org}
	44	account=${RUNNER_ACCOUNT:-ci}
	45	job=flood260
	46
	47	tmp=$(mktemp -d)
	48	fp=
	49	cleanup() {
	50	if [ -n "$fp" ]; then gitbay keys remove "$fp" >/dev/null \|\| echo "remove key $fp by hand" >&2; fi
	51	rm -rf "$tmp"
	52	}
	53	trap cleanup EXIT
	54
	55	echo "==> an expired key"
	56	ssh-keygen -q -t ed25519 -N '' -C auth-flood-260 -f "$tmp/key"
	57	gitbay keys add --label auth-flood-260 --ttl 1s <"$tmp/key.pub" >/dev/null
	58	fp=$(ssh-keygen -lf "$tmp/key.pub" \| awk '{print $2}')
	59	sleep 2
	60
	61	if [ "$mode" = untrusted ]; then
	62	src="${repo%/}/${repo#/}-fork260"
	63	if ! gitbay repo show "$src" --json >/dev/null 2>&1; then
	64	echo "==> forking $repo to $src"
	65	gitbay repo fork "$repo" --name "${repo#*/}-fork260" >/dev/null
	66	fi
	67	branch="flood-260-$(date +%s)"
	68	else
	69	src=$repo
	70	branch=main
	71	fi
	72
	73	echo "==> committing the $job job to $src ($branch)"
	74	git clone -q "ssh://git@$host/$src.git" "$tmp/repo"
	75	cd "$tmp/repo"
	76	[ "$branch" = main ] \|\| git checkout -q -b "$branch"
	77	mkdir -p .gitbay
	78	cp "$tmp/key" .gitbay/flood.key
	79	cat >.gitbay/ci.yml <<EOF
	80	jobs:
	81	$job:
	82	steps:
	83	- echo "GITBAY_SSH=\$GITBAY_SSH"
	84	- sh .gitbay/flood.sh
	85	EOF
	86	cat >.gitbay/flood.sh <<'EOF'
	87	#!/bin/sh
	88	# Written by deploy/runner-auth-flood-test.sh (#260).
	89	set -u
	90	key=/tmp/flood.key
	91	cp .gitbay/flood.key "$key"
	92	chmod 600 "$key"
	93	dest=${GITBAY_SSH#*@}
	94	host=${dest%:*}
	95	port=${dest##*:}
	96	[ "$host" = "$dest" ] && port=22
	97
	98	probe() {
	99	timeout 5 bash -c "exec 3<>/dev/tcp/$1/$2" 2>/dev/null
	100	case $? in
	101	0) echo "open $1:$2" ;;
	102	124) echo "timeout $1:$2" ;;
	103	*) echo "refused $1:$2" ;;
	104	esac
	105	}
	106	getent hosts proxy.golang.org >/dev/null && echo "dns ok" \|\| echo "dns failed"
	107	for t in 127.0.0.1:22 127.0.0.1:2222 "$host:22" "$host:80" "$host:443" "$host:2222" \
	108	10.0.0.1:80 192.168.0.1:80 proxy.golang.org:443 github.com:22; do
	109	probe "${t%:}" "${t##:}"
	110	done
	111
	112	denied=0 refused=0 other=0 i=0
	113	while [ $i -lt 12 ]; do
	114	i=$((i + 1))
	115	out=$(ssh -F /dev/null -i "$key" -o IdentitiesOnly=yes -o BatchMode=yes \
	116	-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ConnectTimeout=5 \
	117	-p "$port" "git@$host" whoami 2>&1)
	118	case $out in
	119	"Permission denied") denied=$((denied + 1)) ;;
	120	"Connection refused") refused=$((refused + 1)) ;;
	121	*) other=$((other + 1)); echo "login $i: $out" ;;
	122	esac
	123	done
	124	echo "logins $denied denied, $refused refused, $other other"
	125	EOF
	126	git add .gitbay
	127	git commit -q -m "ci: auth flood test (#260)"
	128	git push -q origin "$branch"
	129	cd - >/dev/null
	130
	131	if [ "$mode" = untrusted ]; then
	132	gitbay mr create "$repo" --source "$src:$branch" --target main --title "#260 auth flood, untrusted" >/dev/null
	133	fi
	134
	135	# One gitbay call per tick: the CLI shares one connection, and a burst of
	136	# logins is what the limiter is for.
	137	echo "==> waiting for the build"
	138	n=
	139	for _ in $(seq 1 12); do
	140	sleep 5
	141	n=$(gitbay build list "$repo" --job "$job" --limit 1 --json \| jq -r '.data \| (.items // .) \| .[0].number // empty')
	142	[ -n "$n" ] && break
	143	done
	144	[ -n "$n" ] \|\| { echo "no $job build queued on $repo" >&2; exit 1; }
	145	echo " build $n"
	146	status=
	147	for _ in $(seq 1 60); do
	148	sleep 10
	149	status=$(gitbay build show "$repo" "$n" --json \| jq -r .data.status)
	150	case $status in success \| failure) break ;; esac
	151	done
	152	echo " $status"
	153
	154	echo "==> the runner after the build"
	155	seen() { gitbay admin runners --json \| jq -r --arg a "$account" '[.data.runners[] \| select(.username == $a) \| .last_seen] \| max // empty'; }
	156	first=$(seen)
	157	sleep 15
	158	second=$(seen)
	159	echo " $account last seen $first, then $second"
	160
	161	echo "==> build log"
	162	gitbay build log "$repo" "$n" \| sed -n '/^dns /,$p'
	163
	164	echo "==> auth audit, last 15 minutes"
	165	gitbay audit --action auth. --since 15m --json \|
	166	jq -r '.data[] \| "\(.action) \(.data \| fromjson \| .ip // "-")"' \| sort \| uniq -c
	167
	168	echo
	169	fail=0
	170	[ "$status" = success ] \|\| { echo "FAIL: build $n is $status; the runner could not report it"; fail=1; }
	171	[ -n "$second" ] && [ "$second" != "$first" ] \|\| { echo "FAIL: the runner did not poll in 15 seconds after the build"; fail=1; }
	172	if gitbay audit --action auth.throttled --since 15m --json \| jq -e '.data[] \| select((.data \| fromjson \| .ip) == "127.0.0.1")' >/dev/null; then
	173	echo "FAIL: 127.0.0.1, the runner's address, was throttled"
	174	fail=1
	175	fi
	176	[ $fail = 0 ] && echo "PASS ($mode): the build's failed logins did not lock the runner out"
	177	exit $fail

Commit b5ea7bcf97

Commit `b5ea7bcf97`