krz/gitbay

stop tests for the runner's drain !352

merged merged by cmc on 2026-09-08 06:55 UTC · krz/gitbay:runner-stop-test into main

2 files changed, +227 −1

Layout: unified · split

.gitbay/wiki/Admin.org +7 −1
@@ -511,7 +511,13 @@ runner drains a build that is already dead. So =make deploy-runner=
511waits for a running build rather than orphaning it, and a build's result 511waits for a running build rather than orphaning it, and a build's result
512is retried for half a minute if gitbayd is restarting at that moment. A 512is retried for half a minute if gitbayd is restarting at that moment. A
513second SIGTERM ends the runner at once, abandoning the build to the 513second SIGTERM ends the runner at once, abandoning the build to the
514reaper. 514reaper. The suite checks all three: =TestRunnerDrainsOnSIGTERM= signals
515the process, =TestRunnerDropInLetsTheDrainHappen= reads the drop-in's
516=KillMode= and =TimeoutStopSec=, and =TestRunnerDrainUnderSystemd= runs
517the runner as a transient user unit under =systemd-run= and stops it
518under both kill modes. That last one needs a systemd user manager, so it
519skips in the container CI runs in; run it on a Linux host with
520=go test ./e2e -run TestRunnerDrainUnderSystemd -v=.
515 521
516*Validate podman mode on a scratch repository before pointing the runner 522*Validate podman mode on a scratch repository before pointing the runner
517at real ones.* Every deploy that switched the whole instance to 523at real ones.* Every deploy that switched the whole instance to
e2e/runnerstop_test.go added +220
@@ -0,0 +1,220 @@
1package e2e
2
3import (
4 "bytes"
5 "encoding/json"
6 "fmt"
7 "os"
8 "os/exec"
9 "path/filepath"
10 "regexp"
11 "strings"
12 "syscall"
13 "testing"
14 "time"
15)
16
17// The runner drains on SIGTERM: the build in flight runs to its end and
18// is reported, and nothing more is claimed (#179). The drain shipped
19// broken once because only the process was tested and never the unit:
20// systemd's default KillMode signals every process in the cgroup, the
21// step and the log session included, so the runner drained a build
22// whose steps were already dead. deploy/gitbay-runner.override.conf
23// sets KillMode=mixed for that reason, and the systemd test below runs
24// the runner as a real transient unit under both modes (#184).
25
26// stopFixture starts an instance with one repository whose single job
27// sleeps long enough to be stopped mid-step, and returns the admin key
28// that both pushes and runs the runner.
29func stopFixture(t *testing.T) (*instance, string) {
30 t.Helper()
31 inst := startInstance(t)
32 inst.runner = buildRunner(t)
33 key := inst.newKey(t, "alice")
34 inst.admin(t, "admin", "user", "create", "alice", "--key", key+".pub", "--admin")
35 if _, errOut, code := inst.ssh(t, key, "", "repo", "create", "alice/app"); code != 0 {
36 t.Fatalf("repo create: %s", errOut)
37 }
38 work := t.TempDir()
39 env := inst.gitEnv(key)
40 mustGit(t, work, env, "clone", inst.sshURL("alice/app"), "w")
41 dir := filepath.Join(work, "w")
42 os.MkdirAll(filepath.Join(dir, ".gitbay"), 0o755)
43 os.WriteFile(filepath.Join(dir, ".gitbay", "ci.yml"),
44 []byte("jobs:\n slow:\n steps:\n - sleep 4; echo drained\n"), 0o644)
45 mustGit(t, dir, env, "checkout", "-q", "-b", "main")
46 mustGit(t, dir, env, "add", ".")
47 mustGit(t, dir, env, "commit", "-q", "-m", "ci")
48 mustGit(t, dir, env, "push", "-q", "origin", "main")
49 return inst, key
50}
51
52// runnerArgs is the command line the runner tests use, minus the binary.
53func (i *instance) runnerArgs(t *testing.T, key string) []string {
54 opts := fmt.Sprintf("-p %d -i %s -o IdentitiesOnly=yes -o StrictHostKeyChecking=no -o UserKnownHostsFile=%s -o BatchMode=yes",
55 i.port, key, filepath.Join(i.sshDir, "known_hosts"))
56 return []string{
57 "-poll", "200ms",
58 "-isolation", "none",
59 "-remote", "git@127.0.0.1",
60 "-ssh-opts", opts,
61 "-clone-base", fmt.Sprintf("ssh://git@127.0.0.1:%d", i.port),
62 "-workdir", t.TempDir(),
63 }
64}
65
66func (i *instance) buildStatus(t *testing.T, key string) string {
67 t.Helper()
68 out, _, _ := i.ssh(t, key, "", "build", "list", "alice/app", "--json")
69 var env struct {
70 Data []struct {
71 Status string `json:"status"`
72 } `json:"data"`
73 }
74 json.Unmarshal([]byte(out), &env)
75 if len(env.Data) == 0 {
76 return ""
77 }
78 return env.Data[0].Status
79}
80
81func TestRunnerDrainsOnSIGTERM(t *testing.T) {
82 inst, key := stopFixture(t)
83 cmd := exec.Command(inst.runner, inst.runnerArgs(t, key)...)
84 cmd.Env = append(os.Environ(), "GIT_CONFIG_NOSYSTEM=1", "GIT_CONFIG_GLOBAL=/dev/null")
85 var buf bytes.Buffer
86 cmd.Stdout, cmd.Stderr = &buf, &buf
87 if err := cmd.Start(); err != nil {
88 t.Fatal(err)
89 }
90 defer func() { cmd.Process.Kill(); cmd.Wait() }()
91
92 waitFor(t, "the build to be claimed", func() bool { return inst.buildStatus(t, key) == "running" })
93 // The step is asleep. Signal the runner alone, as KillMode=mixed does.
94 if err := cmd.Process.Signal(syscall.SIGTERM); err != nil {
95 t.Fatal(err)
96 }
97 exited := make(chan error, 1)
98 go func() { exited <- cmd.Wait() }()
99 select {
100 case err := <-exited:
101 if err != nil {
102 t.Fatalf("runner exited %v after SIGTERM; output:\n%s", err, buf.String())
103 }
104 case <-time.After(30 * time.Second):
105 t.Fatalf("runner did not exit within 30s of SIGTERM; output:\n%s", buf.String())
106 }
107 if !strings.Contains(buf.String(), "draining") {
108 t.Fatalf("runner did not announce the drain:\n%s", buf.String())
109 }
110 if st := inst.buildStatus(t, key); st != "success" {
111 t.Fatalf("build after drain is %q, want success; runner output:\n%s", st, buf.String())
112 }
113 if out, _, _ := inst.ssh(t, key, "", "build", "log", "alice/app", "1"); !strings.Contains(out, "drained") {
114 t.Fatalf("step did not run to its end:\n%s", out)
115 }
116}
117
118// The drop-in the runner runs under. Read here so a change to it fails a
119// test rather than the next production stop.
120func runnerDropIn(t *testing.T) string {
121 t.Helper()
122 b, err := os.ReadFile(filepath.Join("..", "deploy", "gitbay-runner.override.conf"))
123 if err != nil {
124 t.Fatal(err)
125 }
126 return string(b)
127}
128
129func dropInValue(conf, key string) string {
130 m := regexp.MustCompile(`(?m)^`+key+`=(.*)$`).FindAllStringSubmatch(conf, -1)
131 if len(m) == 0 {
132 return ""
133 }
134 return strings.TrimSpace(m[len(m)-1][1]) // the last assignment wins, as in systemd
135}
136
137// The unit must let the drain happen: the stop signal reaches the runner
138// alone, and the stop timeout outlasts a build plus the report retries.
139func TestRunnerDropInLetsTheDrainHappen(t *testing.T) {
140 conf := runnerDropIn(t)
141 if mode := dropInValue(conf, "KillMode"); mode != "mixed" {
142 t.Fatalf("KillMode=%q, want mixed: control-group signals the step and the log session with the runner", mode)
143 }
144 stop, err := time.ParseDuration(strings.ReplaceAll(dropInValue(conf, "TimeoutStopSec"), "min", "m"))
145 if err != nil {
146 t.Fatalf("TimeoutStopSec: %v", err)
147 }
148 m := regexp.MustCompile(`-timeout (\S+)`).FindStringSubmatch(dropInValue(conf, "ExecStart"))
149 if m == nil {
150 t.Fatal("ExecStart has no -timeout")
151 }
152 build, err := time.ParseDuration(m[1])
153 if err != nil {
154 t.Fatalf("-timeout: %v", err)
155 }
156 // Half a minute of report retries after the build's own limit (#179).
157 if stop < build+30*time.Second {
158 t.Fatalf("TimeoutStopSec %v cannot outlast a %v build and its report retries", stop, build)
159 }
160}
161
162// haveUserSystemd reports whether transient user units can be started
163// here: a Linux host with a systemd user manager for this account.
164func haveUserSystemd(t *testing.T) bool {
165 t.Helper()
166 if _, err := exec.LookPath("systemd-run"); err != nil {
167 return false
168 }
169 return exec.Command("systemd-run", "--user", "--quiet", "--wait", "--collect", "true").Run() == nil
170}
171
172// The runner as a transient unit, stopped by systemd. Under the drop-in's
173// KillMode the build in flight is reported a success; under systemd's
174// default it is not, which is the failure that shipped once.
175func TestRunnerDrainUnderSystemd(t *testing.T) {
176 if !haveUserSystemd(t) {
177 t.Skip("no systemd user manager")
178 }
179 mode := dropInValue(runnerDropIn(t), "KillMode")
180 for _, tc := range []struct {
181 mode string
182 drained bool
183 }{
184 {mode, true},
185 {"control-group", false},
186 } {
187 t.Run(tc.mode, func(t *testing.T) {
188 inst, key := stopFixture(t)
189 unit := fmt.Sprintf("gitbay-runner-test-%d-%s", os.Getpid(), tc.mode)
190 args := append([]string{"--user", "--quiet", "--collect", "--unit", unit,
191 "-p", "KillMode=" + tc.mode, "-p", "TimeoutStopSec=60",
192 "--setenv=GIT_CONFIG_NOSYSTEM=1", "--setenv=GIT_CONFIG_GLOBAL=/dev/null",
193 inst.runner}, inst.runnerArgs(t, key)...)
194 if out, err := exec.Command("systemd-run", args...).CombinedOutput(); err != nil {
195 t.Fatalf("systemd-run: %v\n%s", err, out)
196 }
197 defer exec.Command("systemctl", "--user", "kill", "-s", "SIGKILL", unit).Run()
198
199 waitFor(t, "the build to be claimed", func() bool { return inst.buildStatus(t, key) == "running" })
200 // systemctl stop returns when the unit is down: after the
201 // drain, or after the cgroup was signalled and emptied.
202 stop := exec.Command("systemctl", "--user", "stop", unit)
203 if out, err := stop.CombinedOutput(); err != nil {
204 t.Fatalf("systemctl stop: %v\n%s", err, out)
205 }
206 if tc.drained {
207 if st := inst.buildStatus(t, key); st != "success" {
208 t.Fatalf("KillMode=%s: build after stop is %q, want success", tc.mode, st)
209 }
210 return
211 }
212 // The step was signalled with the runner, so it cannot have
213 // finished: the runner reports a failure, or died before
214 // reporting and left the build running for the reaper.
215 if st := inst.buildStatus(t, key); st == "success" {
216 t.Fatalf("KillMode=%s: build reported success, so the stop test cannot tell the modes apart", tc.mode)
217 }
218 })
219 }
220}