e2e/runnerstop_test.go
220 lines · 8096 bytes
1package e2e
2
3import (
4 "bytes"
5 "encoding/json"
6 "fmt"
7 "os"
8 "os/exec"
9 "path/filepath"
10 "regexp"
11 "strings"
12 "syscall"
13 "testing"
14 "time"
15)
16
17// The runner drains on SIGTERM: the build in flight runs to its end and
18// is reported, and nothing more is claimed (#179). The drain shipped
19// broken once because only the process was tested and never the unit:
20// systemd's default KillMode signals every process in the cgroup, the
21// step and the log session included, so the runner drained a build
22// whose steps were already dead. deploy/gitbay-runner.override.conf
23// sets KillMode=mixed for that reason, and the systemd test below runs
24// the runner as a real transient unit under both modes (#184).
25
26// stopFixture starts an instance with one repository whose single job
27// sleeps long enough to be stopped mid-step, and returns the admin key
28// that both pushes and runs the runner.
29func stopFixture(t *testing.T) (*instance, string) {
30 t.Helper()
31 inst := startInstance(t)
32 inst.runner = buildRunner(t)
33 key := inst.newKey(t, "alice")
34 inst.admin(t, "admin", "user", "create", "alice", "--key", key+".pub", "--admin")
35 if _, errOut, code := inst.ssh(t, key, "", "repo", "create", "alice/app"); code != 0 {
36 t.Fatalf("repo create: %s", errOut)
37 }
38 work := t.TempDir()
39 env := inst.gitEnv(key)
40 mustGit(t, work, env, "clone", inst.sshURL("alice/app"), "w")
41 dir := filepath.Join(work, "w")
42 os.MkdirAll(filepath.Join(dir, ".gitbay"), 0o755)
43 os.WriteFile(filepath.Join(dir, ".gitbay", "ci.yml"),
44 []byte("jobs:\n slow:\n steps:\n - sleep 4; echo drained\n"), 0o644)
45 mustGit(t, dir, env, "checkout", "-q", "-b", "main")
46 mustGit(t, dir, env, "add", ".")
47 mustGit(t, dir, env, "commit", "-q", "-m", "ci")
48 mustGit(t, dir, env, "push", "-q", "origin", "main")
49 return inst, key
50}
51
52// runnerArgs is the command line the runner tests use, minus the binary.
53func (i *instance) runnerArgs(t *testing.T, key string) []string {
54 opts := fmt.Sprintf("-p %d -i %s -o IdentitiesOnly=yes -o StrictHostKeyChecking=no -o UserKnownHostsFile=%s -o BatchMode=yes",
55 i.port, key, filepath.Join(i.sshDir, "known_hosts"))
56 return []string{
57 "-poll", "200ms",
58 "-isolation", "none",
59 "-remote", "git@127.0.0.1",
60 "-ssh-opts", opts,
61 "-clone-base", fmt.Sprintf("ssh://git@127.0.0.1:%d", i.port),
62 "-workdir", t.TempDir(),
63 }
64}
65
66func (i *instance) buildStatus(t *testing.T, key string) string {
67 t.Helper()
68 out, _, _ := i.ssh(t, key, "", "build", "list", "alice/app", "--json")
69 var env struct {
70 Data []struct {
71 Status string `json:"status"`
72 } `json:"data"`
73 }
74 json.Unmarshal([]byte(out), &env)
75 if len(env.Data) == 0 {
76 return ""
77 }
78 return env.Data[0].Status
79}
80
81func TestRunnerDrainsOnSIGTERM(t *testing.T) {
82 inst, key := stopFixture(t)
83 cmd := exec.Command(inst.runner, inst.runnerArgs(t, key)...)
84 cmd.Env = append(os.Environ(), "XDG_CONFIG_HOME="+t.TempDir(), "GIT_CONFIG_NOSYSTEM=1", "GIT_CONFIG_GLOBAL=/dev/null")
85 var buf bytes.Buffer
86 cmd.Stdout, cmd.Stderr = &buf, &buf
87 if err := cmd.Start(); err != nil {
88 t.Fatal(err)
89 }
90 defer func() { cmd.Process.Kill(); cmd.Wait() }()
91
92 waitFor(t, "the build to be claimed", func() bool { return inst.buildStatus(t, key) == "running" })
93 // The step is asleep. Signal the runner alone, as KillMode=mixed does.
94 if err := cmd.Process.Signal(syscall.SIGTERM); err != nil {
95 t.Fatal(err)
96 }
97 exited := make(chan error, 1)
98 go func() { exited <- cmd.Wait() }()
99 select {
100 case err := <-exited:
101 if err != nil {
102 t.Fatalf("runner exited %v after SIGTERM; output:\n%s", err, buf.String())
103 }
104 case <-time.After(30 * time.Second):
105 t.Fatalf("runner did not exit within 30s of SIGTERM; output:\n%s", buf.String())
106 }
107 if !strings.Contains(buf.String(), "draining") {
108 t.Fatalf("runner did not announce the drain:\n%s", buf.String())
109 }
110 if st := inst.buildStatus(t, key); st != "success" {
111 t.Fatalf("build after drain is %q, want success; runner output:\n%s", st, buf.String())
112 }
113 if out, _, _ := inst.ssh(t, key, "", "build", "log", "alice/app", "1"); !strings.Contains(out, "drained") {
114 t.Fatalf("step did not run to its end:\n%s", out)
115 }
116}
117
118// The drop-in the runner runs under. Read here so a change to it fails a
119// test rather than the next production stop.
120func runnerDropIn(t *testing.T) string {
121 t.Helper()
122 b, err := os.ReadFile(filepath.Join("..", "deploy", "gitbay-runner.override.conf"))
123 if err != nil {
124 t.Fatal(err)
125 }
126 return string(b)
127}
128
129func dropInValue(conf, key string) string {
130 m := regexp.MustCompile(`(?m)^`+key+`=(.*)$`).FindAllStringSubmatch(conf, -1)
131 if len(m) == 0 {
132 return ""
133 }
134 return strings.TrimSpace(m[len(m)-1][1]) // the last assignment wins, as in systemd
135}
136
137// The unit must let the drain happen: the stop signal reaches the runner
138// alone, and the stop timeout outlasts a build plus the report retries.
139func TestRunnerDropInLetsTheDrainHappen(t *testing.T) {
140 conf := runnerDropIn(t)
141 if mode := dropInValue(conf, "KillMode"); mode != "mixed" {
142 t.Fatalf("KillMode=%q, want mixed: control-group signals the step and the log session with the runner", mode)
143 }
144 stop, err := time.ParseDuration(strings.ReplaceAll(dropInValue(conf, "TimeoutStopSec"), "min", "m"))
145 if err != nil {
146 t.Fatalf("TimeoutStopSec: %v", err)
147 }
148 m := regexp.MustCompile(`-timeout (\S+)`).FindStringSubmatch(dropInValue(conf, "ExecStart"))
149 if m == nil {
150 t.Fatal("ExecStart has no -timeout")
151 }
152 build, err := time.ParseDuration(m[1])
153 if err != nil {
154 t.Fatalf("-timeout: %v", err)
155 }
156 // Half a minute of report retries after the build's own limit (#179).
157 if stop < build+30*time.Second {
158 t.Fatalf("TimeoutStopSec %v cannot outlast a %v build and its report retries", stop, build)
159 }
160}
161
162// haveUserSystemd reports whether transient user units can be started
163// here: a Linux host with a systemd user manager for this account.
164func haveUserSystemd(t *testing.T) bool {
165 t.Helper()
166 if _, err := exec.LookPath("systemd-run"); err != nil {
167 return false
168 }
169 return exec.Command("systemd-run", "--user", "--quiet", "--wait", "--collect", "true").Run() == nil
170}
171
172// The runner as a transient unit, stopped by systemd. Under the drop-in's
173// KillMode the build in flight is reported a success; under systemd's
174// default it is not, which is the failure that shipped once.
175func TestRunnerDrainUnderSystemd(t *testing.T) {
176 if !haveUserSystemd(t) {
177 t.Skip("no systemd user manager")
178 }
179 mode := dropInValue(runnerDropIn(t), "KillMode")
180 for _, tc := range []struct {
181 mode string
182 drained bool
183 }{
184 {mode, true},
185 {"control-group", false},
186 } {
187 t.Run(tc.mode, func(t *testing.T) {
188 inst, key := stopFixture(t)
189 unit := fmt.Sprintf("gitbay-runner-test-%d-%s", os.Getpid(), tc.mode)
190 args := append([]string{"--user", "--quiet", "--collect", "--unit", unit,
191 "-p", "KillMode=" + tc.mode, "-p", "TimeoutStopSec=60",
192 "--setenv=GIT_CONFIG_NOSYSTEM=1", "--setenv=GIT_CONFIG_GLOBAL=/dev/null",
193 inst.runner}, inst.runnerArgs(t, key)...)
194 if out, err := exec.Command("systemd-run", args...).CombinedOutput(); err != nil {
195 t.Fatalf("systemd-run: %v\n%s", err, out)
196 }
197 defer exec.Command("systemctl", "--user", "kill", "-s", "SIGKILL", unit).Run()
198
199 waitFor(t, "the build to be claimed", func() bool { return inst.buildStatus(t, key) == "running" })
200 // systemctl stop returns when the unit is down: after the
201 // drain, or after the cgroup was signalled and emptied.
202 stop := exec.Command("systemctl", "--user", "stop", unit)
203 if out, err := stop.CombinedOutput(); err != nil {
204 t.Fatalf("systemctl stop: %v\n%s", err, out)
205 }
206 if tc.drained {
207 if st := inst.buildStatus(t, key); st != "success" {
208 t.Fatalf("KillMode=%s: build after stop is %q, want success", tc.mode, st)
209 }
210 return
211 }
212 // The step was signalled with the runner, so it cannot have
213 // finished: the runner reports a failure, or died before
214 // reporting and left the build running for the reaper.
215 if st := inst.buildStatus(t, key); st == "success" {
216 t.Fatalf("KillMode=%s: build reported success, so the stop test cannot tell the modes apart", tc.mode)
217 }
218 })
219 }
220}