e2e/ci_test.go
382 lines · 16253 bytes
1package e2e
2
3import (
4 "bytes"
5 "fmt"
6 "os"
7 "os/exec"
8 "path/filepath"
9 "strings"
10 "testing"
11 "time"
12)
13
14func buildRunner(t *testing.T) string {
15 t.Helper()
16 bin := filepath.Join(t.TempDir(), "gitbay-runner")
17 cmd := exec.Command("go", "build", "-o", bin, "gitbay.org/gitbay/cmd/gitbay-runner")
18 cmd.Dir = ".."
19 if out, err := cmd.CombinedOutput(); err != nil {
20 t.Fatalf("build gitbay-runner: %v\n%s", err, out)
21 }
22 return bin
23}
24
25// runnerOnce processes at most one pending build with the given key.
26func (i *instance) runnerOnce(t *testing.T, key string) string {
27 t.Helper()
28 opts := fmt.Sprintf("-p %d -i %s -o IdentitiesOnly=yes -o StrictHostKeyChecking=no -o UserKnownHostsFile=%s -o BatchMode=yes",
29 i.port, key, filepath.Join(i.sshDir, "known_hosts"))
30 // -isolation none: these tests exercise claiming, logs, statuses and
31 // cancellation, not the sandbox, and the suite must run on a machine
32 // without podman. The isolation tests are in isolation_podman_test.go
33 // and skip visibly when it is absent (#144).
34 cmd := exec.Command(i.runner, "-once",
35 "-remote", "git@127.0.0.1",
36 "-ssh-opts", opts,
37 "-isolation", "none",
38 "-clone-base", fmt.Sprintf("ssh://git@127.0.0.1:%d", i.port),
39 "-workdir", t.TempDir())
40 cmd.Env = append(os.Environ(), "GIT_CONFIG_NOSYSTEM=1", "GIT_CONFIG_GLOBAL=/dev/null")
41 out, err := cmd.CombinedOutput()
42 if err != nil {
43 t.Fatalf("runner: %v\n%s", err, out)
44 }
45 return string(out)
46}
47
48// A failed build mails the repo owner with the log tail; green builds
49// stay silent.
50func TestBuildFailureMail(t *testing.T) {
51 smtp := startFakeSMTP(t)
52 inst := startInstanceWith(t, fmt.Sprintf(
53 "[mail]\nsmtp_host = %q\nfrom = \"noreply@gitbay.test\"\n", smtp.addr))
54 inst.runner = buildRunner(t)
55 aliceKey := inst.newKey(t, "alice")
56 inst.admin(t, "admin", "user", "create", "alice", "--key", aliceKey+".pub",
57 "--email", "alice@example.test", "--verified")
58 runnerKey := inst.newKey(t, "ci")
59 inst.admin(t, "admin", "user", "create", "ci", "--key", runnerKey+".pub", "--admin")
60
61 if _, errOut, code := inst.ssh(t, aliceKey, "", "repo", "create", "alice/app"); code != 0 {
62 t.Fatalf("repo create: %s", errOut)
63 }
64 work := t.TempDir()
65 env := inst.gitEnv(aliceKey)
66 mustGit(t, work, env, "clone", inst.sshURL("alice/app"), "w")
67 dir := filepath.Join(work, "w")
68 os.MkdirAll(filepath.Join(dir, ".gitbay"), 0o755)
69 os.WriteFile(filepath.Join(dir, ".gitbay", "ci.yml"), []byte(
70 "jobs:\n ok:\n steps:\n - echo fine\n broken:\n steps:\n - echo the dataset went stale && false\n"), 0o644)
71 mustGit(t, dir, env, "checkout", "-q", "-b", "main")
72 mustGit(t, dir, env, "add", ".")
73 mustGit(t, dir, env, "commit", "-q", "-m", "base")
74 mustGit(t, dir, env, "push", "-q", "origin", "main")
75
76 inst.runnerOnce(t, runnerKey) // broken (sorts first)
77 inst.runnerOnce(t, runnerKey) // ok
78 mail := smtp.waitFor(t, "alice@example.test", "failed")
79 if !strings.Contains(mail, "broken") || !strings.Contains(mail, "the dataset went stale") ||
80 !strings.Contains(mail, "/alice/app/builds/") {
81 t.Fatalf("failure mail missing detail:\n%s", mail)
82 }
83 // Only the failure mailed: no message mentions the green job.
84 for _, m := range smtp.mailTo("alice@example.test") {
85 if strings.Contains(m, "build") && strings.Contains(m, " ok ") && strings.Contains(m, "failed") == false {
86 t.Fatalf("green build mailed:\n%s", m)
87 }
88 }
89}
90
91func TestCI(t *testing.T) {
92 inst := startInstance(t)
93 inst.runner = buildRunner(t)
94 aliceKey := inst.newKey(t, "alice")
95 inst.admin(t, "admin", "user", "create", "alice", "--key", aliceKey+".pub")
96 runnerKey := inst.newKey(t, "ci")
97 inst.admin(t, "admin", "user", "create", "ci", "--key", runnerKey+".pub", "--admin")
98
99 if _, errOut, code := inst.ssh(t, aliceKey, "", "repo", "create", "alice/app"); code != 0 {
100 t.Fatalf("repo create: %s", errOut)
101 }
102 work := t.TempDir()
103 env := inst.gitEnv(aliceKey)
104 mustGit(t, work, env, "clone", inst.sshURL("alice/app"), "w")
105 dir := filepath.Join(work, "w")
106 os.MkdirAll(filepath.Join(dir, ".gitbay"), 0o755)
107 os.WriteFile(filepath.Join(dir, ".gitbay", "ci.yml"), []byte(
108 "jobs:\n ok:\n steps:\n - echo hello from $GITBAY_JOB\n broken:\n steps:\n - \"false\"\n"), 0o644)
109 os.WriteFile(filepath.Join(dir, "f.txt"), []byte("x\n"), 0o644)
110 mustGit(t, dir, env, "checkout", "-q", "-b", "main")
111 mustGit(t, dir, env, "add", ".")
112 mustGit(t, dir, env, "commit", "-q", "-m", "base")
113 mustGit(t, dir, env, "push", "-q", "origin", "main")
114 sha := strings.TrimSpace(mustGit(t, dir, env, "rev-parse", "HEAD"))
115
116 // The push queued one pending build per job, with pending statuses.
117 out, _, _ := inst.ssh(t, aliceKey, "", "build", "list", "alice/app")
118 if !strings.Contains(out, "broken\tpending") || !strings.Contains(out, "ok\tpending") {
119 t.Fatalf("builds not queued:\n%s", out)
120 }
121 out, _, _ = inst.ssh(t, aliceKey, "", "status", "list", "alice/app", sha)
122 if !strings.Contains(out, "ci/ok") || !strings.Contains(out, "pending") {
123 t.Fatalf("pending statuses missing:\n%s", out)
124 }
125
126 // Non-admins cannot claim jobs.
127 if _, _, code := inst.ssh(t, aliceKey, "", "runner", "next"); code != 4 {
128 t.Fatalf("non-admin claimed a build: exit %d", code)
129 }
130
131 // The runner processes both jobs ("broken" sorts first).
132 inst.runnerOnce(t, runnerKey)
133 inst.runnerOnce(t, runnerKey)
134
135 out, _, _ = inst.ssh(t, aliceKey, "", "build", "list", "alice/app")
136 if !strings.Contains(out, "ok\tsuccess") || !strings.Contains(out, "broken\tfailure") {
137 t.Fatalf("build outcomes wrong:\n%s", out)
138 }
139 // Logs captured the step output and the failure.
140 var okN, brokenN string
141 for _, l := range strings.Split(strings.TrimSpace(out), "\n") {
142 f := strings.Split(l, "\t")
143 if f[1] == "ok" {
144 okN = f[0]
145 } else {
146 brokenN = f[0]
147 }
148 }
149 out, _, _ = inst.ssh(t, aliceKey, "", "build", "log", "alice/app", okN)
150 if !strings.Contains(out, "hello from ok") {
151 t.Fatalf("ok log:\n%s", out)
152 }
153 out, _, _ = inst.ssh(t, aliceKey, "", "build", "log", "alice/app", brokenN)
154 if !strings.Contains(out, "step failed") {
155 t.Fatalf("broken log:\n%s", out)
156 }
157 // Statuses resolved, with target URLs pointing at the build pages.
158 out, _, _ = inst.ssh(t, aliceKey, "", "status", "list", "alice/app", sha, "--json")
159 if !strings.Contains(out, `"ci/ok","state":"success"`) && !strings.Contains(out, `"state":"success"`) {
160 t.Fatalf("status not success:\n%s", out)
161 }
162 if !strings.Contains(out, "/alice/app/builds/") {
163 t.Fatalf("status target url missing:\n%s", out)
164 }
165
166 // Web: list page and log page.
167 status, body := inst.get(t, "/alice/app/builds")
168 if status != 200 || !strings.Contains(body, "ok") || !strings.Contains(body, "failure") {
169 t.Fatalf("builds page: %d\n%s", status, body)
170 }
171 if _, body = inst.get(t, "/alice/app/builds/"+okN); !strings.Contains(body, "hello from ok") {
172 t.Fatalf("build log page:\n%s", body)
173 }
174
175 // --- secrets: stdin in, names-only out, injected into the build env ---
176 if _, errOut, code := inst.ssh(t, aliceKey, "hunter2\n", "repo", "secret", "set", "alice/app", "MY_TOKEN"); code != 0 {
177 t.Fatalf("secret set: %s", errOut)
178 }
179 if _, _, code := inst.ssh(t, aliceKey, "x\n", "repo", "secret", "set", "alice/app", "bad-name"); code != 2 {
180 t.Fatal("bad secret name accepted")
181 }
182 out, _, _ = inst.ssh(t, aliceKey, "", "repo", "secret", "list", "alice/app")
183 if !strings.Contains(out, "MY_TOKEN") || strings.Contains(out, "hunter2") {
184 t.Fatalf("secret list leaked or missed: %s", out)
185 }
186
187 // --- schedules and manual trigger ---
188 os.WriteFile(filepath.Join(dir, ".gitbay", "ci.yml"), []byte(
189 "jobs:\n usesecret:\n steps:\n - echo token=$MY_TOKEN\n nightly:\n schedule: \"0 6 * * 1\"\n steps:\n - echo scheduled ran\n"), 0o644)
190 mustGit(t, dir, env, "add", ".")
191 mustGit(t, dir, env, "commit", "-q", "-m", "secrets and schedule")
192 mustGit(t, dir, env, "push", "-q", "origin", "main")
193
194 // The push queued only the unscheduled job.
195 out, _, _ = inst.ssh(t, aliceKey, "", "build", "list", "alice/app")
196 if !strings.Contains(out, "usesecret\tpending") || strings.Contains(out, "nightly") {
197 t.Fatalf("scheduled job queued on push:\n%s", out)
198 }
199 inst.runnerOnce(t, runnerKey)
200 out, _, _ = inst.ssh(t, aliceKey, "", "build", "list", "alice/app")
201 usecretN := strings.Split(out, "\t")[0]
202 out, _, _ = inst.ssh(t, aliceKey, "", "build", "log", "alice/app", usecretN)
203 if !strings.Contains(out, "token=hunter2") {
204 t.Fatalf("secret not injected:\n%s", out)
205 }
206 // The scheduled job runs on demand via trigger.
207 if _, errOut, code := inst.ssh(t, aliceKey, "", "build", "trigger", "alice/app", "nightly"); code != 0 {
208 t.Fatalf("trigger: %s", errOut)
209 }
210 if _, _, code := inst.ssh(t, aliceKey, "", "build", "trigger", "alice/app", "nosuch"); code != 3 {
211 t.Fatal("triggered a job that does not exist")
212 }
213 inst.runnerOnce(t, runnerKey)
214 out, _, _ = inst.ssh(t, aliceKey, "", "build", "list", "alice/app")
215 if !strings.Contains(out, "nightly\tsuccess") {
216 t.Fatalf("triggered build did not run:\n%s", out)
217 }
218 // Removing the secret stops injection.
219 if _, _, code := inst.ssh(t, aliceKey, "", "repo", "secret", "remove", "alice/app", "MY_TOKEN"); code != 0 {
220 t.Fatal("secret remove failed")
221 }
222
223 // --- tag-triggered jobs ---
224 os.WriteFile(filepath.Join(dir, ".gitbay", "ci.yml"), []byte(
225 "jobs:\n test:\n steps:\n - echo branch build\n publish:\n tags: \"v*\"\n steps:\n - echo publishing $GITBAY_REF\n"), 0o644)
226 mustGit(t, dir, env, "add", ".")
227 mustGit(t, dir, env, "commit", "-q", "-m", "tag job")
228 mustGit(t, dir, env, "push", "-q", "origin", "main")
229 // The branch push queued only the branch job.
230 out, _, _ = inst.ssh(t, aliceKey, "", "build", "list", "alice/app")
231 if strings.Contains(out, "publish") {
232 t.Fatalf("tag job queued on branch push:\n%s", out)
233 }
234 // An annotated tag queues the tag job, with the peeled commit as sha.
235 mustGit(t, dir, env, "tag", "-a", "-m", "rel", "v1.0.0")
236 mustGit(t, dir, env, "push", "-q", "origin", "v1.0.0")
237 headSHA := strings.TrimSpace(mustGit(t, dir, env, "rev-parse", "HEAD"))
238 out, _, _ = inst.ssh(t, aliceKey, "", "build", "list", "alice/app")
239 if !strings.Contains(out, "publish\tpending\t"+headSHA[:10]) || !strings.Contains(out, "v1.0.0") {
240 t.Fatalf("tag build missing or unpeeled:\n%s", out)
241 }
242 // A non-matching tag queues nothing.
243 mustGit(t, dir, env, "tag", "nightly-1")
244 mustGit(t, dir, env, "push", "-q", "origin", "nightly-1")
245 out2, _, _ := inst.ssh(t, aliceKey, "", "build", "list", "alice/app")
246 if strings.Count(out2, "publish") != strings.Count(out, "publish") {
247 t.Fatalf("non-matching tag queued a build:\n%s", out2)
248 }
249 inst.runnerOnce(t, runnerKey) // branch "test" job
250 inst.runnerOnce(t, runnerKey) // tag "publish" job
251 out, _, _ = inst.ssh(t, aliceKey, "", "build", "list", "alice/app")
252 if !strings.Contains(out, "publish\tsuccess") {
253 t.Fatalf("tag build did not run:\n%s", out)
254 }
255 // schedule and tags together are refused.
256 os.WriteFile(filepath.Join(dir, ".gitbay", "ci.yml"), []byte(
257 "jobs:\n both:\n schedule: \"0 6 * * *\"\n tags: \"v*\"\n steps: [echo x]\n"), 0o644)
258 mustGit(t, dir, env, "add", ".")
259 mustGit(t, dir, env, "commit", "-q", "-m", "both triggers")
260 mustGit(t, dir, env, "push", "-q", "origin", "main")
261 shaBoth := strings.TrimSpace(mustGit(t, dir, env, "rev-parse", "HEAD"))
262 out, _, _ = inst.ssh(t, aliceKey, "", "status", "list", "alice/app", shaBoth)
263 if !strings.Contains(out, "ci/config") || !strings.Contains(out, "failure") {
264 t.Fatalf("mutually exclusive triggers not refused:\n%s", out)
265 }
266
267 // A broken ci.yml surfaces as a failed ci/config status.
268 os.WriteFile(filepath.Join(dir, ".gitbay", "ci.yml"), []byte("jobs: {bad name: {steps: [x]}}\n"), 0o644)
269 mustGit(t, dir, env, "add", ".")
270 mustGit(t, dir, env, "commit", "-q", "-m", "break config")
271 mustGit(t, dir, env, "push", "-q", "origin", "main")
272 sha2 := strings.TrimSpace(mustGit(t, dir, env, "rev-parse", "HEAD"))
273 out, _, _ = inst.ssh(t, aliceKey, "", "status", "list", "alice/app", sha2)
274 if !strings.Contains(out, "ci/config") || !strings.Contains(out, "failure") {
275 t.Fatalf("config failure status missing:\n%s", out)
276 }
277}
278
279// runnerJobs runs a runner with -jobs n until it has nothing left to do,
280// and returns its output. Unlike runnerOnce it is not bounded to one
281// build, so it is stopped when the queue drains.
282func (i *instance) runnerJobs(t *testing.T, key, repo string, jobs int) string {
283 t.Helper()
284 opts := fmt.Sprintf("-p %d -i %s -o IdentitiesOnly=yes -o StrictHostKeyChecking=no -o UserKnownHostsFile=%s -o BatchMode=yes",
285 i.port, key, filepath.Join(i.sshDir, "known_hosts"))
286 cmd := exec.Command(i.runner,
287 "-jobs", fmt.Sprint(jobs),
288 "-poll", "200ms",
289 "-isolation", "none",
290 "-remote", "git@127.0.0.1",
291 "-ssh-opts", opts,
292 "-clone-base", fmt.Sprintf("ssh://git@127.0.0.1:%d", i.port),
293 "-workdir", t.TempDir())
294 cmd.Env = append(os.Environ(), "GIT_CONFIG_NOSYSTEM=1", "GIT_CONFIG_GLOBAL=/dev/null")
295 var buf bytes.Buffer
296 cmd.Stdout, cmd.Stderr = &buf, &buf
297 if err := cmd.Start(); err != nil {
298 t.Fatal(err)
299 }
300 defer func() { cmd.Process.Kill(); cmd.Wait() }()
301
302 // Wait for every queued build to leave the pending and running states.
303 deadline := time.Now().Add(60 * time.Second)
304 for time.Now().Before(deadline) {
305 out, _, code := i.ssh(t, key, "", "build", "list", repo, "--json")
306 if code == 0 && !strings.Contains(out, `"status":"pending"`) &&
307 !strings.Contains(out, `"status":"running"`) {
308 break
309 }
310 time.Sleep(200 * time.Millisecond)
311 }
312 return buf.String()
313}
314
315// -jobs N runs N builds at once. ClaimBuild has always been a single
316// transaction that selects and updates, so several workers claiming
317// together is safe; the runner simply never used more than one (#115).
318func TestRunnerConcurrentJobs(t *testing.T) {
319 inst := startInstance(t)
320 inst.runner = buildRunner(t)
321 aliceKey := inst.newKey(t, "alice")
322 inst.admin(t, "admin", "user", "create", "alice", "--key", aliceKey+".pub", "--admin")
323 if _, errOut, code := inst.ssh(t, aliceKey, "", "repo", "create", "alice/app"); code != 0 {
324 t.Fatalf("repo create: %s", errOut)
325 }
326
327 // Four jobs, each sleeping longer than the poll interval, so serial
328 // execution and concurrent execution are distinguishable.
329 ci := "jobs:\n"
330 for _, name := range []string{"one", "two", "three", "four"} {
331 ci += fmt.Sprintf(" %s:\n steps:\n - sleep 1\n - echo done-%s\n", name, name)
332 }
333 env := inst.gitEnv(aliceKey)
334 work := t.TempDir()
335 mustGit(t, work, env, "clone", inst.sshURL("alice/app"), "w")
336 dir := filepath.Join(work, "w")
337 os.MkdirAll(filepath.Join(dir, ".gitbay"), 0o755)
338 os.WriteFile(filepath.Join(dir, ".gitbay", "ci.yml"), []byte(ci), 0o644)
339 mustGit(t, dir, env, "checkout", "-q", "-b", "main")
340 mustGit(t, dir, env, "add", ".")
341 mustGit(t, dir, env, "commit", "-q", "-m", "ci")
342 mustGit(t, dir, env, "push", "-q", "origin", "main")
343
344 out := inst.runnerJobs(t, aliceKey, "alice/app", 4)
345
346 listing, _, code := inst.ssh(t, aliceKey, "", "build", "list", "alice/app", "--json")
347 if code != 0 {
348 t.Fatalf("build list failed:\n%s", out)
349 }
350 for _, name := range []string{"one", "two", "three", "four"} {
351 if !strings.Contains(listing, `"job":"`+name+`"`) {
352 t.Fatalf("%s never ran:\n%s\n%s", name, listing, out)
353 }
354 }
355 if strings.Contains(listing, `"status":"pending"`) || strings.Contains(listing, `"status":"running"`) {
356 t.Fatalf("builds did not finish:\n%s", listing)
357 }
358 // Overlap, asserted from the runner's own log rather than from wall
359 // clock: elapsed time also covers four concurrent clones and the
360 // polling this test does, and would make a slow machine look serial.
361 // Every build announces itself when it starts and again when it
362 // finishes, so concurrency is "a build started before the first one
363 // finished".
364 started, firstFinish := 0, -1
365 for _, line := range strings.Split(out, "\n") {
366 switch {
367 case strings.Contains(line, "alice/app"):
368 started++
369 case strings.Contains(line, ": success"), strings.Contains(line, ": failure"):
370 if firstFinish < 0 {
371 firstFinish = started
372 }
373 }
374 }
375 if started != 4 {
376 t.Fatalf("%d builds started, want 4:\n%s", started, out)
377 }
378 if firstFinish < 2 {
379 t.Errorf("only %d build(s) had started when the first finished; -jobs 4 ran them serially\n%s",
380 firstFinish, out)
381 }
382}