e2e/ci_test.go
384 lines · 16392 bytes
1package e2e
2
3import (
4 "bytes"
5 "fmt"
6 "os"
7 "os/exec"
8 "path/filepath"
9 "strings"
10 "testing"
11 "time"
12)
13
14func buildRunner(t *testing.T) string {
15 t.Helper()
16 bin := filepath.Join(t.TempDir(), "gitbay-runner")
17 cmd := exec.Command("go", "build", "-o", bin, "gitbay.org/gitbay/cmd/gitbay-runner")
18 cmd.Dir = ".."
19 if out, err := cmd.CombinedOutput(); err != nil {
20 t.Fatalf("build gitbay-runner: %v\n%s", err, out)
21 }
22 return bin
23}
24
25// runnerOnce processes at most one pending build with the given key.
26func (i *instance) runnerOnce(t *testing.T, key string, extra ...string) string {
27 t.Helper()
28 opts := fmt.Sprintf("-p %d -i %s -o IdentitiesOnly=yes -o StrictHostKeyChecking=no -o UserKnownHostsFile=%s -o BatchMode=yes",
29 i.port, key, filepath.Join(i.sshDir, "known_hosts"))
30 // -isolation none: these tests exercise claiming, logs, statuses and
31 // cancellation, not the sandbox, and the suite must run on a machine
32 // without podman. The isolation tests are in isolation_podman_test.go
33 // and skip visibly when it is absent (#144).
34 args := []string{"-once",
35 "-remote", "git@127.0.0.1",
36 "-ssh-opts", opts,
37 "-isolation", "none",
38 "-clone-base", fmt.Sprintf("ssh://git@127.0.0.1:%d", i.port),
39 "-workdir", t.TempDir()}
40 args = append(args, extra...)
41 cmd := exec.Command(i.runner, args...)
42 cmd.Env = append(os.Environ(), "XDG_CONFIG_HOME="+t.TempDir(), "GIT_CONFIG_NOSYSTEM=1", "GIT_CONFIG_GLOBAL=/dev/null")
43 out, err := cmd.CombinedOutput()
44 if err != nil {
45 t.Fatalf("runner: %v\n%s", err, out)
46 }
47 return string(out)
48}
49
50// A failed build mails the repo owner with the log tail; green builds
51// stay silent.
52func TestBuildFailureMail(t *testing.T) {
53 smtp := startFakeSMTP(t)
54 inst := startInstanceWith(t, fmt.Sprintf(
55 "[mail]\nsmtp_host = %q\nfrom = \"noreply@gitbay.test\"\n", smtp.addr))
56 inst.runner = buildRunner(t)
57 aliceKey := inst.newKey(t, "alice")
58 inst.admin(t, "admin", "user", "create", "alice", "--key", aliceKey+".pub",
59 "--email", "alice@example.test", "--verified")
60 runnerKey := inst.newKey(t, "ci")
61 inst.admin(t, "admin", "user", "create", "ci", "--key", runnerKey+".pub", "--admin")
62
63 if _, errOut, code := inst.ssh(t, aliceKey, "", "repo", "create", "alice/app"); code != 0 {
64 t.Fatalf("repo create: %s", errOut)
65 }
66 work := t.TempDir()
67 env := inst.gitEnv(aliceKey)
68 mustGit(t, work, env, "clone", inst.sshURL("alice/app"), "w")
69 dir := filepath.Join(work, "w")
70 os.MkdirAll(filepath.Join(dir, ".gitbay"), 0o755)
71 os.WriteFile(filepath.Join(dir, ".gitbay", "ci.yml"), []byte(
72 "jobs:\n ok:\n steps:\n - echo fine\n broken:\n steps:\n - echo the dataset went stale && false\n"), 0o644)
73 mustGit(t, dir, env, "checkout", "-q", "-b", "main")
74 mustGit(t, dir, env, "add", ".")
75 mustGit(t, dir, env, "commit", "-q", "-m", "base")
76 mustGit(t, dir, env, "push", "-q", "origin", "main")
77
78 inst.runnerOnce(t, runnerKey) // broken (sorts first)
79 inst.runnerOnce(t, runnerKey) // ok
80 mail := smtp.waitFor(t, "alice@example.test", "failed")
81 if !strings.Contains(mail, "broken") || !strings.Contains(mail, "the dataset went stale") ||
82 !strings.Contains(mail, "/alice/app/builds/") {
83 t.Fatalf("failure mail missing detail:\n%s", mail)
84 }
85 // Only the failure mailed: no message mentions the green job.
86 for _, m := range smtp.mailTo("alice@example.test") {
87 if strings.Contains(m, "build") && strings.Contains(m, " ok ") && strings.Contains(m, "failed") == false {
88 t.Fatalf("green build mailed:\n%s", m)
89 }
90 }
91}
92
93func TestCI(t *testing.T) {
94 inst := startInstance(t)
95 inst.runner = buildRunner(t)
96 aliceKey := inst.newKey(t, "alice")
97 inst.admin(t, "admin", "user", "create", "alice", "--key", aliceKey+".pub")
98 runnerKey := inst.newKey(t, "ci")
99 inst.admin(t, "admin", "user", "create", "ci", "--key", runnerKey+".pub", "--admin")
100
101 if _, errOut, code := inst.ssh(t, aliceKey, "", "repo", "create", "alice/app"); code != 0 {
102 t.Fatalf("repo create: %s", errOut)
103 }
104 work := t.TempDir()
105 env := inst.gitEnv(aliceKey)
106 mustGit(t, work, env, "clone", inst.sshURL("alice/app"), "w")
107 dir := filepath.Join(work, "w")
108 os.MkdirAll(filepath.Join(dir, ".gitbay"), 0o755)
109 os.WriteFile(filepath.Join(dir, ".gitbay", "ci.yml"), []byte(
110 "jobs:\n ok:\n steps:\n - echo hello from $GITBAY_JOB\n broken:\n steps:\n - \"false\"\n"), 0o644)
111 os.WriteFile(filepath.Join(dir, "f.txt"), []byte("x\n"), 0o644)
112 mustGit(t, dir, env, "checkout", "-q", "-b", "main")
113 mustGit(t, dir, env, "add", ".")
114 mustGit(t, dir, env, "commit", "-q", "-m", "base")
115 mustGit(t, dir, env, "push", "-q", "origin", "main")
116 sha := strings.TrimSpace(mustGit(t, dir, env, "rev-parse", "HEAD"))
117
118 // The push queued one pending build per job, with pending statuses.
119 out, _, _ := inst.ssh(t, aliceKey, "", "build", "list", "alice/app")
120 if !strings.Contains(out, "broken\tpending") || !strings.Contains(out, "ok\tpending") {
121 t.Fatalf("builds not queued:\n%s", out)
122 }
123 out, _, _ = inst.ssh(t, aliceKey, "", "status", "list", "alice/app", sha)
124 if !strings.Contains(out, "ci/ok") || !strings.Contains(out, "pending") {
125 t.Fatalf("pending statuses missing:\n%s", out)
126 }
127
128 // Non-admins cannot claim jobs.
129 if _, _, code := inst.ssh(t, aliceKey, "", "runner", "next"); code != 4 {
130 t.Fatalf("non-admin claimed a build: exit %d", code)
131 }
132
133 // The runner processes both jobs ("broken" sorts first).
134 inst.runnerOnce(t, runnerKey)
135 inst.runnerOnce(t, runnerKey)
136
137 out, _, _ = inst.ssh(t, aliceKey, "", "build", "list", "alice/app")
138 if !strings.Contains(out, "ok\tsuccess") || !strings.Contains(out, "broken\tfailure") {
139 t.Fatalf("build outcomes wrong:\n%s", out)
140 }
141 // Logs captured the step output and the failure.
142 var okN, brokenN string
143 for _, l := range strings.Split(strings.TrimSpace(out), "\n") {
144 f := strings.Split(l, "\t")
145 if f[1] == "ok" {
146 okN = f[0]
147 } else {
148 brokenN = f[0]
149 }
150 }
151 out, _, _ = inst.ssh(t, aliceKey, "", "build", "log", "alice/app", okN)
152 if !strings.Contains(out, "hello from ok") {
153 t.Fatalf("ok log:\n%s", out)
154 }
155 out, _, _ = inst.ssh(t, aliceKey, "", "build", "log", "alice/app", brokenN)
156 if !strings.Contains(out, "step failed") {
157 t.Fatalf("broken log:\n%s", out)
158 }
159 // Statuses resolved, with target URLs pointing at the build pages.
160 out, _, _ = inst.ssh(t, aliceKey, "", "status", "list", "alice/app", sha, "--json")
161 if !strings.Contains(out, `"ci/ok","state":"success"`) && !strings.Contains(out, `"state":"success"`) {
162 t.Fatalf("status not success:\n%s", out)
163 }
164 if !strings.Contains(out, "/alice/app/builds/") {
165 t.Fatalf("status target url missing:\n%s", out)
166 }
167
168 // Web: list page and log page.
169 status, body := inst.get(t, "/alice/app/builds")
170 if status != 200 || !strings.Contains(body, "ok") || !strings.Contains(body, "failure") {
171 t.Fatalf("builds page: %d\n%s", status, body)
172 }
173 if _, body = inst.get(t, "/alice/app/builds/"+okN); !strings.Contains(body, "hello from ok") {
174 t.Fatalf("build log page:\n%s", body)
175 }
176
177 // --- secrets: stdin in, names-only out, injected into the build env ---
178 if _, errOut, code := inst.ssh(t, aliceKey, "hunter2\n", "repo", "secret", "set", "alice/app", "MY_TOKEN"); code != 0 {
179 t.Fatalf("secret set: %s", errOut)
180 }
181 if _, _, code := inst.ssh(t, aliceKey, "x\n", "repo", "secret", "set", "alice/app", "bad-name"); code != 2 {
182 t.Fatal("bad secret name accepted")
183 }
184 out, _, _ = inst.ssh(t, aliceKey, "", "repo", "secret", "list", "alice/app")
185 if !strings.Contains(out, "MY_TOKEN") || strings.Contains(out, "hunter2") {
186 t.Fatalf("secret list leaked or missed: %s", out)
187 }
188
189 // --- schedules and manual trigger ---
190 os.WriteFile(filepath.Join(dir, ".gitbay", "ci.yml"), []byte(
191 "jobs:\n usesecret:\n steps:\n - echo token=$MY_TOKEN\n nightly:\n schedule: \"0 6 * * 1\"\n steps:\n - echo scheduled ran\n"), 0o644)
192 mustGit(t, dir, env, "add", ".")
193 mustGit(t, dir, env, "commit", "-q", "-m", "secrets and schedule")
194 mustGit(t, dir, env, "push", "-q", "origin", "main")
195
196 // The push queued only the unscheduled job.
197 out, _, _ = inst.ssh(t, aliceKey, "", "build", "list", "alice/app")
198 if !strings.Contains(out, "usesecret\tpending") || strings.Contains(out, "nightly") {
199 t.Fatalf("scheduled job queued on push:\n%s", out)
200 }
201 inst.runnerOnce(t, runnerKey)
202 out, _, _ = inst.ssh(t, aliceKey, "", "build", "list", "alice/app")
203 usecretN := strings.Split(out, "\t")[0]
204 out, _, _ = inst.ssh(t, aliceKey, "", "build", "log", "alice/app", usecretN)
205 if !strings.Contains(out, "token=hunter2") {
206 t.Fatalf("secret not injected:\n%s", out)
207 }
208 // The scheduled job runs on demand via trigger.
209 if _, errOut, code := inst.ssh(t, aliceKey, "", "build", "trigger", "alice/app", "nightly"); code != 0 {
210 t.Fatalf("trigger: %s", errOut)
211 }
212 if _, _, code := inst.ssh(t, aliceKey, "", "build", "trigger", "alice/app", "nosuch"); code != 3 {
213 t.Fatal("triggered a job that does not exist")
214 }
215 inst.runnerOnce(t, runnerKey)
216 out, _, _ = inst.ssh(t, aliceKey, "", "build", "list", "alice/app")
217 if !strings.Contains(out, "nightly\tsuccess") {
218 t.Fatalf("triggered build did not run:\n%s", out)
219 }
220 // Removing the secret stops injection.
221 if _, _, code := inst.ssh(t, aliceKey, "", "repo", "secret", "remove", "alice/app", "MY_TOKEN"); code != 0 {
222 t.Fatal("secret remove failed")
223 }
224
225 // --- tag-triggered jobs ---
226 os.WriteFile(filepath.Join(dir, ".gitbay", "ci.yml"), []byte(
227 "jobs:\n test:\n steps:\n - echo branch build\n publish:\n tags: \"v*\"\n steps:\n - echo publishing $GITBAY_REF\n"), 0o644)
228 mustGit(t, dir, env, "add", ".")
229 mustGit(t, dir, env, "commit", "-q", "-m", "tag job")
230 mustGit(t, dir, env, "push", "-q", "origin", "main")
231 // The branch push queued only the branch job.
232 out, _, _ = inst.ssh(t, aliceKey, "", "build", "list", "alice/app")
233 if strings.Contains(out, "publish") {
234 t.Fatalf("tag job queued on branch push:\n%s", out)
235 }
236 // An annotated tag queues the tag job, with the peeled commit as sha.
237 mustGit(t, dir, env, "tag", "-a", "-m", "rel", "v1.0.0")
238 mustGit(t, dir, env, "push", "-q", "origin", "v1.0.0")
239 headSHA := strings.TrimSpace(mustGit(t, dir, env, "rev-parse", "HEAD"))
240 out, _, _ = inst.ssh(t, aliceKey, "", "build", "list", "alice/app")
241 if !strings.Contains(out, "publish\tpending\t"+headSHA[:10]) || !strings.Contains(out, "v1.0.0") {
242 t.Fatalf("tag build missing or unpeeled:\n%s", out)
243 }
244 // A non-matching tag queues nothing.
245 mustGit(t, dir, env, "tag", "nightly-1")
246 mustGit(t, dir, env, "push", "-q", "origin", "nightly-1")
247 out2, _, _ := inst.ssh(t, aliceKey, "", "build", "list", "alice/app")
248 if strings.Count(out2, "publish") != strings.Count(out, "publish") {
249 t.Fatalf("non-matching tag queued a build:\n%s", out2)
250 }
251 inst.runnerOnce(t, runnerKey) // branch "test" job
252 inst.runnerOnce(t, runnerKey) // tag "publish" job
253 out, _, _ = inst.ssh(t, aliceKey, "", "build", "list", "alice/app")
254 if !strings.Contains(out, "publish\tsuccess") {
255 t.Fatalf("tag build did not run:\n%s", out)
256 }
257 // schedule and tags together are refused.
258 os.WriteFile(filepath.Join(dir, ".gitbay", "ci.yml"), []byte(
259 "jobs:\n both:\n schedule: \"0 6 * * *\"\n tags: \"v*\"\n steps: [echo x]\n"), 0o644)
260 mustGit(t, dir, env, "add", ".")
261 mustGit(t, dir, env, "commit", "-q", "-m", "both triggers")
262 mustGit(t, dir, env, "push", "-q", "origin", "main")
263 shaBoth := strings.TrimSpace(mustGit(t, dir, env, "rev-parse", "HEAD"))
264 out, _, _ = inst.ssh(t, aliceKey, "", "status", "list", "alice/app", shaBoth)
265 if !strings.Contains(out, "ci/config") || !strings.Contains(out, "failure") {
266 t.Fatalf("mutually exclusive triggers not refused:\n%s", out)
267 }
268
269 // A broken ci.yml surfaces as a failed ci/config status.
270 os.WriteFile(filepath.Join(dir, ".gitbay", "ci.yml"), []byte("jobs: {bad name: {steps: [x]}}\n"), 0o644)
271 mustGit(t, dir, env, "add", ".")
272 mustGit(t, dir, env, "commit", "-q", "-m", "break config")
273 mustGit(t, dir, env, "push", "-q", "origin", "main")
274 sha2 := strings.TrimSpace(mustGit(t, dir, env, "rev-parse", "HEAD"))
275 out, _, _ = inst.ssh(t, aliceKey, "", "status", "list", "alice/app", sha2)
276 if !strings.Contains(out, "ci/config") || !strings.Contains(out, "failure") {
277 t.Fatalf("config failure status missing:\n%s", out)
278 }
279}
280
281// runnerJobs runs a runner with -jobs n until it has nothing left to do,
282// and returns its output. Unlike runnerOnce it is not bounded to one
283// build, so it is stopped when the queue drains.
284func (i *instance) runnerJobs(t *testing.T, key, repo string, jobs int) string {
285 t.Helper()
286 opts := fmt.Sprintf("-p %d -i %s -o IdentitiesOnly=yes -o StrictHostKeyChecking=no -o UserKnownHostsFile=%s -o BatchMode=yes",
287 i.port, key, filepath.Join(i.sshDir, "known_hosts"))
288 cmd := exec.Command(i.runner,
289 "-jobs", fmt.Sprint(jobs),
290 "-poll", "200ms",
291 "-isolation", "none",
292 "-remote", "git@127.0.0.1",
293 "-ssh-opts", opts,
294 "-clone-base", fmt.Sprintf("ssh://git@127.0.0.1:%d", i.port),
295 "-workdir", t.TempDir())
296 cmd.Env = append(os.Environ(), "XDG_CONFIG_HOME="+t.TempDir(), "GIT_CONFIG_NOSYSTEM=1", "GIT_CONFIG_GLOBAL=/dev/null")
297 var buf bytes.Buffer
298 cmd.Stdout, cmd.Stderr = &buf, &buf
299 if err := cmd.Start(); err != nil {
300 t.Fatal(err)
301 }
302 defer func() { cmd.Process.Kill(); cmd.Wait() }()
303
304 // Wait for every queued build to leave the pending and running states.
305 deadline := time.Now().Add(60 * time.Second)
306 for time.Now().Before(deadline) {
307 out, _, code := i.ssh(t, key, "", "build", "list", repo, "--json")
308 if code == 0 && !strings.Contains(out, `"status":"pending"`) &&
309 !strings.Contains(out, `"status":"running"`) {
310 break
311 }
312 time.Sleep(200 * time.Millisecond)
313 }
314 return buf.String()
315}
316
317// -jobs N runs N builds at once. ClaimBuild has always been a single
318// transaction that selects and updates, so several workers claiming
319// together is safe; the runner simply never used more than one (#115).
320func TestRunnerConcurrentJobs(t *testing.T) {
321 inst := startInstance(t)
322 inst.runner = buildRunner(t)
323 aliceKey := inst.newKey(t, "alice")
324 inst.admin(t, "admin", "user", "create", "alice", "--key", aliceKey+".pub", "--admin")
325 if _, errOut, code := inst.ssh(t, aliceKey, "", "repo", "create", "alice/app"); code != 0 {
326 t.Fatalf("repo create: %s", errOut)
327 }
328
329 // Four jobs, each sleeping longer than the poll interval, so serial
330 // execution and concurrent execution are distinguishable.
331 ci := "jobs:\n"
332 for _, name := range []string{"one", "two", "three", "four"} {
333 ci += fmt.Sprintf(" %s:\n steps:\n - sleep 1\n - echo done-%s\n", name, name)
334 }
335 env := inst.gitEnv(aliceKey)
336 work := t.TempDir()
337 mustGit(t, work, env, "clone", inst.sshURL("alice/app"), "w")
338 dir := filepath.Join(work, "w")
339 os.MkdirAll(filepath.Join(dir, ".gitbay"), 0o755)
340 os.WriteFile(filepath.Join(dir, ".gitbay", "ci.yml"), []byte(ci), 0o644)
341 mustGit(t, dir, env, "checkout", "-q", "-b", "main")
342 mustGit(t, dir, env, "add", ".")
343 mustGit(t, dir, env, "commit", "-q", "-m", "ci")
344 mustGit(t, dir, env, "push", "-q", "origin", "main")
345
346 out := inst.runnerJobs(t, aliceKey, "alice/app", 4)
347
348 listing, _, code := inst.ssh(t, aliceKey, "", "build", "list", "alice/app", "--json")
349 if code != 0 {
350 t.Fatalf("build list failed:\n%s", out)
351 }
352 for _, name := range []string{"one", "two", "three", "four"} {
353 if !strings.Contains(listing, `"job":"`+name+`"`) {
354 t.Fatalf("%s never ran:\n%s\n%s", name, listing, out)
355 }
356 }
357 if strings.Contains(listing, `"status":"pending"`) || strings.Contains(listing, `"status":"running"`) {
358 t.Fatalf("builds did not finish:\n%s", listing)
359 }
360 // Overlap, asserted from the runner's own log rather than from wall
361 // clock: elapsed time also covers four concurrent clones and the
362 // polling this test does, and would make a slow machine look serial.
363 // Every build announces itself when it starts and again when it
364 // finishes, so concurrency is "a build started before the first one
365 // finished".
366 started, firstFinish := 0, -1
367 for _, line := range strings.Split(out, "\n") {
368 switch {
369 case strings.Contains(line, "alice/app"):
370 started++
371 case strings.Contains(line, ": success"), strings.Contains(line, ": failure"):
372 if firstFinish < 0 {
373 firstFinish = started
374 }
375 }
376 }
377 if started != 4 {
378 t.Fatalf("%d builds started, want 4:\n%s", started, out)
379 }
380 if firstFinish < 2 {
381 t.Errorf("only %d build(s) had started when the first finished; -jobs 4 ran them serially\n%s",
382 firstFinish, out)
383 }
384}