e2e/ci_test.go
386 lines · 16714 bytes
1package e2e
2
3import (
4 "bytes"
5 "fmt"
6 "os"
7 "os/exec"
8 "path/filepath"
9 "strings"
10 "testing"
11 "time"
12)
13
14// runnerOnce processes at most one pending build with the given key.
15func (i *instance) runnerOnce(t *testing.T, key string, extra ...string) string {
16 t.Helper()
17 opts := fmt.Sprintf("-p %d -i %s -o IdentitiesOnly=yes -o StrictHostKeyChecking=no -o UserKnownHostsFile=%s -o BatchMode=yes",
18 i.port, key, filepath.Join(i.sshDir, "known_hosts"))
19 // -isolation none: these tests exercise claiming, logs, statuses and
20 // cancellation, not the sandbox, and the suite must run on a machine
21 // without podman. The isolation tests are in isolation_podman_test.go
22 // and skip visibly when it is absent (#144).
23 args := []string{"-once",
24 "-remote", "git@127.0.0.1",
25 "-ssh-opts", opts,
26 "-isolation", "none",
27 "-clone-base", fmt.Sprintf("ssh://git@127.0.0.1:%d", i.port),
28 "-workdir", t.TempDir()}
29 args = append(args, extra...)
30 cmd := exec.Command(i.runner, args...)
31 cmd.Env = append(os.Environ(), "XDG_CONFIG_HOME="+t.TempDir(), "GIT_CONFIG_NOSYSTEM=1", "GIT_CONFIG_GLOBAL=/dev/null")
32 out, err := cmd.CombinedOutput()
33 if err != nil {
34 t.Fatalf("runner: %v\n%s", err, out)
35 }
36 return string(out)
37}
38
39// A failed build mails the repo owner with the log tail; green builds
40// stay silent.
41func TestBuildFailureMail(t *testing.T) {
42 t.Parallel()
43 smtp := startFakeSMTP(t)
44 inst := startInstanceWith(t, fmt.Sprintf(
45 "[mail]\nsmtp_host = %q\nfrom = \"noreply@gitbay.test\"\n", smtp.addr))
46 inst.runner = buildRunner(t)
47 aliceKey := inst.newKey(t, "alice")
48 inst.admin(t, "admin", "user", "create", "alice", "--key", aliceKey+".pub",
49 "--email", "alice@example.test", "--verified")
50 runnerKey := inst.newKey(t, "ci")
51 inst.admin(t, "admin", "user", "create", "ci", "--key", runnerKey+".pub", "--admin")
52
53 if _, errOut, code := inst.ssh(t, aliceKey, "", "repo", "create", "alice/app"); code != 0 {
54 t.Fatalf("repo create: %s", errOut)
55 }
56 work := t.TempDir()
57 env := inst.gitEnv(aliceKey)
58 mustGit(t, work, env, "clone", inst.sshURL("alice/app"), "w")
59 dir := filepath.Join(work, "w")
60 os.MkdirAll(filepath.Join(dir, ".gitbay"), 0o755)
61 os.WriteFile(filepath.Join(dir, ".gitbay", "ci.yml"), []byte(
62 "jobs:\n ok:\n steps:\n - echo fine\n broken:\n steps:\n - echo the dataset went stale && false\n"), 0o644)
63 mustGit(t, dir, env, "checkout", "-q", "-b", "main")
64 mustGit(t, dir, env, "add", ".")
65 mustGit(t, dir, env, "commit", "-q", "-m", "base")
66 mustGit(t, dir, env, "push", "-q", "origin", "main")
67
68 inst.runnerOnce(t, runnerKey) // broken (sorts first)
69 inst.runnerOnce(t, runnerKey) // ok
70 mail := smtp.waitFor(t, "alice@example.test", "failed")
71 if !strings.Contains(mail, "broken") || !strings.Contains(mail, "the dataset went stale") ||
72 !strings.Contains(mail, "/alice/app/builds/") {
73 t.Fatalf("failure mail missing detail:\n%s", mail)
74 }
75 // Only the failure mailed: no message mentions the green job.
76 for _, m := range smtp.mailTo("alice@example.test") {
77 if strings.Contains(m, "build") && strings.Contains(m, " ok ") && strings.Contains(m, "failed") == false {
78 t.Fatalf("green build mailed:\n%s", m)
79 }
80 }
81}
82
83func TestCI(t *testing.T) {
84 t.Parallel()
85 inst := startInstance(t)
86 inst.runner = buildRunner(t)
87 aliceKey := inst.newKey(t, "alice")
88 inst.admin(t, "admin", "user", "create", "alice", "--key", aliceKey+".pub")
89 runnerKey := inst.newKey(t, "ci")
90 inst.admin(t, "admin", "user", "create", "ci", "--key", runnerKey+".pub", "--admin")
91
92 if _, errOut, code := inst.ssh(t, aliceKey, "", "repo", "create", "alice/app"); code != 0 {
93 t.Fatalf("repo create: %s", errOut)
94 }
95 work := t.TempDir()
96 env := inst.gitEnv(aliceKey)
97 mustGit(t, work, env, "clone", inst.sshURL("alice/app"), "w")
98 dir := filepath.Join(work, "w")
99 os.MkdirAll(filepath.Join(dir, ".gitbay"), 0o755)
100 os.WriteFile(filepath.Join(dir, ".gitbay", "ci.yml"), []byte(
101 "jobs:\n ok:\n steps:\n - echo hello from $GITBAY_JOB\n broken:\n steps:\n - \"false\"\n"), 0o644)
102 os.WriteFile(filepath.Join(dir, "f.txt"), []byte("x\n"), 0o644)
103 mustGit(t, dir, env, "checkout", "-q", "-b", "main")
104 mustGit(t, dir, env, "add", ".")
105 mustGit(t, dir, env, "commit", "-q", "-m", "base")
106 mustGit(t, dir, env, "push", "-q", "origin", "main")
107 sha := strings.TrimSpace(mustGit(t, dir, env, "rev-parse", "HEAD"))
108
109 // The push queued one pending build per job, with pending statuses.
110 out, _, _ := inst.ssh(t, aliceKey, "", "build", "list", "alice/app")
111 if !strings.Contains(out, "broken\tpending") || !strings.Contains(out, "ok\tpending") {
112 t.Fatalf("builds not queued:\n%s", out)
113 }
114 out, _, _ = inst.ssh(t, aliceKey, "", "status", "list", "alice/app", sha)
115 if !strings.Contains(out, "ci/ok") || !strings.Contains(out, "pending") {
116 t.Fatalf("pending statuses missing:\n%s", out)
117 }
118
119 // Non-admins cannot claim jobs.
120 if _, _, code := inst.ssh(t, aliceKey, "", "runner", "next"); code != 4 {
121 t.Fatalf("non-admin claimed a build: exit %d", code)
122 }
123
124 // The runner processes both jobs ("broken" sorts first).
125 inst.runnerOnce(t, runnerKey)
126 inst.runnerOnce(t, runnerKey)
127
128 out, _, _ = inst.ssh(t, aliceKey, "", "build", "list", "alice/app")
129 if !strings.Contains(out, "ok\tsuccess") || !strings.Contains(out, "broken\tfailure") {
130 t.Fatalf("build outcomes wrong:\n%s", out)
131 }
132 // Logs captured the step output and the failure.
133 var okN, brokenN string
134 for _, l := range strings.Split(strings.TrimSpace(out), "\n") {
135 f := strings.Split(l, "\t")
136 if f[1] == "ok" {
137 okN = f[0]
138 } else {
139 brokenN = f[0]
140 }
141 }
142 out, _, _ = inst.ssh(t, aliceKey, "", "build", "log", "alice/app", okN)
143 if !strings.Contains(out, "hello from ok") {
144 t.Fatalf("ok log:\n%s", out)
145 }
146 out, _, _ = inst.ssh(t, aliceKey, "", "build", "log", "alice/app", brokenN)
147 if !strings.Contains(out, "step 1/1 failed: exit 1") {
148 t.Fatalf("broken log:\n%s", out)
149 }
150 out, _, _ = inst.ssh(t, aliceKey, "", "build", "show", "alice/app", brokenN)
151 if !strings.Contains(out, "1/1 false (exit 1)") {
152 t.Fatalf("build show does not name the failed step:\n%s", out)
153 }
154 if out, _, _ = inst.ssh(t, aliceKey, "", "build", "log", "alice/app", brokenN, "--step", "failed"); strings.Contains(out, "git clone") || !strings.Contains(out, "exit 1") {
155 t.Fatalf("build log --step failed:\n%s", out)
156 }
157 if _, body := inst.get(t, "/alice/app/builds/"+brokenN); !strings.Contains(body, `id="failed" open`) {
158 t.Fatalf("build page does not open the failed step:\n%s", body)
159 }
160 // Statuses resolved, with target URLs pointing at the build pages.
161 out, _, _ = inst.ssh(t, aliceKey, "", "status", "list", "alice/app", sha, "--json")
162 if !strings.Contains(out, `"ci/ok","state":"success"`) && !strings.Contains(out, `"state":"success"`) {
163 t.Fatalf("status not success:\n%s", out)
164 }
165 if !strings.Contains(out, "/alice/app/builds/") {
166 t.Fatalf("status target url missing:\n%s", out)
167 }
168
169 // Web: list page and log page.
170 status, body := inst.get(t, "/alice/app/builds")
171 if status != 200 || !strings.Contains(body, "ok") || !strings.Contains(body, "failure") {
172 t.Fatalf("builds page: %d\n%s", status, body)
173 }
174 if _, body = inst.get(t, "/alice/app/builds/"+okN); !strings.Contains(body, "hello from ok") {
175 t.Fatalf("build log page:\n%s", body)
176 }
177
178 // --- secrets: stdin in, names-only out, injected into the build env ---
179 if _, errOut, code := inst.ssh(t, aliceKey, "hunter2\n", "repo", "secret", "set", "alice/app", "MY_TOKEN"); code != 0 {
180 t.Fatalf("secret set: %s", errOut)
181 }
182 if _, _, code := inst.ssh(t, aliceKey, "x\n", "repo", "secret", "set", "alice/app", "bad-name"); code != 2 {
183 t.Fatal("bad secret name accepted")
184 }
185 out, _, _ = inst.ssh(t, aliceKey, "", "repo", "secret", "list", "alice/app")
186 if !strings.Contains(out, "MY_TOKEN") || strings.Contains(out, "hunter2") {
187 t.Fatalf("secret list leaked or missed: %s", out)
188 }
189
190 // --- schedules and manual trigger ---
191 os.WriteFile(filepath.Join(dir, ".gitbay", "ci.yml"), []byte(
192 "jobs:\n usesecret:\n steps:\n - echo token=$MY_TOKEN\n nightly:\n schedule: \"0 6 * * 1\"\n steps:\n - echo scheduled ran\n"), 0o644)
193 mustGit(t, dir, env, "add", ".")
194 mustGit(t, dir, env, "commit", "-q", "-m", "secrets and schedule")
195 mustGit(t, dir, env, "push", "-q", "origin", "main")
196
197 // The push queued only the unscheduled job.
198 out, _, _ = inst.ssh(t, aliceKey, "", "build", "list", "alice/app")
199 if !strings.Contains(out, "usesecret\tpending") || strings.Contains(out, "nightly") {
200 t.Fatalf("scheduled job queued on push:\n%s", out)
201 }
202 inst.runnerOnce(t, runnerKey)
203 out, _, _ = inst.ssh(t, aliceKey, "", "build", "list", "alice/app")
204 usecretN := strings.Split(out, "\t")[0]
205 out, _, _ = inst.ssh(t, aliceKey, "", "build", "log", "alice/app", usecretN)
206 if !strings.Contains(out, "token=hunter2") {
207 t.Fatalf("secret not injected:\n%s", out)
208 }
209 // The scheduled job runs on demand via trigger.
210 if _, errOut, code := inst.ssh(t, aliceKey, "", "build", "trigger", "alice/app", "nightly"); code != 0 {
211 t.Fatalf("trigger: %s", errOut)
212 }
213 if _, _, code := inst.ssh(t, aliceKey, "", "build", "trigger", "alice/app", "nosuch"); code != 3 {
214 t.Fatal("triggered a job that does not exist")
215 }
216 inst.runnerOnce(t, runnerKey)
217 out, _, _ = inst.ssh(t, aliceKey, "", "build", "list", "alice/app")
218 if !strings.Contains(out, "nightly\tsuccess") {
219 t.Fatalf("triggered build did not run:\n%s", out)
220 }
221 // Removing the secret stops injection.
222 if _, _, code := inst.ssh(t, aliceKey, "", "repo", "secret", "remove", "alice/app", "MY_TOKEN"); code != 0 {
223 t.Fatal("secret remove failed")
224 }
225
226 // --- tag-triggered jobs ---
227 os.WriteFile(filepath.Join(dir, ".gitbay", "ci.yml"), []byte(
228 "jobs:\n test:\n steps:\n - echo branch build\n publish:\n tags: \"v*\"\n steps:\n - echo publishing $GITBAY_REF\n"), 0o644)
229 mustGit(t, dir, env, "add", ".")
230 mustGit(t, dir, env, "commit", "-q", "-m", "tag job")
231 mustGit(t, dir, env, "push", "-q", "origin", "main")
232 // The branch push queued only the branch job.
233 out, _, _ = inst.ssh(t, aliceKey, "", "build", "list", "alice/app")
234 if strings.Contains(out, "publish") {
235 t.Fatalf("tag job queued on branch push:\n%s", out)
236 }
237 // An annotated tag queues the tag job, with the peeled commit as sha.
238 mustGit(t, dir, env, "tag", "-a", "-m", "rel", "v1.0.0")
239 mustGit(t, dir, env, "push", "-q", "origin", "v1.0.0")
240 headSHA := strings.TrimSpace(mustGit(t, dir, env, "rev-parse", "HEAD"))
241 out, _, _ = inst.ssh(t, aliceKey, "", "build", "list", "alice/app")
242 if !strings.Contains(out, "publish\tpending\t"+headSHA[:10]) || !strings.Contains(out, "v1.0.0") {
243 t.Fatalf("tag build missing or unpeeled:\n%s", out)
244 }
245 // A non-matching tag queues nothing.
246 mustGit(t, dir, env, "tag", "nightly-1")
247 mustGit(t, dir, env, "push", "-q", "origin", "nightly-1")
248 out2, _, _ := inst.ssh(t, aliceKey, "", "build", "list", "alice/app")
249 if strings.Count(out2, "publish") != strings.Count(out, "publish") {
250 t.Fatalf("non-matching tag queued a build:\n%s", out2)
251 }
252 inst.runnerOnce(t, runnerKey) // branch "test" job
253 inst.runnerOnce(t, runnerKey) // tag "publish" job
254 out, _, _ = inst.ssh(t, aliceKey, "", "build", "list", "alice/app")
255 if !strings.Contains(out, "publish\tsuccess") {
256 t.Fatalf("tag build did not run:\n%s", out)
257 }
258 // schedule and tags together are refused.
259 os.WriteFile(filepath.Join(dir, ".gitbay", "ci.yml"), []byte(
260 "jobs:\n both:\n schedule: \"0 6 * * *\"\n tags: \"v*\"\n steps: [echo x]\n"), 0o644)
261 mustGit(t, dir, env, "add", ".")
262 mustGit(t, dir, env, "commit", "-q", "-m", "both triggers")
263 mustGit(t, dir, env, "push", "-q", "origin", "main")
264 shaBoth := strings.TrimSpace(mustGit(t, dir, env, "rev-parse", "HEAD"))
265 out, _, _ = inst.ssh(t, aliceKey, "", "status", "list", "alice/app", shaBoth)
266 if !strings.Contains(out, "ci/config") || !strings.Contains(out, "failure") {
267 t.Fatalf("mutually exclusive triggers not refused:\n%s", out)
268 }
269
270 // A broken ci.yml surfaces as a failed ci/config status.
271 os.WriteFile(filepath.Join(dir, ".gitbay", "ci.yml"), []byte("jobs: {bad name: {steps: [x]}}\n"), 0o644)
272 mustGit(t, dir, env, "add", ".")
273 mustGit(t, dir, env, "commit", "-q", "-m", "break config")
274 mustGit(t, dir, env, "push", "-q", "origin", "main")
275 sha2 := strings.TrimSpace(mustGit(t, dir, env, "rev-parse", "HEAD"))
276 out, _, _ = inst.ssh(t, aliceKey, "", "status", "list", "alice/app", sha2)
277 if !strings.Contains(out, "ci/config") || !strings.Contains(out, "failure") {
278 t.Fatalf("config failure status missing:\n%s", out)
279 }
280}
281
282// runnerJobs runs a runner with -jobs n until it has nothing left to do,
283// and returns its output. Unlike runnerOnce it is not bounded to one
284// build, so it is stopped when the queue drains.
285func (i *instance) runnerJobs(t *testing.T, key, repo string, jobs int) string {
286 t.Helper()
287 opts := fmt.Sprintf("-p %d -i %s -o IdentitiesOnly=yes -o StrictHostKeyChecking=no -o UserKnownHostsFile=%s -o BatchMode=yes",
288 i.port, key, filepath.Join(i.sshDir, "known_hosts"))
289 cmd := exec.Command(i.runner,
290 "-jobs", fmt.Sprint(jobs),
291 "-poll", "200ms",
292 "-isolation", "none",
293 "-remote", "git@127.0.0.1",
294 "-ssh-opts", opts,
295 "-clone-base", fmt.Sprintf("ssh://git@127.0.0.1:%d", i.port),
296 "-workdir", t.TempDir())
297 cmd.Env = append(os.Environ(), "XDG_CONFIG_HOME="+t.TempDir(), "GIT_CONFIG_NOSYSTEM=1", "GIT_CONFIG_GLOBAL=/dev/null")
298 var buf bytes.Buffer
299 cmd.Stdout, cmd.Stderr = &buf, &buf
300 if err := cmd.Start(); err != nil {
301 t.Fatal(err)
302 }
303 defer func() { cmd.Process.Kill(); cmd.Wait() }()
304
305 // Wait for every queued build to leave the pending and running states.
306 deadline := time.Now().Add(60 * time.Second)
307 for time.Now().Before(deadline) {
308 out, _, code := i.ssh(t, key, "", "build", "list", repo, "--json")
309 if code == 0 && !strings.Contains(out, `"status":"pending"`) &&
310 !strings.Contains(out, `"status":"running"`) {
311 break
312 }
313 time.Sleep(200 * time.Millisecond)
314 }
315 return buf.String()
316}
317
318// -jobs N runs N builds at once. ClaimBuild has always been a single
319// transaction that selects and updates, so several workers claiming
320// together is safe; the runner simply never used more than one (#115).
321func TestRunnerConcurrentJobs(t *testing.T) {
322 t.Parallel()
323 inst := startInstance(t)
324 inst.runner = buildRunner(t)
325 aliceKey := inst.newKey(t, "alice")
326 inst.admin(t, "admin", "user", "create", "alice", "--key", aliceKey+".pub", "--admin")
327 if _, errOut, code := inst.ssh(t, aliceKey, "", "repo", "create", "alice/app"); code != 0 {
328 t.Fatalf("repo create: %s", errOut)
329 }
330
331 // Four jobs, each sleeping longer than the poll interval, so serial
332 // execution and concurrent execution are distinguishable.
333 ci := "jobs:\n"
334 for _, name := range []string{"one", "two", "three", "four"} {
335 ci += fmt.Sprintf(" %s:\n steps:\n - sleep 1\n - echo done-%s\n", name, name)
336 }
337 env := inst.gitEnv(aliceKey)
338 work := t.TempDir()
339 mustGit(t, work, env, "clone", inst.sshURL("alice/app"), "w")
340 dir := filepath.Join(work, "w")
341 os.MkdirAll(filepath.Join(dir, ".gitbay"), 0o755)
342 os.WriteFile(filepath.Join(dir, ".gitbay", "ci.yml"), []byte(ci), 0o644)
343 mustGit(t, dir, env, "checkout", "-q", "-b", "main")
344 mustGit(t, dir, env, "add", ".")
345 mustGit(t, dir, env, "commit", "-q", "-m", "ci")
346 mustGit(t, dir, env, "push", "-q", "origin", "main")
347
348 out := inst.runnerJobs(t, aliceKey, "alice/app", 4)
349
350 listing, _, code := inst.ssh(t, aliceKey, "", "build", "list", "alice/app", "--json")
351 if code != 0 {
352 t.Fatalf("build list failed:\n%s", out)
353 }
354 for _, name := range []string{"one", "two", "three", "four"} {
355 if !strings.Contains(listing, `"job":"`+name+`"`) {
356 t.Fatalf("%s never ran:\n%s\n%s", name, listing, out)
357 }
358 }
359 if strings.Contains(listing, `"status":"pending"`) || strings.Contains(listing, `"status":"running"`) {
360 t.Fatalf("builds did not finish:\n%s", listing)
361 }
362 // Overlap, asserted from the runner's own log rather than from wall
363 // clock: elapsed time also covers four concurrent clones and the
364 // polling this test does, and would make a slow machine look serial.
365 // Every build announces itself when it starts and again when it
366 // finishes, so concurrency is "a build started before the first one
367 // finished".
368 started, firstFinish := 0, -1
369 for _, line := range strings.Split(out, "\n") {
370 switch {
371 case strings.Contains(line, "alice/app"):
372 started++
373 case strings.Contains(line, ": success"), strings.Contains(line, ": failure"):
374 if firstFinish < 0 {
375 firstFinish = started
376 }
377 }
378 }
379 if started != 4 {
380 t.Fatalf("%d builds started, want 4:\n%s", started, out)
381 }
382 if firstFinish < 2 {
383 t.Errorf("only %d build(s) had started when the first finished; -jobs 4 ran them serially\n%s",
384 firstFinish, out)
385 }
386}