e2e/ci_test.go
376 lines · 15944 bytes
1package e2e
2
3import (
4 "bytes"
5 "fmt"
6 "os"
7 "os/exec"
8 "path/filepath"
9 "strings"
10 "testing"
11 "time"
12)
13
14func buildRunner(t *testing.T) string {
15 t.Helper()
16 bin := filepath.Join(t.TempDir(), "gitbay-runner")
17 cmd := exec.Command("go", "build", "-o", bin, "gitbay.org/gitbay/cmd/gitbay-runner")
18 cmd.Dir = ".."
19 if out, err := cmd.CombinedOutput(); err != nil {
20 t.Fatalf("build gitbay-runner: %v\n%s", err, out)
21 }
22 return bin
23}
24
25// runnerOnce processes at most one pending build with the given key.
26func (i *instance) runnerOnce(t *testing.T, key string) string {
27 t.Helper()
28 opts := fmt.Sprintf("-p %d -i %s -o IdentitiesOnly=yes -o StrictHostKeyChecking=no -o UserKnownHostsFile=%s -o BatchMode=yes",
29 i.port, key, filepath.Join(i.sshDir, "known_hosts"))
30 cmd := exec.Command(i.runner, "-once",
31 "-remote", "git@127.0.0.1",
32 "-ssh-opts", opts,
33 "-clone-base", fmt.Sprintf("ssh://git@127.0.0.1:%d", i.port),
34 "-workdir", t.TempDir())
35 cmd.Env = append(os.Environ(), "GIT_CONFIG_NOSYSTEM=1", "GIT_CONFIG_GLOBAL=/dev/null")
36 out, err := cmd.CombinedOutput()
37 if err != nil {
38 t.Fatalf("runner: %v\n%s", err, out)
39 }
40 return string(out)
41}
42
43// A failed build mails the repo owner with the log tail; green builds
44// stay silent.
45func TestBuildFailureMail(t *testing.T) {
46 smtp := startFakeSMTP(t)
47 inst := startInstanceWith(t, fmt.Sprintf(
48 "[mail]\nsmtp_host = %q\nfrom = \"noreply@gitbay.test\"\n", smtp.addr))
49 inst.runner = buildRunner(t)
50 aliceKey := inst.newKey(t, "alice")
51 inst.admin(t, "admin", "user", "create", "alice", "--key", aliceKey+".pub",
52 "--email", "alice@example.test", "--verified")
53 runnerKey := inst.newKey(t, "ci")
54 inst.admin(t, "admin", "user", "create", "ci", "--key", runnerKey+".pub", "--admin")
55
56 if _, errOut, code := inst.ssh(t, aliceKey, "", "repo", "create", "alice/app"); code != 0 {
57 t.Fatalf("repo create: %s", errOut)
58 }
59 work := t.TempDir()
60 env := inst.gitEnv(aliceKey)
61 mustGit(t, work, env, "clone", inst.sshURL("alice/app"), "w")
62 dir := filepath.Join(work, "w")
63 os.MkdirAll(filepath.Join(dir, ".gitbay"), 0o755)
64 os.WriteFile(filepath.Join(dir, ".gitbay", "ci.yml"), []byte(
65 "jobs:\n ok:\n steps:\n - echo fine\n broken:\n steps:\n - echo the dataset went stale && false\n"), 0o644)
66 mustGit(t, dir, env, "checkout", "-q", "-b", "main")
67 mustGit(t, dir, env, "add", ".")
68 mustGit(t, dir, env, "commit", "-q", "-m", "base")
69 mustGit(t, dir, env, "push", "-q", "origin", "main")
70
71 inst.runnerOnce(t, runnerKey) // broken (sorts first)
72 inst.runnerOnce(t, runnerKey) // ok
73 mail := smtp.waitFor(t, "alice@example.test", "failed")
74 if !strings.Contains(mail, "broken") || !strings.Contains(mail, "the dataset went stale") ||
75 !strings.Contains(mail, "/alice/app/builds/") {
76 t.Fatalf("failure mail missing detail:\n%s", mail)
77 }
78 // Only the failure mailed: no message mentions the green job.
79 for _, m := range smtp.mailTo("alice@example.test") {
80 if strings.Contains(m, "build") && strings.Contains(m, " ok ") && strings.Contains(m, "failed") == false {
81 t.Fatalf("green build mailed:\n%s", m)
82 }
83 }
84}
85
86func TestCI(t *testing.T) {
87 inst := startInstance(t)
88 inst.runner = buildRunner(t)
89 aliceKey := inst.newKey(t, "alice")
90 inst.admin(t, "admin", "user", "create", "alice", "--key", aliceKey+".pub")
91 runnerKey := inst.newKey(t, "ci")
92 inst.admin(t, "admin", "user", "create", "ci", "--key", runnerKey+".pub", "--admin")
93
94 if _, errOut, code := inst.ssh(t, aliceKey, "", "repo", "create", "alice/app"); code != 0 {
95 t.Fatalf("repo create: %s", errOut)
96 }
97 work := t.TempDir()
98 env := inst.gitEnv(aliceKey)
99 mustGit(t, work, env, "clone", inst.sshURL("alice/app"), "w")
100 dir := filepath.Join(work, "w")
101 os.MkdirAll(filepath.Join(dir, ".gitbay"), 0o755)
102 os.WriteFile(filepath.Join(dir, ".gitbay", "ci.yml"), []byte(
103 "jobs:\n ok:\n steps:\n - echo hello from $GITBAY_JOB\n broken:\n steps:\n - \"false\"\n"), 0o644)
104 os.WriteFile(filepath.Join(dir, "f.txt"), []byte("x\n"), 0o644)
105 mustGit(t, dir, env, "checkout", "-q", "-b", "main")
106 mustGit(t, dir, env, "add", ".")
107 mustGit(t, dir, env, "commit", "-q", "-m", "base")
108 mustGit(t, dir, env, "push", "-q", "origin", "main")
109 sha := strings.TrimSpace(mustGit(t, dir, env, "rev-parse", "HEAD"))
110
111 // The push queued one pending build per job, with pending statuses.
112 out, _, _ := inst.ssh(t, aliceKey, "", "build", "list", "alice/app")
113 if !strings.Contains(out, "broken\tpending") || !strings.Contains(out, "ok\tpending") {
114 t.Fatalf("builds not queued:\n%s", out)
115 }
116 out, _, _ = inst.ssh(t, aliceKey, "", "status", "list", "alice/app", sha)
117 if !strings.Contains(out, "ci/ok") || !strings.Contains(out, "pending") {
118 t.Fatalf("pending statuses missing:\n%s", out)
119 }
120
121 // Non-admins cannot claim jobs.
122 if _, _, code := inst.ssh(t, aliceKey, "", "runner", "next"); code != 4 {
123 t.Fatalf("non-admin claimed a build: exit %d", code)
124 }
125
126 // The runner processes both jobs ("broken" sorts first).
127 inst.runnerOnce(t, runnerKey)
128 inst.runnerOnce(t, runnerKey)
129
130 out, _, _ = inst.ssh(t, aliceKey, "", "build", "list", "alice/app")
131 if !strings.Contains(out, "ok\tsuccess") || !strings.Contains(out, "broken\tfailure") {
132 t.Fatalf("build outcomes wrong:\n%s", out)
133 }
134 // Logs captured the step output and the failure.
135 var okN, brokenN string
136 for _, l := range strings.Split(strings.TrimSpace(out), "\n") {
137 f := strings.Split(l, "\t")
138 if f[1] == "ok" {
139 okN = f[0]
140 } else {
141 brokenN = f[0]
142 }
143 }
144 out, _, _ = inst.ssh(t, aliceKey, "", "build", "log", "alice/app", okN)
145 if !strings.Contains(out, "hello from ok") {
146 t.Fatalf("ok log:\n%s", out)
147 }
148 out, _, _ = inst.ssh(t, aliceKey, "", "build", "log", "alice/app", brokenN)
149 if !strings.Contains(out, "step failed") {
150 t.Fatalf("broken log:\n%s", out)
151 }
152 // Statuses resolved, with target URLs pointing at the build pages.
153 out, _, _ = inst.ssh(t, aliceKey, "", "status", "list", "alice/app", sha, "--json")
154 if !strings.Contains(out, `"ci/ok","state":"success"`) && !strings.Contains(out, `"state":"success"`) {
155 t.Fatalf("status not success:\n%s", out)
156 }
157 if !strings.Contains(out, "/alice/app/builds/") {
158 t.Fatalf("status target url missing:\n%s", out)
159 }
160
161 // Web: list page and log page.
162 status, body := inst.get(t, "/alice/app/builds")
163 if status != 200 || !strings.Contains(body, "ok") || !strings.Contains(body, "failure") {
164 t.Fatalf("builds page: %d\n%s", status, body)
165 }
166 if _, body = inst.get(t, "/alice/app/builds/"+okN); !strings.Contains(body, "hello from ok") {
167 t.Fatalf("build log page:\n%s", body)
168 }
169
170 // --- secrets: stdin in, names-only out, injected into the build env ---
171 if _, errOut, code := inst.ssh(t, aliceKey, "hunter2\n", "repo", "secret", "set", "alice/app", "MY_TOKEN"); code != 0 {
172 t.Fatalf("secret set: %s", errOut)
173 }
174 if _, _, code := inst.ssh(t, aliceKey, "x\n", "repo", "secret", "set", "alice/app", "bad-name"); code != 2 {
175 t.Fatal("bad secret name accepted")
176 }
177 out, _, _ = inst.ssh(t, aliceKey, "", "repo", "secret", "list", "alice/app")
178 if !strings.Contains(out, "MY_TOKEN") || strings.Contains(out, "hunter2") {
179 t.Fatalf("secret list leaked or missed: %s", out)
180 }
181
182 // --- schedules and manual trigger ---
183 os.WriteFile(filepath.Join(dir, ".gitbay", "ci.yml"), []byte(
184 "jobs:\n usesecret:\n steps:\n - echo token=$MY_TOKEN\n nightly:\n schedule: \"0 6 * * 1\"\n steps:\n - echo scheduled ran\n"), 0o644)
185 mustGit(t, dir, env, "add", ".")
186 mustGit(t, dir, env, "commit", "-q", "-m", "secrets and schedule")
187 mustGit(t, dir, env, "push", "-q", "origin", "main")
188
189 // The push queued only the unscheduled job.
190 out, _, _ = inst.ssh(t, aliceKey, "", "build", "list", "alice/app")
191 if !strings.Contains(out, "usesecret\tpending") || strings.Contains(out, "nightly") {
192 t.Fatalf("scheduled job queued on push:\n%s", out)
193 }
194 inst.runnerOnce(t, runnerKey)
195 out, _, _ = inst.ssh(t, aliceKey, "", "build", "list", "alice/app")
196 usecretN := strings.Split(out, "\t")[0]
197 out, _, _ = inst.ssh(t, aliceKey, "", "build", "log", "alice/app", usecretN)
198 if !strings.Contains(out, "token=hunter2") {
199 t.Fatalf("secret not injected:\n%s", out)
200 }
201 // The scheduled job runs on demand via trigger.
202 if _, errOut, code := inst.ssh(t, aliceKey, "", "build", "trigger", "alice/app", "nightly"); code != 0 {
203 t.Fatalf("trigger: %s", errOut)
204 }
205 if _, _, code := inst.ssh(t, aliceKey, "", "build", "trigger", "alice/app", "nosuch"); code != 3 {
206 t.Fatal("triggered a job that does not exist")
207 }
208 inst.runnerOnce(t, runnerKey)
209 out, _, _ = inst.ssh(t, aliceKey, "", "build", "list", "alice/app")
210 if !strings.Contains(out, "nightly\tsuccess") {
211 t.Fatalf("triggered build did not run:\n%s", out)
212 }
213 // Removing the secret stops injection.
214 if _, _, code := inst.ssh(t, aliceKey, "", "repo", "secret", "remove", "alice/app", "MY_TOKEN"); code != 0 {
215 t.Fatal("secret remove failed")
216 }
217
218 // --- tag-triggered jobs ---
219 os.WriteFile(filepath.Join(dir, ".gitbay", "ci.yml"), []byte(
220 "jobs:\n test:\n steps:\n - echo branch build\n publish:\n tags: \"v*\"\n steps:\n - echo publishing $GITBAY_REF\n"), 0o644)
221 mustGit(t, dir, env, "add", ".")
222 mustGit(t, dir, env, "commit", "-q", "-m", "tag job")
223 mustGit(t, dir, env, "push", "-q", "origin", "main")
224 // The branch push queued only the branch job.
225 out, _, _ = inst.ssh(t, aliceKey, "", "build", "list", "alice/app")
226 if strings.Contains(out, "publish") {
227 t.Fatalf("tag job queued on branch push:\n%s", out)
228 }
229 // An annotated tag queues the tag job, with the peeled commit as sha.
230 mustGit(t, dir, env, "tag", "-a", "-m", "rel", "v1.0.0")
231 mustGit(t, dir, env, "push", "-q", "origin", "v1.0.0")
232 headSHA := strings.TrimSpace(mustGit(t, dir, env, "rev-parse", "HEAD"))
233 out, _, _ = inst.ssh(t, aliceKey, "", "build", "list", "alice/app")
234 if !strings.Contains(out, "publish\tpending\t"+headSHA[:10]) || !strings.Contains(out, "v1.0.0") {
235 t.Fatalf("tag build missing or unpeeled:\n%s", out)
236 }
237 // A non-matching tag queues nothing.
238 mustGit(t, dir, env, "tag", "nightly-1")
239 mustGit(t, dir, env, "push", "-q", "origin", "nightly-1")
240 out2, _, _ := inst.ssh(t, aliceKey, "", "build", "list", "alice/app")
241 if strings.Count(out2, "publish") != strings.Count(out, "publish") {
242 t.Fatalf("non-matching tag queued a build:\n%s", out2)
243 }
244 inst.runnerOnce(t, runnerKey) // branch "test" job
245 inst.runnerOnce(t, runnerKey) // tag "publish" job
246 out, _, _ = inst.ssh(t, aliceKey, "", "build", "list", "alice/app")
247 if !strings.Contains(out, "publish\tsuccess") {
248 t.Fatalf("tag build did not run:\n%s", out)
249 }
250 // schedule and tags together are refused.
251 os.WriteFile(filepath.Join(dir, ".gitbay", "ci.yml"), []byte(
252 "jobs:\n both:\n schedule: \"0 6 * * *\"\n tags: \"v*\"\n steps: [echo x]\n"), 0o644)
253 mustGit(t, dir, env, "add", ".")
254 mustGit(t, dir, env, "commit", "-q", "-m", "both triggers")
255 mustGit(t, dir, env, "push", "-q", "origin", "main")
256 shaBoth := strings.TrimSpace(mustGit(t, dir, env, "rev-parse", "HEAD"))
257 out, _, _ = inst.ssh(t, aliceKey, "", "status", "list", "alice/app", shaBoth)
258 if !strings.Contains(out, "ci/config") || !strings.Contains(out, "failure") {
259 t.Fatalf("mutually exclusive triggers not refused:\n%s", out)
260 }
261
262 // A broken ci.yml surfaces as a failed ci/config status.
263 os.WriteFile(filepath.Join(dir, ".gitbay", "ci.yml"), []byte("jobs: {bad name: {steps: [x]}}\n"), 0o644)
264 mustGit(t, dir, env, "add", ".")
265 mustGit(t, dir, env, "commit", "-q", "-m", "break config")
266 mustGit(t, dir, env, "push", "-q", "origin", "main")
267 sha2 := strings.TrimSpace(mustGit(t, dir, env, "rev-parse", "HEAD"))
268 out, _, _ = inst.ssh(t, aliceKey, "", "status", "list", "alice/app", sha2)
269 if !strings.Contains(out, "ci/config") || !strings.Contains(out, "failure") {
270 t.Fatalf("config failure status missing:\n%s", out)
271 }
272}
273
274// runnerJobs runs a runner with -jobs n until it has nothing left to do,
275// and returns its output. Unlike runnerOnce it is not bounded to one
276// build, so it is stopped when the queue drains.
277func (i *instance) runnerJobs(t *testing.T, key, repo string, jobs int) string {
278 t.Helper()
279 opts := fmt.Sprintf("-p %d -i %s -o IdentitiesOnly=yes -o StrictHostKeyChecking=no -o UserKnownHostsFile=%s -o BatchMode=yes",
280 i.port, key, filepath.Join(i.sshDir, "known_hosts"))
281 cmd := exec.Command(i.runner,
282 "-jobs", fmt.Sprint(jobs),
283 "-poll", "200ms",
284 "-remote", "git@127.0.0.1",
285 "-ssh-opts", opts,
286 "-clone-base", fmt.Sprintf("ssh://git@127.0.0.1:%d", i.port),
287 "-workdir", t.TempDir())
288 cmd.Env = append(os.Environ(), "GIT_CONFIG_NOSYSTEM=1", "GIT_CONFIG_GLOBAL=/dev/null")
289 var buf bytes.Buffer
290 cmd.Stdout, cmd.Stderr = &buf, &buf
291 if err := cmd.Start(); err != nil {
292 t.Fatal(err)
293 }
294 defer func() { cmd.Process.Kill(); cmd.Wait() }()
295
296 // Wait for every queued build to leave the pending and running states.
297 deadline := time.Now().Add(60 * time.Second)
298 for time.Now().Before(deadline) {
299 out, _, code := i.ssh(t, key, "", "build", "list", repo, "--json")
300 if code == 0 && !strings.Contains(out, `"status":"pending"`) &&
301 !strings.Contains(out, `"status":"running"`) {
302 break
303 }
304 time.Sleep(200 * time.Millisecond)
305 }
306 return buf.String()
307}
308
309// -jobs N runs N builds at once. ClaimBuild has always been a single
310// transaction that selects and updates, so several workers claiming
311// together is safe; the runner simply never used more than one (#115).
312func TestRunnerConcurrentJobs(t *testing.T) {
313 inst := startInstance(t)
314 inst.runner = buildRunner(t)
315 aliceKey := inst.newKey(t, "alice")
316 inst.admin(t, "admin", "user", "create", "alice", "--key", aliceKey+".pub", "--admin")
317 if _, errOut, code := inst.ssh(t, aliceKey, "", "repo", "create", "alice/app"); code != 0 {
318 t.Fatalf("repo create: %s", errOut)
319 }
320
321 // Four jobs, each sleeping longer than the poll interval, so serial
322 // execution and concurrent execution are distinguishable.
323 ci := "jobs:\n"
324 for _, name := range []string{"one", "two", "three", "four"} {
325 ci += fmt.Sprintf(" %s:\n steps:\n - sleep 1\n - echo done-%s\n", name, name)
326 }
327 env := inst.gitEnv(aliceKey)
328 work := t.TempDir()
329 mustGit(t, work, env, "clone", inst.sshURL("alice/app"), "w")
330 dir := filepath.Join(work, "w")
331 os.MkdirAll(filepath.Join(dir, ".gitbay"), 0o755)
332 os.WriteFile(filepath.Join(dir, ".gitbay", "ci.yml"), []byte(ci), 0o644)
333 mustGit(t, dir, env, "checkout", "-q", "-b", "main")
334 mustGit(t, dir, env, "add", ".")
335 mustGit(t, dir, env, "commit", "-q", "-m", "ci")
336 mustGit(t, dir, env, "push", "-q", "origin", "main")
337
338 out := inst.runnerJobs(t, aliceKey, "alice/app", 4)
339
340 listing, _, code := inst.ssh(t, aliceKey, "", "build", "list", "alice/app", "--json")
341 if code != 0 {
342 t.Fatalf("build list failed:\n%s", out)
343 }
344 for _, name := range []string{"one", "two", "three", "four"} {
345 if !strings.Contains(listing, `"job":"`+name+`"`) {
346 t.Fatalf("%s never ran:\n%s\n%s", name, listing, out)
347 }
348 }
349 if strings.Contains(listing, `"status":"pending"`) || strings.Contains(listing, `"status":"running"`) {
350 t.Fatalf("builds did not finish:\n%s", listing)
351 }
352 // Overlap, asserted from the runner's own log rather than from wall
353 // clock: elapsed time also covers four concurrent clones and the
354 // polling this test does, and would make a slow machine look serial.
355 // Every build announces itself when it starts and again when it
356 // finishes, so concurrency is "a build started before the first one
357 // finished".
358 started, firstFinish := 0, -1
359 for _, line := range strings.Split(out, "\n") {
360 switch {
361 case strings.Contains(line, "alice/app"):
362 started++
363 case strings.Contains(line, ": success"), strings.Contains(line, ": failure"):
364 if firstFinish < 0 {
365 firstFinish = started
366 }
367 }
368 }
369 if started != 4 {
370 t.Fatalf("%d builds started, want 4:\n%s", started, out)
371 }
372 if firstFinish < 2 {
373 t.Errorf("only %d build(s) had started when the first finished; -jobs 4 ran them serially\n%s",
374 firstFinish, out)
375 }
376}