e2e/build_cancel_test.go
175 lines · 8129 bytes
1package e2e
2
3import (
4 "fmt"
5 "os"
6 "os/exec"
7 "path/filepath"
8 "strings"
9 "testing"
10 "time"
11)
12
13// build cancel withdraws a queued build; a running one is the runner's.
14// Cancelling a duplicate of a commit that already passed puts that
15// result back on the commit.
16func TestBuildCancel(t *testing.T) {
17 t.Parallel()
18 inst := startInstance(t)
19 inst.runner = buildRunner(t)
20 aliceKey := inst.newKey(t, "alice")
21 bobKey := inst.newKey(t, "bob")
22 runnerKey := inst.newKey(t, "ci")
23 inst.admin(t, "admin", "user", "create", "alice", "--key", aliceKey+".pub")
24 inst.admin(t, "admin", "user", "create", "bob", "--key", bobKey+".pub")
25 inst.admin(t, "admin", "user", "create", "ci", "--key", runnerKey+".pub", "--admin")
26 if _, _, code := inst.ssh(t, aliceKey, "", "repo", "create", "alice/app"); code != 0 {
27 t.Fatal("repo create failed")
28 }
29 work := t.TempDir()
30 env := inst.gitEnv(aliceKey)
31 mustGit(t, work, env, "clone", inst.sshURL("alice/app"), "w")
32 dir := filepath.Join(work, "w")
33 os.MkdirAll(filepath.Join(dir, ".gitbay"), 0o755)
34 os.WriteFile(filepath.Join(dir, ".gitbay", "ci.yml"), []byte("jobs:\n unit:\n steps:\n - echo fine\n"), 0o644)
35 mustGit(t, dir, env, "checkout", "-q", "-b", "main")
36 mustGit(t, dir, env, "add", ".")
37 mustGit(t, dir, env, "commit", "-q", "-m", "ci")
38 mustGit(t, dir, env, "push", "-q", "origin", "main")
39 sha := strings.Fields(strings.TrimSpace(mustGit(t, dir, env, "ls-remote", "origin", "refs/heads/main")))[0]
40
41 // Build 1 is queued. A reader cannot cancel it; the owner can.
42 if _, _, code := inst.ssh(t, bobKey, "", "build", "cancel", "alice/app", "1"); code != 4 {
43 t.Fatal("reader cancelled a build")
44 }
45 if out, errOut, code := inst.ssh(t, aliceKey, "", "build", "cancel", "alice/app", "1"); code != 0 || !strings.Contains(out, "cancelled alice/app build 1") {
46 t.Fatalf("cancel: exit %d %s%s", code, out, errOut)
47 }
48 if list := inst.buildList(t, aliceKey); !strings.Contains(list, "unit\tcancelled") {
49 t.Fatalf("build list after cancel:\n%s", list)
50 }
51 if out, _, _ := inst.ssh(t, aliceKey, "", "build", "log", "alice/app", "1"); !strings.Contains(out, "cancelled by alice") {
52 t.Fatalf("log after cancel:\n%s", out)
53 }
54 if st, _, _ := inst.ssh(t, aliceKey, "", "status", "list", "alice/app", sha); !strings.Contains(st, "error") || !strings.Contains(st, "cancelled") {
55 t.Fatalf("status after cancel:\n%s", st)
56 }
57 if _, _, code := inst.ssh(t, aliceKey, "", "build", "cancel", "alice/app", "1"); code != 2 {
58 t.Fatal("cancelled a cancelled build")
59 }
60 // Nothing for a runner to claim.
61 if out := inst.runnerOnce(t, runnerKey); strings.Contains(out, "unit") && !strings.Contains(out, "no pending") {
62 t.Fatalf("runner picked up a cancelled build:\n%s", out)
63 }
64 if status, _ := inst.get(t, "/alice/app/badge/build.svg"); status != 200 {
65 t.Fatalf("badge after cancel: %d", status)
66 }
67
68 // Build 2: triggered, run to success. Build 3: the same commit queued
69 // again; cancelling it restores the passed result on the commit.
70 if _, _, code := inst.ssh(t, aliceKey, "", "build", "trigger", "alice/app", "unit"); code != 0 {
71 t.Fatal("trigger failed")
72 }
73 inst.runnerOnce(t, runnerKey)
74 if list := inst.buildList(t, aliceKey); !strings.Contains(list, "unit\tsuccess") {
75 t.Fatalf("build 2 did not pass:\n%s", list)
76 }
77 if _, _, code := inst.ssh(t, aliceKey, "", "build", "trigger", "alice/app", "unit"); code != 0 {
78 t.Fatal("second trigger failed")
79 }
80 if st, _, _ := inst.ssh(t, aliceKey, "", "status", "list", "alice/app", sha); !strings.Contains(st, "pending") {
81 t.Fatalf("status before cancelling the duplicate:\n%s", st)
82 }
83 if _, errOut, code := inst.ssh(t, aliceKey, "", "build", "cancel", "alice/app", "3"); code != 0 {
84 t.Fatalf("cancel duplicate: %s", errOut)
85 }
86 if st, _, _ := inst.ssh(t, aliceKey, "", "status", "list", "alice/app", sha); !strings.Contains(st, "success") || !strings.Contains(st, "passed in build 2") {
87 t.Fatalf("status after cancelling the duplicate:\n%s", st)
88 }
89}
90
91// Cancelling a running build ends it at the runner within seconds: the
92// server closes the log session, the runner kills the step, and its late
93// report lands on a row that already says cancelled.
94func TestBuildCancelRunning(t *testing.T) {
95 t.Parallel()
96 inst := startInstance(t)
97 inst.runner = buildRunner(t)
98 aliceKey := inst.newKey(t, "alice")
99 runnerKey := inst.newKey(t, "ci")
100 inst.admin(t, "admin", "user", "create", "alice", "--key", aliceKey+".pub")
101 inst.admin(t, "admin", "user", "create", "ci", "--key", runnerKey+".pub", "--admin")
102 if _, _, code := inst.ssh(t, aliceKey, "", "repo", "create", "alice/slow"); code != 0 {
103 t.Fatal("repo create failed")
104 }
105 work := t.TempDir()
106 env := inst.gitEnv(aliceKey)
107 mustGit(t, work, env, "clone", inst.sshURL("alice/slow"), "w")
108 dir := filepath.Join(work, "w")
109 os.MkdirAll(filepath.Join(dir, ".gitbay"), 0o755)
110 os.WriteFile(filepath.Join(dir, ".gitbay", "ci.yml"), []byte("jobs:\n slow:\n steps:\n - echo starting\n - sleep 120 & wait\n - echo never\n"), 0o644)
111 mustGit(t, dir, env, "checkout", "-q", "-b", "main")
112 mustGit(t, dir, env, "add", ".")
113 mustGit(t, dir, env, "commit", "-q", "-m", "slow")
114 mustGit(t, dir, env, "push", "-q", "origin", "main")
115 sha := strings.Fields(strings.TrimSpace(mustGit(t, dir, env, "ls-remote", "origin", "refs/heads/main")))[0]
116 slowList := func() string {
117 out, _, _ := inst.ssh(t, aliceKey, "", "build", "list", "alice/slow")
118 return out
119 }
120
121 // A real runner, in the background, claims the build and sits in sleep.
122 // The step forks sleep as a child of the shell, the way dash runs
123 // every command, so the cancel must reach past the shell to end it.
124 opts := fmt.Sprintf("-p %d -i %s -o IdentitiesOnly=yes -o StrictHostKeyChecking=no -o UserKnownHostsFile=%s -o BatchMode=yes",
125 inst.port, runnerKey, filepath.Join(inst.sshDir, "known_hosts"))
126 runner := exec.Command(inst.runner, "-once", "-remote", "git@127.0.0.1", "-ssh-opts", opts,
127 "-isolation", "none",
128 "-clone-base", fmt.Sprintf("ssh://git@127.0.0.1:%d", inst.port), "-workdir", t.TempDir())
129 runner.Env = append(os.Environ(), "XDG_CONFIG_HOME="+t.TempDir(), "GIT_CONFIG_NOSYSTEM=1", "GIT_CONFIG_GLOBAL=/dev/null")
130 var runnerOut strings.Builder
131 runner.Stdout, runner.Stderr = &runnerOut, &runnerOut
132 if err := runner.Start(); err != nil {
133 t.Fatal(err)
134 }
135 exited := make(chan error, 1)
136 go func() { exited <- runner.Wait() }()
137 t.Cleanup(func() { runner.Process.Kill() })
138 deadline := time.Now().Add(30 * time.Second)
139 for !strings.Contains(slowList(), "slow\trunning") {
140 if time.Now().After(deadline) {
141 t.Fatalf("runner never claimed the build:\n%s\nbuild list:\n%s", runnerOut.String(), slowList())
142 }
143 time.Sleep(200 * time.Millisecond)
144 }
145 started := time.Now()
146 out, errOut, code := inst.ssh(t, aliceKey, "", "build", "cancel", "alice/slow", "1")
147 if code != 0 || !strings.Contains(out, "the runner stops at its next check") {
148 t.Fatalf("cancel running: exit %d %s%s", code, out, errOut)
149 }
150 select {
151 case <-exited:
152 case <-time.After(60 * time.Second):
153 t.Fatalf("runner still running 60s after cancel:\n%s", runnerOut.String())
154 }
155 // Well inside the step's 120s sleep: the runner stopped because it
156 // was told to, not because the step ended.
157 if took := time.Since(started); took > 45*time.Second {
158 t.Fatalf("runner took %s to stop", took)
159 }
160 if !strings.Contains(runnerOut.String(), "cancelled") {
161 t.Fatalf("runner did not say it was cancelled:\n%s", runnerOut.String())
162 }
163 // The row, log and status say cancelled, and the runner's late report
164 // changed none of them.
165 if list := slowList(); !strings.Contains(list, "slow\tcancelled") {
166 t.Fatalf("build after cancel:\n%s", list)
167 }
168 log, _, _ := inst.ssh(t, aliceKey, "", "build", "log", "alice/slow", "1")
169 if !strings.Contains(log, "starting") || !strings.Contains(log, "cancelled by alice while running") || strings.Contains(log, "never") {
170 t.Fatalf("log after cancel:\n%s", log)
171 }
172 if st, _, _ := inst.ssh(t, aliceKey, "", "status", "list", "alice/slow", sha); !strings.Contains(st, "error") || !strings.Contains(st, "cancelled") {
173 t.Fatalf("status after cancel:\n%s", st)
174 }
175}