internal/control/runnernext_test.go
338 lines · 12056 bytes
1package control
2
3import (
4 "bytes"
5 "fmt"
6 "os"
7 "path/filepath"
8 "strings"
9 "testing"
10
11 "gitbay.org/gitbay/internal/config"
12 "gitbay.org/gitbay/internal/protocol"
13 "gitbay.org/gitbay/internal/store"
14)
15
16// runnerCtx builds a Ctx good enough to run runRunnerNext directly: an
17// admin user (runnerSession accepts admin as well as scope "runner"), a
18// server root that matches where the test's bare repo lives.
19func runnerCtx(st *store.Store, uid int64, root string) (*Ctx, *bytes.Buffer) {
20 var out bytes.Buffer
21 fp := fmt.Sprintf("SHA256:runner-%d", uid)
22 st.AddSSHKey(uid, fp, "ssh-ed25519", []byte(fp), "full", "") // ErrDuplicateKey on reuse is fine
23 c := &Ctx{
24 User: store.User{ID: uid, Username: "ci", IsAdmin: true},
25 Scope: "full",
26 Source: fp,
27 Store: st,
28 Cfg: config.Config{Server: config.Server{Root: root, SiteURL: "https://x.test"}},
29 Stdin: strings.NewReader(""),
30 Stdout: &out,
31 Stderr: &out,
32 }
33 return c, &out
34}
35
36// A queued build's sha is orphaned by rewinding "main" past it — the shape
37// a force-push leaves, without needing an actual git-receive-pack round
38// trip. The base commit stays reachable, giving one real build behind it.
39func setupOrphanRepo(t *testing.T) (*store.Store, store.Repo, int64, string, string, string) {
40 t.Helper()
41 st, repo, uid := newQueueTestRepo(t)
42 git := gitRunner(t)
43 root := t.TempDir()
44
45 src := filepath.Join(root, "src")
46 os.MkdirAll(src, 0o755)
47 git(root, "init", "-q", "-b", "main", "src")
48 git(src, "commit", "-q", "--allow-empty", "-m", "base")
49 baseSHA := strings.TrimSpace(git(src, "rev-parse", "HEAD"))
50 git(src, "commit", "-q", "--allow-empty", "-m", "orphaned")
51 orphanSHA := strings.TrimSpace(git(src, "rev-parse", "HEAD"))
52
53 dir := RepoDir(root, repo.OwnerName, repo.Name)
54 os.MkdirAll(filepath.Dir(dir), 0o755)
55 git(root, "clone", "-q", "--bare", src, dir)
56 git(dir, "update-ref", "refs/heads/main", baseSHA)
57
58 return st, repo, uid, root, baseSHA, orphanSHA
59}
60
61// A build queued for a sha a force-push orphaned is cancelled at claim
62// time, and the runner gets the next real build instead of an impossible
63// one.
64func TestRunnerNextSkipsOrphanedBuildAndClaimsNext(t *testing.T) {
65 st, repo, uid, root, baseSHA, orphanSHA := setupOrphanRepo(t)
66
67 orphanedID, err := st.CreateBuild(repo.ID, "unit", orphanSHA, "main", "[]", "", "", true)
68 if err != nil {
69 t.Fatal(err)
70 }
71 if err := st.SetCommitStatus(repo.ID, orphanSHA, "ci/unit", "pending", "queued", "https://x.test", uid); err != nil {
72 t.Fatal(err)
73 }
74 realID, err := st.CreateBuild(repo.ID, "unit", baseSHA, "main", "[]", "", "", true)
75 if err != nil {
76 t.Fatal(err)
77 }
78
79 c, out := runnerCtx(st, uid, root)
80 code := runRunnerNext(c, nil)
81 if code != protocol.ExitOK {
82 t.Fatalf("runner next: exit %d, output:\n%s", code, out.String())
83 }
84 if !strings.Contains(out.String(), fmt.Sprintf("build %d: ", realID)) || !strings.Contains(out.String(), baseSHA[:10]) {
85 t.Fatalf("expected the real build handed out, got:\n%s", out.String())
86 }
87
88 orphaned, err := st.BuildByNumber(repo.ID, orphanedID)
89 if err != nil {
90 t.Fatal(err)
91 }
92 if orphaned.Status != "cancelled" {
93 t.Fatalf("orphaned build status = %q, want cancelled", orphaned.Status)
94 }
95 log, _ := st.BuildLog(orphaned.ID)
96 if !strings.Contains(string(log), "not reachable") {
97 t.Fatalf("orphaned build log missing the reason:\n%s", log)
98 }
99 statuses, err := st.ListCommitStatuses(repo.ID, orphanSHA)
100 if err != nil {
101 t.Fatal(err)
102 }
103 if len(statuses) != 1 || statuses[0].State == "pending" {
104 t.Fatalf("orphaned build's commit status still pending: %+v", statuses)
105 }
106
107 real, err := st.BuildByNumber(repo.ID, realID)
108 if err != nil {
109 t.Fatal(err)
110 }
111 if real.Status != "running" {
112 t.Fatalf("real build status = %q, want running (claimed)", real.Status)
113 }
114}
115
116// A build whose sha is genuinely reachable is claimed exactly as before:
117// the reachability check must never reject a healthy build.
118func TestRunnerNextClaimsReachableBuildNormally(t *testing.T) {
119 st, repo, uid, root, baseSHA, _ := setupOrphanRepo(t)
120 id, err := st.CreateBuild(repo.ID, "unit", baseSHA, "main", "[]", "", "", true)
121 if err != nil {
122 t.Fatal(err)
123 }
124
125 c, out := runnerCtx(st, uid, root)
126 code := runRunnerNext(c, nil)
127 if code != protocol.ExitOK {
128 t.Fatalf("runner next: exit %d, output:\n%s", code, out.String())
129 }
130 if strings.Contains(out.String(), "no pending builds") {
131 t.Fatalf("a reachable build was not handed out:\n%s", out.String())
132 }
133 b, err := st.BuildByNumber(repo.ID, id)
134 if err != nil {
135 t.Fatal(err)
136 }
137 if b.Status != "running" {
138 t.Fatalf("reachable build status = %q, want running", b.Status)
139 }
140 log, _ := st.BuildLog(b.ID)
141 if strings.Contains(string(log), "cancelled") {
142 t.Fatalf("a healthy build was cancelled:\n%s", log)
143 }
144}
145
146// A queue built entirely of one orphaned sha, past the loop's cap, still
147// terminates and reports no pending builds — never a spin, never an
148// error — while everything up to the cap is actually resolved rather than
149// left claimed and dangling.
150func TestRunnerNextOrphanedQueuePastCapReportsNoPendingBuilds(t *testing.T) {
151 st, repo, uid, root, _, orphanSHA := setupOrphanRepo(t)
152 total := maxOrphanSkip + 1
153 for i := 0; i < total; i++ {
154 if _, err := st.CreateBuild(repo.ID, fmt.Sprintf("job%d", i), orphanSHA, "main", "[]", "", "", true); err != nil {
155 t.Fatal(err)
156 }
157 }
158
159 c, out := runnerCtx(st, uid, root)
160 code := runRunnerNext(c, nil)
161 if code != protocol.ExitOK {
162 t.Fatalf("runner next: exit %d, output:\n%s", code, out.String())
163 }
164 if !strings.Contains(out.String(), "no pending builds") {
165 t.Fatalf("expected no pending builds, got:\n%s", out.String())
166 }
167
168 builds, err := st.ListBuilds(repo.ID, store.BuildFilter{}, total+1)
169 if err != nil {
170 t.Fatal(err)
171 }
172 var cancelled, pending, other int
173 for _, b := range builds {
174 switch b.Status {
175 case "cancelled":
176 cancelled++
177 case "pending":
178 pending++
179 default:
180 other++
181 }
182 }
183 if other != 0 {
184 t.Fatalf("a build was left claimed rather than resolved: cancelled=%d pending=%d other=%d", cancelled, pending, other)
185 }
186 if cancelled != maxOrphanSkip {
187 t.Fatalf("cancelled %d builds, want the cap of %d", cancelled, maxOrphanSkip)
188 }
189 if pending != total-maxOrphanSkip {
190 t.Fatalf("pending %d builds, want %d left behind by the cap", pending, total-maxOrphanSkip)
191 }
192}
193
194// Reachable erroring — no repository on disk at all — must not read as
195// "unreachable": the ambiguous case is claimable, never cancelled.
196func TestRunnerNextClaimsBuildWhenReachabilityCannotBeChecked(t *testing.T) {
197 st, repo, uid := newQueueTestRepo(t)
198 // No RepoDir created on disk at all: Reachable will fail to even stat
199 // the repository, which must not be read as "orphaned".
200 root := t.TempDir()
201 id, err := st.CreateBuild(repo.ID, "unit", strings.Repeat("a", 40), "main", "[]", "", "", true)
202 if err != nil {
203 t.Fatal(err)
204 }
205
206 c, out := runnerCtx(st, uid, root)
207 code := runRunnerNext(c, nil)
208 if code != protocol.ExitOK {
209 t.Fatalf("runner next: exit %d, output:\n%s", code, out.String())
210 }
211 if strings.Contains(out.String(), "no pending builds") {
212 t.Fatalf("a build was not handed out when reachability could not be checked:\n%s", out.String())
213 }
214 b, err := st.BuildByNumber(repo.ID, id)
215 if err != nil {
216 t.Fatal(err)
217 }
218 if b.Status != "running" {
219 t.Fatalf("build status = %q, want running: an unchecked build must still be claimable", b.Status)
220 }
221}
222
223// runner log records when the stream ended, so a build whose runner then
224// vanishes is failed within minutes rather than at the deadline (#179).
225func TestRunnerLogMarksStreamClosed(t *testing.T) {
226 st, repo, uid := newQueueTestRepo(t)
227 root := t.TempDir()
228 if _, err := st.CreateBuild(repo.ID, "unit", strings.Repeat("a", 40), "main", "[]", "", "", true); err != nil {
229 t.Fatal(err)
230 }
231 b, ok, err := st.ClaimBuild([]int64{repo.ID}, false)
232 if err != nil || !ok {
233 t.Fatalf("claim: %v", err)
234 }
235 c, _ := runnerCtx(st, uid, root)
236 c.Stdin = strings.NewReader("hello\n") // one chunk, then EOF: the stream ends
237 if code := runRunnerLog(c, []string{fmt.Sprint(b.ID)}); code != 0 {
238 t.Fatalf("runner log exited %d", code)
239 }
240 got, _ := st.BuildByID(b.ID)
241 if got.LogClosedAt == "" {
242 t.Fatal("log_closed_at not set when the stream ended")
243 }
244 if got.Status != "running" {
245 t.Errorf("status %s, want still running until the runner reports", got.Status)
246 }
247}
248
249// The claim says whether a build is trusted in so many words. A runner
250// must not infer it from secrets being absent: a trusted repository with
251// no secrets looks the same (#255).
252func TestRunnerNextSaysWhetherTrusted(t *testing.T) {
253 st, repo, uid, root, baseSHA, _ := setupOrphanRepo(t)
254 for _, trusted := range []bool{true, false} {
255 if _, err := st.CreateBuild(repo.ID, "unit", baseSHA, "main", "[]", "", "", trusted); err != nil {
256 t.Fatal(err)
257 }
258 c, out := runnerCtx(st, uid, root)
259 c.JSON = true
260 if code := runRunnerNext(c, []string{"--untrusted"}); code != protocol.ExitOK {
261 t.Fatalf("runner next: exit %d, output:\n%s", code, out.String())
262 }
263 want := fmt.Sprintf(`"trusted":%v`, trusted)
264 if !strings.Contains(out.String(), want) {
265 t.Fatalf("claim of a trusted=%v build lacks %s:\n%s", trusted, want, out.String())
266 }
267 }
268}
269
270// A build on the daemon's own host is given the public destination, not
271// the loopback address its runner polls. The port rides along only when
272// it is not 22, so ssh://$GITBAY_SSH/<owner>/<name>.git is a valid URL
273// either way (#260).
274func TestRunnerNextCarriesPublicSSH(t *testing.T) {
275 st, repo, uid, root, baseSHA, _ := setupOrphanRepo(t)
276 for _, tc := range []struct {
277 port int
278 want string
279 }{
280 {0, `"ssh":"git@x.test"`},
281 {22, `"ssh":"git@x.test"`},
282 {2022, `"ssh":"git@x.test:2022"`},
283 } {
284 if _, err := st.CreateBuild(repo.ID, "unit", baseSHA, "main", "[]", "", "", true); err != nil {
285 t.Fatal(err)
286 }
287 c, out := runnerCtx(st, uid, root) // site_url https://x.test
288 c.Cfg.SSH.Port = tc.port
289 c.JSON = true
290 if code := runRunnerNext(c, nil); code != protocol.ExitOK {
291 t.Fatalf("port %d: runner next: exit %d, output:\n%s", tc.port, code, out.String())
292 }
293 if !strings.Contains(out.String(), tc.want) {
294 t.Fatalf("port %d: claim lacks %s:\n%s", tc.port, tc.want, out.String())
295 }
296 }
297}
298
299// runner done records the failed step and a one-line reason (#266).
300func TestRunnerDoneRecordsFailedStep(t *testing.T) {
301 st, repo, uid, root, baseSHA, _ := setupOrphanRepo(t)
302 n, err := st.CreateBuild(repo.ID, "unit", baseSHA, "main", `["go build ./...","go test ./..."]`, "", "", true)
303 if err != nil {
304 t.Fatal(err)
305 }
306 b, ok, err := st.ClaimBuild([]int64{repo.ID}, false)
307 if err != nil || !ok {
308 t.Fatalf("claim: ok=%v err=%v", ok, err)
309 }
310 c, out := runnerCtx(st, uid, root)
311 if code := runRunnerDone(c, []string{fmt.Sprint(b.ID), "failure", "--step", "2", "--reason", "exit 1\n"}); code != protocol.ExitOK {
312 t.Fatalf("runner done: exit %d\n%s", code, out.String())
313 }
314 got, _ := st.BuildByNumber(repo.ID, n)
315 if got.Status != "failure" || got.FailedStep != 2 || got.FailedReason != "exit 1" {
316 t.Fatalf("status %s step %d reason %q", got.Status, got.FailedStep, got.FailedReason)
317 }
318}
319
320// A report with no flags — an older runner — or with a step past the
321// job's still finishes the build; the step is then recorded as 0.
322func TestRunnerDoneToleratesMissingOrBadStep(t *testing.T) {
323 st, repo, uid, root, baseSHA, _ := setupOrphanRepo(t)
324 for _, extra := range [][]string{nil, {"--step", "9"}} {
325 n, _ := st.CreateBuild(repo.ID, "unit", baseSHA, "main", `["true"]`, "", "", true)
326 b, ok, err := st.ClaimBuild([]int64{repo.ID}, false)
327 if err != nil || !ok {
328 t.Fatalf("claim: ok=%v err=%v", ok, err)
329 }
330 c, out := runnerCtx(st, uid, root)
331 if code := runRunnerDone(c, append([]string{fmt.Sprint(b.ID), "failure"}, extra...)); code != protocol.ExitOK {
332 t.Fatalf("runner done %v: exit %d\n%s", extra, code, out.String())
333 }
334 if got, _ := st.BuildByNumber(repo.ID, n); got.Status != "failure" || got.FailedStep != 0 {
335 t.Fatalf("%v: status %s step %d", extra, got.Status, got.FailedStep)
336 }
337 }
338}