internal/control/build.go
855 lines · 32509 bytes
1package control
2
3import (
4 "encoding/json"
5 "errors"
6 "fmt"
7 "io"
8 "log/slog"
9 "regexp"
10 "strconv"
11 "strings"
12 "time"
13
14 "gitbay.org/gitbay/internal/ci"
15 "gitbay.org/gitbay/internal/gitutil"
16 "gitbay.org/gitbay/internal/policy"
17 "gitbay.org/gitbay/internal/protocol"
18 "gitbay.org/gitbay/internal/store"
19)
20
21func init() {
22 register(Command{Path: []string{"build", "list"},
23 Summary: "list recent builds",
24 Usage: "build list <owner/name>", ReadOnly: true, Run: runBuildList})
25 register(Command{Path: []string{"build", "show"},
26 Summary: "show one build",
27 Usage: "build show <owner/name> <n>", ReadOnly: true, Run: runBuildShow})
28 register(Command{Path: []string{"build", "log"},
29 Summary: "print a build's log",
30 Usage: "build log <owner/name> <n>", ReadOnly: true, Run: runBuildLog})
31
32 register(Command{Path: []string{"build", "jobs"},
33 Summary: "list the jobs a trigger can name",
34 Usage: "build jobs <owner/name>", ReadOnly: true, Run: runBuildJobs})
35
36 register(Command{Path: []string{"build", "cancel"},
37 Summary: "withdraw a queued build before a runner claims it",
38 Usage: "build cancel <owner/name> <n>", Run: runBuildCancel})
39 register(Command{Path: []string{"build", "trigger"},
40 Summary: "queue a job now (scheduled or not)",
41 Usage: "build trigger <owner/name> <job>", Run: runBuildTrigger})
42 // Secrets: set over stdin, listed by name only, injected into the
43 // repo's builds as environment variables. Same discipline as mirror
44 // tokens — the value never appears in argv, logs, or output.
45 register(Command{Path: []string{"repo", "secret", "set"},
46 Summary: "set a build secret",
47 Usage: "repo secret set <owner/name> <NAME> (value on stdin)",
48 ReadsStdin: true, SSHOnly: true, Run: runSecretSet})
49 register(Command{Path: []string{"repo", "secret", "remove"},
50 Summary: "remove a build secret",
51 Usage: "repo secret remove <owner/name> <NAME>", Run: runSecretRemove})
52 register(Command{Path: []string{"repo", "secret", "list"},
53 Summary: "list build secret names",
54 Usage: "repo secret list <owner/name>", ReadOnly: true, Run: runSecretList})
55
56 // Runner commands: the claim/report loop for gitbay-runner. A runner
57 // executes arbitrary repo code, so handing out jobs is the instance
58 // operator's call: a key added with --scope runner, which the
59 // dispatcher confines to these three commands and read-only git, or
60 // an admin key, which a runner host should not hold (#92).
61 register(Command{Path: []string{"runner", "next"},
62 Summary: "claim the oldest pending build this key may run (runner protocol)",
63 Usage: "runner next [--untrusted] [<owner/name>...]", SSHOnly: true, Run: runRunnerNext})
64 register(Command{Path: []string{"runner", "log"},
65 Summary: "append a build's log from stdin",
66 Usage: "runner log <build-id>", SSHOnly: true, ReadsStdin: true, Run: runRunnerLog})
67 register(Command{Path: []string{"runner", "done"},
68 Summary: "finish a build",
69 Usage: "runner done <build-id> success|failure", SSHOnly: true, Run: runRunnerDone})
70}
71
72type BuildOut struct {
73 Number int64 `json:"number"`
74 Job string `json:"job"`
75 Status string `json:"status"`
76 SHA string `json:"sha"`
77 Ref string `json:"ref"`
78 CreatedAt string `json:"created_at"`
79 FinishedAt string `json:"finished_at,omitempty"`
80}
81
82func buildToOut(b store.Build) BuildOut {
83 return BuildOut{b.Number, b.Job, b.Status, b.SHA, b.Ref, b.CreatedAt, b.FinishedAt}
84}
85
86func buildRef(c *Ctx, args []string) (store.Repo, store.Build, int) {
87 if len(args) != 2 {
88 return store.Repo{}, store.Build{}, c.fail(protocol.ExitUsage, "expected <owner/name> <number>")
89 }
90 repo, code := resolveRepo(c, args[0], policy.CanRead)
91 if code >= 0 {
92 return repo, store.Build{}, code
93 }
94 n, err := strconv.ParseInt(args[1], 10, 64)
95 if err != nil {
96 return repo, store.Build{}, c.fail(protocol.ExitUsage, "bad build number %q", args[1])
97 }
98 b, err := c.Store.BuildByNumber(repo.ID, n)
99 if err != nil {
100 return repo, b, c.fail(protocol.ExitNotFound, "no build %d on %s", n, repo.Path())
101 }
102 return repo, b, -1
103}
104
105func runBuildList(c *Ctx, args []string) int {
106 if len(args) != 1 {
107 return c.fail(protocol.ExitUsage, "usage: build list <owner/name>")
108 }
109 repo, code := resolveRepo(c, args[0], policy.CanRead)
110 if code >= 0 {
111 return code
112 }
113 builds, err := c.Store.ListBuilds(repo.ID, 50)
114 if err != nil {
115 return c.fail(protocol.ExitFailure, "%v", err)
116 }
117 var ds []BuildOut
118 for _, b := range builds {
119 ds = append(ds, buildToOut(b))
120 }
121 return c.emit(ds, func(w io.Writer) {
122 for _, d := range ds {
123 fmt.Fprintf(w, "%d\t%s\t%s\t%.10s\t%s\n", d.Number, d.Job, d.Status, d.SHA, d.Ref)
124 }
125 })
126}
127
128func runBuildShow(c *Ctx, args []string) int {
129 _, b, code := buildRef(c, args)
130 if code >= 0 {
131 return code
132 }
133 d := buildToOut(b)
134 return c.emit(d, func(w io.Writer) {
135 fmt.Fprintf(w, "build %d\t%s\t%s\n%.10s on %s\nqueued %s", d.Number, d.Job, d.Status, d.SHA, d.Ref, d.CreatedAt)
136 if d.FinishedAt != "" {
137 fmt.Fprintf(w, ", finished %s", d.FinishedAt)
138 }
139 fmt.Fprintln(w)
140 })
141}
142
143func runBuildLog(c *Ctx, args []string) int {
144 _, b, code := buildRef(c, args)
145 if code >= 0 {
146 return code
147 }
148 log, err := c.Store.BuildLog(b.ID)
149 if err != nil {
150 return c.fail(protocol.ExitFailure, "%v", err)
151 }
152 c.Stdout.Write(log)
153 return protocol.ExitOK
154}
155
156type JobOut struct {
157 Name string `json:"name"`
158 Schedule string `json:"schedule,omitempty"`
159 Tags string `json:"tags,omitempty"`
160}
161
162// repoJobs reads the CI config on the default branch — the same file the
163// scheduler reads — and returns its jobs with the sha they came from.
164func repoJobs(c *Ctx, repo store.Repo) ([]ci.Job, string, int) {
165 dir := RepoDir(c.Cfg.Server.Root, repo.OwnerName, repo.Name)
166 sha, err := gitutil.ResolveRef(dir, "refs/heads/"+repo.DefaultBranch)
167 if err != nil {
168 return nil, "", c.fail(protocol.ExitFailure, "resolving %s: %v", repo.DefaultBranch, err)
169 }
170 raw, err := gitutil.ReadBlob(dir, sha, ci.ConfigPath, 1<<16)
171 if err != nil {
172 return nil, "", c.fail(protocol.ExitNotFound, "%s has no %s on %s", repo.Path(), ci.ConfigPath, repo.DefaultBranch)
173 }
174 jobs, err := ci.Parse(raw)
175 if err != nil {
176 return nil, "", c.failErr(err)
177 }
178 return jobs, sha, -1
179}
180
181// runBuildJobs answers "what can I trigger?". Without it only a surface
182// that can read the repository's git could offer the choice.
183func runBuildJobs(c *Ctx, args []string) int {
184 if len(args) != 1 {
185 return c.fail(protocol.ExitUsage, "usage: build jobs <owner/name>")
186 }
187 repo, code := resolveRepo(c, args[0], policy.CanRead)
188 if code >= 0 {
189 return code
190 }
191 jobs, _, code := repoJobs(c, repo)
192 if code >= 0 {
193 return code
194 }
195 out := make([]JobOut, 0, len(jobs))
196 for _, j := range jobs {
197 out = append(out, JobOut{Name: j.Name, Schedule: j.Schedule, Tags: j.Tags})
198 }
199 return c.emit(out, func(w io.Writer) {
200 for _, j := range out {
201 switch {
202 case j.Schedule != "":
203 fmt.Fprintf(w, "%s\tschedule %s\n", j.Name, j.Schedule)
204 case j.Tags != "":
205 fmt.Fprintf(w, "%s\ttags %s\n", j.Name, j.Tags)
206 default:
207 fmt.Fprintf(w, "%s\ton push\n", j.Name)
208 }
209 }
210 })
211}
212
213func runBuildTrigger(c *Ctx, args []string) int {
214 if len(args) != 2 {
215 return c.fail(protocol.ExitUsage, "usage: build trigger <owner/name> <job>")
216 }
217 repo, code := resolveRepo(c, args[0], policy.CanWrite)
218 if code >= 0 {
219 return code
220 }
221 jobs, sha, code := repoJobs(c, repo)
222 if code >= 0 {
223 return code
224 }
225 for _, j := range jobs {
226 if j.Name != args[1] {
227 continue
228 }
229 steps, _ := json.Marshal(j.Steps)
230 tree, _ := gitutil.ResolveTree(RepoDir(c.Cfg.Server.Root, repo.OwnerName, repo.Name), sha)
231 n, err := c.Store.CreateBuild(repo.ID, j.Name, sha, repo.DefaultBranch, string(steps), j.Image, tree, true)
232 if err != nil {
233 return c.fail(protocol.ExitFailure, "%v", err)
234 }
235 url := fmt.Sprintf("%s/%s/builds/%d", c.Cfg.Server.SiteURL, repo.Path(), n)
236 c.Store.SetCommitStatus(repo.ID, sha, "ci/"+j.Name, "pending", "triggered", url, c.User.ID)
237 return c.emit(map[string]any{"build": n, "job": j.Name, "sha": sha}, func(w io.Writer) {
238 fmt.Fprintf(w, "queued build %d (%s @ %.10s)\n", n, j.Name, sha)
239 })
240 }
241 return c.fail(protocol.ExitNotFound, "no job %q in %s", args[1], ci.ConfigPath)
242}
243
244// secretName is env-var shaped: the value lands in the build environment.
245var secretName = regexp.MustCompile(`^[A-Z_][A-Z0-9_]{0,63}$`)
246
247func runSecretSet(c *Ctx, args []string) int {
248 if len(args) != 2 {
249 return c.fail(protocol.ExitUsage, "usage: repo secret set <owner/name> <NAME> (value on stdin)")
250 }
251 if !secretName.MatchString(args[1]) {
252 return c.fail(protocol.ExitUsage, "secret names are env-var shaped: uppercase letters, digits, _")
253 }
254 repo, code := resolveRepo(c, args[0], policy.CanAdmin)
255 if code >= 0 {
256 return code
257 }
258 raw, err := io.ReadAll(io.LimitReader(c.Stdin, 64<<10))
259 if err != nil {
260 return c.fail(protocol.ExitFailure, "reading secret: %v", err)
261 }
262 value := strings.TrimRight(string(raw), "\n")
263 if value == "" {
264 return c.fail(protocol.ExitUsage, "no value on stdin (pipe it: printf %%s TOKEN | ...)")
265 }
266 if err := c.Store.SetBuildSecret(repo.ID, args[1], value); err != nil {
267 return c.fail(protocol.ExitFailure, "%v", err)
268 }
269 return c.emit(map[string]string{"secret": args[1]}, func(w io.Writer) {
270 fmt.Fprintf(w, "secret %s set on %s\n", args[1], repo.Path())
271 })
272}
273
274func runSecretRemove(c *Ctx, args []string) int {
275 if len(args) != 2 {
276 return c.fail(protocol.ExitUsage, "usage: repo secret remove <owner/name> <NAME>")
277 }
278 repo, code := resolveRepo(c, args[0], policy.CanAdmin)
279 if code >= 0 {
280 return code
281 }
282 if err := c.Store.RemoveBuildSecret(repo.ID, args[1]); err != nil {
283 if errors.Is(err, store.ErrNotFound) {
284 return c.fail(protocol.ExitNotFound, "no secret %s on %s", args[1], repo.Path())
285 }
286 return c.fail(protocol.ExitFailure, "%v", err)
287 }
288 return c.emit(map[string]string{"removed": args[1]}, func(w io.Writer) {
289 fmt.Fprintf(w, "removed %s\n", args[1])
290 })
291}
292
293func runSecretList(c *Ctx, args []string) int {
294 if len(args) != 1 {
295 return c.fail(protocol.ExitUsage, "usage: repo secret list <owner/name>")
296 }
297 repo, code := resolveRepo(c, args[0], policy.CanAdmin)
298 if code >= 0 {
299 return code
300 }
301 names, err := c.Store.ListBuildSecretNames(repo.ID)
302 if err != nil {
303 return c.fail(protocol.ExitFailure, "%v", err)
304 }
305 return c.emit(names, func(w io.Writer) {
306 for _, n := range names {
307 fmt.Fprintln(w, n)
308 }
309 })
310}
311
312// runnerSession resolves the key behind a runner-protocol session. The
313// runner commands are SSHOnly, so Source is the key's fingerprint. An
314// admin key is accepted so an operator can rotate at their own pace; a
315// runner host should hold a key added with --scope runner.
316func runnerSession(c *Ctx) (store.SSHKey, int) {
317 if c.Scope != "runner" && !c.User.IsAdmin {
318 return store.SSHKey{}, c.fail(protocol.ExitDenied, "runner commands need a key added with --scope runner")
319 }
320 key, err := c.Store.SSHKeyByFingerprint(c.Source)
321 if err != nil {
322 return store.SSHKey{}, c.fail(protocol.ExitDenied, "runner commands need an SSH key session")
323 }
324 return key, -1
325}
326
327// runnerAdmin reports whether a session claims builds instance-wide. The
328// bypass is the key, not the account: a scope-runner key is confined to
329// its attachments whoever owns it, including an instance admin.
330func runnerAdmin(c *Ctx) bool {
331 return c.User.IsAdmin && c.Scope != "runner"
332}
333
334// runnerMayBuild reports whether a runner session may act on a
335// repository's builds: an admin key may on any, a runner key on the
336// repositories it is attached to (#184).
337func runnerMayBuild(c *Ctx, key store.SSHKey, repoID int64) (bool, error) {
338 if runnerAdmin(c) {
339 return true, nil
340 }
341 return c.Store.RunnerAttached(key.ID, repoID)
342}
343
344// maxOrphanSkip bounds how many claimed builds runRunnerNext will find
345// unreachable and cancel in one call before giving up. Only fast-forward
346// merges are allowed here, so any branch whose target advances gets
347// rebased and force-pushed, and a stack of branches can do that repeatedly
348// in one sitting — the issue this guards saw five in an afternoon. The cap
349// is well above that, so a real backlog is never cut short, while a
350// repository whose queue is orphaned end to end still returns rather than
351// walking it forever.
352const maxOrphanSkip = 50
353
354func runRunnerNext(c *Ctx, args []string) int {
355 key, code := runnerSession(c)
356 if code >= 0 {
357 return code
358 }
359 f, err := parseFlags(args, flagSpec{Bools: []string{"--untrusted"}, MaxPos: -1,
360 Usage: "runner next [--untrusted] [<owner/name>...]"})
361 if err != nil {
362 return c.fail(protocol.ExitUsage, "%v", err)
363 }
364 // The candidate set. An admin key claims from any repository, narrowed
365 // by the names given. A runner key claims from the repositories it is
366 // attached to; a name outside them is refused, not ignored, so a
367 // misconfigured runner says so instead of idling.
368 var repoIDs []int64
369 for _, arg := range f.Pos {
370 repo, code := resolveRepo(c, arg, policy.CanRead)
371 if code >= 0 {
372 return code
373 }
374 ok, err := runnerMayBuild(c, key, repo.ID)
375 if err != nil {
376 return c.fail(protocol.ExitFailure, "%v", err)
377 }
378 if !ok {
379 return c.fail(protocol.ExitDenied, "this key is not attached to %s", repo.Path())
380 }
381 repoIDs = append(repoIDs, repo.ID)
382 }
383 if !runnerAdmin(c) && len(repoIDs) == 0 {
384 repoIDs, err = c.Store.RunnerRepoIDs(key.ID)
385 if err != nil {
386 return c.fail(protocol.ExitFailure, "%v", err)
387 }
388 if len(repoIDs) == 0 {
389 // Nothing attached: nothing to claim. Still a heartbeat, so
390 // admin runners shows the key polling.
391 c.Store.TouchRunner(key.ID, c.User.ID, "", 0)
392 return c.emit(map[string]any{}, func(w io.Writer) { fmt.Fprintln(w, "no pending builds") })
393 }
394 }
395 untrusted := f.Has("--untrusted")
396 var b store.Build
397 var repo store.Repo
398 var ok bool
399 for attempt := 0; attempt < maxOrphanSkip; attempt++ {
400 b, ok, err = c.Store.ClaimBuild(repoIDs, untrusted)
401 if err != nil {
402 return c.fail(protocol.ExitFailure, "%v", err)
403 }
404 if !ok {
405 break
406 }
407 repo, err = c.Store.RepoByID(b.RepoID)
408 if err != nil {
409 return c.fail(protocol.ExitFailure, "%v", err)
410 }
411 // Only fast-forward merges are allowed here, so a target that
412 // advances gets rebased and force-pushed, orphaning whatever was
413 // queued for the old head: the runner would clone the repo and
414 // fail at checkout with a git internal error that reads exactly
415 // like a real failure. Catch it here instead. A check that itself
416 // fails is not evidence of anything — the build runs for real and
417 // is left to fail on its own terms, never cancelled on a guess.
418 reachable, err := gitutil.Reachable(RepoDir(c.Cfg.Server.Root, repo.OwnerName, repo.Name), b.SHA)
419 if err != nil || reachable {
420 break
421 }
422 if code := cancelOrphanedBuild(c, repo, b); code >= 0 {
423 return code
424 }
425 // Cancelled, not claimed: if the cap is hit right here, the runner
426 // heartbeat below must not record this build as the one handed out.
427 b, ok = store.Build{}, false
428 }
429 // The poll itself is the runner's heartbeat: admin runners reads it.
430 c.Store.TouchRunner(key.ID, c.User.ID, strings.Join(f.Pos, ","), b.ID)
431 if !ok {
432 return c.emit(map[string]any{}, func(w io.Writer) { fmt.Fprintln(w, "no pending builds") })
433 }
434 var steps []string
435 json.Unmarshal([]byte(b.Steps), &steps)
436 // Secrets ride the claim: this channel is admin-only and the values
437 // land in the build's environment, nowhere else.
438 var secrets map[string]string
439 if b.Trusted {
440 secrets, err = c.Store.BuildSecrets(b.RepoID)
441 if err != nil {
442 return c.fail(protocol.ExitFailure, "%v", err)
443 }
444 }
445 d := struct {
446 ID int64 `json:"id"`
447 Repo string `json:"repo"`
448 Number int64 `json:"number"`
449 Job string `json:"job"`
450 SHA string `json:"sha"`
451 Ref string `json:"ref"`
452 Steps []string `json:"steps"`
453 Image string `json:"image,omitempty"`
454 Secrets map[string]string `json:"secrets,omitempty"`
455 }{b.ID, repo.Path(), b.Number, b.Job, b.SHA, b.Ref, steps, b.Image, secrets}
456 return c.emit(d, func(w io.Writer) {
457 fmt.Fprintf(w, "build %d: %s %s @ %.10s\n", d.ID, d.Repo, d.Job, d.SHA)
458 })
459}
460
461func runRunnerLog(c *Ctx, args []string) int {
462 key, code := runnerSession(c)
463 if code >= 0 {
464 return code
465 }
466 if len(args) != 1 {
467 return c.fail(protocol.ExitUsage, "usage: runner log <build-id> (chunk on stdin)")
468 }
469 id, err := strconv.ParseInt(args[0], 10, 64)
470 if err != nil {
471 return c.fail(protocol.ExitUsage, "bad build id %q", args[0])
472 }
473 if b, err := c.Store.BuildByID(id); err != nil {
474 return c.fail(protocol.ExitNotFound, "no build %d", id)
475 } else if ok, err := runnerMayBuild(c, key, b.RepoID); err != nil {
476 return c.fail(protocol.ExitFailure, "%v", err)
477 } else if !ok {
478 return c.fail(protocol.ExitDenied, "this key is not attached to the build's repository")
479 }
480 // Stream stdin into the log in chunks so long builds appear live. An
481 // append that fails drops its chunk and the loop keeps draining: ending
482 // the session here breaks the runner's pipe, and a broken pipe is how a
483 // transient SQLITE_BUSY used to fail the build the log belonged to.
484 //
485 // The session is also how a running build is cancelled: while it is
486 // open the build's row is watched, and when the row stops saying
487 // running the session ends with ExitNotFound, which the runner reads as
488 // "stop this build". Any other end of the session is a lost stream.
489 type chunk struct {
490 data []byte
491 err error
492 }
493 chunks := make(chan chunk, 4)
494 go func() {
495 buf := make([]byte, 64<<10)
496 for {
497 n, rerr := c.Stdin.Read(buf)
498 if n > 0 {
499 chunks <- chunk{data: append([]byte(nil), buf[:n]...)}
500 }
501 if rerr != nil {
502 chunks <- chunk{err: rerr}
503 return
504 }
505 }
506 }()
507 watch := time.NewTicker(2 * time.Second)
508 defer watch.Stop()
509 dropped := 0
510 for {
511 select {
512 case ch := <-chunks:
513 if len(ch.data) > 0 {
514 if err := c.Store.AppendBuildLog(id, ch.data); err != nil {
515 dropped++
516 slog.Warn("appending build log", "build", id, "err", err)
517 }
518 }
519 if ch.err != nil {
520 if dropped > 0 {
521 slog.Warn("build log incomplete", "build", id, "dropped_chunks", dropped)
522 }
523 // The stream ending is the last thing the server hears
524 // from a runner that is about to die; note the time so
525 // the scheduler can fail the build if no outcome follows.
526 if err := c.Store.MarkBuildLogClosed(id); err != nil {
527 slog.Warn("marking build log closed", "build", id, "err", err)
528 }
529 return c.emit(map[string]string{"log": "ok"}, func(w io.Writer) {})
530 }
531 case <-watch.C:
532 if b, err := c.Store.BuildByID(id); err == nil && b.Status != "running" {
533 return c.fail(protocol.ExitNotFound, "build %d is %s; stop", id, b.Status)
534 }
535 }
536 }
537}
538
539func runRunnerDone(c *Ctx, args []string) int {
540 key, code := runnerSession(c)
541 if code >= 0 {
542 return code
543 }
544 if len(args) != 2 || (args[1] != "success" && args[1] != "failure") {
545 return c.fail(protocol.ExitUsage, "usage: runner done <build-id> success|failure")
546 }
547 id, err := strconv.ParseInt(args[0], 10, 64)
548 if err != nil {
549 return c.fail(protocol.ExitUsage, "bad build id %q", args[0])
550 }
551 b, err := c.Store.BuildByID(id)
552 if err != nil {
553 return c.fail(protocol.ExitNotFound, "no build %d", id)
554 }
555 if ok, err := runnerMayBuild(c, key, b.RepoID); err != nil {
556 return c.fail(protocol.ExitFailure, "%v", err)
557 } else if !ok {
558 return c.fail(protocol.ExitDenied, "this key is not attached to the build's repository")
559 }
560 // Cancelled underneath the runner: its report is late, not wrong.
561 // The row, the status and the log were settled by the cancel.
562 if b.Status == "cancelled" {
563 c.Store.RunnerDone(key.ID)
564 return c.emit(map[string]any{"build": b.Number, "status": "cancelled"}, func(w io.Writer) {
565 fmt.Fprintf(w, "build %d was cancelled\n", b.Number)
566 })
567 }
568 if err := c.Store.FinishBuild(id, args[1]); err != nil {
569 return c.fail(protocol.ExitFailure, "finishing build %d: %v", id, err)
570 }
571 c.Store.RunnerDone(key.ID)
572 repo, err := c.Store.RepoByID(b.RepoID)
573 if err != nil {
574 return c.fail(protocol.ExitFailure, "%v", err)
575 }
576 url := fmt.Sprintf("%s/%s/builds/%d", c.Cfg.Server.SiteURL, repo.Path(), b.Number)
577 desc := "build " + args[1]
578 if err := c.Store.SetCommitStatus(repo.ID, b.SHA, "ci/"+b.Job, args[1], desc, url, c.User.ID); err != nil {
579 return c.fail(protocol.ExitFailure, "%v", err)
580 }
581 c.Store.RecordEvent(repo.ID, c.User.ID, "build."+args[1],
582 fmt.Sprintf(`{"number":%d,"job":%q}`, b.Number, b.Job))
583 // A red build mails the repo's notify targets with the log tail — a
584 // failed scheduled job must not wait to be noticed.
585 if args[1] == "failure" {
586 if targets, err := c.Store.RepoNotifyTargets(repo); err == nil {
587 tail := ""
588 if log, err := c.Store.BuildLog(id); err == nil && len(log) > 0 {
589 if len(log) > 2000 {
590 log = log[len(log)-2000:]
591 }
592 tail = string(log)
593 }
594 notify(c, targets, notice{repo: repo, kind: "build",
595 subject: fmt.Sprintf("[%s] build %d failed: %s on %s", repo.Path(), b.Number, b.Job, b.Ref),
596 action: fmt.Sprintf("build %d failed: %s on %s", b.Number, b.Job, b.Ref),
597 body: fmt.Sprintf("job %s failed at %.10s.\n\n…%s\n\n%s\n", b.Job, b.SHA, tail, url),
598 path: fmt.Sprintf("%s/builds/%d", repo.Path(), b.Number)})
599 }
600 }
601 return c.emit(map[string]any{"build": b.Number, "status": args[1]}, func(w io.Writer) {
602 fmt.Fprintf(w, "build %d %s\n", b.Number, args[1])
603 })
604}
605
606// QueueBranchBuilds reads .gitbay/ci.yml at sha and creates one pending
607// build per push job, with a pending commit status the runner resolves.
608// A broken config surfaces as a failed "ci/config" status, not silence.
609//
610// Both paths that move a branch call this: post-receive for a push, and
611// the merge path for a merge, which updates the ref directly and so never
612// reaches a hook. old is the branch's sha before this update, the diff
613// base a job's path filters run against; a new branch has no prior
614// commit and sends old as empty or all zeros. queueJobs falls back to
615// the merge base with the default branch in that case, so a filter
616// still applies to a branch's first push — the shape most changes have,
617// since branch-then-MR is the normal workflow here.
618func QueueBranchBuilds(
619 st *store.Store, root, siteURL string,
620 repo store.Repo, userID int64, branch, old, sha string, now time.Time,
621) {
622 queueJobs(st, root, siteURL, repo, userID, branch, old, sha, now, true, branch == repo.DefaultBranch, true)
623}
624
625// QueueMRBuilds queues the push jobs for a merge request head fetched
626// from another repository, which the target holds at
627// refs/merge-requests/<n>/head, so a fork's merge request has ci/<job>
628// statuses for require-checks to gate on (#98). The head is untrusted:
629// its build runs without the target's secrets. A same-repository head is
630// the branch push's job and is not queued here; a failed one is rebuilt
631// when it lands, not when it is proposed.
632func QueueMRBuilds(
633 st *store.Store, root, siteURL string,
634 repo store.Repo, userID, n int64, sha string,
635) {
636 // No old sha, and unlike QueueBranchBuilds, no merge-base fallback
637 // either: this deliberately keeps failing open and running every
638 // job. require_checks refuses a merge when an MR head has no
639 // statuses at all (mr.go), so filtering a head down to zero jobs
640 // would make it unmergeable rather than just unfiltered (#172).
641 queueJobs(st, root, siteURL, repo, userID, mrHeadRef(n), "", sha, time.Now(), false, false, false)
642}
643
644// skipReason names why a job's path filters excluded this push, mirroring
645// the order ci.Selected checks them in: an unmatched paths list rules a
646// job out before paths-ignore is even considered.
647func skipReason(j ci.Job, changed []string) string {
648 if len(j.Paths) > 0 {
649 hit := false
650 for _, f := range changed {
651 for _, p := range j.Paths {
652 if ci.Match(p, f) {
653 hit = true
654 }
655 }
656 }
657 if !hit {
658 return "no changed file matches paths"
659 }
660 }
661 return "every changed file matched paths-ignore"
662}
663
664func queueJobs(
665 st *store.Store, root, siteURL string,
666 repo store.Repo, userID int64, ref, old, sha string, now time.Time,
667 trusted, syncSchedules, deriveMergeBase bool,
668) {
669 dir := RepoDir(root, repo.OwnerName, repo.Name)
670 raw, err := gitutil.ReadBlob(dir, sha, ci.ConfigPath, 1<<16)
671 if err != nil {
672 return // no CI config at this commit
673 }
674 jobs, err := ci.Parse(raw)
675 if err != nil {
676 st.SetCommitStatus(repo.ID, sha, "ci/config", "failure", err.Error(), "", userID)
677 return
678 }
679 // A build is a fact about a commit, not a ref: a job has no branch
680 // filter, so a commit that already passed a job on another branch has
681 // nothing left to prove when a fast-forward lands it here, and one
682 // still queued or running there will say soon enough. A failed,
683 // abandoned or cancelled build does not count; that commit runs again.
684 built, err := st.BuildsForCommit(repo.ID, sha)
685 if err != nil {
686 built = nil
687 }
688 // A job's result is a property of the tree, not the commit: a rebase
689 // onto a base that touched nothing the branch did gives every commit
690 // a new sha and the same tree, and re-running the suite over it
691 // proves nothing it did not already prove (#177). A success recorded
692 // against the tree stands for the new commit.
693 tree, _ := gitutil.ResolveTree(dir, sha)
694 // The changed-file list a job's path filters run against, computed
695 // once and only if some job actually declares one. When the diff
696 // base does not exist or the diff itself fails, filtered stays
697 // false and every job runs: a filter that cannot be evaluated must
698 // not silently skip CI.
699 //
700 // A branch's first push has no old sha, but a diff base still
701 // exists: the merge base with the default branch. Without deriving
702 // one, every job runs on every new branch, and since branch-then-MR
703 // is the normal workflow, that is the push path filters matter most
704 // for. The merge base of the default branch's tip with itself is
705 // the tip, carrying no diff — that covers the default branch's own
706 // first push on a fresh repository, and must fail open rather than
707 // read as "nothing changed".
708 filtered := false
709 var changed []string
710 for _, j := range jobs {
711 if len(j.Paths) == 0 && len(j.PathsIgnore) == 0 {
712 continue
713 }
714 diffOld := old
715 // A force-push rewrote the branch, so the old tip is not an
716 // ancestor of the new one and old..new is not "what this push
717 // changed" — it is the difference between two histories. After a
718 // rebase that is whatever the new base added, typically nothing
719 // the branch itself touched, so every path filter concludes its
720 // job is unnecessary and the branch reads as green without its
721 // suite having run (#176). The merge base is the honest base:
722 // the filter is deciding about the branch's relationship to its
723 // target, which is what the merge base expresses.
724 if ci.HasDiffBase(diffOld) && deriveMergeBase {
725 if ok, err := gitutil.IsAncestor(dir, diffOld, sha); err != nil || !ok {
726 diffOld = ""
727 }
728 }
729 if !ci.HasDiffBase(diffOld) && deriveMergeBase {
730 if base, err := gitutil.MergeBase(dir, "refs/heads/"+repo.DefaultBranch, sha); err == nil && base != sha {
731 diffOld = base
732 }
733 }
734 if ci.HasDiffBase(diffOld) {
735 if files, err := gitutil.DiffFiles(dir, diffOld, sha); err == nil {
736 changed, filtered = files, true
737 }
738 }
739 break
740 }
741 var schedules []store.Schedule
742 for _, j := range jobs {
743 // Tag jobs run on matching tag pushes only.
744 if j.Tags != "" {
745 continue
746 }
747 if b, ok := built[j.Name]; ok && (b.Status == "success" || b.Status == "pending" || b.Status == "running") {
748 continue
749 }
750 if prev, ok, _ := st.SuccessBuildForTree(repo.ID, tree, j.Name); ok && prev.SHA != sha {
751 url := fmt.Sprintf("%s/%s/builds/%d", siteURL, repo.Path(), prev.Number)
752 st.SetCommitStatus(repo.ID, sha, "ci/"+j.Name, "success",
753 fmt.Sprintf("passed in build %d as %.10s, same tree", prev.Number, prev.SHA), url, userID)
754 continue
755 }
756 // Scheduled jobs run on their cron, not on push; a default-branch
757 // push (re)registers them.
758 if j.Schedule != "" {
759 if syncSchedules {
760 schedules = append(schedules, store.Schedule{
761 RepoID: repo.ID, Job: j.Name, Cron: j.Schedule,
762 NextRun: ci.NextRun(j.Schedule, now),
763 })
764 }
765 continue
766 }
767 // A filter that excludes this push is not silence: it satisfies
768 // require_checks with a skipped status instead of leaving the
769 // commit with none at all, which the gate refuses outright (#172).
770 if filtered && !ci.Selected(j, changed) {
771 st.SetCommitStatus(repo.ID, sha, "ci/"+j.Name, "skipped", skipReason(j, changed), "", userID)
772 continue
773 }
774 steps, _ := json.Marshal(j.Steps)
775 n, err := st.CreateBuild(repo.ID, j.Name, sha, ref, string(steps), j.Image, tree, trusted)
776 if err != nil {
777 slog.Error("queueing build", "repo", repo.Path(), "job", j.Name, "err", err)
778 continue
779 }
780 url := fmt.Sprintf("%s/%s/builds/%d", siteURL, repo.Path(), n)
781 st.SetCommitStatus(repo.ID, sha, "ci/"+j.Name, "pending", "queued", url, userID)
782 }
783 if syncSchedules {
784 if err := st.SyncSchedules(repo.ID, schedules); err != nil {
785 slog.Error("syncing schedules", "repo", repo.Path(), "err", err)
786 }
787 }
788}
789
790// resolveCancelledCommitStatus sets the commit status for a build that was
791// just cancelled: if the commit already passed this job on another ref,
792// that result stands again; otherwise the context reports the
793// cancellation as an error, so the queued status left behind is never
794// pending forever.
795func resolveCancelledCommitStatus(c *Ctx, repo store.Repo, b store.Build) {
796 if prev, ok, err := c.Store.SuccessBuildFor(repo.ID, b.SHA, b.Job); err == nil && ok {
797 url := fmt.Sprintf("%s/%s/builds/%d", c.Cfg.Server.SiteURL, repo.Path(), prev.Number)
798 c.Store.SetCommitStatus(repo.ID, b.SHA, "ci/"+b.Job, "success",
799 fmt.Sprintf("passed in build %d on %s", prev.Number, prev.Ref), url, c.User.ID)
800 return
801 }
802 url := fmt.Sprintf("%s/%s/builds/%d", c.Cfg.Server.SiteURL, repo.Path(), b.Number)
803 c.Store.SetCommitStatus(repo.ID, b.SHA, "ci/"+b.Job, "error", "cancelled", url, c.User.ID)
804}
805
806// cancelOrphanedBuild withdraws a build runRunnerNext claimed and then
807// found unreachable. It leaves the same shape behind as a build cancel a
808// person runs by hand: CancelBuild's status, a log line saying why, and
809// the commit status resolved rather than left pending. Returns -1 to mean
810// "handled, keep going"; anything else is the exit code to return.
811func cancelOrphanedBuild(c *Ctx, repo store.Repo, b store.Build) int {
812 if err := c.Store.CancelBuild(b.ID); err != nil {
813 return c.fail(protocol.ExitFailure, "%v", err)
814 }
815 c.Store.AppendBuildLog(b.ID, []byte(fmt.Sprintf(
816 "cancelled: %.10s is not reachable from any ref; the sha was likely orphaned by a force-push\n", b.SHA)))
817 resolveCancelledCommitStatus(c, repo, b)
818 c.Store.RecordEvent(repo.ID, c.User.ID, "build.cancelled", fmt.Sprintf(`{"number":%d,"job":%q}`, b.Number, b.Job))
819 return -1
820}
821
822func runBuildCancel(c *Ctx, args []string) int {
823 repo, b, code := buildRef(c, args)
824 if code >= 0 {
825 return code
826 }
827 grant, err := c.Store.AccessRole(repo.ID, c.User.ID)
828 if err != nil {
829 return c.fail(protocol.ExitFailure, "%v", err)
830 }
831 if !policy.CanWrite(c.User, repo, grant) {
832 return c.fail(protocol.ExitDenied, "cancelling a build needs write access to %s", repo.Path())
833 }
834 if b.Status != "pending" && b.Status != "running" {
835 return c.fail(protocol.ExitUsage, "build %d is %s; only a queued or running build can be cancelled", b.Number, b.Status)
836 }
837 if err := c.Store.CancelBuild(b.ID); err != nil {
838 return c.fail(protocol.ExitFailure, "%v", err)
839 }
840 if b.Status == "running" {
841 c.Store.AppendBuildLog(b.ID, []byte(fmt.Sprintf("\ncancelled by %s while running; the runner stops at its next check\n", c.User.Username)))
842 } else {
843 c.Store.AppendBuildLog(b.ID, []byte(fmt.Sprintf("cancelled by %s before a runner claimed it\n", c.User.Username)))
844 }
845 // The queued status replaced whatever the commit had for this job.
846 resolveCancelledCommitStatus(c, repo, b)
847 c.Store.RecordEvent(repo.ID, c.User.ID, "build.cancelled", fmt.Sprintf(`{"number":%d,"job":%q}`, b.Number, b.Job))
848 return c.emit(map[string]any{"number": b.Number, "job": b.Job, "status": "cancelled", "was": b.Status}, func(w io.Writer) {
849 if b.Status == "running" {
850 fmt.Fprintf(w, "cancelled %s build %d (%s); the runner stops at its next check\n", repo.Path(), b.Number, b.Job)
851 return
852 }
853 fmt.Fprintf(w, "cancelled %s build %d (%s)\n", repo.Path(), b.Number, b.Job)
854 })
855}