internal/control/build.go
918 lines · 34703 bytes
1package control
2
3import (
4 "encoding/json"
5 "errors"
6 "fmt"
7 "io"
8 "log/slog"
9 "regexp"
10 "slices"
11 "strconv"
12 "strings"
13 "time"
14
15 "gitbay.org/gitbay/internal/ci"
16 "gitbay.org/gitbay/internal/gitutil"
17 "gitbay.org/gitbay/internal/policy"
18 "gitbay.org/gitbay/internal/protocol"
19 "gitbay.org/gitbay/internal/store"
20)
21
22func init() {
23 register(Command{Path: []string{"build", "list"},
24 Summary: "list recent builds",
25 Usage: "build list <owner/name> [--ref <branch>] [--status <state>] [--job <name>] [--limit <n>] [--cursor <c>]", ReadOnly: true, Run: runBuildList})
26 register(Command{Path: []string{"build", "show"},
27 Summary: "show one build",
28 Usage: "build show <owner/name> <n>", ReadOnly: true, Run: runBuildShow})
29 register(Command{Path: []string{"build", "log"},
30 Summary: "print a build's log, or follow it until the build ends",
31 Usage: "build log <owner/name> <n> [--follow]", ReadOnly: true, Run: runBuildLog})
32
33 register(Command{Path: []string{"build", "jobs"},
34 Summary: "list the jobs a trigger can name",
35 Usage: "build jobs <owner/name>", ReadOnly: true, Run: runBuildJobs})
36
37 register(Command{Path: []string{"build", "cancel"},
38 Summary: "withdraw a queued build before a runner claims it",
39 Usage: "build cancel <owner/name> <n>", Run: runBuildCancel})
40 register(Command{Path: []string{"build", "trigger"},
41 Summary: "queue a job now (scheduled or not)",
42 Usage: "build trigger <owner/name> <job>", Run: runBuildTrigger})
43 // Secrets: set over stdin, listed by name only, injected into the
44 // repo's builds as environment variables. Same discipline as mirror
45 // tokens — the value never appears in argv, logs, or output.
46 register(Command{Path: []string{"repo", "secret", "set"},
47 Summary: "set a build secret",
48 Usage: "repo secret set <owner/name> <NAME> (value on stdin)",
49 ReadsStdin: true, Run: runSecretSet})
50 register(Command{Path: []string{"repo", "secret", "remove"},
51 Summary: "remove a build secret",
52 Usage: "repo secret remove <owner/name> <NAME>", Run: runSecretRemove})
53 register(Command{Path: []string{"repo", "secret", "list"},
54 Summary: "list build secret names",
55 Usage: "repo secret list <owner/name>", ReadOnly: true, Run: runSecretList})
56
57 // Runner commands: the claim/report loop for gitbay-runner. A runner
58 // executes arbitrary repo code, so handing out jobs is the instance
59 // operator's call: a key added with --scope runner, which the
60 // dispatcher confines to these three commands and read-only git, or
61 // an admin key, which a runner host should not hold (#92).
62 register(Command{Path: []string{"runner", "next"},
63 Summary: "claim the oldest pending build this key may run (runner protocol)",
64 Usage: "runner next [--untrusted] [<owner/name>...]", Run: runRunnerNext})
65 register(Command{Path: []string{"runner", "log"},
66 Summary: "append a build's log from stdin",
67 Usage: "runner log <build-id>", ReadsStdin: true, Run: runRunnerLog})
68 register(Command{Path: []string{"runner", "done"},
69 Summary: "finish a build",
70 Usage: "runner done <build-id> success|failure", Run: runRunnerDone})
71}
72
73type BuildOut struct {
74 Number int64 `json:"number"`
75 Job string `json:"job"`
76 Status string `json:"status"`
77 SHA string `json:"sha"`
78 Ref string `json:"ref"`
79 CreatedAt string `json:"created_at"`
80 FinishedAt string `json:"finished_at,omitempty"`
81 // Subject is the first line of the commit's message, so a build
82 // names what it ran on rather than only its sha (#241). It is empty
83 // when the commit is no longer in the repository.
84 Subject string `json:"subject,omitempty"`
85}
86
87func buildToOut(b store.Build) BuildOut {
88 return BuildOut{Number: b.Number, Job: b.Job, Status: b.Status, SHA: b.SHA,
89 Ref: b.Ref, CreatedAt: b.CreatedAt, FinishedAt: b.FinishedAt}
90}
91
92func buildRef(c *Ctx, args []string) (store.Repo, store.Build, int) {
93 if len(args) != 2 {
94 return store.Repo{}, store.Build{}, c.usageWith("expected <owner/name> <number>")
95 }
96 repo, code := resolveRepo(c, args[0], policy.CanRead)
97 if code >= 0 {
98 return repo, store.Build{}, code
99 }
100 n, err := strconv.ParseInt(args[1], 10, 64)
101 if err != nil {
102 return repo, store.Build{}, c.fail(protocol.ExitUsage, "bad build number %q", args[1])
103 }
104 b, err := c.Store.BuildByNumber(repo.ID, n)
105 if err != nil {
106 return repo, b, c.fail(protocol.ExitNotFound, "no build %d on %s", n, repo.Path())
107 }
108 return repo, b, -1
109}
110
111// buildStatuses is the vocabulary --status accepts, and what a bad value
112// is told to pick from.
113var buildStatuses = []string{"pending", "running", "success", "failure", "cancelled"}
114
115// buildPage is how many builds one page of build list returns when no
116// --limit is given. The cap has always been there; what it is now
117// reachable past, with --cursor (#244).
118const buildPage = 50
119
120func runBuildList(c *Ctx, args []string) int {
121 args, p, code := parsePageFlags(c, args, "build", true)
122 if code >= 0 {
123 return code
124 }
125 f, err := parseFlags(args, flagSpec{Values: []string{"--ref", "--status", "--job"}, MaxPos: 1, Usage: c.Cmd.Usage})
126 if err != nil {
127 return c.fail(protocol.ExitUsage, "%v", err)
128 }
129 path := f.pos(0)
130 if path == "" {
131 return c.usage()
132 }
133 status := f.Value("--status")
134 if f.Has("--status") && !slices.Contains(buildStatuses, status) {
135 return c.fail(protocol.ExitUsage, "--status must be one of %s", strings.Join(buildStatuses, ", "))
136 }
137 repo, code := resolveRepo(c, path, policy.CanRead)
138 if code >= 0 {
139 return code
140 }
141 limit := p.queryLimit()
142 if limit == 0 {
143 limit = buildPage
144 }
145 filter := store.BuildFilter{Ref: f.Value("--ref"), Status: status, Job: f.Value("--job"), Before: p.keyInt()}
146 builds, err := c.Store.ListBuilds(repo.ID, filter, limit)
147 if err != nil {
148 return c.fail(protocol.ExitFailure, "%v", err)
149 }
150 builds, next := trimPage(p, builds, "build", func(b store.Build) string {
151 return strconv.FormatInt(b.Number, 10)
152 })
153 var ds []BuildOut
154 for _, b := range builds {
155 ds = append(ds, buildToOut(b))
156 }
157 subjects := buildSubjects(c, repo, ds)
158 for i := range ds {
159 ds[i].Subject = subjects[ds[i].SHA]
160 }
161 return c.emitPage(p, ds, next, func(w io.Writer) {
162 for _, d := range ds {
163 fmt.Fprintf(w, "%d\t%s\t%s\t%.10s\t%s\t%s\n", d.Number, d.Job, d.Status, d.SHA, d.Ref, d.Subject)
164 }
165 })
166}
167
168// buildSubjects reads the commit subject of each distinct sha on a page
169// of builds. Several jobs of one push share a commit, so the set is
170// usually far smaller than the page.
171func buildSubjects(c *Ctx, repo store.Repo, ds []BuildOut) map[string]string {
172 seen := map[string]bool{}
173 var shas []string
174 for _, d := range ds {
175 if d.SHA != "" && !seen[d.SHA] {
176 seen[d.SHA] = true
177 shas = append(shas, d.SHA)
178 }
179 }
180 return gitutil.Subjects(RepoDir(c.Cfg.Server.Root, repo.OwnerName, repo.Name), shas)
181}
182
183func runBuildShow(c *Ctx, args []string) int {
184 _, b, code := buildRef(c, args)
185 if code >= 0 {
186 return code
187 }
188 d := buildToOut(b)
189 return c.emit(d, func(w io.Writer) {
190 fmt.Fprintf(w, "build %d\t%s\t%s\n%.10s on %s\nqueued %s", d.Number, d.Job, d.Status, d.SHA, d.Ref, d.CreatedAt)
191 if d.FinishedAt != "" {
192 fmt.Fprintf(w, ", finished %s", d.FinishedAt)
193 }
194 fmt.Fprintln(w)
195 })
196}
197
198func runBuildLog(c *Ctx, args []string) int {
199 f, err := parseFlags(args, flagSpec{Bools: []string{"--follow"}, MaxPos: 2, Usage: c.Cmd.Usage})
200 if err != nil {
201 return c.fail(protocol.ExitUsage, "%v", err)
202 }
203 repo, b, code := buildRef(c, f.Pos)
204 if code >= 0 {
205 return code
206 }
207 if f.Has("--follow") {
208 return followBuildLog(c, repo, b)
209 }
210 log, err := c.Store.BuildLog(b.ID)
211 if err != nil {
212 return c.fail(protocol.ExitFailure, "%v", err)
213 }
214 c.Stdout.Write(log)
215 return protocol.ExitOK
216}
217
218type JobOut struct {
219 Name string `json:"name"`
220 Schedule string `json:"schedule,omitempty"`
221 Tags string `json:"tags,omitempty"`
222}
223
224// repoJobs reads the CI config on the default branch — the same file the
225// scheduler reads — and returns its jobs with the sha they came from.
226func repoJobs(c *Ctx, repo store.Repo) ([]ci.Job, string, int) {
227 dir := RepoDir(c.Cfg.Server.Root, repo.OwnerName, repo.Name)
228 sha, err := gitutil.ResolveRef(dir, "refs/heads/"+repo.DefaultBranch)
229 if err != nil {
230 return nil, "", c.fail(protocol.ExitFailure, "resolving %s: %v", repo.DefaultBranch, err)
231 }
232 raw, err := gitutil.ReadBlob(dir, sha, ci.ConfigPath, 1<<16)
233 if err != nil {
234 return nil, "", c.fail(protocol.ExitNotFound, "%s has no %s on %s", repo.Path(), ci.ConfigPath, repo.DefaultBranch)
235 }
236 jobs, err := ci.Parse(raw)
237 if err != nil {
238 return nil, "", c.failErr(err)
239 }
240 return jobs, sha, -1
241}
242
243// runBuildJobs answers "what can I trigger?". Without it only a surface
244// that can read the repository's git could offer the choice.
245func runBuildJobs(c *Ctx, args []string) int {
246 if len(args) != 1 {
247 return c.usage()
248 }
249 repo, code := resolveRepo(c, args[0], policy.CanRead)
250 if code >= 0 {
251 return code
252 }
253 jobs, _, code := repoJobs(c, repo)
254 if code >= 0 {
255 return code
256 }
257 out := make([]JobOut, 0, len(jobs))
258 for _, j := range jobs {
259 out = append(out, JobOut{Name: j.Name, Schedule: j.Schedule, Tags: j.Tags})
260 }
261 return c.emit(out, func(w io.Writer) {
262 for _, j := range out {
263 switch {
264 case j.Schedule != "":
265 fmt.Fprintf(w, "%s\tschedule %s\n", j.Name, j.Schedule)
266 case j.Tags != "":
267 fmt.Fprintf(w, "%s\ttags %s\n", j.Name, j.Tags)
268 default:
269 fmt.Fprintf(w, "%s\ton push\n", j.Name)
270 }
271 }
272 })
273}
274
275func runBuildTrigger(c *Ctx, args []string) int {
276 if len(args) != 2 {
277 return c.usage()
278 }
279 repo, code := resolveRepo(c, args[0], policy.CanWrite)
280 if code >= 0 {
281 return code
282 }
283 jobs, sha, code := repoJobs(c, repo)
284 if code >= 0 {
285 return code
286 }
287 for _, j := range jobs {
288 if j.Name != args[1] {
289 continue
290 }
291 steps, _ := json.Marshal(j.Steps)
292 tree, _ := gitutil.ResolveTree(RepoDir(c.Cfg.Server.Root, repo.OwnerName, repo.Name), sha)
293 n, err := c.Store.CreateBuild(repo.ID, j.Name, sha, repo.DefaultBranch, string(steps), j.Image, tree, true)
294 if err != nil {
295 return c.fail(protocol.ExitFailure, "%v", err)
296 }
297 url := fmt.Sprintf("%s/%s/builds/%d", c.Cfg.Server.SiteURL, repo.Path(), n)
298 c.Store.SetCommitStatus(repo.ID, sha, "ci/"+j.Name, "pending", "triggered", url, c.User.ID)
299 return c.emit(map[string]any{"build": n, "job": j.Name, "sha": sha}, func(w io.Writer) {
300 fmt.Fprintf(w, "queued build %d (%s @ %.10s)\n", n, j.Name, sha)
301 })
302 }
303 return c.fail(protocol.ExitNotFound, "no job %q in %s", args[1], ci.ConfigPath)
304}
305
306// secretName is env-var shaped: the value lands in the build environment.
307var secretName = regexp.MustCompile(`^[A-Z_][A-Z0-9_]{0,63}$`)
308
309func runSecretSet(c *Ctx, args []string) int {
310 if len(args) != 2 {
311 return c.usage()
312 }
313 if !secretName.MatchString(args[1]) {
314 return c.fail(protocol.ExitUsage, "secret names are env-var shaped: uppercase letters, digits, _")
315 }
316 repo, code := resolveRepo(c, args[0], policy.CanAdmin)
317 if code >= 0 {
318 return code
319 }
320 raw, err := io.ReadAll(io.LimitReader(c.Stdin, 64<<10))
321 if err != nil {
322 return c.fail(protocol.ExitFailure, "reading secret: %v", err)
323 }
324 value := strings.TrimRight(string(raw), "\n")
325 if value == "" {
326 return c.fail(protocol.ExitUsage, "no value on stdin (pipe it: printf %%s TOKEN | ...)")
327 }
328 if err := c.Store.SetBuildSecret(repo.ID, args[1], value); err != nil {
329 return c.fail(protocol.ExitFailure, "%v", err)
330 }
331 return c.emit(map[string]string{"secret": args[1]}, func(w io.Writer) {
332 fmt.Fprintf(w, "secret %s set on %s\n", args[1], repo.Path())
333 })
334}
335
336func runSecretRemove(c *Ctx, args []string) int {
337 if len(args) != 2 {
338 return c.usage()
339 }
340 repo, code := resolveRepo(c, args[0], policy.CanAdmin)
341 if code >= 0 {
342 return code
343 }
344 if err := c.Store.RemoveBuildSecret(repo.ID, args[1]); err != nil {
345 if errors.Is(err, store.ErrNotFound) {
346 return c.fail(protocol.ExitNotFound, "no secret %s on %s", args[1], repo.Path())
347 }
348 return c.fail(protocol.ExitFailure, "%v", err)
349 }
350 return c.emit(map[string]string{"removed": args[1]}, func(w io.Writer) {
351 fmt.Fprintf(w, "removed %s\n", args[1])
352 })
353}
354
355func runSecretList(c *Ctx, args []string) int {
356 if len(args) != 1 {
357 return c.usage()
358 }
359 repo, code := resolveRepo(c, args[0], policy.CanAdmin)
360 if code >= 0 {
361 return code
362 }
363 names, err := c.Store.ListBuildSecretNames(repo.ID)
364 if err != nil {
365 return c.fail(protocol.ExitFailure, "%v", err)
366 }
367 return c.emit(names, func(w io.Writer) {
368 for _, n := range names {
369 fmt.Fprintln(w, n)
370 }
371 })
372}
373
374// runnerSession resolves the key behind a runner-protocol session:
375// Source is the key's fingerprint. A build is claimed by a key, so a
376// session without one is told so plainly rather than half-running. An
377// admin key is accepted so an operator can rotate at their own pace; a
378// runner host should hold a key added with --scope runner.
379func runnerSession(c *Ctx) (store.SSHKey, int) {
380 if c.Scope != "runner" && !c.User.IsAdmin {
381 return store.SSHKey{}, c.fail(protocol.ExitDenied, "runner commands need a key added with --scope runner")
382 }
383 key, err := c.Store.SSHKeyByFingerprint(c.Source)
384 if err != nil {
385 return store.SSHKey{}, c.fail(protocol.ExitDenied, "runner commands need an SSH key session")
386 }
387 return key, -1
388}
389
390// runnerAdmin reports whether a session claims builds instance-wide. The
391// bypass is the key, not the account: a scope-runner key is confined to
392// its attachments whoever owns it, including an instance admin.
393func runnerAdmin(c *Ctx) bool {
394 return c.User.IsAdmin && c.Scope != "runner"
395}
396
397// runnerMayBuild reports whether a runner session may act on a
398// repository's builds: an admin key may on any, a runner key on the
399// repositories it is attached to (#184).
400func runnerMayBuild(c *Ctx, key store.SSHKey, repoID int64) (bool, error) {
401 if runnerAdmin(c) {
402 return true, nil
403 }
404 return c.Store.RunnerAttached(key.ID, repoID)
405}
406
407// maxOrphanSkip bounds how many claimed builds runRunnerNext will find
408// unreachable and cancel in one call before giving up. Only fast-forward
409// merges are allowed here, so any branch whose target advances gets
410// rebased and force-pushed, and a stack of branches can do that repeatedly
411// in one sitting — the issue this guards saw five in an afternoon. The cap
412// is well above that, so a real backlog is never cut short, while a
413// repository whose queue is orphaned end to end still returns rather than
414// walking it forever.
415const maxOrphanSkip = 50
416
417func runRunnerNext(c *Ctx, args []string) int {
418 key, code := runnerSession(c)
419 if code >= 0 {
420 return code
421 }
422 f, err := parseFlags(args, flagSpec{Bools: []string{"--untrusted"}, MaxPos: -1,
423 Usage: "runner next [--untrusted] [<owner/name>...]"})
424 if err != nil {
425 return c.fail(protocol.ExitUsage, "%v", err)
426 }
427 // The candidate set. An admin key claims from any repository, narrowed
428 // by the names given. A runner key claims from the repositories it is
429 // attached to; a name outside them is refused, not ignored, so a
430 // misconfigured runner says so instead of idling.
431 var repoIDs []int64
432 for _, arg := range f.Pos {
433 repo, code := resolveRepo(c, arg, policy.CanRead)
434 if code >= 0 {
435 return code
436 }
437 ok, err := runnerMayBuild(c, key, repo.ID)
438 if err != nil {
439 return c.fail(protocol.ExitFailure, "%v", err)
440 }
441 if !ok {
442 return c.fail(protocol.ExitDenied, "this key is not attached to %s; a repository admin attaches it with repo runner add", repo.Path())
443 }
444 repoIDs = append(repoIDs, repo.ID)
445 }
446 if !runnerAdmin(c) && len(repoIDs) == 0 {
447 repoIDs, err = c.Store.RunnerRepoIDs(key.ID)
448 if err != nil {
449 return c.fail(protocol.ExitFailure, "%v", err)
450 }
451 if len(repoIDs) == 0 {
452 // Nothing attached: nothing to claim. Still a heartbeat, so
453 // admin runners shows the key polling.
454 c.Store.TouchRunner(key.ID, c.User.ID, "", 0)
455 return c.emit(map[string]any{}, func(w io.Writer) { fmt.Fprintln(w, "no pending builds") })
456 }
457 }
458 untrusted := f.Has("--untrusted")
459 var b store.Build
460 var repo store.Repo
461 var ok bool
462 for attempt := 0; attempt < maxOrphanSkip; attempt++ {
463 b, ok, err = c.Store.ClaimBuild(repoIDs, untrusted)
464 if err != nil {
465 return c.fail(protocol.ExitFailure, "%v", err)
466 }
467 if !ok {
468 break
469 }
470 repo, err = c.Store.RepoByID(b.RepoID)
471 if err != nil {
472 return c.fail(protocol.ExitFailure, "%v", err)
473 }
474 // Only fast-forward merges are allowed here, so a target that
475 // advances gets rebased and force-pushed, orphaning whatever was
476 // queued for the old head: the runner would clone the repo and
477 // fail at checkout with a git internal error that reads exactly
478 // like a real failure. Catch it here instead. A check that itself
479 // fails is not evidence of anything — the build runs for real and
480 // is left to fail on its own terms, never cancelled on a guess.
481 reachable, err := gitutil.Reachable(RepoDir(c.Cfg.Server.Root, repo.OwnerName, repo.Name), b.SHA)
482 if err != nil || reachable {
483 break
484 }
485 if code := cancelOrphanedBuild(c, repo, b); code >= 0 {
486 return code
487 }
488 // Cancelled, not claimed: if the cap is hit right here, the runner
489 // heartbeat below must not record this build as the one handed out.
490 b, ok = store.Build{}, false
491 }
492 // The poll itself is the runner's heartbeat: admin runners reads it.
493 c.Store.TouchRunner(key.ID, c.User.ID, strings.Join(f.Pos, ","), b.ID)
494 if !ok {
495 return c.emit(map[string]any{}, func(w io.Writer) { fmt.Fprintln(w, "no pending builds") })
496 }
497 var steps []string
498 json.Unmarshal([]byte(b.Steps), &steps)
499 // Secrets ride the claim: this channel is admin-only and the values
500 // land in the build's environment, nowhere else.
501 var secrets map[string]string
502 if b.Trusted {
503 secrets, err = c.Store.BuildSecrets(b.RepoID)
504 if err != nil {
505 return c.fail(protocol.ExitFailure, "%v", err)
506 }
507 }
508 d := struct {
509 ID int64 `json:"id"`
510 Repo string `json:"repo"`
511 Number int64 `json:"number"`
512 Job string `json:"job"`
513 SHA string `json:"sha"`
514 Ref string `json:"ref"`
515 Steps []string `json:"steps"`
516 Image string `json:"image,omitempty"`
517 Secrets map[string]string `json:"secrets,omitempty"`
518 }{b.ID, repo.Path(), b.Number, b.Job, b.SHA, b.Ref, steps, b.Image, secrets}
519 return c.emit(d, func(w io.Writer) {
520 fmt.Fprintf(w, "build %d: %s %s @ %.10s\n", d.ID, d.Repo, d.Job, d.SHA)
521 })
522}
523
524func runRunnerLog(c *Ctx, args []string) int {
525 key, code := runnerSession(c)
526 if code >= 0 {
527 return code
528 }
529 if len(args) != 1 {
530 return c.usage()
531 }
532 id, err := strconv.ParseInt(args[0], 10, 64)
533 if err != nil {
534 return c.fail(protocol.ExitUsage, "bad build id %q", args[0])
535 }
536 if b, err := c.Store.BuildByID(id); err != nil {
537 return c.fail(protocol.ExitNotFound, "no build %d", id)
538 } else if ok, err := runnerMayBuild(c, key, b.RepoID); err != nil {
539 return c.fail(protocol.ExitFailure, "%v", err)
540 } else if !ok {
541 return c.fail(protocol.ExitDenied, "this key is not attached to the build's repository; a repository admin attaches it with repo runner add")
542 }
543 // Stream stdin into the log in chunks so long builds appear live. An
544 // append that fails drops its chunk and the loop keeps draining: ending
545 // the session here breaks the runner's pipe, and a broken pipe is how a
546 // transient SQLITE_BUSY used to fail the build the log belonged to.
547 //
548 // The session is also how a running build is cancelled: while it is
549 // open the build's row is watched, and when the row stops saying
550 // running the session ends with ExitNotFound, which the runner reads as
551 // "stop this build". Any other end of the session is a lost stream.
552 type chunk struct {
553 data []byte
554 err error
555 }
556 chunks := make(chan chunk, 4)
557 go func() {
558 buf := make([]byte, 64<<10)
559 for {
560 n, rerr := c.Stdin.Read(buf)
561 if n > 0 {
562 chunks <- chunk{data: append([]byte(nil), buf[:n]...)}
563 }
564 if rerr != nil {
565 chunks <- chunk{err: rerr}
566 return
567 }
568 }
569 }()
570 watch := time.NewTicker(2 * time.Second)
571 defer watch.Stop()
572 dropped := 0
573 for {
574 select {
575 case ch := <-chunks:
576 if len(ch.data) > 0 {
577 if err := c.Store.AppendBuildLog(id, ch.data); err != nil {
578 dropped++
579 slog.Warn("appending build log", "build", id, "err", err)
580 }
581 }
582 if ch.err != nil {
583 if dropped > 0 {
584 slog.Warn("build log incomplete", "build", id, "dropped_chunks", dropped)
585 }
586 // The stream ending is the last thing the server hears
587 // from a runner that is about to die; note the time so
588 // the scheduler can fail the build if no outcome follows.
589 if err := c.Store.MarkBuildLogClosed(id); err != nil {
590 slog.Warn("marking build log closed", "build", id, "err", err)
591 }
592 return c.emit(map[string]string{"log": "ok"}, func(w io.Writer) {})
593 }
594 case <-watch.C:
595 if b, err := c.Store.BuildByID(id); err == nil && b.Status != "running" {
596 return c.fail(protocol.ExitNotFound, "build %d is %s; stop", id, b.Status)
597 }
598 }
599 }
600}
601
602func runRunnerDone(c *Ctx, args []string) int {
603 key, code := runnerSession(c)
604 if code >= 0 {
605 return code
606 }
607 if len(args) != 2 || (args[1] != "success" && args[1] != "failure") {
608 return c.usage()
609 }
610 id, err := strconv.ParseInt(args[0], 10, 64)
611 if err != nil {
612 return c.fail(protocol.ExitUsage, "bad build id %q", args[0])
613 }
614 b, err := c.Store.BuildByID(id)
615 if err != nil {
616 return c.fail(protocol.ExitNotFound, "no build %d", id)
617 }
618 if ok, err := runnerMayBuild(c, key, b.RepoID); err != nil {
619 return c.fail(protocol.ExitFailure, "%v", err)
620 } else if !ok {
621 return c.fail(protocol.ExitDenied, "this key is not attached to the build's repository; a repository admin attaches it with repo runner add")
622 }
623 // Cancelled underneath the runner: its report is late, not wrong.
624 // The row, the status and the log were settled by the cancel.
625 if b.Status == "cancelled" {
626 c.Store.RunnerDone(key.ID)
627 return c.emit(map[string]any{"build": b.Number, "status": "cancelled"}, func(w io.Writer) {
628 fmt.Fprintf(w, "build %d was cancelled\n", b.Number)
629 })
630 }
631 if err := c.Store.FinishBuild(id, args[1]); err != nil {
632 return c.fail(protocol.ExitFailure, "finishing build %d: %v", id, err)
633 }
634 c.Store.RunnerDone(key.ID)
635 repo, err := c.Store.RepoByID(b.RepoID)
636 if err != nil {
637 return c.fail(protocol.ExitFailure, "%v", err)
638 }
639 url := fmt.Sprintf("%s/%s/builds/%d", c.Cfg.Server.SiteURL, repo.Path(), b.Number)
640 desc := "build " + args[1]
641 if err := c.Store.SetCommitStatus(repo.ID, b.SHA, "ci/"+b.Job, args[1], desc, url, c.User.ID); err != nil {
642 return c.fail(protocol.ExitFailure, "%v", err)
643 }
644 c.Store.RecordEvent(repo.ID, c.User.ID, "build."+args[1],
645 fmt.Sprintf(`{"number":%d,"job":%q,"sha":%q}`, b.Number, b.Job, b.SHA))
646 // A red build mails the repo's notify targets with the log tail — a
647 // failed scheduled job must not wait to be noticed.
648 if args[1] == "failure" {
649 if targets, err := c.Store.RepoNotifyTargets(repo); err == nil {
650 tail := ""
651 if log, err := c.Store.BuildLog(id); err == nil && len(log) > 0 {
652 if len(log) > 2000 {
653 log = log[len(log)-2000:]
654 }
655 tail = string(log)
656 }
657 notify(c, targets, notice{repo: repo, kind: "build",
658 subject: fmt.Sprintf("[%s] build %d failed: %s on %s", repo.Path(), b.Number, b.Job, b.Ref),
659 action: fmt.Sprintf("build %d failed: %s on %s", b.Number, b.Job, b.Ref),
660 body: fmt.Sprintf("job %s failed at %.10s.\n\n…%s\n\n%s\n", b.Job, b.SHA, tail, url),
661 path: fmt.Sprintf("%s/builds/%d", repo.Path(), b.Number)})
662 }
663 }
664 return c.emit(map[string]any{"build": b.Number, "status": args[1]}, func(w io.Writer) {
665 fmt.Fprintf(w, "build %d %s\n", b.Number, args[1])
666 })
667}
668
669// QueueBranchBuilds reads .gitbay/ci.yml at sha and creates one pending
670// build per push job, with a pending commit status the runner resolves.
671// A broken config surfaces as a failed "ci/config" status, not silence.
672//
673// Both paths that move a branch call this: post-receive for a push, and
674// the merge path for a merge, which updates the ref directly and so never
675// reaches a hook. old is the branch's sha before this update, the diff
676// base a job's path filters run against; a new branch has no prior
677// commit and sends old as empty or all zeros. queueJobs falls back to
678// the merge base with the default branch in that case, so a filter
679// still applies to a branch's first push — the shape most changes have,
680// since branch-then-MR is the normal workflow here.
681func QueueBranchBuilds(
682 st *store.Store, root, siteURL string,
683 repo store.Repo, userID int64, branch, old, sha string, now time.Time,
684) {
685 queueJobs(st, root, siteURL, repo, userID, branch, old, sha, now, true, branch == repo.DefaultBranch, true)
686}
687
688// QueueMRBuilds queues the push jobs for a merge request head fetched
689// from another repository, which the target holds at
690// refs/merge-requests/<n>/head, so a fork's merge request has ci/<job>
691// statuses for require-checks to gate on (#98). The head is untrusted:
692// its build runs without the target's secrets. A same-repository head is
693// the branch push's job and is not queued here; a failed one is rebuilt
694// when it lands, not when it is proposed.
695func QueueMRBuilds(
696 st *store.Store, root, siteURL string,
697 repo store.Repo, userID, n int64, sha string,
698) {
699 // No old sha, and unlike QueueBranchBuilds, no merge-base fallback
700 // either: this deliberately keeps failing open and running every
701 // job. require_checks refuses a merge when an MR head has no
702 // statuses at all (mr.go), so filtering a head down to zero jobs
703 // would make it unmergeable rather than just unfiltered (#172).
704 queueJobs(st, root, siteURL, repo, userID, mrHeadRef(n), "", sha, time.Now(), false, false, false)
705}
706
707// skipReason names why a job's path filters excluded this push, mirroring
708// the order ci.Selected checks them in: an unmatched paths list rules a
709// job out before paths-ignore is even considered.
710func skipReason(j ci.Job, changed []string) string {
711 if len(j.Paths) > 0 {
712 hit := false
713 for _, f := range changed {
714 for _, p := range j.Paths {
715 if ci.Match(p, f) {
716 hit = true
717 }
718 }
719 }
720 if !hit {
721 return "no changed file matches paths"
722 }
723 }
724 return "every changed file matched paths-ignore"
725}
726
727func queueJobs(
728 st *store.Store, root, siteURL string,
729 repo store.Repo, userID int64, ref, old, sha string, now time.Time,
730 trusted, syncSchedules, deriveMergeBase bool,
731) {
732 dir := RepoDir(root, repo.OwnerName, repo.Name)
733 raw, err := gitutil.ReadBlob(dir, sha, ci.ConfigPath, 1<<16)
734 if err != nil {
735 return // no CI config at this commit
736 }
737 jobs, err := ci.Parse(raw)
738 if err != nil {
739 st.SetCommitStatus(repo.ID, sha, "ci/config", "failure", err.Error(), "", userID)
740 return
741 }
742 // A build is a fact about a commit, not a ref: a job has no branch
743 // filter, so a commit that already passed a job on another branch has
744 // nothing left to prove when a fast-forward lands it here, and one
745 // still queued or running there will say soon enough. A failed,
746 // abandoned or cancelled build does not count; that commit runs again.
747 built, err := st.BuildsForCommit(repo.ID, sha)
748 if err != nil {
749 built = nil
750 }
751 // A job's result is a property of the tree, not the commit: a rebase
752 // onto a base that touched nothing the branch did gives every commit
753 // a new sha and the same tree, and re-running the suite over it
754 // proves nothing it did not already prove (#177). A success recorded
755 // against the tree stands for the new commit.
756 tree, _ := gitutil.ResolveTree(dir, sha)
757 // The changed-file list a job's path filters run against, computed
758 // once and only if some job actually declares one. When the diff
759 // base does not exist or the diff itself fails, filtered stays
760 // false and every job runs: a filter that cannot be evaluated must
761 // not silently skip CI.
762 //
763 // A branch's first push has no old sha, but a diff base still
764 // exists: the merge base with the default branch. Without deriving
765 // one, every job runs on every new branch, and since branch-then-MR
766 // is the normal workflow, that is the push path filters matter most
767 // for. The merge base of the default branch's tip with itself is
768 // the tip, carrying no diff — that covers the default branch's own
769 // first push on a fresh repository, and must fail open rather than
770 // read as "nothing changed".
771 filtered := false
772 var changed []string
773 for _, j := range jobs {
774 if len(j.Paths) == 0 && len(j.PathsIgnore) == 0 {
775 continue
776 }
777 diffOld := old
778 // A force-push rewrote the branch, so the old tip is not an
779 // ancestor of the new one and old..new is not "what this push
780 // changed" — it is the difference between two histories. After a
781 // rebase that is whatever the new base added, typically nothing
782 // the branch itself touched, so every path filter concludes its
783 // job is unnecessary and the branch reads as green without its
784 // suite having run (#176). The merge base is the honest base:
785 // the filter is deciding about the branch's relationship to its
786 // target, which is what the merge base expresses.
787 if ci.HasDiffBase(diffOld) && deriveMergeBase {
788 if ok, err := gitutil.IsAncestor(dir, diffOld, sha); err != nil || !ok {
789 diffOld = ""
790 }
791 }
792 if !ci.HasDiffBase(diffOld) && deriveMergeBase {
793 if base, err := gitutil.MergeBase(dir, "refs/heads/"+repo.DefaultBranch, sha); err == nil && base != sha {
794 diffOld = base
795 }
796 }
797 if ci.HasDiffBase(diffOld) {
798 if files, err := gitutil.DiffFiles(dir, diffOld, sha); err == nil {
799 changed, filtered = files, true
800 }
801 }
802 break
803 }
804 var schedules []store.Schedule
805 for _, j := range jobs {
806 // Tag jobs run on matching tag pushes only.
807 if j.Tags != "" {
808 continue
809 }
810 if b, ok := built[j.Name]; ok && (b.Status == "success" || b.Status == "pending" || b.Status == "running") {
811 continue
812 }
813 if prev, ok, _ := st.SuccessBuildForTree(repo.ID, tree, j.Name); ok && prev.SHA != sha {
814 url := fmt.Sprintf("%s/%s/builds/%d", siteURL, repo.Path(), prev.Number)
815 st.SetCommitStatus(repo.ID, sha, "ci/"+j.Name, "success",
816 fmt.Sprintf("passed in build %d as %.10s, same tree", prev.Number, prev.SHA), url, userID)
817 continue
818 }
819 // Scheduled jobs run on their cron, not on push; a default-branch
820 // push (re)registers them.
821 if j.Schedule != "" {
822 if syncSchedules {
823 schedules = append(schedules, store.Schedule{
824 RepoID: repo.ID, Job: j.Name, Cron: j.Schedule,
825 NextRun: ci.NextRun(j.Schedule, now),
826 })
827 }
828 continue
829 }
830 // A filter that excludes this push is not silence: it satisfies
831 // require_checks with a skipped status instead of leaving the
832 // commit with none at all, which the gate refuses outright (#172).
833 if filtered && !ci.Selected(j, changed) {
834 st.SetCommitStatus(repo.ID, sha, "ci/"+j.Name, "skipped", skipReason(j, changed), "", userID)
835 continue
836 }
837 steps, _ := json.Marshal(j.Steps)
838 n, err := st.CreateBuild(repo.ID, j.Name, sha, ref, string(steps), j.Image, tree, trusted)
839 if err != nil {
840 slog.Error("queueing build", "repo", repo.Path(), "job", j.Name, "err", err)
841 continue
842 }
843 url := fmt.Sprintf("%s/%s/builds/%d", siteURL, repo.Path(), n)
844 st.SetCommitStatus(repo.ID, sha, "ci/"+j.Name, "pending", "queued", url, userID)
845 }
846 if syncSchedules {
847 if err := st.SyncSchedules(repo.ID, schedules); err != nil {
848 slog.Error("syncing schedules", "repo", repo.Path(), "err", err)
849 }
850 }
851}
852
853// resolveCancelledCommitStatus sets the commit status for a build that was
854// just cancelled: if the commit already passed this job on another ref,
855// that result stands again; otherwise the context reports the
856// cancellation as an error, so the queued status left behind is never
857// pending forever.
858func resolveCancelledCommitStatus(c *Ctx, repo store.Repo, b store.Build) {
859 if prev, ok, err := c.Store.SuccessBuildFor(repo.ID, b.SHA, b.Job); err == nil && ok {
860 url := fmt.Sprintf("%s/%s/builds/%d", c.Cfg.Server.SiteURL, repo.Path(), prev.Number)
861 c.Store.SetCommitStatus(repo.ID, b.SHA, "ci/"+b.Job, "success",
862 fmt.Sprintf("passed in build %d on %s", prev.Number, prev.Ref), url, c.User.ID)
863 return
864 }
865 url := fmt.Sprintf("%s/%s/builds/%d", c.Cfg.Server.SiteURL, repo.Path(), b.Number)
866 c.Store.SetCommitStatus(repo.ID, b.SHA, "ci/"+b.Job, "error", "cancelled", url, c.User.ID)
867}
868
869// cancelOrphanedBuild withdraws a build runRunnerNext claimed and then
870// found unreachable. It leaves the same shape behind as a build cancel a
871// person runs by hand: CancelBuild's status, a log line saying why, and
872// the commit status resolved rather than left pending. Returns -1 to mean
873// "handled, keep going"; anything else is the exit code to return.
874func cancelOrphanedBuild(c *Ctx, repo store.Repo, b store.Build) int {
875 if err := c.Store.CancelBuild(b.ID); err != nil {
876 return c.fail(protocol.ExitFailure, "%v", err)
877 }
878 c.Store.AppendBuildLog(b.ID, []byte(fmt.Sprintf(
879 "cancelled: %.10s is not reachable from any ref; the sha was likely orphaned by a force-push\n", b.SHA)))
880 resolveCancelledCommitStatus(c, repo, b)
881 c.Store.RecordEvent(repo.ID, c.User.ID, "build.cancelled", fmt.Sprintf(`{"number":%d,"job":%q,"sha":%q}`, b.Number, b.Job, b.SHA))
882 return -1
883}
884
885func runBuildCancel(c *Ctx, args []string) int {
886 repo, b, code := buildRef(c, args)
887 if code >= 0 {
888 return code
889 }
890 grant, err := c.Store.AccessRole(repo.ID, c.User.ID)
891 if err != nil {
892 return c.fail(protocol.ExitFailure, "%v", err)
893 }
894 if !policy.CanWrite(c.User, repo, grant) {
895 return c.fail(protocol.ExitDenied, "cancelling a build needs write access to %s; ask its owner", repo.Path())
896 }
897 if b.Status != "pending" && b.Status != "running" {
898 return c.fail(protocol.ExitUsage, "build %d is %s; only a queued or running build can be cancelled", b.Number, b.Status)
899 }
900 if err := c.Store.CancelBuild(b.ID); err != nil {
901 return c.fail(protocol.ExitFailure, "%v", err)
902 }
903 if b.Status == "running" {
904 c.Store.AppendBuildLog(b.ID, []byte(fmt.Sprintf("\ncancelled by %s while running; the runner stops at its next check\n", c.User.Username)))
905 } else {
906 c.Store.AppendBuildLog(b.ID, []byte(fmt.Sprintf("cancelled by %s before a runner claimed it\n", c.User.Username)))
907 }
908 // The queued status replaced whatever the commit had for this job.
909 resolveCancelledCommitStatus(c, repo, b)
910 c.Store.RecordEvent(repo.ID, c.User.ID, "build.cancelled", fmt.Sprintf(`{"number":%d,"job":%q,"sha":%q}`, b.Number, b.Job, b.SHA))
911 return c.emit(map[string]any{"number": b.Number, "job": b.Job, "status": "cancelled", "was": b.Status}, func(w io.Writer) {
912 if b.Status == "running" {
913 fmt.Fprintf(w, "cancelled %s build %d (%s); the runner stops at its next check\n", repo.Path(), b.Number, b.Job)
914 return
915 }
916 fmt.Fprintf(w, "cancelled %s build %d (%s)\n", repo.Path(), b.Number, b.Job)
917 })
918}