internal/control/build.go
964 lines · 36156 bytes
1package control
2
3import (
4 "encoding/json"
5 "errors"
6 "fmt"
7 "io"
8 "log/slog"
9 "regexp"
10 "slices"
11 "strconv"
12 "strings"
13 "time"
14
15 "gitbay.org/gitbay/internal/ci"
16 "gitbay.org/gitbay/internal/gitutil"
17 "gitbay.org/gitbay/internal/policy"
18 "gitbay.org/gitbay/internal/protocol"
19 "gitbay.org/gitbay/internal/store"
20)
21
22func init() {
23 register(Command{Path: []string{"build", "list"},
24 Summary: "list recent builds",
25 Usage: "build list <owner/name> [--ref <branch>] [--status <state>] [--job <name>] [--limit <n>] [--cursor <c>]",
26 Flags: []Flag{
27 {"--ref", "<branch>", "only builds on this branch", ""},
28 {"--status", "<state>", "only builds in this state", ""},
29 {"--job", "<name>", "only this job", ""},
30 {"--limit", "<n>", "rows per page", "50"},
31 {"--cursor", "<c>", "continue from the previous page", ""},
32 },
33 Examples: []string{"build list krz/gitbay --status failure"},
34 ReadOnly: true, Run: runBuildList})
35 register(Command{Path: []string{"build", "show"},
36 Summary: "show one build",
37 Usage: "build show <owner/name> <n>",
38 Examples: []string{"build show krz/gitbay 431"},
39 ReadOnly: true, Run: runBuildShow})
40 register(Command{Path: []string{"build", "log"},
41 Summary: "print a build's log, or follow it until the build ends",
42 Usage: "build log <owner/name> <n> [--follow]",
43 Flags: []Flag{
44 {"--follow", "", "stream the log until the build ends", ""},
45 },
46 Examples: []string{"build log krz/gitbay 431 --follow"},
47 ReadOnly: true, Run: runBuildLog})
48
49 register(Command{Path: []string{"build", "jobs"},
50 Summary: "list the jobs a trigger can name",
51 Usage: "build jobs <owner/name>",
52 Examples: []string{"build jobs krz/gitbay"},
53 ReadOnly: true, Run: runBuildJobs})
54
55 register(Command{Path: []string{"build", "cancel"},
56 Summary: "withdraw a queued build before a runner claims it",
57 Usage: "build cancel <owner/name> <n>",
58 Examples: []string{"build cancel krz/gitbay 431"},
59 Run: runBuildCancel})
60 register(Command{Path: []string{"build", "trigger"},
61 Summary: "queue a job now (scheduled or not)",
62 Usage: "build trigger <owner/name> <job>",
63 Examples: []string{"build trigger krz/gitbay vuln"},
64 Run: runBuildTrigger})
65 // Secrets: set over stdin, listed by name only, injected into the
66 // repo's builds as environment variables. Same discipline as mirror
67 // tokens — the value never appears in argv, logs, or output.
68 register(Command{Path: []string{"repo", "secret", "set"},
69 Summary: "set a build secret",
70 Usage: "repo secret set <owner/name> <NAME> (value on stdin)",
71 Examples: []string{"repo secret set krz/gitbay DEPLOY_TOKEN"},
72 ReadsStdin: true, Run: runSecretSet})
73 register(Command{Path: []string{"repo", "secret", "remove"},
74 Summary: "remove a build secret",
75 Usage: "repo secret remove <owner/name> <NAME>",
76 Examples: []string{"repo secret remove krz/gitbay DEPLOY_TOKEN"},
77 Run: runSecretRemove})
78 register(Command{Path: []string{"repo", "secret", "list"},
79 Summary: "list build secret names",
80 Usage: "repo secret list <owner/name>",
81 Examples: []string{"repo secret list krz/gitbay"},
82 ReadOnly: true, Run: runSecretList})
83
84 // Runner commands: the claim/report loop for gitbay-runner. A runner
85 // executes arbitrary repo code, so handing out jobs is the instance
86 // operator's call: a key added with --scope runner, which the
87 // dispatcher confines to these three commands and read-only git, or
88 // an admin key, which a runner host should not hold (#92).
89 register(Command{Path: []string{"runner", "next"},
90 Summary: "claim the oldest pending build this key may run (runner protocol)",
91 Usage: "runner next [--untrusted] [<owner/name>...]",
92 Flags: []Flag{
93 {"--untrusted", "", "this runner may build a fork's merge request head", ""},
94 },
95 Examples: []string{"runner next krz/gitbay"},
96 Run: runRunnerNext})
97 register(Command{Path: []string{"runner", "log"},
98 Summary: "append a build's log from stdin",
99 Usage: "runner log <build-id>",
100 Examples: []string{"runner log 431"},
101 ReadsStdin: true, Run: runRunnerLog})
102 register(Command{Path: []string{"runner", "done"},
103 Summary: "finish a build",
104 Usage: "runner done <build-id> success|failure",
105 Examples: []string{"runner done 431 success"},
106 Run: runRunnerDone})
107}
108
109type BuildOut struct {
110 Number int64 `json:"number"`
111 Job string `json:"job"`
112 Status string `json:"status"`
113 SHA string `json:"sha"`
114 Ref string `json:"ref"`
115 CreatedAt string `json:"created_at"`
116 FinishedAt string `json:"finished_at,omitempty"`
117 // Subject is the first line of the commit's message, so a build
118 // names what it ran on rather than only its sha (#241). It is empty
119 // when the commit is no longer in the repository.
120 Subject string `json:"subject,omitempty"`
121}
122
123func buildToOut(b store.Build) BuildOut {
124 return BuildOut{Number: b.Number, Job: b.Job, Status: b.Status, SHA: b.SHA,
125 Ref: b.Ref, CreatedAt: b.CreatedAt, FinishedAt: b.FinishedAt}
126}
127
128func buildRef(c *Ctx, args []string) (store.Repo, store.Build, int) {
129 if len(args) != 2 {
130 return store.Repo{}, store.Build{}, c.usageWith("expected <owner/name> <number>")
131 }
132 repo, code := resolveRepo(c, args[0], policy.CanRead)
133 if code >= 0 {
134 return repo, store.Build{}, code
135 }
136 n, err := strconv.ParseInt(args[1], 10, 64)
137 if err != nil {
138 return repo, store.Build{}, c.fail(protocol.ExitUsage, "bad build number %q", args[1])
139 }
140 b, err := c.Store.BuildByNumber(repo.ID, n)
141 if err != nil {
142 return repo, b, c.fail(protocol.ExitNotFound, "no build %d on %s", n, repo.Path())
143 }
144 return repo, b, -1
145}
146
147// buildStatuses is the vocabulary --status accepts, and what a bad value
148// is told to pick from.
149var buildStatuses = []string{"pending", "running", "success", "failure", "cancelled"}
150
151// buildPage is how many builds one page of build list returns when no
152// --limit is given. The cap has always been there; what it is now
153// reachable past, with --cursor (#244).
154const buildPage = 50
155
156func runBuildList(c *Ctx, args []string) int {
157 args, p, code := parsePageFlags(c, args, "build", true)
158 if code >= 0 {
159 return code
160 }
161 f, err := parseFlags(args, flagSpec{Values: []string{"--ref", "--status", "--job"}, MaxPos: 1, Usage: c.Cmd.Usage})
162 if err != nil {
163 return c.fail(protocol.ExitUsage, "%v", err)
164 }
165 path := f.pos(0)
166 if path == "" {
167 return c.usage()
168 }
169 status := f.Value("--status")
170 if f.Has("--status") && !slices.Contains(buildStatuses, status) {
171 return c.fail(protocol.ExitUsage, "--status must be one of %s", strings.Join(buildStatuses, ", "))
172 }
173 repo, code := resolveRepo(c, path, policy.CanRead)
174 if code >= 0 {
175 return code
176 }
177 limit := p.queryLimit()
178 if limit == 0 {
179 limit = buildPage
180 }
181 filter := store.BuildFilter{Ref: f.Value("--ref"), Status: status, Job: f.Value("--job"), Before: p.keyInt()}
182 builds, err := c.Store.ListBuilds(repo.ID, filter, limit)
183 if err != nil {
184 return c.fail(protocol.ExitFailure, "%v", err)
185 }
186 builds, next := trimPage(p, builds, "build", func(b store.Build) string {
187 return strconv.FormatInt(b.Number, 10)
188 })
189 var ds []BuildOut
190 for _, b := range builds {
191 ds = append(ds, buildToOut(b))
192 }
193 subjects := buildSubjects(c, repo, ds)
194 for i := range ds {
195 ds[i].Subject = subjects[ds[i].SHA]
196 }
197 return c.emitPage(p, ds, next, func(w io.Writer) {
198 tb := c.table(w, "#", "JOB", "STATUS", "SHA", "REF", "TITLE")
199 for _, d := range ds {
200 tb.row(cRef(fmt.Sprintf("%d", d.Number)), cText(d.Job), cState(d.Status), cRef(fmt.Sprintf("%.10s", d.SHA)), cText(d.Ref), cFlex(d.Subject))
201 }
202 tb.flush()
203 })
204}
205
206// buildSubjects reads the commit subject of each distinct sha on a page
207// of builds. Several jobs of one push share a commit, so the set is
208// usually far smaller than the page.
209func buildSubjects(c *Ctx, repo store.Repo, ds []BuildOut) map[string]string {
210 seen := map[string]bool{}
211 var shas []string
212 for _, d := range ds {
213 if d.SHA != "" && !seen[d.SHA] {
214 seen[d.SHA] = true
215 shas = append(shas, d.SHA)
216 }
217 }
218 return gitutil.Subjects(RepoDir(c.Cfg.Server.Root, repo.OwnerName, repo.Name), shas)
219}
220
221func runBuildShow(c *Ctx, args []string) int {
222 repo, b, code := buildRef(c, args)
223 if code >= 0 {
224 return code
225 }
226 d := buildToOut(b)
227 return c.emit(d, func(w io.Writer) {
228 v := c.view(w)
229 v.title(fmt.Sprintf("#%d", d.Number), d.Job, d.Status)
230 v.fields(
231 "sha", fmt.Sprintf("%.10s", d.SHA),
232 "ref", d.Ref,
233 "queued", c.when(d.CreatedAt),
234 "finished", c.when(d.FinishedAt),
235 "url", c.siteURL(repo.Path(), "builds", strconv.FormatInt(d.Number, 10)),
236 )
237 })
238}
239
240func runBuildLog(c *Ctx, args []string) int {
241 f, err := parseFlags(args, flagSpec{Bools: []string{"--follow"}, MaxPos: 2, Usage: c.Cmd.Usage})
242 if err != nil {
243 return c.fail(protocol.ExitUsage, "%v", err)
244 }
245 repo, b, code := buildRef(c, f.Pos)
246 if code >= 0 {
247 return code
248 }
249 if f.Has("--follow") {
250 return followBuildLog(c, repo, b)
251 }
252 log, err := c.Store.BuildLog(b.ID)
253 if err != nil {
254 return c.fail(protocol.ExitFailure, "%v", err)
255 }
256 c.Stdout.Write(log)
257 return protocol.ExitOK
258}
259
260type JobOut struct {
261 Name string `json:"name"`
262 Schedule string `json:"schedule,omitempty"`
263 Tags string `json:"tags,omitempty"`
264}
265
266// repoJobs reads the CI config on the default branch — the same file the
267// scheduler reads — and returns its jobs with the sha they came from.
268func repoJobs(c *Ctx, repo store.Repo) ([]ci.Job, string, int) {
269 dir := RepoDir(c.Cfg.Server.Root, repo.OwnerName, repo.Name)
270 sha, err := gitutil.ResolveRef(dir, "refs/heads/"+repo.DefaultBranch)
271 if err != nil {
272 return nil, "", c.fail(protocol.ExitFailure, "resolving %s: %v", repo.DefaultBranch, err)
273 }
274 raw, err := gitutil.ReadBlob(dir, sha, ci.ConfigPath, 1<<16)
275 if err != nil {
276 return nil, "", c.fail(protocol.ExitNotFound, "%s has no %s on %s", repo.Path(), ci.ConfigPath, repo.DefaultBranch)
277 }
278 jobs, err := ci.Parse(raw)
279 if err != nil {
280 return nil, "", c.failErr(err)
281 }
282 return jobs, sha, -1
283}
284
285// runBuildJobs answers "what can I trigger?". Without it only a surface
286// that can read the repository's git could offer the choice.
287func runBuildJobs(c *Ctx, args []string) int {
288 if len(args) != 1 {
289 return c.usage()
290 }
291 repo, code := resolveRepo(c, args[0], policy.CanRead)
292 if code >= 0 {
293 return code
294 }
295 jobs, _, code := repoJobs(c, repo)
296 if code >= 0 {
297 return code
298 }
299 out := make([]JobOut, 0, len(jobs))
300 for _, j := range jobs {
301 out = append(out, JobOut{Name: j.Name, Schedule: j.Schedule, Tags: j.Tags})
302 }
303 return c.emit(out, func(w io.Writer) {
304 tb := c.table(w, "NAME", "WHEN")
305 for _, j := range out {
306 when := "on push"
307 switch {
308 case j.Schedule != "":
309 when = "schedule " + j.Schedule
310 case j.Tags != "":
311 when = "tags " + j.Tags
312 }
313 tb.row(cRef(j.Name), cText(when))
314 }
315 tb.flush()
316 })
317}
318
319func runBuildTrigger(c *Ctx, args []string) int {
320 if len(args) != 2 {
321 return c.usage()
322 }
323 repo, code := resolveRepo(c, args[0], policy.CanWrite)
324 if code >= 0 {
325 return code
326 }
327 jobs, sha, code := repoJobs(c, repo)
328 if code >= 0 {
329 return code
330 }
331 for _, j := range jobs {
332 if j.Name != args[1] {
333 continue
334 }
335 steps, _ := json.Marshal(j.Steps)
336 tree, _ := gitutil.ResolveTree(RepoDir(c.Cfg.Server.Root, repo.OwnerName, repo.Name), sha)
337 n, err := c.Store.CreateBuild(repo.ID, j.Name, sha, repo.DefaultBranch, string(steps), j.Image, tree, true)
338 if err != nil {
339 return c.fail(protocol.ExitFailure, "%v", err)
340 }
341 url := fmt.Sprintf("%s/%s/builds/%d", c.Cfg.Server.SiteURL, repo.Path(), n)
342 c.Store.SetCommitStatus(repo.ID, sha, "ci/"+j.Name, "pending", "triggered", url, c.User.ID)
343 return c.emit(map[string]any{"build": n, "job": j.Name, "sha": sha}, func(w io.Writer) {
344 fmt.Fprintf(w, "queued build %d (%s @ %.10s)\n", n, j.Name, sha)
345 })
346 }
347 return c.fail(protocol.ExitNotFound, "no job %q in %s", args[1], ci.ConfigPath)
348}
349
350// secretName is env-var shaped: the value lands in the build environment.
351var secretName = regexp.MustCompile(`^[A-Z_][A-Z0-9_]{0,63}$`)
352
353func runSecretSet(c *Ctx, args []string) int {
354 if len(args) != 2 {
355 return c.usage()
356 }
357 if !secretName.MatchString(args[1]) {
358 return c.fail(protocol.ExitUsage, "secret names are env-var shaped: uppercase letters, digits, _")
359 }
360 repo, code := resolveRepo(c, args[0], policy.CanAdmin)
361 if code >= 0 {
362 return code
363 }
364 raw, err := io.ReadAll(io.LimitReader(c.Stdin, 64<<10))
365 if err != nil {
366 return c.fail(protocol.ExitFailure, "reading secret: %v", err)
367 }
368 value := strings.TrimRight(string(raw), "\n")
369 if value == "" {
370 return c.fail(protocol.ExitUsage, "no value on stdin (pipe it: printf %%s TOKEN | ...)")
371 }
372 if err := c.Store.SetBuildSecret(repo.ID, args[1], value); err != nil {
373 return c.fail(protocol.ExitFailure, "%v", err)
374 }
375 return c.emit(map[string]string{"secret": args[1]}, func(w io.Writer) {
376 fmt.Fprintf(w, "secret %s set on %s\n", args[1], repo.Path())
377 })
378}
379
380func runSecretRemove(c *Ctx, args []string) int {
381 if len(args) != 2 {
382 return c.usage()
383 }
384 repo, code := resolveRepo(c, args[0], policy.CanAdmin)
385 if code >= 0 {
386 return code
387 }
388 if err := c.Store.RemoveBuildSecret(repo.ID, args[1]); err != nil {
389 if errors.Is(err, store.ErrNotFound) {
390 return c.fail(protocol.ExitNotFound, "no secret %s on %s", args[1], repo.Path())
391 }
392 return c.fail(protocol.ExitFailure, "%v", err)
393 }
394 return c.emit(map[string]string{"removed": args[1]}, func(w io.Writer) {
395 fmt.Fprintf(w, "removed %s\n", args[1])
396 })
397}
398
399func runSecretList(c *Ctx, args []string) int {
400 if len(args) != 1 {
401 return c.usage()
402 }
403 repo, code := resolveRepo(c, args[0], policy.CanAdmin)
404 if code >= 0 {
405 return code
406 }
407 names, err := c.Store.ListBuildSecretNames(repo.ID)
408 if err != nil {
409 return c.fail(protocol.ExitFailure, "%v", err)
410 }
411 return c.emit(names, func(w io.Writer) {
412 tb := c.table(w, "NAME")
413 for _, n := range names {
414 tb.row(cRef(n))
415 }
416 tb.flush()
417 })
418}
419
420// runnerSession resolves the key behind a runner-protocol session:
421// Source is the key's fingerprint. A build is claimed by a key, so a
422// session without one is told so plainly rather than half-running. An
423// admin key is accepted so an operator can rotate at their own pace; a
424// runner host should hold a key added with --scope runner.
425func runnerSession(c *Ctx) (store.SSHKey, int) {
426 if c.Scope != "runner" && !c.User.IsAdmin {
427 return store.SSHKey{}, c.fail(protocol.ExitDenied, "runner commands need a key added with --scope runner")
428 }
429 key, err := c.Store.SSHKeyByFingerprint(c.Source)
430 if err != nil {
431 return store.SSHKey{}, c.fail(protocol.ExitDenied, "runner commands need an SSH key session")
432 }
433 return key, -1
434}
435
436// runnerAdmin reports whether a session claims builds instance-wide. The
437// bypass is the key, not the account: a scope-runner key is confined to
438// its attachments whoever owns it, including an instance admin.
439func runnerAdmin(c *Ctx) bool {
440 return c.User.IsAdmin && c.Scope != "runner"
441}
442
443// runnerMayBuild reports whether a runner session may act on a
444// repository's builds: an admin key may on any, a runner key on the
445// repositories it is attached to (#184).
446func runnerMayBuild(c *Ctx, key store.SSHKey, repoID int64) (bool, error) {
447 if runnerAdmin(c) {
448 return true, nil
449 }
450 return c.Store.RunnerAttached(key.ID, repoID)
451}
452
453// maxOrphanSkip bounds how many claimed builds runRunnerNext will find
454// unreachable and cancel in one call before giving up. Only fast-forward
455// merges are allowed here, so any branch whose target advances gets
456// rebased and force-pushed, and a stack of branches can do that repeatedly
457// in one sitting — the issue this guards saw five in an afternoon. The cap
458// is well above that, so a real backlog is never cut short, while a
459// repository whose queue is orphaned end to end still returns rather than
460// walking it forever.
461const maxOrphanSkip = 50
462
463func runRunnerNext(c *Ctx, args []string) int {
464 key, code := runnerSession(c)
465 if code >= 0 {
466 return code
467 }
468 f, err := parseFlags(args, flagSpec{Bools: []string{"--untrusted"}, MaxPos: -1,
469 Usage: "runner next [--untrusted] [<owner/name>...]"})
470 if err != nil {
471 return c.fail(protocol.ExitUsage, "%v", err)
472 }
473 // The candidate set. An admin key claims from any repository, narrowed
474 // by the names given. A runner key claims from the repositories it is
475 // attached to; a name outside them is refused, not ignored, so a
476 // misconfigured runner says so instead of idling.
477 var repoIDs []int64
478 for _, arg := range f.Pos {
479 repo, code := resolveRepo(c, arg, policy.CanRead)
480 if code >= 0 {
481 return code
482 }
483 ok, err := runnerMayBuild(c, key, repo.ID)
484 if err != nil {
485 return c.fail(protocol.ExitFailure, "%v", err)
486 }
487 if !ok {
488 return c.fail(protocol.ExitDenied, "this key is not attached to %s; a repository admin attaches it with repo runner add", repo.Path())
489 }
490 repoIDs = append(repoIDs, repo.ID)
491 }
492 if !runnerAdmin(c) && len(repoIDs) == 0 {
493 repoIDs, err = c.Store.RunnerRepoIDs(key.ID)
494 if err != nil {
495 return c.fail(protocol.ExitFailure, "%v", err)
496 }
497 if len(repoIDs) == 0 {
498 // Nothing attached: nothing to claim. Still a heartbeat, so
499 // admin runners shows the key polling.
500 c.Store.TouchRunner(key.ID, c.User.ID, "", 0)
501 return c.emit(map[string]any{}, func(w io.Writer) { fmt.Fprintln(w, "no pending builds") })
502 }
503 }
504 untrusted := f.Has("--untrusted")
505 var b store.Build
506 var repo store.Repo
507 var ok bool
508 for attempt := 0; attempt < maxOrphanSkip; attempt++ {
509 b, ok, err = c.Store.ClaimBuild(repoIDs, untrusted)
510 if err != nil {
511 return c.fail(protocol.ExitFailure, "%v", err)
512 }
513 if !ok {
514 break
515 }
516 repo, err = c.Store.RepoByID(b.RepoID)
517 if err != nil {
518 return c.fail(protocol.ExitFailure, "%v", err)
519 }
520 // Only fast-forward merges are allowed here, so a target that
521 // advances gets rebased and force-pushed, orphaning whatever was
522 // queued for the old head: the runner would clone the repo and
523 // fail at checkout with a git internal error that reads exactly
524 // like a real failure. Catch it here instead. A check that itself
525 // fails is not evidence of anything — the build runs for real and
526 // is left to fail on its own terms, never cancelled on a guess.
527 reachable, err := gitutil.Reachable(RepoDir(c.Cfg.Server.Root, repo.OwnerName, repo.Name), b.SHA)
528 if err != nil || reachable {
529 break
530 }
531 if code := cancelOrphanedBuild(c, repo, b); code >= 0 {
532 return code
533 }
534 // Cancelled, not claimed: if the cap is hit right here, the runner
535 // heartbeat below must not record this build as the one handed out.
536 b, ok = store.Build{}, false
537 }
538 // The poll itself is the runner's heartbeat: admin runners reads it.
539 c.Store.TouchRunner(key.ID, c.User.ID, strings.Join(f.Pos, ","), b.ID)
540 if !ok {
541 return c.emit(map[string]any{}, func(w io.Writer) { fmt.Fprintln(w, "no pending builds") })
542 }
543 var steps []string
544 json.Unmarshal([]byte(b.Steps), &steps)
545 // Secrets ride the claim: this channel is admin-only and the values
546 // land in the build's environment, nowhere else.
547 var secrets map[string]string
548 if b.Trusted {
549 secrets, err = c.Store.BuildSecrets(b.RepoID)
550 if err != nil {
551 return c.fail(protocol.ExitFailure, "%v", err)
552 }
553 }
554 d := struct {
555 ID int64 `json:"id"`
556 Repo string `json:"repo"`
557 Number int64 `json:"number"`
558 Job string `json:"job"`
559 SHA string `json:"sha"`
560 Ref string `json:"ref"`
561 Steps []string `json:"steps"`
562 Image string `json:"image,omitempty"`
563 Secrets map[string]string `json:"secrets,omitempty"`
564 }{b.ID, repo.Path(), b.Number, b.Job, b.SHA, b.Ref, steps, b.Image, secrets}
565 return c.emit(d, func(w io.Writer) {
566 fmt.Fprintf(w, "build %d: %s %s @ %.10s\n", d.ID, d.Repo, d.Job, d.SHA)
567 })
568}
569
570func runRunnerLog(c *Ctx, args []string) int {
571 key, code := runnerSession(c)
572 if code >= 0 {
573 return code
574 }
575 if len(args) != 1 {
576 return c.usage()
577 }
578 id, err := strconv.ParseInt(args[0], 10, 64)
579 if err != nil {
580 return c.fail(protocol.ExitUsage, "bad build id %q", args[0])
581 }
582 if b, err := c.Store.BuildByID(id); err != nil {
583 return c.fail(protocol.ExitNotFound, "no build %d", id)
584 } else if ok, err := runnerMayBuild(c, key, b.RepoID); err != nil {
585 return c.fail(protocol.ExitFailure, "%v", err)
586 } else if !ok {
587 return c.fail(protocol.ExitDenied, "this key is not attached to the build's repository; a repository admin attaches it with repo runner add")
588 }
589 // Stream stdin into the log in chunks so long builds appear live. An
590 // append that fails drops its chunk and the loop keeps draining: ending
591 // the session here breaks the runner's pipe, and a broken pipe is how a
592 // transient SQLITE_BUSY used to fail the build the log belonged to.
593 //
594 // The session is also how a running build is cancelled: while it is
595 // open the build's row is watched, and when the row stops saying
596 // running the session ends with ExitNotFound, which the runner reads as
597 // "stop this build". Any other end of the session is a lost stream.
598 type chunk struct {
599 data []byte
600 err error
601 }
602 chunks := make(chan chunk, 4)
603 go func() {
604 buf := make([]byte, 64<<10)
605 for {
606 n, rerr := c.Stdin.Read(buf)
607 if n > 0 {
608 chunks <- chunk{data: append([]byte(nil), buf[:n]...)}
609 }
610 if rerr != nil {
611 chunks <- chunk{err: rerr}
612 return
613 }
614 }
615 }()
616 watch := time.NewTicker(2 * time.Second)
617 defer watch.Stop()
618 dropped := 0
619 for {
620 select {
621 case ch := <-chunks:
622 if len(ch.data) > 0 {
623 if err := c.Store.AppendBuildLog(id, ch.data); err != nil {
624 dropped++
625 slog.Warn("appending build log", "build", id, "err", err)
626 }
627 }
628 if ch.err != nil {
629 if dropped > 0 {
630 slog.Warn("build log incomplete", "build", id, "dropped_chunks", dropped)
631 }
632 // The stream ending is the last thing the server hears
633 // from a runner that is about to die; note the time so
634 // the scheduler can fail the build if no outcome follows.
635 if err := c.Store.MarkBuildLogClosed(id); err != nil {
636 slog.Warn("marking build log closed", "build", id, "err", err)
637 }
638 return c.emit(map[string]string{"log": "ok"}, func(w io.Writer) {})
639 }
640 case <-watch.C:
641 if b, err := c.Store.BuildByID(id); err == nil && b.Status != "running" {
642 return c.fail(protocol.ExitNotFound, "build %d is %s; stop", id, b.Status)
643 }
644 }
645 }
646}
647
648func runRunnerDone(c *Ctx, args []string) int {
649 key, code := runnerSession(c)
650 if code >= 0 {
651 return code
652 }
653 if len(args) != 2 || (args[1] != "success" && args[1] != "failure") {
654 return c.usage()
655 }
656 id, err := strconv.ParseInt(args[0], 10, 64)
657 if err != nil {
658 return c.fail(protocol.ExitUsage, "bad build id %q", args[0])
659 }
660 b, err := c.Store.BuildByID(id)
661 if err != nil {
662 return c.fail(protocol.ExitNotFound, "no build %d", id)
663 }
664 if ok, err := runnerMayBuild(c, key, b.RepoID); err != nil {
665 return c.fail(protocol.ExitFailure, "%v", err)
666 } else if !ok {
667 return c.fail(protocol.ExitDenied, "this key is not attached to the build's repository; a repository admin attaches it with repo runner add")
668 }
669 // Cancelled underneath the runner: its report is late, not wrong.
670 // The row, the status and the log were settled by the cancel.
671 if b.Status == "cancelled" {
672 c.Store.RunnerDone(key.ID)
673 return c.emit(map[string]any{"build": b.Number, "status": "cancelled"}, func(w io.Writer) {
674 fmt.Fprintf(w, "build %d was cancelled\n", b.Number)
675 })
676 }
677 if err := c.Store.FinishBuild(id, args[1]); err != nil {
678 return c.fail(protocol.ExitFailure, "finishing build %d: %v", id, err)
679 }
680 c.Store.RunnerDone(key.ID)
681 repo, err := c.Store.RepoByID(b.RepoID)
682 if err != nil {
683 return c.fail(protocol.ExitFailure, "%v", err)
684 }
685 url := fmt.Sprintf("%s/%s/builds/%d", c.Cfg.Server.SiteURL, repo.Path(), b.Number)
686 desc := "build " + args[1]
687 if err := c.Store.SetCommitStatus(repo.ID, b.SHA, "ci/"+b.Job, args[1], desc, url, c.User.ID); err != nil {
688 return c.fail(protocol.ExitFailure, "%v", err)
689 }
690 c.Store.RecordEvent(repo.ID, c.User.ID, "build."+args[1],
691 fmt.Sprintf(`{"number":%d,"job":%q,"sha":%q}`, b.Number, b.Job, b.SHA))
692 // A red build mails the repo's notify targets with the log tail — a
693 // failed scheduled job must not wait to be noticed.
694 if args[1] == "failure" {
695 if targets, err := c.Store.RepoNotifyTargets(repo); err == nil {
696 tail := ""
697 if log, err := c.Store.BuildLog(id); err == nil && len(log) > 0 {
698 if len(log) > 2000 {
699 log = log[len(log)-2000:]
700 }
701 tail = string(log)
702 }
703 notify(c, targets, notice{repo: repo, kind: "build",
704 subject: fmt.Sprintf("[%s] build %d failed: %s on %s", repo.Path(), b.Number, b.Job, b.Ref),
705 action: fmt.Sprintf("build %d failed: %s on %s", b.Number, b.Job, b.Ref),
706 body: fmt.Sprintf("job %s failed at %.10s.\n\n…%s\n\n%s\n", b.Job, b.SHA, tail, url),
707 path: fmt.Sprintf("%s/builds/%d", repo.Path(), b.Number)})
708 }
709 }
710 return c.emit(map[string]any{"build": b.Number, "status": args[1]}, func(w io.Writer) {
711 fmt.Fprintf(w, "build %d %s\n", b.Number, args[1])
712 })
713}
714
715// QueueBranchBuilds reads .gitbay/ci.yml at sha and creates one pending
716// build per push job, with a pending commit status the runner resolves.
717// A broken config surfaces as a failed "ci/config" status, not silence.
718//
719// Both paths that move a branch call this: post-receive for a push, and
720// the merge path for a merge, which updates the ref directly and so never
721// reaches a hook. old is the branch's sha before this update, the diff
722// base a job's path filters run against; a new branch has no prior
723// commit and sends old as empty or all zeros. queueJobs falls back to
724// the merge base with the default branch in that case, so a filter
725// still applies to a branch's first push — the shape most changes have,
726// since branch-then-MR is the normal workflow here.
727func QueueBranchBuilds(
728 st *store.Store, root, siteURL string,
729 repo store.Repo, userID int64, branch, old, sha string, now time.Time,
730) {
731 queueJobs(st, root, siteURL, repo, userID, branch, old, sha, now, true, branch == repo.DefaultBranch, true)
732}
733
734// QueueMRBuilds queues the push jobs for a merge request head fetched
735// from another repository, which the target holds at
736// refs/merge-requests/<n>/head, so a fork's merge request has ci/<job>
737// statuses for require-checks to gate on (#98). The head is untrusted:
738// its build runs without the target's secrets. A same-repository head is
739// the branch push's job and is not queued here; a failed one is rebuilt
740// when it lands, not when it is proposed.
741func QueueMRBuilds(
742 st *store.Store, root, siteURL string,
743 repo store.Repo, userID, n int64, sha string,
744) {
745 // No old sha, and unlike QueueBranchBuilds, no merge-base fallback
746 // either: this deliberately keeps failing open and running every
747 // job. require_checks refuses a merge when an MR head has no
748 // statuses at all (mr.go), so filtering a head down to zero jobs
749 // would make it unmergeable rather than just unfiltered (#172).
750 queueJobs(st, root, siteURL, repo, userID, mrHeadRef(n), "", sha, time.Now(), false, false, false)
751}
752
753// skipReason names why a job's path filters excluded this push, mirroring
754// the order ci.Selected checks them in: an unmatched paths list rules a
755// job out before paths-ignore is even considered.
756func skipReason(j ci.Job, changed []string) string {
757 if len(j.Paths) > 0 {
758 hit := false
759 for _, f := range changed {
760 for _, p := range j.Paths {
761 if ci.Match(p, f) {
762 hit = true
763 }
764 }
765 }
766 if !hit {
767 return "no changed file matches paths"
768 }
769 }
770 return "every changed file matched paths-ignore"
771}
772
773func queueJobs(
774 st *store.Store, root, siteURL string,
775 repo store.Repo, userID int64, ref, old, sha string, now time.Time,
776 trusted, syncSchedules, deriveMergeBase bool,
777) {
778 dir := RepoDir(root, repo.OwnerName, repo.Name)
779 raw, err := gitutil.ReadBlob(dir, sha, ci.ConfigPath, 1<<16)
780 if err != nil {
781 return // no CI config at this commit
782 }
783 jobs, err := ci.Parse(raw)
784 if err != nil {
785 st.SetCommitStatus(repo.ID, sha, "ci/config", "failure", err.Error(), "", userID)
786 return
787 }
788 // A build is a fact about a commit, not a ref: a job has no branch
789 // filter, so a commit that already passed a job on another branch has
790 // nothing left to prove when a fast-forward lands it here, and one
791 // still queued or running there will say soon enough. A failed,
792 // abandoned or cancelled build does not count; that commit runs again.
793 built, err := st.BuildsForCommit(repo.ID, sha)
794 if err != nil {
795 built = nil
796 }
797 // A job's result is a property of the tree, not the commit: a rebase
798 // onto a base that touched nothing the branch did gives every commit
799 // a new sha and the same tree, and re-running the suite over it
800 // proves nothing it did not already prove (#177). A success recorded
801 // against the tree stands for the new commit.
802 tree, _ := gitutil.ResolveTree(dir, sha)
803 // The changed-file list a job's path filters run against, computed
804 // once and only if some job actually declares one. When the diff
805 // base does not exist or the diff itself fails, filtered stays
806 // false and every job runs: a filter that cannot be evaluated must
807 // not silently skip CI.
808 //
809 // A branch's first push has no old sha, but a diff base still
810 // exists: the merge base with the default branch. Without deriving
811 // one, every job runs on every new branch, and since branch-then-MR
812 // is the normal workflow, that is the push path filters matter most
813 // for. The merge base of the default branch's tip with itself is
814 // the tip, carrying no diff — that covers the default branch's own
815 // first push on a fresh repository, and must fail open rather than
816 // read as "nothing changed".
817 filtered := false
818 var changed []string
819 for _, j := range jobs {
820 if len(j.Paths) == 0 && len(j.PathsIgnore) == 0 {
821 continue
822 }
823 diffOld := old
824 // A force-push rewrote the branch, so the old tip is not an
825 // ancestor of the new one and old..new is not "what this push
826 // changed" — it is the difference between two histories. After a
827 // rebase that is whatever the new base added, typically nothing
828 // the branch itself touched, so every path filter concludes its
829 // job is unnecessary and the branch reads as green without its
830 // suite having run (#176). The merge base is the honest base:
831 // the filter is deciding about the branch's relationship to its
832 // target, which is what the merge base expresses.
833 if ci.HasDiffBase(diffOld) && deriveMergeBase {
834 if ok, err := gitutil.IsAncestor(dir, diffOld, sha); err != nil || !ok {
835 diffOld = ""
836 }
837 }
838 if !ci.HasDiffBase(diffOld) && deriveMergeBase {
839 if base, err := gitutil.MergeBase(dir, "refs/heads/"+repo.DefaultBranch, sha); err == nil && base != sha {
840 diffOld = base
841 }
842 }
843 if ci.HasDiffBase(diffOld) {
844 if files, err := gitutil.DiffFiles(dir, diffOld, sha); err == nil {
845 changed, filtered = files, true
846 }
847 }
848 break
849 }
850 var schedules []store.Schedule
851 for _, j := range jobs {
852 // Tag jobs run on matching tag pushes only.
853 if j.Tags != "" {
854 continue
855 }
856 if b, ok := built[j.Name]; ok && (b.Status == "success" || b.Status == "pending" || b.Status == "running") {
857 continue
858 }
859 if prev, ok, _ := st.SuccessBuildForTree(repo.ID, tree, j.Name); ok && prev.SHA != sha {
860 url := fmt.Sprintf("%s/%s/builds/%d", siteURL, repo.Path(), prev.Number)
861 st.SetCommitStatus(repo.ID, sha, "ci/"+j.Name, "success",
862 fmt.Sprintf("passed in build %d as %.10s, same tree", prev.Number, prev.SHA), url, userID)
863 continue
864 }
865 // Scheduled jobs run on their cron, not on push; a default-branch
866 // push (re)registers them.
867 if j.Schedule != "" {
868 if syncSchedules {
869 schedules = append(schedules, store.Schedule{
870 RepoID: repo.ID, Job: j.Name, Cron: j.Schedule,
871 NextRun: ci.NextRun(j.Schedule, now),
872 })
873 }
874 continue
875 }
876 // A filter that excludes this push is not silence: it satisfies
877 // require_checks with a skipped status instead of leaving the
878 // commit with none at all, which the gate refuses outright (#172).
879 if filtered && !ci.Selected(j, changed) {
880 st.SetCommitStatus(repo.ID, sha, "ci/"+j.Name, "skipped", skipReason(j, changed), "", userID)
881 continue
882 }
883 steps, _ := json.Marshal(j.Steps)
884 n, err := st.CreateBuild(repo.ID, j.Name, sha, ref, string(steps), j.Image, tree, trusted)
885 if err != nil {
886 slog.Error("queueing build", "repo", repo.Path(), "job", j.Name, "err", err)
887 continue
888 }
889 url := fmt.Sprintf("%s/%s/builds/%d", siteURL, repo.Path(), n)
890 st.SetCommitStatus(repo.ID, sha, "ci/"+j.Name, "pending", "queued", url, userID)
891 }
892 if syncSchedules {
893 if err := st.SyncSchedules(repo.ID, schedules); err != nil {
894 slog.Error("syncing schedules", "repo", repo.Path(), "err", err)
895 }
896 }
897}
898
899// resolveCancelledCommitStatus sets the commit status for a build that was
900// just cancelled: if the commit already passed this job on another ref,
901// that result stands again; otherwise the context reports the
902// cancellation as an error, so the queued status left behind is never
903// pending forever.
904func resolveCancelledCommitStatus(c *Ctx, repo store.Repo, b store.Build) {
905 if prev, ok, err := c.Store.SuccessBuildFor(repo.ID, b.SHA, b.Job); err == nil && ok {
906 url := fmt.Sprintf("%s/%s/builds/%d", c.Cfg.Server.SiteURL, repo.Path(), prev.Number)
907 c.Store.SetCommitStatus(repo.ID, b.SHA, "ci/"+b.Job, "success",
908 fmt.Sprintf("passed in build %d on %s", prev.Number, prev.Ref), url, c.User.ID)
909 return
910 }
911 url := fmt.Sprintf("%s/%s/builds/%d", c.Cfg.Server.SiteURL, repo.Path(), b.Number)
912 c.Store.SetCommitStatus(repo.ID, b.SHA, "ci/"+b.Job, "error", "cancelled", url, c.User.ID)
913}
914
915// cancelOrphanedBuild withdraws a build runRunnerNext claimed and then
916// found unreachable. It leaves the same shape behind as a build cancel a
917// person runs by hand: CancelBuild's status, a log line saying why, and
918// the commit status resolved rather than left pending. Returns -1 to mean
919// "handled, keep going"; anything else is the exit code to return.
920func cancelOrphanedBuild(c *Ctx, repo store.Repo, b store.Build) int {
921 if err := c.Store.CancelBuild(b.ID); err != nil {
922 return c.fail(protocol.ExitFailure, "%v", err)
923 }
924 c.Store.AppendBuildLog(b.ID, []byte(fmt.Sprintf(
925 "cancelled: %.10s is not reachable from any ref; the sha was likely orphaned by a force-push\n", b.SHA)))
926 resolveCancelledCommitStatus(c, repo, b)
927 c.Store.RecordEvent(repo.ID, c.User.ID, "build.cancelled", fmt.Sprintf(`{"number":%d,"job":%q,"sha":%q}`, b.Number, b.Job, b.SHA))
928 return -1
929}
930
931func runBuildCancel(c *Ctx, args []string) int {
932 repo, b, code := buildRef(c, args)
933 if code >= 0 {
934 return code
935 }
936 grant, err := c.Store.AccessRole(repo.ID, c.User.ID)
937 if err != nil {
938 return c.fail(protocol.ExitFailure, "%v", err)
939 }
940 if !policy.CanWrite(c.User, repo, grant) {
941 return c.fail(protocol.ExitDenied, "cancelling a build needs write access to %s; ask its owner", repo.Path())
942 }
943 if b.Status != "pending" && b.Status != "running" {
944 return c.fail(protocol.ExitUsage, "build %d is %s; only a queued or running build can be cancelled", b.Number, b.Status)
945 }
946 if err := c.Store.CancelBuild(b.ID); err != nil {
947 return c.fail(protocol.ExitFailure, "%v", err)
948 }
949 if b.Status == "running" {
950 c.Store.AppendBuildLog(b.ID, []byte(fmt.Sprintf("\ncancelled by %s while running; the runner stops at its next check\n", c.User.Username)))
951 } else {
952 c.Store.AppendBuildLog(b.ID, []byte(fmt.Sprintf("cancelled by %s before a runner claimed it\n", c.User.Username)))
953 }
954 // The queued status replaced whatever the commit had for this job.
955 resolveCancelledCommitStatus(c, repo, b)
956 c.Store.RecordEvent(repo.ID, c.User.ID, "build.cancelled", fmt.Sprintf(`{"number":%d,"job":%q,"sha":%q}`, b.Number, b.Job, b.SHA))
957 return c.emit(map[string]any{"number": b.Number, "job": b.Job, "status": "cancelled", "was": b.Status}, func(w io.Writer) {
958 if b.Status == "running" {
959 fmt.Fprintf(w, "cancelled %s build %d (%s); the runner stops at its next check\n", repo.Path(), b.Number, b.Job)
960 return
961 }
962 fmt.Fprintf(w, "cancelled %s build %d (%s)\n", repo.Path(), b.Number, b.Job)
963 })
964}