internal/control/build.go
1095 lines · 41311 bytes
1package control
2
3import (
4 "encoding/json"
5 "errors"
6 "fmt"
7 "io"
8 "log/slog"
9 "net"
10 "regexp"
11 "slices"
12 "strconv"
13 "strings"
14 "time"
15
16 "gitbay.org/gitbay/internal/ci"
17 "gitbay.org/gitbay/internal/gitutil"
18 "gitbay.org/gitbay/internal/policy"
19 "gitbay.org/gitbay/internal/protocol"
20 "gitbay.org/gitbay/internal/store"
21)
22
23func init() {
24 register(Command{Path: []string{"build", "list"},
25 Summary: "list recent builds",
26 Usage: "build list <owner/name> [--ref <branch>] [--status <state>] [--job <name>] [--limit <n>] [--cursor <c>]",
27 Flags: []Flag{
28 {"--ref", "<branch>", "only builds on this branch", ""},
29 {"--status", "<state>", "only builds in this state", ""},
30 {"--job", "<name>", "only this job", ""},
31 {"--limit", "<n>", "rows per page", "50"},
32 {"--cursor", "<c>", "continue from the previous page", ""},
33 },
34 Examples: []string{"build list krz/gitbay --status failure"},
35 ReadOnly: true, Run: runBuildList})
36 register(Command{Path: []string{"build", "show"},
37 Summary: "show one build",
38 Usage: "build show <owner/name> <n>",
39 Examples: []string{"build show krz/gitbay 431"},
40 ReadOnly: true, Run: runBuildShow})
41 register(Command{Path: []string{"build", "log"},
42 Summary: "print a build's log, or follow it until the build ends",
43 Usage: "build log <owner/name> <n> [--follow] [--step <step>|failed] [--tail <lines>]",
44 Flags: []Flag{
45 {"--follow", "", "stream the log until the build ends", ""},
46 {"--step", "<step>|failed", "only one step's output: 0 for the setup, a step number, or the one that failed", ""},
47 {"--tail", "<lines>", "only the last lines", ""},
48 },
49 Examples: []string{"build log krz/gitbay 431 --follow", "build log krz/gitbay 431 --step failed --tail 40"},
50 ReadOnly: true, Run: runBuildLog})
51
52 register(Command{Path: []string{"build", "jobs"},
53 Summary: "list the jobs a trigger can name",
54 Usage: "build jobs <owner/name>",
55 Examples: []string{"build jobs krz/gitbay"},
56 ReadOnly: true, Run: runBuildJobs})
57
58 register(Command{Path: []string{"build", "cancel"},
59 Summary: "withdraw a queued build before a runner claims it",
60 Usage: "build cancel <owner/name> <n>",
61 Examples: []string{"build cancel krz/gitbay 431"},
62 Run: runBuildCancel})
63 register(Command{Path: []string{"build", "trigger"},
64 Summary: "queue a job now (scheduled or not)",
65 Usage: "build trigger <owner/name> <job>",
66 Examples: []string{"build trigger krz/gitbay vuln"},
67 Run: runBuildTrigger})
68 // Secrets: set over stdin, listed by name only, injected into the
69 // repo's builds as environment variables. Same discipline as mirror
70 // tokens — the value never appears in argv, logs, or output.
71 register(Command{Path: []string{"repo", "secret", "set"},
72 NeedsRecentSignIn: true,
73 Summary: "set a build secret",
74 Usage: "repo secret set <owner/name> <NAME> (value on stdin)",
75 Examples: []string{"repo secret set krz/gitbay DEPLOY_TOKEN"},
76 ReadsStdin: true, Run: runSecretSet})
77 register(Command{Path: []string{"repo", "secret", "remove"},
78 Summary: "remove a build secret",
79 Usage: "repo secret remove <owner/name> <NAME>",
80 Examples: []string{"repo secret remove krz/gitbay DEPLOY_TOKEN"},
81 Run: runSecretRemove})
82 register(Command{Path: []string{"repo", "secret", "list"},
83 Summary: "list build secret names",
84 Usage: "repo secret list <owner/name>",
85 Examples: []string{"repo secret list krz/gitbay"},
86 ReadOnly: true, Run: runSecretList})
87
88 // Runner commands: the claim/report loop for gitbay-runner. A runner
89 // executes arbitrary repo code, so handing out jobs is the instance
90 // operator's call: a key added with --scope runner, which the
91 // dispatcher confines to these three commands and read-only git, or
92 // an admin key, which a runner host should not hold (#92).
93 register(Command{Path: []string{"runner", "next"},
94 Summary: "claim the oldest pending build this key may run (runner protocol)",
95 Usage: "runner next [--untrusted] [<owner/name>...]",
96 Flags: []Flag{
97 {"--untrusted", "", "this runner may build a fork's merge request head", ""},
98 },
99 Examples: []string{"runner next krz/gitbay"},
100 Run: runRunnerNext})
101 register(Command{Path: []string{"runner", "log"},
102 Summary: "append a build's log from stdin",
103 Usage: "runner log <build-id>",
104 Examples: []string{"runner log 431"},
105 ReadsStdin: true, Run: runRunnerLog})
106 register(Command{Path: []string{"runner", "done"},
107 Summary: "finish a build",
108 Usage: "runner done <build-id> success|failure [--step <n>] [--reason <text>]",
109 Flags: []Flag{
110 {"--step", "<n>", "the 1-based step a failed build stopped at", ""},
111 {"--reason", "<text>", "how it failed, one line", ""},
112 },
113 Examples: []string{"runner done 431 success", "runner done 431 failure --step 3 --reason 'exit 1'"},
114 Run: runRunnerDone})
115}
116
117type BuildOut struct {
118 Number int64 `json:"number"`
119 Job string `json:"job"`
120 Status string `json:"status"`
121 SHA string `json:"sha"`
122 Ref string `json:"ref"`
123 CreatedAt string `json:"created_at"`
124 FinishedAt string `json:"finished_at,omitempty"`
125 // Subject is the first line of the commit's message, so a build
126 // names what it ran on rather than only its sha (#241). It is empty
127 // when the commit is no longer in the repository.
128 Subject string `json:"subject,omitempty"`
129 // FailedStep is the 1-based step a failed build stopped at, 0 when
130 // none; FailedReason says how ("exit 1") (#266).
131 FailedStep int `json:"failed_step,omitempty"`
132 FailedReason string `json:"failed_reason,omitempty"`
133 // DurationS is how long the build ran, once it has a start and a
134 // finish.
135 DurationS int64 `json:"duration_s,omitempty"`
136 // Steps are the job's commands; build show only.
137 Steps []string `json:"steps,omitempty"`
138}
139
140func buildToOut(b store.Build) BuildOut {
141 return BuildOut{Number: b.Number, Job: b.Job, Status: b.Status, SHA: b.SHA,
142 Ref: b.Ref, CreatedAt: b.CreatedAt, FinishedAt: b.FinishedAt,
143 FailedStep: b.FailedStep, FailedReason: b.FailedReason,
144 DurationS: int64(b.Elapsed() / time.Second)}
145}
146
147func buildRef(c *Ctx, args []string) (store.Repo, store.Build, int) {
148 if len(args) != 2 {
149 return store.Repo{}, store.Build{}, c.usageWith("expected <owner/name> <number>")
150 }
151 repo, code := resolveRepo(c, args[0], policy.CanRead)
152 if code >= 0 {
153 return repo, store.Build{}, code
154 }
155 n, err := strconv.ParseInt(args[1], 10, 64)
156 if err != nil {
157 return repo, store.Build{}, c.fail(protocol.ExitUsage, "bad build number %q", args[1])
158 }
159 b, err := c.Store.BuildByNumber(repo.ID, n)
160 if err != nil {
161 return repo, b, c.fail(protocol.ExitNotFound, "no build %d on %s", n, repo.Path())
162 }
163 return repo, b, -1
164}
165
166// buildStatuses is the vocabulary --status accepts, and what a bad value
167// is told to pick from.
168var buildStatuses = []string{"pending", "running", "success", "failure", "cancelled"}
169
170// buildPage is how many builds one page of build list returns when no
171// --limit is given. The cap has always been there; what it is now
172// reachable past, with --cursor (#244).
173const buildPage = 50
174
175func runBuildList(c *Ctx, args []string) int {
176 args, p, code := parsePageFlags(c, args, "build", true)
177 if code >= 0 {
178 return code
179 }
180 f, err := c.parseArgs(args, flagSpec{Values: []string{"--ref", "--status", "--job"}, MaxPos: 1, Usage: c.Cmd.Usage})
181 if err != nil {
182 return c.fail(protocol.ExitUsage, "%v", err)
183 }
184 path := f.pos(0)
185 if path == "" {
186 return c.usage()
187 }
188 status := f.Value("--status")
189 if f.Has("--status") && !slices.Contains(buildStatuses, status) {
190 return c.fail(protocol.ExitUsage, "--status must be one of %s", strings.Join(buildStatuses, ", "))
191 }
192 repo, code := resolveRepo(c, path, policy.CanRead)
193 if code >= 0 {
194 return code
195 }
196 limit := p.queryLimit()
197 if limit == 0 {
198 limit = buildPage
199 }
200 filter := store.BuildFilter{Ref: f.Value("--ref"), Status: status, Job: f.Value("--job"), Before: p.keyInt()}
201 builds, err := c.Store.ListBuilds(repo.ID, filter, limit)
202 if err != nil {
203 return c.fail(protocol.ExitFailure, "%v", err)
204 }
205 builds, next := trimPage(p, builds, "build", func(b store.Build) string {
206 return strconv.FormatInt(b.Number, 10)
207 })
208 var ds []BuildOut
209 for _, b := range builds {
210 ds = append(ds, buildToOut(b))
211 }
212 subjects := buildSubjects(c, repo, ds)
213 for i := range ds {
214 ds[i].Subject = subjects[ds[i].SHA]
215 }
216 return c.emitPage(p, ds, next, func(w io.Writer) {
217 tb := c.table(w, "#", "JOB", "STATUS", "SHA", "REF", "TITLE")
218 for _, d := range ds {
219 tb.row(cRef(fmt.Sprintf("%d", d.Number)), cText(d.Job), cState(d.Status), cRef(fmt.Sprintf("%.10s", d.SHA)), cText(d.Ref), cFlex(d.Subject))
220 }
221 tb.flush()
222 })
223}
224
225// buildSubjects reads the commit subject of each distinct sha on a page
226// of builds. Several jobs of one push share a commit, so the set is
227// usually far smaller than the page.
228func buildSubjects(c *Ctx, repo store.Repo, ds []BuildOut) map[string]string {
229 seen := map[string]bool{}
230 var shas []string
231 for _, d := range ds {
232 if d.SHA != "" && !seen[d.SHA] {
233 seen[d.SHA] = true
234 shas = append(shas, d.SHA)
235 }
236 }
237 return gitutil.Subjects(RepoDir(c.Cfg.Server.Root, repo.OwnerName, repo.Name), shas)
238}
239
240func runBuildShow(c *Ctx, args []string) int {
241 repo, b, code := buildRef(c, args)
242 if code >= 0 {
243 return code
244 }
245 d := buildToOut(b)
246 json.Unmarshal([]byte(b.Steps), &d.Steps)
247 return c.emit(d, func(w io.Writer) {
248 failedStep, failed := "", ""
249 if d.FailedStep > 0 && d.FailedStep <= len(d.Steps) {
250 step, _, _ := strings.Cut(d.Steps[d.FailedStep-1], "\n")
251 failedStep = fmt.Sprintf("%d/%d %s", d.FailedStep, len(d.Steps), step)
252 if d.FailedReason != "" {
253 failedStep += " (" + d.FailedReason + ")"
254 }
255 } else {
256 failed = d.FailedReason
257 }
258 duration := ""
259 if d.DurationS > 0 {
260 duration = (time.Duration(d.DurationS) * time.Second).String()
261 }
262 v := c.view(w)
263 v.title(fmt.Sprintf("#%d", d.Number), d.Job, d.Status)
264 v.fields(
265 "sha", fmt.Sprintf("%.10s", d.SHA),
266 "ref", d.Ref,
267 "queued", c.when(d.CreatedAt),
268 "finished", c.when(d.FinishedAt),
269 "duration", duration,
270 "failed step", failedStep,
271 "failed", failed,
272 "url", c.siteURL(repo.Path(), "builds", strconv.FormatInt(d.Number, 10)),
273 )
274 })
275}
276
277func runBuildLog(c *Ctx, args []string) int {
278 f, err := c.parseArgs(args, flagSpec{Bools: []string{"--follow"}, Values: []string{"--step", "--tail"}, MaxPos: 2, Usage: c.Cmd.Usage})
279 if err != nil {
280 return c.fail(protocol.ExitUsage, "%v", err)
281 }
282 repo, b, code := buildRef(c, f.Pos)
283 if code >= 0 {
284 return code
285 }
286 if f.Has("--follow") {
287 if f.Has("--step") || f.Has("--tail") {
288 return c.fail(protocol.ExitUsage, "--step and --tail read the stored log; drop --follow")
289 }
290 return followBuildLog(c, repo, b)
291 }
292 tail := 0
293 if f.Has("--tail") {
294 if tail, err = strconv.Atoi(f.Value("--tail")); err != nil || tail < 1 {
295 return c.fail(protocol.ExitUsage, "--tail takes a number of lines, 1 or more")
296 }
297 }
298 log, err := c.Store.BuildLog(b.ID)
299 if err != nil {
300 return c.fail(protocol.ExitFailure, "%v", err)
301 }
302 if f.Has("--step") {
303 var steps []string
304 json.Unmarshal([]byte(b.Steps), &steps)
305 sections := SplitBuildLog(string(log), steps)
306 at := -1
307 if want := f.Value("--step"); want == "failed" {
308 if at = FailedSection(sections, b.Status, b.FailedStep); at < 0 {
309 return c.fail(protocol.ExitNotFound, "build %d did not fail", b.Number)
310 }
311 } else {
312 n, err := strconv.Atoi(want)
313 if err != nil || n < 0 || n > len(steps) {
314 return c.fail(protocol.ExitUsage, "--step takes 0 (the setup) to %d, or failed", len(steps))
315 }
316 for i, s := range sections {
317 if s.N == n {
318 at = i
319 }
320 }
321 if at < 0 {
322 return c.fail(protocol.ExitNotFound, "build %d has no output for step %d", b.Number, n)
323 }
324 }
325 log = []byte(sections[at].Text)
326 }
327 if tail > 0 {
328 log = tailLines(log, tail)
329 }
330 c.Stdout.Write(log)
331 return protocol.ExitOK
332}
333
334type JobOut struct {
335 Name string `json:"name"`
336 Schedule string `json:"schedule,omitempty"`
337 Tags string `json:"tags,omitempty"`
338}
339
340// repoJobs reads the CI config on the default branch — the same file the
341// scheduler reads — and returns its jobs with the sha they came from.
342func repoJobs(c *Ctx, repo store.Repo) ([]ci.Job, string, int) {
343 dir := RepoDir(c.Cfg.Server.Root, repo.OwnerName, repo.Name)
344 sha, err := gitutil.ResolveRef(dir, "refs/heads/"+repo.DefaultBranch)
345 if err != nil {
346 return nil, "", c.fail(protocol.ExitFailure, "resolving %s: %v", repo.DefaultBranch, err)
347 }
348 raw, err := gitutil.ReadBlob(dir, sha, ci.ConfigPath, 1<<16)
349 if err != nil {
350 return nil, "", c.fail(protocol.ExitNotFound, "%s has no %s on %s", repo.Path(), ci.ConfigPath, repo.DefaultBranch)
351 }
352 jobs, err := ci.Parse(raw)
353 if err != nil {
354 return nil, "", c.failErr(err)
355 }
356 return jobs, sha, -1
357}
358
359// runBuildJobs answers "what can I trigger?". Without it only a surface
360// that can read the repository's git could offer the choice.
361func runBuildJobs(c *Ctx, args []string) int {
362 if len(args) != 1 {
363 return c.usage()
364 }
365 repo, code := resolveRepo(c, args[0], policy.CanRead)
366 if code >= 0 {
367 return code
368 }
369 jobs, _, code := repoJobs(c, repo)
370 if code >= 0 {
371 return code
372 }
373 out := make([]JobOut, 0, len(jobs))
374 for _, j := range jobs {
375 out = append(out, JobOut{Name: j.Name, Schedule: j.Schedule, Tags: j.Tags})
376 }
377 return c.emit(out, func(w io.Writer) {
378 tb := c.table(w, "NAME", "WHEN")
379 for _, j := range out {
380 when := "on push"
381 switch {
382 case j.Schedule != "":
383 when = "schedule " + j.Schedule
384 case j.Tags != "":
385 when = "tags " + j.Tags
386 }
387 tb.row(cRef(j.Name), cText(when))
388 }
389 tb.flush()
390 })
391}
392
393func runBuildTrigger(c *Ctx, args []string) int {
394 if len(args) != 2 {
395 return c.usage()
396 }
397 repo, code := resolveRepo(c, args[0], policy.CanWrite)
398 if code >= 0 {
399 return code
400 }
401 jobs, sha, code := repoJobs(c, repo)
402 if code >= 0 {
403 return code
404 }
405 for _, j := range jobs {
406 if j.Name != args[1] {
407 continue
408 }
409 steps, _ := json.Marshal(j.Steps)
410 tree, _ := gitutil.ResolveTree(RepoDir(c.Cfg.Server.Root, repo.OwnerName, repo.Name), sha)
411 n, err := c.Store.CreateBuild(repo.ID, j.Name, sha, repo.DefaultBranch, string(steps), j.Image, tree, true)
412 if err != nil {
413 return c.fail(protocol.ExitFailure, "%v", err)
414 }
415 url := fmt.Sprintf("%s/%s/builds/%d", c.Cfg.Server.SiteURL, repo.Path(), n)
416 c.Store.SetCommitStatus(repo.ID, sha, "ci/"+j.Name, "pending", "triggered", url, c.User.ID)
417 return c.emit(map[string]any{"build": n, "job": j.Name, "sha": sha}, func(w io.Writer) {
418 fmt.Fprintf(w, "queued build %d (%s @ %.10s)\n", n, j.Name, sha)
419 })
420 }
421 return c.fail(protocol.ExitNotFound, "no job %q in %s", args[1], ci.ConfigPath)
422}
423
424// secretName is env-var shaped: the value lands in the build environment.
425var secretName = regexp.MustCompile(`^[A-Z_][A-Z0-9_]{0,63}$`)
426
427func runSecretSet(c *Ctx, args []string) int {
428 if len(args) != 2 {
429 return c.usage()
430 }
431 if !secretName.MatchString(args[1]) {
432 return c.fail(protocol.ExitUsage, "secret names are env-var shaped: uppercase letters, digits, _")
433 }
434 repo, code := resolveRepo(c, args[0], policy.CanAdmin)
435 if code >= 0 {
436 return code
437 }
438 raw, err := io.ReadAll(io.LimitReader(c.Stdin, 64<<10))
439 if err != nil {
440 return c.fail(protocol.ExitFailure, "reading secret: %v", err)
441 }
442 value := strings.TrimRight(string(raw), "\n")
443 if value == "" {
444 return c.fail(protocol.ExitUsage, "no value on stdin (pipe it: printf %%s TOKEN | ...)")
445 }
446 if err := c.Store.SetBuildSecret(repo.ID, args[1], value); err != nil {
447 return c.fail(protocol.ExitFailure, "%v", err)
448 }
449 return c.emit(map[string]string{"secret": args[1]}, func(w io.Writer) {
450 fmt.Fprintf(w, "secret %s set on %s\n", args[1], repo.Path())
451 })
452}
453
454func runSecretRemove(c *Ctx, args []string) int {
455 if len(args) != 2 {
456 return c.usage()
457 }
458 repo, code := resolveRepo(c, args[0], policy.CanAdmin)
459 if code >= 0 {
460 return code
461 }
462 if err := c.Store.RemoveBuildSecret(repo.ID, args[1]); err != nil {
463 if errors.Is(err, store.ErrNotFound) {
464 return c.fail(protocol.ExitNotFound, "no secret %s on %s", args[1], repo.Path())
465 }
466 return c.fail(protocol.ExitFailure, "%v", err)
467 }
468 return c.emit(map[string]string{"removed": args[1]}, func(w io.Writer) {
469 fmt.Fprintf(w, "removed %s\n", args[1])
470 })
471}
472
473func runSecretList(c *Ctx, args []string) int {
474 if len(args) != 1 {
475 return c.usage()
476 }
477 repo, code := resolveRepo(c, args[0], policy.CanAdmin)
478 if code >= 0 {
479 return code
480 }
481 names, err := c.Store.ListBuildSecretNames(repo.ID)
482 if err != nil {
483 return c.fail(protocol.ExitFailure, "%v", err)
484 }
485 return c.emit(names, func(w io.Writer) {
486 tb := c.table(w, "NAME")
487 for _, n := range names {
488 tb.row(cRef(n))
489 }
490 tb.flush()
491 })
492}
493
494// runnerSession resolves the key behind a runner-protocol session:
495// Source is the key's fingerprint. A build is claimed by a key, so a
496// session without one is told so plainly rather than half-running. An
497// admin key is accepted so an operator can rotate at their own pace; a
498// runner host should hold a key added with --scope runner.
499func runnerSession(c *Ctx) (store.SSHKey, int) {
500 if c.Scope != "runner" && !c.User.IsAdmin {
501 return store.SSHKey{}, c.fail(protocol.ExitDenied, "runner commands need a key added with --scope runner")
502 }
503 key, err := c.Store.SSHKeyByFingerprint(c.Source)
504 if err != nil {
505 return store.SSHKey{}, c.fail(protocol.ExitDenied, "runner commands need an SSH key session")
506 }
507 return key, -1
508}
509
510// runnerAdmin reports whether a session claims builds instance-wide. The
511// bypass is the key, not the account: a scope-runner key is confined to
512// its attachments whoever owns it, including an instance admin.
513func runnerAdmin(c *Ctx) bool {
514 return c.User.IsAdmin && c.Scope != "runner"
515}
516
517// runnerMayBuild reports whether a runner session may act on a
518// repository's builds: an admin key may on any, a runner key on the
519// repositories it is attached to (#184).
520func runnerMayBuild(c *Ctx, key store.SSHKey, repoID int64) (bool, error) {
521 if runnerAdmin(c) {
522 return true, nil
523 }
524 return c.Store.RunnerAttached(key.ID, repoID)
525}
526
527// maxOrphanSkip bounds how many claimed builds runRunnerNext will find
528// unreachable and cancel in one call before giving up. Only fast-forward
529// merges are allowed here, so any branch whose target advances gets
530// rebased and force-pushed, and a stack of branches can do that repeatedly
531// in one sitting — the issue this guards saw five in an afternoon. The cap
532// is well above that, so a real backlog is never cut short, while a
533// repository whose queue is orphaned end to end still returns rather than
534// walking it forever.
535const maxOrphanSkip = 50
536
537// publicSSH is the instance's ssh destination as anyone outside reaches
538// it. A runner on the daemon's own host polls over loopback and takes
539// the port from it for its builds' GITBAY_SSH, which names pasta's
540// address for the host (#260). The port is added only when it is not
541// 22: hutch and orgo build ssh://$GITBAY_SSH/... URLs, valid in both
542// forms. Empty when site_url is not set.
543func publicSSH(c *Ctx) string {
544 host := c.Cfg.SiteHost()
545 if host == "" {
546 return ""
547 }
548 if p := c.Cfg.SSH.Port; p != 0 && p != 22 {
549 return "git@" + net.JoinHostPort(host, strconv.Itoa(p))
550 }
551 return "git@" + host
552}
553
554func runRunnerNext(c *Ctx, args []string) int {
555 key, code := runnerSession(c)
556 if code >= 0 {
557 return code
558 }
559 f, err := c.parseArgs(args, flagSpec{Bools: []string{"--untrusted"}, MaxPos: -1,
560 Usage: "runner next [--untrusted] [<owner/name>...]"})
561 if err != nil {
562 return c.fail(protocol.ExitUsage, "%v", err)
563 }
564 // The candidate set. An admin key claims from any repository, narrowed
565 // by the names given. A runner key claims from the repositories it is
566 // attached to; a name outside them is refused, not ignored, so a
567 // misconfigured runner says so instead of idling.
568 var repoIDs []int64
569 for _, arg := range f.Pos {
570 repo, code := resolveRepo(c, arg, policy.CanRead)
571 if code >= 0 {
572 return code
573 }
574 ok, err := runnerMayBuild(c, key, repo.ID)
575 if err != nil {
576 return c.fail(protocol.ExitFailure, "%v", err)
577 }
578 if !ok {
579 return c.fail(protocol.ExitDenied, "this key is not attached to %s; a repository admin attaches it with repo runner add", repo.Path())
580 }
581 repoIDs = append(repoIDs, repo.ID)
582 }
583 if !runnerAdmin(c) && len(repoIDs) == 0 {
584 repoIDs, err = c.Store.RunnerRepoIDs(key.ID)
585 if err != nil {
586 return c.fail(protocol.ExitFailure, "%v", err)
587 }
588 if len(repoIDs) == 0 {
589 // Nothing attached: nothing to claim. Still a heartbeat, so
590 // admin runners shows the key polling.
591 c.Store.TouchRunner(key.ID, c.User.ID, "", 0)
592 return c.emit(map[string]any{}, func(w io.Writer) { fmt.Fprintln(w, "no pending builds") })
593 }
594 }
595 untrusted := f.Has("--untrusted")
596 var b store.Build
597 var repo store.Repo
598 var ok bool
599 for attempt := 0; attempt < maxOrphanSkip; attempt++ {
600 b, ok, err = c.Store.ClaimBuild(repoIDs, untrusted)
601 if err != nil {
602 return c.fail(protocol.ExitFailure, "%v", err)
603 }
604 if !ok {
605 break
606 }
607 repo, err = c.Store.RepoByID(b.RepoID)
608 if err != nil {
609 return c.fail(protocol.ExitFailure, "%v", err)
610 }
611 // Only fast-forward merges are allowed here, so a target that
612 // advances gets rebased and force-pushed, orphaning whatever was
613 // queued for the old head: the runner would clone the repo and
614 // fail at checkout with a git internal error that reads exactly
615 // like a real failure. Catch it here instead. A check that itself
616 // fails is not evidence of anything — the build runs for real and
617 // is left to fail on its own terms, never cancelled on a guess.
618 reachable, err := gitutil.Reachable(RepoDir(c.Cfg.Server.Root, repo.OwnerName, repo.Name), b.SHA)
619 if err != nil || reachable {
620 break
621 }
622 if code := cancelOrphanedBuild(c, repo, b); code >= 0 {
623 return code
624 }
625 // Cancelled, not claimed: if the cap is hit right here, the runner
626 // heartbeat below must not record this build as the one handed out.
627 b, ok = store.Build{}, false
628 }
629 // The poll itself is the runner's heartbeat: admin runners reads it.
630 c.Store.TouchRunner(key.ID, c.User.ID, strings.Join(f.Pos, ","), b.ID)
631 if !ok {
632 return c.emit(map[string]any{}, func(w io.Writer) { fmt.Fprintln(w, "no pending builds") })
633 }
634 var steps []string
635 json.Unmarshal([]byte(b.Steps), &steps)
636 // Secrets ride the claim: this channel is admin-only and the values
637 // land in the build's environment, nowhere else.
638 var secrets map[string]string
639 if b.Trusted {
640 secrets, err = c.Store.BuildSecrets(b.RepoID)
641 if err != nil {
642 return c.fail(protocol.ExitFailure, "%v", err)
643 }
644 }
645 d := struct {
646 ID int64 `json:"id"`
647 Repo string `json:"repo"`
648 Number int64 `json:"number"`
649 Job string `json:"job"`
650 SHA string `json:"sha"`
651 Ref string `json:"ref"`
652 Steps []string `json:"steps"`
653 Image string `json:"image,omitempty"`
654 // Trusted is always sent: a runner decides a build's home and
655 // secrets from it, and reads a missing field as untrusted (#255).
656 Trusted bool `json:"trusted"`
657 // SSH is the instance's public destination; a runner polling
658 // over loopback takes its port for the build's GITBAY_SSH (#260).
659 SSH string `json:"ssh,omitempty"`
660 Secrets map[string]string `json:"secrets,omitempty"`
661 }{ID: b.ID, Repo: repo.Path(), Number: b.Number, Job: b.Job, SHA: b.SHA, Ref: b.Ref,
662 Steps: steps, Image: b.Image, Trusted: b.Trusted, SSH: publicSSH(c), Secrets: secrets}
663 return c.emit(d, func(w io.Writer) {
664 fmt.Fprintf(w, "build %d: %s %s @ %.10s\n", d.ID, d.Repo, d.Job, d.SHA)
665 })
666}
667
668func runRunnerLog(c *Ctx, args []string) int {
669 key, code := runnerSession(c)
670 if code >= 0 {
671 return code
672 }
673 if len(args) != 1 {
674 return c.usage()
675 }
676 id, err := strconv.ParseInt(args[0], 10, 64)
677 if err != nil {
678 return c.fail(protocol.ExitUsage, "bad build id %q", args[0])
679 }
680 if b, err := c.Store.BuildByID(id); err != nil {
681 return c.fail(protocol.ExitNotFound, "no build %d", id)
682 } else if ok, err := runnerMayBuild(c, key, b.RepoID); err != nil {
683 return c.fail(protocol.ExitFailure, "%v", err)
684 } else if !ok {
685 return c.fail(protocol.ExitDenied, "this key is not attached to the build's repository; a repository admin attaches it with repo runner add")
686 }
687 // Stream stdin into the log in chunks so long builds appear live. An
688 // append that fails drops its chunk and the loop keeps draining: ending
689 // the session here breaks the runner's pipe, and a broken pipe is how a
690 // transient SQLITE_BUSY used to fail the build the log belonged to.
691 //
692 // The session is also how a running build is cancelled: while it is
693 // open the build's row is watched, and when the row stops saying
694 // running the session ends with ExitNotFound, which the runner reads as
695 // "stop this build". Any other end of the session is a lost stream.
696 type chunk struct {
697 data []byte
698 err error
699 }
700 chunks := make(chan chunk, 4)
701 go func() {
702 buf := make([]byte, 64<<10)
703 for {
704 n, rerr := c.Stdin.Read(buf)
705 if n > 0 {
706 chunks <- chunk{data: append([]byte(nil), buf[:n]...)}
707 }
708 if rerr != nil {
709 chunks <- chunk{err: rerr}
710 return
711 }
712 }
713 }()
714 watch := time.NewTicker(2 * time.Second)
715 defer watch.Stop()
716 dropped := 0
717 for {
718 select {
719 case ch := <-chunks:
720 if len(ch.data) > 0 {
721 if err := c.Store.AppendBuildLog(id, ch.data); err != nil {
722 dropped++
723 slog.Warn("appending build log", "build", id, "err", err)
724 }
725 }
726 if ch.err != nil {
727 if dropped > 0 {
728 slog.Warn("build log incomplete", "build", id, "dropped_chunks", dropped)
729 }
730 // The stream ending is the last thing the server hears
731 // from a runner that is about to die; note the time so
732 // the scheduler can fail the build if no outcome follows.
733 if err := c.Store.MarkBuildLogClosed(id); err != nil {
734 slog.Warn("marking build log closed", "build", id, "err", err)
735 }
736 return c.emit(map[string]string{"log": "ok"}, func(w io.Writer) {})
737 }
738 case <-watch.C:
739 if b, err := c.Store.BuildByID(id); err == nil && b.Status != "running" {
740 return c.fail(protocol.ExitNotFound, "build %d is %s; stop", id, b.Status)
741 }
742 }
743 }
744}
745
746func runRunnerDone(c *Ctx, args []string) int {
747 key, code := runnerSession(c)
748 if code >= 0 {
749 return code
750 }
751 f, err := c.parseArgs(args, flagSpec{Values: []string{"--step", "--reason"}, MaxPos: 2, Usage: c.Cmd.Usage})
752 if err != nil {
753 return c.fail(protocol.ExitUsage, "%v", err)
754 }
755 if len(f.Pos) != 2 || (f.Pos[1] != "success" && f.Pos[1] != "failure") {
756 return c.usage()
757 }
758 outcome := f.Pos[1]
759 id, err := strconv.ParseInt(f.Pos[0], 10, 64)
760 if err != nil {
761 return c.fail(protocol.ExitUsage, "bad build id %q", f.Pos[0])
762 }
763 b, err := c.Store.BuildByID(id)
764 if err != nil {
765 return c.fail(protocol.ExitNotFound, "no build %d", id)
766 }
767 if ok, err := runnerMayBuild(c, key, b.RepoID); err != nil {
768 return c.fail(protocol.ExitFailure, "%v", err)
769 } else if !ok {
770 return c.fail(protocol.ExitDenied, "this key is not attached to the build's repository; a repository admin attaches it with repo runner add")
771 }
772 // Cancelled underneath the runner: its report is late, not wrong.
773 // The row, the status and the log were settled by the cancel.
774 if b.Status == "cancelled" {
775 c.Store.RunnerDone(key.ID)
776 return c.emit(map[string]any{"build": b.Number, "status": "cancelled"}, func(w io.Writer) {
777 fmt.Fprintf(w, "build %d was cancelled\n", b.Number)
778 })
779 }
780 if outcome == "failure" {
781 // A step the job does not have is recorded as none rather than
782 // refused: refusing would lose the outcome over a detail (#266).
783 var steps []string
784 json.Unmarshal([]byte(b.Steps), &steps)
785 step, _ := strconv.Atoi(f.Value("--step"))
786 if step < 0 || step > len(steps) {
787 step = 0
788 }
789 if err := c.Store.SetBuildFailure(id, step, failureReason(f.Value("--reason"))); err != nil && !errors.Is(err, store.ErrNotFound) {
790 return c.fail(protocol.ExitFailure, "recording build %d's failure: %v", id, err)
791 }
792 }
793 if err := c.Store.FinishBuild(id, outcome); err != nil {
794 return c.fail(protocol.ExitFailure, "finishing build %d: %v", id, err)
795 }
796 c.Store.RunnerDone(key.ID)
797 repo, err := c.Store.RepoByID(b.RepoID)
798 if err != nil {
799 return c.fail(protocol.ExitFailure, "%v", err)
800 }
801 url := fmt.Sprintf("%s/%s/builds/%d", c.Cfg.Server.SiteURL, repo.Path(), b.Number)
802 desc := "build " + outcome
803 if err := c.Store.SetCommitStatus(repo.ID, b.SHA, "ci/"+b.Job, outcome, desc, url, c.User.ID); err != nil {
804 return c.fail(protocol.ExitFailure, "%v", err)
805 }
806 c.Store.RecordEvent(repo.ID, c.User.ID, "build."+outcome,
807 fmt.Sprintf(`{"number":%d,"job":%q,"sha":%q}`, b.Number, b.Job, b.SHA))
808 // A red build mails the repo's notify targets with the log tail — a
809 // failed scheduled job must not wait to be noticed.
810 if outcome == "failure" {
811 if targets, err := c.Store.RepoNotifyTargets(repo); err == nil {
812 tail := ""
813 if log, err := c.Store.BuildLog(id); err == nil && len(log) > 0 {
814 if len(log) > 2000 {
815 log = log[len(log)-2000:]
816 }
817 tail = string(log)
818 }
819 notify(c, targets, notice{repo: repo, kind: "build",
820 subject: fmt.Sprintf("[%s] build %d failed: %s on %s", repo.Path(), b.Number, b.Job, b.Ref),
821 action: fmt.Sprintf("build %d failed: %s on %s", b.Number, b.Job, b.Ref),
822 body: fmt.Sprintf("job %s failed at %.10s.\n\n…%s\n\n%s\n", b.Job, b.SHA, tail, url),
823 path: fmt.Sprintf("%s/builds/%d", repo.Path(), b.Number)})
824 }
825 }
826 return c.emit(map[string]any{"build": b.Number, "status": outcome}, func(w io.Writer) {
827 fmt.Fprintf(w, "build %d %s\n", b.Number, outcome)
828 })
829}
830
831// failureReason keeps a runner's reason to one line of at most 200
832// bytes: it is shown on the build page and by build show.
833func failureReason(s string) string {
834 s = strings.Join(strings.Fields(s), " ")
835 if len(s) > 200 {
836 s = s[:200]
837 }
838 return strings.ToValidUTF8(s, "")
839}
840
841// QueueBranchBuilds reads .gitbay/ci.yml at sha and creates one pending
842// build per push job, with a pending commit status the runner resolves.
843// A broken config surfaces as a failed "ci/config" status, not silence.
844//
845// Both paths that move a branch call this: post-receive for a push, and
846// the merge path for a merge, which updates the ref directly and so never
847// reaches a hook. old is the branch's sha before this update, the diff
848// base a job's path filters run against; a new branch has no prior
849// commit and sends old as empty or all zeros. queueJobs falls back to
850// the merge base with the default branch in that case, so a filter
851// still applies to a branch's first push — the shape most changes have,
852// since branch-then-MR is the normal workflow here.
853func QueueBranchBuilds(
854 st *store.Store, root, siteURL string,
855 repo store.Repo, userID int64, branch, old, sha string, now time.Time,
856) {
857 queueJobs(st, root, siteURL, repo, userID, branch, old, sha, now, true, branch == repo.DefaultBranch, true)
858}
859
860// QueueMRBuilds queues the push jobs for a merge request head fetched
861// from another repository, which the target holds at
862// refs/merge-requests/<n>/head, so a fork's merge request has ci/<job>
863// statuses for require-checks to gate on (#98). The head is untrusted:
864// its build runs without the target's secrets. A same-repository head is
865// the branch push's job and is not queued here; a failed one is rebuilt
866// when it lands, not when it is proposed.
867func QueueMRBuilds(
868 st *store.Store, root, siteURL string,
869 repo store.Repo, userID, n int64, sha string,
870) {
871 // No old sha, and unlike QueueBranchBuilds, no merge-base fallback
872 // either: this deliberately keeps failing open and running every
873 // job. require_checks refuses a merge when an MR head has no
874 // statuses at all (mr.go), so filtering a head down to zero jobs
875 // would make it unmergeable rather than just unfiltered (#172).
876 queueJobs(st, root, siteURL, repo, userID, mrHeadRef(n), "", sha, time.Now(), false, false, false)
877}
878
879// skipReason names why a job's path filters excluded this push, mirroring
880// the order ci.Selected checks them in: an unmatched paths list rules a
881// job out before paths-ignore is even considered.
882func skipReason(j ci.Job, changed []string) string {
883 if len(j.Paths) > 0 {
884 hit := false
885 for _, f := range changed {
886 for _, p := range j.Paths {
887 if ci.Match(p, f) {
888 hit = true
889 }
890 }
891 }
892 if !hit {
893 return "no changed file matches paths"
894 }
895 }
896 return "every changed file matched paths-ignore"
897}
898
899func queueJobs(
900 st *store.Store, root, siteURL string,
901 repo store.Repo, userID int64, ref, old, sha string, now time.Time,
902 trusted, syncSchedules, deriveMergeBase bool,
903) {
904 dir := RepoDir(root, repo.OwnerName, repo.Name)
905 raw, err := gitutil.ReadBlob(dir, sha, ci.ConfigPath, 1<<16)
906 if err != nil {
907 return // no CI config at this commit
908 }
909 jobs, err := ci.Parse(raw)
910 if err != nil {
911 st.SetCommitStatus(repo.ID, sha, "ci/config", "failure", err.Error(), "", userID)
912 return
913 }
914 // A build is a fact about a commit, not a ref: a job has no branch
915 // filter, so a commit that already passed a job on another branch has
916 // nothing left to prove when a fast-forward lands it here, and one
917 // still queued or running there will say soon enough. A failed,
918 // abandoned or cancelled build does not count; that commit runs again.
919 built, err := st.BuildsForCommit(repo.ID, sha)
920 if err != nil {
921 built = nil
922 }
923 // A job's result is a property of the tree, not the commit: a rebase
924 // onto a base that touched nothing the branch did gives every commit
925 // a new sha and the same tree, and re-running the suite over it
926 // proves nothing it did not already prove (#177). A success recorded
927 // against the tree stands for the new commit.
928 tree, _ := gitutil.ResolveTree(dir, sha)
929 // The changed-file list a job's path filters run against, computed
930 // once and only if some job actually declares one. When the diff
931 // base does not exist or the diff itself fails, filtered stays
932 // false and every job runs: a filter that cannot be evaluated must
933 // not silently skip CI.
934 //
935 // A branch's first push has no old sha, but a diff base still
936 // exists: the merge base with the default branch. Without deriving
937 // one, every job runs on every new branch, and since branch-then-MR
938 // is the normal workflow, that is the push path filters matter most
939 // for. The merge base of the default branch's tip with itself is
940 // the tip, carrying no diff — that covers the default branch's own
941 // first push on a fresh repository, and must fail open rather than
942 // read as "nothing changed".
943 filtered := false
944 var changed []string
945 for _, j := range jobs {
946 if len(j.Paths) == 0 && len(j.PathsIgnore) == 0 {
947 continue
948 }
949 diffOld := old
950 // A force-push rewrote the branch, so the old tip is not an
951 // ancestor of the new one and old..new is not "what this push
952 // changed" — it is the difference between two histories. After a
953 // rebase that is whatever the new base added, typically nothing
954 // the branch itself touched, so every path filter concludes its
955 // job is unnecessary and the branch reads as green without its
956 // suite having run (#176). The merge base is the honest base:
957 // the filter is deciding about the branch's relationship to its
958 // target, which is what the merge base expresses.
959 if ci.HasDiffBase(diffOld) && deriveMergeBase {
960 if ok, err := gitutil.IsAncestor(dir, diffOld, sha); err != nil || !ok {
961 diffOld = ""
962 }
963 }
964 if !ci.HasDiffBase(diffOld) && deriveMergeBase {
965 if base, err := gitutil.MergeBase(dir, "refs/heads/"+repo.DefaultBranch, sha); err == nil && base != sha {
966 diffOld = base
967 }
968 }
969 if ci.HasDiffBase(diffOld) {
970 if files, err := gitutil.DiffFiles(dir, diffOld, sha); err == nil {
971 changed, filtered = files, true
972 }
973 }
974 break
975 }
976 var schedules []store.Schedule
977 for _, j := range jobs {
978 // Tag jobs run on matching tag pushes only.
979 if j.Tags != "" {
980 continue
981 }
982 // A build of this commit that passed, or is queued or running,
983 // stands for it — unless this queue is trusted and that build was
984 // not: a fork's head that lands on a branch is built again as the
985 // repository's own (#258).
986 if b, ok := built[j.Name]; ok && (b.Trusted || !trusted) &&
987 (b.Status == "success" || b.Status == "pending" || b.Status == "running") {
988 continue
989 }
990 if prev, ok, _ := st.SuccessBuildForTree(repo.ID, tree, j.Name, j.Image); ok && prev.SHA != sha {
991 url := fmt.Sprintf("%s/%s/builds/%d", siteURL, repo.Path(), prev.Number)
992 st.SetCommitStatus(repo.ID, sha, "ci/"+j.Name, "success",
993 fmt.Sprintf("passed in build %d as %.10s, same tree", prev.Number, prev.SHA), url, userID)
994 continue
995 }
996 // Scheduled jobs run on their cron, not on push; a default-branch
997 // push (re)registers them.
998 if j.Schedule != "" {
999 if syncSchedules {
1000 schedules = append(schedules, store.Schedule{
1001 RepoID: repo.ID, Job: j.Name, Cron: j.Schedule,
1002 NextRun: ci.NextRun(j.Schedule, now),
1003 })
1004 }
1005 continue
1006 }
1007 // A filter that excludes this push is not silence: it satisfies
1008 // require_checks with a skipped status instead of leaving the
1009 // commit with none at all, which the gate refuses outright (#172).
1010 if filtered && !ci.Selected(j, changed) {
1011 st.SetCommitStatus(repo.ID, sha, "ci/"+j.Name, "skipped", skipReason(j, changed), "", userID)
1012 continue
1013 }
1014 steps, _ := json.Marshal(j.Steps)
1015 n, err := st.CreateBuild(repo.ID, j.Name, sha, ref, string(steps), j.Image, tree, trusted)
1016 if err != nil {
1017 slog.Error("queueing build", "repo", repo.Path(), "job", j.Name, "err", err)
1018 continue
1019 }
1020 url := fmt.Sprintf("%s/%s/builds/%d", siteURL, repo.Path(), n)
1021 st.SetCommitStatus(repo.ID, sha, "ci/"+j.Name, "pending", "queued", url, userID)
1022 }
1023 if syncSchedules {
1024 if err := st.SyncSchedules(repo.ID, schedules); err != nil {
1025 slog.Error("syncing schedules", "repo", repo.Path(), "err", err)
1026 }
1027 }
1028}
1029
1030// resolveCancelledCommitStatus sets the commit status for a build that was
1031// just cancelled: if the commit already passed this job on another ref,
1032// that result stands again; otherwise the context reports the
1033// cancellation as an error, so the queued status left behind is never
1034// pending forever.
1035func resolveCancelledCommitStatus(c *Ctx, repo store.Repo, b store.Build) {
1036 if prev, ok, err := c.Store.SuccessBuildFor(repo.ID, b.SHA, b.Job); err == nil && ok {
1037 url := fmt.Sprintf("%s/%s/builds/%d", c.Cfg.Server.SiteURL, repo.Path(), prev.Number)
1038 c.Store.SetCommitStatus(repo.ID, b.SHA, "ci/"+b.Job, "success",
1039 fmt.Sprintf("passed in build %d on %s", prev.Number, prev.Ref), url, c.User.ID)
1040 return
1041 }
1042 url := fmt.Sprintf("%s/%s/builds/%d", c.Cfg.Server.SiteURL, repo.Path(), b.Number)
1043 c.Store.SetCommitStatus(repo.ID, b.SHA, "ci/"+b.Job, "error", "cancelled", url, c.User.ID)
1044}
1045
1046// cancelOrphanedBuild withdraws a build runRunnerNext claimed and then
1047// found unreachable. It leaves the same shape behind as a build cancel a
1048// person runs by hand: CancelBuild's status, a log line saying why, and
1049// the commit status resolved rather than left pending. Returns -1 to mean
1050// "handled, keep going"; anything else is the exit code to return.
1051func cancelOrphanedBuild(c *Ctx, repo store.Repo, b store.Build) int {
1052 if err := c.Store.CancelBuild(b.ID); err != nil {
1053 return c.fail(protocol.ExitFailure, "%v", err)
1054 }
1055 c.Store.AppendBuildLog(b.ID, []byte(fmt.Sprintf(
1056 "cancelled: %.10s is not reachable from any ref; the sha was likely orphaned by a force-push\n", b.SHA)))
1057 resolveCancelledCommitStatus(c, repo, b)
1058 c.Store.RecordEvent(repo.ID, c.User.ID, "build.cancelled", fmt.Sprintf(`{"number":%d,"job":%q,"sha":%q}`, b.Number, b.Job, b.SHA))
1059 return -1
1060}
1061
1062func runBuildCancel(c *Ctx, args []string) int {
1063 repo, b, code := buildRef(c, args)
1064 if code >= 0 {
1065 return code
1066 }
1067 grant, err := c.Store.AccessRole(repo.ID, c.User.ID)
1068 if err != nil {
1069 return c.fail(protocol.ExitFailure, "%v", err)
1070 }
1071 if !policy.CanWrite(c.User, repo, grant) {
1072 return c.fail(protocol.ExitDenied, "cancelling a build needs write access to %s; ask its owner", repo.Path())
1073 }
1074 if b.Status != "pending" && b.Status != "running" {
1075 return c.fail(protocol.ExitUsage, "build %d is %s; only a queued or running build can be cancelled", b.Number, b.Status)
1076 }
1077 if err := c.Store.CancelBuild(b.ID); err != nil {
1078 return c.fail(protocol.ExitFailure, "%v", err)
1079 }
1080 if b.Status == "running" {
1081 c.Store.AppendBuildLog(b.ID, []byte(fmt.Sprintf("\ncancelled by %s while running; the runner stops at its next check\n", c.User.Username)))
1082 } else {
1083 c.Store.AppendBuildLog(b.ID, []byte(fmt.Sprintf("cancelled by %s before a runner claimed it\n", c.User.Username)))
1084 }
1085 // The queued status replaced whatever the commit had for this job.
1086 resolveCancelledCommitStatus(c, repo, b)
1087 c.Store.RecordEvent(repo.ID, c.User.ID, "build.cancelled", fmt.Sprintf(`{"number":%d,"job":%q,"sha":%q}`, b.Number, b.Job, b.SHA))
1088 return c.emit(map[string]any{"number": b.Number, "job": b.Job, "status": "cancelled", "was": b.Status}, func(w io.Writer) {
1089 if b.Status == "running" {
1090 fmt.Fprintf(w, "cancelled %s build %d (%s); the runner stops at its next check\n", repo.Path(), b.Number, b.Job)
1091 return
1092 }
1093 fmt.Fprintf(w, "cancelled %s build %d (%s)\n", repo.Path(), b.Number, b.Job)
1094 })
1095}