internal/control/build.go
514 lines · 17191 bytes
1package control
2
3import (
4 "encoding/json"
5 "errors"
6 "fmt"
7 "io"
8 "log/slog"
9 "regexp"
10 "strconv"
11 "strings"
12 "time"
13
14 "gitbay.org/gitbay/internal/ci"
15 "gitbay.org/gitbay/internal/gitutil"
16 "gitbay.org/gitbay/internal/policy"
17 "gitbay.org/gitbay/internal/protocol"
18 "gitbay.org/gitbay/internal/store"
19)
20
21func init() {
22 register(Command{Path: []string{"build", "list"},
23 Summary: "list recent builds",
24 Usage: "build list <owner/name>", ReadOnly: true, Run: runBuildList})
25 register(Command{Path: []string{"build", "show"},
26 Summary: "show one build",
27 Usage: "build show <owner/name> <n>", ReadOnly: true, Run: runBuildShow})
28 register(Command{Path: []string{"build", "log"},
29 Summary: "print a build's log",
30 Usage: "build log <owner/name> <n>", ReadOnly: true, Run: runBuildLog})
31
32 register(Command{Path: []string{"build", "jobs"},
33 Summary: "list the jobs a trigger can name",
34 Usage: "build jobs <owner/name>", ReadOnly: true, Run: runBuildJobs})
35
36 register(Command{Path: []string{"build", "trigger"},
37 Summary: "queue a job now (scheduled or not)",
38 Usage: "build trigger <owner/name> <job>", Run: runBuildTrigger})
39 // Secrets: set over stdin, listed by name only, injected into the
40 // repo's builds as environment variables. Same discipline as mirror
41 // tokens — the value never appears in argv, logs, or output.
42 register(Command{Path: []string{"repo", "secret", "set"},
43 Summary: "set a build secret",
44 Usage: "repo secret set <owner/name> <NAME> (value on stdin)",
45 ReadsStdin: true, SSHOnly: true, Run: runSecretSet})
46 register(Command{Path: []string{"repo", "secret", "remove"},
47 Summary: "remove a build secret",
48 Usage: "repo secret remove <owner/name> <NAME>", Run: runSecretRemove})
49 register(Command{Path: []string{"repo", "secret", "list"},
50 Summary: "list build secret names",
51 Usage: "repo secret list <owner/name>", ReadOnly: true, Run: runSecretList})
52
53 // Runner commands: the claim/report loop for gitbay-runner. Admin-only —
54 // a runner executes arbitrary repo code, so handing out jobs is the
55 // instance operator's call.
56 register(Command{Path: []string{"runner", "next"},
57 Summary: "claim the oldest pending build (runner protocol)",
58 Usage: "runner next [<owner/name>...]", SSHOnly: true, Run: runRunnerNext})
59 register(Command{Path: []string{"runner", "log"},
60 Summary: "append a build's log from stdin",
61 Usage: "runner log <build-id>", SSHOnly: true, ReadsStdin: true, Run: runRunnerLog})
62 register(Command{Path: []string{"runner", "done"},
63 Summary: "finish a build",
64 Usage: "runner done <build-id> success|failure", SSHOnly: true, Run: runRunnerDone})
65}
66
67type buildOut struct {
68 Number int64 `json:"number"`
69 Job string `json:"job"`
70 Status string `json:"status"`
71 SHA string `json:"sha"`
72 Ref string `json:"ref"`
73 CreatedAt string `json:"created_at"`
74 FinishedAt string `json:"finished_at,omitempty"`
75}
76
77func buildToOut(b store.Build) buildOut {
78 return buildOut{b.Number, b.Job, b.Status, b.SHA, b.Ref, b.CreatedAt, b.FinishedAt}
79}
80
81func buildRef(c *Ctx, args []string) (store.Repo, store.Build, int) {
82 if len(args) != 2 {
83 return store.Repo{}, store.Build{}, c.fail(protocol.ExitUsage, "expected <owner/name> <number>")
84 }
85 repo, code := resolveRepo(c, args[0], policy.CanRead)
86 if code >= 0 {
87 return repo, store.Build{}, code
88 }
89 n, err := strconv.ParseInt(args[1], 10, 64)
90 if err != nil {
91 return repo, store.Build{}, c.fail(protocol.ExitUsage, "bad build number %q", args[1])
92 }
93 b, err := c.Store.BuildByNumber(repo.ID, n)
94 if err != nil {
95 return repo, b, c.fail(protocol.ExitNotFound, "no build %d on %s", n, repo.Path())
96 }
97 return repo, b, -1
98}
99
100func runBuildList(c *Ctx, args []string) int {
101 if len(args) != 1 {
102 return c.fail(protocol.ExitUsage, "usage: build list <owner/name>")
103 }
104 repo, code := resolveRepo(c, args[0], policy.CanRead)
105 if code >= 0 {
106 return code
107 }
108 builds, err := c.Store.ListBuilds(repo.ID, 50)
109 if err != nil {
110 return c.fail(protocol.ExitFailure, "%v", err)
111 }
112 var ds []buildOut
113 for _, b := range builds {
114 ds = append(ds, buildToOut(b))
115 }
116 return c.emit(ds, func(w io.Writer) {
117 for _, d := range ds {
118 fmt.Fprintf(w, "%d\t%s\t%s\t%.10s\t%s\n", d.Number, d.Job, d.Status, d.SHA, d.Ref)
119 }
120 })
121}
122
123func runBuildShow(c *Ctx, args []string) int {
124 _, b, code := buildRef(c, args)
125 if code >= 0 {
126 return code
127 }
128 d := buildToOut(b)
129 return c.emit(d, func(w io.Writer) {
130 fmt.Fprintf(w, "build %d\t%s\t%s\n%.10s on %s\nqueued %s", d.Number, d.Job, d.Status, d.SHA, d.Ref, d.CreatedAt)
131 if d.FinishedAt != "" {
132 fmt.Fprintf(w, ", finished %s", d.FinishedAt)
133 }
134 fmt.Fprintln(w)
135 })
136}
137
138func runBuildLog(c *Ctx, args []string) int {
139 _, b, code := buildRef(c, args)
140 if code >= 0 {
141 return code
142 }
143 log, err := c.Store.BuildLog(b.ID)
144 if err != nil {
145 return c.fail(protocol.ExitFailure, "%v", err)
146 }
147 c.Stdout.Write(log)
148 return protocol.ExitOK
149}
150
151type jobOut struct {
152 Name string `json:"name"`
153 Schedule string `json:"schedule,omitempty"`
154 Tags string `json:"tags,omitempty"`
155}
156
157// repoJobs reads the CI config on the default branch — the same file the
158// scheduler reads — and returns its jobs with the sha they came from.
159func repoJobs(c *Ctx, repo store.Repo) ([]ci.Job, string, int) {
160 dir := RepoDir(c.Cfg.Server.Root, repo.OwnerName, repo.Name)
161 sha, err := gitutil.ResolveRef(dir, "refs/heads/"+repo.DefaultBranch)
162 if err != nil {
163 return nil, "", c.fail(protocol.ExitFailure, "resolving %s: %v", repo.DefaultBranch, err)
164 }
165 raw, err := gitutil.ReadBlob(dir, sha, ci.ConfigPath, 1<<16)
166 if err != nil {
167 return nil, "", c.fail(protocol.ExitNotFound, "%s has no %s on %s", repo.Path(), ci.ConfigPath, repo.DefaultBranch)
168 }
169 jobs, err := ci.Parse(raw)
170 if err != nil {
171 return nil, "", c.fail(protocol.ExitUsage, "%v", err)
172 }
173 return jobs, sha, -1
174}
175
176// runBuildJobs answers "what can I trigger?". Without it only a surface
177// that can read the repository's git could offer the choice.
178func runBuildJobs(c *Ctx, args []string) int {
179 if len(args) != 1 {
180 return c.fail(protocol.ExitUsage, "usage: build jobs <owner/name>")
181 }
182 repo, code := resolveRepo(c, args[0], policy.CanRead)
183 if code >= 0 {
184 return code
185 }
186 jobs, _, code := repoJobs(c, repo)
187 if code >= 0 {
188 return code
189 }
190 out := make([]jobOut, 0, len(jobs))
191 for _, j := range jobs {
192 out = append(out, jobOut{Name: j.Name, Schedule: j.Schedule, Tags: j.Tags})
193 }
194 return c.emit(out, func(w io.Writer) {
195 for _, j := range out {
196 switch {
197 case j.Schedule != "":
198 fmt.Fprintf(w, "%s\tschedule %s\n", j.Name, j.Schedule)
199 case j.Tags != "":
200 fmt.Fprintf(w, "%s\ttags %s\n", j.Name, j.Tags)
201 default:
202 fmt.Fprintf(w, "%s\ton push\n", j.Name)
203 }
204 }
205 })
206}
207
208func runBuildTrigger(c *Ctx, args []string) int {
209 if len(args) != 2 {
210 return c.fail(protocol.ExitUsage, "usage: build trigger <owner/name> <job>")
211 }
212 repo, code := resolveRepo(c, args[0], policy.CanWrite)
213 if code >= 0 {
214 return code
215 }
216 jobs, sha, code := repoJobs(c, repo)
217 if code >= 0 {
218 return code
219 }
220 for _, j := range jobs {
221 if j.Name != args[1] {
222 continue
223 }
224 steps, _ := json.Marshal(j.Steps)
225 n, err := c.Store.CreateBuild(repo.ID, j.Name, sha, repo.DefaultBranch, string(steps))
226 if err != nil {
227 return c.fail(protocol.ExitFailure, "%v", err)
228 }
229 url := fmt.Sprintf("%s/%s/builds/%d", c.Cfg.Server.SiteURL, repo.Path(), n)
230 c.Store.SetCommitStatus(repo.ID, sha, "ci/"+j.Name, "pending", "triggered", url, c.User.ID)
231 return c.emit(map[string]any{"build": n, "job": j.Name, "sha": sha}, func(w io.Writer) {
232 fmt.Fprintf(w, "queued build %d (%s @ %.10s)\n", n, j.Name, sha)
233 })
234 }
235 return c.fail(protocol.ExitNotFound, "no job %q in %s", args[1], ci.ConfigPath)
236}
237
238// secretName is env-var shaped: the value lands in the build environment.
239var secretName = regexp.MustCompile(`^[A-Z_][A-Z0-9_]{0,63}$`)
240
241func runSecretSet(c *Ctx, args []string) int {
242 if len(args) != 2 {
243 return c.fail(protocol.ExitUsage, "usage: repo secret set <owner/name> <NAME> (value on stdin)")
244 }
245 if !secretName.MatchString(args[1]) {
246 return c.fail(protocol.ExitUsage, "secret names are env-var shaped: uppercase letters, digits, _")
247 }
248 repo, code := resolveRepo(c, args[0], policy.CanAdmin)
249 if code >= 0 {
250 return code
251 }
252 raw, err := io.ReadAll(io.LimitReader(c.Stdin, 64<<10))
253 if err != nil {
254 return c.fail(protocol.ExitFailure, "reading secret: %v", err)
255 }
256 value := strings.TrimRight(string(raw), "\n")
257 if value == "" {
258 return c.fail(protocol.ExitUsage, "no value on stdin (pipe it: printf %%s TOKEN | ...)")
259 }
260 if err := c.Store.SetBuildSecret(repo.ID, args[1], value); err != nil {
261 return c.fail(protocol.ExitFailure, "%v", err)
262 }
263 return c.emit(map[string]string{"secret": args[1]}, func(w io.Writer) {
264 fmt.Fprintf(w, "secret %s set on %s\n", args[1], repo.Path())
265 })
266}
267
268func runSecretRemove(c *Ctx, args []string) int {
269 if len(args) != 2 {
270 return c.fail(protocol.ExitUsage, "usage: repo secret remove <owner/name> <NAME>")
271 }
272 repo, code := resolveRepo(c, args[0], policy.CanAdmin)
273 if code >= 0 {
274 return code
275 }
276 if err := c.Store.RemoveBuildSecret(repo.ID, args[1]); err != nil {
277 if errors.Is(err, store.ErrNotFound) {
278 return c.fail(protocol.ExitNotFound, "no secret %s on %s", args[1], repo.Path())
279 }
280 return c.fail(protocol.ExitFailure, "%v", err)
281 }
282 return c.emit(map[string]string{"removed": args[1]}, func(w io.Writer) {
283 fmt.Fprintf(w, "removed %s\n", args[1])
284 })
285}
286
287func runSecretList(c *Ctx, args []string) int {
288 if len(args) != 1 {
289 return c.fail(protocol.ExitUsage, "usage: repo secret list <owner/name>")
290 }
291 repo, code := resolveRepo(c, args[0], policy.CanAdmin)
292 if code >= 0 {
293 return code
294 }
295 names, err := c.Store.ListBuildSecretNames(repo.ID)
296 if err != nil {
297 return c.fail(protocol.ExitFailure, "%v", err)
298 }
299 return c.emit(names, func(w io.Writer) {
300 for _, n := range names {
301 fmt.Fprintln(w, n)
302 }
303 })
304}
305
306func requireRunner(c *Ctx) int {
307 if !c.User.IsAdmin {
308 return c.fail(protocol.ExitDenied, "runner commands are for instance-admin runner accounts")
309 }
310 return -1
311}
312
313func runRunnerNext(c *Ctx, args []string) int {
314 if code := requireRunner(c); code >= 0 {
315 return code
316 }
317 // Resolve anything a previous runner claimed and never reported, so a
318 // killed runner does not leave a build running and a commit pending forever.
319 if stale, err := c.Store.ReapStaleBuilds(); err != nil {
320 return c.fail(protocol.ExitFailure, "%v", err)
321 } else {
322 for _, sb := range stale {
323 repo, err := c.Store.RepoByID(sb.RepoID)
324 if err != nil {
325 continue
326 }
327 url := fmt.Sprintf("%s/%s/builds/%d", c.Cfg.Server.SiteURL, repo.Path(), sb.Number)
328 c.Store.SetCommitStatus(repo.ID, sb.SHA, "ci/"+sb.Job, "failure",
329 "build abandoned", url, c.User.ID)
330 }
331 }
332 // A runner may limit itself to named repositories. The operator chooses
333 // what a given runner executes by how they start it; this is scoping the
334 // runner asks for, not an ACL the server holds over it.
335 var repoIDs []int64
336 for _, arg := range args {
337 repo, code := resolveRepo(c, arg, policy.CanRead)
338 if code >= 0 {
339 return code
340 }
341 repoIDs = append(repoIDs, repo.ID)
342 }
343 b, ok, err := c.Store.ClaimBuild(repoIDs)
344 if err != nil {
345 return c.fail(protocol.ExitFailure, "%v", err)
346 }
347 if !ok {
348 return c.emit(map[string]any{}, func(w io.Writer) { fmt.Fprintln(w, "no pending builds") })
349 }
350 repo, err := c.Store.RepoByID(b.RepoID)
351 if err != nil {
352 return c.fail(protocol.ExitFailure, "%v", err)
353 }
354 var steps []string
355 json.Unmarshal([]byte(b.Steps), &steps)
356 // Secrets ride the claim: this channel is admin-only and the values
357 // land in the build's environment, nowhere else.
358 secrets, err := c.Store.BuildSecrets(b.RepoID)
359 if err != nil {
360 return c.fail(protocol.ExitFailure, "%v", err)
361 }
362 d := struct {
363 ID int64 `json:"id"`
364 Repo string `json:"repo"`
365 Number int64 `json:"number"`
366 Job string `json:"job"`
367 SHA string `json:"sha"`
368 Ref string `json:"ref"`
369 Steps []string `json:"steps"`
370 Secrets map[string]string `json:"secrets,omitempty"`
371 }{b.ID, repo.Path(), b.Number, b.Job, b.SHA, b.Ref, steps, secrets}
372 return c.emit(d, func(w io.Writer) {
373 fmt.Fprintf(w, "build %d: %s %s @ %.10s\n", d.ID, d.Repo, d.Job, d.SHA)
374 })
375}
376
377func runRunnerLog(c *Ctx, args []string) int {
378 if code := requireRunner(c); code >= 0 {
379 return code
380 }
381 if len(args) != 1 {
382 return c.fail(protocol.ExitUsage, "usage: runner log <build-id> (chunk on stdin)")
383 }
384 id, err := strconv.ParseInt(args[0], 10, 64)
385 if err != nil {
386 return c.fail(protocol.ExitUsage, "bad build id %q", args[0])
387 }
388 // Stream stdin into the log in chunks so long builds appear live. An
389 // append that fails drops its chunk and the loop keeps draining: ending
390 // the session here breaks the runner's pipe, and a broken pipe is how a
391 // transient SQLITE_BUSY used to fail the build the log belonged to.
392 buf := make([]byte, 64<<10)
393 dropped := 0
394 for {
395 n, rerr := c.Stdin.Read(buf)
396 if n > 0 {
397 if err := c.Store.AppendBuildLog(id, buf[:n]); err != nil {
398 dropped++
399 slog.Warn("appending build log", "build", id, "err", err)
400 }
401 }
402 if rerr != nil {
403 break
404 }
405 }
406 if dropped > 0 {
407 slog.Warn("build log incomplete", "build", id, "dropped_chunks", dropped)
408 }
409 return c.emit(map[string]string{"log": "ok"}, func(w io.Writer) {})
410}
411
412func runRunnerDone(c *Ctx, args []string) int {
413 if code := requireRunner(c); code >= 0 {
414 return code
415 }
416 if len(args) != 2 || (args[1] != "success" && args[1] != "failure") {
417 return c.fail(protocol.ExitUsage, "usage: runner done <build-id> success|failure")
418 }
419 id, err := strconv.ParseInt(args[0], 10, 64)
420 if err != nil {
421 return c.fail(protocol.ExitUsage, "bad build id %q", args[0])
422 }
423 b, err := c.Store.BuildByID(id)
424 if err != nil {
425 return c.fail(protocol.ExitNotFound, "no build %d", id)
426 }
427 if err := c.Store.FinishBuild(id, args[1]); err != nil {
428 return c.fail(protocol.ExitFailure, "finishing build %d: %v", id, err)
429 }
430 repo, err := c.Store.RepoByID(b.RepoID)
431 if err != nil {
432 return c.fail(protocol.ExitFailure, "%v", err)
433 }
434 url := fmt.Sprintf("%s/%s/builds/%d", c.Cfg.Server.SiteURL, repo.Path(), b.Number)
435 desc := "build " + args[1]
436 if err := c.Store.SetCommitStatus(repo.ID, b.SHA, "ci/"+b.Job, args[1], desc, url, c.User.ID); err != nil {
437 return c.fail(protocol.ExitFailure, "%v", err)
438 }
439 c.Store.RecordEvent(repo.ID, c.User.ID, "build."+args[1],
440 fmt.Sprintf(`{"number":%d,"job":%q}`, b.Number, b.Job))
441 // A red build mails the repo's notify targets with the log tail — a
442 // failed scheduled job must not wait to be noticed.
443 if args[1] == "failure" {
444 if targets, err := c.Store.RepoNotifyTargets(repo); err == nil {
445 tail := ""
446 if log, err := c.Store.BuildLog(id); err == nil && len(log) > 0 {
447 if len(log) > 2000 {
448 log = log[len(log)-2000:]
449 }
450 tail = string(log)
451 }
452 notifyUsers(c, targets,
453 fmt.Sprintf("[%s] build %d failed: %s on %s", repo.Path(), b.Number, b.Job, b.Ref),
454 fmt.Sprintf("job %s failed at %.10s.\n\n…%s\n\n%s\n", b.Job, b.SHA, tail, url))
455 }
456 }
457 return c.emit(map[string]any{"build": b.Number, "status": args[1]}, func(w io.Writer) {
458 fmt.Fprintf(w, "build %d %s\n", b.Number, args[1])
459 })
460}
461
462// QueueBranchBuilds reads .gitbay/ci.yml at sha and creates one pending
463// build per push job, with a pending commit status the runner resolves.
464// A broken config surfaces as a failed "ci/config" status, not silence.
465//
466// Both paths that move a branch call this: post-receive for a push, and
467// the merge path for a merge, which updates the ref directly and so never
468// reaches a hook.
469func QueueBranchBuilds(
470 st *store.Store, root, siteURL string,
471 repo store.Repo, userID int64, branch, sha string, now time.Time,
472) {
473 dir := RepoDir(root, repo.OwnerName, repo.Name)
474 raw, err := gitutil.ReadBlob(dir, sha, ci.ConfigPath, 1<<16)
475 if err != nil {
476 return // no CI config at this commit
477 }
478 jobs, err := ci.Parse(raw)
479 if err != nil {
480 st.SetCommitStatus(repo.ID, sha, "ci/config", "failure", err.Error(), "", userID)
481 return
482 }
483 var schedules []store.Schedule
484 for _, j := range jobs {
485 // Tag jobs run on matching tag pushes only.
486 if j.Tags != "" {
487 continue
488 }
489 // Scheduled jobs run on their cron, not on push; a default-branch
490 // push (re)registers them.
491 if j.Schedule != "" {
492 if branch == repo.DefaultBranch {
493 schedules = append(schedules, store.Schedule{
494 RepoID: repo.ID, Job: j.Name, Cron: j.Schedule,
495 NextRun: ci.NextRun(j.Schedule, now),
496 })
497 }
498 continue
499 }
500 steps, _ := json.Marshal(j.Steps)
501 n, err := st.CreateBuild(repo.ID, j.Name, sha, branch, string(steps))
502 if err != nil {
503 slog.Error("queueing build", "repo", repo.Path(), "job", j.Name, "err", err)
504 continue
505 }
506 url := fmt.Sprintf("%s/%s/builds/%d", siteURL, repo.Path(), n)
507 st.SetCommitStatus(repo.ID, sha, "ci/"+j.Name, "pending", "queued", url, userID)
508 }
509 if branch == repo.DefaultBranch {
510 if err := st.SyncSchedules(repo.ID, schedules); err != nil {
511 slog.Error("syncing schedules", "repo", repo.Path(), "err", err)
512 }
513 }
514}