internal/control/build.go
439 lines · 14545 bytes
1package control
2
3import (
4 "encoding/json"
5 "errors"
6 "fmt"
7 "io"
8 "regexp"
9 "strconv"
10 "strings"
11
12 "gitbay.org/gitbay/internal/ci"
13 "gitbay.org/gitbay/internal/gitutil"
14 "gitbay.org/gitbay/internal/policy"
15 "gitbay.org/gitbay/internal/protocol"
16 "gitbay.org/gitbay/internal/store"
17)
18
19func init() {
20 register(Command{Path: []string{"build", "list"},
21 Summary: "list recent builds",
22 Usage: "build list <owner/name>", ReadOnly: true, Run: runBuildList})
23 register(Command{Path: []string{"build", "show"},
24 Summary: "show one build",
25 Usage: "build show <owner/name> <n>", ReadOnly: true, Run: runBuildShow})
26 register(Command{Path: []string{"build", "log"},
27 Summary: "print a build's log",
28 Usage: "build log <owner/name> <n>", ReadOnly: true, Run: runBuildLog})
29
30 register(Command{Path: []string{"build", "jobs"},
31 Summary: "list the jobs a trigger can name",
32 Usage: "build jobs <owner/name>", ReadOnly: true, Run: runBuildJobs})
33
34 register(Command{Path: []string{"build", "trigger"},
35 Summary: "queue a job now (scheduled or not)",
36 Usage: "build trigger <owner/name> <job>", Run: runBuildTrigger})
37 // Secrets: set over stdin, listed by name only, injected into the
38 // repo's builds as environment variables. Same discipline as mirror
39 // tokens — the value never appears in argv, logs, or output.
40 register(Command{Path: []string{"repo", "secret", "set"},
41 Summary: "set a build secret",
42 Usage: "repo secret set <owner/name> <NAME> (value on stdin)",
43 ReadsStdin: true, SSHOnly: true, Run: runSecretSet})
44 register(Command{Path: []string{"repo", "secret", "remove"},
45 Summary: "remove a build secret",
46 Usage: "repo secret remove <owner/name> <NAME>", Run: runSecretRemove})
47 register(Command{Path: []string{"repo", "secret", "list"},
48 Summary: "list build secret names",
49 Usage: "repo secret list <owner/name>", ReadOnly: true, Run: runSecretList})
50
51 // Runner commands: the claim/report loop for gitbay-runner. Admin-only —
52 // a runner executes arbitrary repo code, so handing out jobs is the
53 // instance operator's call.
54 register(Command{Path: []string{"runner", "next"},
55 Summary: "claim the oldest pending build (runner protocol)",
56 Usage: "runner next", SSHOnly: true, Run: runRunnerNext})
57 register(Command{Path: []string{"runner", "log"},
58 Summary: "append a build's log from stdin",
59 Usage: "runner log <build-id>", SSHOnly: true, ReadsStdin: true, Run: runRunnerLog})
60 register(Command{Path: []string{"runner", "done"},
61 Summary: "finish a build",
62 Usage: "runner done <build-id> success|failure", SSHOnly: true, Run: runRunnerDone})
63}
64
65type buildOut struct {
66 Number int64 `json:"number"`
67 Job string `json:"job"`
68 Status string `json:"status"`
69 SHA string `json:"sha"`
70 Ref string `json:"ref"`
71 CreatedAt string `json:"created_at"`
72 FinishedAt string `json:"finished_at,omitempty"`
73}
74
75func buildToOut(b store.Build) buildOut {
76 return buildOut{b.Number, b.Job, b.Status, b.SHA, b.Ref, b.CreatedAt, b.FinishedAt}
77}
78
79func buildRef(c *Ctx, args []string) (store.Repo, store.Build, int) {
80 if len(args) != 2 {
81 return store.Repo{}, store.Build{}, c.fail(protocol.ExitUsage, "expected <owner/name> <number>")
82 }
83 repo, code := resolveRepo(c, args[0], policy.CanRead)
84 if code >= 0 {
85 return repo, store.Build{}, code
86 }
87 n, err := strconv.ParseInt(args[1], 10, 64)
88 if err != nil {
89 return repo, store.Build{}, c.fail(protocol.ExitUsage, "bad build number %q", args[1])
90 }
91 b, err := c.Store.BuildByNumber(repo.ID, n)
92 if err != nil {
93 return repo, b, c.fail(protocol.ExitNotFound, "no build %d on %s", n, repo.Path())
94 }
95 return repo, b, -1
96}
97
98func runBuildList(c *Ctx, args []string) int {
99 if len(args) != 1 {
100 return c.fail(protocol.ExitUsage, "usage: build list <owner/name>")
101 }
102 repo, code := resolveRepo(c, args[0], policy.CanRead)
103 if code >= 0 {
104 return code
105 }
106 builds, err := c.Store.ListBuilds(repo.ID, 50)
107 if err != nil {
108 return c.fail(protocol.ExitFailure, "%v", err)
109 }
110 var ds []buildOut
111 for _, b := range builds {
112 ds = append(ds, buildToOut(b))
113 }
114 return c.emit(ds, func(w io.Writer) {
115 for _, d := range ds {
116 fmt.Fprintf(w, "%d\t%s\t%s\t%.10s\t%s\n", d.Number, d.Job, d.Status, d.SHA, d.Ref)
117 }
118 })
119}
120
121func runBuildShow(c *Ctx, args []string) int {
122 _, b, code := buildRef(c, args)
123 if code >= 0 {
124 return code
125 }
126 d := buildToOut(b)
127 return c.emit(d, func(w io.Writer) {
128 fmt.Fprintf(w, "build %d\t%s\t%s\n%.10s on %s\nqueued %s", d.Number, d.Job, d.Status, d.SHA, d.Ref, d.CreatedAt)
129 if d.FinishedAt != "" {
130 fmt.Fprintf(w, ", finished %s", d.FinishedAt)
131 }
132 fmt.Fprintln(w)
133 })
134}
135
136func runBuildLog(c *Ctx, args []string) int {
137 _, b, code := buildRef(c, args)
138 if code >= 0 {
139 return code
140 }
141 log, err := c.Store.BuildLog(b.ID)
142 if err != nil {
143 return c.fail(protocol.ExitFailure, "%v", err)
144 }
145 c.Stdout.Write(log)
146 return protocol.ExitOK
147}
148
149type jobOut struct {
150 Name string `json:"name"`
151 Schedule string `json:"schedule,omitempty"`
152 Tags string `json:"tags,omitempty"`
153}
154
155// repoJobs reads the CI config on the default branch — the same file the
156// scheduler reads — and returns its jobs with the sha they came from.
157func repoJobs(c *Ctx, repo store.Repo) ([]ci.Job, string, int) {
158 dir := RepoDir(c.Cfg.Server.Root, repo.OwnerName, repo.Name)
159 sha, err := gitutil.ResolveRef(dir, "refs/heads/"+repo.DefaultBranch)
160 if err != nil {
161 return nil, "", c.fail(protocol.ExitFailure, "resolving %s: %v", repo.DefaultBranch, err)
162 }
163 raw, err := gitutil.ReadBlob(dir, sha, ci.ConfigPath, 1<<16)
164 if err != nil {
165 return nil, "", c.fail(protocol.ExitNotFound, "%s has no %s on %s", repo.Path(), ci.ConfigPath, repo.DefaultBranch)
166 }
167 jobs, err := ci.Parse(raw)
168 if err != nil {
169 return nil, "", c.fail(protocol.ExitUsage, "%v", err)
170 }
171 return jobs, sha, -1
172}
173
174// runBuildJobs answers "what can I trigger?". Without it only a surface
175// that can read the repository's git could offer the choice.
176func runBuildJobs(c *Ctx, args []string) int {
177 if len(args) != 1 {
178 return c.fail(protocol.ExitUsage, "usage: build jobs <owner/name>")
179 }
180 repo, code := resolveRepo(c, args[0], policy.CanRead)
181 if code >= 0 {
182 return code
183 }
184 jobs, _, code := repoJobs(c, repo)
185 if code >= 0 {
186 return code
187 }
188 out := make([]jobOut, 0, len(jobs))
189 for _, j := range jobs {
190 out = append(out, jobOut{Name: j.Name, Schedule: j.Schedule, Tags: j.Tags})
191 }
192 return c.emit(out, func(w io.Writer) {
193 for _, j := range out {
194 switch {
195 case j.Schedule != "":
196 fmt.Fprintf(w, "%s\tschedule %s\n", j.Name, j.Schedule)
197 case j.Tags != "":
198 fmt.Fprintf(w, "%s\ttags %s\n", j.Name, j.Tags)
199 default:
200 fmt.Fprintf(w, "%s\ton push\n", j.Name)
201 }
202 }
203 })
204}
205
206func runBuildTrigger(c *Ctx, args []string) int {
207 if len(args) != 2 {
208 return c.fail(protocol.ExitUsage, "usage: build trigger <owner/name> <job>")
209 }
210 repo, code := resolveRepo(c, args[0], policy.CanWrite)
211 if code >= 0 {
212 return code
213 }
214 jobs, sha, code := repoJobs(c, repo)
215 if code >= 0 {
216 return code
217 }
218 for _, j := range jobs {
219 if j.Name != args[1] {
220 continue
221 }
222 steps, _ := json.Marshal(j.Steps)
223 n, err := c.Store.CreateBuild(repo.ID, j.Name, sha, repo.DefaultBranch, string(steps))
224 if err != nil {
225 return c.fail(protocol.ExitFailure, "%v", err)
226 }
227 url := fmt.Sprintf("%s/%s/builds/%d", c.Cfg.Server.SiteURL, repo.Path(), n)
228 c.Store.SetCommitStatus(repo.ID, sha, "ci/"+j.Name, "pending", "triggered", url, c.User.ID)
229 return c.emit(map[string]any{"build": n, "job": j.Name, "sha": sha}, func(w io.Writer) {
230 fmt.Fprintf(w, "queued build %d (%s @ %.10s)\n", n, j.Name, sha)
231 })
232 }
233 return c.fail(protocol.ExitNotFound, "no job %q in %s", args[1], ci.ConfigPath)
234}
235
236// secretName is env-var shaped: the value lands in the build environment.
237var secretName = regexp.MustCompile(`^[A-Z_][A-Z0-9_]{0,63}$`)
238
239func runSecretSet(c *Ctx, args []string) int {
240 if len(args) != 2 {
241 return c.fail(protocol.ExitUsage, "usage: repo secret set <owner/name> <NAME> (value on stdin)")
242 }
243 if !secretName.MatchString(args[1]) {
244 return c.fail(protocol.ExitUsage, "secret names are env-var shaped: uppercase letters, digits, _")
245 }
246 repo, code := resolveRepo(c, args[0], policy.CanAdmin)
247 if code >= 0 {
248 return code
249 }
250 raw, err := io.ReadAll(io.LimitReader(c.Stdin, 64<<10))
251 if err != nil {
252 return c.fail(protocol.ExitFailure, "reading secret: %v", err)
253 }
254 value := strings.TrimRight(string(raw), "\n")
255 if value == "" {
256 return c.fail(protocol.ExitUsage, "no value on stdin (pipe it: printf %%s TOKEN | ...)")
257 }
258 if err := c.Store.SetBuildSecret(repo.ID, args[1], value); err != nil {
259 return c.fail(protocol.ExitFailure, "%v", err)
260 }
261 return c.emit(map[string]string{"secret": args[1]}, func(w io.Writer) {
262 fmt.Fprintf(w, "secret %s set on %s\n", args[1], repo.Path())
263 })
264}
265
266func runSecretRemove(c *Ctx, args []string) int {
267 if len(args) != 2 {
268 return c.fail(protocol.ExitUsage, "usage: repo secret remove <owner/name> <NAME>")
269 }
270 repo, code := resolveRepo(c, args[0], policy.CanAdmin)
271 if code >= 0 {
272 return code
273 }
274 if err := c.Store.RemoveBuildSecret(repo.ID, args[1]); err != nil {
275 if errors.Is(err, store.ErrNotFound) {
276 return c.fail(protocol.ExitNotFound, "no secret %s on %s", args[1], repo.Path())
277 }
278 return c.fail(protocol.ExitFailure, "%v", err)
279 }
280 return c.emit(map[string]string{"removed": args[1]}, func(w io.Writer) {
281 fmt.Fprintf(w, "removed %s\n", args[1])
282 })
283}
284
285func runSecretList(c *Ctx, args []string) int {
286 if len(args) != 1 {
287 return c.fail(protocol.ExitUsage, "usage: repo secret list <owner/name>")
288 }
289 repo, code := resolveRepo(c, args[0], policy.CanAdmin)
290 if code >= 0 {
291 return code
292 }
293 names, err := c.Store.ListBuildSecretNames(repo.ID)
294 if err != nil {
295 return c.fail(protocol.ExitFailure, "%v", err)
296 }
297 return c.emit(names, func(w io.Writer) {
298 for _, n := range names {
299 fmt.Fprintln(w, n)
300 }
301 })
302}
303
304func requireRunner(c *Ctx) int {
305 if !c.User.IsAdmin {
306 return c.fail(protocol.ExitDenied, "runner commands are for instance-admin runner accounts")
307 }
308 return -1
309}
310
311func runRunnerNext(c *Ctx, args []string) int {
312 if code := requireRunner(c); code >= 0 {
313 return code
314 }
315 // Resolve anything a previous runner claimed and never reported, so a
316 // killed runner does not leave a build running and a commit pending forever.
317 if stale, err := c.Store.ReapStaleBuilds(); err != nil {
318 return c.fail(protocol.ExitFailure, "%v", err)
319 } else {
320 for _, sb := range stale {
321 repo, err := c.Store.RepoByID(sb.RepoID)
322 if err != nil {
323 continue
324 }
325 url := fmt.Sprintf("%s/%s/builds/%d", c.Cfg.Server.SiteURL, repo.Path(), sb.Number)
326 c.Store.SetCommitStatus(repo.ID, sb.SHA, "ci/"+sb.Job, "failure",
327 "build abandoned", url, c.User.ID)
328 }
329 }
330 b, ok, err := c.Store.ClaimBuild()
331 if err != nil {
332 return c.fail(protocol.ExitFailure, "%v", err)
333 }
334 if !ok {
335 return c.emit(map[string]any{}, func(w io.Writer) { fmt.Fprintln(w, "no pending builds") })
336 }
337 repo, err := c.Store.RepoByID(b.RepoID)
338 if err != nil {
339 return c.fail(protocol.ExitFailure, "%v", err)
340 }
341 var steps []string
342 json.Unmarshal([]byte(b.Steps), &steps)
343 // Secrets ride the claim: this channel is admin-only and the values
344 // land in the build's environment, nowhere else.
345 secrets, err := c.Store.BuildSecrets(b.RepoID)
346 if err != nil {
347 return c.fail(protocol.ExitFailure, "%v", err)
348 }
349 d := struct {
350 ID int64 `json:"id"`
351 Repo string `json:"repo"`
352 Number int64 `json:"number"`
353 Job string `json:"job"`
354 SHA string `json:"sha"`
355 Ref string `json:"ref"`
356 Steps []string `json:"steps"`
357 Secrets map[string]string `json:"secrets,omitempty"`
358 }{b.ID, repo.Path(), b.Number, b.Job, b.SHA, b.Ref, steps, secrets}
359 return c.emit(d, func(w io.Writer) {
360 fmt.Fprintf(w, "build %d: %s %s @ %.10s\n", d.ID, d.Repo, d.Job, d.SHA)
361 })
362}
363
364func runRunnerLog(c *Ctx, args []string) int {
365 if code := requireRunner(c); code >= 0 {
366 return code
367 }
368 if len(args) != 1 {
369 return c.fail(protocol.ExitUsage, "usage: runner log <build-id> (chunk on stdin)")
370 }
371 id, err := strconv.ParseInt(args[0], 10, 64)
372 if err != nil {
373 return c.fail(protocol.ExitUsage, "bad build id %q", args[0])
374 }
375 // Stream stdin into the log in chunks so long builds appear live.
376 buf := make([]byte, 64<<10)
377 for {
378 n, rerr := c.Stdin.Read(buf)
379 if n > 0 {
380 if err := c.Store.AppendBuildLog(id, buf[:n]); err != nil {
381 return c.fail(protocol.ExitFailure, "%v", err)
382 }
383 }
384 if rerr != nil {
385 break
386 }
387 }
388 return c.emit(map[string]string{"log": "ok"}, func(w io.Writer) {})
389}
390
391func runRunnerDone(c *Ctx, args []string) int {
392 if code := requireRunner(c); code >= 0 {
393 return code
394 }
395 if len(args) != 2 || (args[1] != "success" && args[1] != "failure") {
396 return c.fail(protocol.ExitUsage, "usage: runner done <build-id> success|failure")
397 }
398 id, err := strconv.ParseInt(args[0], 10, 64)
399 if err != nil {
400 return c.fail(protocol.ExitUsage, "bad build id %q", args[0])
401 }
402 b, err := c.Store.BuildByID(id)
403 if err != nil {
404 return c.fail(protocol.ExitNotFound, "no build %d", id)
405 }
406 if err := c.Store.FinishBuild(id, args[1]); err != nil {
407 return c.fail(protocol.ExitFailure, "finishing build %d: %v", id, err)
408 }
409 repo, err := c.Store.RepoByID(b.RepoID)
410 if err != nil {
411 return c.fail(protocol.ExitFailure, "%v", err)
412 }
413 url := fmt.Sprintf("%s/%s/builds/%d", c.Cfg.Server.SiteURL, repo.Path(), b.Number)
414 desc := "build " + args[1]
415 if err := c.Store.SetCommitStatus(repo.ID, b.SHA, "ci/"+b.Job, args[1], desc, url, c.User.ID); err != nil {
416 return c.fail(protocol.ExitFailure, "%v", err)
417 }
418 c.Store.RecordEvent(repo.ID, c.User.ID, "build."+args[1],
419 fmt.Sprintf(`{"number":%d,"job":%q}`, b.Number, b.Job))
420 // A red build mails the repo's notify targets with the log tail — a
421 // failed scheduled job must not wait to be noticed.
422 if args[1] == "failure" {
423 if targets, err := c.Store.RepoNotifyTargets(repo); err == nil {
424 tail := ""
425 if log, err := c.Store.BuildLog(id); err == nil && len(log) > 0 {
426 if len(log) > 2000 {
427 log = log[len(log)-2000:]
428 }
429 tail = string(log)
430 }
431 notifyUsers(c, targets,
432 fmt.Sprintf("[%s] build %d failed: %s on %s", repo.Path(), b.Number, b.Job, b.Ref),
433 fmt.Sprintf("job %s failed at %.10s.\n\n…%s\n\n%s\n", b.Job, b.SHA, tail, url))
434 }
435 }
436 return c.emit(map[string]any{"build": b.Number, "status": args[1]}, func(w io.Writer) {
437 fmt.Fprintf(w, "build %d %s\n", b.Number, args[1])
438 })
439}