internal/control/build.go
428 lines · 14386 bytes
1package control
2
3import (
4 "encoding/json"
5 "errors"
6 "fmt"
7 "io"
8 "regexp"
9 "strconv"
10 "strings"
11
12 "gitbay.org/gitbay/internal/ci"
13 "gitbay.org/gitbay/internal/gitutil"
14 "gitbay.org/gitbay/internal/policy"
15 "gitbay.org/gitbay/internal/protocol"
16 "gitbay.org/gitbay/internal/store"
17)
18
19func init() {
20 register(Command{Path: []string{"build", "list"},
21 Summary: "list recent builds: build list <owner/name>", ReadOnly: true, Run: runBuildList})
22 register(Command{Path: []string{"build", "show"},
23 Summary: "show one build: build show <owner/name> <n>", ReadOnly: true, Run: runBuildShow})
24 register(Command{Path: []string{"build", "log"},
25 Summary: "print a build's log: build log <owner/name> <n>", ReadOnly: true, Run: runBuildLog})
26
27 register(Command{Path: []string{"build", "jobs"},
28 Summary: "list the jobs a trigger can name: build jobs <owner/name>", ReadOnly: true, Run: runBuildJobs})
29
30 register(Command{Path: []string{"build", "trigger"},
31 Summary: "queue a job now (scheduled or not): build trigger <owner/name> <job>", Run: runBuildTrigger})
32 // Secrets: set over stdin, listed by name only, injected into the
33 // repo's builds as environment variables. Same discipline as mirror
34 // tokens — the value never appears in argv, logs, or output.
35 register(Command{Path: []string{"repo", "secret", "set"},
36 Summary: "set a build secret: repo secret set <owner/name> <NAME> (value on stdin)",
37 ReadsStdin: true, SSHOnly: true, Run: runSecretSet})
38 register(Command{Path: []string{"repo", "secret", "remove"},
39 Summary: "remove a build secret: repo secret remove <owner/name> <NAME>", Run: runSecretRemove})
40 register(Command{Path: []string{"repo", "secret", "list"},
41 Summary: "list build secret names: repo secret list <owner/name>", ReadOnly: true, Run: runSecretList})
42
43 // Runner commands: the claim/report loop for gitbay-runner. Admin-only —
44 // a runner executes arbitrary repo code, so handing out jobs is the
45 // instance operator's call.
46 register(Command{Path: []string{"runner", "next"},
47 Summary: "claim the oldest pending build (runner protocol)", SSHOnly: true, Run: runRunnerNext})
48 register(Command{Path: []string{"runner", "log"},
49 Summary: "append a build's log from stdin: runner log <build-id>", SSHOnly: true, ReadsStdin: true, Run: runRunnerLog})
50 register(Command{Path: []string{"runner", "done"},
51 Summary: "finish a build: runner done <build-id> success|failure", SSHOnly: true, Run: runRunnerDone})
52}
53
54type buildOut struct {
55 Number int64 `json:"number"`
56 Job string `json:"job"`
57 Status string `json:"status"`
58 SHA string `json:"sha"`
59 Ref string `json:"ref"`
60 CreatedAt string `json:"created_at"`
61 FinishedAt string `json:"finished_at,omitempty"`
62}
63
64func buildToOut(b store.Build) buildOut {
65 return buildOut{b.Number, b.Job, b.Status, b.SHA, b.Ref, b.CreatedAt, b.FinishedAt}
66}
67
68func buildRef(c *Ctx, args []string) (store.Repo, store.Build, int) {
69 if len(args) != 2 {
70 return store.Repo{}, store.Build{}, c.fail(protocol.ExitUsage, "expected <owner/name> <number>")
71 }
72 repo, code := resolveRepo(c, args[0], policy.CanRead)
73 if code >= 0 {
74 return repo, store.Build{}, code
75 }
76 n, err := strconv.ParseInt(args[1], 10, 64)
77 if err != nil {
78 return repo, store.Build{}, c.fail(protocol.ExitUsage, "bad build number %q", args[1])
79 }
80 b, err := c.Store.BuildByNumber(repo.ID, n)
81 if err != nil {
82 return repo, b, c.fail(protocol.ExitNotFound, "no build %d on %s", n, repo.Path())
83 }
84 return repo, b, -1
85}
86
87func runBuildList(c *Ctx, args []string) int {
88 if len(args) != 1 {
89 return c.fail(protocol.ExitUsage, "usage: build list <owner/name>")
90 }
91 repo, code := resolveRepo(c, args[0], policy.CanRead)
92 if code >= 0 {
93 return code
94 }
95 builds, err := c.Store.ListBuilds(repo.ID, 50)
96 if err != nil {
97 return c.fail(protocol.ExitFailure, "%v", err)
98 }
99 var ds []buildOut
100 for _, b := range builds {
101 ds = append(ds, buildToOut(b))
102 }
103 return c.emit(ds, func(w io.Writer) {
104 for _, d := range ds {
105 fmt.Fprintf(w, "%d\t%s\t%s\t%.10s\t%s\n", d.Number, d.Job, d.Status, d.SHA, d.Ref)
106 }
107 })
108}
109
110func runBuildShow(c *Ctx, args []string) int {
111 _, b, code := buildRef(c, args)
112 if code >= 0 {
113 return code
114 }
115 d := buildToOut(b)
116 return c.emit(d, func(w io.Writer) {
117 fmt.Fprintf(w, "build %d\t%s\t%s\n%.10s on %s\nqueued %s", d.Number, d.Job, d.Status, d.SHA, d.Ref, d.CreatedAt)
118 if d.FinishedAt != "" {
119 fmt.Fprintf(w, ", finished %s", d.FinishedAt)
120 }
121 fmt.Fprintln(w)
122 })
123}
124
125func runBuildLog(c *Ctx, args []string) int {
126 _, b, code := buildRef(c, args)
127 if code >= 0 {
128 return code
129 }
130 log, err := c.Store.BuildLog(b.ID)
131 if err != nil {
132 return c.fail(protocol.ExitFailure, "%v", err)
133 }
134 c.Stdout.Write(log)
135 return protocol.ExitOK
136}
137
138type jobOut struct {
139 Name string `json:"name"`
140 Schedule string `json:"schedule,omitempty"`
141 Tags string `json:"tags,omitempty"`
142}
143
144// repoJobs reads the CI config on the default branch — the same file the
145// scheduler reads — and returns its jobs with the sha they came from.
146func repoJobs(c *Ctx, repo store.Repo) ([]ci.Job, string, int) {
147 dir := RepoDir(c.Cfg.Server.Root, repo.OwnerName, repo.Name)
148 sha, err := gitutil.ResolveRef(dir, "refs/heads/"+repo.DefaultBranch)
149 if err != nil {
150 return nil, "", c.fail(protocol.ExitFailure, "resolving %s: %v", repo.DefaultBranch, err)
151 }
152 raw, err := gitutil.ReadBlob(dir, sha, ci.ConfigPath, 1<<16)
153 if err != nil {
154 return nil, "", c.fail(protocol.ExitNotFound, "%s has no %s on %s", repo.Path(), ci.ConfigPath, repo.DefaultBranch)
155 }
156 jobs, err := ci.Parse(raw)
157 if err != nil {
158 return nil, "", c.fail(protocol.ExitUsage, "%v", err)
159 }
160 return jobs, sha, -1
161}
162
163// runBuildJobs answers "what can I trigger?". Without it only a surface
164// that can read the repository's git could offer the choice.
165func runBuildJobs(c *Ctx, args []string) int {
166 if len(args) != 1 {
167 return c.fail(protocol.ExitUsage, "usage: build jobs <owner/name>")
168 }
169 repo, code := resolveRepo(c, args[0], policy.CanRead)
170 if code >= 0 {
171 return code
172 }
173 jobs, _, code := repoJobs(c, repo)
174 if code >= 0 {
175 return code
176 }
177 out := make([]jobOut, 0, len(jobs))
178 for _, j := range jobs {
179 out = append(out, jobOut{Name: j.Name, Schedule: j.Schedule, Tags: j.Tags})
180 }
181 return c.emit(out, func(w io.Writer) {
182 for _, j := range out {
183 switch {
184 case j.Schedule != "":
185 fmt.Fprintf(w, "%s\tschedule %s\n", j.Name, j.Schedule)
186 case j.Tags != "":
187 fmt.Fprintf(w, "%s\ttags %s\n", j.Name, j.Tags)
188 default:
189 fmt.Fprintf(w, "%s\ton push\n", j.Name)
190 }
191 }
192 })
193}
194
195func runBuildTrigger(c *Ctx, args []string) int {
196 if len(args) != 2 {
197 return c.fail(protocol.ExitUsage, "usage: build trigger <owner/name> <job>")
198 }
199 repo, code := resolveRepo(c, args[0], policy.CanWrite)
200 if code >= 0 {
201 return code
202 }
203 jobs, sha, code := repoJobs(c, repo)
204 if code >= 0 {
205 return code
206 }
207 for _, j := range jobs {
208 if j.Name != args[1] {
209 continue
210 }
211 steps, _ := json.Marshal(j.Steps)
212 n, err := c.Store.CreateBuild(repo.ID, j.Name, sha, repo.DefaultBranch, string(steps))
213 if err != nil {
214 return c.fail(protocol.ExitFailure, "%v", err)
215 }
216 url := fmt.Sprintf("%s/%s/builds/%d", c.Cfg.Server.SiteURL, repo.Path(), n)
217 c.Store.SetCommitStatus(repo.ID, sha, "ci/"+j.Name, "pending", "triggered", url, c.User.ID)
218 return c.emit(map[string]any{"build": n, "job": j.Name, "sha": sha}, func(w io.Writer) {
219 fmt.Fprintf(w, "queued build %d (%s @ %.10s)\n", n, j.Name, sha)
220 })
221 }
222 return c.fail(protocol.ExitNotFound, "no job %q in %s", args[1], ci.ConfigPath)
223}
224
225// secretName is env-var shaped: the value lands in the build environment.
226var secretName = regexp.MustCompile(`^[A-Z_][A-Z0-9_]{0,63}$`)
227
228func runSecretSet(c *Ctx, args []string) int {
229 if len(args) != 2 {
230 return c.fail(protocol.ExitUsage, "usage: repo secret set <owner/name> <NAME> (value on stdin)")
231 }
232 if !secretName.MatchString(args[1]) {
233 return c.fail(protocol.ExitUsage, "secret names are env-var shaped: uppercase letters, digits, _")
234 }
235 repo, code := resolveRepo(c, args[0], policy.CanAdmin)
236 if code >= 0 {
237 return code
238 }
239 raw, err := io.ReadAll(io.LimitReader(c.Stdin, 64<<10))
240 if err != nil {
241 return c.fail(protocol.ExitFailure, "reading secret: %v", err)
242 }
243 value := strings.TrimRight(string(raw), "\n")
244 if value == "" {
245 return c.fail(protocol.ExitUsage, "no value on stdin (pipe it: printf %%s TOKEN | ...)")
246 }
247 if err := c.Store.SetBuildSecret(repo.ID, args[1], value); err != nil {
248 return c.fail(protocol.ExitFailure, "%v", err)
249 }
250 return c.emit(map[string]string{"secret": args[1]}, func(w io.Writer) {
251 fmt.Fprintf(w, "secret %s set on %s\n", args[1], repo.Path())
252 })
253}
254
255func runSecretRemove(c *Ctx, args []string) int {
256 if len(args) != 2 {
257 return c.fail(protocol.ExitUsage, "usage: repo secret remove <owner/name> <NAME>")
258 }
259 repo, code := resolveRepo(c, args[0], policy.CanAdmin)
260 if code >= 0 {
261 return code
262 }
263 if err := c.Store.RemoveBuildSecret(repo.ID, args[1]); err != nil {
264 if errors.Is(err, store.ErrNotFound) {
265 return c.fail(protocol.ExitNotFound, "no secret %s on %s", args[1], repo.Path())
266 }
267 return c.fail(protocol.ExitFailure, "%v", err)
268 }
269 return c.emit(map[string]string{"removed": args[1]}, func(w io.Writer) {
270 fmt.Fprintf(w, "removed %s\n", args[1])
271 })
272}
273
274func runSecretList(c *Ctx, args []string) int {
275 if len(args) != 1 {
276 return c.fail(protocol.ExitUsage, "usage: repo secret list <owner/name>")
277 }
278 repo, code := resolveRepo(c, args[0], policy.CanAdmin)
279 if code >= 0 {
280 return code
281 }
282 names, err := c.Store.ListBuildSecretNames(repo.ID)
283 if err != nil {
284 return c.fail(protocol.ExitFailure, "%v", err)
285 }
286 return c.emit(names, func(w io.Writer) {
287 for _, n := range names {
288 fmt.Fprintln(w, n)
289 }
290 })
291}
292
293func requireRunner(c *Ctx) int {
294 if !c.User.IsAdmin {
295 return c.fail(protocol.ExitDenied, "runner commands are for instance-admin runner accounts")
296 }
297 return -1
298}
299
300func runRunnerNext(c *Ctx, args []string) int {
301 if code := requireRunner(c); code >= 0 {
302 return code
303 }
304 // Resolve anything a previous runner claimed and never reported, so a
305 // killed runner does not leave a build running and a commit pending forever.
306 if stale, err := c.Store.ReapStaleBuilds(); err != nil {
307 return c.fail(protocol.ExitFailure, "%v", err)
308 } else {
309 for _, sb := range stale {
310 repo, err := c.Store.RepoByID(sb.RepoID)
311 if err != nil {
312 continue
313 }
314 url := fmt.Sprintf("%s/%s/builds/%d", c.Cfg.Server.SiteURL, repo.Path(), sb.Number)
315 c.Store.SetCommitStatus(repo.ID, sb.SHA, "ci/"+sb.Job, "failure",
316 "build abandoned", url, c.User.ID)
317 }
318 }
319 b, ok, err := c.Store.ClaimBuild()
320 if err != nil {
321 return c.fail(protocol.ExitFailure, "%v", err)
322 }
323 if !ok {
324 return c.emit(map[string]any{}, func(w io.Writer) { fmt.Fprintln(w, "no pending builds") })
325 }
326 repo, err := c.Store.RepoByID(b.RepoID)
327 if err != nil {
328 return c.fail(protocol.ExitFailure, "%v", err)
329 }
330 var steps []string
331 json.Unmarshal([]byte(b.Steps), &steps)
332 // Secrets ride the claim: this channel is admin-only and the values
333 // land in the build's environment, nowhere else.
334 secrets, err := c.Store.BuildSecrets(b.RepoID)
335 if err != nil {
336 return c.fail(protocol.ExitFailure, "%v", err)
337 }
338 d := struct {
339 ID int64 `json:"id"`
340 Repo string `json:"repo"`
341 Number int64 `json:"number"`
342 Job string `json:"job"`
343 SHA string `json:"sha"`
344 Ref string `json:"ref"`
345 Steps []string `json:"steps"`
346 Secrets map[string]string `json:"secrets,omitempty"`
347 }{b.ID, repo.Path(), b.Number, b.Job, b.SHA, b.Ref, steps, secrets}
348 return c.emit(d, func(w io.Writer) {
349 fmt.Fprintf(w, "build %d: %s %s @ %.10s\n", d.ID, d.Repo, d.Job, d.SHA)
350 })
351}
352
353func runRunnerLog(c *Ctx, args []string) int {
354 if code := requireRunner(c); code >= 0 {
355 return code
356 }
357 if len(args) != 1 {
358 return c.fail(protocol.ExitUsage, "usage: runner log <build-id> (chunk on stdin)")
359 }
360 id, err := strconv.ParseInt(args[0], 10, 64)
361 if err != nil {
362 return c.fail(protocol.ExitUsage, "bad build id %q", args[0])
363 }
364 // Stream stdin into the log in chunks so long builds appear live.
365 buf := make([]byte, 64<<10)
366 for {
367 n, rerr := c.Stdin.Read(buf)
368 if n > 0 {
369 if err := c.Store.AppendBuildLog(id, buf[:n]); err != nil {
370 return c.fail(protocol.ExitFailure, "%v", err)
371 }
372 }
373 if rerr != nil {
374 break
375 }
376 }
377 return c.emit(map[string]string{"log": "ok"}, func(w io.Writer) {})
378}
379
380func runRunnerDone(c *Ctx, args []string) int {
381 if code := requireRunner(c); code >= 0 {
382 return code
383 }
384 if len(args) != 2 || (args[1] != "success" && args[1] != "failure") {
385 return c.fail(protocol.ExitUsage, "usage: runner done <build-id> success|failure")
386 }
387 id, err := strconv.ParseInt(args[0], 10, 64)
388 if err != nil {
389 return c.fail(protocol.ExitUsage, "bad build id %q", args[0])
390 }
391 b, err := c.Store.BuildByID(id)
392 if err != nil {
393 return c.fail(protocol.ExitNotFound, "no build %d", id)
394 }
395 if err := c.Store.FinishBuild(id, args[1]); err != nil {
396 return c.fail(protocol.ExitFailure, "finishing build %d: %v", id, err)
397 }
398 repo, err := c.Store.RepoByID(b.RepoID)
399 if err != nil {
400 return c.fail(protocol.ExitFailure, "%v", err)
401 }
402 url := fmt.Sprintf("%s/%s/builds/%d", c.Cfg.Server.SiteURL, repo.Path(), b.Number)
403 desc := "build " + args[1]
404 if err := c.Store.SetCommitStatus(repo.ID, b.SHA, "ci/"+b.Job, args[1], desc, url, c.User.ID); err != nil {
405 return c.fail(protocol.ExitFailure, "%v", err)
406 }
407 c.Store.RecordEvent(repo.ID, c.User.ID, "build."+args[1],
408 fmt.Sprintf(`{"number":%d,"job":%q}`, b.Number, b.Job))
409 // A red build mails the repo's notify targets with the log tail — a
410 // failed scheduled job must not wait to be noticed.
411 if args[1] == "failure" {
412 if targets, err := c.Store.RepoNotifyTargets(repo); err == nil {
413 tail := ""
414 if log, err := c.Store.BuildLog(id); err == nil && len(log) > 0 {
415 if len(log) > 2000 {
416 log = log[len(log)-2000:]
417 }
418 tail = string(log)
419 }
420 notifyUsers(c, targets,
421 fmt.Sprintf("[%s] build %d failed: %s on %s", repo.Path(), b.Number, b.Job, b.Ref),
422 fmt.Sprintf("job %s failed at %.10s.\n\n…%s\n\n%s\n", b.Job, b.SHA, tail, url))
423 }
424 }
425 return c.emit(map[string]any{"build": b.Number, "status": args[1]}, func(w io.Writer) {
426 fmt.Fprintf(w, "build %d %s\n", b.Number, args[1])
427 })
428}