cmd/gitbay-runner/main.go
448 lines · 15651 bytes
1// gitbay-runner executes CI builds queued by a gitbay server. It polls over
2// SSH — the same authenticated channel everything else uses — claims one
3// build at a time, clones the repo, runs each step with `sh -c`, streams the
4// combined output back, and reports success or failure.
5//
6// The account behind the runner's key must be an instance admin: a runner
7// executes arbitrary repo code, so handing out jobs is the operator's call.
8// v1 runs steps directly on the host under this process's user; run it as a
9// dedicated unprivileged user.
10package main
11
12import (
13 "encoding/json"
14 "flag"
15 "fmt"
16 "io"
17 "log"
18 "os"
19 "os/exec"
20 "path/filepath"
21 "strings"
22 "sync"
23 "syscall"
24 "time"
25
26 "gitbay.org/gitbay/internal/buildinfo"
27 "gitbay.org/gitbay/internal/toolpath"
28)
29
30type job struct {
31 ID int64 `json:"id"`
32 Repo string `json:"repo"`
33 Number int64 `json:"number"`
34 Job string `json:"job"`
35 SHA string `json:"sha"`
36 Ref string `json:"ref"`
37 Steps []string `json:"steps"`
38 Image string `json:"image"`
39 Secrets map[string]string `json:"secrets"`
40}
41
42type runner struct {
43 remote string // ssh destination, e.g. git@gitbay.org
44 sshOpts []string
45 cloneBase string // e.g. ssh://git@gitbay.org
46 workdir string
47 timeout time.Duration
48 // image is the container image for a job that names none, and
49 // isolation selects how steps run: "podman" or "none".
50 image string
51 isolation string
52 // repos limits which repositories this runner claims builds for. Empty
53 // means any, which is what a runner on the server itself wants; a runner
54 // somewhere that should not execute every repository's steps names them.
55 repos []string
56}
57
58func main() {
59 var (
60 remote = flag.String("remote", "git@gitbay.org", "ssh destination of the gitbay server")
61 sshOpts = flag.String("ssh-opts", "", "extra ssh options, space-separated (also used for git clone)")
62 cloneBase = flag.String("clone-base", "", "clone URL prefix (default ssh://<remote>)")
63 workdir = flag.String("workdir", defaultWorkdir(), "build workspace root")
64 poll = flag.Duration("poll", 5*time.Second, "idle poll interval")
65 timeout = flag.Duration("timeout", 30*time.Minute, "per-build time limit")
66 repos = flag.String("repos", "", "only claim builds for these repositories, comma-separated owner/name (default: any)")
67 once = flag.Bool("once", false, "process at most one build, then exit")
68 jobs = flag.Int("jobs", 1, "builds to run at once")
69 image = flag.String("image", "", "default container image for jobs that name none")
70 isolation = flag.String("isolation", "podman", "how steps run: podman, or none for no container")
71 version = flag.Bool("version", false, "print the commit this binary was built from, then exit")
72 )
73 flag.Parse()
74 if *version {
75 fmt.Println(buildinfo.String())
76 return
77 }
78 // The runner links internal/store, so it goes stale on changes that never
79 // touch cmd/gitbay-runner. Say which commit is running.
80 log.Printf("gitbay-runner %s", buildinfo.String())
81 r := &runner{
82 remote: *remote,
83 cloneBase: *cloneBase,
84 workdir: *workdir,
85 timeout: *timeout,
86 image: *image,
87 isolation: *isolation,
88 }
89 if err := r.checkIsolation(); err != nil {
90 // Refusing to start is the point. A runner that quietly fell back
91 // to running repository code on the host would drop isolation
92 // with nothing to surface it, which is worse than a stopped
93 // runner: the operator sees a failed unit either way, but only
94 // one of them is honest about why (#144).
95 log.Fatalf("isolation: %v", err)
96 }
97 if *sshOpts != "" {
98 r.sshOpts = strings.Fields(*sshOpts)
99 }
100 for _, name := range strings.Split(*repos, ",") {
101 if name = strings.TrimSpace(name); name != "" {
102 r.repos = append(r.repos, name)
103 }
104 }
105 if r.cloneBase == "" {
106 r.cloneBase = "ssh://" + *remote
107 }
108 // 0o700, not 0o755: a build's checkout and its secrets-bearing
109 // environment are this user's business alone, and the default sits
110 // beside other users' data on a shared host.
111 if err := os.MkdirAll(r.workdir, 0o700); err != nil {
112 log.Fatal(err)
113 }
114 if err := checkWorkdir(r.workdir); err != nil {
115 log.Fatal(err)
116 }
117 n := *jobs
118 if n < 1 {
119 log.Fatal("-jobs must be at least 1")
120 }
121 if *once {
122 // "at most one build" is one build, whatever -jobs says.
123 n = 1
124 }
125 // `runner next` claims inside one transaction, so several workers
126 // claiming at once is already safe; the runner just never used that.
127 // Each build works in its own build-<id> directory, so they do not
128 // meet on disk either.
129 var wg sync.WaitGroup
130 for i := 0; i < n; i++ {
131 wg.Add(1)
132 go func(i int) {
133 defer wg.Done()
134 // Spread the idle polls across the interval rather than
135 // having every worker wake together: n workers asking the
136 // same question in the same instant is n times the load for
137 // one answer.
138 if n > 1 {
139 time.Sleep(time.Duration(i) * *poll / time.Duration(n))
140 }
141 for {
142 ran, err := r.step()
143 if err != nil {
144 log.Printf("runner: %v", err)
145 }
146 if *once {
147 return
148 }
149 if !ran {
150 time.Sleep(*poll)
151 }
152 }
153 }(i)
154 }
155 wg.Wait()
156}
157
158// step claims and executes at most one build. ran reports whether there was
159// one, so the caller knows when to idle.
160func (r *runner) step() (bool, error) {
161 out, err := r.ssh(nil, append([]string{"runner", "next"}, append(r.repos, "--json")...)...)
162 if err != nil {
163 return false, fmt.Errorf("claiming build: %w (%s)", err, out)
164 }
165 var env struct {
166 Data job `json:"data"`
167 }
168 if err := json.Unmarshal([]byte(out), &env); err != nil {
169 return false, fmt.Errorf("parsing job: %w", err)
170 }
171 if env.Data.ID == 0 {
172 return false, nil
173 }
174 j := env.Data
175 log.Printf("build %d: %s %s @ %.10s", j.ID, j.Repo, j.Job, j.SHA)
176 status := "failure"
177 if r.run(j) {
178 status = "success"
179 }
180 if out, err := r.ssh(nil, "runner", "done", fmt.Sprint(j.ID), status); err != nil {
181 return true, fmt.Errorf("reporting build %d: %w (%s)", j.ID, err, out)
182 }
183 log.Printf("build %d: %s", j.ID, status)
184 return true, nil
185}
186
187// logSink forwards a build's output to the server and swallows any error
188// doing so. os/exec surfaces a write failure on a step's stdout through
189// cmd.Wait(), so a sink that can fail is a sink that can fail the build it
190// was only recording — a restart or a dropped session used to turn a green
191// suite red, with the explaining line written to the same dead pipe. Losing
192// log lines is the acceptable failure here; losing the build is not.
193type logSink struct {
194 mu sync.Mutex
195 w io.Writer // nil once a write has failed
196}
197
198func (s *logSink) Write(p []byte) (int, error) {
199 s.mu.Lock()
200 defer s.mu.Unlock()
201 if s.w != nil {
202 if _, err := s.w.Write(p); err != nil {
203 s.w = nil
204 }
205 }
206 return len(p), nil
207}
208
209// broken reports whether the stream was lost, so a build can say its log is
210// incomplete rather than appear to have simply stopped.
211func (s *logSink) broken() bool {
212 s.mu.Lock()
213 defer s.mu.Unlock()
214 return s.w == nil
215}
216
217// run clones, checks out, and executes the steps, streaming output to the
218// server. Returns whether every step succeeded.
219func (r *runner) run(j job) bool {
220 dir := filepath.Join(r.workdir, fmt.Sprintf("build-%d", j.ID))
221 defer os.RemoveAll(dir)
222
223 // A build's HOME. Not the workspace, which is removed after every
224 // build: the Go module cache, the sonar scanner and every other tool
225 // cache live under HOME, so a per-build one re-downloads the world
226 // each time. Not the runner's own home either, where its SSH key and
227 // credential dotfiles are. A directory beside the workspaces is
228 // neither.
229 //
230 // It is shared by every build on this runner, so a step can poison a
231 // cache another repository's build will read. That is already true of
232 // anything a step can reach as this user — see the wiki's
233 // Threat-Model on the runner — and is what container isolation (#144)
234 // is for; -repos is the control until then.
235 buildHome := filepath.Join(r.workdir, "home")
236 if err := os.MkdirAll(buildHome, 0o700); err != nil {
237 log.Printf("build %d: build home: %v", j.ID, err)
238 return false
239 }
240
241 // One long-lived `runner log` session receives the whole stream.
242 logCmd := exec.Command(toolpath.Look("ssh"), append(r.sshOpts, r.remote, "runner", "log", fmt.Sprint(j.ID))...)
243 pipe, err := logCmd.StdinPipe()
244 if err != nil {
245 log.Printf("build %d: log pipe: %v", j.ID, err)
246 return false
247 }
248 sink := &logSink{w: pipe}
249 logCmd.Stdout, logCmd.Stderr = io.Discard, io.Discard
250 if err := logCmd.Start(); err != nil {
251 log.Printf("build %d: log stream: %v", j.ID, err)
252 return false
253 }
254 // The server ends the log session with exit 3 when the build is
255 // cancelled; any other end is a lost stream, which the sink absorbs.
256 cancelled := make(chan struct{})
257 logExited := make(chan struct{})
258 go func() {
259 defer close(logExited)
260 err := logCmd.Wait()
261 if ee, ok := err.(*exec.ExitError); ok && ee.ExitCode() == 3 {
262 close(cancelled)
263 return
264 }
265 if err != nil {
266 log.Printf("build %d: log session ended: %v", j.ID, err)
267 }
268 }()
269 // runStep starts cmd and waits for it, the cancel signal, or the
270 // deadline. Every phase goes through it, so a cancel during the clone
271 // lands as fast as one during a step.
272 runStep := func(cmd *exec.Cmd, deadline time.Time) (bool, string) {
273 select {
274 case <-cancelled:
275 return false, "cancelled"
276 default:
277 }
278 ownProcessGroup(cmd)
279 if err := cmd.Start(); err != nil {
280 return false, fmt.Sprintf("start: %v", err)
281 }
282 done := make(chan error, 1)
283 go func() { done <- cmd.Wait() }()
284 // After a kill, Wait returns once every holder of the log pipe is
285 // gone; the group kill makes that prompt, and the cap makes sure a
286 // straggler cannot hold the build open.
287 reap := func() {
288 killTree(cmd)
289 select {
290 case <-done:
291 case <-time.After(10 * time.Second):
292 }
293 }
294 select {
295 case err := <-done:
296 if err != nil {
297 return false, fmt.Sprintf("step failed: %v", err)
298 }
299 return true, ""
300 case <-cancelled:
301 reap()
302 return false, "cancelled"
303 case <-time.After(time.Until(deadline)):
304 reap()
305 return false, fmt.Sprintf("build timed out after %s", r.timeout)
306 }
307 }
308 defer func() {
309 select {
310 case <-cancelled:
311 log.Printf("build %d: cancelled", j.ID)
312 default:
313 if sink.broken() {
314 log.Printf("build %d: log stream lost; stored log is incomplete", j.ID)
315 }
316 }
317 pipe.Close()
318 <-logExited
319 }()
320
321 gitSSH := strings.TrimSpace("ssh " + strings.Join(r.sshOpts, " "))
322 cloneURL := r.cloneBase + "/" + j.Repo + ".git"
323 deadline := time.Now().Add(r.timeout)
324 fmt.Fprintf(sink, "$ git clone %s (%.10s)\n", cloneURL, j.SHA)
325 // A merge request head lives under refs/merge-requests/, which a
326 // clone does not fetch; ask for the ref before checking out.
327 steps := [][]string{{"clone", "-q", cloneURL, dir}}
328 if strings.HasPrefix(j.Ref, "refs/") {
329 steps = append(steps, []string{"-C", dir, "fetch", "-q", "origin", j.Ref})
330 }
331 steps = append(steps, []string{"-C", dir, "checkout", "-q", j.SHA})
332 for _, args := range steps {
333 cmd := exec.Command(toolpath.Look("git"), args...)
334 cmd.Env = append(os.Environ(), "GIT_SSH_COMMAND="+gitSSH, "GIT_TERMINAL_PROMPT=0")
335 cmd.Stdout, cmd.Stderr = sink, sink
336 if ok, why := runStep(cmd, deadline); !ok {
337 fmt.Fprintf(sink, "git %s: %s\n", args[0], why)
338 return false
339 }
340 }
341
342 env := stepEnv(j, buildHome)
343 return r.runSteps(j, dir, env, sink, deadline, runStep)
344}
345
346// stepEnv builds the environment a build step runs with. It is
347// constructed, not inherited: os.Environ() would hand repository content
348// the runner's entire environment, including anything an operator set on
349// the service (#144).
350//
351// HOME is a build home shared by this runner's builds, not the runner's
352// own: tools read credentials out of dotfiles — .netrc, .npmrc,
353// .gitconfig — and a build has no business finding the runner's. It is
354// not the workspace either, because the workspace is deleted after every
355// build and every tool cache lives under HOME.
356//
357// PATH is the one thing carried over: without it a step cannot find the
358// tools the host was provisioned with.
359func stepEnv(j job, home string) []string {
360 path := os.Getenv("PATH")
361 if path == "" {
362 path = "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
363 }
364 env := []string{
365 "PATH=" + path,
366 "HOME=" + home,
367 "LANG=C.UTF-8",
368 "CI=true",
369 "GITBAY_REPO=" + j.Repo,
370 "GITBAY_SHA=" + j.SHA,
371 "GITBAY_REF=" + j.Ref,
372 "GITBAY_JOB=" + j.Job,
373 }
374 // The server sends secrets only for a trusted build — a merge request
375 // head from a fork arrives with none — so this loop is empty exactly
376 // when it should be.
377 for name, value := range j.Secrets {
378 env = append(env, name+"="+value)
379 }
380 return env
381}
382
383// ssh runs one control command against the server and returns stdout.// ssh runs one control command against the server and returns stdout.
384func (r *runner) ssh(stdin io.Reader, args ...string) (string, error) {
385 cmd := exec.Command(toolpath.Look("ssh"), append(append(r.sshOpts, r.remote), args...)...)
386 if stdin != nil {
387 cmd.Stdin = stdin
388 }
389 var out, errOut strings.Builder
390 cmd.Stdout, cmd.Stderr = &out, &errOut
391 if err := cmd.Run(); err != nil {
392 return out.String() + errOut.String(), err
393 }
394 return out.String(), nil
395}
396
397// defaultWorkdir picks a build workspace that another local user cannot
398// have created first.
399//
400// The default used to be <tmp>/gitbay-runner: a fixed name inside a
401// world-writable directory, created with MkdirAll, which succeeds against
402// an existing directory whoever owns it. On a shared host another user
403// could have made it — or symlinked it — before the runner started, and
404// this is the process that clones repositories and exports build secrets
405// into step environments (go:S5445, #153).
406//
407// The user's cache directory is not world-writable and is per-user by
408// construction. Falling back to tmp keeps a runner working where HOME is
409// unset, and checkWorkdir refuses the unsafe cases there.
410func defaultWorkdir() string {
411 if cache, err := os.UserCacheDir(); err == nil && cache != "" {
412 return filepath.Join(cache, "gitbay-runner")
413 }
414 return filepath.Join(os.TempDir(), "gitbay-runner")
415}
416
417// checkWorkdir makes sure the workspace is a directory this user owns
418// privately. MkdirAll is happy with one that already exists, so being
419// able to create it proves nothing about who made it.
420//
421// A directory we own that is merely too permissive is tightened rather
422// than refused: every runner before this one created its workspace 0755,
423// so refusing would take the runner down on upgrade to fix a permission
424// we are entitled to change. What cannot be repaired — a symlink, or
425// something owned by someone else — is refused, because those are what an
426// attacker leaves behind and neither is ours to correct.
427func checkWorkdir(dir string) error {
428 fi, err := os.Lstat(dir)
429 if err != nil {
430 return err
431 }
432 if fi.Mode()&os.ModeSymlink != 0 {
433 return fmt.Errorf("workdir %s is a symlink; point -workdir at a real directory", dir)
434 }
435 if !fi.IsDir() {
436 return fmt.Errorf("workdir %s is not a directory", dir)
437 }
438 if st, ok := fi.Sys().(*syscall.Stat_t); ok && int(st.Uid) != os.Getuid() {
439 return fmt.Errorf("workdir %s is owned by uid %d, not this process's %d", dir, st.Uid, os.Getuid())
440 }
441 if perm := fi.Mode().Perm(); perm&0o077 != 0 {
442 log.Printf("workdir %s was mode %04o; tightening to 0700 (builds and their secrets are this user's alone)", dir, perm)
443 if err := os.Chmod(dir, 0o700); err != nil {
444 return fmt.Errorf("tightening workdir %s: %w", dir, err)
445 }
446 }
447 return nil
448}