cmd/gitbay-runner/isolate.go
251 lines · 10150 bytes
1package main
2
3import (
4 "fmt"
5 "io"
6 "log"
7 "os"
8 "os/exec"
9 "path/filepath"
10 "strings"
11 "time"
12
13 "gitbay.org/gitbay/internal/toolpath"
14)
15
16// Isolation modes. podman runs a job's steps in a container; none runs
17// them on the host as the runner's user, which is what the runner did
18// before #144 and what a private instance may still choose.
19const (
20 isolationPodman = "podman"
21 isolationNone = "none"
22)
23
24// Images are provisioned, never pulled at build time.
25//
26// The service runs with RestrictSUIDSGID=yes, so podman cannot unpack an
27// image layer containing a setuid or setgid file — which is almost every
28// distribution image (chage, passwd, su). A pull from inside the service
29// fails deep in the unpack with "operation not permitted" on some file
30// nobody has heard of.
31//
32// Keeping that flag and provisioning images deliberately is the better
33// half of the trade, and not only because it is one less hardening
34// concession: on an instance where anyone can push a ci.yml, `image:`
35// would otherwise be "fetch and run this arbitrary image from the
36// internet". An operator pulls or builds what they will allow, and a
37// build chooses among those. --pull=never makes that explicit rather
38// than leaving it to whether a pull happens to fail (#144).
39
40// checkIsolation fails the runner at start-up rather than at the first
41// build, and refuses anything it does not recognise. There is no silent
42// fallback from podman to the host: dropping isolation without saying so
43// is the failure mode this whole change exists to prevent (#144).
44func (r *runner) checkIsolation() error {
45 switch r.isolation {
46 case isolationNone:
47 log.Printf("WARNING: -isolation none: build steps run on this host as %s, "+
48 "with no container. Only do this where every repository is trusted.", currentUser())
49 return nil
50 case isolationPodman:
51 // Configuration before environment: a missing -image is the
52 // operator's to fix whatever the host looks like, and saying so
53 // first means the message does not depend on which machine this
54 // is.
55 //
56 // No built-in default image: one that is not provisioned here
57 // would fail every build with --pull=never, and guessing which
58 // image an operator has is worse than asking.
59 if r.image == "" {
60 return fmt.Errorf("-isolation podman needs -image <ref>, the image a job runs in " +
61 "when it names none. It must already be present on this host: " +
62 "pull or build it as the runner's user, since the service cannot unpack images")
63 }
64 bin := toolpath.Look("podman")
65 out, err := exec.Command(bin, "info", "--format", "{{.Host.Security.Rootless}}").CombinedOutput()
66 if err != nil {
67 return fmt.Errorf("podman is required by -isolation podman but does not work here: %v\n%s\n"+
68 "prepare the host with deploy/runner-podman-setup.sh, or pass -isolation none "+
69 "if every repository on this instance is trusted", err, strings.TrimSpace(string(out)))
70 }
71 log.Printf("isolation: podman (rootless=%s), default image %s, images must be provisioned locally",
72 strings.TrimSpace(string(out)), r.image)
73 return nil
74 default:
75 return fmt.Errorf("unknown -isolation %q: podman or none", r.isolation)
76 }
77}
78
79// runSteps executes a job's steps and reports whether all succeeded. The
80// clone has already happened, outside any container and with the runner's
81// key: the container never sees GIT_SSH_COMMAND, the key, or the runner's
82// environment — it gets the workspace and nothing else.
83type stepRunner func(cmd *exec.Cmd, deadline time.Time) (bool, string)
84
85func (r *runner) runSteps(j job, dir string, env []string, sink io.Writer, deadline time.Time, runStep stepRunner) bool {
86 if r.isolation == isolationNone {
87 for _, step := range j.Steps {
88 fmt.Fprintf(sink, "$ %s\n", step)
89 cmd := exec.Command(toolpath.Look("sh"), "-c", step)
90 cmd.Dir, cmd.Env = dir, env
91 cmd.Stdout, cmd.Stderr = sink, sink
92 if ok, why := runStep(cmd, deadline); !ok {
93 fmt.Fprintf(sink, "%s\n", why)
94 return false
95 }
96 }
97 return true
98 }
99 return r.runStepsPodman(j, dir, env, sink, deadline, runStep)
100}
101
102// runStepsPodman starts one container for the whole job and runs each
103// step in it with `podman exec`. One container per job, not per step,
104// because steps share state — a build step writes what a test step reads
105// — and per-step containers would break that.
106func (r *runner) runStepsPodman(j job, dir string, env []string, sink io.Writer, deadline time.Time, runStep stepRunner) bool {
107 podman := toolpath.Look("podman")
108 image := j.Image
109 if image == "" {
110 image = r.image
111 }
112
113 // Secrets must not reach argv: /proc is world-readable, and this
114 // codebase keeps them on stdin or in files everywhere else. An env
115 // file outside the workspace holds them instead — outside because the
116 // workspace is bind mounted, and a file of secrets sitting in the
117 // checkout is one `cat` from a build's own log.
118 envFile := filepath.Join(r.workdir, fmt.Sprintf("env-%d", j.ID))
119 if err := writeEnvFile(envFile, env); err != nil {
120 fmt.Fprintf(sink, "preparing the build environment: %v\n", err)
121 return false
122 }
123 defer os.Remove(envFile)
124
125 // The build's cgroup carries its limits; podman's own cgroup handling
126 // is off because it never worked here (#188). Every podman process
127 // for this build starts inside the cgroup, exec included: one started
128 // from the runner's cgroup would run the step outside the limit.
129 var cgroupFD *os.File
130 if r.cgroups != nil {
131 dir, f, err := r.cgroups.create(j.ID, r.memory, r.cpus)
132 if err != nil {
133 fmt.Fprintf(sink, "preparing the build cgroup: %v\n", err)
134 return false
135 }
136 cgroupFD = f
137 defer f.Close()
138 defer r.cgroups.remove(dir)
139 }
140
141 name := fmt.Sprintf("gitbay-build-%d", j.ID)
142 // --rm so a container cannot outlive its build; the explicit rm below
143 // covers the case where the daemon-less run itself fails.
144 args := append(r.podmanGlobal(), "run", "--detach", "--rm", "--pull=never", "--cgroups=disabled")
145 args = append(args,
146 "--name", name,
147 "--env-file", envFile,
148 "--volume", dir+":/workspace:rw",
149 // The build home holds the tool caches (Go modules, the sonar
150 // scanner) that must outlive a build; HOME in env points at it.
151 // Mounted at the same path so HOME resolves identically with and
152 // without a container. Only this directory — never the workdir
153 // above it, which holds other builds' workspaces.
154 "--volume", envHome(env)+":"+envHome(env)+":rw",
155 "--workdir", "/workspace",
156 "--entrypoint", "sh",
157 image, "-c", "sleep infinity")
158 start := exec.Command(podman, args...)
159 start.Env = []string{"PATH=" + os.Getenv("PATH"), "HOME=" + r.podmanHome()}
160 intoCgroup(start, cgroupFD)
161 if out, err := start.CombinedOutput(); err != nil {
162 // A missing image lands here, and it is the common case worth
163 // explaining: this runner never pulls, so an image it does not
164 // have is an operator's job to provision, not a transient error
165 // to retry.
166 msg := strings.TrimSpace(string(out))
167 fmt.Fprintf(sink, "starting the build container from %s failed:\n%s\n", image, msg)
168 if strings.Contains(msg, "no such image") || strings.Contains(msg, "image not known") ||
169 strings.Contains(msg, "unable to find") {
170 fmt.Fprintf(sink, "\nThis runner does not pull images. Ask an operator to provision %s "+
171 "on the runner host (podman pull, or podman build) before a job names it.\n", image)
172 }
173 return false
174 }
175 defer exec.Command(podman, append(r.podmanGlobal(), "rm", "--force", name)...).Run()
176
177 for _, step := range j.Steps {
178 fmt.Fprintf(sink, "$ %s\n", step)
179 cmd := exec.Command(podman, append(r.podmanGlobal(), "exec", "--workdir", "/workspace", name, "sh", "-c", step)...)
180 cmd.Env = []string{"PATH=" + os.Getenv("PATH"), "HOME=" + r.podmanHome()}
181 intoCgroup(cmd, cgroupFD)
182 cmd.Stdout, cmd.Stderr = sink, sink
183 if ok, why := runStep(cmd, deadline); !ok {
184 fmt.Fprintf(sink, "%s\n", why)
185 return false
186 }
187 }
188 return true
189}
190
191// podmanGlobal are the flags every podman invocation needs, before the
192// subcommand.
193//
194// The cgroup manager is cgroupfs, not systemd: the runner is a *system*
195// service, so there is no user session and no user@<uid>.service slice
196// for podman to create a scope under. With the systemd manager crun
197// fails with "create directory .../libpod-<id>.scope/container: No such
198// file or directory". The service's own cgroup is delegated
199// (Delegate=yes in the drop-in), which is what cgroupfs needs (#144).
200func (r *runner) podmanGlobal() []string {
201 // Storage paths are left to podman. They are recorded in its
202 // database at first use, so passing --root or --runroot later fails
203 // with "database configuration mismatch" — as does introducing an
204 // XDG_RUNTIME_DIR the database was not initialised with. Changing
205 // either means `podman system reset` and rebuilding the images.
206 return []string{"--cgroup-manager=cgroupfs"}
207}
208
209// envHome returns the HOME the step environment carries.
210func envHome(env []string) string {
211 for _, e := range env {
212 if strings.HasPrefix(e, "HOME=") {
213 return strings.TrimPrefix(e, "HOME=")
214 }
215 }
216 return ""
217}
218
219// podmanHome is where podman keeps its own storage: the runner's home,
220// not a build's. The container store is the runner's business, and a
221// build never sees this path.
222func (r *runner) podmanHome() string {
223 if h, err := os.UserHomeDir(); err == nil && h != "" {
224 return h
225 }
226 return "/var/lib/gitbay-runner"
227}
228
229// writeEnvFile writes KEY=VALUE lines for podman --env-file, readable
230// only by this user. Values containing a newline are refused rather than
231// silently truncated: the format has no escape for one, and a secret that
232// half-arrives is worse than a failed build.
233func writeEnvFile(path string, env []string) error {
234 var b strings.Builder
235 for _, e := range env {
236 if strings.ContainsAny(e, "\n\r") {
237 name, _, _ := strings.Cut(e, "=")
238 return fmt.Errorf("%s contains a newline, which an env file cannot carry", name)
239 }
240 b.WriteString(e)
241 b.WriteByte('\n')
242 }
243 return os.WriteFile(path, []byte(b.String()), 0o600)
244}
245
246func currentUser() string {
247 if u := os.Getenv("USER"); u != "" {
248 return u
249 }
250 return fmt.Sprintf("uid %d", os.Getuid())
251}