cmd/gitbay-runner/isolate.go
284 lines · 11280 bytes
1package main
2
3import (
4 "fmt"
5 "io"
6 "log"
7 "os"
8 "os/exec"
9 "path/filepath"
10 "strings"
11 "time"
12
13 "gitbay.org/gitbay/internal/toolpath"
14)
15
16// Isolation modes. podman runs a job's steps in a container; none runs
17// them on the host as the runner's user, which is what the runner did
18// before #144 and what a private instance may still choose.
19const (
20 isolationPodman = "podman"
21 isolationNone = "none"
22)
23
24// Images are provisioned, never pulled at build time.
25//
26// The service runs with RestrictSUIDSGID=yes, so podman cannot unpack an
27// image layer containing a setuid or setgid file — which is almost every
28// distribution image (chage, passwd, su). A pull from inside the service
29// fails deep in the unpack with "operation not permitted" on some file
30// nobody has heard of.
31//
32// Keeping that flag and provisioning images deliberately is the better
33// half of the trade, and not only because it is one less hardening
34// concession: on an instance where anyone can push a ci.yml, `image:`
35// would otherwise be "fetch and run this arbitrary image from the
36// internet". An operator pulls or builds what they will allow, and a
37// build chooses among those. --pull=never makes that explicit rather
38// than leaving it to whether a pull happens to fail (#144).
39
40// checkIsolation fails the runner at start-up rather than at the first
41// build, and refuses anything it does not recognise. There is no silent
42// fallback from podman to the host: dropping isolation without saying so
43// is the failure mode this whole change exists to prevent (#144).
44func (r *runner) checkIsolation() error {
45 switch r.isolation {
46 case isolationNone:
47 log.Printf("WARNING: -isolation none: build steps run on this host as %s, "+
48 "with no container. Only do this where every repository is trusted.", currentUser())
49 return nil
50 case isolationPodman:
51 // Configuration before environment: a missing -image is the
52 // operator's to fix whatever the host looks like, and saying so
53 // first means the message does not depend on which machine this
54 // is.
55 //
56 // No built-in default image: one that is not provisioned here
57 // would fail every build with --pull=never, and guessing which
58 // image an operator has is worse than asking.
59 if r.image == "" {
60 return fmt.Errorf("-isolation podman needs -image <ref>, the image a job runs in " +
61 "when it names none. It must already be present on this host: " +
62 "pull or build it as the runner's user, since the service cannot unpack images")
63 }
64 bin := toolpath.Look("podman")
65 out, err := exec.Command(bin, "info", "--format", "{{.Host.Security.Rootless}}").CombinedOutput()
66 if err != nil {
67 return fmt.Errorf("podman is required by -isolation podman but does not work here: %v\n%s\n"+
68 "prepare the host with deploy/runner-podman-setup.sh, or pass -isolation none "+
69 "if every repository on this instance is trusted", err, strings.TrimSpace(string(out)))
70 }
71 log.Printf("isolation: podman (rootless=%s), default image %s, images must be provisioned locally",
72 strings.TrimSpace(string(out)), r.image)
73 return nil
74 default:
75 return fmt.Errorf("unknown -isolation %q: podman or none", r.isolation)
76 }
77}
78
79// runSteps executes a job's steps and reports whether all succeeded. The
80// clone has already happened, outside any container and with the runner's
81// key: the container never sees GIT_SSH_COMMAND, the key, or the runner's
82// environment — it gets the workspace and nothing else.
83type stepRunner func(cmd *exec.Cmd, deadline time.Time) (bool, string)
84
85func (r *runner) runSteps(j job, dir string, env []string, sink io.Writer, deadline time.Time, runStep stepRunner) bool {
86 if r.isolation == isolationNone {
87 for _, step := range j.Steps {
88 fmt.Fprintf(sink, "$ %s\n", step)
89 cmd := exec.Command(toolpath.Look("sh"), "-c", step)
90 cmd.Dir, cmd.Env = dir, env
91 cmd.Stdout, cmd.Stderr = sink, sink
92 if ok, why := runStep(cmd, deadline); !ok {
93 fmt.Fprintf(sink, "%s\n", why)
94 return false
95 }
96 }
97 return true
98 }
99 return r.runStepsPodman(j, dir, env, sink, deadline, runStep)
100}
101
102// runStepsPodman starts one container for the whole job and runs each
103// step in it with `podman exec`. One container per job, not per step,
104// because steps share state — a build step writes what a test step reads
105// — and per-step containers would break that.
106func (r *runner) runStepsPodman(j job, dir string, env []string, sink io.Writer, deadline time.Time, runStep stepRunner) bool {
107 podman := toolpath.Look("podman")
108 image := j.Image
109 if image == "" {
110 image = r.image
111 }
112
113 // Secrets must not reach argv: /proc is world-readable, and this
114 // codebase keeps them on stdin or in files everywhere else. An env
115 // file outside the workspace holds them instead — outside because the
116 // workspace is bind mounted, and a file of secrets sitting in the
117 // checkout is one `cat` from a build's own log.
118 //
119 // A value with a newline in it — a private key — cannot go in the
120 // file, which has no escape for one. Those are named on the command
121 // line with --env NAME and valued in the podman process's own
122 // environment, which podman copies into the container.
123 fileEnv, inherit := splitEnv(env)
124 envFile := filepath.Join(r.workdir, fmt.Sprintf("env-%d", j.ID))
125 if err := writeEnvFile(envFile, fileEnv); err != nil {
126 fmt.Fprintf(sink, "preparing the build environment: %v\n", err)
127 return false
128 }
129 defer os.Remove(envFile)
130
131 // The build's cgroup carries its limits; podman's own cgroup handling
132 // is off because it never worked here (#188). Every podman process
133 // for this build starts inside the cgroup, exec included: one started
134 // from the runner's cgroup would run the step outside the limit.
135 var cgroupFD *os.File
136 if r.cgroups != nil {
137 dir, f, err := r.cgroups.create(j.ID, r.memory, r.cpus)
138 if err != nil {
139 fmt.Fprintf(sink, "preparing the build cgroup: %v\n", err)
140 return false
141 }
142 cgroupFD = f
143 defer f.Close()
144 defer r.cgroups.remove(dir)
145 }
146
147 name := fmt.Sprintf("gitbay-build-%d", j.ID)
148 // --rm so a container cannot outlive its build; the explicit rm below
149 // covers the case where the daemon-less run itself fails.
150 args := append(r.podmanGlobal(), "run", "--detach", "--rm", "--pull=never", "--cgroups=disabled")
151 args = append(args,
152 "--name", name,
153 "--env-file", envFile)
154 args = append(args, inheritArgs(inherit)...)
155 args = append(args,
156 "--volume", dir+":/workspace:rw",
157 // The build home holds the tool caches (Go modules, the sonar
158 // scanner) that must outlive a build; HOME in env points at it.
159 // Mounted at the same path so HOME resolves identically with and
160 // without a container. Only this directory — never the workdir
161 // above it, which holds other builds' workspaces.
162 "--volume", envHome(env)+":"+envHome(env)+":rw",
163 "--workdir", "/workspace",
164 "--entrypoint", "sh",
165 image, "-c", "sleep infinity")
166 start := exec.Command(podman, args...)
167 start.Env = append([]string{"PATH=" + os.Getenv("PATH"), "HOME=" + r.podmanHome()}, inherit...)
168 intoCgroup(start, cgroupFD)
169 if out, err := start.CombinedOutput(); err != nil {
170 // A missing image lands here, and it is the common case worth
171 // explaining: this runner never pulls, so an image it does not
172 // have is an operator's job to provision, not a transient error
173 // to retry.
174 msg := strings.TrimSpace(string(out))
175 fmt.Fprintf(sink, "starting the build container from %s failed:\n%s\n", image, msg)
176 if strings.Contains(msg, "no such image") || strings.Contains(msg, "image not known") ||
177 strings.Contains(msg, "unable to find") {
178 fmt.Fprintf(sink, "\nThis runner does not pull images. Ask an operator to provision %s "+
179 "on the runner host (podman pull, or podman build) before a job names it.\n", image)
180 }
181 return false
182 }
183 defer exec.Command(podman, append(r.podmanGlobal(), "rm", "--force", name)...).Run()
184
185 for _, step := range j.Steps {
186 fmt.Fprintf(sink, "$ %s\n", step)
187 cmd := exec.Command(podman, append(r.podmanGlobal(), "exec", "--workdir", "/workspace", name, "sh", "-c", step)...)
188 cmd.Env = []string{"PATH=" + os.Getenv("PATH"), "HOME=" + r.podmanHome()}
189 intoCgroup(cmd, cgroupFD)
190 cmd.Stdout, cmd.Stderr = sink, sink
191 if ok, why := runStep(cmd, deadline); !ok {
192 fmt.Fprintf(sink, "%s\n", why)
193 return false
194 }
195 }
196 return true
197}
198
199// podmanGlobal are the flags every podman invocation needs, before the
200// subcommand.
201//
202// The cgroup manager is cgroupfs, not systemd: the runner is a *system*
203// service, so there is no user session and no user@<uid>.service slice
204// for podman to create a scope under. With the systemd manager crun
205// fails with "create directory .../libpod-<id>.scope/container: No such
206// file or directory". The service's own cgroup is delegated
207// (Delegate=yes in the drop-in), which is what cgroupfs needs (#144).
208func (r *runner) podmanGlobal() []string {
209 // Storage paths are left to podman. They are recorded in its
210 // database at first use, so passing --root or --runroot later fails
211 // with "database configuration mismatch" — as does introducing an
212 // XDG_RUNTIME_DIR the database was not initialised with. Changing
213 // either means `podman system reset` and rebuilding the images.
214 return []string{"--cgroup-manager=cgroupfs"}
215}
216
217// envHome returns the HOME the step environment carries.
218func envHome(env []string) string {
219 for _, e := range env {
220 if strings.HasPrefix(e, "HOME=") {
221 return strings.TrimPrefix(e, "HOME=")
222 }
223 }
224 return ""
225}
226
227// podmanHome is where podman keeps its own storage: the runner's home,
228// not a build's. The container store is the runner's business, and a
229// build never sees this path.
230func (r *runner) podmanHome() string {
231 if h, err := os.UserHomeDir(); err == nil && h != "" {
232 return h
233 }
234 return "/var/lib/gitbay-runner"
235}
236
237// splitEnv separates the entries an env file can carry from those whose
238// value holds a newline, which podman must inherit from its environment.
239func splitEnv(env []string) (file, inherit []string) {
240 for _, e := range env {
241 if strings.ContainsAny(e, "\n\r") {
242 inherit = append(inherit, e)
243 } else {
244 file = append(file, e)
245 }
246 }
247 return file, inherit
248}
249
250// inheritArgs names each inherited variable for podman run: --env NAME
251// with no value makes podman take it from its own environment, so the
252// value never appears on a command line.
253func inheritArgs(inherit []string) []string {
254 var args []string
255 for _, e := range inherit {
256 name, _, _ := strings.Cut(e, "=")
257 args = append(args, "--env", name)
258 }
259 return args
260}
261
262// writeEnvFile writes KEY=VALUE lines for podman --env-file, readable
263// only by this user. Values containing a newline are refused rather than
264// silently truncated: the format has no escape for one, and a secret that
265// half-arrives is worse than a failed build.
266func writeEnvFile(path string, env []string) error {
267 var b strings.Builder
268 for _, e := range env {
269 if strings.ContainsAny(e, "\n\r") {
270 name, _, _ := strings.Cut(e, "=")
271 return fmt.Errorf("%s contains a newline, which an env file cannot carry", name)
272 }
273 b.WriteString(e)
274 b.WriteByte('\n')
275 }
276 return os.WriteFile(path, []byte(b.String()), 0o600)
277}
278
279func currentUser() string {
280 if u := os.Getenv("USER"); u != "" {
281 return u
282 }
283 return fmt.Sprintf("uid %d", os.Getuid())
284}