cmd/gitbay-runner/isolate.go
285 lines · 11618 bytes
1package main
2
3import (
4 "fmt"
5 "io"
6 "log"
7 "os"
8 "os/exec"
9 "path/filepath"
10 "strings"
11 "time"
12
13 "gitbay.org/gitbay/internal/toolpath"
14)
15
16// Isolation modes. podman runs a job's steps in a container; none runs
17// them on the host as the runner's user, which is what the runner did
18// before #144 and what a private instance may still choose.
19const (
20 isolationPodman = "podman"
21 isolationNone = "none"
22)
23
24// Images are provisioned, never pulled at build time.
25//
26// The service runs with RestrictSUIDSGID=yes, so podman cannot unpack an
27// image layer containing a setuid or setgid file — which is almost every
28// distribution image (chage, passwd, su). A pull from inside the service
29// fails deep in the unpack with "operation not permitted" on some file
30// nobody has heard of.
31//
32// Keeping that flag and provisioning images deliberately is the better
33// half of the trade, and not only because it is one less hardening
34// concession: on an instance where anyone can push a ci.yml, `image:`
35// would otherwise be "fetch and run this arbitrary image from the
36// internet". An operator pulls or builds what they will allow, and a
37// build chooses among those. --pull=never makes that explicit rather
38// than leaving it to whether a pull happens to fail (#144).
39
40// checkIsolation fails the runner at start-up rather than at the first
41// build, and refuses anything it does not recognise. There is no silent
42// fallback from podman to the host: dropping isolation without saying so
43// is the failure mode this whole change exists to prevent (#144).
44func (r *runner) checkIsolation() error {
45 switch r.isolation {
46 case isolationNone:
47 log.Printf("WARNING: -isolation none: build steps run on this host as %s, "+
48 "with no container. Only do this where every repository is trusted.", currentUser())
49 return nil
50 case isolationPodman:
51 // Configuration before environment: a missing -image is the
52 // operator's to fix whatever the host looks like, and saying so
53 // first means the message does not depend on which machine this
54 // is.
55 //
56 // No built-in default image: one that is not provisioned here
57 // would fail every build with --pull=never, and guessing which
58 // image an operator has is worse than asking.
59 if r.image == "" {
60 return fmt.Errorf("-isolation podman needs -image <ref>, the image a job runs in " +
61 "when it names none. It must already be present on this host: " +
62 "pull or build it as the runner's user, since the service cannot unpack images")
63 }
64 bin := toolpath.Look("podman")
65 out, err := exec.Command(bin, "info", "--format", "{{.Host.Security.Rootless}}").CombinedOutput()
66 if err != nil {
67 return fmt.Errorf("podman is required by -isolation podman but does not work here: %v\n%s\n"+
68 "prepare the host with deploy/runner-podman-setup.sh, or pass -isolation none "+
69 "if every repository on this instance is trusted", err, strings.TrimSpace(string(out)))
70 }
71 log.Printf("isolation: podman (rootless=%s), default image %s, images must be provisioned locally",
72 strings.TrimSpace(string(out)), r.image)
73 return nil
74 default:
75 return fmt.Errorf("unknown -isolation %q: podman or none", r.isolation)
76 }
77}
78
79// runSteps executes a job's steps. Returns nil when every step succeeded.
80// The clone has already happened, outside any container and with the
81// runner's key: the container never sees GIT_SSH_COMMAND, the key, or the
82// runner's environment — it gets the workspace and nothing else.
83type stepRunner func(cmd *exec.Cmd, deadline time.Time) (bool, string)
84
85func (r *runner) runSteps(j job, dir string, env []string, sink io.Writer, deadline time.Time, runStep stepRunner) *failure {
86 if r.isolation == isolationNone {
87 for i, step := range j.Steps {
88 fmt.Fprintf(sink, "$ %s\n", step)
89 cmd := exec.Command(toolpath.Look("sh"), "-c", step)
90 cmd.Dir, cmd.Env = dir, env
91 cmd.Stdout, cmd.Stderr = sink, sink
92 if ok, why := runStep(cmd, deadline); !ok {
93 fmt.Fprintf(sink, "step %d/%d failed: %s\n", i+1, len(j.Steps), why)
94 return &failure{Step: i + 1, Reason: why}
95 }
96 }
97 return nil
98 }
99 return r.runStepsPodman(j, dir, env, sink, deadline, runStep)
100}
101
102// runStepsPodman starts one container for the whole job and runs each
103// step in it with `podman exec`. One container per job, not per step,
104// because steps share state — a build step writes what a test step reads
105// — and per-step containers would break that.
106func (r *runner) runStepsPodman(j job, dir string, env []string, sink io.Writer, deadline time.Time, runStep stepRunner) *failure {
107 podman := toolpath.Look("podman")
108 image := j.Image
109 if image == "" {
110 image = r.image
111 }
112
113 // Secrets must not reach argv: /proc is world-readable, and this
114 // codebase keeps them on stdin or in files everywhere else. An env
115 // file outside the workspace holds them instead — outside because the
116 // workspace is bind mounted, and a file of secrets sitting in the
117 // checkout is one `cat` from a build's own log.
118 //
119 // A value with a newline in it — a private key — cannot go in the
120 // file, which has no escape for one. Those are named on the command
121 // line with --env NAME and valued in the podman process's own
122 // environment, which podman copies into the container.
123 fileEnv, inherit := splitEnv(env)
124 envFile := filepath.Join(r.workdir, fmt.Sprintf("env-%d", j.ID))
125 if err := writeEnvFile(envFile, fileEnv); err != nil {
126 fmt.Fprintf(sink, "preparing the build environment: %v\n", err)
127 return &failure{Reason: "preparing the build environment failed"}
128 }
129 defer os.Remove(envFile)
130
131 // The build's cgroup carries its limits; podman's own cgroup handling
132 // is off because it never worked here (#188). Every podman process
133 // for this build starts inside the cgroup, exec included: one started
134 // from the runner's cgroup would run the step outside the limit.
135 var cgroupFD *os.File
136 if r.cgroups != nil {
137 dir, f, err := r.cgroups.create(j.ID, r.memory, r.cpus)
138 if err != nil {
139 fmt.Fprintf(sink, "preparing the build cgroup: %v\n", err)
140 return &failure{Reason: "preparing the build cgroup failed"}
141 }
142 cgroupFD = f
143 defer f.Close()
144 defer r.cgroups.remove(dir)
145 }
146
147 name := fmt.Sprintf("gitbay-build-%d", j.ID)
148 // --rm so a container cannot outlive its build; the explicit rm below
149 // covers the case where the daemon-less run itself fails.
150 args := append(r.podmanGlobal(), "run", "--detach", "--rm", "--pull=never", "--cgroups=disabled")
151 args = append(args,
152 "--name", name,
153 "--env-file", envFile)
154 args = append(args, r.buildNetwork()...)
155 args = append(args, inheritArgs(inherit)...)
156 args = append(args,
157 "--volume", dir+":/workspace:rw",
158 // The build home holds the tool caches (Go modules, the sonar
159 // scanner) that must outlive a build; HOME in env points at it.
160 // Mounted at the same path so HOME resolves identically with and
161 // without a container. Only this directory — never the workdir
162 // above it, which holds other builds' workspaces.
163 "--volume", envHome(env)+":"+envHome(env)+":rw",
164 "--workdir", "/workspace",
165 "--entrypoint", "sh",
166 image, "-c", "sleep infinity")
167 start := exec.Command(podman, args...)
168 start.Env = append([]string{"PATH=" + os.Getenv("PATH"), "HOME=" + r.podmanHome()}, inherit...)
169 intoCgroup(start, cgroupFD)
170 if out, err := start.CombinedOutput(); err != nil {
171 // A missing image lands here, and it is the common case worth
172 // explaining: this runner never pulls, so an image it does not
173 // have is an operator's job to provision, not a transient error
174 // to retry.
175 msg := strings.TrimSpace(string(out))
176 fmt.Fprintf(sink, "starting the build container from %s failed:\n%s\n", image, msg)
177 if strings.Contains(msg, "no such image") || strings.Contains(msg, "image not known") ||
178 strings.Contains(msg, "unable to find") {
179 fmt.Fprintf(sink, "\nThis runner does not pull images. Ask an operator to provision %s "+
180 "on the runner host (podman pull, or podman build) before a job names it.\n", image)
181 }
182 return &failure{Reason: "starting the build container failed"}
183 }
184 defer exec.Command(podman, append(r.podmanGlobal(), "rm", "--force", name)...).Run()
185
186 for i, step := range j.Steps {
187 fmt.Fprintf(sink, "$ %s\n", step)
188 cmd := exec.Command(podman, append(r.podmanGlobal(), "exec", "--workdir", "/workspace", name, "sh", "-c", step)...)
189 cmd.Env = []string{"PATH=" + os.Getenv("PATH"), "HOME=" + r.podmanHome()}
190 intoCgroup(cmd, cgroupFD)
191 cmd.Stdout, cmd.Stderr = sink, sink
192 if ok, why := runStep(cmd, deadline); !ok {
193 fmt.Fprintf(sink, "step %d/%d failed: %s\n", i+1, len(j.Steps), why)
194 return &failure{Step: i + 1, Reason: why}
195 }
196 }
197 return nil
198}
199
200// podmanGlobal are the flags every podman invocation needs, before the
201// subcommand.
202//
203// The cgroup manager is cgroupfs, not systemd: the runner is a *system*
204// service, so there is no user session and no user@<uid>.service slice
205// for podman to create a scope under. With the systemd manager crun
206// fails with "create directory .../libpod-<id>.scope/container: No such
207// file or directory". The service's own cgroup is delegated
208// (Delegate=yes in the drop-in), which is what cgroupfs needs (#144).
209func (r *runner) podmanGlobal() []string {
210 // Storage paths are left to podman. They are recorded in its
211 // database at first use, so passing --root or --runroot later fails
212 // with "database configuration mismatch" — as does introducing an
213 // XDG_RUNTIME_DIR the database was not initialised with. Changing
214 // either means `podman system reset` and rebuilding the images.
215 return []string{"--cgroup-manager=cgroupfs"}
216}
217
218// envHome returns the HOME the step environment carries.
219func envHome(env []string) string {
220 for _, e := range env {
221 if strings.HasPrefix(e, "HOME=") {
222 return strings.TrimPrefix(e, "HOME=")
223 }
224 }
225 return ""
226}
227
228// podmanHome is where podman keeps its own storage: the runner's home,
229// not a build's. The container store is the runner's business, and a
230// build never sees this path.
231func (r *runner) podmanHome() string {
232 if h, err := os.UserHomeDir(); err == nil && h != "" {
233 return h
234 }
235 return "/var/lib/gitbay-runner"
236}
237
238// splitEnv separates the entries an env file can carry from those whose
239// value holds a newline, which podman must inherit from its environment.
240func splitEnv(env []string) (file, inherit []string) {
241 for _, e := range env {
242 if strings.ContainsAny(e, "\n\r") {
243 inherit = append(inherit, e)
244 } else {
245 file = append(file, e)
246 }
247 }
248 return file, inherit
249}
250
251// inheritArgs names each inherited variable for podman run: --env NAME
252// with no value makes podman take it from its own environment, so the
253// value never appears on a command line.
254func inheritArgs(inherit []string) []string {
255 var args []string
256 for _, e := range inherit {
257 name, _, _ := strings.Cut(e, "=")
258 args = append(args, "--env", name)
259 }
260 return args
261}
262
263// writeEnvFile writes KEY=VALUE lines for podman --env-file, readable
264// only by this user. Values containing a newline are refused rather than
265// silently truncated: the format has no escape for one, and a secret that
266// half-arrives is worse than a failed build.
267func writeEnvFile(path string, env []string) error {
268 var b strings.Builder
269 for _, e := range env {
270 if strings.ContainsAny(e, "\n\r") {
271 name, _, _ := strings.Cut(e, "=")
272 return fmt.Errorf("%s contains a newline, which an env file cannot carry", name)
273 }
274 b.WriteString(e)
275 b.WriteByte('\n')
276 }
277 return os.WriteFile(path, []byte(b.String()), 0o600)
278}
279
280func currentUser() string {
281 if u := os.Getenv("USER"); u != "" {
282 return u
283 }
284 return fmt.Sprintf("uid %d", os.Getuid())
285}