cmd/gitbay-runner/isolate.go

  1package main
  2
  3import (
  4	"fmt"
  5	"io"
  6	"log"
  7	"os"
  8	"os/exec"
  9	"path/filepath"
 10	"strings"
 11	"time"
 12
 13	"gitbay.org/gitbay/internal/toolpath"
 14)
 15
 16// Isolation modes. podman runs a job's steps in a container; none runs
 17// them on the host as the runner's user, which is what the runner did
 18// before #144 and what a private instance may still choose.
 19const (
 20	isolationPodman = "podman"
 21	isolationNone   = "none"
 22)
 23
 24// Images are provisioned, never pulled at build time.
 25//
 26// The service runs with RestrictSUIDSGID=yes, so podman cannot unpack an
 27// image layer containing a setuid or setgid file — which is almost every
 28// distribution image (chage, passwd, su). A pull from inside the service
 29// fails deep in the unpack with "operation not permitted" on some file
 30// nobody has heard of.
 31//
 32// Keeping that flag and provisioning images deliberately is the better
 33// half of the trade, and not only because it is one less hardening
 34// concession: on an instance where anyone can push a ci.yml, `image:`
 35// would otherwise be "fetch and run this arbitrary image from the
 36// internet". An operator pulls or builds what they will allow, and a
 37// build chooses among those. --pull=never makes that explicit rather
 38// than leaving it to whether a pull happens to fail (#144).
 39
 40// checkIsolation fails the runner at start-up rather than at the first
 41// build, and refuses anything it does not recognise. There is no silent
 42// fallback from podman to the host: dropping isolation without saying so
 43// is the failure mode this whole change exists to prevent (#144).
 44func (r *runner) checkIsolation() error {
 45	switch r.isolation {
 46	case isolationNone:
 47		log.Printf("WARNING: -isolation none: build steps run on this host as %s, "+
 48			"with no container. Only do this where every repository is trusted.", currentUser())
 49		return nil
 50	case isolationPodman:
 51		// Configuration before environment: a missing -image is the
 52		// operator's to fix whatever the host looks like, and saying so
 53		// first means the message does not depend on which machine this
 54		// is.
 55		//
 56		// No built-in default image: one that is not provisioned here
 57		// would fail every build with --pull=never, and guessing which
 58		// image an operator has is worse than asking.
 59		if r.image == "" {
 60			return fmt.Errorf("-isolation podman needs -image <ref>, the image a job runs in " +
 61				"when it names none. It must already be present on this host: " +
 62				"pull or build it as the runner's user, since the service cannot unpack images")
 63		}
 64		bin := toolpath.Look("podman")
 65		out, err := exec.Command(bin, "info", "--format", "{{.Host.Security.Rootless}}").CombinedOutput()
 66		if err != nil {
 67			return fmt.Errorf("podman is required by -isolation podman but does not work here: %v\n%s\n"+
 68				"prepare the host with deploy/runner-podman-setup.sh, or pass -isolation none "+
 69				"if every repository on this instance is trusted", err, strings.TrimSpace(string(out)))
 70		}
 71		log.Printf("isolation: podman (rootless=%s), default image %s, images must be provisioned locally",
 72			strings.TrimSpace(string(out)), r.image)
 73		return nil
 74	default:
 75		return fmt.Errorf("unknown -isolation %q: podman or none", r.isolation)
 76	}
 77}
 78
 79// runSteps executes a job's steps and reports whether all succeeded. The
 80// clone has already happened, outside any container and with the runner's
 81// key: the container never sees GIT_SSH_COMMAND, the key, or the runner's
 82// environment — it gets the workspace and nothing else.
 83type stepRunner func(cmd *exec.Cmd, deadline time.Time) (bool, string)
 84
 85func (r *runner) runSteps(j job, dir string, env []string, sink io.Writer, deadline time.Time, runStep stepRunner) bool {
 86	if r.isolation == isolationNone {
 87		for _, step := range j.Steps {
 88			fmt.Fprintf(sink, "$ %s\n", step)
 89			cmd := exec.Command(toolpath.Look("sh"), "-c", step)
 90			cmd.Dir, cmd.Env = dir, env
 91			cmd.Stdout, cmd.Stderr = sink, sink
 92			if ok, why := runStep(cmd, deadline); !ok {
 93				fmt.Fprintf(sink, "%s\n", why)
 94				return false
 95			}
 96		}
 97		return true
 98	}
 99	return r.runStepsPodman(j, dir, env, sink, deadline, runStep)
100}
101
102// runStepsPodman starts one container for the whole job and runs each
103// step in it with `podman exec`. One container per job, not per step,
104// because steps share state — a build step writes what a test step reads
105// — and per-step containers would break that.
106func (r *runner) runStepsPodman(j job, dir string, env []string, sink io.Writer, deadline time.Time, runStep stepRunner) bool {
107	podman := toolpath.Look("podman")
108	image := j.Image
109	if image == "" {
110		image = r.image
111	}
112
113	// Secrets must not reach argv: /proc is world-readable, and this
114	// codebase keeps them on stdin or in files everywhere else. An env
115	// file outside the workspace holds them instead — outside because the
116	// workspace is bind mounted, and a file of secrets sitting in the
117	// checkout is one `cat` from a build's own log.
118	envFile := filepath.Join(r.workdir, fmt.Sprintf("env-%d", j.ID))
119	if err := writeEnvFile(envFile, env); err != nil {
120		fmt.Fprintf(sink, "preparing the build environment: %v\n", err)
121		return false
122	}
123	defer os.Remove(envFile)
124
125	// The build's cgroup carries its limits; podman's own cgroup handling
126	// is off because it never worked here (#188). Every podman process
127	// for this build starts inside the cgroup, exec included: one started
128	// from the runner's cgroup would run the step outside the limit.
129	var cgroupFD *os.File
130	if r.cgroups != nil {
131		dir, f, err := r.cgroups.create(j.ID, r.memory, r.cpus)
132		if err != nil {
133			fmt.Fprintf(sink, "preparing the build cgroup: %v\n", err)
134			return false
135		}
136		cgroupFD = f
137		defer f.Close()
138		defer r.cgroups.remove(dir)
139	}
140
141	name := fmt.Sprintf("gitbay-build-%d", j.ID)
142	// --rm so a container cannot outlive its build; the explicit rm below
143	// covers the case where the daemon-less run itself fails.
144	args := append(r.podmanGlobal(), "run", "--detach", "--rm", "--pull=never", "--cgroups=disabled")
145	args = append(args,
146		"--name", name,
147		"--env-file", envFile,
148		"--volume", dir+":/workspace:rw",
149		// The build home holds the tool caches (Go modules, the sonar
150		// scanner) that must outlive a build; HOME in env points at it.
151		// Mounted at the same path so HOME resolves identically with and
152		// without a container. Only this directory — never the workdir
153		// above it, which holds other builds' workspaces.
154		"--volume", envHome(env)+":"+envHome(env)+":rw",
155		"--workdir", "/workspace",
156		"--entrypoint", "sh",
157		image, "-c", "sleep infinity")
158	start := exec.Command(podman, args...)
159	start.Env = []string{"PATH=" + os.Getenv("PATH"), "HOME=" + r.podmanHome()}
160	intoCgroup(start, cgroupFD)
161	if out, err := start.CombinedOutput(); err != nil {
162		// A missing image lands here, and it is the common case worth
163		// explaining: this runner never pulls, so an image it does not
164		// have is an operator's job to provision, not a transient error
165		// to retry.
166		msg := strings.TrimSpace(string(out))
167		fmt.Fprintf(sink, "starting the build container from %s failed:\n%s\n", image, msg)
168		if strings.Contains(msg, "no such image") || strings.Contains(msg, "image not known") ||
169			strings.Contains(msg, "unable to find") {
170			fmt.Fprintf(sink, "\nThis runner does not pull images. Ask an operator to provision %s "+
171				"on the runner host (podman pull, or podman build) before a job names it.\n", image)
172		}
173		return false
174	}
175	defer exec.Command(podman, append(r.podmanGlobal(), "rm", "--force", name)...).Run()
176
177	for _, step := range j.Steps {
178		fmt.Fprintf(sink, "$ %s\n", step)
179		cmd := exec.Command(podman, append(r.podmanGlobal(), "exec", "--workdir", "/workspace", name, "sh", "-c", step)...)
180		cmd.Env = []string{"PATH=" + os.Getenv("PATH"), "HOME=" + r.podmanHome()}
181		intoCgroup(cmd, cgroupFD)
182		cmd.Stdout, cmd.Stderr = sink, sink
183		if ok, why := runStep(cmd, deadline); !ok {
184			fmt.Fprintf(sink, "%s\n", why)
185			return false
186		}
187	}
188	return true
189}
190
191// podmanGlobal are the flags every podman invocation needs, before the
192// subcommand.
193//
194// The cgroup manager is cgroupfs, not systemd: the runner is a *system*
195// service, so there is no user session and no user@<uid>.service slice
196// for podman to create a scope under. With the systemd manager crun
197// fails with "create directory .../libpod-<id>.scope/container: No such
198// file or directory". The service's own cgroup is delegated
199// (Delegate=yes in the drop-in), which is what cgroupfs needs (#144).
200func (r *runner) podmanGlobal() []string {
201	// Storage paths are left to podman. They are recorded in its
202	// database at first use, so passing --root or --runroot later fails
203	// with "database configuration mismatch" — as does introducing an
204	// XDG_RUNTIME_DIR the database was not initialised with. Changing
205	// either means `podman system reset` and rebuilding the images.
206	return []string{"--cgroup-manager=cgroupfs"}
207}
208
209// envHome returns the HOME the step environment carries.
210func envHome(env []string) string {
211	for _, e := range env {
212		if strings.HasPrefix(e, "HOME=") {
213			return strings.TrimPrefix(e, "HOME=")
214		}
215	}
216	return ""
217}
218
219// podmanHome is where podman keeps its own storage: the runner's home,
220// not a build's. The container store is the runner's business, and a
221// build never sees this path.
222func (r *runner) podmanHome() string {
223	if h, err := os.UserHomeDir(); err == nil && h != "" {
224		return h
225	}
226	return "/var/lib/gitbay-runner"
227}
228
229// writeEnvFile writes KEY=VALUE lines for podman --env-file, readable
230// only by this user. Values containing a newline are refused rather than
231// silently truncated: the format has no escape for one, and a secret that
232// half-arrives is worse than a failed build.
233func writeEnvFile(path string, env []string) error {
234	var b strings.Builder
235	for _, e := range env {
236		if strings.ContainsAny(e, "\n\r") {
237			name, _, _ := strings.Cut(e, "=")
238			return fmt.Errorf("%s contains a newline, which an env file cannot carry", name)
239		}
240		b.WriteString(e)
241		b.WriteByte('\n')
242	}
243	return os.WriteFile(path, []byte(b.String()), 0o600)
244}
245
246func currentUser() string {
247	if u := os.Getenv("USER"); u != "" {
248		return u
249	}
250	return fmt.Sprintf("uid %d", os.Getuid())
251}