cmd/gitbay-runner/isolate.go

  1package main
  2
  3import (
  4	"fmt"
  5	"io"
  6	"log"
  7	"os"
  8	"os/exec"
  9	"path/filepath"
 10	"strings"
 11	"time"
 12
 13	"gitbay.org/gitbay/internal/toolpath"
 14)
 15
 16// Isolation modes. podman runs a job's steps in a container; none runs
 17// them on the host as the runner's user, which is what the runner did
 18// before #144 and what a private instance may still choose.
 19const (
 20	isolationPodman = "podman"
 21	isolationNone   = "none"
 22)
 23
 24// Images are provisioned, never pulled at build time.
 25//
 26// The service runs with RestrictSUIDSGID=yes, so podman cannot unpack an
 27// image layer containing a setuid or setgid file — which is almost every
 28// distribution image (chage, passwd, su). A pull from inside the service
 29// fails deep in the unpack with "operation not permitted" on some file
 30// nobody has heard of.
 31//
 32// Keeping that flag and provisioning images deliberately is the better
 33// half of the trade, and not only because it is one less hardening
 34// concession: on an instance where anyone can push a ci.yml, `image:`
 35// would otherwise be "fetch and run this arbitrary image from the
 36// internet". An operator pulls or builds what they will allow, and a
 37// build chooses among those. --pull=never makes that explicit rather
 38// than leaving it to whether a pull happens to fail (#144).
 39
 40// checkIsolation fails the runner at start-up rather than at the first
 41// build, and refuses anything it does not recognise. There is no silent
 42// fallback from podman to the host: dropping isolation without saying so
 43// is the failure mode this whole change exists to prevent (#144).
 44func (r *runner) checkIsolation() error {
 45	switch r.isolation {
 46	case isolationNone:
 47		log.Printf("WARNING: -isolation none: build steps run on this host as %s, "+
 48			"with no container. Only do this where every repository is trusted.", currentUser())
 49		return nil
 50	case isolationPodman:
 51		// Configuration before environment: a missing -image is the
 52		// operator's to fix whatever the host looks like, and saying so
 53		// first means the message does not depend on which machine this
 54		// is.
 55		//
 56		// No built-in default image: one that is not provisioned here
 57		// would fail every build with --pull=never, and guessing which
 58		// image an operator has is worse than asking.
 59		if r.image == "" {
 60			return fmt.Errorf("-isolation podman needs -image <ref>, the image a job runs in " +
 61				"when it names none. It must already be present on this host: " +
 62				"pull or build it as the runner's user, since the service cannot unpack images")
 63		}
 64		bin := toolpath.Look("podman")
 65		out, err := exec.Command(bin, "info", "--format", "{{.Host.Security.Rootless}}").CombinedOutput()
 66		if err != nil {
 67			return fmt.Errorf("podman is required by -isolation podman but does not work here: %v\n%s\n"+
 68				"prepare the host with deploy/runner-podman-setup.sh, or pass -isolation none "+
 69				"if every repository on this instance is trusted", err, strings.TrimSpace(string(out)))
 70		}
 71		log.Printf("isolation: podman (rootless=%s), default image %s, images must be provisioned locally",
 72			strings.TrimSpace(string(out)), r.image)
 73		return nil
 74	default:
 75		return fmt.Errorf("unknown -isolation %q: podman or none", r.isolation)
 76	}
 77}
 78
 79// runSteps executes a job's steps and reports whether all succeeded. The
 80// clone has already happened, outside any container and with the runner's
 81// key: the container never sees GIT_SSH_COMMAND, the key, or the runner's
 82// environment — it gets the workspace and nothing else.
 83type stepRunner func(cmd *exec.Cmd, deadline time.Time) (bool, string)
 84
 85func (r *runner) runSteps(j job, dir string, env []string, sink io.Writer, deadline time.Time, runStep stepRunner) bool {
 86	if r.isolation == isolationNone {
 87		for _, step := range j.Steps {
 88			fmt.Fprintf(sink, "$ %s\n", step)
 89			cmd := exec.Command(toolpath.Look("sh"), "-c", step)
 90			cmd.Dir, cmd.Env = dir, env
 91			cmd.Stdout, cmd.Stderr = sink, sink
 92			if ok, why := runStep(cmd, deadline); !ok {
 93				fmt.Fprintf(sink, "%s\n", why)
 94				return false
 95			}
 96		}
 97		return true
 98	}
 99	return r.runStepsPodman(j, dir, env, sink, deadline, runStep)
100}
101
102// runStepsPodman starts one container for the whole job and runs each
103// step in it with `podman exec`. One container per job, not per step,
104// because steps share state — a build step writes what a test step reads
105// — and per-step containers would break that.
106func (r *runner) runStepsPodman(j job, dir string, env []string, sink io.Writer, deadline time.Time, runStep stepRunner) bool {
107	podman := toolpath.Look("podman")
108	image := j.Image
109	if image == "" {
110		image = r.image
111	}
112
113	// Secrets must not reach argv: /proc is world-readable, and this
114	// codebase keeps them on stdin or in files everywhere else. An env
115	// file outside the workspace holds them instead — outside because the
116	// workspace is bind mounted, and a file of secrets sitting in the
117	// checkout is one `cat` from a build's own log.
118	envFile := filepath.Join(r.workdir, fmt.Sprintf("env-%d", j.ID))
119	if err := writeEnvFile(envFile, env); err != nil {
120		fmt.Fprintf(sink, "preparing the build environment: %v\n", err)
121		return false
122	}
123	defer os.Remove(envFile)
124
125	name := fmt.Sprintf("gitbay-build-%d", j.ID)
126	// --rm so a container cannot outlive its build; the explicit rm below
127	// covers the case where the daemon-less run itself fails.
128	start := exec.Command(podman, append(r.podmanGlobal(), "run", "--detach", "--rm",
129		"--pull=never",
130		"--name", name,
131		"--env-file", envFile,
132		"--volume", dir+":/workspace:rw",
133		"--workdir", "/workspace",
134		"--entrypoint", "sh",
135		image, "-c", "sleep infinity")...)
136	start.Env = []string{"PATH=" + os.Getenv("PATH"), "HOME=" + r.podmanHome()}
137	if out, err := start.CombinedOutput(); err != nil {
138		// A missing image lands here, and it is the common case worth
139		// explaining: this runner never pulls, so an image it does not
140		// have is an operator's job to provision, not a transient error
141		// to retry.
142		msg := strings.TrimSpace(string(out))
143		fmt.Fprintf(sink, "starting the build container from %s failed:\n%s\n", image, msg)
144		if strings.Contains(msg, "no such image") || strings.Contains(msg, "image not known") ||
145			strings.Contains(msg, "unable to find") {
146			fmt.Fprintf(sink, "\nThis runner does not pull images. Ask an operator to provision %s "+
147				"on the runner host (podman pull, or podman build) before a job names it.\n", image)
148		}
149		return false
150	}
151	defer exec.Command(podman, append(r.podmanGlobal(), "rm", "--force", name)...).Run()
152
153	for _, step := range j.Steps {
154		fmt.Fprintf(sink, "$ %s\n", step)
155		cmd := exec.Command(podman, append(r.podmanGlobal(), "exec", "--workdir", "/workspace", name, "sh", "-c", step)...)
156		cmd.Env = []string{"PATH=" + os.Getenv("PATH"), "HOME=" + r.podmanHome()}
157		cmd.Stdout, cmd.Stderr = sink, sink
158		if ok, why := runStep(cmd, deadline); !ok {
159			fmt.Fprintf(sink, "%s\n", why)
160			return false
161		}
162	}
163	return true
164}
165
166// podmanGlobal are the flags every podman invocation needs, before the
167// subcommand.
168//
169// The cgroup manager is cgroupfs, not systemd: the runner is a *system*
170// service, so there is no user session and no user@<uid>.service slice
171// for podman to create a scope under. With the systemd manager crun
172// fails with "create directory .../libpod-<id>.scope/container: No such
173// file or directory". The service's own cgroup is delegated
174// (Delegate=yes in the drop-in), which is what cgroupfs needs (#144).
175func (r *runner) podmanGlobal() []string {
176	// Storage paths are left to podman. They are recorded in its
177	// database at first use, so passing --root or --runroot later fails
178	// with "database configuration mismatch" — as does introducing an
179	// XDG_RUNTIME_DIR the database was not initialised with. Changing
180	// either means `podman system reset` and rebuilding the images.
181	return []string{"--cgroup-manager=cgroupfs"}
182}
183
184// podmanHome is where podman keeps its own storage: the runner's home,
185// not a build's. The container store is the runner's business, and a
186// build never sees this path.
187func (r *runner) podmanHome() string {
188	if h, err := os.UserHomeDir(); err == nil && h != "" {
189		return h
190	}
191	return "/var/lib/gitbay-runner"
192}
193
194// writeEnvFile writes KEY=VALUE lines for podman --env-file, readable
195// only by this user. Values containing a newline are refused rather than
196// silently truncated: the format has no escape for one, and a secret that
197// half-arrives is worse than a failed build.
198func writeEnvFile(path string, env []string) error {
199	var b strings.Builder
200	for _, e := range env {
201		if strings.ContainsAny(e, "\n\r") {
202			name, _, _ := strings.Cut(e, "=")
203			return fmt.Errorf("%s contains a newline, which an env file cannot carry", name)
204		}
205		b.WriteString(e)
206		b.WriteByte('\n')
207	}
208	return os.WriteFile(path, []byte(b.String()), 0o600)
209}
210
211func currentUser() string {
212	if u := os.Getenv("USER"); u != "" {
213		return u
214	}
215	return fmt.Sprintf("uid %d", os.Getuid())
216}