Commit `44c2c11f58`

44c2c11f585b1aed351756bec2d5d2ea1f1c1e68

parent: 74b01ee3dc

Verified · cmc ci/build: success ci/test: failure

cmc <hello@cleberg.net> · 2026-09-07 00:27 UTC

runner, deploy, wiki: images are provisioned, never pulled by a build

--pull=never, and -image is required under podman rather than defaulting
to something the host may not have. RestrictSUIDSGID stops the service
unpacking layers with setuid files, and on an open instance image: should
choose among what an operator allows rather than fetch from anywhere.

Ref #144

Layout: unified · split

.gitbay/wiki/Admin.org +19 −4

	@@ -432,10 +432,25 @@ The tag is deliberate rather than =:latest=: changing the file means
432	bumping the tag in =.gitbay/ci.yml=, so a running branch's image does not	432	bumping the tag in =.gitbay/ci.yml=, so a running branch's image does not
433	change under it.	433	change under it.
434		434
435	=-image= sets the default image for jobs that name none	435	=-image= names the image a job runs in when it declares none, and is
436	(=docker.io/library/debian:stable-slim= if unset); a job overrides it	436	required under =-isolation podman=: there is no built-in default,
437	with =image:= in =.gitbay/ci.yml=, validated as a reference so a config	437	because an image this host does not have would fail every build. A job
438	file cannot turn it into podman arguments.	438	overrides it with =image:= in =.gitbay/ci.yml=, validated as a reference
		439	so a config file cannot turn it into podman arguments.
		440
		441	Images are provisioned, never pulled by a build. The runner passes
		442	=--pull=never=. Two reasons, and the second is the better one: the
		443	service runs with =RestrictSUIDSGID=yes= so podman cannot unpack a layer
		444	holding a setuid file, which is nearly every distribution image; and on
		445	an instance where anyone can push a =ci.yml=, =image:= would otherwise
		446	mean "fetch and run anything from the internet". An operator pulls or
		447	builds what is allowed and a build picks among those. A job naming an
		448	image the host does not have fails with a message saying so.
		449
		450	#+begin_src sh
		451	su - ci-runner -s /bin/sh -c "podman pull docker.io/library/alpine:3.20"
		452	su - ci-runner -s /bin/sh -c "podman images"
		453	#+end_src
439		454
440	Prepare a host before pointing an isolating runner at it:	455	Prepare a host before pointing an isolating runner at it:
441		456

.gitbay/wiki/Threat-Model.org +5

   arbitrary repository code, and under podman that code no longer runs in
   the runner's process context. Under =-isolation none= there is no
   container and the flag should be on.
+- *Images are provisioned by the operator, not fetched by a build.* The
+  runner passes =--pull=never=, so =image:= chooses among what the host
+  already has rather than naming anything on the internet. On an
+  instance with open registration that is the difference between a
+  curated set and arbitrary code from a registry nobody vetted.
 Under =-isolation none=, anything a step can do as the runner's user a
 pushed =ci.yml= can do. Under podman a step is confined to its

cmd/gitbay-runner/env_test.go +2 −1

 // has no user slice to work in. Every invocation must say so, or crun
 // fails creating the container's scope (#144).
 func TestPodmanUsesCgroupfs(t *testing.T) {
-	got := podmanGlobal()
+	r := &runner{}
+	got := r.podmanGlobal()
 	found := false
 	for _, f := range got {
 		if f == "--cgroup-manager=cgroupfs" {

cmd/gitbay-runner/isolate.go +50 −15

 	isolationNone   = "none"
 )
-// defaultImage is used when neither the job nor -image names one. Chosen
+// Images are provisioned, never pulled at build time.
-// for being small and having a shell; anything a build actually needs it
+//
-// declares with `image:`.
+// The service runs with RestrictSUIDSGID=yes, so podman cannot unpack an
-const defaultImage = "docker.io/library/debian:stable-slim"
+// image layer containing a setuid or setgid file — which is almost every
+// distribution image (chage, passwd, su). A pull from inside the service
+// fails deep in the unpack with "operation not permitted" on some file
+// nobody has heard of.
+//
+// Keeping that flag and provisioning images deliberately is the better
+// half of the trade, and not only because it is one less hardening
+// concession: on an instance where anyone can push a ci.yml, `image:`
+// would otherwise be "fetch and run this arbitrary image from the
+// internet". An operator pulls or builds what they will allow, and a
+// build chooses among those. --pull=never makes that explicit rather
+// than leaving it to whether a pull happens to fail (#144).
 // checkIsolation fails the runner at start-up rather than at the first
 // build, and refuses anything it does not recognise. There is no silent
 			"with no container. Only do this where every repository is trusted.", currentUser())
 		return nil
 	case isolationPodman:
+		// Configuration before environment: a missing -image is the
+		// operator's to fix whatever the host looks like, and saying so
+		// first means the message does not depend on which machine this
+		// is.
+		//
+		// No built-in default image: one that is not provisioned here
+		// would fail every build with --pull=never, and guessing which
+		// image an operator has is worse than asking.
+		if r.image == "" {
+			return fmt.Errorf("-isolation podman needs -image <ref>, the image a job runs in " +
+				"when it names none. It must already be present on this host: " +
+				"pull or build it as the runner's user, since the service cannot unpack images")
+		}
 		bin := toolpath.Look("podman")
 		out, err := exec.Command(bin, "info", "--format", "{{.Host.Security.Rootless}}").CombinedOutput()
 		if err != nil {
 				"prepare the host with deploy/runner-podman-setup.sh, or pass -isolation none "+
 				"if every repository on this instance is trusted", err, strings.TrimSpace(string(out)))
 		}
-		if r.image == "" {
+		log.Printf("isolation: podman (rootless=%s), default image %s, images must be provisioned locally",
-			r.image = defaultImage
-		}
-		log.Printf("isolation: podman (rootless=%s), default image %s",
 			strings.TrimSpace(string(out)), r.image)
 		return nil
 	default:
 	name := fmt.Sprintf("gitbay-build-%d", j.ID)
 	// --rm so a container cannot outlive its build; the explicit rm below
 	// covers the case where the daemon-less run itself fails.
-	start := exec.Command(podman, append(podmanGlobal(), "run", "--detach", "--rm",
+	start := exec.Command(podman, append(r.podmanGlobal(), "run", "--detach", "--rm",
+		"--pull=never",
 		"--name", name,
 		"--env-file", envFile,
 		"--volume", dir+":/workspace:rw",
 		image, "-c", "sleep infinity")...)
 	start.Env = []string{"PATH=" + os.Getenv("PATH"), "HOME=" + r.podmanHome()}
 	if out, err := start.CombinedOutput(); err != nil {
-		// A pull failure lands here. Fail the build with what podman
+		// A missing image lands here, and it is the common case worth
-		// said; do not retry and do not fall back to another image.
+		// explaining: this runner never pulls, so an image it does not
-		fmt.Fprintf(sink, "starting the build container from %s failed:\n%s\n", image, strings.TrimSpace(string(out)))
+		// have is an operator's job to provision, not a transient error
+		// to retry.
+		msg := strings.TrimSpace(string(out))
+		fmt.Fprintf(sink, "starting the build container from %s failed:\n%s\n", image, msg)
+		if strings.Contains(msg, "no such image") || strings.Contains(msg, "image not known") ||
+			strings.Contains(msg, "unable to find") {
+			fmt.Fprintf(sink, "\nThis runner does not pull images. Ask an operator to provision %s "+
+				"on the runner host (podman pull, or podman build) before a job names it.\n", image)
+		}
 		return false
 	}
-	defer exec.Command(podman, append(podmanGlobal(), "rm", "--force", name)...).Run()
+	defer exec.Command(podman, append(r.podmanGlobal(), "rm", "--force", name)...).Run()
 	for _, step := range j.Steps {
 		fmt.Fprintf(sink, "$ %s\n", step)
-		cmd := exec.Command(podman, append(podmanGlobal(), "exec", "--workdir", "/workspace", name, "sh", "-c", step)...)
+		cmd := exec.Command(podman, append(r.podmanGlobal(), "exec", "--workdir", "/workspace", name, "sh", "-c", step)...)
 		cmd.Env = []string{"PATH=" + os.Getenv("PATH"), "HOME=" + r.podmanHome()}
 		cmd.Stdout, cmd.Stderr = sink, sink
 		if ok, why := runStep(cmd, deadline); !ok {
 // fails with "create directory .../libpod-<id>.scope/container: No such
 // file or directory". The service's own cgroup is delegated
 // (Delegate=yes in the drop-in), which is what cgroupfs needs (#144).
-func podmanGlobal() []string {
+func (r *runner) podmanGlobal() []string {
+	// Storage paths are left to podman. They are recorded in its
+	// database at first use, so passing --root or --runroot later fails
+	// with "database configuration mismatch" — as does introducing an
+	// XDG_RUNTIME_DIR the database was not initialised with. Changing
+	// either means `podman system reset` and rebuilding the images.
 	return []string{"--cgroup-manager=cgroupfs"}
 }

deploy/gitbay-runner.override.conf +12 −5

 # and read-only git, and the sandboxing below keeps a step from
 # touching the system outside its workspace.
 #
-# Delegate=yes
+# Delegate=yes and the storage path below are what rootless podman needs
-# The service unit's ExecStart carries -isolation; podman is the default,
-# and a runner that cannot find one refuses to start rather than running
-# repository code on the host. Prepare the host first
-# (deploy/runner-podman-setup.sh). and the storage path below are what rootless podman needs
 # (#144): it manages its own cgroups for a container, and its image and
 # container store lives under the runner's home, which ProtectSystem
 # would otherwise make read-only. Prepare the host with
 # deploy/runner-podman-setup.sh before deploying a runner that isolates.
 [Service]
+# ExecStart is overridden here rather than left in the unit so the flags
+# and the sandboxing that has to match them live in one file: -isolation
+# podman needs NoNewPrivileges=no below, and -image needs an image the
+# host has been given (deploy/runner-podman-setup.sh, Containerfile.ci).
+# Deliberately no XDG_RUNTIME_DIR. podman records its run root in its
+# database at first use, so setting one later fails with "database
+# configuration mismatch"; the runner's storage was initialised without
+# it and works. Change it only together with `podman system reset` and a
+# rebuild of the images (#144).
+ExecStart=
+ExecStart=/usr/local/bin/gitbay-runner -remote git@127.0.0.1 -workdir /var/lib/gitbay-runner/work -poll 5s -timeout 45m -repos krz/gitbay -isolation podman -image localhost/gitbay-ci:1
 Nice=10
 CPUWeight=30
 IOWeight=30

e2e/isolation_podman_test.go +50 −8

 // havePodman reports whether a working rootless podman is on this
 // machine. The skip is loud on purpose: an isolation test that quietly
 // does not run is how isolation regresses (#144).
+// provisionedImage returns an image present on this host, since the
+// runner never pulls one (#144). Tests must use what is provisioned, the
+// same rule builds follow.
+func provisionedImage(t *testing.T) string {
+	t.Helper()
+	for _, img := range []string{"localhost/gitbay-ci:1", "docker.io/library/debian:stable-slim", "docker.io/library/alpine:latest"} {
+		if err := exec.Command("podman", "image", "exists", img).Run(); err == nil {
+			return img
+		}
+	}
+	t.Log("SKIPPING ISOLATION TEST: podman has no image this test can use. " +
+		"Provision one (podman build -t localhost/gitbay-ci:1 -f deploy/Containerfile.ci). " +
+		"The container path is NOT covered by this run.")
+	return ""
+}
 func havePodman(t *testing.T) bool {
 	t.Helper()
 	if _, err := exec.LookPath("podman"); err != nil {
 func TestRunnerRefusesToStartWithoutPodman(t *testing.T) {
 	bin := buildRunner(t)
 	cmd := exec.Command(bin, "-once", "-remote", "git@127.0.0.1",
-		"-isolation", "podman", "-workdir", t.TempDir())
+		"-isolation", "podman", "-image", "localhost/whatever:1", "-workdir", t.TempDir())
 	// An empty PATH is the reliable way to make podman missing whether or
 	// not this machine has one.
 	cmd.Env = []string{"PATH=" + t.TempDir(), "HOME=" + t.TempDir()}
 	inst.admin(t, "admin", "user", "create", "ci", "--key", runnerKey+".pub", "--admin")
 	inst.ssh(t, aliceKey, "", "repo", "create", "alice/app")
+	image := provisionedImage(t)
+	if image == "" {
+		t.Skip("no provisioned image")
+	}
 	env := inst.gitEnv(aliceKey)
 	work := t.TempDir()
 	mustGit(t, work, env, "clone", inst.sshURL("alice/app"), "w")
 	// The step tries to read the key the runner authenticates with, and
 	// to list the runner's home. Both must fail inside the container.
 	os.WriteFile(filepath.Join(dir, ".gitbay", "ci.yml"), []byte(
-		"jobs:\n  peek:\n    image: docker.io/library/debian:stable-slim\n    steps:\n"+
+		"jobs:\n  peek:\n    image: "+image+"\n    steps:\n"+
 			"      - 'if cat "+runnerKey+" 2>/dev/null; then echo LEAKED-KEY; exit 1; fi; echo no-key'\n"+
 			"      - 'echo HOME=$HOME; ls /workspace'\n"), 0o644)
 	mustGit(t, dir, env, "checkout", "-q", "-b", "main")
 	runnerPodmanOnce(t, inst, runnerKey)
 	out, _, _ := inst.ssh(t, aliceKey, "", "build", "list", "alice/app")
+	log, _, _ := inst.ssh(t, aliceKey, "", "build", "log", "alice/app", "1")
 	if !strings.Contains(out, "success") {
-		t.Fatalf("the containerised build did not pass:\n%s", out)
+		// Without the log this says only "it failed", which cost two CI
+		// rounds to diagnose the first time.
+		t.Fatalf("the containerised build did not pass:\n%s\nbuild log:\n%s", out, log)
 	}
-	log, _, _ := inst.ssh(t, aliceKey, "", "build", "log", "alice/app", "1")
 	if strings.Contains(log, "LEAKED-KEY") {
 		t.Errorf("a step read the runner's ssh key:\n%s", log)
 	}
 	}
 }
-// A pull failure fails the build and says why, rather than retrying or
+// An image this runner does not have fails the build and says an
-// silently choosing another image.
+// operator must provision it, rather than pulling it.
-func TestPodmanPullFailureFailsTheBuild(t *testing.T) {
+func TestPodmanMissingImageFailsTheBuild(t *testing.T) {
 	if !havePodman(t) {
 		t.Skip("no podman")
 	}
 	}
 	log, _, _ := inst.ssh(t, aliceKey, "", "build", "log", "alice/app", "1")
 	if !strings.Contains(log, "gitbay-no-such-image") {
-		t.Errorf("the log does not name the image that could not be pulled:\n%s", log)
+		t.Errorf("the log does not name the missing image:\n%s", log)
+	}
+	if !strings.Contains(log, "does not pull images") {
+		t.Errorf("the log does not say an operator must provision it:\n%s", log)
 	}
 	if strings.Contains(log, "unreachable") {
 		t.Error("a step ran despite the image failing to start")
 		"-remote", "git@127.0.0.1",
 		"-ssh-opts", opts,
 		"-isolation", "podman",
+		"-image", "localhost/gitbay-ci:1",
 		"-clone-base", fmt.Sprintf("ssh://git@127.0.0.1:%d", inst.port),
 		"-workdir", t.TempDir())
 	cmd.Env = append(os.Environ(), "GIT_CONFIG_NOSYSTEM=1", "GIT_CONFIG_GLOBAL=/dev/null")
 		t.Fatalf("runner: %v\n%s", err, out)
 	}
 }
+// Under podman the runner insists on a default image rather than
+// guessing one: with --pull=never an image the host does not have fails
+// every job that names none.
+func TestRunnerRefusesPodmanWithoutAnImage(t *testing.T) {
+	bin := buildRunner(t)
+	cmd := exec.Command(bin, "-once", "-remote", "git@127.0.0.1",
+		"-isolation", "podman", "-workdir", t.TempDir())
+	out, err := cmd.CombinedOutput()
+	if err == nil {
+		t.Fatalf("the runner started in podman mode with no -image:\n%s", out)
+	}
+	if !strings.Contains(string(out), "-image") {
+		t.Errorf("refusal does not name the missing flag:\n%s", out)
+	}
+}

Commit 44c2c11f58

Commit `44c2c11f58`