runner, deploy, wiki: a build home per repository, and a memory cap on bay1 !332

merged merged by cmc on 2026-09-07 19:47 UTC · krz/gitbay:runner-home-memory-184 into main

6 files changed, +128 −23

Layout: unified · split

.gitbay/wiki/Admin.org +19 −2

	@@ -391,6 +391,11 @@ gitbay-runner -remote git@gitbay.org -repos krz/site,krz/docs \
391	-workdir /var/lib/gitbay-runner/work	391	-workdir /var/lib/gitbay-runner/work
392	#+end_src	392	#+end_src
393		393
		394	gitbay.org's runner is scoped: it builds the forge's own repositories
		395	and the isolation canary, nothing else, because it shares the host with
		396	the forge. A =.gitbay/ci.yml= in another repository there queues builds
		397	no runner claims. Whether that changes is krz/gitbay#184.
		398
394	Naming no repositories is the old behaviour and stays the right choice for	399	Naming no repositories is the old behaviour and stays the right choice for
395	the runner on the server itself. The scoping is what the runner asks for,	400	the runner on the server itself. The scoping is what the runner asks for,
396	not an ACL the server holds over it: a runner account is admin by	401	not an ACL the server holds over it: a runner account is admin by
	@@ -444,8 +449,20 @@ overrides it with =image:= in =.gitbay/ci.yml=, validated as a reference
444	so a config file cannot turn it into podman arguments.	449	so a config file cannot turn it into podman arguments.
445		450
446	=-cpus= and =-memory= cap one build's container (podman's own units,	451	=-cpus= and =-memory= cap one build's container (podman's own units,
447	e.g. =-cpus 2 -memory 4g=); unset means uncapped. bay1 runs =-cpus 3=	452	e.g. =-cpus 2 -memory 4g=); unset means uncapped. They only take effect
448	and no memory cap, since the e2e suite needs most of the host's 7GB.	453	where podman can create a cgroup for the container. Under rootless
		454	podman with the cgroupfs manager, which is what a system service gets,
		455	it cannot: the container runs inside the service's own cgroup and both
		456	flags are accepted and ignored (krz/gitbay#188). Cap the unit instead.
		457	bay1's drop-in sets =MemoryMax=6G= and =CPUQuota=300%= on a 7.7GB
		458	four-core host with no swap: the memory cap is what keeps the forge
		459	alive when a build allocates without bound, and it sits above the e2e
		460	suite's 5GB peak rather than at a fair share. =OOMPolicy=continue= keeps
		461	systemd from stopping the runner when a build is OOM-killed.
		462
		463	Each repository gets its own build home under the runner's workdir,
		464	mounted into its containers as =HOME=. Caches persist between builds of
		465	one repository and are never read by another's.
449		466
450	Images are provisioned, never pulled by a build. The runner passes	467	Images are provisioned, never pulled by a build. The runner passes
451	=--pull=never=. Two reasons, and the second is the better one: the	468	=--pull=never=. Two reasons, and the second is the better one: the

.gitbay/wiki/FAQ.org +6

	@@ -17,3 +17,9 @@
17	email patch flow (revisit only if sourcehut-style demand appears),	17	email patch flow (revisit only if sourcehut-style demand appears),
18	federation and Postgres (no need at this scale). Recorded so the	18	federation and Postgres (no need at this scale). Recorded so the
19	absence reads as a decision, not an oversight.	19	absence reads as a decision, not an oversight.
		20	- Does CI run for my repository on gitbay.org? :: Not yet. The runner
		21	there is scoped to the forge's own repositories and its isolation
		22	canary, since it shares the host with the forge. A =.gitbay/ci.yml=
		23	in your repository queues builds nothing claims. krz/gitbay#184 is
		24	where that gets decided. A self-hosted instance runs the same runner
		25	for whichever repositories its operator names.

.gitbay/wiki/Threat-Model.org +7 −5

	@@ -166,11 +166,13 @@ runner, polling over SSH, clones the commit and runs its steps.
166		166
167	Under =-isolation none=, anything a step can do as the runner's user a	167	Under =-isolation none=, anything a step can do as the runner's user a
168	pushed =ci.yml= can do. Under podman a step is confined to its	168	pushed =ci.yml= can do. Under podman a step is confined to its
169	container and the bind-mounted workspace, but the build home's caches	169	container, the bind-mounted workspace and the repository's own build
170	are shared between builds, so one build can still leave something a	170	home, so what a build leaves in a cache is read only by later builds of
171	later build reads. Treat the runner host as executing untrusted code:	171	the same repository. Treat the runner host as executing untrusted code
172	keep it off the daemon's host where the database lives, or scope it to	172	all the same: keep it off the daemon's host where the database lives,
173	repositories whose writers you trust.	173	or scope it to repositories whose writers you trust. gitbay.org does
		174	the latter — its runner builds only the repositories the operator
		175	names.
174		176
175	* What has not been audited	177	* What has not been audited
176		178

cmd/gitbay-runner/home_test.go added +53

@@ -0,0 +1,53 @@
	1	package main
	2
	3	import (
	4	"os"
	5	"path/filepath"
	6	"testing"
	7	)
	8
	9	// The build home is per repository: one shared home let a step poison
	10	// the module cache or plant a .gitconfig that another repository's build
	11	// would honour (#184).
	12	func TestBuildHomeIsPerRepository(t *testing.T) {
	13	work := t.TempDir()
	14	a, err := buildHomeFor(work, "alice/app")
	15	if err != nil {
	16	t.Fatal(err)
	17	}
	18	b, err := buildHomeFor(work, "bob/app")
	19	if err != nil {
	20	t.Fatal(err)
	21	}
	22	if a == b {
	23	t.Fatalf("two repositories share a build home: %s", a)
	24	}
	25	for _, dir := range []string{a, b} {
	26	rel, err := filepath.Rel(filepath.Join(work, "home"), dir)
	27	if err != nil \|\| rel == "." \|\| filepath.IsAbs(rel) \|\| rel[0] == '.' {
	28	t.Fatalf("build home %s is not under %s/home", dir, work)
	29	}
	30	st, err := os.Stat(dir)
	31	if err != nil {
	32	t.Fatalf("build home not created: %v", err)
	33	}
	34	if st.Mode().Perm() != 0o700 {
	35	t.Fatalf("build home mode %o, want 0700", st.Mode().Perm())
	36	}
	37	}
	38	// The same repository gets the same home back: that is what makes it
	39	// a cache.
	40	again, _ := buildHomeFor(work, "alice/app")
	41	if again != a {
	42	t.Fatalf("build home moved between builds: %s then %s", a, again)
	43	}
	44	}
	45
	46	// A repository path is server-validated, but the home must still never
	47	// resolve outside the runner's home root.
	48	func TestBuildHomeRefusesTraversal(t *testing.T) {
	49	work := t.TempDir()
	50	if _, err := buildHomeFor(work, "../../etc"); err == nil {
	51	t.Fatal("a traversing repository path produced a build home")
	52	}
	53	}

cmd/gitbay-runner/main.go +26 −12

 	// credential dotfiles are. A directory beside the workspaces is
 	// neither.
 	//
-	// It is shared by every build on this runner, so a step can poison a
+	// One per repository: shared across repositories, a step could poison
-	// cache another repository's build will read. That is already true of
+	// the module cache or plant a .gitconfig that another repository's
-	// anything a step can reach as this user — see the wiki's
+	// build would honour, and the container mounts the home read-write
-	// Threat-Model on the runner — and is what container isolation (#144)
+	// (#184).
-	// is for; -repos is the control until then.
+	buildHome, err := buildHomeFor(r.workdir, j.Repo)
-	buildHome := filepath.Join(r.workdir, "home")
+	if err != nil {
-	if err := os.MkdirAll(buildHome, 0o700); err != nil {
 		log.Printf("build %d: build home: %v", j.ID, err)
 		return false
 	}
 // the runner's entire environment, including anything an operator set on
 // the service (#144).
 //
-// HOME is a build home shared by this runner's builds, not the runner's
+// HOME is the repository's build home, not the runner's own: tools read
-// own: tools read credentials out of dotfiles — .netrc, .npmrc,
+// credentials out of dotfiles — .netrc, .npmrc, .gitconfig — and a build
-// .gitconfig — and a build has no business finding the runner's. It is
+// has no business finding the runner's. It is not the workspace either,
-// not the workspace either, because the workspace is deleted after every
+// because the workspace is deleted after every build and every tool
-// build and every tool cache lives under HOME.
+// cache lives under HOME.
 //
 // PATH is the one thing carried over: without it a step cannot find the
 // tools the host was provisioned with.
+// buildHomeFor is the build home for one repository: <workdir>/home/<owner>/<name>,
+// created on first use. The repository path comes from the server, but a
+// home must still never resolve outside the home root.
+func buildHomeFor(workdir, repo string) (string, error) {
+	root := filepath.Join(workdir, "home")
+	dir := filepath.Join(root, filepath.FromSlash(repo))
+	if rel, err := filepath.Rel(root, dir); err != nil || rel == "." || strings.HasPrefix(rel, "..") {
+		return "", fmt.Errorf("repository path %q escapes the build home root", repo)
+	}
+	if err := os.MkdirAll(dir, 0o700); err != nil {
+		return "", err
+	}
+	return dir, nil
+}
 func stepEnv(j job, home string) []string {
 	path := os.Getenv("PATH")
 	if path == "" {

deploy/gitbay-runner.override.conf +17 −4

 # repositories never claims a build it is not scoped to, so the canary
 # must be listed or its scheduled build waits forever.
 #
-# -cpus 3 of the host's 4 leaves a core for gitbayd and sshd while a build
+# Resource caps are on the unit, not on the container. Under rootless
-# runs. No -memory: the e2e suite peaks past 5GB of the 7GB, and a cap
+# podman with the cgroupfs manager the container runs inside this
-# that kills it is an outage rather than a limit (#144).
+# service's own cgroup: no child cgroup is created, so podman's --cpus
+# and --memory are accepted and never applied (#188). MemoryMax and
+# CPUQuota below bound the runner and every build together, which is
+# what keeps the forge alive when a build allocates without bound.
+#
+# 6G of the host's 7.7GB, no swap: the e2e suite peaks past 5GB, so the
+# cap sits above that rather than at a fair share. CPUQuota=300% is
+# three of the four cores, leaving one for gitbayd and sshd (#144, #184).
+# OOMPolicy=continue: systemd's default stops the whole service when any
+# process in it is OOM-killed, which would end the runner mid-build; the
+# build fails and the runner carries on.
+MemoryMax=6G
+CPUQuota=300%
+OOMPolicy=continue
 #
 # ExecStart is overridden here rather than left in the unit so the flags
 # and the sandboxing that has to match them live in one file: -isolation
 # when it exits is killed.
 KillMode=mixed
 ExecStart=
-ExecStart=/usr/local/bin/gitbay-runner -remote git@127.0.0.1 -workdir /var/lib/gitbay-runner/work -poll 5s -timeout 45m -repos krz/gitbay,cmc/ci-smoke -isolation podman -image localhost/gitbay-ci:1 -cpus 3
+ExecStart=/usr/local/bin/gitbay-runner -remote git@127.0.0.1 -workdir /var/lib/gitbay-runner/work -poll 5s -timeout 45m -repos krz/gitbay,cmc/ci-smoke -isolation podman -image localhost/gitbay-ci:1
 Nice=10
 CPUWeight=30
 IOWeight=30