Commit `89708196e4`

89708196e47e2f04123bb04a052dbea733619647

parent: 18da8c33b3

Verified · cmc

cmc <hello@cleberg.net> · 2026-09-29 03:37 UTC

runner: builds run under a cgroup per trust class

Ref #260

Layout: unified · split

cmd/gitbay-runner/cgroup.go +14

		@@ -17,6 +17,20 @@ import (
17	17	// process for that build from inside it with podman's own cgroup handling
18	18	// off. What follows is the portable half: parsing and the file writes.
19	19
	20	// buildClasses are the cgroups a build is placed under, by the claim's
	21	// trust flag. deploy/gitbay-runner-builds.nft matches a build's sockets,
	22	// pasta's included, by these two cgroups (#260), so the names are fixed.
	23	var buildClasses = []string{"trusted", "untrusted"}
	24
	25	// buildCgroupDir is build id's cgroup under the runner's builds cgroup.
	26	func buildCgroupDir(builds string, id int64, trusted bool) string {
	27	class := buildClasses[1]
	28	if trusted {
	29	class = buildClasses[0]
	30	}
	31	return filepath.Join(builds, class, fmt.Sprintf("build-%d", id))
	32	}
	33
20	34	// memoryBytes parses podman's memory units — a whole number with an
21	35	// optional b, k, m or g suffix — into bytes.
22	36	func memoryBytes(s string) (int64, error) {

cmd/gitbay-runner/cgroup_linux.go +21 −6

		@@ -17,7 +17,8 @@ import (
17	17	// Delegate=yes hands the runner its service cgroup; the runner parks
18	18	// itself in a leaf so the service cgroup can enable controllers for
19	19	// children (a cgroup may hold processes or controller-enabled children,
20		// not both), and creates one child per build under builds/.
	20	// not both), and creates one child per build under builds/trusted or
	21	// builds/untrusted.
21	22	type buildCgroups struct {
22	23	builds string // <service cgroup>/builds
23	24	}
		@@ -25,7 +26,11 @@ type buildCgroups struct {
25	26	const cgroupControllers = "+cpu +memory +pids"
26	27
27	28	// prepareBuildCgroups moves the runner into <own>/runner, enables the
28		// controllers on its original cgroup, and creates builds/. It fails
	29	// controllers on its original cgroup, and creates builds/ with a child
	30	// per trust class. The unit's drop-in may have created those before the
	31	// runner started, to load the builds nftables table against them; they
	32	// are used as found, never recreated, since the table holds their ids.
	33	// It fails
29	34	// where the cgroup is not writable, which is a unit without
30	35	// Delegate=yes; the caller decides whether that is fatal.
31	36	func prepareBuildCgroups() (*buildCgroups, error) {
		@@ -55,13 +60,23 @@ func prepareBuildCgroups() (*buildCgroups, error) {
55	60	if err := os.WriteFile(filepath.Join(builds, "cgroup.subtree_control"), []byte(cgroupControllers), 0o644); err != nil {
56	61	return nil, fmt.Errorf("enabling controllers on %s: %w", builds, err)
57	62	}
	63	for _, class := range buildClasses {
	64	dir := filepath.Join(builds, class)
	65	if err := os.MkdirAll(dir, 0o755); err != nil {
	66	return nil, err
	67	}
	68	if err := os.WriteFile(filepath.Join(dir, "cgroup.subtree_control"), []byte(cgroupControllers), 0o644); err != nil {
	69	return nil, fmt.Errorf("enabling controllers on %s: %w", dir, err)
	70	}
	71	}
58	72	return &buildCgroups{builds: builds}, nil
59	73	}
60	74
61		// create makes the cgroup for one build with its limits written, and
62		// returns its path and an open directory fd for placing processes.
63		func (c buildCgroups) create(id int64, memory, cpus string) (string, os.File, error) {
64		dir := filepath.Join(c.builds, fmt.Sprintf("build-%d", id))
	75	// create makes the cgroup for one build under its trust class with its
	76	// limits written, and returns its path and an open directory fd for
	77	// placing processes.
	78	func (c buildCgroups) create(id int64, trusted bool, memory, cpus string) (string, os.File, error) {
	79	dir := buildCgroupDir(c.builds, id, trusted)
65	80	if err := os.Mkdir(dir, 0o755); err != nil {
66	81	return "", nil, err
67	82	}

cmd/gitbay-runner/cgroup_other.go +1 −1

		@@ -14,7 +14,7 @@ func prepareBuildCgroups() (*buildCgroups, error) {
14	14	return nil, errors.New("build cgroups need Linux")
15	15	}
16	16
17		func (c buildCgroups) create(id int64, memory, cpus string) (string, os.File, error) {
	17	func (c buildCgroups) create(id int64, trusted bool, memory, cpus string) (string, os.File, error) {
18	18	return "", nil, errors.New("build cgroups need Linux")
19	19	}
20	20

cmd/gitbay-runner/cgroup_test.go +44

		@@ -1,8 +1,10 @@
1	1	package main
2	2
3	3	import (
	4	"fmt"
4	5	"os"
5	6	"path/filepath"
	7	"strings"
6	8	"testing"
7	9	)
8	10
		@@ -90,3 +92,45 @@ func TestWriteLimits(t *testing.T) {
90	92	t.Error("a bad memory limit was accepted")
91	93	}
92	94	}
	95
	96	// A build's cgroup sits under its trust class, which is what the builds
	97	// nftables table matches on (#260).
	98	func TestBuildCgroupDir(t *testing.T) {
	99	builds := "/sys/fs/cgroup/system.slice/gitbay-runner.service/builds"
	100	if got := buildCgroupDir(builds, 7, true); got != builds+"/trusted/build-7" {
	101	t.Errorf("trusted: %s", got)
	102	}
	103	if got := buildCgroupDir(builds, 8, false); got != builds+"/untrusted/build-8" {
	104	t.Errorf("untrusted: %s", got)
	105	}
	106	}
	107
	108	// The builds table and the drop-in that creates its cgroups and loads it
	109	// name the same class cgroups the runner places builds in. A rename on
	110	// one side alone would leave builds unmatched, with only the uid table
	111	// between them and the host.
	112	func TestBuildsTableNamesTheClassCgroups(t *testing.T) {
	113	read := func(name string) string {
	114	t.Helper()
	115	b, err := os.ReadFile(filepath.Join("..", "..", "deploy", name))
	116	if err != nil {
	117	t.Fatal(err)
	118	}
	119	return string(b)
	120	}
	121	const unit = "system.slice/gitbay-runner.service"
	122	table, dropin := read("gitbay-runner-builds.nft"), read("gitbay-runner.override.conf")
	123	for _, class := range buildClasses {
	124	match := fmt.Sprintf(`socket cgroupv2 level 4 "%s/builds/%s" jump %s`, unit, class, class)
	125	if !strings.Contains(table, match) {
	126	t.Errorf("gitbay-runner-builds.nft lacks %q", match)
	127	}
	128	dir := "/sys/fs/cgroup/" + unit + "/builds/" + class
	129	if !strings.Contains(dropin, dir) {
	130	t.Errorf("the drop-in does not create %s", dir)
	131	}
	132	}
	133	if !strings.Contains(dropin, "ExecStartPre=+/usr/sbin/nft -f /etc/gitbay-runner/builds.nft") {
	134	t.Error("the drop-in does not load the builds table")
	135	}
	136	}

cmd/gitbay-runner/isolate.go +1 −1

		@@ -134,7 +134,7 @@ func (r *runner) runStepsPodman(j job, dir string, env []string, sink io.Writer,
134	134	// from the runner's cgroup would run the step outside the limit.
135	135	var cgroupFD *os.File
136	136	if r.cgroups != nil {
137		dir, f, err := r.cgroups.create(j.ID, r.memory, r.cpus)
	137	dir, f, err := r.cgroups.create(j.ID, j.Trusted, r.memory, r.cpus)
138	138	if err != nil {
139	139	fmt.Fprintf(sink, "preparing the build cgroup: %v\n", err)
140	140	return &failure{Reason: "preparing the build cgroup failed"}

cmd/gitbay-runner/main.go +3 −2

		@@ -557,8 +557,9 @@ func (r *runner) buildSSH(public string) string {
557	557	// polls from; the SSH auth limiter counts failures per source address
558	558	// (#260). podman passes --no-map-gw to pasta by default; it is stated
559	559	// here so the build's view of the host does not depend on that default.
560		// The host's nftables table (deploy/gitbay-runner-egress.nft) limits
561		// what a build reaches on the host to 22, 80 and 443.
	560	// The host's nftables tables (deploy/gitbay-runner-egress.nft and
	561	// gitbay-runner-builds.nft) limit what a build reaches on the host to
	562	// 22, 80 and 443, and an untrusted build to nothing but DNS.
562	563	func (r *runner) buildNetwork() []string {
563	564	if !r.loopbackRemote() {
564	565	return nil

Commit 89708196e4

Commit `89708196e4`