Commit `89708196e4`

89708196e47e2f04123bb04a052dbea733619647

parent: 18da8c33b3

Verified · cmc

cmc <hello@cleberg.net> · 2026-09-29 03:37 UTC

runner: builds run under a cgroup per trust class

Ref #260

Layout: unified · split

cmd/gitbay-runner/cgroup.go +14

	@@ -17,6 +17,20 @@ import (
17	// process for that build from inside it with podman's own cgroup handling	17	// process for that build from inside it with podman's own cgroup handling
18	// off. What follows is the portable half: parsing and the file writes.	18	// off. What follows is the portable half: parsing and the file writes.
19		19
		20	// buildClasses are the cgroups a build is placed under, by the claim's
		21	// trust flag. deploy/gitbay-runner-builds.nft matches a build's sockets,
		22	// pasta's included, by these two cgroups (#260), so the names are fixed.
		23	var buildClasses = []string{"trusted", "untrusted"}
		24
		25	// buildCgroupDir is build id's cgroup under the runner's builds cgroup.
		26	func buildCgroupDir(builds string, id int64, trusted bool) string {
		27	class := buildClasses[1]
		28	if trusted {
		29	class = buildClasses[0]
		30	}
		31	return filepath.Join(builds, class, fmt.Sprintf("build-%d", id))
		32	}
		33
20	// memoryBytes parses podman's memory units — a whole number with an	34	// memoryBytes parses podman's memory units — a whole number with an
21	// optional b, k, m or g suffix — into bytes.	35	// optional b, k, m or g suffix — into bytes.
22	func memoryBytes(s string) (int64, error) {	36	func memoryBytes(s string) (int64, error) {

cmd/gitbay-runner/cgroup_linux.go +21 −6

 // Delegate=yes hands the runner its service cgroup; the runner parks
 // itself in a leaf so the service cgroup can enable controllers for
 // children (a cgroup may hold processes or controller-enabled children,
-// not both), and creates one child per build under builds/.
+// not both), and creates one child per build under builds/trusted or
+// builds/untrusted.
 type buildCgroups struct {
 	builds string // <service cgroup>/builds
 }
 const cgroupControllers = "+cpu +memory +pids"
 // prepareBuildCgroups moves the runner into <own>/runner, enables the
-// controllers on its original cgroup, and creates builds/. It fails
+// controllers on its original cgroup, and creates builds/ with a child
+// per trust class. The unit's drop-in may have created those before the
+// runner started, to load the builds nftables table against them; they
+// are used as found, never recreated, since the table holds their ids.
+// It fails
 // where the cgroup is not writable, which is a unit without
 // Delegate=yes; the caller decides whether that is fatal.
 func prepareBuildCgroups() (*buildCgroups, error) {
 	if err := os.WriteFile(filepath.Join(builds, "cgroup.subtree_control"), []byte(cgroupControllers), 0o644); err != nil {
 		return nil, fmt.Errorf("enabling controllers on %s: %w", builds, err)
 	}
+	for _, class := range buildClasses {
+		dir := filepath.Join(builds, class)
+		if err := os.MkdirAll(dir, 0o755); err != nil {
+			return nil, err
+		}
+		if err := os.WriteFile(filepath.Join(dir, "cgroup.subtree_control"), []byte(cgroupControllers), 0o644); err != nil {
+			return nil, fmt.Errorf("enabling controllers on %s: %w", dir, err)
+		}
+	}
 	return &buildCgroups{builds: builds}, nil
 }
-// create makes the cgroup for one build with its limits written, and
+// create makes the cgroup for one build under its trust class with its
-// returns its path and an open directory fd for placing processes.
+// limits written, and returns its path and an open directory fd for
-func (c *buildCgroups) create(id int64, memory, cpus string) (string, *os.File, error) {
+// placing processes.
-	dir := filepath.Join(c.builds, fmt.Sprintf("build-%d", id))
+func (c *buildCgroups) create(id int64, trusted bool, memory, cpus string) (string, *os.File, error) {
+	dir := buildCgroupDir(c.builds, id, trusted)
 	if err := os.Mkdir(dir, 0o755); err != nil {
 		return "", nil, err
 	}

cmd/gitbay-runner/cgroup_other.go +1 −1

	@@ -14,7 +14,7 @@ func prepareBuildCgroups() (*buildCgroups, error) {
14	return nil, errors.New("build cgroups need Linux")	14	return nil, errors.New("build cgroups need Linux")
15	}	15	}
16		16
17	func (c buildCgroups) create(id int64, memory, cpus string) (string, os.File, error) {	17	func (c buildCgroups) create(id int64, trusted bool, memory, cpus string) (string, os.File, error) {
18	return "", nil, errors.New("build cgroups need Linux")	18	return "", nil, errors.New("build cgroups need Linux")
19	}	19	}
20		20

cmd/gitbay-runner/cgroup_test.go +44

	@@ -1,8 +1,10 @@
1	package main	1	package main
2		2
3	import (	3	import (
		4	"fmt"
4	"os"	5	"os"
5	"path/filepath"	6	"path/filepath"
		7	"strings"
6	"testing"	8	"testing"
7	)	9	)
8		10
	@@ -90,3 +92,45 @@ func TestWriteLimits(t *testing.T) {
90	t.Error("a bad memory limit was accepted")	92	t.Error("a bad memory limit was accepted")
91	}	93	}
92	}	94	}
		95
		96	// A build's cgroup sits under its trust class, which is what the builds
		97	// nftables table matches on (#260).
		98	func TestBuildCgroupDir(t *testing.T) {
		99	builds := "/sys/fs/cgroup/system.slice/gitbay-runner.service/builds"
		100	if got := buildCgroupDir(builds, 7, true); got != builds+"/trusted/build-7" {
		101	t.Errorf("trusted: %s", got)
		102	}
		103	if got := buildCgroupDir(builds, 8, false); got != builds+"/untrusted/build-8" {
		104	t.Errorf("untrusted: %s", got)
		105	}
		106	}
		107
		108	// The builds table and the drop-in that creates its cgroups and loads it
		109	// name the same class cgroups the runner places builds in. A rename on
		110	// one side alone would leave builds unmatched, with only the uid table
		111	// between them and the host.
		112	func TestBuildsTableNamesTheClassCgroups(t *testing.T) {
		113	read := func(name string) string {
		114	t.Helper()
		115	b, err := os.ReadFile(filepath.Join("..", "..", "deploy", name))
		116	if err != nil {
		117	t.Fatal(err)
		118	}
		119	return string(b)
		120	}
		121	const unit = "system.slice/gitbay-runner.service"
		122	table, dropin := read("gitbay-runner-builds.nft"), read("gitbay-runner.override.conf")
		123	for _, class := range buildClasses {
		124	match := fmt.Sprintf(`socket cgroupv2 level 4 "%s/builds/%s" jump %s`, unit, class, class)
		125	if !strings.Contains(table, match) {
		126	t.Errorf("gitbay-runner-builds.nft lacks %q", match)
		127	}
		128	dir := "/sys/fs/cgroup/" + unit + "/builds/" + class
		129	if !strings.Contains(dropin, dir) {
		130	t.Errorf("the drop-in does not create %s", dir)
		131	}
		132	}
		133	if !strings.Contains(dropin, "ExecStartPre=+/usr/sbin/nft -f /etc/gitbay-runner/builds.nft") {
		134	t.Error("the drop-in does not load the builds table")
		135	}
		136	}

cmd/gitbay-runner/isolate.go +1 −1

	@@ -134,7 +134,7 @@ func (r *runner) runStepsPodman(j job, dir string, env []string, sink io.Writer,
134	// from the runner's cgroup would run the step outside the limit.	134	// from the runner's cgroup would run the step outside the limit.
135	var cgroupFD *os.File	135	var cgroupFD *os.File
136	if r.cgroups != nil {	136	if r.cgroups != nil {
137	dir, f, err := r.cgroups.create(j.ID, r.memory, r.cpus)	137	dir, f, err := r.cgroups.create(j.ID, j.Trusted, r.memory, r.cpus)
138	if err != nil {	138	if err != nil {
139	fmt.Fprintf(sink, "preparing the build cgroup: %v\n", err)	139	fmt.Fprintf(sink, "preparing the build cgroup: %v\n", err)
140	return &failure{Reason: "preparing the build cgroup failed"}	140	return &failure{Reason: "preparing the build cgroup failed"}

cmd/gitbay-runner/main.go +3 −2

 // polls from; the SSH auth limiter counts failures per source address
 // (#260). podman passes --no-map-gw to pasta by default; it is stated
 // here so the build's view of the host does not depend on that default.
-// The host's nftables table (deploy/gitbay-runner-egress.nft) limits
+// The host's nftables tables (deploy/gitbay-runner-egress.nft and
-// what a build reaches on the host to 22, 80 and 443.
+// gitbay-runner-builds.nft) limit what a build reaches on the host to
+// 22, 80 and 443, and an untrusted build to nothing but DNS.
 func (r *runner) buildNetwork() []string {
 	if !r.loopbackRemote() {
 		return nil

Commit 89708196e4

Commit `89708196e4`