krz/gitbay

runner: the runner owns a cgroup per build, so -memory and -cpus apply !334

merged merged by cmc on 2026-09-07 20:58 UTC · krz/gitbay:build-cgroups-188 into main

10 files changed, +394 −60

Layout: unified · split

.gitbay/wiki/Admin.org +15 −11
@@ -448,17 +448,21 @@ because an image this host does not have would fail every build. A job
448overrides it with =image:= in =.gitbay/ci.yml=, validated as a reference 448overrides it with =image:= in =.gitbay/ci.yml=, validated as a reference
449so a config file cannot turn it into podman arguments. 449so a config file cannot turn it into podman arguments.
450 450
451=-cpus= and =-memory= cap one build's container (podman's own units, 451=-cpus= and =-memory= cap one build (podman's units, e.g. =-cpus 2
452e.g. =-cpus 2 -memory 4g=); unset means uncapped. They only take effect 452-memory 4g=); unset means uncapped. The runner applies them itself: it
453where podman can create a cgroup for the container. Under rootless 453creates a cgroup per build under its own delegated service cgroup,
454podman with the cgroupfs manager, which is what a system service gets, 454writes the limits there, and starts every podman process for the build
455it cannot: the container runs inside the service's own cgroup and both 455inside it, with podman's cgroup handling off. Podman's own =--memory=
456flags are accepted and ignored (krz/gitbay#188). Cap the unit instead. 456and =--cpus= never applied under rootless cgroupfs, which is what a
457bay1's drop-in sets =MemoryMax=6G= and =CPUQuota=300%= on a 7.7GB 457system service gets (krz/gitbay#188). The unit therefore needs
458four-core host with no swap: the memory cap is what keeps the forge 458=Delegate=yes=, which the drop-in sets; without it the runner refuses
459alive when a build allocates without bound, and it sits above the e2e 459to start when a limit is set, and logs that builds run unconfined when
460suite's 5GB peak rather than at a fair share. =OOMPolicy=continue= keeps 460none is. bay1 runs =-cpus 3 -memory 6g= per build inside =MemoryMax=6G=
461systemd from stopping the runner when a build is OOM-killed. 461and =CPUQuota=300%= on the unit, on a 7.7GB four-core host with no
462swap: the memory cap is what keeps the forge alive when a build
463allocates without bound, and it sits above the e2e suite's 5GB peak
464rather than at a fair share. =OOMPolicy=continue= keeps systemd from
465stopping the runner when a build is OOM-killed.
462 466
463Each repository gets its own build home under the runner's workdir, 467Each repository gets its own build home under the runner's workdir,
464mounted into its containers as =HOME=. Caches persist between builds of 468mounted into its containers as =HOME=. Caches persist between builds of
cmd/gitbay-runner/cgroup.go added +87
@@ -0,0 +1,87 @@
1package main
2
3import (
4 "fmt"
5 "os"
6 "path/filepath"
7 "strconv"
8 "strings"
9)
10
11// Build limits are cgroup v2 files the runner writes itself. Podman's
12// --memory and --cpus never applied here: under rootless podman with the
13// cgroupfs manager the container starts inside the service's own cgroup
14// and crun cannot create a child, so the flags were accepted and ignored
15// (#188). The runner owns a cgroup per build under its delegated service
16// cgroup instead, writes the limits into it, and starts every podman
17// process for that build from inside it with podman's own cgroup handling
18// off. What follows is the portable half: parsing and the file writes.
19
20// memoryBytes parses podman's memory units — a whole number with an
21// optional b, k, m or g suffix — into bytes.
22func memoryBytes(s string) (int64, error) {
23 if s == "" {
24 return 0, fmt.Errorf("empty memory limit")
25 }
26 num, unit := s, ""
27 if last := s[len(s)-1]; last < '0' || last > '9' {
28 num, unit = s[:len(s)-1], strings.ToLower(s[len(s)-1:])
29 }
30 n, err := strconv.ParseInt(num, 10, 64)
31 if err != nil || n <= 0 {
32 return 0, fmt.Errorf("memory limit %q: want a whole number of b, k, m or g", s)
33 }
34 shift := map[string]uint{"": 0, "b": 0, "k": 10, "m": 20, "g": 30}
35 sh, ok := shift[unit]
36 if !ok {
37 return 0, fmt.Errorf("memory limit %q: unit %q is not b, k, m or g", s, unit)
38 }
39 return n << sh, nil
40}
41
42// cpuMax renders a CPU count, whole or fractional, as cgroup v2's
43// "<quota> <period>" over a 100ms period.
44func cpuMax(s string) (string, error) {
45 const period = 100000
46 f, err := strconv.ParseFloat(s, 64)
47 if err != nil || f <= 0 {
48 return "", fmt.Errorf("cpu limit %q: want a positive number of CPUs", s)
49 }
50 return fmt.Sprintf("%d %d", int64(f*period+0.5), period), nil
51}
52
53// ownCgroupPath reads the cgroup v2 path out of /proc/self/cgroup
54// contents. A v1 hierarchy has more than the one "0::" line and is not
55// something the runner manages.
56func ownCgroupPath(procSelfCgroup string) (string, error) {
57 lines := strings.Split(strings.TrimSpace(procSelfCgroup), "\n")
58 if len(lines) != 1 || !strings.HasPrefix(lines[0], "0::/") {
59 return "", fmt.Errorf("not a cgroup v2 host: /proc/self/cgroup is %q", strings.TrimSpace(procSelfCgroup))
60 }
61 return strings.TrimPrefix(lines[0], "0::"), nil
62}
63
64// writeLimits writes the requested limits into a cgroup directory. An
65// unset limit writes nothing, so the build inherits whatever the unit
66// allows rather than getting "max".
67func writeLimits(dir, memory, cpus string) error {
68 if memory != "" {
69 n, err := memoryBytes(memory)
70 if err != nil {
71 return err
72 }
73 if err := os.WriteFile(filepath.Join(dir, "memory.max"), []byte(strconv.FormatInt(n, 10)), 0o644); err != nil {
74 return err
75 }
76 }
77 if cpus != "" {
78 v, err := cpuMax(cpus)
79 if err != nil {
80 return err
81 }
82 if err := os.WriteFile(filepath.Join(dir, "cpu.max"), []byte(v), 0o644); err != nil {
83 return err
84 }
85 }
86 return nil
87}
cmd/gitbay-runner/cgroup_linux.go added +109
@@ -0,0 +1,109 @@
1//go:build linux
2
3package main
4
5import (
6 "fmt"
7 "log"
8 "os"
9 "os/exec"
10 "path/filepath"
11 "strconv"
12 "syscall"
13 "time"
14)
15
16// buildCgroups is the runner's own cgroup subtree for builds. The unit's
17// Delegate=yes hands the runner its service cgroup; the runner parks
18// itself in a leaf so the service cgroup can enable controllers for
19// children (a cgroup may hold processes or controller-enabled children,
20// not both), and creates one child per build under builds/.
21type buildCgroups struct {
22 builds string // <service cgroup>/builds
23}
24
25const cgroupControllers = "+cpu +memory +pids"
26
27// prepareBuildCgroups moves the runner into <own>/runner, enables the
28// controllers on its original cgroup, and creates builds/. It fails
29// where the cgroup is not writable, which is a unit without
30// Delegate=yes; the caller decides whether that is fatal.
31func prepareBuildCgroups() (*buildCgroups, error) {
32 raw, err := os.ReadFile("/proc/self/cgroup")
33 if err != nil {
34 return nil, err
35 }
36 own, err := ownCgroupPath(string(raw))
37 if err != nil {
38 return nil, err
39 }
40 root := filepath.Join("/sys/fs/cgroup", own)
41 leaf := filepath.Join(root, "runner")
42 if err := os.MkdirAll(leaf, 0o755); err != nil {
43 return nil, fmt.Errorf("%s is not writable; the unit needs Delegate=yes: %w", root, err)
44 }
45 if err := os.WriteFile(filepath.Join(leaf, "cgroup.procs"), []byte(strconv.Itoa(os.Getpid())), 0o644); err != nil {
46 return nil, fmt.Errorf("moving into %s: %w", leaf, err)
47 }
48 if err := os.WriteFile(filepath.Join(root, "cgroup.subtree_control"), []byte(cgroupControllers), 0o644); err != nil {
49 return nil, fmt.Errorf("enabling controllers on %s: %w", root, err)
50 }
51 builds := filepath.Join(root, "builds")
52 if err := os.MkdirAll(builds, 0o755); err != nil {
53 return nil, err
54 }
55 if err := os.WriteFile(filepath.Join(builds, "cgroup.subtree_control"), []byte(cgroupControllers), 0o644); err != nil {
56 return nil, fmt.Errorf("enabling controllers on %s: %w", builds, err)
57 }
58 return &buildCgroups{builds: builds}, nil
59}
60
61// create makes the cgroup for one build with its limits written, and
62// returns its path and an open directory fd for placing processes.
63func (c *buildCgroups) create(id int64, memory, cpus string) (string, *os.File, error) {
64 dir := filepath.Join(c.builds, fmt.Sprintf("build-%d", id))
65 if err := os.Mkdir(dir, 0o755); err != nil {
66 return "", nil, err
67 }
68 if err := writeLimits(dir, memory, cpus); err != nil {
69 os.Remove(dir)
70 return "", nil, err
71 }
72 f, err := os.Open(dir)
73 if err != nil {
74 os.Remove(dir)
75 return "", nil, err
76 }
77 return dir, f, nil
78}
79
80// remove kills whatever is still in the build's cgroup — conmon, the
81// pause process, a step's stray child — and removes it. rmdir fails
82// until the kernel has reaped every process, so it retries briefly.
83func (c *buildCgroups) remove(dir string) {
84 if err := os.WriteFile(filepath.Join(dir, "cgroup.kill"), []byte("1"), 0o644); err != nil {
85 log.Printf("cgroup %s: kill: %v", dir, err)
86 }
87 for i := 0; i < 50; i++ {
88 if err := os.Remove(dir); err == nil {
89 return
90 }
91 time.Sleep(100 * time.Millisecond)
92 }
93 log.Printf("cgroup %s: still populated after kill; left in place", dir)
94}
95
96// intoCgroup starts cmd inside the cgroup fd refers to, so podman,
97// conmon and everything they start inherit the build's limits. A podman
98// exec started from the runner's own cgroup lands there, outside the
99// limit, which is why every invocation for a build goes through this.
100func intoCgroup(cmd *exec.Cmd, f *os.File) {
101 if f == nil {
102 return
103 }
104 if cmd.SysProcAttr == nil {
105 cmd.SysProcAttr = &syscall.SysProcAttr{}
106 }
107 cmd.SysProcAttr.UseCgroupFD = true
108 cmd.SysProcAttr.CgroupFD = int(f.Fd())
109}
cmd/gitbay-runner/cgroup_other.go added +23
@@ -0,0 +1,23 @@
1//go:build !linux
2
3package main
4
5import (
6 "errors"
7 "os"
8 "os/exec"
9)
10
11type buildCgroups struct{}
12
13func prepareBuildCgroups() (*buildCgroups, error) {
14 return nil, errors.New("build cgroups need Linux")
15}
16
17func (c *buildCgroups) create(id int64, memory, cpus string) (string, *os.File, error) {
18 return "", nil, errors.New("build cgroups need Linux")
19}
20
21func (c *buildCgroups) remove(dir string) {}
22
23func intoCgroup(cmd *exec.Cmd, f *os.File) {}
cmd/gitbay-runner/cgroup_test.go added +92
@@ -0,0 +1,92 @@
1package main
2
3import (
4 "os"
5 "path/filepath"
6 "testing"
7)
8
9// Podman's --memory and --cpus never applied under rootless cgroupfs: the
10// container ran in the service's own cgroup and crun could not create a
11// child (#188). The runner now owns the build cgroups itself and places
12// podman inside one, so the limits are written by the runner in cgroup
13// v2's own units.
14func TestMemoryBytes(t *testing.T) {
15 cases := map[string]int64{
16 "64m": 64 << 20,
17 "6g": 6 << 30,
18 "512k": 512 << 10,
19 "100b": 100,
20 "4096": 4096,
21 "1G": 1 << 30,
22 }
23 for in, want := range cases {
24 got, err := memoryBytes(in)
25 if err != nil || got != want {
26 t.Errorf("memoryBytes(%q) = %d, %v; want %d", in, got, err, want)
27 }
28 }
29 for _, bad := range []string{"", "lots", "6gb", "-1g", "1.5g"} {
30 if _, err := memoryBytes(bad); err == nil {
31 t.Errorf("memoryBytes(%q) accepted", bad)
32 }
33 }
34}
35
36func TestCPUMax(t *testing.T) {
37 cases := map[string]string{
38 "3": "300000 100000",
39 "1": "100000 100000",
40 "1.5": "150000 100000",
41 "0.25": "25000 100000",
42 }
43 for in, want := range cases {
44 got, err := cpuMax(in)
45 if err != nil || got != want {
46 t.Errorf("cpuMax(%q) = %q, %v; want %q", in, got, err, want)
47 }
48 }
49 for _, bad := range []string{"", "0", "-1", "two"} {
50 if _, err := cpuMax(bad); err == nil {
51 t.Errorf("cpuMax(%q) accepted", bad)
52 }
53 }
54}
55
56// /proc/self/cgroup on cgroup v2 is one line, "0::<path>".
57func TestOwnCgroupPath(t *testing.T) {
58 got, err := ownCgroupPath("0::/system.slice/gitbay-runner.service\n")
59 if err != nil || got != "/system.slice/gitbay-runner.service" {
60 t.Fatalf("ownCgroupPath = %q, %v", got, err)
61 }
62 // A v1 hierarchy, or anything else, is not something the runner
63 // manages.
64 if _, err := ownCgroupPath("12:memory:/user.slice\n0::/init.scope\n"); err == nil {
65 t.Fatal("v1 hierarchy accepted")
66 }
67}
68
69// Limits land as cgroup v2 interface files in the build's directory;
70// an unset limit writes nothing, which means "inherit", not "max".
71func TestWriteLimits(t *testing.T) {
72 dir := t.TempDir()
73 if err := writeLimits(dir, "6g", "3"); err != nil {
74 t.Fatal(err)
75 }
76 if got, _ := os.ReadFile(filepath.Join(dir, "memory.max")); string(got) != "6442450944" {
77 t.Errorf("memory.max = %q", got)
78 }
79 if got, _ := os.ReadFile(filepath.Join(dir, "cpu.max")); string(got) != "300000 100000" {
80 t.Errorf("cpu.max = %q", got)
81 }
82 empty := t.TempDir()
83 if err := writeLimits(empty, "", ""); err != nil {
84 t.Fatal(err)
85 }
86 if entries, _ := os.ReadDir(empty); len(entries) != 0 {
87 t.Errorf("unset limits wrote %v", entries)
88 }
89 if err := writeLimits(t.TempDir(), "lots", ""); err == nil {
90 t.Error("a bad memory limit was accepted")
91 }
92}
cmd/gitbay-runner/env_test.go −13
@@ -118,19 +118,6 @@ func TestEnvHomeFindsHome(t *testing.T) {
118 } 118 }
119} 119}
120 120
121// A limit is passed to podman only when set; unset means uncapped, not a
122// default that could kill the suite.
123func TestLimitArgs(t *testing.T) {
124 if got := (&runner{}).limitArgs(); len(got) != 0 {
125 t.Errorf("no limits set, got %v", got)
126 }
127 got := (&runner{memory: "4g", cpus: "2"}).limitArgs()
128 want := []string{"--memory", "4g", "--cpus", "2"}
129 if strings.Join(got, " ") != strings.Join(want, " ") {
130 t.Errorf("limitArgs = %v, want %v", got, want)
131 }
132}
133
134// Closing stop drains: the build in flight finishes and is reported, and 121// Closing stop drains: the build in flight finishes and is reported, and
135// no further build is claimed (#179). 122// no further build is claimed (#179).
136func TestServeDrainsOnStop(t *testing.T) { 123func TestServeDrainsOnStop(t *testing.T) {
cmd/gitbay-runner/isolate.go +19 −19
@@ -122,11 +122,26 @@ func (r *runner) runStepsPodman(j job, dir string, env []string, sink io.Writer,
122 } 122 }
123 defer os.Remove(envFile) 123 defer os.Remove(envFile)
124 124
125 // The build's cgroup carries its limits; podman's own cgroup handling
126 // is off because it never worked here (#188). Every podman process
127 // for this build starts inside the cgroup, exec included: one started
128 // from the runner's cgroup would run the step outside the limit.
129 var cgroupFD *os.File
130 if r.cgroups != nil {
131 dir, f, err := r.cgroups.create(j.ID, r.memory, r.cpus)
132 if err != nil {
133 fmt.Fprintf(sink, "preparing the build cgroup: %v\n", err)
134 return false
135 }
136 cgroupFD = f
137 defer f.Close()
138 defer r.cgroups.remove(dir)
139 }
140
125 name := fmt.Sprintf("gitbay-build-%d", j.ID) 141 name := fmt.Sprintf("gitbay-build-%d", j.ID)
126 // --rm so a container cannot outlive its build; the explicit rm below 142 // --rm so a container cannot outlive its build; the explicit rm below
127 // covers the case where the daemon-less run itself fails. 143 // covers the case where the daemon-less run itself fails.
128 args := append(r.podmanGlobal(), "run", "--detach", "--rm", "--pull=never") 144 args := append(r.podmanGlobal(), "run", "--detach", "--rm", "--pull=never", "--cgroups=disabled")
129 args = append(args, r.limitArgs()...)
130 args = append(args, 145 args = append(args,
131 "--name", name, 146 "--name", name,
132 "--env-file", envFile, 147 "--env-file", envFile,
@@ -142,6 +157,7 @@ func (r *runner) runStepsPodman(j job, dir string, env []string, sink io.Writer,
142 image, "-c", "sleep infinity") 157 image, "-c", "sleep infinity")
143 start := exec.Command(podman, args...) 158 start := exec.Command(podman, args...)
144 start.Env = []string{"PATH=" + os.Getenv("PATH"), "HOME=" + r.podmanHome()} 159 start.Env = []string{"PATH=" + os.Getenv("PATH"), "HOME=" + r.podmanHome()}
160 intoCgroup(start, cgroupFD)
145 if out, err := start.CombinedOutput(); err != nil { 161 if out, err := start.CombinedOutput(); err != nil {
146 // A missing image lands here, and it is the common case worth 162 // A missing image lands here, and it is the common case worth
147 // explaining: this runner never pulls, so an image it does not 163 // explaining: this runner never pulls, so an image it does not
@@ -162,6 +178,7 @@ func (r *runner) runStepsPodman(j job, dir string, env []string, sink io.Writer,
162 fmt.Fprintf(sink, "$ %s\n", step) 178 fmt.Fprintf(sink, "$ %s\n", step)
163 cmd := exec.Command(podman, append(r.podmanGlobal(), "exec", "--workdir", "/workspace", name, "sh", "-c", step)...) 179 cmd := exec.Command(podman, append(r.podmanGlobal(), "exec", "--workdir", "/workspace", name, "sh", "-c", step)...)
164 cmd.Env = []string{"PATH=" + os.Getenv("PATH"), "HOME=" + r.podmanHome()} 180 cmd.Env = []string{"PATH=" + os.Getenv("PATH"), "HOME=" + r.podmanHome()}
181 intoCgroup(cmd, cgroupFD)
165 cmd.Stdout, cmd.Stderr = sink, sink 182 cmd.Stdout, cmd.Stderr = sink, sink
166 if ok, why := runStep(cmd, deadline); !ok { 183 if ok, why := runStep(cmd, deadline); !ok {
167 fmt.Fprintf(sink, "%s\n", why) 184 fmt.Fprintf(sink, "%s\n", why)
@@ -189,23 +206,6 @@ func (r *runner) podmanGlobal() []string {
189 return []string{"--cgroup-manager=cgroupfs"} 206 return []string{"--cgroup-manager=cgroupfs"}
190} 207}
191 208
192// limitArgs caps one build's container. The service's CPUWeight and
193// IOWeight shape the service against other services, not one build
194// against the host, and the threat model lists resource exhaustion as
195// unaddressed. Memory is deliberately uncapped by default: the e2e suite
196// peaks past 5GB on a 7GB host, and a cap that kills the suite is an
197// outage, not a limit.
198func (r *runner) limitArgs() []string {
199 var args []string
200 if r.memory != "" {
201 args = append(args, "--memory", r.memory)
202 }
203 if r.cpus != "" {
204 args = append(args, "--cpus", r.cpus)
205 }
206 return args
207}
208
209// envHome returns the HOME the step environment carries. 209// envHome returns the HOME the step environment carries.
210func envHome(env []string) string { 210func envHome(env []string) string {
211 for _, e := range env { 211 for _, e := range env {
cmd/gitbay-runner/main.go +23 −3
@@ -50,9 +50,12 @@ type runner struct {
50 // isolation selects how steps run: "podman" or "none". 50 // isolation selects how steps run: "podman" or "none".
51 image string 51 image string
52 isolation string 52 isolation string
53 // memory and cpus cap one build's container; empty means no cap. 53 // memory and cpus cap one build's cgroup; empty means no cap.
54 memory string 54 memory string
55 cpus string 55 cpus string
56 // cgroups is the runner's build cgroup subtree, nil where the unit
57 // is not delegated and builds run unconfined in the service cgroup.
58 cgroups *buildCgroups
56 // stepFn is step, replaceable by tests. 59 // stepFn is step, replaceable by tests.
57 stepFn func() (bool, error) 60 stepFn func() (bool, error)
58 // repos limits which repositories this runner claims builds for. Empty 61 // repos limits which repositories this runner claims builds for. Empty
@@ -74,8 +77,8 @@ func main() {
74 jobs = flag.Int("jobs", 1, "builds to run at once") 77 jobs = flag.Int("jobs", 1, "builds to run at once")
75 image = flag.String("image", "", "default container image for jobs that name none") 78 image = flag.String("image", "", "default container image for jobs that name none")
76 isolation = flag.String("isolation", "podman", "how steps run: podman, or none for no container") 79 isolation = flag.String("isolation", "podman", "how steps run: podman, or none for no container")
77 memory = flag.String("memory", "", "memory limit per build container, e.g. 4g (podman only; default unlimited)") 80 memory = flag.String("memory", "", "memory limit per build, e.g. 4g (podman only, needs a delegated cgroup; default unlimited)")
78 cpus = flag.String("cpus", "", "CPU limit per build container, e.g. 2 (podman only; default unlimited)") 81 cpus = flag.String("cpus", "", "CPU limit per build, e.g. 2 (podman only, needs a delegated cgroup; default unlimited)")
79 version = flag.Bool("version", false, "print the commit this binary was built from, then exit") 82 version = flag.Bool("version", false, "print the commit this binary was built from, then exit")
80 ) 83 )
81 flag.Parse() 84 flag.Parse()
@@ -96,6 +99,23 @@ func main() {
96 memory: *memory, 99 memory: *memory,
97 cpus: *cpus, 100 cpus: *cpus,
98 } 101 }
102 if r.isolation == isolationPodman {
103 // Before podman runs anything: its pause process lands in the
104 // cgroup of the first invocation, and that must be the runner's
105 // leaf, not a build's.
106 cg, err := prepareBuildCgroups()
107 switch {
108 case err == nil:
109 r.cgroups = cg
110 case r.memory != "" || r.cpus != "":
111 // Limits that cannot be applied are refused, not dropped:
112 // a runner that accepted -memory and ran uncapped is what
113 // #188 was.
114 log.Fatalf("-memory/-cpus: build cgroups unavailable: %v", err)
115 default:
116 log.Printf("build cgroups unavailable (%v); builds run unconfined in the service cgroup", err)
117 }
118 }
99 if err := r.checkIsolation(); err != nil { 119 if err := r.checkIsolation(); err != nil {
100 // Refusing to start is the point. A runner that quietly fell back 120 // Refusing to start is the point. A runner that quietly fell back
101 // to running repository code on the host would drop isolation 121 // to running repository code on the host would drop isolation
cmd/gitbay-runner/proc_unix.go +4 −1
@@ -13,7 +13,10 @@ import (
13// child: dash forks a single command rather than exec'ing it, so on a 13// child: dash forks a single command rather than exec'ing it, so on a
14// Debian host "sh -c 'sleep 120'" survived its shell by two minutes. 14// Debian host "sh -c 'sleep 120'" survived its shell by two minutes.
15func ownProcessGroup(cmd *exec.Cmd) { 15func ownProcessGroup(cmd *exec.Cmd) {
16 cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true} 16 if cmd.SysProcAttr == nil {
17 cmd.SysProcAttr = &syscall.SysProcAttr{}
18 }
19 cmd.SysProcAttr.Setpgid = true
17} 20}
18 21
19func killTree(cmd *exec.Cmd) { 22func killTree(cmd *exec.Cmd) {
deploy/gitbay-runner.override.conf +22 −13
@@ -27,19 +27,20 @@
27# repositories never claims a build it is not scoped to, so the canary 27# repositories never claims a build it is not scoped to, so the canary
28# must be listed or its scheduled build waits forever. 28# must be listed or its scheduled build waits forever.
29# 29#
30# Resource caps are on the unit, not on the container. Under rootless 30# Two layers of resource caps. MemoryMax and CPUQuota bound the unit —
31# podman with the cgroupfs manager the container runs inside this 31# the runner and every build together — which is what keeps the forge
32# service's own cgroup: no child cgroup is created, so podman's --cpus 32# alive when a build allocates without bound. -memory and -cpus on
33# and --memory are accepted and never applied (#188). MemoryMax and 33# ExecStart cap each build's own cgroup, which the runner creates under
34# CPUQuota below bound the runner and every build together, which is 34# this unit's delegated cgroup (Delegate=yes below); podman's own --memory
35# what keeps the forge alive when a build allocates without bound. 35# and --cpus never applied here, since under rootless cgroupfs the
36# container ran in the service cgroup itself (#188).
36# 37#
37# 6G of the host's 7.7GB, no swap: the e2e suite peaks past 5GB, so the 38# 6G of the host's 7.7GB, no swap: the e2e suite peaks past 5GB, so the
38# cap sits above that rather than at a fair share. CPUQuota=300% is 39# cap sits above that rather than at a fair share. CPUQuota=300% and
39# three of the four cores, leaving one for gitbayd and sshd (#144, #184). 40# -cpus 3 are three of the four cores, leaving one for gitbayd and sshd
40# OOMPolicy=continue: systemd's default stops the whole service when any 41# (#144, #184). OOMPolicy=continue: systemd's default stops the whole
41# process in it is OOM-killed, which would end the runner mid-build; the 42# service when any process in it is OOM-killed, which would end the
42# build fails and the runner carries on. 43# runner mid-build; the build fails and the runner carries on.
43MemoryMax=6G 44MemoryMax=6G
44CPUQuota=300% 45CPUQuota=300%
45OOMPolicy=continue 46OOMPolicy=continue
@@ -70,7 +71,7 @@ TimeoutStopSec=50min
70# when it exits is killed. 71# when it exits is killed.
71KillMode=mixed 72KillMode=mixed
72ExecStart= 73ExecStart=
73ExecStart=/usr/local/bin/gitbay-runner -remote git@127.0.0.1 -workdir /var/lib/gitbay-runner/work -poll 5s -timeout 45m -repos krz/gitbay,cmc/ci-smoke -isolation podman -image localhost/gitbay-ci:1 74ExecStart=/usr/local/bin/gitbay-runner -remote git@127.0.0.1 -workdir /var/lib/gitbay-runner/work -poll 5s -timeout 45m -repos krz/gitbay,cmc/ci-smoke -isolation podman -image localhost/gitbay-ci:1 -cpus 3 -memory 6g
74Nice=10 75Nice=10
75CPUWeight=30 76CPUWeight=30
76IOWeight=30 77IOWeight=30
@@ -102,7 +103,15 @@ ProtectSystem=full
102# as this user — the container now covers: a build gets its own proc, 103# as this user — the container now covers: a build gets its own proc,
103# with those paths masked by the runtime. Under -isolation none, set it 104# with those paths masked by the runtime. Under -isolation none, set it
104# back to yes. 105# back to yes.
105ProtectControlGroups=yes 106# ProtectControlGroups is off because the runner writes cgroups: it
107# creates one per build under this unit's delegated cgroup to carry
108# -memory and -cpus (#188). The flag mounts /sys/fs/cgroup read-only in
109# the unit's namespace, which makes even a delegated cgroup unwritable,
110# and the runner then refuses to start when a limit is set. Delegate=yes
111# already hands this unit its subtree; what the flag protected beyond
112# that is other units' cgroups, which are root-owned and not writable
113# by this user regardless.
114ProtectControlGroups=no
106RestrictSUIDSGID=yes 115RestrictSUIDSGID=yes
107Delegate=yes 116Delegate=yes
108# The runner's home is /var/lib/gitbay-runner (see the Admin page), and 117# The runner's home is /var/lib/gitbay-runner (see the Admin page), and