Commit c14c881e38

c14c881e38e9bf5bc2ed34b791f2396c30e6e7fb

parent: 0caaaedf8f

Verified · cmc ci/build: success ci/test: success

cmc <hello@cleberg.net> · 2026-09-07 20:37 UTC

runner, deploy, wiki: the runner owns a cgroup per build, so -memory and -cpus apply

Podman's --memory and --cpus never applied under rootless cgroupfs: the
container started inside the service's own cgroup, crun could not
create a child, and the flags were accepted and ignored. The runner now
parks itself in a leaf of its delegated service cgroup, enables the
controllers, creates builds/build-<id> with the limits written in
cgroup v2's units, and starts every podman process for the build inside
it via CLONE_INTO_CGROUP, with podman's cgroup handling off. A podman
exec started from the runner's cgroup would run the step outside the
limit, which is why exec goes through it too. The cgroup is killed and
removed after the build.

Without a delegated cgroup the runner refuses to start when a limit is
set, and logs that builds run unconfined when none is.

bay1's ExecStart gets -cpus 3 -memory 6g back, inside the unit's
MemoryMax and CPUQuota.

Closes #188

Layout: unified · split

.gitbay/wiki/Admin.org +15 −11
@@ -448,17 +448,21 @@ because an image this host does not have would fail every build. A job
448overrides it with =image:= in =.gitbay/ci.yml=, validated as a reference 448overrides it with =image:= in =.gitbay/ci.yml=, validated as a reference
449so a config file cannot turn it into podman arguments. 449so a config file cannot turn it into podman arguments.
450 450
451=-cpus= and =-memory= cap one build's container (podman's own units, 451=-cpus= and =-memory= cap one build (podman's units, e.g. =-cpus 2
452e.g. =-cpus 2 -memory 4g=); unset means uncapped. They only take effect 452-memory 4g=); unset means uncapped. The runner applies them itself: it
453where podman can create a cgroup for the container. Under rootless 453creates a cgroup per build under its own delegated service cgroup,
454podman with the cgroupfs manager, which is what a system service gets, 454writes the limits there, and starts every podman process for the build
455it cannot: the container runs inside the service's own cgroup and both 455inside it, with podman's cgroup handling off. Podman's own =--memory=
456flags are accepted and ignored (krz/gitbay#188). Cap the unit instead. 456and =--cpus= never applied under rootless cgroupfs, which is what a
457bay1's drop-in sets =MemoryMax=6G= and =CPUQuota=300%= on a 7.7GB 457system service gets (krz/gitbay#188). The unit therefore needs
458four-core host with no swap: the memory cap is what keeps the forge 458=Delegate=yes=, which the drop-in sets; without it the runner refuses
459alive when a build allocates without bound, and it sits above the e2e 459to start when a limit is set, and logs that builds run unconfined when
460suite's 5GB peak rather than at a fair share. =OOMPolicy=continue= keeps 460none is. bay1 runs =-cpus 3 -memory 6g= per build inside =MemoryMax=6G=
461systemd from stopping the runner when a build is OOM-killed. 461and =CPUQuota=300%= on the unit, on a 7.7GB four-core host with no
462swap: the memory cap is what keeps the forge alive when a build
463allocates without bound, and it sits above the e2e suite's 5GB peak
464rather than at a fair share. =OOMPolicy=continue= keeps systemd from
465stopping the runner when a build is OOM-killed.
462 466
463Each repository gets its own build home under the runner's workdir, 467Each repository gets its own build home under the runner's workdir,
464mounted into its containers as =HOME=. Caches persist between builds of 468mounted into its containers as =HOME=. Caches persist between builds of
cmd/gitbay-runner/cgroup.go added +87
@@ -0,0 +1,87 @@
1package main
2
3import (
4 "fmt"
5 "os"
6 "path/filepath"
7 "strconv"
8 "strings"
9)
10
11// Build limits are cgroup v2 files the runner writes itself. Podman's
12// --memory and --cpus never applied here: under rootless podman with the
13// cgroupfs manager the container starts inside the service's own cgroup
14// and crun cannot create a child, so the flags were accepted and ignored
15// (#188). The runner owns a cgroup per build under its delegated service
16// cgroup instead, writes the limits into it, and starts every podman
17// process for that build from inside it with podman's own cgroup handling
18// off. What follows is the portable half: parsing and the file writes.
19
20// memoryBytes parses podman's memory units — a whole number with an
21// optional b, k, m or g suffix — into bytes.
22func memoryBytes(s string) (int64, error) {
23 if s == "" {
24 return 0, fmt.Errorf("empty memory limit")
25 }
26 num, unit := s, ""
27 if last := s[len(s)-1]; last < '0' || last > '9' {
28 num, unit = s[:len(s)-1], strings.ToLower(s[len(s)-1:])
29 }
30 n, err := strconv.ParseInt(num, 10, 64)
31 if err != nil || n <= 0 {
32 return 0, fmt.Errorf("memory limit %q: want a whole number of b, k, m or g", s)
33 }
34 shift := map[string]uint{"": 0, "b": 0, "k": 10, "m": 20, "g": 30}
35 sh, ok := shift[unit]
36 if !ok {
37 return 0, fmt.Errorf("memory limit %q: unit %q is not b, k, m or g", s, unit)
38 }
39 return n << sh, nil
40}
41
42// cpuMax renders a CPU count, whole or fractional, as cgroup v2's
43// "<quota> <period>" over a 100ms period.
44func cpuMax(s string) (string, error) {
45 const period = 100000
46 f, err := strconv.ParseFloat(s, 64)
47 if err != nil || f <= 0 {
48 return "", fmt.Errorf("cpu limit %q: want a positive number of CPUs", s)
49 }
50 return fmt.Sprintf("%d %d", int64(f*period+0.5), period), nil
51}
52
53// ownCgroupPath reads the cgroup v2 path out of /proc/self/cgroup
54// contents. A v1 hierarchy has more than the one "0::" line and is not
55// something the runner manages.
56func ownCgroupPath(procSelfCgroup string) (string, error) {
57 lines := strings.Split(strings.TrimSpace(procSelfCgroup), "\n")
58 if len(lines) != 1 || !strings.HasPrefix(lines[0], "0::/") {
59 return "", fmt.Errorf("not a cgroup v2 host: /proc/self/cgroup is %q", strings.TrimSpace(procSelfCgroup))
60 }
61 return strings.TrimPrefix(lines[0], "0::"), nil
62}
63
64// writeLimits writes the requested limits into a cgroup directory. An
65// unset limit writes nothing, so the build inherits whatever the unit
66// allows rather than getting "max".
67func writeLimits(dir, memory, cpus string) error {
68 if memory != "" {
69 n, err := memoryBytes(memory)
70 if err != nil {
71 return err
72 }
73 if err := os.WriteFile(filepath.Join(dir, "memory.max"), []byte(strconv.FormatInt(n, 10)), 0o644); err != nil {
74 return err
75 }
76 }
77 if cpus != "" {
78 v, err := cpuMax(cpus)
79 if err != nil {
80 return err
81 }
82 if err := os.WriteFile(filepath.Join(dir, "cpu.max"), []byte(v), 0o644); err != nil {
83 return err
84 }
85 }
86 return nil
87}
cmd/gitbay-runner/cgroup_linux.go added +109
@@ -0,0 +1,109 @@
1//go:build linux
2
3package main
4
5import (
6 "fmt"
7 "log"
8 "os"
9 "os/exec"
10 "path/filepath"
11 "strconv"
12 "syscall"
13 "time"
14)
15
16// buildCgroups is the runner's own cgroup subtree for builds. The unit's
17// Delegate=yes hands the runner its service cgroup; the runner parks
18// itself in a leaf so the service cgroup can enable controllers for
19// children (a cgroup may hold processes or controller-enabled children,
20// not both), and creates one child per build under builds/.
21type buildCgroups struct {
22 builds string // <service cgroup>/builds
23}
24
25const cgroupControllers = "+cpu +memory +pids"
26
27// prepareBuildCgroups moves the runner into <own>/runner, enables the
28// controllers on its original cgroup, and creates builds/. It fails
29// where the cgroup is not writable, which is a unit without
30// Delegate=yes; the caller decides whether that is fatal.
31func prepareBuildCgroups() (*buildCgroups, error) {
32 raw, err := os.ReadFile("/proc/self/cgroup")
33 if err != nil {
34 return nil, err
35 }
36 own, err := ownCgroupPath(string(raw))
37 if err != nil {
38 return nil, err
39 }
40 root := filepath.Join("/sys/fs/cgroup", own)
41 leaf := filepath.Join(root, "runner")
42 if err := os.MkdirAll(leaf, 0o755); err != nil {
43 return nil, fmt.Errorf("%s is not writable; the unit needs Delegate=yes: %w", root, err)
44 }
45 if err := os.WriteFile(filepath.Join(leaf, "cgroup.procs"), []byte(strconv.Itoa(os.Getpid())), 0o644); err != nil {
46 return nil, fmt.Errorf("moving into %s: %w", leaf, err)
47 }
48 if err := os.WriteFile(filepath.Join(root, "cgroup.subtree_control"), []byte(cgroupControllers), 0o644); err != nil {
49 return nil, fmt.Errorf("enabling controllers on %s: %w", root, err)
50 }
51 builds := filepath.Join(root, "builds")
52 if err := os.MkdirAll(builds, 0o755); err != nil {
53 return nil, err
54 }
55 if err := os.WriteFile(filepath.Join(builds, "cgroup.subtree_control"), []byte(cgroupControllers), 0o644); err != nil {
56 return nil, fmt.Errorf("enabling controllers on %s: %w", builds, err)
57 }
58 return &buildCgroups{builds: builds}, nil
59}
60
61// create makes the cgroup for one build with its limits written, and
62// returns its path and an open directory fd for placing processes.
63func (c *buildCgroups) create(id int64, memory, cpus string) (string, *os.File, error) {
64 dir := filepath.Join(c.builds, fmt.Sprintf("build-%d", id))
65 if err := os.Mkdir(dir, 0o755); err != nil {
66 return "", nil, err
67 }
68 if err := writeLimits(dir, memory, cpus); err != nil {
69 os.Remove(dir)
70 return "", nil, err
71 }
72 f, err := os.Open(dir)
73 if err != nil {
74 os.Remove(dir)
75 return "", nil, err
76 }
77 return dir, f, nil
78}
79
80// remove kills whatever is still in the build's cgroup — conmon, the
81// pause process, a step's stray child — and removes it. rmdir fails
82// until the kernel has reaped every process, so it retries briefly.
83func (c *buildCgroups) remove(dir string) {
84 if err := os.WriteFile(filepath.Join(dir, "cgroup.kill"), []byte("1"), 0o644); err != nil {
85 log.Printf("cgroup %s: kill: %v", dir, err)
86 }
87 for i := 0; i < 50; i++ {
88 if err := os.Remove(dir); err == nil {
89 return
90 }
91 time.Sleep(100 * time.Millisecond)
92 }
93 log.Printf("cgroup %s: still populated after kill; left in place", dir)
94}
95
96// intoCgroup starts cmd inside the cgroup fd refers to, so podman,
97// conmon and everything they start inherit the build's limits. A podman
98// exec started from the runner's own cgroup lands there, outside the
99// limit, which is why every invocation for a build goes through this.
100func intoCgroup(cmd *exec.Cmd, f *os.File) {
101 if f == nil {
102 return
103 }
104 if cmd.SysProcAttr == nil {
105 cmd.SysProcAttr = &syscall.SysProcAttr{}
106 }
107 cmd.SysProcAttr.UseCgroupFD = true
108 cmd.SysProcAttr.CgroupFD = int(f.Fd())
109}
cmd/gitbay-runner/cgroup_other.go added +23
@@ -0,0 +1,23 @@
1//go:build !linux
2
3package main
4
5import (
6 "errors"
7 "os"
8 "os/exec"
9)
10
11type buildCgroups struct{}
12
13func prepareBuildCgroups() (*buildCgroups, error) {
14 return nil, errors.New("build cgroups need Linux")
15}
16
17func (c *buildCgroups) create(id int64, memory, cpus string) (string, *os.File, error) {
18 return "", nil, errors.New("build cgroups need Linux")
19}
20
21func (c *buildCgroups) remove(dir string) {}
22
23func intoCgroup(cmd *exec.Cmd, f *os.File) {}
cmd/gitbay-runner/cgroup_test.go added +92
@@ -0,0 +1,92 @@
1package main
2
3import (
4 "os"
5 "path/filepath"
6 "testing"
7)
8
9// Podman's --memory and --cpus never applied under rootless cgroupfs: the
10// container ran in the service's own cgroup and crun could not create a
11// child (#188). The runner now owns the build cgroups itself and places
12// podman inside one, so the limits are written by the runner in cgroup
13// v2's own units.
14func TestMemoryBytes(t *testing.T) {
15 cases := map[string]int64{
16 "64m": 64 << 20,
17 "6g": 6 << 30,
18 "512k": 512 << 10,
19 "100b": 100,
20 "4096": 4096,
21 "1G": 1 << 30,
22 }
23 for in, want := range cases {
24 got, err := memoryBytes(in)
25 if err != nil || got != want {
26 t.Errorf("memoryBytes(%q) = %d, %v; want %d", in, got, err, want)
27 }
28 }
29 for _, bad := range []string{"", "lots", "6gb", "-1g", "1.5g"} {
30 if _, err := memoryBytes(bad); err == nil {
31 t.Errorf("memoryBytes(%q) accepted", bad)
32 }
33 }
34}
35
36func TestCPUMax(t *testing.T) {
37 cases := map[string]string{
38 "3": "300000 100000",
39 "1": "100000 100000",
40 "1.5": "150000 100000",
41 "0.25": "25000 100000",
42 }
43 for in, want := range cases {
44 got, err := cpuMax(in)
45 if err != nil || got != want {
46 t.Errorf("cpuMax(%q) = %q, %v; want %q", in, got, err, want)
47 }
48 }
49 for _, bad := range []string{"", "0", "-1", "two"} {
50 if _, err := cpuMax(bad); err == nil {
51 t.Errorf("cpuMax(%q) accepted", bad)
52 }
53 }
54}
55
56// /proc/self/cgroup on cgroup v2 is one line, "0::<path>".
57func TestOwnCgroupPath(t *testing.T) {
58 got, err := ownCgroupPath("0::/system.slice/gitbay-runner.service\n")
59 if err != nil || got != "/system.slice/gitbay-runner.service" {
60 t.Fatalf("ownCgroupPath = %q, %v", got, err)
61 }
62 // A v1 hierarchy, or anything else, is not something the runner
63 // manages.
64 if _, err := ownCgroupPath("12:memory:/user.slice\n0::/init.scope\n"); err == nil {
65 t.Fatal("v1 hierarchy accepted")
66 }
67}
68
69// Limits land as cgroup v2 interface files in the build's directory;
70// an unset limit writes nothing, which means "inherit", not "max".
71func TestWriteLimits(t *testing.T) {
72 dir := t.TempDir()
73 if err := writeLimits(dir, "6g", "3"); err != nil {
74 t.Fatal(err)
75 }
76 if got, _ := os.ReadFile(filepath.Join(dir, "memory.max")); string(got) != "6442450944" {
77 t.Errorf("memory.max = %q", got)
78 }
79 if got, _ := os.ReadFile(filepath.Join(dir, "cpu.max")); string(got) != "300000 100000" {
80 t.Errorf("cpu.max = %q", got)
81 }
82 empty := t.TempDir()
83 if err := writeLimits(empty, "", ""); err != nil {
84 t.Fatal(err)
85 }
86 if entries, _ := os.ReadDir(empty); len(entries) != 0 {
87 t.Errorf("unset limits wrote %v", entries)
88 }
89 if err := writeLimits(t.TempDir(), "lots", ""); err == nil {
90 t.Error("a bad memory limit was accepted")
91 }
92}
cmd/gitbay-runner/env_test.go −13
@@ -118,19 +118,6 @@ func TestEnvHomeFindsHome(t *testing.T) {
118 } 118 }
119} 119}
120 120
121// A limit is passed to podman only when set; unset means uncapped, not a
122// default that could kill the suite.
123func TestLimitArgs(t *testing.T) {
124 if got := (&runner{}).limitArgs(); len(got) != 0 {
125 t.Errorf("no limits set, got %v", got)
126 }
127 got := (&runner{memory: "4g", cpus: "2"}).limitArgs()
128 want := []string{"--memory", "4g", "--cpus", "2"}
129 if strings.Join(got, " ") != strings.Join(want, " ") {
130 t.Errorf("limitArgs = %v, want %v", got, want)
131 }
132}
133
134// Closing stop drains: the build in flight finishes and is reported, and 121// Closing stop drains: the build in flight finishes and is reported, and
135// no further build is claimed (#179). 122// no further build is claimed (#179).
136func TestServeDrainsOnStop(t *testing.T) { 123func TestServeDrainsOnStop(t *testing.T) {
cmd/gitbay-runner/isolate.go +19 −19
@@ -122,11 +122,26 @@ func (r *runner) runStepsPodman(j job, dir string, env []string, sink io.Writer,
122 } 122 }
123 defer os.Remove(envFile) 123 defer os.Remove(envFile)
124 124
125 // The build's cgroup carries its limits; podman's own cgroup handling
126 // is off because it never worked here (#188). Every podman process
127 // for this build starts inside the cgroup, exec included: one started
128 // from the runner's cgroup would run the step outside the limit.
129 var cgroupFD *os.File
130 if r.cgroups != nil {
131 dir, f, err := r.cgroups.create(j.ID, r.memory, r.cpus)
132 if err != nil {
133 fmt.Fprintf(sink, "preparing the build cgroup: %v\n", err)
134 return false
135 }
136 cgroupFD = f
137 defer f.Close()
138 defer r.cgroups.remove(dir)
139 }
140
125 name := fmt.Sprintf("gitbay-build-%d", j.ID) 141 name := fmt.Sprintf("gitbay-build-%d", j.ID)
126 // --rm so a container cannot outlive its build; the explicit rm below 142 // --rm so a container cannot outlive its build; the explicit rm below
127 // covers the case where the daemon-less run itself fails. 143 // covers the case where the daemon-less run itself fails.
128 args := append(r.podmanGlobal(), "run", "--detach", "--rm", "--pull=never") 144 args := append(r.podmanGlobal(), "run", "--detach", "--rm", "--pull=never", "--cgroups=disabled")
129 args = append(args, r.limitArgs()...)
130 args = append(args, 145 args = append(args,
131 "--name", name, 146 "--name", name,
132 "--env-file", envFile, 147 "--env-file", envFile,
@@ -142,6 +157,7 @@ func (r *runner) runStepsPodman(j job, dir string, env []string, sink io.Writer,
142 image, "-c", "sleep infinity") 157 image, "-c", "sleep infinity")
143 start := exec.Command(podman, args...) 158 start := exec.Command(podman, args...)
144 start.Env = []string{"PATH=" + os.Getenv("PATH"), "HOME=" + r.podmanHome()} 159 start.Env = []string{"PATH=" + os.Getenv("PATH"), "HOME=" + r.podmanHome()}
160 intoCgroup(start, cgroupFD)
145 if out, err := start.CombinedOutput(); err != nil { 161 if out, err := start.CombinedOutput(); err != nil {
146 // A missing image lands here, and it is the common case worth 162 // A missing image lands here, and it is the common case worth
147 // explaining: this runner never pulls, so an image it does not 163 // explaining: this runner never pulls, so an image it does not
@@ -162,6 +178,7 @@ func (r *runner) runStepsPodman(j job, dir string, env []string, sink io.Writer,
162 fmt.Fprintf(sink, "$ %s\n", step) 178 fmt.Fprintf(sink, "$ %s\n", step)
163 cmd := exec.Command(podman, append(r.podmanGlobal(), "exec", "--workdir", "/workspace", name, "sh", "-c", step)...) 179 cmd := exec.Command(podman, append(r.podmanGlobal(), "exec", "--workdir", "/workspace", name, "sh", "-c", step)...)
164 cmd.Env = []string{"PATH=" + os.Getenv("PATH"), "HOME=" + r.podmanHome()} 180 cmd.Env = []string{"PATH=" + os.Getenv("PATH"), "HOME=" + r.podmanHome()}
181 intoCgroup(cmd, cgroupFD)
165 cmd.Stdout, cmd.Stderr = sink, sink 182 cmd.Stdout, cmd.Stderr = sink, sink
166 if ok, why := runStep(cmd, deadline); !ok { 183 if ok, why := runStep(cmd, deadline); !ok {
167 fmt.Fprintf(sink, "%s\n", why) 184 fmt.Fprintf(sink, "%s\n", why)
@@ -189,23 +206,6 @@ func (r *runner) podmanGlobal() []string {
189 return []string{"--cgroup-manager=cgroupfs"} 206 return []string{"--cgroup-manager=cgroupfs"}
190} 207}
191 208
192// limitArgs caps one build's container. The service's CPUWeight and
193// IOWeight shape the service against other services, not one build
194// against the host, and the threat model lists resource exhaustion as
195// unaddressed. Memory is deliberately uncapped by default: the e2e suite
196// peaks past 5GB on a 7GB host, and a cap that kills the suite is an
197// outage, not a limit.
198func (r *runner) limitArgs() []string {
199 var args []string
200 if r.memory != "" {
201 args = append(args, "--memory", r.memory)
202 }
203 if r.cpus != "" {
204 args = append(args, "--cpus", r.cpus)
205 }
206 return args
207}
208
209// envHome returns the HOME the step environment carries. 209// envHome returns the HOME the step environment carries.
210func envHome(env []string) string { 210func envHome(env []string) string {
211 for _, e := range env { 211 for _, e := range env {
cmd/gitbay-runner/main.go +23 −3
@@ -50,9 +50,12 @@ type runner struct {
50 // isolation selects how steps run: "podman" or "none". 50 // isolation selects how steps run: "podman" or "none".
51 image string 51 image string
52 isolation string 52 isolation string
53 // memory and cpus cap one build's container; empty means no cap. 53 // memory and cpus cap one build's cgroup; empty means no cap.
54 memory string 54 memory string
55 cpus string 55 cpus string
56 // cgroups is the runner's build cgroup subtree, nil where the unit
57 // is not delegated and builds run unconfined in the service cgroup.
58 cgroups *buildCgroups
56 // stepFn is step, replaceable by tests. 59 // stepFn is step, replaceable by tests.
57 stepFn func() (bool, error) 60 stepFn func() (bool, error)
58 // repos limits which repositories this runner claims builds for. Empty 61 // repos limits which repositories this runner claims builds for. Empty
@@ -74,8 +77,8 @@ func main() {
74 jobs = flag.Int("jobs", 1, "builds to run at once") 77 jobs = flag.Int("jobs", 1, "builds to run at once")
75 image = flag.String("image", "", "default container image for jobs that name none") 78 image = flag.String("image", "", "default container image for jobs that name none")
76 isolation = flag.String("isolation", "podman", "how steps run: podman, or none for no container") 79 isolation = flag.String("isolation", "podman", "how steps run: podman, or none for no container")
77 memory = flag.String("memory", "", "memory limit per build container, e.g. 4g (podman only; default unlimited)") 80 memory = flag.String("memory", "", "memory limit per build, e.g. 4g (podman only, needs a delegated cgroup; default unlimited)")
78 cpus = flag.String("cpus", "", "CPU limit per build container, e.g. 2 (podman only; default unlimited)") 81 cpus = flag.String("cpus", "", "CPU limit per build, e.g. 2 (podman only, needs a delegated cgroup; default unlimited)")
79 version = flag.Bool("version", false, "print the commit this binary was built from, then exit") 82 version = flag.Bool("version", false, "print the commit this binary was built from, then exit")
80 ) 83 )
81 flag.Parse() 84 flag.Parse()
@@ -96,6 +99,23 @@ func main() {
96 memory: *memory, 99 memory: *memory,
97 cpus: *cpus, 100 cpus: *cpus,
98 } 101 }
102 if r.isolation == isolationPodman {
103 // Before podman runs anything: its pause process lands in the
104 // cgroup of the first invocation, and that must be the runner's
105 // leaf, not a build's.
106 cg, err := prepareBuildCgroups()
107 switch {
108 case err == nil:
109 r.cgroups = cg
110 case r.memory != "" || r.cpus != "":
111 // Limits that cannot be applied are refused, not dropped:
112 // a runner that accepted -memory and ran uncapped is what
113 // #188 was.
114 log.Fatalf("-memory/-cpus: build cgroups unavailable: %v", err)
115 default:
116 log.Printf("build cgroups unavailable (%v); builds run unconfined in the service cgroup", err)
117 }
118 }
99 if err := r.checkIsolation(); err != nil { 119 if err := r.checkIsolation(); err != nil {
100 // Refusing to start is the point. A runner that quietly fell back 120 // Refusing to start is the point. A runner that quietly fell back
101 // to running repository code on the host would drop isolation 121 // to running repository code on the host would drop isolation
cmd/gitbay-runner/proc_unix.go +4 −1
@@ -13,7 +13,10 @@ import (
13// child: dash forks a single command rather than exec'ing it, so on a 13// child: dash forks a single command rather than exec'ing it, so on a
14// Debian host "sh -c 'sleep 120'" survived its shell by two minutes. 14// Debian host "sh -c 'sleep 120'" survived its shell by two minutes.
15func ownProcessGroup(cmd *exec.Cmd) { 15func ownProcessGroup(cmd *exec.Cmd) {
16 cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true} 16 if cmd.SysProcAttr == nil {
17 cmd.SysProcAttr = &syscall.SysProcAttr{}
18 }
19 cmd.SysProcAttr.Setpgid = true
17} 20}
18 21
19func killTree(cmd *exec.Cmd) { 22func killTree(cmd *exec.Cmd) {
deploy/gitbay-runner.override.conf +13 −12
@@ -27,19 +27,20 @@
27# repositories never claims a build it is not scoped to, so the canary 27# repositories never claims a build it is not scoped to, so the canary
28# must be listed or its scheduled build waits forever. 28# must be listed or its scheduled build waits forever.
29# 29#
30# Resource caps are on the unit, not on the container. Under rootless 30# Two layers of resource caps. MemoryMax and CPUQuota bound the unit —
31# podman with the cgroupfs manager the container runs inside this 31# the runner and every build together — which is what keeps the forge
32# service's own cgroup: no child cgroup is created, so podman's --cpus 32# alive when a build allocates without bound. -memory and -cpus on
33# and --memory are accepted and never applied (#188). MemoryMax and 33# ExecStart cap each build's own cgroup, which the runner creates under
34# CPUQuota below bound the runner and every build together, which is 34# this unit's delegated cgroup (Delegate=yes below); podman's own --memory
35# what keeps the forge alive when a build allocates without bound. 35# and --cpus never applied here, since under rootless cgroupfs the
36# container ran in the service cgroup itself (#188).
36# 37#
37# 6G of the host's 7.7GB, no swap: the e2e suite peaks past 5GB, so the 38# 6G of the host's 7.7GB, no swap: the e2e suite peaks past 5GB, so the
38# cap sits above that rather than at a fair share. CPUQuota=300% is 39# cap sits above that rather than at a fair share. CPUQuota=300% and
39# three of the four cores, leaving one for gitbayd and sshd (#144, #184). 40# -cpus 3 are three of the four cores, leaving one for gitbayd and sshd
40# OOMPolicy=continue: systemd's default stops the whole service when any 41# (#144, #184). OOMPolicy=continue: systemd's default stops the whole
41# process in it is OOM-killed, which would end the runner mid-build; the 42# service when any process in it is OOM-killed, which would end the
42# build fails and the runner carries on. 43# runner mid-build; the build fails and the runner carries on.
43MemoryMax=6G 44MemoryMax=6G
44CPUQuota=300% 45CPUQuota=300%
45OOMPolicy=continue 46OOMPolicy=continue
@@ -70,7 +71,7 @@ TimeoutStopSec=50min
70# when it exits is killed. 71# when it exits is killed.
71KillMode=mixed 72KillMode=mixed
72ExecStart= 73ExecStart=
73ExecStart=/usr/local/bin/gitbay-runner -remote git@127.0.0.1 -workdir /var/lib/gitbay-runner/work -poll 5s -timeout 45m -repos krz/gitbay,cmc/ci-smoke -isolation podman -image localhost/gitbay-ci:1 74ExecStart=/usr/local/bin/gitbay-runner -remote git@127.0.0.1 -workdir /var/lib/gitbay-runner/work -poll 5s -timeout 45m -repos krz/gitbay,cmc/ci-smoke -isolation podman -image localhost/gitbay-ci:1 -cpus 3 -memory 6g
74Nice=10 75Nice=10
75CPUWeight=30 76CPUWeight=30
76IOWeight=30 77IOWeight=30