Commit c14c881e38
Verified · cmc ci/build: success ci/test: success
.gitbay/wiki/Admin.org +15 −11
| @@ -448,17 +448,21 @@ because an image this host does not have would fail every build. A job | ||
| 448 | 448 | overrides it with =image:= in =.gitbay/ci.yml=, validated as a reference |
| 449 | 449 | so a config file cannot turn it into podman arguments. |
| 450 | 450 | |
| 451 | =-cpus= and =-memory= cap one build's container (podman's own units, | |
| 452 | e.g. =-cpus 2 -memory 4g=); unset means uncapped. They only take effect | |
| 453 | where podman can create a cgroup for the container. Under rootless | |
| 454 | podman with the cgroupfs manager, which is what a system service gets, | |
| 455 | it cannot: the container runs inside the service's own cgroup and both | |
| 456 | flags are accepted and ignored (krz/gitbay#188). Cap the unit instead. | |
| 457 | bay1's drop-in sets =MemoryMax=6G= and =CPUQuota=300%= on a 7.7GB | |
| 458 | four-core host with no swap: the memory cap is what keeps the forge | |
| 459 | alive when a build allocates without bound, and it sits above the e2e | |
| 460 | suite's 5GB peak rather than at a fair share. =OOMPolicy=continue= keeps | |
| 461 | systemd from stopping the runner when a build is OOM-killed. | |
| 451 | =-cpus= and =-memory= cap one build (podman's units, e.g. =-cpus 2 | |
| 452 | -memory 4g=); unset means uncapped. The runner applies them itself: it | |
| 453 | creates a cgroup per build under its own delegated service cgroup, | |
| 454 | writes the limits there, and starts every podman process for the build | |
| 455 | inside it, with podman's cgroup handling off. Podman's own =--memory= | |
| 456 | and =--cpus= never applied under rootless cgroupfs, which is what a | |
| 457 | system service gets (krz/gitbay#188). The unit therefore needs | |
| 458 | =Delegate=yes=, which the drop-in sets; without it the runner refuses | |
| 459 | to start when a limit is set, and logs that builds run unconfined when | |
| 460 | none is. bay1 runs =-cpus 3 -memory 6g= per build inside =MemoryMax=6G= | |
| 461 | and =CPUQuota=300%= on the unit, on a 7.7GB four-core host with no | |
| 462 | swap: the memory cap is what keeps the forge alive when a build | |
| 463 | allocates without bound, and it sits above the e2e suite's 5GB peak | |
| 464 | rather than at a fair share. =OOMPolicy=continue= keeps systemd from | |
| 465 | stopping the runner when a build is OOM-killed. | |
| 462 | 466 | |
| 463 | 467 | Each repository gets its own build home under the runner's workdir, |
| 464 | 468 | mounted into its containers as =HOME=. Caches persist between builds of |
cmd/gitbay-runner/cgroup.go added +87
| @@ -0,0 +1,87 @@ | ||
| 1 | package main | |
| 2 | ||
| 3 | import ( | |
| 4 | "fmt" | |
| 5 | "os" | |
| 6 | "path/filepath" | |
| 7 | "strconv" | |
| 8 | "strings" | |
| 9 | ) | |
| 10 | ||
| 11 | // Build limits are cgroup v2 files the runner writes itself. Podman's | |
| 12 | // --memory and --cpus never applied here: under rootless podman with the | |
| 13 | // cgroupfs manager the container starts inside the service's own cgroup | |
| 14 | // and crun cannot create a child, so the flags were accepted and ignored | |
| 15 | // (#188). The runner owns a cgroup per build under its delegated service | |
| 16 | // cgroup instead, writes the limits into it, and starts every podman | |
| 17 | // process for that build from inside it with podman's own cgroup handling | |
| 18 | // off. What follows is the portable half: parsing and the file writes. | |
| 19 | ||
| 20 | // memoryBytes parses podman's memory units — a whole number with an | |
| 21 | // optional b, k, m or g suffix — into bytes. | |
| 22 | func memoryBytes(s string) (int64, error) { | |
| 23 | if s == "" { | |
| 24 | return 0, fmt.Errorf("empty memory limit") | |
| 25 | } | |
| 26 | num, unit := s, "" | |
| 27 | if last := s[len(s)-1]; last < '0' || last > '9' { | |
| 28 | num, unit = s[:len(s)-1], strings.ToLower(s[len(s)-1:]) | |
| 29 | } | |
| 30 | n, err := strconv.ParseInt(num, 10, 64) | |
| 31 | if err != nil || n <= 0 { | |
| 32 | return 0, fmt.Errorf("memory limit %q: want a whole number of b, k, m or g", s) | |
| 33 | } | |
| 34 | shift := map[string]uint{"": 0, "b": 0, "k": 10, "m": 20, "g": 30} | |
| 35 | sh, ok := shift[unit] | |
| 36 | if !ok { | |
| 37 | return 0, fmt.Errorf("memory limit %q: unit %q is not b, k, m or g", s, unit) | |
| 38 | } | |
| 39 | return n << sh, nil | |
| 40 | } | |
| 41 | ||
| 42 | // cpuMax renders a CPU count, whole or fractional, as cgroup v2's | |
| 43 | // "<quota> <period>" over a 100ms period. | |
| 44 | func cpuMax(s string) (string, error) { | |
| 45 | const period = 100000 | |
| 46 | f, err := strconv.ParseFloat(s, 64) | |
| 47 | if err != nil || f <= 0 { | |
| 48 | return "", fmt.Errorf("cpu limit %q: want a positive number of CPUs", s) | |
| 49 | } | |
| 50 | return fmt.Sprintf("%d %d", int64(f*period+0.5), period), nil | |
| 51 | } | |
| 52 | ||
| 53 | // ownCgroupPath reads the cgroup v2 path out of /proc/self/cgroup | |
| 54 | // contents. A v1 hierarchy has more than the one "0::" line and is not | |
| 55 | // something the runner manages. | |
| 56 | func ownCgroupPath(procSelfCgroup string) (string, error) { | |
| 57 | lines := strings.Split(strings.TrimSpace(procSelfCgroup), "\n") | |
| 58 | if len(lines) != 1 || !strings.HasPrefix(lines[0], "0::/") { | |
| 59 | return "", fmt.Errorf("not a cgroup v2 host: /proc/self/cgroup is %q", strings.TrimSpace(procSelfCgroup)) | |
| 60 | } | |
| 61 | return strings.TrimPrefix(lines[0], "0::"), nil | |
| 62 | } | |
| 63 | ||
| 64 | // writeLimits writes the requested limits into a cgroup directory. An | |
| 65 | // unset limit writes nothing, so the build inherits whatever the unit | |
| 66 | // allows rather than getting "max". | |
| 67 | func writeLimits(dir, memory, cpus string) error { | |
| 68 | if memory != "" { | |
| 69 | n, err := memoryBytes(memory) | |
| 70 | if err != nil { | |
| 71 | return err | |
| 72 | } | |
| 73 | if err := os.WriteFile(filepath.Join(dir, "memory.max"), []byte(strconv.FormatInt(n, 10)), 0o644); err != nil { | |
| 74 | return err | |
| 75 | } | |
| 76 | } | |
| 77 | if cpus != "" { | |
| 78 | v, err := cpuMax(cpus) | |
| 79 | if err != nil { | |
| 80 | return err | |
| 81 | } | |
| 82 | if err := os.WriteFile(filepath.Join(dir, "cpu.max"), []byte(v), 0o644); err != nil { | |
| 83 | return err | |
| 84 | } | |
| 85 | } | |
| 86 | return nil | |
| 87 | } | |
cmd/gitbay-runner/cgroup_linux.go added +109
| @@ -0,0 +1,109 @@ | ||
| 1 | //go:build linux | |
| 2 | ||
| 3 | package main | |
| 4 | ||
| 5 | import ( | |
| 6 | "fmt" | |
| 7 | "log" | |
| 8 | "os" | |
| 9 | "os/exec" | |
| 10 | "path/filepath" | |
| 11 | "strconv" | |
| 12 | "syscall" | |
| 13 | "time" | |
| 14 | ) | |
| 15 | ||
| 16 | // buildCgroups is the runner's own cgroup subtree for builds. The unit's | |
| 17 | // Delegate=yes hands the runner its service cgroup; the runner parks | |
| 18 | // itself in a leaf so the service cgroup can enable controllers for | |
| 19 | // children (a cgroup may hold processes or controller-enabled children, | |
| 20 | // not both), and creates one child per build under builds/. | |
| 21 | type buildCgroups struct { | |
| 22 | builds string // <service cgroup>/builds | |
| 23 | } | |
| 24 | ||
| 25 | const cgroupControllers = "+cpu +memory +pids" | |
| 26 | ||
| 27 | // prepareBuildCgroups moves the runner into <own>/runner, enables the | |
| 28 | // controllers on its original cgroup, and creates builds/. It fails | |
| 29 | // where the cgroup is not writable, which is a unit without | |
| 30 | // Delegate=yes; the caller decides whether that is fatal. | |
| 31 | func prepareBuildCgroups() (*buildCgroups, error) { | |
| 32 | raw, err := os.ReadFile("/proc/self/cgroup") | |
| 33 | if err != nil { | |
| 34 | return nil, err | |
| 35 | } | |
| 36 | own, err := ownCgroupPath(string(raw)) | |
| 37 | if err != nil { | |
| 38 | return nil, err | |
| 39 | } | |
| 40 | root := filepath.Join("/sys/fs/cgroup", own) | |
| 41 | leaf := filepath.Join(root, "runner") | |
| 42 | if err := os.MkdirAll(leaf, 0o755); err != nil { | |
| 43 | return nil, fmt.Errorf("%s is not writable; the unit needs Delegate=yes: %w", root, err) | |
| 44 | } | |
| 45 | if err := os.WriteFile(filepath.Join(leaf, "cgroup.procs"), []byte(strconv.Itoa(os.Getpid())), 0o644); err != nil { | |
| 46 | return nil, fmt.Errorf("moving into %s: %w", leaf, err) | |
| 47 | } | |
| 48 | if err := os.WriteFile(filepath.Join(root, "cgroup.subtree_control"), []byte(cgroupControllers), 0o644); err != nil { | |
| 49 | return nil, fmt.Errorf("enabling controllers on %s: %w", root, err) | |
| 50 | } | |
| 51 | builds := filepath.Join(root, "builds") | |
| 52 | if err := os.MkdirAll(builds, 0o755); err != nil { | |
| 53 | return nil, err | |
| 54 | } | |
| 55 | if err := os.WriteFile(filepath.Join(builds, "cgroup.subtree_control"), []byte(cgroupControllers), 0o644); err != nil { | |
| 56 | return nil, fmt.Errorf("enabling controllers on %s: %w", builds, err) | |
| 57 | } | |
| 58 | return &buildCgroups{builds: builds}, nil | |
| 59 | } | |
| 60 | ||
| 61 | // create makes the cgroup for one build with its limits written, and | |
| 62 | // returns its path and an open directory fd for placing processes. | |
| 63 | func (c *buildCgroups) create(id int64, memory, cpus string) (string, *os.File, error) { | |
| 64 | dir := filepath.Join(c.builds, fmt.Sprintf("build-%d", id)) | |
| 65 | if err := os.Mkdir(dir, 0o755); err != nil { | |
| 66 | return "", nil, err | |
| 67 | } | |
| 68 | if err := writeLimits(dir, memory, cpus); err != nil { | |
| 69 | os.Remove(dir) | |
| 70 | return "", nil, err | |
| 71 | } | |
| 72 | f, err := os.Open(dir) | |
| 73 | if err != nil { | |
| 74 | os.Remove(dir) | |
| 75 | return "", nil, err | |
| 76 | } | |
| 77 | return dir, f, nil | |
| 78 | } | |
| 79 | ||
| 80 | // remove kills whatever is still in the build's cgroup — conmon, the | |
| 81 | // pause process, a step's stray child — and removes it. rmdir fails | |
| 82 | // until the kernel has reaped every process, so it retries briefly. | |
| 83 | func (c *buildCgroups) remove(dir string) { | |
| 84 | if err := os.WriteFile(filepath.Join(dir, "cgroup.kill"), []byte("1"), 0o644); err != nil { | |
| 85 | log.Printf("cgroup %s: kill: %v", dir, err) | |
| 86 | } | |
| 87 | for i := 0; i < 50; i++ { | |
| 88 | if err := os.Remove(dir); err == nil { | |
| 89 | return | |
| 90 | } | |
| 91 | time.Sleep(100 * time.Millisecond) | |
| 92 | } | |
| 93 | log.Printf("cgroup %s: still populated after kill; left in place", dir) | |
| 94 | } | |
| 95 | ||
| 96 | // intoCgroup starts cmd inside the cgroup fd refers to, so podman, | |
| 97 | // conmon and everything they start inherit the build's limits. A podman | |
| 98 | // exec started from the runner's own cgroup lands there, outside the | |
| 99 | // limit, which is why every invocation for a build goes through this. | |
| 100 | func intoCgroup(cmd *exec.Cmd, f *os.File) { | |
| 101 | if f == nil { | |
| 102 | return | |
| 103 | } | |
| 104 | if cmd.SysProcAttr == nil { | |
| 105 | cmd.SysProcAttr = &syscall.SysProcAttr{} | |
| 106 | } | |
| 107 | cmd.SysProcAttr.UseCgroupFD = true | |
| 108 | cmd.SysProcAttr.CgroupFD = int(f.Fd()) | |
| 109 | } | |
cmd/gitbay-runner/cgroup_other.go added +23
| @@ -0,0 +1,23 @@ | ||
| 1 | //go:build !linux | |
| 2 | ||
| 3 | package main | |
| 4 | ||
| 5 | import ( | |
| 6 | "errors" | |
| 7 | "os" | |
| 8 | "os/exec" | |
| 9 | ) | |
| 10 | ||
| 11 | type buildCgroups struct{} | |
| 12 | ||
| 13 | func prepareBuildCgroups() (*buildCgroups, error) { | |
| 14 | return nil, errors.New("build cgroups need Linux") | |
| 15 | } | |
| 16 | ||
| 17 | func (c *buildCgroups) create(id int64, memory, cpus string) (string, *os.File, error) { | |
| 18 | return "", nil, errors.New("build cgroups need Linux") | |
| 19 | } | |
| 20 | ||
| 21 | func (c *buildCgroups) remove(dir string) {} | |
| 22 | ||
| 23 | func intoCgroup(cmd *exec.Cmd, f *os.File) {} | |
cmd/gitbay-runner/cgroup_test.go added +92
| @@ -0,0 +1,92 @@ | ||
| 1 | package main | |
| 2 | ||
| 3 | import ( | |
| 4 | "os" | |
| 5 | "path/filepath" | |
| 6 | "testing" | |
| 7 | ) | |
| 8 | ||
| 9 | // Podman's --memory and --cpus never applied under rootless cgroupfs: the | |
| 10 | // container ran in the service's own cgroup and crun could not create a | |
| 11 | // child (#188). The runner now owns the build cgroups itself and places | |
| 12 | // podman inside one, so the limits are written by the runner in cgroup | |
| 13 | // v2's own units. | |
| 14 | func TestMemoryBytes(t *testing.T) { | |
| 15 | cases := map[string]int64{ | |
| 16 | "64m": 64 << 20, | |
| 17 | "6g": 6 << 30, | |
| 18 | "512k": 512 << 10, | |
| 19 | "100b": 100, | |
| 20 | "4096": 4096, | |
| 21 | "1G": 1 << 30, | |
| 22 | } | |
| 23 | for in, want := range cases { | |
| 24 | got, err := memoryBytes(in) | |
| 25 | if err != nil || got != want { | |
| 26 | t.Errorf("memoryBytes(%q) = %d, %v; want %d", in, got, err, want) | |
| 27 | } | |
| 28 | } | |
| 29 | for _, bad := range []string{"", "lots", "6gb", "-1g", "1.5g"} { | |
| 30 | if _, err := memoryBytes(bad); err == nil { | |
| 31 | t.Errorf("memoryBytes(%q) accepted", bad) | |
| 32 | } | |
| 33 | } | |
| 34 | } | |
| 35 | ||
| 36 | func TestCPUMax(t *testing.T) { | |
| 37 | cases := map[string]string{ | |
| 38 | "3": "300000 100000", | |
| 39 | "1": "100000 100000", | |
| 40 | "1.5": "150000 100000", | |
| 41 | "0.25": "25000 100000", | |
| 42 | } | |
| 43 | for in, want := range cases { | |
| 44 | got, err := cpuMax(in) | |
| 45 | if err != nil || got != want { | |
| 46 | t.Errorf("cpuMax(%q) = %q, %v; want %q", in, got, err, want) | |
| 47 | } | |
| 48 | } | |
| 49 | for _, bad := range []string{"", "0", "-1", "two"} { | |
| 50 | if _, err := cpuMax(bad); err == nil { | |
| 51 | t.Errorf("cpuMax(%q) accepted", bad) | |
| 52 | } | |
| 53 | } | |
| 54 | } | |
| 55 | ||
| 56 | // /proc/self/cgroup on cgroup v2 is one line, "0::<path>". | |
| 57 | func TestOwnCgroupPath(t *testing.T) { | |
| 58 | got, err := ownCgroupPath("0::/system.slice/gitbay-runner.service\n") | |
| 59 | if err != nil || got != "/system.slice/gitbay-runner.service" { | |
| 60 | t.Fatalf("ownCgroupPath = %q, %v", got, err) | |
| 61 | } | |
| 62 | // A v1 hierarchy, or anything else, is not something the runner | |
| 63 | // manages. | |
| 64 | if _, err := ownCgroupPath("12:memory:/user.slice\n0::/init.scope\n"); err == nil { | |
| 65 | t.Fatal("v1 hierarchy accepted") | |
| 66 | } | |
| 67 | } | |
| 68 | ||
| 69 | // Limits land as cgroup v2 interface files in the build's directory; | |
| 70 | // an unset limit writes nothing, which means "inherit", not "max". | |
| 71 | func TestWriteLimits(t *testing.T) { | |
| 72 | dir := t.TempDir() | |
| 73 | if err := writeLimits(dir, "6g", "3"); err != nil { | |
| 74 | t.Fatal(err) | |
| 75 | } | |
| 76 | if got, _ := os.ReadFile(filepath.Join(dir, "memory.max")); string(got) != "6442450944" { | |
| 77 | t.Errorf("memory.max = %q", got) | |
| 78 | } | |
| 79 | if got, _ := os.ReadFile(filepath.Join(dir, "cpu.max")); string(got) != "300000 100000" { | |
| 80 | t.Errorf("cpu.max = %q", got) | |
| 81 | } | |
| 82 | empty := t.TempDir() | |
| 83 | if err := writeLimits(empty, "", ""); err != nil { | |
| 84 | t.Fatal(err) | |
| 85 | } | |
| 86 | if entries, _ := os.ReadDir(empty); len(entries) != 0 { | |
| 87 | t.Errorf("unset limits wrote %v", entries) | |
| 88 | } | |
| 89 | if err := writeLimits(t.TempDir(), "lots", ""); err == nil { | |
| 90 | t.Error("a bad memory limit was accepted") | |
| 91 | } | |
| 92 | } | |
cmd/gitbay-runner/env_test.go −13
| @@ -118,19 +118,6 @@ func TestEnvHomeFindsHome(t *testing.T) { | ||
| 118 | 118 | } |
| 119 | 119 | } |
| 120 | 120 | |
| 121 | // A limit is passed to podman only when set; unset means uncapped, not a | |
| 122 | // default that could kill the suite. | |
| 123 | func TestLimitArgs(t *testing.T) { | |
| 124 | if got := (&runner{}).limitArgs(); len(got) != 0 { | |
| 125 | t.Errorf("no limits set, got %v", got) | |
| 126 | } | |
| 127 | got := (&runner{memory: "4g", cpus: "2"}).limitArgs() | |
| 128 | want := []string{"--memory", "4g", "--cpus", "2"} | |
| 129 | if strings.Join(got, " ") != strings.Join(want, " ") { | |
| 130 | t.Errorf("limitArgs = %v, want %v", got, want) | |
| 131 | } | |
| 132 | } | |
| 133 | ||
| 134 | 121 | // Closing stop drains: the build in flight finishes and is reported, and |
| 135 | 122 | // no further build is claimed (#179). |
| 136 | 123 | func TestServeDrainsOnStop(t *testing.T) { |
cmd/gitbay-runner/isolate.go +19 −19
| @@ -122,11 +122,26 @@ func (r *runner) runStepsPodman(j job, dir string, env []string, sink io.Writer, | ||
| 122 | 122 | } |
| 123 | 123 | defer os.Remove(envFile) |
| 124 | 124 | |
| 125 | // The build's cgroup carries its limits; podman's own cgroup handling | |
| 126 | // is off because it never worked here (#188). Every podman process | |
| 127 | // for this build starts inside the cgroup, exec included: one started | |
| 128 | // from the runner's cgroup would run the step outside the limit. | |
| 129 | var cgroupFD *os.File | |
| 130 | if r.cgroups != nil { | |
| 131 | dir, f, err := r.cgroups.create(j.ID, r.memory, r.cpus) | |
| 132 | if err != nil { | |
| 133 | fmt.Fprintf(sink, "preparing the build cgroup: %v\n", err) | |
| 134 | return false | |
| 135 | } | |
| 136 | cgroupFD = f | |
| 137 | defer f.Close() | |
| 138 | defer r.cgroups.remove(dir) | |
| 139 | } | |
| 140 | ||
| 125 | 141 | name := fmt.Sprintf("gitbay-build-%d", j.ID) |
| 126 | 142 | // --rm so a container cannot outlive its build; the explicit rm below |
| 127 | 143 | // covers the case where the daemon-less run itself fails. |
| 128 | args := append(r.podmanGlobal(), "run", "--detach", "--rm", "--pull=never") | |
| 129 | args = append(args, r.limitArgs()...) | |
| 144 | args := append(r.podmanGlobal(), "run", "--detach", "--rm", "--pull=never", "--cgroups=disabled") | |
| 130 | 145 | args = append(args, |
| 131 | 146 | "--name", name, |
| 132 | 147 | "--env-file", envFile, |
| @@ -142,6 +157,7 @@ func (r *runner) runStepsPodman(j job, dir string, env []string, sink io.Writer, | ||
| 142 | 157 | image, "-c", "sleep infinity") |
| 143 | 158 | start := exec.Command(podman, args...) |
| 144 | 159 | start.Env = []string{"PATH=" + os.Getenv("PATH"), "HOME=" + r.podmanHome()} |
| 160 | intoCgroup(start, cgroupFD) | |
| 145 | 161 | if out, err := start.CombinedOutput(); err != nil { |
| 146 | 162 | // A missing image lands here, and it is the common case worth |
| 147 | 163 | // explaining: this runner never pulls, so an image it does not |
| @@ -162,6 +178,7 @@ func (r *runner) runStepsPodman(j job, dir string, env []string, sink io.Writer, | ||
| 162 | 178 | fmt.Fprintf(sink, "$ %s\n", step) |
| 163 | 179 | cmd := exec.Command(podman, append(r.podmanGlobal(), "exec", "--workdir", "/workspace", name, "sh", "-c", step)...) |
| 164 | 180 | cmd.Env = []string{"PATH=" + os.Getenv("PATH"), "HOME=" + r.podmanHome()} |
| 181 | intoCgroup(cmd, cgroupFD) | |
| 165 | 182 | cmd.Stdout, cmd.Stderr = sink, sink |
| 166 | 183 | if ok, why := runStep(cmd, deadline); !ok { |
| 167 | 184 | fmt.Fprintf(sink, "%s\n", why) |
| @@ -189,23 +206,6 @@ func (r *runner) podmanGlobal() []string { | ||
| 189 | 206 | return []string{"--cgroup-manager=cgroupfs"} |
| 190 | 207 | } |
| 191 | 208 | |
| 192 | // limitArgs caps one build's container. The service's CPUWeight and | |
| 193 | // IOWeight shape the service against other services, not one build | |
| 194 | // against the host, and the threat model lists resource exhaustion as | |
| 195 | // unaddressed. Memory is deliberately uncapped by default: the e2e suite | |
| 196 | // peaks past 5GB on a 7GB host, and a cap that kills the suite is an | |
| 197 | // outage, not a limit. | |
| 198 | func (r *runner) limitArgs() []string { | |
| 199 | var args []string | |
| 200 | if r.memory != "" { | |
| 201 | args = append(args, "--memory", r.memory) | |
| 202 | } | |
| 203 | if r.cpus != "" { | |
| 204 | args = append(args, "--cpus", r.cpus) | |
| 205 | } | |
| 206 | return args | |
| 207 | } | |
| 208 | ||
| 209 | 209 | // envHome returns the HOME the step environment carries. |
| 210 | 210 | func envHome(env []string) string { |
| 211 | 211 | for _, e := range env { |
cmd/gitbay-runner/main.go +23 −3
| @@ -50,9 +50,12 @@ type runner struct { | ||
| 50 | 50 | // isolation selects how steps run: "podman" or "none". |
| 51 | 51 | image string |
| 52 | 52 | isolation string |
| 53 | // memory and cpus cap one build's container; empty means no cap. | |
| 53 | // memory and cpus cap one build's cgroup; empty means no cap. | |
| 54 | 54 | memory string |
| 55 | 55 | cpus string |
| 56 | // cgroups is the runner's build cgroup subtree, nil where the unit | |
| 57 | // is not delegated and builds run unconfined in the service cgroup. | |
| 58 | cgroups *buildCgroups | |
| 56 | 59 | // stepFn is step, replaceable by tests. |
| 57 | 60 | stepFn func() (bool, error) |
| 58 | 61 | // repos limits which repositories this runner claims builds for. Empty |
| @@ -74,8 +77,8 @@ func main() { | ||
| 74 | 77 | jobs = flag.Int("jobs", 1, "builds to run at once") |
| 75 | 78 | image = flag.String("image", "", "default container image for jobs that name none") |
| 76 | 79 | isolation = flag.String("isolation", "podman", "how steps run: podman, or none for no container") |
| 77 | memory = flag.String("memory", "", "memory limit per build container, e.g. 4g (podman only; default unlimited)") | |
| 78 | cpus = flag.String("cpus", "", "CPU limit per build container, e.g. 2 (podman only; default unlimited)") | |
| 80 | memory = flag.String("memory", "", "memory limit per build, e.g. 4g (podman only, needs a delegated cgroup; default unlimited)") | |
| 81 | cpus = flag.String("cpus", "", "CPU limit per build, e.g. 2 (podman only, needs a delegated cgroup; default unlimited)") | |
| 79 | 82 | version = flag.Bool("version", false, "print the commit this binary was built from, then exit") |
| 80 | 83 | ) |
| 81 | 84 | flag.Parse() |
| @@ -96,6 +99,23 @@ func main() { | ||
| 96 | 99 | memory: *memory, |
| 97 | 100 | cpus: *cpus, |
| 98 | 101 | } |
| 102 | if r.isolation == isolationPodman { | |
| 103 | // Before podman runs anything: its pause process lands in the | |
| 104 | // cgroup of the first invocation, and that must be the runner's | |
| 105 | // leaf, not a build's. | |
| 106 | cg, err := prepareBuildCgroups() | |
| 107 | switch { | |
| 108 | case err == nil: | |
| 109 | r.cgroups = cg | |
| 110 | case r.memory != "" || r.cpus != "": | |
| 111 | // Limits that cannot be applied are refused, not dropped: | |
| 112 | // a runner that accepted -memory and ran uncapped is what | |
| 113 | // #188 was. | |
| 114 | log.Fatalf("-memory/-cpus: build cgroups unavailable: %v", err) | |
| 115 | default: | |
| 116 | log.Printf("build cgroups unavailable (%v); builds run unconfined in the service cgroup", err) | |
| 117 | } | |
| 118 | } | |
| 99 | 119 | if err := r.checkIsolation(); err != nil { |
| 100 | 120 | // Refusing to start is the point. A runner that quietly fell back |
| 101 | 121 | // to running repository code on the host would drop isolation |
cmd/gitbay-runner/proc_unix.go +4 −1
| @@ -13,7 +13,10 @@ import ( | ||
| 13 | 13 | // child: dash forks a single command rather than exec'ing it, so on a |
| 14 | 14 | // Debian host "sh -c 'sleep 120'" survived its shell by two minutes. |
| 15 | 15 | func ownProcessGroup(cmd *exec.Cmd) { |
| 16 | cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true} | |
| 16 | if cmd.SysProcAttr == nil { | |
| 17 | cmd.SysProcAttr = &syscall.SysProcAttr{} | |
| 18 | } | |
| 19 | cmd.SysProcAttr.Setpgid = true | |
| 17 | 20 | } |
| 18 | 21 | |
| 19 | 22 | func killTree(cmd *exec.Cmd) { |
deploy/gitbay-runner.override.conf +13 −12
| @@ -27,19 +27,20 @@ | ||
| 27 | 27 | # repositories never claims a build it is not scoped to, so the canary |
| 28 | 28 | # must be listed or its scheduled build waits forever. |
| 29 | 29 | # |
| 30 | # Resource caps are on the unit, not on the container. Under rootless | |
| 31 | # podman with the cgroupfs manager the container runs inside this | |
| 32 | # service's own cgroup: no child cgroup is created, so podman's --cpus | |
| 33 | # and --memory are accepted and never applied (#188). MemoryMax and | |
| 34 | # CPUQuota below bound the runner and every build together, which is | |
| 35 | # what keeps the forge alive when a build allocates without bound. | |
| 30 | # Two layers of resource caps. MemoryMax and CPUQuota bound the unit — | |
| 31 | # the runner and every build together — which is what keeps the forge | |
| 32 | # alive when a build allocates without bound. -memory and -cpus on | |
| 33 | # ExecStart cap each build's own cgroup, which the runner creates under | |
| 34 | # this unit's delegated cgroup (Delegate=yes below); podman's own --memory | |
| 35 | # and --cpus never applied here, since under rootless cgroupfs the | |
| 36 | # container ran in the service cgroup itself (#188). | |
| 36 | 37 | # |
| 37 | 38 | # 6G of the host's 7.7GB, no swap: the e2e suite peaks past 5GB, so the |
| 38 | # cap sits above that rather than at a fair share. CPUQuota=300% is | |
| 39 | # three of the four cores, leaving one for gitbayd and sshd (#144, #184). | |
| 40 | # OOMPolicy=continue: systemd's default stops the whole service when any | |
| 41 | # process in it is OOM-killed, which would end the runner mid-build; the | |
| 42 | # build fails and the runner carries on. | |
| 39 | # cap sits above that rather than at a fair share. CPUQuota=300% and | |
| 40 | # -cpus 3 are three of the four cores, leaving one for gitbayd and sshd | |
| 41 | # (#144, #184). OOMPolicy=continue: systemd's default stops the whole | |
| 42 | # service when any process in it is OOM-killed, which would end the | |
| 43 | # runner mid-build; the build fails and the runner carries on. | |
| 43 | 44 | MemoryMax=6G |
| 44 | 45 | CPUQuota=300% |
| 45 | 46 | OOMPolicy=continue |
| @@ -70,7 +71,7 @@ TimeoutStopSec=50min | ||
| 70 | 71 | # when it exits is killed. |
| 71 | 72 | KillMode=mixed |
| 72 | 73 | ExecStart= |
| 73 | ExecStart=/usr/local/bin/gitbay-runner -remote git@127.0.0.1 -workdir /var/lib/gitbay-runner/work -poll 5s -timeout 45m -repos krz/gitbay,cmc/ci-smoke -isolation podman -image localhost/gitbay-ci:1 | |
| 74 | ExecStart=/usr/local/bin/gitbay-runner -remote git@127.0.0.1 -workdir /var/lib/gitbay-runner/work -poll 5s -timeout 45m -repos krz/gitbay,cmc/ci-smoke -isolation podman -image localhost/gitbay-ci:1 -cpus 3 -memory 6g | |
| 74 | 75 | Nice=10 |
| 75 | 76 | CPUWeight=30 |
| 76 | 77 | IOWeight=30 |