Commit 89eba6adea
Verified · cmc
Layout: unified · split
.gitbay/wiki/Admin.org +4
| @@ -438,6 +438,10 @@ because an image this host does not have would fail every build. A job | |||
| 438 | overrides it with =image:= in =.gitbay/ci.yml=, validated as a reference | 438 | overrides it with =image:= in =.gitbay/ci.yml=, validated as a reference |
| 439 | so a config file cannot turn it into podman arguments. | 439 | so a config file cannot turn it into podman arguments. |
| 440 | 440 | ||
| 441 | =-cpus= and =-memory= cap one build's container (podman's own units, | ||
| 442 | e.g. =-cpus 2 -memory 4g=); unset means uncapped. bay1 runs =-cpus 3= | ||
| 443 | and no memory cap, since the e2e suite needs most of the host's 7GB. | ||
| 444 | |||
| 441 | *Images are provisioned, never pulled by a build.* The runner passes | 445 | *Images are provisioned, never pulled by a build.* The runner passes |
| 442 | =--pull=never=. Two reasons, and the second is the better one: the | 446 | =--pull=never=. Two reasons, and the second is the better one: the |
| 443 | service runs with =RestrictSUIDSGID=yes= so podman cannot unpack a layer | 447 | service runs with =RestrictSUIDSGID=yes= so podman cannot unpack a layer |
cmd/gitbay-runner/env_test.go +13
| @@ -116,3 +116,16 @@ func TestEnvHomeFindsHome(t *testing.T) { | |||
| 116 | t.Errorf("envHome with no HOME = %q, want empty", got) | 116 | t.Errorf("envHome with no HOME = %q, want empty", got) |
| 117 | } | 117 | } |
| 118 | } | 118 | } |
| 119 | |||
| 120 | // A limit is passed to podman only when set; unset means uncapped, not a | ||
| 121 | // default that could kill the suite. | ||
| 122 | func TestLimitArgs(t *testing.T) { | ||
| 123 | if got := (&runner{}).limitArgs(); len(got) != 0 { | ||
| 124 | t.Errorf("no limits set, got %v", got) | ||
| 125 | } | ||
| 126 | got := (&runner{memory: "4g", cpus: "2"}).limitArgs() | ||
| 127 | want := []string{"--memory", "4g", "--cpus", "2"} | ||
| 128 | if strings.Join(got, " ") != strings.Join(want, " ") { | ||
| 129 | t.Errorf("limitArgs = %v, want %v", got, want) | ||
| 130 | } | ||
| 131 | } | ||
cmd/gitbay-runner/isolate.go +23 −4
| @@ -125,8 +125,9 @@ func (r *runner) runStepsPodman(j job, dir string, env []string, sink io.Writer, | |||
| 125 | name := fmt.Sprintf("gitbay-build-%d", j.ID) | 125 | name := fmt.Sprintf("gitbay-build-%d", j.ID) |
| 126 | // --rm so a container cannot outlive its build; the explicit rm below | 126 | // --rm so a container cannot outlive its build; the explicit rm below |
| 127 | // covers the case where the daemon-less run itself fails. | 127 | // covers the case where the daemon-less run itself fails. |
| 128 | start := exec.Command(podman, append(r.podmanGlobal(), "run", "--detach", "--rm", | 128 | args := append(r.podmanGlobal(), "run", "--detach", "--rm", "--pull=never") |
| 129 | "--pull=never", | 129 | args = append(args, r.limitArgs()...) |
| 130 | args = append(args, | ||
| 130 | "--name", name, | 131 | "--name", name, |
| 131 | "--env-file", envFile, | 132 | "--env-file", envFile, |
| 132 | "--volume", dir+":/workspace:rw", | 133 | "--volume", dir+":/workspace:rw", |
| @@ -138,7 +139,8 @@ func (r *runner) runStepsPodman(j job, dir string, env []string, sink io.Writer, | |||
| 138 | "--volume", envHome(env)+":"+envHome(env)+":rw", | 139 | "--volume", envHome(env)+":"+envHome(env)+":rw", |
| 139 | "--workdir", "/workspace", | 140 | "--workdir", "/workspace", |
| 140 | "--entrypoint", "sh", | 141 | "--entrypoint", "sh", |
| 141 | image, "-c", "sleep infinity")...) | 142 | image, "-c", "sleep infinity") |
| 143 | start := exec.Command(podman, args...) | ||
| 142 | start.Env = []string{"PATH=" + os.Getenv("PATH"), "HOME=" + r.podmanHome()} | 144 | start.Env = []string{"PATH=" + os.Getenv("PATH"), "HOME=" + r.podmanHome()} |
| 143 | if out, err := start.CombinedOutput(); err != nil { | 145 | if out, err := start.CombinedOutput(); err != nil { |
| 144 | // A missing image lands here, and it is the common case worth | 146 | // A missing image lands here, and it is the common case worth |
| @@ -187,7 +189,24 @@ func (r *runner) podmanGlobal() []string { | |||
| 187 | return []string{"--cgroup-manager=cgroupfs"} | 189 | return []string{"--cgroup-manager=cgroupfs"} |
| 188 | } | 190 | } |
| 189 | 191 | ||
| 190 | // env_home returns the HOME the step environment carries. | 192 | // limitArgs caps one build's container. The service's CPUWeight and |
| 193 | // IOWeight shape the service against other services, not one build | ||
| 194 | // against the host, and the threat model lists resource exhaustion as | ||
| 195 | // unaddressed. Memory is deliberately uncapped by default: the e2e suite | ||
| 196 | // peaks past 5GB on a 7GB host, and a cap that kills the suite is an | ||
| 197 | // outage, not a limit. | ||
| 198 | func (r *runner) limitArgs() []string { | ||
| 199 | var args []string | ||
| 200 | if r.memory != "" { | ||
| 201 | args = append(args, "--memory", r.memory) | ||
| 202 | } | ||
| 203 | if r.cpus != "" { | ||
| 204 | args = append(args, "--cpus", r.cpus) | ||
| 205 | } | ||
| 206 | return args | ||
| 207 | } | ||
| 208 | |||
| 209 | // envHome returns the HOME the step environment carries. | ||
| 191 | func envHome(env []string) string { | 210 | func envHome(env []string) string { |
| 192 | for _, e := range env { | 211 | for _, e := range env { |
| 193 | if strings.HasPrefix(e, "HOME=") { | 212 | if strings.HasPrefix(e, "HOME=") { |
cmd/gitbay-runner/main.go +7
| @@ -49,6 +49,9 @@ type runner struct { | |||
| 49 | // isolation selects how steps run: "podman" or "none". | 49 | // isolation selects how steps run: "podman" or "none". |
| 50 | image string | 50 | image string |
| 51 | isolation string | 51 | isolation string |
| 52 | // memory and cpus cap one build's container; empty means no cap. | ||
| 53 | memory string | ||
| 54 | cpus string | ||
| 52 | // repos limits which repositories this runner claims builds for. Empty | 55 | // repos limits which repositories this runner claims builds for. Empty |
| 53 | // means any, which is what a runner on the server itself wants; a runner | 56 | // means any, which is what a runner on the server itself wants; a runner |
| 54 | // somewhere that should not execute every repository's steps names them. | 57 | // somewhere that should not execute every repository's steps names them. |
| @@ -68,6 +71,8 @@ func main() { | |||
| 68 | jobs = flag.Int("jobs", 1, "builds to run at once") | 71 | jobs = flag.Int("jobs", 1, "builds to run at once") |
| 69 | image = flag.String("image", "", "default container image for jobs that name none") | 72 | image = flag.String("image", "", "default container image for jobs that name none") |
| 70 | isolation = flag.String("isolation", "podman", "how steps run: podman, or none for no container") | 73 | isolation = flag.String("isolation", "podman", "how steps run: podman, or none for no container") |
| 74 | memory = flag.String("memory", "", "memory limit per build container, e.g. 4g (podman only; default unlimited)") | ||
| 75 | cpus = flag.String("cpus", "", "CPU limit per build container, e.g. 2 (podman only; default unlimited)") | ||
| 71 | version = flag.Bool("version", false, "print the commit this binary was built from, then exit") | 76 | version = flag.Bool("version", false, "print the commit this binary was built from, then exit") |
| 72 | ) | 77 | ) |
| 73 | flag.Parse() | 78 | flag.Parse() |
| @@ -85,6 +90,8 @@ func main() { | |||
| 85 | timeout: *timeout, | 90 | timeout: *timeout, |
| 86 | image: *image, | 91 | image: *image, |
| 87 | isolation: *isolation, | 92 | isolation: *isolation, |
| 93 | memory: *memory, | ||
| 94 | cpus: *cpus, | ||
| 88 | } | 95 | } |
| 89 | if err := r.checkIsolation(); err != nil { | 96 | if err := r.checkIsolation(); err != nil { |
| 90 | // Refusing to start is the point. A runner that quietly fell back | 97 | // Refusing to start is the point. A runner that quietly fell back |
deploy/gitbay-runner.override.conf +5 −1
| @@ -23,6 +23,10 @@ | |||
| 23 | # would otherwise make read-only. Prepare the host with | 23 | # would otherwise make read-only. Prepare the host with |
| 24 | # deploy/runner-podman-setup.sh before deploying a runner that isolates. | 24 | # deploy/runner-podman-setup.sh before deploying a runner that isolates. |
| 25 | [Service] | 25 | [Service] |
| 26 | # -cpus 3 of the host's 4 leaves a core for gitbayd and sshd while a build | ||
| 27 | # runs. No -memory: the e2e suite peaks past 5GB of the 7GB, and a cap | ||
| 28 | # that kills it is an outage rather than a limit (#144). | ||
| 29 | # | ||
| 26 | # ExecStart is overridden here rather than left in the unit so the flags | 30 | # ExecStart is overridden here rather than left in the unit so the flags |
| 27 | # and the sandboxing that has to match them live in one file: -isolation | 31 | # and the sandboxing that has to match them live in one file: -isolation |
| 28 | # podman needs NoNewPrivileges=no below, and -image needs an image the | 32 | # podman needs NoNewPrivileges=no below, and -image needs an image the |
| @@ -37,7 +41,7 @@ | |||
| 37 | # End it with the service. | 41 | # End it with the service. |
| 38 | ExecStopPost=-/usr/bin/pkill -u ci-runner -x catatonit | 42 | ExecStopPost=-/usr/bin/pkill -u ci-runner -x catatonit |
| 39 | ExecStart= | 43 | ExecStart= |
| 40 | ExecStart=/usr/local/bin/gitbay-runner -remote git@127.0.0.1 -workdir /var/lib/gitbay-runner/work -poll 5s -timeout 45m -repos krz/gitbay -isolation podman -image localhost/gitbay-ci:1 | 44 | ExecStart=/usr/local/bin/gitbay-runner -remote git@127.0.0.1 -workdir /var/lib/gitbay-runner/work -poll 5s -timeout 45m -repos krz/gitbay -isolation podman -image localhost/gitbay-ci:1 -cpus 3 |
| 41 | Nice=10 | 45 | Nice=10 |
| 42 | CPUWeight=30 | 46 | CPUWeight=30 |
| 43 | IOWeight=30 | 47 | IOWeight=30 |