deploy/gitbay-runner.override.conf

  1# Drop-in for gitbay-runner.service, installed by `make deploy-runner` to
  2# /etc/systemd/system/gitbay-runner.service.d/override.conf.
  3#
  4# A build must never starve the host: the e2e suite alone starts sixty
  5# daemon instances, and with nothing holding it back a deploy's scp on
  6# the admin sshd stalled at 1%. Lower CPU and IO weight keep sshd,
  7# gitbayd and the backup timers responsive while a build runs.
  8#
  9# These weights are for the service, not per build, so `-jobs N` divides
 10# them among N builds rather than taking N times as much. Raising -jobs
 11# does not need them raised; it makes each build slower, not the host
 12# busier.
 13#
 14# A build runs whatever the repository's ci.yml says, as the runner's
 15# own user. Keep that user unprivileged: its key is added with
 16# `keys add --scope runner`, which confines it to the runner protocol
 17# and read-only git, and the sandboxing below keeps a step from
 18# touching the system outside its workspace.
 19#
 20# Delegate=yes and the storage path below are what rootless podman needs
 21# (#144): it manages its own cgroups for a container, and its image and
 22# container store lives under the runner's home, which ProtectSystem
 23# would otherwise make read-only. Prepare the host with
 24# deploy/runner-podman-setup.sh before deploying a runner that isolates.
 25[Unit]
 26# The host egress rule (#260, gitbay-runner-egress.nft) limits what this
 27# unit's user reaches on the host: 127.0.0.1:22 for the runner, the
 28# forge's public 22, 80 and 443 for builds, nothing else. The builds
 29# table (ExecStartPre below) narrows that per build. Required, so
 30# the runner does not start without it: a table that failed to load must
 31# not mean builds reach the admin sshd.
 32Requires=gitbay-runner-egress.service
 33After=gitbay-runner-egress.service
 34
 35[Service]
 36# The runner polls as a non-admin account with a runner-scoped key, and
 37# claims only the repositories that key is attached to (`repo runner
 38# add`): krz/gitbay, krz/hutch, krz/keycask, krz/orgo, krz/skunky-art
 39# and cmc/cleberg.net. The attachments are the boundary, so ExecStart
 40# names no -repos. To validate a runner change, create a scratch
 41# repository, attach this key to it, and run with -repos naming only
 42# that repository until the change is proven (Admin wiki, CI runner).
 43#
 44# Two layers of resource caps. MemoryMax and CPUQuota bound the unit —
 45# the runner and every build together — which is what keeps the forge
 46# alive when a build allocates without bound. -memory and -cpus on
 47# ExecStart cap each build's own cgroup, which the runner creates under
 48# this unit's delegated cgroup (Delegate=yes below); podman's own --memory
 49# and --cpus never applied here, since under rootless cgroupfs the
 50# container ran in the service cgroup itself (#188).
 51#
 52# 6G of the host's 7.7GB, no swap: the e2e suite peaks past 5GB, so the
 53# cap sits above that rather than at a fair share. CPUQuota=300% and
 54# -cpus 3 are three of the four cores, leaving one for gitbayd and sshd
 55# (#144, #184). OOMPolicy=continue: systemd's default stops the whole
 56# service when any process in it is OOM-killed, which would end the
 57# runner mid-build; the build fails and the runner carries on.
 58MemoryMax=6G
 59CPUQuota=300%
 60OOMPolicy=continue
 61#
 62# ExecStart is overridden here rather than left in the unit so the flags
 63# and the sandboxing that has to match them live in one file: -isolation
 64# podman needs NoNewPrivileges=no below, and -image needs an image the
 65# host has been given (deploy/runner-podman-setup.sh, Containerfile.ci).
 66# podman's run root is pinned under the runner's home by storage.conf
 67# (runner-podman-setup.sh), not taken from XDG_RUNTIME_DIR or /tmp: this
 68# unit has PrivateTmp, so a /tmp run root is a per-instance tmpfs.
 69#
 70# The cgroupfs manager puts podman's pause process under the user slice,
 71# outside this unit's cgroup, so a stop does not end it and the next
 72# start joins its namespaces — including a /tmp that no longer exists.
 73# End it with the service.
 74ExecStopPost=-/usr/bin/pkill -u ci-runner -x catatonit
 75# The builds table (#260, gitbay-runner-builds.nft) matches a build's
 76# traffic by the cgroup the runner starts it in, and nftables turns a
 77# cgroup path into the cgroup's id when the table loads. This unit's
 78# cgroup is new on every start, so the two class cgroups are created
 79# here, handed to the runner's user, and the table loaded against them,
 80# before the runner starts. As root (+), so the mkdir does not depend on
 81# when systemd hands the delegated cgroup to the unit's user. A table
 82# that fails to load stops the start, as the uid table's Requires= does.
 83ExecStartPre=+/usr/bin/mkdir -p /sys/fs/cgroup/system.slice/gitbay-runner.service/builds/trusted /sys/fs/cgroup/system.slice/gitbay-runner.service/builds/untrusted
 84ExecStartPre=+/usr/bin/chown -R ci-runner:ci-runner /sys/fs/cgroup/system.slice/gitbay-runner.service/builds
 85ExecStartPre=+/usr/sbin/nft -f /etc/gitbay-runner/builds.nft
 86# On stop the runner drains: it claims nothing more and finishes the
 87# build in flight, then exits. Give it long enough — the per-build limit
 88# is 45m plus half a minute of report retries — before systemd kills it.
 89# `make deploy-runner` therefore waits for a running build (#179).
 90TimeoutStopSec=50min
 91# The drain only works if the stop signal reaches the runner alone. The
 92# default control-group mode sends SIGTERM to every process in the
 93# cgroup at once — the ssh session streaming the log and the build's
 94# container with it — so the runner drained a build whose steps were
 95# already dead. mixed signals the main process only; whatever is left
 96# when it exits is killed.
 97KillMode=mixed
 98# -untrusted: this runner isolates in podman, so it takes merge request
 99# heads from forks; a runner without a container must not.
100ExecStart=
101ExecStart=/usr/local/bin/gitbay-runner -remote git@127.0.0.1 -workdir /var/lib/gitbay-runner/work -poll 5s -timeout 45m -isolation podman -image localhost/gitbay-ci:2 -cpus 3 -memory 6g -untrusted
102Nice=10
103CPUWeight=30
104IOWeight=30
105# NoNewPrivileges is off, and that is a deliberate trade (#144).
106#
107# Rootless podman sets up its user namespace with newuidmap, a setuid
108# helper; NoNewPrivileges=yes blocks it and podman fails with
109# "newuidmap: write to uid_map failed: Operation not permitted", so the
110# runner refuses to start. The choice is between this flag and running
111# builds in containers at all.
112#
113# Containers are the stronger boundary by a wide margin. NoNewPrivileges
114# constrained a process that was already executing arbitrary repository
115# code as this user; a container confines that code to an image and a
116# bind-mounted workspace. What is lost is one hardening layer on the
117# runner process itself, which is ours rather than a build's — a build no
118# longer runs in this process's context at all.
119#
120# Under -isolation none there is no container, and this flag should be
121# yes. Set it back if you run that way.
122NoNewPrivileges=no
123ProtectSystem=full
124# ProtectKernelTunables is off, for the same reason NoNewPrivileges is
125# (#144). It overmounts /proc/sys and friends in this unit's namespace,
126# and the kernel then refuses a fresh proc mount in any child user
127# namespace ("mount too revealing"): crun fails with "mount `proc` to
128# `proc`: Operation not permitted". There is no podman setting for it.
129# What the flag protected — /proc/sys from a build running on the host
130# as this user — the container now covers: a build gets its own proc,
131# with those paths masked by the runtime. Under -isolation none, set it
132# back to yes.
133# ProtectControlGroups is off because the runner writes cgroups: it
134# creates one per build under this unit's delegated cgroup to carry
135# -memory and -cpus (#188). The flag mounts /sys/fs/cgroup read-only in
136# the unit's namespace, which makes even a delegated cgroup unwritable,
137# and the runner then refuses to start when a limit is set. Delegate=yes
138# already hands this unit its subtree; what the flag protected beyond
139# that is other units' cgroups, which are root-owned and not writable
140# by this user regardless.
141ProtectControlGroups=no
142RestrictSUIDSGID=yes
143Delegate=yes
144# The runner's home is /var/lib/gitbay-runner (see the Admin page), and
145# the leading - makes a missing path ignored rather than fatal: this
146# drop-in installs on hosts that have not been prepared for podman yet,
147# and a unit that refuses to start would stop every build on the
148# instance.
149ReadWritePaths=-/var/lib/gitbay-runner/.local/share/containers -/var/lib/gitbay-runner/.config/containers