deploy/gitbay-runner.override.conf

  1# Drop-in for gitbay-runner.service, installed by `make deploy-runner` to
  2# /etc/systemd/system/gitbay-runner.service.d/override.conf.
  3#
  4# A build must never starve the host: the e2e suite alone starts sixty
  5# daemon instances, and with nothing holding it back a deploy's scp on
  6# the admin sshd stalled at 1%. Lower CPU and IO weight keep sshd,
  7# gitbayd and the backup timers responsive while a build runs.
  8#
  9# These weights are for the service, not per build, so `-jobs N` divides
 10# them among N builds rather than taking N times as much. Raising -jobs
 11# does not need them raised; it makes each build slower, not the host
 12# busier.
 13#
 14# A build runs whatever the repository's ci.yml says, as the runner's
 15# own user. Keep that user unprivileged: its key is added with
 16# `keys add --scope runner`, which confines it to the runner protocol
 17# and read-only git, and the sandboxing below keeps a step from
 18# touching the system outside its workspace.
 19#
 20# Delegate=yes and the storage path below are what rootless podman needs
 21# (#144): it manages its own cgroups for a container, and its image and
 22# container store lives under the runner's home, which ProtectSystem
 23# would otherwise make read-only. Prepare the host with
 24# deploy/runner-podman-setup.sh before deploying a runner that isolates.
 25[Service]
 26# cmc/ci-smoke is the nightly isolation canary; a runner scoped to named
 27# repositories never claims a build it is not scoped to, so the canary
 28# must be listed or its scheduled build waits forever.
 29#
 30# Resource caps are on the unit, not on the container. Under rootless
 31# podman with the cgroupfs manager the container runs inside this
 32# service's own cgroup: no child cgroup is created, so podman's --cpus
 33# and --memory are accepted and never applied (#188). MemoryMax and
 34# CPUQuota below bound the runner and every build together, which is
 35# what keeps the forge alive when a build allocates without bound.
 36#
 37# 6G of the host's 7.7GB, no swap: the e2e suite peaks past 5GB, so the
 38# cap sits above that rather than at a fair share. CPUQuota=300% is
 39# three of the four cores, leaving one for gitbayd and sshd (#144, #184).
 40# OOMPolicy=continue: systemd's default stops the whole service when any
 41# process in it is OOM-killed, which would end the runner mid-build; the
 42# build fails and the runner carries on.
 43MemoryMax=6G
 44CPUQuota=300%
 45OOMPolicy=continue
 46#
 47# ExecStart is overridden here rather than left in the unit so the flags
 48# and the sandboxing that has to match them live in one file: -isolation
 49# podman needs NoNewPrivileges=no below, and -image needs an image the
 50# host has been given (deploy/runner-podman-setup.sh, Containerfile.ci).
 51# podman's run root is pinned under the runner's home by storage.conf
 52# (runner-podman-setup.sh), not taken from XDG_RUNTIME_DIR or /tmp: this
 53# unit has PrivateTmp, so a /tmp run root is a per-instance tmpfs.
 54#
 55# The cgroupfs manager puts podman's pause process under the user slice,
 56# outside this unit's cgroup, so a stop does not end it and the next
 57# start joins its namespaces — including a /tmp that no longer exists.
 58# End it with the service.
 59ExecStopPost=-/usr/bin/pkill -u ci-runner -x catatonit
 60# On stop the runner drains: it claims nothing more and finishes the
 61# build in flight, then exits. Give it long enough — the per-build limit
 62# is 45m plus half a minute of report retries — before systemd kills it.
 63# `make deploy-runner` therefore waits for a running build (#179).
 64TimeoutStopSec=50min
 65# The drain only works if the stop signal reaches the runner alone. The
 66# default control-group mode sends SIGTERM to every process in the
 67# cgroup at once — the ssh session streaming the log and the build's
 68# container with it — so the runner drained a build whose steps were
 69# already dead. mixed signals the main process only; whatever is left
 70# when it exits is killed.
 71KillMode=mixed
 72ExecStart=
 73ExecStart=/usr/local/bin/gitbay-runner -remote git@127.0.0.1 -workdir /var/lib/gitbay-runner/work -poll 5s -timeout 45m -repos krz/gitbay,cmc/ci-smoke -isolation podman -image localhost/gitbay-ci:1
 74Nice=10
 75CPUWeight=30
 76IOWeight=30
 77# NoNewPrivileges is off, and that is a deliberate trade (#144).
 78#
 79# Rootless podman sets up its user namespace with newuidmap, a setuid
 80# helper; NoNewPrivileges=yes blocks it and podman fails with
 81# "newuidmap: write to uid_map failed: Operation not permitted", so the
 82# runner refuses to start. The choice is between this flag and running
 83# builds in containers at all.
 84#
 85# Containers are the stronger boundary by a wide margin. NoNewPrivileges
 86# constrained a process that was already executing arbitrary repository
 87# code as this user; a container confines that code to an image and a
 88# bind-mounted workspace. What is lost is one hardening layer on the
 89# runner process itself, which is ours rather than a build's — a build no
 90# longer runs in this process's context at all.
 91#
 92# Under -isolation none there is no container, and this flag should be
 93# yes. Set it back if you run that way.
 94NoNewPrivileges=no
 95ProtectSystem=full
 96# ProtectKernelTunables is off, for the same reason NoNewPrivileges is
 97# (#144). It overmounts /proc/sys and friends in this unit's namespace,
 98# and the kernel then refuses a fresh proc mount in any child user
 99# namespace ("mount too revealing"): crun fails with "mount `proc` to
100# `proc`: Operation not permitted". There is no podman setting for it.
101# What the flag protected — /proc/sys from a build running on the host
102# as this user — the container now covers: a build gets its own proc,
103# with those paths masked by the runtime. Under -isolation none, set it
104# back to yes.
105ProtectControlGroups=yes
106RestrictSUIDSGID=yes
107Delegate=yes
108# The runner's home is /var/lib/gitbay-runner (see the Admin page), and
109# the leading - makes a missing path ignored rather than fatal: this
110# drop-in installs on hosts that have not been prepared for podman yet,
111# and a unit that refuses to start would stop every build on the
112# instance.
113ReadWritePaths=-/var/lib/gitbay-runner/.local/share/containers -/var/lib/gitbay-runner/.config/containers