deploy: prepare a runner host for rootless podman !291

merged merged by cmc on 2026-09-06 20:03 UTC · krz/gitbay:runner-podman-host-144 into main

5 files changed, +142 −0

Layout: unified · split

.gitbay/wiki/Admin.org +39

	@@ -406,6 +406,45 @@ deploy, an archive publish, an automated MR branch — needs an explicit
406	grant on that repo: =repo access grant <owner/name> ci write=. Private	406	grant on that repo: =repo access grant <owner/name> ci write=. Private
407	repos likewise need at least read for the clone.	407	repos likewise need at least read for the clone.
408		408
		409	** Container isolation
		410
		411	Builds currently run as the runner's own user with no container; the
		412	service drop-in and =-repos= are the controls (krz/gitbay#144). The step
		413	environment is constructed rather than inherited, so a build sees =PATH=,
		414	=HOME= (its workspace), =LANG=, =CI=, its =GITBAY_*= variables and its
		415	secrets and nothing else — but a step can still read what that user can
		416	read, and concurrent builds share a =-workdir=.
		417
		418	Rootless podman is the chosen remedy; the host preparation ships ahead of
		419	the runner that uses it, so the order is fixed:
		420
		421	#+begin_src sh
		422	ssh -p 2222 root@<host> 'sh -s' < deploy/runner-podman-setup.sh
		423	make deploy-runner
		424	#+end_src
		425
		426	The script installs podman, delegates a subuid/subgid range to
		427	=ci-runner=, checks that user namespaces are enabled rather than
		428	assuming, enables lingering, and verifies rootless podman actually runs
		429	as that user. It is idempotent.
		430
		431	*Do not deploy an isolating runner to a host that has not been
		432	prepared.* The runner is specified to refuse to start without a working
		433	podman rather than fall back to running builds unsandboxed — a fallback
		434	that silently drops isolation is worse than a stopped runner, because
		435	nothing surfaces it. On an unprepared host that refusal stops every
		436	build on the instance.
		437
		438	The service drop-in carries =Delegate=yes= for rootless cgroup
		439	management and =ReadWritePaths= for podman's store under
		440	=/var/lib/gitbay-runner=, which =ProtectSystem=full= would otherwise
		441	make read-only. Those paths are prefixed =-= so they are ignored when
		442	absent: the drop-in installs on unprepared hosts too, and a unit that
		443	refused to start would stop every build.
		444	=gitbay-runner-prune.timer= prunes unused images weekly, as the runner's
		445	user: rootless storage belongs to that user, and root's prune would not
		446	see it. An unpruned image store on a 40GB host is a slow outage.
		447
409	* LFS storage	448	* LFS storage
410		449
411	Objects live content-addressed under =[lfs] root= (default	450	Objects live content-addressed under =[lfs] root= (default

deploy/gitbay-runner-prune.service added +13

@@ -0,0 +1,13 @@
	1	# Paired with gitbay-runner-prune.timer. Runs as the runner's own user,
	2	# because rootless podman's store belongs to that user and root's prune
	3	# would not see it.
	4	[Unit]
	5	Description=Prune unused podman images on the CI runner
	6
	7	[Service]
	8	Type=oneshot
	9	User=ci-runner
	10	# Images not used by a container and older than a week. A build that
	11	# names an image again re-pulls it; the cost is one pull, not a failure.
	12	ExecStart=/usr/bin/podman image prune --all --force --filter until=168h
	13	ExecStart=/usr/bin/podman container prune --force

deploy/gitbay-runner-prune.timer added +13

@@ -0,0 +1,13 @@
	1	# Podman's image store grows without bound: every `image:` a repository
	2	# names is pulled and kept. On a 40GB host that is a slow outage, so
	3	# prune weekly (#144).
	4	[Unit]
	5	Description=Prune unused podman images on the CI runner
	6
	7	[Timer]
	8	OnCalendar=Sun 04:00
	9	RandomizedDelaySec=30m
	10	Persistent=true
	11
	12	[Install]
	13	WantedBy=timers.target

deploy/gitbay-runner.override.conf +13

	@@ -16,6 +16,12 @@
16	# `keys add --scope runner`, which confines it to the runner protocol	16	# `keys add --scope runner`, which confines it to the runner protocol
17	# and read-only git, and the sandboxing below keeps a step from	17	# and read-only git, and the sandboxing below keeps a step from
18	# touching the system outside its workspace.	18	# touching the system outside its workspace.
		19	#
		20	# Delegate=yes and the storage path below are what rootless podman needs
		21	# (#144): it manages its own cgroups for a container, and its image and
		22	# container store lives under the runner's home, which ProtectSystem
		23	# would otherwise make read-only. Prepare the host with
		24	# deploy/runner-podman-setup.sh before deploying a runner that isolates.
19	[Service]	25	[Service]
20	Nice=10	26	Nice=10
21	CPUWeight=30	27	CPUWeight=30
	@@ -25,3 +31,10 @@ ProtectSystem=full
25	ProtectKernelTunables=yes	31	ProtectKernelTunables=yes
26	ProtectControlGroups=yes	32	ProtectControlGroups=yes
27	RestrictSUIDSGID=yes	33	RestrictSUIDSGID=yes
		34	Delegate=yes
		35	# The runner's home is /var/lib/gitbay-runner (see the Admin page), and
		36	# the leading - makes a missing path ignored rather than fatal: this
		37	# drop-in installs on hosts that have not been prepared for podman yet,
		38	# and a unit that refuses to start would stop every build on the
		39	# instance.
		40	ReadWritePaths=-/var/lib/gitbay-runner/.local/share/containers -/var/lib/gitbay-runner/.config/containers

deploy/runner-podman-setup.sh added +64

@@ -0,0 +1,64 @@
	1	#!/bin/sh
	2	# Prepare a runner host for container-isolated builds (#144).
	3	#
	4	# Run this on the runner host as root BEFORE deploying a gitbay-runner
	5	# that requires isolation. The runner refuses to start without a working
	6	# podman rather than falling back to running builds unsandboxed, so the
	7	# order matters: prepare the host, then `make deploy-runner`.
	8	#
	9	# ssh -p 2222 root@bay1 'sh -s' < deploy/runner-podman-setup.sh
	10	#
	11	# Idempotent: safe to re-run.
	12	set -eu
	13
	14	RUNNER_USER="${RUNNER_USER:-ci-runner}"
	15
	16	# The runner's home is wherever the account was created with; podman's
	17	# store lives under it and the systemd drop-in names the same path.
	18
	19	if ! id "$RUNNER_USER" >/dev/null 2>&1; then
	20	echo "no such user: $RUNNER_USER" >&2
	21	exit 1
	22	fi
	23
	24	echo "==> installing podman"
	25	if ! command -v podman >/dev/null 2>&1; then
	26	apt-get update
	27	DEBIAN_FRONTEND=noninteractive apt-get install -y podman uidmap
	28	fi
	29	podman --version
	30
	31	# Rootless podman maps container uids into a range delegated to the user.
	32	# Without these the runner's `podman run` fails with a mapping error.
	33	echo "==> subuid/subgid for $RUNNER_USER"
	34	for f in /etc/subuid /etc/subgid; do
	35	if ! grep -q "^$RUNNER_USER:" "$f" 2>/dev/null; then
	36	echo "$RUNNER_USER:200000:65536" >>"$f"
	37	echo " added to $f"
	38	else
	39	echo " already in $f"
	40	fi
	41	done
	42
	43	# User namespaces are what rootless podman is built on. Debian 13 enables
	44	# them by default; check rather than assume, because a build silently
	45	# running as the host user is exactly what this is meant to prevent.
	46	echo "==> kernel support"
	47	max_ns=$(cat /proc/sys/user/max_user_namespaces 2>/dev/null \|\| echo 0)
	48	if [ "$max_ns" -lt 1 ]; then
	49	echo "user namespaces are disabled (user.max_user_namespaces=$max_ns);" >&2
	50	echo "rootless podman cannot work until they are enabled" >&2
	51	exit 1
	52	fi
	53	echo " max_user_namespaces=$max_ns"
	54
	55	# Lingering keeps the user's systemd session — and so podman's storage
	56	# and any running container — alive when nobody is logged in.
	57	echo "==> lingering for $RUNNER_USER"
	58	loginctl enable-linger "$RUNNER_USER"
	59
	60	echo "==> verifying rootless podman as $RUNNER_USER"
	61	su - "$RUNNER_USER" -s /bin/sh -c 'podman info --format "{{.Host.Security.Rootless}}"'
	62
	63	echo
	64	echo "host is ready; now: make deploy-runner"