deploy: pin podman's run root; end the pause process with the service !310

merged merged by cmc on 2026-09-07 01:52 UTC · krz/gitbay:runner-podman-storage into main

4 files changed, +80 −13

Layout: unified · split

.gitbay/wiki/Admin.org +17

		@@ -469,6 +469,23 @@ cannot call =newuidmap= and the runner refuses to start. That is a
469	469	considered trade, explained in the file and in the Threat-Model; if you
470	470	run with =-isolation none=, set it back to =yes=.
471	471
	472	*Validate podman mode on a scratch repository before pointing the runner
	473	at real ones.* Every deploy that switched the whole instance to
	474	containers and failed took CI down with it. Instead: create a throwaway
	475	repository the runner account can read (public, or granted read — a
	476	private one is "not found" to the runner and the build stays pending),
	477	give it one job that names the CI image, and deploy the runner with
	478	=-repos= naming only that repository. The production unit, with its real
	479	hardening, then claims nothing else; other repositories' builds queue
	480	until =-repos= is switched back, which is a pause, not an outage.
	481
	482	#+begin_src sh
	483	gitbay repo create cmc/ci-smoke # then push a .gitbay/ci.yml naming the image
	484	sed -i 's#-repos krz/gitbay #-repos cmc/ci-smoke #' /etc/systemd/system/gitbay-runner.service.d/override.conf
	485	systemctl daemon-reload && systemctl restart gitbay-runner
	486	gitbay build log cmc/ci-smoke 1 # green: switch -repos back, redeploy
	487	#+end_src
	488
472	489	*Do not deploy an isolating runner to a host that has not been
473	490	prepared.* The runner is specified to refuse to start without a working
474	491	podman rather than fall back to running builds unsandboxed — a fallback

.gitbay/wiki/Threat-Model.org +7 −3

		@@ -145,9 +145,13 @@ runner, polling over SSH, clones the commit and runs its steps.
145	145	repository is trusted; there is no automatic fallback to it — a runner
146	146	configured for podman that cannot find one refuses to start, because
147	147	dropping isolation silently is worse than a stopped runner. The
148		systemd drop-in still adds =ProtectSystem=full= and the kernel and
149		cgroup protections, and =-repos= still limits a runner to named
150		repositories. =NoNewPrivileges= is off: rootless podman sets up its
	148	systemd drop-in still adds =ProtectSystem=full= and the cgroup
	149	protections, and =-repos= still limits a runner to named
	150	repositories. =ProtectKernelTunables= is off: it overmounts =/proc=
	151	in the unit's namespace and the kernel then refuses a proc mount in
	152	any child user namespace, which every rootless container needs; the
	153	container masks the same paths for the build itself.
	154	=NoNewPrivileges= is off: rootless podman sets up its
151	155	namespace with the setuid =newuidmap=, which that flag blocks, so the
152	156	choice is between it and containers at all. Containers are the stronger
153	157	boundary — the flag constrained a process that was already running

deploy/gitbay-runner.override.conf +18 −6

		@@ -27,11 +27,15 @@
27	27	# and the sandboxing that has to match them live in one file: -isolation
28	28	# podman needs NoNewPrivileges=no below, and -image needs an image the
29	29	# host has been given (deploy/runner-podman-setup.sh, Containerfile.ci).
30		# Deliberately no XDG_RUNTIME_DIR. podman records its run root in its
31		# database at first use, so setting one later fails with "database
32		# configuration mismatch"; the runner's storage was initialised without
33		# it and works. Change it only together with `podman system reset` and a
34		# rebuild of the images (#144).
	30	# podman's run root is pinned under the runner's home by storage.conf
	31	# (runner-podman-setup.sh), not taken from XDG_RUNTIME_DIR or /tmp: this
	32	# unit has PrivateTmp, so a /tmp run root is a per-instance tmpfs.
	33	#
	34	# The cgroupfs manager puts podman's pause process under the user slice,
	35	# outside this unit's cgroup, so a stop does not end it and the next
	36	# start joins its namespaces — including a /tmp that no longer exists.
	37	# End it with the service.
	38	ExecStopPost=-/usr/bin/pkill -u ci-runner -x catatonit
35	39	ExecStart=
36	40	ExecStart=/usr/local/bin/gitbay-runner -remote git@127.0.0.1 -workdir /var/lib/gitbay-runner/work -poll 5s -timeout 45m -repos krz/gitbay -isolation podman -image localhost/gitbay-ci:1
37	41	Nice=10
		@@ -56,7 +60,15 @@ IOWeight=30
56	60	# yes. Set it back if you run that way.
57	61	NoNewPrivileges=no
58	62	ProtectSystem=full
59		ProtectKernelTunables=yes
	63	# ProtectKernelTunables is off, for the same reason NoNewPrivileges is
	64	# (#144). It overmounts /proc/sys and friends in this unit's namespace,
	65	# and the kernel then refuses a fresh proc mount in any child user
	66	# namespace ("mount too revealing"): crun fails with "mount `proc` to
	67	# `proc`: Operation not permitted". There is no podman setting for it.
	68	# What the flag protected — /proc/sys from a build running on the host
	69	# as this user — the container now covers: a build gets its own proc,
	70	# with those paths masked by the runtime. Under -isolation none, set it
	71	# back to yes.
60	72	ProtectControlGroups=yes
61	73	RestrictSUIDSGID=yes
62	74	Delegate=yes

deploy/runner-podman-setup.sh +38 −4

		@@ -52,13 +52,47 @@ if [ "$max_ns" -lt 1 ]; then
52	52	fi
53	53	echo " max_user_namespaces=$max_ns"
54	54
55		# Lingering keeps the user's systemd session — and so podman's storage
56		# and any running container — alive when nobody is logged in.
	55	# podman's storage paths are pinned in storage.conf, both graphroot and
	56	# runroot, under the runner's home. Left to podman, the run root is
	57	# $XDG_RUNTIME_DIR or /tmp/storage-run-<uid>; the service runs with
	58	# PrivateTmp, so that is a per-instance tmpfs, and podman's pause process
	59	# (which the cgroupfs manager places outside the service cgroup) can
	60	# outlive a restart holding a dead /tmp — after which every podman
	61	# command, in any context, fails with "mkdir ...: no such file or
	62	# directory". A run root under the home directory is valid in every
	63	# namespace and needs neither lingering nor /tmp.
	64	#
	65	# podman records the run root at first use. Changing it later needs
	66	# `podman system reset --force` as the runner user and a rebuild of the
	67	# images; this script does not do that for you.
	68	home=$(getent passwd "$RUNNER_USER" \| cut -d: -f6)
	69	conf="$home/.config/containers/storage.conf"
	70	echo "==> storage config in $conf"
	71	install -d -o "$RUNNER_USER" -g "$RUNNER_USER" -m 700 "$home/.config/containers"
	72	printf '[storage]\ndriver = "overlay"\ngraphroot = "%s/.local/share/containers/storage"\nrunroot = "%s/.local/share/containers/run"\n' "$home" "$home" >"$conf"
	73	chown "$RUNNER_USER:$RUNNER_USER" "$conf"
	74	echo " written"
	75
	76	# podman sets net.ipv4.ping_group_range in every container by default,
	77	# for unprivileged ping. The service runs with ProtectKernelTunables, so
	78	# /proc/sys is read-only and crun fails to start the container with
	79	# "open /proc/sys/net/ipv4/ping_group_range: Read-only file system". A
	80	# build has no use for ping; drop the default rather than the hardening.
	81	cconf="$home/.config/containers/containers.conf"
	82	echo "==> container defaults in $cconf"
	83	printf '[containers]\ndefault_sysctls = []\n' >"$cconf"
	84	chown "$RUNNER_USER:$RUNNER_USER" "$cconf"
	85	echo " written"
	86
	87	# Lingering keeps the user's systemd session alive when nobody is logged
	88	# in, which podman's pause process relies on.
57	89	echo "==> lingering for $RUNNER_USER"
58	90	loginctl enable-linger "$RUNNER_USER"
59	91
60	92	echo "==> verifying rootless podman as $RUNNER_USER"
61		su - "$RUNNER_USER" -s /bin/sh -c 'podman info --format "{{.Host.Security.Rootless}}"'
	93	# The verification fails rather than passing with \|\| true: a host that
	94	# reports ready and is not is the outage this script exists to prevent.
	95	su - "$RUNNER_USER" -s /bin/sh -c "podman info --format 'rootless={{.Host.Security.Rootless}} runroot={{.Store.RunRoot}}'"
62	96
63	97	echo
64		echo "host is ready; now: make deploy-runner"
	98	echo "host is ready. Build the CI image (deploy/Containerfile.ci), then: make deploy-runner"