Commit `4be4bd5917`

4be4bd5917a9487f980c0af7ef15e099c18487bf

parent: 89708196e4

Verified · cmc

cmc <hello@cleberg.net> · 2026-09-29 03:37 UTC

runner host: build egress by cgroup, narrower for untrusted builds

Ref #260

Layout: unified · split

Makefile +5

		@@ -77,9 +77,13 @@ deploy-runner: preflight
77	77	./deploy/copy.sh $(HOST) $(PORT) deploy/gitbay-runner-prune.timer /etc/systemd/system/gitbay-runner-prune.timer
78	78	./deploy/copy.sh $(HOST) $(PORT) deploy/gitbay-runner-egress.nft /etc/gitbay-runner/egress.nft
79	79	./deploy/copy.sh $(HOST) $(PORT) deploy/gitbay-runner-egress.service /etc/systemd/system/gitbay-runner-egress.service
	80	./deploy/copy.sh $(HOST) $(PORT) deploy/gitbay-runner-builds.nft /etc/gitbay-runner/builds.nft
80	81	@echo "==> loading the egress rule"
	82	@# builds.nft names the runner's class cgroups, which may not exist yet;
	83	@# its syntax is checked against system.slice, which always does.
81	84	ssh -p $(PORT) root@$(HOST) 'set -eu; \
82	85	nft -c -f /etc/gitbay-runner/egress.nft; \
	86	sed "s\|level 4 \"system.slice/gitbay-runner.service/builds/[a-z]*\"\|level 1 \"system.slice\"\|" /etc/gitbay-runner/builds.nft \| nft -c -f /dev/stdin; \
83	87	systemctl daemon-reload; \
84	88	systemctl enable gitbay-runner-egress.service; \
85	89	systemctl reload-or-restart gitbay-runner-egress.service'
		@@ -89,5 +93,6 @@ deploy-runner: preflight
89	93	mv /usr/local/bin/gitbay-runner.new /usr/local/bin/gitbay-runner; \
90	94	systemctl enable --now gitbay-runner-prune.timer; \
91	95	systemctl restart gitbay-runner; \
	96	nft list table inet gitbay_builds >/dev/null; \
92	97	systemctl --no-pager --lines=3 status gitbay-runner; \
93	98	systemctl --no-pager list-timers gitbay-runner-prune.timer'

deploy/gitbay-runner-builds.nft added +94

		@@ -0,0 +1,94 @@
	1	#!/usr/sbin/nft -f
	2	# Egress for CI builds by trust (#260). Installed as
	3	# /etc/gitbay-runner/builds.nft by `make deploy-runner` and loaded by the
	4	# runner unit's ExecStartPre (gitbay-runner.override.conf), after the
	5	# cgroups it names exist.
	6	#
	7	# gitbay-runner-egress.nft cannot tell a build from its runner: both run
	8	# as ci-runner. This table can. The runner starts every podman process
	9	# for a build inside builds/trusted/build-<id> or
	10	# builds/untrusted/build-<id> under its service cgroup, and pasta,
	11	# which podman starts, inherits that cgroup; so every socket pasta opens
	12	# for a build carries it. The runner itself, its clone and its log
	13	# stream run in <service>/runner and never match here.
	14	#
	15	# nftables resolves a cgroup path to the cgroup's id when the table
	16	# loads. The runner's service cgroup is new on every start, so the drop-in
	17	# creates builds/trusted and builds/untrusted and loads this file on
	18	# every start, and the runner never recreates them. A table loaded
	19	# against an earlier start's cgroups matches nothing, and builds then
	20	# get only the uid table.
	21	#
	22	# The uid table still applies to builds; a packet must pass both. This
	23	# table only takes away. Both hook output with policy accept, so their
	24	# relative order does not matter.
	25	#
	26	# Trusted builds (a branch of the repository, with its secrets):
	27	# internet open, any port.
	28	# host, public 22, 80 and 443: the forge at GITBAY_SSH
	29	# (169.254.1.2, which pasta translates to the public
	30	# address). hutch and orgo publish over 22.
	31	# host, loopback 53 only: the host's resolver, where pasta forwards
	32	# a build's DNS when the host's nameserver is a
	33	# loopback address.
	34	# private ranges closed (RFC 1918, CGNAT, link-local, ULA).
	35	# Untrusted builds (a fork's merge request head, no secrets):
	36	# internet 80 and 443 over TCP, and 53: enough to fetch
	37	# modules and packages, not to send mail or reach
	38	# ssh elsewhere.
	39	# host loopback 53 only. No forge: an untrusted build has
	40	# no key to use there, and a failing login from it
	41	# would count against the host's public address.
	42	# private ranges closed.
	43	# -isolation none builds run in the runner's own cgroup and get only the
	44	# uid table; such a runner must not take -untrusted.
	45	#
	46	# A host whose /etc/resolv.conf names a nameserver in a private range
	47	# needs that address let through here, or builds resolve nothing.
	48	#
	49	# The first line creates the table if it is missing, so the delete never
	50	# fails; the file then replaces it in one transaction.
	51
	52	table inet gitbay_builds
	53	delete table inet gitbay_builds
	54
	55	table inet gitbay_builds {
	56	set private4 {
	57	type ipv4_addr
	58	flags interval
	59	elements = { 0.0.0.0/8, 10.0.0.0/8, 100.64.0.0/10, 169.254.0.0/16, 172.16.0.0/12, 192.168.0.0/16 }
	60	}
	61
	62	set private6 {
	63	type ipv6_addr
	64	flags interval
	65	elements = { fc00::/7, fe80::/10 }
	66	}
	67
	68	chain output {
	69	type filter hook output priority filter; policy accept;
	70	socket cgroupv2 level 4 "system.slice/gitbay-runner.service/builds/trusted" jump trusted
	71	socket cgroupv2 level 4 "system.slice/gitbay-runner.service/builds/untrusted" jump untrusted
	72	}
	73
	74	chain trusted {
	75	oifname "lo" ip daddr 127.0.0.0/8 meta l4proto { tcp, udp } th dport 53 return
	76	oifname "lo" ip6 daddr ::1 meta l4proto { tcp, udp } th dport 53 return
	77	oifname "lo" ip daddr != 127.0.0.0/8 tcp dport { 22, 80, 443 } return
	78	oifname "lo" ip6 daddr != ::1 tcp dport { 22, 80, 443 } return
	79	oifname "lo" counter reject
	80	ip daddr @private4 counter reject
	81	ip6 daddr @private6 counter reject
	82	}
	83
	84	chain untrusted {
	85	oifname "lo" ip daddr 127.0.0.0/8 meta l4proto { tcp, udp } th dport 53 return
	86	oifname "lo" ip6 daddr ::1 meta l4proto { tcp, udp } th dport 53 return
	87	oifname "lo" counter reject
	88	ip daddr @private4 counter reject
	89	ip6 daddr @private6 counter reject
	90	meta l4proto { tcp, udp } th dport 53 return
	91	tcp dport { 80, 443 } return
	92	counter reject
	93	}
	94	}

deploy/gitbay-runner-egress.nft +2 −1

		@@ -9,7 +9,8 @@
9	9	# nftables sees them exactly as it sees the runner's own ssh, so this
10	10	# table cannot tell a build from its runner. It limits what that user
11	11	# reaches on this host, and the runner needs little: 127.0.0.1:22, to
12		# poll, clone and stream logs.
	12	# poll, clone and stream logs. gitbay-runner-builds.nft tells them apart
	13	# by cgroup and narrows this per build, trusted or not.
13	14	#
14	15	# Every packet to one of the host's own addresses, loopback or public,
15	16	# leaves through lo, so the output hook sees host-bound traffic as

deploy/gitbay-runner-egress.service +7

		@@ -13,6 +13,11 @@
13	13	#
14	14	# Stop uses destroy, which succeeds when the table is already gone (a
15	15	# flush ruleset removes it); delete would fail and leave the unit failed.
	16	#
	17	# The builds table (gitbay-runner-builds.nft) is loaded by the runner's
	18	# own start, against its cgroups. Reload loads it again when those
	19	# cgroups exist, so after a restart of nftables.service one reload puts
	20	# both tables back.
16	21	[Unit]
17	22	Description=Host egress rule for CI builds
18	23	After=nftables.service ufw.service
		@@ -23,7 +28,9 @@ Type=oneshot
23	28	RemainAfterExit=yes
24	29	ExecStart=/usr/sbin/nft -f /etc/gitbay-runner/egress.nft
25	30	ExecReload=/usr/sbin/nft -f /etc/gitbay-runner/egress.nft
	31	ExecReload=/bin/sh -c 'if [ -d /sys/fs/cgroup/system.slice/gitbay-runner.service/builds/untrusted ]; then exec /usr/sbin/nft -f /etc/gitbay-runner/builds.nft; fi'
26	32	ExecStop=/usr/sbin/nft destroy table inet gitbay_runner
	33	ExecStop=/usr/sbin/nft destroy table inet gitbay_builds
27	34
28	35	[Install]
29	36	WantedBy=multi-user.target

deploy/gitbay-runner.override.conf +13 −1

		@@ -25,7 +25,8 @@
25	25	[Unit]
26	26	# The host egress rule (#260, gitbay-runner-egress.nft) limits what this
27	27	# unit's user reaches on the host: 127.0.0.1:22 for the runner, the
28		# forge's public 22, 80 and 443 for builds, nothing else. Required, so
	28	# forge's public 22, 80 and 443 for builds, nothing else. The builds
	29	# table (ExecStartPre below) narrows that per build. Required, so
29	30	# the runner does not start without it: a table that failed to load must
30	31	# not mean builds reach the admin sshd.
31	32	Requires=gitbay-runner-egress.service
		@@ -71,6 +72,17 @@ OOMPolicy=continue
71	72	# start joins its namespaces — including a /tmp that no longer exists.
72	73	# End it with the service.
73	74	ExecStopPost=-/usr/bin/pkill -u ci-runner -x catatonit
	75	# The builds table (#260, gitbay-runner-builds.nft) matches a build's
	76	# traffic by the cgroup the runner starts it in, and nftables turns a
	77	# cgroup path into the cgroup's id when the table loads. This unit's
	78	# cgroup is new on every start, so the two class cgroups are created
	79	# here, handed to the runner's user, and the table loaded against them,
	80	# before the runner starts. As root (+), so the mkdir does not depend on
	81	# when systemd hands the delegated cgroup to the unit's user. A table
	82	# that fails to load stops the start, as the uid table's Requires= does.
	83	ExecStartPre=+/usr/bin/mkdir -p /sys/fs/cgroup/system.slice/gitbay-runner.service/builds/trusted /sys/fs/cgroup/system.slice/gitbay-runner.service/builds/untrusted
	84	ExecStartPre=+/usr/bin/chown -R ci-runner:ci-runner /sys/fs/cgroup/system.slice/gitbay-runner.service/builds
	85	ExecStartPre=+/usr/sbin/nft -f /etc/gitbay-runner/builds.nft
74	86	# On stop the runner drains: it claims nothing more and finishes the
75	87	# build in flight, then exits. Give it long enough — the per-build limit
76	88	# is 45m plus half a minute of report retries — before systemd kills it.

Commit 4be4bd5917

Commit `4be4bd5917`