deploy/cloud-init.yaml
343 lines · 11837 bytes
1#cloud-config
2# gitbay VPS bootstrap (Ubuntu 24.04).
3#
4# What this does on first boot:
5# - moves the host's admin sshd to port 2222 (gitbay's embedded SSH
6# listener owns port 22) — CONNECT ON 2222 AFTER FIRST BOOT
7# - creates the unprivileged gitbay user and directory layout
8# - installs /etc/gitbay/config.toml, the systemd unit (with
9# CAP_NET_BIND_SERVICE so ports 22/80/443 work without root), and a
10# nightly backup timer
11# - opens ufw for 22, 80, 443, 2222
12#
13# It does NOT install the gitbayd binary (it is not hosted anywhere yet);
14# scp it to /usr/local/bin/gitbayd afterward and `systemctl start gitbayd`.
15
16package_update: true
17packages:
18 - git
19 - ufw
20 - unattended-upgrades
21 - fail2ban
22 # The CI runner shares this host and the suite drives them; without them
23 # the LFS and signature tests skip themselves and CI goes green having
24 # tested less.
25 - git-lfs
26 - gnupg
27
28write_files:
29 # Admin sshd on 2222. Ubuntu 24.04 socket-activates sshd, so the port
30 # must change in BOTH sshd_config and the socket unit.
31 - path: /etc/ssh/sshd_config.d/60-gitbay-port.conf
32 content: |
33 Port 2222
34 PasswordAuthentication no
35 # Throttle unauthenticated connection floods on the admin sshd
36 # (gitbayd's own port 22 is throttled by limits.ssh_auth_rate).
37 MaxStartups 10:30:60
38 MaxAuthTries 3
39 LoginGraceTime 20
40
41 # OS security patches applied automatically; reboot at 04:30 if needed.
42 - path: /etc/apt/apt.conf.d/51gitbay-unattended
43 content: |
44 Unattended-Upgrade::Allowed-Origins { "${distro_id}:${distro_codename}-security"; };
45 Unattended-Upgrade::Automatic-Reboot "true";
46 Unattended-Upgrade::Automatic-Reboot-Time "04:30";
47 APT::Periodic::Update-Package-Lists "1";
48 APT::Periodic::Unattended-Upgrade "1";
49
50 # fail2ban watches the admin sshd for auth failures.
51 - path: /etc/fail2ban/jail.d/gitbay.conf
52 content: |
53 [sshd]
54 enabled = true
55 port = 2222
56 backend = systemd
57 maxretry = 5
58 bantime = 1h
59
60 # Heartbeat: disk/service/cert status to journald every run, and to a
61 # webhook as well if one is set in /etc/gitbay/monitor.url. Exits non-zero
62 # on an alert so the unit shows up in systemctl --failed.
63 - path: /usr/local/bin/gitbay-monitor.sh
64 permissions: "0755"
65 content: |
66 #!/bin/sh
67 set -eu
68 disk=$(df -P /var/lib/gitbay | awk 'NR==2{print $5}')
69 svc=$(systemctl is-active gitbayd || true)
70 # Soonest ACME cert expiry. Reporting whichever name sorted first said
71 # nothing about the one actually about to lapse, and the cache is under
72 # acme/, so this read autocert/ and reported n/a forever.
73 cert=/var/lib/gitbay/acme
74 exp="n/a"
75 days=""
76 if [ -d "$cert" ]; then
77 soonest=""
78 for f in "$cert"/*; do
79 [ -f "$f" ] || continue
80 case "${f##*/}" in acme_account*) continue ;; esac
81 end=$(openssl x509 -enddate -noout -in "$f" 2>/dev/null | cut -d= -f2 || true)
82 [ -n "$end" ] || continue
83 secs=$(date -u -d "$end" +%s 2>/dev/null || true)
84 [ -n "$secs" ] || continue
85 if [ -z "$soonest" ] || [ "$secs" -lt "$soonest" ]; then
86 soonest="$secs"
87 exp="$end"
88 fi
89 done
90 if [ -n "$soonest" ]; then
91 days=$(( (soonest - $(date -u +%s)) / 86400 ))
92 fi
93 fi
94 # Backups are timers, and a timer failing quietly is the most
95 # damaging silent failure this host has. Age of the newest full
96 # archive and the newest database snapshot, in hours.
97 now=$(date -u +%s)
98 age_h() {
99 f=$(ls -t "$1"/*.tar.gz 2>/dev/null | head -1)
100 [ -n "$f" ] || { echo ""; return; }
101 echo $(( (now - $(stat -c %Y "$f")) / 3600 ))
102 }
103 full_age=$(age_h /var/backups/gitbay)
104 db_age=$(age_h /var/backups/gitbay/db)
105 # The daemon's own word, from inside the process.
106 site=$(sed -n 's/^site_url *= *"\(.*\)"/\1/p' /etc/gitbay/config.toml | head -1)
107 health="n/a"
108 if [ -n "$site" ]; then
109 health=$(curl -fsS -m 10 "$site/healthz" 2>/dev/null | grep -o '"ok":[a-z]*' | head -1 | cut -d: -f2)
110 [ -n "$health" ] || health="unreachable"
111 fi
112 alert=""
113 if [ "$svc" != "active" ]; then
114 alert="gitbayd is $svc; "
115 fi
116 if [ "$health" != "true" ]; then
117 alert="${alert}healthz $health; "
118 fi
119 if [ -z "$full_age" ] || [ "$full_age" -ge 25 ]; then
120 alert="${alert}full backup ${full_age:-missing}h old; "
121 fi
122 if [ -z "$db_age" ] || [ "$db_age" -ge 2 ]; then
123 alert="${alert}db snapshot ${db_age:-missing}h old; "
124 fi
125 pct=$(echo "$disk" | tr -d '%')
126 if [ "$pct" -ge 85 ]; then
127 alert="${alert}disk ${disk}; "
128 fi
129 if [ -n "$days" ] && [ "$days" -lt 21 ]; then
130 alert="${alert}cert expires in ${days}d; "
131 fi
132 # journald always gets the reading, so an unset webhook cannot make a
133 # sick host look like a quiet one.
134 echo "disk=$disk service=$svc healthz=$health cert_expires=$exp${days:+ cert_days=$days} full_backup_h=${full_age:-missing} db_snapshot_h=${db_age:-missing}"
135 url_file=/etc/gitbay/monitor.url
136 if [ -f "$url_file" ]; then
137 body=$(printf '{"disk":"%s","service":"%s","healthz":"%s","cert_expires":"%s","full_backup_h":"%s","db_snapshot_h":"%s","alert":"%s"}' "$disk" "$svc" "$health" "$exp" "${full_age:-missing}" "${db_age:-missing}" "$alert")
138 if ! curl -fsS -m 10 -H 'Content-Type: application/json' -d "$body" "$(cat "$url_file")" >/dev/null; then
139 echo "monitor webhook post failed" >&2
140 fi
141 fi
142 if [ -n "$alert" ]; then
143 echo "$alert" >&2
144 exit 1
145 fi
146
147 - path: /etc/systemd/system/gitbay-monitor.service
148 content: |
149 [Unit]
150 Description=gitbay host heartbeat
151 [Service]
152 Type=oneshot
153 ExecStart=/usr/local/bin/gitbay-monitor.sh
154
155 - path: /etc/systemd/system/gitbay-monitor.timer
156 content: |
157 [Unit]
158 Description=gitbay host heartbeat
159 [Timer]
160 OnCalendar=*-*-* *:00:00 UTC
161 Persistent=true
162 [Install]
163 WantedBy=timers.target
164 - path: /etc/systemd/system/ssh.socket.d/override.conf
165 content: |
166 [Socket]
167 ListenStream=
168 ListenStream=2222
169
170 - path: /etc/gitbay/config.toml
171 permissions: "0640"
172 content: |
173 [server]
174 root = "/var/lib/gitbay"
175 site_url = "https://gitbay.org"
176
177 [ssh]
178 mode = "embedded"
179 port = 22
180
181 [http]
182 addr = ":443"
183 tls = "acme"
184 acme_email = "hello@gitbay.org"
185 acme_http_addr = ":80"
186
187 [web]
188 mode = "view_only"
189
190 [registration]
191 mode = "closed"
192
193 - path: /etc/systemd/system/gitbayd.service
194 content: |
195 [Unit]
196 Description=gitbay forge daemon
197 After=network-online.target
198 Wants=network-online.target
199
200 [Service]
201 User=gitbay
202 Group=gitbay
203 ExecStart=/usr/local/bin/gitbayd --config /etc/gitbay/config.toml serve
204 Restart=on-failure
205 RestartSec=3
206
207 # Bind 22/80/443 without root; no privilege escalation afterward.
208 AmbientCapabilities=CAP_NET_BIND_SERVICE
209 CapabilityBoundingSet=CAP_NET_BIND_SERVICE
210 NoNewPrivileges=yes
211 ProtectSystem=strict
212 ProtectHome=yes
213 ReadWritePaths=/var/lib/gitbay /var/backups/gitbay
214 PrivateTmp=yes
215 ProtectKernelTunables=yes
216 ProtectKernelModules=yes
217 ProtectControlGroups=yes
218 ProtectHostname=yes
219 ProtectClock=yes
220 ProtectKernelLogs=yes
221 RestrictSUIDSGID=yes
222 RestrictNamespaces=yes
223 RestrictRealtime=yes
224 LockPersonality=yes
225 MemoryDenyWriteExecute=yes
226 PrivateDevices=yes
227 # IPv4/IPv6 for listeners and outbound git/ssh; UNIX for the hook socket.
228 RestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX
229 # Allow only ordinary service syscalls; the daemon spawns git and ssh,
230 # so keep @process/@exec available (both are within @system-service).
231 SystemCallFilter=@system-service
232 SystemCallErrorNumber=EPERM
233 SystemCallArchitectures=native
234
235 [Install]
236 WantedBy=multi-user.target
237
238 - path: /usr/local/bin/gitbay-backup.sh
239 permissions: "0755"
240 content: |
241 #!/bin/sh
242 # Nightly consistent backup; keeps the last 7 locally.
243 # To ship offsite, add an rclone/s3 upload of $out here.
244 set -eu
245 # The archive carries the database, so it gets the database's mode.
246 umask 027
247 dir=/var/backups/gitbay
248 out="$dir/gitbay-$(date -u +%Y%m%d-%H%M%S).tar.gz"
249 /usr/local/bin/gitbayd --config /etc/gitbay/config.toml admin backup --out "$out"
250 ls -1t "$dir"/gitbay-*.tar.gz | tail -n +8 | xargs -r rm --
251
252 # Hourly database-only snapshot. The nightly full backup below is the one
253 # that can rebuild the host; this one exists because the database holds
254 # issues, merge requests and comments, which unlike the repositories have
255 # no second copy anywhere. 48 of them is two days at a few MB each.
256 - path: /usr/local/bin/gitbay-db-backup.sh
257 permissions: "0755"
258 content: |
259 #!/bin/sh
260 set -eu
261 # The archive is the whole database, so it gets the database's mode.
262 umask 027
263 dir=/var/backups/gitbay/db
264 mkdir -p "$dir"
265 chmod 0750 "$dir"
266 out="$dir/gitbay-db-$(date -u +%Y%m%d-%H%M%S).tar.gz"
267 /usr/local/bin/gitbayd --config /etc/gitbay/config.toml admin backup --db-only --out "$out"
268 ls -1t "$dir"/gitbay-db-*.tar.gz | tail -n +49 | xargs -r rm --
269
270 - path: /etc/systemd/system/gitbay-db-backup.service
271 content: |
272 [Unit]
273 Description=gitbay hourly database backup
274 [Service]
275 Type=oneshot
276 User=gitbay
277 ExecStart=/usr/local/bin/gitbay-db-backup.sh
278
279 - path: /etc/systemd/system/gitbay-db-backup.timer
280 content: |
281 [Unit]
282 Description=gitbay hourly database backup
283 [Timer]
284 OnCalendar=*-*-* *:20:00 UTC
285 RandomizedDelaySec=5m
286 Persistent=true
287 [Install]
288 WantedBy=timers.target
289
290 - path: /etc/systemd/system/gitbay-backup.service
291 content: |
292 [Unit]
293 Description=gitbay nightly backup
294 [Service]
295 Type=oneshot
296 User=gitbay
297 ExecStart=/usr/local/bin/gitbay-backup.sh
298
299 - path: /etc/systemd/system/gitbay-backup.timer
300 content: |
301 [Unit]
302 Description=gitbay nightly backup
303 [Timer]
304 OnCalendar=*-*-* 09:00:00 UTC
305 RandomizedDelaySec=15m
306 Persistent=true
307 [Install]
308 WantedBy=timers.target
309
310 - path: /etc/systemd/system/gitbay-gc.service
311 content: |
312 [Unit]
313 Description=gitbay weekly repository maintenance
314 [Service]
315 Type=oneshot
316 User=gitbay
317 ExecStart=/usr/local/bin/gitbayd --config /etc/gitbay/config.toml admin gc
318
319 - path: /etc/systemd/system/gitbay-gc.timer
320 content: |
321 [Unit]
322 Description=gitbay weekly repository maintenance
323 [Timer]
324 OnCalendar=Sun *-*-* 07:00:00 UTC
325 RandomizedDelaySec=30m
326 Persistent=true
327 [Install]
328 WantedBy=timers.target
329
330runcmd:
331 - adduser --system --group --home /var/lib/gitbay --shell /usr/sbin/nologin gitbay
332 - install -d -o gitbay -g gitbay -m 750 /var/lib/gitbay /var/backups/gitbay
333 - chgrp gitbay /etc/gitbay/config.toml /etc/gitbay
334 - ufw allow 22/tcp
335 - ufw allow 80/tcp
336 - ufw allow 443/tcp
337 - ufw allow 2222/tcp
338 - ufw --force enable
339 - systemctl daemon-reload
340 - systemctl restart ssh.socket || systemctl restart ssh
341 - systemctl enable gitbayd gitbay-backup.timer gitbay-db-backup.timer gitbay-gc.timer gitbay-monitor.timer
342 - systemctl start gitbay-backup.timer gitbay-db-backup.timer gitbay-gc.timer gitbay-monitor.timer
343 - systemctl enable --now unattended-upgrades fail2ban