deploy/cloud-init.yaml
352 lines · 12242 bytes
1#cloud-config
2# gitbay VPS bootstrap (Ubuntu 24.04).
3#
4# What this does on first boot:
5# - moves the host's admin sshd to port 2222 (gitbay's embedded SSH
6# listener owns port 22) — CONNECT ON 2222 AFTER FIRST BOOT
7# - creates the unprivileged gitbay user and directory layout
8# - installs /etc/gitbay/config.toml, the systemd unit (with
9# CAP_NET_BIND_SERVICE so ports 22/80/443 work without root), and a
10# nightly backup timer
11# - opens ufw for 22, 80, 443, 2222
12#
13# It does NOT install the gitbayd binary (it is not hosted anywhere yet);
14# scp it to /usr/local/bin/gitbayd afterward and `systemctl start gitbayd`.
15
16package_update: true
17packages:
18 - git
19 - ufw
20 - unattended-upgrades
21 - fail2ban
22 # The CI runner shares this host and the suite drives them; without them
23 # the LFS and signature tests skip themselves and CI goes green having
24 # tested less.
25 - git-lfs
26 - gnupg
27
28write_files:
29 # Admin sshd on 2222. Ubuntu 24.04 socket-activates sshd, so the port
30 # must change in BOTH sshd_config and the socket unit.
31 - path: /etc/ssh/sshd_config.d/60-gitbay-port.conf
32 content: |
33 Port 2222
34 PasswordAuthentication no
35 # Throttle unauthenticated connection floods on the admin sshd
36 # (gitbayd's own port 22 is throttled by limits.ssh_auth_rate).
37 MaxStartups 10:30:60
38 MaxAuthTries 3
39 LoginGraceTime 20
40
41 # OS security patches applied automatically; reboot at 04:30 if needed.
42 - path: /etc/apt/apt.conf.d/51gitbay-unattended
43 content: |
44 Unattended-Upgrade::Allowed-Origins { "${distro_id}:${distro_codename}-security"; };
45 Unattended-Upgrade::Automatic-Reboot "true";
46 Unattended-Upgrade::Automatic-Reboot-Time "04:30";
47 APT::Periodic::Update-Package-Lists "1";
48 APT::Periodic::Unattended-Upgrade "1";
49
50 # fail2ban watches the admin sshd for auth failures.
51 - path: /etc/fail2ban/jail.d/gitbay.conf
52 content: |
53 [sshd]
54 enabled = true
55 port = 2222
56 backend = systemd
57 maxretry = 5
58 bantime = 1h
59
60 # Heartbeat: disk/service/cert status to journald every run, and to a
61 # webhook as well if one is set in /etc/gitbay/monitor.url. Exits non-zero
62 # on an alert so the unit shows up in systemctl --failed.
63 - path: /usr/local/bin/gitbay-monitor.sh
64 permissions: "0755"
65 content: |
66 #!/bin/sh
67 set -eu
68 disk=$(df -P /var/lib/gitbay | awk 'NR==2{print $5}')
69 svc=$(systemctl is-active gitbayd || true)
70 # Soonest ACME cert expiry. Reporting whichever name sorted first said
71 # nothing about the one actually about to lapse, and the cache is under
72 # acme/, so this read autocert/ and reported n/a forever.
73 cert=/var/lib/gitbay/acme
74 exp="n/a"
75 days=""
76 if [ -d "$cert" ]; then
77 soonest=""
78 for f in "$cert"/*; do
79 [ -f "$f" ] || continue
80 case "${f##*/}" in acme_account*) continue ;; esac
81 end=$(openssl x509 -enddate -noout -in "$f" 2>/dev/null | cut -d= -f2 || true)
82 [ -n "$end" ] || continue
83 secs=$(date -u -d "$end" +%s 2>/dev/null || true)
84 [ -n "$secs" ] || continue
85 if [ -z "$soonest" ] || [ "$secs" -lt "$soonest" ]; then
86 soonest="$secs"
87 exp="$end"
88 fi
89 done
90 if [ -n "$soonest" ]; then
91 days=$(( (soonest - $(date -u +%s)) / 86400 ))
92 fi
93 fi
94 # Backups are timers, and a timer failing quietly is the most
95 # damaging silent failure this host has. Age of the newest full
96 # archive and the newest database snapshot, in hours.
97 now=$(date -u +%s)
98 age_h() {
99 f=$(ls -t "$1"/*.tar.gz 2>/dev/null | head -1)
100 [ -n "$f" ] || { echo ""; return; }
101 echo $(( (now - $(stat -c %Y "$f")) / 3600 ))
102 }
103 full_age=$(age_h /var/backups/gitbay)
104 db_age=$(age_h /var/backups/gitbay/db)
105 # The daemon's own word, from inside the process.
106 site=$(sed -n 's/^site_url *= *"\(.*\)"/\1/p' /etc/gitbay/config.toml | head -1)
107 health="n/a"
108 if [ -n "$site" ]; then
109 health=$(curl -fsS -m 10 "$site/healthz" 2>/dev/null | grep -o '"ok":[a-z]*' | head -1 | cut -d: -f2)
110 [ -n "$health" ] || health="unreachable"
111 fi
112 alert=""
113 if [ "$svc" != "active" ]; then
114 alert="gitbayd is $svc; "
115 fi
116 if [ "$health" != "true" ]; then
117 alert="${alert}healthz $health; "
118 fi
119 if [ -z "$full_age" ] || [ "$full_age" -ge 25 ]; then
120 alert="${alert}full backup ${full_age:-missing}h old; "
121 fi
122 if [ -z "$db_age" ] || [ "$db_age" -ge 2 ]; then
123 alert="${alert}db snapshot ${db_age:-missing}h old; "
124 fi
125 pct=$(echo "$disk" | tr -d '%')
126 if [ "$pct" -ge 85 ]; then
127 alert="${alert}disk ${disk}; "
128 fi
129 if [ -n "$days" ] && [ "$days" -lt 21 ]; then
130 alert="${alert}cert expires in ${days}d; "
131 fi
132 # journald always gets the reading, so an unset webhook cannot make a
133 # sick host look like a quiet one.
134 echo "disk=$disk service=$svc healthz=$health cert_expires=$exp${days:+ cert_days=$days} full_backup_h=${full_age:-missing} db_snapshot_h=${db_age:-missing}"
135 url_file=/etc/gitbay/monitor.url
136 if [ -f "$url_file" ]; then
137 body=$(printf '{"disk":"%s","service":"%s","healthz":"%s","cert_expires":"%s","full_backup_h":"%s","db_snapshot_h":"%s","alert":"%s"}' "$disk" "$svc" "$health" "$exp" "${full_age:-missing}" "${db_age:-missing}" "$alert")
138 if ! curl -fsS -m 10 -H 'Content-Type: application/json' -d "$body" "$(cat "$url_file")" >/dev/null; then
139 echo "monitor webhook post failed" >&2
140 fi
141 fi
142 if [ -n "$alert" ]; then
143 echo "$alert" >&2
144 exit 1
145 fi
146
147 - path: /etc/systemd/system/gitbay-monitor.service
148 content: |
149 [Unit]
150 Description=gitbay host heartbeat
151 [Service]
152 Type=oneshot
153 ExecStart=/usr/local/bin/gitbay-monitor.sh
154
155 - path: /etc/systemd/system/gitbay-monitor.timer
156 content: |
157 [Unit]
158 Description=gitbay host heartbeat
159 [Timer]
160 OnCalendar=*-*-* *:00:00 UTC
161 Persistent=true
162 [Install]
163 WantedBy=timers.target
164 - path: /etc/systemd/system/ssh.socket.d/override.conf
165 content: |
166 [Socket]
167 ListenStream=
168 ListenStream=2222
169
170 - path: /etc/gitbay/config.toml
171 permissions: "0640"
172 content: |
173 [server]
174 root = "/var/lib/gitbay"
175 site_url = "https://gitbay.org"
176
177 [ssh]
178 mode = "embedded"
179 port = 22
180
181 [http]
182 addr = ":443"
183 tls = "acme"
184 acme_email = "hello@gitbay.org"
185 acme_http_addr = ":80"
186
187 [web]
188 mode = "view_only"
189
190 [registration]
191 mode = "closed"
192
193 # How long the append-only tables keep a row. Unset means forever,
194 # which is the default: growing is a decision, but so is deleting an
195 # audit trail. Expired sessions and tokens are swept either way.
196 # [retention]
197 # audit = "8760h" # a year
198 # events = "4380h" # six months
199 # webhook_deliveries = "720h" # a month
200 # mail = "720h"
201
202 - path: /etc/systemd/system/gitbayd.service
203 content: |
204 [Unit]
205 Description=gitbay forge daemon
206 After=network-online.target
207 Wants=network-online.target
208
209 [Service]
210 User=gitbay
211 Group=gitbay
212 ExecStart=/usr/local/bin/gitbayd --config /etc/gitbay/config.toml serve
213 Restart=on-failure
214 RestartSec=3
215
216 # Bind 22/80/443 without root; no privilege escalation afterward.
217 AmbientCapabilities=CAP_NET_BIND_SERVICE
218 CapabilityBoundingSet=CAP_NET_BIND_SERVICE
219 NoNewPrivileges=yes
220 ProtectSystem=strict
221 ProtectHome=yes
222 ReadWritePaths=/var/lib/gitbay /var/backups/gitbay
223 PrivateTmp=yes
224 ProtectKernelTunables=yes
225 ProtectKernelModules=yes
226 ProtectControlGroups=yes
227 ProtectHostname=yes
228 ProtectClock=yes
229 ProtectKernelLogs=yes
230 RestrictSUIDSGID=yes
231 RestrictNamespaces=yes
232 RestrictRealtime=yes
233 LockPersonality=yes
234 MemoryDenyWriteExecute=yes
235 PrivateDevices=yes
236 # IPv4/IPv6 for listeners and outbound git/ssh; UNIX for the hook socket.
237 RestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX
238 # Allow only ordinary service syscalls; the daemon spawns git and ssh,
239 # so keep @process/@exec available (both are within @system-service).
240 SystemCallFilter=@system-service
241 SystemCallErrorNumber=EPERM
242 SystemCallArchitectures=native
243
244 [Install]
245 WantedBy=multi-user.target
246
247 - path: /usr/local/bin/gitbay-backup.sh
248 permissions: "0755"
249 content: |
250 #!/bin/sh
251 # Nightly consistent backup; keeps the last 7 locally.
252 # To ship offsite, add an rclone/s3 upload of $out here.
253 set -eu
254 # The archive carries the database, so it gets the database's mode.
255 umask 027
256 dir=/var/backups/gitbay
257 out="$dir/gitbay-$(date -u +%Y%m%d-%H%M%S).tar.gz"
258 /usr/local/bin/gitbayd --config /etc/gitbay/config.toml admin backup --out "$out"
259 ls -1t "$dir"/gitbay-*.tar.gz | tail -n +8 | xargs -r rm --
260
261 # Hourly database-only snapshot. The nightly full backup below is the one
262 # that can rebuild the host; this one exists because the database holds
263 # issues, merge requests and comments, which unlike the repositories have
264 # no second copy anywhere. 48 of them is two days at a few MB each.
265 - path: /usr/local/bin/gitbay-db-backup.sh
266 permissions: "0755"
267 content: |
268 #!/bin/sh
269 set -eu
270 # The archive is the whole database, so it gets the database's mode.
271 umask 027
272 dir=/var/backups/gitbay/db
273 mkdir -p "$dir"
274 chmod 0750 "$dir"
275 out="$dir/gitbay-db-$(date -u +%Y%m%d-%H%M%S).tar.gz"
276 /usr/local/bin/gitbayd --config /etc/gitbay/config.toml admin backup --db-only --out "$out"
277 ls -1t "$dir"/gitbay-db-*.tar.gz | tail -n +49 | xargs -r rm --
278
279 - path: /etc/systemd/system/gitbay-db-backup.service
280 content: |
281 [Unit]
282 Description=gitbay hourly database backup
283 [Service]
284 Type=oneshot
285 User=gitbay
286 ExecStart=/usr/local/bin/gitbay-db-backup.sh
287
288 - path: /etc/systemd/system/gitbay-db-backup.timer
289 content: |
290 [Unit]
291 Description=gitbay hourly database backup
292 [Timer]
293 OnCalendar=*-*-* *:20:00 UTC
294 RandomizedDelaySec=5m
295 Persistent=true
296 [Install]
297 WantedBy=timers.target
298
299 - path: /etc/systemd/system/gitbay-backup.service
300 content: |
301 [Unit]
302 Description=gitbay nightly backup
303 [Service]
304 Type=oneshot
305 User=gitbay
306 ExecStart=/usr/local/bin/gitbay-backup.sh
307
308 - path: /etc/systemd/system/gitbay-backup.timer
309 content: |
310 [Unit]
311 Description=gitbay nightly backup
312 [Timer]
313 OnCalendar=*-*-* 09:00:00 UTC
314 RandomizedDelaySec=15m
315 Persistent=true
316 [Install]
317 WantedBy=timers.target
318
319 - path: /etc/systemd/system/gitbay-gc.service
320 content: |
321 [Unit]
322 Description=gitbay weekly repository maintenance
323 [Service]
324 Type=oneshot
325 User=gitbay
326 ExecStart=/usr/local/bin/gitbayd --config /etc/gitbay/config.toml admin gc
327
328 - path: /etc/systemd/system/gitbay-gc.timer
329 content: |
330 [Unit]
331 Description=gitbay weekly repository maintenance
332 [Timer]
333 OnCalendar=Sun *-*-* 07:00:00 UTC
334 RandomizedDelaySec=30m
335 Persistent=true
336 [Install]
337 WantedBy=timers.target
338
339runcmd:
340 - adduser --system --group --home /var/lib/gitbay --shell /usr/sbin/nologin gitbay
341 - install -d -o gitbay -g gitbay -m 750 /var/lib/gitbay /var/backups/gitbay
342 - chgrp gitbay /etc/gitbay/config.toml /etc/gitbay
343 - ufw allow 22/tcp
344 - ufw allow 80/tcp
345 - ufw allow 443/tcp
346 - ufw allow 2222/tcp
347 - ufw --force enable
348 - systemctl daemon-reload
349 - systemctl restart ssh.socket || systemctl restart ssh
350 - systemctl enable gitbayd gitbay-backup.timer gitbay-db-backup.timer gitbay-gc.timer gitbay-monitor.timer
351 - systemctl start gitbay-backup.timer gitbay-db-backup.timer gitbay-gc.timer gitbay-monitor.timer
352 - systemctl enable --now unattended-upgrades fail2ban