deploy/cloud-init.yaml
357 lines · 12505 bytes
1#cloud-config
2# gitbay VPS bootstrap (Ubuntu 24.04).
3#
4# What this does on first boot:
5# - moves the host's admin sshd to port 2222 (gitbay's embedded SSH
6# listener owns port 22) — CONNECT ON 2222 AFTER FIRST BOOT
7# - creates the unprivileged gitbay user and directory layout
8# - installs /etc/gitbay/config.toml, the systemd unit (with
9# CAP_NET_BIND_SERVICE so ports 22/80/443 work without root), and a
10# nightly backup timer
11# - opens ufw for 22, 80, 443, 2222
12#
13# It does NOT install the gitbayd binary (it is not hosted anywhere yet);
14# scp it to /usr/local/bin/gitbayd afterward, then create the secret key
15# and hand it to the daemon user before starting:
16# gitbayd --config /etc/gitbay/config.toml admin secrets init
17# chown gitbay:gitbay /etc/gitbay/secret.key
18# systemctl start gitbayd
19# On a restore, put the key file's off-host copy there instead of init.
20
21package_update: true
22packages:
23 - git
24 - ufw
25 - unattended-upgrades
26 - fail2ban
27 # The CI runner shares this host and the suite drives them; without them
28 # the LFS and signature tests skip themselves and CI goes green having
29 # tested less.
30 - git-lfs
31 - gnupg
32
33write_files:
34 # Admin sshd on 2222. Ubuntu 24.04 socket-activates sshd, so the port
35 # must change in BOTH sshd_config and the socket unit.
36 - path: /etc/ssh/sshd_config.d/60-gitbay-port.conf
37 content: |
38 Port 2222
39 PasswordAuthentication no
40 # Throttle unauthenticated connection floods on the admin sshd
41 # (gitbayd's own port 22 is throttled by limits.ssh_auth_rate).
42 MaxStartups 10:30:60
43 MaxAuthTries 3
44 LoginGraceTime 20
45
46 # OS security patches applied automatically; reboot at 04:30 if needed.
47 - path: /etc/apt/apt.conf.d/51gitbay-unattended
48 content: |
49 Unattended-Upgrade::Allowed-Origins { "${distro_id}:${distro_codename}-security"; };
50 Unattended-Upgrade::Automatic-Reboot "true";
51 Unattended-Upgrade::Automatic-Reboot-Time "04:30";
52 APT::Periodic::Update-Package-Lists "1";
53 APT::Periodic::Unattended-Upgrade "1";
54
55 # fail2ban watches the admin sshd for auth failures.
56 - path: /etc/fail2ban/jail.d/gitbay.conf
57 content: |
58 [sshd]
59 enabled = true
60 port = 2222
61 backend = systemd
62 maxretry = 5
63 bantime = 1h
64
65 # Heartbeat: disk/service/cert status to journald every run, and to a
66 # webhook as well if one is set in /etc/gitbay/monitor.url. Exits non-zero
67 # on an alert so the unit shows up in systemctl --failed.
68 - path: /usr/local/bin/gitbay-monitor.sh
69 permissions: "0755"
70 content: |
71 #!/bin/sh
72 set -eu
73 disk=$(df -P /var/lib/gitbay | awk 'NR==2{print $5}')
74 svc=$(systemctl is-active gitbayd || true)
75 # Soonest ACME cert expiry. Reporting whichever name sorted first said
76 # nothing about the one actually about to lapse, and the cache is under
77 # acme/, so this read autocert/ and reported n/a forever.
78 cert=/var/lib/gitbay/acme
79 exp="n/a"
80 days=""
81 if [ -d "$cert" ]; then
82 soonest=""
83 for f in "$cert"/*; do
84 [ -f "$f" ] || continue
85 case "${f##*/}" in acme_account*) continue ;; esac
86 end=$(openssl x509 -enddate -noout -in "$f" 2>/dev/null | cut -d= -f2 || true)
87 [ -n "$end" ] || continue
88 secs=$(date -u -d "$end" +%s 2>/dev/null || true)
89 [ -n "$secs" ] || continue
90 if [ -z "$soonest" ] || [ "$secs" -lt "$soonest" ]; then
91 soonest="$secs"
92 exp="$end"
93 fi
94 done
95 if [ -n "$soonest" ]; then
96 days=$(( (soonest - $(date -u +%s)) / 86400 ))
97 fi
98 fi
99 # Backups are timers, and a timer failing quietly is the most
100 # damaging silent failure this host has. Age of the newest full
101 # archive and the newest database snapshot, in hours.
102 now=$(date -u +%s)
103 age_h() {
104 f=$(ls -t "$1"/*.tar.gz "$1"/*.tar.gz.age 2>/dev/null | head -1)
105 [ -n "$f" ] || { echo ""; return; }
106 echo $(( (now - $(stat -c %Y "$f")) / 3600 ))
107 }
108 full_age=$(age_h /var/backups/gitbay)
109 db_age=$(age_h /var/backups/gitbay/db)
110 # The daemon's own word, from inside the process.
111 site=$(sed -n 's/^site_url *= *"\(.*\)"/\1/p' /etc/gitbay/config.toml | head -1)
112 health="n/a"
113 if [ -n "$site" ]; then
114 health=$(curl -fsS -m 10 "$site/healthz" 2>/dev/null | grep -o '"ok":[a-z]*' | head -1 | cut -d: -f2)
115 [ -n "$health" ] || health="unreachable"
116 fi
117 alert=""
118 if [ "$svc" != "active" ]; then
119 alert="gitbayd is $svc; "
120 fi
121 if [ "$health" != "true" ]; then
122 alert="${alert}healthz $health; "
123 fi
124 if [ -z "$full_age" ] || [ "$full_age" -ge 25 ]; then
125 alert="${alert}full backup ${full_age:-missing}h old; "
126 fi
127 if [ -z "$db_age" ] || [ "$db_age" -ge 2 ]; then
128 alert="${alert}db snapshot ${db_age:-missing}h old; "
129 fi
130 pct=$(echo "$disk" | tr -d '%')
131 if [ "$pct" -ge 85 ]; then
132 alert="${alert}disk ${disk}; "
133 fi
134 if [ -n "$days" ] && [ "$days" -lt 21 ]; then
135 alert="${alert}cert expires in ${days}d; "
136 fi
137 # journald always gets the reading, so an unset webhook cannot make a
138 # sick host look like a quiet one.
139 echo "disk=$disk service=$svc healthz=$health cert_expires=$exp${days:+ cert_days=$days} full_backup_h=${full_age:-missing} db_snapshot_h=${db_age:-missing}"
140 url_file=/etc/gitbay/monitor.url
141 if [ -f "$url_file" ]; then
142 body=$(printf '{"disk":"%s","service":"%s","healthz":"%s","cert_expires":"%s","full_backup_h":"%s","db_snapshot_h":"%s","alert":"%s"}' "$disk" "$svc" "$health" "$exp" "${full_age:-missing}" "${db_age:-missing}" "$alert")
143 if ! curl -fsS -m 10 -H 'Content-Type: application/json' -d "$body" "$(cat "$url_file")" >/dev/null; then
144 echo "monitor webhook post failed" >&2
145 fi
146 fi
147 if [ -n "$alert" ]; then
148 echo "$alert" >&2
149 exit 1
150 fi
151
152 - path: /etc/systemd/system/gitbay-monitor.service
153 content: |
154 [Unit]
155 Description=gitbay host heartbeat
156 [Service]
157 Type=oneshot
158 ExecStart=/usr/local/bin/gitbay-monitor.sh
159
160 - path: /etc/systemd/system/gitbay-monitor.timer
161 content: |
162 [Unit]
163 Description=gitbay host heartbeat
164 [Timer]
165 OnCalendar=*-*-* *:00:00 UTC
166 Persistent=true
167 [Install]
168 WantedBy=timers.target
169 - path: /etc/systemd/system/ssh.socket.d/override.conf
170 content: |
171 [Socket]
172 ListenStream=
173 ListenStream=2222
174
175 - path: /etc/gitbay/config.toml
176 permissions: "0640"
177 content: |
178 [server]
179 root = "/var/lib/gitbay"
180 site_url = "https://gitbay.org"
181
182 [ssh]
183 mode = "embedded"
184 port = 22
185
186 [http]
187 addr = ":443"
188 tls = "acme"
189 acme_email = "hello@gitbay.org"
190 acme_http_addr = ":80"
191
192 [web]
193 mode = "view_only"
194
195 [registration]
196 mode = "closed"
197
198 # How long the append-only tables keep a row. Unset means forever,
199 # which is the default: growing is a decision, but so is deleting an
200 # audit trail. Expired sessions and tokens are swept either way.
201 # [retention]
202 # audit = "8760h" # a year
203 # events = "4380h" # six months
204 # webhook_deliveries = "720h" # a month
205 # mail = "720h"
206
207 - path: /etc/systemd/system/gitbayd.service
208 content: |
209 [Unit]
210 Description=gitbay forge daemon
211 After=network-online.target
212 Wants=network-online.target
213
214 [Service]
215 User=gitbay
216 Group=gitbay
217 ExecStart=/usr/local/bin/gitbayd --config /etc/gitbay/config.toml serve
218 Restart=on-failure
219 RestartSec=3
220
221 # Bind 22/80/443 without root; no privilege escalation afterward.
222 AmbientCapabilities=CAP_NET_BIND_SERVICE
223 CapabilityBoundingSet=CAP_NET_BIND_SERVICE
224 NoNewPrivileges=yes
225 ProtectSystem=strict
226 ProtectHome=yes
227 ReadWritePaths=/var/lib/gitbay /var/backups/gitbay
228 PrivateTmp=yes
229 ProtectKernelTunables=yes
230 ProtectKernelModules=yes
231 ProtectControlGroups=yes
232 ProtectHostname=yes
233 ProtectClock=yes
234 ProtectKernelLogs=yes
235 RestrictSUIDSGID=yes
236 RestrictNamespaces=yes
237 RestrictRealtime=yes
238 LockPersonality=yes
239 MemoryDenyWriteExecute=yes
240 PrivateDevices=yes
241 # IPv4/IPv6 for listeners and outbound git/ssh; UNIX for the hook socket.
242 RestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX
243 # Allow only ordinary service syscalls; the daemon spawns git and ssh,
244 # so keep @process/@exec available (both are within @system-service).
245 SystemCallFilter=@system-service
246 SystemCallErrorNumber=EPERM
247 SystemCallArchitectures=native
248
249 [Install]
250 WantedBy=multi-user.target
251
252 - path: /usr/local/bin/gitbay-backup.sh
253 permissions: "0755"
254 content: |
255 #!/bin/sh
256 # Nightly consistent backup; keeps the last 7 locally.
257 # To ship offsite, add an rclone/s3 upload of $out here.
258 set -eu
259 # gitbayd writes the archive 0600, owned by the backup user.
260 umask 027
261 dir=/var/backups/gitbay
262 out="$dir/gitbay-$(date -u +%Y%m%d-%H%M%S).tar.gz"
263 /usr/local/bin/gitbayd --config /etc/gitbay/config.toml admin backup --out "$out"
264 ls -1t "$dir"/gitbay-*.tar.gz* | tail -n +8 | xargs -r rm --
265
266 # Hourly database-only snapshot. The nightly full backup below is the one
267 # that can rebuild the host; this one exists because the database holds
268 # issues, merge requests and comments, which unlike the repositories have
269 # no second copy anywhere. 48 of them is two days at a few MB each.
270 - path: /usr/local/bin/gitbay-db-backup.sh
271 permissions: "0755"
272 content: |
273 #!/bin/sh
274 set -eu
275 # gitbayd writes the archive 0600, owned by the backup user.
276 umask 027
277 dir=/var/backups/gitbay/db
278 mkdir -p "$dir"
279 chmod 0750 "$dir"
280 out="$dir/gitbay-db-$(date -u +%Y%m%d-%H%M%S).tar.gz"
281 /usr/local/bin/gitbayd --config /etc/gitbay/config.toml admin backup --db-only --out "$out"
282 ls -1t "$dir"/gitbay-db-*.tar.gz* | tail -n +49 | xargs -r rm --
283
284 - path: /etc/systemd/system/gitbay-db-backup.service
285 content: |
286 [Unit]
287 Description=gitbay hourly database backup
288 [Service]
289 Type=oneshot
290 User=gitbay
291 ExecStart=/usr/local/bin/gitbay-db-backup.sh
292
293 - path: /etc/systemd/system/gitbay-db-backup.timer
294 content: |
295 [Unit]
296 Description=gitbay hourly database backup
297 [Timer]
298 OnCalendar=*-*-* *:20:00 UTC
299 RandomizedDelaySec=5m
300 Persistent=true
301 [Install]
302 WantedBy=timers.target
303
304 - path: /etc/systemd/system/gitbay-backup.service
305 content: |
306 [Unit]
307 Description=gitbay nightly backup
308 [Service]
309 Type=oneshot
310 User=gitbay
311 ExecStart=/usr/local/bin/gitbay-backup.sh
312
313 - path: /etc/systemd/system/gitbay-backup.timer
314 content: |
315 [Unit]
316 Description=gitbay nightly backup
317 [Timer]
318 OnCalendar=*-*-* 09:00:00 UTC
319 RandomizedDelaySec=15m
320 Persistent=true
321 [Install]
322 WantedBy=timers.target
323
324 - path: /etc/systemd/system/gitbay-gc.service
325 content: |
326 [Unit]
327 Description=gitbay weekly repository maintenance
328 [Service]
329 Type=oneshot
330 User=gitbay
331 ExecStart=/usr/local/bin/gitbayd --config /etc/gitbay/config.toml admin gc
332
333 - path: /etc/systemd/system/gitbay-gc.timer
334 content: |
335 [Unit]
336 Description=gitbay weekly repository maintenance
337 [Timer]
338 OnCalendar=Sun *-*-* 07:00:00 UTC
339 RandomizedDelaySec=30m
340 Persistent=true
341 [Install]
342 WantedBy=timers.target
343
344runcmd:
345 - adduser --system --group --home /var/lib/gitbay --shell /usr/sbin/nologin gitbay
346 - install -d -o gitbay -g gitbay -m 750 /var/lib/gitbay /var/backups/gitbay
347 - chgrp gitbay /etc/gitbay/config.toml /etc/gitbay
348 - ufw allow 22/tcp
349 - ufw allow 80/tcp
350 - ufw allow 443/tcp
351 - ufw allow 2222/tcp
352 - ufw --force enable
353 - systemctl daemon-reload
354 - systemctl restart ssh.socket || systemctl restart ssh
355 - systemctl enable gitbayd gitbay-backup.timer gitbay-db-backup.timer gitbay-gc.timer gitbay-monitor.timer
356 - systemctl start gitbay-backup.timer gitbay-db-backup.timer gitbay-gc.timer gitbay-monitor.timer
357 - systemctl enable --now unattended-upgrades fail2ban