#cloud-config
# gitbay VPS bootstrap (Ubuntu 24.04).
#
# What this does on first boot:
#   - moves the host's admin sshd to port 2222 (gitbay's embedded SSH
#     listener owns port 22) — CONNECT ON 2222 AFTER FIRST BOOT
#   - creates the unprivileged gitbay user and directory layout
#   - installs /etc/gitbay/config.toml, the systemd unit (with
#     CAP_NET_BIND_SERVICE so ports 22/80/443 work without root), and a
#     nightly backup timer
#   - opens ufw for 22, 80, 443, 2222
#
# It does NOT install the gitbayd binary (it is not hosted anywhere yet);
# scp it to /usr/local/bin/gitbayd afterward, then create the secret key
# and hand it to the daemon user before starting:
#   gitbayd --config /etc/gitbay/config.toml admin secrets init
#   chown gitbay:gitbay /etc/gitbay/secret.key
#   systemctl start gitbayd
# On a restore, put the key file's off-host copy there instead of init.

package_update: true
packages:
  - git
  - ufw
  - unattended-upgrades
  - fail2ban
  # The CI runner shares this host and the suite drives them; without them
  # the LFS and signature tests skip themselves and CI goes green having
  # tested less.
  - git-lfs
  - gnupg

write_files:
  # Admin sshd on 2222. Ubuntu 24.04 socket-activates sshd, so the port
  # must change in BOTH sshd_config and the socket unit.
  - path: /etc/ssh/sshd_config.d/60-gitbay-port.conf
    content: |
      Port 2222
      PasswordAuthentication no
      # Throttle unauthenticated connection floods on the admin sshd
      # (gitbayd's own port 22 is throttled by limits.ssh_auth_rate).
      MaxStartups 10:30:60
      MaxAuthTries 3
      LoginGraceTime 20

  # OS security patches applied automatically; reboot at 04:30 if needed.
  - path: /etc/apt/apt.conf.d/51gitbay-unattended
    content: |
      Unattended-Upgrade::Allowed-Origins { "${distro_id}:${distro_codename}-security"; };
      Unattended-Upgrade::Automatic-Reboot "true";
      Unattended-Upgrade::Automatic-Reboot-Time "04:30";
      APT::Periodic::Update-Package-Lists "1";
      APT::Periodic::Unattended-Upgrade "1";

  # fail2ban watches the admin sshd for auth failures.
  - path: /etc/fail2ban/jail.d/gitbay.conf
    content: |
      [sshd]
      enabled = true
      port    = 2222
      backend = systemd
      maxretry = 5
      bantime  = 1h

  # Heartbeat: disk/service/cert status to journald every run, and to a
  # webhook as well if one is set in /etc/gitbay/monitor.url. Exits non-zero
  # on an alert so the unit shows up in systemctl --failed.
  - path: /usr/local/bin/gitbay-monitor.sh
    permissions: "0755"
    content: |
      #!/bin/sh
      set -eu
      disk=$(df -P /var/lib/gitbay | awk 'NR==2{print $5}')
      svc=$(systemctl is-active gitbayd || true)
      # Soonest ACME cert expiry. Reporting whichever name sorted first said
      # nothing about the one actually about to lapse, and the cache is under
      # acme/, so this read autocert/ and reported n/a forever.
      cert=/var/lib/gitbay/acme
      exp="n/a"
      days=""
      if [ -d "$cert" ]; then
        soonest=""
        for f in "$cert"/*; do
          [ -f "$f" ] || continue
          case "${f##*/}" in acme_account*) continue ;; esac
          end=$(openssl x509 -enddate -noout -in "$f" 2>/dev/null | cut -d= -f2 || true)
          [ -n "$end" ] || continue
          secs=$(date -u -d "$end" +%s 2>/dev/null || true)
          [ -n "$secs" ] || continue
          if [ -z "$soonest" ] || [ "$secs" -lt "$soonest" ]; then
            soonest="$secs"
            exp="$end"
          fi
        done
        if [ -n "$soonest" ]; then
          days=$(( (soonest - $(date -u +%s)) / 86400 ))
        fi
      fi
      # Backups are timers, and a timer failing quietly is the most
      # damaging silent failure this host has. Age of the newest full
      # archive and the newest database snapshot, in hours.
      now=$(date -u +%s)
      age_h() {
        f=$(ls -t "$1"/*.tar.gz "$1"/*.tar.gz.age 2>/dev/null | head -1)
        [ -n "$f" ] || { echo ""; return; }
        echo $(( (now - $(stat -c %Y "$f")) / 3600 ))
      }
      full_age=$(age_h /var/backups/gitbay)
      db_age=$(age_h /var/backups/gitbay/db)
      # The daemon's own word, from inside the process.
      site=$(sed -n 's/^site_url *= *"\(.*\)"/\1/p' /etc/gitbay/config.toml | head -1)
      health="n/a"
      if [ -n "$site" ]; then
        health=$(curl -fsS -m 10 "$site/healthz" 2>/dev/null | grep -o '"ok":[a-z]*' | head -1 | cut -d: -f2)
        [ -n "$health" ] || health="unreachable"
      fi
      alert=""
      if [ "$svc" != "active" ]; then
        alert="gitbayd is $svc; "
      fi
      if [ "$health" != "true" ]; then
        alert="${alert}healthz $health; "
      fi
      if [ -z "$full_age" ] || [ "$full_age" -ge 25 ]; then
        alert="${alert}full backup ${full_age:-missing}h old; "
      fi
      if [ -z "$db_age" ] || [ "$db_age" -ge 2 ]; then
        alert="${alert}db snapshot ${db_age:-missing}h old; "
      fi
      pct=$(echo "$disk" | tr -d '%')
      if [ "$pct" -ge 85 ]; then
        alert="${alert}disk ${disk}; "
      fi
      if [ -n "$days" ] && [ "$days" -lt 21 ]; then
        alert="${alert}cert expires in ${days}d; "
      fi
      # journald always gets the reading, so an unset webhook cannot make a
      # sick host look like a quiet one.
      echo "disk=$disk service=$svc healthz=$health cert_expires=$exp${days:+ cert_days=$days} full_backup_h=${full_age:-missing} db_snapshot_h=${db_age:-missing}"
      url_file=/etc/gitbay/monitor.url
      if [ -f "$url_file" ]; then
        body=$(printf '{"disk":"%s","service":"%s","healthz":"%s","cert_expires":"%s","full_backup_h":"%s","db_snapshot_h":"%s","alert":"%s"}' "$disk" "$svc" "$health" "$exp" "${full_age:-missing}" "${db_age:-missing}" "$alert")
        if ! curl -fsS -m 10 -H 'Content-Type: application/json' -d "$body" "$(cat "$url_file")" >/dev/null; then
          echo "monitor webhook post failed" >&2
        fi
      fi
      if [ -n "$alert" ]; then
        echo "$alert" >&2
        exit 1
      fi

  - path: /etc/systemd/system/gitbay-monitor.service
    content: |
      [Unit]
      Description=gitbay host heartbeat
      [Service]
      Type=oneshot
      ExecStart=/usr/local/bin/gitbay-monitor.sh

  - path: /etc/systemd/system/gitbay-monitor.timer
    content: |
      [Unit]
      Description=gitbay host heartbeat
      [Timer]
      OnCalendar=*-*-* *:00:00 UTC
      Persistent=true
      [Install]
      WantedBy=timers.target
  - path: /etc/systemd/system/ssh.socket.d/override.conf
    content: |
      [Socket]
      ListenStream=
      ListenStream=2222

  - path: /etc/gitbay/config.toml
    permissions: "0640"
    content: |
      [server]
      root = "/var/lib/gitbay"
      site_url = "https://gitbay.org"

      [ssh]
      mode = "embedded"
      port = 22

      [http]
      addr = ":443"
      tls = "acme"
      acme_email = "hello@gitbay.org"
      acme_http_addr = ":80"

      [web]
      mode = "view_only"

      [registration]
      mode = "closed"

      # How long the append-only tables keep a row. Unset means forever,
      # which is the default: growing is a decision, but so is deleting an
      # audit trail. Expired sessions and tokens are swept either way.
      # [retention]
      # audit = "8760h"              # a year
      # events = "4380h"             # six months
      # webhook_deliveries = "720h"  # a month
      # mail = "720h"

  - path: /etc/systemd/system/gitbayd.service
    content: |
      [Unit]
      Description=gitbay forge daemon
      After=network-online.target
      Wants=network-online.target

      [Service]
      User=gitbay
      Group=gitbay
      ExecStart=/usr/local/bin/gitbayd --config /etc/gitbay/config.toml serve
      Restart=on-failure
      RestartSec=3

      # Bind 22/80/443 without root; no privilege escalation afterward.
      AmbientCapabilities=CAP_NET_BIND_SERVICE
      CapabilityBoundingSet=CAP_NET_BIND_SERVICE
      NoNewPrivileges=yes
      ProtectSystem=strict
      ProtectHome=yes
      ReadWritePaths=/var/lib/gitbay /var/backups/gitbay
      PrivateTmp=yes
      ProtectKernelTunables=yes
      ProtectKernelModules=yes
      ProtectControlGroups=yes
      ProtectHostname=yes
      ProtectClock=yes
      ProtectKernelLogs=yes
      RestrictSUIDSGID=yes
      RestrictNamespaces=yes
      RestrictRealtime=yes
      LockPersonality=yes
      MemoryDenyWriteExecute=yes
      PrivateDevices=yes
      # IPv4/IPv6 for listeners and outbound git/ssh; UNIX for the hook socket.
      RestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX
      # Allow only ordinary service syscalls; the daemon spawns git and ssh,
      # so keep @process/@exec available (both are within @system-service).
      SystemCallFilter=@system-service
      SystemCallErrorNumber=EPERM
      SystemCallArchitectures=native

      [Install]
      WantedBy=multi-user.target

  - path: /usr/local/bin/gitbay-backup.sh
    permissions: "0755"
    content: |
      #!/bin/sh
      # Nightly consistent backup; keeps the last 7 locally.
      # To ship offsite, add an rclone/s3 upload of $out here.
      set -eu
      # gitbayd writes the archive 0600, owned by the backup user.
      umask 027
      dir=/var/backups/gitbay
      out="$dir/gitbay-$(date -u +%Y%m%d-%H%M%S).tar.gz"
      /usr/local/bin/gitbayd --config /etc/gitbay/config.toml admin backup --out "$out"
      ls -1t "$dir"/gitbay-*.tar.gz* | tail -n +8 | xargs -r rm --

  # Hourly database-only snapshot. The nightly full backup below is the one
  # that can rebuild the host; this one exists because the database holds
  # issues, merge requests and comments, which unlike the repositories have
  # no second copy anywhere. 48 of them is two days at a few MB each.
  - path: /usr/local/bin/gitbay-db-backup.sh
    permissions: "0755"
    content: |
      #!/bin/sh
      set -eu
      # gitbayd writes the archive 0600, owned by the backup user.
      umask 027
      dir=/var/backups/gitbay/db
      mkdir -p "$dir"
      chmod 0750 "$dir"
      out="$dir/gitbay-db-$(date -u +%Y%m%d-%H%M%S).tar.gz"
      /usr/local/bin/gitbayd --config /etc/gitbay/config.toml admin backup --db-only --out "$out"
      ls -1t "$dir"/gitbay-db-*.tar.gz* | tail -n +49 | xargs -r rm --

  - path: /etc/systemd/system/gitbay-db-backup.service
    content: |
      [Unit]
      Description=gitbay hourly database backup
      [Service]
      Type=oneshot
      User=gitbay
      ExecStart=/usr/local/bin/gitbay-db-backup.sh

  - path: /etc/systemd/system/gitbay-db-backup.timer
    content: |
      [Unit]
      Description=gitbay hourly database backup
      [Timer]
      OnCalendar=*-*-* *:20:00 UTC
      RandomizedDelaySec=5m
      Persistent=true
      [Install]
      WantedBy=timers.target

  - path: /etc/systemd/system/gitbay-backup.service
    content: |
      [Unit]
      Description=gitbay nightly backup
      [Service]
      Type=oneshot
      User=gitbay
      ExecStart=/usr/local/bin/gitbay-backup.sh

  - path: /etc/systemd/system/gitbay-backup.timer
    content: |
      [Unit]
      Description=gitbay nightly backup
      [Timer]
      OnCalendar=*-*-* 09:00:00 UTC
      RandomizedDelaySec=15m
      Persistent=true
      [Install]
      WantedBy=timers.target

  - path: /etc/systemd/system/gitbay-gc.service
    content: |
      [Unit]
      Description=gitbay weekly repository maintenance
      [Service]
      Type=oneshot
      User=gitbay
      ExecStart=/usr/local/bin/gitbayd --config /etc/gitbay/config.toml admin gc

  - path: /etc/systemd/system/gitbay-gc.timer
    content: |
      [Unit]
      Description=gitbay weekly repository maintenance
      [Timer]
      OnCalendar=Sun *-*-* 07:00:00 UTC
      RandomizedDelaySec=30m
      Persistent=true
      [Install]
      WantedBy=timers.target

runcmd:
  - adduser --system --group --home /var/lib/gitbay --shell /usr/sbin/nologin gitbay
  - install -d -o gitbay -g gitbay -m 750 /var/lib/gitbay /var/backups/gitbay
  - chgrp gitbay /etc/gitbay/config.toml /etc/gitbay
  - ufw allow 22/tcp
  - ufw allow 80/tcp
  - ufw allow 443/tcp
  - ufw allow 2222/tcp
  - ufw --force enable
  - systemctl daemon-reload
  - systemctl restart ssh.socket || systemctl restart ssh
  - systemctl enable gitbayd gitbay-backup.timer gitbay-db-backup.timer gitbay-gc.timer gitbay-monitor.timer
  - systemctl start gitbay-backup.timer gitbay-db-backup.timer gitbay-gc.timer gitbay-monitor.timer
  - systemctl enable --now unattended-upgrades fail2ban