Commit `fff6fb16e4`

fff6fb16e4a7173ab00ffc469b90b1223d1297f1

parent: 2480262fcd

Verified · cmc ci/build: success ci/test: success ci/vuln: success

cmc <hello@cleberg.net> · 2026-09-01T04:39:02Z

Make host monitoring speak, scan for vulnerabilities in CI, snapshot the database hourly

The three items #28 was narrowed to.

gitbay-monitor.sh exited 0 at its second line whenever /etc/gitbay/monitor.url
was absent, which it has been on bay1 since the script landed: disk, service
and cert results were computed hourly and dropped, and an unset webhook looked
exactly like a healthy host. It now writes the reading to journald every run,
says so when a webhook post fails instead of swallowing it, and exits non-zero
on an alert so the unit surfaces in systemctl --failed. The webhook is a second
channel rather than the only one.

govulncheck lived only in deploy/audit.sh, which is run by hand. It is a CI job
now, separate from test so an advisory published against unchanged code cannot
be mistaken for a test failure.

store.Open chmods the database to 0640. The directory above it is 0750, so 0644
was not a live exposure, but the file carries token hashes and addresses.
SQLite gives the -wal and -shm files the main file's mode, so all three follow.

admin backup --db-only writes the SQLite snapshot without the repositories,
and gitbay-db-backup.timer runs it hourly keeping 48. The full nightly archive
and its 7-day retention are untouched. Hourly full archives would have been
550MB each, and holding the existing 7-day window at that rate needs 92GB;
splitting it costs ~250MB and puts the tighter recovery point on the data that
warrants it, since repositories are git and have other copies while issues,
merge requests and comments do not. Continuous replication was considered and
rejected: it would leave the repositories on the nightly archive, so a restore
could produce a database referencing commits the repository backup lacks.
Recorded on the wiki's Admin page.

Ref #28

.gitbay/ci.yml +8

		@@ -15,3 +15,11 @@ jobs:
15	15	steps:
16	16	- missing=""; for t in git git-lfs gpg; do command -v "$t" >/dev/null \|\| missing="$missing $t"; done; test -x /usr/sbin/sshd \|\| missing="$missing sshd"; test -z "$missing" \|\| { echo "runner is missing:$missing"; exit 1; }
17	17	- go test ./... -count=1 -timeout 20m
	18	# Vulnerability scanning, separate from test so a newly published advisory
	19	# against unchanged code does not mask a real test failure. It fails the
	20	# build on purpose: an advisory that only lands in a report nobody reads is
	21	# the state this replaced. @latest matches deploy/audit.sh, so a run scans
	22	# against the database as it is today, not as it was at commit time.
	23	vuln:
	24	steps:
	25	- go run golang.org/x/vuln/cmd/govulncheck@latest ./...

cmd/gitbayd/backup.go +44 −29

		@@ -27,6 +27,7 @@ import (
27	27	// rows pointing at objects the archive never captured.
28	28	func backupCmd() *cobra.Command {
29	29	var out string
	30	var dbOnly bool
30	31	cmd := &cobra.Command{
31	32	Use: "backup",
32	33	Short: "write a consistent backup archive (database snapshot first, then repositories)",
		@@ -34,6 +35,12 @@ func backupCmd() *cobra.Command {
34	35	all repositories, and the SSH host keys. Transient state (hook socket,
35	36	regenerated hook scripts, askpass helper, WAL files) is excluded.
36	37
	38	--db-only writes the database snapshot alone. It is seconds and megabytes
	39	rather than minutes and gigabytes, which is what makes a frequent schedule
	40	affordable, and the database is the copy of issues, merge requests and
	41	comments that exists nowhere else. Repositories are not in such an archive,
	42	so it supplements a full backup and does not replace one.
	43
37	44	Restore: extract into an empty directory, point server.root at it, start
38	45	gitbayd. Host keys are preserved, so clients keep their known_hosts entries.`,
39	46	RunE: func(cmd *cobra.Command, args []string) error {
		@@ -44,14 +51,15 @@ gitbayd. Host keys are preserved, so clients keep their known_hosts entries.`,
44	51	if out == "" {
45	52	out = fmt.Sprintf("gitbay-backup-%s.tar.gz", time.Now().UTC().Format("20060102-150405"))
46	53	}
47		return runBackup(cfg, out)
	54	return runBackup(cfg, out, dbOnly)
48	55	},
49	56	}
50	57	cmd.Flags().StringVar(&out, "out", "", "output archive path (default gitbay-backup-<utc timestamp>.tar.gz)")
	58	cmd.Flags().BoolVar(&dbOnly, "db-only", false, "archive the database snapshot alone, without repositories")
51	59	return cmd
52	60	}
53	61
54		func runBackup(cfg config.Config, out string) error {
	62	func runBackup(cfg config.Config, out string, dbOnly bool) error {
55	63	st, err := openStore(cfg)
56	64	if err != nil {
57	65	return err
		@@ -79,42 +87,45 @@ func runBackup(cfg config.Config, out string) error {
79	87	}
80	88
81	89	// 2. Everything under the root except transient or regenerated state.
	90	// Skipped entirely for --db-only.
82	91	skip := map[string]bool{
83	92	"gitbay.db": true, "gitbay.db-wal": true, "gitbay.db-shm": true,
84	93	"hook.sock": true, "askpass.sh": true, "hooks": true,
85	94	}
86	95	repoCount := 0
87	96	root := cfg.Server.Root
88		err = filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error {
89		if err != nil {
90		return err
91		}
92		rel, err := filepath.Rel(root, path)
93		if err != nil {
94		return err
95		}
96		if rel == "." {
97		return nil
98		}
99		if top, _, _ := strings.Cut(rel, string(filepath.Separator)); skip[top] {
100		if d.IsDir() {
101		return filepath.SkipDir
	97	if !dbOnly {
	98	err = filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error {
	99	if err != nil {
	100	return err
102	101	}
103		return nil
104		}
105		if !d.Type().IsRegular() && !d.IsDir() {
106		return nil // sockets, symlinks
107		}
108		if d.IsDir() {
109		if strings.HasSuffix(rel, ".git") {
110		repoCount++
	102	rel, err := filepath.Rel(root, path)
	103	if err != nil {
	104	return err
	105	}
	106	if rel == "." {
	107	return nil
	108	}
	109	if top, _, _ := strings.Cut(rel, string(filepath.Separator)); skip[top] {
	110	if d.IsDir() {
	111	return filepath.SkipDir
	112	}
	113	return nil
	114	}
	115	if !d.Type().IsRegular() && !d.IsDir() {
	116	return nil // sockets, symlinks
	117	}
	118	if d.IsDir() {
	119	if strings.HasSuffix(rel, ".git") {
	120	repoCount++
	121	}
	122	return nil // directories are implied by member paths
111	123	}
112		return nil // directories are implied by member paths
	124	return addFile(tw, path, filepath.ToSlash(rel))
	125	})
	126	if err != nil {
	127	return err
113	128	}
114		return addFile(tw, path, filepath.ToSlash(rel))
115		})
116		if err != nil {
117		return err
118	129	}
119	130	if err := tw.Close(); err != nil {
120	131	return err
		@@ -127,6 +138,10 @@ func runBackup(cfg config.Config, out string) error {
127	138	}
128	139
129	140	info, _ := os.Stat(out)
	141	if dbOnly {
	142	fmt.Printf("wrote %s (database only, %.1f MB)\n", out, float64(info.Size())/1e6)
	143	return nil
	144	}
130	145	fmt.Printf("wrote %s (%d repositories, %.1f MB)\n", out, repoCount, float64(info.Size())/1e6)
131	146	return nil
132	147	}

cmd/gitbayd/backup_test.go added +96

		@@ -0,0 +1,96 @@
	1	package main
	2
	3	import (
	4	"archive/tar"
	5	"compress/gzip"
	6	"io"
	7	"os"
	8	"path/filepath"
	9	"sort"
	10	"testing"
	11
	12	"gitbay.org/gitbay/internal/config"
	13	)
	14
	15	// members lists the archive's entries by name.
	16	func members(t *testing.T, path string) []string {
	17	t.Helper()
	18	f, err := os.Open(path)
	19	if err != nil {
	20	t.Fatal(err)
	21	}
	22	defer f.Close()
	23	gz, err := gzip.NewReader(f)
	24	if err != nil {
	25	t.Fatal(err)
	26	}
	27	var names []string
	28	tr := tar.NewReader(gz)
	29	for {
	30	hdr, err := tr.Next()
	31	if err == io.EOF {
	32	break
	33	}
	34	if err != nil {
	35	t.Fatal(err)
	36	}
	37	names = append(names, hdr.Name)
	38	}
	39	sort.Strings(names)
	40	return names
	41	}
	42
	43	// --db-only is what makes an hourly schedule affordable, so it has to leave
	44	// the repositories out and still carry a restorable database.
	45	func TestBackupDBOnlyOmitsRepositories(t *testing.T) {
	46	root := t.TempDir()
	47	cfg := config.Config{Server: config.Server{Root: root}}
	48	s, err := openStore(cfg)
	49	if err != nil {
	50	t.Fatal(err)
	51	}
	52	s.Close()
	53
	54	repo := filepath.Join(root, "repos", "krz", "thing.git")
	55	if err := os.MkdirAll(repo, 0o750); err != nil {
	56	t.Fatal(err)
	57	}
	58	if err := os.WriteFile(filepath.Join(repo, "HEAD"), []byte("ref: refs/heads/main\n"), 0o640); err != nil {
	59	t.Fatal(err)
	60	}
	61
	62	full := filepath.Join(t.TempDir(), "full.tar.gz")
	63	if err := runBackup(cfg, full, false); err != nil {
	64	t.Fatalf("full backup: %v", err)
	65	}
	66	dbOnly := filepath.Join(t.TempDir(), "db.tar.gz")
	67	if err := runBackup(cfg, dbOnly, true); err != nil {
	68	t.Fatalf("db-only backup: %v", err)
	69	}
	70
	71	fullNames := members(t, full)
	72	if len(fullNames) < 2 {
	73	t.Fatalf("full backup carries only %v", fullNames)
	74	}
	75	var sawRepo bool
	76	for _, n := range fullNames {
	77	if n == "repos/krz/thing.git/HEAD" {
	78	sawRepo = true
	79	}
	80	}
	81	if !sawRepo {
	82	t.Errorf("full backup is missing the repository: %v", fullNames)
	83	}
	84
	85	if got := members(t, dbOnly); len(got) != 1 \|\| got[0] != "gitbay.db" {
	86	t.Errorf("db-only backup carries %v, want [gitbay.db]", got)
	87	}
	88
	89	fi, err := os.Stat(dbOnly)
	90	if err != nil {
	91	t.Fatal(err)
	92	}
	93	if fi.Size() == 0 {
	94	t.Error("db-only backup is empty")
	95	}
	96	}

deploy/cloud-init.yaml +63 −12

		@@ -57,16 +57,14 @@ write_files:
57	57	maxretry = 5
58	58	bantime = 1h
59	59
60		# Heartbeat: post disk/service/cert status to a webhook if one is set in
61		# /etc/gitbay/monitor.url. Silent when the file is absent.
	60	# Heartbeat: disk/service/cert status to journald every run, and to a
	61	# webhook as well if one is set in /etc/gitbay/monitor.url. Exits non-zero
	62	# on an alert so the unit shows up in systemctl --failed.
62	63	- path: /usr/local/bin/gitbay-monitor.sh
63	64	permissions: "0755"
64	65	content: \|
65	66	#!/bin/sh
66	67	set -eu
67		url_file=/etc/gitbay/monitor.url
68		[ -f "$url_file" ] \|\| exit 0
69		url=$(cat "$url_file")
70	68	disk=$(df -P /var/lib/gitbay \| awk 'NR==2{print $5}')
71	69	svc=$(systemctl is-active gitbayd \|\| true)
72	70	# Days until the ACME cert expires, if autocert cached one.
		@@ -74,14 +72,32 @@ write_files:
74	72	exp="n/a"
75	73	if [ -d "$cert" ]; then
76	74	f=$(ls -1 "$cert" 2>/dev/null \| grep -v acme_account \| head -1 \|\| true)
77		[ -n "$f" ] && exp=$(openssl x509 -enddate -noout -in "$cert/$f" 2>/dev/null \| cut -d= -f2 \|\| echo n/a)
	75	if [ -n "$f" ]; then
	76	exp=$(openssl x509 -enddate -noout -in "$cert/$f" 2>/dev/null \| cut -d= -f2 \|\| echo n/a)
	77	fi
78	78	fi
79	79	alert=""
80		[ "$svc" != "active" ] && alert="gitbayd is $svc; "
	80	if [ "$svc" != "active" ]; then
	81	alert="gitbayd is $svc; "
	82	fi
81	83	pct=$(echo "$disk" \| tr -d '%')
82		[ "$pct" -ge 85 ] && alert="${alert}disk ${disk}; "
83		body=$(printf '{"disk":"%s","service":"%s","cert_expires":"%s","alert":"%s"}' "$disk" "$svc" "$exp" "$alert")
84		curl -fsS -m 10 -H 'Content-Type: application/json' -d "$body" "$url" >/dev/null 2>&1 \|\| true
	84	if [ "$pct" -ge 85 ]; then
	85	alert="${alert}disk ${disk}; "
	86	fi
	87	# journald always gets the reading, so an unset webhook cannot make a
	88	# sick host look like a quiet one.
	89	echo "disk=$disk service=$svc cert_expires=$exp"
	90	url_file=/etc/gitbay/monitor.url
	91	if [ -f "$url_file" ]; then
	92	body=$(printf '{"disk":"%s","service":"%s","cert_expires":"%s","alert":"%s"}' "$disk" "$svc" "$exp" "$alert")
	93	if ! curl -fsS -m 10 -H 'Content-Type: application/json' -d "$body" "$(cat "$url_file")" >/dev/null; then
	94	echo "monitor webhook post failed" >&2
	95	fi
	96	fi
	97	if [ -n "$alert" ]; then
	98	echo "$alert" >&2
	99	exit 1
	100	fi
85	101
86	102	- path: /etc/systemd/system/gitbay-monitor.service
87	103	content: \|
		@@ -186,6 +202,41 @@ write_files:
186	202	/usr/local/bin/gitbayd --config /etc/gitbay/config.toml admin backup --out "$out"
187	203	ls -1t "$dir"/gitbay-*.tar.gz \| tail -n +8 \| xargs -r rm --
188	204
	205	# Hourly database-only snapshot. The nightly full backup below is the one
	206	# that can rebuild the host; this one exists because the database holds
	207	# issues, merge requests and comments, which unlike the repositories have
	208	# no second copy anywhere. 48 of them is two days at a few MB each.
	209	- path: /usr/local/bin/gitbay-db-backup.sh
	210	permissions: "0755"
	211	content: \|
	212	#!/bin/sh
	213	set -eu
	214	dir=/var/backups/gitbay/db
	215	mkdir -p "$dir"
	216	out="$dir/gitbay-db-$(date -u +%Y%m%d-%H%M%S).tar.gz"
	217	/usr/local/bin/gitbayd --config /etc/gitbay/config.toml admin backup --db-only --out "$out"
	218	ls -1t "$dir"/gitbay-db-*.tar.gz \| tail -n +49 \| xargs -r rm --
	219
	220	- path: /etc/systemd/system/gitbay-db-backup.service
	221	content: \|
	222	[Unit]
	223	Description=gitbay hourly database backup
	224	[Service]
	225	Type=oneshot
	226	User=gitbay
	227	ExecStart=/usr/local/bin/gitbay-db-backup.sh
	228
	229	- path: /etc/systemd/system/gitbay-db-backup.timer
	230	content: \|
	231	[Unit]
	232	Description=gitbay hourly database backup
	233	[Timer]
	234	OnCalendar=--* *:20:00 UTC
	235	RandomizedDelaySec=5m
	236	Persistent=true
	237	[Install]
	238	WantedBy=timers.target
	239
189	240	- path: /etc/systemd/system/gitbay-backup.service
190	241	content: \|
191	242	[Unit]
		@@ -237,6 +288,6 @@ runcmd:
237	288	- ufw --force enable
238	289	- systemctl daemon-reload
239	290	- systemctl restart ssh.socket \|\| systemctl restart ssh
240		- systemctl enable gitbayd gitbay-backup.timer gitbay-gc.timer gitbay-monitor.timer
241		- systemctl start gitbay-backup.timer gitbay-gc.timer gitbay-monitor.timer
	291	- systemctl enable gitbayd gitbay-backup.timer gitbay-db-backup.timer gitbay-gc.timer gitbay-monitor.timer
	292	- systemctl start gitbay-backup.timer gitbay-db-backup.timer gitbay-gc.timer gitbay-monitor.timer
242	293	- systemctl enable --now unattended-upgrades fail2ban

internal/store/store.go +12

		@@ -4,8 +4,10 @@ package store
4	4	import (
5	5	"database/sql"
6	6	"embed"
	7	"errors"
7	8	"fmt"
8	9	"io/fs"
	10	"os"
9	11	"sort"
10	12	"strconv"
11	13	"strings"
		@@ -35,6 +37,16 @@ func Open(path string) (*Store, error) {
35	37	db.Close()
36	38	return nil, err
37	39	}
	40	// SQLite creates the file 0666&~umask, so it lands 0644 by default. The
	41	// directory above it is the real boundary, but the file holds token
	42	// hashes, addresses and private repo names and has no business being
	43	// world-readable on its own.
	44	if path != ":memory:" {
	45	if err := os.Chmod(path, 0o640); err != nil && !errors.Is(err, fs.ErrNotExist) {
	46	db.Close()
	47	return nil, err
	48	}
	49	}
38	50	return &Store{DB: db}, nil
39	51	}
40	52

internal/store/store_test.go +19

		@@ -1,6 +1,7 @@
1	1	package store
2	2
3	3	import (
	4	"os"
4	5	"path/filepath"
5	6	"strings"
6	7	"testing"
		@@ -96,3 +97,21 @@ func TestForeignKeysEnforced(t *testing.T) {
96	97	t.Fatal("insert with dangling user_id succeeded; foreign keys are off")
97	98	}
98	99	}
	100
	101	// The database file carries token hashes, addresses and private repo names.
	102	// The directory above it is the real boundary; this is the second one.
	103	func TestDatabaseFileIsNotWorldReadable(t *testing.T) {
	104	path := filepath.Join(t.TempDir(), "gitbay.db")
	105	s, err := Open(path)
	106	if err != nil {
	107	t.Fatal(err)
	108	}
	109	defer s.Close()
	110	fi, err := os.Stat(path)
	111	if err != nil {
	112	t.Fatal(err)
	113	}
	114	if mode := fi.Mode().Perm(); mode&0o007 != 0 {
	115	t.Errorf("database mode %04o is other-readable", mode)
	116	}
	117	}

Commit fff6fb16e4

Commit `fff6fb16e4`