krz/gitbay

Make host monitoring speak, scan for vulnerabilities in CI, snapshot the database hourly !145

merged merged by cmc on 2026-09-01 04:45 UTC · krz/gitbay:sec-28-remaining into main

6 files changed, +242 −41

Layout: unified · split

.gitbay/ci.yml +8
@@ -15,3 +15,11 @@ jobs:
15 steps: 15 steps:
16 - missing=""; for t in git git-lfs gpg; do command -v "$t" >/dev/null || missing="$missing $t"; done; test -x /usr/sbin/sshd || missing="$missing sshd"; test -z "$missing" || { echo "runner is missing:$missing"; exit 1; } 16 - missing=""; for t in git git-lfs gpg; do command -v "$t" >/dev/null || missing="$missing $t"; done; test -x /usr/sbin/sshd || missing="$missing sshd"; test -z "$missing" || { echo "runner is missing:$missing"; exit 1; }
17 - go test ./... -count=1 -timeout 20m 17 - go test ./... -count=1 -timeout 20m
18 # Vulnerability scanning, separate from test so a newly published advisory
19 # against unchanged code does not mask a real test failure. It fails the
20 # build on purpose: an advisory that only lands in a report nobody reads is
21 # the state this replaced. @latest matches deploy/audit.sh, so a run scans
22 # against the database as it is today, not as it was at commit time.
23 vuln:
24 steps:
25 - go run golang.org/x/vuln/cmd/govulncheck@latest ./...
cmd/gitbayd/backup.go +44 −29
@@ -27,6 +27,7 @@ import (
27// rows pointing at objects the archive never captured. 27// rows pointing at objects the archive never captured.
28func backupCmd() *cobra.Command { 28func backupCmd() *cobra.Command {
29 var out string 29 var out string
30 var dbOnly bool
30 cmd := &cobra.Command{ 31 cmd := &cobra.Command{
31 Use: "backup", 32 Use: "backup",
32 Short: "write a consistent backup archive (database snapshot first, then repositories)", 33 Short: "write a consistent backup archive (database snapshot first, then repositories)",
@@ -34,6 +35,12 @@ func backupCmd() *cobra.Command {
34all repositories, and the SSH host keys. Transient state (hook socket, 35all repositories, and the SSH host keys. Transient state (hook socket,
35regenerated hook scripts, askpass helper, WAL files) is excluded. 36regenerated hook scripts, askpass helper, WAL files) is excluded.
36 37
38--db-only writes the database snapshot alone. It is seconds and megabytes
39rather than minutes and gigabytes, which is what makes a frequent schedule
40affordable, and the database is the copy of issues, merge requests and
41comments that exists nowhere else. Repositories are not in such an archive,
42so it supplements a full backup and does not replace one.
43
37Restore: extract into an empty directory, point server.root at it, start 44Restore: extract into an empty directory, point server.root at it, start
38gitbayd. Host keys are preserved, so clients keep their known_hosts entries.`, 45gitbayd. Host keys are preserved, so clients keep their known_hosts entries.`,
39 RunE: func(cmd *cobra.Command, args []string) error { 46 RunE: func(cmd *cobra.Command, args []string) error {
@@ -44,14 +51,15 @@ gitbayd. Host keys are preserved, so clients keep their known_hosts entries.`,
44 if out == "" { 51 if out == "" {
45 out = fmt.Sprintf("gitbay-backup-%s.tar.gz", time.Now().UTC().Format("20060102-150405")) 52 out = fmt.Sprintf("gitbay-backup-%s.tar.gz", time.Now().UTC().Format("20060102-150405"))
46 } 53 }
47 return runBackup(cfg, out) 54 return runBackup(cfg, out, dbOnly)
48 }, 55 },
49 } 56 }
50 cmd.Flags().StringVar(&out, "out", "", "output archive path (default gitbay-backup-<utc timestamp>.tar.gz)") 57 cmd.Flags().StringVar(&out, "out", "", "output archive path (default gitbay-backup-<utc timestamp>.tar.gz)")
58 cmd.Flags().BoolVar(&dbOnly, "db-only", false, "archive the database snapshot alone, without repositories")
51 return cmd 59 return cmd
52} 60}
53 61
54func runBackup(cfg config.Config, out string) error { 62func runBackup(cfg config.Config, out string, dbOnly bool) error {
55 st, err := openStore(cfg) 63 st, err := openStore(cfg)
56 if err != nil { 64 if err != nil {
57 return err 65 return err
@@ -79,42 +87,45 @@ func runBackup(cfg config.Config, out string) error {
79 } 87 }
80 88
81 // 2. Everything under the root except transient or regenerated state. 89 // 2. Everything under the root except transient or regenerated state.
90 // Skipped entirely for --db-only.
82 skip := map[string]bool{ 91 skip := map[string]bool{
83 "gitbay.db": true, "gitbay.db-wal": true, "gitbay.db-shm": true, 92 "gitbay.db": true, "gitbay.db-wal": true, "gitbay.db-shm": true,
84 "hook.sock": true, "askpass.sh": true, "hooks": true, 93 "hook.sock": true, "askpass.sh": true, "hooks": true,
85 } 94 }
86 repoCount := 0 95 repoCount := 0
87 root := cfg.Server.Root 96 root := cfg.Server.Root
88 err = filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error { 97 if !dbOnly {
89 if err != nil { 98 err = filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error {
90 return err 99 if err != nil {
91 } 100 return err
92 rel, err := filepath.Rel(root, path)
93 if err != nil {
94 return err
95 }
96 if rel == "." {
97 return nil
98 }
99 if top, _, _ := strings.Cut(rel, string(filepath.Separator)); skip[top] {
100 if d.IsDir() {
101 return filepath.SkipDir
102 } 101 }
103 return nil 102 rel, err := filepath.Rel(root, path)
104 } 103 if err != nil {
105 if !d.Type().IsRegular() && !d.IsDir() { 104 return err
106 return nil // sockets, symlinks 105 }
107 } 106 if rel == "." {
108 if d.IsDir() { 107 return nil
109 if strings.HasSuffix(rel, ".git") { 108 }
110 repoCount++ 109 if top, _, _ := strings.Cut(rel, string(filepath.Separator)); skip[top] {
110 if d.IsDir() {
111 return filepath.SkipDir
112 }
113 return nil
114 }
115 if !d.Type().IsRegular() && !d.IsDir() {
116 return nil // sockets, symlinks
117 }
118 if d.IsDir() {
119 if strings.HasSuffix(rel, ".git") {
120 repoCount++
121 }
122 return nil // directories are implied by member paths
111 } 123 }
112 return nil // directories are implied by member paths 124 return addFile(tw, path, filepath.ToSlash(rel))
125 })
126 if err != nil {
127 return err
113 } 128 }
114 return addFile(tw, path, filepath.ToSlash(rel))
115 })
116 if err != nil {
117 return err
118 } 129 }
119 if err := tw.Close(); err != nil { 130 if err := tw.Close(); err != nil {
120 return err 131 return err
@@ -127,6 +138,10 @@ func runBackup(cfg config.Config, out string) error {
127 } 138 }
128 139
129 info, _ := os.Stat(out) 140 info, _ := os.Stat(out)
141 if dbOnly {
142 fmt.Printf("wrote %s (database only, %.1f MB)\n", out, float64(info.Size())/1e6)
143 return nil
144 }
130 fmt.Printf("wrote %s (%d repositories, %.1f MB)\n", out, repoCount, float64(info.Size())/1e6) 145 fmt.Printf("wrote %s (%d repositories, %.1f MB)\n", out, repoCount, float64(info.Size())/1e6)
131 return nil 146 return nil
132} 147}
cmd/gitbayd/backup_test.go added +96
@@ -0,0 +1,96 @@
1package main
2
3import (
4 "archive/tar"
5 "compress/gzip"
6 "io"
7 "os"
8 "path/filepath"
9 "sort"
10 "testing"
11
12 "gitbay.org/gitbay/internal/config"
13)
14
15// members lists the archive's entries by name.
16func members(t *testing.T, path string) []string {
17 t.Helper()
18 f, err := os.Open(path)
19 if err != nil {
20 t.Fatal(err)
21 }
22 defer f.Close()
23 gz, err := gzip.NewReader(f)
24 if err != nil {
25 t.Fatal(err)
26 }
27 var names []string
28 tr := tar.NewReader(gz)
29 for {
30 hdr, err := tr.Next()
31 if err == io.EOF {
32 break
33 }
34 if err != nil {
35 t.Fatal(err)
36 }
37 names = append(names, hdr.Name)
38 }
39 sort.Strings(names)
40 return names
41}
42
43// --db-only is what makes an hourly schedule affordable, so it has to leave
44// the repositories out and still carry a restorable database.
45func TestBackupDBOnlyOmitsRepositories(t *testing.T) {
46 root := t.TempDir()
47 cfg := config.Config{Server: config.Server{Root: root}}
48 s, err := openStore(cfg)
49 if err != nil {
50 t.Fatal(err)
51 }
52 s.Close()
53
54 repo := filepath.Join(root, "repos", "krz", "thing.git")
55 if err := os.MkdirAll(repo, 0o750); err != nil {
56 t.Fatal(err)
57 }
58 if err := os.WriteFile(filepath.Join(repo, "HEAD"), []byte("ref: refs/heads/main\n"), 0o640); err != nil {
59 t.Fatal(err)
60 }
61
62 full := filepath.Join(t.TempDir(), "full.tar.gz")
63 if err := runBackup(cfg, full, false); err != nil {
64 t.Fatalf("full backup: %v", err)
65 }
66 dbOnly := filepath.Join(t.TempDir(), "db.tar.gz")
67 if err := runBackup(cfg, dbOnly, true); err != nil {
68 t.Fatalf("db-only backup: %v", err)
69 }
70
71 fullNames := members(t, full)
72 if len(fullNames) < 2 {
73 t.Fatalf("full backup carries only %v", fullNames)
74 }
75 var sawRepo bool
76 for _, n := range fullNames {
77 if n == "repos/krz/thing.git/HEAD" {
78 sawRepo = true
79 }
80 }
81 if !sawRepo {
82 t.Errorf("full backup is missing the repository: %v", fullNames)
83 }
84
85 if got := members(t, dbOnly); len(got) != 1 || got[0] != "gitbay.db" {
86 t.Errorf("db-only backup carries %v, want [gitbay.db]", got)
87 }
88
89 fi, err := os.Stat(dbOnly)
90 if err != nil {
91 t.Fatal(err)
92 }
93 if fi.Size() == 0 {
94 t.Error("db-only backup is empty")
95 }
96}
deploy/cloud-init.yaml +63 −12
@@ -57,16 +57,14 @@ write_files:
57 maxretry = 5 57 maxretry = 5
58 bantime = 1h 58 bantime = 1h
59 59
60 # Heartbeat: post disk/service/cert status to a webhook if one is set in 60 # Heartbeat: disk/service/cert status to journald every run, and to a
61 # /etc/gitbay/monitor.url. Silent when the file is absent. 61 # webhook as well if one is set in /etc/gitbay/monitor.url. Exits non-zero
62 # on an alert so the unit shows up in systemctl --failed.
62 - path: /usr/local/bin/gitbay-monitor.sh 63 - path: /usr/local/bin/gitbay-monitor.sh
63 permissions: "0755" 64 permissions: "0755"
64 content: | 65 content: |
65 #!/bin/sh 66 #!/bin/sh
66 set -eu 67 set -eu
67 url_file=/etc/gitbay/monitor.url
68 [ -f "$url_file" ] || exit 0
69 url=$(cat "$url_file")
70 disk=$(df -P /var/lib/gitbay | awk 'NR==2{print $5}') 68 disk=$(df -P /var/lib/gitbay | awk 'NR==2{print $5}')
71 svc=$(systemctl is-active gitbayd || true) 69 svc=$(systemctl is-active gitbayd || true)
72 # Days until the ACME cert expires, if autocert cached one. 70 # Days until the ACME cert expires, if autocert cached one.
@@ -74,14 +72,32 @@ write_files:
74 exp="n/a" 72 exp="n/a"
75 if [ -d "$cert" ]; then 73 if [ -d "$cert" ]; then
76 f=$(ls -1 "$cert" 2>/dev/null | grep -v acme_account | head -1 || true) 74 f=$(ls -1 "$cert" 2>/dev/null | grep -v acme_account | head -1 || true)
77 [ -n "$f" ] && exp=$(openssl x509 -enddate -noout -in "$cert/$f" 2>/dev/null | cut -d= -f2 || echo n/a) 75 if [ -n "$f" ]; then
76 exp=$(openssl x509 -enddate -noout -in "$cert/$f" 2>/dev/null | cut -d= -f2 || echo n/a)
77 fi
78 fi 78 fi
79 alert="" 79 alert=""
80 [ "$svc" != "active" ] && alert="gitbayd is $svc; " 80 if [ "$svc" != "active" ]; then
81 alert="gitbayd is $svc; "
82 fi
81 pct=$(echo "$disk" | tr -d '%') 83 pct=$(echo "$disk" | tr -d '%')
82 [ "$pct" -ge 85 ] && alert="${alert}disk ${disk}; " 84 if [ "$pct" -ge 85 ]; then
83 body=$(printf '{"disk":"%s","service":"%s","cert_expires":"%s","alert":"%s"}' "$disk" "$svc" "$exp" "$alert") 85 alert="${alert}disk ${disk}; "
84 curl -fsS -m 10 -H 'Content-Type: application/json' -d "$body" "$url" >/dev/null 2>&1 || true 86 fi
87 # journald always gets the reading, so an unset webhook cannot make a
88 # sick host look like a quiet one.
89 echo "disk=$disk service=$svc cert_expires=$exp"
90 url_file=/etc/gitbay/monitor.url
91 if [ -f "$url_file" ]; then
92 body=$(printf '{"disk":"%s","service":"%s","cert_expires":"%s","alert":"%s"}' "$disk" "$svc" "$exp" "$alert")
93 if ! curl -fsS -m 10 -H 'Content-Type: application/json' -d "$body" "$(cat "$url_file")" >/dev/null; then
94 echo "monitor webhook post failed" >&2
95 fi
96 fi
97 if [ -n "$alert" ]; then
98 echo "$alert" >&2
99 exit 1
100 fi
85 101
86 - path: /etc/systemd/system/gitbay-monitor.service 102 - path: /etc/systemd/system/gitbay-monitor.service
87 content: | 103 content: |
@@ -186,6 +202,41 @@ write_files:
186 /usr/local/bin/gitbayd --config /etc/gitbay/config.toml admin backup --out "$out" 202 /usr/local/bin/gitbayd --config /etc/gitbay/config.toml admin backup --out "$out"
187 ls -1t "$dir"/gitbay-*.tar.gz | tail -n +8 | xargs -r rm -- 203 ls -1t "$dir"/gitbay-*.tar.gz | tail -n +8 | xargs -r rm --
188 204
205 # Hourly database-only snapshot. The nightly full backup below is the one
206 # that can rebuild the host; this one exists because the database holds
207 # issues, merge requests and comments, which unlike the repositories have
208 # no second copy anywhere. 48 of them is two days at a few MB each.
209 - path: /usr/local/bin/gitbay-db-backup.sh
210 permissions: "0755"
211 content: |
212 #!/bin/sh
213 set -eu
214 dir=/var/backups/gitbay/db
215 mkdir -p "$dir"
216 out="$dir/gitbay-db-$(date -u +%Y%m%d-%H%M%S).tar.gz"
217 /usr/local/bin/gitbayd --config /etc/gitbay/config.toml admin backup --db-only --out "$out"
218 ls -1t "$dir"/gitbay-db-*.tar.gz | tail -n +49 | xargs -r rm --
219
220 - path: /etc/systemd/system/gitbay-db-backup.service
221 content: |
222 [Unit]
223 Description=gitbay hourly database backup
224 [Service]
225 Type=oneshot
226 User=gitbay
227 ExecStart=/usr/local/bin/gitbay-db-backup.sh
228
229 - path: /etc/systemd/system/gitbay-db-backup.timer
230 content: |
231 [Unit]
232 Description=gitbay hourly database backup
233 [Timer]
234 OnCalendar=*-*-* *:20:00 UTC
235 RandomizedDelaySec=5m
236 Persistent=true
237 [Install]
238 WantedBy=timers.target
239
189 - path: /etc/systemd/system/gitbay-backup.service 240 - path: /etc/systemd/system/gitbay-backup.service
190 content: | 241 content: |
191 [Unit] 242 [Unit]
@@ -237,6 +288,6 @@ runcmd:
237 - ufw --force enable 288 - ufw --force enable
238 - systemctl daemon-reload 289 - systemctl daemon-reload
239 - systemctl restart ssh.socket || systemctl restart ssh 290 - systemctl restart ssh.socket || systemctl restart ssh
240 - systemctl enable gitbayd gitbay-backup.timer gitbay-gc.timer gitbay-monitor.timer 291 - systemctl enable gitbayd gitbay-backup.timer gitbay-db-backup.timer gitbay-gc.timer gitbay-monitor.timer
241 - systemctl start gitbay-backup.timer gitbay-gc.timer gitbay-monitor.timer 292 - systemctl start gitbay-backup.timer gitbay-db-backup.timer gitbay-gc.timer gitbay-monitor.timer
242 - systemctl enable --now unattended-upgrades fail2ban 293 - systemctl enable --now unattended-upgrades fail2ban
internal/store/store.go +12
@@ -4,8 +4,10 @@ package store
4import ( 4import (
5 "database/sql" 5 "database/sql"
6 "embed" 6 "embed"
7 "errors"
7 "fmt" 8 "fmt"
8 "io/fs" 9 "io/fs"
10 "os"
9 "sort" 11 "sort"
10 "strconv" 12 "strconv"
11 "strings" 13 "strings"
@@ -35,6 +37,16 @@ func Open(path string) (*Store, error) {
35 db.Close() 37 db.Close()
36 return nil, err 38 return nil, err
37 } 39 }
40 // SQLite creates the file 0666&~umask, so it lands 0644 by default. The
41 // directory above it is the real boundary, but the file holds token
42 // hashes, addresses and private repo names and has no business being
43 // world-readable on its own.
44 if path != ":memory:" {
45 if err := os.Chmod(path, 0o640); err != nil && !errors.Is(err, fs.ErrNotExist) {
46 db.Close()
47 return nil, err
48 }
49 }
38 return &Store{DB: db}, nil 50 return &Store{DB: db}, nil
39} 51}
40 52
internal/store/store_test.go +19
@@ -1,6 +1,7 @@
1package store 1package store
2 2
3import ( 3import (
4 "os"
4 "path/filepath" 5 "path/filepath"
5 "strings" 6 "strings"
6 "testing" 7 "testing"
@@ -96,3 +97,21 @@ func TestForeignKeysEnforced(t *testing.T) {
96 t.Fatal("insert with dangling user_id succeeded; foreign keys are off") 97 t.Fatal("insert with dangling user_id succeeded; foreign keys are off")
97 } 98 }
98} 99}
100
101// The database file carries token hashes, addresses and private repo names.
102// The directory above it is the real boundary; this is the second one.
103func TestDatabaseFileIsNotWorldReadable(t *testing.T) {
104 path := filepath.Join(t.TempDir(), "gitbay.db")
105 s, err := Open(path)
106 if err != nil {
107 t.Fatal(err)
108 }
109 defer s.Close()
110 fi, err := os.Stat(path)
111 if err != nil {
112 t.Fatal(err)
113 }
114 if mode := fi.Mode().Perm(); mode&0o007 != 0 {
115 t.Errorf("database mode %04o is other-readable", mode)
116 }
117}