krz/gitbay

A CLI-first git forge. cli forge git self-hosted https://gitbay.org

Commit d201d6acb1

d201d6acb1a350c3ef7f5a2a66df1aa0f7ac6ea8

parent: 55cbdaf81e

Verified · cmc

cmc <hello@cleberg.net> · 2026-09-28 22:25 UTC 

backup: archive each repository's refs before its objects

Ref #259

Layout: unified · split

.gitbay/wiki/Architecture/08-Operations.org +4 −2
@@ -52,8 +52,10 @@ the product activity feed, not an audit trail.
52| Database only | hourly | 48 | SQLite snapshot; age-encrypted when =[backup] age_recipients= is set | 52| Database only | hourly | 48 | SQLite snapshot; age-encrypted when =[backup] age_recipients= is set |
53| Offsite (restic)| nightly | per prune policy | =/var/lib/gitbay= and a staged database copy, to object storage | 53| Offsite (restic)| nightly | per prune policy | =/var/lib/gitbay= and a staged database copy, to object storage |
54 54
55- The database snapshot is taken before repositories are read, so a 55- The database snapshot is taken before repositories are read, and
56 push during the backup leaves only unreferenced objects 56 each repository's HEAD, packed-refs and refs/ are archived before its
57 objects, so every archived ref finds the objects it reaches. A push
58 during the backup is missing or present as unreferenced objects
57 (=cmd/gitbayd/backup.go=). 59 (=cmd/gitbayd/backup.go=).
58- Excluded: WAL files, the hook socket, askpass scripts, generated 60- Excluded: WAL files, the hook socket, askpass scripts, generated
59 hooks. 61 hooks.
.gitbay/wiki/Threat-Model.org +6 −3
@@ -262,9 +262,12 @@ assume has been checked.
262 pixel against a viewer. A profile is the wider surface of the two: it 262 pixel against a viewer. A profile is the wider surface of the two: it
263 is linked from every commit and issue its owner touches. Documented; 263 is linked from every commit and issue its owner touches. Documented;
264 proxying is future work. 264 proxying is future work.
265- Backups are consistent per the DB-snapshot-first ordering, and 265- Backups snapshot the database first, and repository deletes and
266 repository deletes and moves wait out a full backup; a push during 266 moves wait out a full backup. Each repository's refs are archived
267 one leaves only unreferenced objects (see [[Admin]]). 267 before its objects, so every archived ref finds the objects it
268 reaches. A push during a backup may be missing from the archive, or
269 present as objects no archived ref names, and a repository's refs may
270 be newer than the database snapshot (see [[Admin]]).
268- The audit log lives in the database the daemon writes, so anyone with 271- The audit log lives in the database the daemon writes, so anyone with
269 the daemon user's access can change it. The hash chain makes an edited 272 the daemon user's access can change it. The hash chain makes an edited
270 or removed row show as a break under =gitbayd admin audit verify=, 273 or removed row show as a break under =gitbayd admin audit verify=,
cmd/gitbayd/backup.go +73 −8
@@ -4,6 +4,7 @@ import (
4 "archive/tar" 4 "archive/tar"
5 "bufio" 5 "bufio"
6 "compress/gzip" 6 "compress/gzip"
7 "errors"
7 "fmt" 8 "fmt"
8 "io" 9 "io"
9 "io/fs" 10 "io/fs"
@@ -25,10 +26,11 @@ import (
25// every repository and the SSH host keys. Restore by extracting the archive 26// every repository and the SSH host keys. Restore by extracting the archive
26// into a fresh server.root. 27// into a fresh server.root.
27// 28//
28// Ordering: the database is snapshotted BEFORE the repositories are read. 29// Ordering: the database is snapshotted BEFORE the repositories are read,
29// A push that lands mid-backup then shows up only as unreferenced git 30// and each repository's refs before its objects. A push that lands
30// objects in the archive (harmless); the reverse order could leave database 31// mid-backup then shows up only as unreferenced git objects in the archive
31// rows pointing at objects the archive never captured. 32// (harmless) or not at all; the reverse order could leave database rows or
33// refs pointing at objects the archive never captured.
32func backupCmd() *cobra.Command { 34func backupCmd() *cobra.Command {
33 var out, verify, identity string 35 var out, verify, identity string
34 var dbOnly bool 36 var dbOnly bool
@@ -180,15 +182,29 @@ func runBackup(cfg config.Config, out string, dbOnly bool) error {
180 if !d.Type().IsRegular() && !d.IsDir() { 182 if !d.Type().IsRegular() && !d.IsDir() {
181 return nil // sockets, symlinks 183 return nil // sockets, symlinks
182 } 184 }
183 if d.IsDir() { 185 // A repository's refs were archived on entering it.
184 if strings.HasSuffix(rel, ".git") { 186 if strings.HasSuffix(filepath.Dir(rel), ".git") && refNames[d.Name()] {
185 repoCount++ 187 if d.IsDir() {
188 return filepath.SkipDir
186 } 189 }
190 return nil
191 }
192 if d.IsDir() {
187 // A directory entry, even for one that holds no file (a 193 // A directory entry, even for one that holds no file (a
188 // bare repository's refs/heads and refs/tags once every 194 // bare repository's refs/heads and refs/tags once every
189 // ref is packed), so extraction recreates it: git's own 195 // ref is packed), so extraction recreates it: git's own
190 // repository discovery needs refs/ to exist. 196 // repository discovery needs refs/ to exist.
191 return addDir(tw, path, filepath.ToSlash(rel)) 197 if err := addDir(tw, path, filepath.ToSlash(rel)); err != nil {
198 return err
199 }
200 if strings.HasSuffix(rel, ".git") {
201 repoCount++
202 if err := addRefs(tw, path, filepath.ToSlash(rel)); err != nil {
203 return err
204 }
205 afterRefs(path)
206 }
207 return nil
192 } 208 }
193 return addFile(tw, path, filepath.ToSlash(rel)) 209 return addFile(tw, path, filepath.ToSlash(rel))
194 }) 210 })
@@ -230,6 +246,55 @@ func runBackup(cfg config.Config, out string, dbOnly bool) error {
230 return nil 246 return nil
231} 247}
232 248
249// refNames are what a repository's refs are read from. WalkDir would
250// reach objects/ before packed-refs and refs/, so a push landing mid-walk
251// could leave an archived ref naming objects the archive lacks. addRefs
252// archives these first on entering the repository; objects are only ever
253// added, so the walk that follows finds every object those refs reach.
254var refNames = map[string]bool{"HEAD": true, "packed-refs": true, "refs": true}
255
256// afterRefs runs between a repository's refs and the rest of it. Tests
257// use it to write into the repository at that point.
258var afterRefs = func(repo string) {}
259
260// addRefs archives HEAD, packed-refs and refs/ of the repository at
261// path, whichever exist.
262func addRefs(tw *tar.Writer, path, name string) error {
263 for _, f := range []string{"HEAD", "packed-refs"} {
264 fi, err := os.Lstat(filepath.Join(path, f))
265 if errors.Is(err, fs.ErrNotExist) || err == nil && !fi.Mode().IsRegular() {
266 continue
267 }
268 if err != nil {
269 return err
270 }
271 if err := addFile(tw, filepath.Join(path, f), name+"/"+f); err != nil {
272 return err
273 }
274 }
275 refs := filepath.Join(path, "refs")
276 if _, err := os.Lstat(refs); errors.Is(err, fs.ErrNotExist) {
277 return nil
278 }
279 return filepath.WalkDir(refs, func(p string, d fs.DirEntry, err error) error {
280 if err != nil {
281 return err
282 }
283 rel, err := filepath.Rel(path, p)
284 if err != nil {
285 return err
286 }
287 member := name + "/" + filepath.ToSlash(rel)
288 switch {
289 case d.IsDir():
290 return addDir(tw, p, member)
291 case d.Type().IsRegular():
292 return addFile(tw, p, member)
293 }
294 return nil
295 })
296}
297
233// syncDir makes a rename in dir durable. 298// syncDir makes a rename in dir durable.
234func syncDir(dir string) error { 299func syncDir(dir string) error {
235 d, err := os.Open(dir) 300 d, err := os.Open(dir)
cmd/gitbayd/backup_test.go +65
@@ -483,3 +483,68 @@ func TestBackupPreservesPackedRefDirs(t *testing.T) {
483 } 483 }
484 gitIn(t, restoredRepo, "fsck", "--connectivity-only", "--no-progress", "--no-dangling") 484 gitIn(t, restoredRepo, "fsck", "--connectivity-only", "--no-progress", "--no-dangling")
485} 485}
486
487// A commit pushed after a repository's refs are archived and before its
488// objects are leaves the archive with the earlier refs and every object
489// they reach, plus the new ones unreferenced (#259).
490func TestBackupArchivesRefsBeforeObjects(t *testing.T) {
491 cfg := testConfig(t)
492 st, err := openStore(cfg)
493 if err != nil {
494 t.Fatal(err)
495 }
496 uid, err := st.CreateUser("krz", false)
497 if err != nil {
498 t.Fatal(err)
499 }
500 if _, err := st.CreateRepo("user", uid, "thing", "public"); err != nil {
501 t.Fatal(err)
502 }
503 st.Close()
504
505 work := t.TempDir()
506 gitIn(t, work, "init", "-q", "-b", "main")
507 if err := os.WriteFile(filepath.Join(work, "a.txt"), []byte("a\n"), 0o644); err != nil {
508 t.Fatal(err)
509 }
510 gitIn(t, work, "add", "a.txt")
511 gitIn(t, work, "commit", "-q", "-m", "one")
512 dir := filepath.Join(cfg.Server.Root, "repos", "krz", "thing.git")
513 gitIn(t, work, "clone", "-q", "--bare", work, dir)
514 first := gitIn(t, dir, "rev-parse", "refs/heads/main")
515
516 var second string
517 afterRefs = func(repo string) {
518 if repo != dir {
519 return
520 }
521 if err := os.WriteFile(filepath.Join(work, "b.txt"), []byte("b\n"), 0o644); err != nil {
522 t.Fatal(err)
523 }
524 gitIn(t, work, "add", "b.txt")
525 gitIn(t, work, "commit", "-q", "-m", "two")
526 gitIn(t, work, "push", "-q", dir, "main")
527 second = gitIn(t, dir, "rev-parse", "refs/heads/main")
528 }
529 t.Cleanup(func() { afterRefs = func(string) {} })
530
531 archive := filepath.Join(t.TempDir(), "b.tar.gz")
532 if err := runBackup(cfg, archive, false); err != nil {
533 t.Fatal(err)
534 }
535 if second == "" || second == first {
536 t.Fatal("the push between the refs and the objects did not happen")
537 }
538 if err := verifyBackup(archive, ""); err != nil {
539 t.Fatalf("verify: %v", err)
540 }
541 restored := t.TempDir()
542 if out, err := exec.Command("tar", "-xzf", archive, "-C", restored).CombinedOutput(); err != nil {
543 t.Fatalf("extract: %v\n%s", err, out)
544 }
545 repo := filepath.Join(restored, "repos", "krz", "thing.git")
546 if got := gitIn(t, repo, "rev-parse", "refs/heads/main"); got != first {
547 t.Errorf("archived main is %s, want %s from before the push", got, first)
548 }
549 gitIn(t, repo, "cat-file", "-e", second)
550}