cmd/gitbayd/backup.go
651 lines · 20630 bytes
1package main
2
3import (
4 "archive/tar"
5 "bufio"
6 "compress/gzip"
7 "errors"
8 "fmt"
9 "io"
10 "io/fs"
11 "os"
12 "path"
13 "path/filepath"
14 "regexp"
15 "strings"
16 "time"
17
18 "filippo.io/age"
19 "github.com/spf13/cobra"
20
21 "gitbay.org/gitbay/internal/backuplock"
22 "gitbay.org/gitbay/internal/config"
23 "gitbay.org/gitbay/internal/gitutil"
24 "gitbay.org/gitbay/internal/store"
25)
26
27// backupCmd produces one tar.gz holding a consistent database snapshot plus
28// every repository and the SSH host keys. Restore by extracting the archive
29// into a fresh server.root.
30//
31// Ordering: the database is snapshotted BEFORE the repositories are read,
32// and each repository's refs before its objects. A push that lands
33// mid-backup then shows up only as unreferenced git objects in the archive
34// (harmless) or not at all; the reverse order could leave database rows or
35// refs pointing at objects the archive never captured.
36func backupCmd() *cobra.Command {
37 var out, verify, identity string
38 var dbOnly bool
39 cmd := &cobra.Command{
40 Use: "backup",
41 Short: "write a consistent backup archive (database snapshot first, then repositories)",
42 Long: `Writes a tar.gz of the server root: a consistent SQLite snapshot,
43all repositories, and the SSH host keys. Transient state (hook socket,
44regenerated hook scripts, askpass helper, WAL files) is excluded.
45
46--db-only writes the database snapshot alone. It is seconds and megabytes
47rather than minutes and gigabytes, which is what makes a frequent schedule
48affordable, and the database is the copy of issues, merge requests and
49comments that exists nowhere else. Repositories are not in such an archive,
50so it supplements a full backup and does not replace one.
51
52Restore: extract into an empty directory, point server.root at it,
53restore server.secret_key_file from its own backup (mode 0600, owned by
54the daemon user), start gitbayd. No archive carries the key file, and
55without it gitbayd refuses to start. Host keys are preserved, so clients
56keep their known_hosts entries.
57
58With [backup] age_recipients set, the archive is encrypted to those age
59public keys and its name ends in .age. --verify then needs --identity
60<file> holding a matching private key, which is kept off the host.`,
61 RunE: func(cmd *cobra.Command, args []string) error {
62 if verify != "" {
63 return verifyBackup(verify, identity)
64 }
65 cfg, err := config.Load(configPath)
66 if err != nil {
67 return err
68 }
69 return runBackup(cfg, archivePath(out, cfg, time.Now()), dbOnly)
70 },
71 }
72 cmd.Flags().StringVar(&out, "out", "", "output archive path (default gitbay-backup-<utc timestamp>.tar.gz; .age is appended when [backup] age_recipients is set)")
73 cmd.Flags().BoolVar(&dbOnly, "db-only", false, "archive the database snapshot alone, without repositories")
74 cmd.Flags().StringVar(&verify, "verify", "", "check an archive instead of writing one: database integrity, its repositories against the archive's, and git connectivity of each")
75 cmd.Flags().StringVar(&identity, "identity", "", "with --verify: an age identity file that opens an encrypted archive")
76 return cmd
77}
78
79// archivePath is where the archive goes: out, or a timestamped name,
80// ending in .age when the archive is encrypted.
81func archivePath(out string, cfg config.Config, now time.Time) string {
82 if out == "" {
83 out = fmt.Sprintf("gitbay-backup-%s.tar.gz", now.UTC().Format("20060102-150405"))
84 }
85 if len(cfg.Backup.AgeRecipients) > 0 && !strings.HasSuffix(out, ".age") {
86 out += ".age"
87 }
88 return out
89}
90
91func runBackup(cfg config.Config, out string, dbOnly bool) error {
92 var rs []age.Recipient
93 if len(cfg.Backup.AgeRecipients) > 0 {
94 var err error
95 if rs, err = cfg.Backup.Recipients(); err != nil {
96 return err
97 }
98 } else if strings.HasSuffix(out, ".age") {
99 return fmt.Errorf("%s ends in .age but [backup] age_recipients is not set, so the archive would not be encrypted", out)
100 }
101
102 dir := filepath.Dir(out)
103 // An archive under the root would be in the next full backup's walk.
104 if config.Within(cfg.Server.Root, dir) {
105 return fmt.Errorf("%s is inside server.root %s; write the archive elsewhere", out, cfg.Server.Root)
106 }
107 removeStale(dir, time.Now().Add(-staleAge))
108
109 // Deletes, renames and transfers wait until the walk finishes, so
110 // every repository the snapshot names is still on disk when the walk
111 // reaches it (#259). A database-only archive reads no repository.
112 if !dbOnly {
113 release, err := backuplock.Hold(cfg.Server.Root)
114 if err != nil {
115 return fmt.Errorf("backup lock: %w", err)
116 }
117 defer release()
118 }
119
120 // VACUUM INTO copies sealed values as they are, so the backup needs
121 // no key file, and it migrates nothing. store.Open would create a
122 // missing database, so its absence is checked first.
123 dbFile := filepath.Join(cfg.Server.Root, "gitbay.db")
124 if _, err := os.Stat(dbFile); err != nil {
125 return fmt.Errorf("database: %w", err)
126 }
127 st, err := store.Open(dbFile)
128 if err != nil {
129 return err
130 }
131 defer st.Close()
132
133 // 1. Consistent database snapshot, before any repository is read. It
134 // goes in a fresh 0700 directory beside the archive.
135 snapDir, err := os.MkdirTemp(dir, ".gitbay-snap-")
136 if err != nil {
137 return err
138 }
139 defer os.RemoveAll(snapDir)
140 snap := filepath.Join(snapDir, "gitbay.db")
141 if err := snapshotDB(st, snap); err != nil {
142 return fmt.Errorf("database snapshot: %w", err)
143 }
144
145 // The archive is written to a temporary name beside out and renamed
146 // once complete, so a failed run leaves no partial archive behind.
147 f, err := os.CreateTemp(dir, "."+filepath.Base(out)+".tmp-")
148 if err != nil {
149 return err
150 }
151 done := false
152 defer func() {
153 if !done {
154 f.Close()
155 os.Remove(f.Name())
156 }
157 }()
158 var sink io.Writer = f
159 var enc io.WriteCloser
160 if len(rs) > 0 {
161 if enc, err = age.Encrypt(f, rs...); err != nil {
162 return err
163 }
164 sink = enc
165 }
166 gz := gzip.NewWriter(sink)
167 tw := tar.NewWriter(gz)
168
169 if err := addFile(tw, snap, "gitbay.db"); err != nil {
170 return err
171 }
172
173 // 2. Everything under the root except transient or regenerated state.
174 // Skipped entirely for --db-only.
175 skip := map[string]bool{
176 "gitbay.db": true, "gitbay.db-wal": true, "gitbay.db-shm": true,
177 "hook.sock": true, "askpass.sh": true, "hooks": true,
178 backuplock.Name: true,
179 }
180 repoCount := 0
181 root := cfg.Server.Root
182 if !dbOnly {
183 err = filepath.WalkDir(root, func(path string, d fs.DirEntry, walkErr error) error {
184 rel, err := filepath.Rel(root, path)
185 if err != nil {
186 return err
187 }
188 if walkErr != nil {
189 if vanished(walkErr, rel) {
190 return nil
191 }
192 return walkErr
193 }
194 if rel == "." {
195 return nil
196 }
197 if top, _, _ := strings.Cut(rel, string(filepath.Separator)); skip[top] {
198 if d.IsDir() {
199 return filepath.SkipDir
200 }
201 return nil
202 }
203 if !d.Type().IsRegular() && !d.IsDir() {
204 return nil // sockets, symlinks
205 }
206 // A repository's refs were archived on entering it.
207 if strings.HasSuffix(filepath.Dir(rel), ".git") && refNames[d.Name()] {
208 if d.IsDir() {
209 return filepath.SkipDir
210 }
211 return nil
212 }
213 if d.IsDir() {
214 // A directory entry, even for one that holds no file (a
215 // bare repository's refs/heads and refs/tags once every
216 // ref is packed), so extraction recreates it: git's own
217 // repository discovery needs refs/ to exist.
218 if err := addDir(tw, path, filepath.ToSlash(rel)); err != nil {
219 if vanished(err, rel) {
220 return filepath.SkipDir
221 }
222 return err
223 }
224 if strings.HasSuffix(rel, ".git") {
225 repoCount++
226 if err := addRefs(tw, path, filepath.ToSlash(rel)); err != nil {
227 return err
228 }
229 afterRefs(path)
230 }
231 return nil
232 }
233 beforeAdd(path)
234 if err := addFile(tw, path, filepath.ToSlash(rel)); !vanished(err, rel) {
235 return err
236 }
237 return nil
238 })
239 if err != nil {
240 return err
241 }
242 }
243 if err := tw.Close(); err != nil {
244 return err
245 }
246 if err := gz.Close(); err != nil {
247 return err
248 }
249 if enc != nil {
250 if err := enc.Close(); err != nil {
251 return err
252 }
253 }
254 if err := f.Sync(); err != nil {
255 return err
256 }
257 if err := f.Close(); err != nil {
258 return err
259 }
260 if err := os.Rename(f.Name(), out); err != nil {
261 return err
262 }
263 done = true
264 if err := syncDir(dir); err != nil {
265 return err
266 }
267
268 info, _ := os.Stat(out)
269 if dbOnly {
270 fmt.Printf("wrote %s (database only, %.1f MB)\n", out, float64(info.Size())/1e6)
271 return nil
272 }
273 fmt.Printf("wrote %s (%d repositories, %.1f MB)\n", out, repoCount, float64(info.Size())/1e6)
274 return nil
275}
276
277// staleAge is how old a snapshot directory or temporary archive must be
278// before a later run removes it. A run that is still writing one is
279// younger than this.
280const staleAge = 24 * time.Hour
281
282// tmpArchive is a temporary archive's name: os.CreateTemp's pattern
283// "."+base+".tmp-" followed by the digits it appends.
284var tmpArchive = regexp.MustCompile(`^\..+\.tmp-[0-9]+$`)
285
286// removeStale removes what a killed run left in dir: snapshot
287// directories and temporary archives last modified before cutoff.
288func removeStale(dir string, cutoff time.Time) {
289 ents, err := os.ReadDir(dir)
290 if err != nil {
291 return
292 }
293 for _, e := range ents {
294 name := e.Name()
295 snap := e.IsDir() && strings.HasPrefix(name, ".gitbay-snap-")
296 tmp := e.Type().IsRegular() && tmpArchive.MatchString(name)
297 if !snap && !tmp {
298 continue
299 }
300 info, err := e.Info()
301 if err != nil || !info.ModTime().Before(cutoff) {
302 continue
303 }
304 p := filepath.Join(dir, name)
305 if err := os.RemoveAll(p); err != nil {
306 fmt.Fprintf(os.Stderr, "removing stale %s: %v\n", p, err)
307 continue
308 }
309 fmt.Fprintf(os.Stderr, "removed stale %s\n", p)
310 }
311}
312
313// refNames are what a repository's refs are read from. WalkDir would
314// reach objects/ before packed-refs and refs/, so a push landing mid-walk
315// could leave an archived ref naming objects the archive lacks. addRefs
316// archives these first on entering the repository; objects are only ever
317// added, so the walk that follows finds every object those refs reach.
318var refNames = map[string]bool{"HEAD": true, "packed-refs": true, "refs": true}
319
320// afterRefs runs between a repository's refs and the rest of it. Tests
321// use it to write into the repository at that point.
322var afterRefs = func(repo string) {}
323
324// addRefs archives HEAD, refs/ and packed-refs of the repository at
325// path, whichever exist. refs/ is read before packed-refs, the order git
326// reads them in: pack-refs writes packed-refs before deleting the loose
327// refs it packed, so a ref moving between the two is caught in one.
328func addRefs(tw *tar.Writer, path, name string) error {
329 if err := addRegular(tw, path, name, "HEAD"); err != nil {
330 return err
331 }
332 refs := filepath.Join(path, "refs")
333 if _, err := os.Lstat(refs); err == nil {
334 if err := addTree(tw, path, name, refs); err != nil {
335 return err
336 }
337 } else if !errors.Is(err, fs.ErrNotExist) {
338 return err
339 }
340 return addRegular(tw, path, name, "packed-refs")
341}
342
343// addRegular archives the regular file f in the repository at path, if
344// it exists.
345func addRegular(tw *tar.Writer, path, name, f string) error {
346 fi, err := os.Lstat(filepath.Join(path, f))
347 if errors.Is(err, fs.ErrNotExist) || err == nil && !fi.Mode().IsRegular() {
348 return nil
349 }
350 if err != nil {
351 return err
352 }
353 return addFile(tw, filepath.Join(path, f), name+"/"+f)
354}
355
356// addTree archives the directory refs inside the repository at path.
357func addTree(tw *tar.Writer, path, name, refs string) error {
358 return filepath.WalkDir(refs, func(p string, d fs.DirEntry, err error) error {
359 if err != nil {
360 return err
361 }
362 rel, err := filepath.Rel(path, p)
363 if err != nil {
364 return err
365 }
366 member := name + "/" + filepath.ToSlash(rel)
367 switch {
368 case d.IsDir():
369 return addDir(tw, p, member)
370 case d.Type().IsRegular():
371 return addFile(tw, p, member)
372 }
373 return nil
374 })
375}
376
377// beforeAdd runs before each file the walk archives outside refs. Tests
378// use it to remove a file between listing and reading.
379var beforeAdd = func(path string) {}
380
381// vanished reports a file or directory under a repository's objects/
382// that went between the walk listing it and reading it: a pack or loose
383// object a concurrent gc or receive.autogc removed. The walk skips it.
384// Refs archived earlier reach only objects that are still reachable, and
385// a repack writes those into a new pack before removing the old one; if
386// one is lost regardless, verify's fsck reports it.
387func vanished(err error, rel string) bool {
388 if !errors.Is(err, fs.ErrNotExist) {
389 return false
390 }
391 parts := strings.Split(filepath.ToSlash(rel), "/")
392 for i := 0; i+2 < len(parts); i++ {
393 if strings.HasSuffix(parts[i], ".git") && parts[i+1] == "objects" {
394 return true
395 }
396 }
397 return false
398}
399
400// syncDir makes a rename in dir durable.
401func syncDir(dir string) error {
402 d, err := os.Open(dir)
403 if err != nil {
404 return err
405 }
406 defer d.Close()
407 return d.Sync()
408}
409
410// snapshotDB writes a consistent copy of the live database. VACUUM INTO
411// takes a read snapshot, so concurrent daemon writes are safe under WAL.
412func snapshotDB(st *store.Store, dest string) error {
413 quoted := strings.ReplaceAll(dest, "'", "''")
414 _, err := st.DB.Exec(fmt.Sprintf("VACUUM INTO '%s'", quoted))
415 return err
416}
417
418// addFile opens before writing the header, so a file removed after the
419// walk listed it fails before the archive has a member for it.
420func addFile(tw *tar.Writer, path, name string) error {
421 src, err := os.Open(path)
422 if err != nil {
423 return err
424 }
425 defer src.Close()
426 info, err := src.Stat()
427 if err != nil {
428 return err
429 }
430 hdr, err := tar.FileInfoHeader(info, "")
431 if err != nil {
432 return err
433 }
434 hdr.Name = name
435 if err := tw.WriteHeader(hdr); err != nil {
436 return err
437 }
438 _, err = io.CopyN(tw, src, hdr.Size)
439 return err
440}
441
442// addDir writes a directory entry, so an empty directory survives
443// extraction. The mode never exceeds 0755, whatever the source directory
444// carries.
445func addDir(tw *tar.Writer, path, name string) error {
446 info, err := os.Stat(path)
447 if err != nil {
448 return err
449 }
450 hdr, err := tar.FileInfoHeader(info, "")
451 if err != nil {
452 return err
453 }
454 hdr.Name = name + "/"
455 hdr.Mode = hdr.Mode&^0o777 | hdr.Mode&0o755
456 return tw.WriteHeader(hdr)
457}
458
459// verifyBackup reads an archive back, decrypting it with identity when it
460// is encrypted: the database snapshot must pass SQLite's integrity check,
461// every repository it names must be in the archive, and each of those
462// must pass git fsck --connectivity-only. A database-only archive is
463// checked for integrity alone and says so. Repositories are extracted to
464// a temporary directory for the check, so it needs free space for them.
465func verifyBackup(path, identity string) error {
466 f, err := os.Open(path)
467 if err != nil {
468 return err
469 }
470 defer f.Close()
471 plain, err := archiveReader(f, path, identity)
472 if err != nil {
473 return err
474 }
475 gz, err := gzip.NewReader(plain)
476 if err != nil {
477 return fmt.Errorf("%s: not a gzip archive: %w", path, err)
478 }
479 tr := tar.NewReader(gz)
480 tmp, err := os.MkdirTemp("", "gitbay-verify-")
481 if err != nil {
482 return err
483 }
484 defer os.RemoveAll(tmp)
485 dbPath := ""
486 inArchive := map[string]bool{}
487 members := 0
488 for {
489 h, err := tr.Next()
490 if err == io.EOF {
491 break
492 }
493 if err != nil {
494 return fmt.Errorf("%s: archive damaged after %d members: %w", path, members, err)
495 }
496 members++
497 switch {
498 case h.Name == "gitbay.db":
499 dbPath = filepath.Join(tmp, "gitbay.db")
500 if err := extractTo(tr, dbPath); err != nil {
501 return fmt.Errorf("%s: extracting the database: %w", path, err)
502 }
503 case strings.HasPrefix(h.Name, "repos/"):
504 trimmed := strings.TrimSuffix(h.Name, "/")
505 // repos/<owner>/<name>.git/HEAD marks one repository present.
506 parts := strings.Split(trimmed, "/")
507 if len(parts) == 4 && parts[3] == "HEAD" && strings.HasSuffix(parts[2], ".git") {
508 inArchive[parts[1]+"/"+strings.TrimSuffix(parts[2], ".git")] = true
509 }
510 if !filepath.IsLocal(trimmed) {
511 return fmt.Errorf("%s: member %q leaves the archive root", path, h.Name)
512 }
513 if borrowsObjects(trimmed) {
514 continue
515 }
516 dest := filepath.Join(tmp, filepath.FromSlash(trimmed))
517 switch h.Typeflag {
518 case tar.TypeDir:
519 // The archive's directory modes do not matter to fsck, and
520 // a hostile one would stop RemoveAll cleaning up.
521 if err := os.MkdirAll(dest, 0o700); err != nil {
522 return fmt.Errorf("%s: creating %s: %w", path, h.Name, err)
523 }
524 case tar.TypeReg:
525 if err := extractTo(tr, dest); err != nil {
526 return fmt.Errorf("%s: extracting %s: %w", path, h.Name, err)
527 }
528 }
529 }
530 }
531 // Read to the end so gzip checks its trailer and age its final chunk.
532 if _, err := io.Copy(io.Discard, gz); err != nil {
533 return fmt.Errorf("%s: archive truncated or damaged: %w", path, err)
534 }
535 if err := gz.Close(); err != nil {
536 return fmt.Errorf("%s: archive truncated or damaged: %w", path, err)
537 }
538 if dbPath == "" {
539 return fmt.Errorf("%s: no gitbay.db in the archive", path)
540 }
541 st, err := store.Open(dbPath)
542 if err != nil {
543 return fmt.Errorf("%s: database does not open: %w", path, err)
544 }
545 defer st.Close()
546 var integrity string
547 if err := st.DB.QueryRow("PRAGMA integrity_check").Scan(&integrity); err != nil {
548 return fmt.Errorf("%s: integrity check: %w", path, err)
549 }
550 if integrity != "ok" {
551 return fmt.Errorf("%s: database integrity: %s", path, integrity)
552 }
553 repos, err := st.ListAllRepos()
554 if err != nil {
555 return err
556 }
557 if len(inArchive) == 0 {
558 fmt.Printf("%s: database only; integrity ok, %d repositories in the database, none in the archive\n", path, len(repos))
559 return nil
560 }
561 var missing []string
562 for _, r := range repos {
563 if !inArchive[r.Path()] {
564 missing = append(missing, r.Path())
565 }
566 }
567 extra := len(inArchive) - (len(repos) - len(missing))
568 fmt.Printf("%s: integrity ok, %d repositories in the database, %d in the archive\n", path, len(repos), len(inArchive))
569 if len(missing) > 0 {
570 return fmt.Errorf("%s: %d repositories the database names are not in the archive: %s", path, len(missing), strings.Join(missing, ", "))
571 }
572 if extra > 0 {
573 fmt.Printf("%d repositories in the archive that the database does not name (created after the snapshot)\n", extra)
574 }
575 var broken []string
576 for _, r := range repos {
577 dir := filepath.Join(tmp, "repos", r.OwnerName, r.Name+".git")
578 if err := gitutil.FsckConnectivity(dir); err != nil {
579 fmt.Fprintf(os.Stderr, "%s: %v\n", r.Path(), err)
580 broken = append(broken, r.Path())
581 }
582 }
583 if len(broken) > 0 {
584 return fmt.Errorf("%s: %d repositories fail the connectivity check: %s", path, len(broken), strings.Join(broken, ", "))
585 }
586 fmt.Printf("connectivity ok on %d repositories\n", len(repos))
587 return nil
588}
589
590// borrowsObjects reports an archive member that would point git at
591// objects or refs outside the extracted repository: alternates, or a
592// commondir directly in a *.git directory. gitbay writes none, and one in
593// a hostile archive would have fsck read another repository on the host,
594// so verify leaves them out. The comparison ignores case, as a
595// case-insensitive filesystem would.
596func borrowsObjects(name string) bool {
597 name = strings.ToLower(filepath.ToSlash(filepath.Clean(name)))
598 if dir, base := path.Split(name); base == "commondir" && strings.HasSuffix(strings.TrimSuffix(dir, "/"), ".git") {
599 return true
600 }
601 return strings.HasSuffix(name, "/objects/info/alternates") ||
602 strings.HasSuffix(name, "/objects/info/http-alternates")
603}
604
605// extractTo writes one archive member to dest, owner-only.
606func extractTo(r io.Reader, dest string) error {
607 if err := os.MkdirAll(filepath.Dir(dest), 0o700); err != nil {
608 return err
609 }
610 w, err := os.OpenFile(dest, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o600)
611 if err != nil {
612 return err
613 }
614 if _, err := io.Copy(w, r); err != nil {
615 w.Close()
616 return err
617 }
618 return w.Close()
619}
620
621const ageHeader = "age-encryption.org/v1\n"
622
623// archiveReader returns the archive's gzip stream, decrypting it first
624// when it is an age file.
625func archiveReader(f io.Reader, path, identity string) (io.Reader, error) {
626 br := bufio.NewReader(f)
627 head, _ := br.Peek(len(ageHeader))
628 if string(head) != ageHeader {
629 if identity != "" {
630 fmt.Fprintf(os.Stderr, "%s is not encrypted; --identity was not used\n", path)
631 }
632 return br, nil
633 }
634 if identity == "" {
635 return nil, fmt.Errorf("%s is encrypted; pass --identity <file> with the private key for one of its recipients", path)
636 }
637 idf, err := os.Open(identity)
638 if err != nil {
639 return nil, err
640 }
641 defer idf.Close()
642 ids, err := age.ParseIdentities(idf)
643 if err != nil {
644 return nil, fmt.Errorf("%s: %w", identity, err)
645 }
646 r, err := age.Decrypt(br, ids...)
647 if err != nil {
648 return nil, fmt.Errorf("%s: decrypting: %w", path, err)
649 }
650 return r, nil
651}