krz/gitbay

A CLI-first git forge. cli forge git self-hosted https://gitbay.org

Commit a27671383a

a27671383a2c67dc48e642cdf8306e469ba72d09

parent: c09a124633

Verified · cmc

cmc <hello@cleberg.net> · 2026-09-29 03:29 UTC 

gitbayd: admin restore-drill restores an archive, verifies it, reports elapsed time and the newest activity

Ref #259

Layout: unified · split

cmd/gitbayd/backup.go +28 −12
@@ -467,6 +467,19 @@ func addDir(tw *tar.Writer, path, name string) error {
467// checked for integrity alone and says so. Repositories are extracted to 467// checked for integrity alone and says so. Repositories are extracted to
468// a temporary directory for the check, so it needs free space for them. 468// a temporary directory for the check, so it needs free space for them.
469func verifyBackup(path, identity string) error { 469func verifyBackup(path, identity string) error {
470 tmp, err := os.MkdirTemp("", "gitbay-verify-")
471 if err != nil {
472 return err
473 }
474 defer os.RemoveAll(tmp)
475 return checkArchive(path, identity, tmp, false)
476}
477
478// checkArchive extracts the archive at path into dest and runs verify's
479// checks on it. With full unset it extracts only the database and the
480// repositories; with full set, every member, so dest is a restored
481// server.root. Alternates and commondir are left out either way.
482func checkArchive(path, identity, dest string, full bool) error {
470 f, err := os.Open(path) 483 f, err := os.Open(path)
471 if err != nil { 484 if err != nil {
472 return err 485 return err
@@ -481,11 +494,6 @@ func verifyBackup(path, identity string) error {
481 return fmt.Errorf("%s: not a gzip archive: %w", path, err) 494 return fmt.Errorf("%s: not a gzip archive: %w", path, err)
482 } 495 }
483 tr := tar.NewReader(gz) 496 tr := tar.NewReader(gz)
484 tmp, err := os.MkdirTemp("", "gitbay-verify-")
485 if err != nil {
486 return err
487 }
488 defer os.RemoveAll(tmp)
489 dbPath := "" 497 dbPath := ""
490 inArchive := map[string]bool{} 498 inArchive := map[string]bool{}
491 members := 0 499 members := 0
@@ -502,7 +510,7 @@ func verifyBackup(path, identity string) error {
502 members++ 510 members++
503 switch { 511 switch {
504 case h.Name == "gitbay.db": 512 case h.Name == "gitbay.db":
505 dbPath = filepath.Join(tmp, "gitbay.db") 513 dbPath = filepath.Join(dest, "gitbay.db")
506 if err := extractTo(tr, dbPath); err != nil { 514 if err := extractTo(tr, dbPath); err != nil {
507 return fmt.Errorf("%s: extracting the database: %w", path, err) 515 return fmt.Errorf("%s: extracting the database: %w", path, err)
508 } 516 }
@@ -510,18 +518,26 @@ func verifyBackup(path, identity string) error {
510 // Objects are named by their sha256, so each is checked as it 518 // Objects are named by their sha256, so each is checked as it
511 // streams past and none is extracted. 519 // streams past and none is extracted.
512 lfsObjects++ 520 lfsObjects++
521 if !filepath.IsLocal(h.Name) {
522 return fmt.Errorf("%s: member %q leaves the archive root", path, h.Name)
523 }
513 sum := sha256.New() 524 sum := sha256.New()
514 if _, err := io.Copy(sum, tr); err != nil { 525 if full {
526 err = extractTo(io.TeeReader(tr, sum), filepath.Join(dest, filepath.FromSlash(h.Name)))
527 } else {
528 _, err = io.Copy(sum, tr)
529 }
530 if err != nil {
515 return fmt.Errorf("%s: reading %s: %w", path, h.Name, err) 531 return fmt.Errorf("%s: reading %s: %w", path, h.Name, err)
516 } 532 }
517 if hex.EncodeToString(sum.Sum(nil)) != filepath.Base(h.Name) { 533 if hex.EncodeToString(sum.Sum(nil)) != filepath.Base(h.Name) {
518 badLFS = append(badLFS, h.Name) 534 badLFS = append(badLFS, h.Name)
519 } 535 }
520 case strings.HasPrefix(h.Name, "repos/"): 536 case full || strings.HasPrefix(h.Name, "repos/"):
521 trimmed := strings.TrimSuffix(h.Name, "/") 537 trimmed := strings.TrimSuffix(h.Name, "/")
522 // repos/<owner>/<name>.git/HEAD marks one repository present. 538 // repos/<owner>/<name>.git/HEAD marks one repository present.
523 parts := strings.Split(trimmed, "/") 539 parts := strings.Split(trimmed, "/")
524 if len(parts) == 4 && parts[3] == "HEAD" && strings.HasSuffix(parts[2], ".git") { 540 if len(parts) == 4 && parts[0] == "repos" && parts[3] == "HEAD" && strings.HasSuffix(parts[2], ".git") {
525 inArchive[parts[1]+"/"+strings.TrimSuffix(parts[2], ".git")] = true 541 inArchive[parts[1]+"/"+strings.TrimSuffix(parts[2], ".git")] = true
526 } 542 }
527 if !filepath.IsLocal(trimmed) { 543 if !filepath.IsLocal(trimmed) {
@@ -530,7 +546,7 @@ func verifyBackup(path, identity string) error {
530 if borrowsObjects(trimmed) { 546 if borrowsObjects(trimmed) {
531 continue 547 continue
532 } 548 }
533 dest := filepath.Join(tmp, filepath.FromSlash(trimmed)) 549 dest := filepath.Join(dest, filepath.FromSlash(trimmed))
534 switch h.Typeflag { 550 switch h.Typeflag {
535 case tar.TypeDir: 551 case tar.TypeDir:
536 // The archive's directory modes do not matter to fsck, and 552 // The archive's directory modes do not matter to fsck, and
@@ -592,7 +608,7 @@ func verifyBackup(path, identity string) error {
592 var failed []error 608 var failed []error
593 var broken []string 609 var broken []string
594 for _, r := range repos { 610 for _, r := range repos {
595 dir := filepath.Join(tmp, "repos", r.OwnerName, r.Name+".git") 611 dir := filepath.Join(dest, "repos", r.OwnerName, r.Name+".git")
596 if err := gitutil.FsckConnectivity(dir); err != nil { 612 if err := gitutil.FsckConnectivity(dir); err != nil {
597 fmt.Fprintf(os.Stderr, "%s: %v\n", r.Path(), err) 613 fmt.Fprintf(os.Stderr, "%s: %v\n", r.Path(), err)
598 broken = append(broken, r.Path()) 614 broken = append(broken, r.Path())
@@ -603,7 +619,7 @@ func verifyBackup(path, identity string) error {
603 } else { 619 } else {
604 fmt.Printf("connectivity ok on %d repositories\n", len(repos)) 620 fmt.Printf("connectivity ok on %d repositories\n", len(repos))
605 } 621 }
606 assets, badAssets, err := checkReleaseAssets(st, repos, tmp) 622 assets, badAssets, err := checkReleaseAssets(st, repos, dest)
607 if err != nil { 623 if err != nil {
608 return err 624 return err
609 } 625 }
cmd/gitbayd/main.go +1
@@ -455,6 +455,7 @@ func adminCmd() *cobra.Command {
455 hostCmd("runners [--json]", "runner accounts: last poll, scope, the build each holds", "admin", "runners"), 455 hostCmd("runners [--json]", "runner accounts: last poll, scope, the build each holds", "admin", "runners"),
456 auditCmd, 456 auditCmd,
457 backupCmd(), 457 backupCmd(),
458 restoreDrillCmd(),
458 secretsCmd(), 459 secretsCmd(),
459 gcCmd(), 460 gcCmd(),
460 adminMigrateCommitRefsCmd(), 461 adminMigrateCommitRefsCmd(),
cmd/gitbayd/restoredrill.go added +107
@@ -0,0 +1,107 @@
1package main
2
3import (
4 "errors"
5 "fmt"
6 "io/fs"
7 "os"
8 "path/filepath"
9 "strings"
10 "time"
11
12 "github.com/spf13/cobra"
13
14 "gitbay.org/gitbay/internal/store"
15)
16
17// restoreDrillCmd rehearses the archive half of a restore: extract a
18// full archive into an empty directory, run verify's checks on what was
19// extracted, and report the elapsed time and the newest activity the
20// restored database holds.
21func restoreDrillCmd() *cobra.Command {
22 var into, identity string
23 cmd := &cobra.Command{
24 Use: "restore-drill <archive> --into <dir>",
25 Short: "restore a full archive into an empty directory, verify it, report elapsed time and the newest recovered activity",
26 Long: `Extracts every member of a full backup archive into --into, which must
27be empty or absent, and runs the checks of backup --verify on the
28extracted copy: database integrity, every repository present and
29passing git fsck --connectivity-only, release assets and LFS object
30digests. It then prints the newest issue, comment and push in the
31restored database, which is the recovery point, and the elapsed time.
32
33The result is a server.root a gitbayd can be pointed at. The archive
34does not carry server.secret_key_file or config.toml; the Admin wiki's
35Restore drill section covers those and the offsite path.`,
36 Args: cobra.ExactArgs(1),
37 RunE: func(cmd *cobra.Command, args []string) error {
38 if into == "" {
39 return errors.New("--into <dir> is required")
40 }
41 return restoreDrill(args[0], identity, into)
42 },
43 }
44 cmd.Flags().StringVar(&into, "into", "", "empty or absent directory to restore into")
45 cmd.Flags().StringVar(&identity, "identity", "", "an age identity file that opens an encrypted archive")
46 return cmd
47}
48
49func restoreDrill(archive, identity, into string) error {
50 start := time.Now()
51 ents, err := os.ReadDir(into)
52 switch {
53 case errors.Is(err, fs.ErrNotExist):
54 if err := os.MkdirAll(into, 0o700); err != nil {
55 return err
56 }
57 case err != nil:
58 return err
59 case len(ents) > 0:
60 return fmt.Errorf("%s is not empty; restore into an empty or absent directory", into)
61 }
62 checkErr := checkArchive(archive, identity, into, true)
63 if ents, err := os.ReadDir(into); err == nil {
64 var names []string
65 for _, e := range ents {
66 names = append(names, e.Name())
67 }
68 fmt.Printf("restored into %s: %s\n", into, strings.Join(names, " "))
69 }
70 // store.Open would create a missing database.
71 db := filepath.Join(into, "gitbay.db")
72 if _, err := os.Stat(db); err == nil {
73 if err := printNewest(db); err != nil {
74 checkErr = errors.Join(checkErr, err)
75 }
76 }
77 fmt.Printf("elapsed %s\n", time.Since(start).Round(100*time.Millisecond))
78 return checkErr
79}
80
81// newest are the recovered timestamps a drill records.
82var newest = []struct{ label, query string }{
83 {"issue", "SELECT MAX(created_at) FROM issues"},
84 {"issue comment", "SELECT MAX(created_at) FROM issue_comments"},
85 {"merge request comment", "SELECT MAX(created_at) FROM mr_comments"},
86 {"push", "SELECT MAX(created_at) FROM events WHERE kind = 'push'"},
87}
88
89func printNewest(db string) error {
90 st, err := store.Open(db)
91 if err != nil {
92 return err
93 }
94 defer st.Close()
95 for _, n := range newest {
96 var at *string
97 if err := st.DB.QueryRow(n.query).Scan(&at); err != nil {
98 return fmt.Errorf("newest %s: %w", n.label, err)
99 }
100 v := "none"
101 if at != nil {
102 v = *at
103 }
104 fmt.Printf("newest %s: %s\n", n.label, v)
105 }
106 return nil
107}
cmd/gitbayd/restoredrill_test.go added +63
@@ -0,0 +1,63 @@
1package main
2
3import (
4 "os"
5 "path/filepath"
6 "strings"
7 "testing"
8)
9
10// A drill restores a full archive into an empty directory as a usable
11// root, verifies it, and refuses a directory that already holds files.
12func TestRestoreDrill(t *testing.T) {
13 cfg := testConfig(t)
14 root := cfg.Server.Root
15 st, err := openStore(cfg)
16 if err != nil {
17 t.Fatal(err)
18 }
19 uid, err := st.CreateUser("krz", false)
20 if err != nil {
21 t.Fatal(err)
22 }
23 rid, err := st.CreateRepo("user", uid, "thing", "public")
24 if err != nil {
25 t.Fatal(err)
26 }
27 if _, err := st.CreateIssue(rid, uid, "bug", "", "md"); err != nil {
28 t.Fatal(err)
29 }
30 if err := st.RecordEvent(rid, uid, "push", "{}"); err != nil {
31 t.Fatal(err)
32 }
33 st.Close()
34 work := t.TempDir()
35 gitIn(t, work, "init", "-q", "-b", "main")
36 writeFile(t, filepath.Join(work, "a.txt"), []byte("a\n"))
37 gitIn(t, work, "add", "a.txt")
38 gitIn(t, work, "commit", "-q", "-m", "one")
39 gitIn(t, work, "clone", "-q", "--bare", work, filepath.Join(root, "repos", "krz", "thing.git"))
40 writeFile(t, filepath.Join(root, "ssh", "host_ed25519"), []byte("key\n"))
41
42 archive := filepath.Join(t.TempDir(), "b.tar.gz")
43 if err := runBackup(cfg, archive, false); err != nil {
44 t.Fatal(err)
45 }
46 into := filepath.Join(t.TempDir(), "restored")
47 if err := restoreDrill(archive, "", into); err != nil {
48 t.Fatal(err)
49 }
50 for _, p := range []string{"gitbay.db", "ssh/host_ed25519", "repos/krz/thing.git/HEAD"} {
51 if _, err := os.Stat(filepath.Join(into, p)); err != nil {
52 t.Errorf("restored root lacks %s: %v", p, err)
53 }
54 }
55 if got := gitIn(t, filepath.Join(into, "repos", "krz", "thing.git"), "log", "--format=%s", "main"); got != "one" {
56 t.Errorf("restored log %q", got)
57 }
58
59 err = restoreDrill(archive, "", into)
60 if err == nil || !strings.Contains(err.Error(), "not empty") {
61 t.Fatalf("drill into a non-empty directory: %v", err)
62 }
63}