krz/gitbay

A CLI-first git forge. cli forge git self-hosted https://gitbay.org

Commit 4c11a67a9f

4c11a67a9fd655dac5708e3ea95d53919610e1bc

parent: 24c0aad9bf

Verified · cmc

cmc <hello@cleberg.net> · 2026-08-24 00:19 UTC 

admin backup: consistent archive with restore verified

- gitbayd admin backup [--out]: one tar.gz holding a consistent SQLite
  snapshot (VACUUM INTO, safe against a live daemon under WAL), every
  repository, and the SSH host keys; transient state excluded
  (hook.sock, regenerated hooks dir, askpass helper, WAL/SHM)
- snapshot ordering: database BEFORE repositories, so a push landing
  mid-backup yields orphaned git objects rather than database rows
  pointing at objects the archive missed (the plan stated the reverse
  order; its own rationale requires this one)
- e2e restores the archive into a fresh root and proves it: same host
  key passes strict checking, identity/repo/tag/issue all present,
  and the restored instance accepts new pushes (hooks self-regenerate)

Layout: unified · split

cmd/gitbayd/backup.go added +162
@@ -0,0 +1,162 @@
1package main
2
3import (
4 "archive/tar"
5 "compress/gzip"
6 "fmt"
7 "io"
8 "io/fs"
9 "os"
10 "path/filepath"
11 "strings"
12 "time"
13
14 "github.com/spf13/cobra"
15
16 "gitbay.org/gitbay/internal/config"
17 "gitbay.org/gitbay/internal/store"
18)
19
20// backupCmd produces one tar.gz holding a consistent database snapshot plus
21// every repository and the SSH host keys. Restore by extracting the archive
22// into a fresh server.root.
23//
24// Ordering: the database is snapshotted BEFORE the repositories are read.
25// A push that lands mid-backup then shows up only as unreferenced git
26// objects in the archive (harmless); the reverse order could leave database
27// rows pointing at objects the archive never captured.
28func backupCmd() *cobra.Command {
29 var out string
30 cmd := &cobra.Command{
31 Use: "backup",
32 Short: "write a consistent backup archive (database snapshot first, then repositories)",
33 Long: `Writes a tar.gz of the server root: a consistent SQLite snapshot,
34all repositories, and the SSH host keys. Transient state (hook socket,
35regenerated hook scripts, askpass helper, WAL files) is excluded.
36
37Restore: extract into an empty directory, point server.root at it, start
38gitbayd. Host keys are preserved, so clients keep their known_hosts entries.`,
39 RunE: func(cmd *cobra.Command, args []string) error {
40 cfg, err := config.Load(configPath)
41 if err != nil {
42 return err
43 }
44 if out == "" {
45 out = fmt.Sprintf("gitbay-backup-%s.tar.gz", time.Now().UTC().Format("20060102-150405"))
46 }
47 return runBackup(cfg, out)
48 },
49 }
50 cmd.Flags().StringVar(&out, "out", "", "output archive path (default gitbay-backup-<utc timestamp>.tar.gz)")
51 return cmd
52}
53
54func runBackup(cfg config.Config, out string) error {
55 st, err := openStore(cfg)
56 if err != nil {
57 return err
58 }
59 defer st.Close()
60
61 // 1. Consistent database snapshot, before any repository is read.
62 snap := filepath.Join(os.TempDir(), fmt.Sprintf("gitbay-snap-%d.db", os.Getpid()))
63 os.Remove(snap)
64 defer os.Remove(snap)
65 if err := snapshotDB(st, snap); err != nil {
66 return fmt.Errorf("database snapshot: %w", err)
67 }
68
69 f, err := os.Create(out)
70 if err != nil {
71 return err
72 }
73 defer f.Close()
74 gz := gzip.NewWriter(f)
75 tw := tar.NewWriter(gz)
76
77 if err := addFile(tw, snap, "gitbay.db"); err != nil {
78 return err
79 }
80
81 // 2. Everything under the root except transient or regenerated state.
82 skip := map[string]bool{
83 "gitbay.db": true, "gitbay.db-wal": true, "gitbay.db-shm": true,
84 "hook.sock": true, "askpass.sh": true, "hooks": true,
85 }
86 repoCount := 0
87 root := cfg.Server.Root
88 err = filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error {
89 if err != nil {
90 return err
91 }
92 rel, err := filepath.Rel(root, path)
93 if err != nil {
94 return err
95 }
96 if rel == "." {
97 return nil
98 }
99 if top, _, _ := strings.Cut(rel, string(filepath.Separator)); skip[top] {
100 if d.IsDir() {
101 return filepath.SkipDir
102 }
103 return nil
104 }
105 if !d.Type().IsRegular() && !d.IsDir() {
106 return nil // sockets, symlinks
107 }
108 if d.IsDir() {
109 if strings.HasSuffix(rel, ".git") {
110 repoCount++
111 }
112 return nil // directories are implied by member paths
113 }
114 return addFile(tw, path, filepath.ToSlash(rel))
115 })
116 if err != nil {
117 return err
118 }
119 if err := tw.Close(); err != nil {
120 return err
121 }
122 if err := gz.Close(); err != nil {
123 return err
124 }
125 if err := f.Close(); err != nil {
126 return err
127 }
128
129 info, _ := os.Stat(out)
130 fmt.Printf("wrote %s (%d repositories, %.1f MB)\n", out, repoCount, float64(info.Size())/1e6)
131 return nil
132}
133
134// snapshotDB writes a consistent copy of the live database. VACUUM INTO
135// takes a read snapshot, so concurrent daemon writes are safe under WAL.
136func snapshotDB(st *store.Store, dest string) error {
137 quoted := strings.ReplaceAll(dest, "'", "''")
138 _, err := st.DB.Exec(fmt.Sprintf("VACUUM INTO '%s'", quoted))
139 return err
140}
141
142func addFile(tw *tar.Writer, path, name string) error {
143 info, err := os.Stat(path)
144 if err != nil {
145 return err
146 }
147 hdr, err := tar.FileInfoHeader(info, "")
148 if err != nil {
149 return err
150 }
151 hdr.Name = name
152 if err := tw.WriteHeader(hdr); err != nil {
153 return err
154 }
155 src, err := os.Open(path)
156 if err != nil {
157 return err
158 }
159 defer src.Close()
160 _, err = io.Copy(tw, src)
161 return err
162}
cmd/gitbayd/main.go +1 −1
@@ -215,7 +215,7 @@ func adminCmd() *cobra.Command {
215 userCmd, 215 userCmd,
216 emailCmd, 216 emailCmd,
217 notImplemented("invite", "issue registration invites"), 217 notImplemented("invite", "issue registration invites"),
218 notImplemented("backup", "consistent backup: repos first, then database"), 218 backupCmd(),
219 notImplemented("gc", "run git gc across repositories"), 219 notImplemented("gc", "run git gc across repositories"),
220 notImplemented("stats", "instance statistics"), 220 notImplemented("stats", "instance statistics"),
221 ) 221 )
e2e/backup_test.go added +174
@@ -0,0 +1,174 @@
1package e2e
2
3import (
4 "fmt"
5 "net"
6 "os"
7 "os/exec"
8 "path/filepath"
9 "strings"
10 "testing"
11 "time"
12)
13
14func TestAdminBackup(t *testing.T) {
15 inst := startInstance(t)
16 aliceKey := inst.newKey(t, "alice")
17 inst.admin(t, "admin", "user", "create", "alice",
18 "--key", aliceKey+".pub", "--email", "alice@example.test", "--verified")
19
20 // Content worth backing up: a repo with commits and a tag, and an issue.
21 if _, errOut, code := inst.ssh(t, aliceKey, "", "repo", "create", "alice/keep"); code != 0 {
22 t.Fatalf("repo create: %s", errOut)
23 }
24 work := t.TempDir()
25 env := inst.gitEnv(aliceKey)
26 mustGit(t, work, env, "clone", inst.sshURL("alice/keep"), "w")
27 dir := filepath.Join(work, "w")
28 os.WriteFile(filepath.Join(dir, "data.txt"), []byte("precious\n"), 0o644)
29 mustGit(t, dir, env, "checkout", "-q", "-b", "main")
30 mustGit(t, dir, env, "add", ".")
31 mustGit(t, dir, env, "commit", "-q", "-m", "keep me")
32 mustGit(t, dir, env, "tag", "v1")
33 mustGit(t, dir, env, "push", "-q", "origin", "main", "v1")
34 if _, _, code := inst.ssh(t, aliceKey, "", "issue", "create", "alice/keep", "--title", "'survives backup'"); code != 0 {
35 t.Fatal("issue create failed")
36 }
37
38 // Back up while the daemon is running.
39 archive := filepath.Join(t.TempDir(), "backup.tar.gz")
40 out := inst.admin(t, "admin", "backup", "--out", archive)
41 if !strings.Contains(out, "1 repositories") {
42 t.Fatalf("backup summary: %s", out)
43 }
44
45 // The archive holds the snapshot, the repo, and the host key — and none
46 // of the transient state.
47 list, err := exec.Command("tar", "-tzf", archive).Output()
48 if err != nil {
49 t.Fatal(err)
50 }
51 names := string(list)
52 for _, want := range []string{"gitbay.db", "repos/alice/keep.git/", "ssh/host_ed25519"} {
53 if !strings.Contains(names, want) {
54 t.Fatalf("archive missing %s:\n%s", want, names)
55 }
56 }
57 for _, line := range strings.Split(strings.TrimSpace(names), "\n") {
58 // Top-level transient state must be absent; a repo's own inert
59 // sample hooks directory (keep.git/hooks/) is fine.
60 for _, banned := range []string{"hook.sock", "hooks/", "askpass.sh", "gitbay.db-wal"} {
61 if line == banned || strings.HasPrefix(line, banned) {
62 t.Fatalf("archive contains transient state %s:\n%s", line, names)
63 }
64 }
65 }
66
67 // Restore: extract into a fresh root and serve from it.
68 root2 := t.TempDir()
69 if outB, err := exec.Command("tar", "-xzf", archive, "-C", root2).CombinedOutput(); err != nil {
70 t.Fatalf("extract: %v\n%s", err, outB)
71 }
72 port2 := freePort(t)
73 httpPort2 := freePort(t)
74 config2 := filepath.Join(root2, "config.toml")
75 cfg := fmt.Sprintf(`
76[server]
77root = %q
78site_url = "https://gitbay.test"
79[ssh]
80port = %d
81[http]
82addr = "127.0.0.1:%d"
83tls = "off"
84`, root2, port2, httpPort2)
85 if err := os.WriteFile(config2, []byte(cfg), 0o600); err != nil {
86 t.Fatal(err)
87 }
88 proc2 := exec.Command(inst.gitbayd, "--config", config2, "serve")
89 proc2.Stderr = os.Stderr
90 if err := proc2.Start(); err != nil {
91 t.Fatal(err)
92 }
93 t.Cleanup(func() { proc2.Process.Kill(); proc2.Wait() })
94 deadline := time.Now().Add(10 * time.Second)
95 for {
96 conn, err := net.DialTimeout("tcp", fmt.Sprintf("127.0.0.1:%d", port2), 200*time.Millisecond)
97 if err == nil {
98 conn.Close()
99 break
100 }
101 if time.Now().After(deadline) {
102 t.Fatal("restored gitbayd did not start")
103 }
104 time.Sleep(50 * time.Millisecond)
105 }
106
107 // Strict host key checking against the ORIGINAL instance's host key:
108 // the preserved key means the restored server is cryptographically the
109 // same host. known_hosts entries are per host:port, so rebind the
110 // original entry to the new port.
111 khRaw, err := os.ReadFile(filepath.Join(inst.sshDir, "known_hosts"))
112 if err != nil {
113 t.Fatal(err)
114 }
115 fields := strings.Fields(strings.SplitN(string(khRaw), "\n", 2)[0])
116 if len(fields) < 3 {
117 t.Fatalf("unexpected known_hosts: %q", khRaw)
118 }
119 kh2 := filepath.Join(t.TempDir(), "known_hosts")
120 entry := fmt.Sprintf("[127.0.0.1]:%d %s %s\n", port2, fields[1], fields[2])
121 if err := os.WriteFile(kh2, []byte(entry), 0o600); err != nil {
122 t.Fatal(err)
123 }
124 ssh2 := func(args ...string) (string, string, int) {
125 base := []string{
126 "-p", fmt.Sprint(port2), "-i", aliceKey,
127 "-o", "IdentitiesOnly=yes",
128 "-o", "UserKnownHostsFile=" + kh2,
129 "-o", "StrictHostKeyChecking=yes",
130 "-o", "BatchMode=yes",
131 "git@127.0.0.1",
132 }
133 cmd := exec.Command("ssh", append(base, args...)...)
134 var o, e strings.Builder
135 cmd.Stdout, cmd.Stderr = &o, &e
136 err := cmd.Run()
137 code := 0
138 if ee, ok := err.(*exec.ExitError); ok {
139 code = ee.ExitCode()
140 } else if err != nil {
141 t.Fatalf("ssh: %v", err)
142 }
143 return o.String(), e.String(), code
144 }
145
146 // Identity, repo data, and issue all survived.
147 out2, errOut, code := ssh2("whoami")
148 if code != 0 || strings.TrimSpace(out2) != "alice" {
149 t.Fatalf("whoami on restored instance: exit %d, %q, %s", code, out2, errOut)
150 }
151 if out2, _, code = ssh2("repo", "log", "alice/keep"); code != 0 || !strings.Contains(out2, "keep me") {
152 t.Fatalf("restored log: %d\n%s", code, out2)
153 }
154 if out2, _, code = ssh2("issue", "show", "alice/keep", "1"); code != 0 || !strings.Contains(out2, "survives backup") {
155 t.Fatalf("restored issue: %d\n%s", code, out2)
156 }
157
158 // The restored instance accepts new pushes: hooks were regenerated at
159 // startup, not restored from the archive.
160 env2 := append(os.Environ(),
161 fmt.Sprintf("GIT_SSH_COMMAND=ssh -i %s -o IdentitiesOnly=yes -o StrictHostKeyChecking=yes -o UserKnownHostsFile=%s -o BatchMode=yes",
162 aliceKey, kh2),
163 "GIT_CONFIG_NOSYSTEM=1", "GIT_CONFIG_GLOBAL=/dev/null",
164 "GIT_AUTHOR_NAME=t", "GIT_AUTHOR_EMAIL=t@example.test",
165 "GIT_COMMITTER_NAME=t", "GIT_COMMITTER_EMAIL=t@example.test")
166 work2 := t.TempDir()
167 mustGit(t, work2, env2, "clone", fmt.Sprintf("ssh://git@127.0.0.1:%d/alice/keep.git", port2), "w")
168 dir2 := filepath.Join(work2, "w")
169 if data, _ := os.ReadFile(filepath.Join(dir2, "data.txt")); string(data) != "precious\n" {
170 t.Fatalf("restored content: %q", data)
171 }
172 mustGit(t, dir2, env2, "commit", "-q", "--allow-empty", "-m", "post-restore")
173 mustGit(t, dir2, env2, "push", "-q", "origin", "main")
174}