cmd/gitbayd/backup.go
389 lines · 11358 bytes
1package main
2
3import (
4 "archive/tar"
5 "bufio"
6 "compress/gzip"
7 "fmt"
8 "io"
9 "io/fs"
10 "os"
11 "path/filepath"
12 "strings"
13 "time"
14
15 "filippo.io/age"
16 "github.com/spf13/cobra"
17
18 "gitbay.org/gitbay/internal/config"
19 "gitbay.org/gitbay/internal/store"
20)
21
22// backupCmd produces one tar.gz holding a consistent database snapshot plus
23// every repository and the SSH host keys. Restore by extracting the archive
24// into a fresh server.root.
25//
26// Ordering: the database is snapshotted BEFORE the repositories are read.
27// A push that lands mid-backup then shows up only as unreferenced git
28// objects in the archive (harmless); the reverse order could leave database
29// rows pointing at objects the archive never captured.
30func backupCmd() *cobra.Command {
31 var out, verify, identity string
32 var dbOnly bool
33 cmd := &cobra.Command{
34 Use: "backup",
35 Short: "write a consistent backup archive (database snapshot first, then repositories)",
36 Long: `Writes a tar.gz of the server root: a consistent SQLite snapshot,
37all repositories, and the SSH host keys. Transient state (hook socket,
38regenerated hook scripts, askpass helper, WAL files) is excluded.
39
40--db-only writes the database snapshot alone. It is seconds and megabytes
41rather than minutes and gigabytes, which is what makes a frequent schedule
42affordable, and the database is the copy of issues, merge requests and
43comments that exists nowhere else. Repositories are not in such an archive,
44so it supplements a full backup and does not replace one.
45
46Restore: extract into an empty directory, point server.root at it, start
47gitbayd. Host keys are preserved, so clients keep their known_hosts entries.
48
49With [backup] age_recipients set, the archive is encrypted to those age
50public keys and its name ends in .age. --verify then needs --identity
51<file> holding a matching private key, which is kept off the host.`,
52 RunE: func(cmd *cobra.Command, args []string) error {
53 if verify != "" {
54 return verifyBackup(verify, identity)
55 }
56 cfg, err := config.Load(configPath)
57 if err != nil {
58 return err
59 }
60 return runBackup(cfg, archivePath(out, cfg, time.Now()), dbOnly)
61 },
62 }
63 cmd.Flags().StringVar(&out, "out", "", "output archive path (default gitbay-backup-<utc timestamp>.tar.gz; .age is appended when [backup] age_recipients is set)")
64 cmd.Flags().BoolVar(&dbOnly, "db-only", false, "archive the database snapshot alone, without repositories")
65 cmd.Flags().StringVar(&verify, "verify", "", "check an archive instead of writing one: database integrity, and its repositories against the archive's")
66 cmd.Flags().StringVar(&identity, "identity", "", "with --verify: an age identity file that opens an encrypted archive")
67 return cmd
68}
69
70// archivePath is where the archive goes: out, or a timestamped name,
71// ending in .age when the archive is encrypted.
72func archivePath(out string, cfg config.Config, now time.Time) string {
73 if out == "" {
74 out = fmt.Sprintf("gitbay-backup-%s.tar.gz", now.UTC().Format("20060102-150405"))
75 }
76 if len(cfg.Backup.AgeRecipients) > 0 && !strings.HasSuffix(out, ".age") {
77 out += ".age"
78 }
79 return out
80}
81
82func runBackup(cfg config.Config, out string, dbOnly bool) error {
83 var rs []age.Recipient
84 if len(cfg.Backup.AgeRecipients) > 0 {
85 var err error
86 if rs, err = cfg.Backup.Recipients(); err != nil {
87 return err
88 }
89 } else if strings.HasSuffix(out, ".age") {
90 return fmt.Errorf("%s ends in .age but [backup] age_recipients is not set, so the archive would not be encrypted", out)
91 }
92
93 st, err := openStore(cfg)
94 if err != nil {
95 return err
96 }
97 defer st.Close()
98
99 // 1. Consistent database snapshot, before any repository is read. It
100 // goes in a fresh 0700 directory beside the archive.
101 dir := filepath.Dir(out)
102 snapDir, err := os.MkdirTemp(dir, ".gitbay-snap-")
103 if err != nil {
104 return err
105 }
106 defer os.RemoveAll(snapDir)
107 snap := filepath.Join(snapDir, "gitbay.db")
108 if err := snapshotDB(st, snap); err != nil {
109 return fmt.Errorf("database snapshot: %w", err)
110 }
111
112 // The archive is written to a temporary name beside out and renamed
113 // once complete, so a failed run leaves no partial archive behind.
114 f, err := os.CreateTemp(dir, "."+filepath.Base(out)+".tmp-")
115 if err != nil {
116 return err
117 }
118 done := false
119 defer func() {
120 if !done {
121 f.Close()
122 os.Remove(f.Name())
123 }
124 }()
125 var sink io.Writer = f
126 var enc io.WriteCloser
127 if len(rs) > 0 {
128 if enc, err = age.Encrypt(f, rs...); err != nil {
129 return err
130 }
131 sink = enc
132 }
133 gz := gzip.NewWriter(sink)
134 tw := tar.NewWriter(gz)
135
136 if err := addFile(tw, snap, "gitbay.db"); err != nil {
137 return err
138 }
139
140 // 2. Everything under the root except transient or regenerated state.
141 // Skipped entirely for --db-only.
142 skip := map[string]bool{
143 "gitbay.db": true, "gitbay.db-wal": true, "gitbay.db-shm": true,
144 "hook.sock": true, "askpass.sh": true, "hooks": true,
145 }
146 repoCount := 0
147 root := cfg.Server.Root
148 if !dbOnly {
149 err = filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error {
150 if err != nil {
151 return err
152 }
153 rel, err := filepath.Rel(root, path)
154 if err != nil {
155 return err
156 }
157 if rel == "." {
158 return nil
159 }
160 if top, _, _ := strings.Cut(rel, string(filepath.Separator)); skip[top] {
161 if d.IsDir() {
162 return filepath.SkipDir
163 }
164 return nil
165 }
166 if !d.Type().IsRegular() && !d.IsDir() {
167 return nil // sockets, symlinks
168 }
169 if d.IsDir() {
170 if strings.HasSuffix(rel, ".git") {
171 repoCount++
172 }
173 return nil // directories are implied by member paths
174 }
175 return addFile(tw, path, filepath.ToSlash(rel))
176 })
177 if err != nil {
178 return err
179 }
180 }
181 if err := tw.Close(); err != nil {
182 return err
183 }
184 if err := gz.Close(); err != nil {
185 return err
186 }
187 if enc != nil {
188 if err := enc.Close(); err != nil {
189 return err
190 }
191 }
192 if err := f.Sync(); err != nil {
193 return err
194 }
195 if err := f.Close(); err != nil {
196 return err
197 }
198 if err := os.Rename(f.Name(), out); err != nil {
199 return err
200 }
201 done = true
202 if err := syncDir(dir); err != nil {
203 return err
204 }
205
206 info, _ := os.Stat(out)
207 if dbOnly {
208 fmt.Printf("wrote %s (database only, %.1f MB)\n", out, float64(info.Size())/1e6)
209 return nil
210 }
211 fmt.Printf("wrote %s (%d repositories, %.1f MB)\n", out, repoCount, float64(info.Size())/1e6)
212 return nil
213}
214
215// syncDir makes a rename in dir durable.
216func syncDir(dir string) error {
217 d, err := os.Open(dir)
218 if err != nil {
219 return err
220 }
221 defer d.Close()
222 return d.Sync()
223}
224
225// snapshotDB writes a consistent copy of the live database. VACUUM INTO
226// takes a read snapshot, so concurrent daemon writes are safe under WAL.
227func snapshotDB(st *store.Store, dest string) error {
228 quoted := strings.ReplaceAll(dest, "'", "''")
229 _, err := st.DB.Exec(fmt.Sprintf("VACUUM INTO '%s'", quoted))
230 return err
231}
232
233func addFile(tw *tar.Writer, path, name string) error {
234 info, err := os.Stat(path)
235 if err != nil {
236 return err
237 }
238 hdr, err := tar.FileInfoHeader(info, "")
239 if err != nil {
240 return err
241 }
242 hdr.Name = name
243 if err := tw.WriteHeader(hdr); err != nil {
244 return err
245 }
246 src, err := os.Open(path)
247 if err != nil {
248 return err
249 }
250 defer src.Close()
251 _, err = io.Copy(tw, src)
252 return err
253}
254
255// verifyBackup reads an archive back, decrypting it with identity when it
256// is encrypted: the database snapshot must pass
257// SQLite's integrity check, and every repository it names must be in the
258// archive. A database-only archive is checked for integrity alone and
259// says so. Nothing is written except a temporary copy of the database.
260func verifyBackup(path, identity string) error {
261 f, err := os.Open(path)
262 if err != nil {
263 return err
264 }
265 defer f.Close()
266 plain, err := archiveReader(f, path, identity)
267 if err != nil {
268 return err
269 }
270 gz, err := gzip.NewReader(plain)
271 if err != nil {
272 return fmt.Errorf("%s: not a gzip archive: %w", path, err)
273 }
274 tr := tar.NewReader(gz)
275 tmp, err := os.MkdirTemp("", "gitbay-verify-")
276 if err != nil {
277 return err
278 }
279 defer os.RemoveAll(tmp)
280 dbPath := ""
281 inArchive := map[string]bool{}
282 members := 0
283 for {
284 h, err := tr.Next()
285 if err == io.EOF {
286 break
287 }
288 if err != nil {
289 return fmt.Errorf("%s: archive damaged after %d members: %w", path, members, err)
290 }
291 members++
292 switch {
293 case h.Name == "gitbay.db":
294 dbPath = filepath.Join(tmp, "gitbay.db")
295 w, err := os.Create(dbPath)
296 if err != nil {
297 return err
298 }
299 if _, err := io.Copy(w, tr); err != nil {
300 w.Close()
301 return fmt.Errorf("%s: extracting the database: %w", path, err)
302 }
303 w.Close()
304 case strings.HasPrefix(h.Name, "repos/"):
305 // repos/<owner>/<name>.git/HEAD marks one repository present.
306 parts := strings.Split(h.Name, "/")
307 if len(parts) == 4 && parts[3] == "HEAD" && strings.HasSuffix(parts[2], ".git") {
308 inArchive[parts[1]+"/"+strings.TrimSuffix(parts[2], ".git")] = true
309 }
310 }
311 }
312 // Read to the end so gzip checks its trailer and age its final chunk.
313 if _, err := io.Copy(io.Discard, gz); err != nil {
314 return fmt.Errorf("%s: archive truncated or damaged: %w", path, err)
315 }
316 if err := gz.Close(); err != nil {
317 return fmt.Errorf("%s: archive truncated or damaged: %w", path, err)
318 }
319 if dbPath == "" {
320 return fmt.Errorf("%s: no gitbay.db in the archive", path)
321 }
322 st, err := store.Open(dbPath)
323 if err != nil {
324 return fmt.Errorf("%s: database does not open: %w", path, err)
325 }
326 defer st.Close()
327 var integrity string
328 if err := st.DB.QueryRow("PRAGMA integrity_check").Scan(&integrity); err != nil {
329 return fmt.Errorf("%s: integrity check: %w", path, err)
330 }
331 if integrity != "ok" {
332 return fmt.Errorf("%s: database integrity: %s", path, integrity)
333 }
334 repos, err := st.ListAllRepos()
335 if err != nil {
336 return err
337 }
338 if len(inArchive) == 0 {
339 fmt.Printf("%s: database only; integrity ok, %d repositories in the database, none in the archive\n", path, len(repos))
340 return nil
341 }
342 var missing []string
343 for _, r := range repos {
344 if !inArchive[r.Path()] {
345 missing = append(missing, r.Path())
346 }
347 }
348 extra := len(inArchive) - (len(repos) - len(missing))
349 fmt.Printf("%s: integrity ok, %d repositories in the database, %d in the archive\n", path, len(repos), len(inArchive))
350 if len(missing) > 0 {
351 return fmt.Errorf("%s: %d repositories the database names are not in the archive: %s", path, len(missing), strings.Join(missing, ", "))
352 }
353 if extra > 0 {
354 fmt.Printf("%d repositories in the archive that the database does not name (deleted after the snapshot)\n", extra)
355 }
356 return nil
357}
358
359const ageHeader = "age-encryption.org/v1\n"
360
361// archiveReader returns the archive's gzip stream, decrypting it first
362// when it is an age file.
363func archiveReader(f io.Reader, path, identity string) (io.Reader, error) {
364 br := bufio.NewReader(f)
365 head, _ := br.Peek(len(ageHeader))
366 if string(head) != ageHeader {
367 if identity != "" {
368 fmt.Fprintf(os.Stderr, "%s is not encrypted; --identity was not used\n", path)
369 }
370 return br, nil
371 }
372 if identity == "" {
373 return nil, fmt.Errorf("%s is encrypted; pass --identity <file> with the private key for one of its recipients", path)
374 }
375 idf, err := os.Open(identity)
376 if err != nil {
377 return nil, err
378 }
379 defer idf.Close()
380 ids, err := age.ParseIdentities(idf)
381 if err != nil {
382 return nil, fmt.Errorf("%s: %w", identity, err)
383 }
384 r, err := age.Decrypt(br, ids...)
385 if err != nil {
386 return nil, fmt.Errorf("%s: decrypting: %w", path, err)
387 }
388 return r, nil
389}