| @@ -15,7 +15,9 @@ import ( |
| 15 | "filippo.io/age" |
15 | "filippo.io/age" |
| 16 | "github.com/spf13/cobra" |
16 | "github.com/spf13/cobra" |
| 17 | |
17 | |
| |
18 | "gitbay.org/gitbay/internal/backuplock" |
| 18 | "gitbay.org/gitbay/internal/config" |
19 | "gitbay.org/gitbay/internal/config" |
| |
20 | "gitbay.org/gitbay/internal/gitutil" |
| 19 | "gitbay.org/gitbay/internal/store" |
21 | "gitbay.org/gitbay/internal/store" |
| 20 | ) |
22 | ) |
| 21 | |
23 | |
| @@ -62,7 +64,7 @@ public keys and its name ends in .age. --verify then needs --identity |
| 62 | } |
64 | } |
| 63 | cmd.Flags().StringVar(&out, "out", "", "output archive path (default gitbay-backup-<utc timestamp>.tar.gz; .age is appended when [backup] age_recipients is set)") |
65 | cmd.Flags().StringVar(&out, "out", "", "output archive path (default gitbay-backup-<utc timestamp>.tar.gz; .age is appended when [backup] age_recipients is set)") |
| 64 | cmd.Flags().BoolVar(&dbOnly, "db-only", false, "archive the database snapshot alone, without repositories") |
66 | cmd.Flags().BoolVar(&dbOnly, "db-only", false, "archive the database snapshot alone, without repositories") |
| 65 | cmd.Flags().StringVar(&verify, "verify", "", "check an archive instead of writing one: database integrity, and its repositories against the archive's") |
67 | cmd.Flags().StringVar(&verify, "verify", "", "check an archive instead of writing one: database integrity, its repositories against the archive's, and git connectivity of each") |
| 66 | cmd.Flags().StringVar(&identity, "identity", "", "with --verify: an age identity file that opens an encrypted archive") |
68 | cmd.Flags().StringVar(&identity, "identity", "", "with --verify: an age identity file that opens an encrypted archive") |
| 67 | return cmd |
69 | return cmd |
| 68 | } |
70 | } |
| @@ -90,6 +92,17 @@ func runBackup(cfg config.Config, out string, dbOnly bool) error { |
| 90 | return fmt.Errorf("%s ends in .age but [backup] age_recipients is not set, so the archive would not be encrypted", out) |
92 | return fmt.Errorf("%s ends in .age but [backup] age_recipients is not set, so the archive would not be encrypted", out) |
| 91 | } |
93 | } |
| 92 | |
94 | |
| |
95 | // Deletes, renames and transfers wait until the walk finishes, so |
| |
96 | // every repository the snapshot names is still on disk when the walk |
| |
97 | // reaches it (#259). A database-only archive reads no repository. |
| |
98 | if !dbOnly { |
| |
99 | release, err := backuplock.Hold(cfg.Server.Root) |
| |
100 | if err != nil { |
| |
101 | return fmt.Errorf("backup lock: %w", err) |
| |
102 | } |
| |
103 | defer release() |
| |
104 | } |
| |
105 | |
| 93 | st, err := openStore(cfg) |
106 | st, err := openStore(cfg) |
| 94 | if err != nil { |
107 | if err != nil { |
| 95 | return err |
108 | return err |
| @@ -142,6 +155,7 @@ func runBackup(cfg config.Config, out string, dbOnly bool) error { |
| 142 | skip := map[string]bool{ |
155 | skip := map[string]bool{ |
| 143 | "gitbay.db": true, "gitbay.db-wal": true, "gitbay.db-shm": true, |
156 | "gitbay.db": true, "gitbay.db-wal": true, "gitbay.db-shm": true, |
| 144 | "hook.sock": true, "askpass.sh": true, "hooks": true, |
157 | "hook.sock": true, "askpass.sh": true, "hooks": true, |
| |
158 | backuplock.Name: true, |
| 145 | } |
159 | } |
| 146 | repoCount := 0 |
160 | repoCount := 0 |
| 147 | root := cfg.Server.Root |
161 | root := cfg.Server.Root |
| @@ -170,7 +184,11 @@ func runBackup(cfg config.Config, out string, dbOnly bool) error { |
| 170 | if strings.HasSuffix(rel, ".git") { |
184 | if strings.HasSuffix(rel, ".git") { |
| 171 | repoCount++ |
185 | repoCount++ |
| 172 | } |
186 | } |
| 173 | return nil // directories are implied by member paths |
187 | // A directory entry, even for one that holds no file (a |
| |
188 | // bare repository's refs/heads and refs/tags once every |
| |
189 | // ref is packed), so extraction recreates it: git's own |
| |
190 | // repository discovery needs refs/ to exist. |
| |
191 | return addDir(tw, path, filepath.ToSlash(rel)) |
| 174 | } |
192 | } |
| 175 | return addFile(tw, path, filepath.ToSlash(rel)) |
193 | return addFile(tw, path, filepath.ToSlash(rel)) |
| 176 | }) |
194 | }) |
| @@ -252,11 +270,29 @@ func addFile(tw *tar.Writer, path, name string) error { |
| 252 | return err |
270 | return err |
| 253 | } |
271 | } |
| 254 | |
272 | |
| |
273 | // addDir writes a directory entry, so an empty directory survives |
| |
274 | // extraction. The mode never exceeds 0755, whatever the source directory |
| |
275 | // carries. |
| |
276 | func addDir(tw *tar.Writer, path, name string) error { |
| |
277 | info, err := os.Stat(path) |
| |
278 | if err != nil { |
| |
279 | return err |
| |
280 | } |
| |
281 | hdr, err := tar.FileInfoHeader(info, "") |
| |
282 | if err != nil { |
| |
283 | return err |
| |
284 | } |
| |
285 | hdr.Name = name + "/" |
| |
286 | hdr.Mode = hdr.Mode&^0o777 | hdr.Mode&0o755 |
| |
287 | return tw.WriteHeader(hdr) |
| |
288 | } |
| |
289 | |
| 255 | // verifyBackup reads an archive back, decrypting it with identity when it |
290 | // verifyBackup reads an archive back, decrypting it with identity when it |
| 256 | // is encrypted: the database snapshot must pass |
291 | // is encrypted: the database snapshot must pass SQLite's integrity check, |
| 257 | // SQLite's integrity check, and every repository it names must be in the |
292 | // every repository it names must be in the archive, and each of those |
| 258 | // archive. A database-only archive is checked for integrity alone and |
293 | // must pass git fsck --connectivity-only. A database-only archive is |
| 259 | // says so. Nothing is written except a temporary copy of the database. |
294 | // checked for integrity alone and says so. Repositories are extracted to |
| |
295 | // a temporary directory for the check, so it needs free space for them. |
| 260 | func verifyBackup(path, identity string) error { |
296 | func verifyBackup(path, identity string) error { |
| 261 | f, err := os.Open(path) |
297 | f, err := os.Open(path) |
| 262 | if err != nil { |
298 | if err != nil { |
| @@ -292,21 +328,32 @@ func verifyBackup(path, identity string) error { |
| 292 | switch { |
328 | switch { |
| 293 | case h.Name == "gitbay.db": |
329 | case h.Name == "gitbay.db": |
| 294 | dbPath = filepath.Join(tmp, "gitbay.db") |
330 | dbPath = filepath.Join(tmp, "gitbay.db") |
| 295 | w, err := os.Create(dbPath) |
331 | if err := extractTo(tr, dbPath); err != nil { |
| 296 | if err != nil { |
| |
| 297 | return err |
| |
| 298 | } |
| |
| 299 | if _, err := io.Copy(w, tr); err != nil { |
| |
| 300 | w.Close() |
| |
| 301 | return fmt.Errorf("%s: extracting the database: %w", path, err) |
332 | return fmt.Errorf("%s: extracting the database: %w", path, err) |
| 302 | } |
333 | } |
| 303 | w.Close() |
| |
| 304 | case strings.HasPrefix(h.Name, "repos/"): |
334 | case strings.HasPrefix(h.Name, "repos/"): |
| |
335 | trimmed := strings.TrimSuffix(h.Name, "/") |
| 305 | // repos/<owner>/<name>.git/HEAD marks one repository present. |
336 | // repos/<owner>/<name>.git/HEAD marks one repository present. |
| 306 | parts := strings.Split(h.Name, "/") |
337 | parts := strings.Split(trimmed, "/") |
| 307 | if len(parts) == 4 && parts[3] == "HEAD" && strings.HasSuffix(parts[2], ".git") { |
338 | if len(parts) == 4 && parts[3] == "HEAD" && strings.HasSuffix(parts[2], ".git") { |
| 308 | inArchive[parts[1]+"/"+strings.TrimSuffix(parts[2], ".git")] = true |
339 | inArchive[parts[1]+"/"+strings.TrimSuffix(parts[2], ".git")] = true |
| 309 | } |
340 | } |
| |
341 | if !filepath.IsLocal(trimmed) { |
| |
342 | return fmt.Errorf("%s: member %q leaves the archive root", path, h.Name) |
| |
343 | } |
| |
344 | dest := filepath.Join(tmp, filepath.FromSlash(trimmed)) |
| |
345 | switch h.Typeflag { |
| |
346 | case tar.TypeDir: |
| |
347 | // The archive's directory modes do not matter to fsck, and |
| |
348 | // a hostile one would stop RemoveAll cleaning up. |
| |
349 | if err := os.MkdirAll(dest, 0o700); err != nil { |
| |
350 | return fmt.Errorf("%s: creating %s: %w", path, h.Name, err) |
| |
351 | } |
| |
352 | case tar.TypeReg: |
| |
353 | if err := extractTo(tr, dest); err != nil { |
| |
354 | return fmt.Errorf("%s: extracting %s: %w", path, h.Name, err) |
| |
355 | } |
| |
356 | } |
| 310 | } |
357 | } |
| 311 | } |
358 | } |
| 312 | // Read to the end so gzip checks its trailer and age its final chunk. |
359 | // Read to the end so gzip checks its trailer and age its final chunk. |
| @@ -351,11 +398,39 @@ func verifyBackup(path, identity string) error { |
| 351 | return fmt.Errorf("%s: %d repositories the database names are not in the archive: %s", path, len(missing), strings.Join(missing, ", ")) |
398 | return fmt.Errorf("%s: %d repositories the database names are not in the archive: %s", path, len(missing), strings.Join(missing, ", ")) |
| 352 | } |
399 | } |
| 353 | if extra > 0 { |
400 | if extra > 0 { |
| 354 | fmt.Printf("%d repositories in the archive that the database does not name (deleted after the snapshot)\n", extra) |
401 | fmt.Printf("%d repositories in the archive that the database does not name (created after the snapshot)\n", extra) |
| 355 | } |
402 | } |
| |
403 | var broken []string |
| |
404 | for _, r := range repos { |
| |
405 | dir := filepath.Join(tmp, "repos", r.OwnerName, r.Name+".git") |
| |
406 | if err := gitutil.FsckConnectivity(dir); err != nil { |
| |
407 | fmt.Fprintf(os.Stderr, "%s: %v\n", r.Path(), err) |
| |
408 | broken = append(broken, r.Path()) |
| |
409 | } |
| |
410 | } |
| |
411 | if len(broken) > 0 { |
| |
412 | return fmt.Errorf("%s: %d repositories fail the connectivity check: %s", path, len(broken), strings.Join(broken, ", ")) |
| |
413 | } |
| |
414 | fmt.Printf("connectivity ok on %d repositories\n", len(repos)) |
| 356 | return nil |
415 | return nil |
| 357 | } |
416 | } |
| 358 | |
417 | |
| |
418 | // extractTo writes one archive member to dest, owner-only. |
| |
419 | func extractTo(r io.Reader, dest string) error { |
| |
420 | if err := os.MkdirAll(filepath.Dir(dest), 0o700); err != nil { |
| |
421 | return err |
| |
422 | } |
| |
423 | w, err := os.OpenFile(dest, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o600) |
| |
424 | if err != nil { |
| |
425 | return err |
| |
426 | } |
| |
427 | if _, err := io.Copy(w, r); err != nil { |
| |
428 | w.Close() |
| |
429 | return err |
| |
430 | } |
| |
431 | return w.Close() |
| |
432 | } |
| |
433 | |
| 359 | const ageHeader = "age-encryption.org/v1\n" |
434 | const ageHeader = "age-encryption.org/v1\n" |
| 360 | |
435 | |
| 361 | // archiveReader returns the archive's gzip stream, decrypting it first |
436 | // archiveReader returns the archive's gzip stream, decrypting it first |