| @@ -15,7 +15,9 @@ import ( |
| 15 | 15 | "filippo.io/age" |
| 16 | 16 | "github.com/spf13/cobra" |
| 17 | 17 | |
| 18 | "gitbay.org/gitbay/internal/backuplock" |
| 18 | 19 | "gitbay.org/gitbay/internal/config" |
| 20 | "gitbay.org/gitbay/internal/gitutil" |
| 19 | 21 | "gitbay.org/gitbay/internal/store" |
| 20 | 22 | ) |
| 21 | 23 | |
| @@ -62,7 +64,7 @@ public keys and its name ends in .age. --verify then needs --identity |
| 62 | 64 | } |
| 63 | 65 | cmd.Flags().StringVar(&out, "out", "", "output archive path (default gitbay-backup-<utc timestamp>.tar.gz; .age is appended when [backup] age_recipients is set)") |
| 64 | 66 | cmd.Flags().BoolVar(&dbOnly, "db-only", false, "archive the database snapshot alone, without repositories") |
| 65 | | cmd.Flags().StringVar(&verify, "verify", "", "check an archive instead of writing one: database integrity, and its repositories against the archive's") |
| 67 | cmd.Flags().StringVar(&verify, "verify", "", "check an archive instead of writing one: database integrity, its repositories against the archive's, and git connectivity of each") |
| 66 | 68 | cmd.Flags().StringVar(&identity, "identity", "", "with --verify: an age identity file that opens an encrypted archive") |
| 67 | 69 | return cmd |
| 68 | 70 | } |
| @@ -90,6 +92,17 @@ func runBackup(cfg config.Config, out string, dbOnly bool) error { |
| 90 | 92 | return fmt.Errorf("%s ends in .age but [backup] age_recipients is not set, so the archive would not be encrypted", out) |
| 91 | 93 | } |
| 92 | 94 | |
| 95 | // Deletes, renames and transfers wait until the walk finishes, so |
| 96 | // every repository the snapshot names is still on disk when the walk |
| 97 | // reaches it (#259). A database-only archive reads no repository. |
| 98 | if !dbOnly { |
| 99 | release, err := backuplock.Hold(cfg.Server.Root) |
| 100 | if err != nil { |
| 101 | return fmt.Errorf("backup lock: %w", err) |
| 102 | } |
| 103 | defer release() |
| 104 | } |
| 105 | |
| 93 | 106 | st, err := openStore(cfg) |
| 94 | 107 | if err != nil { |
| 95 | 108 | return err |
| @@ -142,6 +155,7 @@ func runBackup(cfg config.Config, out string, dbOnly bool) error { |
| 142 | 155 | skip := map[string]bool{ |
| 143 | 156 | "gitbay.db": true, "gitbay.db-wal": true, "gitbay.db-shm": true, |
| 144 | 157 | "hook.sock": true, "askpass.sh": true, "hooks": true, |
| 158 | backuplock.Name: true, |
| 145 | 159 | } |
| 146 | 160 | repoCount := 0 |
| 147 | 161 | root := cfg.Server.Root |
| @@ -170,7 +184,11 @@ func runBackup(cfg config.Config, out string, dbOnly bool) error { |
| 170 | 184 | if strings.HasSuffix(rel, ".git") { |
| 171 | 185 | repoCount++ |
| 172 | 186 | } |
| 173 | | return nil // directories are implied by member paths |
| 187 | // A directory entry, even for one that holds no file (a |
| 188 | // bare repository's refs/heads and refs/tags once every |
| 189 | // ref is packed), so extraction recreates it: git's own |
| 190 | // repository discovery needs refs/ to exist. |
| 191 | return addDir(tw, path, filepath.ToSlash(rel)) |
| 174 | 192 | } |
| 175 | 193 | return addFile(tw, path, filepath.ToSlash(rel)) |
| 176 | 194 | }) |
| @@ -252,11 +270,29 @@ func addFile(tw *tar.Writer, path, name string) error { |
| 252 | 270 | return err |
| 253 | 271 | } |
| 254 | 272 | |
| 273 | // addDir writes a directory entry, so an empty directory survives |
| 274 | // extraction. The mode never exceeds 0755, whatever the source directory |
| 275 | // carries. |
| 276 | func addDir(tw *tar.Writer, path, name string) error { |
| 277 | info, err := os.Stat(path) |
| 278 | if err != nil { |
| 279 | return err |
| 280 | } |
| 281 | hdr, err := tar.FileInfoHeader(info, "") |
| 282 | if err != nil { |
| 283 | return err |
| 284 | } |
| 285 | hdr.Name = name + "/" |
| 286 | hdr.Mode = hdr.Mode&^0o777 | hdr.Mode&0o755 |
| 287 | return tw.WriteHeader(hdr) |
| 288 | } |
| 289 | |
| 255 | 290 | // verifyBackup reads an archive back, decrypting it with identity when it |
| 256 | | // is encrypted: the database snapshot must pass |
| 257 | | // SQLite's integrity check, and every repository it names must be in the |
| 258 | | // archive. A database-only archive is checked for integrity alone and |
| 259 | | // says so. Nothing is written except a temporary copy of the database. |
| 291 | // is encrypted: the database snapshot must pass SQLite's integrity check, |
| 292 | // every repository it names must be in the archive, and each of those |
| 293 | // must pass git fsck --connectivity-only. A database-only archive is |
| 294 | // checked for integrity alone and says so. Repositories are extracted to |
| 295 | // a temporary directory for the check, so it needs free space for them. |
| 260 | 296 | func verifyBackup(path, identity string) error { |
| 261 | 297 | f, err := os.Open(path) |
| 262 | 298 | if err != nil { |
| @@ -292,21 +328,32 @@ func verifyBackup(path, identity string) error { |
| 292 | 328 | switch { |
| 293 | 329 | case h.Name == "gitbay.db": |
| 294 | 330 | dbPath = filepath.Join(tmp, "gitbay.db") |
| 295 | | w, err := os.Create(dbPath) |
| 296 | | if err != nil { |
| 297 | | return err |
| 298 | | } |
| 299 | | if _, err := io.Copy(w, tr); err != nil { |
| 300 | | w.Close() |
| 331 | if err := extractTo(tr, dbPath); err != nil { |
| 301 | 332 | return fmt.Errorf("%s: extracting the database: %w", path, err) |
| 302 | 333 | } |
| 303 | | w.Close() |
| 304 | 334 | case strings.HasPrefix(h.Name, "repos/"): |
| 335 | trimmed := strings.TrimSuffix(h.Name, "/") |
| 305 | 336 | // repos/<owner>/<name>.git/HEAD marks one repository present. |
| 306 | | parts := strings.Split(h.Name, "/") |
| 337 | parts := strings.Split(trimmed, "/") |
| 307 | 338 | if len(parts) == 4 && parts[3] == "HEAD" && strings.HasSuffix(parts[2], ".git") { |
| 308 | 339 | inArchive[parts[1]+"/"+strings.TrimSuffix(parts[2], ".git")] = true |
| 309 | 340 | } |
| 341 | if !filepath.IsLocal(trimmed) { |
| 342 | return fmt.Errorf("%s: member %q leaves the archive root", path, h.Name) |
| 343 | } |
| 344 | dest := filepath.Join(tmp, filepath.FromSlash(trimmed)) |
| 345 | switch h.Typeflag { |
| 346 | case tar.TypeDir: |
| 347 | // The archive's directory modes do not matter to fsck, and |
| 348 | // a hostile one would stop RemoveAll cleaning up. |
| 349 | if err := os.MkdirAll(dest, 0o700); err != nil { |
| 350 | return fmt.Errorf("%s: creating %s: %w", path, h.Name, err) |
| 351 | } |
| 352 | case tar.TypeReg: |
| 353 | if err := extractTo(tr, dest); err != nil { |
| 354 | return fmt.Errorf("%s: extracting %s: %w", path, h.Name, err) |
| 355 | } |
| 356 | } |
| 310 | 357 | } |
| 311 | 358 | } |
| 312 | 359 | // Read to the end so gzip checks its trailer and age its final chunk. |
| @@ -351,11 +398,39 @@ func verifyBackup(path, identity string) error { |
| 351 | 398 | return fmt.Errorf("%s: %d repositories the database names are not in the archive: %s", path, len(missing), strings.Join(missing, ", ")) |
| 352 | 399 | } |
| 353 | 400 | if extra > 0 { |
| 354 | | fmt.Printf("%d repositories in the archive that the database does not name (deleted after the snapshot)\n", extra) |
| 401 | fmt.Printf("%d repositories in the archive that the database does not name (created after the snapshot)\n", extra) |
| 355 | 402 | } |
| 403 | var broken []string |
| 404 | for _, r := range repos { |
| 405 | dir := filepath.Join(tmp, "repos", r.OwnerName, r.Name+".git") |
| 406 | if err := gitutil.FsckConnectivity(dir); err != nil { |
| 407 | fmt.Fprintf(os.Stderr, "%s: %v\n", r.Path(), err) |
| 408 | broken = append(broken, r.Path()) |
| 409 | } |
| 410 | } |
| 411 | if len(broken) > 0 { |
| 412 | return fmt.Errorf("%s: %d repositories fail the connectivity check: %s", path, len(broken), strings.Join(broken, ", ")) |
| 413 | } |
| 414 | fmt.Printf("connectivity ok on %d repositories\n", len(repos)) |
| 356 | 415 | return nil |
| 357 | 416 | } |
| 358 | 417 | |
| 418 | // extractTo writes one archive member to dest, owner-only. |
| 419 | func extractTo(r io.Reader, dest string) error { |
| 420 | if err := os.MkdirAll(filepath.Dir(dest), 0o700); err != nil { |
| 421 | return err |
| 422 | } |
| 423 | w, err := os.OpenFile(dest, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o600) |
| 424 | if err != nil { |
| 425 | return err |
| 426 | } |
| 427 | if _, err := io.Copy(w, r); err != nil { |
| 428 | w.Close() |
| 429 | return err |
| 430 | } |
| 431 | return w.Close() |
| 432 | } |
| 433 | |
| 359 | 434 | const ageHeader = "age-encryption.org/v1\n" |
| 360 | 435 | |
| 361 | 436 | // archiveReader returns the archive's gzip stream, decrypting it first |