| @@ -4,6 +4,8 @@ import ( |
| 4 | "archive/tar" |
4 | "archive/tar" |
| 5 | "bufio" |
5 | "bufio" |
| 6 | "compress/gzip" |
6 | "compress/gzip" |
| |
7 | "crypto/sha256" |
| |
8 | "encoding/hex" |
| 7 | "errors" |
9 | "errors" |
| 8 | "fmt" |
10 | "fmt" |
| 9 | "io" |
11 | "io" |
| @@ -12,6 +14,7 @@ import ( |
| 12 | "path" |
14 | "path" |
| 13 | "path/filepath" |
15 | "path/filepath" |
| 14 | "regexp" |
16 | "regexp" |
| |
17 | "strconv" |
| 15 | "strings" |
18 | "strings" |
| 16 | "time" |
19 | "time" |
| 17 | |
20 | |
| @@ -71,7 +74,7 @@ public keys and its name ends in .age. --verify then needs --identity |
| 71 | } |
74 | } |
| 72 | cmd.Flags().StringVar(&out, "out", "", "output archive path (default gitbay-backup-<utc timestamp>.tar.gz; .age is appended when [backup] age_recipients is set)") |
75 | cmd.Flags().StringVar(&out, "out", "", "output archive path (default gitbay-backup-<utc timestamp>.tar.gz; .age is appended when [backup] age_recipients is set)") |
| 73 | cmd.Flags().BoolVar(&dbOnly, "db-only", false, "archive the database snapshot alone, without repositories") |
76 | cmd.Flags().BoolVar(&dbOnly, "db-only", false, "archive the database snapshot alone, without repositories") |
| 74 | cmd.Flags().StringVar(&verify, "verify", "", "check an archive instead of writing one: database integrity, its repositories against the archive's, and git connectivity of each") |
77 | cmd.Flags().StringVar(&verify, "verify", "", "check an archive instead of writing one: database integrity, its repositories against the archive's, git connectivity of each, release assets and LFS object digests") |
| 75 | cmd.Flags().StringVar(&identity, "identity", "", "with --verify: an age identity file that opens an encrypted archive") |
78 | cmd.Flags().StringVar(&identity, "identity", "", "with --verify: an age identity file that opens an encrypted archive") |
| 76 | return cmd |
79 | return cmd |
| 77 | } |
80 | } |
| @@ -459,7 +462,8 @@ func addDir(tw *tar.Writer, path, name string) error { |
| 459 | // verifyBackup reads an archive back, decrypting it with identity when it |
462 | // verifyBackup reads an archive back, decrypting it with identity when it |
| 460 | // is encrypted: the database snapshot must pass SQLite's integrity check, |
463 | // is encrypted: the database snapshot must pass SQLite's integrity check, |
| 461 | // every repository it names must be in the archive, and each of those |
464 | // every repository it names must be in the archive, and each of those |
| 462 | // must pass git fsck --connectivity-only. A database-only archive is |
465 | // must pass git fsck --connectivity-only; release assets must match the |
| |
466 | // database and LFS objects their names. A database-only archive is |
| 463 | // checked for integrity alone and says so. Repositories are extracted to |
467 | // checked for integrity alone and says so. Repositories are extracted to |
| 464 | // a temporary directory for the check, so it needs free space for them. |
468 | // a temporary directory for the check, so it needs free space for them. |
| 465 | func verifyBackup(path, identity string) error { |
469 | func verifyBackup(path, identity string) error { |
| @@ -485,6 +489,8 @@ func verifyBackup(path, identity string) error { |
| 485 | dbPath := "" |
489 | dbPath := "" |
| 486 | inArchive := map[string]bool{} |
490 | inArchive := map[string]bool{} |
| 487 | members := 0 |
491 | members := 0 |
| |
492 | lfsObjects := 0 |
| |
493 | var badLFS []string |
| 488 | for { |
494 | for { |
| 489 | h, err := tr.Next() |
495 | h, err := tr.Next() |
| 490 | if err == io.EOF { |
496 | if err == io.EOF { |
| @@ -500,6 +506,17 @@ func verifyBackup(path, identity string) error { |
| 500 | if err := extractTo(tr, dbPath); err != nil { |
506 | if err := extractTo(tr, dbPath); err != nil { |
| 501 | return fmt.Errorf("%s: extracting the database: %w", path, err) |
507 | return fmt.Errorf("%s: extracting the database: %w", path, err) |
| 502 | } |
508 | } |
| |
509 | case strings.HasPrefix(h.Name, "lfs/") && h.Typeflag == tar.TypeReg: |
| |
510 | // Objects are named by their sha256, so each is checked as it |
| |
511 | // streams past and none is extracted. |
| |
512 | lfsObjects++ |
| |
513 | sum := sha256.New() |
| |
514 | if _, err := io.Copy(sum, tr); err != nil { |
| |
515 | return fmt.Errorf("%s: reading %s: %w", path, h.Name, err) |
| |
516 | } |
| |
517 | if hex.EncodeToString(sum.Sum(nil)) != filepath.Base(h.Name) { |
| |
518 | badLFS = append(badLFS, h.Name) |
| |
519 | } |
| 503 | case strings.HasPrefix(h.Name, "repos/"): |
520 | case strings.HasPrefix(h.Name, "repos/"): |
| 504 | trimmed := strings.TrimSuffix(h.Name, "/") |
521 | trimmed := strings.TrimSuffix(h.Name, "/") |
| 505 | // repos/<owner>/<name>.git/HEAD marks one repository present. |
522 | // repos/<owner>/<name>.git/HEAD marks one repository present. |
| @@ -572,6 +589,7 @@ func verifyBackup(path, identity string) error { |
| 572 | if extra > 0 { |
589 | if extra > 0 { |
| 573 | fmt.Printf("%d repositories in the archive that the database does not name (created after the snapshot)\n", extra) |
590 | fmt.Printf("%d repositories in the archive that the database does not name (created after the snapshot)\n", extra) |
| 574 | } |
591 | } |
| |
592 | var failed []error |
| 575 | var broken []string |
593 | var broken []string |
| 576 | for _, r := range repos { |
594 | for _, r := range repos { |
| 577 | dir := filepath.Join(tmp, "repos", r.OwnerName, r.Name+".git") |
595 | dir := filepath.Join(tmp, "repos", r.OwnerName, r.Name+".git") |
| @@ -581,10 +599,70 @@ func verifyBackup(path, identity string) error { |
| 581 | } |
599 | } |
| 582 | } |
600 | } |
| 583 | if len(broken) > 0 { |
601 | if len(broken) > 0 { |
| 584 | return fmt.Errorf("%s: %d repositories fail the connectivity check: %s", path, len(broken), strings.Join(broken, ", ")) |
602 | failed = append(failed, fmt.Errorf("%s: %d repositories fail the connectivity check: %s", path, len(broken), strings.Join(broken, ", "))) |
| |
603 | } else { |
| |
604 | fmt.Printf("connectivity ok on %d repositories\n", len(repos)) |
| 585 | } |
605 | } |
| 586 | fmt.Printf("connectivity ok on %d repositories\n", len(repos)) |
606 | assets, badAssets, err := checkReleaseAssets(st, repos, tmp) |
| 587 | return nil |
607 | if err != nil { |
| |
608 | return err |
| |
609 | } |
| |
610 | if len(badAssets) > 0 { |
| |
611 | failed = append(failed, fmt.Errorf("%s: %d release assets missing or not matching their digest: %s", path, len(badAssets), strings.Join(badAssets, ", "))) |
| |
612 | } else { |
| |
613 | fmt.Printf("release assets ok: %d\n", assets) |
| |
614 | } |
| |
615 | // LFS objects are named by pointer files in git history, not by the |
| |
616 | // database, so this checks the archived objects' digests and not that |
| |
617 | // every pointer has its object. With [lfs] root outside server.root |
| |
618 | // the archive carries none. |
| |
619 | if len(badLFS) > 0 { |
| |
620 | failed = append(failed, fmt.Errorf("%s: %d LFS objects do not match their digest: %s", path, len(badLFS), strings.Join(badLFS, ", "))) |
| |
621 | } else { |
| |
622 | fmt.Printf("LFS objects ok: %d\n", lfsObjects) |
| |
623 | } |
| |
624 | return errors.Join(failed...) |
| |
625 | } |
| |
626 | |
| |
627 | // checkReleaseAssets checks that every release asset the database names |
| |
628 | // is under root, extracted, with its recorded size and sha256. It |
| |
629 | // returns how many the database names and those that fail. |
| |
630 | func checkReleaseAssets(st *store.Store, repos []store.Repo, root string) (int, []string, error) { |
| |
631 | byID := map[int64]store.Repo{} |
| |
632 | for _, r := range repos { |
| |
633 | byID[r.ID] = r |
| |
634 | } |
| |
635 | rows, err := st.DB.Query(`SELECT rl.repo_id, a.release_id, a.name, a.size, a.sha256 |
| |
636 | FROM release_assets a JOIN releases rl ON rl.id = a.release_id |
| |
637 | ORDER BY rl.repo_id, a.release_id, a.name`) |
| |
638 | if err != nil { |
| |
639 | return 0, nil, err |
| |
640 | } |
| |
641 | defer rows.Close() |
| |
642 | n := 0 |
| |
643 | var bad []string |
| |
644 | for rows.Next() { |
| |
645 | var repoID, relID, size int64 |
| |
646 | var name, want string |
| |
647 | if err := rows.Scan(&repoID, &relID, &name, &size, &want); err != nil { |
| |
648 | return 0, nil, err |
| |
649 | } |
| |
650 | n++ |
| |
651 | r := byID[repoID] |
| |
652 | label := fmt.Sprintf("%s release %d %s", r.Path(), relID, name) |
| |
653 | f, err := os.Open(filepath.Join(root, "repos", r.OwnerName, r.Name+".git", "gitbay-releases", strconv.FormatInt(relID, 10), name)) |
| |
654 | if err != nil { |
| |
655 | bad = append(bad, label) |
| |
656 | continue |
| |
657 | } |
| |
658 | sum := sha256.New() |
| |
659 | got, err := io.Copy(sum, f) |
| |
660 | f.Close() |
| |
661 | if err != nil || got != size || hex.EncodeToString(sum.Sum(nil)) != want { |
| |
662 | bad = append(bad, label) |
| |
663 | } |
| |
664 | } |
| |
665 | return n, bad, rows.Err() |
| 588 | } |
666 | } |
| 589 | |
667 | |
| 590 | // borrowsObjects reports an archive member that would point git at |
668 | // borrowsObjects reports an archive member that would point git at |