| @@ -4,6 +4,8 @@ import ( |
| 4 | 4 | "archive/tar" |
| 5 | 5 | "bufio" |
| 6 | 6 | "compress/gzip" |
| 7 | "crypto/sha256" |
| 8 | "encoding/hex" |
| 7 | 9 | "errors" |
| 8 | 10 | "fmt" |
| 9 | 11 | "io" |
| @@ -12,6 +14,7 @@ import ( |
| 12 | 14 | "path" |
| 13 | 15 | "path/filepath" |
| 14 | 16 | "regexp" |
| 17 | "strconv" |
| 15 | 18 | "strings" |
| 16 | 19 | "time" |
| 17 | 20 | |
| @@ -71,7 +74,7 @@ public keys and its name ends in .age. --verify then needs --identity |
| 71 | 74 | } |
| 72 | 75 | cmd.Flags().StringVar(&out, "out", "", "output archive path (default gitbay-backup-<utc timestamp>.tar.gz; .age is appended when [backup] age_recipients is set)") |
| 73 | 76 | cmd.Flags().BoolVar(&dbOnly, "db-only", false, "archive the database snapshot alone, without repositories") |
| 74 | | cmd.Flags().StringVar(&verify, "verify", "", "check an archive instead of writing one: database integrity, its repositories against the archive's, and git connectivity of each") |
| 77 | cmd.Flags().StringVar(&verify, "verify", "", "check an archive instead of writing one: database integrity, its repositories against the archive's, git connectivity of each, release assets and LFS object digests") |
| 75 | 78 | cmd.Flags().StringVar(&identity, "identity", "", "with --verify: an age identity file that opens an encrypted archive") |
| 76 | 79 | return cmd |
| 77 | 80 | } |
| @@ -459,7 +462,8 @@ func addDir(tw *tar.Writer, path, name string) error { |
| 459 | 462 | // verifyBackup reads an archive back, decrypting it with identity when it |
| 460 | 463 | // is encrypted: the database snapshot must pass SQLite's integrity check, |
| 461 | 464 | // every repository it names must be in the archive, and each of those |
| 462 | | // must pass git fsck --connectivity-only. A database-only archive is |
| 465 | // must pass git fsck --connectivity-only; release assets must match the |
| 466 | // database and LFS objects their names. A database-only archive is |
| 463 | 467 | // checked for integrity alone and says so. Repositories are extracted to |
| 464 | 468 | // a temporary directory for the check, so it needs free space for them. |
| 465 | 469 | func verifyBackup(path, identity string) error { |
| @@ -485,6 +489,8 @@ func verifyBackup(path, identity string) error { |
| 485 | 489 | dbPath := "" |
| 486 | 490 | inArchive := map[string]bool{} |
| 487 | 491 | members := 0 |
| 492 | lfsObjects := 0 |
| 493 | var badLFS []string |
| 488 | 494 | for { |
| 489 | 495 | h, err := tr.Next() |
| 490 | 496 | if err == io.EOF { |
| @@ -500,6 +506,17 @@ func verifyBackup(path, identity string) error { |
| 500 | 506 | if err := extractTo(tr, dbPath); err != nil { |
| 501 | 507 | return fmt.Errorf("%s: extracting the database: %w", path, err) |
| 502 | 508 | } |
| 509 | case strings.HasPrefix(h.Name, "lfs/") && h.Typeflag == tar.TypeReg: |
| 510 | // Objects are named by their sha256, so each is checked as it |
| 511 | // streams past and none is extracted. |
| 512 | lfsObjects++ |
| 513 | sum := sha256.New() |
| 514 | if _, err := io.Copy(sum, tr); err != nil { |
| 515 | return fmt.Errorf("%s: reading %s: %w", path, h.Name, err) |
| 516 | } |
| 517 | if hex.EncodeToString(sum.Sum(nil)) != filepath.Base(h.Name) { |
| 518 | badLFS = append(badLFS, h.Name) |
| 519 | } |
| 503 | 520 | case strings.HasPrefix(h.Name, "repos/"): |
| 504 | 521 | trimmed := strings.TrimSuffix(h.Name, "/") |
| 505 | 522 | // repos/<owner>/<name>.git/HEAD marks one repository present. |
| @@ -572,6 +589,7 @@ func verifyBackup(path, identity string) error { |
| 572 | 589 | if extra > 0 { |
| 573 | 590 | fmt.Printf("%d repositories in the archive that the database does not name (created after the snapshot)\n", extra) |
| 574 | 591 | } |
| 592 | var failed []error |
| 575 | 593 | var broken []string |
| 576 | 594 | for _, r := range repos { |
| 577 | 595 | dir := filepath.Join(tmp, "repos", r.OwnerName, r.Name+".git") |
| @@ -581,10 +599,70 @@ func verifyBackup(path, identity string) error { |
| 581 | 599 | } |
| 582 | 600 | } |
| 583 | 601 | if len(broken) > 0 { |
| 584 | | return fmt.Errorf("%s: %d repositories fail the connectivity check: %s", path, len(broken), strings.Join(broken, ", ")) |
| 602 | failed = append(failed, fmt.Errorf("%s: %d repositories fail the connectivity check: %s", path, len(broken), strings.Join(broken, ", "))) |
| 603 | } else { |
| 604 | fmt.Printf("connectivity ok on %d repositories\n", len(repos)) |
| 585 | 605 | } |
| 586 | | fmt.Printf("connectivity ok on %d repositories\n", len(repos)) |
| 587 | | return nil |
| 606 | assets, badAssets, err := checkReleaseAssets(st, repos, tmp) |
| 607 | if err != nil { |
| 608 | return err |
| 609 | } |
| 610 | if len(badAssets) > 0 { |
| 611 | failed = append(failed, fmt.Errorf("%s: %d release assets missing or not matching their digest: %s", path, len(badAssets), strings.Join(badAssets, ", "))) |
| 612 | } else { |
| 613 | fmt.Printf("release assets ok: %d\n", assets) |
| 614 | } |
| 615 | // LFS objects are named by pointer files in git history, not by the |
| 616 | // database, so this checks the archived objects' digests and not that |
| 617 | // every pointer has its object. With [lfs] root outside server.root |
| 618 | // the archive carries none. |
| 619 | if len(badLFS) > 0 { |
| 620 | failed = append(failed, fmt.Errorf("%s: %d LFS objects do not match their digest: %s", path, len(badLFS), strings.Join(badLFS, ", "))) |
| 621 | } else { |
| 622 | fmt.Printf("LFS objects ok: %d\n", lfsObjects) |
| 623 | } |
| 624 | return errors.Join(failed...) |
| 625 | } |
| 626 | |
| 627 | // checkReleaseAssets checks that every release asset the database names |
| 628 | // is under root, extracted, with its recorded size and sha256. It |
| 629 | // returns how many the database names and those that fail. |
| 630 | func checkReleaseAssets(st *store.Store, repos []store.Repo, root string) (int, []string, error) { |
| 631 | byID := map[int64]store.Repo{} |
| 632 | for _, r := range repos { |
| 633 | byID[r.ID] = r |
| 634 | } |
| 635 | rows, err := st.DB.Query(`SELECT rl.repo_id, a.release_id, a.name, a.size, a.sha256 |
| 636 | FROM release_assets a JOIN releases rl ON rl.id = a.release_id |
| 637 | ORDER BY rl.repo_id, a.release_id, a.name`) |
| 638 | if err != nil { |
| 639 | return 0, nil, err |
| 640 | } |
| 641 | defer rows.Close() |
| 642 | n := 0 |
| 643 | var bad []string |
| 644 | for rows.Next() { |
| 645 | var repoID, relID, size int64 |
| 646 | var name, want string |
| 647 | if err := rows.Scan(&repoID, &relID, &name, &size, &want); err != nil { |
| 648 | return 0, nil, err |
| 649 | } |
| 650 | n++ |
| 651 | r := byID[repoID] |
| 652 | label := fmt.Sprintf("%s release %d %s", r.Path(), relID, name) |
| 653 | f, err := os.Open(filepath.Join(root, "repos", r.OwnerName, r.Name+".git", "gitbay-releases", strconv.FormatInt(relID, 10), name)) |
| 654 | if err != nil { |
| 655 | bad = append(bad, label) |
| 656 | continue |
| 657 | } |
| 658 | sum := sha256.New() |
| 659 | got, err := io.Copy(sum, f) |
| 660 | f.Close() |
| 661 | if err != nil || got != size || hex.EncodeToString(sum.Sum(nil)) != want { |
| 662 | bad = append(bad, label) |
| 663 | } |
| 664 | } |
| 665 | return n, bad, rows.Err() |
| 588 | 666 | } |
| 589 | 667 | |
| 590 | 668 | // borrowsObjects reports an archive member that would point git at |