backup: verify release assets and LFS; admin restore-drill !518

merged merged by cmc on 2026-09-29 03:37 UTC · krz/gitbay:backup-verify-259 into main

10 files changed, +459 −38

Layout: unified · split

.gitbay/wiki/Admin.org +78 −19

	@@ -491,11 +491,18 @@ each one it removes.
491		491
492	=--verify= reads an archive back: the snapshot must pass SQLite's	492	=--verify= reads an archive back: the snapshot must pass SQLite's
493	integrity check, every repository the snapshot names must be in the	493	integrity check, every repository the snapshot names must be in the
494	archive, and each must pass =git fsck --connectivity-only=. It	494	archive, and each must pass =git fsck --connectivity-only=. Every
495	extracts the repositories to a temporary directory for that, so it	495	release asset the snapshot names must be in its repository with its
496	needs free space the size of the repositories. A database-only archive	496	recorded size and sha256, and every LFS object in the archive must
497	is checked for integrity and says so. Exit is non-zero on damage, a	497	hash to its name. LFS objects are named by pointer files in git
498	missing repository or a missing object.	498	history rather than by the database, so a pointer whose object was
		499	never uploaded is not caught, and with =[lfs] root= outside
		500	=server.root= the archive holds no LFS objects at all. It extracts the
		501	repositories to a temporary directory for the checks, so it needs free
		502	space the size of the repositories. A database-only archive is checked
		503	for integrity and says so. Exit is non-zero on damage, a missing
		504	repository, a missing object, or a missing or altered asset or LFS
		505	object.
499		506
500	A full backup holds =<root>/backup.lock= from its database snapshot to	507	A full backup holds =<root>/backup.lock= from its database snapshot to
501	its last repository. While it runs, =repo delete=, =repo rename=,	508	its last repository. While it runs, =repo delete=, =repo rename=,
	@@ -653,28 +660,80 @@ gitbayd admin secrets rotate # new key, reseal, retire the old one (as root)
653		660
654	** Restore drill	661	** Restore drill
655		662
656	A restore onto a clean host, run quarterly and after any change to the	663	A restore onto a clean host, run quarterly (January, April, July,
657	backup code (=cmd/gitbayd/backup.go=, the offsite job), and recorded	664	October) and after any change to the backup code
658	below. The disaster it rehearses is losing bay1, so the local archives	665	(=cmd/gitbayd/backup.go=, =cmd/gitbayd/restoredrill.go=, the offsite
659	are gone with it and the sources are the main offsite restic	666	job), and recorded below. The disaster it rehearses is losing bay1, so
660	repository (repositories, LFS, the staged database, =config.toml=),	667	the local archives are gone with it and the sources are the main
661	the off-host copy of =secret.key= and =apns.p8= (a keys repository once	668	offsite restic repository (repositories, LFS, the staged database,
662	runbook D creates it; until then the operator's hand-made copy), and	669	=config.toml=), the off-host copy of =secret.key= and =apns.p8= (a keys
663	the operator's password manager (=offsite.env=, the keys repository's	670	repository once runbook D creates it; until then the operator's
664	password and token once it exists, =backup-identity.txt=). The steps are in the data-at-rest	671	hand-made copy), and the operator's password manager (=offsite.env=,
665	plan's operator runbook	672	the keys repository's password and token once it exists,
666	(=docs/plans/2026-09-27-data-at-rest-and-backup.md=).	673	=backup-identity.txt=). The full steps are runbook C of the
		674	data-at-rest plan (=docs/plans/2026-09-27-data-at-rest-and-backup.md=).
		675
		676	=gitbayd admin restore-drill= does the archive half. It extracts a
		677	full archive into an empty or absent directory, runs every =--verify=
		678	check on the extracted copy, and prints what was restored, the newest
		679	issue, issue comment, merge request comment and push in the restored
		680	database, and the elapsed time. Exit is non-zero if any check fails.
		681
		682	#+begin_src sh
		683	gitbayd admin restore-drill /var/backups/gitbay/gitbay-20260927-090000.tar.gz --into /srv/drill
		684	gitbayd admin restore-drill <archive>.tar.gz.age --identity backup-identity.txt --into /srv/drill
		685	#+end_src
		686
		687	What to restore, and from where:
		688
		689	\| Item \| Source \| Path on the drill host \|
		690	\|-------------------------------+-------------------------------------------------------------+------------------------------------------------\|
		691	\| Database \| staged copy in =/var/lib/gitbay-stage= (restic), or an archive \| =<root>/gitbay.db= \|
		692	\| Repositories \| =/var/lib/gitbay/repos= (restic), or an archive \| =<root>/repos= \|
		693	\| LFS objects \| =/var/lib/gitbay/lfs= (restic), or an archive \| =<root>/lfs= (or =[lfs] root=) \|
		694	\| Release assets \| inside each repository (=gitbay-releases/=) \| with the repositories \|
		695	\| Host keys \| =/var/lib/gitbay/ssh= (restic), or an archive \| =<root>/ssh= (or =[ssh] host_keys=) \|
		696	\| =config.toml= \| =/var/lib/gitbay-stage/config.toml= (restic) \| =/etc/gitbay/config.toml= \|
		697	\| =secret.key=, =apns.p8= \| the off-host copy; no archive or main snapshot carries them \| =/etc/gitbay/=, mode 0600, owned by =gitbay= \|
		698
		699	From the offsite path (restic is not run by any gitbay command):
		700
		701	#+begin_src sh
		702	restic restore latest --target / --include /var/lib/gitbay --include /var/lib/gitbay-stage
		703	cp /var/lib/gitbay-stage/gitbay.db /var/lib/gitbay/gitbay.db
		704	gitbayd --config /etc/gitbay/config.toml admin backup --out /tmp/drill.tar.gz
		705	gitbayd admin restore-drill /tmp/drill.tar.gz --into /tmp/drill-root
		706	#+end_src
		707
		708	The second archive is how the restic tree gets the same checks and
		709	timestamps; =/tmp/drill-root= is discarded afterwards.
		710
		711	What to check, each a column below:
		712
		713	- DB integrity, connectivity, release assets, LFS: =restore-drill=
		714	prints =integrity ok=, =connectivity ok on N repositories=, =release
		715	assets ok: N=, =LFS objects ok: N=. Compare N with =gitbayd admin
		716	stats --json= on the source at the snapshot time.
		717	- Secrets: =gitbayd --config /etc/gitbay/config.toml admin secrets
		718	check= opens every value.
		719	- Host key: =ssh-keyscan -p 22 <drill-host>= matches the source's
		720	fingerprint.
		721	- Config: =gitbayd --config /etc/gitbay/config.toml check-config=.
		722	- Service: =ssh -p 22 git@<drill-host> whoami= and a =git clone= over
		723	SSH succeed.
667		724
668	Time to service runs from the clean host's first root login to the	725	Time to service runs from the clean host's first root login to the
669	first successful =git clone= over SSH from it. The recovery point is	726	first successful =git clone= over SSH from it. The recovery point is
670	the time of the newest restic snapshot restored.	727	the time of the newest restic snapshot restored; record beside it the
		728	newest issue, comment and push =restore-drill= printed, which show how
		729	much activity the restore carries.
671		730
672	No drill has been run yet; the procedure above is written but	731	No drill has been run yet; the procedure above is written but
673	unexercised, and #259 stays open until the first row below is	732	unexercised, and #259 stays open until the first row below is
674	recorded.	733	recorded.
675		734
676	\| Date \| Host \| Snapshot restored (UTC) \| Time to service \| DB integrity \| Connectivity \| LFS \| Release assets \| Host key \| Secrets \| Notes \|	735	\| Date \| Host \| Snapshot restored (UTC) \| Newest issue / comment / push \| Time to service \| DB integrity \| Connectivity \| LFS \| Release assets \| Host key \| Secrets \| Notes \|
677	\|------+------+-------------------------+-----------------+--------------+--------------+-----+----------------+----------+---------+-------\|	736	\|------+------+-------------------------+-------------------------------+-----------------+--------------+--------------+-----+----------------+----------+---------+-------\|
678		737
679	* Upgrades	738	* Upgrades
680		739

.gitbay/wiki/Architecture/08-Operations.org +6 −2

 - Excluded: WAL files, the hook socket, askpass scripts, generated
   hooks.
 - =gitbayd admin backup --verify= checks SQLite integrity, that every
-  repository the database names is present, and =git fsck
+  repository the database names is present, =git fsck
-  --connectivity-only= on each (=backup.go=).
+  --connectivity-only= on each, release assets against their recorded
+  sha256, and LFS objects against their names (=backup.go=).
+  =gitbayd admin restore-drill= runs the same checks on a full
+  extraction and reports elapsed time and the newest recovered
+  activity (=restoredrill.go=).
 - Repository deletes, renames and transfers refuse while a full backup
   runs (=internal/backuplock=), so the snapshot and the walk agree.
 - The host's restic credentials are append-only; the key that can

.gitbay/wiki/Architecture/09-Controls.org +1 −1

	@@ -99,6 +99,6 @@ chapter names of OWASP ASVS 4.0 where one fits.
99	\| Concurrency limit on git pack generation \| in place \| global, per-principal, bounded queue across SSH, HTTP and git:// (=internal/packlimit=); not in system SSH mode \|	99	\| Concurrency limit on git pack generation \| in place \| global, per-principal, bounded queue across SSH, HTTP and git:// (=internal/packlimit=); not in system SSH mode \|
100	\| Service hardening \| in place \| systemd sandboxing ([[file:03-Deployment.org][3]]) \|	100	\| Service hardening \| in place \| systemd sandboxing ([[file:03-Deployment.org][3]]) \|
101	\| Backups offsite and append-only \| in place \| restic with append-only credentials (documented) \|	101	\| Backups offsite and append-only \| in place \| restic with append-only credentials (documented) \|
102	\| Restore tested \| gap \| #259 \|	102	\| Restore tested \| gap \| tooling in place (=admin restore-drill=, Admin wiki "Restore drill"); clean-host drill pending (#259) \|
103	\| Migrations validated before commit \| gap \| foreign-key check runs after commit (#261) \|	103	\| Migrations validated before commit \| gap \| foreign-key check runs after commit (#261) \|
104	\| Signed, reviewed changes to production \| in place \| signed commits, =require-mr=, ff-only merges, clean-tree deploys \|	104	\| Signed, reviewed changes to production \| in place \| signed commits, =require-mr=, ff-only merges, clean-tree deploys \|

.gitbay/wiki/Architecture/10-Known-Gaps.org +1 −1

	@@ -10,7 +10,7 @@ what the 2026-09-27 review found; remove a row when its issue closes.
10		10
11	\| Issue \| Area \| Gap \| Severity \|	11	\| Issue \| Area \| Gap \| Severity \|
12	\|-------+------------------+-----------------------------------------------------------------------+----------\|	12	\|-------+------------------+-----------------------------------------------------------------------+----------\|
13	\| #259 \| Recovery \| No restore has been exercised; the drill is written (Admin wiki) and not yet run \| high \|	13	\| #259 \| Recovery \| No restore has been exercised; the procedure and tooling (=admin restore-drill=, =backup --verify=) are in place, the clean-host drill is pending \| high \|
14	\| #260 \| CI network \| Builds share the runner's source address; no egress policy \| medium \|	14	\| #260 \| CI network \| Builds share the runner's source address; no egress policy \| medium \|
15	\| #261 \| Various \| Migration foreign-key check after commit; three web writes bypass dispatch; documentation drift \| medium \|	15	\| #261 \| Various \| Migration foreign-key check after commit; three web writes bypass dispatch; documentation drift \| medium \|
16		16

CHANGELOG.org +12

	@@ -4,6 +4,18 @@ Versioning follows semver from v0.1.0. Database migrations run
4	automatically on daemon start; upgrade notes appear per release when	4	automatically on daemon start; upgrade notes appear per release when
5	anything beyond "replace the binary and restart" is needed.	5	anything beyond "replace the binary and restart" is needed.
6		6
		7	* Unreleased
		8
		9	- =gitbayd admin backup --verify= also checks every release asset the
		10	database names against its recorded size and sha256, and every
		11	archived LFS object against its name (#259).
		12	- =gitbayd admin restore-drill <archive> --into <dir>= extracts a full
		13	archive into an empty directory, runs the =--verify= checks on the
		14	extracted copy, and prints the newest issue, comment and push it
		15	recovered and the elapsed time. The Admin wiki's Restore drill
		16	section lists what to restore, what to check and where to record it
		17	(#259).
		18
7	* v1.37.0 — 2026-09-29	19	* v1.37.0 — 2026-09-29
8		20
9	Findings from the 2026-09-27 architecture review.	21	Findings from the 2026-09-27 architecture review.

cmd/gitbayd/backup.go +115 −15

	@@ -4,6 +4,8 @@ import (
4	"archive/tar"	4	"archive/tar"
5	"bufio"	5	"bufio"
6	"compress/gzip"	6	"compress/gzip"
		7	"crypto/sha256"
		8	"encoding/hex"
7	"errors"	9	"errors"
8	"fmt"	10	"fmt"
9	"io"	11	"io"
	@@ -12,6 +14,7 @@ import (
12	"path"	14	"path"
13	"path/filepath"	15	"path/filepath"
14	"regexp"	16	"regexp"
		17	"strconv"
15	"strings"	18	"strings"
16	"time"	19	"time"
17		20
	@@ -21,6 +24,7 @@ import (
21	"gitbay.org/gitbay/internal/backuplock"	24	"gitbay.org/gitbay/internal/backuplock"
22	"gitbay.org/gitbay/internal/config"	25	"gitbay.org/gitbay/internal/config"
23	"gitbay.org/gitbay/internal/gitutil"	26	"gitbay.org/gitbay/internal/gitutil"
		27	"gitbay.org/gitbay/internal/lfs"
24	"gitbay.org/gitbay/internal/store"	28	"gitbay.org/gitbay/internal/store"
25	)	29	)
26		30
	@@ -71,7 +75,7 @@ public keys and its name ends in .age. --verify then needs --identity
71	}	75	}
72	cmd.Flags().StringVar(&out, "out", "", "output archive path (default gitbay-backup-<utc timestamp>.tar.gz; .age is appended when [backup] age_recipients is set)")	76	cmd.Flags().StringVar(&out, "out", "", "output archive path (default gitbay-backup-<utc timestamp>.tar.gz; .age is appended when [backup] age_recipients is set)")
73	cmd.Flags().BoolVar(&dbOnly, "db-only", false, "archive the database snapshot alone, without repositories")	77	cmd.Flags().BoolVar(&dbOnly, "db-only", false, "archive the database snapshot alone, without repositories")
74	cmd.Flags().StringVar(&verify, "verify", "", "check an archive instead of writing one: database integrity, its repositories against the archive's, and git connectivity of each")	78	cmd.Flags().StringVar(&verify, "verify", "", "check an archive instead of writing one: database integrity, its repositories against the archive's, git connectivity of each, release assets and LFS object digests")
75	cmd.Flags().StringVar(&identity, "identity", "", "with --verify: an age identity file that opens an encrypted archive")	79	cmd.Flags().StringVar(&identity, "identity", "", "with --verify: an age identity file that opens an encrypted archive")
76	return cmd	80	return cmd
77	}	81	}
	@@ -459,10 +463,24 @@ func addDir(tw *tar.Writer, path, name string) error {
459	// verifyBackup reads an archive back, decrypting it with identity when it	463	// verifyBackup reads an archive back, decrypting it with identity when it
460	// is encrypted: the database snapshot must pass SQLite's integrity check,	464	// is encrypted: the database snapshot must pass SQLite's integrity check,
461	// every repository it names must be in the archive, and each of those	465	// every repository it names must be in the archive, and each of those
462	// must pass git fsck --connectivity-only. A database-only archive is	466	// must pass git fsck --connectivity-only; release assets must match the
		467	// database and LFS objects their names. A database-only archive is
463	// checked for integrity alone and says so. Repositories are extracted to	468	// checked for integrity alone and says so. Repositories are extracted to
464	// a temporary directory for the check, so it needs free space for them.	469	// a temporary directory for the check, so it needs free space for them.
465	func verifyBackup(path, identity string) error {	470	func verifyBackup(path, identity string) error {
		471	tmp, err := os.MkdirTemp("", "gitbay-verify-")
		472	if err != nil {
		473	return err
		474	}
		475	defer os.RemoveAll(tmp)
		476	return checkArchive(path, identity, tmp, false)
		477	}
		478
		479	// checkArchive extracts the archive at path into dest and runs verify's
		480	// checks on it. With full unset it extracts only the database and the
		481	// repositories; with full set, every member, so dest is a restored
		482	// server.root. Alternates and commondir are left out either way.
		483	func checkArchive(path, identity, dest string, full bool) error {
466	f, err := os.Open(path)	484	f, err := os.Open(path)
467	if err != nil {	485	if err != nil {
468	return err	486	return err
	@@ -477,14 +495,11 @@ func verifyBackup(path, identity string) error {
477	return fmt.Errorf("%s: not a gzip archive: %w", path, err)	495	return fmt.Errorf("%s: not a gzip archive: %w", path, err)
478	}	496	}
479	tr := tar.NewReader(gz)	497	tr := tar.NewReader(gz)
480	tmp, err := os.MkdirTemp("", "gitbay-verify-")
481	if err != nil {
482	return err
483	}
484	defer os.RemoveAll(tmp)
485	dbPath := ""	498	dbPath := ""
486	inArchive := map[string]bool{}	499	inArchive := map[string]bool{}
487	members := 0	500	members := 0
		501	lfsObjects := 0
		502	var badLFS []string
488	for {	503	for {
489	h, err := tr.Next()	504	h, err := tr.Next()
490	if err == io.EOF {	505	if err == io.EOF {
	@@ -496,15 +511,35 @@ func verifyBackup(path, identity string) error {
496	members++	511	members++
497	switch {	512	switch {
498	case h.Name == "gitbay.db":	513	case h.Name == "gitbay.db":
499	dbPath = filepath.Join(tmp, "gitbay.db")	514	dbPath = filepath.Join(dest, "gitbay.db")
500	if err := extractTo(tr, dbPath); err != nil {	515	if err := extractTo(tr, dbPath); err != nil {
501	return fmt.Errorf("%s: extracting the database: %w", path, err)	516	return fmt.Errorf("%s: extracting the database: %w", path, err)
502	}	517	}
503	case strings.HasPrefix(h.Name, "repos/"):	518	case strings.HasPrefix(h.Name, "lfs/") && h.Typeflag == tar.TypeReg && lfs.OIDPat.MatchString(filepath.Base(h.Name)):
		519	// Objects are named by their sha256, so each is checked as it
		520	// streams past and none is extracted. Other names, such as an
		521	// upload's .upload-* staging file, are not objects.
		522	lfsObjects++
		523	if !filepath.IsLocal(h.Name) {
		524	return fmt.Errorf("%s: member %q leaves the archive root", path, h.Name)
		525	}
		526	sum := sha256.New()
		527	if full {
		528	err = extractTo(io.TeeReader(tr, sum), filepath.Join(dest, filepath.FromSlash(h.Name)))
		529	} else {
		530	_, err = io.Copy(sum, tr)
		531	}
		532	if err != nil {
		533	return fmt.Errorf("%s: reading %s: %w", path, h.Name, err)
		534	}
		535	if hex.EncodeToString(sum.Sum(nil)) != filepath.Base(h.Name) {
		536	badLFS = append(badLFS, h.Name)
		537	}
		538	case full \|\| strings.HasPrefix(h.Name, "repos/"):
504	trimmed := strings.TrimSuffix(h.Name, "/")	539	trimmed := strings.TrimSuffix(h.Name, "/")
505	// repos/<owner>/<name>.git/HEAD marks one repository present.	540	// repos/<owner>/<name>.git/HEAD marks one repository present.
506	parts := strings.Split(trimmed, "/")	541	parts := strings.Split(trimmed, "/")
507	if len(parts) == 4 && parts[3] == "HEAD" && strings.HasSuffix(parts[2], ".git") {	542	if len(parts) == 4 && parts[0] == "repos" && parts[3] == "HEAD" && strings.HasSuffix(parts[2], ".git") {
508	inArchive[parts[1]+"/"+strings.TrimSuffix(parts[2], ".git")] = true	543	inArchive[parts[1]+"/"+strings.TrimSuffix(parts[2], ".git")] = true
509	}	544	}
510	if !filepath.IsLocal(trimmed) {	545	if !filepath.IsLocal(trimmed) {
	@@ -513,7 +548,7 @@ func verifyBackup(path, identity string) error {
513	if borrowsObjects(trimmed) {	548	if borrowsObjects(trimmed) {
514	continue	549	continue
515	}	550	}
516	dest := filepath.Join(tmp, filepath.FromSlash(trimmed))	551	dest := filepath.Join(dest, filepath.FromSlash(trimmed))
517	switch h.Typeflag {	552	switch h.Typeflag {
518	case tar.TypeDir:	553	case tar.TypeDir:
519	// The archive's directory modes do not matter to fsck, and	554	// The archive's directory modes do not matter to fsck, and
	@@ -572,19 +607,84 @@ func verifyBackup(path, identity string) error {
572	if extra > 0 {	607	if extra > 0 {
573	fmt.Printf("%d repositories in the archive that the database does not name (created after the snapshot)\n", extra)	608	fmt.Printf("%d repositories in the archive that the database does not name (created after the snapshot)\n", extra)
574	}	609	}
		610	var failed []error
575	var broken []string	611	var broken []string
576	for _, r := range repos {	612	for _, r := range repos {
577	dir := filepath.Join(tmp, "repos", r.OwnerName, r.Name+".git")	613	dir := filepath.Join(dest, "repos", r.OwnerName, r.Name+".git")
578	if err := gitutil.FsckConnectivity(dir); err != nil {	614	if err := gitutil.FsckConnectivity(dir); err != nil {
579	fmt.Fprintf(os.Stderr, "%s: %v\n", r.Path(), err)	615	fmt.Fprintf(os.Stderr, "%s: %v\n", r.Path(), err)
580	broken = append(broken, r.Path())	616	broken = append(broken, r.Path())
581	}	617	}
582	}	618	}
583	if len(broken) > 0 {	619	if len(broken) > 0 {
584	return fmt.Errorf("%s: %d repositories fail the connectivity check: %s", path, len(broken), strings.Join(broken, ", "))	620	failed = append(failed, fmt.Errorf("%s: %d repositories fail the connectivity check: %s", path, len(broken), strings.Join(broken, ", ")))
		621	} else {
		622	fmt.Printf("connectivity ok on %d repositories\n", len(repos))
585	}	623	}
586	fmt.Printf("connectivity ok on %d repositories\n", len(repos))	624	assets, badAssets, err := checkReleaseAssets(st, repos, dest)
587	return nil	625	if err != nil {
		626	return err
		627	}
		628	if len(badAssets) > 0 {
		629	failed = append(failed, fmt.Errorf("%s: %d release assets missing or not matching their digest: %s", path, len(badAssets), strings.Join(badAssets, ", ")))
		630	} else {
		631	fmt.Printf("release assets ok: %d\n", assets)
		632	}
		633	// LFS objects are named by pointer files in git history, not by the
		634	// database, so this checks the archived objects' digests and not that
		635	// every pointer has its object. With [lfs] root outside server.root
		636	// the archive carries none.
		637	if len(badLFS) > 0 {
		638	failed = append(failed, fmt.Errorf("%s: %d LFS objects do not match their digest: %s", path, len(badLFS), strings.Join(badLFS, ", ")))
		639	} else {
		640	fmt.Printf("LFS objects ok: %d\n", lfsObjects)
		641	}
		642	return errors.Join(failed...)
		643	}
		644
		645	// checkReleaseAssets checks that every release asset the database names
		646	// is under root, extracted, with its recorded size and sha256. It
		647	// returns how many the database names and those that fail.
		648	func checkReleaseAssets(st *store.Store, repos []store.Repo, root string) (int, []string, error) {
		649	byID := map[int64]store.Repo{}
		650	for _, r := range repos {
		651	byID[r.ID] = r
		652	}
		653	rows, err := st.DB.Query(`SELECT rl.repo_id, a.release_id, a.name, a.size, a.sha256
		654	FROM release_assets a JOIN releases rl ON rl.id = a.release_id
		655	ORDER BY rl.repo_id, a.release_id, a.name`)
		656	if err != nil {
		657	return 0, nil, err
		658	}
		659	defer rows.Close()
		660	n := 0
		661	var bad []string
		662	for rows.Next() {
		663	var repoID, relID, size int64
		664	var name, want string
		665	if err := rows.Scan(&repoID, &relID, &name, &size, &want); err != nil {
		666	return 0, nil, err
		667	}
		668	n++
		669	r, ok := byID[repoID]
		670	owner := r.Path()
		671	if !ok {
		672	owner = fmt.Sprintf("repository %d", repoID)
		673	}
		674	label := fmt.Sprintf("%s release %d %s", owner, relID, name)
		675	f, err := os.Open(filepath.Join(root, "repos", r.OwnerName, r.Name+".git", "gitbay-releases", strconv.FormatInt(relID, 10), name))
		676	if err != nil {
		677	bad = append(bad, label)
		678	continue
		679	}
		680	sum := sha256.New()
		681	got, err := io.Copy(sum, f)
		682	f.Close()
		683	if err != nil \|\| got != size \|\| hex.EncodeToString(sum.Sum(nil)) != want {
		684	bad = append(bad, label)
		685	}
		686	}
		687	return n, bad, rows.Err()
588	}	688	}
589		689
590	// borrowsObjects reports an archive member that would point git at	690	// borrowsObjects reports an archive member that would point git at

cmd/gitbayd/backup_test.go +75

	@@ -3,6 +3,8 @@ package main
3	import (	3	import (
4	"archive/tar"	4	"archive/tar"
5	"compress/gzip"	5	"compress/gzip"
		6	"crypto/sha256"
		7	"encoding/hex"
6	"errors"	8	"errors"
7	"io"	9	"io"
8	"io/fs"	10	"io/fs"
	@@ -10,6 +12,7 @@ import (
10	"os/exec"	12	"os/exec"
11	"path/filepath"	13	"path/filepath"
12	"sort"	14	"sort"
		15	"strconv"
13	"strings"	16	"strings"
14	"testing"	17	"testing"
15	"time"	18	"time"
	@@ -840,3 +843,75 @@ func TestVerifyIgnoresCommondir(t *testing.T) {
840	t.Fatalf("verify of a repository whose objects are only in its commondir: %v", err)	843	t.Fatalf("verify of a repository whose objects are only in its commondir: %v", err)
841	}	844	}
842	}	845	}
		846
		847	// verify checks each release asset the database names against its
		848	// recorded digest, and each archived LFS object against its name.
		849	func TestVerifyChecksReleaseAssetsAndLFS(t *testing.T) {
		850	cfg := testConfig(t)
		851	root := cfg.Server.Root
		852	st, err := openStore(cfg)
		853	if err != nil {
		854	t.Fatal(err)
		855	}
		856	uid, err := st.CreateUser("krz", false)
		857	if err != nil {
		858	t.Fatal(err)
		859	}
		860	rid, err := st.CreateRepo("user", uid, "thing", "public")
		861	if err != nil {
		862	t.Fatal(err)
		863	}
		864	relID, err := st.CreateRelease(rid, "v1", "v1", "", uid, "md")
		865	if err != nil {
		866	t.Fatal(err)
		867	}
		868	asset := []byte("binary\n")
		869	sum := sha256.Sum256(asset)
		870	if err := st.AddReleaseAsset(relID, "tool", int64(len(asset)), hex.EncodeToString(sum[:])); err != nil {
		871	t.Fatal(err)
		872	}
		873	st.Close()
		874	dir := filepath.Join(root, "repos", "krz", "thing.git")
		875	gitIn(t, root, "init", "-q", "--bare", dir)
		876	assetFile := filepath.Join(dir, "gitbay-releases", strconv.FormatInt(relID, 10), "tool")
		877	writeFile(t, assetFile, asset)
		878	obj := []byte("large\n")
		879	oid := sha256.Sum256(obj)
		880	o := hex.EncodeToString(oid[:])
		881	writeFile(t, filepath.Join(root, "lfs", o[:2], o[2:4], o), obj)
		882	// An upload in progress stages a temporary file beside the objects.
		883	writeFile(t, filepath.Join(root, "lfs", o[:2], o[2:4], ".upload-123"), []byte("partial"))
		884
		885	good := filepath.Join(t.TempDir(), "good.tar.gz")
		886	if err := runBackup(cfg, good, false); err != nil {
		887	t.Fatal(err)
		888	}
		889	if err := verifyBackup(good, ""); err != nil {
		890	t.Fatalf("intact archive: %v", err)
		891	}
		892
		893	writeFile(t, assetFile, []byte("tampered\n"))
		894	wrong := strings.Repeat("0", 64)
		895	writeFile(t, filepath.Join(root, "lfs", "00", "00", wrong), obj)
		896	bad := filepath.Join(t.TempDir(), "bad.tar.gz")
		897	if err := runBackup(cfg, bad, false); err != nil {
		898	t.Fatal(err)
		899	}
		900	err = verifyBackup(bad, "")
		901	if err == nil \|\| !strings.Contains(err.Error(), "krz/thing release") \|\| !strings.Contains(err.Error(), wrong) {
		902	t.Fatalf("archive with a bad asset and LFS object: %v", err)
		903	}
		904	if strings.Contains(err.Error(), o) \|\| strings.Contains(err.Error(), ".upload-") {
		905	t.Fatalf("intact LFS object or upload staging file reported: %v", err)
		906	}
		907	}
		908
		909	func writeFile(t *testing.T, p string, b []byte) {
		910	t.Helper()
		911	if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil {
		912	t.Fatal(err)
		913	}
		914	if err := os.WriteFile(p, b, 0o644); err != nil {
		915	t.Fatal(err)
		916	}
		917	}

cmd/gitbayd/main.go +1

	@@ -455,6 +455,7 @@ func adminCmd() *cobra.Command {
455	hostCmd("runners [--json]", "runner accounts: last poll, scope, the build each holds", "admin", "runners"),	455	hostCmd("runners [--json]", "runner accounts: last poll, scope, the build each holds", "admin", "runners"),
456	auditCmd,	456	auditCmd,
457	backupCmd(),	457	backupCmd(),
		458	restoreDrillCmd(),
458	secretsCmd(),	459	secretsCmd(),
459	gcCmd(),	460	gcCmd(),
460	adminMigrateCommitRefsCmd(),	461	adminMigrateCommitRefsCmd(),

cmd/gitbayd/restoredrill.go added +107

@@ -0,0 +1,107 @@
	1	package main
	2
	3	import (
	4	"errors"
	5	"fmt"
	6	"io/fs"
	7	"os"
	8	"path/filepath"
	9	"strings"
	10	"time"
	11
	12	"github.com/spf13/cobra"
	13
	14	"gitbay.org/gitbay/internal/store"
	15	)
	16
	17	// restoreDrillCmd rehearses the archive half of a restore: extract a
	18	// full archive into an empty directory, run verify's checks on what was
	19	// extracted, and report the elapsed time and the newest activity the
	20	// restored database holds.
	21	func restoreDrillCmd() *cobra.Command {
	22	var into, identity string
	23	cmd := &cobra.Command{
	24	Use: "restore-drill <archive> --into <dir>",
	25	Short: "restore a full archive into an empty directory, verify it, report elapsed time and the newest recovered activity",
	26	Long: `Extracts every member of a full backup archive into --into, which must
	27	be empty or absent, and runs the checks of backup --verify on the
	28	extracted copy: database integrity, every repository present and
	29	passing git fsck --connectivity-only, release assets and LFS object
	30	digests. It then prints the newest issue, comment and push in the
	31	restored database, which is the recovery point, and the elapsed time.
	32
	33	The result is a server.root a gitbayd can be pointed at. The archive
	34	does not carry server.secret_key_file or config.toml; the Admin wiki's
	35	Restore drill section covers those and the offsite path.`,
	36	Args: cobra.ExactArgs(1),
	37	RunE: func(cmd *cobra.Command, args []string) error {
	38	if into == "" {
	39	return errors.New("--into <dir> is required")
	40	}
	41	return restoreDrill(args[0], identity, into)
	42	},
	43	}
	44	cmd.Flags().StringVar(&into, "into", "", "empty or absent directory to restore into")
	45	cmd.Flags().StringVar(&identity, "identity", "", "an age identity file that opens an encrypted archive")
	46	return cmd
	47	}
	48
	49	func restoreDrill(archive, identity, into string) error {
	50	start := time.Now()
	51	ents, err := os.ReadDir(into)
	52	switch {
	53	case errors.Is(err, fs.ErrNotExist):
	54	if err := os.MkdirAll(into, 0o700); err != nil {
	55	return err
	56	}
	57	case err != nil:
	58	return err
	59	case len(ents) > 0:
	60	return fmt.Errorf("%s is not empty; restore into an empty or absent directory", into)
	61	}
	62	checkErr := checkArchive(archive, identity, into, true)
	63	if ents, err := os.ReadDir(into); err == nil {
	64	var names []string
	65	for _, e := range ents {
	66	names = append(names, e.Name())
	67	}
	68	fmt.Printf("restored into %s: %s\n", into, strings.Join(names, " "))
	69	}
	70	// store.Open would create a missing database.
	71	db := filepath.Join(into, "gitbay.db")
	72	if _, err := os.Stat(db); err == nil {
	73	if err := printNewest(db); err != nil {
	74	checkErr = errors.Join(checkErr, err)
	75	}
	76	}
	77	fmt.Printf("elapsed %s\n", time.Since(start).Round(100*time.Millisecond))
	78	return checkErr
	79	}
	80
	81	// newest are the recovered timestamps a drill records.
	82	var newest = []struct{ label, query string }{
	83	{"issue", "SELECT MAX(created_at) FROM issues"},
	84	{"issue comment", "SELECT MAX(created_at) FROM issue_comments"},
	85	{"merge request comment", "SELECT MAX(created_at) FROM mr_comments"},
	86	{"push", "SELECT MAX(created_at) FROM events WHERE kind = 'push'"},
	87	}
	88
	89	func printNewest(db string) error {
	90	st, err := store.Open(db)
	91	if err != nil {
	92	return err
	93	}
	94	defer st.Close()
	95	for _, n := range newest {
	96	var at *string
	97	if err := st.DB.QueryRow(n.query).Scan(&at); err != nil {
	98	return fmt.Errorf("newest %s: %w", n.label, err)
	99	}
	100	v := "none"
	101	if at != nil {
	102	v = *at
	103	}
	104	fmt.Printf("newest %s: %s\n", n.label, v)
	105	}
	106	return nil
	107	}

cmd/gitbayd/restoredrill_test.go added +63

@@ -0,0 +1,63 @@
	1	package main
	2
	3	import (
	4	"os"
	5	"path/filepath"
	6	"strings"
	7	"testing"
	8	)
	9
	10	// A drill restores a full archive into an empty directory as a usable
	11	// root, verifies it, and refuses a directory that already holds files.
	12	func TestRestoreDrill(t *testing.T) {
	13	cfg := testConfig(t)
	14	root := cfg.Server.Root
	15	st, err := openStore(cfg)
	16	if err != nil {
	17	t.Fatal(err)
	18	}
	19	uid, err := st.CreateUser("krz", false)
	20	if err != nil {
	21	t.Fatal(err)
	22	}
	23	rid, err := st.CreateRepo("user", uid, "thing", "public")
	24	if err != nil {
	25	t.Fatal(err)
	26	}
	27	if _, err := st.CreateIssue(rid, uid, "bug", "", "md"); err != nil {
	28	t.Fatal(err)
	29	}
	30	if err := st.RecordEvent(rid, uid, "push", "{}"); err != nil {
	31	t.Fatal(err)
	32	}
	33	st.Close()
	34	work := t.TempDir()
	35	gitIn(t, work, "init", "-q", "-b", "main")
	36	writeFile(t, filepath.Join(work, "a.txt"), []byte("a\n"))
	37	gitIn(t, work, "add", "a.txt")
	38	gitIn(t, work, "commit", "-q", "-m", "one")
	39	gitIn(t, work, "clone", "-q", "--bare", work, filepath.Join(root, "repos", "krz", "thing.git"))
	40	writeFile(t, filepath.Join(root, "ssh", "host_ed25519"), []byte("key\n"))
	41
	42	archive := filepath.Join(t.TempDir(), "b.tar.gz")
	43	if err := runBackup(cfg, archive, false); err != nil {
	44	t.Fatal(err)
	45	}
	46	into := filepath.Join(t.TempDir(), "restored")
	47	if err := restoreDrill(archive, "", into); err != nil {
	48	t.Fatal(err)
	49	}
	50	for _, p := range []string{"gitbay.db", "ssh/host_ed25519", "repos/krz/thing.git/HEAD"} {
	51	if _, err := os.Stat(filepath.Join(into, p)); err != nil {
	52	t.Errorf("restored root lacks %s: %v", p, err)
	53	}
	54	}
	55	if got := gitIn(t, filepath.Join(into, "repos", "krz", "thing.git"), "log", "--format=%s", "main"); got != "one" {
	56	t.Errorf("restored log %q", got)
	57	}
	58
	59	err = restoreDrill(archive, "", into)
	60	if err == nil \|\| !strings.Contains(err.Error(), "not empty") {
	61	t.Fatalf("drill into a non-empty directory: %v", err)
	62	}
	63	}