!518 · krz/gitbay

.gitbay/wiki/Admin.org +78 −19

		@@ -491,11 +491,18 @@ each one it removes.
491	491
492	492	=--verify= reads an archive back: the snapshot must pass SQLite's
493	493	integrity check, every repository the snapshot names must be in the
494		archive, and each must pass =git fsck --connectivity-only=. It
495		extracts the repositories to a temporary directory for that, so it
496		needs free space the size of the repositories. A database-only archive
497		is checked for integrity and says so. Exit is non-zero on damage, a
498		missing repository or a missing object.
	494	archive, and each must pass =git fsck --connectivity-only=. Every
	495	release asset the snapshot names must be in its repository with its
	496	recorded size and sha256, and every LFS object in the archive must
	497	hash to its name. LFS objects are named by pointer files in git
	498	history rather than by the database, so a pointer whose object was
	499	never uploaded is not caught, and with =[lfs] root= outside
	500	=server.root= the archive holds no LFS objects at all. It extracts the
	501	repositories to a temporary directory for the checks, so it needs free
	502	space the size of the repositories. A database-only archive is checked
	503	for integrity and says so. Exit is non-zero on damage, a missing
	504	repository, a missing object, or a missing or altered asset or LFS
	505	object.
499	506
500	507	A full backup holds =<root>/backup.lock= from its database snapshot to
501	508	its last repository. While it runs, =repo delete=, =repo rename=,
		@@ -653,28 +660,80 @@ gitbayd admin secrets rotate # new key, reseal, retire the old one (as root)
653	660
654	661	** Restore drill
655	662
656		A restore onto a clean host, run quarterly and after any change to the
657		backup code (=cmd/gitbayd/backup.go=, the offsite job), and recorded
658		below. The disaster it rehearses is losing bay1, so the local archives
659		are gone with it and the sources are the main offsite restic
660		repository (repositories, LFS, the staged database, =config.toml=),
661		the off-host copy of =secret.key= and =apns.p8= (a keys repository once
662		runbook D creates it; until then the operator's hand-made copy), and
663		the operator's password manager (=offsite.env=, the keys repository's
664		password and token once it exists, =backup-identity.txt=). The steps are in the data-at-rest
665		plan's operator runbook
666		(=docs/plans/2026-09-27-data-at-rest-and-backup.md=).
	663	A restore onto a clean host, run quarterly (January, April, July,
	664	October) and after any change to the backup code
	665	(=cmd/gitbayd/backup.go=, =cmd/gitbayd/restoredrill.go=, the offsite
	666	job), and recorded below. The disaster it rehearses is losing bay1, so
	667	the local archives are gone with it and the sources are the main
	668	offsite restic repository (repositories, LFS, the staged database,
	669	=config.toml=), the off-host copy of =secret.key= and =apns.p8= (a keys
	670	repository once runbook D creates it; until then the operator's
	671	hand-made copy), and the operator's password manager (=offsite.env=,
	672	the keys repository's password and token once it exists,
	673	=backup-identity.txt=). The full steps are runbook C of the
	674	data-at-rest plan (=docs/plans/2026-09-27-data-at-rest-and-backup.md=).
	675
	676	=gitbayd admin restore-drill= does the archive half. It extracts a
	677	full archive into an empty or absent directory, runs every =--verify=
	678	check on the extracted copy, and prints what was restored, the newest
	679	issue, issue comment, merge request comment and push in the restored
	680	database, and the elapsed time. Exit is non-zero if any check fails.
	681
	682	#+begin_src sh
	683	gitbayd admin restore-drill /var/backups/gitbay/gitbay-20260927-090000.tar.gz --into /srv/drill
	684	gitbayd admin restore-drill <archive>.tar.gz.age --identity backup-identity.txt --into /srv/drill
	685	#+end_src
	686
	687	What to restore, and from where:
	688
	689	\| Item \| Source \| Path on the drill host \|
	690	\|-------------------------------+-------------------------------------------------------------+------------------------------------------------\|
	691	\| Database \| staged copy in =/var/lib/gitbay-stage= (restic), or an archive \| =<root>/gitbay.db= \|
	692	\| Repositories \| =/var/lib/gitbay/repos= (restic), or an archive \| =<root>/repos= \|
	693	\| LFS objects \| =/var/lib/gitbay/lfs= (restic), or an archive \| =<root>/lfs= (or =[lfs] root=) \|
	694	\| Release assets \| inside each repository (=gitbay-releases/=) \| with the repositories \|
	695	\| Host keys \| =/var/lib/gitbay/ssh= (restic), or an archive \| =<root>/ssh= (or =[ssh] host_keys=) \|
	696	\| =config.toml= \| =/var/lib/gitbay-stage/config.toml= (restic) \| =/etc/gitbay/config.toml= \|
	697	\| =secret.key=, =apns.p8= \| the off-host copy; no archive or main snapshot carries them \| =/etc/gitbay/=, mode 0600, owned by =gitbay= \|
	698
	699	From the offsite path (restic is not run by any gitbay command):
	700
	701	#+begin_src sh
	702	restic restore latest --target / --include /var/lib/gitbay --include /var/lib/gitbay-stage
	703	cp /var/lib/gitbay-stage/gitbay.db /var/lib/gitbay/gitbay.db
	704	gitbayd --config /etc/gitbay/config.toml admin backup --out /tmp/drill.tar.gz
	705	gitbayd admin restore-drill /tmp/drill.tar.gz --into /tmp/drill-root
	706	#+end_src
	707
	708	The second archive is how the restic tree gets the same checks and
	709	timestamps; =/tmp/drill-root= is discarded afterwards.
	710
	711	What to check, each a column below:
	712
	713	- DB integrity, connectivity, release assets, LFS: =restore-drill=
	714	prints =integrity ok=, =connectivity ok on N repositories=, =release
	715	assets ok: N=, =LFS objects ok: N=. Compare N with =gitbayd admin
	716	stats --json= on the source at the snapshot time.
	717	- Secrets: =gitbayd --config /etc/gitbay/config.toml admin secrets
	718	check= opens every value.
	719	- Host key: =ssh-keyscan -p 22 <drill-host>= matches the source's
	720	fingerprint.
	721	- Config: =gitbayd --config /etc/gitbay/config.toml check-config=.
	722	- Service: =ssh -p 22 git@<drill-host> whoami= and a =git clone= over
	723	SSH succeed.
667	724
668	725	Time to service runs from the clean host's first root login to the
669	726	first successful =git clone= over SSH from it. The recovery point is
670		the time of the newest restic snapshot restored.
	727	the time of the newest restic snapshot restored; record beside it the
	728	newest issue, comment and push =restore-drill= printed, which show how
	729	much activity the restore carries.
671	730
672	731	No drill has been run yet; the procedure above is written but
673	732	unexercised, and #259 stays open until the first row below is
674	733	recorded.
675	734
676		\| Date \| Host \| Snapshot restored (UTC) \| Time to service \| DB integrity \| Connectivity \| LFS \| Release assets \| Host key \| Secrets \| Notes \|
677		\|------+------+-------------------------+-----------------+--------------+--------------+-----+----------------+----------+---------+-------\|
	735	\| Date \| Host \| Snapshot restored (UTC) \| Newest issue / comment / push \| Time to service \| DB integrity \| Connectivity \| LFS \| Release assets \| Host key \| Secrets \| Notes \|
	736	\|------+------+-------------------------+-------------------------------+-----------------+--------------+--------------+-----+----------------+----------+---------+-------\|
678	737
679	738	* Upgrades
680	739

.gitbay/wiki/Architecture/08-Operations.org +6 −2

		@@ -62,8 +62,12 @@ the product activity feed, not an audit trail.
62	62	- Excluded: WAL files, the hook socket, askpass scripts, generated
63	63	hooks.
64	64	- =gitbayd admin backup --verify= checks SQLite integrity, that every
65		repository the database names is present, and =git fsck
66		--connectivity-only= on each (=backup.go=).
	65	repository the database names is present, =git fsck
	66	--connectivity-only= on each, release assets against their recorded
	67	sha256, and LFS objects against their names (=backup.go=).
	68	=gitbayd admin restore-drill= runs the same checks on a full
	69	extraction and reports elapsed time and the newest recovered
	70	activity (=restoredrill.go=).
67	71	- Repository deletes, renames and transfers refuse while a full backup
68	72	runs (=internal/backuplock=), so the snapshot and the walk agree.
69	73	- The host's restic credentials are append-only; the key that can

.gitbay/wiki/Architecture/09-Controls.org +1 −1

		@@ -99,6 +99,6 @@ chapter names of OWASP ASVS 4.0 where one fits.
99	99	\| Concurrency limit on git pack generation \| in place \| global, per-principal, bounded queue across SSH, HTTP and git:// (=internal/packlimit=); not in system SSH mode \|
100	100	\| Service hardening \| in place \| systemd sandboxing ([[file:03-Deployment.org][3]]) \|
101	101	\| Backups offsite and append-only \| in place \| restic with append-only credentials (documented) \|
102		\| Restore tested \| gap \| #259 \|
	102	\| Restore tested \| gap \| tooling in place (=admin restore-drill=, Admin wiki "Restore drill"); clean-host drill pending (#259) \|
103	103	\| Migrations validated before commit \| gap \| foreign-key check runs after commit (#261) \|
104	104	\| Signed, reviewed changes to production \| in place \| signed commits, =require-mr=, ff-only merges, clean-tree deploys \|

.gitbay/wiki/Architecture/10-Known-Gaps.org +1 −1

		@@ -10,7 +10,7 @@ what the 2026-09-27 review found; remove a row when its issue closes.
10	10
11	11	\| Issue \| Area \| Gap \| Severity \|
12	12	\|-------+------------------+-----------------------------------------------------------------------+----------\|
13		\| #259 \| Recovery \| No restore has been exercised; the drill is written (Admin wiki) and not yet run \| high \|
	13	\| #259 \| Recovery \| No restore has been exercised; the procedure and tooling (=admin restore-drill=, =backup --verify=) are in place, the clean-host drill is pending \| high \|
14	14	\| #260 \| CI network \| Builds share the runner's source address; no egress policy \| medium \|
15	15	\| #261 \| Various \| Migration foreign-key check after commit; three web writes bypass dispatch; documentation drift \| medium \|
16	16

CHANGELOG.org +12

		@@ -4,6 +4,18 @@ Versioning follows semver from v0.1.0. Database migrations run
4	4	automatically on daemon start; upgrade notes appear per release when
5	5	anything beyond "replace the binary and restart" is needed.
6	6
	7	* Unreleased
	8
	9	- =gitbayd admin backup --verify= also checks every release asset the
	10	database names against its recorded size and sha256, and every
	11	archived LFS object against its name (#259).
	12	- =gitbayd admin restore-drill <archive> --into <dir>= extracts a full
	13	archive into an empty directory, runs the =--verify= checks on the
	14	extracted copy, and prints the newest issue, comment and push it
	15	recovered and the elapsed time. The Admin wiki's Restore drill
	16	section lists what to restore, what to check and where to record it
	17	(#259).
	18
7	19	* v1.37.0 — 2026-09-29
8	20
9	21	Findings from the 2026-09-27 architecture review.

cmd/gitbayd/backup.go +115 −15

		@@ -4,6 +4,8 @@ import (
4	4	"archive/tar"
5	5	"bufio"
6	6	"compress/gzip"
	7	"crypto/sha256"
	8	"encoding/hex"
7	9	"errors"
8	10	"fmt"
9	11	"io"
		@@ -12,6 +14,7 @@ import (
12	14	"path"
13	15	"path/filepath"
14	16	"regexp"
	17	"strconv"
15	18	"strings"
16	19	"time"
17	20
		@@ -21,6 +24,7 @@ import (
21	24	"gitbay.org/gitbay/internal/backuplock"
22	25	"gitbay.org/gitbay/internal/config"
23	26	"gitbay.org/gitbay/internal/gitutil"
	27	"gitbay.org/gitbay/internal/lfs"
24	28	"gitbay.org/gitbay/internal/store"
25	29	)
26	30
		@@ -71,7 +75,7 @@ public keys and its name ends in .age. --verify then needs --identity
71	75	}
72	76	cmd.Flags().StringVar(&out, "out", "", "output archive path (default gitbay-backup-<utc timestamp>.tar.gz; .age is appended when [backup] age_recipients is set)")
73	77	cmd.Flags().BoolVar(&dbOnly, "db-only", false, "archive the database snapshot alone, without repositories")
74		cmd.Flags().StringVar(&verify, "verify", "", "check an archive instead of writing one: database integrity, its repositories against the archive's, and git connectivity of each")
	78	cmd.Flags().StringVar(&verify, "verify", "", "check an archive instead of writing one: database integrity, its repositories against the archive's, git connectivity of each, release assets and LFS object digests")
75	79	cmd.Flags().StringVar(&identity, "identity", "", "with --verify: an age identity file that opens an encrypted archive")
76	80	return cmd
77	81	}
		@@ -459,10 +463,24 @@ func addDir(tw *tar.Writer, path, name string) error {
459	463	// verifyBackup reads an archive back, decrypting it with identity when it
460	464	// is encrypted: the database snapshot must pass SQLite's integrity check,
461	465	// every repository it names must be in the archive, and each of those
462		// must pass git fsck --connectivity-only. A database-only archive is
	466	// must pass git fsck --connectivity-only; release assets must match the
	467	// database and LFS objects their names. A database-only archive is
463	468	// checked for integrity alone and says so. Repositories are extracted to
464	469	// a temporary directory for the check, so it needs free space for them.
465	470	func verifyBackup(path, identity string) error {
	471	tmp, err := os.MkdirTemp("", "gitbay-verify-")
	472	if err != nil {
	473	return err
	474	}
	475	defer os.RemoveAll(tmp)
	476	return checkArchive(path, identity, tmp, false)
	477	}
	478
	479	// checkArchive extracts the archive at path into dest and runs verify's
	480	// checks on it. With full unset it extracts only the database and the
	481	// repositories; with full set, every member, so dest is a restored
	482	// server.root. Alternates and commondir are left out either way.
	483	func checkArchive(path, identity, dest string, full bool) error {
466	484	f, err := os.Open(path)
467	485	if err != nil {
468	486	return err
		@@ -477,14 +495,11 @@ func verifyBackup(path, identity string) error {
477	495	return fmt.Errorf("%s: not a gzip archive: %w", path, err)
478	496	}
479	497	tr := tar.NewReader(gz)
480		tmp, err := os.MkdirTemp("", "gitbay-verify-")
481		if err != nil {
482		return err
483		}
484		defer os.RemoveAll(tmp)
485	498	dbPath := ""
486	499	inArchive := map[string]bool{}
487	500	members := 0
	501	lfsObjects := 0
	502	var badLFS []string
488	503	for {
489	504	h, err := tr.Next()
490	505	if err == io.EOF {
		@@ -496,15 +511,35 @@ func verifyBackup(path, identity string) error {
496	511	members++
497	512	switch {
498	513	case h.Name == "gitbay.db":
499		dbPath = filepath.Join(tmp, "gitbay.db")
	514	dbPath = filepath.Join(dest, "gitbay.db")
500	515	if err := extractTo(tr, dbPath); err != nil {
501	516	return fmt.Errorf("%s: extracting the database: %w", path, err)
502	517	}
503		case strings.HasPrefix(h.Name, "repos/"):
	518	case strings.HasPrefix(h.Name, "lfs/") && h.Typeflag == tar.TypeReg && lfs.OIDPat.MatchString(filepath.Base(h.Name)):
	519	// Objects are named by their sha256, so each is checked as it
	520	// streams past and none is extracted. Other names, such as an
	521	// upload's .upload-* staging file, are not objects.
	522	lfsObjects++
	523	if !filepath.IsLocal(h.Name) {
	524	return fmt.Errorf("%s: member %q leaves the archive root", path, h.Name)
	525	}
	526	sum := sha256.New()
	527	if full {
	528	err = extractTo(io.TeeReader(tr, sum), filepath.Join(dest, filepath.FromSlash(h.Name)))
	529	} else {
	530	_, err = io.Copy(sum, tr)
	531	}
	532	if err != nil {
	533	return fmt.Errorf("%s: reading %s: %w", path, h.Name, err)
	534	}
	535	if hex.EncodeToString(sum.Sum(nil)) != filepath.Base(h.Name) {
	536	badLFS = append(badLFS, h.Name)
	537	}
	538	case full \|\| strings.HasPrefix(h.Name, "repos/"):
504	539	trimmed := strings.TrimSuffix(h.Name, "/")
505	540	// repos/<owner>/<name>.git/HEAD marks one repository present.
506	541	parts := strings.Split(trimmed, "/")
507		if len(parts) == 4 && parts[3] == "HEAD" && strings.HasSuffix(parts[2], ".git") {
	542	if len(parts) == 4 && parts[0] == "repos" && parts[3] == "HEAD" && strings.HasSuffix(parts[2], ".git") {
508	543	inArchive[parts[1]+"/"+strings.TrimSuffix(parts[2], ".git")] = true
509	544	}
510	545	if !filepath.IsLocal(trimmed) {
		@@ -513,7 +548,7 @@ func verifyBackup(path, identity string) error {
513	548	if borrowsObjects(trimmed) {
514	549	continue
515	550	}
516		dest := filepath.Join(tmp, filepath.FromSlash(trimmed))
	551	dest := filepath.Join(dest, filepath.FromSlash(trimmed))
517	552	switch h.Typeflag {
518	553	case tar.TypeDir:
519	554	// The archive's directory modes do not matter to fsck, and
		@@ -572,19 +607,84 @@ func verifyBackup(path, identity string) error {
572	607	if extra > 0 {
573	608	fmt.Printf("%d repositories in the archive that the database does not name (created after the snapshot)\n", extra)
574	609	}
	610	var failed []error
575	611	var broken []string
576	612	for _, r := range repos {
577		dir := filepath.Join(tmp, "repos", r.OwnerName, r.Name+".git")
	613	dir := filepath.Join(dest, "repos", r.OwnerName, r.Name+".git")
578	614	if err := gitutil.FsckConnectivity(dir); err != nil {
579	615	fmt.Fprintf(os.Stderr, "%s: %v\n", r.Path(), err)
580	616	broken = append(broken, r.Path())
581	617	}
582	618	}
583	619	if len(broken) > 0 {
584		return fmt.Errorf("%s: %d repositories fail the connectivity check: %s", path, len(broken), strings.Join(broken, ", "))
	620	failed = append(failed, fmt.Errorf("%s: %d repositories fail the connectivity check: %s", path, len(broken), strings.Join(broken, ", ")))
	621	} else {
	622	fmt.Printf("connectivity ok on %d repositories\n", len(repos))
585	623	}
586		fmt.Printf("connectivity ok on %d repositories\n", len(repos))
587		return nil
	624	assets, badAssets, err := checkReleaseAssets(st, repos, dest)
	625	if err != nil {
	626	return err
	627	}
	628	if len(badAssets) > 0 {
	629	failed = append(failed, fmt.Errorf("%s: %d release assets missing or not matching their digest: %s", path, len(badAssets), strings.Join(badAssets, ", ")))
	630	} else {
	631	fmt.Printf("release assets ok: %d\n", assets)
	632	}
	633	// LFS objects are named by pointer files in git history, not by the
	634	// database, so this checks the archived objects' digests and not that
	635	// every pointer has its object. With [lfs] root outside server.root
	636	// the archive carries none.
	637	if len(badLFS) > 0 {
	638	failed = append(failed, fmt.Errorf("%s: %d LFS objects do not match their digest: %s", path, len(badLFS), strings.Join(badLFS, ", ")))
	639	} else {
	640	fmt.Printf("LFS objects ok: %d\n", lfsObjects)
	641	}
	642	return errors.Join(failed...)
	643	}
	644
	645	// checkReleaseAssets checks that every release asset the database names
	646	// is under root, extracted, with its recorded size and sha256. It
	647	// returns how many the database names and those that fail.
	648	func checkReleaseAssets(st *store.Store, repos []store.Repo, root string) (int, []string, error) {
	649	byID := map[int64]store.Repo{}
	650	for _, r := range repos {
	651	byID[r.ID] = r
	652	}
	653	rows, err := st.DB.Query(`SELECT rl.repo_id, a.release_id, a.name, a.size, a.sha256
	654	FROM release_assets a JOIN releases rl ON rl.id = a.release_id
	655	ORDER BY rl.repo_id, a.release_id, a.name`)
	656	if err != nil {
	657	return 0, nil, err
	658	}
	659	defer rows.Close()
	660	n := 0
	661	var bad []string
	662	for rows.Next() {
	663	var repoID, relID, size int64
	664	var name, want string
	665	if err := rows.Scan(&repoID, &relID, &name, &size, &want); err != nil {
	666	return 0, nil, err
	667	}
	668	n++
	669	r, ok := byID[repoID]
	670	owner := r.Path()
	671	if !ok {
	672	owner = fmt.Sprintf("repository %d", repoID)
	673	}
	674	label := fmt.Sprintf("%s release %d %s", owner, relID, name)
	675	f, err := os.Open(filepath.Join(root, "repos", r.OwnerName, r.Name+".git", "gitbay-releases", strconv.FormatInt(relID, 10), name))
	676	if err != nil {
	677	bad = append(bad, label)
	678	continue
	679	}
	680	sum := sha256.New()
	681	got, err := io.Copy(sum, f)
	682	f.Close()
	683	if err != nil \|\| got != size \|\| hex.EncodeToString(sum.Sum(nil)) != want {
	684	bad = append(bad, label)
	685	}
	686	}
	687	return n, bad, rows.Err()
588	688	}
589	689
590	690	// borrowsObjects reports an archive member that would point git at

cmd/gitbayd/backup_test.go +75

		@@ -3,6 +3,8 @@ package main
3	3	import (
4	4	"archive/tar"
5	5	"compress/gzip"
	6	"crypto/sha256"
	7	"encoding/hex"
6	8	"errors"
7	9	"io"
8	10	"io/fs"
		@@ -10,6 +12,7 @@ import (
10	12	"os/exec"
11	13	"path/filepath"
12	14	"sort"
	15	"strconv"
13	16	"strings"
14	17	"testing"
15	18	"time"
		@@ -840,3 +843,75 @@ func TestVerifyIgnoresCommondir(t *testing.T) {
840	843	t.Fatalf("verify of a repository whose objects are only in its commondir: %v", err)
841	844	}
842	845	}
	846
	847	// verify checks each release asset the database names against its
	848	// recorded digest, and each archived LFS object against its name.
	849	func TestVerifyChecksReleaseAssetsAndLFS(t *testing.T) {
	850	cfg := testConfig(t)
	851	root := cfg.Server.Root
	852	st, err := openStore(cfg)
	853	if err != nil {
	854	t.Fatal(err)
	855	}
	856	uid, err := st.CreateUser("krz", false)
	857	if err != nil {
	858	t.Fatal(err)
	859	}
	860	rid, err := st.CreateRepo("user", uid, "thing", "public")
	861	if err != nil {
	862	t.Fatal(err)
	863	}
	864	relID, err := st.CreateRelease(rid, "v1", "v1", "", uid, "md")
	865	if err != nil {
	866	t.Fatal(err)
	867	}
	868	asset := []byte("binary\n")
	869	sum := sha256.Sum256(asset)
	870	if err := st.AddReleaseAsset(relID, "tool", int64(len(asset)), hex.EncodeToString(sum[:])); err != nil {
	871	t.Fatal(err)
	872	}
	873	st.Close()
	874	dir := filepath.Join(root, "repos", "krz", "thing.git")
	875	gitIn(t, root, "init", "-q", "--bare", dir)
	876	assetFile := filepath.Join(dir, "gitbay-releases", strconv.FormatInt(relID, 10), "tool")
	877	writeFile(t, assetFile, asset)
	878	obj := []byte("large\n")
	879	oid := sha256.Sum256(obj)
	880	o := hex.EncodeToString(oid[:])
	881	writeFile(t, filepath.Join(root, "lfs", o[:2], o[2:4], o), obj)
	882	// An upload in progress stages a temporary file beside the objects.
	883	writeFile(t, filepath.Join(root, "lfs", o[:2], o[2:4], ".upload-123"), []byte("partial"))
	884
	885	good := filepath.Join(t.TempDir(), "good.tar.gz")
	886	if err := runBackup(cfg, good, false); err != nil {
	887	t.Fatal(err)
	888	}
	889	if err := verifyBackup(good, ""); err != nil {
	890	t.Fatalf("intact archive: %v", err)
	891	}
	892
	893	writeFile(t, assetFile, []byte("tampered\n"))
	894	wrong := strings.Repeat("0", 64)
	895	writeFile(t, filepath.Join(root, "lfs", "00", "00", wrong), obj)
	896	bad := filepath.Join(t.TempDir(), "bad.tar.gz")
	897	if err := runBackup(cfg, bad, false); err != nil {
	898	t.Fatal(err)
	899	}
	900	err = verifyBackup(bad, "")
	901	if err == nil \|\| !strings.Contains(err.Error(), "krz/thing release") \|\| !strings.Contains(err.Error(), wrong) {
	902	t.Fatalf("archive with a bad asset and LFS object: %v", err)
	903	}
	904	if strings.Contains(err.Error(), o) \|\| strings.Contains(err.Error(), ".upload-") {
	905	t.Fatalf("intact LFS object or upload staging file reported: %v", err)
	906	}
	907	}
	908
	909	func writeFile(t *testing.T, p string, b []byte) {
	910	t.Helper()
	911	if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil {
	912	t.Fatal(err)
	913	}
	914	if err := os.WriteFile(p, b, 0o644); err != nil {
	915	t.Fatal(err)
	916	}
	917	}

cmd/gitbayd/main.go +1

		@@ -455,6 +455,7 @@ func adminCmd() *cobra.Command {
455	455	hostCmd("runners [--json]", "runner accounts: last poll, scope, the build each holds", "admin", "runners"),
456	456	auditCmd,
457	457	backupCmd(),
	458	restoreDrillCmd(),
458	459	secretsCmd(),
459	460	gcCmd(),
460	461	adminMigrateCommitRefsCmd(),

cmd/gitbayd/restoredrill.go added +107

		@@ -0,0 +1,107 @@
	1	package main
	2
	3	import (
	4	"errors"
	5	"fmt"
	6	"io/fs"
	7	"os"
	8	"path/filepath"
	9	"strings"
	10	"time"
	11
	12	"github.com/spf13/cobra"
	13
	14	"gitbay.org/gitbay/internal/store"
	15	)
	16
	17	// restoreDrillCmd rehearses the archive half of a restore: extract a
	18	// full archive into an empty directory, run verify's checks on what was
	19	// extracted, and report the elapsed time and the newest activity the
	20	// restored database holds.
	21	func restoreDrillCmd() *cobra.Command {
	22	var into, identity string
	23	cmd := &cobra.Command{
	24	Use: "restore-drill <archive> --into <dir>",
	25	Short: "restore a full archive into an empty directory, verify it, report elapsed time and the newest recovered activity",
	26	Long: `Extracts every member of a full backup archive into --into, which must
	27	be empty or absent, and runs the checks of backup --verify on the
	28	extracted copy: database integrity, every repository present and
	29	passing git fsck --connectivity-only, release assets and LFS object
	30	digests. It then prints the newest issue, comment and push in the
	31	restored database, which is the recovery point, and the elapsed time.
	32
	33	The result is a server.root a gitbayd can be pointed at. The archive
	34	does not carry server.secret_key_file or config.toml; the Admin wiki's
	35	Restore drill section covers those and the offsite path.`,
	36	Args: cobra.ExactArgs(1),
	37	RunE: func(cmd *cobra.Command, args []string) error {
	38	if into == "" {
	39	return errors.New("--into <dir> is required")
	40	}
	41	return restoreDrill(args[0], identity, into)
	42	},
	43	}
	44	cmd.Flags().StringVar(&into, "into", "", "empty or absent directory to restore into")
	45	cmd.Flags().StringVar(&identity, "identity", "", "an age identity file that opens an encrypted archive")
	46	return cmd
	47	}
	48
	49	func restoreDrill(archive, identity, into string) error {
	50	start := time.Now()
	51	ents, err := os.ReadDir(into)
	52	switch {
	53	case errors.Is(err, fs.ErrNotExist):
	54	if err := os.MkdirAll(into, 0o700); err != nil {
	55	return err
	56	}
	57	case err != nil:
	58	return err
	59	case len(ents) > 0:
	60	return fmt.Errorf("%s is not empty; restore into an empty or absent directory", into)
	61	}
	62	checkErr := checkArchive(archive, identity, into, true)
	63	if ents, err := os.ReadDir(into); err == nil {
	64	var names []string
	65	for _, e := range ents {
	66	names = append(names, e.Name())
	67	}
	68	fmt.Printf("restored into %s: %s\n", into, strings.Join(names, " "))
	69	}
	70	// store.Open would create a missing database.
	71	db := filepath.Join(into, "gitbay.db")
	72	if _, err := os.Stat(db); err == nil {
	73	if err := printNewest(db); err != nil {
	74	checkErr = errors.Join(checkErr, err)
	75	}
	76	}
	77	fmt.Printf("elapsed %s\n", time.Since(start).Round(100*time.Millisecond))
	78	return checkErr
	79	}
	80
	81	// newest are the recovered timestamps a drill records.
	82	var newest = []struct{ label, query string }{
	83	{"issue", "SELECT MAX(created_at) FROM issues"},
	84	{"issue comment", "SELECT MAX(created_at) FROM issue_comments"},
	85	{"merge request comment", "SELECT MAX(created_at) FROM mr_comments"},
	86	{"push", "SELECT MAX(created_at) FROM events WHERE kind = 'push'"},
	87	}
	88
	89	func printNewest(db string) error {
	90	st, err := store.Open(db)
	91	if err != nil {
	92	return err
	93	}
	94	defer st.Close()
	95	for _, n := range newest {
	96	var at *string
	97	if err := st.DB.QueryRow(n.query).Scan(&at); err != nil {
	98	return fmt.Errorf("newest %s: %w", n.label, err)
	99	}
	100	v := "none"
	101	if at != nil {
	102	v = *at
	103	}
	104	fmt.Printf("newest %s: %s\n", n.label, v)
	105	}
	106	return nil
	107	}

cmd/gitbayd/restoredrill_test.go added +63

		@@ -0,0 +1,63 @@
	1	package main
	2
	3	import (
	4	"os"
	5	"path/filepath"
	6	"strings"
	7	"testing"
	8	)
	9
	10	// A drill restores a full archive into an empty directory as a usable
	11	// root, verifies it, and refuses a directory that already holds files.
	12	func TestRestoreDrill(t *testing.T) {
	13	cfg := testConfig(t)
	14	root := cfg.Server.Root
	15	st, err := openStore(cfg)
	16	if err != nil {
	17	t.Fatal(err)
	18	}
	19	uid, err := st.CreateUser("krz", false)
	20	if err != nil {
	21	t.Fatal(err)
	22	}
	23	rid, err := st.CreateRepo("user", uid, "thing", "public")
	24	if err != nil {
	25	t.Fatal(err)
	26	}
	27	if _, err := st.CreateIssue(rid, uid, "bug", "", "md"); err != nil {
	28	t.Fatal(err)
	29	}
	30	if err := st.RecordEvent(rid, uid, "push", "{}"); err != nil {
	31	t.Fatal(err)
	32	}
	33	st.Close()
	34	work := t.TempDir()
	35	gitIn(t, work, "init", "-q", "-b", "main")
	36	writeFile(t, filepath.Join(work, "a.txt"), []byte("a\n"))
	37	gitIn(t, work, "add", "a.txt")
	38	gitIn(t, work, "commit", "-q", "-m", "one")
	39	gitIn(t, work, "clone", "-q", "--bare", work, filepath.Join(root, "repos", "krz", "thing.git"))
	40	writeFile(t, filepath.Join(root, "ssh", "host_ed25519"), []byte("key\n"))
	41
	42	archive := filepath.Join(t.TempDir(), "b.tar.gz")
	43	if err := runBackup(cfg, archive, false); err != nil {
	44	t.Fatal(err)
	45	}
	46	into := filepath.Join(t.TempDir(), "restored")
	47	if err := restoreDrill(archive, "", into); err != nil {
	48	t.Fatal(err)
	49	}
	50	for _, p := range []string{"gitbay.db", "ssh/host_ed25519", "repos/krz/thing.git/HEAD"} {
	51	if _, err := os.Stat(filepath.Join(into, p)); err != nil {
	52	t.Errorf("restored root lacks %s: %v", p, err)
	53	}
	54	}
	55	if got := gitIn(t, filepath.Join(into, "repos", "krz", "thing.git"), "log", "--format=%s", "main"); got != "one" {
	56	t.Errorf("restored log %q", got)
	57	}
	58
	59	err = restoreDrill(archive, "", into)
	60	if err == nil \|\| !strings.Contains(err.Error(), "not empty") {
	61	t.Fatalf("drill into a non-empty directory: %v", err)
	62	}
	63	}

backup: verify release assets and LFS; admin restore-drill !518