backup: hold moves off, refs before objects, directory entries, verify connectivity !504

merged merged by cmc on 2026-09-28 23:09 UTC · krz/gitbay:backup-verify-lock into main

23 files changed, +1394 −115

Layout: unified · split

.gitbay/wiki/Admin.org +81 −10

	@@ -444,12 +444,46 @@ One archive: a consistent SQLite snapshot (taken before the
444	repositories are read, so the database never references objects the	444	repositories are read, so the database never references objects the
445	archive missed), every repository, and the SSH host keys. Excluded:	445	archive missed), every repository, and the SSH host keys. Excluded:
446	hook socket, regenerated hook scripts, WAL files. Safe to run against a	446	hook socket, regenerated hook scripts, WAL files. Safe to run against a
447	live daemon.	447	live daemon. =--out= must be outside =server.root=, or the next full
		448	backup would carry the archive. Each run removes the snapshot
		449	directories (=.gitbay-snap-=) and temporary archives (=..tmp-*=) a
		450	killed run left beside its archive once they are a day old, and prints
		451	each one it removes.
448		452
449	=--verify= reads an archive back: the snapshot must pass SQLite's	453	=--verify= reads an archive back: the snapshot must pass SQLite's
450	integrity check, and every repository the snapshot names must be in the	454	integrity check, every repository the snapshot names must be in the
451	archive. A database-only archive is checked for integrity and says so.	455	archive, and each must pass =git fsck --connectivity-only=. It
452	Exit is non-zero on damage or a missing repository.	456	extracts the repositories to a temporary directory for that, so it
		457	needs free space the size of the repositories. A database-only archive
		458	is checked for integrity and says so. Exit is non-zero on damage, a
		459	missing repository or a missing object.
		460
		461	A full backup holds =<root>/backup.lock= from its database snapshot to
		462	its last repository. While it runs, =repo delete=, =repo rename=,
		463	=repo transfer=, =admin repo delete=, =org rename=, =admin mr prune=
		464	and =gitbayd admin gc= refuse with "a backup is running"; retry when it
		465	finishes. Database-only backups take no lock. A pack that git's own
		466	automatic gc removes during the walk is skipped; each repository's refs
		467	are archived before its objects, so the refs still find their objects,
		468	and =--verify= reports it if one does not.
		469
		470	Verifying an archive of unknown origin: run it as an unprivileged
		471	user. The connectivity check runs git with =--git-dir= on each
		472	extracted repository, so a directory that is not a repository fails
		473	rather than git checking an enclosing one. =objects/info/alternates=
		474	and a =commondir= directly in a =*.git= directory are not extracted,
		475	so an archive cannot use either to have git read another repository's
		476	objects or refs on the host. Git still reads each archived
		477	repository's own =config=.
		478
		479	Archives carry a directory entry for every directory, including an
		480	empty one, so a bare repository whose refs are all packed restores as
		481	a repository. Archives written before this release do not: extracting
		482	one can leave a repository's =refs/= directory missing, which stops
		483	git from recognizing it as a repository at all. =gitbayd admin backup
		484	--verify <archive>= names the repositories this affects; the fix is
		485	=mkdir -p <root>/repos/<owner>/<name>.git/refs= for each one, after
		486	which it opens normally.
453		487
454	With =[backup] age_recipients= set the archive is =<name>.tar.gz.age=	488	With =[backup] age_recipients= set the archive is =<name>.tar.gz.age=
455	and =--verify= needs the private key:	489	and =--verify= needs the private key:
	@@ -464,8 +498,11 @@ restic credentials), so verifying an encrypted archive happens there
464	or on a restore host.	498	or on a restore host.
465		499
466	Restore: extract into an empty directory, point =server.root= at it,	500	Restore: extract into an empty directory, point =server.root= at it,
467	start gitbayd. Host keys are preserved, so clients keep their	501	restore =server.secret_key_file= from its own copy (mode 0600, owned
468	known_hosts entries; hooks regenerate at startup.	502	by the account gitbayd runs as), then start gitbayd. No archive carries
		503	the key file, and without it gitbayd refuses to start. Host keys are
		504	preserved, so clients keep their known_hosts entries; hooks regenerate
		505	at startup.
469		506
470	** Schedule and recovery point	507	** Schedule and recovery point
471		508
	@@ -493,8 +530,10 @@ too.
493	** Offsite copies	530	** Offsite copies
494		531
495	bay1 also takes a nightly restic snapshot of =/var/lib/gitbay= and	532	bay1 also takes a nightly restic snapshot of =/var/lib/gitbay= and
496	=/var/lib/gitbay-stage= (the staged database copy) to an S3 bucket at	533	=/var/lib/gitbay-stage= (a database copy and =config.toml=, staged
497	Scaleway, with a key that can only add snapshots. The key that can	534	there) to an S3 bucket at Scaleway, with a key that can only add
		535	snapshots. Nothing else under =/etc/gitbay= is in it: not the secret
		536	key file, not =apns.p8=. The key that can
498	remove them lives on the operator's machine, in	537	remove them lives on the operator's machine, in
499	=~/.config/gitbay/offsite.env=, and never on bay1: a compromised host	538	=~/.config/gitbay/offsite.env=, and never on bay1: a compromised host
500	cannot destroy its own history. Forgetting, pruning and rewriting all	539	cannot destroy its own history. Forgetting, pruning and rewriting all
	@@ -543,7 +582,10 @@ each value prefixed with the id of the key that sealed it
543	=server.root=, and therefore in neither the local archives nor the	582	=server.root=, and therefore in neither the local archives nor the
544	main restic repository. It must be copied off the host separately;	583	main restic repository. It must be copied off the host separately;
545	without it a restored database's secrets cannot be opened, and	584	without it a restored database's secrets cannot be opened, and
546	gitbayd refuses to start against them.	585	gitbayd refuses to start against them. A separate restic repository
		586	for it and =apns.p8= is planned (runbook D of the data-at-rest plan)
		587	and not yet in place, so today the only off-host copy is one the
		588	operator makes by hand after =init= and after every =rotate=.
547		589
548	#+begin_src sh	590	#+begin_src sh
549	gitbayd admin secrets init # once; deploy/install.sh does it on first install	591	gitbayd admin secrets init # once; deploy/install.sh does it on first install
	@@ -554,6 +596,8 @@ gitbayd admin secrets rotate # new key, reseal, retire the old one (as root)
554	- Missing file: every gitbayd process that opens the database refuses	596	- Missing file: every gitbayd process that opens the database refuses
555	to run and names the path, including =serve= and, in system mode,	597	to run and names the path, including =serve= and, in system mode,
556	=authorized-keys=. =migrate= does not need it.	598	=authorized-keys=. =migrate= does not need it.
		599	- Backups: =admin backup= and =admin backup --verify= do not need the
		600	key file; a restore does.
557	- Wrong key: =serve= stops at startup naming the first row that does	601	- Wrong key: =serve= stops at startup naming the first row that does
558	not open; =secrets check= does the same without starting anything.	602	not open; =secrets check= does the same without starting anything.
559	- Upgrade: the first start after the upgrade seals every value still	603	- Upgrade: the first start after the upgrade seals every value still
	@@ -562,10 +606,37 @@ gitbayd admin secrets rotate # new key, reseal, retire the old one (as root)
562	transaction, then removes the old keys. Run it as root, since it	606	transaction, then removes the old keys. Run it as root, since it
563	replaces the key file in =/etc/gitbay=; the file keeps its owner.	607	replaces the key file in =/etc/gitbay=; the file keeps its owner.
564	The daemon re-reads the file when it changes, so it needs no	608	The daemon re-reads the file when it changes, so it needs no
565	restart. Copy the new file off the host afterwards.	609	restart. Copy the new file off the host afterwards. =init= and
		610	=rotate= hold an flock on =<key file>.lock= while they run, so a
		611	second run waits for the first.
566	- Push devices are looked up by the SHA-256 of their token	612	- Push devices are looked up by the SHA-256 of their token
567	(=push_devices.token_hash=), since two seals of one token differ.	613	(=push_devices.token_hash=), since two seals of one token differ.
568		614
		615	** Restore drill
		616
		617	A restore onto a clean host, run quarterly and after any change to the
		618	backup code (=cmd/gitbayd/backup.go=, the offsite job), and recorded
		619	below. The disaster it rehearses is losing bay1, so the local archives
		620	are gone with it and the sources are the main offsite restic
		621	repository (repositories, LFS, the staged database, =config.toml=),
		622	the off-host copy of =secret.key= and =apns.p8= (a keys repository once
		623	runbook D creates it; until then the operator's hand-made copy), and
		624	the operator's password manager (=offsite.env=, the keys repository's
		625	password and token once it exists, =backup-identity.txt=). The steps are in the data-at-rest
		626	plan's operator runbook
		627	(=docs/plans/2026-09-27-data-at-rest-and-backup.md=).
		628
		629	Time to service runs from the clean host's first root login to the
		630	first successful =git clone= over SSH from it. The recovery point is
		631	the time of the newest restic snapshot restored.
		632
		633	No drill has been run yet; the procedure above is written but
		634	unexercised, and #259 stays open until the first row below is
		635	recorded.
		636
		637	\| Date \| Host \| Snapshot restored (UTC) \| Time to service \| DB integrity \| Connectivity \| LFS \| Release assets \| Host key \| Secrets \| Notes \|
		638	\|------+------+-------------------------+-----------------+--------------+--------------+-----+----------------+----------+---------+-------\|
		639
569	* Upgrades	640	* Upgrades
570		641
571	Replace the binary, restart the unit. Migrations apply automatically and	642	Replace the binary, restart the unit. Migrations apply automatically and

.gitbay/wiki/Architecture/06-Data-and-Cryptography.org +1 −1

	@@ -45,7 +45,7 @@ throttling (=internal/sshd/sshd.go=).
45	\| API tokens, sessions, login links, email codes, invites \| SHA-256 of a 256-bit random value; the value is shown once and never stored (=internal/store/sessions.go=) \|	45	\| API tokens, sessions, login links, email codes, invites \| SHA-256 of a 256-bit random value; the value is shown once and never stored (=internal/store/sessions.go=) \|
46	\| CI secrets, webhook secrets, mirror tokens, APNs device tokens \| AES-256-GCM under a key file outside the database and outside =server.root=; additional data binds table, column and row (=internal/seal=, =internal/store/secrets.go=) \|	46	\| CI secrets, webhook secrets, mirror tokens, APNs device tokens \| AES-256-GCM under a key file outside the database and outside =server.root=; additional data binds table, column and row (=internal/seal=, =internal/store/secrets.go=) \|
47	\| SQLite file \| mode 0640, directory 0750 \|	47	\| SQLite file \| mode 0640, directory 0750 \|
48	\| Backups \| local archives age-encrypted when =[backup] age_recipients= is set; restic encrypts the offsite copy; neither carries the secret key file, whose offsite copy is a separate keys repository \|	48	\| Backups \| local archives age-encrypted when =[backup] age_recipients= is set; restic encrypts the offsite copy; neither carries the secret key file, which is copied off the host by hand until a separate keys repository is set up \|
49	\| Disk \| no application-level encryption; any disk encryption is the host's \|	49	\| Disk \| no application-level encryption; any disk encryption is the host's \|
50		50
51	The database file or a backup read by anyone other than the =gitbay=	51	The database file or a backup read by anyone other than the =gitbay=

.gitbay/wiki/Architecture/08-Operations.org +12 −7

 | Database only   | hourly   | 48   | SQLite snapshot; age-encrypted when =[backup] age_recipients= is set |
 | Offsite (restic)| nightly  | per prune policy | =/var/lib/gitbay= and a staged database copy, to object storage |
-- The database snapshot is taken before repositories are read, so a
+- The database snapshot is taken before repositories are read, and
-  push during the backup leaves only unreferenced objects
+  each repository's HEAD, refs/ and packed-refs are archived before its
+  objects, so every archived ref finds the objects it reaches, unless
+  git's own automatic gc after a push repacks during the walk; the
+  archive can then miss objects, and =--verify= reports it. A push
+  during the backup is missing or present as unreferenced objects
   (=cmd/gitbayd/backup.go=).
 - Excluded: WAL files, the hook socket, askpass scripts, generated
   hooks.
-- =gitbayd admin backup --verify= checks SQLite integrity and that every
+- =gitbayd admin backup --verify= checks SQLite integrity, that every
-  repository the database names is present (=backup.go=). It does
+  repository the database names is present, and =git fsck
-  not check git object connectivity.
+  --connectivity-only= on each (=backup.go=).
+- Repository deletes, renames and transfers refuse while a full backup
+  runs (=internal/backuplock=), so the snapshot and the walk agree.
 - The host's restic credentials are append-only; the key that can
   delete or prune snapshots is held off the host, so a compromised host
   cannot destroy its own history (documented: Admin wiki).
 - Recovery point: about one hour for database-only data (issues, merge
   requests, reviews), one day for repositories.
-- Recovery time: not measured. No restore onto a clean host has been
+- Recovery time: see the Admin wiki's Restore drill table.
-  recorded (#259).
 Restore procedure: extract the archive into an empty directory, point
 =server.root= at it, start =gitbayd=; hooks regenerate and the host key

.gitbay/wiki/Architecture/10-Known-Gaps.org +1 −1

	@@ -10,7 +10,7 @@ what the 2026-09-27 review found; remove a row when its issue closes.
10		10
11	\| Issue \| Area \| Gap \| Severity \|	11	\| Issue \| Area \| Gap \| Severity \|
12	\|-------+------------------+-----------------------------------------------------------------------+----------\|	12	\|-------+------------------+-----------------------------------------------------------------------+----------\|
13	\| #259 \| Recovery \| No restore has been exercised; verification does not check git connectivity \| high \|	13	\| #259 \| Recovery \| No restore has been exercised; the drill is written (Admin wiki) and not yet run \| high \|
14	\| #260 \| CI network \| Builds share the runner's source address; no egress policy \| medium \|	14	\| #260 \| CI network \| Builds share the runner's source address; no egress policy \| medium \|
15	\| #261 \| Various \| Migration foreign-key check after commit; three web writes bypass dispatch; documentation drift \| medium \|	15	\| #261 \| Various \| Migration foreign-key check after commit; three web writes bypass dispatch; documentation drift \| medium \|
16	\| #262 \| Availability \| No limit on concurrent git pack generation \| high \|	16	\| #262 \| Availability \| No limit on concurrent git pack generation \| high \|

.gitbay/wiki/Threat-Model.org +8 −3

   pixel against a viewer. A profile is the wider surface of the two: it
   is linked from every commit and issue its owner touches. Documented;
   proxying is future work.
-- Backups are consistent per the DB-snapshot-first ordering but are not a
+- Backups snapshot the database first, and repository deletes and
-  single atomic snapshot; a few orphaned git objects are possible and
+  moves wait out a full backup. Each repository's refs are archived
-  harmless (see [[Admin]]).
+  before its objects, so every archived ref finds the objects it
+  reaches, unless git's own automatic gc after a push repacks during
+  the walk: the archive can then miss objects, and =--verify= reports
+  it. A push during a backup may be missing from the archive, or
+  present as objects no archived ref names, and a repository's refs may
+  be newer than the database snapshot (see [[Admin]]).
 - The audit log lives in the database the daemon writes, so anyone with
   the daemon user's access can change it. The hash chain makes an edited
   or removed row show as a break under =gitbayd admin audit verify=,

CHANGELOG.org +51 −16

	@@ -29,6 +29,29 @@ timeout (#256, #257, #276, #277).
29	=--scope full=. A script that mints a token and then writes with it	29	=--scope full=. A script that mints a token and then writes with it
30	must add =--scope full=. Existing tokens keep their scope.	30	must add =--scope full=. Existing tokens keep their scope.
31		31
		32	Upgrade note. Upgrade the instance before the CLI: an older server
		33	refuses the CLI's leading =--path== argument as an unknown command,
		34	for the eighteen commands whose CLI path differs from the registry's
		35	(the =gitbay auth ...= commands and =repo topics list=).
		36
		37	Upgrade note. gitbayd needs =server.secret_key_file= (default
		38	=/etc/gitbay/secret.key=) and refuses to start without it. Before
		39	replacing the binary, run =gitbayd admin secrets init= as root and
		40	=chown gitbay:gitbay /etc/gitbay/secret.key= (=deploy/install.sh= does
		41	both when the file is missing). The first start seals the stored
		42	secrets. Back the key file up separately: =admin backup= archives do
		43	not carry it (see the Admin wiki, "Secret key"). Downgrading to an
		44	earlier release after values are sealed is not supported: an older
		45	gitbayd reads a sealed value's =gbs1:...= prefix as the literal
		46	secret.
		47
		48	Upgrade note. Archives now carry a directory entry for every
		49	directory, so a bare repository whose refs are all packed restores as
		50	a repository. An archive written before this release lacks those
		51	entries; if extracting one leaves a repository's =refs/= directory
		52	missing, =gitbayd admin backup --verify <archive>= names it, and
		53	=mkdir -p <root>/repos/<owner>/<name>.git/refs= fixes it.
		54
32	- A token with a =--ttl= is refused on every command that creates a	55	- A token with a =--ttl= is refused on every command that creates a
33	credential: tokens, keys, deploy keys, runner keys, login links,	56	credential: tokens, keys, deploy keys, runner keys, login links,
34	invites, accounts and verified addresses (#257).	57	invites, accounts and verified addresses (#257).
	@@ -85,10 +108,6 @@ must add =--scope full=. Existing tokens keep their scope.
85	separately (#275).	108	separately (#275).
86	- Audit retention deletes by id, up to the newest row older than the	109	- Audit retention deletes by id, up to the newest row older than the
87	retention, so a clock step back cannot leave a gap in the chain (#275).	110	retention, so a clock step back cannot leave a gap in the chain (#275).
88	Upgrade note. Upgrade the instance before the CLI: an older server
89	refuses the CLI's leading =--path== argument as an unknown command,
90	for the eighteen commands whose CLI path differs from the registry's
91	(the =gitbay auth ...= commands and =repo topics list=).
92	- =dashboard= and =feed= print activity as sentences	111	- =dashboard= and =feed= print activity as sentences
93	(=cmc opened issue krz/gitbay#12=) instead of raw event payloads,	112	(=cmc opened issue krz/gitbay#12=) instead of raw event payloads,
94	and a labelled event names its labels there and on the web feed. An	113	and a labelled event names its labels there and on the web feed. An
	@@ -137,18 +156,6 @@ for the eighteen commands whose CLI path differs from the registry's
137	previous" link on each revision after the first, so a reviewer whose	156	previous" link on each revision after the first, so a reviewer whose
138	approval a force-push staled can see what changed without leaving the	157	approval a force-push staled can see what changed without leaving the
139	browser (#269).	158	browser (#269).
140	- Untrusted builds (merge requests from forks) get a fresh HOME removed after the build and no secrets; trusted builds keep a per-repository home under =<workdir>/trusted-home=. Deploy gitbayd before the runner; the old shared homes under the runner's workdir can be deleted. (#255)
141	- =status set= refuses =ci/= contexts, which belong to the instance's builds. Build results are reused only from trusted builds on the same image. =repo settings require-contexts= names status contexts that must report green; setting any turns require-checks on, and one not yet reported counts as pending. (#258)
142	Upgrade note. gitbayd needs =server.secret_key_file= (default
143	=/etc/gitbay/secret.key=) and refuses to start without it. Before
144	replacing the binary, run =gitbayd admin secrets init= as root and
145	=chown gitbay:gitbay /etc/gitbay/secret.key= (=deploy/install.sh= does
146	both when the file is missing). The first start seals the stored
147	secrets. Back the key file up separately: =admin backup= archives do
148	not carry it (see the Admin wiki, "Secret key"). Downgrading to an
149	earlier release after values are sealed is not supported: an older
150	gitbayd reads a sealed value's =gbs1:...= prefix as the literal
151	secret.
152	- CI secrets, webhook secrets, mirror tokens and push device tokens are	159	- CI secrets, webhook secrets, mirror tokens and push device tokens are
153	stored sealed with AES-256-GCM (#273). =gitbayd admin secrets	160	stored sealed with AES-256-GCM (#273). =gitbayd admin secrets
154	init\|rotate\|check=.	161	init\|rotate\|check=.
	@@ -180,6 +187,34 @@ secret.
180	- =gitbayd admin backup= encrypts archives to =[backup] age_recipients=	187	- =gitbayd admin backup= encrypts archives to =[backup] age_recipients=
181	when set (#274); =--verify= takes =--identity <file>=. Archive names	188	when set (#274); =--verify= takes =--identity <file>=. Archive names
182	gain =.age=; the shipped backup scripts and monitor match both.	189	gain =.age=; the shipped backup scripts and monitor match both.
		190	- A full backup holds =<root>/backup.lock= from its database snapshot
		191	to its last repository; =repo delete=, =repo rename=, =repo
		192	transfer=, =admin repo delete= and =org rename= refuse with "a
		193	backup is running" while it runs. =--verify= now also runs =git
		194	fsck --connectivity-only= on each archived repository and names any
		195	that fail. (#259)
		196	- A full backup archives each repository's HEAD, =refs/= and
		197	=packed-refs= before its objects, so a push during the backup cannot
		198	leave an archived ref naming objects the archive lacks. The exception
		199	is git's own automatic gc after a push repacking during the walk: the
		200	archive can then miss objects, and =--verify= reports it. (#259)
		201	- =gitbayd admin gc= and =admin mr prune= refuse with "a backup is
		202	running" during a full backup. A pack or loose object that git's
		203	automatic gc removes while the backup walks is skipped instead of
		204	failing the run; =--verify= reports it if a ref needed it. (#259)
		205	- =gitbayd admin backup= and =--verify= no longer need the secret key
		206	file: sealed values are copied as they are. A missing database is
		207	refused instead of created. (#259)
		208	- =gitbayd admin backup= refuses an =--out= inside =server.root=, and
		209	removes the snapshot directories and temporary archives a killed run
		210	left beside its archive once they are a day old. (#259)
		211	- =gitbayd admin backup --verify= runs fsck with =--git-dir=, so a
		212	directory that is not a repository fails instead of git checking an
		213	enclosing one, and does not extract =objects/info/alternates= or a
		214	repository's =commondir=, so neither can point fsck at another
		215	repository on the host. (#259)
		216	- =gitbayd admin secrets init= and =rotate= hold an flock on =<key
		217	file>.lock=, so two runs at once serialize. (#273)
183		218
184	* v1.36.0 — 2026-09-23	219	* v1.36.0 — 2026-09-23
185		220

cmd/gitbayd/backup.go +295 −33

	@@ -4,18 +4,23 @@ import (
4	"archive/tar"	4	"archive/tar"
5	"bufio"	5	"bufio"
6	"compress/gzip"	6	"compress/gzip"
		7	"errors"
7	"fmt"	8	"fmt"
8	"io"	9	"io"
9	"io/fs"	10	"io/fs"
10	"os"	11	"os"
		12	"path"
11	"path/filepath"	13	"path/filepath"
		14	"regexp"
12	"strings"	15	"strings"
13	"time"	16	"time"
14		17
15	"filippo.io/age"	18	"filippo.io/age"
16	"github.com/spf13/cobra"	19	"github.com/spf13/cobra"
17		20
		21	"gitbay.org/gitbay/internal/backuplock"
18	"gitbay.org/gitbay/internal/config"	22	"gitbay.org/gitbay/internal/config"
		23	"gitbay.org/gitbay/internal/gitutil"
19	"gitbay.org/gitbay/internal/store"	24	"gitbay.org/gitbay/internal/store"
20	)	25	)
21		26
	@@ -23,10 +28,11 @@ import (
23	// every repository and the SSH host keys. Restore by extracting the archive	28	// every repository and the SSH host keys. Restore by extracting the archive
24	// into a fresh server.root.	29	// into a fresh server.root.
25	//	30	//
26	// Ordering: the database is snapshotted BEFORE the repositories are read.	31	// Ordering: the database is snapshotted BEFORE the repositories are read,
27	// A push that lands mid-backup then shows up only as unreferenced git	32	// and each repository's refs before its objects. A push that lands
28	// objects in the archive (harmless); the reverse order could leave database	33	// mid-backup then shows up only as unreferenced git objects in the archive
29	// rows pointing at objects the archive never captured.	34	// (harmless) or not at all; the reverse order could leave database rows or
		35	// refs pointing at objects the archive never captured.
30	func backupCmd() *cobra.Command {	36	func backupCmd() *cobra.Command {
31	var out, verify, identity string	37	var out, verify, identity string
32	var dbOnly bool	38	var dbOnly bool
	@@ -43,8 +49,11 @@ affordable, and the database is the copy of issues, merge requests and
43	comments that exists nowhere else. Repositories are not in such an archive,	49	comments that exists nowhere else. Repositories are not in such an archive,
44	so it supplements a full backup and does not replace one.	50	so it supplements a full backup and does not replace one.
45		51
46	Restore: extract into an empty directory, point server.root at it, start	52	Restore: extract into an empty directory, point server.root at it,
47	gitbayd. Host keys are preserved, so clients keep their known_hosts entries.	53	restore server.secret_key_file from its own backup (mode 0600, owned by
		54	the daemon user), start gitbayd. No archive carries the key file, and
		55	without it gitbayd refuses to start. Host keys are preserved, so clients
		56	keep their known_hosts entries.
48		57
49	With [backup] age_recipients set, the archive is encrypted to those age	58	With [backup] age_recipients set, the archive is encrypted to those age
50	public keys and its name ends in .age. --verify then needs --identity	59	public keys and its name ends in .age. --verify then needs --identity
	@@ -62,7 +71,7 @@ public keys and its name ends in .age. --verify then needs --identity
62	}	71	}
63	cmd.Flags().StringVar(&out, "out", "", "output archive path (default gitbay-backup-<utc timestamp>.tar.gz; .age is appended when [backup] age_recipients is set)")	72	cmd.Flags().StringVar(&out, "out", "", "output archive path (default gitbay-backup-<utc timestamp>.tar.gz; .age is appended when [backup] age_recipients is set)")
64	cmd.Flags().BoolVar(&dbOnly, "db-only", false, "archive the database snapshot alone, without repositories")	73	cmd.Flags().BoolVar(&dbOnly, "db-only", false, "archive the database snapshot alone, without repositories")
65	cmd.Flags().StringVar(&verify, "verify", "", "check an archive instead of writing one: database integrity, and its repositories against the archive's")	74	cmd.Flags().StringVar(&verify, "verify", "", "check an archive instead of writing one: database integrity, its repositories against the archive's, and git connectivity of each")
66	cmd.Flags().StringVar(&identity, "identity", "", "with --verify: an age identity file that opens an encrypted archive")	75	cmd.Flags().StringVar(&identity, "identity", "", "with --verify: an age identity file that opens an encrypted archive")
67	return cmd	76	return cmd
68	}	77	}
	@@ -90,7 +99,32 @@ func runBackup(cfg config.Config, out string, dbOnly bool) error {
90	return fmt.Errorf("%s ends in .age but [backup] age_recipients is not set, so the archive would not be encrypted", out)	99	return fmt.Errorf("%s ends in .age but [backup] age_recipients is not set, so the archive would not be encrypted", out)
91	}	100	}
92		101
93	st, err := openStore(cfg)	102	dir := filepath.Dir(out)
		103	// An archive under the root would be in the next full backup's walk.
		104	if config.Within(cfg.Server.Root, dir) {
		105	return fmt.Errorf("%s is inside server.root %s; write the archive elsewhere", out, cfg.Server.Root)
		106	}
		107	removeStale(dir, time.Now().Add(-staleAge))
		108
		109	// Deletes, renames and transfers wait until the walk finishes, so
		110	// every repository the snapshot names is still on disk when the walk
		111	// reaches it (#259). A database-only archive reads no repository.
		112	if !dbOnly {
		113	release, err := backuplock.Hold(cfg.Server.Root)
		114	if err != nil {
		115	return fmt.Errorf("backup lock: %w", err)
		116	}
		117	defer release()
		118	}
		119
		120	// VACUUM INTO copies sealed values as they are, so the backup needs
		121	// no key file, and it migrates nothing. store.Open would create a
		122	// missing database, so its absence is checked first.
		123	dbFile := filepath.Join(cfg.Server.Root, "gitbay.db")
		124	if _, err := os.Stat(dbFile); err != nil {
		125	return fmt.Errorf("database: %w", err)
		126	}
		127	st, err := store.Open(dbFile)
94	if err != nil {	128	if err != nil {
95	return err	129	return err
96	}	130	}
	@@ -98,7 +132,6 @@ func runBackup(cfg config.Config, out string, dbOnly bool) error {
98		132
99	// 1. Consistent database snapshot, before any repository is read. It	133	// 1. Consistent database snapshot, before any repository is read. It
100	// goes in a fresh 0700 directory beside the archive.	134	// goes in a fresh 0700 directory beside the archive.
101	dir := filepath.Dir(out)
102	snapDir, err := os.MkdirTemp(dir, ".gitbay-snap-")	135	snapDir, err := os.MkdirTemp(dir, ".gitbay-snap-")
103	if err != nil {	136	if err != nil {
104	return err	137	return err
	@@ -142,18 +175,22 @@ func runBackup(cfg config.Config, out string, dbOnly bool) error {
142	skip := map[string]bool{	175	skip := map[string]bool{
143	"gitbay.db": true, "gitbay.db-wal": true, "gitbay.db-shm": true,	176	"gitbay.db": true, "gitbay.db-wal": true, "gitbay.db-shm": true,
144	"hook.sock": true, "askpass.sh": true, "hooks": true,	177	"hook.sock": true, "askpass.sh": true, "hooks": true,
		178	backuplock.Name: true,
145	}	179	}
146	repoCount := 0	180	repoCount := 0
147	root := cfg.Server.Root	181	root := cfg.Server.Root
148	if !dbOnly {	182	if !dbOnly {
149	err = filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error {	183	err = filepath.WalkDir(root, func(path string, d fs.DirEntry, walkErr error) error {
150	if err != nil {
151	return err
152	}
153	rel, err := filepath.Rel(root, path)	184	rel, err := filepath.Rel(root, path)
154	if err != nil {	185	if err != nil {
155	return err	186	return err
156	}	187	}
		188	if walkErr != nil {
		189	if vanished(walkErr, rel) {
		190	return nil
		191	}
		192	return walkErr
		193	}
157	if rel == "." {	194	if rel == "." {
158	return nil	195	return nil
159	}	196	}
	@@ -166,13 +203,38 @@ func runBackup(cfg config.Config, out string, dbOnly bool) error {
166	if !d.Type().IsRegular() && !d.IsDir() {	203	if !d.Type().IsRegular() && !d.IsDir() {
167	return nil // sockets, symlinks	204	return nil // sockets, symlinks
168	}	205	}
		206	// A repository's refs were archived on entering it.
		207	if strings.HasSuffix(filepath.Dir(rel), ".git") && refNames[d.Name()] {
		208	if d.IsDir() {
		209	return filepath.SkipDir
		210	}
		211	return nil
		212	}
169	if d.IsDir() {	213	if d.IsDir() {
		214	// A directory entry, even for one that holds no file (a
		215	// bare repository's refs/heads and refs/tags once every
		216	// ref is packed), so extraction recreates it: git's own
		217	// repository discovery needs refs/ to exist.
		218	if err := addDir(tw, path, filepath.ToSlash(rel)); err != nil {
		219	if vanished(err, rel) {
		220	return filepath.SkipDir
		221	}
		222	return err
		223	}
170	if strings.HasSuffix(rel, ".git") {	224	if strings.HasSuffix(rel, ".git") {
171	repoCount++	225	repoCount++
		226	if err := addRefs(tw, path, filepath.ToSlash(rel)); err != nil {
		227	return err
		228	}
		229	afterRefs(path)
172	}	230	}
173	return nil // directories are implied by member paths	231	return nil
174	}	232	}
175	return addFile(tw, path, filepath.ToSlash(rel))	233	beforeAdd(path)
		234	if err := addFile(tw, path, filepath.ToSlash(rel)); !vanished(err, rel) {
		235	return err
		236	}
		237	return nil
176	})	238	})
177	if err != nil {	239	if err != nil {
178	return err	240	return err
	@@ -212,6 +274,129 @@ func runBackup(cfg config.Config, out string, dbOnly bool) error {
212	return nil	274	return nil
213	}	275	}
214		276
		277	// staleAge is how old a snapshot directory or temporary archive must be
		278	// before a later run removes it. A run that is still writing one is
		279	// younger than this.
		280	const staleAge = 24 * time.Hour
		281
		282	// tmpArchive is a temporary archive's name: os.CreateTemp's pattern
		283	// "."+base+".tmp-" followed by the digits it appends.
		284	var tmpArchive = regexp.MustCompile(`^\..+\.tmp-[0-9]+$`)
		285
		286	// removeStale removes what a killed run left in dir: snapshot
		287	// directories and temporary archives last modified before cutoff.
		288	func removeStale(dir string, cutoff time.Time) {
		289	ents, err := os.ReadDir(dir)
		290	if err != nil {
		291	return
		292	}
		293	for _, e := range ents {
		294	name := e.Name()
		295	snap := e.IsDir() && strings.HasPrefix(name, ".gitbay-snap-")
		296	tmp := e.Type().IsRegular() && tmpArchive.MatchString(name)
		297	if !snap && !tmp {
		298	continue
		299	}
		300	info, err := e.Info()
		301	if err != nil \|\| !info.ModTime().Before(cutoff) {
		302	continue
		303	}
		304	p := filepath.Join(dir, name)
		305	if err := os.RemoveAll(p); err != nil {
		306	fmt.Fprintf(os.Stderr, "removing stale %s: %v\n", p, err)
		307	continue
		308	}
		309	fmt.Fprintf(os.Stderr, "removed stale %s\n", p)
		310	}
		311	}
		312
		313	// refNames are what a repository's refs are read from. WalkDir would
		314	// reach objects/ before packed-refs and refs/, so a push landing mid-walk
		315	// could leave an archived ref naming objects the archive lacks. addRefs
		316	// archives these first on entering the repository; objects are only ever
		317	// added, so the walk that follows finds every object those refs reach.
		318	var refNames = map[string]bool{"HEAD": true, "packed-refs": true, "refs": true}
		319
		320	// afterRefs runs between a repository's refs and the rest of it. Tests
		321	// use it to write into the repository at that point.
		322	var afterRefs = func(repo string) {}
		323
		324	// addRefs archives HEAD, refs/ and packed-refs of the repository at
		325	// path, whichever exist. refs/ is read before packed-refs, the order git
		326	// reads them in: pack-refs writes packed-refs before deleting the loose
		327	// refs it packed, so a ref moving between the two is caught in one.
		328	func addRefs(tw *tar.Writer, path, name string) error {
		329	if err := addRegular(tw, path, name, "HEAD"); err != nil {
		330	return err
		331	}
		332	refs := filepath.Join(path, "refs")
		333	if _, err := os.Lstat(refs); err == nil {
		334	if err := addTree(tw, path, name, refs); err != nil {
		335	return err
		336	}
		337	} else if !errors.Is(err, fs.ErrNotExist) {
		338	return err
		339	}
		340	return addRegular(tw, path, name, "packed-refs")
		341	}
		342
		343	// addRegular archives the regular file f in the repository at path, if
		344	// it exists.
		345	func addRegular(tw *tar.Writer, path, name, f string) error {
		346	fi, err := os.Lstat(filepath.Join(path, f))
		347	if errors.Is(err, fs.ErrNotExist) \|\| err == nil && !fi.Mode().IsRegular() {
		348	return nil
		349	}
		350	if err != nil {
		351	return err
		352	}
		353	return addFile(tw, filepath.Join(path, f), name+"/"+f)
		354	}
		355
		356	// addTree archives the directory refs inside the repository at path.
		357	func addTree(tw *tar.Writer, path, name, refs string) error {
		358	return filepath.WalkDir(refs, func(p string, d fs.DirEntry, err error) error {
		359	if err != nil {
		360	return err
		361	}
		362	rel, err := filepath.Rel(path, p)
		363	if err != nil {
		364	return err
		365	}
		366	member := name + "/" + filepath.ToSlash(rel)
		367	switch {
		368	case d.IsDir():
		369	return addDir(tw, p, member)
		370	case d.Type().IsRegular():
		371	return addFile(tw, p, member)
		372	}
		373	return nil
		374	})
		375	}
		376
		377	// beforeAdd runs before each file the walk archives outside refs. Tests
		378	// use it to remove a file between listing and reading.
		379	var beforeAdd = func(path string) {}
		380
		381	// vanished reports a file or directory under a repository's objects/
		382	// that went between the walk listing it and reading it: a pack or loose
		383	// object a concurrent gc or receive.autogc removed. The walk skips it.
		384	// Refs archived earlier reach only objects that are still reachable, and
		385	// a repack writes those into a new pack before removing the old one; if
		386	// one is lost regardless, verify's fsck reports it.
		387	func vanished(err error, rel string) bool {
		388	if !errors.Is(err, fs.ErrNotExist) {
		389	return false
		390	}
		391	parts := strings.Split(filepath.ToSlash(rel), "/")
		392	for i := 0; i+2 < len(parts); i++ {
		393	if strings.HasSuffix(parts[i], ".git") && parts[i+1] == "objects" {
		394	return true
		395	}
		396	}
		397	return false
		398	}
		399
215	// syncDir makes a rename in dir durable.	400	// syncDir makes a rename in dir durable.
216	func syncDir(dir string) error {	401	func syncDir(dir string) error {
217	d, err := os.Open(dir)	402	d, err := os.Open(dir)
	@@ -230,8 +415,15 @@ func snapshotDB(st *store.Store, dest string) error {
230	return err	415	return err
231	}	416	}
232		417
		418	// addFile opens before writing the header, so a file removed after the
		419	// walk listed it fails before the archive has a member for it.
233	func addFile(tw *tar.Writer, path, name string) error {	420	func addFile(tw *tar.Writer, path, name string) error {
234	info, err := os.Stat(path)	421	src, err := os.Open(path)
		422	if err != nil {
		423	return err
		424	}
		425	defer src.Close()
		426	info, err := src.Stat()
235	if err != nil {	427	if err != nil {
236	return err	428	return err
237	}	429	}
	@@ -243,20 +435,33 @@ func addFile(tw *tar.Writer, path, name string) error {
243	if err := tw.WriteHeader(hdr); err != nil {	435	if err := tw.WriteHeader(hdr); err != nil {
244	return err	436	return err
245	}	437	}
246	src, err := os.Open(path)	438	_, err = io.CopyN(tw, src, hdr.Size)
		439	return err
		440	}
		441
		442	// addDir writes a directory entry, so an empty directory survives
		443	// extraction. The mode never exceeds 0755, whatever the source directory
		444	// carries.
		445	func addDir(tw *tar.Writer, path, name string) error {
		446	info, err := os.Stat(path)
247	if err != nil {	447	if err != nil {
248	return err	448	return err
249	}	449	}
250	defer src.Close()	450	hdr, err := tar.FileInfoHeader(info, "")
251	_, err = io.Copy(tw, src)	451	if err != nil {
252	return err	452	return err
		453	}
		454	hdr.Name = name + "/"
		455	hdr.Mode = hdr.Mode&^0o777 \| hdr.Mode&0o755
		456	return tw.WriteHeader(hdr)
253	}	457	}
254		458
255	// verifyBackup reads an archive back, decrypting it with identity when it	459	// verifyBackup reads an archive back, decrypting it with identity when it
256	// is encrypted: the database snapshot must pass	460	// is encrypted: the database snapshot must pass SQLite's integrity check,
257	// SQLite's integrity check, and every repository it names must be in the	461	// every repository it names must be in the archive, and each of those
258	// archive. A database-only archive is checked for integrity alone and	462	// must pass git fsck --connectivity-only. A database-only archive is
259	// says so. Nothing is written except a temporary copy of the database.	463	// checked for integrity alone and says so. Repositories are extracted to
		464	// a temporary directory for the check, so it needs free space for them.
260	func verifyBackup(path, identity string) error {	465	func verifyBackup(path, identity string) error {
261	f, err := os.Open(path)	466	f, err := os.Open(path)
262	if err != nil {	467	if err != nil {
	@@ -292,21 +497,35 @@ func verifyBackup(path, identity string) error {
292	switch {	497	switch {
293	case h.Name == "gitbay.db":	498	case h.Name == "gitbay.db":
294	dbPath = filepath.Join(tmp, "gitbay.db")	499	dbPath = filepath.Join(tmp, "gitbay.db")
295	w, err := os.Create(dbPath)	500	if err := extractTo(tr, dbPath); err != nil {
296	if err != nil {
297	return err
298	}
299	if _, err := io.Copy(w, tr); err != nil {
300	w.Close()
301	return fmt.Errorf("%s: extracting the database: %w", path, err)	501	return fmt.Errorf("%s: extracting the database: %w", path, err)
302	}	502	}
303	w.Close()
304	case strings.HasPrefix(h.Name, "repos/"):	503	case strings.HasPrefix(h.Name, "repos/"):
		504	trimmed := strings.TrimSuffix(h.Name, "/")
305	// repos/<owner>/<name>.git/HEAD marks one repository present.	505	// repos/<owner>/<name>.git/HEAD marks one repository present.
306	parts := strings.Split(h.Name, "/")	506	parts := strings.Split(trimmed, "/")
307	if len(parts) == 4 && parts[3] == "HEAD" && strings.HasSuffix(parts[2], ".git") {	507	if len(parts) == 4 && parts[3] == "HEAD" && strings.HasSuffix(parts[2], ".git") {
308	inArchive[parts[1]+"/"+strings.TrimSuffix(parts[2], ".git")] = true	508	inArchive[parts[1]+"/"+strings.TrimSuffix(parts[2], ".git")] = true
309	}	509	}
		510	if !filepath.IsLocal(trimmed) {
		511	return fmt.Errorf("%s: member %q leaves the archive root", path, h.Name)
		512	}
		513	if borrowsObjects(trimmed) {
		514	continue
		515	}
		516	dest := filepath.Join(tmp, filepath.FromSlash(trimmed))
		517	switch h.Typeflag {
		518	case tar.TypeDir:
		519	// The archive's directory modes do not matter to fsck, and
		520	// a hostile one would stop RemoveAll cleaning up.
		521	if err := os.MkdirAll(dest, 0o700); err != nil {
		522	return fmt.Errorf("%s: creating %s: %w", path, h.Name, err)
		523	}
		524	case tar.TypeReg:
		525	if err := extractTo(tr, dest); err != nil {
		526	return fmt.Errorf("%s: extracting %s: %w", path, h.Name, err)
		527	}
		528	}
310	}	529	}
311	}	530	}
312	// Read to the end so gzip checks its trailer and age its final chunk.	531	// Read to the end so gzip checks its trailer and age its final chunk.
	@@ -351,11 +570,54 @@ func verifyBackup(path, identity string) error {
351	return fmt.Errorf("%s: %d repositories the database names are not in the archive: %s", path, len(missing), strings.Join(missing, ", "))	570	return fmt.Errorf("%s: %d repositories the database names are not in the archive: %s", path, len(missing), strings.Join(missing, ", "))
352	}	571	}
353	if extra > 0 {	572	if extra > 0 {
354	fmt.Printf("%d repositories in the archive that the database does not name (deleted after the snapshot)\n", extra)	573	fmt.Printf("%d repositories in the archive that the database does not name (created after the snapshot)\n", extra)
		574	}
		575	var broken []string
		576	for _, r := range repos {
		577	dir := filepath.Join(tmp, "repos", r.OwnerName, r.Name+".git")
		578	if err := gitutil.FsckConnectivity(dir); err != nil {
		579	fmt.Fprintf(os.Stderr, "%s: %v\n", r.Path(), err)
		580	broken = append(broken, r.Path())
		581	}
		582	}
		583	if len(broken) > 0 {
		584	return fmt.Errorf("%s: %d repositories fail the connectivity check: %s", path, len(broken), strings.Join(broken, ", "))
355	}	585	}
		586	fmt.Printf("connectivity ok on %d repositories\n", len(repos))
356	return nil	587	return nil
357	}	588	}
358		589
		590	// borrowsObjects reports an archive member that would point git at
		591	// objects or refs outside the extracted repository: alternates, or a
		592	// commondir directly in a *.git directory. gitbay writes none, and one in
		593	// a hostile archive would have fsck read another repository on the host,
		594	// so verify leaves them out. The comparison ignores case, as a
		595	// case-insensitive filesystem would.
		596	func borrowsObjects(name string) bool {
		597	name = strings.ToLower(filepath.ToSlash(filepath.Clean(name)))
		598	if dir, base := path.Split(name); base == "commondir" && strings.HasSuffix(strings.TrimSuffix(dir, "/"), ".git") {
		599	return true
		600	}
		601	return strings.HasSuffix(name, "/objects/info/alternates") \|\|
		602	strings.HasSuffix(name, "/objects/info/http-alternates")
		603	}
		604
		605	// extractTo writes one archive member to dest, owner-only.
		606	func extractTo(r io.Reader, dest string) error {
		607	if err := os.MkdirAll(filepath.Dir(dest), 0o700); err != nil {
		608	return err
		609	}
		610	w, err := os.OpenFile(dest, os.O_WRONLY\|os.O_CREATE\|os.O_TRUNC, 0o600)
		611	if err != nil {
		612	return err
		613	}
		614	if _, err := io.Copy(w, r); err != nil {
		615	w.Close()
		616	return err
		617	}
		618	return w.Close()
		619	}
		620
359	const ageHeader = "age-encryption.org/v1\n"	621	const ageHeader = "age-encryption.org/v1\n"
360		622
361	// archiveReader returns the archive's gzip stream, decrypting it first	623	// archiveReader returns the archive's gzip stream, decrypting it first

cmd/gitbayd/backup_test.go +516

	@@ -5,7 +5,9 @@ import (
5	"compress/gzip"	5	"compress/gzip"
6	"errors"	6	"errors"
7	"io"	7	"io"
		8	"io/fs"
8	"os"	9	"os"
		10	"os/exec"
9	"path/filepath"	11	"path/filepath"
10	"sort"	12	"sort"
11	"strings"	13	"strings"
	@@ -14,7 +16,9 @@ import (
14		16
15	"filippo.io/age"	17	"filippo.io/age"
16		18
		19	"gitbay.org/gitbay/internal/backuplock"
17	"gitbay.org/gitbay/internal/config"	20	"gitbay.org/gitbay/internal/config"
		21	"gitbay.org/gitbay/internal/gitutil"
18	)	22	)
19		23
20	// members lists the archive's entries by name.	24	// members lists the archive's entries by name.
	@@ -324,3 +328,515 @@ func TestArchivePath(t *testing.T) {
324	}	328	}
325	}	329	}
326	}	330	}
		331
		332	func gitIn(t *testing.T, dir string, args ...string) string {
		333	t.Helper()
		334	cmd := exec.Command("git", append([]string{"-C", dir}, args...)...)
		335	cmd.Env = append(os.Environ(),
		336	"GIT_AUTHOR_NAME=t", "GIT_AUTHOR_EMAIL=t@e",
		337	"GIT_COMMITTER_NAME=t", "GIT_COMMITTER_EMAIL=t@e")
		338	out, err := cmd.CombinedOutput()
		339	if err != nil {
		340	t.Fatalf("git %v: %v\n%s", args, err, out)
		341	}
		342	return strings.TrimSpace(string(out))
		343	}
		344
		345	// verify runs git's connectivity check on every repository the
		346	// database names: a repository missing an object fails it.
		347	func TestVerifyChecksConnectivity(t *testing.T) {
		348	cfg := testConfig(t)
		349	st, err := openStore(cfg)
		350	if err != nil {
		351	t.Fatal(err)
		352	}
		353	uid, err := st.CreateUser("krz", false)
		354	if err != nil {
		355	t.Fatal(err)
		356	}
		357	if _, err := st.CreateRepo("user", uid, "thing", "public"); err != nil {
		358	t.Fatal(err)
		359	}
		360	st.Close()
		361
		362	work := t.TempDir()
		363	gitIn(t, work, "init", "-q", "-b", "main")
		364	if err := os.WriteFile(filepath.Join(work, "a.txt"), []byte("a\n"), 0o644); err != nil {
		365	t.Fatal(err)
		366	}
		367	gitIn(t, work, "add", "a.txt")
		368	gitIn(t, work, "commit", "-q", "-m", "one")
		369	dir := filepath.Join(cfg.Server.Root, "repos", "krz", "thing.git")
		370	gitIn(t, work, "clone", "-q", "--bare", work, dir)
		371
		372	good := filepath.Join(t.TempDir(), "good.tar.gz")
		373	if err := runBackup(cfg, good, false); err != nil {
		374	t.Fatal(err)
		375	}
		376	if err := verifyBackup(good, ""); err != nil {
		377	t.Fatalf("intact archive: %v", err)
		378	}
		379
		380	blob := gitIn(t, dir, "rev-parse", "HEAD:a.txt")
		381	if err := os.Remove(filepath.Join(dir, "objects", blob[:2], blob[2:])); err != nil {
		382	t.Fatal(err)
		383	}
		384	bad := filepath.Join(t.TempDir(), "bad.tar.gz")
		385	if err := runBackup(cfg, bad, false); err != nil {
		386	t.Fatal(err)
		387	}
		388	err = verifyBackup(bad, "")
		389	if err == nil \|\| !strings.Contains(err.Error(), "krz/thing") \|\| !strings.Contains(err.Error(), "connectivity") {
		390	t.Fatalf("archive with a missing blob: %v", err)
		391	}
		392	}
		393
		394	// A full backup waits for a delete under way, and does not archive its
		395	// own lock file.
		396	func TestFullBackupWaitsForRepositoryMoves(t *testing.T) {
		397	cfg := testConfig(t)
		398	s, err := openStore(cfg)
		399	if err != nil {
		400	t.Fatal(err)
		401	}
		402	s.Close()
		403	inFlight, err := backuplock.TryShared(cfg.Server.Root)
		404	if err != nil {
		405	t.Fatal(err)
		406	}
		407	out := filepath.Join(t.TempDir(), "b.tar.gz")
		408	done := make(chan error, 1)
		409	go func() { done <- runBackup(cfg, out, false) }()
		410	select {
		411	case err := <-done:
		412	t.Fatalf("backup finished while a delete held the lock: %v", err)
		413	case <-time.After(200 * time.Millisecond):
		414	}
		415	inFlight()
		416	select {
		417	case err := <-done:
		418	if err != nil {
		419	t.Fatal(err)
		420	}
		421	case <-time.After(10 * time.Second):
		422	t.Fatal("backup never started after the delete finished")
		423	}
		424	for _, n := range members(t, out) {
		425	if n == backuplock.Name {
		426	t.Fatalf("archive carries %s", n)
		427	}
		428	}
		429	}
		430
		431	// A repository with every ref packed keeps its empty refs/heads and
		432	// refs/tags directories through backup and extraction, the same as a real
		433	// restore would: git needs refs/ to recognize a bare repository at all,
		434	// even when every ref lives in packed-refs (#259).
		435	func TestBackupPreservesPackedRefDirs(t *testing.T) {
		436	cfg := testConfig(t)
		437	st, err := openStore(cfg)
		438	if err != nil {
		439	t.Fatal(err)
		440	}
		441	uid, err := st.CreateUser("krz", false)
		442	if err != nil {
		443	t.Fatal(err)
		444	}
		445	if _, err := st.CreateRepo("user", uid, "thing", "public"); err != nil {
		446	t.Fatal(err)
		447	}
		448	st.Close()
		449
		450	work := t.TempDir()
		451	gitIn(t, work, "init", "-q", "-b", "main")
		452	if err := os.WriteFile(filepath.Join(work, "a.txt"), []byte("a\n"), 0o644); err != nil {
		453	t.Fatal(err)
		454	}
		455	gitIn(t, work, "add", "a.txt")
		456	gitIn(t, work, "commit", "-q", "-m", "one")
		457	dir := filepath.Join(cfg.Server.Root, "repos", "krz", "thing.git")
		458	gitIn(t, work, "clone", "-q", "--bare", work, dir)
		459	gitIn(t, dir, "pack-refs", "--all")
		460	entries, err := os.ReadDir(filepath.Join(dir, "refs", "heads"))
		461	if err != nil {
		462	t.Fatal(err)
		463	}
		464	if len(entries) != 0 {
		465	t.Fatalf("refs/heads not empty after pack-refs --all: %v", entries)
		466	}
		467
		468	archive := filepath.Join(t.TempDir(), "b.tar.gz")
		469	if err := runBackup(cfg, archive, false); err != nil {
		470	t.Fatal(err)
		471	}
		472
		473	// verify sees the archive exactly as a restore would: no workaround.
		474	if err := verifyBackup(archive, ""); err != nil {
		475	t.Fatalf("verify: %v", err)
		476	}
		477
		478	restored := t.TempDir()
		479	if out, err := exec.Command("tar", "-xzf", archive, "-C", restored).CombinedOutput(); err != nil {
		480	t.Fatalf("extract: %v\n%s", err, out)
		481	}
		482	restoredRepo := filepath.Join(restored, "repos", "krz", "thing.git")
		483	if got := gitIn(t, restoredRepo, "rev-parse", "--verify", "HEAD"); got == "" {
		484	t.Fatal("rev-parse --verify HEAD returned nothing after restore")
		485	}
		486	gitIn(t, restoredRepo, "fsck", "--connectivity-only", "--no-progress", "--no-dangling")
		487	}
		488
		489	// A commit pushed after a repository's refs are archived and before its
		490	// objects are leaves the archive with the earlier refs and every object
		491	// they reach, plus the new ones unreferenced (#259).
		492	func TestBackupArchivesRefsBeforeObjects(t *testing.T) {
		493	cfg := testConfig(t)
		494	st, err := openStore(cfg)
		495	if err != nil {
		496	t.Fatal(err)
		497	}
		498	uid, err := st.CreateUser("krz", false)
		499	if err != nil {
		500	t.Fatal(err)
		501	}
		502	if _, err := st.CreateRepo("user", uid, "thing", "public"); err != nil {
		503	t.Fatal(err)
		504	}
		505	st.Close()
		506
		507	work := t.TempDir()
		508	gitIn(t, work, "init", "-q", "-b", "main")
		509	if err := os.WriteFile(filepath.Join(work, "a.txt"), []byte("a\n"), 0o644); err != nil {
		510	t.Fatal(err)
		511	}
		512	gitIn(t, work, "add", "a.txt")
		513	gitIn(t, work, "commit", "-q", "-m", "one")
		514	dir := filepath.Join(cfg.Server.Root, "repos", "krz", "thing.git")
		515	gitIn(t, work, "clone", "-q", "--bare", work, dir)
		516	first := gitIn(t, dir, "rev-parse", "refs/heads/main")
		517
		518	var second string
		519	afterRefs = func(repo string) {
		520	if repo != dir {
		521	return
		522	}
		523	if err := os.WriteFile(filepath.Join(work, "b.txt"), []byte("b\n"), 0o644); err != nil {
		524	t.Fatal(err)
		525	}
		526	gitIn(t, work, "add", "b.txt")
		527	gitIn(t, work, "commit", "-q", "-m", "two")
		528	gitIn(t, work, "push", "-q", dir, "main")
		529	second = gitIn(t, dir, "rev-parse", "refs/heads/main")
		530	}
		531	t.Cleanup(func() { afterRefs = func(string) {} })
		532
		533	archive := filepath.Join(t.TempDir(), "b.tar.gz")
		534	if err := runBackup(cfg, archive, false); err != nil {
		535	t.Fatal(err)
		536	}
		537	if second == "" \|\| second == first {
		538	t.Fatal("the push between the refs and the objects did not happen")
		539	}
		540	if err := verifyBackup(archive, ""); err != nil {
		541	t.Fatalf("verify: %v", err)
		542	}
		543	restored := t.TempDir()
		544	if out, err := exec.Command("tar", "-xzf", archive, "-C", restored).CombinedOutput(); err != nil {
		545	t.Fatalf("extract: %v\n%s", err, out)
		546	}
		547	repo := filepath.Join(restored, "repos", "krz", "thing.git")
		548	if got := gitIn(t, repo, "rev-parse", "refs/heads/main"); got != first {
		549	t.Errorf("archived main is %s, want %s from before the push", got, first)
		550	}
		551	gitIn(t, repo, "cat-file", "-e", second)
		552	}
		553
		554	// A pack removed between the walk listing it and reading it is skipped,
		555	// and verify's fsck then reports what it held; a vanished file outside
		556	// objects/ still fails the backup.
		557	func TestBackupSkipsVanishedObjects(t *testing.T) {
		558	cfg := testConfig(t)
		559	st, err := openStore(cfg)
		560	if err != nil {
		561	t.Fatal(err)
		562	}
		563	uid, err := st.CreateUser("krz", false)
		564	if err != nil {
		565	t.Fatal(err)
		566	}
		567	if _, err := st.CreateRepo("user", uid, "thing", "public"); err != nil {
		568	t.Fatal(err)
		569	}
		570	st.Close()
		571
		572	work := t.TempDir()
		573	gitIn(t, work, "init", "-q", "-b", "main")
		574	if err := os.WriteFile(filepath.Join(work, "a.txt"), []byte("a\n"), 0o644); err != nil {
		575	t.Fatal(err)
		576	}
		577	gitIn(t, work, "add", "a.txt")
		578	gitIn(t, work, "commit", "-q", "-m", "one")
		579	dir := filepath.Join(cfg.Server.Root, "repos", "krz", "thing.git")
		580	gitIn(t, work, "clone", "-q", "--bare", work, dir)
		581	gitIn(t, dir, "repack", "-q", "-a", "-d")
		582
		583	removed := ""
		584	beforeAdd = func(path string) {
		585	if removed == "" && strings.HasSuffix(path, ".pack") {
		586	removed = path
		587	os.Remove(path)
		588	}
		589	}
		590	t.Cleanup(func() { beforeAdd = func(string) {} })
		591	archive := filepath.Join(t.TempDir(), "b.tar.gz")
		592	if err := runBackup(cfg, archive, false); err != nil {
		593	t.Fatalf("backup with a vanished pack: %v", err)
		594	}
		595	if removed == "" {
		596	t.Fatal("no pack was archived")
		597	}
		598	for _, n := range members(t, archive) {
		599	if strings.HasSuffix(n, ".pack") {
		600	t.Errorf("archive carries %s", n)
		601	}
		602	}
		603	if err := verifyBackup(archive, ""); err == nil \|\| !strings.Contains(err.Error(), "connectivity") {
		604	t.Errorf("verify of an archive missing its pack: %v", err)
		605	}
		606
		607	beforeAdd = func(path string) {
		608	if strings.HasSuffix(path, filepath.Join("thing.git", "config")) {
		609	os.Remove(path)
		610	}
		611	}
		612	err = runBackup(cfg, filepath.Join(t.TempDir(), "c.tar.gz"), false)
		613	if !errors.Is(err, fs.ErrNotExist) {
		614	t.Fatalf("backup with a vanished config: %v, want not-exist", err)
		615	}
		616	}
		617
		618	// gc refuses while a full backup holds the lock.
		619	func TestGCRefusedDuringBackup(t *testing.T) {
		620	cfg := testConfig(t)
		621	release, err := backuplock.Hold(cfg.Server.Root)
		622	if err != nil {
		623	t.Fatal(err)
		624	}
		625	defer release()
		626	if err := runGC(cfg, "", false, false); !errors.Is(err, backuplock.ErrBusy) {
		627	t.Fatalf("gc during a backup: %v", err)
		628	}
		629	}
		630
		631	// A backup and its verify need no key file: sealed values are copied as
		632	// they are. A missing database is refused rather than created.
		633	func TestBackupNeedsNoKeyFile(t *testing.T) {
		634	cfg := testConfig(t)
		635	out := filepath.Join(t.TempDir(), "b.tar.gz")
		636	if err := runBackup(cfg, out, false); !errors.Is(err, fs.ErrNotExist) {
		637	t.Fatalf("backup without a database: %v", err)
		638	}
		639	if _, err := os.Stat(filepath.Join(cfg.Server.Root, "gitbay.db")); !os.IsNotExist(err) {
		640	t.Fatalf("backup created a database: %v", err)
		641	}
		642	s, err := openStore(cfg)
		643	if err != nil {
		644	t.Fatal(err)
		645	}
		646	s.Close()
		647	if err := os.Remove(cfg.Server.SecretKeyFile); err != nil {
		648	t.Fatal(err)
		649	}
		650	if err := runBackup(cfg, out, false); err != nil {
		651	t.Fatalf("backup without the key file: %v", err)
		652	}
		653	if err := verifyBackup(out, ""); err != nil {
		654	t.Fatalf("verify without the key file: %v", err)
		655	}
		656	}
		657
		658	// A run removes what a killed run left beside the archive once it is a
		659	// day old, and leaves younger ones, which may belong to a run under way.
		660	func TestBackupRemovesStaleTemporaries(t *testing.T) {
		661	cfg := testConfig(t)
		662	s, err := openStore(cfg)
		663	if err != nil {
		664	t.Fatal(err)
		665	}
		666	s.Close()
		667	dir := t.TempDir()
		668	old := time.Now().Add(-25 * time.Hour)
		669	mk := func(name string, isDir bool, mtime time.Time) {
		670	p := filepath.Join(dir, name)
		671	if isDir {
		672	if err := os.Mkdir(p, 0o700); err != nil {
		673	t.Fatal(err)
		674	}
		675	} else if err := os.WriteFile(p, []byte("x"), 0o600); err != nil {
		676	t.Fatal(err)
		677	}
		678	if err := os.Chtimes(p, mtime, mtime); err != nil {
		679	t.Fatal(err)
		680	}
		681	}
		682	mk(".gitbay-snap-old", true, old)
		683	mk(".b.tar.gz.tmp-123", false, old)
		684	mk(".gitbay-snap-new", true, time.Now())
		685	mk(".b.tar.gz.tmp-456", false, time.Now())
		686	mk(".keep", false, old)
		687	mk(".notes.tmp-draft", false, old)
		688	if err := runBackup(cfg, filepath.Join(dir, "b.tar.gz"), true); err != nil {
		689	t.Fatal(err)
		690	}
		691	got := leftovers(t, dir)
		692	want := []string{".b.tar.gz.tmp-456", ".gitbay-snap-new", ".keep", ".notes.tmp-draft"}
		693	sort.Strings(got)
		694	if strings.Join(got, " ") != strings.Join(want, " ") {
		695	t.Errorf("left %v, want %v", got, want)
		696	}
		697	}
		698
		699	// An archive written under server.root, directly or through a symlink,
		700	// would be in the next full backup, so it is refused.
		701	func TestBackupRefusesOutputInsideRoot(t *testing.T) {
		702	cfg := testConfig(t)
		703	s, err := openStore(cfg)
		704	if err != nil {
		705	t.Fatal(err)
		706	}
		707	s.Close()
		708	link := filepath.Join(t.TempDir(), "link")
		709	if err := os.Symlink(cfg.Server.Root, link); err != nil {
		710	t.Fatal(err)
		711	}
		712	for _, out := range []string{
		713	filepath.Join(cfg.Server.Root, "b.tar.gz"),
		714	filepath.Join(cfg.Server.Root, "backups", "b.tar.gz"),
		715	filepath.Join(link, "b.tar.gz"),
		716	} {
		717	if err := runBackup(cfg, out, true); err == nil \|\| !strings.Contains(err.Error(), "inside server.root") {
		718	t.Errorf("%s: %v", out, err)
		719	}
		720	}
		721	}
		722
		723	// verify does not extract objects/info/alternates, so an archived
		724	// repository cannot borrow objects from paths outside the archive.
		725	func TestVerifyIgnoresAlternates(t *testing.T) {
		726	cfg := testConfig(t)
		727	st, err := openStore(cfg)
		728	if err != nil {
		729	t.Fatal(err)
		730	}
		731	uid, err := st.CreateUser("krz", false)
		732	if err != nil {
		733	t.Fatal(err)
		734	}
		735	if _, err := st.CreateRepo("user", uid, "thing", "public"); err != nil {
		736	t.Fatal(err)
		737	}
		738	st.Close()
		739
		740	work := t.TempDir()
		741	gitIn(t, work, "init", "-q", "-b", "main")
		742	if err := os.WriteFile(filepath.Join(work, "a.txt"), []byte("a\n"), 0o644); err != nil {
		743	t.Fatal(err)
		744	}
		745	gitIn(t, work, "add", "a.txt")
		746	gitIn(t, work, "commit", "-q", "-m", "one")
		747	dir := filepath.Join(cfg.Server.Root, "repos", "krz", "thing.git")
		748	gitIn(t, work, "clone", "-q", "--bare", "--shared", work, dir)
		749	if _, err := os.Stat(filepath.Join(dir, "objects", "info", "alternates")); err != nil {
		750	t.Fatal(err)
		751	}
		752	gitIn(t, dir, "fsck", "--connectivity-only", "--no-progress")
		753
		754	archive := filepath.Join(t.TempDir(), "b.tar.gz")
		755	if err := runBackup(cfg, archive, false); err != nil {
		756	t.Fatal(err)
		757	}
		758	if err := verifyBackup(archive, ""); err == nil \|\| !strings.Contains(err.Error(), "connectivity") {
		759	t.Fatalf("verify of a repository whose objects are only in an alternate: %v", err)
		760	}
		761	}
		762
		763	func TestBorrowsObjectsMember(t *testing.T) {
		764	for name, want := range map[string]bool{
		765	"repos/a/b.git/objects/info/alternates": true,
		766	"repos/a/b.git/objects/info/./alternates": true,
		767	"repos/a/b.git/objects/info/Alternates": true,
		768	"repos/a/b.git/objects/info/http-alternates": true,
		769	"repos/a/b.git/objects/info/packs": false,
		770	"repos/a/b.git/refs/heads/alternates": false,
		771	"repos/a/b.git/commondir": true,
		772	"repos/a/b.git/CommonDir": true,
		773	"repos/a/b.git/refs/heads/commondir": false,
		774	} {
		775	if got := borrowsObjects(name); got != want {
		776	t.Errorf("borrowsObjects(%q) = %v, want %v", name, got, want)
		777	}
		778	}
		779	}
		780
		781	// verify does not extract a commondir, so an archived repository with
		782	// none of its own objects cannot pass by pointing git at a repository on
		783	// the host.
		784	func TestVerifyIgnoresCommondir(t *testing.T) {
		785	cfg := testConfig(t)
		786	st, err := openStore(cfg)
		787	if err != nil {
		788	t.Fatal(err)
		789	}
		790	uid, err := st.CreateUser("krz", false)
		791	if err != nil {
		792	t.Fatal(err)
		793	}
		794	if _, err := st.CreateRepo("user", uid, "thing", "public"); err != nil {
		795	t.Fatal(err)
		796	}
		797	st.Close()
		798
		799	work := t.TempDir()
		800	gitIn(t, work, "init", "-q", "-b", "main")
		801	if err := os.WriteFile(filepath.Join(work, "a.txt"), []byte("a\n"), 0o644); err != nil {
		802	t.Fatal(err)
		803	}
		804	gitIn(t, work, "add", "a.txt")
		805	gitIn(t, work, "commit", "-q", "-m", "one")
		806	host := filepath.Join(t.TempDir(), "host.git")
		807	gitIn(t, work, "clone", "-q", "--bare", work, host)
		808	gitIn(t, host, "pack-refs", "--all")
		809
		810	// A repository whose refs are its own and whose objects directory is
		811	// empty, borrowing everything else from host through commondir.
		812	dir := filepath.Join(cfg.Server.Root, "repos", "krz", "thing.git")
		813	for _, d := range []string{"objects", "refs"} {
		814	if err := os.MkdirAll(filepath.Join(dir, d), 0o755); err != nil {
		815	t.Fatal(err)
		816	}
		817	}
		818	packed, err := os.ReadFile(filepath.Join(host, "packed-refs"))
		819	if err != nil {
		820	t.Fatal(err)
		821	}
		822	for name, body := range map[string]string{
		823	"HEAD": "ref: refs/heads/main\n",
		824	"packed-refs": string(packed),
		825	"commondir": host + "\n",
		826	} {
		827	if err := os.WriteFile(filepath.Join(dir, name), []byte(body), 0o644); err != nil {
		828	t.Fatal(err)
		829	}
		830	}
		831	if err := gitutil.FsckConnectivity(dir); err != nil {
		832	t.Fatalf("with commondir on the host the repository should pass: %v", err)
		833	}
		834
		835	archive := filepath.Join(t.TempDir(), "b.tar.gz")
		836	if err := runBackup(cfg, archive, false); err != nil {
		837	t.Fatal(err)
		838	}
		839	if err := verifyBackup(archive, ""); err == nil \|\| !strings.Contains(err.Error(), "connectivity") {
		840	t.Fatalf("verify of a repository whose objects are only in its commondir: %v", err)
		841	}
		842	}

cmd/gitbayd/maint.go +51 −38

 	"github.com/spf13/cobra"
+	"gitbay.org/gitbay/internal/backuplock"
 	"gitbay.org/gitbay/internal/config"
 	"gitbay.org/gitbay/internal/control"
 	"gitbay.org/gitbay/internal/gitutil"
 			if err != nil {
 				return err
 			}
-			st, err := openStore(cfg)
+			return runGC(cfg, repoPath, aggressive, withLFS)
-			if err != nil {
-				return err
-			}
-			defer st.Close()
-			var repos []store.Repo
-			if repoPath != "" {
-				r, err := st.RepoByPath(repoPath)
-				if err != nil {
-					return fmt.Errorf("no repository %q", repoPath)
-				}
-				repos = []store.Repo{r}
-			} else if repos, err = st.ListAllRepos(); err != nil {
-				return err
-			}
-			var before, after int64
-			for _, r := range repos {
-				dir := control.RepoDir(cfg.Server.Root, r.OwnerName, r.Name)
-				b := gitutil.DirSize(dir)
-				gcArgs := []string{"-C", dir, "gc", "--quiet"}
-				if aggressive {
-					gcArgs = append(gcArgs, "--aggressive")
-				}
-				if out, err := exec.Command(toolpath.Look("git"), gcArgs...).CombinedOutput(); err != nil {
-					fmt.Fprintf(os.Stderr, "%s: gc failed: %v\n%s", r.Path(), err, out)
-					continue
-				}
-				a := gitutil.DirSize(dir)
-				before, after = before+b, after+a
-				fmt.Printf("%s\t%s -> %s\n", r.Path(), human(b), human(a))
-			}
-			fmt.Printf("total\t%s -> %s (freed %s)\n", human(before), human(after), human(before-after))
-			if !withLFS {
-				return nil
-			}
-			return gcLFS(cfg, st)
 		},
 	}
 	cmd.Flags().StringVar(&repoPath, "repo", "", "one repository (owner/name) instead of all")
 	return cmd
 }
+// runGC refuses while a full backup runs: a prune removing a pack the
+// backup has listed but not yet read would leave it out of the archive
+// (#259).
+func runGC(cfg config.Config, repoPath string, aggressive, withLFS bool) error {
+	release, err := backuplock.TryShared(cfg.Server.Root)
+	if err != nil {
+		return err
+	}
+	defer release()
+	st, err := openStore(cfg)
+	if err != nil {
+		return err
+	}
+	defer st.Close()
+	var repos []store.Repo
+	if repoPath != "" {
+		r, err := st.RepoByPath(repoPath)
+		if err != nil {
+			return fmt.Errorf("no repository %q", repoPath)
+		}
+		repos = []store.Repo{r}
+	} else if repos, err = st.ListAllRepos(); err != nil {
+		return err
+	}
+	var before, after int64
+	for _, r := range repos {
+		dir := control.RepoDir(cfg.Server.Root, r.OwnerName, r.Name)
+		b := gitutil.DirSize(dir)
+		gcArgs := []string{"-C", dir, "gc", "--quiet"}
+		if aggressive {
+			gcArgs = append(gcArgs, "--aggressive")
+		}
+		if out, err := exec.Command(toolpath.Look("git"), gcArgs...).CombinedOutput(); err != nil {
+			fmt.Fprintf(os.Stderr, "%s: gc failed: %v\n%s", r.Path(), err, out)
+			continue
+		}
+		a := gitutil.DirSize(dir)
+		before, after = before+b, after+a
+		fmt.Printf("%s\t%s -> %s\n", r.Path(), human(b), human(a))
+	}
+	fmt.Printf("total\t%s -> %s (freed %s)\n", human(before), human(after), human(before-after))
+	if !withLFS {
+		return nil
+	}
+	return gcLFS(cfg, st)
+}
 func human(b int64) string {
 	switch {
 	case b >= 1<<30:

cmd/gitbayd/secrets.go +27

	@@ -65,6 +65,11 @@ keeps its owner. Copy the new file off the host afterwards.`,
65	// the owner of server.root, since the daemon reads it as that user.	65	// the owner of server.root, since the daemon reads it as that user.
66	func initSecrets(cfg config.Config, w io.Writer) error {	66	func initSecrets(cfg config.Config, w io.Writer) error {
67	path := cfg.Server.SecretKeyFile	67	path := cfg.Server.SecretKeyFile
		68	unlock, err := lockKeyFile(path)
		69	if err != nil {
		70	return err
		71	}
		72	defer unlock()
68	if _, err := os.Lstat(path); err == nil {	73	if _, err := os.Lstat(path); err == nil {
69	return fmt.Errorf("%s already exists; gitbayd admin secrets rotate replaces its key", path)	74	return fmt.Errorf("%s already exists; gitbayd admin secrets rotate replaces its key", path)
70	} else if !errors.Is(err, fs.ErrNotExist) {	75	} else if !errors.Is(err, fs.ErrNotExist) {
	@@ -108,6 +113,11 @@ func initSecrets(cfg config.Config, w io.Writer) error {
108	// finishes the job.	113	// finishes the job.
109	func rotateSecrets(cfg config.Config, w io.Writer) error {	114	func rotateSecrets(cfg config.Config, w io.Writer) error {
110	path := cfg.Server.SecretKeyFile	115	path := cfg.Server.SecretKeyFile
		116	unlock, err := lockKeyFile(path)
		117	if err != nil {
		118	return err
		119	}
		120	defer unlock()
111	old, err := seal.ReadKeys(path)	121	old, err := seal.ReadKeys(path)
112	if err != nil {	122	if err != nil {
113	return err	123	return err
	@@ -152,6 +162,23 @@ func rotateSecrets(cfg config.Config, w io.Writer) error {
152	return nil	162	return nil
153	}	163	}
154		164
		165	// lockKeyFile takes an exclusive flock on <path>.lock, waiting for
		166	// another init or rotate to finish. Two rotations interleaved would each
		167	// write a file without the other's new key, and values resealed under
		168	// the lost one would no longer open. O_NOFOLLOW refuses a symlink planted
		169	// at the lock's name.
		170	func lockKeyFile(path string) (func(), error) {
		171	f, err := os.OpenFile(path+".lock", os.O_RDWR\|os.O_CREATE\|syscall.O_NOFOLLOW, 0o600)
		172	if err != nil {
		173	return nil, fmt.Errorf("key file lock: %w", err)
		174	}
		175	if err := syscall.Flock(int(f.Fd()), syscall.LOCK_EX); err != nil {
		176	f.Close()
		177	return nil, fmt.Errorf("key file lock: %w", err)
		178	}
		179	return func() { f.Close() }, nil
		180	}
		181
155	// checkSecrets opens every stored secret and prints, per column, how	182	// checkSecrets opens every stored secret and prints, per column, how
156	// many values each key sealed and every value that does not open. Any	183	// many values each key sealed and every value that does not open. Any
157	// such value is an error.	184	// such value is an error.

cmd/gitbayd/secrets_test.go +48

	@@ -7,6 +7,7 @@ import (
7	"path/filepath"	7	"path/filepath"
8	"strings"	8	"strings"
9	"testing"	9	"testing"
		10	"time"
10		11
11	"gitbay.org/gitbay/internal/config"	12	"gitbay.org/gitbay/internal/config"
12	"gitbay.org/gitbay/internal/seal"	13	"gitbay.org/gitbay/internal/seal"
	@@ -174,3 +175,50 @@ func assertNoKeyMaterial(t *testing.T, out string, keys []seal.Key) {
174	}	175	}
175	}	176	}
176	}	177	}
		178
		179	// A rotate waits while another holds the key file's lock, and two run
		180	// at once both finish with every value still opening.
		181	func TestRotateSecretsSerializes(t *testing.T) {
		182	cfg := testConfig(t)
		183	st, repoID := storeWithSecret(t, cfg)
		184	st.Close()
		185
		186	unlock, err := lockKeyFile(cfg.Server.SecretKeyFile)
		187	if err != nil {
		188	t.Fatal(err)
		189	}
		190	done := make(chan error, 2)
		191	go func() { done <- rotateSecrets(cfg, &bytes.Buffer{}) }()
		192	go func() { done <- rotateSecrets(cfg, &bytes.Buffer{}) }()
		193	select {
		194	case err := <-done:
		195	t.Fatalf("rotate finished while the lock was held: %v", err)
		196	case <-time.After(200 * time.Millisecond):
		197	}
		198	unlock()
		199	for range 2 {
		200	select {
		201	case err := <-done:
		202	if err != nil {
		203	t.Fatalf("rotate: %v", err)
		204	}
		205	case <-time.After(30 * time.Second):
		206	t.Fatal("rotate never finished")
		207	}
		208	}
		209	keys, err := seal.ReadKeys(cfg.Server.SecretKeyFile)
		210	if err != nil \|\| len(keys) != 1 {
		211	t.Fatalf("key file after two rotations: %v, %v", keys, err)
		212	}
		213	st, err = openStore(cfg)
		214	if err != nil {
		215	t.Fatal(err)
		216	}
		217	defer st.Close()
		218	if got, err := st.BuildSecrets(repoID); err != nil \|\| got["TOKEN"] != "v1" {
		219	t.Fatalf("value after two rotations: %v, %v", got, err)
		220	}
		221	if fi, err := os.Lstat(cfg.Server.SecretKeyFile + ".lock"); err != nil \|\| fi.Mode().Perm() != 0o600 {
		222	t.Fatalf("lock file: %v, %v", fi, err)
		223	}
		224	}

cmd/gitbayd/system.go +2 −2

	@@ -16,8 +16,8 @@ import (
16		16
17	// authorizedKeysCmd backs sshd's AuthorizedKeysCommand in system mode:	17	// authorizedKeysCmd backs sshd's AuthorizedKeysCommand in system mode:
18	//	18	//
19	// AuthorizedKeysCommand /usr/bin/gitbayd --config /etc/gitbay/config.toml authorized-keys %t %k	19	// AuthorizedKeysCommand /usr/local/bin/gitbayd --config /etc/gitbay/config.toml authorized-keys %t %k
20	// AuthorizedKeysCommandUser git	20	// AuthorizedKeysCommandUser gitbay
21	//	21	//
22	// It prints a forced-command authorized_keys line for registered keys and	22	// It prints a forced-command authorized_keys line for registered keys and
23	// nothing for unknown ones — so unknown keys fail authentication inside	23	// nothing for unknown ones — so unknown keys fail authentication inside

deploy/cloud-init.yaml +6 −1

 #   - opens ufw for 22, 80, 443, 2222
 #
 # It does NOT install the gitbayd binary (it is not hosted anywhere yet);
-# scp it to /usr/local/bin/gitbayd afterward and `systemctl start gitbayd`.
+# scp it to /usr/local/bin/gitbayd afterward, then create the secret key
+# and hand it to the daemon user before starting:
+#   gitbayd --config /etc/gitbay/config.toml admin secrets init
+#   chown gitbay:gitbay /etc/gitbay/secret.key
+#   systemctl start gitbayd
+# On a restore, put the key file's off-host copy there instead of init.
 package_update: true
 packages:

e2e/backup_test.go +4

 		t.Fatal("the archived database carries the build secret in clear")
 	}
+	if out := inst.admin(t, "admin", "backup", "--verify", archive); !strings.Contains(out, "connectivity ok on 1 repositories") {
+		t.Fatalf("verify: %s", out)
+	}
 	// Restore: extract into a fresh root and serve from it.
 	root2 := t.TempDir()
 	if outB, err := exec.Command("tar", "-xzf", archive, "-C", root2).CombinedOutput(); err != nil {

internal/backuplock/backuplock.go added +59

@@ -0,0 +1,59 @@
	1	// Package backuplock keeps repository deletes, renames, transfers and
	2	// prunes out of a full backup's way (#259). The backup runs in its own
	3	// process (gitbayd admin backup) and a delete in the daemon's, so the
	4	// lock is flock(2) on a file under server.root: the backup holds it
	5	// exclusively from its database snapshot until the last repository is
	6	// archived, and each delete, move or prune holds it shared while it runs.
	7	package backuplock
	8
	9	import (
	10	"errors"
	11	"os"
	12	"path/filepath"
	13	"syscall"
	14	)
	15
	16	// Name is the lock file under server.root. Backups skip it.
	17	const Name = "backup.lock"
	18
	19	// ErrBusy is TryShared's answer while a backup holds the lock.
	20	var ErrBusy = errors.New("a backup is running; repositories cannot be deleted, renamed, moved or pruned until it finishes, usually within minutes")
	21
	22	// open opens the lock file read-only, which is all flock needs, so the
	23	// daemon's user can lock a file a root-run backup created. O_NOFOLLOW
	24	// refuses a symlink planted at Name instead of following it, since the
	25	// path is inside server.root where only the daemon's own user writes.
	26	func open(root string) (*os.File, error) {
	27	return os.OpenFile(filepath.Join(root, Name), os.O_RDONLY\|os.O_CREATE\|syscall.O_NOFOLLOW, 0o644)
	28	}
	29
	30	// Hold takes the lock exclusively, waiting for deletes and moves under
	31	// way to finish. Closing the file releases it.
	32	func Hold(root string) (func(), error) {
	33	f, err := open(root)
	34	if err != nil {
	35	return nil, err
	36	}
	37	if err := syscall.Flock(int(f.Fd()), syscall.LOCK_EX); err != nil {
	38	f.Close()
	39	return nil, err
	40	}
	41	return func() { f.Close() }, nil
	42	}
	43
	44	// TryShared takes the lock shared without waiting: ErrBusy while a
	45	// backup holds it.
	46	func TryShared(root string) (func(), error) {
	47	f, err := open(root)
	48	if err != nil {
	49	return nil, err
	50	}
	51	if err := syscall.Flock(int(f.Fd()), syscall.LOCK_SH\|syscall.LOCK_NB); err != nil {
	52	f.Close()
	53	if errors.Is(err, syscall.EWOULDBLOCK) {
	54	return nil, ErrBusy
	55	}
	56	return nil, err
	57	}
	58	return func() { f.Close() }, nil
	59	}

internal/backuplock/backuplock_test.go added +69

@@ -0,0 +1,69 @@
	1	package backuplock
	2
	3	import (
	4	"errors"
	5	"testing"
	6	"time"
	7	)
	8
	9	func TestTrySharedRefusedWhileHeld(t *testing.T) {
	10	root := t.TempDir()
	11	release, err := Hold(root)
	12	if err != nil {
	13	t.Fatal(err)
	14	}
	15	if _, err := TryShared(root); !errors.Is(err, ErrBusy) {
	16	t.Fatalf("TryShared during a backup: %v", err)
	17	}
	18	release()
	19	r, err := TryShared(root)
	20	if err != nil {
	21	t.Fatalf("TryShared after the backup: %v", err)
	22	}
	23	r()
	24	}
	25
	26	func TestSharedHoldersCoexist(t *testing.T) {
	27	root := t.TempDir()
	28	a, err := TryShared(root)
	29	if err != nil {
	30	t.Fatal(err)
	31	}
	32	defer a()
	33	b, err := TryShared(root)
	34	if err != nil {
	35	t.Fatalf("second shared holder: %v", err)
	36	}
	37	b()
	38	}
	39
	40	// A backup waits for a delete already under way.
	41	func TestHoldWaitsForSharedHolder(t *testing.T) {
	42	root := t.TempDir()
	43	shared, err := TryShared(root)
	44	if err != nil {
	45	t.Fatal(err)
	46	}
	47	got := make(chan struct{})
	48	go func() {
	49	release, err := Hold(root)
	50	if err != nil {
	51	t.Error(err)
	52	close(got)
	53	return
	54	}
	55	close(got)
	56	release()
	57	}()
	58	select {
	59	case <-got:
	60	t.Fatal("Hold returned while a shared holder was in")
	61	case <-time.After(100 * time.Millisecond):
	62	}
	63	shared()
	64	select {
	65	case <-got:
	66	case <-time.After(5 * time.Second):
	67	t.Fatal("Hold never returned after the shared holder left")
	68	}
	69	}

internal/config/config.go +3 −3

internal/control/admin.go +8

	@@ -655,6 +655,14 @@ func runAdminMRPrune(c *Ctx, args []string) int {
655	mrs = append(mrs, mr)	655	mrs = append(mrs, mr)
656	}	656	}
657		657
		658	// The prune would remove objects a running full backup has listed
		659	// and not yet read.
		660	release, code := holdOffBackup(c)
		661	if code >= 0 {
		662	return code
		663	}
		664	defer release()
		665
658	// The record is written as each ref goes, not after the gc: a failure	666	// The record is written as each ref goes, not after the gc: a failure
659	// past this point leaves refs deleted, and the audit log and the MR	667	// past this point leaves refs deleted, and the audit log and the MR
660	// thread must say so. Re-running the same command finishes the job.	668	// thread must say so. Re-running the same command finishes the job.

internal/control/backuplock_test.go added +57

@@ -0,0 +1,57 @@
	1	package control
	2
	3	import (
	4	"strings"
	5	"testing"
	6
	7	"gitbay.org/gitbay/internal/backuplock"
	8	"gitbay.org/gitbay/internal/protocol"
	9	)
	10
	11	func TestRepoDeleteAndRenameRefusedDuringBackup(t *testing.T) {
	12	st, repo, uid := newQueueTestRepo(t)
	13	owner, err := st.UserByID(uid)
	14	if err != nil {
	15	t.Fatal(err)
	16	}
	17	root := t.TempDir()
	18	release, err := backuplock.Hold(root)
	19	if err != nil {
	20	t.Fatal(err)
	21	}
	22	for _, argv := range [][]string{
	23	{"repo", "rename", repo.Path(), "renamed"},
	24	{"repo", "delete", repo.Path(), "--yes"},
	25	} {
	26	c, errOut := pruneCtx(st, root, owner)
	27	if code := Dispatch(c, argv); code != protocol.ExitFailure \|\| !strings.Contains(errOut.String(), "a backup is running") {
	28	t.Fatalf("%v during a backup: exit %d, %s", argv, code, errOut)
	29	}
	30	}
	31	if got, err := st.RepoByID(repo.ID); err != nil \|\| got.Name != repo.Name {
	32	t.Fatalf("repository changed during a backup: %+v, %v", got, err)
	33	}
	34	release()
	35
	36	c, errOut := pruneCtx(st, root, owner)
	37	if code := Dispatch(c, []string{"repo", "delete", repo.Path(), "--yes"}); code != protocol.ExitOK {
	38	t.Fatalf("delete after the backup: exit %d, %s", code, errOut)
	39	}
	40	}
	41
	42	func TestAdminMRPruneRefusedDuringBackup(t *testing.T) {
	43	st, repo, root, headSHA := prunedRepo(t)
	44	dir := RepoDir(root, repo.OwnerName, repo.Name)
	45	release, err := backuplock.Hold(root)
	46	if err != nil {
	47	t.Fatal(err)
	48	}
	49	defer release()
	50	c, errOut := pruneCtx(st, root, rootUser(t, st))
	51	if code := Dispatch(c, []string{"admin", "mr", "prune", repo.Path(), "1", "--yes"}); code != protocol.ExitFailure \|\| !strings.Contains(errOut.String(), "a backup is running") {
	52	t.Fatalf("prune during a backup: exit %d, %s", code, errOut)
	53	}
	54	if !refExists(dir, mrHeadRef(1)) \|\| !objectExists(dir, headSHA) {
	55	t.Fatal("prune during a backup changed the repository")
	56	}
	57	}

internal/control/org.go +5

 	if _, err := os.Stat(newDir); err == nil {
 		return c.fail(protocol.ExitFailure, "repository directory %s already exists", newName)
 	}
+	release, lockCode := holdOffBackup(c)
+	if lockCode >= 0 {
+		return lockCode
+	}
+	defer release()
 	if err := c.Store.RenameOrg(org.ID, newName); err != nil {
 		return c.failErr(err)
 	}

internal/control/repo.go +28

 	"strconv"
 	"strings"
+	"gitbay.org/gitbay/internal/backuplock"
 	"gitbay.org/gitbay/internal/gitutil"
 	"gitbay.org/gitbay/internal/policy"
 	"gitbay.org/gitbay/internal/protocol"
 	if _, err := os.Stat(newDir); err == nil {
 		return c.fail(protocol.ExitFailure, "repository directory already exists at %s/%s", newOwner, repo.Name)
 	}
+	release, lockCode := holdOffBackup(c)
+	if lockCode >= 0 {
+		return lockCode
+	}
+	defer release()
 	// The directory moves before the record changes: a move that fails
 	// leaves nothing to undo, whereas the record's change into an org
 	// folds labels and milestones into the org's rows, which a revert
 	if _, err := os.Stat(newDir); err == nil {
 		return c.fail(protocol.ExitFailure, "repository directory already exists at %s/%s", repo.OwnerName, newName)
 	}
+	release, lockCode := holdOffBackup(c)
+	if lockCode >= 0 {
+		return lockCode
+	}
+	defer release()
 	if err := c.Store.RenameRepo(repo.ID, newName); err != nil {
 		return c.failErr(err)
 	}
 // webhooks; an instance that needs to hear about it wants the audit log
 // (#112).
 func deleteRepo(c *Ctx, repo store.Repo) int {
+	release, lockCode := holdOffBackup(c)
+	if lockCode >= 0 {
+		return lockCode
+	}
+	defer release()
 	// Open MRs sourced from this repo keep working (targets own the
 	// objects) but must show that the source is gone.
 	if err := c.Store.MarkSourceGoneForRepo(repo.ID); err != nil {
 	})
 }
+// holdOffBackup keeps a full backup from starting while a repository
+// directory moves or goes, and refuses while one runs: the backup's
+// database snapshot names every repository its walk then archives
+// (#259). The caller defers the returned release.
+func holdOffBackup(c *Ctx) (func(), int) {
+	release, err := backuplock.TryShared(c.Cfg.Server.Root)
+	if err != nil {
+		return nil, c.fail(protocol.ExitFailure, "%v", err)
+	}
+	return release, -1
+}
 func runAccessGrant(c *Ctx, args []string) int {
 	if len(args) != 3 || !slices.Contains([]string{"read", "write", "admin"}, args[2]) {
 		return c.usage()

internal/gitutil/fsck_test.go added +49

@@ -0,0 +1,49 @@
	1	package gitutil
	2
	3	import (
	4	"os"
	5	"os/exec"
	6	"path/filepath"
	7	"strings"
	8	"testing"
	9	)
	10
	11	func TestFsckConnectivityFindsAMissingObject(t *testing.T) {
	12	dir := t.TempDir()
	13	git(t, dir, "init", "-q", "-b", "main")
	14	write(t, dir, "a.txt", "a\n")
	15	git(t, dir, "add", "a.txt")
	16	git(t, dir, "commit", "-q", "-m", "one")
	17	gitDir := filepath.Join(dir, ".git")
	18	if err := FsckConnectivity(gitDir); err != nil {
	19	t.Fatalf("intact repository: %v", err)
	20	}
	21	out, err := exec.Command("git", "-C", dir, "rev-parse", "HEAD:a.txt").Output()
	22	if err != nil {
	23	t.Fatal(err)
	24	}
	25	blob := strings.TrimSpace(string(out))
	26	if err := os.Remove(filepath.Join(dir, ".git", "objects", blob[:2], blob[2:])); err != nil {
	27	t.Fatal(err)
	28	}
	29	if err := FsckConnectivity(gitDir); err == nil {
	30	t.Fatal("a repository missing a blob passed")
	31	}
	32	}
	33
	34	// A directory that is not a repository fails, even inside one: git does
	35	// not search upward and check the enclosing repository instead.
	36	func TestFsckConnectivityRefusesANonRepository(t *testing.T) {
	37	dir := t.TempDir()
	38	git(t, dir, "init", "-q", "-b", "main")
	39	write(t, dir, "a.txt", "a\n")
	40	git(t, dir, "add", "a.txt")
	41	git(t, dir, "commit", "-q", "-m", "one")
	42	inner := filepath.Join(dir, "repos", "x.git")
	43	if err := os.MkdirAll(inner, 0o755); err != nil {
	44	t.Fatal(err)
	45	}
	46	if err := FsckConnectivity(inner); err == nil {
	47	t.Fatal("an empty directory inside a repository passed")
	48	}
	49	}

internal/gitutil/merge.go +13

	@@ -66,6 +66,19 @@ func PruneNow(dir string) error {
66	return nil	66	return nil
67	}	67	}
68		68
		69	// FsckConnectivity checks that every object reachable from the
		70	// repository's refs is present, without reading blob contents. A backup
		71	// verify runs it on each archived repository (#259). dir is the git
		72	// directory itself; it is passed as --git-dir so that a directory that is
		73	// not a repository fails instead of git checking one enclosing it.
		74	func FsckConnectivity(dir string) error {
		75	cmd := exec.Command(toolpath.Look("git"), "--git-dir="+dir, "fsck", "--connectivity-only", "--no-progress", "--no-dangling")
		76	if out, err := cmd.CombinedOutput(); err != nil {
		77	return fmt.Errorf("fsck --connectivity-only: %v\n%s", err, out)
		78	}
		79	return nil
		80	}
		81
69	// RevListRange returns commits in old..new, newest first.	82	// RevListRange returns commits in old..new, newest first.
70	func RevListRange(dir, old, new string) ([]string, error) {	83	func RevListRange(dir, old, new string) ([]string, error) {
71	cmd := exec.Command(toolpath.Look("git"), "-C", dir, "rev-list", "--end-of-options", new, "^"+old)	84	cmd := exec.Command(toolpath.Look("git"), "-C", dir, "rev-list", "--end-of-options", new, "^"+old)

	@@ -359,12 +359,12 @@ func Load(path string) (Config, error) {
359	return cfg, cfg.Validate()	359	return cfg, cfg.Validate()
360	}	360	}
361		361
362	// within reports whether path is dir or below it. Both are compared as	362	// Within reports whether path is dir or below it. Both are compared as
363	// cleaned absolute paths (a relative path resolves against the working	363	// cleaned absolute paths (a relative path resolves against the working
364	// directory, same as every other path in this config), with symlinks	364	// directory, same as every other path in this config), with symlinks
365	// resolved where the path exists on disk, so a path that reaches into dir	365	// resolved where the path exists on disk, so a path that reaches into dir
366	// through a symlink, or through "..", is still reported as inside.	366	// through a symlink, or through "..", is still reported as inside.
367	func within(dir, path string) bool {	367	func Within(dir, path string) bool {
368	dir, path = resolvePath(dir), resolvePath(path)	368	dir, path = resolvePath(dir), resolvePath(path)
369	rel, err := filepath.Rel(dir, path)	369	rel, err := filepath.Rel(dir, path)
370	return err == nil && rel != ".." && !strings.HasPrefix(rel, ".."+string(filepath.Separator))	370	return err == nil && rel != ".." && !strings.HasPrefix(rel, ".."+string(filepath.Separator))
	@@ -430,7 +430,7 @@ func (c Config) Validate() error {
430	switch {	430	switch {
431	case c.Server.SecretKeyFile == "":	431	case c.Server.SecretKeyFile == "":
432	errs = append(errs, errors.New("server.secret_key_file is required"))	432	errs = append(errs, errors.New("server.secret_key_file is required"))
433	case within(c.Server.Root, c.Server.SecretKeyFile):	433	case Within(c.Server.Root, c.Server.SecretKeyFile):
434	errs = append(errs, fmt.Errorf("server.secret_key_file %q is inside server.root: backups of the root would carry the key beside the values it seals", c.Server.SecretKeyFile))	434	errs = append(errs, fmt.Errorf("server.secret_key_file %q is inside server.root: backups of the root would carry the key beside the values it seals", c.Server.SecretKeyFile))
435	}	435	}
436	if err := oneOf("ssh.mode", c.SSH.Mode, "embedded", "system"); err != nil {	436	if err := oneOf("ssh.mode", c.SSH.Mode, "embedded", "system"); err != nil {