backup: hold moves off, refs before objects, directory entries, verify connectivity !504

merged merged by cmc on 2026-09-28 23:09 UTC · krz/gitbay:backup-verify-lock into main

23 files changed, +1394 −115

Layout: unified · split

.gitbay/wiki/Admin.org +81 −10

		@@ -444,12 +444,46 @@ One archive: a consistent SQLite snapshot (taken before the
444	444	repositories are read, so the database never references objects the
445	445	archive missed), every repository, and the SSH host keys. Excluded:
446	446	hook socket, regenerated hook scripts, WAL files. Safe to run against a
447		live daemon.
	447	live daemon. =--out= must be outside =server.root=, or the next full
	448	backup would carry the archive. Each run removes the snapshot
	449	directories (=.gitbay-snap-=) and temporary archives (=..tmp-*=) a
	450	killed run left beside its archive once they are a day old, and prints
	451	each one it removes.
448	452
449	453	=--verify= reads an archive back: the snapshot must pass SQLite's
450		integrity check, and every repository the snapshot names must be in the
451		archive. A database-only archive is checked for integrity and says so.
452		Exit is non-zero on damage or a missing repository.
	454	integrity check, every repository the snapshot names must be in the
	455	archive, and each must pass =git fsck --connectivity-only=. It
	456	extracts the repositories to a temporary directory for that, so it
	457	needs free space the size of the repositories. A database-only archive
	458	is checked for integrity and says so. Exit is non-zero on damage, a
	459	missing repository or a missing object.
	460
	461	A full backup holds =<root>/backup.lock= from its database snapshot to
	462	its last repository. While it runs, =repo delete=, =repo rename=,
	463	=repo transfer=, =admin repo delete=, =org rename=, =admin mr prune=
	464	and =gitbayd admin gc= refuse with "a backup is running"; retry when it
	465	finishes. Database-only backups take no lock. A pack that git's own
	466	automatic gc removes during the walk is skipped; each repository's refs
	467	are archived before its objects, so the refs still find their objects,
	468	and =--verify= reports it if one does not.
	469
	470	Verifying an archive of unknown origin: run it as an unprivileged
	471	user. The connectivity check runs git with =--git-dir= on each
	472	extracted repository, so a directory that is not a repository fails
	473	rather than git checking an enclosing one. =objects/info/alternates=
	474	and a =commondir= directly in a =*.git= directory are not extracted,
	475	so an archive cannot use either to have git read another repository's
	476	objects or refs on the host. Git still reads each archived
	477	repository's own =config=.
	478
	479	Archives carry a directory entry for every directory, including an
	480	empty one, so a bare repository whose refs are all packed restores as
	481	a repository. Archives written before this release do not: extracting
	482	one can leave a repository's =refs/= directory missing, which stops
	483	git from recognizing it as a repository at all. =gitbayd admin backup
	484	--verify <archive>= names the repositories this affects; the fix is
	485	=mkdir -p <root>/repos/<owner>/<name>.git/refs= for each one, after
	486	which it opens normally.
453	487
454	488	With =[backup] age_recipients= set the archive is =<name>.tar.gz.age=
455	489	and =--verify= needs the private key:
		@@ -464,8 +498,11 @@ restic credentials), so verifying an encrypted archive happens there
464	498	or on a restore host.
465	499
466	500	Restore: extract into an empty directory, point =server.root= at it,
467		start gitbayd. Host keys are preserved, so clients keep their
468		known_hosts entries; hooks regenerate at startup.
	501	restore =server.secret_key_file= from its own copy (mode 0600, owned
	502	by the account gitbayd runs as), then start gitbayd. No archive carries
	503	the key file, and without it gitbayd refuses to start. Host keys are
	504	preserved, so clients keep their known_hosts entries; hooks regenerate
	505	at startup.
469	506
470	507	** Schedule and recovery point
471	508
		@@ -493,8 +530,10 @@ too.
493	530	** Offsite copies
494	531
495	532	bay1 also takes a nightly restic snapshot of =/var/lib/gitbay= and
496		=/var/lib/gitbay-stage= (the staged database copy) to an S3 bucket at
497		Scaleway, with a key that can only add snapshots. The key that can
	533	=/var/lib/gitbay-stage= (a database copy and =config.toml=, staged
	534	there) to an S3 bucket at Scaleway, with a key that can only add
	535	snapshots. Nothing else under =/etc/gitbay= is in it: not the secret
	536	key file, not =apns.p8=. The key that can
498	537	remove them lives on the operator's machine, in
499	538	=~/.config/gitbay/offsite.env=, and never on bay1: a compromised host
500	539	cannot destroy its own history. Forgetting, pruning and rewriting all
		@@ -543,7 +582,10 @@ each value prefixed with the id of the key that sealed it
543	582	=server.root=, and therefore in neither the local archives nor the
544	583	main restic repository. It must be copied off the host separately;
545	584	without it a restored database's secrets cannot be opened, and
546		gitbayd refuses to start against them.
	585	gitbayd refuses to start against them. A separate restic repository
	586	for it and =apns.p8= is planned (runbook D of the data-at-rest plan)
	587	and not yet in place, so today the only off-host copy is one the
	588	operator makes by hand after =init= and after every =rotate=.
547	589
548	590	#+begin_src sh
549	591	gitbayd admin secrets init # once; deploy/install.sh does it on first install
		@@ -554,6 +596,8 @@ gitbayd admin secrets rotate # new key, reseal, retire the old one (as root)
554	596	- Missing file: every gitbayd process that opens the database refuses
555	597	to run and names the path, including =serve= and, in system mode,
556	598	=authorized-keys=. =migrate= does not need it.
	599	- Backups: =admin backup= and =admin backup --verify= do not need the
	600	key file; a restore does.
557	601	- Wrong key: =serve= stops at startup naming the first row that does
558	602	not open; =secrets check= does the same without starting anything.
559	603	- Upgrade: the first start after the upgrade seals every value still
		@@ -562,10 +606,37 @@ gitbayd admin secrets rotate # new key, reseal, retire the old one (as root)
562	606	transaction, then removes the old keys. Run it as root, since it
563	607	replaces the key file in =/etc/gitbay=; the file keeps its owner.
564	608	The daemon re-reads the file when it changes, so it needs no
565		restart. Copy the new file off the host afterwards.
	609	restart. Copy the new file off the host afterwards. =init= and
	610	=rotate= hold an flock on =<key file>.lock= while they run, so a
	611	second run waits for the first.
566	612	- Push devices are looked up by the SHA-256 of their token
567	613	(=push_devices.token_hash=), since two seals of one token differ.
568	614
	615	** Restore drill
	616
	617	A restore onto a clean host, run quarterly and after any change to the
	618	backup code (=cmd/gitbayd/backup.go=, the offsite job), and recorded
	619	below. The disaster it rehearses is losing bay1, so the local archives
	620	are gone with it and the sources are the main offsite restic
	621	repository (repositories, LFS, the staged database, =config.toml=),
	622	the off-host copy of =secret.key= and =apns.p8= (a keys repository once
	623	runbook D creates it; until then the operator's hand-made copy), and
	624	the operator's password manager (=offsite.env=, the keys repository's
	625	password and token once it exists, =backup-identity.txt=). The steps are in the data-at-rest
	626	plan's operator runbook
	627	(=docs/plans/2026-09-27-data-at-rest-and-backup.md=).
	628
	629	Time to service runs from the clean host's first root login to the
	630	first successful =git clone= over SSH from it. The recovery point is
	631	the time of the newest restic snapshot restored.
	632
	633	No drill has been run yet; the procedure above is written but
	634	unexercised, and #259 stays open until the first row below is
	635	recorded.
	636
	637	\| Date \| Host \| Snapshot restored (UTC) \| Time to service \| DB integrity \| Connectivity \| LFS \| Release assets \| Host key \| Secrets \| Notes \|
	638	\|------+------+-------------------------+-----------------+--------------+--------------+-----+----------------+----------+---------+-------\|
	639
569	640	* Upgrades
570	641
571	642	Replace the binary, restart the unit. Migrations apply automatically and

.gitbay/wiki/Architecture/06-Data-and-Cryptography.org +1 −1

		@@ -45,7 +45,7 @@ throttling (=internal/sshd/sshd.go=).
45	45	\| API tokens, sessions, login links, email codes, invites \| SHA-256 of a 256-bit random value; the value is shown once and never stored (=internal/store/sessions.go=) \|
46	46	\| CI secrets, webhook secrets, mirror tokens, APNs device tokens \| AES-256-GCM under a key file outside the database and outside =server.root=; additional data binds table, column and row (=internal/seal=, =internal/store/secrets.go=) \|
47	47	\| SQLite file \| mode 0640, directory 0750 \|
48		\| Backups \| local archives age-encrypted when =[backup] age_recipients= is set; restic encrypts the offsite copy; neither carries the secret key file, whose offsite copy is a separate keys repository \|
	48	\| Backups \| local archives age-encrypted when =[backup] age_recipients= is set; restic encrypts the offsite copy; neither carries the secret key file, which is copied off the host by hand until a separate keys repository is set up \|
49	49	\| Disk \| no application-level encryption; any disk encryption is the host's \|
50	50
51	51	The database file or a backup read by anyone other than the =gitbay=

.gitbay/wiki/Architecture/08-Operations.org +12 −7

		@@ -52,21 +52,26 @@ the product activity feed, not an audit trail.
52	52	\| Database only \| hourly \| 48 \| SQLite snapshot; age-encrypted when =[backup] age_recipients= is set \|
53	53	\| Offsite (restic)\| nightly \| per prune policy \| =/var/lib/gitbay= and a staged database copy, to object storage \|
54	54
55		- The database snapshot is taken before repositories are read, so a
56		push during the backup leaves only unreferenced objects
	55	- The database snapshot is taken before repositories are read, and
	56	each repository's HEAD, refs/ and packed-refs are archived before its
	57	objects, so every archived ref finds the objects it reaches, unless
	58	git's own automatic gc after a push repacks during the walk; the
	59	archive can then miss objects, and =--verify= reports it. A push
	60	during the backup is missing or present as unreferenced objects
57	61	(=cmd/gitbayd/backup.go=).
58	62	- Excluded: WAL files, the hook socket, askpass scripts, generated
59	63	hooks.
60		- =gitbayd admin backup --verify= checks SQLite integrity and that every
61		repository the database names is present (=backup.go=). It does
62		not check git object connectivity.
	64	- =gitbayd admin backup --verify= checks SQLite integrity, that every
	65	repository the database names is present, and =git fsck
	66	--connectivity-only= on each (=backup.go=).
	67	- Repository deletes, renames and transfers refuse while a full backup
	68	runs (=internal/backuplock=), so the snapshot and the walk agree.
63	69	- The host's restic credentials are append-only; the key that can
64	70	delete or prune snapshots is held off the host, so a compromised host
65	71	cannot destroy its own history (documented: Admin wiki).
66	72	- Recovery point: about one hour for database-only data (issues, merge
67	73	requests, reviews), one day for repositories.
68		- Recovery time: not measured. No restore onto a clean host has been
69		recorded (#259).
	74	- Recovery time: see the Admin wiki's Restore drill table.
70	75
71	76	Restore procedure: extract the archive into an empty directory, point
72	77	=server.root= at it, start =gitbayd=; hooks regenerate and the host key

.gitbay/wiki/Architecture/10-Known-Gaps.org +1 −1

		@@ -10,7 +10,7 @@ what the 2026-09-27 review found; remove a row when its issue closes.
10	10
11	11	\| Issue \| Area \| Gap \| Severity \|
12	12	\|-------+------------------+-----------------------------------------------------------------------+----------\|
13		\| #259 \| Recovery \| No restore has been exercised; verification does not check git connectivity \| high \|
	13	\| #259 \| Recovery \| No restore has been exercised; the drill is written (Admin wiki) and not yet run \| high \|
14	14	\| #260 \| CI network \| Builds share the runner's source address; no egress policy \| medium \|
15	15	\| #261 \| Various \| Migration foreign-key check after commit; three web writes bypass dispatch; documentation drift \| medium \|
16	16	\| #262 \| Availability \| No limit on concurrent git pack generation \| high \|

.gitbay/wiki/Threat-Model.org +8 −3

		@@ -262,9 +262,14 @@ assume has been checked.
262	262	pixel against a viewer. A profile is the wider surface of the two: it
263	263	is linked from every commit and issue its owner touches. Documented;
264	264	proxying is future work.
265		- Backups are consistent per the DB-snapshot-first ordering but are not a
266		single atomic snapshot; a few orphaned git objects are possible and
267		harmless (see [[Admin]]).
	265	- Backups snapshot the database first, and repository deletes and
	266	moves wait out a full backup. Each repository's refs are archived
	267	before its objects, so every archived ref finds the objects it
	268	reaches, unless git's own automatic gc after a push repacks during
	269	the walk: the archive can then miss objects, and =--verify= reports
	270	it. A push during a backup may be missing from the archive, or
	271	present as objects no archived ref names, and a repository's refs may
	272	be newer than the database snapshot (see [[Admin]]).
268	273	- The audit log lives in the database the daemon writes, so anyone with
269	274	the daemon user's access can change it. The hash chain makes an edited
270	275	or removed row show as a break under =gitbayd admin audit verify=,

CHANGELOG.org +51 −16

		@@ -29,6 +29,29 @@ timeout (#256, #257, #276, #277).
29	29	=--scope full=. A script that mints a token and then writes with it
30	30	must add =--scope full=. Existing tokens keep their scope.
31	31
	32	Upgrade note. Upgrade the instance before the CLI: an older server
	33	refuses the CLI's leading =--path== argument as an unknown command,
	34	for the eighteen commands whose CLI path differs from the registry's
	35	(the =gitbay auth ...= commands and =repo topics list=).
	36
	37	Upgrade note. gitbayd needs =server.secret_key_file= (default
	38	=/etc/gitbay/secret.key=) and refuses to start without it. Before
	39	replacing the binary, run =gitbayd admin secrets init= as root and
	40	=chown gitbay:gitbay /etc/gitbay/secret.key= (=deploy/install.sh= does
	41	both when the file is missing). The first start seals the stored
	42	secrets. Back the key file up separately: =admin backup= archives do
	43	not carry it (see the Admin wiki, "Secret key"). Downgrading to an
	44	earlier release after values are sealed is not supported: an older
	45	gitbayd reads a sealed value's =gbs1:...= prefix as the literal
	46	secret.
	47
	48	Upgrade note. Archives now carry a directory entry for every
	49	directory, so a bare repository whose refs are all packed restores as
	50	a repository. An archive written before this release lacks those
	51	entries; if extracting one leaves a repository's =refs/= directory
	52	missing, =gitbayd admin backup --verify <archive>= names it, and
	53	=mkdir -p <root>/repos/<owner>/<name>.git/refs= fixes it.
	54
32	55	- A token with a =--ttl= is refused on every command that creates a
33	56	credential: tokens, keys, deploy keys, runner keys, login links,
34	57	invites, accounts and verified addresses (#257).
		@@ -85,10 +108,6 @@ must add =--scope full=. Existing tokens keep their scope.
85	108	separately (#275).
86	109	- Audit retention deletes by id, up to the newest row older than the
87	110	retention, so a clock step back cannot leave a gap in the chain (#275).
88		Upgrade note. Upgrade the instance before the CLI: an older server
89		refuses the CLI's leading =--path== argument as an unknown command,
90		for the eighteen commands whose CLI path differs from the registry's
91		(the =gitbay auth ...= commands and =repo topics list=).
92	111	- =dashboard= and =feed= print activity as sentences
93	112	(=cmc opened issue krz/gitbay#12=) instead of raw event payloads,
94	113	and a labelled event names its labels there and on the web feed. An
		@@ -137,18 +156,6 @@ for the eighteen commands whose CLI path differs from the registry's
137	156	previous" link on each revision after the first, so a reviewer whose
138	157	approval a force-push staled can see what changed without leaving the
139	158	browser (#269).
140		- Untrusted builds (merge requests from forks) get a fresh HOME removed after the build and no secrets; trusted builds keep a per-repository home under =<workdir>/trusted-home=. Deploy gitbayd before the runner; the old shared homes under the runner's workdir can be deleted. (#255)
141		- =status set= refuses =ci/= contexts, which belong to the instance's builds. Build results are reused only from trusted builds on the same image. =repo settings require-contexts= names status contexts that must report green; setting any turns require-checks on, and one not yet reported counts as pending. (#258)
142		Upgrade note. gitbayd needs =server.secret_key_file= (default
143		=/etc/gitbay/secret.key=) and refuses to start without it. Before
144		replacing the binary, run =gitbayd admin secrets init= as root and
145		=chown gitbay:gitbay /etc/gitbay/secret.key= (=deploy/install.sh= does
146		both when the file is missing). The first start seals the stored
147		secrets. Back the key file up separately: =admin backup= archives do
148		not carry it (see the Admin wiki, "Secret key"). Downgrading to an
149		earlier release after values are sealed is not supported: an older
150		gitbayd reads a sealed value's =gbs1:...= prefix as the literal
151		secret.
152	159	- CI secrets, webhook secrets, mirror tokens and push device tokens are
153	160	stored sealed with AES-256-GCM (#273). =gitbayd admin secrets
154	161	init\|rotate\|check=.
		@@ -180,6 +187,34 @@ secret.
180	187	- =gitbayd admin backup= encrypts archives to =[backup] age_recipients=
181	188	when set (#274); =--verify= takes =--identity <file>=. Archive names
182	189	gain =.age=; the shipped backup scripts and monitor match both.
	190	- A full backup holds =<root>/backup.lock= from its database snapshot
	191	to its last repository; =repo delete=, =repo rename=, =repo
	192	transfer=, =admin repo delete= and =org rename= refuse with "a
	193	backup is running" while it runs. =--verify= now also runs =git
	194	fsck --connectivity-only= on each archived repository and names any
	195	that fail. (#259)
	196	- A full backup archives each repository's HEAD, =refs/= and
	197	=packed-refs= before its objects, so a push during the backup cannot
	198	leave an archived ref naming objects the archive lacks. The exception
	199	is git's own automatic gc after a push repacking during the walk: the
	200	archive can then miss objects, and =--verify= reports it. (#259)
	201	- =gitbayd admin gc= and =admin mr prune= refuse with "a backup is
	202	running" during a full backup. A pack or loose object that git's
	203	automatic gc removes while the backup walks is skipped instead of
	204	failing the run; =--verify= reports it if a ref needed it. (#259)
	205	- =gitbayd admin backup= and =--verify= no longer need the secret key
	206	file: sealed values are copied as they are. A missing database is
	207	refused instead of created. (#259)
	208	- =gitbayd admin backup= refuses an =--out= inside =server.root=, and
	209	removes the snapshot directories and temporary archives a killed run
	210	left beside its archive once they are a day old. (#259)
	211	- =gitbayd admin backup --verify= runs fsck with =--git-dir=, so a
	212	directory that is not a repository fails instead of git checking an
	213	enclosing one, and does not extract =objects/info/alternates= or a
	214	repository's =commondir=, so neither can point fsck at another
	215	repository on the host. (#259)
	216	- =gitbayd admin secrets init= and =rotate= hold an flock on =<key
	217	file>.lock=, so two runs at once serialize. (#273)
183	218
184	219	* v1.36.0 — 2026-09-23
185	220

cmd/gitbayd/backup.go +295 −33

		@@ -4,18 +4,23 @@ import (
4	4	"archive/tar"
5	5	"bufio"
6	6	"compress/gzip"
	7	"errors"
7	8	"fmt"
8	9	"io"
9	10	"io/fs"
10	11	"os"
	12	"path"
11	13	"path/filepath"
	14	"regexp"
12	15	"strings"
13	16	"time"
14	17
15	18	"filippo.io/age"
16	19	"github.com/spf13/cobra"
17	20
	21	"gitbay.org/gitbay/internal/backuplock"
18	22	"gitbay.org/gitbay/internal/config"
	23	"gitbay.org/gitbay/internal/gitutil"
19	24	"gitbay.org/gitbay/internal/store"
20	25	)
21	26
		@@ -23,10 +28,11 @@ import (
23	28	// every repository and the SSH host keys. Restore by extracting the archive
24	29	// into a fresh server.root.
25	30	//
26		// Ordering: the database is snapshotted BEFORE the repositories are read.
27		// A push that lands mid-backup then shows up only as unreferenced git
28		// objects in the archive (harmless); the reverse order could leave database
29		// rows pointing at objects the archive never captured.
	31	// Ordering: the database is snapshotted BEFORE the repositories are read,
	32	// and each repository's refs before its objects. A push that lands
	33	// mid-backup then shows up only as unreferenced git objects in the archive
	34	// (harmless) or not at all; the reverse order could leave database rows or
	35	// refs pointing at objects the archive never captured.
30	36	func backupCmd() *cobra.Command {
31	37	var out, verify, identity string
32	38	var dbOnly bool
		@@ -43,8 +49,11 @@ affordable, and the database is the copy of issues, merge requests and
43	49	comments that exists nowhere else. Repositories are not in such an archive,
44	50	so it supplements a full backup and does not replace one.
45	51
46		Restore: extract into an empty directory, point server.root at it, start
47		gitbayd. Host keys are preserved, so clients keep their known_hosts entries.
	52	Restore: extract into an empty directory, point server.root at it,
	53	restore server.secret_key_file from its own backup (mode 0600, owned by
	54	the daemon user), start gitbayd. No archive carries the key file, and
	55	without it gitbayd refuses to start. Host keys are preserved, so clients
	56	keep their known_hosts entries.
48	57
49	58	With [backup] age_recipients set, the archive is encrypted to those age
50	59	public keys and its name ends in .age. --verify then needs --identity
		@@ -62,7 +71,7 @@ public keys and its name ends in .age. --verify then needs --identity
62	71	}
63	72	cmd.Flags().StringVar(&out, "out", "", "output archive path (default gitbay-backup-<utc timestamp>.tar.gz; .age is appended when [backup] age_recipients is set)")
64	73	cmd.Flags().BoolVar(&dbOnly, "db-only", false, "archive the database snapshot alone, without repositories")
65		cmd.Flags().StringVar(&verify, "verify", "", "check an archive instead of writing one: database integrity, and its repositories against the archive's")
	74	cmd.Flags().StringVar(&verify, "verify", "", "check an archive instead of writing one: database integrity, its repositories against the archive's, and git connectivity of each")
66	75	cmd.Flags().StringVar(&identity, "identity", "", "with --verify: an age identity file that opens an encrypted archive")
67	76	return cmd
68	77	}
		@@ -90,7 +99,32 @@ func runBackup(cfg config.Config, out string, dbOnly bool) error {
90	99	return fmt.Errorf("%s ends in .age but [backup] age_recipients is not set, so the archive would not be encrypted", out)
91	100	}
92	101
93		st, err := openStore(cfg)
	102	dir := filepath.Dir(out)
	103	// An archive under the root would be in the next full backup's walk.
	104	if config.Within(cfg.Server.Root, dir) {
	105	return fmt.Errorf("%s is inside server.root %s; write the archive elsewhere", out, cfg.Server.Root)
	106	}
	107	removeStale(dir, time.Now().Add(-staleAge))
	108
	109	// Deletes, renames and transfers wait until the walk finishes, so
	110	// every repository the snapshot names is still on disk when the walk
	111	// reaches it (#259). A database-only archive reads no repository.
	112	if !dbOnly {
	113	release, err := backuplock.Hold(cfg.Server.Root)
	114	if err != nil {
	115	return fmt.Errorf("backup lock: %w", err)
	116	}
	117	defer release()
	118	}
	119
	120	// VACUUM INTO copies sealed values as they are, so the backup needs
	121	// no key file, and it migrates nothing. store.Open would create a
	122	// missing database, so its absence is checked first.
	123	dbFile := filepath.Join(cfg.Server.Root, "gitbay.db")
	124	if _, err := os.Stat(dbFile); err != nil {
	125	return fmt.Errorf("database: %w", err)
	126	}
	127	st, err := store.Open(dbFile)
94	128	if err != nil {
95	129	return err
96	130	}
		@@ -98,7 +132,6 @@ func runBackup(cfg config.Config, out string, dbOnly bool) error {
98	132
99	133	// 1. Consistent database snapshot, before any repository is read. It
100	134	// goes in a fresh 0700 directory beside the archive.
101		dir := filepath.Dir(out)
102	135	snapDir, err := os.MkdirTemp(dir, ".gitbay-snap-")
103	136	if err != nil {
104	137	return err
		@@ -142,18 +175,22 @@ func runBackup(cfg config.Config, out string, dbOnly bool) error {
142	175	skip := map[string]bool{
143	176	"gitbay.db": true, "gitbay.db-wal": true, "gitbay.db-shm": true,
144	177	"hook.sock": true, "askpass.sh": true, "hooks": true,
	178	backuplock.Name: true,
145	179	}
146	180	repoCount := 0
147	181	root := cfg.Server.Root
148	182	if !dbOnly {
149		err = filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error {
150		if err != nil {
151		return err
152		}
	183	err = filepath.WalkDir(root, func(path string, d fs.DirEntry, walkErr error) error {
153	184	rel, err := filepath.Rel(root, path)
154	185	if err != nil {
155	186	return err
156	187	}
	188	if walkErr != nil {
	189	if vanished(walkErr, rel) {
	190	return nil
	191	}
	192	return walkErr
	193	}
157	194	if rel == "." {
158	195	return nil
159	196	}
		@@ -166,13 +203,38 @@ func runBackup(cfg config.Config, out string, dbOnly bool) error {
166	203	if !d.Type().IsRegular() && !d.IsDir() {
167	204	return nil // sockets, symlinks
168	205	}
	206	// A repository's refs were archived on entering it.
	207	if strings.HasSuffix(filepath.Dir(rel), ".git") && refNames[d.Name()] {
	208	if d.IsDir() {
	209	return filepath.SkipDir
	210	}
	211	return nil
	212	}
169	213	if d.IsDir() {
	214	// A directory entry, even for one that holds no file (a
	215	// bare repository's refs/heads and refs/tags once every
	216	// ref is packed), so extraction recreates it: git's own
	217	// repository discovery needs refs/ to exist.
	218	if err := addDir(tw, path, filepath.ToSlash(rel)); err != nil {
	219	if vanished(err, rel) {
	220	return filepath.SkipDir
	221	}
	222	return err
	223	}
170	224	if strings.HasSuffix(rel, ".git") {
171	225	repoCount++
	226	if err := addRefs(tw, path, filepath.ToSlash(rel)); err != nil {
	227	return err
	228	}
	229	afterRefs(path)
172	230	}
173		return nil // directories are implied by member paths
	231	return nil
174	232	}
175		return addFile(tw, path, filepath.ToSlash(rel))
	233	beforeAdd(path)
	234	if err := addFile(tw, path, filepath.ToSlash(rel)); !vanished(err, rel) {
	235	return err
	236	}
	237	return nil
176	238	})
177	239	if err != nil {
178	240	return err
		@@ -212,6 +274,129 @@ func runBackup(cfg config.Config, out string, dbOnly bool) error {
212	274	return nil
213	275	}
214	276
	277	// staleAge is how old a snapshot directory or temporary archive must be
	278	// before a later run removes it. A run that is still writing one is
	279	// younger than this.
	280	const staleAge = 24 * time.Hour
	281
	282	// tmpArchive is a temporary archive's name: os.CreateTemp's pattern
	283	// "."+base+".tmp-" followed by the digits it appends.
	284	var tmpArchive = regexp.MustCompile(`^\..+\.tmp-[0-9]+$`)
	285
	286	// removeStale removes what a killed run left in dir: snapshot
	287	// directories and temporary archives last modified before cutoff.
	288	func removeStale(dir string, cutoff time.Time) {
	289	ents, err := os.ReadDir(dir)
	290	if err != nil {
	291	return
	292	}
	293	for _, e := range ents {
	294	name := e.Name()
	295	snap := e.IsDir() && strings.HasPrefix(name, ".gitbay-snap-")
	296	tmp := e.Type().IsRegular() && tmpArchive.MatchString(name)
	297	if !snap && !tmp {
	298	continue
	299	}
	300	info, err := e.Info()
	301	if err != nil \|\| !info.ModTime().Before(cutoff) {
	302	continue
	303	}
	304	p := filepath.Join(dir, name)
	305	if err := os.RemoveAll(p); err != nil {
	306	fmt.Fprintf(os.Stderr, "removing stale %s: %v\n", p, err)
	307	continue
	308	}
	309	fmt.Fprintf(os.Stderr, "removed stale %s\n", p)
	310	}
	311	}
	312
	313	// refNames are what a repository's refs are read from. WalkDir would
	314	// reach objects/ before packed-refs and refs/, so a push landing mid-walk
	315	// could leave an archived ref naming objects the archive lacks. addRefs
	316	// archives these first on entering the repository; objects are only ever
	317	// added, so the walk that follows finds every object those refs reach.
	318	var refNames = map[string]bool{"HEAD": true, "packed-refs": true, "refs": true}
	319
	320	// afterRefs runs between a repository's refs and the rest of it. Tests
	321	// use it to write into the repository at that point.
	322	var afterRefs = func(repo string) {}
	323
	324	// addRefs archives HEAD, refs/ and packed-refs of the repository at
	325	// path, whichever exist. refs/ is read before packed-refs, the order git
	326	// reads them in: pack-refs writes packed-refs before deleting the loose
	327	// refs it packed, so a ref moving between the two is caught in one.
	328	func addRefs(tw *tar.Writer, path, name string) error {
	329	if err := addRegular(tw, path, name, "HEAD"); err != nil {
	330	return err
	331	}
	332	refs := filepath.Join(path, "refs")
	333	if _, err := os.Lstat(refs); err == nil {
	334	if err := addTree(tw, path, name, refs); err != nil {
	335	return err
	336	}
	337	} else if !errors.Is(err, fs.ErrNotExist) {
	338	return err
	339	}
	340	return addRegular(tw, path, name, "packed-refs")
	341	}
	342
	343	// addRegular archives the regular file f in the repository at path, if
	344	// it exists.
	345	func addRegular(tw *tar.Writer, path, name, f string) error {
	346	fi, err := os.Lstat(filepath.Join(path, f))
	347	if errors.Is(err, fs.ErrNotExist) \|\| err == nil && !fi.Mode().IsRegular() {
	348	return nil
	349	}
	350	if err != nil {
	351	return err
	352	}
	353	return addFile(tw, filepath.Join(path, f), name+"/"+f)
	354	}
	355
	356	// addTree archives the directory refs inside the repository at path.
	357	func addTree(tw *tar.Writer, path, name, refs string) error {
	358	return filepath.WalkDir(refs, func(p string, d fs.DirEntry, err error) error {
	359	if err != nil {
	360	return err
	361	}
	362	rel, err := filepath.Rel(path, p)
	363	if err != nil {
	364	return err
	365	}
	366	member := name + "/" + filepath.ToSlash(rel)
	367	switch {
	368	case d.IsDir():
	369	return addDir(tw, p, member)
	370	case d.Type().IsRegular():
	371	return addFile(tw, p, member)
	372	}
	373	return nil
	374	})
	375	}
	376
	377	// beforeAdd runs before each file the walk archives outside refs. Tests
	378	// use it to remove a file between listing and reading.
	379	var beforeAdd = func(path string) {}
	380
	381	// vanished reports a file or directory under a repository's objects/
	382	// that went between the walk listing it and reading it: a pack or loose
	383	// object a concurrent gc or receive.autogc removed. The walk skips it.
	384	// Refs archived earlier reach only objects that are still reachable, and
	385	// a repack writes those into a new pack before removing the old one; if
	386	// one is lost regardless, verify's fsck reports it.
	387	func vanished(err error, rel string) bool {
	388	if !errors.Is(err, fs.ErrNotExist) {
	389	return false
	390	}
	391	parts := strings.Split(filepath.ToSlash(rel), "/")
	392	for i := 0; i+2 < len(parts); i++ {
	393	if strings.HasSuffix(parts[i], ".git") && parts[i+1] == "objects" {
	394	return true
	395	}
	396	}
	397	return false
	398	}
	399
215	400	// syncDir makes a rename in dir durable.
216	401	func syncDir(dir string) error {
217	402	d, err := os.Open(dir)
		@@ -230,8 +415,15 @@ func snapshotDB(st *store.Store, dest string) error {
230	415	return err
231	416	}
232	417
	418	// addFile opens before writing the header, so a file removed after the
	419	// walk listed it fails before the archive has a member for it.
233	420	func addFile(tw *tar.Writer, path, name string) error {
234		info, err := os.Stat(path)
	421	src, err := os.Open(path)
	422	if err != nil {
	423	return err
	424	}
	425	defer src.Close()
	426	info, err := src.Stat()
235	427	if err != nil {
236	428	return err
237	429	}
		@@ -243,20 +435,33 @@ func addFile(tw *tar.Writer, path, name string) error {
243	435	if err := tw.WriteHeader(hdr); err != nil {
244	436	return err
245	437	}
246		src, err := os.Open(path)
	438	_, err = io.CopyN(tw, src, hdr.Size)
	439	return err
	440	}
	441
	442	// addDir writes a directory entry, so an empty directory survives
	443	// extraction. The mode never exceeds 0755, whatever the source directory
	444	// carries.
	445	func addDir(tw *tar.Writer, path, name string) error {
	446	info, err := os.Stat(path)
247	447	if err != nil {
248	448	return err
249	449	}
250		defer src.Close()
251		_, err = io.Copy(tw, src)
252		return err
	450	hdr, err := tar.FileInfoHeader(info, "")
	451	if err != nil {
	452	return err
	453	}
	454	hdr.Name = name + "/"
	455	hdr.Mode = hdr.Mode&^0o777 \| hdr.Mode&0o755
	456	return tw.WriteHeader(hdr)
253	457	}
254	458
255	459	// verifyBackup reads an archive back, decrypting it with identity when it
256		// is encrypted: the database snapshot must pass
257		// SQLite's integrity check, and every repository it names must be in the
258		// archive. A database-only archive is checked for integrity alone and
259		// says so. Nothing is written except a temporary copy of the database.
	460	// is encrypted: the database snapshot must pass SQLite's integrity check,
	461	// every repository it names must be in the archive, and each of those
	462	// must pass git fsck --connectivity-only. A database-only archive is
	463	// checked for integrity alone and says so. Repositories are extracted to
	464	// a temporary directory for the check, so it needs free space for them.
260	465	func verifyBackup(path, identity string) error {
261	466	f, err := os.Open(path)
262	467	if err != nil {
		@@ -292,21 +497,35 @@ func verifyBackup(path, identity string) error {
292	497	switch {
293	498	case h.Name == "gitbay.db":
294	499	dbPath = filepath.Join(tmp, "gitbay.db")
295		w, err := os.Create(dbPath)
296		if err != nil {
297		return err
298		}
299		if _, err := io.Copy(w, tr); err != nil {
300		w.Close()
	500	if err := extractTo(tr, dbPath); err != nil {
301	501	return fmt.Errorf("%s: extracting the database: %w", path, err)
302	502	}
303		w.Close()
304	503	case strings.HasPrefix(h.Name, "repos/"):
	504	trimmed := strings.TrimSuffix(h.Name, "/")
305	505	// repos/<owner>/<name>.git/HEAD marks one repository present.
306		parts := strings.Split(h.Name, "/")
	506	parts := strings.Split(trimmed, "/")
307	507	if len(parts) == 4 && parts[3] == "HEAD" && strings.HasSuffix(parts[2], ".git") {
308	508	inArchive[parts[1]+"/"+strings.TrimSuffix(parts[2], ".git")] = true
309	509	}
	510	if !filepath.IsLocal(trimmed) {
	511	return fmt.Errorf("%s: member %q leaves the archive root", path, h.Name)
	512	}
	513	if borrowsObjects(trimmed) {
	514	continue
	515	}
	516	dest := filepath.Join(tmp, filepath.FromSlash(trimmed))
	517	switch h.Typeflag {
	518	case tar.TypeDir:
	519	// The archive's directory modes do not matter to fsck, and
	520	// a hostile one would stop RemoveAll cleaning up.
	521	if err := os.MkdirAll(dest, 0o700); err != nil {
	522	return fmt.Errorf("%s: creating %s: %w", path, h.Name, err)
	523	}
	524	case tar.TypeReg:
	525	if err := extractTo(tr, dest); err != nil {
	526	return fmt.Errorf("%s: extracting %s: %w", path, h.Name, err)
	527	}
	528	}
310	529	}
311	530	}
312	531	// Read to the end so gzip checks its trailer and age its final chunk.
		@@ -351,11 +570,54 @@ func verifyBackup(path, identity string) error {
351	570	return fmt.Errorf("%s: %d repositories the database names are not in the archive: %s", path, len(missing), strings.Join(missing, ", "))
352	571	}
353	572	if extra > 0 {
354		fmt.Printf("%d repositories in the archive that the database does not name (deleted after the snapshot)\n", extra)
	573	fmt.Printf("%d repositories in the archive that the database does not name (created after the snapshot)\n", extra)
	574	}
	575	var broken []string
	576	for _, r := range repos {
	577	dir := filepath.Join(tmp, "repos", r.OwnerName, r.Name+".git")
	578	if err := gitutil.FsckConnectivity(dir); err != nil {
	579	fmt.Fprintf(os.Stderr, "%s: %v\n", r.Path(), err)
	580	broken = append(broken, r.Path())
	581	}
	582	}
	583	if len(broken) > 0 {
	584	return fmt.Errorf("%s: %d repositories fail the connectivity check: %s", path, len(broken), strings.Join(broken, ", "))
355	585	}
	586	fmt.Printf("connectivity ok on %d repositories\n", len(repos))
356	587	return nil
357	588	}
358	589
	590	// borrowsObjects reports an archive member that would point git at
	591	// objects or refs outside the extracted repository: alternates, or a
	592	// commondir directly in a *.git directory. gitbay writes none, and one in
	593	// a hostile archive would have fsck read another repository on the host,
	594	// so verify leaves them out. The comparison ignores case, as a
	595	// case-insensitive filesystem would.
	596	func borrowsObjects(name string) bool {
	597	name = strings.ToLower(filepath.ToSlash(filepath.Clean(name)))
	598	if dir, base := path.Split(name); base == "commondir" && strings.HasSuffix(strings.TrimSuffix(dir, "/"), ".git") {
	599	return true
	600	}
	601	return strings.HasSuffix(name, "/objects/info/alternates") \|\|
	602	strings.HasSuffix(name, "/objects/info/http-alternates")
	603	}
	604
	605	// extractTo writes one archive member to dest, owner-only.
	606	func extractTo(r io.Reader, dest string) error {
	607	if err := os.MkdirAll(filepath.Dir(dest), 0o700); err != nil {
	608	return err
	609	}
	610	w, err := os.OpenFile(dest, os.O_WRONLY\|os.O_CREATE\|os.O_TRUNC, 0o600)
	611	if err != nil {
	612	return err
	613	}
	614	if _, err := io.Copy(w, r); err != nil {
	615	w.Close()
	616	return err
	617	}
	618	return w.Close()
	619	}
	620
359	621	const ageHeader = "age-encryption.org/v1\n"
360	622
361	623	// archiveReader returns the archive's gzip stream, decrypting it first

cmd/gitbayd/backup_test.go +516

		@@ -5,7 +5,9 @@ import (
5	5	"compress/gzip"
6	6	"errors"
7	7	"io"
	8	"io/fs"
8	9	"os"
	10	"os/exec"
9	11	"path/filepath"
10	12	"sort"
11	13	"strings"
		@@ -14,7 +16,9 @@ import (
14	16
15	17	"filippo.io/age"
16	18
	19	"gitbay.org/gitbay/internal/backuplock"
17	20	"gitbay.org/gitbay/internal/config"
	21	"gitbay.org/gitbay/internal/gitutil"
18	22	)
19	23
20	24	// members lists the archive's entries by name.
		@@ -324,3 +328,515 @@ func TestArchivePath(t *testing.T) {
324	328	}
325	329	}
326	330	}
	331
	332	func gitIn(t *testing.T, dir string, args ...string) string {
	333	t.Helper()
	334	cmd := exec.Command("git", append([]string{"-C", dir}, args...)...)
	335	cmd.Env = append(os.Environ(),
	336	"GIT_AUTHOR_NAME=t", "GIT_AUTHOR_EMAIL=t@e",
	337	"GIT_COMMITTER_NAME=t", "GIT_COMMITTER_EMAIL=t@e")
	338	out, err := cmd.CombinedOutput()
	339	if err != nil {
	340	t.Fatalf("git %v: %v\n%s", args, err, out)
	341	}
	342	return strings.TrimSpace(string(out))
	343	}
	344
	345	// verify runs git's connectivity check on every repository the
	346	// database names: a repository missing an object fails it.
	347	func TestVerifyChecksConnectivity(t *testing.T) {
	348	cfg := testConfig(t)
	349	st, err := openStore(cfg)
	350	if err != nil {
	351	t.Fatal(err)
	352	}
	353	uid, err := st.CreateUser("krz", false)
	354	if err != nil {
	355	t.Fatal(err)
	356	}
	357	if _, err := st.CreateRepo("user", uid, "thing", "public"); err != nil {
	358	t.Fatal(err)
	359	}
	360	st.Close()
	361
	362	work := t.TempDir()
	363	gitIn(t, work, "init", "-q", "-b", "main")
	364	if err := os.WriteFile(filepath.Join(work, "a.txt"), []byte("a\n"), 0o644); err != nil {
	365	t.Fatal(err)
	366	}
	367	gitIn(t, work, "add", "a.txt")
	368	gitIn(t, work, "commit", "-q", "-m", "one")
	369	dir := filepath.Join(cfg.Server.Root, "repos", "krz", "thing.git")
	370	gitIn(t, work, "clone", "-q", "--bare", work, dir)
	371
	372	good := filepath.Join(t.TempDir(), "good.tar.gz")
	373	if err := runBackup(cfg, good, false); err != nil {
	374	t.Fatal(err)
	375	}
	376	if err := verifyBackup(good, ""); err != nil {
	377	t.Fatalf("intact archive: %v", err)
	378	}
	379
	380	blob := gitIn(t, dir, "rev-parse", "HEAD:a.txt")
	381	if err := os.Remove(filepath.Join(dir, "objects", blob[:2], blob[2:])); err != nil {
	382	t.Fatal(err)
	383	}
	384	bad := filepath.Join(t.TempDir(), "bad.tar.gz")
	385	if err := runBackup(cfg, bad, false); err != nil {
	386	t.Fatal(err)
	387	}
	388	err = verifyBackup(bad, "")
	389	if err == nil \|\| !strings.Contains(err.Error(), "krz/thing") \|\| !strings.Contains(err.Error(), "connectivity") {
	390	t.Fatalf("archive with a missing blob: %v", err)
	391	}
	392	}
	393
	394	// A full backup waits for a delete under way, and does not archive its
	395	// own lock file.
	396	func TestFullBackupWaitsForRepositoryMoves(t *testing.T) {
	397	cfg := testConfig(t)
	398	s, err := openStore(cfg)
	399	if err != nil {
	400	t.Fatal(err)
	401	}
	402	s.Close()
	403	inFlight, err := backuplock.TryShared(cfg.Server.Root)
	404	if err != nil {
	405	t.Fatal(err)
	406	}
	407	out := filepath.Join(t.TempDir(), "b.tar.gz")
	408	done := make(chan error, 1)
	409	go func() { done <- runBackup(cfg, out, false) }()
	410	select {
	411	case err := <-done:
	412	t.Fatalf("backup finished while a delete held the lock: %v", err)
	413	case <-time.After(200 * time.Millisecond):
	414	}
	415	inFlight()
	416	select {
	417	case err := <-done:
	418	if err != nil {
	419	t.Fatal(err)
	420	}
	421	case <-time.After(10 * time.Second):
	422	t.Fatal("backup never started after the delete finished")
	423	}
	424	for _, n := range members(t, out) {
	425	if n == backuplock.Name {
	426	t.Fatalf("archive carries %s", n)
	427	}
	428	}
	429	}
	430
	431	// A repository with every ref packed keeps its empty refs/heads and
	432	// refs/tags directories through backup and extraction, the same as a real
	433	// restore would: git needs refs/ to recognize a bare repository at all,
	434	// even when every ref lives in packed-refs (#259).
	435	func TestBackupPreservesPackedRefDirs(t *testing.T) {
	436	cfg := testConfig(t)
	437	st, err := openStore(cfg)
	438	if err != nil {
	439	t.Fatal(err)
	440	}
	441	uid, err := st.CreateUser("krz", false)
	442	if err != nil {
	443	t.Fatal(err)
	444	}
	445	if _, err := st.CreateRepo("user", uid, "thing", "public"); err != nil {
	446	t.Fatal(err)
	447	}
	448	st.Close()
	449
	450	work := t.TempDir()
	451	gitIn(t, work, "init", "-q", "-b", "main")
	452	if err := os.WriteFile(filepath.Join(work, "a.txt"), []byte("a\n"), 0o644); err != nil {
	453	t.Fatal(err)
	454	}
	455	gitIn(t, work, "add", "a.txt")
	456	gitIn(t, work, "commit", "-q", "-m", "one")
	457	dir := filepath.Join(cfg.Server.Root, "repos", "krz", "thing.git")
	458	gitIn(t, work, "clone", "-q", "--bare", work, dir)
	459	gitIn(t, dir, "pack-refs", "--all")
	460	entries, err := os.ReadDir(filepath.Join(dir, "refs", "heads"))
	461	if err != nil {
	462	t.Fatal(err)
	463	}
	464	if len(entries) != 0 {
	465	t.Fatalf("refs/heads not empty after pack-refs --all: %v", entries)
	466	}
	467
	468	archive := filepath.Join(t.TempDir(), "b.tar.gz")
	469	if err := runBackup(cfg, archive, false); err != nil {
	470	t.Fatal(err)
	471	}
	472
	473	// verify sees the archive exactly as a restore would: no workaround.
	474	if err := verifyBackup(archive, ""); err != nil {
	475	t.Fatalf("verify: %v", err)
	476	}
	477
	478	restored := t.TempDir()
	479	if out, err := exec.Command("tar", "-xzf", archive, "-C", restored).CombinedOutput(); err != nil {
	480	t.Fatalf("extract: %v\n%s", err, out)
	481	}
	482	restoredRepo := filepath.Join(restored, "repos", "krz", "thing.git")
	483	if got := gitIn(t, restoredRepo, "rev-parse", "--verify", "HEAD"); got == "" {
	484	t.Fatal("rev-parse --verify HEAD returned nothing after restore")
	485	}
	486	gitIn(t, restoredRepo, "fsck", "--connectivity-only", "--no-progress", "--no-dangling")
	487	}
	488
	489	// A commit pushed after a repository's refs are archived and before its
	490	// objects are leaves the archive with the earlier refs and every object
	491	// they reach, plus the new ones unreferenced (#259).
	492	func TestBackupArchivesRefsBeforeObjects(t *testing.T) {
	493	cfg := testConfig(t)
	494	st, err := openStore(cfg)
	495	if err != nil {
	496	t.Fatal(err)
	497	}
	498	uid, err := st.CreateUser("krz", false)
	499	if err != nil {
	500	t.Fatal(err)
	501	}
	502	if _, err := st.CreateRepo("user", uid, "thing", "public"); err != nil {
	503	t.Fatal(err)
	504	}
	505	st.Close()
	506
	507	work := t.TempDir()
	508	gitIn(t, work, "init", "-q", "-b", "main")
	509	if err := os.WriteFile(filepath.Join(work, "a.txt"), []byte("a\n"), 0o644); err != nil {
	510	t.Fatal(err)
	511	}
	512	gitIn(t, work, "add", "a.txt")
	513	gitIn(t, work, "commit", "-q", "-m", "one")
	514	dir := filepath.Join(cfg.Server.Root, "repos", "krz", "thing.git")
	515	gitIn(t, work, "clone", "-q", "--bare", work, dir)
	516	first := gitIn(t, dir, "rev-parse", "refs/heads/main")
	517
	518	var second string
	519	afterRefs = func(repo string) {
	520	if repo != dir {
	521	return
	522	}
	523	if err := os.WriteFile(filepath.Join(work, "b.txt"), []byte("b\n"), 0o644); err != nil {
	524	t.Fatal(err)
	525	}
	526	gitIn(t, work, "add", "b.txt")
	527	gitIn(t, work, "commit", "-q", "-m", "two")
	528	gitIn(t, work, "push", "-q", dir, "main")
	529	second = gitIn(t, dir, "rev-parse", "refs/heads/main")
	530	}
	531	t.Cleanup(func() { afterRefs = func(string) {} })
	532
	533	archive := filepath.Join(t.TempDir(), "b.tar.gz")
	534	if err := runBackup(cfg, archive, false); err != nil {
	535	t.Fatal(err)
	536	}
	537	if second == "" \|\| second == first {
	538	t.Fatal("the push between the refs and the objects did not happen")
	539	}
	540	if err := verifyBackup(archive, ""); err != nil {
	541	t.Fatalf("verify: %v", err)
	542	}
	543	restored := t.TempDir()
	544	if out, err := exec.Command("tar", "-xzf", archive, "-C", restored).CombinedOutput(); err != nil {
	545	t.Fatalf("extract: %v\n%s", err, out)
	546	}
	547	repo := filepath.Join(restored, "repos", "krz", "thing.git")
	548	if got := gitIn(t, repo, "rev-parse", "refs/heads/main"); got != first {
	549	t.Errorf("archived main is %s, want %s from before the push", got, first)
	550	}
	551	gitIn(t, repo, "cat-file", "-e", second)
	552	}
	553
	554	// A pack removed between the walk listing it and reading it is skipped,
	555	// and verify's fsck then reports what it held; a vanished file outside
	556	// objects/ still fails the backup.
	557	func TestBackupSkipsVanishedObjects(t *testing.T) {
	558	cfg := testConfig(t)
	559	st, err := openStore(cfg)
	560	if err != nil {
	561	t.Fatal(err)
	562	}
	563	uid, err := st.CreateUser("krz", false)
	564	if err != nil {
	565	t.Fatal(err)
	566	}
	567	if _, err := st.CreateRepo("user", uid, "thing", "public"); err != nil {
	568	t.Fatal(err)
	569	}
	570	st.Close()
	571
	572	work := t.TempDir()
	573	gitIn(t, work, "init", "-q", "-b", "main")
	574	if err := os.WriteFile(filepath.Join(work, "a.txt"), []byte("a\n"), 0o644); err != nil {
	575	t.Fatal(err)
	576	}
	577	gitIn(t, work, "add", "a.txt")
	578	gitIn(t, work, "commit", "-q", "-m", "one")
	579	dir := filepath.Join(cfg.Server.Root, "repos", "krz", "thing.git")
	580	gitIn(t, work, "clone", "-q", "--bare", work, dir)
	581	gitIn(t, dir, "repack", "-q", "-a", "-d")
	582
	583	removed := ""
	584	beforeAdd = func(path string) {
	585	if removed == "" && strings.HasSuffix(path, ".pack") {
	586	removed = path
	587	os.Remove(path)
	588	}
	589	}
	590	t.Cleanup(func() { beforeAdd = func(string) {} })
	591	archive := filepath.Join(t.TempDir(), "b.tar.gz")
	592	if err := runBackup(cfg, archive, false); err != nil {
	593	t.Fatalf("backup with a vanished pack: %v", err)
	594	}
	595	if removed == "" {
	596	t.Fatal("no pack was archived")
	597	}
	598	for _, n := range members(t, archive) {
	599	if strings.HasSuffix(n, ".pack") {
	600	t.Errorf("archive carries %s", n)
	601	}
	602	}
	603	if err := verifyBackup(archive, ""); err == nil \|\| !strings.Contains(err.Error(), "connectivity") {
	604	t.Errorf("verify of an archive missing its pack: %v", err)
	605	}
	606
	607	beforeAdd = func(path string) {
	608	if strings.HasSuffix(path, filepath.Join("thing.git", "config")) {
	609	os.Remove(path)
	610	}
	611	}
	612	err = runBackup(cfg, filepath.Join(t.TempDir(), "c.tar.gz"), false)
	613	if !errors.Is(err, fs.ErrNotExist) {
	614	t.Fatalf("backup with a vanished config: %v, want not-exist", err)
	615	}
	616	}
	617
	618	// gc refuses while a full backup holds the lock.
	619	func TestGCRefusedDuringBackup(t *testing.T) {
	620	cfg := testConfig(t)
	621	release, err := backuplock.Hold(cfg.Server.Root)
	622	if err != nil {
	623	t.Fatal(err)
	624	}
	625	defer release()
	626	if err := runGC(cfg, "", false, false); !errors.Is(err, backuplock.ErrBusy) {
	627	t.Fatalf("gc during a backup: %v", err)
	628	}
	629	}
	630
	631	// A backup and its verify need no key file: sealed values are copied as
	632	// they are. A missing database is refused rather than created.
	633	func TestBackupNeedsNoKeyFile(t *testing.T) {
	634	cfg := testConfig(t)
	635	out := filepath.Join(t.TempDir(), "b.tar.gz")
	636	if err := runBackup(cfg, out, false); !errors.Is(err, fs.ErrNotExist) {
	637	t.Fatalf("backup without a database: %v", err)
	638	}
	639	if _, err := os.Stat(filepath.Join(cfg.Server.Root, "gitbay.db")); !os.IsNotExist(err) {
	640	t.Fatalf("backup created a database: %v", err)
	641	}
	642	s, err := openStore(cfg)
	643	if err != nil {
	644	t.Fatal(err)
	645	}
	646	s.Close()
	647	if err := os.Remove(cfg.Server.SecretKeyFile); err != nil {
	648	t.Fatal(err)
	649	}
	650	if err := runBackup(cfg, out, false); err != nil {
	651	t.Fatalf("backup without the key file: %v", err)
	652	}
	653	if err := verifyBackup(out, ""); err != nil {
	654	t.Fatalf("verify without the key file: %v", err)
	655	}
	656	}
	657
	658	// A run removes what a killed run left beside the archive once it is a
	659	// day old, and leaves younger ones, which may belong to a run under way.
	660	func TestBackupRemovesStaleTemporaries(t *testing.T) {
	661	cfg := testConfig(t)
	662	s, err := openStore(cfg)
	663	if err != nil {
	664	t.Fatal(err)
	665	}
	666	s.Close()
	667	dir := t.TempDir()
	668	old := time.Now().Add(-25 * time.Hour)
	669	mk := func(name string, isDir bool, mtime time.Time) {
	670	p := filepath.Join(dir, name)
	671	if isDir {
	672	if err := os.Mkdir(p, 0o700); err != nil {
	673	t.Fatal(err)
	674	}
	675	} else if err := os.WriteFile(p, []byte("x"), 0o600); err != nil {
	676	t.Fatal(err)
	677	}
	678	if err := os.Chtimes(p, mtime, mtime); err != nil {
	679	t.Fatal(err)
	680	}
	681	}
	682	mk(".gitbay-snap-old", true, old)
	683	mk(".b.tar.gz.tmp-123", false, old)
	684	mk(".gitbay-snap-new", true, time.Now())
	685	mk(".b.tar.gz.tmp-456", false, time.Now())
	686	mk(".keep", false, old)
	687	mk(".notes.tmp-draft", false, old)
	688	if err := runBackup(cfg, filepath.Join(dir, "b.tar.gz"), true); err != nil {
	689	t.Fatal(err)
	690	}
	691	got := leftovers(t, dir)
	692	want := []string{".b.tar.gz.tmp-456", ".gitbay-snap-new", ".keep", ".notes.tmp-draft"}
	693	sort.Strings(got)
	694	if strings.Join(got, " ") != strings.Join(want, " ") {
	695	t.Errorf("left %v, want %v", got, want)
	696	}
	697	}
	698
	699	// An archive written under server.root, directly or through a symlink,
	700	// would be in the next full backup, so it is refused.
	701	func TestBackupRefusesOutputInsideRoot(t *testing.T) {
	702	cfg := testConfig(t)
	703	s, err := openStore(cfg)
	704	if err != nil {
	705	t.Fatal(err)
	706	}
	707	s.Close()
	708	link := filepath.Join(t.TempDir(), "link")
	709	if err := os.Symlink(cfg.Server.Root, link); err != nil {
	710	t.Fatal(err)
	711	}
	712	for _, out := range []string{
	713	filepath.Join(cfg.Server.Root, "b.tar.gz"),
	714	filepath.Join(cfg.Server.Root, "backups", "b.tar.gz"),
	715	filepath.Join(link, "b.tar.gz"),
	716	} {
	717	if err := runBackup(cfg, out, true); err == nil \|\| !strings.Contains(err.Error(), "inside server.root") {
	718	t.Errorf("%s: %v", out, err)
	719	}
	720	}
	721	}
	722
	723	// verify does not extract objects/info/alternates, so an archived
	724	// repository cannot borrow objects from paths outside the archive.
	725	func TestVerifyIgnoresAlternates(t *testing.T) {
	726	cfg := testConfig(t)
	727	st, err := openStore(cfg)
	728	if err != nil {
	729	t.Fatal(err)
	730	}
	731	uid, err := st.CreateUser("krz", false)
	732	if err != nil {
	733	t.Fatal(err)
	734	}
	735	if _, err := st.CreateRepo("user", uid, "thing", "public"); err != nil {
	736	t.Fatal(err)
	737	}
	738	st.Close()
	739
	740	work := t.TempDir()
	741	gitIn(t, work, "init", "-q", "-b", "main")
	742	if err := os.WriteFile(filepath.Join(work, "a.txt"), []byte("a\n"), 0o644); err != nil {
	743	t.Fatal(err)
	744	}
	745	gitIn(t, work, "add", "a.txt")
	746	gitIn(t, work, "commit", "-q", "-m", "one")
	747	dir := filepath.Join(cfg.Server.Root, "repos", "krz", "thing.git")
	748	gitIn(t, work, "clone", "-q", "--bare", "--shared", work, dir)
	749	if _, err := os.Stat(filepath.Join(dir, "objects", "info", "alternates")); err != nil {
	750	t.Fatal(err)
	751	}
	752	gitIn(t, dir, "fsck", "--connectivity-only", "--no-progress")
	753
	754	archive := filepath.Join(t.TempDir(), "b.tar.gz")
	755	if err := runBackup(cfg, archive, false); err != nil {
	756	t.Fatal(err)
	757	}
	758	if err := verifyBackup(archive, ""); err == nil \|\| !strings.Contains(err.Error(), "connectivity") {
	759	t.Fatalf("verify of a repository whose objects are only in an alternate: %v", err)
	760	}
	761	}
	762
	763	func TestBorrowsObjectsMember(t *testing.T) {
	764	for name, want := range map[string]bool{
	765	"repos/a/b.git/objects/info/alternates": true,
	766	"repos/a/b.git/objects/info/./alternates": true,
	767	"repos/a/b.git/objects/info/Alternates": true,
	768	"repos/a/b.git/objects/info/http-alternates": true,
	769	"repos/a/b.git/objects/info/packs": false,
	770	"repos/a/b.git/refs/heads/alternates": false,
	771	"repos/a/b.git/commondir": true,
	772	"repos/a/b.git/CommonDir": true,
	773	"repos/a/b.git/refs/heads/commondir": false,
	774	} {
	775	if got := borrowsObjects(name); got != want {
	776	t.Errorf("borrowsObjects(%q) = %v, want %v", name, got, want)
	777	}
	778	}
	779	}
	780
	781	// verify does not extract a commondir, so an archived repository with
	782	// none of its own objects cannot pass by pointing git at a repository on
	783	// the host.
	784	func TestVerifyIgnoresCommondir(t *testing.T) {
	785	cfg := testConfig(t)
	786	st, err := openStore(cfg)
	787	if err != nil {
	788	t.Fatal(err)
	789	}
	790	uid, err := st.CreateUser("krz", false)
	791	if err != nil {
	792	t.Fatal(err)
	793	}
	794	if _, err := st.CreateRepo("user", uid, "thing", "public"); err != nil {
	795	t.Fatal(err)
	796	}
	797	st.Close()
	798
	799	work := t.TempDir()
	800	gitIn(t, work, "init", "-q", "-b", "main")
	801	if err := os.WriteFile(filepath.Join(work, "a.txt"), []byte("a\n"), 0o644); err != nil {
	802	t.Fatal(err)
	803	}
	804	gitIn(t, work, "add", "a.txt")
	805	gitIn(t, work, "commit", "-q", "-m", "one")
	806	host := filepath.Join(t.TempDir(), "host.git")
	807	gitIn(t, work, "clone", "-q", "--bare", work, host)
	808	gitIn(t, host, "pack-refs", "--all")
	809
	810	// A repository whose refs are its own and whose objects directory is
	811	// empty, borrowing everything else from host through commondir.
	812	dir := filepath.Join(cfg.Server.Root, "repos", "krz", "thing.git")
	813	for _, d := range []string{"objects", "refs"} {
	814	if err := os.MkdirAll(filepath.Join(dir, d), 0o755); err != nil {
	815	t.Fatal(err)
	816	}
	817	}
	818	packed, err := os.ReadFile(filepath.Join(host, "packed-refs"))
	819	if err != nil {
	820	t.Fatal(err)
	821	}
	822	for name, body := range map[string]string{
	823	"HEAD": "ref: refs/heads/main\n",
	824	"packed-refs": string(packed),
	825	"commondir": host + "\n",
	826	} {
	827	if err := os.WriteFile(filepath.Join(dir, name), []byte(body), 0o644); err != nil {
	828	t.Fatal(err)
	829	}
	830	}
	831	if err := gitutil.FsckConnectivity(dir); err != nil {
	832	t.Fatalf("with commondir on the host the repository should pass: %v", err)
	833	}
	834
	835	archive := filepath.Join(t.TempDir(), "b.tar.gz")
	836	if err := runBackup(cfg, archive, false); err != nil {
	837	t.Fatal(err)
	838	}
	839	if err := verifyBackup(archive, ""); err == nil \|\| !strings.Contains(err.Error(), "connectivity") {
	840	t.Fatalf("verify of a repository whose objects are only in its commondir: %v", err)
	841	}
	842	}

cmd/gitbayd/maint.go +51 −38

		@@ -8,6 +8,7 @@ import (
8	8
9	9	"github.com/spf13/cobra"
10	10
	11	"gitbay.org/gitbay/internal/backuplock"
11	12	"gitbay.org/gitbay/internal/config"
12	13	"gitbay.org/gitbay/internal/control"
13	14	"gitbay.org/gitbay/internal/gitutil"
		@@ -27,44 +28,7 @@ func gcCmd() *cobra.Command {
27	28	if err != nil {
28	29	return err
29	30	}
30		st, err := openStore(cfg)
31		if err != nil {
32		return err
33		}
34		defer st.Close()
35
36		var repos []store.Repo
37		if repoPath != "" {
38		r, err := st.RepoByPath(repoPath)
39		if err != nil {
40		return fmt.Errorf("no repository %q", repoPath)
41		}
42		repos = []store.Repo{r}
43		} else if repos, err = st.ListAllRepos(); err != nil {
44		return err
45		}
46
47		var before, after int64
48		for _, r := range repos {
49		dir := control.RepoDir(cfg.Server.Root, r.OwnerName, r.Name)
50		b := gitutil.DirSize(dir)
51		gcArgs := []string{"-C", dir, "gc", "--quiet"}
52		if aggressive {
53		gcArgs = append(gcArgs, "--aggressive")
54		}
55		if out, err := exec.Command(toolpath.Look("git"), gcArgs...).CombinedOutput(); err != nil {
56		fmt.Fprintf(os.Stderr, "%s: gc failed: %v\n%s", r.Path(), err, out)
57		continue
58		}
59		a := gitutil.DirSize(dir)
60		before, after = before+b, after+a
61		fmt.Printf("%s\t%s -> %s\n", r.Path(), human(b), human(a))
62		}
63		fmt.Printf("total\t%s -> %s (freed %s)\n", human(before), human(after), human(before-after))
64		if !withLFS {
65		return nil
66		}
67		return gcLFS(cfg, st)
	31	return runGC(cfg, repoPath, aggressive, withLFS)
68	32	},
69	33	}
70	34	cmd.Flags().StringVar(&repoPath, "repo", "", "one repository (owner/name) instead of all")
		@@ -73,6 +37,55 @@ func gcCmd() *cobra.Command {
73	37	return cmd
74	38	}
75	39
	40	// runGC refuses while a full backup runs: a prune removing a pack the
	41	// backup has listed but not yet read would leave it out of the archive
	42	// (#259).
	43	func runGC(cfg config.Config, repoPath string, aggressive, withLFS bool) error {
	44	release, err := backuplock.TryShared(cfg.Server.Root)
	45	if err != nil {
	46	return err
	47	}
	48	defer release()
	49	st, err := openStore(cfg)
	50	if err != nil {
	51	return err
	52	}
	53	defer st.Close()
	54
	55	var repos []store.Repo
	56	if repoPath != "" {
	57	r, err := st.RepoByPath(repoPath)
	58	if err != nil {
	59	return fmt.Errorf("no repository %q", repoPath)
	60	}
	61	repos = []store.Repo{r}
	62	} else if repos, err = st.ListAllRepos(); err != nil {
	63	return err
	64	}
	65
	66	var before, after int64
	67	for _, r := range repos {
	68	dir := control.RepoDir(cfg.Server.Root, r.OwnerName, r.Name)
	69	b := gitutil.DirSize(dir)
	70	gcArgs := []string{"-C", dir, "gc", "--quiet"}
	71	if aggressive {
	72	gcArgs = append(gcArgs, "--aggressive")
	73	}
	74	if out, err := exec.Command(toolpath.Look("git"), gcArgs...).CombinedOutput(); err != nil {
	75	fmt.Fprintf(os.Stderr, "%s: gc failed: %v\n%s", r.Path(), err, out)
	76	continue
	77	}
	78	a := gitutil.DirSize(dir)
	79	before, after = before+b, after+a
	80	fmt.Printf("%s\t%s -> %s\n", r.Path(), human(b), human(a))
	81	}
	82	fmt.Printf("total\t%s -> %s (freed %s)\n", human(before), human(after), human(before-after))
	83	if !withLFS {
	84	return nil
	85	}
	86	return gcLFS(cfg, st)
	87	}
	88
76	89	func human(b int64) string {
77	90	switch {
78	91	case b >= 1<<30:

cmd/gitbayd/secrets.go +27

		@@ -65,6 +65,11 @@ keeps its owner. Copy the new file off the host afterwards.`,
65	65	// the owner of server.root, since the daemon reads it as that user.
66	66	func initSecrets(cfg config.Config, w io.Writer) error {
67	67	path := cfg.Server.SecretKeyFile
	68	unlock, err := lockKeyFile(path)
	69	if err != nil {
	70	return err
	71	}
	72	defer unlock()
68	73	if _, err := os.Lstat(path); err == nil {
69	74	return fmt.Errorf("%s already exists; gitbayd admin secrets rotate replaces its key", path)
70	75	} else if !errors.Is(err, fs.ErrNotExist) {
		@@ -108,6 +113,11 @@ func initSecrets(cfg config.Config, w io.Writer) error {
108	113	// finishes the job.
109	114	func rotateSecrets(cfg config.Config, w io.Writer) error {
110	115	path := cfg.Server.SecretKeyFile
	116	unlock, err := lockKeyFile(path)
	117	if err != nil {
	118	return err
	119	}
	120	defer unlock()
111	121	old, err := seal.ReadKeys(path)
112	122	if err != nil {
113	123	return err
		@@ -152,6 +162,23 @@ func rotateSecrets(cfg config.Config, w io.Writer) error {
152	162	return nil
153	163	}
154	164
	165	// lockKeyFile takes an exclusive flock on <path>.lock, waiting for
	166	// another init or rotate to finish. Two rotations interleaved would each
	167	// write a file without the other's new key, and values resealed under
	168	// the lost one would no longer open. O_NOFOLLOW refuses a symlink planted
	169	// at the lock's name.
	170	func lockKeyFile(path string) (func(), error) {
	171	f, err := os.OpenFile(path+".lock", os.O_RDWR\|os.O_CREATE\|syscall.O_NOFOLLOW, 0o600)
	172	if err != nil {
	173	return nil, fmt.Errorf("key file lock: %w", err)
	174	}
	175	if err := syscall.Flock(int(f.Fd()), syscall.LOCK_EX); err != nil {
	176	f.Close()
	177	return nil, fmt.Errorf("key file lock: %w", err)
	178	}
	179	return func() { f.Close() }, nil
	180	}
	181
155	182	// checkSecrets opens every stored secret and prints, per column, how
156	183	// many values each key sealed and every value that does not open. Any
157	184	// such value is an error.

cmd/gitbayd/secrets_test.go +48

		@@ -7,6 +7,7 @@ import (
7	7	"path/filepath"
8	8	"strings"
9	9	"testing"
	10	"time"
10	11
11	12	"gitbay.org/gitbay/internal/config"
12	13	"gitbay.org/gitbay/internal/seal"
		@@ -174,3 +175,50 @@ func assertNoKeyMaterial(t *testing.T, out string, keys []seal.Key) {
174	175	}
175	176	}
176	177	}
	178
	179	// A rotate waits while another holds the key file's lock, and two run
	180	// at once both finish with every value still opening.
	181	func TestRotateSecretsSerializes(t *testing.T) {
	182	cfg := testConfig(t)
	183	st, repoID := storeWithSecret(t, cfg)
	184	st.Close()
	185
	186	unlock, err := lockKeyFile(cfg.Server.SecretKeyFile)
	187	if err != nil {
	188	t.Fatal(err)
	189	}
	190	done := make(chan error, 2)
	191	go func() { done <- rotateSecrets(cfg, &bytes.Buffer{}) }()
	192	go func() { done <- rotateSecrets(cfg, &bytes.Buffer{}) }()
	193	select {
	194	case err := <-done:
	195	t.Fatalf("rotate finished while the lock was held: %v", err)
	196	case <-time.After(200 * time.Millisecond):
	197	}
	198	unlock()
	199	for range 2 {
	200	select {
	201	case err := <-done:
	202	if err != nil {
	203	t.Fatalf("rotate: %v", err)
	204	}
	205	case <-time.After(30 * time.Second):
	206	t.Fatal("rotate never finished")
	207	}
	208	}
	209	keys, err := seal.ReadKeys(cfg.Server.SecretKeyFile)
	210	if err != nil \|\| len(keys) != 1 {
	211	t.Fatalf("key file after two rotations: %v, %v", keys, err)
	212	}
	213	st, err = openStore(cfg)
	214	if err != nil {
	215	t.Fatal(err)
	216	}
	217	defer st.Close()
	218	if got, err := st.BuildSecrets(repoID); err != nil \|\| got["TOKEN"] != "v1" {
	219	t.Fatalf("value after two rotations: %v, %v", got, err)
	220	}
	221	if fi, err := os.Lstat(cfg.Server.SecretKeyFile + ".lock"); err != nil \|\| fi.Mode().Perm() != 0o600 {
	222	t.Fatalf("lock file: %v, %v", fi, err)
	223	}
	224	}

cmd/gitbayd/system.go +2 −2

		@@ -16,8 +16,8 @@ import (
16	16
17	17	// authorizedKeysCmd backs sshd's AuthorizedKeysCommand in system mode:
18	18	//
19		// AuthorizedKeysCommand /usr/bin/gitbayd --config /etc/gitbay/config.toml authorized-keys %t %k
20		// AuthorizedKeysCommandUser git
	19	// AuthorizedKeysCommand /usr/local/bin/gitbayd --config /etc/gitbay/config.toml authorized-keys %t %k
	20	// AuthorizedKeysCommandUser gitbay
21	21	//
22	22	// It prints a forced-command authorized_keys line for registered keys and
23	23	// nothing for unknown ones — so unknown keys fail authentication inside

deploy/cloud-init.yaml +6 −1

		@@ -11,7 +11,12 @@
11	11	# - opens ufw for 22, 80, 443, 2222
12	12	#
13	13	# It does NOT install the gitbayd binary (it is not hosted anywhere yet);
14		# scp it to /usr/local/bin/gitbayd afterward and `systemctl start gitbayd`.
	14	# scp it to /usr/local/bin/gitbayd afterward, then create the secret key
	15	# and hand it to the daemon user before starting:
	16	# gitbayd --config /etc/gitbay/config.toml admin secrets init
	17	# chown gitbay:gitbay /etc/gitbay/secret.key
	18	# systemctl start gitbayd
	19	# On a restore, put the key file's off-host copy there instead of init.
15	20
16	21	package_update: true
17	22	packages:

e2e/backup_test.go +4

		@@ -87,6 +87,10 @@ func TestAdminBackup(t *testing.T) {
87	87	t.Fatal("the archived database carries the build secret in clear")
88	88	}
89	89
	90	if out := inst.admin(t, "admin", "backup", "--verify", archive); !strings.Contains(out, "connectivity ok on 1 repositories") {
	91	t.Fatalf("verify: %s", out)
	92	}
	93
90	94	// Restore: extract into a fresh root and serve from it.
91	95	root2 := t.TempDir()
92	96	if outB, err := exec.Command("tar", "-xzf", archive, "-C", root2).CombinedOutput(); err != nil {

internal/backuplock/backuplock.go added +59

		@@ -0,0 +1,59 @@
	1	// Package backuplock keeps repository deletes, renames, transfers and
	2	// prunes out of a full backup's way (#259). The backup runs in its own
	3	// process (gitbayd admin backup) and a delete in the daemon's, so the
	4	// lock is flock(2) on a file under server.root: the backup holds it
	5	// exclusively from its database snapshot until the last repository is
	6	// archived, and each delete, move or prune holds it shared while it runs.
	7	package backuplock
	8
	9	import (
	10	"errors"
	11	"os"
	12	"path/filepath"
	13	"syscall"
	14	)
	15
	16	// Name is the lock file under server.root. Backups skip it.
	17	const Name = "backup.lock"
	18
	19	// ErrBusy is TryShared's answer while a backup holds the lock.
	20	var ErrBusy = errors.New("a backup is running; repositories cannot be deleted, renamed, moved or pruned until it finishes, usually within minutes")
	21
	22	// open opens the lock file read-only, which is all flock needs, so the
	23	// daemon's user can lock a file a root-run backup created. O_NOFOLLOW
	24	// refuses a symlink planted at Name instead of following it, since the
	25	// path is inside server.root where only the daemon's own user writes.
	26	func open(root string) (*os.File, error) {
	27	return os.OpenFile(filepath.Join(root, Name), os.O_RDONLY\|os.O_CREATE\|syscall.O_NOFOLLOW, 0o644)
	28	}
	29
	30	// Hold takes the lock exclusively, waiting for deletes and moves under
	31	// way to finish. Closing the file releases it.
	32	func Hold(root string) (func(), error) {
	33	f, err := open(root)
	34	if err != nil {
	35	return nil, err
	36	}
	37	if err := syscall.Flock(int(f.Fd()), syscall.LOCK_EX); err != nil {
	38	f.Close()
	39	return nil, err
	40	}
	41	return func() { f.Close() }, nil
	42	}
	43
	44	// TryShared takes the lock shared without waiting: ErrBusy while a
	45	// backup holds it.
	46	func TryShared(root string) (func(), error) {
	47	f, err := open(root)
	48	if err != nil {
	49	return nil, err
	50	}
	51	if err := syscall.Flock(int(f.Fd()), syscall.LOCK_SH\|syscall.LOCK_NB); err != nil {
	52	f.Close()
	53	if errors.Is(err, syscall.EWOULDBLOCK) {
	54	return nil, ErrBusy
	55	}
	56	return nil, err
	57	}
	58	return func() { f.Close() }, nil
	59	}

internal/backuplock/backuplock_test.go added +69

		@@ -0,0 +1,69 @@
	1	package backuplock
	2
	3	import (
	4	"errors"
	5	"testing"
	6	"time"
	7	)
	8
	9	func TestTrySharedRefusedWhileHeld(t *testing.T) {
	10	root := t.TempDir()
	11	release, err := Hold(root)
	12	if err != nil {
	13	t.Fatal(err)
	14	}
	15	if _, err := TryShared(root); !errors.Is(err, ErrBusy) {
	16	t.Fatalf("TryShared during a backup: %v", err)
	17	}
	18	release()
	19	r, err := TryShared(root)
	20	if err != nil {
	21	t.Fatalf("TryShared after the backup: %v", err)
	22	}
	23	r()
	24	}
	25
	26	func TestSharedHoldersCoexist(t *testing.T) {
	27	root := t.TempDir()
	28	a, err := TryShared(root)
	29	if err != nil {
	30	t.Fatal(err)
	31	}
	32	defer a()
	33	b, err := TryShared(root)
	34	if err != nil {
	35	t.Fatalf("second shared holder: %v", err)
	36	}
	37	b()
	38	}
	39
	40	// A backup waits for a delete already under way.
	41	func TestHoldWaitsForSharedHolder(t *testing.T) {
	42	root := t.TempDir()
	43	shared, err := TryShared(root)
	44	if err != nil {
	45	t.Fatal(err)
	46	}
	47	got := make(chan struct{})
	48	go func() {
	49	release, err := Hold(root)
	50	if err != nil {
	51	t.Error(err)
	52	close(got)
	53	return
	54	}
	55	close(got)
	56	release()
	57	}()
	58	select {
	59	case <-got:
	60	t.Fatal("Hold returned while a shared holder was in")
	61	case <-time.After(100 * time.Millisecond):
	62	}
	63	shared()
	64	select {
	65	case <-got:
	66	case <-time.After(5 * time.Second):
	67	t.Fatal("Hold never returned after the shared holder left")
	68	}
	69	}

internal/config/config.go +3 −3

		@@ -359,12 +359,12 @@ func Load(path string) (Config, error) {
359	359	return cfg, cfg.Validate()
360	360	}
361	361
362		// within reports whether path is dir or below it. Both are compared as
	362	// Within reports whether path is dir or below it. Both are compared as
363	363	// cleaned absolute paths (a relative path resolves against the working
364	364	// directory, same as every other path in this config), with symlinks
365	365	// resolved where the path exists on disk, so a path that reaches into dir
366	366	// through a symlink, or through "..", is still reported as inside.
367		func within(dir, path string) bool {
	367	func Within(dir, path string) bool {
368	368	dir, path = resolvePath(dir), resolvePath(path)
369	369	rel, err := filepath.Rel(dir, path)
370	370	return err == nil && rel != ".." && !strings.HasPrefix(rel, ".."+string(filepath.Separator))
		@@ -430,7 +430,7 @@ func (c Config) Validate() error {
430	430	switch {
431	431	case c.Server.SecretKeyFile == "":
432	432	errs = append(errs, errors.New("server.secret_key_file is required"))
433		case within(c.Server.Root, c.Server.SecretKeyFile):
	433	case Within(c.Server.Root, c.Server.SecretKeyFile):
434	434	errs = append(errs, fmt.Errorf("server.secret_key_file %q is inside server.root: backups of the root would carry the key beside the values it seals", c.Server.SecretKeyFile))
435	435	}
436	436	if err := oneOf("ssh.mode", c.SSH.Mode, "embedded", "system"); err != nil {

internal/control/admin.go +8

		@@ -655,6 +655,14 @@ func runAdminMRPrune(c *Ctx, args []string) int {
655	655	mrs = append(mrs, mr)
656	656	}
657	657
	658	// The prune would remove objects a running full backup has listed
	659	// and not yet read.
	660	release, code := holdOffBackup(c)
	661	if code >= 0 {
	662	return code
	663	}
	664	defer release()
	665
658	666	// The record is written as each ref goes, not after the gc: a failure
659	667	// past this point leaves refs deleted, and the audit log and the MR
660	668	// thread must say so. Re-running the same command finishes the job.

internal/control/backuplock_test.go added +57

		@@ -0,0 +1,57 @@
	1	package control
	2
	3	import (
	4	"strings"
	5	"testing"
	6
	7	"gitbay.org/gitbay/internal/backuplock"
	8	"gitbay.org/gitbay/internal/protocol"
	9	)
	10
	11	func TestRepoDeleteAndRenameRefusedDuringBackup(t *testing.T) {
	12	st, repo, uid := newQueueTestRepo(t)
	13	owner, err := st.UserByID(uid)
	14	if err != nil {
	15	t.Fatal(err)
	16	}
	17	root := t.TempDir()
	18	release, err := backuplock.Hold(root)
	19	if err != nil {
	20	t.Fatal(err)
	21	}
	22	for _, argv := range [][]string{
	23	{"repo", "rename", repo.Path(), "renamed"},
	24	{"repo", "delete", repo.Path(), "--yes"},
	25	} {
	26	c, errOut := pruneCtx(st, root, owner)
	27	if code := Dispatch(c, argv); code != protocol.ExitFailure \|\| !strings.Contains(errOut.String(), "a backup is running") {
	28	t.Fatalf("%v during a backup: exit %d, %s", argv, code, errOut)
	29	}
	30	}
	31	if got, err := st.RepoByID(repo.ID); err != nil \|\| got.Name != repo.Name {
	32	t.Fatalf("repository changed during a backup: %+v, %v", got, err)
	33	}
	34	release()
	35
	36	c, errOut := pruneCtx(st, root, owner)
	37	if code := Dispatch(c, []string{"repo", "delete", repo.Path(), "--yes"}); code != protocol.ExitOK {
	38	t.Fatalf("delete after the backup: exit %d, %s", code, errOut)
	39	}
	40	}
	41
	42	func TestAdminMRPruneRefusedDuringBackup(t *testing.T) {
	43	st, repo, root, headSHA := prunedRepo(t)
	44	dir := RepoDir(root, repo.OwnerName, repo.Name)
	45	release, err := backuplock.Hold(root)
	46	if err != nil {
	47	t.Fatal(err)
	48	}
	49	defer release()
	50	c, errOut := pruneCtx(st, root, rootUser(t, st))
	51	if code := Dispatch(c, []string{"admin", "mr", "prune", repo.Path(), "1", "--yes"}); code != protocol.ExitFailure \|\| !strings.Contains(errOut.String(), "a backup is running") {
	52	t.Fatalf("prune during a backup: exit %d, %s", code, errOut)
	53	}
	54	if !refExists(dir, mrHeadRef(1)) \|\| !objectExists(dir, headSHA) {
	55	t.Fatal("prune during a backup changed the repository")
	56	}
	57	}

internal/control/org.go +5

		@@ -167,6 +167,11 @@ func runOrgRename(c *Ctx, args []string) int {
167	167	if _, err := os.Stat(newDir); err == nil {
168	168	return c.fail(protocol.ExitFailure, "repository directory %s already exists", newName)
169	169	}
	170	release, lockCode := holdOffBackup(c)
	171	if lockCode >= 0 {
	172	return lockCode
	173	}
	174	defer release()
170	175	if err := c.Store.RenameOrg(org.ID, newName); err != nil {
171	176	return c.failErr(err)
172	177	}

internal/control/repo.go +28

		@@ -11,6 +11,7 @@ import (
11	11	"strconv"
12	12	"strings"
13	13
	14	"gitbay.org/gitbay/internal/backuplock"
14	15	"gitbay.org/gitbay/internal/gitutil"
15	16	"gitbay.org/gitbay/internal/policy"
16	17	"gitbay.org/gitbay/internal/protocol"
		@@ -513,6 +514,11 @@ func runRepoTransfer(c *Ctx, args []string) int {
513	514	if _, err := os.Stat(newDir); err == nil {
514	515	return c.fail(protocol.ExitFailure, "repository directory already exists at %s/%s", newOwner, repo.Name)
515	516	}
	517	release, lockCode := holdOffBackup(c)
	518	if lockCode >= 0 {
	519	return lockCode
	520	}
	521	defer release()
516	522	// The directory moves before the record changes: a move that fails
517	523	// leaves nothing to undo, whereas the record's change into an org
518	524	// folds labels and milestones into the org's rows, which a revert
		@@ -557,6 +563,11 @@ func runRepoRename(c *Ctx, args []string) int {
557	563	if _, err := os.Stat(newDir); err == nil {
558	564	return c.fail(protocol.ExitFailure, "repository directory already exists at %s/%s", repo.OwnerName, newName)
559	565	}
	566	release, lockCode := holdOffBackup(c)
	567	if lockCode >= 0 {
	568	return lockCode
	569	}
	570	defer release()
560	571	if err := c.Store.RenameRepo(repo.ID, newName); err != nil {
561	572	return c.failErr(err)
562	573	}
		@@ -609,6 +620,11 @@ func runRepoDelete(c *Ctx, args []string) int {
609	620	// webhooks; an instance that needs to hear about it wants the audit log
610	621	// (#112).
611	622	func deleteRepo(c *Ctx, repo store.Repo) int {
	623	release, lockCode := holdOffBackup(c)
	624	if lockCode >= 0 {
	625	return lockCode
	626	}
	627	defer release()
612	628	// Open MRs sourced from this repo keep working (targets own the
613	629	// objects) but must show that the source is gone.
614	630	if err := c.Store.MarkSourceGoneForRepo(repo.ID); err != nil {
		@@ -625,6 +641,18 @@ func deleteRepo(c *Ctx, repo store.Repo) int {
625	641	})
626	642	}
627	643
	644	// holdOffBackup keeps a full backup from starting while a repository
	645	// directory moves or goes, and refuses while one runs: the backup's
	646	// database snapshot names every repository its walk then archives
	647	// (#259). The caller defers the returned release.
	648	func holdOffBackup(c *Ctx) (func(), int) {
	649	release, err := backuplock.TryShared(c.Cfg.Server.Root)
	650	if err != nil {
	651	return nil, c.fail(protocol.ExitFailure, "%v", err)
	652	}
	653	return release, -1
	654	}
	655
628	656	func runAccessGrant(c *Ctx, args []string) int {
629	657	if len(args) != 3 \|\| !slices.Contains([]string{"read", "write", "admin"}, args[2]) {
630	658	return c.usage()

internal/gitutil/fsck_test.go added +49

		@@ -0,0 +1,49 @@
	1	package gitutil
	2
	3	import (
	4	"os"
	5	"os/exec"
	6	"path/filepath"
	7	"strings"
	8	"testing"
	9	)
	10
	11	func TestFsckConnectivityFindsAMissingObject(t *testing.T) {
	12	dir := t.TempDir()
	13	git(t, dir, "init", "-q", "-b", "main")
	14	write(t, dir, "a.txt", "a\n")
	15	git(t, dir, "add", "a.txt")
	16	git(t, dir, "commit", "-q", "-m", "one")
	17	gitDir := filepath.Join(dir, ".git")
	18	if err := FsckConnectivity(gitDir); err != nil {
	19	t.Fatalf("intact repository: %v", err)
	20	}
	21	out, err := exec.Command("git", "-C", dir, "rev-parse", "HEAD:a.txt").Output()
	22	if err != nil {
	23	t.Fatal(err)
	24	}
	25	blob := strings.TrimSpace(string(out))
	26	if err := os.Remove(filepath.Join(dir, ".git", "objects", blob[:2], blob[2:])); err != nil {
	27	t.Fatal(err)
	28	}
	29	if err := FsckConnectivity(gitDir); err == nil {
	30	t.Fatal("a repository missing a blob passed")
	31	}
	32	}
	33
	34	// A directory that is not a repository fails, even inside one: git does
	35	// not search upward and check the enclosing repository instead.
	36	func TestFsckConnectivityRefusesANonRepository(t *testing.T) {
	37	dir := t.TempDir()
	38	git(t, dir, "init", "-q", "-b", "main")
	39	write(t, dir, "a.txt", "a\n")
	40	git(t, dir, "add", "a.txt")
	41	git(t, dir, "commit", "-q", "-m", "one")
	42	inner := filepath.Join(dir, "repos", "x.git")
	43	if err := os.MkdirAll(inner, 0o755); err != nil {
	44	t.Fatal(err)
	45	}
	46	if err := FsckConnectivity(inner); err == nil {
	47	t.Fatal("an empty directory inside a repository passed")
	48	}
	49	}

internal/gitutil/merge.go +13

		@@ -66,6 +66,19 @@ func PruneNow(dir string) error {
66	66	return nil
67	67	}
68	68
	69	// FsckConnectivity checks that every object reachable from the
	70	// repository's refs is present, without reading blob contents. A backup
	71	// verify runs it on each archived repository (#259). dir is the git
	72	// directory itself; it is passed as --git-dir so that a directory that is
	73	// not a repository fails instead of git checking one enclosing it.
	74	func FsckConnectivity(dir string) error {
	75	cmd := exec.Command(toolpath.Look("git"), "--git-dir="+dir, "fsck", "--connectivity-only", "--no-progress", "--no-dangling")
	76	if out, err := cmd.CombinedOutput(); err != nil {
	77	return fmt.Errorf("fsck --connectivity-only: %v\n%s", err, out)
	78	}
	79	return nil
	80	}
	81
69	82	// RevListRange returns commits in old..new, newest first.
70	83	func RevListRange(dir, old, new string) ([]string, error) {
71	84	cmd := exec.Command(toolpath.Look("git"), "-C", dir, "rev-list", "--end-of-options", new, "^"+old)