Commit `8994b7770e`

8994b7770ea48df9f349ddcd0f2c3c72979d7280

parent: 32fb00e679

Verified · cmc ci/build: success ci/test: success

cmc <hello@cleberg.net> · 2026-09-28 22:38 UTC

backup: verify skips commondir; refs/ before packed-refs; tighter stale match

Ref #259

Layout: unified · split

.gitbay/wiki/Admin.org +4 −3

		@@ -470,9 +470,10 @@ and =--verify= reports it if one does not.
470	470	Verifying an archive of unknown origin: run it as an unprivileged
471	471	user. The connectivity check runs git with =--git-dir= on each
472	472	extracted repository, so a directory that is not a repository fails
473		rather than git checking an enclosing one, and =objects/info/alternates=
474		members are not extracted, so an archive cannot point git at object
475		stores elsewhere on the host. Git still reads each archived
	473	rather than git checking an enclosing one. =objects/info/alternates=
	474	and a =commondir= directly in a =*.git= directory are not extracted,
	475	so an archive cannot use either to have git read another repository's
	476	objects or refs on the host. Git still reads each archived
476	477	repository's own =config=.
477	478
478	479	Archives carry a directory entry for every directory, including an

.gitbay/wiki/Architecture/08-Operations.org +4 −2

		@@ -53,8 +53,10 @@ the product activity feed, not an audit trail.
53	53	\| Offsite (restic)\| nightly \| per prune policy \| =/var/lib/gitbay= and a staged database copy, to object storage \|
54	54
55	55	- The database snapshot is taken before repositories are read, and
56		each repository's HEAD, packed-refs and refs/ are archived before its
57		objects, so every archived ref finds the objects it reaches. A push
	56	each repository's HEAD, refs/ and packed-refs are archived before its
	57	objects, so every archived ref finds the objects it reaches, unless
	58	git's own automatic gc after a push repacks during the walk; the
	59	archive can then miss objects, and =--verify= reports it. A push
58	60	during the backup is missing or present as unreferenced objects
59	61	(=cmd/gitbayd/backup.go=).
60	62	- Excluded: WAL files, the hook socket, askpass scripts, generated

.gitbay/wiki/Threat-Model.org +3 −1

		@@ -265,7 +265,9 @@ assume has been checked.
265	265	- Backups snapshot the database first, and repository deletes and
266	266	moves wait out a full backup. Each repository's refs are archived
267	267	before its objects, so every archived ref finds the objects it
268		reaches. A push during a backup may be missing from the archive, or
	268	reaches, unless git's own automatic gc after a push repacks during
	269	the walk: the archive can then miss objects, and =--verify= reports
	270	it. A push during a backup may be missing from the archive, or
269	271	present as objects no archived ref names, and a repository's refs may
270	272	be newer than the database snapshot (see [[Admin]]).
271	273	- The audit log lives in the database the daemon writes, so anyone with

CHANGELOG.org +8 −4

		@@ -193,9 +193,11 @@ missing, =gitbayd admin backup --verify <archive>= names it, and
193	193	backup is running" while it runs. =--verify= now also runs =git
194	194	fsck --connectivity-only= on each archived repository and names any
195	195	that fail. (#259)
196		- A full backup archives each repository's HEAD, =packed-refs= and
197		=refs/= before its objects, so a push during the backup cannot leave
198		an archived ref naming objects the archive lacks. (#259)
	196	- A full backup archives each repository's HEAD, =refs/= and
	197	=packed-refs= before its objects, so a push during the backup cannot
	198	leave an archived ref naming objects the archive lacks. The exception
	199	is git's own automatic gc after a push repacking during the walk: the
	200	archive can then miss objects, and =--verify= reports it. (#259)
199	201	- =gitbayd admin gc= and =admin mr prune= refuse with "a backup is
200	202	running" during a full backup. A pack or loose object that git's
201	203	automatic gc removes while the backup walks is skipped instead of
		@@ -208,7 +210,9 @@ missing, =gitbayd admin backup --verify <archive>= names it, and
208	210	left beside its archive once they are a day old. (#259)
209	211	- =gitbayd admin backup --verify= runs fsck with =--git-dir=, so a
210	212	directory that is not a repository fails instead of git checking an
211		enclosing one, and does not extract =objects/info/alternates=. (#259)
	213	enclosing one, and does not extract =objects/info/alternates= or a
	214	repository's =commondir=, so neither can point fsck at another
	215	repository on the host. (#259)
212	216	- =gitbayd admin secrets init= and =rotate= hold an flock on =<key
213	217	file>.lock=, so two runs at once serialize. (#273)
214	218

cmd/gitbayd/backup.go +46 −21

		@@ -9,7 +9,9 @@ import (
9	9	"io"
10	10	"io/fs"
11	11	"os"
	12	"path"
12	13	"path/filepath"
	14	"regexp"
13	15	"strings"
14	16	"time"
15	17
		@@ -277,6 +279,10 @@ func runBackup(cfg config.Config, out string, dbOnly bool) error {
277	279	// younger than this.
278	280	const staleAge = 24 * time.Hour
279	281
	282	// tmpArchive is a temporary archive's name: os.CreateTemp's pattern
	283	// "."+base+".tmp-" followed by the digits it appends.
	284	var tmpArchive = regexp.MustCompile(`^\..+\.tmp-[0-9]+$`)
	285
280	286	// removeStale removes what a killed run left in dir: snapshot
281	287	// directories and temporary archives last modified before cutoff.
282	288	func removeStale(dir string, cutoff time.Time) {
		@@ -287,7 +293,7 @@ func removeStale(dir string, cutoff time.Time) {
287	293	for _, e := range ents {
288	294	name := e.Name()
289	295	snap := e.IsDir() && strings.HasPrefix(name, ".gitbay-snap-")
290		tmp := e.Type().IsRegular() && strings.HasPrefix(name, ".") && strings.Contains(name, ".tmp-")
	296	tmp := e.Type().IsRegular() && tmpArchive.MatchString(name)
291	297	if !snap && !tmp {
292	298	continue
293	299	}
		@@ -315,25 +321,40 @@ var refNames = map[string]bool{"HEAD": true, "packed-refs": true, "refs": true}
315	321	// use it to write into the repository at that point.
316	322	var afterRefs = func(repo string) {}
317	323
318		// addRefs archives HEAD, packed-refs and refs/ of the repository at
319		// path, whichever exist.
	324	// addRefs archives HEAD, refs/ and packed-refs of the repository at
	325	// path, whichever exist. refs/ is read before packed-refs, the order git
	326	// reads them in: pack-refs writes packed-refs before deleting the loose
	327	// refs it packed, so a ref moving between the two is caught in one.
320	328	func addRefs(tw *tar.Writer, path, name string) error {
321		for _, f := range []string{"HEAD", "packed-refs"} {
322		fi, err := os.Lstat(filepath.Join(path, f))
323		if errors.Is(err, fs.ErrNotExist) \|\| err == nil && !fi.Mode().IsRegular() {
324		continue
325		}
326		if err != nil {
327		return err
328		}
329		if err := addFile(tw, filepath.Join(path, f), name+"/"+f); err != nil {
	329	if err := addRegular(tw, path, name, "HEAD"); err != nil {
	330	return err
	331	}
	332	refs := filepath.Join(path, "refs")
	333	if _, err := os.Lstat(refs); err == nil {
	334	if err := addTree(tw, path, name, refs); err != nil {
330	335	return err
331	336	}
	337	} else if !errors.Is(err, fs.ErrNotExist) {
	338	return err
332	339	}
333		refs := filepath.Join(path, "refs")
334		if _, err := os.Lstat(refs); errors.Is(err, fs.ErrNotExist) {
	340	return addRegular(tw, path, name, "packed-refs")
	341	}
	342
	343	// addRegular archives the regular file f in the repository at path, if
	344	// it exists.
	345	func addRegular(tw *tar.Writer, path, name, f string) error {
	346	fi, err := os.Lstat(filepath.Join(path, f))
	347	if errors.Is(err, fs.ErrNotExist) \|\| err == nil && !fi.Mode().IsRegular() {
335	348	return nil
336	349	}
	350	if err != nil {
	351	return err
	352	}
	353	return addFile(tw, filepath.Join(path, f), name+"/"+f)
	354	}
	355
	356	// addTree archives the directory refs inside the repository at path.
	357	func addTree(tw *tar.Writer, path, name, refs string) error {
337	358	return filepath.WalkDir(refs, func(p string, d fs.DirEntry, err error) error {
338	359	if err != nil {
339	360	return err
		@@ -489,7 +510,7 @@ func verifyBackup(path, identity string) error {
489	510	if !filepath.IsLocal(trimmed) {
490	511	return fmt.Errorf("%s: member %q leaves the archive root", path, h.Name)
491	512	}
492		if alternates(trimmed) {
	513	if borrowsObjects(trimmed) {
493	514	continue
494	515	}
495	516	dest := filepath.Join(tmp, filepath.FromSlash(trimmed))
		@@ -566,13 +587,17 @@ func verifyBackup(path, identity string) error {
566	587	return nil
567	588	}
568	589
569		// alternates reports an archive member that would point git at object
570		// stores outside the extracted repository. gitbay writes none, and one in
571		// a hostile archive would have fsck read other paths, so verify leaves
572		// them out. The comparison ignores case, as a case-insensitive
573		// filesystem would.
574		func alternates(name string) bool {
	590	// borrowsObjects reports an archive member that would point git at
	591	// objects or refs outside the extracted repository: alternates, or a
	592	// commondir directly in a *.git directory. gitbay writes none, and one in
	593	// a hostile archive would have fsck read another repository on the host,
	594	// so verify leaves them out. The comparison ignores case, as a
	595	// case-insensitive filesystem would.
	596	func borrowsObjects(name string) bool {
575	597	name = strings.ToLower(filepath.ToSlash(filepath.Clean(name)))
	598	if dir, base := path.Split(name); base == "commondir" && strings.HasSuffix(strings.TrimSuffix(dir, "/"), ".git") {
	599	return true
	600	}
576	601	return strings.HasSuffix(name, "/objects/info/alternates") \|\|
577	602	strings.HasSuffix(name, "/objects/info/http-alternates")
578	603	}

cmd/gitbayd/backup_test.go +74 −6

		@@ -18,6 +18,7 @@ import (
18	18
19	19	"gitbay.org/gitbay/internal/backuplock"
20	20	"gitbay.org/gitbay/internal/config"
	21	"gitbay.org/gitbay/internal/gitutil"
21	22	)
22	23
23	24	// members lists the archive's entries by name.
		@@ -679,15 +680,16 @@ func TestBackupRemovesStaleTemporaries(t *testing.T) {
679	680	}
680	681	}
681	682	mk(".gitbay-snap-old", true, old)
682		mk(".b.tar.gz.tmp-old", false, old)
	683	mk(".b.tar.gz.tmp-123", false, old)
683	684	mk(".gitbay-snap-new", true, time.Now())
684		mk(".b.tar.gz.tmp-new", false, time.Now())
	685	mk(".b.tar.gz.tmp-456", false, time.Now())
685	686	mk(".keep", false, old)
	687	mk(".notes.tmp-draft", false, old)
686	688	if err := runBackup(cfg, filepath.Join(dir, "b.tar.gz"), true); err != nil {
687	689	t.Fatal(err)
688	690	}
689	691	got := leftovers(t, dir)
690		want := []string{".b.tar.gz.tmp-new", ".gitbay-snap-new", ".keep"}
	692	want := []string{".b.tar.gz.tmp-456", ".gitbay-snap-new", ".keep", ".notes.tmp-draft"}
691	693	sort.Strings(got)
692	694	if strings.Join(got, " ") != strings.Join(want, " ") {
693	695	t.Errorf("left %v, want %v", got, want)
		@@ -758,7 +760,7 @@ func TestVerifyIgnoresAlternates(t *testing.T) {
758	760	}
759	761	}
760	762
761		func TestAlternatesMember(t *testing.T) {
	763	func TestBorrowsObjectsMember(t *testing.T) {
762	764	for name, want := range map[string]bool{
763	765	"repos/a/b.git/objects/info/alternates": true,
764	766	"repos/a/b.git/objects/info/./alternates": true,
		@@ -766,9 +768,75 @@ func TestAlternatesMember(t *testing.T) {
766	768	"repos/a/b.git/objects/info/http-alternates": true,
767	769	"repos/a/b.git/objects/info/packs": false,
768	770	"repos/a/b.git/refs/heads/alternates": false,
	771	"repos/a/b.git/commondir": true,
	772	"repos/a/b.git/CommonDir": true,
	773	"repos/a/b.git/refs/heads/commondir": false,
769	774	} {
770		if got := alternates(name); got != want {
771		t.Errorf("alternates(%q) = %v, want %v", name, got, want)
	775	if got := borrowsObjects(name); got != want {
	776	t.Errorf("borrowsObjects(%q) = %v, want %v", name, got, want)
772	777	}
773	778	}
774	779	}
	780
	781	// verify does not extract a commondir, so an archived repository with
	782	// none of its own objects cannot pass by pointing git at a repository on
	783	// the host.
	784	func TestVerifyIgnoresCommondir(t *testing.T) {
	785	cfg := testConfig(t)
	786	st, err := openStore(cfg)
	787	if err != nil {
	788	t.Fatal(err)
	789	}
	790	uid, err := st.CreateUser("krz", false)
	791	if err != nil {
	792	t.Fatal(err)
	793	}
	794	if _, err := st.CreateRepo("user", uid, "thing", "public"); err != nil {
	795	t.Fatal(err)
	796	}
	797	st.Close()
	798
	799	work := t.TempDir()
	800	gitIn(t, work, "init", "-q", "-b", "main")
	801	if err := os.WriteFile(filepath.Join(work, "a.txt"), []byte("a\n"), 0o644); err != nil {
	802	t.Fatal(err)
	803	}
	804	gitIn(t, work, "add", "a.txt")
	805	gitIn(t, work, "commit", "-q", "-m", "one")
	806	host := filepath.Join(t.TempDir(), "host.git")
	807	gitIn(t, work, "clone", "-q", "--bare", work, host)
	808	gitIn(t, host, "pack-refs", "--all")
	809
	810	// A repository whose refs are its own and whose objects directory is
	811	// empty, borrowing everything else from host through commondir.
	812	dir := filepath.Join(cfg.Server.Root, "repos", "krz", "thing.git")
	813	for _, d := range []string{"objects", "refs"} {
	814	if err := os.MkdirAll(filepath.Join(dir, d), 0o755); err != nil {
	815	t.Fatal(err)
	816	}
	817	}
	818	packed, err := os.ReadFile(filepath.Join(host, "packed-refs"))
	819	if err != nil {
	820	t.Fatal(err)
	821	}
	822	for name, body := range map[string]string{
	823	"HEAD": "ref: refs/heads/main\n",
	824	"packed-refs": string(packed),
	825	"commondir": host + "\n",
	826	} {
	827	if err := os.WriteFile(filepath.Join(dir, name), []byte(body), 0o644); err != nil {
	828	t.Fatal(err)
	829	}
	830	}
	831	if err := gitutil.FsckConnectivity(dir); err != nil {
	832	t.Fatalf("with commondir on the host the repository should pass: %v", err)
	833	}
	834
	835	archive := filepath.Join(t.TempDir(), "b.tar.gz")
	836	if err := runBackup(cfg, archive, false); err != nil {
	837	t.Fatal(err)
	838	}
	839	if err := verifyBackup(archive, ""); err == nil \|\| !strings.Contains(err.Error(), "connectivity") {
	840	t.Fatalf("verify of a repository whose objects are only in its commondir: %v", err)
	841	}
	842	}

Commit 8994b7770e

Commit `8994b7770e`