Commit `12a6859743`

12a6859743fa1d5f99bfc6549d76efbdd919c4ca

parent: 03e5ee3161

Verified · cmc

cmc <hello@cleberg.net> · 2026-09-28 21:25 UTC

backup: encrypt archives to [backup] age_recipients; --verify --identity

Ref #274

Layout: unified · split

cmd/gitbayd/backup.go +134 −18

		@@ -2,6 +2,7 @@ package main
2	2
3	3	import (
4	4	"archive/tar"
	5	"bufio"
5	6	"compress/gzip"
6	7	"fmt"
7	8	"io"
		@@ -11,6 +12,7 @@ import (
11	12	"strings"
12	13	"time"
13	14
	15	"filippo.io/age"
14	16	"github.com/spf13/cobra"
15	17
16	18	"gitbay.org/gitbay/internal/config"
		@@ -26,7 +28,7 @@ import (
26	28	// objects in the archive (harmless); the reverse order could leave database
27	29	// rows pointing at objects the archive never captured.
28	30	func backupCmd() *cobra.Command {
29		var out, verify string
	31	var out, verify, identity string
30	32	var dbOnly bool
31	33	cmd := &cobra.Command{
32	34	Use: "backup",
		@@ -42,48 +44,93 @@ comments that exists nowhere else. Repositories are not in such an archive,
42	44	so it supplements a full backup and does not replace one.
43	45
44	46	Restore: extract into an empty directory, point server.root at it, start
45		gitbayd. Host keys are preserved, so clients keep their known_hosts entries.`,
	47	gitbayd. Host keys are preserved, so clients keep their known_hosts entries.
	48
	49	With [backup] age_recipients set, the archive is encrypted to those age
	50	public keys and its name ends in .age. --verify then needs --identity
	51	<file> holding a matching private key, which is kept off the host.`,
46	52	RunE: func(cmd *cobra.Command, args []string) error {
47	53	if verify != "" {
48		return verifyBackup(verify)
	54	return verifyBackup(verify, identity)
49	55	}
50	56	cfg, err := config.Load(configPath)
51	57	if err != nil {
52	58	return err
53	59	}
54		if out == "" {
55		out = fmt.Sprintf("gitbay-backup-%s.tar.gz", time.Now().UTC().Format("20060102-150405"))
56		}
57		return runBackup(cfg, out, dbOnly)
	60	return runBackup(cfg, archivePath(out, cfg, time.Now()), dbOnly)
58	61	},
59	62	}
60		cmd.Flags().StringVar(&out, "out", "", "output archive path (default gitbay-backup-<utc timestamp>.tar.gz)")
	63	cmd.Flags().StringVar(&out, "out", "", "output archive path (default gitbay-backup-<utc timestamp>.tar.gz; .age is appended when [backup] age_recipients is set)")
61	64	cmd.Flags().BoolVar(&dbOnly, "db-only", false, "archive the database snapshot alone, without repositories")
62	65	cmd.Flags().StringVar(&verify, "verify", "", "check an archive instead of writing one: database integrity, and its repositories against the archive's")
	66	cmd.Flags().StringVar(&identity, "identity", "", "with --verify: an age identity file that opens an encrypted archive")
63	67	return cmd
64	68	}
65	69
	70	// archivePath is where the archive goes: out, or a timestamped name,
	71	// ending in .age when the archive is encrypted.
	72	func archivePath(out string, cfg config.Config, now time.Time) string {
	73	if out == "" {
	74	out = fmt.Sprintf("gitbay-backup-%s.tar.gz", now.UTC().Format("20060102-150405"))
	75	}
	76	if len(cfg.Backup.AgeRecipients) > 0 && !strings.HasSuffix(out, ".age") {
	77	out += ".age"
	78	}
	79	return out
	80	}
	81
66	82	func runBackup(cfg config.Config, out string, dbOnly bool) error {
	83	var rs []age.Recipient
	84	if len(cfg.Backup.AgeRecipients) > 0 {
	85	var err error
	86	if rs, err = cfg.Backup.Recipients(); err != nil {
	87	return err
	88	}
	89	} else if strings.HasSuffix(out, ".age") {
	90	return fmt.Errorf("%s ends in .age but [backup] age_recipients is not set, so the archive would not be encrypted", out)
	91	}
	92
67	93	st, err := openStore(cfg)
68	94	if err != nil {
69	95	return err
70	96	}
71	97	defer st.Close()
72	98
73		// 1. Consistent database snapshot, before any repository is read.
74		snap := filepath.Join(os.TempDir(), fmt.Sprintf("gitbay-snap-%d.db", os.Getpid()))
75		os.Remove(snap)
76		defer os.Remove(snap)
	99	// 1. Consistent database snapshot, before any repository is read. It
	100	// goes in a fresh 0700 directory beside the archive.
	101	dir := filepath.Dir(out)
	102	snapDir, err := os.MkdirTemp(dir, ".gitbay-snap-")
	103	if err != nil {
	104	return err
	105	}
	106	defer os.RemoveAll(snapDir)
	107	snap := filepath.Join(snapDir, "gitbay.db")
77	108	if err := snapshotDB(st, snap); err != nil {
78	109	return fmt.Errorf("database snapshot: %w", err)
79	110	}
80	111
81		f, err := os.Create(out)
	112	// The archive is written to a temporary name beside out and renamed
	113	// once complete, so a failed run leaves no partial archive behind.
	114	f, err := os.CreateTemp(dir, "."+filepath.Base(out)+".tmp-")
82	115	if err != nil {
83	116	return err
84	117	}
85		defer f.Close()
86		gz := gzip.NewWriter(f)
	118	done := false
	119	defer func() {
	120	if !done {
	121	f.Close()
	122	os.Remove(f.Name())
	123	}
	124	}()
	125	var sink io.Writer = f
	126	var enc io.WriteCloser
	127	if len(rs) > 0 {
	128	if enc, err = age.Encrypt(f, rs...); err != nil {
	129	return err
	130	}
	131	sink = enc
	132	}
	133	gz := gzip.NewWriter(sink)
87	134	tw := tar.NewWriter(gz)
88	135
89	136	if err := addFile(tw, snap, "gitbay.db"); err != nil {
		@@ -137,9 +184,24 @@ func runBackup(cfg config.Config, out string, dbOnly bool) error {
137	184	if err := gz.Close(); err != nil {
138	185	return err
139	186	}
	187	if enc != nil {
	188	if err := enc.Close(); err != nil {
	189	return err
	190	}
	191	}
	192	if err := f.Sync(); err != nil {
	193	return err
	194	}
140	195	if err := f.Close(); err != nil {
141	196	return err
142	197	}
	198	if err := os.Rename(f.Name(), out); err != nil {
	199	return err
	200	}
	201	done = true
	202	if err := syncDir(dir); err != nil {
	203	return err
	204	}
143	205
144	206	info, _ := os.Stat(out)
145	207	if dbOnly {
		@@ -150,6 +212,16 @@ func runBackup(cfg config.Config, out string, dbOnly bool) error {
150	212	return nil
151	213	}
152	214
	215	// syncDir makes a rename in dir durable.
	216	func syncDir(dir string) error {
	217	d, err := os.Open(dir)
	218	if err != nil {
	219	return err
	220	}
	221	defer d.Close()
	222	return d.Sync()
	223	}
	224
153	225	// snapshotDB writes a consistent copy of the live database. VACUUM INTO
154	226	// takes a read snapshot, so concurrent daemon writes are safe under WAL.
155	227	func snapshotDB(st *store.Store, dest string) error {
		@@ -180,17 +252,22 @@ func addFile(tw *tar.Writer, path, name string) error {
180	252	return err
181	253	}
182	254
183		// verifyBackup reads an archive back: the database snapshot must pass
	255	// verifyBackup reads an archive back, decrypting it with identity when it
	256	// is encrypted: the database snapshot must pass
184	257	// SQLite's integrity check, and every repository it names must be in the
185	258	// archive. A database-only archive is checked for integrity alone and
186	259	// says so. Nothing is written except a temporary copy of the database.
187		func verifyBackup(path string) error {
	260	func verifyBackup(path, identity string) error {
188	261	f, err := os.Open(path)
189	262	if err != nil {
190	263	return err
191	264	}
192	265	defer f.Close()
193		gz, err := gzip.NewReader(f)
	266	plain, err := archiveReader(f, path, identity)
	267	if err != nil {
	268	return err
	269	}
	270	gz, err := gzip.NewReader(plain)
194	271	if err != nil {
195	272	return fmt.Errorf("%s: not a gzip archive: %w", path, err)
196	273	}
		@@ -232,6 +309,13 @@ func verifyBackup(path string) error {
232	309	}
233	310	}
234	311	}
	312	// Read to the end so gzip checks its trailer and age its final chunk.
	313	if _, err := io.Copy(io.Discard, gz); err != nil {
	314	return fmt.Errorf("%s: archive truncated or damaged: %w", path, err)
	315	}
	316	if err := gz.Close(); err != nil {
	317	return fmt.Errorf("%s: archive truncated or damaged: %w", path, err)
	318	}
235	319	if dbPath == "" {
236	320	return fmt.Errorf("%s: no gitbay.db in the archive", path)
237	321	}
		@@ -271,3 +355,35 @@ func verifyBackup(path string) error {
271	355	}
272	356	return nil
273	357	}
	358
	359	const ageHeader = "age-encryption.org/v1\n"
	360
	361	// archiveReader returns the archive's gzip stream, decrypting it first
	362	// when it is an age file.
	363	func archiveReader(f io.Reader, path, identity string) (io.Reader, error) {
	364	br := bufio.NewReader(f)
	365	head, _ := br.Peek(len(ageHeader))
	366	if string(head) != ageHeader {
	367	if identity != "" {
	368	fmt.Fprintf(os.Stderr, "%s is not encrypted; --identity was not used\n", path)
	369	}
	370	return br, nil
	371	}
	372	if identity == "" {
	373	return nil, fmt.Errorf("%s is encrypted; pass --identity <file> with the private key for one of its recipients", path)
	374	}
	375	idf, err := os.Open(identity)
	376	if err != nil {
	377	return nil, err
	378	}
	379	defer idf.Close()
	380	ids, err := age.ParseIdentities(idf)
	381	if err != nil {
	382	return nil, fmt.Errorf("%s: %w", identity, err)
	383	}
	384	r, err := age.Decrypt(br, ids...)
	385	if err != nil {
	386	return nil, fmt.Errorf("%s: decrypting: %w", path, err)
	387	}
	388	return r, nil
	389	}

cmd/gitbayd/backup_test.go +232

		@@ -3,11 +3,18 @@ package main
3	3	import (
4	4	"archive/tar"
5	5	"compress/gzip"
	6	"errors"
6	7	"io"
7	8	"os"
8	9	"path/filepath"
9	10	"sort"
	11	"strings"
10	12	"testing"
	13	"time"
	14
	15	"filippo.io/age"
	16
	17	"gitbay.org/gitbay/internal/config"
11	18	)
12	19
13	20	// members lists the archive's entries by name.
		@@ -92,3 +99,228 @@ func TestBackupDBOnlyOmitsRepositories(t *testing.T) {
92	99	t.Error("db-only backup is empty")
93	100	}
94	101	}
	102
	103	func TestBackupEncryptedToAgeRecipient(t *testing.T) {
	104	cfg := testConfig(t)
	105	id, err := age.GenerateX25519Identity()
	106	if err != nil {
	107	t.Fatal(err)
	108	}
	109	cfg.Backup.AgeRecipients = []string{id.Recipient().String()}
	110	s, err := openStore(cfg)
	111	if err != nil {
	112	t.Fatal(err)
	113	}
	114	s.Close()
	115
	116	out := filepath.Join(t.TempDir(), "b.tar.gz.age")
	117	if err := runBackup(cfg, out, true); err != nil {
	118	t.Fatal(err)
	119	}
	120	head := make([]byte, 22)
	121	f, err := os.Open(out)
	122	if err != nil {
	123	t.Fatal(err)
	124	}
	125	_, err = io.ReadFull(f, head)
	126	f.Close()
	127	if err != nil {
	128	t.Fatal(err)
	129	}
	130	if string(head) != "age-encryption.org/v1\n" {
	131	t.Fatalf("archive is not age-encrypted: %q", head)
	132	}
	133
	134	if err := verifyBackup(out, ""); err == nil \|\| !strings.Contains(err.Error(), "--identity") {
	135	t.Fatalf("verify without an identity: %v", err)
	136	}
	137	idFile := filepath.Join(t.TempDir(), "backup-identity.txt")
	138	if err := os.WriteFile(idFile, []byte(id.String()+"\n"), 0o600); err != nil {
	139	t.Fatal(err)
	140	}
	141	if err := verifyBackup(out, idFile); err != nil {
	142	t.Fatalf("verify with the identity: %v", err)
	143	}
	144	other, err := age.GenerateX25519Identity()
	145	if err != nil {
	146	t.Fatal(err)
	147	}
	148	otherFile := filepath.Join(t.TempDir(), "other.txt")
	149	if err := os.WriteFile(otherFile, []byte(other.String()+"\n"), 0o600); err != nil {
	150	t.Fatal(err)
	151	}
	152	var noMatch *age.NoIdentityMatchError
	153	if err := verifyBackup(out, otherFile); !errors.As(err, &noMatch) {
	154	t.Fatalf("verify with another identity: %v, want a no-identity-match error", err)
	155	}
	156	}
	157
	158	// leftovers lists what a backup run left in dir besides the archive.
	159	func leftovers(t *testing.T, dir string) []string {
	160	t.Helper()
	161	ents, err := os.ReadDir(dir)
	162	if err != nil {
	163	t.Fatal(err)
	164	}
	165	var names []string
	166	for _, e := range ents {
	167	if strings.HasPrefix(e.Name(), ".") {
	168	names = append(names, e.Name())
	169	}
	170	}
	171	return names
	172	}
	173
	174	// The snapshot directory and the archive's temporary file are removed
	175	// whether the run succeeds or fails, and a failed run leaves no archive.
	176	func TestBackupLeavesNoTemporaries(t *testing.T) {
	177	cfg := testConfig(t)
	178	id, err := age.GenerateX25519Identity()
	179	if err != nil {
	180	t.Fatal(err)
	181	}
	182	cfg.Backup.AgeRecipients = []string{id.Recipient().String()}
	183	s, err := openStore(cfg)
	184	if err != nil {
	185	t.Fatal(err)
	186	}
	187	s.Close()
	188
	189	dir := t.TempDir()
	190	out := filepath.Join(dir, "ok.tar.gz.age")
	191	if err := runBackup(cfg, out, false); err != nil {
	192	t.Fatal(err)
	193	}
	194	if got := leftovers(t, dir); len(got) != 0 {
	195	t.Errorf("after a successful run: %v", got)
	196	}
	197	fi, err := os.Stat(out)
	198	if err != nil {
	199	t.Fatal(err)
	200	}
	201	if fi.Mode().Perm() != 0o600 {
	202	t.Errorf("archive mode %v, want 0600", fi.Mode().Perm())
	203	}
	204
	205	// A file the walk cannot read fails the run after the snapshot and
	206	// the temporary archive exist. Root reads a mode-0 file, so the case
	207	// needs an unprivileged user.
	208	if os.Geteuid() == 0 {
	209	t.Log("running as root: skipping the mid-walk failure case")
	210	} else {
	211	unreadable := filepath.Join(cfg.Server.Root, "unreadable")
	212	if err := os.WriteFile(unreadable, []byte("x"), 0o000); err != nil {
	213	t.Fatal(err)
	214	}
	215	failed := filepath.Join(dir, "failed.tar.gz.age")
	216	err := runBackup(cfg, failed, false)
	217	os.Remove(unreadable)
	218	if err == nil {
	219	t.Fatal("backup with an unreadable file succeeded")
	220	}
	221	if _, err := os.Stat(failed); !os.IsNotExist(err) {
	222	t.Errorf("failed run left an archive: %v", err)
	223	}
	224	if got := leftovers(t, dir); len(got) != 0 {
	225	t.Errorf("after a failed run: %v", got)
	226	}
	227	}
	228
	229	bad := cfg
	230	bad.Backup.AgeRecipients = []string{"age1x"}
	231	if err := runBackup(bad, filepath.Join(dir, "bad.tar.gz.age"), true); err == nil {
	232	t.Fatal("backup with a bad recipient succeeded")
	233	}
	234	if got := leftovers(t, dir); len(got) != 0 {
	235	t.Errorf("after a bad recipient: %v", got)
	236	}
	237	}
	238
	239	func TestBackupRefusesAgeNameWithoutRecipients(t *testing.T) {
	240	cfg := testConfig(t)
	241	out := filepath.Join(t.TempDir(), "b.tar.gz.age")
	242	err := runBackup(cfg, out, true)
	243	if err == nil \|\| !strings.Contains(err.Error(), "age_recipients") {
	244	t.Fatalf("got %v, want a refusal naming age_recipients", err)
	245	}
	246	}
	247
	248	// A truncated archive fails verification even when the tar stream's end
	249	// markers survive: gzip's trailer and age's final chunk are checked.
	250	func TestVerifyRejectsTruncatedArchive(t *testing.T) {
	251	cfg := testConfig(t)
	252	s, err := openStore(cfg)
	253	if err != nil {
	254	t.Fatal(err)
	255	}
	256	s.Close()
	257	dir := t.TempDir()
	258	plain := filepath.Join(dir, "p.tar.gz")
	259	if err := runBackup(cfg, plain, true); err != nil {
	260	t.Fatal(err)
	261	}
	262	id, err := age.GenerateX25519Identity()
	263	if err != nil {
	264	t.Fatal(err)
	265	}
	266	enc := cfg
	267	enc.Backup.AgeRecipients = []string{id.Recipient().String()}
	268	sealed := filepath.Join(dir, "e.tar.gz.age")
	269	if err := runBackup(enc, sealed, true); err != nil {
	270	t.Fatal(err)
	271	}
	272	idFile := filepath.Join(dir, "id.txt")
	273	if err := os.WriteFile(idFile, []byte(id.String()+"\n"), 0o600); err != nil {
	274	t.Fatal(err)
	275	}
	276	if err := verifyBackup(plain, ""); err != nil {
	277	t.Fatalf("intact plain archive: %v", err)
	278	}
	279	if err := verifyBackup(sealed, idFile); err != nil {
	280	t.Fatalf("intact encrypted archive: %v", err)
	281	}
	282
	283	for _, c := range []struct {
	284	src string
	285	cut int
	286	identity string
	287	}{
	288	{plain, 1, ""},
	289	{sealed, 1, idFile},
	290	{sealed, 100, idFile},
	291	} {
	292	data, err := os.ReadFile(c.src)
	293	if err != nil {
	294	t.Fatal(err)
	295	}
	296	short := filepath.Join(dir, "short-"+filepath.Base(c.src))
	297	if err := os.WriteFile(short, data[:len(data)-c.cut], 0o600); err != nil {
	298	t.Fatal(err)
	299	}
	300	if err := verifyBackup(short, c.identity); err == nil {
	301	t.Errorf("%s cut by %d bytes verified", filepath.Base(c.src), c.cut)
	302	}
	303	}
	304	}
	305
	306	func TestArchivePath(t *testing.T) {
	307	now := time.Date(2026, 9, 27, 9, 0, 0, 0, time.UTC)
	308	plain := testConfig(t)
	309	enc := plain
	310	enc.Backup.AgeRecipients = []string{"age1x"}
	311	for _, c := range []struct {
	312	out string
	313	cfg config.Config
	314	want string
	315	}{
	316	{"", plain, "gitbay-backup-20260927-090000.tar.gz"},
	317	{"", enc, "gitbay-backup-20260927-090000.tar.gz.age"},
	318	{"/b/x.tar.gz", enc, "/b/x.tar.gz.age"},
	319	{"/b/x.tar.gz.age", enc, "/b/x.tar.gz.age"},
	320	{"/b/x.tar.gz", plain, "/b/x.tar.gz"},
	321	} {
	322	if got := archivePath(c.out, c.cfg, now); got != c.want {
	323	t.Errorf("archivePath(%q) = %q, want %q", c.out, got, c.want)
	324	}
	325	}
	326	}

Commit 12a6859743

Commit `12a6859743`