Commit 24b9cdbca1

24b9cdbca155aed12860c629bd4db58d9927ad8f

parent: 1ec2c9cfb7

Verified · cmc

cmc <hello@cleberg.net> · 2026-09-28 09:02 UTC

gitbayd: load the secret key file; admin secrets init, rotate, check

serve seals values still in clear, or under a retired key, before it
starts listening, and refuses to start when that fails.

Ref #273

Layout: unified · split

cmd/gitbayd/backup_test.go +2 −4
@@ -8,8 +8,6 @@ import (
88 "path/filepath"
99 "sort"
1010 "testing"
11
12 "gitbay.org/gitbay/internal/config"
1311)
1412
1513// members lists the archive's entries by name.
@@ -43,8 +41,8 @@ func members(t *testing.T, path string) []string {
4341// --db-only is what makes an hourly schedule affordable, so it has to leave
4442// the repositories out and still carry a restorable database.
4543func TestBackupDBOnlyOmitsRepositories(t *testing.T) {
46 root := t.TempDir()
47 cfg := config.Config{Server: config.Server{Root: root}}
44 cfg := testConfig(t)
45 root := cfg.Server.Root
4846 s, err := openStore(cfg)
4947 if err != nil {
5048 t.Fatal(err)
cmd/gitbayd/main.go +24
@@ -4,8 +4,10 @@ package main
44
55import (
66 "context"
7 "errors"
78 "fmt"
89 "io"
10 "io/fs"
911 "log/slog"
1012 "net"
1113 "net/http"
@@ -31,6 +33,7 @@ import (
3133 "gitbay.org/gitbay/internal/mirror"
3234 "gitbay.org/gitbay/internal/notify"
3335 "gitbay.org/gitbay/internal/push"
36 "gitbay.org/gitbay/internal/seal"
3437 "gitbay.org/gitbay/internal/sshd"
3538 "gitbay.org/gitbay/internal/store"
3639 "gitbay.org/gitbay/internal/toolpath"
@@ -38,10 +41,21 @@ import (
3841)
3942
4043func openStore(cfg config.Config) (*store.Store, error) {
44 // The key file seals the secret columns (#273). Without it the
45 // database's secrets cannot be read or written, so nothing that
46 // opens the database runs.
47 keys, err := seal.Load(cfg.Server.SecretKeyFile)
48 if errors.Is(err, fs.ErrNotExist) {
49 return nil, fmt.Errorf("secret key file %s does not exist: create it with gitbayd admin secrets init, or restore it from its off-host copy (backups do not carry it)", cfg.Server.SecretKeyFile)
50 }
51 if err != nil {
52 return nil, fmt.Errorf("secret key file: %w", err)
53 }
4154 s, err := store.Open(filepath.Join(cfg.Server.Root, "gitbay.db"))
4255 if err != nil {
4356 return nil, err
4457 }
58 s.SetKeyring(keys)
4559 // Say so when the schema moves. A restart migrates in silence otherwise,
4660 // which makes an unexpected schema version hard to attribute to the deploy
4761 // that caused it.
@@ -144,6 +158,15 @@ func serveCmd() *cobra.Command {
144158 // audit row outside the database the daemon can write. Rows
145159 // are logged at Info, which the default handler always emits.
146160 st.AuditJournal = slog.Default()
161 // Values stored before sealing existed, or under a key a
162 // rotation retired, are sealed under the current key before
163 // anything reads them. A value the key file cannot open
164 // stops the start here rather than failing each delivery.
165 if n, err := st.ResealSecrets(); err != nil {
166 return fmt.Errorf("sealing secrets under %s: %w (gitbayd admin secrets check lists every value that does not open)", cfg.Server.SecretKeyFile, err)
167 } else if n > 0 {
168 slog.Info("sealed secret values", "count", n)
169 }
147170
148171 // Regenerate hook scripts so a moved binary self-heals, then
149172 // start the hook policy socket.
@@ -422,6 +445,7 @@ func adminCmd() *cobra.Command {
422445 hostCmd("runners [--json]", "runner accounts: last poll, scope, the build each holds", "admin", "runners"),
423446 auditCmd,
424447 backupCmd(),
448 secretsCmd(),
425449 gcCmd(),
426450 adminMigrateCommitRefsCmd(),
427451 adminMigrateProfileAboutCmd(),
cmd/gitbayd/main_test.go +1 −3
@@ -6,15 +6,13 @@ import (
66 "strconv"
77 "strings"
88 "testing"
9
10 "gitbay.org/gitbay/internal/config"
119)
1210
1311// A restart that moves the schema says so. Migrations used to run in silence,
1412// which left an unexpected user_version with nothing in the journal tying it to
1513// the deploy that applied it.
1614func TestOpenStoreLogsSchemaMigration(t *testing.T) {
17 cfg := config.Config{Server: config.Server{Root: t.TempDir()}}
15 cfg := testConfig(t)
1816
1917 var buf bytes.Buffer
2018 prev := slog.Default()
cmd/gitbayd/secrets.go added +196
@@ -0,0 +1,196 @@
1package main
2
3import (
4 "errors"
5 "fmt"
6 "io"
7 "io/fs"
8 "os"
9 "sort"
10 "strings"
11 "syscall"
12
13 "github.com/spf13/cobra"
14
15 "gitbay.org/gitbay/internal/config"
16 "gitbay.org/gitbay/internal/seal"
17)
18
19// secretsCmd manages the key file that seals CI secrets, webhook
20// secrets, mirror tokens and push device tokens in the database. No
21// subcommand prints key material, only key ids.
22func secretsCmd() *cobra.Command {
23 cmd := &cobra.Command{
24 Use: "secrets",
25 Short: "the key file that seals secrets stored in the database",
26 }
27 run := func(f func(config.Config, io.Writer) error) func(*cobra.Command, []string) error {
28 return func(cmd *cobra.Command, args []string) error {
29 cfg, err := config.Load(configPath)
30 if err != nil {
31 return err
32 }
33 return f(cfg, os.Stdout)
34 }
35 }
36 cmd.AddCommand(
37 &cobra.Command{
38 Use: "init",
39 Short: "create the key file (server.secret_key_file) with one new key",
40 Long: `Creates server.secret_key_file, mode 0600, holding one new key. Run as
41root, the file is given to the owner of server.root, the daemon's user.
42Refuses when the file exists.`,
43 RunE: run(initSecrets),
44 },
45 &cobra.Command{
46 Use: "rotate",
47 Short: "seal every secret under a new key and retire the old ones",
48 Long: `Adds a new key to the key file, reseals every value under it in one
49transaction, then removes the old keys from the file. A running daemon
50re-reads the file when it changes, so no restart is needed. Run as the
51user that can replace the key file (root, for /etc/gitbay); the file
52keeps its owner. Copy the new file off the host afterwards.`,
53 RunE: run(rotateSecrets),
54 },
55 &cobra.Command{
56 Use: "check",
57 Short: "open every stored secret and count them per column by key; exit 1 if any does not open",
58 RunE: run(checkSecrets),
59 },
60 )
61 return cmd
62}
63
64// initSecrets writes a new key file. Run as root, it hands the file to
65// the owner of server.root, since the daemon reads it as that user.
66func initSecrets(cfg config.Config, w io.Writer) error {
67 path := cfg.Server.SecretKeyFile
68 if _, err := os.Lstat(path); err == nil {
69 return fmt.Errorf("%s already exists; gitbayd admin secrets rotate replaces its key", path)
70 } else if !errors.Is(err, fs.ErrNotExist) {
71 return err
72 }
73 uid, gid := -1, -1
74 if os.Geteuid() == 0 {
75 fi, err := os.Stat(cfg.Server.Root)
76 if err != nil {
77 return fmt.Errorf("the key file is given to the owner of server.root: %w", err)
78 }
79 st, ok := fi.Sys().(*syscall.Stat_t)
80 if !ok {
81 return fmt.Errorf("cannot read the owner of %s", cfg.Server.Root)
82 }
83 uid, gid = int(st.Uid), int(st.Gid)
84 }
85 k, err := seal.NewKey()
86 if err != nil {
87 return err
88 }
89 if err := seal.WriteKeys(path, []seal.Key{k}); err != nil {
90 return err
91 }
92 if uid >= 0 {
93 if err := os.Chown(path, uid, gid); err != nil {
94 // A root-owned file left behind would make a re-run refuse.
95 os.Remove(path)
96 return fmt.Errorf("could not give %s to the owner of %s, so it was removed: %w", path, cfg.Server.Root, err)
97 }
98 }
99 fmt.Fprintf(w, "wrote %s (key %s). Copy it off this host: backups do not carry it, and a restored database's secrets do not open without it.\n", path, k.ID)
100 return nil
101}
102
103// rotateSecrets adds a key, reseals under it, then drops the old keys.
104// Each step leaves a file that opens every stored value: after the first
105// write the file holds old and new keys; the reseal is one transaction;
106// the last write happens only after the reseal committed and every value
107// is confirmed under the new key. Interrupted anywhere, running it again
108// finishes the job.
109func rotateSecrets(cfg config.Config, w io.Writer) error {
110 path := cfg.Server.SecretKeyFile
111 old, err := seal.ReadKeys(path)
112 if err != nil {
113 return err
114 }
115 next, err := seal.NewKey()
116 if err != nil {
117 return err
118 }
119 if err := seal.WriteKeys(path, append(old, next)); err != nil {
120 return err
121 }
122 st, err := openStore(cfg)
123 if err != nil {
124 return err
125 }
126 defer st.Close()
127 keep := fmt.Sprintf("the key file holds the old keys and %s; run rotate again", next.ID)
128 n, err := st.ResealSecrets()
129 if err != nil {
130 return fmt.Errorf("resealing: %w (%s)", err, keep)
131 }
132 // Guards against a value sealed outside the reseal transaction under
133 // an old key; no test reaches it, since that needs a hook between the
134 // two calls.
135 use, err := st.SecretKeyUse()
136 if err != nil {
137 return fmt.Errorf("checking the reseal: %w (%s)", err, keep)
138 }
139 for id, c := range use {
140 if id != next.ID {
141 return fmt.Errorf("%d values are not under %s after the reseal (%s)", c, next.ID, keep)
142 }
143 }
144 if err := seal.WriteKeys(path, []seal.Key{next}); err != nil {
145 return err
146 }
147 retired := make([]string, len(old))
148 for i, k := range old {
149 retired[i] = k.ID
150 }
151 fmt.Fprintf(w, "key %s: resealed %d values; retired %s. Copy %s off this host.\n", next.ID, n, strings.Join(retired, ", "), path)
152 return nil
153}
154
155// checkSecrets opens every stored secret and prints, per column, how
156// many values each key sealed and every value that does not open. Any
157// such value is an error.
158func checkSecrets(cfg config.Config, w io.Writer) error {
159 st, err := openStore(cfg)
160 if err != nil {
161 return err
162 }
163 defer st.Close()
164 report, err := st.SecretReport()
165 if err != nil {
166 return err
167 }
168 failed := 0
169 for _, u := range report {
170 ids := make([]string, 0, len(u.ByKey))
171 for id := range u.ByKey {
172 ids = append(ids, id)
173 }
174 sort.Strings(ids)
175 var parts []string
176 for _, id := range ids {
177 if id == "" {
178 parts = append(parts, fmt.Sprintf("clear %d (sealed when the daemon next starts)", u.ByKey[id]))
179 } else {
180 parts = append(parts, fmt.Sprintf("key %s %d", id, u.ByKey[id]))
181 }
182 }
183 if len(parts) == 0 {
184 parts = []string{"none"}
185 }
186 fmt.Fprintf(w, "%s: %s\n", u.Column, strings.Join(parts, ", "))
187 for _, f := range u.Failed {
188 fmt.Fprintf(w, "%s row %d: %s\n", u.Column, f.RowID, f.Err)
189 failed++
190 }
191 }
192 if failed > 0 {
193 return fmt.Errorf("%s does not open %d stored values", cfg.Server.SecretKeyFile, failed)
194 }
195 return nil
196}
cmd/gitbayd/secrets_test.go added +176
@@ -0,0 +1,176 @@
1package main
2
3import (
4 "bytes"
5 "encoding/base64"
6 "os"
7 "path/filepath"
8 "strings"
9 "testing"
10
11 "gitbay.org/gitbay/internal/config"
12 "gitbay.org/gitbay/internal/seal"
13 "gitbay.org/gitbay/internal/store"
14)
15
16func TestOpenStoreRefusesWithoutKeyFile(t *testing.T) {
17 cfg := testConfig(t)
18 cfg.Server.SecretKeyFile = filepath.Join(t.TempDir(), "absent.key")
19 _, err := openStore(cfg)
20 if err == nil || !strings.Contains(err.Error(), "gitbayd admin secrets init") || !strings.Contains(err.Error(), cfg.Server.SecretKeyFile) {
21 t.Fatalf("openStore without a key file: %v", err)
22 }
23}
24
25// storeWithSecret opens cfg's store and stores one build secret.
26func storeWithSecret(t *testing.T, cfg config.Config) (*store.Store, int64) {
27 t.Helper()
28 st, err := openStore(cfg)
29 if err != nil {
30 t.Fatal(err)
31 }
32 uid, err := st.CreateUser("alice", false)
33 if err != nil {
34 t.Fatal(err)
35 }
36 repoID, err := st.CreateRepo("user", uid, "app", "public")
37 if err != nil {
38 t.Fatal(err)
39 }
40 if err := st.SetBuildSecret(repoID, "TOKEN", "v1"); err != nil {
41 t.Fatal(err)
42 }
43 return st, repoID
44}
45
46func TestRotateSecrets(t *testing.T) {
47 cfg := testConfig(t)
48 before, err := seal.ReadKeys(cfg.Server.SecretKeyFile)
49 if err != nil {
50 t.Fatal(err)
51 }
52 st, repoID := storeWithSecret(t, cfg)
53 st.Close()
54
55 var out bytes.Buffer
56 if err := rotateSecrets(cfg, &out); err != nil {
57 t.Fatalf("rotate: %v", err)
58 }
59 after, err := seal.ReadKeys(cfg.Server.SecretKeyFile)
60 if err != nil {
61 t.Fatal(err)
62 }
63 if len(after) != 1 || after[0].ID == before[0].ID {
64 t.Fatalf("key file after rotation holds %v, before %v", after, before)
65 }
66 assertNoKeyMaterial(t, out.String(), append(before, after...))
67 st, err = openStore(cfg)
68 if err != nil {
69 t.Fatal(err)
70 }
71 defer st.Close()
72 use, err := st.SecretKeyUse()
73 if err != nil || use[after[0].ID] != 1 || len(use) != 1 {
74 t.Fatalf("SecretKeyUse after rotation = %v, %v", use, err)
75 }
76 if got, _ := st.BuildSecrets(repoID); got["TOKEN"] != "v1" {
77 t.Fatalf("value after rotation: %v", got)
78 }
79}
80
81// A reseal that fails leaves the old keys in the file, so every value
82// still opens.
83func TestRotateSecretsKeepsOldKeysWhenResealFails(t *testing.T) {
84 cfg := testConfig(t)
85 before, err := seal.ReadKeys(cfg.Server.SecretKeyFile)
86 if err != nil {
87 t.Fatal(err)
88 }
89 st, repoID := storeWithSecret(t, cfg)
90 // A second value sealed under a key the file does not hold.
91 if _, err := st.DB.Exec("INSERT INTO build_secrets (repo_id, name, value) VALUES (?, 'BAD', 'gbs1:deadbeef:AAAA')", repoID); err != nil {
92 t.Fatal(err)
93 }
94 st.Close()
95
96 if err := rotateSecrets(cfg, &bytes.Buffer{}); err == nil {
97 t.Fatal("rotate succeeded over a value that does not open")
98 }
99 after, err := seal.ReadKeys(cfg.Server.SecretKeyFile)
100 if err != nil {
101 t.Fatal(err)
102 }
103 if len(after) != 2 || after[0].ID != before[0].ID {
104 t.Fatalf("key file after a failed rotation holds %v", after)
105 }
106}
107
108func TestInitSecrets(t *testing.T) {
109 cfg := testConfig(t)
110 cfg.Server.SecretKeyFile = filepath.Join(t.TempDir(), "secret.key")
111 var out bytes.Buffer
112 if err := initSecrets(cfg, &out); err != nil {
113 t.Fatal(err)
114 }
115 fi, err := os.Stat(cfg.Server.SecretKeyFile)
116 if err != nil {
117 t.Fatal(err)
118 }
119 if fi.Mode().Perm() != 0o600 {
120 t.Fatalf("mode %04o", fi.Mode().Perm())
121 }
122 keys, err := seal.ReadKeys(cfg.Server.SecretKeyFile)
123 if err != nil || len(keys) != 1 {
124 t.Fatalf("keys %v, %v", keys, err)
125 }
126 if !strings.Contains(out.String(), keys[0].ID) {
127 t.Fatalf("output does not name the key id: %q", out.String())
128 }
129 assertNoKeyMaterial(t, out.String(), keys)
130 if err := initSecrets(cfg, &out); err == nil || !strings.Contains(err.Error(), "already exists") {
131 t.Fatalf("second init: %v", err)
132 }
133 if again, _ := seal.ReadKeys(cfg.Server.SecretKeyFile); again[0].ID != keys[0].ID {
134 t.Fatal("second init replaced the key")
135 }
136}
137
138func TestCheckSecrets(t *testing.T) {
139 cfg := testConfig(t)
140 keys, err := seal.ReadKeys(cfg.Server.SecretKeyFile)
141 if err != nil {
142 t.Fatal(err)
143 }
144 st, repoID := storeWithSecret(t, cfg)
145
146 var out bytes.Buffer
147 if err := checkSecrets(cfg, &out); err != nil {
148 t.Fatalf("check: %v\n%s", err, out.String())
149 }
150 if !strings.Contains(out.String(), "build_secrets.value: key "+keys[0].ID+" 1") {
151 t.Fatalf("check output:\n%s", out.String())
152 }
153 assertNoKeyMaterial(t, out.String(), keys)
154
155 if _, err := st.DB.Exec("INSERT INTO build_secrets (repo_id, name, value) VALUES (?, 'BAD', 'gbs1:deadbeef:AAAA')", repoID); err != nil {
156 t.Fatal(err)
157 }
158 st.Close()
159 out.Reset()
160 err = checkSecrets(cfg, &out)
161 if err == nil || !strings.Contains(err.Error(), "does not open 1 stored value") {
162 t.Fatalf("check over a value that does not open: %v", err)
163 }
164 if !strings.Contains(out.String(), "deadbeef") || !strings.Contains(out.String(), "build_secrets.value row") {
165 t.Fatalf("check output does not name the failing row:\n%s", out.String())
166 }
167}
168
169func assertNoKeyMaterial(t *testing.T, out string, keys []seal.Key) {
170 t.Helper()
171 for _, k := range keys {
172 if strings.Contains(out, base64.StdEncoding.EncodeToString(k.Secret)) {
173 t.Fatalf("output carries key %s's secret", k.ID)
174 }
175 }
176}
cmd/gitbayd/testconfig_test.go added +24
@@ -0,0 +1,24 @@
1package main
2
3import (
4 "path/filepath"
5 "testing"
6
7 "gitbay.org/gitbay/internal/config"
8 "gitbay.org/gitbay/internal/seal"
9)
10
11// testConfig is a config with a fresh root and a key file outside it,
12// the minimum openStore accepts.
13func testConfig(t *testing.T) config.Config {
14 t.Helper()
15 key := filepath.Join(t.TempDir(), "secret.key")
16 k, err := seal.NewKey()
17 if err != nil {
18 t.Fatal(err)
19 }
20 if err := seal.WriteKeys(key, []seal.Key{k}); err != nil {
21 t.Fatal(err)
22 }
23 return config.Config{Server: config.Server{Root: t.TempDir(), SecretKeyFile: key}}
24}
internal/store/secrets.go +46 −7
@@ -181,22 +181,61 @@ func (s *Store) ResealSecrets() (int, error) {
181181 return n, tx.Commit()
182182}
183183
184// SecretKeyUse counts the values in the secret columns by the id of the
185// key that sealed them ("" for a value still in clear), opening each
186// one, so a wrong or incomplete key file is an error naming the row.
187func (s *Store) SecretKeyUse() (map[string]int, error) {
188 use := map[string]int{}
184// SecretColumnUse is one secret column's values by the id of the key
185// that sealed them ("" for a value still in clear), and the values that
186// do not open under the loaded key file.
187type SecretColumnUse struct {
188 Column string // "<table>.<column>"
189 ByKey map[string]int
190 Failed []SecretFailure
191}
192
193// SecretFailure is a stored value that does not open.
194type SecretFailure struct {
195 RowID int64
196 Err error
197}
198
199// SecretReport opens every value in the secret columns and counts them
200// per column by key id. A value that does not open is listed rather than
201// ending the scan.
202func (s *Store) SecretReport() ([]SecretColumnUse, error) {
203 var out []SecretColumnUse
189204 for _, c := range secretColumns {
190205 rows, err := secretRows(s.DB, c)
191206 if err != nil {
192207 return nil, err
193208 }
209 u := SecretColumnUse{Column: c.table + "." + c.column, ByKey: map[string]int{}}
194210 for _, r := range rows {
195211 if _, err := s.openValue(r.aad, r.value); err != nil {
196 return nil, fmt.Errorf("%s.%s row %d: %w", c.table, c.column, r.rowid, err)
212 u.Failed = append(u.Failed, SecretFailure{RowID: r.rowid, Err: err})
213 continue
197214 }
198215 id, _ := seal.KeyID(r.value)
199 use[id]++
216 u.ByKey[id]++
217 }
218 out = append(out, u)
219 }
220 return out, nil
221}
222
223// SecretKeyUse counts the values in the secret columns by the id of the
224// key that sealed them ("" for a value still in clear), opening each
225// one, so a wrong or incomplete key file is an error naming the row.
226func (s *Store) SecretKeyUse() (map[string]int, error) {
227 report, err := s.SecretReport()
228 if err != nil {
229 return nil, err
230 }
231 use := map[string]int{}
232 for _, u := range report {
233 if len(u.Failed) > 0 {
234 f := u.Failed[0]
235 return nil, fmt.Errorf("%s row %d: %w", u.Column, f.RowID, f.Err)
236 }
237 for id, n := range u.ByKey {
238 use[id] += n
200239 }
201240 }
202241 return use, nil
internal/store/store.go +3 −1
@@ -58,7 +58,9 @@ type Store struct {
5858// no failures, and readers, which WAL keeps out of the way, are
5959// unaffected (#121).
6060func Open(path string) (*Store, error) {
61 dsn := path + "?_txlock=immediate&_pragma=journal_mode(WAL)&_pragma=foreign_keys(ON)&_pragma=busy_timeout(5000)"
61 // synchronous(FULL): a commit is durable before it returns, which
62 // secret key rotation needs before it drops the old keys (#273).
63 dsn := path + "?_txlock=immediate&_pragma=journal_mode(WAL)&_pragma=synchronous(FULL)&_pragma=foreign_keys(ON)&_pragma=busy_timeout(5000)"
6264 if path == ":memory:" {
6365 dsn = ":memory:?_txlock=immediate&_pragma=foreign_keys(ON)"
6466 }