Commit 24b9cdbca1
Verified · cmc
Layout: unified · split
cmd/gitbayd/backup_test.go +2 −4
| @@ -8,8 +8,6 @@ import ( | |||
| 8 | "path/filepath" | 8 | "path/filepath" |
| 9 | "sort" | 9 | "sort" |
| 10 | "testing" | 10 | "testing" |
| 11 | |||
| 12 | "gitbay.org/gitbay/internal/config" | ||
| 13 | ) | 11 | ) |
| 14 | 12 | ||
| 15 | // members lists the archive's entries by name. | 13 | // members lists the archive's entries by name. |
| @@ -43,8 +41,8 @@ func members(t *testing.T, path string) []string { | |||
| 43 | // --db-only is what makes an hourly schedule affordable, so it has to leave | 41 | // --db-only is what makes an hourly schedule affordable, so it has to leave |
| 44 | // the repositories out and still carry a restorable database. | 42 | // the repositories out and still carry a restorable database. |
| 45 | func TestBackupDBOnlyOmitsRepositories(t *testing.T) { | 43 | func TestBackupDBOnlyOmitsRepositories(t *testing.T) { |
| 46 | root := t.TempDir() | 44 | cfg := testConfig(t) |
| 47 | cfg := config.Config{Server: config.Server{Root: root}} | 45 | root := cfg.Server.Root |
| 48 | s, err := openStore(cfg) | 46 | s, err := openStore(cfg) |
| 49 | if err != nil { | 47 | if err != nil { |
| 50 | t.Fatal(err) | 48 | t.Fatal(err) |
cmd/gitbayd/main.go +24
| @@ -4,8 +4,10 @@ package main | |||
| 4 | 4 | ||
| 5 | import ( | 5 | import ( |
| 6 | "context" | 6 | "context" |
| 7 | "errors" | ||
| 7 | "fmt" | 8 | "fmt" |
| 8 | "io" | 9 | "io" |
| 10 | "io/fs" | ||
| 9 | "log/slog" | 11 | "log/slog" |
| 10 | "net" | 12 | "net" |
| 11 | "net/http" | 13 | "net/http" |
| @@ -31,6 +33,7 @@ import ( | |||
| 31 | "gitbay.org/gitbay/internal/mirror" | 33 | "gitbay.org/gitbay/internal/mirror" |
| 32 | "gitbay.org/gitbay/internal/notify" | 34 | "gitbay.org/gitbay/internal/notify" |
| 33 | "gitbay.org/gitbay/internal/push" | 35 | "gitbay.org/gitbay/internal/push" |
| 36 | "gitbay.org/gitbay/internal/seal" | ||
| 34 | "gitbay.org/gitbay/internal/sshd" | 37 | "gitbay.org/gitbay/internal/sshd" |
| 35 | "gitbay.org/gitbay/internal/store" | 38 | "gitbay.org/gitbay/internal/store" |
| 36 | "gitbay.org/gitbay/internal/toolpath" | 39 | "gitbay.org/gitbay/internal/toolpath" |
| @@ -38,10 +41,21 @@ import ( | |||
| 38 | ) | 41 | ) |
| 39 | 42 | ||
| 40 | func openStore(cfg config.Config) (*store.Store, error) { | 43 | func openStore(cfg config.Config) (*store.Store, error) { |
| 44 | // The key file seals the secret columns (#273). Without it the | ||
| 45 | // database's secrets cannot be read or written, so nothing that | ||
| 46 | // opens the database runs. | ||
| 47 | keys, err := seal.Load(cfg.Server.SecretKeyFile) | ||
| 48 | if errors.Is(err, fs.ErrNotExist) { | ||
| 49 | return nil, fmt.Errorf("secret key file %s does not exist: create it with gitbayd admin secrets init, or restore it from its off-host copy (backups do not carry it)", cfg.Server.SecretKeyFile) | ||
| 50 | } | ||
| 51 | if err != nil { | ||
| 52 | return nil, fmt.Errorf("secret key file: %w", err) | ||
| 53 | } | ||
| 41 | s, err := store.Open(filepath.Join(cfg.Server.Root, "gitbay.db")) | 54 | s, err := store.Open(filepath.Join(cfg.Server.Root, "gitbay.db")) |
| 42 | if err != nil { | 55 | if err != nil { |
| 43 | return nil, err | 56 | return nil, err |
| 44 | } | 57 | } |
| 58 | s.SetKeyring(keys) | ||
| 45 | // Say so when the schema moves. A restart migrates in silence otherwise, | 59 | // Say so when the schema moves. A restart migrates in silence otherwise, |
| 46 | // which makes an unexpected schema version hard to attribute to the deploy | 60 | // which makes an unexpected schema version hard to attribute to the deploy |
| 47 | // that caused it. | 61 | // that caused it. |
| @@ -144,6 +158,15 @@ func serveCmd() *cobra.Command { | |||
| 144 | // audit row outside the database the daemon can write. Rows | 158 | // audit row outside the database the daemon can write. Rows |
| 145 | // are logged at Info, which the default handler always emits. | 159 | // are logged at Info, which the default handler always emits. |
| 146 | st.AuditJournal = slog.Default() | 160 | st.AuditJournal = slog.Default() |
| 161 | // Values stored before sealing existed, or under a key a | ||
| 162 | // rotation retired, are sealed under the current key before | ||
| 163 | // anything reads them. A value the key file cannot open | ||
| 164 | // stops the start here rather than failing each delivery. | ||
| 165 | if n, err := st.ResealSecrets(); err != nil { | ||
| 166 | return fmt.Errorf("sealing secrets under %s: %w (gitbayd admin secrets check lists every value that does not open)", cfg.Server.SecretKeyFile, err) | ||
| 167 | } else if n > 0 { | ||
| 168 | slog.Info("sealed secret values", "count", n) | ||
| 169 | } | ||
| 147 | 170 | ||
| 148 | // Regenerate hook scripts so a moved binary self-heals, then | 171 | // Regenerate hook scripts so a moved binary self-heals, then |
| 149 | // start the hook policy socket. | 172 | // start the hook policy socket. |
| @@ -422,6 +445,7 @@ func adminCmd() *cobra.Command { | |||
| 422 | hostCmd("runners [--json]", "runner accounts: last poll, scope, the build each holds", "admin", "runners"), | 445 | hostCmd("runners [--json]", "runner accounts: last poll, scope, the build each holds", "admin", "runners"), |
| 423 | auditCmd, | 446 | auditCmd, |
| 424 | backupCmd(), | 447 | backupCmd(), |
| 448 | secretsCmd(), | ||
| 425 | gcCmd(), | 449 | gcCmd(), |
| 426 | adminMigrateCommitRefsCmd(), | 450 | adminMigrateCommitRefsCmd(), |
| 427 | adminMigrateProfileAboutCmd(), | 451 | adminMigrateProfileAboutCmd(), |
cmd/gitbayd/main_test.go +1 −3
| @@ -6,15 +6,13 @@ import ( | |||
| 6 | "strconv" | 6 | "strconv" |
| 7 | "strings" | 7 | "strings" |
| 8 | "testing" | 8 | "testing" |
| 9 | |||
| 10 | "gitbay.org/gitbay/internal/config" | ||
| 11 | ) | 9 | ) |
| 12 | 10 | ||
| 13 | // A restart that moves the schema says so. Migrations used to run in silence, | 11 | // A restart that moves the schema says so. Migrations used to run in silence, |
| 14 | // which left an unexpected user_version with nothing in the journal tying it to | 12 | // which left an unexpected user_version with nothing in the journal tying it to |
| 15 | // the deploy that applied it. | 13 | // the deploy that applied it. |
| 16 | func TestOpenStoreLogsSchemaMigration(t *testing.T) { | 14 | func TestOpenStoreLogsSchemaMigration(t *testing.T) { |
| 17 | cfg := config.Config{Server: config.Server{Root: t.TempDir()}} | 15 | cfg := testConfig(t) |
| 18 | 16 | ||
| 19 | var buf bytes.Buffer | 17 | var buf bytes.Buffer |
| 20 | prev := slog.Default() | 18 | prev := slog.Default() |
cmd/gitbayd/secrets.go added +196
| @@ -0,0 +1,196 @@ | |||
| 1 | package main | ||
| 2 | |||
| 3 | import ( | ||
| 4 | "errors" | ||
| 5 | "fmt" | ||
| 6 | "io" | ||
| 7 | "io/fs" | ||
| 8 | "os" | ||
| 9 | "sort" | ||
| 10 | "strings" | ||
| 11 | "syscall" | ||
| 12 | |||
| 13 | "github.com/spf13/cobra" | ||
| 14 | |||
| 15 | "gitbay.org/gitbay/internal/config" | ||
| 16 | "gitbay.org/gitbay/internal/seal" | ||
| 17 | ) | ||
| 18 | |||
| 19 | // secretsCmd manages the key file that seals CI secrets, webhook | ||
| 20 | // secrets, mirror tokens and push device tokens in the database. No | ||
| 21 | // subcommand prints key material, only key ids. | ||
| 22 | func secretsCmd() *cobra.Command { | ||
| 23 | cmd := &cobra.Command{ | ||
| 24 | Use: "secrets", | ||
| 25 | Short: "the key file that seals secrets stored in the database", | ||
| 26 | } | ||
| 27 | run := func(f func(config.Config, io.Writer) error) func(*cobra.Command, []string) error { | ||
| 28 | return func(cmd *cobra.Command, args []string) error { | ||
| 29 | cfg, err := config.Load(configPath) | ||
| 30 | if err != nil { | ||
| 31 | return err | ||
| 32 | } | ||
| 33 | return f(cfg, os.Stdout) | ||
| 34 | } | ||
| 35 | } | ||
| 36 | cmd.AddCommand( | ||
| 37 | &cobra.Command{ | ||
| 38 | Use: "init", | ||
| 39 | Short: "create the key file (server.secret_key_file) with one new key", | ||
| 40 | Long: `Creates server.secret_key_file, mode 0600, holding one new key. Run as | ||
| 41 | root, the file is given to the owner of server.root, the daemon's user. | ||
| 42 | Refuses when the file exists.`, | ||
| 43 | RunE: run(initSecrets), | ||
| 44 | }, | ||
| 45 | &cobra.Command{ | ||
| 46 | Use: "rotate", | ||
| 47 | Short: "seal every secret under a new key and retire the old ones", | ||
| 48 | Long: `Adds a new key to the key file, reseals every value under it in one | ||
| 49 | transaction, then removes the old keys from the file. A running daemon | ||
| 50 | re-reads the file when it changes, so no restart is needed. Run as the | ||
| 51 | user that can replace the key file (root, for /etc/gitbay); the file | ||
| 52 | keeps its owner. Copy the new file off the host afterwards.`, | ||
| 53 | RunE: run(rotateSecrets), | ||
| 54 | }, | ||
| 55 | &cobra.Command{ | ||
| 56 | Use: "check", | ||
| 57 | Short: "open every stored secret and count them per column by key; exit 1 if any does not open", | ||
| 58 | RunE: run(checkSecrets), | ||
| 59 | }, | ||
| 60 | ) | ||
| 61 | return cmd | ||
| 62 | } | ||
| 63 | |||
| 64 | // initSecrets writes a new key file. Run as root, it hands the file to | ||
| 65 | // the owner of server.root, since the daemon reads it as that user. | ||
| 66 | func initSecrets(cfg config.Config, w io.Writer) error { | ||
| 67 | path := cfg.Server.SecretKeyFile | ||
| 68 | if _, err := os.Lstat(path); err == nil { | ||
| 69 | return fmt.Errorf("%s already exists; gitbayd admin secrets rotate replaces its key", path) | ||
| 70 | } else if !errors.Is(err, fs.ErrNotExist) { | ||
| 71 | return err | ||
| 72 | } | ||
| 73 | uid, gid := -1, -1 | ||
| 74 | if os.Geteuid() == 0 { | ||
| 75 | fi, err := os.Stat(cfg.Server.Root) | ||
| 76 | if err != nil { | ||
| 77 | return fmt.Errorf("the key file is given to the owner of server.root: %w", err) | ||
| 78 | } | ||
| 79 | st, ok := fi.Sys().(*syscall.Stat_t) | ||
| 80 | if !ok { | ||
| 81 | return fmt.Errorf("cannot read the owner of %s", cfg.Server.Root) | ||
| 82 | } | ||
| 83 | uid, gid = int(st.Uid), int(st.Gid) | ||
| 84 | } | ||
| 85 | k, err := seal.NewKey() | ||
| 86 | if err != nil { | ||
| 87 | return err | ||
| 88 | } | ||
| 89 | if err := seal.WriteKeys(path, []seal.Key{k}); err != nil { | ||
| 90 | return err | ||
| 91 | } | ||
| 92 | if uid >= 0 { | ||
| 93 | if err := os.Chown(path, uid, gid); err != nil { | ||
| 94 | // A root-owned file left behind would make a re-run refuse. | ||
| 95 | os.Remove(path) | ||
| 96 | return fmt.Errorf("could not give %s to the owner of %s, so it was removed: %w", path, cfg.Server.Root, err) | ||
| 97 | } | ||
| 98 | } | ||
| 99 | fmt.Fprintf(w, "wrote %s (key %s). Copy it off this host: backups do not carry it, and a restored database's secrets do not open without it.\n", path, k.ID) | ||
| 100 | return nil | ||
| 101 | } | ||
| 102 | |||
| 103 | // rotateSecrets adds a key, reseals under it, then drops the old keys. | ||
| 104 | // Each step leaves a file that opens every stored value: after the first | ||
| 105 | // write the file holds old and new keys; the reseal is one transaction; | ||
| 106 | // the last write happens only after the reseal committed and every value | ||
| 107 | // is confirmed under the new key. Interrupted anywhere, running it again | ||
| 108 | // finishes the job. | ||
| 109 | func rotateSecrets(cfg config.Config, w io.Writer) error { | ||
| 110 | path := cfg.Server.SecretKeyFile | ||
| 111 | old, err := seal.ReadKeys(path) | ||
| 112 | if err != nil { | ||
| 113 | return err | ||
| 114 | } | ||
| 115 | next, err := seal.NewKey() | ||
| 116 | if err != nil { | ||
| 117 | return err | ||
| 118 | } | ||
| 119 | if err := seal.WriteKeys(path, append(old, next)); err != nil { | ||
| 120 | return err | ||
| 121 | } | ||
| 122 | st, err := openStore(cfg) | ||
| 123 | if err != nil { | ||
| 124 | return err | ||
| 125 | } | ||
| 126 | defer st.Close() | ||
| 127 | keep := fmt.Sprintf("the key file holds the old keys and %s; run rotate again", next.ID) | ||
| 128 | n, err := st.ResealSecrets() | ||
| 129 | if err != nil { | ||
| 130 | return fmt.Errorf("resealing: %w (%s)", err, keep) | ||
| 131 | } | ||
| 132 | // Guards against a value sealed outside the reseal transaction under | ||
| 133 | // an old key; no test reaches it, since that needs a hook between the | ||
| 134 | // two calls. | ||
| 135 | use, err := st.SecretKeyUse() | ||
| 136 | if err != nil { | ||
| 137 | return fmt.Errorf("checking the reseal: %w (%s)", err, keep) | ||
| 138 | } | ||
| 139 | for id, c := range use { | ||
| 140 | if id != next.ID { | ||
| 141 | return fmt.Errorf("%d values are not under %s after the reseal (%s)", c, next.ID, keep) | ||
| 142 | } | ||
| 143 | } | ||
| 144 | if err := seal.WriteKeys(path, []seal.Key{next}); err != nil { | ||
| 145 | return err | ||
| 146 | } | ||
| 147 | retired := make([]string, len(old)) | ||
| 148 | for i, k := range old { | ||
| 149 | retired[i] = k.ID | ||
| 150 | } | ||
| 151 | fmt.Fprintf(w, "key %s: resealed %d values; retired %s. Copy %s off this host.\n", next.ID, n, strings.Join(retired, ", "), path) | ||
| 152 | return nil | ||
| 153 | } | ||
| 154 | |||
| 155 | // checkSecrets opens every stored secret and prints, per column, how | ||
| 156 | // many values each key sealed and every value that does not open. Any | ||
| 157 | // such value is an error. | ||
| 158 | func checkSecrets(cfg config.Config, w io.Writer) error { | ||
| 159 | st, err := openStore(cfg) | ||
| 160 | if err != nil { | ||
| 161 | return err | ||
| 162 | } | ||
| 163 | defer st.Close() | ||
| 164 | report, err := st.SecretReport() | ||
| 165 | if err != nil { | ||
| 166 | return err | ||
| 167 | } | ||
| 168 | failed := 0 | ||
| 169 | for _, u := range report { | ||
| 170 | ids := make([]string, 0, len(u.ByKey)) | ||
| 171 | for id := range u.ByKey { | ||
| 172 | ids = append(ids, id) | ||
| 173 | } | ||
| 174 | sort.Strings(ids) | ||
| 175 | var parts []string | ||
| 176 | for _, id := range ids { | ||
| 177 | if id == "" { | ||
| 178 | parts = append(parts, fmt.Sprintf("clear %d (sealed when the daemon next starts)", u.ByKey[id])) | ||
| 179 | } else { | ||
| 180 | parts = append(parts, fmt.Sprintf("key %s %d", id, u.ByKey[id])) | ||
| 181 | } | ||
| 182 | } | ||
| 183 | if len(parts) == 0 { | ||
| 184 | parts = []string{"none"} | ||
| 185 | } | ||
| 186 | fmt.Fprintf(w, "%s: %s\n", u.Column, strings.Join(parts, ", ")) | ||
| 187 | for _, f := range u.Failed { | ||
| 188 | fmt.Fprintf(w, "%s row %d: %s\n", u.Column, f.RowID, f.Err) | ||
| 189 | failed++ | ||
| 190 | } | ||
| 191 | } | ||
| 192 | if failed > 0 { | ||
| 193 | return fmt.Errorf("%s does not open %d stored values", cfg.Server.SecretKeyFile, failed) | ||
| 194 | } | ||
| 195 | return nil | ||
| 196 | } | ||
cmd/gitbayd/secrets_test.go added +176
| @@ -0,0 +1,176 @@ | |||
| 1 | package main | ||
| 2 | |||
| 3 | import ( | ||
| 4 | "bytes" | ||
| 5 | "encoding/base64" | ||
| 6 | "os" | ||
| 7 | "path/filepath" | ||
| 8 | "strings" | ||
| 9 | "testing" | ||
| 10 | |||
| 11 | "gitbay.org/gitbay/internal/config" | ||
| 12 | "gitbay.org/gitbay/internal/seal" | ||
| 13 | "gitbay.org/gitbay/internal/store" | ||
| 14 | ) | ||
| 15 | |||
| 16 | func TestOpenStoreRefusesWithoutKeyFile(t *testing.T) { | ||
| 17 | cfg := testConfig(t) | ||
| 18 | cfg.Server.SecretKeyFile = filepath.Join(t.TempDir(), "absent.key") | ||
| 19 | _, err := openStore(cfg) | ||
| 20 | if err == nil || !strings.Contains(err.Error(), "gitbayd admin secrets init") || !strings.Contains(err.Error(), cfg.Server.SecretKeyFile) { | ||
| 21 | t.Fatalf("openStore without a key file: %v", err) | ||
| 22 | } | ||
| 23 | } | ||
| 24 | |||
| 25 | // storeWithSecret opens cfg's store and stores one build secret. | ||
| 26 | func storeWithSecret(t *testing.T, cfg config.Config) (*store.Store, int64) { | ||
| 27 | t.Helper() | ||
| 28 | st, err := openStore(cfg) | ||
| 29 | if err != nil { | ||
| 30 | t.Fatal(err) | ||
| 31 | } | ||
| 32 | uid, err := st.CreateUser("alice", false) | ||
| 33 | if err != nil { | ||
| 34 | t.Fatal(err) | ||
| 35 | } | ||
| 36 | repoID, err := st.CreateRepo("user", uid, "app", "public") | ||
| 37 | if err != nil { | ||
| 38 | t.Fatal(err) | ||
| 39 | } | ||
| 40 | if err := st.SetBuildSecret(repoID, "TOKEN", "v1"); err != nil { | ||
| 41 | t.Fatal(err) | ||
| 42 | } | ||
| 43 | return st, repoID | ||
| 44 | } | ||
| 45 | |||
| 46 | func TestRotateSecrets(t *testing.T) { | ||
| 47 | cfg := testConfig(t) | ||
| 48 | before, err := seal.ReadKeys(cfg.Server.SecretKeyFile) | ||
| 49 | if err != nil { | ||
| 50 | t.Fatal(err) | ||
| 51 | } | ||
| 52 | st, repoID := storeWithSecret(t, cfg) | ||
| 53 | st.Close() | ||
| 54 | |||
| 55 | var out bytes.Buffer | ||
| 56 | if err := rotateSecrets(cfg, &out); err != nil { | ||
| 57 | t.Fatalf("rotate: %v", err) | ||
| 58 | } | ||
| 59 | after, err := seal.ReadKeys(cfg.Server.SecretKeyFile) | ||
| 60 | if err != nil { | ||
| 61 | t.Fatal(err) | ||
| 62 | } | ||
| 63 | if len(after) != 1 || after[0].ID == before[0].ID { | ||
| 64 | t.Fatalf("key file after rotation holds %v, before %v", after, before) | ||
| 65 | } | ||
| 66 | assertNoKeyMaterial(t, out.String(), append(before, after...)) | ||
| 67 | st, err = openStore(cfg) | ||
| 68 | if err != nil { | ||
| 69 | t.Fatal(err) | ||
| 70 | } | ||
| 71 | defer st.Close() | ||
| 72 | use, err := st.SecretKeyUse() | ||
| 73 | if err != nil || use[after[0].ID] != 1 || len(use) != 1 { | ||
| 74 | t.Fatalf("SecretKeyUse after rotation = %v, %v", use, err) | ||
| 75 | } | ||
| 76 | if got, _ := st.BuildSecrets(repoID); got["TOKEN"] != "v1" { | ||
| 77 | t.Fatalf("value after rotation: %v", got) | ||
| 78 | } | ||
| 79 | } | ||
| 80 | |||
| 81 | // A reseal that fails leaves the old keys in the file, so every value | ||
| 82 | // still opens. | ||
| 83 | func TestRotateSecretsKeepsOldKeysWhenResealFails(t *testing.T) { | ||
| 84 | cfg := testConfig(t) | ||
| 85 | before, err := seal.ReadKeys(cfg.Server.SecretKeyFile) | ||
| 86 | if err != nil { | ||
| 87 | t.Fatal(err) | ||
| 88 | } | ||
| 89 | st, repoID := storeWithSecret(t, cfg) | ||
| 90 | // A second value sealed under a key the file does not hold. | ||
| 91 | if _, err := st.DB.Exec("INSERT INTO build_secrets (repo_id, name, value) VALUES (?, 'BAD', 'gbs1:deadbeef:AAAA')", repoID); err != nil { | ||
| 92 | t.Fatal(err) | ||
| 93 | } | ||
| 94 | st.Close() | ||
| 95 | |||
| 96 | if err := rotateSecrets(cfg, &bytes.Buffer{}); err == nil { | ||
| 97 | t.Fatal("rotate succeeded over a value that does not open") | ||
| 98 | } | ||
| 99 | after, err := seal.ReadKeys(cfg.Server.SecretKeyFile) | ||
| 100 | if err != nil { | ||
| 101 | t.Fatal(err) | ||
| 102 | } | ||
| 103 | if len(after) != 2 || after[0].ID != before[0].ID { | ||
| 104 | t.Fatalf("key file after a failed rotation holds %v", after) | ||
| 105 | } | ||
| 106 | } | ||
| 107 | |||
| 108 | func TestInitSecrets(t *testing.T) { | ||
| 109 | cfg := testConfig(t) | ||
| 110 | cfg.Server.SecretKeyFile = filepath.Join(t.TempDir(), "secret.key") | ||
| 111 | var out bytes.Buffer | ||
| 112 | if err := initSecrets(cfg, &out); err != nil { | ||
| 113 | t.Fatal(err) | ||
| 114 | } | ||
| 115 | fi, err := os.Stat(cfg.Server.SecretKeyFile) | ||
| 116 | if err != nil { | ||
| 117 | t.Fatal(err) | ||
| 118 | } | ||
| 119 | if fi.Mode().Perm() != 0o600 { | ||
| 120 | t.Fatalf("mode %04o", fi.Mode().Perm()) | ||
| 121 | } | ||
| 122 | keys, err := seal.ReadKeys(cfg.Server.SecretKeyFile) | ||
| 123 | if err != nil || len(keys) != 1 { | ||
| 124 | t.Fatalf("keys %v, %v", keys, err) | ||
| 125 | } | ||
| 126 | if !strings.Contains(out.String(), keys[0].ID) { | ||
| 127 | t.Fatalf("output does not name the key id: %q", out.String()) | ||
| 128 | } | ||
| 129 | assertNoKeyMaterial(t, out.String(), keys) | ||
| 130 | if err := initSecrets(cfg, &out); err == nil || !strings.Contains(err.Error(), "already exists") { | ||
| 131 | t.Fatalf("second init: %v", err) | ||
| 132 | } | ||
| 133 | if again, _ := seal.ReadKeys(cfg.Server.SecretKeyFile); again[0].ID != keys[0].ID { | ||
| 134 | t.Fatal("second init replaced the key") | ||
| 135 | } | ||
| 136 | } | ||
| 137 | |||
| 138 | func TestCheckSecrets(t *testing.T) { | ||
| 139 | cfg := testConfig(t) | ||
| 140 | keys, err := seal.ReadKeys(cfg.Server.SecretKeyFile) | ||
| 141 | if err != nil { | ||
| 142 | t.Fatal(err) | ||
| 143 | } | ||
| 144 | st, repoID := storeWithSecret(t, cfg) | ||
| 145 | |||
| 146 | var out bytes.Buffer | ||
| 147 | if err := checkSecrets(cfg, &out); err != nil { | ||
| 148 | t.Fatalf("check: %v\n%s", err, out.String()) | ||
| 149 | } | ||
| 150 | if !strings.Contains(out.String(), "build_secrets.value: key "+keys[0].ID+" 1") { | ||
| 151 | t.Fatalf("check output:\n%s", out.String()) | ||
| 152 | } | ||
| 153 | assertNoKeyMaterial(t, out.String(), keys) | ||
| 154 | |||
| 155 | if _, err := st.DB.Exec("INSERT INTO build_secrets (repo_id, name, value) VALUES (?, 'BAD', 'gbs1:deadbeef:AAAA')", repoID); err != nil { | ||
| 156 | t.Fatal(err) | ||
| 157 | } | ||
| 158 | st.Close() | ||
| 159 | out.Reset() | ||
| 160 | err = checkSecrets(cfg, &out) | ||
| 161 | if err == nil || !strings.Contains(err.Error(), "does not open 1 stored value") { | ||
| 162 | t.Fatalf("check over a value that does not open: %v", err) | ||
| 163 | } | ||
| 164 | if !strings.Contains(out.String(), "deadbeef") || !strings.Contains(out.String(), "build_secrets.value row") { | ||
| 165 | t.Fatalf("check output does not name the failing row:\n%s", out.String()) | ||
| 166 | } | ||
| 167 | } | ||
| 168 | |||
| 169 | func assertNoKeyMaterial(t *testing.T, out string, keys []seal.Key) { | ||
| 170 | t.Helper() | ||
| 171 | for _, k := range keys { | ||
| 172 | if strings.Contains(out, base64.StdEncoding.EncodeToString(k.Secret)) { | ||
| 173 | t.Fatalf("output carries key %s's secret", k.ID) | ||
| 174 | } | ||
| 175 | } | ||
| 176 | } | ||
cmd/gitbayd/testconfig_test.go added +24
| @@ -0,0 +1,24 @@ | |||
| 1 | package main | ||
| 2 | |||
| 3 | import ( | ||
| 4 | "path/filepath" | ||
| 5 | "testing" | ||
| 6 | |||
| 7 | "gitbay.org/gitbay/internal/config" | ||
| 8 | "gitbay.org/gitbay/internal/seal" | ||
| 9 | ) | ||
| 10 | |||
| 11 | // testConfig is a config with a fresh root and a key file outside it, | ||
| 12 | // the minimum openStore accepts. | ||
| 13 | func testConfig(t *testing.T) config.Config { | ||
| 14 | t.Helper() | ||
| 15 | key := filepath.Join(t.TempDir(), "secret.key") | ||
| 16 | k, err := seal.NewKey() | ||
| 17 | if err != nil { | ||
| 18 | t.Fatal(err) | ||
| 19 | } | ||
| 20 | if err := seal.WriteKeys(key, []seal.Key{k}); err != nil { | ||
| 21 | t.Fatal(err) | ||
| 22 | } | ||
| 23 | return config.Config{Server: config.Server{Root: t.TempDir(), SecretKeyFile: key}} | ||
| 24 | } | ||
internal/store/secrets.go +46 −7
| @@ -181,22 +181,61 @@ func (s *Store) ResealSecrets() (int, error) { | |||
| 181 | return n, tx.Commit() | 181 | return n, tx.Commit() |
| 182 | } | 182 | } |
| 183 | 183 | ||
| 184 | // SecretKeyUse counts the values in the secret columns by the id of the | 184 | // SecretColumnUse is one secret column's values by the id of the key |
| 185 | // key that sealed them ("" for a value still in clear), opening each | 185 | // that sealed them ("" for a value still in clear), and the values that |
| 186 | // one, so a wrong or incomplete key file is an error naming the row. | 186 | // do not open under the loaded key file. |
| 187 | func (s *Store) SecretKeyUse() (map[string]int, error) { | 187 | type SecretColumnUse struct { |
| 188 | use := map[string]int{} | 188 | Column string // "<table>.<column>" |
| 189 | ByKey map[string]int | ||
| 190 | Failed []SecretFailure | ||
| 191 | } | ||
| 192 | |||
| 193 | // SecretFailure is a stored value that does not open. | ||
| 194 | type SecretFailure struct { | ||
| 195 | RowID int64 | ||
| 196 | Err error | ||
| 197 | } | ||
| 198 | |||
| 199 | // SecretReport opens every value in the secret columns and counts them | ||
| 200 | // per column by key id. A value that does not open is listed rather than | ||
| 201 | // ending the scan. | ||
| 202 | func (s *Store) SecretReport() ([]SecretColumnUse, error) { | ||
| 203 | var out []SecretColumnUse | ||
| 189 | for _, c := range secretColumns { | 204 | for _, c := range secretColumns { |
| 190 | rows, err := secretRows(s.DB, c) | 205 | rows, err := secretRows(s.DB, c) |
| 191 | if err != nil { | 206 | if err != nil { |
| 192 | return nil, err | 207 | return nil, err |
| 193 | } | 208 | } |
| 209 | u := SecretColumnUse{Column: c.table + "." + c.column, ByKey: map[string]int{}} | ||
| 194 | for _, r := range rows { | 210 | for _, r := range rows { |
| 195 | if _, err := s.openValue(r.aad, r.value); err != nil { | 211 | if _, err := s.openValue(r.aad, r.value); err != nil { |
| 196 | return nil, fmt.Errorf("%s.%s row %d: %w", c.table, c.column, r.rowid, err) | 212 | u.Failed = append(u.Failed, SecretFailure{RowID: r.rowid, Err: err}) |
| 213 | continue | ||
| 197 | } | 214 | } |
| 198 | id, _ := seal.KeyID(r.value) | 215 | id, _ := seal.KeyID(r.value) |
| 199 | use[id]++ | 216 | u.ByKey[id]++ |
| 217 | } | ||
| 218 | out = append(out, u) | ||
| 219 | } | ||
| 220 | return out, nil | ||
| 221 | } | ||
| 222 | |||
| 223 | // SecretKeyUse counts the values in the secret columns by the id of the | ||
| 224 | // key that sealed them ("" for a value still in clear), opening each | ||
| 225 | // one, so a wrong or incomplete key file is an error naming the row. | ||
| 226 | func (s *Store) SecretKeyUse() (map[string]int, error) { | ||
| 227 | report, err := s.SecretReport() | ||
| 228 | if err != nil { | ||
| 229 | return nil, err | ||
| 230 | } | ||
| 231 | use := map[string]int{} | ||
| 232 | for _, u := range report { | ||
| 233 | if len(u.Failed) > 0 { | ||
| 234 | f := u.Failed[0] | ||
| 235 | return nil, fmt.Errorf("%s row %d: %w", u.Column, f.RowID, f.Err) | ||
| 236 | } | ||
| 237 | for id, n := range u.ByKey { | ||
| 238 | use[id] += n | ||
| 200 | } | 239 | } |
| 201 | } | 240 | } |
| 202 | return use, nil | 241 | return use, nil |
internal/store/store.go +3 −1
| @@ -58,7 +58,9 @@ type Store struct { | |||
| 58 | // no failures, and readers, which WAL keeps out of the way, are | 58 | // no failures, and readers, which WAL keeps out of the way, are |
| 59 | // unaffected (#121). | 59 | // unaffected (#121). |
| 60 | func Open(path string) (*Store, error) { | 60 | func Open(path string) (*Store, error) { |
| 61 | dsn := path + "?_txlock=immediate&_pragma=journal_mode(WAL)&_pragma=foreign_keys(ON)&_pragma=busy_timeout(5000)" | 61 | // synchronous(FULL): a commit is durable before it returns, which |
| 62 | // secret key rotation needs before it drops the old keys (#273). | ||
| 63 | dsn := path + "?_txlock=immediate&_pragma=journal_mode(WAL)&_pragma=synchronous(FULL)&_pragma=foreign_keys(ON)&_pragma=busy_timeout(5000)" | ||
| 62 | if path == ":memory:" { | 64 | if path == ":memory:" { |
| 63 | dsn = ":memory:?_txlock=immediate&_pragma=foreign_keys(ON)" | 65 | dsn = ":memory:?_txlock=immediate&_pragma=foreign_keys(ON)" |
| 64 | } | 66 | } |