cmd/gitbayd/main.go
661 lines · 21555 bytes
1// gitbayd is the forge server daemon. The same binary also runs in hook mode
2// (invoked by git via core.hooksPath) and hosts the host-local admin commands.
3package main
4
5import (
6 "context"
7 "errors"
8 "fmt"
9 "io"
10 "io/fs"
11 "log/slog"
12 "net"
13 "net/http"
14 "os"
15 "os/signal"
16 "path/filepath"
17 "strconv"
18 "strings"
19 "syscall"
20 "time"
21
22 "github.com/spf13/cobra"
23 "golang.org/x/crypto/acme/autocert"
24
25 "gitbay.org/gitbay/internal/buildinfo"
26 "gitbay.org/gitbay/internal/ci"
27 "gitbay.org/gitbay/internal/config"
28 "gitbay.org/gitbay/internal/control"
29 "gitbay.org/gitbay/internal/deps"
30 "gitbay.org/gitbay/internal/gitd"
31 "gitbay.org/gitbay/internal/hookd"
32 "gitbay.org/gitbay/internal/httpd"
33 "gitbay.org/gitbay/internal/mailin"
34 "gitbay.org/gitbay/internal/mirror"
35 "gitbay.org/gitbay/internal/notify"
36 "gitbay.org/gitbay/internal/packlimit"
37 "gitbay.org/gitbay/internal/push"
38 "gitbay.org/gitbay/internal/seal"
39 "gitbay.org/gitbay/internal/sshd"
40 "gitbay.org/gitbay/internal/store"
41 "gitbay.org/gitbay/internal/symbols"
42 "gitbay.org/gitbay/internal/toolpath"
43 "gitbay.org/gitbay/internal/webhook"
44)
45
46func openStore(cfg config.Config) (*store.Store, error) {
47 // The key file seals the secret columns (#273). Without it the
48 // database's secrets cannot be read or written, so nothing that
49 // opens the database runs.
50 keys, err := seal.Load(cfg.Server.SecretKeyFile)
51 if errors.Is(err, fs.ErrNotExist) {
52 return nil, fmt.Errorf("secret key file %s does not exist: create it with gitbayd admin secrets init, or restore it from its off-host copy (backups do not carry it)", cfg.Server.SecretKeyFile)
53 }
54 if err != nil {
55 return nil, fmt.Errorf("secret key file: %w", err)
56 }
57 s, err := store.Open(filepath.Join(cfg.Server.Root, "gitbay.db"))
58 if err != nil {
59 return nil, err
60 }
61 s.SetKeyring(keys)
62 // Say so when the schema moves. A restart migrates in silence otherwise,
63 // which makes an unexpected schema version hard to attribute to the deploy
64 // that caused it.
65 before, err := s.Version()
66 if err != nil {
67 s.Close()
68 return nil, err
69 }
70 if err := s.MigrateUp(); err != nil {
71 s.Close()
72 return nil, err
73 }
74 after, err := s.Version()
75 if err != nil {
76 s.Close()
77 return nil, err
78 }
79 if after != before {
80 note, err := s.TakeMigrationNote()
81 if err != nil {
82 s.Close()
83 return nil, err
84 }
85 args := []any{"from", before, "to", after}
86 if note != "" {
87 args = append(args, "note", note)
88 }
89 slog.Info("schema migrated", args...)
90 }
91 return s, nil
92}
93
94var configPath string
95
96func main() {
97 root := &cobra.Command{
98 Use: "gitbayd",
99 Short: "gitbay server daemon",
100 SilenceUsage: true,
101 SilenceErrors: true,
102 }
103 root.PersistentFlags().StringVar(&configPath, "config", "/etc/gitbay/config.toml", "path to config file")
104
105 root.AddCommand(
106 checkConfigCmd(),
107 serveCmd(),
108 migrateCmd(),
109 adminCmd(),
110 hookCmd(),
111 authorizedKeysCmd(),
112 shellCmd(),
113 versionCmd(),
114 )
115
116 if err := root.Execute(); err != nil {
117 fmt.Fprintln(os.Stderr, "gitbayd:", err)
118 os.Exit(1)
119 }
120}
121
122func checkConfigCmd() *cobra.Command {
123 var noHost bool
124 cmd := &cobra.Command{
125 Use: "check-config",
126 Short: "validate the configuration and exit",
127 RunE: func(cmd *cobra.Command, args []string) error {
128 cfg, err := config.Load(configPath)
129 if err != nil {
130 return err
131 }
132 if !noHost {
133 if err := cfg.CheckHost(); err != nil {
134 return err
135 }
136 }
137 fmt.Println("config ok")
138 return nil
139 },
140 }
141 cmd.Flags().BoolVar(&noHost, "no-host-checks", false, "skip host environment probes (port binding, paths)")
142 return cmd
143}
144
145func serveCmd() *cobra.Command {
146 return &cobra.Command{
147 Use: "serve",
148 Short: "run the ssh, http, and git listeners",
149 RunE: func(cmd *cobra.Command, args []string) error {
150 // First line of every run: the journal then says which commit is
151 // serving, without rebuilding the binary to find out.
152 logBuild()
153 // The tools this daemon shells out to are resolved once, at
154 // package init. Say so now rather than failing on whichever
155 // request first needed git.
156 if err := toolpath.Verify(); err != nil {
157 return fmt.Errorf("required tools missing: %w", err)
158 }
159 cfg, err := config.Load(configPath)
160 if err != nil {
161 return err
162 }
163 warnIfUnmerged(cfg)
164 st, err := openStore(cfg)
165 if err != nil {
166 return err
167 }
168 defer st.Close()
169 // The daemon's stderr is the service journal: a copy of each
170 // audit row outside the database the daemon can write. Rows
171 // are logged at Info, which the default handler always emits.
172 st.AuditJournal = slog.Default()
173 // Values stored before sealing existed, or under a key a
174 // rotation retired, are sealed under the current key before
175 // anything reads them. A value the key file cannot open
176 // stops the start here rather than failing each delivery.
177 if n, err := st.ResealSecrets(); err != nil {
178 return fmt.Errorf("sealing secrets under %s: %w (gitbayd admin secrets check lists every value that does not open)", cfg.Server.SecretKeyFile, err)
179 } else if n > 0 {
180 slog.Info("sealed secret values", "count", n)
181 }
182
183 // Regenerate hook scripts so a moved binary self-heals, then
184 // start the hook policy socket.
185 self, err := os.Executable()
186 if err != nil {
187 return err
188 }
189 if err := hookd.WriteHookScripts(control.HooksDir(cfg.Server.Root), self); err != nil {
190 return err
191 }
192 stopHookd, err := hookd.Serve(cfg, st)
193 if err != nil {
194 return err
195 }
196 defer stopHookd()
197
198 // SIGTERM is how a deploy restarts the daemon: stop accepting,
199 // let what is in flight finish, exit 0 (#105).
200 ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
201 defer stop()
202
203 // Outbound webhook deliveries, and the mail queue when SMTP is
204 // configured, share one retry base. It is overridable for
205 // tests via GITBAY_WEBHOOK_RETRY_BASE; notify.DefaultRetryBase
206 // names the production default so nothing else has to guess it.
207 retryBase := notify.DefaultRetryBase
208 if v := os.Getenv("GITBAY_WEBHOOK_RETRY_BASE"); v != "" {
209 if d, err := time.ParseDuration(v); err == nil {
210 retryBase = d
211 }
212 }
213 whCtx, whCancel := context.WithCancel(context.Background())
214 defer whCancel()
215 go webhook.New(st, cfg.Webhooks.AllowLocal, retryBase).Run(whCtx)
216 if cfg.Mail.SMTPHost != "" {
217 go notify.New(st, cfg, retryBase).Run(whCtx)
218 }
219 if in := cfg.Mail.Inbound; in.Enabled {
220 // An unreadable password file is a misconfiguration:
221 // refuse to start rather than poll and fail every tick.
222 if _, err := in.Password(); err != nil {
223 return err
224 }
225 if !in.Authenticated() {
226 slog.Warn("mail reply: [mail.inbound] require_dkim and trusted_authserv_id are unset, so a reply's From is not authenticated; set one on any instance reachable from the internet")
227 }
228 go (&mailin.Poller{P: &mailin.Processor{St: st, Cfg: cfg}, In: in}).Run(whCtx)
229 }
230 if cfg.Push.Enabled {
231 p, err := push.New(st, cfg.Push, cfg.Server.SiteURL, retryBase)
232 if err != nil {
233 // Config validation already parsed the key, so this
234 // is not a misconfiguration; fail loudly rather than
235 // running with a silent delivery route.
236 slog.Error("push: starting deliverer", "err", err)
237 } else {
238 go p.Run(whCtx)
239 }
240 }
241 go mirror.New(st, cfg).Run(whCtx)
242 if d := cfg.Registration.PendingExpiryDuration(); d > 0 {
243 go reapPending(whCtx, st, d)
244 }
245 go sweep(whCtx, st, cfg)
246 go (&ci.Scheduler{St: st, SiteURL: cfg.Server.SiteURL,
247 RepoDir: func(owner, name string) string {
248 return control.RepoDir(cfg.Server.Root, owner, name)
249 }}).Run(whCtx)
250 go deps.New(st, cfg, func(owner, name string) string {
251 return control.RepoDir(cfg.Server.Root, owner, name)
252 }, buildinfo.String()).Run(whCtx)
253 go symbols.New(st, func(owner, name string) string {
254 return control.RepoDir(cfg.Server.Root, owner, name)
255 }).Run(whCtx)
256
257 // One pack-generation budget for SSH, smart HTTP and git://.
258 // Anonymous clients ("ip:" principals) share all but one
259 // slot, so an account can always get the last.
260 packMax, packPer, packQueue, packWait := cfg.Limits.PackLimits()
261 packs := packlimit.New(packMax, packPer, packQueue, packWait)
262 if packMax > 1 {
263 packs.CapClass("ip:", packMax-1)
264 }
265 // receive-pack has a budget of its own. Parallel pushes
266 // from one account (scripts, bots, several terminals)
267 // queue, up to half the queue, rather than being refused
268 // once one is waiting.
269 pushMax, pushPer, pushQueue, pushWait := cfg.Limits.PushLimits()
270 pushes := packlimit.New(pushMax, pushPer, pushQueue, pushWait)
271 pushes.Name("push")
272 pushes.CapQueue(pushPerQueue(pushQueue))
273
274 errCh := make(chan error, 3)
275 var sshSrv *sshd.Server
276 var sshLn, gitLn net.Listener
277 if cfg.SSH.Mode == "embedded" {
278 srv, err := sshd.New(cfg, st, packs, pushes)
279 if err != nil {
280 return err
281 }
282 ln, err := net.Listen("tcp", net.JoinHostPort("", strconv.Itoa(cfg.SSH.Port)))
283 if err != nil {
284 return err
285 }
286 slog.Info("ssh listening", "addr", ln.Addr())
287 sshSrv, sshLn = srv, ln
288 go func() { errCh <- srv.Serve(ln) }()
289 } else {
290 // system mode: the host sshd owns the SSH port and invokes
291 // this binary via AuthorizedKeysCommand + forced command.
292 slog.Info("ssh handled by host sshd (ssh.mode = system)")
293 }
294
295 web := httpd.New(cfg, st, packs)
296 // Header and idle timeouts bound what an idle or slow client can
297 // hold open. No write timeout: archives and upload-pack stream
298 // for as long as they take (#104).
299 hs := &http.Server{
300 Addr: cfg.HTTP.Addr,
301 Handler: web.Handler(),
302 ReadHeaderTimeout: 10 * time.Second,
303 IdleTimeout: 2 * time.Minute,
304 MaxHeaderBytes: 64 << 10,
305 }
306 go func() {
307 slog.Info("http listening", "addr", cfg.HTTP.Addr, "tls", cfg.HTTP.TLS)
308 switch cfg.HTTP.TLS {
309 case "off":
310 errCh <- hs.ListenAndServe()
311 case "files":
312 hs.TLSConfig = serverTLS(nil)
313 errCh <- hs.ListenAndServeTLS(cfg.HTTP.CertFile, cfg.HTTP.KeyFile)
314 case "acme":
315 host := cfg.SiteHost()
316 stripPort := func(hp string) string {
317 if h, _, err := net.SplitHostPort(hp); err == nil {
318 return h
319 }
320 return hp
321 }
322 // Beyond the site host, allow <owner>.<pages domain>
323 // for owners that exist — certs come on demand per
324 // subdomain, no wildcard needed.
325 hostPolicy := func(ctx context.Context, h string) error {
326 if h == host {
327 return nil
328 }
329 if pd := cfg.Pages.Domain; pd != "" {
330 if h == pd {
331 return nil // apex: serves a redirect to the forge
332 }
333 if owner, ok := strings.CutSuffix(h, "."+pd); ok &&
334 !strings.Contains(owner, ".") && st.OwnerExists(owner) {
335 return nil
336 }
337 }
338 // Custom pages domains: certs only for claimed hosts.
339 if _, err := st.PageDomainRepo(h); err == nil {
340 return nil
341 }
342 return fmt.Errorf("host %q not served here", h)
343 }
344 m := &autocert.Manager{
345 Prompt: autocert.AcceptTOS,
346 Cache: autocert.DirCache(filepath.Join(cfg.Server.Root, "acme")),
347 HostPolicy: hostPolicy,
348 Email: cfg.HTTP.ACMEEmail,
349 }
350 // TLS-ALPN-01 rides the HTTPS port itself. The optional
351 // plain-HTTP listener adds HTTP-01 and a redirect; losing
352 // it (port 80 taken, no privileges) is not fatal.
353 if addr := cfg.HTTP.ACMEHTTPAddr; addr != "" && addr != "off" {
354 redirect := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
355 // Pages hosts redirect to themselves, not the
356 // forge host.
357 target := host
358 if hostPolicy(r.Context(), stripPort(r.Host)) == nil {
359 target = stripPort(r.Host)
360 }
361 http.Redirect(w, r, "https://"+target+r.URL.RequestURI(), http.StatusMovedPermanently)
362 })
363 go func() {
364 slog.Info("acme http listening", "addr", addr)
365 acmeHTTP := &http.Server{Addr: addr, Handler: m.HTTPHandler(redirect),
366 ReadHeaderTimeout: 10 * time.Second, IdleTimeout: time.Minute}
367 if err := acmeHTTP.ListenAndServe(); err != nil {
368 slog.Warn("acme http listener failed; continuing with TLS-ALPN only", "err", err)
369 }
370 }()
371 }
372 hs.TLSConfig = serverTLS(m.TLSConfig())
373 errCh <- hs.ListenAndServeTLS("", "")
374 }
375 }()
376
377 if cfg.GitDaemon.Enabled {
378 gln, err := net.Listen("tcp", net.JoinHostPort("", strconv.Itoa(cfg.GitDaemon.Port)))
379 if err != nil {
380 return err
381 }
382 slog.Info("git-daemon listening", "addr", gln.Addr())
383 gitLn = gln
384 go func() { errCh <- gitd.New(cfg, st, packs).Serve(gln) }()
385 }
386
387 select {
388 case err := <-errCh:
389 return err
390 case <-ctx.Done():
391 }
392 slog.Info("shutting down")
393 stop()
394 for _, ln := range []net.Listener{sshLn, gitLn} {
395 if ln != nil {
396 ln.Close()
397 }
398 }
399 // Follows run until a build ends; end them first so the drain
400 // waits only for work that finishes.
401 web.Stop()
402 if sshSrv != nil {
403 sshSrv.Stop()
404 }
405 drain, cancel := context.WithTimeout(context.Background(), 30*time.Second)
406 defer cancel()
407 if err := hs.Shutdown(drain); err != nil {
408 slog.Warn("http shutdown", "err", err)
409 }
410 if sshSrv != nil {
411 if err := sshSrv.Shutdown(drain); err != nil {
412 slog.Warn("ssh shutdown", "err", err)
413 }
414 }
415 return nil
416 },
417 }
418}
419
420func migrateCmd() *cobra.Command {
421 var to int
422 cmd := &cobra.Command{
423 Use: "migrate",
424 Short: "apply schema migrations",
425 RunE: func(cmd *cobra.Command, args []string) error {
426 cfg, err := config.Load(configPath)
427 if err != nil {
428 return err
429 }
430 s, err := store.Open(cfg.Server.Root + "/gitbay.db")
431 if err != nil {
432 return err
433 }
434 defer s.Close()
435 if err := s.MigrateTo(to); err != nil {
436 return err
437 }
438 v, err := s.Version()
439 if err != nil {
440 return err
441 }
442 fmt.Println("schema version", v)
443 return nil
444 },
445 }
446 cmd.Flags().IntVar(&to, "to", -1, "target schema version (-1 = latest)")
447 return cmd
448}
449
450func adminCmd() *cobra.Command {
451 admin := &cobra.Command{
452 Use: "admin",
453 Short: "host-local administration",
454 }
455 userCmd := &cobra.Command{Use: "user", Short: "manage users"}
456 userCmd.AddCommand(
457 hostUserCreateCmd(),
458 hostCmd("list [--state active|pending|disabled|admin] [--limit n] [--cursor c]", "list accounts", "admin", "user", "list"),
459 hostCmd("show <username>", "show an account: keys, emails, orgs, tokens, sessions", "admin", "user", "show"),
460 hostCmd("disable <username>", "suspend an account: keys and sessions refused until re-enabled", "admin", "user", "disable"),
461 hostCmd("enable <username>", "restore a suspended account", "admin", "user", "enable"),
462 hostCmd("delete <username> --yes", "delete an account that anchors nothing (keys, emails, and sessions go with it)", "admin", "user", "delete"),
463 hostCmd("promote <username>", "make an account an instance admin", "admin", "user", "promote"),
464 hostCmd("demote <username>", "remove instance admin from an account (never the last one)", "admin", "user", "demote"),
465 hostCmd("limits <username> [--repos n|default] [--bytes n|default]", "show or set repository and storage caps", "admin", "user", "limits"),
466 )
467 emailCmd := &cobra.Command{Use: "email", Short: "manage user emails"}
468 emailCmd.AddCommand(hostCmd("verify <username> <address>", "mark an email verified by admin assertion", "admin", "email", "verify"))
469 repoCmd := &cobra.Command{Use: "repo", Short: "any repository, for moderation (audited)"}
470 repoCmd.AddCommand(
471 hostCmd("list [--owner o] [--visibility public|private] [--limit n] [--cursor c]", "every repository with size and last push", "admin", "repo", "list"),
472 hostCmd("archive <owner/name>", "archive a repository", "admin", "repo", "archive"),
473 hostCmd("unarchive <owner/name>", "unarchive a repository", "admin", "repo", "unarchive"),
474 hostCmd("visibility <owner/name> public|private", "set a repository's visibility", "admin", "repo", "visibility"),
475 hostCmd("delete <owner/name> --yes", "delete a repository", "admin", "repo", "delete"),
476 )
477 configCmd := &cobra.Command{Use: "config", Short: "the configuration in effect"}
478 configCmd.AddCommand(configShowCmd())
479 auditCmd := hostCmd("audit [--limit n] [--json]", "print the security audit log, newest first", "audit")
480 auditCmd.AddCommand(auditVerifyCmd())
481 admin.AddCommand(
482 userCmd,
483 emailCmd,
484 repoCmd,
485 configCmd,
486 hostCmd("invite --email <address>", "issue a registration invite and email its code", "admin", "invite"),
487 hostCmd("stats [--json]", "instance statistics: counts and per-repository disk usage", "admin", "stats"),
488 hostCmd("runners [--json]", "runner accounts: last poll, scope, the build each holds", "admin", "runners"),
489 auditCmd,
490 backupCmd(),
491 restoreDrillCmd(),
492 secretsCmd(),
493 gcCmd(),
494 adminMigrateCommitRefsCmd(),
495 adminMigrateProfileAboutCmd(),
496 adminBackfillActivityCmd(),
497 )
498 return admin
499}
500
501// hostCmd runs a registry command as the host itself: an admin context
502// with no account behind it, so audit rows carry no actor and the source
503// "host". Arguments pass through untouched; the registry owns the flags,
504// which is what keeps this surface and an admin's SSH session from
505// drifting.
506func hostCmd(use, short string, path ...string) *cobra.Command {
507 return &cobra.Command{
508 Use: use,
509 Short: short,
510 DisableFlagParsing: true,
511 RunE: func(cmd *cobra.Command, args []string) error {
512 for _, a := range args {
513 if a == "--help" || a == "-h" {
514 return cmd.Help()
515 }
516 }
517 return runAsHost(path, args, os.Stdin)
518 },
519 }
520}
521
522// hostArgs pulls the root's --config out of args: with flag parsing off,
523// cobra hands the persistent flag through untouched.
524func hostArgs(args []string) []string {
525 var rest []string
526 for i := 0; i < len(args); i++ {
527 switch {
528 case args[i] == "--config" && i+1 < len(args):
529 configPath = args[i+1]
530 i++
531 case strings.HasPrefix(args[i], "--config="):
532 configPath = strings.TrimPrefix(args[i], "--config=")
533 default:
534 rest = append(rest, args[i])
535 }
536 }
537 return rest
538}
539
540func runAsHost(path, args []string, stdin io.Reader) error {
541 args = hostArgs(args)
542 cfg, err := config.Load(configPath)
543 if err != nil {
544 return err
545 }
546 st, err := openStore(cfg)
547 if err != nil {
548 return err
549 }
550 c := &control.Ctx{
551 User: store.User{Username: "host", IsAdmin: true},
552 Scope: "full",
553 Store: st,
554 Cfg: cfg,
555 Stdin: stdin,
556 Stdout: os.Stdout,
557 Stderr: os.Stderr,
558 Source: "host",
559 }
560 code := control.Dispatch(c, append(append([]string{}, path...), args...))
561 st.Close()
562 if code != 0 {
563 os.Exit(code)
564 }
565 return nil
566}
567
568// hostUserCreateCmd keeps --key <path>, which the registry command cannot
569// take (no file paths over SSH): the file becomes the command's stdin.
570func hostUserCreateCmd() *cobra.Command {
571 return &cobra.Command{
572 Use: "create <username> [--admin] [--email <address> [--verified]] [--key <file>]",
573 Short: "create a user (host-local bootstrap; the only path in closed mode)",
574 DisableFlagParsing: true,
575 RunE: func(cmd *cobra.Command, args []string) error {
576 var stdin io.Reader = os.Stdin
577 var rest []string
578 args = hostArgs(args)
579 for i := 0; i < len(args); i++ {
580 switch {
581 case args[i] == "--help" || args[i] == "-h":
582 return cmd.Help()
583 case args[i] == "--key" && i+1 < len(args) && args[i+1] != "-":
584 f, err := os.Open(args[i+1])
585 if err != nil {
586 return err
587 }
588 defer f.Close()
589 stdin = f
590 rest = append(rest, "--key", "-")
591 i++
592 default:
593 rest = append(rest, args[i])
594 }
595 }
596 return runAsHost([]string{"admin", "user", "create"}, rest, stdin)
597 },
598 }
599}
600
601// sweep prunes expired sessions and tokens, and rows past their
602// configured retention, hourly and once at start. GITBAY_SWEEP_TICK
603// shortens the interval for tests.
604func sweep(ctx context.Context, st *store.Store, cfg config.Config) {
605 tick := time.Hour
606 if v := os.Getenv("GITBAY_SWEEP_TICK"); v != "" {
607 if d, err := time.ParseDuration(v); err == nil {
608 tick = d
609 }
610 }
611 audit, events, deliveries, mail, push := cfg.Retention.Durations()
612 r := store.Retention{Audit: audit, Events: events,
613 WebhookDeliveries: deliveries, Mail: mail, Push: push}
614 t := time.NewTicker(tick)
615 defer t.Stop()
616 for {
617 swept, err := st.Sweep(r, time.Now())
618 if err != nil {
619 slog.Error("sweeping", "err", err, "removed", swept.Total())
620 } else if n := swept.Total(); n > 0 {
621 slog.Info("swept expired rows", "removed", n, "tables", swept)
622 }
623 select {
624 case <-ctx.Done():
625 return
626 case <-t.C:
627 }
628 }
629}
630
631// reapPending removes self-registered accounts still unverified after
632// maxAge, hourly and once at start. GITBAY_REAP_TICK shortens the
633// interval for tests.
634func reapPending(ctx context.Context, st *store.Store, maxAge time.Duration) {
635 tick := time.Hour
636 if v := os.Getenv("GITBAY_REAP_TICK"); v != "" {
637 if d, err := time.ParseDuration(v); err == nil {
638 tick = d
639 }
640 }
641 t := time.NewTicker(tick)
642 defer t.Stop()
643 for {
644 if removed, err := st.ReapPendingUsers(maxAge); err != nil {
645 slog.Error("reaping pending accounts", "err", err)
646 } else if len(removed) > 0 {
647 slog.Info("removed unverified accounts", "users", removed)
648 }
649 select {
650 case <-ctx.Done():
651 return
652 case <-t.C:
653 }
654 }
655}
656
657// pushPerQueue is how many pushes one principal may have waiting: half
658// the queue, at least one.
659func pushPerQueue(queue int) int {
660 return max(1, queue/2)
661}