cmd/gitbayd/main.go
561 lines · 17507 bytes
1// gitbayd is the forge server daemon. The same binary also runs in hook mode
2// (invoked by git via core.hooksPath) and hosts the host-local admin commands.
3package main
4
5import (
6 "context"
7 "fmt"
8 "io"
9 "log/slog"
10 "net"
11 "net/http"
12 "os"
13 "os/signal"
14 "path/filepath"
15 "strconv"
16 "strings"
17 "syscall"
18 "time"
19
20 "github.com/spf13/cobra"
21 "golang.org/x/crypto/acme/autocert"
22
23 "gitbay.org/gitbay/internal/buildinfo"
24 "gitbay.org/gitbay/internal/ci"
25 "gitbay.org/gitbay/internal/config"
26 "gitbay.org/gitbay/internal/control"
27 "gitbay.org/gitbay/internal/deps"
28 "gitbay.org/gitbay/internal/gitd"
29 "gitbay.org/gitbay/internal/hookd"
30 "gitbay.org/gitbay/internal/httpd"
31 "gitbay.org/gitbay/internal/mirror"
32 "gitbay.org/gitbay/internal/notify"
33 "gitbay.org/gitbay/internal/sshd"
34 "gitbay.org/gitbay/internal/store"
35 "gitbay.org/gitbay/internal/toolpath"
36 "gitbay.org/gitbay/internal/webhook"
37)
38
39func openStore(cfg config.Config) (*store.Store, error) {
40 s, err := store.Open(filepath.Join(cfg.Server.Root, "gitbay.db"))
41 if err != nil {
42 return nil, err
43 }
44 // Say so when the schema moves. A restart migrates in silence otherwise,
45 // which makes an unexpected schema version hard to attribute to the deploy
46 // that caused it.
47 before, err := s.Version()
48 if err != nil {
49 s.Close()
50 return nil, err
51 }
52 if err := s.MigrateUp(); err != nil {
53 s.Close()
54 return nil, err
55 }
56 after, err := s.Version()
57 if err != nil {
58 s.Close()
59 return nil, err
60 }
61 if after != before {
62 slog.Info("schema migrated", "from", before, "to", after)
63 }
64 return s, nil
65}
66
67var configPath string
68
69func main() {
70 root := &cobra.Command{
71 Use: "gitbayd",
72 Short: "gitbay server daemon",
73 SilenceUsage: true,
74 SilenceErrors: true,
75 }
76 root.PersistentFlags().StringVar(&configPath, "config", "/etc/gitbay/config.toml", "path to config file")
77
78 root.AddCommand(
79 checkConfigCmd(),
80 serveCmd(),
81 migrateCmd(),
82 adminCmd(),
83 hookCmd(),
84 authorizedKeysCmd(),
85 shellCmd(),
86 versionCmd(),
87 )
88
89 if err := root.Execute(); err != nil {
90 fmt.Fprintln(os.Stderr, "gitbayd:", err)
91 os.Exit(1)
92 }
93}
94
95func checkConfigCmd() *cobra.Command {
96 var noHost bool
97 cmd := &cobra.Command{
98 Use: "check-config",
99 Short: "validate the configuration and exit",
100 RunE: func(cmd *cobra.Command, args []string) error {
101 cfg, err := config.Load(configPath)
102 if err != nil {
103 return err
104 }
105 if !noHost {
106 if err := cfg.CheckHost(); err != nil {
107 return err
108 }
109 }
110 fmt.Println("config ok")
111 return nil
112 },
113 }
114 cmd.Flags().BoolVar(&noHost, "no-host-checks", false, "skip host environment probes (port binding, paths)")
115 return cmd
116}
117
118func serveCmd() *cobra.Command {
119 return &cobra.Command{
120 Use: "serve",
121 Short: "run the ssh, http, and git listeners",
122 RunE: func(cmd *cobra.Command, args []string) error {
123 // First line of every run: the journal then says which commit is
124 // serving, without rebuilding the binary to find out.
125 logBuild()
126 // The tools this daemon shells out to are resolved once, at
127 // package init. Say so now rather than failing on whichever
128 // request first needed git.
129 if err := toolpath.Verify(); err != nil {
130 return fmt.Errorf("required tools missing: %w", err)
131 }
132 cfg, err := config.Load(configPath)
133 if err != nil {
134 return err
135 }
136 warnIfUnmerged(cfg)
137 st, err := openStore(cfg)
138 if err != nil {
139 return err
140 }
141 defer st.Close()
142
143 // Regenerate hook scripts so a moved binary self-heals, then
144 // start the hook policy socket.
145 self, err := os.Executable()
146 if err != nil {
147 return err
148 }
149 if err := hookd.WriteHookScripts(control.HooksDir(cfg.Server.Root), self); err != nil {
150 return err
151 }
152 stopHookd, err := hookd.Serve(cfg, st)
153 if err != nil {
154 return err
155 }
156 defer stopHookd()
157
158 // SIGTERM is how a deploy restarts the daemon: stop accepting,
159 // let what is in flight finish, exit 0 (#105).
160 ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
161 defer stop()
162
163 // Outbound webhook deliveries, and the mail queue when SMTP is
164 // configured, share one retry base. It is overridable for
165 // tests via GITBAY_WEBHOOK_RETRY_BASE; notify.DefaultRetryBase
166 // names the production default so nothing else has to guess it.
167 retryBase := notify.DefaultRetryBase
168 if v := os.Getenv("GITBAY_WEBHOOK_RETRY_BASE"); v != "" {
169 if d, err := time.ParseDuration(v); err == nil {
170 retryBase = d
171 }
172 }
173 whCtx, whCancel := context.WithCancel(context.Background())
174 defer whCancel()
175 go webhook.New(st, cfg.Webhooks.AllowLocal, retryBase).Run(whCtx)
176 if cfg.Mail.SMTPHost != "" {
177 go notify.New(st, cfg, retryBase).Run(whCtx)
178 }
179 go mirror.New(st, cfg).Run(whCtx)
180 if d := cfg.Registration.PendingExpiryDuration(); d > 0 {
181 go reapPending(whCtx, st, d)
182 }
183 go sweep(whCtx, st, cfg)
184 go (&ci.Scheduler{St: st, SiteURL: cfg.Server.SiteURL,
185 RepoDir: func(owner, name string) string {
186 return control.RepoDir(cfg.Server.Root, owner, name)
187 }}).Run(whCtx)
188 go deps.New(st, cfg, func(owner, name string) string {
189 return control.RepoDir(cfg.Server.Root, owner, name)
190 }, buildinfo.String()).Run(whCtx)
191
192 errCh := make(chan error, 3)
193 var sshSrv *sshd.Server
194 var sshLn, gitLn net.Listener
195 if cfg.SSH.Mode == "embedded" {
196 srv, err := sshd.New(cfg, st)
197 if err != nil {
198 return err
199 }
200 ln, err := net.Listen("tcp", net.JoinHostPort("", strconv.Itoa(cfg.SSH.Port)))
201 if err != nil {
202 return err
203 }
204 slog.Info("ssh listening", "addr", ln.Addr())
205 sshSrv, sshLn = srv, ln
206 go func() { errCh <- srv.Serve(ln) }()
207 } else {
208 // system mode: the host sshd owns the SSH port and invokes
209 // this binary via AuthorizedKeysCommand + forced command.
210 slog.Info("ssh handled by host sshd (ssh.mode = system)")
211 }
212
213 web := httpd.New(cfg, st)
214 // Header and idle timeouts bound what an idle or slow client can
215 // hold open. No write timeout: archives and upload-pack stream
216 // for as long as they take (#104).
217 hs := &http.Server{
218 Addr: cfg.HTTP.Addr,
219 Handler: web.Handler(),
220 ReadHeaderTimeout: 10 * time.Second,
221 IdleTimeout: 2 * time.Minute,
222 MaxHeaderBytes: 64 << 10,
223 }
224 go func() {
225 slog.Info("http listening", "addr", cfg.HTTP.Addr, "tls", cfg.HTTP.TLS)
226 switch cfg.HTTP.TLS {
227 case "off":
228 errCh <- hs.ListenAndServe()
229 case "files":
230 errCh <- hs.ListenAndServeTLS(cfg.HTTP.CertFile, cfg.HTTP.KeyFile)
231 case "acme":
232 host := cfg.SiteHost()
233 stripPort := func(hp string) string {
234 if h, _, err := net.SplitHostPort(hp); err == nil {
235 return h
236 }
237 return hp
238 }
239 // Beyond the site host, allow <owner>.<pages domain>
240 // for owners that exist — certs come on demand per
241 // subdomain, no wildcard needed.
242 hostPolicy := func(ctx context.Context, h string) error {
243 if h == host {
244 return nil
245 }
246 if pd := cfg.Pages.Domain; pd != "" {
247 if h == pd {
248 return nil // apex: serves a redirect to the forge
249 }
250 if owner, ok := strings.CutSuffix(h, "."+pd); ok &&
251 !strings.Contains(owner, ".") && st.OwnerExists(owner) {
252 return nil
253 }
254 }
255 // Custom pages domains: certs only for claimed hosts.
256 if _, err := st.PageDomainRepo(h); err == nil {
257 return nil
258 }
259 return fmt.Errorf("host %q not served here", h)
260 }
261 m := &autocert.Manager{
262 Prompt: autocert.AcceptTOS,
263 Cache: autocert.DirCache(filepath.Join(cfg.Server.Root, "acme")),
264 HostPolicy: hostPolicy,
265 Email: cfg.HTTP.ACMEEmail,
266 }
267 // TLS-ALPN-01 rides the HTTPS port itself. The optional
268 // plain-HTTP listener adds HTTP-01 and a redirect; losing
269 // it (port 80 taken, no privileges) is not fatal.
270 if addr := cfg.HTTP.ACMEHTTPAddr; addr != "" && addr != "off" {
271 redirect := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
272 // Pages hosts redirect to themselves, not the
273 // forge host.
274 target := host
275 if hostPolicy(r.Context(), stripPort(r.Host)) == nil {
276 target = stripPort(r.Host)
277 }
278 http.Redirect(w, r, "https://"+target+r.URL.RequestURI(), http.StatusMovedPermanently)
279 })
280 go func() {
281 slog.Info("acme http listening", "addr", addr)
282 acmeHTTP := &http.Server{Addr: addr, Handler: m.HTTPHandler(redirect),
283 ReadHeaderTimeout: 10 * time.Second, IdleTimeout: time.Minute}
284 if err := acmeHTTP.ListenAndServe(); err != nil {
285 slog.Warn("acme http listener failed; continuing with TLS-ALPN only", "err", err)
286 }
287 }()
288 }
289 hs.TLSConfig = m.TLSConfig()
290 errCh <- hs.ListenAndServeTLS("", "")
291 }
292 }()
293
294 if cfg.GitDaemon.Enabled {
295 gln, err := net.Listen("tcp", net.JoinHostPort("", strconv.Itoa(cfg.GitDaemon.Port)))
296 if err != nil {
297 return err
298 }
299 slog.Info("git-daemon listening", "addr", gln.Addr())
300 gitLn = gln
301 go func() { errCh <- gitd.New(cfg, st).Serve(gln) }()
302 }
303
304 select {
305 case err := <-errCh:
306 return err
307 case <-ctx.Done():
308 }
309 slog.Info("shutting down")
310 stop()
311 for _, ln := range []net.Listener{sshLn, gitLn} {
312 if ln != nil {
313 ln.Close()
314 }
315 }
316 drain, cancel := context.WithTimeout(context.Background(), 30*time.Second)
317 defer cancel()
318 if err := hs.Shutdown(drain); err != nil {
319 slog.Warn("http shutdown", "err", err)
320 }
321 if sshSrv != nil {
322 if err := sshSrv.Shutdown(drain); err != nil {
323 slog.Warn("ssh shutdown", "err", err)
324 }
325 }
326 return nil
327 },
328 }
329}
330
331func migrateCmd() *cobra.Command {
332 var to int
333 cmd := &cobra.Command{
334 Use: "migrate",
335 Short: "apply schema migrations",
336 RunE: func(cmd *cobra.Command, args []string) error {
337 cfg, err := config.Load(configPath)
338 if err != nil {
339 return err
340 }
341 s, err := store.Open(cfg.Server.Root + "/gitbay.db")
342 if err != nil {
343 return err
344 }
345 defer s.Close()
346 if err := s.MigrateTo(to); err != nil {
347 return err
348 }
349 v, err := s.Version()
350 if err != nil {
351 return err
352 }
353 fmt.Println("schema version", v)
354 return nil
355 },
356 }
357 cmd.Flags().IntVar(&to, "to", -1, "target schema version (-1 = latest)")
358 return cmd
359}
360
361func adminCmd() *cobra.Command {
362 admin := &cobra.Command{
363 Use: "admin",
364 Short: "host-local administration",
365 }
366 userCmd := &cobra.Command{Use: "user", Short: "manage users"}
367 userCmd.AddCommand(
368 hostUserCreateCmd(),
369 hostCmd("list [--state active|pending|disabled|admin] [--limit n] [--cursor c]", "list accounts", "admin", "user", "list"),
370 hostCmd("show <username>", "show an account: keys, emails, orgs, tokens, sessions", "admin", "user", "show"),
371 hostCmd("disable <username>", "suspend an account: keys and sessions refused until re-enabled", "admin", "user", "disable"),
372 hostCmd("enable <username>", "restore a suspended account", "admin", "user", "enable"),
373 hostCmd("delete <username> --yes", "delete an account that anchors nothing (keys, emails, and sessions go with it)", "admin", "user", "delete"),
374 hostCmd("promote <username>", "make an account an instance admin", "admin", "user", "promote"),
375 hostCmd("demote <username>", "remove instance admin from an account (never the last one)", "admin", "user", "demote"),
376 hostCmd("limits <username> [--repos n|default] [--bytes n|default]", "show or set repository and storage caps", "admin", "user", "limits"),
377 )
378 emailCmd := &cobra.Command{Use: "email", Short: "manage user emails"}
379 emailCmd.AddCommand(hostCmd("verify <username> <address>", "mark an email verified by admin assertion", "admin", "email", "verify"))
380 repoCmd := &cobra.Command{Use: "repo", Short: "any repository, for moderation (audited)"}
381 repoCmd.AddCommand(
382 hostCmd("list [--owner o] [--visibility public|private] [--limit n] [--cursor c]", "every repository with size and last push", "admin", "repo", "list"),
383 hostCmd("archive <owner/name>", "archive a repository", "admin", "repo", "archive"),
384 hostCmd("unarchive <owner/name>", "unarchive a repository", "admin", "repo", "unarchive"),
385 hostCmd("visibility <owner/name> public|private", "set a repository's visibility", "admin", "repo", "visibility"),
386 hostCmd("delete <owner/name> --yes", "delete a repository", "admin", "repo", "delete"),
387 )
388 configCmd := &cobra.Command{Use: "config", Short: "the configuration in effect"}
389 configCmd.AddCommand(configShowCmd())
390 admin.AddCommand(
391 userCmd,
392 emailCmd,
393 repoCmd,
394 configCmd,
395 hostCmd("invite --email <address>", "issue a registration invite and email its code", "admin", "invite"),
396 hostCmd("stats [--json]", "instance statistics: counts and per-repository disk usage", "admin", "stats"),
397 hostCmd("runners [--json]", "runner accounts: last poll, scope, the build each holds", "admin", "runners"),
398 hostCmd("audit [--limit n] [--json]", "print the security audit log, newest first", "audit"),
399 backupCmd(),
400 gcCmd(),
401 adminMigrateCommitRefsCmd(),
402 adminBackfillActivityCmd(),
403 )
404 return admin
405}
406
407// hostCmd runs a registry command as the host itself: an admin context
408// with no account behind it, so audit rows carry no actor and the source
409// "host". Arguments pass through untouched; the registry owns the flags,
410// which is what keeps this surface and an admin's SSH session from
411// drifting.
412func hostCmd(use, short string, path ...string) *cobra.Command {
413 return &cobra.Command{
414 Use: use,
415 Short: short,
416 DisableFlagParsing: true,
417 RunE: func(cmd *cobra.Command, args []string) error {
418 for _, a := range args {
419 if a == "--help" || a == "-h" {
420 return cmd.Help()
421 }
422 }
423 return runAsHost(path, args, os.Stdin)
424 },
425 }
426}
427
428// hostArgs pulls the root's --config out of args: with flag parsing off,
429// cobra hands the persistent flag through untouched.
430func hostArgs(args []string) []string {
431 var rest []string
432 for i := 0; i < len(args); i++ {
433 switch {
434 case args[i] == "--config" && i+1 < len(args):
435 configPath = args[i+1]
436 i++
437 case strings.HasPrefix(args[i], "--config="):
438 configPath = strings.TrimPrefix(args[i], "--config=")
439 default:
440 rest = append(rest, args[i])
441 }
442 }
443 return rest
444}
445
446func runAsHost(path, args []string, stdin io.Reader) error {
447 args = hostArgs(args)
448 cfg, err := config.Load(configPath)
449 if err != nil {
450 return err
451 }
452 st, err := openStore(cfg)
453 if err != nil {
454 return err
455 }
456 c := &control.Ctx{
457 User: store.User{Username: "host", IsAdmin: true},
458 Scope: "full",
459 Store: st,
460 Cfg: cfg,
461 Stdin: stdin,
462 Stdout: os.Stdout,
463 Stderr: os.Stderr,
464 Source: "host",
465 }
466 code := control.Dispatch(c, append(append([]string{}, path...), args...))
467 st.Close()
468 if code != 0 {
469 os.Exit(code)
470 }
471 return nil
472}
473
474// hostUserCreateCmd keeps --key <path>, which the registry command cannot
475// take (no file paths over SSH): the file becomes the command's stdin.
476func hostUserCreateCmd() *cobra.Command {
477 return &cobra.Command{
478 Use: "create <username> [--admin] [--email <address> [--verified]] [--key <file>]",
479 Short: "create a user (host-local bootstrap; the only path in closed mode)",
480 DisableFlagParsing: true,
481 RunE: func(cmd *cobra.Command, args []string) error {
482 var stdin io.Reader = os.Stdin
483 var rest []string
484 args = hostArgs(args)
485 for i := 0; i < len(args); i++ {
486 switch {
487 case args[i] == "--help" || args[i] == "-h":
488 return cmd.Help()
489 case args[i] == "--key" && i+1 < len(args) && args[i+1] != "-":
490 f, err := os.Open(args[i+1])
491 if err != nil {
492 return err
493 }
494 defer f.Close()
495 stdin = f
496 rest = append(rest, "--key", "-")
497 i++
498 default:
499 rest = append(rest, args[i])
500 }
501 }
502 return runAsHost([]string{"admin", "user", "create"}, rest, stdin)
503 },
504 }
505}
506
507// sweep prunes expired sessions and tokens, and rows past their
508// configured retention, hourly and once at start. GITBAY_SWEEP_TICK
509// shortens the interval for tests.
510func sweep(ctx context.Context, st *store.Store, cfg config.Config) {
511 tick := time.Hour
512 if v := os.Getenv("GITBAY_SWEEP_TICK"); v != "" {
513 if d, err := time.ParseDuration(v); err == nil {
514 tick = d
515 }
516 }
517 audit, events, deliveries, mail := cfg.Retention.Durations()
518 r := store.Retention{Audit: audit, Events: events,
519 WebhookDeliveries: deliveries, Mail: mail}
520 t := time.NewTicker(tick)
521 defer t.Stop()
522 for {
523 swept, err := st.Sweep(r, time.Now())
524 if err != nil {
525 slog.Error("sweeping", "err", err, "removed", swept.Total())
526 } else if n := swept.Total(); n > 0 {
527 slog.Info("swept expired rows", "removed", n, "tables", swept)
528 }
529 select {
530 case <-ctx.Done():
531 return
532 case <-t.C:
533 }
534 }
535}
536
537// reapPending removes self-registered accounts still unverified after
538// maxAge, hourly and once at start. GITBAY_REAP_TICK shortens the
539// interval for tests.
540func reapPending(ctx context.Context, st *store.Store, maxAge time.Duration) {
541 tick := time.Hour
542 if v := os.Getenv("GITBAY_REAP_TICK"); v != "" {
543 if d, err := time.ParseDuration(v); err == nil {
544 tick = d
545 }
546 }
547 t := time.NewTicker(tick)
548 defer t.Stop()
549 for {
550 if removed, err := st.ReapPendingUsers(maxAge); err != nil {
551 slog.Error("reaping pending accounts", "err", err)
552 } else if len(removed) > 0 {
553 slog.Info("removed unverified accounts", "users", removed)
554 }
555 select {
556 case <-ctx.Done():
557 return
558 case <-t.C:
559 }
560 }
561}