cmd/gitbayd/main.go
552 lines · 17042 bytes
1// gitbayd is the forge server daemon. The same binary also runs in hook mode
2// (invoked by git via core.hooksPath) and hosts the host-local admin commands.
3package main
4
5import (
6 "context"
7 "fmt"
8 "io"
9 "log/slog"
10 "net"
11 "net/http"
12 "os"
13 "os/signal"
14 "path/filepath"
15 "strconv"
16 "strings"
17 "syscall"
18 "time"
19
20 "github.com/spf13/cobra"
21 "golang.org/x/crypto/acme/autocert"
22
23 "gitbay.org/gitbay/internal/buildinfo"
24 "gitbay.org/gitbay/internal/ci"
25 "gitbay.org/gitbay/internal/config"
26 "gitbay.org/gitbay/internal/control"
27 "gitbay.org/gitbay/internal/deps"
28 "gitbay.org/gitbay/internal/gitd"
29 "gitbay.org/gitbay/internal/hookd"
30 "gitbay.org/gitbay/internal/httpd"
31 "gitbay.org/gitbay/internal/mirror"
32 "gitbay.org/gitbay/internal/notify"
33 "gitbay.org/gitbay/internal/sshd"
34 "gitbay.org/gitbay/internal/store"
35 "gitbay.org/gitbay/internal/webhook"
36)
37
38func openStore(cfg config.Config) (*store.Store, error) {
39 s, err := store.Open(filepath.Join(cfg.Server.Root, "gitbay.db"))
40 if err != nil {
41 return nil, err
42 }
43 // Say so when the schema moves. A restart migrates in silence otherwise,
44 // which makes an unexpected schema version hard to attribute to the deploy
45 // that caused it.
46 before, err := s.Version()
47 if err != nil {
48 s.Close()
49 return nil, err
50 }
51 if err := s.MigrateUp(); err != nil {
52 s.Close()
53 return nil, err
54 }
55 after, err := s.Version()
56 if err != nil {
57 s.Close()
58 return nil, err
59 }
60 if after != before {
61 slog.Info("schema migrated", "from", before, "to", after)
62 }
63 return s, nil
64}
65
66var configPath string
67
68func main() {
69 root := &cobra.Command{
70 Use: "gitbayd",
71 Short: "gitbay server daemon",
72 SilenceUsage: true,
73 SilenceErrors: true,
74 }
75 root.PersistentFlags().StringVar(&configPath, "config", "/etc/gitbay/config.toml", "path to config file")
76
77 root.AddCommand(
78 checkConfigCmd(),
79 serveCmd(),
80 migrateCmd(),
81 adminCmd(),
82 hookCmd(),
83 authorizedKeysCmd(),
84 shellCmd(),
85 versionCmd(),
86 )
87
88 if err := root.Execute(); err != nil {
89 fmt.Fprintln(os.Stderr, "gitbayd:", err)
90 os.Exit(1)
91 }
92}
93
94func checkConfigCmd() *cobra.Command {
95 var noHost bool
96 cmd := &cobra.Command{
97 Use: "check-config",
98 Short: "validate the configuration and exit",
99 RunE: func(cmd *cobra.Command, args []string) error {
100 cfg, err := config.Load(configPath)
101 if err != nil {
102 return err
103 }
104 if !noHost {
105 if err := cfg.CheckHost(); err != nil {
106 return err
107 }
108 }
109 fmt.Println("config ok")
110 return nil
111 },
112 }
113 cmd.Flags().BoolVar(&noHost, "no-host-checks", false, "skip host environment probes (port binding, paths)")
114 return cmd
115}
116
117func serveCmd() *cobra.Command {
118 return &cobra.Command{
119 Use: "serve",
120 Short: "run the ssh, http, and git listeners",
121 RunE: func(cmd *cobra.Command, args []string) error {
122 // First line of every run: the journal then says which commit is
123 // serving, without rebuilding the binary to find out.
124 logBuild()
125 cfg, err := config.Load(configPath)
126 if err != nil {
127 return err
128 }
129 warnIfUnmerged(cfg)
130 st, err := openStore(cfg)
131 if err != nil {
132 return err
133 }
134 defer st.Close()
135
136 // Regenerate hook scripts so a moved binary self-heals, then
137 // start the hook policy socket.
138 self, err := os.Executable()
139 if err != nil {
140 return err
141 }
142 if err := hookd.WriteHookScripts(control.HooksDir(cfg.Server.Root), self); err != nil {
143 return err
144 }
145 stopHookd, err := hookd.Serve(cfg, st)
146 if err != nil {
147 return err
148 }
149 defer stopHookd()
150
151 // SIGTERM is how a deploy restarts the daemon: stop accepting,
152 // let what is in flight finish, exit 0 (#105).
153 ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
154 defer stop()
155
156 // Outbound webhook deliveries. The retry base is overridable
157 // for tests via GITBAY_WEBHOOK_RETRY_BASE.
158 retryBase := 30 * time.Second
159 if v := os.Getenv("GITBAY_WEBHOOK_RETRY_BASE"); v != "" {
160 if d, err := time.ParseDuration(v); err == nil {
161 retryBase = d
162 }
163 }
164 whCtx, whCancel := context.WithCancel(context.Background())
165 defer whCancel()
166 go webhook.New(st, cfg.Webhooks.AllowLocal, retryBase).Run(whCtx)
167 if cfg.Mail.SMTPHost != "" {
168 go notify.New(st, cfg, retryBase).Run(whCtx)
169 }
170 go mirror.New(st, cfg).Run(whCtx)
171 if d := cfg.Registration.PendingExpiryDuration(); d > 0 {
172 go reapPending(whCtx, st, d)
173 }
174 go sweep(whCtx, st, cfg)
175 go (&ci.Scheduler{St: st, SiteURL: cfg.Server.SiteURL,
176 RepoDir: func(owner, name string) string {
177 return control.RepoDir(cfg.Server.Root, owner, name)
178 }}).Run(whCtx)
179 go deps.New(st, cfg, func(owner, name string) string {
180 return control.RepoDir(cfg.Server.Root, owner, name)
181 }, buildinfo.String()).Run(whCtx)
182
183 errCh := make(chan error, 3)
184 var sshSrv *sshd.Server
185 var sshLn, gitLn net.Listener
186 if cfg.SSH.Mode == "embedded" {
187 srv, err := sshd.New(cfg, st)
188 if err != nil {
189 return err
190 }
191 ln, err := net.Listen("tcp", net.JoinHostPort("", strconv.Itoa(cfg.SSH.Port)))
192 if err != nil {
193 return err
194 }
195 slog.Info("ssh listening", "addr", ln.Addr())
196 sshSrv, sshLn = srv, ln
197 go func() { errCh <- srv.Serve(ln) }()
198 } else {
199 // system mode: the host sshd owns the SSH port and invokes
200 // this binary via AuthorizedKeysCommand + forced command.
201 slog.Info("ssh handled by host sshd (ssh.mode = system)")
202 }
203
204 web := httpd.New(cfg, st)
205 // Header and idle timeouts bound what an idle or slow client can
206 // hold open. No write timeout: archives and upload-pack stream
207 // for as long as they take (#104).
208 hs := &http.Server{
209 Addr: cfg.HTTP.Addr,
210 Handler: web.Handler(),
211 ReadHeaderTimeout: 10 * time.Second,
212 IdleTimeout: 2 * time.Minute,
213 MaxHeaderBytes: 64 << 10,
214 }
215 go func() {
216 slog.Info("http listening", "addr", cfg.HTTP.Addr, "tls", cfg.HTTP.TLS)
217 switch cfg.HTTP.TLS {
218 case "off":
219 errCh <- hs.ListenAndServe()
220 case "files":
221 errCh <- hs.ListenAndServeTLS(cfg.HTTP.CertFile, cfg.HTTP.KeyFile)
222 case "acme":
223 host := cfg.SiteHost()
224 stripPort := func(hp string) string {
225 if h, _, err := net.SplitHostPort(hp); err == nil {
226 return h
227 }
228 return hp
229 }
230 // Beyond the site host, allow <owner>.<pages domain>
231 // for owners that exist — certs come on demand per
232 // subdomain, no wildcard needed.
233 hostPolicy := func(ctx context.Context, h string) error {
234 if h == host {
235 return nil
236 }
237 if pd := cfg.Pages.Domain; pd != "" {
238 if h == pd {
239 return nil // apex: serves a redirect to the forge
240 }
241 if owner, ok := strings.CutSuffix(h, "."+pd); ok &&
242 !strings.Contains(owner, ".") && st.OwnerExists(owner) {
243 return nil
244 }
245 }
246 // Custom pages domains: certs only for claimed hosts.
247 if _, err := st.PageDomainRepo(h); err == nil {
248 return nil
249 }
250 return fmt.Errorf("host %q not served here", h)
251 }
252 m := &autocert.Manager{
253 Prompt: autocert.AcceptTOS,
254 Cache: autocert.DirCache(filepath.Join(cfg.Server.Root, "acme")),
255 HostPolicy: hostPolicy,
256 Email: cfg.HTTP.ACMEEmail,
257 }
258 // TLS-ALPN-01 rides the HTTPS port itself. The optional
259 // plain-HTTP listener adds HTTP-01 and a redirect; losing
260 // it (port 80 taken, no privileges) is not fatal.
261 if addr := cfg.HTTP.ACMEHTTPAddr; addr != "" && addr != "off" {
262 redirect := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
263 // Pages hosts redirect to themselves, not the
264 // forge host.
265 target := host
266 if hostPolicy(r.Context(), stripPort(r.Host)) == nil {
267 target = stripPort(r.Host)
268 }
269 http.Redirect(w, r, "https://"+target+r.URL.RequestURI(), http.StatusMovedPermanently)
270 })
271 go func() {
272 slog.Info("acme http listening", "addr", addr)
273 acmeHTTP := &http.Server{Addr: addr, Handler: m.HTTPHandler(redirect),
274 ReadHeaderTimeout: 10 * time.Second, IdleTimeout: time.Minute}
275 if err := acmeHTTP.ListenAndServe(); err != nil {
276 slog.Warn("acme http listener failed; continuing with TLS-ALPN only", "err", err)
277 }
278 }()
279 }
280 hs.TLSConfig = m.TLSConfig()
281 errCh <- hs.ListenAndServeTLS("", "")
282 }
283 }()
284
285 if cfg.GitDaemon.Enabled {
286 gln, err := net.Listen("tcp", net.JoinHostPort("", strconv.Itoa(cfg.GitDaemon.Port)))
287 if err != nil {
288 return err
289 }
290 slog.Info("git-daemon listening", "addr", gln.Addr())
291 gitLn = gln
292 go func() { errCh <- gitd.New(cfg, st).Serve(gln) }()
293 }
294
295 select {
296 case err := <-errCh:
297 return err
298 case <-ctx.Done():
299 }
300 slog.Info("shutting down")
301 stop()
302 for _, ln := range []net.Listener{sshLn, gitLn} {
303 if ln != nil {
304 ln.Close()
305 }
306 }
307 drain, cancel := context.WithTimeout(context.Background(), 30*time.Second)
308 defer cancel()
309 if err := hs.Shutdown(drain); err != nil {
310 slog.Warn("http shutdown", "err", err)
311 }
312 if sshSrv != nil {
313 if err := sshSrv.Shutdown(drain); err != nil {
314 slog.Warn("ssh shutdown", "err", err)
315 }
316 }
317 return nil
318 },
319 }
320}
321
322func migrateCmd() *cobra.Command {
323 var to int
324 cmd := &cobra.Command{
325 Use: "migrate",
326 Short: "apply schema migrations",
327 RunE: func(cmd *cobra.Command, args []string) error {
328 cfg, err := config.Load(configPath)
329 if err != nil {
330 return err
331 }
332 s, err := store.Open(cfg.Server.Root + "/gitbay.db")
333 if err != nil {
334 return err
335 }
336 defer s.Close()
337 if err := s.MigrateTo(to); err != nil {
338 return err
339 }
340 v, err := s.Version()
341 if err != nil {
342 return err
343 }
344 fmt.Println("schema version", v)
345 return nil
346 },
347 }
348 cmd.Flags().IntVar(&to, "to", -1, "target schema version (-1 = latest)")
349 return cmd
350}
351
352func adminCmd() *cobra.Command {
353 admin := &cobra.Command{
354 Use: "admin",
355 Short: "host-local administration",
356 }
357 userCmd := &cobra.Command{Use: "user", Short: "manage users"}
358 userCmd.AddCommand(
359 hostUserCreateCmd(),
360 hostCmd("list [--state active|pending|disabled|admin] [--limit n] [--cursor c]", "list accounts", "admin", "user", "list"),
361 hostCmd("show <username>", "show an account: keys, emails, orgs, tokens, sessions", "admin", "user", "show"),
362 hostCmd("disable <username>", "suspend an account: keys and sessions refused until re-enabled", "admin", "user", "disable"),
363 hostCmd("enable <username>", "restore a suspended account", "admin", "user", "enable"),
364 hostCmd("delete <username> --yes", "delete an account that anchors nothing (keys, emails, and sessions go with it)", "admin", "user", "delete"),
365 hostCmd("promote <username>", "make an account an instance admin", "admin", "user", "promote"),
366 hostCmd("demote <username>", "remove instance admin from an account (never the last one)", "admin", "user", "demote"),
367 hostCmd("limits <username> [--repos n|default] [--bytes n|default]", "show or set repository and storage caps", "admin", "user", "limits"),
368 )
369 emailCmd := &cobra.Command{Use: "email", Short: "manage user emails"}
370 emailCmd.AddCommand(hostCmd("verify <username> <address>", "mark an email verified by admin assertion", "admin", "email", "verify"))
371 repoCmd := &cobra.Command{Use: "repo", Short: "any repository, for moderation (audited)"}
372 repoCmd.AddCommand(
373 hostCmd("list [--owner o] [--visibility public|private] [--limit n] [--cursor c]", "every repository with size and last push", "admin", "repo", "list"),
374 hostCmd("archive <owner/name>", "archive a repository", "admin", "repo", "archive"),
375 hostCmd("unarchive <owner/name>", "unarchive a repository", "admin", "repo", "unarchive"),
376 hostCmd("visibility <owner/name> public|private", "set a repository's visibility", "admin", "repo", "visibility"),
377 hostCmd("delete <owner/name> --yes", "delete a repository", "admin", "repo", "delete"),
378 )
379 configCmd := &cobra.Command{Use: "config", Short: "the configuration in effect"}
380 configCmd.AddCommand(configShowCmd())
381 admin.AddCommand(
382 userCmd,
383 emailCmd,
384 repoCmd,
385 configCmd,
386 hostCmd("invite --email <address>", "issue a registration invite and email its code", "admin", "invite"),
387 hostCmd("stats [--json]", "instance statistics: counts and per-repository disk usage", "admin", "stats"),
388 hostCmd("runners [--json]", "runner accounts: last poll, scope, the build each holds", "admin", "runners"),
389 hostCmd("audit [--limit n] [--json]", "print the security audit log, newest first", "audit"),
390 backupCmd(),
391 gcCmd(),
392 adminMigrateCommitRefsCmd(),
393 adminBackfillActivityCmd(),
394 )
395 return admin
396}
397
398// hostCmd runs a registry command as the host itself: an admin context
399// with no account behind it, so audit rows carry no actor and the source
400// "host". Arguments pass through untouched; the registry owns the flags,
401// which is what keeps this surface and an admin's SSH session from
402// drifting.
403func hostCmd(use, short string, path ...string) *cobra.Command {
404 return &cobra.Command{
405 Use: use,
406 Short: short,
407 DisableFlagParsing: true,
408 RunE: func(cmd *cobra.Command, args []string) error {
409 for _, a := range args {
410 if a == "--help" || a == "-h" {
411 return cmd.Help()
412 }
413 }
414 return runAsHost(path, args, os.Stdin)
415 },
416 }
417}
418
419// hostArgs pulls the root's --config out of args: with flag parsing off,
420// cobra hands the persistent flag through untouched.
421func hostArgs(args []string) []string {
422 var rest []string
423 for i := 0; i < len(args); i++ {
424 switch {
425 case args[i] == "--config" && i+1 < len(args):
426 configPath = args[i+1]
427 i++
428 case strings.HasPrefix(args[i], "--config="):
429 configPath = strings.TrimPrefix(args[i], "--config=")
430 default:
431 rest = append(rest, args[i])
432 }
433 }
434 return rest
435}
436
437func runAsHost(path, args []string, stdin io.Reader) error {
438 args = hostArgs(args)
439 cfg, err := config.Load(configPath)
440 if err != nil {
441 return err
442 }
443 st, err := openStore(cfg)
444 if err != nil {
445 return err
446 }
447 c := &control.Ctx{
448 User: store.User{Username: "host", IsAdmin: true},
449 Scope: "full",
450 Store: st,
451 Cfg: cfg,
452 Stdin: stdin,
453 Stdout: os.Stdout,
454 Stderr: os.Stderr,
455 Source: "host",
456 }
457 code := control.Dispatch(c, append(append([]string{}, path...), args...))
458 st.Close()
459 if code != 0 {
460 os.Exit(code)
461 }
462 return nil
463}
464
465// hostUserCreateCmd keeps --key <path>, which the registry command cannot
466// take (no file paths over SSH): the file becomes the command's stdin.
467func hostUserCreateCmd() *cobra.Command {
468 return &cobra.Command{
469 Use: "create <username> [--admin] [--email <address> [--verified]] [--key <file>]",
470 Short: "create a user (host-local bootstrap; the only path in closed mode)",
471 DisableFlagParsing: true,
472 RunE: func(cmd *cobra.Command, args []string) error {
473 var stdin io.Reader = os.Stdin
474 var rest []string
475 args = hostArgs(args)
476 for i := 0; i < len(args); i++ {
477 switch {
478 case args[i] == "--help" || args[i] == "-h":
479 return cmd.Help()
480 case args[i] == "--key" && i+1 < len(args) && args[i+1] != "-":
481 f, err := os.Open(args[i+1])
482 if err != nil {
483 return err
484 }
485 defer f.Close()
486 stdin = f
487 rest = append(rest, "--key", "-")
488 i++
489 default:
490 rest = append(rest, args[i])
491 }
492 }
493 return runAsHost([]string{"admin", "user", "create"}, rest, stdin)
494 },
495 }
496}
497
498// sweep prunes expired sessions and tokens, and rows past their
499// configured retention, hourly and once at start. GITBAY_SWEEP_TICK
500// shortens the interval for tests.
501func sweep(ctx context.Context, st *store.Store, cfg config.Config) {
502 tick := time.Hour
503 if v := os.Getenv("GITBAY_SWEEP_TICK"); v != "" {
504 if d, err := time.ParseDuration(v); err == nil {
505 tick = d
506 }
507 }
508 audit, events, deliveries, mail := cfg.Retention.Durations()
509 r := store.Retention{Audit: audit, Events: events,
510 WebhookDeliveries: deliveries, Mail: mail}
511 t := time.NewTicker(tick)
512 defer t.Stop()
513 for {
514 swept, err := st.Sweep(r, time.Now())
515 if err != nil {
516 slog.Error("sweeping", "err", err, "removed", swept.Total())
517 } else if n := swept.Total(); n > 0 {
518 slog.Info("swept expired rows", "removed", n, "tables", swept)
519 }
520 select {
521 case <-ctx.Done():
522 return
523 case <-t.C:
524 }
525 }
526}
527
528// reapPending removes self-registered accounts still unverified after
529// maxAge, hourly and once at start. GITBAY_REAP_TICK shortens the
530// interval for tests.
531func reapPending(ctx context.Context, st *store.Store, maxAge time.Duration) {
532 tick := time.Hour
533 if v := os.Getenv("GITBAY_REAP_TICK"); v != "" {
534 if d, err := time.ParseDuration(v); err == nil {
535 tick = d
536 }
537 }
538 t := time.NewTicker(tick)
539 defer t.Stop()
540 for {
541 if removed, err := st.ReapPendingUsers(maxAge); err != nil {
542 slog.Error("reaping pending accounts", "err", err)
543 } else if len(removed) > 0 {
544 slog.Info("removed unverified accounts", "users", removed)
545 }
546 select {
547 case <-ctx.Done():
548 return
549 case <-t.C:
550 }
551 }
552}