krz/gitbay

gitbayd: header and idle timeouts, and a drain on SIGTERM !193

merged merged by cmc on 2026-09-03 17:10 UTC · krz/gitbay:serve-shutdown into main

3 files changed, +106 −4

Layout: unified · split

cmd/gitbayd/main.go +47 −3
@@ -10,9 +10,11 @@ import (
1010 "net"
1111 "net/http"
1212 "os"
13 "os/signal"
1314 "path/filepath"
1415 "strconv"
1516 "strings"
17 "syscall"
1618 "time"
1719
1820 "github.com/spf13/cobra"
@@ -146,6 +148,11 @@ func serveCmd() *cobra.Command {
146148 }
147149 defer stopHookd()
148150
151 // SIGTERM is how a deploy restarts the daemon: stop accepting,
152 // let what is in flight finish, exit 0 (#105).
153 ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
154 defer stop()
155
149156 // Outbound webhook deliveries. The retry base is overridable
150157 // for tests via GITBAY_WEBHOOK_RETRY_BASE.
151158 retryBase := 30 * time.Second
@@ -173,6 +180,8 @@ func serveCmd() *cobra.Command {
173180 }, buildinfo.String()).Run(whCtx)
174181
175182 errCh := make(chan error, 3)
183 var sshSrv *sshd.Server
184 var sshLn, gitLn net.Listener
176185 if cfg.SSH.Mode == "embedded" {
177186 srv, err := sshd.New(cfg, st)
178187 if err != nil {
@@ -183,6 +192,7 @@ func serveCmd() *cobra.Command {
183192 return err
184193 }
185194 slog.Info("ssh listening", "addr", ln.Addr())
195 sshSrv, sshLn = srv, ln
186196 go func() { errCh <- srv.Serve(ln) }()
187197 } else {
188198 // system mode: the host sshd owns the SSH port and invokes
@@ -191,7 +201,16 @@ func serveCmd() *cobra.Command {
191201 }
192202
193203 web := httpd.New(cfg, st)
194 hs := &http.Server{Addr: cfg.HTTP.Addr, Handler: web.Handler()}
204 // Header and idle timeouts bound what an idle or slow client can
205 // hold open. No write timeout: archives and upload-pack stream
206 // for as long as they take (#104).
207 hs := &http.Server{
208 Addr: cfg.HTTP.Addr,
209 Handler: web.Handler(),
210 ReadHeaderTimeout: 10 * time.Second,
211 IdleTimeout: 2 * time.Minute,
212 MaxHeaderBytes: 64 << 10,
213 }
195214 go func() {
196215 slog.Info("http listening", "addr", cfg.HTTP.Addr, "tls", cfg.HTTP.TLS)
197216 switch cfg.HTTP.TLS {
@@ -250,7 +269,9 @@ func serveCmd() *cobra.Command {
250269 })
251270 go func() {
252271 slog.Info("acme http listening", "addr", addr)
253 if err := http.ListenAndServe(addr, m.HTTPHandler(redirect)); err != nil {
272 acmeHTTP := &http.Server{Addr: addr, Handler: m.HTTPHandler(redirect),
273 ReadHeaderTimeout: 10 * time.Second, IdleTimeout: time.Minute}
274 if err := acmeHTTP.ListenAndServe(); err != nil {
254275 slog.Warn("acme http listener failed; continuing with TLS-ALPN only", "err", err)
255276 }
256277 }()
@@ -266,10 +287,33 @@ func serveCmd() *cobra.Command {
266287 return err
267288 }
268289 slog.Info("git-daemon listening", "addr", gln.Addr())
290 gitLn = gln
269291 go func() { errCh <- gitd.New(cfg, st).Serve(gln) }()
270292 }
271293
272 return <-errCh
294 select {
295 case err := <-errCh:
296 return err
297 case <-ctx.Done():
298 }
299 slog.Info("shutting down")
300 stop()
301 for _, ln := range []net.Listener{sshLn, gitLn} {
302 if ln != nil {
303 ln.Close()
304 }
305 }
306 drain, cancel := context.WithTimeout(context.Background(), 30*time.Second)
307 defer cancel()
308 if err := hs.Shutdown(drain); err != nil {
309 slog.Warn("http shutdown", "err", err)
310 }
311 if sshSrv != nil {
312 if err := sshSrv.Shutdown(drain); err != nil {
313 slog.Warn("ssh shutdown", "err", err)
314 }
315 }
316 return nil
273317 },
274318 }
275319}
e2e/shutdown_test.go added +34
@@ -0,0 +1,34 @@
1package e2e
2
3import (
4 "net/http"
5 "syscall"
6 "testing"
7 "time"
8)
9
10// SIGTERM is how a deploy restarts the daemon. It used to be a plain
11// kill: no listener closed, no request or push allowed to finish. The
12// daemon now stops its listeners, drains, and exits 0 (#105).
13func TestServeStopsOnSIGTERM(t *testing.T) {
14 inst := startInstance(t)
15 if resp, err := http.Get(inst.base() + "/healthz"); err != nil || resp.StatusCode != 200 {
16 t.Fatalf("healthz before shutdown: %v", err)
17 }
18 if err := inst.proc.Process.Signal(syscall.SIGTERM); err != nil {
19 t.Fatal(err)
20 }
21 done := make(chan error, 1)
22 go func() { done <- inst.proc.Wait() }()
23 select {
24 case err := <-done:
25 if err != nil {
26 t.Fatalf("daemon did not exit cleanly on SIGTERM: %v", err)
27 }
28 case <-time.After(15 * time.Second):
29 t.Fatal("daemon still running 15s after SIGTERM")
30 }
31 if _, err := http.Get(inst.base() + "/healthz"); err == nil {
32 t.Fatal("http listener still answering after shutdown")
33 }
34}
internal/sshd/sshd.go +25 −1
@@ -3,6 +3,7 @@
33package sshd
44
55import (
6 "context"
67 "crypto/ed25519"
78 "crypto/rand"
89 "encoding/base64"
@@ -16,6 +17,7 @@ import (
1617 "path/filepath"
1718 "strconv"
1819 "strings"
20 "sync"
1921 "time"
2022
2123 "golang.org/x/crypto/ssh"
@@ -34,6 +36,7 @@ type Server struct {
3436 st *store.Store
3537 sshCfg *ssh.ServerConfig
3638 authLimiter *rateLimiter
39 sessions sync.WaitGroup // accepted connections still being served
3740}
3841
3942func New(cfg config.Config, st *store.Store) (*Server, error) {
@@ -139,7 +142,28 @@ func (s *Server) Serve(ln net.Listener) error {
139142 if err != nil {
140143 return err
141144 }
142 go s.handleConn(conn)
145 s.sessions.Add(1)
146 go func() {
147 defer s.sessions.Done()
148 s.handleConn(conn)
149 }()
150 }
151}
152
153// Shutdown waits for every accepted connection to finish, or for ctx. The
154// caller closes the listener first; a push in flight completes rather
155// than being cut mid-pack.
156func (s *Server) Shutdown(ctx context.Context) error {
157 done := make(chan struct{})
158 go func() {
159 s.sessions.Wait()
160 close(done)
161 }()
162 select {
163 case <-done:
164 return nil
165 case <-ctx.Done():
166 return ctx.Err()
143167 }
144168}
145169