Commit `bd49b87fce`

bd49b87fce895e9f0a7588152548fb6e1821ac7d

parent: 6c6c6248c1

Verified · cmc

cmc <hello@cleberg.net> · 2026-09-28 22:53 UTC

httpd: web archives take a pack slot

Ref #262

Layout: unified · split

.gitbay/wiki/Admin.org +3 −2

		@@ -210,7 +210,8 @@ push=.
210	210	repositories may take; a push may be no larger than what is left.
211	211	- =pack_concurrency= (3), =pack_per_principal= (2), =pack_queue= (32),
212	212	=pack_queue_wait= (="60s"=) — git pack generation (clones, fetches,
213		=git archive --remote=) over SSH, smart HTTP and git:// shares one
	213	=git archive --remote=, web archive downloads) over SSH, smart HTTP
	214	and git:// shares one
214	215	budget: this many at once, this many per account (per client
215	216	address when anonymous: an IPv4 address, or an IPv6 /64), and this
216	217	many waiting for at most the wait. Anonymous clients together hold
		@@ -220,7 +221,7 @@ push=.
220	221	gets 503 with =Retry-After: 30=, git:// an =ERR= line. A queued
221	222	client that disconnects leaves the queue; a running clone whose
222	223	client disconnects is killed. Ref listings (info/refs, protocol v2
223		=ls-refs=), pushes and web archives are outside the budget. For the
	224	=ls-refs=), pushes and =repo download= are outside the budget. For the
224	225	three counts 0 means the default and a negative value turns that
225	226	bound off. The defaults suit a four-core host; see [[Performance]].
226	227	With =ssh.mode = "system"= each SSH session is its own process and

CHANGELOG.org +4 −1

		@@ -224,12 +224,15 @@ missing, =gitbayd admin backup --verify <archive>= names it, and
224	224	"the server is busy…" and exits 1, HTTP gets 503 with
225	225	=Retry-After: 30=, and git:// gets an =ERR= line. Operators: the
226	226	defaults are tuned for a four-core host; set the three counts to -1
227		to turn the limit off. Ref listings, pushes and web archives are
	227	to turn the limit off. Ref listings, pushes and =repo download= are
228	228	unaffected. Under =ssh.mode = "system"= SSH clones are not counted,
229	229	since each session is its own process (#262).
230	230	- Anonymous clones are counted per IPv4 address or IPv6 /64, and
231	231	together hold at most =pack_concurrency= − 1 slots, so a signed-in
232	232	client can always get the last one (#262).
	233	- Web archive downloads (=/{owner}/{repo}/archive/{ref}.tar.gz=) take
	234	a pack slot, answer 503 with =Retry-After: 30= when none is free, and
	235	are killed when the client leaves or stops reading (#262).
233	236
234	237	* v1.36.0 — 2026-09-23
235	238

internal/gitutil/read.go +6 −1

		@@ -132,12 +132,17 @@ var ErrArchiveTooLarge = errors.New("archive exceeds the size limit")
132	132	// Archive streams a tar.gz of ref to w, within archiveTimeout and
133	133	// MaxArchiveBytes. Past either, git is killed and the error says which.
134	134	func Archive(dir, ref, prefix string, w io.Writer) error {
	135	return ArchiveUntil(dir, ref, prefix, w, nil)
	136	}
	137
	138	// ArchiveUntil is Archive, killing git when stop closes.
	139	func ArchiveUntil(dir, ref, prefix string, w io.Writer, stop <-chan struct{}) error {
135	140	ctx, cancel := context.WithTimeout(context.Background(), archiveTimeout)
136	141	defer cancel()
137	142	cmd := exec.CommandContext(ctx, toolpath.Look("git"), "-C", dir, "archive", "--format=tar.gz", "--prefix="+prefix+"/", "--end-of-options", ref)
138	143	lw := &cappedWriter{w: w, left: MaxArchiveBytes, stop: cancel}
139	144	cmd.Stdout = lw
140		err := cmd.Run()
	145	err := RunUntil(cmd, stop)
141	146	switch {
142	147	case lw.exceeded:
143	148	return ErrArchiveTooLarge

internal/httpd/packlimit_test.go +60

		@@ -286,3 +286,63 @@ func TestFetchKilledWhenClientStopsReading(t *testing.T) {
286	286	// The 32MB pack cannot fit in the socket buffers; nothing is read.
287	287	ended(t, s, finished, 20*time.Second)
288	288	}
	289
	290	func getArchive(s Server, w http.ResponseWriter, r http.Request) {
	291	r.SetPathValue("owner", "alice")
	292	r.SetPathValue("repo", "app")
	293	r.SetPathValue("file", "main.tar.gz")
	294	s.archive(w, r)
	295	}
	296
	297	// A web archive takes a pack slot: busy is 503, and the slot is free
	298	// again once the archive is written.
	299	func TestArchiveTakesASlot(t *testing.T) {
	300	s := limitedServer(t)
	301	seed(t, s, 10)
	302	hold, err := s.packs.Acquire(nil, "ip:elsewhere")
	303	if err != nil {
	304	t.Fatal(err)
	305	}
	306	w := httptest.NewRecorder()
	307	getArchive(s, w, httptest.NewRequest("GET", "/alice/app/archive/main.tar.gz", nil))
	308	if w.Code != http.StatusServiceUnavailable \|\| w.Header().Get("Retry-After") == "" {
	309	t.Fatalf("busy: status %d, Retry-After %q", w.Code, w.Header().Get("Retry-After"))
	310	}
	311	hold()
	312	w = httptest.NewRecorder()
	313	getArchive(s, w, httptest.NewRequest("GET", "/alice/app/archive/main.tar.gz", nil))
	314	if w.Code != http.StatusOK \|\| w.Header().Get("Content-Type") != "application/gzip" \|\| w.Body.Len() == 0 {
	315	t.Fatalf("free: status %d, type %q, %d bytes", w.Code, w.Header().Get("Content-Type"), w.Body.Len())
	316	}
	317	hold, err = s.packs.Acquire(nil, "ip:elsewhere")
	318	if err != nil {
	319	t.Fatalf("slot not released after the archive: %v", err)
	320	}
	321	hold()
	322	}
	323
	324	// An archive whose client stops reading is cut after StallDeadline.
	325	func TestArchiveKilledWhenClientStopsReading(t *testing.T) {
	326	stallAfter(t, 500*time.Millisecond)
	327	s := limitedServer(t)
	328	sizes := make([]int, 16)
	329	for i := range sizes {
	330	sizes[i] = 2 << 20
	331	}
	332	seed(t, s, sizes...)
	333	finished := make(chan struct{})
	334	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
	335	getArchive(s, w, r)
	336	close(finished)
	337	}))
	338	defer srv.Close()
	339	conn, err := net.Dial("tcp", srv.Listener.Addr().String())
	340	if err != nil {
	341	t.Fatal(err)
	342	}
	343	defer conn.Close()
	344	conn.(*net.TCPConn).SetReadBuffer(4096)
	345	fmt.Fprintf(conn, "GET /alice/app/archive/main.tar.gz HTTP/1.1\r\nHost: x\r\n\r\n")
	346	// The 32MB archive cannot fit in the socket buffers; nothing is read.
	347	ended(t, s, finished, 20*time.Second)
	348	}

internal/httpd/smart.go +52 −42

		@@ -146,52 +146,14 @@ func (s Server) uploadPack(w http.ResponseWriter, r http.Request) {
146	146	cancel := r.Context().Done()
147	147	out := io.Writer(w)
148	148	if !lsRefs(br) {
149		// A queued clone waits at most the limiter's wait, and Stop ends
150		// the wait so it does not hold up a restart's drain. net/http
151		// notices a departed client only after the body is read, so
152		// that rarely ends it.
153		release, err := s.packs.Acquire(s.until(r), s.packPrincipal(r))
154		if err != nil {
155		msg := "the server is restarting; try again in a minute"
156		if errors.Is(err, packlimit.ErrBusy) {
157		msg = "the server is busy: it is at its limit of concurrent clones and fetches; try again in a minute"
158		}
159		w.Header().Set("Retry-After", "30")
160		http.Error(w, msg, http.StatusServiceUnavailable)
	149	o, kill, finish, ok := s.packSlot(w, r)
	150	if !ok {
161	151	return
162	152	}
163	153	// Deferred before git runs, so it fires after git has exited
164	154	// and been waited for.
165		defer release()
166		var stalled <-chan struct{}
167		var unwatch func()
168		out, stalled, unwatch = s.packs.Watch(w)
169		defer unwatch()
170		kill := make(chan struct{})
171		finished := make(chan struct{})
172		exited := make(chan struct{})
173		// The watcher must not touch w once the handler has returned.
174		defer func() {
175		close(finished)
176		<-exited
177		}()
178		go func() {
179		defer close(exited)
180		select {
181		case <-finished:
182		return
183		case <-r.Context().Done():
184		case <-stalled:
185		// A write blocked on a client that stopped reading, or a
186		// read of a body it stopped sending, outlives git;
187		// expired deadlines end both copies, so Wait returns.
188		rc := http.NewResponseController(w)
189		rc.SetReadDeadline(time.Now())
190		rc.SetWriteDeadline(time.Now())
191		}
192		close(kill)
193		}()
194		cancel = kill
	155	defer finish()
	156	out, cancel = o, kill
195	157	}
196	158	w.Header().Set("Content-Type", "application/x-git-upload-pack-result")
197	159	w.Header().Set("Cache-Control", "no-cache")
		@@ -203,6 +165,54 @@ func (s Server) uploadPack(w http.ResponseWriter, r http.Request) {
203	165	gitutil.RunUntil(cmd, cancel)
204	166	}
205	167
	168	// packSlot takes a pack-generation slot for r, answering 503 with
	169	// Retry-After when none comes free. A queued request waits at most the
	170	// limiter's wait, and Stop ends the wait so it does not hold up a
	171	// restart's drain; net/http notices a departed client only after the
	172	// body is read, so that rarely ends it. On success out is w watched for
	173	// stalls, kill closes when git must stop — the client left, or no write
	174	// to it completed for packlimit.StallDeadline — and finish, called once
	175	// git has exited, releases the slot.
	176	func (s Server) packSlot(w http.ResponseWriter, r http.Request) (out io.Writer, kill <-chan struct{}, finish func(), ok bool) {
	177	release, err := s.packs.Acquire(s.until(r), s.packPrincipal(r))
	178	if err != nil {
	179	msg := "the server is restarting; try again in a minute"
	180	if errors.Is(err, packlimit.ErrBusy) {
	181	msg = "the server is busy: it is at its limit of concurrent clones and fetches; try again in a minute"
	182	}
	183	w.Header().Set("Retry-After", "30")
	184	http.Error(w, msg, http.StatusServiceUnavailable)
	185	return nil, nil, nil, false
	186	}
	187	out, stalled, unwatch := s.packs.Watch(w)
	188	killed := make(chan struct{})
	189	finished := make(chan struct{})
	190	exited := make(chan struct{})
	191	go func() {
	192	defer close(exited)
	193	select {
	194	case <-finished:
	195	return
	196	case <-r.Context().Done():
	197	case <-stalled:
	198	// A write blocked on a client that stopped reading, or a
	199	// read of a body it stopped sending, outlives git;
	200	// expired deadlines end both copies, so Wait returns.
	201	rc := http.NewResponseController(w)
	202	rc.SetReadDeadline(time.Now())
	203	rc.SetWriteDeadline(time.Now())
	204	}
	205	close(killed)
	206	}()
	207	return out, killed, func() {
	208	// The watcher must not touch w once the handler has returned.
	209	close(finished)
	210	<-exited
	211	unwatch()
	212	release()
	213	}, true
	214	}
	215
206	216	// lsRefs reports whether a protocol v2 request is a ref listing, which
207	217	// generates no pack. Its first pkt-line is "command=ls-refs".
208	218	func lsRefs(br *bufio.Reader) bool {

internal/httpd/web.go +6 −1

		@@ -2340,10 +2340,15 @@ func (s Server) archive(w http.ResponseWriter, r http.Request) {
2340	2340	s.notFound(w, r)
2341	2341	return
2342	2342	}
	2343	out, kill, finish, ok := s.packSlot(w, r)
	2344	if !ok {
	2345	return
	2346	}
	2347	defer finish()
2343	2348	prefix := fmt.Sprintf("%s-%s", p.Repo.Name, ref)
2344	2349	w.Header().Set("Content-Type", "application/gzip")
2345	2350	w.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=%q", prefix+".tar.gz"))
2346		gitutil.Archive(p.Dir, ref, prefix, w)
	2351	gitutil.ArchiveUntil(p.Dir, ref, prefix, out, kill)
2347	2352	}
2348	2353
2349	2354	func policyCanAdmin(u store.User, repo store.Repo, grant string) bool {

Commit bd49b87fce

Commit `bd49b87fce`