Commit `bd49b87fce`

bd49b87fce895e9f0a7588152548fb6e1821ac7d

parent: 6c6c6248c1

Verified · cmc

cmc <hello@cleberg.net> · 2026-09-28 22:53 UTC

httpd: web archives take a pack slot

Ref #262

Layout: unified · split

.gitbay/wiki/Admin.org +3 −2

   repositories may take; a push may be no larger than what is left.
 - =pack_concurrency= (3), =pack_per_principal= (2), =pack_queue= (32),
   =pack_queue_wait= (="60s"=) — git pack generation (clones, fetches,
-  =git archive --remote=) over SSH, smart HTTP and git:// shares one
+  =git archive --remote=, web archive downloads) over SSH, smart HTTP
+  and git:// shares one
   budget: this many at once, this many per account (per client
   address when anonymous: an IPv4 address, or an IPv6 /64), and this
   many waiting for at most the wait. Anonymous clients together hold
   gets 503 with =Retry-After: 30=, git:// an =ERR= line. A queued
   client that disconnects leaves the queue; a running clone whose
   client disconnects is killed. Ref listings (info/refs, protocol v2
-  =ls-refs=), pushes and web archives are outside the budget. For the
+  =ls-refs=), pushes and =repo download= are outside the budget. For the
   three counts 0 means the default and a negative value turns that
   bound off. The defaults suit a four-core host; see [[Performance]].
   With =ssh.mode = "system"= each SSH session is its own process and

CHANGELOG.org +4 −1

   "the server is busy…" and exits 1, HTTP gets 503 with
   =Retry-After: 30=, and git:// gets an =ERR= line. *Operators:* the
   defaults are tuned for a four-core host; set the three counts to -1
-  to turn the limit off. Ref listings, pushes and web archives are
+  to turn the limit off. Ref listings, pushes and =repo download= are
   unaffected. Under =ssh.mode = "system"= SSH clones are not counted,
   since each session is its own process (#262).
 - Anonymous clones are counted per IPv4 address or IPv6 /64, and
   together hold at most =pack_concurrency= − 1 slots, so a signed-in
   client can always get the last one (#262).
+- Web archive downloads (=/{owner}/{repo}/archive/{ref}.tar.gz=) take
+  a pack slot, answer 503 with =Retry-After: 30= when none is free, and
+  are killed when the client leaves or stops reading (#262).
 * v1.36.0 — 2026-09-23

internal/gitutil/read.go +6 −1

 // Archive streams a tar.gz of ref to w, within archiveTimeout and
 // MaxArchiveBytes. Past either, git is killed and the error says which.
 func Archive(dir, ref, prefix string, w io.Writer) error {
+	return ArchiveUntil(dir, ref, prefix, w, nil)
+}
+// ArchiveUntil is Archive, killing git when stop closes.
+func ArchiveUntil(dir, ref, prefix string, w io.Writer, stop <-chan struct{}) error {
 	ctx, cancel := context.WithTimeout(context.Background(), archiveTimeout)
 	defer cancel()
 	cmd := exec.CommandContext(ctx, toolpath.Look("git"), "-C", dir, "archive", "--format=tar.gz", "--prefix="+prefix+"/", "--end-of-options", ref)
 	lw := &cappedWriter{w: w, left: MaxArchiveBytes, stop: cancel}
 	cmd.Stdout = lw
-	err := cmd.Run()
+	err := RunUntil(cmd, stop)
 	switch {
 	case lw.exceeded:
 		return ErrArchiveTooLarge

internal/httpd/packlimit_test.go +60

	@@ -286,3 +286,63 @@ func TestFetchKilledWhenClientStopsReading(t *testing.T) {
286	// The 32MB pack cannot fit in the socket buffers; nothing is read.	286	// The 32MB pack cannot fit in the socket buffers; nothing is read.
287	ended(t, s, finished, 20*time.Second)	287	ended(t, s, finished, 20*time.Second)
288	}	288	}
		289
		290	func getArchive(s Server, w http.ResponseWriter, r http.Request) {
		291	r.SetPathValue("owner", "alice")
		292	r.SetPathValue("repo", "app")
		293	r.SetPathValue("file", "main.tar.gz")
		294	s.archive(w, r)
		295	}
		296
		297	// A web archive takes a pack slot: busy is 503, and the slot is free
		298	// again once the archive is written.
		299	func TestArchiveTakesASlot(t *testing.T) {
		300	s := limitedServer(t)
		301	seed(t, s, 10)
		302	hold, err := s.packs.Acquire(nil, "ip:elsewhere")
		303	if err != nil {
		304	t.Fatal(err)
		305	}
		306	w := httptest.NewRecorder()
		307	getArchive(s, w, httptest.NewRequest("GET", "/alice/app/archive/main.tar.gz", nil))
		308	if w.Code != http.StatusServiceUnavailable \|\| w.Header().Get("Retry-After") == "" {
		309	t.Fatalf("busy: status %d, Retry-After %q", w.Code, w.Header().Get("Retry-After"))
		310	}
		311	hold()
		312	w = httptest.NewRecorder()
		313	getArchive(s, w, httptest.NewRequest("GET", "/alice/app/archive/main.tar.gz", nil))
		314	if w.Code != http.StatusOK \|\| w.Header().Get("Content-Type") != "application/gzip" \|\| w.Body.Len() == 0 {
		315	t.Fatalf("free: status %d, type %q, %d bytes", w.Code, w.Header().Get("Content-Type"), w.Body.Len())
		316	}
		317	hold, err = s.packs.Acquire(nil, "ip:elsewhere")
		318	if err != nil {
		319	t.Fatalf("slot not released after the archive: %v", err)
		320	}
		321	hold()
		322	}
		323
		324	// An archive whose client stops reading is cut after StallDeadline.
		325	func TestArchiveKilledWhenClientStopsReading(t *testing.T) {
		326	stallAfter(t, 500*time.Millisecond)
		327	s := limitedServer(t)
		328	sizes := make([]int, 16)
		329	for i := range sizes {
		330	sizes[i] = 2 << 20
		331	}
		332	seed(t, s, sizes...)
		333	finished := make(chan struct{})
		334	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		335	getArchive(s, w, r)
		336	close(finished)
		337	}))
		338	defer srv.Close()
		339	conn, err := net.Dial("tcp", srv.Listener.Addr().String())
		340	if err != nil {
		341	t.Fatal(err)
		342	}
		343	defer conn.Close()
		344	conn.(*net.TCPConn).SetReadBuffer(4096)
		345	fmt.Fprintf(conn, "GET /alice/app/archive/main.tar.gz HTTP/1.1\r\nHost: x\r\n\r\n")
		346	// The 32MB archive cannot fit in the socket buffers; nothing is read.
		347	ended(t, s, finished, 20*time.Second)
		348	}

internal/httpd/smart.go +52 −42

 	cancel := r.Context().Done()
 	out := io.Writer(w)
 	if !lsRefs(br) {
-		// A queued clone waits at most the limiter's wait, and Stop ends
+		o, kill, finish, ok := s.packSlot(w, r)
-		// the wait so it does not hold up a restart's drain. net/http
+		if !ok {
-		// notices a departed client only after the body is read, so
-		// that rarely ends it.
-		release, err := s.packs.Acquire(s.until(r), s.packPrincipal(r))
-		if err != nil {
-			msg := "the server is restarting; try again in a minute"
-			if errors.Is(err, packlimit.ErrBusy) {
-				msg = "the server is busy: it is at its limit of concurrent clones and fetches; try again in a minute"
-			}
-			w.Header().Set("Retry-After", "30")
-			http.Error(w, msg, http.StatusServiceUnavailable)
 			return
 		}
 		// Deferred before git runs, so it fires after git has exited
 		// and been waited for.
-		defer release()
+		defer finish()
-		var stalled <-chan struct{}
+		out, cancel = o, kill
-		var unwatch func()
-		out, stalled, unwatch = s.packs.Watch(w)
-		defer unwatch()
-		kill := make(chan struct{})
-		finished := make(chan struct{})
-		exited := make(chan struct{})
-		// The watcher must not touch w once the handler has returned.
-		defer func() {
-			close(finished)
-			<-exited
-		}()
-		go func() {
-			defer close(exited)
-			select {
-			case <-finished:
-				return
-			case <-r.Context().Done():
-			case <-stalled:
-				// A write blocked on a client that stopped reading, or a
-				// read of a body it stopped sending, outlives git;
-				// expired deadlines end both copies, so Wait returns.
-				rc := http.NewResponseController(w)
-				rc.SetReadDeadline(time.Now())
-				rc.SetWriteDeadline(time.Now())
-			}
-			close(kill)
-		}()
-		cancel = kill
 	}
 	w.Header().Set("Content-Type", "application/x-git-upload-pack-result")
 	w.Header().Set("Cache-Control", "no-cache")
 	gitutil.RunUntil(cmd, cancel)
 }
+// packSlot takes a pack-generation slot for r, answering 503 with
+// Retry-After when none comes free. A queued request waits at most the
+// limiter's wait, and Stop ends the wait so it does not hold up a
+// restart's drain; net/http notices a departed client only after the
+// body is read, so that rarely ends it. On success out is w watched for
+// stalls, kill closes when git must stop — the client left, or no write
+// to it completed for packlimit.StallDeadline — and finish, called once
+// git has exited, releases the slot.
+func (s *Server) packSlot(w http.ResponseWriter, r *http.Request) (out io.Writer, kill <-chan struct{}, finish func(), ok bool) {
+	release, err := s.packs.Acquire(s.until(r), s.packPrincipal(r))
+	if err != nil {
+		msg := "the server is restarting; try again in a minute"
+		if errors.Is(err, packlimit.ErrBusy) {
+			msg = "the server is busy: it is at its limit of concurrent clones and fetches; try again in a minute"
+		}
+		w.Header().Set("Retry-After", "30")
+		http.Error(w, msg, http.StatusServiceUnavailable)
+		return nil, nil, nil, false
+	}
+	out, stalled, unwatch := s.packs.Watch(w)
+	killed := make(chan struct{})
+	finished := make(chan struct{})
+	exited := make(chan struct{})
+	go func() {
+		defer close(exited)
+		select {
+		case <-finished:
+			return
+		case <-r.Context().Done():
+		case <-stalled:
+			// A write blocked on a client that stopped reading, or a
+			// read of a body it stopped sending, outlives git;
+			// expired deadlines end both copies, so Wait returns.
+			rc := http.NewResponseController(w)
+			rc.SetReadDeadline(time.Now())
+			rc.SetWriteDeadline(time.Now())
+		}
+		close(killed)
+	}()
+	return out, killed, func() {
+		// The watcher must not touch w once the handler has returned.
+		close(finished)
+		<-exited
+		unwatch()
+		release()
+	}, true
+}
 // lsRefs reports whether a protocol v2 request is a ref listing, which
 // generates no pack. Its first pkt-line is "command=ls-refs".
 func lsRefs(br *bufio.Reader) bool {

internal/httpd/web.go +6 −1

 		s.notFound(w, r)
 		return
 	}
+	out, kill, finish, ok := s.packSlot(w, r)
+	if !ok {
+		return
+	}
+	defer finish()
 	prefix := fmt.Sprintf("%s-%s", p.Repo.Name, ref)
 	w.Header().Set("Content-Type", "application/gzip")
 	w.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=%q", prefix+".tar.gz"))
-	gitutil.Archive(p.Dir, ref, prefix, w)
+	gitutil.ArchiveUntil(p.Dir, ref, prefix, out, kill)
 }
 func policyCanAdmin(u store.User, repo store.Repo, grant string) bool {

Commit bd49b87fce

Commit `bd49b87fce`