Commit `5763dc3cfe`

5763dc3cfeadf491d6eda3b6a91623f86396a134

parent: 8a00f7c55c

Verified · cmc

cmc <hello@cleberg.net> · 2026-07-15 01:22 UTC

feat: throttle and time out outbound DeviantArt requests

DeviantArt fronts its API with CloudFront + WAF, which bans egress IPs
that hit it too hard. devianter issues requests with a bare http.Client,
so unbounded concurrent handlers each pulled ~150-200 KB of JSON, which
both risked a ban and could exhaust the process under a bot flood.

Wrap the default transport to bound rate and concurrency for
deviantart.com and add timeouts. Other hosts (wixmp image CDN) pass
straight through, so media stays fast, and ProxyFromEnvironment is
preserved so HTTPS_PROXY egress still works.

Layout: unified · split

app/httpclient.go added +68

		@@ -0,0 +1,68 @@
	1	package app
	2
	3	import (
	4	"net/http"
	5	"strings"
	6	"sync"
	7	"time"
	8	)
	9
	10	// DeviantArt fronts its API with AWS CloudFront + WAF, which bans egress IPs that
	11	// hit it too hard. Under a bot flood, unbounded concurrent handlers each fetch
	12	// ~150-200 KB of DA JSON, which both hammers that IP (risking a ban) and can OOM
	13	// the process. devianter makes its requests with a bare &http.Client{}, so they go
	14	// through http.DefaultTransport — we wrap it here to bound the rate and concurrency
	15	// of calls to deviantart.com and to add timeouts. Requests to other hosts (e.g.
	16	// wixmp image CDN) are passed straight through, so media stays fast.
	17	//
	18	// http.ProxyFromEnvironment is preserved, so HTTPS_PROXY (VPN egress) still applies.
	19
	20	// Tunables (kept in source; safe defaults). Lower is gentler on the DA IP.
	21	var (
	22	daMinInterval = 400 * time.Millisecond // minimum gap between DA request starts
	23	daMaxConcurrent = 2 // max simultaneous in-flight DA requests
	24	)
	25
	26	type daThrottle struct {
	27	base http.RoundTripper
	28	sem chan struct{}
	29	mu sync.Mutex
	30	last time.Time
	31	}
	32
	33	func (t daThrottle) RoundTrip(req http.Request) (*http.Response, error) {
	34	// Only throttle DeviantArt's WAF-protected API host; let everything else fly.
	35	if !strings.Contains(req.URL.Hostname(), "deviantart.com") {
	36	return t.base.RoundTrip(req)
	37	}
	38
	39	// Concurrency cap: block until a slot frees up (backpressure under floods).
	40	t.sem <- struct{}{}
	41	defer func() { <-t.sem }()
	42
	43	// Rate cap: enforce a minimum interval between request starts.
	44	t.mu.Lock()
	45	if wait := daMinInterval - time.Since(t.last); wait > 0 {
	46	time.Sleep(wait)
	47	}
	48	t.last = time.Now()
	49	t.mu.Unlock()
	50
	51	return t.base.RoundTrip(req)
	52	}
	53
	54	// InstallDAThrottle wraps http.DefaultTransport with the rate/concurrency limits and
	55	// timeouts above. Call once at startup, before any DeviantArt request is made.
	56	func InstallDAThrottle() {
	57	// Clone the default transport so we keep its Proxy (ProxyFromEnvironment) and
	58	// connection-pool defaults, then tighten timeouts to bound hung connections.
	59	base := http.DefaultTransport.(*http.Transport).Clone()
	60	base.TLSHandshakeTimeout = 10 * time.Second
	61	base.ResponseHeaderTimeout = 20 * time.Second
	62	base.ExpectContinueTimeout = 2 * time.Second
	63
	64	http.DefaultTransport = &daThrottle{
	65	base: base,
	66	sem: make(chan struct{}, daMaxConcurrent),
	67	}
	68	}

app/httpclient_test.go added +137

		@@ -0,0 +1,137 @@
	1	package app
	2
	3	import (
	4	"net/http"
	5	"net/http/httptest"
	6	"sync"
	7	"testing"
	8	"time"
	9	)
	10
	11	// stubTransport records how many requests reached it and returns an empty 200.
	12	type stubTransport struct {
	13	mu sync.Mutex
	14	n int
	15	}
	16
	17	func (s stubTransport) RoundTrip(req http.Request) (*http.Response, error) {
	18	s.mu.Lock()
	19	s.n++
	20	s.mu.Unlock()
	21	return httptest.NewRecorder().Result(), nil
	22	}
	23
	24	func newTestThrottle(base http.RoundTripper, gap time.Duration, max int) *daThrottle {
	25	return &daThrottle{base: base, sem: make(chan struct{}, max)}
	26	}
	27
	28	// DeviantArt requests must be spaced by at least daMinInterval.
	29	func TestThrottleRateLimitsDeviantArt(t *testing.T) {
	30	stub := &stubTransport{}
	31	tr := newTestThrottle(stub, daMinInterval, daMaxConcurrent)
	32
	33	start := time.Now()
	34	const n = 3
	35	for i := 0; i < n; i++ {
	36	req, _ := http.NewRequest("GET", "https://www.deviantart.com/_puppy/x", nil)
	37	if _, err := tr.RoundTrip(req); err != nil {
	38	t.Fatalf("unexpected error: %v", err)
	39	}
	40	}
	41	elapsed := time.Since(start)
	42
	43	// n requests => at least (n-1) gaps between them.
	44	if want := time.Duration(n-1) * daMinInterval; elapsed < want {
	45	t.Errorf("DA requests were not throttled: %d requests took %v, want >= %v", n, elapsed, want)
	46	}
	47	if stub.n != n {
	48	t.Errorf("expected all %d requests to reach the base transport, got %d", n, stub.n)
	49	}
	50	}
	51
	52	// Non-DA hosts (e.g. the wixmp image CDN) must not be slowed down.
	53	func TestThrottleSkipsOtherHosts(t *testing.T) {
	54	stub := &stubTransport{}
	55	tr := newTestThrottle(stub, daMinInterval, daMaxConcurrent)
	56
	57	start := time.Now()
	58	for i := 0; i < 5; i++ {
	59	req, _ := http.NewRequest("GET", "https://images-wixmp-ed30a86b8c4ca887773594c2.wixmp.com/f/x.jpg", nil)
	60	if _, err := tr.RoundTrip(req); err != nil {
	61	t.Fatalf("unexpected error: %v", err)
	62	}
	63	}
	64
	65	if elapsed := time.Since(start); elapsed >= daMinInterval {
	66	t.Errorf("non-DA host was throttled: 5 requests took %v, want < %v", elapsed, daMinInterval)
	67	}
	68	if stub.n != 5 {
	69	t.Errorf("expected 5 requests through, got %d", stub.n)
	70	}
	71	}
	72
	73	// Concurrent callers must never exceed daMaxConcurrent in-flight DA requests.
	74	func TestThrottleCapsConcurrency(t *testing.T) {
	75	var (
	76	mu sync.Mutex
	77	inFlight int
	78	peak int
	79	)
	80	counting := roundTripFunc(func(req http.Request) (http.Response, error) {
	81	mu.Lock()
	82	inFlight++
	83	if inFlight > peak {
	84	peak = inFlight
	85	}
	86	mu.Unlock()
	87
	88	time.Sleep(20 * time.Millisecond) // hold the slot
	89
	90	mu.Lock()
	91	inFlight--
	92	mu.Unlock()
	93	return httptest.NewRecorder().Result(), nil
	94	})
	95
	96	tr := newTestThrottle(counting, daMinInterval, daMaxConcurrent)
	97
	98	var wg sync.WaitGroup
	99	for i := 0; i < 6; i++ {
	100	wg.Add(1)
	101	go func() {
	102	defer wg.Done()
	103	req, _ := http.NewRequest("GET", "https://www.deviantart.com/_puppy/x", nil)
	104	tr.RoundTrip(req)
	105	}()
	106	}
	107	wg.Wait()
	108
	109	if peak > daMaxConcurrent {
	110	t.Errorf("concurrency cap breached: peak %d in-flight DA requests, max %d", peak, daMaxConcurrent)
	111	}
	112	}
	113
	114	type roundTripFunc func(http.Request) (http.Response, error)
	115
	116	func (f roundTripFunc) RoundTrip(r http.Request) (http.Response, error) { return f(r) }
	117
	118	// InstallDAThrottle must preserve proxy-from-environment so HTTPS_PROXY (VPN
	119	// egress) keeps working, and must not panic on a repeat call.
	120	func TestInstallDAThrottlePreservesProxy(t *testing.T) {
	121	orig := http.DefaultTransport
	122	defer func() { http.DefaultTransport = orig }()
	123
	124	InstallDAThrottle()
	125
	126	th, ok := http.DefaultTransport.(*daThrottle)
	127	if !ok {
	128	t.Fatalf("DefaultTransport was not wrapped, got %T", http.DefaultTransport)
	129	}
	130	base, ok := th.base.(*http.Transport)
	131	if !ok {
	132	t.Fatalf("base transport is not *http.Transport, got %T", th.base)
	133	}
	134	if base.Proxy == nil {
	135	t.Error("base transport lost its Proxy func: HTTPS_PROXY / VPN egress would break")
	136	}
	137	}

main.go +4

		@@ -17,6 +17,10 @@ func main() {
17	17	app.ExecuteConfig()
18	18	static.CopyTemplatesToMemory()
19	19
	20	// Rate/concurrency-limit + time-out outbound DeviantArt requests so bot floods
	21	// can't exhaust the process or get our egress IP banned by CloudFront/WAF.
	22	app.InstallDAThrottle()
	23
20	24	go func() {
21	25	for {
22	26	err := devianter.UpdateCSRF()

Commit 5763dc3cfe

Commit `5763dc3cfe`