app/httpclient.go

  1package app
  2
  3import (
  4	"net/http"
  5	"net/url"
  6	"strings"
  7	"sync"
  8	"time"
  9)
 10
 11// DeviantArt fronts its API with AWS CloudFront + WAF, which bans egress IPs that
 12// hit it too hard. Under a bot flood, unbounded concurrent handlers each fetch
 13// ~150-200 KB of DA JSON, which both hammers that IP (risking a ban) and can OOM
 14// the process. devianter makes its requests with a bare &http.Client{}, so they go
 15// through http.DefaultTransport — we wrap it here to bound the rate and concurrency
 16// of calls to deviantart.com and to add timeouts. Requests to other hosts (e.g.
 17// wixmp image CDN) are passed straight through, so media stays fast.
 18//
 19// http.ProxyFromEnvironment is preserved, so HTTPS_PROXY (VPN egress) still applies.
 20
 21// Tunables, set from the upstream config block by ExecuteConfig; these are
 22// the defaults for a config that omits it. Slower is gentler on the egress
 23// address, which DeviantArt bans when it asks too often.
 24var (
 25	daMinInterval   = 400 * time.Millisecond // minimum gap between DA request starts
 26	daMaxConcurrent = 2                      // max simultaneous in-flight DA requests
 27)
 28
 29// downloadTimeout bounds a single outbound fetch end to end, so that a stalled
 30// CDN connection cannot pin a request handler open indefinitely.
 31const downloadTimeout = 60 * time.Second
 32
 33type daThrottle struct {
 34	base http.RoundTripper
 35	sem  chan struct{}
 36	mu   sync.Mutex
 37	last time.Time
 38}
 39
 40// RoundTrip applies the rate and concurrency limits to DeviantArt requests and
 41// passes everything else straight through to the base transport.
 42func (t *daThrottle) RoundTrip(req *http.Request) (*http.Response, error) {
 43	// Only throttle DeviantArt's WAF-protected API host; let everything else fly.
 44	if !strings.Contains(req.URL.Hostname(), "deviantart.com") {
 45		return t.base.RoundTrip(req)
 46	}
 47
 48	// Concurrency cap: block until a slot frees up (backpressure under floods).
 49	t.sem <- struct{}{}
 50	defer func() { <-t.sem }()
 51
 52	// Rate cap: enforce a minimum interval between request starts.
 53	t.mu.Lock()
 54	if wait := daMinInterval - time.Since(t.last); wait > 0 {
 55		time.Sleep(wait)
 56	}
 57	t.last = time.Now()
 58	t.mu.Unlock()
 59
 60	return t.base.RoundTrip(req)
 61}
 62
 63// baseTransport is the tuned transport installed by InstallDAThrottle, kept so
 64// that per-client transports (see ProxiedTransport) inherit the same timeouts
 65// instead of silently bypassing them.
 66var baseTransport *http.Transport
 67
 68// tunedTransport clones the current default transport, preserving its Proxy
 69// (ProxyFromEnvironment) and connection-pool defaults, and tightens timeouts to
 70// bound hung connections.
 71func tunedTransport() *http.Transport {
 72	base, ok := http.DefaultTransport.(*http.Transport)
 73	if !ok {
 74		// Already wrapped, or a non-standard transport is installed. Start from a
 75		// fresh one rather than panicking on a type assertion.
 76		base = &http.Transport{Proxy: http.ProxyFromEnvironment}
 77	}
 78
 79	t := base.Clone()
 80	t.TLSHandshakeTimeout = 10 * time.Second
 81	t.ResponseHeaderTimeout = 20 * time.Second
 82	t.ExpectContinueTimeout = 2 * time.Second
 83	return t
 84}
 85
 86// daCache is the API response cache shared by every transport, or nil when
 87// api-cache.enabled is false.
 88var daCache *apiCache
 89
 90// chain wraps base with the throttle and, when enabled, the cache in front
 91// of it, so a hit never spends a throttle slot.
 92func chain(base http.RoundTripper) http.RoundTripper {
 93	rt := throttled(base)
 94	if daCache != nil {
 95		return daCache.transport(rt)
 96	}
 97	return rt
 98}
 99
100// logCacheStatsForever prints one line an hour so an operator can see the
101// cache working without an endpoint. Run it in its own goroutine.
102func logCacheStatsForever(c *apiCache) {
103	for {
104		time.Sleep(time.Hour)
105		hits, misses, stale, entries, held := c.stats()
106		println("api cache:", hits, "hits,", misses, "misses,", stale, "served stale,", entries, "entries,", held>>20, "MB held")
107	}
108}
109
110// InstallDAThrottle wraps http.DefaultTransport with the rate/concurrency limits
111// and timeouts above, and with the API response cache when it is enabled. Call
112// once at startup, after ExecuteConfig and before any DeviantArt request.
113func InstallDAThrottle() {
114	baseTransport = tunedTransport()
115	if CFG.APICache.Enabled {
116		daCache = newAPICache(CFG.APICache.MaxSize<<20, apiCacheTTL, apiCacheStale)
117		go logCacheStatsForever(daCache)
118	}
119	http.DefaultTransport = chain(baseTransport)
120}
121
122// throttled wraps base with the DeviantArt rate and concurrency limits.
123func throttled(base http.RoundTripper) http.RoundTripper {
124	return &daThrottle{base: base, sem: make(chan struct{}, daMaxConcurrent)}
125}
126
127// ProxiedTransport returns a throttled transport routing through proxy. Downloads
128// configured with download-proxy go through here so they keep the timeouts and
129// limits that InstallDAThrottle installs on the default transport.
130func ProxiedTransport(proxy *url.URL) http.RoundTripper {
131	var base *http.Transport
132	if baseTransport != nil {
133		base = baseTransport.Clone()
134	} else {
135		base = tunedTransport()
136	}
137	base.Proxy = http.ProxyURL(proxy)
138	return chain(base)
139}