krz/skunky-art

  1package app
  2
  3import (
  4	"net/http"
  5	"net/url"
  6	"strings"
  7	"sync"
  8	"time"
  9)
 10
 11// DeviantArt fronts its API with AWS CloudFront + WAF, which bans egress IPs that
 12// hit it too hard. Under a bot flood, unbounded concurrent handlers each fetch
 13// ~150-200 KB of DA JSON, which both hammers that IP (risking a ban) and can OOM
 14// the process. devianter makes its requests with a bare &http.Client{}, so they go
 15// through http.DefaultTransport — we wrap it here to bound the rate and concurrency
 16// of calls to deviantart.com and to add timeouts. Requests to other hosts (e.g.
 17// wixmp image CDN) are passed straight through, so media stays fast.
 18//
 19// http.ProxyFromEnvironment is preserved, so HTTPS_PROXY (VPN egress) still applies.
 20
 21// Tunables (kept in source; safe defaults). Lower is gentler on the DA IP.
 22var (
 23	daMinInterval   = 400 * time.Millisecond // minimum gap between DA request starts
 24	daMaxConcurrent = 2                      // max simultaneous in-flight DA requests
 25)
 26
 27// downloadTimeout bounds a single outbound fetch end to end, so that a stalled
 28// CDN connection cannot pin a request handler open indefinitely.
 29const downloadTimeout = 60 * time.Second
 30
 31type daThrottle struct {
 32	base http.RoundTripper
 33	sem  chan struct{}
 34	mu   sync.Mutex
 35	last time.Time
 36}
 37
 38// RoundTrip applies the rate and concurrency limits to DeviantArt requests and
 39// passes everything else straight through to the base transport.
 40func (t *daThrottle) RoundTrip(req *http.Request) (*http.Response, error) {
 41	// Only throttle DeviantArt's WAF-protected API host; let everything else fly.
 42	if !strings.Contains(req.URL.Hostname(), "deviantart.com") {
 43		return t.base.RoundTrip(req)
 44	}
 45
 46	// Concurrency cap: block until a slot frees up (backpressure under floods).
 47	t.sem <- struct{}{}
 48	defer func() { <-t.sem }()
 49
 50	// Rate cap: enforce a minimum interval between request starts.
 51	t.mu.Lock()
 52	if wait := daMinInterval - time.Since(t.last); wait > 0 {
 53		time.Sleep(wait)
 54	}
 55	t.last = time.Now()
 56	t.mu.Unlock()
 57
 58	return t.base.RoundTrip(req)
 59}
 60
 61// baseTransport is the tuned transport installed by InstallDAThrottle, kept so
 62// that per-client transports (see ProxiedTransport) inherit the same timeouts
 63// instead of silently bypassing them.
 64var baseTransport *http.Transport
 65
 66// tunedTransport clones the current default transport, preserving its Proxy
 67// (ProxyFromEnvironment) and connection-pool defaults, and tightens timeouts to
 68// bound hung connections.
 69func tunedTransport() *http.Transport {
 70	base, ok := http.DefaultTransport.(*http.Transport)
 71	if !ok {
 72		// Already wrapped, or a non-standard transport is installed. Start from a
 73		// fresh one rather than panicking on a type assertion.
 74		base = &http.Transport{Proxy: http.ProxyFromEnvironment}
 75	}
 76
 77	t := base.Clone()
 78	t.TLSHandshakeTimeout = 10 * time.Second
 79	t.ResponseHeaderTimeout = 20 * time.Second
 80	t.ExpectContinueTimeout = 2 * time.Second
 81	return t
 82}
 83
 84// InstallDAThrottle wraps http.DefaultTransport with the rate/concurrency limits and
 85// timeouts above. Call once at startup, before any DeviantArt request is made.
 86func InstallDAThrottle() {
 87	baseTransport = tunedTransport()
 88	http.DefaultTransport = throttled(baseTransport)
 89}
 90
 91// throttled wraps base with the DeviantArt rate and concurrency limits.
 92func throttled(base http.RoundTripper) http.RoundTripper {
 93	return &daThrottle{base: base, sem: make(chan struct{}, daMaxConcurrent)}
 94}
 95
 96// ProxiedTransport returns a throttled transport routing through proxy. Downloads
 97// configured with download-proxy go through here so they keep the timeouts and
 98// limits that InstallDAThrottle installs on the default transport.
 99func ProxiedTransport(proxy *url.URL) http.RoundTripper {
100	var base *http.Transport
101	if baseTransport != nil {
102		base = baseTransport.Clone()
103	} else {
104		base = tunedTransport()
105	}
106	base.Proxy = http.ProxyURL(proxy)
107	return throttled(base)
108}