krz/skunky-art

Alternative privacy frontend for DeviantArt. deviantart frontend privacy web https://art.krz.sh

Commit 5763dc3cfe

5763dc3cfeadf491d6eda3b6a91623f86396a134

parent: 8a00f7c55c

Verified · cmc

cmc <hello@cleberg.net> · 2026-07-15 01:22 UTC 

feat: throttle and time out outbound DeviantArt requests

DeviantArt fronts its API with CloudFront + WAF, which bans egress IPs
that hit it too hard. devianter issues requests with a bare http.Client,
so unbounded concurrent handlers each pulled ~150-200 KB of JSON, which
both risked a ban and could exhaust the process under a bot flood.

Wrap the default transport to bound rate and concurrency for
deviantart.com and add timeouts. Other hosts (wixmp image CDN) pass
straight through, so media stays fast, and ProxyFromEnvironment is
preserved so HTTPS_PROXY egress still works.

Layout: unified · split

app/httpclient.go added +68
@@ -0,0 +1,68 @@
1package app
2
3import (
4 "net/http"
5 "strings"
6 "sync"
7 "time"
8)
9
10// DeviantArt fronts its API with AWS CloudFront + WAF, which bans egress IPs that
11// hit it too hard. Under a bot flood, unbounded concurrent handlers each fetch
12// ~150-200 KB of DA JSON, which both hammers that IP (risking a ban) and can OOM
13// the process. devianter makes its requests with a bare &http.Client{}, so they go
14// through http.DefaultTransport — we wrap it here to bound the rate and concurrency
15// of calls to deviantart.com and to add timeouts. Requests to other hosts (e.g.
16// wixmp image CDN) are passed straight through, so media stays fast.
17//
18// http.ProxyFromEnvironment is preserved, so HTTPS_PROXY (VPN egress) still applies.
19
20// Tunables (kept in source; safe defaults). Lower is gentler on the DA IP.
21var (
22 daMinInterval = 400 * time.Millisecond // minimum gap between DA request starts
23 daMaxConcurrent = 2 // max simultaneous in-flight DA requests
24)
25
26type daThrottle struct {
27 base http.RoundTripper
28 sem chan struct{}
29 mu sync.Mutex
30 last time.Time
31}
32
33func (t *daThrottle) RoundTrip(req *http.Request) (*http.Response, error) {
34 // Only throttle DeviantArt's WAF-protected API host; let everything else fly.
35 if !strings.Contains(req.URL.Hostname(), "deviantart.com") {
36 return t.base.RoundTrip(req)
37 }
38
39 // Concurrency cap: block until a slot frees up (backpressure under floods).
40 t.sem <- struct{}{}
41 defer func() { <-t.sem }()
42
43 // Rate cap: enforce a minimum interval between request starts.
44 t.mu.Lock()
45 if wait := daMinInterval - time.Since(t.last); wait > 0 {
46 time.Sleep(wait)
47 }
48 t.last = time.Now()
49 t.mu.Unlock()
50
51 return t.base.RoundTrip(req)
52}
53
54// InstallDAThrottle wraps http.DefaultTransport with the rate/concurrency limits and
55// timeouts above. Call once at startup, before any DeviantArt request is made.
56func InstallDAThrottle() {
57 // Clone the default transport so we keep its Proxy (ProxyFromEnvironment) and
58 // connection-pool defaults, then tighten timeouts to bound hung connections.
59 base := http.DefaultTransport.(*http.Transport).Clone()
60 base.TLSHandshakeTimeout = 10 * time.Second
61 base.ResponseHeaderTimeout = 20 * time.Second
62 base.ExpectContinueTimeout = 2 * time.Second
63
64 http.DefaultTransport = &daThrottle{
65 base: base,
66 sem: make(chan struct{}, daMaxConcurrent),
67 }
68}
app/httpclient_test.go added +137
@@ -0,0 +1,137 @@
1package app
2
3import (
4 "net/http"
5 "net/http/httptest"
6 "sync"
7 "testing"
8 "time"
9)
10
11// stubTransport records how many requests reached it and returns an empty 200.
12type stubTransport struct {
13 mu sync.Mutex
14 n int
15}
16
17func (s *stubTransport) RoundTrip(req *http.Request) (*http.Response, error) {
18 s.mu.Lock()
19 s.n++
20 s.mu.Unlock()
21 return httptest.NewRecorder().Result(), nil
22}
23
24func newTestThrottle(base http.RoundTripper, gap time.Duration, max int) *daThrottle {
25 return &daThrottle{base: base, sem: make(chan struct{}, max)}
26}
27
28// DeviantArt requests must be spaced by at least daMinInterval.
29func TestThrottleRateLimitsDeviantArt(t *testing.T) {
30 stub := &stubTransport{}
31 tr := newTestThrottle(stub, daMinInterval, daMaxConcurrent)
32
33 start := time.Now()
34 const n = 3
35 for i := 0; i < n; i++ {
36 req, _ := http.NewRequest("GET", "https://www.deviantart.com/_puppy/x", nil)
37 if _, err := tr.RoundTrip(req); err != nil {
38 t.Fatalf("unexpected error: %v", err)
39 }
40 }
41 elapsed := time.Since(start)
42
43 // n requests => at least (n-1) gaps between them.
44 if want := time.Duration(n-1) * daMinInterval; elapsed < want {
45 t.Errorf("DA requests were not throttled: %d requests took %v, want >= %v", n, elapsed, want)
46 }
47 if stub.n != n {
48 t.Errorf("expected all %d requests to reach the base transport, got %d", n, stub.n)
49 }
50}
51
52// Non-DA hosts (e.g. the wixmp image CDN) must not be slowed down.
53func TestThrottleSkipsOtherHosts(t *testing.T) {
54 stub := &stubTransport{}
55 tr := newTestThrottle(stub, daMinInterval, daMaxConcurrent)
56
57 start := time.Now()
58 for i := 0; i < 5; i++ {
59 req, _ := http.NewRequest("GET", "https://images-wixmp-ed30a86b8c4ca887773594c2.wixmp.com/f/x.jpg", nil)
60 if _, err := tr.RoundTrip(req); err != nil {
61 t.Fatalf("unexpected error: %v", err)
62 }
63 }
64
65 if elapsed := time.Since(start); elapsed >= daMinInterval {
66 t.Errorf("non-DA host was throttled: 5 requests took %v, want < %v", elapsed, daMinInterval)
67 }
68 if stub.n != 5 {
69 t.Errorf("expected 5 requests through, got %d", stub.n)
70 }
71}
72
73// Concurrent callers must never exceed daMaxConcurrent in-flight DA requests.
74func TestThrottleCapsConcurrency(t *testing.T) {
75 var (
76 mu sync.Mutex
77 inFlight int
78 peak int
79 )
80 counting := roundTripFunc(func(req *http.Request) (*http.Response, error) {
81 mu.Lock()
82 inFlight++
83 if inFlight > peak {
84 peak = inFlight
85 }
86 mu.Unlock()
87
88 time.Sleep(20 * time.Millisecond) // hold the slot
89
90 mu.Lock()
91 inFlight--
92 mu.Unlock()
93 return httptest.NewRecorder().Result(), nil
94 })
95
96 tr := newTestThrottle(counting, daMinInterval, daMaxConcurrent)
97
98 var wg sync.WaitGroup
99 for i := 0; i < 6; i++ {
100 wg.Add(1)
101 go func() {
102 defer wg.Done()
103 req, _ := http.NewRequest("GET", "https://www.deviantart.com/_puppy/x", nil)
104 tr.RoundTrip(req)
105 }()
106 }
107 wg.Wait()
108
109 if peak > daMaxConcurrent {
110 t.Errorf("concurrency cap breached: peak %d in-flight DA requests, max %d", peak, daMaxConcurrent)
111 }
112}
113
114type roundTripFunc func(*http.Request) (*http.Response, error)
115
116func (f roundTripFunc) RoundTrip(r *http.Request) (*http.Response, error) { return f(r) }
117
118// InstallDAThrottle must preserve proxy-from-environment so HTTPS_PROXY (VPN
119// egress) keeps working, and must not panic on a repeat call.
120func TestInstallDAThrottlePreservesProxy(t *testing.T) {
121 orig := http.DefaultTransport
122 defer func() { http.DefaultTransport = orig }()
123
124 InstallDAThrottle()
125
126 th, ok := http.DefaultTransport.(*daThrottle)
127 if !ok {
128 t.Fatalf("DefaultTransport was not wrapped, got %T", http.DefaultTransport)
129 }
130 base, ok := th.base.(*http.Transport)
131 if !ok {
132 t.Fatalf("base transport is not *http.Transport, got %T", th.base)
133 }
134 if base.Proxy == nil {
135 t.Error("base transport lost its Proxy func: HTTPS_PROXY / VPN egress would break")
136 }
137}
main.go +4
@@ -17,6 +17,10 @@ func main() {
17 app.ExecuteConfig() 17 app.ExecuteConfig()
18 static.CopyTemplatesToMemory() 18 static.CopyTemplatesToMemory()
19 19
20 // Rate/concurrency-limit + time-out outbound DeviantArt requests so bot floods
21 // can't exhaust the process or get our egress IP banned by CloudFront/WAF.
22 app.InstallDAThrottle()
23
20 go func() { 24 go func() {
21 for { 25 for {
22 err := devianter.UpdateCSRF() 26 err := devianter.UpdateCSRF()