krz/skunky-art

Alternative privacy frontend for DeviantArt. deviantart frontend privacy web https://art.krz.sh

Commit 4261b937e4

4261b937e4b0e550d83fd57dabb7a21e6ba69d46

parent: 1b89e1fe37

Verified · cmc ci/build: success ci/lint: success ci/test: success

cmc <hello@cleberg.net> · 2026-09-11 02:52 UTC 

Add robots.txt and a per-client rate limit

Page, feed and API requests draw on a token bucket per client address;
media, avatars and static files do not. Over budget answers 429 with
Retry-After. Behind a reverse proxy the client is the rightmost
X-Forwarded-For entry, trusted only when the connection itself is from
a loopback or private address. rate-limit.per-minute (default 60) and
burst (default 20); 0 turns it off.

/robots.txt disallows search, the API, user and group pages, media and
paginated URLs, with a 10 second crawl delay.

Closes #10

Layout: unified · split

SETUP.md +14
@@ -34,6 +34,20 @@ Time units:
34 recently used entries are dropped past this. 34 recently used entries are dropped past this.
35 * `ttl` — how long a response is reused, in the time units above. Default 35 * `ttl` — how long a response is reused, in the time units above. Default
36 `5i`. 36 `5i`.
37* `rate-limit` — Per-client budget for page, feed and API requests, so one
38 crawler cannot spend the whole upstream budget. Media, avatars and static
39 files are not counted. Over budget answers 429 with `Retry-After`.
40 Behind a reverse proxy the client is taken from the rightmost
41 `X-Forwarded-For` entry, but only when the connection itself comes from a
42 loopback or private address; a direct client's header is ignored.
43 * `per-minute` — sustained requests per minute per client, default 60.
44 `0` turns the limit off.
45 * `burst` — how many requests a client can make at once before the rate
46 applies, default 20.
47
48`/robots.txt` is served automatically. It disallows search, the API, user
49and group pages, media and paginated URLs, and asks for a 10 second crawl
50delay; the index, daily deviations and posts stay crawlable.
37* `static-path` — This setting determines path to static, which will be copied to RAM when SkunkyArt is started. Useless if you're use binary compiled with 'embed' tag. 51* `static-path` — This setting determines path to static, which will be copied to RAM when SkunkyArt is started. Useless if you're use binary compiled with 'embed' tag.
38* `download-proxy` — Outbound proxy used when fetching media from DeviantArt's 52* `download-proxy` — Outbound proxy used when fetching media from DeviantArt's
39 CDN. Leave empty (`""`) unless you actually run a proxy: if this points at 53 CDN. Leave empty (`""`) unless you actually run a proxy: if this points at
app/config.go +28 −12
@@ -34,20 +34,26 @@ type apiCacheConfig struct {
34 TTL string `json:"ttl"` 34 TTL string `json:"ttl"`
35} 35}
36 36
37type rateLimitConfig struct {
38 PerMinute int `json:"per-minute"`
39 Burst int `json:"burst"`
40}
41
37type config struct { 42type config struct {
38 cfg string 43 cfg string
39 Listen string `json:"listen"` 44 Listen string `json:"listen"`
40 URI string `json:"uri"` 45 URI string `json:"uri"`
41 Cache cacheConfig `json:"cache"` 46 Cache cacheConfig `json:"cache"`
42 APICache apiCacheConfig `json:"api-cache"` 47 APICache apiCacheConfig `json:"api-cache"`
43 Proxy bool `json:"proxy"` 48 RateLimit rateLimitConfig `json:"rate-limit"`
44 Nsfw bool `json:"nsfw"` 49 Proxy bool `json:"proxy"`
45 HideAI bool `json:"hide-ai"` 50 Nsfw bool `json:"nsfw"`
46 Theme string `json:"theme"` 51 HideAI bool `json:"hide-ai"`
47 Language string `json:"language"` 52 Theme string `json:"theme"`
48 UserAgent string `json:"user-agent"` 53 Language string `json:"language"`
49 DownloadProxy string `json:"download-proxy"` 54 UserAgent string `json:"user-agent"`
50 StaticPath string `json:"static-path"` 55 DownloadProxy string `json:"download-proxy"`
56 StaticPath string `json:"static-path"`
51} 57}
52 58
53// CFG is the running instance's configuration, holding the defaults below until 59// CFG is the running instance's configuration, holding the defaults below until
@@ -68,6 +74,10 @@ var CFG = config{
68 MaxSize: 64, 74 MaxSize: 64,
69 TTL: "5i", 75 TTL: "5i",
70 }, 76 },
77 RateLimit: rateLimitConfig{
78 PerMinute: 60,
79 Burst: 20,
80 },
71 StaticPath: "static", 81 StaticPath: "static",
72 UserAgent: "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.36", 82 UserAgent: "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.36",
73 Proxy: true, 83 Proxy: true,
@@ -192,6 +202,12 @@ func ExecuteConfig() {
192 apiCacheTTL = d 202 apiCacheTTL = d
193 } 203 }
194 204
205 // per-minute 0 turns the limit off; a burst below one token would
206 // refuse every request, so it is floored to one.
207 if CFG.RateLimit.PerMinute > 0 {
208 daLimiter = newRateLimiter(CFG.RateLimit.PerMinute, max(CFG.RateLimit.Burst, 1))
209 }
210
195 static.StaticPath = CFG.StaticPath 211 static.StaticPath = CFG.StaticPath
196 devianter.UserAgent = CFG.UserAgent 212 devianter.UserAgent = CFG.UserAgent
197 } 213 }
app/ratelimit.go added +132
@@ -0,0 +1,132 @@
1package app
2
3import (
4 "net"
5 "net/http"
6 "strings"
7 "sync"
8 "time"
9)
10
11// rateLimiter is a token bucket per client address. It bounds how many page,
12// feed and API requests one client can make, so a single crawler cannot spend
13// the whole upstream budget and turn the DeviantArt throttle into latency for
14// everyone else.
15type rateLimiter struct {
16 perSecond float64
17 burst float64
18 now func() time.Time
19
20 mu sync.Mutex
21 buckets map[string]*bucket
22 inserts int
23}
24
25type bucket struct {
26 tokens float64
27 last time.Time
28}
29
30// bucketIdle is how long a client can go unseen before its bucket is dropped.
31const bucketIdle = 10 * time.Minute
32
33func newRateLimiter(perMinute, burst int) *rateLimiter {
34 return &rateLimiter{
35 perSecond: float64(perMinute) / 60,
36 burst: float64(burst),
37 now: time.Now,
38 buckets: map[string]*bucket{},
39 }
40}
41
42// allow reports whether client may make a request now, spending one token if
43// so. A client's first request finds a full bucket.
44func (l *rateLimiter) allow(client string) bool {
45 l.mu.Lock()
46 defer l.mu.Unlock()
47
48 now := l.now()
49 b := l.buckets[client]
50 if b == nil {
51 b = &bucket{tokens: l.burst, last: now}
52 l.buckets[client] = b
53 l.inserts++
54 if l.inserts%1000 == 0 {
55 l.prune(now)
56 }
57 } else {
58 b.tokens = min(l.burst, b.tokens+now.Sub(b.last).Seconds()*l.perSecond)
59 b.last = now
60 }
61
62 if b.tokens < 1 {
63 return false
64 }
65 b.tokens--
66 return true
67}
68
69// prune drops buckets idle past bucketIdle. The caller holds mu.
70func (l *rateLimiter) prune(now time.Time) {
71 for client, b := range l.buckets {
72 if now.Sub(b.last) > bucketIdle {
73 delete(l.buckets, client)
74 }
75 }
76}
77
78// clientAddr is the address a request is limited by. Behind a reverse proxy
79// every connection arrives from the proxy, so when the connection is from a
80// loopback or private address the client is the rightmost X-Forwarded-For
81// entry, which is the one that proxy appended. A direct client's connection is
82// not from a private address, so its own header is never trusted.
83func clientAddr(r *http.Request) string {
84 host, _, err := net.SplitHostPort(r.RemoteAddr)
85 if err != nil {
86 host = r.RemoteAddr
87 }
88 ip := net.ParseIP(host)
89 if ip == nil || (!ip.IsLoopback() && !ip.IsPrivate()) {
90 return host
91 }
92 xff := r.Header.Get("X-Forwarded-For")
93 if xff == "" {
94 return host
95 }
96 parts := strings.Split(xff, ",")
97 if forwarded := strings.TrimSpace(parts[len(parts)-1]); forwarded != "" {
98 return forwarded
99 }
100 return host
101}
102
103// limited reports whether an endpoint counts against the client's budget.
104// Media, avatars and static assets do not: one listing page loads twenty
105// thumbnails, which would exhaust any sensible page budget.
106func limited(endpoint string) bool {
107 switch endpoint {
108 case "media", "stylesheet", "favicon.ico", "robots.txt":
109 return false
110 }
111 return true
112}
113
114// robotsTXT keeps crawlers off the pages that fan out into unbounded upstream
115// requests. The index, daily deviations and posts stay allowed, so the instance
116// is still discoverable.
117func robotsTXT(base string) string {
118 var b strings.Builder
119 b.WriteString("User-agent: *\n")
120 for _, p := range []string{"search", "api", "group_user", "media", "*?p="} {
121 b.WriteString("Disallow: ")
122 b.WriteString(base)
123 b.WriteString(p)
124 b.WriteString("\n")
125 }
126 b.WriteString("Crawl-delay: 10\n")
127 return b.String()
128}
129
130// daLimiter is the running instance's limiter, or nil when rate-limit.per-minute
131// is 0. Set by ExecuteConfig.
132var daLimiter *rateLimiter
app/ratelimit_test.go added +142
@@ -0,0 +1,142 @@
1package app
2
3import (
4 "net/http"
5 "net/http/httptest"
6 "strings"
7 "testing"
8 "time"
9)
10
11func TestRateLimiterAllowsBurstThenRefuses(t *testing.T) {
12 l := newRateLimiter(60, 3)
13 now := time.Now()
14 l.now = func() time.Time { return now }
15
16 for i := range 3 {
17 if !l.allow("c") {
18 t.Fatalf("request %d refused inside the burst", i+1)
19 }
20 }
21 if l.allow("c") {
22 t.Error("request past the burst allowed")
23 }
24 if !l.allow("other") {
25 t.Error("a different client was refused by the first one's budget")
26 }
27}
28
29func TestRateLimiterRefillsAtTheConfiguredRate(t *testing.T) {
30 l := newRateLimiter(60, 1) // one token per second
31 now := time.Now()
32 l.now = func() time.Time { return now }
33
34 l.allow("c")
35 if l.allow("c") {
36 t.Fatal("second immediate request allowed with burst 1")
37 }
38 now = now.Add(500 * time.Millisecond)
39 if l.allow("c") {
40 t.Error("allowed after half a second, want a full second per token")
41 }
42 now = now.Add(600 * time.Millisecond)
43 if !l.allow("c") {
44 t.Error("refused after more than a second, want one token refilled")
45 }
46}
47
48func TestRateLimiterPrunesIdleBuckets(t *testing.T) {
49 l := newRateLimiter(60, 1)
50 now := time.Now()
51 l.now = func() time.Time { return now }
52
53 l.allow("old")
54 now = now.Add(bucketIdle + time.Minute)
55 l.prune(now)
56
57 if _, ok := l.buckets["old"]; ok {
58 t.Error("idle bucket survived a prune")
59 }
60}
61
62func TestClientAddrTrustsForwardedForOnlyBehindAProxy(t *testing.T) {
63 cases := []struct{ remote, xff, want string }{
64 {"127.0.0.1:1234", "203.0.113.5", "203.0.113.5"},
65 {"10.0.0.2:1234", "198.51.100.7, 203.0.113.5", "203.0.113.5"},
66 {"127.0.0.1:1234", "", "127.0.0.1"},
67 {"203.0.113.9:1234", "198.51.100.7", "203.0.113.9"},
68 {"[::1]:1234", "203.0.113.5", "203.0.113.5"},
69 }
70 for _, c := range cases {
71 r := httptest.NewRequest(http.MethodGet, "/", nil)
72 r.RemoteAddr = c.remote
73 if c.xff != "" {
74 r.Header.Set("X-Forwarded-For", c.xff)
75 }
76 if got := clientAddr(r); got != c.want {
77 t.Errorf("remote %s xff %q: got %s, want %s", c.remote, c.xff, got, c.want)
78 }
79 }
80}
81
82func TestRobotsTXTCarriesTheBaseURI(t *testing.T) {
83 out := robotsTXT("/art/")
84 for _, want := range []string{"Disallow: /art/search\n", "Disallow: /art/api\n", "Disallow: /art/*?p=\n", "Crawl-delay: 10\n"} {
85 if !strings.Contains(out, want) {
86 t.Errorf("robots.txt lacks %q:\n%s", want, out)
87 }
88 }
89}
90
91// TestHandlerLimitsPagesButNotMedia drives the real handler: a client past its
92// budget gets 429 on a page with Retry-After and no Cache-Control, while its
93// stylesheet, favicon and avatar requests are never counted.
94func TestHandlerLimitsPagesButNotMedia(t *testing.T) {
95 limiter, uri := daLimiter, CFG.URI
96 daLimiter = newRateLimiter(60, 2)
97 CFG.URI = "/"
98 defer func() { daLimiter, CFG.URI = limiter, uri }()
99
100 orig := fetchAvatar
101 fetchAvatar = func(string, rune) (string, error) { return "png", nil }
102 defer func() { fetchAvatar = orig }()
103
104 from := func(target string) *httptest.ResponseRecorder {
105 loadTemplates()
106 rec := httptest.NewRecorder()
107 req := httptest.NewRequest(http.MethodGet, target, nil)
108 req.RemoteAddr = "203.0.113.5:4000"
109 Handler()(rec, req)
110 return rec
111 }
112
113 from("/about")
114 from("/about")
115 third := from("/about")
116 if third.Code != 429 {
117 t.Fatalf("third page request: status %d, want 429", third.Code)
118 }
119 if third.Header().Get("Retry-After") == "" || third.Header().Get("Cache-Control") != "" {
120 t.Errorf("429 headers: Retry-After %q, Cache-Control %q", third.Header().Get("Retry-After"), third.Header().Get("Cache-Control"))
121 }
122
123 for _, target := range []string{"/stylesheet", "/favicon.ico", "/media/emojitar/alice?type=a", "/robots.txt"} {
124 if rec := from(target); rec.Code != 200 {
125 t.Errorf("%s: status %d while limited, want 200 (exempt)", target, rec.Code)
126 }
127 }
128}
129
130func TestHandlerServesRobotsTXT(t *testing.T) {
131 uri := CFG.URI
132 CFG.URI = "/"
133 defer func() { CFG.URI = uri }()
134
135 rec := serve(t, "/robots.txt")
136 if rec.Code != 200 || !strings.HasPrefix(rec.Body.String(), "User-agent: *\n") {
137 t.Errorf("robots.txt: status %d body %q", rec.Code, rec.Body.String())
138 }
139 if ct := rec.Header().Get("Content-Type"); !strings.HasPrefix(ct, "text/plain") {
140 t.Errorf("Content-Type %q, want text/plain", ct)
141 }
142}
app/router.go +10
@@ -137,6 +137,12 @@ func Handler() http.HandlerFunc {
137 w.Header().Add("X-Frame-Options", "DENY") 137 w.Header().Add("X-Frame-Options", "DENY")
138 w.Header().Set("Cache-Control", cacheControlPage) 138 w.Header().Set("Cache-Control", cacheControlPage)
139 139
140 if daLimiter != nil && limited(skunky.Endpoint) && !daLimiter.allow(clientAddr(r)) {
141 w.Header().Set("Retry-After", "60")
142 skunky.ReturnHTTPError(http.StatusTooManyRequests)
143 return
144 }
145
140 switch skunky.Endpoint { 146 switch skunky.Endpoint {
141 // main 147 // main
142 case "": 148 case "":
@@ -182,6 +188,10 @@ func Handler() http.HandlerFunc {
182 case "favicon.ico": 188 case "favicon.ico":
183 w.Header().Set("Cache-Control", cacheControlAssets) 189 w.Header().Set("Cache-Control", cacheControlAssets)
184 _, _ = w.Write(open("images/logo.png")) 190 _, _ = w.Write(open("images/logo.png"))
191 case "robots.txt":
192 w.Header().Set("Cache-Control", cacheControlAssets)
193 w.Header().Set("Content-Type", "text/plain; charset=utf-8")
194 wr(w, robotsTXT(CFG.URI))
185 195
186 // API 196 // API
187 case "api": 197 case "api":
config.example.json +4
@@ -14,6 +14,10 @@
14 "max-size": 64, 14 "max-size": 64,
15 "ttl": "5i" 15 "ttl": "5i"
16 }, 16 },
17 "rate-limit": {
18 "per-minute": 60,
19 "burst": 20
20 },
17 "static-path": "static", 21 "static-path": "static",
18 "download-proxy": "", 22 "download-proxy": "",
19 "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.36", 23 "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.36",