Commit `4261b937e4`

4261b937e4b0e550d83fd57dabb7a21e6ba69d46

parent: 1b89e1fe37

Verified · cmc ci/build: success ci/lint: success ci/test: success

cmc <hello@cleberg.net> · 2026-09-11 02:52 UTC

Add robots.txt and a per-client rate limit

Page, feed and API requests draw on a token bucket per client address;
media, avatars and static files do not. Over budget answers 429 with
Retry-After. Behind a reverse proxy the client is the rightmost
X-Forwarded-For entry, trusted only when the connection itself is from
a loopback or private address. rate-limit.per-minute (default 60) and
burst (default 20); 0 turns it off.

/robots.txt disallows search, the API, user and group pages, media and
paginated URLs, with a 10 second crawl delay.

Closes #10

Layout: unified · split

SETUP.md +14

		@@ -34,6 +34,20 @@ Time units:
34	34	recently used entries are dropped past this.
35	35	* `ttl` — how long a response is reused, in the time units above. Default
36	36	`5i`.
	37	* `rate-limit` — Per-client budget for page, feed and API requests, so one
	38	crawler cannot spend the whole upstream budget. Media, avatars and static
	39	files are not counted. Over budget answers 429 with `Retry-After`.
	40	Behind a reverse proxy the client is taken from the rightmost
	41	`X-Forwarded-For` entry, but only when the connection itself comes from a
	42	loopback or private address; a direct client's header is ignored.
	43	* `per-minute` — sustained requests per minute per client, default 60.
	44	`0` turns the limit off.
	45	* `burst` — how many requests a client can make at once before the rate
	46	applies, default 20.
	47
	48	`/robots.txt` is served automatically. It disallows search, the API, user
	49	and group pages, media and paginated URLs, and asks for a 10 second crawl
	50	delay; the index, daily deviations and posts stay crawlable.
37	51	* `static-path` — This setting determines path to static, which will be copied to RAM when SkunkyArt is started. Useless if you're use binary compiled with 'embed' tag.
38	52	* `download-proxy` — Outbound proxy used when fetching media from DeviantArt's
39	53	CDN. Leave empty (`""`) unless you actually run a proxy: if this points at

app/config.go +28 −12

		@@ -34,20 +34,26 @@ type apiCacheConfig struct {
34	34	TTL string `json:"ttl"`
35	35	}
36	36
	37	type rateLimitConfig struct {
	38	PerMinute int `json:"per-minute"`
	39	Burst int `json:"burst"`
	40	}
	41
37	42	type config struct {
38	43	cfg string
39		Listen string `json:"listen"`
40		URI string `json:"uri"`
41		Cache cacheConfig `json:"cache"`
42		APICache apiCacheConfig `json:"api-cache"`
43		Proxy bool `json:"proxy"`
44		Nsfw bool `json:"nsfw"`
45		HideAI bool `json:"hide-ai"`
46		Theme string `json:"theme"`
47		Language string `json:"language"`
48		UserAgent string `json:"user-agent"`
49		DownloadProxy string `json:"download-proxy"`
50		StaticPath string `json:"static-path"`
	44	Listen string `json:"listen"`
	45	URI string `json:"uri"`
	46	Cache cacheConfig `json:"cache"`
	47	APICache apiCacheConfig `json:"api-cache"`
	48	RateLimit rateLimitConfig `json:"rate-limit"`
	49	Proxy bool `json:"proxy"`
	50	Nsfw bool `json:"nsfw"`
	51	HideAI bool `json:"hide-ai"`
	52	Theme string `json:"theme"`
	53	Language string `json:"language"`
	54	UserAgent string `json:"user-agent"`
	55	DownloadProxy string `json:"download-proxy"`
	56	StaticPath string `json:"static-path"`
51	57	}
52	58
53	59	// CFG is the running instance's configuration, holding the defaults below until
		@@ -68,6 +74,10 @@ var CFG = config{
68	74	MaxSize: 64,
69	75	TTL: "5i",
70	76	},
	77	RateLimit: rateLimitConfig{
	78	PerMinute: 60,
	79	Burst: 20,
	80	},
71	81	StaticPath: "static",
72	82	UserAgent: "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.36",
73	83	Proxy: true,
		@@ -192,6 +202,12 @@ func ExecuteConfig() {
192	202	apiCacheTTL = d
193	203	}
194	204
	205	// per-minute 0 turns the limit off; a burst below one token would
	206	// refuse every request, so it is floored to one.
	207	if CFG.RateLimit.PerMinute > 0 {
	208	daLimiter = newRateLimiter(CFG.RateLimit.PerMinute, max(CFG.RateLimit.Burst, 1))
	209	}
	210
195	211	static.StaticPath = CFG.StaticPath
196	212	devianter.UserAgent = CFG.UserAgent
197	213	}

app/ratelimit.go added +132

		@@ -0,0 +1,132 @@
	1	package app
	2
	3	import (
	4	"net"
	5	"net/http"
	6	"strings"
	7	"sync"
	8	"time"
	9	)
	10
	11	// rateLimiter is a token bucket per client address. It bounds how many page,
	12	// feed and API requests one client can make, so a single crawler cannot spend
	13	// the whole upstream budget and turn the DeviantArt throttle into latency for
	14	// everyone else.
	15	type rateLimiter struct {
	16	perSecond float64
	17	burst float64
	18	now func() time.Time
	19
	20	mu sync.Mutex
	21	buckets map[string]*bucket
	22	inserts int
	23	}
	24
	25	type bucket struct {
	26	tokens float64
	27	last time.Time
	28	}
	29
	30	// bucketIdle is how long a client can go unseen before its bucket is dropped.
	31	const bucketIdle = 10 * time.Minute
	32
	33	func newRateLimiter(perMinute, burst int) *rateLimiter {
	34	return &rateLimiter{
	35	perSecond: float64(perMinute) / 60,
	36	burst: float64(burst),
	37	now: time.Now,
	38	buckets: map[string]*bucket{},
	39	}
	40	}
	41
	42	// allow reports whether client may make a request now, spending one token if
	43	// so. A client's first request finds a full bucket.
	44	func (l *rateLimiter) allow(client string) bool {
	45	l.mu.Lock()
	46	defer l.mu.Unlock()
	47
	48	now := l.now()
	49	b := l.buckets[client]
	50	if b == nil {
	51	b = &bucket{tokens: l.burst, last: now}
	52	l.buckets[client] = b
	53	l.inserts++
	54	if l.inserts%1000 == 0 {
	55	l.prune(now)
	56	}
	57	} else {
	58	b.tokens = min(l.burst, b.tokens+now.Sub(b.last).Seconds()*l.perSecond)
	59	b.last = now
	60	}
	61
	62	if b.tokens < 1 {
	63	return false
	64	}
	65	b.tokens--
	66	return true
	67	}
	68
	69	// prune drops buckets idle past bucketIdle. The caller holds mu.
	70	func (l *rateLimiter) prune(now time.Time) {
	71	for client, b := range l.buckets {
	72	if now.Sub(b.last) > bucketIdle {
	73	delete(l.buckets, client)
	74	}
	75	}
	76	}
	77
	78	// clientAddr is the address a request is limited by. Behind a reverse proxy
	79	// every connection arrives from the proxy, so when the connection is from a
	80	// loopback or private address the client is the rightmost X-Forwarded-For
	81	// entry, which is the one that proxy appended. A direct client's connection is
	82	// not from a private address, so its own header is never trusted.
	83	func clientAddr(r *http.Request) string {
	84	host, _, err := net.SplitHostPort(r.RemoteAddr)
	85	if err != nil {
	86	host = r.RemoteAddr
	87	}
	88	ip := net.ParseIP(host)
	89	if ip == nil \|\| (!ip.IsLoopback() && !ip.IsPrivate()) {
	90	return host
	91	}
	92	xff := r.Header.Get("X-Forwarded-For")
	93	if xff == "" {
	94	return host
	95	}
	96	parts := strings.Split(xff, ",")
	97	if forwarded := strings.TrimSpace(parts[len(parts)-1]); forwarded != "" {
	98	return forwarded
	99	}
	100	return host
	101	}
	102
	103	// limited reports whether an endpoint counts against the client's budget.
	104	// Media, avatars and static assets do not: one listing page loads twenty
	105	// thumbnails, which would exhaust any sensible page budget.
	106	func limited(endpoint string) bool {
	107	switch endpoint {
	108	case "media", "stylesheet", "favicon.ico", "robots.txt":
	109	return false
	110	}
	111	return true
	112	}
	113
	114	// robotsTXT keeps crawlers off the pages that fan out into unbounded upstream
	115	// requests. The index, daily deviations and posts stay allowed, so the instance
	116	// is still discoverable.
	117	func robotsTXT(base string) string {
	118	var b strings.Builder
	119	b.WriteString("User-agent: *\n")
	120	for _, p := range []string{"search", "api", "group_user", "media", "*?p="} {
	121	b.WriteString("Disallow: ")
	122	b.WriteString(base)
	123	b.WriteString(p)
	124	b.WriteString("\n")
	125	}
	126	b.WriteString("Crawl-delay: 10\n")
	127	return b.String()
	128	}
	129
	130	// daLimiter is the running instance's limiter, or nil when rate-limit.per-minute
	131	// is 0. Set by ExecuteConfig.
	132	var daLimiter *rateLimiter

app/ratelimit_test.go added +142

		@@ -0,0 +1,142 @@
	1	package app
	2
	3	import (
	4	"net/http"
	5	"net/http/httptest"
	6	"strings"
	7	"testing"
	8	"time"
	9	)
	10
	11	func TestRateLimiterAllowsBurstThenRefuses(t *testing.T) {
	12	l := newRateLimiter(60, 3)
	13	now := time.Now()
	14	l.now = func() time.Time { return now }
	15
	16	for i := range 3 {
	17	if !l.allow("c") {
	18	t.Fatalf("request %d refused inside the burst", i+1)
	19	}
	20	}
	21	if l.allow("c") {
	22	t.Error("request past the burst allowed")
	23	}
	24	if !l.allow("other") {
	25	t.Error("a different client was refused by the first one's budget")
	26	}
	27	}
	28
	29	func TestRateLimiterRefillsAtTheConfiguredRate(t *testing.T) {
	30	l := newRateLimiter(60, 1) // one token per second
	31	now := time.Now()
	32	l.now = func() time.Time { return now }
	33
	34	l.allow("c")
	35	if l.allow("c") {
	36	t.Fatal("second immediate request allowed with burst 1")
	37	}
	38	now = now.Add(500 * time.Millisecond)
	39	if l.allow("c") {
	40	t.Error("allowed after half a second, want a full second per token")
	41	}
	42	now = now.Add(600 * time.Millisecond)
	43	if !l.allow("c") {
	44	t.Error("refused after more than a second, want one token refilled")
	45	}
	46	}
	47
	48	func TestRateLimiterPrunesIdleBuckets(t *testing.T) {
	49	l := newRateLimiter(60, 1)
	50	now := time.Now()
	51	l.now = func() time.Time { return now }
	52
	53	l.allow("old")
	54	now = now.Add(bucketIdle + time.Minute)
	55	l.prune(now)
	56
	57	if _, ok := l.buckets["old"]; ok {
	58	t.Error("idle bucket survived a prune")
	59	}
	60	}
	61
	62	func TestClientAddrTrustsForwardedForOnlyBehindAProxy(t *testing.T) {
	63	cases := []struct{ remote, xff, want string }{
	64	{"127.0.0.1:1234", "203.0.113.5", "203.0.113.5"},
	65	{"10.0.0.2:1234", "198.51.100.7, 203.0.113.5", "203.0.113.5"},
	66	{"127.0.0.1:1234", "", "127.0.0.1"},
	67	{"203.0.113.9:1234", "198.51.100.7", "203.0.113.9"},
	68	{"[::1]:1234", "203.0.113.5", "203.0.113.5"},
	69	}
	70	for _, c := range cases {
	71	r := httptest.NewRequest(http.MethodGet, "/", nil)
	72	r.RemoteAddr = c.remote
	73	if c.xff != "" {
	74	r.Header.Set("X-Forwarded-For", c.xff)
	75	}
	76	if got := clientAddr(r); got != c.want {
	77	t.Errorf("remote %s xff %q: got %s, want %s", c.remote, c.xff, got, c.want)
	78	}
	79	}
	80	}
	81
	82	func TestRobotsTXTCarriesTheBaseURI(t *testing.T) {
	83	out := robotsTXT("/art/")
	84	for _, want := range []string{"Disallow: /art/search\n", "Disallow: /art/api\n", "Disallow: /art/*?p=\n", "Crawl-delay: 10\n"} {
	85	if !strings.Contains(out, want) {
	86	t.Errorf("robots.txt lacks %q:\n%s", want, out)
	87	}
	88	}
	89	}
	90
	91	// TestHandlerLimitsPagesButNotMedia drives the real handler: a client past its
	92	// budget gets 429 on a page with Retry-After and no Cache-Control, while its
	93	// stylesheet, favicon and avatar requests are never counted.
	94	func TestHandlerLimitsPagesButNotMedia(t *testing.T) {
	95	limiter, uri := daLimiter, CFG.URI
	96	daLimiter = newRateLimiter(60, 2)
	97	CFG.URI = "/"
	98	defer func() { daLimiter, CFG.URI = limiter, uri }()
	99
	100	orig := fetchAvatar
	101	fetchAvatar = func(string, rune) (string, error) { return "png", nil }
	102	defer func() { fetchAvatar = orig }()
	103
	104	from := func(target string) *httptest.ResponseRecorder {
	105	loadTemplates()
	106	rec := httptest.NewRecorder()
	107	req := httptest.NewRequest(http.MethodGet, target, nil)
	108	req.RemoteAddr = "203.0.113.5:4000"
	109	Handler()(rec, req)
	110	return rec
	111	}
	112
	113	from("/about")
	114	from("/about")
	115	third := from("/about")
	116	if third.Code != 429 {
	117	t.Fatalf("third page request: status %d, want 429", third.Code)
	118	}
	119	if third.Header().Get("Retry-After") == "" \|\| third.Header().Get("Cache-Control") != "" {
	120	t.Errorf("429 headers: Retry-After %q, Cache-Control %q", third.Header().Get("Retry-After"), third.Header().Get("Cache-Control"))
	121	}
	122
	123	for _, target := range []string{"/stylesheet", "/favicon.ico", "/media/emojitar/alice?type=a", "/robots.txt"} {
	124	if rec := from(target); rec.Code != 200 {
	125	t.Errorf("%s: status %d while limited, want 200 (exempt)", target, rec.Code)
	126	}
	127	}
	128	}
	129
	130	func TestHandlerServesRobotsTXT(t *testing.T) {
	131	uri := CFG.URI
	132	CFG.URI = "/"
	133	defer func() { CFG.URI = uri }()
	134
	135	rec := serve(t, "/robots.txt")
	136	if rec.Code != 200 \|\| !strings.HasPrefix(rec.Body.String(), "User-agent: *\n") {
	137	t.Errorf("robots.txt: status %d body %q", rec.Code, rec.Body.String())
	138	}
	139	if ct := rec.Header().Get("Content-Type"); !strings.HasPrefix(ct, "text/plain") {
	140	t.Errorf("Content-Type %q, want text/plain", ct)
	141	}
	142	}

app/router.go +10

		@@ -137,6 +137,12 @@ func Handler() http.HandlerFunc {
137	137	w.Header().Add("X-Frame-Options", "DENY")
138	138	w.Header().Set("Cache-Control", cacheControlPage)
139	139
	140	if daLimiter != nil && limited(skunky.Endpoint) && !daLimiter.allow(clientAddr(r)) {
	141	w.Header().Set("Retry-After", "60")
	142	skunky.ReturnHTTPError(http.StatusTooManyRequests)
	143	return
	144	}
	145
140	146	switch skunky.Endpoint {
141	147	// main
142	148	case "":
		@@ -182,6 +188,10 @@ func Handler() http.HandlerFunc {
182	188	case "favicon.ico":
183	189	w.Header().Set("Cache-Control", cacheControlAssets)
184	190	_, _ = w.Write(open("images/logo.png"))
	191	case "robots.txt":
	192	w.Header().Set("Cache-Control", cacheControlAssets)
	193	w.Header().Set("Content-Type", "text/plain; charset=utf-8")
	194	wr(w, robotsTXT(CFG.URI))
185	195
186	196	// API
187	197	case "api":

config.example.json +4

		@@ -14,6 +14,10 @@
14	14	"max-size": 64,
15	15	"ttl": "5i"
16	16	},
	17	"rate-limit": {
	18	"per-minute": 60,
	19	"burst": 20
	20	},
17	21	"static-path": "static",
18	22	"download-proxy": "",
19	23	"user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.36",

Commit 4261b937e4

Commit `4261b937e4`