Commit `5b2c8a11b2`

5b2c8a11b2195cecc635ba205c7f001452d842c7

parent: ec6fbe95a9

Verified · cmc ci/build: success ci/lint: success ci/test: success

cmc <hello@cleberg.net> · 2026-09-11 19:04 UTC

Make the upstream throttle configurable

upstream.min-interval-ms (default 400) and upstream.max-concurrent
(default 2) replace the source constants, so an instance that
DeviantArt bans can slow its own requests down without a rebuild.

Layout: unified · split

SETUP.md +6

		@@ -50,6 +50,12 @@ below apply. A file named with `-c` must exist.
50	50	`0` turns the limit off.
51	51	* `burst` — How many requests a client can make at once before the rate
52	52	applies, default 20.
	53	* `upstream` — How fast the instance itself talks to DeviantArt, whichever
	54	client asked. DeviantArt bans an address that asks too often, so an
	55	instance that gets banned should slow this down before anything else.
	56	* `min-interval-ms` — Minimum gap between two requests to DeviantArt, in
	57	milliseconds. Default 400.
	58	* `max-concurrent` — Requests to DeviantArt in flight at once. Default 2.
53	59	* `static-path` — Directory of templates, styles and catalogues, read into
54	60	memory at startup. Default `static`. Ignored by a binary built with the
55	61	`embed` tag.

app/config.go +16

		@@ -40,6 +40,12 @@ type rateLimitConfig struct {
40	40	Burst int `json:"burst"`
41	41	}
42	42
	43	// upstreamConfig paces the instance's own requests to DeviantArt.
	44	type upstreamConfig struct {
	45	MinIntervalMS int `json:"min-interval-ms"`
	46	MaxConcurrent int `json:"max-concurrent"`
	47	}
	48
43	49	type config struct {
44	50	cfg string
45	51	Listen string `json:"listen"`
		@@ -47,6 +53,7 @@ type config struct {
47	53	Cache cacheConfig `json:"cache"`
48	54	APICache apiCacheConfig `json:"api-cache"`
49	55	RateLimit rateLimitConfig `json:"rate-limit"`
	56	Upstream upstreamConfig `json:"upstream"`
50	57	Proxy bool `json:"proxy"`
51	58	Nsfw bool `json:"nsfw"`
52	59	HideAI bool `json:"hide-ai"`
		@@ -82,6 +89,10 @@ var CFG = config{
82	89	PerMinute: 60,
83	90	Burst: 20,
84	91	},
	92	Upstream: upstreamConfig{
	93	MinIntervalMS: 400,
	94	MaxConcurrent: 2,
	95	},
85	96	StaticPath: "static",
86	97	UserAgent: "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.36",
87	98	Proxy: true,
		@@ -234,6 +245,11 @@ func ExecuteConfig() {
234	245	daLimiter = newRateLimiter(CFG.RateLimit.PerMinute, max(CFG.RateLimit.Burst, 1))
235	246	}
236	247
	248	// Read here so InstallDAThrottle, which runs next, builds the throttle
	249	// from the file rather than the source defaults.
	250	daMinInterval = time.Duration(max(CFG.Upstream.MinIntervalMS, 0)) * time.Millisecond
	251	daMaxConcurrent = max(CFG.Upstream.MaxConcurrent, 1)
	252
237	253	static.StaticPath = CFG.StaticPath
238	254	devianter.UserAgent = CFG.UserAgent
239	255	}

app/config_test.go +27

		@@ -1,6 +1,7 @@
1	1	package app
2	2
3	3	import (
	4	"os"
4	5	"testing"
5	6	"time"
6	7	)
		@@ -80,3 +81,29 @@ func TestExecuteConfigExitsOnAMissingExplicitFile(t *testing.T) {
80	81	t.Error("a missing file named with -c started the instance, want an exit")
81	82	}
82	83	}
	84
	85	func TestUpstreamDefaults(t *testing.T) {
	86	if CFG.Upstream.MinIntervalMS != 400 \|\| CFG.Upstream.MaxConcurrent != 2 {
	87	t.Errorf("defaults are %+v, want 400 ms and 2 in flight", CFG.Upstream)
	88	}
	89	}
	90
	91	// TestUpstreamConfigSetsTheThrottle pins that the file's values reach the
	92	// throttle: the tunables used to be source constants.
	93	func TestUpstreamConfigSetsTheThrottle(t *testing.T) {
	94	dir := t.TempDir()
	95	path := dir + "/config.json"
	96	if err := os.WriteFile(path, []byte(`{"upstream": {"min-interval-ms": 1500, "max-concurrent": 1}}`), 0600); err != nil {
	97	t.Fatal(err)
	98	}
	99	withScratchConfig(t, path, true)
	100	interval, concurrent := daMinInterval, daMaxConcurrent
	101	t.Cleanup(func() { daMinInterval, daMaxConcurrent = interval, concurrent })
	102	captureExit(t)
	103
	104	ExecuteConfig()
	105
	106	if daMinInterval != 1500*time.Millisecond \|\| daMaxConcurrent != 1 {
	107	t.Errorf("throttle is %v / %d, want 1.5s / 1 from the file", daMinInterval, daMaxConcurrent)
	108	}
	109	}

app/httpclient.go +3 −1

		@@ -18,7 +18,9 @@ import (
18	18	//
19	19	// http.ProxyFromEnvironment is preserved, so HTTPS_PROXY (VPN egress) still applies.
20	20
21		// Tunables (kept in source; safe defaults). Lower is gentler on the DA IP.
	21	// Tunables, set from the upstream config block by ExecuteConfig; these are
	22	// the defaults for a config that omits it. Slower is gentler on the egress
	23	// address, which DeviantArt bans when it asks too often.
22	24	var (
23	25	daMinInterval = 400 * time.Millisecond // minimum gap between DA request starts
24	26	daMaxConcurrent = 2 // max simultaneous in-flight DA requests

config.example.json +4

		@@ -18,6 +18,10 @@
18	18	"per-minute": 60,
19	19	"burst": 20
20	20	},
	21	"upstream": {
	22	"min-interval-ms": 400,
	23	"max-concurrent": 2
	24	},
21	25	"static-path": "static",
22	26	"download-proxy": "",
23	27	"user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.36",

Commit 5b2c8a11b2

Commit `5b2c8a11b2`