Proxy random media with the token in the query, and bootstrap before listening !28

merged merged by cmc on 2026-09-11 18:57 UTC · krz/skunky-art:fix/random-media into main

11 files changed, +121 −282

Layout: unified · split

SETUP.md −5

     recently used entries are dropped past this.
   * `ttl` — How long a response is reused, in the time units below. Default
     `5i`.
-  * `stale` — How long past `ttl` a response is kept to be served when
-    DeviantArt fails or blocks the instance, so a short ban does not take
-    the daily deviations, popular searches and feeds down. Default `1h`;
-    `0i` keeps nothing past `ttl`. After a block the instance also leaves
-    DeviantArt alone for a minute instead of retrying every request.
 * `rate-limit` — Per-client budget for page, feed and API requests, so one
   crawler cannot spend the whole upstream budget. Media, avatars and static
   files are not counted. Over budget answers 429 with `Retry-After`.

app/api.go +15 −6

 import (
 	"encoding/json"
 	"math/rand"
+	"net/url"
 	"strings"
 	"github.com/krazywarez/devianter"
 		return
 	}
-	if CFG.Proxy {
+	if !CFG.Proxy {
-		mediaURL = mediaURL[21:]
-		dot := strings.Index(mediaURL, ".")
-		a.main.Writer.Header().Del("Content-Type")
-		a.main.DownloadAndSendMedia(mediaURL[:dot], mediaURL[dot+11:])
-	} else {
 		a.main.Writer.Header().Add("Location", mediaURL)
 		a.main.Writer.WriteHeader(302)
+		return
+	}
+	// Parsed, not sliced: the signing token has to reach wixmp as a query
+	// parameter. Passing the raw tail as the path put "?token=..." inside
+	// the path, which wixmp answers with 401.
+	u, err := url.Parse(mediaURL)
+	if err != nil {
+		a.Error("bad media url", 502)
+		return
 	}
+	subdomain := strings.TrimSuffix(strings.TrimPrefix(u.Host, "images-wixmp-"), ".wixmp.com")
+	a.main.Writer.Header().Del("Content-Type")
+	a.main.downloadAndSendMedia(subdomain, strings.TrimPrefix(u.Path, "/"), u.Query().Get("token"))
 }
 // fetchDailyDeviations is devianter.GetDailyDeviations behind a variable so

app/api_test.go +37

	@@ -1,7 +1,10 @@
1	package app	1	package app
2		2
3	import (	3	import (
		4	"net/http"
4	"net/http/httptest"	5	"net/http/httptest"
		6	"net/url"
		7	"strings"
5	"testing"	8	"testing"
6		9
7	"github.com/krazywarez/devianter"	10	"github.com/krazywarez/devianter"
	@@ -108,3 +111,37 @@ func TestRandomHonoursNSFW(t *testing.T) {
108	t.Errorf("status %d, Location %q; want 404 and no media", w.Code, w.Header().Get("Location"))	111	t.Errorf("status %d, Location %q; want 404 and no media", w.Code, w.Header().Get("Location"))
109	}	112	}
110	}	113	}
		114
		115	// TestSendMediaProxiesWithTheTokenInTheQuery is the regression test for
		116	// /api/random answering 401 with proxying on: the signing token was passed
		117	// inside the path, so wixmp never saw it as a parameter.
		118	func TestSendMediaProxiesWithTheTokenInTheQuery(t *testing.T) {
		119	proxy, cache := CFG.Proxy, CFG.Cache.Enabled
		120	CFG.Proxy, CFG.Cache.Enabled = true, false
		121	defer func() { CFG.Proxy, CFG.Cache.Enabled = proxy, cache }()
		122
		123	var fetched string
		124	orig := fetchMedia
		125	fetchMedia = func(u string) Downloaded {
		126	fetched = u
		127	return Downloaded{Status: 200, Body: []byte("png"), Headers: http.Header{"Content-Type": {"image/png"}}}
		128	}
		129	defer func() { fetchMedia = orig }()
		130
		131	d := fullviewDeviation()
		132	d.Media.Token = []string{"tok.en.sig"}
		133
		134	w := httptest.NewRecorder()
		135	API{main: &skunkyart{Writer: w, Args: url.Values{}}}.sendMedia(d)
		136
		137	u, err := url.Parse(fetched)
		138	if err != nil \|\| u.Host != "images-wixmp-abc.wixmp.com" {
		139	t.Fatalf("fetched %q, want a wixmp URL on the deviation's subdomain", fetched)
		140	}
		141	if strings.Contains(u.Path, "token") \|\| u.Query().Get("token") == "" {
		142	t.Errorf("token not passed as a query parameter: path %q query %q", u.Path, u.RawQuery)
		143	}
		144	if w.Code != 200 \|\| w.Body.String() != "png" {
		145	t.Errorf("response %d %q, want the proxied image", w.Code, w.Body.String())
		146	}
		147	}

app/apicache.go +28 −98

 type apiCache struct {
 	maxBytes int64
 	ttl      time.Duration
-	// stale is how long past ttl an entry is kept to be served when upstream
+	now      func() time.Time
-	// is blocked or unreachable. Zero keeps nothing past ttl.
-	stale time.Duration
-	now   func() time.Time
-	mu        sync.Mutex
+	mu      sync.Mutex
-	entries   map[string]*cacheEntry
+	entries map[string]*cacheEntry
-	lru       *list.List // front is most recently used
+	lru     *list.List // front is most recently used
-	held      int64
+	held    int64
-	hits      int64
+	hits    int64
-	misses    int64
+	misses  int64
-	staleHits int64
-	// blockedUntil is set when DeviantArt answers with a block. Until then a
-	// miss with nothing stale to serve gets blockResp back without an
-	// upstream call, so a banned instance stops hammering the WAF.
-	blockedUntil time.Time
-	blockResp    *cacheEntry
 	flight singleflight.Group
 }
-// blockBackoff is how long upstream is left alone after a block response.
-const blockBackoff = time.Minute
 // cacheEntry is one buffered response. header is a clone of the upstream
 // header; body is the whole body, read once.
 type cacheEntry struct {
 	elem    *list.Element
 }
-func newAPICache(maxBytes int64, ttl, stale time.Duration) *apiCache {
+func newAPICache(maxBytes int64, ttl time.Duration) *apiCache {
 	return &apiCache{
 		maxBytes: maxBytes,
 		ttl:      ttl,
-		stale:    stale,
 		now:      time.Now,
 		entries:  map[string]*cacheEntry{},
 		lru:      list.New(),
 	base  http.RoundTripper
 }
-// RoundTrip serves a fresh hit from memory. A miss is fetched once per key
+// RoundTrip serves a hit from memory. A miss is fetched once per key however
-// however many callers are waiting, buffered, stored if it is a 200, and
+// many callers are waiting, buffered, stored if it is a 200, and handed to
-// handed to every waiter as its own response.
+// every waiter as its own response.
-//
-// When upstream fails or answers with a block, a stale entry is served
-// instead if one is still held, so a short ban does not take the popular
-// pages down. A block also starts a backoff during which misses with nothing
-// stale get the block response back without an upstream call.
 func (t *cachedTransport) RoundTrip(req *http.Request) (*http.Response, error) {
 	if !cacheable(req) {
 		return t.base.RoundTrip(req)
 	}
 	key := cacheKey(req)
-	old, fresh := t.cache.get(key)
+	if e := t.cache.get(key); e != nil {
-	if fresh {
+		return e.response(req), nil
-		return old.response(req), nil
-	}
-	if blocked := t.cache.blockedResponse(); blocked != nil {
-		if old != nil {
-			t.cache.countStale()
-			return old.response(req), nil
-		}
-		return blocked.response(req), nil
 	}
 	v, err, _ := t.cache.flight.Do(key, func() (any, error) {
 			return nil, err
 		}
 		e := &cacheEntry{key: key, status: resp.StatusCode, header: resp.Header.Clone(), body: body}
-		switch e.status {
+		if e.status == http.StatusOK {
-		case http.StatusOK:
 			t.cache.put(e)
-		case http.StatusForbidden, http.StatusTooManyRequests:
-			t.cache.block(e)
 		}
 		return e, nil
 	})
 	if err != nil {
-		if old != nil {
-			t.cache.countStale()
-			return old.response(req), nil
-		}
 		return nil, err
 	}
 	e, ok := v.(*cacheEntry)
 	if !ok {
 		return nil, io.ErrUnexpectedEOF
 	}
-	if e.status != http.StatusOK && old != nil {
-		t.cache.countStale()
-		return old.response(req), nil
-	}
 	return e.response(req), nil
 }
-// block records a block response and starts the backoff.
-func (c *apiCache) block(e *cacheEntry) {
-	c.mu.Lock()
-	defer c.mu.Unlock()
-	c.blockedUntil = c.now().Add(blockBackoff)
-	c.blockResp = e
-}
-// blockedResponse returns the last block response while the backoff runs,
-// or nil once it is over.
-func (c *apiCache) blockedResponse() *cacheEntry {
-	c.mu.Lock()
-	defer c.mu.Unlock()
-	if c.blockResp != nil && c.now().Before(c.blockedUntil) {
-		return c.blockResp
-	}
-	return nil
-}
-func (c *apiCache) countStale() {
-	c.mu.Lock()
-	c.staleHits++
-	c.mu.Unlock()
-}
 // response builds a fresh http.Response over the buffered body, so each
 // caller can read and close its own.
 func (e *cacheEntry) response(req *http.Request) *http.Response {
 	}
 }
-// get returns the entry for key and whether it is still fresh. A fresh hit is
+// get returns the live entry for key, marking it most recently used, or nil.
-// marked most recently used. An entry past ttl but within the stale window is
+// An expired entry is dropped on the way out.
-// returned as not fresh, for the caller to fall back on; one past the stale
+func (c *apiCache) get(key string) *cacheEntry {
-// window is dropped.
-func (c *apiCache) get(key string) (*cacheEntry, bool) {
 	c.mu.Lock()
 	defer c.mu.Unlock()
 	e := c.entries[key]
 	if e == nil {
 		c.misses++
-		return nil, false
+		return nil
 	}
-	now := c.now()
+	if !c.now().Before(e.expires) {
-	if now.Before(e.expires) {
+		c.remove(e)
-		c.lru.MoveToFront(e.elem)
+		c.misses++
-		c.hits++
+		return nil
-		return e, true
-	}
-	c.misses++
-	if now.Before(e.expires.Add(c.stale)) {
-		return e, false
 	}
-	c.remove(e)
+	c.lru.MoveToFront(e.elem)
-	return nil, false
+	c.hits++
+	return e
 }
 // put stores e, evicting from the least recently used end until it fits. A
 	c.held -= int64(len(e.body))
 }
-// stats reports the counters for the hourly log line. stale counts the
+// stats reports the counters for the hourly log line.
-// misses that were answered from an expired entry because upstream failed.
+func (c *apiCache) stats() (hits, misses int64, entries int, held int64) {
-func (c *apiCache) stats() (hits, misses, stale int64, entries int, held int64) {
 	c.mu.Lock()
 	defer c.mu.Unlock()
-	return c.hits, c.misses, c.staleHits, len(c.entries), c.held
+	return c.hits, c.misses, len(c.entries), c.held
 }

app/apicache_stale_test.go deleted −132

	@@ -1,132 +0,0 @@
1	package app
2
3	import (
4	"errors"
5	"net/http"
6	"testing"
7	"time"
8	)
9
10	// failingRT is an upstream that can be switched between a scripted status and
11	// a transport error mid-test.
12	type failingRT struct {
13	fakeRT
14	err error
15	}
16
17	func (f failingRT) RoundTrip(r http.Request) (*http.Response, error) {
18	f.mu.Lock()
19	err := f.err
20	f.mu.Unlock()
21	if err != nil {
22	f.mu.Lock()
23	f.calls++
24	f.mu.Unlock()
25	return nil, err
26	}
27	return f.fakeRT.RoundTrip(r)
28	}
29
30	func (f *failingRT) set(status int, body string, err error) {
31	f.mu.Lock()
32	defer f.mu.Unlock()
33	f.status, f.body, f.err = status, body, err
34	}
35
36	// staleCache returns a cache with a one minute TTL and one hour stale window
37	// over a controllable clock, warmed with one good response for puppyURL.
38	func staleCache(t testing.T) (apiCache, failingRT, http.RoundTripper, time.Time) {
39	t.Helper()
40	up := &failingRT{}
41	up.set(200, `{"good":1}`, nil)
42	c := newAPICache(1<<20, time.Minute, time.Hour)
43	now := time.Now()
44	c.now = func() time.Time { return now }
45	rt := c.transport(up)
46	get(t, rt, puppyURL)
47	return c, up, rt, &now
48	}
49
50	func TestStaleEntryServedWhenUpstreamBlocks(t *testing.T) {
51	c, up, rt, now := staleCache(t)
52	now = now.Add(2 time.Minute) // past ttl, inside the stale window
53	up.set(403, "<html>blocked</html>", nil)
54
55	status, body := get(t, rt, puppyURL)
56
57	if status != 200 \|\| body != `{"good":1}` {
58	t.Errorf("got %d %q, want the stale 200 body", status, body)
59	}
60	if up.count() != 2 {
61	t.Errorf("upstream called %d times, want 2: one warm-up and one attempt that hit the block", up.count())
62	}
63	if _, _, stale, _, _ := c.stats(); stale != 1 {
64	t.Errorf("stale counter is %d, want 1", stale)
65	}
66	}
67
68	func TestStaleEntryServedOnTransportError(t *testing.T) {
69	_, up, rt, now := staleCache(t)
70	now = now.Add(2 time.Minute)
71	up.set(0, "", errors.New("dial tcp: connection refused"))
72
73	status, body := get(t, rt, puppyURL)
74
75	if status != 200 \|\| body != `{"good":1}` {
76	t.Errorf("got %d %q, want the stale 200 body", status, body)
77	}
78	}
79
80	func TestBlockBackoffSkipsUpstream(t *testing.T) {
81	_, up, rt, now := staleCache(t)
82	now = now.Add(2 time.Minute)
83	up.set(403, "<html>blocked</html>", nil)
84	get(t, rt, puppyURL) // triggers the block
85	calls := up.count()
86
87	other := "https://www.deviantart.com/_puppy/dabrowse/search/all?q=other"
88	status, body := get(t, rt, other)
89	if status != 403 \|\| body != "<html>blocked</html>" {
90	t.Errorf("uncached key during backoff got %d %q, want the block response", status, body)
91	}
92	if up.count() != calls {
93	t.Errorf("upstream called during backoff (%d -> %d), want none", calls, up.count())
94	}
95
96	*now = now.Add(blockBackoff + time.Second)
97	up.set(200, `{"back":1}`, nil)
98	if status, _ := get(t, rt, other); status != 200 \|\| up.count() != calls+1 {
99	t.Errorf("after backoff: status %d, calls %d, want 200 and one more upstream call", status, up.count())
100	}
101	}
102
103	func TestStaleEntryDroppedAfterTheWindow(t *testing.T) {
104	c, up, rt, now := staleCache(t)
105	*now = now.Add(time.Minute + time.Hour + time.Second) // past ttl and stale
106	up.set(403, "<html>blocked</html>", nil)
107
108	status, _ := get(t, rt, puppyURL)
109
110	if status != 403 {
111	t.Errorf("got %d, want the 403 passed through once nothing stale is held", status)
112	}
113	if _, _, _, entries, _ := c.stats(); entries != 0 {
114	t.Errorf("%d entries held, want 0", entries)
115	}
116	}
117
118	func TestZeroStaleKeepsNothingPastTTL(t *testing.T) {
119	up := &failingRT{}
120	up.set(200, "x", nil)
121	c := newAPICache(1<<20, time.Minute, 0)
122	now := time.Now()
123	c.now = func() time.Time { return now }
124	rt := c.transport(up)
125	get(t, rt, puppyURL)
126	now = now.Add(2 * time.Minute)
127	up.set(403, "blocked", nil)
128
129	if status, _ := get(t, rt, puppyURL); status != 403 {
130	t.Errorf("got %d with stale 0, want 403: nothing may be served past ttl", status)
131	}
132	}

app/apicache_test.go +16 −18

 func TestSecondRequestIsServedFromCache(t *testing.T) {
 	up := &fakeRT{status: 200, body: `{"a":1}`}
-	rt := newAPICache(1<<20, time.Minute, 0).transport(up)
+	rt := newAPICache(1<<20, time.Minute).transport(up)
 	get(t, rt, puppyURL)
 	status, body := get(t, rt, puppyURL)
 func TestExpiredEntryIsRefetched(t *testing.T) {
 	up := &fakeRT{status: 200, body: `{}`}
-	c := newAPICache(1<<20, time.Minute, 0)
+	c := newAPICache(1<<20, time.Minute)
 	now := time.Now()
 	c.now = func() time.Time { return now }
 	rt := c.transport(up)
 	}
 }
-// A 500 here rather than a 403: a 403 is a block and starts the backoff,
-// which is covered in apicache_stale_test.go.
 func TestNon200IsNotStored(t *testing.T) {
-	up := &fakeRT{status: 500, body: "upstream broke"}
+	up := &fakeRT{status: 403, body: "blocked"}
-	rt := newAPICache(1<<20, time.Minute, 0).transport(up)
+	rt := newAPICache(1<<20, time.Minute).transport(up)
 	status, body := get(t, rt, puppyURL)
 	get(t, rt, puppyURL)
-	if status != 500 || body != "upstream broke" {
+	if status != 403 || body != "blocked" {
-		t.Errorf("first response is %d %q, want the upstream 500 passed through", status, body)
+		t.Errorf("first response is %d %q, want the upstream 403 passed through", status, body)
 	}
 	if up.count() != 2 {
-		t.Errorf("upstream called %d times, want 2: a 500 must not be cached", up.count())
+		t.Errorf("upstream called %d times, want 2: a 403 must not be cached", up.count())
 	}
 }
 func TestBypassesSessionAndOtherHosts(t *testing.T) {
 	up := &fakeRT{status: 200, body: "x"}
-	rt := newAPICache(1<<20, time.Minute, 0).transport(up)
+	rt := newAPICache(1<<20, time.Minute).transport(up)
 	for _, url := range []string{
 		"https://www.deviantart.com/_puppy",
 func TestKeyIgnoresCSRFToken(t *testing.T) {
 	up := &fakeRT{status: 200, body: "x"}
-	rt := newAPICache(1<<20, time.Minute, 0).transport(up)
+	rt := newAPICache(1<<20, time.Minute).transport(up)
 	get(t, rt, puppyURL)
 	get(t, rt, strings.Replace(puppyURL, "csrf_token=abc", "csrf_token=def", 1))
 func TestByteBoundEvictsLeastRecentlyUsed(t *testing.T) {
 	up := &fakeRT{status: 200, body: strings.Repeat("x", 100)}
-	rt := newAPICache(250, time.Minute, 0).transport(up)
+	rt := newAPICache(250, time.Minute).transport(up)
 	a := "https://www.deviantart.com/_puppy/a?p=1"
 	b := "https://www.deviantart.com/_puppy/b?p=1"
 	c := "https://www.deviantart.com/_puppy/c?p=1"
 func TestConcurrentMissesMakeOneUpstreamCall(t *testing.T) {
 	up := &fakeRT{status: 200, body: "x", delay: 50 * time.Millisecond}
-	rt := newAPICache(1<<20, time.Minute, 0).transport(up)
+	rt := newAPICache(1<<20, time.Minute).transport(up)
 	var wg sync.WaitGroup
 	for range 20 {
 func TestStatsCountHitsAndMisses(t *testing.T) {
 	up := &fakeRT{status: 200, body: "abc"}
-	c := newAPICache(1<<20, time.Minute, 0)
+	c := newAPICache(1<<20, time.Minute)
 	rt := c.transport(up)
 	get(t, rt, puppyURL)
 	get(t, rt, puppyURL)
 	get(t, rt, puppyURL)
-	hits, misses, stale, entries, held := c.stats()
+	hits, misses, entries, held := c.stats()
-	if hits != 2 || misses != 1 || stale != 0 || entries != 1 || held != 3 {
+	if hits != 2 || misses != 1 || entries != 1 || held != 3 {
-		t.Errorf("stats = %d hits, %d misses, %d stale, %d entries, %d bytes; want 2, 1, 0, 1, 3", hits, misses, stale, entries, held)
+		t.Errorf("stats = %d hits, %d misses, %d entries, %d bytes; want 2, 1, 1, 3", hits, misses, entries, held)
 	}
 }
 func TestHitDoesNotConsumeAThrottleSlot(t *testing.T) {
 	up := &fakeRT{status: 200, body: "x"}
 	th := &daThrottle{base: up, sem: make(chan struct{}, 1)}
-	rt := newAPICache(1<<20, time.Minute, 0).transport(th)
+	rt := newAPICache(1<<20, time.Minute).transport(th)
 	get(t, rt, puppyURL) // populate through the throttle

app/cache.go +10 −3

 // client, serving it from the on-disk or in-memory cache when enabled. It
 // responds 403 when proxying is turned off for this instance.
 func (s skunkyart) DownloadAndSendMedia(subdomain, path string) {
-	mediaURL, ok := buildMediaURL(subdomain, path, s.Args.Get("token"))
+	s.downloadAndSendMedia(subdomain, path, s.Args.Get("token"))
+}
+// fetchMedia is Download behind a variable so tests can script the CDN.
+var fetchMedia = Download
+func (s skunkyart) downloadAndSendMedia(subdomain, path, token string) {
+	mediaURL, ok := buildMediaURL(subdomain, path, token)
 	if !ok {
 		s.ReturnHTTPError(400)
 		return
 			memPut(key, response)
 		}
 	case CFG.Proxy:
-		dwnld := Download(mediaURL)
+		dwnld := fetchMedia(mediaURL)
 		if dwnld.Status != 200 {
 			s.ReturnHTTPError(dwnld.Status)
 			return
 		}
 	}
-	dwnld := Download(mediaURL)
+	dwnld := fetchMedia(mediaURL)
 	if dwnld.Status != 200 || !strings.HasPrefix(dwnld.Headers.Get("Content-Type"), "image") {
 		s.ReturnHTTPError(dwnld.Status)
 		return nil, false

app/config.go +2 −12

 	Enabled bool   `json:"enabled"`
 	MaxSize int64  `json:"max-size"`
 	TTL     string `json:"ttl"`
-	Stale   string `json:"stale"`
 }
 type rateLimitConfig struct {
 		Enabled: true,
 		MaxSize: 64,
 		TTL:     "5i",
-		Stale:   "1h",
 	},
 	RateLimit: rateLimitConfig{
 		PerMinute: 60,
 var lifetimeParsed int64
-// apiCacheTTL and apiCacheStale are api-cache.ttl and api-cache.stale parsed,
+// apiCacheTTL is api-cache.ttl parsed, set by ExecuteConfig.
-// set by ExecuteConfig.
+var apiCacheTTL time.Duration
-var apiCacheTTL, apiCacheStale time.Duration
 // parseLifetime reads a duration in the config's unit syntax: a number
 // followed by i (minutes), h (hours), d (days), w (weeks), m (30-day
 			exit("config: api-cache.ttl: "+err.Error(), 1)
 		}
 		apiCacheTTL = d
-		if CFG.APICache.Stale != "" {
-			d, err := parseLifetime(CFG.APICache.Stale)
-			if err != nil {
-				exit("config: api-cache.stale: "+err.Error(), 1)
-			}
-			apiCacheStale = d
-		}
 	}
 	// per-minute 0 turns the limit off; a burst below one token would

app/config_test.go +2 −2

	@@ -32,8 +32,8 @@ func TestParseLifetimeRejectsBadInput(t *testing.T) {
32	}	32	}
33		33
34	func TestAPICacheDefaults(t *testing.T) {	34	func TestAPICacheDefaults(t *testing.T) {
35	if !CFG.APICache.Enabled \|\| CFG.APICache.MaxSize != 64 \|\| CFG.APICache.TTL != "5i" \|\| CFG.APICache.Stale != "1h" {	35	if !CFG.APICache.Enabled \|\| CFG.APICache.MaxSize != 64 \|\| CFG.APICache.TTL != "5i" {
36	t.Errorf("defaults are %+v, want enabled, 64 MB, 5i, stale 1h", CFG.APICache)	36	t.Errorf("defaults are %+v, want enabled, 64 MB, 5i", CFG.APICache)
37	}	37	}
38	}	38	}
39		39

app/httpclient.go +3 −3

	@@ -100,8 +100,8 @@ func chain(base http.RoundTripper) http.RoundTripper {
100	func logCacheStatsForever(c *apiCache) {	100	func logCacheStatsForever(c *apiCache) {
101	for {	101	for {
102	time.Sleep(time.Hour)	102	time.Sleep(time.Hour)
103	hits, misses, stale, entries, held := c.stats()	103	hits, misses, entries, held := c.stats()
104	println("api cache:", hits, "hits,", misses, "misses,", stale, "served stale,", entries, "entries,", held>>20, "MB held")	104	println("api cache:", hits, "hits,", misses, "misses,", entries, "entries,", held>>20, "MB held")
105	}	105	}
106	}	106	}
107		107
	@@ -111,7 +111,7 @@ func logCacheStatsForever(c *apiCache) {
111	func InstallDAThrottle() {	111	func InstallDAThrottle() {
112	baseTransport = tunedTransport()	112	baseTransport = tunedTransport()
113	if CFG.APICache.Enabled {	113	if CFG.APICache.Enabled {
114	daCache = newAPICache(CFG.APICache.MaxSize<<20, apiCacheTTL, apiCacheStale)	114	daCache = newAPICache(CFG.APICache.MaxSize<<20, apiCacheTTL)
115	go logCacheStatsForever(daCache)	115	go logCacheStatsForever(daCache)
116	}	116	}
117	http.DefaultTransport = chain(baseTransport)	117	http.DefaultTransport = chain(baseTransport)

main.go +8 −3

 	// and let the request escape the throttle and the configured User-Agent.
 	go app.RefreshInstances()
+	// The first session bootstrap runs before the listener opens: requests
+	// that arrive before it finishes go to DeviantArt without a token and
+	// fail, which showed up as 502s for the first seconds after a restart.
+	if err := devianter.UpdateCSRF(); err != nil {
+		println(err.Error())
+	}
 	go func() {
 		for {
-			err := devianter.UpdateCSRF()
+			time.Sleep(12 * time.Hour)
-			if err != nil {
+			if err := devianter.UpdateCSRF(); err != nil {
 				println(err.Error())
 			}
-			time.Sleep(12 * time.Hour)
 		}
 	}()