!28 · krz/skunky-art

SETUP.md −5

		@@ -35,11 +35,6 @@ below apply. A file named with `-c` must exist.
35	35	recently used entries are dropped past this.
36	36	* `ttl` — How long a response is reused, in the time units below. Default
37	37	`5i`.
38		* `stale` — How long past `ttl` a response is kept to be served when
39		DeviantArt fails or blocks the instance, so a short ban does not take
40		the daily deviations, popular searches and feeds down. Default `1h`;
41		`0i` keeps nothing past `ttl`. After a block the instance also leaves
42		DeviantArt alone for a minute instead of retrying every request.
43	38	* `rate-limit` — Per-client budget for page, feed and API requests, so one
44	39	crawler cannot spend the whole upstream budget. Media, avatars and static
45	40	files are not counted. Over budget answers 429 with `Retry-After`.

app/api.go +15 −6

		@@ -3,6 +3,7 @@ package app
3	3	import (
4	4	"encoding/json"
5	5	"math/rand"
	6	"net/url"
6	7	"strings"
7	8
8	9	"github.com/krazywarez/devianter"
		@@ -52,15 +53,23 @@ func (a API) sendMedia(d *devianter.Deviation) {
52	53	return
53	54	}
54	55
55		if CFG.Proxy {
56		mediaURL = mediaURL[21:]
57		dot := strings.Index(mediaURL, ".")
58		a.main.Writer.Header().Del("Content-Type")
59		a.main.DownloadAndSendMedia(mediaURL[:dot], mediaURL[dot+11:])
60		} else {
	56	if !CFG.Proxy {
61	57	a.main.Writer.Header().Add("Location", mediaURL)
62	58	a.main.Writer.WriteHeader(302)
	59	return
	60	}
	61
	62	// Parsed, not sliced: the signing token has to reach wixmp as a query
	63	// parameter. Passing the raw tail as the path put "?token=..." inside
	64	// the path, which wixmp answers with 401.
	65	u, err := url.Parse(mediaURL)
	66	if err != nil {
	67	a.Error("bad media url", 502)
	68	return
63	69	}
	70	subdomain := strings.TrimSuffix(strings.TrimPrefix(u.Host, "images-wixmp-"), ".wixmp.com")
	71	a.main.Writer.Header().Del("Content-Type")
	72	a.main.downloadAndSendMedia(subdomain, strings.TrimPrefix(u.Path, "/"), u.Query().Get("token"))
64	73	}
65	74
66	75	// fetchDailyDeviations is devianter.GetDailyDeviations behind a variable so

app/api_test.go +37

		@@ -1,7 +1,10 @@
1	1	package app
2	2
3	3	import (
	4	"net/http"
4	5	"net/http/httptest"
	6	"net/url"
	7	"strings"
5	8	"testing"
6	9
7	10	"github.com/krazywarez/devianter"
		@@ -108,3 +111,37 @@ func TestRandomHonoursNSFW(t *testing.T) {
108	111	t.Errorf("status %d, Location %q; want 404 and no media", w.Code, w.Header().Get("Location"))
109	112	}
110	113	}
	114
	115	// TestSendMediaProxiesWithTheTokenInTheQuery is the regression test for
	116	// /api/random answering 401 with proxying on: the signing token was passed
	117	// inside the path, so wixmp never saw it as a parameter.
	118	func TestSendMediaProxiesWithTheTokenInTheQuery(t *testing.T) {
	119	proxy, cache := CFG.Proxy, CFG.Cache.Enabled
	120	CFG.Proxy, CFG.Cache.Enabled = true, false
	121	defer func() { CFG.Proxy, CFG.Cache.Enabled = proxy, cache }()
	122
	123	var fetched string
	124	orig := fetchMedia
	125	fetchMedia = func(u string) Downloaded {
	126	fetched = u
	127	return Downloaded{Status: 200, Body: []byte("png"), Headers: http.Header{"Content-Type": {"image/png"}}}
	128	}
	129	defer func() { fetchMedia = orig }()
	130
	131	d := fullviewDeviation()
	132	d.Media.Token = []string{"tok.en.sig"}
	133
	134	w := httptest.NewRecorder()
	135	API{main: &skunkyart{Writer: w, Args: url.Values{}}}.sendMedia(d)
	136
	137	u, err := url.Parse(fetched)
	138	if err != nil \|\| u.Host != "images-wixmp-abc.wixmp.com" {
	139	t.Fatalf("fetched %q, want a wixmp URL on the deviation's subdomain", fetched)
	140	}
	141	if strings.Contains(u.Path, "token") \|\| u.Query().Get("token") == "" {
	142	t.Errorf("token not passed as a query parameter: path %q query %q", u.Path, u.RawQuery)
	143	}
	144	if w.Code != 200 \|\| w.Body.String() != "png" {
	145	t.Errorf("response %d %q, want the proxied image", w.Code, w.Body.String())
	146	}
	147	}

app/apicache.go +28 −98

		@@ -21,31 +21,18 @@ import (
21	21	type apiCache struct {
22	22	maxBytes int64
23	23	ttl time.Duration
24		// stale is how long past ttl an entry is kept to be served when upstream
25		// is blocked or unreachable. Zero keeps nothing past ttl.
26		stale time.Duration
27		now func() time.Time
	24	now func() time.Time
28	25
29		mu sync.Mutex
30		entries map[string]*cacheEntry
31		lru *list.List // front is most recently used
32		held int64
33		hits int64
34		misses int64
35		staleHits int64
36
37		// blockedUntil is set when DeviantArt answers with a block. Until then a
38		// miss with nothing stale to serve gets blockResp back without an
39		// upstream call, so a banned instance stops hammering the WAF.
40		blockedUntil time.Time
41		blockResp *cacheEntry
	26	mu sync.Mutex
	27	entries map[string]*cacheEntry
	28	lru *list.List // front is most recently used
	29	held int64
	30	hits int64
	31	misses int64
42	32
43	33	flight singleflight.Group
44	34	}
45	35
46		// blockBackoff is how long upstream is left alone after a block response.
47		const blockBackoff = time.Minute
48
49	36	// cacheEntry is one buffered response. header is a clone of the upstream
50	37	// header; body is the whole body, read once.
51	38	type cacheEntry struct {
		@@ -57,11 +44,10 @@ type cacheEntry struct {
57	44	elem *list.Element
58	45	}
59	46
60		func newAPICache(maxBytes int64, ttl, stale time.Duration) *apiCache {
	47	func newAPICache(maxBytes int64, ttl time.Duration) *apiCache {
61	48	return &apiCache{
62	49	maxBytes: maxBytes,
63	50	ttl: ttl,
64		stale: stale,
65	51	now: time.Now,
66	52	entries: map[string]*cacheEntry{},
67	53	lru: list.New(),
		@@ -101,29 +87,16 @@ type cachedTransport struct {
101	87	base http.RoundTripper
102	88	}
103	89
104		// RoundTrip serves a fresh hit from memory. A miss is fetched once per key
105		// however many callers are waiting, buffered, stored if it is a 200, and
106		// handed to every waiter as its own response.
107		//
108		// When upstream fails or answers with a block, a stale entry is served
109		// instead if one is still held, so a short ban does not take the popular
110		// pages down. A block also starts a backoff during which misses with nothing
111		// stale get the block response back without an upstream call.
	90	// RoundTrip serves a hit from memory. A miss is fetched once per key however
	91	// many callers are waiting, buffered, stored if it is a 200, and handed to
	92	// every waiter as its own response.
112	93	func (t cachedTransport) RoundTrip(req http.Request) (*http.Response, error) {
113	94	if !cacheable(req) {
114	95	return t.base.RoundTrip(req)
115	96	}
116	97	key := cacheKey(req)
117		old, fresh := t.cache.get(key)
118		if fresh {
119		return old.response(req), nil
120		}
121		if blocked := t.cache.blockedResponse(); blocked != nil {
122		if old != nil {
123		t.cache.countStale()
124		return old.response(req), nil
125		}
126		return blocked.response(req), nil
	98	if e := t.cache.get(key); e != nil {
	99	return e.response(req), nil
127	100	}
128	101
129	102	v, err, _ := t.cache.flight.Do(key, func() (any, error) {
		@@ -137,57 +110,21 @@ func (t cachedTransport) RoundTrip(req http.Request) (*http.Response, error) {
137	110	return nil, err
138	111	}
139	112	e := &cacheEntry{key: key, status: resp.StatusCode, header: resp.Header.Clone(), body: body}
140		switch e.status {
141		case http.StatusOK:
	113	if e.status == http.StatusOK {
142	114	t.cache.put(e)
143		case http.StatusForbidden, http.StatusTooManyRequests:
144		t.cache.block(e)
145	115	}
146	116	return e, nil
147	117	})
148	118	if err != nil {
149		if old != nil {
150		t.cache.countStale()
151		return old.response(req), nil
152		}
153	119	return nil, err
154	120	}
155	121	e, ok := v.(*cacheEntry)
156	122	if !ok {
157	123	return nil, io.ErrUnexpectedEOF
158	124	}
159		if e.status != http.StatusOK && old != nil {
160		t.cache.countStale()
161		return old.response(req), nil
162		}
163	125	return e.response(req), nil
164	126	}
165	127
166		// block records a block response and starts the backoff.
167		func (c apiCache) block(e cacheEntry) {
168		c.mu.Lock()
169		defer c.mu.Unlock()
170		c.blockedUntil = c.now().Add(blockBackoff)
171		c.blockResp = e
172		}
173
174		// blockedResponse returns the last block response while the backoff runs,
175		// or nil once it is over.
176		func (c apiCache) blockedResponse() cacheEntry {
177		c.mu.Lock()
178		defer c.mu.Unlock()
179		if c.blockResp != nil && c.now().Before(c.blockedUntil) {
180		return c.blockResp
181		}
182		return nil
183		}
184
185		func (c *apiCache) countStale() {
186		c.mu.Lock()
187		c.staleHits++
188		c.mu.Unlock()
189		}
190
191	128	// response builds a fresh http.Response over the buffered body, so each
192	129	// caller can read and close its own.
193	130	func (e cacheEntry) response(req http.Request) *http.Response {
		@@ -204,31 +141,25 @@ func (e cacheEntry) response(req http.Request) *http.Response {
204	141	}
205	142	}
206	143
207		// get returns the entry for key and whether it is still fresh. A fresh hit is
208		// marked most recently used. An entry past ttl but within the stale window is
209		// returned as not fresh, for the caller to fall back on; one past the stale
210		// window is dropped.
211		func (c apiCache) get(key string) (cacheEntry, bool) {
	144	// get returns the live entry for key, marking it most recently used, or nil.
	145	// An expired entry is dropped on the way out.
	146	func (c apiCache) get(key string) cacheEntry {
212	147	c.mu.Lock()
213	148	defer c.mu.Unlock()
214	149
215	150	e := c.entries[key]
216	151	if e == nil {
217	152	c.misses++
218		return nil, false
	153	return nil
219	154	}
220		now := c.now()
221		if now.Before(e.expires) {
222		c.lru.MoveToFront(e.elem)
223		c.hits++
224		return e, true
225		}
226		c.misses++
227		if now.Before(e.expires.Add(c.stale)) {
228		return e, false
	155	if !c.now().Before(e.expires) {
	156	c.remove(e)
	157	c.misses++
	158	return nil
229	159	}
230		c.remove(e)
231		return nil, false
	160	c.lru.MoveToFront(e.elem)
	161	c.hits++
	162	return e
232	163	}
233	164
234	165	// put stores e, evicting from the least recently used end until it fits. A
		@@ -265,10 +196,9 @@ func (c apiCache) remove(e cacheEntry) {
265	196	c.held -= int64(len(e.body))
266	197	}
267	198
268		// stats reports the counters for the hourly log line. stale counts the
269		// misses that were answered from an expired entry because upstream failed.
270		func (c *apiCache) stats() (hits, misses, stale int64, entries int, held int64) {
	199	// stats reports the counters for the hourly log line.
	200	func (c *apiCache) stats() (hits, misses int64, entries int, held int64) {
271	201	c.mu.Lock()
272	202	defer c.mu.Unlock()
273		return c.hits, c.misses, c.staleHits, len(c.entries), c.held
	203	return c.hits, c.misses, len(c.entries), c.held
274	204	}

app/apicache_stale_test.go deleted −132

		@@ -1,132 +0,0 @@
1		package app
2
3		import (
4		"errors"
5		"net/http"
6		"testing"
7		"time"
8		)
9
10		// failingRT is an upstream that can be switched between a scripted status and
11		// a transport error mid-test.
12		type failingRT struct {
13		fakeRT
14		err error
15		}
16
17		func (f failingRT) RoundTrip(r http.Request) (*http.Response, error) {
18		f.mu.Lock()
19		err := f.err
20		f.mu.Unlock()
21		if err != nil {
22		f.mu.Lock()
23		f.calls++
24		f.mu.Unlock()
25		return nil, err
26		}
27		return f.fakeRT.RoundTrip(r)
28		}
29
30		func (f *failingRT) set(status int, body string, err error) {
31		f.mu.Lock()
32		defer f.mu.Unlock()
33		f.status, f.body, f.err = status, body, err
34		}
35
36		// staleCache returns a cache with a one minute TTL and one hour stale window
37		// over a controllable clock, warmed with one good response for puppyURL.
38		func staleCache(t testing.T) (apiCache, failingRT, http.RoundTripper, time.Time) {
39		t.Helper()
40		up := &failingRT{}
41		up.set(200, `{"good":1}`, nil)
42		c := newAPICache(1<<20, time.Minute, time.Hour)
43		now := time.Now()
44		c.now = func() time.Time { return now }
45		rt := c.transport(up)
46		get(t, rt, puppyURL)
47		return c, up, rt, &now
48		}
49
50		func TestStaleEntryServedWhenUpstreamBlocks(t *testing.T) {
51		c, up, rt, now := staleCache(t)
52		now = now.Add(2 time.Minute) // past ttl, inside the stale window
53		up.set(403, "<html>blocked</html>", nil)
54
55		status, body := get(t, rt, puppyURL)
56
57		if status != 200 \|\| body != `{"good":1}` {
58		t.Errorf("got %d %q, want the stale 200 body", status, body)
59		}
60		if up.count() != 2 {
61		t.Errorf("upstream called %d times, want 2: one warm-up and one attempt that hit the block", up.count())
62		}
63		if _, _, stale, _, _ := c.stats(); stale != 1 {
64		t.Errorf("stale counter is %d, want 1", stale)
65		}
66		}
67
68		func TestStaleEntryServedOnTransportError(t *testing.T) {
69		_, up, rt, now := staleCache(t)
70		now = now.Add(2 time.Minute)
71		up.set(0, "", errors.New("dial tcp: connection refused"))
72
73		status, body := get(t, rt, puppyURL)
74
75		if status != 200 \|\| body != `{"good":1}` {
76		t.Errorf("got %d %q, want the stale 200 body", status, body)
77		}
78		}
79
80		func TestBlockBackoffSkipsUpstream(t *testing.T) {
81		_, up, rt, now := staleCache(t)
82		now = now.Add(2 time.Minute)
83		up.set(403, "<html>blocked</html>", nil)
84		get(t, rt, puppyURL) // triggers the block
85		calls := up.count()
86
87		other := "https://www.deviantart.com/_puppy/dabrowse/search/all?q=other"
88		status, body := get(t, rt, other)
89		if status != 403 \|\| body != "<html>blocked</html>" {
90		t.Errorf("uncached key during backoff got %d %q, want the block response", status, body)
91		}
92		if up.count() != calls {
93		t.Errorf("upstream called during backoff (%d -> %d), want none", calls, up.count())
94		}
95
96		*now = now.Add(blockBackoff + time.Second)
97		up.set(200, `{"back":1}`, nil)
98		if status, _ := get(t, rt, other); status != 200 \|\| up.count() != calls+1 {
99		t.Errorf("after backoff: status %d, calls %d, want 200 and one more upstream call", status, up.count())
100		}
101		}
102
103		func TestStaleEntryDroppedAfterTheWindow(t *testing.T) {
104		c, up, rt, now := staleCache(t)
105		*now = now.Add(time.Minute + time.Hour + time.Second) // past ttl and stale
106		up.set(403, "<html>blocked</html>", nil)
107
108		status, _ := get(t, rt, puppyURL)
109
110		if status != 403 {
111		t.Errorf("got %d, want the 403 passed through once nothing stale is held", status)
112		}
113		if _, _, _, entries, _ := c.stats(); entries != 0 {
114		t.Errorf("%d entries held, want 0", entries)
115		}
116		}
117
118		func TestZeroStaleKeepsNothingPastTTL(t *testing.T) {
119		up := &failingRT{}
120		up.set(200, "x", nil)
121		c := newAPICache(1<<20, time.Minute, 0)
122		now := time.Now()
123		c.now = func() time.Time { return now }
124		rt := c.transport(up)
125		get(t, rt, puppyURL)
126		now = now.Add(2 * time.Minute)
127		up.set(403, "blocked", nil)
128
129		if status, _ := get(t, rt, puppyURL); status != 403 {
130		t.Errorf("got %d with stale 0, want 403: nothing may be served past ttl", status)
131		}
132		}

app/apicache_test.go +16 −18

		@@ -56,7 +56,7 @@ func get(t *testing.T, rt http.RoundTripper, url string) (int, string) {
56	56
57	57	func TestSecondRequestIsServedFromCache(t *testing.T) {
58	58	up := &fakeRT{status: 200, body: `{"a":1}`}
59		rt := newAPICache(1<<20, time.Minute, 0).transport(up)
	59	rt := newAPICache(1<<20, time.Minute).transport(up)
60	60
61	61	get(t, rt, puppyURL)
62	62	status, body := get(t, rt, puppyURL)
		@@ -71,7 +71,7 @@ func TestSecondRequestIsServedFromCache(t *testing.T) {
71	71
72	72	func TestExpiredEntryIsRefetched(t *testing.T) {
73	73	up := &fakeRT{status: 200, body: `{}`}
74		c := newAPICache(1<<20, time.Minute, 0)
	74	c := newAPICache(1<<20, time.Minute)
75	75	now := time.Now()
76	76	c.now = func() time.Time { return now }
77	77	rt := c.transport(up)
		@@ -85,26 +85,24 @@ func TestExpiredEntryIsRefetched(t *testing.T) {
85	85	}
86	86	}
87	87
88		// A 500 here rather than a 403: a 403 is a block and starts the backoff,
89		// which is covered in apicache_stale_test.go.
90	88	func TestNon200IsNotStored(t *testing.T) {
91		up := &fakeRT{status: 500, body: "upstream broke"}
92		rt := newAPICache(1<<20, time.Minute, 0).transport(up)
	89	up := &fakeRT{status: 403, body: "blocked"}
	90	rt := newAPICache(1<<20, time.Minute).transport(up)
93	91
94	92	status, body := get(t, rt, puppyURL)
95	93	get(t, rt, puppyURL)
96	94
97		if status != 500 \|\| body != "upstream broke" {
98		t.Errorf("first response is %d %q, want the upstream 500 passed through", status, body)
	95	if status != 403 \|\| body != "blocked" {
	96	t.Errorf("first response is %d %q, want the upstream 403 passed through", status, body)
99	97	}
100	98	if up.count() != 2 {
101		t.Errorf("upstream called %d times, want 2: a 500 must not be cached", up.count())
	99	t.Errorf("upstream called %d times, want 2: a 403 must not be cached", up.count())
102	100	}
103	101	}
104	102
105	103	func TestBypassesSessionAndOtherHosts(t *testing.T) {
106	104	up := &fakeRT{status: 200, body: "x"}
107		rt := newAPICache(1<<20, time.Minute, 0).transport(up)
	105	rt := newAPICache(1<<20, time.Minute).transport(up)
108	106
109	107	for _, url := range []string{
110	108	"https://www.deviantart.com/_puppy",
		@@ -121,7 +119,7 @@ func TestBypassesSessionAndOtherHosts(t *testing.T) {
121	119
122	120	func TestKeyIgnoresCSRFToken(t *testing.T) {
123	121	up := &fakeRT{status: 200, body: "x"}
124		rt := newAPICache(1<<20, time.Minute, 0).transport(up)
	122	rt := newAPICache(1<<20, time.Minute).transport(up)
125	123
126	124	get(t, rt, puppyURL)
127	125	get(t, rt, strings.Replace(puppyURL, "csrf_token=abc", "csrf_token=def", 1))
		@@ -133,7 +131,7 @@ func TestKeyIgnoresCSRFToken(t *testing.T) {
133	131
134	132	func TestByteBoundEvictsLeastRecentlyUsed(t *testing.T) {
135	133	up := &fakeRT{status: 200, body: strings.Repeat("x", 100)}
136		rt := newAPICache(250, time.Minute, 0).transport(up)
	134	rt := newAPICache(250, time.Minute).transport(up)
137	135	a := "https://www.deviantart.com/_puppy/a?p=1"
138	136	b := "https://www.deviantart.com/_puppy/b?p=1"
139	137	c := "https://www.deviantart.com/_puppy/c?p=1"
		@@ -153,7 +151,7 @@ func TestByteBoundEvictsLeastRecentlyUsed(t *testing.T) {
153	151
154	152	func TestConcurrentMissesMakeOneUpstreamCall(t *testing.T) {
155	153	up := &fakeRT{status: 200, body: "x", delay: 50 * time.Millisecond}
156		rt := newAPICache(1<<20, time.Minute, 0).transport(up)
	154	rt := newAPICache(1<<20, time.Minute).transport(up)
157	155
158	156	var wg sync.WaitGroup
159	157	for range 20 {
		@@ -168,16 +166,16 @@ func TestConcurrentMissesMakeOneUpstreamCall(t *testing.T) {
168	166
169	167	func TestStatsCountHitsAndMisses(t *testing.T) {
170	168	up := &fakeRT{status: 200, body: "abc"}
171		c := newAPICache(1<<20, time.Minute, 0)
	169	c := newAPICache(1<<20, time.Minute)
172	170	rt := c.transport(up)
173	171
174	172	get(t, rt, puppyURL)
175	173	get(t, rt, puppyURL)
176	174	get(t, rt, puppyURL)
177	175
178		hits, misses, stale, entries, held := c.stats()
179		if hits != 2 \|\| misses != 1 \|\| stale != 0 \|\| entries != 1 \|\| held != 3 {
180		t.Errorf("stats = %d hits, %d misses, %d stale, %d entries, %d bytes; want 2, 1, 0, 1, 3", hits, misses, stale, entries, held)
	176	hits, misses, entries, held := c.stats()
	177	if hits != 2 \|\| misses != 1 \|\| entries != 1 \|\| held != 3 {
	178	t.Errorf("stats = %d hits, %d misses, %d entries, %d bytes; want 2, 1, 1, 3", hits, misses, entries, held)
181	179	}
182	180	}
183	181
		@@ -187,7 +185,7 @@ func TestStatsCountHitsAndMisses(t *testing.T) {
187	185	func TestHitDoesNotConsumeAThrottleSlot(t *testing.T) {
188	186	up := &fakeRT{status: 200, body: "x"}
189	187	th := &daThrottle{base: up, sem: make(chan struct{}, 1)}
190		rt := newAPICache(1<<20, time.Minute, 0).transport(th)
	188	rt := newAPICache(1<<20, time.Minute).transport(th)
191	189
192	190	get(t, rt, puppyURL) // populate through the throttle
193	191

app/cache.go +10 −3

		@@ -194,7 +194,14 @@ func buildMediaURL(subdomain, path, token string) (string, bool) {
194	194	// client, serving it from the on-disk or in-memory cache when enabled. It
195	195	// responds 403 when proxying is turned off for this instance.
196	196	func (s skunkyart) DownloadAndSendMedia(subdomain, path string) {
197		mediaURL, ok := buildMediaURL(subdomain, path, s.Args.Get("token"))
	197	s.downloadAndSendMedia(subdomain, path, s.Args.Get("token"))
	198	}
	199
	200	// fetchMedia is Download behind a variable so tests can script the CDN.
	201	var fetchMedia = Download
	202
	203	func (s skunkyart) downloadAndSendMedia(subdomain, path, token string) {
	204	mediaURL, ok := buildMediaURL(subdomain, path, token)
198	205	if !ok {
199	206	s.ReturnHTTPError(400)
200	207	return
		@@ -225,7 +232,7 @@ func (s skunkyart) DownloadAndSendMedia(subdomain, path string) {
225	232	memPut(key, response)
226	233	}
227	234	case CFG.Proxy:
228		dwnld := Download(mediaURL)
	235	dwnld := fetchMedia(mediaURL)
229	236	if dwnld.Status != 200 {
230	237	s.ReturnHTTPError(dwnld.Status)
231	238	return
		@@ -258,7 +265,7 @@ func (s skunkyart) loadOrFetchMedia(filePath, mediaURL string) ([]byte, bool) {
258	265	}
259	266	}
260	267
261		dwnld := Download(mediaURL)
	268	dwnld := fetchMedia(mediaURL)
262	269	if dwnld.Status != 200 \|\| !strings.HasPrefix(dwnld.Headers.Get("Content-Type"), "image") {
263	270	s.ReturnHTTPError(dwnld.Status)
264	271	return nil, false

app/config.go +2 −12

		@@ -32,7 +32,6 @@ type apiCacheConfig struct {
32	32	Enabled bool `json:"enabled"`
33	33	MaxSize int64 `json:"max-size"`
34	34	TTL string `json:"ttl"`
35		Stale string `json:"stale"`
36	35	}
37	36
38	37	type rateLimitConfig struct {
		@@ -76,7 +75,6 @@ var CFG = config{
76	75	Enabled: true,
77	76	MaxSize: 64,
78	77	TTL: "5i",
79		Stale: "1h",
80	78	},
81	79	RateLimit: rateLimitConfig{
82	80	PerMinute: 60,
		@@ -90,9 +88,8 @@ var CFG = config{
90	88
91	89	var lifetimeParsed int64
92	90
93		// apiCacheTTL and apiCacheStale are api-cache.ttl and api-cache.stale parsed,
94		// set by ExecuteConfig.
95		var apiCacheTTL, apiCacheStale time.Duration
	91	// apiCacheTTL is api-cache.ttl parsed, set by ExecuteConfig.
	92	var apiCacheTTL time.Duration
96	93
97	94	// parseLifetime reads a duration in the config's unit syntax: a number
98	95	// followed by i (minutes), h (hours), d (days), w (weeks), m (30-day
		@@ -219,13 +216,6 @@ func ExecuteConfig() {
219	216	exit("config: api-cache.ttl: "+err.Error(), 1)
220	217	}
221	218	apiCacheTTL = d
222		if CFG.APICache.Stale != "" {
223		d, err := parseLifetime(CFG.APICache.Stale)
224		if err != nil {
225		exit("config: api-cache.stale: "+err.Error(), 1)
226		}
227		apiCacheStale = d
228		}
229	219	}
230	220
231	221	// per-minute 0 turns the limit off; a burst below one token would

app/config_test.go +2 −2

		@@ -32,8 +32,8 @@ func TestParseLifetimeRejectsBadInput(t *testing.T) {
32	32	}
33	33
34	34	func TestAPICacheDefaults(t *testing.T) {
35		if !CFG.APICache.Enabled \|\| CFG.APICache.MaxSize != 64 \|\| CFG.APICache.TTL != "5i" \|\| CFG.APICache.Stale != "1h" {
36		t.Errorf("defaults are %+v, want enabled, 64 MB, 5i, stale 1h", CFG.APICache)
	35	if !CFG.APICache.Enabled \|\| CFG.APICache.MaxSize != 64 \|\| CFG.APICache.TTL != "5i" {
	36	t.Errorf("defaults are %+v, want enabled, 64 MB, 5i", CFG.APICache)
37	37	}
38	38	}
39	39

app/httpclient.go +3 −3

		@@ -100,8 +100,8 @@ func chain(base http.RoundTripper) http.RoundTripper {
100	100	func logCacheStatsForever(c *apiCache) {
101	101	for {
102	102	time.Sleep(time.Hour)
103		hits, misses, stale, entries, held := c.stats()
104		println("api cache:", hits, "hits,", misses, "misses,", stale, "served stale,", entries, "entries,", held>>20, "MB held")
	103	hits, misses, entries, held := c.stats()
	104	println("api cache:", hits, "hits,", misses, "misses,", entries, "entries,", held>>20, "MB held")
105	105	}
106	106	}
107	107
		@@ -111,7 +111,7 @@ func logCacheStatsForever(c *apiCache) {
111	111	func InstallDAThrottle() {
112	112	baseTransport = tunedTransport()
113	113	if CFG.APICache.Enabled {
114		daCache = newAPICache(CFG.APICache.MaxSize<<20, apiCacheTTL, apiCacheStale)
	114	daCache = newAPICache(CFG.APICache.MaxSize<<20, apiCacheTTL)
115	115	go logCacheStatsForever(daCache)
116	116	}
117	117	http.DefaultTransport = chain(baseTransport)

main.go +8 −3

		@@ -39,13 +39,18 @@ func main() {
39	39	// and let the request escape the throttle and the configured User-Agent.
40	40	go app.RefreshInstances()
41	41
	42	// The first session bootstrap runs before the listener opens: requests
	43	// that arrive before it finishes go to DeviantArt without a token and
	44	// fail, which showed up as 502s for the first seconds after a restart.
	45	if err := devianter.UpdateCSRF(); err != nil {
	46	println(err.Error())
	47	}
42	48	go func() {
43	49	for {
44		err := devianter.UpdateCSRF()
45		if err != nil {
	50	time.Sleep(12 * time.Hour)
	51	if err := devianter.UpdateCSRF(); err != nil {
46	52	println(err.Error())
47	53	}
48		time.Sleep(12 * time.Hour)
49	54	}
50	55	}()
51	56

Proxy random media with the token in the query, and bootstrap before listening !28