app/cache.go
372 lines · 11119 bytes · executable
1package app
2
3import (
4 "crypto/sha1" //nolint:gosec // G505: SHA-1 is a cache-key hash here, not a security primitive
5 "encoding/base64"
6 "encoding/hex"
7 "encoding/json"
8 "io"
9 "net/url"
10 "os"
11 "regexp"
12 "sort"
13 "strconv"
14 "strings"
15 "sync"
16 "time"
17)
18
19type file struct {
20 Score int
21 Content []byte
22}
23
24// tempFS is the in-memory media cache, guarded by mx. A plain Mutex rather than
25// an RWMutex on purpose: every operation here mutates something (a read bumps
26// Score), and the previous code took an RLock to write, which is not exclusive.
27var tempFS = make(map[[20]byte]*file)
28var mx sync.Mutex
29
30// memGet returns the cached body for key and raises its score so that popular
31// entries outlive the janitor, or nil when the entry is absent or still empty.
32func memGet(key [20]byte) []byte {
33 mx.Lock()
34 defer mx.Unlock()
35
36 f := tempFS[key]
37 if f == nil || f.Content == nil {
38 return nil
39 }
40 f.Score += 2
41 return f.Content
42}
43
44// memPut caches body under key. An empty body is not cached, so a failed fetch
45// cannot poison the cache with a zero-length image.
46func memPut(key [20]byte, body []byte) {
47 if len(body) == 0 {
48 return
49 }
50
51 mx.Lock()
52 defer mx.Unlock()
53 tempFS[key] = &file{Content: body}
54}
55
56// InitMemCacheJanitor ages the in-memory cache forever, dropping entries whose
57// score has run out. Run it in its own goroutine, once, and only when memcache
58// is enabled.
59//
60// One loop ages the whole map. The previous design started a goroutine per
61// cached file, each looping until its own entry was evicted, and each touching
62// the map without holding mx — a concurrent map read and write, which the Go
63// runtime treats as a fatal error that recover cannot catch.
64func InitMemCacheJanitor() {
65 for {
66 time.Sleep(1 * time.Minute)
67 ageMemCache()
68 }
69}
70
71// ageMemCache runs one round of aging: every entry loses a point, and entries
72// that are already out of points are dropped. An entry starts at zero, so a body
73// nothing asks for again is gone within a round.
74func ageMemCache() {
75 mx.Lock()
76 defer mx.Unlock()
77
78 for k, f := range tempFS {
79 if f.Score <= 0 {
80 delete(tempFS, k)
81 continue
82 }
83 f.Score--
84 }
85}
86
87// mediaSubdomain matches the one hostname label wixmp media URLs vary: a hex
88// string, sometimes with dashes. Anything outside that set is rejected rather
89// than escaped, because this label is what selects the host to fetch from.
90var mediaSubdomain = regexp.MustCompile(`^[a-zA-Z0-9-]+$`)
91
92// blurConstraint reports the minimum blur radius a wixmp media token demands, or
93// 0 if it demands none.
94//
95// DeviantArt signs mature-content media with a watermark-service token whose obj
96// carries a "blur": ">=N" constraint. wixmp then rejects a plain /v1/fit
97// transform with 403 unless it includes a matching blur_N operation, so this is
98// what tells buildMediaURL when to add one. A token it cannot parse yields 0,
99// leaving the URL untouched — the same behaviour as before this check existed.
100func blurConstraint(token string) int {
101 // A JWT is header.payload.signature; the claims are the middle segment,
102 // base64url-encoded without padding.
103 parts := strings.SplitN(token, ".", 3)
104 if len(parts) < 2 {
105 return 0
106 }
107 payload, err := base64.RawURLEncoding.DecodeString(parts[1])
108 if err != nil {
109 return 0
110 }
111
112 var claims struct {
113 Obj [][]struct {
114 Blur string `json:"blur"`
115 } `json:"obj"`
116 }
117 if json.Unmarshal(payload, &claims) != nil ||
118 len(claims.Obj) == 0 || len(claims.Obj[0]) == 0 {
119 return 0
120 }
121
122 // The constraint reads like ">=10"; take its digits as the radius, which is
123 // the minimum the token accepts.
124 n := 0
125 for _, c := range claims.Obj[0][0].Blur {
126 if c >= '0' && c <= '9' {
127 n = n*10 + int(c-'0')
128 }
129 }
130 return n
131}
132
133// addBlurToTransform inserts a blur_n operation into a wixmp /v1/fit transform,
134// turning e.g. w_1280,h_1920 into w_1280,h_1920,blur_n. It returns path
135// unchanged when it carries no /v1/fit transform (GIFs and oversized originals
136// are served without one) or already blurs.
137func addBlurToTransform(path string, n int) string {
138 const marker = "/v1/fit/"
139 start := strings.Index(path, marker)
140 if start < 0 {
141 return path
142 }
143 ops := start + len(marker)
144 end := strings.IndexByte(path[ops:], '/')
145 if end < 0 {
146 return path
147 }
148 end += ops
149 if strings.Contains(path[ops:end], "blur_") {
150 return path
151 }
152 return path[:end] + ",blur_" + strconv.Itoa(n) + path[end:]
153}
154
155// buildMediaURL returns the wixmp CDN URL for one media item, reporting false
156// when subdomain is not a bare hostname label.
157//
158// subdomain and path arrive already percent-decoded from the request path, so
159// they can carry the characters that end a host. Concatenated into a URL string,
160// a subdomain of "x@attacker.example#" reparses as host attacker.example, with
161// "images-wixmp-x" demoted to userinfo and the intended host to a fragment —
162// pointing the fetch at whatever the caller names, including addresses reachable
163// only from the instance itself.
164func buildMediaURL(subdomain, path, token string) (string, bool) {
165 if !mediaSubdomain.MatchString(subdomain) {
166 return "", false
167 }
168
169 // Mature media is signed with a token that only authorizes a blurred render;
170 // without a matching blur op in the transform wixmp answers 403. Add the op
171 // the token demands, and only then, so unconstrained media is left as-is.
172 if n := blurConstraint(token); n > 0 {
173 path = addBlurToTransform(path, n)
174 }
175
176 // Fields rather than concatenation: String escapes the path, so a decoded
177 // "#" or "?" in it stays part of the path instead of ending it. The host is
178 // checked above rather than escaped, because url.URL passes it through
179 // verbatim.
180 u := url.URL{
181 Scheme: "https",
182 Host: "images-wixmp-" + subdomain + ".wixmp.com",
183 Path: "/" + path,
184 }
185 if token != "" {
186 u.RawQuery = url.Values{"token": {token}}.Encode()
187 }
188 return u.String(), true
189}
190
191// DownloadAndSendMedia proxies one image from DeviantArt's wixmp CDN to the
192// client, serving it from the on-disk or in-memory cache when enabled. It
193// responds 403 when proxying is turned off for this instance.
194func (s skunkyart) DownloadAndSendMedia(subdomain, path string) {
195 s.downloadAndSendMedia(subdomain, path, s.Args.Get("token"))
196}
197
198// fetchMedia is Download behind a variable so tests can script the CDN.
199var fetchMedia = Download
200
201func (s skunkyart) downloadAndSendMedia(subdomain, path, token string) {
202 mediaURL, ok := buildMediaURL(subdomain, path, token)
203 if !ok {
204 s.ReturnHTTPError(400)
205 return
206 }
207
208 var response []byte
209
210 switch {
211 case CFG.Cache.Enabled:
212 key := sha1.Sum([]byte(subdomain + path)) //nolint:gosec // G401: cache-key hash, not a security primitive
213 filePath := cacheFilePath(key)
214
215 if CFG.Cache.MemCache {
216 if cached := memGet(key); cached != nil {
217 response = cached
218 break
219 }
220 }
221
222 body, ok := s.loadOrFetchMedia(filePath, mediaURL)
223 if !ok {
224 // loadOrFetchMedia has already written the error response.
225 return
226 }
227 response = body
228
229 if CFG.Cache.MemCache {
230 memPut(key, response)
231 }
232 case CFG.Proxy:
233 dwnld := fetchMedia(mediaURL)
234 if dwnld.Status != 200 {
235 s.ReturnHTTPError(dwnld.Status)
236 return
237 }
238 response = dwnld.Body
239 default:
240 s.Writer.Header().Del("Cache-Control")
241 s.Writer.WriteHeader(403)
242 response = []byte(esc(T(s.Lang, "error.proxy")))
243 }
244
245 _, _ = s.Writer.Write(response)
246}
247
248// loadOrFetchMedia returns the media body for filePath, preferring the on-disk
249// cache and falling back to fetching mediaURL, which it then writes back to the
250// cache. It reports false when it has already written an error response, so the
251// caller must not write anything further.
252func (s skunkyart) loadOrFetchMedia(filePath, mediaURL string) ([]byte, bool) {
253 // filePath is built from a SHA-1 of the request, not from user input, so it
254 // cannot escape the cache directory.
255 if f, err := os.Open(filePath); err == nil { //nolint:gosec // G304: path is a hash, not user-controlled
256 defer func() { try(f.Close()) }()
257
258 if body, err := io.ReadAll(f); err == nil {
259 return body, true
260 } else {
261 // An unreadable cache entry is not fatal; re-fetch it instead.
262 try(err)
263 }
264 }
265
266 dwnld := fetchMedia(mediaURL)
267 if dwnld.Status != 200 || !strings.HasPrefix(dwnld.Headers.Get("Content-Type"), "image") {
268 s.ReturnHTTPError(dwnld.Status)
269 return nil, false
270 }
271
272 try(os.WriteFile(filePath, dwnld.Body, 0600))
273 return dwnld.Body, true
274}
275
276// InitCacheSystem runs the cache rotation loop forever: every update-interval
277// seconds it drops files past their lifetime and, when the cache is over
278// max-size, the oldest files until it fits. Run it in its own goroutine.
279func InitCacheSystem() {
280 c := &CFG.Cache
281 for {
282 if err := rotateCache(c.Path, time.Duration(lifetimeParsed)*time.Millisecond, c.MaxSize, time.Now()); err != nil {
283 println("cache rotation:", err.Error())
284 }
285 time.Sleep(time.Second * time.Duration(c.UpdateInterval))
286 }
287}
288
289// rotateCache does one rotation pass over dir. Files whose modification time
290// is more than lifetime ago are removed (lifetime 0 keeps everything). If the
291// remaining files exceed maxSize bytes (0 for no cap), the oldest are removed
292// until they fit. Only files are touched, never the directory: in the
293// container it is a bind mount, which cannot be removed, and the old
294// remove-and-recreate logged an error every pass.
295func rotateCache(dir string, lifetime time.Duration, maxSize int64, now time.Time) error {
296 entries, err := os.ReadDir(dir)
297 if err != nil {
298 if os.IsNotExist(err) {
299 return os.Mkdir(dir, 0700)
300 }
301 return err
302 }
303
304 type cached struct {
305 path string
306 size int64
307 mod time.Time
308 }
309 var files []cached
310 var total int64
311 for _, e := range entries {
312 if e.IsDir() {
313 continue
314 }
315 info, err := e.Info()
316 if err != nil {
317 continue
318 }
319 f := cached{path: dir + "/" + e.Name(), size: info.Size(), mod: info.ModTime()}
320 if lifetime > 0 && !f.mod.Add(lifetime).After(now) {
321 try(os.Remove(f.path))
322 continue
323 }
324 files = append(files, f)
325 total += f.size
326 }
327
328 if maxSize <= 0 || total <= maxSize {
329 return nil
330 }
331 sort.Slice(files, func(i, j int) bool { return files[i].mod.Before(files[j].mod) })
332 for _, f := range files {
333 if total <= maxSize {
334 break
335 }
336 try(os.Remove(f.path))
337 total -= f.size
338 }
339 return nil
340}
341
342// cacheFilePath is where the body cached under key lives on disk.
343func cacheFilePath(key [20]byte) string {
344 return CFG.Cache.Path + "/" + hex.EncodeToString(key[:])
345}
346
347// cachedBody returns the body stored under key, from memory when memcache is
348// on and otherwise from disk, or nil when there is none.
349func cachedBody(key [20]byte) []byte {
350 if CFG.Cache.MemCache {
351 if body := memGet(key); body != nil {
352 return body
353 }
354 }
355 // The path is a hash of the key, not user input.
356 body, err := os.ReadFile(cacheFilePath(key)) //nolint:gosec // G304
357 if err != nil || len(body) == 0 {
358 return nil
359 }
360 if CFG.Cache.MemCache {
361 memPut(key, body)
362 }
363 return body
364}
365
366// storeBody writes body under key to disk and, when memcache is on, memory.
367func storeBody(key [20]byte, body []byte) {
368 try(os.WriteFile(cacheFilePath(key), body, 0600))
369 if CFG.Cache.MemCache {
370 memPut(key, body)
371 }
372}