app/cache.go
374 lines · 11175 bytes · executable
1package app
2
3// TODO: implement JSON caching and clean up the code.
4
5import (
6 "crypto/sha1" //nolint:gosec // G505: SHA-1 is a cache-key hash here, not a security primitive
7 "encoding/base64"
8 "encoding/hex"
9 "encoding/json"
10 "io"
11 "net/url"
12 "os"
13 "regexp"
14 "sort"
15 "strconv"
16 "strings"
17 "sync"
18 "time"
19)
20
21type file struct {
22 Score int
23 Content []byte
24}
25
26// tempFS is the in-memory media cache, guarded by mx. A plain Mutex rather than
27// an RWMutex on purpose: every operation here mutates something (a read bumps
28// Score), and the previous code took an RLock to write, which is not exclusive.
29var tempFS = make(map[[20]byte]*file)
30var mx sync.Mutex
31
32// memGet returns the cached body for key and raises its score so that popular
33// entries outlive the janitor, or nil when the entry is absent or still empty.
34func memGet(key [20]byte) []byte {
35 mx.Lock()
36 defer mx.Unlock()
37
38 f := tempFS[key]
39 if f == nil || f.Content == nil {
40 return nil
41 }
42 f.Score += 2
43 return f.Content
44}
45
46// memPut caches body under key. An empty body is not cached, so a failed fetch
47// cannot poison the cache with a zero-length image.
48func memPut(key [20]byte, body []byte) {
49 if len(body) == 0 {
50 return
51 }
52
53 mx.Lock()
54 defer mx.Unlock()
55 tempFS[key] = &file{Content: body}
56}
57
58// InitMemCacheJanitor ages the in-memory cache forever, dropping entries whose
59// score has run out. Run it in its own goroutine, once, and only when memcache
60// is enabled.
61//
62// One loop ages the whole map. The previous design started a goroutine per
63// cached file, each looping until its own entry was evicted, and each touching
64// the map without holding mx — a concurrent map read and write, which the Go
65// runtime treats as a fatal error that recover cannot catch.
66func InitMemCacheJanitor() {
67 for {
68 time.Sleep(1 * time.Minute)
69 ageMemCache()
70 }
71}
72
73// ageMemCache runs one round of aging: every entry loses a point, and entries
74// that are already out of points are dropped. An entry starts at zero, so a body
75// nothing asks for again is gone within a round.
76func ageMemCache() {
77 mx.Lock()
78 defer mx.Unlock()
79
80 for k, f := range tempFS {
81 if f.Score <= 0 {
82 delete(tempFS, k)
83 continue
84 }
85 f.Score--
86 }
87}
88
89// mediaSubdomain matches the one hostname label wixmp media URLs vary: a hex
90// string, sometimes with dashes. Anything outside that set is rejected rather
91// than escaped, because this label is what selects the host to fetch from.
92var mediaSubdomain = regexp.MustCompile(`^[a-zA-Z0-9-]+$`)
93
94// blurConstraint reports the minimum blur radius a wixmp media token demands, or
95// 0 if it demands none.
96//
97// DeviantArt signs mature-content media with a watermark-service token whose obj
98// carries a "blur": ">=N" constraint. wixmp then rejects a plain /v1/fit
99// transform with 403 unless it includes a matching blur_N operation, so this is
100// what tells buildMediaURL when to add one. A token it cannot parse yields 0,
101// leaving the URL untouched — the same behaviour as before this check existed.
102func blurConstraint(token string) int {
103 // A JWT is header.payload.signature; the claims are the middle segment,
104 // base64url-encoded without padding.
105 parts := strings.SplitN(token, ".", 3)
106 if len(parts) < 2 {
107 return 0
108 }
109 payload, err := base64.RawURLEncoding.DecodeString(parts[1])
110 if err != nil {
111 return 0
112 }
113
114 var claims struct {
115 Obj [][]struct {
116 Blur string `json:"blur"`
117 } `json:"obj"`
118 }
119 if json.Unmarshal(payload, &claims) != nil ||
120 len(claims.Obj) == 0 || len(claims.Obj[0]) == 0 {
121 return 0
122 }
123
124 // The constraint reads like ">=10"; take its digits as the radius, which is
125 // the minimum the token accepts.
126 n := 0
127 for _, c := range claims.Obj[0][0].Blur {
128 if c >= '0' && c <= '9' {
129 n = n*10 + int(c-'0')
130 }
131 }
132 return n
133}
134
135// addBlurToTransform inserts a blur_n operation into a wixmp /v1/fit transform,
136// turning e.g. w_1280,h_1920 into w_1280,h_1920,blur_n. It returns path
137// unchanged when it carries no /v1/fit transform (GIFs and oversized originals
138// are served without one) or already blurs.
139func addBlurToTransform(path string, n int) string {
140 const marker = "/v1/fit/"
141 start := strings.Index(path, marker)
142 if start < 0 {
143 return path
144 }
145 ops := start + len(marker)
146 end := strings.IndexByte(path[ops:], '/')
147 if end < 0 {
148 return path
149 }
150 end += ops
151 if strings.Contains(path[ops:end], "blur_") {
152 return path
153 }
154 return path[:end] + ",blur_" + strconv.Itoa(n) + path[end:]
155}
156
157// buildMediaURL returns the wixmp CDN URL for one media item, reporting false
158// when subdomain is not a bare hostname label.
159//
160// subdomain and path arrive already percent-decoded from the request path, so
161// they can carry the characters that end a host. Concatenated into a URL string,
162// a subdomain of "x@attacker.example#" reparses as host attacker.example, with
163// "images-wixmp-x" demoted to userinfo and the intended host to a fragment —
164// pointing the fetch at whatever the caller names, including addresses reachable
165// only from the instance itself.
166func buildMediaURL(subdomain, path, token string) (string, bool) {
167 if !mediaSubdomain.MatchString(subdomain) {
168 return "", false
169 }
170
171 // Mature media is signed with a token that only authorizes a blurred render;
172 // without a matching blur op in the transform wixmp answers 403. Add the op
173 // the token demands, and only then, so unconstrained media is left as-is.
174 if n := blurConstraint(token); n > 0 {
175 path = addBlurToTransform(path, n)
176 }
177
178 // Fields rather than concatenation: String escapes the path, so a decoded
179 // "#" or "?" in it stays part of the path instead of ending it. The host is
180 // checked above rather than escaped, because url.URL passes it through
181 // verbatim.
182 u := url.URL{
183 Scheme: "https",
184 Host: "images-wixmp-" + subdomain + ".wixmp.com",
185 Path: "/" + path,
186 }
187 if token != "" {
188 u.RawQuery = url.Values{"token": {token}}.Encode()
189 }
190 return u.String(), true
191}
192
193// DownloadAndSendMedia proxies one image from DeviantArt's wixmp CDN to the
194// client, serving it from the on-disk or in-memory cache when enabled. It
195// responds 403 when proxying is turned off for this instance.
196func (s skunkyart) DownloadAndSendMedia(subdomain, path string) {
197 s.downloadAndSendMedia(subdomain, path, s.Args.Get("token"))
198}
199
200// fetchMedia is Download behind a variable so tests can script the CDN.
201var fetchMedia = Download
202
203func (s skunkyart) downloadAndSendMedia(subdomain, path, token string) {
204 mediaURL, ok := buildMediaURL(subdomain, path, token)
205 if !ok {
206 s.ReturnHTTPError(400)
207 return
208 }
209
210 var response []byte
211
212 switch {
213 case CFG.Cache.Enabled:
214 key := sha1.Sum([]byte(subdomain + path)) //nolint:gosec // G401: cache-key hash, not a security primitive
215 filePath := cacheFilePath(key)
216
217 if CFG.Cache.MemCache {
218 if cached := memGet(key); cached != nil {
219 response = cached
220 break
221 }
222 }
223
224 body, ok := s.loadOrFetchMedia(filePath, mediaURL)
225 if !ok {
226 // loadOrFetchMedia has already written the error response.
227 return
228 }
229 response = body
230
231 if CFG.Cache.MemCache {
232 memPut(key, response)
233 }
234 case CFG.Proxy:
235 dwnld := fetchMedia(mediaURL)
236 if dwnld.Status != 200 {
237 s.ReturnHTTPError(dwnld.Status)
238 return
239 }
240 response = dwnld.Body
241 default:
242 s.Writer.Header().Del("Cache-Control")
243 s.Writer.WriteHeader(403)
244 response = []byte(esc(T(s.Lang, "error.proxy")))
245 }
246
247 _, _ = s.Writer.Write(response)
248}
249
250// loadOrFetchMedia returns the media body for filePath, preferring the on-disk
251// cache and falling back to fetching mediaURL, which it then writes back to the
252// cache. It reports false when it has already written an error response, so the
253// caller must not write anything further.
254func (s skunkyart) loadOrFetchMedia(filePath, mediaURL string) ([]byte, bool) {
255 // filePath is built from a SHA-1 of the request, not from user input, so it
256 // cannot escape the cache directory.
257 if f, err := os.Open(filePath); err == nil { //nolint:gosec // G304: path is a hash, not user-controlled
258 defer func() { try(f.Close()) }()
259
260 if body, err := io.ReadAll(f); err == nil {
261 return body, true
262 } else {
263 // An unreadable cache entry is not fatal; re-fetch it instead.
264 try(err)
265 }
266 }
267
268 dwnld := fetchMedia(mediaURL)
269 if dwnld.Status != 200 || !strings.HasPrefix(dwnld.Headers.Get("Content-Type"), "image") {
270 s.ReturnHTTPError(dwnld.Status)
271 return nil, false
272 }
273
274 try(os.WriteFile(filePath, dwnld.Body, 0600))
275 return dwnld.Body, true
276}
277
278// InitCacheSystem runs the cache rotation loop forever: every update-interval
279// seconds it drops files past their lifetime and, when the cache is over
280// max-size, the oldest files until it fits. Run it in its own goroutine.
281func InitCacheSystem() {
282 c := &CFG.Cache
283 for {
284 if err := rotateCache(c.Path, time.Duration(lifetimeParsed)*time.Millisecond, c.MaxSize, time.Now()); err != nil {
285 println("cache rotation:", err.Error())
286 }
287 time.Sleep(time.Second * time.Duration(c.UpdateInterval))
288 }
289}
290
291// rotateCache does one rotation pass over dir. Files whose modification time
292// is more than lifetime ago are removed (lifetime 0 keeps everything). If the
293// remaining files exceed maxSize bytes (0 for no cap), the oldest are removed
294// until they fit. Only files are touched, never the directory: in the
295// container it is a bind mount, which cannot be removed, and the old
296// remove-and-recreate logged an error every pass.
297func rotateCache(dir string, lifetime time.Duration, maxSize int64, now time.Time) error {
298 entries, err := os.ReadDir(dir)
299 if err != nil {
300 if os.IsNotExist(err) {
301 return os.Mkdir(dir, 0700)
302 }
303 return err
304 }
305
306 type cached struct {
307 path string
308 size int64
309 mod time.Time
310 }
311 var files []cached
312 var total int64
313 for _, e := range entries {
314 if e.IsDir() {
315 continue
316 }
317 info, err := e.Info()
318 if err != nil {
319 continue
320 }
321 f := cached{path: dir + "/" + e.Name(), size: info.Size(), mod: info.ModTime()}
322 if lifetime > 0 && !f.mod.Add(lifetime).After(now) {
323 try(os.Remove(f.path))
324 continue
325 }
326 files = append(files, f)
327 total += f.size
328 }
329
330 if maxSize <= 0 || total <= maxSize {
331 return nil
332 }
333 sort.Slice(files, func(i, j int) bool { return files[i].mod.Before(files[j].mod) })
334 for _, f := range files {
335 if total <= maxSize {
336 break
337 }
338 try(os.Remove(f.path))
339 total -= f.size
340 }
341 return nil
342}
343
344// cacheFilePath is where the body cached under key lives on disk.
345func cacheFilePath(key [20]byte) string {
346 return CFG.Cache.Path + "/" + hex.EncodeToString(key[:])
347}
348
349// cachedBody returns the body stored under key, from memory when memcache is
350// on and otherwise from disk, or nil when there is none.
351func cachedBody(key [20]byte) []byte {
352 if CFG.Cache.MemCache {
353 if body := memGet(key); body != nil {
354 return body
355 }
356 }
357 // The path is a hash of the key, not user input.
358 body, err := os.ReadFile(cacheFilePath(key)) //nolint:gosec // G304
359 if err != nil || len(body) == 0 {
360 return nil
361 }
362 if CFG.Cache.MemCache {
363 memPut(key, body)
364 }
365 return body
366}
367
368// storeBody writes body under key to disk and, when memcache is on, memory.
369func storeBody(key [20]byte, body []byte) {
370 try(os.WriteFile(cacheFilePath(key), body, 0600))
371 if CFG.Cache.MemCache {
372 memPut(key, body)
373 }
374}