app/cache.go
357 lines · 10524 bytes · executable
1package app
2
3// TODO: implement JSON caching and clean up the code.
4
5import (
6 "crypto/sha1" //nolint:gosec // G505: SHA-1 is a cache-key hash here, not a security primitive
7 "encoding/base64"
8 "encoding/hex"
9 "encoding/json"
10 "io"
11 "net/url"
12 "os"
13 "regexp"
14 "strconv"
15 "strings"
16 "sync"
17 "syscall"
18 "time"
19)
20
21type file struct {
22 Score int
23 Content []byte
24}
25
26// tempFS is the in-memory media cache, guarded by mx. A plain Mutex rather than
27// an RWMutex on purpose: every operation here mutates something (a read bumps
28// Score), and the previous code took an RLock to write, which is not exclusive.
29var tempFS = make(map[[20]byte]*file)
30var mx sync.Mutex
31
32// memGet returns the cached body for key and raises its score so that popular
33// entries outlive the janitor, or nil when the entry is absent or still empty.
34func memGet(key [20]byte) []byte {
35 mx.Lock()
36 defer mx.Unlock()
37
38 f := tempFS[key]
39 if f == nil || f.Content == nil {
40 return nil
41 }
42 f.Score += 2
43 return f.Content
44}
45
46// memPut caches body under key. An empty body is not cached, so a failed fetch
47// cannot poison the cache with a zero-length image.
48func memPut(key [20]byte, body []byte) {
49 if len(body) == 0 {
50 return
51 }
52
53 mx.Lock()
54 defer mx.Unlock()
55 tempFS[key] = &file{Content: body}
56}
57
58// InitMemCacheJanitor ages the in-memory cache forever, dropping entries whose
59// score has run out. Run it in its own goroutine, once, and only when memcache
60// is enabled.
61//
62// One loop ages the whole map. The previous design started a goroutine per
63// cached file, each looping until its own entry was evicted, and each touching
64// the map without holding mx — a concurrent map read and write, which the Go
65// runtime treats as a fatal error that recover cannot catch.
66func InitMemCacheJanitor() {
67 for {
68 time.Sleep(1 * time.Minute)
69 ageMemCache()
70 }
71}
72
73// ageMemCache runs one round of aging: every entry loses a point, and entries
74// that are already out of points are dropped. An entry starts at zero, so a body
75// nothing asks for again is gone within a round.
76func ageMemCache() {
77 mx.Lock()
78 defer mx.Unlock()
79
80 for k, f := range tempFS {
81 if f.Score <= 0 {
82 delete(tempFS, k)
83 continue
84 }
85 f.Score--
86 }
87}
88
89// mediaSubdomain matches the one hostname label wixmp media URLs vary: a hex
90// string, sometimes with dashes. Anything outside that set is rejected rather
91// than escaped, because this label is what selects the host to fetch from.
92var mediaSubdomain = regexp.MustCompile(`^[a-zA-Z0-9-]+$`)
93
94// blurConstraint reports the minimum blur radius a wixmp media token demands, or
95// 0 if it demands none.
96//
97// DeviantArt signs mature-content media with a watermark-service token whose obj
98// carries a "blur": ">=N" constraint. wixmp then rejects a plain /v1/fit
99// transform with 403 unless it includes a matching blur_N operation, so this is
100// what tells buildMediaURL when to add one. A token it cannot parse yields 0,
101// leaving the URL untouched — the same behaviour as before this check existed.
102func blurConstraint(token string) int {
103 // A JWT is header.payload.signature; the claims are the middle segment,
104 // base64url-encoded without padding.
105 parts := strings.SplitN(token, ".", 3)
106 if len(parts) < 2 {
107 return 0
108 }
109 payload, err := base64.RawURLEncoding.DecodeString(parts[1])
110 if err != nil {
111 return 0
112 }
113
114 var claims struct {
115 Obj [][]struct {
116 Blur string `json:"blur"`
117 } `json:"obj"`
118 }
119 if json.Unmarshal(payload, &claims) != nil ||
120 len(claims.Obj) == 0 || len(claims.Obj[0]) == 0 {
121 return 0
122 }
123
124 // The constraint reads like ">=10"; take its digits as the radius, which is
125 // the minimum the token accepts.
126 n := 0
127 for _, c := range claims.Obj[0][0].Blur {
128 if c >= '0' && c <= '9' {
129 n = n*10 + int(c-'0')
130 }
131 }
132 return n
133}
134
135// addBlurToTransform inserts a blur_n operation into a wixmp /v1/fit transform,
136// turning e.g. w_1280,h_1920 into w_1280,h_1920,blur_n. It returns path
137// unchanged when it carries no /v1/fit transform (GIFs and oversized originals
138// are served without one) or already blurs.
139func addBlurToTransform(path string, n int) string {
140 const marker = "/v1/fit/"
141 start := strings.Index(path, marker)
142 if start < 0 {
143 return path
144 }
145 ops := start + len(marker)
146 end := strings.IndexByte(path[ops:], '/')
147 if end < 0 {
148 return path
149 }
150 end += ops
151 if strings.Contains(path[ops:end], "blur_") {
152 return path
153 }
154 return path[:end] + ",blur_" + strconv.Itoa(n) + path[end:]
155}
156
157// buildMediaURL returns the wixmp CDN URL for one media item, reporting false
158// when subdomain is not a bare hostname label.
159//
160// subdomain and path arrive already percent-decoded from the request path, so
161// they can carry the characters that end a host. Concatenated into a URL string,
162// a subdomain of "x@attacker.example#" reparses as host attacker.example, with
163// "images-wixmp-x" demoted to userinfo and the intended host to a fragment —
164// pointing the fetch at whatever the caller names, including addresses reachable
165// only from the instance itself.
166func buildMediaURL(subdomain, path, token string) (string, bool) {
167 if !mediaSubdomain.MatchString(subdomain) {
168 return "", false
169 }
170
171 // Mature media is signed with a token that only authorizes a blurred render;
172 // without a matching blur op in the transform wixmp answers 403. Add the op
173 // the token demands, and only then, so unconstrained media is left as-is.
174 if n := blurConstraint(token); n > 0 {
175 path = addBlurToTransform(path, n)
176 }
177
178 // Fields rather than concatenation: String escapes the path, so a decoded
179 // "#" or "?" in it stays part of the path instead of ending it. The host is
180 // checked above rather than escaped, because url.URL passes it through
181 // verbatim.
182 u := url.URL{
183 Scheme: "https",
184 Host: "images-wixmp-" + subdomain + ".wixmp.com",
185 Path: "/" + path,
186 }
187 if token != "" {
188 u.RawQuery = url.Values{"token": {token}}.Encode()
189 }
190 return u.String(), true
191}
192
193// DownloadAndSendMedia proxies one image from DeviantArt's wixmp CDN to the
194// client, serving it from the on-disk or in-memory cache when enabled. It
195// responds 403 when proxying is turned off for this instance.
196func (s skunkyart) DownloadAndSendMedia(subdomain, path string) {
197 s.downloadAndSendMedia(subdomain, path, s.Args.Get("token"))
198}
199
200// fetchMedia is Download behind a variable so tests can script the CDN.
201var fetchMedia = Download
202
203func (s skunkyart) downloadAndSendMedia(subdomain, path, token string) {
204 mediaURL, ok := buildMediaURL(subdomain, path, token)
205 if !ok {
206 s.ReturnHTTPError(400)
207 return
208 }
209
210 var response []byte
211
212 switch {
213 case CFG.Cache.Enabled:
214 key := sha1.Sum([]byte(subdomain + path)) //nolint:gosec // G401: cache-key hash, not a security primitive
215 filePath := cacheFilePath(key)
216
217 if CFG.Cache.MemCache {
218 if cached := memGet(key); cached != nil {
219 response = cached
220 break
221 }
222 }
223
224 body, ok := s.loadOrFetchMedia(filePath, mediaURL)
225 if !ok {
226 // loadOrFetchMedia has already written the error response.
227 return
228 }
229 response = body
230
231 if CFG.Cache.MemCache {
232 memPut(key, response)
233 }
234 case CFG.Proxy:
235 dwnld := fetchMedia(mediaURL)
236 if dwnld.Status != 200 {
237 s.ReturnHTTPError(dwnld.Status)
238 return
239 }
240 response = dwnld.Body
241 default:
242 s.Writer.Header().Del("Cache-Control")
243 s.Writer.WriteHeader(403)
244 response = []byte(esc(T(s.Lang, "error.proxy")))
245 }
246
247 _, _ = s.Writer.Write(response)
248}
249
250// loadOrFetchMedia returns the media body for filePath, preferring the on-disk
251// cache and falling back to fetching mediaURL, which it then writes back to the
252// cache. It reports false when it has already written an error response, so the
253// caller must not write anything further.
254func (s skunkyart) loadOrFetchMedia(filePath, mediaURL string) ([]byte, bool) {
255 // filePath is built from a SHA-1 of the request, not from user input, so it
256 // cannot escape the cache directory.
257 if f, err := os.Open(filePath); err == nil { //nolint:gosec // G304: path is a hash, not user-controlled
258 defer func() { try(f.Close()) }()
259
260 if body, err := io.ReadAll(f); err == nil {
261 return body, true
262 } else {
263 // An unreadable cache entry is not fatal; re-fetch it instead.
264 try(err)
265 }
266 }
267
268 dwnld := fetchMedia(mediaURL)
269 if dwnld.Status != 200 || !strings.HasPrefix(dwnld.Headers.Get("Content-Type"), "image") {
270 s.ReturnHTTPError(dwnld.Status)
271 return nil, false
272 }
273
274 try(os.WriteFile(filePath, dwnld.Body, 0600))
275 return dwnld.Body, true
276}
277
278// InitCacheSystem runs the cache rotation loop forever, evicting files past
279// their lifetime and emptying the cache when it outgrows max-size. Run it in its
280// own goroutine.
281func InitCacheSystem() {
282 c := &CFG.Cache
283 for {
284 dir, err := os.ReadDir(c.Path)
285 if err != nil {
286 if os.IsNotExist(err) {
287 try(os.Mkdir(c.Path, 0700))
288 continue
289 }
290 println(err.Error())
291 }
292
293 var total int64
294 for _, file := range dir {
295 fileName := c.Path + "/" + file.Name()
296 fileInfo, err := file.Info()
297 try(err)
298
299 if c.Lifetime != "" {
300 now := time.Now().UnixMilli()
301
302 // Sys() is platform-specific and only documented to be a
303 // *syscall.Stat_t on unix; skip rotation rather than panic
304 // if the filesystem reports something else.
305 if stat, ok := fileInfo.Sys().(*syscall.Stat_t); ok {
306 if statTime(stat)+lifetimeParsed <= now {
307 try(os.RemoveAll(fileName))
308 }
309 }
310 }
311
312 total += fileInfo.Size()
313 // if c.MaxSize != 0 && fileInfo.Size() > c.MaxSize {
314 // try(os.RemoveAll(fileName))
315 // }
316 }
317
318 if c.MaxSize != 0 && total > c.MaxSize {
319 try(os.RemoveAll(c.Path))
320 try(os.Mkdir(c.Path, 0700))
321 }
322
323 time.Sleep(time.Second * time.Duration(c.UpdateInterval))
324 }
325}
326
327// cacheFilePath is where the body cached under key lives on disk.
328func cacheFilePath(key [20]byte) string {
329 return CFG.Cache.Path + "/" + hex.EncodeToString(key[:])
330}
331
332// cachedBody returns the body stored under key, from memory when memcache is
333// on and otherwise from disk, or nil when there is none.
334func cachedBody(key [20]byte) []byte {
335 if CFG.Cache.MemCache {
336 if body := memGet(key); body != nil {
337 return body
338 }
339 }
340 // The path is a hash of the key, not user input.
341 body, err := os.ReadFile(cacheFilePath(key)) //nolint:gosec // G304
342 if err != nil || len(body) == 0 {
343 return nil
344 }
345 if CFG.Cache.MemCache {
346 memPut(key, body)
347 }
348 return body
349}
350
351// storeBody writes body under key to disk and, when memcache is on, memory.
352func storeBody(key [20]byte, body []byte) {
353 try(os.WriteFile(cacheFilePath(key), body, 0600))
354 if CFG.Cache.MemCache {
355 memPut(key, body)
356 }
357}