Commit `e2ccebb244`

e2ccebb244d78221ddfb5296e9cd9fcd7a7ed107

parent: e8ee7ce359

Verified · cmc ci/build: success ci/lint: success ci/test: success

cmc <hello@cleberg.net> · 2026-09-11 19:25 UTC

Trim the media cache to its cap instead of emptying it

Rotation is one pass in rotateCache: files past their lifetime go, and
when the rest exceed max-size the oldest go until they fit. It never
removes the directory, which in the container is a bind mount and
logged unlinkat and mkdir errors on every pass. Ages come from the
modification time, so the per-platform ctime files are gone.

Closes #36

Layout: unified · split

SETUP.md +1 −1

		@@ -22,7 +22,7 @@ below apply. A file named with `-c` must exist.
22	22	* `lifetime` — How long a cached file is kept, in the time units below.
23	23	Default `1w`.
24	24	* `max-size` — Cache size cap in megabytes, default 200. When the cache
25		grows past it the whole directory is emptied, not trimmed.
	25	grows past it, the oldest files are removed until it fits.
26	26	* `update-interval` — Seconds between rotation passes, default 1. Each
27	27	pass reads the whole cache directory, so raise it on a large cache.
28	28	* `api-cache` — In-memory cache of DeviantArt API responses. Every page,

app/cache.go +55 −38

		@@ -11,10 +11,10 @@ import (
11	11	"net/url"
12	12	"os"
13	13	"regexp"
	14	"sort"
14	15	"strconv"
15	16	"strings"
16	17	"sync"
17		"syscall"
18	18	"time"
19	19	)
20	20
		@@ -275,53 +275,70 @@ func (s skunkyart) loadOrFetchMedia(filePath, mediaURL string) ([]byte, bool) {
275	275	return dwnld.Body, true
276	276	}
277	277
278		// InitCacheSystem runs the cache rotation loop forever, evicting files past
279		// their lifetime and emptying the cache when it outgrows max-size. Run it in its
280		// own goroutine.
	278	// InitCacheSystem runs the cache rotation loop forever: every update-interval
	279	// seconds it drops files past their lifetime and, when the cache is over
	280	// max-size, the oldest files until it fits. Run it in its own goroutine.
281	281	func InitCacheSystem() {
282	282	c := &CFG.Cache
283	283	for {
284		dir, err := os.ReadDir(c.Path)
285		if err != nil {
286		if os.IsNotExist(err) {
287		try(os.Mkdir(c.Path, 0700))
288		continue
289		}
290		println(err.Error())
	284	if err := rotateCache(c.Path, time.Duration(lifetimeParsed)*time.Millisecond, c.MaxSize, time.Now()); err != nil {
	285	println("cache rotation:", err.Error())
291	286	}
	287	time.Sleep(time.Second * time.Duration(c.UpdateInterval))
	288	}
	289	}
292	290
293		var total int64
294		for _, file := range dir {
295		fileName := c.Path + "/" + file.Name()
296		fileInfo, err := file.Info()
297		try(err)
298
299		if c.Lifetime != "" {
300		now := time.Now().UnixMilli()
301
302		// Sys() is platform-specific and only documented to be a
303		// *syscall.Stat_t on unix; skip rotation rather than panic
304		// if the filesystem reports something else.
305		if stat, ok := fileInfo.Sys().(*syscall.Stat_t); ok {
306		if statTime(stat)+lifetimeParsed <= now {
307		try(os.RemoveAll(fileName))
308		}
309		}
310		}
311
312		total += fileInfo.Size()
313		// if c.MaxSize != 0 && fileInfo.Size() > c.MaxSize {
314		// try(os.RemoveAll(fileName))
315		// }
	291	// rotateCache does one rotation pass over dir. Files whose modification time
	292	// is more than lifetime ago are removed (lifetime 0 keeps everything). If the
	293	// remaining files exceed maxSize bytes (0 for no cap), the oldest are removed
	294	// until they fit. Only files are touched, never the directory: in the
	295	// container it is a bind mount, which cannot be removed, and the old
	296	// remove-and-recreate logged an error every pass.
	297	func rotateCache(dir string, lifetime time.Duration, maxSize int64, now time.Time) error {
	298	entries, err := os.ReadDir(dir)
	299	if err != nil {
	300	if os.IsNotExist(err) {
	301	return os.Mkdir(dir, 0700)
316	302	}
	303	return err
	304	}
317	305
318		if c.MaxSize != 0 && total > c.MaxSize {
319		try(os.RemoveAll(c.Path))
320		try(os.Mkdir(c.Path, 0700))
	306	type cached struct {
	307	path string
	308	size int64
	309	mod time.Time
	310	}
	311	var files []cached
	312	var total int64
	313	for _, e := range entries {
	314	if e.IsDir() {
	315	continue
	316	}
	317	info, err := e.Info()
	318	if err != nil {
	319	continue
	320	}
	321	f := cached{path: dir + "/" + e.Name(), size: info.Size(), mod: info.ModTime()}
	322	if lifetime > 0 && !f.mod.Add(lifetime).After(now) {
	323	try(os.Remove(f.path))
	324	continue
321	325	}
	326	files = append(files, f)
	327	total += f.size
	328	}
322	329
323		time.Sleep(time.Second * time.Duration(c.UpdateInterval))
	330	if maxSize <= 0 \|\| total <= maxSize {
	331	return nil
	332	}
	333	sort.Slice(files, func(i, j int) bool { return files[i].mod.Before(files[j].mod) })
	334	for _, f := range files {
	335	if total <= maxSize {
	336	break
	337	}
	338	try(os.Remove(f.path))
	339	total -= f.size
324	340	}
	341	return nil
325	342	}
326	343
327	344	// cacheFilePath is where the body cached under key lives on disk.

app/cache_rotate_test.go added +90

		@@ -0,0 +1,90 @@
	1	package app
	2
	3	import (
	4	"os"
	5	"path/filepath"
	6	"testing"
	7	"time"
	8	)
	9
	10	// seed writes a cache file of size bytes with the given age.
	11	func seed(t *testing.T, dir, name string, size int, age time.Duration, now time.Time) {
	12	t.Helper()
	13	p := filepath.Join(dir, name)
	14	if err := os.WriteFile(p, make([]byte, size), 0600); err != nil {
	15	t.Fatal(err)
	16	}
	17	if err := os.Chtimes(p, now.Add(-age), now.Add(-age)); err != nil {
	18	t.Fatal(err)
	19	}
	20	}
	21
	22	func names(t *testing.T, dir string) []string {
	23	t.Helper()
	24	entries, err := os.ReadDir(dir)
	25	if err != nil {
	26	t.Fatal(err)
	27	}
	28	var out []string
	29	for _, e := range entries {
	30	out = append(out, e.Name())
	31	}
	32	return out
	33	}
	34
	35	func TestRotateCacheDropsExpiredFiles(t *testing.T) {
	36	dir, now := t.TempDir(), time.Now()
	37	seed(t, dir, "old", 10, 2*time.Hour, now)
	38	seed(t, dir, "new", 10, time.Minute, now)
	39
	40	if err := rotateCache(dir, time.Hour, 0, now); err != nil {
	41	t.Fatal(err)
	42	}
	43	if got := names(t, dir); len(got) != 1 \|\| got[0] != "new" {
	44	t.Errorf("files after rotation: %v, want only new", got)
	45	}
	46	}
	47
	48	// TestRotateCacheTrimsOldestToTheCap is the regression test for #36: over the
	49	// cap the oldest files go until the rest fit, and the directory itself is
	50	// never removed.
	51	func TestRotateCacheTrimsOldestToTheCap(t *testing.T) {
	52	dir, now := t.TempDir(), time.Now()
	53	seed(t, dir, "oldest", 100, 3*time.Hour, now)
	54	seed(t, dir, "middle", 100, 2*time.Hour, now)
	55	seed(t, dir, "newest", 100, time.Hour, now)
	56
	57	if err := rotateCache(dir, 0, 250, now); err != nil {
	58	t.Fatal(err)
	59	}
	60	got := names(t, dir)
	61	if len(got) != 2 \|\| got[0] != "middle" \|\| got[1] != "newest" {
	62	t.Errorf("files after rotation: %v, want middle and newest", got)
	63	}
	64	if _, err := os.Stat(dir); err != nil {
	65	t.Errorf("cache directory removed: %v", err)
	66	}
	67	}
	68
	69	func TestRotateCacheExpiredFilesDoNotCountTowardTheCap(t *testing.T) {
	70	dir, now := t.TempDir(), time.Now()
	71	seed(t, dir, "expired", 200, 2*time.Hour, now)
	72	seed(t, dir, "kept", 100, time.Minute, now)
	73
	74	if err := rotateCache(dir, time.Hour, 150, now); err != nil {
	75	t.Fatal(err)
	76	}
	77	if got := names(t, dir); len(got) != 1 \|\| got[0] != "kept" {
	78	t.Errorf("files after rotation: %v, want only kept (the expired file must not push it over the cap)", got)
	79	}
	80	}
	81
	82	func TestRotateCacheCreatesAMissingDirectory(t *testing.T) {
	83	dir := filepath.Join(t.TempDir(), "cache")
	84	if err := rotateCache(dir, 0, 0, time.Now()); err != nil {
	85	t.Fatal(err)
	86	}
	87	if _, err := os.Stat(dir); err != nil {
	88	t.Errorf("directory not created: %v", err)
	89	}
	90	}

app/stat-darwin.go deleted −14

		@@ -1,14 +0,0 @@
1		//go:build darwin
2
3		package app
4
5		import (
6		"syscall"
7		"time"
8		)
9
10		// macOS names the ctime field Ctimespec rather than Ctim, so it needs its own
11		// variant to let the project build and test on a Mac (deploys are Linux).
12		func statTime(stat *syscall.Stat_t) int64 {
13		return time.Unix(stat.Ctimespec.Unix()).UnixMilli()
14		}

app/stat-freebsd.go deleted −12

		@@ -1,12 +0,0 @@
1		//go:build freebsd
2
3		package app
4
5		import (
6		"syscall"
7		"time"
8		)
9
10		func statTime(stat *syscall.Stat_t) int64 {
11		return time.Unix(stat.Ctimespec.Unix()).UnixMilli()
12		}

app/stat.go deleted −12

		@@ -1,12 +0,0 @@
1		//go:build !freebsd && !darwin
2
3		package app
4
5		import (
6		"syscall"
7		"time"
8		)
9
10		func statTime(stat *syscall.Stat_t) int64 {
11		return time.Unix(stat.Ctim.Unix()).UnixMilli()
12		}

Commit e2ccebb244

Commit `e2ccebb244`