Commit `e2ccebb244`

e2ccebb244d78221ddfb5296e9cd9fcd7a7ed107

parent: e8ee7ce359

Verified · cmc ci/build: success ci/lint: success ci/test: success

cmc <hello@cleberg.net> · 2026-09-11 19:25 UTC

Trim the media cache to its cap instead of emptying it

Rotation is one pass in rotateCache: files past their lifetime go, and
when the rest exceed max-size the oldest go until they fit. It never
removes the directory, which in the container is a bind mount and
logged unlinkat and mkdir errors on every pass. Ages come from the
modification time, so the per-platform ctime files are gone.

Closes #36

Layout: unified · split

SETUP.md +1 −1

	@@ -22,7 +22,7 @@ below apply. A file named with `-c` must exist.
22	* `lifetime` — How long a cached file is kept, in the time units below.	22	* `lifetime` — How long a cached file is kept, in the time units below.
23	Default `1w`.	23	Default `1w`.
24	* `max-size` — Cache size cap in megabytes, default 200. When the cache	24	* `max-size` — Cache size cap in megabytes, default 200. When the cache
25	grows past it the whole directory is emptied, not trimmed.	25	grows past it, the oldest files are removed until it fits.
26	* `update-interval` — Seconds between rotation passes, default 1. Each	26	* `update-interval` — Seconds between rotation passes, default 1. Each
27	pass reads the whole cache directory, so raise it on a large cache.	27	pass reads the whole cache directory, so raise it on a large cache.
28	* `api-cache` — In-memory cache of DeviantArt API responses. Every page,	28	* `api-cache` — In-memory cache of DeviantArt API responses. Every page,

app/cache.go +55 −38

	@@ -11,10 +11,10 @@ import (
11	"net/url"	11	"net/url"
12	"os"	12	"os"
13	"regexp"	13	"regexp"
		14	"sort"
14	"strconv"	15	"strconv"
15	"strings"	16	"strings"
16	"sync"	17	"sync"
17	"syscall"
18	"time"	18	"time"
19	)	19	)
20		20
	@@ -275,53 +275,70 @@ func (s skunkyart) loadOrFetchMedia(filePath, mediaURL string) ([]byte, bool) {
275	return dwnld.Body, true	275	return dwnld.Body, true
276	}	276	}
277		277
278	// InitCacheSystem runs the cache rotation loop forever, evicting files past	278	// InitCacheSystem runs the cache rotation loop forever: every update-interval
279	// their lifetime and emptying the cache when it outgrows max-size. Run it in its	279	// seconds it drops files past their lifetime and, when the cache is over
280	// own goroutine.	280	// max-size, the oldest files until it fits. Run it in its own goroutine.
281	func InitCacheSystem() {	281	func InitCacheSystem() {
282	c := &CFG.Cache	282	c := &CFG.Cache
283	for {	283	for {
284	dir, err := os.ReadDir(c.Path)	284	if err := rotateCache(c.Path, time.Duration(lifetimeParsed)*time.Millisecond, c.MaxSize, time.Now()); err != nil {
285	if err != nil {	285	println("cache rotation:", err.Error())
286	if os.IsNotExist(err) {
287	try(os.Mkdir(c.Path, 0700))
288	continue
289	}
290	println(err.Error())
291	}	286	}
		287	time.Sleep(time.Second * time.Duration(c.UpdateInterval))
		288	}
		289	}
292		290
293	var total int64	291	// rotateCache does one rotation pass over dir. Files whose modification time
294	for _, file := range dir {	292	// is more than lifetime ago are removed (lifetime 0 keeps everything). If the
295	fileName := c.Path + "/" + file.Name()	293	// remaining files exceed maxSize bytes (0 for no cap), the oldest are removed
296	fileInfo, err := file.Info()	294	// until they fit. Only files are touched, never the directory: in the
297	try(err)	295	// container it is a bind mount, which cannot be removed, and the old
298		296	// remove-and-recreate logged an error every pass.
299	if c.Lifetime != "" {	297	func rotateCache(dir string, lifetime time.Duration, maxSize int64, now time.Time) error {
300	now := time.Now().UnixMilli()	298	entries, err := os.ReadDir(dir)
301		299	if err != nil {
302	// Sys() is platform-specific and only documented to be a	300	if os.IsNotExist(err) {
303	// *syscall.Stat_t on unix; skip rotation rather than panic	301	return os.Mkdir(dir, 0700)
304	// if the filesystem reports something else.
305	if stat, ok := fileInfo.Sys().(*syscall.Stat_t); ok {
306	if statTime(stat)+lifetimeParsed <= now {
307	try(os.RemoveAll(fileName))
308	}
309	}
310	}
311
312	total += fileInfo.Size()
313	// if c.MaxSize != 0 && fileInfo.Size() > c.MaxSize {
314	// try(os.RemoveAll(fileName))
315	// }
316	}	302	}
		303	return err
		304	}
317		305
318	if c.MaxSize != 0 && total > c.MaxSize {	306	type cached struct {
319	try(os.RemoveAll(c.Path))	307	path string
320	try(os.Mkdir(c.Path, 0700))	308	size int64
		309	mod time.Time
		310	}
		311	var files []cached
		312	var total int64
		313	for _, e := range entries {
		314	if e.IsDir() {
		315	continue
		316	}
		317	info, err := e.Info()
		318	if err != nil {
		319	continue
		320	}
		321	f := cached{path: dir + "/" + e.Name(), size: info.Size(), mod: info.ModTime()}
		322	if lifetime > 0 && !f.mod.Add(lifetime).After(now) {
		323	try(os.Remove(f.path))
		324	continue
321	}	325	}
		326	files = append(files, f)
		327	total += f.size
		328	}
322		329
323	time.Sleep(time.Second * time.Duration(c.UpdateInterval))	330	if maxSize <= 0 \|\| total <= maxSize {
		331	return nil
		332	}
		333	sort.Slice(files, func(i, j int) bool { return files[i].mod.Before(files[j].mod) })
		334	for _, f := range files {
		335	if total <= maxSize {
		336	break
		337	}
		338	try(os.Remove(f.path))
		339	total -= f.size
324	}	340	}
		341	return nil
325	}	342	}
326		343
327	// cacheFilePath is where the body cached under key lives on disk.	344	// cacheFilePath is where the body cached under key lives on disk.

app/cache_rotate_test.go added +90

@@ -0,0 +1,90 @@
	1	package app
	2
	3	import (
	4	"os"
	5	"path/filepath"
	6	"testing"
	7	"time"
	8	)
	9
	10	// seed writes a cache file of size bytes with the given age.
	11	func seed(t *testing.T, dir, name string, size int, age time.Duration, now time.Time) {
	12	t.Helper()
	13	p := filepath.Join(dir, name)
	14	if err := os.WriteFile(p, make([]byte, size), 0600); err != nil {
	15	t.Fatal(err)
	16	}
	17	if err := os.Chtimes(p, now.Add(-age), now.Add(-age)); err != nil {
	18	t.Fatal(err)
	19	}
	20	}
	21
	22	func names(t *testing.T, dir string) []string {
	23	t.Helper()
	24	entries, err := os.ReadDir(dir)
	25	if err != nil {
	26	t.Fatal(err)
	27	}
	28	var out []string
	29	for _, e := range entries {
	30	out = append(out, e.Name())
	31	}
	32	return out
	33	}
	34
	35	func TestRotateCacheDropsExpiredFiles(t *testing.T) {
	36	dir, now := t.TempDir(), time.Now()
	37	seed(t, dir, "old", 10, 2*time.Hour, now)
	38	seed(t, dir, "new", 10, time.Minute, now)
	39
	40	if err := rotateCache(dir, time.Hour, 0, now); err != nil {
	41	t.Fatal(err)
	42	}
	43	if got := names(t, dir); len(got) != 1 \|\| got[0] != "new" {
	44	t.Errorf("files after rotation: %v, want only new", got)
	45	}
	46	}
	47
	48	// TestRotateCacheTrimsOldestToTheCap is the regression test for #36: over the
	49	// cap the oldest files go until the rest fit, and the directory itself is
	50	// never removed.
	51	func TestRotateCacheTrimsOldestToTheCap(t *testing.T) {
	52	dir, now := t.TempDir(), time.Now()
	53	seed(t, dir, "oldest", 100, 3*time.Hour, now)
	54	seed(t, dir, "middle", 100, 2*time.Hour, now)
	55	seed(t, dir, "newest", 100, time.Hour, now)
	56
	57	if err := rotateCache(dir, 0, 250, now); err != nil {
	58	t.Fatal(err)
	59	}
	60	got := names(t, dir)
	61	if len(got) != 2 \|\| got[0] != "middle" \|\| got[1] != "newest" {
	62	t.Errorf("files after rotation: %v, want middle and newest", got)
	63	}
	64	if _, err := os.Stat(dir); err != nil {
	65	t.Errorf("cache directory removed: %v", err)
	66	}
	67	}
	68
	69	func TestRotateCacheExpiredFilesDoNotCountTowardTheCap(t *testing.T) {
	70	dir, now := t.TempDir(), time.Now()
	71	seed(t, dir, "expired", 200, 2*time.Hour, now)
	72	seed(t, dir, "kept", 100, time.Minute, now)
	73
	74	if err := rotateCache(dir, time.Hour, 150, now); err != nil {
	75	t.Fatal(err)
	76	}
	77	if got := names(t, dir); len(got) != 1 \|\| got[0] != "kept" {
	78	t.Errorf("files after rotation: %v, want only kept (the expired file must not push it over the cap)", got)
	79	}
	80	}
	81
	82	func TestRotateCacheCreatesAMissingDirectory(t *testing.T) {
	83	dir := filepath.Join(t.TempDir(), "cache")
	84	if err := rotateCache(dir, 0, 0, time.Now()); err != nil {
	85	t.Fatal(err)
	86	}
	87	if _, err := os.Stat(dir); err != nil {
	88	t.Errorf("directory not created: %v", err)
	89	}
	90	}

app/stat-darwin.go deleted −14

	@@ -1,14 +0,0 @@
1	//go:build darwin
2
3	package app
4
5	import (
6	"syscall"
7	"time"
8	)
9
10	// macOS names the ctime field Ctimespec rather than Ctim, so it needs its own
11	// variant to let the project build and test on a Mac (deploys are Linux).
12	func statTime(stat *syscall.Stat_t) int64 {
13	return time.Unix(stat.Ctimespec.Unix()).UnixMilli()
14	}

app/stat-freebsd.go deleted −12

	@@ -1,12 +0,0 @@
1	//go:build freebsd
2
3	package app
4
5	import (
6	"syscall"
7	"time"
8	)
9
10	func statTime(stat *syscall.Stat_t) int64 {
11	return time.Unix(stat.Ctimespec.Unix()).UnixMilli()
12	}

app/stat.go deleted −12

	@@ -1,12 +0,0 @@
1	//go:build !freebsd && !darwin
2
3	package app
4
5	import (
6	"syscall"
7	"time"
8	)
9
10	func statTime(stat *syscall.Stat_t) int64 {
11	return time.Unix(stat.Ctim.Unix()).UnixMilli()
12	}

Commit e2ccebb244

Commit `e2ccebb244`