Commit `51386c203f`

51386c203f67e5ced14804b138d522e961c739d9

parent: 0b531fd6ad

Verified · cmc ci/build: success ci/test: failure ci/vuln: success

cmc <hello@cleberg.net> · 2026-09-02 02:55 UTC

lfs: count it in stats, and gc --lfs for orphaned objects

admin stats walked repository directories only, so the LFS root was
missing from the disk total, and the Admin page recorded that gc never
collected an orphaned object.

stats carries lfs_bytes. gc --lfs scans every repository's objects for
LFS pointers, every object rather than the reachable ones since an
unreachable blob is still one gc has not removed, and deletes stored
objects no pointer names. Objects younger than a day are kept: git lfs
uploads before it pushes, so a fresh unreferenced object is a push in
progress, not an orphan.

Closes #79

Layout: unified · split

cmd/gitbayd/maint.go +48 −3

		@@ -4,21 +4,23 @@ import (
4	4	"fmt"
5	5	"os"
6	6	"os/exec"
	7	"time"
7	8
8	9	"github.com/spf13/cobra"
9	10
10	11	"gitbay.org/gitbay/internal/config"
11	12	"gitbay.org/gitbay/internal/control"
12	13	"gitbay.org/gitbay/internal/gitutil"
	14	"gitbay.org/gitbay/internal/lfs"
13	15	"gitbay.org/gitbay/internal/store"
14	16	)
15	17
16	18	func gcCmd() *cobra.Command {
17	19	var repoPath string
18		var aggressive bool
	20	var aggressive, withLFS bool
19	21	cmd := &cobra.Command{
20	22	Use: "gc",
21		Short: "repack and prune repositories (git gc)",
	23	Short: "repack and prune repositories (git gc); --lfs removes unreferenced LFS objects",
22	24	RunE: func(cmd *cobra.Command, args []string) error {
23	25	cfg, err := config.Load(configPath)
24	26	if err != nil {
		@@ -58,11 +60,15 @@ func gcCmd() *cobra.Command {
58	60	fmt.Printf("%s\t%s -> %s\n", r.Path(), human(b), human(a))
59	61	}
60	62	fmt.Printf("total\t%s -> %s (freed %s)\n", human(before), human(after), human(before-after))
61		return nil
	63	if !withLFS {
	64	return nil
	65	}
	66	return gcLFS(cfg, st)
62	67	},
63	68	}
64	69	cmd.Flags().StringVar(&repoPath, "repo", "", "one repository (owner/name) instead of all")
65	70	cmd.Flags().BoolVar(&aggressive, "aggressive", false, "more thorough repack (slow; rarely needed)")
	71	cmd.Flags().BoolVar(&withLFS, "lfs", false, "also remove LFS objects no repository references (older than a day)")
66	72	return cmd
67	73	}
68	74
		@@ -78,3 +84,42 @@ func human(b int64) string {
78	84	return fmt.Sprintf("%d B", b)
79	85	}
80	86	}
	87
	88	// lfsOrphanAge keeps an object uploaded ahead of the push that references
	89	// it: git lfs uploads first and pushes second.
	90	const lfsOrphanAge = 24 * time.Hour
	91
	92	// gcLFS removes LFS objects no repository's pointers name. Every
	93	// repository is scanned, whatever --repo said: an object is shared across
	94	// repositories by content, so only the whole set says it is unreferenced.
	95	func gcLFS(cfg config.Config, st *store.Store) error {
	96	repos, err := st.ListAllRepos()
	97	if err != nil {
	98	return err
	99	}
	100	referenced := map[string]bool{}
	101	for _, r := range repos {
	102	oids, err := gitutil.LFSPointerOIDs(control.RepoDir(cfg.Server.Root, r.OwnerName, r.Name))
	103	if err != nil {
	104	return fmt.Errorf("%s: scanning for LFS pointers: %w", r.Path(), err)
	105	}
	106	for _, o := range oids {
	107	referenced[o] = true
	108	}
	109	}
	110	blobs := lfs.LocalStore{Root: lfs.RootFor(cfg.LFS.Root, cfg.Server.Root)}
	111	orphans, err := blobs.Orphans(referenced, lfsOrphanAge)
	112	if err != nil {
	113	return err
	114	}
	115	var freed int64
	116	for _, o := range orphans {
	117	if err := blobs.Delete(o.OID); err != nil {
	118	fmt.Fprintf(os.Stderr, "lfs %s: %v\n", o.OID, err)
	119	continue
	120	}
	121	freed += o.Size
	122	}
	123	fmt.Printf("lfs\t%d referenced, removed %d orphans (%s)\n", len(referenced), len(orphans), human(freed))
	124	return nil
	125	}

e2e/reap_test.go +52

		@@ -1,6 +1,8 @@
1	1	package e2e
2	2
3	3	import (
	4	"crypto/sha256"
	5	"encoding/hex"
4	6	"encoding/json"
5	7	"fmt"
6	8	"os"
		@@ -152,3 +154,53 @@ func TestHealthz(t *testing.T) {
152	154	t.Fatalf("healthz registered as a username: %s", errOut)
153	155	}
154	156	}
	157
	158	// gc --lfs removes objects no pointer names, keeps referenced ones, and
	159	// leaves anything young enough to be an upload ahead of its push.
	160	func TestGCLFSOrphans(t *testing.T) {
	161	inst := startInstance(t)
	162	aliceKey := inst.newKey(t, "alice")
	163	inst.admin(t, "admin", "user", "create", "alice", "--key", aliceKey+".pub")
	164	if _, _, code := inst.ssh(t, aliceKey, "", "repo", "create", "alice/big"); code != 0 {
	165	t.Fatal("repo create failed")
	166	}
	167	put := func(content string, age time.Duration) string {
	168	t.Helper()
	169	sum := sha256.Sum256([]byte(content))
	170	oid := hex.EncodeToString(sum[:])
	171	path := filepath.Join(inst.root, "lfs", oid[:2], oid[2:4], oid)
	172	os.MkdirAll(filepath.Dir(path), 0o755)
	173	os.WriteFile(path, []byte(content), 0o644)
	174	os.Chtimes(path, time.Now().Add(-age), time.Now().Add(-age))
	175	return oid
	176	}
	177	kept := put("referenced payload", 48*time.Hour)
	178	orphan := put("nobody points here", 48*time.Hour)
	179	young := put("just uploaded", time.Minute)
	180
	181	work := t.TempDir()
	182	env := inst.gitEnv(aliceKey)
	183	mustGit(t, work, env, "clone", inst.sshURL("alice/big"), "w")
	184	dir := filepath.Join(work, "w")
	185	pointer := fmt.Sprintf("version https://git-lfs.github.com/spec/v1\noid sha256:%s\nsize %d\n", kept, len("referenced payload"))
	186	os.WriteFile(filepath.Join(dir, "data.bin"), []byte(pointer), 0o644)
	187	mustGit(t, dir, env, "checkout", "-q", "-b", "main")
	188	mustGit(t, dir, env, "add", ".")
	189	mustGit(t, dir, env, "commit", "-q", "-m", "pointer")
	190	mustGit(t, dir, env, "push", "-q", "origin", "main")
	191
	192	if out := inst.admin(t, "admin", "stats", "--json"); !strings.Contains(out, `"lfs_bytes":`) \|\| strings.Contains(out, `"lfs_bytes":0`) {
	193	t.Fatalf("stats lfs bytes:\n%s", out)
	194	}
	195	out := inst.admin(t, "admin", "gc", "--lfs")
	196	if !strings.Contains(out, "lfs\t1 referenced, removed 1 orphans") {
	197	t.Fatalf("gc --lfs:\n%s", out)
	198	}
	199	exists := func(oid string) bool {
	200	_, err := os.Stat(filepath.Join(inst.root, "lfs", oid[:2], oid[2:4], oid))
	201	return err == nil
	202	}
	203	if !exists(kept) \|\| exists(orphan) \|\| !exists(young) {
	204	t.Fatalf("after gc: kept=%v orphan=%v young=%v", exists(kept), exists(orphan), exists(young))
	205	}
	206	}

internal/control/adminhost.go +4 −1

		@@ -9,6 +9,7 @@ import (
9	9	"golang.org/x/crypto/ssh"
10	10
11	11	"gitbay.org/gitbay/internal/gitutil"
	12	"gitbay.org/gitbay/internal/lfs"
12	13	"gitbay.org/gitbay/internal/mail"
13	14	"gitbay.org/gitbay/internal/policy"
14	15	"gitbay.org/gitbay/internal/protocol"
		@@ -301,9 +302,11 @@ func runAdminStats(c *Ctx, args []string) int {
301	302	Counts store.Counts `json:"counts"`
302	303	DBBytes int64 `json:"db_bytes"`
303	304	RepoBytes int64 `json:"repo_bytes"`
	305	LFSBytes int64 `json:"lfs_bytes"`
304	306	Repos []repoDisk `json:"repos"`
305	307	}
306	308	d := out{Counts: counts, Repos: []repoDisk{}}
	309	d.LFSBytes = lfs.LocalStore{Root: lfs.RootFor(c.Cfg.LFS.Root, c.Cfg.Server.Root)}.Size()
307	310	for _, r := range repos {
308	311	b := gitutil.DirSize(RepoDir(c.Cfg.Server.Root, r.OwnerName, r.Name))
309	312	d.Repos = append(d.Repos, repoDisk{r.Path(), b})
		@@ -316,7 +319,7 @@ func runAdminStats(c *Ctx, args []string) int {
316	319	fmt.Fprintf(w, "users %d · orgs %d · repos %d · issues %d (%d open) · MRs %d (%d open)\n",
317	320	counts.Users, counts.Orgs, counts.Repos,
318	321	counts.Issues, counts.OpenIssues, counts.MRs, counts.OpenMRs)
319		fmt.Fprintf(w, "database %s · repositories %s\n\n", humanBytes(d.DBBytes), humanBytes(d.RepoBytes))
	322	fmt.Fprintf(w, "database %s · repositories %s · lfs %s\n\n", humanBytes(d.DBBytes), humanBytes(d.RepoBytes), humanBytes(d.LFSBytes))
320	323	for _, r := range d.Repos {
321	324	fmt.Fprintf(w, "%s\t%s\n", r.Path, humanBytes(r.Bytes))
322	325	}

internal/gitutil/lfs.go added +77

		@@ -0,0 +1,77 @@
	1	package gitutil
	2
	3	import (
	4	"bufio"
	5	"bytes"
	6	"os/exec"
	7	"regexp"
	8	"strconv"
	9	"strings"
	10	)
	11
	12	// lfsPointerMax bounds a pointer file; real ones are around 130 bytes.
	13	const lfsPointerMax = 1024
	14
	15	var lfsOIDLine = regexp.MustCompile(`(?m)^oid sha256:([0-9a-f]{64})$`)
	16
	17	// LFSPointerOIDs returns every LFS object id referenced by a pointer blob
	18	// anywhere in the repository: every object, not just the reachable ones,
	19	// since an unreachable blob is still an object gc has not removed.
	20	func LFSPointerOIDs(dir string) ([]string, error) {
	21	list := exec.Command("git", "-C", dir, "cat-file", "--batch-all-objects",
	22	"--batch-check=%(objecttype) %(objectsize) %(objectname)")
	23	out, err := list.Output()
	24	if err != nil {
	25	return nil, err
	26	}
	27	var small []string
	28	sc := bufio.NewScanner(bytes.NewReader(out))
	29	for sc.Scan() {
	30	f := strings.Fields(sc.Text())
	31	if len(f) != 3 \|\| f[0] != "blob" {
	32	continue
	33	}
	34	if n, err := strconv.Atoi(f[1]); err == nil && n <= lfsPointerMax {
	35	small = append(small, f[2])
	36	}
	37	}
	38	if len(small) == 0 {
	39	return nil, nil
	40	}
	41	cat := exec.Command("git", "-C", dir, "cat-file", "--batch")
	42	cat.Stdin = strings.NewReader(strings.Join(small, "\n") + "\n")
	43	out, err = cat.Output()
	44	if err != nil {
	45	return nil, err
	46	}
	47	// --batch output: "<sha> blob <size>\n<content>\n" per object.
	48	var oids []string
	49	rest := out
	50	for len(rest) > 0 {
	51	nl := bytes.IndexByte(rest, '\n')
	52	if nl < 0 {
	53	break
	54	}
	55	hdr := strings.Fields(string(rest[:nl]))
	56	rest = rest[nl+1:]
	57	if len(hdr) != 3 {
	58	break
	59	}
	60	size, _ := strconv.Atoi(hdr[2])
	61	if size > len(rest) {
	62	break
	63	}
	64	body := rest[:size]
	65	rest = rest[size:]
	66	if len(rest) > 0 && rest[0] == '\n' {
	67	rest = rest[1:]
	68	}
	69	if !bytes.HasPrefix(body, []byte("version https://git-lfs.github.com/spec/")) {
	70	continue
	71	}
	72	if m := lfsOIDLine.FindSubmatch(body); m != nil {
	73	oids = append(oids, string(m[1]))
	74	}
	75	}
	76	return oids, nil
	77	}

internal/httpd/lfs.go +1 −6

		@@ -5,7 +5,6 @@ import (
5	5	"fmt"
6	6	"io"
7	7	"net/http"
8		"path/filepath"
9	8	"strings"
10	9	"time"
11	10
		@@ -21,11 +20,7 @@ import (
21	20	const lfsMediaType = "application/vnd.git-lfs+json"
22	21
23	22	func (s *Server) lfsStore() lfs.BlobStore {
24		root := s.cfg.LFS.Root
25		if root == "" {
26		root = filepath.Join(s.cfg.Server.Root, "lfs")
27		}
28		return lfs.LocalStore{Root: root}
	23	return lfs.LocalStore{Root: lfs.RootFor(s.cfg.LFS.Root, s.cfg.Server.Root)}
29	24	}
30	25
31	26	func (s *Server) lfsMaxObject() int64 {

internal/lfs/lfs.go +54

		@@ -20,6 +20,7 @@ import (
20	20	"encoding/hex"
21	21	"fmt"
22	22	"io"
	23	"io/fs"
23	24	"os"
24	25	"path/filepath"
25	26	"regexp"
		@@ -173,3 +174,56 @@ func NewSecret() string {
173	174	rand.Read(buf)
174	175	return hex.EncodeToString(buf)
175	176	}
	177
	178	// Orphans lists objects in the store that no repository references and
	179	// that are older than minAge: an object uploaded ahead of the push that
	180	// will reference it is not an orphan yet. referenced holds the object ids
	181	// every repository's pointers name.
	182	func (s LocalStore) Orphans(referenced map[string]bool, minAge time.Duration) ([]Orphan, error) {
	183	cutoff := time.Now().Add(-minAge)
	184	var out []Orphan
	185	err := filepath.WalkDir(s.Root, func(path string, d fs.DirEntry, err error) error {
	186	if err != nil \|\| d.IsDir() {
	187	return nil
	188	}
	189	oid := d.Name()
	190	if !OIDPat.MatchString(oid) \|\| referenced[oid] {
	191	return nil
	192	}
	193	info, err := d.Info()
	194	if err != nil \|\| info.ModTime().After(cutoff) {
	195	return nil
	196	}
	197	out = append(out, Orphan{OID: oid, Size: info.Size()})
	198	return nil
	199	})
	200	return out, err
	201	}
	202
	203	// Orphan is one unreferenced object.
	204	type Orphan struct {
	205	OID string
	206	Size int64
	207	}
	208
	209	// Size sums every object in the store.
	210	func (s LocalStore) Size() int64 {
	211	var total int64
	212	filepath.WalkDir(s.Root, func(_ string, d fs.DirEntry, err error) error {
	213	if err == nil && !d.IsDir() {
	214	if fi, err := d.Info(); err == nil {
	215	total += fi.Size()
	216	}
	217	}
	218	return nil
	219	})
	220	return total
	221	}
	222
	223	// RootFor is the store root a configuration implies.
	224	func RootFor(lfsRoot, serverRoot string) string {
	225	if lfsRoot != "" {
	226	return lfsRoot
	227	}
	228	return filepath.Join(serverRoot, "lfs")
	229	}

Commit 51386c203f

Commit `51386c203f`